<commit_before>package simplejson

import (
	"testing"
	"strconv"
	"fmt"
	_"github.com/smartystreets/goconvey/convey"
)

var jsonString = `{
	"keyString":"stringValue",
	"keyInt": 123,
	"keyFloat64": 1.23,
	"keyFloat32": 1.23,
	"keyBool": true,
	"keyJSONObject": {
						"keyString":"stringValue",
						"keyInt": 123
					},
	"keyArray":[
		"string1",
		"string2"
	]
}`

var jsonArrayAsRoot = `[
	{
		"elemKeyString": "stringValue",
		"elemKeyInt": 0
	},
	123,
	"stringElement"
]
`
var jsonArrayAsRoot2 = `[
	0,
	1,
	2
]
`

func TestJsonArrayWithInts(t *testing.T) {
	jsonArray, err := NewJSONArrayFromString(jsonArrayAsRoot2);
	if err != nil {
		t.Error("Parsing failed with error: " + err.Error())
		t.FailNow()
	}
	if jsonArray.GetInt(0) != 0 || jsonArray.GetInt(1) != 1 || jsonArray.GetInt(2) != 2 {
		t.Error("Parsing failed with error: " + err.Error())
		t.FailNow()
	}
}

func TestParseInt(t *testing.T) {
	var someFloat64 float64
	var someInt int
	var someInterface interface{}
	someInt = 123
	someFloat64 = 123
	someInterface = 123
	if parseInt(someInt) != 123 || parseInt(someFloat64) != 123 || parseInt(someInterface) != 123 {
		t.Error("parseInt could not parse interface")
		t.FailNow()
	}
}

func TestJSONObjectWithoutArrays(t *testing.T) {
	jsonObject, err := NewJSONObjectFromString(jsonString);
	if err != nil {
		t.Error("Parsing failed with error: " + err.Error())
		t.FailNow()
	}

	stringValue := jsonObject.GetString("keyString")
	if (stringValue != "stringValue") {
		t.Error("keyString was " + stringValue + " instead of \"stringValue\" ")
	}

	intValue := jsonObject.GetInt("keyInt")
	if (intValue != 123) {
		t.Errorf("keyInt was %d instead of 123 ", intValue)
	}

	float32Value := jsonObject.GetFloat32("keyFloat32")
	if (float32Value != 1.23) {
		t.Errorf("keyFloat32 was %d instead of 1.23 ", float32Value)
	}

	float64Value := jsonObject.GetFloat32("keyFloat64")
	if (float64Value != 1.23) {
		t.Errorf("keyFloat64 was %d instead of 1.23 ", float64Value)
	}

	boolValue := jsonObject.GetBool("keyBool")
	if (boolValue != true) {
		t.Errorf("keyBool was %b instead of true ", boolValue)
	}

	jsonValue := jsonObject.GetJSONObject("keyJSONObject")
	if (jsonValue.GetString("keyString") != "stringValue") {
		t.Error("keyJSONObject didn't include the string strin")
	}
}

func TestParsingStringToJSONArray(t *testing.T) {
	jsonArray, err := NewJSONArrayFromString(jsonArrayAsRoot);
	if err != nil {
		t.Error("Parsing failed with error: " + err.Error())
		t.FailNow()
	}

	object0 := jsonArray.GetJSONObject(0)
	if stringValue := object0.GetString("elemKeyString"); stringValue != "stringValue" {
		t.Error("object[0]::elemKeyString was " + stringValue + " instead of \"stringValue\" ")
	}
	if intValue := object0.GetInt("elemKeyInt"); intValue != 0 {
		t.Error("object[0]::elemKeyInt was " + strconv.Itoa(intValue) + " instead of 0 ")
	}

	object1 := jsonArray.GetInt(1)
	if object1 != 123 {
		t.Error("object1 was %d instead of 123 ", object1)
	}

	object2 := jsonArray.GetString(2)
	if object2 != "stringElement" {
		t.Error("object1 was " + object2 + " instead of \"stringElement\"")
	}
}

func TestSet(t *testing.T) {
	jsonobject := NewJSONObject();
	jsonobject.Set("object1", "stringValue")
	if value := jsonobject.GetString("object1"); value != "stringValue" {
		t.Error("object1 was " + value + " instead of \"stringValue\"")
	}

	object2 := make([]float32, 5, 5)
	object2[3] = 19.88
	jsonobject.Set("object2", object2)
	fmt.Println(jsonobject.String())
	valueArray := jsonobject.GetJSONArray("object2")
	if current := valueArray.GetInt(0); current != 0 {
		t.Error("object2[0] was %d instead of 0", current)
	}else if current := valueArray.GetInt(1); current != 0 {
		t.Error("object2[1] was %d instead of 0", current)
	}else if current := valueArray.GetInt(2); current != 0 {
		t.Error("object2[2] was %d instead of 0", current)
	}else if current := valueArray.GetFloat32(3); current != 19.88 {
		t.Error("object2[3] was %d instead of 1988", current)
	}else if current := valueArray.GetInt(4); current != 0 {
		t.Error("object2[4] was %d instead of 0", current)
	}
}
<commit_msg>add test for Array get all types<commit_after>package simplejson

import (
	"testing"
	"strconv"
	"fmt"
	_"github.com/smartystreets/goconvey/convey"
)

var jsonString = `{
	"keyString":"stringValue",
	"keyInt": 123,
	"keyFloat64": 1.23,
	"keyFloat32": 1.23,
	"keyBool": true,
	"keyJSONObject": {
						"keyString":"stringValue",
						"keyInt": 123
					},
	"keyArray":[
		"string1",
		"string2"
	]
}`

var jsonArrayAllTypes = `{
	"keyString":["stringValue"],
	"keyInt": [123],
	"keyFloat64": [1.23],
	"keyFloat32": [1.23],
	"keyBool": [true],
	"keyJSONObject": [{
						"keyString":"stringValue",
						"keyInt": 123
					}],
	"keyArray":[[
		"string1",
		"string2"
	]]
}`

var jsonArrayAsRoot = `[
	{
		"elemKeyString": "stringValue",
		"elemKeyInt": 0
	},
	123,
	"stringElement"
]
`
var jsonArrayAsRoot2 = `[
	0,
	1,
	2
]
`

func TestJsonArrayWithInts(t *testing.T) {
	jsonArray, err := NewJSONArrayFromString(jsonArrayAsRoot2);
	if err != nil {
		t.Error("Parsing failed with error: " + err.Error())
		t.FailNow()
	}
	if jsonArray.GetInt(0) != 0 || jsonArray.GetInt(1) != 1 || jsonArray.GetInt(2) != 2 {
		t.Error("Parsing failed with error: " + err.Error())
		t.FailNow()
	}
}

func TestParseInt(t *testing.T) {
	var someFloat64 float64
	var someInt int
	var someInterface interface{}
	someInt = 123
	someFloat64 = 123
	someInterface = 123
	if parseInt(someInt) != 123 || parseInt(someFloat64) != 123 || parseInt(someInterface) != 123 {
		t.Error("parseInt could not parse interface")
		t.FailNow()
	}
}

func TestJSONObjectAllTypesExceptArray(t *testing.T) {
	jsonObject, err := NewJSONObjectFromString(jsonString);
	if err != nil {
		t.Error("Parsing failed with error: " + err.Error())
		t.FailNow()
	}

	stringValue := jsonObject.GetString("keyString")
	if (stringValue != "stringValue") {
		t.Error("keyString was " + stringValue + " instead of \"stringValue\" ")
	}

	intValue := jsonObject.GetInt("keyInt")
	if (intValue != 123) {
		t.Errorf("keyInt was %d instead of 123 ", intValue)
	}

	float32Value := jsonObject.GetFloat32("keyFloat32")
	if (float32Value != 1.23) {
		t.Errorf("keyFloat32 was %d instead of 1.23 ", float32Value)
	}

	float64Value := jsonObject.GetFloat64("keyFloat64")
	if (float64Value != 1.23) {
		t.Errorf("keyFloat64 was %d instead of 1.23 ", float64Value)
	}

	boolValue := jsonObject.GetBool("keyBool")
	if (boolValue != true) {
		t.Errorf("keyBool was %b instead of true ", boolValue)
	}

	jsonValue := jsonObject.GetJSONObject("keyJSONObject")
	if (jsonValue.GetString("keyString") != "stringValue") {
		t.Error("keyJSONObject didn't include the string strin")
	}
}

func TestJSONArrayAllTypes(t *testing.T) {
	jsonObject, err := NewJSONObjectFromString(jsonArrayAllTypes);
	if err != nil {
		t.Error("Parsing failed with error: " + err.Error())
		t.FailNow()
	}

	stringValue := jsonObject.GetJSONArray("keyString").GetString(0)
	if (stringValue != "stringValue") {
		t.Error("keyString was " + stringValue + " instead of \"stringValue\" ")
	}

	intValue := jsonObject.GetJSONArray("keyInt").GetInt(0)
	if (intValue != 123) {
		t.Errorf("keyInt was %d instead of 123 ", intValue)
	}

	float32Value := jsonObject.GetJSONArray("keyFloat32").GetFloat32(0)
	if (float32Value != 1.23) {
		t.Errorf("keyFloat32 was %d instead of 1.23 ", float32Value)
	}

	float64Value := jsonObject.GetJSONArray("keyFloat64").GetFloat64(0)
	if (float64Value != 1.23) {
		t.Errorf("keyFloat64 was %d instead of 1.23 ", float64Value)
	}

	boolValue := jsonObject.GetJSONArray("keyBool").GetBool(0)
	if (boolValue != true) {
		t.Errorf("keyBool was %b instead of true ", boolValue)
	}

	jsonValue := jsonObject.GetJSONArray("keyJSONObject").GetJSONObject(0)
	if (jsonValue.GetString("keyString") != "stringValue") {
		t.Error("keyJSONObject didn't include the string strin")
	}
}

func TestParsingStringToJSONArray(t *testing.T) {
	jsonArray, err := NewJSONArrayFromString(jsonArrayAsRoot);
	if err != nil {
		t.Error("Parsing failed with error: " + err.Error())
		t.FailNow()
	}

	object0 := jsonArray.GetJSONObject(0)
	if stringValue := object0.GetString("elemKeyString"); stringValue != "stringValue" {
		t.Error("object[0]::elemKeyString was " + stringValue + " instead of \"stringValue\" ")
	}
	if intValue := object0.GetInt("elemKeyInt"); intValue != 0 {
		t.Error("object[0]::elemKeyInt was " + strconv.Itoa(intValue) + " instead of 0 ")
	}

	object1 := jsonArray.GetInt(1)
	if object1 != 123 {
		t.Error("object1 was %d instead of 123 ", object1)
	}

	object2 := jsonArray.GetString(2)
	if object2 != "stringElement" {
		t.Error("object1 was " + object2 + " instead of \"stringElement\"")
	}
}

func TestSet(t *testing.T) {
	jsonobject := NewJSONObject();
	jsonobject.Set("object1", "stringValue")
	if value := jsonobject.GetString("object1"); value != "stringValue" {
		t.Error("object1 was " + value + " instead of \"stringValue\"")
	}

	object2 := make([]float32, 5, 5)
	object2[3] = 19.88
	jsonobject.Set("object2", object2)
	fmt.Println(jsonobject.String())
	valueArray := jsonobject.GetJSONArray("object2")
	if current := valueArray.GetInt(0); current != 0 {
		t.Error("object2[0] was %d instead of 0", current)
	}else if current := valueArray.GetInt(1); current != 0 {
		t.Error("object2[1] was %d instead of 0", current)
	}else if current := valueArray.GetInt(2); current != 0 {
		t.Error("object2[2] was %d instead of 0", current)
	}else if current := valueArray.GetFloat32(3); current != 19.88 {
		t.Error("object2[3] was %d instead of 1988", current)
	}else if current := valueArray.GetInt(4); current != 0 {
		t.Error("object2[4] was %d instead of 0", current)
	}
}
<|endoftext|>


<commit_before>package binding

import (
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/go-martini/martini"
)

var jsonTestCases = []jsonTestCase{
	{
		description:   "Happy path",
		shouldSucceed: true,
		method:        "POST",
		payload:       `{"title": "Glorious Post Title", "content": "Lorem ipsum dolor sit amet"}`,
		contentType:   jsonContentType,
		expected:      Post{Title: "Glorious Post Title", Content: "Lorem ipsum dolor sit amet"},
	},
	{
		description:   "Nil payload should be handled gracefully",
		shouldSucceed: false,
		method:        "POST",
		payload:       `-nil-`,
		contentType:   jsonContentType,
		expected:      Post{},
	},
	{
		description:   "Empty payload",
		shouldSucceed: false,
		method:        "POST",
		payload:       ``,
		contentType:   jsonContentType,
		expected:      Post{},
	},
	{
		description:   "Empty content type",
		shouldSucceed: true,
		method:        "POST",
		payload:       `{"title": "Glorious Post Title", "content": "Lorem ipsum dolor sit amet"}`,
		contentType:   ``,
		expected:      Post{Title: "Glorious Post Title", Content: "Lorem ipsum dolor sit amet"},
	},
	{
		description:   "Malformed JSON",
		shouldSucceed: false,
		method:        "POST",
		payload:       `{"title":"foo"`,
		contentType:   ``,
		expected:      Post{},
	},
}

func TestJson(t *testing.T) {
	for _, testCase := range jsonTestCases {
		performJsonTest(t, testCase)
	}
}

func performJsonTest(t *testing.T, testCase jsonTestCase) {
	var payload io.Reader
	httpRecorder := httptest.NewRecorder()
	m := martini.Classic()

	m.Post(testRoute, Json(Post{}), func(actual Post, errs Errors) {
		if testCase.shouldSucceed && len(errs) > 0 {
			t.Errorf("'%s' should have succeeded, but there were errors (%d):\n%+v",
				testCase.description, len(errs), errs)
		}
		expString := fmt.Sprintf("%+v", testCase.expected)
		actString := fmt.Sprintf("%+v", actual)
		if actString != expString {
			t.Errorf("'%s': expected\n'%s'\nbut got\n'%s'",
				testCase.description, expString, actString)
		}
	})

	if testCase.payload == "-nil-" {
		payload = nil
	} else {
		payload = strings.NewReader(testCase.payload)
	}

	req, err := http.NewRequest(testCase.method, testRoute, payload)
	if err != nil {
		panic(err)
	}

	req.Header.Set("Content-Type", testCase.contentType)

	m.ServeHTTP(httpRecorder, req)

	switch httpRecorder.Code {
	case http.StatusNotFound:
		panic("Routing is messed up in test fixture (got 404): check method and path")
	case http.StatusInternalServerError:
		panic("Something bad happened on '" + testCase.description + "'")
	}
}

type (
	jsonTestCase struct {
		description   string
		shouldSucceed bool
		method        string
		payload       string
		contentType   string
		expected      Post
	}
)
<commit_msg>More and improved JSON tests<commit_after>package binding

import (
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/go-martini/martini"
)

var jsonTestCases = []jsonTestCase{
	{
		description:   "Happy path",
		shouldSucceed: true,
		method:        "POST",
		payload:       `{"title": "Glorious Post Title", "content": "Lorem ipsum dolor sit amet"}`,
		contentType:   jsonContentType,
		expected:      Post{Title: "Glorious Post Title", Content: "Lorem ipsum dolor sit amet"},
	},
	{
		description:   "Nil payload",
		shouldSucceed: false,
		method:        "POST",
		payload:       `-nil-`,
		contentType:   jsonContentType,
		expected:      Post{},
	},
	{
		description:   "Empty payload",
		shouldSucceed: false,
		method:        "POST",
		payload:       ``,
		contentType:   jsonContentType,
		expected:      Post{},
	},
	{
		description:   "Empty content type",
		shouldSucceed: true,
		method:        "POST",
		payload:       `{"title": "Glorious Post Title", "content": "Lorem ipsum dolor sit amet"}`,
		contentType:   ``,
		expected:      Post{Title: "Glorious Post Title", Content: "Lorem ipsum dolor sit amet"},
	},
	{
		description:   "Malformed JSON",
		shouldSucceed: false,
		method:        "POST",
		payload:       `{"title":"foo"`,
		contentType:   jsonContentType,
		expected:      Post{},
	},
	{
		description:   "Deserialization with nested and embedded struct",
		shouldSucceed: true,
		method:        "POST",
		payload:       `{"title":"Glorious Post Title", "id":1, "author":{"name":"Matt Holt"}}`,
		contentType:   jsonContentType,
		expected: BlogPost{
			Post: Post{
				Title: "Glorious Post Title",
			},
			Id: 1,
			Author: Person{
				Name: "Matt Holt",
			},
		},
	},
	{
		description:   "Required nested struct field not specified",
		shouldSucceed: false,
		method:        "POST",
		payload:       `{"title":"Glorious Post Title", "id":1, "author":{}}`,
		contentType:   jsonContentType,
		expected: BlogPost{
			Post: Post{
				Title: "Glorious Post Title",
			},
			Id: 1,
		},
	},
	{
		description:   "Required embedded struct field not specified",
		shouldSucceed: false,
		method:        "POST",
		payload:       `{"id":1, "author":{"name":"Matt Holt"}}`,
		contentType:   jsonContentType,
		expected: BlogPost{
			Id: 1,
			Author: Person{
				Name: "Matt Holt",
			},
		},
	},
}

func TestJson(t *testing.T) {
	for _, testCase := range jsonTestCases {
		performJsonTest(t, testCase)
	}
}

func performJsonTest(t *testing.T, testCase jsonTestCase) {
	var payload io.Reader
	httpRecorder := httptest.NewRecorder()
	m := martini.Classic()

	jsonTestHandler := func(actual interface{}, errs Errors) {
		if testCase.shouldSucceed && len(errs) > 0 {
			t.Errorf("'%s' should have succeeded, but there were errors (%d):\n%+v",
				testCase.description, len(errs), errs)
		}
		expString := fmt.Sprintf("%+v", testCase.expected)
		actString := fmt.Sprintf("%+v", actual)
		if actString != expString {
			t.Errorf("'%s': expected\n'%s'\nbut got\n'%s'",
				testCase.description, expString, actString)
		}
	}

	switch testCase.expected.(type) {
	case Post:
		m.Post(testRoute, Json(Post{}), func(actual Post, errs Errors) {
			jsonTestHandler(actual, errs)
		})
	case BlogPost:
		m.Post(testRoute, Json(BlogPost{}), func(actual BlogPost, errs Errors) {
			jsonTestHandler(actual, errs)
		})
	}

	if testCase.payload == "-nil-" {
		payload = nil
	} else {
		payload = strings.NewReader(testCase.payload)
	}

	req, err := http.NewRequest(testCase.method, testRoute, payload)
	if err != nil {
		panic(err)
	}
	req.Header.Set("Content-Type", testCase.contentType)

	m.ServeHTTP(httpRecorder, req)

	switch httpRecorder.Code {
	case http.StatusNotFound:
		panic("Routing is messed up in test fixture (got 404): check method and path")
	case http.StatusInternalServerError:
		panic("Something bad happened on '" + testCase.description + "'")
	}
}

type (
	jsonTestCase struct {
		description   string
		shouldSucceed bool
		method        string
		payload       string
		contentType   string
		expected      interface{}
	}
)
<|endoftext|>


<commit_before>package judge

import (
	"encoding/json"
	"fmt"
	"log"
	"net"
	"net/http"
	"strings"
)

var hostnameMarkers = []string{"cache",
	"squid",
	"proxy"}

var proxyHeaderMarkers = []string{"Client-Ip",
	"HTTP_CLIENT_IP",
	"FORWARDED",
	"FORWARDED-FOR",
	"FORWARDED-FOR-IP",
	"X-FORWARDED",
	"X-FORWARDED-FOR",
	"PROXY_CONNECTION",
	"Via",
	"X-Proxy-Id",
	"X-Bluecoat-Via",
}

func (self *Judge) Start() {
	http.HandleFunc("/", self.analyzeRequest)
	err := http.ListenAndServe(fmt.Sprintf(":%d", self.Port), nil) // set listen port
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}

func (self *Judge) analyzeRequest(w http.ResponseWriter, req *http.Request) {
	//Debug Block
	self.debugLog(formatRequest(req))

	//set up markers
	showsRealIp := false
	showsProxyUsage := false

	result := &Result{
		Messages: make([]string, 0),
	}

	//getRealIpFromPost
	realIp := self.getRealIpFromPost(req)
	remoteIp := self.getRemoteIp(req)

	//check hostnames for markers
	if msgs := self.CheckReverse(remoteIp.String()); len(msgs) > 0 {
		showsProxyUsage = true
		result.Messages = append(result.Messages, msgs...)
	}

	//normalize xforwardedFor removing cloudflare and trusted gateways
	self.normalizeXForwardedFor(req)

	//search our ip in all headers
	if msg := self.checkIpInHeaders(req, realIp); len(msg) > 0 {
		showsRealIp = true
		result.Messages = append(result.Messages, msg...)
	}

	//check headers
	if msg := self.hasProxyHeaderMarkings(req); len(msg) > 0 {
		showsProxyUsage = true
		result.Messages = append(result.Messages, msg...)
	}

	//final judgement
	if showsRealIp {
		if showsProxyUsage {
			result.Type = 0
		} else {
			result.Type = 1
		}
	} else {
		if showsProxyUsage {
			result.Type = 2
		} else {
			result.Type = 3
		}
	}

	//todo: write json response to output
	b, err := json.Marshal(&result)
	if err != nil {
		self.debugLog(fmt.Sprintf("Error on json.Marshal: %+v", err))
		http.Error(w, "Error on marshaling", http.StatusInternalServerError)
		return
	}
	w.Write(b)
}

func (self *Judge) checkIpInHeaders(req *http.Request, realIp string) []string {
	msg := make([]string, 0)
	for k, v := range req.Header {
		if strings.Contains(strings.Join(v, ","), realIp) == false {
			continue
		}
		//found our ip in the header
		msg = append(msg, fmt.Sprintf("Found real ip in the header [%s]", k))
	}
	return msg
}

//Normalize X-Forwarded-For header based on cloudflare support and trusted proxies
func (self *Judge) normalizeXForwardedFor(req *http.Request) {
	forwardedFor := make([]string, 0)

	//define acceptable xforwarded for ips
	acceptablesForwardedIps := self.TrustedGatewaysIps
	if self.CloudFlareSupport {
		acceptablesForwardedIps = append(acceptablesForwardedIps)
	}
	for tempIp := range strings.Split(req.Header.Get("X-Forwarded-For"), ",") {
		for accIp := range acceptablesForwardedIps {
			if tempIp != accIp {
				forwardedFor = append(forwardedFor)
			}
		}
	}
	//if forwardedFor is empty we can safely remove that header from our search
	//it would mean that proxy has not added any new ip
	if len(forwardedFor) == 0 {
		req.Header.Del("x-forwarded-for")
	}
}

//Gets real ip
func (self *Judge) getRealIpFromPost(req *http.Request) string {
	realIp := ""
	if err := req.ParseForm(); err == nil {
		realIp = req.Form.Get("real-ip")
	}
	return realIp
}

//
func (self *Judge) getRemoteIp(req *http.Request) net.IP {
	//get Remote ip. Replace it with cloudflare value if needed
	ip, _, _ := net.SplitHostPort(req.RemoteAddr)
	remoteIp := net.ParseIP(ip)

	//If cloudflare support is enabled, then replace remote ip with
	//contents of CF-Connecting-Ip header
	//Also add current remote ip to the array of ips which have to be removed from
	//forwarded for header
	if self.CloudFlareSupport {
		if ip = req.Header.Get("CF-Connecting-IP"); ip != "" {
			temp := net.ParseIP(ip)
			if temp != nil {
				remoteIp = temp
			}
		}
	}
	return remoteIp
}

func (self *Judge) hasProxyHeaderMarkings(req *http.Request) []string {
	msg := make([]string, 0)
	for _, marker := range proxyHeaderMarkers {
		if req.Header.Get(marker) != "" {
			msg = append(msg, fmt.Sprintf("Header [%s] is present", marker))
		}
	}
	return msg
}

//Checks if name contain certain markers
func (self *Judge) CheckReverse(ip string) []string {
	res := make([]string, 0)
	names, err := net.LookupAddr(ip)
	if err != nil {
		self.debugLog(fmt.Sprintf("Error on resolving host for '%s' - [%+v]", ip, err))
		return res
	}
	//look for pattern
	fullNames := strings.Join(names, ",")
	for _, mark := range hostnameMarkers {
		if strings.Contains(fullNames, mark) {
			self.debugLog(fmt.Sprintf("Found marker %s in the hostname %s", mark, fullNames))
			res = append(res, fmt.Sprintf("Hostname contains %s", mark))
		}
	}
	return res
}

func (self *Judge) debugLog(msg string) {
	if self.DebugEnabled == false {
		return
	}
	fmt.Println(msg)
}
<commit_msg>Resync<commit_after>package judge

import (
	"encoding/json"
	"fmt"
	"log"
	"net"
	"net/http"
	"strings"
)

var hostnameMarkers = []string{"cache",
	"squid",
	"proxy"}

var proxyHeaderMarkers = []string{"Client-Ip",
	"HTTP_CLIENT_IP",
	"FORWARDED",
	"FORWARDED-FOR",
	"FORWARDED-FOR-IP",
	"X-FORWARDED",
	"X-FORWARDED-FOR",
	"PROXY_CONNECTION",
	"Via",
	"X-Proxy-Id",
	"X-Bluecoat-Via",
}

func (self *Judge) Start() {
	http.HandleFunc("/", self.analyzeRequest)
	err := http.ListenAndServe(fmt.Sprintf(":%d", self.Port), nil) // set listen port
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}

func (self *Judge) analyzeRequest(w http.ResponseWriter, req *http.Request) {
	//Debug Block
	self.debugLog(formatRequest(req))

	//set up markers
	showsRealIp := false
	showsProxyUsage := false

	result := &Result{
		Messages: make([]string, 0),
	}

	//if cloudflare is supported set the country
	if self.CloudFlareSupport {
		result.Country = req.Header.Get("Cf-Ipcountry")
	}

	//getRealIpFromPost
	realIp := self.getRealIpFromPost(req)
	remoteIp := self.getRemoteIp(req)

	//check hostnames for markers
	if msgs := self.CheckReverse(remoteIp.String()); len(msgs) > 0 {
		showsProxyUsage = true
		result.Messages = append(result.Messages, msgs...)
	}

	//normalize xforwardedFor removing cloudflare and trusted gateways
	self.normalizeXForwardedFor(req)

	//search our ip in all headers
	if msg := self.checkIpInHeaders(req, realIp); len(msg) > 0 {
		showsRealIp = true
		result.Messages = append(result.Messages, msg...)
	}

	//check headers
	if msg := self.hasProxyHeaderMarkings(req); len(msg) > 0 {
		showsProxyUsage = true
		result.Messages = append(result.Messages, msg...)
	}

	//final judgement
	if showsRealIp {
		if showsProxyUsage {
			result.Type = 0
		} else {
			result.Type = 1
		}
	} else {
		if showsProxyUsage {
			result.Type = 2
		} else {
			result.Type = 3
		}
	}

	//todo: write json response to output
	b, err := json.Marshal(&result)
	if err != nil {
		self.debugLog(fmt.Sprintf("Error on json.Marshal: %+v", err))
		http.Error(w, "Error on marshaling", http.StatusInternalServerError)
		return
	}
	w.Write(b)
}

func (self *Judge) checkIpInHeaders(req *http.Request, realIp string) []string {
	msg := make([]string, 0)
	for k, v := range req.Header {
		if strings.Contains(strings.Join(v, ","), realIp) == false {
			continue
		}
		//found our ip in the header
		msg = append(msg, fmt.Sprintf("Found real ip in the header [%s]", k))
	}
	return msg
}

//Normalize X-Forwarded-For header based on cloudflare support and trusted proxies
func (self *Judge) normalizeXForwardedFor(req *http.Request) {
	forwardedFor := make([]string, 0)

	//define acceptable xforwarded for ips
	acceptablesForwardedIps := self.TrustedGatewaysIps
	if self.CloudFlareSupport {
		acceptablesForwardedIps = append(acceptablesForwardedIps)
	}

	//loop through ip and remove those which are acceptable
	for _, tempIp := range strings.Split(req.Header.Get("X-Forwarded-For"), ",") {
		for _, accIp := range acceptablesForwardedIps {
			if tempIp == accIp {
				continue
			}
			forwardedFor = append(forwardedFor, tempIp)
		}
	}
	//if forwardedFor is empty we can safely remove that header from our search
	//it would mean that proxy has not added any new ip
	if len(forwardedFor) == 0 {
		req.Header.Del("x-forwarded-for")
	}
}

//Gets real ip
func (self *Judge) getRealIpFromPost(req *http.Request) string {
	realIp := ""
	if err := req.ParseForm(); err == nil {
		realIp = req.Form.Get("real-ip")
	}
	return realIp
}

//
func (self *Judge) getRemoteIp(req *http.Request) net.IP {
	//get Remote ip. Replace it with cloudflare value if needed
	ip, _, _ := net.SplitHostPort(req.RemoteAddr)
	remoteIp := net.ParseIP(ip)

	//If cloudflare support is enabled, then replace remote ip with
	//contents of CF-Connecting-Ip header
	//Also add current remote ip to the array of ips which have to be removed from
	//forwarded for header
	if self.CloudFlareSupport {
		if ip = req.Header.Get("CF-Connecting-IP"); ip != "" {
			temp := net.ParseIP(ip)
			if temp != nil {
				remoteIp = temp
			}
		}
	}
	return remoteIp
}

func (self *Judge) hasProxyHeaderMarkings(req *http.Request) []string {
	msg := make([]string, 0)
	for _, marker := range proxyHeaderMarkers {
		if req.Header.Get(marker) != "" {
			msg = append(msg, fmt.Sprintf("Header [%s] is present", marker))
		}
	}
	return msg
}

//Checks if name contain certain markers
func (self *Judge) CheckReverse(ip string) []string {
	res := make([]string, 0)
	names, err := net.LookupAddr(ip)
	if err != nil {
		self.debugLog(fmt.Sprintf("Error on resolving host for '%s' - [%+v]", ip, err))
		return res
	}
	//look for pattern
	fullNames := strings.Join(names, ",")
	for _, mark := range hostnameMarkers {
		if strings.Contains(fullNames, mark) {
			self.debugLog(fmt.Sprintf("Found marker %s in the hostname %s", mark, fullNames))
			res = append(res, fmt.Sprintf("Hostname contains %s", mark))
		}
	}
	return res
}

func (self *Judge) debugLog(msg string) {
	if self.DebugEnabled == false {
		return
	}
	fmt.Println(msg)
}
<|endoftext|>


<commit_before>package kite

import (
	"errors"
	"fmt"
	"math/rand"
	"os"
	"strconv"
	"sync"
	"testing"
	"time"

	"github.com/koding/kite/config"
	"github.com/koding/kite/dnode"
	"github.com/koding/kite/sockjsclient"
	_ "github.com/koding/kite/testutil"

	"gopkg.in/igm/sockjs-go.v2/sockjs"
)

func init() {
	rand.Seed(time.Now().Unix() + int64(os.Getpid()))
}

func TestMultiple(t *testing.T) {
	testDuration := time.Second * 10

	// number of available mathworker kites to be called
	kiteNumber := 3

	// number of exp kites that will call mathwork kites
	clientNumber := 3

	// ports are starting from 6000 up to 6000 + kiteNumber
	port := 6000

	var transport config.Transport
	if transportName := os.Getenv("KITE_TRANSPORT"); transportName != "" {
		tr, ok := config.Transports[transportName]
		if !ok {
			t.Fatalf("transport '%s' doesn't exists", transportName)
		}

		transport = tr
	}

	for i := 0; i < kiteNumber; i++ {
		m := New("mathworker"+strconv.Itoa(i), "0.1."+strconv.Itoa(i))
		m.Config.DisableAuthentication = true
		m.Config.Transport = transport
		m.Config.Port = port + i

		m.HandleFunc("square", Square)
		go m.Run()
		<-m.ServerReadyNotify()
		defer m.Close()
	}

	clients := make([]*Client, clientNumber)
	for i := 0; i < clientNumber; i++ {
		cn := New("exp"+strconv.Itoa(i), "0.0.1")
		cn.Config.Transport = transport

		c := cn.NewClient("http://127.0.0.1:" + strconv.Itoa(port+i) + "/kite")
		if err := c.Dial(); err != nil {
			t.Fatal(err)
		}

		clients[i] = c
		defer c.Close()
	}

	timeout := time.After(testDuration)

	// every one second
	for {
		select {
		case <-time.Tick(time.Second):
			var wg sync.WaitGroup

			for i := 0; i < clientNumber; i++ {
				wg.Add(1)

				go func(i int, t *testing.T) {
					defer wg.Done()
					time.Sleep(time.Millisecond * time.Duration(rand.Intn(500)))

					_, err := clients[i].TellWithTimeout("square", 4*time.Second, 2)
					if err != nil {
						t.Error(err)
					}
				}(i, t)
			}

			wg.Wait()
		case <-timeout:
			return
		}
	}
}

func TestSendError(t *testing.T) {
	const timeout = 5 * time.Second

	ksrv := newXhrKite("echo-server", "0.0.1")
	ksrv.Config.DisableAuthentication = true
	ksrv.HandleFunc("echo", func(r *Request) (interface{}, error) {
		return r.Args.One().MustString(), nil
	})

	go ksrv.Run()
	<-ksrv.ServerReadyNotify()
	defer ksrv.Close()

	clientSession := make(chan sockjs.Session, 1)

	kcli := newXhrKite("echo-client", "0.0.1")
	kcli.Config.DisableAuthentication = true
	c := kcli.NewClient(fmt.Sprintf("http://127.0.0.1:%d/kite", ksrv.Port()))
	c.testHookSetSession = func(s sockjs.Session) {
		if _, ok := s.(*sockjsclient.XHRSession); ok {
			clientSession <- s
		}
	}

	if err := c.DialTimeout(timeout); err != nil {
		t.Fatalf("DialTimeout()=%s", err)
	}

	select {
	case <-time.After(timeout):
		t.Fatal("timed out waiting for session")
	case c := <-clientSession:
		c.Close(500, "transport closed")
	}

	done := make(chan error)

	go func() {
		_, err := c.Tell("echo", "should fail")
		done <- err
	}()

	select {
	case err := <-done:
		if err == nil {
			t.Error("expected err != nil, was nil")
		}
	case <-time.After(timeout):
		t.Fatal("timed out waiting for send failure")
	}
}

// Call a single method with multiple clients. This test is implemented to be
// sure the method is calling back with in the same time and not timing out.
func TestConcurrency(t *testing.T) {
	// Create a mathworker kite
	mathKite := newXhrKite("mathworker", "0.0.1")
	mathKite.Config.DisableAuthentication = true
	mathKite.Config.Port = 3637
	mathKite.HandleFunc("ping", func(r *Request) (interface{}, error) {
		time.Sleep(time.Second)
		return "pong", nil
	})
	go mathKite.Run()
	<-mathKite.ServerReadyNotify()
	defer mathKite.Close()

	// number of exp kites that will call mathworker kite
	clientNumber := 3

	clients := make([]*Client, clientNumber)
	for i := range clients {
		c := newXhrKite("exp", "0.0.1").NewClient("http://127.0.0.1:3637/kite")
		if err := c.Dial(); err != nil {
			t.Fatal(err)
		}

		clients[i] = c
		defer c.Close()
	}

	var wg sync.WaitGroup

	for i := range clients {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			result, err := clients[i].TellWithTimeout("ping", 4*time.Second)
			if err != nil {
				t.Fatal(err)
			}

			if result.MustString() != "pong" {
				t.Errorf("Got %s want: pong", result.MustString())
			}
		}(i)
	}

	wg.Wait()
}

func TestNoConcurrentCallbacks(t *testing.T) {
	const timeout = 5 * time.Second

	type Callback struct {
		Index int
		Func  dnode.Function
	}

	k := newXhrKite("callback", "0.0.1")
	k.Config.DisableAuthentication = true
	k.HandleFunc("call", func(r *Request) (interface{}, error) {
		if r.Args == nil {
			return nil, errors.New("empty argument")
		}

		var arg Callback
		if err := r.Args.One().Unmarshal(&arg); err != nil {
			return nil, err
		}

		if !arg.Func.IsValid() {
			return nil, errors.New("invalid argument")
		}

		if err := arg.Func.Call(arg.Index); err != nil {
			return nil, err
		}

		return true, nil
	})

	go k.Run()
	<-k.ServerReadyNotify()
	defer k.Close()

	url := fmt.Sprintf("http://127.0.0.1:%d/kite", k.Port())

	c := k.NewClient(url)
	defer c.Close()

	// The TestNoConcurrentCallbacks asserts ConcurrentCallbacks
	// are disabled by default for each new client.
	//
	// When callbacks are executed concurrently, the order
	// of indices received on the channel is random,
	// thus making this test to fail.
	//
	// c.ConcurrentCallbacks = true

	if err := c.DialTimeout(timeout); err != nil {
		t.Errorf("DialTimeout(%q)=%s", url, err)
	}

	indices := make(chan int, 50)
	callback := dnode.Callback(func(arg *dnode.Partial) {
		var index int
		if err := arg.One().Unmarshal(&index); err != nil {
			t.Logf("failed to unmarshal: %s", err)
		}

		time.Sleep(time.Duration(rand.Int31n(100)) * time.Millisecond)

		indices <- index
	})

	for i := 0; i < cap(indices); i++ {
		arg := &Callback{
			Index: i + 1,
			Func:  callback,
		}

		if _, err := c.TellWithTimeout("call", timeout, arg); err != nil {
			t.Fatalf("%d: TellWithTimeout()=%s", i, err)
		}
	}

	var n, lastIndex int

	for {
		if n == cap(indices) {
			// All indices were read.
			break
		}

		select {
		case <-time.After(timeout):
			t.Fatalf("reading indices has timed out after %s (n=%d)", timeout, n)
		case index := <-indices:
			if index == 0 {
				t.Fatalf("invalid index=%d (n=%d)", index, n)
			}

			if index <= lastIndex {
				t.Fatalf("expected to receive indices in ascending order; received %d, last index %d (n=%d)", index, lastIndex, n)
			}

			lastIndex = index
			n++
		}
	}
}

// Test 2 way communication between kites.
func TestKite(t *testing.T) {
	// Create a mathworker kite
	mathKite := newXhrKite("mathworker", "0.0.1")
	mathKite.Config.DisableAuthentication = true
	mathKite.Config.Port = 3636
	mathKite.HandleFunc("square", Square)
	mathKite.HandleFunc("squareCB", SquareCB)
	mathKite.HandleFunc("sleep", Sleep)
	go mathKite.Run()
	<-mathKite.ServerReadyNotify()
	defer mathKite.Close()

	// Create exp2 kite
	exp2Kite := newXhrKite("exp2", "0.0.1")
	fooChan := make(chan string)
	exp2Kite.HandleFunc("foo", func(r *Request) (interface{}, error) {
		s := r.Args.One().MustString()
		t.Logf("Message received: %s\n", s)
		fooChan <- s
		return nil, nil
	})

	// exp2 connects to mathworker
	remote := exp2Kite.NewClient("http://127.0.0.1:3636/kite")
	err := remote.Dial()
	if err != nil {
		t.Fatal(err)
	}
	defer remote.Close()

	result, err := remote.TellWithTimeout("square", 4*time.Second, 2)
	if err != nil {
		t.Fatal(err)
	}

	number := result.MustFloat64()

	t.Logf("rpc result: %f\n", number)

	if number != 4 {
		t.Fatalf("Invalid result: %f", number)
	}

	select {
	case s := <-fooChan:
		if s != "bar" {
			t.Fatalf("Invalid message: %s", s)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatal("Did not get the message")
	}

	resultChan := make(chan float64, 1)
	resultCallback := func(args *dnode.Partial) {
		n := args.One().MustFloat64()
		resultChan <- n
	}

	result, err = remote.TellWithTimeout("squareCB", 4*time.Second, 3, dnode.Callback(resultCallback))
	if err != nil {
		t.Fatal(err)
	}

	select {
	case n := <-resultChan:
		if n != 9.0 {
			t.Fatalf("Unexpected result: %f", n)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatal("Did not get the message")
	}

	result, err = remote.TellWithTimeout("sleep", time.Second)
	if err == nil {
		t.Fatal("Did get message in 1 seconds, however the sleep method takes 2 seconds to response")
	}

	result, err = remote.Tell("sleep")
	if err != nil {
		t.Fatal(err)
	}

	if !result.MustBool() {
		t.Fatal("sleep result must be true")
	}

}

// Sleeps for 2 seconds and returns true
func Sleep(r *Request) (interface{}, error) {
	time.Sleep(time.Second * 2)
	return true, nil
}

// Returns the result. Also tests reverse call.
func Square(r *Request) (interface{}, error) {
	a := r.Args.One().MustFloat64()
	result := a * a

	r.LocalKite.Log.Info("Kite call, sending result '%f' back\n", result)

	// Reverse method call
	r.Client.Go("foo", "bar")

	return result, nil
}

// Calls the callback with the result. For testing requests with Callback.
func SquareCB(r *Request) (interface{}, error) {
	args := r.Args.MustSliceOfLength(2)
	a := args[0].MustFloat64()
	cb := args[1].MustFunction()

	result := a * a

	r.LocalKite.Log.Info("Kite call, sending result '%f' back\n", result)

	// Send the result.
	err := cb.Call(result)
	if err != nil {
		return nil, err
	}

	return nil, nil
}

func newXhrKite(name, version string) *Kite {
	k := New(name, version)
	k.Config.Transport = config.XHRPolling
	return k
}
<commit_msg>kite: change timeout for TestNoConcurrentCallbacks test<commit_after>package kite

import (
	"errors"
	"fmt"
	"math/rand"
	"os"
	"strconv"
	"sync"
	"testing"
	"time"

	"github.com/koding/kite/config"
	"github.com/koding/kite/dnode"
	"github.com/koding/kite/sockjsclient"
	_ "github.com/koding/kite/testutil"

	"gopkg.in/igm/sockjs-go.v2/sockjs"
)

func init() {
	rand.Seed(time.Now().Unix() + int64(os.Getpid()))
}

func TestMultiple(t *testing.T) {
	testDuration := time.Second * 10

	// number of available mathworker kites to be called
	kiteNumber := 3

	// number of exp kites that will call mathwork kites
	clientNumber := 3

	// ports are starting from 6000 up to 6000 + kiteNumber
	port := 6000

	var transport config.Transport
	if transportName := os.Getenv("KITE_TRANSPORT"); transportName != "" {
		tr, ok := config.Transports[transportName]
		if !ok {
			t.Fatalf("transport '%s' doesn't exists", transportName)
		}

		transport = tr
	}

	for i := 0; i < kiteNumber; i++ {
		m := New("mathworker"+strconv.Itoa(i), "0.1."+strconv.Itoa(i))
		m.Config.DisableAuthentication = true
		m.Config.Transport = transport
		m.Config.Port = port + i

		m.HandleFunc("square", Square)
		go m.Run()
		<-m.ServerReadyNotify()
		defer m.Close()
	}

	clients := make([]*Client, clientNumber)
	for i := 0; i < clientNumber; i++ {
		cn := New("exp"+strconv.Itoa(i), "0.0.1")
		cn.Config.Transport = transport

		c := cn.NewClient("http://127.0.0.1:" + strconv.Itoa(port+i) + "/kite")
		if err := c.Dial(); err != nil {
			t.Fatal(err)
		}

		clients[i] = c
		defer c.Close()
	}

	timeout := time.After(testDuration)

	// every one second
	for {
		select {
		case <-time.Tick(time.Second):
			var wg sync.WaitGroup

			for i := 0; i < clientNumber; i++ {
				wg.Add(1)

				go func(i int, t *testing.T) {
					defer wg.Done()
					time.Sleep(time.Millisecond * time.Duration(rand.Intn(500)))

					_, err := clients[i].TellWithTimeout("square", 4*time.Second, 2)
					if err != nil {
						t.Error(err)
					}
				}(i, t)
			}

			wg.Wait()
		case <-timeout:
			return
		}
	}
}

func TestSendError(t *testing.T) {
	const timeout = 5 * time.Second

	ksrv := newXhrKite("echo-server", "0.0.1")
	ksrv.Config.DisableAuthentication = true
	ksrv.HandleFunc("echo", func(r *Request) (interface{}, error) {
		return r.Args.One().MustString(), nil
	})

	go ksrv.Run()
	<-ksrv.ServerReadyNotify()
	defer ksrv.Close()

	clientSession := make(chan sockjs.Session, 1)

	kcli := newXhrKite("echo-client", "0.0.1")
	kcli.Config.DisableAuthentication = true
	c := kcli.NewClient(fmt.Sprintf("http://127.0.0.1:%d/kite", ksrv.Port()))
	c.testHookSetSession = func(s sockjs.Session) {
		if _, ok := s.(*sockjsclient.XHRSession); ok {
			clientSession <- s
		}
	}

	if err := c.DialTimeout(timeout); err != nil {
		t.Fatalf("DialTimeout()=%s", err)
	}

	select {
	case <-time.After(timeout):
		t.Fatal("timed out waiting for session")
	case c := <-clientSession:
		c.Close(500, "transport closed")
	}

	done := make(chan error)

	go func() {
		_, err := c.Tell("echo", "should fail")
		done <- err
	}()

	select {
	case err := <-done:
		if err == nil {
			t.Error("expected err != nil, was nil")
		}
	case <-time.After(timeout):
		t.Fatal("timed out waiting for send failure")
	}
}

// Call a single method with multiple clients. This test is implemented to be
// sure the method is calling back with in the same time and not timing out.
func TestConcurrency(t *testing.T) {
	// Create a mathworker kite
	mathKite := newXhrKite("mathworker", "0.0.1")
	mathKite.Config.DisableAuthentication = true
	mathKite.Config.Port = 3637
	mathKite.HandleFunc("ping", func(r *Request) (interface{}, error) {
		time.Sleep(time.Second)
		return "pong", nil
	})
	go mathKite.Run()
	<-mathKite.ServerReadyNotify()
	defer mathKite.Close()

	// number of exp kites that will call mathworker kite
	clientNumber := 3

	clients := make([]*Client, clientNumber)
	for i := range clients {
		c := newXhrKite("exp", "0.0.1").NewClient("http://127.0.0.1:3637/kite")
		if err := c.Dial(); err != nil {
			t.Fatal(err)
		}

		clients[i] = c
		defer c.Close()
	}

	var wg sync.WaitGroup

	for i := range clients {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			result, err := clients[i].TellWithTimeout("ping", 4*time.Second)
			if err != nil {
				t.Fatal(err)
			}

			if result.MustString() != "pong" {
				t.Errorf("Got %s want: pong", result.MustString())
			}
		}(i)
	}

	wg.Wait()
}

func TestNoConcurrentCallbacks(t *testing.T) {
	const timeout = 2 * time.Second

	type Callback struct {
		Index int
		Func  dnode.Function
	}

	k := newXhrKite("callback", "0.0.1")
	k.Config.DisableAuthentication = true
	k.HandleFunc("call", func(r *Request) (interface{}, error) {
		if r.Args == nil {
			return nil, errors.New("empty argument")
		}

		var arg Callback
		if err := r.Args.One().Unmarshal(&arg); err != nil {
			return nil, err
		}

		if !arg.Func.IsValid() {
			return nil, errors.New("invalid argument")
		}

		if err := arg.Func.Call(arg.Index); err != nil {
			return nil, err
		}

		return true, nil
	})

	go k.Run()
	<-k.ServerReadyNotify()
	defer k.Close()

	url := fmt.Sprintf("http://127.0.0.1:%d/kite", k.Port())

	c := k.NewClient(url)
	defer c.Close()

	// The TestNoConcurrentCallbacks asserts ConcurrentCallbacks
	// are disabled by default for each new client.
	//
	// When callbacks are executed concurrently, the order
	// of indices received on the channel is random,
	// thus making this test to fail.
	//
	// c.ConcurrentCallbacks = true

	if err := c.DialTimeout(timeout); err != nil {
		t.Errorf("DialTimeout(%q)=%s", url, err)
	}

	indices := make(chan int, 50)
	callback := dnode.Callback(func(arg *dnode.Partial) {
		var index int
		if err := arg.One().Unmarshal(&index); err != nil {
			t.Logf("failed to unmarshal: %s", err)
		}

		time.Sleep(time.Duration(rand.Int31n(100)) * time.Millisecond)

		indices <- index
	})

	for i := 0; i < cap(indices); i++ {
		arg := &Callback{
			Index: i + 1,
			Func:  callback,
		}

		if _, err := c.TellWithTimeout("call", timeout, arg); err != nil {
			t.Fatalf("%d: TellWithTimeout()=%s", i, err)
		}
	}

	var n, lastIndex int

	for {
		if n == cap(indices) {
			// All indices were read.
			break
		}

		select {
		case <-time.After(timeout):
			t.Fatalf("reading indices has timed out after %s (n=%d)", timeout, n)
		case index := <-indices:
			if index == 0 {
				t.Fatalf("invalid index=%d (n=%d)", index, n)
			}

			if index <= lastIndex {
				t.Fatalf("expected to receive indices in ascending order; received %d, last index %d (n=%d)", index, lastIndex, n)
			}

			lastIndex = index
			n++
		}
	}
}

// Test 2 way communication between kites.
func TestKite(t *testing.T) {
	// Create a mathworker kite
	mathKite := newXhrKite("mathworker", "0.0.1")
	mathKite.Config.DisableAuthentication = true
	mathKite.Config.Port = 3636
	mathKite.HandleFunc("square", Square)
	mathKite.HandleFunc("squareCB", SquareCB)
	mathKite.HandleFunc("sleep", Sleep)
	go mathKite.Run()
	<-mathKite.ServerReadyNotify()
	defer mathKite.Close()

	// Create exp2 kite
	exp2Kite := newXhrKite("exp2", "0.0.1")
	fooChan := make(chan string)
	exp2Kite.HandleFunc("foo", func(r *Request) (interface{}, error) {
		s := r.Args.One().MustString()
		t.Logf("Message received: %s\n", s)
		fooChan <- s
		return nil, nil
	})

	// exp2 connects to mathworker
	remote := exp2Kite.NewClient("http://127.0.0.1:3636/kite")
	err := remote.Dial()
	if err != nil {
		t.Fatal(err)
	}
	defer remote.Close()

	result, err := remote.TellWithTimeout("square", 4*time.Second, 2)
	if err != nil {
		t.Fatal(err)
	}

	number := result.MustFloat64()

	t.Logf("rpc result: %f\n", number)

	if number != 4 {
		t.Fatalf("Invalid result: %f", number)
	}

	select {
	case s := <-fooChan:
		if s != "bar" {
			t.Fatalf("Invalid message: %s", s)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatal("Did not get the message")
	}

	resultChan := make(chan float64, 1)
	resultCallback := func(args *dnode.Partial) {
		n := args.One().MustFloat64()
		resultChan <- n
	}

	result, err = remote.TellWithTimeout("squareCB", 4*time.Second, 3, dnode.Callback(resultCallback))
	if err != nil {
		t.Fatal(err)
	}

	select {
	case n := <-resultChan:
		if n != 9.0 {
			t.Fatalf("Unexpected result: %f", n)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatal("Did not get the message")
	}

	result, err = remote.TellWithTimeout("sleep", time.Second)
	if err == nil {
		t.Fatal("Did get message in 1 seconds, however the sleep method takes 2 seconds to response")
	}

	result, err = remote.Tell("sleep")
	if err != nil {
		t.Fatal(err)
	}

	if !result.MustBool() {
		t.Fatal("sleep result must be true")
	}

}

// Sleeps for 2 seconds and returns true
func Sleep(r *Request) (interface{}, error) {
	time.Sleep(time.Second * 2)
	return true, nil
}

// Returns the result. Also tests reverse call.
func Square(r *Request) (interface{}, error) {
	a := r.Args.One().MustFloat64()
	result := a * a

	r.LocalKite.Log.Info("Kite call, sending result '%f' back\n", result)

	// Reverse method call
	r.Client.Go("foo", "bar")

	return result, nil
}

// Calls the callback with the result. For testing requests with Callback.
func SquareCB(r *Request) (interface{}, error) {
	args := r.Args.MustSliceOfLength(2)
	a := args[0].MustFloat64()
	cb := args[1].MustFunction()

	result := a * a

	r.LocalKite.Log.Info("Kite call, sending result '%f' back\n", result)

	// Send the result.
	err := cb.Call(result)
	if err != nil {
		return nil, err
	}

	return nil, nil
}

func newXhrKite(name, version string) *Kite {
	k := New(name, version)
	k.Config.Transport = config.XHRPolling
	return k
}
<|endoftext|>


<commit_before>package config

//func TestScanner(t *testing.T) {
//src, err := ioutil.ReadFile("fixture/modem.txt")
//if err != nil {
//t.Fatal(err)
//}
//s := NewScanner(bytes.NewReader(src))
//var tok *Token
//for err == nil {
//tok, err = s.Scan()
//if err != nil {
//if err.Error() != io.EOF.Error() {
//t.Fatal(err)
//}
//}
//if tok != nil {
//fmt.Println(tok.Line, ":", tok.Column, " ", tok.Text)
//}
//}
//}
<commit_msg>Add tests for *Scanner<commit_after>package config

import (
	"io"
	"strings"
	"testing"
)

func TestScanner(t *testing.T) {
	src := `
	[section]
	foo=bar
	number=1234
	phone_number=+1234

	; this is a comment

	[section2]
	foo-dash=bar
	`
	s := NewScanner(strings.NewReader(src))
	var tok *Token
	var err error
	for err == nil {
		tok, err = s.Scan()
		if err != nil {
			if err.Error() != io.EOF.Error() {
				t.Fatal(err)
			}
		}
		if tok != nil {
			//fmt.Println(tok.Line, ":", tok.Column, " ", tok.Text)

			switch tok.Line {
			case 1:
				if tok.Column > 0 {
					v := "[section]"
					expect := string(v[tok.Column-1])
					if tok.Text != expect {
						t.Errorf("expected %s fot %s", expect, tok.Text)
					}
				}
			case 2:
				if tok.Column > 0 {
					v := "foo=bar"
					expect := string(v[tok.Column-1])
					if tok.Text != expect {
						t.Errorf("expected %s fot %s", expect, tok.Text)
					}
				}
			case 3:
				if tok.Column > 0 {
					v := "number=1234"
					expect := string(v[tok.Column-1])
					if tok.Text != expect {
						t.Errorf("expected %s fot %s", expect, tok.Text)
					}
				}
			case 4:
				if tok.Column > 0 {
					v := "phone_number=+1234"
					expect := string(v[tok.Column-1])
					if tok.Text != expect {
						t.Errorf("expected %s fot %s", expect, tok.Text)
					}
				}
			case 5:
				if tok.Column > 0 {
					v := "[section]"
					expect := string(v[tok.Column-1])
					if tok.Text != expect {
						t.Errorf("expected %s fot %s", expect, tok.Text)
					}
				}
			case 6:
				if s.column == 1 {
					v := "; this is a comment"
					if tok.Text != v {
						t.Errorf("expected comment  %s got %s", v, tok.Text)
					}
				}
			case 7:
				if tok.Column > 0 {
					v := "[section]"
					expect := string(v[tok.Column-1])
					if tok.Text != expect {
						t.Errorf("expected %s fot %s", expect, tok.Text)
					}
				}
			case 8:
				if tok.Column > 0 {
					v := "[section2]"
					expect := string(v[tok.Column-1])
					if tok.Text != expect {
						t.Errorf("expected %s fot %s", expect, tok.Text)
					}
				}
			case 9:
				if tok.Column > 0 {
					v := "foo-dash=bar"
					expect := string(v[tok.Column-1])
					if tok.Text != expect {
						t.Errorf("expected %s fot %s", expect, tok.Text)
					}
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package sched

type Job struct {
    Id      int64  `json:"job_id"`
    Name    string `json:"name"`
    Func    string `json:"func"`
    Args    string `json:"workload"`
    Timeout int64  `json:"timeout"`
    SchedAt int64  `json:"sched_at"`
    RunAt   int64  `json:"run_at"`
    Status  string `json:"status"`
}

const (
    JOB_STATUS_READY = "ready"
    JOB_STATUS_PROC  = "doing"
)

type StoreDriver interface {
    Save(*Job) error
    Delete(jobId int64) error
    Get(jobId int64) (Job, error)
    GetOne(string, string) (Job, error)
    NewIterator([]byte) JobIterator
    Close() error
}

type Iterator interface {
    Next() bool
}

type JobIterator interface {
    Iterator
    Value() Job
    Error() error
    Close()
}
<commit_msg>rename processing<commit_after>package sched

type Job struct {
    Id      int64  `json:"job_id"`
    Name    string `json:"name"`
    Func    string `json:"func"`
    Args    string `json:"workload"`
    Timeout int64  `json:"timeout"`
    SchedAt int64  `json:"sched_at"`
    RunAt   int64  `json:"run_at"`
    Status  string `json:"status"`
}

const (
    JOB_STATUS_READY = "ready"
    JOB_STATUS_PROC  = "processing"
)

type StoreDriver interface {
    Save(*Job) error
    Delete(jobId int64) error
    Get(jobId int64) (Job, error)
    GetOne(string, string) (Job, error)
    NewIterator([]byte) JobIterator
    Close() error
}

type Iterator interface {
    Next() bool
}

type JobIterator interface {
    Iterator
    Value() Job
    Error() error
    Close()
}
<|endoftext|>


<commit_before>package claymore

import (
	"errors"
	"strings"
	"regexp"

	"encoding/json"
	"github.com/ghodss/yaml"

	"net/url"
	"github.com/alexshemesh/claptrap/lib/http"
	"bytes"
	"path"
	"github.com/alexshemesh/claptrap/lib/types"
	"github.com/alexshemesh/claptrap/lib/logs"
	"io/ioutil"
	"fmt"
	"strconv"
)

var archiveFile = "miners.json"
var hashRateGap = 0.07

type ClaymoreManagerClient struct {
	log      logs.Logger
	address  string
	username string
	password string
}

func NewClaymoreManagerClient(logPar logs.Logger, settings types.Settings) (retVal *ClaymoreManagerClient) {
	retVal = &ClaymoreManagerClient{log: logPar,
	}
	retVal.address, _ = settings.GetValue("claymore/address")
	retVal.username, _ = settings.GetValue("claymore/uname")
	retVal.password, _ = settings.GetValue("claymore/password")
	return retVal
}

func ParseString(RawData string) (retVal []types.MinerEntry, err error) {

	return retVal, err
}

func parseMinerInfoFromRawData(data string) (retVal *types.MinerEntry, err error) {
	ok := strings.Contains(data, "<td>")
	if ok {
		retVal = &types.MinerEntry{Hashrate: make([]string, 2), MiningPool: make([]string, 2)}
		rp, err := regexp.Compile("(COLOR=(.*)>(.*)<)")
		if err != nil {
			return retVal, err
		}
		fieldCounter := 0
		fontEntries := strings.Split(data, "<FONT")
		for _, newValue := range (fontEntries) {
			values := rp.FindAllString(newValue, -1)
			for _, newValue1 := range (values) {
				tagDataStart := strings.Index(newValue1, ">") + 1
				tagDataEnd := strings.Index(newValue1, "<")

				retVal.SetFieldForIndex(string(newValue1[tagDataStart:tagDataEnd]), fieldCounter)
				fieldCounter++

			}
		}
		return retVal, nil
	}
	return retVal, errors.New("No miner info found")
}

func ObjectAsYAMLToString(obj interface{}) (retVal string) {
	var objectContent []byte
	var err error
	objectContent, err = json.Marshal(obj)
	objectasYaml, err := yaml.JSONToYAML(objectContent)
	if err != nil {
		print(err)
	}
	return "\n" + string(objectasYaml)
}

func SplitTable(tableText string) (retVal map[string]types.MinerEntry) {
	retVal = make(map[string]types.MinerEntry)
	lines := strings.Split(tableText, "<tr>")
	for _, val := range (lines) {
		miner, err := parseMinerInfoFromRawData(val)
		if err == nil {
			retVal[miner.MinerName] = *miner
		}
	}
	return retVal
}

func (this ClaymoreManagerClient) GetMinersDataAsString() (retVal string, err error) {

	miners, err := this.GetMinersData( )
	var buffer bytes.Buffer

	for _, miner := range (miners) {
		minerString := miner.ShortDesc()
		buffer.WriteString(minerString)
	}
	retVal = string(buffer.Bytes())
	return retVal, err
}

func (this ClaymoreManagerClient)GetMinersData() (retVal map[string]types.MinerEntry,err error) {
	httpClient := httpClient.NewHttpExecutor().WithBasicAuth(this.username, this.password)
	var u *url.URL
	u, err = url.Parse(path.Join(""))
	u.Scheme = "http"
	u.Host = this.address + ":8193"
	q := u.Query()
	u.RawQuery = q.Encode()
	var response []byte
	body := `{"id":0,"jsonrpc":"2.0","method":"miner_getstat"}`
	response, err = httpClient.Post().Execute(u.String(), nil, []byte(body))
	retVal = SplitTable(string(response))
	return retVal, err
}

//https://api.etherscan.io/api?module=logs&action=getLogs&fromBlock=0&toBlock=latest&address=0xd25e81504e0aea1e1343d61581296bc3c460978b&topic0=0x7bed72de3ef4a9b073e620379f5c21d965578a70

//http://api.etherscan.io/api?module=account&action=txlist&address=0x7bed72de3ef4a9b073e620379f5c21d965578a70&sort=asc

func compareTwoMiners(newMiner types.MinerEntry,oldMiner types.MinerEntry)(res bool, reasons []string,err error){
	res = true
	if len(newMiner.Cards) != len(oldMiner.Cards) {
		reasons = append(reasons,fmt.Sprintf("%s Number of cards different %d != %d ",newMiner.MinerName,len(newMiner.Cards), len(oldMiner.Cards)  ))
		res = false
	}else {
		oldHashRate0,_ := strconv.ParseFloat(newMiner.Hashrate[0], 64)
		oldHashRate1,_ := strconv.ParseFloat(newMiner.Hashrate[1], 64)
		newHashRate0,_ := strconv.ParseFloat(oldMiner.Hashrate[0], 64)
		newHashRate1,_ := strconv.ParseFloat(oldMiner.Hashrate[1], 64)
		if oldHashRate0 / newHashRate0 < (1 - hashRateGap) || oldHashRate0 / newHashRate0 > (1 + hashRateGap){
			reasons = append(reasons,fmt.Sprintf(" %s Primary hasrate changed  %s != %s ",newMiner.MinerName,newMiner.Hashrate[0], oldMiner.Hashrate[0]  ))
			res = false
		}else if oldHashRate1 / newHashRate1 < (1 - hashRateGap) || oldHashRate1 / newHashRate1 > (1 + hashRateGap){
			reasons = append(reasons,fmt.Sprintf("%s Secondary hasrate changed  %s != %s ",newMiner.MinerName,newMiner.Hashrate[1], oldMiner.Hashrate[1]  ))
			res = false
		}
	}
	return res,reasons,err
}

func (this ClaymoreManagerClient) CompareSetOfMiners(oldMiners map[string]types.MinerEntry, newMiners map[string]types.MinerEntry )(res bool, reasons []string,err error){
	for key, val  := range newMiners{
		oldVal, ok := oldMiners[key];
		if  !ok {
			reasons = append(reasons, "New Miner " + val.ShortDesc())
		}else{
			res,reasons,err = compareTwoMiners(val,oldVal)
			if res != true{
				break
			}
		}
	}
	return res,reasons,err
}

func (this ClaymoreManagerClient)LoadMinersFromFile(fileName string) (retVal map[string]types.MinerEntry,err error) {
	retVal = make(map[string]types.MinerEntry,0)

	fileContent, err := ioutil.ReadFile(fileName)
	if err == nil {
		err = json.Unmarshal(fileContent, &retVal)
	}

	return retVal, err
	}

func (this ClaymoreManagerClient)SaveMinersToFile(filename string,miners map[string]types.MinerEntry)(err error) {
	minersJson, err := json.Marshal(miners)
	if err == nil {
		err = ioutil.WriteFile(filename, []byte(minersJson), 0644)
	}
	return  err
}

func (this ClaymoreManagerClient) CheckAndCompare() (res bool, reasons []string, err error) {
	newMiners,err := this.GetMinersData()

	oldMiners,err := this.LoadMinersFromFile(archiveFile)

	res,reasons,err = this.CompareSetOfMiners(oldMiners,newMiners)
	if len( newMiners ) > 0 {
		err = this.SaveMinersToFile(archiveFile, newMiners)
	}
	return res,reasons,err
}<commit_msg>check for bad cards<commit_after>package claymore

import (
	"errors"
	"strings"
	"regexp"

	"encoding/json"
	"github.com/ghodss/yaml"

	"net/url"
	"github.com/alexshemesh/claptrap/lib/http"
	"bytes"
	"path"
	"github.com/alexshemesh/claptrap/lib/types"
	"github.com/alexshemesh/claptrap/lib/logs"
	"io/ioutil"
	"fmt"
	"strconv"
)

var archiveFile = "miners.json"
var hashRateGap = 0.07

type ClaymoreManagerClient struct {
	log      logs.Logger
	address  string
	username string
	password string
}

func NewClaymoreManagerClient(logPar logs.Logger, settings types.Settings) (retVal *ClaymoreManagerClient) {
	retVal = &ClaymoreManagerClient{log: logPar,
	}
	retVal.address, _ = settings.GetValue("claymore/address")
	retVal.username, _ = settings.GetValue("claymore/uname")
	retVal.password, _ = settings.GetValue("claymore/password")
	return retVal
}

func ParseString(RawData string) (retVal []types.MinerEntry, err error) {

	return retVal, err
}

func parseMinerInfoFromRawData(data string) (retVal *types.MinerEntry, err error) {
	ok := strings.Contains(data, "<td>")
	if ok {
		retVal = &types.MinerEntry{Hashrate: make([]string, 2), MiningPool: make([]string, 2)}
		rp, err := regexp.Compile("(COLOR=(.*)>(.*)<)")
		if err != nil {
			return retVal, err
		}
		fieldCounter := 0
		fontEntries := strings.Split(data, "<FONT")
		for _, newValue := range (fontEntries) {
			values := rp.FindAllString(newValue, -1)
			for _, newValue1 := range (values) {
				tagDataStart := strings.Index(newValue1, ">") + 1
				tagDataEnd := strings.Index(newValue1, "<")

				retVal.SetFieldForIndex(string(newValue1[tagDataStart:tagDataEnd]), fieldCounter)
				fieldCounter++

			}
		}
		return retVal, nil
	}
	return retVal, errors.New("No miner info found")
}

func ObjectAsYAMLToString(obj interface{}) (retVal string) {
	var objectContent []byte
	var err error
	objectContent, err = json.Marshal(obj)
	objectasYaml, err := yaml.JSONToYAML(objectContent)
	if err != nil {
		print(err)
	}
	return "\n" + string(objectasYaml)
}

func SplitTable(tableText string) (retVal map[string]types.MinerEntry) {
	retVal = make(map[string]types.MinerEntry)
	lines := strings.Split(tableText, "<tr>")
	for _, val := range (lines) {
		miner, err := parseMinerInfoFromRawData(val)
		if err == nil {
			retVal[miner.MinerName] = *miner
		}
	}
	return retVal
}

func (this ClaymoreManagerClient) GetMinersDataAsString() (retVal string, err error) {

	miners, err := this.GetMinersData( )
	var buffer bytes.Buffer

	for _, miner := range (miners) {
		minerString := miner.ShortDesc()
		buffer.WriteString(minerString)
	}
	retVal = string(buffer.Bytes())
	return retVal, err
}

func (this ClaymoreManagerClient)GetMinersData() (retVal map[string]types.MinerEntry,err error) {
	httpClient := httpClient.NewHttpExecutor().WithBasicAuth(this.username, this.password)
	var u *url.URL
	u, err = url.Parse(path.Join(""))
	u.Scheme = "http"
	u.Host = this.address + ":8193"
	q := u.Query()
	u.RawQuery = q.Encode()
	var response []byte
	body := `{"id":0,"jsonrpc":"2.0","method":"miner_getstat"}`
	response, err = httpClient.Post().Execute(u.String(), nil, []byte(body))
	retVal = SplitTable(string(response))
	return retVal, err
}

//https://api.etherscan.io/api?module=logs&action=getLogs&fromBlock=0&toBlock=latest&address=0xd25e81504e0aea1e1343d61581296bc3c460978b&topic0=0x7bed72de3ef4a9b073e620379f5c21d965578a70

//http://api.etherscan.io/api?module=account&action=txlist&address=0x7bed72de3ef4a9b073e620379f5c21d965578a70&sort=asc

func compareTwoMiners(newMiner types.MinerEntry,oldMiner types.MinerEntry)(res bool, reasons []string,err error){
	res = true
	if len(newMiner.Cards) != len(oldMiner.Cards) {
		reasons = append(reasons,fmt.Sprintf("%s Number of cards different %d -> %d ",newMiner.MinerName, len(oldMiner.Cards),len(newMiner.Cards)  ))
		res = false
	}else {
		oldHashRate0,_ := strconv.ParseFloat(newMiner.Hashrate[0], 64)
		oldHashRate1,_ := strconv.ParseFloat(newMiner.Hashrate[1], 64)
		newHashRate0,_ := strconv.ParseFloat(oldMiner.Hashrate[0], 64)
		newHashRate1,_ := strconv.ParseFloat(oldMiner.Hashrate[1], 64)
		if oldHashRate0 / newHashRate0 < (1 - hashRateGap) || oldHashRate0 / newHashRate0 > (1 + hashRateGap){
			reasons = append(reasons,fmt.Sprintf(" %s Primary hashrate changed  %s -> %s ",newMiner.MinerName, oldMiner.Hashrate[0],newMiner.Hashrate[0]  ))
			res = false
		}else if oldHashRate1 / newHashRate1 < (1 - hashRateGap) || oldHashRate1 / newHashRate1 > (1 + hashRateGap){
			reasons = append(reasons,fmt.Sprintf("%s Secondary hashrate changed  %s -> %s ",newMiner.MinerName,oldMiner.Hashrate[1],newMiner.Hashrate[1] ))
			res = false
		}
	}
	return res,reasons,err
}

func (this ClaymoreManagerClient) CompareSetOfMiners(oldMiners map[string]types.MinerEntry, newMiners map[string]types.MinerEntry )(res bool, reasons []string,err error){
	for key, val  := range newMiners{
		oldVal, ok := oldMiners[key];
		if  !ok {
			reasons = append(reasons, "New Miner " + val.ShortDesc())
		}else{
			res,reasons,err = compareTwoMiners(val,oldVal)
			if res != true{
				break
			}
		}
	}
	return res,reasons,err
}

func (this ClaymoreManagerClient)LoadMinersFromFile(fileName string) (retVal map[string]types.MinerEntry,err error) {
	retVal = make(map[string]types.MinerEntry,0)

	fileContent, err := ioutil.ReadFile(fileName)
	if err == nil {
		err = json.Unmarshal(fileContent, &retVal)
	}

	return retVal, err
	}

func (this ClaymoreManagerClient)SaveMinersToFile(filename string,miners map[string]types.MinerEntry)(err error) {
	minersJson, err := json.Marshal(miners)
	if err == nil {
		err = ioutil.WriteFile(filename, []byte(minersJson), 0644)
	}
	return  err
}

func (this ClaymoreManagerClient) CheckAndCompare() (res bool, reasons []string, err error) {
	newMiners,err := this.GetMinersData()

	oldMiners,err := this.LoadMinersFromFile(archiveFile)

	res,reasons,err = this.CompareSetOfMiners(oldMiners,newMiners)
	if len( newMiners ) > 0 {
		err = this.SaveMinersToFile(archiveFile, newMiners)
	}
	return res,reasons,err
}<|endoftext|>


<commit_before>package transports

import (
	"errors"
	"fmt"
	"net/http"
	"strings"

	"github.com/golang/protobuf/proto"
	"github.com/matiasinsaurralde/transports/marshalers/protos"
	// "log"
)

type ProtobufMarshaler struct {
}

func (marshaler ProtobufMarshaler) Marshal(i *interface{}) (error, interface{}) {
	var err error
	var r interface{}

	if i == nil {
		err = errors.New(MarshalerNilTypeError)
		return err, r
	}

	switch t := (*i).(type) {
	case *http.Request:
		request := (*i).(*http.Request)
		requestProto := &transportsProto.HttpRequest{
			Method: proto.String(request.Method),
			Url:    proto.String(request.URL.String()),
			Proto:  proto.String(request.Proto),
		}
		r, err = proto.Marshal(requestProto)
	case *http.Response:
	default:
		message := fmt.Sprintf(MarshalerTypeNotSupportedError)
		typestr := fmt.Sprintf("%T", t)
		err = errors.New(strings.Join([]string{message, typestr}, " "))
	}

	return err, r
}

func (marshaler ProtobufMarshaler) Unmarshal() {
	return
}
<commit_msg>Implementing Protobuf Unmarshal()<commit_after>package transports

import (
	"errors"
	"fmt"
	"net/http"
	"strings"

	"github.com/golang/protobuf/proto"
	"github.com/matiasinsaurralde/transports/marshalers/protos"
)

type ProtobufMarshaler struct {
}

func (marshaler ProtobufMarshaler) Marshal(i *interface{}) (error, interface{}) {
	var err error
	var r interface{}

	if i == nil {
		err = errors.New(MarshalerNilTypeError)
		return err, r
	}

	switch t := (*i).(type) {
	case *http.Request:
		request := (*i).(*http.Request)
		requestProto := &transportsProto.HttpRequest{
			Method: proto.String(request.Method),
			Url:    proto.String(request.URL.String()),
			Proto:  proto.String(request.Proto),
		}
		r, err = proto.Marshal(requestProto)
	case *http.Response:
	default:
		message := fmt.Sprintf(MarshalerTypeNotSupportedError)
		typestr := fmt.Sprintf("%T", t)
		err = errors.New(strings.Join([]string{message, typestr}, " "))
	}

	return err, r
}

func (marshaler ProtobufMarshaler) Unmarshal(i *interface{}) (error, interface{}) {
	var err error
	var r interface{}

	switch t := (*i).(type) {
	case []byte:
	default:
		message := fmt.Sprintf(MarshalerTypeNotSupportedError)
		typestr := fmt.Sprintf("%T", t)
		err = errors.New(strings.Join([]string{message, typestr}, " "))
	}

	buffer := (*i).([]byte)

	HttpResponse := &transportsProto.HttpResponse{}
	err = proto.Unmarshal(buffer, HttpResponse)

	if err == nil {
		r = HttpResponse
		return err, r
	}

	HttpRequest := &transportsProto.HttpRequest{}
	err = proto.Unmarshal(buffer, HttpRequest)

	if err == nil {
		r = HttpRequest
		return err, r
	}

	return err, nil
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2014 The foocsim Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kvdb

import (
	"fmt"
	"github.com/lpabon/bufferio"
	"github.com/lpabon/godbc"
	"os"
	"syscall"
)

const (
	KB = 1024
	MB = 1024 * KB
	GB = 1024 * MB
	TB = 1024 * GB
)

type IoSegment struct {
	max          uint64
	size         uint64
	metadatasize uint64
	datasize     uint64
}

type IoEntryKey struct {
	key string
}

type KVIoDB struct {
	size       uint64
	blocksize  uint64
	segment    IoSegment
	current    uint64
	entry      uint64
	maxentries uint64
	segmentbuf []byte
	data       *bufferio.BufferIO
	meta       *bufferio.BufferIO
	fp         *os.File
}

func NewKVIoDB(dbpath string, blocks uint64, blocksize uint32) *KVIoDB {

	var err error

	db := &KVIoDB{}
	db.size = blocks * uint64(blocksize)
	db.blocksize = uint64(blocksize)
	db.segment.metadatasize = 4 * KB
	db.segment.datasize = 4 * MB
	db.maxentries = db.segment.datasize / db.blocksize
	db.segment.size = db.segment.metadatasize + db.segment.datasize
	db.segment.max = db.size / uint64(db.segment.size)

	db.segmentbuf = make([]byte, db.segment.size)
	db.data = bufferio.NewBufferIO(db.segmentbuf[:db.segment.datasize])
	db.meta = bufferio.NewBufferIO(db.segmentbuf[db.segment.datasize:])

	db.fp, err = os.OpenFile(dbpath, syscall.O_DIRECT|os.O_CREATE|os.O_RDWR, os.ModePerm)
	godbc.Check(err == nil)

	return db
}

func (c *KVIoDB) sync() {
	c.fp.WriteAt(c.segmentbuf, int64(c.current))
	c.current += c.segment.size
	c.current = c.current % c.size
	c.data.Reset()
	c.meta.Reset()
}

func (c *KVIoDB) Close() {
	c.sync()
	c.fp.Close()
}

func (c *KVIoDB) Put(key, val []byte, index uint64) error {

	var keyentry IoEntryKey

	c.data.Write(val)
	keyentry.key = string(key)
	c.meta.WriteDataLE(keyentry)
	c.entry++

	if c.entry > c.maxentries {
		c.sync()
	}

	return nil
}

func (c *KVIoDB) Get(key []byte, index uint64) ([]byte, error) {

	buf := make([]byte, c.blocksize)
	offset := (index*c.blocksize + (index/c.maxentries)*c.segment.metadatasize)
	n, err := c.fp.ReadAt(buf, int64(offset))
	godbc.Check(uint64(n) == c.blocksize,
		fmt.Sprintf("Read %v expected %v from location %v",
			n, c.blocksize, offset))
	godbc.Check(err == nil)

	return buf, nil
}

func (c *KVIoDB) Delete(key []byte, index uint64) error {
	// nothing to do
	return nil
}
<commit_msg>Read hit from current ram buffer<commit_after>//
// Copyright (c) 2014 The foocsim Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kvdb

import (
	"fmt"
	"github.com/lpabon/bufferio"
	"github.com/lpabon/godbc"
	"os"
	"syscall"
)

const (
	KB = 1024
	MB = 1024 * KB
	GB = 1024 * MB
	TB = 1024 * GB
)

type IoSegment struct {
	max          uint64
	size         uint64
	metadatasize uint64
	datasize     uint64
}

type IoEntryKey struct {
	key string
}

type KVIoDB struct {
	size       uint64
	blocksize  uint64
	segment    IoSegment
	current    uint64
	entry      uint64
	maxentries uint64
	segmentbuf []byte
	data       *bufferio.BufferIO
	meta       *bufferio.BufferIO
	fp         *os.File
}

func NewKVIoDB(dbpath string, blocks uint64, blocksize uint32) *KVIoDB {

	var err error

	db := &KVIoDB{}
	db.size = blocks * uint64(blocksize)
	db.blocksize = uint64(blocksize)
	db.segment.metadatasize = 4 * KB
	db.segment.datasize = 4 * MB
	db.maxentries = db.segment.datasize / db.blocksize
	db.segment.size = db.segment.metadatasize + db.segment.datasize
	db.segment.max = db.size / uint64(db.segment.size)

	db.segmentbuf = make([]byte, db.segment.size)
	db.data = bufferio.NewBufferIO(db.segmentbuf[:db.segment.datasize])
	db.meta = bufferio.NewBufferIO(db.segmentbuf[db.segment.datasize:])

	db.fp, err = os.OpenFile(dbpath, syscall.O_DIRECT|os.O_CREATE|os.O_RDWR, os.ModePerm)
	godbc.Check(err == nil)

	return db
}

func (c *KVIoDB) sync() {
	c.fp.WriteAt(c.segmentbuf, int64(c.current))
	c.current += c.segment.size
	c.current = c.current % c.size
	c.data.Reset()
	c.meta.Reset()
}

func (c *KVIoDB) Close() {
	c.sync()
	c.fp.Close()
}

func (c *KVIoDB) Put(key, val []byte, index uint64) error {

	var keyentry IoEntryKey

	c.data.Write(val)
	keyentry.key = string(key)
	c.meta.WriteDataLE(keyentry)
	c.entry++

	if c.entry > c.maxentries {
		c.sync()
	}

	return nil
}

func (c *KVIoDB) Get(key []byte, index uint64) ([]byte, error) {

	var n int
	var err error

	buf := make([]byte, c.blocksize)
	offset := (index*c.blocksize + (index/c.maxentries)*c.segment.metadatasize)

	if (offset > c.current) && (offset < (c.current + c.segment.size)) {
		n, err = c.data.ReadAt(buf, int64(offset-c.current))
		fmt.Printf("+RAM\n")
	} else {
		n, err = c.fp.ReadAt(buf, int64(offset))
	}
	godbc.Check(uint64(n) == c.blocksize,
		fmt.Sprintf("Read %v expected %v from location %v",
			n, c.blocksize, offset))
	godbc.Check(err == nil)

	return buf, nil
}

func (c *KVIoDB) Delete(key []byte, index uint64) error {
	// nothing to do
	return nil
}
<|endoftext|>


<commit_before><commit_msg>=修复websocket库的bug<commit_after><|endoftext|>


<commit_before>package main

import (
	"bufio"
	"os"
)

func initRIO() *RObject {
	obj := &RObject{}
	obj.name = "RIO"
	obj.ivars = make(map[string]Object)
	obj.class = nil
	obj.methods = make(map[string]*RMethod)

	// RString method initialization
	obj.methods["readlines"] = &RMethod{gofunc: RIO_readlines}

	return obj
}

// IO.readlines(filename)
// [RString]
func RIO_readlines(vm *GobiesVM, receiver Object, v []Object) Object {
	filename := v[0].(*RObject).val.str

	obj := RArray_new(vm, receiver, nil)

	input, _ := os.Open(filename)
	scanner := bufio.NewScanner(input)

	for scanner.Scan() {
		dummy_obj := make([]Object, 1, 1)
		dummy_obj[0] = scanner.Text()
		rstr := RString_new(vm, receiver, dummy_obj)
		dummy_obj[0] = rstr
		RArray_append(vm, obj, dummy_obj)
	}

	return obj
}
<commit_msg>[IO] revise readlines() for large files<commit_after>package main

import (
	"bufio"
	"os"
)

func initRIO() *RObject {
	obj := &RObject{}
	obj.name = "RIO"
	obj.ivars = make(map[string]Object)
	obj.class = nil
	obj.methods = make(map[string]*RMethod)

	// RString method initialization
	obj.methods["readlines"] = &RMethod{gofunc: RIO_readlines}

	return obj
}

// IO.readlines(filename)
// [RString]
func RIO_readlines(vm *GobiesVM, receiver Object, v []Object) Object {
	filename := v[0].(*RObject).val.str

	obj := RArray_new(vm, receiver, nil)

	input, _ := os.Open(filename)
	defer input.Close()

	reader := bufio.NewReader(input)

	line, err := reader.ReadString('\n')
	for ; err == nil; line, err = reader.ReadString('\n') {
		dummy_obj := make([]Object, 1, 1)
		dummy_obj[0] = line
		rstr := RString_new(vm, receiver, dummy_obj)
		dummy_obj[0] = rstr
		RArray_append(vm, obj, dummy_obj)
	}

	return obj
}
<|endoftext|>


<commit_before>package taskfile

// Defer is the parameters to a defer operation.
// It can be exactly one of:
// - A string command
// - A task call
type Defer struct {
	Cmd  string
	Call *Call
}

// isValid returns true when Defer describes a valid action.
// In order for a Defer to be valid, one of Cmd or Call.Task
// must be non-empty.
func (d *Defer) isValid() bool {
	return d.Cmd != "" || (d.Call != nil && d.Call.Task != "")
}

// UnmarshalYAML implements yaml.Unmarshaler interface
func (d *Defer) UnmarshalYAML(unmarshal func(interface{}) error) error {
	var cmd string
	if err := unmarshal(&cmd); err == nil {
		d.Cmd = cmd
		return nil
	}
	var taskCall struct {
		Task string
		Vars *Vars
	}
	if err := unmarshal(&taskCall); err != nil {
		return err
	}
	d.Call = &Call{
		Task: taskCall.Task,
		Vars: taskCall.Vars,
	}
	return nil
}
<commit_msg>remove extra file<commit_after><|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/lomik/carbon-clickhouse/carbon"
	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
	"github.com/lomik/zapwriter"
	"go.uber.org/zap"

	_ "net/http/pprof"
)

// Version of carbon-clickhouse
const Version = "0.11.0.custom.2"

func httpServe(addr string) (func(), error) {
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, err
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, nil)
	return func() { listener.Close() }, nil
}

func main() {
	var err error

	/* CONFIG start */

	configFile := flag.String("config", "/etc/carbon-clickhouse/carbon-clickhouse.conf", "Filename of config")
	printDefaultConfig := flag.Bool("config-print-default", false, "Print default config")
	checkConfig := flag.Bool("check-config", false, "Check config and exit")
	printVersion := flag.Bool("version", false, "Print version")
	cat := flag.String("cat", "", "Print RowBinary file in TabSeparated format")
	bincat := flag.String("recover", "", "Read all good records from corrupted data file. Write binary data to stdout")

	flag.Parse()

	if *printVersion {
		fmt.Print(Version)
		return
	}

	if *cat != "" {
		reader, err := RowBinary.NewReader(*cat, false)
		if err != nil {
			log.Fatal(err)
		}

		for {
			metric, err := reader.ReadRecord()
			if err != nil {
				if err == io.EOF {
					return
				}
				log.Fatal(err)
			}

			fmt.Printf("%s\t%#v\t%d\t%s\t%d\n",
				string(metric),
				reader.Value(),
				reader.Timestamp(),
				reader.DaysString(),
				reader.Version(),
			)
		}
	}

	if *bincat != "" {
		reader, err := RowBinary.NewReader(*bincat, false)
		if err != nil {
			log.Fatal(err)
		}

		io.Copy(os.Stdout, reader)
		return
	}

	if *printDefaultConfig {
		if err = carbon.PrintDefaultConfig(); err != nil {
			log.Fatal(err)
		}
		return
	}

	app := carbon.New(*configFile)

	if err = app.ParseConfig(); err != nil {
		log.Fatal(err)
	}

	// config parsed successfully. Exit in check-only mode
	if *checkConfig {
		return
	}

	cfg := app.Config

	if err = zapwriter.ApplyConfig(cfg.Logging); err != nil {
		log.Fatal(err)
	}

	mainLogger := zapwriter.Logger("main")

	/* CONFIG end */

	// pprof
	if cfg.Pprof.Enabled {
		_, err = httpServe(cfg.Pprof.Listen)
		if err != nil {
			mainLogger.Fatal("pprof listen failed", zap.Error(err))
		}
	}

	if err = app.Start(); err != nil {
		mainLogger.Fatal("app start failed", zap.Error(err))
	} else {
		mainLogger.Info("app started")
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)

		for {
			s := <-c
			switch s {
			case syscall.SIGUSR1:
				mainLogger.Info("SIGUSR1 received. Clear tree cache")
				app.Reset()
			case syscall.SIGUSR2:
				mainLogger.Info("SIGUSR2 received. Ignoring")
			case syscall.SIGHUP:
				mainLogger.Info("SIGHUP received. Ignoring")
			}
		}
	}()

	app.Loop()

	mainLogger.Info("app stopped")
}
<commit_msg>AD-12849 don't change Version<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/lomik/carbon-clickhouse/carbon"
	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
	"github.com/lomik/zapwriter"
	"go.uber.org/zap"

	_ "net/http/pprof"
)

// Version of carbon-clickhouse
const Version = "0.11.0"

func httpServe(addr string) (func(), error) {
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, err
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, nil)
	return func() { listener.Close() }, nil
}

func main() {
	var err error

	/* CONFIG start */

	configFile := flag.String("config", "/etc/carbon-clickhouse/carbon-clickhouse.conf", "Filename of config")
	printDefaultConfig := flag.Bool("config-print-default", false, "Print default config")
	checkConfig := flag.Bool("check-config", false, "Check config and exit")
	printVersion := flag.Bool("version", false, "Print version")
	cat := flag.String("cat", "", "Print RowBinary file in TabSeparated format")
	bincat := flag.String("recover", "", "Read all good records from corrupted data file. Write binary data to stdout")

	flag.Parse()

	if *printVersion {
		fmt.Print(Version)
		return
	}

	if *cat != "" {
		reader, err := RowBinary.NewReader(*cat, false)
		if err != nil {
			log.Fatal(err)
		}

		for {
			metric, err := reader.ReadRecord()
			if err != nil {
				if err == io.EOF {
					return
				}
				log.Fatal(err)
			}

			fmt.Printf("%s\t%#v\t%d\t%s\t%d\n",
				string(metric),
				reader.Value(),
				reader.Timestamp(),
				reader.DaysString(),
				reader.Version(),
			)
		}
	}

	if *bincat != "" {
		reader, err := RowBinary.NewReader(*bincat, false)
		if err != nil {
			log.Fatal(err)
		}

		io.Copy(os.Stdout, reader)
		return
	}

	if *printDefaultConfig {
		if err = carbon.PrintDefaultConfig(); err != nil {
			log.Fatal(err)
		}
		return
	}

	app := carbon.New(*configFile)

	if err = app.ParseConfig(); err != nil {
		log.Fatal(err)
	}

	// config parsed successfully. Exit in check-only mode
	if *checkConfig {
		return
	}

	cfg := app.Config

	if err = zapwriter.ApplyConfig(cfg.Logging); err != nil {
		log.Fatal(err)
	}

	mainLogger := zapwriter.Logger("main")

	/* CONFIG end */

	// pprof
	if cfg.Pprof.Enabled {
		_, err = httpServe(cfg.Pprof.Listen)
		if err != nil {
			mainLogger.Fatal("pprof listen failed", zap.Error(err))
		}
	}

	if err = app.Start(); err != nil {
		mainLogger.Fatal("app start failed", zap.Error(err))
	} else {
		mainLogger.Info("app started")
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)

		for {
			s := <-c
			switch s {
			case syscall.SIGUSR1:
				mainLogger.Info("SIGUSR1 received. Clear tree cache")
				app.Reset()
			case syscall.SIGUSR2:
				mainLogger.Info("SIGUSR2 received. Ignoring")
			case syscall.SIGHUP:
				mainLogger.Info("SIGHUP received. Ignoring")
			}
		}
	}()

	app.Loop()

	mainLogger.Info("app stopped")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/lomik/carbon-clickhouse/carbon"
	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
	"github.com/lomik/zapwriter"
	"go.uber.org/zap"

	_ "net/http/pprof"
)

// Version of carbon-clickhouse
const Version = "0.8.1"

func httpServe(addr string) (func(), error) {
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, err
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, nil)
	return func() { listener.Close() }, nil
}

func main() {
	var err error

	/* CONFIG start */

	configFile := flag.String("config", "/etc/carbon-clickhouse/carbon-clickhouse.conf", "Filename of config")
	printDefaultConfig := flag.Bool("config-print-default", false, "Print default config")
	checkConfig := flag.Bool("check-config", false, "Check config and exit")
	printVersion := flag.Bool("version", false, "Print version")
	cat := flag.String("cat", "", "Print RowBinary file in TabSeparated format")
	bincat := flag.String("recover", "", "Read all good records from corrupted data file. Write binary data to stdout")

	flag.Parse()

	if *printVersion {
		fmt.Print(Version)
		return
	}

	if *cat != "" {
		reader, err := RowBinary.NewReader(*cat)
		if err != nil {
			log.Fatal(err)
		}

		for {
			metric, err := reader.ReadRecord()
			if err != nil {
				if err == io.EOF {
					return
				}
				log.Fatal(err)
			}

			fmt.Printf("%s\t%#v\t%d\t%s\t%d\n",
				string(metric),
				reader.Value(),
				reader.Timestamp(),
				reader.DaysString(),
				reader.Version(),
			)
		}
	}

	if *bincat != "" {
		reader, err := RowBinary.NewReader(*bincat)
		if err != nil {
			log.Fatal(err)
		}

		io.Copy(os.Stdout, reader)
		return
	}

	if *printDefaultConfig {
		if err = carbon.PrintDefaultConfig(); err != nil {
			log.Fatal(err)
		}
		return
	}

	app := carbon.New(*configFile)

	if err = app.ParseConfig(); err != nil {
		log.Fatal(err)
	}

	// config parsed successfully. Exit in check-only mode
	if *checkConfig {
		return
	}

	cfg := app.Config

	if err = zapwriter.ApplyConfig(cfg.Logging); err != nil {
		log.Fatal(err)
	}

	mainLogger := zapwriter.Logger("main")

	/* CONFIG end */

	// pprof
	if cfg.Pprof.Enabled {
		_, err = httpServe(cfg.Pprof.Listen)
		if err != nil {
			mainLogger.Fatal("pprof listen failed", zap.Error(err))
		}
	}

	if err = app.Start(); err != nil {
		mainLogger.Fatal("app start failed", zap.Error(err))
	} else {
		mainLogger.Info("app started")
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)

		for {
			s := <-c
			switch s {
			case syscall.SIGUSR1:
				mainLogger.Info("SIGUSR1 received. Clear tree cache")
				app.Reset()
			case syscall.SIGUSR2:
				mainLogger.Info("SIGUSR2 received. Ignoring")
			case syscall.SIGHUP:
				mainLogger.Info("SIGHUP received. Ignoring")
			}
		}
	}()

	app.Loop()

	mainLogger.Info("app stopped")
}
<commit_msg>version 0.8.2<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/lomik/carbon-clickhouse/carbon"
	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
	"github.com/lomik/zapwriter"
	"go.uber.org/zap"

	_ "net/http/pprof"
)

// Version of carbon-clickhouse
const Version = "0.8.2"

func httpServe(addr string) (func(), error) {
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, err
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, nil)
	return func() { listener.Close() }, nil
}

func main() {
	var err error

	/* CONFIG start */

	configFile := flag.String("config", "/etc/carbon-clickhouse/carbon-clickhouse.conf", "Filename of config")
	printDefaultConfig := flag.Bool("config-print-default", false, "Print default config")
	checkConfig := flag.Bool("check-config", false, "Check config and exit")
	printVersion := flag.Bool("version", false, "Print version")
	cat := flag.String("cat", "", "Print RowBinary file in TabSeparated format")
	bincat := flag.String("recover", "", "Read all good records from corrupted data file. Write binary data to stdout")

	flag.Parse()

	if *printVersion {
		fmt.Print(Version)
		return
	}

	if *cat != "" {
		reader, err := RowBinary.NewReader(*cat)
		if err != nil {
			log.Fatal(err)
		}

		for {
			metric, err := reader.ReadRecord()
			if err != nil {
				if err == io.EOF {
					return
				}
				log.Fatal(err)
			}

			fmt.Printf("%s\t%#v\t%d\t%s\t%d\n",
				string(metric),
				reader.Value(),
				reader.Timestamp(),
				reader.DaysString(),
				reader.Version(),
			)
		}
	}

	if *bincat != "" {
		reader, err := RowBinary.NewReader(*bincat)
		if err != nil {
			log.Fatal(err)
		}

		io.Copy(os.Stdout, reader)
		return
	}

	if *printDefaultConfig {
		if err = carbon.PrintDefaultConfig(); err != nil {
			log.Fatal(err)
		}
		return
	}

	app := carbon.New(*configFile)

	if err = app.ParseConfig(); err != nil {
		log.Fatal(err)
	}

	// config parsed successfully. Exit in check-only mode
	if *checkConfig {
		return
	}

	cfg := app.Config

	if err = zapwriter.ApplyConfig(cfg.Logging); err != nil {
		log.Fatal(err)
	}

	mainLogger := zapwriter.Logger("main")

	/* CONFIG end */

	// pprof
	if cfg.Pprof.Enabled {
		_, err = httpServe(cfg.Pprof.Listen)
		if err != nil {
			mainLogger.Fatal("pprof listen failed", zap.Error(err))
		}
	}

	if err = app.Start(); err != nil {
		mainLogger.Fatal("app start failed", zap.Error(err))
	} else {
		mainLogger.Info("app started")
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)

		for {
			s := <-c
			switch s {
			case syscall.SIGUSR1:
				mainLogger.Info("SIGUSR1 received. Clear tree cache")
				app.Reset()
			case syscall.SIGUSR2:
				mainLogger.Info("SIGUSR2 received. Ignoring")
			case syscall.SIGHUP:
				mainLogger.Info("SIGHUP received. Ignoring")
			}
		}
	}()

	app.Loop()

	mainLogger.Info("app stopped")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/lomik/carbon-clickhouse/carbon"
	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
	"github.com/lomik/zapwriter"
	"go.uber.org/zap"

	_ "net/http/pprof"
)

// Version of carbon-clickhouse
const Version = "0.11.1"

func httpServe(addr string) (func(), error) {
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, err
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, nil)
	return func() { listener.Close() }, nil
}

func main() {
	var err error

	/* CONFIG start */

	configFile := flag.String("config", "/etc/carbon-clickhouse/carbon-clickhouse.conf", "Filename of config")
	printDefaultConfig := flag.Bool("config-print-default", false, "Print default config")
	checkConfig := flag.Bool("check-config", false, "Check config and exit")
	printVersion := flag.Bool("version", false, "Print version")
	cat := flag.String("cat", "", "Print RowBinary file in TabSeparated format")
	bincat := flag.String("recover", "", "Read all good records from corrupted data file. Write binary data to stdout")

	flag.Parse()

	if *printVersion {
		fmt.Print(Version)
		return
	}

	if *cat != "" {
		reader, err := RowBinary.NewReader(*cat, false)
		if err != nil {
			log.Fatal(err)
		}

		for {
			metric, err := reader.ReadRecord()
			if err != nil {
				if err == io.EOF {
					return
				}
				log.Fatal(err)
			}

			fmt.Printf("%s\t%#v\t%d\t%s\t%d\n",
				string(metric),
				reader.Value(),
				reader.Timestamp(),
				reader.DaysString(),
				reader.Version(),
			)
		}
	}

	if *bincat != "" {
		reader, err := RowBinary.NewReader(*bincat, false)
		if err != nil {
			log.Fatal(err)
		}

		io.Copy(os.Stdout, reader)
		return
	}

	if *printDefaultConfig {
		if err = carbon.PrintDefaultConfig(); err != nil {
			log.Fatal(err)
		}
		return
	}

	app := carbon.New(*configFile)

	if err = app.ParseConfig(); err != nil {
		log.Fatal(err)
	}

	// config parsed successfully. Exit in check-only mode
	if *checkConfig {
		return
	}

	cfg := app.Config

	if err = zapwriter.ApplyConfig(cfg.Logging); err != nil {
		log.Fatal(err)
	}

	mainLogger := zapwriter.Logger("main")

	/* CONFIG end */

	// pprof
	if cfg.Pprof.Enabled {
		_, err = httpServe(cfg.Pprof.Listen)
		if err != nil {
			mainLogger.Fatal("pprof listen failed", zap.Error(err))
		}
	}

	if err = app.Start(); err != nil {
		mainLogger.Fatal("app start failed", zap.Error(err))
	} else {
		mainLogger.Info("app started")
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)

		for {
			s := <-c
			switch s {
			case syscall.SIGUSR1:
				mainLogger.Info("SIGUSR1 received. Clear tree cache")
				app.Reset()
			case syscall.SIGUSR2:
				mainLogger.Info("SIGUSR2 received. Ignoring")
			case syscall.SIGHUP:
				mainLogger.Info("SIGHUP received. Ignoring")
			}
		}
	}()

	app.Loop()

	mainLogger.Info("app stopped")
}
<commit_msg>periodic return memory to os<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"runtime/debug"
	"syscall"
	"time"

	"github.com/lomik/carbon-clickhouse/carbon"
	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
	"github.com/lomik/zapwriter"
	"go.uber.org/zap"

	_ "net/http/pprof"
)

// Version of carbon-clickhouse
const Version = "0.11.1"

func httpServe(addr string) (func(), error) {
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, err
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		return nil, err
	}

	go http.Serve(listener, nil)
	return func() { listener.Close() }, nil
}

func main() {
	var err error

	/* CONFIG start */

	configFile := flag.String("config", "/etc/carbon-clickhouse/carbon-clickhouse.conf", "Filename of config")
	printDefaultConfig := flag.Bool("config-print-default", false, "Print default config")
	checkConfig := flag.Bool("check-config", false, "Check config and exit")
	printVersion := flag.Bool("version", false, "Print version")
	cat := flag.String("cat", "", "Print RowBinary file in TabSeparated format")
	bincat := flag.String("recover", "", "Read all good records from corrupted data file. Write binary data to stdout")

	flag.Parse()

	if *printVersion {
		fmt.Print(Version)
		return
	}

	if *cat != "" {
		reader, err := RowBinary.NewReader(*cat, false)
		if err != nil {
			log.Fatal(err)
		}

		for {
			metric, err := reader.ReadRecord()
			if err != nil {
				if err == io.EOF {
					return
				}
				log.Fatal(err)
			}

			fmt.Printf("%s\t%#v\t%d\t%s\t%d\n",
				string(metric),
				reader.Value(),
				reader.Timestamp(),
				reader.DaysString(),
				reader.Version(),
			)
		}
	}

	if *bincat != "" {
		reader, err := RowBinary.NewReader(*bincat, false)
		if err != nil {
			log.Fatal(err)
		}

		io.Copy(os.Stdout, reader)
		return
	}

	if *printDefaultConfig {
		if err = carbon.PrintDefaultConfig(); err != nil {
			log.Fatal(err)
		}
		return
	}

	app := carbon.New(*configFile)

	if err = app.ParseConfig(); err != nil {
		log.Fatal(err)
	}

	// config parsed successfully. Exit in check-only mode
	if *checkConfig {
		return
	}

	cfg := app.Config

	if err = zapwriter.ApplyConfig(cfg.Logging); err != nil {
		log.Fatal(err)
	}

	mainLogger := zapwriter.Logger("main")

	/* CONFIG end */

	// pprof
	if cfg.Pprof.Enabled {
		_, err = httpServe(cfg.Pprof.Listen)
		if err != nil {
			mainLogger.Fatal("pprof listen failed", zap.Error(err))
		}
	}

	if err = app.Start(); err != nil {
		mainLogger.Fatal("app start failed", zap.Error(err))
	} else {
		mainLogger.Info("app started")
	}

	go func() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)

		for {
			s := <-c
			switch s {
			case syscall.SIGUSR1:
				mainLogger.Info("SIGUSR1 received. Clear tree cache")
				app.Reset()
			case syscall.SIGUSR2:
				mainLogger.Info("SIGUSR2 received. Ignoring")
			case syscall.SIGHUP:
				mainLogger.Info("SIGHUP received. Ignoring")
			}
		}
	}()

	go func() {
		for {
			debug.FreeOSMemory()
			time.Sleep(30 * time.Minute)
		}
	}()

	app.Loop()

	mainLogger.Info("app stopped")
}
<|endoftext|>