<commit_before>package jsonmap

import "testing"

type InnerThing struct {
	Foo   string
	AnInt int
	ABool bool
}

type OuterThing struct {
	Bar        string
	InnerThing InnerThing
}

type UnregisteredThing struct {
}

var InnerThingTypeMap = TypeMap{
	InnerThing{},
	[]MappedField{
		{
			StructFieldName: "Foo",
			JSONFieldName:   "foo",
			Validator:       String(1, 12),
			Optional:        true,
		},
		{
			StructFieldName: "AnInt",
			JSONFieldName:   "an_int",
			Validator:       Integer(0, 10),
			Optional:        true,
		},
		{
			StructFieldName: "ABool",
			JSONFieldName:   "a_bool",
			Validator:       Boolean(),
			Optional:        true,
		},
	},
}

var OuterThingTypeMap = TypeMap{
	OuterThing{},
	[]MappedField{
		{
			StructFieldName: "Bar",
			JSONFieldName:   "bar",
			Validator:       String(1, 255),
		},
		{
			StructFieldName: "InnerThing",
			JSONFieldName:   "inner_thing",
			Contains:        InnerThingTypeMap,
		},
	},
}

var TestTypeMapper = NewTypeMapper(
	InnerThingTypeMap,
	OuterThingTypeMap,
)

func TestValidateInnerThing(t *testing.T) {
	v := &InnerThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"foo": "fooz", "an_int": 10, "a_bool": true}`), v)
	if err != nil {
		t.Fatal(err)
	}
	if v.Foo != "fooz" {
		t.Fatal("Field Foo does not have expected value 'fooz':", v.Foo)
	}
}

func TestValidateOuterThing(t *testing.T) {
	v := &OuterThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"bar": "bazam", "inner_thing": {"foo": "fooz"}}`), v)
	if err != nil {
		t.Fatal(err)
	}
	if v.InnerThing.Foo != "fooz" {
		t.Fatal("Inner field Foo does not have expected value 'fooz':", v.InnerThing.Foo)
	}
	if v.Bar != "bazam" {
		t.Fatal("Outer field Bar does not have expected value 'bazam':", v.Bar)
	}
}

func TestValidateUnregisteredThing(t *testing.T) {
	defer func() {
		if r := recover(); r == nil {
			t.Fatal("No panic")
		}
	}()
	v := &UnregisteredThing{}
	TestTypeMapper.Unmarshal([]byte(`{}`), v)
	t.Fatal("Unexpected success")
}

func TestValidateStringTypeMismatch(t *testing.T) {
	v := &InnerThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"foo": 12.0}`), v)
	if err == nil {
		t.Fatal("Unexpected success")
	}
	if err.Error() != "error validating field 'foo': not a string" {
		t.Fatal("Unexpected error message:", err.Error())
	}
}

func TestValidateStringTooShort(t *testing.T) {
	v := &InnerThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"foo": ""}`), v)
	if err == nil {
		t.Fatal("Unexpected success")
	}
	if err.Error() != "error validating field 'foo': too short, must be at least 1 characters" {
		t.Fatal("Unexpected error message:", err.Error())
	}
}

func TestValidateStringTooLong(t *testing.T) {
	v := &InnerThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"foo": "thisvalueistoolong"}`), v)
	if err == nil {
		t.Fatal("Unexpected success")
	}
	if err.Error() != "error validating field 'foo': too long, may not be more than 12 characters" {
		t.Fatal("Unexpected error message:", err.Error())
	}
}

func TestValidateBooleanTypeMismatch(t *testing.T) {
	v := &InnerThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"a_bool": 12.0}`), v)
	if err == nil {
		t.Fatal("Unexpected success")
	}
	if err.Error() != "error validating field 'a_bool': not a boolean" {
		t.Fatal("Unexpected error message:", err.Error())
	}
}

func TestValidateIntegerTypeMismatch(t *testing.T) {
	v := &InnerThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"an_int": false}`), v)
	if err == nil {
		t.Fatal("Unexpected success")
	}
	if err.Error() != "error validating field 'an_int': not an integer" {
		t.Fatal("Unexpected error message:", err.Error())
	}
}

func TestValidateIntegerNumericTypeMismatch(t *testing.T) {
	v := &InnerThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"an_int": 12.1}`), v)
	if err == nil {
		t.Fatal("Unexpected success")
	}
	if err.Error() != "error validating field 'an_int': not an integer" {
		t.Fatal("Unexpected error message:", err.Error())
	}
}

func TestValidateIntegerTooSmall(t *testing.T) {
	v := &InnerThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"an_int": -1}`), v)
	if err == nil {
		t.Fatal("Unexpected success")
	}
	if err.Error() != "error validating field 'an_int': too small, must be at least 0" {
		t.Fatal("Unexpected error message:", err.Error())
	}
}

func TestValidateIntegerTooLarge(t *testing.T) {
	v := &InnerThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"an_int": 2048}`), v)
	if err == nil {
		t.Fatal("Unexpected success")
	}
	if err.Error() != "error validating field 'an_int': too large, may not be larger than 10" {
		t.Fatal("Unexpected error message:", err.Error())
	}
}

func TestMarshalInnerThing(t *testing.T) {
	v := &InnerThing{
		Foo:   "bar",
		AnInt: 7,
		ABool: true,
	}
	data, err := TestTypeMapper.Marshal(v)
	if err != nil {
		t.Fatal(err)
	}
	if string(data) != `{"a_bool":true,"an_int":7,"foo":"bar"}` {
		t.Fatal("Unexpected Marshal output:", string(data))
	}
}

func TestMarshalOuterThing(t *testing.T) {
	v := &OuterThing{
		Bar: "bang",
		InnerThing: InnerThing{
			Foo:   "bar",
			AnInt: 3,
			ABool: false,
		},
	}
	data, err := TestTypeMapper.Marshal(v)
	if err != nil {
		t.Fatal(err)
	}
	if string(data) != `{"bar":"bang","inner_thing":{"a_bool":false,"an_int":3,"foo":"bar"}}` {
		t.Fatal("Unexpected Marshal output:", string(data))
	}
}
<commit_msg>Add a test of serializing with a pointer to a nested struct<commit_after>package jsonmap

import "testing"

type InnerThing struct {
	Foo   string
	AnInt int
	ABool bool
}

type OuterThing struct {
	InnerThing InnerThing
}

type OuterPointerThing struct {
	InnerThing *InnerThing
}

type UnregisteredThing struct {
}

var InnerThingTypeMap = TypeMap{
	InnerThing{},
	[]MappedField{
		{
			StructFieldName: "Foo",
			JSONFieldName:   "foo",
			Validator:       String(1, 12),
			Optional:        true,
		},
		{
			StructFieldName: "AnInt",
			JSONFieldName:   "an_int",
			Validator:       Integer(0, 10),
			Optional:        true,
		},
		{
			StructFieldName: "ABool",
			JSONFieldName:   "a_bool",
			Validator:       Boolean(),
			Optional:        true,
		},
	},
}

var OuterThingTypeMap = TypeMap{
	OuterThing{},
	[]MappedField{
		{
			StructFieldName: "InnerThing",
			JSONFieldName:   "inner_thing",
			Contains:        InnerThingTypeMap,
		},
	},
}

var OuterPointerThingTypeMap = TypeMap{
	OuterPointerThing{},
	[]MappedField{
		{
			StructFieldName: "InnerThing",
			JSONFieldName:   "inner_thing",
			Contains:        InnerThingTypeMap,
		},
	},
}

var TestTypeMapper = NewTypeMapper(
	InnerThingTypeMap,
	OuterThingTypeMap,
	OuterPointerThingTypeMap,
)

func TestValidateInnerThing(t *testing.T) {
	v := &InnerThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"foo": "fooz", "an_int": 10, "a_bool": true}`), v)
	if err != nil {
		t.Fatal(err)
	}
	if v.Foo != "fooz" {
		t.Fatal("Field Foo does not have expected value 'fooz':", v.Foo)
	}
}

func TestValidateOuterThing(t *testing.T) {
	v := &OuterThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"inner_thing": {"foo": "fooz"}}`), v)
	if err != nil {
		t.Fatal(err)
	}
	if v.InnerThing.Foo != "fooz" {
		t.Fatal("Inner field Foo does not have expected value 'fooz':", v.InnerThing.Foo)
	}
}

func TestValidateUnregisteredThing(t *testing.T) {
	defer func() {
		if r := recover(); r == nil {
			t.Fatal("No panic")
		}
	}()
	v := &UnregisteredThing{}
	TestTypeMapper.Unmarshal([]byte(`{}`), v)
	t.Fatal("Unexpected success")
}

func TestValidateStringTypeMismatch(t *testing.T) {
	v := &InnerThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"foo": 12.0}`), v)
	if err == nil {
		t.Fatal("Unexpected success")
	}
	if err.Error() != "error validating field 'foo': not a string" {
		t.Fatal("Unexpected error message:", err.Error())
	}
}

func TestValidateStringTooShort(t *testing.T) {
	v := &InnerThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"foo": ""}`), v)
	if err == nil {
		t.Fatal("Unexpected success")
	}
	if err.Error() != "error validating field 'foo': too short, must be at least 1 characters" {
		t.Fatal("Unexpected error message:", err.Error())
	}
}

func TestValidateStringTooLong(t *testing.T) {
	v := &InnerThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"foo": "thisvalueistoolong"}`), v)
	if err == nil {
		t.Fatal("Unexpected success")
	}
	if err.Error() != "error validating field 'foo': too long, may not be more than 12 characters" {
		t.Fatal("Unexpected error message:", err.Error())
	}
}

func TestValidateBooleanTypeMismatch(t *testing.T) {
	v := &InnerThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"a_bool": 12.0}`), v)
	if err == nil {
		t.Fatal("Unexpected success")
	}
	if err.Error() != "error validating field 'a_bool': not a boolean" {
		t.Fatal("Unexpected error message:", err.Error())
	}
}

func TestValidateIntegerTypeMismatch(t *testing.T) {
	v := &InnerThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"an_int": false}`), v)
	if err == nil {
		t.Fatal("Unexpected success")
	}
	if err.Error() != "error validating field 'an_int': not an integer" {
		t.Fatal("Unexpected error message:", err.Error())
	}
}

func TestValidateIntegerNumericTypeMismatch(t *testing.T) {
	v := &InnerThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"an_int": 12.1}`), v)
	if err == nil {
		t.Fatal("Unexpected success")
	}
	if err.Error() != "error validating field 'an_int': not an integer" {
		t.Fatal("Unexpected error message:", err.Error())
	}
}

func TestValidateIntegerTooSmall(t *testing.T) {
	v := &InnerThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"an_int": -1}`), v)
	if err == nil {
		t.Fatal("Unexpected success")
	}
	if err.Error() != "error validating field 'an_int': too small, must be at least 0" {
		t.Fatal("Unexpected error message:", err.Error())
	}
}

func TestValidateIntegerTooLarge(t *testing.T) {
	v := &InnerThing{}
	err := TestTypeMapper.Unmarshal([]byte(`{"an_int": 2048}`), v)
	if err == nil {
		t.Fatal("Unexpected success")
	}
	if err.Error() != "error validating field 'an_int': too large, may not be larger than 10" {
		t.Fatal("Unexpected error message:", err.Error())
	}
}

func TestMarshalInnerThing(t *testing.T) {
	v := &InnerThing{
		Foo:   "bar",
		AnInt: 7,
		ABool: true,
	}
	data, err := TestTypeMapper.Marshal(v)
	if err != nil {
		t.Fatal(err)
	}
	if string(data) != `{"a_bool":true,"an_int":7,"foo":"bar"}` {
		t.Fatal("Unexpected Marshal output:", string(data))
	}
}

func TestMarshalOuterThing(t *testing.T) {
	v := &OuterThing{
		InnerThing: InnerThing{
			Foo:   "bar",
			AnInt: 3,
			ABool: false,
		},
	}
	data, err := TestTypeMapper.Marshal(v)
	if err != nil {
		t.Fatal(err)
	}
	if string(data) != `{"inner_thing":{"a_bool":false,"an_int":3,"foo":"bar"}}` {
		t.Fatal("Unexpected Marshal output:", string(data))
	}
}

func TestMarshalOuterPointerThing(t *testing.T) {
	v := &OuterPointerThing{
		InnerThing: &InnerThing{
			Foo:   "bar",
			AnInt: 3,
			ABool: false,
		},
	}
	data, err := TestTypeMapper.Marshal(v)
	if err != nil {
		t.Fatal(err)
	}
	if string(data) != `{"inner_thing":{"a_bool":false,"an_int":3,"foo":"bar"}}` {
		t.Fatal("Unexpected Marshal output:", string(data))
	}
}
<|endoftext|>


<commit_before>//
//   date  : 2016-05-13
//   author: xjdrew
//

package k1

import (
	"net"
	"sync"
	"time"

	"github.com/miekg/dns"
)

// hijacked domain
type DomainRecord struct {
	Hostname string // hostname
	Proxy    string // proxy

	IP      net.IP // nat ip
	RealIP  net.IP // real ip
	Hits    int
	Expires time.Time

	answer *dns.A // cache dns answer
}

func (record *DomainRecord) SetRealIP(msg *dns.Msg) {
	if record.RealIP != nil {
		return
	}

	var ip net.IP
	for _, item := range msg.Answer {
		switch answer := item.(type) {
		case *dns.A:
			ip = answer.A
			break
		}
	}
	record.RealIP = ip
	logger.Debugf("[dns] %s real ip: %s", record.Hostname, ip)
}

func (record *DomainRecord) Answer(request *dns.Msg) *dns.Msg {
	rsp := new(dns.Msg)
	rsp.SetReply(request)
	rsp.Answer = append(rsp.Answer, record.answer)
	return rsp
}

func (record *DomainRecord) Touch() {
	record.Hits++
	record.Expires = time.Now().Add(dnsDefaultTtl * time.Second)
}

type DnsTable struct {
	// dns ip pool
	ipPool *DnsIPPool

	// hijacked domain records
	records     map[string]*DomainRecord // domain -> record
	ip2Domain   map[string]string        // ip -> domain: map hijacked ip address to domain
	recordsLock sync.Mutex               // protect records and ip2Domain

	nonProxyDomains map[string]time.Time // non proxy domain
	npdLock         sync.Mutex           // protect non proxy domain
}

func (c *DnsTable) get(domain string) *DomainRecord {
	record := c.records[domain]
	if record != nil {
		record.Touch()
	}
	return record
}

func (c *DnsTable) GetByIP(ip net.IP) *DomainRecord {
	c.recordsLock.Lock()
	defer c.recordsLock.Unlock()
	if domain, ok := c.ip2Domain[ip.String()]; ok {
		return c.get(domain)
	}
	return nil
}

func (c *DnsTable) Contains(ip net.IP) bool {
	return c.ipPool.Contains(ip)
}

func (c *DnsTable) Get(domain string) *DomainRecord {
	c.recordsLock.Lock()
	defer c.recordsLock.Unlock()
	return c.get(domain)
}

// forge a IPv4 dns reply
func forgeIPv4Answer(domain string, ip net.IP) *dns.A {
	rr := new(dns.A)
	rr.Hdr = dns.RR_Header{Name: dns.Fqdn(domain), Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: dnsDefaultTtl}
	rr.A = ip.To4()
	return rr
}

func (c *DnsTable) Set(domain string, proxy string) *DomainRecord {
	c.recordsLock.Lock()
	defer c.recordsLock.Unlock()
	record := c.records[domain]
	if record != nil {
		return record
	}

	// alloc a ip
	ip := c.ipPool.Alloc(domain)
	if ip == nil {
		logger.Errorf("[dns] ip space is used up, domain:%s", domain)
		return nil
	}

	record = new(DomainRecord)
	record.IP = ip
	record.Hostname = domain
	record.Proxy = proxy
	record.answer = forgeIPv4Answer(domain, ip)

	record.Touch()

	c.records[domain] = record
	c.ip2Domain[ip.String()] = domain
	logger.Debugf("[dns] hijack %s -> %s", domain, ip.String())
	return record
}

func (c *DnsTable) IsNonProxyDomain(domain string) bool {
	c.npdLock.Lock()
	defer c.npdLock.Unlock()
	_, ok := c.nonProxyDomains[domain]
	return ok
}

func (c *DnsTable) SetNonProxyDomain(domain string, ttl uint32) {
	c.npdLock.Lock()
	defer c.npdLock.Unlock()
	c.nonProxyDomains[domain] = time.Now().Add(time.Duration(ttl) * time.Second)
	logger.Debugf("[dns] set non proxy domain: %s, ttl: %d", domain, ttl)
}

func (c *DnsTable) clearExpiredNonProxyDomain(now time.Time) {
	c.npdLock.Lock()
	defer c.npdLock.Unlock()
	for domain, expired := range c.nonProxyDomains {
		if expired.Before(now) {
			delete(c.nonProxyDomains, domain)
			logger.Debugf("[dns] release non proxy domain: %s", domain)
		}
	}
}

func (c *DnsTable) clearExpiredDomain(now time.Time) {
	c.recordsLock.Lock()
	defer c.recordsLock.Unlock()

	threshold := 1000
	if threshold > c.ipPool.Capacity()/10 {
		threshold = c.ipPool.Capacity() / 10
	}

	if len(c.records) <= threshold {
		return
	}

	for domain, record := range c.records {
		if !record.Expires.Before(now) {
			continue
		}
		delete(c.records, domain)
		delete(c.ip2Domain, record.IP.String())
		c.ipPool.Release(record.IP)
		logger.Debugf("[dns] release %s -> %s, hit: %d", domain, record.IP.String(), record.Hits)
	}
}

func (c *DnsTable) Serve() error {
	tick := time.Tick(60 * time.Second)
	for now := range tick {
		c.clearExpiredDomain(now)
		c.clearExpiredNonProxyDomain(now)
	}
	return nil
}

func NewDnsTable(ip net.IP, subnet *net.IPNet) *DnsTable {
	c := new(DnsTable)
	c.ipPool = NewDnsIPPool(ip, subnet)
	c.records = make(map[string]*DomainRecord)
	c.ip2Domain = make(map[string]string)
	c.nonProxyDomains = make(map[string]time.Time)
	return c
}
<commit_msg>fix recursion warning (#18)<commit_after>//
//   date  : 2016-05-13
//   author: xjdrew
//

package k1

import (
	"net"
	"sync"
	"time"

	"github.com/miekg/dns"
)

// hijacked domain
type DomainRecord struct {
	Hostname string // hostname
	Proxy    string // proxy

	IP      net.IP // nat ip
	RealIP  net.IP // real ip
	Hits    int
	Expires time.Time

	answer *dns.A // cache dns answer
}

func (record *DomainRecord) SetRealIP(msg *dns.Msg) {
	if record.RealIP != nil {
		return
	}

	var ip net.IP
	for _, item := range msg.Answer {
		switch answer := item.(type) {
		case *dns.A:
			ip = answer.A
			break
		}
	}
	record.RealIP = ip
	logger.Debugf("[dns] %s real ip: %s", record.Hostname, ip)
}

func (record *DomainRecord) Answer(request *dns.Msg) *dns.Msg {
	rsp := new(dns.Msg)
	rsp.SetReply(request)
	rsp.RecursionAvailable = true
	rsp.Answer = append(rsp.Answer, record.answer)
	return rsp
}

func (record *DomainRecord) Touch() {
	record.Hits++
	record.Expires = time.Now().Add(dnsDefaultTtl * time.Second)
}

type DnsTable struct {
	// dns ip pool
	ipPool *DnsIPPool

	// hijacked domain records
	records     map[string]*DomainRecord // domain -> record
	ip2Domain   map[string]string        // ip -> domain: map hijacked ip address to domain
	recordsLock sync.Mutex               // protect records and ip2Domain

	nonProxyDomains map[string]time.Time // non proxy domain
	npdLock         sync.Mutex           // protect non proxy domain
}

func (c *DnsTable) get(domain string) *DomainRecord {
	record := c.records[domain]
	if record != nil {
		record.Touch()
	}
	return record
}

func (c *DnsTable) GetByIP(ip net.IP) *DomainRecord {
	c.recordsLock.Lock()
	defer c.recordsLock.Unlock()
	if domain, ok := c.ip2Domain[ip.String()]; ok {
		return c.get(domain)
	}
	return nil
}

func (c *DnsTable) Contains(ip net.IP) bool {
	return c.ipPool.Contains(ip)
}

func (c *DnsTable) Get(domain string) *DomainRecord {
	c.recordsLock.Lock()
	defer c.recordsLock.Unlock()
	return c.get(domain)
}

// forge a IPv4 dns reply
func forgeIPv4Answer(domain string, ip net.IP) *dns.A {
	rr := new(dns.A)
	rr.Hdr = dns.RR_Header{Name: dns.Fqdn(domain), Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: dnsDefaultTtl}
	rr.A = ip.To4()
	return rr
}

func (c *DnsTable) Set(domain string, proxy string) *DomainRecord {
	c.recordsLock.Lock()
	defer c.recordsLock.Unlock()
	record := c.records[domain]
	if record != nil {
		return record
	}

	// alloc a ip
	ip := c.ipPool.Alloc(domain)
	if ip == nil {
		logger.Errorf("[dns] ip space is used up, domain:%s", domain)
		return nil
	}

	record = new(DomainRecord)
	record.IP = ip
	record.Hostname = domain
	record.Proxy = proxy
	record.answer = forgeIPv4Answer(domain, ip)

	record.Touch()

	c.records[domain] = record
	c.ip2Domain[ip.String()] = domain
	logger.Debugf("[dns] hijack %s -> %s", domain, ip.String())
	return record
}

func (c *DnsTable) IsNonProxyDomain(domain string) bool {
	c.npdLock.Lock()
	defer c.npdLock.Unlock()
	_, ok := c.nonProxyDomains[domain]
	return ok
}

func (c *DnsTable) SetNonProxyDomain(domain string, ttl uint32) {
	c.npdLock.Lock()
	defer c.npdLock.Unlock()
	c.nonProxyDomains[domain] = time.Now().Add(time.Duration(ttl) * time.Second)
	logger.Debugf("[dns] set non proxy domain: %s, ttl: %d", domain, ttl)
}

func (c *DnsTable) clearExpiredNonProxyDomain(now time.Time) {
	c.npdLock.Lock()
	defer c.npdLock.Unlock()
	for domain, expired := range c.nonProxyDomains {
		if expired.Before(now) {
			delete(c.nonProxyDomains, domain)
			logger.Debugf("[dns] release non proxy domain: %s", domain)
		}
	}
}

func (c *DnsTable) clearExpiredDomain(now time.Time) {
	c.recordsLock.Lock()
	defer c.recordsLock.Unlock()

	threshold := 1000
	if threshold > c.ipPool.Capacity()/10 {
		threshold = c.ipPool.Capacity() / 10
	}

	if len(c.records) <= threshold {
		return
	}

	for domain, record := range c.records {
		if !record.Expires.Before(now) {
			continue
		}
		delete(c.records, domain)
		delete(c.ip2Domain, record.IP.String())
		c.ipPool.Release(record.IP)
		logger.Debugf("[dns] release %s -> %s, hit: %d", domain, record.IP.String(), record.Hits)
	}
}

func (c *DnsTable) Serve() error {
	tick := time.Tick(60 * time.Second)
	for now := range tick {
		c.clearExpiredDomain(now)
		c.clearExpiredNonProxyDomain(now)
	}
	return nil
}

func NewDnsTable(ip net.IP, subnet *net.IPNet) *DnsTable {
	c := new(DnsTable)
	c.ipPool = NewDnsIPPool(ip, subnet)
	c.records = make(map[string]*DomainRecord)
	c.ip2Domain = make(map[string]string)
	c.nonProxyDomains = make(map[string]time.Time)
	return c
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"log"
	"net/http"
	"net/url"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/buptmiao/eago"
)

func ByrBBSCrawler() *eago.Crawler {
	userid := "inaadversity"
	passwd := "362622365"
	crawler := eago.NewCrawler("byrbbs")
	crawler.SetDepth(3).AddSeedUrls(
		//"https://bbs.byr.cn/section/ajax_list.json?uid=" + userid + "&root=sec-0",
		//"https://bbs.byr.cn/section/ajax_list.json?uid="+userid+"&root=sec-1",
		//"https://bbs.byr.cn/section/ajax_list.json?uid=" + userid + "&root=sec-2",
		//"https://bbs.byr.cn/section/ajax_list.json?uid="+userid+"&root=sec-3",
		//"https://bbs.byr.cn/section/ajax_list.json?uid="+userid+"&root=sec-4",
		//"https://bbs.byr.cn/section/ajax_list.json?uid="+userid+"&root=sec-5",
		//"https://bbs.byr.cn/section/ajax_list.json?uid="+userid+"&root=sec-6",
		//"https://bbs.byr.cn/section/ajax_list.json?uid="+userid+"&root=sec-7",
		//"https://bbs.byr.cn/section/ajax_list.json?uid="+userid+"&root=sec-8",
		//"https://bbs.byr.cn/section/ajax_list.json?uid="+userid+"&root=sec-9",
		"https://bbs.byr.cn/board/ACM_ICPC",
	)
	crawler.StartWith(func() []*eago.UrlRequest {
		params := url.Values{}
		params.Set("id", userid)
		params.Set("passwd", passwd)
		//params.Set("mode", "0")
		//params.Set("CookieDate", "0")
		header := http.Header{}
		header.Set("Host", "bbs.byr.cn")
		header.Set("X-Requested-With", "XMLHttpRequest")
		header.Set("Connection", "keep-alive")
		header.Set("content-type", "application/x-www-form-urlencoded; charset=UTF-8")
		req := eago.NewUrlRequest("https://bbs.byr.cn/user/ajax_login.json", "POST", "byrbbs", "login", 1)
		req.SetParams(params.Encode())
		req.SetHeader(header)
		return []*eago.UrlRequest{req}
	})
	crawler.AddParser("login", func(resp *eago.UrlResponse) (urls []*eago.UrlRequest) {
		log.Println("Url:", resp.Src.Url)
		unicode, _ := eago.GBKtoUTF(resp.Body)
		log.Println("Body:", unicode)
		log.Println("Content-Length:", len(resp.Body))
		log.Println("cookies***********", resp.Resp.Header)
		log.Println("cookies***********", resp.Resp.Header["Set-Cookie"])
		res := make([]*eago.UrlRequest, 0, 10)
		for _, v := range crawler.SeedUrls {
			req := eago.NewUrlRequest(v, "GET", "byrbbs", "step2", 1)
			header := http.Header{}
			cookies := ""
			for _, v := range resp.Resp.Header["Set-Cookie"] {
				cookies += strings.Split(v, ";")[0] + ";"
			}
			cookies = strings.TrimRight(cookies, ";")
			log.Println("cookies***********", cookies)
			header.Set("Cookie", cookies)
			header.Set("X-Requested-With", "XMLHttpRequest")

			req.SetDepth(1).SetHeader(header)
			res = append(res, req)
		}
		return res
	})
	crawler.AddParser("step1", func(resp *eago.UrlResponse) (urls []*eago.UrlRequest) {
		log.Println("Url:", resp.Src.Url)
		unicode, _ := eago.GBKtoUTF(resp.Body)
		log.Println("Body:", unicode)
		log.Println("Content-Length:", len(resp.Body))
		type board struct {
			Tag string `json:"t, omitempty"`
			Id  string `json:"id, omitempty"`
		}
		boards := make([]board, 0)
		err := json.Unmarshal([]byte(unicode), &boards)
		if err != nil {
			log.Println(err)
			return nil
		}
		res := make([]*eago.UrlRequest, 0, 10)
		for _, v := range boards {
			log.Println(v.Tag)
			doc, err := goquery.NewDocumentFromReader(strings.NewReader(v.Tag))
			if err != nil {
				continue
			}
			suburl := doc.Find("a").AttrOr("href", "")
			title := doc.Find("a").AttrOr("title", "")
			if v.Id != "" {
				url := "https://bbs.byr.cn/section/ajax_list.json?uid=" + userid + "&root=" + v.Id
				log.Println("url:", url)
				req := eago.NewUrlRequest(url, "GET", "byrbbs", "step1", 1)
				header := http.Header{}
				header.Set("X-Requested-With", "XMLHttpRequest")
				req.SetDepth(2).SetHeader(header)
				res = append(res, req)
			} else {
				url := "https://bbs.byr.cn" + suburl + "?_uid=" + userid
				req := eago.NewUrlRequest(url, "GET", "byrbbs", "step2", 1)
				header := http.Header{}
				header.Set("X-Requested-With", "XMLHttpRequest")
				req.SetDepth(2).SetHeader(header)
				res = append(res, req)
			}
			log.Println(suburl, title)

		}
		return res
	})
	crawler.AddParser("step2", func(resp *eago.UrlResponse) (urls []*eago.UrlRequest) {
		log.Println("Url:", resp.Src.Url)
		unicode, _ := eago.GBKtoUTF(resp.Body)
		log.Println("Body:", unicode)
		log.Println("Content-Length:", len(resp.Body))
		return nil
	})
	store := eago.NewDefaultStore(eago.GetRedisClient())
	crawler.SetStorage(store)
	return crawler
}

func main() {
	eago.LoadConfig()
	node := eago.GetNodeInstance()
	cluster := eago.GetClusterInstance()

	bbs := ByrBBSCrawler()
	node.AddCrawler(bbs)

	eago.NewRpcServer().Start()
	// Discover will Block the execution, until a master node
	// is found, or become master itself.
	cluster.Discover()
	// start the Http Server
	eago.NewHttpServer(node).Serve()
}
<commit_msg>Update byrbbs.go<commit_after>package main

import (
	"encoding/json"
	"log"
	"net/http"
	"net/url"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/buptmiao/eago"
)

func ByrBBSCrawler() *eago.Crawler {
	userid := ""
	passwd := ""
	crawler := eago.NewCrawler("byrbbs")
	crawler.SetDepth(3).AddSeedUrls(
		//"https://bbs.byr.cn/section/ajax_list.json?uid=" + userid + "&root=sec-0",
		//"https://bbs.byr.cn/section/ajax_list.json?uid="+userid+"&root=sec-1",
		//"https://bbs.byr.cn/section/ajax_list.json?uid=" + userid + "&root=sec-2",
		//"https://bbs.byr.cn/section/ajax_list.json?uid="+userid+"&root=sec-3",
		//"https://bbs.byr.cn/section/ajax_list.json?uid="+userid+"&root=sec-4",
		//"https://bbs.byr.cn/section/ajax_list.json?uid="+userid+"&root=sec-5",
		//"https://bbs.byr.cn/section/ajax_list.json?uid="+userid+"&root=sec-6",
		//"https://bbs.byr.cn/section/ajax_list.json?uid="+userid+"&root=sec-7",
		//"https://bbs.byr.cn/section/ajax_list.json?uid="+userid+"&root=sec-8",
		//"https://bbs.byr.cn/section/ajax_list.json?uid="+userid+"&root=sec-9",
		"https://bbs.byr.cn/board/ACM_ICPC",
	)
	crawler.StartWith(func() []*eago.UrlRequest {
		params := url.Values{}
		params.Set("id", userid)
		params.Set("passwd", passwd)
		//params.Set("mode", "0")
		//params.Set("CookieDate", "0")
		header := http.Header{}
		header.Set("Host", "bbs.byr.cn")
		header.Set("X-Requested-With", "XMLHttpRequest")
		header.Set("Connection", "keep-alive")
		header.Set("content-type", "application/x-www-form-urlencoded; charset=UTF-8")
		req := eago.NewUrlRequest("https://bbs.byr.cn/user/ajax_login.json", "POST", "byrbbs", "login", 1)
		req.SetParams(params.Encode())
		req.SetHeader(header)
		return []*eago.UrlRequest{req}
	})
	crawler.AddParser("login", func(resp *eago.UrlResponse) (urls []*eago.UrlRequest) {
		log.Println("Url:", resp.Src.Url)
		unicode, _ := eago.GBKtoUTF(resp.Body)
		log.Println("Body:", unicode)
		log.Println("Content-Length:", len(resp.Body))
		log.Println("cookies***********", resp.Resp.Header)
		log.Println("cookies***********", resp.Resp.Header["Set-Cookie"])
		res := make([]*eago.UrlRequest, 0, 10)
		for _, v := range crawler.SeedUrls {
			req := eago.NewUrlRequest(v, "GET", "byrbbs", "step2", 1)
			header := http.Header{}
			cookies := ""
			for _, v := range resp.Resp.Header["Set-Cookie"] {
				cookies += strings.Split(v, ";")[0] + ";"
			}
			cookies = strings.TrimRight(cookies, ";")
			log.Println("cookies***********", cookies)
			header.Set("Cookie", cookies)
			header.Set("X-Requested-With", "XMLHttpRequest")

			req.SetDepth(1).SetHeader(header)
			res = append(res, req)
		}
		return res
	})
	crawler.AddParser("step1", func(resp *eago.UrlResponse) (urls []*eago.UrlRequest) {
		log.Println("Url:", resp.Src.Url)
		unicode, _ := eago.GBKtoUTF(resp.Body)
		log.Println("Body:", unicode)
		log.Println("Content-Length:", len(resp.Body))
		type board struct {
			Tag string `json:"t, omitempty"`
			Id  string `json:"id, omitempty"`
		}
		boards := make([]board, 0)
		err := json.Unmarshal([]byte(unicode), &boards)
		if err != nil {
			log.Println(err)
			return nil
		}
		res := make([]*eago.UrlRequest, 0, 10)
		for _, v := range boards {
			log.Println(v.Tag)
			doc, err := goquery.NewDocumentFromReader(strings.NewReader(v.Tag))
			if err != nil {
				continue
			}
			suburl := doc.Find("a").AttrOr("href", "")
			title := doc.Find("a").AttrOr("title", "")
			if v.Id != "" {
				url := "https://bbs.byr.cn/section/ajax_list.json?uid=" + userid + "&root=" + v.Id
				log.Println("url:", url)
				req := eago.NewUrlRequest(url, "GET", "byrbbs", "step1", 1)
				header := http.Header{}
				header.Set("X-Requested-With", "XMLHttpRequest")
				req.SetDepth(2).SetHeader(header)
				res = append(res, req)
			} else {
				url := "https://bbs.byr.cn" + suburl + "?_uid=" + userid
				req := eago.NewUrlRequest(url, "GET", "byrbbs", "step2", 1)
				header := http.Header{}
				header.Set("X-Requested-With", "XMLHttpRequest")
				req.SetDepth(2).SetHeader(header)
				res = append(res, req)
			}
			log.Println(suburl, title)

		}
		return res
	})
	crawler.AddParser("step2", func(resp *eago.UrlResponse) (urls []*eago.UrlRequest) {
		log.Println("Url:", resp.Src.Url)
		unicode, _ := eago.GBKtoUTF(resp.Body)
		log.Println("Body:", unicode)
		log.Println("Content-Length:", len(resp.Body))
		return nil
	})
	store := eago.NewDefaultStore(eago.GetRedisClient())
	crawler.SetStorage(store)
	return crawler
}

func main() {
	eago.LoadConfig()
	node := eago.GetNodeInstance()
	cluster := eago.GetClusterInstance()

	bbs := ByrBBSCrawler()
	node.AddCrawler(bbs)

	eago.NewRpcServer().Start()
	// Discover will Block the execution, until a master node
	// is found, or become master itself.
	cluster.Discover()
	// start the Http Server
	eago.NewHttpServer(node).Serve()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"github.com/apache/beam/sdks/go/pkg/beam"
	"github.com/apache/beam/sdks/go/pkg/beam/io/textio"
	"github.com/apache/beam/sdks/go/pkg/beam/runners/beamexec"
	"github.com/apache/beam/sdks/go/pkg/beam/transforms/debug"
	"log"
	"os"
	"strings"
)

var (
	input = flag.String("input", os.ExpandEnv("$GOPATH/src/github.com/apache/beam/sdks/go/data/haiku/old_pond.txt"), "Files to read.")
	count = flag.Int("count", 2, "Number of trees")
	depth = flag.Int("depth", 3, "Depth of each tree")
)

// Forest builds a forest of processing nodes with flatten "branches".
func Forest(p *beam.Pipeline) error {
	for i := 0; i < *count; i++ {
		t, err := tree(p, *depth)
		if err != nil {
			return err
		}
		if err := debug.Print0(p, t); err != nil {
			return err
		}
	}
	return nil
}

func tree(p *beam.Pipeline, depth int) (beam.PCollection, error) {
	if depth <= 0 {
		return leaf(p)
	}
	a, err := tree(p, depth-1)
	if err != nil {
		return beam.PCollection{}, err
	}
	b, err := tree(p, depth-1)
	if err != nil {
		return beam.PCollection{}, err
	}
	c, err := tree(p, depth-2)
	if err != nil {
		return beam.PCollection{}, err
	}
	return beam.Flatten(p, a, b, c)
}

func leaf(p *beam.Pipeline) (beam.PCollection, error) {
	lines, err := textio.Immediate(p, *input)
	if err != nil {
		return beam.PCollection{}, err
	}
	return beam.ParDo(p, strings.ToUpper, lines)
}

func main() {
	flag.Parse()
	ctx := context.Background()
	beamexec.Init(ctx)

	log.Print("Running wordcap")

	p := beam.NewPipeline()
	if err := Forest(p); err != nil {
		log.Fatalf("Failed to construct job: %v", err)
	}
	if err := beamexec.Run(ctx, p); err != nil {
		log.Fatalf("Failed to execute job: %v", err)
	}
}
<commit_msg>Make forest example print less<commit_after>package main

import (
	"context"
	"flag"
	"github.com/apache/beam/sdks/go/pkg/beam"
	"github.com/apache/beam/sdks/go/pkg/beam/io/textio"
	"github.com/apache/beam/sdks/go/pkg/beam/runners/beamexec"
	"github.com/apache/beam/sdks/go/pkg/beam/transforms/count"
	"github.com/apache/beam/sdks/go/pkg/beam/transforms/debug"
	"log"
	"os"
	"strings"
)

var (
	input = flag.String("input", os.ExpandEnv("$GOPATH/src/github.com/apache/beam/sdks/go/data/haiku/old_pond.txt"), "Files to read.")
	n     = flag.Int("count", 2, "Number of trees")
	depth = flag.Int("depth", 3, "Depth of each tree")
)

// Forest builds a forest of processing nodes with flatten "branches".
func Forest(p *beam.Pipeline) error {
	for i := 0; i < *n; i++ {
		t, err := tree(p, *depth)
		if err != nil {
			return err
		}
		deduped, err := count.Dedup(p, t)
		if err != nil {
			return err
		}
		if err := debug.Print0(p, deduped); err != nil {
			return err
		}
	}
	return nil
}

func tree(p *beam.Pipeline, depth int) (beam.PCollection, error) {
	if depth <= 0 {
		return leaf(p)
	}
	a, err := tree(p, depth-1)
	if err != nil {
		return beam.PCollection{}, err
	}
	b, err := tree(p, depth-1)
	if err != nil {
		return beam.PCollection{}, err
	}
	c, err := tree(p, depth-2)
	if err != nil {
		return beam.PCollection{}, err
	}
	return beam.Flatten(p, a, b, c)
}

func leaf(p *beam.Pipeline) (beam.PCollection, error) {
	lines, err := textio.Immediate(p, *input)
	if err != nil {
		return beam.PCollection{}, err
	}
	return beam.ParDo(p, strings.ToUpper, lines)
}

func main() {
	flag.Parse()
	ctx := context.Background()
	beamexec.Init(ctx)

	log.Print("Running forest")

	p := beam.NewPipeline()
	if err := Forest(p); err != nil {
		log.Fatalf("Failed to construct job: %v", err)
	}
	if err := beamexec.Run(ctx, p); err != nil {
		log.Fatalf("Failed to execute job: %v", err)
	}
}
<|endoftext|>


<commit_before>package neo4j

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"koding/tools/config"
	"labix.org/v2/mgo/bson"
	"log"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"
)

var (
	BASE_URL         = config.Current.Neo4j.Write + ":" + strconv.Itoa(config.Current.Neo4j.Port)
	INDEX_NODE_PATH  = "/db/data/index/node/koding"
	UNIQUE_NODE_PATH = "/db/data/index/node/koding?unique"
	INDEX_PATH       = "/db/data/index/node"
	NODE_URL         = "/db/data/node"
	MAX_RETRIES      = 5
	TIMEOUT          = 3
)

type Relationship struct {
	Id         bson.ObjectId `bson:"_id,omitempty"`
	TargetId   bson.ObjectId `bson:"targetId,omitempty"`
	TargetName string        `bson:"targetName"`
	SourceId   bson.ObjectId `bson:"sourceId,omitempty"`
	SourceName string        `bson:"sourceName"`
	As         string
	Data       bson.Binary
}

// Setup the dial timeout
func dialTimeout(timeout time.Duration) func(network, addr string) (c net.Conn, err error) {
	return func(netw, addr string) (net.Conn, error) {
		conn, err := net.DialTimeout(netw, addr, timeout)
		if err != nil {
			return nil, err
		}
		return conn, nil
	}
}

// Gets URL and string data to be sent and makes POST request
// reads response body and returns as string
func sendRequest(requestType, url, data string, attempt int) string {

	// Set the timeout
	timeOut := time.Duration(TIMEOUT) * time.Second

	transport := http.Transport{
		Dial: dialTimeout(timeOut),
	}

	client := http.Client{
		Transport: &transport,
	}

	//convert string into bytestream
	dataByte := strings.NewReader(data)
	req, err := http.NewRequest(requestType, url, dataByte)

	// read response body
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json")
	log.Print(fmt.Sprintf("Attempt [%v/%v] to req %v", MAX_RETRIES, attempt, url))
	res, err := client.Do(req)
	if err != nil && attempt <= MAX_RETRIES {
		log.Print(err)
		attempt++
		sendRequest(requestType, url, data, attempt)
	}
	if err != nil && attempt > MAX_RETRIES {
		panic(fmt.Sprintf("req to %v timed out after %v retries", url, attempt))
	}

	body, _ := ioutil.ReadAll(res.Body)

	defer res.Body.Close()

	return string(body)

}

// connect source and target with relation property
// response will be object
func CreateRelationship(relation, source, target string) map[string]interface{} {

	relationshipData := fmt.Sprintf(`{"to" : "%s", "type" : "%s" }`, target, relation)
	relRes := sendRequest("POST", fmt.Sprintf("%s", source), relationshipData, 1)

	relNode, err := jsonDecode(relRes)
	if err != nil {
		log.Println("Problem with relation response", relRes)
	}

	return relNode
}

// connect source and target with relation property
// response will be object
func CreateRelationshipWithData(relation, source, target, data string) map[string]interface{} {

	relationshipData := fmt.Sprintf(`{"to" : "%s", "type" : "%s", "data" : %s }`, target, relation, data)
	relRes := sendRequest("POST", fmt.Sprintf("%s", source), relationshipData, 1)

	relNode, err := jsonDecode(relRes)
	if err != nil {
		log.Println("Problem with relation response", relRes)
	}

	return relNode
}

// creates a unique node with given id and node name
// response will be Object
func CreateUniqueNode(id string, name string) map[string]interface{} {

	url := BASE_URL + UNIQUE_NODE_PATH

	postData := generatePostJsonData(id, name)

	response := sendRequest("POST", url, postData, 1)

	node, err := jsonDecode(response)
	if err != nil {
		log.Println("Problem with unique node creation response", response)
	}

	return node
}

// deletes a relation between two node using relationship info
func DeleteRelationship(sourceId, targetId, relationship string) bool {

	//get source node information
	sourceInfo := GetNode(sourceId)

	//get target node information
	targetInfo := GetNode(targetId)

	if len(sourceInfo) < 1 || len(targetInfo) < 1 {
		return false
	}

	if _, ok := sourceInfo[0]["self"]; !ok {
		return false
	}

	if _, ok := targetInfo[0]["self"]; !ok {
		return false
	}

	// create  url to get relationship information of source node
	relationshipsURL := fmt.Sprintf("%s", sourceInfo[0]["self"]) + "/relationships/all/" + relationship

	//this request returns objects in an array
	response := sendRequest("GET", relationshipsURL, "", 1)
	//so use json array decoder
	relationships, err := jsonArrayDecode(response)
	if err != nil {
		log.Println("Problem with unique node creation response", response)
		return false
	}

	if len(relationships) < 1 {
		return false
	}

	if _, ok := relationships[0]["self"]; !ok {
		return false
	}

	foundNode := false

	for _, relation := range relationships {
		if relation["end"] == targetInfo[0]["self"] {
			toBeDeletedRelationURL := fmt.Sprintf("%s", relation["self"])
			deletionResponse := sendRequest("DELETE", toBeDeletedRelationURL, "", 1)
			log.Println(deletionResponse)
			foundNode = true

			break
		}
	}

	if !foundNode {
		log.Println("not found!", relationships[0]["self"])
	}

	return true
}

// gets node from neo4j with given unique node id
//response will be object
func GetNode(id string) []map[string]interface{} {

	url := BASE_URL + INDEX_NODE_PATH + "/id/" + id

	response := sendRequest("GET", url, "", 1)

	nodeData, err := jsonArrayDecode(response)
	if err != nil {
		log.Println("Problem with response", response)
	}

	return nodeData
}

// updates node with given data
// response will be object
func UpdateNode(id, propertiesJSON string) map[string]interface{} {

	node := GetNode(id)

	if len(node) < 1 {
		return nil
	}

	//if self is not there!
	if _, ok := node[0]["self"]; !ok {
		return nil
	}

	// create  url to get relationship information of source node
	propertiesURL := fmt.Sprintf("%s", node[0]["self"]) + "/properties"

	response := sendRequest("PUT", propertiesURL, propertiesJSON, 1)
	if response != "" {
		log.Println(response)
		res, err := jsonDecode(response)
		if err != nil {
			log.Println("Problem with response", err, res)
		}
	}

	return make(map[string]interface{})
}

func DeleteNode(id string) bool {

	node := GetNode(id)

	if len(node) < 1 {
		return false
	}

	//if self is not there!
	if _, ok := node[0]["self"]; !ok {
		return false
	}

	nodeURL := fmt.Sprintf("%s", node[0]["self"])

	relationshipsURL := nodeURL + "/relationships/all"

	response := sendRequest("GET", relationshipsURL, "", 1)

	relations, err := jsonArrayDecode(response)
	if err != nil {
		log.Println("Problem with response", response)
		return false
	}

	for _, relation := range relations {
		if _, ok := relation["self"]; ok {
			relationshipURL := fmt.Sprintf("%s", relation["self"])
			sendRequest("DELETE", relationshipURL, "", 1)
		}
	}

	sendRequest("DELETE", nodeURL, "", 1)

	return true
}

// creates a unique tree head node to hold all nodes
// it is called once during runtime while initializing
func CreateUniqueIndex(name string) {
	//create unique index
	url := BASE_URL + INDEX_PATH

	bd := sendRequest("POST", url, `{"name":"`+name+`"}`, 1)

	log.Println("Created unique index for data", bd)
}

// This is a custom json string generator as http request body to neo4j
func generatePostJsonData(id, name string) string {
	return fmt.Sprintf(`{ "key" : "id", "value" : "%s", "properties" : { "id" : "%s", "name" : "%s" } }`, id, id, name)
}

//here, mapping of decoded json
func jsonArrayDecode(data string) ([]map[string]interface{}, error) {
	var source []map[string]interface{}

	err := json.Unmarshal([]byte(data), &source)
	if err != nil {
		log.Println("Marshalling error:", err)
		return nil, err
	}

	return source, nil
}

//here, mapping of decoded json
func jsonDecode(data string) (map[string]interface{}, error) {
	var source map[string]interface{}

	err := json.Unmarshal([]byte(data), &source)
	if err != nil {
		log.Println("Marshalling error:", err)
		return nil, err
	}

	return source, nil
}

var NotAllowedNames = []string{
	"CStatusActivity",
	"CFolloweeBucketActivity",
	"CFollowerBucketActivity",
	"CCodeSnipActivity",
	"CDiscussionActivity",
	"CReplieeBucketActivity",
	"CReplierBucketActivity",
	"CBlogPostActivity",
	"CNewMemberBucketActivity",
	"CTutorialActivity",
	"CLikeeBucketActivity",
	"CLikerBucketActivity",
	"CInstalleeBucketActivity",
	"CInstallerBucketActivity",
	"CActivity",
	"CRunnableActivity",
	"JAppStorage",
	"JFeed",
}
<commit_msg>Add deadline to allow timeout if the connection hangs<commit_after>package neo4j

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"koding/tools/config"
	"labix.org/v2/mgo/bson"
	"log"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"
)

var (
	BASE_URL         = config.Current.Neo4j.Write + ":" + strconv.Itoa(config.Current.Neo4j.Port)
	INDEX_NODE_PATH  = "/db/data/index/node/koding"
	UNIQUE_NODE_PATH = "/db/data/index/node/koding?unique"
	INDEX_PATH       = "/db/data/index/node"
	NODE_URL         = "/db/data/node"
	MAX_RETRIES      = 5
	TIMEOUT          = 3
	DEADLINE         = 30
)

type Relationship struct {
	Id         bson.ObjectId `bson:"_id,omitempty"`
	TargetId   bson.ObjectId `bson:"targetId,omitempty"`
	TargetName string        `bson:"targetName"`
	SourceId   bson.ObjectId `bson:"sourceId,omitempty"`
	SourceName string        `bson:"sourceName"`
	As         string
	Data       bson.Binary
}

// Setup the dial timeout
func dialTimeout(timeout time.Duration, deadline time.Duration) func(network, addr string) (c net.Conn, err error) {
	return func(netw, addr string) (net.Conn, error) {
		conn, err := net.DialTimeout(netw, addr, timeout)
		if err != nil {
			return nil, err
		}
		conn.SetDeadline(time.Now().Add(deadline))
		return conn, nil
	}
}

// Gets URL and string data to be sent and makes POST request
// reads response body and returns as string
func sendRequest(requestType, url, data string, attempt int) string {

	// Set the timeout & deadline
	timeOut := time.Duration(TIMEOUT) * time.Second
    deadLine := time.Duration(DEADLINE) * time.Second

	transport := http.Transport{
		Dial: dialTimeout(timeOut, deadLine),
	}

	client := http.Client{
		Transport: &transport,
	}

	//convert string into bytestream
	dataByte := strings.NewReader(data)
	req, err := http.NewRequest(requestType, url, dataByte)

	// read response body
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json")
	log.Print(fmt.Sprintf("Attempt [%v/%v] to req %v", MAX_RETRIES, attempt, url))
	res, err := client.Do(req)
	if err != nil && attempt <= MAX_RETRIES {
		log.Print(err)
		attempt++
		sendRequest(requestType, url, data, attempt)
	}
	if err != nil && attempt > MAX_RETRIES {
		panic(fmt.Sprintf("req to %v timed out after %v retries", url, attempt))
	}

	body, _ := ioutil.ReadAll(res.Body)

	defer res.Body.Close()

	return string(body)

}

// connect source and target with relation property
// response will be object
func CreateRelationship(relation, source, target string) map[string]interface{} {

	relationshipData := fmt.Sprintf(`{"to" : "%s", "type" : "%s" }`, target, relation)
	relRes := sendRequest("POST", fmt.Sprintf("%s", source), relationshipData, 1)

	relNode, err := jsonDecode(relRes)
	if err != nil {
		log.Println("Problem with relation response", relRes)
	}

	return relNode
}

// connect source and target with relation property
// response will be object
func CreateRelationshipWithData(relation, source, target, data string) map[string]interface{} {

	relationshipData := fmt.Sprintf(`{"to" : "%s", "type" : "%s", "data" : %s }`, target, relation, data)
	relRes := sendRequest("POST", fmt.Sprintf("%s", source), relationshipData, 1)

	relNode, err := jsonDecode(relRes)
	if err != nil {
		log.Println("Problem with relation response", relRes)
	}

	return relNode
}

// creates a unique node with given id and node name
// response will be Object
func CreateUniqueNode(id string, name string) map[string]interface{} {

	url := BASE_URL + UNIQUE_NODE_PATH

	postData := generatePostJsonData(id, name)

	response := sendRequest("POST", url, postData, 1)

	node, err := jsonDecode(response)
	if err != nil {
		log.Println("Problem with unique node creation response", response)
	}

	return node
}

// deletes a relation between two node using relationship info
func DeleteRelationship(sourceId, targetId, relationship string) bool {

	//get source node information
	sourceInfo := GetNode(sourceId)

	//get target node information
	targetInfo := GetNode(targetId)

	if len(sourceInfo) < 1 || len(targetInfo) < 1 {
		return false
	}

	if _, ok := sourceInfo[0]["self"]; !ok {
		return false
	}

	if _, ok := targetInfo[0]["self"]; !ok {
		return false
	}

	// create  url to get relationship information of source node
	relationshipsURL := fmt.Sprintf("%s", sourceInfo[0]["self"]) + "/relationships/all/" + relationship

	//this request returns objects in an array
	response := sendRequest("GET", relationshipsURL, "", 1)
	//so use json array decoder
	relationships, err := jsonArrayDecode(response)
	if err != nil {
		log.Println("Problem with unique node creation response", response)
		return false
	}

	if len(relationships) < 1 {
		return false
	}

	if _, ok := relationships[0]["self"]; !ok {
		return false
	}

	foundNode := false

	for _, relation := range relationships {
		if relation["end"] == targetInfo[0]["self"] {
			toBeDeletedRelationURL := fmt.Sprintf("%s", relation["self"])
			deletionResponse := sendRequest("DELETE", toBeDeletedRelationURL, "", 1)
			log.Println(deletionResponse)
			foundNode = true

			break
		}
	}

	if !foundNode {
		log.Println("not found!", relationships[0]["self"])
	}

	return true
}

// gets node from neo4j with given unique node id
//response will be object
func GetNode(id string) []map[string]interface{} {

	url := BASE_URL + INDEX_NODE_PATH + "/id/" + id

	response := sendRequest("GET", url, "", 1)

	nodeData, err := jsonArrayDecode(response)
	if err != nil {
		log.Println("Problem with response", response)
	}

	return nodeData
}

// updates node with given data
// response will be object
func UpdateNode(id, propertiesJSON string) map[string]interface{} {

	node := GetNode(id)

	if len(node) < 1 {
		return nil
	}

	//if self is not there!
	if _, ok := node[0]["self"]; !ok {
		return nil
	}

	// create  url to get relationship information of source node
	propertiesURL := fmt.Sprintf("%s", node[0]["self"]) + "/properties"

	response := sendRequest("PUT", propertiesURL, propertiesJSON, 1)
	if response != "" {
		log.Println(response)
		res, err := jsonDecode(response)
		if err != nil {
			log.Println("Problem with response", err, res)
		}
	}

	return make(map[string]interface{})
}

func DeleteNode(id string) bool {

	node := GetNode(id)

	if len(node) < 1 {
		return false
	}

	//if self is not there!
	if _, ok := node[0]["self"]; !ok {
		return false
	}

	nodeURL := fmt.Sprintf("%s", node[0]["self"])

	relationshipsURL := nodeURL + "/relationships/all"

	response := sendRequest("GET", relationshipsURL, "", 1)

	relations, err := jsonArrayDecode(response)
	if err != nil {
		log.Println("Problem with response", response)
		return false
	}

	for _, relation := range relations {
		if _, ok := relation["self"]; ok {
			relationshipURL := fmt.Sprintf("%s", relation["self"])
			sendRequest("DELETE", relationshipURL, "", 1)
		}
	}

	sendRequest("DELETE", nodeURL, "", 1)

	return true
}

// creates a unique tree head node to hold all nodes
// it is called once during runtime while initializing
func CreateUniqueIndex(name string) {
	//create unique index
	url := BASE_URL + INDEX_PATH

	bd := sendRequest("POST", url, `{"name":"`+name+`"}`, 1)

	log.Println("Created unique index for data", bd)
}

// This is a custom json string generator as http request body to neo4j
func generatePostJsonData(id, name string) string {
	return fmt.Sprintf(`{ "key" : "id", "value" : "%s", "properties" : { "id" : "%s", "name" : "%s" } }`, id, id, name)
}

//here, mapping of decoded json
func jsonArrayDecode(data string) ([]map[string]interface{}, error) {
	var source []map[string]interface{}

	err := json.Unmarshal([]byte(data), &source)
	if err != nil {
		log.Println("Marshalling error:", err)
		return nil, err
	}

	return source, nil
}

//here, mapping of decoded json
func jsonDecode(data string) (map[string]interface{}, error) {
	var source map[string]interface{}

	err := json.Unmarshal([]byte(data), &source)
	if err != nil {
		log.Println("Marshalling error:", err)
		return nil, err
	}

	return source, nil
}

var NotAllowedNames = []string{
	"CStatusActivity",
	"CFolloweeBucketActivity",
	"CFollowerBucketActivity",
	"CCodeSnipActivity",
	"CDiscussionActivity",
	"CReplieeBucketActivity",
	"CReplierBucketActivity",
	"CBlogPostActivity",
	"CNewMemberBucketActivity",
	"CTutorialActivity",
	"CLikeeBucketActivity",
	"CLikerBucketActivity",
	"CInstalleeBucketActivity",
	"CInstallerBucketActivity",
	"CActivity",
	"CRunnableActivity",
	"JAppStorage",
	"JFeed",
}
<|endoftext|>


<commit_before>package google

import (
	"fmt"
	"log"
	"regexp"
	"strings"

	"github.com/hashicorp/terraform/helper/schema"
	"google.golang.org/api/compute/v1"
)

var instancesSelfLinkPattern = regexp.MustCompile(fmt.Sprintf(zonalLinkBasePattern, "instances"))

func resourceComputeTargetPool() *schema.Resource {
	return &schema.Resource{
		Create: resourceComputeTargetPoolCreate,
		Read:   resourceComputeTargetPoolRead,
		Delete: resourceComputeTargetPoolDelete,
		Update: resourceComputeTargetPoolUpdate,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"backup_pool": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: false,
			},

			"description": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},

			"failover_ratio": {
				Type:     schema.TypeFloat,
				Optional: true,
				ForceNew: true,
			},

			"health_checks": {
				Type:     schema.TypeList,
				Optional: true,
				ForceNew: false,
				MaxItems: 1,
				Elem: &schema.Schema{
					Type:             schema.TypeString,
					DiffSuppressFunc: compareSelfLinkOrResourceName,
				},
			},

			"instances": {
				Type:     schema.TypeList,
				Optional: true,
				Computed: true,
				ForceNew: false,
				Elem:     &schema.Schema{Type: schema.TypeString},
				DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool {
					// instances are stored in state as "zone/name"
					oldParts := strings.Split(old, "/")

					// instances can also be specified in the config as a URL
					if parts := instancesSelfLinkPattern.FindStringSubmatch(new); len(oldParts) == 2 && len(parts) == 4 {
						// parts[0] = full match
						// parts[1] = project
						// parts[2] = zone
						// parts[3] = instance name

						oZone, oName := oldParts[0], oldParts[1]
						nZone, nName := parts[2], parts[3]
						if oZone == nZone && oName == nName {
							return true
						}
					}
					return false
				},
			},

			"project": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Computed: true,
			},

			"region": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Computed: true,
			},

			"self_link": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"session_affinity": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Default:  "NONE",
			},
		},
	}
}

// Healthchecks need to exist before being referred to from the target pool.
func convertHealthChecks(healthChecks []interface{}, d *schema.ResourceData, config *Config) ([]string, error) {
	if healthChecks == nil || len(healthChecks) == 0 {
		return []string{}, nil
	}

	hc, err := ParseHttpHealthCheckFieldValue(healthChecks[0].(string), d, config)
	if err != nil {
		return nil, err
	}

	return []string{hc.RelativeLink()}, nil
}

// Instances do not need to exist yet, so we simply generate URLs.
// Instances can be full URLS or zone/name
func convertInstancesToUrls(config *Config, project string, names []string) ([]string, error) {
	urls := make([]string, len(names))
	for i, name := range names {
		if strings.HasPrefix(name, "https://www.googleapis.com/compute/v1/") {
			urls[i] = name
		} else {
			splitName := strings.Split(name, "/")
			if len(splitName) != 2 {
				return nil, fmt.Errorf("Invalid instance name, require URL or zone/name: %s", name)
			} else {
				urls[i] = fmt.Sprintf(
					"https://www.googleapis.com/compute/v1/projects/%s/zones/%s/instances/%s",
					project, splitName[0], splitName[1])
			}
		}
	}
	return urls, nil
}

func resourceComputeTargetPoolCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	region, err := getRegion(d, config)
	if err != nil {
		return err
	}

	project, err := getProject(d, config)
	if err != nil {
		return err
	}

	hchkUrls, err := convertHealthChecks(d.Get("health_checks").([]interface{}), d, config)
	if err != nil {
		return err
	}

	instanceUrls, err := convertInstancesToUrls(
		config, project, convertStringArr(d.Get("instances").([]interface{})))
	if err != nil {
		return err
	}

	// Build the parameter
	tpool := &compute.TargetPool{
		BackupPool:      d.Get("backup_pool").(string),
		Description:     d.Get("description").(string),
		HealthChecks:    hchkUrls,
		Instances:       instanceUrls,
		Name:            d.Get("name").(string),
		SessionAffinity: d.Get("session_affinity").(string),
	}
	if d.Get("failover_ratio") != nil {
		tpool.FailoverRatio = d.Get("failover_ratio").(float64)
	}
	log.Printf("[DEBUG] TargetPool insert request: %#v", tpool)
	op, err := config.clientCompute.TargetPools.Insert(
		project, region, tpool).Do()
	if err != nil {
		return fmt.Errorf("Error creating TargetPool: %s", err)
	}

	// It probably maybe worked, so store the ID now
	d.SetId(tpool.Name)

	err = computeOperationWait(config.clientCompute, op, project, "Creating Target Pool")
	if err != nil {
		return err
	}
	return resourceComputeTargetPoolRead(d, meta)
}

func calcAddRemove(from []string, to []string) ([]string, []string) {
	add := make([]string, 0)
	remove := make([]string, 0)
	for _, u := range to {
		found := false
		for _, v := range from {
			if u == v {
				found = true
				break
			}
		}
		if !found {
			add = append(add, u)
		}
	}
	for _, u := range from {
		found := false
		for _, v := range to {
			if u == v {
				found = true
				break
			}
		}
		if !found {
			remove = append(remove, u)
		}
	}
	return add, remove
}

func resourceComputeTargetPoolUpdate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	region, err := getRegion(d, config)
	if err != nil {
		return err
	}

	project, err := getProject(d, config)
	if err != nil {
		return err
	}

	d.Partial(true)

	if d.HasChange("health_checks") {

		from_, to_ := d.GetChange("health_checks")
		fromUrls, err := convertHealthChecks(from_.([]interface{}), d, config)
		if err != nil {
			return err
		}
		toUrls, err := convertHealthChecks(to_.([]interface{}), d, config)
		if err != nil {
			return err
		}
		add, remove := calcAddRemove(fromUrls, toUrls)

		removeReq := &compute.TargetPoolsRemoveHealthCheckRequest{
			HealthChecks: make([]*compute.HealthCheckReference, len(remove)),
		}
		for i, v := range remove {
			removeReq.HealthChecks[i] = &compute.HealthCheckReference{HealthCheck: v}
		}
		op, err := config.clientCompute.TargetPools.RemoveHealthCheck(
			project, region, d.Id(), removeReq).Do()
		if err != nil {
			return fmt.Errorf("Error updating health_check: %s", err)
		}

		err = computeOperationWait(config.clientCompute, op, project, "Updating Target Pool")
		if err != nil {
			return err
		}
		addReq := &compute.TargetPoolsAddHealthCheckRequest{
			HealthChecks: make([]*compute.HealthCheckReference, len(add)),
		}
		for i, v := range add {
			addReq.HealthChecks[i] = &compute.HealthCheckReference{HealthCheck: v}
		}
		op, err = config.clientCompute.TargetPools.AddHealthCheck(
			project, region, d.Id(), addReq).Do()
		if err != nil {
			return fmt.Errorf("Error updating health_check: %s", err)
		}

		err = computeOperationWait(config.clientCompute, op, project, "Updating Target Pool")
		if err != nil {
			return err
		}
		d.SetPartial("health_checks")
	}

	if d.HasChange("instances") {

		from_, to_ := d.GetChange("instances")
		from := convertStringArr(from_.([]interface{}))
		to := convertStringArr(to_.([]interface{}))
		fromUrls, err := convertInstancesToUrls(config, project, from)
		if err != nil {
			return err
		}
		toUrls, err := convertInstancesToUrls(config, project, to)
		if err != nil {
			return err
		}
		add, remove := calcAddRemove(fromUrls, toUrls)

		addReq := &compute.TargetPoolsAddInstanceRequest{
			Instances: make([]*compute.InstanceReference, len(add)),
		}
		for i, v := range add {
			addReq.Instances[i] = &compute.InstanceReference{Instance: v}
		}
		op, err := config.clientCompute.TargetPools.AddInstance(
			project, region, d.Id(), addReq).Do()
		if err != nil {
			return fmt.Errorf("Error updating instances: %s", err)
		}

		err = computeOperationWait(config.clientCompute, op, project, "Updating Target Pool")
		if err != nil {
			return err
		}
		removeReq := &compute.TargetPoolsRemoveInstanceRequest{
			Instances: make([]*compute.InstanceReference, len(remove)),
		}
		for i, v := range remove {
			removeReq.Instances[i] = &compute.InstanceReference{Instance: v}
		}
		op, err = config.clientCompute.TargetPools.RemoveInstance(
			project, region, d.Id(), removeReq).Do()
		if err != nil {
			return fmt.Errorf("Error updating instances: %s", err)
		}
		err = computeOperationWait(config.clientCompute, op, project, "Updating Target Pool")
		if err != nil {
			return err
		}
		d.SetPartial("instances")
	}

	if d.HasChange("backup_pool") {
		bpool_name := d.Get("backup_pool").(string)
		tref := &compute.TargetReference{
			Target: bpool_name,
		}
		op, err := config.clientCompute.TargetPools.SetBackup(
			project, region, d.Id(), tref).Do()
		if err != nil {
			return fmt.Errorf("Error updating backup_pool: %s", err)
		}

		err = computeOperationWait(config.clientCompute, op, project, "Updating Target Pool")
		if err != nil {
			return err
		}
		d.SetPartial("backup_pool")
	}

	d.Partial(false)

	return resourceComputeTargetPoolRead(d, meta)
}

func convertInstancesFromUrls(urls []string) []string {
	result := make([]string, 0, len(urls))
	for _, url := range urls {
		urlArray := strings.Split(url, "/")
		instance := fmt.Sprintf("%s/%s", urlArray[len(urlArray)-3], urlArray[len(urlArray)-1])
		result = append(result, instance)
	}
	return result
}

func resourceComputeTargetPoolRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	region, err := getRegion(d, config)
	if err != nil {
		return err
	}

	project, err := getProject(d, config)
	if err != nil {
		return err
	}

	tpool, err := config.clientCompute.TargetPools.Get(
		project, region, d.Id()).Do()
	if err != nil {
		return handleNotFoundError(err, d, fmt.Sprintf("Target Pool %q", d.Get("name").(string)))
	}

	regionUrl := strings.Split(tpool.Region, "/")
	d.Set("self_link", tpool.SelfLink)
	d.Set("backup_pool", tpool.BackupPool)
	d.Set("description", tpool.Description)
	d.Set("failover_ratio", tpool.FailoverRatio)
	d.Set("health_checks", tpool.HealthChecks)
	if tpool.Instances != nil {
		d.Set("instances", convertInstancesFromUrls(tpool.Instances))
	} else {
		d.Set("instances", nil)
	}
	d.Set("name", tpool.Name)
	d.Set("region", regionUrl[len(regionUrl)-1])
	d.Set("session_affinity", tpool.SessionAffinity)
	d.Set("project", project)
	return nil
}

func resourceComputeTargetPoolDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	region, err := getRegion(d, config)
	if err != nil {
		return err
	}

	project, err := getProject(d, config)
	if err != nil {
		return err
	}

	// Delete the TargetPool
	op, err := config.clientCompute.TargetPools.Delete(
		project, region, d.Id()).Do()
	if err != nil {
		return fmt.Errorf("Error deleting TargetPool: %s", err)
	}

	err = computeOperationWait(config.clientCompute, op, project, "Deleting Target Pool")
	if err != nil {
		return err
	}
	d.SetId("")
	return nil
}
<commit_msg>more descriptive error message for health check not found (#883)<commit_after>package google

import (
	"fmt"
	"log"
	"regexp"
	"strings"

	"github.com/hashicorp/terraform/helper/schema"
	"google.golang.org/api/compute/v1"
	"google.golang.org/api/googleapi"
)

var instancesSelfLinkPattern = regexp.MustCompile(fmt.Sprintf(zonalLinkBasePattern, "instances"))

func resourceComputeTargetPool() *schema.Resource {
	return &schema.Resource{
		Create: resourceComputeTargetPoolCreate,
		Read:   resourceComputeTargetPoolRead,
		Delete: resourceComputeTargetPoolDelete,
		Update: resourceComputeTargetPoolUpdate,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"backup_pool": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: false,
			},

			"description": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},

			"failover_ratio": {
				Type:     schema.TypeFloat,
				Optional: true,
				ForceNew: true,
			},

			"health_checks": {
				Type:     schema.TypeList,
				Optional: true,
				ForceNew: false,
				MaxItems: 1,
				Elem: &schema.Schema{
					Type:             schema.TypeString,
					DiffSuppressFunc: compareSelfLinkOrResourceName,
				},
			},

			"instances": {
				Type:     schema.TypeList,
				Optional: true,
				Computed: true,
				ForceNew: false,
				Elem:     &schema.Schema{Type: schema.TypeString},
				DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool {
					// instances are stored in state as "zone/name"
					oldParts := strings.Split(old, "/")

					// instances can also be specified in the config as a URL
					if parts := instancesSelfLinkPattern.FindStringSubmatch(new); len(oldParts) == 2 && len(parts) == 4 {
						// parts[0] = full match
						// parts[1] = project
						// parts[2] = zone
						// parts[3] = instance name

						oZone, oName := oldParts[0], oldParts[1]
						nZone, nName := parts[2], parts[3]
						if oZone == nZone && oName == nName {
							return true
						}
					}
					return false
				},
			},

			"project": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Computed: true,
			},

			"region": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Computed: true,
			},

			"self_link": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"session_affinity": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Default:  "NONE",
			},
		},
	}
}

// Healthchecks need to exist before being referred to from the target pool.
func convertHealthChecks(healthChecks []interface{}, d *schema.ResourceData, config *Config) ([]string, error) {
	if healthChecks == nil || len(healthChecks) == 0 {
		return []string{}, nil
	}

	hc, err := ParseHttpHealthCheckFieldValue(healthChecks[0].(string), d, config)
	if err != nil {
		return nil, err
	}

	return []string{hc.RelativeLink()}, nil
}

// Instances do not need to exist yet, so we simply generate URLs.
// Instances can be full URLS or zone/name
func convertInstancesToUrls(config *Config, project string, names []string) ([]string, error) {
	urls := make([]string, len(names))
	for i, name := range names {
		if strings.HasPrefix(name, "https://www.googleapis.com/compute/v1/") {
			urls[i] = name
		} else {
			splitName := strings.Split(name, "/")
			if len(splitName) != 2 {
				return nil, fmt.Errorf("Invalid instance name, require URL or zone/name: %s", name)
			} else {
				urls[i] = fmt.Sprintf(
					"https://www.googleapis.com/compute/v1/projects/%s/zones/%s/instances/%s",
					project, splitName[0], splitName[1])
			}
		}
	}
	return urls, nil
}

func resourceComputeTargetPoolCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	region, err := getRegion(d, config)
	if err != nil {
		return err
	}

	project, err := getProject(d, config)
	if err != nil {
		return err
	}

	hchkUrls, err := convertHealthChecks(d.Get("health_checks").([]interface{}), d, config)
	if err != nil {
		return err
	}

	instanceUrls, err := convertInstancesToUrls(
		config, project, convertStringArr(d.Get("instances").([]interface{})))
	if err != nil {
		return err
	}

	// Build the parameter
	tpool := &compute.TargetPool{
		BackupPool:      d.Get("backup_pool").(string),
		Description:     d.Get("description").(string),
		HealthChecks:    hchkUrls,
		Instances:       instanceUrls,
		Name:            d.Get("name").(string),
		SessionAffinity: d.Get("session_affinity").(string),
	}
	if d.Get("failover_ratio") != nil {
		tpool.FailoverRatio = d.Get("failover_ratio").(float64)
	}
	log.Printf("[DEBUG] TargetPool insert request: %#v", tpool)
	op, err := config.clientCompute.TargetPools.Insert(
		project, region, tpool).Do()
	if err != nil {
		if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 404 && strings.Contains(gerr.Message, "httpHealthChecks") {
			return fmt.Errorf("Health check %s is not a valid HTTP health check", d.Get("health_checks").([]interface{})[0])
		}
		return fmt.Errorf("Error creating TargetPool: %s", err)
	}

	// It probably maybe worked, so store the ID now
	d.SetId(tpool.Name)

	err = computeOperationWait(config.clientCompute, op, project, "Creating Target Pool")
	if err != nil {
		return err
	}
	return resourceComputeTargetPoolRead(d, meta)
}

func calcAddRemove(from []string, to []string) ([]string, []string) {
	add := make([]string, 0)
	remove := make([]string, 0)
	for _, u := range to {
		found := false
		for _, v := range from {
			if u == v {
				found = true
				break
			}
		}
		if !found {
			add = append(add, u)
		}
	}
	for _, u := range from {
		found := false
		for _, v := range to {
			if u == v {
				found = true
				break
			}
		}
		if !found {
			remove = append(remove, u)
		}
	}
	return add, remove
}

func resourceComputeTargetPoolUpdate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	region, err := getRegion(d, config)
	if err != nil {
		return err
	}

	project, err := getProject(d, config)
	if err != nil {
		return err
	}

	d.Partial(true)

	if d.HasChange("health_checks") {

		from_, to_ := d.GetChange("health_checks")
		fromUrls, err := convertHealthChecks(from_.([]interface{}), d, config)
		if err != nil {
			return err
		}
		toUrls, err := convertHealthChecks(to_.([]interface{}), d, config)
		if err != nil {
			return err
		}
		add, remove := calcAddRemove(fromUrls, toUrls)

		removeReq := &compute.TargetPoolsRemoveHealthCheckRequest{
			HealthChecks: make([]*compute.HealthCheckReference, len(remove)),
		}
		for i, v := range remove {
			removeReq.HealthChecks[i] = &compute.HealthCheckReference{HealthCheck: v}
		}
		op, err := config.clientCompute.TargetPools.RemoveHealthCheck(
			project, region, d.Id(), removeReq).Do()
		if err != nil {
			return fmt.Errorf("Error updating health_check: %s", err)
		}

		err = computeOperationWait(config.clientCompute, op, project, "Updating Target Pool")
		if err != nil {
			return err
		}
		addReq := &compute.TargetPoolsAddHealthCheckRequest{
			HealthChecks: make([]*compute.HealthCheckReference, len(add)),
		}
		for i, v := range add {
			addReq.HealthChecks[i] = &compute.HealthCheckReference{HealthCheck: v}
		}
		op, err = config.clientCompute.TargetPools.AddHealthCheck(
			project, region, d.Id(), addReq).Do()
		if err != nil {
			return fmt.Errorf("Error updating health_check: %s", err)
		}

		err = computeOperationWait(config.clientCompute, op, project, "Updating Target Pool")
		if err != nil {
			return err
		}
		d.SetPartial("health_checks")
	}

	if d.HasChange("instances") {

		from_, to_ := d.GetChange("instances")
		from := convertStringArr(from_.([]interface{}))
		to := convertStringArr(to_.([]interface{}))
		fromUrls, err := convertInstancesToUrls(config, project, from)
		if err != nil {
			return err
		}
		toUrls, err := convertInstancesToUrls(config, project, to)
		if err != nil {
			return err
		}
		add, remove := calcAddRemove(fromUrls, toUrls)

		addReq := &compute.TargetPoolsAddInstanceRequest{
			Instances: make([]*compute.InstanceReference, len(add)),
		}
		for i, v := range add {
			addReq.Instances[i] = &compute.InstanceReference{Instance: v}
		}
		op, err := config.clientCompute.TargetPools.AddInstance(
			project, region, d.Id(), addReq).Do()
		if err != nil {
			return fmt.Errorf("Error updating instances: %s", err)
		}

		err = computeOperationWait(config.clientCompute, op, project, "Updating Target Pool")
		if err != nil {
			return err
		}
		removeReq := &compute.TargetPoolsRemoveInstanceRequest{
			Instances: make([]*compute.InstanceReference, len(remove)),
		}
		for i, v := range remove {
			removeReq.Instances[i] = &compute.InstanceReference{Instance: v}
		}
		op, err = config.clientCompute.TargetPools.RemoveInstance(
			project, region, d.Id(), removeReq).Do()
		if err != nil {
			return fmt.Errorf("Error updating instances: %s", err)
		}
		err = computeOperationWait(config.clientCompute, op, project, "Updating Target Pool")
		if err != nil {
			return err
		}
		d.SetPartial("instances")
	}

	if d.HasChange("backup_pool") {
		bpool_name := d.Get("backup_pool").(string)
		tref := &compute.TargetReference{
			Target: bpool_name,
		}
		op, err := config.clientCompute.TargetPools.SetBackup(
			project, region, d.Id(), tref).Do()
		if err != nil {
			return fmt.Errorf("Error updating backup_pool: %s", err)
		}

		err = computeOperationWait(config.clientCompute, op, project, "Updating Target Pool")
		if err != nil {
			return err
		}
		d.SetPartial("backup_pool")
	}

	d.Partial(false)

	return resourceComputeTargetPoolRead(d, meta)
}

func convertInstancesFromUrls(urls []string) []string {
	result := make([]string, 0, len(urls))
	for _, url := range urls {
		urlArray := strings.Split(url, "/")
		instance := fmt.Sprintf("%s/%s", urlArray[len(urlArray)-3], urlArray[len(urlArray)-1])
		result = append(result, instance)
	}
	return result
}

func resourceComputeTargetPoolRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	region, err := getRegion(d, config)
	if err != nil {
		return err
	}

	project, err := getProject(d, config)
	if err != nil {
		return err
	}

	tpool, err := config.clientCompute.TargetPools.Get(
		project, region, d.Id()).Do()
	if err != nil {
		return handleNotFoundError(err, d, fmt.Sprintf("Target Pool %q", d.Get("name").(string)))
	}

	regionUrl := strings.Split(tpool.Region, "/")
	d.Set("self_link", tpool.SelfLink)
	d.Set("backup_pool", tpool.BackupPool)
	d.Set("description", tpool.Description)
	d.Set("failover_ratio", tpool.FailoverRatio)
	d.Set("health_checks", tpool.HealthChecks)
	if tpool.Instances != nil {
		d.Set("instances", convertInstancesFromUrls(tpool.Instances))
	} else {
		d.Set("instances", nil)
	}
	d.Set("name", tpool.Name)
	d.Set("region", regionUrl[len(regionUrl)-1])
	d.Set("session_affinity", tpool.SessionAffinity)
	d.Set("project", project)
	return nil
}

func resourceComputeTargetPoolDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	region, err := getRegion(d, config)
	if err != nil {
		return err
	}

	project, err := getProject(d, config)
	if err != nil {
		return err
	}

	// Delete the TargetPool
	op, err := config.clientCompute.TargetPools.Delete(
		project, region, d.Id()).Do()
	if err != nil {
		return fmt.Errorf("Error deleting TargetPool: %s", err)
	}

	err = computeOperationWait(config.clientCompute, op, project, "Deleting Target Pool")
	if err != nil {
		return err
	}
	d.SetId("")
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package conn

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

type HttpResponse struct {
	// The HTTP status code, e.g. 200 or 404.
	StatusCode int

	// The response body. This is the empty slice if the body was empty.
	Body []byte
}

// A connection to a host over HTTP.
type HttpConn interface {
	// Send the supplied request to the service.
	SendRequest(req Request) (resp *HttpResponse, err error)
}

// Return a connection to the supplied endpoint, based on its scheme and host
// fields.
func NewHttpConn(endpoint *url.URL) (HttpConn, error) {
	switch endpoint.Scheme {
	case "http", "https":
	default:
		return nil, fmt.Errorf("Unsupported scheme: %s", endpoint.Scheme)
	}

	return &httpConn{endpoint}, nil
}

type httpConn struct {
	endpoint *url.URL
}

func (c* httpConn) SendRequest(req Request) (resp *HttpResponse, err error) {
	// Create an appropriate URL.
	url := url.URL{
		Scheme:   c.endpoint.Scheme,
		Host:     c.endpoint.Host,
		Path:     "/",
	}

	urlStr := url.String()

	// Create a request to the system HTTP library.
	body := []byte("TODO")
	sysReq, err := http.NewRequest("POST", urlStr, bytes.NewBuffer(body))
	if err != nil {
		return nil, fmt.Errorf("http.NewRequest: %v", err)
	}

	// Set required headers.
	//
	// More info:
	//     http://goo.gl/0aD5S
	sysReq.Header.Set(
		"Content-Type",
		"application/x-www-form-urlencoded; charset=utf-8")

	sysReq.Header.Set(
		"Host",
		url.Host)

	// Call the system HTTP library.
	sysResp, err := http.DefaultClient.Do(sysReq)
	if err != nil {
		return nil, fmt.Errorf("http.DefaultClient.Do: %v", err)
	}

	// Convert the response.
	resp = &HttpResponse{
		StatusCode: sysResp.StatusCode,
	}

	if resp.Body, err = ioutil.ReadAll(sysResp.Body); err != nil {
		return nil, fmt.Errorf("Reading body: %v", err)
	}

	sysResp.Body.Close()

	return resp, nil
}
<commit_msg>Set the request body.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package conn

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

type HttpResponse struct {
	// The HTTP status code, e.g. 200 or 404.
	StatusCode int

	// The response body. This is the empty slice if the body was empty.
	Body []byte
}

// A connection to a host over HTTP.
type HttpConn interface {
	// Send the supplied request to the service.
	SendRequest(req Request) (resp *HttpResponse, err error)
}

// Return a connection to the supplied endpoint, based on its scheme and host
// fields.
func NewHttpConn(endpoint *url.URL) (HttpConn, error) {
	switch endpoint.Scheme {
	case "http", "https":
	default:
		return nil, fmt.Errorf("Unsupported scheme: %s", endpoint.Scheme)
	}

	return &httpConn{endpoint}, nil
}

type httpConn struct {
	endpoint *url.URL
}

func (c* httpConn) SendRequest(req Request) (resp *HttpResponse, err error) {
	// Create an appropriate URL.
	u := url.URL{
		Scheme:   c.endpoint.Scheme,
		Host:     c.endpoint.Host,
		Path:     "/",
	}

	urlStr := u.String()

	// Create an appropriate body.
	bodyVals := url.Values{}
	for key, val := range req {
		bodyVals.Set(key, val)
	}

	body := []byte(bodyVals.Encode())

	// Create a request to the system HTTP library.
	sysReq, err := http.NewRequest("POST", urlStr, bytes.NewBuffer(body))
	if err != nil {
		return nil, fmt.Errorf("http.NewRequest: %v", err)
	}

	// Set required headers.
	//
	// More info:
	//     http://goo.gl/0aD5S
	sysReq.Header.Set(
		"Content-Type",
		"application/x-www-form-urlencoded; charset=utf-8")

	sysReq.Header.Set(
		"Host",
		u.Host)

	// Call the system HTTP library.
	sysResp, err := http.DefaultClient.Do(sysReq)
	if err != nil {
		return nil, fmt.Errorf("http.DefaultClient.Do: %v", err)
	}

	// Convert the response.
	resp = &HttpResponse{
		StatusCode: sysResp.StatusCode,
	}

	if resp.Body, err = ioutil.ReadAll(sysResp.Body); err != nil {
		return nil, fmt.Errorf("Reading body: %v", err)
	}

	sysResp.Body.Close()

	return resp, nil
}
<|endoftext|>


<commit_before>package experiments

import (
	"time"
	"github.com/yaricom/goNEAT/neat/genetics"
	"math"
	"encoding/gob"
	"bytes"
	"reflect"
)

// The structure to represent execution results of one generation
type Generation struct {
	// The generation ID for this epoch
	Id          int
	// The time when epoch was evaluated
	Executed    time.Time
	// The best organism of best species
	Best        *genetics.Organism
	// The flag to indicate whether experiment was solved in this epoch
	Solved      bool

	// The list of organisms fitness values per species in population
	Fitness     Floats
	// The age of organisms per species in population
	Age         Floats
	// The list of organisms complexities per species in population
	Compexity   Floats

	// The number of species in population at the end of this epoch
	Diversity   int

	// The number of evaluations done before winner found
	WinnerEvals int
	// The number of nodes in winner genome or zero if not solved
	WinnerNodes int
	// The numbers of genes (links) in winner genome or zero if not solved
	WinnerGenes int
}

// Collects statistics about given population
func (epoch *Generation) FillPopulationStatistics(pop *genetics.Population) {
	max_fitness := float64(math.MinInt64)
	epoch.Diversity = len(pop.Species)
	epoch.Age = make(Floats, epoch.Diversity)
	epoch.Compexity = make(Floats, epoch.Diversity)
	epoch.Fitness = make(Floats, epoch.Diversity)
	for i, curr_species := range pop.Species {
		epoch.Age[i] = float64(curr_species.Age)
		epoch.Compexity[i] = float64(curr_species.Organisms[0].Phenotype.Complexity())
		epoch.Fitness[i] = curr_species.Organisms[0].Fitness

		// find best organism in epoch if not solved
		if !epoch.Solved && curr_species.Organisms[0].Fitness > max_fitness {
			max_fitness = curr_species.Organisms[0].Fitness
			epoch.Best = curr_species.Organisms[0]
		}
	}
}

// Returns average fitness, age, and complexity among all organisms from population at the end of this epoch
func (epoch Generation) Average() (fitness, age, complexity float64) {
	fitness = epoch.Fitness.Mean()
	age = epoch.Age.Mean()
	complexity = epoch.Compexity.Mean()
	return  fitness, age, complexity
}

// Encodes generation with provided GOB encoder
func (epoch Generation) Encode(enc *gob.Encoder) error {
	err := enc.EncodeValue(reflect.ValueOf(epoch.Id))
	err = enc.EncodeValue(reflect.ValueOf(epoch.Executed))
	err = enc.EncodeValue(reflect.ValueOf(epoch.Solved))
	err = enc.EncodeValue(reflect.ValueOf(epoch.Fitness))
	err = enc.EncodeValue(reflect.ValueOf(epoch.Age))
	err = enc.EncodeValue(reflect.ValueOf(epoch.Compexity))
	err = enc.EncodeValue(reflect.ValueOf(epoch.Diversity))
	err = enc.EncodeValue(reflect.ValueOf(epoch.WinnerEvals))
	err = enc.EncodeValue(reflect.ValueOf(epoch.WinnerNodes))
	err = enc.EncodeValue(reflect.ValueOf(epoch.WinnerGenes))

	if err != nil {
		return err
	}

	// encode best organism
	if epoch.Best != nil {
		err = encodeOrganism(enc, epoch.Best)
	}
	return err
}

func encodeOrganism(enc *gob.Encoder, org *genetics.Organism) error {
	err := enc.Encode(org.Fitness)
	err = enc.Encode(org.OriginalFitness)
	err = enc.Encode(org.IsWinner)
	err = enc.Encode(org.Generation)
	err = enc.Encode(org.ExpectedOffspring)
	err = enc.Encode(org.Error)

	if err != nil {
		return err
	}

	// encode organism genome
	if org.Genotype != nil {
		err = enc.Encode(org.Genotype.Id)
		out_buf := bytes.NewBufferString("")
		org.Genotype.Write(out_buf)
		err = enc.Encode(out_buf.Bytes())
	}

	return err
}

func (epoch *Generation) Decode(dec *gob.Decoder) error {
	err := dec.Decode(&epoch.Id)
	err = dec.Decode(&epoch.Executed)
	err = dec.Decode(&epoch.Solved)
	err = dec.Decode(&epoch.Fitness)
	err = dec.Decode(&epoch.Age)
	err = dec.Decode(&epoch.Compexity)
	err = dec.Decode(&epoch.Diversity)
	err = dec.Decode(&epoch.WinnerEvals)
	err = dec.Decode(&epoch.WinnerNodes)
	err = dec.Decode(&epoch.WinnerGenes)

	if err != nil {
		return err
	}

	// decode organism
	org, err := decodeOrganism(dec)
	if err == nil {
		epoch.Best = org
	}
	return err
}

func decodeOrganism(dec *gob.Decoder) (*genetics.Organism, error) {
	org := genetics.Organism{}
	err := dec.Decode(&org.Fitness)
	err = dec.Decode(&org.OriginalFitness)
	err = dec.Decode(&org.IsWinner)
	err = dec.Decode(&org.Generation)
	err = dec.Decode(&org.ExpectedOffspring)
	err = dec.Decode(&org.Error)

	if err != nil {
		return nil, err
	}

	// decode organism genome
	var gen_id int
	err = dec.Decode(&gen_id)
	var data []byte
	err = dec.Decode(&data)
	gen, err := genetics.ReadGenome(bytes.NewBuffer(data), gen_id)
	org.Genotype = gen

	return &org, err
}

// Generations is a sortable collection of generations by execution time and Id
type Generations []Generation

func (is Generations) Len() int {
	return len(is)
}
func (is Generations) Swap(i, j int) {
	is[i], is[j] = is[j], is[i]
}
func (is Generations) Less(i, j int) bool {
	if is[i].Executed.Equal(is[j].Executed) {
		return is[i].Id < is[j].Id // less is from earlier epochs
	}
	return is[i].Executed.Before(is[j].Executed) // less is from earlier time
}
<commit_msg>Added trial ID to easy associate epoch with trial<commit_after>package experiments

import (
	"time"
	"github.com/yaricom/goNEAT/neat/genetics"
	"math"
	"encoding/gob"
	"bytes"
	"reflect"
)

// The structure to represent execution results of one generation
type Generation struct {
	// The generation ID for this epoch
	Id          int
	// The time when epoch was evaluated
	Executed    time.Time
	// The best organism of best species
	Best        *genetics.Organism
	// The flag to indicate whether experiment was solved in this epoch
	Solved      bool

	// The list of organisms fitness values per species in population
	Fitness     Floats
	// The age of organisms per species in population
	Age         Floats
	// The list of organisms complexities per species in population
	Compexity   Floats

	// The number of species in population at the end of this epoch
	Diversity   int

	// The number of evaluations done before winner found
	WinnerEvals int
	// The number of nodes in winner genome or zero if not solved
	WinnerNodes int
	// The numbers of genes (links) in winner genome or zero if not solved
	WinnerGenes int

	// The ID of Trial this Generation was evaluated in
	TrialId     int
}

// Collects statistics about given population
func (epoch *Generation) FillPopulationStatistics(pop *genetics.Population) {
	max_fitness := float64(math.MinInt64)
	epoch.Diversity = len(pop.Species)
	epoch.Age = make(Floats, epoch.Diversity)
	epoch.Compexity = make(Floats, epoch.Diversity)
	epoch.Fitness = make(Floats, epoch.Diversity)
	for i, curr_species := range pop.Species {
		epoch.Age[i] = float64(curr_species.Age)
		epoch.Compexity[i] = float64(curr_species.Organisms[0].Phenotype.Complexity())
		epoch.Fitness[i] = curr_species.Organisms[0].Fitness

		// find best organism in epoch if not solved
		if !epoch.Solved && curr_species.Organisms[0].Fitness > max_fitness {
			max_fitness = curr_species.Organisms[0].Fitness
			epoch.Best = curr_species.Organisms[0]
		}
	}
}

// Returns average fitness, age, and complexity among all organisms from population at the end of this epoch
func (epoch Generation) Average() (fitness, age, complexity float64) {
	fitness = epoch.Fitness.Mean()
	age = epoch.Age.Mean()
	complexity = epoch.Compexity.Mean()
	return fitness, age, complexity
}

// Encodes generation with provided GOB encoder
func (epoch Generation) Encode(enc *gob.Encoder) error {
	err := enc.EncodeValue(reflect.ValueOf(epoch.Id))
	err = enc.EncodeValue(reflect.ValueOf(epoch.Executed))
	err = enc.EncodeValue(reflect.ValueOf(epoch.Solved))
	err = enc.EncodeValue(reflect.ValueOf(epoch.Fitness))
	err = enc.EncodeValue(reflect.ValueOf(epoch.Age))
	err = enc.EncodeValue(reflect.ValueOf(epoch.Compexity))
	err = enc.EncodeValue(reflect.ValueOf(epoch.Diversity))
	err = enc.EncodeValue(reflect.ValueOf(epoch.WinnerEvals))
	err = enc.EncodeValue(reflect.ValueOf(epoch.WinnerNodes))
	err = enc.EncodeValue(reflect.ValueOf(epoch.WinnerGenes))

	if err != nil {
		return err
	}

	// encode best organism
	if epoch.Best != nil {
		err = encodeOrganism(enc, epoch.Best)
	}
	return err
}

func encodeOrganism(enc *gob.Encoder, org *genetics.Organism) error {
	err := enc.Encode(org.Fitness)
	err = enc.Encode(org.OriginalFitness)
	err = enc.Encode(org.IsWinner)
	err = enc.Encode(org.Generation)
	err = enc.Encode(org.ExpectedOffspring)
	err = enc.Encode(org.Error)

	if err != nil {
		return err
	}

	// encode organism genome
	if org.Genotype != nil {
		err = enc.Encode(org.Genotype.Id)
		out_buf := bytes.NewBufferString("")
		org.Genotype.Write(out_buf)
		err = enc.Encode(out_buf.Bytes())
	}

	return err
}

func (epoch *Generation) Decode(dec *gob.Decoder) error {
	err := dec.Decode(&epoch.Id)
	err = dec.Decode(&epoch.Executed)
	err = dec.Decode(&epoch.Solved)
	err = dec.Decode(&epoch.Fitness)
	err = dec.Decode(&epoch.Age)
	err = dec.Decode(&epoch.Compexity)
	err = dec.Decode(&epoch.Diversity)
	err = dec.Decode(&epoch.WinnerEvals)
	err = dec.Decode(&epoch.WinnerNodes)
	err = dec.Decode(&epoch.WinnerGenes)

	if err != nil {
		return err
	}

	// decode organism
	org, err := decodeOrganism(dec)
	if err == nil {
		epoch.Best = org
	}
	return err
}

func decodeOrganism(dec *gob.Decoder) (*genetics.Organism, error) {
	org := genetics.Organism{}
	err := dec.Decode(&org.Fitness)
	err = dec.Decode(&org.OriginalFitness)
	err = dec.Decode(&org.IsWinner)
	err = dec.Decode(&org.Generation)
	err = dec.Decode(&org.ExpectedOffspring)
	err = dec.Decode(&org.Error)

	if err != nil {
		return nil, err
	}

	// decode organism genome
	var gen_id int
	err = dec.Decode(&gen_id)
	var data []byte
	err = dec.Decode(&data)
	gen, err := genetics.ReadGenome(bytes.NewBuffer(data), gen_id)
	org.Genotype = gen

	return &org, err
}

// Generations is a sortable collection of generations by execution time and Id
type Generations []Generation

func (is Generations) Len() int {
	return len(is)
}
func (is Generations) Swap(i, j int) {
	is[i], is[j] = is[j], is[i]
}
func (is Generations) Less(i, j int) bool {
	if is[i].Executed.Equal(is[j].Executed) {
		return is[i].Id < is[j].Id // less is from earlier epochs
	}
	return is[i].Executed.Before(is[j].Executed) // less is from earlier time
}
<|endoftext|>