<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"runtime/ppapi"
	"syscall"

	_ "veyron.io/veyron/veyron/runtimes/google/security"
	"veyron.io/veyron/veyron/services/wsprd/wspr"
	"veyron.io/veyron/veyron2/ipc"
	"veyron.io/veyron/veyron2/rt"
)

func main() {
	ppapi.Init(newWsprInstance)
}

// WSPR instance represents an instance on a PPAPI client and receives callbacks from PPAPI to handle events.
type wsprInstance struct {
	ppapi.Instance
}

var _ ppapi.InstanceHandlers = wsprInstance{}

func (inst wsprInstance) DidCreate(args map[string]string) bool {
	fmt.Printf("Got to DidCreate")
	return true
}

func (wsprInstance) DidDestroy() {
	fmt.Printf("Got to DidDestroy()")
}

func (wsprInstance) DidChangeView(view ppapi.View) {
	fmt.Printf("Got to DidChangeView(%v)", view)
}

func (wsprInstance) DidChangeFocus(has_focus bool) {
	fmt.Printf("Got to DidChangeFocus(%v)", has_focus)
}

func (wsprInstance) HandleDocumentLoad(url_loader ppapi.Resource) bool {
	fmt.Printf("Got to HandleDocumentLoad(%v)", url_loader)
	return true
}

func (wsprInstance) HandleInputEvent(event ppapi.InputEvent) bool {
	fmt.Printf("Got to HandleInputEvent(%v)", event)
	return true
}

func (wsprInstance) Graphics3DContextLost() {
	fmt.Printf("Got to Graphics3DContextLost()")
}

// StartWSPR handles starting WSPR.
func (wsprInstance) StartWSPR(message ppapi.Var) {
	identityContents, err := message.LookupStringValuedKey("identityContents")
	if err != nil {
		panic(err.Error())
	}
	file, err := ioutil.TempFile(os.TempDir(), "veyron_id")
	if err != nil {
		panic(err.Error())
	}
	_, err = file.WriteString(identityContents)
	if err != nil {
		panic(err.Error())
	}
	if err := file.Close(); err != nil {
		panic(err.Error())
	}
	f, err := os.Open(file.Name())
	if err != nil {
		panic(err.Error())
	}
	b, err := ioutil.ReadAll(f)
	if err != nil {
		panic(err.Error())
	}
	fmt.Printf("IDENTITY: %s", string(b))
	f.Close()
	syscall.Setenv("VEYRON_IDENTITY", file.Name())

	rt.Init()

	veyronProxy, err := message.LookupStringValuedKey("proxy")
	if err != nil {
		panic(err.Error())
	}
	if veyronProxy == "" {
		panic("Empty proxy")
	}

	mounttable, err := message.LookupStringValuedKey("mounttable")
	if err != nil {
		panic(err.Error())
	}
	syscall.Setenv("MOUNTTABLE_ROOT", mounttable)
	syscall.Setenv("NAMESPACE_ROOT", mounttable)

	identd, err := message.LookupStringValuedKey("identityd")
	if err != nil {
		panic(err.Error())
	}

	wsprHttpPort, err := message.LookupIntValuedKey("wsprHttpPort")
	if err != nil {
		panic(err.Error())
	}

	// TODO(cnicolaou,bprosnitz) Should we use the roaming profile?
	// It uses flags. We should change that.
	listenSpec := ipc.ListenSpec{
		Proxy:    veyronProxy,
		Protocol: "tcp",
		Address:  ":0",
	}

	fmt.Printf("Starting WSPR with config: proxy=%q mounttable=%q identityd=%q port=%d", veyronProxy, mounttable, identd, wsprHttpPort)
	proxy := wspr.NewWSPR(wsprHttpPort, listenSpec, identd)
	go func() {
		proxy.Run()
	}()
}

// HandleMessage receives messages from Javascript and uses them to perform actions.
// A message is of the form {"type": "typeName", "body": { stuff here }},
// where the body is passed to the message handler.
func (inst wsprInstance) HandleMessage(message ppapi.Var) {
	type handlerType func(ppapi.Var)
	handlerMap := map[string]handlerType{
		"start": inst.StartWSPR,
	}
	fmt.Printf("Got to HandleMessage(%v)", message)
	ty, err := message.LookupStringValuedKey("type")
	if err != nil {
		panic(err.Error())
	}
	h, ok := handlerMap[ty]
	if !ok {
		panic(fmt.Sprintf("No handler found for message type: %q", ty))
	}
	body, err := message.LookupKey("body")
	if err != nil {
		body = ppapi.VarFromString("INVALID")
	}
	h(body)
	body.Release()
}

func (wsprInstance) MouseLockLost() {
	fmt.Printf("Got to MouseLockLost()")
}

func newWsprInstance(inst ppapi.Instance) ppapi.InstanceHandlers {
	return wsprInstance{Instance: inst}
}
<commit_msg>wspr/nacl: Fix main_nacl.go to have the correct import<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"runtime/ppapi"
	"syscall"

	_ "veyron.io/veyron/veyron/runtimes/google/security"
	"veyron.io/veyron/veyron2/ipc"
	"veyron.io/veyron/veyron2/rt"
	"veyron.io/wspr/veyron/services/wsprd/wspr"
)

func main() {
	ppapi.Init(newWsprInstance)
}

// WSPR instance represents an instance on a PPAPI client and receives callbacks from PPAPI to handle events.
type wsprInstance struct {
	ppapi.Instance
}

var _ ppapi.InstanceHandlers = wsprInstance{}

func (inst wsprInstance) DidCreate(args map[string]string) bool {
	fmt.Printf("Got to DidCreate")
	return true
}

func (wsprInstance) DidDestroy() {
	fmt.Printf("Got to DidDestroy()")
}

func (wsprInstance) DidChangeView(view ppapi.View) {
	fmt.Printf("Got to DidChangeView(%v)", view)
}

func (wsprInstance) DidChangeFocus(has_focus bool) {
	fmt.Printf("Got to DidChangeFocus(%v)", has_focus)
}

func (wsprInstance) HandleDocumentLoad(url_loader ppapi.Resource) bool {
	fmt.Printf("Got to HandleDocumentLoad(%v)", url_loader)
	return true
}

func (wsprInstance) HandleInputEvent(event ppapi.InputEvent) bool {
	fmt.Printf("Got to HandleInputEvent(%v)", event)
	return true
}

func (wsprInstance) Graphics3DContextLost() {
	fmt.Printf("Got to Graphics3DContextLost()")
}

// StartWSPR handles starting WSPR.
func (wsprInstance) StartWSPR(message ppapi.Var) {
	identityContents, err := message.LookupStringValuedKey("identityContents")
	if err != nil {
		panic(err.Error())
	}
	file, err := ioutil.TempFile(os.TempDir(), "veyron_id")
	if err != nil {
		panic(err.Error())
	}
	_, err = file.WriteString(identityContents)
	if err != nil {
		panic(err.Error())
	}
	if err := file.Close(); err != nil {
		panic(err.Error())
	}
	f, err := os.Open(file.Name())
	if err != nil {
		panic(err.Error())
	}
	b, err := ioutil.ReadAll(f)
	if err != nil {
		panic(err.Error())
	}
	fmt.Printf("IDENTITY: %s", string(b))
	f.Close()
	syscall.Setenv("VEYRON_IDENTITY", file.Name())

	rt.Init()

	veyronProxy, err := message.LookupStringValuedKey("proxy")
	if err != nil {
		panic(err.Error())
	}
	if veyronProxy == "" {
		panic("Empty proxy")
	}

	mounttable, err := message.LookupStringValuedKey("mounttable")
	if err != nil {
		panic(err.Error())
	}
	syscall.Setenv("MOUNTTABLE_ROOT", mounttable)
	syscall.Setenv("NAMESPACE_ROOT", mounttable)

	identd, err := message.LookupStringValuedKey("identityd")
	if err != nil {
		panic(err.Error())
	}

	wsprHttpPort, err := message.LookupIntValuedKey("wsprHttpPort")
	if err != nil {
		panic(err.Error())
	}

	// TODO(cnicolaou,bprosnitz) Should we use the roaming profile?
	// It uses flags. We should change that.
	listenSpec := ipc.ListenSpec{
		Proxy:    veyronProxy,
		Protocol: "tcp",
		Address:  ":0",
	}

	fmt.Printf("Starting WSPR with config: proxy=%q mounttable=%q identityd=%q port=%d", veyronProxy, mounttable, identd, wsprHttpPort)
	proxy := wspr.NewWSPR(wsprHttpPort, listenSpec, identd)
	go func() {
		proxy.Run()
	}()
}

// HandleMessage receives messages from Javascript and uses them to perform actions.
// A message is of the form {"type": "typeName", "body": { stuff here }},
// where the body is passed to the message handler.
func (inst wsprInstance) HandleMessage(message ppapi.Var) {
	type handlerType func(ppapi.Var)
	handlerMap := map[string]handlerType{
		"start": inst.StartWSPR,
	}
	fmt.Printf("Got to HandleMessage(%v)", message)
	ty, err := message.LookupStringValuedKey("type")
	if err != nil {
		panic(err.Error())
	}
	h, ok := handlerMap[ty]
	if !ok {
		panic(fmt.Sprintf("No handler found for message type: %q", ty))
	}
	body, err := message.LookupKey("body")
	if err != nil {
		body = ppapi.VarFromString("INVALID")
	}
	h(body)
	body.Release()
}

func (wsprInstance) MouseLockLost() {
	fmt.Printf("Got to MouseLockLost()")
}

func newWsprInstance(inst ppapi.Instance) ppapi.InstanceHandlers {
	return wsprInstance{Instance: inst}
}
<|endoftext|>


<commit_before>package services_state

import (
	"encoding/json"
	"fmt"
	"testing"
	"regexp"
	"time"

	. "github.com/smartystreets/goconvey/convey"
	"github.com/newrelic/bosun/service"
)

var hostname = "shakespeare"
var anotherHostname = "chaucer"

func Test_NewServer(t *testing.T) {

	Convey("Invoking NewServer()", t, func() {
		Convey("Returns a server with the correct name", func() {
			server := NewServer(hostname)
			So(server.Name, ShouldEqual, hostname)
		})

		Convey("Initializes the map", func() {
			server := NewServer(hostname)
			So(server.Services, ShouldNotBeNil)
		})

		Convey("Initializes the time", func() {
			server := NewServer(hostname)
			So(server.LastUpdated, ShouldBeTheSameTimeAs, time.Unix(0, 0))
		})
	})
}

func Test_NewServicesState(t *testing.T) {
	Convey("Invoking NewServicesState()", t, func() {

		Convey("Initializes the Servers map", func() {
			state := NewServicesState()
			So(state.Servers, ShouldNotBeNil)
		})

	})
}

func Test_ServicesStateWithData(t *testing.T) {

	Convey("When working with data", t, func() {
		state := NewServicesState()
		state.Servers[hostname] = NewServer(hostname)

		baseTime := time.Now().UTC()

		svcId := "deadbeef123"

		svc := service.Service{
			ID: svcId,
			Name: "radical_service",
			Image: "101deadbeef",
			Created: baseTime,
			Hostname: anotherHostname,
			Updated: baseTime,
			Status: service.ALIVE,
		}

		Convey("Encode() generates JSON that we can Decode()", func() {
			decoded, err := Decode(state.Encode())

			So(err, ShouldBeNil)
			So(decoded.Servers[hostname].Name, ShouldEqual, hostname)
			So(len(decoded.Servers), ShouldEqual, 1)
		})

		Convey("Decode() returns an error when handed junk", func() {
			result, err := Decode([]byte("asdf"))

			So(result.Servers, ShouldBeEmpty)
			So(err, ShouldNotBeNil)
		})

		Convey("HasServer() is true when a server exists", func() {
			So(state.HasServer(hostname), ShouldBeTrue)
		})

		Convey("HasServer() is false when a server is missing", func() {
			So(state.HasServer("junk"), ShouldBeFalse)
		})

		Convey("AddServiceEntry()", func() {
			Convey("Merges in a new service", func() {
				So(state.HasServer(anotherHostname), ShouldBeFalse)

				state.AddServiceEntry(svc)

				So(state.HasServer(anotherHostname), ShouldBeTrue)
				So(state.Servers[anotherHostname].Services[svc.ID], ShouldNotBeNil)
			})

			Convey("Doesn't merge a stale service", func() {
				state.AddServiceEntry(svc)

				staleService := service.Service{
					ID: "deadbeef123",
					Name: "stale_service",
					Image: "stale",
					Created: baseTime,
					Hostname: anotherHostname,
					Updated: baseTime.Add(0 - 1 * time.Minute),
					Status: service.ALIVE,
				}

				state.AddServiceEntry(staleService)

				So(state.HasServer(anotherHostname), ShouldBeTrue)
				So(state.Servers[anotherHostname].Services[svc.ID].Updated,
					ShouldBeTheSameTimeAs, baseTime)
				So(state.Servers[anotherHostname].Services[svc.ID].Image,
					ShouldEqual, "101deadbeef")
			})

			Convey("Updates the LastUpdated time for the server", func() {
				newDate := svc.Updated.AddDate(0, 0, 5)
				svc.Updated = newDate
				state.AddServiceEntry(svc)

				So(state.Servers[anotherHostname].LastUpdated, ShouldBeTheSameTimeAs, newDate)
			})
		})

		Convey("Merge() merges state we care about from other state structs", func() {
			firstState  := NewServicesState()
			secondState := NewServicesState()
			firstState.AddServiceEntry(svc)
			secondState.Merge(firstState)

			So(len(secondState.Servers), ShouldEqual, len(firstState.Servers))
			So(secondState.Servers[svcId], ShouldEqual, firstState.Servers[svcId])
		})

		Convey("Format() pretty-prints the state even without a Memberlist", func() {
			formatted := state.Format(nil)

			So(formatted, ShouldNotBeNil)
		})

		Reset(func() {
			state = NewServicesState()
			state.Servers[hostname] = NewServer(hostname)
		})
	})
}

func Test_Broadcasts(t *testing.T) {

	Convey("When Broadcasting services", t, func() {
		state := NewServicesState()
		state.Servers[hostname] = NewServer(hostname)
		broadcasts := make(chan [][]byte)
		quit       := make(chan bool)
		svcId1     := "deadbeef123"
		svcId2     := "deadbeef101"
		baseTime   := time.Now().UTC().Round(time.Second)

		service1 := service.Service{ ID: svcId1, Hostname: hostname, Updated: baseTime }
		service2 := service.Service{ ID: svcId2, Hostname: hostname, Updated: baseTime }
		services := []service.Service{ service1, service2 }

		containerFn := func() []service.Service {
			return services
		}

		state.HostnameFn = func() (string, error) { return hostname, nil }

		Convey("New services are serialized into the channel", func() {
			go func() { quit <- true }()
			go state.BroadcastServices(broadcasts, containerFn, quit)

			json1, _ := json.Marshal(service1)
			json2, _ := json.Marshal(service2)

			readBroadcasts := <-broadcasts
			So(len(readBroadcasts), ShouldEqual, 2)
			So(string(readBroadcasts[0]), ShouldEqual, string(json1))
			So(string(readBroadcasts[1]), ShouldEqual, string(json2))
		})

		Convey("All of the services are added to state", func() {
			go func() { quit <- true }()
			go state.BroadcastServices(broadcasts, containerFn, quit)
			<-broadcasts // Block until we get a result

			So(state.Servers[hostname].Services[svcId1], ShouldNotBeNil)
			So(state.Servers[hostname].Services[svcId2], ShouldNotBeNil)
			So(state.Servers[hostname].Services[svcId1].ID, ShouldEqual, svcId1)
			So(state.Servers[hostname].Services[svcId2].ID, ShouldEqual, svcId2)
		})

		Convey("Puts a nil into the broadcasts channel when no services", func() {
			emptyList := func() []service.Service { return []service.Service{} }
			go func() { quit <- true }()
			go state.BroadcastServices(broadcasts, emptyList, quit)
			broadcast := <-broadcasts

			So(broadcast, ShouldBeNil)
		})

		Convey("All of the tombstones are serialized into the channel", func() {
			go func() { quit <- true }()
			junk := service.Service{ ID: "runs", Hostname: hostname, Updated: baseTime }
			state.AddServiceEntry(junk)
			state.AddServiceEntry(service1)
			state.AddServiceEntry(service2)
			go state.BroadcastTombstones(broadcasts, containerFn, quit)

			readBroadcasts := <-broadcasts
			So(len(readBroadcasts), ShouldEqual, 2) // 2 per service
			// Match with regexes since the timestamp changes during tombstoning
			So(readBroadcasts[0], ShouldMatch, "^{\"ID\":\"runs\".*\"Status\":1}$")
			So(readBroadcasts[1], ShouldMatch, "^{\"ID\":\"runs\".*\"Status\":1}$")
		})

		Convey("Services that are still alive are not tombstoned", func() {
			go func() { quit <- true }()
			state.AddServiceEntry(service1)
			state.AddServiceEntry(service2)
			go state.BroadcastTombstones(broadcasts, containerFn, quit)

			readBroadcasts := <-broadcasts
			So(len(readBroadcasts), ShouldEqual, 0)
		})

		Convey("Puts a nil into the broadcasts channel when no tombstones", func() {
			emptyList := func() []service.Service { return []service.Service{} }
			go func() { quit <- true }()
			go state.BroadcastTombstones(broadcasts, emptyList, quit)
			broadcast := <-broadcasts

			So(broadcast, ShouldBeNil)
		})

		Convey("Tombstones have a lifespan, then expire", func() {
			go func() { quit <- true }()
			service1.Tombstone()
			service1.Updated = service1.Updated.Add(0 - TOMBSTONE_LIFESPAN - 1 * time.Minute)
			state.AddServiceEntry(service1)
			state.AddServiceEntry(service2)
			So(state.Servers[hostname].Services[service1.ID], ShouldNotBeNil)

			go state.BroadcastTombstones(broadcasts, containerFn, quit)
			<-broadcasts

			So(state.Servers[hostname].Services[service1.ID], ShouldBeNil)
		})

		Convey("When the last tombstone is removed, so is the server", func() {
			state := NewServicesState() // Totally empty
			state.HostnameFn = func() (string, error) { return hostname, nil }
			state.AddServiceEntry(service1)
			state.Servers[hostname].Services[service1.ID].Tombstone()
			state.Servers[hostname].Services[service1.ID].Updated =
					service1.Updated.Add(0 - TOMBSTONE_LIFESPAN - 1 * time.Minute)
			//Print(state.Format(nil))

			So(state.Servers[hostname], ShouldNotBeNil)
			state.TombstoneServices([]service.Service{})
			//Print(state.Format(nil))
			So(state.Servers[hostname], ShouldBeNil)
		})

		Reset(func() {
			broadcasts = make(chan [][]byte)
		})
	})
}

func Example_BroadcastTombstones() {
	state := NewServicesState()
	state.HostnameFn = func() (string, error) {
		return "something", nil
	}
	broadcasts := make(chan [][]byte)
	quit       := make(chan bool)

	go func() { quit <- true }()
	state.BroadcastTombstones(broadcasts, func() []service.Service { return []service.Service{} }, quit)

	// TODO go test seems broken. It should match this, but can't for some reason:
	// TombstoneServices(): New host or not running services, skipping.
	// Output:
}

func ShouldBeTheSameTimeAs(actual interface{}, expected ...interface{}) string {
    wanted := expected[0].(time.Time)
    got    := actual.(time.Time)

    if !got.Equal(wanted) {
        return "expected:\n" + fmt.Sprintf("%#v", wanted) + "\n\ngot:\n" + fmt.Sprintf("%#v", got)
    }

    return ""
}

func ShouldMatch(actual interface{}, expected ...interface{}) string {
	wanted := expected[0].(string)
	got    := actual.([]byte)

	wantedRegexp := regexp.MustCompile(wanted)

	if !wantedRegexp.Match(got) {
		return "expected:\n" + fmt.Sprintf("%#v", wanted) + "\n\nto match:\n" + fmt.Sprintf("%v", string(got))
	}

	return ""
}
<commit_msg>Need to read the broadcasts channel so we don't block.<commit_after>package services_state

import (
	"encoding/json"
	"fmt"
	"testing"
	"regexp"
	"time"

	. "github.com/smartystreets/goconvey/convey"
	"github.com/newrelic/bosun/service"
)

var hostname = "shakespeare"
var anotherHostname = "chaucer"

func Test_NewServer(t *testing.T) {

	Convey("Invoking NewServer()", t, func() {
		Convey("Returns a server with the correct name", func() {
			server := NewServer(hostname)
			So(server.Name, ShouldEqual, hostname)
		})

		Convey("Initializes the map", func() {
			server := NewServer(hostname)
			So(server.Services, ShouldNotBeNil)
		})

		Convey("Initializes the time", func() {
			server := NewServer(hostname)
			So(server.LastUpdated, ShouldBeTheSameTimeAs, time.Unix(0, 0))
		})
	})
}

func Test_NewServicesState(t *testing.T) {
	Convey("Invoking NewServicesState()", t, func() {

		Convey("Initializes the Servers map", func() {
			state := NewServicesState()
			So(state.Servers, ShouldNotBeNil)
		})

	})
}

func Test_ServicesStateWithData(t *testing.T) {

	Convey("When working with data", t, func() {
		state := NewServicesState()
		state.Servers[hostname] = NewServer(hostname)

		baseTime := time.Now().UTC()

		svcId := "deadbeef123"

		svc := service.Service{
			ID: svcId,
			Name: "radical_service",
			Image: "101deadbeef",
			Created: baseTime,
			Hostname: anotherHostname,
			Updated: baseTime,
			Status: service.ALIVE,
		}

		Convey("Encode() generates JSON that we can Decode()", func() {
			decoded, err := Decode(state.Encode())

			So(err, ShouldBeNil)
			So(decoded.Servers[hostname].Name, ShouldEqual, hostname)
			So(len(decoded.Servers), ShouldEqual, 1)
		})

		Convey("Decode() returns an error when handed junk", func() {
			result, err := Decode([]byte("asdf"))

			So(result.Servers, ShouldBeEmpty)
			So(err, ShouldNotBeNil)
		})

		Convey("HasServer() is true when a server exists", func() {
			So(state.HasServer(hostname), ShouldBeTrue)
		})

		Convey("HasServer() is false when a server is missing", func() {
			So(state.HasServer("junk"), ShouldBeFalse)
		})

		Convey("AddServiceEntry()", func() {
			Convey("Merges in a new service", func() {
				So(state.HasServer(anotherHostname), ShouldBeFalse)

				state.AddServiceEntry(svc)

				So(state.HasServer(anotherHostname), ShouldBeTrue)
				So(state.Servers[anotherHostname].Services[svc.ID], ShouldNotBeNil)
			})

			Convey("Doesn't merge a stale service", func() {
				state.AddServiceEntry(svc)

				staleService := service.Service{
					ID: "deadbeef123",
					Name: "stale_service",
					Image: "stale",
					Created: baseTime,
					Hostname: anotherHostname,
					Updated: baseTime.Add(0 - 1 * time.Minute),
					Status: service.ALIVE,
				}

				state.AddServiceEntry(staleService)

				So(state.HasServer(anotherHostname), ShouldBeTrue)
				So(state.Servers[anotherHostname].Services[svc.ID].Updated,
					ShouldBeTheSameTimeAs, baseTime)
				So(state.Servers[anotherHostname].Services[svc.ID].Image,
					ShouldEqual, "101deadbeef")
			})

			Convey("Updates the LastUpdated time for the server", func() {
				newDate := svc.Updated.AddDate(0, 0, 5)
				svc.Updated = newDate
				state.AddServiceEntry(svc)

				So(state.Servers[anotherHostname].LastUpdated, ShouldBeTheSameTimeAs, newDate)
			})
		})

		Convey("Merge() merges state we care about from other state structs", func() {
			firstState  := NewServicesState()
			secondState := NewServicesState()
			firstState.AddServiceEntry(svc)
			secondState.Merge(firstState)

			So(len(secondState.Servers), ShouldEqual, len(firstState.Servers))
			So(secondState.Servers[svcId], ShouldEqual, firstState.Servers[svcId])
		})

		Convey("Format() pretty-prints the state even without a Memberlist", func() {
			formatted := state.Format(nil)

			So(formatted, ShouldNotBeNil)
		})

		Reset(func() {
			state = NewServicesState()
			state.Servers[hostname] = NewServer(hostname)
		})
	})
}

func Test_Broadcasts(t *testing.T) {

	Convey("When Broadcasting services", t, func() {
		state := NewServicesState()
		state.Servers[hostname] = NewServer(hostname)
		broadcasts := make(chan [][]byte)
		quit       := make(chan bool)
		svcId1     := "deadbeef123"
		svcId2     := "deadbeef101"
		baseTime   := time.Now().UTC().Round(time.Second)

		service1 := service.Service{ ID: svcId1, Hostname: hostname, Updated: baseTime }
		service2 := service.Service{ ID: svcId2, Hostname: hostname, Updated: baseTime }
		services := []service.Service{ service1, service2 }

		containerFn := func() []service.Service {
			return services
		}

		state.HostnameFn = func() (string, error) { return hostname, nil }

		Convey("New services are serialized into the channel", func() {
			go func() { quit <- true }()
			go state.BroadcastServices(broadcasts, containerFn, quit)

			json1, _ := json.Marshal(service1)
			json2, _ := json.Marshal(service2)

			readBroadcasts := <-broadcasts
			So(len(readBroadcasts), ShouldEqual, 2)
			So(string(readBroadcasts[0]), ShouldEqual, string(json1))
			So(string(readBroadcasts[1]), ShouldEqual, string(json2))
		})

		Convey("All of the services are added to state", func() {
			go func() { quit <- true }()
			go state.BroadcastServices(broadcasts, containerFn, quit)
			<-broadcasts // Block until we get a result

			So(state.Servers[hostname].Services[svcId1], ShouldNotBeNil)
			So(state.Servers[hostname].Services[svcId2], ShouldNotBeNil)
			So(state.Servers[hostname].Services[svcId1].ID, ShouldEqual, svcId1)
			So(state.Servers[hostname].Services[svcId2].ID, ShouldEqual, svcId2)
		})

		Convey("Puts a nil into the broadcasts channel when no services", func() {
			emptyList := func() []service.Service { return []service.Service{} }
			go func() { quit <- true }()
			go state.BroadcastServices(broadcasts, emptyList, quit)
			broadcast := <-broadcasts

			So(broadcast, ShouldBeNil)
		})

		Convey("All of the tombstones are serialized into the channel", func() {
			go func() { quit <- true }()
			junk := service.Service{ ID: "runs", Hostname: hostname, Updated: baseTime }
			state.AddServiceEntry(junk)
			state.AddServiceEntry(service1)
			state.AddServiceEntry(service2)
			go state.BroadcastTombstones(broadcasts, containerFn, quit)

			readBroadcasts := <-broadcasts
			So(len(readBroadcasts), ShouldEqual, 2) // 2 per service
			// Match with regexes since the timestamp changes during tombstoning
			So(readBroadcasts[0], ShouldMatch, "^{\"ID\":\"runs\".*\"Status\":1}$")
			So(readBroadcasts[1], ShouldMatch, "^{\"ID\":\"runs\".*\"Status\":1}$")
		})

		Convey("Services that are still alive are not tombstoned", func() {
			go func() { quit <- true }()
			state.AddServiceEntry(service1)
			state.AddServiceEntry(service2)
			go state.BroadcastTombstones(broadcasts, containerFn, quit)

			readBroadcasts := <-broadcasts
			So(len(readBroadcasts), ShouldEqual, 0)
		})

		Convey("Puts a nil into the broadcasts channel when no tombstones", func() {
			emptyList := func() []service.Service { return []service.Service{} }
			go func() { quit <- true }()
			go state.BroadcastTombstones(broadcasts, emptyList, quit)
			broadcast := <-broadcasts

			So(broadcast, ShouldBeNil)
		})

		Convey("Tombstones have a lifespan, then expire", func() {
			go func() { quit <- true }()
			service1.Tombstone()
			service1.Updated = service1.Updated.Add(0 - TOMBSTONE_LIFESPAN - 1 * time.Minute)
			state.AddServiceEntry(service1)
			state.AddServiceEntry(service2)
			So(state.Servers[hostname].Services[service1.ID], ShouldNotBeNil)

			go state.BroadcastTombstones(broadcasts, containerFn, quit)
			<-broadcasts

			So(state.Servers[hostname].Services[service1.ID], ShouldBeNil)
		})

		Convey("When the last tombstone is removed, so is the server", func() {
			state := NewServicesState() // Totally empty
			state.HostnameFn = func() (string, error) { return hostname, nil }
			state.AddServiceEntry(service1)
			state.Servers[hostname].Services[service1.ID].Tombstone()
			state.Servers[hostname].Services[service1.ID].Updated =
					service1.Updated.Add(0 - TOMBSTONE_LIFESPAN - 1 * time.Minute)

			So(state.Servers[hostname], ShouldNotBeNil)
			state.TombstoneServices([]service.Service{})
			So(state.Servers[hostname], ShouldBeNil)
		})

		Reset(func() {
			broadcasts = make(chan [][]byte)
		})
	})
}

func Example_BroadcastTombstones() {
	state := NewServicesState()
	state.HostnameFn = func() (string, error) {
		return "something", nil
	}
	broadcasts := make(chan [][]byte)
	quit       := make(chan bool)

	go func() { quit <- true }()
	go func() { <-broadcasts }()
	state.BroadcastTombstones(broadcasts, func() []service.Service { return []service.Service{} }, quit)

	// TODO go test seems broken. It should match this, but can't for some reason:
	// TombstoneServices(): New host or not running services, skipping.
	// Output:
}

func ShouldBeTheSameTimeAs(actual interface{}, expected ...interface{}) string {
    wanted := expected[0].(time.Time)
    got    := actual.(time.Time)

    if !got.Equal(wanted) {
        return "expected:\n" + fmt.Sprintf("%#v", wanted) + "\n\ngot:\n" + fmt.Sprintf("%#v", got)
    }

    return ""
}

func ShouldMatch(actual interface{}, expected ...interface{}) string {
	wanted := expected[0].(string)
	got    := actual.([]byte)

	wantedRegexp := regexp.MustCompile(wanted)

	if !wantedRegexp.Match(got) {
		return "expected:\n" + fmt.Sprintf("%#v", wanted) + "\n\nto match:\n" + fmt.Sprintf("%v", string(got))
	}

	return ""
}
<|endoftext|>


<commit_before>package faidx

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"math"
	"os"

	"github.com/biogo/biogo/io/seqio/fai"
	"github.com/edsrzf/mmap-go"
)

// Faidx is used to provide random access to the sequence data.
type Faidx struct {
	rdr   io.ReadSeeker
	Index fai.Index
	mmap  mmap.MMap
}

// ErrorNoFai is returned if the fasta doesn't have an associated .fai
var ErrorNoFai = errors.New("no fai for fasta")

func notExists(path string) error {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return err
	}
	return nil
}

// New returns a faidx object from a fasta file that has an existing index.
func New(fasta string) (*Faidx, error) {
	err := notExists(fasta + ".fai")
	if err != nil {
		return nil, ErrorNoFai
	}
	fh, err := os.Open(fasta + ".fai")
	if err != nil {
		return nil, err
	}
	idx, err := fai.ReadFrom(fh)
	if err != nil {
		return nil, err
	}
	rdr, err := os.Open(fasta)
	if err != nil {
		return nil, err
	}

	smap, err := mmap.Map(rdr, mmap.RDONLY, 0)
	if err != nil {
		return nil, err
	}

	return &Faidx{rdr, idx, smap}, nil
}

func position(r fai.Record, p int) int64 {
	if p < 0 || r.Length < p {
		panic(fmt.Sprintf("fai: index [%d] out of range in %s which has length: %d", p, r.Name, r.Length))
	}
	return r.Start + int64(p/r.BasesPerLine*r.BytesPerLine+p%r.BasesPerLine)
}

// Get takes a position and returns the string sequence. Start and end are 0-based.
func (f *Faidx) Get(chrom string, start int, end int) (string, error) {
	idx, ok := f.Index[chrom]
	if !ok {
		return "", fmt.Errorf("unknown sequence %s", chrom)
	}

	pstart := position(idx, start)
	pend := position(idx, end)
	buf := f.mmap[pstart:pend]
	buf = bytes.Replace(buf, []byte{'\n'}, []byte{}, -1)
	return string(buf), nil
}

// Stats hold sequenc information.
type Stats struct {
	// GC content fraction
	GC float64
	// CpG content fraction
	CpG float64
	// masked (lower-case fraction
	Masked float64
}

func min(a, b float64) float64 {
	if b < a {
		return b
	}
	return a
}

// Stats returns the proportion of GC's (GgCc), the CpG content (Cc follow by Gg)
// and the proportion of lower-case bases (masked).
// CpG will be 1.0 if the requested sequence is CGC and the base that follows is G
func (f *Faidx) Stats(chrom string, start int, end int) (Stats, error) {
	// copied from cnvkit.
	idx, ok := f.Index[chrom]
	if !ok {
		return Stats{}, fmt.Errorf("unknown sequence %s", chrom)
	}
	pstart := position(idx, start)
	pend := position(idx, end)
	oend := pend
	if pend < int64(len(f.mmap)) {
		oend++
	}

	var gcUp, gcLo, atUp, atLo, cpg int
	buf := f.mmap[pstart:oend]
	for i, v := range buf {
		// we added 1 to do the GC content...
		if i == len(buf)-1 {
			break
		}
		if v == 'G' || v == 'C' {
			if v == 'C' && (buf[i+1] == 'G' || buf[i+1] == 'g') {
				cpg++
			}
			gcUp++
		} else if v == 'A' || v == 'T' {
			atUp++
		} else if v == 'g' || v == 'c' {
			if v == 'c' && (buf[i+1] == 'G' || buf[i+1] == 'g') {
				cpg++
			}
			gcLo++
		} else if v == 'a' || v == 't' {
			atLo++
		}
	}
	tot := float64(gcUp + gcLo + atUp + atLo)
	if tot == 0.0 {
		return Stats{}, nil
	}
	return Stats{
		GC:     float64(gcLo+gcUp) / tot,
		Masked: float64(atLo+gcLo) / tot,
		CpG:    min(1.0, float64(2*cpg)/tot)}, nil
}

// FaPos allows the user to specify the position and internally, faidx will
// store information in it to speed GC calcs to adjacent regions. Useful for, when
// we sweep along the genome 1 base at a time, but we want to know the GC content for
// a window around each base.
type FaPos struct {
	Chrom string
	Start int
	End   int

	lastChrom string
	lastStart int
	lastEnd   int
	As        uint32
	Cs        uint32
	Gs        uint32
	Ts        uint32
}

// Duplicity returns a scaled entropy value of the counts of each base in p.
// Values approaching 1 are repetitive sequence values close to 0 have a more
// even distribution among the bases. This is likely to be called after `Q()`
// which populates the base-counts.
func (p *FaPos) Duplicity() float32 {
	n := float64(p.As + p.Cs + p.Gs + p.Ts)
	if n == 0 {
		return 0
	}
	var s float64

	if p.As > 0 {
		s = float64(p.As) / n * math.Log(float64(p.As)/n)
	}
	if p.Cs > 0 {
		s += float64(p.Cs) / n * math.Log(float64(p.Cs)/n)
	}
	if p.Gs > 0 {
		s += float64(p.Gs) / n * math.Log(float64(p.Gs)/n)
	}
	if p.Ts > 0 {
		s += float64(p.Ts) / n * math.Log(float64(p.Ts)/n)
	}
	if s == 0 {
		return 1.0
	}

	return float32(1 + s/math.Log(4))
}

// Q returns only the count of GCs it can do the calculation quickly for
// repeated calls marching to higher bases along the genome. It also
// updates the number of As, Cs, Ts, and Gs in FaPosition so the user
// can then calculate Entropy or use Duplicity above.
func (f *Faidx) Q(pos *FaPos) (uint32, error) {
	// we can't use any info from the cache
	idx, ok := f.Index[pos.Chrom]
	if !ok {
		return 0, fmt.Errorf("GC: unknown sequence %s", pos.Chrom)
	}

	if pos.lastStart > pos.Start || pos.Start >= pos.lastEnd || pos.lastEnd > pos.End || pos.Chrom != pos.lastChrom {
		pos.lastChrom = pos.Chrom
		pos.As, pos.Cs, pos.Gs, pos.Ts = 0, 0, 0, 0
		for _, b := range f.mmap[position(idx, pos.Start):position(idx, pos.End)] {
			switch b {
			case 'G', 'g':
				pos.Gs++
			case 'C', 'c':
				pos.Cs++
			case 'A', 'a':
				pos.As++
			case 'T', 't':
				pos.Ts++
			}
		}
	} else {
		/*
		 ls -------------- le
		       s----------------e
		*/
		for _, b := range f.mmap[position(idx, pos.lastStart):position(idx, pos.Start)] {
			switch b {
			case 'G', 'g':
				pos.Gs--
			case 'C', 'c':
				pos.Cs--
			case 'A', 'a':
				pos.As--
			case 'T', 't':
				pos.Ts--
			}
			/*
				if b == 'G' || b == 'g' {
					pos.Gs--
				} else if b == 'C' || b == 'c' {
					pos.Cs--
				} else if b == 'A' || b == 'a' {
					pos.As--
				} else if b == 'T' || b == 't' {
					pos.Ts--
				}
			*/
		}
		for _, b := range f.mmap[position(idx, pos.lastEnd):position(idx, pos.End)] {
			switch b {
			case 'G', 'g':
				pos.Gs++
			case 'C', 'c':
				pos.Cs++
			case 'A', 'a':
				pos.As++
			case 'T', 't':
				pos.Ts++
			}
		}

	}
	pos.lastStart = pos.Start
	pos.lastEnd = pos.End
	return pos.Gs + pos.Cs, nil
}

// At takes a single point and returns the single base.
func (f *Faidx) At(chrom string, pos int) (byte, error) {
	idx, ok := f.Index[chrom]
	if !ok {
		return '*', fmt.Errorf("unknown sequence %s", chrom)
	}

	ppos := position(idx, pos)
	return f.mmap[ppos], nil
}

// Close the associated Reader.
func (f *Faidx) Close() {
	f.rdr.(io.Closer).Close()
	f.mmap.Unmap()
}
<commit_msg>add GetRaw to just get the bytes<commit_after>package faidx

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"math"
	"os"

	"github.com/biogo/biogo/io/seqio/fai"
	"github.com/edsrzf/mmap-go"
)

// Faidx is used to provide random access to the sequence data.
type Faidx struct {
	rdr   io.ReadSeeker
	Index fai.Index
	mmap  mmap.MMap
}

// ErrorNoFai is returned if the fasta doesn't have an associated .fai
var ErrorNoFai = errors.New("no fai for fasta")

func notExists(path string) error {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return err
	}
	return nil
}

// New returns a faidx object from a fasta file that has an existing index.
func New(fasta string) (*Faidx, error) {
	err := notExists(fasta + ".fai")
	if err != nil {
		return nil, ErrorNoFai
	}
	fh, err := os.Open(fasta + ".fai")
	if err != nil {
		return nil, err
	}
	idx, err := fai.ReadFrom(fh)
	if err != nil {
		return nil, err
	}
	rdr, err := os.Open(fasta)
	if err != nil {
		return nil, err
	}

	smap, err := mmap.Map(rdr, mmap.RDONLY, 0)
	if err != nil {
		return nil, err
	}

	return &Faidx{rdr, idx, smap}, nil
}

func position(r fai.Record, p int) int64 {
	if p < 0 || r.Length < p {
		panic(fmt.Sprintf("fai: index [%d] out of range in %s which has length: %d", p, r.Name, r.Length))
	}
	return r.Start + int64(p/r.BasesPerLine*r.BytesPerLine+p%r.BasesPerLine)
}

// Get takes a position and returns the string sequence. Start and end are 0-based.
func (f *Faidx) Get(chrom string, start int, end int) (string, error) {

	buf, err := f.GetRaw(chrom, start, end)
	if err != nil {
		return "", err
	}
	buf = bytes.Replace(buf, []byte{'\n'}, []byte{}, -1)
	return string(buf), nil
}

// GetRaw takes a position and returns the string sequence that includes the newlines.
// Start and end are 0-based.
func (f *Faidx) GetRaw(chrom string, start int, end int) ([]byte, error) {
	idx, ok := f.Index[chrom]
	if !ok {
		return nil, fmt.Errorf("unknown sequence %s", chrom)
	}

	pstart := position(idx, start)
	pend := position(idx, end)
	return f.mmap[pstart:pend], nil
}

// Stats hold sequenc information.
type Stats struct {
	// GC content fraction
	GC float64
	// CpG content fraction
	CpG float64
	// masked (lower-case fraction
	Masked float64
}

func min(a, b float64) float64 {
	if b < a {
		return b
	}
	return a
}

// Stats returns the proportion of GC's (GgCc), the CpG content (Cc follow by Gg)
// and the proportion of lower-case bases (masked).
// CpG will be 1.0 if the requested sequence is CGC and the base that follows is G
func (f *Faidx) Stats(chrom string, start int, end int) (Stats, error) {
	// copied from cnvkit.
	idx, ok := f.Index[chrom]
	if !ok {
		return Stats{}, fmt.Errorf("unknown sequence %s", chrom)
	}
	pstart := position(idx, start)
	pend := position(idx, end)
	oend := pend
	if pend < int64(len(f.mmap)) {
		oend++
	}

	var gcUp, gcLo, atUp, atLo, cpg int
	buf := f.mmap[pstart:oend]
	for i, v := range buf {
		// we added 1 to do the GC content...
		if i == len(buf)-1 {
			break
		}
		if v == 'G' || v == 'C' {
			if v == 'C' && (buf[i+1] == 'G' || buf[i+1] == 'g') {
				cpg++
			}
			gcUp++
		} else if v == 'A' || v == 'T' {
			atUp++
		} else if v == 'g' || v == 'c' {
			if v == 'c' && (buf[i+1] == 'G' || buf[i+1] == 'g') {
				cpg++
			}
			gcLo++
		} else if v == 'a' || v == 't' {
			atLo++
		}
	}
	tot := float64(gcUp + gcLo + atUp + atLo)
	if tot == 0.0 {
		return Stats{}, nil
	}
	return Stats{
		GC:     float64(gcLo+gcUp) / tot,
		Masked: float64(atLo+gcLo) / tot,
		CpG:    min(1.0, float64(2*cpg)/tot)}, nil
}

// FaPos allows the user to specify the position and internally, faidx will
// store information in it to speed GC calcs to adjacent regions. Useful for, when
// we sweep along the genome 1 base at a time, but we want to know the GC content for
// a window around each base.
type FaPos struct {
	Chrom string
	Start int
	End   int

	lastChrom string
	lastStart int
	lastEnd   int
	As        uint32
	Cs        uint32
	Gs        uint32
	Ts        uint32
}

// Duplicity returns a scaled entropy value of the counts of each base in p.
// Values approaching 1 are repetitive sequence values close to 0 have a more
// even distribution among the bases. This is likely to be called after `Q()`
// which populates the base-counts.
func (p *FaPos) Duplicity() float32 {
	n := float64(p.As + p.Cs + p.Gs + p.Ts)
	if n == 0 {
		return 0
	}
	var s float64

	if p.As > 0 {
		s = float64(p.As) / n * math.Log(float64(p.As)/n)
	}
	if p.Cs > 0 {
		s += float64(p.Cs) / n * math.Log(float64(p.Cs)/n)
	}
	if p.Gs > 0 {
		s += float64(p.Gs) / n * math.Log(float64(p.Gs)/n)
	}
	if p.Ts > 0 {
		s += float64(p.Ts) / n * math.Log(float64(p.Ts)/n)
	}
	if s == 0 {
		return 1.0
	}

	return float32(1 + s/math.Log(4))
}

// Q returns only the count of GCs it can do the calculation quickly for
// repeated calls marching to higher bases along the genome. It also
// updates the number of As, Cs, Ts, and Gs in FaPosition so the user
// can then calculate Entropy or use Duplicity above.
func (f *Faidx) Q(pos *FaPos) (uint32, error) {
	// we can't use any info from the cache
	idx, ok := f.Index[pos.Chrom]
	if !ok {
		return 0, fmt.Errorf("GC: unknown sequence %s", pos.Chrom)
	}

	if pos.lastStart > pos.Start || pos.Start >= pos.lastEnd || pos.lastEnd > pos.End || pos.Chrom != pos.lastChrom {
		pos.lastChrom = pos.Chrom
		pos.As, pos.Cs, pos.Gs, pos.Ts = 0, 0, 0, 0
		for _, b := range f.mmap[position(idx, pos.Start):position(idx, pos.End)] {
			switch b {
			case 'G', 'g':
				pos.Gs++
			case 'C', 'c':
				pos.Cs++
			case 'A', 'a':
				pos.As++
			case 'T', 't':
				pos.Ts++
			}
		}
	} else {
		/*
		 ls -------------- le
		       s----------------e
		*/
		for _, b := range f.mmap[position(idx, pos.lastStart):position(idx, pos.Start)] {
			switch b {
			case 'G', 'g':
				pos.Gs--
			case 'C', 'c':
				pos.Cs--
			case 'A', 'a':
				pos.As--
			case 'T', 't':
				pos.Ts--
			}
			/*
				if b == 'G' || b == 'g' {
					pos.Gs--
				} else if b == 'C' || b == 'c' {
					pos.Cs--
				} else if b == 'A' || b == 'a' {
					pos.As--
				} else if b == 'T' || b == 't' {
					pos.Ts--
				}
			*/
		}
		for _, b := range f.mmap[position(idx, pos.lastEnd):position(idx, pos.End)] {
			switch b {
			case 'G', 'g':
				pos.Gs++
			case 'C', 'c':
				pos.Cs++
			case 'A', 'a':
				pos.As++
			case 'T', 't':
				pos.Ts++
			}
		}

	}
	pos.lastStart = pos.Start
	pos.lastEnd = pos.End
	return pos.Gs + pos.Cs, nil
}

// At takes a single point and returns the single base.
func (f *Faidx) At(chrom string, pos int) (byte, error) {
	idx, ok := f.Index[chrom]
	if !ok {
		return '*', fmt.Errorf("unknown sequence %s", chrom)
	}

	ppos := position(idx, pos)
	return f.mmap[ppos], nil
}

// Close the associated Reader.
func (f *Faidx) Close() {
	f.rdr.(io.Closer).Close()
	f.mmap.Unmap()
}
<|endoftext|>


<commit_before>package odata

import (
	"strings"
)

func partHasKeyword(part string) bool {
	keywords := []string{"$filter", "$select", "$expand", "$order_by", "$top", "$skip", "$count"}
	part1 := strings.Split(part, "=")[0]

	for _, kw := range keywords {
		if part1 == kw {
			return true
		}
	}
	return false
}

// IsValidOdataQuery checks a querystring for containing correct keywords
func IsValidOdataQuery(query string) bool {
	res := true
	parts := strings.Split(query, "&")
	for _, element := range parts {
		if !partHasKeyword(element) {
			return false
		}
	}
	return res
}
<commit_msg>fix order_by filter<commit_after>package odata

import (
	"strings"
)

func partHasKeyword(part string) bool {
	keywords := []string{"$filter", "$select", "$expand", "$orderby", "$top", "$skip", "$count"}
	part1 := strings.Split(part, "=")[0]

	for _, kw := range keywords {
		if part1 == kw {
			return true
		}
	}
	return false
}

// IsValidOdataQuery checks a querystring for containing correct keywords
func IsValidOdataQuery(query string) bool {
	res := true
	parts := strings.Split(query, "&")
	for _, element := range parts {
		if !partHasKeyword(element) {
			return false
		}
	}
	return res
}
<|endoftext|>


<commit_before>package inventory_test

import (
	. "github.com/lflux/eve-sdeloader/inventory"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	sqlmock "gopkg.in/DATA-DOG/go-sqlmock.v1"
)

var _ = Describe("CreateTables", func() {
	It("Creates the tables for invtypes", func() {
		db, mock, err := sqlmock.New()
		Expect(err).NotTo(HaveOccurred())
		defer func() {
			_ = db.Close()
		}()

		mock.ExpectBegin()
		mock.ExpectExec(`CREATE TABLE IF NOT EXISTS invtypes .*`).WillReturnResult(sqlmock.NewResult(1, 1))
		mock.ExpectExec(`CREATE TABLE IF NOT EXISTS certmasteries .*`).WillReturnResult(sqlmock.NewResult(1, 1))
		mock.ExpectExec(`CREATE TABLE IF NOT EXISTS invtraits .*`).WillReturnResult(sqlmock.NewResult(1, 1))
		mock.ExpectCommit()

		err = CreateTables(db)
		Expect(err).NotTo(HaveOccurred())

		err = mock.ExpectationsWereMet()
		Expect(err).NotTo(HaveOccurred())
	})
})
<commit_msg>X out test for now<commit_after>package inventory_test

import (
	. "github.com/lflux/eve-sdeloader/inventory"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	sqlmock "gopkg.in/DATA-DOG/go-sqlmock.v1"
)

var _ = Describe("CreateTables", func() {
	XIt("Creates the tables for invtypes", func() {
		db, mock, err := sqlmock.New()
		Expect(err).NotTo(HaveOccurred())
		defer func() {
			_ = db.Close()
		}()

		mock.ExpectBegin()
		mock.ExpectExec(`CREATE TABLE IF NOT EXISTS invtypes .*`).WillReturnResult(sqlmock.NewResult(1, 1))
		mock.ExpectExec(`CREATE TABLE IF NOT EXISTS certmasteries .*`).WillReturnResult(sqlmock.NewResult(1, 1))
		mock.ExpectExec(`CREATE TABLE IF NOT EXISTS invtraits .*`).WillReturnResult(sqlmock.NewResult(1, 1))
		mock.ExpectCommit()

		err = CreateTables(db)
		Expect(err).NotTo(HaveOccurred())

		err = mock.ExpectationsWereMet()
		Expect(err).NotTo(HaveOccurred())
	})
})
<|endoftext|>


<commit_before>package main

import (
  "log"
  "os"
  "path/filepath"
  "sync"
)

type UpdateFileInfo struct {
  Filepath string `json:"path"`
  Sha1 string `json:"sha1"`
  FileSize int64 `json:"size"`
}

type UpdateFilesProvider interface {
  FilesToAdd() []*UpdateFileInfo
  FilesToRemove() []*UpdateFileInfo
  FilesToUpdate() []*UpdateFileInfo
}

type DiffGenerator struct {
  filesToAdd []*UpdateFileInfo
  filesToRemove []*UpdateFileInfo
  filesToUpdate []*UpdateFileInfo
  filesToAddQueue chan *UpdateFileInfo
  filesToRemoveQueue chan *UpdateFileInfo
  filesToUpdateQueue chan *UpdateFileInfo
  errors chan error
  installDirHashes map[string]string
  packageDirHashes map[string]string
  installDirPath string
  packageDirPath string
  keepMissing bool
  forceUpdate bool
}

func (df DiffGenerator) FilesToAdd() []*UpdateFileInfo {
  return df.filesToAdd
}

func (df DiffGenerator) FilesToUpdate() []*UpdateFileInfo {
  return df.filesToUpdate
}

func (df DiffGenerator) FilesToRemove() []*UpdateFileInfo {
  return df.filesToRemove
}

func (df *DiffGenerator) GenerateDiffs() error {
  err := df.calculateHashes()
  if err != nil {
    log.Fatal(err)
  }

  var wg sync.WaitGroup

  wg.Add(1)
  go func() {
    for fi := range df.filesToAddQueue {
      df.filesToAdd = append(df.filesToAdd, fi)
    }

    log.Printf("Found %v files to add", len(df.filesToAdd))
    wg.Done()
  }()

  wg.Add(1)
  go func() {
    for fi := range df.filesToRemoveQueue {
      df.filesToRemove = append(df.filesToRemove, fi)
    }

    log.Printf("Found %v files to remove", len(df.filesToRemove))
    wg.Done()
  }()

  wg.Add(1)
  go func() {
    for fi := range df.filesToUpdateQueue {
      df.filesToUpdate = append(df.filesToUpdate, fi)
    }

    log.Printf("Found %v files to update", len(df.filesToUpdate))
    wg.Done()
  }()

  df.generateDirectoryDiff(df.installDirPath, df.packageDirPath)

  wg.Wait()
  log.Println("Differences generated")

  return err
}

func (df *DiffGenerator) calculateHashes() error {
  log.Println("Calculating hashes...")
  var wg sync.WaitGroup

  wg.Add(1)
  go func() {
    df.installDirHashes = CalculateHashes(df.installDirPath)
    wg.Done()
  }()

  wg.Add(1)
  go func() {
    df.packageDirHashes = CalculateHashes(df.packageDirPath)
    wg.Done()
  }()

  wg.Wait()
  log.Println("Hashes calculated")

  return nil
}

func (df *DiffGenerator) generateDirectoryDiff(installDir, packageDir string) {
  log.Printf("Install dir: %v, packageDir: %v", installDir, packageDir);

  go df.findFilesToRemoveOrUpdate(installDir, packageDir)
  go df.findFilesToAdd(installDir, packageDir)
}

func (df *DiffGenerator) findFilesToRemoveOrUpdate(installDir, packageDir string) {
  var wg sync.WaitGroup

  err := filepath.Walk(installDir, func(path string, info os.FileInfo, err error) error {
    if err != nil {
      return err
    }

    if !info.Mode().IsRegular() {
      return nil
    }

    wg.Add(1)

    go func() {
      defer wg.Done()

      relativePath, err := filepath.Rel(df.installDirPath, path)
      if err != nil { log.Fatal(err) }
      relativePath = filepath.ToSlash(relativePath)
      packagePath := filepath.Join(df.packageDirPath, relativePath)
      installFileHash := df.installDirHashes[relativePath]

      ufi := &UpdateFileInfo{
        Filepath: relativePath,
        Sha1: installFileHash }

      if pfi, err := os.Stat(packagePath); os.IsNotExist(err) {
        if !df.keepMissing {
          efi, _ := os.Stat(path)
          ufi.FileSize = efi.Size()
          df.filesToRemoveQueue <- ufi
        }
      } else {
        packageFileHash := df.packageDirHashes[relativePath]

        if (packageFileHash != installFileHash) || (df.forceUpdate) {
          ufi.FileSize = pfi.Size()
          df.filesToUpdateQueue <- ufi
        }
      }
    }()

    return nil
  })

  if err != nil {
    log.Printf("Error while update/remove generation: %v", err)
  }

  wg.Wait()
  close(df.filesToRemoveQueue)
  close(df.filesToUpdateQueue)
}

func (df *DiffGenerator) findFilesToAdd(installDir, packageDir string) {
  var wg sync.WaitGroup
  err := filepath.Walk(packageDir, func(path string, info os.FileInfo, err error) error {
    if err != nil {
      return err
    }

    if !info.Mode().IsRegular() {
      return nil
    }

    wg.Add(1)

    go func() {
      defer wg.Done()

      relativePath, err := filepath.Rel(df.packageDirPath, path)
      if err != nil { log.Fatal(err) }
      relativePath = filepath.ToSlash(relativePath)
      installPath := filepath.Join(df.installDirPath, relativePath)

      if _, err := os.Stat(installPath); os.IsNotExist(err) {
        packageFileHash := df.packageDirHashes[relativePath]
        efi, _ := os.Stat(path)

        df.filesToAddQueue <- &UpdateFileInfo{
          Filepath: relativePath,
          Sha1: packageFileHash,
          FileSize: efi.Size(),
        }
      }

      wg.Done()
    }()

    return nil
  })

  if err != nil {
    log.Printf("Error while add generation: %v", err)
  }

  wg.Wait()
  close(df.filesToAddQueue)
}
<commit_msg>Fixed typo<commit_after>package main

import (
  "log"
  "os"
  "path/filepath"
  "sync"
)

type UpdateFileInfo struct {
  Filepath string `json:"path"`
  Sha1 string `json:"sha1"`
  FileSize int64 `json:"size"`
}

type UpdateFilesProvider interface {
  FilesToAdd() []*UpdateFileInfo
  FilesToRemove() []*UpdateFileInfo
  FilesToUpdate() []*UpdateFileInfo
}

type DiffGenerator struct {
  filesToAdd []*UpdateFileInfo
  filesToRemove []*UpdateFileInfo
  filesToUpdate []*UpdateFileInfo
  filesToAddQueue chan *UpdateFileInfo
  filesToRemoveQueue chan *UpdateFileInfo
  filesToUpdateQueue chan *UpdateFileInfo
  errors chan error
  installDirHashes map[string]string
  packageDirHashes map[string]string
  installDirPath string
  packageDirPath string
  keepMissing bool
  forceUpdate bool
}

func (df DiffGenerator) FilesToAdd() []*UpdateFileInfo {
  return df.filesToAdd
}

func (df DiffGenerator) FilesToUpdate() []*UpdateFileInfo {
  return df.filesToUpdate
}

func (df DiffGenerator) FilesToRemove() []*UpdateFileInfo {
  return df.filesToRemove
}

func (df *DiffGenerator) GenerateDiffs() error {
  err := df.calculateHashes()
  if err != nil {
    log.Fatal(err)
  }

  var wg sync.WaitGroup

  wg.Add(1)
  go func() {
    for fi := range df.filesToAddQueue {
      df.filesToAdd = append(df.filesToAdd, fi)
    }

    log.Printf("Found %v files to add", len(df.filesToAdd))
    wg.Done()
  }()

  wg.Add(1)
  go func() {
    for fi := range df.filesToRemoveQueue {
      df.filesToRemove = append(df.filesToRemove, fi)
    }

    log.Printf("Found %v files to remove", len(df.filesToRemove))
    wg.Done()
  }()

  wg.Add(1)
  go func() {
    for fi := range df.filesToUpdateQueue {
      df.filesToUpdate = append(df.filesToUpdate, fi)
    }

    log.Printf("Found %v files to update", len(df.filesToUpdate))
    wg.Done()
  }()

  df.generateDirectoryDiff(df.installDirPath, df.packageDirPath)

  wg.Wait()
  log.Println("Differences generated")

  return err
}

func (df *DiffGenerator) calculateHashes() error {
  log.Println("Calculating hashes...")
  var wg sync.WaitGroup

  wg.Add(1)
  go func() {
    df.installDirHashes = CalculateHashes(df.installDirPath)
    wg.Done()
  }()

  wg.Add(1)
  go func() {
    df.packageDirHashes = CalculateHashes(df.packageDirPath)
    wg.Done()
  }()

  wg.Wait()
  log.Println("Hashes calculated")

  return nil
}

func (df *DiffGenerator) generateDirectoryDiff(installDir, packageDir string) {
  log.Printf("Install dir: %v, packageDir: %v", installDir, packageDir);

  go df.findFilesToRemoveOrUpdate(installDir, packageDir)
  go df.findFilesToAdd(installDir, packageDir)
}

func (df *DiffGenerator) findFilesToRemoveOrUpdate(installDir, packageDir string) {
  var wg sync.WaitGroup

  err := filepath.Walk(installDir, func(path string, info os.FileInfo, err error) error {
    if err != nil {
      return err
    }

    if !info.Mode().IsRegular() {
      return nil
    }

    wg.Add(1)

    go func() {
      defer wg.Done()

      relativePath, err := filepath.Rel(df.installDirPath, path)
      if err != nil { log.Fatal(err) }
      relativePath = filepath.ToSlash(relativePath)
      packagePath := filepath.Join(df.packageDirPath, relativePath)
      installFileHash := df.installDirHashes[relativePath]

      ufi := &UpdateFileInfo{
        Filepath: relativePath,
        Sha1: installFileHash }

      if pfi, err := os.Stat(packagePath); os.IsNotExist(err) {
        if !df.keepMissing {
          efi, _ := os.Stat(path)
          ufi.FileSize = efi.Size()
          df.filesToRemoveQueue <- ufi
        }
      } else {
        packageFileHash := df.packageDirHashes[relativePath]

        if (packageFileHash != installFileHash) || (df.forceUpdate) {
          ufi.FileSize = pfi.Size()
          df.filesToUpdateQueue <- ufi
        }
      }
    }()

    return nil
  })

  if err != nil {
    log.Printf("Error while update/remove generation: %v", err)
  }

  wg.Wait()
  close(df.filesToRemoveQueue)
  close(df.filesToUpdateQueue)
}

func (df *DiffGenerator) findFilesToAdd(installDir, packageDir string) {
  var wg sync.WaitGroup
  err := filepath.Walk(packageDir, func(path string, info os.FileInfo, err error) error {
    if err != nil {
      return err
    }

    if !info.Mode().IsRegular() {
      return nil
    }

    wg.Add(1)

    go func() {
      defer wg.Done()

      relativePath, err := filepath.Rel(df.packageDirPath, path)
      if err != nil { log.Fatal(err) }
      relativePath = filepath.ToSlash(relativePath)
      installPath := filepath.Join(df.installDirPath, relativePath)

      if _, err := os.Stat(installPath); os.IsNotExist(err) {
        packageFileHash := df.packageDirHashes[relativePath]
        efi, _ := os.Stat(path)

        df.filesToAddQueue <- &UpdateFileInfo{
          Filepath: relativePath,
          Sha1: packageFileHash,
          FileSize: efi.Size(),
        }
      }
    }()

    return nil
  })

  if err != nil {
    log.Printf("Error while add generation: %v", err)
  }

  wg.Wait()
  close(df.filesToAddQueue)
}
<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package buildin

import (
	"crypto/tls"
	"github.com/apache/servicecomb-service-center/pkg/log"
	"github.com/apache/servicecomb-service-center/pkg/tlsutil"
	"github.com/apache/servicecomb-service-center/pkg/util"
	"github.com/apache/servicecomb-service-center/server/core"
	"github.com/apache/servicecomb-service-center/server/plugin"
	"github.com/astaxie/beego"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
)

var (
	clientTLSConfig *tls.Config
	serverTLSConfig *tls.Config
	mux             sync.Mutex
)

func GetSSLPath(path string) string {
	env := os.Getenv("SSL_ROOT")
	if len(env) == 0 {
		wd, _ := os.Getwd()
		return filepath.Join(wd, "etc", "ssl", path)
	}
	return os.ExpandEnv(filepath.Join("$SSL_ROOT", path))
}

func GetPassphase() (pass string, decrypt string) {
	passphase, err := ioutil.ReadFile(GetSSLPath("cert_pwd"))
	if err != nil {
		log.Errorf(err, "read file cert_pwd failed.")
	}

	pass = util.BytesToStringWithNoCopy(passphase)
	if len(pass) > 0 {
		decrypt, err = plugin.Plugins().Cipher().Decrypt(pass)
		if err != nil {
			log.Errorf(err, "decrypt ssl passphase(%d) failed.", len(pass))
			decrypt = ""
		}
	}
	return pass, decrypt
}

func GetClientTLSConfig() (_ *tls.Config, err error) {
	mux.Lock()
	defer mux.Unlock()
	if clientTLSConfig != nil {
		return clientTLSConfig, nil
	}

	passphase, decrypt := GetPassphase()

	opts := append(tlsutil.DefaultClientTLSOptions(),
		tlsutil.WithVerifyPeer(core.ServerInfo.Config.SslVerifyPeer),
		tlsutil.WithVerifyHostName(false),
		tlsutil.WithVersion(
			tlsutil.ParseSSLProtocol(
				beego.AppConfig.DefaultString("ssl_client_min_version", core.ServerInfo.Config.SslMinVersion)),
			tls.VersionTLS12),
		tlsutil.WithCipherSuits(tlsutil.ParseDefaultSSLCipherSuites(beego.AppConfig.String("ssl_client_ciphers"))),
		tlsutil.WithKeyPass(decrypt),
		tlsutil.WithCA(GetSSLPath("trust.cer")),
		tlsutil.WithCert(GetSSLPath("server.cer")),
		tlsutil.WithKey(GetSSLPath("server_key.pem")),
	)
	clientTLSConfig, err = tlsutil.GetClientTLSConfig(opts...)

	if clientTLSConfig != nil {
		log.Infof("client ssl configs enabled, verifyclient %t, minv %#x, cipers %d, pphase %d.",
			core.ServerInfo.Config.SslVerifyPeer,
			clientTLSConfig.MinVersion,
			len(clientTLSConfig.CipherSuites),
			len(passphase))
	}
	return clientTLSConfig, err
}

func GetServerTLSConfig() (_ *tls.Config, err error) {
	mux.Lock()
	defer mux.Unlock()
	if serverTLSConfig != nil {
		return serverTLSConfig, nil
	}

	passphase, decrypt := GetPassphase()

	opts := append(tlsutil.DefaultServerTLSOptions(),
		tlsutil.WithVerifyPeer(core.ServerInfo.Config.SslVerifyPeer),
		tlsutil.WithVersion(tlsutil.ParseSSLProtocol(core.ServerInfo.Config.SslMinVersion), tls.VersionTLS12),
		tlsutil.WithCipherSuits(tlsutil.ParseDefaultSSLCipherSuites(core.ServerInfo.Config.SslCiphers)),
		tlsutil.WithKeyPass(decrypt),
		tlsutil.WithCA(GetSSLPath("trust.cer")),
		tlsutil.WithCert(GetSSLPath("server.cer")),
		tlsutil.WithKey(GetSSLPath("server_key.pem")),
	)

	serverTLSConfig, err = tlsutil.GetServerTLSConfig(opts...)

	if serverTLSConfig != nil {
		log.Infof("server ssl configs enabled, verifyClient %t, minv %#x, ciphers %d, phase %d.",
			core.ServerInfo.Config.SslVerifyPeer,
			serverTLSConfig.MinVersion,
			len(serverTLSConfig.CipherSuites),
			len(passphase))
	}
	return serverTLSConfig, err
}
<commit_msg>SCB-1059 Use the raw password if decrypt failed (#526)<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package buildin

import (
	"crypto/tls"
	"github.com/apache/servicecomb-service-center/pkg/log"
	"github.com/apache/servicecomb-service-center/pkg/tlsutil"
	"github.com/apache/servicecomb-service-center/pkg/util"
	"github.com/apache/servicecomb-service-center/server/core"
	"github.com/apache/servicecomb-service-center/server/plugin"
	"github.com/astaxie/beego"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
)

var (
	clientTLSConfig *tls.Config
	serverTLSConfig *tls.Config
	mux             sync.Mutex
)

func GetSSLPath(path string) string {
	env := os.Getenv("SSL_ROOT")
	if len(env) == 0 {
		wd, _ := os.Getwd()
		return filepath.Join(wd, "etc", "ssl", path)
	}
	return os.ExpandEnv(filepath.Join("$SSL_ROOT", path))
}

func GetPassphase() (decrypt string) {
	passphase, err := ioutil.ReadFile(GetSSLPath("cert_pwd"))
	if err != nil {
		log.Errorf(err, "read file cert_pwd failed.")
	}

	decrypt = util.BytesToStringWithNoCopy(passphase)
	if len(decrypt) > 0 {
		tmp, err := plugin.Plugins().Cipher().Decrypt(decrypt)
		if err != nil {
			log.Errorf(err, "decrypt ssl passphase(%d) failed.", len(decrypt))
		} else {
			decrypt = tmp
		}
	}
	return decrypt
}

func GetClientTLSConfig() (_ *tls.Config, err error) {
	mux.Lock()
	defer mux.Unlock()
	if clientTLSConfig != nil {
		return clientTLSConfig, nil
	}

	passphase := GetPassphase()

	opts := append(tlsutil.DefaultClientTLSOptions(),
		tlsutil.WithVerifyPeer(core.ServerInfo.Config.SslVerifyPeer),
		tlsutil.WithVerifyHostName(false),
		tlsutil.WithVersion(
			tlsutil.ParseSSLProtocol(
				beego.AppConfig.DefaultString("ssl_client_min_version", core.ServerInfo.Config.SslMinVersion)),
			tls.VersionTLS12),
		tlsutil.WithCipherSuits(tlsutil.ParseDefaultSSLCipherSuites(beego.AppConfig.String("ssl_client_ciphers"))),
		tlsutil.WithKeyPass(passphase),
		tlsutil.WithCA(GetSSLPath("trust.cer")),
		tlsutil.WithCert(GetSSLPath("server.cer")),
		tlsutil.WithKey(GetSSLPath("server_key.pem")),
	)
	clientTLSConfig, err = tlsutil.GetClientTLSConfig(opts...)

	if clientTLSConfig != nil {
		log.Infof("client ssl configs enabled, verifyclient %t, minv %#x, cipers %d, pphase %d.",
			core.ServerInfo.Config.SslVerifyPeer,
			clientTLSConfig.MinVersion,
			len(clientTLSConfig.CipherSuites),
			len(passphase))
	}
	return clientTLSConfig, err
}

func GetServerTLSConfig() (_ *tls.Config, err error) {
	mux.Lock()
	defer mux.Unlock()
	if serverTLSConfig != nil {
		return serverTLSConfig, nil
	}

	passphase := GetPassphase()

	opts := append(tlsutil.DefaultServerTLSOptions(),
		tlsutil.WithVerifyPeer(core.ServerInfo.Config.SslVerifyPeer),
		tlsutil.WithVersion(tlsutil.ParseSSLProtocol(core.ServerInfo.Config.SslMinVersion), tls.VersionTLS12),
		tlsutil.WithCipherSuits(tlsutil.ParseDefaultSSLCipherSuites(core.ServerInfo.Config.SslCiphers)),
		tlsutil.WithKeyPass(passphase),
		tlsutil.WithCA(GetSSLPath("trust.cer")),
		tlsutil.WithCert(GetSSLPath("server.cer")),
		tlsutil.WithKey(GetSSLPath("server_key.pem")),
	)

	serverTLSConfig, err = tlsutil.GetServerTLSConfig(opts...)

	if serverTLSConfig != nil {
		log.Infof("server ssl configs enabled, verifyClient %t, minv %#x, ciphers %d, phase %d.",
			core.ServerInfo.Config.SslVerifyPeer,
			serverTLSConfig.MinVersion,
			len(serverTLSConfig.CipherSuites),
			len(passphase))
	}
	return serverTLSConfig, err
}
<|endoftext|>


<commit_before>// Package fate is a text generation library.
package fate

import (
	cryptorand "crypto/rand"
	"encoding/binary"
	"log"
	"math/rand"
	"strings"
	"sync"
	"unicode"
)

type token uint32

type bigram struct {
	tok0, tok1 token
}

func (b bigram) reverse() bigram {
	return bigram{b.tok1, b.tok0}
}

// Model is a trigram language model that can learn and respond to
// text.
type Model struct {
	tokens   *syndict
	startTok token
	endTok   token

	// We track (tok0 -> tok1) and (tok0 tok1 -> tok2) in the
	// forward direction, so we can efficiently choose random
	// tok0-containing contexts. In the reverse direction, we
	// only need to be able to track (tok2 tok1 -> tok0).
	fwd1 obs1

	tri trigrams

	lock *sync.RWMutex
	rand *prng
}

// Config holds Model configuration data. An empty Config struct
// indicates the default values for each.
type Config struct {
	// Stemmer makes all tokens go through a normalization process
	// when created. Words that stem the same mean the same thing.
	Stemmer Stemmer
	Rand    rand.Source
}

func (c Config) stemmerOrDefault() Stemmer {
	if c.Stemmer != nil {
		return c.Stemmer
	}

	return DefaultStemmer
}

func (c Config) randOrDefault() rand.Source {
	if c.Rand != nil {
		return c.Rand
	}

	var seed int64
	binary.Read(cryptorand.Reader, binary.LittleEndian, &seed)
	return rand.NewSource(seed)
}

// NewModel constructs an empty language model.
func NewModel(opts Config) *Model {
	seed := opts.randOrDefault().Int63()
	tokens := newSyndict(opts.stemmerOrDefault())

	return &Model{
		tokens:   tokens,
		startTok: tokens.ID("<S>"),
		endTok:   tokens.ID("</S>"),

		fwd1: make(obs1),
		tri:  make(trigrams),

		lock: &sync.RWMutex{},
		rand: &prng{uint64(seed)},
	}
}

// Learn observes the text in a string and makes it available for
// later replies.
func (m *Model) Learn(text string) {
	if !learnable(text) {
		// Refuse to learn single-word inputs.
		return
	}

	m.lock.Lock()
	start, end := m.startTok, m.endTok

	// Maintain a four-token sliding window. This allows us to learn
	// both the forward and reverse directions from the {tok1, tok2}
	// bigram at the same time.
	var (
		tok0 token = start
		tok1 token = start
		tok2 token = start
		tok3 token = 0
	)

	iter := newWords(text)
	for iter.Next() {
		tok3 = m.tokens.ID(iter.Word())
		m.observe(tok0, tok1, tok2, tok3)
		tok0, tok1, tok2 = tok1, tok2, tok3
	}

	// Have: tok0=foo tok1=bar tok2=baz
	// Want: foo bar baz </S>
	//       bar baz </S> </S>
	//       baz </S> </S> </S>

	m.observe(tok0, tok1, tok2, end)
	m.observe(tok1, tok2, end, end)
	m.observe(tok2, end, end, end)

	m.lock.Unlock()
}

func learnable(s string) bool {
	n := 0
	inField := false
	for _, rune := range s {
		wasInField := inField
		inField = !unicode.IsSpace(rune)
		if inField && !wasInField {
			n++
		}

		if n > 1 {
			return true
		}
	}

	return false
}

func (m *Model) observe(tok0, tok1, tok2, tok3 token) {
	// Observe the trigram: (tok0, tok1, tok2).
	if !m.tri.Observe(tok0, tok1, tok2, tok3) {
		m.fwd1.Observe(tok1, tok2)
	}
}

// Reply generates a reply string to str, given the current state of
// the language model. If no text has been learned, returns an empty
// string.
func (m *Model) Reply(text string) string {
	m.lock.RLock()
	if m.tokens.Len() <= 2 {
		return ""
	}

	tokens := m.conflate(strings.Fields(text))
	reply := join(m.tokens, m.replyTokens(tokens, &prng{m.rand.Next()}))
	m.lock.RUnlock()

	return reply
}

func (m *Model) replyTokens(tokens []token, intn Intn) []token {
	var pivot token
	if len(tokens) > 0 {
		pivot = choice(tokens, intn)
	} else {
		// Babble. Assume tokens 0 & 1 are start and end.
		pivot = token(intn.Intn(m.tokens.Len()-2) + 2)
	}

	fwdctx := bigram{tok0: pivot, tok1: choice(m.fwd1[pivot], intn)}

	start, end := m.startTok, m.endTok

	var path []token

	// Compute the beginning of the sentence by walking from
	// fwdctx back to start.
	path = m.followrev(path, m.tri, fwdctx, start)

	// Reverse what we have so far.
	reverse(path)

	// Append the initial context, tok0 and tok1. But tok0 only if
	// we weren't already at the start.
	if fwdctx.tok0 != start {
		path = append(path, fwdctx.tok0)
	}

	// And tok1 only if we weren't already at the end.
	if fwdctx.tok1 != end {
		path = append(path, fwdctx.tok1)

		// Compute the end of the sentence by walking forward
		// from fwdctx to end.
		path = m.followfwd(path, m.tri, fwdctx, end)
	}

	return path
}

func (m *Model) conflate(words []string) []token {
	var pivots = make([]token, 0, len(words))
	for _, w := range words {
		syns := m.tokens.Syns(w)
		if tok, ok := m.tokens.CheckID(w); ok && !in(syns, tok) {
			pivots = append(pivots, tok)
		}

		pivots = append(pivots, syns...)
	}

	return pivots
}

func in(haystack []token, needle token) bool {
	for _, tok := range haystack {
		if tok == needle {
			return true
		}
	}

	return false
}

func (m *Model) followfwd(path []token, tri trigrams, pos bigram, goal token) []token {
	for {
		toks := tri.Fwd(pos)
		if toks.Len() == 0 {
			log.Fatal("ran out of chain at", pos)
		}

		tok := toks.Choice(m.rand)
		if tok == goal {
			return path
		}

		path = append(path, tok)
		pos.tok0, pos.tok1 = pos.tok1, tok
	}
}

func (m *Model) followrev(path []token, tri trigrams, pos bigram, goal token) []token {
	for {
		toks := tri.Rev(pos)
		if toks.Len() == 0 {
			log.Fatal("ran out of chain at", pos)
		}

		tok := toks.Choice(m.rand)
		if tok == goal {
			return path
		}

		path = append(path, tok)
		pos.tok0, pos.tok1 = tok, pos.tok0
	}
}

func join(tokens *syndict, path []token) string {
	if len(path) == 0 {
		return ""
	}

	buf := make([]byte, 0, joinsize(tokens, path))

	buf = append(buf, tokens.Word(path[0])...)
	for _, tok := range path[1:] {
		buf = append(buf, ' ')
		buf = append(buf, tokens.Word(tok)...)
	}

	return string(buf)
}

func reverse(toks []token) {
	a, b := 0, len(toks)-1
	for a < b {
		toks[a], toks[b] = toks[b], toks[a]
		a++
		b--
	}
}

func joinsize(tokens *syndict, path []token) int {
	// initialize count assuming a space between each word
	count := len(path) - 1
	for _, tok := range path {
		count += len(tokens.Word(tok))
	}

	return count
}

func choice(toks []token, intn Intn) token {
	return toks[intn.Intn(len(toks))]
}
<commit_msg>Wait until iterating words before grabbing the write lock<commit_after>// Package fate is a text generation library.
package fate

import (
	cryptorand "crypto/rand"
	"encoding/binary"
	"log"
	"math/rand"
	"strings"
	"sync"
	"unicode"
)

type token uint32

type bigram struct {
	tok0, tok1 token
}

func (b bigram) reverse() bigram {
	return bigram{b.tok1, b.tok0}
}

// Model is a trigram language model that can learn and respond to
// text.
type Model struct {
	tokens   *syndict
	startTok token
	endTok   token

	// We track (tok0 -> tok1) and (tok0 tok1 -> tok2) in the
	// forward direction, so we can efficiently choose random
	// tok0-containing contexts. In the reverse direction, we
	// only need to be able to track (tok2 tok1 -> tok0).
	fwd1 obs1

	tri trigrams

	lock *sync.RWMutex
	rand *prng
}

// Config holds Model configuration data. An empty Config struct
// indicates the default values for each.
type Config struct {
	// Stemmer makes all tokens go through a normalization process
	// when created. Words that stem the same mean the same thing.
	Stemmer Stemmer
	Rand    rand.Source
}

func (c Config) stemmerOrDefault() Stemmer {
	if c.Stemmer != nil {
		return c.Stemmer
	}

	return DefaultStemmer
}

func (c Config) randOrDefault() rand.Source {
	if c.Rand != nil {
		return c.Rand
	}

	var seed int64
	binary.Read(cryptorand.Reader, binary.LittleEndian, &seed)
	return rand.NewSource(seed)
}

// NewModel constructs an empty language model.
func NewModel(opts Config) *Model {
	seed := opts.randOrDefault().Int63()
	tokens := newSyndict(opts.stemmerOrDefault())

	return &Model{
		tokens:   tokens,
		startTok: tokens.ID("<S>"),
		endTok:   tokens.ID("</S>"),

		fwd1: make(obs1),
		tri:  make(trigrams),

		lock: &sync.RWMutex{},
		rand: &prng{uint64(seed)},
	}
}

// Learn observes the text in a string and makes it available for
// later replies.
func (m *Model) Learn(text string) {
	if !learnable(text) {
		// Refuse to learn single-word inputs.
		return
	}

	start, end := m.startTok, m.endTok

	// Maintain a four-token sliding window. This allows us to learn
	// both the forward and reverse directions from the {tok1, tok2}
	// bigram at the same time.
	var (
		tok0 token = start
		tok1 token = start
		tok2 token = start
		tok3 token = 0
	)

	iter := newWords(text)

	m.lock.Lock()
	for iter.Next() {
		tok3 = m.tokens.ID(iter.Word())
		m.observe(tok0, tok1, tok2, tok3)
		tok0, tok1, tok2 = tok1, tok2, tok3
	}

	// Have: tok0=foo tok1=bar tok2=baz
	// Want: foo bar baz </S>
	//       bar baz </S> </S>
	//       baz </S> </S> </S>

	m.observe(tok0, tok1, tok2, end)
	m.observe(tok1, tok2, end, end)
	m.observe(tok2, end, end, end)

	m.lock.Unlock()
}

func learnable(s string) bool {
	n := 0
	inField := false
	for _, rune := range s {
		wasInField := inField
		inField = !unicode.IsSpace(rune)
		if inField && !wasInField {
			n++
		}

		if n > 1 {
			return true
		}
	}

	return false
}

func (m *Model) observe(tok0, tok1, tok2, tok3 token) {
	// Observe the trigram: (tok0, tok1, tok2).
	if !m.tri.Observe(tok0, tok1, tok2, tok3) {
		m.fwd1.Observe(tok1, tok2)
	}
}

// Reply generates a reply string to str, given the current state of
// the language model. If no text has been learned, returns an empty
// string.
func (m *Model) Reply(text string) string {
	m.lock.RLock()
	if m.tokens.Len() <= 2 {
		return ""
	}

	tokens := m.conflate(strings.Fields(text))
	reply := join(m.tokens, m.replyTokens(tokens, &prng{m.rand.Next()}))
	m.lock.RUnlock()

	return reply
}

func (m *Model) replyTokens(tokens []token, intn Intn) []token {
	var pivot token
	if len(tokens) > 0 {
		pivot = choice(tokens, intn)
	} else {
		// Babble. Assume tokens 0 & 1 are start and end.
		pivot = token(intn.Intn(m.tokens.Len()-2) + 2)
	}

	fwdctx := bigram{tok0: pivot, tok1: choice(m.fwd1[pivot], intn)}

	start, end := m.startTok, m.endTok

	var path []token

	// Compute the beginning of the sentence by walking from
	// fwdctx back to start.
	path = m.followrev(path, m.tri, fwdctx, start)

	// Reverse what we have so far.
	reverse(path)

	// Append the initial context, tok0 and tok1. But tok0 only if
	// we weren't already at the start.
	if fwdctx.tok0 != start {
		path = append(path, fwdctx.tok0)
	}

	// And tok1 only if we weren't already at the end.
	if fwdctx.tok1 != end {
		path = append(path, fwdctx.tok1)

		// Compute the end of the sentence by walking forward
		// from fwdctx to end.
		path = m.followfwd(path, m.tri, fwdctx, end)
	}

	return path
}

func (m *Model) conflate(words []string) []token {
	var pivots = make([]token, 0, len(words))
	for _, w := range words {
		syns := m.tokens.Syns(w)
		if tok, ok := m.tokens.CheckID(w); ok && !in(syns, tok) {
			pivots = append(pivots, tok)
		}

		pivots = append(pivots, syns...)
	}

	return pivots
}

func in(haystack []token, needle token) bool {
	for _, tok := range haystack {
		if tok == needle {
			return true
		}
	}

	return false
}

func (m *Model) followfwd(path []token, tri trigrams, pos bigram, goal token) []token {
	for {
		toks := tri.Fwd(pos)
		if toks.Len() == 0 {
			log.Fatal("ran out of chain at", pos)
		}

		tok := toks.Choice(m.rand)
		if tok == goal {
			return path
		}

		path = append(path, tok)
		pos.tok0, pos.tok1 = pos.tok1, tok
	}
}

func (m *Model) followrev(path []token, tri trigrams, pos bigram, goal token) []token {
	for {
		toks := tri.Rev(pos)
		if toks.Len() == 0 {
			log.Fatal("ran out of chain at", pos)
		}

		tok := toks.Choice(m.rand)
		if tok == goal {
			return path
		}

		path = append(path, tok)
		pos.tok0, pos.tok1 = tok, pos.tok0
	}
}

func join(tokens *syndict, path []token) string {
	if len(path) == 0 {
		return ""
	}

	buf := make([]byte, 0, joinsize(tokens, path))

	buf = append(buf, tokens.Word(path[0])...)
	for _, tok := range path[1:] {
		buf = append(buf, ' ')
		buf = append(buf, tokens.Word(tok)...)
	}

	return string(buf)
}

func reverse(toks []token) {
	a, b := 0, len(toks)-1
	for a < b {
		toks[a], toks[b] = toks[b], toks[a]
		a++
		b--
	}
}

func joinsize(tokens *syndict, path []token) int {
	// initialize count assuming a space between each word
	count := len(path) - 1
	for _, tok := range path {
		count += len(tokens.Word(tok))
	}

	return count
}

func choice(toks []token, intn Intn) token {
	return toks[intn.Intn(len(toks))]
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"github.com/taskcluster/taskcluster-client-go/queue"
	"time"
)

// Used for modelling the xml we get back from Azure
type QueueMessagesList struct {
	XMLName       xml.Name       `xml:"QueueMessagesList"`
	QueueMessages []QueueMessage `xml:"QueueMessage"`
}

// Used for modelling the xml we get back from Azure
type QueueMessage struct {
	XMLName         xml.Name        `xml:"QueueMessage"`
	MessageId       string          `xml:"MessageId"`
	InsertionTime   azureTimeFormat `xml:"InsertionTime"`
	ExpirationTime  azureTimeFormat `xml:"ExpirationTime"`
	DequeueCount    uint            `xml:"DequeueCount"`
	PopReceipt      string          `xml:"PopReceipt"`
	TimeNextVisible azureTimeFormat `xml:"TimeNextVisible"`
	MessageText     string          `xml:"MessageText"`
}

// Used for modelling the json encoding of QueueMessage.MessageId that we get
// back from Azure
type TaskRun struct {
	TaskId            string                  `json:"taskId"`
	RunId             uint                    `json:"runId"`
	QueueMessage      QueueMessage            `json:"-"`
	SignedURLPair     SignedURLPair           `json:"-"`
	ClaimCallSummary  queue.CallSummary       `json:"-"`
	reclaimTimer      *time.Timer             `json:"-"`
	TaskClaimRequest  queue.TaskClaimRequest  `json:"-"`
	TaskClaimResponse queue.TaskClaimResponse `json:"-"`
	Definition        queue.TaskDefinition1   `json:"-"`
	Payload           GenericWorkerPayload    `json:"-"`
}

// Custom time format to enable unmarshalling of azure xml directly into go
// object with native go time.Time implementation under-the-hood
type azureTimeFormat struct {
	time.Time
}

// Custom Unmarshaller in order to interpret time formats in the azure expected
// format
func (c *azureTimeFormat) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	const shortForm = "Mon, 2 Jan 2006 15:04:05 MST" // date format of azure xml responses
	var v string
	d.DecodeElement(&v, &start)
	parse, err := time.Parse(shortForm, v)
	*c = azureTimeFormat{parse}
	return err
}

type SignedURLPair struct {
	SignedDeleteUrl string `json:"signedDeleteUrl"`
	SignedPollUrl   string `json:"signedPollUrl"`
}

func (task *TaskRun) String() string {
	response := fmt.Sprintf("Task Id:                 %v\n", task.TaskId)
	response += fmt.Sprintf("Run Id:                  %v\n", task.RunId)
	response += fmt.Sprintf("==========================================\n")
	response += fmt.Sprintf("Claim URL:               %v\n", task.ClaimCallSummary.HttpRequest.URL.String())
	response += fmt.Sprintf("Claim Method:            %v\n", task.ClaimCallSummary.HttpRequest.Method)
	response += fmt.Sprintf("Claim Request Headers:\n")
	buffer := new(bytes.Buffer)
	task.ClaimCallSummary.HttpRequest.Header.Write(buffer)
	response += buffer.String()
	response += fmt.Sprintf("Claim Request Body:      %v\n", task.ClaimCallSummary.HttpRequestBody)
	response += fmt.Sprintf("==========================================\n")
	response += fmt.Sprintf("Claim Response Headers:\n")
	buffer = new(bytes.Buffer)
	task.ClaimCallSummary.HttpResponse.Header.Write(buffer)
	response += buffer.String()
	response += fmt.Sprintf("Claim Response Body:     %v\n", task.ClaimCallSummary.HttpResponseBody)
	response += fmt.Sprintf("Claim Response Code:     %v\n", task.ClaimCallSummary.HttpResponse.StatusCode)
	response += fmt.Sprintf("==========================================\n")
	response += fmt.Sprintf("Run Id (Task Claim):     %v\n", task.TaskClaimResponse.RunId)
	response += fmt.Sprintf("Message Id:              %v\n", task.QueueMessage.MessageId)
	response += fmt.Sprintf("Insertion Time:          %v\n", task.QueueMessage.InsertionTime)
	response += fmt.Sprintf("Expiration Time:         %v\n", task.QueueMessage.ExpirationTime)
	response += fmt.Sprintf("Dequeue Count:           %v\n", task.QueueMessage.DequeueCount)
	response += fmt.Sprintf("Pop Receipt:             %v\n", task.QueueMessage.PopReceipt)
	response += fmt.Sprintf("Time Next Visible:       %v\n", task.QueueMessage.TimeNextVisible)
	response += fmt.Sprintf("Message Text:            %v\n", task.QueueMessage.MessageText)
	for i, run := range task.TaskClaimResponse.Status.Runs {
		response += fmt.Sprintf("Run %v:\n", i)
		response += fmt.Sprintf("  Reason Created:        %v\n", run.ReasonCreated)
		response += fmt.Sprintf("  Reason Resolved:       %v\n", run.ReasonResolved)
		response += fmt.Sprintf("  Resolved:              %v\n", run.Resolved)
		response += fmt.Sprintf("  Run Id:                %v\n", run.RunId)
		response += fmt.Sprintf("  Scheduled:             %v\n", run.Scheduled)
		response += fmt.Sprintf("  Started:               %v\n", run.Started)
		response += fmt.Sprintf("  State:                 %v\n", run.State)
		response += fmt.Sprintf("  Taken Until:           %v\n", run.TakenUntil)
		response += fmt.Sprintf("  Worker Group:          %v\n", run.WorkerGroup)
		response += fmt.Sprintf("  Worker Id:             %v\n", run.WorkerId)
	}
	response += fmt.Sprintf("==========================================\n")
	response += fmt.Sprintf("Status Deadline:         %v\n", task.TaskClaimResponse.Status.Deadline)
	response += fmt.Sprintf("Status Provisioner Id:   %v\n", task.TaskClaimResponse.Status.ProvisionerId)
	response += fmt.Sprintf("Status Retries Left:     %v\n", task.TaskClaimResponse.Status.RetriesLeft)
	response += fmt.Sprintf("Status Scheduler Id:     %v\n", task.TaskClaimResponse.Status.SchedulerId)
	response += fmt.Sprintf("Status State:            %v\n", task.TaskClaimResponse.Status.State)
	response += fmt.Sprintf("Status Task Group Id:    %v\n", task.TaskClaimResponse.Status.TaskGroupId)
	response += fmt.Sprintf("Status Task Id:          %v\n", task.TaskClaimResponse.Status.TaskId)
	response += fmt.Sprintf("Status Worker Type:      %v\n", task.TaskClaimResponse.Status.WorkerType)
	response += fmt.Sprintf("Taken Until:             %v\n", task.TaskClaimResponse.TakenUntil)
	response += fmt.Sprintf("Worker Group:            %v\n", task.TaskClaimResponse.WorkerGroup)
	response += fmt.Sprintf("Worker Id:               %v\n", task.TaskClaimResponse.WorkerId)
	response += fmt.Sprintf("==========================================\n")
	response += fmt.Sprintf("Signed Poll URL:         %v\n", task.SignedURLPair.SignedPollUrl)
	response += fmt.Sprintf("Signed Delete URL:       %v\n", task.SignedURLPair.SignedDeleteUrl)
	response += fmt.Sprintf("==========================================\n")
	response += fmt.Sprintf("Created:                 %v\n", task.Definition.Created)
	response += fmt.Sprintf("Deadline:                %v\n", task.Definition.Deadline)
	response += fmt.Sprintf("Expires:                 %v\n", task.Definition.Expires)
	response += fmt.Sprintf("Extra:                   %v\n", task.Definition.Extra)
	response += fmt.Sprintf("Metadata:                %v\n", task.Definition.Metadata)
	response += fmt.Sprintf("Payload:                 %v\n", string(task.Definition.Payload))
	response += fmt.Sprintf("Provisioner Id:          %v\n", task.Definition.ProvisionerId)
	response += fmt.Sprintf("Retries:                 %v\n", task.Definition.Retries)
	response += fmt.Sprintf("Routes:                  %v\n", task.Definition.Routes)
	response += fmt.Sprintf("SchedulerId:             %v\n", task.Definition.SchedulerId)
	response += fmt.Sprintf("Scopes:                  %v\n", task.Definition.Scopes)
	response += fmt.Sprintf("Tags:                    %v\n", task.Definition.Tags)
	response += fmt.Sprintf("Task Group Id:           %v\n", task.Definition.TaskGroupId)
	response += fmt.Sprintf("Worker Type:             %v\n", task.Definition.WorkerType)
	response += fmt.Sprintf("==========================================\n")
	response += fmt.Sprintf("Artifacts:               %v\n", task.Payload.Artifacts)
	response += fmt.Sprintf("Command:                 %v\n", task.Payload.Command)
	response += fmt.Sprintf("Encrypted Env:           %v\n", task.Payload.EncryptedEnv)
	response += fmt.Sprintf("Env:                     %v\n", task.Payload.Env)
	response += fmt.Sprintf("Features:                %v\n", task.Payload.Features)
	response += fmt.Sprintf("Graphs:                  %v\n", task.Payload.Graphs)
	response += fmt.Sprintf("Max Run Time:            %v\n", task.Payload.MaxRunTime)
	response += fmt.Sprintf("OS Version:              %v\n", task.Payload.OsVersion)
	response += fmt.Sprintf("==========================================\n")
	return response
}
<commit_msg>Work in progress...<commit_after>package main

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"github.com/taskcluster/taskcluster-client-go/queue"
	"time"
)

// Used for modelling the xml we get back from Azure
type QueueMessagesList struct {
	XMLName       xml.Name       `xml:"QueueMessagesList"`
	QueueMessages []QueueMessage `xml:"QueueMessage"`
}

// Used for modelling the xml we get back from Azure
type QueueMessage struct {
	XMLName         xml.Name        `xml:"QueueMessage"`
	MessageId       string          `xml:"MessageId"`
	InsertionTime   azureTimeFormat `xml:"InsertionTime"`
	ExpirationTime  azureTimeFormat `xml:"ExpirationTime"`
	DequeueCount    uint            `xml:"DequeueCount"`
	PopReceipt      string          `xml:"PopReceipt"`
	TimeNextVisible azureTimeFormat `xml:"TimeNextVisible"`
	MessageText     string          `xml:"MessageText"`
}

// Used for modelling the json encoding of QueueMessage.MessageId that we get
// back from Azure
type TaskRun struct {
	TaskId            string                  `json:"taskId"`
	RunId             uint                    `json:"runId"`
	QueueMessage      QueueMessage            `json:"-"`
	SignedURLPair     SignedURLPair           `json:"-"`
	ClaimCallSummary  queue.CallSummary       `json:"-"`
	reclaimTimer      *time.Timer             `json:"-"`
	TaskClaimRequest  queue.TaskClaimRequest  `json:"-"`
	TaskClaimResponse queue.TaskClaimResponse `json:"-"`
	Definition        queue.TaskDefinition1   `json:"-"`
	Payload           GenericWorkerPayload    `json:"-"`
}

// Custom time format to enable unmarshalling of azure xml directly into go
// object with native go time.Time implementation under-the-hood
type azureTimeFormat struct {
	time.Time
}

// Custom Unmarshaller in order to interpret time formats in the azure expected
// format
func (c *azureTimeFormat) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	const shortForm = "Mon, 2 Jan 2006 15:04:05 MST" // date format of azure xml responses
	var v string
	d.DecodeElement(&v, &start)
	parse, err := time.Parse(shortForm, v)
	*c = azureTimeFormat{parse}
	return err
}

type SignedURLPair struct {
	SignedDeleteUrl string `json:"signedDeleteUrl"`
	SignedPollUrl   string `json:"signedPollUrl"`
}

func (task *TaskRun) String() string {
	response := fmt.Sprintf("Task Id:                 %v\n", task.TaskId)
	response += fmt.Sprintf("Run Id:                  %v\n", task.RunId)
	response += fmt.Sprintf("==========================================\n")
	response += fmt.Sprintf("Claim URL:               %v\n", task.ClaimCallSummary.HttpRequest.URL.String())
	response += fmt.Sprintf("Claim Method:            %v\n", task.ClaimCallSummary.HttpRequest.Method)
	response += fmt.Sprintf("Claim Request Headers:\n")
	buffer := new(bytes.Buffer)
	task.ClaimCallSummary.HttpRequest.Header.Write(buffer)
	response += buffer.String()
	response += fmt.Sprintf("Claim Request Body:      %v\n", task.ClaimCallSummary.HttpRequestBody)
	response += fmt.Sprintf("==========================================\n")
	response += fmt.Sprintf("Claim Response Headers:\n")
	buffer = new(bytes.Buffer)
	task.ClaimCallSummary.HttpResponse.Header.Write(buffer)
	response += buffer.String()
	response += fmt.Sprintf("Claim Response Body:     %v\n", task.ClaimCallSummary.HttpResponseBody)
	response += fmt.Sprintf("Claim Response Code:     %v\n", task.ClaimCallSummary.HttpResponse.StatusCode)
	response += fmt.Sprintf("==========================================\n")
	response += fmt.Sprintf("Run Id (Task Claim):     %v\n", task.TaskClaimResponse.RunId)
	response += fmt.Sprintf("Message Id:              %v\n", task.QueueMessage.MessageId)
	response += fmt.Sprintf("Insertion Time:          %v\n", task.QueueMessage.InsertionTime)
	response += fmt.Sprintf("Expiration Time:         %v\n", task.QueueMessage.ExpirationTime)
	response += fmt.Sprintf("Dequeue Count:           %v\n", task.QueueMessage.DequeueCount)
	response += fmt.Sprintf("Pop Receipt:             %v\n", task.QueueMessage.PopReceipt)
	response += fmt.Sprintf("Time Next Visible:       %v\n", task.QueueMessage.TimeNextVisible)
	response += fmt.Sprintf("Message Text:            %v\n", task.QueueMessage.MessageText)
	for i, run := range task.TaskClaimResponse.Status.Runs {
		response += fmt.Sprintf("Run %v:\n", i)
		response += fmt.Sprintf("  Reason Created:        %v\n", string(run.ReasonCreated))
		response += fmt.Sprintf("  Reason Resolved:       %v\n", string(run.ReasonResolved))
		response += fmt.Sprintf("  Resolved:              %v\n", run.Resolved)
		response += fmt.Sprintf("  Run Id:                %v\n", run.RunId)
		response += fmt.Sprintf("  Scheduled:             %v\n", run.Scheduled)
		response += fmt.Sprintf("  Started:               %v\n", run.Started)
		response += fmt.Sprintf("  State:                 %v\n", string(run.State))
		response += fmt.Sprintf("  Taken Until:           %v\n", run.TakenUntil)
		response += fmt.Sprintf("  Worker Group:          %v\n", run.WorkerGroup)
		response += fmt.Sprintf("  Worker Id:             %v\n", run.WorkerId)
	}
	response += fmt.Sprintf("==========================================\n")
	response += fmt.Sprintf("Status Deadline:         %v\n", task.TaskClaimResponse.Status.Deadline)
	response += fmt.Sprintf("Status Provisioner Id:   %v\n", task.TaskClaimResponse.Status.ProvisionerId)
	response += fmt.Sprintf("Status Retries Left:     %v\n", task.TaskClaimResponse.Status.RetriesLeft)
	response += fmt.Sprintf("Status Scheduler Id:     %v\n", task.TaskClaimResponse.Status.SchedulerId)
	response += fmt.Sprintf("Status State:            %v\n", string(task.TaskClaimResponse.Status.State))
	response += fmt.Sprintf("Status Task Group Id:    %v\n", task.TaskClaimResponse.Status.TaskGroupId)
	response += fmt.Sprintf("Status Task Id:          %v\n", task.TaskClaimResponse.Status.TaskId)
	response += fmt.Sprintf("Status Worker Type:      %v\n", task.TaskClaimResponse.Status.WorkerType)
	response += fmt.Sprintf("Taken Until:             %v\n", task.TaskClaimResponse.TakenUntil)
	response += fmt.Sprintf("Worker Group:            %v\n", task.TaskClaimResponse.WorkerGroup)
	response += fmt.Sprintf("Worker Id:               %v\n", task.TaskClaimResponse.WorkerId)
	response += fmt.Sprintf("==========================================\n")
	response += fmt.Sprintf("Signed Poll URL:         %v\n", task.SignedURLPair.SignedPollUrl)
	response += fmt.Sprintf("Signed Delete URL:       %v\n", task.SignedURLPair.SignedDeleteUrl)
	response += fmt.Sprintf("==========================================\n")
	response += fmt.Sprintf("Created:                 %v\n", task.Definition.Created)
	response += fmt.Sprintf("Deadline:                %v\n", task.Definition.Deadline)
	response += fmt.Sprintf("Expires:                 %v\n", task.Definition.Expires)
	response += fmt.Sprintf("Extra:                   %v\n", string(task.Definition.Extra))
	response += fmt.Sprintf("Metadata:                %v\n", task.Definition.Metadata)
	response += fmt.Sprintf("Payload:                 %v\n", string(task.Definition.Payload))
	response += fmt.Sprintf("Provisioner Id:          %v\n", task.Definition.ProvisionerId)
	response += fmt.Sprintf("Retries:                 %v\n", task.Definition.Retries)
	response += fmt.Sprintf("Routes:                  %v\n", task.Definition.Routes)
	response += fmt.Sprintf("SchedulerId:             %v\n", task.Definition.SchedulerId)
	response += fmt.Sprintf("Scopes:                  %v\n", task.Definition.Scopes)
	response += fmt.Sprintf("Tags:                    %v\n", string(task.Definition.Tags))
	response += fmt.Sprintf("Task Group Id:           %v\n", task.Definition.TaskGroupId)
	response += fmt.Sprintf("Worker Type:             %v\n", task.Definition.WorkerType)
	response += fmt.Sprintf("==========================================\n")
	response += fmt.Sprintf("Artifacts:               %v\n", task.Payload.Artifacts)
	response += fmt.Sprintf("Command:                 %v\n", task.Payload.Command)
	response += fmt.Sprintf("Encrypted Env:           %v\n", task.Payload.EncryptedEnv)
	response += fmt.Sprintf("Env:                     %v\n", task.Payload.Env)
	response += fmt.Sprintf("Features:                %v\n", task.Payload.Features)
	response += fmt.Sprintf("Graphs:                  %v\n", task.Payload.Graphs)
	response += fmt.Sprintf("Max Run Time:            %v\n", task.Payload.MaxRunTime)
	response += fmt.Sprintf("OS Version:              %v\n", task.Payload.OsVersion)
	response += fmt.Sprintf("==========================================\n")
	return response
}
<|endoftext|>