<commit_before>package main

import (
	"errors"
	"log"
	"net/url"
	"os"
	"sort"
	"sync"

	"github.com/flynn/flynn-controller/client"
	ct "github.com/flynn/flynn-controller/types"
	"github.com/flynn/flynn-host/types"
	"github.com/flynn/go-dockerclient"
	"github.com/flynn/go-flynn/cluster"
)

func main() {
	cc, err := controller.New(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	cl, err := cluster.NewClient()
	if err != nil {
		log.Fatal(err)
	}
	c := newContext(cl)
	// TODO: initial load of data
	// TODO: periodic full cluster sync for anti-entropy
	go c.watchFormations(cc)
}

func newContext(cl clusterClient) *context {
	return &context{
		clusterClient: cl,
		formations:    NewFormations(),
		hosts:         newHostClients(),
		jobs:          newJobMap(),
	}
}

type context struct {
	clusterClient
	formations *Formations

	hosts *hostClients
	jobs  *jobMap
}

type clusterClient interface {
	ListHosts() (map[string]host.Host, error)
	AddJobs(req *host.AddJobsReq) (*host.AddJobsRes, error)
	ConnectHost(id string) (cluster.Host, error)
}

type formationStreamer interface {
	StreamFormations() (<-chan *ct.ExpandedFormation, *error)
}

func (c *context) watchFormations(fs formationStreamer) {
	ch, _ := fs.StreamFormations()

	for ef := range ch {
		f := NewFormation(c, ef)
		c.formations.Add(f)
		go f.Rectify()
	}

	// TODO: log disconnect and restart
	// TODO: trigger cluster sync
}

func (c *context) watchHost(id string) {
	if !c.hosts.Add(id) {
		return
	}
	defer c.hosts.Remove(id)

	h, err := c.ConnectHost(id)
	if err != nil {
		// TODO: log/handle error
	}
	c.hosts.Set(id, h)

	ch := make(chan *host.Event)
	h.StreamEvents("all", ch)
	for event := range ch {
		if event.Event != "error" && event.Event != "stop" {
			continue
		}
		job := c.jobs.Get(id, event.JobID)
		if job == nil {
			continue
		}

		c.jobs.Remove(id, event.JobID)
		go job.Formation.RemoveJob(job.Type, id, event.JobID)
	}
	// TODO: check error/reconnect
}

func newHostClients() *hostClients {
	return &hostClients{hosts: make(map[string]cluster.Host)}
}

type hostClients struct {
	hosts map[string]cluster.Host
	mtx   sync.RWMutex
}

func (h *hostClients) Add(id string) bool {
	h.mtx.Lock()
	defer h.mtx.Unlock()
	if _, exists := h.hosts[id]; exists {
		return false
	}
	h.hosts[id] = nil
	return true
}

func (h *hostClients) Set(id string, client cluster.Host) {
	h.mtx.Lock()
	h.hosts[id] = client
	h.mtx.Unlock()
}

func (h *hostClients) Remove(id string) {
	h.mtx.Lock()
	delete(h.hosts, id)
	h.mtx.Unlock()
}

func (h *hostClients) Get(id string) cluster.Host {
	h.mtx.RLock()
	defer h.mtx.RUnlock()
	return h.hosts[id]
}

func newJobMap() *jobMap {
	return &jobMap{jobs: make(map[jobKey]*Job)}
}

type jobMap struct {
	jobs map[jobKey]*Job
	mtx  sync.RWMutex
}

func (m *jobMap) Add(hostID, jobID string, job *Job) {
	m.mtx.Lock()
	m.jobs[jobKey{hostID, jobID}] = job
	m.mtx.Unlock()
}

func (m *jobMap) Remove(host, job string) {
	m.mtx.Lock()
	delete(m.jobs, jobKey{host, job})
	m.mtx.Unlock()
}

func (m *jobMap) Get(host, job string) *Job {
	m.mtx.RLock()
	defer m.mtx.RUnlock()
	return m.jobs[jobKey{host, job}]
}

type jobKey struct {
	hostID, jobID string
}

type formationKey struct {
	appID, releaseID string
}

func NewFormations() *Formations {
	return &Formations{formations: make(map[formationKey]*Formation)}
}

type Formations struct {
	formations map[formationKey]*Formation
	mtx        sync.RWMutex
}

func (fs *Formations) Get(appID, releaseID string) *Formation {
	fs.mtx.RLock()
	defer fs.mtx.RUnlock()
	return fs.formations[formationKey{appID, releaseID}]
}

func (fs *Formations) Add(f *Formation) {
	fs.mtx.Lock()
	fs.formations[f.key()] = f
	fs.mtx.Unlock()
}

func (fs *Formations) Delete(f *Formation) {
	fs.mtx.Lock()
	delete(fs.formations, f.key())
	fs.mtx.Unlock()
}

func NewFormation(c *context, ef *ct.ExpandedFormation) *Formation {
	return &Formation{
		App:       ef.App,
		Release:   ef.Release,
		Artifact:  ef.Artifact,
		Processes: ef.Processes,
		jobs:      make(jobTypeMap),
		c:         c,
	}
}

type Job struct {
	Type      string
	Formation *Formation
}

type jobTypeMap map[string]map[jobKey]*Job

func (m jobTypeMap) Add(typ, host, id string) *Job {
	jobs, ok := m[typ]
	if !ok {
		jobs = make(map[jobKey]*Job)
		m[typ] = jobs
	}
	job := &Job{Type: typ}
	jobs[jobKey{host, id}] = job
	return job
}

func (m jobTypeMap) Remove(typ, host, id string) {
	if jobs, ok := m[typ]; ok {
		delete(jobs, jobKey{host, id})
	}
}

func (m jobTypeMap) Get(typ, host, id string) *Job {
	return m[typ][jobKey{host, id}]
}

type Formation struct {
	mtx       sync.Mutex
	App       *ct.App
	Release   *ct.Release
	Artifact  *ct.Artifact
	Processes map[string]int

	jobs jobTypeMap
	c    *context
}

func (f *Formation) key() formationKey {
	return formationKey{f.App.ID, f.Release.ID}
}

func (f *Formation) Rectify() {
	f.mtx.Lock()
	defer f.mtx.Unlock()
	f.rectify()
}

func (f *Formation) RemoveJob(typ, hostID, jobID string) {
	f.mtx.Lock()
	defer f.mtx.Unlock()

	f.jobs.Remove(typ, hostID, jobID)
	f.rectify()
}

func (f *Formation) rectify() {
	// update job counts
	for t, expected := range f.Processes {
		diff := expected - len(f.jobs[t])
		if diff > 0 {
			f.add(diff, t)
		} else if diff < 0 {
			f.remove(-diff, t)
		}
	}

	// remove process types
	for t, jobs := range f.jobs {
		if _, exists := f.Processes[t]; !exists {
			f.remove(len(jobs), t)
		}
	}
}

func (f *Formation) add(n int, name string) {
	config, err := f.jobConfig(name)
	if err != nil {
		// TODO: log/handle error
	}
	for i := 0; i < n; i++ {
		hosts, err := f.c.ListHosts()
		if err != nil {
			// TODO: log/handle error
		}
		if len(hosts) == 0 {
			// TODO: log/handle error
		}
		hostCounts := make(map[string]int, len(hosts))
		for _, h := range hosts {
			hostCounts[h.ID] = 0
			for _, job := range h.Jobs {
				if f.jobType(job) != name {
					continue
				}
				hostCounts[h.ID]++
			}
		}
		sh := make(sortHosts, 0, len(hosts))
		for id, count := range hostCounts {
			sh = append(sh, sortHost{id, count})
		}
		sh.Sort()

		h := hosts[sh[0].ID]
		go f.c.watchHost(h.ID)

		job := f.jobs.Add(name, h.ID, config.ID)
		job.Formation = f
		f.c.jobs.Add(h.ID, config.ID, job)

		res, err := f.c.AddJobs(&host.AddJobsReq{HostJobs: map[string][]*host.Job{h.ID: {config}}})
		if err != nil || !res.Success {
			f.jobs.Remove(name, h.ID, config.ID)
			f.c.jobs.Remove(h.ID, config.ID)
			// TODO: log/handle error
		}
	}
}

func (f *Formation) jobType(job *host.Job) string {
	if job.Attributes["flynn-controller.app"] != f.App.ID ||
		job.Attributes["flynn-controller.release"] != f.Release.ID {
		return ""
	}
	return job.Attributes["flynn-controller.type"]
}

func (f *Formation) remove(n int, name string) {
	i := 0
	for k := range f.jobs[name] {
		// TODO: robust host handling
		if err := f.c.hosts.Get(k.hostID).StopJob(k.jobID); err != nil {
			// TODO: log/handle error
		}
		f.jobs.Remove(name, k.hostID, k.jobID)
		f.c.jobs.Remove(k.hostID, k.jobID)
		if i++; i == n {
			break
		}
	}
}

func (f *Formation) jobConfig(name string) (*host.Job, error) {
	t := f.Release.Processes[name]
	image, err := dockerImage(f.Artifact.URI)
	if err != nil {
		return nil, err
	}
	return &host.Job{
		ID:       cluster.RandomJobID(""),
		TCPPorts: t.Ports.TCP,
		Attributes: map[string]string{
			"flynn-controller.app":     f.App.ID,
			"flynn-controller.release": f.Release.ID,
			"flynn-controller.type":    name,
		},
		Config: &docker.Config{
			Cmd:   t.Cmd,
			Env:   formatEnv(f.Release.Env, t.Env),
			Image: image,
		},
	}, nil
}

func dockerImage(uri string) (string, error) {
	// TODO: ID refs (see https://github.com/dotcloud/docker/issues/4106)
	u, err := url.Parse(uri)
	if err != nil {
		return "", err
	}
	if u.Scheme != "docker" {
		return "", errors.New("scheduler: only docker artifact URIs are currently supported")
	}
	var suffix string
	if tag := u.Query().Get("tag"); tag != "" {
		suffix = ":" + tag
	}
	return u.Host + suffix, nil
}

func formatEnv(envs ...map[string]string) []string {
	env := make(map[string]string)
	for _, e := range envs {
		for k, v := range e {
			env[k] = v
		}
	}
	res := make([]string, 0, len(env))
	for k, v := range env {
		res = append(res, k+"="+v)
	}
	return res
}

type sortHost struct {
	ID   string
	Jobs int
}

type sortHosts []sortHost

func (h sortHosts) Len() int           { return len(h) }
func (h sortHosts) Less(i, j int) bool { return h[i].Jobs < h[j].Jobs }
func (h sortHosts) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
func (h sortHosts) Sort()              { sort.Sort(h) }
<commit_msg>controller/scheduler: Don't run watchFormations in goroutine<commit_after>package main

import (
	"errors"
	"log"
	"net/url"
	"os"
	"sort"
	"sync"

	"github.com/flynn/flynn-controller/client"
	ct "github.com/flynn/flynn-controller/types"
	"github.com/flynn/flynn-host/types"
	"github.com/flynn/go-dockerclient"
	"github.com/flynn/go-flynn/cluster"
)

func main() {
	cc, err := controller.New(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	cl, err := cluster.NewClient()
	if err != nil {
		log.Fatal(err)
	}
	c := newContext(cl)
	// TODO: initial load of data
	// TODO: periodic full cluster sync for anti-entropy
	c.watchFormations(cc)
}

func newContext(cl clusterClient) *context {
	return &context{
		clusterClient: cl,
		formations:    NewFormations(),
		hosts:         newHostClients(),
		jobs:          newJobMap(),
	}
}

type context struct {
	clusterClient
	formations *Formations

	hosts *hostClients
	jobs  *jobMap
}

type clusterClient interface {
	ListHosts() (map[string]host.Host, error)
	AddJobs(req *host.AddJobsReq) (*host.AddJobsRes, error)
	ConnectHost(id string) (cluster.Host, error)
}

type formationStreamer interface {
	StreamFormations() (<-chan *ct.ExpandedFormation, *error)
}

func (c *context) watchFormations(fs formationStreamer) {
	ch, _ := fs.StreamFormations()

	for ef := range ch {
		f := NewFormation(c, ef)
		c.formations.Add(f)
		go f.Rectify()
	}

	// TODO: log disconnect and restart
	// TODO: trigger cluster sync
}

func (c *context) watchHost(id string) {
	if !c.hosts.Add(id) {
		return
	}
	defer c.hosts.Remove(id)

	h, err := c.ConnectHost(id)
	if err != nil {
		// TODO: log/handle error
	}
	c.hosts.Set(id, h)

	ch := make(chan *host.Event)
	h.StreamEvents("all", ch)
	for event := range ch {
		if event.Event != "error" && event.Event != "stop" {
			continue
		}
		job := c.jobs.Get(id, event.JobID)
		if job == nil {
			continue
		}

		c.jobs.Remove(id, event.JobID)
		go job.Formation.RemoveJob(job.Type, id, event.JobID)
	}
	// TODO: check error/reconnect
}

func newHostClients() *hostClients {
	return &hostClients{hosts: make(map[string]cluster.Host)}
}

type hostClients struct {
	hosts map[string]cluster.Host
	mtx   sync.RWMutex
}

func (h *hostClients) Add(id string) bool {
	h.mtx.Lock()
	defer h.mtx.Unlock()
	if _, exists := h.hosts[id]; exists {
		return false
	}
	h.hosts[id] = nil
	return true
}

func (h *hostClients) Set(id string, client cluster.Host) {
	h.mtx.Lock()
	h.hosts[id] = client
	h.mtx.Unlock()
}

func (h *hostClients) Remove(id string) {
	h.mtx.Lock()
	delete(h.hosts, id)
	h.mtx.Unlock()
}

func (h *hostClients) Get(id string) cluster.Host {
	h.mtx.RLock()
	defer h.mtx.RUnlock()
	return h.hosts[id]
}

func newJobMap() *jobMap {
	return &jobMap{jobs: make(map[jobKey]*Job)}
}

type jobMap struct {
	jobs map[jobKey]*Job
	mtx  sync.RWMutex
}

func (m *jobMap) Add(hostID, jobID string, job *Job) {
	m.mtx.Lock()
	m.jobs[jobKey{hostID, jobID}] = job
	m.mtx.Unlock()
}

func (m *jobMap) Remove(host, job string) {
	m.mtx.Lock()
	delete(m.jobs, jobKey{host, job})
	m.mtx.Unlock()
}

func (m *jobMap) Get(host, job string) *Job {
	m.mtx.RLock()
	defer m.mtx.RUnlock()
	return m.jobs[jobKey{host, job}]
}

type jobKey struct {
	hostID, jobID string
}

type formationKey struct {
	appID, releaseID string
}

func NewFormations() *Formations {
	return &Formations{formations: make(map[formationKey]*Formation)}
}

type Formations struct {
	formations map[formationKey]*Formation
	mtx        sync.RWMutex
}

func (fs *Formations) Get(appID, releaseID string) *Formation {
	fs.mtx.RLock()
	defer fs.mtx.RUnlock()
	return fs.formations[formationKey{appID, releaseID}]
}

func (fs *Formations) Add(f *Formation) {
	fs.mtx.Lock()
	fs.formations[f.key()] = f
	fs.mtx.Unlock()
}

func (fs *Formations) Delete(f *Formation) {
	fs.mtx.Lock()
	delete(fs.formations, f.key())
	fs.mtx.Unlock()
}

func NewFormation(c *context, ef *ct.ExpandedFormation) *Formation {
	return &Formation{
		App:       ef.App,
		Release:   ef.Release,
		Artifact:  ef.Artifact,
		Processes: ef.Processes,
		jobs:      make(jobTypeMap),
		c:         c,
	}
}

type Job struct {
	Type      string
	Formation *Formation
}

type jobTypeMap map[string]map[jobKey]*Job

func (m jobTypeMap) Add(typ, host, id string) *Job {
	jobs, ok := m[typ]
	if !ok {
		jobs = make(map[jobKey]*Job)
		m[typ] = jobs
	}
	job := &Job{Type: typ}
	jobs[jobKey{host, id}] = job
	return job
}

func (m jobTypeMap) Remove(typ, host, id string) {
	if jobs, ok := m[typ]; ok {
		delete(jobs, jobKey{host, id})
	}
}

func (m jobTypeMap) Get(typ, host, id string) *Job {
	return m[typ][jobKey{host, id}]
}

type Formation struct {
	mtx       sync.Mutex
	App       *ct.App
	Release   *ct.Release
	Artifact  *ct.Artifact
	Processes map[string]int

	jobs jobTypeMap
	c    *context
}

func (f *Formation) key() formationKey {
	return formationKey{f.App.ID, f.Release.ID}
}

func (f *Formation) Rectify() {
	f.mtx.Lock()
	defer f.mtx.Unlock()
	f.rectify()
}

func (f *Formation) RemoveJob(typ, hostID, jobID string) {
	f.mtx.Lock()
	defer f.mtx.Unlock()

	f.jobs.Remove(typ, hostID, jobID)
	f.rectify()
}

func (f *Formation) rectify() {
	// update job counts
	for t, expected := range f.Processes {
		diff := expected - len(f.jobs[t])
		if diff > 0 {
			f.add(diff, t)
		} else if diff < 0 {
			f.remove(-diff, t)
		}
	}

	// remove process types
	for t, jobs := range f.jobs {
		if _, exists := f.Processes[t]; !exists {
			f.remove(len(jobs), t)
		}
	}
}

func (f *Formation) add(n int, name string) {
	config, err := f.jobConfig(name)
	if err != nil {
		// TODO: log/handle error
	}
	for i := 0; i < n; i++ {
		hosts, err := f.c.ListHosts()
		if err != nil {
			// TODO: log/handle error
		}
		if len(hosts) == 0 {
			// TODO: log/handle error
		}
		hostCounts := make(map[string]int, len(hosts))
		for _, h := range hosts {
			hostCounts[h.ID] = 0
			for _, job := range h.Jobs {
				if f.jobType(job) != name {
					continue
				}
				hostCounts[h.ID]++
			}
		}
		sh := make(sortHosts, 0, len(hosts))
		for id, count := range hostCounts {
			sh = append(sh, sortHost{id, count})
		}
		sh.Sort()

		h := hosts[sh[0].ID]
		go f.c.watchHost(h.ID)

		job := f.jobs.Add(name, h.ID, config.ID)
		job.Formation = f
		f.c.jobs.Add(h.ID, config.ID, job)

		res, err := f.c.AddJobs(&host.AddJobsReq{HostJobs: map[string][]*host.Job{h.ID: {config}}})
		if err != nil || !res.Success {
			f.jobs.Remove(name, h.ID, config.ID)
			f.c.jobs.Remove(h.ID, config.ID)
			// TODO: log/handle error
		}
	}
}

func (f *Formation) jobType(job *host.Job) string {
	if job.Attributes["flynn-controller.app"] != f.App.ID ||
		job.Attributes["flynn-controller.release"] != f.Release.ID {
		return ""
	}
	return job.Attributes["flynn-controller.type"]
}

func (f *Formation) remove(n int, name string) {
	i := 0
	for k := range f.jobs[name] {
		// TODO: robust host handling
		if err := f.c.hosts.Get(k.hostID).StopJob(k.jobID); err != nil {
			// TODO: log/handle error
		}
		f.jobs.Remove(name, k.hostID, k.jobID)
		f.c.jobs.Remove(k.hostID, k.jobID)
		if i++; i == n {
			break
		}
	}
}

func (f *Formation) jobConfig(name string) (*host.Job, error) {
	t := f.Release.Processes[name]
	image, err := dockerImage(f.Artifact.URI)
	if err != nil {
		return nil, err
	}
	return &host.Job{
		ID:       cluster.RandomJobID(""),
		TCPPorts: t.Ports.TCP,
		Attributes: map[string]string{
			"flynn-controller.app":     f.App.ID,
			"flynn-controller.release": f.Release.ID,
			"flynn-controller.type":    name,
		},
		Config: &docker.Config{
			Cmd:   t.Cmd,
			Env:   formatEnv(f.Release.Env, t.Env),
			Image: image,
		},
	}, nil
}

func dockerImage(uri string) (string, error) {
	// TODO: ID refs (see https://github.com/dotcloud/docker/issues/4106)
	u, err := url.Parse(uri)
	if err != nil {
		return "", err
	}
	if u.Scheme != "docker" {
		return "", errors.New("scheduler: only docker artifact URIs are currently supported")
	}
	var suffix string
	if tag := u.Query().Get("tag"); tag != "" {
		suffix = ":" + tag
	}
	return u.Host + suffix, nil
}

func formatEnv(envs ...map[string]string) []string {
	env := make(map[string]string)
	for _, e := range envs {
		for k, v := range e {
			env[k] = v
		}
	}
	res := make([]string, 0, len(env))
	for k, v := range env {
		res = append(res, k+"="+v)
	}
	return res
}

type sortHost struct {
	ID   string
	Jobs int
}

type sortHosts []sortHost

func (h sortHosts) Len() int           { return len(h) }
func (h sortHosts) Less(i, j int) bool { return h[i].Jobs < h[j].Jobs }
func (h sortHosts) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
func (h sortHosts) Sort()              { sort.Sort(h) }
<|endoftext|>


<commit_before>package protocol

import (
	"bytes"
	"errors"
)

// ErrorList is a helper struct for collecting multiple errors.
type ErrorList struct {
	errors            []error
	descriptionPrefix string
}

func NewErrorList(descriptionPrefix string) *ErrorList {
	return &ErrorList{
		descriptionPrefix: descriptionPrefix,
		errors:            []error{},
	}

}

// Add adds an error.
func (l *ErrorList) Add(err error) {
	l.errors = append(l.errors, err)
}

// ErrorOrNil returns an error containing the information of all errors in the list,
// of nil if the list is empty.
func (l *ErrorList) ErrorOrNil() error {
	if len(l.errors) == 0 {
		return nil
	}
	buffer := bytes.Buffer{}
	buffer.WriteString(l.descriptionPrefix)
	for i, err := range l.errors {
		buffer.WriteString(err.Error())
		if i+1 < len(l.errors) {
			buffer.WriteString("; ")
		}
	}
	return errors.New(buffer.String())
}
<commit_msg>implementing Error() function of error interface<commit_after>package protocol

import (
	"bytes"
	"errors"
)

// ErrorList is a helper struct for collecting multiple errors.
type ErrorList struct {
	errors            []error
	descriptionPrefix string
}

func NewErrorList(descriptionPrefix string) *ErrorList {
	return &ErrorList{
		descriptionPrefix: descriptionPrefix,
		errors:            []error{},
	}
}

// Add adds an error.
func (l *ErrorList) Add(err error) {
	l.errors = append(l.errors, err)
}

// ErrorOrNil returns an error containing the information of all errors in the list,
// of nil if the list is empty.
func (l *ErrorList) ErrorOrNil() error {
	if len(l.errors) == 0 {
		return nil
	}
	return errors.New(l.Error())
}

func (l *ErrorList) Error() string {
	if len(l.errors) == 0 {
		return ""
	}
	buffer := bytes.Buffer{}
	buffer.WriteString(l.descriptionPrefix)
	for i, err := range l.errors {
		buffer.WriteString(err.Error())
		if i+1 < len(l.errors) {
			buffer.WriteString("; ")
		}
	}
	return buffer.String()
}
<|endoftext|>


<commit_before>package eventbus

import (
	"encoding/json"
	"log"
	"testing"
	"time"
)

type message struct {
	Name string
}

func TestEventBusEmit(t *testing.T) {
	bus, err := NewEventBus()
	if err != nil {
		t.Errorf("Expected to initialize EventBus %s", err)
	}

	if err := bus.Emit("topic", &message{Name: "event"}); err != nil {
		t.Errorf("Expected to emit message %s", err)
	}
}

func TestEventBusOn(t *testing.T) {
	bus, err := NewEventBus()
	if err != nil {
		t.Errorf("Expected to initialize EventBus %s", err)
	}

	testHandler := func(msg []byte) error {
		log.Print("on handler")
		m := message{}
		if err := json.Unmarshal(msg, &m); err != nil {
			t.Errorf("Expected to unmarshal a message %s", err)
		}

		return nil
	}

	if err := bus.On("topic", "channel", testHandler); err != nil {
		t.Errorf("Expected to listen a message %s", err)
	}

	time.Sleep(200 * time.Millisecond)

	if err := bus.Emit("topic", &message{Name: "event"}); err != nil {
		t.Errorf("Expected to emit message %s", err)
	}

	time.Sleep(200 * time.Millisecond)
}
<commit_msg>Remove extra log<commit_after>package eventbus

import (
	"encoding/json"
	"testing"
	"time"
)

type message struct {
	Name string
}

func TestEventBusEmit(t *testing.T) {
	bus, err := NewEventBus()
	if err != nil {
		t.Errorf("Expected to initialize EventBus %s", err)
	}

	if err := bus.Emit("topic", &message{Name: "event"}); err != nil {
		t.Errorf("Expected to emit message %s", err)
	}
}

func TestEventBusOn(t *testing.T) {
	bus, err := NewEventBus()
	if err != nil {
		t.Errorf("Expected to initialize EventBus %s", err)
	}

	testHandler := func(msg []byte) error {
		m := message{}
		if err := json.Unmarshal(msg, &m); err != nil {
			t.Errorf("Expected to unmarshal a message %s", err)
		}

		return nil
	}

	if err := bus.On("topic", "channel", testHandler); err != nil {
		t.Errorf("Expected to listen a message %s", err)
	}

	time.Sleep(200 * time.Millisecond)

	if err := bus.Emit("topic", &message{Name: "event"}); err != nil {
		t.Errorf("Expected to emit message %s", err)
	}

	time.Sleep(200 * time.Millisecond)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/signal"
	"syscall"
	"time"
)

type scheduler struct {
	t    <-chan time.Time
	quit chan struct{}
	f    func()
}

func main() {

	scheduler := scheduler{
		t:    time.NewTicker(time.Second * 1).C,
		quit: make(chan struct{}),
		f: func() {
			start := time.Now()
			fmt.Printf("Start: %q ", start.Format(time.RFC3339))
			time.Sleep(time.Second * 4)
			fmt.Printf("Elapsed: %q\n", time.Since(start))
		},
	}

	go func() {
		for {
			select {
			case <-scheduler.t:
				scheduler.f()
			case <-scheduler.quit:
				return
			}
		}
	}()

	// exit on signal
	block := make(chan os.Signal, 1)
	signal.Notify(block, os.Interrupt, os.Kill, syscall.SIGTERM)
	signalType := <-block
	fmt.Printf("%q signal received.", signalType)
	signal.Stop(block)
	os.Exit(0)
}
<commit_msg>	modified:   main.go<commit_after>package main

import (
	"fmt"
	"os"
	"os/signal"
	"syscall"
	"time"
)

type scheduler struct {
	t    <-chan time.Time
	quit chan struct{}
	f    func()
}

func myFunc() func() {
	return func() {
		start := time.Now()
		fmt.Printf("Start: %q ", start.Format(time.RFC3339))
		time.Sleep(time.Second * 4)
		fmt.Printf("Elapsed: %q\n", time.Since(start))
	}
}

func main() {

	scheduler := scheduler{
		t:    time.NewTicker(time.Second * 1).C,
		quit: make(chan struct{}),
		f:    myFunc(),
	}

	go func() {
		for {
			select {
			case <-scheduler.t:
				scheduler.f()
			case <-scheduler.quit:
				return
			}
		}
	}()

	// exit on signal
	block := make(chan os.Signal, 1)
	signal.Notify(block, os.Interrupt, os.Kill, syscall.SIGTERM)
	signalType := <-block
	fmt.Printf("%q signal received.", signalType)
	signal.Stop(block)
	os.Exit(0)
}
<|endoftext|>


<commit_before>package drivers

// Info represents information about a storage driver.
type Info struct {
	Name                  string
	Version               string
	VolumeTypes           []VolumeType // Supported volume types.
	Remote                bool         // Whether the driver uses a remote backing store.
	OptimizedImages       bool         // Whether driver stores images as separate volume.
	PreservesInodes       bool         // Whether driver preserves inodes when volumes are moved hosts.
	BlockBacking          bool         // Whether driver uses block devices as backing store.
	RunningQuotaResize    bool         // Whether quota resize is supported whilst instance running.
	RunningSnapshotFreeze bool         // Whether instance should be frozen during snapshot if running.
}

// VolumeFiller provides a struct for filling a volume.
type VolumeFiller struct {
	Fill func(mountPath, rootBlockPath string) error // Function to fill the volume.

	Fingerprint string // If the Filler will unpack an image, it should be this fingerprint.
}
<commit_msg>lxd/storage/drivers/driver/types: Adds DirectIO indicator to driver info struct<commit_after>package drivers

// Info represents information about a storage driver.
type Info struct {
	Name                  string
	Version               string
	VolumeTypes           []VolumeType // Supported volume types.
	Remote                bool         // Whether the driver uses a remote backing store.
	OptimizedImages       bool         // Whether driver stores images as separate volume.
	PreservesInodes       bool         // Whether driver preserves inodes when volumes are moved hosts.
	BlockBacking          bool         // Whether driver uses block devices as backing store.
	RunningQuotaResize    bool         // Whether quota resize is supported whilst instance running.
	RunningSnapshotFreeze bool         // Whether instance should be frozen during snapshot if running.
	DirectIO              bool         // Whether the driver supports direct I/O.
}

// VolumeFiller provides a struct for filling a volume.
type VolumeFiller struct {
	Fill func(mountPath, rootBlockPath string) error // Function to fill the volume.

	Fingerprint string // If the Filler will unpack an image, it should be this fingerprint.
}
<|endoftext|>


<commit_before>package gpool

import (
	"errors"
	"fmt"
	"runtime"
	"sync"
	"testing"
	"time"
)

var errTestError = errors.New("test error")

var failJob = func(c chan bool) (interface{}, error) {
	return nil, errTestError
}

var goodJob = func(c chan bool) (interface{}, error) {
	time.Sleep(time.Second / 4)
	return nil, nil
}

func Test_Pool_Wait(t *testing.T) {
	p := NewPool(1)
	ok := make(chan bool)
	go func() {
		p.Wait()
		close(ok)
	}()
	p.Kill()
	p.Wait()
	p.Wait()
	<-ok
}

func Test_Propagated_Pool_Jobs(t *testing.T) {
	p := NewPool(1)
	defer p.Destroy()
	p.Send(NewJob(Identifier("Testing"), failJob))
	e := p.Close()
	if e != nil {
		t.Fatal(e)
	}
	e = p.Wait()
	if e == nil {
		t.Fatal("Nil error")
	} else if RealError(e) != errTestError {
		t.Fatalf("wrong error want %#v, got %#v", errTestError, RealError(e))
	}
	t.Log(e)
	if len(p.Jobs(Finished)) > 0 {
		t.Fatal("Job present in Finished jobs")
	}
	if len(p.Jobs(Failed)) != 1 {
		t.Fatal("Job not present in Failed jobs")
	}
}

func Test_NonPropagated_Pool_Jobs(t *testing.T) {
	p := NewNonPropagatingPool(1)
	defer p.Destroy()
	p.Send(NewJob(Identifier("Testing"), failJob))
	e := p.Close()
	if e != nil {
		t.Fatal(e)
	}
	e = p.Wait()
	if e != nil {
		t.Fatal("unexpected error")
	}
	if len(p.Jobs(Finished)) > 0 {
		t.Fatal("Job present in Finished jobs")
	}
	fj := p.Jobs(Failed)
	if len(fj) != 1 {
		t.Fatal("Job not present in Failed jobs")
	}
	if RealError(fj[0].Error) != errTestError {
		t.Fatal("expected test error, got ", RealError(fj[0].Error))
	}
}

func Test_Pool_Load(t *testing.T) {
	p := NewPool(runtime.NumCPU())
	defer p.Destroy()
	const wrks = 1000000
	for i := 0; i < wrks; i++ {
		p.Send(NewJob(Identifier("Testing"), func(c chan bool) (interface{}, error) {
			return nil, nil
		}))
	}
	e := p.Close()
	if e != nil {
		t.Fatal(e)
	}
	e = p.Wait()
	if e != nil {
		t.Fatal(e)
	}
	s := p.Jobs(Finished)
	if len(s) != wrks {
		t.Fatal("not enough jobs, wanted ", wrks, " got", len(s))
	}
}

func Test_PoolError(t *testing.T) {
	p := NewPool(2)
	defer p.Destroy()
	for range make([]int, 40) {
		p.Send(NewJob(Identifier("Testing"), failJob))
	}
	p.Close()
	e := p.Wait()
	if err, ok := e.(PoolError); !ok {
		t.Fatal("error is not a pool error")
	} else {
		if err.E != errTestError {
			t.Fatal(err.E)
		}
	}
	if l := len(p.Jobs(Failed)); l != 1 {
		t.Fatal("wanted 1 failed job, got ", l)
	}
}

func Test_Pool_Send_Serial(t *testing.T) {
	p := NewPool(1)
	defer p.Destroy()
	for range make([]int, 20) {
		p.Send(NewJob(Identifier("Testing"), goodJob))
	}
	e := p.Close()
	if e != nil {
		t.Fatal(e)
	}
	e = p.Wait()
	if e != nil {
		t.Fatal(e)
	}
	s := p.Jobs(Finished)
	if len(s) != 20 {
		t.Fatal("not enough jobs, wanted 20 got", len(s))
	}
}

func Test_Pool_Send_Concurrent(t *testing.T) {
	p := NewPool(1)
	defer p.Destroy()
	wg := &sync.WaitGroup{}
	for range make([]int, 20) {
		wg.Add(1)
		go func() {
			defer wg.Done()
			p.Send(NewJob(Identifier("Testing"), goodJob))
		}()
	}
	wg.Wait()
	e := p.Close()
	if e != nil {
		t.Fatal(e)
	}
	e = p.Wait()
	if e != nil {
		t.Fatal(e)
	}
	s := p.Jobs(Finished)
	if len(s) != 20 {
		t.Fatal("not enough jobs, wanted 20 got", len(s))
	}
}

func Test_Pool_Healthy(t *testing.T) {
	p := NewPool(1)
	defer p.Destroy()
	if ok := p.Healthy(); !ok {
		t.Fatal("pool unexpectedly closed")
	}
	e := p.Close()
	if e != nil {
		t.Fatal(e)
	}
	if ok := p.Healthy(); ok {
		t.Fatal("pool not closed")
	}
}

func Test_Pool_Error(t *testing.T) {
	p := NewPool(1)
	defer p.Destroy()
	e := p.Send(NewJob(Identifier("Testing"), failJob))
	if e != nil {
		t.Fatal(e)
	}
	e = p.Close()
	if e != nil {
		t.Fatal(e)
	}
	e = p.Wait()
	if e == nil {
		t.Fatal("expected error")
	}

	if e := p.Error(); e == nil {
		t.Fatal("no pool error")
	}

}

func Test_Pool_Kill(t *testing.T) {
	p := NewPool(1)
	defer p.Destroy()
	cancelled := make(chan bool)
	p.Send(NewJob(Identifier("Testing"), func(c chan bool) (interface{}, error) {
		<-c
		close(cancelled)
		return nil, nil
	}))
	p.Kill()
	select {
	case <-cancelled:
		return
	case <-time.After(2 * time.Second):
		t.Fatal("no job response after 2 seconds")
	}
	e := p.Close()
	if e != nil {
		t.Fatal("expected error, got", e)
	}
	e = p.Wait()
	if e != ErrKilled {
		t.Fatal("expected ErrKilled, got", e)
	}
}

//
func Test_Pool_Grow(t *testing.T) {
	p := NewPool(2)
	defer p.Destroy()
	if c := p.Workers(); c != 2 {
		t.Fatal("wanted 2 workers, got", c)
	}
	e := p.Grow(2)
	if e != nil {
		t.Fatal(e)
	}
	if c := p.Workers(); c != 4 {
		t.Fatal("wanted 4 workers, got", c)
	}
	p.Kill()
	p.Wait()
}

//
//func Test_Pool_Shrink_Neg(t *testing.T) {
//	p := NewPool(4)
//	defer p.Destroy()
//	if c, _ := p.mgr.workers(); c != 4 {
//		t.Fatal("wanted 4 workers, got", c)
//	}
//	e := p.Shrink(4)
//	if e != ErrWorkerCount {
//		t.Fatal("wanted ErrWorkerCount, got", e)
//	}
//	p.Kill()
//	p.Wait()
//}
//
//func Test_Pool_Resize(t *testing.T) {
//	p := NewPool(1)
//	defer p.Destroy()
//	e := p.Resize(5)
//	if e != nil {
//		t.Fatal(e)
//	}
//
//	if c, _ := p.mgr.workers(); c != 5 {
//		t.Fatal("worker state incorrect, wanted 5, got", c)
//	}
//	p.Kill()
//	p.Wait()
//}

func Test_Pool_State(t *testing.T) {
	p := NewPool(1)
	defer p.Destroy()
	ok := make(chan bool)

	job := NewJob(Identifier("Testing"), func(c chan bool) (interface{}, error) {
		select {
		case <-ok:
		case <-c:
		}

		return nil, nil
	})
	e := p.Send(job)
	if e != nil {
		t.Fatal(e)
	}

	s := p.Jobs("")

	close(ok)

	if len(s) != 1 {
		t.Fatalf("expected 1 job, got %s", len(s))
	}
	if s[0].State != Executing {
		t.Fatalf("expected Executing, got %s", s[0].State)
	}
	p.Kill()
	p.Wait()
}

//func Test_Pool_NRunning(t *testing.T) {
//	p := NewPool(2)
//	defer p.Destroy()
//	if c, _ := p.mgr.workers(); c != 2 {
//		t.Fatal("wanted 2 workers, got", c)
//	}
//	p.Kill()
//	p.Wait()
//}

func Example() {
	// Create a Pool with 5 workers
	p := NewPool(5)

	// Example PoolJobFn.
	// After 10 seconds the job will return Hello, World!
	JobFn := func(c chan bool) (interface{}, error) {
		<-time.After(10 * time.Second)
		return "Hello, World!", nil
	}
	// Create a Job with an Identifier
	Job := NewJob(
		Identifier("MyPoolJob"), JobFn,
	)
	// Send it to the Pool
	p.Send(Job)

	// Close the pool after all messages are sent
	p.Close()

	// Wait for the pool to finish
	e := p.Wait()
	if e != nil {
		// Do something with errors here
	}

	// Iterate over jobs that have finished and print the output
	for _, j := range p.Jobs(Finished) {
		if s, ok := j.Output.(string); ok {
			fmt.Println(s) // Outputs: Hello, World!
		}
	}
}

func doBenchMarkSubmit(b *testing.B, Workers int, N int) {
	p := NewPool(Workers)
	defer p.Wait()
	j := NewJob(Identifier("Benchmark"), func(c chan bool) (interface{}, error) {
		return nil, nil
	})
	b.ResetTimer()
	for i := 0; i < N; i++ {
		e := p.Send(j)
		if e != nil {
			b.Fatal(e)
		}
	}
	p.Close()
	if err := p.Wait(); err != nil {
		b.Fatal(err)
	}
}

func BenchmarkSubmit_1(b *testing.B) {
	doBenchMarkSubmit(b, 1, b.N)
}

func BenchmarkSubmit_10(b *testing.B) {
	doBenchMarkSubmit(b, 10, b.N)
}

func BenchmarkSubmit_100(b *testing.B) {
	doBenchMarkSubmit(b, 100, b.N)
}
<commit_msg>cannot guarantee failed job count<commit_after>package gpool

import (
	"errors"
	"fmt"
	"runtime"
	"sync"
	"testing"
	"time"
)

var errTestError = errors.New("test error")

var failJob = func(c chan bool) (interface{}, error) {
	return nil, errTestError
}

var goodJob = func(c chan bool) (interface{}, error) {
	time.Sleep(time.Second / 4)
	return nil, nil
}

func Test_Pool_Wait(t *testing.T) {
	p := NewPool(1)
	ok := make(chan bool)
	go func() {
		p.Wait()
		close(ok)
	}()
	p.Kill()
	p.Wait()
	p.Wait()
	<-ok
}

func Test_Propagated_Pool_Jobs(t *testing.T) {
	p := NewPool(1)
	defer p.Destroy()
	p.Send(NewJob(Identifier("Testing"), failJob))
	e := p.Close()
	if e != nil {
		t.Fatal(e)
	}
	e = p.Wait()
	if e == nil {
		t.Fatal("Nil error")
	} else if RealError(e) != errTestError {
		t.Fatalf("wrong error want %#v, got %#v", errTestError, RealError(e))
	}
	t.Log(e)
	if len(p.Jobs(Finished)) > 0 {
		t.Fatal("Job present in Finished jobs")
	}
	if len(p.Jobs(Failed)) != 1 {
		t.Fatal("Job not present in Failed jobs")
	}
}

func Test_NonPropagated_Pool_Jobs(t *testing.T) {
	p := NewNonPropagatingPool(1)
	defer p.Destroy()
	p.Send(NewJob(Identifier("Testing"), failJob))
	e := p.Close()
	if e != nil {
		t.Fatal(e)
	}
	e = p.Wait()
	if e != nil {
		t.Fatal("unexpected error")
	}
	if len(p.Jobs(Finished)) > 0 {
		t.Fatal("Job present in Finished jobs")
	}
	fj := p.Jobs(Failed)
	if len(fj) != 1 {
		t.Fatal("Job not present in Failed jobs")
	}
	if RealError(fj[0].Error) != errTestError {
		t.Fatal("expected test error, got ", RealError(fj[0].Error))
	}
}

func Test_Pool_Load(t *testing.T) {
	p := NewPool(runtime.NumCPU())
	defer p.Destroy()
	const wrks = 1000000
	for i := 0; i < wrks; i++ {
		p.Send(NewJob(Identifier("Testing"), func(c chan bool) (interface{}, error) {
			return nil, nil
		}))
	}
	e := p.Close()
	if e != nil {
		t.Fatal(e)
	}
	e = p.Wait()
	if e != nil {
		t.Fatal(e)
	}
	s := p.Jobs(Finished)
	if len(s) != wrks {
		t.Fatal("not enough jobs, wanted ", wrks, " got", len(s))
	}
}

func Test_PoolError(t *testing.T) {
	p := NewPool(2)
	defer p.Destroy()
	p.Send(NewJob(Identifier("Testing"), failJob))
	p.Close()
	e := p.Wait()
	if err, ok := e.(PoolError); !ok {
		t.Fatal("error is not a pool error")
	} else {
		if err.E != errTestError {
			t.Fatal(err.E)
		}
	}
	if l := len(p.Jobs(Failed)); l != 1 {
		t.Fatal("wanted 1 failed job, got ", l)
	}
}

func Test_Pool_Send_Serial(t *testing.T) {
	p := NewPool(1)
	defer p.Destroy()
	for range make([]int, 20) {
		p.Send(NewJob(Identifier("Testing"), goodJob))
	}
	e := p.Close()
	if e != nil {
		t.Fatal(e)
	}
	e = p.Wait()
	if e != nil {
		t.Fatal(e)
	}
	s := p.Jobs(Finished)
	if len(s) != 20 {
		t.Fatal("not enough jobs, wanted 20 got", len(s))
	}
}

func Test_Pool_Send_Concurrent(t *testing.T) {
	p := NewPool(1)
	defer p.Destroy()
	wg := &sync.WaitGroup{}
	for range make([]int, 20) {
		wg.Add(1)
		go func() {
			defer wg.Done()
			p.Send(NewJob(Identifier("Testing"), goodJob))
		}()
	}
	wg.Wait()
	e := p.Close()
	if e != nil {
		t.Fatal(e)
	}
	e = p.Wait()
	if e != nil {
		t.Fatal(e)
	}
	s := p.Jobs(Finished)
	if len(s) != 20 {
		t.Fatal("not enough jobs, wanted 20 got", len(s))
	}
}

func Test_Pool_Healthy(t *testing.T) {
	p := NewPool(1)
	defer p.Destroy()
	if ok := p.Healthy(); !ok {
		t.Fatal("pool unexpectedly closed")
	}
	e := p.Close()
	if e != nil {
		t.Fatal(e)
	}
	if ok := p.Healthy(); ok {
		t.Fatal("pool not closed")
	}
}

func Test_Pool_Error(t *testing.T) {
	p := NewPool(1)
	defer p.Destroy()
	e := p.Send(NewJob(Identifier("Testing"), failJob))
	if e != nil {
		t.Fatal(e)
	}
	e = p.Close()
	if e != nil {
		t.Fatal(e)
	}
	e = p.Wait()
	if e == nil {
		t.Fatal("expected error")
	}

	if e := p.Error(); e == nil {
		t.Fatal("no pool error")
	}

}

func Test_Pool_Kill(t *testing.T) {
	p := NewPool(1)
	defer p.Destroy()
	cancelled := make(chan bool)
	p.Send(NewJob(Identifier("Testing"), func(c chan bool) (interface{}, error) {
		<-c
		close(cancelled)
		return nil, nil
	}))
	p.Kill()
	select {
	case <-cancelled:
		return
	case <-time.After(2 * time.Second):
		t.Fatal("no job response after 2 seconds")
	}
	e := p.Close()
	if e != nil {
		t.Fatal("expected error, got", e)
	}
	e = p.Wait()
	if e != ErrKilled {
		t.Fatal("expected ErrKilled, got", e)
	}
}

//
func Test_Pool_Grow(t *testing.T) {
	p := NewPool(2)
	defer p.Destroy()
	if c := p.Workers(); c != 2 {
		t.Fatal("wanted 2 workers, got", c)
	}
	e := p.Grow(2)
	if e != nil {
		t.Fatal(e)
	}
	if c := p.Workers(); c != 4 {
		t.Fatal("wanted 4 workers, got", c)
	}
	p.Kill()
	p.Wait()
}

//
//func Test_Pool_Shrink_Neg(t *testing.T) {
//	p := NewPool(4)
//	defer p.Destroy()
//	if c, _ := p.mgr.workers(); c != 4 {
//		t.Fatal("wanted 4 workers, got", c)
//	}
//	e := p.Shrink(4)
//	if e != ErrWorkerCount {
//		t.Fatal("wanted ErrWorkerCount, got", e)
//	}
//	p.Kill()
//	p.Wait()
//}
//
//func Test_Pool_Resize(t *testing.T) {
//	p := NewPool(1)
//	defer p.Destroy()
//	e := p.Resize(5)
//	if e != nil {
//		t.Fatal(e)
//	}
//
//	if c, _ := p.mgr.workers(); c != 5 {
//		t.Fatal("worker state incorrect, wanted 5, got", c)
//	}
//	p.Kill()
//	p.Wait()
//}

func Test_Pool_State(t *testing.T) {
	p := NewPool(1)
	defer p.Destroy()
	ok := make(chan bool)

	job := NewJob(Identifier("Testing"), func(c chan bool) (interface{}, error) {
		select {
		case <-ok:
		case <-c:
		}

		return nil, nil
	})
	e := p.Send(job)
	if e != nil {
		t.Fatal(e)
	}

	s := p.Jobs("")

	close(ok)

	if len(s) != 1 {
		t.Fatalf("expected 1 job, got %s", len(s))
	}
	if s[0].State != Executing {
		t.Fatalf("expected Executing, got %s", s[0].State)
	}
	p.Kill()
	p.Wait()
}

//func Test_Pool_NRunning(t *testing.T) {
//	p := NewPool(2)
//	defer p.Destroy()
//	if c, _ := p.mgr.workers(); c != 2 {
//		t.Fatal("wanted 2 workers, got", c)
//	}
//	p.Kill()
//	p.Wait()
//}

func Example() {
	// Create a Pool with 5 workers
	p := NewPool(5)

	// Example PoolJobFn.
	// After 10 seconds the job will return Hello, World!
	JobFn := func(c chan bool) (interface{}, error) {
		<-time.After(10 * time.Second)
		return "Hello, World!", nil
	}
	// Create a Job with an Identifier
	Job := NewJob(
		Identifier("MyPoolJob"), JobFn,
	)
	// Send it to the Pool
	p.Send(Job)

	// Close the pool after all messages are sent
	p.Close()

	// Wait for the pool to finish
	e := p.Wait()
	if e != nil {
		// Do something with errors here
	}

	// Iterate over jobs that have finished and print the output
	for _, j := range p.Jobs(Finished) {
		if s, ok := j.Output.(string); ok {
			fmt.Println(s) // Outputs: Hello, World!
		}
	}
}

func doBenchMarkSubmit(b *testing.B, Workers int, N int) {
	p := NewPool(Workers)
	defer p.Wait()
	j := NewJob(Identifier("Benchmark"), func(c chan bool) (interface{}, error) {
		return nil, nil
	})
	b.ResetTimer()
	for i := 0; i < N; i++ {
		e := p.Send(j)
		if e != nil {
			b.Fatal(e)
		}
	}
	p.Close()
	if err := p.Wait(); err != nil {
		b.Fatal(err)
	}
}

func BenchmarkSubmit_1(b *testing.B) {
	doBenchMarkSubmit(b, 1, b.N)
}

func BenchmarkSubmit_10(b *testing.B) {
	doBenchMarkSubmit(b, 10, b.N)
}

func BenchmarkSubmit_100(b *testing.B) {
	doBenchMarkSubmit(b, 100, b.N)
}
<|endoftext|>


<commit_before>package controllers

import (
	"net/http"
	"encoding/json"
	"github.com/wandi34/wallets-as-a-service/backend/common"
	"fmt"
	"github.com/blockcypher/gobcy"
	"github.com/wandi34/wallets-as-a-service/backend/data"
	"github.com/wandi34/wallets-as-a-service/backend/models"
	"gopkg.in/mgo.v2/bson"
	"strconv"
	"bytes"
	"errors"
)

func CreateTransaction(w http.ResponseWriter, r *http.Request) {
	var dataResource CreateTransactionResource
	// Decode the incoming Transaction json
	err := json.NewDecoder(r.Body).Decode(&dataResource)
	if err != nil {
		common.DisplayAppError(
			w,
			err,
			"Invalid body",
			500,
		)
		return
	}
	sourceAddress := dataResource.Data.SourceAddress
	targetAddress := dataResource.Data.TargetAddress
	amount, err := strconv.Atoi(dataResource.Data.Amount)
	//Post New TXSkeleton
	skel, err := bcy.NewTX(gobcy.TempNewTX(sourceAddress, targetAddress, amount), false)
	if err != nil {
		common.DisplayAppError(w, err, "Tx Error", 400)
	}
	//Sign it locally
	context := NewContext()
	defer context.Close()
	col := context.DbCollection("accounts")
	repo := &data.AccountRepository{C: col}
	// Authenticate the login user
	result := models.Account{}
	err = repo.C.Find(bson.M{"wallet.address": sourceAddress}).One(&result)
	fmt.Println(len(skel.ToSign))
	// Decrypt private key
	password := dataResource.Data.Password
	if !bytes.Equal(common.GetMd5Hash(password), result.PwHash) {
		err := errors.New("wrong password")
		common.DisplayAppError(w, err, "Given password is wrong", 400)
		return
	}
	privateKey, _ := common.Decrypt(common.GetMd5Hash(dataResource.Data.Password), []byte(result.Wallet.Private))

	//Sign all open transactions with private key
	var signingKeys []string
	for i := 0;i < len(skel.ToSign);i++{
		signingKeys = append(signingKeys, string(privateKey[:]))
	}
	err = skel.Sign(signingKeys)
	if err != nil {
		common.DisplayAppError(w, err, "Signing Tx Error", 400)
	}
	//Send TXSkeleton
	skel, err = bcy.SendTX(skel)
	if err != nil {
		common.DisplayAppError(w, err, "Sending Tx Error", 400)
	}
	fmt.Printf("%+v\n", skel)

	j, _ := json.Marshal(skel)
	w.Write(j)
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusCreated)
}

<commit_msg>Fix multiple write header calls<commit_after>package controllers

import (
	"net/http"
	"encoding/json"
	"github.com/wandi34/wallets-as-a-service/backend/common"
	"fmt"
	"github.com/blockcypher/gobcy"
	"github.com/wandi34/wallets-as-a-service/backend/data"
	"github.com/wandi34/wallets-as-a-service/backend/models"
	"gopkg.in/mgo.v2/bson"
	"strconv"
	"bytes"
	"errors"
)

func CreateTransaction(w http.ResponseWriter, r *http.Request) {
	var dataResource CreateTransactionResource
	// Decode the incoming Transaction json
	err := json.NewDecoder(r.Body).Decode(&dataResource)
	if err != nil {
		common.DisplayAppError(
			w,
			err,
			"Invalid body",
			500,
		)
		return
	}
	sourceAddress := dataResource.Data.SourceAddress
	targetAddress := dataResource.Data.TargetAddress
	amount, err := strconv.Atoi(dataResource.Data.Amount)
	//Post New TXSkeleton
	skel, err := bcy.NewTX(gobcy.TempNewTX(sourceAddress, targetAddress, amount), false)
	if err != nil {
		common.DisplayAppError(w, err, "Tx Error", 400)
		return
	}
	//Sign it locally
	context := NewContext()
	defer context.Close()
	col := context.DbCollection("accounts")
	repo := &data.AccountRepository{C: col}
	// Authenticate the login user
	result := models.Account{}
	err = repo.C.Find(bson.M{"wallet.address": sourceAddress}).One(&result)
	fmt.Println(len(skel.ToSign))
	// Decrypt private key
	password := dataResource.Data.Password
	if !bytes.Equal(common.GetMd5Hash(password), result.PwHash) {
		err := errors.New("wrong password")
		common.DisplayAppError(w, err, "Given password is wrong", 400)
		return
	}
	privateKey, _ := common.Decrypt(common.GetMd5Hash(dataResource.Data.Password), []byte(result.Wallet.Private))

	//Sign all open transactions with private key
	var signingKeys []string
	for i := 0;i < len(skel.ToSign);i++{
		signingKeys = append(signingKeys, string(privateKey[:]))
	}
	err = skel.Sign(signingKeys)
	if err != nil {
		common.DisplayAppError(w, err, "Signing Tx Error", 400)
		return
	}
	//Send TXSkeleton
	skel, err = bcy.SendTX(skel)
	if err != nil {
		common.DisplayAppError(w, err, "Sending Tx Error", 400)
		return
	}
	fmt.Printf("%+v\n", skel)

	j, _ := json.Marshal(skel)
	w.Write(j)
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusCreated)
}

<|endoftext|>


<commit_before>package utils

import (
	"fmt"
	"log"

	"github.com/matcornic/subify/common/config"
	logger "github.com/spf13/jwalterweatherman"
)

// VerbosePrintln only prints log if verbose mode is enabled
func VerbosePrintln(logger *log.Logger, log string) {
	if config.Verbose && log != "" {
		logger.Println(log)
	}
}

// Exit exits the application and logs the given message
func Exit(format string, args ...interface{}) {
	ExitVerbose("", format, args...)
}

// ExitPrintError displays an error message on stderr and exit 1
// Eventually prints more details about the error if verbose mode is enabled
func ExitPrintError(err error, format string, args ...interface{}) {
	ExitVerbose(fmt.Sprint(err), format, args...)
}

// ExitVerbose displays an error message on stderr and exit 1
// Eventually prints more details if any verbose details are given and verbose mode is enabled
func ExitVerbose(verboseLog string, format string, args ...interface{}) {
	VerbosePrintln(logger.ERROR, verboseLog)
	if !config.Verbose {
		logger.ERROR.Println("Run subify with --verbose option to get more information about the error")
	}
	logger.FATAL.Printf(format)
}
<commit_msg>Exits the program with an error code != 0 when using utils exit function<commit_after>package utils

import (
	"fmt"
	"log"
	"os"

	"github.com/matcornic/subify/common/config"
	logger "github.com/spf13/jwalterweatherman"
)

// VerbosePrintln only prints log if verbose mode is enabled
func VerbosePrintln(logger *log.Logger, log string) {
	if config.Verbose && log != "" {
		logger.Println(log)
	}
}

// Exit exits the application and logs the given message
func Exit(format string, args ...interface{}) {
	ExitVerbose("", format, args...)
}

// ExitPrintError displays an error message on stderr and exit 1
// Eventually prints more details about the error if verbose mode is enabled
func ExitPrintError(err error, format string, args ...interface{}) {
	ExitVerbose(fmt.Sprint(err), format, args...)
}

// ExitVerbose displays an error message on stderr and exit 1
// Eventually prints more details if any verbose details are given and verbose mode is enabled
func ExitVerbose(verboseLog string, format string, args ...interface{}) {
	VerbosePrintln(logger.ERROR, verboseLog)
	if !config.Verbose {
		logger.ERROR.Println("Run subify with --verbose option to get more information about the error")
	}
	logger.FATAL.Printf(format)
	os.Exit(-1)
}
<|endoftext|>


<commit_before>package fs

import (
	"encoding/base64"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"reflect"
	"testing"

	"github.com/koding/kite"
	"github.com/koding/kite/dnode"
)

var (
	fs     *kite.Kite
	remote *kite.Client
)

func init() {
	fs = kite.New("fs", "0.0.1")
	fs.Config.DisableAuthentication = true
	fs.Config.Port = 3636
	fs.HandleFunc("readDirectory", ReadDirectory)
	fs.HandleFunc("glob", Glob)
	fs.HandleFunc("readFile", ReadFile)
	fs.HandleFunc("writeFile", WriteFile)
	fs.HandleFunc("uniquePath", UniquePath)
	fs.HandleFunc("getInfo", GetInfo)
	fs.HandleFunc("setPermissions", SetPermissions)
	fs.HandleFunc("remove", Remove)
	fs.HandleFunc("rename", Rename)
	fs.HandleFunc("createDirectory", CreateDirectory)
	fs.HandleFunc("move", Move)
	fs.HandleFunc("copy", Copy)

	go fs.Run()
	<-fs.ServerReadyNotify()

	client := kite.New("client", "0.0.1")
	client.Config.DisableAuthentication = true
	remote = client.NewClientString("ws://127.0.0.1:3636")
	err := remote.Dial()
	if err != nil {
		log.Fatal("err")
	}
}

func TestReadDirectory(t *testing.T) {
	testDir := "."

	files, err := ioutil.ReadDir(testDir)
	if err != nil {
		t.Fatal(err)
	}

	currentFiles := make([]string, len(files))
	for i, f := range files {
		currentFiles[i] = f.Name()
	}

	resp, err := remote.Tell("readDirectory", struct {
		Path     string
		OnChange dnode.Function
	}{
		Path:     testDir,
		OnChange: dnode.Function{},
	})

	if err != nil {
		t.Fatal(err)
	}

	f, err := resp.Map()
	if err != nil {
		t.Fatal(err)
	}

	entries, err := f["files"].SliceOfLength(len(files))
	if err != nil {
		t.Fatal(err)
	}

	respFiles := make([]string, len(files))
	for i, e := range entries {
		f := &FileEntry{}
		err := e.Unmarshal(f)
		if err != nil {
			t.Fatal(err)
		}

		respFiles[i] = f.Name
	}

	if !reflect.DeepEqual(respFiles, currentFiles) {
		t.Error("got %+v, expected %+v", respFiles, currentFiles)
	}
}

func TestGlob(t *testing.T) {
	testGlob := "*"

	files, err := glob(testGlob)
	if err != nil {
		t.Fatal(err)
	}

	resp, err := remote.Tell("glob", struct {
		Pattern string
	}{
		Pattern: testGlob,
	})
	if err != nil {
		t.Fatal(err)
	}

	var r []string
	err = resp.Unmarshal(&r)
	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(r, files) {
		t.Errorf("got %+v, expected %+v", r, files)
	}
}

func TestReadFile(t *testing.T) {
	testFile := "testdata/testfile1.txt"

	content, err := ioutil.ReadFile(testFile)
	if err != nil {
		t.Error(err)
	}

	resp, err := remote.Tell("readFile", struct {
		Path string
	}{
		Path: testFile,
	})
	if err != nil {
		t.Fatal(err)
	}

	buf := resp.MustMap()["content"].MustString()

	s, err := base64.StdEncoding.DecodeString(buf)
	if err != nil {
		t.Error(err)
	}

	if string(s) != string(content) {
		t.Errorf("got %s, expecting %s", string(s), string(content))
	}

}

func TestWriteFile(t *testing.T) {
	testFile, err := ioutil.TempFile("", "")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(testFile.Name())

	content := []byte("hello kite")

	t.Log("writeFile write to a  file")
	resp, err := remote.Tell("writeFile", struct {
		Path           string
		Content        []byte
		DoNotOverwrite bool
		Append         bool
	}{
		Path:    testFile.Name(),
		Content: content,
	})
	if err != nil {
		t.Fatal(err)
	}

	if int(resp.MustFloat64()) != len(content) {
		t.Errorf("content len is wrong. got %d expected %d", int(resp.MustFloat64()), len(content))
	}

	buf, err := ioutil.ReadFile(testFile.Name())
	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(buf, content) {
		t.Errorf("content is wrong. got '%s' expected '%s'", string(buf), string(content))
	}

	t.Log("writeFile try to write if DoNotOverwrite is enabled")
	resp, err = remote.Tell("writeFile", struct {
		Path           string
		Content        []byte
		DoNotOverwrite bool
		Append         bool
	}{
		Path:           testFile.Name(),
		Content:        content,
		DoNotOverwrite: true,
	})
	if err == nil {
		t.Fatal("DoNotOverwrite is enabled, it shouldn't open the file", err)
	}

	t.Log("writeFile append to an existing file")
	resp, err = remote.Tell("writeFile", struct {
		Path           string
		Content        []byte
		DoNotOverwrite bool
		Append         bool
	}{
		Path:    testFile.Name(),
		Content: content,
		Append:  true,
	})
	if err != nil {
		t.Fatal(err)
	}

	buf, err = ioutil.ReadFile(testFile.Name())
	if err != nil {
		t.Fatal(err)
	}

	ap := string(content) + string(content)
	if !reflect.DeepEqual(buf, []byte(ap)) {
		t.Errorf("content is wrong. got '%s' expected '%s'", string(buf), ap)
	}
}

func TestUniquePath(t *testing.T) {
	testFile := "testdata/testfile1.txt"
	tempFiles := []string{}

	defer func() {
		for _, f := range tempFiles {
			os.Remove(f)
		}
	}()

	uniqueFile := func() string {
		resp, err := remote.Tell("uniquePath", struct {
			Path string
		}{
			Path: testFile,
		})
		if err != nil {
			t.Fatal(err)
		}

		s := resp.MustString()

		tempFiles = append(tempFiles, s) // add to remove them later

		// create the file now, the next call to uniquePath should generate a
		// different name when this files exits.
		err = ioutil.WriteFile(s, []byte("test111"), 0755)
		if err != nil {
			t.Fatal(err)
		}

		return s
	}

	file1 := uniqueFile()
	file2 := uniqueFile()

	if file1 == file2 {
		t.Error("files should be different, got the same %s", file1)
	}
}

func TestGetInfo(t *testing.T) {
	testFile := "testdata/testfile1.txt"

	resp, err := remote.Tell("getInfo", struct {
		Path string
	}{
		Path: testFile,
	})
	if err != nil {
		t.Fatal(err)
	}

	f := &FileEntry{}
	err = resp.Unmarshal(f)
	if err != nil {
		t.Fatal(err)
	}

	if f.Name != filepath.Base(testFile) {
		t.Errorf("got %s expecting %s", f.Name, testFile)
	}
}

func TestSetPermissions(t *testing.T)  {}
func TestRemove(t *testing.T)          {}
func TestRename(t *testing.T)          {}
func TestCreateDirectory(t *testing.T) {}
func TestMove(t *testing.T)            {}
func TestCopy(t *testing.T)            {}
<commit_msg>kite-handler/fs: add tests for remove and rename<commit_after>package fs

import (
	"encoding/base64"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"reflect"
	"testing"

	"github.com/koding/kite"
	"github.com/koding/kite/dnode"
)

var (
	fs     *kite.Kite
	remote *kite.Client
)

func init() {
	fs = kite.New("fs", "0.0.1")
	fs.Config.DisableAuthentication = true
	fs.Config.Port = 3636
	fs.HandleFunc("readDirectory", ReadDirectory)
	fs.HandleFunc("glob", Glob)
	fs.HandleFunc("readFile", ReadFile)
	fs.HandleFunc("writeFile", WriteFile)
	fs.HandleFunc("uniquePath", UniquePath)
	fs.HandleFunc("getInfo", GetInfo)
	fs.HandleFunc("setPermissions", SetPermissions)
	fs.HandleFunc("remove", Remove)
	fs.HandleFunc("rename", Rename)
	fs.HandleFunc("createDirectory", CreateDirectory)
	fs.HandleFunc("move", Move)
	fs.HandleFunc("copy", Copy)

	go fs.Run()
	<-fs.ServerReadyNotify()

	client := kite.New("client", "0.0.1")
	client.Config.DisableAuthentication = true
	remote = client.NewClientString("ws://127.0.0.1:3636")
	err := remote.Dial()
	if err != nil {
		log.Fatal("err")
	}
}

func TestReadDirectory(t *testing.T) {
	testDir := "."

	files, err := ioutil.ReadDir(testDir)
	if err != nil {
		t.Fatal(err)
	}

	currentFiles := make([]string, len(files))
	for i, f := range files {
		currentFiles[i] = f.Name()
	}

	resp, err := remote.Tell("readDirectory", struct {
		Path     string
		OnChange dnode.Function
	}{
		Path:     testDir,
		OnChange: dnode.Function{},
	})

	if err != nil {
		t.Fatal(err)
	}

	f, err := resp.Map()
	if err != nil {
		t.Fatal(err)
	}

	entries, err := f["files"].SliceOfLength(len(files))
	if err != nil {
		t.Fatal(err)
	}

	respFiles := make([]string, len(files))
	for i, e := range entries {
		f := &FileEntry{}
		err := e.Unmarshal(f)
		if err != nil {
			t.Fatal(err)
		}

		respFiles[i] = f.Name
	}

	if !reflect.DeepEqual(respFiles, currentFiles) {
		t.Error("got %+v, expected %+v", respFiles, currentFiles)
	}
}

func TestGlob(t *testing.T) {
	testGlob := "*"

	files, err := glob(testGlob)
	if err != nil {
		t.Fatal(err)
	}

	resp, err := remote.Tell("glob", struct {
		Pattern string
	}{
		Pattern: testGlob,
	})
	if err != nil {
		t.Fatal(err)
	}

	var r []string
	err = resp.Unmarshal(&r)
	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(r, files) {
		t.Errorf("got %+v, expected %+v", r, files)
	}
}

func TestReadFile(t *testing.T) {
	testFile := "testdata/testfile1.txt"

	content, err := ioutil.ReadFile(testFile)
	if err != nil {
		t.Error(err)
	}

	resp, err := remote.Tell("readFile", struct {
		Path string
	}{
		Path: testFile,
	})
	if err != nil {
		t.Fatal(err)
	}

	buf := resp.MustMap()["content"].MustString()

	s, err := base64.StdEncoding.DecodeString(buf)
	if err != nil {
		t.Error(err)
	}

	if string(s) != string(content) {
		t.Errorf("got %s, expecting %s", string(s), string(content))
	}

}

func TestWriteFile(t *testing.T) {
	testFile, err := ioutil.TempFile("", "")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(testFile.Name())

	content := []byte("hello kite")

	t.Log("writeFile write to a  file")
	resp, err := remote.Tell("writeFile", struct {
		Path           string
		Content        []byte
		DoNotOverwrite bool
		Append         bool
	}{
		Path:    testFile.Name(),
		Content: content,
	})
	if err != nil {
		t.Fatal(err)
	}

	if int(resp.MustFloat64()) != len(content) {
		t.Errorf("content len is wrong. got %d expected %d", int(resp.MustFloat64()), len(content))
	}

	buf, err := ioutil.ReadFile(testFile.Name())
	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(buf, content) {
		t.Errorf("content is wrong. got '%s' expected '%s'", string(buf), string(content))
	}

	t.Log("writeFile try to write if DoNotOverwrite is enabled")
	resp, err = remote.Tell("writeFile", struct {
		Path           string
		Content        []byte
		DoNotOverwrite bool
		Append         bool
	}{
		Path:           testFile.Name(),
		Content:        content,
		DoNotOverwrite: true,
	})
	if err == nil {
		t.Fatal("DoNotOverwrite is enabled, it shouldn't open the file", err)
	}

	t.Log("writeFile append to an existing file")
	resp, err = remote.Tell("writeFile", struct {
		Path           string
		Content        []byte
		DoNotOverwrite bool
		Append         bool
	}{
		Path:    testFile.Name(),
		Content: content,
		Append:  true,
	})
	if err != nil {
		t.Fatal(err)
	}

	buf, err = ioutil.ReadFile(testFile.Name())
	if err != nil {
		t.Fatal(err)
	}

	ap := string(content) + string(content)
	if !reflect.DeepEqual(buf, []byte(ap)) {
		t.Errorf("content is wrong. got '%s' expected '%s'", string(buf), ap)
	}
}

func TestUniquePath(t *testing.T) {
	testFile := "testdata/testfile1.txt"
	tempFiles := []string{}

	defer func() {
		for _, f := range tempFiles {
			os.Remove(f)
		}
	}()

	uniqueFile := func() string {
		resp, err := remote.Tell("uniquePath", struct {
			Path string
		}{
			Path: testFile,
		})
		if err != nil {
			t.Fatal(err)
		}

		s := resp.MustString()

		tempFiles = append(tempFiles, s) // add to remove them later

		// create the file now, the next call to uniquePath should generate a
		// different name when this files exits.
		err = ioutil.WriteFile(s, []byte("test111"), 0755)
		if err != nil {
			t.Fatal(err)
		}

		return s
	}

	file1 := uniqueFile()
	file2 := uniqueFile()

	if file1 == file2 {
		t.Error("files should be different, got the same %s", file1)
	}
}

func TestGetInfo(t *testing.T) {
	testFile := "testdata/testfile1.txt"

	resp, err := remote.Tell("getInfo", struct {
		Path string
	}{
		Path: testFile,
	})
	if err != nil {
		t.Fatal(err)
	}

	f := &FileEntry{}
	err = resp.Unmarshal(f)
	if err != nil {
		t.Fatal(err)
	}

	if f.Name != filepath.Base(testFile) {
		t.Errorf("got %s expecting %s", f.Name, testFile)
	}
}

func TestSetPermissions(t *testing.T) {}

func TestRemove(t *testing.T) {
	testFile, err := ioutil.TempFile("", "")
	if err != nil {
		t.Fatal(err)
	}

	resp, err := remote.Tell("remove", struct {
		Path string
	}{
		Path: testFile.Name(),
	})
	if err != nil {
		t.Fatal(err)
	}

	if !resp.MustBool() {
		t.Fatal("removing should return true")
	}

	ok, err := exists(testFile.Name())
	if err != nil {
		t.Error(err)
	}

	if ok {
		t.Fatalf("file still does exists %s", testFile.Name())
	}

}

func TestRename(t *testing.T) {
	testFile, err := ioutil.TempFile("", "")
	if err != nil {
		t.Fatal(err)
	}

	testNewPath := "kite.txt"
	defer os.Remove(testNewPath)

	resp, err := remote.Tell("rename", struct {
		OldPath string
		NewPath string
	}{
		OldPath: testFile.Name(),
		NewPath: testNewPath,
	})
	if err != nil {
		t.Fatal(err)
	}

	if !resp.MustBool() {
		t.Fatal("renaming should return true")
	}

	ok, err := exists(testFile.Name())
	if err != nil {
		t.Error(err)
	}

	if ok {
		t.Fatalf("file still does exists %s", testFile.Name())
	}

	ok, err = exists(testNewPath)
	if err != nil {
		t.Error(err)
	}

	if !ok {
		t.Fatalf("file does exists %s", testNewPath)
	}

}
func TestCreateDirectory(t *testing.T) {}
func TestMove(t *testing.T)            {}
func TestCopy(t *testing.T)            {}

func exists(file string) (bool, error) {
	_, err := os.Stat(file)
	if err == nil {
		return true, nil // file exist
	}

	if os.IsNotExist(err) {
		return false, nil // file does not exist
	}

	return false, err
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"
	"time"
)

func TestGetLatestVersionFromURL(t *testing.T) {
	clearCache()

	expected := "1.1.3"

	cleanupTestServer := setupTestServer(expected)
	defer cleanupTestServer()

	v, err := getLatestVersionFromURL()
	if err != nil {
		t.Fatal(err)
	}

	if v.String() != expected {
		t.Errorf("expected latest version to be %q, was %q", expected, v)
	}
}

func TestGetLatestVersionFromCache(t *testing.T) {
	expected := "1.1.3"

	setupTestCache(expected)
	defer clearCache()

	v, _, ok := getLatestVersionFromCache()
	if !ok {
		t.Errorf("could not read version from cache!")
	}

	if v.String() != expected {
		t.Errorf("expected latest version to be %q, was %q", expected, v)
	}
}

func TestGetLatestVersionWithCache(t *testing.T) {
	expected := "1.1.3"

	setupTestCache(expected)
	defer clearCache()

	cleanupTestServer := setupTestServer("2.0.0")
	defer cleanupTestServer()

	v, err := getLatestVersion()
	if err != nil {
		t.Fatal(err)
	}

	if v.String() != expected {
		t.Errorf("expected latest version to be %q, was %q", expected, v)
	}
}

func TestGetLatestVersionWithoutCache(t *testing.T) {
	clearCache()

	expected := "2.0.0"

	cleanupTestServer := setupTestServer(expected)
	defer cleanupTestServer()

	v, err := getLatestVersion()
	if err != nil {
		t.Fatal(err)
	}

	if v.String() != expected {
		t.Errorf("expected latest version to be %q, was %q", expected, v)
	}

	cv, _, ok := getLatestVersionFromCache()
	if !ok {
		t.Errorf("could not read version from cache!")
	}

	if cv.String() != expected {
		t.Errorf("expected cached version after update to be %q, was %q", expected, v)
	}
}

func TestGetLatestVersionWithInvalidCache(t *testing.T) {
	setupTestCache("1.1.3")
	invalidateCache()
	defer clearCache()

	expected := "2.0.0"

	cleanupTestServer := setupTestServer(expected)
	defer cleanupTestServer()

	v, err := getLatestVersion()
	if err != nil {
		t.Fatal(err)
	}

	if v.String() != expected {
		t.Errorf("expected latest version to be %q, was %q", expected, v)
	}

	cv, _, ok := getLatestVersionFromCache()
	if !ok {
		t.Errorf("could not read version from cache!")
	}

	if cv.String() != expected {
		t.Errorf("expected cached version after update to be %q, was %q", expected, v)
	}
}

func ExampleCheckForUpdateWithUpdateAvailable() {
	clearCache()

	resetVersion := setupFakeVersion("1.1.3")
	defer resetVersion()

	expected := "2.0.0"

	cleanupTestServer := setupTestServer(expected)
	defer cleanupTestServer()

	CheckForUpdate()

	// Output:
	// Please consider updating the PhraseApp CLI client (1.1.3 < 2.0.0)
	// You can get the latest version from https://phraseapp.com/en/cli.
}

func ExampleCheckForUpdateWithNoUpdateAvailable() {
	clearCache()

	latest := "1.7.0"

	resetVersion := setupFakeVersion(latest)
	defer resetVersion()

	cleanupTestServer := setupTestServer(latest)
	defer cleanupTestServer()

	CheckForUpdate()

	// Output:
}

func ExampleCheckForUpdateWithDevVersion() {
	clearCache()

	resetVersion := setupFakeVersion("1.1.3-dev")
	defer resetVersion()

	cleanupTestServer := setupTestServer("2.0.0")
	defer cleanupTestServer()

	CheckForUpdate()

	// Output: You're running a development version (1.1.3-dev) of the PhraseApp client! Latest version is 2.0.0.
}

func setupTestServer(version string) func() {
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Location", "https://github.com/phrase/phraseapp-client/releases/tag/"+version)
		w.WriteHeader(http.StatusFound)
	}))

	old := releasesURL
	releasesURL = s.URL

	return func() {
		s.Close()
		releasesURL = old
	}
}

func setupTestCache(version string) {
	ioutil.WriteFile(versionCacheFilename, []byte(version), 0600)
}

func invalidateCache() {
	info, _ := os.Stat(versionCacheFilename)
	os.Chtimes(versionCacheFilename, time.Now(), info.ModTime().Add(-48*time.Hour))
}

func clearCache() {
	os.Remove(versionCacheFilename)
}

func setupFakeVersion(version string) func() {
	tmp := PHRASEAPP_CLIENT_VERSION
	PHRASEAPP_CLIENT_VERSION = version

	return func() {
		PHRASEAPP_CLIENT_VERSION = tmp
	}
}
<commit_msg>fix tests<commit_after>package main

import (
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"
	"time"
)

func TestGetLatestVersionFromURL(t *testing.T) {
	clearCache()

	expected := "1.1.3"

	cleanupTestServer := setupTestServer(expected)
	defer cleanupTestServer()

	v, err := getLatestVersionFromURL()
	if err != nil {
		t.Fatal(err)
	}

	if v.String() != expected {
		t.Errorf("expected latest version to be %q, was %q", expected, v)
	}
}

func TestGetLatestVersionFromCache(t *testing.T) {
	expected := "1.1.3"

	setupTestCache(expected)
	defer clearCache()

	v, _, err := getLatestVersionFromCache()
	if err != nil {
		t.Errorf("could not read version from cache!")
	}

	if v.String() != expected {
		t.Errorf("expected latest version to be %q, was %q", expected, v)
	}
}

func TestGetLatestVersionWithCache(t *testing.T) {
	expected := "1.1.3"

	setupTestCache(expected)
	defer clearCache()

	cleanupTestServer := setupTestServer("2.0.0")
	defer cleanupTestServer()

	v, err := getLatestVersion()
	if err != nil {
		t.Fatal(err)
	}

	if v.String() != expected {
		t.Errorf("expected latest version to be %q, was %q", expected, v)
	}
}

func TestGetLatestVersionWithoutCache(t *testing.T) {
	clearCache()

	expected := "2.0.0"

	cleanupTestServer := setupTestServer(expected)
	defer cleanupTestServer()

	v, err := getLatestVersion()
	if err != nil {
		t.Fatal(err)
	}

	if v.String() != expected {
		t.Errorf("expected latest version to be %q, was %q", expected, v)
	}

	cv, _, err := getLatestVersionFromCache()
	if err != nil {
		t.Errorf("could not read version from cache!")
	}

	if cv.String() != expected {
		t.Errorf("expected cached version after update to be %q, was %q", expected, v)
	}
}

func TestGetLatestVersionWithInvalidCache(t *testing.T) {
	setupTestCache("1.1.3")
	invalidateCache()
	defer clearCache()

	expected := "2.0.0"

	cleanupTestServer := setupTestServer(expected)
	defer cleanupTestServer()

	v, err := getLatestVersion()
	if err != nil {
		t.Fatal(err)
	}

	if v.String() != expected {
		t.Errorf("expected latest version to be %q, was %q", expected, v)
	}

	cv, _, err := getLatestVersionFromCache()
	if err != nil {
		t.Errorf("could not read version from cache!")
	}

	if cv.String() != expected {
		t.Errorf("expected cached version after update to be %q, was %q", expected, v)
	}
}

func ExampleCheckForUpdateWithUpdateAvailable() {
	clearCache()

	resetVersion := setupFakeVersion("1.1.3")
	defer resetVersion()

	expected := "2.0.0"

	cleanupTestServer := setupTestServer(expected)
	defer cleanupTestServer()

	CheckForUpdate()

	// Output:
	// Please consider updating the PhraseApp CLI client (1.1.3 < 2.0.0)
	// You can get the latest version from https://phraseapp.com/en/cli.
}

func ExampleCheckForUpdateWithNoUpdateAvailable() {
	clearCache()

	latest := "1.7.0"

	resetVersion := setupFakeVersion(latest)
	defer resetVersion()

	cleanupTestServer := setupTestServer(latest)
	defer cleanupTestServer()

	CheckForUpdate()

	// Output:
}

func ExampleCheckForUpdateWithDevVersion() {
	clearCache()

	resetVersion := setupFakeVersion("1.1.3-dev")
	defer resetVersion()

	cleanupTestServer := setupTestServer("2.0.0")
	defer cleanupTestServer()

	CheckForUpdate()

	// Output: You're running a development version (1.1.3-dev) of the PhraseApp client! Latest version is 2.0.0.
}

func setupTestServer(version string) func() {
	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Location", "https://github.com/phrase/phraseapp-client/releases/tag/"+version)
		w.WriteHeader(http.StatusFound)
	}))

	old := releasesURL
	releasesURL = s.URL

	return func() {
		s.Close()
		releasesURL = old
	}
}

func setupTestCache(version string) {
	ioutil.WriteFile(versionCacheFilename, []byte(version), 0600)
}

func invalidateCache() {
	info, _ := os.Stat(versionCacheFilename)
	os.Chtimes(versionCacheFilename, time.Now(), info.ModTime().Add(-48*time.Hour))
}

func clearCache() {
	os.Remove(versionCacheFilename)
}

func setupFakeVersion(version string) func() {
	tmp := PHRASEAPP_CLIENT_VERSION
	PHRASEAPP_CLIENT_VERSION = version

	return func() {
		PHRASEAPP_CLIENT_VERSION = tmp
	}
}
<|endoftext|>


<commit_before>package testing

var FluxEndToEndSkipList = map[string]map[string]string{
	"universe": {
		// TODO(adam) determine the reason for these test failures.
		"cov":                      "Reason TBD",
		"covariance":               "Reason TBD",
		"cumulative_sum":           "Reason TBD",
		"cumulative_sum_default":   "Reason TBD",
		"cumulative_sum_noop":      "Reason TBD",
		"drop_non_existent":        "Reason TBD",
		"first":                    "Reason TBD",
		"highestAverage":           "Reason TBD",
		"highestMax":               "Reason TBD",
		"histogram":                "Reason TBD",
		"histogram_normalize":      "Reason TBD",
		"histogram_quantile":       "Reason TBD",
		"join":                     "Reason TBD",
		"join_across_measurements": "Reason TBD",
		"join_agg":                 "Reason TBD",
		"keep_non_existent":        "Reason TBD",
		"key_values":               "Reason TBD",
		"key_values_host_name":     "Reason TBD",
		"last":                     "Reason TBD",
		"lowestAverage":            "Reason TBD",
		"max":                      "Reason TBD",
		"min":                      "Reason TBD",
		"sample":                   "Reason TBD",
		"selector_preserve_time":   "Reason TBD",
		"shift":                    "Reason TBD",
		"shift_negative_duration":  "Reason TBD",
		"task_per_line":            "Reason TBD",
		"top":                      "Reason TBD",
		"union":                    "Reason TBD",
		"union_heterogeneous":      "Reason TBD",
		"unique":                   "Reason TBD",
		"distinct":                 "Reason TBD",

		// it appears these occur when writing the input data.  `to` may not be null safe.
		"fill_bool":   "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_float":  "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_int":    "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_string": "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_time":   "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_uint":   "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"window_null": "failed to read meta data: panic: interface conversion: interface {} is nil, not float64",

		// these may just be missing calls to range() in the tests.  easy to fix in a new PR.
		"group_nulls":           "unbounded test",
		"integral":              "unbounded test",
		"integral_columns":      "unbounded test",
		"map":                   "unbounded test",
		"join_missing_on_col":   "unbounded test",
		"join_use_previous":     "unbounded test (https://github.com/influxdata/flux/issues/2996)"
		"rowfn_with_import":     "unbounded test",
		"merge_filter_flag_on":  "unbounded test (https://github.com/influxdata/flux/issues/2995)",
		"merge_filter_flag_off": "unbounded test (https://github.com/influxdata/flux/issues/2994)",

		// the following tests have a difference between the CSV-decoded input table, and the storage-retrieved version of that table
		"columns":            "group key mismatch",
		"set":                "column order mismatch",
		"simple_max":         "_stop missing from expected output",
		"derivative":         "time bounds mismatch (engine uses now() instead of bounds on input table)",
		"difference_columns": "data write/read path loses columns x and y",
		"keys":               "group key mismatch",

		// failed to read meta data errors: the CSV encoding is incomplete probably due to data schema errors.  needs more detailed investigation to find root cause of error
		// "filter_by_regex":             "failed to read metadata",
		// "filter_by_tags":              "failed to read metadata",
		"group":                       "failed to read metadata",
		"group_except":                "failed to read metadata",
		"group_ungroup":               "failed to read metadata",
		"pivot_mean":                  "failed to read metadata",
		"histogram_quantile_minvalue": "failed to read meta data: no column with label _measurement exists",
		"increase":                    "failed to read meta data: table has no _value column",

		"string_max":                  "error: invalid use of function: *functions.MaxSelector has no implementation for type string (https://github.com/influxdata/platform/issues/224)",
		"null_as_value":               "null not supported as value in influxql (https://github.com/influxdata/platform/issues/353)",
		"string_interp":               "string interpolation not working as expected in flux (https://github.com/influxdata/platform/issues/404)",
		"to":                          "to functions are not supported in the testing framework (https://github.com/influxdata/flux/issues/77)",
		"covariance_missing_column_1": "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"covariance_missing_column_2": "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"drop_before_rename":          "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"drop_referenced":             "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"yield":                       "yield requires special test case (https://github.com/influxdata/flux/issues/535)",

		"window_group_mean_ungroup": "window trigger optimization modifies sort order of its output tables (https://github.com/influxdata/flux/issues/1067)",

		"median_column": "failing in different ways (https://github.com/influxdata/influxdb/issues/13909)",
		"dynamic_query": "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",

		"to_int":  "dateTime conversion issue: https://github.com/influxdata/influxdb/issues/14575",
		"to_uint": "dateTime conversion issue: https://github.com/influxdata/influxdb/issues/14575",

		"holt_winters_panic": "Expected output is an empty table which breaks the testing framework (https://github.com/influxdata/influxdb/issues/14749)",
		"map_nulls":          "to cannot write null values",
	},
	"experimental": {
		"set":       "Reason TBD",
		"join":      "unbounded test",
		"alignTime": "unbounded test",
	},
	"experimental/geo": {
		"filterRowsNotStrict": "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"filterRowsStrict":    "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"gridFilterLevel":     "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"gridFilter":          "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"groupByArea":         "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"filterRowsPivoted":   "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"shapeDataWithFilter": "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"shapeData":           "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
	},
	"regexp": {
		"replaceAllString": "Reason TBD",
	},
	"http": {
		"http_endpoint": "need ability to test side effects in e2e tests: (https://github.com/influxdata/flux/issues/1723)",
	},
	"influxdata/influxdb/v1": {
		"show_measurements": "flaky test (https://github.com/influxdata/influxdb/issues/15450)",
		"show_tag_values":   "flaky test (https://github.com/influxdata/influxdb/issues/15450)",
		"show_tag_keys":     "flaky test (https://github.com/influxdata/influxdb/issues/15450)",
	},
	"influxdata/influxdb/monitor": {
		"state_changes_big_any_to_any":     "unbounded test",
		"state_changes_big_info_to_ok":     "unbounded test",
		"state_changes_big_ok_to_info":     "unbounded test",
		"state_changes_any_to_any":         "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
		"state_changes_info_to_any":        "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
		"state_changes_invalid_any_to_any": "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
		"state_changes":                    "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
	},
	"influxdata/influxdb/secrets": {
		"secrets": "Cannot inject custom deps into the test framework so the secrets don't lookup correctly",
	},
	"internal/promql": {
		"join": "unbounded test",
	},
	"testing/chronograf": {
		"buckets":                "unbounded test",
		"aggregate_window_count": "flakey test: https://github.com/influxdata/influxdb/issues/18463",
	},
	"testing/kapacitor": {
		"fill_default": "unknown field type for f1",
	},
	"testing/pandas": {
		"extract_regexp_findStringIndex": "pandas. map does not correctly handled returned arrays (https://github.com/influxdata/flux/issues/1387)",
		"partition_strings_splitN":       "pandas. map does not correctly handled returned arrays (https://github.com/influxdata/flux/issues/1387)",
	},
	"testing/promql": {
		"emptyTable": "tests a source",
		"year":       "flakey test: https://github.com/influxdata/influxdb/issues/15667",
	},
}

type PerTestFeatureFlagMap = map[string]map[string]map[string]string

var FluxEndToEndFeatureFlags = PerTestFeatureFlagMap{
	"planner": {
		"window_count_push": {
			"pushDownWindowAggregateCount": "true",
		},
		"window_sum_push": {
			"pushDownWindowAggregateSum": "true",
		},
		"bare_count_push": {
			"pushDownWindowAggregateCount": "true",
		},
		"bare_sum_push": {
			"pushDownWindowAggregateSum": "true",
		},
	},
}
<commit_msg>chore: fix accidental typo<commit_after>package testing

var FluxEndToEndSkipList = map[string]map[string]string{
	"universe": {
		// TODO(adam) determine the reason for these test failures.
		"cov":                      "Reason TBD",
		"covariance":               "Reason TBD",
		"cumulative_sum":           "Reason TBD",
		"cumulative_sum_default":   "Reason TBD",
		"cumulative_sum_noop":      "Reason TBD",
		"drop_non_existent":        "Reason TBD",
		"first":                    "Reason TBD",
		"highestAverage":           "Reason TBD",
		"highestMax":               "Reason TBD",
		"histogram":                "Reason TBD",
		"histogram_normalize":      "Reason TBD",
		"histogram_quantile":       "Reason TBD",
		"join":                     "Reason TBD",
		"join_across_measurements": "Reason TBD",
		"join_agg":                 "Reason TBD",
		"keep_non_existent":        "Reason TBD",
		"key_values":               "Reason TBD",
		"key_values_host_name":     "Reason TBD",
		"last":                     "Reason TBD",
		"lowestAverage":            "Reason TBD",
		"max":                      "Reason TBD",
		"min":                      "Reason TBD",
		"sample":                   "Reason TBD",
		"selector_preserve_time":   "Reason TBD",
		"shift":                    "Reason TBD",
		"shift_negative_duration":  "Reason TBD",
		"task_per_line":            "Reason TBD",
		"top":                      "Reason TBD",
		"union":                    "Reason TBD",
		"union_heterogeneous":      "Reason TBD",
		"unique":                   "Reason TBD",
		"distinct":                 "Reason TBD",

		// it appears these occur when writing the input data.  `to` may not be null safe.
		"fill_bool":   "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_float":  "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_int":    "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_string": "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_time":   "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"fill_uint":   "failed to read meta data: panic: interface conversion: interface {} is nil, not uint64",
		"window_null": "failed to read meta data: panic: interface conversion: interface {} is nil, not float64",

		// these may just be missing calls to range() in the tests.  easy to fix in a new PR.
		"group_nulls":           "unbounded test",
		"integral":              "unbounded test",
		"integral_columns":      "unbounded test",
		"map":                   "unbounded test",
		"join_missing_on_col":   "unbounded test",
		"join_use_previous":     "unbounded test (https://github.com/influxdata/flux/issues/2996)",
		"rowfn_with_import":     "unbounded test",
		"merge_filter_flag_on":  "unbounded test (https://github.com/influxdata/flux/issues/2995)",
		"merge_filter_flag_off": "unbounded test (https://github.com/influxdata/flux/issues/2994)",

		// the following tests have a difference between the CSV-decoded input table, and the storage-retrieved version of that table
		"columns":            "group key mismatch",
		"set":                "column order mismatch",
		"simple_max":         "_stop missing from expected output",
		"derivative":         "time bounds mismatch (engine uses now() instead of bounds on input table)",
		"difference_columns": "data write/read path loses columns x and y",
		"keys":               "group key mismatch",

		// failed to read meta data errors: the CSV encoding is incomplete probably due to data schema errors.  needs more detailed investigation to find root cause of error
		// "filter_by_regex":             "failed to read metadata",
		// "filter_by_tags":              "failed to read metadata",
		"group":                       "failed to read metadata",
		"group_except":                "failed to read metadata",
		"group_ungroup":               "failed to read metadata",
		"pivot_mean":                  "failed to read metadata",
		"histogram_quantile_minvalue": "failed to read meta data: no column with label _measurement exists",
		"increase":                    "failed to read meta data: table has no _value column",

		"string_max":                  "error: invalid use of function: *functions.MaxSelector has no implementation for type string (https://github.com/influxdata/platform/issues/224)",
		"null_as_value":               "null not supported as value in influxql (https://github.com/influxdata/platform/issues/353)",
		"string_interp":               "string interpolation not working as expected in flux (https://github.com/influxdata/platform/issues/404)",
		"to":                          "to functions are not supported in the testing framework (https://github.com/influxdata/flux/issues/77)",
		"covariance_missing_column_1": "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"covariance_missing_column_2": "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"drop_before_rename":          "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"drop_referenced":             "need to support known errors in new test framework (https://github.com/influxdata/flux/issues/536)",
		"yield":                       "yield requires special test case (https://github.com/influxdata/flux/issues/535)",

		"window_group_mean_ungroup": "window trigger optimization modifies sort order of its output tables (https://github.com/influxdata/flux/issues/1067)",

		"median_column": "failing in different ways (https://github.com/influxdata/influxdb/issues/13909)",
		"dynamic_query": "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",

		"to_int":  "dateTime conversion issue: https://github.com/influxdata/influxdb/issues/14575",
		"to_uint": "dateTime conversion issue: https://github.com/influxdata/influxdb/issues/14575",

		"holt_winters_panic": "Expected output is an empty table which breaks the testing framework (https://github.com/influxdata/influxdb/issues/14749)",
		"map_nulls":          "to cannot write null values",
	},
	"experimental": {
		"set":       "Reason TBD",
		"join":      "unbounded test",
		"alignTime": "unbounded test",
	},
	"experimental/geo": {
		"filterRowsNotStrict": "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"filterRowsStrict":    "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"gridFilterLevel":     "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"gridFilter":          "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"groupByArea":         "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"filterRowsPivoted":   "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"shapeDataWithFilter": "tableFind does not work in e2e tests: https://github.com/influxdata/influxdb/issues/13975",
		"shapeData":           "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
	},
	"regexp": {
		"replaceAllString": "Reason TBD",
	},
	"http": {
		"http_endpoint": "need ability to test side effects in e2e tests: (https://github.com/influxdata/flux/issues/1723)",
	},
	"influxdata/influxdb/v1": {
		"show_measurements": "flaky test (https://github.com/influxdata/influxdb/issues/15450)",
		"show_tag_values":   "flaky test (https://github.com/influxdata/influxdb/issues/15450)",
		"show_tag_keys":     "flaky test (https://github.com/influxdata/influxdb/issues/15450)",
	},
	"influxdata/influxdb/monitor": {
		"state_changes_big_any_to_any":     "unbounded test",
		"state_changes_big_info_to_ok":     "unbounded test",
		"state_changes_big_ok_to_info":     "unbounded test",
		"state_changes_any_to_any":         "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
		"state_changes_info_to_any":        "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
		"state_changes_invalid_any_to_any": "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
		"state_changes":                    "test run before to() is finished: https://github.com/influxdata/influxdb/issues/13975",
	},
	"influxdata/influxdb/secrets": {
		"secrets": "Cannot inject custom deps into the test framework so the secrets don't lookup correctly",
	},
	"internal/promql": {
		"join": "unbounded test",
	},
	"testing/chronograf": {
		"buckets":                "unbounded test",
		"aggregate_window_count": "flakey test: https://github.com/influxdata/influxdb/issues/18463",
	},
	"testing/kapacitor": {
		"fill_default": "unknown field type for f1",
	},
	"testing/pandas": {
		"extract_regexp_findStringIndex": "pandas. map does not correctly handled returned arrays (https://github.com/influxdata/flux/issues/1387)",
		"partition_strings_splitN":       "pandas. map does not correctly handled returned arrays (https://github.com/influxdata/flux/issues/1387)",
	},
	"testing/promql": {
		"emptyTable": "tests a source",
		"year":       "flakey test: https://github.com/influxdata/influxdb/issues/15667",
	},
}

type PerTestFeatureFlagMap = map[string]map[string]map[string]string

var FluxEndToEndFeatureFlags = PerTestFeatureFlagMap{
	"planner": {
		"window_count_push": {
			"pushDownWindowAggregateCount": "true",
		},
		"window_sum_push": {
			"pushDownWindowAggregateSum": "true",
		},
		"bare_count_push": {
			"pushDownWindowAggregateCount": "true",
		},
		"bare_sum_push": {
			"pushDownWindowAggregateSum": "true",
		},
	},
}
<|endoftext|>


<commit_before>package whproxy

import (
	"bufio"
	"io"
	"net/http"
	"regexp"
	"sync"

	"github.com/gorilla/websocket"
	"github.com/taskcluster/webhooktunnel/util"
	"github.com/taskcluster/webhooktunnel/wsmux"
)

var (
	registerRe = regexp.MustCompile("^/register/(\\w+)/?$")
	serveRe    = regexp.MustCompile("^/(\\w+)/?(.*)$")
)

// Config for Proxy. Accepts a websocket.Upgrader and a Logger.
// Default value for Upgrade ReadBufferSize and WriteBufferSize is 1024 bytes.
// Default Logger is NilLogger.
type Config struct {
	Upgrader websocket.Upgrader
	Logger   util.Logger
}

// Proxy is used to send http and ws requests to workers.
// New proxy can be created by using whproxy.New()
type Proxy struct {
	m        sync.RWMutex
	pool     map[string]*wsmux.Session
	upgrader websocket.Upgrader
	logger   util.Logger
	handler  http.Handler
}

func (p *Proxy) validateRequest(r *http.Request) error {
	return nil
}

// New returns a pointer to a new proxy instance
func New(conf Config) *Proxy {
	p := &Proxy{
		pool:     make(map[string]*wsmux.Session),
		upgrader: conf.Upgrader,
		logger:   conf.Logger,
	}

	if p.logger == nil {
		p.logger = &util.NilLogger{}
	}

	p.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// register will be matched first
		if registerRe.MatchString(r.URL.Path) { // matches "/register/(\w+)/?$"
			id := registerRe.FindStringSubmatch(r.URL.Path)[1]
			p.register(w, r, id)
		} else if serveRe.MatchString(r.URL.Path) { // matches "/{id}/{path}"
			matches := serveRe.FindStringSubmatch(r.URL.Path)
			id, path := matches[1], matches[2]
			p.serveRequest(w, r, id, path)
		} else { // if not register request or worker request, not found
			http.NotFound(w, r)
		}
	})

	return p
}

// GetHandler returns the router assosciated with the proxy
func (p *Proxy) GetHandler() http.Handler {
	return p.handler
}

// getWorkerSession returns true if a session with the given id is present
func (p *Proxy) getWorkerSession(id string) (*wsmux.Session, bool) {
	p.m.RLock()
	defer p.m.RUnlock()
	s, ok := p.pool[id]
	return s, ok
}

// addWorker adds a new worker to the pool
func (p *Proxy) addWorker(id string, conn *websocket.Conn, config wsmux.Config) error {
	p.m.Lock()
	defer p.m.Unlock()
	if _, ok := p.pool[id]; ok {
		return ErrDuplicateWorker
	}
	p.pool[id] = wsmux.Server(conn, config)
	p.logger.Printf("worker with id %s registered on proxy", id)
	return nil
}

// register is used to connect a worker to the proxy so that it can start serving API endpoints.
// The request must contain the worker ID in the url.
// The request is validated by the proxy and the http connection is upgraded to websocket.
func (p *Proxy) register(w http.ResponseWriter, r *http.Request, id string) {
	if !websocket.IsWebSocketUpgrade(r) {
		http.NotFound(w, r)
		return
	}

	if err := p.validateRequest(r); err != nil {
		http.Error(w, "invalid request", 401)
		return
	}

	if _, ok := p.getWorkerSession(id); ok {
		http.Error(w, "duplicate worker", 401)
		return
	}

	conn, err := p.upgrader.Upgrade(w, r, nil)
	if err != nil {
		panic(err)
	}

	// add worker after connection is established
	_ = p.addWorker(id, conn, wsmux.Config{StreamBufferSize: 64 * 1024})
}

// serveRequest serves worker endpoints to viewers
func (p *Proxy) serveRequest(w http.ResponseWriter, r *http.Request, id string, path string) {
	session, ok := p.getWorkerSession(id)

	// 404 if worker is not registered on this proxy
	if !ok {
		// DHT code will be added here
		http.Error(w, "worker not found", 404)
		return
	}

	// Open a stream to the worker session
	reqStream, err := session.Open()
	if err != nil {
		http.Error(w, "could not connect to the worker", 500)
		return
	}

	// set original path as header
	r.Header.Set("x-webhooktunnel-original-path", r.URL.Path)

	// check for a websocket request
	if websocket.IsWebSocketUpgrade(r) {
		_ = websocketProxy(w, r, reqStream, p.upgrader)
		return
	}

	// rewrite path for worker and write request
	r.URL.Path = "/" + path
	err = r.Write(reqStream)
	if err != nil {
		http.Error(w, "error sending request to worker", 500)
		return
	}

	// read response from worker
	bufReader := bufio.NewReader(reqStream)
	resp, err := http.ReadResponse(bufReader, r)
	if err != nil {
		http.Error(w, "error sending response", 500)
		return
	}

	// manually proxy response
	// clear responseWriter headers and write response headers instead
	for k := range w.Header() {
		w.Header().Del(k)
	}
	for k, v := range resp.Header {
		w.Header()[k] = v
	}

	// dump headers
	w.WriteHeader(resp.StatusCode)

	// stream body to viewer
	if resp.Body != nil {
		_, err := io.Copy(w, resp.Body)
		if err != nil {
			return
		}
	}
}
<commit_msg>remove session when worker connection is closed<commit_after>package whproxy

import (
	"bufio"
	"io"
	"net/http"
	"regexp"
	"sync"

	"github.com/gorilla/websocket"
	"github.com/taskcluster/webhooktunnel/util"
	"github.com/taskcluster/webhooktunnel/wsmux"
)

var (
	registerRe = regexp.MustCompile("^/register/(\\w+)/?$")
	serveRe    = regexp.MustCompile("^/(\\w+)/?(.*)$")
)

// Config for Proxy. Accepts a websocket.Upgrader and a Logger.
// Default value for Upgrade ReadBufferSize and WriteBufferSize is 1024 bytes.
// Default Logger is NilLogger.
type Config struct {
	Upgrader websocket.Upgrader
	Logger   util.Logger
}

// Proxy is used to send http and ws requests to workers.
// New proxy can be created by using whproxy.New()
type Proxy struct {
	m               sync.RWMutex
	pool            map[string]*wsmux.Session
	upgrader        websocket.Upgrader
	logger          util.Logger
	handler         http.Handler
	onSessionRemove func(string)
}

func (p *Proxy) validateRequest(r *http.Request) error {
	return nil
}

// New returns a pointer to a new proxy instance
func New(conf Config) *Proxy {
	p := &Proxy{
		pool:     make(map[string]*wsmux.Session),
		upgrader: conf.Upgrader,
		logger:   conf.Logger,
	}

	if p.logger == nil {
		p.logger = &util.NilLogger{}
	}

	p.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// register will be matched first
		if registerRe.MatchString(r.URL.Path) { // matches "/register/(\w+)/?$"
			id := registerRe.FindStringSubmatch(r.URL.Path)[1]
			p.register(w, r, id)
		} else if serveRe.MatchString(r.URL.Path) { // matches "/{id}/{path}"
			matches := serveRe.FindStringSubmatch(r.URL.Path)
			id, path := matches[1], matches[2]
			p.serveRequest(w, r, id, path)
		} else { // if not register request or worker request, not found
			http.NotFound(w, r)
		}
	})

	return p
}

func (p *Proxy) SetSessionRemoveHandler(h func(string)) {
	p.m.Lock()
	defer p.m.Unlock()
	p.onSessionRemove = h
}

// GetHandler returns the router assosciated with the proxy
func (p *Proxy) GetHandler() http.Handler {
	return p.handler
}

// getWorkerSession returns true if a session with the given id is present
func (p *Proxy) getWorkerSession(id string) (*wsmux.Session, bool) {
	p.m.RLock()
	defer p.m.RUnlock()
	s, ok := p.pool[id]
	return s, ok
}

// addWorker adds a new worker to the pool
func (p *Proxy) addWorker(id string, conn *websocket.Conn, config wsmux.Config) error {
	p.m.Lock()
	defer p.m.Unlock()
	if _, ok := p.pool[id]; ok {
		return ErrDuplicateWorker
	}
	p.pool[id] = wsmux.Server(conn, config)
	p.logger.Printf("worker with id %s registered on proxy", id)
	return nil
}

// removeWorker is an idempotent operation which deletes a worker from the proxy's
// worker pool
func (p *Proxy) removeWorker(id string) {
	p.m.Lock()
	defer p.m.Unlock()
	delete(p.pool, id)
	p.logger.Printf("worker with id %s removed from proxy", id)
}

// register is used to connect a worker to the proxy so that it can start serving API endpoints.
// The request must contain the worker ID in the url.
// The request is validated by the proxy and the http connection is upgraded to websocket.
func (p *Proxy) register(w http.ResponseWriter, r *http.Request, id string) {
	if !websocket.IsWebSocketUpgrade(r) {
		http.NotFound(w, r)
		return
	}

	if err := p.validateRequest(r); err != nil {
		http.Error(w, "invalid request", 401)
		return
	}

	if _, ok := p.getWorkerSession(id); ok {
		http.Error(w, "duplicate worker", 401)
		return
	}

	conn, err := p.upgrader.Upgrade(w, r, nil)
	if err != nil {
		panic(err)
	}

	// generate config
	conf := wsmux.Config{
		StreamBufferSize: 64 * 1024,
		RemoteCloseCallback: func() {
			p.removeWorker(id)
			if p.onSessionRemove != nil {
				p.onSessionRemove(id)
			}
		},
	}
	// add worker after connection is established
	_ = p.addWorker(id, conn, conf)
}

// serveRequest serves worker endpoints to viewers
func (p *Proxy) serveRequest(w http.ResponseWriter, r *http.Request, id string, path string) {
	session, ok := p.getWorkerSession(id)

	// 404 if worker is not registered on this proxy
	if !ok {
		// DHT code will be added here
		http.Error(w, "worker not found", 404)
		return
	}

	// Open a stream to the worker session
	reqStream, err := session.Open()
	if err != nil {
		http.Error(w, "could not connect to the worker", 500)
		return
	}

	// set original path as header
	r.Header.Set("x-webhooktunnel-original-path", r.URL.Path)

	// check for a websocket request
	if websocket.IsWebSocketUpgrade(r) {
		_ = websocketProxy(w, r, reqStream, p.upgrader)
		return
	}

	// rewrite path for worker and write request
	r.URL.Path = "/" + path
	err = r.Write(reqStream)
	if err != nil {
		http.Error(w, "error sending request to worker", 500)
		return
	}

	// read response from worker
	bufReader := bufio.NewReader(reqStream)
	resp, err := http.ReadResponse(bufReader, r)
	if err != nil {
		http.Error(w, "error sending response", 500)
		return
	}

	// manually proxy response
	// clear responseWriter headers and write response headers instead
	for k := range w.Header() {
		w.Header().Del(k)
	}
	for k, v := range resp.Header {
		w.Header()[k] = v
	}

	// dump headers
	w.WriteHeader(resp.StatusCode)

	// stream body to viewer
	if resp.Body != nil {
		_, err := io.Copy(w, resp.Body)
		if err != nil {
			return
		}
	}
}
<|endoftext|>