<commit_before>package http_crawler_test

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"

	. "github.com/alphagov/govuk_crawler_worker/http_crawler"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

func testServer(status int, body string) *httptest.Server {
	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(status)
		fmt.Fprintln(w, body)
	}))
}

var _ = Describe("Crawl", func() {
	var (
		crawler    *Crawler
		crawlerErr error
	)

	BeforeEach(func() {
		crawler, crawlerErr = NewCrawler("http://127.0.0.1/")

		Expect(crawlerErr).To(BeNil())
		Expect(crawler).ToNot(BeNil())
	})

	Describe("RetryStatusCodes", func() {
		It("should return a fixed int array with values 429, 500..599", func() {
			statusCodes := RetryStatusCodes()

			Expect(len(statusCodes)).To(Equal(101))
			Expect(statusCodes[0]).To(Equal(429))
			Expect(statusCodes[1]).To(Equal(500))
			Expect(statusCodes[100]).To(Equal(599))
		})
	})

	Describe("NewCrawler()", func() {
		It("doesn't allow providing empty URLs", func() {
			badCrawler, err := NewCrawler("")

			Expect(err).ToNot(BeNil())
			Expect(badCrawler).To(BeNil())
		})

		It("provides a new crawler that accepts the provided host", func() {
			GOVUKCrawler, err := NewCrawler("https://www.gov.uk/")

			Expect(err).To(BeNil())
			Expect(GOVUKCrawler.RootURL.Host).To(Equal("www.gov.uk"))
		})
	})

	Describe("Crawler.Crawl()", func() {
		It("specifies a user agent when making a request", func() {
			userAgentTestServer := func(httpStatus int) *httptest.Server {
				return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					w.WriteHeader(httpStatus)
					fmt.Fprintln(w, r.UserAgent())
				}))
			}

			ts := userAgentTestServer(http.StatusOK)
			defer ts.Close()

			body, err := crawler.Crawl(ts.URL)

			Expect(err).To(BeNil())
			Expect(strings.HasPrefix((string(body)), "GOV.UK Crawler Worker")).To(BeTrue())
		})

		It("returns a body with no errors for 200 OK responses", func() {
			ts := testServer(http.StatusOK, "Hello world")
			defer ts.Close()

			body, err := crawler.Crawl(ts.URL)

			Expect(err).To(BeNil())
			Expect(strings.TrimSpace(string(body))).To(Equal("Hello world"))
		})

		It("doesn't allow crawling a URL that doesn't match the root URL", func() {
			body, err := crawler.Crawl("http://google.com/foo")

			Expect(err).To(Equal(CannotCrawlURL))
			Expect(body).To(Equal([]byte{}))
		})

		Describe("returning a retry error", func() {
			It("returns a retry error if we get a response code of Too Many Requests", func() {
				ts := testServer(429, "Too Many Requests")
				defer ts.Close()

				body, err := crawler.Crawl(ts.URL)

				Expect(err).To(Equal(RetryRequestError))
				Expect(body).To(Equal([]byte{}))
			})

			It("returns a retry error if we get a response code of Internal Server Error", func() {
				ts := testServer(http.StatusInternalServerError, "Internal Server Error")
				defer ts.Close()

				body, err := crawler.Crawl(ts.URL)

				Expect(err).To(Equal(RetryRequestError))
				Expect(body).To(Equal([]byte{}))
			})

			It("returns a retry error if we get a response code of Gateway Timeout", func() {
				ts := testServer(http.StatusGatewayTimeout, "Gateway Timeout")
				defer ts.Close()

				body, err := crawler.Crawl(ts.URL)

				Expect(err).To(Equal(RetryRequestError))
				Expect(body).To(Equal([]byte{}))
			})
		})
	})
})
<commit_msg>Split HTTP server handler into own variable<commit_after>package http_crawler_test

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"

	. "github.com/alphagov/govuk_crawler_worker/http_crawler"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

func testServer(status int, body string) *httptest.Server {
	handler := func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(status)
		fmt.Fprintln(w, body)
	}
	return httptest.NewServer(http.HandlerFunc(handler))
}

var _ = Describe("Crawl", func() {
	var (
		crawler    *Crawler
		crawlerErr error
	)

	BeforeEach(func() {
		crawler, crawlerErr = NewCrawler("http://127.0.0.1/")

		Expect(crawlerErr).To(BeNil())
		Expect(crawler).ToNot(BeNil())
	})

	Describe("RetryStatusCodes", func() {
		It("should return a fixed int array with values 429, 500..599", func() {
			statusCodes := RetryStatusCodes()

			Expect(len(statusCodes)).To(Equal(101))
			Expect(statusCodes[0]).To(Equal(429))
			Expect(statusCodes[1]).To(Equal(500))
			Expect(statusCodes[100]).To(Equal(599))
		})
	})

	Describe("NewCrawler()", func() {
		It("doesn't allow providing empty URLs", func() {
			badCrawler, err := NewCrawler("")

			Expect(err).ToNot(BeNil())
			Expect(badCrawler).To(BeNil())
		})

		It("provides a new crawler that accepts the provided host", func() {
			GOVUKCrawler, err := NewCrawler("https://www.gov.uk/")

			Expect(err).To(BeNil())
			Expect(GOVUKCrawler.RootURL.Host).To(Equal("www.gov.uk"))
		})
	})

	Describe("Crawler.Crawl()", func() {
		It("specifies a user agent when making a request", func() {
			userAgentTestServer := func(httpStatus int) *httptest.Server {
				return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					w.WriteHeader(httpStatus)
					fmt.Fprintln(w, r.UserAgent())
				}))
			}

			ts := userAgentTestServer(http.StatusOK)
			defer ts.Close()

			body, err := crawler.Crawl(ts.URL)

			Expect(err).To(BeNil())
			Expect(strings.HasPrefix((string(body)), "GOV.UK Crawler Worker")).To(BeTrue())
		})

		It("returns a body with no errors for 200 OK responses", func() {
			ts := testServer(http.StatusOK, "Hello world")
			defer ts.Close()

			body, err := crawler.Crawl(ts.URL)

			Expect(err).To(BeNil())
			Expect(strings.TrimSpace(string(body))).To(Equal("Hello world"))
		})

		It("doesn't allow crawling a URL that doesn't match the root URL", func() {
			body, err := crawler.Crawl("http://google.com/foo")

			Expect(err).To(Equal(CannotCrawlURL))
			Expect(body).To(Equal([]byte{}))
		})

		Describe("returning a retry error", func() {
			It("returns a retry error if we get a response code of Too Many Requests", func() {
				ts := testServer(429, "Too Many Requests")
				defer ts.Close()

				body, err := crawler.Crawl(ts.URL)

				Expect(err).To(Equal(RetryRequestError))
				Expect(body).To(Equal([]byte{}))
			})

			It("returns a retry error if we get a response code of Internal Server Error", func() {
				ts := testServer(http.StatusInternalServerError, "Internal Server Error")
				defer ts.Close()

				body, err := crawler.Crawl(ts.URL)

				Expect(err).To(Equal(RetryRequestError))
				Expect(body).To(Equal([]byte{}))
			})

			It("returns a retry error if we get a response code of Gateway Timeout", func() {
				ts := testServer(http.StatusGatewayTimeout, "Gateway Timeout")
				defer ts.Close()

				body, err := crawler.Crawl(ts.URL)

				Expect(err).To(Equal(RetryRequestError))
				Expect(body).To(Equal([]byte{}))
			})
		})
	})
})
<|endoftext|>


<commit_before>package orchestrators

import (
	"encoding/json"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"sort"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/camptocamp/bivac/handler"
	"github.com/camptocamp/bivac/volume"

	log "github.com/Sirupsen/logrus"
	"github.com/rancher/go-rancher/v2"
	"golang.org/x/net/websocket"
)

// CattleOrchestrator implements a container orchestrator for Cattle
type CattleOrchestrator struct {
	Handler *handler.Bivac
	Client  *client.RancherClient
}

// NewCattleOrchestrator creates a Cattle client
func NewCattleOrchestrator(c *handler.Bivac) (o *CattleOrchestrator) {
	var err error
	o = &CattleOrchestrator{
		Handler: c,
	}

	o.Client, err = client.NewRancherClient(&client.ClientOpts{
		Url:       o.Handler.Config.Cattle.URL,
		AccessKey: o.Handler.Config.Cattle.AccessKey,
		SecretKey: o.Handler.Config.Cattle.SecretKey,
	})
	if err != nil {
		log.Errorf("failed to create a new Rancher client: %s", err)
	}

	return
}

// GetName returns the orchestrator name
func (*CattleOrchestrator) GetName() string {
	return "Cattle"
}

// GetHandler returns the Orchestrator's handler
func (o *CattleOrchestrator) GetHandler() *handler.Bivac {
	return o.Handler
}

// GetVolumes returns the Cattle volumes
func (o *CattleOrchestrator) GetVolumes() (volumes []*volume.Volume, err error) {
	c := o.Handler

	vs, err := o.Client.Volume.List(&client.ListOpts{})
	if err != nil {
		log.Errorf("failed to list volumes: %s", err)
	}

	var mountpoint string
	for _, v := range vs.Data {
		if len(v.Mounts) < 1 {
			mountpoint = "/data"
		} else {
			mountpoint = v.Mounts[0].Path
		}

		var hostID, hostname string
		var spc *client.StoragePoolCollection
		err := o.rawAPICall("GET", v.Links["storagePools"], &spc)
		if err != nil {
			log.Errorf("failed to retrieve storage pool from volume %s: %s", v.Name, err)
			continue
		}

		if len(spc.Data) == 0 {
			log.Errorf("no storage pool for the volume %s: %s", v.Name, err)
			continue
		}

		if len(spc.Data[0].HostIds) == 0 {
			log.Errorf("no host for the volume %s: %s", v.Name, err)
			continue
		}

		hostID = spc.Data[0].HostIds[0]

		h, err := o.Client.Host.ById(hostID)
		if err != nil {
			log.Errorf("failed to retrieve host from id %s: %s", hostID, err)
			hostname = ""
		} else {
			hostname = h.Hostname
		}

		nv := &volume.Volume{
			Config:     &volume.Config{},
			Mountpoint: mountpoint,
			Name:       v.Name,
			HostBind:   hostID,
			Hostname:   hostname,
		}

		v := volume.NewVolume(nv, c.Config, c.Hostname)
		if b, r, s := o.blacklistedVolume(v); b {
			log.WithFields(log.Fields{
				"volume": v.Name,
				"reason": r,
				"source": s,
			}).Info("Ignoring volume")
			continue
		}
		volumes = append(volumes, v)
	}
	return
}

// LaunchContainer starts a containe using the Cattle orchestrator
func (o *CattleOrchestrator) LaunchContainer(image string, env map[string]string, cmd []string, volumes []*volume.Volume) (state int, stdout string, err error) {
	environment := make(map[string]interface{}, len(env))
	for envKey, envVal := range env {
		environment[envKey] = envVal
	}

	var hostbind string
	if len(volumes) > 0 {
		hostbind = volumes[0].HostBind
	} else {
		hostbind = ""
	}

	cvs := []string{}
	for _, v := range volumes {
		cvs = append(cvs, v.Name+":"+v.Mountpoint)
	}

	container, err := o.Client.Container.Create(&client.Container{
		HostId:      hostbind,
		ImageUuid:   "docker:" + image,
		Command:     cmd,
		Environment: environment,
		RestartPolicy: &client.RestartPolicy{
			MaximumRetryCount: 1,
			Name:              "on-failure",
		},
		DataVolumes: cvs,
	})
	if err != nil {
		log.Errorf("failed to create worker container: %s", err)
	}

	defer o.DeleteWorker(container)

	stopped := false
	terminated := false
	for !terminated {
		container, err := o.Client.Container.ById(container.Id)
		if err != nil {
			log.Errorf("failed to inspect worker: %s", err)
		}

		// This workaround is awful but it's the only way to know if the container failed.
		if container.State == "stopped" {
			if container.StartCount == 1 {
				if stopped == false {
					stopped = true
					time.Sleep(5 * time.Second)
				} else {
					terminated = true
					state = 0
				}
			} else {
				state = 1
				terminated = true
			}
		}
	}

	var hostAccess *client.HostAccess
	err = o.rawAPICall("POST", container.Links["self"]+"/?action=logs", &hostAccess)
	if err != nil {
		log.Errorf("failed to read response from rancher: %s", err)
	}

	origin := o.Handler.Config.Cattle.URL

	u, err := url.Parse(hostAccess.Url)
	if err != nil {
		log.Errorf("failed to parse rancher server url: %s", err)
	}
	q := u.Query()
	q.Set("token", hostAccess.Token)
	u.RawQuery = q.Encode()

	ws, err := websocket.Dial(u.String(), "", origin)
	if err != nil {
		log.Errorf("failed to open websocket with rancher server: %s", err)
	}

	var data = make([]byte, 1024)
	var n int
	if n, err = ws.Read(data); err != nil && err.Error() != "EOF" {
		log.Errorf("failed to retrieve logs: %s", err)
	}
	stdout = string(data[:n])
	log.WithFields(log.Fields{
		"container": container.Id,
		"volumes":   strings.Join(cvs[:], ","),
		"cmd":       strings.Join(cmd[:], " "),
	}).Debug(stdout)
	return
}

// DeleteWorker deletes a worker
func (o *CattleOrchestrator) DeleteWorker(container *client.Container) {
	err := o.Client.Container.Delete(container)
	if err != nil {
		log.Errorf("failed to delete worker: %s", err)
	}
	removed := false
	for !removed {
		container, err := o.Client.Container.ById(container.Id)
		if err != nil {
			log.Errorf("failed to inspect worker: %s", err)
		}
		if container.Removed != "" {
			removed = true
		}
	}
	return
}

// GetMountedVolumes returns mounted volumes
func (o *CattleOrchestrator) GetMountedVolumes() (containers []*volume.MountedVolumes, err error) {
	c, err := o.Client.Container.List(&client.ListOpts{})
	if err != nil {
		log.Errorf("failed to list containers: %s", err)
	}

	for _, container := range c.Data {
		mv := &volume.MountedVolumes{
			ContainerID: container.Id,
			Volumes:     make(map[string]string),
		}
		for _, mount := range container.Mounts {
			mv.Volumes[mount.VolumeName] = mount.Path
		}
		containers = append(containers, mv)
	}
	return
}

// ContainerExec executes a command in a container
func (o *CattleOrchestrator) ContainerExec(mountedVolumes *volume.MountedVolumes, command []string) (err error) {

	container, err := o.Client.Container.ById(mountedVolumes.ContainerID)
	if err != nil {
		log.Errorf("failed to retrieve container: %s", err)
		return
	}

	hostAccess, err := o.Client.Container.ActionExecute(container, &client.ContainerExec{
		AttachStdin:  false,
		AttachStdout: true,
		Command:      command,
		Tty:          false,
	})
	if err != nil {
		log.Errorf("failed to prepare command execution in container: %s", err)
		return
	}

	origin := o.Handler.Config.Cattle.URL

	u, err := url.Parse(hostAccess.Url)
	if err != nil {
		log.Errorf("failed to parse rancher server url: %s", err)
	}
	q := u.Query()
	q.Set("token", hostAccess.Token)
	u.RawQuery = q.Encode()

	ws, err := websocket.Dial(u.String(), "", origin)
	if err != nil {
		log.Errorf("failed to open websocket with rancher server: %s", err)
	}

	var data = make([]byte, 1024)
	var n int
	if n, err = ws.Read(data); err != nil && err.Error() != "EOF" {
		log.Errorf("failed to retrieve logs: %s", err)
	}
	log.WithFields(log.Fields{
		"container": mountedVolumes.ContainerID,
		"cmd":       strings.Join(command[:], " "),
	}).Debug(string(data[:n]))
	return
}

func (o *CattleOrchestrator) blacklistedVolume(vol *volume.Volume) (bool, string, string) {

	defaultBlacklistedVolumes := []string{
		"duplicity_cache",
		"restic_cache",
		"duplicity-cache",
		"restic-cache",
		"lost+found",
	}

	if utf8.RuneCountInString(vol.Name) == 64 || utf8.RuneCountInString(vol.Name) == 0 {
		return true, "unnamed", ""
	}

	if strings.Contains(vol.Name, "/") {
		return true, "blacklisted", "path"
	}

	list := o.Handler.Config.VolumesBlacklist
	list = append(list, defaultBlacklistedVolumes...)
	sort.Strings(list)
	i := sort.SearchStrings(list, vol.Name)
	if i < len(list) && list[i] == vol.Name {
		return true, "blacklisted", "blacklist config"
	}

	if vol.Config.Ignore {
		return true, "blacklisted", "volume config"
	}

	return false, "", ""
}

func (o *CattleOrchestrator) rawAPICall(method, endpoint string, object interface{}) (err error) {
	// TODO: Use go-rancher.
	// It was impossible to use it, maybe a problem in go-rancher or a lack of documentation.
	clientHTTP := &http.Client{}
	v := url.Values{}
	req, err := http.NewRequest(method, endpoint, strings.NewReader(v.Encode()))
	req.SetBasicAuth(o.Handler.Config.Cattle.AccessKey, o.Handler.Config.Cattle.SecretKey)
	resp, err := clientHTTP.Do(req)
	if err != nil {
		log.Errorf("failed to execute POST request: %s", err)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Errorf("failed to read response from rancher: %s", err)
	}
	err = json.Unmarshal(body, object)
	if err != nil {
		log.Errorf("failed to unmarshal: %s", err)
	}
	return
}

func detectCattle() bool {
	_, err := net.LookupHost("rancher-metadata")
	if err != nil {
		return false
	}
	return true
}
<commit_msg>Fix pagination issue when using Cattle orchestrator (fixes #187)<commit_after>package orchestrators

import (
	"encoding/json"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"sort"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/camptocamp/bivac/handler"
	"github.com/camptocamp/bivac/volume"

	log "github.com/Sirupsen/logrus"
	"github.com/rancher/go-rancher/v2"
	"golang.org/x/net/websocket"
)

// CattleOrchestrator implements a container orchestrator for Cattle
type CattleOrchestrator struct {
	Handler *handler.Bivac
	Client  *client.RancherClient
}

// NewCattleOrchestrator creates a Cattle client
func NewCattleOrchestrator(c *handler.Bivac) (o *CattleOrchestrator) {
	var err error
	o = &CattleOrchestrator{
		Handler: c,
	}

	o.Client, err = client.NewRancherClient(&client.ClientOpts{
		Url:       o.Handler.Config.Cattle.URL,
		AccessKey: o.Handler.Config.Cattle.AccessKey,
		SecretKey: o.Handler.Config.Cattle.SecretKey,
	})
	if err != nil {
		log.Errorf("failed to create a new Rancher client: %s", err)
	}

	return
}

// GetName returns the orchestrator name
func (*CattleOrchestrator) GetName() string {
	return "Cattle"
}

// GetHandler returns the Orchestrator's handler
func (o *CattleOrchestrator) GetHandler() *handler.Bivac {
	return o.Handler
}

// GetVolumes returns the Cattle volumes
func (o *CattleOrchestrator) GetVolumes() (volumes []*volume.Volume, err error) {
	c := o.Handler

	vs, err := o.Client.Volume.List(&client.ListOpts{
		Filters: map[string]interface{}{
			"limit": -2,
			"all":   true,
		},
	})
	if err != nil {
		log.Errorf("failed to list volumes: %s", err)
	}

	var mountpoint string
	for _, v := range vs.Data {
		if len(v.Mounts) < 1 {
			mountpoint = "/data"
		} else {
			mountpoint = v.Mounts[0].Path
		}

		var hostID, hostname string
		var spc *client.StoragePoolCollection
		err := o.rawAPICall("GET", v.Links["storagePools"], &spc)
		if err != nil {
			log.Errorf("failed to retrieve storage pool from volume %s: %s", v.Name, err)
			continue
		}

		if len(spc.Data) == 0 {
			log.Errorf("no storage pool for the volume %s: %s", v.Name, err)
			continue
		}

		if len(spc.Data[0].HostIds) == 0 {
			log.Errorf("no host for the volume %s: %s", v.Name, err)
			continue
		}

		hostID = spc.Data[0].HostIds[0]

		h, err := o.Client.Host.ById(hostID)
		if err != nil {
			log.Errorf("failed to retrieve host from id %s: %s", hostID, err)
			hostname = ""
		} else {
			hostname = h.Hostname
		}

		nv := &volume.Volume{
			Config:     &volume.Config{},
			Mountpoint: mountpoint,
			Name:       v.Name,
			HostBind:   hostID,
			Hostname:   hostname,
		}

		v := volume.NewVolume(nv, c.Config, c.Hostname)
		if b, r, s := o.blacklistedVolume(v); b {
			log.WithFields(log.Fields{
				"volume": v.Name,
				"reason": r,
				"source": s,
			}).Info("Ignoring volume")
			continue
		}
		volumes = append(volumes, v)
	}
	return
}

// LaunchContainer starts a containe using the Cattle orchestrator
func (o *CattleOrchestrator) LaunchContainer(image string, env map[string]string, cmd []string, volumes []*volume.Volume) (state int, stdout string, err error) {
	environment := make(map[string]interface{}, len(env))
	for envKey, envVal := range env {
		environment[envKey] = envVal
	}

	var hostbind string
	if len(volumes) > 0 {
		hostbind = volumes[0].HostBind
	} else {
		hostbind = ""
	}

	cvs := []string{}
	for _, v := range volumes {
		cvs = append(cvs, v.Name+":"+v.Mountpoint)
	}

	container, err := o.Client.Container.Create(&client.Container{
		HostId:      hostbind,
		ImageUuid:   "docker:" + image,
		Command:     cmd,
		Environment: environment,
		RestartPolicy: &client.RestartPolicy{
			MaximumRetryCount: 1,
			Name:              "on-failure",
		},
		DataVolumes: cvs,
	})
	if err != nil {
		log.Errorf("failed to create worker container: %s", err)
	}

	defer o.DeleteWorker(container)

	stopped := false
	terminated := false
	for !terminated {
		container, err := o.Client.Container.ById(container.Id)
		if err != nil {
			log.Errorf("failed to inspect worker: %s", err)
		}

		// This workaround is awful but it's the only way to know if the container failed.
		if container.State == "stopped" {
			if container.StartCount == 1 {
				if stopped == false {
					stopped = true
					time.Sleep(5 * time.Second)
				} else {
					terminated = true
					state = 0
				}
			} else {
				state = 1
				terminated = true
			}
		}
	}

	var hostAccess *client.HostAccess
	err = o.rawAPICall("POST", container.Links["self"]+"/?action=logs", &hostAccess)
	if err != nil {
		log.Errorf("failed to read response from rancher: %s", err)
	}

	origin := o.Handler.Config.Cattle.URL

	u, err := url.Parse(hostAccess.Url)
	if err != nil {
		log.Errorf("failed to parse rancher server url: %s", err)
	}
	q := u.Query()
	q.Set("token", hostAccess.Token)
	u.RawQuery = q.Encode()

	ws, err := websocket.Dial(u.String(), "", origin)
	if err != nil {
		log.Errorf("failed to open websocket with rancher server: %s", err)
	}

	var data = make([]byte, 1024)
	var n int
	if n, err = ws.Read(data); err != nil && err.Error() != "EOF" {
		log.Errorf("failed to retrieve logs: %s", err)
	}
	stdout = string(data[:n])
	log.WithFields(log.Fields{
		"container": container.Id,
		"volumes":   strings.Join(cvs[:], ","),
		"cmd":       strings.Join(cmd[:], " "),
	}).Debug(stdout)
	return
}

// DeleteWorker deletes a worker
func (o *CattleOrchestrator) DeleteWorker(container *client.Container) {
	err := o.Client.Container.Delete(container)
	if err != nil {
		log.Errorf("failed to delete worker: %s", err)
	}
	removed := false
	for !removed {
		container, err := o.Client.Container.ById(container.Id)
		if err != nil {
			log.Errorf("failed to inspect worker: %s", err)
		}
		if container.Removed != "" {
			removed = true
		}
	}
	return
}

// GetMountedVolumes returns mounted volumes
func (o *CattleOrchestrator) GetMountedVolumes() (containers []*volume.MountedVolumes, err error) {
	c, err := o.Client.Container.List(&client.ListOpts{
		Filters: map[string]interface{}{
			"limit": -2,
			"all":   true,
		},
	})

	if err != nil {
		log.Errorf("failed to list containers: %s", err)
	}

	for _, container := range c.Data {
		mv := &volume.MountedVolumes{
			ContainerID: container.Id,
			Volumes:     make(map[string]string),
		}
		for _, mount := range container.Mounts {
			mv.Volumes[mount.VolumeName] = mount.Path
		}
		containers = append(containers, mv)
	}
	return
}

// ContainerExec executes a command in a container
func (o *CattleOrchestrator) ContainerExec(mountedVolumes *volume.MountedVolumes, command []string) (err error) {

	container, err := o.Client.Container.ById(mountedVolumes.ContainerID)
	if err != nil {
		log.Errorf("failed to retrieve container: %s", err)
		return
	}

	hostAccess, err := o.Client.Container.ActionExecute(container, &client.ContainerExec{
		AttachStdin:  false,
		AttachStdout: true,
		Command:      command,
		Tty:          false,
	})
	if err != nil {
		log.Errorf("failed to prepare command execution in container: %s", err)
		return
	}

	origin := o.Handler.Config.Cattle.URL

	u, err := url.Parse(hostAccess.Url)
	if err != nil {
		log.Errorf("failed to parse rancher server url: %s", err)
	}
	q := u.Query()
	q.Set("token", hostAccess.Token)
	u.RawQuery = q.Encode()

	ws, err := websocket.Dial(u.String(), "", origin)
	if err != nil {
		log.Errorf("failed to open websocket with rancher server: %s", err)
	}

	var data = make([]byte, 1024)
	var n int
	if n, err = ws.Read(data); err != nil && err.Error() != "EOF" {
		log.Errorf("failed to retrieve logs: %s", err)
	}
	log.WithFields(log.Fields{
		"container": mountedVolumes.ContainerID,
		"cmd":       strings.Join(command[:], " "),
	}).Debug(string(data[:n]))
	return
}

func (o *CattleOrchestrator) blacklistedVolume(vol *volume.Volume) (bool, string, string) {

	defaultBlacklistedVolumes := []string{
		"duplicity_cache",
		"restic_cache",
		"duplicity-cache",
		"restic-cache",
		"lost+found",
	}

	if utf8.RuneCountInString(vol.Name) == 64 || utf8.RuneCountInString(vol.Name) == 0 {
		return true, "unnamed", ""
	}

	if strings.Contains(vol.Name, "/") {
		return true, "blacklisted", "path"
	}

	list := o.Handler.Config.VolumesBlacklist
	list = append(list, defaultBlacklistedVolumes...)
	sort.Strings(list)
	i := sort.SearchStrings(list, vol.Name)
	if i < len(list) && list[i] == vol.Name {
		return true, "blacklisted", "blacklist config"
	}

	if vol.Config.Ignore {
		return true, "blacklisted", "volume config"
	}

	return false, "", ""
}

func (o *CattleOrchestrator) rawAPICall(method, endpoint string, object interface{}) (err error) {
	// TODO: Use go-rancher.
	// It was impossible to use it, maybe a problem in go-rancher or a lack of documentation.
	clientHTTP := &http.Client{}
	v := url.Values{}
	req, err := http.NewRequest(method, endpoint, strings.NewReader(v.Encode()))
	req.SetBasicAuth(o.Handler.Config.Cattle.AccessKey, o.Handler.Config.Cattle.SecretKey)
	resp, err := clientHTTP.Do(req)
	if err != nil {
		log.Errorf("failed to execute POST request: %s", err)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Errorf("failed to read response from rancher: %s", err)
	}
	err = json.Unmarshal(body, object)
	if err != nil {
		log.Errorf("failed to unmarshal: %s", err)
	}
	return
}

func detectCattle() bool {
	_, err := net.LookupHost("rancher-metadata")
	if err != nil {
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package node

import (
	"bytes"
	"fmt"
	"github.com/AutoRoute/l2"
)

// The layer two protocol takes a layer two device and returns the hash of the
// Public Key of all neighbors it can find.
type NeighborFinder interface {
	Find(l2.FrameReadWriter) <-chan string
}

type NeighborData struct {
	pk PublicKey
}

func NewNeighborData(pk PublicKey) NeighborData {
	return NeighborData{pk}
}

func (n NeighborData) Find(mac string, frw l2.FrameReadWriter) <-chan string {
	c := make(chan string)
	// Broadcast Hash
	broadcastAddr := l2.macToBytesOrDie("ff:ff:ff:ff:ff:ff")
	localAddr := l2.macToBytesOrDie(mac) // TODO: decide on mac passing before merging
	var protocol uint16 = 31337          // TODO: add real protocol
	publicKeyHash := []byte(n.pk.Hash())
	initFrame := l2.NewEthFrame(broadcastAddr, localAddr, protocol, publicKeyHash)
	fmt.Println("Broadcasting packet.")
	var err error = frw.WriteFrame(initFrame) // TODO: check errors
	if err != nil {
		panic(err)
	}
	fmt.Println("Broadcasted packet.")
	// Process Loop
	go func() {
		for {
			fmt.Println("Receiving packet.")
			newInstanceFrame, _ := frw.ReadFrame()
			src := newInstanceFrame.Source()
			dest := newInstanceFrame.Destination()
			fmt.Printf("Received packet from %v.\n", src)
			fmt.Printf("Received packet to %v.\n", dest)
			if newInstanceFrame.Type() != protocol {
				continue // Throw away if protocols don't match
			}
			if bytes.Equal(src, localAddr) {
				continue // Throw away if from me
			}
			if !(bytes.Equal(dest, localAddr) || bytes.Equal(dest, broadcastAddr)) {
				continue // Throw away if it wasn't to me or the broadcast address
			}
			c <- string(newInstanceFrame.Data())
			if bytes.Equal(dest, broadcastAddr) { // Respond if to broadcast addr
				var p PublicKey = pktest("test2") // TODO: pass public key
				publicKeyHash := []byte(p.Hash())
				initFrame := l2.NewEthFrame(src, localAddr, 31337, publicKeyHash) // TODO: add real protocol
				fmt.Printf("Sending response packet %v.\n", src)
				var err error = frw.WriteFrame(initFrame) // TODO: check errors
				if err != nil {
					panic(err)
				}
				fmt.Println("Sent response packet.")
			}
		}
	}()
	return c
}
<commit_msg>Patched neighbor_finder.go with version from master.<commit_after>package node

import (
	"bytes"
	"fmt"
	"github.com/AutoRoute/l2"
)

// The layer two protocol takes a layer two device and returns the hash of the
// Public Key of all neighbors it can find.
type NeighborFinder interface {
	Find(l2.FrameReadWriter) <-chan string
}

type NeighborData struct {
	pk PublicKey
}

func NewNeighborData(pk PublicKey) NeighborData {
	return NeighborData{pk}
}
func (n NeighborData) Find(mac string, frw l2.FrameReadWriter) (<-chan string, error) {
	c := make(chan string)
	// Broadcast Hash
	broadcastAddr, errb := l2.MacToBytes("ff:ff:ff:ff:ff:ff")
	if errb != nil {
		return c, errb
	}
	localAddr, errl := l2.MacToBytes(mac) // TODO: decide on mac passing before merging
	if errl != nil {
		return c, errl
	}
	var protocol uint16 = 31337 // TODO: add real protocol
	publicKeyHash := []byte(n.pk.Hash())
	initFrame := l2.NewEthFrame(broadcastAddr, localAddr, protocol, publicKeyHash)
	fmt.Println("Broadcasting packet.")
	var err error = frw.WriteFrame(initFrame) // TODO: check errors
	if err != nil {
		panic(err)
	}
	fmt.Println("Broadcasted packet.")
	// Process Loop
	go func() {
		for {
			fmt.Println("Receiving packet.")
			newInstanceFrame, _ := frw.ReadFrame()
			src := newInstanceFrame.Source()
			dest := newInstanceFrame.Destination()
			fmt.Printf("Received packet from %v.\n", src)
			fmt.Printf("Received packet to %v.\n", dest)
			if newInstanceFrame.Type() != protocol {
				continue // Throw away if protocols don't match
			}
			if bytes.Equal(src, localAddr) {
				continue // Throw away if from me
			}
			if !(bytes.Equal(dest, localAddr) || bytes.Equal(dest, broadcastAddr)) {
				continue // Throw away if it wasn't to me or the broadcast address
			}
			c <- string(newInstanceFrame.Data())
			if bytes.Equal(dest, broadcastAddr) { // Respond if to broadcast addr
				var p PublicKey = pktest("test2") // TODO: pass public key
				publicKeyHash := []byte(p.Hash())
				initFrame := l2.NewEthFrame(src, localAddr, 31337, publicKeyHash) // TODO: add real protocol
				fmt.Printf("Sending response packet %v.\n", src)
				var err error = frw.WriteFrame(initFrame) // TODO: check errors
				if err != nil {
					panic(err)
				}
				fmt.Println("Sent response packet.")
			}
		}
	}()
	return c, nil // TODO: return channel error?
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"fmt"
	"reflect"
	"sort"
	"strings"
	"testing"

	"github.com/prometheus/client_golang/prometheus"
	dto "github.com/prometheus/client_model/go"
	"github.com/prometheus/common/expfmt"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apis/extensions"
)

type mockDeploymentStore struct {
	f func() ([]extensions.Deployment, error)
}

func (ds mockDeploymentStore) List() (deployments []extensions.Deployment, err error) {
	return ds.f()
}

func TestDeploymentCollector(t *testing.T) {
	// Fixed metadata on type and help text. We prepend this to every expected
	// output so we only have to modify a single place when doing adjustments.
	const metadata = `
		# HELP deployment_replicas The number of replicas per deployment.
		# TYPE deployment_replicas gauge
		# HELP deployment_replicas_available The number of available replicas per deployment.
		# TYPE deployment_replicas_available gauge
	`
	cases := []struct {
		depls []extensions.Deployment
		want  string
	}{
		{
			depls: []extensions.Deployment{
				{
					ObjectMeta: api.ObjectMeta{
						Name:      "depl1",
						Namespace: "ns1",
					},
					Status: extensions.DeploymentStatus{
						Replicas:          15,
						AvailableReplicas: 10,
					},
				}, {
					ObjectMeta: api.ObjectMeta{
						Name:      "depl2",
						Namespace: "ns2",
					},
					Status: extensions.DeploymentStatus{
						Replicas:          10,
						AvailableReplicas: 5,
					},
				}, {
					ObjectMeta: api.ObjectMeta{
						Name:      "depl3",
						Namespace: "ns2",
					},
					Status: extensions.DeploymentStatus{
						Replicas:          1,
						AvailableReplicas: 0,
					},
				},
			},
			want: metadata + `
				deployment_replicas{namespace="ns1",deployment="depl1"} 15
				deployment_replicas{namespace="ns2",deployment="depl2"} 10
				deployment_replicas{namespace="ns2",deployment="depl3"} 1
				deployment_replicas_available{namespace="ns2",deployment="depl2"} 5
				deployment_replicas_available{namespace="ns1",deployment="depl1"} 10
				deployment_replicas_available{namespace="ns2",deployment="depl3"} 0
			`,
		},
	}
	for _, c := range cases {
		dc := &deploymentCollector{
			store: mockDeploymentStore{
				f: func() ([]extensions.Deployment, error) { return c.depls, nil },
			},
		}
		if err := gatherAndCompare(dc, c.want); err != nil {
			t.Errorf("unexpected collecting result:\n%s", err)
		}
	}
}

// gatherAndCompare retrieves all metrics exposed by a collector and compares it
// to an expected output in the Prometheus text exposition format.
func gatherAndCompare(c prometheus.Collector, expected string) error {
	expected = removeUnusedWhitespace(expected)

	reg := prometheus.NewPedanticRegistry()
	if err := reg.Register(c); err != nil {
		return fmt.Errorf("registering collector failed: %s", err)
	}
	metrics, err := reg.Gather()
	if err != nil {
		return fmt.Errorf("gathering metrics failed: %s", err)
	}
	var tp expfmt.TextParser
	expectedMetrics, err := tp.TextToMetricFamilies(bytes.NewReader([]byte(expected)))
	if err != nil {
		return fmt.Errorf("parsing expected metrics failed: %s", err)
	}

	// Compare the sorted gathering result with the parsed expected result.
	// Apply the same normalization to the expected output as the client library
	// does to the gathering output.
	if !reflect.DeepEqual(metrics, normalizeMetricFamilies(expectedMetrics)) {
		// Encode the gathered output to the readbale text format for comparison.
		var buf bytes.Buffer
		enc := expfmt.NewEncoder(&buf, expfmt.FmtText)
		for _, mf := range metrics {
			if err := enc.Encode(mf); err != nil {
				return fmt.Errorf("encoding result failed: %s", err)
			}
		}

		return fmt.Errorf(`
metric output does not match expectation; want:

%s

got:

%s
`, expected, buf.String())
	}
	return nil
}

func removeUnusedWhitespace(s string) string {
	var (
		trimmedLine  string
		trimmedLines []string
		lines        []string = strings.Split(s, "\n")
	)

	for _, l := range lines {
		trimmedLine = strings.TrimSpace(l)

		if len(trimmedLine) > 0 {
			trimmedLines = append(trimmedLines, trimmedLine)
		}
	}

	// The Prometheus metrics representation parser expects an empty line at the
	// end otherwise fails with an unexpected EOF error.
	return strings.Join(trimmedLines, "\n") + "\n"
}

// The below sorting code is copied form the Prometheus client library modulo the added
// label pair sorting.
// https://github.com/prometheus/client_golang/blob/ea6e1db4cb8127eeb0b6954f7320363e5451820f/prometheus/registry.go#L642-L684

// metricSorter is a sortable slice of *dto.Metric.
type metricSorter []*dto.Metric

func (s metricSorter) Len() int {
	return len(s)
}

func (s metricSorter) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s metricSorter) Less(i, j int) bool {
	if len(s[i].Label) != len(s[j].Label) {
		return len(s[i].Label) < len(s[j].Label)
	}
	for n, lp := range s[i].Label {
		vi := lp.GetValue()
		vj := s[j].Label[n].GetValue()
		if vi != vj {
			return vi < vj
		}
	}

	if s[i].TimestampMs == nil {
		return false
	}
	if s[j].TimestampMs == nil {
		return true
	}
	return s[i].GetTimestampMs() < s[j].GetTimestampMs()
}

// normalizeMetricFamilies returns a MetricFamily slice whith empty
// MetricFamilies pruned and the remaining MetricFamilies sorted by name within
// the slice, with the contained Metrics sorted within each MetricFamily.
func normalizeMetricFamilies(metricFamiliesByName map[string]*dto.MetricFamily) []*dto.MetricFamily {
	for _, mf := range metricFamiliesByName {
		sort.Sort(metricSorter(mf.Metric))
	}
	names := make([]string, 0, len(metricFamiliesByName))
	for name, mf := range metricFamiliesByName {
		if len(mf.Metric) > 0 {
			names = append(names, name)
			for _, m := range mf.Metric {
				sort.Sort(prometheus.LabelPairSorter(m.Label))
			}
		}
	}
	sort.Strings(names)
	result := make([]*dto.MetricFamily, 0, len(names))
	for _, name := range names {
		result = append(result, metricFamiliesByName[name])
	}
	return result
}
<commit_msg>Fix metric equality checks<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"fmt"
	"reflect"
	"sort"
	"strings"
	"testing"

	"github.com/prometheus/client_golang/prometheus"
	dto "github.com/prometheus/client_model/go"
	"github.com/prometheus/common/expfmt"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apis/extensions"
)

type mockDeploymentStore struct {
	f func() ([]extensions.Deployment, error)
}

func (ds mockDeploymentStore) List() (deployments []extensions.Deployment, err error) {
	return ds.f()
}

func TestDeploymentCollector(t *testing.T) {
	// Fixed metadata on type and help text. We prepend this to every expected
	// output so we only have to modify a single place when doing adjustments.
	const metadata = `
		# HELP deployment_replicas The number of replicas per deployment.
		# TYPE deployment_replicas gauge
		# HELP deployment_replicas_available The number of available replicas per deployment.
		# TYPE deployment_replicas_available gauge
	`
	cases := []struct {
		depls []extensions.Deployment
		want  string
	}{
		{
			depls: []extensions.Deployment{
				{
					ObjectMeta: api.ObjectMeta{
						Name:      "depl1",
						Namespace: "ns1",
					},
					Status: extensions.DeploymentStatus{
						Replicas:          15,
						AvailableReplicas: 10,
					},
				}, {
					ObjectMeta: api.ObjectMeta{
						Name:      "depl2",
						Namespace: "ns2",
					},
					Status: extensions.DeploymentStatus{
						Replicas:          10,
						AvailableReplicas: 5,
					},
				}, {
					ObjectMeta: api.ObjectMeta{
						Name:      "depl3",
						Namespace: "ns2",
					},
					Status: extensions.DeploymentStatus{
						Replicas:          1,
						AvailableReplicas: 0,
					},
				},
			},
			want: metadata + `
				deployment_replicas{namespace="ns1",deployment="depl1"} 15
				deployment_replicas{namespace="ns2",deployment="depl2"} 10
				deployment_replicas{namespace="ns2",deployment="depl3"} 1
				deployment_replicas_available{namespace="ns2",deployment="depl2"} 5
				deployment_replicas_available{namespace="ns1",deployment="depl1"} 10
				deployment_replicas_available{namespace="ns2",deployment="depl3"} 0
			`,
		},
	}
	for _, c := range cases {
		dc := &deploymentCollector{
			store: mockDeploymentStore{
				f: func() ([]extensions.Deployment, error) { return c.depls, nil },
			},
		}
		if err := gatherAndCompare(dc, c.want); err != nil {
			t.Errorf("unexpected collecting result:\n%s", err)
		}
	}
}

// gatherAndCompare retrieves all metrics exposed by a collector and compares it
// to an expected output in the Prometheus text exposition format.
func gatherAndCompare(c prometheus.Collector, expected string) error {
	expected = removeUnusedWhitespace(expected)

	reg := prometheus.NewPedanticRegistry()
	if err := reg.Register(c); err != nil {
		return fmt.Errorf("registering collector failed: %s", err)
	}
	metrics, err := reg.Gather()
	if err != nil {
		return fmt.Errorf("gathering metrics failed: %s", err)
	}
	var tp expfmt.TextParser
	expectedMetrics, err := tp.TextToMetricFamilies(bytes.NewReader([]byte(expected)))
	if err != nil {
		return fmt.Errorf("parsing expected metrics failed: %s", err)
	}

	if !reflect.DeepEqual(metrics, normalizeMetricFamilies(expectedMetrics)) {
		// Encode the gathered output to the readbale text format for comparison.
		var buf1 bytes.Buffer
		enc := expfmt.NewEncoder(&buf1, expfmt.FmtText)
		for _, mf := range metrics {
			if err := enc.Encode(mf); err != nil {
				return fmt.Errorf("encoding result failed: %s", err)
			}
		}
		// Encode normalized expected metrics again to generate them in the same ordering
		// the registry does to spot differences more easily.
		var buf2 bytes.Buffer
		enc = expfmt.NewEncoder(&buf2, expfmt.FmtText)
		for _, mf := range normalizeMetricFamilies(expectedMetrics) {
			if err := enc.Encode(mf); err != nil {
				return fmt.Errorf("encoding result failed: %s", err)
			}
		}

		return fmt.Errorf(`
metric output does not match expectation; want:

%s

got:

%s
`, buf2.String(), buf1.String())
	}
	return nil
}

func removeUnusedWhitespace(s string) string {
	var (
		trimmedLine  string
		trimmedLines []string
		lines        []string = strings.Split(s, "\n")
	)

	for _, l := range lines {
		trimmedLine = strings.TrimSpace(l)

		if len(trimmedLine) > 0 {
			trimmedLines = append(trimmedLines, trimmedLine)
		}
	}

	// The Prometheus metrics representation parser expects an empty line at the
	// end otherwise fails with an unexpected EOF error.
	return strings.Join(trimmedLines, "\n") + "\n"
}

// The below sorting code is copied form the Prometheus client library modulo the added
// label pair sorting.
// https://github.com/prometheus/client_golang/blob/ea6e1db4cb8127eeb0b6954f7320363e5451820f/prometheus/registry.go#L642-L684

// metricSorter is a sortable slice of *dto.Metric.
type metricSorter []*dto.Metric

func (s metricSorter) Len() int {
	return len(s)
}

func (s metricSorter) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s metricSorter) Less(i, j int) bool {
	sort.Sort(prometheus.LabelPairSorter(s[i].Label))
	sort.Sort(prometheus.LabelPairSorter(s[j].Label))

	if len(s[i].Label) != len(s[j].Label) {
		return len(s[i].Label) < len(s[j].Label)
	}

	for n, lp := range s[i].Label {
		vi := lp.GetValue()
		vj := s[j].Label[n].GetValue()
		if vi != vj {
			return vi < vj
		}
	}

	if s[i].TimestampMs == nil {
		return false
	}
	if s[j].TimestampMs == nil {
		return true
	}
	return s[i].GetTimestampMs() < s[j].GetTimestampMs()
}

// normalizeMetricFamilies returns a MetricFamily slice whith empty
// MetricFamilies pruned and the remaining MetricFamilies sorted by name within
// the slice, with the contained Metrics sorted within each MetricFamily.
func normalizeMetricFamilies(metricFamiliesByName map[string]*dto.MetricFamily) []*dto.MetricFamily {
	for _, mf := range metricFamiliesByName {
		sort.Sort(metricSorter(mf.Metric))
	}
	names := make([]string, 0, len(metricFamiliesByName))
	for name, mf := range metricFamiliesByName {
		if len(mf.Metric) > 0 {
			names = append(names, name)
		}
	}
	sort.Strings(names)
	result := make([]*dto.MetricFamily, 0, len(names))
	for _, name := range names {
		result = append(result, metricFamiliesByName[name])
	}
	return result
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/xml"
	"fmt"
	"io"
	"runtime"
	"strings"
)

type JUnitTestSuite struct {
	XMLName    xml.Name        `xml:"testsuite"`
	Tests      int             `xml:"tests,attr"`
	Failures   int             `xml:"failures,attr"`
	Time       string          `xml:"time,attr"`
	Name       string          `xml:"name,attr"`
	Properties []JUnitProperty `xml:"properties>property,omitempty"`
	TestCases  []JUnitTestCase
}

type JUnitTestCase struct {
	XMLName   xml.Name `xml:"testcase"`
	Classname string   `xml:"classname,attr"`
	Name      string   `xml:"name,attr"`
	Time      string   `xml:"time,attr"`
	Failure   string  `xml:"failure,omitempty"`
}

type JUnitProperty struct {
	Name  string `xml:"name,attr"`
	Value string `xml:"value,attr"`
}

func NewJUnitProperty(name, value string) JUnitProperty {
	return JUnitProperty{
		Name:  name,
		Value: value,
	}
}

func JUnitReportXML(report *Report, w io.Writer) error {
	suites := []JUnitTestSuite{}

	// convert Report to JUnit test suites
	for _, pkg := range report.Packages {
		ts := JUnitTestSuite{
			Tests:      len(pkg.Tests),
			Failures:   0,
			Time:       formatTime(pkg.Time),
			Name:       pkg.Name,
			Properties: []JUnitProperty{},
			TestCases:  []JUnitTestCase{},
		}

		classname := pkg.Name
		if idx := strings.LastIndex(classname, "/"); idx > -1 && idx < len(pkg.Name) {
			classname = pkg.Name[idx+1:]
		}

		// properties
		ts.Properties = append(ts.Properties, NewJUnitProperty("go.version", runtime.Version()))

		// individual test cases
		for _, test := range pkg.Tests {
			testCase := JUnitTestCase{
				Classname: classname,
				Name:      test.Name,
				Time:      formatTime(test.Time),
				Failure:   "",
			}

			if test.Result == FAIL {
				ts.Failures += 1

				// TODO: set error message
				testCase.Failure = "Failed"
			}

			ts.TestCases = append(ts.TestCases, testCase)
		}

		suites = append(suites, ts)
	}

	// to xml
	bytes, err := xml.MarshalIndent(suites, "", "\t")
	if err != nil {
		return err
	}

	writer := bufio.NewWriter(w)

	// remove newline from xml.Header, because xml.MarshalIndent starts with a newline
	writer.WriteString(xml.Header[:len(xml.Header)-1])
	writer.Write(bytes)
	writer.WriteByte('\n')
	writer.Flush()

	return nil
}

func countFailures(tests []Test) (result int) {
	for _, test := range tests {
		if test.Result == FAIL {
			result += 1
		}
	}
	return
}

func formatTime(time int) string {
	return fmt.Sprintf("%.3f", float64(time)/1000.0)
}
<commit_msg>Add xsd link to junit-formatter<commit_after>package main

import (
	"bufio"
	"encoding/xml"
	"fmt"
	"io"
	"runtime"
	"strings"
)

type JUnitTestSuite struct {
	XMLName    xml.Name        `xml:"testsuite"`
	Tests      int             `xml:"tests,attr"`
	Failures   int             `xml:"failures,attr"`
	Time       string          `xml:"time,attr"`
	Name       string          `xml:"name,attr"`
	Properties []JUnitProperty `xml:"properties>property,omitempty"`
	TestCases  []JUnitTestCase
}

type JUnitTestCase struct {
	XMLName   xml.Name `xml:"testcase"`
	Classname string   `xml:"classname,attr"`
	Name      string   `xml:"name,attr"`
	Time      string   `xml:"time,attr"`
	Failure   string  `xml:"failure,omitempty"`
}

type JUnitProperty struct {
	Name  string `xml:"name,attr"`
	Value string `xml:"value,attr"`
}

func NewJUnitProperty(name, value string) JUnitProperty {
	return JUnitProperty{
		Name:  name,
		Value: value,
	}
}

// JUnitReportXML writes a junit xml representation of the given report to w
// in the format described at http://windyroad.org/dl/Open%20Source/JUnit.xsd
func JUnitReportXML(report *Report, w io.Writer) error {
	suites := []JUnitTestSuite{}

	// convert Report to JUnit test suites
	for _, pkg := range report.Packages {
		ts := JUnitTestSuite{
			Tests:      len(pkg.Tests),
			Failures:   0,
			Time:       formatTime(pkg.Time),
			Name:       pkg.Name,
			Properties: []JUnitProperty{},
			TestCases:  []JUnitTestCase{},
		}

		classname := pkg.Name
		if idx := strings.LastIndex(classname, "/"); idx > -1 && idx < len(pkg.Name) {
			classname = pkg.Name[idx+1:]
		}

		// properties
		ts.Properties = append(ts.Properties, NewJUnitProperty("go.version", runtime.Version()))

		// individual test cases
		for _, test := range pkg.Tests {
			testCase := JUnitTestCase{
				Classname: classname,
				Name:      test.Name,
				Time:      formatTime(test.Time),
				Failure:   "",
			}

			if test.Result == FAIL {
				ts.Failures += 1

				// TODO: set error message
				testCase.Failure = "Failed"
			}

			ts.TestCases = append(ts.TestCases, testCase)
		}

		suites = append(suites, ts)
	}

	// to xml
	bytes, err := xml.MarshalIndent(suites, "", "\t")
	if err != nil {
		return err
	}

	writer := bufio.NewWriter(w)

	// remove newline from xml.Header, because xml.MarshalIndent starts with a newline
	writer.WriteString(xml.Header[:len(xml.Header)-1])
	writer.Write(bytes)
	writer.WriteByte('\n')
	writer.Flush()

	return nil
}

func countFailures(tests []Test) (result int) {
	for _, test := range tests {
		if test.Result == FAIL {
			result += 1
		}
	}
	return
}

func formatTime(time int) string {
	return fmt.Sprintf("%.3f", float64(time)/1000.0)
}
<|endoftext|>


<commit_before>package vivoupdater

import (
	"bytes"
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"fmt"
	"log"
	"time"

	"github.com/DCSO/fluxline"
	"github.com/Shopify/sarama"
)

var Subscriber *KafkaSubscriber

func GetSubscriber() *KafkaSubscriber {
	return Subscriber
}

func SetSubscriber(s *KafkaSubscriber) {
	Subscriber = s
}

func SetupConsumer(ks *KafkaSubscriber) error {
	SetSubscriber(ks)
	return nil
}

func NewTLSConfig(ks *KafkaSubscriber) (*tls.Config, error) {
	tlsConfig := tls.Config{}
	// Load client cert
	cert, err := tls.X509KeyPair([]byte(ks.ClientCert), []byte(ks.ClientKey))

	if err != nil {
		fmt.Println("error reading cert")
		return &tlsConfig, err
	}
	tlsConfig.Certificates = []tls.Certificate{cert}

	// Load CA cert - should get bytes from vault
	caCert := []byte(ks.ServerCert)

	if err != nil {
		fmt.Println("error reading caCert")
		return &tlsConfig, err
	}
	caCertPool := x509.NewCertPool()
	caCertPool.AppendCertsFromPEM(caCert)
	tlsConfig.RootCAs = caCertPool

	tlsConfig.BuildNameToCertificate()
	return &tlsConfig, err
}

func GetCertsFromVault(env string, config *VaultConfig, kafka *KafkaSubscriber) {
	if len(config.Token) == 0 {
		err := FetchToken(config)
		if err != nil {
			fmt.Printf("Unable to fetch token from vault for role_id and secret_id:\n  %s\n", err)
		}
	}

	secrets := SecretsMap(env)
	// NOTE: reads values 'into' a struct
	var values Secrets
	//NOTE: order matters, needs token
	err := FetchSecrets(config, secrets, &values)
	if err != nil {
		log.Fatal(err)
	}

	kafka.ClientCert = values.KafkaClientCert
	kafka.ClientKey = values.KafkaClientKey
	kafka.ServerCert = values.KafkaServerCert
}

type ConsumerGroupHandler struct {
	Context context.Context
	Logger  *log.Logger
	Updates chan UpdateMessage
	Cancel  context.CancelFunc
}

func (c ConsumerGroupHandler) Setup(sess sarama.ConsumerGroupSession) error {
	c.Logger.Printf("Consumer Setup callback:%s\n", sess.MemberID())
	return nil
}

func (c ConsumerGroupHandler) Cleanup(sess sarama.ConsumerGroupSession) error {
	c.Logger.Printf("Consumer Cleanup callback:%s\n", sess.MemberID())
	return nil
}

// https://github.com/Shopify/sarama/issues/1192
func (c ConsumerGroupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession,
	claim sarama.ConsumerGroupClaim) error {

	um := UpdateMessage{}

	for {
		select {
		case msg := <-claim.Messages():
			err := json.Unmarshal(msg.Value, &um)
			if err != nil {
				return err
			}
			// NOTE: could check for "" but have not seen blank ones in log
			c.Logger.Printf("uri received: %v\n", um.Triple.Subject)
			c.Updates <- um
			// marking offset
			sess.MarkMessage(msg, "")
		case <-sess.Context().Done():
			err := sess.Context().Err()
			return err
		}
	}
}

type KafkaSubscriber struct {
	Brokers    []string
	Topic      string
	ClientCert string
	ClientKey  string
	ServerCert string
	ClientID   string
	GroupName  string
}

func StartConsumer(ctx context.Context, ks KafkaSubscriber, handler ConsumerGroupHandler) error {
	sarama.Logger = handler.Logger

	tlsConfig, err := NewTLSConfig(&ks)

	if err != nil {
		handler.Logger.Fatal(err)
		return err
	}

	consumerConfig := sarama.NewConfig()
	consumerConfig.ClientID = ks.ClientID
	consumerConfig.Version = sarama.V1_0_0_0
	consumerConfig.Net.TLS.Enable = true
	consumerConfig.Net.TLS.Config = tlsConfig
	consumerConfig.Consumer.Return.Errors = true

	consumerConfig.Net.ReadTimeout = (10 * time.Second)
	consumerConfig.Net.DialTimeout = (10 * time.Second)
	consumerConfig.Net.WriteTimeout = (10 * time.Second)

	// not sure a good number of retries
	consumerConfig.Metadata.Retry.Max = 3
	consumerConfig.Metadata.Retry.Backoff = (10 * time.Second)
	consumerConfig.Metadata.RefreshFrequency = (15 * time.Minute)

	// set rebalance timeout?  - default 60ms
	//consumerConfig.Consumer.Group.Rebalance.Timeout = time.Duration(6000 * time.Millisecond)
	// set a max wait time??
	//consumerConfig.Consumer.MaxWaitTime = time.Duration(305000 * time.Millisecond)
	consumerConfig.Consumer.Offsets.Initial = sarama.OffsetNewest
	//consumerConfig.Consumer.Offsets.Initial = sarama.OffsetOldest
	client, err := sarama.NewClient(ks.Brokers, consumerConfig)
	if err != nil {
		handler.Logger.Fatalf("CLIENT ERROR:%v\n", err)
		return err
	}
	defer func() { _ = client.Close() }()

	// Start a new consumer group
	group, err := sarama.NewConsumerGroupFromClient(ks.GroupName, client)
	if err != nil {
		handler.Logger.Printf("GROUP ERROR:%v\n", err)
		return err
	}
	defer func() { _ = group.Close() }()

	// NOTE: sometimes this gives:
	// CONSUME ERR:kafka server:
	// "A rebalance for the group is in progress. Please re-join the group.
	// Closing Client, Error while closing connection to broker i/o timeout"

	// see sarama docs of this function about rebalance, retries etc...
	err = group.Consume(ctx, []string{ks.Topic}, handler)
	if err != nil {
		// NOTE: sarama example panics here
		handler.Logger.Printf("CONSUME ERR:%v\n", err)
		return err
	}
	return nil
}

func (ks KafkaSubscriber) Subscribe(ctx context.Context,
	logger *log.Logger, updates chan UpdateMessage) error {
	// NOTE: need to use channel to send to batcher
	handler := ConsumerGroupHandler{Logger: logger, Updates: updates}
	err := StartConsumer(ctx, ks, handler)
	if err != nil {
		logger.Printf("start-consumer error: %v\n", err)
		return err
	}
	return nil
}

var producer sarama.AsyncProducer

func GetProducer() sarama.AsyncProducer {
	return producer
}

func SetProducer(p sarama.AsyncProducer) {
	producer = p
}

func SetupProducer(ks *KafkaSubscriber) error {
	tlsConfig, err := NewTLSConfig(ks)
	if err != nil {
		return err
	}
	producerConfig := sarama.NewConfig()
	producerConfig.ClientID = ks.ClientID
	producerConfig.Version = sarama.V1_0_0_0
	producerConfig.Net.TLS.Enable = true
	producerConfig.Net.TLS.Config = tlsConfig

	producer, err := sarama.NewAsyncProducer(ks.Brokers, producerConfig)
	if err != nil {
		return err
	}

	SetProducer(producer)
	return nil
}

func Produce(topic string, val string) {
	msg := &sarama.ProducerMessage{Topic: topic, Value: sarama.StringEncoder(val)}
	prod := GetProducer()
	// NOTE: async
	prod.Input() <- msg
}

func FluxLine(measurement string, c interface{}, tags map[string]string) (bytes.Buffer, error) {
	var b bytes.Buffer
	encoder := fluxline.NewEncoder(&b)
	err := encoder.Encode(measurement, c, tags)
	if err != nil {
		return b, err
	}
	return b, nil
}

type IndexMetrics struct {
	Start time.Time
	End   time.Time
	Uris  []string
	Name  string
}

func SendMetrics(metrics IndexMetrics, logger *log.Logger) {
	rt := (metrics.End.Sub(metrics.Start).Seconds() * 1000.0)

	d := struct {
		Duration float64 `influx:"duration"`
		Count    int64   `influx:"count"`
	}{
		Duration: rt,
		Count:    int64(len(metrics.Uris)),
	}

	tags := map[string]string{
		"indexer": metrics.Name,
	}

	line, err := FluxLine("vivoupdater.update", d, tags)
	if err == nil {
		Produce(MetricsTopic, line.String())
	}

}
<commit_msg>FDP-2601: try creating struct each time in consumer loop message (instead of outside)<commit_after>package vivoupdater

import (
	"bytes"
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"fmt"
	"log"
	"time"

	"github.com/DCSO/fluxline"
	"github.com/Shopify/sarama"
)

var Subscriber *KafkaSubscriber

func GetSubscriber() *KafkaSubscriber {
	return Subscriber
}

func SetSubscriber(s *KafkaSubscriber) {
	Subscriber = s
}

func SetupConsumer(ks *KafkaSubscriber) error {
	SetSubscriber(ks)
	return nil
}

func NewTLSConfig(ks *KafkaSubscriber) (*tls.Config, error) {
	tlsConfig := tls.Config{}
	// Load client cert
	cert, err := tls.X509KeyPair([]byte(ks.ClientCert), []byte(ks.ClientKey))

	if err != nil {
		fmt.Println("error reading cert")
		return &tlsConfig, err
	}
	tlsConfig.Certificates = []tls.Certificate{cert}

	// Load CA cert - should get bytes from vault
	caCert := []byte(ks.ServerCert)

	if err != nil {
		fmt.Println("error reading caCert")
		return &tlsConfig, err
	}
	caCertPool := x509.NewCertPool()
	caCertPool.AppendCertsFromPEM(caCert)
	tlsConfig.RootCAs = caCertPool

	tlsConfig.BuildNameToCertificate()
	return &tlsConfig, err
}

func GetCertsFromVault(env string, config *VaultConfig, kafka *KafkaSubscriber) {
	if len(config.Token) == 0 {
		err := FetchToken(config)
		if err != nil {
			fmt.Printf("Unable to fetch token from vault for role_id and secret_id:\n  %s\n", err)
		}
	}

	secrets := SecretsMap(env)
	// NOTE: reads values 'into' a struct
	var values Secrets
	//NOTE: order matters, needs token
	err := FetchSecrets(config, secrets, &values)
	if err != nil {
		log.Fatal(err)
	}

	kafka.ClientCert = values.KafkaClientCert
	kafka.ClientKey = values.KafkaClientKey
	kafka.ServerCert = values.KafkaServerCert
}

type ConsumerGroupHandler struct {
	Context context.Context
	Logger  *log.Logger
	Updates chan UpdateMessage
	//Cancel  context.CancelFunc
}

func (c ConsumerGroupHandler) Setup(sess sarama.ConsumerGroupSession) error {
	c.Logger.Printf("Consumer Setup callback:%s\n", sess.MemberID())
	return nil
}

func (c ConsumerGroupHandler) Cleanup(sess sarama.ConsumerGroupSession) error {
	c.Logger.Printf("Consumer Cleanup callback:%s\n", sess.MemberID())
	return nil
}

// https://github.com/Shopify/sarama/issues/1192
func (c ConsumerGroupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession,
	claim sarama.ConsumerGroupClaim) error {

	for {
		select {
		case msg := <-claim.Messages():
			var um UpdateMessage
			err := json.Unmarshal(msg.Value, &um)
			if err != nil {
				return err
			}
			// NOTE: could check for "" but have not seen blank ones in log
			c.Logger.Printf("uri consumed: %v\n", um.Triple.Subject)
			c.Updates <- um
			// marking offset
			sess.MarkMessage(msg, "")
		case <-sess.Context().Done():
			err := sess.Context().Err()
			return err
		}
	}
}

type KafkaSubscriber struct {
	Brokers    []string
	Topic      string
	ClientCert string
	ClientKey  string
	ServerCert string
	ClientID   string
	GroupName  string
}

func StartConsumer(ctx context.Context, ks KafkaSubscriber, handler ConsumerGroupHandler) error {
	sarama.Logger = handler.Logger

	tlsConfig, err := NewTLSConfig(&ks)

	if err != nil {
		handler.Logger.Fatal(err)
		return err
	}

	consumerConfig := sarama.NewConfig()
	consumerConfig.ClientID = ks.ClientID
	consumerConfig.Version = sarama.V1_0_0_0
	consumerConfig.Net.TLS.Enable = true
	consumerConfig.Net.TLS.Config = tlsConfig
	consumerConfig.Consumer.Return.Errors = true

	consumerConfig.Net.ReadTimeout = (10 * time.Second)
	consumerConfig.Net.DialTimeout = (10 * time.Second)
	consumerConfig.Net.WriteTimeout = (10 * time.Second)

	// not sure a good number of retries
	consumerConfig.Metadata.Retry.Max = 3
	consumerConfig.Metadata.Retry.Backoff = (10 * time.Second)
	consumerConfig.Metadata.RefreshFrequency = (15 * time.Minute)

	// set rebalance timeout?  - default 60ms
	//consumerConfig.Consumer.Group.Rebalance.Timeout = time.Duration(6000 * time.Millisecond)
	// set a max wait time??
	//consumerConfig.Consumer.MaxWaitTime = time.Duration(305000 * time.Millisecond)
	consumerConfig.Consumer.Offsets.Initial = sarama.OffsetNewest
	//consumerConfig.Consumer.Offsets.Initial = sarama.OffsetOldest
	client, err := sarama.NewClient(ks.Brokers, consumerConfig)
	if err != nil {
		handler.Logger.Fatalf("CLIENT ERROR:%v\n", err)
		return err
	}
	defer func() { _ = client.Close() }()

	// Start a new consumer group
	group, err := sarama.NewConsumerGroupFromClient(ks.GroupName, client)
	if err != nil {
		handler.Logger.Printf("GROUP ERROR:%v\n", err)
		return err
	}
	defer func() { _ = group.Close() }()

	// NOTE: sometimes this gives:
	// CONSUME ERR:kafka server:
	// "A rebalance for the group is in progress. Please re-join the group.
	// Closing Client, Error while closing connection to broker i/o timeout"

	// see sarama docs of this function about rebalance, retries etc...
	err = group.Consume(ctx, []string{ks.Topic}, handler)
	if err != nil {
		// NOTE: sarama example panics here
		handler.Logger.Printf("CONSUME ERR:%v\n", err)
		return err
	}
	return nil
}

func (ks KafkaSubscriber) Subscribe(ctx context.Context,
	logger *log.Logger, updates chan UpdateMessage) error {
	// NOTE: need to use channel to send to batcher
	handler := ConsumerGroupHandler{Logger: logger, Updates: updates}
	err := StartConsumer(ctx, ks, handler)
	if err != nil {
		logger.Printf("start-consumer error: %v\n", err)
		return err
	}
	return nil
}

var producer sarama.AsyncProducer

func GetProducer() sarama.AsyncProducer {
	return producer
}

func SetProducer(p sarama.AsyncProducer) {
	producer = p
}

func SetupProducer(ks *KafkaSubscriber) error {
	tlsConfig, err := NewTLSConfig(ks)
	if err != nil {
		return err
	}
	producerConfig := sarama.NewConfig()
	producerConfig.ClientID = ks.ClientID
	producerConfig.Version = sarama.V1_0_0_0
	producerConfig.Net.TLS.Enable = true
	producerConfig.Net.TLS.Config = tlsConfig

	producer, err := sarama.NewAsyncProducer(ks.Brokers, producerConfig)
	if err != nil {
		return err
	}

	SetProducer(producer)
	return nil
}

func Produce(topic string, val string) {
	msg := &sarama.ProducerMessage{Topic: topic, Value: sarama.StringEncoder(val)}
	prod := GetProducer()
	// NOTE: async
	prod.Input() <- msg
}

func FluxLine(measurement string, c interface{}, tags map[string]string) (bytes.Buffer, error) {
	var b bytes.Buffer
	encoder := fluxline.NewEncoder(&b)
	err := encoder.Encode(measurement, c, tags)
	if err != nil {
		return b, err
	}
	return b, nil
}

type IndexMetrics struct {
	Start time.Time
	End   time.Time
	Uris  []string
	Name  string
}

func SendMetrics(metrics IndexMetrics, logger *log.Logger) {
	rt := (metrics.End.Sub(metrics.Start).Seconds() * 1000.0)

	d := struct {
		Duration float64 `influx:"duration"`
		Count    int64   `influx:"count"`
	}{
		Duration: rt,
		Count:    int64(len(metrics.Uris)),
	}

	tags := map[string]string{
		"indexer": metrics.Name,
	}

	line, err := FluxLine("vivoupdater.update", d, tags)
	if err == nil {
		Produce(MetricsTopic, line.String())
	}

}
<|endoftext|>


<commit_before>package tcp

import (
	"errors"
	"time"

	"github.com/hsheth2/logs"
)

func (c *TCB) packetSender() {
	// TODO: deal with data in urgSend buffers
	c.sendBufferUpdate.L.Lock()
	defer c.sendBufferUpdate.L.Lock()

	for {
		//ch logs.Trace.Println(c.Hash(), "Beginning send with sendBuffer len:", len(c.sendBuffer))
		if len(c.sendBuffer) > 0 {
			sz := uint16(min(uint64(len(c.sendBuffer)), uint64(c.maxSegSize)))
			data := c.sendBuffer[:sz]
			c.sendBuffer = c.sendBuffer[sz:]
			c.sendData(data, len(c.sendBuffer) == 0)
			continue
		}
		c.sendFinished.Broadcast(true)
		if c.stopSending {
			//ch logs.Trace.Println(c.Hash(), "Stopping packet sender; all pending sends have completed")
			return
		}
		c.sendBufferUpdate.Wait()
	}
}

func (c *TCB) sendData(data []byte, push bool) (err error) {
	//ch logs.Trace.Println(c.Hash(), "Sending Data with len:", len(data))
	var flags uint8 = TCP_ACK
	if push {
		//ch logs.Trace.Println(c.Hash(), "Data send with PSH flag")
		flags |= TCP_PSH
	}
	psh_packet := &TCP_Packet{
		header: &TCP_Header{
			seq:     c.seqNum,
			ack:     c.ackNum,
			flags:   flags,
			urg:     0,
			options: []byte{},
		},
		payload: data,
	}
	c.seqAckMutex.Lock()
	c.seqNum += uint32(len(data))
	c.seqAckMutex.Unlock()
	err = c.sendWithRetransmit(psh_packet)
	if err != nil {
		logs.Error.Println(c.Hash(), err)
	}
	return err
}

func (c *TCB) sendWithRetransmit(data *TCP_Packet) error {
	// send the first packet
	err := c.sendPacket(data)
	if err != nil { // try at least twice
		c.sendPacket(data)
	}

	go func() error {
		// ack listeners
		ackFound := make(chan bool, 1)
		killAckListen := make(chan bool, 1)
		c.listenForAck(ackFound, killAckListen, data.header.seq+data.getPayloadSize())

		// timers and timeouts
		resendTimerChan := make(chan bool, TCP_RESEND_LIMIT)
		timeout := make(chan bool, 1)
		killTimer := make(chan bool, 1)
		resendTimer(resendTimerChan, timeout, killTimer, c.resendDelay)

		// resend if needed
		for {
			select {
			case <-ackFound:
				killTimer <- true
				return nil
			case <-resendTimerChan:
				c.sendPacket(data)
			case <-timeout:
				// TODO deal with a resend timeout fully
				killAckListen <- true
				logs.Error.Println(c.Hash(), "Resend of packet seq", data.header.seq, "timed out")
				return errors.New("Resend timed out")
			}
		}
	}()

	return nil
}

func (c *TCB) listenForAck(successOut chan<- bool, end <-chan bool, targetAck uint32) {
	//ch logs.Trace.Println(c.Hash(), "Listening for ack:", targetAck)
	in := c.recentAckUpdate.Register(ACK_BUF_SZ)
	go func(in chan interface{}, successOut chan<- bool, end <-chan bool, targetAck uint32) {
		defer c.recentAckUpdate.Unregister(in)
		for {
			select {
			case v := <-in:
				//ch logs.Trace.Println(c.Hash(), "Ack listener got ack: ", v.(uint32))
				if v.(uint32) >= targetAck {
					//ch logs.Trace.Println(c.Hash(), "Killing the resender for ack:", targetAck)
					successOut <- true
					return
				}
			case <-end:
				return
			}
		}
	}(in, successOut, end, targetAck)
}

func resendTimer(timerOutput, timeout chan<- bool, finished <-chan bool, delay time.Duration) {
	for i := 0; i < TCP_RESEND_LIMIT; i++ {
		select {
		case <-time.After(delay):
			timerOutput <- true
			delay *= 2 // increase the delay after each resend
		case <-finished:
			return
		}
	}
	timeout <- true
}

func (c *TCB) sendPacket(d *TCP_Packet) error {
	// Requires that seq, ack, flags, urg, and options are set
	// Will set everything else

	d.header.srcport = c.lport
	d.header.dstport = c.rport
	c.windowMutex.RLock()
	d.header.window = c.getWindow() // TODO improve the window field calculation
	c.windowMutex.RUnlock()
	d.rip = c.ipAddress
	d.lip = c.srcIP

	pay, err := d.Marshal_TCP_Packet()
	if err != nil {
		logs.Error.Println(c.Hash(), err)
		return err
	}

	err = c.writer.WriteTo(pay)

	if err != nil {
		logs.Error.Println(c.Hash(), err)
		return err
	}

	return nil
}

func (c *TCB) sendResetFlag(seq, ack uint32, flag uint8) error {
	//ch logs.Trace.Println(c.Hash(), "Sending RST with seq: ", seq, " and ack: ", ack)
	rst := &TCP_Packet{
		header: &TCP_Header{
			seq:     seq,
			ack:     ack,
			flags:   flag,
			urg:     0,
			options: []byte{},
		},
		payload: []byte{},
	}

	return c.sendPacket(rst)
}

func (c *TCB) sendReset(seq, ack uint32) error {
	return c.sendResetFlag(seq, ack, TCP_RST)
}

func (c *TCB) sendAck(seq, ack uint32) error {
	//ch logs.Trace.Println(c.Hash(), "Sending ACK with seq: ", seq, " and ack: ", ack)
	ack_packet := &TCP_Packet{
		header: &TCP_Header{
			seq:     seq,
			ack:     ack,
			flags:   TCP_ACK,
			urg:     0,
			options: []byte{},
		},
		payload: []byte{},
	}
	return c.sendPacket(ack_packet)
}

func (c *TCB) sendFin(seq, ack uint32) error {
	//ch logs.Trace.Println(c.Hash(), "Sending FIN with seq: ", seq, " and ack: ", ack)
	fin_packet := &TCP_Packet{
		header: &TCP_Header{
			seq:     seq,
			ack:     ack,
			flags:   TCP_ACK | TCP_FIN,
			urg:     0,
			options: []byte{},
		},
		payload: []byte{},
	}
	return c.sendPacket(fin_packet)
}
<commit_msg>Made TCP compatible<commit_after>package tcp

import (
	"errors"
	"time"

	"github.com/hsheth2/logs"
)

func (c *TCB) packetSender() {
	// TODO: deal with data in urgSend buffers
	c.sendBufferUpdate.L.Lock()
	defer c.sendBufferUpdate.L.Lock()

	for {
		//ch logs.Trace.Println(c.Hash(), "Beginning send with sendBuffer len:", len(c.sendBuffer))
		if len(c.sendBuffer) > 0 {
			sz := uint16(min(uint64(len(c.sendBuffer)), uint64(c.maxSegSize)))
			data := c.sendBuffer[:sz]
			c.sendBuffer = c.sendBuffer[sz:]
			c.sendData(data, len(c.sendBuffer) == 0)
			continue
		}
		c.sendFinished.Broadcast(true)
		if c.stopSending {
			//ch logs.Trace.Println(c.Hash(), "Stopping packet sender; all pending sends have completed")
			return
		}
		c.sendBufferUpdate.Wait()
	}
}

func (c *TCB) sendData(data []byte, push bool) (err error) {
	//ch logs.Trace.Println(c.Hash(), "Sending Data with len:", len(data))
	var flags uint8 = TCP_ACK
	if push {
		//ch logs.Trace.Println(c.Hash(), "Data send with PSH flag")
		flags |= TCP_PSH
	}
	psh_packet := &TCP_Packet{
		header: &TCP_Header{
			seq:     c.seqNum,
			ack:     c.ackNum,
			flags:   flags,
			urg:     0,
			options: []byte{},
		},
		payload: data,
	}
	c.seqAckMutex.Lock()
	c.seqNum += uint32(len(data))
	c.seqAckMutex.Unlock()
	err = c.sendWithRetransmit(psh_packet)
	if err != nil {
		logs.Error.Println(c.Hash(), err)
	}
	return err
}

func (c *TCB) sendWithRetransmit(data *TCP_Packet) error {
	// send the first packet
	err := c.sendPacket(data)
	if err != nil { // try at least twice
		c.sendPacket(data)
	}

	go func() error {
		// ack listeners
		ackFound := make(chan bool, 1)
		killAckListen := make(chan bool, 1)
		c.listenForAck(ackFound, killAckListen, data.header.seq+data.getPayloadSize())

		// timers and timeouts
		resendTimerChan := make(chan bool, TCP_RESEND_LIMIT)
		timeout := make(chan bool, 1)
		killTimer := make(chan bool, 1)
		resendTimer(resendTimerChan, timeout, killTimer, c.resendDelay)

		// resend if needed
		for {
			select {
			case <-ackFound:
				killTimer <- true
				return nil
			case <-resendTimerChan:
				c.sendPacket(data)
			case <-timeout:
				// TODO deal with a resend timeout fully
				killAckListen <- true
				logs.Error.Println(c.Hash(), "Resend of packet seq", data.header.seq, "timed out")
				return errors.New("Resend timed out")
			}
		}
	}()

	return nil
}

func (c *TCB) listenForAck(successOut chan<- bool, end <-chan bool, targetAck uint32) {
	//ch logs.Trace.Println(c.Hash(), "Listening for ack:", targetAck)
	in := c.recentAckUpdate.Register(ACK_BUF_SZ)
	go func(in chan interface{}, successOut chan<- bool, end <-chan bool, targetAck uint32) {
		defer c.recentAckUpdate.Unregister(in)
		for {
			select {
			case v := <-in:
				//ch logs.Trace.Println(c.Hash(), "Ack listener got ack: ", v.(uint32))
				if v.(uint32) >= targetAck {
					//ch logs.Trace.Println(c.Hash(), "Killing the resender for ack:", targetAck)
					successOut <- true
					return
				}
			case <-end:
				return
			}
		}
	}(in, successOut, end, targetAck)
}

func resendTimer(timerOutput, timeout chan<- bool, finished <-chan bool, delay time.Duration) {
	for i := 0; i < TCP_RESEND_LIMIT; i++ {
		select {
		case <-time.After(delay):
			timerOutput <- true
			delay *= 2 // increase the delay after each resend
		case <-finished:
			return
		}
	}
	timeout <- true
}

func (c *TCB) sendPacket(d *TCP_Packet) error {
	// Requires that seq, ack, flags, urg, and options are set
	// Will set everything else

	d.header.srcport = c.lport
	d.header.dstport = c.rport
	c.windowMutex.RLock()
	d.header.window = c.getWindow() // TODO improve the window field calculation
	c.windowMutex.RUnlock()
	d.rip = c.ipAddress
	d.lip = c.srcIP

	pay, err := d.Marshal_TCP_Packet()
	if err != nil {
		logs.Error.Println(c.Hash(), err)
		return err
	}

	n, err := c.writer.WriteTo(pay)
	if n != len(pay) {
		return errors.New("Not all data written successfully")
	}

	if err != nil {
		logs.Error.Println(c.Hash(), err)
		return err
	}

	return nil
}

func (c *TCB) sendResetFlag(seq, ack uint32, flag uint8) error {
	//ch logs.Trace.Println(c.Hash(), "Sending RST with seq: ", seq, " and ack: ", ack)
	rst := &TCP_Packet{
		header: &TCP_Header{
			seq:     seq,
			ack:     ack,
			flags:   flag,
			urg:     0,
			options: []byte{},
		},
		payload: []byte{},
	}

	return c.sendPacket(rst)
}

func (c *TCB) sendReset(seq, ack uint32) error {
	return c.sendResetFlag(seq, ack, TCP_RST)
}

func (c *TCB) sendAck(seq, ack uint32) error {
	//ch logs.Trace.Println(c.Hash(), "Sending ACK with seq: ", seq, " and ack: ", ack)
	ack_packet := &TCP_Packet{
		header: &TCP_Header{
			seq:     seq,
			ack:     ack,
			flags:   TCP_ACK,
			urg:     0,
			options: []byte{},
		},
		payload: []byte{},
	}
	return c.sendPacket(ack_packet)
}

func (c *TCB) sendFin(seq, ack uint32) error {
	//ch logs.Trace.Println(c.Hash(), "Sending FIN with seq: ", seq, " and ack: ", ack)
	fin_packet := &TCP_Packet{
		header: &TCP_Header{
			seq:     seq,
			ack:     ack,
			flags:   TCP_ACK | TCP_FIN,
			urg:     0,
			options: []byte{},
		},
		payload: []byte{},
	}
	return c.sendPacket(fin_packet)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"runtime"
	"sync"
	"sync/atomic"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/getlantern/golog"
	"github.com/oxtoacart/tdb"
	. "github.com/oxtoacart/tdb/expr"
)

var (
	log = golog.LoggerFor("tdbdemo")
)

func main() {
	epoch := time.Date(2015, time.January, 1, 0, 0, 0, 0, time.UTC)

	tmpDir, err := ioutil.TempDir("", "tdbtest")
	if err != nil {
		log.Fatal(err)
	}
	defer os.RemoveAll(tmpDir)

	log.Debugf("Writing data to %v", tmpDir)

	numReporters := 5000
	uniquesPerReporter := 100
	uniquesPerPeriod := 20
	reportingPeriods := 1000
	reportingInterval := time.Millisecond
	resolution := reportingInterval * 5
	hotPeriod := resolution * 10
	retainPeriods := 2
	retentionPeriod := time.Duration(retainPeriods) * reportingInterval
	numWriters := 4
	db := tdb.NewDB(&tdb.DBOpts{
		Dir:       tmpDir,
		BatchSize: 1000,
	})
	err = db.CreateTable("test", resolution, hotPeriod, retentionPeriod, tdb.DerivedField{
		Name: "iii",
		Expr: Avg(Calc("ii / i")),
	})
	if err != nil {
		log.Fatal(err)
	}

	inserts := int64(0)
	start := time.Now()

	report := func() {
		stats := db.TableStats("test")
		delta := time.Now().Sub(start)
		start = time.Now()
		i := atomic.SwapInt64(&inserts, 0)
		var ms runtime.MemStats
		runtime.ReadMemStats(&ms)
		preGC := float64(ms.HeapAlloc) / 1024.0 / 1024.0
		runtime.GC()
		runtime.ReadMemStats(&ms)
		postGC := float64(ms.HeapAlloc) / 1024.0 / 1024.0
		fmt.Printf(`
%s inserts at %s inserts per second
Hot Keys: %s     Archived Buckets: %s
HeapAlloc pre/post GC %f/%f MiB
`,
			humanize.Comma(i), humanize.Comma(i/int64(delta.Seconds())),
			humanize.Comma(stats.HotKeys), humanize.Comma(stats.ArchivedBuckets),
			preGC, postGC)
	}

	go func() {
		for {
			tk := time.NewTicker(30 * time.Second)
			for range tk.C {
				report()
			}
		}
	}()

	go func() {
		for {
			tk := time.NewTicker(10 * time.Second)
			for range tk.C {
				count := 0
				now := db.Now("test")
				q := &tdb.Query{
					Table:  "test",
					Fields: []string{"i"},
					From:   now.Add(-2 * hotPeriod),
					OnValues: func(key map[string]interface{}, field string, vals []float64) {
						count++
					},
				}
				start := time.Now()
				err := db.RunQuery(q)
				delta := time.Now().Sub(start)
				if err != nil {
					log.Errorf("Unable to run query: %v", err)
					continue
				}
				fmt.Printf("\nQuery at %v returned %v in %v\n", now, humanize.Comma(int64(count)), delta)
			}
		}
	}()

	var wg sync.WaitGroup
	wg.Add(numWriters)
	for _w := 0; _w < numWriters; _w++ {
		go func() {
			defer wg.Done()
			for i := 0; i < reportingPeriods; i++ {
				ts := epoch.Add(time.Duration(i) * reportingInterval)
				for r := 0; r < numReporters/numWriters; r++ {
					for u := 0; u < uniquesPerPeriod; u++ {
						p := &tdb.Point{
							Ts: ts,
							Dims: map[string]interface{}{
								"r": rand.Intn(numReporters),
								"u": rand.Intn(uniquesPerReporter),
								"b": rand.Float64() > 0.99,
							},
							Vals: map[string]float64{
								"i":  float64(rand.Intn(100000)),
								"ii": float64(rand.Intn(100)),
							},
						}
						ierr := db.Insert("test", p)
						if ierr != nil {
							log.Errorf("Unable to insert: %v", err)
							return
						}
						atomic.AddInt64(&inserts, 1)
					}
				}
				fmt.Print(".")
			}
		}()
	}

	wg.Wait()
	report()
}
<commit_msg>Updated tdbdemo<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"runtime"
	"sync"
	"sync/atomic"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/getlantern/golog"
	"github.com/oxtoacart/tdb"
	. "github.com/oxtoacart/tdb/expr"
)

var (
	log = golog.LoggerFor("tdbdemo")
)

func main() {
	epoch := time.Date(2015, time.January, 1, 0, 0, 0, 0, time.UTC)

	tmpDir, err := ioutil.TempDir("", "tdbtest")
	if err != nil {
		log.Fatal(err)
	}
	defer os.RemoveAll(tmpDir)

	log.Debugf("Writing data to %v", tmpDir)

	numReporters := 5000
	uniquesPerReporter := 100
	uniquesPerPeriod := 20
	reportingPeriods := 1000
	reportingInterval := time.Millisecond
	resolution := reportingInterval * 5
	hotPeriod := resolution * 10
	retainPeriods := 2
	retentionPeriod := time.Duration(retainPeriods) * reportingInterval
	numWriters := 4
	db := tdb.NewDB(&tdb.DBOpts{
		Dir:       tmpDir,
		BatchSize: 1000,
	})
	err = db.CreateTable("test", resolution, hotPeriod, retentionPeriod, map[string]Expr{
		"i":   Sum("i"),
		"ii":  Sum("ii"),
		"iii": Avg(Div("ii", "i")),
	})
	if err != nil {
		log.Fatal(err)
	}

	inserts := int64(0)
	start := time.Now()

	report := func() {
		stats := db.TableStats("test")
		delta := time.Now().Sub(start)
		start = time.Now()
		i := atomic.SwapInt64(&inserts, 0)
		var ms runtime.MemStats
		runtime.ReadMemStats(&ms)
		preGC := float64(ms.HeapAlloc) / 1024.0 / 1024.0
		runtime.GC()
		runtime.ReadMemStats(&ms)
		postGC := float64(ms.HeapAlloc) / 1024.0 / 1024.0
		fmt.Printf(`
%s inserts at %s inserts per second
Hot Keys: %s     Archived Buckets: %s
HeapAlloc pre/post GC %f/%f MiB
`,
			humanize.Comma(i), humanize.Comma(i/int64(delta.Seconds())),
			humanize.Comma(stats.HotKeys), humanize.Comma(stats.ArchivedBuckets),
			preGC, postGC)
	}

	go func() {
		for {
			tk := time.NewTicker(30 * time.Second)
			for range tk.C {
				report()
			}
		}
	}()

	go func() {
		for {
			tk := time.NewTicker(10 * time.Second)
			for range tk.C {
				count := 0
				now := db.Now("test")
				q := &tdb.Query{
					Table:  "test",
					Fields: []string{"i"},
					From:   now.Add(-2 * hotPeriod),
					OnValues: func(key map[string]interface{}, field string, vals []float64) {
						count++
					},
				}
				start := time.Now()
				err := db.RunQuery(q)
				delta := time.Now().Sub(start)
				if err != nil {
					log.Errorf("Unable to run query: %v", err)
					continue
				}
				fmt.Printf("\nQuery at %v returned %v in %v\n", now, humanize.Comma(int64(count)), delta)
			}
		}
	}()

	var wg sync.WaitGroup
	wg.Add(numWriters)
	for _w := 0; _w < numWriters; _w++ {
		go func() {
			defer wg.Done()
			for i := 0; i < reportingPeriods; i++ {
				ts := epoch.Add(time.Duration(i) * reportingInterval)
				for r := 0; r < numReporters/numWriters; r++ {
					for u := 0; u < uniquesPerPeriod; u++ {
						p := &tdb.Point{
							Ts: ts,
							Dims: map[string]interface{}{
								"r": rand.Intn(numReporters),
								"u": rand.Intn(uniquesPerReporter),
								"b": rand.Float64() > 0.99,
							},
							Vals: map[string]float64{
								"i":  float64(rand.Intn(100000)),
								"ii": float64(rand.Intn(100)),
							},
						}
						ierr := db.Insert("test", p)
						if ierr != nil {
							log.Errorf("Unable to insert: %v", err)
							return
						}
						atomic.AddInt64(&inserts, 1)
					}
				}
				fmt.Print(".")
			}
		}()
	}

	wg.Wait()
	report()
}
<|endoftext|>


<commit_before>package discover

import (
	"net/http"
	"log"

	"github.com/coreos/go-etcd/etcd"
	"github.com/flynn/rpcplus"
)

const (
	HeartbeatIntervalSecs = 5
	MissedHearbeatTTL     = 5
)

type ServiceUpdate struct {
	Name   string
	Addr   string
	Online bool
	Attrs  map[string]string
}

type Args struct {
	Name  string
	Addr  string
	Attrs map[string]string
}

type UpdateStream interface {
	Chan() chan *ServiceUpdate
	Close()
}

type DiscoveryBackend interface {
	Subscribe(name string) (UpdateStream, error)
	Register(name string, addr string, attrs map[string]string) error
	Unregister(name string, addr string) error
	Heartbeat(name string, addr string) error
}

type Agent struct {
	Backend DiscoveryBackend
	Address string
}

func NewServer(addr string) *Agent {
	return &Agent{
		Backend: &EtcdBackend{Client: etcd.NewClient(nil)},
		Address: addr,
	}
}

func ListenAndServe(server *Agent) error {
	rpcplus.HandleHTTP()
	err := rpcplus.Register(server)
	if err != nil {
		return err
	}
	return http.ListenAndServe(server.Address, nil)
}

func (s *Agent) Subscribe(args *Args, stream rpcplus.Stream) error {
	updates, err := s.Backend.Subscribe(args.Name)
	if err != nil {
		log.Println("Subscribe: ", err)
		stream.Send <- &ServiceUpdate{} // be sure to unblock client
		return err
	}
	for update := range updates.Chan() {
		select {
		case stream.Send <- update:
		case <-stream.Error:
			updates.Close()
			return nil
		}
	}
	return nil
}

func (s *Agent) Register(args *Args, ret *struct{}) error {
	err := s.Backend.Register(args.Name, args.Addr, args.Attrs)
	if err != nil {
		log.Println("Register: ", err)
	}
	return err
}

func (s *Agent) Unregister(args *Args, ret *struct{}) error {
	err := s.Backend.Unregister(args.Name, args.Addr)
	if err != nil {
		log.Println("Unregister: ", err)
	}
	return err
}

func (s *Agent) Heartbeat(args *Args, ret *struct{}) error {
	err := s.Backend.Heartbeat(args.Name, args.Addr)
	if err != nil {
		log.Println("Heartbeat: ", err)
	}
	return err
}
<commit_msg>Use comborpc<commit_after>package discover

import (
	"log"
	"net/http"

	"github.com/coreos/go-etcd/etcd"
	"github.com/flynn/rpcplus"
	rpc "github.com/flynn/rpcplus/comborpc"
)

const (
	HeartbeatIntervalSecs = 5
	MissedHearbeatTTL     = 5
)

type ServiceUpdate struct {
	Name   string
	Addr   string
	Online bool
	Attrs  map[string]string
}

type Args struct {
	Name  string
	Addr  string
	Attrs map[string]string
}

type UpdateStream interface {
	Chan() chan *ServiceUpdate
	Close()
}

type DiscoveryBackend interface {
	Subscribe(name string) (UpdateStream, error)
	Register(name string, addr string, attrs map[string]string) error
	Unregister(name string, addr string) error
	Heartbeat(name string, addr string) error
}

type Agent struct {
	Backend DiscoveryBackend
	Address string
}

func NewServer(addr string) *Agent {
	return &Agent{
		Backend: &EtcdBackend{Client: etcd.NewClient(nil)},
		Address: addr,
	}
}

func ListenAndServe(server *Agent) error {
	rpc.HandleHTTP()
	if err := rpc.Register(server); err != nil {
		return err
	}
	return http.ListenAndServe(server.Address, nil)
}

func (s *Agent) Subscribe(args *Args, stream rpcplus.Stream) error {
	updates, err := s.Backend.Subscribe(args.Name)
	if err != nil {
		log.Println("Subscribe: ", err)
		stream.Send <- &ServiceUpdate{} // be sure to unblock client
		return err
	}
	for update := range updates.Chan() {
		select {
		case stream.Send <- update:
		case <-stream.Error:
			updates.Close()
			return nil
		}
	}
	return nil
}

func (s *Agent) Register(args *Args, ret *struct{}) error {
	err := s.Backend.Register(args.Name, args.Addr, args.Attrs)
	if err != nil {
		log.Println("Register: ", err)
	}
	return err
}

func (s *Agent) Unregister(args *Args, ret *struct{}) error {
	err := s.Backend.Unregister(args.Name, args.Addr)
	if err != nil {
		log.Println("Unregister: ", err)
	}
	return err
}

func (s *Agent) Heartbeat(args *Args, ret *struct{}) error {
	err := s.Backend.Heartbeat(args.Name, args.Addr)
	if err != nil {
		log.Println("Heartbeat: ", err)
	}
	return err
}
<|endoftext|>


<commit_before>// +build !windows

/*
 Copyright 2019 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package tty

import (
	"github.com/pufferpanel/apufferi/v4"
	"github.com/pufferpanel/pufferd/v2/environments/envs"
)

type EnvironmentFactory struct {
	envs.EnvironmentFactory
}

func (ef EnvironmentFactory) Create(id string) envs.Environment {
	t := &tty{
		BaseEnvironment: &envs.BaseEnvironment{
			TypeWithMetadata: apufferi.TypeWithMetadata{
				Type: "tty",
			},
		},
	}
	t.BaseEnvironment.ExecutionFunction = t.ttyExecuteAsync
	return t
}

func (ef EnvironmentFactory) Key() string {
	return "tty"
}
<commit_msg>fix tty<commit_after>// +build !windows

/*
 Copyright 2019 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package tty

import (
	"github.com/pufferpanel/pufferd/v2/environments/envs"
)

type EnvironmentFactory struct {
	envs.EnvironmentFactory
}

func (ef EnvironmentFactory) Create(id string) envs.Environment {
	t := &tty{
		BaseEnvironment: &envs.BaseEnvironment{Type: "tty"},
	}
	t.BaseEnvironment.ExecutionFunction = t.ttyExecuteAsync
	return t
}

func (ef EnvironmentFactory) Key() string {
	return "tty"
}
<|endoftext|>