<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Network interface identification for Linux

package net

import (
	"os"
	"syscall"
	"unsafe"
)

// If the ifindex is zero, interfaceTable returns mappings of all
// network interfaces.  Otherwise it returns a mapping of a specific
// interface.
func interfaceTable(ifindex int) ([]Interface, error) {
	tab, err := syscall.NetlinkRIB(syscall.RTM_GETLINK, syscall.AF_UNSPEC)
	if err != nil {
		return nil, os.NewSyscallError("netlink rib", err)
	}

	msgs, err := syscall.ParseNetlinkMessage(tab)
	if err != nil {
		return nil, os.NewSyscallError("netlink message", err)
	}

	var ift []Interface
	for _, m := range msgs {
		switch m.Header.Type {
		case syscall.NLMSG_DONE:
			goto done
		case syscall.RTM_NEWLINK:
			ifim := (*syscall.IfInfomsg)(unsafe.Pointer(&m.Data[0]))
			if ifindex == 0 || ifindex == int(ifim.Index) {
				attrs, err := syscall.ParseNetlinkRouteAttr(&m)
				if err != nil {
					return nil, os.NewSyscallError("netlink routeattr", err)
				}
				ifi := newLink(ifim, attrs)
				ift = append(ift, ifi)
			}
		}
	}
done:
	return ift, nil
}

func newLink(ifim *syscall.IfInfomsg, attrs []syscall.NetlinkRouteAttr) Interface {
	ifi := Interface{Index: int(ifim.Index), Flags: linkFlags(ifim.Flags)}
	for _, a := range attrs {
		switch a.Attr.Type {
		case syscall.IFLA_ADDRESS:
			var nonzero bool
			for _, b := range a.Value {
				if b != 0 {
					nonzero = true
				}
			}
			if nonzero {
				ifi.HardwareAddr = a.Value[:]
			}
		case syscall.IFLA_IFNAME:
			ifi.Name = string(a.Value[:len(a.Value)-1])
		case syscall.IFLA_MTU:
			ifi.MTU = int(*(*uint32)(unsafe.Pointer(&a.Value[:4][0])))
		}
	}
	return ifi
}

func linkFlags(rawFlags uint32) Flags {
	var f Flags
	if rawFlags&syscall.IFF_UP != 0 {
		f |= FlagUp
	}
	if rawFlags&syscall.IFF_BROADCAST != 0 {
		f |= FlagBroadcast
	}
	if rawFlags&syscall.IFF_LOOPBACK != 0 {
		f |= FlagLoopback
	}
	if rawFlags&syscall.IFF_POINTOPOINT != 0 {
		f |= FlagPointToPoint
	}
	if rawFlags&syscall.IFF_MULTICAST != 0 {
		f |= FlagMulticast
	}
	return f
}

// If the ifindex is zero, interfaceAddrTable returns addresses
// for all network interfaces.  Otherwise it returns addresses
// for a specific interface.
func interfaceAddrTable(ifindex int) ([]Addr, error) {
	tab, err := syscall.NetlinkRIB(syscall.RTM_GETADDR, syscall.AF_UNSPEC)
	if err != nil {
		return nil, os.NewSyscallError("netlink rib", err)
	}

	msgs, err := syscall.ParseNetlinkMessage(tab)
	if err != nil {
		return nil, os.NewSyscallError("netlink message", err)
	}

	ifat, err := addrTable(msgs, ifindex)
	if err != nil {
		return nil, err
	}
	return ifat, nil
}

func addrTable(msgs []syscall.NetlinkMessage, ifindex int) ([]Addr, error) {
	var ifat []Addr
	for _, m := range msgs {
		switch m.Header.Type {
		case syscall.NLMSG_DONE:
			goto done
		case syscall.RTM_NEWADDR:
			ifam := (*syscall.IfAddrmsg)(unsafe.Pointer(&m.Data[0]))
			if ifindex == 0 || ifindex == int(ifam.Index) {
				attrs, err := syscall.ParseNetlinkRouteAttr(&m)
				if err != nil {
					return nil, os.NewSyscallError("netlink routeattr", err)
				}
				ifat = append(ifat, newAddr(attrs, int(ifam.Family), int(ifam.Prefixlen)))
			}
		}
	}
done:
	return ifat, nil
}

func newAddr(attrs []syscall.NetlinkRouteAttr, family, pfxlen int) Addr {
	ifa := &IPNet{}
	for _, a := range attrs {
		switch a.Attr.Type {
		case syscall.IFA_ADDRESS:
			switch family {
			case syscall.AF_INET:
				ifa.IP = IPv4(a.Value[0], a.Value[1], a.Value[2], a.Value[3])
				ifa.Mask = CIDRMask(pfxlen, 8*IPv4len)
			case syscall.AF_INET6:
				ifa.IP = make(IP, IPv6len)
				copy(ifa.IP, a.Value[:])
				ifa.Mask = CIDRMask(pfxlen, 8*IPv6len)
			}
		}
	}
	return ifa
}

// If the ifindex is zero, interfaceMulticastAddrTable returns
// addresses for all network interfaces.  Otherwise it returns
// addresses for a specific interface.
func interfaceMulticastAddrTable(ifindex int) ([]Addr, error) {
	var (
		err error
		ifi *Interface
	)
	if ifindex > 0 {
		ifi, err = InterfaceByIndex(ifindex)
		if err != nil {
			return nil, err
		}
	}
	ifmat4 := parseProcNetIGMP("/proc/net/igmp", ifi)
	ifmat6 := parseProcNetIGMP6("/proc/net/igmp6", ifi)
	return append(ifmat4, ifmat6...), nil
}

func parseProcNetIGMP(path string, ifi *Interface) []Addr {
	fd, err := open(path)
	if err != nil {
		return nil
	}
	defer fd.close()

	var (
		ifmat []Addr
		name  string
	)
	fd.readLine() // skip first line
	b := make([]byte, IPv4len)
	for l, ok := fd.readLine(); ok; l, ok = fd.readLine() {
		f := splitAtBytes(l, " :\r\t\n")
		if len(f) < 4 {
			continue
		}
		switch {
		case l[0] != ' ' && l[0] != '\t': // new interface line
			name = f[1]
		case len(f[0]) == 8:
			if ifi == nil || name == ifi.Name {
				// The Linux kernel puts the IP
				// address in /proc/net/igmp in native
				// endianness.
				for i := 0; i+1 < len(f[0]); i += 2 {
					b[i/2], _ = xtoi2(f[0][i:i+2], 0)
				}
				i := *(*uint32)(unsafe.Pointer(&b[:4][0]))
				ifma := IPAddr{IP: IPv4(byte(i>>24), byte(i>>16), byte(i>>8), byte(i))}
				ifmat = append(ifmat, ifma.toAddr())
			}
		}
	}
	return ifmat
}

func parseProcNetIGMP6(path string, ifi *Interface) []Addr {
	fd, err := open(path)
	if err != nil {
		return nil
	}
	defer fd.close()

	var ifmat []Addr
	b := make([]byte, IPv6len)
	for l, ok := fd.readLine(); ok; l, ok = fd.readLine() {
		f := splitAtBytes(l, " \r\t\n")
		if len(f) < 6 {
			continue
		}
		if ifi == nil || f[1] == ifi.Name {
			for i := 0; i+1 < len(f[2]); i += 2 {
				b[i/2], _ = xtoi2(f[2][i:i+2], 0)
			}
			ifma := IPAddr{IP: IP{b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], b[10], b[11], b[12], b[13], b[14], b[15]}}
			ifmat = append(ifmat, ifma.toAddr())
		}
	}
	return ifmat
}
<commit_msg>net: return correct point-to-point interface address on linux<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Network interface identification for Linux

package net

import (
	"os"
	"syscall"
	"unsafe"
)

// If the ifindex is zero, interfaceTable returns mappings of all
// network interfaces.  Otherwise it returns a mapping of a specific
// interface.
func interfaceTable(ifindex int) ([]Interface, error) {
	tab, err := syscall.NetlinkRIB(syscall.RTM_GETLINK, syscall.AF_UNSPEC)
	if err != nil {
		return nil, os.NewSyscallError("netlink rib", err)
	}

	msgs, err := syscall.ParseNetlinkMessage(tab)
	if err != nil {
		return nil, os.NewSyscallError("netlink message", err)
	}

	var ift []Interface
	for _, m := range msgs {
		switch m.Header.Type {
		case syscall.NLMSG_DONE:
			goto done
		case syscall.RTM_NEWLINK:
			ifim := (*syscall.IfInfomsg)(unsafe.Pointer(&m.Data[0]))
			if ifindex == 0 || ifindex == int(ifim.Index) {
				attrs, err := syscall.ParseNetlinkRouteAttr(&m)
				if err != nil {
					return nil, os.NewSyscallError("netlink routeattr", err)
				}
				ifi := newLink(ifim, attrs)
				ift = append(ift, ifi)
			}
		}
	}
done:
	return ift, nil
}

func newLink(ifim *syscall.IfInfomsg, attrs []syscall.NetlinkRouteAttr) Interface {
	ifi := Interface{Index: int(ifim.Index), Flags: linkFlags(ifim.Flags)}
	for _, a := range attrs {
		switch a.Attr.Type {
		case syscall.IFLA_ADDRESS:
			var nonzero bool
			for _, b := range a.Value {
				if b != 0 {
					nonzero = true
				}
			}
			if nonzero {
				ifi.HardwareAddr = a.Value[:]
			}
		case syscall.IFLA_IFNAME:
			ifi.Name = string(a.Value[:len(a.Value)-1])
		case syscall.IFLA_MTU:
			ifi.MTU = int(*(*uint32)(unsafe.Pointer(&a.Value[:4][0])))
		}
	}
	return ifi
}

func linkFlags(rawFlags uint32) Flags {
	var f Flags
	if rawFlags&syscall.IFF_UP != 0 {
		f |= FlagUp
	}
	if rawFlags&syscall.IFF_BROADCAST != 0 {
		f |= FlagBroadcast
	}
	if rawFlags&syscall.IFF_LOOPBACK != 0 {
		f |= FlagLoopback
	}
	if rawFlags&syscall.IFF_POINTOPOINT != 0 {
		f |= FlagPointToPoint
	}
	if rawFlags&syscall.IFF_MULTICAST != 0 {
		f |= FlagMulticast
	}
	return f
}

// If the ifindex is zero, interfaceAddrTable returns addresses
// for all network interfaces.  Otherwise it returns addresses
// for a specific interface.
func interfaceAddrTable(ifindex int) ([]Addr, error) {
	tab, err := syscall.NetlinkRIB(syscall.RTM_GETADDR, syscall.AF_UNSPEC)
	if err != nil {
		return nil, os.NewSyscallError("netlink rib", err)
	}

	msgs, err := syscall.ParseNetlinkMessage(tab)
	if err != nil {
		return nil, os.NewSyscallError("netlink message", err)
	}

	ifat, err := addrTable(msgs, ifindex)
	if err != nil {
		return nil, err
	}
	return ifat, nil
}

func addrTable(msgs []syscall.NetlinkMessage, ifindex int) ([]Addr, error) {
	var ifat []Addr
	for _, m := range msgs {
		switch m.Header.Type {
		case syscall.NLMSG_DONE:
			goto done
		case syscall.RTM_NEWADDR:
			ifam := (*syscall.IfAddrmsg)(unsafe.Pointer(&m.Data[0]))
			ifi, err := InterfaceByIndex(int(ifam.Index))
			if err != nil {
				return nil, err
			}
			if ifindex == 0 || ifindex == int(ifam.Index) {
				attrs, err := syscall.ParseNetlinkRouteAttr(&m)
				if err != nil {
					return nil, os.NewSyscallError("netlink routeattr", err)
				}
				ifat = append(ifat, newAddr(attrs, ifi, ifam))
			}
		}
	}
done:
	return ifat, nil
}

func newAddr(attrs []syscall.NetlinkRouteAttr, ifi *Interface, ifam *syscall.IfAddrmsg) Addr {
	ifa := &IPNet{}
	for _, a := range attrs {
		if ifi.Flags&FlagPointToPoint != 0 && a.Attr.Type == syscall.IFA_LOCAL ||
			ifi.Flags&FlagPointToPoint == 0 && a.Attr.Type == syscall.IFA_ADDRESS {
			switch ifam.Family {
			case syscall.AF_INET:
				ifa.IP = IPv4(a.Value[0], a.Value[1], a.Value[2], a.Value[3])
				ifa.Mask = CIDRMask(int(ifam.Prefixlen), 8*IPv4len)
			case syscall.AF_INET6:
				ifa.IP = make(IP, IPv6len)
				copy(ifa.IP, a.Value[:])
				ifa.Mask = CIDRMask(int(ifam.Prefixlen), 8*IPv6len)
			}
		}
	}
	return ifa
}

// If the ifindex is zero, interfaceMulticastAddrTable returns
// addresses for all network interfaces.  Otherwise it returns
// addresses for a specific interface.
func interfaceMulticastAddrTable(ifindex int) ([]Addr, error) {
	var (
		err error
		ifi *Interface
	)
	if ifindex > 0 {
		ifi, err = InterfaceByIndex(ifindex)
		if err != nil {
			return nil, err
		}
	}
	ifmat4 := parseProcNetIGMP("/proc/net/igmp", ifi)
	ifmat6 := parseProcNetIGMP6("/proc/net/igmp6", ifi)
	return append(ifmat4, ifmat6...), nil
}

func parseProcNetIGMP(path string, ifi *Interface) []Addr {
	fd, err := open(path)
	if err != nil {
		return nil
	}
	defer fd.close()

	var (
		ifmat []Addr
		name  string
	)
	fd.readLine() // skip first line
	b := make([]byte, IPv4len)
	for l, ok := fd.readLine(); ok; l, ok = fd.readLine() {
		f := splitAtBytes(l, " :\r\t\n")
		if len(f) < 4 {
			continue
		}
		switch {
		case l[0] != ' ' && l[0] != '\t': // new interface line
			name = f[1]
		case len(f[0]) == 8:
			if ifi == nil || name == ifi.Name {
				// The Linux kernel puts the IP
				// address in /proc/net/igmp in native
				// endianness.
				for i := 0; i+1 < len(f[0]); i += 2 {
					b[i/2], _ = xtoi2(f[0][i:i+2], 0)
				}
				i := *(*uint32)(unsafe.Pointer(&b[:4][0]))
				ifma := IPAddr{IP: IPv4(byte(i>>24), byte(i>>16), byte(i>>8), byte(i))}
				ifmat = append(ifmat, ifma.toAddr())
			}
		}
	}
	return ifmat
}

func parseProcNetIGMP6(path string, ifi *Interface) []Addr {
	fd, err := open(path)
	if err != nil {
		return nil
	}
	defer fd.close()

	var ifmat []Addr
	b := make([]byte, IPv6len)
	for l, ok := fd.readLine(); ok; l, ok = fd.readLine() {
		f := splitAtBytes(l, " \r\t\n")
		if len(f) < 6 {
			continue
		}
		if ifi == nil || f[1] == ifi.Name {
			for i := 0; i+1 < len(f[2]); i += 2 {
				b[i/2], _ = xtoi2(f[2][i:i+2], 0)
			}
			ifma := IPAddr{IP: IP{b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], b[10], b[11], b[12], b[13], b[14], b[15]}}
			ifmat = append(ifmat, ifma.toAddr())
		}
	}
	return ifmat
}
<|endoftext|>


<commit_before>package s3

// Tests for the PFS' S3 emulation API. Note that, in calls to
// `tu.UniqueString`, all lowercase characters are used, unlike in other
// tests. This is in order to generate repo names that are also valid bucket
// names. Otherwise minio complains that the bucket name is not valid.

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"testing"

	minio "github.com/minio/minio-go"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/server/pfs/server"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	tu "github.com/pachyderm/pachyderm/src/server/pkg/testutil"
)

func serve(t *testing.T, pc *client.APIClient) (*http.Server, uint16) {
	port := tu.UniquePort()
	srv := Server(pc, port)

	go func() {
		if err := srv.ListenAndServe(); err != http.ErrServerClosed {
			t.Fatalf("http server returned an error: %v", err)
		}
	}()

	// Wait for the server to start
	require.NoError(t, backoff.Retry(func() error {
		c := &http.Client{}
		res, err := c.Get(fmt.Sprintf("http://127.0.0.1:%d/_ping", port))
		if err != nil {
			return err
		} else if res.StatusCode != 200 {
			return fmt.Errorf("Unexpected status code: %d", res.StatusCode)
		}
		return nil
	}, backoff.NewTestingBackOff()))

	return srv, port
}

func getObject(c *minio.Client, repo, branch, file string) (string, error) {
	obj, err := c.GetObject(repo, fmt.Sprintf("%s/%s", branch, file))
	if err != nil {
		return "", err
	}
	defer func() { err = obj.Close() }()
	bytes, err := ioutil.ReadAll(obj)
	if err != nil {
		return "", err
	}
	return string(bytes), err
}

func TestGetFile(t *testing.T) {
	repo := tu.UniqueString("testgetfile")
	pc := server.GetPachClient(t)
	require.NoError(t, pc.CreateRepo(repo))
	_, err := pc.PutFile(repo, "master", "file", strings.NewReader("content"))
	require.NoError(t, err)

	srv, port := serve(t, pc)
	c, err := minio.New(fmt.Sprintf("127.0.0.1:%d", port), "id", "secret", false)
	require.NoError(t, err)

	fetchedContent, err := getObject(c, repo, "master", "file")
	require.NoError(t, err)
	require.Equal(t, "content", fetchedContent)

	require.NoError(t, srv.Close())
}

func TestGetFileInBranch(t *testing.T) {
	repo := tu.UniqueString("testgetfileinbranch")
	pc := server.GetPachClient(t)
	require.NoError(t, pc.CreateRepo(repo))
	require.NoError(t, pc.CreateBranch(repo, "branch", "", nil))
	_, err := pc.PutFile(repo, "branch", "file", strings.NewReader("content"))
	require.NoError(t, err)

	srv, port := serve(t, pc)
	c, err := minio.New(fmt.Sprintf("127.0.0.1:%d", port), "id", "secret", false)
	require.NoError(t, err)

	fetchedContent, err := getObject(c, repo, "branch", "file")
	require.NoError(t, err)
	require.Equal(t, "content", fetchedContent)

	require.NoError(t, srv.Close())
}

func TestNonExistingBranch(t *testing.T) {
	repo := tu.UniqueString("testnonexistingbranch")
	pc := server.GetPachClient(t)
	require.NoError(t, pc.CreateRepo(repo))

	srv, port := serve(t, pc)
	c, err := minio.New(fmt.Sprintf("127.0.0.1:%d", port), "id", "secret", false)
	require.NoError(t, err)

	_, err = getObject(c, repo, "branch", "file")
	require.YesError(t, err)
	require.Equal(t, err.Error(), "The specified key does not exist.")
	require.NoError(t, srv.Close())
}

func TestNonExistingRepo(t *testing.T) {
	repo := tu.UniqueString("testnonexistingrepo")
	pc := server.GetPachClient(t)

	srv, port := serve(t, pc)
	c, err := minio.New(fmt.Sprintf("127.0.0.1:%d", port), "id", "secret", false)
	require.NoError(t, err)

	_, err = getObject(c, repo, "master", "file")
	require.YesError(t, err)
	require.Equal(t, err.Error(), "The specified bucket does not exist.")
	require.NoError(t, srv.Close())
}

func TestPutObject(t *testing.T) {
	repo := tu.UniqueString("testputobject")
	pc := server.GetPachClient(t)
	require.NoError(t, pc.CreateRepo(repo))
	require.NoError(t, pc.CreateBranch(repo, "branch", "", nil))

	srv, port := serve(t, pc)
	c, err := minio.New(fmt.Sprintf("127.0.0.1:%d", port), "id", "secret", false)
	require.NoError(t, err)

	_, err = c.PutObject(repo, fmt.Sprintf("%s/%s", "branch", "file"), strings.NewReader("content"), "text/plain")
	require.NoError(t, err)

	fetchedContent, err := getObject(c, repo, "branch", "file")
	require.NoError(t, err)
	require.Equal(t, "content", fetchedContent)

	require.NoError(t, srv.Close())
}
<commit_msg>Added test for listing buckets<commit_after>package s3

// Tests for the PFS' S3 emulation API. Note that, in calls to
// `tu.UniqueString`, all lowercase characters are used, unlike in other
// tests. This is in order to generate repo names that are also valid bucket
// names. Otherwise minio complains that the bucket name is not valid.

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"testing"
	"time"

	minio "github.com/minio/minio-go"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/server/pfs/server"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	tu "github.com/pachyderm/pachyderm/src/server/pkg/testutil"
)

func serve(t *testing.T, pc *client.APIClient) (*http.Server, uint16) {
	port := tu.UniquePort()
	srv := Server(pc, port)

	go func() {
		if err := srv.ListenAndServe(); err != http.ErrServerClosed {
			t.Fatalf("http server returned an error: %v", err)
		}
	}()

	// Wait for the server to start
	require.NoError(t, backoff.Retry(func() error {
		c := &http.Client{}
		res, err := c.Get(fmt.Sprintf("http://127.0.0.1:%d/_ping", port))
		if err != nil {
			return err
		} else if res.StatusCode != 200 {
			return fmt.Errorf("Unexpected status code: %d", res.StatusCode)
		}
		return nil
	}, backoff.NewTestingBackOff()))

	return srv, port
}

func getObject(c *minio.Client, repo, branch, file string) (string, error) {
	obj, err := c.GetObject(repo, fmt.Sprintf("%s/%s", branch, file))
	if err != nil {
		return "", err
	}
	defer func() { err = obj.Close() }()
	bytes, err := ioutil.ReadAll(obj)
	if err != nil {
		return "", err
	}
	return string(bytes), err
}

func TestListBuckets(t *testing.T) {
	startTime := time.Now()

	pc := server.GetPachClient(t)
	repo1 := tu.UniqueString("testlistbuckets1")
	require.NoError(t, pc.CreateRepo(repo1))
	repo2 := tu.UniqueString("testlistbuckets2")
	require.NoError(t, pc.CreateRepo(repo2))

	endTime := time.Now()

	srv, port := serve(t, pc)
	c, err := minio.New(fmt.Sprintf("127.0.0.1:%d", port), "id", "secret", false)
	require.NoError(t, err)

	buckets, err := c.ListBuckets()
	require.NoError(t, err)
	require.Equal(t, 2, len(buckets))

	for _, bucket := range buckets {
		require.EqualOneOf(t, []string{repo1, repo2}, bucket.Name)
		require.True(t, startTime.Before(bucket.CreationDate))
		require.True(t, endTime.After(bucket.CreationDate))
	}

	require.NoError(t, srv.Close())
}

func TestGetFile(t *testing.T) {
	repo := tu.UniqueString("testgetfile")
	pc := server.GetPachClient(t)
	require.NoError(t, pc.CreateRepo(repo))
	_, err := pc.PutFile(repo, "master", "file", strings.NewReader("content"))
	require.NoError(t, err)

	srv, port := serve(t, pc)
	c, err := minio.New(fmt.Sprintf("127.0.0.1:%d", port), "id", "secret", false)
	require.NoError(t, err)

	fetchedContent, err := getObject(c, repo, "master", "file")
	require.NoError(t, err)
	require.Equal(t, "content", fetchedContent)

	require.NoError(t, srv.Close())
}

func TestGetFileInBranch(t *testing.T) {
	repo := tu.UniqueString("testgetfileinbranch")
	pc := server.GetPachClient(t)
	require.NoError(t, pc.CreateRepo(repo))
	require.NoError(t, pc.CreateBranch(repo, "branch", "", nil))
	_, err := pc.PutFile(repo, "branch", "file", strings.NewReader("content"))
	require.NoError(t, err)

	srv, port := serve(t, pc)
	c, err := minio.New(fmt.Sprintf("127.0.0.1:%d", port), "id", "secret", false)
	require.NoError(t, err)

	fetchedContent, err := getObject(c, repo, "branch", "file")
	require.NoError(t, err)
	require.Equal(t, "content", fetchedContent)

	require.NoError(t, srv.Close())
}

func TestNonExistingBranch(t *testing.T) {
	repo := tu.UniqueString("testnonexistingbranch")
	pc := server.GetPachClient(t)
	require.NoError(t, pc.CreateRepo(repo))

	srv, port := serve(t, pc)
	c, err := minio.New(fmt.Sprintf("127.0.0.1:%d", port), "id", "secret", false)
	require.NoError(t, err)

	_, err = getObject(c, repo, "branch", "file")
	require.YesError(t, err)
	require.Equal(t, err.Error(), "The specified key does not exist.")
	require.NoError(t, srv.Close())
}

func TestNonExistingRepo(t *testing.T) {
	repo := tu.UniqueString("testnonexistingrepo")
	pc := server.GetPachClient(t)

	srv, port := serve(t, pc)
	c, err := minio.New(fmt.Sprintf("127.0.0.1:%d", port), "id", "secret", false)
	require.NoError(t, err)

	_, err = getObject(c, repo, "master", "file")
	require.YesError(t, err)
	require.Equal(t, err.Error(), "The specified bucket does not exist.")
	require.NoError(t, srv.Close())
}

func TestPutObject(t *testing.T) {
	repo := tu.UniqueString("testputobject")
	pc := server.GetPachClient(t)
	require.NoError(t, pc.CreateRepo(repo))
	require.NoError(t, pc.CreateBranch(repo, "branch", "", nil))

	srv, port := serve(t, pc)
	c, err := minio.New(fmt.Sprintf("127.0.0.1:%d", port), "id", "secret", false)
	require.NoError(t, err)

	_, err = c.PutObject(repo, fmt.Sprintf("%s/%s", "branch", "file"), strings.NewReader("content"), "text/plain")
	require.NoError(t, err)

	fetchedContent, err := getObject(c, repo, "branch", "file")
	require.NoError(t, err)
	require.Equal(t, "content", fetchedContent)

	require.NoError(t, srv.Close())
}
<|endoftext|>


<commit_before>// This package implements a provisioner for Packer that executes a
// saltstack highstate within the remote machine
package salt

import (
	"fmt"
	"github.com/mitchellh/iochan"
	"github.com/mitchellh/mapstructure"
	"github.com/mitchellh/packer/packer"
	"io"
	"log"
	"sort"
	"strings"
)

var Ui packer.Ui

type config struct {
	// If true, skips installing Salt. Defaults to false.
	SkipInstall bool `mapstructure:"skip_install"`
}

type Provisioner struct {
	config config
}

func (p *Provisioner) Prepare(raws ...interface{}) error {
	var md mapstructure.Metadata
	decoderConfig := &mapstructure.DecoderConfig{
		Metadata: &md,
		Result:   &p.config,
	}

	decoder, err := mapstructure.NewDecoder(decoderConfig)
	if err != nil {
		return err
	}

	for _, raw := range raws {
		err := decoder.Decode(raw)
		if err != nil {
			return err
		}
	}

	// Accumulate any errors
	errs := make([]error, 0)

	// Unused keys are errors
	if len(md.Unused) > 0 {
		sort.Strings(md.Unused)
		for _, unused := range md.Unused {
			if unused != "type" && !strings.HasPrefix(unused, "packer_") {
				errs = append(
					errs, fmt.Errorf("Unknown configuration key: %s", unused))
			}
		}
	}

	if len(errs) > 0 {
		return &packer.MultiError{errs}
	}

	return nil
}

func InstallSalt(comm packer.Communicator) (err error) {
	Ui.Say("Installing Salt")
	cmd := "wget -O - http://bootstrap.saltstack.org | sudo sh"
	if err = executeCommand(cmd, comm); err != nil {
		return fmt.Errorf("Unable to install Salt: %d", err)
	}

	return nil
}

func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
	var err error
	Ui = ui

	if !p.config.SkipInstall {
		if err = InstallSalt(comm); err != nil {
			return fmt.Errorf("Error installing Salt: %s", err)
		}
	}

	return nil
}

func executeCommand(command string, comm packer.Communicator) (err error) {
	// Setup the remote command
	stdout_r, stdout_w := io.Pipe()
	stderr_r, stderr_w := io.Pipe()

	var cmd packer.RemoteCmd
	cmd.Command = command
	cmd.Stdout = stdout_w
	cmd.Stderr = stderr_w

	log.Printf("Executing command: %s", cmd.Command)
	err = comm.Start(&cmd)
	if err != nil {
		return fmt.Errorf("Failed executing command: %s", err)
	}

	exitChan := make(chan int, 1)
	stdoutChan := iochan.DelimReader(stdout_r, '\n')
	stderrChan := iochan.DelimReader(stderr_r, '\n')

	go func() {
		defer stdout_w.Close()
		defer stderr_w.Close()

		cmd.Wait()
		exitChan <- cmd.ExitStatus
	}()

OutputLoop:
	for {
		select {
		case output := <-stderrChan:
			Ui.Message(strings.TrimSpace(output))
		case output := <-stdoutChan:
			Ui.Message(strings.TrimSpace(output))
		case exitStatus := <-exitChan:
			log.Printf("Chef Solo provisioner exited with status %d", exitStatus)

			if exitStatus != 0 {
				return fmt.Errorf("Command exited with non-zero exit status: %d", exitStatus)
			}

			break OutputLoop
		}
	}

	// Make sure we finish off stdout/stderr because we may have gotten
	// a message from the exit channel first.
	for output := range stdoutChan {
		Ui.Message(output)
	}

	for output := range stderrChan {
		Ui.Message(output)
	}

	return nil
}
<commit_msg>provisioner/salt: pass args to bootstrap<commit_after>// This package implements a provisioner for Packer that executes a
// saltstack highstate within the remote machine
package salt

import (
	"fmt"
	"github.com/mitchellh/iochan"
	"github.com/mitchellh/mapstructure"
	"github.com/mitchellh/packer/packer"
	"io"
	"log"
	"sort"
	"strings"
)

var Ui packer.Ui

type config struct {
	// If true, run the salt-bootstrap script
	SkipBootstrap bool   `mapstructure:"skip_bootstrap"`
	BootstrapArgs string `mapstructure:"bootstrap_args"`
}

type Provisioner struct {
	config config
}

func (p *Provisioner) Prepare(raws ...interface{}) error {
	var md mapstructure.Metadata
	decoderConfig := &mapstructure.DecoderConfig{
		Metadata: &md,
		Result:   &p.config,
	}

	decoder, err := mapstructure.NewDecoder(decoderConfig)
	if err != nil {
		return err
	}

	for _, raw := range raws {
		err := decoder.Decode(raw)
		if err != nil {
			return err
		}
	}

	// Accumulate any errors
	errs := make([]error, 0)

	// Unused keys are errors
	if len(md.Unused) > 0 {
		sort.Strings(md.Unused)
		for _, unused := range md.Unused {
			if unused != "type" && !strings.HasPrefix(unused, "packer_") {
				errs = append(
					errs, fmt.Errorf("Unknown configuration key: %s", unused))
			}
		}
	}

	if len(errs) > 0 {
		return &packer.MultiError{errs}
	}

	return nil
}

func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
	var err error
	Ui = ui

	if !p.config.SkipBootstrap {
		cmd := fmt.Sprintf("wget -O - http://bootstrap.saltstack.org | sudo sh -s %s", p.config.BootstrapArgs)
		Ui.Say(fmt.Sprintf("Installing Salt with command %s", cmd))
		if err = executeCommand(cmd, comm); err != nil {
			return fmt.Errorf("Unable to install Salt: %d", err)
		}
	}
	return nil
}

func executeCommand(command string, comm packer.Communicator) (err error) {
	// Setup the remote command
	stdout_r, stdout_w := io.Pipe()
	stderr_r, stderr_w := io.Pipe()

	var cmd packer.RemoteCmd
	cmd.Command = command
	cmd.Stdout = stdout_w
	cmd.Stderr = stderr_w

	log.Printf("Executing command: %s", cmd.Command)
	err = comm.Start(&cmd)
	if err != nil {
		return fmt.Errorf("Failed executing command: %s", err)
	}

	exitChan := make(chan int, 1)
	stdoutChan := iochan.DelimReader(stdout_r, '\n')
	stderrChan := iochan.DelimReader(stderr_r, '\n')

	go func() {
		defer stdout_w.Close()
		defer stderr_w.Close()

		cmd.Wait()
		exitChan <- cmd.ExitStatus
	}()

OutputLoop:
	for {
		select {
		case output := <-stderrChan:
			Ui.Message(strings.TrimSpace(output))
		case output := <-stdoutChan:
			Ui.Message(strings.TrimSpace(output))
		case exitStatus := <-exitChan:
			log.Printf("Chef Solo provisioner exited with status %d", exitStatus)

			if exitStatus != 0 {
				return fmt.Errorf("Command exited with non-zero exit status: %d", exitStatus)
			}

			break OutputLoop
		}
	}

	// Make sure we finish off stdout/stderr because we may have gotten
	// a message from the exit channel first.
	for output := range stdoutChan {
		Ui.Message(output)
	}

	for output := range stderrChan {
		Ui.Message(output)
	}

	return nil
}
<|endoftext|>


<commit_before>// Package gateway implements the actual gateway which will listen for requests
// and forward them to servers wrapped by gatewayrpc
package gateway

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"sync"

	"github.com/gorilla/rpc/v2"
	"github.com/levenlabs/gatewayrpc"
	"github.com/levenlabs/go-llog"
	"github.com/levenlabs/go-srvclient"
	"github.com/levenlabs/golib/rpcutil"
)

type remoteService struct {
	gatewayrpc.Service
	*url.URL
	srv bool
}

// Request contains all the data about an incoming request which is currently
// known. It can be used in conjunction to RequestCallback in order to provide
// extra functionality or checks to request coming through.
//
// Notably, rpc.CodecRequest can be used to read in the arguments of the call
// and write back responses or errors to the client if no forwarding should be
// done.
//
// Also, the Body of the *http.Request will be filled with the raw body of
// content which originally came in, and there's no need to actually close it.
// the *http.Request's URL field will also have been modified to reflect the url
// the request will be being forwarded to.
type Request struct {
	http.ResponseWriter
	*http.Request
	ServiceName string
	gatewayrpc.Method
	rpc.CodecRequest
}

// Gateway is an http.Handler which implements the JSON RPC2 spec, but forwards
// all of its requests onto backend services
type Gateway struct {
	services map[string]remoteService
	mutex    sync.RWMutex
	codecs   map[string]rpc.Codec

	// BackupHandler, if not nil, will be used to handle the requests which
	// don't have a corresponding backend service to forward to (based on their
	// method)
	BackupHandler http.Handler

	// RequestCallback, if not nil, will be called just before actually
	// forwarding a request onto its backend service. If false is returned, the
	// forwarding will not be done. See the Request docstring for more on what
	// is actually possible with this
	RequestCallback func(Request) bool

	// CORSMatch, if not nil, will be used against the Origin header. and if it
	// matches Access-Control-Allow-* headers will be sent back, including an
	// Allow-Access-Control-Origin matching the sent in Origin
	CORSMatch *regexp.Regexp
}

// NewGateway returns an instantiated Gateway object
func NewGateway() Gateway {
	return Gateway{
		services: map[string]remoteService{},
		codecs:   map[string]rpc.Codec{},
	}
}

// returns a copy of the given url, with the host potentially resolved using a
// srv request
func resolveURL(uu *url.URL) (*url.URL, bool) {
	var srv bool
	uu2 := *uu
	host, err := srvclient.SRV(uu2.Host)
	if err == nil {
		srv = true
		uu2.Host = host
	}
	return &uu2, srv
}

// AddURL performs the RPC.GetServices request against the given url, and will
// add all returned services to its mapping.
//
// All DNS will be attempted to be resolved using SRV records first, and will
// use a normal DNS request as a backup
func (g Gateway) AddURL(u string) error {
	uu, err := url.Parse(u)
	if err != nil {
		return err
	}

	uu2, srvOK := resolveURL(uu)

	res := gatewayrpc.GetServicesRes{}
	if err = rpcutil.JSONRPC2Call(uu2.String(), &res, "RPC.GetServices", &struct{}{}); err != nil {
		return err
	}

	for _, srv := range res.Services {
		for m := range srv.Methods {
			llog.Debug("adding method", llog.KV{"service": srv.Name, "method": m})
		}
	}

	g.mutex.Lock()
	defer g.mutex.Unlock()
	for _, srv := range res.Services {
		g.services[srv.Name] = remoteService{
			Service: srv,
			URL:     uu,
			srv:     srvOK,
		}
	}
	return nil
}

// RegisterCodec is used to register an encoder/decoder which will operate on
// requests with the given contentType
func (g Gateway) RegisterCodec(codec rpc.Codec, contentType string) {
	g.codecs[strings.ToLower(contentType)] = codec
}

// returns the string form of the url which should handle this method
// ("Service.MethodName"), as well as the method's details. Returns an error if
// the method couldn't be found
func (g Gateway) getMethod(mStr string) (*url.URL, string, gatewayrpc.Method, error) {
	var m gatewayrpc.Method

	parts := strings.SplitN(mStr, ".", 2)
	if len(parts) != 2 {
		return nil, "", m, errors.New("invalid method endpoint given")
	}
	srvName, mName := parts[0], parts[1]

	g.mutex.RLock()
	rsrv, ok := g.services[srvName]
	g.mutex.RUnlock()
	if !ok {
		return nil, srvName, m, errors.New("unknown service name")
	}

	m, ok = rsrv.Methods[mName]
	if !ok {
		return nil, srvName, m, errors.New("unknown method name")
	}

	if rsrv.srv {
		uu, srvOK := resolveURL(rsrv.URL)
		if !srvOK {
			return nil, srvName, m, errors.New("could not resolve host via srv")
		}
		return uu, srvName, m, nil
	}

	return rsrv.URL, srvName, m, nil
}

// GetMethodURL returns the url which should be used to call the given method
// ("Service.MethodName"). If the service was originally resolved using a srv
// request it will be re-resolved everytime this is called, in order to
// load-balance across instances. Will return an error if the service is
// unknown, or the resolving fails for some reason.
func (g Gateway) GetMethodURL(mStr string) (*url.URL, error) {
	u, _, _, err := g.getMethod(mStr)
	return u, err
}

// We really only need the params part of this, we can get everything else from
// the codec
type serverRequest struct {
	// A Structured value to pass as arguments to the method.
	Params *json.RawMessage `json:"params"`
}

func (g Gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	kv := rpcutil.RequestKV(r)
	llog.Debug("ServeHTTP called", kv)

	// Possibly check CORS and set the headers to send back if it matches
	origin := r.Header.Get("Origin")
	if origin != "" && g.CORSMatch != nil && g.CORSMatch.MatchString(origin) {
		w.Header().Add("Access-Control-Allow-Origin", origin)
		w.Header().Add("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
		w.Header().Add("Access-Control-Allow-Credentials", "true")
		w.Header().Add("Access-Control-Allow-Headers", "DNT, User-Agent, X-Requested-With, Content-Type")
	}

	// We allow OPTIONS so that preflighted requests can get CORS back
	if r.Method == "OPTIONS" {
		return
	}

	if r.Method != "POST" {
		kv["method"] = r.Method
		llog.Warn("Invalid method sent", kv)
		writeErrorf(w, 405, "rpc: POST method required, received %q", r.Method)
		return
	}

	contentType := r.Header.Get("Content-Type")
	idx := strings.Index(contentType, ";")
	if idx != -1 {
		contentType = contentType[:idx]
	}
	codec := g.codecs[strings.ToLower(contentType)]
	if codec == nil {
		kv["contentType"] = contentType
		llog.Warn("Unrecognized Content-Type", kv)
		writeErrorf(w, 415, "rpc: unrecognized Content-Type: %q", contentType)
		return
	}

	bodyCopy := bytes.NewBuffer(make([]byte, 0, r.ContentLength))
	r.Body = ioutil.NopCloser(io.TeeReader(r.Body, bodyCopy))
	codecReq := codec.NewRequest(r)
	r.Body = ioutil.NopCloser(bodyCopy)

	// At this point the original r.Body was read into the codecReq, but the
	// TeeReader has also copied it into bodyCopy, which was then set to r.Body

	m, err := codecReq.Method()
	if err != nil {
		kv["err"] = err
		llog.Warn("Err retrieving method from codec", kv)
		codecReq.WriteError(w, 400, err)
		return
	}

	kv["method"] = m
	llog.Debug("Received method call", kv)

	u, rpcSrvName, rpcMethod, err := g.getMethod(m)
	if err != nil {
		if g.BackupHandler != nil {
			g.BackupHandler.ServeHTTP(w, r)
			return
		}
		kv["err"] = err
		llog.Warn("Error retrieving method from gatway", kv)
		codecReq.WriteError(w, 400, err)
	}
	r.URL = u
	r.RequestURI = ""

	if g.RequestCallback != nil && !g.RequestCallback(Request{
		ResponseWriter: w,
		Request:        r,
		ServiceName:    rpcSrvName,
		Method:         rpcMethod,
		CodecRequest:   codecReq,
	}) {
		return
	}

	res, err := http.DefaultClient.Do(r)
	if err != nil {
		kv["err"] = err
		llog.Error("Error forwarding request", kv)
		codecReq.WriteError(w, 500, err)
		return
	}
	defer res.Body.Close()

	//pass along the content-type
	w.Header().Set("Content-Type", contentType)
	io.Copy(w, res.Body)
	res.Body.Close()
}

func writeErrorf(w http.ResponseWriter, status int, msg string, args ...interface{}) {
	w.WriteHeader(status)
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	fmt.Fprint(w, fmt.Sprintf(msg, args...))
}
<commit_msg>implement polling to periodically refresh the backend services<commit_after>// Package gateway implements the actual gateway which will listen for requests
// and forward them to servers wrapped by gatewayrpc
package gateway

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/gorilla/rpc/v2"
	"github.com/levenlabs/gatewayrpc"
	"github.com/levenlabs/go-llog"
	"github.com/levenlabs/go-srvclient"
	"github.com/levenlabs/golib/rpcutil"
)

type remoteService struct {
	gatewayrpc.Service
	*url.URL
	origURL string
	srv     bool
}

// Request contains all the data about an incoming request which is currently
// known. It can be used in conjunction to RequestCallback in order to provide
// extra functionality or checks to request coming through.
//
// Notably, rpc.CodecRequest can be used to read in the arguments of the call
// and write back responses or errors to the client if no forwarding should be
// done.
//
// Also, the Body of the *http.Request will be filled with the raw body of
// content which originally came in, and there's no need to actually close it.
// the *http.Request's URL field will also have been modified to reflect the url
// the request will be being forwarded to.
type Request struct {
	http.ResponseWriter
	*http.Request
	ServiceName string
	gatewayrpc.Method
	rpc.CodecRequest
}

// Gateway is an http.Handler which implements the JSON RPC2 spec, but forwards
// all of its requests onto backend services
type Gateway struct {
	services map[string]remoteService
	mutex    sync.RWMutex
	codecs   map[string]rpc.Codec
	poll     <-chan time.Time

	// BackupHandler, if not nil, will be used to handle the requests which
	// don't have a corresponding backend service to forward to (based on their
	// method)
	BackupHandler http.Handler

	// RequestCallback, if not nil, will be called just before actually
	// forwarding a request onto its backend service. If false is returned, the
	// forwarding will not be done. See the Request docstring for more on what
	// is actually possible with this
	RequestCallback func(Request) bool

	// CORSMatch, if not nil, will be used against the Origin header. and if it
	// matches Access-Control-Allow-* headers will be sent back, including an
	// Allow-Access-Control-Origin matching the sent in Origin
	CORSMatch *regexp.Regexp
}

// NewGateway returns an instantiated Gateway object
func NewGateway() Gateway {
	return Gateway{
		services: map[string]remoteService{},
		codecs:   map[string]rpc.Codec{},
		poll:     time.Tick(30 * time.Second),
	}
}

// returns a copy of the given url, with the host potentially resolved using a
// srv request
func resolveURL(uu *url.URL) (*url.URL, bool) {
	var srv bool
	uu2 := *uu
	host, err := srvclient.SRV(uu2.Host)
	if err == nil {
		srv = true
		uu2.Host = host
	}
	return &uu2, srv
}

// AddURL performs the RPC.GetServices request against the given url, and will
// add all returned services to its mapping.
//
// All DNS will be attempted to be resolved using SRV records first, and will
// use a normal DNS request as a backup
func (g Gateway) AddURL(u string) error {
	uu, err := url.Parse(u)
	if err != nil {
		return err
	}

	uu2, srvOK := resolveURL(uu)

	res := gatewayrpc.GetServicesRes{}
	if err = rpcutil.JSONRPC2Call(uu2.String(), &res, "RPC.GetServices", &struct{}{}); err != nil {
		return err
	}

	for _, srv := range res.Services {
		for m := range srv.Methods {
			llog.Debug("adding method", llog.KV{"service": srv.Name, "method": m})
		}
	}

	g.mutex.Lock()
	defer g.mutex.Unlock()
	for _, srv := range res.Services {
		g.services[srv.Name] = remoteService{
			Service: srv,
			URL:     uu,
			origURL: u,
			srv:     srvOK,
		}
	}
	return nil
}

func (g Gateway) refreshURLs() {
	llog.Debug("refreshing urls")
	g.mutex.RLock()
	srvs := make([]remoteService, 0, len(g.services))
	for _, srv := range g.services {
		srvs = append(srvs, srv)
	}
	g.mutex.RUnlock()

	for _, srv := range srvs {
		if err := g.AddURL(srv.origURL); err != nil {
			llog.Error("error refreshing url", llog.KV{
				"url": srv.origURL,
				"err": err,
			})
		}
	}
}

// RegisterCodec is used to register an encoder/decoder which will operate on
// requests with the given contentType
func (g Gateway) RegisterCodec(codec rpc.Codec, contentType string) {
	g.codecs[strings.ToLower(contentType)] = codec
}

// returns the string form of the url which should handle this method
// ("Service.MethodName"), as well as the method's details. Returns an error if
// the method couldn't be found
func (g Gateway) getMethod(mStr string) (*url.URL, string, gatewayrpc.Method, error) {
	var m gatewayrpc.Method

	parts := strings.SplitN(mStr, ".", 2)
	if len(parts) != 2 {
		return nil, "", m, errors.New("invalid method endpoint given")
	}
	srvName, mName := parts[0], parts[1]

	g.mutex.RLock()
	rsrv, ok := g.services[srvName]
	g.mutex.RUnlock()
	if !ok {
		return nil, srvName, m, errors.New("unknown service name")
	}

	m, ok = rsrv.Methods[mName]
	if !ok {
		return nil, srvName, m, errors.New("unknown method name")
	}

	if rsrv.srv {
		uu, srvOK := resolveURL(rsrv.URL)
		if !srvOK {
			return nil, srvName, m, errors.New("could not resolve host via srv")
		}
		return uu, srvName, m, nil
	}

	return rsrv.URL, srvName, m, nil
}

// GetMethodURL returns the url which should be used to call the given method
// ("Service.MethodName"). If the service was originally resolved using a srv
// request it will be re-resolved everytime this is called, in order to
// load-balance across instances. Will return an error if the service is
// unknown, or the resolving fails for some reason.
func (g Gateway) GetMethodURL(mStr string) (*url.URL, error) {
	u, _, _, err := g.getMethod(mStr)
	return u, err
}

// We really only need the params part of this, we can get everything else from
// the codec
type serverRequest struct {
	// A Structured value to pass as arguments to the method.
	Params *json.RawMessage `json:"params"`
}

func (g Gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// Periodically we want to refresh the services that gateway knows about. We
	// do it in a new goroutine so we don't block this actual request. We don't
	// want to simply have a dedicated go routine looping over the poll channel
	// to do this because having an http.Handler spawn up its own routine that's
	// making requests and doing stuff is kind of unexpected behavior
	select {
	case <-g.poll:
		go g.refreshURLs()
	default:
	}

	kv := rpcutil.RequestKV(r)
	llog.Debug("ServeHTTP called", kv)

	// Possibly check CORS and set the headers to send back if it matches
	origin := r.Header.Get("Origin")
	if origin != "" && g.CORSMatch != nil && g.CORSMatch.MatchString(origin) {
		w.Header().Add("Access-Control-Allow-Origin", origin)
		w.Header().Add("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
		w.Header().Add("Access-Control-Allow-Credentials", "true")
		w.Header().Add("Access-Control-Allow-Headers", "DNT, User-Agent, X-Requested-With, Content-Type")
	}

	// We allow OPTIONS so that preflighted requests can get CORS back
	if r.Method == "OPTIONS" {
		return
	}

	if r.Method != "POST" {
		kv["method"] = r.Method
		llog.Warn("Invalid method sent", kv)
		writeErrorf(w, 405, "rpc: POST method required, received %q", r.Method)
		return
	}

	contentType := r.Header.Get("Content-Type")
	idx := strings.Index(contentType, ";")
	if idx != -1 {
		contentType = contentType[:idx]
	}
	codec := g.codecs[strings.ToLower(contentType)]
	if codec == nil {
		kv["contentType"] = contentType
		llog.Warn("Unrecognized Content-Type", kv)
		writeErrorf(w, 415, "rpc: unrecognized Content-Type: %q", contentType)
		return
	}

	bodyCopy := bytes.NewBuffer(make([]byte, 0, r.ContentLength))
	r.Body = ioutil.NopCloser(io.TeeReader(r.Body, bodyCopy))
	codecReq := codec.NewRequest(r)
	r.Body = ioutil.NopCloser(bodyCopy)

	// At this point the original r.Body was read into the codecReq, but the
	// TeeReader has also copied it into bodyCopy, which was then set to r.Body

	m, err := codecReq.Method()
	if err != nil {
		kv["err"] = err
		llog.Warn("Err retrieving method from codec", kv)
		codecReq.WriteError(w, 400, err)
		return
	}

	kv["method"] = m
	llog.Debug("Received method call", kv)

	u, rpcSrvName, rpcMethod, err := g.getMethod(m)
	if err != nil {
		if g.BackupHandler != nil {
			g.BackupHandler.ServeHTTP(w, r)
			return
		}
		kv["err"] = err
		llog.Warn("Error retrieving method from gatway", kv)
		codecReq.WriteError(w, 400, err)
	}
	r.URL = u
	r.RequestURI = ""

	if g.RequestCallback != nil && !g.RequestCallback(Request{
		ResponseWriter: w,
		Request:        r,
		ServiceName:    rpcSrvName,
		Method:         rpcMethod,
		CodecRequest:   codecReq,
	}) {
		return
	}

	res, err := http.DefaultClient.Do(r)
	if err != nil {
		kv["err"] = err
		llog.Error("Error forwarding request", kv)
		codecReq.WriteError(w, 500, err)
		return
	}
	defer res.Body.Close()

	//pass along the content-type
	w.Header().Set("Content-Type", contentType)
	io.Copy(w, res.Body)
	res.Body.Close()
}

func writeErrorf(w http.ResponseWriter, status int, msg string, args ...interface{}) {
	w.WriteHeader(status)
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	fmt.Fprint(w, fmt.Sprintf(msg, args...))
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"fmt"
	"strings"
	"golang.org/x/net/websocket"
	"time"
	"encoding/json"
	"net/http"
	"bytes"

	/*"strconv"*/

)
type JSONTime time.Time
type Message struct {

	Solarflare	bool   	`json:"solarFlare"`
	Temperature float64 `json:"temperature"`
	Radiation   int32	`json:"radiation"`
	Stamp		time.Time `json:"stamp"`
}

func readSensorMessages(ws *websocket.Conn, incomingMessages chan string, sensor int) {
	for {
		var in [] byte
			if err := websocket.Message.Receive(ws, &in); err != nil {
				return
			}
			incomingMessages <- string(in)
	}
}

func postAvgReading(msg Message,gc string){
		admin_password := "1234"
/*		log.Println("=========================================")
	    log.Println("game console ulr:",gc)
		log.Println("Aadmin Pass:",admin_password)
		log.Println("AVG Solar Flare:",msg.Solarflare)
		log.Println("AVG Radation:",msg.Radiation)
		log.Println("AVG Temperature:",msg.Temperature)
		log.Println("TimeStamp:",msg.Stamp)

*/		log.Println("=========================================")

		b, err := json.Marshal(msg)
		fmt.Printf("Marshal Json: %s\n",b)

		var jsonStr=[]byte(b)
	    req, err := http.NewRequest("POST", gc, bytes.NewBuffer(jsonStr))
    	req.Header.Set("X-Auth-Token", admin_password)
    	req.Header.Set("Content-Type", "application/json")

		client := &http.Client{}
		resp, err := client.Do(req)
		 if err != nil {
		      panic(err)
		}
		defer resp.Body.Close()

		log.Println("response Status:", resp.Status)
		log.Println("response Headers:", resp.Header)

		time.Sleep(time.Second)
}

func getAvgReading(messagelist [5]string,gc string){

	var msg Message
	var avgMsg Message
	avgMsg.Radiation=0
	avgMsg.Temperature=0
	var avgFlare float32=0

	var c int = 0
	for i := range messagelist {

		   fmt.Println("sendor #",i,":",messagelist[i])
		   b:=[]byte(messagelist[i])

	 	   err := json.Unmarshal(b, &msg)
			if err != nil {
				fmt.Printf("Error: %s\n", err.Error())
				//return
			}else{
				c++;
			/*
				log.Println("Current Solar Flare:",msg.Solarflare)
				log.Println("Current Radation:",msg.Radiation)
				log.Println("current Temperature:",msg.Temperature)
				log.Println("current TimeStamp:",msg.Stamp)
				*/
				avgMsg.Radiation=avgMsg.Radiation+msg.Radiation
				avgMsg.Temperature=avgMsg.Temperature+msg.Temperature
				if msg.Solarflare{
					avgFlare++
				}
				avgMsg.Stamp=msg.Stamp
			}

	}
	if c >0 {
		fmt.Println("counter#:",c)
		log.Println("Total Solar Flare:",avgMsg.Solarflare)
		log.Println("Total Radation:",avgMsg.Radiation)
		log.Println("Total Temperature:",avgMsg.Temperature)
		log.Println("Last TimeStamp:",avgMsg.Stamp)

		avgMsg.Radiation=avgMsg.Radiation/int32(c)
		avgMsg.Temperature=avgMsg.Temperature/float64(c)
		avgFlare=avgFlare/float32(c)
		if avgFlare>=0.5{
				avgMsg.Solarflare = true
		}
		 postAvgReading(avgMsg,gc)

	}
}

func main() {

	//Read Env Variables
	//sensor_endpoints_str :="104.40.93.11:8080,104.40.93.11:8081,104.40.93.11:8082,104.40.93.11:8083"

	sensor_endpoints_str:=os.Getenv("SENSOR_ENDPOINT")
	controller_endpoint := os.Getenv("GC_ENDPOINT")
	//controller_endpoint ="104.40.89.227:8080"

	log.Printf("Current Sensor EndPoints '%s'", sensor_endpoints_str)
	log.Printf("Current Controller '%s'", controller_endpoint)

	//Prepare Game Controller URL
	controller_url := "http://" + controller_endpoint + "/api/readings"
	log.Printf("Current Sensor EndPoints %s ",controller_url)

	sensor_endpoint_list := strings.Split(sensor_endpoints_str, ",")
	for i := 0; i < len(sensor_endpoint_list); i++ {
		sensor_endpoint_list[i] = "ws://" + sensor_endpoint_list[i] +"/ws"
		log.Printf("Sensor EndPoint '%s'", sensor_endpoint_list[i])
	}

	//var message Message
	var chans [5]chan string
	var incomingMessages [5]string

	for i := range chans {
	   chans[i] = make(chan string)
	}

	for i := 0; i < len(sensor_endpoint_list); i++ {
			log.Println("Process Socket:", sensor_endpoint_list[i])
			ws, err := websocket.Dial(sensor_endpoint_list[i], "", sensor_endpoint_list[i])
			if err != nil {
				log.Printf("Dial failed: %s\n", err.Error())

			}else{

				log.Println("Print mesg:", chans[i])
				go readSensorMessages(ws, chans[i],i)

			}
	}

	for {
		select {

		case sensor0 := <- chans[0]:
			//fmt.Println("Message from IncommingMessage1:", sensor0)
			incomingMessages[0]=sensor0

		case sensor1 := <- chans[1]:
			//fmt.Println("Message from IncommingMessage2:", sensor1)
			incomingMessages[1]=sensor1

		case sensor2 := <- chans[2]:
			//fmt.Println("Message from IncommingMessage3:", sensor2)
			incomingMessages[2]=sensor2

		case sensor3 := <- chans[3]:
			//fmt.Println("Message from IncommingMessage4:", sensor3)
			incomingMessages[3]=sensor3

		case sensor4 := <- chans[4]:
			//fmt.Println("Message from IncommingMessage4:", sensor4)
			incomingMessages[4]=sensor4

		default:
			getAvgReading(incomingMessages,controller_url)
		}
	}

}
<commit_msg>Update Avg sensor procedure<commit_after>package main

import (
	"log"
	"os"
	"fmt"
	"strings"
	"golang.org/x/net/websocket"
	"time"
	"encoding/json"
	"net/http"
	"bytes"

	/*"strconv"*/

)
type JSONTime time.Time
type Message struct {

	Solarflare	bool   	`json:"solarFlare"`
	Temperature float64 `json:"temperature"`
	Radiation   int32	`json:"radiation"`
	Stamp		time.Time `json:"stamp"`
}

func readSensorMessages(ws *websocket.Conn, incomingMessages chan string, sensor int) {
	for {
		var in [] byte
			if err := websocket.Message.Receive(ws, &in); err != nil {
				fmt.Println(err.Error())
				return
			}
			incomingMessages <- string(in)
	}
}

func postAvgReading(msg Message,gc string){
		admin_password := "1234"
/*		log.Println("=========================================")
	    log.Println("game console ulr:",gc)
		log.Println("Aadmin Pass:",admin_password)
		log.Println("AVG Solar Flare:",msg.Solarflare)
		log.Println("AVG Radation:",msg.Radiation)
		log.Println("AVG Temperature:",msg.Temperature)
		log.Println("TimeStamp:",msg.Stamp)

*/		log.Println("=========================================")

		b, err := json.Marshal(msg)
		fmt.Printf("Marshal Json: %s\n",b)

		var jsonStr=[]byte(b)
	    req, err := http.NewRequest("POST", gc, bytes.NewBuffer(jsonStr))
    	req.Header.Set("X-Auth-Token", admin_password)
    	req.Header.Set("Content-Type", "application/json")

		client := &http.Client{}
		resp, err := client.Do(req)
		 if err != nil {
		      panic(err)
		}
		defer resp.Body.Close()

		log.Println("response Status:", resp.Status)
		log.Println("response Headers:", resp.Header)

		time.Sleep(time.Second)
}

func getAvgReading(messagelist *[5]string,gc string){

	var msg Message
	var avgMsg Message

	var avgFlare float32=0
	for{
		avgMsg.Radiation=0
		avgMsg.Temperature=0
		avgFlare =0
		var c int = 0
		for i := range messagelist {
			 if len(messagelist[0])>0{
			   fmt.Println("sendor #",i,":",messagelist[i])
			   b:=[]byte(messagelist[i])

		 	   err := json.Unmarshal(b, &msg)
				if err != nil {
					fmt.Printf("Error: %s\n", err.Error())
					//return
				}else{
					c++;
				/*
					log.Println("Current Solar Flare:",msg.Solarflare)
					log.Println("Current Radation:",msg.Radiation)
					log.Println("current Temperature:",msg.Temperature)
					log.Println("current TimeStamp:",msg.Stamp)
					*/
					avgMsg.Radiation=avgMsg.Radiation+msg.Radiation
					avgMsg.Temperature=avgMsg.Temperature+msg.Temperature
					if msg.Solarflare{
						avgFlare++
					}
					avgMsg.Stamp=msg.Stamp
				}
			}

		}
		if c >0 {
			fmt.Println("counter#:",c)
			log.Println("Total Solar Flare:",avgMsg.Solarflare)
			log.Println("Total Radation:",avgMsg.Radiation)
			log.Println("Total Temperature:",avgMsg.Temperature)
			log.Println("Last TimeStamp:",avgMsg.Stamp)

			avgMsg.Radiation=avgMsg.Radiation/int32(c)
			avgMsg.Temperature=avgMsg.Temperature/float64(c)
			avgFlare=avgFlare/float32(c)
			if avgFlare>=0.5{
					avgMsg.Solarflare = true
			}
			 postAvgReading(avgMsg,gc)

		}
	}
}

func main() {

	//Read Env Variables
	//sensor_endpoints_str :="104.40.93.11:8080,104.40.93.11:8081,104.40.93.11:8082,104.40.93.11:8083"

	sensor_endpoints_str:=os.Getenv("SENSOR_ENDPOINT")
	controller_endpoint := os.Getenv("GC_ENDPOINT")
	//controller_endpoint ="104.40.89.227:8080"

	log.Printf("Current Sensor EndPoints '%s'", sensor_endpoints_str)
	log.Printf("Current Controller '%s'", controller_endpoint)

	//Prepare Game Controller URL
	controller_url := "http://" + controller_endpoint + "/api/readings"
	log.Printf("Current Sensor EndPoints %s ",controller_url)

	sensor_endpoint_list := strings.Split(sensor_endpoints_str, ",")
	for i := 0; i < len(sensor_endpoint_list); i++ {
		sensor_endpoint_list[i] = "ws://" + sensor_endpoint_list[i] +"/ws"
		log.Printf("Sensor EndPoint '%s'", sensor_endpoint_list[i])
	}

	//var message Message
	var chans [5]chan string
	var incomingMessages [5]string

	for i := range chans {
	   chans[i] = make(chan string)
	}

	for i := 0; i < len(sensor_endpoint_list); i++ {
			log.Println("Process Socket:", sensor_endpoint_list[i])
			ws, err := websocket.Dial(sensor_endpoint_list[i], "", sensor_endpoint_list[i])
			if err != nil {
				log.Printf("Dial failed: %s\n", err.Error())

			}else{

				log.Println("Print msg:", chans[i])
				go readSensorMessages(ws, chans[i],i)

			}
	}

	go getAvgReading(&incomingMessages,controller_url)

	for {
		select {

		case sensor0 := <- chans[0]:
			//fmt.Println("Message from IncommingMessage1:", sensor0)
			incomingMessages[0]=sensor0

		case sensor1 := <- chans[1]:
			//fmt.Println("Message from IncommingMessage2:", sensor1)
			incomingMessages[1]=sensor1

		case sensor2 := <- chans[2]:
			//fmt.Println("Message from IncommingMessage3:", sensor2)
			incomingMessages[2]=sensor2

		case sensor3 := <- chans[3]:
			//fmt.Println("Message from IncommingMessage4:", sensor3)
			incomingMessages[3]=sensor3

		case sensor4 := <- chans[4]:
			//fmt.Println("Message from IncommingMessage4:", sensor4)
			incomingMessages[4]=sensor4

		//default:

		}
	}

}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package snippets

import (
	"bytes"
	"strings"
	"testing"

	"github.com/GoogleCloudPlatform/golang-samples/internal/testutil"
)

func TestAutoPunctuation(t *testing.T) {
	testutil.SystemTest(t)

	var buf bytes.Buffer
	err := autoPunctuation(&buf, "../testdata/commercial_mono.wav")
	if err != nil {
		t.Fatal(err)
	}

	if got := buf.String(); !strings.Contains(got, "Okay. Sure.") {
		t.Fatalf(`autoPunctuation(../testdata/commercial_mono.wav) = %q; want "Okay. Sure"`, got)
	}
}
<commit_msg>speech: use different check for auto punctuation (#1106)<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package snippets

import (
	"bytes"
	"strings"
	"testing"

	"github.com/GoogleCloudPlatform/golang-samples/internal/testutil"
)

func TestAutoPunctuation(t *testing.T) {
	testutil.SystemTest(t)

	var buf bytes.Buffer
	err := autoPunctuation(&buf, "../testdata/commercial_mono.wav")
	if err != nil {
		t.Fatal(err)
	}

	if got, want := buf.String(), "I'm here"; !strings.Contains(got, want) {
		t.Fatalf(`autoPunctuation(../testdata/commercial_mono.wav) = %q; want %q`, got, want)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"errors"
	"log"
	"net"
	"regexp"
	"strconv"
	"strings"

	"github.com/samalba/dockerclient"
)

type DockerManager struct {
	config *Config
	list   ServiceListProvider
	docker *dockerclient.DockerClient
}

func NewDockerManager(c *Config, list ServiceListProvider, tlsConfig *tls.Config) (*DockerManager, error) {
	docker, err := dockerclient.NewDockerClient(c.dockerHost, tlsConfig)
	if err != nil {
		return nil, err
	}

	return &DockerManager{config: c, list: list, docker: docker}, nil
}

func (d *DockerManager) Start() error {
	ec := make(chan error)
	d.docker.StartMonitorEvents(d.eventCallback, ec)
	go func() {
		for {
			log.Println("Event error", <-ec)
		}
	}()

	containers, err := d.docker.ListContainers(false, false, "")
	if err != nil {
		return errors.New("Error connecting to docker socket: " + err.Error())
	}

	for _, container := range containers {
		service, err := d.getService(container.Id)
		if err != nil {
			log.Println(err)
			continue
		}
		d.list.AddService(container.Id, *service)
	}

	return nil
}

func (d *DockerManager) Stop() {
	d.docker.StopAllMonitorEvents()
}

func (d *DockerManager) getService(id string) (*Service, error) {
	inspect, err := d.docker.InspectContainer(id)
	if err != nil {
		return nil, err
	}

	service := NewService()
	service.Aliases = make([]string, 0)

	service.Image = getImageName(inspect.Config.Image)
	if imageNameIsSHA(service.Image, inspect.Image) {
		log.Println("Warning: Can't route ", id[:10], ", image", service.Image, "is not a tag.")
		service.Image = ""
	}
	service.Name = cleanContainerName(inspect.Name)
	service.Ip = net.ParseIP(inspect.NetworkSettings.IPAddress)

	service = overrideFromEnv(service, splitEnv(inspect.Config.Env))
	if service == nil {
		return nil, errors.New("Skipping " + id)
	}

	return service, nil
}

func (d *DockerManager) eventCallback(event *dockerclient.Event, ec chan error, args ...interface{}) {
	//log.Printf("Received event: %#v %#v\n", *event, args)

	switch event.Status {
	case "die", "stop", "kill":
		// Errors can be ignored here because there can be no-op events.
		d.list.RemoveService(event.Id)
	case "start", "restart":
		service, err := d.getService(event.Id)
		if err != nil {
			ec <- err
			return
		}

		d.list.AddService(event.Id, *service)
	}
}

func getImageName(tag string) string {
	if index := strings.LastIndex(tag, "/"); index != -1 {
		tag = tag[index+1:]
	}
	if index := strings.LastIndex(tag, ":"); index != -1 {
		tag = tag[:index]
	}
	return tag
}

func imageNameIsSHA(image, sha string) bool {
	// Hard to make a judgement on small image names.
	if len(image) < 4 {
		return false
	}
	// Image name is not HEX
	matched, _ := regexp.MatchString("^[0-9a-f]+$", image)
	if !matched {
		return false
	}
	return strings.HasPrefix(sha, image)
}

func cleanContainerName(name string) string {
	return strings.Replace(name, "/", "", -1)
}

func splitEnv(in []string) (out map[string]string) {
	out = make(map[string]string, len(in))
	for _, exp := range in {
		parts := strings.SplitN(exp, "=", 2)
		var value string
		if len(parts) > 1 {
			value = strings.Trim(parts[1], " ") // trim just in case
		}
		out[strings.Trim(parts[0], " ")] = value
	}
	return
}

func overrideFromEnv(in *Service, env map[string]string) (out *Service) {
	var region string
	for k, v := range env {
		if k == "DNSDOCK_IGNORE" || k == "SERVICE_IGNORE" {
			return nil
		}

		if k == "DNSDOCK_ALIAS" {
			in.Aliases = strings.Split(v, ",")
		}

		if k == "DNSDOCK_NAME" {
			in.Name = v
		}

		if k == "SERVICE_TAGS" {
			if len(v) == 0 {
				in.Name = ""
			} else {
				in.Name = strings.Split(v, ",")[0]
			}
		}

		if k == "DNSDOCK_IMAGE" || k == "SERVICE_NAME" {
			in.Image = v
		}

		if k == "DNSDOCK_TTL" {
			if ttl, err := strconv.Atoi(v); err == nil {
				in.Ttl = ttl
			}
		}

		if k == "SERVICE_REGION" {
			region = v
		}
	}

	if len(region) > 0 {
		in.Image = in.Image + "." + region
	}
	out = in
	return
}
<commit_msg>Add support for labels<commit_after>package main

import (
	"crypto/tls"
	"errors"
	"log"
	"net"
	"regexp"
	"strconv"
	"strings"

	"github.com/samalba/dockerclient"
)

type DockerManager struct {
	config *Config
	list   ServiceListProvider
	docker *dockerclient.DockerClient
}

func NewDockerManager(c *Config, list ServiceListProvider, tlsConfig *tls.Config) (*DockerManager, error) {
	docker, err := dockerclient.NewDockerClient(c.dockerHost, tlsConfig)
	if err != nil {
		return nil, err
	}

	return &DockerManager{config: c, list: list, docker: docker}, nil
}

func (d *DockerManager) Start() error {
	ec := make(chan error)
	d.docker.StartMonitorEvents(d.eventCallback, ec)
	go func() {
		for {
			log.Println("Event error", <-ec)
		}
	}()

	containers, err := d.docker.ListContainers(false, false, "")
	if err != nil {
		return errors.New("Error connecting to docker socket: " + err.Error())
	}

	for _, container := range containers {
		service, err := d.getService(container.Id)
		if err != nil {
			log.Println(err)
			continue
		}
		d.list.AddService(container.Id, *service)
	}

	return nil
}

func (d *DockerManager) Stop() {
	d.docker.StopAllMonitorEvents()
}

func (d *DockerManager) getService(id string) (*Service, error) {
	inspect, err := d.docker.InspectContainer(id)
	if err != nil {
		return nil, err
	}

	service := NewService()
	service.Aliases = make([]string, 0)

	service.Image = getImageName(inspect.Config.Image)
	if imageNameIsSHA(service.Image, inspect.Image) {
		log.Println("Warning: Can't route ", id[:10], ", image", service.Image, "is not a tag.")
		service.Image = ""
	}
	service.Name = cleanContainerName(inspect.Name)
	service.Ip = net.ParseIP(inspect.NetworkSettings.IPAddress)

	service = overrideFromLabels(service, inspect.Config.Labels)
	service = overrideFromEnv(service, splitEnv(inspect.Config.Env))
	if service == nil {
		return nil, errors.New("Skipping " + id)
	}

	return service, nil
}

func (d *DockerManager) eventCallback(event *dockerclient.Event, ec chan error, args ...interface{}) {
	//log.Printf("Received event: %#v %#v\n", *event, args)

	switch event.Status {
	case "die", "stop", "kill":
		// Errors can be ignored here because there can be no-op events.
		d.list.RemoveService(event.Id)
	case "start", "restart":
		service, err := d.getService(event.Id)
		if err != nil {
			ec <- err
			return
		}

		d.list.AddService(event.Id, *service)
	}
}

func getImageName(tag string) string {
	if index := strings.LastIndex(tag, "/"); index != -1 {
		tag = tag[index+1:]
	}
	if index := strings.LastIndex(tag, ":"); index != -1 {
		tag = tag[:index]
	}
	return tag
}

func imageNameIsSHA(image, sha string) bool {
	// Hard to make a judgement on small image names.
	if len(image) < 4 {
		return false
	}
	// Image name is not HEX
	matched, _ := regexp.MatchString("^[0-9a-f]+$", image)
	if !matched {
		return false
	}
	return strings.HasPrefix(sha, image)
}

func cleanContainerName(name string) string {
	return strings.Replace(name, "/", "", -1)
}

func splitEnv(in []string) (out map[string]string) {
	out = make(map[string]string, len(in))
	for _, exp := range in {
		parts := strings.SplitN(exp, "=", 2)
		var value string
		if len(parts) > 1 {
			value = strings.Trim(parts[1], " ") // trim just in case
		}
		out[strings.Trim(parts[0], " ")] = value
	}
	return
}

func overrideFromLabels(in *Service, labels map[string]string) (out *Service) {
	var region string
	for k, v := range labels {
		if k == "com.dnsdock.ignore" {
			return nil
		}

		if k == "com.dnsdock.alias" {
			in.Aliases = strings.Split(v, ",")
		}

		if k == "com.dnsdock.name" {
			in.Name = v
		}

		if k == "com.dnsdock.tags" {
			if len(v) == 0 {
				in.Name = ""
			} else {
				in.Name = strings.Split(v, ",")[0]
			}
		}

		if k == "com.dnsdock.image" {
			in.Image = v
		}

		if k == "com.dnsdock.ttl" {
			if ttl, err := strconv.Atoi(v); err == nil {
				in.Ttl = ttl
			}
		}

		if k == "com.dnsdock.region" {
			region = v
		}
	}

	if len(region) > 0 {
		in.Image = in.Image + "." + region
	}
	out = in
	return
}

func overrideFromEnv(in *Service, env map[string]string) (out *Service) {
	var region string
	for k, v := range env {
		if k == "DNSDOCK_IGNORE" || k == "SERVICE_IGNORE" {
			return nil
		}

		if k == "DNSDOCK_ALIAS" {
			in.Aliases = strings.Split(v, ",")
		}

		if k == "DNSDOCK_NAME" {
			in.Name = v
		}

		if k == "SERVICE_TAGS" {
			if len(v) == 0 {
				in.Name = ""
			} else {
				in.Name = strings.Split(v, ",")[0]
			}
		}

		if k == "DNSDOCK_IMAGE" || k == "SERVICE_NAME" {
			in.Image = v
		}

		if k == "DNSDOCK_TTL" {
			if ttl, err := strconv.Atoi(v); err == nil {
				in.Ttl = ttl
			}
		}

		if k == "SERVICE_REGION" {
			region = v
		}
	}

	if len(region) > 0 {
		in.Image = in.Image + "." + region
	}
	out = in
	return
}
<|endoftext|>


<commit_before>// Import this in all *_test.go files to initialize ~/.tendermint_test.

package tendermint_test

import (
	"os"
	"path"
	"strings"

	. "github.com/tendermint/go-common"
	cfg "github.com/tendermint/go-config"
)

func init() {
	// Creates ~/.tendermint_test
	EnsureDir(os.Getenv("HOME")+"/.tendermint_test", 0700)
}

func ResetConfig(path string) {
	rootDir := os.Getenv("HOME") + "/.tendermint_test/" + path
	cfg.ApplyConfig(GetConfig(rootDir))
}

func initTMRoot(rootDir string) {
	// Remove ~/.tendermint_test_bak
	if FileExists(rootDir + "_bak") {
		err := os.RemoveAll(rootDir + "_bak")
		if err != nil {
			PanicSanity(err.Error())
		}
	}
	// Move ~/.tendermint_test to ~/.tendermint_test_bak
	if FileExists(rootDir) {
		err := os.Rename(rootDir, rootDir+"_bak")
		if err != nil {
			PanicSanity(err.Error())
		}
	}
	// Create new dir
	EnsureDir(rootDir, 0700)

	configFilePath := path.Join(rootDir, "config.toml")
	genesisFilePath := path.Join(rootDir, "genesis.json")
	privFilePath := path.Join(rootDir, "priv_validator.json")

	// Write default config file if missing.
	if !FileExists(configFilePath) {
		// Ask user for moniker
		// moniker := cfg.Prompt("Type hostname: ", "anonymous")
		MustWriteFile(configFilePath, []byte(defaultConfig("anonymous")), 0644)
	}
	if !FileExists(genesisFilePath) {
		MustWriteFile(genesisFilePath, []byte(defaultGenesis), 0644)
	}
	// we always overwrite the priv val
	MustWriteFile(privFilePath, []byte(defaultPrivValidator), 0644)
}

func GetConfig(rootDir string) cfg.Config {
	initTMRoot(rootDir)

	configFilePath := path.Join(rootDir, "config.toml")
	mapConfig, err := cfg.ReadMapConfigFromFile(configFilePath)
	if err != nil {
		Exit(Fmt("Could not read config: %v", err))
	}

	// Set defaults or panic
	if mapConfig.IsSet("chain_id") {
		Exit("Cannot set 'chain_id' via config.toml")
	}
	mapConfig.SetDefault("chain_id", "tendermint_test")
	mapConfig.SetDefault("genesis_file", rootDir+"/genesis.json")
	mapConfig.SetDefault("proxy_app", "dummy")
	mapConfig.SetDefault("moniker", "anonymous")
	mapConfig.SetDefault("node_laddr", "0.0.0.0:36656")
	mapConfig.SetDefault("fast_sync", false)
	mapConfig.SetDefault("skip_upnp", true)
	mapConfig.SetDefault("addrbook_file", rootDir+"/addrbook.json")
	mapConfig.SetDefault("priv_validator_file", rootDir+"/priv_validator.json")
	mapConfig.SetDefault("db_backend", "memdb")
	mapConfig.SetDefault("db_dir", rootDir+"/data")
	mapConfig.SetDefault("log_level", "debug")
	mapConfig.SetDefault("rpc_laddr", "0.0.0.0:36657")
	mapConfig.SetDefault("prof_laddr", "")
	mapConfig.SetDefault("revision_file", rootDir+"/revision")
	mapConfig.SetDefault("cswal", rootDir+"/data/cswal")
	mapConfig.SetDefault("cswal_light", false)

	mapConfig.SetDefault("block_size", 10000)
	mapConfig.SetDefault("timeout_propose", 100)
	mapConfig.SetDefault("timeout_propose_delta", 1)
	mapConfig.SetDefault("timeout_prevote", 1)
	mapConfig.SetDefault("timeout_prevote_delta", 1)
	mapConfig.SetDefault("timeout_precommit", 1)
	mapConfig.SetDefault("timeout_precommit_delta", 1)
	mapConfig.SetDefault("timeout_commit", 1)
	mapConfig.SetDefault("mempool_recheck", true)
	mapConfig.SetDefault("mempool_broadcast", true)

	return mapConfig
}

var defaultConfigTmpl = `# This is a TOML config file.
# For more information, see https://github.com/toml-lang/toml

proxy_app = "dummy"
moniker = "__MONIKER__"
node_laddr = "0.0.0.0:36656"
seeds = ""
fast_sync = false
db_backend = "memdb"
log_level = "debug"
rpc_laddr = "0.0.0.0:36657"
`

func defaultConfig(moniker string) (defaultConfig string) {
	defaultConfig = strings.Replace(defaultConfigTmpl, "__MONIKER__", moniker, -1)
	return
}

var defaultGenesis = `{
  "genesis_time": "0001-01-01T00:00:00.000Z",
  "chain_id": "tendermint_test",
  "validators": [
    {
      "pub_key": [
        1,
        "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
      ],
      "amount": 10,
      "name": ""
    }
  ],
  "app_hash": ""
}`

var defaultPrivValidator = `{
  "address": "D028C9981F7A87F3093672BF0D5B0E2A1B3ED456",
  "pub_key": [
    1,
    "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
  ],
  "priv_key": [
    1,
    "27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
  ],
  "last_height": 0,
  "last_round": 0,
  "last_step": 0
}`
<commit_msg>Fix tendermint_test config -- Address() spec has changed<commit_after>// Import this in all *_test.go files to initialize ~/.tendermint_test.

package tendermint_test

import (
	"os"
	"path"
	"strings"

	. "github.com/tendermint/go-common"
	cfg "github.com/tendermint/go-config"
)

func init() {
	// Creates ~/.tendermint_test
	EnsureDir(os.Getenv("HOME")+"/.tendermint_test", 0700)
}

func ResetConfig(path string) {
	rootDir := os.Getenv("HOME") + "/.tendermint_test/" + path
	cfg.ApplyConfig(GetConfig(rootDir))
}

func initTMRoot(rootDir string) {
	// Remove ~/.tendermint_test_bak
	if FileExists(rootDir + "_bak") {
		err := os.RemoveAll(rootDir + "_bak")
		if err != nil {
			PanicSanity(err.Error())
		}
	}
	// Move ~/.tendermint_test to ~/.tendermint_test_bak
	if FileExists(rootDir) {
		err := os.Rename(rootDir, rootDir+"_bak")
		if err != nil {
			PanicSanity(err.Error())
		}
	}
	// Create new dir
	EnsureDir(rootDir, 0700)

	configFilePath := path.Join(rootDir, "config.toml")
	genesisFilePath := path.Join(rootDir, "genesis.json")
	privFilePath := path.Join(rootDir, "priv_validator.json")

	// Write default config file if missing.
	if !FileExists(configFilePath) {
		// Ask user for moniker
		// moniker := cfg.Prompt("Type hostname: ", "anonymous")
		MustWriteFile(configFilePath, []byte(defaultConfig("anonymous")), 0644)
	}
	if !FileExists(genesisFilePath) {
		MustWriteFile(genesisFilePath, []byte(defaultGenesis), 0644)
	}
	// we always overwrite the priv val
	MustWriteFile(privFilePath, []byte(defaultPrivValidator), 0644)
}

func GetConfig(rootDir string) cfg.Config {
	initTMRoot(rootDir)

	configFilePath := path.Join(rootDir, "config.toml")
	mapConfig, err := cfg.ReadMapConfigFromFile(configFilePath)
	if err != nil {
		Exit(Fmt("Could not read config: %v", err))
	}

	// Set defaults or panic
	if mapConfig.IsSet("chain_id") {
		Exit("Cannot set 'chain_id' via config.toml")
	}
	mapConfig.SetDefault("chain_id", "tendermint_test")
	mapConfig.SetDefault("genesis_file", rootDir+"/genesis.json")
	mapConfig.SetDefault("proxy_app", "dummy")
	mapConfig.SetDefault("moniker", "anonymous")
	mapConfig.SetDefault("node_laddr", "0.0.0.0:36656")
	mapConfig.SetDefault("fast_sync", false)
	mapConfig.SetDefault("skip_upnp", true)
	mapConfig.SetDefault("addrbook_file", rootDir+"/addrbook.json")
	mapConfig.SetDefault("priv_validator_file", rootDir+"/priv_validator.json")
	mapConfig.SetDefault("db_backend", "memdb")
	mapConfig.SetDefault("db_dir", rootDir+"/data")
	mapConfig.SetDefault("log_level", "debug")
	mapConfig.SetDefault("rpc_laddr", "0.0.0.0:36657")
	mapConfig.SetDefault("prof_laddr", "")
	mapConfig.SetDefault("revision_file", rootDir+"/revision")
	mapConfig.SetDefault("cswal", rootDir+"/data/cswal")
	mapConfig.SetDefault("cswal_light", false)

	mapConfig.SetDefault("block_size", 10000)
	mapConfig.SetDefault("timeout_propose", 100)
	mapConfig.SetDefault("timeout_propose_delta", 1)
	mapConfig.SetDefault("timeout_prevote", 1)
	mapConfig.SetDefault("timeout_prevote_delta", 1)
	mapConfig.SetDefault("timeout_precommit", 1)
	mapConfig.SetDefault("timeout_precommit_delta", 1)
	mapConfig.SetDefault("timeout_commit", 1)
	mapConfig.SetDefault("mempool_recheck", true)
	mapConfig.SetDefault("mempool_broadcast", true)

	return mapConfig
}

var defaultConfigTmpl = `# This is a TOML config file.
# For more information, see https://github.com/toml-lang/toml

proxy_app = "dummy"
moniker = "__MONIKER__"
node_laddr = "0.0.0.0:36656"
seeds = ""
fast_sync = false
db_backend = "memdb"
log_level = "debug"
rpc_laddr = "0.0.0.0:36657"
`

func defaultConfig(moniker string) (defaultConfig string) {
	defaultConfig = strings.Replace(defaultConfigTmpl, "__MONIKER__", moniker, -1)
	return
}

var defaultGenesis = `{
  "genesis_time": "0001-01-01T00:00:00.000Z",
  "chain_id": "tendermint_test",
  "validators": [
    {
      "pub_key": [
        1,
        "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
      ],
      "amount": 10,
      "name": ""
    }
  ],
  "app_hash": ""
}`

var defaultPrivValidator = `{
  "address": "C12729D1CAC0E791FF12882E11A8A57620509722",
  "pub_key": [
    1,
    "3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
  ],
  "priv_key": [
    1,
    "27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8"
  ],
  "last_height": 0,
  "last_round": 0,
  "last_step": 0
}`
<|endoftext|>


<commit_before>// Copyright 2019 Red Hat, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package validate

import (
	"errors"
	"fmt"
	"reflect"
	"testing"

	"github.com/ajeddeloh/vcontext/path"
	"github.com/ajeddeloh/vcontext/report"
	"github.com/ajeddeloh/vcontext/tree"
)

var (
	dummy = errors.New("dummy error")
	empty = path.New("json")
)

// mkReport generates reports with a single error with the specified context, line, col and kind
func mkReport(err error, c path.ContextPath, k report.Kind, line, col int64) (r report.Report) {
	r.AddOn(c, err, k)
	if line == 0 {
		return
	}
	r.Entries[0].Marker.StartP = &tree.Pos{Line: line, Column: col}
	return
}

// mangleReport wipes the fields we don't populate in our tests so the tests don't need to be as verbose
func mangleReport(r *report.Report) {
	for i := range r.Entries {
		if sp := r.Entries[i].Marker.StartP; sp != nil {
			sp.Index = 0
		}
		r.Entries[i].Marker.EndP = nil
	}
}

type test struct{}

func (t test) Validate(c path.ContextPath) (r report.Report) {
	r.AddOnError(c, dummy)
	return
}

type test2 struct {
	Test test `json:"foobar"`
}

func TestValidateWithContext(t *testing.T) {
	tests := []struct {
		in    interface{}
		inRaw string
		out   report.Report
	}{
		{
			in:  struct{}{},
			out: report.Report{},
		},
		{
			in:  test{},
			out: mkReport(dummy, empty, report.Error, 0, 0),
		},
		{
			in:    test{},
			inRaw: "{   }",
			out:   mkReport(dummy, empty, report.Error, 1, 2),
		},
		{
			in:    struct{}{},
			inRaw: `{"foo":"bar"}`,
			out:   mkReport(fmt.Errorf("Unused key foo"), path.New("json", tree.Key("foo")), report.Warn, 1, 2),
		},
		{
			in:    test2{},
			inRaw: `{"foobar": {}}`,
			out:   mkReport(dummy, path.New("json", "foobar"), report.Error, 1, 13),
		},
	}

	for i, test := range tests {
		r := ValidateWithContext(test.in, []byte(test.inRaw))
		mangleReport(&r)
		if !reflect.DeepEqual(test.out, r) {
			t.Errorf("#%d: bad report: want %+v got %+v", i, test.out, r)
		}
	}
}
<commit_msg>config/validate: add tests for duplicate checking<commit_after>// Copyright 2019 Red Hat, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package validate

import (
	"errors"
	"fmt"
	"reflect"
	"testing"

	ignerrors "github.com/coreos/ignition/v2/config/shared/errors"

	"github.com/ajeddeloh/vcontext/path"
	"github.com/ajeddeloh/vcontext/report"
	"github.com/ajeddeloh/vcontext/tree"
)

var (
	dummy = errors.New("dummy error")
	empty = path.New("json")
)

// mkReport generates reports with a single error with the specified context, line, col and kind
func mkReport(err error, c path.ContextPath, k report.Kind, line, col int64) (r report.Report) {
	r.AddOn(c, err, k)
	if line == 0 {
		return
	}
	r.Entries[0].Marker.StartP = &tree.Pos{Line: line, Column: col}
	return
}

// mangleReport wipes the fields we don't populate in our tests so the tests don't need to be as verbose
func mangleReport(r *report.Report) {
	for i := range r.Entries {
		if sp := r.Entries[i].Marker.StartP; sp != nil {
			sp.Index = 0
		}
		r.Entries[i].Marker.EndP = nil
	}
}

type test struct{}

func (t test) Validate(c path.ContextPath) (r report.Report) {
	r.AddOnError(c, dummy)
	return
}

type testDup struct{}

func (t testDup) Key() string {
	return "same"
}

type test2 struct {
	Test test `json:"foobar"`
}

type test3 struct {
	NoDups []testDup `json:"dups"`
}

type test4 struct {
	Ignored []testDup `json:"dups"`
}

func (t test4) IgnoreDuplicates() map[string]struct{} {
	return map[string]struct{}{
		"Ignored": {},
	}
}

type test5 struct {
	NoDups1 []testDup `json:"dups1"`
	NoDups2 []testDup `json:"dups2"`
}

func (t test5) MergedKeys() map[string]string {
	return map[string]string{
		"NoDups1": "dups",
		"NoDups2": "dups",
	}
}

func TestValidateWithContext(t *testing.T) {
	tests := []struct {
		in    interface{}
		inRaw string
		out   report.Report
	}{
		{
			in: struct{}{},
		},
		{
			in:  test{},
			out: mkReport(dummy, empty, report.Error, 0, 0),
		},
		{
			in:    test{},
			inRaw: "{   }",
			out:   mkReport(dummy, empty, report.Error, 1, 2),
		},
		{
			in:    struct{}{},
			inRaw: `{"foo":"bar"}`,
			out:   mkReport(fmt.Errorf("Unused key foo"), path.New("json", tree.Key("foo")), report.Warn, 1, 2),
		},
		{
			in:    test2{},
			inRaw: `{"foobar": {}}`,
			out:   mkReport(dummy, path.New("json", "foobar"), report.Error, 1, 13),
		},
		{
			in: test3{},
		},
		{
			in: test3{
				NoDups: make([]testDup, 1, 1),
			},
		},
		{
			in: test3{
				NoDups: make([]testDup, 2, 2),
			},
			out: mkReport(ignerrors.ErrDuplicate, path.New("json", "dups", 1), report.Error, 0, 0),
		},
		{
			in: test4{
				Ignored: make([]testDup, 2, 2),
			},
		},
		{
			in: test5{
				NoDups1: make([]testDup, 1, 1),
			},
		},
		{
			in: test5{
				NoDups1: make([]testDup, 1, 1),
				NoDups2: make([]testDup, 1, 1),
			},
			out: mkReport(ignerrors.ErrDuplicate, path.New("json", "dups2", 0), report.Error, 0, 0),
		},
	}

	for i, test := range tests {
		r := ValidateWithContext(test.in, []byte(test.inRaw))
		mangleReport(&r)
		if !reflect.DeepEqual(test.out, r) {
			t.Errorf("#%d: bad report: want %+v got %+v", i, test.out, r)
		}
	}
}
<|endoftext|>


<commit_before>package connections

import (
	"sync"
	"time"

	"github.com/gorilla/websocket"
	"github.com/pquerna/ffjson/ffjson"
	"github.com/sirupsen/logrus"

	"github.com/ivan1993spb/snake-server/broadcast"
	"github.com/ivan1993spb/snake-server/game"
	"github.com/ivan1993spb/snake-server/player"
)

const (
	chanOutputMessageBuffer     = 128
	chanReadMessagesBuffer      = 128
	chanDecodeMessageBuffer     = 128
	chanEncodeMessageBuffer     = 128
	chanProxyInputMessageBuffer = 64
	chanInputMessagesBuffer     = 32
	chanSnakeCommandsBuffer     = 32
	chanMergeBytesBuffer        = 64

	sendInputMessageTimeout = time.Millisecond * 50
)

type ConnectionWorker struct {
	conn   *websocket.Conn
	logger logrus.FieldLogger

	chsInput    []chan InputMessage
	chsInputMux *sync.RWMutex

	flagStarted bool
}

func NewConnectionWorker(conn *websocket.Conn, logger logrus.FieldLogger) *ConnectionWorker {
	return &ConnectionWorker{
		conn:        conn,
		logger:      logger,
		chsInput:    make([]chan InputMessage, 0),
		chsInputMux: &sync.RWMutex{},
	}
}

type ErrStartConnectionWorker string

func (e ErrStartConnectionWorker) Error() string {
	return "error start connection worker: " + string(e)
}

func (cw *ConnectionWorker) Start(stop <-chan struct{}, game *game.Game, broadcast *broadcast.GroupBroadcast, gameBytes <-chan []byte) error {
	if cw.flagStarted {
		return ErrStartConnectionWorker("connection worker already started")
	}

	cw.flagStarted = true

	broadcast.BroadcastMessage("user joined your game group")

	// Input
	chInputBytes, chStop := cw.read()
	chInputMessages := cw.decode(chInputBytes, chStop)
	cw.broadcastInputMessage(chInputMessages, chStop)
	chCommands := cw.listenSnakeCommands(chStop, cw.input(chStop, chanInputMessagesBuffer))
	cw.listenPlayerBroadcasts(chStop, cw.input(chStop, chanInputMessagesBuffer), broadcast)

	p := player.NewPlayer(cw.logger, game.World())

	// Output
	chPlayer := p.Start(chStop, chCommands)
	chOutputBytes := cw.encode(chStop, cw.listenPlayer(chStop, chPlayer))
	cw.write(cw.mergeBytesCh(chStop, chOutputBytes, gameBytes), chStop)

	select {
	case <-chStop:
		// On connection error
	case <-stop:
		// External stop
	}

	broadcast.BroadcastMessage("user left your game group")

	cw.stopInputs()

	return nil
}

func (cw *ConnectionWorker) stopInputs() {
	cw.chsInputMux.Lock()
	defer cw.chsInputMux.Unlock()

	for _, ch := range cw.chsInput {
		close(ch)
	}

	cw.chsInput = cw.chsInput[:0]
}

func (cw *ConnectionWorker) read() (<-chan []byte, <-chan struct{}) {
	chout := make(chan []byte, chanReadMessagesBuffer)
	chstop := make(chan struct{}, 0)

	go func() {
		defer close(chout)
		defer close(chstop)

		for {
			messageType, data, err := cw.conn.ReadMessage()
			if err != nil {
				if !websocket.IsCloseError(err, websocket.CloseNormalClosure) {
					cw.logger.Errorln("read input message error:", err)
				}
				return
			}

			if websocket.TextMessage != messageType {
				cw.logger.Warning("unexpected input message type")
				continue
			}

			chout <- data
		}
	}()

	return chout, chstop
}

func (cw *ConnectionWorker) decode(chin <-chan []byte, stop <-chan struct{}) <-chan InputMessage {
	chout := make(chan InputMessage, chanDecodeMessageBuffer)

	go func() {
		defer close(chout)

		var decoder = ffjson.NewDecoder()

		for {
			select {
			case data, ok := <-chin:
				if !ok {
					return
				}

				var inputMessage InputMessage
				if err := decoder.Decode(data, &inputMessage); err != nil {
					cw.logger.Errorln("decode input message error:", err)
				} else {
					select {
					case <-stop:
						return
					case chout <- inputMessage:
					}
				}
			case <-stop:
				return
			}
		}
	}()

	return chout
}

func (cw *ConnectionWorker) broadcastInputMessage(chin <-chan InputMessage, stop <-chan struct{}) {
	go func() {
		for {
			select {
			case inputMessage, ok := <-chin:
				if !ok {
					return
				}

				cw.doBroadcast(inputMessage, stop)
			case <-stop:
				return
			}
		}
	}()
}

func (cw *ConnectionWorker) doBroadcast(message InputMessage, stop <-chan struct{}) {
	cw.chsInputMux.RLock()
	defer cw.chsInputMux.RUnlock()

	for _, ch := range cw.chsInput {
		select {
		case ch <- message:
		case <-stop:
			return
		}
	}
}

func (cw *ConnectionWorker) input(stop <-chan struct{}, buffer uint) <-chan InputMessage {
	chProxy := make(chan InputMessage, chanProxyInputMessageBuffer)

	cw.chsInputMux.Lock()
	cw.chsInput = append(cw.chsInput, chProxy)
	cw.chsInputMux.Unlock()

	chout := make(chan InputMessage, buffer)

	go func() {
		defer close(chout)
		defer func() {
			go func() {
				for range chProxy {
				}
			}()

			cw.chsInputMux.Lock()
			for i := range cw.chsInput {
				if cw.chsInput[i] == chProxy {
					cw.chsInput = append(cw.chsInput[:i], cw.chsInput[i+1:]...)
					close(chProxy)
					break
				}
			}
			cw.chsInputMux.Unlock()
		}()

		for {
			select {
			case <-stop:
				return
			case inputMessage, ok := <-chProxy:
				if !ok {
					return
				}
				cw.sendInputMessage(chout, inputMessage, stop, sendInputMessageTimeout)
			}
		}
	}()

	return chout
}

func (cw *ConnectionWorker) sendInputMessage(ch chan InputMessage, inputMessage InputMessage, stop <-chan struct{}, timeout time.Duration) {
	const tickSize = 5

	var timer = time.NewTimer(timeout)
	defer timer.Stop()

	var ticker = time.NewTicker(timeout / tickSize)
	defer ticker.Stop()

	if cap(ch) == 0 {
		select {
		case ch <- inputMessage:
		case <-stop:
		case <-timer.C:
		}
	} else {
		for {
			select {
			case ch <- inputMessage:
				return
			case <-stop:
				return
			case <-timer.C:
				return
			case <-ticker.C:
				if len(ch) == cap(ch) {
					select {
					case <-ch:
					case ch <- inputMessage:
						return
					case <-stop:
						return
					case <-timer.C:
						return
					}
				}
			}
		}
	}
}

func (cw *ConnectionWorker) mergeBytesCh(stop <-chan struct{}, chins ...<-chan []byte) <-chan []byte {
	chout := make(chan []byte, chanMergeBytesBuffer)

	wg := sync.WaitGroup{}
	wg.Add(len(chins))

	for _, chin := range chins {
		go func(chin <-chan []byte) {
			defer wg.Done()

			for {
				select {
				case <-stop:
					return
				case data, ok := <-chin:
					if !ok {
						return
					}

					select {
					case chout <- data:
					case <-stop:
						return
					}
				}
			}
		}(chin)
	}

	go func() {
		wg.Wait()
		close(chout)
	}()

	return chout
}

func (cw *ConnectionWorker) write(chin <-chan []byte, stop <-chan struct{}) {
	go func() {
		for {
			select {
			case data, ok := <-chin:
				if !ok {
					return
				}

				if err := cw.conn.WriteMessage(websocket.TextMessage, data); err != nil {
					if !websocket.IsCloseError(err, websocket.CloseNormalClosure) {
						cw.logger.Errorln("write output message error:", err)
					}
					return
				}
			case <-stop:
				return
			}
		}
	}()
}

func (cw *ConnectionWorker) encode(stop <-chan struct{}, chins ...<-chan OutputMessage) <-chan []byte {
	chout := make(chan []byte, chanEncodeMessageBuffer)

	wg := sync.WaitGroup{}
	wg.Add(len(chins))

	for _, chin := range chins {
		go func(chin <-chan OutputMessage) {
			defer wg.Done()
			for {
				select {
				case <-stop:
					return
				case message, ok := <-chin:
					if !ok {
						return
					}

					if data, err := ffjson.Marshal(message); err != nil {
						cw.logger.Errorln("encode output message error:", err)
					} else {
						select {
						case <-stop:
							return
						case chout <- data:
						}
					}
				}
			}
		}(chin)
	}

	go func() {
		wg.Wait()
		close(chout)
	}()

	return chout
}

func (cw *ConnectionWorker) listenPlayer(stop <-chan struct{}, chin <-chan player.Message) <-chan OutputMessage {
	chout := make(chan OutputMessage, chanOutputMessageBuffer)

	go func() {
		defer close(chout)

		for {
			select {
			case event, ok := <-chin:
				if !ok {
					return
				}

				outputMessage := OutputMessage{
					Type:    OutputMessageTypePlayer,
					Payload: event,
				}

				select {
				case chout <- outputMessage:
				case <-stop:
					return
				}
			case <-stop:
				return
			}
		}
	}()

	return chout
}

func (cw *ConnectionWorker) listenSnakeCommands(stop <-chan struct{}, chin <-chan InputMessage) <-chan string {
	chout := make(chan string, chanSnakeCommandsBuffer)

	go func() {
		defer close(chout)

		for {
			select {
			case message, ok := <-chin:
				if !ok {
					return
				}

				if message.Type == InputMessageTypeSnakeCommand {
					select {
					case chout <- message.Payload:
					case <-stop:
						return
					}
				}
			case <-stop:
				return
			}
		}
	}()

	return chout
}

func (cw *ConnectionWorker) listenPlayerBroadcasts(stop <-chan struct{}, chin <-chan InputMessage, b *broadcast.GroupBroadcast) {
	go func() {
		for {
			select {
			case message, ok := <-chin:
				if !ok {
					return
				}

				if message.Type == InputMessageTypeBroadcast {
					b.BroadcastMessage(broadcast.BroadcastMessage(message.Payload))
				}
			case <-stop:
				return
			}
		}
	}()
}
<commit_msg>Increase chan buffers in ConnectionWorker<commit_after>package connections

import (
	"sync"
	"time"

	"github.com/gorilla/websocket"
	"github.com/pquerna/ffjson/ffjson"
	"github.com/sirupsen/logrus"

	"github.com/ivan1993spb/snake-server/broadcast"
	"github.com/ivan1993spb/snake-server/game"
	"github.com/ivan1993spb/snake-server/player"
)

const (
	chanOutputMessageBuffer     = 256
	chanReadMessagesBuffer      = 128
	chanDecodeMessageBuffer     = 128
	chanEncodeMessageBuffer     = 1024
	chanProxyInputMessageBuffer = 64
	chanInputMessagesBuffer     = 64
	chanSnakeCommandsBuffer     = 64
	chanMergeBytesBuffer        = 1024

	sendInputMessageTimeout = time.Millisecond * 50
)

type ConnectionWorker struct {
	conn   *websocket.Conn
	logger logrus.FieldLogger

	chsInput    []chan InputMessage
	chsInputMux *sync.RWMutex

	flagStarted bool
}

func NewConnectionWorker(conn *websocket.Conn, logger logrus.FieldLogger) *ConnectionWorker {
	return &ConnectionWorker{
		conn:        conn,
		logger:      logger,
		chsInput:    make([]chan InputMessage, 0),
		chsInputMux: &sync.RWMutex{},
	}
}

type ErrStartConnectionWorker string

func (e ErrStartConnectionWorker) Error() string {
	return "error start connection worker: " + string(e)
}

func (cw *ConnectionWorker) Start(stop <-chan struct{}, game *game.Game, broadcast *broadcast.GroupBroadcast, gameBytes <-chan []byte) error {
	if cw.flagStarted {
		return ErrStartConnectionWorker("connection worker already started")
	}

	cw.flagStarted = true

	broadcast.BroadcastMessage("user joined your game group")

	// Input
	chInputBytes, chStop := cw.read()
	chInputMessages := cw.decode(chInputBytes, chStop)
	cw.broadcastInputMessage(chInputMessages, chStop)
	chCommands := cw.listenSnakeCommands(chStop, cw.input(chStop, chanInputMessagesBuffer))
	cw.listenPlayerBroadcasts(chStop, cw.input(chStop, chanInputMessagesBuffer), broadcast)

	p := player.NewPlayer(cw.logger, game.World())

	// Output
	chPlayer := p.Start(chStop, chCommands)
	chOutputBytes := cw.encode(chStop, cw.listenPlayer(chStop, chPlayer))
	cw.write(cw.mergeBytesCh(chStop, chOutputBytes, gameBytes), chStop)

	select {
	case <-chStop:
		// On connection error
	case <-stop:
		// External stop
	}

	broadcast.BroadcastMessage("user left your game group")

	cw.stopInputs()

	return nil
}

func (cw *ConnectionWorker) stopInputs() {
	cw.chsInputMux.Lock()
	defer cw.chsInputMux.Unlock()

	for _, ch := range cw.chsInput {
		close(ch)
	}

	cw.chsInput = cw.chsInput[:0]
}

func (cw *ConnectionWorker) read() (<-chan []byte, <-chan struct{}) {
	chout := make(chan []byte, chanReadMessagesBuffer)
	chstop := make(chan struct{}, 0)

	go func() {
		defer close(chout)
		defer close(chstop)

		for {
			messageType, data, err := cw.conn.ReadMessage()
			if err != nil {
				if !websocket.IsCloseError(err, websocket.CloseNormalClosure) {
					cw.logger.Errorln("read input message error:", err)
				}
				return
			}

			if websocket.TextMessage != messageType {
				cw.logger.Warning("unexpected input message type")
				continue
			}

			chout <- data
		}
	}()

	return chout, chstop
}

func (cw *ConnectionWorker) decode(chin <-chan []byte, stop <-chan struct{}) <-chan InputMessage {
	chout := make(chan InputMessage, chanDecodeMessageBuffer)

	go func() {
		defer close(chout)

		var decoder = ffjson.NewDecoder()

		for {
			select {
			case data, ok := <-chin:
				if !ok {
					return
				}

				var inputMessage InputMessage
				if err := decoder.Decode(data, &inputMessage); err != nil {
					cw.logger.Errorln("decode input message error:", err)
				} else {
					select {
					case <-stop:
						return
					case chout <- inputMessage:
					}
				}
			case <-stop:
				return
			}
		}
	}()

	return chout
}

func (cw *ConnectionWorker) broadcastInputMessage(chin <-chan InputMessage, stop <-chan struct{}) {
	go func() {
		for {
			select {
			case inputMessage, ok := <-chin:
				if !ok {
					return
				}

				cw.doBroadcast(inputMessage, stop)
			case <-stop:
				return
			}
		}
	}()
}

func (cw *ConnectionWorker) doBroadcast(message InputMessage, stop <-chan struct{}) {
	cw.chsInputMux.RLock()
	defer cw.chsInputMux.RUnlock()

	for _, ch := range cw.chsInput {
		select {
		case ch <- message:
		case <-stop:
			return
		}
	}
}

func (cw *ConnectionWorker) input(stop <-chan struct{}, buffer uint) <-chan InputMessage {
	chProxy := make(chan InputMessage, chanProxyInputMessageBuffer)

	cw.chsInputMux.Lock()
	cw.chsInput = append(cw.chsInput, chProxy)
	cw.chsInputMux.Unlock()

	chout := make(chan InputMessage, buffer)

	go func() {
		defer close(chout)
		defer func() {
			go func() {
				for range chProxy {
				}
			}()

			cw.chsInputMux.Lock()
			for i := range cw.chsInput {
				if cw.chsInput[i] == chProxy {
					cw.chsInput = append(cw.chsInput[:i], cw.chsInput[i+1:]...)
					close(chProxy)
					break
				}
			}
			cw.chsInputMux.Unlock()
		}()

		for {
			select {
			case <-stop:
				return
			case inputMessage, ok := <-chProxy:
				if !ok {
					return
				}
				cw.sendInputMessage(chout, inputMessage, stop, sendInputMessageTimeout)
			}
		}
	}()

	return chout
}

func (cw *ConnectionWorker) sendInputMessage(ch chan InputMessage, inputMessage InputMessage, stop <-chan struct{}, timeout time.Duration) {
	const tickSize = 5

	var timer = time.NewTimer(timeout)
	defer timer.Stop()

	var ticker = time.NewTicker(timeout / tickSize)
	defer ticker.Stop()

	if cap(ch) == 0 {
		select {
		case ch <- inputMessage:
		case <-stop:
		case <-timer.C:
		}
	} else {
		for {
			select {
			case ch <- inputMessage:
				return
			case <-stop:
				return
			case <-timer.C:
				return
			case <-ticker.C:
				if len(ch) == cap(ch) {
					select {
					case <-ch:
					case ch <- inputMessage:
						return
					case <-stop:
						return
					case <-timer.C:
						return
					}
				}
			}
		}
	}
}

func (cw *ConnectionWorker) mergeBytesCh(stop <-chan struct{}, chins ...<-chan []byte) <-chan []byte {
	chout := make(chan []byte, chanMergeBytesBuffer)

	wg := sync.WaitGroup{}
	wg.Add(len(chins))

	for _, chin := range chins {
		go func(chin <-chan []byte) {
			defer wg.Done()

			for {
				select {
				case <-stop:
					return
				case data, ok := <-chin:
					if !ok {
						return
					}

					select {
					case chout <- data:
					case <-stop:
						return
					}
				}
			}
		}(chin)
	}

	go func() {
		wg.Wait()
		close(chout)
	}()

	return chout
}

func (cw *ConnectionWorker) write(chin <-chan []byte, stop <-chan struct{}) {
	go func() {
		for {
			select {
			case data, ok := <-chin:
				if !ok {
					return
				}

				if err := cw.conn.WriteMessage(websocket.TextMessage, data); err != nil {
					if !websocket.IsCloseError(err, websocket.CloseNormalClosure) {
						cw.logger.Errorln("write output message error:", err)
					}
					return
				}
			case <-stop:
				return
			}
		}
	}()
}

func (cw *ConnectionWorker) encode(stop <-chan struct{}, chins ...<-chan OutputMessage) <-chan []byte {
	chout := make(chan []byte, chanEncodeMessageBuffer)

	wg := sync.WaitGroup{}
	wg.Add(len(chins))

	for _, chin := range chins {
		go func(chin <-chan OutputMessage) {
			defer wg.Done()
			for {
				select {
				case <-stop:
					return
				case message, ok := <-chin:
					if !ok {
						return
					}

					if data, err := ffjson.Marshal(message); err != nil {
						cw.logger.Errorln("encode output message error:", err)
					} else {
						select {
						case <-stop:
							return
						case chout <- data:
						}
					}
				}
			}
		}(chin)
	}

	go func() {
		wg.Wait()
		close(chout)
	}()

	return chout
}

func (cw *ConnectionWorker) listenPlayer(stop <-chan struct{}, chin <-chan player.Message) <-chan OutputMessage {
	chout := make(chan OutputMessage, chanOutputMessageBuffer)

	go func() {
		defer close(chout)

		for {
			select {
			case event, ok := <-chin:
				if !ok {
					return
				}

				outputMessage := OutputMessage{
					Type:    OutputMessageTypePlayer,
					Payload: event,
				}

				select {
				case chout <- outputMessage:
				case <-stop:
					return
				}
			case <-stop:
				return
			}
		}
	}()

	return chout
}

func (cw *ConnectionWorker) listenSnakeCommands(stop <-chan struct{}, chin <-chan InputMessage) <-chan string {
	chout := make(chan string, chanSnakeCommandsBuffer)

	go func() {
		defer close(chout)

		for {
			select {
			case message, ok := <-chin:
				if !ok {
					return
				}

				if message.Type == InputMessageTypeSnakeCommand {
					select {
					case chout <- message.Payload:
					case <-stop:
						return
					}
				}
			case <-stop:
				return
			}
		}
	}()

	return chout
}

func (cw *ConnectionWorker) listenPlayerBroadcasts(stop <-chan struct{}, chin <-chan InputMessage, b *broadcast.GroupBroadcast) {
	go func() {
		for {
			select {
			case message, ok := <-chin:
				if !ok {
					return
				}

				if message.Type == InputMessageTypeBroadcast {
					b.BroadcastMessage(broadcast.BroadcastMessage(message.Payload))
				}
			case <-stop:
				return
			}
		}
	}()
}
<|endoftext|>