<commit_before>package bild

import (
	"image"
	"image/color"
)

type normColor struct {
	r, g, b, a float64
}

// Add returns an image with the added color values of two images
func Add(a image.Image, b image.Image) *image.RGBA {
	dst := blendOperation(a, b, func(c0 color.RGBA, c1 color.RGBA) color.RGBA {

		r0 := float64(c0.R)
		g0 := float64(c0.G)
		b0 := float64(c0.B)
		a0 := float64(c0.A)

		r1 := float64(c1.R)
		g1 := float64(c1.G)
		b1 := float64(c1.B)
		a1 := float64(c1.A)

		r2 := uint8(clampFloat64(r0+r1, 0, 255))
		g2 := uint8(clampFloat64(g0+g1, 0, 255))
		b2 := uint8(clampFloat64(b0+b1, 0, 255))
		a2 := uint8(clampFloat64(a0+a1, 0, 255))

		return color.RGBA{r2, g2, b2, a2}
	})

	return dst
}

// Multiply returns an image with the normalized color values of two images multiplied
func Multiply(a image.Image, b image.Image) *image.RGBA {
	dst := blendOperation(a, b, func(c0 color.RGBA, c1 color.RGBA) color.RGBA {

		r0 := float64(c0.R) / 255
		g0 := float64(c0.G) / 255
		b0 := float64(c0.B) / 255
		a0 := float64(c0.A) / 255

		r1 := float64(c1.R) / 255
		g1 := float64(c1.G) / 255
		b1 := float64(c1.B) / 255
		a1 := float64(c1.A) / 255

		r2 := uint8(r0 * r1 * 255)
		g2 := uint8(g0 * g1 * 255)
		b2 := uint8(b0 * b1 * 255)
		a2 := uint8(a0 * a1 * 255)

		return color.RGBA{r2, g2, b2, a2}
	})

	return dst
}

// Overlay returns an image that combines Multiply and Screen blend modes
func Overlay(a image.Image, b image.Image) *image.RGBA {
	dst := blendOperation(a, b, func(c0 color.RGBA, c1 color.RGBA) color.RGBA {

		r0 := float64(c0.R) / 255
		g0 := float64(c0.G) / 255
		b0 := float64(c0.B) / 255
		a0 := float64(c0.A) / 255

		r1 := float64(c1.R) / 255
		g1 := float64(c1.G) / 255
		b1 := float64(c1.B) / 255
		a1 := float64(c1.A) / 255

		var r2, g2, b2, a2 uint8
		if 0.3*r0+0.6*g0+0.1*b0 < 0.5 {
			r2 = uint8(clampFloat64(r0*r1*2*255, 0, 255))
			g2 = uint8(clampFloat64(g0*g1*2*255, 0, 255))
			b2 = uint8(clampFloat64(b0*b1*2*255, 0, 255))
			a2 = uint8(clampFloat64(a0*a1*2*255, 0, 255))
		} else {
			r2 = uint8(clampFloat64((1-(2*(1-r0)*(1-r1)))*255, 0, 255))
			g2 = uint8(clampFloat64((1-(2*(1-g0)*(1-g1)))*255, 0, 255))
			b2 = uint8(clampFloat64((1-(2*(1-b0)*(1-b1)))*255, 0, 255))
			a2 = uint8(clampFloat64((1-(2*(1-a0)*(1-a1)))*255, 0, 255))
		}

		return color.RGBA{r2, g2, b2, a2}
	})

	return dst
}

func blendOperation(a image.Image, b image.Image, fn func(color.RGBA, color.RGBA) color.RGBA) *image.RGBA {
	// Currently only equal size images are supported
	if a.Bounds() != b.Bounds() {
		panic("blend operation: only equal size images are supported")
	}

	bounds := a.Bounds()
	srcA := CloneAsRGBA(a)
	srcB := CloneAsRGBA(b)

	dst := image.NewRGBA(bounds)

	w, h := bounds.Max.X, bounds.Max.Y

	parallelize(h, func(start, end int) {
		for x := 0; x < w; x++ {
			for y := start; y < end; y++ {
				pos := y*dst.Stride + x*4
				result := fn(color.RGBA{srcA.Pix[pos+0], srcA.Pix[pos+1], srcA.Pix[pos+2], srcA.Pix[pos+3]},
					color.RGBA{srcB.Pix[pos+0], srcB.Pix[pos+1], srcB.Pix[pos+2], srcB.Pix[pos+3]})

				dst.Pix[pos+0] = result.R
				dst.Pix[pos+1] = result.G
				dst.Pix[pos+2] = result.B
				dst.Pix[pos+3] = result.A
			}

		}
	})

	return dst
}
<commit_msg>Added soft light and screen<commit_after>package bild

import (
	"image"
	"image/color"
)

type normColor struct {
	r, g, b, a float64
}

// Add returns an image with the added color values of two images
func Add(a image.Image, b image.Image) *image.RGBA {
	dst := blendOperation(a, b, func(c0 color.RGBA, c1 color.RGBA) color.RGBA {

		r0 := float64(c0.R)
		g0 := float64(c0.G)
		b0 := float64(c0.B)
		a0 := float64(c0.A)

		r1 := float64(c1.R)
		g1 := float64(c1.G)
		b1 := float64(c1.B)
		a1 := float64(c1.A)

		r2 := uint8(clampFloat64(r0+r1, 0, 255))
		g2 := uint8(clampFloat64(g0+g1, 0, 255))
		b2 := uint8(clampFloat64(b0+b1, 0, 255))
		a2 := uint8(clampFloat64(a0+a1, 0, 255))

		return color.RGBA{r2, g2, b2, a2}
	})

	return dst
}

// Multiply returns an image with the normalized color values of two images multiplied
func Multiply(a image.Image, b image.Image) *image.RGBA {
	dst := blendOperation(a, b, func(c0 color.RGBA, c1 color.RGBA) color.RGBA {

		r0 := float64(c0.R)
		g0 := float64(c0.G)
		b0 := float64(c0.B)
		a0 := float64(c0.A)

		r1 := float64(c1.R)
		g1 := float64(c1.G)
		b1 := float64(c1.B)
		a1 := float64(c1.A)

		r2 := uint8(r0 * r1 / 255)
		g2 := uint8(g0 * g1 / 255)
		b2 := uint8(b0 * b1 / 255)
		a2 := uint8(a0 * a1 / 255)

		return color.RGBA{r2, g2, b2, a2}
	})

	return dst
}

// Overlay returns an image that combines Multiply and Screen blend modes
func Overlay(a image.Image, b image.Image) *image.RGBA {
	dst := blendOperation(a, b, func(c0 color.RGBA, c1 color.RGBA) color.RGBA {

		r0 := float64(c0.R) / 255
		g0 := float64(c0.G) / 255
		b0 := float64(c0.B) / 255
		a0 := float64(c0.A) / 255

		r1 := float64(c1.R) / 255
		g1 := float64(c1.G) / 255
		b1 := float64(c1.B) / 255
		a1 := float64(c1.A) / 255

		var r2, g2, b2, a2 uint8
		if 0.3*r0+0.6*g0+0.1*b0 < 0.5 {
			r2 = uint8(clampFloat64(r0*r1*2*255, 0, 255))
			g2 = uint8(clampFloat64(g0*g1*2*255, 0, 255))
			b2 = uint8(clampFloat64(b0*b1*2*255, 0, 255))
			a2 = uint8(clampFloat64(a0*a1*2*255, 0, 255))
		} else {
			r2 = uint8(clampFloat64((1-(2*(1-r0)*(1-r1)))*255, 0, 255))
			g2 = uint8(clampFloat64((1-(2*(1-g0)*(1-g1)))*255, 0, 255))
			b2 = uint8(clampFloat64((1-(2*(1-b0)*(1-b1)))*255, 0, 255))
			a2 = uint8(clampFloat64((1-(2*(1-a0)*(1-a1)))*255, 0, 255))
		}

		return color.RGBA{r2, g2, b2, a2}
	})

	return dst
}

// SoftLight returns an image has the Soft Light blend mode applied
func SoftLight(a image.Image, b image.Image) *image.RGBA {
	dst := blendOperation(a, b, func(c0 color.RGBA, c1 color.RGBA) color.RGBA {

		r0 := float64(c0.R) / 255
		g0 := float64(c0.G) / 255
		b0 := float64(c0.B) / 255
		a0 := float64(c0.A) / 255

		r1 := float64(c1.R) / 255
		g1 := float64(c1.G) / 255
		b1 := float64(c1.B) / 255
		a1 := float64(c1.A) / 255

		r2 := uint8(clampFloat64(((1-2*r1)*r0*r0+2*r0*r1)*255, 0, 255))
		g2 := uint8(clampFloat64(((1-2*g1)*g0*g0+2*g0*g1)*255, 0, 255))
		b2 := uint8(clampFloat64(((1-2*b1)*b0*b0+2*b0*b1)*255, 0, 255))
		a2 := uint8(clampFloat64(((1-2*a1)*a0*a0+2*a0*a1)*255, 0, 255))

		return color.RGBA{r2, g2, b2, a2}
	})

	return dst
}

// Screen returns an image that has the screen blend mode applied
func Screen(a image.Image, b image.Image) *image.RGBA {
	dst := blendOperation(a, b, func(c0 color.RGBA, c1 color.RGBA) color.RGBA {

		r0 := float64(c0.R) / 255
		g0 := float64(c0.G) / 255
		b0 := float64(c0.B) / 255
		a0 := float64(c0.A) / 255

		r1 := float64(c1.R) / 255
		g1 := float64(c1.G) / 255
		b1 := float64(c1.B) / 255
		a1 := float64(c1.A) / 255

		r2 := uint8(clampFloat64((1-(1-r0)*(1-r1))*255, 0, 255))
		g2 := uint8(clampFloat64((1-(1-g0)*(1-g1))*255, 0, 255))
		b2 := uint8(clampFloat64((1-(1-b0)*(1-b1))*255, 0, 255))
		a2 := uint8(clampFloat64((1-(1-a0)*(1-a1))*255, 0, 255))

		return color.RGBA{r2, g2, b2, a2}
	})

	return dst
}

func blendOperation(a image.Image, b image.Image, fn func(color.RGBA, color.RGBA) color.RGBA) *image.RGBA {
	// Currently only equal size images are supported
	if a.Bounds() != b.Bounds() {
		panic("blend operation: only equal size images are supported")
	}

	bounds := a.Bounds()
	srcA := CloneAsRGBA(a)
	srcB := CloneAsRGBA(b)

	dst := image.NewRGBA(bounds)

	w, h := bounds.Max.X, bounds.Max.Y

	parallelize(h, func(start, end int) {
		for x := 0; x < w; x++ {
			for y := start; y < end; y++ {
				pos := y*dst.Stride + x*4
				result := fn(color.RGBA{srcA.Pix[pos+0], srcA.Pix[pos+1], srcA.Pix[pos+2], srcA.Pix[pos+3]},
					color.RGBA{srcB.Pix[pos+0], srcB.Pix[pos+1], srcB.Pix[pos+2], srcB.Pix[pos+3]})

				dst.Pix[pos+0] = result.R
				dst.Pix[pos+1] = result.G
				dst.Pix[pos+2] = result.B
				dst.Pix[pos+3] = result.A
			}

		}
	})

	return dst
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"os"
	"path"

	"github.com/lxc/lxd/shared"
)

const SECCOMP_HEADER = `
2
`

const DEFAULT_SECCOMP_POLICY = `
reject_force_umount  # comment this to allow umount -f;  not recommended
[all]
kexec_load errno 38
open_by_handle_at errno 38
init_module errno 38
finit_module errno 38
delete_module errno 38
`
const COMPAT_BLOCKING_POLICY = `
[x86_64]
compat_sys_rt_sigaction errno 38
stub_x32_rt_sigreturn errno 38
compat_sys_ioctl errno 38
compat_sys_readv errno 38
compat_sys_writev errno 38
compat_sys_recvfrom errno 38
compat_sys_sendmsg errno 38
compat_sys_recvmsg errno 38
stub_x32_execve errno 38
compat_sys_ptrace errno 38
compat_sys_rt_sigpending errno 38
compat_sys_rt_sigtimedwait errno 38
compat_sys_rt_sigqueueinfo errno 38
compat_sys_sigaltstack errno 38
compat_sys_timer_create errno 38
compat_sys_mq_notify errno 38
compat_sys_kexec_load errno 38
compat_sys_waitid errno 38
compat_sys_set_robust_list errno 38
compat_sys_get_robust_list errno 38
compat_sys_vmsplice errno 38
compat_sys_move_pages errno 38
compat_sys_preadv64 errno 38
compat_sys_pwritev64 errno 38
compat_sys_rt_tgsigqueueinfo errno 38
compat_sys_recvmmsg errno 38
compat_sys_sendmmsg errno 38
compat_sys_process_vm_readv errno 38
compat_sys_process_vm_writev errno 38
compat_sys_setsockopt errno 38
compat_sys_getsockopt errno 38
compat_sys_io_setup errno 38
compat_sys_io_submit errno 38
stub_x32_execveat errno 38
`

var seccompPath = shared.VarPath("security", "seccomp")

func SeccompProfilePath(c container) string {
	return path.Join(seccompPath, c.Name())
}

func ContainerNeedsSeccomp(c container) bool {
	config := c.ExpandedConfig()

	keys := []string{
		"raw.seccomp",
		"security.syscalls.whitelist",
		"security.syscalls.blacklist",
	}

	for _, k := range keys {
		_, hasKey := config[k]
		if hasKey {
			return true
		}
	}

	compat := config["security.syscalls.blacklist_compat"]
	if shared.IsTrue(compat) {
		return true
	}

	/* this are enabled by default, so if the keys aren't present, that
	 * means "true"
	 */
	default_, ok := config["security.syscalls.blacklist_default"]
	if !ok || shared.IsTrue(default_) {
		return true
	}

	return false
}

func getSeccompProfileContent(c container) string {
	config := c.ExpandedConfig()

	raw := config["raw.seccomp"]
	if raw != "" {
		return raw
	}

	policy := SECCOMP_HEADER

	whitelist := config["security.syscalls.whitelist"]
	if whitelist != "" {
		policy += "whitelist\n[all]\n"
		policy += whitelist
		return policy
	}

	policy += "blacklist\n"

	default_, ok := config["security.syscalls.blacklist_default"]
	if !ok || shared.IsTrue(default_) {
		policy += DEFAULT_SECCOMP_POLICY
	}

	compat := config["security.syscalls.blacklist_compat"]
	if shared.IsTrue(compat) {
		policy += COMPAT_BLOCKING_POLICY
	}

	return policy
}

func SeccompCreateProfile(c container) error {
	/* Unlike apparmor, there is no way to "cache" profiles, and profiles
	 * are automatically unloaded when a task dies. Thus, we don't need to
	 * unload them when a container stops, and we don't have to worry about
	 * the mtime on the file for any compiler purpose, so let's just write
	 * out the profile.
	 */
	if !ContainerNeedsSeccomp(c) {
		return nil
	}

	profile := getSeccompProfileContent(c)
	if err := os.MkdirAll(seccompPath, 0700); err != nil {
		return err
	}

	return ioutil.WriteFile(SeccompProfilePath(c), []byte(profile), 0600)
}

func SeccompDeleteProfile(c container) {
	/* similar to AppArmor, if we've never started this container, the
	 * delete can fail and that's ok.
	 */
	os.Remove(SeccompProfilePath(c))
}
<commit_msg>render seccomp policy correctly on other 64 bit arches<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"

	"github.com/lxc/lxd/shared"
)

const SECCOMP_HEADER = `
2
`

const DEFAULT_SECCOMP_POLICY = `
reject_force_umount  # comment this to allow umount -f;  not recommended
[all]
kexec_load errno 38
open_by_handle_at errno 38
init_module errno 38
finit_module errno 38
delete_module errno 38
`
const COMPAT_BLOCKING_POLICY = `
[%s]
compat_sys_rt_sigaction errno 38
stub_x32_rt_sigreturn errno 38
compat_sys_ioctl errno 38
compat_sys_readv errno 38
compat_sys_writev errno 38
compat_sys_recvfrom errno 38
compat_sys_sendmsg errno 38
compat_sys_recvmsg errno 38
stub_x32_execve errno 38
compat_sys_ptrace errno 38
compat_sys_rt_sigpending errno 38
compat_sys_rt_sigtimedwait errno 38
compat_sys_rt_sigqueueinfo errno 38
compat_sys_sigaltstack errno 38
compat_sys_timer_create errno 38
compat_sys_mq_notify errno 38
compat_sys_kexec_load errno 38
compat_sys_waitid errno 38
compat_sys_set_robust_list errno 38
compat_sys_get_robust_list errno 38
compat_sys_vmsplice errno 38
compat_sys_move_pages errno 38
compat_sys_preadv64 errno 38
compat_sys_pwritev64 errno 38
compat_sys_rt_tgsigqueueinfo errno 38
compat_sys_recvmmsg errno 38
compat_sys_sendmmsg errno 38
compat_sys_process_vm_readv errno 38
compat_sys_process_vm_writev errno 38
compat_sys_setsockopt errno 38
compat_sys_getsockopt errno 38
compat_sys_io_setup errno 38
compat_sys_io_submit errno 38
stub_x32_execveat errno 38
`

var seccompPath = shared.VarPath("security", "seccomp")

func SeccompProfilePath(c container) string {
	return path.Join(seccompPath, c.Name())
}

func ContainerNeedsSeccomp(c container) bool {
	config := c.ExpandedConfig()

	keys := []string{
		"raw.seccomp",
		"security.syscalls.whitelist",
		"security.syscalls.blacklist",
	}

	for _, k := range keys {
		_, hasKey := config[k]
		if hasKey {
			return true
		}
	}

	compat := config["security.syscalls.blacklist_compat"]
	if shared.IsTrue(compat) {
		return true
	}

	/* this are enabled by default, so if the keys aren't present, that
	 * means "true"
	 */
	default_, ok := config["security.syscalls.blacklist_default"]
	if !ok || shared.IsTrue(default_) {
		return true
	}

	return false
}

func getSeccompProfileContent(c container) (string, error) {
	config := c.ExpandedConfig()

	raw := config["raw.seccomp"]
	if raw != "" {
		return raw, nil
	}

	policy := SECCOMP_HEADER

	whitelist := config["security.syscalls.whitelist"]
	if whitelist != "" {
		policy += "whitelist\n[all]\n"
		policy += whitelist
		return policy, nil
	}

	policy += "blacklist\n"

	default_, ok := config["security.syscalls.blacklist_default"]
	if !ok || shared.IsTrue(default_) {
		policy += DEFAULT_SECCOMP_POLICY
	}

	compat := config["security.syscalls.blacklist_compat"]
	if shared.IsTrue(compat) {
		arch, err := shared.ArchitectureName(c.Architecture())
		if err != nil {
			return "", err
		}
		policy += fmt.Sprintf(COMPAT_BLOCKING_POLICY, arch)
	}

	return policy, nil
}

func SeccompCreateProfile(c container) error {
	/* Unlike apparmor, there is no way to "cache" profiles, and profiles
	 * are automatically unloaded when a task dies. Thus, we don't need to
	 * unload them when a container stops, and we don't have to worry about
	 * the mtime on the file for any compiler purpose, so let's just write
	 * out the profile.
	 */
	if !ContainerNeedsSeccomp(c) {
		return nil
	}

	profile, err := getSeccompProfileContent(c)
	if err != nil {
		return nil
	}

	if err := os.MkdirAll(seccompPath, 0700); err != nil {
		return err
	}

	return ioutil.WriteFile(SeccompProfilePath(c), []byte(profile), 0600)
}

func SeccompDeleteProfile(c container) {
	/* similar to AppArmor, if we've never started this container, the
	 * delete can fail and that's ok.
	 */
	os.Remove(SeccompProfilePath(c))
}
<|endoftext|>


<commit_before>package xlate

import (
	"reflect"
	"testing"

	"github.com/polydawn/go-xlate/dest/tok"
)

func TestMapperSetup(t *testing.T) {
	type BB struct {
		Z string
	}
	type AA struct {
		X string
		Y BB
	}

	type testRow struct {
		title  string
		setup  MapperSetup
		expect interface{} // err type or nil
	}
	for _, tr := range []testRow{{
		title:  "empty mapper setups are acceptable",
		setup:  MapperSetup{},
		expect: nil,
	}, {
		title: "short simple mapper setups are acceptable",
		setup: MapperSetup{
			{AA{}, Map_Wildcard_toString},
			{BB{}, Map_Wildcard_toString},
		},
		expect: nil,
	}, {
		title: "mapper setups with nil funcs are rejected",
		setup: MapperSetup{
			{AA{}, nil},
		},
		expect: &ErrNilMappingFunc{MapperSetupRow{AA{}, nil}},
	}, {
		title: "mapper setups with repeat types are rejected",
		setup: MapperSetup{
			{AA{}, Map_Wildcard_toStringOfType},
			{AA{}, Map_Wildcard_toString},
		},
		expect: &ErrMapperSetupNotUnique{MapperSetupRow{AA{}, Map_Wildcard_toString}},
	}} {
		err := capturePanics(func() {
			NewMapper(tr.setup)
		})
		if !stringyEquality(tr.expect, err) {
			t.Errorf("test %q FAILED:\n\texpected     %#v\n\terrored with %#v",
				tr.title, tr.expect, err)
		}
	}
}

func TestMapperBasics(t *testing.T) {
	type BB struct {
		Z string
	}
	type AA struct {
		X string
		Y BB
	}

	type testRow struct {
		title     string
		thing     interface{}
		mapper    *Mapper
		expect    tok.Tokens
		expectErr error
	}
	for _, tr := range []testRow{{
		title: "test a single thing at its zero value",
		thing: BB{},
		mapper: NewMapper(MapperSetup{
			{BB{}, Map_Wildcard_toString},
		}),
		expect: tok.Tokens{
			{tok.TokenKind_ValString, "{}"},
		},
	}, {
		title: "test a single thing that has no handler",
		thing: AA{},
		mapper: NewMapper(MapperSetup{
			{BB{}, Map_Wildcard_toString},
		}),
		expectErr: &ErrMissingMappingFunc{reflect.TypeOf(AA{})},
	}, {
		title: "test a custom func plus recursion at its zero value",
		thing: AA{},
		mapper: NewMapper(MapperSetup{
			// a fairly bizarre hack of recursion, trying to avoid triggering anything too fancy yet.
			{AA{}, func(dispatch *Mapper, dest Destination, input interface{}) {
				dest.OpenMap()
				dest.WriteMapKey("type")
				dest.WriteString("AA")
				dest.WriteMapKey("data")
				dispatch.Map(dest, input.(AA).Y)
				dest.CloseMap()
			}},
			{BB{}, Map_Wildcard_toString},
		}),
		expect: tok.Tokens{
			{tok.TokenKind_OpenMap, nil},
			{tok.TokenKind_MapKey, "type"},
			{tok.TokenKind_ValString, "AA"},
			{tok.TokenKind_MapKey, "data"},
			{tok.TokenKind_ValString, "{}"},
			{tok.TokenKind_CloseMap, nil},
		},
	}} {
		tokDest := tok.NewDestination()
		err := capturePanics(func() {
			tr.mapper.Map(
				tokDest,
				tr.thing,
			)
		})
		if !stringyEquality(tr.expectErr, err) {
			t.Errorf("test %q FAILED:\n\texpected     %#v\n\terrored with %#v",
				tr.title, tr.expectErr, err)
		}
		if !stringyEquality(tr.expect, tokDest.Tokens) {
			t.Errorf("test %q FAILED:\n\texpected: %#v\n\tactual:   %#v",
				tr.title, tr.expect, tokDest.Tokens)
		}
	}
}
<commit_msg>Inquiry into dispatchery based on interfaces.<commit_after>package xlate

import (
	"fmt"
	"reflect"
	"testing"

	"github.com/polydawn/go-xlate/dest/tok"
)

func TestMapperSetup(t *testing.T) {
	type BB struct {
		Z string
	}
	type AA struct {
		X string
		Y BB
	}

	type testRow struct {
		title  string
		setup  MapperSetup
		expect interface{} // err type or nil
	}
	for _, tr := range []testRow{{
		title:  "empty mapper setups are acceptable",
		setup:  MapperSetup{},
		expect: nil,
	}, {
		title: "short simple mapper setups are acceptable",
		setup: MapperSetup{
			{AA{}, Map_Wildcard_toString},
			{BB{}, Map_Wildcard_toString},
		},
		expect: nil,
	}, {
		title: "mapper setups with nil funcs are rejected",
		setup: MapperSetup{
			{AA{}, nil},
		},
		expect: &ErrNilMappingFunc{MapperSetupRow{AA{}, nil}},
	}, {
		title: "mapper setups with repeat types are rejected",
		setup: MapperSetup{
			{AA{}, Map_Wildcard_toStringOfType},
			{AA{}, Map_Wildcard_toString},
		},
		expect: &ErrMapperSetupNotUnique{MapperSetupRow{AA{}, Map_Wildcard_toString}},
	}} {
		err := capturePanics(func() {
			NewMapper(tr.setup)
		})
		if !stringyEquality(tr.expect, err) {
			t.Errorf("test %q FAILED:\n\texpected     %#v\n\terrored with %#v",
				tr.title, tr.expect, err)
		}
	}
}

func TestMapperBasics(t *testing.T) {
	type BB struct {
		Z string
	}
	type AA struct {
		X string
		Y BB
	}

	type testRow struct {
		title     string
		thing     interface{}
		mapper    *Mapper
		expect    tok.Tokens
		expectErr error
	}
	for _, tr := range []testRow{{
		title: "test a single thing at its zero value",
		thing: BB{},
		mapper: NewMapper(MapperSetup{
			{BB{}, Map_Wildcard_toString},
		}),
		expect: tok.Tokens{
			{tok.TokenKind_ValString, "{}"},
		},
	}, {
		title: "test a single thing that has no handler",
		thing: AA{},
		mapper: NewMapper(MapperSetup{
			{BB{}, Map_Wildcard_toString},
		}),
		expectErr: &ErrMissingMappingFunc{reflect.TypeOf(AA{})},
	}, {
		title: "test a custom func plus recursion at its zero value",
		thing: AA{},
		mapper: NewMapper(MapperSetup{
			// a fairly bizarre hack of recursion, trying to avoid triggering anything too fancy yet.
			{AA{}, func(dispatch *Mapper, dest Destination, input interface{}) {
				dest.OpenMap()
				dest.WriteMapKey("type")
				dest.WriteString("AA")
				dest.WriteMapKey("data")
				dispatch.Map(dest, input.(AA).Y)
				dest.CloseMap()
			}},
			{BB{}, Map_Wildcard_toString},
		}),
		expect: tok.Tokens{
			{tok.TokenKind_OpenMap, nil},
			{tok.TokenKind_MapKey, "type"},
			{tok.TokenKind_ValString, "AA"},
			{tok.TokenKind_MapKey, "data"},
			{tok.TokenKind_ValString, "{}"},
			{tok.TokenKind_CloseMap, nil},
		},
	}} {
		tokDest := tok.NewDestination()
		err := capturePanics(func() {
			tr.mapper.Map(
				tokDest,
				tr.thing,
			)
		})
		if !stringyEquality(tr.expectErr, err) {
			t.Errorf("test %q FAILED:\n\texpected     %#v\n\terrored with %#v",
				tr.title, tr.expectErr, err)
		}
		if !stringyEquality(tr.expect, tokDest.Tokens) {
			t.Errorf("test %q FAILED:\n\texpected: %#v\n\tactual:   %#v",
				tr.title, tr.expect, tokDest.Tokens)
		}
	}
}

func TestMapperInterfaceDispatch(t *testing.T) {
	type Iface interface{}
	type Top struct {
		X Iface
	}
	type AA struct {
		Y string
	}
	type BB struct {
		Z string
	}

	fmt.Printf(";; %v -- %v\n", reflect.TypeOf(Iface(nil)), nil)
	fmt.Printf(";; %v -- %v\n", reflect.TypeOf(Iface(BB{})), reflect.TypeOf(Iface(BB{})).Kind())
	var fuk Iface
	fmt.Printf(";; %v -- %v\n", reflect.TypeOf(fuk), nil)
	fmt.Printf(";; %v -- %v\n", reflect.TypeOf(&fuk), reflect.TypeOf(&fuk).Kind())
	fmt.Printf(";; %v -- %v\n", reflect.TypeOf(&fuk).Elem(), reflect.TypeOf(&fuk).Elem().Kind())

	type testRow struct {
		title     string
		thing     interface{}
		mapper    *Mapper
		expect    tok.Tokens
		expectErr error
	}
	for _, tr := range []testRow{{
		title: "test a single thing at its zero value",
		thing: Top{AA{"whee"}},
		mapper: NewMapper(MapperSetup{
			{Top{}, func(dispatch *Mapper, dest Destination, input interface{}) {
				dispatch.Map(dest, Iface(input.(Top).X))
			}},
			//{Iface(nil), Map_Wildcard_toString},
			{AA{}, Map_Wildcard_toString},
		}),
		expect: tok.Tokens{
			{tok.TokenKind_ValString, "{}"},
		},
	}} {
		tokDest := tok.NewDestination()
		err := capturePanics(func() {
			tr.mapper.Map(
				tokDest,
				tr.thing,
			)
		})
		if !stringyEquality(tr.expectErr, err) {
			t.Errorf("test %q FAILED:\n\texpected     %#v\n\terrored with %v",
				tr.title, tr.expectErr, err)
		}
		if !stringyEquality(tr.expect, tokDest.Tokens) {
			t.Errorf("test %q FAILED:\n\texpected: %#v\n\tactual:   %#v",
				tr.title, tr.expect, tokDest.Tokens)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	ggproto "github.com/gogo/protobuf/proto"
	p2p_crypto "github.com/libp2p/go-libp2p-crypto"
	p2p_host "github.com/libp2p/go-libp2p-host"
	p2p_peer "github.com/libp2p/go-libp2p-peer"
	p2p_pstore "github.com/libp2p/go-libp2p-peerstore"
	mc "github.com/mediachain/concat/mc"
	mcq "github.com/mediachain/concat/mc/query"
	pb "github.com/mediachain/concat/proto"
	multiaddr "github.com/multiformats/go-multiaddr"
	multihash "github.com/multiformats/go-multihash"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"
	"sync"
	"time"
)

type Node struct {
	mc.PeerIdentity
	publisher mc.PublisherIdentity
	info      string
	status    int
	laddr     multiaddr.Multiaddr
	host      p2p_host.Host
	netCtx    context.Context
	netCancel context.CancelFunc
	dir       *p2p_pstore.PeerInfo
	natCfg    mc.NATConfig
	home      string
	db        StatementDB
	ds        Datastore
	auth      PeerAuth
	mx        sync.Mutex
	counter   int
}

type StatementDB interface {
	Open(home string) error
	Put(*pb.Statement) error
	PutBatch([]*pb.Statement) error
	Get(id string) (*pb.Statement, error)
	Query(*mcq.Query) ([]interface{}, error)
	QueryStream(context.Context, *mcq.Query) (<-chan interface{}, error)
	QueryOne(*mcq.Query) (interface{}, error)
	Merge(*pb.Statement) (bool, error)
	MergeBatch([]*pb.Statement) (int, error)
	Delete(*mcq.Query) (int, error)
	Close() error
}

type Key multihash.Multihash
type Datastore interface {
	Open(home string) error
	Put(data []byte) (Key, error)
	PutBatch(batch [][]byte) ([]Key, error)
	Has(Key) (bool, error)
	Get(Key) ([]byte, error)
	Delete(Key) error
	Close()
}

type PeerAuth struct {
	peers map[p2p_peer.ID][]string
	mx    sync.Mutex
}

type NodeInfo struct {
	Peer      string `json:"peer"`
	Publisher string `json:"publisher"`
	Info      string `json:"info"`
}

var (
	UnknownStatement = errors.New("Unknown statement")
	UnknownObject    = errors.New("Unknown Object")
	BadStatementBody = errors.New("Unrecognized statement body")
	BadQuery         = errors.New("Unexpected query")
	BadState         = errors.New("Unrecognized state")
	BadMethod        = errors.New("Unsupported method")
	BadNamespace     = errors.New("Illegal namespace")
	BadResult        = errors.New("Bad result set")
	BadStatement     = errors.New("Bad statement; verification failed")
	NoResult         = errors.New("Empty result set")
	MissingData      = errors.New("Missing statement metadata")
	UnexpectedData   = errors.New("Unexpected data object")
	BadData          = errors.New("Bad data object; hash mismatch")
	BadPush          = errors.New("Bad push value; unexpected object")
	BadResponse      = errors.New("Bad response; unexpected object")
	BadRuleset       = errors.New("Bad auth ruleset; unexpected object")
)

const (
	StatusOffline = iota
	StatusOnline
	StatusPublic
)

var statusString = []string{"offline", "online", "public"}

type PushError string

func (s PushError) Error() string {
	return string(s)
}

type StreamError struct {
	Err string `json:"error"`
}

func (s StreamError) Error() string {
	return s.Err
}

func sendStreamError(ctx context.Context, ch chan interface{}, what string) {
	select {
	case ch <- StreamError{what}:
	case <-ctx.Done():
	}
}

func (node *Node) stmtCounter() int {
	node.mx.Lock()
	counter := node.counter
	node.counter++
	node.mx.Unlock()
	return counter
}

func (node *Node) doPublish(ns string, body interface{}) (string, error) {
	stmt, err := node.makeStatement(ns, body)
	if err != nil {
		return "", err
	}

	err = node.db.Put(stmt)
	return stmt.Id, err
}

func (node *Node) doPublishBatch(ns string, lst []interface{}) ([]string, error) {
	stmts := make([]*pb.Statement, len(lst))
	sids := make([]string, len(lst))
	for x, body := range lst {
		stmt, err := node.makeStatement(ns, body)
		if err != nil {
			return nil, err
		}
		stmts[x] = stmt
		sids[x] = stmt.Id
	}

	err := node.db.PutBatch(stmts)
	if err != nil {
		return nil, err
	}

	return sids, err
}

func (node *Node) makeStatement(ns string, body interface{}) (*pb.Statement, error) {
	stmt := new(pb.Statement)
	pid := node.publisher.ID58
	ts := time.Now().Unix()
	counter := node.stmtCounter()
	stmt.Id = fmt.Sprintf("%s:%d:%d", pid, ts, counter)
	stmt.Publisher = pid
	stmt.Namespace = ns
	stmt.Timestamp = ts
	switch body := body.(type) {
	case *pb.SimpleStatement:
		stmt.Body = &pb.StatementBody{&pb.StatementBody_Simple{body}}

	case *pb.CompoundStatement:
		stmt.Body = &pb.StatementBody{&pb.StatementBody_Compound{body}}

	case *pb.EnvelopeStatement:
		stmt.Body = &pb.StatementBody{&pb.StatementBody_Envelope{body}}

	case *pb.ArchiveStatement:
		stmt.Body = &pb.StatementBody{&pb.StatementBody_Archive{body}}

	default:
		return nil, BadStatementBody
	}

	err := node.signStatement(stmt)
	if err != nil {
		return nil, err
	}

	return stmt, nil
}

func (node *Node) signStatement(stmt *pb.Statement) error {
	bytes, err := ggproto.Marshal(stmt)
	if err != nil {
		return err
	}

	sig, err := node.publisher.PrivKey.Sign(bytes)
	if err != nil {
		return err
	}

	stmt.Signature = sig
	return nil
}

func (node *Node) checkStatement(stmt *pb.Statement) bool {
	return stmt.Id != "" &&
		stmt.Publisher != "" &&
		stmt.Namespace != "" &&
		stmt.Timestamp > 0 &&
		stmt.Body != nil &&
		stmt.Signature != nil
}

func (node *Node) verifyStatement(stmt *pb.Statement) (bool, error) {
	pubk, err := mc.PublisherKey(stmt.Publisher)
	if err != nil {
		return false, err
	}

	return node.verifyStatementSig(stmt, pubk)
}

func (node *Node) verifyStatementCacheKeys(stmt *pb.Statement, pkcache map[string]p2p_crypto.PubKey) (bool, error) {
	var pubk p2p_crypto.PubKey
	var err error

	pubk, ok := pkcache[stmt.Publisher]
	if !ok {
		pubk, err = mc.PublisherKey(stmt.Publisher)
		if err != nil {
			return false, err
		}
		pkcache[stmt.Publisher] = pubk
	}

	return node.verifyStatementSig(stmt, pubk)
}

func (node *Node) verifyStatementSig(stmt *pb.Statement, pubk p2p_crypto.PubKey) (bool, error) {
	sig := stmt.Signature
	stmt.Signature = nil
	bytes, err := ggproto.Marshal(stmt)
	stmt.Signature = sig

	if err != nil {
		return false, err
	}

	return pubk.Verify(bytes, sig)
}

func (node *Node) openDB() error {
	node.db = &SQLiteDB{}
	return node.db.Open(node.home)
}

func (node *Node) openDS() error {
	node.ds = &RocksDS{}
	return node.ds.Open(node.home)
}

// persistent configuration
type NodeConfig struct {
	Info string                 `json:"info,omitempty"`
	NAT  string                 `json:"nat,omitempty"`
	Dir  string                 `json:"dir,omitempty"`
	Auth map[string]interface{} `json:"auth,omitempty"`
}

func (node *Node) saveConfig() error {
	var cfg NodeConfig
	cfg.Info = node.info
	cfg.NAT = node.natCfg.String()
	if node.dir != nil {
		cfg.Dir = mc.FormatHandle(*node.dir)
	}
	cfg.Auth = node.auth.toJSON()

	bytes, err := json.Marshal(cfg)
	if err != nil {
		return err
	}

	cfgpath := path.Join(node.home, "config.json")
	return ioutil.WriteFile(cfgpath, bytes, 0644)
}

func (node *Node) loadConfig() error {
	cfgpath := path.Join(node.home, "config.json")

	bytes, err := ioutil.ReadFile(cfgpath)
	switch {
	case os.IsNotExist(err):
		return nil
	case err != nil:
		return err
	}

	var cfg NodeConfig
	err = json.Unmarshal(bytes, &cfg)
	if err != nil {
		return err
	}

	node.info = cfg.Info

	natCfg, err := mc.NATConfigFromString(cfg.NAT)
	if err != nil {
		return err
	}
	node.natCfg = natCfg

	if cfg.Dir != "" {
		pinfo, err := mc.ParseHandle(cfg.Dir)
		if err != nil {
			return err
		}
		node.dir = &pinfo
	}

	err = node.auth.fromJSON(cfg.Auth)
	return err
}

func (node *Node) doShutdown() {
	log.Printf("Node shutdown")
	err := node.db.Close()
	if err != nil {
		log.Printf("Error closing StatementDB: %s", err.Error())
	}
	node.ds.Close()
	os.Exit(0)
}

func (auth *PeerAuth) fromJSON(rmap map[string]interface{}) error {
	auth.mx.Lock()
	defer auth.mx.Unlock()

	auth.peers = make(map[p2p_peer.ID][]string)
	for id58, xrules := range rmap {
		pid, err := p2p_peer.IDB58Decode(id58)
		if err != nil {
			return err
		}

		rules, ok := xrules.([]string)
		if !ok {
			return BadRuleset
		}

		auth.peers[pid] = rules
	}

	return nil
}

func (auth *PeerAuth) toJSON() map[string]interface{} {
	auth.mx.Lock()
	defer auth.mx.Unlock()

	rmap := make(map[string]interface{})
	for pid, rules := range auth.peers {
		rmap[pid.Pretty()] = rules
	}

	return rmap
}

func (auth *PeerAuth) authorize(pid p2p_peer.ID, nss []string) bool {
	if len(nss) == 0 {
		return false
	}

	auth.mx.Lock()
	defer auth.mx.Unlock()

	rules := auth.peers[pid]
	if len(rules) == 0 {
		return false
	}

	for _, ns := range nss {
		if !auth.authorizeAllow(rules, ns) {
			return false
		}
	}

	return true
}

func (auth *PeerAuth) authorizeAllow(rules []string, ns string) bool {
	for _, rule := range rules {
		switch {
		case rule == "*":
			return true

		case strings.HasSuffix(rule, ".*"):
			if strings.HasPrefix(ns, rule[:len(rule)-2]) {
				return true
			}

		case rule == ns:
			return true
		}
	}

	return false
}

func (auth *PeerAuth) getRules(pid p2p_peer.ID) []string {
	auth.mx.Lock()
	defer auth.mx.Unlock()
	return auth.peers[pid]
}

func (auth *PeerAuth) setRules(pid p2p_peer.ID, rules []string) {
	auth.mx.Lock()
	auth.peers[pid] = rules
	auth.mx.Unlock()
}

func (auth *PeerAuth) clearRules(pid p2p_peer.ID) {
	auth.mx.Lock()
	delete(auth.peers, pid)
	auth.mx.Unlock()
}
<commit_msg>mcnode: fix PeerAuth json unmarshalling: interface{} is the new void*<commit_after>package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	ggproto "github.com/gogo/protobuf/proto"
	p2p_crypto "github.com/libp2p/go-libp2p-crypto"
	p2p_host "github.com/libp2p/go-libp2p-host"
	p2p_peer "github.com/libp2p/go-libp2p-peer"
	p2p_pstore "github.com/libp2p/go-libp2p-peerstore"
	mc "github.com/mediachain/concat/mc"
	mcq "github.com/mediachain/concat/mc/query"
	pb "github.com/mediachain/concat/proto"
	multiaddr "github.com/multiformats/go-multiaddr"
	multihash "github.com/multiformats/go-multihash"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"
	"sync"
	"time"
)

type Node struct {
	mc.PeerIdentity
	publisher mc.PublisherIdentity
	info      string
	status    int
	laddr     multiaddr.Multiaddr
	host      p2p_host.Host
	netCtx    context.Context
	netCancel context.CancelFunc
	dir       *p2p_pstore.PeerInfo
	natCfg    mc.NATConfig
	home      string
	db        StatementDB
	ds        Datastore
	auth      PeerAuth
	mx        sync.Mutex
	counter   int
}

type StatementDB interface {
	Open(home string) error
	Put(*pb.Statement) error
	PutBatch([]*pb.Statement) error
	Get(id string) (*pb.Statement, error)
	Query(*mcq.Query) ([]interface{}, error)
	QueryStream(context.Context, *mcq.Query) (<-chan interface{}, error)
	QueryOne(*mcq.Query) (interface{}, error)
	Merge(*pb.Statement) (bool, error)
	MergeBatch([]*pb.Statement) (int, error)
	Delete(*mcq.Query) (int, error)
	Close() error
}

type Key multihash.Multihash
type Datastore interface {
	Open(home string) error
	Put(data []byte) (Key, error)
	PutBatch(batch [][]byte) ([]Key, error)
	Has(Key) (bool, error)
	Get(Key) ([]byte, error)
	Delete(Key) error
	Close()
}

type PeerAuth struct {
	peers map[p2p_peer.ID][]string
	mx    sync.Mutex
}

type NodeInfo struct {
	Peer      string `json:"peer"`
	Publisher string `json:"publisher"`
	Info      string `json:"info"`
}

var (
	UnknownStatement = errors.New("Unknown statement")
	UnknownObject    = errors.New("Unknown Object")
	BadStatementBody = errors.New("Unrecognized statement body")
	BadQuery         = errors.New("Unexpected query")
	BadState         = errors.New("Unrecognized state")
	BadMethod        = errors.New("Unsupported method")
	BadNamespace     = errors.New("Illegal namespace")
	BadResult        = errors.New("Bad result set")
	BadStatement     = errors.New("Bad statement; verification failed")
	NoResult         = errors.New("Empty result set")
	MissingData      = errors.New("Missing statement metadata")
	UnexpectedData   = errors.New("Unexpected data object")
	BadData          = errors.New("Bad data object; hash mismatch")
	BadPush          = errors.New("Bad push value; unexpected object")
	BadResponse      = errors.New("Bad response; unexpected object")
	BadRuleset       = errors.New("Bad auth ruleset; unexpected object")
)

const (
	StatusOffline = iota
	StatusOnline
	StatusPublic
)

var statusString = []string{"offline", "online", "public"}

type PushError string

func (s PushError) Error() string {
	return string(s)
}

type StreamError struct {
	Err string `json:"error"`
}

func (s StreamError) Error() string {
	return s.Err
}

func sendStreamError(ctx context.Context, ch chan interface{}, what string) {
	select {
	case ch <- StreamError{what}:
	case <-ctx.Done():
	}
}

func (node *Node) stmtCounter() int {
	node.mx.Lock()
	counter := node.counter
	node.counter++
	node.mx.Unlock()
	return counter
}

func (node *Node) doPublish(ns string, body interface{}) (string, error) {
	stmt, err := node.makeStatement(ns, body)
	if err != nil {
		return "", err
	}

	err = node.db.Put(stmt)
	return stmt.Id, err
}

func (node *Node) doPublishBatch(ns string, lst []interface{}) ([]string, error) {
	stmts := make([]*pb.Statement, len(lst))
	sids := make([]string, len(lst))
	for x, body := range lst {
		stmt, err := node.makeStatement(ns, body)
		if err != nil {
			return nil, err
		}
		stmts[x] = stmt
		sids[x] = stmt.Id
	}

	err := node.db.PutBatch(stmts)
	if err != nil {
		return nil, err
	}

	return sids, err
}

func (node *Node) makeStatement(ns string, body interface{}) (*pb.Statement, error) {
	stmt := new(pb.Statement)
	pid := node.publisher.ID58
	ts := time.Now().Unix()
	counter := node.stmtCounter()
	stmt.Id = fmt.Sprintf("%s:%d:%d", pid, ts, counter)
	stmt.Publisher = pid
	stmt.Namespace = ns
	stmt.Timestamp = ts
	switch body := body.(type) {
	case *pb.SimpleStatement:
		stmt.Body = &pb.StatementBody{&pb.StatementBody_Simple{body}}

	case *pb.CompoundStatement:
		stmt.Body = &pb.StatementBody{&pb.StatementBody_Compound{body}}

	case *pb.EnvelopeStatement:
		stmt.Body = &pb.StatementBody{&pb.StatementBody_Envelope{body}}

	case *pb.ArchiveStatement:
		stmt.Body = &pb.StatementBody{&pb.StatementBody_Archive{body}}

	default:
		return nil, BadStatementBody
	}

	err := node.signStatement(stmt)
	if err != nil {
		return nil, err
	}

	return stmt, nil
}

func (node *Node) signStatement(stmt *pb.Statement) error {
	bytes, err := ggproto.Marshal(stmt)
	if err != nil {
		return err
	}

	sig, err := node.publisher.PrivKey.Sign(bytes)
	if err != nil {
		return err
	}

	stmt.Signature = sig
	return nil
}

func (node *Node) checkStatement(stmt *pb.Statement) bool {
	return stmt.Id != "" &&
		stmt.Publisher != "" &&
		stmt.Namespace != "" &&
		stmt.Timestamp > 0 &&
		stmt.Body != nil &&
		stmt.Signature != nil
}

func (node *Node) verifyStatement(stmt *pb.Statement) (bool, error) {
	pubk, err := mc.PublisherKey(stmt.Publisher)
	if err != nil {
		return false, err
	}

	return node.verifyStatementSig(stmt, pubk)
}

func (node *Node) verifyStatementCacheKeys(stmt *pb.Statement, pkcache map[string]p2p_crypto.PubKey) (bool, error) {
	var pubk p2p_crypto.PubKey
	var err error

	pubk, ok := pkcache[stmt.Publisher]
	if !ok {
		pubk, err = mc.PublisherKey(stmt.Publisher)
		if err != nil {
			return false, err
		}
		pkcache[stmt.Publisher] = pubk
	}

	return node.verifyStatementSig(stmt, pubk)
}

func (node *Node) verifyStatementSig(stmt *pb.Statement, pubk p2p_crypto.PubKey) (bool, error) {
	sig := stmt.Signature
	stmt.Signature = nil
	bytes, err := ggproto.Marshal(stmt)
	stmt.Signature = sig

	if err != nil {
		return false, err
	}

	return pubk.Verify(bytes, sig)
}

func (node *Node) openDB() error {
	node.db = &SQLiteDB{}
	return node.db.Open(node.home)
}

func (node *Node) openDS() error {
	node.ds = &RocksDS{}
	return node.ds.Open(node.home)
}

// persistent configuration
type NodeConfig struct {
	Info string                 `json:"info,omitempty"`
	NAT  string                 `json:"nat,omitempty"`
	Dir  string                 `json:"dir,omitempty"`
	Auth map[string]interface{} `json:"auth,omitempty"`
}

func (node *Node) saveConfig() error {
	var cfg NodeConfig
	cfg.Info = node.info
	cfg.NAT = node.natCfg.String()
	if node.dir != nil {
		cfg.Dir = mc.FormatHandle(*node.dir)
	}
	cfg.Auth = node.auth.toJSON()

	bytes, err := json.Marshal(cfg)
	if err != nil {
		return err
	}

	cfgpath := path.Join(node.home, "config.json")
	return ioutil.WriteFile(cfgpath, bytes, 0644)
}

func (node *Node) loadConfig() error {
	cfgpath := path.Join(node.home, "config.json")

	bytes, err := ioutil.ReadFile(cfgpath)
	switch {
	case os.IsNotExist(err):
		return nil
	case err != nil:
		return err
	}

	var cfg NodeConfig
	err = json.Unmarshal(bytes, &cfg)
	if err != nil {
		return err
	}

	node.info = cfg.Info

	natCfg, err := mc.NATConfigFromString(cfg.NAT)
	if err != nil {
		return err
	}
	node.natCfg = natCfg

	if cfg.Dir != "" {
		pinfo, err := mc.ParseHandle(cfg.Dir)
		if err != nil {
			return err
		}
		node.dir = &pinfo
	}

	err = node.auth.fromJSON(cfg.Auth)
	return err
}

func (node *Node) doShutdown() {
	log.Printf("Node shutdown")
	err := node.db.Close()
	if err != nil {
		log.Printf("Error closing StatementDB: %s", err.Error())
	}
	node.ds.Close()
	os.Exit(0)
}

func (auth *PeerAuth) fromJSON(rmap map[string]interface{}) error {
	auth.mx.Lock()
	defer auth.mx.Unlock()

	auth.peers = make(map[p2p_peer.ID][]string)
	for id58, xrules := range rmap {
		pid, err := p2p_peer.IDB58Decode(id58)
		if err != nil {
			return err
		}

		xxrules, ok := xrules.([]interface{})
		if !ok {
			return BadRuleset
		}

		rules := make([]string, len(xxrules))
		for x, xxrule := range xxrules {
			rule, ok := xxrule.(string)
			if !ok {
				return BadRuleset
			}

			rules[x] = rule
		}

		auth.peers[pid] = rules
	}

	return nil
}

func (auth *PeerAuth) toJSON() map[string]interface{} {
	auth.mx.Lock()
	defer auth.mx.Unlock()

	rmap := make(map[string]interface{})
	for pid, rules := range auth.peers {
		rmap[pid.Pretty()] = rules
	}

	return rmap
}

func (auth *PeerAuth) authorize(pid p2p_peer.ID, nss []string) bool {
	if len(nss) == 0 {
		return false
	}

	auth.mx.Lock()
	defer auth.mx.Unlock()

	rules := auth.peers[pid]
	if len(rules) == 0 {
		return false
	}

	for _, ns := range nss {
		if !auth.authorizeAllow(rules, ns) {
			return false
		}
	}

	return true
}

func (auth *PeerAuth) authorizeAllow(rules []string, ns string) bool {
	for _, rule := range rules {
		switch {
		case rule == "*":
			return true

		case strings.HasSuffix(rule, ".*"):
			if strings.HasPrefix(ns, rule[:len(rule)-2]) {
				return true
			}

		case rule == ns:
			return true
		}
	}

	return false
}

func (auth *PeerAuth) getRules(pid p2p_peer.ID) []string {
	auth.mx.Lock()
	defer auth.mx.Unlock()
	return auth.peers[pid]
}

func (auth *PeerAuth) setRules(pid p2p_peer.ID, rules []string) {
	auth.mx.Lock()
	auth.peers[pid] = rules
	auth.mx.Unlock()
}

func (auth *PeerAuth) clearRules(pid p2p_peer.ID) {
	auth.mx.Lock()
	delete(auth.peers, pid)
	auth.mx.Unlock()
}
<|endoftext|>


<commit_before>package minion

import (
	"bytes"
	"encoding/json"
	"log"
	"math/rand"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/dnaeon/backoff"
	"github.com/dnaeon/gru/classifier"
	"github.com/dnaeon/gru/task"
	"github.com/dnaeon/gru/utils"

	etcdclient "github.com/coreos/etcd/client"
	"github.com/pborman/uuid"
	"golang.org/x/net/context"
)

// EtcdMinionSpace is the keyspace in etcd used by minions
const EtcdMinionSpace = "/gru/minion"

// Etcd Minion
type etcdMinion struct {
	// Name of the minion
	name string

	// Minion root directory in etcd
	rootDir string

	// Minion queue directory in etcd
	queueDir string

	// Log directory of previously executed tasks
	logDir string

	// Classifier directory in etcd
	classifierDir string

	// Minion unique identifier
	id uuid.UUID

	// KeysAPI client to etcd
	kapi etcdclient.KeysAPI

	// Channel over which tasks are sent for processing
	taskQueue chan *task.Task

	// Channel used to signal shutdown time
	done chan struct{}
}

// NewEtcdMinion creates a new minion with etcd backend
func NewEtcdMinion(name string, cfg etcdclient.Config) Minion {
	c, err := etcdclient.New(cfg)
	if err != nil {
		log.Fatal(err)
	}

	kapi := etcdclient.NewKeysAPI(c)
	id := utils.GenerateUUID(name)
	rootDir := filepath.Join(EtcdMinionSpace, id.String())
	queueDir := filepath.Join(rootDir, "queue")
	classifierDir := filepath.Join(rootDir, "classifier")
	logDir := filepath.Join(rootDir, "log")
	taskQueue := make(chan *task.Task)
	done := make(chan struct{})

	m := &etcdMinion{
		name:          name,
		rootDir:       rootDir,
		queueDir:      queueDir,
		classifierDir: classifierDir,
		logDir:        logDir,
		id:            id,
		kapi:          kapi,
		taskQueue:     taskQueue,
		done:          done,
	}

	return m
}

// Checks for any pending tasks and sends them
// for processing if there are any
func (m *etcdMinion) checkQueue() error {
	opts := &etcdclient.GetOptions{
		Recursive: true,
		Sort:      true,
	}

	// Get backlog tasks if any
	// If the directory key in etcd is missing that is okay, since
	// it means there are no pending tasks for processing
	resp, err := m.kapi.Get(context.Background(), m.queueDir, opts)
	if err != nil {
		if eerr, ok := err.(etcdclient.Error); !ok || eerr.Code == etcdclient.ErrorCodeKeyNotFound {
			return err
		}
	}

	backlog := resp.Node.Nodes
	if len(backlog) == 0 {
		// No backlog tasks found
		return nil
	}

	log.Printf("Found %d pending tasks in queue", len(backlog))
	for _, node := range backlog {
		t, err := EtcdUnmarshalTask(node)
		m.kapi.Delete(context.Background(), node.Key, nil)

		if err != nil {
			continue
		}

		m.taskQueue <- t
	}

	return nil
}

// Runs periodic jobs such as refreshing classifiers and
// updating the lastseen time every five minutes
func (m *etcdMinion) periodicRunner() {
	schedule := time.Minute * 5
	ticker := time.NewTicker(schedule)
	log.Printf("Periodic scheduler set to run every %s\n", schedule)

	for {
		select {
		case <-m.done:
			break
		case now := <-ticker.C:
			// Run periodic jobs
			m.classify()
			m.checkQueue()
			m.SetLastseen(now.Unix())
		}
	}
}

// Processes new tasks
func (m *etcdMinion) processTask(t *task.Task) error {
	var buf bytes.Buffer

	// Update state of task to indicate that we are now processing it
	t.State = task.TaskStateProcessing
	m.SaveTaskResult(t)

	cmd := exec.Command(t.Command, t.Args...)
	cmd.Stdout = &buf
	cmd.Stderr = &buf

	log.Printf("Processing task %s\n", t.TaskID)

	cmdError := cmd.Run()
	t.TimeProcessed = time.Now().Unix()
	t.Result = buf.String()

	if cmdError != nil {
		log.Printf("Failed to process task %s\n", t.TaskID)
		t.Error = cmdError.Error()
		t.State = task.TaskStateFailed
	} else {
		log.Printf("Finished processing task %s\n", t.TaskID)
		t.State = task.TaskStateSuccess
	}

	m.SaveTaskResult(t)

	return cmdError
}

// Classifies the minion
func (m *etcdMinion) classify() error {
	for key := range classifier.Registry {
		klassifier, err := classifier.Get(key)

		if err != nil {
			log.Printf("Failed to get classifier %s: %s\n", key, err)
			continue
		}

		m.SetClassifier(klassifier)
	}

	return nil
}

// EtcdUnmarshalTask unmarshals tasks from an etcd node
func EtcdUnmarshalTask(node *etcdclient.Node) (*task.Task, error) {
	task := new(task.Task)
	err := json.Unmarshal([]byte(node.Value), &task)

	return task, err
}

// ID returns the minion unique identifier
func (m *etcdMinion) ID() uuid.UUID {
	return m.id
}

// SetName sets the human-readable name of the minion in etcd
func (m *etcdMinion) SetName(name string) error {
	nameKey := filepath.Join(m.rootDir, "name")
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), nameKey, name, opts)
	if err != nil {
		log.Printf("Failed to set name of minion: %s\n", err)
	}

	return err
}

// SetLastseen sets the time the minion was last seen in
// seconds since the Epoch
func (m *etcdMinion) SetLastseen(s int64) error {
	lastseenKey := filepath.Join(m.rootDir, "lastseen")
	lastseenValue := strconv.FormatInt(s, 10)
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), lastseenKey, lastseenValue, opts)
	if err != nil {
		log.Printf("Failed to set lastseen time: %s\n", err)
	}

	return err
}

// SetClassifier sets a classifier for the minion in etcd
func (m *etcdMinion) SetClassifier(c *classifier.Classifier) error {
	// Classifiers in etcd expire after an hour
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
		TTL:       time.Hour,
	}

	// Serialize classifier to JSON and save it in etcd
	data, err := json.Marshal(c)
	if err != nil {
		log.Printf("Failed to serialize classifier %s: %s\n", c.Key, err)
		return err
	}

	// Classifier key in etcd
	klassifierKey := filepath.Join(m.classifierDir, c.Key)
	_, err = m.kapi.Set(context.Background(), klassifierKey, string(data), opts)

	if err != nil {
		log.Printf("Failed to set classifier %s: %s\n", c.Key, err)
		return err
	}

	return nil
}

// TaskListener monitors etcd for new tasks
func (m *etcdMinion) TaskListener(c chan<- *task.Task) error {
	log.Printf("Task listener is watching %s\n", m.queueDir)

	rand.Seed(time.Now().UTC().UnixNano())
	b := backoff.Backoff{
		Min:    1 * time.Second,
		Max:    10 * time.Minute,
		Factor: 2.0,
		Jitter: true,
	}

	watcherOpts := &etcdclient.WatcherOptions{
		Recursive: true,
	}
	watcher := m.kapi.Watcher(m.queueDir, watcherOpts)

	for {
		resp, err := watcher.Next(context.Background())
		if err != nil {
			// Use a backoff and retry later again
			duration := b.Duration()
			log.Printf("%s, retrying in %s\n", err, duration)
			time.Sleep(duration)
			continue
		}

		// Reset the backoff counter on successful receive
		b.Reset()

		// Ignore "delete" events when removing a task from the queue
		action := strings.ToLower(resp.Action)
		if strings.EqualFold(action, "delete") {
			continue
		}

		// Unmarshal and remove task from the queue
		t, err := EtcdUnmarshalTask(resp.Node)
		m.kapi.Delete(context.Background(), resp.Node.Key, nil)

		if err != nil {
			log.Printf("Received invalid task %s: %s\n", resp.Node.Key, err)
			continue
		}

		log.Printf("Received task %s\n", t.TaskID)
		c <- t
	}

	return nil
}

// TaskRunner processes new tasks
func (m *etcdMinion) TaskRunner(c <-chan *task.Task) error {
	log.Println("Starting task runner")

	for {
		select {
		case <-m.done:
			break
		case t := <-c:
			t.State = task.TaskStateQueued
			t.TimeReceived = time.Now().Unix()
			m.SaveTaskResult(t)

			if t.IsConcurrent {
				go m.processTask(t)
			} else {
				m.processTask(t)
			}
		}
	}

	return nil
}

// SaveTaskResult stores the result of a task in etcd
func (m *etcdMinion) SaveTaskResult(t *task.Task) error {
	taskKey := filepath.Join(m.logDir, t.TaskID.String())

	data, err := json.Marshal(t)
	if err != nil {
		log.Printf("Failed to serialize task %s: %s\n", t.TaskID, err)
		return err
	}

	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err = m.kapi.Set(context.Background(), taskKey, string(data), opts)
	if err != nil {
		log.Printf("Failed to save task %s: %s\n", t.TaskID, err)
	}

	return err
}

// Seve starts the minion
func (m *etcdMinion) Serve() error {
	err := m.SetName(m.name)
	if err != nil {
		return err
	}

	now := time.Now().Unix()
	err = m.SetLastseen(now)
	if err != nil {
		return err
	}

	// Start minion services
	go m.classify()
	go m.checkQueue()
	go m.periodicRunner()
	go m.TaskRunner(m.taskQueue)
	go m.TaskListener(m.taskQueue)

	log.Printf("Minion %s is ready to serve", m.ID())

	return nil
}

// Stop shutdowns the minions and its services
func (m *etcdMinion) Stop() error {
	log.Println("Minion is shutting down")

	close(m.taskQueue)
	close(m.done)

	return nil
}
<commit_msg>minion: etcd minions are now able to process tasks with catalog<commit_after>package minion

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"math/rand"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/dnaeon/backoff"
	"github.com/dnaeon/gru/classifier"
	"github.com/dnaeon/gru/task"
	"github.com/dnaeon/gru/utils"

	etcdclient "github.com/coreos/etcd/client"
	"github.com/pborman/uuid"
	"golang.org/x/net/context"
)

// EtcdMinionSpace is the keyspace in etcd used by minions
const EtcdMinionSpace = "/gru/minion"

// Etcd Minion
type etcdMinion struct {
	// Name of the minion
	name string

	// Minion root directory in etcd
	rootDir string

	// Minion queue directory in etcd
	queueDir string

	// Log directory of previously executed tasks
	logDir string

	// Classifier directory in etcd
	classifierDir string

	// Minion unique identifier
	id uuid.UUID

	// KeysAPI client to etcd
	kapi etcdclient.KeysAPI

	// Channel over which tasks are sent for processing
	taskQueue chan *task.Task

	// Channel used to signal shutdown time
	done chan struct{}
}

// NewEtcdMinion creates a new minion with etcd backend
func NewEtcdMinion(name string, cfg etcdclient.Config) Minion {
	c, err := etcdclient.New(cfg)
	if err != nil {
		log.Fatal(err)
	}

	kapi := etcdclient.NewKeysAPI(c)
	id := utils.GenerateUUID(name)
	rootDir := filepath.Join(EtcdMinionSpace, id.String())
	queueDir := filepath.Join(rootDir, "queue")
	classifierDir := filepath.Join(rootDir, "classifier")
	logDir := filepath.Join(rootDir, "log")
	taskQueue := make(chan *task.Task)
	done := make(chan struct{})

	m := &etcdMinion{
		name:          name,
		rootDir:       rootDir,
		queueDir:      queueDir,
		classifierDir: classifierDir,
		logDir:        logDir,
		id:            id,
		kapi:          kapi,
		taskQueue:     taskQueue,
		done:          done,
	}

	return m
}

// Checks for any pending tasks and sends them
// for processing if there are any
func (m *etcdMinion) checkQueue() error {
	opts := &etcdclient.GetOptions{
		Recursive: true,
		Sort:      true,
	}

	// Get backlog tasks if any
	// If the directory key in etcd is missing that is okay, since
	// it means there are no pending tasks for processing
	resp, err := m.kapi.Get(context.Background(), m.queueDir, opts)
	if err != nil {
		if eerr, ok := err.(etcdclient.Error); !ok || eerr.Code == etcdclient.ErrorCodeKeyNotFound {
			return err
		}
	}

	backlog := resp.Node.Nodes
	if len(backlog) == 0 {
		// No backlog tasks found
		return nil
	}

	log.Printf("Found %d pending tasks in queue", len(backlog))
	for _, node := range backlog {
		t, err := EtcdUnmarshalTask(node)
		m.kapi.Delete(context.Background(), node.Key, nil)

		if err != nil {
			continue
		}

		m.taskQueue <- t
	}

	return nil
}

// Runs periodic jobs such as refreshing classifiers and
// updating the lastseen time every five minutes
func (m *etcdMinion) periodicRunner() {
	schedule := time.Minute * 5
	ticker := time.NewTicker(schedule)
	log.Printf("Periodic scheduler set to run every %s\n", schedule)

	for {
		select {
		case <-m.done:
			break
		case now := <-ticker.C:
			// Run periodic jobs
			m.classify()
			m.checkQueue()
			m.SetLastseen(now.Unix())
		}
	}
}

// Processes new tasks
func (m *etcdMinion) processTask(t *task.Task) error {
	// Update state of task to indicate that we are now processing it
	t.State = task.TaskStateProcessing
	m.SaveTaskResult(t)

	log.Printf("Processing task %s\n", t.TaskID)

	var buf bytes.Buffer
	fmt.Fprintf(&buf, "Loaded %d resources from catalog", t.Catalog.Len())
	err := t.Catalog.Run(&buf)
	t.TimeProcessed = time.Now().Unix()
	t.Result = buf.String()

	if err != nil {
		log.Printf("Failed to process task %s: %s\n", t.TaskID, err)
		t.State = task.TaskStateFailed
	} else {
		log.Printf("Finished processing task %s\n", t.TaskID)
		t.State = task.TaskStateSuccess
	}

	m.SaveTaskResult(t)

	return err
}

// Classifies the minion
func (m *etcdMinion) classify() error {
	for key := range classifier.Registry {
		klassifier, err := classifier.Get(key)

		if err != nil {
			log.Printf("Failed to get classifier %s: %s\n", key, err)
			continue
		}

		m.SetClassifier(klassifier)
	}

	return nil
}

// EtcdUnmarshalTask unmarshals tasks from an etcd node
func EtcdUnmarshalTask(node *etcdclient.Node) (*task.Task, error) {
	task := new(task.Task)
	err := json.Unmarshal([]byte(node.Value), &task)

	return task, err
}

// ID returns the minion unique identifier
func (m *etcdMinion) ID() uuid.UUID {
	return m.id
}

// SetName sets the human-readable name of the minion in etcd
func (m *etcdMinion) SetName(name string) error {
	nameKey := filepath.Join(m.rootDir, "name")
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), nameKey, name, opts)
	if err != nil {
		log.Printf("Failed to set name of minion: %s\n", err)
	}

	return err
}

// SetLastseen sets the time the minion was last seen in
// seconds since the Epoch
func (m *etcdMinion) SetLastseen(s int64) error {
	lastseenKey := filepath.Join(m.rootDir, "lastseen")
	lastseenValue := strconv.FormatInt(s, 10)
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), lastseenKey, lastseenValue, opts)
	if err != nil {
		log.Printf("Failed to set lastseen time: %s\n", err)
	}

	return err
}

// SetClassifier sets a classifier for the minion in etcd
func (m *etcdMinion) SetClassifier(c *classifier.Classifier) error {
	// Classifiers in etcd expire after an hour
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
		TTL:       time.Hour,
	}

	// Serialize classifier to JSON and save it in etcd
	data, err := json.Marshal(c)
	if err != nil {
		log.Printf("Failed to serialize classifier %s: %s\n", c.Key, err)
		return err
	}

	// Classifier key in etcd
	klassifierKey := filepath.Join(m.classifierDir, c.Key)
	_, err = m.kapi.Set(context.Background(), klassifierKey, string(data), opts)

	if err != nil {
		log.Printf("Failed to set classifier %s: %s\n", c.Key, err)
		return err
	}

	return nil
}

// TaskListener monitors etcd for new tasks
func (m *etcdMinion) TaskListener(c chan<- *task.Task) error {
	log.Printf("Task listener is watching %s\n", m.queueDir)

	rand.Seed(time.Now().UTC().UnixNano())
	b := backoff.Backoff{
		Min:    1 * time.Second,
		Max:    10 * time.Minute,
		Factor: 2.0,
		Jitter: true,
	}

	watcherOpts := &etcdclient.WatcherOptions{
		Recursive: true,
	}
	watcher := m.kapi.Watcher(m.queueDir, watcherOpts)

	for {
		resp, err := watcher.Next(context.Background())
		if err != nil {
			// Use a backoff and retry later again
			duration := b.Duration()
			log.Printf("%s, retrying in %s\n", err, duration)
			time.Sleep(duration)
			continue
		}

		// Reset the backoff counter on successful receive
		b.Reset()

		// Ignore "delete" events when removing a task from the queue
		action := strings.ToLower(resp.Action)
		if strings.EqualFold(action, "delete") {
			continue
		}

		// Unmarshal and remove task from the queue
		t, err := EtcdUnmarshalTask(resp.Node)
		m.kapi.Delete(context.Background(), resp.Node.Key, nil)

		if err != nil {
			log.Printf("Received invalid task %s: %s\n", resp.Node.Key, err)
			continue
		}

		log.Printf("Received task %s\n", t.TaskID)
		c <- t
	}

	return nil
}

// TaskRunner processes new tasks
func (m *etcdMinion) TaskRunner(c <-chan *task.Task) error {
	log.Println("Starting task runner")

	for {
		select {
		case <-m.done:
			break
		case t := <-c:
			t.State = task.TaskStateQueued
			t.TimeReceived = time.Now().Unix()
			m.SaveTaskResult(t)

			if t.IsConcurrent {
				go m.processTask(t)
			} else {
				m.processTask(t)
			}
		}
	}

	return nil
}

// SaveTaskResult stores the result of a task in etcd
func (m *etcdMinion) SaveTaskResult(t *task.Task) error {
	taskKey := filepath.Join(m.logDir, t.TaskID.String())

	data, err := json.Marshal(t)
	if err != nil {
		log.Printf("Failed to serialize task %s: %s\n", t.TaskID, err)
		return err
	}

	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err = m.kapi.Set(context.Background(), taskKey, string(data), opts)
	if err != nil {
		log.Printf("Failed to save task %s: %s\n", t.TaskID, err)
	}

	return err
}

// Seve starts the minion
func (m *etcdMinion) Serve() error {
	err := m.SetName(m.name)
	if err != nil {
		return err
	}

	now := time.Now().Unix()
	err = m.SetLastseen(now)
	if err != nil {
		return err
	}

	// Start minion services
	go m.classify()
	go m.checkQueue()
	go m.periodicRunner()
	go m.TaskRunner(m.taskQueue)
	go m.TaskListener(m.taskQueue)

	log.Printf("Minion %s is ready to serve", m.ID())

	return nil
}

// Stop shutdowns the minions and its services
func (m *etcdMinion) Stop() error {
	log.Println("Minion is shutting down")

	close(m.taskQueue)
	close(m.done)

	return nil
}
<|endoftext|>


<commit_before>package filemetadata

import (
	"io/ioutil"
	"os"
	"testing"
	"time"

	"github.com/bazelbuild/remote-apis-sdks/go/digest"
	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"
)

func TestCompute(t *testing.T) {
	tests := []struct {
		name       string
		contents   string
		executable bool
	}{
		{
			name:     "empty",
			contents: "",
		},
		{
			name:     "non-executable",
			contents: "bla",
		},
		{
			name:       "executable",
			contents:   "foo",
			executable: true,
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			margin := 20 * time.Millisecond
			before := time.Now().Truncate(margin)
			filename, err := createFile(t, tc.executable, tc.contents)
			if err != nil {
				t.Fatalf("Failed to create tmp file for testing digests: %v", err)
			}
			after := time.Now().Truncate(margin).Add(margin)
			got, err := Compute(filename)
			if err != nil {
				t.Errorf("Compute(%v) failed. Got error: %v", filename, err)
			}
			want := &Metadata{
				Digest:       digest.NewFromBlob([]byte(tc.contents)),
				IsExecutable: tc.executable,
			}
			if diff := cmp.Diff(want, got, cmpopts.IgnoreFields(Metadata{}, "MTime")); diff != "" {
				t.Errorf("Compute(%v) returned diff. (-want +got)\n%s", filename, diff)
			}
			if got.MTime.Before(before) || got.MTime.After(after) {
				t.Errorf("Compute(%v) returned MTime %v, expected time in (%v, %v).", filename, got.MTime, before, after)
			}
		})
	}
}

func createFile(t *testing.T, executable bool, contents string) (string, error) {
	t.Helper()
	perm := os.FileMode(0666)
	if executable {
		perm = os.FileMode(0766)
	}
	tmpFile, err := ioutil.TempFile(os.TempDir(), "")
	if err != nil {
		return "", err
	}
	if err := tmpFile.Chmod(perm); err != nil {
		return "", err
	}
	if err := tmpFile.Close(); err != nil {
		return "", err
	}
	filename := tmpFile.Name()
	if err = ioutil.WriteFile(filename, []byte(contents), os.ModeTemporary); err != nil {
		return "", err
	}
	return filename, nil
}
<commit_msg>Deflaking file metadata test (#50)<commit_after>package filemetadata

import (
	"io/ioutil"
	"os"
	"testing"
	"time"

	"github.com/bazelbuild/remote-apis-sdks/go/digest"
	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"
)

func TestCompute(t *testing.T) {
	tests := []struct {
		name       string
		contents   string
		executable bool
	}{
		{
			name:     "empty",
			contents: "",
		},
		{
			name:     "non-executable",
			contents: "bla",
		},
		{
			name:       "executable",
			contents:   "foo",
			executable: true,
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			margin := time.Second
			before := time.Now().Truncate(margin)
			filename, err := createFile(t, tc.executable, tc.contents)
			if err != nil {
				t.Fatalf("Failed to create tmp file for testing digests: %v", err)
			}
			after := time.Now().Truncate(margin).Add(margin)
			got, err := Compute(filename)
			if err != nil {
				t.Errorf("Compute(%v) failed. Got error: %v", filename, err)
			}
			want := &Metadata{
				Digest:       digest.NewFromBlob([]byte(tc.contents)),
				IsExecutable: tc.executable,
			}
			if diff := cmp.Diff(want, got, cmpopts.IgnoreFields(Metadata{}, "MTime")); diff != "" {
				t.Errorf("Compute(%v) returned diff. (-want +got)\n%s", filename, diff)
			}
			if got.MTime.Before(before) || got.MTime.After(after) {
				t.Errorf("Compute(%v) returned MTime %v, expected time in (%v, %v).", filename, got.MTime, before, after)
			}
		})
	}
}

func createFile(t *testing.T, executable bool, contents string) (string, error) {
	t.Helper()
	perm := os.FileMode(0666)
	if executable {
		perm = os.FileMode(0766)
	}
	tmpFile, err := ioutil.TempFile(os.TempDir(), "")
	if err != nil {
		return "", err
	}
	if err := tmpFile.Chmod(perm); err != nil {
		return "", err
	}
	if err := tmpFile.Close(); err != nil {
		return "", err
	}
	filename := tmpFile.Name()
	if err = ioutil.WriteFile(filename, []byte(contents), os.ModeTemporary); err != nil {
		return "", err
	}
	return filename, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/versions"
	"github.com/docker/docker/pkg/integration/checker"
	"github.com/go-check/check"
)

var expectedNetworkInterfaceStats = strings.Split("rx_bytes rx_dropped rx_errors rx_packets tx_bytes tx_dropped tx_errors tx_packets", " ")

func (s *DockerSuite) TestApiStatsNoStreamGetCpu(c *check.C) {
	out, _ := dockerCmd(c, "run", "-d", "busybox", "/bin/sh", "-c", "while true;do echo 'Hello'; usleep 100000; done")

	id := strings.TrimSpace(out)
	c.Assert(waitRun(id), checker.IsNil)

	resp, body, err := sockRequestRaw("GET", fmt.Sprintf("/containers/%s/stats?stream=false", id), nil, "")
	c.Assert(err, checker.IsNil)
	c.Assert(resp.StatusCode, checker.Equals, http.StatusOK)
	c.Assert(resp.Header.Get("Content-Type"), checker.Equals, "application/json")

	var v *types.Stats
	err = json.NewDecoder(body).Decode(&v)
	c.Assert(err, checker.IsNil)
	body.Close()

	var cpuPercent = 0.0

	if daemonPlatform != "windows" {
		cpuDelta := float64(v.CPUStats.CPUUsage.TotalUsage - v.PreCPUStats.CPUUsage.TotalUsage)
		systemDelta := float64(v.CPUStats.SystemUsage - v.PreCPUStats.SystemUsage)
		cpuPercent = (cpuDelta / systemDelta) * float64(len(v.CPUStats.CPUUsage.PercpuUsage)) * 100.0
	} else {
		// Max number of 100ns intervals between the previous time read and now
		possIntervals := uint64(v.Read.Sub(v.PreRead).Nanoseconds()) // Start with number of ns intervals
		possIntervals /= 100                                         // Convert to number of 100ns intervals
		possIntervals *= uint64(v.NumProcs)                          // Multiple by the number of processors

		// Intervals used
		intervalsUsed := v.CPUStats.CPUUsage.TotalUsage - v.PreCPUStats.CPUUsage.TotalUsage

		// Percentage avoiding divide-by-zero
		if possIntervals > 0 {
			cpuPercent = float64(intervalsUsed) / float64(possIntervals) * 100.0
		}
	}

	c.Assert(cpuPercent, check.Not(checker.Equals), 0.0, check.Commentf("docker stats with no-stream get cpu usage failed: was %v", cpuPercent))
}

func (s *DockerSuite) TestApiStatsStoppedContainerInGoroutines(c *check.C) {
	out, _ := dockerCmd(c, "run", "-d", "busybox", "/bin/sh", "-c", "echo 1")
	id := strings.TrimSpace(out)

	getGoRoutines := func() int {
		_, body, err := sockRequestRaw("GET", fmt.Sprintf("/info"), nil, "")
		c.Assert(err, checker.IsNil)
		info := types.Info{}
		err = json.NewDecoder(body).Decode(&info)
		c.Assert(err, checker.IsNil)
		body.Close()
		return info.NGoroutines
	}

	// When the HTTP connection is closed, the number of goroutines should not increase.
	routines := getGoRoutines()
	_, body, err := sockRequestRaw("GET", fmt.Sprintf("/containers/%s/stats", id), nil, "")
	c.Assert(err, checker.IsNil)
	body.Close()

	t := time.After(30 * time.Second)
	for {
		select {
		case <-t:
			c.Assert(getGoRoutines(), checker.LessOrEqualThan, routines)
			return
		default:
			if n := getGoRoutines(); n <= routines {
				return
			}
			time.Sleep(200 * time.Millisecond)
		}
	}
}

func (s *DockerSuite) TestApiStatsNetworkStats(c *check.C) {
	testRequires(c, SameHostDaemon)

	out, _ := runSleepingContainer(c)
	id := strings.TrimSpace(out)
	c.Assert(waitRun(id), checker.IsNil)

	// Retrieve the container address
	net := "bridge"
	if daemonPlatform == "windows" {
		net = "nat"
	}
	contIP := findContainerIP(c, id, net)
	numPings := 1

	var preRxPackets uint64
	var preTxPackets uint64
	var postRxPackets uint64
	var postTxPackets uint64

	// Get the container networking stats before and after pinging the container
	nwStatsPre := getNetworkStats(c, id)
	for _, v := range nwStatsPre {
		preRxPackets += v.RxPackets
		preTxPackets += v.TxPackets
	}

	countParam := "-c"
	if runtime.GOOS == "windows" {
		countParam = "-n" // Ping count parameter is -n on Windows
	}
	pingout, err := exec.Command("ping", contIP, countParam, strconv.Itoa(numPings)).CombinedOutput()
	if err != nil && runtime.GOOS == "linux" {
		// If it fails then try a work-around, but just for linux.
		// If this fails too then go back to the old error for reporting.
		//
		// The ping will sometimes fail due to an apparmor issue where it
		// denies access to the libc.so.6 shared library - running it
		// via /lib64/ld-linux-x86-64.so.2 seems to work around it.
		pingout2, err2 := exec.Command("/lib64/ld-linux-x86-64.so.2", "/bin/ping", contIP, "-c", strconv.Itoa(numPings)).CombinedOutput()
		if err2 == nil {
			pingout = pingout2
			err = err2
		}
	}
	c.Assert(err, checker.IsNil)
	pingouts := string(pingout[:])
	nwStatsPost := getNetworkStats(c, id)
	for _, v := range nwStatsPost {
		postRxPackets += v.RxPackets
		postTxPackets += v.TxPackets
	}

	// Verify the stats contain at least the expected number of packets
	// On Linux, account for ARP.
	expRxPkts := preRxPackets + uint64(numPings)
	expTxPkts := preTxPackets + uint64(numPings)
	if daemonPlatform != "windows" {
		expRxPkts++
		expTxPkts++
	}
	c.Assert(postTxPackets, checker.GreaterOrEqualThan, expTxPkts,
		check.Commentf("Reported less TxPackets than expected. Expected >= %d. Found %d. %s", expTxPkts, postTxPackets, pingouts))
	c.Assert(postRxPackets, checker.GreaterOrEqualThan, expRxPkts,
		check.Commentf("Reported less Txbytes than expected. Expected >= %d. Found %d. %s", expRxPkts, postRxPackets, pingouts))
}

func (s *DockerSuite) TestApiStatsNetworkStatsVersioning(c *check.C) {
	testRequires(c, SameHostDaemon)

	out, _ := runSleepingContainer(c)
	id := strings.TrimSpace(out)
	c.Assert(waitRun(id), checker.IsNil)
	wg := sync.WaitGroup{}

	for i := 17; i <= 21; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			apiVersion := fmt.Sprintf("v1.%d", i)
			statsJSONBlob := getVersionedStats(c, id, apiVersion)
			if versions.LessThan(apiVersion, "v1.21") {
				c.Assert(jsonBlobHasLTv121NetworkStats(statsJSONBlob), checker.Equals, true,
					check.Commentf("Stats JSON blob from API %s %#v does not look like a <v1.21 API stats structure", apiVersion, statsJSONBlob))
			} else {
				c.Assert(jsonBlobHasGTE121NetworkStats(statsJSONBlob), checker.Equals, true,
					check.Commentf("Stats JSON blob from API %s %#v does not look like a >=v1.21 API stats structure", apiVersion, statsJSONBlob))
			}
		}()
	}
	wg.Wait()
}

func getNetworkStats(c *check.C, id string) map[string]types.NetworkStats {
	var st *types.StatsJSON

	_, body, err := sockRequestRaw("GET", fmt.Sprintf("/containers/%s/stats?stream=false", id), nil, "")
	c.Assert(err, checker.IsNil)

	err = json.NewDecoder(body).Decode(&st)
	c.Assert(err, checker.IsNil)
	body.Close()

	return st.Networks
}

// getVersionedStats returns stats result for the
// container with id using an API call with version apiVersion. Since the
// stats result type differs between API versions, we simply return
// map[string]interface{}.
func getVersionedStats(c *check.C, id string, apiVersion string) map[string]interface{} {
	stats := make(map[string]interface{})

	_, body, err := sockRequestRaw("GET", fmt.Sprintf("/%s/containers/%s/stats?stream=false", apiVersion, id), nil, "")
	c.Assert(err, checker.IsNil)
	defer body.Close()

	err = json.NewDecoder(body).Decode(&stats)
	c.Assert(err, checker.IsNil, check.Commentf("failed to decode stat: %s", err))

	return stats
}

func jsonBlobHasLTv121NetworkStats(blob map[string]interface{}) bool {
	networkStatsIntfc, ok := blob["network"]
	if !ok {
		return false
	}
	networkStats, ok := networkStatsIntfc.(map[string]interface{})
	if !ok {
		return false
	}
	for _, expectedKey := range expectedNetworkInterfaceStats {
		if _, ok := networkStats[expectedKey]; !ok {
			return false
		}
	}
	return true
}

func jsonBlobHasGTE121NetworkStats(blob map[string]interface{}) bool {
	networksStatsIntfc, ok := blob["networks"]
	if !ok {
		return false
	}
	networksStats, ok := networksStatsIntfc.(map[string]interface{})
	if !ok {
		return false
	}
	for _, networkInterfaceStatsIntfc := range networksStats {
		networkInterfaceStats, ok := networkInterfaceStatsIntfc.(map[string]interface{})
		if !ok {
			return false
		}
		for _, expectedKey := range expectedNetworkInterfaceStats {
			if _, ok := networkInterfaceStats[expectedKey]; !ok {
				return false
			}
		}
	}
	return true
}

func (s *DockerSuite) TestApiStatsContainerNotFound(c *check.C) {
	testRequires(c, DaemonIsLinux)

	status, _, err := sockRequest("GET", "/containers/nonexistent/stats", nil)
	c.Assert(err, checker.IsNil)
	c.Assert(status, checker.Equals, http.StatusNotFound)

	status, _, err = sockRequest("GET", "/containers/nonexistent/stats?stream=0", nil)
	c.Assert(err, checker.IsNil)
	c.Assert(status, checker.Equals, http.StatusNotFound)
}

func (s *DockerSuite) TestApiStatsNoStreamConnectedContainers(c *check.C) {
	testRequires(c, DaemonIsLinux)

	out1, _ := runSleepingContainer(c)
	id1 := strings.TrimSpace(out1)
	c.Assert(waitRun(id1), checker.IsNil)

	out2, _ := runSleepingContainer(c, "--net", "container:"+id1)
	id2 := strings.TrimSpace(out2)
	c.Assert(waitRun(id2), checker.IsNil)

	ch := make(chan error)
	go func() {
		resp, body, err := sockRequestRaw("GET", fmt.Sprintf("/containers/%s/stats?stream=false", id2), nil, "")
		defer body.Close()
		if err != nil {
			ch <- err
		}
		if resp.StatusCode != http.StatusOK {
			ch <- fmt.Errorf("Invalid StatusCode %v", resp.StatusCode)
		}
		if resp.Header.Get("Content-Type") != "application/json" {
			ch <- fmt.Errorf("Invalid 'Content-Type' %v", resp.Header.Get("Content-Type"))
		}
		var v *types.Stats
		if err := json.NewDecoder(body).Decode(&v); err != nil {
			ch <- err
		}
		ch <- nil
	}()

	select {
	case err := <-ch:
		c.Assert(err, checker.IsNil, check.Commentf("Error in stats remote API: %v", err))
	case <-time.After(15 * time.Second):
		c.Fatalf("Stats did not return after timeout")
	}
}
<commit_msg>Fix datarace in ApiStatsnetworkStatsVersioning<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/versions"
	"github.com/docker/docker/pkg/integration/checker"
	"github.com/go-check/check"
)

var expectedNetworkInterfaceStats = strings.Split("rx_bytes rx_dropped rx_errors rx_packets tx_bytes tx_dropped tx_errors tx_packets", " ")

func (s *DockerSuite) TestApiStatsNoStreamGetCpu(c *check.C) {
	out, _ := dockerCmd(c, "run", "-d", "busybox", "/bin/sh", "-c", "while true;do echo 'Hello'; usleep 100000; done")

	id := strings.TrimSpace(out)
	c.Assert(waitRun(id), checker.IsNil)

	resp, body, err := sockRequestRaw("GET", fmt.Sprintf("/containers/%s/stats?stream=false", id), nil, "")
	c.Assert(err, checker.IsNil)
	c.Assert(resp.StatusCode, checker.Equals, http.StatusOK)
	c.Assert(resp.Header.Get("Content-Type"), checker.Equals, "application/json")

	var v *types.Stats
	err = json.NewDecoder(body).Decode(&v)
	c.Assert(err, checker.IsNil)
	body.Close()

	var cpuPercent = 0.0

	if daemonPlatform != "windows" {
		cpuDelta := float64(v.CPUStats.CPUUsage.TotalUsage - v.PreCPUStats.CPUUsage.TotalUsage)
		systemDelta := float64(v.CPUStats.SystemUsage - v.PreCPUStats.SystemUsage)
		cpuPercent = (cpuDelta / systemDelta) * float64(len(v.CPUStats.CPUUsage.PercpuUsage)) * 100.0
	} else {
		// Max number of 100ns intervals between the previous time read and now
		possIntervals := uint64(v.Read.Sub(v.PreRead).Nanoseconds()) // Start with number of ns intervals
		possIntervals /= 100                                         // Convert to number of 100ns intervals
		possIntervals *= uint64(v.NumProcs)                          // Multiple by the number of processors

		// Intervals used
		intervalsUsed := v.CPUStats.CPUUsage.TotalUsage - v.PreCPUStats.CPUUsage.TotalUsage

		// Percentage avoiding divide-by-zero
		if possIntervals > 0 {
			cpuPercent = float64(intervalsUsed) / float64(possIntervals) * 100.0
		}
	}

	c.Assert(cpuPercent, check.Not(checker.Equals), 0.0, check.Commentf("docker stats with no-stream get cpu usage failed: was %v", cpuPercent))
}

func (s *DockerSuite) TestApiStatsStoppedContainerInGoroutines(c *check.C) {
	out, _ := dockerCmd(c, "run", "-d", "busybox", "/bin/sh", "-c", "echo 1")
	id := strings.TrimSpace(out)

	getGoRoutines := func() int {
		_, body, err := sockRequestRaw("GET", fmt.Sprintf("/info"), nil, "")
		c.Assert(err, checker.IsNil)
		info := types.Info{}
		err = json.NewDecoder(body).Decode(&info)
		c.Assert(err, checker.IsNil)
		body.Close()
		return info.NGoroutines
	}

	// When the HTTP connection is closed, the number of goroutines should not increase.
	routines := getGoRoutines()
	_, body, err := sockRequestRaw("GET", fmt.Sprintf("/containers/%s/stats", id), nil, "")
	c.Assert(err, checker.IsNil)
	body.Close()

	t := time.After(30 * time.Second)
	for {
		select {
		case <-t:
			c.Assert(getGoRoutines(), checker.LessOrEqualThan, routines)
			return
		default:
			if n := getGoRoutines(); n <= routines {
				return
			}
			time.Sleep(200 * time.Millisecond)
		}
	}
}

func (s *DockerSuite) TestApiStatsNetworkStats(c *check.C) {
	testRequires(c, SameHostDaemon)

	out, _ := runSleepingContainer(c)
	id := strings.TrimSpace(out)
	c.Assert(waitRun(id), checker.IsNil)

	// Retrieve the container address
	net := "bridge"
	if daemonPlatform == "windows" {
		net = "nat"
	}
	contIP := findContainerIP(c, id, net)
	numPings := 1

	var preRxPackets uint64
	var preTxPackets uint64
	var postRxPackets uint64
	var postTxPackets uint64

	// Get the container networking stats before and after pinging the container
	nwStatsPre := getNetworkStats(c, id)
	for _, v := range nwStatsPre {
		preRxPackets += v.RxPackets
		preTxPackets += v.TxPackets
	}

	countParam := "-c"
	if runtime.GOOS == "windows" {
		countParam = "-n" // Ping count parameter is -n on Windows
	}
	pingout, err := exec.Command("ping", contIP, countParam, strconv.Itoa(numPings)).CombinedOutput()
	if err != nil && runtime.GOOS == "linux" {
		// If it fails then try a work-around, but just for linux.
		// If this fails too then go back to the old error for reporting.
		//
		// The ping will sometimes fail due to an apparmor issue where it
		// denies access to the libc.so.6 shared library - running it
		// via /lib64/ld-linux-x86-64.so.2 seems to work around it.
		pingout2, err2 := exec.Command("/lib64/ld-linux-x86-64.so.2", "/bin/ping", contIP, "-c", strconv.Itoa(numPings)).CombinedOutput()
		if err2 == nil {
			pingout = pingout2
			err = err2
		}
	}
	c.Assert(err, checker.IsNil)
	pingouts := string(pingout[:])
	nwStatsPost := getNetworkStats(c, id)
	for _, v := range nwStatsPost {
		postRxPackets += v.RxPackets
		postTxPackets += v.TxPackets
	}

	// Verify the stats contain at least the expected number of packets
	// On Linux, account for ARP.
	expRxPkts := preRxPackets + uint64(numPings)
	expTxPkts := preTxPackets + uint64(numPings)
	if daemonPlatform != "windows" {
		expRxPkts++
		expTxPkts++
	}
	c.Assert(postTxPackets, checker.GreaterOrEqualThan, expTxPkts,
		check.Commentf("Reported less TxPackets than expected. Expected >= %d. Found %d. %s", expTxPkts, postTxPackets, pingouts))
	c.Assert(postRxPackets, checker.GreaterOrEqualThan, expRxPkts,
		check.Commentf("Reported less Txbytes than expected. Expected >= %d. Found %d. %s", expRxPkts, postRxPackets, pingouts))
}

func (s *DockerSuite) TestApiStatsNetworkStatsVersioning(c *check.C) {
	testRequires(c, SameHostDaemon)

	out, _ := runSleepingContainer(c)
	id := strings.TrimSpace(out)
	c.Assert(waitRun(id), checker.IsNil)
	wg := sync.WaitGroup{}

	// Windows API versions prior to 1.21 doesn't support stats.
	startAt := 17
	if daemonPlatform == "windows" {
		startAt = 21
	}

	for i := startAt; i <= 21; i++ {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			apiVersion := fmt.Sprintf("v1.%d", i)
			statsJSONBlob := getVersionedStats(c, id, apiVersion)
			if versions.LessThan(apiVersion, "v1.21") {
				c.Assert(jsonBlobHasLTv121NetworkStats(statsJSONBlob), checker.Equals, true,
					check.Commentf("Stats JSON blob from API %s %#v does not look like a <v1.21 API stats structure", apiVersion, statsJSONBlob))
			} else {
				c.Assert(jsonBlobHasGTE121NetworkStats(statsJSONBlob), checker.Equals, true,
					check.Commentf("Stats JSON blob from API %s %#v does not look like a >=v1.21 API stats structure", apiVersion, statsJSONBlob))
			}
		}(i)
	}
	wg.Wait()
}

func getNetworkStats(c *check.C, id string) map[string]types.NetworkStats {
	var st *types.StatsJSON

	_, body, err := sockRequestRaw("GET", fmt.Sprintf("/containers/%s/stats?stream=false", id), nil, "")
	c.Assert(err, checker.IsNil)

	err = json.NewDecoder(body).Decode(&st)
	c.Assert(err, checker.IsNil)
	body.Close()

	return st.Networks
}

// getVersionedStats returns stats result for the
// container with id using an API call with version apiVersion. Since the
// stats result type differs between API versions, we simply return
// map[string]interface{}.
func getVersionedStats(c *check.C, id string, apiVersion string) map[string]interface{} {
	stats := make(map[string]interface{})

	_, body, err := sockRequestRaw("GET", fmt.Sprintf("/%s/containers/%s/stats?stream=false", apiVersion, id), nil, "")
	c.Assert(err, checker.IsNil)
	defer body.Close()

	err = json.NewDecoder(body).Decode(&stats)
	c.Assert(err, checker.IsNil, check.Commentf("failed to decode stat: %s", err))

	return stats
}

func jsonBlobHasLTv121NetworkStats(blob map[string]interface{}) bool {
	networkStatsIntfc, ok := blob["network"]
	if !ok {
		return false
	}
	networkStats, ok := networkStatsIntfc.(map[string]interface{})
	if !ok {
		return false
	}
	for _, expectedKey := range expectedNetworkInterfaceStats {
		if _, ok := networkStats[expectedKey]; !ok {
			return false
		}
	}
	return true
}

func jsonBlobHasGTE121NetworkStats(blob map[string]interface{}) bool {
	networksStatsIntfc, ok := blob["networks"]
	if !ok {
		return false
	}
	networksStats, ok := networksStatsIntfc.(map[string]interface{})
	if !ok {
		return false
	}
	for _, networkInterfaceStatsIntfc := range networksStats {
		networkInterfaceStats, ok := networkInterfaceStatsIntfc.(map[string]interface{})
		if !ok {
			return false
		}
		for _, expectedKey := range expectedNetworkInterfaceStats {
			if _, ok := networkInterfaceStats[expectedKey]; !ok {
				return false
			}
		}
	}
	return true
}

func (s *DockerSuite) TestApiStatsContainerNotFound(c *check.C) {
	testRequires(c, DaemonIsLinux)

	status, _, err := sockRequest("GET", "/containers/nonexistent/stats", nil)
	c.Assert(err, checker.IsNil)
	c.Assert(status, checker.Equals, http.StatusNotFound)

	status, _, err = sockRequest("GET", "/containers/nonexistent/stats?stream=0", nil)
	c.Assert(err, checker.IsNil)
	c.Assert(status, checker.Equals, http.StatusNotFound)
}

func (s *DockerSuite) TestApiStatsNoStreamConnectedContainers(c *check.C) {
	testRequires(c, DaemonIsLinux)

	out1, _ := runSleepingContainer(c)
	id1 := strings.TrimSpace(out1)
	c.Assert(waitRun(id1), checker.IsNil)

	out2, _ := runSleepingContainer(c, "--net", "container:"+id1)
	id2 := strings.TrimSpace(out2)
	c.Assert(waitRun(id2), checker.IsNil)

	ch := make(chan error)
	go func() {
		resp, body, err := sockRequestRaw("GET", fmt.Sprintf("/containers/%s/stats?stream=false", id2), nil, "")
		defer body.Close()
		if err != nil {
			ch <- err
		}
		if resp.StatusCode != http.StatusOK {
			ch <- fmt.Errorf("Invalid StatusCode %v", resp.StatusCode)
		}
		if resp.Header.Get("Content-Type") != "application/json" {
			ch <- fmt.Errorf("Invalid 'Content-Type' %v", resp.Header.Get("Content-Type"))
		}
		var v *types.Stats
		if err := json.NewDecoder(body).Decode(&v); err != nil {
			ch <- err
		}
		ch <- nil
	}()

	select {
	case err := <-ch:
		c.Assert(err, checker.IsNil, check.Commentf("Error in stats remote API: %v", err))
	case <-time.After(15 * time.Second):
		c.Fatalf("Stats did not return after timeout")
	}
}
<|endoftext|>