<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package lxc_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	stdtesting "testing"

	"github.com/loggo/loggo"
	gc "launchpad.net/gocheck"
	"launchpad.net/golxc"
	"launchpad.net/goyaml"

	"launchpad.net/juju-core/container"
	"launchpad.net/juju-core/container/lxc"
	lxctesting "launchpad.net/juju-core/container/lxc/testing"
	containertesting "launchpad.net/juju-core/container/testing"
	instancetest "launchpad.net/juju-core/instance/testing"
	jc "launchpad.net/juju-core/testing/checkers"
	"launchpad.net/juju-core/testing/testbase"
)

func Test(t *stdtesting.T) {
	gc.TestingT(t)
}

type LxcSuite struct {
	lxctesting.TestSuite
}

var _ = gc.Suite(&LxcSuite{})

func (s *LxcSuite) SetUpSuite(c *gc.C) {
	s.TestSuite.SetUpSuite(c)
	tmpDir := c.MkDir()
	restore := testbase.PatchEnvironment("PATH", tmpDir)
	s.AddSuiteCleanup(func(*gc.C) { restore() })
	err := ioutil.WriteFile(
		filepath.Join(tmpDir, "apt-config"),
		[]byte(aptConfigScript),
		0755)
	c.Assert(err, gc.IsNil)
}

func (s *LxcSuite) SetUpTest(c *gc.C) {
	s.TestSuite.SetUpTest(c)
	loggo.GetLogger("juju.container.lxc").SetLogLevel(loggo.TRACE)
}

const (
	aptHTTPProxy     = "http://1.2.3.4:3142"
	configProxyExtra = `Acquire::https::Proxy "false";
Acquire::ftp::Proxy "false";`
)

var (
	configHttpProxy = fmt.Sprintf(`Acquire::http::Proxy "%s";`, aptHTTPProxy)
	aptConfigScript = fmt.Sprintf("#!/bin/sh\n echo '%s\n%s'", configHttpProxy, configProxyExtra)
)

func (s *LxcSuite) TestStartContainer(c *gc.C) {
	manager := lxc.NewContainerManager(container.ManagerConfig{})
	instance := containertesting.StartContainer(c, manager, "1/lxc/0")

	name := string(instance.Id())
	// Check our container config files.
	lxcConfContents, err := ioutil.ReadFile(filepath.Join(s.ContainerDir, name, "lxc.conf"))
	c.Assert(err, gc.IsNil)
	c.Assert(string(lxcConfContents), jc.Contains, "lxc.network.link = nic42")

	cloudInitFilename := filepath.Join(s.ContainerDir, name, "cloud-init")
	data := containertesting.AssertCloudInit(c, cloudInitFilename)

	x := make(map[interface{}]interface{})
	err = goyaml.Unmarshal(data, &x)
	c.Assert(err, gc.IsNil)

	c.Assert(x["apt_proxy"], gc.Equals, aptHTTPProxy)

	var scripts []string
	for _, s := range x["runcmd"].([]interface{}) {
		scripts = append(scripts, s.(string))
	}

	c.Assert(scripts[len(scripts)-4:], gc.DeepEquals, []string{
		"start jujud-machine-1-lxc-0",
		"install -D -m 644 /dev/null '/etc/apt/apt.conf.d/99proxy-extra'",
		fmt.Sprintf(`printf '%%s\n' '%s' > '/etc/apt/apt.conf.d/99proxy-extra'`, configProxyExtra),
		"ifconfig",
	})

	// Check the mount point has been created inside the container.
	c.Assert(filepath.Join(s.LxcDir, name, "rootfs/var/log/juju"), jc.IsDirectory)
	// Check that the config file is linked in the restart dir.
	expectedLinkLocation := filepath.Join(s.RestartDir, name+".conf")
	expectedTarget := filepath.Join(s.LxcDir, name, "config")
	linkInfo, err := os.Lstat(expectedLinkLocation)
	c.Assert(err, gc.IsNil)
	c.Assert(linkInfo.Mode()&os.ModeSymlink, gc.Equals, os.ModeSymlink)

	location, err := os.Readlink(expectedLinkLocation)
	c.Assert(err, gc.IsNil)
	c.Assert(location, gc.Equals, expectedTarget)
}

func (s *LxcSuite) TestContainerState(c *gc.C) {
	manager := lxc.NewContainerManager(container.ManagerConfig{})
	instance := containertesting.StartContainer(c, manager, "1/lxc/0")

	// The mock container will be immediately "running".
	c.Assert(instance.Status(), gc.Equals, string(golxc.StateRunning))

	// StopContainer stops and then destroys the container, putting it
	// into "unknown" state.
	err := manager.StopContainer(instance)
	c.Assert(err, gc.IsNil)
	c.Assert(instance.Status(), gc.Equals, string(golxc.StateUnknown))
}

func (s *LxcSuite) TestStopContainer(c *gc.C) {
	manager := lxc.NewContainerManager(container.ManagerConfig{})
	instance := containertesting.StartContainer(c, manager, "1/lxc/0")

	err := manager.StopContainer(instance)
	c.Assert(err, gc.IsNil)

	name := string(instance.Id())
	// Check that the container dir is no longer in the container dir
	c.Assert(filepath.Join(s.ContainerDir, name), jc.DoesNotExist)
	// but instead, in the removed container dir
	c.Assert(filepath.Join(s.RemovedDir, name), jc.IsDirectory)
}

func (s *LxcSuite) TestStopContainerNameClash(c *gc.C) {
	manager := lxc.NewContainerManager(container.ManagerConfig{})
	instance := containertesting.StartContainer(c, manager, "1/lxc/0")

	name := string(instance.Id())
	targetDir := filepath.Join(s.RemovedDir, name)
	err := os.MkdirAll(targetDir, 0755)
	c.Assert(err, gc.IsNil)

	err = manager.StopContainer(instance)
	c.Assert(err, gc.IsNil)

	// Check that the container dir is no longer in the container dir
	c.Assert(filepath.Join(s.ContainerDir, name), jc.DoesNotExist)
	// but instead, in the removed container dir with a ".1" suffix as there was already a directory there.
	c.Assert(filepath.Join(s.RemovedDir, fmt.Sprintf("%s.1", name)), jc.IsDirectory)
}

func (s *LxcSuite) TestNamedManagerPrefix(c *gc.C) {
	manager := lxc.NewContainerManager(container.ManagerConfig{Name: "eric"})
	instance := containertesting.StartContainer(c, manager, "1/lxc/0")
	c.Assert(string(instance.Id()), gc.Equals, "eric-machine-1-lxc-0")
}

func (s *LxcSuite) TestListContainers(c *gc.C) {
	foo := lxc.NewContainerManager(container.ManagerConfig{Name: "foo"})
	bar := lxc.NewContainerManager(container.ManagerConfig{Name: "bar"})

	foo1 := containertesting.StartContainer(c, foo, "1/lxc/0")
	foo2 := containertesting.StartContainer(c, foo, "1/lxc/1")
	foo3 := containertesting.StartContainer(c, foo, "1/lxc/2")

	bar1 := containertesting.StartContainer(c, bar, "1/lxc/0")
	bar2 := containertesting.StartContainer(c, bar, "1/lxc/1")

	result, err := foo.ListContainers()
	c.Assert(err, gc.IsNil)
	instancetest.MatchInstances(c, result, foo1, foo2, foo3)

	result, err = bar.ListContainers()
	c.Assert(err, gc.IsNil)
	instancetest.MatchInstances(c, result, bar1, bar2)
}

func (s *LxcSuite) TestStartContainerAutostarts(c *gc.C) {
	manager := lxc.NewContainerManager(container.ManagerConfig{})
	instance := containertesting.StartContainer(c, manager, "1/lxc/0")
	autostartLink := lxc.RestartSymlink(string(instance.Id()))
	c.Assert(autostartLink, jc.IsSymlink)
}

func (s *LxcSuite) TestStartContainerNoRestartDir(c *gc.C) {
	err := os.Remove(s.RestartDir)
	c.Assert(err, gc.IsNil)

	manager := lxc.NewContainerManager(container.ManagerConfig{})
	instance := containertesting.StartContainer(c, manager, "1/lxc/0")
	autostartLink := lxc.RestartSymlink(string(instance.Id()))

	config := lxc.NetworkConfigTemplate("foo", "bar")
	expected := `
lxc.network.type = foo
lxc.network.link = bar
lxc.network.flags = up
lxc.start.auto = 1
`
	c.Assert(config, gc.Equals, expected)
	c.Assert(autostartLink, jc.DoesNotExist)
}

func (s *LxcSuite) TestStopContainerRemovesAutostartLink(c *gc.C) {
	manager := lxc.NewContainerManager(container.ManagerConfig{})
	instance := containertesting.StartContainer(c, manager, "1/lxc/0")
	err := manager.StopContainer(instance)
	c.Assert(err, gc.IsNil)
	autostartLink := lxc.RestartSymlink(string(instance.Id()))
	c.Assert(autostartLink, jc.SymlinkDoesNotExist)
}

func (s *LxcSuite) TestStopContainerNoRestartDir(c *gc.C) {
	err := os.Remove(s.RestartDir)
	c.Assert(err, gc.IsNil)

	manager := lxc.NewContainerManager(container.ManagerConfig{})
	instance := containertesting.StartContainer(c, manager, "1/lxc/0")
	err = manager.StopContainer(instance)
	c.Assert(err, gc.IsNil)
}

type NetworkSuite struct {
	testbase.LoggingSuite
}

var _ = gc.Suite(&NetworkSuite{})

func (*NetworkSuite) TestGenerateNetworkConfig(c *gc.C) {
	for _, test := range []struct {
		config *container.NetworkConfig
		net    string
		link   string
	}{{
		config: nil,
		net:    "veth",
		link:   "lxcbr0",
	}, {
		config: lxc.DefaultNetworkConfig(),
		net:    "veth",
		link:   "lxcbr0",
	}, {
		config: container.BridgeNetworkConfig("foo"),
		net:    "veth",
		link:   "foo",
	}, {
		config: container.PhysicalNetworkConfig("foo"),
		net:    "phys",
		link:   "foo",
	}} {
		config := lxc.GenerateNetworkConfig(test.config)
		c.Assert(config, jc.Contains, fmt.Sprintf("lxc.network.type = %s\n", test.net))
		c.Assert(config, jc.Contains, fmt.Sprintf("lxc.network.link = %s\n", test.link))
	}
}

func (*NetworkSuite) TestNetworkConfigTemplate(c *gc.C) {
	config := lxc.NetworkConfigTemplate("foo", "bar")
	//In the past, the entire lxc.conf file was just networking. With the addition
	//of the auto start, we now have to have better isolate this test. As such, we
	//parse the conf template results and just get the results that start with
	//'lxc.network' as that is what the test cares about.
	obtained := []string{}
	for _, value := range strings.Split(config, "\n") {
		if strings.HasPrefix(value, "lxc.network") {
			obtained = append(obtained, value)
		}
	}
	expected := []string{
		"lxc.network.type = foo",
		"lxc.network.link = bar",
		"lxc.network.flags = up",
	}
	c.Assert(obtained, gc.DeepEquals, expected)
}
<commit_msg>Nothing special for container proxies.<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package lxc_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	stdtesting "testing"

	"github.com/loggo/loggo"
	gc "launchpad.net/gocheck"
	"launchpad.net/golxc"
	"launchpad.net/goyaml"

	"launchpad.net/juju-core/container"
	"launchpad.net/juju-core/container/lxc"
	lxctesting "launchpad.net/juju-core/container/lxc/testing"
	containertesting "launchpad.net/juju-core/container/testing"
	instancetest "launchpad.net/juju-core/instance/testing"
	jc "launchpad.net/juju-core/testing/checkers"
	"launchpad.net/juju-core/testing/testbase"
)

func Test(t *stdtesting.T) {
	gc.TestingT(t)
}

type LxcSuite struct {
	lxctesting.TestSuite
}

var _ = gc.Suite(&LxcSuite{})

func (s *LxcSuite) SetUpTest(c *gc.C) {
	s.TestSuite.SetUpTest(c)
	loggo.GetLogger("juju.container.lxc").SetLogLevel(loggo.TRACE)
}

func (s *LxcSuite) TestStartContainer(c *gc.C) {
	manager := lxc.NewContainerManager(container.ManagerConfig{})
	instance := containertesting.StartContainer(c, manager, "1/lxc/0")

	name := string(instance.Id())
	// Check our container config files.
	lxcConfContents, err := ioutil.ReadFile(filepath.Join(s.ContainerDir, name, "lxc.conf"))
	c.Assert(err, gc.IsNil)
	c.Assert(string(lxcConfContents), jc.Contains, "lxc.network.link = nic42")

	cloudInitFilename := filepath.Join(s.ContainerDir, name, "cloud-init")
	data := containertesting.AssertCloudInit(c, cloudInitFilename)

	x := make(map[interface{}]interface{})
	err = goyaml.Unmarshal(data, &x)
	c.Assert(err, gc.IsNil)

	var scripts []string
	for _, s := range x["runcmd"].([]interface{}) {
		scripts = append(scripts, s.(string))
	}

	c.Assert(scripts[len(scripts)-2:], gc.DeepEquals, []string{
		"start jujud-machine-1-lxc-0",
		"ifconfig",
	})

	// Check the mount point has been created inside the container.
	c.Assert(filepath.Join(s.LxcDir, name, "rootfs/var/log/juju"), jc.IsDirectory)
	// Check that the config file is linked in the restart dir.
	expectedLinkLocation := filepath.Join(s.RestartDir, name+".conf")
	expectedTarget := filepath.Join(s.LxcDir, name, "config")
	linkInfo, err := os.Lstat(expectedLinkLocation)
	c.Assert(err, gc.IsNil)
	c.Assert(linkInfo.Mode()&os.ModeSymlink, gc.Equals, os.ModeSymlink)

	location, err := os.Readlink(expectedLinkLocation)
	c.Assert(err, gc.IsNil)
	c.Assert(location, gc.Equals, expectedTarget)
}

func (s *LxcSuite) TestContainerState(c *gc.C) {
	manager := lxc.NewContainerManager(container.ManagerConfig{})
	instance := containertesting.StartContainer(c, manager, "1/lxc/0")

	// The mock container will be immediately "running".
	c.Assert(instance.Status(), gc.Equals, string(golxc.StateRunning))

	// StopContainer stops and then destroys the container, putting it
	// into "unknown" state.
	err := manager.StopContainer(instance)
	c.Assert(err, gc.IsNil)
	c.Assert(instance.Status(), gc.Equals, string(golxc.StateUnknown))
}

func (s *LxcSuite) TestStopContainer(c *gc.C) {
	manager := lxc.NewContainerManager(container.ManagerConfig{})
	instance := containertesting.StartContainer(c, manager, "1/lxc/0")

	err := manager.StopContainer(instance)
	c.Assert(err, gc.IsNil)

	name := string(instance.Id())
	// Check that the container dir is no longer in the container dir
	c.Assert(filepath.Join(s.ContainerDir, name), jc.DoesNotExist)
	// but instead, in the removed container dir
	c.Assert(filepath.Join(s.RemovedDir, name), jc.IsDirectory)
}

func (s *LxcSuite) TestStopContainerNameClash(c *gc.C) {
	manager := lxc.NewContainerManager(container.ManagerConfig{})
	instance := containertesting.StartContainer(c, manager, "1/lxc/0")

	name := string(instance.Id())
	targetDir := filepath.Join(s.RemovedDir, name)
	err := os.MkdirAll(targetDir, 0755)
	c.Assert(err, gc.IsNil)

	err = manager.StopContainer(instance)
	c.Assert(err, gc.IsNil)

	// Check that the container dir is no longer in the container dir
	c.Assert(filepath.Join(s.ContainerDir, name), jc.DoesNotExist)
	// but instead, in the removed container dir with a ".1" suffix as there was already a directory there.
	c.Assert(filepath.Join(s.RemovedDir, fmt.Sprintf("%s.1", name)), jc.IsDirectory)
}

func (s *LxcSuite) TestNamedManagerPrefix(c *gc.C) {
	manager := lxc.NewContainerManager(container.ManagerConfig{Name: "eric"})
	instance := containertesting.StartContainer(c, manager, "1/lxc/0")
	c.Assert(string(instance.Id()), gc.Equals, "eric-machine-1-lxc-0")
}

func (s *LxcSuite) TestListContainers(c *gc.C) {
	foo := lxc.NewContainerManager(container.ManagerConfig{Name: "foo"})
	bar := lxc.NewContainerManager(container.ManagerConfig{Name: "bar"})

	foo1 := containertesting.StartContainer(c, foo, "1/lxc/0")
	foo2 := containertesting.StartContainer(c, foo, "1/lxc/1")
	foo3 := containertesting.StartContainer(c, foo, "1/lxc/2")

	bar1 := containertesting.StartContainer(c, bar, "1/lxc/0")
	bar2 := containertesting.StartContainer(c, bar, "1/lxc/1")

	result, err := foo.ListContainers()
	c.Assert(err, gc.IsNil)
	instancetest.MatchInstances(c, result, foo1, foo2, foo3)

	result, err = bar.ListContainers()
	c.Assert(err, gc.IsNil)
	instancetest.MatchInstances(c, result, bar1, bar2)
}

func (s *LxcSuite) TestStartContainerAutostarts(c *gc.C) {
	manager := lxc.NewContainerManager(container.ManagerConfig{})
	instance := containertesting.StartContainer(c, manager, "1/lxc/0")
	autostartLink := lxc.RestartSymlink(string(instance.Id()))
	c.Assert(autostartLink, jc.IsSymlink)
}

func (s *LxcSuite) TestStartContainerNoRestartDir(c *gc.C) {
	err := os.Remove(s.RestartDir)
	c.Assert(err, gc.IsNil)

	manager := lxc.NewContainerManager(container.ManagerConfig{})
	instance := containertesting.StartContainer(c, manager, "1/lxc/0")
	autostartLink := lxc.RestartSymlink(string(instance.Id()))

	config := lxc.NetworkConfigTemplate("foo", "bar")
	expected := `
lxc.network.type = foo
lxc.network.link = bar
lxc.network.flags = up
lxc.start.auto = 1
`
	c.Assert(config, gc.Equals, expected)
	c.Assert(autostartLink, jc.DoesNotExist)
}

func (s *LxcSuite) TestStopContainerRemovesAutostartLink(c *gc.C) {
	manager := lxc.NewContainerManager(container.ManagerConfig{})
	instance := containertesting.StartContainer(c, manager, "1/lxc/0")
	err := manager.StopContainer(instance)
	c.Assert(err, gc.IsNil)
	autostartLink := lxc.RestartSymlink(string(instance.Id()))
	c.Assert(autostartLink, jc.SymlinkDoesNotExist)
}

func (s *LxcSuite) TestStopContainerNoRestartDir(c *gc.C) {
	err := os.Remove(s.RestartDir)
	c.Assert(err, gc.IsNil)

	manager := lxc.NewContainerManager(container.ManagerConfig{})
	instance := containertesting.StartContainer(c, manager, "1/lxc/0")
	err = manager.StopContainer(instance)
	c.Assert(err, gc.IsNil)
}

type NetworkSuite struct {
	testbase.LoggingSuite
}

var _ = gc.Suite(&NetworkSuite{})

func (*NetworkSuite) TestGenerateNetworkConfig(c *gc.C) {
	for _, test := range []struct {
		config *container.NetworkConfig
		net    string
		link   string
	}{{
		config: nil,
		net:    "veth",
		link:   "lxcbr0",
	}, {
		config: lxc.DefaultNetworkConfig(),
		net:    "veth",
		link:   "lxcbr0",
	}, {
		config: container.BridgeNetworkConfig("foo"),
		net:    "veth",
		link:   "foo",
	}, {
		config: container.PhysicalNetworkConfig("foo"),
		net:    "phys",
		link:   "foo",
	}} {
		config := lxc.GenerateNetworkConfig(test.config)
		c.Assert(config, jc.Contains, fmt.Sprintf("lxc.network.type = %s\n", test.net))
		c.Assert(config, jc.Contains, fmt.Sprintf("lxc.network.link = %s\n", test.link))
	}
}

func (*NetworkSuite) TestNetworkConfigTemplate(c *gc.C) {
	config := lxc.NetworkConfigTemplate("foo", "bar")
	//In the past, the entire lxc.conf file was just networking. With the addition
	//of the auto start, we now have to have better isolate this test. As such, we
	//parse the conf template results and just get the results that start with
	//'lxc.network' as that is what the test cares about.
	obtained := []string{}
	for _, value := range strings.Split(config, "\n") {
		if strings.HasPrefix(value, "lxc.network") {
			obtained = append(obtained, value)
		}
	}
	expected := []string{
		"lxc.network.type = foo",
		"lxc.network.link = bar",
		"lxc.network.flags = up",
	}
	c.Assert(obtained, gc.DeepEquals, expected)
}
<|endoftext|>


<commit_before>package dragonfly_test

import (
	//"fmt"
	"github.com/denniscollective/dragonfly.go/dragonfly"
	"testing"
)

const stubB64Job string = "W1siZmYiLCIvVXNlcnMvZGVubmlzL3dvcmtzcGFjZS96aXZpdHkvcHVibGljL2NvbnRlbnQvcGhvdG9zZXRzL29yaWdpbmFsc19hcmNoaXZlLzAwMC8wMDAwMDAvMDAwMDAwMDA3LzAwMDAwMDAwMjQtaC1vcmlnLmpwZyJdLFsicCIsInRodW1iIiwiMjB4MjAiXV0"

func TestFetch(t *testing.T) {
	job, _ := dragonfly.Decode(stubB64Job)
	file, err := job.Apply()

	if err != nil {
		t.Error("job.Apply failed")
	}

	if len(file.Name()) < 10 {
		t.Error("expected a file Object")
	}

}

func TestDecodeThingThatNeedsTwoEquals(t *testing.T) {
	jobstr := "W1siZmYiLCIvVXNlcnMvZGVubmlzL3dvcmtzcGFjZS96aXZpdHkvcHVibGljL2ltYWdlcy9pY29ucy9kZWZhdWx0XzI1Ni5qcGciXSxbInAiLCJ0aHVtYiIsIjgweDgwIyJdXQ"
	job, err := dragonfly.Decode(jobstr)

	if err != nil {
		t.Errorf("Deconde job got error %s", err)
	}

	if len(job.Steps) != 2 {
		t.Error("job should have two steps")
	}

}

func TestDecodeDragonfly(t *testing.T) {
	job, err := dragonfly.Decode(stubB64Job)

	if err != nil {
		t.Errorf("Deconde job got error %s", err)
	}

	if len(job.Steps) != 2 {
		t.Error("job should have two steps")
	}

	if job.Steps[0].Command != "ff" {
		t.Error("the first test of the stub is supposed to be fetch File")
	}

	if args := job.Steps[1].Args; args[0] != "thumb" && args[1] != "20x20" {
		t.Error("second step should be a resize to thumbnail 20x20 job")
	}
}

func TestDecodeFailse(t *testing.T) {
	t.Skip()
	job, err := dragonfly.Decode("this is y i'm hawt")
	if err == nil {
		t.Error("Decode errors aren't propagating")
	}

	if job != nil {
		t.Error("Decode should return nil when it has an error")
	}
}
<commit_msg>refactor test<commit_after>package dragonfly_test

import (
	//"fmt"
	"github.com/denniscollective/dragonfly.go/dragonfly"
	"testing"
)

const stubB64Job string = "W1siZmYiLCIvVXNlcnMvZGVubmlzL3dvcmtzcGFjZS96aXZpdHkvcHVibGljL2NvbnRlbnQvcGhvdG9zZXRzL29yaWdpbmFsc19hcmNoaXZlLzAwMC8wMDAwMDAvMDAwMDAwMDA3LzAwMDAwMDAwMjQtaC1vcmlnLmpwZyJdLFsicCIsInRodW1iIiwiMjB4MjAiXV0"

func TestFetch(t *testing.T) {
	file, err := dragonfly.ImageFor(stubB64Job)

	if err != nil {
		t.Error("ImgFor(stub) failed")
	}

	if len(file.Name()) < 10 {
		t.Error("expected a file Object")
	}

}

func TestDecodeThingThatNeedsTwoEquals(t *testing.T) {
	jobstr := "W1siZmYiLCIvVXNlcnMvZGVubmlzL3dvcmtzcGFjZS96aXZpdHkvcHVibGljL2ltYWdlcy9pY29ucy9kZWZhdWx0XzI1Ni5qcGciXSxbInAiLCJ0aHVtYiIsIjgweDgwIyJdXQ"
	job, err := dragonfly.Decode(jobstr)

	if err != nil {
		t.Errorf("Deconde job got error %s", err)
	}

	if len(job.Steps) != 2 {
		t.Error("job should have two steps")
	}

}

func TestDecodeDragonfly(t *testing.T) {
	job, err := dragonfly.Decode(stubB64Job)

	if err != nil {
		t.Errorf("Deconde job got error %s", err)
	}

	if len(job.Steps) != 2 {
		t.Error("job should have two steps")
	}

	if job.Steps[0].Command != "ff" {
		t.Error("the first test of the stub is supposed to be fetch File")
	}

	if args := job.Steps[1].Args; args[0] != "thumb" && args[1] != "20x20" {
		t.Error("second step should be a resize to thumbnail 20x20 job")
	}
}

func TestDecodeFailse(t *testing.T) {
	t.Skip()
	job, err := dragonfly.Decode("this is y i'm hawt")
	if err == nil {
		t.Error("Decode errors aren't propagating")
	}

	if job != nil {
		t.Error("Decode should return nil when it has an error")
	}
}
<|endoftext|>


<commit_before>package dos

import (
	"os"
	"syscall"
)

func GetFileAttributesFromFileInfo(status os.FileInfo) uint32 {
	return status.Sys().(*syscall.Win32FileAttributeData).FileAttributes
}

func GetFileAttributes(path string) (uint32, error) {
	cpath, cpathErr := syscall.UTF16PtrFromString(path)
	if cpathErr != nil {
		return 0, cpathErr
	}
	return syscall.GetFileAttributes(cpath)
}

func SetFileAttributes(path string, attr uint32) error {
	cpath, cpathErr := syscall.UTF16PtrFromString(path)
	if cpathErr != nil {
		return cpathErr
	}
	return syscall.SetFileAttributes(cpath, attr)
}
<commit_msg>dos.GetFileAttributesFromFileInfo can read *FindFiles<commit_after>package dos

import (
	"os"
	"syscall"
)

func GetFileAttributesFromFileInfo(status os.FileInfo) uint32 {
	if it, ok := status.Sys().(*syscall.Win32FileAttributeData); ok {
		return it.FileAttributes
	} else if it, ok := status.(*FindFiles); ok {
		return it.data1.FileAttributes
	} else {
		panic("Can not get fileatttribute")
	}
}

func GetFileAttributes(path string) (uint32, error) {
	cpath, cpathErr := syscall.UTF16PtrFromString(path)
	if cpathErr != nil {
		return 0, cpathErr
	}
	return syscall.GetFileAttributes(cpath)
}

func SetFileAttributes(path string, attr uint32) error {
	cpath, cpathErr := syscall.UTF16PtrFromString(path)
	if cpathErr != nil {
		return cpathErr
	}
	return syscall.SetFileAttributes(cpath, attr)
}
<|endoftext|>


<commit_before>package docker

import (
	"archive/tar"
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"github.com/dotcloud/docker/utils"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
)

type Archive io.Reader

type Compression uint32

const (
	Uncompressed Compression = iota
	Bzip2
	Gzip
	Xz
)

func DetectCompression(source []byte) Compression {
	for _, c := range source[:10] {
		utils.Debugf("%x", c)
	}

	sourceLen := len(source)
	for compression, m := range map[Compression][]byte{
		Bzip2: {0x42, 0x5A, 0x68},
		Gzip:  {0x1F, 0x8B, 0x08},
		Xz:    {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
	} {
		fail := false
		if len(m) > sourceLen {
			utils.Debugf("Len too short")
			continue
		}
		i := 0
		for _, b := range m {
			if b != source[i] {
				fail = true
				break
			}
			i++
		}
		if !fail {
			return compression
		}
	}
	return Uncompressed
}

func (compression *Compression) Flag() string {
	switch *compression {
	case Bzip2:
		return "j"
	case Gzip:
		return "z"
	case Xz:
		return "J"
	}
	return ""
}

func (compression *Compression) Extension() string {
	switch *compression {
	case Uncompressed:
		return "tar"
	case Bzip2:
		return "tar.bz2"
	case Gzip:
		return "tar.gz"
	case Xz:
		return "tar.xz"
	}
	return ""
}

// Tar creates an archive from the directory at `path`, and returns it as a
// stream of bytes.
func Tar(path string, compression Compression) (io.Reader, error) {
	return TarFilter(path, compression, nil)
}

// Tar creates an archive from the directory at `path`, only including files whose relative
// paths are included in `filter`. If `filter` is nil, then all files are included.
func TarFilter(path string, compression Compression, filter []string) (io.Reader, error) {
	args := []string{"tar", "--numeric-owner", "-f", "-", "-C", path}
	if filter == nil {
		filter = []string{"."}
	}
	for _, f := range filter {
		args = append(args, "-c"+compression.Flag(), f)
	}
	return CmdStream(exec.Command(args[0], args[1:]...))
}

// Untar reads a stream of bytes from `archive`, parses it as a tar archive,
// and unpacks it into the directory at `path`.
// The archive may be compressed with one of the following algorithgms:
//  identity (uncompressed), gzip, bzip2, xz.
// FIXME: specify behavior when target path exists vs. doesn't exist.
func Untar(archive io.Reader, path string) error {
	if archive == nil {
		return fmt.Errorf("Empty archive")
	}
	bufferedArchive := bufio.NewReaderSize(archive, 10)
	buf, err := bufferedArchive.Peek(10)
	if err != nil {
		return err
	}
	compression := DetectCompression(buf)

	utils.Debugf("Archive compression detected: %s", compression.Extension())

	cmd := exec.Command("tar", "--numeric-owner", "-f", "-", "-C", path, "-x"+compression.Flag())
	cmd.Stdin = bufferedArchive
	// Hardcode locale environment for predictable outcome regardless of host configuration.
	//   (see https://github.com/dotcloud/docker/issues/355)
	cmd.Env = []string{"LANG=en_US.utf-8", "LC_ALL=en_US.utf-8"}
	output, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("%s: %s", err, output)
	}
	return nil
}

// TarUntar is a convenience function which calls Tar and Untar, with
// the output of one piped into the other. If either Tar or Untar fails,
// TarUntar aborts and returns the error.
func TarUntar(src string, filter []string, dst string) error {
	utils.Debugf("TarUntar(%s %s %s)", src, filter, dst)
	archive, err := TarFilter(src, Uncompressed, filter)
	if err != nil {
		return err
	}
	return Untar(archive, dst)
}

// UntarPath is a convenience function which looks for an archive
// at filesystem path `src`, and unpacks it at `dst`.
func UntarPath(src, dst string) error {
	if archive, err := os.Open(src); err != nil {
		return err
	} else if err := Untar(archive, dst); err != nil {
		return err
	}
	return nil
}

// CopyWithTar creates a tar archive of filesystem path `src`, and
// unpacks it at filesystem path `dst`.
// The archive is streamed directly with fixed buffering and no
// intermediary disk IO.
//
func CopyWithTar(src, dst string) error {
	srcSt, err := os.Stat(src)
	if err != nil {
		return err
	}
	if !srcSt.IsDir() {
		return CopyFileWithTar(src, dst)
	}
	// Create dst, copy src's content into it
	utils.Debugf("Creating dest directory: %s", dst)
	if err := os.MkdirAll(dst, 0700); err != nil && !os.IsExist(err) {
		return err
	}
	utils.Debugf("Calling TarUntar(%s, %s)", src, dst)
	return TarUntar(src, nil, dst)
}

// CopyFileWithTar emulates the behavior of the 'cp' command-line
// for a single file. It copies a regular file from path `src` to
// path `dst`, and preserves all its metadata.
//
// If `dst` ends with a trailing slash '/', the final destination path
// will be `dst/base(src)`.
func CopyFileWithTar(src, dst string) error {
	utils.Debugf("CopyFileWithTar(%s, %s)", src, dst)
	srcSt, err := os.Stat(src)
	if err != nil {
		return err
	}
	if srcSt.IsDir() {
		return fmt.Errorf("Can't copy a directory")
	}
	// Clean up the trailing /
	if dst[len(dst)-1] == '/' {
		dst = path.Join(dst, filepath.Base(src))
	}
	// Create the holding directory if necessary
	if err := os.MkdirAll(filepath.Dir(dst), 0700); err != nil && !os.IsExist(err) {
		return err
	}
	buf := new(bytes.Buffer)
	tw := tar.NewWriter(buf)
	hdr, err := tar.FileInfoHeader(srcSt, "")
	if err != nil {
		return err
	}
	hdr.Name = filepath.Base(dst)
	if err := tw.WriteHeader(hdr); err != nil {
		return err
	}
	srcF, err := os.Open(src)
	if err != nil {
		return err
	}
	if _, err := io.Copy(tw, srcF); err != nil {
		return err
	}
	tw.Close()
	return Untar(buf, filepath.Dir(dst))
}

// CmdStream executes a command, and returns its stdout as a stream.
// If the command fails to run or doesn't complete successfully, an error
// will be returned, including anything written on stderr.
func CmdStream(cmd *exec.Cmd) (io.Reader, error) {
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return nil, err
	}
	pipeR, pipeW := io.Pipe()
	errChan := make(chan []byte)
	// Collect stderr, we will use it in case of an error
	go func() {
		errText, e := ioutil.ReadAll(stderr)
		if e != nil {
			errText = []byte("(...couldn't fetch stderr: " + e.Error() + ")")
		}
		errChan <- errText
	}()
	// Copy stdout to the returned pipe
	go func() {
		_, err := io.Copy(pipeW, stdout)
		if err != nil {
			pipeW.CloseWithError(err)
		}
		errText := <-errChan
		if err := cmd.Wait(); err != nil {
			pipeW.CloseWithError(errors.New(err.Error() + ": " + string(errText)))
		} else {
			pipeW.Close()
		}
	}()
	// Run the command and return the pipe
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	return pipeR, nil
}

// NewTempArchive reads the content of src into a temporary file, and returns the contents
// of that file as an archive. The archive can only be read once - as soon as reading completes,
// the file will be deleted.
func NewTempArchive(src Archive, dir string) (*TempArchive, error) {
	f, err := ioutil.TempFile(dir, "")
	if err != nil {
		return nil, err
	}
	if _, err := io.Copy(f, src); err != nil {
		return nil, err
	}
	if _, err := f.Seek(0, 0); err != nil {
		return nil, err
	}
	st, err := f.Stat()
	if err != nil {
		return nil, err
	}
	size := st.Size()
	return &TempArchive{f, size}, nil
}

type TempArchive struct {
	*os.File
	Size int64 // Pre-computed from Stat().Size() as a convenience
}

func (archive *TempArchive) Read(data []byte) (int, error) {
	n, err := archive.File.Read(data)
	if err != nil {
		os.Remove(archive.File.Name())
	}
	return n, err
}
<commit_msg>Use fmt.Errorf when appropriate.<commit_after>package docker

import (
	"archive/tar"
	"bufio"
	"bytes"
	"fmt"
	"github.com/dotcloud/docker/utils"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
)

type Archive io.Reader

type Compression uint32

const (
	Uncompressed Compression = iota
	Bzip2
	Gzip
	Xz
)

func DetectCompression(source []byte) Compression {
	for _, c := range source[:10] {
		utils.Debugf("%x", c)
	}

	sourceLen := len(source)
	for compression, m := range map[Compression][]byte{
		Bzip2: {0x42, 0x5A, 0x68},
		Gzip:  {0x1F, 0x8B, 0x08},
		Xz:    {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
	} {
		fail := false
		if len(m) > sourceLen {
			utils.Debugf("Len too short")
			continue
		}
		i := 0
		for _, b := range m {
			if b != source[i] {
				fail = true
				break
			}
			i++
		}
		if !fail {
			return compression
		}
	}
	return Uncompressed
}

func (compression *Compression) Flag() string {
	switch *compression {
	case Bzip2:
		return "j"
	case Gzip:
		return "z"
	case Xz:
		return "J"
	}
	return ""
}

func (compression *Compression) Extension() string {
	switch *compression {
	case Uncompressed:
		return "tar"
	case Bzip2:
		return "tar.bz2"
	case Gzip:
		return "tar.gz"
	case Xz:
		return "tar.xz"
	}
	return ""
}

// Tar creates an archive from the directory at `path`, and returns it as a
// stream of bytes.
func Tar(path string, compression Compression) (io.Reader, error) {
	return TarFilter(path, compression, nil)
}

// Tar creates an archive from the directory at `path`, only including files whose relative
// paths are included in `filter`. If `filter` is nil, then all files are included.
func TarFilter(path string, compression Compression, filter []string) (io.Reader, error) {
	args := []string{"tar", "--numeric-owner", "-f", "-", "-C", path}
	if filter == nil {
		filter = []string{"."}
	}
	for _, f := range filter {
		args = append(args, "-c"+compression.Flag(), f)
	}
	return CmdStream(exec.Command(args[0], args[1:]...))
}

// Untar reads a stream of bytes from `archive`, parses it as a tar archive,
// and unpacks it into the directory at `path`.
// The archive may be compressed with one of the following algorithgms:
//  identity (uncompressed), gzip, bzip2, xz.
// FIXME: specify behavior when target path exists vs. doesn't exist.
func Untar(archive io.Reader, path string) error {
	if archive == nil {
		return fmt.Errorf("Empty archive")
	}
	bufferedArchive := bufio.NewReaderSize(archive, 10)
	buf, err := bufferedArchive.Peek(10)
	if err != nil {
		return err
	}
	compression := DetectCompression(buf)

	utils.Debugf("Archive compression detected: %s", compression.Extension())

	cmd := exec.Command("tar", "--numeric-owner", "-f", "-", "-C", path, "-x"+compression.Flag())
	cmd.Stdin = bufferedArchive
	// Hardcode locale environment for predictable outcome regardless of host configuration.
	//   (see https://github.com/dotcloud/docker/issues/355)
	cmd.Env = []string{"LANG=en_US.utf-8", "LC_ALL=en_US.utf-8"}
	output, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("%s: %s", err, output)
	}
	return nil
}

// TarUntar is a convenience function which calls Tar and Untar, with
// the output of one piped into the other. If either Tar or Untar fails,
// TarUntar aborts and returns the error.
func TarUntar(src string, filter []string, dst string) error {
	utils.Debugf("TarUntar(%s %s %s)", src, filter, dst)
	archive, err := TarFilter(src, Uncompressed, filter)
	if err != nil {
		return err
	}
	return Untar(archive, dst)
}

// UntarPath is a convenience function which looks for an archive
// at filesystem path `src`, and unpacks it at `dst`.
func UntarPath(src, dst string) error {
	if archive, err := os.Open(src); err != nil {
		return err
	} else if err := Untar(archive, dst); err != nil {
		return err
	}
	return nil
}

// CopyWithTar creates a tar archive of filesystem path `src`, and
// unpacks it at filesystem path `dst`.
// The archive is streamed directly with fixed buffering and no
// intermediary disk IO.
//
func CopyWithTar(src, dst string) error {
	srcSt, err := os.Stat(src)
	if err != nil {
		return err
	}
	if !srcSt.IsDir() {
		return CopyFileWithTar(src, dst)
	}
	// Create dst, copy src's content into it
	utils.Debugf("Creating dest directory: %s", dst)
	if err := os.MkdirAll(dst, 0700); err != nil && !os.IsExist(err) {
		return err
	}
	utils.Debugf("Calling TarUntar(%s, %s)", src, dst)
	return TarUntar(src, nil, dst)
}

// CopyFileWithTar emulates the behavior of the 'cp' command-line
// for a single file. It copies a regular file from path `src` to
// path `dst`, and preserves all its metadata.
//
// If `dst` ends with a trailing slash '/', the final destination path
// will be `dst/base(src)`.
func CopyFileWithTar(src, dst string) error {
	utils.Debugf("CopyFileWithTar(%s, %s)", src, dst)
	srcSt, err := os.Stat(src)
	if err != nil {
		return err
	}
	if srcSt.IsDir() {
		return fmt.Errorf("Can't copy a directory")
	}
	// Clean up the trailing /
	if dst[len(dst)-1] == '/' {
		dst = path.Join(dst, filepath.Base(src))
	}
	// Create the holding directory if necessary
	if err := os.MkdirAll(filepath.Dir(dst), 0700); err != nil && !os.IsExist(err) {
		return err
	}
	buf := new(bytes.Buffer)
	tw := tar.NewWriter(buf)
	hdr, err := tar.FileInfoHeader(srcSt, "")
	if err != nil {
		return err
	}
	hdr.Name = filepath.Base(dst)
	if err := tw.WriteHeader(hdr); err != nil {
		return err
	}
	srcF, err := os.Open(src)
	if err != nil {
		return err
	}
	if _, err := io.Copy(tw, srcF); err != nil {
		return err
	}
	tw.Close()
	return Untar(buf, filepath.Dir(dst))
}

// CmdStream executes a command, and returns its stdout as a stream.
// If the command fails to run or doesn't complete successfully, an error
// will be returned, including anything written on stderr.
func CmdStream(cmd *exec.Cmd) (io.Reader, error) {
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return nil, err
	}
	pipeR, pipeW := io.Pipe()
	errChan := make(chan []byte)
	// Collect stderr, we will use it in case of an error
	go func() {
		errText, e := ioutil.ReadAll(stderr)
		if e != nil {
			errText = []byte("(...couldn't fetch stderr: " + e.Error() + ")")
		}
		errChan <- errText
	}()
	// Copy stdout to the returned pipe
	go func() {
		_, err := io.Copy(pipeW, stdout)
		if err != nil {
			pipeW.CloseWithError(err)
		}
		errText := <-errChan
		if err := cmd.Wait(); err != nil {
			pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errText))
		} else {
			pipeW.Close()
		}
	}()
	// Run the command and return the pipe
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	return pipeR, nil
}

// NewTempArchive reads the content of src into a temporary file, and returns the contents
// of that file as an archive. The archive can only be read once - as soon as reading completes,
// the file will be deleted.
func NewTempArchive(src Archive, dir string) (*TempArchive, error) {
	f, err := ioutil.TempFile(dir, "")
	if err != nil {
		return nil, err
	}
	if _, err := io.Copy(f, src); err != nil {
		return nil, err
	}
	if _, err := f.Seek(0, 0); err != nil {
		return nil, err
	}
	st, err := f.Stat()
	if err != nil {
		return nil, err
	}
	size := st.Size()
	return &TempArchive{f, size}, nil
}

type TempArchive struct {
	*os.File
	Size int64 // Pre-computed from Stat().Size() as a convenience
}

func (archive *TempArchive) Read(data []byte) (int, error) {
	n, err := archive.File.Read(data)
	if err != nil {
		os.Remove(archive.File.Name())
	}
	return n, err
}
<|endoftext|>


<commit_before>package ipam

import (
	"context"
	"fmt"
	"net"
	"sync"
	"testing"

	"github.com/docker/libnetwork/ipamapi"
	"github.com/stretchr/testify/assert"
	"golang.org/x/sync/semaphore"
)

const (
	all = iota
	even
	odd
)

type releaseMode uint

type testContext struct {
	a      *Allocator
	opts   map[string]string
	ipList []*net.IPNet
	ipMap  map[string]bool
	pid    string
	maxIP  int
}

func newTestContext(t *testing.T, mask int, options map[string]string) *testContext {
	a, err := getAllocator(true)
	if err != nil {
		t.Fatal(err)
	}
	a.addrSpaces["giallo"] = &addrSpace{
		id:      dsConfigKey + "/" + "giallo",
		ds:      a.addrSpaces[localAddressSpace].ds,
		alloc:   a.addrSpaces[localAddressSpace].alloc,
		scope:   a.addrSpaces[localAddressSpace].scope,
		subnets: map[SubnetKey]*PoolData{},
	}

	network := fmt.Sprintf("192.168.100.0/%d", mask)
	// total ips 2^(32-mask) - 2 (network and broadcast)
	totalIps := 1<<uint(32-mask) - 2

	pid, _, _, err := a.RequestPool("giallo", network, "", nil, false)
	if err != nil {
		t.Fatal(err)
	}

	return &testContext{
		a:      a,
		opts:   options,
		ipList: make([]*net.IPNet, 0, totalIps),
		ipMap:  make(map[string]bool),
		pid:    pid,
		maxIP:  totalIps,
	}
}

func TestDebug(t *testing.T) {
	tctx := newTestContext(t, 23, map[string]string{ipamapi.AllocSerialPrefix: "true"})
	tctx.a.RequestAddress(tctx.pid, nil, map[string]string{ipamapi.AllocSerialPrefix: "true"})
	tctx.a.RequestAddress(tctx.pid, nil, map[string]string{ipamapi.AllocSerialPrefix: "true"})
}

func TestFullAllocateRelease(t *testing.T) {
	for _, parallelism := range []int64{2, 4, 8} {
		for _, mask := range []int{29, 25, 24, 21} {
			tctx := newTestContext(t, mask, map[string]string{ipamapi.AllocSerialPrefix: "true"})
			allocate(t, tctx, parallelism)
			release(t, tctx, all, parallelism)
		}
	}
}

func TestOddAllocateRelease(t *testing.T) {
	for _, parallelism := range []int64{2, 4, 8} {
		for _, mask := range []int{29, 25, 24, 21} {
			tctx := newTestContext(t, mask, map[string]string{ipamapi.AllocSerialPrefix: "true"})
			allocate(t, tctx, parallelism)
			release(t, tctx, odd, parallelism)
		}
	}
}

func TestFullAllocateSerialReleaseParallel(t *testing.T) {
	for _, parallelism := range []int64{1, 4, 8} {
		tctx := newTestContext(t, 23, map[string]string{ipamapi.AllocSerialPrefix: "true"})
		allocate(t, tctx, 1)
		release(t, tctx, all, parallelism)
	}
}

func TestOddAllocateSerialReleaseParallel(t *testing.T) {
	for _, parallelism := range []int64{1, 4, 8} {
		tctx := newTestContext(t, 23, map[string]string{ipamapi.AllocSerialPrefix: "true"})
		allocate(t, tctx, 1)
		release(t, tctx, odd, parallelism)
	}
}

func TestEvenAllocateSerialReleaseParallel(t *testing.T) {
	for _, parallelism := range []int64{1, 4, 8} {
		tctx := newTestContext(t, 23, map[string]string{ipamapi.AllocSerialPrefix: "true"})
		allocate(t, tctx, 1)
		release(t, tctx, even, parallelism)
	}
}

func allocate(t *testing.T, tctx *testContext, parallel int64) {
	// Allocate the whole space
	parallelExec := semaphore.NewWeighted(parallel)
	routineNum := tctx.maxIP + 10
	ch := make(chan *net.IPNet, routineNum)
	var id int
	var wg sync.WaitGroup
	// routine loop
	for {
		wg.Add(1)
		go func(id int) {
			parallelExec.Acquire(context.Background(), 1)
			ip, _, _ := tctx.a.RequestAddress(tctx.pid, nil, tctx.opts)
			ch <- ip
			parallelExec.Release(1)
			wg.Done()
		}(id)
		id++
		if id == routineNum {
			break
		}
	}

	// give time to all the go routines to finish
	wg.Wait()

	// process results
	for i := 0; i < routineNum; i++ {
		ip := <-ch
		if ip == nil {
			continue
		}
		if there, ok := tctx.ipMap[ip.String()]; ok && there {
			t.Fatalf("Got duplicate IP %s", ip.String())
			break
		}
		tctx.ipList = append(tctx.ipList, ip)
		tctx.ipMap[ip.String()] = true
	}

	assert.Len(t, tctx.ipList, tctx.maxIP)
	if len(tctx.ipList) != tctx.maxIP {
		t.Fatal("missmatch number allocation")
	}
}

func release(t *testing.T, tctx *testContext, mode releaseMode, parallel int64) {
	var startIndex, increment, stopIndex, length int
	switch mode {
	case all:
		startIndex = 0
		increment = 1
		stopIndex = tctx.maxIP - 1
		length = tctx.maxIP
	case odd, even:
		if mode == odd {
			startIndex = 1
		}
		increment = 2
		stopIndex = tctx.maxIP - 1
		length = tctx.maxIP / 2
		if tctx.maxIP%2 > 0 {
			length++
		}
	default:
		t.Fatal("unsupported mode yet")
	}

	ipIndex := make([]int, 0, length)
	// calculate the index to release from the ipList
	for i := startIndex; ; i += increment {
		ipIndex = append(ipIndex, i)
		if i+increment > stopIndex {
			break
		}
	}

	var id int
	parallelExec := semaphore.NewWeighted(parallel)
	ch := make(chan *net.IPNet, len(ipIndex))
	wg := sync.WaitGroup{}
	for index := range ipIndex {
		wg.Add(1)
		go func(id, index int) {
			parallelExec.Acquire(context.Background(), 1)
			// logrus.Errorf("index %v", index)
			// logrus.Errorf("list %v", tctx.ipList)
			err := tctx.a.ReleaseAddress(tctx.pid, tctx.ipList[index].IP)
			if err != nil {
				t.Fatalf("routine %d got %v", id, err)
			}
			ch <- tctx.ipList[index]
			parallelExec.Release(1)
			wg.Done()
		}(id, index)
		id++
	}
	wg.Wait()

	for i := 0; i < len(ipIndex); i++ {
		ip := <-ch

		// check if it is really free
		_, _, err := tctx.a.RequestAddress(tctx.pid, ip.IP, nil)
		assert.NoError(t, err, "ip %v not properly released", ip)
		if err != nil {
			t.Fatalf("ip %v not properly released, error:%v", ip, err)
		}
		err = tctx.a.ReleaseAddress(tctx.pid, ip.IP)
		assert.NoError(t, err)

		if there, ok := tctx.ipMap[ip.String()]; !ok || !there {
			t.Fatalf("ip %v got double deallocated", ip)
		}
		tctx.ipMap[ip.String()] = false
		for j, v := range tctx.ipList {
			if v == ip {
				tctx.ipList = append(tctx.ipList[:j], tctx.ipList[j+1:]...)
				break
			}
		}
	}

	assert.Len(t, tctx.ipList, tctx.maxIP-length)
}
<commit_msg>Add test for parallel request pool<commit_after>package ipam

import (
	"context"
	"fmt"
	"math/rand"
	"net"
	"sort"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/docker/libnetwork/ipamapi"
	"github.com/stretchr/testify/assert"
	"golang.org/x/sync/semaphore"
)

const (
	all = iota
	even
	odd
)

type releaseMode uint

type testContext struct {
	a      *Allocator
	opts   map[string]string
	ipList []*net.IPNet
	ipMap  map[string]bool
	pid    string
	maxIP  int
}

func newTestContext(t *testing.T, mask int, options map[string]string) *testContext {
	a, err := getAllocator(false)
	if err != nil {
		t.Fatal(err)
	}
	a.addrSpaces["giallo"] = &addrSpace{
		id:      dsConfigKey + "/" + "giallo",
		ds:      a.addrSpaces[localAddressSpace].ds,
		alloc:   a.addrSpaces[localAddressSpace].alloc,
		scope:   a.addrSpaces[localAddressSpace].scope,
		subnets: map[SubnetKey]*PoolData{},
	}

	network := fmt.Sprintf("192.168.100.0/%d", mask)
	// total ips 2^(32-mask) - 2 (network and broadcast)
	totalIps := 1<<uint(32-mask) - 2

	pid, _, _, err := a.RequestPool("giallo", network, "", nil, false)
	if err != nil {
		t.Fatal(err)
	}

	return &testContext{
		a:      a,
		opts:   options,
		ipList: make([]*net.IPNet, 0, totalIps),
		ipMap:  make(map[string]bool),
		pid:    pid,
		maxIP:  totalIps,
	}
}

func TestDebug(t *testing.T) {
	tctx := newTestContext(t, 23, map[string]string{ipamapi.AllocSerialPrefix: "true"})
	tctx.a.RequestAddress(tctx.pid, nil, map[string]string{ipamapi.AllocSerialPrefix: "true"})
	tctx.a.RequestAddress(tctx.pid, nil, map[string]string{ipamapi.AllocSerialPrefix: "true"})
}

type op struct {
	id   int32
	add  bool
	name string
}

func (o *op) String() string {
	return fmt.Sprintf("%+v", *o)
}

func TestRequestPoolParallel(t *testing.T) {
	a, err := getAllocator(false)
	if err != nil {
		t.Fatal(err)
	}
	var operationIndex int32
	ch := make(chan *op, 240)
	for i := 0; i < 120; i++ {
		go func(t *testing.T, a *Allocator, ch chan *op) {
			name, _, _, err := a.RequestPool("GlobalDefault", "", "", nil, false)
			if err != nil {
				t.Fatalf("request error %v", err)
			}
			idx := atomic.AddInt32(&operationIndex, 1)
			ch <- &op{idx, true, name}
			time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
			idx = atomic.AddInt32(&operationIndex, 1)
			err = a.ReleasePool(name)
			if err != nil {
				t.Fatalf("relase error %v", err)
			}
			ch <- &op{idx, false, name}
		}(t, a, ch)
	}

	// map of events
	m := make(map[string][]*op)
	for i := 0; i < 240; i++ {
		x := <-ch
		ops, ok := m[x.name]
		if !ok {
			ops = make([]*op, 0, 10)
		}
		ops = append(ops, x)
		m[x.name] = ops
	}

	// Post processing to avoid event reordering on the channel
	for pool, ops := range m {
		sort.Slice(ops[:], func(i, j int) bool {
			return ops[i].id < ops[j].id
		})
		expected := true
		for _, op := range ops {
			if op.add != expected {
				t.Fatalf("Operations for %v not valid %v, operations %v", pool, op, ops)
			}
			expected = !expected
		}
	}
}

func TestFullAllocateRelease(t *testing.T) {
	for _, parallelism := range []int64{2, 4, 8} {
		for _, mask := range []int{29, 25, 24, 21} {
			tctx := newTestContext(t, mask, map[string]string{ipamapi.AllocSerialPrefix: "true"})
			allocate(t, tctx, parallelism)
			release(t, tctx, all, parallelism)
		}
	}
}

func TestOddAllocateRelease(t *testing.T) {
	for _, parallelism := range []int64{2, 4, 8} {
		for _, mask := range []int{29, 25, 24, 21} {
			tctx := newTestContext(t, mask, map[string]string{ipamapi.AllocSerialPrefix: "true"})
			allocate(t, tctx, parallelism)
			release(t, tctx, odd, parallelism)
		}
	}
}

func TestFullAllocateSerialReleaseParallel(t *testing.T) {
	for _, parallelism := range []int64{1, 4, 8} {
		tctx := newTestContext(t, 23, map[string]string{ipamapi.AllocSerialPrefix: "true"})
		allocate(t, tctx, 1)
		release(t, tctx, all, parallelism)
	}
}

func TestOddAllocateSerialReleaseParallel(t *testing.T) {
	for _, parallelism := range []int64{1, 4, 8} {
		tctx := newTestContext(t, 23, map[string]string{ipamapi.AllocSerialPrefix: "true"})
		allocate(t, tctx, 1)
		release(t, tctx, odd, parallelism)
	}
}

func TestEvenAllocateSerialReleaseParallel(t *testing.T) {
	for _, parallelism := range []int64{1, 4, 8} {
		tctx := newTestContext(t, 23, map[string]string{ipamapi.AllocSerialPrefix: "true"})
		allocate(t, tctx, 1)
		release(t, tctx, even, parallelism)
	}
}

func allocate(t *testing.T, tctx *testContext, parallel int64) {
	// Allocate the whole space
	parallelExec := semaphore.NewWeighted(parallel)
	routineNum := tctx.maxIP + 10
	ch := make(chan *net.IPNet, routineNum)
	var id int
	var wg sync.WaitGroup
	// routine loop
	for {
		wg.Add(1)
		go func(id int) {
			parallelExec.Acquire(context.Background(), 1)
			ip, _, _ := tctx.a.RequestAddress(tctx.pid, nil, tctx.opts)
			ch <- ip
			parallelExec.Release(1)
			wg.Done()
		}(id)
		id++
		if id == routineNum {
			break
		}
	}

	// give time to all the go routines to finish
	wg.Wait()

	// process results
	for i := 0; i < routineNum; i++ {
		ip := <-ch
		if ip == nil {
			continue
		}
		if there, ok := tctx.ipMap[ip.String()]; ok && there {
			t.Fatalf("Got duplicate IP %s", ip.String())
			break
		}
		tctx.ipList = append(tctx.ipList, ip)
		tctx.ipMap[ip.String()] = true
	}

	assert.Len(t, tctx.ipList, tctx.maxIP)
	if len(tctx.ipList) != tctx.maxIP {
		t.Fatal("missmatch number allocation")
	}
}

func release(t *testing.T, tctx *testContext, mode releaseMode, parallel int64) {
	var startIndex, increment, stopIndex, length int
	switch mode {
	case all:
		startIndex = 0
		increment = 1
		stopIndex = tctx.maxIP - 1
		length = tctx.maxIP
	case odd, even:
		if mode == odd {
			startIndex = 1
		}
		increment = 2
		stopIndex = tctx.maxIP - 1
		length = tctx.maxIP / 2
		if tctx.maxIP%2 > 0 {
			length++
		}
	default:
		t.Fatal("unsupported mode yet")
	}

	ipIndex := make([]int, 0, length)
	// calculate the index to release from the ipList
	for i := startIndex; ; i += increment {
		ipIndex = append(ipIndex, i)
		if i+increment > stopIndex {
			break
		}
	}

	var id int
	parallelExec := semaphore.NewWeighted(parallel)
	ch := make(chan *net.IPNet, len(ipIndex))
	wg := sync.WaitGroup{}
	for index := range ipIndex {
		wg.Add(1)
		go func(id, index int) {
			parallelExec.Acquire(context.Background(), 1)
			// logrus.Errorf("index %v", index)
			// logrus.Errorf("list %v", tctx.ipList)
			err := tctx.a.ReleaseAddress(tctx.pid, tctx.ipList[index].IP)
			if err != nil {
				t.Fatalf("routine %d got %v", id, err)
			}
			ch <- tctx.ipList[index]
			parallelExec.Release(1)
			wg.Done()
		}(id, index)
		id++
	}
	wg.Wait()

	for i := 0; i < len(ipIndex); i++ {
		ip := <-ch

		// check if it is really free
		_, _, err := tctx.a.RequestAddress(tctx.pid, ip.IP, nil)
		assert.NoError(t, err, "ip %v not properly released", ip)
		if err != nil {
			t.Fatalf("ip %v not properly released, error:%v", ip, err)
		}
		err = tctx.a.ReleaseAddress(tctx.pid, ip.IP)
		assert.NoError(t, err)

		if there, ok := tctx.ipMap[ip.String()]; !ok || !there {
			t.Fatalf("ip %v got double deallocated", ip)
		}
		tctx.ipMap[ip.String()] = false
		for j, v := range tctx.ipList {
			if v == ip {
				tctx.ipList = append(tctx.ipList[:j], tctx.ipList[j+1:]...)
				break
			}
		}
	}

	assert.Len(t, tctx.ipList, tctx.maxIP-length)
}
<|endoftext|>


<commit_before>// Package eval handles evaluation of nodes and consists the runtime of the
// shell.
package eval

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"strings"
	"syscall"
	"unicode/utf8"

	"github.com/elves/elvish/errutil"
	"github.com/elves/elvish/parse"
	"github.com/elves/elvish/store"
)

const FnPrefix = "&"

// ns is a namespace.
type ns map[string]Variable

// Evaler is used to evaluate elvish sources. It maintains runtime context
// shared among all evalCtx instances.
type Evaler struct {
	global      ns
	mod         map[string]ns
	searchPaths []string
	store       *store.Store
	Editor      Editor
}

// evalCtx maintains an Evaler along with its runtime context. After creation
// an evalCtx is not modified, and new instances are created when needed.
type evalCtx struct {
	*Evaler
	name, text, context string

	local, up ns
	ports     []*port
}

// NewEvaler creates a new Evaler.
func NewEvaler(st *store.Store) *Evaler {
	// Construct searchPaths
	var searchPaths []string
	if path := os.Getenv("PATH"); path != "" {
		searchPaths = strings.Split(path, ":")
	} else {
		searchPaths = []string{"/bin"}
	}

	// Construct initial global namespace
	pid := String(strconv.Itoa(syscall.Getpid()))
	paths := NewList()
	paths.appendStrings(searchPaths)
	global := ns{
		"pid":   newPtrVariable(pid),
		"ok":    newPtrVariable(OK),
		"true":  newPtrVariable(Bool(true)),
		"false": newPtrVariable(Bool(false)),
		"paths": newPtrVariable(paths),
	}
	for _, b := range builtinFns {
		global[FnPrefix+b.Name] = newPtrVariable(b)
	}

	return &Evaler{global, map[string]ns{}, searchPaths, st, nil}
}

func PprintError(e error) {
	switch e := e.(type) {
	case nil:
		fmt.Print("\033[32mok\033[m")
	case multiError:
		fmt.Print("(")
		for i, c := range e.errors {
			if i > 0 {
				fmt.Print(" | ")
			}
			PprintError(c.inner)
		}
		fmt.Print(")")
	case flow:
		fmt.Print("\033[33m" + e.Error() + "\033[m")
	default:
		fmt.Print("\033[31;1m" + e.Error() + "\033[m")
	}
}

const (
	outChanSize   = 32
	outChanLeader = "▶ "
)

// newTopEvalCtx creates a top-level evalCtx.
func newTopEvalCtx(ev *Evaler, name, text string) (*evalCtx, chan bool) {
	ch := make(chan Value, outChanSize)
	done := make(chan bool, 1)
	go func() {
		for v := range ch {
			fmt.Printf("%s%s\n", outChanLeader, v.Repr())
		}
		done <- true
	}()

	return &evalCtx{
		ev,
		name, text, "top",
		ev.global, ns{},
		[]*port{{f: os.Stdin},
			{f: os.Stdout, ch: ch, closeCh: true}, {f: os.Stderr}},
	}, done
}

// fork returns a modified copy of ec. The ports are copied deeply, with
// shouldClose flags reset, and the context is changed to the given value.
// Other fields are copied shallowly.
func (ec *evalCtx) fork(newContext string) *evalCtx {
	newPorts := make([]*port, len(ec.ports))
	for i, p := range ec.ports {
		newPorts[i] = &port{p.f, p.ch, false, false}
	}
	return &evalCtx{
		ec.Evaler,
		ec.name, ec.text, newContext,
		ec.local, ec.up,
		newPorts,
	}
}

// port returns ec.ports[i] or nil if i is out of range. This makes it possible
// to treat ec.ports as if it has an infinite tail of nil's.
func (ec *evalCtx) port(i int) *port {
	if i >= len(ec.ports) {
		return nil
	}
	return ec.ports[i]
}

// growPorts makes the size of ec.ports at least n, adding nil's if necessary.
func (ec *evalCtx) growPorts(n int) {
	if len(ec.ports) >= n {
		return
	}
	ports := ec.ports
	ec.ports = make([]*port, n)
	copy(ec.ports, ports)
}

func makeScope(s ns) scope {
	sc := scope{}
	for name, _ := range s {
		sc[name] = true
	}
	return sc
}

// Eval evaluates a chunk node n. The supplied name and text are used in
// diagnostic messages.
func (ev *Evaler) Eval(name, text string, n *parse.Chunk) error {
	return ev.evalWithOut(name, text, n, nil)
}

func (ev *Evaler) evalWithOut(name, text string, n *parse.Chunk, out *port) error {
	op, err := compile(name, text, makeScope(ev.global), n)
	if err != nil {
		return err
	}

	ec, outdone := newTopEvalCtx(ev, name, text)
	if out != nil {
		outdone = nil
		ec.ports[1] = out
	}
	ex := ec.peval(op)
	ec.closePorts()
	if outdone != nil {
		<-outdone
	}

	return ex
}

// peval evaluates an op in a protected environment so that calls to errorf are
// wrapped in an Error.
func (ec *evalCtx) peval(op op) (ex error) {
	defer errutil.Catch(&ex)
	op(ec)
	return nil
}

// errorf stops the ec.eval immediately by panicking with a diagnostic message.
// The panic is supposed to be caught by ec.eval.
func (ec *evalCtx) errorf(p int, format string, args ...interface{}) {
	throw(errutil.NewContextualError(
		fmt.Sprintf("%s (%s)", ec.name, ec.context), "error",
		ec.text, p, format, args...))
}

// mustSingleString returns a String if that is the only element of vs.
// Otherwise it errors.
func (ec *evalCtx) mustSingleString(vs []Value, what string, p int) String {
	if len(vs) != 1 {
		ec.errorf(p, "Expect exactly one word for %s, got %d", what, len(vs))
	}
	v, ok := vs[0].(String)
	if !ok {
		ec.errorf(p, "Expect string for %s, got %s", what, vs[0])
	}
	return v
}

// SourceText evaluates a chunk of elvish source.
func (ev *Evaler) SourceText(name, src, dir string) error {
	n, err := parse.Parse(name, src)
	if err != nil {
		return err
	}
	return ev.Eval(name, src, n)
}

func readFileUTF8(fname string) (string, error) {
	bytes, err := ioutil.ReadFile(fname)
	if err != nil {
		return "", err
	}
	if !utf8.Valid(bytes) {
		return "", fmt.Errorf("%s: source is not valid UTF-8", fname)
	}
	return string(bytes), nil
}

// Source evaluates the content of a file.
func (ev *Evaler) Source(fname string) error {
	src, err := readFileUTF8(fname)
	if err != nil {
		return err
	}
	return ev.SourceText(fname, src, path.Dir(fname))
}

// Global returns the global namespace.
func (ev *Evaler) Global() ns {
	return ev.global
}

// ResolveVar resolves a variable. When the variable cannot be found, nil is
// returned.
func (ec *evalCtx) ResolveVar(ns, name string) Variable {
	if ns == "env" {
		return newEnvVariable(name)
	}
	if mod, ok := ec.mod[ns]; ok {
		return mod[name]
	}

	may := func(n string) bool {
		return ns == "" || ns == n
	}
	if may("local") {
		if v, ok := ec.local[name]; ok {
			return v
		}
	}
	if may("up") {
		if v, ok := ec.up[name]; ok {
			return v
		}
	}
	return nil
}
<commit_msg>eval: implement output redir correctly in Evaler.evalWithChanOut<commit_after>// Package eval handles evaluation of nodes and consists the runtime of the
// shell.
package eval

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"strings"
	"syscall"
	"unicode/utf8"

	"github.com/elves/elvish/errutil"
	"github.com/elves/elvish/parse"
	"github.com/elves/elvish/store"
)

const FnPrefix = "&"

// ns is a namespace.
type ns map[string]Variable

// Evaler is used to evaluate elvish sources. It maintains runtime context
// shared among all evalCtx instances.
type Evaler struct {
	global      ns
	mod         map[string]ns
	searchPaths []string
	store       *store.Store
	Editor      Editor
}

// evalCtx maintains an Evaler along with its runtime context. After creation
// an evalCtx is not modified, and new instances are created when needed.
type evalCtx struct {
	*Evaler
	name, text, context string

	local, up ns
	ports     []*port
}

// NewEvaler creates a new Evaler.
func NewEvaler(st *store.Store) *Evaler {
	// Construct searchPaths
	var searchPaths []string
	if path := os.Getenv("PATH"); path != "" {
		searchPaths = strings.Split(path, ":")
	} else {
		searchPaths = []string{"/bin"}
	}

	// Construct initial global namespace
	pid := String(strconv.Itoa(syscall.Getpid()))
	paths := NewList()
	paths.appendStrings(searchPaths)
	global := ns{
		"pid":   newPtrVariable(pid),
		"ok":    newPtrVariable(OK),
		"true":  newPtrVariable(Bool(true)),
		"false": newPtrVariable(Bool(false)),
		"paths": newPtrVariable(paths),
	}
	for _, b := range builtinFns {
		global[FnPrefix+b.Name] = newPtrVariable(b)
	}

	return &Evaler{global, map[string]ns{}, searchPaths, st, nil}
}

func PprintError(e error) {
	switch e := e.(type) {
	case nil:
		fmt.Print("\033[32mok\033[m")
	case multiError:
		fmt.Print("(")
		for i, c := range e.errors {
			if i > 0 {
				fmt.Print(" | ")
			}
			PprintError(c.inner)
		}
		fmt.Print(")")
	case flow:
		fmt.Print("\033[33m" + e.Error() + "\033[m")
	default:
		fmt.Print("\033[31;1m" + e.Error() + "\033[m")
	}
}

const (
	outChanSize   = 32
	outChanLeader = "▶ "
)

// newTopEvalCtx creates a top-level evalCtx.
func newTopEvalCtx(ev *Evaler, name, text string) (*evalCtx, chan bool) {
	ch := make(chan Value, outChanSize)
	done := make(chan bool, 1)
	go func() {
		for v := range ch {
			fmt.Printf("%s%s\n", outChanLeader, v.Repr())
		}
		done <- true
	}()

	return &evalCtx{
		ev,
		name, text, "top",
		ev.global, ns{},
		[]*port{{f: os.Stdin},
			{f: os.Stdout, ch: ch, closeCh: true}, {f: os.Stderr}},
	}, done
}

// fork returns a modified copy of ec. The ports are copied deeply, with
// shouldClose flags reset, and the context is changed to the given value.
// Other fields are copied shallowly.
func (ec *evalCtx) fork(newContext string) *evalCtx {
	newPorts := make([]*port, len(ec.ports))
	for i, p := range ec.ports {
		newPorts[i] = &port{p.f, p.ch, false, false}
	}
	return &evalCtx{
		ec.Evaler,
		ec.name, ec.text, newContext,
		ec.local, ec.up,
		newPorts,
	}
}

// port returns ec.ports[i] or nil if i is out of range. This makes it possible
// to treat ec.ports as if it has an infinite tail of nil's.
func (ec *evalCtx) port(i int) *port {
	if i >= len(ec.ports) {
		return nil
	}
	return ec.ports[i]
}

// growPorts makes the size of ec.ports at least n, adding nil's if necessary.
func (ec *evalCtx) growPorts(n int) {
	if len(ec.ports) >= n {
		return
	}
	ports := ec.ports
	ec.ports = make([]*port, n)
	copy(ec.ports, ports)
}

func makeScope(s ns) scope {
	sc := scope{}
	for name, _ := range s {
		sc[name] = true
	}
	return sc
}

// Eval evaluates a chunk node n. The supplied name and text are used in
// diagnostic messages.
func (ev *Evaler) Eval(name, text string, n *parse.Chunk) error {
	return ev.evalWithOut(name, text, n, nil)
}

func (ev *Evaler) evalWithOut(name, text string, n *parse.Chunk, out *port) error {
	op, err := compile(name, text, makeScope(ev.global), n)
	if err != nil {
		out.close()
		return err
	}

	ec, outdone := newTopEvalCtx(ev, name, text)
	if out != nil {
		ec.ports[1].close()
		ec.ports[1] = out
	}
	ex := ec.peval(op)
	ec.closePorts()
	<-outdone

	return ex
}

// peval evaluates an op in a protected environment so that calls to errorf are
// wrapped in an Error.
func (ec *evalCtx) peval(op op) (ex error) {
	defer errutil.Catch(&ex)
	op(ec)
	return nil
}

// errorf stops the ec.eval immediately by panicking with a diagnostic message.
// The panic is supposed to be caught by ec.eval.
func (ec *evalCtx) errorf(p int, format string, args ...interface{}) {
	throw(errutil.NewContextualError(
		fmt.Sprintf("%s (%s)", ec.name, ec.context), "error",
		ec.text, p, format, args...))
}

// mustSingleString returns a String if that is the only element of vs.
// Otherwise it errors.
func (ec *evalCtx) mustSingleString(vs []Value, what string, p int) String {
	if len(vs) != 1 {
		ec.errorf(p, "Expect exactly one word for %s, got %d", what, len(vs))
	}
	v, ok := vs[0].(String)
	if !ok {
		ec.errorf(p, "Expect string for %s, got %s", what, vs[0])
	}
	return v
}

// SourceText evaluates a chunk of elvish source.
func (ev *Evaler) SourceText(name, src, dir string) error {
	n, err := parse.Parse(name, src)
	if err != nil {
		return err
	}
	return ev.Eval(name, src, n)
}

func readFileUTF8(fname string) (string, error) {
	bytes, err := ioutil.ReadFile(fname)
	if err != nil {
		return "", err
	}
	if !utf8.Valid(bytes) {
		return "", fmt.Errorf("%s: source is not valid UTF-8", fname)
	}
	return string(bytes), nil
}

// Source evaluates the content of a file.
func (ev *Evaler) Source(fname string) error {
	src, err := readFileUTF8(fname)
	if err != nil {
		return err
	}
	return ev.SourceText(fname, src, path.Dir(fname))
}

// Global returns the global namespace.
func (ev *Evaler) Global() ns {
	return ev.global
}

// ResolveVar resolves a variable. When the variable cannot be found, nil is
// returned.
func (ec *evalCtx) ResolveVar(ns, name string) Variable {
	if ns == "env" {
		return newEnvVariable(name)
	}
	if mod, ok := ec.mod[ns]; ok {
		return mod[name]
	}

	may := func(n string) bool {
		return ns == "" || ns == n
	}
	if may("local") {
		if v, ok := ec.local[name]; ok {
			return v
		}
	}
	if may("up") {
		if v, ok := ec.up[name]; ok {
			return v
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Package eval handles evaluation of nodes and consists the runtime of the
// shell.
package eval

//go:generate ./gen-embedded-modules

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"unicode/utf8"

	"github.com/elves/elvish/parse"
	"github.com/elves/elvish/store"
	"github.com/elves/elvish/sys"
	"github.com/elves/elvish/util"
)

var logger = util.GetLogger("[eval] ")

// FnPrefix is the prefix for the variable names of functions. Defining a
// function "foo" is equivalent to setting a variable named FnPrefix + "foo".
const FnPrefix = "&"

// Namespace is a map from name to variables.
type Namespace map[string]Variable

// Evaler is used to evaluate elvish sources. It maintains runtime context
// shared among all evalCtx instances.
type Evaler struct {
	Global  Namespace
	Modules map[string]Namespace
	Store   *store.Store
	Editor  Editor
	DataDir string
	intCh   chan struct{}
}

// EvalCtx maintains an Evaler along with its runtime context. After creation
// an EvalCtx is seldom modified, and new instances are created when needed.
type EvalCtx struct {
	*Evaler
	name, srcName, src string

	local, up   Namespace
	ports       []*Port
	positionals []Value

	begin, end int
	traceback  *util.SourceContext

	background bool
}

// NewEvaler creates a new Evaler.
func NewEvaler(st *store.Store, dataDir string) *Evaler {
	return &Evaler{Namespace{}, map[string]Namespace{}, st, nil, dataDir, nil}
}

func (ev *Evaler) searchPaths() []string {
	return builtinNamespace["paths"].(*EnvPathList).get()
}

const (
	outChanSize    = 32
	outChanLeader  = "▶ "
	falseIndicator = "✗"
	initIndent     = NoPretty
)

// NewTopEvalCtx creates a top-level evalCtx.
func NewTopEvalCtx(ev *Evaler, name, text string, ports []*Port) *EvalCtx {
	return &EvalCtx{
		ev, "top",
		name, text,
		ev.Global, Namespace{},
		ports, nil,
		0, len(text), nil, false,
	}
}

// fork returns a modified copy of ec. The ports are forked, and the name is
// changed to the given value. Other fields are copied shallowly.
func (ec *EvalCtx) fork(name string) *EvalCtx {
	newPorts := make([]*Port, len(ec.ports))
	for i, p := range ec.ports {
		newPorts[i] = p.Fork()
	}
	return &EvalCtx{
		ec.Evaler, name,
		ec.srcName, ec.src,
		ec.local, ec.up,
		newPorts, ec.positionals,
		ec.begin, ec.end, ec.traceback, ec.background,
	}
}

// port returns ec.ports[i] or nil if i is out of range. This makes it possible
// to treat ec.ports as if it has an infinite tail of nil's.
func (ec *EvalCtx) port(i int) *Port {
	if i >= len(ec.ports) {
		return nil
	}
	return ec.ports[i]
}

// growPorts makes the size of ec.ports at least n, adding nil's if necessary.
func (ec *EvalCtx) growPorts(n int) {
	if len(ec.ports) >= n {
		return
	}
	ports := ec.ports
	ec.ports = make([]*Port, n)
	copy(ec.ports, ports)
}

func makeScope(s Namespace) scope {
	sc := scope{}
	for name := range s {
		sc[name] = true
	}
	return sc
}

// eval evaluates a chunk node n. The supplied name and text are used in
// diagnostic messages.
func (ev *Evaler) eval(op Op, ports []*Port, name, text string) error {
	ec := NewTopEvalCtx(ev, name, text, ports)
	return ec.PEval(op)
}

func (ec *EvalCtx) Interrupts() <-chan struct{} {
	return ec.intCh
}

// Eval sets up the Evaler and evaluates a chunk. The supplied name and text are
// used in diagnostic messages.
func (ev *Evaler) Eval(op Op, name, text string) error {
	inCh := make(chan Value)
	close(inCh)

	outCh := make(chan Value, outChanSize)
	outDone := make(chan struct{})
	go func() {
		for v := range outCh {
			fmt.Printf("%s%s\n", outChanLeader, v.Repr(initIndent))
		}
		close(outDone)
	}()

	ports := []*Port{
		{File: os.Stdin, Chan: inCh},
		{File: os.Stdout, Chan: outCh},
		{File: os.Stderr, Chan: BlackholeChan},
	}

	// signal.Ignore(syscall.SIGTTIN)

	// Ignore TTOU.
	// When a subprocess in its own process group puts itself in the foreground,
	// the elvish will be in the background. In that case, elvish will move
	// itself back to the foreground by calling tcsetpgrp. However, whenever a
	// background process calls tcsetpgrp (or otherwise attempts to modify the
	// terminal configuration), TTOU will be sent, whose default handler is to
	// stop the process. When the process lives in an orphaned process group
	// (most likely for elvish), the call will outright fail. Therefore, for
	// elvish to be able to move itself back to the foreground, we need to
	// ignore TTOU.
	signal.Ignore(syscall.SIGTTOU)
	stopSigGoroutine := make(chan struct{})
	sigGoRoutineDone := make(chan struct{})
	// Set up intCh.
	ev.intCh = make(chan struct{})
	sigCh := make(chan os.Signal)
	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGQUIT)
	go func() {
		closedIntCh := false
	loop:
		for {
			select {
			case <-sigCh:
				if !closedIntCh {
					close(ev.intCh)
					closedIntCh = true
				}
			case <-stopSigGoroutine:
				break loop
			}
		}
		ev.intCh = nil
		signal.Stop(sigCh)
		close(sigGoRoutineDone)
	}()

	err := ev.eval(op, ports, name, text)
	close(outCh)
	<-outDone
	close(stopSigGoroutine)
	<-sigGoRoutineDone

	// Put myself in foreground, in case some command has put me in background.
	// XXX Should probably use fd of /dev/tty instead of 0.
	if sys.IsATTY(0) {
		err := sys.Tcsetpgrp(0, syscall.Getpgrp())
		if err != nil {
			fmt.Println("failed to put myself in foreground:", err)
		}
	}

	// Un-ignore TTOU.
	signal.Reset(syscall.SIGTTOU)

	return err
}

func summarize(text string) string {
	// TODO Make a proper summary.
	if len(text) < 32 {
		return text
	}
	var b bytes.Buffer
	for i, r := range text {
		if i+len(string(r)) >= 32 {
			break
		}
		b.WriteRune(r)
	}
	return b.String()
}

// Compile compiles elvish code in the global scope. If the error is not nil, it
// always has type CompilationError.
func (ev *Evaler) Compile(n *parse.Chunk, name, text string) (Op, error) {
	return compile(makeScope(ev.Global), n, name, text)
}

// PEval evaluates an op in a protected environment so that calls to errorf are
// wrapped in an Error.
func (ec *EvalCtx) PEval(op Op) (err error) {
	defer catch(&err, ec)
	op.Exec(ec)
	return nil
}

func (ec *EvalCtx) PCall(f Callable, args []Value, opts map[string]Value) (err error) {
	defer catch(&err, ec)
	f.Call(ec, args, opts)
	return nil
}

func (ec *EvalCtx) PCaptureOutput(f Callable, args []Value, opts map[string]Value) (vs []Value, err error) {
	// XXX There is no source.
	return pcaptureOutput(ec, Op{
		func(newec *EvalCtx) { f.Call(newec, args, opts) }, -1, -1})
}

func catch(perr *error, ec *EvalCtx) {
	// NOTE: We have to duplicate instead of calling util.Catch here, since
	// recover can only catch a panic when called directly from a deferred
	// function.
	r := recover()
	if r == nil {
		return
	}
	if exc, ok := r.(util.Thrown); ok {
		err := exc.Error
		if _, ok := err.(*Exception); !ok {
			err = ec.makeException(err)
		}
		*perr = err
	} else if r != nil {
		panic(r)
	}
}

// makeException turns an error into an Exception by adding traceback.
func (ec *EvalCtx) makeException(e error) *Exception {
	return &Exception{e, ec.addTraceback()}
}

func (ec *EvalCtx) addTraceback() *util.SourceContext {
	return &util.SourceContext{
		Name: ec.srcName, Source: ec.src,
		Begin: ec.begin, End: ec.end, Next: ec.traceback,
	}
}

// errorpf stops the ec.eval immediately by panicking with a diagnostic message.
// The panic is supposed to be caught by ec.eval.
func (ec *EvalCtx) errorpf(begin, end int, format string, args ...interface{}) {
	ec.begin, ec.end = begin, end
	throwf(format, args...)
}

// SourceText evaluates a chunk of elvish source.
func (ev *Evaler) SourceText(name, src string) error {
	n, err := parse.Parse(name, src)
	if err != nil {
		return err
	}
	op, err := ev.Compile(n, name, src)
	if err != nil {
		return err
	}
	return ev.Eval(op, name, src)
}

func readFileUTF8(fname string) (string, error) {
	bytes, err := ioutil.ReadFile(fname)
	if err != nil {
		return "", err
	}
	if !utf8.Valid(bytes) {
		return "", fmt.Errorf("%s: source is not valid UTF-8", fname)
	}
	return string(bytes), nil
}

// Source evaluates the content of a file.
func (ev *Evaler) Source(fname string) error {
	src, err := readFileUTF8(fname)
	if err != nil {
		return err
	}
	return ev.SourceText(fname, src)
}

// Builtin returns the builtin namespace.
func Builtin() Namespace {
	return builtinNamespace
}

// ErrStoreUnconnected is thrown by ResolveVar when a shared: variable needs to
// be resolved but the store is not connected.
var ErrStoreUnconnected = errors.New("store unconnected")

// ResolveVar resolves a variable. When the variable cannot be found, nil is
// returned.
func (ec *EvalCtx) ResolveVar(ns, name string) Variable {
	switch ns {
	case "local":
		return ec.getLocal(name)
	case "up":
		return ec.up[name]
	case "builtin":
		return builtinNamespace[name]
	case "":
		if v := ec.getLocal(name); v != nil {
			return v
		}
		if v, ok := ec.up[name]; ok {
			return v
		}
		return builtinNamespace[name]
	case "e":
		if strings.HasPrefix(name, FnPrefix) {
			return NewRoVariable(ExternalCmd{name[len(FnPrefix):]})
		}
	case "E":
		return envVariable{name}
	case "shared":
		if ec.Store == nil {
			throw(ErrStoreUnconnected)
		}
		return sharedVariable{ec.Store, name}
	default:
		if ns, ok := ec.Modules[ns]; ok {
			return ns[name]
		}
	}
	return nil
}

// getLocal finds the named local variable.
func (ec *EvalCtx) getLocal(name string) Variable {
	i, err := strconv.Atoi(name)
	if err == nil {
		// Logger.Println("positional variable", i)
		// Logger.Printf("EvalCtx=%p, args=%v", ec, ec.positionals)
		if i < 0 {
			i += len(ec.positionals)
		}
		if i < 0 || i >= len(ec.positionals) {
			// Logger.Print("out of range")
			return nil
		}
		// Logger.Print("found")
		return NewRoVariable(ec.positionals[i])
	}
	return ec.local[name]
}

var ErrMoreThanOneRest = errors.New("more than one @ lvalue")

// IterateInputs calls the passed function for each input element.
func (ec *EvalCtx) IterateInputs(f func(Value)) {
	var w sync.WaitGroup
	inputs := make(chan Value)

	w.Add(2)
	go func() {
		linesToChan(ec.ports[0].File, inputs)
		w.Done()
	}()
	go func() {
		for v := range ec.ports[0].Chan {
			inputs <- v
		}
		w.Done()
	}()
	go func() {
		w.Wait()
		close(inputs)
	}()

	for v := range inputs {
		f(v)
	}
}

func linesToChan(r io.Reader, ch chan<- Value) {
	filein := bufio.NewReader(r)
	for {
		line, err := filein.ReadString('\n')
		if line != "" {
			ch <- String(strings.TrimSuffix(line, "\n"))
		}
		if err != nil {
			if err != io.EOF {
				logger.Println("error on reading:", err)
			}
			break
		}
	}
}

// OutputChan returns a channel onto which output can be written.
func (ec *EvalCtx) OutputChan() chan<- Value {
	return ec.ports[1].Chan
}
<commit_msg>eval: Let Evaler eval with custom ports.<commit_after>// Package eval handles evaluation of nodes and consists the runtime of the
// shell.
package eval

//go:generate ./gen-embedded-modules

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"unicode/utf8"

	"github.com/elves/elvish/parse"
	"github.com/elves/elvish/store"
	"github.com/elves/elvish/sys"
	"github.com/elves/elvish/util"
)

var logger = util.GetLogger("[eval] ")

// FnPrefix is the prefix for the variable names of functions. Defining a
// function "foo" is equivalent to setting a variable named FnPrefix + "foo".
const FnPrefix = "&"

// Namespace is a map from name to variables.
type Namespace map[string]Variable

// Evaler is used to evaluate elvish sources. It maintains runtime context
// shared among all evalCtx instances.
type Evaler struct {
	Global  Namespace
	Modules map[string]Namespace
	Store   *store.Store
	Editor  Editor
	DataDir string
	intCh   chan struct{}
}

// EvalCtx maintains an Evaler along with its runtime context. After creation
// an EvalCtx is seldom modified, and new instances are created when needed.
type EvalCtx struct {
	*Evaler
	name, srcName, src string

	local, up   Namespace
	ports       []*Port
	positionals []Value

	begin, end int
	traceback  *util.SourceContext

	background bool
}

// NewEvaler creates a new Evaler.
func NewEvaler(st *store.Store, dataDir string) *Evaler {
	return &Evaler{Namespace{}, map[string]Namespace{}, st, nil, dataDir, nil}
}

func (ev *Evaler) searchPaths() []string {
	return builtinNamespace["paths"].(*EnvPathList).get()
}

const (
	outChanSize    = 32
	outChanLeader  = "▶ "
	falseIndicator = "✗"
	initIndent     = NoPretty
)

// NewTopEvalCtx creates a top-level evalCtx.
func NewTopEvalCtx(ev *Evaler, name, text string, ports []*Port) *EvalCtx {
	return &EvalCtx{
		ev, "top",
		name, text,
		ev.Global, Namespace{},
		ports, nil,
		0, len(text), nil, false,
	}
}

// fork returns a modified copy of ec. The ports are forked, and the name is
// changed to the given value. Other fields are copied shallowly.
func (ec *EvalCtx) fork(name string) *EvalCtx {
	newPorts := make([]*Port, len(ec.ports))
	for i, p := range ec.ports {
		newPorts[i] = p.Fork()
	}
	return &EvalCtx{
		ec.Evaler, name,
		ec.srcName, ec.src,
		ec.local, ec.up,
		newPorts, ec.positionals,
		ec.begin, ec.end, ec.traceback, ec.background,
	}
}

// port returns ec.ports[i] or nil if i is out of range. This makes it possible
// to treat ec.ports as if it has an infinite tail of nil's.
func (ec *EvalCtx) port(i int) *Port {
	if i >= len(ec.ports) {
		return nil
	}
	return ec.ports[i]
}

// growPorts makes the size of ec.ports at least n, adding nil's if necessary.
func (ec *EvalCtx) growPorts(n int) {
	if len(ec.ports) >= n {
		return
	}
	ports := ec.ports
	ec.ports = make([]*Port, n)
	copy(ec.ports, ports)
}

func makeScope(s Namespace) scope {
	sc := scope{}
	for name := range s {
		sc[name] = true
	}
	return sc
}

// eval evaluates a chunk node n. The supplied name and text are used in
// diagnostic messages.
func (ev *Evaler) eval(op Op, ports []*Port, name, text string) error {
	ec := NewTopEvalCtx(ev, name, text, ports)
	return ec.PEval(op)
}

func (ec *EvalCtx) Interrupts() <-chan struct{} {
	return ec.intCh
}

// Eval sets up the Evaler with standard ports and evaluates an Op. The supplied
// name and text are used in diagnostic messages.
func (ev *Evaler) Eval(op Op, name, text string) error {
	inCh := make(chan Value)
	close(inCh)

	outCh := make(chan Value, outChanSize)
	outDone := make(chan struct{})
	go func() {
		for v := range outCh {
			fmt.Println(outChanLeader + v.Repr(initIndent))
		}
		close(outDone)
	}()
	defer func() {
		close(outCh)
		<-outDone
	}()

	ports := []*Port{
		{File: os.Stdin, Chan: inCh},
		{File: os.Stdout, Chan: outCh},
		{File: os.Stderr, Chan: BlackholeChan},
	}

	return ev.EvalWithPorts(ports, op, name, text)
}

// EvalWithFiles sets up the Evaler with the given ports and evaluates an Op.
// The supplied name and text are used in diagnostic messages.
func (ev *Evaler) EvalWithPorts(ports []*Port, op Op, name, text string) error {
	// signal.Ignore(syscall.SIGTTIN)

	// Ignore TTOU.
	// When a subprocess in its own process group puts itself in the foreground,
	// the elvish will be in the background. In that case, elvish will move
	// itself back to the foreground by calling tcsetpgrp. However, whenever a
	// background process calls tcsetpgrp (or otherwise attempts to modify the
	// terminal configuration), TTOU will be sent, whose default handler is to
	// stop the process. When the process lives in an orphaned process group
	// (most likely for elvish), the call will outright fail. Therefore, for
	// elvish to be able to move itself back to the foreground, we need to
	// ignore TTOU.
	signal.Ignore(syscall.SIGTTOU)
	stopSigGoroutine := make(chan struct{})
	sigGoRoutineDone := make(chan struct{})
	// Set up intCh.
	ev.intCh = make(chan struct{})
	sigCh := make(chan os.Signal)
	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGQUIT)
	go func() {
		closedIntCh := false
	loop:
		for {
			select {
			case <-sigCh:
				if !closedIntCh {
					close(ev.intCh)
					closedIntCh = true
				}
			case <-stopSigGoroutine:
				break loop
			}
		}
		ev.intCh = nil
		signal.Stop(sigCh)
		close(sigGoRoutineDone)
	}()

	err := ev.eval(op, ports, name, text)

	close(stopSigGoroutine)
	<-sigGoRoutineDone

	// Put myself in foreground, in case some command has put me in background.
	// XXX Should probably use fd of /dev/tty instead of 0.
	if sys.IsATTY(0) {
		err := sys.Tcsetpgrp(0, syscall.Getpgrp())
		if err != nil {
			fmt.Println("failed to put myself in foreground:", err)
		}
	}

	// Un-ignore TTOU.
	signal.Reset(syscall.SIGTTOU)

	return err
}

func summarize(text string) string {
	// TODO Make a proper summary.
	if len(text) < 32 {
		return text
	}
	var b bytes.Buffer
	for i, r := range text {
		if i+len(string(r)) >= 32 {
			break
		}
		b.WriteRune(r)
	}
	return b.String()
}

// Compile compiles elvish code in the global scope. If the error is not nil, it
// always has type CompilationError.
func (ev *Evaler) Compile(n *parse.Chunk, name, text string) (Op, error) {
	return compile(makeScope(ev.Global), n, name, text)
}

// PEval evaluates an op in a protected environment so that calls to errorf are
// wrapped in an Error.
func (ec *EvalCtx) PEval(op Op) (err error) {
	defer catch(&err, ec)
	op.Exec(ec)
	return nil
}

func (ec *EvalCtx) PCall(f Callable, args []Value, opts map[string]Value) (err error) {
	defer catch(&err, ec)
	f.Call(ec, args, opts)
	return nil
}

func (ec *EvalCtx) PCaptureOutput(f Callable, args []Value, opts map[string]Value) (vs []Value, err error) {
	// XXX There is no source.
	return pcaptureOutput(ec, Op{
		func(newec *EvalCtx) { f.Call(newec, args, opts) }, -1, -1})
}

func catch(perr *error, ec *EvalCtx) {
	// NOTE: We have to duplicate instead of calling util.Catch here, since
	// recover can only catch a panic when called directly from a deferred
	// function.
	r := recover()
	if r == nil {
		return
	}
	if exc, ok := r.(util.Thrown); ok {
		err := exc.Error
		if _, ok := err.(*Exception); !ok {
			err = ec.makeException(err)
		}
		*perr = err
	} else if r != nil {
		panic(r)
	}
}

// makeException turns an error into an Exception by adding traceback.
func (ec *EvalCtx) makeException(e error) *Exception {
	return &Exception{e, ec.addTraceback()}
}

func (ec *EvalCtx) addTraceback() *util.SourceContext {
	return &util.SourceContext{
		Name: ec.srcName, Source: ec.src,
		Begin: ec.begin, End: ec.end, Next: ec.traceback,
	}
}

// errorpf stops the ec.eval immediately by panicking with a diagnostic message.
// The panic is supposed to be caught by ec.eval.
func (ec *EvalCtx) errorpf(begin, end int, format string, args ...interface{}) {
	ec.begin, ec.end = begin, end
	throwf(format, args...)
}

// SourceText evaluates a chunk of elvish source.
func (ev *Evaler) SourceText(name, src string) error {
	n, err := parse.Parse(name, src)
	if err != nil {
		return err
	}
	op, err := ev.Compile(n, name, src)
	if err != nil {
		return err
	}
	return ev.Eval(op, name, src)
}

func readFileUTF8(fname string) (string, error) {
	bytes, err := ioutil.ReadFile(fname)
	if err != nil {
		return "", err
	}
	if !utf8.Valid(bytes) {
		return "", fmt.Errorf("%s: source is not valid UTF-8", fname)
	}
	return string(bytes), nil
}

// Source evaluates the content of a file.
func (ev *Evaler) Source(fname string) error {
	src, err := readFileUTF8(fname)
	if err != nil {
		return err
	}
	return ev.SourceText(fname, src)
}

// Builtin returns the builtin namespace.
func Builtin() Namespace {
	return builtinNamespace
}

// ErrStoreUnconnected is thrown by ResolveVar when a shared: variable needs to
// be resolved but the store is not connected.
var ErrStoreUnconnected = errors.New("store unconnected")

// ResolveVar resolves a variable. When the variable cannot be found, nil is
// returned.
func (ec *EvalCtx) ResolveVar(ns, name string) Variable {
	switch ns {
	case "local":
		return ec.getLocal(name)
	case "up":
		return ec.up[name]
	case "builtin":
		return builtinNamespace[name]
	case "":
		if v := ec.getLocal(name); v != nil {
			return v
		}
		if v, ok := ec.up[name]; ok {
			return v
		}
		return builtinNamespace[name]
	case "e":
		if strings.HasPrefix(name, FnPrefix) {
			return NewRoVariable(ExternalCmd{name[len(FnPrefix):]})
		}
	case "E":
		return envVariable{name}
	case "shared":
		if ec.Store == nil {
			throw(ErrStoreUnconnected)
		}
		return sharedVariable{ec.Store, name}
	default:
		if ns, ok := ec.Modules[ns]; ok {
			return ns[name]
		}
	}
	return nil
}

// getLocal finds the named local variable.
func (ec *EvalCtx) getLocal(name string) Variable {
	i, err := strconv.Atoi(name)
	if err == nil {
		// Logger.Println("positional variable", i)
		// Logger.Printf("EvalCtx=%p, args=%v", ec, ec.positionals)
		if i < 0 {
			i += len(ec.positionals)
		}
		if i < 0 || i >= len(ec.positionals) {
			// Logger.Print("out of range")
			return nil
		}
		// Logger.Print("found")
		return NewRoVariable(ec.positionals[i])
	}
	return ec.local[name]
}

var ErrMoreThanOneRest = errors.New("more than one @ lvalue")

// IterateInputs calls the passed function for each input element.
func (ec *EvalCtx) IterateInputs(f func(Value)) {
	var w sync.WaitGroup
	inputs := make(chan Value)

	w.Add(2)
	go func() {
		linesToChan(ec.ports[0].File, inputs)
		w.Done()
	}()
	go func() {
		for v := range ec.ports[0].Chan {
			inputs <- v
		}
		w.Done()
	}()
	go func() {
		w.Wait()
		close(inputs)
	}()

	for v := range inputs {
		f(v)
	}
}

func linesToChan(r io.Reader, ch chan<- Value) {
	filein := bufio.NewReader(r)
	for {
		line, err := filein.ReadString('\n')
		if line != "" {
			ch <- String(strings.TrimSuffix(line, "\n"))
		}
		if err != nil {
			if err != io.EOF {
				logger.Println("error on reading:", err)
			}
			break
		}
	}
}

// OutputChan returns a channel onto which output can be written.
func (ec *EvalCtx) OutputChan() chan<- Value {
	return ec.ports[1].Chan
}
<|endoftext|>


<commit_before>package controllers

import (
	"bytes"
	"fmt"
	"github.com/gin-gonic/gin"
	"github.com/stretchr/testify/assert"
	"gopkg.in/DATA-DOG/go-sqlmock.v1"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/eirka/eirka-libs/audit"
	"github.com/eirka/eirka-libs/db"
	//e "github.com/eirka/eirka-libs/errors"
	"github.com/eirka/eirka-libs/redis"
	"github.com/eirka/eirka-libs/user"
)

func performJwtJsonRequest(r http.Handler, method, path, token string, body []byte) *httptest.ResponseRecorder {
	req, _ := http.NewRequest(method, path, bytes.NewBuffer(body))
	req.Header.Set("X-Real-Ip", "123.0.0.1")
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
	req.Header.Set("Content-Type", "application/json")
	w := httptest.NewRecorder()
	r.ServeHTTP(w, req)
	return w
}

func performJwtFormRequest(r http.Handler, method, path, token string, body bytes.Buffer) *httptest.ResponseRecorder {
	req, _ := http.NewRequest(method, path, &body)
	req.Header.Set("X-Real-Ip", "123.0.0.1")
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
	req.Header.Set("Content-Type", "multipart/form-data")
	w := httptest.NewRecorder()
	r.ServeHTTP(w, req)
	return w
}

func TestEmailController(t *testing.T) {

	var err error

	user.Secret = "secret"

	gin.SetMode(gin.ReleaseMode)

	router := gin.New()

	router.Use(user.Auth(true))

	router.POST("/email", EmailController)

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	rows := sqlmock.NewRows([]string{"name", "email"}).AddRow("test", "old@test.com")
	mock.ExpectQuery(`SELECT user_name,user_email FROM users WHERE user_id`).WillReturnRows(rows)

	mock.ExpectExec("UPDATE users SET user_email").
		WithArgs("cool@test.com", 2).
		WillReturnResult(sqlmock.NewResult(1, 1))

	request := []byte(`{"ib": 1, "email": "test@test.com"}`)

	first := performJsonRequest(router, "POST", "/email", request)

	assert.Equal(t, first.Code, 200, "HTTP request code should match")
	assert.JSONEq(t, first.Body.String(), successMessage(audit.AuditEmailUpdate), "HTTP response should match")

	assert.NoError(t, mock.ExpectationsWereMet(), "An error was not expected")

}
<commit_msg>add controller tests<commit_after>package controllers

import (
	"bytes"
	"fmt"
	"github.com/gin-gonic/gin"
	"github.com/stretchr/testify/assert"
	"gopkg.in/DATA-DOG/go-sqlmock.v1"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/eirka/eirka-libs/audit"
	"github.com/eirka/eirka-libs/db"
	//e "github.com/eirka/eirka-libs/errors"
	"github.com/eirka/eirka-libs/user"
)

func performJwtJsonRequest(r http.Handler, method, path, token string, body []byte) *httptest.ResponseRecorder {
	req, _ := http.NewRequest(method, path, bytes.NewBuffer(body))
	req.Header.Set("X-Real-Ip", "123.0.0.1")
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
	req.Header.Set("Content-Type", "application/json")
	w := httptest.NewRecorder()
	r.ServeHTTP(w, req)
	return w
}

func performJwtFormRequest(r http.Handler, method, path, token string, body bytes.Buffer) *httptest.ResponseRecorder {
	req, _ := http.NewRequest(method, path, &body)
	req.Header.Set("X-Real-Ip", "123.0.0.1")
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
	req.Header.Set("Content-Type", "multipart/form-data")
	w := httptest.NewRecorder()
	r.ServeHTTP(w, req)
	return w
}

func TestEmailController(t *testing.T) {

	var err error

	user.Secret = "secret"

	gin.SetMode(gin.ReleaseMode)

	router := gin.New()

	router.Use(user.Auth(true))

	router.POST("/email", EmailController)

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	rows := sqlmock.NewRows([]string{"name", "email"}).AddRow("test", "old@test.com")
	mock.ExpectQuery(`SELECT user_name,user_email FROM users WHERE user_id`).WillReturnRows(rows)

	mock.ExpectExec("UPDATE users SET user_email").
		WithArgs("cool@test.com", 2).
		WillReturnResult(sqlmock.NewResult(1, 1))

	request := []byte(`{"ib": 1, "email": "test@test.com"}`)

	first := performJsonRequest(router, "POST", "/email", request)

	assert.Equal(t, first.Code, 200, "HTTP request code should match")
	assert.JSONEq(t, first.Body.String(), successMessage(audit.AuditEmailUpdate), "HTTP response should match")

	assert.NoError(t, mock.ExpectationsWereMet(), "An error was not expected")

}
<|endoftext|>


<commit_before>package dnssrv

import (
	"crypto/md5"
	"fmt"
	"net"
	"sort"
	"time"

	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/loadbalancer"
	"github.com/go-kit/kit/log"
)

// Publisher yields endpoints taken from the named DNS SRV record. The name is
// resolved on a fixed schedule. Priorities and weights are ignored.
type Publisher struct {
	name      string
	ttl       time.Duration
	factory   loadbalancer.Factory
	logger    log.Logger
	endpoints chan []endpoint.Endpoint
	quit      chan struct{}
}

// NewPublisher returns a DNS SRV publisher. The name is resolved
// synchronously as part of construction; if that resolution fails, the
// constructor will return an error. The factory is used to convert a
// host:port to a usable endpoint. The logger is used to report DNS and
// factory errors.
func NewPublisher(name string, ttl time.Duration, f loadbalancer.Factory, logger log.Logger) (*Publisher, error) {
	logger = log.NewContext(logger).With("component", "DNS SRV Publisher")
	addrs, md5, err := resolve(name)
	if err != nil {
		return nil, err
	}
	p := &Publisher{
		name:      name,
		ttl:       ttl,
		factory:   f,
		logger:    logger,
		endpoints: make(chan []endpoint.Endpoint),
		quit:      make(chan struct{}),
	}
	go p.loop(lift(addrs, f, logger), md5)
	return p, nil
}

// Stop terminates the publisher.
func (p *Publisher) Stop() {
	close(p.quit)
}

func (p *Publisher) loop(endpoints []endpoint.Endpoint, md5 string) {
	t := newTicker(p.ttl)
	defer t.Stop()
	for {
		select {
		case p.endpoints <- endpoints:

		case <-t.C:
			// TODO should we do this out-of-band?
			addrs, newmd5, err := resolve(p.name)
			if err != nil {
				p.logger.Log("name", p.name, "err", err)
				continue // don't replace good endpoints with bad ones
			}
			if newmd5 == md5 {
				continue // no change
			}
			endpoints = lift(addrs, p.factory, p.logger)
			md5 = newmd5

		case <-p.quit:
			return
		}
	}
}

// Endpoints implements the Publisher interface.
func (p *Publisher) Endpoints() ([]endpoint.Endpoint, error) {
	select {
	case endpoints := <-p.endpoints:
		return endpoints, nil
	case <-p.quit:
		return nil, loadbalancer.ErrPublisherStopped
	}
}

var (
	lookupSRV = net.LookupSRV
	newTicker = time.NewTicker
)

func resolve(name string) (addrs []*net.SRV, md5sum string, err error) {
	_, addrs, err = lookupSRV("", "", name)
	if err != nil {
		return addrs, "", err
	}
	hostports := make([]string, len(addrs))
	for i, addr := range addrs {
		hostports[i] = fmt.Sprintf("%s:%d", addr.Target, addr.Port)
	}
	sort.Sort(sort.StringSlice(hostports))
	h := md5.New()
	for _, hostport := range hostports {
		fmt.Fprintf(h, hostport)
	}
	return addrs, fmt.Sprintf("%x", h.Sum(nil)), nil
}

func lift(addrs []*net.SRV, f loadbalancer.Factory, logger log.Logger) []endpoint.Endpoint {
	endpoints := make([]endpoint.Endpoint, 0, len(addrs))
	for _, addr := range addrs {
		endpoint, err := f(addr2instance(addr))
		if err != nil {
			logger.Log("instance", addr2instance(addr), "err", err)
			continue
		}
		endpoints = append(endpoints, endpoint)
	}
	return endpoints
}

func addr2instance(addr *net.SRV) string {
	return net.JoinHostPort(addr.Target, fmt.Sprint(addr.Port))
}
<commit_msg>s/lift/makeEndpoints/g<commit_after>package dnssrv

import (
	"crypto/md5"
	"fmt"
	"net"
	"sort"
	"time"

	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/loadbalancer"
	"github.com/go-kit/kit/log"
)

// Publisher yields endpoints taken from the named DNS SRV record. The name is
// resolved on a fixed schedule. Priorities and weights are ignored.
type Publisher struct {
	name      string
	ttl       time.Duration
	factory   loadbalancer.Factory
	logger    log.Logger
	endpoints chan []endpoint.Endpoint
	quit      chan struct{}
}

// NewPublisher returns a DNS SRV publisher. The name is resolved
// synchronously as part of construction; if that resolution fails, the
// constructor will return an error. The factory is used to convert a
// host:port to a usable endpoint. The logger is used to report DNS and
// factory errors.
func NewPublisher(name string, ttl time.Duration, f loadbalancer.Factory, logger log.Logger) (*Publisher, error) {
	logger = log.NewContext(logger).With("component", "DNS SRV Publisher")
	addrs, md5, err := resolve(name)
	if err != nil {
		return nil, err
	}
	p := &Publisher{
		name:      name,
		ttl:       ttl,
		factory:   f,
		logger:    logger,
		endpoints: make(chan []endpoint.Endpoint),
		quit:      make(chan struct{}),
	}
	go p.loop(makeEndpoints(addrs, f, logger), md5)
	return p, nil
}

// Stop terminates the publisher.
func (p *Publisher) Stop() {
	close(p.quit)
}

func (p *Publisher) loop(endpoints []endpoint.Endpoint, md5 string) {
	t := newTicker(p.ttl)
	defer t.Stop()
	for {
		select {
		case p.endpoints <- endpoints:

		case <-t.C:
			// TODO should we do this out-of-band?
			addrs, newmd5, err := resolve(p.name)
			if err != nil {
				p.logger.Log("name", p.name, "err", err)
				continue // don't replace good endpoints with bad ones
			}
			if newmd5 == md5 {
				continue // no change
			}
			endpoints = makeEndpoints(addrs, p.factory, p.logger)
			md5 = newmd5

		case <-p.quit:
			return
		}
	}
}

// Endpoints implements the Publisher interface.
func (p *Publisher) Endpoints() ([]endpoint.Endpoint, error) {
	select {
	case endpoints := <-p.endpoints:
		return endpoints, nil
	case <-p.quit:
		return nil, loadbalancer.ErrPublisherStopped
	}
}

var (
	lookupSRV = net.LookupSRV
	newTicker = time.NewTicker
)

func resolve(name string) (addrs []*net.SRV, md5sum string, err error) {
	_, addrs, err = lookupSRV("", "", name)
	if err != nil {
		return addrs, "", err
	}
	hostports := make([]string, len(addrs))
	for i, addr := range addrs {
		hostports[i] = fmt.Sprintf("%s:%d", addr.Target, addr.Port)
	}
	sort.Sort(sort.StringSlice(hostports))
	h := md5.New()
	for _, hostport := range hostports {
		fmt.Fprintf(h, hostport)
	}
	return addrs, fmt.Sprintf("%x", h.Sum(nil)), nil
}

func makeEndpoints(addrs []*net.SRV, f loadbalancer.Factory, logger log.Logger) []endpoint.Endpoint {
	endpoints := make([]endpoint.Endpoint, 0, len(addrs))
	for _, addr := range addrs {
		endpoint, err := f(addr2instance(addr))
		if err != nil {
			logger.Log("instance", addr2instance(addr), "err", err)
			continue
		}
		endpoints = append(endpoints, endpoint)
	}
	return endpoints
}

func addr2instance(addr *net.SRV) string {
	return net.JoinHostPort(addr.Target, fmt.Sprint(addr.Port))
}
<|endoftext|>