<commit_before>package actions

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"testing"

	"github.com/CenturyLinkLabs/panamax-remote-agent-go/agent"
	"github.com/CenturyLinkLabs/panamaxcli/config"
	"github.com/stretchr/testify/assert"
)

type FakeConfig struct {
	Agents              []config.Remote
	SavedName           string
	SavedToken          string
	ErrorForSave        error
	ActiveRemote        *config.Remote
	ActivatedRemoteName string
	ErrorForSetActive   error
}

func (c *FakeConfig) Save(name string, token string) error {
	c.SavedName = name
	c.SavedToken = token
	c.Agents = append(c.Agents, config.Remote{Name: name, Token: token})
	return c.ErrorForSave
}

func (c *FakeConfig) Get(name string) (config.Remote, error) {
	for _, r := range c.Agents {
		if r.Name == name {
			return r, nil
		}
	}

	return config.Remote{}, fmt.Errorf("the remote '%s' does not exist", name)
}

func (c *FakeConfig) Remotes() []config.Remote {
	return c.Agents
}

func (c *FakeConfig) SetActive(name string) error {
	c.ActivatedRemoteName = name
	if c.ErrorForSetActive != nil {
		return c.ErrorForSetActive
	}
	return nil
}

func (c *FakeConfig) Active() *config.Remote {
	return c.ActiveRemote
}

func setupTokenFile(t *testing.T, data string) string {
	tokenFile, err := ioutil.TempFile("", "pmx-test-token")
	tokenFile.WriteString(data)
	assert.NoError(t, err)
	return tokenFile.Name()
}

func TestAddRemote(t *testing.T) {
	tokenFilePath := setupTokenFile(t, "token data")
	defer os.Remove(tokenFilePath)
	fc := FakeConfig{}
	output, err := AddRemote(&fc, "testname", tokenFilePath)

	assert.Equal(t, "testname", fc.SavedName)
	assert.Equal(t, "token data", fc.SavedToken)
	assert.NoError(t, err)
	assert.Equal(t, "Successfully added! 'testname' is now your active remote.", output.ToPrettyOutput())
	assert.Equal(t, "testname", fc.ActivatedRemoteName)
}

func TestInactiveSecondAddRemote(t *testing.T) {
	tokenFilePath := setupTokenFile(t, "token data")
	defer os.Remove(tokenFilePath)
	fc := FakeConfig{Agents: []config.Remote{{Name: "Test"}}}
	output, err := AddRemote(&fc, "testname", tokenFilePath)

	assert.NoError(t, err)
	assert.Equal(t, "Successfully added!", output.ToPrettyOutput())
	assert.Equal(t, "", fc.ActivatedRemoteName)
}

func TestStripsWhitespaceAddRemote(t *testing.T) {
	tokenFilePath := setupTokenFile(t, "\n token data \n\n ")
	defer os.Remove(tokenFilePath)
	fc := FakeConfig{}
	_, err := AddRemote(&fc, "testname", tokenFilePath)

	assert.NoError(t, err)
	assert.Equal(t, "token data", fc.SavedToken)
}

func TestErroredExistingNameAddRemote(t *testing.T) {
	fc := FakeConfig{Agents: []config.Remote{{Name: "name"}}}
	output, err := AddRemote(&fc, "name", "unused")

	assert.Equal(t, "", output.ToPrettyOutput())
	assert.EqualError(t, err, "Name already exists")
}

func TestErroredInvalidNameAddRemote(t *testing.T) {
	fc := FakeConfig{}
	output, err := AddRemote(&fc, "bad name", "unused")
	assert.Equal(t, "", output.ToPrettyOutput())
	assert.EqualError(t, err, "Invalid name")

	_, err = AddRemote(&fc, "bad!", "unused")
	assert.EqualError(t, err, "Invalid name")

	_, err = AddRemote(&fc, "bad/", "unused")
	assert.EqualError(t, err, "Invalid name")
}

func TestErroredMissingFileAddRemote(t *testing.T) {
	fc := FakeConfig{}
	output, err := AddRemote(&fc, "name", "bad/file")
	assert.Equal(t, "", output.ToPrettyOutput())
	assert.EqualError(t, err, "open bad/file: no such file or directory")
}

func TestErroredConfigSaveAddRemote(t *testing.T) {
	tokenFilePath := setupTokenFile(t, "token data")
	defer os.Remove(tokenFilePath)
	fc := FakeConfig{ErrorForSave: errors.New("test error")}
	output, err := AddRemote(&fc, "name", tokenFilePath)

	assert.Equal(t, "", output.ToPrettyOutput())
	assert.EqualError(t, err, "test error")
}

func TestListRemotes(t *testing.T) {
	active := config.Remote{Name: "Active"}
	fc := FakeConfig{
		Agents: []config.Remote{
			{Name: "Test", Endpoint: "http://example.com"},
			active,
		},
		ActiveRemote: &active,
	}
	output := ListRemotes(&fc)

	lo, ok := output.(*ListOutput)
	if assert.True(t, ok) && assert.Len(t, lo.Rows, 2) {
		assert.Equal(t, "", lo.Rows[0]["Active"])
		assert.Equal(t, "Test", lo.Rows[0]["Name"])
		assert.Equal(t, "http://example.com", lo.Rows[0]["Endpoint"])

		assert.Equal(t, "*", lo.Rows[1]["Active"])
		assert.Equal(t, "Active", lo.Rows[1]["Name"])
	}
}

func TestNoActiveListRemotes(t *testing.T) {
	fc := FakeConfig{Agents: []config.Remote{{Name: "Test"}}}
	assert.NotPanics(t, func() {
		ListRemotes(&fc)
	})
}

func TestListRemotesNoRemotes(t *testing.T) {
	fc := FakeConfig{}
	output := ListRemotes(&fc)
	assert.Equal(t, "No remotes", output.ToPrettyOutput())
}

func TestSetActiveRemote(t *testing.T) {
	fc := FakeConfig{}
	output, err := SetActiveRemote(&fc, "Test")

	assert.Equal(t, "Test", fc.ActivatedRemoteName)
	assert.NoError(t, err)
	assert.Equal(t, "Success!", output.ToPrettyOutput())
}

func TestDescribeRemote(t *testing.T) {
	setupFactory()
	fakeClient.Metadata = agent.Metadata{
		Agent: struct {
			Version string `json:"version"`
		}{Version: "0.1"},
		Adapter: struct {
			Version   string `json:version`
			Type      string `json:type`
			IsHealthy bool   `json:isHealthy`
		}{"0.2", "Test", true},
	}
	fakeClient.Deployments = []agent.DeploymentResponseLite{
		{ID: 17, Name: "Test Deployment", ServiceIDs: []string{"1", "2"}},
	}
	r := config.Remote{Name: "Test", Endpoint: "http://example.com"}
	fc := FakeConfig{Agents: []config.Remote{r}}
	output, err := DescribeRemote(&fc, "Test")

	assert.NoError(t, err)
	if assert.Len(t, fakeFactory.NewedRemotes, 2) {
		assert.Equal(t, "Test", fakeFactory.NewedRemotes[0].Name)
	}

	co, ok := output.(*CombinedOutput)
	if assert.True(t, ok) && assert.Len(t, co.Outputs, 2) {
		do, ok := co.Outputs[0].Output.(DetailOutput)
		if assert.True(t, ok) {
			assert.Equal(t, "false", do.Details["Active"])
			assert.Equal(t, "Test", do.Details["Name"])
			assert.Equal(t, "http://example.com", do.Details["Endpoint"])
			assert.Equal(t, "0.1", do.Details["Agent Version"])
			assert.Equal(t, "0.2", do.Details["Adapter Version"])
			assert.Equal(t, "Test", do.Details["Adapter Type"])
			assert.Equal(t, "true", do.Details["Adapter Is Healthy"])
		}

		lo, ok := co.Outputs[1].Output.(*ListOutput)
		if assert.True(t, ok) && assert.Len(t, lo.Rows, 1) {
			r := lo.Rows[0]
			assert.Equal(t, "17", r["ID"])
			assert.Equal(t, "Test Deployment", r["Name"])
		}
	}
}

func TestErroredClientMetadataDescribeRemote(t *testing.T) {
	setupFactory()
	fakeClient.ErrorForMetadata = errors.New("test error")
	r := config.Remote{Name: "Test", Endpoint: "http://example.com"}
	fc := FakeConfig{Agents: []config.Remote{r}}

	output, err := DescribeRemote(&fc, "Test")
	assert.Equal(t, "", output.ToPrettyOutput())
	assert.EqualError(t, err, "test error")
}

func TestErroredClientDeploymentListDescribeRemote(t *testing.T) {
	setupFactory()
	fakeClient.ErrorForDeploymentList = errors.New("test error deployment list")
	r := config.Remote{Name: "Test", Endpoint: "http://example.com"}
	fc := FakeConfig{Agents: []config.Remote{r}}

	output, err := DescribeRemote(&fc, "Test")
	assert.Equal(t, "", output.ToPrettyOutput())
	assert.EqualError(t, err, "test error deployment list")
}

func TestErroredNonexistantDescribeRemote(t *testing.T) {
	fc := FakeConfig{}
	output, err := DescribeRemote(&fc, "Nonexistant")

	assert.EqualError(t, err, "the remote 'Nonexistant' does not exist")
	assert.Equal(t, "", output.ToPrettyOutput())
}

func TestErroredSetActiveRemote(t *testing.T) {
	fc := FakeConfig{ErrorForSetActive: errors.New("Name Not Found")}
	output, err := SetActiveRemote(&fc, "Bad")

	assert.Error(t, err)
	assert.Equal(t, "", output.ToPrettyOutput())
}

func TestGetRemoteToken(t *testing.T) {
	fc := FakeConfig{
		Agents: []config.Remote{{Name: "Test", Token: "Token\nText"}},
	}
	output, err := GetRemoteToken(&fc, "Test")

	assert.NoError(t, err)
	assert.Equal(t, "Token\nText", output.ToPrettyOutput())
}

func TestErroredGetRemoteToken(t *testing.T) {
	fc := FakeConfig{}
	output, err := GetRemoteToken(&fc, "nonexistant")

	assert.EqualError(t, err, "the remote 'nonexistant' does not exist")
	assert.Empty(t, output.ToPrettyOutput())
}
<commit_msg>Fix bad json annotation syntax. Thanks Go Vet!<commit_after>package actions

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"testing"

	"github.com/CenturyLinkLabs/panamax-remote-agent-go/agent"
	"github.com/CenturyLinkLabs/panamaxcli/config"
	"github.com/stretchr/testify/assert"
)

type FakeConfig struct {
	Agents              []config.Remote
	SavedName           string
	SavedToken          string
	ErrorForSave        error
	ActiveRemote        *config.Remote
	ActivatedRemoteName string
	ErrorForSetActive   error
}

func (c *FakeConfig) Save(name string, token string) error {
	c.SavedName = name
	c.SavedToken = token
	c.Agents = append(c.Agents, config.Remote{Name: name, Token: token})
	return c.ErrorForSave
}

func (c *FakeConfig) Get(name string) (config.Remote, error) {
	for _, r := range c.Agents {
		if r.Name == name {
			return r, nil
		}
	}

	return config.Remote{}, fmt.Errorf("the remote '%s' does not exist", name)
}

func (c *FakeConfig) Remotes() []config.Remote {
	return c.Agents
}

func (c *FakeConfig) SetActive(name string) error {
	c.ActivatedRemoteName = name
	if c.ErrorForSetActive != nil {
		return c.ErrorForSetActive
	}
	return nil
}

func (c *FakeConfig) Active() *config.Remote {
	return c.ActiveRemote
}

func setupTokenFile(t *testing.T, data string) string {
	tokenFile, err := ioutil.TempFile("", "pmx-test-token")
	tokenFile.WriteString(data)
	assert.NoError(t, err)
	return tokenFile.Name()
}

func TestAddRemote(t *testing.T) {
	tokenFilePath := setupTokenFile(t, "token data")
	defer os.Remove(tokenFilePath)
	fc := FakeConfig{}
	output, err := AddRemote(&fc, "testname", tokenFilePath)

	assert.Equal(t, "testname", fc.SavedName)
	assert.Equal(t, "token data", fc.SavedToken)
	assert.NoError(t, err)
	assert.Equal(t, "Successfully added! 'testname' is now your active remote.", output.ToPrettyOutput())
	assert.Equal(t, "testname", fc.ActivatedRemoteName)
}

func TestInactiveSecondAddRemote(t *testing.T) {
	tokenFilePath := setupTokenFile(t, "token data")
	defer os.Remove(tokenFilePath)
	fc := FakeConfig{Agents: []config.Remote{{Name: "Test"}}}
	output, err := AddRemote(&fc, "testname", tokenFilePath)

	assert.NoError(t, err)
	assert.Equal(t, "Successfully added!", output.ToPrettyOutput())
	assert.Equal(t, "", fc.ActivatedRemoteName)
}

func TestStripsWhitespaceAddRemote(t *testing.T) {
	tokenFilePath := setupTokenFile(t, "\n token data \n\n ")
	defer os.Remove(tokenFilePath)
	fc := FakeConfig{}
	_, err := AddRemote(&fc, "testname", tokenFilePath)

	assert.NoError(t, err)
	assert.Equal(t, "token data", fc.SavedToken)
}

func TestErroredExistingNameAddRemote(t *testing.T) {
	fc := FakeConfig{Agents: []config.Remote{{Name: "name"}}}
	output, err := AddRemote(&fc, "name", "unused")

	assert.Equal(t, "", output.ToPrettyOutput())
	assert.EqualError(t, err, "Name already exists")
}

func TestErroredInvalidNameAddRemote(t *testing.T) {
	fc := FakeConfig{}
	output, err := AddRemote(&fc, "bad name", "unused")
	assert.Equal(t, "", output.ToPrettyOutput())
	assert.EqualError(t, err, "Invalid name")

	_, err = AddRemote(&fc, "bad!", "unused")
	assert.EqualError(t, err, "Invalid name")

	_, err = AddRemote(&fc, "bad/", "unused")
	assert.EqualError(t, err, "Invalid name")
}

func TestErroredMissingFileAddRemote(t *testing.T) {
	fc := FakeConfig{}
	output, err := AddRemote(&fc, "name", "bad/file")
	assert.Equal(t, "", output.ToPrettyOutput())
	assert.EqualError(t, err, "open bad/file: no such file or directory")
}

func TestErroredConfigSaveAddRemote(t *testing.T) {
	tokenFilePath := setupTokenFile(t, "token data")
	defer os.Remove(tokenFilePath)
	fc := FakeConfig{ErrorForSave: errors.New("test error")}
	output, err := AddRemote(&fc, "name", tokenFilePath)

	assert.Equal(t, "", output.ToPrettyOutput())
	assert.EqualError(t, err, "test error")
}

func TestListRemotes(t *testing.T) {
	active := config.Remote{Name: "Active"}
	fc := FakeConfig{
		Agents: []config.Remote{
			{Name: "Test", Endpoint: "http://example.com"},
			active,
		},
		ActiveRemote: &active,
	}
	output := ListRemotes(&fc)

	lo, ok := output.(*ListOutput)
	if assert.True(t, ok) && assert.Len(t, lo.Rows, 2) {
		assert.Equal(t, "", lo.Rows[0]["Active"])
		assert.Equal(t, "Test", lo.Rows[0]["Name"])
		assert.Equal(t, "http://example.com", lo.Rows[0]["Endpoint"])

		assert.Equal(t, "*", lo.Rows[1]["Active"])
		assert.Equal(t, "Active", lo.Rows[1]["Name"])
	}
}

func TestNoActiveListRemotes(t *testing.T) {
	fc := FakeConfig{Agents: []config.Remote{{Name: "Test"}}}
	assert.NotPanics(t, func() {
		ListRemotes(&fc)
	})
}

func TestListRemotesNoRemotes(t *testing.T) {
	fc := FakeConfig{}
	output := ListRemotes(&fc)
	assert.Equal(t, "No remotes", output.ToPrettyOutput())
}

func TestSetActiveRemote(t *testing.T) {
	fc := FakeConfig{}
	output, err := SetActiveRemote(&fc, "Test")

	assert.Equal(t, "Test", fc.ActivatedRemoteName)
	assert.NoError(t, err)
	assert.Equal(t, "Success!", output.ToPrettyOutput())
}

func TestDescribeRemote(t *testing.T) {
	setupFactory()
	fakeClient.Metadata = agent.Metadata{
		Agent: agent.AgentMetadata{Version: "0.1"},
		Adapter: struct {
			Version   string `json:"version"`
			Type      string `json:"type"`
			IsHealthy bool   `json:"isHealthy"`
		}{"0.2", "Test", true},
	}
	fakeClient.Deployments = []agent.DeploymentResponseLite{
		{ID: 17, Name: "Test Deployment", ServiceIDs: []string{"1", "2"}},
	}
	r := config.Remote{Name: "Test", Endpoint: "http://example.com"}
	fc := FakeConfig{Agents: []config.Remote{r}}
	output, err := DescribeRemote(&fc, "Test")

	assert.NoError(t, err)
	if assert.Len(t, fakeFactory.NewedRemotes, 2) {
		assert.Equal(t, "Test", fakeFactory.NewedRemotes[0].Name)
	}

	co, ok := output.(*CombinedOutput)
	if assert.True(t, ok) && assert.Len(t, co.Outputs, 2) {
		do, ok := co.Outputs[0].Output.(DetailOutput)
		if assert.True(t, ok) {
			assert.Equal(t, "false", do.Details["Active"])
			assert.Equal(t, "Test", do.Details["Name"])
			assert.Equal(t, "http://example.com", do.Details["Endpoint"])
			assert.Equal(t, "0.1", do.Details["Agent Version"])
			assert.Equal(t, "0.2", do.Details["Adapter Version"])
			assert.Equal(t, "Test", do.Details["Adapter Type"])
			assert.Equal(t, "true", do.Details["Adapter Is Healthy"])
		}

		lo, ok := co.Outputs[1].Output.(*ListOutput)
		if assert.True(t, ok) && assert.Len(t, lo.Rows, 1) {
			r := lo.Rows[0]
			assert.Equal(t, "17", r["ID"])
			assert.Equal(t, "Test Deployment", r["Name"])
		}
	}
}

func TestErroredClientMetadataDescribeRemote(t *testing.T) {
	setupFactory()
	fakeClient.ErrorForMetadata = errors.New("test error")
	r := config.Remote{Name: "Test", Endpoint: "http://example.com"}
	fc := FakeConfig{Agents: []config.Remote{r}}

	output, err := DescribeRemote(&fc, "Test")
	assert.Equal(t, "", output.ToPrettyOutput())
	assert.EqualError(t, err, "test error")
}

func TestErroredClientDeploymentListDescribeRemote(t *testing.T) {
	setupFactory()
	fakeClient.ErrorForDeploymentList = errors.New("test error deployment list")
	r := config.Remote{Name: "Test", Endpoint: "http://example.com"}
	fc := FakeConfig{Agents: []config.Remote{r}}

	output, err := DescribeRemote(&fc, "Test")
	assert.Equal(t, "", output.ToPrettyOutput())
	assert.EqualError(t, err, "test error deployment list")
}

func TestErroredNonexistantDescribeRemote(t *testing.T) {
	fc := FakeConfig{}
	output, err := DescribeRemote(&fc, "Nonexistant")

	assert.EqualError(t, err, "the remote 'Nonexistant' does not exist")
	assert.Equal(t, "", output.ToPrettyOutput())
}

func TestErroredSetActiveRemote(t *testing.T) {
	fc := FakeConfig{ErrorForSetActive: errors.New("Name Not Found")}
	output, err := SetActiveRemote(&fc, "Bad")

	assert.Error(t, err)
	assert.Equal(t, "", output.ToPrettyOutput())
}

func TestGetRemoteToken(t *testing.T) {
	fc := FakeConfig{
		Agents: []config.Remote{{Name: "Test", Token: "Token\nText"}},
	}
	output, err := GetRemoteToken(&fc, "Test")

	assert.NoError(t, err)
	assert.Equal(t, "Token\nText", output.ToPrettyOutput())
}

func TestErroredGetRemoteToken(t *testing.T) {
	fc := FakeConfig{}
	output, err := GetRemoteToken(&fc, "nonexistant")

	assert.EqualError(t, err, "the remote 'nonexistant' does not exist")
	assert.Empty(t, output.ToPrettyOutput())
}
<|endoftext|>


<commit_before>package python

import (
	"bytes"
	"encoding/json"
	"fmt"
	"text/template"

	"sourcegraph.com/sourcegraph/srcgraph/config"
	"sourcegraph.com/sourcegraph/srcgraph/container"
	"sourcegraph.com/sourcegraph/srcgraph/dep2"
	"sourcegraph.com/sourcegraph/srcgraph/task2"
	"sourcegraph.com/sourcegraph/srcgraph/unit"
)

func init() {
	dep2.RegisterLister(&fauxPackage{}, dep2.DockerLister{defaultPythonEnv})
	dep2.RegisterResolver(pythonRequirementTargetType, defaultPythonEnv)
}

func (p *pythonEnv) BuildLister(dir string, unit unit.SourceUnit, c *config.Repository, x *task2.Context) (*container.Command, error) {
	dockerfile, err := p.depDockerfile()
	if err != nil {
		return nil, err
	}

	return &container.Command{
		Container: container.Container{
			Dockerfile: dockerfile,
			RunOptions: []string{"-v", dir + ":" + srcRoot},
			Cmd:        []string{"pydep-run.py", srcRoot},
			Stderr:     x.Stderr,
			Stdout:     x.Stdout,
		},
		Transform: func(orig []byte) ([]byte, error) {
			var reqs []requirement
			err := json.NewDecoder(bytes.NewReader(orig)).Decode(&reqs)
			if err != nil {
				return nil, err
			}
			deps := make([]*dep2.RawDependency, len(reqs))
			for i, req := range reqs {
				deps[i] = &dep2.RawDependency{
					TargetType: pythonRequirementTargetType,
					Target:     req,
				}
			}
			return json.Marshal(deps)
		},
	}, nil
}

type requirement struct {
	ProjectName string      `json:"project_name"`
	UnsafeName  string      `json:"unsafe_name"`
	Key         string      `json:"key"`
	Specs       [][2]string `json:"specs"`
	Extras      []string    `json:"extras"`
	RepoURL     string      `json:"repo_url"`
	Packages    []string    `json:"packages"`
	Modules     []string    `json:"modules"`
}

func (p *pythonEnv) Resolve(dep *dep2.RawDependency, c *config.Repository, x *task2.Context) (*dep2.ResolvedTarget, error) {
	switch dep.TargetType {
	case pythonRequirementTargetType:
		var req requirement
		reqJson, _ := json.Marshal(dep.Target)
		json.Unmarshal(reqJson, &req)

		toUnit := &fauxPackage{}
		return &dep2.ResolvedTarget{
			ToRepoCloneURL: req.RepoURL,
			ToUnit:         toUnit.Name(),
			ToUnitType:     unit.Type(toUnit),
		}, nil
	default:
		return nil, fmt.Errorf("Unexpected target type for Python: %+v", dep.TargetType)
	}
}

func (l *pythonEnv) depDockerfile() ([]byte, error) {
	var buf bytes.Buffer
	template.Must(template.New("").Parse(depDockerfile)).Execute(&buf, struct {
		Python string
	}{
		Python: l.PythonVersion,
	})
	return buf.Bytes(), nil
}

const pythonRequirementTargetType = "python-requirement"
const depDockerfile = `FROM ubuntu:13.10
RUN apt-get update
RUN apt-get install -qy curl
RUN apt-get install -qy git
RUN apt-get install -qy {{.Python}}
RUN ln -s $(which {{.Python}}) /usr/bin/python
RUN curl https://raw.github.com/pypa/pip/master/contrib/get-pip.py > get-pip.py
RUN python get-pip.py

RUN pip install git+git://github.com/sourcegraph/pydep@0.0
`
<commit_msg>use revision-specific url, and use the url that the previous url redirected to (the previous url was returning empty when curling)<commit_after>package python

import (
	"bytes"
	"encoding/json"
	"fmt"
	"text/template"

	"sourcegraph.com/sourcegraph/srcgraph/config"
	"sourcegraph.com/sourcegraph/srcgraph/container"
	"sourcegraph.com/sourcegraph/srcgraph/dep2"
	"sourcegraph.com/sourcegraph/srcgraph/task2"
	"sourcegraph.com/sourcegraph/srcgraph/unit"
)

func init() {
	dep2.RegisterLister(&fauxPackage{}, dep2.DockerLister{defaultPythonEnv})
	dep2.RegisterResolver(pythonRequirementTargetType, defaultPythonEnv)
}

func (p *pythonEnv) BuildLister(dir string, unit unit.SourceUnit, c *config.Repository, x *task2.Context) (*container.Command, error) {
	dockerfile, err := p.depDockerfile()
	if err != nil {
		return nil, err
	}

	return &container.Command{
		Container: container.Container{
			Dockerfile: dockerfile,
			RunOptions: []string{"-v", dir + ":" + srcRoot},
			Cmd:        []string{"pydep-run.py", srcRoot},
			Stderr:     x.Stderr,
			Stdout:     x.Stdout,
		},
		Transform: func(orig []byte) ([]byte, error) {
			var reqs []requirement
			err := json.NewDecoder(bytes.NewReader(orig)).Decode(&reqs)
			if err != nil {
				return nil, err
			}
			deps := make([]*dep2.RawDependency, len(reqs))
			for i, req := range reqs {
				deps[i] = &dep2.RawDependency{
					TargetType: pythonRequirementTargetType,
					Target:     req,
				}
			}
			return json.Marshal(deps)
		},
	}, nil
}

type requirement struct {
	ProjectName string      `json:"project_name"`
	UnsafeName  string      `json:"unsafe_name"`
	Key         string      `json:"key"`
	Specs       [][2]string `json:"specs"`
	Extras      []string    `json:"extras"`
	RepoURL     string      `json:"repo_url"`
	Packages    []string    `json:"packages"`
	Modules     []string    `json:"modules"`
}

func (p *pythonEnv) Resolve(dep *dep2.RawDependency, c *config.Repository, x *task2.Context) (*dep2.ResolvedTarget, error) {
	switch dep.TargetType {
	case pythonRequirementTargetType:
		var req requirement
		reqJson, _ := json.Marshal(dep.Target)
		json.Unmarshal(reqJson, &req)

		toUnit := &fauxPackage{}
		return &dep2.ResolvedTarget{
			ToRepoCloneURL: req.RepoURL,
			ToUnit:         toUnit.Name(),
			ToUnitType:     unit.Type(toUnit),
		}, nil
	default:
		return nil, fmt.Errorf("Unexpected target type for Python: %+v", dep.TargetType)
	}
}

func (l *pythonEnv) depDockerfile() ([]byte, error) {
	var buf bytes.Buffer
	template.Must(template.New("").Parse(depDockerfile)).Execute(&buf, struct {
		Python string
	}{
		Python: l.PythonVersion,
	})
	return buf.Bytes(), nil
}

const pythonRequirementTargetType = "python-requirement"
const depDockerfile = `FROM ubuntu:13.10
RUN apt-get update
RUN apt-get install -qy curl
RUN apt-get install -qy git
RUN apt-get install -qy {{.Python}}
RUN ln -s $(which {{.Python}}) /usr/bin/python
RUN curl https://raw.githubusercontent.com/pypa/pip/cdee19c77cf6514d42e2d1b7134f10b8ed36b63a/contrib/get-pip.py > get-pip.py
RUN python get-pip.py

RUN pip install git+git://github.com/sourcegraph/pydep@0.0
`
<|endoftext|>


<commit_before>package qemu

import (
	"context"
	"fmt"
	"log"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/hashicorp/packer/helper/multistep"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/template/interpolate"
)

// stepRun runs the virtual machine
type stepRun struct {
	BootDrive string
	Message   string
}

type qemuArgsTemplateData struct {
	HTTPIP      string
	HTTPPort    uint
	HTTPDir     string
	OutputDir   string
	Name        string
	SSHHostPort uint
}

func (s *stepRun) Run(_ context.Context, state multistep.StateBag) multistep.StepAction {
	driver := state.Get("driver").(Driver)
	ui := state.Get("ui").(packer.Ui)

	ui.Say(s.Message)

	command, err := getCommandArgs(s.BootDrive, state)
	if err != nil {
		err := fmt.Errorf("Error processing QemuArgs: %s", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	ui.Message(fmt.Sprintf("command:=%s", command))
	if err := driver.Qemu(command...); err != nil {
		err := fmt.Errorf("Error launching VM: %s", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	return multistep.ActionContinue
}

func (s *stepRun) Cleanup(state multistep.StateBag) {
	driver := state.Get("driver").(Driver)
	ui := state.Get("ui").(packer.Ui)

	if err := driver.Stop(); err != nil {
		ui.Error(fmt.Sprintf("Error shutting down VM: %s", err))
	}
}

func getCommandArgs(bootDrive string, state multistep.StateBag) ([]string, error) {
	config := state.Get("config").(*Config)
	isoPath := state.Get("iso_path").(string)
	vncIP := state.Get("vnc_ip").(string)
	vncPort := state.Get("vnc_port").(uint)
	ui := state.Get("ui").(packer.Ui)
	driver := state.Get("driver").(Driver)

	vnc := fmt.Sprintf("%s:%d", vncIP, vncPort-5900)
	vmName := config.VMName
	imgPath := filepath.Join(config.OutputDir, vmName)

	defaultArgs := make(map[string]interface{})
	var deviceArgs []string
	var driveArgs []string
	var sshHostPort uint

	defaultArgs["-name"] = vmName
	defaultArgs["-machine"] = fmt.Sprintf("type=%s", config.MachineType)
	if config.Comm.Type != "none" {
		sshHostPort = state.Get("sshHostPort").(uint)
		defaultArgs["-netdev"] = fmt.Sprintf("user,id=user.0,hostfwd=tcp::%v-:%d", sshHostPort, config.Comm.Port())
	} else {
		defaultArgs["-netdev"] = fmt.Sprintf("user,id=user.0")
	}

	qemuVersion, err := driver.Version()
	if err != nil {
		return nil, err
	}
	parts := strings.Split(qemuVersion, ".")
	qemuMajor, err := strconv.Atoi(parts[0])
	if err != nil {
		return nil, err
	}
	if qemuMajor >= 2 {
		if config.DiskInterface == "virtio-scsi" {
			deviceArgs = append(deviceArgs, "virtio-scsi-pci,id=scsi0", "scsi-hd,bus=scsi0.0,drive=drive0")
			driveArgs = append(driveArgs, fmt.Sprintf("if=none,file=%s,id=drive0,cache=%s,discard=%s,format=%s", imgPath, config.DiskCache, config.DiskDiscard, config.Format))
		} else {
			driveArgs = append(driveArgs, fmt.Sprintf("file=%s,if=%s,cache=%s,discard=%s,format=%s", imgPath, config.DiskInterface, config.DiskCache, config.DiskDiscard, config.Format))
		}
	} else {
		driveArgs = append(driveArgs, fmt.Sprintf("file=%s,if=%s,cache=%s,format=%s", imgPath, config.DiskInterface, config.DiskCache, config.Format))
	}

	ui.Message(fmt.Sprintf("config.NetDevice:=%s", config.NetDevice))

	deviceArgs = append(deviceArgs, fmt.Sprintf("%s,netdev=user.0", config.NetDevice))

	if config.Headless == true {
		vncIpRaw, vncIpOk := state.GetOk("vnc_ip")
		vncPortRaw, vncPortOk := state.GetOk("vnc_port")

		if vncIpOk && vncPortOk {
			vncIp := vncIpRaw.(string)
			vncPort := vncPortRaw.(uint)

			ui.Message(fmt.Sprintf(
				"The VM will be run headless, without a GUI. If you want to\n"+
					"view the screen of the VM, connect via VNC without a password to\n"+
					"vnc://%s:%d", vncIp, vncPort))
		} else {
			ui.Message("The VM will be run headless, without a GUI, as configured.\n" +
				"If the run isn't succeeding as you expect, please enable the GUI\n" +
				"to inspect the progress of the build.")
		}
	} else {
		if qemuMajor >= 2 {
			if !config.UseDefaultDisplay {
				defaultArgs["-display"] = "sdl"
			}
		} else {
			ui.Message("WARNING: The version of qemu  on your host doesn't support display mode.\n" +
				"The display parameter will be ignored.")
		}
	}

	defaultArgs["-device"] = deviceArgs
	defaultArgs["-drive"] = driveArgs

	if !config.DiskImage {
		defaultArgs["-cdrom"] = isoPath
	}
	defaultArgs["-boot"] = bootDrive
	defaultArgs["-m"] = "512M"
	defaultArgs["-vnc"] = vnc

	// Append the accelerator to the machine type if it is specified
	if config.Accelerator != "none" {
		defaultArgs["-machine"] = fmt.Sprintf("%s,accel=%s", defaultArgs["-machine"], config.Accelerator)
	} else {
		ui.Message("WARNING: The VM will be started with no hardware acceleration.\n" +
			"The installation may take considerably longer to finish.\n")
	}

	// Determine if we have a floppy disk to attach
	if floppyPathRaw, ok := state.GetOk("floppy_path"); ok {
		defaultArgs["-fda"] = floppyPathRaw.(string)
	} else {
		log.Println("Qemu Builder has no floppy files, not attaching a floppy.")
	}

	inArgs := make(map[string][]string)
	if len(config.QemuArgs) > 0 {
		ui.Say("Overriding defaults Qemu arguments with QemuArgs...")

		httpPort := state.Get("http_port").(uint)
		ctx := config.ctx
		if config.Comm.Type != "none" {
			ctx.Data = qemuArgsTemplateData{
				"10.0.2.2",
				httpPort,
				config.HTTPDir,
				config.OutputDir,
				config.VMName,
				sshHostPort,
			}
		} else {
			ctx.Data = qemuArgsTemplateData{
				HTTPIP:    "10.0.2.2",
				HTTPPort:  httpPort,
				HTTPDir:   config.HTTPDir,
				OutputDir: config.OutputDir,
				Name:      config.VMName,
			}
		}
		newQemuArgs, err := processArgs(config.QemuArgs, &ctx)
		if err != nil {
			return nil, err
		}

		// because qemu supports multiple appearances of the same
		// switch, just different values, each key in the args hash
		// will have an array of string values
		for _, qemuArgs := range newQemuArgs {
			key := qemuArgs[0]
			val := strings.Join(qemuArgs[1:], "")
			if _, ok := inArgs[key]; !ok {
				inArgs[key] = make([]string, 0)
			}
			if len(val) > 0 {
				inArgs[key] = append(inArgs[key], val)
			}
		}
	}

	// get any remaining missing default args from the default settings
	for key := range defaultArgs {
		if _, ok := inArgs[key]; !ok {
			arg := make([]string, 1)
			switch defaultArgs[key].(type) {
			case string:
				arg[0] = defaultArgs[key].(string)
			case []string:
				arg = defaultArgs[key].([]string)
			}
			inArgs[key] = arg
		}
	}

	// Flatten to array of strings
	outArgs := make([]string, 0)
	for key, values := range inArgs {
		if len(values) > 0 {
			for idx := range values {
				outArgs = append(outArgs, key, values[idx])
			}
		} else {
			outArgs = append(outArgs, key)
		}
	}

	return outArgs, nil
}

func processArgs(args [][]string, ctx *interpolate.Context) ([][]string, error) {
	var err error

	if args == nil {
		return make([][]string, 0), err
	}

	newArgs := make([][]string, len(args))
	for argsIdx, rowArgs := range args {
		parms := make([]string, len(rowArgs))
		newArgs[argsIdx] = parms
		for i, parm := range rowArgs {
			parms[i], err = interpolate.Render(parm, ctx)
			if err != nil {
				return nil, err
			}
		}
	}

	return newArgs, err
}
<commit_msg>revert accidental debugging change<commit_after>package qemu

import (
	"context"
	"fmt"
	"log"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/hashicorp/packer/helper/multistep"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/template/interpolate"
)

// stepRun runs the virtual machine
type stepRun struct {
	BootDrive string
	Message   string
}

type qemuArgsTemplateData struct {
	HTTPIP      string
	HTTPPort    uint
	HTTPDir     string
	OutputDir   string
	Name        string
	SSHHostPort uint
}

func (s *stepRun) Run(_ context.Context, state multistep.StateBag) multistep.StepAction {
	driver := state.Get("driver").(Driver)
	ui := state.Get("ui").(packer.Ui)

	ui.Say(s.Message)

	command, err := getCommandArgs(s.BootDrive, state)
	if err != nil {
		err := fmt.Errorf("Error processing QemuArgs: %s", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	if err := driver.Qemu(command...); err != nil {
		err := fmt.Errorf("Error launching VM: %s", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	return multistep.ActionContinue
}

func (s *stepRun) Cleanup(state multistep.StateBag) {
	driver := state.Get("driver").(Driver)
	ui := state.Get("ui").(packer.Ui)

	if err := driver.Stop(); err != nil {
		ui.Error(fmt.Sprintf("Error shutting down VM: %s", err))
	}
}

func getCommandArgs(bootDrive string, state multistep.StateBag) ([]string, error) {
	config := state.Get("config").(*Config)
	isoPath := state.Get("iso_path").(string)
	vncIP := state.Get("vnc_ip").(string)
	vncPort := state.Get("vnc_port").(uint)
	ui := state.Get("ui").(packer.Ui)
	driver := state.Get("driver").(Driver)

	vnc := fmt.Sprintf("%s:%d", vncIP, vncPort-5900)
	vmName := config.VMName
	imgPath := filepath.Join(config.OutputDir, vmName)

	defaultArgs := make(map[string]interface{})
	var deviceArgs []string
	var driveArgs []string
	var sshHostPort uint

	defaultArgs["-name"] = vmName
	defaultArgs["-machine"] = fmt.Sprintf("type=%s", config.MachineType)
	if config.Comm.Type != "none" {
		sshHostPort = state.Get("sshHostPort").(uint)
		defaultArgs["-netdev"] = fmt.Sprintf("user,id=user.0,hostfwd=tcp::%v-:%d", sshHostPort, config.Comm.Port())
	} else {
		defaultArgs["-netdev"] = fmt.Sprintf("user,id=user.0")
	}

	qemuVersion, err := driver.Version()
	if err != nil {
		return nil, err
	}
	parts := strings.Split(qemuVersion, ".")
	qemuMajor, err := strconv.Atoi(parts[0])
	if err != nil {
		return nil, err
	}
	if qemuMajor >= 2 {
		if config.DiskInterface == "virtio-scsi" {
			deviceArgs = append(deviceArgs, "virtio-scsi-pci,id=scsi0", "scsi-hd,bus=scsi0.0,drive=drive0")
			driveArgs = append(driveArgs, fmt.Sprintf("if=none,file=%s,id=drive0,cache=%s,discard=%s,format=%s", imgPath, config.DiskCache, config.DiskDiscard, config.Format))
		} else {
			driveArgs = append(driveArgs, fmt.Sprintf("file=%s,if=%s,cache=%s,discard=%s,format=%s", imgPath, config.DiskInterface, config.DiskCache, config.DiskDiscard, config.Format))
		}
	} else {
		driveArgs = append(driveArgs, fmt.Sprintf("file=%s,if=%s,cache=%s,format=%s", imgPath, config.DiskInterface, config.DiskCache, config.Format))
	}
	deviceArgs = append(deviceArgs, fmt.Sprintf("%s,netdev=user.0", config.NetDevice))

	if config.Headless == true {
		vncIpRaw, vncIpOk := state.GetOk("vnc_ip")
		vncPortRaw, vncPortOk := state.GetOk("vnc_port")

		if vncIpOk && vncPortOk {
			vncIp := vncIpRaw.(string)
			vncPort := vncPortRaw.(uint)

			ui.Message(fmt.Sprintf(
				"The VM will be run headless, without a GUI. If you want to\n"+
					"view the screen of the VM, connect via VNC without a password to\n"+
					"vnc://%s:%d", vncIp, vncPort))
		} else {
			ui.Message("The VM will be run headless, without a GUI, as configured.\n" +
				"If the run isn't succeeding as you expect, please enable the GUI\n" +
				"to inspect the progress of the build.")
		}
	} else {
		if qemuMajor >= 2 {
			if !config.UseDefaultDisplay {
				defaultArgs["-display"] = "sdl"
			}
		} else {
			ui.Message("WARNING: The version of qemu  on your host doesn't support display mode.\n" +
				"The display parameter will be ignored.")
		}
	}

	defaultArgs["-device"] = deviceArgs
	defaultArgs["-drive"] = driveArgs

	if !config.DiskImage {
		defaultArgs["-cdrom"] = isoPath
	}
	defaultArgs["-boot"] = bootDrive
	defaultArgs["-m"] = "512M"
	defaultArgs["-vnc"] = vnc

	// Append the accelerator to the machine type if it is specified
	if config.Accelerator != "none" {
		defaultArgs["-machine"] = fmt.Sprintf("%s,accel=%s", defaultArgs["-machine"], config.Accelerator)
	} else {
		ui.Message("WARNING: The VM will be started with no hardware acceleration.\n" +
			"The installation may take considerably longer to finish.\n")
	}

	// Determine if we have a floppy disk to attach
	if floppyPathRaw, ok := state.GetOk("floppy_path"); ok {
		defaultArgs["-fda"] = floppyPathRaw.(string)
	} else {
		log.Println("Qemu Builder has no floppy files, not attaching a floppy.")
	}

	inArgs := make(map[string][]string)
	if len(config.QemuArgs) > 0 {
		ui.Say("Overriding defaults Qemu arguments with QemuArgs...")

		httpPort := state.Get("http_port").(uint)
		ctx := config.ctx
		if config.Comm.Type != "none" {
			ctx.Data = qemuArgsTemplateData{
				"10.0.2.2",
				httpPort,
				config.HTTPDir,
				config.OutputDir,
				config.VMName,
				sshHostPort,
			}
		} else {
			ctx.Data = qemuArgsTemplateData{
				HTTPIP:    "10.0.2.2",
				HTTPPort:  httpPort,
				HTTPDir:   config.HTTPDir,
				OutputDir: config.OutputDir,
				Name:      config.VMName,
			}
		}
		newQemuArgs, err := processArgs(config.QemuArgs, &ctx)
		if err != nil {
			return nil, err
		}

		// because qemu supports multiple appearances of the same
		// switch, just different values, each key in the args hash
		// will have an array of string values
		for _, qemuArgs := range newQemuArgs {
			key := qemuArgs[0]
			val := strings.Join(qemuArgs[1:], "")
			if _, ok := inArgs[key]; !ok {
				inArgs[key] = make([]string, 0)
			}
			if len(val) > 0 {
				inArgs[key] = append(inArgs[key], val)
			}
		}
	}

	// get any remaining missing default args from the default settings
	for key := range defaultArgs {
		if _, ok := inArgs[key]; !ok {
			arg := make([]string, 1)
			switch defaultArgs[key].(type) {
			case string:
				arg[0] = defaultArgs[key].(string)
			case []string:
				arg = defaultArgs[key].([]string)
			}
			inArgs[key] = arg
		}
	}

	// Flatten to array of strings
	outArgs := make([]string, 0)
	for key, values := range inArgs {
		if len(values) > 0 {
			for idx := range values {
				outArgs = append(outArgs, key, values[idx])
			}
		} else {
			outArgs = append(outArgs, key)
		}
	}

	return outArgs, nil
}

func processArgs(args [][]string, ctx *interpolate.Context) ([][]string, error) {
	var err error

	if args == nil {
		return make([][]string, 0), err
	}

	newArgs := make([][]string, len(args))
	for argsIdx, rowArgs := range args {
		parms := make([]string, len(rowArgs))
		newArgs[argsIdx] = parms
		for i, parm := range rowArgs {
			parms[i], err = interpolate.Render(parm, ctx)
			if err != nil {
				return nil, err
			}
		}
	}

	return newArgs, err
}
<|endoftext|>


<commit_before>package s3proxy

import (
	"context"
	"errors"
	"fmt"
	"io"
	"log"
	"path"

	"github.com/buchgr/bazel-remote/cache"
	"github.com/buchgr/bazel-remote/cache/disk/casblob"
	"github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
)

type uploadReq struct {
	hash string
	size int64
	kind cache.EntryKind
	rc   io.ReadCloser
}

type s3Cache struct {
	mcore        *minio.Core
	prefix       string
	bucket       string
	uploadQueue  chan<- uploadReq
	accessLogger cache.Logger
	errorLogger  cache.Logger
	v2mode       bool
	objectKey    func(hash string, kind cache.EntryKind) string
}

var (
	cacheHits = promauto.NewCounter(prometheus.CounterOpts{
		Name: "bazel_remote_s3_cache_hits",
		Help: "The total number of s3 backend cache hits",
	})
	cacheMisses = promauto.NewCounter(prometheus.CounterOpts{
		Name: "bazel_remote_s3_cache_misses",
		Help: "The total number of s3 backend cache misses",
	})
)

// Used in place of minio's verbose "NoSuchKey" error.
var errNotFound = errors.New("NOT FOUND")

// New returns a new instance of the S3-API based cache
func New(
	// S3CloudStorageConfig struct fields:
	Endpoint string,
	Bucket string,
	Prefix string,
	AccessKeyID string,
	SecretAccessKey string,
	DisableSSL bool,
	IAMRoleEndpoint string,
	Region string,

	storageMode string, accessLogger cache.Logger,
	errorLogger cache.Logger, numUploaders, maxQueuedUploads int) cache.Proxy {

	fmt.Println("Using S3 backend.")

	var minioCore *minio.Core
	var err error

	if AccessKeyID != "" && SecretAccessKey != "" {
		// Initialize minio client object.
		opts := &minio.Options{
			Creds:  credentials.NewStaticV4(AccessKeyID, SecretAccessKey, ""),
			Secure: !DisableSSL,
			Region: Region,
		}
		minioCore, err = minio.NewCore(Endpoint, opts)
		if err != nil {
			log.Fatalln(err)
		}
	} else {
		// Initialize minio client object with IAM credentials
		opts := &minio.Options{
			// This config value may be empty.
			Creds: credentials.NewIAM(IAMRoleEndpoint),

			Region: Region,
			Secure: !DisableSSL,
		}
		minioCore, err = minio.NewCore(Endpoint, opts)
		if err != nil {
			log.Fatalln(err)
		}
	}

	if storageMode != "zstd" && storageMode != "uncompressed" {
		log.Fatalf("Unsupported storage mode for the s3proxy backend: %q, must be one of \"zstd\" or \"uncompressed\"",
			storageMode)
	}

	c := &s3Cache{
		mcore:        minioCore,
		prefix:       Prefix,
		bucket:       Bucket,
		accessLogger: accessLogger,
		errorLogger:  errorLogger,
		v2mode:       storageMode == "zstd",
	}

	if c.v2mode {
		c.objectKey = func(hash string, kind cache.EntryKind) string {
			return objectKeyV2(c.prefix, hash, kind)
		}
	} else {
		c.objectKey = func(hash string, kind cache.EntryKind) string {
			return objectKeyV1(c.prefix, hash, kind)
		}
	}

	if maxQueuedUploads > 0 && numUploaders > 0 {
		uploadQueue := make(chan uploadReq, maxQueuedUploads)
		for uploader := 0; uploader < numUploaders; uploader++ {
			go func() {
				for item := range uploadQueue {
					c.uploadFile(item)
				}
			}()
		}

		c.uploadQueue = uploadQueue
	}

	return c
}

func objectKeyV2(prefix string, hash string, kind cache.EntryKind) string {
	var baseKey string
	if kind == cache.CAS {
		// Use "cas.v2" to distinguish new from old format blobs.
		baseKey = path.Join("cas.v2", hash[:2], hash)
	} else {
		baseKey = path.Join(kind.String(), hash[:2], hash)
	}

	if prefix == "" {
		return baseKey
	}

	return path.Join(prefix, baseKey)
}

func objectKeyV1(prefix string, hash string, kind cache.EntryKind) string {
	if prefix == "" {
		return path.Join(kind.String(), hash[:2], hash)
	}

	return path.Join(prefix, kind.String(), hash[:2], hash)
}

// Helper function for logging responses
func logResponse(log cache.Logger, method, bucket, key string, err error) {
	status := "OK"
	if err != nil {
		status = err.Error()
	}

	log.Printf("S3 %s %s %s %s", method, bucket, key, status)
}

func (c *s3Cache) uploadFile(item uploadReq) {
	_, err := c.mcore.PutObject(
		context.Background(),
		c.bucket,                          // bucketName
		c.objectKey(item.hash, item.kind), // objectName
		item.rc,                           // reader
		item.size,                         // objectSize
		"",                                // md5base64
		"",                                // sha256
		minio.PutObjectOptions{
			UserMetadata: map[string]string{
				"Content-Type": "application/octet-stream",
			},
		}, // metadata
	)

	logResponse(c.accessLogger, "UPLOAD", c.bucket, c.objectKey(item.hash, item.kind), err)

	item.rc.Close()
}

func (c *s3Cache) Put(kind cache.EntryKind, hash string, size int64, rc io.ReadCloser) {
	if c.uploadQueue == nil {
		rc.Close()
		return
	}

	select {
	case c.uploadQueue <- uploadReq{
		hash: hash,
		size: size,
		kind: kind,
		rc:   rc,
	}:
	default:
		c.errorLogger.Printf("too many uploads queued\n")
		rc.Close()
	}
}

func (c *s3Cache) Get(ctx context.Context, kind cache.EntryKind, hash string) (io.ReadCloser, int64, error) {

	rc, info, _, err := c.mcore.GetObject(
		ctx,
		c.bucket,                 // bucketName
		c.objectKey(hash, kind),  // objectName
		minio.GetObjectOptions{}, // opts
	)
	if err != nil {
		if minio.ToErrorResponse(err).Code == "NoSuchKey" {
			cacheMisses.Inc()
			logResponse(c.accessLogger, "DOWNLOAD", c.bucket, c.objectKey(hash, kind), errNotFound)
			return nil, -1, nil
		}
		cacheMisses.Inc()
		logResponse(c.accessLogger, "DOWNLOAD", c.bucket, c.objectKey(hash, kind), err)
		return nil, -1, err
	}
	cacheHits.Inc()

	logResponse(c.accessLogger, "DOWNLOAD", c.bucket, c.objectKey(hash, kind), nil)

	if kind == cache.CAS && c.v2mode {
		return casblob.ExtractLogicalSize(rc)
	}

	return rc, info.Size, nil
}

func (c *s3Cache) Contains(ctx context.Context, kind cache.EntryKind, hash string) (bool, int64) {
	size := int64(-1)
	exists := false

	s, err := c.mcore.StatObject(
		ctx,
		c.bucket,                  // bucketName
		c.objectKey(hash, kind),   // objectName
		minio.StatObjectOptions{}, // opts
	)

	exists = (err == nil)
	if err != nil {
		err = errNotFound
	} else if kind != cache.CAS || !c.v2mode {
		size = s.Size
	}

	logResponse(c.accessLogger, "CONTAINS", c.bucket, c.objectKey(hash, kind), err)

	return exists, size
}
<commit_msg>[s3proxy] Simplify minio client construction<commit_after>package s3proxy

import (
	"context"
	"errors"
	"fmt"
	"io"
	"log"
	"path"

	"github.com/buchgr/bazel-remote/cache"
	"github.com/buchgr/bazel-remote/cache/disk/casblob"
	"github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
)

type uploadReq struct {
	hash string
	size int64
	kind cache.EntryKind
	rc   io.ReadCloser
}

type s3Cache struct {
	mcore        *minio.Core
	prefix       string
	bucket       string
	uploadQueue  chan<- uploadReq
	accessLogger cache.Logger
	errorLogger  cache.Logger
	v2mode       bool
	objectKey    func(hash string, kind cache.EntryKind) string
}

var (
	cacheHits = promauto.NewCounter(prometheus.CounterOpts{
		Name: "bazel_remote_s3_cache_hits",
		Help: "The total number of s3 backend cache hits",
	})
	cacheMisses = promauto.NewCounter(prometheus.CounterOpts{
		Name: "bazel_remote_s3_cache_misses",
		Help: "The total number of s3 backend cache misses",
	})
)

// Used in place of minio's verbose "NoSuchKey" error.
var errNotFound = errors.New("NOT FOUND")

// New returns a new instance of the S3-API based cache
func New(
	// S3CloudStorageConfig struct fields:
	Endpoint string,
	Bucket string,
	Prefix string,
	AccessKeyID string,
	SecretAccessKey string,
	DisableSSL bool,
	IAMRoleEndpoint string,
	Region string,

	storageMode string, accessLogger cache.Logger,
	errorLogger cache.Logger, numUploaders, maxQueuedUploads int) cache.Proxy {

	fmt.Println("Using S3 backend.")

	var minioCore *minio.Core
	var err error
	var creds *credentials.Credentials

	if AccessKeyID != "" && SecretAccessKey != "" {
		creds = credentials.NewStaticV4(AccessKeyID, SecretAccessKey, "")
	} else {
		creds = credentials.NewIAM(IAMRoleEndpoint)
	}

	// Initialize minio client with credentials
	opts := &minio.Options{
		Creds: creds,

		Region: Region,
		Secure: !DisableSSL,
	}
	minioCore, err = minio.NewCore(Endpoint, opts)
	if err != nil {
		log.Fatalln(err)
	}

	if storageMode != "zstd" && storageMode != "uncompressed" {
		log.Fatalf("Unsupported storage mode for the s3proxy backend: %q, must be one of \"zstd\" or \"uncompressed\"",
			storageMode)
	}

	c := &s3Cache{
		mcore:        minioCore,
		prefix:       Prefix,
		bucket:       Bucket,
		accessLogger: accessLogger,
		errorLogger:  errorLogger,
		v2mode:       storageMode == "zstd",
	}

	if c.v2mode {
		c.objectKey = func(hash string, kind cache.EntryKind) string {
			return objectKeyV2(c.prefix, hash, kind)
		}
	} else {
		c.objectKey = func(hash string, kind cache.EntryKind) string {
			return objectKeyV1(c.prefix, hash, kind)
		}
	}

	if maxQueuedUploads > 0 && numUploaders > 0 {
		uploadQueue := make(chan uploadReq, maxQueuedUploads)
		for uploader := 0; uploader < numUploaders; uploader++ {
			go func() {
				for item := range uploadQueue {
					c.uploadFile(item)
				}
			}()
		}

		c.uploadQueue = uploadQueue
	}

	return c
}

func objectKeyV2(prefix string, hash string, kind cache.EntryKind) string {
	var baseKey string
	if kind == cache.CAS {
		// Use "cas.v2" to distinguish new from old format blobs.
		baseKey = path.Join("cas.v2", hash[:2], hash)
	} else {
		baseKey = path.Join(kind.String(), hash[:2], hash)
	}

	if prefix == "" {
		return baseKey
	}

	return path.Join(prefix, baseKey)
}

func objectKeyV1(prefix string, hash string, kind cache.EntryKind) string {
	if prefix == "" {
		return path.Join(kind.String(), hash[:2], hash)
	}

	return path.Join(prefix, kind.String(), hash[:2], hash)
}

// Helper function for logging responses
func logResponse(log cache.Logger, method, bucket, key string, err error) {
	status := "OK"
	if err != nil {
		status = err.Error()
	}

	log.Printf("S3 %s %s %s %s", method, bucket, key, status)
}

func (c *s3Cache) uploadFile(item uploadReq) {
	_, err := c.mcore.PutObject(
		context.Background(),
		c.bucket,                          // bucketName
		c.objectKey(item.hash, item.kind), // objectName
		item.rc,                           // reader
		item.size,                         // objectSize
		"",                                // md5base64
		"",                                // sha256
		minio.PutObjectOptions{
			UserMetadata: map[string]string{
				"Content-Type": "application/octet-stream",
			},
		}, // metadata
	)

	logResponse(c.accessLogger, "UPLOAD", c.bucket, c.objectKey(item.hash, item.kind), err)

	item.rc.Close()
}

func (c *s3Cache) Put(kind cache.EntryKind, hash string, size int64, rc io.ReadCloser) {
	if c.uploadQueue == nil {
		rc.Close()
		return
	}

	select {
	case c.uploadQueue <- uploadReq{
		hash: hash,
		size: size,
		kind: kind,
		rc:   rc,
	}:
	default:
		c.errorLogger.Printf("too many uploads queued\n")
		rc.Close()
	}
}

func (c *s3Cache) Get(ctx context.Context, kind cache.EntryKind, hash string) (io.ReadCloser, int64, error) {

	rc, info, _, err := c.mcore.GetObject(
		ctx,
		c.bucket,                 // bucketName
		c.objectKey(hash, kind),  // objectName
		minio.GetObjectOptions{}, // opts
	)
	if err != nil {
		if minio.ToErrorResponse(err).Code == "NoSuchKey" {
			cacheMisses.Inc()
			logResponse(c.accessLogger, "DOWNLOAD", c.bucket, c.objectKey(hash, kind), errNotFound)
			return nil, -1, nil
		}
		cacheMisses.Inc()
		logResponse(c.accessLogger, "DOWNLOAD", c.bucket, c.objectKey(hash, kind), err)
		return nil, -1, err
	}
	cacheHits.Inc()

	logResponse(c.accessLogger, "DOWNLOAD", c.bucket, c.objectKey(hash, kind), nil)

	if kind == cache.CAS && c.v2mode {
		return casblob.ExtractLogicalSize(rc)
	}

	return rc, info.Size, nil
}

func (c *s3Cache) Contains(ctx context.Context, kind cache.EntryKind, hash string) (bool, int64) {
	size := int64(-1)
	exists := false

	s, err := c.mcore.StatObject(
		ctx,
		c.bucket,                  // bucketName
		c.objectKey(hash, kind),   // objectName
		minio.StatObjectOptions{}, // opts
	)

	exists = (err == nil)
	if err != nil {
		err = errNotFound
	} else if kind != cache.CAS || !c.v2mode {
		size = s.Size
	}

	logResponse(c.accessLogger, "CONTAINS", c.bucket, c.objectKey(hash, kind), err)

	return exists, size
}
<|endoftext|>


<commit_before>package sms

import (
	"log"
	"net/http"
	"os"

	"github.com/julienschmidt/httprouter"
	"github.com/votinginfoproject/sms-web/queue"
)

var q queue.ExternalQueueService

func WireUp(eqs queue.ExternalQueueService) {
	q = eqs
	q.Connect()
}

func Receive(res http.ResponseWriter, req *http.Request, params httprouter.Params) {
	sid := req.Form["AccountSid"]

	if len(sid) == 0 || sid[0] != os.Getenv("TWILIO_SID") {
		res.WriteHeader(http.StatusForbidden)
		log.Print("[FORBIDDEN] : ", req)
	} else {
		number := req.Form["From"][0]
		message := req.Form["Body"][0]
		q.Enqueue(number, message)
	}
}
<commit_msg>nicer forbidden logging<commit_after>package sms

import (
	"log"
	"net/http"
	"os"

	"github.com/julienschmidt/httprouter"
	"github.com/votinginfoproject/sms-web/queue"
)

var q queue.ExternalQueueService

func WireUp(eqs queue.ExternalQueueService) {
	q = eqs
	q.Connect()
}

func Receive(res http.ResponseWriter, req *http.Request, params httprouter.Params) {
	sid := req.Form["AccountSid"]

	if len(sid) == 0 || sid[0] != os.Getenv("TWILIO_SID") {
		res.WriteHeader(http.StatusForbidden)
		log.Printf("[FORBIDDEN] Method: %s - Path: %s - Host: %s, FormData: %s", req.Method, req.URL.RequestURI(), req.Host, req.Form)
	} else {
		number := req.Form["From"][0]
		message := req.Form["Body"][0]
		q.Enqueue(number, message)
	}
}
<|endoftext|>


<commit_before>package nameserver

import (
	"bytes"
	"fmt"
	"math/rand"
	"net"
	"sort"
	"strings"
	"time"

	"github.com/miekg/dns"

	"github.com/weaveworks/weave/net/address"
)

const (
	topDomain        = "."
	reverseDNSdomain = "in-addr.arpa."
	etcResolvConf    = "/etc/resolv.conf"
	udpBuffSize      = uint16(4096)
	minUDPSize       = 512

	DefaultPort          = 53
	DefaultTTL           = 1
	DefaultClientTimeout = 5 * time.Second
)

type DNSServer struct {
	ns     *Nameserver
	domain string
	ttl    uint32
	port   int

	servers   []*dns.Server
	upstream  *dns.ClientConfig
	tcpClient *dns.Client
	udpClient *dns.Client
}

func NewDNSServer(ns *Nameserver, domain string, port int, ttl uint32,
	clientTimeout time.Duration) (*DNSServer, error) {

	s := &DNSServer{
		ns:        ns,
		domain:    dns.Fqdn(domain),
		ttl:       ttl,
		port:      port,
		tcpClient: &dns.Client{Net: "tcp", ReadTimeout: clientTimeout},
		udpClient: &dns.Client{Net: "udp", ReadTimeout: clientTimeout, UDPSize: udpBuffSize},
	}
	var err error
	if s.upstream, err = dns.ClientConfigFromFile(etcResolvConf); err != nil {
		return nil, err
	}

	err = s.listen(port)
	return s, err
}

func (d *DNSServer) String() string {
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "WeaveDNS (%s)\n", d.ns.ourName)
	fmt.Fprintf(&buf, "  listening on port %d, for domain %s\n", d.port, d.domain)
	fmt.Fprintf(&buf, "  response ttl %d\n", d.ttl)
	return buf.String()
}

func (d *DNSServer) listen(port int) error {
	udpListener, err := net.ListenPacket("udp", fmt.Sprintf(":%d", port))
	if err != nil {
		return err
	}
	udpServer := &dns.Server{PacketConn: udpListener, Handler: d.createMux(d.udpClient, minUDPSize)}

	tcpListener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
	if err != nil {
		udpServer.Shutdown()
		return err
	}
	tcpServer := &dns.Server{Listener: tcpListener, Handler: d.createMux(d.tcpClient, -1)}

	d.servers = []*dns.Server{udpServer, tcpServer}
	return nil
}

func (d *DNSServer) ActivateAndServe() {
	for _, server := range d.servers {
		go func(server *dns.Server) {
			server.ActivateAndServe()
		}(server)
	}
}

func (d *DNSServer) Stop() error {
	for _, server := range d.servers {
		if err := server.Shutdown(); err != nil {
			return err
		}
	}
	return nil
}

func (d *DNSServer) errorResponse(r *dns.Msg, code int, w dns.ResponseWriter) {
	m := dns.Msg{}
	m.SetReply(r)
	m.RecursionAvailable = true
	m.Rcode = code

	d.ns.debugf("error response: %+v", m)
	if err := w.WriteMsg(&m); err != nil {
		d.ns.infof("error responding: %v", err)
	}
}

func (d *DNSServer) createMux(client *dns.Client, defaultMaxResponseSize int) *dns.ServeMux {
	m := dns.NewServeMux()
	m.HandleFunc(d.domain, d.handleLocal(defaultMaxResponseSize))
	m.HandleFunc(reverseDNSdomain, d.handleReverse(client, defaultMaxResponseSize))
	m.HandleFunc(topDomain, d.handleRecursive(client, defaultMaxResponseSize))
	return m
}

func (d *DNSServer) handleLocal(defaultMaxResponseSize int) func(dns.ResponseWriter, *dns.Msg) {
	return func(w dns.ResponseWriter, req *dns.Msg) {
		d.ns.debugf("local request: %+v", *req)
		if len(req.Question) != 1 || req.Question[0].Qtype != dns.TypeA {
			d.errorResponse(req, dns.RcodeNameError, w)
			return
		}

		hostname := dns.Fqdn(req.Question[0].Name)
		if strings.Count(hostname, ".") == 1 {
			hostname = hostname + d.domain
		}

		addrs := d.ns.Lookup(hostname)

		response := dns.Msg{}
		response.RecursionAvailable = true
		response.Authoritative = true
		response.SetReply(req)
		response.Answer = make([]dns.RR, len(addrs))

		header := dns.RR_Header{
			Name:   req.Question[0].Name,
			Rrtype: dns.TypeA,
			Class:  dns.ClassINET,
			Ttl:    d.ttl,
		}

		for i, addr := range addrs {
			ip := addr.IP4()
			response.Answer[i] = &dns.A{Hdr: header, A: ip}
		}

		shuffleAnswers(&response.Answer)
		maxResponseSize := getMaxResponseSize(req, defaultMaxResponseSize)
		truncateResponse(&response, maxResponseSize)

		d.ns.debugf("response: %+v", response)
		if err := w.WriteMsg(&response); err != nil {
			d.ns.infof("error responding: %v", err)
		}
	}
}

func (d *DNSServer) handleReverse(client *dns.Client, defaultMaxResponseSize int) func(dns.ResponseWriter, *dns.Msg) {
	return func(w dns.ResponseWriter, req *dns.Msg) {
		d.ns.debugf("reverse request: %+v", *req)
		if len(req.Question) != 1 || req.Question[0].Qtype != dns.TypePTR {
			d.errorResponse(req, dns.RcodeNameError, w)
			return
		}

		ipStr := strings.TrimSuffix(req.Question[0].Name, "."+reverseDNSdomain)
		ip, err := address.ParseIP(ipStr)
		if err != nil {
			d.errorResponse(req, dns.RcodeNameError, w)
			return
		}

		hostname, err := d.ns.ReverseLookup(ip.Reverse())
		if err != nil {
			d.handleRecursive(client, defaultMaxResponseSize)(w, req)
			return
		}

		response := dns.Msg{}
		response.RecursionAvailable = true
		response.Authoritative = true
		response.SetReply(req)

		header := dns.RR_Header{
			Name:   req.Question[0].Name,
			Rrtype: dns.TypePTR,
			Class:  dns.ClassINET,
			Ttl:    d.ttl,
		}

		response.Answer = []dns.RR{&dns.PTR{
			Hdr: header,
			Ptr: hostname,
		}}

		maxResponseSize := getMaxResponseSize(req, defaultMaxResponseSize)
		truncateResponse(&response, maxResponseSize)

		d.ns.debugf("response: %+v", response)
		if err := w.WriteMsg(&response); err != nil {
			d.ns.infof("error responding: %v", err)
		}
	}
}

func (d *DNSServer) handleRecursive(client *dns.Client, defaultMaxResponseSize int) func(dns.ResponseWriter, *dns.Msg) {
	return func(w dns.ResponseWriter, req *dns.Msg) {
		d.ns.debugf("recursive request: %+v", *req)

		// Resolve unqualified names locally
		if len(req.Question) == 1 && req.Question[0].Qtype == dns.TypeA {
			hostname := dns.Fqdn(req.Question[0].Name)
			if strings.Count(hostname, ".") == 1 {
				d.handleLocal(defaultMaxResponseSize)(w, req)
				return
			}
		}

		for _, server := range d.upstream.Servers {
			reqCopy := *req
			reqCopy.Id = dns.Id()
			response, _, err := client.Exchange(&reqCopy, fmt.Sprintf("%s:%s", server, d.upstream.Port))
			if err != nil || response == nil {
				d.ns.debugf("error trying %s: %v", server, err)
				continue
			}
			if response.Rcode != dns.RcodeSuccess && !response.Authoritative {
				d.ns.debugf("non-authoritative error trying %s: %v", server, response.Rcode)
				continue
			}
			d.ns.debugf("response: %+v", response)
			response.SetReply(req)
			if err := w.WriteMsg(response); err != nil {
				d.ns.infof("error responding: %v", err)
			}
			return
		}

		d.errorResponse(req, dns.RcodeServerFailure, w)
	}
}

func shuffleAnswers(answers *[]dns.RR) {
	if len(*answers) <= 1 {
		return
	}

	for i := range *answers {
		j := rand.Intn(i + 1)
		(*answers)[i], (*answers)[j] = (*answers)[j], (*answers)[i]
	}
}

func truncateResponse(response *dns.Msg, maxSize int) {
	if len(response.Answer) <= 1 || maxSize <= 0 {
		return
	}

	// take a copy of answers, as we're going to mutate response
	answers := response.Answer

	// search for smallest i that is too big
	i := sort.Search(len(response.Answer), func(i int) bool {
		// return true if too big
		response.Answer = answers[:i+1]
		return response.Len() > maxSize
	})
	if i == len(answers) {
		response.Answer = answers
		return
	}

	response.Answer = answers[:i]
	response.Truncated = true
}

func getMaxResponseSize(req *dns.Msg, defaultMaxResponseSize int) int {
	if opt := req.IsEdns0(); opt != nil {
		return int(opt.UDPSize())
	}
	return defaultMaxResponseSize
}
<commit_msg>Use Msg.Copy!<commit_after>package nameserver

import (
	"bytes"
	"fmt"
	"math/rand"
	"net"
	"sort"
	"strings"
	"time"

	"github.com/miekg/dns"

	"github.com/weaveworks/weave/net/address"
)

const (
	topDomain        = "."
	reverseDNSdomain = "in-addr.arpa."
	etcResolvConf    = "/etc/resolv.conf"
	udpBuffSize      = uint16(4096)
	minUDPSize       = 512

	DefaultPort          = 53
	DefaultTTL           = 1
	DefaultClientTimeout = 5 * time.Second
)

type DNSServer struct {
	ns     *Nameserver
	domain string
	ttl    uint32
	port   int

	servers   []*dns.Server
	upstream  *dns.ClientConfig
	tcpClient *dns.Client
	udpClient *dns.Client
}

func NewDNSServer(ns *Nameserver, domain string, port int, ttl uint32,
	clientTimeout time.Duration) (*DNSServer, error) {

	s := &DNSServer{
		ns:        ns,
		domain:    dns.Fqdn(domain),
		ttl:       ttl,
		port:      port,
		tcpClient: &dns.Client{Net: "tcp", ReadTimeout: clientTimeout},
		udpClient: &dns.Client{Net: "udp", ReadTimeout: clientTimeout, UDPSize: udpBuffSize},
	}
	var err error
	if s.upstream, err = dns.ClientConfigFromFile(etcResolvConf); err != nil {
		return nil, err
	}

	err = s.listen(port)
	return s, err
}

func (d *DNSServer) String() string {
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "WeaveDNS (%s)\n", d.ns.ourName)
	fmt.Fprintf(&buf, "  listening on port %d, for domain %s\n", d.port, d.domain)
	fmt.Fprintf(&buf, "  response ttl %d\n", d.ttl)
	return buf.String()
}

func (d *DNSServer) listen(port int) error {
	udpListener, err := net.ListenPacket("udp", fmt.Sprintf(":%d", port))
	if err != nil {
		return err
	}
	udpServer := &dns.Server{PacketConn: udpListener, Handler: d.createMux(d.udpClient, minUDPSize)}

	tcpListener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
	if err != nil {
		udpServer.Shutdown()
		return err
	}
	tcpServer := &dns.Server{Listener: tcpListener, Handler: d.createMux(d.tcpClient, -1)}

	d.servers = []*dns.Server{udpServer, tcpServer}
	return nil
}

func (d *DNSServer) ActivateAndServe() {
	for _, server := range d.servers {
		go func(server *dns.Server) {
			server.ActivateAndServe()
		}(server)
	}
}

func (d *DNSServer) Stop() error {
	for _, server := range d.servers {
		if err := server.Shutdown(); err != nil {
			return err
		}
	}
	return nil
}

func (d *DNSServer) errorResponse(r *dns.Msg, code int, w dns.ResponseWriter) {
	m := dns.Msg{}
	m.SetReply(r)
	m.RecursionAvailable = true
	m.Rcode = code

	d.ns.debugf("error response: %+v", m)
	if err := w.WriteMsg(&m); err != nil {
		d.ns.infof("error responding: %v", err)
	}
}

func (d *DNSServer) createMux(client *dns.Client, defaultMaxResponseSize int) *dns.ServeMux {
	m := dns.NewServeMux()
	m.HandleFunc(d.domain, d.handleLocal(defaultMaxResponseSize))
	m.HandleFunc(reverseDNSdomain, d.handleReverse(client, defaultMaxResponseSize))
	m.HandleFunc(topDomain, d.handleRecursive(client, defaultMaxResponseSize))
	return m
}

func (d *DNSServer) handleLocal(defaultMaxResponseSize int) func(dns.ResponseWriter, *dns.Msg) {
	return func(w dns.ResponseWriter, req *dns.Msg) {
		d.ns.debugf("local request: %+v", *req)
		if len(req.Question) != 1 || req.Question[0].Qtype != dns.TypeA {
			d.errorResponse(req, dns.RcodeNameError, w)
			return
		}

		hostname := dns.Fqdn(req.Question[0].Name)
		if strings.Count(hostname, ".") == 1 {
			hostname = hostname + d.domain
		}

		addrs := d.ns.Lookup(hostname)

		response := dns.Msg{}
		response.RecursionAvailable = true
		response.Authoritative = true
		response.SetReply(req)
		response.Answer = make([]dns.RR, len(addrs))

		header := dns.RR_Header{
			Name:   req.Question[0].Name,
			Rrtype: dns.TypeA,
			Class:  dns.ClassINET,
			Ttl:    d.ttl,
		}

		for i, addr := range addrs {
			ip := addr.IP4()
			response.Answer[i] = &dns.A{Hdr: header, A: ip}
		}

		shuffleAnswers(&response.Answer)
		maxResponseSize := getMaxResponseSize(req, defaultMaxResponseSize)
		truncateResponse(&response, maxResponseSize)

		d.ns.debugf("response: %+v", response)
		if err := w.WriteMsg(&response); err != nil {
			d.ns.infof("error responding: %v", err)
		}
	}
}

func (d *DNSServer) handleReverse(client *dns.Client, defaultMaxResponseSize int) func(dns.ResponseWriter, *dns.Msg) {
	return func(w dns.ResponseWriter, req *dns.Msg) {
		d.ns.debugf("reverse request: %+v", *req)
		if len(req.Question) != 1 || req.Question[0].Qtype != dns.TypePTR {
			d.errorResponse(req, dns.RcodeNameError, w)
			return
		}

		ipStr := strings.TrimSuffix(req.Question[0].Name, "."+reverseDNSdomain)
		ip, err := address.ParseIP(ipStr)
		if err != nil {
			d.errorResponse(req, dns.RcodeNameError, w)
			return
		}

		hostname, err := d.ns.ReverseLookup(ip.Reverse())
		if err != nil {
			d.handleRecursive(client, defaultMaxResponseSize)(w, req)
			return
		}

		response := dns.Msg{}
		response.RecursionAvailable = true
		response.Authoritative = true
		response.SetReply(req)

		header := dns.RR_Header{
			Name:   req.Question[0].Name,
			Rrtype: dns.TypePTR,
			Class:  dns.ClassINET,
			Ttl:    d.ttl,
		}

		response.Answer = []dns.RR{&dns.PTR{
			Hdr: header,
			Ptr: hostname,
		}}

		maxResponseSize := getMaxResponseSize(req, defaultMaxResponseSize)
		truncateResponse(&response, maxResponseSize)

		d.ns.debugf("response: %+v", response)
		if err := w.WriteMsg(&response); err != nil {
			d.ns.infof("error responding: %v", err)
		}
	}
}

func (d *DNSServer) handleRecursive(client *dns.Client, defaultMaxResponseSize int) func(dns.ResponseWriter, *dns.Msg) {
	return func(w dns.ResponseWriter, req *dns.Msg) {
		d.ns.debugf("recursive request: %+v", *req)

		// Resolve unqualified names locally
		if len(req.Question) == 1 && req.Question[0].Qtype == dns.TypeA {
			hostname := dns.Fqdn(req.Question[0].Name)
			if strings.Count(hostname, ".") == 1 {
				d.handleLocal(defaultMaxResponseSize)(w, req)
				return
			}
		}

		for _, server := range d.upstream.Servers {
			reqCopy := req.Copy()
			reqCopy.Id = dns.Id()
			response, _, err := client.Exchange(reqCopy, fmt.Sprintf("%s:%s", server, d.upstream.Port))
			if err != nil || response == nil {
				d.ns.debugf("error trying %s: %v", server, err)
				continue
			}
			if response.Rcode != dns.RcodeSuccess && !response.Authoritative {
				d.ns.debugf("non-authoritative error trying %s: %v", server, response.Rcode)
				continue
			}
			d.ns.debugf("response: %+v", response)
			response.SetReply(req)
			if err := w.WriteMsg(response); err != nil {
				d.ns.infof("error responding: %v", err)
			}
			return
		}

		d.errorResponse(req, dns.RcodeServerFailure, w)
	}
}

func shuffleAnswers(answers *[]dns.RR) {
	if len(*answers) <= 1 {
		return
	}

	for i := range *answers {
		j := rand.Intn(i + 1)
		(*answers)[i], (*answers)[j] = (*answers)[j], (*answers)[i]
	}
}

func truncateResponse(response *dns.Msg, maxSize int) {
	if len(response.Answer) <= 1 || maxSize <= 0 {
		return
	}

	// take a copy of answers, as we're going to mutate response
	answers := response.Answer

	// search for smallest i that is too big
	i := sort.Search(len(response.Answer), func(i int) bool {
		// return true if too big
		response.Answer = answers[:i+1]
		return response.Len() > maxSize
	})
	if i == len(answers) {
		response.Answer = answers
		return
	}

	response.Answer = answers[:i]
	response.Truncated = true
}

func getMaxResponseSize(req *dns.Msg, defaultMaxResponseSize int) int {
	if opt := req.IsEdns0(); opt != nil {
		return int(opt.UDPSize())
	}
	return defaultMaxResponseSize
}
<|endoftext|>


<commit_before>package sevendays

import (
	"github.com/juju/gocharm/charmbits/service"
	"github.com/juju/gocharm/hook"
	"github.com/mever/sevendays/httpd"
	"gopkg.in/juju/charm.v5"
)

type Charm struct {
	svc service.Service
	ctx *hook.Context
}

func RegisterHooks(r *hook.Registry) {
	c := Charm{}
	r.RegisterContext(c.setContext, nil)
	r.RegisterHook("start", c.start)
	r.RegisterHook("upgrade-charm", c.start)
	r.RegisterHook("config-changed", c.configChanged)
	c.svc.Register(r.Clone("service"), "", c.startService)

	r.RegisterConfig("user", charm.Option{
		Type:        "string",
		Description: "Provide Steam user name",
		Default:     "",
	})
}

func (c *Charm) start() error {
	return c.svc.Start(c.ctx.CharmDir)
}

func (c *Charm) configChanged() error {
	user, _ := c.ctx.GetConfigString("user")
	ss := httpd.State{
		User: user,
	}

	var reply string
	err := c.svc.Call("RpcServer.Set", ss, &reply)
	c.ctx.Logf(reply)
	return err
}

func (c *Charm) startService(ctx *service.Context, args []string) (hook.Command, error) {
	return startServer(ctx, args)
}

func (c *Charm) setContext(ctx *hook.Context) error {
	c.ctx = ctx
	return nil
}
<commit_msg>[Fixed] first time Charm install causes 'config-changed' hook error<commit_after>package sevendays

import (
	"github.com/juju/gocharm/charmbits/service"
	"github.com/juju/gocharm/hook"
	"github.com/mever/sevendays/httpd"
	"gopkg.in/juju/charm.v5"
)

type Charm struct {
	svc service.Service
	ctx *hook.Context
}

func RegisterHooks(r *hook.Registry) {
	c := Charm{}
	r.RegisterContext(c.setContext, nil)
	r.RegisterHook("start", c.start)
	r.RegisterHook("upgrade-charm", c.start)
	r.RegisterHook("config-changed", c.configChanged)
	c.svc.Register(r.Clone("service"), "", c.startService)

	r.RegisterConfig("user", charm.Option{
		Type:        "string",
		Description: "Provide Steam user name",
		Default:     "",
	})
}

func (c *Charm) start() error {
	return c.svc.Start(c.ctx.CharmDir)
}

func (c *Charm) configChanged() error {
	if !c.svc.Started() {
		if err := c.start(); err != nil {
			return err
		}
	}

	user, _ := c.ctx.GetConfigString("user")
	ss := httpd.State{
		User: user,
	}

	var reply string
	err := c.svc.Call("RpcServer.Set", ss, &reply)
	c.ctx.Logf(reply)
	return err
}

func (c *Charm) startService(ctx *service.Context, args []string) (hook.Command, error) {
	return startServer(ctx, args)
}

func (c *Charm) setContext(ctx *hook.Context) error {
	c.ctx = ctx
	return nil
}
<|endoftext|>


<commit_before>package netevents

import (
	"net"
	"sync"

	"github.com/segmentio/events"
)

type connLogger struct {
	net.Conn
	*events.Logger
	typ  string
	once sync.Once
}

func (conn *connLogger) Base() net.Conn {
	return conn.Conn
}

func (conn *connLogger) Close() error {
	conn.close(1)
	return conn.Conn.Close()
}

func (conn *connLogger) open(depth int) {
	conn.log(depth+1, "opening")
}

func (conn *connLogger) close(depth int) {
	conn.once.Do(func() { conn.log(depth+3, "closing") })
}

func (conn *connLogger) log(depth int, event string) {
	raddr := conn.RemoteAddr()
	laddr := conn.LocalAddr()

	// Here we're doing our best to adjust the call depth to point outside
	// of this package, but if the connection is wrapped again it may not be
	// very meaningful to report the file/line from which the connection was
	// handled or closed.
	logger := *conn.Logger
	logger.CallDepth += depth + 1

	logger.Log("%{local_address}s->%{remote_address}s - %{event}s %{type}s %{protocol}s connection",
		laddr.String(),
		raddr.String(),
		event,
		conn.typ,
		laddr.Network(),
	)
}
<commit_msg>Base => BaseConn<commit_after>package netevents

import (
	"net"
	"sync"

	"github.com/segmentio/events"
)

type connLogger struct {
	net.Conn
	*events.Logger
	typ  string
	once sync.Once
}

func (conn *connLogger) BaseConn() net.Conn {
	return conn.Conn
}

func (conn *connLogger) Close() error {
	conn.close(1)
	return conn.Conn.Close()
}

func (conn *connLogger) open(depth int) {
	conn.log(depth+1, "opening")
}

func (conn *connLogger) close(depth int) {
	conn.once.Do(func() { conn.log(depth+3, "closing") })
}

func (conn *connLogger) log(depth int, event string) {
	raddr := conn.RemoteAddr()
	laddr := conn.LocalAddr()

	// Here we're doing our best to adjust the call depth to point outside
	// of this package, but if the connection is wrapped again it may not be
	// very meaningful to report the file/line from which the connection was
	// handled or closed.
	logger := *conn.Logger
	logger.CallDepth += depth + 1

	logger.Log("%{local_address}s->%{remote_address}s - %{event}s %{type}s %{protocol}s connection",
		laddr.String(),
		raddr.String(),
		event,
		conn.typ,
		laddr.Network(),
	)
}
<|endoftext|>


<commit_before>package upload

import ()

type Uploader struct {
	contentpath string
	sessions    []*Session
}

func NewUploader(dir string) *Uploader {
	return &Uploader{contentpath: dir}
}

// Session returns a session associated with id.
func (u *Uploader) Session(id string) *Session {
	for _, sess := range u.sessions {
		if sess.ID() == id {
			return sess
		}
	}
	return nil
}

func (u *Uploader) AddSession() *Session {
	sess := NewSession(u.contentpath)
	u.sessions = append(u.sessions, sess)
	return sess
}
<commit_msg>uploader: add cleanup method<commit_after>package upload

type Uploader struct {
	contentpath string
	sessions    []*Session
}

func NewUploader(dir string) *Uploader {
	return &Uploader{contentpath: dir}
}

// Session returns a session associated with id.
func (u *Uploader) Session(id string) *Session {
	_, sess := u.findSession(id)
	return sess
}

// AddSession creates a new upload session and adds
// it to sessions list.
func (u *Uploader) AddSession() *Session {
	sess := NewSession(u.contentpath)
	u.sessions = append(u.sessions, sess)
	return sess
}

func (u *Uploader) CleanupSession(id string) {
	i, sess := u.findSession(id)

	if i >= 0 {
		u.sessions[i] = u.sessions[len(u.sessions)-1]
		u.sessions[len(u.sessions)-1] = nil
		u.sessions = u.sessions[:len(u.sessions)-1]
	}
}

// findSession searches session list and returns found session
// and its index in the list.
func (u *Uploader) findSession(id string) (int, *Session) {
	for i, sess := range u.sessions {
		if sess.id == id {
			return i, sess
		}
	}
	return -1, nil
}
<|endoftext|>


<commit_before>// The upstartConfig package creates an upstart configuration file for the executing program.
package upstartConfig

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
)

//Location to write the upstart configuration file to.
var DestinationPath string = "/etc/init"

//Permisions for the upstart configuration file
var Perms string = "0666"

type Config struct {
	ServiceName   string
	ServicePath   string
	StartRunLevel string
	StopRunLevel  string
	EnableEnv     bool
	Script        string
}

func BinaryToIntStr(in bool) string {
	if in {
		return "1"
	}
	return "0"
}

// Gets the name of the currently executing program. This is used for the name of the upstart configuration file, along with the script.
func GetName() string {
	pathParts := strings.Split(os.Args[0], "/")
	return pathParts[len(pathParts)-1]
}

// Gets the name and path of the currently executing program. This is used for determining what program upstart should manage.
func GetPath() string {
	pathParts := strings.Split(os.Args[0], "/")
	return strings.Join(pathParts[0:len(pathParts)-1], "/")
}

// Writes the upstart configuration file.
func Write(options ...func(*Config)) error {
	contents, config := Generate(options...)
	file := fmt.Sprintf("%s/%s.conf", DestinationPath, config.ServiceName)
	return ioutil.WriteFile(file, []byte(contents), 0666)
}

//Generates the upstart configuration file.
func Generate(options ...func(*Config)) (string, *Config) {
	path := GetPath()
	name := GetName()

	config := &Config{
		ServiceName:   name,
		ServicePath:   path,
		StartRunLevel: "2345",
		StopRunLevel:  "!2345",
		EnableEnv:     true,
		Script:        fmt.Sprintf("cd %s; exec ./%s", path, name),
	}

	//populate options into config
	for _, o := range options {
		o(config)
	}

	lines := []string{
		fmt.Sprintf("description \"%s\"", config.ServiceName),
		fmt.Sprintf("start on runlevel [%s]", config.StartRunLevel),
		fmt.Sprintf("stop on runlevel [%s]", config.StopRunLevel),
		fmt.Sprintf("env enabled=%s", BinaryToIntStr(config.EnableEnv)),
		"",
		"respawn",
		"",
		"script",
		config.Script,
		"end script",
	}

	return strings.Join(lines, "\n"), config
}
<commit_msg>adding comments<commit_after>// The upstartConfig package creates an upstart configuration file for the executing program.
//
// For many services written in Go, there is almost no system configuration needed beyond setting up your program to restart if it crashes or the machine reboots.
//
// Using this package, you can integrate that configuration step, removing the need for system configuration tools, or a more complex startup script.
package upstartConfig

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
)

//Location to write the upstart configuration file to.
var DestinationPath string = "/etc/init"

//Permisions for the upstart configuration file
var Perms string = "0666"

//Upstart configuration
type Config struct {
	ServiceName   string
	ServicePath   string
	StartRunLevel string
	StopRunLevel  string
	EnableEnv     bool
	Script        string
}

func BinaryToIntStr(in bool) string {
	if in {
		return "1"
	}
	return "0"
}

// Gets the name of the currently executing program. This is used for the name of the upstart configuration file, along with the script.
func GetName() string {
	pathParts := strings.Split(os.Args[0], "/")
	return pathParts[len(pathParts)-1]
}

// Gets the name and path of the currently executing program. This is used for determining what program upstart should manage.
func GetPath() string {
	pathParts := strings.Split(os.Args[0], "/")
	return strings.Join(pathParts[0:len(pathParts)-1], "/")
}

// Writes the upstart configuration file.
func Write(options ...func(*Config)) error {
	contents, config := Generate(options...)
	file := fmt.Sprintf("%s/%s.conf", DestinationPath, config.ServiceName)
	return ioutil.WriteFile(file, []byte(contents), 0666)
}

//Generates the upstart configuration file.
func Generate(options ...func(*Config)) (string, *Config) {
	path := GetPath()
	name := GetName()

	config := &Config{
		ServiceName:   name,
		ServicePath:   path,
		StartRunLevel: "2345",
		StopRunLevel:  "!2345",
		EnableEnv:     true,
		Script:        fmt.Sprintf("cd %s; exec ./%s", path, name),
	}

	//populate options into config, overriding defaults
	for _, o := range options {
		o(config)
	}

	lines := []string{
		fmt.Sprintf("description \"%s\"", config.ServiceName),
		fmt.Sprintf("start on runlevel [%s]", config.StartRunLevel),
		fmt.Sprintf("stop on runlevel [%s]", config.StopRunLevel),
		fmt.Sprintf("env enabled=%s", BinaryToIntStr(config.EnableEnv)),
		"",
		"respawn",
		"",
		"script",
		config.Script,
		"end script",
	}

	return strings.Join(lines, "\n"), config
}
<|endoftext|>