<commit_before>package mocks

import (
	"encoding/json"
	"io"
	"strconv"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/events"
	"github.com/docker/docker/api/types/image"
	"github.com/docker/docker/api/types/swarm"
	drydocker "github.com/moncho/dry/docker"
)

//DockerDaemonMock mocks a DockerDaemon
type DockerDaemonMock struct {
}

//ContainerByID mock
func (_m *DockerDaemonMock) ContainerByID(id string) *drydocker.Container {
	return nil
}

//Containers mock
func (_m *DockerDaemonMock) Containers(filters []drydocker.ContainerFilter, mode drydocker.SortMode) []*drydocker.Container {

	var containers []*drydocker.Container
	for index := 0; index < 10; index++ {
		containers = append(containers, &drydocker.Container{
			Container: types.Container{ID: strconv.Itoa(index), Names: []string{"Name"},
				Status: "Up and running"},
		})
	}
	for index := 0; index < 10; index++ {
		containers = append(containers, &drydocker.Container{
			Container: types.Container{ID: strconv.Itoa(index), Names: []string{"Name"},
				Status: "Never worked"},
		})
	}
	for _, filter := range filters {
		containers = filter.Apply(containers)
	}
	return containers
}

//DiskUsage mock
func (_m *DockerDaemonMock) DiskUsage() (types.DiskUsage, error) {
	return types.DiskUsage{}, nil
}

// DockerEnv provides a mock function with given fields:
func (_m *DockerDaemonMock) DockerEnv() *drydocker.Env {
	return &drydocker.Env{DockerHost: "dry.io", DockerTLSVerify: false, DockerCertPath: ""}
}

// Events provides a mock function with given fields:
func (_m *DockerDaemonMock) Events() (<-chan events.Message, chan<- struct{}, error) {

	return nil, nil, nil
}

//EventLog mock
func (_m *DockerDaemonMock) EventLog() *drydocker.EventLog {
	return nil
}

//History mock
func (_m *DockerDaemonMock) History(id string) ([]image.HistoryResponseItem, error) {
	return nil, nil
}

//ImageByID mock
func (_m *DockerDaemonMock) ImageByID(id string) (types.ImageSummary, error) {
	return types.ImageSummary{}, nil
}

//Images mock
func (_m *DockerDaemonMock) Images() ([]types.ImageSummary, error) {

	imagesJSON := `[
		 {
						 "ID": "8dfafdbc3a40",
						  "RepoTags":["dry/dry:1"],
							"Created": 1367854155,
							"Size": 100,
							"VirtualSize": 1000,
							"ParentID": "7dfafdbc3a40",
							"RepoDigests": ["8dfafdbc3a40"]
		 },
		 {
						 "ID": "541a0f4efc6f",
						  "RepoTags":["dry/dry:2"],
							"Created": 1367854155,
							"Size": 100,
							"VirtualSize": 1000,
							"ParentID": "7dfafdbc3a40",
							"RepoDigests": ["541a0f4efc6f"]
		 },
		 {
						 "ID": "26380e1ca356",
						  "RepoTags":["dry/dry:3"],
							"Created": 1367854155,
							"Size": 100,
							"VirtualSize": 1000,
							"ParentID": "7dfafdbc3a40",
							"RepoDigests": ["26380e1ca356"]
		 },
		 {
						 "ID": "a3d6e836e86a",
						  "RepoTags":["examplevotingapp_result-app:latest"],
							"Created": 1367844155,
							"Size": 0,
							"VirtualSize": 54543,
							"ParentID": "7dfafdbc3a40",
							"RepoDigests": ["a3d6e836e86a"]
		 },
		 {
						 "ID": "03b4557ad7b9",
						  "RepoTags":["dry/dry:4"],
							"Created": 136123255,
							"Size": 43985,
							"VirtualSize": 1343400,
							"ParentID": "7dfafdbc3a40",
							"RepoDigests": ["03b4557ad7b9"]
		 }
	]`
	var images []types.ImageSummary
	err := json.Unmarshal([]byte(imagesJSON), &images)
	return images, err
}

//ImagesCount mock
func (_m *DockerDaemonMock) ImagesCount() int {
	i, _ := _m.Images()
	return len(i)
}

// Info provides a mock function with given fields:
func (_m *DockerDaemonMock) Info() (types.Info, error) {
	swarmInfo := swarm.Info{LocalNodeState: swarm.LocalNodeStateInactive}
	return types.Info{
		Name:     "test",
		NCPU:     2,
		MemTotal: 1024,
		Swarm:    swarmInfo}, nil
}

// Inspect provides a mock function with given fields: id
func (_m *DockerDaemonMock) Inspect(id string) (types.ContainerJSON, error) {
	return types.ContainerJSON{}, nil
}

// InspectImage mock
func (_m *DockerDaemonMock) InspectImage(name string) (types.ImageInspect, error) {
	return types.ImageInspect{}, nil
}

// IsContainerRunning provides a mock function with given fields: id
func (_m *DockerDaemonMock) IsContainerRunning(id string) bool {
	return false
}

// Kill provides a mock function with given fields: id
func (_m *DockerDaemonMock) Kill(id string) error {
	return nil
}

// Logs provides a mock function with given fields: id
func (_m *DockerDaemonMock) Logs(id string) io.ReadCloser {
	return nil
}

//Networks mock
func (_m *DockerDaemonMock) Networks() ([]types.NetworkResource, error) {
	return nil, nil
}

//NetworkAt mock
func (_m *DockerDaemonMock) NetworkAt(position int) (*types.NetworkResource, error) {
	return nil, nil
}

//NetworksCount mock
func (_m *DockerDaemonMock) NetworksCount() int {
	return 0
}

//NetworkInspect mock
func (_m *DockerDaemonMock) NetworkInspect(id string) (types.NetworkResource, error) {
	return types.NetworkResource{}, nil
}

//Node mock
func (_m *DockerDaemonMock) Node(id string) (*swarm.Node, error) {
	return nil, nil
}

//NodeChangeAvailability mock
func (_m *DockerDaemonMock) NodeChangeAvailability(nodeID string, availability swarm.NodeAvailability) error {
	return nil
}

//Nodes mock
func (_m *DockerDaemonMock) Nodes() ([]swarm.Node, error) {
	return nil, nil
}

//NodeTasks mock
func (_m *DockerDaemonMock) NodeTasks(nodeID string) ([]swarm.Task, error) {
	return nil, nil
}

// Ok mocks OK
func (_m *DockerDaemonMock) Ok() (bool, error) {

	return false, nil
}

//OpenChannel mocks OpenChannel
func (_m *DockerDaemonMock) OpenChannel(container *drydocker.Container) *drydocker.StatsChannel {
	return nil
}

// Prune mocks prune command
func (_m *DockerDaemonMock) Prune() (*drydocker.PruneReport, error) {
	return nil, nil
}

// RestartContainer provides a mock function with given fields: id
func (_m *DockerDaemonMock) RestartContainer(id string) error {

	return nil
}

// Rm provides a mock function with given fields: id
func (_m *DockerDaemonMock) Rm(id string) error {
	return nil
}

// Rmi mock
func (_m *DockerDaemonMock) Rmi(id string, force bool) ([]types.ImageDeleteResponseItem, error) {
	return nil, nil
}

// Refresh provides a mock function with given fields: allContainers
func (_m *DockerDaemonMock) Refresh(notify func(err error)) {
	notify(nil)
}

//RefreshImages mock
func (_m *DockerDaemonMock) RefreshImages() error {
	return nil
}

//RefreshNetworks mock
func (_m *DockerDaemonMock) RefreshNetworks() error {
	return nil
}

// RemoveAllStoppedContainers provides a mock function with given fields:
func (_m *DockerDaemonMock) RemoveAllStoppedContainers() (int, error) {
	return 0, nil

}

//RemoveDanglingImages mock
func (_m *DockerDaemonMock) RemoveDanglingImages() (int, error) {
	return 0, nil
}

//RemoveNetwork mock
func (_m *DockerDaemonMock) RemoveNetwork(id string) error {
	return nil
}

//ResolveNode mock
func (_m *DockerDaemonMock) ResolveNode(id string) (string, error) {
	return "", nil
}

//ResolveService mock
func (_m *DockerDaemonMock) ResolveService(id string) (string, error) {
	return "", nil
}

//RunImage mock
func (_m *DockerDaemonMock) RunImage(image types.ImageSummary, command string) error {
	return nil
}

//Service mock
func (_m *DockerDaemonMock) Service(id string) (*swarm.Service, error) {
	return nil, nil
}

//ServiceLogs mock
func (_m *DockerDaemonMock) ServiceLogs(id string) (io.ReadCloser, error) {
	return nil, nil
}

//Services mock
func (_m *DockerDaemonMock) Services() ([]swarm.Service, error) {
	return nil, nil
}

//ServiceRemove mock
func (_m *DockerDaemonMock) ServiceRemove(id string) error {
	return nil
}

//ServiceScale mock
func (_m *DockerDaemonMock) ServiceScale(id string, scale uint64) error {
	return nil
}

//ServiceTasks mock
func (_m *DockerDaemonMock) ServiceTasks(services ...string) ([]swarm.Task, error) {
	return nil, nil
}

// StopContainer provides a mock function with given fields: id
func (_m *DockerDaemonMock) StopContainer(id string) error {
	return nil
}

// Sort provides a mock function with given fields: sortMode
func (_m *DockerDaemonMock) Sort(sortMode drydocker.SortMode) {

}

//SortImages mock
func (_m *DockerDaemonMock) SortImages(sortMode drydocker.SortMode) {
}

//SortNetworks mock
func (_m *DockerDaemonMock) SortNetworks(sortMode drydocker.SortMode) {
}

//Stacks mock
func (_m *DockerDaemonMock) Stacks() ([]drydocker.Stack, error) {
	return nil, nil
}

//StackTasks empty mock
func (_m *DockerDaemonMock) StackTasks(stack string) ([]swarm.Task, error) {
	return nil, nil
}

//Task empty mock
func (_m *DockerDaemonMock) Task(id string) (swarm.Task, error) {
	return swarm.Task{}, nil
}

//Top function mock
func (_m *DockerDaemonMock) Top(id string) (container.ContainerTopOKBody, error) {

	return container.ContainerTopOKBody{}, nil
}

// Version provides a mock function with given fields:
func (_m *DockerDaemonMock) Version() (*types.Version, error) {

	return &types.Version{
		Version:       "1.0",
		APIVersion:    "1.27",
		Os:            "dry",
		Arch:          "amd64",
		KernelVersion: "42",
	}, nil
}
<commit_msg>Allow to specify a time from which to retrieve logs<commit_after>package mocks

import (
	"encoding/json"
	"io"
	"strconv"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/events"
	"github.com/docker/docker/api/types/image"
	"github.com/docker/docker/api/types/swarm"
	drydocker "github.com/moncho/dry/docker"
)

//DockerDaemonMock mocks a DockerDaemon
type DockerDaemonMock struct {
}

//ContainerByID mock
func (_m *DockerDaemonMock) ContainerByID(id string) *drydocker.Container {
	return nil
}

//Containers mock
func (_m *DockerDaemonMock) Containers(filters []drydocker.ContainerFilter, mode drydocker.SortMode) []*drydocker.Container {

	var containers []*drydocker.Container
	for index := 0; index < 10; index++ {
		containers = append(containers, &drydocker.Container{
			Container: types.Container{ID: strconv.Itoa(index), Names: []string{"Name"},
				Status: "Up and running"},
		})
	}
	for index := 0; index < 10; index++ {
		containers = append(containers, &drydocker.Container{
			Container: types.Container{ID: strconv.Itoa(index), Names: []string{"Name"},
				Status: "Never worked"},
		})
	}
	for _, filter := range filters {
		containers = filter.Apply(containers)
	}
	return containers
}

//DiskUsage mock
func (_m *DockerDaemonMock) DiskUsage() (types.DiskUsage, error) {
	return types.DiskUsage{}, nil
}

// DockerEnv provides a mock function with given fields:
func (_m *DockerDaemonMock) DockerEnv() *drydocker.Env {
	return &drydocker.Env{DockerHost: "dry.io", DockerTLSVerify: false, DockerCertPath: ""}
}

// Events provides a mock function with given fields:
func (_m *DockerDaemonMock) Events() (<-chan events.Message, chan<- struct{}, error) {

	return nil, nil, nil
}

//EventLog mock
func (_m *DockerDaemonMock) EventLog() *drydocker.EventLog {
	return nil
}

//History mock
func (_m *DockerDaemonMock) History(id string) ([]image.HistoryResponseItem, error) {
	return nil, nil
}

//ImageByID mock
func (_m *DockerDaemonMock) ImageByID(id string) (types.ImageSummary, error) {
	return types.ImageSummary{}, nil
}

//Images mock
func (_m *DockerDaemonMock) Images() ([]types.ImageSummary, error) {

	imagesJSON := `[
		 {
						 "ID": "8dfafdbc3a40",
						  "RepoTags":["dry/dry:1"],
							"Created": 1367854155,
							"Size": 100,
							"VirtualSize": 1000,
							"ParentID": "7dfafdbc3a40",
							"RepoDigests": ["8dfafdbc3a40"]
		 },
		 {
						 "ID": "541a0f4efc6f",
						  "RepoTags":["dry/dry:2"],
							"Created": 1367854155,
							"Size": 100,
							"VirtualSize": 1000,
							"ParentID": "7dfafdbc3a40",
							"RepoDigests": ["541a0f4efc6f"]
		 },
		 {
						 "ID": "26380e1ca356",
						  "RepoTags":["dry/dry:3"],
							"Created": 1367854155,
							"Size": 100,
							"VirtualSize": 1000,
							"ParentID": "7dfafdbc3a40",
							"RepoDigests": ["26380e1ca356"]
		 },
		 {
						 "ID": "a3d6e836e86a",
						  "RepoTags":["examplevotingapp_result-app:latest"],
							"Created": 1367844155,
							"Size": 0,
							"VirtualSize": 54543,
							"ParentID": "7dfafdbc3a40",
							"RepoDigests": ["a3d6e836e86a"]
		 },
		 {
						 "ID": "03b4557ad7b9",
						  "RepoTags":["dry/dry:4"],
							"Created": 136123255,
							"Size": 43985,
							"VirtualSize": 1343400,
							"ParentID": "7dfafdbc3a40",
							"RepoDigests": ["03b4557ad7b9"]
		 }
	]`
	var images []types.ImageSummary
	err := json.Unmarshal([]byte(imagesJSON), &images)
	return images, err
}

//ImagesCount mock
func (_m *DockerDaemonMock) ImagesCount() int {
	i, _ := _m.Images()
	return len(i)
}

// Info provides a mock function with given fields:
func (_m *DockerDaemonMock) Info() (types.Info, error) {
	swarmInfo := swarm.Info{LocalNodeState: swarm.LocalNodeStateInactive}
	return types.Info{
		Name:     "test",
		NCPU:     2,
		MemTotal: 1024,
		Swarm:    swarmInfo}, nil
}

// Inspect provides a mock function with given fields: id
func (_m *DockerDaemonMock) Inspect(id string) (types.ContainerJSON, error) {
	return types.ContainerJSON{}, nil
}

// InspectImage mock
func (_m *DockerDaemonMock) InspectImage(name string) (types.ImageInspect, error) {
	return types.ImageInspect{}, nil
}

// IsContainerRunning provides a mock function with given fields: id
func (_m *DockerDaemonMock) IsContainerRunning(id string) bool {
	return false
}

// Kill provides a mock function with given fields: id
func (_m *DockerDaemonMock) Kill(id string) error {
	return nil
}

// Logs provides a mock function with given fields: id
func (_m *DockerDaemonMock) Logs(id, since string) (io.ReadCloser, error) {
	return nil, nil
}

//Networks mock
func (_m *DockerDaemonMock) Networks() ([]types.NetworkResource, error) {
	return nil, nil
}

//NetworkAt mock
func (_m *DockerDaemonMock) NetworkAt(position int) (*types.NetworkResource, error) {
	return nil, nil
}

//NetworksCount mock
func (_m *DockerDaemonMock) NetworksCount() int {
	return 0
}

//NetworkInspect mock
func (_m *DockerDaemonMock) NetworkInspect(id string) (types.NetworkResource, error) {
	return types.NetworkResource{}, nil
}

//Node mock
func (_m *DockerDaemonMock) Node(id string) (*swarm.Node, error) {
	return nil, nil
}

//NodeChangeAvailability mock
func (_m *DockerDaemonMock) NodeChangeAvailability(nodeID string, availability swarm.NodeAvailability) error {
	return nil
}

//Nodes mock
func (_m *DockerDaemonMock) Nodes() ([]swarm.Node, error) {
	return nil, nil
}

//NodeTasks mock
func (_m *DockerDaemonMock) NodeTasks(nodeID string) ([]swarm.Task, error) {
	return nil, nil
}

// Ok mocks OK
func (_m *DockerDaemonMock) Ok() (bool, error) {

	return false, nil
}

//OpenChannel mocks OpenChannel
func (_m *DockerDaemonMock) OpenChannel(container *drydocker.Container) *drydocker.StatsChannel {
	return nil
}

// Prune mocks prune command
func (_m *DockerDaemonMock) Prune() (*drydocker.PruneReport, error) {
	return nil, nil
}

// RestartContainer provides a mock function with given fields: id
func (_m *DockerDaemonMock) RestartContainer(id string) error {

	return nil
}

// Rm provides a mock function with given fields: id
func (_m *DockerDaemonMock) Rm(id string) error {
	return nil
}

// Rmi mock
func (_m *DockerDaemonMock) Rmi(id string, force bool) ([]types.ImageDeleteResponseItem, error) {
	return nil, nil
}

// Refresh provides a mock function with given fields: allContainers
func (_m *DockerDaemonMock) Refresh(notify func(err error)) {
	notify(nil)
}

//RefreshImages mock
func (_m *DockerDaemonMock) RefreshImages() error {
	return nil
}

//RefreshNetworks mock
func (_m *DockerDaemonMock) RefreshNetworks() error {
	return nil
}

// RemoveAllStoppedContainers provides a mock function with given fields:
func (_m *DockerDaemonMock) RemoveAllStoppedContainers() (int, error) {
	return 0, nil

}

//RemoveDanglingImages mock
func (_m *DockerDaemonMock) RemoveDanglingImages() (int, error) {
	return 0, nil
}

//RemoveNetwork mock
func (_m *DockerDaemonMock) RemoveNetwork(id string) error {
	return nil
}

//ResolveNode mock
func (_m *DockerDaemonMock) ResolveNode(id string) (string, error) {
	return "", nil
}

//ResolveService mock
func (_m *DockerDaemonMock) ResolveService(id string) (string, error) {
	return "", nil
}

//RunImage mock
func (_m *DockerDaemonMock) RunImage(image types.ImageSummary, command string) error {
	return nil
}

//Service mock
func (_m *DockerDaemonMock) Service(id string) (*swarm.Service, error) {
	return nil, nil
}

//ServiceLogs mock
func (_m *DockerDaemonMock) ServiceLogs(id, since string) (io.ReadCloser, error) {
	return nil, nil
}

//Services mock
func (_m *DockerDaemonMock) Services() ([]swarm.Service, error) {
	return nil, nil
}

//ServiceRemove mock
func (_m *DockerDaemonMock) ServiceRemove(id string) error {
	return nil
}

//ServiceScale mock
func (_m *DockerDaemonMock) ServiceScale(id string, scale uint64) error {
	return nil
}

//ServiceTasks mock
func (_m *DockerDaemonMock) ServiceTasks(services ...string) ([]swarm.Task, error) {
	return nil, nil
}

// StopContainer provides a mock function with given fields: id
func (_m *DockerDaemonMock) StopContainer(id string) error {
	return nil
}

// Sort provides a mock function with given fields: sortMode
func (_m *DockerDaemonMock) Sort(sortMode drydocker.SortMode) {

}

//SortImages mock
func (_m *DockerDaemonMock) SortImages(sortMode drydocker.SortMode) {
}

//SortNetworks mock
func (_m *DockerDaemonMock) SortNetworks(sortMode drydocker.SortMode) {
}

//Stacks mock
func (_m *DockerDaemonMock) Stacks() ([]drydocker.Stack, error) {
	return nil, nil
}

//StackTasks empty mock
func (_m *DockerDaemonMock) StackTasks(stack string) ([]swarm.Task, error) {
	return nil, nil
}

//Task empty mock
func (_m *DockerDaemonMock) Task(id string) (swarm.Task, error) {
	return swarm.Task{}, nil
}

//Top function mock
func (_m *DockerDaemonMock) Top(id string) (container.ContainerTopOKBody, error) {

	return container.ContainerTopOKBody{}, nil
}

// Version provides a mock function with given fields:
func (_m *DockerDaemonMock) Version() (*types.Version, error) {

	return &types.Version{
		Version:       "1.0",
		APIVersion:    "1.27",
		Os:            "dry",
		Arch:          "amd64",
		KernelVersion: "42",
	}, nil
}
<|endoftext|>


<commit_before>package models

import (
	"errors"

	"code.cloudfoundry.org/bbs/format"
)

func (*VolumePlacement) Version() format.Version {
	return format.V1
}

func (*VolumePlacement) Validate() error {
	return nil
}

// while volume mounts are experimental, we should never persist a "old" volume
// mount to the db layer, so the handler must convert old data models to the new ones
// when volume mounts are no longer experimental, this validation strategy must be reconsidered
func (v *VolumeMount) Validate() error {
	var ve ValidationError
	if v.Driver == "" {
		ve = ve.Append(errors.New("invalid volume_mount driver"))
	}
	if !(v.Mode == "r" || v.Mode == "rw") {
		ve = ve.Append(errors.New("invalid volume_mount mode"))
	}
	if v.Shared != nil && v.Shared.VolumeId == "" {
		ve = ve.Append(errors.New("invalid volume_mount volume id"))
	}

	if !ve.Empty() {
		return ve
	}

	return nil
}
<commit_msg>Remove comment referencing volume mounts as experimental<commit_after>package models

import (
	"errors"

	"code.cloudfoundry.org/bbs/format"
)

func (*VolumePlacement) Version() format.Version {
	return format.V1
}

func (*VolumePlacement) Validate() error {
	return nil
}

func (v *VolumeMount) Validate() error {
	var ve ValidationError
	if v.Driver == "" {
		ve = ve.Append(errors.New("invalid volume_mount driver"))
	}
	if !(v.Mode == "r" || v.Mode == "rw") {
		ve = ve.Append(errors.New("invalid volume_mount mode"))
	}
	if v.Shared != nil && v.Shared.VolumeId == "" {
		ve = ve.Append(errors.New("invalid volume_mount volume id"))
	}

	if !ve.Empty() {
		return ve
	}

	return nil
}
<|endoftext|>


<commit_before>package s3

import (
    "io/ioutil"
    "os"
    "strings"
    "testing"
    "fmt"

    minio "github.com/minio/minio-go"

    tu "github.com/pachyderm/pachyderm/src/server/pkg/testutil"
    "github.com/pachyderm/pachyderm/src/client"
    "github.com/pachyderm/pachyderm/src/client/pkg/require"
    "github.com/pachyderm/pachyderm/src/client/pfs"
)

type workerTestState struct {
	pachClient *client.APIClient
	minioClient *minio.Client
	inputRepo string
	outputRepo string
	inputMasterCommit *pfs.Commit
	inputDevelopCommit *pfs.Commit
	outputCommit *pfs.Commit
}

func workerListBuckets(t *testing.T, s *workerTestState) {
	// create a repo - this should not show up list buckets with the worker
	// driver
    repo := tu.UniqueString("testlistbuckets1")
    require.NoError(t, s.pachClient.CreateRepo(repo))
    require.NoError(t, s.pachClient.CreateBranch(repo, "master", "", nil))

    buckets, err := s.minioClient.ListBuckets()
    require.NoError(t, err)

    actualBucketNames := []string{}
    for _, bucket := range buckets {
    	actualBucketNames = append(actualBucketNames, bucket.Name)
    }

    require.ElementsEqual(t, []string{"in1", "in2", "out"}, actualBucketNames)
}

func workerGetObject(t *testing.T, s *workerTestState) {
    fetchedContent, err := getObject(t, s.minioClient, "in1", "file")
    require.NoError(t, err)
    require.Equal(t, "foo", fetchedContent)
}

func workerGetObjectOutputRepo(t *testing.T, s *workerTestState) {
    _, err := getObject(t, s.minioClient, "out", "file")
    keyNotFoundError(t, err)
}

func workerStatObject(t *testing.T, s *workerTestState) {
    info, err := s.minioClient.StatObject("in1", "file", minio.StatObjectOptions{})
    require.NoError(t, err)
    require.True(t, len(info.ETag) > 0)
    require.Equal(t, "text/plain; charset=utf-8", info.ContentType)
    require.Equal(t, int64(3), info.Size)
}

func workerPutObject(t *testing.T, s *workerTestState) {
    r := strings.NewReader("content1")
    _, err := s.minioClient.PutObject("out", "file", r, int64(r.Len()), minio.PutObjectOptions{ContentType: "text/plain"})
    require.NoError(t, err)

    // this should act as a PFS PutFileOverwrite
    r2 := strings.NewReader("content2")
    _, err = s.minioClient.PutObject("out", "file", r2, int64(r2.Len()), minio.PutObjectOptions{ContentType: "text/plain"})
    require.NoError(t, err)

    _, err = getObject(t, s.minioClient, "out", "file")
    keyNotFoundError(t, err)
}

func workerPutObjectInputRepo(t *testing.T, s *workerTestState) {
    r := strings.NewReader("content1")
    _, err := s.minioClient.PutObject("in1", "file", r, int64(r.Len()), minio.PutObjectOptions{ContentType: "text/plain"})
    notImplementedError(t, err)
}

func workerRemoveObject(t *testing.T, s *workerTestState) {
    _, err := s.pachClient.PutFile(s.outputRepo, s.outputCommit.ID, "file", strings.NewReader("content"))
    require.NoError(t, err)

    // as per PFS semantics, the second delete should be a no-op
    require.NoError(t, s.minioClient.RemoveObject("out", "file"))
    require.NoError(t, s.minioClient.RemoveObject("out", "file"))
}

func workerRemoveObjectInputRepo(t *testing.T, s *workerTestState) {
    err := s.minioClient.RemoveObject("in1", "file")
    notImplementedError(t, err)
}

// Tests inserting and getting files over 64mb in size
func workerLargeObjects(t *testing.T, s *workerTestState) {
    // create a temporary file to put ~65mb of contents into it
    inputFile, err := ioutil.TempFile("", "pachyderm-test-large-objects-input-*")
    require.NoError(t, err)
    defer os.Remove(inputFile.Name())
    n, err := inputFile.WriteString(strings.Repeat("no tv and no beer make homer something something.\n", 1363149))
    require.NoError(t, err)
    require.Equal(t, n, 68157450)
    require.NoError(t, inputFile.Sync())

    // first ensure that putting into a repo that doesn't exist triggers an
    // error
    _, err = s.minioClient.FPutObject("foobar", "file", inputFile.Name(), minio.PutObjectOptions{
        ContentType: "text/plain",
    })
    bucketNotFoundError(t, err)

    // now try putting into a legit repo
    l, err := s.minioClient.FPutObject("out", "file", inputFile.Name(), minio.PutObjectOptions{
        ContentType: "text/plain",
    })
    require.NoError(t, err)
    require.Equal(t, int(l), 68157450)

    // try getting an object that does not exist
    err = s.minioClient.FGetObject("foobar", "file", "foo", minio.GetObjectOptions{})
    bucketNotFoundError(t, err)

    // get the file that does exist, doesn't work because we're reading from
    // an output repo
    outputFile, err := ioutil.TempFile("", "pachyderm-test-large-objects-output-*")
    require.NoError(t, err)
    defer os.Remove(outputFile.Name())
    err = s.minioClient.FGetObject("out", "file", outputFile.Name(), minio.GetObjectOptions{})
    keyNotFoundError(t, err)
}

func workerMakeBucket(t *testing.T, s *workerTestState) {
    repo := tu.UniqueString("testmakebucket")
    notImplementedError(t, s.minioClient.MakeBucket(repo, ""))
}

func workerBucketExists(t *testing.T, s *workerTestState) {
    exists, err := s.minioClient.BucketExists("in1")
    require.NoError(t, err)
    require.True(t, exists)

    exists, err = s.minioClient.BucketExists("out")
    require.NoError(t, err)
    require.True(t, exists)

    exists, err = s.minioClient.BucketExists("foobar")
    require.NoError(t, err)
    require.False(t, exists)
}

func workerRemoveBucket(t *testing.T, s *workerTestState) {
    notImplementedError(t, s.minioClient.RemoveBucket("in1"))
    notImplementedError(t, s.minioClient.RemoveBucket("out"))
}

func workerListObjectsPaginated(t *testing.T, s *workerTestState) {
    // Request that will list all files in root
    ch := s.minioClient.ListObjects("in2", "", false, make(chan struct{}))
    expectedFiles := []string{}
    for i := 0; i <= 1000; i++ {
        expectedFiles = append(expectedFiles, fmt.Sprintf("%d", i))
    }
    checkListObjects(t, ch, nil, nil, expectedFiles, []string{"dir/"})

    // Request that will list all files starting with 1
    ch = s.minioClient.ListObjects("in2", "1", false, make(chan struct{}))
    expectedFiles = []string{}
    for i := 0; i <= 1000; i++ {
        file := fmt.Sprintf("%d", i)
        if strings.HasPrefix(file, "1") {
            expectedFiles = append(expectedFiles, file)
        }
    }
    checkListObjects(t, ch, nil, nil, expectedFiles, []string{})

    // Request that will list all files in a directory
    ch = s.minioClient.ListObjects("in2", "dir/", false, make(chan struct{}))
    expectedFiles = []string{}
    for i := 0; i < 10; i++ {
        expectedFiles = append(expectedFiles, fmt.Sprintf("dir/%d", i))
    }
    checkListObjects(t, ch, nil, nil, expectedFiles, []string{})
}

// func workerListObjectsRecursive(t *testing.T, s *workerTestState) {
//     // `startTime` and `endTime` will be used to ensure that an object's
//     // `LastModified` date is correct. A few minutes are subtracted/added to
//     // each to tolerate the node time not being the same as the host time.
//     startTime := time.Now().Add(time.Duration(-5) * time.Minute)
//     repo := tu.UniqueString("testlistobjectsrecursive")
//     require.NoError(t, s.pachClient.CreateRepo(repo))
//     require.NoError(t, s.pachClient.CreateBranch(repo, "branch", "", nil))
//     require.NoError(t, s.pachClient.CreateBranch(repo, "emptybranch", "", nil))
//     commit, err := s.pachClient.StartCommit(repo, "master")
//     require.NoError(t, err)
//     putListFileTestObject(t, s.pachClient, repo, commit.ID, "", 0)
//     putListFileTestObject(t, s.pachClient, repo, commit.ID, "rootdir/", 1)
//     putListFileTestObject(t, s.pachClient, repo, commit.ID, "rootdir/subdir/", 2)
//     putListFileTestObject(t, s.pachClient, repo, "branch", "", 3)
//     require.NoError(t, s.pachClient.FinishCommit(repo, commit.ID))
//     endTime := time.Now().Add(time.Duration(5) * time.Minute)

//     // Request that will list all files in master
//     expectedFiles := []string{"0", "rootdir/1", "rootdir/subdir/2"}
//     ch := s.minioClient.ListObjects(fmt.Sprintf("master.%s", repo), "", true, make(chan struct{}))
//     checkListObjects(t, ch, startTime, endTime, expectedFiles, []string{})

//     // Requests that will list all files in rootdir
//     expectedFiles = []string{"rootdir/1", "rootdir/subdir/2"}
//     ch = s.minioClient.ListObjects(fmt.Sprintf("master.%s", repo), "r", true, make(chan struct{}))
//     checkListObjects(t, ch, startTime, endTime, expectedFiles, []string{})
//     ch = s.minioClient.ListObjects(fmt.Sprintf("master.%s", repo), "rootdir", true, make(chan struct{}))
//     checkListObjects(t, ch, startTime, endTime, expectedFiles, []string{})
//     ch = s.minioClient.ListObjects(fmt.Sprintf("master.%s", repo), "rootdir/", true, make(chan struct{}))
//     checkListObjects(t, ch, startTime, endTime, expectedFiles, []string{})

//     // Requests that will list all files in subdir
//     expectedFiles = []string{"rootdir/subdir/2"}
//     ch = s.minioClient.ListObjects(fmt.Sprintf("master.%s", repo), "rootdir/s", true, make(chan struct{}))
//     checkListObjects(t, ch, startTime, endTime, expectedFiles, []string{})
//     ch = s.minioClient.ListObjects(fmt.Sprintf("master.%s", repo), "rootdir/subdir", true, make(chan struct{}))
//     checkListObjects(t, ch, startTime, endTime, expectedFiles, []string{})
//     ch = s.minioClient.ListObjects(fmt.Sprintf("master.%s", repo), "rootdir/subdir/", true, make(chan struct{}))
//     checkListObjects(t, ch, startTime, endTime, expectedFiles, []string{})
//     ch = s.minioClient.ListObjects(fmt.Sprintf("master.%s", repo), "rootdir/subdir/2", true, make(chan struct{}))
//     checkListObjects(t, ch, startTime, endTime, expectedFiles, []string{})
// }

func TestWorkerDriver(t *testing.T) {
    if testing.Short() {
        t.Skip("Skipping integration tests in short mode")
    }

    pachClient, err := client.NewForTest()
    require.NoError(t, err)

    inputRepo := tu.UniqueString("testworkerdriverinput")
    require.NoError(t, pachClient.CreateRepo(inputRepo))
    outputRepo := tu.UniqueString("testworkerdriveroutput")
    require.NoError(t, pachClient.CreateRepo(outputRepo))

    // create a master branch on the input repo
    inputMasterCommit, err := pachClient.StartCommit(inputRepo, "master")
	require.NoError(t, err)
	_, err = pachClient.PutFile(inputRepo, inputMasterCommit.ID, "file", strings.NewReader("foo"))
	require.NoError(t, err)
	require.NoError(t, pachClient.FinishCommit(inputRepo, inputMasterCommit.ID))

	// create a develop branch on the input repo
    inputDevelopCommit, err := pachClient.StartCommit(inputRepo, "develop")
	require.NoError(t, err)
	for i := 0; i <= 1000; i++ {
        putListFileTestObject(t, pachClient, inputRepo, inputDevelopCommit.ID, "", i)
    }
    for i := 0; i < 10; i++ {
        putListFileTestObject(t, pachClient, inputRepo, inputDevelopCommit.ID, "dir/", i)
    }
	require.NoError(t, pachClient.FinishCommit(inputRepo, inputDevelopCommit.ID))

	// create the output branch
    outputCommit, err := pachClient.StartCommit(outputRepo, "master")
	require.NoError(t, err)

    driver := NewWorkerDriver(
        []*Bucket{
            &Bucket{
                Repo: inputRepo,
                Commit: inputMasterCommit.ID,
                Name: "in1",
            },
            &Bucket{
                Repo: inputRepo,
                Commit: inputDevelopCommit.ID,
                Name: "in2",
            },
        },
        &Bucket{
            Repo: outputRepo,
            Commit: outputCommit.ID,
            Name: "out",
        },
    )

    testRunner(t, "worker", driver, func(t *testing.T, pachClient *client.APIClient, minioClient *minio.Client) {
    	s := &workerTestState{
			pachClient: pachClient,
			minioClient: minioClient,
			inputRepo: inputRepo,
			outputRepo: outputRepo,
			inputMasterCommit: inputMasterCommit,
			inputDevelopCommit: inputDevelopCommit,
			outputCommit: outputCommit,
    	}

        t.Run("ListBuckets", func(t *testing.T) {
            workerListBuckets(t, s)
        })
        t.Run("GetObject", func(t *testing.T) {
            workerGetObject(t, s)
        })
        t.Run("GetObjectOutputRepo", func(t *testing.T) {
            workerGetObjectOutputRepo(t, s)
        })
        t.Run("StatObject", func(t *testing.T) {
            workerStatObject(t, s)
        })
        t.Run("PutObject", func(t *testing.T) {
            workerPutObject(t, s)
        })
        t.Run("PutObjectInputRepo", func(t *testing.T) {
            workerPutObjectInputRepo(t, s)
        })
        t.Run("RemoveObject", func(t *testing.T) {
            workerRemoveObject(t, s)
        })
        t.Run("RemoveObjectInputRepo", func(t *testing.T) {
            workerRemoveObjectInputRepo(t, s)
        })
        t.Run("LargeObjects", func(t *testing.T) {
            workerLargeObjects(t, s)
        })
        t.Run("MakeBucket", func(t *testing.T) {
            workerMakeBucket(t, s)
        })
        t.Run("BucketExists", func(t *testing.T) {
            workerBucketExists(t, s)
        })
        t.Run("RemoveBucket", func(t *testing.T) {
            workerRemoveBucket(t, s)
        })
        t.Run("ListObjectsPaginated", func(t *testing.T) {
            workerListObjectsPaginated(t, s)
        })
        // t.Run("ListObjectsRecursive", func(t *testing.T) {
        //     workerListObjectsRecursive(t, s)
        // })
    })
}
<commit_msg>Test for recursively listing files<commit_after>package s3

import (
	"io/ioutil"
	"os"
	"strings"
	"testing"
	"fmt"

	minio "github.com/minio/minio-go"

	tu "github.com/pachyderm/pachyderm/src/server/pkg/testutil"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pfs"
)

type workerTestState struct {
	pachClient *client.APIClient
	minioClient *minio.Client
	inputRepo string
	outputRepo string
	inputMasterCommit *pfs.Commit
	inputDevelopCommit *pfs.Commit
	outputCommit *pfs.Commit
}

func workerListBuckets(t *testing.T, s *workerTestState) {
	// create a repo - this should not show up list buckets with the worker
	// driver
	repo := tu.UniqueString("testlistbuckets1")
	require.NoError(t, s.pachClient.CreateRepo(repo))
	require.NoError(t, s.pachClient.CreateBranch(repo, "master", "", nil))

	buckets, err := s.minioClient.ListBuckets()
	require.NoError(t, err)

	actualBucketNames := []string{}
	for _, bucket := range buckets {
		actualBucketNames = append(actualBucketNames, bucket.Name)
	}

	require.ElementsEqual(t, []string{"in1", "in2", "out"}, actualBucketNames)
}

func workerGetObject(t *testing.T, s *workerTestState) {
	fetchedContent, err := getObject(t, s.minioClient, "in1", "0")
	require.NoError(t, err)
	require.Equal(t, "0\n", fetchedContent)
}

func workerGetObjectOutputRepo(t *testing.T, s *workerTestState) {
	_, err := getObject(t, s.minioClient, "out", "file")
	keyNotFoundError(t, err)
}

func workerStatObject(t *testing.T, s *workerTestState) {
	info, err := s.minioClient.StatObject("in1", "0", minio.StatObjectOptions{})
	require.NoError(t, err)
	require.True(t, len(info.ETag) > 0)
	require.Equal(t, "text/plain; charset=utf-8", info.ContentType)
	require.Equal(t, int64(2), info.Size)
}

func workerPutObject(t *testing.T, s *workerTestState) {
	r := strings.NewReader("content1")
	_, err := s.minioClient.PutObject("out", "file", r, int64(r.Len()), minio.PutObjectOptions{ContentType: "text/plain"})
	require.NoError(t, err)

	// this should act as a PFS PutFileOverwrite
	r2 := strings.NewReader("content2")
	_, err = s.minioClient.PutObject("out", "file", r2, int64(r2.Len()), minio.PutObjectOptions{ContentType: "text/plain"})
	require.NoError(t, err)

	_, err = getObject(t, s.minioClient, "out", "file")
	keyNotFoundError(t, err)
}

func workerPutObjectInputRepo(t *testing.T, s *workerTestState) {
	r := strings.NewReader("content1")
	_, err := s.minioClient.PutObject("in1", "0", r, int64(r.Len()), minio.PutObjectOptions{ContentType: "text/plain"})
	notImplementedError(t, err)
}

func workerRemoveObject(t *testing.T, s *workerTestState) {
	_, err := s.pachClient.PutFile(s.outputRepo, s.outputCommit.ID, "file", strings.NewReader("content"))
	require.NoError(t, err)

	// as per PFS semantics, the second delete should be a no-op
	require.NoError(t, s.minioClient.RemoveObject("out", "file"))
	require.NoError(t, s.minioClient.RemoveObject("out", "file"))
}

func workerRemoveObjectInputRepo(t *testing.T, s *workerTestState) {
	err := s.minioClient.RemoveObject("in1", "0")
	notImplementedError(t, err)
}

// Tests inserting and getting files over 64mb in size
func workerLargeObjects(t *testing.T, s *workerTestState) {
	// create a temporary file to put ~65mb of contents into it
	inputFile, err := ioutil.TempFile("", "pachyderm-test-large-objects-input-*")
	require.NoError(t, err)
	defer os.Remove(inputFile.Name())
	n, err := inputFile.WriteString(strings.Repeat("no tv and no beer make homer something something.\n", 1363149))
	require.NoError(t, err)
	require.Equal(t, n, 68157450)
	require.NoError(t, inputFile.Sync())

	// first ensure that putting into a repo that doesn't exist triggers an
	// error
	_, err = s.minioClient.FPutObject("foobar", "file", inputFile.Name(), minio.PutObjectOptions{
		ContentType: "text/plain",
	})
	bucketNotFoundError(t, err)

	// now try putting into a legit repo
	l, err := s.minioClient.FPutObject("out", "file", inputFile.Name(), minio.PutObjectOptions{
		ContentType: "text/plain",
	})
	require.NoError(t, err)
	require.Equal(t, int(l), 68157450)

	// try getting an object that does not exist
	err = s.minioClient.FGetObject("foobar", "file", "foo", minio.GetObjectOptions{})
	bucketNotFoundError(t, err)

	// get the file that does exist, doesn't work because we're reading from
	// an output repo
	outputFile, err := ioutil.TempFile("", "pachyderm-test-large-objects-output-*")
	require.NoError(t, err)
	defer os.Remove(outputFile.Name())
	err = s.minioClient.FGetObject("out", "file", outputFile.Name(), minio.GetObjectOptions{})
	keyNotFoundError(t, err)
}

func workerMakeBucket(t *testing.T, s *workerTestState) {
	repo := tu.UniqueString("testmakebucket")
	notImplementedError(t, s.minioClient.MakeBucket(repo, ""))
}

func workerBucketExists(t *testing.T, s *workerTestState) {
	exists, err := s.minioClient.BucketExists("in1")
	require.NoError(t, err)
	require.True(t, exists)

	exists, err = s.minioClient.BucketExists("out")
	require.NoError(t, err)
	require.True(t, exists)

	exists, err = s.minioClient.BucketExists("foobar")
	require.NoError(t, err)
	require.False(t, exists)
}

func workerRemoveBucket(t *testing.T, s *workerTestState) {
	notImplementedError(t, s.minioClient.RemoveBucket("in1"))
	notImplementedError(t, s.minioClient.RemoveBucket("out"))
}

func workerListObjectsPaginated(t *testing.T, s *workerTestState) {
	// Request that will list all files in root
	ch := s.minioClient.ListObjects("in2", "", false, make(chan struct{}))
	expectedFiles := []string{}
	for i := 0; i <= 1000; i++ {
		expectedFiles = append(expectedFiles, fmt.Sprintf("%d", i))
	}
	checkListObjects(t, ch, nil, nil, expectedFiles, []string{"dir/"})

	// Request that will list all files starting with 1
	ch = s.minioClient.ListObjects("in2", "1", false, make(chan struct{}))
	expectedFiles = []string{}
	for i := 0; i <= 1000; i++ {
		file := fmt.Sprintf("%d", i)
		if strings.HasPrefix(file, "1") {
			expectedFiles = append(expectedFiles, file)
		}
	}
	checkListObjects(t, ch, nil, nil, expectedFiles, []string{})

	// Request that will list all files in a directory
	ch = s.minioClient.ListObjects("in2", "dir/", false, make(chan struct{}))
	expectedFiles = []string{}
	for i := 0; i < 10; i++ {
		expectedFiles = append(expectedFiles, fmt.Sprintf("dir/%d", i))
	}
	checkListObjects(t, ch, nil, nil, expectedFiles, []string{})
}

func workerListObjectsRecursive(t *testing.T, s *workerTestState) {
	// Request that will list all files in master
	expectedFiles := []string{"0", "rootdir/1", "rootdir/subdir/2"}
	ch := s.minioClient.ListObjects("in1", "", true, make(chan struct{}))
	checkListObjects(t, ch, nil, nil, expectedFiles, []string{})

	// Requests that will list all files in rootdir
	expectedFiles = []string{"rootdir/1", "rootdir/subdir/2"}
	ch = s.minioClient.ListObjects("in1", "r", true, make(chan struct{}))
	checkListObjects(t, ch, nil, nil, expectedFiles, []string{})
	ch = s.minioClient.ListObjects("in1", "rootdir", true, make(chan struct{}))
	checkListObjects(t, ch, nil, nil, expectedFiles, []string{})
	ch = s.minioClient.ListObjects("in1", "rootdir/", true, make(chan struct{}))
	checkListObjects(t, ch, nil, nil, expectedFiles, []string{})

	// Requests that will list all files in subdir
	expectedFiles = []string{"rootdir/subdir/2"}
	ch = s.minioClient.ListObjects("in1", "rootdir/s", true, make(chan struct{}))
	checkListObjects(t, ch, nil, nil, expectedFiles, []string{})
	ch = s.minioClient.ListObjects("in1", "rootdir/subdir", true, make(chan struct{}))
	checkListObjects(t, ch, nil, nil, expectedFiles, []string{})
	ch = s.minioClient.ListObjects("in1", "rootdir/subdir/", true, make(chan struct{}))
	checkListObjects(t, ch, nil, nil, expectedFiles, []string{})
	ch = s.minioClient.ListObjects("in1", "rootdir/subdir/2", true, make(chan struct{}))
	checkListObjects(t, ch, nil, nil, expectedFiles, []string{})
}

func TestWorkerDriver(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	pachClient, err := client.NewForTest()
	require.NoError(t, err)

	inputRepo := tu.UniqueString("testworkerdriverinput")
	require.NoError(t, pachClient.CreateRepo(inputRepo))
	outputRepo := tu.UniqueString("testworkerdriveroutput")
	require.NoError(t, pachClient.CreateRepo(outputRepo))

	// create a master branch on the input repo
	inputMasterCommit, err := pachClient.StartCommit(inputRepo, "master")
	require.NoError(t, err)
	putListFileTestObject(t, pachClient, inputRepo, inputMasterCommit.ID, "", 0)
	putListFileTestObject(t, pachClient, inputRepo, inputMasterCommit.ID, "rootdir/", 1)
	putListFileTestObject(t, pachClient, inputRepo, inputMasterCommit.ID, "rootdir/subdir/", 2)
	require.NoError(t, pachClient.FinishCommit(inputRepo, inputMasterCommit.ID))

	// create a develop branch on the input repo
	inputDevelopCommit, err := pachClient.StartCommit(inputRepo, "develop")
	require.NoError(t, err)
	for i := 0; i <= 1000; i++ {
		putListFileTestObject(t, pachClient, inputRepo, inputDevelopCommit.ID, "", i)
	}
	for i := 0; i < 10; i++ {
		putListFileTestObject(t, pachClient, inputRepo, inputDevelopCommit.ID, "dir/", i)
	}
	require.NoError(t, pachClient.FinishCommit(inputRepo, inputDevelopCommit.ID))

	// create the output branch
	outputCommit, err := pachClient.StartCommit(outputRepo, "master")
	require.NoError(t, err)

	driver := NewWorkerDriver(
		[]*Bucket{
			&Bucket{
				Repo: inputRepo,
				Commit: inputMasterCommit.ID,
				Name: "in1",
			},
			&Bucket{
				Repo: inputRepo,
				Commit: inputDevelopCommit.ID,
				Name: "in2",
			},
		},
		&Bucket{
			Repo: outputRepo,
			Commit: outputCommit.ID,
			Name: "out",
		},
	)

	testRunner(t, "worker", driver, func(t *testing.T, pachClient *client.APIClient, minioClient *minio.Client) {
		s := &workerTestState{
			pachClient: pachClient,
			minioClient: minioClient,
			inputRepo: inputRepo,
			outputRepo: outputRepo,
			inputMasterCommit: inputMasterCommit,
			inputDevelopCommit: inputDevelopCommit,
			outputCommit: outputCommit,
		}

		t.Run("ListBuckets", func(t *testing.T) {
			workerListBuckets(t, s)
		})
		t.Run("GetObject", func(t *testing.T) {
			workerGetObject(t, s)
		})
		t.Run("GetObjectOutputRepo", func(t *testing.T) {
			workerGetObjectOutputRepo(t, s)
		})
		t.Run("StatObject", func(t *testing.T) {
			workerStatObject(t, s)
		})
		t.Run("PutObject", func(t *testing.T) {
			workerPutObject(t, s)
		})
		t.Run("PutObjectInputRepo", func(t *testing.T) {
			workerPutObjectInputRepo(t, s)
		})
		t.Run("RemoveObject", func(t *testing.T) {
			workerRemoveObject(t, s)
		})
		t.Run("RemoveObjectInputRepo", func(t *testing.T) {
			workerRemoveObjectInputRepo(t, s)
		})
		t.Run("LargeObjects", func(t *testing.T) {
			workerLargeObjects(t, s)
		})
		t.Run("MakeBucket", func(t *testing.T) {
			workerMakeBucket(t, s)
		})
		t.Run("BucketExists", func(t *testing.T) {
			workerBucketExists(t, s)
		})
		t.Run("RemoveBucket", func(t *testing.T) {
			workerRemoveBucket(t, s)
		})
		t.Run("ListObjectsPaginated", func(t *testing.T) {
			workerListObjectsPaginated(t, s)
		})
		t.Run("ListObjectsRecursive", func(t *testing.T) {
			workerListObjectsRecursive(t, s)
		})
	})
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os/exec"
	"strconv"
)

import (
	yaml "github.com/kylelemons/go-gypsy/yaml"
)

var configFile string
func init() {
	const (
		defaultConfigFile = "/etc/jojo.yaml"
		usage             = "full path to config file"
	)
	flag.StringVar(&configFile, "config", defaultConfigFile, usage)
	flag.StringVar(&configFile, "c", defaultConfigFile, usage+" (shorthand)")
}

var port uint64
func init() {
	const (
		defaultPort = 3000
		usage       = "port to serve on"
	)
	flag.Uint64Var(&port, "port", defaultPort, usage)
	flag.Uint64Var(&port, "p", defaultPort, usage+" (shorthand)")
}

func scriptHandler(script string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
                log.Printf("[Info] %s %s %s %s", r.Proto, r.Method, r.Host, r.URL.Path)
                log.Printf("[DEBUG] %s", r.Header)
                fmt.Fprintf(w, "Hey, I'm going to call: %s", script)
                log.Printf("[Info] Running %s", script)
                cmd := exec.Command(script, "")
                var out bytes.Buffer
                cmd.Stdout = &out
                cmd.Stderr = &out
                err := cmd.Run()
                if err != nil {
                        log.Fatal(err)
                }
                fmt.Fprintf(w, "Results: %s", out.String())
                log.Printf("[Info] Results: %s", out.String())
	}
}

func loadConfig() {
	config := yaml.ConfigFile(configFile)
	numRoutes, err := config.Count("routes")
	if err != nil {
		log.Fatalf("Error %s", err)
	}
	for i := 0; i < numRoutes; i++ {
		url, _ := config.Get(fmt.Sprintf("routes[%d].url", i))
		script, _ := config.Get(fmt.Sprintf("routes[%d].script", i))
		http.HandleFunc(url, scriptHandler(script))
	}
}

func main() {
	flag.Parse()
	loadConfig()
	log.Println("[INFO] Starting server on port "+strconv.FormatUint(port, 10))
	log.Fatalf("[FATAL] %s", http.ListenAndServe(":"+strconv.FormatUint(port, 10), nil))
}
<commit_msg>stop dying on non-zero script exit status<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os/exec"
	"strconv"
)

import (
	yaml "github.com/kylelemons/go-gypsy/yaml"
)

var configFile string
func init() {
	const (
		defaultConfigFile = "/etc/jojo.yaml"
		usage             = "full path to config file"
	)
	flag.StringVar(&configFile, "config", defaultConfigFile, usage)
	flag.StringVar(&configFile, "c", defaultConfigFile, usage+" (shorthand)")
}

var port uint64
func init() {
	const (
		defaultPort = 3000
		usage       = "port to serve on"
	)
	flag.Uint64Var(&port, "port", defaultPort, usage)
	flag.Uint64Var(&port, "p", defaultPort, usage+" (shorthand)")
}

func scriptHandler(script string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
                log.Printf("[Info] %s %s %s %s", r.Proto, r.Method, r.Host, r.URL.Path)
                log.Printf("[DEBUG] %s", r.Header)
                fmt.Fprintf(w, "
Hey, I'm going to call: %s
", script)
                log.Printf("[Info] Running %s", script)
                cmd := exec.Command(script, "")
                var out bytes.Buffer
                cmd.Stdout = &out
                cmd.Stderr = &out
                err := cmd.Run()
                if err != nil {
                  fmt.Fprintf(w, "
Got an error: %s
", err)
                  log.Printf("[ERROR] %s", err)
                }
                fmt.Fprintf(w, "
Results: %s
", out.String())
                log.Printf("[Info] Results: %s", out.String())
	}
}

func loadConfig() {
	config := yaml.ConfigFile(configFile)
	numRoutes, err := config.Count("routes")
	if err != nil {
		log.Fatalf("Error %s", err)
	}
	for i := 0; i < numRoutes; i++ {
		url, _ := config.Get(fmt.Sprintf("routes[%d].url", i))
		script, _ := config.Get(fmt.Sprintf("routes[%d].script", i))
		http.HandleFunc(url, scriptHandler(script))
	}
}

func main() {
	flag.Parse()
	loadConfig()
	log.Println("[INFO] Starting server on port "+strconv.FormatUint(port, 10))
	log.Fatalf("[FATAL] %s", http.ListenAndServe(":"+strconv.FormatUint(port, 10), nil))
}
<|endoftext|>


<commit_before>package sdp

import (
	"fmt"
	"net"
	"time"
)

// Constants for SDP attributes used in JSEP
const (
	AttrKeyIdentity        = "identity"
	AttrKeyGroup           = "group"
	AttrKeySsrc            = "ssrc"
	AttrKeySsrcGroup       = "ssrc-group"
	AttrKeyMsidSemantic    = "msid-semantic"
	AttrKeyConnectionSetup = "setup"
	AttrKeyMID             = "mid"
	AttrKeyICELite         = "ice-lite"
	AttrKeyRtcpMux         = "rtcp-mux"
	AttrKeyRtcpRsize       = "rtcp-rsize"
)

// Constants for semantic tokens used in JSEP
const (
	SemanticTokenLipSynchronization     = "LS"
	SemanticTokenFlowIdentification     = "FID"
	SemanticTokenForwardErrorCorrection = "FEC"
	SemanticTokenWebRTCMediaStreams     = "WMS"
)

// API to match draft-ietf-rtcweb-jsep
// Move to webrtc or its own package?

// NewJSEPSessionDescription creates a new SessionDescription with
// some settings that are required by the JSEP spec.
func NewJSEPSessionDescription(fingerprint string, identity bool) *SessionDescription {
	d := &SessionDescription{
		Version: 0,
		Origin: Origin{
			Username:       "-",
			SessionID:      newSessionID(),
			SessionVersion: uint64(time.Now().Unix()),
			NetworkType:    "IN",
			AddressType:    "IP4",
			UnicastAddress: "0.0.0.0",
		},
		SessionName: "-",
		TimeDescriptions: []TimeDescription{
			{
				Timing: Timing{
					StartTime: 0,
					StopTime:  0,
				},
				RepeatTimes: nil,
			},
		},
		Attributes: []Attribute{
			// 	"Attribute(ice-options:trickle)", // TODO: implement trickle ICE
			Attribute("fingerprint:sha-256 " + fingerprint),
		},
	}

	if identity {
		d.WithPropertyAttribute(AttrKeyIdentity)
	}

	return d
}

// WithPropertyAttribute adds a property attribute 'a=key' to the session description
func (s *SessionDescription) WithPropertyAttribute(key string) *SessionDescription {
	s.Attributes = append(s.Attributes, Attribute(key))
	return s
}

// WithValueAttribute adds a value attribute 'a=key:value' to the session description
func (s *SessionDescription) WithValueAttribute(key, value string) *SessionDescription {
	s.Attributes = append(s.Attributes, Attribute(fmt.Sprintf("%s:%s", key, value)))
	return s
}

// WithMedia adds a media description to the session description
func (s *SessionDescription) WithMedia(md *MediaDescription) *SessionDescription {
	s.MediaDescriptions = append(s.MediaDescriptions, md)
	return s
}

// NewJSEPMediaDescription creates a new MediaName with
// some settings that are required by the JSEP spec.
func NewJSEPMediaDescription(codecType string, codecPrefs []string) *MediaDescription {
	// TODO: handle codecPrefs
	d := &MediaDescription{
		MediaName: MediaName{
			Media:  codecType,
			Port:   RangedPort{Value: 9},
			Protos: []string{"UDP", "TLS", "RTP", "SAVPF"},
		},
		ConnectionInformation: &ConnectionInformation{
			NetworkType: "IN",
			AddressType: "IP4",
			Address: &Address{
				IP: net.ParseIP("0.0.0.0"),
			},
		},
	}
	return d
}

// WithPropertyAttribute adds a property attribute 'a=key' to the media description
func (d *MediaDescription) WithPropertyAttribute(key string) *MediaDescription {
	d.Attributes = append(d.Attributes, Attribute(key))
	return d
}

// WithValueAttribute adds a value attribute 'a=key:value' to the media description
func (d *MediaDescription) WithValueAttribute(key, value string) *MediaDescription {
	d.Attributes = append(d.Attributes, Attribute(fmt.Sprintf("%s:%s", key, value)))
	return d
}

// WithICECredentials adds ICE credentials to the media description
func (d *MediaDescription) WithICECredentials(username, password string) *MediaDescription {
	return d.
		WithValueAttribute("ice-ufrag", username).
		WithValueAttribute("ice-pwd", password)
}

// WithCodec adds codec information to the media description
func (d *MediaDescription) WithCodec(payloadType uint8, name string, clockrate uint32, channels uint16, fmtp string) *MediaDescription {
	d.MediaName.Formats = append(d.MediaName.Formats, int(payloadType))
	rtpmap := fmt.Sprintf("%d %s/%d", payloadType, name, clockrate)
	if channels > 0 {
		rtpmap = rtpmap + fmt.Sprintf("/%d", channels)
	}
	d.WithValueAttribute("rtpmap", rtpmap)
	if fmtp != "" {
		d.WithValueAttribute("fmtp", fmt.Sprintf("%d %s", payloadType, fmtp))
	}
	return d
}

// WithMediaSource adds media source information to the media description
func (d *MediaDescription) WithMediaSource(ssrc uint32, cname, streamLabel, label string) *MediaDescription {
	return d.
		WithValueAttribute("ssrc", fmt.Sprintf("%d cname:%s", ssrc, cname)). // Deprecated but not phased out?
		WithValueAttribute("ssrc", fmt.Sprintf("%d msid:%s %s", ssrc, streamLabel, label)).
		WithValueAttribute("ssrc", fmt.Sprintf("%d mslabel:%s", ssrc, streamLabel)). // Deprecated but not phased out?
		WithValueAttribute("ssrc", fmt.Sprintf("%d label:%s", ssrc, label))          // Deprecated but not phased out?
}

// WithCandidate adds an ICE candidate to the media description
func (d *MediaDescription) WithCandidate(value string) *MediaDescription {
	return d.WithValueAttribute("candidate", value)
}
<commit_msg>DTLS: Switch to native DTLS<commit_after>package sdp

import (
	"fmt"
	"net"
	"time"
)

// Constants for SDP attributes used in JSEP
const (
	AttrKeyIdentity        = "identity"
	AttrKeyGroup           = "group"
	AttrKeySsrc            = "ssrc"
	AttrKeySsrcGroup       = "ssrc-group"
	AttrKeyMsidSemantic    = "msid-semantic"
	AttrKeyConnectionSetup = "setup"
	AttrKeyMID             = "mid"
	AttrKeyICELite         = "ice-lite"
	AttrKeyRtcpMux         = "rtcp-mux"
	AttrKeyRtcpRsize       = "rtcp-rsize"
)

// Constants for semantic tokens used in JSEP
const (
	SemanticTokenLipSynchronization     = "LS"
	SemanticTokenFlowIdentification     = "FID"
	SemanticTokenForwardErrorCorrection = "FEC"
	SemanticTokenWebRTCMediaStreams     = "WMS"
)

// API to match draft-ietf-rtcweb-jsep
// Move to webrtc or its own package?

// NewJSEPSessionDescription creates a new SessionDescription with
// some settings that are required by the JSEP spec.
func NewJSEPSessionDescription(identity bool) *SessionDescription {
	d := &SessionDescription{
		Version: 0,
		Origin: Origin{
			Username:       "-",
			SessionID:      newSessionID(),
			SessionVersion: uint64(time.Now().Unix()),
			NetworkType:    "IN",
			AddressType:    "IP4",
			UnicastAddress: "0.0.0.0",
		},
		SessionName: "-",
		TimeDescriptions: []TimeDescription{
			{
				Timing: Timing{
					StartTime: 0,
					StopTime:  0,
				},
				RepeatTimes: nil,
			},
		},
		Attributes: []Attribute{
			// 	"Attribute(ice-options:trickle)", // TODO: implement trickle ICE
		},
	}

	if identity {
		d.WithPropertyAttribute(AttrKeyIdentity)
	}

	return d
}

// WithPropertyAttribute adds a property attribute 'a=key' to the session description
func (s *SessionDescription) WithPropertyAttribute(key string) *SessionDescription {
	s.Attributes = append(s.Attributes, Attribute(key))
	return s
}

// WithValueAttribute adds a value attribute 'a=key:value' to the session description
func (s *SessionDescription) WithValueAttribute(key, value string) *SessionDescription {
	s.Attributes = append(s.Attributes, Attribute(fmt.Sprintf("%s:%s", key, value)))
	return s
}

// WithFingerprint adds a fingerprint to the session description
func (s *SessionDescription) WithFingerprint(algorithm, value string) *SessionDescription {
	return s.WithValueAttribute("fingerprint", algorithm+" "+value)
}

// WithMedia adds a media description to the session description
func (s *SessionDescription) WithMedia(md *MediaDescription) *SessionDescription {
	s.MediaDescriptions = append(s.MediaDescriptions, md)
	return s
}

// NewJSEPMediaDescription creates a new MediaName with
// some settings that are required by the JSEP spec.
func NewJSEPMediaDescription(codecType string, codecPrefs []string) *MediaDescription {
	// TODO: handle codecPrefs
	d := &MediaDescription{
		MediaName: MediaName{
			Media:  codecType,
			Port:   RangedPort{Value: 9},
			Protos: []string{"UDP", "TLS", "RTP", "SAVPF"},
		},
		ConnectionInformation: &ConnectionInformation{
			NetworkType: "IN",
			AddressType: "IP4",
			Address: &Address{
				IP: net.ParseIP("0.0.0.0"),
			},
		},
	}
	return d
}

// WithPropertyAttribute adds a property attribute 'a=key' to the media description
func (d *MediaDescription) WithPropertyAttribute(key string) *MediaDescription {
	d.Attributes = append(d.Attributes, Attribute(key))
	return d
}

// WithValueAttribute adds a value attribute 'a=key:value' to the media description
func (d *MediaDescription) WithValueAttribute(key, value string) *MediaDescription {
	d.Attributes = append(d.Attributes, Attribute(fmt.Sprintf("%s:%s", key, value)))
	return d
}

// WithICECredentials adds ICE credentials to the media description
func (d *MediaDescription) WithICECredentials(username, password string) *MediaDescription {
	return d.
		WithValueAttribute("ice-ufrag", username).
		WithValueAttribute("ice-pwd", password)
}

// WithCodec adds codec information to the media description
func (d *MediaDescription) WithCodec(payloadType uint8, name string, clockrate uint32, channels uint16, fmtp string) *MediaDescription {
	d.MediaName.Formats = append(d.MediaName.Formats, int(payloadType))
	rtpmap := fmt.Sprintf("%d %s/%d", payloadType, name, clockrate)
	if channels > 0 {
		rtpmap = rtpmap + fmt.Sprintf("/%d", channels)
	}
	d.WithValueAttribute("rtpmap", rtpmap)
	if fmtp != "" {
		d.WithValueAttribute("fmtp", fmt.Sprintf("%d %s", payloadType, fmtp))
	}
	return d
}

// WithMediaSource adds media source information to the media description
func (d *MediaDescription) WithMediaSource(ssrc uint32, cname, streamLabel, label string) *MediaDescription {
	return d.
		WithValueAttribute("ssrc", fmt.Sprintf("%d cname:%s", ssrc, cname)). // Deprecated but not phased out?
		WithValueAttribute("ssrc", fmt.Sprintf("%d msid:%s %s", ssrc, streamLabel, label)).
		WithValueAttribute("ssrc", fmt.Sprintf("%d mslabel:%s", ssrc, streamLabel)). // Deprecated but not phased out?
		WithValueAttribute("ssrc", fmt.Sprintf("%d label:%s", ssrc, label))          // Deprecated but not phased out?
}

// WithCandidate adds an ICE candidate to the media description
func (d *MediaDescription) WithCandidate(value string) *MediaDescription {
	return d.WithValueAttribute("candidate", value)
}
<|endoftext|>


<commit_before>/*
Package jsonware has a middleware setup for doing RESTful reqeuests with net/http.
It makes it easy to unobtrusively serialize and deserialize json in a type safe
manner, and handle errors from the internal handlers including logging.
*/
package jsonware

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"reflect"
	"strings"
)

/*
JSONHandler handles json api endpoint restful requests. It can be constructed
by passing a suitable function into the JSON function.

Error handling is handled differently depending on whether or not JSONErr was
used to report the error. See JSONErr documentation to understand how to control
error handling.

To log cloaked errors, use the Log() function to set an io.Writer to log to
when reporting a real internal server error.

	// Register a JSONHandler.
	http.Handle("/", JSON(myHandler).Log(myLogger))
*/
type JSONHandler struct {
	logger io.Writer
	fn     reflect.Value
	in     reflect.Type
}

// Log sets the JSONHandler's logging io.Writer for writing out cloaked errors.
func (j *JSONHandler) Log(logger io.Writer) *JSONHandler {
	j.logger = logger
	return j
}

/*
JSONErr can be used in a JSONHandler to override the error mechanism in
JSONHandler's ServeHTTP method. If a status is set it will obey it,
otherwise it will assume 200 OK. The error message will be relayed to the
client. If you wish to use a general server error, simply return error from
the handler.

	func handler(w http.ResponseWriter, r *http.Request) {
		return nil, nil // No response from ServeHTTP, use w to create one.
	}

	func handler(w http.ResponseWriter, r *http.Request) {
		return nil, errors.New("hi") // 500 Response with cloaked+logged error.
	}

	func handler(w http.ResponseWriter, r *http.Request) {
		return nil, JSONErr{Err: errors.New("hi")} // 200 Response with error output to client
	}

	func handler(w http.ResponseWriter, r *http.Request) {
		return nil, JSONErr{400, errors.New("hi")} // 400 Response with error output to client
	}
*/
type JSONErr struct {
	Status int
	Err    error
}

// Error returns Error() from the internal error.
func (j JSONErr) Error() string {
	return j.Err.Error()
}

// ServeHTTP serves an http response, see JSONHandler documentation for details.
func (j JSONHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// Ensure request accepts json
	ah := r.Header.Get("Accept")
	if !strings.Contains(ah, "*/*") && !strings.Contains(ah, "application/json") {
		w.Header().Set("Content-Type", "text/plain")
		w.WriteHeader(http.StatusBadRequest)
		io.WriteString(w, "this endpoint only responds to json-accepting clients")
		return
	}

	w.Header().Set("Content-Type", "application/json")

	// Ensure request follows REST principles.
	deserialize := j.fn.Type().NumIn() == 3
	switch {
	case deserialize && !isDataMethod(r.Method):
		fallthrough
	case !deserialize && isDataMethod(r.Method):
		writeJSONError(w, j.logger, JSONErr{
			http.StatusBadRequest,
			fmt.Errorf("invalid http method to this endpoint: %s", r.Method),
		})
		return
	}

	// Set up arguments for handler call.
	in := []reflect.Value{
		reflect.ValueOf(w), reflect.ValueOf(r),
	}
	var deserializeTo reflect.Value
	if deserialize {
		switch j.in.Kind() {
		case reflect.Slice, reflect.Map:
			deserializeTo = reflect.New(j.in)
			in = append(in, deserializeTo.Elem())
		case reflect.Ptr:
			deserializeTo = reflect.New(j.in.Elem())
			in = append(in, deserializeTo)
		}
	}

	// Do json deserialization of body.
	if deserialize {
		dec := json.NewDecoder(r.Body)
		defer r.Body.Close()

		if err := dec.Decode(deserializeTo.Interface()); err != nil {
			writeJSONError(w, j.logger, JSONErr{
				http.StatusBadRequest,
				fmt.Errorf("could not deserialize json request body"),
			})
			return
		}
	}

	out := j.fn.Call(in)

	// Handle error return value
	if !out[1].IsNil() {
		writeJSONError(w, j.logger, out[1].Interface().(error))
		return
	}

	// Serialize the interface{} return value
	if !out[0].IsNil() {
		enc := json.NewEncoder(w)
		if err := enc.Encode(out[0].Interface()); err != nil {
			writeJSONError(w, j.logger, JSONErr{
				http.StatusInternalServerError,
				fmt.Errorf("problem preparing response"),
			})
			return
		}
	}
}

// isDataMethod returns true if method is one of POST PUT or PATCH
func isDataMethod(method string) bool {
	return method == "POST" || method == "PATCH" || method == "PUT"
}

// jsonSanitizer does basic sanitization (except unicode) of json values.
var jsonSanitizer = strings.NewReplacer(
	`"`, `\"`, `\`, `\\`, `/`, `\/`, "\x08", "\\b",
	"\x12", "\\f", "\n", "\\n", "\r", "\\r", "\t", "\\t",
)

// writeJSONError writes an error out to the response.
func writeJSONError(w http.ResponseWriter, logger io.Writer, err error) {
	switch e := err.(type) {
	case JSONErr:
		if e.Status != 0 {
			w.WriteHeader(e.Status)
		} else {
			w.WriteHeader(http.StatusOK)
		}
		fmt.Fprintf(w, `{ "error": "%v" }`, jsonSanitizer.Replace(e.Error()))
	default:
		if logger != nil {
			fmt.Fprintf(logger, "internal error: %v", err)
		}
		w.WriteHeader(http.StatusInternalServerError)
		io.WriteString(w, `{ "error": "an internal server error occurred" }`)
	}
}

/*
JSON changes a function into a JSONHandler.
Acceptable forms of the input function:

	GET (Note: all variant return types also work with POST/PATCH/PUT)
	func Fn(w http.ResponseWriter, r *http.Request) (interface{}, error)
	func Fn(w http.ResponseWriter, r *http.Request) (*MyStruct, error)
	func Fn(w http.ResponseWriter, r *http.Request) ([]*MyStruct, error)
	func Fn(w http.ResponseWriter, r *http.Request) (map[string]*MyStruct, error)

	POST/PUT/PATCH
	func Fn(w http.ResponseWriter, r *http.Request, m *MyStruct) (interface{}, error)
	func Fn(w http.ResponseWriter, r *http.Request, m []*MyStruct) (interface{}, error)
	func Fn(w http.ResponseWriter, r *http.Request, m map[string]*MyStruct) (interface{}, error)
*/
func JSON(fn interface{}) *JSONHandler {
	typ := reflect.TypeOf(fn)
	if typ.Kind() != reflect.Func {
		panic("Can only register functions.")
	}

	var p1, p2, p3 reflect.Type

	switch typ.NumIn() {
	case 3:
		p3 = typ.In(2)
		if p3.Kind() != reflect.Ptr && p3.Kind() != reflect.Map && p3.Kind() != reflect.Slice {
			panic("Third argument must be an *object, map, or slice")
		}

		fallthrough
	case 2:
		p1, p2 = typ.In(0), typ.In(1)
		if "http.ResponseWriter" != p1.String() {
			panic("First argument must be an http.ResponseWriter")
		}

		if "*http.Request" != p2.String() {
			panic("Second argument must be a *http.Request")
		}
	default:
		panic("Handler must have 2-3 arguments: ResponseWriter, Request, [Object]")
	}

	if typ.NumOut() != 2 {
		panic("Handler must have two returns: *object or interface{}, and error")
	}

	o1, o2 := typ.Out(0), typ.Out(1)

	if "interface {}" != o1.String() && o1.Kind() != reflect.Ptr && o1.Kind() != reflect.Slice && o1.Kind() != reflect.Map {
		panic("First return must be an empty *object, map, slice or interface{}")
	}

	if "error" != o2.String() {
		panic("Second return must be an error")
	}

	return &JSONHandler{fn: reflect.ValueOf(fn), in: p3}
}
<commit_msg>Stop using defer for no reason.<commit_after>/*
Package jsonware has a middleware setup for doing RESTful reqeuests with net/http.
It makes it easy to unobtrusively serialize and deserialize json in a type safe
manner, and handle errors from the internal handlers including logging.
*/
package jsonware

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"reflect"
	"strings"
)

/*
JSONHandler handles json api endpoint restful requests. It can be constructed
by passing a suitable function into the JSON function.

Error handling is handled differently depending on whether or not JSONErr was
used to report the error. See JSONErr documentation to understand how to control
error handling.

To log cloaked errors, use the Log() function to set an io.Writer to log to
when reporting a real internal server error.

	// Register a JSONHandler.
	http.Handle("/", JSON(myHandler).Log(myLogger))
*/
type JSONHandler struct {
	logger io.Writer
	fn     reflect.Value
	in     reflect.Type
}

// Log sets the JSONHandler's logging io.Writer for writing out cloaked errors.
func (j *JSONHandler) Log(logger io.Writer) *JSONHandler {
	j.logger = logger
	return j
}

/*
JSONErr can be used in a JSONHandler to override the error mechanism in
JSONHandler's ServeHTTP method. If a status is set it will obey it,
otherwise it will assume 200 OK. The error message will be relayed to the
client. If you wish to use a general server error, simply return error from
the handler.

	func handler(w http.ResponseWriter, r *http.Request) {
		return nil, nil // No response from ServeHTTP, use w to create one.
	}

	func handler(w http.ResponseWriter, r *http.Request) {
		return nil, errors.New("hi") // 500 Response with cloaked+logged error.
	}

	func handler(w http.ResponseWriter, r *http.Request) {
		return nil, JSONErr{Err: errors.New("hi")} // 200 Response with error output to client
	}

	func handler(w http.ResponseWriter, r *http.Request) {
		return nil, JSONErr{400, errors.New("hi")} // 400 Response with error output to client
	}
*/
type JSONErr struct {
	Status int
	Err    error
}

// Error returns Error() from the internal error.
func (j JSONErr) Error() string {
	return j.Err.Error()
}

// ServeHTTP serves an http response, see JSONHandler documentation for details.
func (j JSONHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// Ensure request accepts json
	ah := r.Header.Get("Accept")
	if !strings.Contains(ah, "*/*") && !strings.Contains(ah, "application/json") {
		w.Header().Set("Content-Type", "text/plain")
		w.WriteHeader(http.StatusBadRequest)
		io.WriteString(w, "this endpoint only responds to json-accepting clients")
		return
	}

	w.Header().Set("Content-Type", "application/json")

	// Ensure request follows REST principles.
	deserialize := j.fn.Type().NumIn() == 3
	switch {
	case deserialize && !isDataMethod(r.Method):
		fallthrough
	case !deserialize && isDataMethod(r.Method):
		writeJSONError(w, j.logger, JSONErr{
			http.StatusBadRequest,
			fmt.Errorf("invalid http method to this endpoint: %s", r.Method),
		})
		return
	}

	// Set up arguments for handler call.
	in := []reflect.Value{
		reflect.ValueOf(w), reflect.ValueOf(r),
	}
	var deserializeTo reflect.Value
	if deserialize {
		switch j.in.Kind() {
		case reflect.Slice, reflect.Map:
			deserializeTo = reflect.New(j.in)
			in = append(in, deserializeTo.Elem())
		case reflect.Ptr:
			deserializeTo = reflect.New(j.in.Elem())
			in = append(in, deserializeTo)
		}
	}

	// Do json deserialization of body.
	if deserialize {
		dec := json.NewDecoder(r.Body)

		if err := dec.Decode(deserializeTo.Interface()); err != nil {
			writeJSONError(w, j.logger, JSONErr{
				http.StatusBadRequest,
				fmt.Errorf("could not deserialize json request body"),
			})
			return
		}
		r.Body.Close()
	}

	out := j.fn.Call(in)

	// Handle error return value
	if !out[1].IsNil() {
		writeJSONError(w, j.logger, out[1].Interface().(error))
		return
	}

	// Serialize the interface{} return value
	if !out[0].IsNil() {
		enc := json.NewEncoder(w)
		if err := enc.Encode(out[0].Interface()); err != nil {
			writeJSONError(w, j.logger, JSONErr{
				http.StatusInternalServerError,
				fmt.Errorf("problem preparing response"),
			})
			return
		}
	}
}

// isDataMethod returns true if method is one of POST PUT or PATCH
func isDataMethod(method string) bool {
	return method == "POST" || method == "PATCH" || method == "PUT"
}

// jsonSanitizer does basic sanitization (except unicode) of json values.
var jsonSanitizer = strings.NewReplacer(
	`"`, `\"`, `\`, `\\`, `/`, `\/`, "\x08", "\\b",
	"\x12", "\\f", "\n", "\\n", "\r", "\\r", "\t", "\\t",
)

// writeJSONError writes an error out to the response.
func writeJSONError(w http.ResponseWriter, logger io.Writer, err error) {
	switch e := err.(type) {
	case JSONErr:
		if e.Status != 0 {
			w.WriteHeader(e.Status)
		} else {
			w.WriteHeader(http.StatusOK)
		}
		fmt.Fprintf(w, `{ "error": "%v" }`, jsonSanitizer.Replace(e.Error()))
	default:
		if logger != nil {
			fmt.Fprintf(logger, "internal error: %v", err)
		}
		w.WriteHeader(http.StatusInternalServerError)
		io.WriteString(w, `{ "error": "an internal server error occurred" }`)
	}
}

/*
JSON changes a function into a JSONHandler.
Acceptable forms of the input function:

	GET (Note: all variant return types also work with POST/PATCH/PUT)
	func Fn(w http.ResponseWriter, r *http.Request) (interface{}, error)
	func Fn(w http.ResponseWriter, r *http.Request) (*MyStruct, error)
	func Fn(w http.ResponseWriter, r *http.Request) ([]*MyStruct, error)
	func Fn(w http.ResponseWriter, r *http.Request) (map[string]*MyStruct, error)

	POST/PUT/PATCH
	func Fn(w http.ResponseWriter, r *http.Request, m *MyStruct) (interface{}, error)
	func Fn(w http.ResponseWriter, r *http.Request, m []*MyStruct) (interface{}, error)
	func Fn(w http.ResponseWriter, r *http.Request, m map[string]*MyStruct) (interface{}, error)
*/
func JSON(fn interface{}) *JSONHandler {
	typ := reflect.TypeOf(fn)
	if typ.Kind() != reflect.Func {
		panic("Can only register functions.")
	}

	var p1, p2, p3 reflect.Type

	switch typ.NumIn() {
	case 3:
		p3 = typ.In(2)
		if p3.Kind() != reflect.Ptr && p3.Kind() != reflect.Map && p3.Kind() != reflect.Slice {
			panic("Third argument must be an *object, map, or slice")
		}

		fallthrough
	case 2:
		p1, p2 = typ.In(0), typ.In(1)
		if "http.ResponseWriter" != p1.String() {
			panic("First argument must be an http.ResponseWriter")
		}

		if "*http.Request" != p2.String() {
			panic("Second argument must be a *http.Request")
		}
	default:
		panic("Handler must have 2-3 arguments: ResponseWriter, Request, [Object]")
	}

	if typ.NumOut() != 2 {
		panic("Handler must have two returns: *object or interface{}, and error")
	}

	o1, o2 := typ.Out(0), typ.Out(1)

	if "interface {}" != o1.String() && o1.Kind() != reflect.Ptr && o1.Kind() != reflect.Slice && o1.Kind() != reflect.Map {
		panic("First return must be an empty *object, map, slice or interface{}")
	}

	if "error" != o2.String() {
		panic("Second return must be an error")
	}

	return &JSONHandler{fn: reflect.ValueOf(fn), in: p3}
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris

package mime

import (
	"bufio"
	"os"
	"strings"
)

func init() {
	osInitMime = initMimeUnix
}

var typeFiles = []string{
	"/etc/mime.types",
	"/etc/apache2/mime.types",
	"/etc/apache/mime.types",
}

func loadMimeFile(filename string) {
	f, err := os.Open(filename)
	if err != nil {
		return
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		fields := strings.Fields(scanner.Text())
		if len(fields) <= 1 || fields[0][0] == '#' {
			continue
		}
		mimeType := fields[0]
		for _, ext := range fields[1:] {
			if ext[0] == '#' {
				break
			}
			setExtensionType("."+ext, mimeType)
		}
	}
	if err := scanner.Err(); err != nil {
		panic(err)
	}
}

func initMimeUnix() {
	for _, filename := range typeFiles {
		loadMimeFile(filename)
	}
}

func initMimeForTests() map[string]string {
	typeFiles = []string{"testdata/test.types"}
	return map[string]string{
		".T1":  "application/test",
		".t2":  "text/test; charset=utf-8",
		".png": "image/png",
	}
}
<commit_msg>mime: add wasm architecture<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris

package mime

import (
	"bufio"
	"os"
	"strings"
)

func init() {
	osInitMime = initMimeUnix
}

var typeFiles = []string{
	"/etc/mime.types",
	"/etc/apache2/mime.types",
	"/etc/apache/mime.types",
}

func loadMimeFile(filename string) {
	f, err := os.Open(filename)
	if err != nil {
		return
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		fields := strings.Fields(scanner.Text())
		if len(fields) <= 1 || fields[0][0] == '#' {
			continue
		}
		mimeType := fields[0]
		for _, ext := range fields[1:] {
			if ext[0] == '#' {
				break
			}
			setExtensionType("."+ext, mimeType)
		}
	}
	if err := scanner.Err(); err != nil {
		panic(err)
	}
}

func initMimeUnix() {
	for _, filename := range typeFiles {
		loadMimeFile(filename)
	}
}

func initMimeForTests() map[string]string {
	typeFiles = []string{"testdata/test.types"}
	return map[string]string{
		".T1":  "application/test",
		".t2":  "text/test; charset=utf-8",
		".png": "image/png",
	}
}
<|endoftext|>


<commit_before>// Package lile provides helper methods to quickly create RPC based services
// that have metrics, tracing and pub/sub support
package lile

import (
	"net"
	"net/http"
	"os"
	"time"

	"github.com/grpc-ecosystem/go-grpc-middleware"
	"github.com/grpc-ecosystem/go-grpc-prometheus"
	"github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc"
	"github.com/lileio/lile/fromenv"
	"github.com/prometheus/client_golang/prometheus"
	uuid "github.com/satori/go.uuid"
	"github.com/sirupsen/logrus"
	"google.golang.org/grpc"
)

var service = NewService("lile")

type registerImplementation func(s *grpc.Server)

// Service is a grpc compatible server with extra features
type Service struct {
	Name string
	Port string
	// Interceptors
	UnaryInts  []grpc.UnaryServerInterceptor
	StreamInts []grpc.StreamServerInterceptor
	// The RPC server implementation
	GRPCImplementation registerImplementation
}

func defaultOptions(n string) Service {
	return Service{
		Name:               n,
		Port:               ":8000",
		GRPCImplementation: func(s *grpc.Server) {},
	}
}

// NewService creates a lile service with N options
func NewService(name string) Service {
	return defaultOptions(name)
}

func GlobalService() *Service {
	return &service
}

func Name(n string) {
	service.Name = n
}

func Port(n string) {
	service.Port = n
}

// AddUnaryInterceptor adds a unary interceptor to the RPC server
func AddUnaryInterceptor(unint grpc.UnaryServerInterceptor) {
	service.UnaryInts = append(service.UnaryInts, unint)
}

// AddStreamInterceptor adds a stream interceptor to the RPC server
func AddStreamInterceptor(sint grpc.StreamServerInterceptor) {
	service.StreamInts = append(service.StreamInts, sint)
}

func Server(r registerImplementation) {
	service.GRPCImplementation = r
}

func Serve() error {
	lis, err := net.Listen("tcp", service.Port)
	if err != nil {
		return err
	}

	logrus.Infof("Serving gRPC on %s", service.Port)
	return CreateServer().Serve(lis)
}

func CreateServer() *grpc.Server {
	// Default interceptors, [prometheus, opentracing]
	AddUnaryInterceptor(grpc_prometheus.UnaryServerInterceptor)
	AddStreamInterceptor(grpc_prometheus.StreamServerInterceptor)
	AddUnaryInterceptor(otgrpc.OpenTracingServerInterceptor(
		fromenv.Tracer(service.Name)))

	gs := grpc.NewServer(
		grpc.UnaryInterceptor(
			grpc_middleware.ChainUnaryServer(service.UnaryInts...)),
		grpc.StreamInterceptor(
			grpc_middleware.ChainStreamServer(service.StreamInts...)),
	)

	service.GRPCImplementation(gs)

	grpc_prometheus.EnableHandlingTimeHistogram()
	grpc_prometheus.Register(gs)
	http.Handle("/metrics", prometheus.Handler())
	logrus.Infof("Prometheus metrics at :9000/metrics")
	port := "9000"
	if p := os.Getenv("PROMETHEUS_PORT"); p != "" {
		port = p
	}
	go http.ListenAndServe(":"+port, nil)

	return gs
}

// NewTestServer is a helper function to create a gRPC server on a unix socket
// it returns the socket location and a func to call which starts the server
func NewTestServer(s *grpc.Server) (string, func()) {
	// Create a temp random unix socket
	uid, err := uuid.NewV1()
	if err != nil {
		panic(err)
	}

	skt := "/tmp/" + uid.String()

	ln, err := net.Listen("unix", skt)
	if err != nil {
		panic(err)
	}

	return skt, func() {
		s.Serve(ln)
	}
}

// TestConn is a connection that connects to a socket based connection
func TestConn(addr string) *grpc.ClientConn {
	conn, err := grpc.Dial(
		addr,
		grpc.WithDialer(func(addr string, d time.Duration) (net.Conn, error) {
			return net.Dial("unix", addr)
		}),
		grpc.WithInsecure(),
		grpc.WithTimeout(1*time.Second),
		grpc.WithBlock(),
	)

	if err != nil {
		panic(err)
	}

	return conn
}
<commit_msg>Add windows test transport support<commit_after>// Package lile provides helper methods to quickly create RPC based services
// that have metrics, tracing and pub/sub support
package lile

import (
	"net"
	"net/http"
	"os"
	"runtime"
	"time"

	"github.com/grpc-ecosystem/go-grpc-middleware"
	"github.com/grpc-ecosystem/go-grpc-prometheus"
	"github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc"
	"github.com/lileio/lile/fromenv"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/natefinch/npipe"
	uuid "github.com/satori/go.uuid"
	"github.com/sirupsen/logrus"
	"google.golang.org/grpc"
)

var service = NewService("lile")

type registerImplementation func(s *grpc.Server)

// Service is a grpc compatible server with extra features
type Service struct {
	Name string
	Port string
	// Interceptors
	UnaryInts  []grpc.UnaryServerInterceptor
	StreamInts []grpc.StreamServerInterceptor
	// The RPC server implementation
	GRPCImplementation registerImplementation
}

func defaultOptions(n string) Service {
	return Service{
		Name:               n,
		Port:               ":8000",
		GRPCImplementation: func(s *grpc.Server) {},
	}
}

// NewService creates a lile service with N options
func NewService(name string) Service {
	return defaultOptions(name)
}

func GlobalService() *Service {
	return &service
}

func Name(n string) {
	service.Name = n
}

func Port(n string) {
	service.Port = n
}

// AddUnaryInterceptor adds a unary interceptor to the RPC server
func AddUnaryInterceptor(unint grpc.UnaryServerInterceptor) {
	service.UnaryInts = append(service.UnaryInts, unint)
}

// AddStreamInterceptor adds a stream interceptor to the RPC server
func AddStreamInterceptor(sint grpc.StreamServerInterceptor) {
	service.StreamInts = append(service.StreamInts, sint)
}

func Server(r registerImplementation) {
	service.GRPCImplementation = r
}

func Serve() error {
	lis, err := net.Listen("tcp", service.Port)
	if err != nil {
		return err
	}

	logrus.Infof("Serving gRPC on %s", service.Port)
	return CreateServer().Serve(lis)
}

func CreateServer() *grpc.Server {
	// Default interceptors, [prometheus, opentracing]
	AddUnaryInterceptor(grpc_prometheus.UnaryServerInterceptor)
	AddStreamInterceptor(grpc_prometheus.StreamServerInterceptor)
	AddUnaryInterceptor(otgrpc.OpenTracingServerInterceptor(
		fromenv.Tracer(service.Name)))

	gs := grpc.NewServer(
		grpc.UnaryInterceptor(
			grpc_middleware.ChainUnaryServer(service.UnaryInts...)),
		grpc.StreamInterceptor(
			grpc_middleware.ChainStreamServer(service.StreamInts...)),
	)

	service.GRPCImplementation(gs)

	grpc_prometheus.EnableHandlingTimeHistogram()
	grpc_prometheus.Register(gs)
	http.Handle("/metrics", prometheus.Handler())
	logrus.Infof("Prometheus metrics at :9000/metrics")
	port := "9000"
	if p := os.Getenv("PROMETHEUS_PORT"); p != "" {
		port = p
	}
	go http.ListenAndServe(":"+port, nil)

	return gs
}

//   Creates a server listener dependent on the underlying platform. Windows
// hosts will have a Windows Named pipe, anything else gets a UNIX socket
func getTestServerTransportOrPanic()(string, net.Listener) {
	var uniqueAddress string
	var serverListener net.Listener
	var err error

	// Create a temp random unix socket
	uid, err := uuid.NewV1()
	if err != nil {
		panic(err)
	}
	uniqueAddress = uid.String()

	if runtime.GOOS == "windows" {
		uniqueAddress = `\\.\pipe\` + uniqueAddress
		serverListener, err = npipe.Listen(uniqueAddress)

	} else {
		uniqueAddress = "/tmp/" + uniqueAddress
		serverListener, err = net.Listen("unix", uniqueAddress)

	}

	if err != nil {
		panic(err)
	}

	return uniqueAddress, serverListener
}

//   NewTestServer is a helper function to create a gRPC server on a non-network
// socket and it returns the socket location and a func to call which starts
// the server
func NewTestServer(s *grpc.Server) (string, func()) {

	socketAddress, listener := getTestServerTransportOrPanic()

	return socketAddress, func() {
		s.Serve(listener)
	}
}

//   Returns a dialer function for the underlying platform. Returns a Windows
// named pipe if asked for Windows, else a UNIX socket
func getDialerFunctionForPlatform(platform string)(
		func(string)(net.Conn, error)) {

	var dialFunc func(string)(net.Conn, error)

	if platform == "windows" {
		dialFunc = func(addr string)(net.Conn, error) {
			return npipe.Dial(addr)
		}
	} else {
		dialFunc = func(addr string)(net.Conn, error) {
			return net.Dial("unix", addr)
		}
	}

	return dialFunc
}

// TestConn is a connection that connects to a socket based connection
func TestConn(addr string) *grpc.ClientConn {

	dialFunc := getDialerFunctionForPlatform(runtime.GOOS)

	conn, err := grpc.Dial(
		addr,
		grpc.WithDialer(func(addr string, d time.Duration) (net.Conn, error) {
			return dialFunc(addr)
		}),
		grpc.WithInsecure(),
		grpc.WithTimeout(1*time.Second),
		grpc.WithBlock(),
	)

	if err != nil {
		panic(err)
	}

	return conn
}
<|endoftext|>