<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"code.google.com/p/go-uuid/uuid"
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/fsouza/go-dockerclient"
	"github.com/wsxiaoys/terminal/color"
	"io"
	"net/url"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"text/template"
)

var (
	distros = map[string]string{
		"ubuntu_lucid": "ubuntu:10.04",
		"ubuntu:10.04": "ubuntu:10.04",

		"ubuntu_precise": "ubuntu:12.04",
		"ubuntu:12.04":   "ubuntu:12.04",

		"ubuntu_raring": "ubuntu:13.04",
		"ubuntu:13.04":  "ubuntu:13.04",

		"ubuntu_trusty": "ubuntu:14.04",
		"ubuntu:14.04":  "ubuntu:14.04",

		"centos:6.6": "centos:6.6",
	}

	docker_client   *docker.Client
	docker_endpoint string
	red             func(string) string
	green           func(string) string
	light_green     func(string) string
)

const image_tag string = "ruby_build"

func init() {
	u, err := url.Parse(os.Getenv("DOCKER_HOST"))
	if err != nil {
		panic(err)
	}
	u.Scheme = "https"
	docker_endpoint = u.String()

	c, err := docker.NewClient(docker_endpoint)

	tr, err := NewHTTPSClient(os.Getenv("DOCKER_CERT_PATH"))
	if err != nil {
		panic(err)
	}
	c.HTTPClient = tr

	if err != nil {
		panic(err)
	}
	docker_client = c
}

func main() {

	app := cli.NewApp()
	app.Name = "build_ruby"
	app.Usage = "Build ruby debs from source for Ubuntu"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "ruby, r",
			Value: "",
			Usage: "Required. The version to build, eg. 2.1.0 (for recent versions with no patch release) or 2.0.0-p451",
		},
		cli.StringFlag{
			Name:  "distro, d",
			Value: "ubuntu:12.04",
			Usage: "Which distro to use for the build",
		},
		cli.StringFlag{
			Name:  "arch, a",
			Value: "amd64",
			Usage: "Arch to use in package filename, eg: 'none', 'all', 'amd64' etc.",
		},
		cli.StringFlag{
			Name:  "iteration, i",
			Value: "",
			Usage: "eg: 37s~precise",
		},
		cli.IntFlag{
			Name:  "cpus, c",
			Usage: "The number of CPUs to use for the make process, defaults to the number in the local machine. Change if you're running on a remote docker host",
		},
	}
	app.Action = buildRuby
	app.Run(os.Args)
}

func buildRuby(c *cli.Context) {
	if c.String("ruby") == "" {
		color.Fprintln(os.Stderr, "@{r!}You didn't specify a Ruby version to build!")
		cli.ShowAppHelp(c)
		os.Exit(1)
	}

	if distros[c.String("distro")] == "" {
		color.Fprintln(os.Stderr, "@{r!}You specified a distro that I don't know how to build for")
		cli.ShowAppHelp(c)
		os.Exit(1)
	}

	var parallel_make_tasks int
	if c.Int("cpus") != 0 {
		parallel_make_tasks = c.Int("cpus")
	} else {
		parallel_make_tasks = runtime.NumCPU()
	}

	var patch_file_full_paths []string = patchFilePathsFromRubyVersion(c.String("ruby"))

	var dockerfile *bytes.Buffer = dockerFileFromTemplate(distros[c.String("distro")], c.String("ruby"), c.String("arch"), c.String("iteration"), fileBasePaths(patch_file_full_paths), parallel_make_tasks)
	color.Println("@{g!}Using Dockerfile:")
	color.Printf("@{gc}%s\n", dockerfile)

	var build_tarfile *bytes.Buffer = createTarFileFromDockerfile(dockerfile, patch_file_full_paths)

	image_name := fmt.Sprintf("ruby_build_%s_image", uuid.NewRandom())
	opts := docker.BuildImageOptions{
		Name:                image_name,
		NoCache:             true,
		SuppressOutput:      false,
		RmTmpContainer:      true,
		ForceRmTmpContainer: true,
		InputStream:         build_tarfile,
		OutputStream:        os.Stdout,
	}
	if err := docker_client.BuildImage(opts); err != nil {
		panic(err)
	}
	color.Printf("@{g!}Created image with name %s\n", image_name)

	image, err := docker_client.InspectImage(image_name)
	if err != nil {
		panic(err)
	}

	/*
		Create a container with the created image id

		This seems like a hack. We need a "container" to enable us to copy the ruby
		package out, but I can't see how to do this without needed to run a command
		or just use specify an image ID directly, hence the noop 'date' command.

	*/

	color.Printf("@{g!}Creating container with from image id %s\n", image.ID)
	config := docker.Config{AttachStdout: false, AttachStdin: false, Image: image.ID, Cmd: []string{"date"}}
	container_name := fmt.Sprintf("ruby_build_%s_container", uuid.NewRandom())
	create_container_opts := docker.CreateContainerOptions{Name: container_name, Config: &config}
	container, err := docker_client.CreateContainer(create_container_opts)
	if err != nil {
		panic(err)
	}

	// See https://github.com/wjessop/build_ruby/issues/2
	if err := docker_client.StopContainer(container.ID, 1); err != nil {
		// panic(err)
		color.Printf("@{f}Failed to stop container %d, error was: %s\n", container.ID, err.Error())
	}

	copyPackageFromContainerToLocalFs(container, rubyPackageFileName(c.String("ruby"), c.String("iteration"), c.String("arch"), c.String("distro")))

	color.Println("@{g!}Removing container:", container.ID)
	if err := docker_client.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID, RemoveVolumes: true, Force: false}); err != nil {
		panic(err)
	}
}

func patchFilePathsFromRubyVersion(version string) []string {
	var patch_files []string
	for _, name := range AssetNames() {
		if strings.Contains(name, fmt.Sprintf("/%s/", version)) {
			patch_files = append(patch_files, name)
		}
	}

	color.Printf("@{g}Found patch files for current Ruby version: %v\n", patch_files)

	return patch_files
}

func createTarFileFromDockerfile(dockerfile *bytes.Buffer, patch_file_paths []string) *bytes.Buffer {
	// Create a buffer to write our archive to.
	buf := new(bytes.Buffer)

	// Create a new tar archive.
	tw := tar.NewWriter(buf)

	// Add the Dockerfile
	hdr := &tar.Header{
		Name: "Dockerfile",
		Size: int64(dockerfile.Len()),
	}

	if err := tw.WriteHeader(hdr); err != nil {
		panic(err)
	}

	if _, err := tw.Write(dockerfile.Bytes()); err != nil {
		panic(err)
	}

	for _, path := range patch_file_paths {
		color.Printf("@{g}Adding patch file to the tar: %s (at path %s)\n", patch_file_paths, filepath.Base(path))

		asset_bytes, err := Asset(path)
		if err != nil {
			panic(err)
		}

		// We store the patch files flat in the root dir, hence the Base call
		hdr := &tar.Header{
			Name: filepath.Base(path),
			Size: int64(len(asset_bytes)),
		}
		if err := tw.WriteHeader(hdr); err != nil {
			panic(err)
		}

		if _, err := tw.Write(asset_bytes); err != nil {
			panic(err)
		}
	}

	// Make sure to check the error on Close.
	if err := tw.Close(); err != nil {
		panic(err)
	}

	return buf
}

func copyPackageFromContainerToLocalFs(container *docker.Container, filename string) {
	color.Println("@{g!}Copying package out of the container")

	var buf bytes.Buffer
	if err := docker_client.CopyFromContainer(docker.CopyFromContainerOptions{
		Container:    container.ID,
		Resource:     filename,
		OutputStream: &buf,
	}); err != nil {
		panic(err)
	}

	buffer := bytes.NewReader(buf.Bytes())

	var tar_out *tar.Reader = tar.NewReader(buffer)
	tar_header, err := tar_out.Next()
	if err != nil {
		panic(err)
	}

	color.Printf("@{g!}Extracting pckage file %s (%d bytes)\n", tar_header.Name, tar_header.Size)

	out, err := os.Create(filename)
	if err != nil {
		panic(err)
	}
	defer out.Close()

	io.Copy(out, tar_out)
}

func rubyPackageFileName(version, iteration, arch string, distro string) string {
	var formatted_iteration = ""
	if iteration != "" {
		formatted_iteration = "_" + iteration
	}

	var formatted_arch = ""
	if arch != "none" {
		formatted_arch = "_" + arch
	}
	return "ruby-" + version + formatted_iteration + formatted_arch + packageFormat(distro)
}

func packageFormat(distro string) string {
	if strings.Contains(distro, "centos") || strings.Contains(distro, "rhel") {
		return ".rpm"
	} else {
		return ".deb"
	}
}

func packageFormat(distro string) string {
	if strings.Contains(distro, "centos") || strings.Contains(distro, "rhel") {
		return ".rpm"
	} else {
		return ".deb"
	}
}

func dockerFileFromTemplate(distro, ruby_version, arch, iteration string, patches []string, parallel_make_jobs int) *bytes.Buffer {
	type buildVars struct {
		Distro      string
		RubyVersion string
		Arch        string
		Iteration   string
		DownloadUrl string
		FileName    string
		NumCPU      int
		Patches     []string
	}

	var formatted_iteration = ""
	if iteration != "" {
		formatted_iteration = fmt.Sprintf("--iteration %s \\", iteration)
	}

	download_url := rubyDownloadUrl(ruby_version)
	dockerfile_vars := buildVars{distro, ruby_version, arch, formatted_iteration, download_url, rubyPackageFileName(ruby_version, iteration, arch, distro), patches, parallel_make_jobs}

	// This would be way better as a look up table, or with a more formal lookup process
	var template_location string
	switch distro {
	case "ubuntu:10.04":
		template_location = "data/Dockerfile-lucid.template"
	case "centos:6.6":
		template_location = "data/Dockerfile-centos.template"
	default:
		template_location = "data/Dockerfile.template"
	}

	dockerfile_template, err := Asset(template_location)
	if err != nil {
		panic(err)
	}
	if len(dockerfile_template) == 0 {
		panic("Couldn't find Dockerfile template in bindata")
	}

	tmpl, err := template.New("dockerfile_template").Parse(string(dockerfile_template))
	if err != nil {
		panic(err)
	}

	buf := new(bytes.Buffer)

	err = tmpl.Execute(buf, dockerfile_vars)
	if err != nil {
		panic(err)
	}

	return buf
}

func rubyDownloadUrl(version string) string {
	// eg:
	// http://cache.ruby-lang.org/pub/ruby/2.1/ruby-2.1.1.tar.gz
	// http://cache.ruby-lang.org/pub/ruby/2.0/ruby-2.0.0-p451.tar.gz

	v := majorMinorVersionOnly(version)
	return "http://cache.ruby-lang.org/pub/ruby/" + v + "/ruby-" + version + ".tar.gz"
}

func majorMinorVersionOnly(full_version string) string {
	return strings.Join(strings.SplitN(full_version, ".", 3)[0:2], ".")
}

func fileBasePaths(full_paths []string) (base_paths []string) {
	for _, p := range full_paths {
		base_paths = append(base_paths, filepath.Base(p))
	}

	return
}
<commit_msg>Sort patch files so they get applied in order<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"code.google.com/p/go-uuid/uuid"
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/fsouza/go-dockerclient"
	"github.com/wsxiaoys/terminal/color"
	"io"
	"net/url"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"text/template"
)

var (
	distros = map[string]string{
		"ubuntu_lucid": "ubuntu:10.04",
		"ubuntu:10.04": "ubuntu:10.04",

		"ubuntu_precise": "ubuntu:12.04",
		"ubuntu:12.04":   "ubuntu:12.04",

		"ubuntu_raring": "ubuntu:13.04",
		"ubuntu:13.04":  "ubuntu:13.04",

		"ubuntu_trusty": "ubuntu:14.04",
		"ubuntu:14.04":  "ubuntu:14.04",

		"centos:6.6": "centos:6.6",
	}

	docker_client   *docker.Client
	docker_endpoint string
	red             func(string) string
	green           func(string) string
	light_green     func(string) string
)

const image_tag string = "ruby_build"

func init() {
	u, err := url.Parse(os.Getenv("DOCKER_HOST"))
	if err != nil {
		panic(err)
	}
	u.Scheme = "https"
	docker_endpoint = u.String()

	c, err := docker.NewClient(docker_endpoint)

	tr, err := NewHTTPSClient(os.Getenv("DOCKER_CERT_PATH"))
	if err != nil {
		panic(err)
	}
	c.HTTPClient = tr

	if err != nil {
		panic(err)
	}
	docker_client = c
}

func main() {

	app := cli.NewApp()
	app.Name = "build_ruby"
	app.Usage = "Build ruby debs from source for Ubuntu"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "ruby, r",
			Value: "",
			Usage: "Required. The version to build, eg. 2.1.0 (for recent versions with no patch release) or 2.0.0-p451",
		},
		cli.StringFlag{
			Name:  "distro, d",
			Value: "ubuntu:12.04",
			Usage: "Which distro to use for the build",
		},
		cli.StringFlag{
			Name:  "arch, a",
			Value: "amd64",
			Usage: "Arch to use in package filename, eg: 'none', 'all', 'amd64' etc.",
		},
		cli.StringFlag{
			Name:  "iteration, i",
			Value: "",
			Usage: "eg: 37s~precise",
		},
		cli.IntFlag{
			Name:  "cpus, c",
			Usage: "The number of CPUs to use for the make process, defaults to the number in the local machine. Change if you're running on a remote docker host",
		},
	}
	app.Action = buildRuby
	app.Run(os.Args)
}

func buildRuby(c *cli.Context) {
	if c.String("ruby") == "" {
		color.Fprintln(os.Stderr, "@{r!}You didn't specify a Ruby version to build!")
		cli.ShowAppHelp(c)
		os.Exit(1)
	}

	if distros[c.String("distro")] == "" {
		color.Fprintln(os.Stderr, "@{r!}You specified a distro that I don't know how to build for")
		cli.ShowAppHelp(c)
		os.Exit(1)
	}

	var parallel_make_tasks int
	if c.Int("cpus") != 0 {
		parallel_make_tasks = c.Int("cpus")
	} else {
		parallel_make_tasks = runtime.NumCPU()
	}

	var patch_file_full_paths []string = patchFilePathsFromRubyVersion(c.String("ruby"))

	var dockerfile *bytes.Buffer = dockerFileFromTemplate(distros[c.String("distro")], c.String("ruby"), c.String("arch"), c.String("iteration"), fileBasePaths(patch_file_full_paths), parallel_make_tasks)
	color.Println("@{g!}Using Dockerfile:")
	color.Printf("@{gc}%s\n", dockerfile)

	var build_tarfile *bytes.Buffer = createTarFileFromDockerfile(dockerfile, patch_file_full_paths)

	image_name := fmt.Sprintf("ruby_build_%s_image", uuid.NewRandom())
	opts := docker.BuildImageOptions{
		Name:                image_name,
		NoCache:             true,
		SuppressOutput:      false,
		RmTmpContainer:      true,
		ForceRmTmpContainer: true,
		InputStream:         build_tarfile,
		OutputStream:        os.Stdout,
	}
	if err := docker_client.BuildImage(opts); err != nil {
		panic(err)
	}
	color.Printf("@{g!}Created image with name %s\n", image_name)

	image, err := docker_client.InspectImage(image_name)
	if err != nil {
		panic(err)
	}

	/*
		Create a container with the created image id

		This seems like a hack. We need a "container" to enable us to copy the ruby
		package out, but I can't see how to do this without needed to run a command
		or just use specify an image ID directly, hence the noop 'date' command.

	*/

	color.Printf("@{g!}Creating container with from image id %s\n", image.ID)
	config := docker.Config{AttachStdout: false, AttachStdin: false, Image: image.ID, Cmd: []string{"date"}}
	container_name := fmt.Sprintf("ruby_build_%s_container", uuid.NewRandom())
	create_container_opts := docker.CreateContainerOptions{Name: container_name, Config: &config}
	container, err := docker_client.CreateContainer(create_container_opts)
	if err != nil {
		panic(err)
	}

	// See https://github.com/wjessop/build_ruby/issues/2
	if err := docker_client.StopContainer(container.ID, 1); err != nil {
		// panic(err)
		color.Printf("@{f}Failed to stop container %d, error was: %s\n", container.ID, err.Error())
	}

	copyPackageFromContainerToLocalFs(container, rubyPackageFileName(c.String("ruby"), c.String("iteration"), c.String("arch"), c.String("distro")))

	color.Println("@{g!}Removing container:", container.ID)
	if err := docker_client.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID, RemoveVolumes: true, Force: false}); err != nil {
		panic(err)
	}
}

func patchFilePathsFromRubyVersion(version string) []string {
	var patch_files []string
	for _, name := range AssetNames() {
		if strings.Contains(name, fmt.Sprintf("/%s/", version)) {
			patch_files = append(patch_files, name)
		}
	}

	sort.Strings(patch_files)
	color.Printf("@{g}Found patch files for current Ruby version: %v\n", patch_files)
	return patch_files
}

func createTarFileFromDockerfile(dockerfile *bytes.Buffer, patch_file_paths []string) *bytes.Buffer {
	// Create a buffer to write our archive to.
	buf := new(bytes.Buffer)

	// Create a new tar archive.
	tw := tar.NewWriter(buf)

	// Add the Dockerfile
	hdr := &tar.Header{
		Name: "Dockerfile",
		Size: int64(dockerfile.Len()),
	}

	if err := tw.WriteHeader(hdr); err != nil {
		panic(err)
	}

	if _, err := tw.Write(dockerfile.Bytes()); err != nil {
		panic(err)
	}

	for _, path := range patch_file_paths {
		color.Printf("@{g}Adding patch file to the tar: %s (at path %s)\n", patch_file_paths, filepath.Base(path))

		asset_bytes, err := Asset(path)
		if err != nil {
			panic(err)
		}

		// We store the patch files flat in the root dir, hence the Base call
		hdr := &tar.Header{
			Name: filepath.Base(path),
			Size: int64(len(asset_bytes)),
		}
		if err := tw.WriteHeader(hdr); err != nil {
			panic(err)
		}

		if _, err := tw.Write(asset_bytes); err != nil {
			panic(err)
		}
	}

	// Make sure to check the error on Close.
	if err := tw.Close(); err != nil {
		panic(err)
	}

	return buf
}

func copyPackageFromContainerToLocalFs(container *docker.Container, filename string) {
	color.Println("@{g!}Copying package out of the container")

	var buf bytes.Buffer
	if err := docker_client.CopyFromContainer(docker.CopyFromContainerOptions{
		Container:    container.ID,
		Resource:     filename,
		OutputStream: &buf,
	}); err != nil {
		panic(err)
	}

	buffer := bytes.NewReader(buf.Bytes())

	var tar_out *tar.Reader = tar.NewReader(buffer)
	tar_header, err := tar_out.Next()
	if err != nil {
		panic(err)
	}

	color.Printf("@{g!}Extracting pckage file %s (%d bytes)\n", tar_header.Name, tar_header.Size)

	out, err := os.Create(filename)
	if err != nil {
		panic(err)
	}
	defer out.Close()

	io.Copy(out, tar_out)
}

func rubyPackageFileName(version, iteration, arch string, distro string) string {
	var formatted_iteration = ""
	if iteration != "" {
		formatted_iteration = "_" + iteration
	}

	var formatted_arch = ""
	if arch != "none" {
		formatted_arch = "_" + arch
	}
	return "ruby-" + version + formatted_iteration + formatted_arch + packageFormat(distro)
}

func packageFormat(distro string) string {
	if strings.Contains(distro, "centos") || strings.Contains(distro, "rhel") {
		return ".rpm"
	} else {
		return ".deb"
	}
}

func dockerFileFromTemplate(distro, ruby_version, arch, iteration string, patches []string, parallel_make_jobs int) *bytes.Buffer {
	type buildVars struct {
		Distro      string
		RubyVersion string
		Arch        string
		Iteration   string
		DownloadUrl string
		FileName    string
		NumCPU      int
		Patches     []string
	}

	var formatted_iteration = ""
	if iteration != "" {
		formatted_iteration = fmt.Sprintf("--iteration %s \\", iteration)
	}

	download_url := rubyDownloadUrl(ruby_version)
	dockerfile_vars := buildVars{distro, ruby_version, arch, formatted_iteration, download_url, rubyPackageFileName(ruby_version, iteration, arch, distro), patches, parallel_make_jobs}

	// This would be way better as a look up table, or with a more formal lookup process
	var template_location string
	switch distro {
	case "ubuntu:10.04":
		template_location = "data/Dockerfile-lucid.template"
	case "centos:6.6":
		template_location = "data/Dockerfile-centos.template"
	default:
		template_location = "data/Dockerfile.template"
	}

	dockerfile_template, err := Asset(template_location)
	if err != nil {
		panic(err)
	}
	if len(dockerfile_template) == 0 {
		panic("Couldn't find Dockerfile template in bindata")
	}

	tmpl, err := template.New("dockerfile_template").Parse(string(dockerfile_template))
	if err != nil {
		panic(err)
	}

	buf := new(bytes.Buffer)

	err = tmpl.Execute(buf, dockerfile_vars)
	if err != nil {
		panic(err)
	}

	return buf
}

func rubyDownloadUrl(version string) string {
	// eg:
	// http://cache.ruby-lang.org/pub/ruby/2.1/ruby-2.1.1.tar.gz
	// http://cache.ruby-lang.org/pub/ruby/2.0/ruby-2.0.0-p451.tar.gz

	v := majorMinorVersionOnly(version)
	return "http://cache.ruby-lang.org/pub/ruby/" + v + "/ruby-" + version + ".tar.gz"
}

func majorMinorVersionOnly(full_version string) string {
	return strings.Join(strings.SplitN(full_version, ".", 3)[0:2], ".")
}

func fileBasePaths(full_paths []string) (base_paths []string) {
	for _, p := range full_paths {
		base_paths = append(base_paths, filepath.Base(p))
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"os"
	"testing"

	assert "github.com/stretchr/testify/require"
)

func TestResizeImage(t *testing.T) {
	tmpfile, err := ioutil.TempFile("", "resized_image")
	assert.NoError(t, err)
	defer os.Remove(tmpfile.Name())

	err = resizeImage(nil, "./content/images/about/avatar.jpg", tmpfile.Name(), 100)
	assert.NoError(t, err)
}
<commit_msg>Add test for without MozJPEG<commit_after>package main

import (
	"io/ioutil"
	"os"
	"testing"

	assert "github.com/stretchr/testify/require"
)

func TestResizeImage(t *testing.T) {
	tmpfile, err := ioutil.TempFile("", "resized_image")
	assert.NoError(t, err)
	defer os.Remove(tmpfile.Name())

	err = resizeImage(nil, "./content/images/about/avatar.jpg", tmpfile.Name(), 100)
	assert.NoError(t, err)
}

func TestResizeImage_NoMozJPEG(t *testing.T) {
	if conf.MozJPEGBin == "" {
		return
	}

	oldMozJPEGBin := conf.MozJPEGBin
	defer func() {
		conf.MozJPEGBin = oldMozJPEGBin
	}()
	conf.MozJPEGBin = ""

	tmpfile, err := ioutil.TempFile("", "resized_image")
	assert.NoError(t, err)
	defer os.Remove(tmpfile.Name())

	err = resizeImage(nil, "./content/images/about/avatar.jpg", tmpfile.Name(), 100)
	assert.NoError(t, err)
}
<|endoftext|>


<commit_before>package customer

import (
	"strconv"

	"rental"
)

// Customer store customer information
type Customer struct {
	name    string
	rentals []*rental.Rental
}

// NewCustomer create new customer with name
func NewCustomer(name string) *Customer {
	return &Customer{
		name:    name,
		rentals: make([]*rental.Rental, 0, 3),
	}
}

// AddRental adds rental r to customer rentals
func (c *Customer) AddRental(r *rental.Rental) {
	c.rentals = append(c.rentals, r)
}

// Name return customer's name
func (c *Customer) Name() string {
	return c.name
}

// Statement return string statement of customer order
func (c *Customer) Statement() string {
	var (
		totalAmount          float64
		frequentRenterPoints int
	)
	result := "Rental Record for " + c.Name() + "\n"
	for _, each := range c.rentals {
		// show figures for this rental
		frequentRenterPoints += each.GetFrequentRenterPoints()
		result += "\t" + each.Movie().Title() + "\t" +
			strconv.FormatFloat(each.GetCharge(), 'f', -1, 64) + "\n"
		totalAmount += each.GetCharge()
	}
	// add footer lines
	result += "Amount owed is " +
		strconv.FormatFloat(totalAmount, 'f', -1, 64) + "\n"
	result += "You earned " + strconv.Itoa(frequentRenterPoints) +
		" frequentrenterpoints renter points"
	return result
}
<commit_msg>refactor(VideoStore) Replace temp var in Statement<commit_after>package customer

import (
	"strconv"

	"rental"
)

// Customer store customer information
type Customer struct {
	name    string
	rentals []*rental.Rental
}

// NewCustomer create new customer with name
func NewCustomer(name string) *Customer {
	return &Customer{
		name:    name,
		rentals: make([]*rental.Rental, 0, 3),
	}
}

// AddRental adds rental r to customer rentals
func (c *Customer) AddRental(r *rental.Rental) {
	c.rentals = append(c.rentals, r)
}

// Name return customer's name
func (c *Customer) Name() string {
	return c.name
}

// Statement return string statement of customer order
func (c *Customer) Statement() string {
	var frequentRenterPoints int
	result := "Rental Record for " + c.Name() + "\n"
	for _, each := range c.rentals {
		// show figures for this rental
		frequentRenterPoints += each.GetFrequentRenterPoints()
		result += "\t" + each.Movie().Title() + "\t" +
			strconv.FormatFloat(each.GetCharge(), 'f', -1, 64) + "\n"
	}
	// add footer lines
	result += "Amount owed is " +
		strconv.FormatFloat(c.getTotalCharge(), 'f', -1, 64) + "\n"
	result += "You earned " + strconv.Itoa(frequentRenterPoints) +
		" frequentrenterpoints renter points"
	return result
}

func (c *Customer) getTotalCharge() float64 {
	var result float64
	for _, each := range c.rentals {
		result += each.GetCharge()
	}
	return result
}
<|endoftext|>


<commit_before>package data_types

import (
  "encoding/json"
  "fmt"
  "net"
)

type PortNumber uint16

type UInt8List []uint8

func (e UInt8List) MarshalJSON() ([]byte, error) {
  if e == nil {
    return json.Marshal(nil)
  } else {
    a := make([]int, len(e))
    for i, v := range e {
      a[i] = int(v)
    }
    return json.Marshal(a)
  }
}

type PortRange struct {
  LowerPort *PortNumber `json:"lower-port"`
  UpperPort *PortNumber `json:"upper-port"`
}

type IPPrefix struct {
  IP     net.IP
  Length int
}

type IPv4Prefix IPPrefix
type IPv6Prefix IPPrefix

type PortRangeOrOperator struct {
  LowerPort  *PortNumber `json:"lower-port"`
  UpperPort  *PortNumber `json:"upper-port"`
  Operator   *Operator   `json:"operator"`
  Port       *PortNumber `json:"port"`
}

type Fragment struct {
  Operator *OperatorBit `json:"operator"`
  Type *FragmentType `json:"type"`
}

type FlagsBitmask struct {
  Operator *OperatorBit `json:"operator"`
  Bitmask uint16 `json:"bitmask"`
}

type Empty int

const (
  Empty_Value Empty = 0
)

func (e Empty) String() string {
  return "empty"
}

func (e Empty) MarshalJSON() ([]byte, error) {
  return json.Marshal([]interface{} { nil })
}

func (p *Empty) UnmarshalJSON(data []byte) error {
  var a []interface{}
  err := json.Unmarshal(data, &a)
  if err != nil {
    return fmt.Errorf("Could not unmarshal as empty: %v", data)
  }

  if len(a) == 1 && a[0] == nil {
    *p = Empty_Value
    return nil
  } else {
    return fmt.Errorf("Unexpected empty: %v", a)
  }
}

func (e IPPrefix) String() string {
  return fmt.Sprintf("%s/%d", e.IP.String(), e.Length)
}

func (e IPPrefix) MarshalJSON() ([]byte, error) {
  return json.Marshal(e.String())
}

func (p *IPPrefix) UnmarshalJSON(data []byte) error {
  var s string
  err := json.Unmarshal(data, &s)
  if err != nil {
    return fmt.Errorf("Could not unmarshal as string: %v", data)
  }

  ip, net, err := net.ParseCIDR(s)
  if err != nil {
    return fmt.Errorf("Bad ip-prefix: %v", s)
  }
  ones, _ := net.Mask.Size()
  *p = IPPrefix{ ip, ones }
  return nil
}

func (e IPv4Prefix) String() string {
  return IPPrefix(e).String()
}

func (e IPv4Prefix) MarshalJSON() ([]byte, error) {
  return json.Marshal(IPPrefix(e))
}

func (p *IPv4Prefix) UnmarshalJSON(data []byte) error {
  var x IPPrefix
  err := json.Unmarshal(data, &x)
  if err != nil {
    return fmt.Errorf("Could not unmarshal as ip-prefix: %v", data)
  }

  if len(x.IP) != net.IPv4len {
    return fmt.Errorf("Bad ipv4-prefix: %v", x)
  }
  *p = IPv4Prefix(x)
  return nil
}

func (e IPv6Prefix) String() string {
  return IPPrefix(e).String()
}

func (e IPv6Prefix) MarshalJSON() ([]byte, error) {
  return json.Marshal(IPPrefix(e))
}

func (p *IPv6Prefix) UnmarshalJSON(data []byte) error {
  var x IPPrefix
  err := json.Unmarshal(data, &x)
  if err != nil {
    return fmt.Errorf("Could not unmarshal as ip-prefix: %v", data)
  }

  if len(x.IP) != net.IPv6len {
    return fmt.Errorf("Bad ipv6-prefix: %v", x)
  }
  *p = IPv6Prefix(x)
  return nil
}
<commit_msg>Fix bad request when PUT ACL with ipv4 or ipv6 destination-network #165<commit_after>package data_types

import (
  "encoding/json"
  "fmt"
  "net"
)

type PortNumber uint16

type UInt8List []uint8

func (e UInt8List) MarshalJSON() ([]byte, error) {
  if e == nil {
    return json.Marshal(nil)
  } else {
    a := make([]int, len(e))
    for i, v := range e {
      a[i] = int(v)
    }
    return json.Marshal(a)
  }
}

type PortRange struct {
  LowerPort *PortNumber `json:"lower-port"`
  UpperPort *PortNumber `json:"upper-port"`
}

type IPPrefix struct {
  IP     net.IP
  Length int
}

type IPv4Prefix IPPrefix
type IPv6Prefix IPPrefix

type PortRangeOrOperator struct {
  LowerPort  *PortNumber `json:"lower-port"`
  UpperPort  *PortNumber `json:"upper-port"`
  Operator   *Operator   `json:"operator"`
  Port       *PortNumber `json:"port"`
}

type Fragment struct {
  Operator *OperatorBit `json:"operator"`
  Type *FragmentType `json:"type"`
}

type FlagsBitmask struct {
  Operator *OperatorBit `json:"operator"`
  Bitmask uint16 `json:"bitmask"`
}

type Empty int

const (
  Empty_Value Empty = 0
)

func (e Empty) String() string {
  return "empty"
}

func (e Empty) MarshalJSON() ([]byte, error) {
  return json.Marshal([]interface{} { nil })
}

func (p *Empty) UnmarshalJSON(data []byte) error {
  var a []interface{}
  err := json.Unmarshal(data, &a)
  if err != nil {
    return fmt.Errorf("Could not unmarshal as empty: %v", data)
  }

  if len(a) == 1 && a[0] == nil {
    *p = Empty_Value
    return nil
  } else {
    return fmt.Errorf("Unexpected empty: %v", a)
  }
}

func (e IPPrefix) String() string {
  return fmt.Sprintf("%s/%d", e.IP.String(), e.Length)
}

func (e IPPrefix) MarshalJSON() ([]byte, error) {
  return json.Marshal(e.String())
}

func (p *IPPrefix) UnmarshalJSON(data []byte) error {
  var s string
  err := json.Unmarshal(data, &s)
  if err != nil {
    return fmt.Errorf("Could not unmarshal as string: %v", data)
  }

  ip, net, err := net.ParseCIDR(s)
  if err != nil {
    return fmt.Errorf("Bad ip-prefix: %v", s)
  }
  ones, _ := net.Mask.Size()
  *p = IPPrefix{ ip, ones }
  return nil
}

func (e IPv4Prefix) String() string {
  return IPPrefix(e).String()
}

func (e IPv4Prefix) MarshalJSON() ([]byte, error) {
  return json.Marshal(IPPrefix(e))
}

func (p *IPv4Prefix) UnmarshalJSON(data []byte) error {
  var x IPPrefix
  err := json.Unmarshal(data, &x)
  if err != nil {
    return fmt.Errorf("Could not unmarshal as ip-prefix: %v", data)
  }

  if x.IP.To4() == nil {
    return fmt.Errorf("Bad ipv4-prefix: %v", x)
  }
  *p = IPv4Prefix(x)
  return nil
}

func (e IPv6Prefix) String() string {
  return IPPrefix(e).String()
}

func (e IPv6Prefix) MarshalJSON() ([]byte, error) {
  return json.Marshal(IPPrefix(e))
}

func (p *IPv6Prefix) UnmarshalJSON(data []byte) error {
  var x IPPrefix
  err := json.Unmarshal(data, &x)
  if err != nil {
    return fmt.Errorf("Could not unmarshal as ip-prefix: %v", data)
  }

  if x.IP.To4() != nil {
    return fmt.Errorf("Bad ipv6-prefix: %v", x)
  }
  *p = IPv6Prefix(x)
  return nil
}
<|endoftext|>


<commit_before>package etcd

import (
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"path"
	"strings"
	"sync"
	"time"
)

// Errors introduced by handling requests
var (
	ErrRequestCancelled = errors.New("sending request is cancelled")
)

type RawRequest struct {
	Method       string
	RelativePath string
	Values       url.Values
	Cancel       <-chan bool
}

// NewRawRequest returns a new RawRequest
func NewRawRequest(method, relativePath string, values url.Values, cancel <-chan bool) *RawRequest {
	return &RawRequest{
		Method:       method,
		RelativePath: relativePath,
		Values:       values,
		Cancel:       cancel,
	}
}

// getCancelable issues a cancelable GET request
func (c *Client) getCancelable(key string, options Options,
	cancel <-chan bool) (*RawResponse, error) {
	logger.Debugf("get %s [%s]", key, c.cluster.Leader)
	p := keyToPath(key)

	// If consistency level is set to STRONG, append
	// the `consistent` query string.
	if c.config.Consistency == STRONG_CONSISTENCY {
		options["consistent"] = true
	}

	str, err := options.toParameters(VALID_GET_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	req := NewRawRequest("GET", p, nil, cancel)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// get issues a GET request
func (c *Client) get(key string, options Options) (*RawResponse, error) {
	return c.getCancelable(key, options, nil)
}

// put issues a PUT request
func (c *Client) put(key string, value string, ttl uint64,
	options Options) (*RawResponse, error) {

	logger.Debugf("put %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader)
	p := keyToPath(key)

	str, err := options.toParameters(VALID_PUT_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	req := NewRawRequest("PUT", p, buildValues(value, ttl), nil)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// post issues a POST request
func (c *Client) post(key string, value string, ttl uint64) (*RawResponse, error) {
	logger.Debugf("post %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader)
	p := keyToPath(key)

	req := NewRawRequest("POST", p, buildValues(value, ttl), nil)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// delete issues a DELETE request
func (c *Client) delete(key string, options Options) (*RawResponse, error) {
	logger.Debugf("delete %s [%s]", key, c.cluster.Leader)
	p := keyToPath(key)

	str, err := options.toParameters(VALID_DELETE_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	req := NewRawRequest("DELETE", p, nil, nil)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// SendRequest sends a HTTP request and returns a Response as defined by etcd
func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) {

	var req *http.Request
	var resp *http.Response
	var httpPath string
	var err error
	var respBody []byte

	reqs := make([]http.Request, 0)
	resps := make([]http.Response, 0)

	checkRetry := c.CheckRetry
	if checkRetry == nil {
		checkRetry = DefaultCheckRetry
	}

	cancelled := make(chan bool, 1)
	reqLock := new(sync.Mutex)

	if rr.Cancel != nil {
		cancelRoutine := make(chan bool)
		defer close(cancelRoutine)

		go func() {
			select {
			case <-rr.Cancel:
				cancelled <-true
				logger.Debug("send.request is cancelled")
			case <-cancelRoutine:
				return
			}

			// Repeat canceling request until this thread is stopped
			// because we have no idea about whether it succeeds.
			for {
				reqLock.Lock()
				c.httpClient.Transport.(*http.Transport).CancelRequest(req)
				reqLock.Unlock()

				select {
				case <-time.After(100 * time.Millisecond):
				case <-cancelRoutine:
					return
				}
			}
		}()
	}

	// if we connect to a follower, we will retry until we find a leader
	for attempt := 0; ; attempt++ {
		select {
		case <-cancelled:
			return nil, ErrRequestCancelled
		default:
		}

		logger.Debug("begin attempt", attempt, "for", rr.RelativePath)

		if rr.Method == "GET" && c.config.Consistency == WEAK_CONSISTENCY {
			// If it's a GET and consistency level is set to WEAK,
			// then use a random machine.
			httpPath = c.getHttpPath(true, rr.RelativePath)
		} else {
			// Else use the leader.
			httpPath = c.getHttpPath(false, rr.RelativePath)
		}

		// Return a cURL command if curlChan is set
		if c.cURLch != nil {
			command := fmt.Sprintf("curl -X %s %s", rr.Method, httpPath)
			for key, value := range rr.Values {
				command += fmt.Sprintf(" -d %s=%s", key, value[0])
			}
			c.sendCURL(command)
		}

		logger.Debug("send.request.to ", httpPath, " | method ", rr.Method)

		reqLock.Lock()
		if rr.Values == nil {
			req, _ = http.NewRequest(rr.Method, httpPath, nil)
		} else {
			req, _ = http.NewRequest(rr.Method, httpPath,
				strings.NewReader(rr.Values.Encode()))

			req.Header.Set("Content-Type",
				"application/x-www-form-urlencoded; param=value")
		}
		reqLock.Unlock()

		resp, err = c.httpClient.Do(req)
		// If the request was cancelled, return ErrRequestCancelled directly
		select {
		case <-cancelled:
			return nil, ErrRequestCancelled
		default:
		}

		reqs = append(reqs, *req)

		// network error, change a machine!
		if err != nil {
			logger.Debug("network error:", err.Error())
			resps = append(resps, http.Response{})
			if checkErr := checkRetry(c.cluster, reqs, resps, err); checkErr != nil {
				return nil, checkErr
			}

			c.cluster.switchLeader(attempt % len(c.cluster.Machines))
			continue
		}

		// if there is no error, it should receive response
		resps = append(resps, *resp)
		defer resp.Body.Close()
		logger.Debug("recv.response.from", httpPath)

		if validHttpStatusCode[resp.StatusCode] {
			// try to read byte code and break the loop
			respBody, err = ioutil.ReadAll(resp.Body)
			if err == nil {
				logger.Debug("recv.success.", httpPath)
				break
			}
		}

		// if resp is TemporaryRedirect, set the new leader and retry
		if resp.StatusCode == http.StatusTemporaryRedirect {
			u, err := resp.Location()

			if err != nil {
				logger.Warning(err)
			} else {
				// Update cluster leader based on redirect location
				// because it should point to the leader address
				c.cluster.updateLeaderFromURL(u)
				logger.Debug("recv.response.relocate", u.String())
			}
			continue
		}

		if checkErr := checkRetry(c.cluster, reqs, resps,
			errors.New("Unexpected HTTP status code")); checkErr != nil {
			return nil, checkErr
		}
	}

	r := &RawResponse{
		StatusCode: resp.StatusCode,
		Body:       respBody,
		Header:     resp.Header,
	}

	return r, nil
}

// DefaultCheckRetry checks retry cases
// If it has retried 2 * machine number, stop to retry it anymore
// If resp is nil, sleep for 200ms
// If status code is InternalServerError, sleep for 200ms.
func DefaultCheckRetry(cluster *Cluster, reqs []http.Request,
	resps []http.Response, err error) error {

	if len(reqs) >= 2*len(cluster.Machines) {
		return newError(ErrCodeEtcdNotReachable,
			"Tried to connect to each peer twice and failed", 0)
	}

	resp := &resps[len(resps)-1]

	if resp == nil {
		time.Sleep(time.Millisecond * 200)
		return nil
	}

	code := resp.StatusCode
	if code == http.StatusInternalServerError {
		time.Sleep(time.Millisecond * 200)

	}

	logger.Warning("bad response status code", code)
	return nil
}

func (c *Client) getHttpPath(random bool, s ...string) string {
	var machine string
	if random {
		machine = c.cluster.Machines[rand.Intn(len(c.cluster.Machines))]
	} else {
		machine = c.cluster.Leader
	}

	fullPath := machine + "/" + version
	for _, seg := range s {
		fullPath = fullPath + "/" + seg
	}

	return fullPath
}

// buildValues builds a url.Values map according to the given value and ttl
func buildValues(value string, ttl uint64) url.Values {
	v := url.Values{}

	if value != "" {
		v.Set("value", value)
	}

	if ttl > 0 {
		v.Set("ttl", fmt.Sprintf("%v", ttl))
	}

	return v
}

// convert key string to http path exclude version
// for example: key[foo] -> path[keys/foo]
// key[/] -> path[keys/]
func keyToPath(key string) string {
	p := path.Join("keys", key)

	// corner case: if key is "/" or "//" ect
	// path join will clear the tailing "/"
	// we need to add it back
	if p == "keys" {
		p = "keys/"
	}

	return p
}
<commit_msg>bump(go-etcd): b4cb8c977c7b725ecf7c24ee9be24936ff68d1e4<commit_after>package etcd

import (
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"path"
	"strings"
	"sync"
	"time"
)

// Errors introduced by handling requests
var (
	ErrRequestCancelled = errors.New("sending request is cancelled")
)

type RawRequest struct {
	Method       string
	RelativePath string
	Values       url.Values
	Cancel       <-chan bool
}

// NewRawRequest returns a new RawRequest
func NewRawRequest(method, relativePath string, values url.Values, cancel <-chan bool) *RawRequest {
	return &RawRequest{
		Method:       method,
		RelativePath: relativePath,
		Values:       values,
		Cancel:       cancel,
	}
}

// getCancelable issues a cancelable GET request
func (c *Client) getCancelable(key string, options Options,
	cancel <-chan bool) (*RawResponse, error) {
	logger.Debugf("get %s [%s]", key, c.cluster.Leader)
	p := keyToPath(key)

	// If consistency level is set to STRONG, append
	// the `consistent` query string.
	if c.config.Consistency == STRONG_CONSISTENCY {
		options["consistent"] = true
	}

	str, err := options.toParameters(VALID_GET_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	req := NewRawRequest("GET", p, nil, cancel)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// get issues a GET request
func (c *Client) get(key string, options Options) (*RawResponse, error) {
	return c.getCancelable(key, options, nil)
}

// put issues a PUT request
func (c *Client) put(key string, value string, ttl uint64,
	options Options) (*RawResponse, error) {

	logger.Debugf("put %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader)
	p := keyToPath(key)

	str, err := options.toParameters(VALID_PUT_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	req := NewRawRequest("PUT", p, buildValues(value, ttl), nil)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// post issues a POST request
func (c *Client) post(key string, value string, ttl uint64) (*RawResponse, error) {
	logger.Debugf("post %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader)
	p := keyToPath(key)

	req := NewRawRequest("POST", p, buildValues(value, ttl), nil)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// delete issues a DELETE request
func (c *Client) delete(key string, options Options) (*RawResponse, error) {
	logger.Debugf("delete %s [%s]", key, c.cluster.Leader)
	p := keyToPath(key)

	str, err := options.toParameters(VALID_DELETE_OPTIONS)
	if err != nil {
		return nil, err
	}
	p += str

	req := NewRawRequest("DELETE", p, nil, nil)
	resp, err := c.SendRequest(req)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// SendRequest sends a HTTP request and returns a Response as defined by etcd
func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) {

	var req *http.Request
	var resp *http.Response
	var httpPath string
	var err error
	var respBody []byte

	reqs := make([]http.Request, 0)
	resps := make([]http.Response, 0)

	checkRetry := c.CheckRetry
	if checkRetry == nil {
		checkRetry = DefaultCheckRetry
	}

	cancelled := make(chan bool, 1)
	reqLock := new(sync.Mutex)

	if rr.Cancel != nil {
		cancelRoutine := make(chan bool)
		defer close(cancelRoutine)

		go func() {
			select {
			case <-rr.Cancel:
				cancelled <-true
				logger.Debug("send.request is cancelled")
			case <-cancelRoutine:
				return
			}

			// Repeat canceling request until this thread is stopped
			// because we have no idea about whether it succeeds.
			for {
				reqLock.Lock()
				c.httpClient.Transport.(*http.Transport).CancelRequest(req)
				reqLock.Unlock()

				select {
				case <-time.After(100 * time.Millisecond):
				case <-cancelRoutine:
					return
				}
			}
		}()
	}

	// if we connect to a follower, we will retry until we find a leader
	for attempt := 0; ; attempt++ {
		select {
		case <-cancelled:
			return nil, ErrRequestCancelled
		default:
		}

		logger.Debug("begin attempt", attempt, "for", rr.RelativePath)

		if rr.Method == "GET" && c.config.Consistency == WEAK_CONSISTENCY {
			// If it's a GET and consistency level is set to WEAK,
			// then use a random machine.
			httpPath = c.getHttpPath(true, rr.RelativePath)
		} else {
			// Else use the leader.
			httpPath = c.getHttpPath(false, rr.RelativePath)
		}

		// Return a cURL command if curlChan is set
		if c.cURLch != nil {
			command := fmt.Sprintf("curl -X %s %s", rr.Method, httpPath)
			for key, value := range rr.Values {
				command += fmt.Sprintf(" -d %s=%s", key, value[0])
			}
			c.sendCURL(command)
		}

		logger.Debug("send.request.to ", httpPath, " | method ", rr.Method)

		reqLock.Lock()
		if rr.Values == nil {
			req, err = http.NewRequest(rr.Method, httpPath, nil)
		} else {
			req, err = http.NewRequest(rr.Method, httpPath,
				strings.NewReader(rr.Values.Encode()))

			req.Header.Set("Content-Type",
				"application/x-www-form-urlencoded; param=value")
		}
		reqLock.Unlock()

		if err != nil {
			return nil, err
		}

		resp, err = c.httpClient.Do(req)
		// If the request was cancelled, return ErrRequestCancelled directly
		select {
		case <-cancelled:
			return nil, ErrRequestCancelled
		default:
		}

		reqs = append(reqs, *req)

		// network error, change a machine!
		if err != nil {
			logger.Debug("network error:", err.Error())
			resps = append(resps, http.Response{})
			if checkErr := checkRetry(c.cluster, reqs, resps, err); checkErr != nil {
				return nil, checkErr
			}

			c.cluster.switchLeader(attempt % len(c.cluster.Machines))
			continue
		}

		// if there is no error, it should receive response
		resps = append(resps, *resp)
		defer resp.Body.Close()
		logger.Debug("recv.response.from", httpPath)

		if validHttpStatusCode[resp.StatusCode] {
			// try to read byte code and break the loop
			respBody, err = ioutil.ReadAll(resp.Body)
			if err == nil {
				logger.Debug("recv.success.", httpPath)
				break
			}
		}

		// if resp is TemporaryRedirect, set the new leader and retry
		if resp.StatusCode == http.StatusTemporaryRedirect {
			u, err := resp.Location()

			if err != nil {
				logger.Warning(err)
			} else {
				// Update cluster leader based on redirect location
				// because it should point to the leader address
				c.cluster.updateLeaderFromURL(u)
				logger.Debug("recv.response.relocate", u.String())
			}
			continue
		}

		if checkErr := checkRetry(c.cluster, reqs, resps,
			errors.New("Unexpected HTTP status code")); checkErr != nil {
			return nil, checkErr
		}
	}

	r := &RawResponse{
		StatusCode: resp.StatusCode,
		Body:       respBody,
		Header:     resp.Header,
	}

	return r, nil
}

// DefaultCheckRetry checks retry cases
// If it has retried 2 * machine number, stop to retry it anymore
// If resp is nil, sleep for 200ms
// If status code is InternalServerError, sleep for 200ms.
func DefaultCheckRetry(cluster *Cluster, reqs []http.Request,
	resps []http.Response, err error) error {

	if len(reqs) >= 2*len(cluster.Machines) {
		return newError(ErrCodeEtcdNotReachable,
			"Tried to connect to each peer twice and failed", 0)
	}

	resp := &resps[len(resps)-1]

	if resp == nil {
		time.Sleep(time.Millisecond * 200)
		return nil
	}

	code := resp.StatusCode
	if code == http.StatusInternalServerError {
		time.Sleep(time.Millisecond * 200)

	}

	logger.Warning("bad response status code", code)
	return nil
}

func (c *Client) getHttpPath(random bool, s ...string) string {
	var machine string
	if random {
		machine = c.cluster.Machines[rand.Intn(len(c.cluster.Machines))]
	} else {
		machine = c.cluster.Leader
	}

	fullPath := machine + "/" + version
	for _, seg := range s {
		fullPath = fullPath + "/" + seg
	}

	return fullPath
}

// buildValues builds a url.Values map according to the given value and ttl
func buildValues(value string, ttl uint64) url.Values {
	v := url.Values{}

	if value != "" {
		v.Set("value", value)
	}

	if ttl > 0 {
		v.Set("ttl", fmt.Sprintf("%v", ttl))
	}

	return v
}

// convert key string to http path exclude version
// for example: key[foo] -> path[keys/foo]
// key[/] -> path[keys/]
func keyToPath(key string) string {
	p := path.Join("keys", key)

	// corner case: if key is "/" or "//" ect
	// path join will clear the tailing "/"
	// we need to add it back
	if p == "keys" {
		p = "keys/"
	}

	return p
}
<|endoftext|>


<commit_before><commit_msg>Temporary fix for the deep equals usage inside bolt storage<commit_after><|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cipher

import "io"

// The Stream* objects are so simple that all their members are public. Users
// can create them themselves.

// StreamReader wraps a Stream into an io.Reader. It calls XORKeyStream
// to process each slice of data which passes through.
type StreamReader struct {
	S Stream
	R io.Reader
}

func (r StreamReader) Read(dst []byte) (n int, err error) {
	n, err = r.R.Read(dst)
	r.S.XORKeyStream(dst[:n], dst[:n])
	return
}

// StreamWriter wraps a Stream into an io.Writer. It calls XORKeyStream
// to process each slice of data which passes through. If any Write call
// returns short then the StreamWriter is out of sync and must be discarded.
type StreamWriter struct {
	S   Stream
	W   io.Writer
	Err error // unused
}

func (w StreamWriter) Write(src []byte) (n int, err error) {
	c := make([]byte, len(src))
	w.S.XORKeyStream(c, src)
	n, err = w.W.Write(c)
	if n != len(src) {
		if err == nil { // should never happen
			err = io.ErrShortWrite
		}
	}
	return
}

func (w StreamWriter) Close() error {
	// This saves us from either requiring a WriteCloser or having a
	// StreamWriterCloser.
	return w.W.(io.Closer).Close()
}
<commit_msg>crypto/cipher: StreamWriter.Closer docs + behavior change<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cipher

import "io"

// The Stream* objects are so simple that all their members are public. Users
// can create them themselves.

// StreamReader wraps a Stream into an io.Reader. It calls XORKeyStream
// to process each slice of data which passes through.
type StreamReader struct {
	S Stream
	R io.Reader
}

func (r StreamReader) Read(dst []byte) (n int, err error) {
	n, err = r.R.Read(dst)
	r.S.XORKeyStream(dst[:n], dst[:n])
	return
}

// StreamWriter wraps a Stream into an io.Writer. It calls XORKeyStream
// to process each slice of data which passes through. If any Write call
// returns short then the StreamWriter is out of sync and must be discarded.
// A StreamWriter has no internal buffering; Close does not need
// to be called to flush write data.
type StreamWriter struct {
	S   Stream
	W   io.Writer
	Err error // unused
}

func (w StreamWriter) Write(src []byte) (n int, err error) {
	c := make([]byte, len(src))
	w.S.XORKeyStream(c, src)
	n, err = w.W.Write(c)
	if n != len(src) {
		if err == nil { // should never happen
			err = io.ErrShortWrite
		}
	}
	return
}

// Close closes the underlying Writer and returns its Close return value, if the Writer
// is also an io.Closer. Otherwise it returns nil.
func (w StreamWriter) Close() error {
	if c, ok := w.W.(io.Closer); ok {
		return c.Close()
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Sockets for Windows

package net

import (
	"syscall"
)

func setKernelSpecificSockopt(s syscall.Handle, f int) {
	// Allow reuse of recently-used addresses and ports.
	syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)

	// Allow broadcast.
	syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1)

	if f == syscall.AF_INET6 {
		// using ip, tcp, udp, etc.
		// allow both protocols even if the OS default is otherwise.
		syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, 0)
	}
}
<commit_msg>net: do not set SO_REUSEADDR for windows<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Sockets for Windows

package net

import (
	"syscall"
)

func setKernelSpecificSockopt(s syscall.Handle, f int) {
	// Allow broadcast.
	syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1)

	if f == syscall.AF_INET6 {
		// using ip, tcp, udp, etc.
		// allow both protocols even if the OS default is otherwise.
		syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, 0)
	}
}
<|endoftext|>


<commit_before>package fluentd

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/plugins/inputs"
)

const (
	measurement  = "fluentd"
	description  = "Read metrics exposed by fluentd in_monitor plugin"
	sampleConfig = `
  ## This plugin reads information exposed by fluentd (using /api/plugins.json endpoint).
  ##
  ## Endpoint:
  ## - only one URI is allowed
  ## - https is not supported
  endpoint = "http://localhost:24220/api/plugins.json"

  ## Define which plugins have to be excluded (based on "type" field - e.g. monitor_agent)
  exclude = [
	  "monitor_agent",
	  "dummy",
  ]
`
)

// Fluentd - plugin main structure
type Fluentd struct {
	Endpoint string
	Exclude  []string
	client   *http.Client
}

type endpointInfo struct {
	Payload []pluginData `json:"plugins"`
}

type pluginData struct {
	PluginID              string   `json:"plugin_id"`
	PluginType            string   `json:"type"`
	PluginCategory        string   `json:"plugin_category"`
	RetryCount            *float64 `json:"retry_count"`
	BufferQueueLength     *float64 `json:"buffer_queue_length"`
	BufferTotalQueuedSize *float64 `json:"buffer_total_queued_size"`
}

// parse JSON from fluentd Endpoint
// Parameters:
// 		data: unprocessed json recivied from endpoint
//
// Returns:
//		pluginData:		slice that contains parsed plugins
//		error:			error that may have occurred
func parse(data []byte) (datapointArray []pluginData, err error) {
	var endpointData endpointInfo

	if err = json.Unmarshal(data, &endpointData); err != nil {
		err = fmt.Errorf("Processing JSON structure")
		return
	}

	for _, point := range endpointData.Payload {
		datapointArray = append(datapointArray, point)
	}

	return
}

// Description - display description
func (h *Fluentd) Description() string { return description }

// SampleConfig - generate configuretion
func (h *Fluentd) SampleConfig() string { return sampleConfig }

// Gather - Main code responsible for gathering, processing and creating metrics
func (h *Fluentd) Gather(acc telegraf.Accumulator) error {

	_, err := url.Parse(h.Endpoint)
	if err != nil {
		return fmt.Errorf("Invalid URL \"%s\"", h.Endpoint)
	}

	if h.client == nil {

		tr := &http.Transport{
			ResponseHeaderTimeout: time.Duration(3 * time.Second),
		}

		client := &http.Client{
			Transport: tr,
			Timeout:   time.Duration(4 * time.Second),
		}

		h.client = client
	}

	resp, err := h.client.Get(h.Endpoint)

	if err != nil {
		return fmt.Errorf("Unable to perform HTTP client GET on \"%s\": %s", h.Endpoint, err)
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		return fmt.Errorf("Unable to read the HTTP body \"%s\": %s", string(body), err)
	}

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("http status ok not met")
	}

	dataPoints, err := parse(body)

	if err != nil {
		return fmt.Errorf("Problem with parsing")
	}

	// Go through all plugins one by one
	for _, p := range dataPoints {

		skip := false

		// Check if this specific type was excluded in configuration
		for _, exclude := range h.Exclude {
			if exclude == p.PluginType {
				skip = true
			}
		}

		// If not, create new metric and add it to Accumulator
		if !skip {
			tmpFields := make(map[string]interface{})

			tmpTags := map[string]string{
				"plugin_id":       p.PluginID,
				"plugin_category": p.PluginCategory,
				"plugin_type":     p.PluginType,
			}

			if p.BufferQueueLength != nil {
				tmpFields["buffer_queue_length"] = p.BufferQueueLength

			}
			if p.RetryCount != nil {
				tmpFields["retry_count"] = p.RetryCount
			}

			if p.BufferTotalQueuedSize != nil {
				tmpFields["buffer_total_queued_size"] = p.BufferTotalQueuedSize
			}

			if !((p.BufferQueueLength == nil) && (p.RetryCount == nil) && (p.BufferTotalQueuedSize == nil)) {
				acc.AddFields(measurement, tmpFields, tmpTags)
			}
		}
	}

	return nil
}

func init() {
	inputs.Add("fluentd", func() telegraf.Input { return &Fluentd{} })
}
<commit_msg>Fix optional field types in fluentd input<commit_after>package fluentd

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/plugins/inputs"
)

const (
	measurement  = "fluentd"
	description  = "Read metrics exposed by fluentd in_monitor plugin"
	sampleConfig = `
  ## This plugin reads information exposed by fluentd (using /api/plugins.json endpoint).
  ##
  ## Endpoint:
  ## - only one URI is allowed
  ## - https is not supported
  endpoint = "http://localhost:24220/api/plugins.json"

  ## Define which plugins have to be excluded (based on "type" field - e.g. monitor_agent)
  exclude = [
	  "monitor_agent",
	  "dummy",
  ]
`
)

// Fluentd - plugin main structure
type Fluentd struct {
	Endpoint string
	Exclude  []string
	client   *http.Client
}

type endpointInfo struct {
	Payload []pluginData `json:"plugins"`
}

type pluginData struct {
	PluginID              string   `json:"plugin_id"`
	PluginType            string   `json:"type"`
	PluginCategory        string   `json:"plugin_category"`
	RetryCount            *float64 `json:"retry_count"`
	BufferQueueLength     *float64 `json:"buffer_queue_length"`
	BufferTotalQueuedSize *float64 `json:"buffer_total_queued_size"`
}

// parse JSON from fluentd Endpoint
// Parameters:
// 		data: unprocessed json recivied from endpoint
//
// Returns:
//		pluginData:		slice that contains parsed plugins
//		error:			error that may have occurred
func parse(data []byte) (datapointArray []pluginData, err error) {
	var endpointData endpointInfo

	if err = json.Unmarshal(data, &endpointData); err != nil {
		err = fmt.Errorf("Processing JSON structure")
		return
	}

	for _, point := range endpointData.Payload {
		datapointArray = append(datapointArray, point)
	}

	return
}

// Description - display description
func (h *Fluentd) Description() string { return description }

// SampleConfig - generate configuretion
func (h *Fluentd) SampleConfig() string { return sampleConfig }

// Gather - Main code responsible for gathering, processing and creating metrics
func (h *Fluentd) Gather(acc telegraf.Accumulator) error {

	_, err := url.Parse(h.Endpoint)
	if err != nil {
		return fmt.Errorf("Invalid URL \"%s\"", h.Endpoint)
	}

	if h.client == nil {

		tr := &http.Transport{
			ResponseHeaderTimeout: time.Duration(3 * time.Second),
		}

		client := &http.Client{
			Transport: tr,
			Timeout:   time.Duration(4 * time.Second),
		}

		h.client = client
	}

	resp, err := h.client.Get(h.Endpoint)

	if err != nil {
		return fmt.Errorf("Unable to perform HTTP client GET on \"%s\": %s", h.Endpoint, err)
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		return fmt.Errorf("Unable to read the HTTP body \"%s\": %s", string(body), err)
	}

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("http status ok not met")
	}

	dataPoints, err := parse(body)

	if err != nil {
		return fmt.Errorf("Problem with parsing")
	}

	// Go through all plugins one by one
	for _, p := range dataPoints {

		skip := false

		// Check if this specific type was excluded in configuration
		for _, exclude := range h.Exclude {
			if exclude == p.PluginType {
				skip = true
			}
		}

		// If not, create new metric and add it to Accumulator
		if !skip {
			tmpFields := make(map[string]interface{})

			tmpTags := map[string]string{
				"plugin_id":       p.PluginID,
				"plugin_category": p.PluginCategory,
				"plugin_type":     p.PluginType,
			}

			if p.BufferQueueLength != nil {
				tmpFields["buffer_queue_length"] = *p.BufferQueueLength

			}
			if p.RetryCount != nil {
				tmpFields["retry_count"] = *p.RetryCount
			}

			if p.BufferTotalQueuedSize != nil {
				tmpFields["buffer_total_queued_size"] = *p.BufferTotalQueuedSize
			}

			if !((p.BufferQueueLength == nil) && (p.RetryCount == nil) && (p.BufferTotalQueuedSize == nil)) {
				acc.AddFields(measurement, tmpFields, tmpTags)
			}
		}
	}

	return nil
}

func init() {
	inputs.Add("fluentd", func() telegraf.Input { return &Fluentd{} })
}
<|endoftext|>


<commit_before>package execd

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"strings"
	"time"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/config"
	"github.com/influxdata/telegraf/internal/process"
	"github.com/influxdata/telegraf/plugins/parsers"
	"github.com/influxdata/telegraf/plugins/processors"
	"github.com/influxdata/telegraf/plugins/serializers"
)

const sampleConfig = `
	## Program to run as daemon
	## eg: command = ["/path/to/your_program", "arg1", "arg2"]
	command = ["cat"]

  ## Delay before the process is restarted after an unexpected termination
  restart_delay = "10s"
`

type Execd struct {
	Command      []string        `toml:"command"`
	RestartDelay config.Duration `toml:"restart_delay"`
	Log          telegraf.Logger

	parserConfig     *parsers.Config
	parser           parsers.Parser
	serializerConfig *serializers.Config
	serializer       serializers.Serializer
	acc              telegraf.Accumulator
	process          *process.Process
}

func New() *Execd {
	return &Execd{
		RestartDelay: config.Duration(10 * time.Second),
		parserConfig: &parsers.Config{
			DataFormat: "influx",
		},
		serializerConfig: &serializers.Config{
			DataFormat: "influx",
		},
	}
}

func (e *Execd) SampleConfig() string {
	return sampleConfig
}

func (e *Execd) Description() string {
	return "Run executable as long-running processor plugin"
}

func (e *Execd) Start(acc telegraf.Accumulator) error {
	var err error
	e.parser, err = parsers.NewParser(e.parserConfig)
	if err != nil {
		return fmt.Errorf("error creating parser: %w", err)
	}
	e.serializer, err = serializers.NewSerializer(e.serializerConfig)
	if err != nil {
		return fmt.Errorf("error creating serializer: %w", err)
	}
	e.acc = acc

	e.process, err = process.New(e.Command)
	if err != nil {
		return fmt.Errorf("error creating new process: %w", err)
	}
	e.process.Log = e.Log
	e.process.RestartDelay = time.Duration(e.RestartDelay)
	e.process.ReadStdoutFn = e.cmdReadOut
	e.process.ReadStderrFn = e.cmdReadErr

	if err = e.process.Start(); err != nil {
		// if there was only one argument, and it contained spaces, warn the user
		// that they may have configured it wrong.
		if len(e.Command) == 1 && strings.Contains(e.Command[0], " ") {
			e.Log.Warn("The processors.execd Command contained spaces but no arguments. " +
				"This setting expects the program and arguments as an array of strings, " +
				"not as a space-delimited string. See the plugin readme for an example.")
		}
		return fmt.Errorf("failed to start process %s: %w", e.Command, err)
	}

	return nil
}

func (e *Execd) Add(m telegraf.Metric, acc telegraf.Accumulator) error {
	b, err := e.serializer.Serialize(m)
	if err != nil {
		return fmt.Errorf("metric serializing error: %w", err)
	}

	_, err = e.process.Stdin.Write(b)
	if err != nil {
		return fmt.Errorf("error writing to process stdin: %w", err)
	}

	// We cannot maintain tracking metrics at the moment because input/output
	// is done asynchronously and we don't have any metric metadata to tie the
	// output metric back to the original input metric.
	m.Drop()
	return nil
}

func (e *Execd) Stop() error {
	e.process.Stop()
	return nil
}

func (e *Execd) cmdReadOut(out io.Reader) {
	scanner := bufio.NewScanner(out)

	for scanner.Scan() {
		metrics, err := e.parser.Parse(scanner.Bytes())
		if err != nil {
			e.Log.Errorf("Parse error: %s", err)
		}

		for _, metric := range metrics {
			e.acc.AddMetric(metric)
		}
	}

	if err := scanner.Err(); err != nil {
		e.Log.Errorf("Error reading stdout: %s", err)
	}
}

func (e *Execd) cmdReadErr(out io.Reader) {
	scanner := bufio.NewScanner(out)

	for scanner.Scan() {
		e.Log.Errorf("stderr: %q", scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		e.Log.Errorf("Error reading stderr: %s", err)
	}
}

func (e *Execd) Init() error {
	if len(e.Command) == 0 {
		return errors.New("no command specified")
	}
	return nil
}

func init() {
	processors.AddStreaming("execd", func() telegraf.StreamingProcessor {
		return New()
	})
}
<commit_msg>Increasing the metric buffer (#8145)<commit_after>package execd

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"strings"
	"time"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/config"
	"github.com/influxdata/telegraf/internal/process"
	"github.com/influxdata/telegraf/plugins/parsers"
	"github.com/influxdata/telegraf/plugins/processors"
	"github.com/influxdata/telegraf/plugins/serializers"
)

const sampleConfig = `
	## Program to run as daemon
	## eg: command = ["/path/to/your_program", "arg1", "arg2"]
	command = ["cat"]

  ## Delay before the process is restarted after an unexpected termination
  restart_delay = "10s"
`

type Execd struct {
	Command      []string        `toml:"command"`
	RestartDelay config.Duration `toml:"restart_delay"`
	Log          telegraf.Logger

	parserConfig     *parsers.Config
	parser           parsers.Parser
	serializerConfig *serializers.Config
	serializer       serializers.Serializer
	acc              telegraf.Accumulator
	process          *process.Process
}

func New() *Execd {
	return &Execd{
		RestartDelay: config.Duration(10 * time.Second),
		parserConfig: &parsers.Config{
			DataFormat: "influx",
		},
		serializerConfig: &serializers.Config{
			DataFormat: "influx",
		},
	}
}

func (e *Execd) SampleConfig() string {
	return sampleConfig
}

func (e *Execd) Description() string {
	return "Run executable as long-running processor plugin"
}

func (e *Execd) Start(acc telegraf.Accumulator) error {
	var err error
	e.parser, err = parsers.NewParser(e.parserConfig)
	if err != nil {
		return fmt.Errorf("error creating parser: %w", err)
	}
	e.serializer, err = serializers.NewSerializer(e.serializerConfig)
	if err != nil {
		return fmt.Errorf("error creating serializer: %w", err)
	}
	e.acc = acc

	e.process, err = process.New(e.Command)
	if err != nil {
		return fmt.Errorf("error creating new process: %w", err)
	}
	e.process.Log = e.Log
	e.process.RestartDelay = time.Duration(e.RestartDelay)
	e.process.ReadStdoutFn = e.cmdReadOut
	e.process.ReadStderrFn = e.cmdReadErr

	if err = e.process.Start(); err != nil {
		// if there was only one argument, and it contained spaces, warn the user
		// that they may have configured it wrong.
		if len(e.Command) == 1 && strings.Contains(e.Command[0], " ") {
			e.Log.Warn("The processors.execd Command contained spaces but no arguments. " +
				"This setting expects the program and arguments as an array of strings, " +
				"not as a space-delimited string. See the plugin readme for an example.")
		}
		return fmt.Errorf("failed to start process %s: %w", e.Command, err)
	}

	return nil
}

func (e *Execd) Add(m telegraf.Metric, acc telegraf.Accumulator) error {
	b, err := e.serializer.Serialize(m)
	if err != nil {
		return fmt.Errorf("metric serializing error: %w", err)
	}

	_, err = e.process.Stdin.Write(b)
	if err != nil {
		return fmt.Errorf("error writing to process stdin: %w", err)
	}

	// We cannot maintain tracking metrics at the moment because input/output
	// is done asynchronously and we don't have any metric metadata to tie the
	// output metric back to the original input metric.
	m.Drop()
	return nil
}

func (e *Execd) Stop() error {
	e.process.Stop()
	return nil
}

func (e *Execd) cmdReadOut(out io.Reader) {
	scanner := bufio.NewScanner(out)
	scanBuf := make([]byte, 4096)
	scanner.Buffer(scanBuf, 262144)

	for scanner.Scan() {
		metrics, err := e.parser.Parse(scanner.Bytes())
		if err != nil {
			e.Log.Errorf("Parse error: %s", err)
		}

		for _, metric := range metrics {
			e.acc.AddMetric(metric)
		}
	}

	if err := scanner.Err(); err != nil {
		e.Log.Errorf("Error reading stdout: %s", err)
	}
}

func (e *Execd) cmdReadErr(out io.Reader) {
	scanner := bufio.NewScanner(out)

	for scanner.Scan() {
		e.Log.Errorf("stderr: %q", scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		e.Log.Errorf("Error reading stderr: %s", err)
	}
}

func (e *Execd) Init() error {
	if len(e.Command) == 0 {
		return errors.New("no command specified")
	}
	return nil
}

func init() {
	processors.AddStreaming("execd", func() telegraf.StreamingProcessor {
		return New()
	})
}
<|endoftext|>


<commit_before>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by lister-gen. DO NOT EDIT.

package v1alpha1

import (
	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/tools/cache"
	v1alpha1 "k8s.io/sample-controller/pkg/apis/samplecontroller/v1alpha1"
)

// FooLister helps list Foos.
type FooLister interface {
	// List lists all Foos in the indexer.
	List(selector labels.Selector) (ret []*v1alpha1.Foo, err error)
	// Foos returns an object that can list and get Foos.
	Foos(namespace string) FooNamespaceLister
	FooListerExpansion
}

// fooLister implements the FooLister interface.
type fooLister struct {
	indexer cache.Indexer
}

// NewFooLister returns a new FooLister.
func NewFooLister(indexer cache.Indexer) FooLister {
	return &fooLister{indexer: indexer}
}

// List lists all Foos in the indexer.
func (s *fooLister) List(selector labels.Selector) (ret []*v1alpha1.Foo, err error) {
	err = cache.ListAll(s.indexer, selector, func(m interface{}) {
		ret = append(ret, m.(*v1alpha1.Foo))
	})
	return ret, err
}

// Foos returns an object that can list and get Foos.
func (s *fooLister) Foos(namespace string) FooNamespaceLister {
	return fooNamespaceLister{indexer: s.indexer, namespace: namespace}
}

// FooNamespaceLister helps list and get Foos.
type FooNamespaceLister interface {
	// List lists all Foos in the indexer for a given namespace.
	List(selector labels.Selector) (ret []*v1alpha1.Foo, err error)
	// Get retrieves the Foo from the indexer for a given namespace and name.
	Get(name string) (*v1alpha1.Foo, error)
	FooNamespaceListerExpansion
}

// fooNamespaceLister implements the FooNamespaceLister
// interface.
type fooNamespaceLister struct {
	indexer   cache.Indexer
	namespace string
}

// List lists all Foos in the indexer for a given namespace.
func (s fooNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.Foo, err error) {
	err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) {
		ret = append(ret, m.(*v1alpha1.Foo))
	})
	return ret, err
}

// Get retrieves the Foo from the indexer for a given namespace and name.
func (s fooNamespaceLister) Get(name string) (*v1alpha1.Foo, error) {
	obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name)
	if err != nil {
		return nil, err
	}
	if !exists {
		return nil, errors.NewNotFound(v1alpha1.Resource("foo"), name)
	}
	return obj.(*v1alpha1.Foo), nil
}
<commit_msg>Re-generate all listers<commit_after>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by lister-gen. DO NOT EDIT.

package v1alpha1

import (
	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/tools/cache"
	v1alpha1 "k8s.io/sample-controller/pkg/apis/samplecontroller/v1alpha1"
)

// FooLister helps list Foos.
// All objects returned here must be treated as read-only.
type FooLister interface {
	// List lists all Foos in the indexer.
	// Objects returned here must be treated as read-only.
	List(selector labels.Selector) (ret []*v1alpha1.Foo, err error)
	// Foos returns an object that can list and get Foos.
	Foos(namespace string) FooNamespaceLister
	FooListerExpansion
}

// fooLister implements the FooLister interface.
type fooLister struct {
	indexer cache.Indexer
}

// NewFooLister returns a new FooLister.
func NewFooLister(indexer cache.Indexer) FooLister {
	return &fooLister{indexer: indexer}
}

// List lists all Foos in the indexer.
func (s *fooLister) List(selector labels.Selector) (ret []*v1alpha1.Foo, err error) {
	err = cache.ListAll(s.indexer, selector, func(m interface{}) {
		ret = append(ret, m.(*v1alpha1.Foo))
	})
	return ret, err
}

// Foos returns an object that can list and get Foos.
func (s *fooLister) Foos(namespace string) FooNamespaceLister {
	return fooNamespaceLister{indexer: s.indexer, namespace: namespace}
}

// FooNamespaceLister helps list and get Foos.
// All objects returned here must be treated as read-only.
type FooNamespaceLister interface {
	// List lists all Foos in the indexer for a given namespace.
	// Objects returned here must be treated as read-only.
	List(selector labels.Selector) (ret []*v1alpha1.Foo, err error)
	// Get retrieves the Foo from the indexer for a given namespace and name.
	// Objects returned here must be treated as read-only.
	Get(name string) (*v1alpha1.Foo, error)
	FooNamespaceListerExpansion
}

// fooNamespaceLister implements the FooNamespaceLister
// interface.
type fooNamespaceLister struct {
	indexer   cache.Indexer
	namespace string
}

// List lists all Foos in the indexer for a given namespace.
func (s fooNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.Foo, err error) {
	err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) {
		ret = append(ret, m.(*v1alpha1.Foo))
	})
	return ret, err
}

// Get retrieves the Foo from the indexer for a given namespace and name.
func (s fooNamespaceLister) Get(name string) (*v1alpha1.Foo, error) {
	obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name)
	if err != nil {
		return nil, err
	}
	if !exists {
		return nil, errors.NewNotFound(v1alpha1.Resource("foo"), name)
	}
	return obj.(*v1alpha1.Foo), nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package addons

const (
	KubeProxyConfigMap = `
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-proxy
  namespace: kube-system
  labels:
    app: kube-proxy
data:
  kubeconfig.conf: |
    apiVersion: v1
    kind: Config
    clusters:
    - cluster:
        certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        server: {{ .MasterEndpoint }}
      name: default
    contexts:
    - context:
        cluster: default
        namespace: default
        user: default
      name: default
    current-context: default
    users:
    - name: default
      user:
        tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
`

	KubeProxyDaemonSet = `
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  labels:
    k8s-app: kube-proxy
  name: kube-proxy
  namespace: kube-system
spec:
  selector:
    matchLabels:
      k8s-app: kube-proxy
  template:
    metadata:
      labels:
        k8s-app: kube-proxy
    spec:
      containers:
      - name: kube-proxy
        image: {{ .Image }}
        imagePullPolicy: IfNotPresent
        command:
        - /usr/local/bin/kube-proxy
        - --kubeconfig=/var/lib/kube-proxy/kubeconfig.conf
        {{ .ClusterCIDR }}
        securityContext:
          privileged: true
        volumeMounts:
        - mountPath: /var/lib/kube-proxy
          name: kube-proxy
      hostNetwork: true
      serviceAccountName: kube-proxy
      # TODO: Why doesn't the Decoder recognize this new field and decode it properly? Right now it's ignored
      # tolerations:
      # - key: {{ .MasterTaintKey }}
      #   effect: NoSchedule
      volumes:
      - name: kube-proxy
        configMap:
          name: kube-proxy
`

	KubeDNSVersion = "1.14.2"

	KubeDNSDeployment = `

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: kube-dns
  namespace: kube-system
  labels:
    k8s-app: kube-dns
spec:
  # replicas: not specified here:
  # 1. In order to make Addon Manager do not reconcile this replicas parameter.
  # 2. Default is 1.
  # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
  strategy:
    rollingUpdate:
      maxSurge: 10%
      maxUnavailable: 0
  selector:
    matchLabels:
      k8s-app: kube-dns
  template:
    metadata:
      labels:
        k8s-app: kube-dns
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      volumes:
      - name: kube-dns-config
        configMap:
          name: kube-dns
          optional: true
      containers:
      - name: kubedns
        image: {{ .ImageRepository }}/k8s-dns-kube-dns-{{ .Arch }}:{{ .Version }}
        imagePullPolicy: IfNotPresent
        resources:
          # TODO: Set memory limits when we've profiled the container for large
          # clusters, then set request = limit to keep this container in
          # guaranteed class. Currently, this container falls into the
          # "burstable" category so the kubelet doesn't backoff from restarting it.
          limits:
            memory: 170Mi
          requests:
            cpu: 100m
            memory: 70Mi
        livenessProbe:
          httpGet:
            path: /healthcheck/kubedns
            port: 10054
            scheme: HTTP
          initialDelaySeconds: 60
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        readinessProbe:
          httpGet:
            path: /readiness
            port: 8081
            scheme: HTTP
          # we poll on pod startup for the Kubernetes master service and
          # only setup the /readiness HTTP server once that's available.
          initialDelaySeconds: 3
          timeoutSeconds: 5
        args:
        - --domain={{ .DNSDomain }}.
        - --dns-port=10053
        - --config-dir=/kube-dns-config
        - --v=2
        env:
        - name: PROMETHEUS_PORT
          value: "10055"
        ports:
        - containerPort: 10053
          name: dns-local
          protocol: UDP
        - containerPort: 10053
          name: dns-tcp-local
          protocol: TCP
        - containerPort: 10055
          name: metrics
          protocol: TCP
        volumeMounts:
        - name: kube-dns-config
          mountPath: /kube-dns-config
      - name: dnsmasq
        image: {{ .ImageRepository }}/k8s-dns-dnsmasq-nanny-{{ .Arch }}:{{ .Version }}
        imagePullPolicy: IfNotPresent
        livenessProbe:
          httpGet:
            path: /healthcheck/dnsmasq
            port: 10054
            scheme: HTTP
          initialDelaySeconds: 60
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        args:
        - -v=2
        - -logtostderr
        - -configDir=/etc/k8s/dns/dnsmasq-nanny
        - -restartDnsmasq=true
        - --
        - -k
        - --cache-size=1000
        - --log-facility=-
        - --server=/{{ .DNSDomain }}/127.0.0.1#10053
        - --server=/in-addr.arpa/127.0.0.1#10053
        - --server=/ip6.arpa/127.0.0.1#10053
        ports:
        - containerPort: 53
          name: dns
          protocol: UDP
        - containerPort: 53
          name: dns-tcp
          protocol: TCP
        # see: https://github.com/kubernetes/kubernetes/issues/29055 for details
        resources:
          requests:
            cpu: 150m
            memory: 20Mi
        volumeMounts:
        - name: kube-dns-config
          mountPath: /etc/k8s/dns/dnsmasq-nanny
      - name: sidecar
        image: {{ .ImageRepository }}/k8s-dns-sidecar-{{ .Arch }}:{{ .Version }}
        imagePullPolicy: IfNotPresent
        livenessProbe:
          httpGet:
            path: /metrics
            port: 10054
            scheme: HTTP
          initialDelaySeconds: 60
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        args:
        - --v=2
        - --logtostderr
        - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.{{ .DNSDomain }},5,A
        - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.{{ .DNSDomain }},5,A
        ports:
        - containerPort: 10054
          name: metrics
          protocol: TCP
        resources:
          requests:
            memory: 20Mi
            cpu: 10m
      dnsPolicy: Default  # Don't use cluster DNS.
      serviceAccountName: kube-dns
      # TODO: Why doesn't the Decoder recognize this new field and decode it properly? Right now it's ignored
      # tolerations:
      # - key: CriticalAddonsOnly
      #   operator: Exists
      # - key: {{ .MasterTaintKey }}
      #   effect: NoSchedule
      # TODO: Remove this affinity field as soon as we are using manifest lists
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: beta.kubernetes.io/arch
                operator: In
                values:
                - {{ .Arch }}
`

	KubeDNSService = `
apiVersion: v1
kind: Service
metadata:
  labels:
    k8s-app: kube-dns
    kubernetes.io/cluster-service: "true"
    kubernetes.io/name: "KubeDNS"
  name: kube-dns
  namespace: kube-system
  # Without this resourceVersion value, an update of the Service between versions will yield:
  #   Service "kube-dns" is invalid: metadata.resourceVersion: Invalid value: "": must be specified for an update
  resourceVersion: "0"
spec:
  clusterIP: {{ .DNSIP }}
  ports:
  - name: dns
    port: 53
    protocol: UDP
    targetPort: 53
  - name: dns-tcp
    port: 53
    protocol: TCP
    targetPort: 53
  selector:
    k8s-app: kube-dns
`
)
<commit_msg>Enable iptables -w in kubeadm selfhosted<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package addons

const (
	KubeProxyConfigMap = `
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-proxy
  namespace: kube-system
  labels:
    app: kube-proxy
data:
  kubeconfig.conf: |
    apiVersion: v1
    kind: Config
    clusters:
    - cluster:
        certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        server: {{ .MasterEndpoint }}
      name: default
    contexts:
    - context:
        cluster: default
        namespace: default
        user: default
      name: default
    current-context: default
    users:
    - name: default
      user:
        tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
`

	KubeProxyDaemonSet = `
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  labels:
    k8s-app: kube-proxy
  name: kube-proxy
  namespace: kube-system
spec:
  selector:
    matchLabels:
      k8s-app: kube-proxy
  template:
    metadata:
      labels:
        k8s-app: kube-proxy
    spec:
      containers:
      - name: kube-proxy
        image: {{ .Image }}
        imagePullPolicy: IfNotPresent
        command:
        - /usr/local/bin/kube-proxy
        - --kubeconfig=/var/lib/kube-proxy/kubeconfig.conf
        {{ .ClusterCIDR }}
        volumeMounts:
        - mountPath: /var/lib/kube-proxy
          name: kube-proxy
        # TODO: Make this a file hostpath mount
        - mountPath: /run/xtables.lock
          name: xtables-lock
          readOnly: false
      securityContext:
        privileged: true
      hostNetwork: true
      serviceAccountName: kube-proxy
      # TODO: Why doesn't the Decoder recognize this new field and decode it properly? Right now it's ignored
      # tolerations:
      # - key: {{ .MasterTaintKey }}
      #   effect: NoSchedule
      volumes:
      - name: kube-proxy
        configMap:
          name: kube-proxy
      - name: xtables-lock
        hostPath:
          path: /run/xtables.lock
`

	KubeDNSVersion = "1.14.2"

	KubeDNSDeployment = `

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: kube-dns
  namespace: kube-system
  labels:
    k8s-app: kube-dns
spec:
  # replicas: not specified here:
  # 1. In order to make Addon Manager do not reconcile this replicas parameter.
  # 2. Default is 1.
  # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
  strategy:
    rollingUpdate:
      maxSurge: 10%
      maxUnavailable: 0
  selector:
    matchLabels:
      k8s-app: kube-dns
  template:
    metadata:
      labels:
        k8s-app: kube-dns
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      volumes:
      - name: kube-dns-config
        configMap:
          name: kube-dns
          optional: true
      containers:
      - name: kubedns
        image: {{ .ImageRepository }}/k8s-dns-kube-dns-{{ .Arch }}:{{ .Version }}
        imagePullPolicy: IfNotPresent
        resources:
          # TODO: Set memory limits when we've profiled the container for large
          # clusters, then set request = limit to keep this container in
          # guaranteed class. Currently, this container falls into the
          # "burstable" category so the kubelet doesn't backoff from restarting it.
          limits:
            memory: 170Mi
          requests:
            cpu: 100m
            memory: 70Mi
        livenessProbe:
          httpGet:
            path: /healthcheck/kubedns
            port: 10054
            scheme: HTTP
          initialDelaySeconds: 60
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        readinessProbe:
          httpGet:
            path: /readiness
            port: 8081
            scheme: HTTP
          # we poll on pod startup for the Kubernetes master service and
          # only setup the /readiness HTTP server once that's available.
          initialDelaySeconds: 3
          timeoutSeconds: 5
        args:
        - --domain={{ .DNSDomain }}.
        - --dns-port=10053
        - --config-dir=/kube-dns-config
        - --v=2
        env:
        - name: PROMETHEUS_PORT
          value: "10055"
        ports:
        - containerPort: 10053
          name: dns-local
          protocol: UDP
        - containerPort: 10053
          name: dns-tcp-local
          protocol: TCP
        - containerPort: 10055
          name: metrics
          protocol: TCP
        volumeMounts:
        - name: kube-dns-config
          mountPath: /kube-dns-config
      - name: dnsmasq
        image: {{ .ImageRepository }}/k8s-dns-dnsmasq-nanny-{{ .Arch }}:{{ .Version }}
        imagePullPolicy: IfNotPresent
        livenessProbe:
          httpGet:
            path: /healthcheck/dnsmasq
            port: 10054
            scheme: HTTP
          initialDelaySeconds: 60
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        args:
        - -v=2
        - -logtostderr
        - -configDir=/etc/k8s/dns/dnsmasq-nanny
        - -restartDnsmasq=true
        - --
        - -k
        - --cache-size=1000
        - --log-facility=-
        - --server=/{{ .DNSDomain }}/127.0.0.1#10053
        - --server=/in-addr.arpa/127.0.0.1#10053
        - --server=/ip6.arpa/127.0.0.1#10053
        ports:
        - containerPort: 53
          name: dns
          protocol: UDP
        - containerPort: 53
          name: dns-tcp
          protocol: TCP
        # see: https://github.com/kubernetes/kubernetes/issues/29055 for details
        resources:
          requests:
            cpu: 150m
            memory: 20Mi
        volumeMounts:
        - name: kube-dns-config
          mountPath: /etc/k8s/dns/dnsmasq-nanny
      - name: sidecar
        image: {{ .ImageRepository }}/k8s-dns-sidecar-{{ .Arch }}:{{ .Version }}
        imagePullPolicy: IfNotPresent
        livenessProbe:
          httpGet:
            path: /metrics
            port: 10054
            scheme: HTTP
          initialDelaySeconds: 60
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        args:
        - --v=2
        - --logtostderr
        - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.{{ .DNSDomain }},5,A
        - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.{{ .DNSDomain }},5,A
        ports:
        - containerPort: 10054
          name: metrics
          protocol: TCP
        resources:
          requests:
            memory: 20Mi
            cpu: 10m
      dnsPolicy: Default  # Don't use cluster DNS.
      serviceAccountName: kube-dns
      # TODO: Why doesn't the Decoder recognize this new field and decode it properly? Right now it's ignored
      # tolerations:
      # - key: CriticalAddonsOnly
      #   operator: Exists
      # - key: {{ .MasterTaintKey }}
      #   effect: NoSchedule
      # TODO: Remove this affinity field as soon as we are using manifest lists
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: beta.kubernetes.io/arch
                operator: In
                values:
                - {{ .Arch }}
`

	KubeDNSService = `
apiVersion: v1
kind: Service
metadata:
  labels:
    k8s-app: kube-dns
    kubernetes.io/cluster-service: "true"
    kubernetes.io/name: "KubeDNS"
  name: kube-dns
  namespace: kube-system
  # Without this resourceVersion value, an update of the Service between versions will yield:
  #   Service "kube-dns" is invalid: metadata.resourceVersion: Invalid value: "": must be specified for an update
  resourceVersion: "0"
spec:
  clusterIP: {{ .DNSIP }}
  ports:
  - name: dns
    port: 53
    protocol: UDP
    targetPort: 53
  - name: dns-tcp
    port: 53
    protocol: TCP
    targetPort: 53
  selector:
    k8s-app: kube-dns
`
)
<|endoftext|>