<commit_before>package nfpm

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"testing"

	"github.com/goreleaser/goreleaser/config"
	"github.com/goreleaser/goreleaser/context"
	"github.com/goreleaser/goreleaser/internal/artifact"
	"github.com/goreleaser/goreleaser/internal/testlib"
	"github.com/stretchr/testify/assert"
)

func TestDescription(t *testing.T) {
	assert.NotEmpty(t, Pipe{}.String())
}

func TestRunPipeNoFormats(t *testing.T) {
	var ctx = &context.Context{
		Git: context.GitInfo{
			CurrentTag: "v1.0.0",
		},
		Config:      config.Project{},
		Parallelism: runtime.NumCPU(),
	}
	testlib.AssertSkipped(t, Pipe{}.Run(ctx))
}

func TestRunPipeInvalidFormat(t *testing.T) {
	var ctx = context.New(config.Project{
		ProjectName: "nope",
		NFPM: config.NFPM{
			Bindir:  "/usr/bin",
			Formats: []string{"nope"},
			NFPMOverridables: config.NFPMOverridables{
				NameTemplate: defaultNameTemplate,
				Files:        map[string]string{},
			},
		},
	})
	ctx.Git = context.GitInfo{
		CurrentTag: "v1.2.3",
	}
	for _, goos := range []string{"linux", "darwin"} {
		for _, goarch := range []string{"amd64", "386"} {
			ctx.Artifacts.Add(artifact.Artifact{
				Name:   "mybin",
				Path:   "whatever",
				Goarch: goarch,
				Goos:   goos,
				Type:   artifact.Binary,
			})
		}
	}
	assert.Contains(t, Pipe{}.Run(ctx).Error(), `no packager registered for the format nope`)
}

func TestRunPipe(t *testing.T) {
	folder, err := ioutil.TempDir("", "archivetest")
	assert.NoError(t, err)
	var dist = filepath.Join(folder, "dist")
	assert.NoError(t, os.Mkdir(dist, 0755))
	assert.NoError(t, os.Mkdir(filepath.Join(dist, "mybin"), 0755))
	var binPath = filepath.Join(dist, "mybin", "mybin")
	_, err = os.Create(binPath)
	assert.NoError(t, err)
	var ctx = context.New(config.Project{
		ProjectName: "mybin",
		Dist:        dist,
		NFPM: config.NFPM{
			Bindir:      "/usr/bin",
			Formats:     []string{"deb", "rpm"},
			Description: "Some description",
			License:     "MIT",
			Maintainer:  "me@me",
			Vendor:      "asdf",
			Homepage:    "https://goreleaser.github.io",
			NFPMOverridables: config.NFPMOverridables{
				NameTemplate: defaultNameTemplate,
				Dependencies: []string{"make"},
				Recommends:   []string{"svn"},
				Suggests:     []string{"bzr"},
				Conflicts:    []string{"git"},
				EmptyFolders: []string{"/var/log/foobar"},
				Files: map[string]string{
					"./testdata/testfile.txt": "/usr/share/testfile.txt",
				},
				ConfigFiles: map[string]string{
					"./testdata/testfile.txt": "/etc/nope.conf",
				},
			},
			Overrides: map[string]config.NFPMOverridables{
				"rpm": {
					ConfigFiles: map[string]string{
						"./testdata/testfile.txt": "/etc/nope-rpm.conf",
					},
				},
			},
		},
	})
	ctx.Git = context.GitInfo{CurrentTag: "v1.0.0"}
	for _, goos := range []string{"linux", "darwin"} {
		for _, goarch := range []string{"amd64", "386"} {
			ctx.Artifacts.Add(artifact.Artifact{
				Name:   "mybin",
				Path:   binPath,
				Goarch: goarch,
				Goos:   goos,
				Type:   artifact.Binary,
			})
		}
	}
	assert.NoError(t, Pipe{}.Run(ctx))
	assert.Len(t, ctx.Config.NFPM.Files, 1, "should not modify the config file list")
}

func TestInvalidNameTemplate(t *testing.T) {
	var ctx = &context.Context{
		Parallelism: runtime.NumCPU(),
		Artifacts:   artifact.New(),
		Config: config.Project{
			NFPM: config.NFPM{
				NFPMOverridables: config.NFPMOverridables{NameTemplate: "{{.Foo}"},
				Formats:          []string{"deb"},
			},
		},
	}
	ctx.Artifacts.Add(artifact.Artifact{
		Name:   "mybin",
		Goos:   "linux",
		Goarch: "amd64",
		Type:   artifact.Binary,
	})
	assert.Contains(t, Pipe{}.Run(ctx).Error(), `template: {{.Foo}:1: unexpected "}" in operand`)
}

func TestCreateFileDoesntExist(t *testing.T) {
	folder, err := ioutil.TempDir("", "archivetest")
	assert.NoError(t, err)
	var dist = filepath.Join(folder, "dist")
	assert.NoError(t, os.Mkdir(dist, 0755))
	assert.NoError(t, os.Mkdir(filepath.Join(dist, "mybin"), 0755))
	var ctx = context.New(config.Project{
		Dist:        dist,
		ProjectName: "asd",
		NFPM: config.NFPM{
			Formats: []string{"deb", "rpm"},
			NFPMOverridables: config.NFPMOverridables{
				Files: map[string]string{
					"testdata/testfile.txt": "/var/lib/test/testfile.txt",
				},
			},
		},
	})
	ctx.Git = context.GitInfo{
		CurrentTag: "v1.2.3",
	}
	ctx.Artifacts.Add(artifact.Artifact{
		Name:   "mybin",
		Path:   filepath.Join(dist, "mybin", "mybin"),
		Goos:   "linux",
		Goarch: "amd64",
		Type:   artifact.Binary,
	})
	assert.Contains(t, Pipe{}.Run(ctx).Error(), `dist/mybin/mybin: file does not exist`)
}

func TestInvalidConfig(t *testing.T) {
	folder, err := ioutil.TempDir("", "archivetest")
	assert.NoError(t, err)
	var dist = filepath.Join(folder, "dist")
	assert.NoError(t, os.Mkdir(dist, 0755))
	assert.NoError(t, os.Mkdir(filepath.Join(dist, "mybin"), 0755))
	var ctx = context.New(config.Project{
		Dist: dist,
		NFPM: config.NFPM{
			Formats: []string{"deb"},
		},
	})
	ctx.Artifacts.Add(artifact.Artifact{
		Name:   "mybin",
		Path:   filepath.Join(dist, "mybin", "mybin"),
		Goos:   "linux",
		Goarch: "amd64",
		Type:   artifact.Binary,
	})
	assert.Contains(t, Pipe{}.Run(ctx).Error(), `invalid nfpm config: package name cannot be empty`)
}

func TestDefault(t *testing.T) {
	var ctx = &context.Context{
		Config: config.Project{
			NFPM: config.NFPM{},
		},
	}
	assert.NoError(t, Pipe{}.Default(ctx))
	assert.Equal(t, "/usr/local/bin", ctx.Config.NFPM.Bindir)
	assert.Equal(t, defaultNameTemplate, ctx.Config.NFPM.NameTemplate)
}

func TestDefaultSet(t *testing.T) {
	var ctx = &context.Context{
		Config: config.Project{
			NFPM: config.NFPM{
				Bindir: "/bin",
				NFPMOverridables: config.NFPMOverridables{
					NameTemplate: "foo",
				},
			},
		},
	}
	assert.NoError(t, Pipe{}.Default(ctx))
	assert.Equal(t, "/bin", ctx.Config.NFPM.Bindir)
	assert.Equal(t, "foo", ctx.Config.NFPM.NameTemplate)
}

func TestOverrides(t *testing.T) {
	var ctx = &context.Context{
		Config: config.Project{
			NFPM: config.NFPM{
				Bindir: "/bin",
				NFPMOverridables: config.NFPMOverridables{
					NameTemplate: "foo",
				},
				Overrides: map[string]config.NFPMOverridables{
					"deb": {
						NameTemplate: "bar",
					},
				},
			},
		},
	}
	assert.NoError(t, Pipe{}.Default(ctx))
	merged, err := mergeOverrides(ctx, "deb")
	assert.NoError(t, err)
	assert.Equal(t, "/bin", ctx.Config.NFPM.Bindir)
	assert.Equal(t, "foo", ctx.Config.NFPM.NameTemplate)
	assert.Equal(t, "bar", ctx.Config.NFPM.Overrides["deb"].NameTemplate)
	assert.Equal(t, "bar", merged.NameTemplate)
}
<commit_msg>test: name template overrides on nfpm pipe<commit_after>package nfpm

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"testing"

	"github.com/goreleaser/goreleaser/config"
	"github.com/goreleaser/goreleaser/context"
	"github.com/goreleaser/goreleaser/internal/artifact"
	"github.com/goreleaser/goreleaser/internal/testlib"
	"github.com/stretchr/testify/assert"
)

func TestDescription(t *testing.T) {
	assert.NotEmpty(t, Pipe{}.String())
}

func TestRunPipeNoFormats(t *testing.T) {
	var ctx = &context.Context{
		Git: context.GitInfo{
			CurrentTag: "v1.0.0",
		},
		Config:      config.Project{},
		Parallelism: runtime.NumCPU(),
	}
	testlib.AssertSkipped(t, Pipe{}.Run(ctx))
}

func TestRunPipeInvalidFormat(t *testing.T) {
	var ctx = context.New(config.Project{
		ProjectName: "nope",
		NFPM: config.NFPM{
			Bindir:  "/usr/bin",
			Formats: []string{"nope"},
			NFPMOverridables: config.NFPMOverridables{
				NameTemplate: defaultNameTemplate,
				Files:        map[string]string{},
			},
		},
	})
	ctx.Git = context.GitInfo{
		CurrentTag: "v1.2.3",
	}
	for _, goos := range []string{"linux", "darwin"} {
		for _, goarch := range []string{"amd64", "386"} {
			ctx.Artifacts.Add(artifact.Artifact{
				Name:   "mybin",
				Path:   "whatever",
				Goarch: goarch,
				Goos:   goos,
				Type:   artifact.Binary,
			})
		}
	}
	assert.Contains(t, Pipe{}.Run(ctx).Error(), `no packager registered for the format nope`)
}

func TestRunPipe(t *testing.T) {
	folder, err := ioutil.TempDir("", "archivetest")
	assert.NoError(t, err)
	var dist = filepath.Join(folder, "dist")
	assert.NoError(t, os.Mkdir(dist, 0755))
	assert.NoError(t, os.Mkdir(filepath.Join(dist, "mybin"), 0755))
	var binPath = filepath.Join(dist, "mybin", "mybin")
	_, err = os.Create(binPath)
	assert.NoError(t, err)
	var ctx = context.New(config.Project{
		ProjectName: "mybin",
		Dist:        dist,
		NFPM: config.NFPM{
			Bindir:      "/usr/bin",
			Formats:     []string{"deb", "rpm"},
			Description: "Some description",
			License:     "MIT",
			Maintainer:  "me@me",
			Vendor:      "asdf",
			Homepage:    "https://goreleaser.github.io",
			NFPMOverridables: config.NFPMOverridables{
				NameTemplate: defaultNameTemplate,
				Dependencies: []string{"make"},
				Recommends:   []string{"svn"},
				Suggests:     []string{"bzr"},
				Conflicts:    []string{"git"},
				EmptyFolders: []string{"/var/log/foobar"},
				Files: map[string]string{
					"./testdata/testfile.txt": "/usr/share/testfile.txt",
				},
				ConfigFiles: map[string]string{
					"./testdata/testfile.txt": "/etc/nope.conf",
				},
				Replacements: map[string]string{
					"linux": "Tux",
				},
			},
			Overrides: map[string]config.NFPMOverridables{
				"rpm": {
					ConfigFiles: map[string]string{
						"./testdata/testfile.txt": "/etc/nope-rpm.conf",
					},
				},
			},
		},
	})
	ctx.Version = "1.0.0"
	ctx.Git = context.GitInfo{CurrentTag: "v1.0.0"}
	for _, goos := range []string{"linux", "darwin"} {
		for _, goarch := range []string{"amd64", "386"} {
			ctx.Artifacts.Add(artifact.Artifact{
				Name:   "mybin",
				Path:   binPath,
				Goarch: goarch,
				Goos:   goos,
				Type:   artifact.Binary,
			})
		}
	}
	assert.NoError(t, Pipe{}.Run(ctx))
	var packages = ctx.Artifacts.Filter(artifact.ByType(artifact.LinuxPackage)).List()
	assert.Len(t, packages, 4)
	for _, pkg := range packages {
		assert.Contains(t, pkg.Name, "mybin_1.0.0_Tux_", "linux should have been replaced by Tux")
	}
	assert.Len(t, ctx.Config.NFPM.Files, 1, "should not modify the config file list")
}

func TestInvalidNameTemplate(t *testing.T) {
	var ctx = &context.Context{
		Parallelism: runtime.NumCPU(),
		Artifacts:   artifact.New(),
		Config: config.Project{
			NFPM: config.NFPM{
				NFPMOverridables: config.NFPMOverridables{NameTemplate: "{{.Foo}"},
				Formats:          []string{"deb"},
			},
		},
	}
	ctx.Artifacts.Add(artifact.Artifact{
		Name:   "mybin",
		Goos:   "linux",
		Goarch: "amd64",
		Type:   artifact.Binary,
	})
	assert.Contains(t, Pipe{}.Run(ctx).Error(), `template: {{.Foo}:1: unexpected "}" in operand`)
}

func TestCreateFileDoesntExist(t *testing.T) {
	folder, err := ioutil.TempDir("", "archivetest")
	assert.NoError(t, err)
	var dist = filepath.Join(folder, "dist")
	assert.NoError(t, os.Mkdir(dist, 0755))
	assert.NoError(t, os.Mkdir(filepath.Join(dist, "mybin"), 0755))
	var ctx = context.New(config.Project{
		Dist:        dist,
		ProjectName: "asd",
		NFPM: config.NFPM{
			Formats: []string{"deb", "rpm"},
			NFPMOverridables: config.NFPMOverridables{
				Files: map[string]string{
					"testdata/testfile.txt": "/var/lib/test/testfile.txt",
				},
			},
		},
	})
	ctx.Git = context.GitInfo{
		CurrentTag: "v1.2.3",
	}
	ctx.Artifacts.Add(artifact.Artifact{
		Name:   "mybin",
		Path:   filepath.Join(dist, "mybin", "mybin"),
		Goos:   "linux",
		Goarch: "amd64",
		Type:   artifact.Binary,
	})
	assert.Contains(t, Pipe{}.Run(ctx).Error(), `dist/mybin/mybin: file does not exist`)
}

func TestInvalidConfig(t *testing.T) {
	folder, err := ioutil.TempDir("", "archivetest")
	assert.NoError(t, err)
	var dist = filepath.Join(folder, "dist")
	assert.NoError(t, os.Mkdir(dist, 0755))
	assert.NoError(t, os.Mkdir(filepath.Join(dist, "mybin"), 0755))
	var ctx = context.New(config.Project{
		Dist: dist,
		NFPM: config.NFPM{
			Formats: []string{"deb"},
		},
	})
	ctx.Artifacts.Add(artifact.Artifact{
		Name:   "mybin",
		Path:   filepath.Join(dist, "mybin", "mybin"),
		Goos:   "linux",
		Goarch: "amd64",
		Type:   artifact.Binary,
	})
	assert.Contains(t, Pipe{}.Run(ctx).Error(), `invalid nfpm config: package name cannot be empty`)
}

func TestDefault(t *testing.T) {
	var ctx = &context.Context{
		Config: config.Project{
			NFPM: config.NFPM{},
		},
	}
	assert.NoError(t, Pipe{}.Default(ctx))
	assert.Equal(t, "/usr/local/bin", ctx.Config.NFPM.Bindir)
	assert.Equal(t, defaultNameTemplate, ctx.Config.NFPM.NameTemplate)
}

func TestDefaultSet(t *testing.T) {
	var ctx = &context.Context{
		Config: config.Project{
			NFPM: config.NFPM{
				Bindir: "/bin",
				NFPMOverridables: config.NFPMOverridables{
					NameTemplate: "foo",
				},
			},
		},
	}
	assert.NoError(t, Pipe{}.Default(ctx))
	assert.Equal(t, "/bin", ctx.Config.NFPM.Bindir)
	assert.Equal(t, "foo", ctx.Config.NFPM.NameTemplate)
}

func TestOverrides(t *testing.T) {
	var ctx = &context.Context{
		Config: config.Project{
			NFPM: config.NFPM{
				Bindir: "/bin",
				NFPMOverridables: config.NFPMOverridables{
					NameTemplate: "foo",
				},
				Overrides: map[string]config.NFPMOverridables{
					"deb": {
						NameTemplate: "bar",
					},
				},
			},
		},
	}
	assert.NoError(t, Pipe{}.Default(ctx))
	merged, err := mergeOverrides(ctx, "deb")
	assert.NoError(t, err)
	assert.Equal(t, "/bin", ctx.Config.NFPM.Bindir)
	assert.Equal(t, "foo", ctx.Config.NFPM.NameTemplate)
	assert.Equal(t, "bar", ctx.Config.NFPM.Overrides["deb"].NameTemplate)
	assert.Equal(t, "bar", merged.NameTemplate)
}
<|endoftext|>


<commit_before>package sign

import (
	"bytes"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"

	"github.com/goreleaser/goreleaser/internal/artifact"

	"github.com/goreleaser/goreleaser/config"
	"github.com/goreleaser/goreleaser/context"

	"github.com/stretchr/testify/assert"
)

func TestDescription(t *testing.T) {
	assert.NotEmpty(t, Pipe{}.String())
}

func TestSignDefault(t *testing.T) {
	ctx := &context.Context{}
	Pipe{}.Default(ctx)
	assert.Equal(t, ctx.Config.Sign.Cmd, "gpg")
	assert.Equal(t, ctx.Config.Sign.Signature, "${artifact}.sig")
	assert.Equal(t, ctx.Config.Sign.Args, []string{"--output", "$signature", "--detach-sig", "$artifact"})
	assert.Equal(t, ctx.Config.Sign.Artifacts, "none")
}

func TestSignDisabled(t *testing.T) {
	ctx := &context.Context{}
	ctx.Config.Sign.Artifacts = "none"
	err := Pipe{}.Run(ctx)
	assert.EqualError(t, err, "artifact signing disabled")
}

func TestSignInvalidArtifacts(t *testing.T) {
	ctx := &context.Context{}
	ctx.Config.Sign.Artifacts = "foo"
	err := Pipe{}.Run(ctx)
	assert.EqualError(t, err, "invalid list of artifacts to sign: foo")
}

func TestSignArtifacts(t *testing.T) {
	// fix permission on keyring dir to suppress warning about insecure permissions
	assert.NoError(t, os.Chmod(keyring, 0700))

	tests := []struct {
		desc       string
		ctx        *context.Context
		signatures []string
	}{
		{
			desc: "sign all artifacts",
			ctx: context.New(
				config.Project{
					Sign: config.Sign{Artifacts: "all"},
				},
			),
			signatures: []string{"artifact1.sig", "artifact2.sig", "checksum.sig"},
		},
		{
			desc: "sign only checksums",
			ctx: context.New(
				config.Project{
					Sign: config.Sign{Artifacts: "checksum"},
				},
			),
			signatures: []string{"checksum.sig"},
		},
	}

	for _, tt := range tests {
		t.Run(tt.desc, func(t *testing.T) {
			testSign(t, tt.ctx, tt.signatures)
		})
	}
}

const keyring = "testdata/gnupg"
const user = "nopass"

func testSign(t *testing.T, ctx *context.Context, signatures []string) {
	// create temp dir for file and signature
	tmpdir, err := ioutil.TempDir("", "goreleaser")
	assert.NoError(t, err)
	defer os.RemoveAll(tmpdir)

	ctx.Config.Dist = tmpdir

	// create some fake artifacts
	for _, f := range []string{"artifact1", "artifact2", "checksum"} {
		file := filepath.Join(tmpdir, f)
		assert.NoError(t, ioutil.WriteFile(file, []byte("foo"), 0644))
	}
	ctx.Artifacts.Add(artifact.Artifact{
		Name: "artifact1",
		Path: filepath.Join(tmpdir, "artifact1"),
		Type: artifact.UploadableArchive,
	})
	ctx.Artifacts.Add(artifact.Artifact{
		Name: "artifact2",
		Path: filepath.Join(tmpdir, "artifact2"),
		Type: artifact.UploadableArchive,
	})
	ctx.Artifacts.Add(artifact.Artifact{
		Name: "checksum",
		Path: filepath.Join(tmpdir, "checksum"),
		Type: artifact.Checksum,
	})

	// configure the pipeline
	// make sure we are using the test keyring
	assert.NoError(t, Pipe{}.Default(ctx))
	ctx.Config.Sign.Args = append([]string{"--homedir", keyring}, ctx.Config.Sign.Args...)

	// run the pipeline
	assert.NoError(t, Pipe{}.Run(ctx))

	// verify that only the artifacts and the signatures are in the dist dir
	files, err := ioutil.ReadDir(tmpdir)
	assert.NoError(t, err)
	gotFiles := []string{}
	for _, f := range files {
		gotFiles = append(gotFiles, f.Name())
	}

	assert.Contains(t, gotFiles, signatures)

	// verify the signatures
	for _, sig := range signatures {
		verifySignature(t, ctx, sig)
	}
}

func verifySignature(t *testing.T, ctx *context.Context, sig string) {
	artifact := sig[:len(sig)-len(".sig")]

	// verify signature was made with key for usesr 'nopass'
	cmd := exec.Command("gpg", "--homedir", keyring, "--verify", filepath.Join(ctx.Config.Dist, sig), filepath.Join(ctx.Config.Dist, artifact))
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Log(string(out))
		t.Fatal("verify: ", err)
	}

	// check if the signature matches the user we expect to do this properly we
	// might need to have either separate keyrings or export the key from the
	// keyring before we do the verification. For now we punt and look in the
	// output.
	if !bytes.Contains(out, []byte(user)) {
		t.Fatalf("signature is not from %s", user)
	}
}
<commit_msg>test: fix sign pipe tests (hopefully)<commit_after>package sign

import (
	"bytes"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"testing"

	"github.com/goreleaser/goreleaser/internal/artifact"

	"github.com/goreleaser/goreleaser/config"
	"github.com/goreleaser/goreleaser/context"

	"github.com/stretchr/testify/assert"
)

func TestDescription(t *testing.T) {
	assert.NotEmpty(t, Pipe{}.String())
}

func TestSignDefault(t *testing.T) {
	ctx := &context.Context{}
	Pipe{}.Default(ctx)
	assert.Equal(t, ctx.Config.Sign.Cmd, "gpg")
	assert.Equal(t, ctx.Config.Sign.Signature, "${artifact}.sig")
	assert.Equal(t, ctx.Config.Sign.Args, []string{"--output", "$signature", "--detach-sig", "$artifact"})
	assert.Equal(t, ctx.Config.Sign.Artifacts, "none")
}

func TestSignDisabled(t *testing.T) {
	ctx := &context.Context{}
	ctx.Config.Sign.Artifacts = "none"
	err := Pipe{}.Run(ctx)
	assert.EqualError(t, err, "artifact signing disabled")
}

func TestSignInvalidArtifacts(t *testing.T) {
	ctx := &context.Context{}
	ctx.Config.Sign.Artifacts = "foo"
	err := Pipe{}.Run(ctx)
	assert.EqualError(t, err, "invalid list of artifacts to sign: foo")
}

func TestSignArtifacts(t *testing.T) {
	// fix permission on keyring dir to suppress warning about insecure permissions
	assert.NoError(t, os.Chmod(keyring, 0700))

	tests := []struct {
		desc       string
		ctx        *context.Context
		signatures []string
	}{
		{
			desc: "sign all artifacts",
			ctx: context.New(
				config.Project{
					Sign: config.Sign{Artifacts: "all"},
				},
			),
			signatures: []string{"artifact1.sig", "artifact2.sig", "checksum.sig"},
		},
		{
			desc: "sign only checksums",
			ctx: context.New(
				config.Project{
					Sign: config.Sign{Artifacts: "checksum"},
				},
			),
			signatures: []string{"checksum.sig"},
		},
	}

	for _, tt := range tests {
		t.Run(tt.desc, func(t *testing.T) {
			testSign(t, tt.ctx, tt.signatures)
		})
	}
}

const keyring = "testdata/gnupg"
const user = "nopass"

func testSign(t *testing.T, ctx *context.Context, signatures []string) {
	// create temp dir for file and signature
	tmpdir, err := ioutil.TempDir("", "goreleaser")
	assert.NoError(t, err)
	defer os.RemoveAll(tmpdir)

	ctx.Config.Dist = tmpdir

	// create some fake artifacts
	var artifacts = []string{"artifact1", "artifact2", "checksum"}
	for _, f := range artifacts {
		file := filepath.Join(tmpdir, f)
		assert.NoError(t, ioutil.WriteFile(file, []byte("foo"), 0644))
	}
	ctx.Artifacts.Add(artifact.Artifact{
		Name: "artifact1",
		Path: filepath.Join(tmpdir, "artifact1"),
		Type: artifact.UploadableArchive,
	})
	ctx.Artifacts.Add(artifact.Artifact{
		Name: "artifact2",
		Path: filepath.Join(tmpdir, "artifact2"),
		Type: artifact.UploadableArchive,
	})
	ctx.Artifacts.Add(artifact.Artifact{
		Name: "checksum",
		Path: filepath.Join(tmpdir, "checksum"),
		Type: artifact.Checksum,
	})

	// configure the pipeline
	// make sure we are using the test keyring
	assert.NoError(t, Pipe{}.Default(ctx))
	ctx.Config.Sign.Args = append([]string{"--homedir", keyring}, ctx.Config.Sign.Args...)

	// run the pipeline
	assert.NoError(t, Pipe{}.Run(ctx))

	// verify that only the artifacts and the signatures are in the dist dir
	files, err := ioutil.ReadDir(tmpdir)
	assert.NoError(t, err)
	gotFiles := []string{}
	for _, f := range files {
		gotFiles = append(gotFiles, f.Name())
	}
	wantFiles := append(artifacts, signatures...)
	sort.Strings(wantFiles)
	assert.Equal(t, wantFiles, gotFiles)

	// verify the signatures
	for _, sig := range signatures {
		verifySignature(t, ctx, sig)
	}

	var signArtifacts []string
	for _, sig := range ctx.Artifacts.Filter(artifact.ByType(artifact.Signature)).List() {
		signArtifacts = append(signArtifacts, sig.Name)
	}
	// check signature is an artifact
	assert.Equal(t, signArtifacts, signatures)
}

func verifySignature(t *testing.T, ctx *context.Context, sig string) {
	artifact := sig[:len(sig)-len(".sig")]

	// verify signature was made with key for usesr 'nopass'
	cmd := exec.Command("gpg", "--homedir", keyring, "--verify", filepath.Join(ctx.Config.Dist, sig), filepath.Join(ctx.Config.Dist, artifact))
	out, err := cmd.CombinedOutput()
	assert.NoError(t, err)

	// check if the signature matches the user we expect to do this properly we
	// might need to have either separate keyrings or export the key from the
	// keyring before we do the verification. For now we punt and look in the
	// output.
	if !bytes.Contains(out, []byte(user)) {
		t.Fatalf("signature is not from %s", user)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package install installs the v1 monolithic api, making it available as an
// option to all of the API encoding/decoding machinery.
package install

import (
	"fmt"
	"strings"

	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api/latest"
	"k8s.io/kubernetes/pkg/util/sets"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/meta"
	"k8s.io/kubernetes/pkg/api/registered"
	apiutil "k8s.io/kubernetes/pkg/api/util"
	"k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/runtime"
)

// userResources is a group of resources mostly used by a kubectl user
var userResources = []string{"rc", "svc", "pods", "pvc"}

const importPrefix = "k8s.io/kubernetes/pkg/api"

var accessor = meta.NewAccessor()

func init() {
	groupMeta, err := latest.RegisterGroup("")
	if err != nil {
		glog.V(4).Infof("%v", err)
		return
	}
	// Use the first API version in the list of registered versions as the latest.
	registeredGroupVersions := registered.GroupVersionsForGroup("")
	groupVersion := registeredGroupVersions[0]
	*groupMeta = latest.GroupMeta{
		GroupVersion: groupVersion,
		Group:        apiutil.GetGroup(groupVersion),
		Version:      apiutil.GetVersion(groupVersion),
		Codec:        runtime.CodecFor(api.Scheme, groupVersion),
	}
	var versions []string
	var groupVersions []string
	for i := len(registeredGroupVersions) - 1; i >= 0; i-- {
		versions = append(versions, apiutil.GetVersion(registeredGroupVersions[i]))
		groupVersions = append(groupVersions, registeredGroupVersions[i])
	}
	groupMeta.Versions = versions
	groupMeta.GroupVersions = groupVersions

	groupMeta.SelfLinker = runtime.SelfLinker(accessor)

	// the list of kinds that are scoped at the root of the api hierarchy
	// if a kind is not enumerated here, it is assumed to have a namespace scope
	// the list of kinds that are scoped at the root of the api hierarchy
	// if a kind is not enumerated here, it is assumed to have a namespace scope
	rootScoped := sets.NewString(
		"Node",
		"Minion",
		"Namespace",
		"PersistentVolume",
		"ComponentStatus",
	)

	// these kinds should be excluded from the list of resources
	ignoredKinds := sets.NewString(
		"ListOptions",
		"DeleteOptions",
		"Status",
		"PodLogOptions",
		"PodExecOptions",
		"PodAttachOptions",
		"PodProxyOptions",
		"ThirdPartyResource",
		"ThirdPartyResourceData",
		"ThirdPartyResourceList")

	mapper := api.NewDefaultRESTMapper("", versions, interfacesFor, importPrefix, ignoredKinds, rootScoped)
	// setup aliases for groups of resources
	mapper.AddResourceAlias("all", userResources...)
	groupMeta.RESTMapper = mapper
	api.RegisterRESTMapper(groupMeta.RESTMapper)
	groupMeta.InterfacesFor = interfacesFor
}

// InterfacesFor returns the default Codec and ResourceVersioner for a given version
// string, or an error if the version is not known.
func interfacesFor(version string) (*meta.VersionInterfaces, error) {
	switch version {
	case "v1":
		return &meta.VersionInterfaces{
			Codec:            v1.Codec,
			ObjectConvertor:  api.Scheme,
			MetadataAccessor: accessor,
		}, nil
	default:
		{
			g, _ := latest.Group("")
			return nil, fmt.Errorf("unsupported storage version: %s (valid: %s)", version, strings.Join(g.Versions, ", "))
		}
	}
}
<commit_msg>Remove useless codes about Minion kind<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package install installs the v1 monolithic api, making it available as an
// option to all of the API encoding/decoding machinery.
package install

import (
	"fmt"
	"strings"

	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api/latest"
	"k8s.io/kubernetes/pkg/util/sets"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/meta"
	"k8s.io/kubernetes/pkg/api/registered"
	apiutil "k8s.io/kubernetes/pkg/api/util"
	"k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/runtime"
)

// userResources is a group of resources mostly used by a kubectl user
var userResources = []string{"rc", "svc", "pods", "pvc"}

const importPrefix = "k8s.io/kubernetes/pkg/api"

var accessor = meta.NewAccessor()

func init() {
	groupMeta, err := latest.RegisterGroup("")
	if err != nil {
		glog.V(4).Infof("%v", err)
		return
	}
	// Use the first API version in the list of registered versions as the latest.
	registeredGroupVersions := registered.GroupVersionsForGroup("")
	groupVersion := registeredGroupVersions[0]
	*groupMeta = latest.GroupMeta{
		GroupVersion: groupVersion,
		Group:        apiutil.GetGroup(groupVersion),
		Version:      apiutil.GetVersion(groupVersion),
		Codec:        runtime.CodecFor(api.Scheme, groupVersion),
	}
	var versions []string
	var groupVersions []string
	for i := len(registeredGroupVersions) - 1; i >= 0; i-- {
		versions = append(versions, apiutil.GetVersion(registeredGroupVersions[i]))
		groupVersions = append(groupVersions, registeredGroupVersions[i])
	}
	groupMeta.Versions = versions
	groupMeta.GroupVersions = groupVersions

	groupMeta.SelfLinker = runtime.SelfLinker(accessor)

	// the list of kinds that are scoped at the root of the api hierarchy
	// if a kind is not enumerated here, it is assumed to have a namespace scope
	rootScoped := sets.NewString(
		"Node",
		"Namespace",
		"PersistentVolume",
		"ComponentStatus",
	)

	// these kinds should be excluded from the list of resources
	ignoredKinds := sets.NewString(
		"ListOptions",
		"DeleteOptions",
		"Status",
		"PodLogOptions",
		"PodExecOptions",
		"PodAttachOptions",
		"PodProxyOptions",
		"ThirdPartyResource",
		"ThirdPartyResourceData",
		"ThirdPartyResourceList")

	mapper := api.NewDefaultRESTMapper("", versions, interfacesFor, importPrefix, ignoredKinds, rootScoped)
	// setup aliases for groups of resources
	mapper.AddResourceAlias("all", userResources...)
	groupMeta.RESTMapper = mapper
	api.RegisterRESTMapper(groupMeta.RESTMapper)
	groupMeta.InterfacesFor = interfacesFor
}

// InterfacesFor returns the default Codec and ResourceVersioner for a given version
// string, or an error if the version is not known.
func interfacesFor(version string) (*meta.VersionInterfaces, error) {
	switch version {
	case "v1":
		return &meta.VersionInterfaces{
			Codec:            v1.Codec,
			ObjectConvertor:  api.Scheme,
			MetadataAccessor: accessor,
		}, nil
	default:
		{
			g, _ := latest.Group("")
			return nil, fmt.Errorf("unsupported storage version: %s (valid: %s)", version, strings.Join(g.Versions, ", "))
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cluster

import (
	"fmt"
	"math"
	"time"

	"github.com/coreos/etcd-operator/pkg/util/etcdutil"
	"github.com/coreos/etcd-operator/pkg/util/k8sutil"

	"github.com/coreos/etcd-operator/pkg/util/retryutil"
	"github.com/pborman/uuid"
)

func (c *Cluster) addOneSelfHostedMember() error {
	c.status.AppendScalingUpCondition(c.members.Size(), c.cluster.Spec.Size)

	newMemberName := etcdutil.CreateMemberName(c.cluster.Metadata.Name, c.memberCounter)
	c.memberCounter++

	peerURL := "http://$(MY_POD_IP):2380"
	initialCluster := append(c.members.PeerURLPairs(), newMemberName+"="+peerURL)

	pod := k8sutil.NewSelfHostedEtcdPod(newMemberName, initialCluster, c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner())
	pod = k8sutil.PodWithAddMemberInitContainer(pod, c.members.ClientURLs(), newMemberName, []string{peerURL}, c.cluster.Spec)

	_, err := c.config.KubeCli.CoreV1().Pods(c.cluster.Metadata.Namespace).Create(pod)
	if err != nil {
		return err
	}
	// wait for the new pod to start and add itself into the etcd cluster.
	oldN := c.members.Size()
	err = retryutil.Retry(5*time.Second, math.MaxInt64, func() (bool, error) {
		err = c.updateMembers(c.members)
		if err != nil {
			c.logger.Errorf("add self hosted member: fail to update members: %v", err)
			if err == errUnexpectedUnreadyMember {
				return false, nil
			}
			return false, err
		}
		if c.members.Size() > oldN {
			return true, nil
		}
		return false, nil
	})
	if err != nil {
		return err
	}

	c.logger.Infof("added a self-hosted member (%s)", newMemberName)
	return nil
}

func (c *Cluster) newSelfHostedSeedMember() error {
	newMemberName := fmt.Sprintf("%s-%04d", c.cluster.Metadata.Name, c.memberCounter)
	c.memberCounter++
	initialCluster := []string{newMemberName + "=http://$(MY_POD_IP):2380"}

	pod := k8sutil.NewSelfHostedEtcdPod(newMemberName, initialCluster, c.cluster.Metadata.Name, "new", uuid.New(), c.cluster.Spec, c.cluster.AsOwner())
	_, err := k8sutil.CreateAndWaitPod(c.config.KubeCli, c.cluster.Metadata.Namespace, pod, 30*time.Second)
	if err != nil {
		return err
	}

	c.logger.Infof("self-hosted cluster created with seed member (%s)", newMemberName)
	return nil
}

func (c *Cluster) migrateBootMember() error {
	endpoint := c.cluster.Spec.SelfHosted.BootMemberClientEndpoint

	c.logger.Infof("migrating boot member (%s)", endpoint)

	resp, err := etcdutil.ListMembers([]string{endpoint})
	if err != nil {
		return fmt.Errorf("failed to list members from boot member (%v)", err)
	}
	if len(resp.Members) != 1 {
		return fmt.Errorf("boot cluster contains more than one member")
	}
	bootMember := resp.Members[0]

	initialCluster := make([]string, 0)
	for _, purl := range bootMember.PeerURLs {
		initialCluster = append(initialCluster, fmt.Sprintf("%s=%s", bootMember.Name, purl))
	}

	// create the  member inside Kubernetes for migration
	newMemberName := fmt.Sprintf("%s-%04d", c.cluster.Metadata.Name, c.memberCounter)
	c.memberCounter++

	peerURL := "http://$(MY_POD_IP):2380"
	initialCluster = append(initialCluster, newMemberName+"="+peerURL)

	pod := k8sutil.NewSelfHostedEtcdPod(newMemberName, initialCluster, c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner())
	pod = k8sutil.PodWithAddMemberInitContainer(pod, []string{endpoint}, newMemberName, []string{peerURL}, c.cluster.Spec)
	pod, err = k8sutil.CreateAndWaitPod(c.config.KubeCli, c.cluster.Metadata.Namespace, pod, 30*time.Second)
	if err != nil {
		return err
	}

	go func() {
		// TODO: a shorter timeout?
		delay := 60 * time.Second
		c.logger.Infof("wait %v before removing the boot member", delay)
		time.Sleep(delay)

		err = etcdutil.RemoveMember([]string{"http://" + pod.Status.PodIP + ":2379"}, bootMember.ID)
		if err != nil {
			c.logger.Errorf("boot member migration: failed to remove the boot member (%v)", err)
		}
	}()

	c.logger.Infof("self-hosted cluster created with boot member (%s)", endpoint)

	return nil
}
<commit_msg>selfHosted: add commnets<commit_after>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cluster

import (
	"fmt"
	"math"
	"time"

	"github.com/coreos/etcd-operator/pkg/util/etcdutil"
	"github.com/coreos/etcd-operator/pkg/util/k8sutil"

	"github.com/coreos/etcd-operator/pkg/util/retryutil"
	"github.com/pborman/uuid"
)

func (c *Cluster) addOneSelfHostedMember() error {
	c.status.AppendScalingUpCondition(c.members.Size(), c.cluster.Spec.Size)

	newMemberName := etcdutil.CreateMemberName(c.cluster.Metadata.Name, c.memberCounter)
	c.memberCounter++

	peerURL := "http://$(MY_POD_IP):2380"
	initialCluster := append(c.members.PeerURLPairs(), newMemberName+"="+peerURL)

	pod := k8sutil.NewSelfHostedEtcdPod(newMemberName, initialCluster, c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner())
	pod = k8sutil.PodWithAddMemberInitContainer(pod, c.members.ClientURLs(), newMemberName, []string{peerURL}, c.cluster.Spec)

	_, err := c.config.KubeCli.CoreV1().Pods(c.cluster.Metadata.Namespace).Create(pod)
	if err != nil {
		return err
	}
	// wait for the new pod to start and add itself into the etcd cluster.
	oldN := c.members.Size()
	err = retryutil.Retry(5*time.Second, math.MaxInt64, func() (bool, error) {
		err = c.updateMembers(c.members)
		if err != nil {
			c.logger.Errorf("add self hosted member: fail to update members: %v", err)
			if err == errUnexpectedUnreadyMember {
				return false, nil
			}
			return false, err
		}
		if c.members.Size() > oldN {
			return true, nil
		}
		return false, nil
	})
	if err != nil {
		return err
	}

	c.logger.Infof("added a self-hosted member (%s)", newMemberName)
	return nil
}

func (c *Cluster) newSelfHostedSeedMember() error {
	newMemberName := fmt.Sprintf("%s-%04d", c.cluster.Metadata.Name, c.memberCounter)
	c.memberCounter++
	initialCluster := []string{newMemberName + "=http://$(MY_POD_IP):2380"}

	pod := k8sutil.NewSelfHostedEtcdPod(newMemberName, initialCluster, c.cluster.Metadata.Name, "new", uuid.New(), c.cluster.Spec, c.cluster.AsOwner())
	_, err := k8sutil.CreateAndWaitPod(c.config.KubeCli, c.cluster.Metadata.Namespace, pod, 30*time.Second)
	if err != nil {
		return err
	}

	c.logger.Infof("self-hosted cluster created with seed member (%s)", newMemberName)
	return nil
}

func (c *Cluster) migrateBootMember() error {
	endpoint := c.cluster.Spec.SelfHosted.BootMemberClientEndpoint

	c.logger.Infof("migrating boot member (%s)", endpoint)

	resp, err := etcdutil.ListMembers([]string{endpoint})
	if err != nil {
		return fmt.Errorf("failed to list members from boot member (%v)", err)
	}
	if len(resp.Members) != 1 {
		return fmt.Errorf("boot cluster contains more than one member")
	}
	bootMember := resp.Members[0]

	initialCluster := make([]string, 0)
	for _, purl := range bootMember.PeerURLs {
		initialCluster = append(initialCluster, fmt.Sprintf("%s=%s", bootMember.Name, purl))
	}

	// create the  member inside Kubernetes for migration
	newMemberName := fmt.Sprintf("%s-%04d", c.cluster.Metadata.Name, c.memberCounter)
	c.memberCounter++

	peerURL := "http://$(MY_POD_IP):2380"
	initialCluster = append(initialCluster, newMemberName+"="+peerURL)

	pod := k8sutil.NewSelfHostedEtcdPod(newMemberName, initialCluster, c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner())
	pod = k8sutil.PodWithAddMemberInitContainer(pod, []string{endpoint}, newMemberName, []string{peerURL}, c.cluster.Spec)
	pod, err = k8sutil.CreateAndWaitPod(c.config.KubeCli, c.cluster.Metadata.Namespace, pod, 30*time.Second)
	if err != nil {
		return err
	}

	go func() {
		// TODO: a shorter timeout?
		// Waiting here for cluster to get stable:
		// - etcd data are replicated;
		// - cluster TPR state has switched to "Running"
		delay := 60 * time.Second
		c.logger.Infof("wait %v before removing the boot member", delay)
		time.Sleep(delay)

		err = etcdutil.RemoveMember([]string{"http://" + pod.Status.PodIP + ":2379"}, bootMember.ID)
		if err != nil {
			c.logger.Errorf("boot member migration: failed to remove the boot member (%v)", err)
		}
	}()

	c.logger.Infof("self-hosted cluster created with boot member (%s)", endpoint)

	return nil
}
<|endoftext|>


<commit_before>package errors_test

import (
	"fmt"

	"github.com/vardius/go-api-boilerplate/pkg/errors"
)

func ExampleNew() {
	err := errors.New("example")

	fmt.Printf("%s\n", err)

	// Output:
	// example
}

func ExampleWrap() {
	subErr := errors.New("example")
	err := errors.Wrap(subErr)

	fmt.Printf("%s\n", err)

	// Output:
	// example
}
<commit_msg>Update example tests to not print output<commit_after>package errors_test

import (
	"fmt"

	"github.com/vardius/go-api-boilerplate/pkg/errors"
)

func ExampleNew() {
	err := errors.New("example")

	fmt.Printf("%s\n", err)
}

func ExampleWrap() {
	subErr := errors.New("example")
	err := errors.Wrap(subErr)

	fmt.Printf("%s\n", err)
}
<|endoftext|>


<commit_before>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package id3v2

import (
	"io/ioutil"
	"testing"
)

func BenchmarkParseAllFrames(b *testing.B) {
	if err := resetMP3Tag(); err != nil {
		b.Fatal("Error while reseting mp3 file:", err)
	}
	b.ResetTimer()

	for n := 0; n < b.N; n++ {
		tag, err := Open(mp3Name, defaultOpts)
		if tag == nil || err != nil {
			b.Fatal("Error while opening mp3 file:", err)
		}
		if err = tag.Close(); err != nil {
			b.Error("Error while closing a tag:", err)
		}
	}
}

func BenchmarkParseAndWriteCommonCase(b *testing.B) {
	frontCover, err := ioutil.ReadFile(frontCoverName)
	if err != nil {
		b.Error("Error while reading front cover file")
	}
	b.ResetTimer()

	// We use b.N+1, because in first iteration we just reset tag
	// and set common frames. Also timer will be resetted.
	for n := 0; n < b.N+1; n++ {
		tag, err := Open(mp3Name, defaultOpts)
		if tag == nil || err != nil {
			b.Fatal("Error while opening mp3 file:", err)
		}
		defer tag.Close()

		// Delete all frames in first iteration
		if n == 0 {
			tag.DeleteAllFrames()
		}

		tag.SetTitle("Title")
		tag.SetArtist("Artist")
		tag.SetYear("2016")

		// Set front cover
		pic := PictureFrame{
			Encoding:    ENUTF8,
			MimeType:    "image/jpeg",
			PictureType: PTFrontCover,
			Description: "Front cover",
			Picture:     frontCover,
		}
		tag.AddAttachedPicture(pic)

		if err = tag.Save(); err != nil {
			b.Error("Error while saving a tag:", err)
		}

		// Reset timer because we just resetting tag in first iteration
		if n == 0 {
			b.ResetTimer()
		}
	}
}

func BenchmarkParseAndWriteManyFrames(b *testing.B) {
	frontCover, err := ioutil.ReadFile(frontCoverName)
	if err != nil {
		b.Error("Error while reading front cover file")
	}
	b.ResetTimer()

	// We use b.N+1, because in first iteration we just reset tag
	// and set many frames. Also timer will be resetted.
	for n := 0; n < b.N+1; n++ {
		tag, err := Open(mp3Name, defaultOpts)
		if tag == nil || err != nil {
			b.Fatal("Error while opening mp3 file:", err)
		}
		defer tag.Close()

		// Delete all frames in first iteration.
		if n == 0 {
			tag.DeleteAllFrames()
		}

		tag.SetTitle("Title")
		tag.SetArtist("Artist")
		tag.SetAlbum("Album")
		tag.SetYear("2016")
		tag.SetGenre("Genre")

		// Set front cover.
		pic := PictureFrame{
			Encoding:    ENUTF8,
			MimeType:    "image/jpeg",
			PictureType: PTFrontCover,
			Description: "Front cover",
			Picture:     frontCover,
		}
		tag.AddAttachedPicture(pic)

		// Set USLT.
		uslt := UnsynchronisedLyricsFrame{
			Encoding:          ENUTF8,
			Language:          "eng",
			ContentDescriptor: "Content descriptor",
			Lyrics:            "bogem/id3v2",
		}
		tag.AddUnsynchronisedLyricsFrame(uslt)

		// Set comment.
		comm := CommentFrame{
			Encoding:    ENUTF8,
			Language:    "eng",
			Description: "Short description",
			Text:        "The actual text",
		}
		tag.AddCommentFrame(comm)

		if err = tag.Save(); err != nil {
			b.Error("Error while saving a tag:", err)
		}

		// Reset timer because we just reset file in first iteration.
		if n == 0 {
			b.ResetTimer()
		}
	}
}

func BenchmarkReuseTag(b *testing.B) {
	frontCover, err := ioutil.ReadFile(frontCoverName)
	if err != nil {
		b.Error("Error while reading front cover file")
	}
	tag := NewEmptyTag()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tag.DeleteAllFrames()

		tag.SetTitle("Title")
		tag.SetArtist("Artist")
		tag.SetAlbum("Album")
		tag.SetYear("2016")
		tag.SetGenre("Genre")

		// Set front cover.
		pic := PictureFrame{
			Encoding:    ENUTF8,
			MimeType:    "image/jpeg",
			PictureType: PTFrontCover,
			Description: "Front cover",
			Picture:     frontCover,
		}
		tag.AddAttachedPicture(pic)

		// Set USLT.
		uslt := UnsynchronisedLyricsFrame{
			Encoding:          ENUTF8,
			Language:          "eng",
			ContentDescriptor: "Content descriptor",
			Lyrics:            "bogem/id3v2",
		}
		tag.AddUnsynchronisedLyricsFrame(uslt)

		// Set comment.
		comm := CommentFrame{
			Encoding:    ENUTF8,
			Language:    "eng",
			Description: "Short description",
			Text:        "The actual text",
		}
		tag.AddCommentFrame(comm)
	}
}
<commit_msg>Add BenchmarkParseArtistAndTitle<commit_after>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package id3v2

import (
	"io/ioutil"
	"testing"
)

func BenchmarkParseAllFrames(b *testing.B) {
	if err := resetMP3Tag(); err != nil {
		b.Fatal("Error while reseting mp3 file:", err)
	}
	b.ResetTimer()

	for n := 0; n < b.N; n++ {
		tag, err := Open(mp3Name, defaultOpts)
		if tag == nil || err != nil {
			b.Fatal("Error while opening mp3 file:", err)
		}
		if err = tag.Close(); err != nil {
			b.Error("Error while closing a tag:", err)
		}
	}
}

func BenchmarkParseArtistAndTitle(b *testing.B) {
	if err := resetMP3Tag(); err != nil {
		b.Fatal("Error while reseting mp3 file:", err)
	}
	b.ResetTimer()

	for n := 0; n < b.N; n++ {
		tag, err := Open(mp3Name, Options{Parse: true, ParseFrames: []string{"Artist", "Title"}})
		if tag == nil || err != nil {
			b.Fatal("Error while opening mp3 file:", err)
		}
		if err = tag.Close(); err != nil {
			b.Error("Error while closing a tag:", err)
		}
	}
}

func BenchmarkParseAndWriteCommonCase(b *testing.B) {
	frontCover, err := ioutil.ReadFile(frontCoverName)
	if err != nil {
		b.Error("Error while reading front cover file")
	}
	b.ResetTimer()

	// We use b.N+1, because in first iteration we just reset tag
	// and set common frames. Also timer will be resetted.
	for n := 0; n < b.N+1; n++ {
		tag, err := Open(mp3Name, defaultOpts)
		if tag == nil || err != nil {
			b.Fatal("Error while opening mp3 file:", err)
		}
		defer tag.Close()

		// Delete all frames in first iteration
		if n == 0 {
			tag.DeleteAllFrames()
		}

		tag.SetTitle("Title")
		tag.SetArtist("Artist")
		tag.SetYear("2016")

		// Set front cover
		pic := PictureFrame{
			Encoding:    ENUTF8,
			MimeType:    "image/jpeg",
			PictureType: PTFrontCover,
			Description: "Front cover",
			Picture:     frontCover,
		}
		tag.AddAttachedPicture(pic)

		if err = tag.Save(); err != nil {
			b.Error("Error while saving a tag:", err)
		}

		// Reset timer because we just resetting tag in first iteration
		if n == 0 {
			b.ResetTimer()
		}
	}
}

func BenchmarkParseAndWriteManyFrames(b *testing.B) {
	frontCover, err := ioutil.ReadFile(frontCoverName)
	if err != nil {
		b.Error("Error while reading front cover file")
	}
	b.ResetTimer()

	// We use b.N+1, because in first iteration we just reset tag
	// and set many frames. Also timer will be resetted.
	for n := 0; n < b.N+1; n++ {
		tag, err := Open(mp3Name, defaultOpts)
		if tag == nil || err != nil {
			b.Fatal("Error while opening mp3 file:", err)
		}
		defer tag.Close()

		// Delete all frames in first iteration.
		if n == 0 {
			tag.DeleteAllFrames()
		}

		tag.SetTitle("Title")
		tag.SetArtist("Artist")
		tag.SetAlbum("Album")
		tag.SetYear("2016")
		tag.SetGenre("Genre")

		// Set front cover.
		pic := PictureFrame{
			Encoding:    ENUTF8,
			MimeType:    "image/jpeg",
			PictureType: PTFrontCover,
			Description: "Front cover",
			Picture:     frontCover,
		}
		tag.AddAttachedPicture(pic)

		// Set USLT.
		uslt := UnsynchronisedLyricsFrame{
			Encoding:          ENUTF8,
			Language:          "eng",
			ContentDescriptor: "Content descriptor",
			Lyrics:            "bogem/id3v2",
		}
		tag.AddUnsynchronisedLyricsFrame(uslt)

		// Set comment.
		comm := CommentFrame{
			Encoding:    ENUTF8,
			Language:    "eng",
			Description: "Short description",
			Text:        "The actual text",
		}
		tag.AddCommentFrame(comm)

		if err = tag.Save(); err != nil {
			b.Error("Error while saving a tag:", err)
		}

		// Reset timer because we just reset file in first iteration.
		if n == 0 {
			b.ResetTimer()
		}
	}
}

func BenchmarkReuseTag(b *testing.B) {
	frontCover, err := ioutil.ReadFile(frontCoverName)
	if err != nil {
		b.Error("Error while reading front cover file")
	}
	tag := NewEmptyTag()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tag.DeleteAllFrames()

		tag.SetTitle("Title")
		tag.SetArtist("Artist")
		tag.SetAlbum("Album")
		tag.SetYear("2016")
		tag.SetGenre("Genre")

		// Set front cover.
		pic := PictureFrame{
			Encoding:    ENUTF8,
			MimeType:    "image/jpeg",
			PictureType: PTFrontCover,
			Description: "Front cover",
			Picture:     frontCover,
		}
		tag.AddAttachedPicture(pic)

		// Set USLT.
		uslt := UnsynchronisedLyricsFrame{
			Encoding:          ENUTF8,
			Language:          "eng",
			ContentDescriptor: "Content descriptor",
			Lyrics:            "bogem/id3v2",
		}
		tag.AddUnsynchronisedLyricsFrame(uslt)

		// Set comment.
		comm := CommentFrame{
			Encoding:    ENUTF8,
			Language:    "eng",
			Description: "Short description",
			Text:        "The actual text",
		}
		tag.AddCommentFrame(comm)
	}
}
<|endoftext|>


<commit_before>package myslave

import (
	"context"
	"fmt"
	"time"

	"github.com/funkygao/dbus/pkg/model"
	log "github.com/funkygao/log4go"
	uuid "github.com/satori/go.uuid"
	"github.com/siddontang/go-mysql/mysql"
	"github.com/siddontang/go-mysql/replication"
)

// StopReplication stops the slave and do necessary cleanups.
func (m *MySlave) StopReplication() {
	if m.isMaster {
		m.r.Close()

		if err := m.p.Flush(); err != nil {
			log.Error("[%s] flush: %s", m.name, err)
		}
	}

	m.leaveCluster()
	m.isMaster = false
}

// StartReplication start the mysql binlog replication.
// TODO graceful shutdown
// TODO GTID
func (m *MySlave) StartReplication(ready chan struct{}) {
	m.joinClusterAndBecomeMaster() // block till become master
	m.isMaster = true

	m.rowsEvent = make(chan *model.RowsEvent, m.c.Int("event_buffer_len", 100))
	m.errors = make(chan error, 1)

	m.r = replication.NewBinlogSyncer(&replication.BinlogSyncerConfig{
		ServerID:        uint32(m.c.Int("server_id", 137)), // 137 unique enough?
		Flavor:          m.c.String("flavor", mysql.MySQLFlavor),
		Host:            m.host,
		Port:            m.port,
		User:            m.user,
		Password:        m.passwd,
		SemiSyncEnabled: m.c.Bool("semi_sync", false),
	})

	// resume replication position from the checkpoint
	file, offset, err := m.p.Committed()
	if err != nil {
		close(ready)
		m.emitFatalError(err)
		return
	}

	var syncer *replication.BinlogStreamer
	if m.GTID {
		// SELECT @@gtid_mode
		// SHOW GLOBAL VARIABLES LIKE 'SERVER_UUID'
		var masterUuid uuid.UUID
		// 07c93cd7-a7d3-12a5-94e1-a0369a7c3790:1-313225133
		set, _ := mysql.ParseMysqlGTIDSet(fmt.Sprintf("%s:%d-%d", masterUuid.String(), 1, 2))
		syncer, err = m.r.StartSyncGTID(set)
	} else {
		syncer, err = m.r.StartSync(mysql.Position{
			Name: file,
			Pos:  offset,
		})
	}

	if err != nil {
		// unrecoverable err encountered
		// e,g.
		// ERROR 1045 (28000): Access denied for user 'xx'@'1.1.1.1'
		close(ready)
		m.emitFatalError(err)
		return
	}

	close(ready)
	log.Trace("[%s] ready to receive binlog stream", m.name)

	timeout := time.Second
	maxTimeout := time.Minute
	for {
		// what if the conn broken?
		// 2017/02/08 08:31:39 binlogsyncer.go:529: [info] receive EOF packet, retry ReadPacket
		// 2017/02/08 08:31:39 binlogsyncer.go:484: [error] connection was bad
		// 2017/02/08 08:31:40 binlogsyncer.go:446: [info] begin to re-sync from (mysql.000003, 2492)
		// 2017/02/08 08:31:40 binlogsyncer.go:130: [info] register slave for master server localhost:3306
		// 2017/02/08 08:31:40 binlogsyncer.go:502: [error] retry sync err: dial tcp [::1]:3306: getsockopt: connection refused, wait 1s and retry again
		// 2017/02/08 08:31:42 binlogsyncer.go:446: [info] begin to re-sync from (mysql.000003, 2492)
		// 2017/02/08 08:31:42 binlogsyncer.go:130: [info] register slave for master server localhost:3306
		// 2017/02/08 08:31:42 binlogsyncer.go:502: [error] retry sync err: dial tcp [::1]:3306: getsockopt: connection refused, wait 1s and retry again
		//
		// or invalid position sent?
		// 2017/02/08 08:42:09 binlogstreamer.go:47: [error] close sync with err: ERROR 1236 (HY000): Client requested master to start replication from position > file size; the first event 'mysql.000004' at 2274, the last event read from '/usr/local/var/mysql.000004' at 4, the last byte read from '/usr/local/var/mysql.000004' at 4.
		//
		// [02/08/17 17:39:20] [EROR] ( mysqlbinlog.go:64) backoff 5s: invalid table id 2413, no correspond table map event
		// mysql> SHOW BINLOG EVENTS IN '8623306-bin.008517' FROM 117752230 limit 10;
		// +--------------------+-----------+------------+-----------+-------------+-----------------------------------------------------------------------------+
		// | Log_name           | Pos       | Event_type | Server_id | End_log_pos | Info                                                                        |
		// +--------------------+-----------+------------+-----------+-------------+-----------------------------------------------------------------------------+
		// | 8623306-bin.008517 | 117732471 | Query      |  81413306 |   117732534 | BEGIN                                                                       |
		// | 8623306-bin.008517 | 117732534 | Table_map  |  81413306 |   117732590 | table_id: 2925 (zabbix.history)                                             |
		// | 8623306-bin.008517 | 117732590 | Write_rows |  81413306 |   117735750 | table_id: 2925 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117735750 | Table_map  |  81413306 |   117735810 | table_id: 2968 (zabbix.history_uint)                                        |
		// | 8623306-bin.008517 | 117735810 | Write_rows |  81413306 |   117744020 | table_id: 2968                                                              |
		// | 8623306-bin.008517 | 117744020 | Write_rows |  81413306 |   117752230 | table_id: 2968                                                              |
		// | 8623306-bin.008517 | 117752230 | Write_rows |  81413306 |   117755340 | table_id: 2968 flags: STMT_END_F                                            | <- checkpoint
		// | 8623306-bin.008517 | 117755340 | Xid        |  81413306 |   117755371 | COMMIT /* xid=2243095633 */                                                 |
		// | 8623306-bin.008517 | 117755371 | Gtid       |  81413306 |   117755419 | SET @@SESSION.GTID_NEXT= 'a0f36bb1-fdbb-11e5-8413-a0369f7c3bb4:16393135780' |
		// | 8623306-bin.008517 | 117755419 | Query      |  81413306 |   117755482 | BEGIN                                                                       |
		// | 8623306-bin.008517 | 117755482 | Table_map  |  81413306 |   117755538 | table_id: 2925 (zabbix.history)                                             |
		// | 8623306-bin.008517 | 117755538 | Write_rows |  81413306 |   117758123 | table_id: 2925 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117758123 | Table_map  |  81413306 |   117758183 | table_id: 2968 (zabbix.history_uint)                                        |
		// | 8623306-bin.008517 | 117758183 | Write_rows |  81413306 |   117766393 | table_id: 2968                                                              |
		// | 8623306-bin.008517 | 117766393 | Write_rows |  81413306 |   117772728 | table_id: 2968 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117772728 | Xid        |  81413306 |   117772759 | COMMIT /* xid=2243095636 */                                                 |
		// +--------------------+-----------+------------+-----------+-------------+-----------------------------------------------------------------------------+
		ctx, cancel := context.WithTimeout(context.Background(), timeout)
		ev, err := syncer.GetEvent(ctx)
		cancel()

		if err != nil {
			if err == context.DeadlineExceeded {
				if timeout < maxTimeout {
					// backoff
					timeout *= 2
				}

				continue
			}

			// TODO try out all the cases
			m.emitFatalError(err)
			return
		}

		m.m.Events.Mark(1)

		// insert into tbtest values(1) will trigger the following events:
		// QueryEvent    BEGIN, Log position: 4800
		// TableMapEvent Schema: mydb, Table: tbtest, TableID: 81, Log position: 4844
		// RowsEvent     Values: 0:1, TableID: 81, Log position: 4884
		// XIDEvent      XID: 356, Log position: 4915 // COMMMIT
		switch e := ev.Event.(type) {
		case *replication.RotateEvent:
			// e,g.
			// Position: 4
			// Next log name: mysql.000002
			file = string(e.NextLogName)
			log.Trace("[%s] rotate to (%s, %d)", m.name, file, e.Position)

		case *replication.RowsEvent:
			m.m.TPS.Mark(1)
			m.m.Lag.Update(time.Now().Unix() - int64(ev.Header.Timestamp))
			m.handleRowsEvent(file, ev.Header, e)

		case *replication.QueryEvent:
			// DDL comes this way
			// e,g. create table y(id int)
			// e,g. BEGIN
			// e,g. flush tables

		case *replication.XIDEvent:
			// e,g. xid: 1293

		case *replication.FormatDescriptionEvent:
			// Version: 4
			// Server version: 5.6.23-log
			// Checksum algorithm: 1

		case *replication.TableMapEvent:
			// e,g.
			// TableID: 170
			// TableID size: 6
			// Flags: 1
			// Schema: test
			// Table: y
			// Column count: 1
			// Column type:
			// 00000000  03
			// NULL bitmap:
			// 00000000  01
			//
			// https://github.com/siddontang/go-mysql/issues/48
			// If we have processed TableMapEvent and then restart, the following
			// rows event may miss the table id when we sync from last saved position.

		case *replication.GenericEvent:
			// known misc types of event

		case *replication.GTIDEvent:
			// only GTID mode will recv this event

		default:
			log.Warn("[%s] unexpected event: %+v", m.name, e)
		}
	}

}

func (m *MySlave) emitFatalError(err error) {
	m.errors <- err
	if m.r != nil {
		m.r.Close()
	}
}
<commit_msg>add comment<commit_after>package myslave

import (
	"context"
	"fmt"
	"time"

	"github.com/funkygao/dbus/pkg/model"
	log "github.com/funkygao/log4go"
	uuid "github.com/satori/go.uuid"
	"github.com/siddontang/go-mysql/mysql"
	"github.com/siddontang/go-mysql/replication"
)

// StopReplication stops the slave and do necessary cleanups.
func (m *MySlave) StopReplication() {
	if m.isMaster {
		m.r.Close()

		if err := m.p.Flush(); err != nil {
			log.Error("[%s] flush: %s", m.name, err)
		}
	}

	m.leaveCluster()
	m.isMaster = false
}

// StartReplication start the mysql binlog replication.
// TODO graceful shutdown
// TODO GTID
func (m *MySlave) StartReplication(ready chan struct{}) {
	m.joinClusterAndBecomeMaster() // block till become master
	m.isMaster = true

	m.rowsEvent = make(chan *model.RowsEvent, m.c.Int("event_buffer_len", 100))
	m.errors = make(chan error, 1)

	m.r = replication.NewBinlogSyncer(&replication.BinlogSyncerConfig{
		ServerID:        uint32(m.c.Int("server_id", 137)), // 137 unique enough?
		Flavor:          m.c.String("flavor", mysql.MySQLFlavor),
		Host:            m.host,
		Port:            m.port,
		User:            m.user,
		Password:        m.passwd,
		SemiSyncEnabled: m.c.Bool("semi_sync", false),
	})

	// resume replication position from the checkpoint
	file, offset, err := m.p.Committed()
	if err != nil {
		close(ready)
		m.emitFatalError(err)
		return
	}

	var syncer *replication.BinlogStreamer
	if m.GTID {
		// SELECT @@gtid_mode
		// SHOW GLOBAL VARIABLES LIKE 'SERVER_UUID'
		var masterUuid uuid.UUID
		// 07c93cd7-a7d3-12a5-94e1-a0369a7c3790:1-313225133
		set, _ := mysql.ParseMysqlGTIDSet(fmt.Sprintf("%s:%d-%d", masterUuid.String(), 1, 2))
		syncer, err = m.r.StartSyncGTID(set)
	} else {
		syncer, err = m.r.StartSync(mysql.Position{
			Name: file,
			Pos:  offset,
		})
	}

	if err != nil {
		// unrecoverable err encountered
		// e,g.
		// ERROR 1045 (28000): Access denied for user 'xx'@'1.1.1.1'
		close(ready)
		m.emitFatalError(err)
		return
	}

	close(ready)
	log.Trace("[%s] ready to receive binlog stream", m.name)

	timeout := time.Second
	maxTimeout := time.Minute
	for {
		// what if the conn broken?
		// 2017/02/08 08:31:39 binlogsyncer.go:529: [info] receive EOF packet, retry ReadPacket
		// 2017/02/08 08:31:39 binlogsyncer.go:484: [error] connection was bad
		// 2017/02/08 08:31:40 binlogsyncer.go:446: [info] begin to re-sync from (mysql.000003, 2492)
		// 2017/02/08 08:31:40 binlogsyncer.go:130: [info] register slave for master server localhost:3306
		// 2017/02/08 08:31:40 binlogsyncer.go:502: [error] retry sync err: dial tcp [::1]:3306: getsockopt: connection refused, wait 1s and retry again
		// 2017/02/08 08:31:42 binlogsyncer.go:446: [info] begin to re-sync from (mysql.000003, 2492)
		// 2017/02/08 08:31:42 binlogsyncer.go:130: [info] register slave for master server localhost:3306
		// 2017/02/08 08:31:42 binlogsyncer.go:502: [error] retry sync err: dial tcp [::1]:3306: getsockopt: connection refused, wait 1s and retry again
		//
		// or invalid position sent?
		// 2017/02/08 08:42:09 binlogstreamer.go:47: [error] close sync with err: ERROR 1236 (HY000): Client requested master to start replication from position > file size; the first event 'mysql.000004' at 2274, the last event read from '/usr/local/var/mysql.000004' at 4, the last byte read from '/usr/local/var/mysql.000004' at 4.
		//
		// [02/08/17 17:39:20] [EROR] ( mysqlbinlog.go:64) backoff 5s: invalid table id 2413, no correspond table map event
		// mysql> SHOW BINLOG EVENTS IN '8623306-bin.008517' FROM 117752230 limit 10;
		// +--------------------+-----------+------------+-----------+-------------+-----------------------------------------------------------------------------+
		// | Log_name           | Pos       | Event_type | Server_id | End_log_pos | Info                                                                        |
		// +--------------------+-----------+------------+-----------+-------------+-----------------------------------------------------------------------------+
		// | 8623306-bin.008517 | 117732471 | Query      |  81413306 |   117732534 | BEGIN                                                                       |
		// | 8623306-bin.008517 | 117732534 | Table_map  |  81413306 |   117732590 | table_id: 2925 (zabbix.history)                                             |
		// | 8623306-bin.008517 | 117732590 | Write_rows |  81413306 |   117735750 | table_id: 2925 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117735750 | Table_map  |  81413306 |   117735810 | table_id: 2968 (zabbix.history_uint)                                        |
		// | 8623306-bin.008517 | 117735810 | Write_rows |  81413306 |   117744020 | table_id: 2968                                                              |
		// | 8623306-bin.008517 | 117744020 | Write_rows |  81413306 |   117752230 | table_id: 2968                                                              |
		// | 8623306-bin.008517 | 117752230 | Write_rows |  81413306 |   117755340 | table_id: 2968 flags: STMT_END_F                                            | <- checkpoint
		// | 8623306-bin.008517 | 117755340 | Xid        |  81413306 |   117755371 | COMMIT /* xid=2243095633 */                                                 |
		// | 8623306-bin.008517 | 117755371 | Gtid       |  81413306 |   117755419 | SET @@SESSION.GTID_NEXT= 'a0f36bb1-fdbb-11e5-8413-a0369f7c3bb4:16393135780' |
		// | 8623306-bin.008517 | 117755419 | Query      |  81413306 |   117755482 | BEGIN                                                                       |
		// | 8623306-bin.008517 | 117755482 | Table_map  |  81413306 |   117755538 | table_id: 2925 (zabbix.history)                                             |
		// | 8623306-bin.008517 | 117755538 | Write_rows |  81413306 |   117758123 | table_id: 2925 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117758123 | Table_map  |  81413306 |   117758183 | table_id: 2968 (zabbix.history_uint)                                        |
		// | 8623306-bin.008517 | 117758183 | Write_rows |  81413306 |   117766393 | table_id: 2968                                                              |
		// | 8623306-bin.008517 | 117766393 | Write_rows |  81413306 |   117772728 | table_id: 2968 flags: STMT_END_F                                            |
		// | 8623306-bin.008517 | 117772728 | Xid        |  81413306 |   117772759 | COMMIT /* xid=2243095636 */                                                 |
		// +--------------------+-----------+------------+-----------+-------------+-----------------------------------------------------------------------------+
		ctx, cancel := context.WithTimeout(context.Background(), timeout)
		ev, err := syncer.GetEvent(ctx)
		cancel()

		if err != nil {
			if err == context.DeadlineExceeded {
				if timeout < maxTimeout {
					// backoff
					timeout *= 2
				}

				continue
			}

			// TODO try out all the cases
			m.emitFatalError(err)
			return
		}

		m.m.Events.Mark(1)

		// insert into tbtest values(1) will trigger the following events:
		// QueryEvent    BEGIN, Log position: 4800
		// TableMapEvent Schema: mydb, Table: tbtest, TableID: 81, Log position: 4844
		// RowsEvent     Values: 0:1, TableID: 81, Log position: 4884
		// XIDEvent      XID: 356, Log position: 4915 // COMMMIT
		switch e := ev.Event.(type) {
		case *replication.RotateEvent:
			// e,g.
			// Position: 4
			// Next log name: mysql.000002
			file = string(e.NextLogName)
			// e.Position is End_log_pos(i,e. next log position)
			log.Trace("[%s] rotate to (%s, %d)", m.name, file, e.Position)

		case *replication.RowsEvent:
			m.m.TPS.Mark(1)
			m.m.Lag.Update(time.Now().Unix() - int64(ev.Header.Timestamp))
			m.handleRowsEvent(file, ev.Header, e)

		case *replication.QueryEvent:
			// DDL comes this way
			// e,g. create table y(id int)
			// e,g. BEGIN
			// e,g. flush tables

		case *replication.XIDEvent:
			// e,g. COMMIT /* xid=403013040 */

		case *replication.FormatDescriptionEvent:
			// Version: 4
			// Server version: 5.6.23-log
			// Checksum algorithm: 1

		case *replication.TableMapEvent:
			// e,g.
			// TableID: 170
			// TableID size: 6
			// Flags: 1
			// Schema: test
			// Table: y
			// Column count: 1
			// Column type:
			// 00000000  03
			// NULL bitmap:
			// 00000000  01
			//
			// https://github.com/siddontang/go-mysql/issues/48
			// If we have processed TableMapEvent and then restart, the following
			// rows event may miss the table id when we sync from last saved position.

		case *replication.GenericEvent:
			// known misc types of event

		case *replication.GTIDEvent:
			// only GTID mode will recv this event

		default:
			log.Warn("[%s] unexpected event: %+v", m.name, e)
		}
	}

}

func (m *MySlave) emitFatalError(err error) {
	m.errors <- err
	if m.r != nil {
		m.r.Close()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	httpgzip "github.com/daaku/go.httpgzip"
	"github.com/urfave/cli/v2"

	"crypto/ecdsa"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"math/big"
	"net"
	"strings"
	"time"
)

func logHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.Method, r.RequestURI)
		h.ServeHTTP(w, r)
	})
}

func customHeadersHandler(h http.Handler, headersMap map[string]string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		for key, value := range headersMap {
			w.Header().Set(key, value)
		}
		h.ServeHTTP(w, r)
	})
}

func publicKey(priv interface{}) interface{} {
	switch k := priv.(type) {
	case *rsa.PrivateKey:
		return &k.PublicKey
	case *ecdsa.PrivateKey:
		return &k.PublicKey
	default:
		return nil
	}
}

func pemBlockForKey(priv interface{}) *pem.Block {
	switch k := priv.(type) {
	case *rsa.PrivateKey:
		return &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)}
	case *ecdsa.PrivateKey:
		b, err := x509.MarshalECPrivateKey(k)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Unable to marshal ECDSA private key: %v", err)
			os.Exit(2)
		}
		return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b}
	default:
		return nil
	}
}

// This is extracted from src/crypto/tls/generate_cert.go, keeping only the
// bare minimum needed to create a usable cert for localhost development.
func generateSelfSignedCert() {
	host := "localhost"
	validFor := 365 * 24 * time.Hour
	rsaBits := 2048

	var priv interface{}
	var err error
	priv, err = rsa.GenerateKey(rand.Reader, rsaBits)

	if err != nil {
		log.Fatalf("failed to generate private key: %s", err)
	}

	notBefore := time.Now()
	notAfter := notBefore.Add(validFor)

	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		log.Fatalf("failed to generate serial number: %s", err)
	}

	template := x509.Certificate{
		SerialNumber: serialNumber,
		Subject: pkix.Name{
			Organization: []string{"Acme Co"},
		},
		NotBefore: notBefore,
		NotAfter:  notAfter,

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
	}

	hosts := strings.Split(host, ",")
	for _, h := range hosts {
		if ip := net.ParseIP(h); ip != nil {
			template.IPAddresses = append(template.IPAddresses, ip)
		} else {
			template.DNSNames = append(template.DNSNames, h)
		}
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, publicKey(priv), priv)
	if err != nil {
		log.Fatalf("Failed to create certificate: %s", err)
	}

	certOut, err := os.Create("cert.pem")
	if err != nil {
		log.Fatalf("failed to open cert.pem for writing: %s", err)
	}
	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
	certOut.Close()
	log.Print("written cert.pem\n")

	keyOut, err := os.OpenFile("key.pem", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		log.Print("failed to open key.pem for writing:", err)
		return
	}
	pem.Encode(keyOut, pemBlockForKey(priv))
	keyOut.Close()
	log.Print("written key.pem\n")
}

func main() {
	var gzip bool
	var port int
	var path string = "./"
	var logging bool
	var http2 bool
	var headerFlags cli.StringSlice
	var headersMap map[string]string
	var stop = make(chan os.Signal, 1)
	var err error
	var server *http.Server

	app := &cli.App{
		Name:    "serve",
		Usage:   "deliver content of current directory via http/https",
		Version: "1.1.1",

		Flags: []cli.Flag{
			&cli.BoolFlag{
				Name:        "gzip",
				Aliases:     []string{"g"},
				Usage:       "enable gzip encoding",
				Destination: &gzip,
			},
			&cli.IntFlag{
				Name:        "port",
				Aliases:     []string{"p"},
				Usage:       "specify port for listening",
				Value:       8080,
				Destination: &port,
			},
			&cli.BoolFlag{
				Name:        "logging",
				Aliases:     []string{"l"},
				Usage:       "enable logging output",
				Destination: &logging,
			},
			&cli.BoolFlag{
				Name:        "http2",
				Aliases:     []string{"2"},
				Usage:       "enable http2, this generates a self signed certificate, if one isn't already present; cert.pem, key.pem",
				Destination: &http2,
			},
			&cli.StringSliceFlag{
				Name:        "header",
				Aliases:     []string{"H"},
				Usage:       "custom header(s) to add to the response (can be repeated multiple times)",
				Destination: &headerFlags,
			},
		},

		Action: func(c *cli.Context) error {
			if c.NArg() > 0 {
				path = c.Args().Get(0)
			}

			handler := http.FileServer(http.Dir(path))

			if gzip {
				handler = httpgzip.NewHandler(handler)
			}

			if logging {
				handler = logHandler(handler)
			}

			headers := headerFlags.Value()
			if len(headers) > 0 {
				headersMap = make(map[string]string)
				for _, header := range headers {
					idx := strings.Index(header, ":")
					if idx < 0 {
						return fmt.Errorf("Invalid header: %s", header)
					}
					k := header[:idx]
					v := header[idx+1:]
					headersMap[strings.TrimSpace(k)] = strings.TrimSpace(v)
				}
				handler = customHeadersHandler(handler, headersMap)
			}

			server = &http.Server{
				Addr:    fmt.Sprintf(":%d", port),
				Handler: handler,
			}

			serveHttps := func() {
				if _, err := os.Stat("cert.pem"); os.IsNotExist(err) {
					generateSelfSignedCert()
				}

				err = server.ListenAndServeTLS("cert.pem", "key.pem")
				if err != nil {
					log.Fatalf("ListenAndServeTLS error: %s", err)
				}
			}

			serveHttp := func() {
				err = server.ListenAndServe()
				if err != nil {
					log.Fatalf("ListenAndServe error: %s", err)
				}
			}

			if http2 {
				go serveHttps()
			} else {
				go serveHttp()
			}

			log.Println(fmt.Sprintf("Serving content of %s on localhost:%v ...", path, port))

			signal.Notify(stop, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)

			<-stop

			log.Print("Stopping.")

			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
			defer cancel()
			if err := server.Shutdown(ctx); err != nil {
				log.Fatalf("%s", err)
			}

			return nil
		},
	}

	err = app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Bump version number<commit_after>package main

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	httpgzip "github.com/daaku/go.httpgzip"
	"github.com/urfave/cli/v2"

	"crypto/ecdsa"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"math/big"
	"net"
	"strings"
	"time"
)

func logHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.Method, r.RequestURI)
		h.ServeHTTP(w, r)
	})
}

func customHeadersHandler(h http.Handler, headersMap map[string]string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		for key, value := range headersMap {
			w.Header().Set(key, value)
		}
		h.ServeHTTP(w, r)
	})
}

func publicKey(priv interface{}) interface{} {
	switch k := priv.(type) {
	case *rsa.PrivateKey:
		return &k.PublicKey
	case *ecdsa.PrivateKey:
		return &k.PublicKey
	default:
		return nil
	}
}

func pemBlockForKey(priv interface{}) *pem.Block {
	switch k := priv.(type) {
	case *rsa.PrivateKey:
		return &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)}
	case *ecdsa.PrivateKey:
		b, err := x509.MarshalECPrivateKey(k)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Unable to marshal ECDSA private key: %v", err)
			os.Exit(2)
		}
		return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b}
	default:
		return nil
	}
}

// This is extracted from src/crypto/tls/generate_cert.go, keeping only the
// bare minimum needed to create a usable cert for localhost development.
func generateSelfSignedCert() {
	host := "localhost"
	validFor := 365 * 24 * time.Hour
	rsaBits := 2048

	var priv interface{}
	var err error
	priv, err = rsa.GenerateKey(rand.Reader, rsaBits)

	if err != nil {
		log.Fatalf("failed to generate private key: %s", err)
	}

	notBefore := time.Now()
	notAfter := notBefore.Add(validFor)

	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		log.Fatalf("failed to generate serial number: %s", err)
	}

	template := x509.Certificate{
		SerialNumber: serialNumber,
		Subject: pkix.Name{
			Organization: []string{"Acme Co"},
		},
		NotBefore: notBefore,
		NotAfter:  notAfter,

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
	}

	hosts := strings.Split(host, ",")
	for _, h := range hosts {
		if ip := net.ParseIP(h); ip != nil {
			template.IPAddresses = append(template.IPAddresses, ip)
		} else {
			template.DNSNames = append(template.DNSNames, h)
		}
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, publicKey(priv), priv)
	if err != nil {
		log.Fatalf("Failed to create certificate: %s", err)
	}

	certOut, err := os.Create("cert.pem")
	if err != nil {
		log.Fatalf("failed to open cert.pem for writing: %s", err)
	}
	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
	certOut.Close()
	log.Print("written cert.pem\n")

	keyOut, err := os.OpenFile("key.pem", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		log.Print("failed to open key.pem for writing:", err)
		return
	}
	pem.Encode(keyOut, pemBlockForKey(priv))
	keyOut.Close()
	log.Print("written key.pem\n")
}

func main() {
	var gzip bool
	var port int
	var path string = "./"
	var logging bool
	var http2 bool
	var headerFlags cli.StringSlice
	var headersMap map[string]string
	var stop = make(chan os.Signal, 1)
	var err error
	var server *http.Server

	app := &cli.App{
		Name:    "serve",
		Usage:   "deliver content of current directory via http/https",
		Version: "1.2.0",

		Flags: []cli.Flag{
			&cli.BoolFlag{
				Name:        "gzip",
				Aliases:     []string{"g"},
				Usage:       "enable gzip encoding",
				Destination: &gzip,
			},
			&cli.IntFlag{
				Name:        "port",
				Aliases:     []string{"p"},
				Usage:       "specify port for listening",
				Value:       8080,
				Destination: &port,
			},
			&cli.BoolFlag{
				Name:        "logging",
				Aliases:     []string{"l"},
				Usage:       "enable logging output",
				Destination: &logging,
			},
			&cli.BoolFlag{
				Name:        "http2",
				Aliases:     []string{"2"},
				Usage:       "enable http2, this generates a self signed certificate, if one isn't already present; cert.pem, key.pem",
				Destination: &http2,
			},
			&cli.StringSliceFlag{
				Name:        "header",
				Aliases:     []string{"H"},
				Usage:       "custom header(s) to add to the response (can be repeated multiple times)",
				Destination: &headerFlags,
			},
		},

		Action: func(c *cli.Context) error {
			if c.NArg() > 0 {
				path = c.Args().Get(0)
			}

			handler := http.FileServer(http.Dir(path))

			if gzip {
				handler = httpgzip.NewHandler(handler)
			}

			if logging {
				handler = logHandler(handler)
			}

			headers := headerFlags.Value()
			if len(headers) > 0 {
				headersMap = make(map[string]string)
				for _, header := range headers {
					idx := strings.Index(header, ":")
					if idx < 0 {
						return fmt.Errorf("Invalid header: %s", header)
					}
					k := header[:idx]
					v := header[idx+1:]
					headersMap[strings.TrimSpace(k)] = strings.TrimSpace(v)
				}
				handler = customHeadersHandler(handler, headersMap)
			}

			server = &http.Server{
				Addr:    fmt.Sprintf(":%d", port),
				Handler: handler,
			}

			serveHttps := func() {
				if _, err := os.Stat("cert.pem"); os.IsNotExist(err) {
					generateSelfSignedCert()
				}

				err = server.ListenAndServeTLS("cert.pem", "key.pem")
				if err != nil {
					log.Fatalf("ListenAndServeTLS error: %s", err)
				}
			}

			serveHttp := func() {
				err = server.ListenAndServe()
				if err != nil {
					log.Fatalf("ListenAndServe error: %s", err)
				}
			}

			if http2 {
				go serveHttps()
			} else {
				go serveHttp()
			}

			log.Println(fmt.Sprintf("Serving content of %s on localhost:%v ...", path, port))

			signal.Notify(stop, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)

			<-stop

			log.Print("Stopping.")

			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
			defer cancel()
			if err := server.Shutdown(ctx); err != nil {
				log.Fatalf("%s", err)
			}

			return nil
		},
	}

	err = app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"log"
	"math/rand"
	"net/http"
	"strconv"

	"github.com/stripe/sequins/blocks"
)

// serveKey is the entrypoint for incoming HTTP requests. It looks up the value
// locally, for, failing that, asks a peer that has it. If the request was
// already proxied to us, it is not proxied further.
func (vs *version) serveKey(w http.ResponseWriter, r *http.Request, key string) {
	// If we don't have any data for this version at all, that's a 404.
	if vs.numPartitions == 0 {
		vs.serveNotFound(w)
		return
	}

	partition, alternatePartition := blocks.KeyPartition(key, vs.numPartitions)
	bs := vs.getBlockStore()
	if bs != nil && (vs.hasPartition(partition) || vs.hasPartition(alternatePartition)) {
		record, err := bs.Get(key)
		if err != nil {
			vs.serveError(w, key, err)
			return
		}

		vs.serveLocal(w, key, record)
	} else if r.URL.Query().Get("proxy") == "" {
		vs.serveProxied(w, r, key, partition, alternatePartition)
	} else {
		vs.serveError(w, key, errProxiedIncorrectly)
	}
}

func (vs *version) serveLocal(w http.ResponseWriter, key string, record *blocks.Record) {
	if record == nil {
		vs.serveNotFound(w)
		return
	}

	defer record.Close()
	w.Header().Set(versionHeader, vs.name)
	w.Header().Set("Content-Length", strconv.FormatUint(record.ValueLen, 10))
	w.Header().Set("Last-Modified", vs.created.UTC().Format(http.TimeFormat))
	_, err := io.Copy(w, record)
	if err != nil {
		// We already wrote a 200 OK, so not much we can do here except log.
		log.Printf("Error streaming response for /%s/%s (version %s): %s", vs.db, key, vs.name)
	}
}

func (vs *version) serveProxied(w http.ResponseWriter, r *http.Request,
	key string, partition, alternatePartition int) {

	// Shuffle the peers, so we try them in a random order.
	// TODO: We don't want to blacklist nodes, but we can weight them lower
	peers := shuffle(vs.partitions.getPeers(partition))
	if len(peers) == 0 {
		log.Printf("No peers available for /%s/%s (version %s)", vs.db, key, vs.name)
		w.WriteHeader(http.StatusBadGateway)
		return
	}

	resp, peer, err := vs.proxy(r, peers)
	if err == nil && resp.StatusCode == 404 && alternatePartition != partition {
		log.Println("Trying alternate partition for pathological key", key)

		resp.Body.Close()
		alternatePeers := shuffle(vs.partitions.getPeers(alternatePartition))
		resp, peer, err = vs.proxy(r, alternatePeers)
	}

	if err == errNoAvailablePeers {
		// Either something is wrong with sharding, or all peers errored for some
		// other reason. 502
		log.Printf("No peers available for /%s/%s (version %s)", vs.db, key, vs.name)
		w.WriteHeader(http.StatusBadGateway)
		return
	} else if err == errProxyTimeout {
		// All of our peers failed us. 504.
		log.Printf("All peers timed out for /%s/%s (version %s)", vs.db, key, vs.name)
		w.WriteHeader(http.StatusGatewayTimeout)
		return
	} else if err != nil {
		// Some other error. 500.
		vs.serveError(w, key, err)
		return
	}

	// Proxying can produce inconsistent versions if something is broken. Use the
	// one the peer set.
	w.Header().Set(versionHeader, resp.Header.Get(versionHeader))
	w.Header().Set(proxyHeader, peer)
	w.Header().Set("Content-Length", resp.Header.Get("Content-Length"))
	w.Header().Set("Last-Modified", vs.created.UTC().Format(http.TimeFormat))
	w.WriteHeader(resp.StatusCode)

	// TODO: Apparently in 1.7 the client always asks for gzip by default. If our
	// client asks for gzip too, we should be able to pass through without
	// decompressing.
	defer resp.Body.Close()
	_, err = io.Copy(w, resp.Body)
	if err != nil {
		// We already wrote a 200 OK, so not much we can do here except log.
		log.Printf("Error copying response from peer for /%s/%s (version %s): %s", vs.db, key, vs.name)
	}
}

func (vs *version) serveNotFound(w http.ResponseWriter) {
	w.Header().Set(versionHeader, vs.name)
	w.WriteHeader(http.StatusNotFound)
}

func (vs *version) serveError(w http.ResponseWriter, key string, err error) {
	log.Printf("Error fetching value for /%s/%s: %s\n", vs.db, key, err)
	w.WriteHeader(http.StatusInternalServerError)
}

func shuffle(vs []string) []string {
	shuffled := make([]string, len(vs))
	perm := rand.Perm(len(vs))
	for i, v := range perm {
		shuffled[v] = vs[i]
	}

	return shuffled
}
<commit_msg>Correctly print error when failing to proxy<commit_after>package main

import (
	"io"
	"log"
	"math/rand"
	"net/http"
	"strconv"

	"github.com/stripe/sequins/blocks"
)

// serveKey is the entrypoint for incoming HTTP requests. It looks up the value
// locally, for, failing that, asks a peer that has it. If the request was
// already proxied to us, it is not proxied further.
func (vs *version) serveKey(w http.ResponseWriter, r *http.Request, key string) {
	// If we don't have any data for this version at all, that's a 404.
	if vs.numPartitions == 0 {
		vs.serveNotFound(w)
		return
	}

	partition, alternatePartition := blocks.KeyPartition(key, vs.numPartitions)
	bs := vs.getBlockStore()
	if bs != nil && (vs.hasPartition(partition) || vs.hasPartition(alternatePartition)) {
		record, err := bs.Get(key)
		if err != nil {
			vs.serveError(w, key, err)
			return
		}

		vs.serveLocal(w, key, record)
	} else if r.URL.Query().Get("proxy") == "" {
		vs.serveProxied(w, r, key, partition, alternatePartition)
	} else {
		vs.serveError(w, key, errProxiedIncorrectly)
	}
}

func (vs *version) serveLocal(w http.ResponseWriter, key string, record *blocks.Record) {
	if record == nil {
		vs.serveNotFound(w)
		return
	}

	defer record.Close()
	w.Header().Set(versionHeader, vs.name)
	w.Header().Set("Content-Length", strconv.FormatUint(record.ValueLen, 10))
	w.Header().Set("Last-Modified", vs.created.UTC().Format(http.TimeFormat))
	_, err := io.Copy(w, record)
	if err != nil {
		// We already wrote a 200 OK, so not much we can do here except log.
		log.Printf("Error streaming response for /%s/%s (version %s): %s", vs.db, key, vs.name)
	}
}

func (vs *version) serveProxied(w http.ResponseWriter, r *http.Request,
	key string, partition, alternatePartition int) {

	// Shuffle the peers, so we try them in a random order.
	// TODO: We don't want to blacklist nodes, but we can weight them lower
	peers := shuffle(vs.partitions.getPeers(partition))
	if len(peers) == 0 {
		log.Printf("No peers available for /%s/%s (version %s)", vs.db, key, vs.name)
		w.WriteHeader(http.StatusBadGateway)
		return
	}

	resp, peer, err := vs.proxy(r, peers)
	if err == nil && resp.StatusCode == 404 && alternatePartition != partition {
		log.Println("Trying alternate partition for pathological key", key)

		resp.Body.Close()
		alternatePeers := shuffle(vs.partitions.getPeers(alternatePartition))
		resp, peer, err = vs.proxy(r, alternatePeers)
	}

	if err == errNoAvailablePeers {
		// Either something is wrong with sharding, or all peers errored for some
		// other reason. 502
		log.Printf("No peers available for /%s/%s (version %s)", vs.db, key, vs.name)
		w.WriteHeader(http.StatusBadGateway)
		return
	} else if err == errProxyTimeout {
		// All of our peers failed us. 504.
		log.Printf("All peers timed out for /%s/%s (version %s)", vs.db, key, vs.name)
		w.WriteHeader(http.StatusGatewayTimeout)
		return
	} else if err != nil {
		// Some other error. 500.
		vs.serveError(w, key, err)
		return
	}

	// Proxying can produce inconsistent versions if something is broken. Use the
	// one the peer set.
	w.Header().Set(versionHeader, resp.Header.Get(versionHeader))
	w.Header().Set(proxyHeader, peer)
	w.Header().Set("Content-Length", resp.Header.Get("Content-Length"))
	w.Header().Set("Last-Modified", vs.created.UTC().Format(http.TimeFormat))
	w.WriteHeader(resp.StatusCode)

	// TODO: Apparently in 1.7 the client always asks for gzip by default. If our
	// client asks for gzip too, we should be able to pass through without
	// decompressing.
	defer resp.Body.Close()
	_, err = io.Copy(w, resp.Body)
	if err != nil {
		// We already wrote a 200 OK, so not much we can do here except log.
		log.Printf("Error copying response from peer for /%s/%s (version %s): %s", vs.db, key, vs.name, err)
	}
}

func (vs *version) serveNotFound(w http.ResponseWriter) {
	w.Header().Set(versionHeader, vs.name)
	w.WriteHeader(http.StatusNotFound)
}

func (vs *version) serveError(w http.ResponseWriter, key string, err error) {
	log.Printf("Error fetching value for /%s/%s: %s\n", vs.db, key, err)
	w.WriteHeader(http.StatusInternalServerError)
}

func shuffle(vs []string) []string {
	shuffled := make([]string, len(vs))
	perm := rand.Perm(len(vs))
	for i, v := range perm {
		shuffled[v] = vs[i]
	}

	return shuffled
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resource

import (
	"context"
	"fmt"
	"reflect"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/dynamic"
)

// CRDGetter is a function that can download the list of GVK for all
// CRDs.
type CRDGetter func() ([]schema.GroupKind, error)

func CRDFromDynamic(client dynamic.Interface) CRDGetter {
	return func() ([]schema.GroupKind, error) {
		list, err := client.Resource(schema.GroupVersionResource{
			Group:    "apiextensions.k8s.io",
			Version:  "v1beta1",
			Resource: "customresourcedefinitions",
		}).List(context.TODO(), metav1.ListOptions{})
		if err != nil {
			return nil, fmt.Errorf("failed to list CRDs: %v", err)
		}
		if list == nil {
			return nil, nil
		}

		gks := []schema.GroupKind{}

		// We need to parse the list to get the gvk, I guess that's fine.
		for _, crd := range (*list).Items {
			// Look for group, version, and kind
			group, _, _ := unstructured.NestedString(crd.Object, "spec", "group")
			kind, _, _ := unstructured.NestedString(crd.Object, "spec", "names", "kind")

			gks = append(gks, schema.GroupKind{
				Group: group,
				Kind:  kind,
			})
		}

		return gks, nil
	}
}

// CRDFinder keeps a cache of known CRDs and finds a given GVK in the
// list.
type CRDFinder interface {
	HasCRD(gvk schema.GroupKind) (bool, error)
}

func NewCRDFinder(getter CRDGetter) CRDFinder {
	return &crdFinder{
		getter: getter,
	}
}

type crdFinder struct {
	getter CRDGetter
	cache  *[]schema.GroupKind
}

func (f *crdFinder) cacheCRDs() error {
	if f.cache != nil {
		return nil
	}

	list, err := f.getter()
	if err != nil {
		return err
	}
	f.cache = &list
	return nil
}

func (f *crdFinder) findCRD(gvk schema.GroupKind) bool {
	for _, crd := range *f.cache {
		if reflect.DeepEqual(gvk, crd) {
			return true
		}
	}
	return false
}

func (f *crdFinder) HasCRD(gvk schema.GroupKind) (bool, error) {
	if err := f.cacheCRDs(); err != nil {
		return false, err
	}
	return f.findCRD(gvk), nil
}
<commit_msg>update CLI helper finding CRDs to use CRD v1<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resource

import (
	"context"
	"fmt"
	"reflect"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/dynamic"
)

// CRDGetter is a function that can download the list of GVK for all
// CRDs.
type CRDGetter func() ([]schema.GroupKind, error)

func CRDFromDynamic(client dynamic.Interface) CRDGetter {
	return func() ([]schema.GroupKind, error) {
		list, err := client.Resource(schema.GroupVersionResource{
			Group:    "apiextensions.k8s.io",
			Version:  "v1",
			Resource: "customresourcedefinitions",
		}).List(context.TODO(), metav1.ListOptions{})
		if err != nil {
			return nil, fmt.Errorf("failed to list CRDs: %v", err)
		}
		if list == nil {
			return nil, nil
		}

		gks := []schema.GroupKind{}

		// We need to parse the list to get the gvk, I guess that's fine.
		for _, crd := range (*list).Items {
			// Look for group, version, and kind
			group, _, _ := unstructured.NestedString(crd.Object, "spec", "group")
			kind, _, _ := unstructured.NestedString(crd.Object, "spec", "names", "kind")

			gks = append(gks, schema.GroupKind{
				Group: group,
				Kind:  kind,
			})
		}

		return gks, nil
	}
}

// CRDFinder keeps a cache of known CRDs and finds a given GVK in the
// list.
type CRDFinder interface {
	HasCRD(gvk schema.GroupKind) (bool, error)
}

func NewCRDFinder(getter CRDGetter) CRDFinder {
	return &crdFinder{
		getter: getter,
	}
}

type crdFinder struct {
	getter CRDGetter
	cache  *[]schema.GroupKind
}

func (f *crdFinder) cacheCRDs() error {
	if f.cache != nil {
		return nil
	}

	list, err := f.getter()
	if err != nil {
		return err
	}
	f.cache = &list
	return nil
}

func (f *crdFinder) findCRD(gvk schema.GroupKind) bool {
	for _, crd := range *f.cache {
		if reflect.DeepEqual(gvk, crd) {
			return true
		}
	}
	return false
}

func (f *crdFinder) HasCRD(gvk schema.GroupKind) (bool, error) {
	if err := f.cacheCRDs(); err != nil {
		return false, err
	}
	return f.findCRD(gvk), nil
}
<|endoftext|>