<commit_before>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package images

import (
	"context"
	"fmt"
	"os"
	"os/exec"
	"path"

	flag "github.com/spf13/pflag"

	"github.com/jetstack/cert-manager/hack/release/pkg/bazel"
	"github.com/jetstack/cert-manager/hack/release/pkg/flags"
	logf "github.com/jetstack/cert-manager/hack/release/pkg/log"
	"github.com/jetstack/cert-manager/hack/release/pkg/util"
)

var (
	Default = &Plugin{}

	supportedGoArch     = []string{"amd64", "arm64", "arm"}
	supportedComponents = []string{"acmesolver", "controller", "webhook", "cainjector"}
	log                 = logf.Log.WithName("images")
)

type Plugin struct {
	// The list of images to build (e.g. acmesolver, controller, webhook)
	Components []string

	// If true, the built images will be exported to the configured docker
	// daemon when the Build() method is called.
	ExportToDocker bool

	// List of architectures to build images for
	GoArch []string

	// DockerConfig is a path to a directory containing a config.json file that
	// is used for Docker authentication
	DockerConfig string

	// TODO: add GOOS support once the build system supports more than linux

	// built is set to true if Build() has completed successfully
	built bool
	// configFileName is computed based on the DockerConfig field
	configFileName string
}

func (g *Plugin) AddFlags(fs *flag.FlagSet) {
	fs.BoolVar(&g.ExportToDocker, "images.export", false, "if true, images will be exported to the currently configured docker daemon")
	fs.StringSliceVar(&g.Components, "images.components", []string{"acmesolver", "controller", "webhook"}, "the list of components to build images for")
	fs.StringSliceVar(&g.GoArch, "images.goarch", []string{"amd64", "arm64", "arm"}, "list of architectures to build images for")
	fs.StringVar(&g.DockerConfig, "images.docker-config", "", "path to a directory containing a docker config.json file used when pushing images")
}

func (g *Plugin) Validate() []error {
	var errs []error

	// validate goarch flag
	for _, a := range g.GoArch {
		valid := false
		for _, sa := range supportedGoArch {
			if a == sa {
				valid = true
				break
			}
		}
		if !valid {
			errs = append(errs, fmt.Errorf("invalid goarch value %q", a))
		}
	}

	// validate components flag
	for _, a := range g.Components {
		valid := false
		for _, sa := range supportedComponents {
			if a == sa {
				valid = true
				break
			}
		}
		if !valid {
			errs = append(errs, fmt.Errorf("invalid component name %q", a))
		}
	}

	return errs
}

func (g *Plugin) InitPublish() []error {
	var errs []error

	if g.DockerConfig != "" {
		configFileName := path.Join(g.DockerConfig, "config.json")
		f, err := os.Stat(configFileName)
		if err != nil {
			return []error{fmt.Errorf("error checking config file: %v", err)}
		}
		if f.IsDir() {
			return []error{fmt.Errorf("docker config.json is not a file")}
		}
		g.configFileName = g.DockerConfig
	}

	return errs
}

func (g *Plugin) Build(ctx context.Context) error {
	_, err := g.build(ctx)
	if err != nil {
		return err
	}

	if g.ExportToDocker {
		log.Info("Exporting docker images to local docker daemon")
		if err := g.exportToDocker(ctx); err != nil {
			return err
		}
	} else {
		log.Info("skipping exporting docker images to docker daemon")
	}

	return nil
}

func (g *Plugin) Publish(ctx context.Context) error {
	log.Info("running publish for image plugin")
	// this case should never be reached, but we check it to be safe
	if !g.built {
		if _, err := g.build(ctx); err != nil {
			return err
		}
	}

	log.Info("pushing images")
	targets := g.generateTargets()
	err := g.pushImages(ctx, targets)
	if err != nil {
		return err
	}

	log.Info("published all docker images")

	return nil
}

func (g *Plugin) Complete() error {
	log = log.WithName("default-flags")

	if g.DockerConfig == "" {
		g.DockerConfig = os.Getenv("DOCKER_CONFIG")
		if g.DockerConfig != "" {
			log.Info("set default value", "flag", "images.docker-config", "value", g.DockerConfig)
		}
	}
	return nil
}

func (g *Plugin) build(ctx context.Context) (imageTargets, error) {
	targets := g.generateTargets()

	bazelTargets := targets.bazelTargets()
	log := log.WithValues("images", bazelTargets)
	log.Info("building bazel image targets")

	// set the os and arch to linux/amd64
	// whilst we might be building cross-arch binaries, cgo only depends on
	// particular OS settings and not arch, so just by setting this to 'linux'
	// we can fix cross builds on platforms other than linux
	// if we support alternate OS values in future, this will need updating
	// with a call to BuildPlatformE per *OS*.
	err := bazel.Default.BuildPlatformE(ctx, log, "linux", "amd64", bazelTargets...)
	if err != nil {
		return nil, fmt.Errorf("error building docker images (%v): %v", targets, err)
	}

	g.built = true
	return targets, nil
}

func (g *Plugin) exportToDocker(ctx context.Context) error {
	targets := g.generateTargets()
	log.WithValues("images", targets.bazelExportTargets()).Info("exporting images to docker daemon")
	for _, target := range targets.bazelExportTargets() {
		log := log.WithValues("target", target)
		log.Info("exporting image to docker daemon")
		// set the os and arch to linux/amd64
		// whilst we might be building cross-arch binaries, cgo only depends on
		// particular OS settings and not arch, so just by setting this to 'linux'
		// we can fix cross builds on platforms other than linux
		// if we support alternate OS values in future, this will need updating
		// with a call to BuildPlatformE per *OS*.
		err := bazel.Default.RunPlatformE(ctx, log, "linux", "amd64", target)
		if err != nil {
			return fmt.Errorf("error exporting image %q to docker daemon: %v", target, err)
		}
	}

	log.WithValues("images", targets.exportedImageNames()).Info("exported all docker images")

	return nil
}

// generateTargets generates a list of Bazel target names that must be
// built for this invocation of the image builder
func (g *Plugin) generateTargets() imageTargets {
	var targets []imageTarget
	for _, c := range g.Components {
		for _, a := range g.GoArch {
			targets = append(targets, imageTarget{c, "linux", a})
		}
	}
	return targets
}

// pushImages will push the images built for this release to the registry
// TODO: add support for calling container_push targets instead of just 'docker push'
func (p *Plugin) pushImages(ctx context.Context, targets imageTargets) error {
	err := p.exportToDocker(ctx)
	if err != nil {
		return err
	}

	var images []string
	for _, t := range targets {
		images = append(images, t.exportedImageNames()...)
	}

	log.WithValues("images", images).Info("pushing docker images")
	for _, img := range images {
		log := log.WithValues("image", img)
		log.Info("pushing docker image")
		args := []string{}
		if p.configFileName != "" {
			args = append(args, "--config", p.configFileName)
		}
		args = append(args, "push", img)
		cmd := exec.CommandContext(ctx, "docker", args...)
		err := util.RunE(log, cmd)
		if err != nil {
			return err
		}
	}

	return nil
}

type imageTargets []imageTarget

func (i imageTargets) bazelTargets() []string {
	out := make([]string, len(i))
	for idx, target := range i {
		out[idx] = target.bazelTarget()
	}
	return out
}

func (i imageTargets) bazelExportTargets() []string {
	out := make([]string, len(i))
	for idx, target := range i {
		out[idx] = target.bazelExportTarget()
	}
	return out
}

func (i imageTargets) exportedImageNames() []string {
	out := make([]string, 0)
	for _, target := range i {
		out = append(out, target.exportedImageNames()...)
	}
	return out
}

type imageTarget struct {
	name, os, arch string
}

func (i imageTarget) bazelTarget() string {
	return fmt.Sprintf("//cmd/%s:image.%s-%s", i.name, i.os, i.arch)
}

func (i imageTarget) bazelExportTarget() string {
	return fmt.Sprintf("//cmd/%s:image.%s-%s.export", i.name, i.os, i.arch)
}

func (i imageTarget) exportedImageNames() []string {
	if i.arch == "amd64" {
		return []string{
			fmt.Sprintf("%s/cert-manager-%s:%s", flags.Default.DockerRepo, i.name, flags.Default.AppVersion),
			fmt.Sprintf("%s/cert-manager-%s:%s", flags.Default.DockerRepo, i.name, flags.Default.GitCommitRef),
		}
	}

	return []string{
		fmt.Sprintf("%s/cert-manager-%s-%s:%s", flags.Default.DockerRepo, i.name, i.arch, flags.Default.AppVersion),
		fmt.Sprintf("%s/cert-manager-%s-%s:%s", flags.Default.DockerRepo, i.name, i.arch, flags.Default.GitCommitRef),
	}
}
<commit_msg>Add cainjector to list of images to build<commit_after>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package images

import (
	"context"
	"fmt"
	"os"
	"os/exec"
	"path"

	flag "github.com/spf13/pflag"

	"github.com/jetstack/cert-manager/hack/release/pkg/bazel"
	"github.com/jetstack/cert-manager/hack/release/pkg/flags"
	logf "github.com/jetstack/cert-manager/hack/release/pkg/log"
	"github.com/jetstack/cert-manager/hack/release/pkg/util"
)

var (
	Default = &Plugin{}

	supportedGoArch     = []string{"amd64", "arm64", "arm"}
	supportedComponents = []string{"acmesolver", "controller", "webhook", "cainjector"}
	log                 = logf.Log.WithName("images")
)

type Plugin struct {
	// The list of images to build (e.g. acmesolver, controller, webhook)
	Components []string

	// If true, the built images will be exported to the configured docker
	// daemon when the Build() method is called.
	ExportToDocker bool

	// List of architectures to build images for
	GoArch []string

	// DockerConfig is a path to a directory containing a config.json file that
	// is used for Docker authentication
	DockerConfig string

	// TODO: add GOOS support once the build system supports more than linux

	// built is set to true if Build() has completed successfully
	built bool
	// configFileName is computed based on the DockerConfig field
	configFileName string
}

func (g *Plugin) AddFlags(fs *flag.FlagSet) {
	fs.BoolVar(&g.ExportToDocker, "images.export", false, "if true, images will be exported to the currently configured docker daemon")
	fs.StringSliceVar(&g.Components, "images.components", []string{"acmesolver", "controller", "webhook", "cainjector"}, "the list of components to build images for")
	fs.StringSliceVar(&g.GoArch, "images.goarch", []string{"amd64", "arm64", "arm"}, "list of architectures to build images for")
	fs.StringVar(&g.DockerConfig, "images.docker-config", "", "path to a directory containing a docker config.json file used when pushing images")
}

func (g *Plugin) Validate() []error {
	var errs []error

	// validate goarch flag
	for _, a := range g.GoArch {
		valid := false
		for _, sa := range supportedGoArch {
			if a == sa {
				valid = true
				break
			}
		}
		if !valid {
			errs = append(errs, fmt.Errorf("invalid goarch value %q", a))
		}
	}

	// validate components flag
	for _, a := range g.Components {
		valid := false
		for _, sa := range supportedComponents {
			if a == sa {
				valid = true
				break
			}
		}
		if !valid {
			errs = append(errs, fmt.Errorf("invalid component name %q", a))
		}
	}

	return errs
}

func (g *Plugin) InitPublish() []error {
	var errs []error

	if g.DockerConfig != "" {
		configFileName := path.Join(g.DockerConfig, "config.json")
		f, err := os.Stat(configFileName)
		if err != nil {
			return []error{fmt.Errorf("error checking config file: %v", err)}
		}
		if f.IsDir() {
			return []error{fmt.Errorf("docker config.json is not a file")}
		}
		g.configFileName = g.DockerConfig
	}

	return errs
}

func (g *Plugin) Build(ctx context.Context) error {
	_, err := g.build(ctx)
	if err != nil {
		return err
	}

	if g.ExportToDocker {
		log.Info("Exporting docker images to local docker daemon")
		if err := g.exportToDocker(ctx); err != nil {
			return err
		}
	} else {
		log.Info("skipping exporting docker images to docker daemon")
	}

	return nil
}

func (g *Plugin) Publish(ctx context.Context) error {
	log.Info("running publish for image plugin")
	// this case should never be reached, but we check it to be safe
	if !g.built {
		if _, err := g.build(ctx); err != nil {
			return err
		}
	}

	log.Info("pushing images")
	targets := g.generateTargets()
	err := g.pushImages(ctx, targets)
	if err != nil {
		return err
	}

	log.Info("published all docker images")

	return nil
}

func (g *Plugin) Complete() error {
	log = log.WithName("default-flags")

	if g.DockerConfig == "" {
		g.DockerConfig = os.Getenv("DOCKER_CONFIG")
		if g.DockerConfig != "" {
			log.Info("set default value", "flag", "images.docker-config", "value", g.DockerConfig)
		}
	}
	return nil
}

func (g *Plugin) build(ctx context.Context) (imageTargets, error) {
	targets := g.generateTargets()

	bazelTargets := targets.bazelTargets()
	log := log.WithValues("images", bazelTargets)
	log.Info("building bazel image targets")

	// set the os and arch to linux/amd64
	// whilst we might be building cross-arch binaries, cgo only depends on
	// particular OS settings and not arch, so just by setting this to 'linux'
	// we can fix cross builds on platforms other than linux
	// if we support alternate OS values in future, this will need updating
	// with a call to BuildPlatformE per *OS*.
	err := bazel.Default.BuildPlatformE(ctx, log, "linux", "amd64", bazelTargets...)
	if err != nil {
		return nil, fmt.Errorf("error building docker images (%v): %v", targets, err)
	}

	g.built = true
	return targets, nil
}

func (g *Plugin) exportToDocker(ctx context.Context) error {
	targets := g.generateTargets()
	log.WithValues("images", targets.bazelExportTargets()).Info("exporting images to docker daemon")
	for _, target := range targets.bazelExportTargets() {
		log := log.WithValues("target", target)
		log.Info("exporting image to docker daemon")
		// set the os and arch to linux/amd64
		// whilst we might be building cross-arch binaries, cgo only depends on
		// particular OS settings and not arch, so just by setting this to 'linux'
		// we can fix cross builds on platforms other than linux
		// if we support alternate OS values in future, this will need updating
		// with a call to BuildPlatformE per *OS*.
		err := bazel.Default.RunPlatformE(ctx, log, "linux", "amd64", target)
		if err != nil {
			return fmt.Errorf("error exporting image %q to docker daemon: %v", target, err)
		}
	}

	log.WithValues("images", targets.exportedImageNames()).Info("exported all docker images")

	return nil
}

// generateTargets generates a list of Bazel target names that must be
// built for this invocation of the image builder
func (g *Plugin) generateTargets() imageTargets {
	var targets []imageTarget
	for _, c := range g.Components {
		for _, a := range g.GoArch {
			targets = append(targets, imageTarget{c, "linux", a})
		}
	}
	return targets
}

// pushImages will push the images built for this release to the registry
// TODO: add support for calling container_push targets instead of just 'docker push'
func (p *Plugin) pushImages(ctx context.Context, targets imageTargets) error {
	err := p.exportToDocker(ctx)
	if err != nil {
		return err
	}

	var images []string
	for _, t := range targets {
		images = append(images, t.exportedImageNames()...)
	}

	log.WithValues("images", images).Info("pushing docker images")
	for _, img := range images {
		log := log.WithValues("image", img)
		log.Info("pushing docker image")
		args := []string{}
		if p.configFileName != "" {
			args = append(args, "--config", p.configFileName)
		}
		args = append(args, "push", img)
		cmd := exec.CommandContext(ctx, "docker", args...)
		err := util.RunE(log, cmd)
		if err != nil {
			return err
		}
	}

	return nil
}

type imageTargets []imageTarget

func (i imageTargets) bazelTargets() []string {
	out := make([]string, len(i))
	for idx, target := range i {
		out[idx] = target.bazelTarget()
	}
	return out
}

func (i imageTargets) bazelExportTargets() []string {
	out := make([]string, len(i))
	for idx, target := range i {
		out[idx] = target.bazelExportTarget()
	}
	return out
}

func (i imageTargets) exportedImageNames() []string {
	out := make([]string, 0)
	for _, target := range i {
		out = append(out, target.exportedImageNames()...)
	}
	return out
}

type imageTarget struct {
	name, os, arch string
}

func (i imageTarget) bazelTarget() string {
	return fmt.Sprintf("//cmd/%s:image.%s-%s", i.name, i.os, i.arch)
}

func (i imageTarget) bazelExportTarget() string {
	return fmt.Sprintf("//cmd/%s:image.%s-%s.export", i.name, i.os, i.arch)
}

func (i imageTarget) exportedImageNames() []string {
	if i.arch == "amd64" {
		return []string{
			fmt.Sprintf("%s/cert-manager-%s:%s", flags.Default.DockerRepo, i.name, flags.Default.AppVersion),
			fmt.Sprintf("%s/cert-manager-%s:%s", flags.Default.DockerRepo, i.name, flags.Default.GitCommitRef),
		}
	}

	return []string{
		fmt.Sprintf("%s/cert-manager-%s-%s:%s", flags.Default.DockerRepo, i.name, i.arch, flags.Default.AppVersion),
		fmt.Sprintf("%s/cert-manager-%s-%s:%s", flags.Default.DockerRepo, i.name, i.arch, flags.Default.GitCommitRef),
	}
}
<|endoftext|>


<commit_before>package algorithmia

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

type FileAttributes struct {
	FileName     string `json:"filename"`
	LastModified string `json:"last_modified"`
	Size         int64  `json:"size"`
}

type DataFile struct {
	DataObjectType

	path         string
	url          string
	lastModified time.Time
	size         int64

	client *Client
}

func NewDataFile(client *Client, dataUrl string) *DataFile {
	p := strings.TrimSpace(dataUrl)
	if strings.HasPrefix(p, "data://") {
		p = p[len("data://"):]
	} else if strings.HasPrefix(p, "/") {
		p = p[1:]
	}
	return &DataFile{
		DataObjectType: File,
		client:         client,
		path:           p,
		url:            getUrl(p),
	}
}

func (f *DataFile) SetAttributes(attr *FileAttributes) error {
	//%Y-%m-%dT%H:%M:%S.000Z
	t, err := time.Parse("2006-01-02T15:04:05.000Z", attr.LastModified)
	if err != nil {
		return err
	}
	f.lastModified = t
	f.size = attr.Size
	return nil
}

//Get file from the data api
func (f *DataFile) File() (*os.File, error) {
	if exists, err := f.Exists(); err != nil {
		return nil, err
	} else if !exists {
		return nil, errors.New(fmt.Sprint("file does not exist -", f.path))
	}

	resp, err := f.client.getHelper(f.url, url.Values{})
	if err != nil {
		return nil, err
	}

	rf, err := ioutil.TempFile(os.TempDir(), "algorithmia")
	if err != nil {
		return nil, err
	}
	_, err = io.Copy(rf, resp.Body)
	if err != nil {
		rf.Close()
		return nil, err
	}
	return rf, nil
}

func (f *DataFile) Exists() (bool, error) {
	resp, err := f.client.headHelper(f.url)
	return resp.StatusCode == http.StatusOK, err
}

func (f *DataFile) Name() (string, error) {
	_, name, err := getParentAndBase(f.path)
	return name, err
}

func (f *DataFile) Bytes() ([]byte, error) {
	if exists, err := f.Exists(); err != nil {
		return nil, err
	} else if !exists {
		return nil, errors.New(fmt.Sprint("file does not exist -", f.path))
	}

	resp, err := f.client.getHelper(f.url, url.Values{})
	if err != nil {
		return nil, err
	}

	return ioutil.ReadAll(resp.Body)
}

func (f *DataFile) StringContents() (string, error) {
	if b, err := f.Bytes(); err != nil {
		return "", err
	} else {
		return string(b), nil
	}
}

func (f *DataFile) Json(x interface{}) error {
	if exists, err := f.Exists(); err != nil {
		return err
	} else if !exists {
		return errors.New(fmt.Sprint("file does not exist -", f.path))
	}

	resp, err := f.client.getHelper(f.url, url.Values{})
	if err != nil {
		return err
	}

	return getJson(resp, x)
}

//Post to data api
func (f *DataFile) Put(data []byte) error {
	resp, err := f.client.putHelper(f.url, data)
	if err != nil {
		return err
	}

	b, err := getRaw(resp)
	if err != nil {
		return err
	}

	err = errorFromJsonData(b)
	if err != nil {
		return err
	}

	return nil
}

//Post json to data api
func (f *DataFile) PutJson(data interface{}) error {
	b, err := json.Marshal(data)
	if err != nil {
		return err
	}
	return f.Put(b)
}

//Post file to data api
func (f *DataFile) PutFile(fpath string) error {
	b, err := ioutil.ReadFile(fpath)
	if err != nil {
		return err
	}
	return f.Put(b)
}

//Delete from data api
func (f *DataFile) Delete() error {
	resp, err := f.client.deleteHelper(f.url)
	if err != nil {
		return err
	}

	if err := errorFromResponse(resp); err != nil {
		return err
	}

	return nil
}

func (f *DataFile) Path() string {
	return f.path
}

func (f *DataFile) Url() string {
	return f.url
}

func (f *DataFile) LastModified() time.Time {
	return f.lastModified
}

func (f *DataFile) Size() int64 {
	return f.size
}
<commit_msg>DataFile Put argument type is now interface{}, for []byte use PutBytes instead<commit_after>package algorithmia

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

type FileAttributes struct {
	FileName     string `json:"filename"`
	LastModified string `json:"last_modified"`
	Size         int64  `json:"size"`
}

type DataFile struct {
	DataObjectType

	path         string
	url          string
	lastModified time.Time
	size         int64

	client *Client
}

func NewDataFile(client *Client, dataUrl string) *DataFile {
	p := strings.TrimSpace(dataUrl)
	if strings.HasPrefix(p, "data://") {
		p = p[len("data://"):]
	} else if strings.HasPrefix(p, "/") {
		p = p[1:]
	}
	return &DataFile{
		DataObjectType: File,
		client:         client,
		path:           p,
		url:            getUrl(p),
	}
}

func (f *DataFile) SetAttributes(attr *FileAttributes) error {
	//%Y-%m-%dT%H:%M:%S.000Z
	t, err := time.Parse("2006-01-02T15:04:05.000Z", attr.LastModified)
	if err != nil {
		return err
	}
	f.lastModified = t
	f.size = attr.Size
	return nil
}

//Get file from the data api
func (f *DataFile) File() (*os.File, error) {
	if exists, err := f.Exists(); err != nil {
		return nil, err
	} else if !exists {
		return nil, errors.New(fmt.Sprint("file does not exist -", f.path))
	}

	resp, err := f.client.getHelper(f.url, url.Values{})
	if err != nil {
		return nil, err
	}

	rf, err := ioutil.TempFile(os.TempDir(), "algorithmia")
	if err != nil {
		return nil, err
	}
	_, err = io.Copy(rf, resp.Body)
	if err != nil {
		rf.Close()
		return nil, err
	}
	name := rf.Name()
	if rf.Close(); err != nil {
		return nil, err
	}

	return os.Open(name)
}

func (f *DataFile) Exists() (bool, error) {
	resp, err := f.client.headHelper(f.url)
	return resp.StatusCode == http.StatusOK, err
}

func (f *DataFile) Name() (string, error) {
	_, name, err := getParentAndBase(f.path)
	return name, err
}

func (f *DataFile) Bytes() ([]byte, error) {
	if exists, err := f.Exists(); err != nil {
		return nil, err
	} else if !exists {
		return nil, errors.New(fmt.Sprint("file does not exist -", f.path))
	}

	resp, err := f.client.getHelper(f.url, url.Values{})
	if err != nil {
		return nil, err
	}

	return ioutil.ReadAll(resp.Body)
}

func (f *DataFile) StringContents() (string, error) {
	if b, err := f.Bytes(); err != nil {
		return "", err
	} else {
		return string(b), nil
	}
}

func (f *DataFile) Json(x interface{}) error {
	if exists, err := f.Exists(); err != nil {
		return err
	} else if !exists {
		return errors.New(fmt.Sprint("file does not exist -", f.path))
	}

	resp, err := f.client.getHelper(f.url, url.Values{})
	if err != nil {
		return err
	}

	return getJson(resp, x)
}

//Post to data api
func (f *DataFile) Put(data interface{}) error {
	switch dt := data.(type) {
	case string:
		return f.PutBytes([]byte(dt))
	case []byte:
		return f.PutBytes(dt)
	default:
		return f.PutJson(data)
	}
}

func (f *DataFile) PutBytes(data []byte) error {
	resp, err := f.client.putHelper(f.url, data)
	if err != nil {
		return err
	}

	b, err := getRaw(resp)
	if err != nil {
		return err
	}

	err = errorFromJsonData(b)
	if err != nil {
		return err
	}

	return nil
}

//Post json to data api
func (f *DataFile) PutJson(data interface{}) error {
	b, err := json.Marshal(data)
	if err != nil {
		return err
	}
	return f.PutBytes(b)
}

//Post file to data api
func (f *DataFile) PutFile(fpath string) error {
	b, err := ioutil.ReadFile(fpath)
	if err != nil {
		return err
	}
	return f.PutBytes(b)
}

//Delete from data api
func (f *DataFile) Delete() error {
	resp, err := f.client.deleteHelper(f.url)
	if err != nil {
		return err
	}

	if err := errorFromResponse(resp); err != nil {
		return err
	}

	return nil
}

func (f *DataFile) Path() string {
	return f.path
}

func (f *DataFile) Url() string {
	return f.url
}

func (f *DataFile) LastModified() time.Time {
	return f.lastModified
}

func (f *DataFile) Size() int64 {
	return f.size
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/urlfetch"
	"io/ioutil"
	"net/http"
)

type FacebookAttachment struct {
	/* image, audio, video, file or location */
	Type string `json:"type,omitempty"`

	/* multimedia or location payload */
	Payload string `json:"payload,omitempty"`
}

type FacebookQuickReply struct {
	/* Custom data provided by the app */
	Payload string `json:"payload,omitempty"`
}

type FacebookMessage struct {

	/* Indicates the message sent from the page itself */
	IsEcho bool `json:"is_echo,omitempty"`

	/* ID of the app from which the message was sent */
	AppId string `json:"app_id,omitempty"`

	/* Custom string passed to the Send API as the metadata field */
	Metadata string `json:"metadata,omitempty"`

	/* Message ID */
	Mid string `json:"mid,omitempty"`

	/* Message sequence number */
	Seq int64 `json:"seq,omitempty"`

	/* Text of message */
	Text string `json:"text,omitempty"`

	/* Array containing attachment data */
	Attachments *[]FacebookAttachment `json:"attachment,omitempty"`

	/* Optional custom data provided by the sending app */
	QuickReply *FacebookQuickReply `json:"quick_reply,omitempty"`
}

type FacebookPerson struct {
	Id string `json:"id,omitempty"`
}

type FacebookPostback struct {
	/* payload parameter that was defined with the button */
	Payload string `json:"payload,omitempty"`
}

type FacebookOptIn struct {
	/* data-ref parameter that was defined with the entry point */
	Ref string `json:"ref,omitempty"`
}

type FacebookAccountLinking struct {
	/* linked or unlinked */
	status string `json:"status,omitempty"`

	/* Value of pass-through authorization_code provided in the Linking Account flow */
	AuthorizationCode string `json:"authorization_code,omitempty"`
}

type FacebookDelivery struct {
	/* Array containing message IDs of messages that were delivered. Field may not be present. */
	Mids []string `json:"mids,omitempty"`

	/* All messages that were sent before this timestamp were delivered */
	Watermark int64 `json:"watermark,omitempty"`

	/* Sequence number */
	Seq int64 `json:"seq,omitempty"`
}

type FacebookRead struct {
	/* All messages that were sent before this timestamp were read */
	Watermark int64 `json:"watermark,omitempty"`

	/* Sequence number */
	Seq int64 `json:"seq,omitempty"`
}

type FacebookMessaging struct {
	/* Sender user ID */
	Sender *FacebookPerson `json:"sender,omitempty"`

	/* Recipient user ID */
	Recipient *FacebookPerson `json:"recipient,omitempty"`

	/* Time of messaging (epoch time in milliseconds) */
	Timestamp int64 `json:"timestamp,omitempty"`

	/* Message Received Payload */
	Message *FacebookMessage `json:"message,omitempty"`

	/* Postback Received Payload */
	Postback *FacebookPostback `json:"postback,omitempty"`

	/* Authentification Payload */
	OptIn *FacebookOptIn `json:"optin,omitempty"`

	/* Account Linking Payload */
	AccountLinking *FacebookAccountLinking `json:"account_linking,omitempty"`

	/* Message Delivered Payload */
	Delivery *FacebookDelivery `json:"delivery,omitempty"`

	/* Message Read Payload */
	Read *FacebookRead `json:"read,omitempty"`
}

type FacebookEntry struct {
	/* Page ID of page */
	Id string `json:"id,omitempty"`

	/* Time of update (epoch time in milliseconds) */
	Time int64 `json:"time,omitempty"`

	/* Array containing objects related to messaging */
	Messaging []FacebookMessaging `json:"messaging,omitempty"`
}

type FacebookPayload struct {
	Object string `json:"object,omitempty"`

	/* Array containing event data */
	Entry []FacebookEntry `json:"entry,omitempty"`
}

func FacebookCallbackHandler(w http.ResponseWriter, r *http.Request) {

	c := appengine.NewContext(r)
	log.Debugf(c, ">>> Facebook Callback Handler")

	if r.Method == "GET" {
		FacebookCallbackGETHandler(w, r)
	} else if r.Method == "POST" {
		FacebookCallbackPOSTHandler(w, r)
	} else {
		log.Errorf(c, "Error, unkown method: %v", r.Method)
		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
		return
	}

}

func FacebookCallbackGETHandler(w http.ResponseWriter, r *http.Request) {

	c := appengine.NewContext(r)
	log.Debugf(c, ">>>> FacebookCallbackGETHandler")

	mode := r.FormValue("hub.mode")
	log.Debugf(c, "Hub Mode: %v", mode)

	challenge := r.FormValue("hub.challenge")
	log.Debugf(c, "Hub Challenge: %v", challenge)

	verify_token := r.FormValue("hub.verify_token")
	log.Debugf(c, "Hub Verify Token: %v", verify_token)

	if verify_token != VERIFY_TOKEN {
		log.Errorf(c, "Error, bad verification token: %v", verify_token)
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}

	if mode != "subscribe" {
		log.Errorf(c, "Error, bad mode: %v", mode)
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}

	fmt.Fprintf(w, "%v", challenge)
}

func FacebookCallbackPOSTHandler(w http.ResponseWriter, r *http.Request) {

	c := appengine.NewContext(r)
	log.Debugf(c, ">>>> FacebookCallbackPOSTHandler")

	var payload FacebookPayload
	err := UnmarshalRequest(c, r, &payload)
	if err != nil {
		log.Errorf(c, "Error reading JSON: %v", err)
		http.Error(w, "Internal Server Error: "+err.Error(), http.StatusInternalServerError)
		return
	}

	if payload.Object == "page" {
		log.Debugf(c, "Received %v entries", len(payload.Entry))

		for i, e := range payload.Entry {
			log.Debugf(c, "Entry #%v", i)
			log.Debugf(c, "Message on page %v", e.Id)
			log.Debugf(c, "Time: %v", e.Time)

			if len(e.Messaging) > 0 {
				log.Debugf(c, "Received %v messages", len(e.Messaging))

				for _, m := range e.Messaging {
					log.Debugf(c, "Sender %v Recipient %v", m.Sender.Id, m.Recipient.Id)

					if m.Message != nil {
						log.Debugf(c, " --- Message Received")
						log.Debugf(c, "Message Id: %v", m.Message.Mid)
						log.Debugf(c, "Message Seq: %v", m.Message.Seq)
						log.Debugf(c, "Message: %v", m.Message.Text)

						err = TreatMessage(c, m.Sender.Id, m.Message.Text)
						if err != nil {
							log.Errorf(c, "Error, sending message: %v", err)
						}
					}

					if m.Delivery != nil {
						log.Debugf(c, " --- Message Delivered")
						log.Debugf(c, "Message Ids: %v", m.Delivery.Mids)
						log.Debugf(c, "Message Seq: %v", m.Delivery.Seq)
						log.Debugf(c, "Watermark: %v", m.Delivery.Watermark)
					}

				}
			} else {
				log.Warningf(c, "No message in payload - feature not developed!")
			}
		}

	} else {
		log.Errorf(c, "Error, unkown Object type: %v", payload.Object)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}

	fmt.Fprintf(w, "")
}

func TreatMessage(c context.Context, recipientId string, message string) error {
	response := "Hello " + message
	return SendFacebookMessage(c, recipientId, response)
}

func SendFacebookMessage(c context.Context, recipientId string, message string) error {

	messaging := FacebookMessaging{
		Recipient: &FacebookPerson{
			Id: recipientId,
		},
		Message: &FacebookMessage{
			Text: message,
		},
	}

	jsonData, err := json.Marshal(messaging)
	if err != nil {
		log.Errorf(c, "Error converting to JSON: %v", err)
		return err
	}

	URL := "https://graph.facebook.com/v2.6/me/messages?access_token=" + PAGE_ACCESS_TOKEN
	log.Debugf(c, "Calling %v", URL)
	resp, err := urlfetch.Client(c).Post(URL, "application/json", bytes.NewReader(jsonData))
	if err != nil {
		log.Errorf(c, "Error posting message: %v", err)
		return err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Errorf(c, "Error reading response: %v", err)
		return err
	}

	log.Debugf(c, "Facebook response: %s", body)

	return nil

}
<commit_msg>simplified TreatMessage and rename GetResponse<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/urlfetch"
	"io/ioutil"
	"net/http"
)

type FacebookAttachment struct {
	/* image, audio, video, file or location */
	Type string `json:"type,omitempty"`

	/* multimedia or location payload */
	Payload string `json:"payload,omitempty"`
}

type FacebookQuickReply struct {
	/* Custom data provided by the app */
	Payload string `json:"payload,omitempty"`
}

type FacebookMessage struct {

	/* Indicates the message sent from the page itself */
	IsEcho bool `json:"is_echo,omitempty"`

	/* ID of the app from which the message was sent */
	AppId string `json:"app_id,omitempty"`

	/* Custom string passed to the Send API as the metadata field */
	Metadata string `json:"metadata,omitempty"`

	/* Message ID */
	Mid string `json:"mid,omitempty"`

	/* Message sequence number */
	Seq int64 `json:"seq,omitempty"`

	/* Text of message */
	Text string `json:"text,omitempty"`

	/* Array containing attachment data */
	Attachments *[]FacebookAttachment `json:"attachment,omitempty"`

	/* Optional custom data provided by the sending app */
	QuickReply *FacebookQuickReply `json:"quick_reply,omitempty"`
}

type FacebookPerson struct {
	Id string `json:"id,omitempty"`
}

type FacebookPostback struct {
	/* payload parameter that was defined with the button */
	Payload string `json:"payload,omitempty"`
}

type FacebookOptIn struct {
	/* data-ref parameter that was defined with the entry point */
	Ref string `json:"ref,omitempty"`
}

type FacebookAccountLinking struct {
	/* linked or unlinked */
	status string `json:"status,omitempty"`

	/* Value of pass-through authorization_code provided in the Linking Account flow */
	AuthorizationCode string `json:"authorization_code,omitempty"`
}

type FacebookDelivery struct {
	/* Array containing message IDs of messages that were delivered. Field may not be present. */
	Mids []string `json:"mids,omitempty"`

	/* All messages that were sent before this timestamp were delivered */
	Watermark int64 `json:"watermark,omitempty"`

	/* Sequence number */
	Seq int64 `json:"seq,omitempty"`
}

type FacebookRead struct {
	/* All messages that were sent before this timestamp were read */
	Watermark int64 `json:"watermark,omitempty"`

	/* Sequence number */
	Seq int64 `json:"seq,omitempty"`
}

type FacebookMessaging struct {
	/* Sender user ID */
	Sender *FacebookPerson `json:"sender,omitempty"`

	/* Recipient user ID */
	Recipient *FacebookPerson `json:"recipient,omitempty"`

	/* Time of messaging (epoch time in milliseconds) */
	Timestamp int64 `json:"timestamp,omitempty"`

	/* Message Received Payload */
	Message *FacebookMessage `json:"message,omitempty"`

	/* Postback Received Payload */
	Postback *FacebookPostback `json:"postback,omitempty"`

	/* Authentification Payload */
	OptIn *FacebookOptIn `json:"optin,omitempty"`

	/* Account Linking Payload */
	AccountLinking *FacebookAccountLinking `json:"account_linking,omitempty"`

	/* Message Delivered Payload */
	Delivery *FacebookDelivery `json:"delivery,omitempty"`

	/* Message Read Payload */
	Read *FacebookRead `json:"read,omitempty"`
}

type FacebookEntry struct {
	/* Page ID of page */
	Id string `json:"id,omitempty"`

	/* Time of update (epoch time in milliseconds) */
	Time int64 `json:"time,omitempty"`

	/* Array containing objects related to messaging */
	Messaging []FacebookMessaging `json:"messaging,omitempty"`
}

type FacebookPayload struct {
	Object string `json:"object,omitempty"`

	/* Array containing event data */
	Entry []FacebookEntry `json:"entry,omitempty"`
}

func FacebookCallbackHandler(w http.ResponseWriter, r *http.Request) {

	c := appengine.NewContext(r)
	log.Debugf(c, ">>> Facebook Callback Handler")

	if r.Method == "GET" {
		FacebookCallbackGETHandler(w, r)
	} else if r.Method == "POST" {
		FacebookCallbackPOSTHandler(w, r)
	} else {
		log.Errorf(c, "Error, unkown method: %v", r.Method)
		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
		return
	}

}

func FacebookCallbackGETHandler(w http.ResponseWriter, r *http.Request) {

	c := appengine.NewContext(r)
	log.Debugf(c, ">>>> FacebookCallbackGETHandler")

	mode := r.FormValue("hub.mode")
	log.Debugf(c, "Hub Mode: %v", mode)

	challenge := r.FormValue("hub.challenge")
	log.Debugf(c, "Hub Challenge: %v", challenge)

	verify_token := r.FormValue("hub.verify_token")
	log.Debugf(c, "Hub Verify Token: %v", verify_token)

	if verify_token != VERIFY_TOKEN {
		log.Errorf(c, "Error, bad verification token: %v", verify_token)
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}

	if mode != "subscribe" {
		log.Errorf(c, "Error, bad mode: %v", mode)
		http.Error(w, "Bad Request", http.StatusBadRequest)
		return
	}

	fmt.Fprintf(w, "%v", challenge)
}

func FacebookCallbackPOSTHandler(w http.ResponseWriter, r *http.Request) {

	c := appengine.NewContext(r)
	log.Debugf(c, ">>>> FacebookCallbackPOSTHandler")

	var payload FacebookPayload
	err := UnmarshalRequest(c, r, &payload)
	if err != nil {
		log.Errorf(c, "Error reading JSON: %v", err)
		http.Error(w, "Internal Server Error: "+err.Error(), http.StatusInternalServerError)
		return
	}

	if payload.Object == "page" {
		log.Debugf(c, "Received %v entries", len(payload.Entry))

		for i, e := range payload.Entry {
			log.Debugf(c, "Entry #%v", i)
			log.Debugf(c, "Message on page %v", e.Id)
			log.Debugf(c, "Time: %v", e.Time)

			if len(e.Messaging) > 0 {
				log.Debugf(c, "Received %v messages", len(e.Messaging))

				for _, m := range e.Messaging {
					log.Debugf(c, "Sender %v Recipient %v", m.Sender.Id, m.Recipient.Id)

					if m.Message != nil {
						log.Debugf(c, " --- Message Received")
						log.Debugf(c, "Message Id: %v", m.Message.Mid)
						log.Debugf(c, "Message Seq: %v", m.Message.Seq)
						log.Debugf(c, "Message: %v", m.Message.Text)

						response := GetResponse(c, m.Sender.Id, m.Message.Text)

						err = SendFacebookMessage(c, m.Sender.Id, response)
						if err != nil {
							log.Errorf(c, "Error, sending message: %v", err)
						}
					}

					if m.Delivery != nil {
						log.Debugf(c, " --- Message Delivered")
						log.Debugf(c, "Message Ids: %v", m.Delivery.Mids)
						log.Debugf(c, "Message Seq: %v", m.Delivery.Seq)
						log.Debugf(c, "Watermark: %v", m.Delivery.Watermark)
					}

				}
			} else {
				log.Warningf(c, "No message in payload - feature not developed!")
			}
		}

	} else {
		log.Errorf(c, "Error, unkown Object type: %v", payload.Object)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}

	fmt.Fprintf(w, "")
}

func GetResponse(c context.Context, recipientId string, message string) string {
	response := "Hello " + message
	return response
}

func SendFacebookMessage(c context.Context, recipientId string, message string) error {

	messaging := FacebookMessaging{
		Recipient: &FacebookPerson{
			Id: recipientId,
		},
		Message: &FacebookMessage{
			Text: message,
		},
	}

	jsonData, err := json.Marshal(messaging)
	if err != nil {
		log.Errorf(c, "Error converting to JSON: %v", err)
		return err
	}

	URL := "https://graph.facebook.com/v2.6/me/messages?access_token=" + PAGE_ACCESS_TOKEN
	log.Debugf(c, "Calling %v", URL)
	resp, err := urlfetch.Client(c).Post(URL, "application/json", bytes.NewReader(jsonData))
	if err != nil {
		log.Errorf(c, "Error posting message: %v", err)
		return err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Errorf(c, "Error reading response: %v", err)
		return err
	}

	log.Debugf(c, "Facebook response: %s", body)

	return nil

}
<|endoftext|>


<commit_before>package reservation

import (
	"fmt"
	"os"
	"sync"
	"testing"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/stretchr/testify/assert"
)

var redisTestURL = os.Getenv("REDIS_TEST_URL")

func TestNewManager(t *testing.T) {
	manager, err := NewManager(redisTestURL, "test-worker")
	assert.Nil(t, err)
	// If manager is not nil, NewManager was able to succesfully ping local redis
	assert.NotNil(t, manager)
}

func setUp(t *testing.T) (*Manager, string) {
	conn, err := redis.DialTimeout("tcp", redisTestURL, 15*time.Second, 10*time.Second, 10*time.Second)
	message := fmt.Sprintf("Could not connect to redis at %s. Ensure redis is running at this location (maybe locally)?", redisTestURL)
	assert.Nil(t, err, message)
	defer conn.Close()
	conn.Do("FLUSHALL")
	manager, err := NewManager(redisTestURL, "test-worker")
	assert.Nil(t, err)
	resourceID := "12345"
	return manager, resourceID
}

func TestManagerLockCreate(t *testing.T) {
	manager, resourceID := setUp(t)

	// Make a new reservation
	reservation, err := manager.Lock(resourceID)
	assert.Nil(t, err)

	// Assert reservation created with correct field values
	assert.NotNil(t, reservation)

	// Make the same reservation again and ensure error returned
	dupeReservation, err := manager.Lock(resourceID)
	assert.Nil(t, dupeReservation)
	assert.EqualError(t, err, fmt.Sprintf("Reservation already exists for resource %s", resourceID))

	// Release the first reservation and ensure we can make a second reservation
	err = reservation.Release()
	assert.Nil(t, err)
	reservation, err = manager.Lock(resourceID)
	assert.NotNil(t, reservation)
}

func TestManagerLockConcurrentRequests(t *testing.T) {
	// Test to ensure no race conditions when making many concurrent lock requests
	manager, resourceID := setUp(t)

	var wg sync.WaitGroup

	hold := make(chan struct{})

	// Make 100 simultaneous requests for locks
	numErrors := 0
	numReservations := 0
	for i := 0; i < 100; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			<-hold // try to read from channel to block the goroutine
			reservation, err := manager.Lock(resourceID)
			if err != nil {
				numErrors++
			}
			if reservation != nil {
				numReservations++
			}
		}()
	}
	close(hold) // close channel so all goroutines make requests at once
	wg.Wait()

	assert.Equal(t, numReservations, 1, "Expected only one reservation to be made")
	assert.Equal(t, numErrors, 99, "Expected 99 errors")
}

func TestReservationTTL(t *testing.T) {
	manager, resourceID := setUp(t)

	// This setup would never happen in production, but simulates orphaned reservations
	// being left in redis if (for example) the process dies unexpectedly without releasing
	// the reservation
	manager.Heartbeat = 100 * time.Second
	manager.TTL = 1 * time.Second

	// Create a reservation
	reservation, err := manager.Lock(resourceID)
	assert.Nil(t, err)

	// Assert we can make a new reservation after TTL expires
	time.Sleep(2 * time.Second)
	reservation, err = manager.Lock(resourceID)
	assert.NotNil(t, reservation)
	assert.Nil(t, err)
}

func TestReservationExtend(t *testing.T) {
	manager, resourceID := setUp(t)

	manager.Heartbeat = 500 * time.Millisecond
	manager.TTL = 1 * time.Second

	// Create a reservation
	reservation, err := manager.Lock(resourceID)
	assert.Nil(t, err)

	// Assert we cannot make a new reservation because TTL has been extended
	time.Sleep(2 * time.Second)
	reservation, err = manager.Lock(resourceID)
	assert.Nil(t, reservation)
	assert.EqualError(t, err, fmt.Sprintf("Reservation already exists for resource %s", resourceID))
}
<commit_msg>race condition test: check for exact error message<commit_after>package reservation

import (
	"fmt"
	"os"
	"sync"
	"testing"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/stretchr/testify/assert"
)

var redisTestURL = os.Getenv("REDIS_TEST_URL")

func TestNewManager(t *testing.T) {
	manager, err := NewManager(redisTestURL, "test-worker")
	assert.Nil(t, err)
	// If manager is not nil, NewManager was able to succesfully ping local redis
	assert.NotNil(t, manager)
}

func setUp(t *testing.T) (*Manager, string) {
	conn, err := redis.DialTimeout("tcp", redisTestURL, 15*time.Second, 10*time.Second, 10*time.Second)
	message := fmt.Sprintf("Could not connect to redis at %s. Ensure redis is running at this location (maybe locally)?", redisTestURL)
	assert.Nil(t, err, message)
	defer conn.Close()
	conn.Do("FLUSHALL")
	manager, err := NewManager(redisTestURL, "test-worker")
	assert.Nil(t, err)
	resourceID := "12345"
	return manager, resourceID
}

func TestManagerLockCreate(t *testing.T) {
	manager, resourceID := setUp(t)

	// Make a new reservation
	reservation, err := manager.Lock(resourceID)
	assert.Nil(t, err)

	// Assert reservation created with correct field values
	assert.NotNil(t, reservation)

	// Make the same reservation again and ensure error returned
	dupeReservation, err := manager.Lock(resourceID)
	assert.Nil(t, dupeReservation)
	assert.EqualError(t, err, fmt.Sprintf("Reservation already exists for resource %s", resourceID))

	// Release the first reservation and ensure we can make a second reservation
	err = reservation.Release()
	assert.Nil(t, err)
	reservation, err = manager.Lock(resourceID)
	assert.NotNil(t, reservation)
}

func TestManagerLockConcurrentRequests(t *testing.T) {
	// Test to ensure no race conditions when making many concurrent lock requests
	manager, resourceID := setUp(t)

	var wg sync.WaitGroup

	hold := make(chan struct{})

	// Make 100 simultaneous requests for locks
	numErrors := 0
	numReservations := 0
	for i := 0; i < 100; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			<-hold // try to read from channel to block the goroutine
			reservation, err := manager.Lock(resourceID)
			expectedErr := fmt.Sprintf("Reservation already exists for resource %s", resourceID)
			if err != nil && err.Error() == expectedErr {
				numErrors++
			}
			if reservation != nil {
				numReservations++
			}
		}()
	}
	close(hold) // close channel so all goroutines make requests at once
	wg.Wait()

	assert.Equal(t, numReservations, 1, "Expected only one reservation to be made")
	assert.Equal(t, numErrors, 99, "Expected 99 errors")
}

func TestReservationTTL(t *testing.T) {
	manager, resourceID := setUp(t)

	// This setup would never happen in production, but simulates orphaned reservations
	// being left in redis if (for example) the process dies unexpectedly without releasing
	// the reservation
	manager.Heartbeat = 100 * time.Second
	manager.TTL = 1 * time.Second

	// Create a reservation
	reservation, err := manager.Lock(resourceID)
	assert.Nil(t, err)

	// Assert we can make a new reservation after TTL expires
	time.Sleep(2 * time.Second)
	reservation, err = manager.Lock(resourceID)
	assert.NotNil(t, reservation)
	assert.Nil(t, err)
}

func TestReservationExtend(t *testing.T) {
	manager, resourceID := setUp(t)

	manager.Heartbeat = 500 * time.Millisecond
	manager.TTL = 1 * time.Second

	// Create a reservation
	reservation, err := manager.Lock(resourceID)
	assert.Nil(t, err)

	// Assert we cannot make a new reservation because TTL has been extended
	time.Sleep(2 * time.Second)
	reservation, err = manager.Lock(resourceID)
	assert.Nil(t, reservation)
	assert.EqualError(t, err, fmt.Sprintf("Reservation already exists for resource %s", resourceID))
}
<|endoftext|>


<commit_before>package firehosenozzle

import (
	"crypto/tls"
	"time"

	"github.com/cloudfoundry/noaa/consumer"
	noaerrors "github.com/cloudfoundry/noaa/errors"
	"github.com/cloudfoundry/sonde-go/events"
	"github.com/gorilla/websocket"
	"github.com/prometheus/common/log"

	"github.com/cloudfoundry-community/firehose_exporter/metrics"
)

type FirehoseNozzle struct {
	url                string
	skipSSLValidation  bool
	subscriptionID     string
	idleTimeout        time.Duration
	minRetryDelay      time.Duration
	maxRetryDelay      time.Duration
	maxRetryCount      int
	authTokenRefresher consumer.TokenRefresher
	metricsStore       *metrics.Store
	errs               <-chan error
	messages           <-chan *events.Envelope
	consumer           *consumer.Consumer
}

func New(
	url string,
	skipSSLValidation bool,
	subscriptionID string,
	idleTimeout time.Duration,
	minRetryDelay time.Duration,
	maxRetryDelay time.Duration,
	maxRetryCount int,
	authTokenRefresher consumer.TokenRefresher,
	metricsStore *metrics.Store,
) *FirehoseNozzle {
	return &FirehoseNozzle{
		url:                url,
		skipSSLValidation:  skipSSLValidation,
		subscriptionID:     subscriptionID,
		idleTimeout:        idleTimeout,
		minRetryDelay:      minRetryDelay,
		maxRetryDelay:      maxRetryDelay,
		maxRetryCount:      maxRetryCount,
		authTokenRefresher: authTokenRefresher,
		metricsStore:       metricsStore,
		errs:               make(<-chan error),
		messages:           make(<-chan *events.Envelope),
	}
}

func (n *FirehoseNozzle) Start() error {
	log.Info("Starting Firehose Nozzle...")
	n.consumeFirehose()
	err := n.parseEnvelopes()
	log.Info("Firehose Nozzle shutting down...")
	return err
}

func (n *FirehoseNozzle) consumeFirehose() {
	n.consumer = consumer.New(
		n.url,
		&tls.Config{InsecureSkipVerify: n.skipSSLValidation},
		nil,
	)
	n.consumer.RefreshTokenFrom(n.authTokenRefresher)
	n.consumer.SetDebugPrinter(DebugPrinter{})
	if n.idleTimeout > 0 {
		n.consumer.SetIdleTimeout(n.idleTimeout)
	}
	if n.minRetryDelay > 0 {
		n.consumer.SetMinRetryDelay(n.minRetryDelay)
	}
	if n.maxRetryDelay > 0 {
		n.consumer.SetMaxRetryDelay(n.maxRetryDelay)
	}
	if n.maxRetryCount > 0 {
		n.consumer.SetMaxRetryCount(n.maxRetryCount)
	}
	n.messages, n.errs = n.consumer.Firehose(n.subscriptionID, "")
}

func (n *FirehoseNozzle) parseEnvelopes() error {
	for {
		select {
		case envelope := <-n.messages:
			n.handleMessage(envelope)
			n.metricsStore.AddMetric(envelope)
		case err := <-n.errs:
			retryError := n.handleError(err)
			if !retryError {
				return err
			}
		}
	}
}

func (n *FirehoseNozzle) handleMessage(envelope *events.Envelope) {
	if envelope.GetEventType() == events.Envelope_CounterEvent && envelope.CounterEvent.GetName() == "TruncatingBuffer.DroppedMessages" && envelope.GetOrigin() == "doppler" {
		log.Infof("We've intercepted an upstream message which indicates that the Nozzle or the TrafficController is not keeping up. Please try scaling up the Nozzle.")
		n.metricsStore.AlertSlowConsumerError()
	}
}

func (n *FirehoseNozzle) handleError(err error) bool {
	log.Errorf("Error while reading from the Firehose: %v", err)

	switch err.(type) {
	case noaerrors.RetryError:
		switch noaRetryError := err.(noaerrors.RetryError).Err.(type) {
		case *websocket.CloseError:
			switch noaRetryError.Code {
			case websocket.ClosePolicyViolation:
				log.Errorf("Nozzle couldn't keep up. Please try scaling up the Nozzle.")
				n.metricsStore.AlertSlowConsumerError()
			}
		}
		return true
	}

	log.Info("Closing connection with Firehose...")
	n.consumer.Close()

	return false
}

type DebugPrinter struct{}

func (dp DebugPrinter) Print(title, dump string) {
	log.Debugf("%s: %s", title, dump)
}
<commit_msg>Subscribe to only to metrics feed from firehose<commit_after>package firehosenozzle

import (
	"crypto/tls"
	"time"

	"github.com/cloudfoundry/noaa/consumer"
	noaerrors "github.com/cloudfoundry/noaa/errors"
	"github.com/cloudfoundry/sonde-go/events"
	"github.com/gorilla/websocket"
	"github.com/prometheus/common/log"

	"github.com/cloudfoundry-community/firehose_exporter/metrics"
)

type FirehoseNozzle struct {
	url                string
	skipSSLValidation  bool
	subscriptionID     string
	idleTimeout        time.Duration
	minRetryDelay      time.Duration
	maxRetryDelay      time.Duration
	maxRetryCount      int
	authTokenRefresher consumer.TokenRefresher
	metricsStore       *metrics.Store
	errs               <-chan error
	messages           <-chan *events.Envelope
	consumer           *consumer.Consumer
}

func New(
	url string,
	skipSSLValidation bool,
	subscriptionID string,
	idleTimeout time.Duration,
	minRetryDelay time.Duration,
	maxRetryDelay time.Duration,
	maxRetryCount int,
	authTokenRefresher consumer.TokenRefresher,
	metricsStore *metrics.Store,
) *FirehoseNozzle {
	return &FirehoseNozzle{
		url:                url,
		skipSSLValidation:  skipSSLValidation,
		subscriptionID:     subscriptionID,
		idleTimeout:        idleTimeout,
		minRetryDelay:      minRetryDelay,
		maxRetryDelay:      maxRetryDelay,
		maxRetryCount:      maxRetryCount,
		authTokenRefresher: authTokenRefresher,
		metricsStore:       metricsStore,
		errs:               make(<-chan error),
		messages:           make(<-chan *events.Envelope),
	}
}

func (n *FirehoseNozzle) Start() error {
	log.Info("Starting Firehose Nozzle...")
	n.consumeFirehose()
	err := n.parseEnvelopes()
	log.Info("Firehose Nozzle shutting down...")
	return err
}

func (n *FirehoseNozzle) consumeFirehose() {
	n.consumer = consumer.New(
		n.url,
		&tls.Config{InsecureSkipVerify: n.skipSSLValidation},
		nil,
	)
	n.consumer.RefreshTokenFrom(n.authTokenRefresher)
	n.consumer.SetDebugPrinter(DebugPrinter{})
	if n.idleTimeout > 0 {
		n.consumer.SetIdleTimeout(n.idleTimeout)
	}
	if n.minRetryDelay > 0 {
		n.consumer.SetMinRetryDelay(n.minRetryDelay)
	}
	if n.maxRetryDelay > 0 {
		n.consumer.SetMaxRetryDelay(n.maxRetryDelay)
	}
	if n.maxRetryCount > 0 {
		n.consumer.SetMaxRetryCount(n.maxRetryCount)
	}
	n.messages, n.errs = n.consumer.FilteredFirehose(n.subscriptionID, "", consumer.Metrics)
}

func (n *FirehoseNozzle) parseEnvelopes() error {
	for {
		select {
		case envelope := <-n.messages:
			n.handleMessage(envelope)
			n.metricsStore.AddMetric(envelope)
		case err := <-n.errs:
			retryError := n.handleError(err)
			if !retryError {
				return err
			}
		}
	}
}

func (n *FirehoseNozzle) handleMessage(envelope *events.Envelope) {
	if envelope.GetEventType() == events.Envelope_CounterEvent && envelope.CounterEvent.GetName() == "TruncatingBuffer.DroppedMessages" && envelope.GetOrigin() == "doppler" {
		log.Infof("We've intercepted an upstream message which indicates that the Nozzle or the TrafficController is not keeping up. Please try scaling up the Nozzle.")
		n.metricsStore.AlertSlowConsumerError()
	}
}

func (n *FirehoseNozzle) handleError(err error) bool {
	log.Errorf("Error while reading from the Firehose: %v", err)

	switch err.(type) {
	case noaerrors.RetryError:
		switch noaRetryError := err.(noaerrors.RetryError).Err.(type) {
		case *websocket.CloseError:
			switch noaRetryError.Code {
			case websocket.ClosePolicyViolation:
				log.Errorf("Nozzle couldn't keep up. Please try scaling up the Nozzle.")
				n.metricsStore.AlertSlowConsumerError()
			}
		}
		return true
	}

	log.Info("Closing connection with Firehose...")
	n.consumer.Close()

	return false
}

type DebugPrinter struct{}

func (dp DebugPrinter) Print(title, dump string) {
	log.Debugf("%s: %s", title, dump)
}
<|endoftext|>


<commit_before>package file

import (
	"fmt"
	"io"
	"os"
)

//IsFileExists 目录或文件是否存在
func IsFileExists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}
	return true, nil
}

//IsFile 判断是否为文件
func IsFile(path string) (bool, error) {
	info, err := os.Stat(path)
	if err != nil {
		return false, err
	}
	return info.Mode().IsRegular(), nil
}

//CopyFile 文件拷贝,优先使用hard link
func CopyFile(src, dest string) (err error) {
	srcInfo, err := os.Stat(src)
	if err != nil {
		return
	}
	if !srcInfo.Mode().IsRegular() {
		return fmt.Errorf("CopyFile: non-regular source file %s (%q)", srcInfo.Name(), srcInfo.Mode().String())
	}
	destInfo, err := os.Stat(dest)
	if err != nil {
		if !os.IsNotExist(err) {
			return
		}
	} else {
		if !(destInfo.Mode().IsRegular()) {
			return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", destInfo.Name(), destInfo.Mode().String())
		}
		if os.SameFile(srcInfo, destInfo) {
			return
		}
	}
	if err = os.Link(src, dest); err == nil {
		return
	}
	err = CopyFileContents(src, dest)
	return
}

//CopyFileContents 文件内容拷贝
func CopyFileContents(src, dest string) (err error) {
	in, err := os.Open(src)
	if err != nil {
		return
	}
	defer in.Close()
	out, err := os.Create(dest)
	if err != nil {
		return
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return
	}
	err = out.Sync()
	return
}
<commit_msg>更新部分方法命名<commit_after>package file

import (
	"fmt"
	"io"
	"os"
)

//IsExists 目录或文件是否存在
func IsExists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}
	return true, nil
}

//IsFile 判断是否为文件
func IsFile(path string) (bool, error) {
	info, err := os.Stat(path)
	if err != nil {
		return false, err
	}
	return info.Mode().IsRegular(), nil
}

//Copy 文件拷贝,优先使用hard link
func Copy(src, dest string) (err error) {
	srcInfo, err := os.Stat(src)
	if err != nil {
		return
	}
	if !srcInfo.Mode().IsRegular() {
		return fmt.Errorf("CopyFile: non-regular source file %s (%q)", srcInfo.Name(), srcInfo.Mode().String())
	}
	destInfo, err := os.Stat(dest)
	if err != nil {
		if !os.IsNotExist(err) {
			return
		}
	} else {
		if !(destInfo.Mode().IsRegular()) {
			return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", destInfo.Name(), destInfo.Mode().String())
		}
		if os.SameFile(srcInfo, destInfo) {
			return
		}
	}
	if err = os.Link(src, dest); err == nil {
		return
	}
	err = CopyContents(src, dest)
	return
}

//CopyFileContents 文件内容拷贝
func CopyContents(src, dest string) (err error) {
	in, err := os.Open(src)
	if err != nil {
		return
	}
	defer in.Close()
	out, err := os.Create(dest)
	if err != nil {
		return
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return
	}
	err = out.Sync()
	return
}
<|endoftext|>


<commit_before>package file

import "strings"
import "regexp"

type Line []rune

func (line Line) Dup() Line {
	strLine := string(line)
	return Line(strLine)
}

func (line Line) ToString() string {
	return string(line)
}

func (line Line) CommonStart(other Line) Line {
	for k, r := range line {
		if k >= len(other) || other[k] != r {
			return line[:k].Dup()
		}
	}
	return line.Dup()
}

func (line Line) Search(term string, start, end int) (int, int) {
	if len(line) == 0 {
		return -1, -1
	}
	if end < 0 || end >= len(line) {
		end = len(line) + end
		if end < 0 || end < start {
			return -1, -1
		}
	}
	n := len(term)
	var startCol, endCol int
	target := string(line[start : end+1])
	if term[0:1] == "/" && term[n-1:n] == "/" {
		re, err := regexp.Compile(term[1 : n-1])
		if err != nil {
			return -1, -1
		}
		cols := re.FindStringIndex(target)
		if cols == nil {
			return -1, -1
		}
		startCol = cols[0]
		endCol = cols[1]
	} else {
		strLine := strings.ToLower(target)
		term = strings.ToLower(term)
		startCol = strings.Index(strLine, term)
		endCol = startCol + len(term)
	}
	if startCol < 0 {
		return startCol, endCol
	} else {
		return startCol + start, endCol + start
	}
}

func (line Line) tabs2spaces() Line {
	strLine := string(line)
	strLine = strings.Replace(strLine, "\t", "    ", -1)
	return Line(strLine)
}
<commit_msg>cleanup file/line<commit_after>package file

import "strings"
import "regexp"

type Line []rune

// Dup returns a new Line with the same content.
func (line Line) Dup() Line {
	strLine := string(line)
	return Line(strLine)
}

// ToString converts a Line into a string.
func (line Line) ToString() string {
	return string(line)
}

// CommonStart returns the sub-line that is common between
// two lines.
func (line Line) CommonStart(other Line) Line {
	for k, r := range line {
		if k >= len(other) || other[k] != r {
			return line[:k].Dup()
		}
	}
	return line.Dup()
}

// Search returns the start/end positions for a search term.
// A -1 indicates no match.
func (line Line) Search(term string, start, end int) (int, int) {

	// if line is empty, there is no match.
	if len(line) == 0 {
		return -1, -1
	}

	// Negative end indicates "from end of line".
	if end < 0 || end >= len(line) {
		end = len(line) + end
		if end < 0 || end < start {
			return -1, -1
		}
	}

	return line.search(term, start, end)

}

func isRegex(term string) bool {
	n := len(term)
	return (term[0:1] == "/" && term[n-1:n] == "/")
}

func (line Line) search(term string, start, end int) (int, int) {

	n := len(term)
	var startCol, endCol int
	target := string(line[start : end+1])

	if isRegex(term) {
		re, err := regexp.Compile(term[1 : n-1])
		if err != nil {
			return -1, -1
		}
		cols := re.FindStringIndex(target)
		if cols == nil {
			return -1, -1
		}
		startCol = cols[0]
		endCol = cols[1]
	} else {
		strLine := strings.ToLower(target)
		term = strings.ToLower(term)
		startCol = strings.Index(strLine, term)
		endCol = startCol + len(term)
	}
	if startCol < 0 {
		return startCol, endCol
	} else {
		return startCol + start, endCol + start
	}
}

func (line Line) tabs2spaces() Line {
	strLine := string(line)
	strLine = strings.Replace(strLine, "\t", "    ", -1)
	return Line(strLine)
}
<|endoftext|>


<commit_before>package anidb

import (
	"fmt"
	"github.com/Kovensky/go-anidb/misc"
	"github.com/Kovensky/go-anidb/udp"
	"github.com/Kovensky/go-fscache"
	"image"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

var _ cacheable = &File{}

func (f *File) setCachedTS(ts time.Time) {
	f.Cached = ts
}

func (f *File) IsStale() bool {
	if f == nil {
		return true
	}
	if f.Incomplete {
		return time.Now().Sub(f.Cached) > FileIncompleteCacheDuration
	}
	return time.Now().Sub(f.Cached) > FileCacheDuration
}

func cacheFile(f *File) {
	CacheSet(f.FID, "fid", "by-ed2k", f.Ed2kHash, f.Filesize)
	CacheSet(f, "fid", f.FID)
}

type FID int

func (fid FID) File() *File {
	var f File
	if CacheGet(&f, "fid", fid) == nil {
		return &f
	}
	return nil
}

// Prefetches the Anime, Episode and Group that this
// file is linked to using the given AniDB instance.
//
// Returns a channel where this file will be sent to
// when the prefetching is done; if the file is nil,
// the channel will return nil.
func (f *File) Prefetch(adb *AniDB) <-chan *File {
	ch := make(chan *File, 1)
	go func() {
		if f != nil {
			a := adb.AnimeByID(f.AID)
			g := adb.GroupByID(f.GID)
			<-a
			<-g
			ch <- f
		}
		close(ch)
	}()
	return ch
}

// Retrieves a File by its FID. Uses the UDP API.
func (adb *AniDB) FileByID(fid FID) <-chan *File {
	key := []fscache.CacheKey{"fid", fid}

	ch := make(chan *File, 1)

	if fid < 1 {
		ch <- nil
		close(ch)
		return ch
	}

	ic := make(chan notification, 1)
	go func() { ch <- (<-ic).(*File); close(ch) }()
	if intentMap.Intent(ic, key...) {
		return ch
	}

	if !Cache.IsValid(InvalidKeyCacheDuration, key...) {
		intentMap.NotifyClose((*File)(nil), key...)
		return ch
	}

	f := fid.File()
	if !f.IsStale() {
		intentMap.NotifyClose(f, key...)
		return ch
	}

	go func() {
		reply := <-adb.udp.SendRecv("FILE",
			paramMap{
				"fid":   fid,
				"fmask": fileFmask,
				"amask": fileAmask,
			})

		if reply.Error() == nil {
			adb.parseFileResponse(&f, reply, false)

			cacheFile(f)
		} else if reply.Code() == 320 {
			Cache.SetInvalid(key...)
		}

		intentMap.NotifyClose(f, key...)
	}()
	return ch
}

var validEd2kHash = regexp.MustCompile(`\A[[:xdigit:]]{32}\z`)

// Retrieves a File by its Ed2kHash + Filesize combination. Uses the UDP API.
func (adb *AniDB) FileByEd2kSize(ed2k string, size int64) <-chan *File {
	key := []fscache.CacheKey{"fid", "by-ed2k", ed2k, size}

	ch := make(chan *File, 1)

	if size < 1 || !validEd2kHash.MatchString(ed2k) {
		ch <- nil
		close(ch)
		return ch
	}
	// AniDB always uses lower case hashes
	ed2k = strings.ToLower(ed2k)

	ic := make(chan notification, 1)
	go func() {
		fid := (<-ic).(FID)
		if fid > 0 {
			ch <- <-adb.FileByID(fid)
		}
		close(ch)
	}()
	if intentMap.Intent(ic, key...) {
		return ch
	}

	if !Cache.IsValid(InvalidKeyCacheDuration, key...) {
		intentMap.NotifyClose(FID(0), key...)
		return ch
	}

	fid := FID(0)

	switch ts, err := Cache.Get(&fid, key...); {
	case err == nil && time.Now().Sub(ts) < FileCacheDuration:
		intentMap.NotifyClose(fid, key...)
		return ch
	}

	go func() {
		reply := <-adb.udp.SendRecv("FILE",
			paramMap{
				"ed2k":  ed2k,
				"size":  size,
				"fmask": fileFmask,
				"amask": fileAmask,
			})

		var f *File
		if reply.Error() == nil {
			adb.parseFileResponse(&f, reply, false)

			fid = f.FID

			cacheFile(f)
		} else if reply.Code() == 320 { // file not found
			Cache.SetInvalid(key...)
		} else if reply.Code() == 322 { // multiple files found
			panic("Don't know what to do with " + strings.Join(reply.Lines(), "\n"))
		}

		intentMap.NotifyClose(fid, key...)
	}()
	return ch
}

var fileFmask = "7fda7fe8"
var fileAmask = "00008000"

const (
	stateCRCOK = 1 << iota
	stateCRCERR
	stateV2
	stateV3
	stateV4
	stateV5
	stateUncensored
	stateCensored
)

func sanitizeCodec(codec string) string {
	switch codec {
	case "MP3 CBR":
		return "MP3"
	case "WMV9 (also WMV3)":
		return "WMV9"
	case "Ogg (Vorbis)":
		return "Vorbis"
	case "H264/AVC":
		return "H.264"
	}
	return codec
}

var opedRE = regexp.MustCompile(`\A(Opening|Ending)(?: (\d+))?\z`)

func (adb *AniDB) parseFileResponse(f **File, reply udpapi.APIReply, calledFromFIDsByGID bool) bool {
	if reply.Error() != nil {
		return false
	}
	if reply.Truncated() {
		panic("Truncated")
	}

	uidChan := make(chan UID, 1)
	if adb.udp.credentials != nil {
		go func() { uidChan <- <-adb.GetUserUID(decrypt(adb.udp.credentials.username)) }()
	} else {
		uidChan <- 0
		close(uidChan)
	}

	parts := strings.Split(reply.Lines()[1], "|")
	ints := make([]int64, len(parts))
	for i, p := range parts {
		ints[i], _ = strconv.ParseInt(p, 10, 64)
	}

	partial := false

	rels := strings.Split(parts[5], "'")
	relList := make([]EID, 0, len(parts[5]))
	related := make(RelatedEpisodes, len(parts[5]))
	for _, rel := range rels {
		r := strings.Split(rel, ",")
		if len(r) < 2 {
			continue
		}

		eid, _ := strconv.ParseInt(r[0], 10, 32)
		pct, _ := strconv.ParseInt(r[1], 10, 32)
		relList = append(relList, EID(eid))
		related[EID(eid)] = float32(pct) / 100

		if pct != 100 {
			partial = true
		}
	}

	epno := misc.ParseEpisodeList(parts[24])
	fid := FID(ints[0])
	aid := AID(ints[1])
	eid := EID(ints[2])
	gid := GID(ints[3])
	lid := LID(ints[4])

	if !epno[0].Start.ContainsEpisodes(epno[0].End) || len(epno) > 1 || len(relList) > 0 {
		// epno is broken -- we need to sanitize it
		thisEp := <-adb.EpisodeByID(eid)
		bad := false
		if thisEp != nil {
			parts := make([]string, 1, len(relList)+1)
			parts[0] = thisEp.Episode.String()

			// everything after this SHOULD be cache hits now, unless this is somehow
			// linked with an EID from a different anime (*stares at Haruhi*).
			// We don't want to use eps from different AIDs anyway, so that makes
			// the job easier.

			// We check if the related episodes are all in sequence from this one.
			// If they are, we build a new epno with the sequence. Otherwise,
			// our epno will only have the primary episode.

			// gather the episode numbers
			for _, eid := range relList {
				if ep := eid.Episode(); ep != nil && ep.AID == thisEp.AID {
					parts = append(parts, ep.Episode.String())
				} else {
					bad = true
					break
				}
			}

			test := misc.EpisodeList{}
			// only if we didn't break the loop
			if !bad {
				test = misc.ParseEpisodeList(strings.Join(parts, ","))
			}

			if partial {
				if calledFromFIDsByGID {
					epno = test
					adb.Logger.Printf("UDP!!! FID %d is only part of episode %s with no complementary files", fid, epno)
				} else if len(test) == 1 && test[0].Start.Number == test[0].End.Number {
					fids := []int{}

					for fid := range adb.FIDsByGID(thisEp, gid) {
						fids = append(fids, int(fid))
					}
					if len(fids) >= 1 && fids[0] == 0 {
						fids = fids[1:]
						// Only entry was API error
						if len(fids) == 0 {
							return false
						}
					}
					sort.Sort(sort.IntSlice(fids))
					idx := sort.SearchInts(fids, int(fid))
					if idx == len(fids) {
						panic(fmt.Sprintf("FID %d couldn't locate itself", fid))
					}

					epno = test

					// equate pointers
					epno[0].End = epno[0].Start

					epno[0].Start.Parts = len(fids)
					epno[0].Start.Part = idx
				} else {
					panic(fmt.Sprintf("Don't know what to do with partial episode %s (EID %d)", test, eid))
				}
			} else {
				// if they're all in sequence, then we'll only have a single range in the list
				if len(test) == 1 {
					epno = test
				} else {
					// use only the primary epno then
					epno = misc.ParseEpisodeList(thisEp.Episode.String())
				}
			}
		}
	}

	epstr := epno.String()
	if len(epno) == 1 && epno[0].Type == misc.EpisodeTypeCredits && epno[0].Len() == 1 {
		typ := ""
		n := 0

		if ep := <-adb.EpisodeByID(eid); ep == nil {
		} else if m := opedRE.FindStringSubmatch(ep.Titles["en"]); len(m) > 2 {
			num, err := strconv.ParseInt(m[2], 10, 32)
			if err == nil {
				n = int(num)
			}

			typ = m[1]
		}

		gobi := fmt.Sprintf("%d", n)
		if n == 0 {
			gobi = ""
		}

		switch typ {
		case "Opening":
			epstr = "OP" + gobi
		case "Ending":
			epstr = "ED" + gobi
		}
	}

	version := FileVersion(1)
	switch i := ints[7]; {
	case i&stateV5 != 0:
		version = 5
	case i&stateV4 != 0:
		version = 4
	case i&stateV3 != 0:
		version = 3
	case i&stateV2 != 0:
		version = 2
	}

	codecs := strings.Split(parts[14], "'")
	bitrates := strings.Split(parts[15], "'")
	alangs := strings.Split(parts[20], "'")
	streams := make([]AudioStream, len(codecs))
	for i := range streams {
		br, _ := strconv.ParseInt(bitrates[i], 10, 32)
		streams[i] = AudioStream{
			Bitrate:  int(br),
			Codec:    sanitizeCodec(codecs[i]),
			Language: Language(alangs[i]),
		}
	}

	sl := strings.Split(parts[21], "'")
	slangs := make([]Language, len(sl))
	for i := range sl {
		slangs[i] = Language(sl[i])
	}

	depth := int(ints[12])
	if depth == 0 {
		depth = 8
	}
	res := strings.Split(parts[18], "x")
	width, _ := strconv.ParseInt(res[0], 10, 32)
	height, _ := strconv.ParseInt(res[1], 10, 32)
	video := VideoInfo{
		Bitrate:    int(ints[17]),
		Codec:      sanitizeCodec(parts[16]),
		ColorDepth: depth,
		Resolution: image.Rect(0, 0, int(width), int(height)),
	}

	lidMap := LIDMap{}
	if *f != nil {
		lidMap = (*f).LID
	}

	uid := <-uidChan
	if uid != 0 {
		lidMap[uid] = lid
	}

	*f = &File{
		FID: fid,

		AID: aid,
		EID: eid,
		GID: gid,
		LID: lidMap,

		EpisodeString: epstr,
		EpisodeNumber: epno,

		RelatedEpisodes: related,
		Deprecated:      ints[6] != 0,

		CRCMatch:   ints[7]&stateCRCOK != 0,
		BadCRC:     ints[7]&stateCRCERR != 0,
		Version:    version,
		Uncensored: ints[7]&stateUncensored != 0,
		Censored:   ints[7]&stateCensored != 0,

		Incomplete: video.Resolution.Empty(),

		Filesize: ints[8],
		Ed2kHash: parts[9],
		SHA1Hash: parts[10],
		CRC32:    parts[11],

		Source: FileSource(parts[13]),

		AudioStreams:      streams,
		SubtitleLanguages: slangs,
		VideoInfo:         video,
		FileExtension:     parts[19],

		Length:  time.Duration(ints[22]) * time.Second,
		AirDate: time.Unix(ints[23], 0),
	}
	return true
}
<commit_msg>anidb: Also cache the EID-AID relationship when caching FILE results<commit_after>package anidb

import (
	"fmt"
	"github.com/Kovensky/go-anidb/misc"
	"github.com/Kovensky/go-anidb/udp"
	"github.com/Kovensky/go-fscache"
	"image"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

var _ cacheable = &File{}

func (f *File) setCachedTS(ts time.Time) {
	f.Cached = ts
}

func (f *File) IsStale() bool {
	if f == nil {
		return true
	}
	if f.Incomplete {
		return time.Now().Sub(f.Cached) > FileIncompleteCacheDuration
	}
	return time.Now().Sub(f.Cached) > FileCacheDuration
}

func cacheFile(f *File) {
	CacheSet(f.AID, "aid", "by-eid", f.EID)
	CacheSet(f.FID, "fid", "by-ed2k", f.Ed2kHash, f.Filesize)
	CacheSet(f, "fid", f.FID)
}

type FID int

func (fid FID) File() *File {
	var f File
	if CacheGet(&f, "fid", fid) == nil {
		return &f
	}
	return nil
}

// Prefetches the Anime, Episode and Group that this
// file is linked to using the given AniDB instance.
//
// Returns a channel where this file will be sent to
// when the prefetching is done; if the file is nil,
// the channel will return nil.
func (f *File) Prefetch(adb *AniDB) <-chan *File {
	ch := make(chan *File, 1)
	go func() {
		if f != nil {
			a := adb.AnimeByID(f.AID)
			g := adb.GroupByID(f.GID)
			<-a
			<-g
			ch <- f
		}
		close(ch)
	}()
	return ch
}

// Retrieves a File by its FID. Uses the UDP API.
func (adb *AniDB) FileByID(fid FID) <-chan *File {
	key := []fscache.CacheKey{"fid", fid}

	ch := make(chan *File, 1)

	if fid < 1 {
		ch <- nil
		close(ch)
		return ch
	}

	ic := make(chan notification, 1)
	go func() { ch <- (<-ic).(*File); close(ch) }()
	if intentMap.Intent(ic, key...) {
		return ch
	}

	if !Cache.IsValid(InvalidKeyCacheDuration, key...) {
		intentMap.NotifyClose((*File)(nil), key...)
		return ch
	}

	f := fid.File()
	if !f.IsStale() {
		intentMap.NotifyClose(f, key...)
		return ch
	}

	go func() {
		reply := <-adb.udp.SendRecv("FILE",
			paramMap{
				"fid":   fid,
				"fmask": fileFmask,
				"amask": fileAmask,
			})

		if reply.Error() == nil {
			adb.parseFileResponse(&f, reply, false)

			cacheFile(f)
		} else if reply.Code() == 320 {
			Cache.SetInvalid(key...)
		}

		intentMap.NotifyClose(f, key...)
	}()
	return ch
}

var validEd2kHash = regexp.MustCompile(`\A[[:xdigit:]]{32}\z`)

// Retrieves a File by its Ed2kHash + Filesize combination. Uses the UDP API.
func (adb *AniDB) FileByEd2kSize(ed2k string, size int64) <-chan *File {
	key := []fscache.CacheKey{"fid", "by-ed2k", ed2k, size}

	ch := make(chan *File, 1)

	if size < 1 || !validEd2kHash.MatchString(ed2k) {
		ch <- nil
		close(ch)
		return ch
	}
	// AniDB always uses lower case hashes
	ed2k = strings.ToLower(ed2k)

	ic := make(chan notification, 1)
	go func() {
		fid := (<-ic).(FID)
		if fid > 0 {
			ch <- <-adb.FileByID(fid)
		}
		close(ch)
	}()
	if intentMap.Intent(ic, key...) {
		return ch
	}

	if !Cache.IsValid(InvalidKeyCacheDuration, key...) {
		intentMap.NotifyClose(FID(0), key...)
		return ch
	}

	fid := FID(0)

	switch ts, err := Cache.Get(&fid, key...); {
	case err == nil && time.Now().Sub(ts) < FileCacheDuration:
		intentMap.NotifyClose(fid, key...)
		return ch
	}

	go func() {
		reply := <-adb.udp.SendRecv("FILE",
			paramMap{
				"ed2k":  ed2k,
				"size":  size,
				"fmask": fileFmask,
				"amask": fileAmask,
			})

		var f *File
		if reply.Error() == nil {
			adb.parseFileResponse(&f, reply, false)

			fid = f.FID

			cacheFile(f)
		} else if reply.Code() == 320 { // file not found
			Cache.SetInvalid(key...)
		} else if reply.Code() == 322 { // multiple files found
			panic("Don't know what to do with " + strings.Join(reply.Lines(), "\n"))
		}

		intentMap.NotifyClose(fid, key...)
	}()
	return ch
}

var fileFmask = "7fda7fe8"
var fileAmask = "00008000"

const (
	stateCRCOK = 1 << iota
	stateCRCERR
	stateV2
	stateV3
	stateV4
	stateV5
	stateUncensored
	stateCensored
)

func sanitizeCodec(codec string) string {
	switch codec {
	case "MP3 CBR":
		return "MP3"
	case "WMV9 (also WMV3)":
		return "WMV9"
	case "Ogg (Vorbis)":
		return "Vorbis"
	case "H264/AVC":
		return "H.264"
	}
	return codec
}

var opedRE = regexp.MustCompile(`\A(Opening|Ending)(?: (\d+))?\z`)

func (adb *AniDB) parseFileResponse(f **File, reply udpapi.APIReply, calledFromFIDsByGID bool) bool {
	if reply.Error() != nil {
		return false
	}
	if reply.Truncated() {
		panic("Truncated")
	}

	uidChan := make(chan UID, 1)
	if adb.udp.credentials != nil {
		go func() { uidChan <- <-adb.GetUserUID(decrypt(adb.udp.credentials.username)) }()
	} else {
		uidChan <- 0
		close(uidChan)
	}

	parts := strings.Split(reply.Lines()[1], "|")
	ints := make([]int64, len(parts))
	for i, p := range parts {
		ints[i], _ = strconv.ParseInt(p, 10, 64)
	}

	partial := false

	rels := strings.Split(parts[5], "'")
	relList := make([]EID, 0, len(parts[5]))
	related := make(RelatedEpisodes, len(parts[5]))
	for _, rel := range rels {
		r := strings.Split(rel, ",")
		if len(r) < 2 {
			continue
		}

		eid, _ := strconv.ParseInt(r[0], 10, 32)
		pct, _ := strconv.ParseInt(r[1], 10, 32)
		relList = append(relList, EID(eid))
		related[EID(eid)] = float32(pct) / 100

		if pct != 100 {
			partial = true
		}
	}

	epno := misc.ParseEpisodeList(parts[24])
	fid := FID(ints[0])
	aid := AID(ints[1])
	eid := EID(ints[2])
	gid := GID(ints[3])
	lid := LID(ints[4])

	if !epno[0].Start.ContainsEpisodes(epno[0].End) || len(epno) > 1 || len(relList) > 0 {
		// epno is broken -- we need to sanitize it
		thisEp := <-adb.EpisodeByID(eid)
		bad := false
		if thisEp != nil {
			parts := make([]string, 1, len(relList)+1)
			parts[0] = thisEp.Episode.String()

			// everything after this SHOULD be cache hits now, unless this is somehow
			// linked with an EID from a different anime (*stares at Haruhi*).
			// We don't want to use eps from different AIDs anyway, so that makes
			// the job easier.

			// We check if the related episodes are all in sequence from this one.
			// If they are, we build a new epno with the sequence. Otherwise,
			// our epno will only have the primary episode.

			// gather the episode numbers
			for _, eid := range relList {
				if ep := eid.Episode(); ep != nil && ep.AID == thisEp.AID {
					parts = append(parts, ep.Episode.String())
				} else {
					bad = true
					break
				}
			}

			test := misc.EpisodeList{}
			// only if we didn't break the loop
			if !bad {
				test = misc.ParseEpisodeList(strings.Join(parts, ","))
			}

			if partial {
				if calledFromFIDsByGID {
					epno = test
					adb.Logger.Printf("UDP!!! FID %d is only part of episode %s with no complementary files", fid, epno)
				} else if len(test) == 1 && test[0].Start.Number == test[0].End.Number {
					fids := []int{}

					for fid := range adb.FIDsByGID(thisEp, gid) {
						fids = append(fids, int(fid))
					}
					if len(fids) >= 1 && fids[0] == 0 {
						fids = fids[1:]
						// Only entry was API error
						if len(fids) == 0 {
							return false
						}
					}
					sort.Sort(sort.IntSlice(fids))
					idx := sort.SearchInts(fids, int(fid))
					if idx == len(fids) {
						panic(fmt.Sprintf("FID %d couldn't locate itself", fid))
					}

					epno = test

					// equate pointers
					epno[0].End = epno[0].Start

					epno[0].Start.Parts = len(fids)
					epno[0].Start.Part = idx
				} else {
					panic(fmt.Sprintf("Don't know what to do with partial episode %s (EID %d)", test, eid))
				}
			} else {
				// if they're all in sequence, then we'll only have a single range in the list
				if len(test) == 1 {
					epno = test
				} else {
					// use only the primary epno then
					epno = misc.ParseEpisodeList(thisEp.Episode.String())
				}
			}
		}
	}

	epstr := epno.String()
	if len(epno) == 1 && epno[0].Type == misc.EpisodeTypeCredits && epno[0].Len() == 1 {
		typ := ""
		n := 0

		if ep := <-adb.EpisodeByID(eid); ep == nil {
		} else if m := opedRE.FindStringSubmatch(ep.Titles["en"]); len(m) > 2 {
			num, err := strconv.ParseInt(m[2], 10, 32)
			if err == nil {
				n = int(num)
			}

			typ = m[1]
		}

		gobi := fmt.Sprintf("%d", n)
		if n == 0 {
			gobi = ""
		}

		switch typ {
		case "Opening":
			epstr = "OP" + gobi
		case "Ending":
			epstr = "ED" + gobi
		}
	}

	version := FileVersion(1)
	switch i := ints[7]; {
	case i&stateV5 != 0:
		version = 5
	case i&stateV4 != 0:
		version = 4
	case i&stateV3 != 0:
		version = 3
	case i&stateV2 != 0:
		version = 2
	}

	codecs := strings.Split(parts[14], "'")
	bitrates := strings.Split(parts[15], "'")
	alangs := strings.Split(parts[20], "'")
	streams := make([]AudioStream, len(codecs))
	for i := range streams {
		br, _ := strconv.ParseInt(bitrates[i], 10, 32)
		streams[i] = AudioStream{
			Bitrate:  int(br),
			Codec:    sanitizeCodec(codecs[i]),
			Language: Language(alangs[i]),
		}
	}

	sl := strings.Split(parts[21], "'")
	slangs := make([]Language, len(sl))
	for i := range sl {
		slangs[i] = Language(sl[i])
	}

	depth := int(ints[12])
	if depth == 0 {
		depth = 8
	}
	res := strings.Split(parts[18], "x")
	width, _ := strconv.ParseInt(res[0], 10, 32)
	height, _ := strconv.ParseInt(res[1], 10, 32)
	video := VideoInfo{
		Bitrate:    int(ints[17]),
		Codec:      sanitizeCodec(parts[16]),
		ColorDepth: depth,
		Resolution: image.Rect(0, 0, int(width), int(height)),
	}

	lidMap := LIDMap{}
	if *f != nil {
		lidMap = (*f).LID
	}

	uid := <-uidChan
	if uid != 0 {
		lidMap[uid] = lid
	}

	*f = &File{
		FID: fid,

		AID: aid,
		EID: eid,
		GID: gid,
		LID: lidMap,

		EpisodeString: epstr,
		EpisodeNumber: epno,

		RelatedEpisodes: related,
		Deprecated:      ints[6] != 0,

		CRCMatch:   ints[7]&stateCRCOK != 0,
		BadCRC:     ints[7]&stateCRCERR != 0,
		Version:    version,
		Uncensored: ints[7]&stateUncensored != 0,
		Censored:   ints[7]&stateCensored != 0,

		Incomplete: video.Resolution.Empty(),

		Filesize: ints[8],
		Ed2kHash: parts[9],
		SHA1Hash: parts[10],
		CRC32:    parts[11],

		Source: FileSource(parts[13]),

		AudioStreams:      streams,
		SubtitleLanguages: slangs,
		VideoInfo:         video,
		FileExtension:     parts[19],

		Length:  time.Duration(ints[22]) * time.Second,
		AirDate: time.Unix(ints[23], 0),
	}
	return true
}
<|endoftext|>