<commit_before>package provider

import (
	"errors"
	"net/http"
	"strconv"
	"strings"
	"text/template"
	"time"

	"golang.org/x/net/context"

	"github.com/BurntSushi/ty/fun"
	log "github.com/Sirupsen/logrus"
	"github.com/cenkalti/backoff"
	"github.com/containous/traefik/safe"
	"github.com/containous/traefik/types"
	"github.com/docker/engine-api/client"
	dockertypes "github.com/docker/engine-api/types"
	eventtypes "github.com/docker/engine-api/types/events"
	"github.com/docker/engine-api/types/filters"
	"github.com/docker/go-connections/sockets"
	"github.com/docker/go-connections/tlsconfig"
	"github.com/vdemeester/docker-events"
)

// DockerAPIVersion is a constant holding the version of the Docker API traefik will use
const DockerAPIVersion string = "1.21"

// Docker holds configurations of the Docker provider.
type Docker struct {
	BaseProvider
	Endpoint string     `description:"Docker server endpoint. Can be a tcp or a unix socket endpoint"`
	Domain   string     `description:"Default domain used"`
	TLS      *DockerTLS `description:"Enable Docker TLS support"`
}

// DockerTLS holds TLS specific configurations
type DockerTLS struct {
	CA                 string `description:"TLS CA"`
	Cert               string `description:"TLS cert"`
	Key                string `description:"TLS key"`
	InsecureSkipVerify bool   `description:"TLS insecure skip verify"`
}

func (provider *Docker) createClient() (client.APIClient, error) {
	var httpClient *http.Client
	httpHeaders := map[string]string{
		// FIXME(vdemeester) use version here O:)
		"User-Agent": "Traefik",
	}
	if provider.TLS != nil {
		tlsOptions := tlsconfig.Options{
			CAFile:             provider.TLS.CA,
			CertFile:           provider.TLS.Cert,
			KeyFile:            provider.TLS.Key,
			InsecureSkipVerify: provider.TLS.InsecureSkipVerify,
		}
		config, err := tlsconfig.Client(tlsOptions)
		if err != nil {
			return nil, err
		}
		tr := &http.Transport{
			TLSClientConfig: config,
		}
		proto, addr, _, err := client.ParseHost(provider.Endpoint)
		if err != nil {
			return nil, err
		}

		sockets.ConfigureTransport(tr, proto, addr)

		httpClient = &http.Client{
			Transport: tr,
		}
	}
	return client.NewClient(provider.Endpoint, DockerAPIVersion, httpClient, httpHeaders)
}

// Provide allows the provider to provide configurations to traefik
// using the given configuration channel.
func (provider *Docker) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints []types.Constraint) error {
	provider.Constraints = append(provider.Constraints, constraints...)
	// TODO register this routine in pool, and watch for stop channel
	safe.Go(func() {
		operation := func() error {
			var err error

			dockerClient, err := provider.createClient()
			if err != nil {
				log.Errorf("Failed to create a client for docker, error: %s", err)
				return err
			}

			ctx := context.Background()
			version, err := dockerClient.ServerVersion(ctx)
			log.Debugf("Docker connection established with docker %s (API %s)", version.Version, version.APIVersion)
			containers, err := listContainers(ctx, dockerClient)
			if err != nil {
				log.Errorf("Failed to list containers for docker, error %s", err)
				return err
			}
			configuration := provider.loadDockerConfig(containers)
			configurationChan <- types.ConfigMessage{
				ProviderName:  "docker",
				Configuration: configuration,
			}
			if provider.Watch {
				ctx, cancel := context.WithCancel(ctx)
				f := filters.NewArgs()
				f.Add("type", "container")
				options := dockertypes.EventsOptions{
					Filters: f,
				}
				eventHandler := events.NewHandler(events.ByAction)
				startStopHandle := func(m eventtypes.Message) {
					log.Debugf("Docker event received %+v", m)
					containers, err := listContainers(ctx, dockerClient)
					if err != nil {
						log.Errorf("Failed to list containers for docker, error %s", err)
						// Call cancel to get out of the monitor
						cancel()
						return
					}
					configuration := provider.loadDockerConfig(containers)
					if configuration != nil {
						configurationChan <- types.ConfigMessage{
							ProviderName:  "docker",
							Configuration: configuration,
						}
					}
				}
				eventHandler.Handle("start", startStopHandle)
				eventHandler.Handle("die", startStopHandle)

				errChan := events.MonitorWithHandler(ctx, dockerClient, options, eventHandler)
				pool.Go(func(stop chan bool) {
					for {
						select {
						case <-stop:
							cancel()
							return
						}
					}
				})
				if err := <-errChan; err != nil {
					return err
				}
			}
			return nil
		}
		notify := func(err error, time time.Duration) {
			log.Errorf("Docker connection error %+v, retrying in %s", err, time)
		}
		err := backoff.RetryNotify(operation, backoff.NewExponentialBackOff(), notify)
		if err != nil {
			log.Fatalf("Cannot connect to docker server %+v", err)
		}
	})

	return nil
}

func (provider *Docker) loadDockerConfig(containersInspected []dockertypes.ContainerJSON) *types.Configuration {
	var DockerFuncMap = template.FuncMap{
		"getBackend":        provider.getBackend,
		"getIPAddress":      provider.getIPAddress,
		"getPort":           provider.getPort,
		"getWeight":         provider.getWeight,
		"getDomain":         provider.getDomain,
		"getProtocol":       provider.getProtocol,
		"getPassHostHeader": provider.getPassHostHeader,
		"getPriority":       provider.getPriority,
		"getEntryPoints":    provider.getEntryPoints,
		"getFrontendRule":   provider.getFrontendRule,
		"replace":           replace,
	}

	// filter containers
	filteredContainers := fun.Filter(provider.ContainerFilter, containersInspected).([]dockertypes.ContainerJSON)

	frontends := map[string][]dockertypes.ContainerJSON{}
	for _, container := range filteredContainers {
		frontendName := provider.getFrontendName(container)
		frontends[frontendName] = append(frontends[frontendName], container)
	}

	templateObjects := struct {
		Containers []dockertypes.ContainerJSON
		Frontends  map[string][]dockertypes.ContainerJSON
		Domain     string
	}{
		filteredContainers,
		frontends,
		provider.Domain,
	}

	configuration, err := provider.getConfiguration("templates/docker.tmpl", DockerFuncMap, templateObjects)
	if err != nil {
		log.Error(err)
	}
	return configuration
}

// ContainerFilter checks if container have to be exposed
func (provider *Docker) ContainerFilter(container dockertypes.ContainerJSON) bool {
	_, err := strconv.Atoi(container.Config.Labels["traefik.port"])
	if len(container.NetworkSettings.Ports) == 0 && err != nil {
		log.Debugf("Filtering container without port and no traefik.port label %s", container.Name)
		return false
	}
	if len(container.NetworkSettings.Ports) > 1 && err != nil {
		log.Debugf("Filtering container with more than 1 port and no traefik.port label %s", container.Name)
		return false
	}

	if container.Config.Labels["traefik.enable"] == "false" {
		log.Debugf("Filtering disabled container %s", container.Name)
		return false
	}

	constraintTags := strings.Split(container.Config.Labels["traefik.tags"], ",")
	ok, failingConstraint := provider.MatchConstraints(constraintTags)
	if ok == false {
		if failingConstraint != nil {
			log.Debugf("Container %v pruned by '%v' constraint", container.Name, failingConstraint.String())
		}
		return false
	}

	return true
}

func (provider *Docker) getFrontendName(container dockertypes.ContainerJSON) string {
	// Replace '.' with '-' in quoted keys because of this issue https://github.com/BurntSushi/toml/issues/78
	return normalize(provider.getFrontendRule(container))
}

// GetFrontendRule returns the frontend rule for the specified container, using
// it's label. It returns a default one (Host) if the label is not present.
func (provider *Docker) getFrontendRule(container dockertypes.ContainerJSON) string {
	// ⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠
	// TODO: backwards compatibility with DEPRECATED rule.Value
	if value, ok := container.Config.Labels["traefik.frontend.value"]; ok {
		log.Warnf("Label traefik.frontend.value=%s is DEPRECATED (will be removed in v1.0.0), please refer to the rule label: https://github.com/containous/traefik/blob/master/docs/index.md#docker", value)
		rule, _ := container.Config.Labels["traefik.frontend.rule"]
		return rule + ":" + value
	}
	// ⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠

	if label, err := getLabel(container, "traefik.frontend.rule"); err == nil {
		return label
	}
	return "Host:" + provider.getSubDomain(container.Name) + "." + provider.Domain
}

func (provider *Docker) getBackend(container dockertypes.ContainerJSON) string {
	if label, err := getLabel(container, "traefik.backend"); err == nil {
		return label
	}
	return normalize(container.Name)
}

func (provider *Docker) getIPAddress(container dockertypes.ContainerJSON) string {
	if label, err := getLabel(container, "traefik.docker.network"); err == nil && label != "" {
		networks := container.NetworkSettings.Networks
		if networks != nil {
			network := networks[label]
			if network != nil {
				return network.IPAddress
			}
		}
	}
	for _, network := range container.NetworkSettings.Networks {
		return network.IPAddress
	}
	return ""
}

func (provider *Docker) getPort(container dockertypes.ContainerJSON) string {
	if label, err := getLabel(container, "traefik.port"); err == nil {
		return label
	}
	for key := range container.NetworkSettings.Ports {
		return key.Port()
	}
	return ""
}

func (provider *Docker) getWeight(container dockertypes.ContainerJSON) string {
	if label, err := getLabel(container, "traefik.weight"); err == nil {
		return label
	}
	return "1"
}

func (provider *Docker) getDomain(container dockertypes.ContainerJSON) string {
	if label, err := getLabel(container, "traefik.domain"); err == nil {
		return label
	}
	return provider.Domain
}

func (provider *Docker) getProtocol(container dockertypes.ContainerJSON) string {
	if label, err := getLabel(container, "traefik.protocol"); err == nil {
		return label
	}
	return "http"
}

func (provider *Docker) getPassHostHeader(container dockertypes.ContainerJSON) string {
	if passHostHeader, err := getLabel(container, "traefik.frontend.passHostHeader"); err == nil {
		return passHostHeader
	}
	return "true"
}

func (provider *Docker) getPriority(container dockertypes.ContainerJSON) string {
	if priority, err := getLabel(container, "traefik.frontend.priority"); err == nil {
		return priority
	}
	return "0"
}

func (provider *Docker) getEntryPoints(container dockertypes.ContainerJSON) []string {
	if entryPoints, err := getLabel(container, "traefik.frontend.entryPoints"); err == nil {
		return strings.Split(entryPoints, ",")
	}
	return []string{}
}

func getLabel(container dockertypes.ContainerJSON, label string) (string, error) {
	for key, value := range container.Config.Labels {
		if key == label {
			return value, nil
		}
	}
	return "", errors.New("Label not found:" + label)
}

func getLabels(container dockertypes.ContainerJSON, labels []string) (map[string]string, error) {
	var globalErr error
	foundLabels := map[string]string{}
	for _, label := range labels {
		foundLabel, err := getLabel(container, label)
		// Error out only if one of them is defined.
		if err != nil {
			globalErr = errors.New("Label not found: " + label)
			continue
		}
		foundLabels[label] = foundLabel

	}
	return foundLabels, globalErr
}

func listContainers(ctx context.Context, dockerClient client.APIClient) ([]dockertypes.ContainerJSON, error) {
	containerList, err := dockerClient.ContainerList(ctx, dockertypes.ContainerListOptions{})
	if err != nil {
		return []dockertypes.ContainerJSON{}, err
	}
	containersInspected := []dockertypes.ContainerJSON{}

	// get inspect containers
	for _, container := range containerList {
		containerInspected, err := dockerClient.ContainerInspect(ctx, container.ID)
		if err != nil {
			log.Warnf("Failed to inpsect container %s, error: %s", container.ID, err)
		} else {
			containersInspected = append(containersInspected, containerInspected)
		}
	}
	return containersInspected, nil
}

// Escape beginning slash "/", convert all others to dash "-"
func (provider *Docker) getSubDomain(name string) string {
	return strings.Replace(strings.TrimPrefix(name, "/"), "/", "-", -1)
}
<commit_msg>fix(constainrs,docker): Syntax<commit_after>package provider

import (
	"errors"
	"net/http"
	"strconv"
	"strings"
	"text/template"
	"time"

	"golang.org/x/net/context"

	"github.com/BurntSushi/ty/fun"
	log "github.com/Sirupsen/logrus"
	"github.com/cenkalti/backoff"
	"github.com/containous/traefik/safe"
	"github.com/containous/traefik/types"
	"github.com/docker/engine-api/client"
	dockertypes "github.com/docker/engine-api/types"
	eventtypes "github.com/docker/engine-api/types/events"
	"github.com/docker/engine-api/types/filters"
	"github.com/docker/go-connections/sockets"
	"github.com/docker/go-connections/tlsconfig"
	"github.com/vdemeester/docker-events"
)

// DockerAPIVersion is a constant holding the version of the Docker API traefik will use
const DockerAPIVersion string = "1.21"

// Docker holds configurations of the Docker provider.
type Docker struct {
	BaseProvider
	Endpoint string     `description:"Docker server endpoint. Can be a tcp or a unix socket endpoint"`
	Domain   string     `description:"Default domain used"`
	TLS      *DockerTLS `description:"Enable Docker TLS support"`
}

// DockerTLS holds TLS specific configurations
type DockerTLS struct {
	CA                 string `description:"TLS CA"`
	Cert               string `description:"TLS cert"`
	Key                string `description:"TLS key"`
	InsecureSkipVerify bool   `description:"TLS insecure skip verify"`
}

func (provider *Docker) createClient() (client.APIClient, error) {
	var httpClient *http.Client
	httpHeaders := map[string]string{
		// FIXME(vdemeester) use version here O:)
		"User-Agent": "Traefik",
	}
	if provider.TLS != nil {
		tlsOptions := tlsconfig.Options{
			CAFile:             provider.TLS.CA,
			CertFile:           provider.TLS.Cert,
			KeyFile:            provider.TLS.Key,
			InsecureSkipVerify: provider.TLS.InsecureSkipVerify,
		}
		config, err := tlsconfig.Client(tlsOptions)
		if err != nil {
			return nil, err
		}
		tr := &http.Transport{
			TLSClientConfig: config,
		}
		proto, addr, _, err := client.ParseHost(provider.Endpoint)
		if err != nil {
			return nil, err
		}

		sockets.ConfigureTransport(tr, proto, addr)

		httpClient = &http.Client{
			Transport: tr,
		}
	}
	return client.NewClient(provider.Endpoint, DockerAPIVersion, httpClient, httpHeaders)
}

// Provide allows the provider to provide configurations to traefik
// using the given configuration channel.
func (provider *Docker) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints []types.Constraint) error {
	provider.Constraints = append(provider.Constraints, constraints...)
	// TODO register this routine in pool, and watch for stop channel
	safe.Go(func() {
		operation := func() error {
			var err error

			dockerClient, err := provider.createClient()
			if err != nil {
				log.Errorf("Failed to create a client for docker, error: %s", err)
				return err
			}

			ctx := context.Background()
			version, err := dockerClient.ServerVersion(ctx)
			log.Debugf("Docker connection established with docker %s (API %s)", version.Version, version.APIVersion)
			containers, err := listContainers(ctx, dockerClient)
			if err != nil {
				log.Errorf("Failed to list containers for docker, error %s", err)
				return err
			}
			configuration := provider.loadDockerConfig(containers)
			configurationChan <- types.ConfigMessage{
				ProviderName:  "docker",
				Configuration: configuration,
			}
			if provider.Watch {
				ctx, cancel := context.WithCancel(ctx)
				f := filters.NewArgs()
				f.Add("type", "container")
				options := dockertypes.EventsOptions{
					Filters: f,
				}
				eventHandler := events.NewHandler(events.ByAction)
				startStopHandle := func(m eventtypes.Message) {
					log.Debugf("Docker event received %+v", m)
					containers, err := listContainers(ctx, dockerClient)
					if err != nil {
						log.Errorf("Failed to list containers for docker, error %s", err)
						// Call cancel to get out of the monitor
						cancel()
						return
					}
					configuration := provider.loadDockerConfig(containers)
					if configuration != nil {
						configurationChan <- types.ConfigMessage{
							ProviderName:  "docker",
							Configuration: configuration,
						}
					}
				}
				eventHandler.Handle("start", startStopHandle)
				eventHandler.Handle("die", startStopHandle)

				errChan := events.MonitorWithHandler(ctx, dockerClient, options, eventHandler)
				pool.Go(func(stop chan bool) {
					for {
						select {
						case <-stop:
							cancel()
							return
						}
					}
				})
				if err := <-errChan; err != nil {
					return err
				}
			}
			return nil
		}
		notify := func(err error, time time.Duration) {
			log.Errorf("Docker connection error %+v, retrying in %s", err, time)
		}
		err := backoff.RetryNotify(operation, backoff.NewExponentialBackOff(), notify)
		if err != nil {
			log.Fatalf("Cannot connect to docker server %+v", err)
		}
	})

	return nil
}

func (provider *Docker) loadDockerConfig(containersInspected []dockertypes.ContainerJSON) *types.Configuration {
	var DockerFuncMap = template.FuncMap{
		"getBackend":        provider.getBackend,
		"getIPAddress":      provider.getIPAddress,
		"getPort":           provider.getPort,
		"getWeight":         provider.getWeight,
		"getDomain":         provider.getDomain,
		"getProtocol":       provider.getProtocol,
		"getPassHostHeader": provider.getPassHostHeader,
		"getPriority":       provider.getPriority,
		"getEntryPoints":    provider.getEntryPoints,
		"getFrontendRule":   provider.getFrontendRule,
		"replace":           replace,
	}

	// filter containers
	filteredContainers := fun.Filter(provider.ContainerFilter, containersInspected).([]dockertypes.ContainerJSON)

	frontends := map[string][]dockertypes.ContainerJSON{}
	for _, container := range filteredContainers {
		frontendName := provider.getFrontendName(container)
		frontends[frontendName] = append(frontends[frontendName], container)
	}

	templateObjects := struct {
		Containers []dockertypes.ContainerJSON
		Frontends  map[string][]dockertypes.ContainerJSON
		Domain     string
	}{
		filteredContainers,
		frontends,
		provider.Domain,
	}

	configuration, err := provider.getConfiguration("templates/docker.tmpl", DockerFuncMap, templateObjects)
	if err != nil {
		log.Error(err)
	}
	return configuration
}

// ContainerFilter checks if container have to be exposed
func (provider *Docker) ContainerFilter(container dockertypes.ContainerJSON) bool {
	_, err := strconv.Atoi(container.Config.Labels["traefik.port"])
	if len(container.NetworkSettings.Ports) == 0 && err != nil {
		log.Debugf("Filtering container without port and no traefik.port label %s", container.Name)
		return false
	}
	if len(container.NetworkSettings.Ports) > 1 && err != nil {
		log.Debugf("Filtering container with more than 1 port and no traefik.port label %s", container.Name)
		return false
	}

	if container.Config.Labels["traefik.enable"] == "false" {
		log.Debugf("Filtering disabled container %s", container.Name)
		return false
	}

	constraintTags := strings.Split(container.Config.Labels["traefik.tags"], ",")
	if ok, failingConstraint := provider.MatchConstraints(constraintTags); !ok {
		if failingConstraint != nil {
			log.Debugf("Container %v pruned by '%v' constraint", container.Name, failingConstraint.String())
		}
		return false
	}

	return true
}

func (provider *Docker) getFrontendName(container dockertypes.ContainerJSON) string {
	// Replace '.' with '-' in quoted keys because of this issue https://github.com/BurntSushi/toml/issues/78
	return normalize(provider.getFrontendRule(container))
}

// GetFrontendRule returns the frontend rule for the specified container, using
// it's label. It returns a default one (Host) if the label is not present.
func (provider *Docker) getFrontendRule(container dockertypes.ContainerJSON) string {
	// ⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠
	// TODO: backwards compatibility with DEPRECATED rule.Value
	if value, ok := container.Config.Labels["traefik.frontend.value"]; ok {
		log.Warnf("Label traefik.frontend.value=%s is DEPRECATED (will be removed in v1.0.0), please refer to the rule label: https://github.com/containous/traefik/blob/master/docs/index.md#docker", value)
		rule, _ := container.Config.Labels["traefik.frontend.rule"]
		return rule + ":" + value
	}
	// ⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠⚠

	if label, err := getLabel(container, "traefik.frontend.rule"); err == nil {
		return label
	}
	return "Host:" + provider.getSubDomain(container.Name) + "." + provider.Domain
}

func (provider *Docker) getBackend(container dockertypes.ContainerJSON) string {
	if label, err := getLabel(container, "traefik.backend"); err == nil {
		return label
	}
	return normalize(container.Name)
}

func (provider *Docker) getIPAddress(container dockertypes.ContainerJSON) string {
	if label, err := getLabel(container, "traefik.docker.network"); err == nil && label != "" {
		networks := container.NetworkSettings.Networks
		if networks != nil {
			network := networks[label]
			if network != nil {
				return network.IPAddress
			}
		}
	}
	for _, network := range container.NetworkSettings.Networks {
		return network.IPAddress
	}
	return ""
}

func (provider *Docker) getPort(container dockertypes.ContainerJSON) string {
	if label, err := getLabel(container, "traefik.port"); err == nil {
		return label
	}
	for key := range container.NetworkSettings.Ports {
		return key.Port()
	}
	return ""
}

func (provider *Docker) getWeight(container dockertypes.ContainerJSON) string {
	if label, err := getLabel(container, "traefik.weight"); err == nil {
		return label
	}
	return "1"
}

func (provider *Docker) getDomain(container dockertypes.ContainerJSON) string {
	if label, err := getLabel(container, "traefik.domain"); err == nil {
		return label
	}
	return provider.Domain
}

func (provider *Docker) getProtocol(container dockertypes.ContainerJSON) string {
	if label, err := getLabel(container, "traefik.protocol"); err == nil {
		return label
	}
	return "http"
}

func (provider *Docker) getPassHostHeader(container dockertypes.ContainerJSON) string {
	if passHostHeader, err := getLabel(container, "traefik.frontend.passHostHeader"); err == nil {
		return passHostHeader
	}
	return "true"
}

func (provider *Docker) getPriority(container dockertypes.ContainerJSON) string {
	if priority, err := getLabel(container, "traefik.frontend.priority"); err == nil {
		return priority
	}
	return "0"
}

func (provider *Docker) getEntryPoints(container dockertypes.ContainerJSON) []string {
	if entryPoints, err := getLabel(container, "traefik.frontend.entryPoints"); err == nil {
		return strings.Split(entryPoints, ",")
	}
	return []string{}
}

func getLabel(container dockertypes.ContainerJSON, label string) (string, error) {
	for key, value := range container.Config.Labels {
		if key == label {
			return value, nil
		}
	}
	return "", errors.New("Label not found:" + label)
}

func getLabels(container dockertypes.ContainerJSON, labels []string) (map[string]string, error) {
	var globalErr error
	foundLabels := map[string]string{}
	for _, label := range labels {
		foundLabel, err := getLabel(container, label)
		// Error out only if one of them is defined.
		if err != nil {
			globalErr = errors.New("Label not found: " + label)
			continue
		}
		foundLabels[label] = foundLabel

	}
	return foundLabels, globalErr
}

func listContainers(ctx context.Context, dockerClient client.APIClient) ([]dockertypes.ContainerJSON, error) {
	containerList, err := dockerClient.ContainerList(ctx, dockertypes.ContainerListOptions{})
	if err != nil {
		return []dockertypes.ContainerJSON{}, err
	}
	containersInspected := []dockertypes.ContainerJSON{}

	// get inspect containers
	for _, container := range containerList {
		containerInspected, err := dockerClient.ContainerInspect(ctx, container.ID)
		if err != nil {
			log.Warnf("Failed to inpsect container %s, error: %s", container.ID, err)
		} else {
			containersInspected = append(containersInspected, containerInspected)
		}
	}
	return containersInspected, nil
}

// Escape beginning slash "/", convert all others to dash "-"
func (provider *Docker) getSubDomain(name string) string {
	return strings.Replace(strings.TrimPrefix(name, "/"), "/", "-", -1)
}
<|endoftext|>


<commit_before>package middleware

import (
	"github.com/gin-gonic/gin"
	"strings"

	"github.com/eirka/eirka-libs/redis"
)

var (
	RedisKeyIndex = make(map[string]RedisKey)
	RedisKeys     = []RedisKey{
		{base: "tag", fieldcount: 2, hash: true, expire: true},
		{base: "thread", fieldcount: 2, hash: true, expire: false},
		{base: "post", fieldcount: 2, hash: true, expire: false},
		{base: "directory", fieldcount: 1, hash: true, expire: false},
		{base: "index", fieldcount: 1, hash: true, expire: false},
		{base: "image", fieldcount: 1, hash: true, expire: false},
		{base: "tags", fieldcount: 1, hash: true, expire: false},
		{base: "favorited", fieldcount: 1, hash: false, expire: true},
		{base: "new", fieldcount: 1, hash: false, expire: true},
		{base: "popular", fieldcount: 1, hash: false, expire: true},
		{base: "imageboards", fieldcount: 1, hash: false, expire: true},
	}
)

func init() {
	// key index map
	for _, key := range RedisKeys {
		RedisKeyIndex[key.base] = key
	}
}

// Cache will check for the key in Redis and serve it. If not found, it will
// take the marshalled JSON from the controller and set it in Redis
func Cache() gin.HandlerFunc {
	return func(c *gin.Context) {
		var result []byte
		var err error

		// bool for analytics middleware
		c.Set("cached", false)

		// break cache if there is a query
		if c.Request.URL.RawQuery != "" {
			c.Next()
			return
		}

		// Trim leading / from path and split
		params := strings.Split(strings.Trim(c.Request.URL.Path, "/"), "/")

		// get the keyname
		key, ok := RedisKeyIndex[params[0]]
		if !ok {
			c.Next()
			return
		}

		// set the key minus the base
		key.SetKey(params[1:]...)

		result, err = key.Get()
		if err == redis.ErrCacheMiss {
			// go to the controller
			c.Next()

			// Check if there was an error from the controller
			_, controllerError := c.Get("controllerError")
			if controllerError {
				c.Abort()
				return
			}

			err = key.Set(c.MustGet("data").([]byte))
			if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

		} else if err != nil {
			c.Error(err)
			c.Abort()
			return
		}

		// if we made it this far then the page was cached
		c.Set("cached", true)

		c.Writer.Header().Set("Content-Type", "application/json")
		c.Writer.Write(result)
		c.Abort()
		return
	}

}

type RedisKey struct {
	base       string
	fieldcount int
	hash       bool
	expire     bool
	key        string
	hashid     string
}

func (r *RedisKey) SetKey(ids ...string) {

	// create our key
	r.key = strings.Join([]string{r.base, strings.Join(ids[:r.fieldcount], ":")}, ":")

	// get our hash id
	if r.hash {
		r.hashid = strings.Join(ids[r.fieldcount:], "")
	}

	return
}

func (r *RedisKey) Get() (result []byte, err error) {

	if r.hash {
		return redis.RedisCache.HGet(r.key, r.hashid)
	} else {
		return redis.RedisCache.Get(r.key)
	}

	return
}

func (r *RedisKey) Set(data []byte) (err error) {

	if r.hash {
		err = redis.RedisCache.HMSet(r.key, r.hashid, data)
	} else {
		err = redis.RedisCache.Set(r.key, data)
	}

	if r.expire {
		return redis.RedisCache.Expire(r.key, 600)
	}

	return
}
<commit_msg>use explicit keys<commit_after>package middleware

import (
	"github.com/gin-gonic/gin"
	"strings"

	"github.com/eirka/eirka-libs/redis"
)

var (
	RedisKeyIndex = make(map[string]RedisKey)
	RedisKeys     = []RedisKey{
		{base: "tag", fieldcount: 2, hash: true, expire: true},
		{base: "thread", fieldcount: 2, hash: true, expire: false},
		{base: "post", fieldcount: 2, hash: true, expire: false},
		{base: "directory", fieldcount: 1, hash: true, expire: false},
		{base: "index", fieldcount: 1, hash: true, expire: false},
		{base: "image", fieldcount: 1, hash: true, expire: false},
		{base: "tags", fieldcount: 1, hash: true, expire: false},
		{base: "favorited", fieldcount: 1, hash: false, expire: true},
		{base: "new", fieldcount: 1, hash: false, expire: true},
		{base: "popular", fieldcount: 1, hash: false, expire: true},
		{base: "imageboards", fieldcount: 0, hash: false, expire: true},
	}
)

func init() {
	// key index map
	for _, key := range RedisKeys {
		RedisKeyIndex[key.base] = key
	}
}

// Cache will check for the key in Redis and serve it. If not found, it will
// take the marshalled JSON from the controller and set it in Redis
func Cache() gin.HandlerFunc {
	return func(c *gin.Context) {
		var result []byte
		var err error

		// bool for analytics middleware
		c.Set("cached", false)

		// break cache if there is a query
		if c.Request.URL.RawQuery != "" {
			c.Next()
			return
		}

		// Trim leading / from path and split
		params := strings.Split(strings.Trim(c.Request.URL.Path, "/"), "/")

		// get the keyname
		key, ok := RedisKeyIndex[params[0]]
		if !ok {
			c.Next()
			return
		}

		// set the key minus the base
		key.SetKey(params[1:]...)

		result, err = key.Get()
		if err == redis.ErrCacheMiss {
			// go to the controller
			c.Next()

			// Check if there was an error from the controller
			_, controllerError := c.Get("controllerError")
			if controllerError {
				c.Abort()
				return
			}

			err = key.Set(c.MustGet("data").([]byte))
			if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

		} else if err != nil {
			c.Error(err)
			c.Abort()
			return
		}

		// if we made it this far then the page was cached
		c.Set("cached", true)

		c.Writer.Header().Set("Content-Type", "application/json")
		c.Writer.Write(result)
		c.Abort()
		return
	}

}

type RedisKey struct {
	base       string
	fieldcount int
	hash       bool
	expire     bool
	key        string
	hashid     string
}

func (r *RedisKey) SetKey(ids ...string) {

	// create our key
	r.key = strings.Join([]string{r.base, strings.Join(ids[:r.fieldcount], ":")}, ":")

	// get our hash id
	if r.hash {
		r.hashid = strings.Join(ids[r.fieldcount:], "")
	}

	return
}

func (r *RedisKey) Get() (result []byte, err error) {

	if r.hash {
		return redis.RedisCache.HGet(r.key, r.hashid)
	} else {
		return redis.RedisCache.Get(r.key)
	}

	return
}

func (r *RedisKey) Set(data []byte) (err error) {

	if r.hash {
		err = redis.RedisCache.HMSet(r.key, r.hashid, data)
	} else {
		err = redis.RedisCache.Set(r.key, data)
	}

	if r.expire {
		return redis.RedisCache.Expire(r.key, 600)
	}

	return
}
<|endoftext|>


<commit_before>package githubprovider

import (
	"fmt"
	"strings"

	"github.com/google/go-github/github"
	"github.com/hashicorp/terraform/helper/schema"
)

// required field are here for adding a user to the organization
func resourceGithubAddUser() *schema.Resource {
	return &schema.Resource{
		Create: resourceGithubAddUserCreate,
		Read:   resourceGithubAddUserRead,
		Update: resourceGithubAddUserCreate,
		Delete: resourceGithubAddUserDelete,

		Schema: map[string]*schema.Schema{
			"username": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},

			// role is the required for the membership
			// its value is member as default.
			"role": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Default:  "member",
			},

			// repos is the repos that the organization has
			"repos": &schema.Schema{
				Type:     schema.TypeList,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Required: true,
			},

			// organization is the name of the organization
			"organization": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},

			"teams": &schema.Schema{
				Type:     schema.TypeList,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Required: true,
			},

			// title is the title of the SSH Key
			"title": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},

			// SSHKey is the public key of the user
			"SSHKey": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
		},
	}
}

// GetTeamIDs gets the teams id of the organization
func GetTeamIDs(client *github.Client, org string, teamNames []string) ([]int, error) {
	currentPage := 1

	var teamIDs []int

	for {
		options := &github.ListOptions{
			PerPage: 100,
			Page:    currentPage,
		}

		teams, resp, err := client.Organizations.ListTeams(org, options)
		if err != nil {
			return nil, err
		}

		if len(teams) == 0 {
			break
		}
		// Iterate over all teams and add current user to related team(s)
		for i, team := range teams {
			for _, teamName := range teamNames {
				if *team.Name == teamName {
					teamIDs = append(teamIDs, *teams[i].ID)
				}
			}
		}

		if currentPage == resp.LastPage {
			break
		}

		currentPage = resp.NextPage
	}

	return teamIDs, nil
}

// resourceGithubAddUserCreate adds the user to the organization & the teams
func resourceGithubAddUserCreate(d *schema.ResourceData, meta interface{}) error {
	clientOrg := meta.(*Clients).OrgClient
	client := meta.(*Clients).UserClient

	org := d.Get("organization").(string)
	user := d.Get("username").(string)
	teamNames := interfaceToStringSlice(d.Get("teams"))
	role := d.Get("role").(string)

	if err := checkScopePermissions(client, user); err != nil {
		return err
	}

	teamIDs, err := GetTeamIDs(clientOrg, org, teamNames)

	optAddOrgMembership := &github.OrganizationAddTeamMembershipOptions{
		Role: role,
	}

	for _, teamID := range teamIDs {
		_, _, err := clientOrg.Organizations.AddTeamMembership(teamID, user, optAddOrgMembership)
		if err != nil {
			return err
		}
	}

	active := "active"

	membership := &github.Membership{
		// state should be active to add the user into organization
		State: &active,

		// Role is the required for the membership
		Role: &role,
	}

	// EditOrgMembership edits the membership for user in specified organization.
	// if user is authenticated, we dont need to set 1.parameter as user
	_, _, err = client.Organizations.EditOrgMembership("", org, membership)
	if err != nil {
		return err
	}

	for _, repo := range interfaceToStringSlice(d.Get("repos")) {
		// Creates a fork for the authenticated user.
		_, _, err = client.Repositories.CreateFork(org, repo, nil)
		if err != nil {
			return err
		}
	}

	title := d.Get("title").(string)
	keySSH := d.Get("SSHKey").(string)

	key := &github.Key{
		Title: &title,
		Key:   &keySSH,
	}

	// CreateKey creates a public key. Requires that you are authenticated via Basic Auth,
	// or OAuth with at least `write:public_key` scope.
	//
	// If SSH key is already set up, when u try to add same SSHKEY then
	//you are gonna get 422: Validation error.
	_, _, err = client.Users.CreateKey(key)
	if err != nil && !isErr422ValidationFailed(err) {
		return err
	}

	d.SetId(user)

	return nil
}

func resourceGithubAddUserRead(d *schema.ResourceData, meta interface{}) error {
	org := d.Get("organization").(string)
	user := d.Get("username").(string)
	role := d.Get("role").(string)
	teamNames := interfaceToStringSlice(d.Get("teams"))
	repos := interfaceToStringSlice(d.Get("repos"))
	fmt.Println("org: %v, user: %v,role: %v, teamnames: %v, repos: %v",
		org,
		user,
		role,
		teamNames,
		repos,
	)

	return nil
}

// resourceGithubAddUserCreate removes the user from the organization & the teams
func resourceGithubAddUserDelete(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*Clients).OrgClient

	user := d.Get("username").(string)
	org := d.Get("organization").(string)

	// Removing a user from this list will remove them from all teams and
	// they will no longer have any access to the organization’s repositories.
	_, err := client.Organizations.RemoveMember(org, user)
	return err
}

func getKeyID(client *github.Client, user, title string) (int, error) {
	keys, _, err := client.Users.ListKeys(user, nil)
	if err != nil {
		return 0, err
	}

	for _, key := range keys {
		if *key.Title == title {
			return *key.ID, nil
		}
	}

	return 0, err
}

// interfaceToStringSlice converts the interface to slice of string
func interfaceToStringSlice(s interface{}) []string {
	slice, ok := s.([]interface{})
	if !ok {
		return nil
	}

	sslice := make([]string, len(slice))
	for i := range slice {
		sslice[i] = slice[i].(string)
	}

	return sslice
}

func checkScopePermissions(client *github.Client, username string) error {
	arr, err := getScopes(client, username)
	if err != nil {
		return err
	}

	// we created 2-dimensional array for scopes.
	scopeArray := [][]string{
		// if user enables one of this scopes, then its OK to go..
		{"write:public_key", "admin:public_key"},
		{"user"},
		{"repo", "public_repo"},
		{"admin:org"},
	}

	for _, scopeElement := range scopeArray {
		if !(isInArray(arr, scopeElement)) {
			scopeErr := fmt.Errorf("Could not find required scope :", scopeElement)
			return scopeErr
		}
	}

	return nil
}

func getScopes(client *github.Client, username string) ([]string, error) {
	var scopes []string
	_, resp, err := client.Users.Get(username)
	if err != nil {
		return scopes, err
	}

	list := resp.Header.Get("X-Oauth-Scopes")
	scopes = strings.Split(list, ", ")

	return scopes, nil
}

func isInArray(arr, item []string) bool {
	for _, a := range arr {
		for _, i := range item {
			if a == i {
				return true
			}
		}
	}
	return false
}

// isErr422ValidationFailed return true if error contains the string:
// '422 Validation Failed'. This error is special cased so we can ignore it on
// when it occurs during rebuilding of stack template.
func isErr422ValidationFailed(err error) bool {
	return err != nil && strings.Contains(err.Error(), "422 Validation Failed")
}
<commit_msg>terraform/github: add checking for team names<commit_after>package githubprovider

import (
	"errors"
	"fmt"
	"strings"

	"github.com/google/go-github/github"
	"github.com/hashicorp/terraform/helper/schema"
)

// required field are here for adding a user to the organization
func resourceGithubAddUser() *schema.Resource {
	return &schema.Resource{
		Create: resourceGithubAddUserCreate,
		Read:   resourceGithubAddUserRead,
		Update: resourceGithubAddUserCreate,
		Delete: resourceGithubAddUserDelete,

		Schema: map[string]*schema.Schema{
			"username": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},

			// role is the required for the membership
			// its value is member as default.
			"role": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Default:  "member",
			},

			// repos is the repos that the organization has
			"repos": &schema.Schema{
				Type:     schema.TypeList,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Required: true,
			},

			// organization is the name of the organization
			"organization": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},

			"teams": &schema.Schema{
				Type:     schema.TypeList,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Required: true,
			},

			// title is the title of the SSH Key
			"title": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},

			// SSHKey is the public key of the user
			"SSHKey": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
		},
	}
}

// GetTeamIDs gets the teams id of the organization
func GetTeamIDs(client *github.Client, org string, teamNames []string) ([]int, error) {
	currentPage := 1

	var teamIDs []int

	for {
		options := &github.ListOptions{
			PerPage: 100,
			Page:    currentPage,
		}

		teams, resp, err := client.Organizations.ListTeams(org, options)
		if err != nil {
			return nil, err
		}

		if len(teams) == 0 {
			break
		}
		// Iterate over all teams and add current user to related team(s)
		for i, team := range teams {
			for _, teamName := range teamNames {
				if *team.Name == teamName {
					teamIDs = append(teamIDs, *teams[i].ID)
				}
			}
		}

		if currentPage == resp.LastPage {
			break
		}

		currentPage = resp.NextPage
	}

	return teamIDs, nil
}

// resourceGithubAddUserCreate adds the user to the organization & the teams
func resourceGithubAddUserCreate(d *schema.ResourceData, meta interface{}) error {
	clientOrg := meta.(*Clients).OrgClient
	client := meta.(*Clients).UserClient

	org := d.Get("organization").(string)
	user := d.Get("username").(string)
	teamNames := interfaceToStringSlice(d.Get("teams"))
	role := d.Get("role").(string)

	if err := checkScopePermissions(client, user); err != nil {
		return err
	}

	if len(teamNames) == 0 {
		return errors.New("team name is not defined")
	}

	teamIDs, err := GetTeamIDs(clientOrg, org, teamNames)

	optAddOrgMembership := &github.OrganizationAddTeamMembershipOptions{
		Role: role,
	}

	if len(teamNames) != len(teamIDs) {
		return errors.New("team name is not found")
	}

	for _, teamID := range teamIDs {
		_, _, err := clientOrg.Organizations.AddTeamMembership(teamID, user, optAddOrgMembership)
		if err != nil {
			return err
		}
	}

	active := "active"

	membership := &github.Membership{
		// state should be active to add the user into organization
		State: &active,

		// Role is the required for the membership
		Role: &role,
	}

	// EditOrgMembership edits the membership for user in specified organization.
	// if user is authenticated, we dont need to set 1.parameter as user
	_, _, err = client.Organizations.EditOrgMembership("", org, membership)
	if err != nil {
		return err
	}

	for _, repo := range interfaceToStringSlice(d.Get("repos")) {
		// Creates a fork for the authenticated user.
		_, _, err = client.Repositories.CreateFork(org, repo, nil)
		if err != nil {
			return err
		}
	}

	title := d.Get("title").(string)
	keySSH := d.Get("SSHKey").(string)

	key := &github.Key{
		Title: &title,
		Key:   &keySSH,
	}

	// CreateKey creates a public key. Requires that you are authenticated via Basic Auth,
	// or OAuth with at least `write:public_key` scope.
	//
	// If SSH key is already set up, when u try to add same SSHKEY then
	//you are gonna get 422: Validation error.
	_, _, err = client.Users.CreateKey(key)
	if err != nil && !isErr422ValidationFailed(err) {
		return err
	}

	d.SetId(user)

	return nil
}

func resourceGithubAddUserRead(d *schema.ResourceData, meta interface{}) error {
	org := d.Get("organization").(string)
	user := d.Get("username").(string)
	role := d.Get("role").(string)
	teamNames := interfaceToStringSlice(d.Get("teams"))
	repos := interfaceToStringSlice(d.Get("repos"))
	fmt.Println("org: %v, user: %v,role: %v, teamnames: %v, repos: %v",
		org,
		user,
		role,
		teamNames,
		repos,
	)

	return nil
}

// resourceGithubAddUserCreate removes the user from the organization & the teams
func resourceGithubAddUserDelete(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*Clients).OrgClient

	user := d.Get("username").(string)
	org := d.Get("organization").(string)

	// Removing a user from this list will remove them from all teams and
	// they will no longer have any access to the organization’s repositories.
	_, err := client.Organizations.RemoveMember(org, user)
	return err
}

func getKeyID(client *github.Client, user, title string) (int, error) {
	keys, _, err := client.Users.ListKeys(user, nil)
	if err != nil {
		return 0, err
	}

	for _, key := range keys {
		if *key.Title == title {
			return *key.ID, nil
		}
	}

	return 0, err
}

// interfaceToStringSlice converts the interface to slice of string
func interfaceToStringSlice(s interface{}) []string {
	slice, ok := s.([]interface{})
	if !ok {
		return nil
	}

	sslice := make([]string, len(slice))
	for i := range slice {
		sslice[i] = slice[i].(string)
	}

	return sslice
}

func checkScopePermissions(client *github.Client, username string) error {
	arr, err := getScopes(client, username)
	if err != nil {
		return err
	}

	// we created 2-dimensional array for scopes.
	scopeArray := [][]string{
		// if user enables one of this scopes, then its OK to go..
		{"write:public_key", "admin:public_key"},
		{"user"},
		{"repo", "public_repo"},
		{"admin:org"},
	}

	for _, scopeElement := range scopeArray {
		if !(isInArray(arr, scopeElement)) {
			scopeErr := fmt.Errorf("Could not find required scope :", scopeElement)
			return scopeErr
		}
	}

	return nil
}

func getScopes(client *github.Client, username string) ([]string, error) {
	var scopes []string
	_, resp, err := client.Users.Get(username)
	if err != nil {
		return scopes, err
	}

	list := resp.Header.Get("X-Oauth-Scopes")
	scopes = strings.Split(list, ", ")

	return scopes, nil
}

func isInArray(arr, item []string) bool {
	for _, a := range arr {
		for _, i := range item {
			if a == i {
				return true
			}
		}
	}
	return false
}

// isErr422ValidationFailed return true if error contains the string:
// '422 Validation Failed'. This error is special cased so we can ignore it on
// when it occurs during rebuilding of stack template.
func isErr422ValidationFailed(err error) bool {
	return err != nil && strings.Contains(err.Error(), "422 Validation Failed")
}
<|endoftext|>


<commit_before>package contractor

import (
	"os"
	"path/filepath"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/renter/proto"
	"github.com/NebulousLabs/Sia/persist"
	"github.com/NebulousLabs/Sia/types"
)

// contractorPersist defines what Contractor data persists across sessions.
type contractorPersist struct {
	Allowance     modules.Allowance         `json:"allowance"`
	BlockHeight   types.BlockHeight         `json:"blockheight"`
	CurrentPeriod types.BlockHeight         `json:"currentperiod"`
	LastChange    modules.ConsensusChangeID `json:"lastchange"`
	OldContracts  []modules.RenterContract  `json:"oldcontracts"`
	RenewedIDs    map[string]string         `json:"renewedids"`
}

// persistData returns the data in the Contractor that will be saved to disk.
func (c *Contractor) persistData() contractorPersist {
	data := contractorPersist{
		Allowance:     c.allowance,
		BlockHeight:   c.blockHeight,
		CurrentPeriod: c.currentPeriod,
		LastChange:    c.lastChange,
		RenewedIDs:    make(map[string]string),
	}
	for _, contract := range c.oldContracts {
		data.OldContracts = append(data.OldContracts, contract)
	}
	for oldID, newID := range c.renewedIDs {
		data.RenewedIDs[oldID.String()] = newID.String()
	}
	return data
}

// load loads the Contractor persistence data from disk.
func (c *Contractor) load() error {
	var data contractorPersist
	err := c.persist.load(&data)
	if err != nil {
		return err
	}
	c.allowance = data.Allowance
	c.blockHeight = data.BlockHeight
	c.currentPeriod = data.CurrentPeriod
	c.lastChange = data.LastChange
	for _, contract := range data.OldContracts {
		c.oldContracts[contract.ID] = contract
	}
	for oldString, newString := range data.RenewedIDs {
		var oldHash, newHash crypto.Hash
		oldHash.LoadString(oldString)
		newHash.LoadString(newString)
		c.renewedIDs[types.FileContractID(oldHash)] = types.FileContractID(newHash)
	}

	return nil
}

// save saves the Contractor persistence data to disk.
func (c *Contractor) save() error {
	return c.persist.save(c.persistData())
}

// saveSync saves the Contractor persistence data to disk and then syncs to disk.
func (c *Contractor) saveSync() error {
	return c.persist.save(c.persistData())
}

// convertPersist converts the pre-v1.3.1 contractor persist formats to the new
// formats.
func convertPersist(dir string) error {
	// Try loading v1.3.1 persist. If it has the correct version number, no
	// further action is necessary.
	persistPath := filepath.Join(dir, "contractor.json")
	err := persist.LoadJSON(persistMeta, nil, persistPath)
	if err == nil {
		return nil
	}

	// Try loading v1.3.0 persist (journal).
	journalPath := filepath.Join(dir, "contractor.journal")
	if _, err := os.Stat(journalPath); os.IsNotExist(err) {
		// no journal file found; assume this is a fresh install
		return nil
	}
	var p journalPersist
	j, err := openJournal(journalPath, &p)
	if err != nil {
		return err
	}
	j.Close()
	// convert to v1.3.1 format and save
	data := contractorPersist{
		Allowance:     p.Allowance,
		BlockHeight:   p.BlockHeight,
		CurrentPeriod: p.CurrentPeriod,
		LastChange:    p.LastChange,
		RenewedIDs:    p.RenewedIDs,
	}
	for _, c := range p.OldContracts {
		data.OldContracts = append(data.OldContracts, modules.RenterContract{
			ID:               c.ID,
			HostPublicKey:    c.HostPublicKey,
			StartHeight:      c.StartHeight,
			EndHeight:        c.EndHeight(),
			RenterFunds:      c.RenterFunds(),
			DownloadSpending: c.DownloadSpending,
			StorageSpending:  c.StorageSpending,
			UploadSpending:   c.UploadSpending,
			TotalCost:        c.TotalCost,
			ContractFee:      c.ContractFee,
			TxnFee:           c.TxnFee,
			SiafundFee:       c.SiafundFee,
		})
	}
	err = persist.SaveJSON(persistMeta, data, persistPath)
	if err != nil {
		return err
	}

	// convert contracts to contract files
	for _, c := range p.Contracts {
		if err := proto.ConvertV130Contract(c, dir); err != nil {
			return err
		}
	}

	// delete the journal file
	os.RemoveAll(journalPath)
	return nil
}
<commit_msg>save converted contracts to right directory<commit_after>package contractor

import (
	"os"
	"path/filepath"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/renter/proto"
	"github.com/NebulousLabs/Sia/persist"
	"github.com/NebulousLabs/Sia/types"
)

// contractorPersist defines what Contractor data persists across sessions.
type contractorPersist struct {
	Allowance     modules.Allowance         `json:"allowance"`
	BlockHeight   types.BlockHeight         `json:"blockheight"`
	CurrentPeriod types.BlockHeight         `json:"currentperiod"`
	LastChange    modules.ConsensusChangeID `json:"lastchange"`
	OldContracts  []modules.RenterContract  `json:"oldcontracts"`
	RenewedIDs    map[string]string         `json:"renewedids"`
}

// persistData returns the data in the Contractor that will be saved to disk.
func (c *Contractor) persistData() contractorPersist {
	data := contractorPersist{
		Allowance:     c.allowance,
		BlockHeight:   c.blockHeight,
		CurrentPeriod: c.currentPeriod,
		LastChange:    c.lastChange,
		RenewedIDs:    make(map[string]string),
	}
	for _, contract := range c.oldContracts {
		data.OldContracts = append(data.OldContracts, contract)
	}
	for oldID, newID := range c.renewedIDs {
		data.RenewedIDs[oldID.String()] = newID.String()
	}
	return data
}

// load loads the Contractor persistence data from disk.
func (c *Contractor) load() error {
	var data contractorPersist
	err := c.persist.load(&data)
	if err != nil {
		return err
	}
	c.allowance = data.Allowance
	c.blockHeight = data.BlockHeight
	c.currentPeriod = data.CurrentPeriod
	c.lastChange = data.LastChange
	for _, contract := range data.OldContracts {
		c.oldContracts[contract.ID] = contract
	}
	for oldString, newString := range data.RenewedIDs {
		var oldHash, newHash crypto.Hash
		oldHash.LoadString(oldString)
		newHash.LoadString(newString)
		c.renewedIDs[types.FileContractID(oldHash)] = types.FileContractID(newHash)
	}

	return nil
}

// save saves the Contractor persistence data to disk.
func (c *Contractor) save() error {
	return c.persist.save(c.persistData())
}

// saveSync saves the Contractor persistence data to disk and then syncs to disk.
func (c *Contractor) saveSync() error {
	return c.persist.save(c.persistData())
}

// convertPersist converts the pre-v1.3.1 contractor persist formats to the new
// formats.
func convertPersist(dir string) error {
	// Try loading v1.3.1 persist. If it has the correct version number, no
	// further action is necessary.
	persistPath := filepath.Join(dir, "contractor.json")
	err := persist.LoadJSON(persistMeta, nil, persistPath)
	if err == nil {
		return nil
	}

	// Try loading v1.3.0 persist (journal).
	journalPath := filepath.Join(dir, "contractor.journal")
	if _, err := os.Stat(journalPath); os.IsNotExist(err) {
		// no journal file found; assume this is a fresh install
		return nil
	}
	var p journalPersist
	j, err := openJournal(journalPath, &p)
	if err != nil {
		return err
	}
	j.Close()
	// convert to v1.3.1 format and save
	data := contractorPersist{
		Allowance:     p.Allowance,
		BlockHeight:   p.BlockHeight,
		CurrentPeriod: p.CurrentPeriod,
		LastChange:    p.LastChange,
		RenewedIDs:    p.RenewedIDs,
	}
	for _, c := range p.OldContracts {
		data.OldContracts = append(data.OldContracts, modules.RenterContract{
			ID:               c.ID,
			HostPublicKey:    c.HostPublicKey,
			StartHeight:      c.StartHeight,
			EndHeight:        c.EndHeight(),
			RenterFunds:      c.RenterFunds(),
			DownloadSpending: c.DownloadSpending,
			StorageSpending:  c.StorageSpending,
			UploadSpending:   c.UploadSpending,
			TotalCost:        c.TotalCost,
			ContractFee:      c.ContractFee,
			TxnFee:           c.TxnFee,
			SiafundFee:       c.SiafundFee,
		})
	}
	err = persist.SaveJSON(persistMeta, data, persistPath)
	if err != nil {
		return err
	}

	// convert contracts to contract files
	for _, c := range p.Contracts {
		if err := proto.ConvertV130Contract(c, filepath.Join(dir, "contracts")); err != nil {
			return err
		}
	}

	// delete the journal file
	os.RemoveAll(journalPath)
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

/*

ABOUT THIS TEST
===============

This test plays part in testing an upgrade path from a previous version/tag. It takes a GitHub workflow file to complete the functionality.
What's in this file is the setting up of a cluster, sharded and unsharded keyspace, creating and populating some tables, then testing retrieval of data.
The twist here is that you can run this test over pre-existing vtdataroot, which means this test can reuse existing etcd, existing tables, existing mysql,
in which case it will not attempt to create keyspaces/schemas/tables, nor will it populate table data. Instead, it will only check for retrieval of data.

The game is to setup the cluster with a stable version (say `v8.0.0`), take it down (and preserve data), then setup a new cluster with a new version (namely the branch/PR head) and attempt to read the data.

Both executions must force some settings so that both reuse same directories, ports, etc. An invocation will look like:
go test ./go/test/endtoend/versionupgrade80/upgrade80_test.go --keep-data -force-vtdataroot /tmp/vtdataroot/vtroot_10901 --force-port-start 11900 --force-base-tablet-uid 1190

*/

package versionupgrade

import (
	"flag"
	"fmt"
	"os"
	"path"
	"testing"

	"vitess.io/vitess/go/mysql"

	"vitess.io/vitess/go/test/endtoend/cluster"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var (
	clusterInstance       *cluster.LocalProcessCluster
	vtParams              mysql.ConnParams
	hostname              = "localhost"
	keyspaceName          = "ks"
	cell                  = "zone1"
	schemaChangeDirectory = ""
	totalTableCount       = 4
	createTable           = `
		CREATE TABLE %s (
			id bigint(20) NOT NULL,
			msg varchar(64),
			PRIMARY KEY (id)
		) ENGINE=InnoDB;
		`
	insertIntoTable = `
		INSERT INTO %s (id, msg) VALUES (17, 'abc');
		`
	selectFromTable = `
		SELECT id, msg FROM %s LIMIT 1;
		`
)

// TestMain is the main entry point
func TestMain(m *testing.M) {
	defer cluster.PanicHandler(nil)
	flag.Parse()

	exitcode, err := func() (int, error) {
		clusterInstance = cluster.NewCluster(cell, hostname)
		schemaChangeDirectory = path.Join("/tmp", fmt.Sprintf("schema_change_dir_%d", clusterInstance.GetAndReserveTabletUID()))
		defer os.RemoveAll(schemaChangeDirectory)
		defer clusterInstance.Teardown()

		if _, err := os.Stat(schemaChangeDirectory); os.IsNotExist(err) {
			_ = os.Mkdir(schemaChangeDirectory, 0700)
		}

		clusterInstance.VtctldExtraArgs = []string{
			"-schema_change_dir", schemaChangeDirectory,
			"-schema_change_controller", "local",
			"-schema_change_check_interval", "1"}

		if err := clusterInstance.StartTopo(); err != nil {
			return 1, err
		}

		// Start keyspace
		keyspace := &cluster.Keyspace{
			Name: keyspaceName,
		}

		if err := clusterInstance.StartUnshardedKeyspace(*keyspace, 2, true); err != nil {
			return 1, err
		}
		if err := clusterInstance.StartKeyspace(*keyspace, []string{"1"}, 1, false); err != nil {
			return 1, err
		}

		vtgateInstance := clusterInstance.NewVtgateInstance()
		// set the gateway we want to use
		vtgateInstance.GatewayImplementation = "tabletgateway"
		// Start vtgate
		if err := vtgateInstance.Setup(); err != nil {
			return 1, err
		}
		// ensure it is torn down during cluster TearDown
		clusterInstance.VtgateProcess = *vtgateInstance
		vtParams = mysql.ConnParams{
			Host: clusterInstance.Hostname,
			Port: clusterInstance.VtgateMySQLPort,
		}

		return m.Run(), nil
	}()
	if err != nil {
		fmt.Printf("%v\n", err)
		os.Exit(1)
	} else {
		os.Exit(exitcode)
	}

}

func TestShards(t *testing.T) {
	defer cluster.PanicHandler(t)
	assert.Equal(t, 2, len(clusterInstance.Keyspaces[0].Shards))
}

func TestDeploySchema(t *testing.T) {
	defer cluster.PanicHandler(t)

	if clusterInstance.ReusingVTDATAROOT {
		// we assume data is already deployed
		return
	}
	// Create n tables, populate
	for i := 0; i < totalTableCount; i++ {
		tableName := fmt.Sprintf("vt_upgrade_test_%02d", i)

		{
			sqlQuery := fmt.Sprintf(createTable, tableName)
			err := clusterInstance.VtctlclientProcess.ApplySchema(keyspaceName, sqlQuery)
			require.Nil(t, err)
		}
		for i := range clusterInstance.Keyspaces[0].Shards {
			sqlQuery := fmt.Sprintf(insertIntoTable, tableName)
			tablet := clusterInstance.Keyspaces[0].Shards[i].Vttablets[0]
			_, err := tablet.VttabletProcess.QueryTablet(sqlQuery, keyspaceName, true)
			require.Nil(t, err)
		}
	}

	checkTables(t, "", totalTableCount)
}

func TestTablesExist(t *testing.T) {
	defer cluster.PanicHandler(t)

	checkTables(t, "", totalTableCount)
}

// checkTables checks the number of tables in the first two shards.
func checkTables(t *testing.T, showTableName string, expectCount int) {
	for i := range clusterInstance.Keyspaces[0].Shards {
		checkTablesCount(t, clusterInstance.Keyspaces[0].Shards[i].Vttablets[0], showTableName, expectCount)
	}
}

// checkTablesCount checks the number of tables in the given tablet
func checkTablesCount(t *testing.T, tablet *cluster.Vttablet, showTableName string, expectCount int) {
	query := fmt.Sprintf(`show tables like '%%%s%%';`, showTableName)
	queryResult, err := tablet.VttabletProcess.QueryTablet(query, keyspaceName, true)
	require.Nil(t, err)
	assert.Equal(t, expectCount, len(queryResult.Rows))
}

// TestTablesData checks the data in tables
func TestTablesData(t *testing.T) {
	// Create n tables, populate
	for i := 0; i < totalTableCount; i++ {
		tableName := fmt.Sprintf("vt_upgrade_test_%02d", i)

		for i := range clusterInstance.Keyspaces[0].Shards {
			sqlQuery := fmt.Sprintf(selectFromTable, tableName)
			tablet := clusterInstance.Keyspaces[0].Shards[i].Vttablets[0]
			queryResult, err := tablet.VttabletProcess.QueryTablet(sqlQuery, keyspaceName, true)
			require.Nil(t, err)
			require.NotNil(t, queryResult)
			row := queryResult.Named().Row()
			require.NotNil(t, row)
			require.Equal(t, int64(17), row.AsInt64("id", 0))
			require.Equal(t, "abc", row.AsString("msg", ""))
		}
	}
}
<commit_msg>adding debug info<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

/*

ABOUT THIS TEST
===============

This test plays part in testing an upgrade path from a previous version/tag. It takes a GitHub workflow file to complete the functionality.
What's in this file is the setting up of a cluster, sharded and unsharded keyspace, creating and populating some tables, then testing retrieval of data.
The twist here is that you can run this test over pre-existing vtdataroot, which means this test can reuse existing etcd, existing tables, existing mysql,
in which case it will not attempt to create keyspaces/schemas/tables, nor will it populate table data. Instead, it will only check for retrieval of data.

The game is to setup the cluster with a stable version (say `v8.0.0`), take it down (and preserve data), then setup a new cluster with a new version (namely the branch/PR head) and attempt to read the data.

Both executions must force some settings so that both reuse same directories, ports, etc. An invocation will look like:
go test ./go/test/endtoend/versionupgrade80/upgrade80_test.go --keep-data -force-vtdataroot /tmp/vtdataroot/vtroot_10901 --force-port-start 11900 --force-base-tablet-uid 1190

*/

package versionupgrade

import (
	"flag"
	"fmt"
	"os"
	"path"
	"testing"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/vt/log"

	"vitess.io/vitess/go/test/endtoend/cluster"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var (
	clusterInstance       *cluster.LocalProcessCluster
	vtParams              mysql.ConnParams
	hostname              = "localhost"
	keyspaceName          = "ks"
	cell                  = "zone1"
	schemaChangeDirectory = ""
	totalTableCount       = 4
	createTable           = `
		CREATE TABLE %s (
			id bigint(20) NOT NULL,
			msg varchar(64),
			PRIMARY KEY (id)
		) ENGINE=InnoDB;
		`
	insertIntoTable = `
		INSERT INTO %s (id, msg) VALUES (17, 'abc');
		`
	selectFromTable = `
		SELECT id, msg FROM %s LIMIT 1;
		`
)

// TestMain is the main entry point
func TestMain(m *testing.M) {
	defer cluster.PanicHandler(nil)
	flag.Parse()

	exitcode, err := func() (int, error) {
		clusterInstance = cluster.NewCluster(cell, hostname)
		schemaChangeDirectory = path.Join("/tmp", fmt.Sprintf("schema_change_dir_%d", clusterInstance.GetAndReserveTabletUID()))
		defer os.RemoveAll(schemaChangeDirectory)
		defer clusterInstance.Teardown()

		if _, err := os.Stat(schemaChangeDirectory); os.IsNotExist(err) {
			_ = os.Mkdir(schemaChangeDirectory, 0700)
		}

		clusterInstance.VtctldExtraArgs = []string{
			"-schema_change_dir", schemaChangeDirectory,
			"-schema_change_controller", "local",
			"-schema_change_check_interval", "1"}

		if err := clusterInstance.StartTopo(); err != nil {
			return 1, err
		}

		// Start keyspace
		keyspace := &cluster.Keyspace{
			Name: keyspaceName,
		}

		if err := clusterInstance.StartUnshardedKeyspace(*keyspace, 2, true); err != nil {
			return 1, err
		}
		if err := clusterInstance.StartKeyspace(*keyspace, []string{"1"}, 1, false); err != nil {
			return 1, err
		}

		vtgateInstance := clusterInstance.NewVtgateInstance()
		// set the gateway we want to use
		vtgateInstance.GatewayImplementation = "tabletgateway"
		// Start vtgate
		if err := vtgateInstance.Setup(); err != nil {
			return 1, err
		}
		// ensure it is torn down during cluster TearDown
		clusterInstance.VtgateProcess = *vtgateInstance
		vtParams = mysql.ConnParams{
			Host: clusterInstance.Hostname,
			Port: clusterInstance.VtgateMySQLPort,
		}

		return m.Run(), nil
	}()
	if err != nil {
		fmt.Printf("%v\n", err)
		os.Exit(1)
	} else {
		os.Exit(exitcode)
	}

}

func TestShards(t *testing.T) {
	defer cluster.PanicHandler(t)
	assert.Equal(t, 2, len(clusterInstance.Keyspaces[0].Shards))
}

func TestDeploySchema(t *testing.T) {
	defer cluster.PanicHandler(t)

	if clusterInstance.ReusingVTDATAROOT {
		// we assume data is already deployed
		return
	}
	// Create n tables, populate
	for i := 0; i < totalTableCount; i++ {
		tableName := fmt.Sprintf("vt_upgrade_test_%02d", i)

		{
			sqlQuery := fmt.Sprintf(createTable, tableName)
			err := clusterInstance.VtctlclientProcess.ApplySchema(keyspaceName, sqlQuery)
			if err != nil {
				log.Infof("ERROR: %+v", err)
			}
			require.Nil(t, err)
		}
		for i := range clusterInstance.Keyspaces[0].Shards {
			sqlQuery := fmt.Sprintf(insertIntoTable, tableName)
			tablet := clusterInstance.Keyspaces[0].Shards[i].Vttablets[0]
			_, err := tablet.VttabletProcess.QueryTablet(sqlQuery, keyspaceName, true)
			require.Nil(t, err)
		}
	}

	checkTables(t, "", totalTableCount)
}

func TestTablesExist(t *testing.T) {
	defer cluster.PanicHandler(t)

	checkTables(t, "", totalTableCount)
}

// checkTables checks the number of tables in the first two shards.
func checkTables(t *testing.T, showTableName string, expectCount int) {
	for i := range clusterInstance.Keyspaces[0].Shards {
		checkTablesCount(t, clusterInstance.Keyspaces[0].Shards[i].Vttablets[0], showTableName, expectCount)
	}
}

// checkTablesCount checks the number of tables in the given tablet
func checkTablesCount(t *testing.T, tablet *cluster.Vttablet, showTableName string, expectCount int) {
	query := fmt.Sprintf(`show tables like '%%%s%%';`, showTableName)
	queryResult, err := tablet.VttabletProcess.QueryTablet(query, keyspaceName, true)
	require.Nil(t, err)
	assert.Equal(t, expectCount, len(queryResult.Rows))
}

// TestTablesData checks the data in tables
func TestTablesData(t *testing.T) {
	// Create n tables, populate
	for i := 0; i < totalTableCount; i++ {
		tableName := fmt.Sprintf("vt_upgrade_test_%02d", i)

		for i := range clusterInstance.Keyspaces[0].Shards {
			sqlQuery := fmt.Sprintf(selectFromTable, tableName)
			tablet := clusterInstance.Keyspaces[0].Shards[i].Vttablets[0]
			queryResult, err := tablet.VttabletProcess.QueryTablet(sqlQuery, keyspaceName, true)
			require.Nil(t, err)
			require.NotNil(t, queryResult)
			row := queryResult.Named().Row()
			require.NotNil(t, row)
			require.Equal(t, int64(17), row.AsInt64("id", 0))
			require.Equal(t, "abc", row.AsString("msg", ""))
		}
	}
}
<|endoftext|>


<commit_before>package channels

import "io"

type Ds3ObjectReadCloser struct {
	io.Reader
}

func NewDs3ObjectReadCloserDecorator(reader io.Reader) io.ReadCloser {
	return Ds3ObjectReadCloser { Reader: reader }
}

func (Ds3ObjectReadCloser Ds3ObjectReadCloser) Close() error {
	return nil
}

<commit_msg>In Close(), actually close the reader if it implements io.ReadCloser.  JIRA NEO-537<commit_after>package channels

import "io"

type Ds3ObjectReadCloser struct {
	io.Reader
}

func NewDs3ObjectReadCloserDecorator(reader io.Reader) io.ReadCloser {
	return Ds3ObjectReadCloser { Reader: reader }
}

func (Ds3ObjectReadCloser Ds3ObjectReadCloser) Close() error {
	readCloser, isReadCloser := Ds3ObjectReadCloser.Reader.(io.ReadCloser)
	if isReadCloser && readCloser != nil {
		return readCloser.Close()
	}
	return nil
}

<|endoftext|>


<commit_before>package twelve

import (
	"fmt"
	"strings"
	"testing"
)

type testCase struct {
	input    int
	expected string
}

var testCases = []testCase{
	{1, "On the first day of Christmas my true love gave to me: a Partridge in a Pear Tree."},
	{2, "On the second day of Christmas my true love gave to me: two Turtle Doves, and a Partridge in a Pear Tree."},
	{3, "On the third day of Christmas my true love gave to me: three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
	{4, "On the fourth day of Christmas my true love gave to me: four Calling Birds, three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
	{5, "On the fifth day of Christmas my true love gave to me: five Gold Rings, four Calling Birds, three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
	{6, "On the sixth day of Christmas my true love gave to me: six Geese-a-Laying, five Gold Rings, four Calling Birds, three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
	{7, "On the seventh day of Christmas my true love gave to me: seven Swans-a-Swimming, six Geese-a-Laying, five Gold Rings, four Calling Birds, three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
	{8, "On the eighth day of Christmas my true love gave to me: eight Maids-a-Milking, seven Swans-a-Swimming, six Geese-a-Laying, five Gold Rings, four Calling Birds, three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
	{9, "On the ninth day of Christmas my true love gave to me: nine Ladies Dancing, eight Maids-a-Milking, seven Swans-a-Swimming, six Geese-a-Laying, five Gold Rings, four Calling Birds, three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
	{10, "On the tenth day of Christmas my true love gave to me: ten Lords-a-Leaping, nine Ladies Dancing, eight Maids-a-Milking, seven Swans-a-Swimming, six Geese-a-Laying, five Gold Rings, four Calling Birds, three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
	{11, "On the eleventh day of Christmas my true love gave to me: eleven Pipers Piping, ten Lords-a-Leaping, nine Ladies Dancing, eight Maids-a-Milking, seven Swans-a-Swimming, six Geese-a-Laying, five Gold Rings, four Calling Birds, three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
	{12, "On the twelfth day of Christmas my true love gave to me: twelve Drummers Drumming, eleven Pipers Piping, ten Lords-a-Leaping, nine Ladies Dancing, eight Maids-a-Milking, seven Swans-a-Swimming, six Geese-a-Laying, five Gold Rings, four Calling Birds, three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
}

// diff compares two multi-line strings and returns a helpful comment
func diff(got, want string) string {
	g := strings.Split(got, "\n")
	w := strings.Split(want, "\n")
	for i := 0; ; i++ {
		switch {
		case i < len(g) && i < len(w):
			if g[i] == w[i] {
				continue
			}
			return fmt.Sprintf("-- first difference in line %d:\n"+
				"-- got : %q\n-- want: %q\n", i+1, g[i], w[i])
		case i < len(g):
			return fmt.Sprintf("-- got %d extra lines after line %d:\n"+
				"-- first extra line: %q\n", len(g)-len(w), i, g[i])
		case i < len(w):
			return fmt.Sprintf("-- got %d correct lines, want %d more lines:\n"+
				"-- want next: %q\n", i, len(w)-i, w[i])
		default:
			return "no differences found"
		}
	}
}

func TestSong(t *testing.T) {
	var verses []string
	for _, test := range testCases {
		verses = append(verses, test.expected)
	}
	var expected = strings.Join(verses, "\n")
	actual := Song()
	if expected != actual {
		t.Fatalf("Song() =\n%s\n  want:\n%s\n%s", actual, expected, diff(actual, expected))
	}
}

func TestVerse(t *testing.T) {
	for _, test := range testCases {
		actual := Verse(test.input)
		if actual != test.expected {
			t.Errorf("Twelve Days test [%d], expected [%s], actual [%s]", test.input, test.expected, actual)
		}
	}
}
<commit_msg>twelve days: add benchmark<commit_after>package twelve

import (
	"fmt"
	"strings"
	"testing"
)

type testCase struct {
	input    int
	expected string
}

var testCases = []testCase{
	{1, "On the first day of Christmas my true love gave to me: a Partridge in a Pear Tree."},
	{2, "On the second day of Christmas my true love gave to me: two Turtle Doves, and a Partridge in a Pear Tree."},
	{3, "On the third day of Christmas my true love gave to me: three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
	{4, "On the fourth day of Christmas my true love gave to me: four Calling Birds, three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
	{5, "On the fifth day of Christmas my true love gave to me: five Gold Rings, four Calling Birds, three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
	{6, "On the sixth day of Christmas my true love gave to me: six Geese-a-Laying, five Gold Rings, four Calling Birds, three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
	{7, "On the seventh day of Christmas my true love gave to me: seven Swans-a-Swimming, six Geese-a-Laying, five Gold Rings, four Calling Birds, three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
	{8, "On the eighth day of Christmas my true love gave to me: eight Maids-a-Milking, seven Swans-a-Swimming, six Geese-a-Laying, five Gold Rings, four Calling Birds, three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
	{9, "On the ninth day of Christmas my true love gave to me: nine Ladies Dancing, eight Maids-a-Milking, seven Swans-a-Swimming, six Geese-a-Laying, five Gold Rings, four Calling Birds, three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
	{10, "On the tenth day of Christmas my true love gave to me: ten Lords-a-Leaping, nine Ladies Dancing, eight Maids-a-Milking, seven Swans-a-Swimming, six Geese-a-Laying, five Gold Rings, four Calling Birds, three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
	{11, "On the eleventh day of Christmas my true love gave to me: eleven Pipers Piping, ten Lords-a-Leaping, nine Ladies Dancing, eight Maids-a-Milking, seven Swans-a-Swimming, six Geese-a-Laying, five Gold Rings, four Calling Birds, three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
	{12, "On the twelfth day of Christmas my true love gave to me: twelve Drummers Drumming, eleven Pipers Piping, ten Lords-a-Leaping, nine Ladies Dancing, eight Maids-a-Milking, seven Swans-a-Swimming, six Geese-a-Laying, five Gold Rings, four Calling Birds, three French Hens, two Turtle Doves, and a Partridge in a Pear Tree."},
}

// diff compares two multi-line strings and returns a helpful comment
func diff(got, want string) string {
	g := strings.Split(got, "\n")
	w := strings.Split(want, "\n")
	for i := 0; ; i++ {
		switch {
		case i < len(g) && i < len(w):
			if g[i] == w[i] {
				continue
			}
			return fmt.Sprintf("-- first difference in line %d:\n"+
				"-- got : %q\n-- want: %q\n", i+1, g[i], w[i])
		case i < len(g):
			return fmt.Sprintf("-- got %d extra lines after line %d:\n"+
				"-- first extra line: %q\n", len(g)-len(w), i, g[i])
		case i < len(w):
			return fmt.Sprintf("-- got %d correct lines, want %d more lines:\n"+
				"-- want next: %q\n", i, len(w)-i, w[i])
		default:
			return "no differences found"
		}
	}
}

func TestSong(t *testing.T) {
	var verses []string
	for _, test := range testCases {
		verses = append(verses, test.expected)
	}
	var expected = strings.Join(verses, "\n")
	actual := Song()
	if expected != actual {
		t.Fatalf("Song() =\n%s\n  want:\n%s\n%s", actual, expected, diff(actual, expected))
	}
}

func TestVerse(t *testing.T) {
	for _, test := range testCases {
		actual := Verse(test.input)
		if actual != test.expected {
			t.Errorf("Twelve Days test [%d], expected [%s], actual [%s]", test.input, test.expected, actual)
		}
	}
}

func BenchmarkVerse(b *testing.B) {
	for i := 0; i < b.N; i++ {
		for _, test := range testCases {
			Verse(test.input)
		}
	}
}

func BenchmarkSong(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Song()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"fmt"
	"hash/fnv"
	"log"
	"os"
	"strings"
	"time"
)

const (
	SEED_SIZE = 16
)

func check_result(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

type QueryPos struct {
	name    int
	pos     int
	content string
}

func string_to_hash(s string) uint32 {
	h := fnv.New32a()
	h.Write([]byte(s))
	return h.Sum32()
}

func addSequence(content string, sequence int, hash map[uint32][]QueryPos) {
	// for an exact match we only have to hash the start of the query
	position := 0
	kmer := content[position : position+SEED_SIZE]
	kmer_hash := string_to_hash(kmer)
	entry := QueryPos{name: sequence, pos: position, content: content}
	query, ok := hash[kmer_hash]
	if !ok {
		query = make([]QueryPos, 0)
	}
	query = append(query, entry)
	hash[kmer_hash] = query
}

func searchSequence(content string, hash map[uint32][]QueryPos, sequence_name string, genome_filename string, sequence_names []string) {
	for position := 0; position <= len(content)-SEED_SIZE; position += 1 {
		kmer := content[position : position+SEED_SIZE]
		kmer_hash := string_to_hash(kmer)
		query, ok := hash[kmer_hash]
		if ok {
			for _, candidate := range query {
				if position+len(candidate.content) <= len(content) && content[position:position+len(candidate.content)] == candidate.content {
					fmt.Fprintf(os.Stdout, "%s\t%s\t%s\t%v\n", genome_filename, sequence_name, sequence_names[candidate.name], position)
				}
			}
		}
	}
}

func main() {
	sequences_filename := os.Args[1]
	fmt.Fprintf(os.Stderr, "%s: processing: %s\n", time.Now().String(), sequences_filename)

	file, err := os.Open(sequences_filename)
	check_result(err)
	defer file.Close()

	gr, err := gzip.NewReader(file)
	check_result(err)
	defer gr.Close()

	scanner := bufio.NewScanner(gr)
	lines := 0
	sequence_count := 0

	var sequence_names []string
	var query_kmers map[uint32][]QueryPos = make(map[uint32][]QueryPos)
	var content *bytes.Buffer

	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, ">") {
			if content != nil {
				addSequence(content.String(), sequence_count-1, query_kmers)
			}
			sequence_names = append(sequence_names, line[1:])
			sequence_count++
			content = new(bytes.Buffer)
		} else {
			(*content).WriteString(line)
		}
		lines++
		if lines%1000000 == 0 {
			fmt.Fprintf(os.Stderr, "%s: processing: %s: %v lines %v sequences. %v kmers\n", time.Now().String(), sequences_filename, lines, len(sequence_names), len(query_kmers))
			// debug.FreeOSMemory()
		}
	}
	addSequence(content.String(), sequence_count-1, query_kmers)
	fmt.Fprintf(os.Stderr, "%s: done processing: %s: %v lines %v sequences.\n", time.Now().String(), sequences_filename, lines, len(sequence_names))

	// process each genome - list matching sequences
	fmt.Fprintf(os.Stdout, "Genome\tGenomeContig\tQueryContig\tGenomePosition\n")
	for _, genome_filename := range os.Args[2:] {
		fmt.Fprintf(os.Stderr, "%s: processing genome: %s\n", time.Now().String(), genome_filename)
		file, err := os.Open(genome_filename)
		check_result(err)
		defer file.Close()

		lines = 0
		sequence_count = 0
		content = new(bytes.Buffer)
		r := bufio.NewReader(file)
		scanner := bufio.NewScanner(r)
		var sequence_name string
		for scanner.Scan() {
			line := scanner.Text()
			if strings.HasPrefix(line, ">") {
				if content != nil {
					searchSequence(content.String(), query_kmers, sequence_name, genome_filename, sequence_names)
				}
				sequence_name = line[1:]
				sequence_count++
				content = new(bytes.Buffer)
			} else {
				(*content).WriteString(line)
			}
			lines++
		}
		searchSequence(content.String(), query_kmers, sequence_name, genome_filename, sequence_names)
		fmt.Fprintf(os.Stderr, "%s: done genome: %s. %v lines. %v sequences.\n", time.Now().String(), genome_filename, lines, sequence_count)
	}
}
<commit_msg>usage message<commit_after>package main

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"hash/fnv"
	"log"
	"os"
	"strings"
	"time"
)

const (
	SEED_SIZE = 16
)

func check_result(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

type QueryPos struct {
	name    int
	pos     int
	content string
}

func string_to_hash(s string) uint32 {
	h := fnv.New32a()
	h.Write([]byte(s))
	return h.Sum32()
}

func addSequence(content string, sequence int, hash map[uint32][]QueryPos) {
	// for an exact match we only have to hash the start of the query
	position := 0
	kmer := content[position : position+SEED_SIZE]
	kmer_hash := string_to_hash(kmer)
	entry := QueryPos{name: sequence, pos: position, content: content}
	query, ok := hash[kmer_hash]
	if !ok {
		query = make([]QueryPos, 0)
	}
	query = append(query, entry)
	hash[kmer_hash] = query
}

func searchSequence(content string, hash map[uint32][]QueryPos, sequence_name string, genome_filename string, sequence_names []string) {
	for position := 0; position <= len(content)-SEED_SIZE; position += 1 {
		kmer := content[position : position+SEED_SIZE]
		kmer_hash := string_to_hash(kmer)
		query, ok := hash[kmer_hash]
		if ok {
			for _, candidate := range query {
				if position+len(candidate.content) <= len(content) && content[position:position+len(candidate.content)] == candidate.content {
					fmt.Fprintf(os.Stdout, "%s\t%s\t%s\t%v\n", genome_filename, sequence_name, sequence_names[candidate.name], position)
				}
			}
		}
	}
}

func main() {
	if len(os.Args) < 2 {
		fmt.Fprintf(os.Stderr, "Usage: fastaquery queries.fq.gz genome.fa [genome.fa...]\n")
		flag.PrintDefaults()
		os.Exit(1)
	}
	sequences_filename := os.Args[1]
	fmt.Fprintf(os.Stderr, "%s: processing: %s\n", time.Now().String(), sequences_filename)

	file, err := os.Open(sequences_filename)
	check_result(err)
	defer file.Close()

	gr, err := gzip.NewReader(file)
	check_result(err)
	defer gr.Close()

	scanner := bufio.NewScanner(gr)
	lines := 0
	sequence_count := 0

	var sequence_names []string
	var query_kmers map[uint32][]QueryPos = make(map[uint32][]QueryPos)
	var content *bytes.Buffer

	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, ">") {
			if content != nil {
				addSequence(content.String(), sequence_count-1, query_kmers)
			}
			sequence_names = append(sequence_names, line[1:])
			sequence_count++
			content = new(bytes.Buffer)
		} else {
			(*content).WriteString(line)
		}
		lines++
		if lines%1000000 == 0 {
			fmt.Fprintf(os.Stderr, "%s: processing: %s: %v lines %v sequences. %v kmers\n", time.Now().String(), sequences_filename, lines, len(sequence_names), len(query_kmers))
			// debug.FreeOSMemory()
		}
	}
	addSequence(content.String(), sequence_count-1, query_kmers)
	fmt.Fprintf(os.Stderr, "%s: done processing: %s: %v lines %v sequences.\n", time.Now().String(), sequences_filename, lines, len(sequence_names))

	// process each genome - list matching sequences
	fmt.Fprintf(os.Stdout, "Genome\tGenomeContig\tQueryContig\tGenomePosition\n")
	for _, genome_filename := range os.Args[2:] {
		fmt.Fprintf(os.Stderr, "%s: processing genome: %s\n", time.Now().String(), genome_filename)
		file, err := os.Open(genome_filename)
		check_result(err)
		defer file.Close()

		lines = 0
		sequence_count = 0
		content = new(bytes.Buffer)
		r := bufio.NewReader(file)
		scanner := bufio.NewScanner(r)
		var sequence_name string
		for scanner.Scan() {
			line := scanner.Text()
			if strings.HasPrefix(line, ">") {
				if content != nil {
					searchSequence(content.String(), query_kmers, sequence_name, genome_filename, sequence_names)
				}
				sequence_name = line[1:]
				sequence_count++
				content = new(bytes.Buffer)
			} else {
				(*content).WriteString(line)
			}
			lines++
		}
		searchSequence(content.String(), query_kmers, sequence_name, genome_filename, sequence_names)
		fmt.Fprintf(os.Stderr, "%s: done genome: %s. %v lines. %v sequences.\n", time.Now().String(), genome_filename, lines, sequence_count)
	}
}
<|endoftext|>


<commit_before>package golog

import "testing"

func TestAsserta(t *testing.T) {
    db0 := NewDatabase()
    db1 := db0.Asserta(NewTerm("alpha"))
    db2 := db1.Asserta(NewTerm("beta"))
    db3 := db2.Asserta(NewTerm("gamma", NewTerm("greek to me")))

    // do we have the right number of clauses?
    if db0.ClauseCount() != 0 {
        t.Errorf("db0: wrong number of clauses: %d", db0.ClauseCount())
    }
    if db1.ClauseCount() != 1 {
        t.Errorf("db0: wrong number of clauses: %d", db0.ClauseCount())
    }
    if db2.ClauseCount() != 2 {
        t.Errorf("db0: wrong number of clauses: %d", db0.ClauseCount())
    }
    if db3.ClauseCount() != 3 {
        t.Errorf("db0: wrong number of clauses: %d", db0.ClauseCount())
    }
}
<commit_msg>Test against a clause head and body<commit_after>package golog

import "testing"

func TestAsserta(t *testing.T) {
    db0 := NewDatabase()
    db1 := db0.Asserta(NewTerm("alpha"))
    db2 := db1.Asserta(NewTerm("beta"))

    foo := NewTerm("foo", NewTerm("one"), NewTerm("two"))
    body := NewTerm("alpha")
    db3 := db2.Asserta(NewTerm(":-", foo, body))
    t.Logf(db3.String()) // helpful for debugging

    // do we have the right number of clauses?
    if db0.ClauseCount() != 0 {
        t.Errorf("db0: wrong number of clauses: %d", db0.ClauseCount())
    }
    if db1.ClauseCount() != 1 {
        t.Errorf("db0: wrong number of clauses: %d", db0.ClauseCount())
    }
    if db2.ClauseCount() != 2 {
        t.Errorf("db0: wrong number of clauses: %d", db0.ClauseCount())
    }
    if db3.ClauseCount() != 3 {
        t.Errorf("db0: wrong number of clauses: %d", db0.ClauseCount())
    }
}
<|endoftext|>


<commit_before>// Copyright 2017 by the contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package healthcheck

import (
	"net/http"
)

// Check is a health/readiness check.
type Check func() error

// Handler is an http.Handler with additional methods that register health and
// readiness checks. It handles handle "/live" and "/ready" HTTP
// endpoints.
type Handler interface {
	// The Handler is an http.Handler, so it can be exposed directly and handle
	// /live and /ready endpoints.
	http.Handler

	// AddLivenessCheck adds a check that indicates that this instance of the
	// application should be destroyed or restarted. A failed liveness check
	// indicates that this instance is unhealthy, not some upstream dependency.
	// Every liveness check is also included as a readiness check.
	AddLivenessCheck(name string, check Check)

	// AddReadinessCheck adds a check that indicates that this instance of the
	// application is currently unable to serve requests because of an upstream
	// or some transient failure. If a readiness check fails, this instance
	// should no longer receiver requests, but should not be restarted or
	// destroyed.
	AddReadinessCheck(name string, check Check)

	// LiveEndpoint is the HTTP handler for just the /live endpoint, which is
	// useful if you need to attach it into your own HTTP handler tree.
	LiveEndpoint(http.ResponseWriter, *http.Request)

	// ReadyEndpoint is the HTTP handler for just the /ready endpoint, which is
	// useful if you need to attach it into your own HTTP handler tree.
	ReadyEndpoint(http.ResponseWriter, *http.Request)
}

// Trigger represents a health check that is tripped by a discrete event such
// as a batch job failing.
type Trigger interface {
	// Trip sets the current status of the Trigger to the specified error
	Trip(error)

	// Reset the trigger to a healthy state (shorthand for Trip(nil)).
	Reset()

	// Check returns a check function suitable for `Handler.AddHealthCheck`.
	Check() Check
}
<commit_msg>Drop Trigger interface.<commit_after>// Copyright 2017 by the contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package healthcheck

import (
	"net/http"
)

// Check is a health/readiness check.
type Check func() error

// Handler is an http.Handler with additional methods that register health and
// readiness checks. It handles handle "/live" and "/ready" HTTP
// endpoints.
type Handler interface {
	// The Handler is an http.Handler, so it can be exposed directly and handle
	// /live and /ready endpoints.
	http.Handler

	// AddLivenessCheck adds a check that indicates that this instance of the
	// application should be destroyed or restarted. A failed liveness check
	// indicates that this instance is unhealthy, not some upstream dependency.
	// Every liveness check is also included as a readiness check.
	AddLivenessCheck(name string, check Check)

	// AddReadinessCheck adds a check that indicates that this instance of the
	// application is currently unable to serve requests because of an upstream
	// or some transient failure. If a readiness check fails, this instance
	// should no longer receiver requests, but should not be restarted or
	// destroyed.
	AddReadinessCheck(name string, check Check)

	// LiveEndpoint is the HTTP handler for just the /live endpoint, which is
	// useful if you need to attach it into your own HTTP handler tree.
	LiveEndpoint(http.ResponseWriter, *http.Request)

	// ReadyEndpoint is the HTTP handler for just the /ready endpoint, which is
	// useful if you need to attach it into your own HTTP handler tree.
	ReadyEndpoint(http.ResponseWriter, *http.Request)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"github.com/fsouza/inv_bots/lib"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"log"
	"os"
	"text/template"
	"time"
)

const (
	dbName                = "bovespa_plantao_empresas"
	newsCollName          = "news"
	notificationsCollName = "notifications"
)

var emailTemplate = template.Must(template.New("report").Parse(`Subject: {{.subject}}
To: {{.recipient}}
From: {{.sender}}

{{.subject}}

{{.link}}`))

var (
	baseURL    string
	sender     string
	password   string
	recipient  string
	tickerTime time.Duration
)

type News struct {
	ID    string `bson:"_id"`
	Title string
	Date  time.Time
}

type Notification struct {
	NewsID    string
	Date      time.Time
	Recipient string
}

func init() {
	flag.StringVar(&sender, "s", "", "Email address of the sender, for authentication in Gmail")
	flag.StringVar(&password, "p", "", "Email password of the sender, for authentication in Gmail")
	flag.StringVar(&recipient, "r", "", "Email address of the recipient")
	flag.StringVar(&baseURL, "u", "", "Base URL")
	flag.DurationVar(&tickerTime, "t", time.Minute, "Ticker interval")
}

func connect() (*mgo.Session, error) {
	return mgo.Dial("localhost:27017")
}

func notificationsCollection(session *mgo.Session) *mgo.Collection {
	collection := session.DB(dbName).C(notificationsCollName)
	collection.EnsureIndex(mgo.Index{Key: []string{"newsid"}, Background: true})
	return collection
}

func poolRecords(ticker <-chan time.Time) {
	for _ = range ticker {
		records := getRecords()
		if len(records) > 0 {
			notifyRecords(records)
		}
	}
}

func getNotificatedNews() ([]string, error) {
	session, err := connect()
	if err != nil {
		return nil, err
	}
	defer session.Close()
	collection := notificationsCollection(session)
	var notifications []Notification
	err = collection.Find(nil).Select(bson.M{"newsid": 1}).All(&notifications)
	if err != nil {
		return nil, err
	}
	result := make([]string, len(notifications))
	for i, notification := range notifications {
		result[i] = notification.NewsID
	}
	return result, nil
}

func getRecords() []News {
	session, err := connect()
	if err != nil {
		log.Printf("ERROR: %s", err)
		return nil
	}
	defer session.Close()
	notificated, err := getNotificatedNews()
	if err != nil {
		log.Printf("ERROR: %s", err)
		return nil
	}
	collection := session.DB(dbName).C(newsCollName)
	var newsList []News
	query := bson.M{
		"_id":   bson.M{"$nin": notificated},
		"title": bson.M{"$regex": "^fii.*(relatorio|informe)", "$options": "i"},
	}
	err = collection.Find(query).All(&newsList)
	if err != nil {
		log.Printf("ERROR: %s", err)
		return nil
	}
	return newsList
}

func notifyRecords(newsList []News) {
	mailSender, err := lib.NewGmailSender(sender, password)
	if err != nil {
		log.Printf("ERROR: %s", err)
		return
	}
	defer mailSender.Close()
	session, err := connect()
	if err != nil {
		log.Printf("ERROR: %s", err)
		return
	}
	defer session.Close()
	for _, news := range newsList {
		var body bytes.Buffer
		emailTemplate.Execute(&body, map[string]string{
			"subject":   news.Title,
			"recipient": recipient,
			"sender":    sender,
			"link":      baseURL + news.ID,
		})
		err := mailSender.SendMail(recipient, body.Bytes())
		if err != nil {
			log.Printf("ERROR: %s", err)
			return
		}
		collection := notificationsCollection(session)
		collection.Insert(Notification{NewsID: news.ID, Recipient: recipient, Date: time.Now()})
	}
}

func main() {
	var failures int
	flag.Parse()
	if sender == "" {
		log.Print("Please provide the sender")
		failures++
	}
	if recipient == "" {
		log.Print("Please provide the recipient")
		failures++
	}
	if password == "" {
		log.Print("Please provide the password")
		failures++
	}
	if baseURL == "" {
		log.Print("Please provide the base URL")
		failures++
	}
	if failures > 0 {
		os.Exit(2)
	}
	poolRecords(time.Tick(tickerTime))
}
<commit_msg>fii_report: expand regexp for "carta"<commit_after>package main

import (
	"bytes"
	"flag"
	"github.com/fsouza/inv_bots/lib"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"log"
	"os"
	"text/template"
	"time"
)

const (
	dbName                = "bovespa_plantao_empresas"
	newsCollName          = "news"
	notificationsCollName = "notifications"
)

var emailTemplate = template.Must(template.New("report").Parse(`Subject: {{.subject}}
To: {{.recipient}}
From: {{.sender}}

{{.subject}}

{{.link}}`))

var (
	baseURL    string
	sender     string
	password   string
	recipient  string
	tickerTime time.Duration
)

type News struct {
	ID    string `bson:"_id"`
	Title string
	Date  time.Time
}

type Notification struct {
	NewsID    string
	Date      time.Time
	Recipient string
}

func init() {
	flag.StringVar(&sender, "s", "", "Email address of the sender, for authentication in Gmail")
	flag.StringVar(&password, "p", "", "Email password of the sender, for authentication in Gmail")
	flag.StringVar(&recipient, "r", "", "Email address of the recipient")
	flag.StringVar(&baseURL, "u", "", "Base URL")
	flag.DurationVar(&tickerTime, "t", time.Minute, "Ticker interval")
}

func connect() (*mgo.Session, error) {
	return mgo.Dial("localhost:27017")
}

func notificationsCollection(session *mgo.Session) *mgo.Collection {
	collection := session.DB(dbName).C(notificationsCollName)
	collection.EnsureIndex(mgo.Index{Key: []string{"newsid"}, Background: true})
	return collection
}

func poolRecords(ticker <-chan time.Time) {
	for _ = range ticker {
		records := getRecords()
		if len(records) > 0 {
			notifyRecords(records)
		}
	}
}

func getNotificatedNews() ([]string, error) {
	session, err := connect()
	if err != nil {
		return nil, err
	}
	defer session.Close()
	collection := notificationsCollection(session)
	var notifications []Notification
	err = collection.Find(nil).Select(bson.M{"newsid": 1}).All(&notifications)
	if err != nil {
		return nil, err
	}
	result := make([]string, len(notifications))
	for i, notification := range notifications {
		result[i] = notification.NewsID
	}
	return result, nil
}

func getRecords() []News {
	session, err := connect()
	if err != nil {
		log.Printf("ERROR: %s", err)
		return nil
	}
	defer session.Close()
	notificated, err := getNotificatedNews()
	if err != nil {
		log.Printf("ERROR: %s", err)
		return nil
	}
	collection := session.DB(dbName).C(newsCollName)
	var newsList []News
	query := bson.M{
		"_id":   bson.M{"$nin": notificated},
		"title": bson.M{"$regex": "^fii.*(relatorio|informe|carta)", "$options": "i"},
	}
	err = collection.Find(query).All(&newsList)
	if err != nil {
		log.Printf("ERROR: %s", err)
		return nil
	}
	return newsList
}

func notifyRecords(newsList []News) {
	mailSender, err := lib.NewGmailSender(sender, password)
	if err != nil {
		log.Printf("ERROR: %s", err)
		return
	}
	defer mailSender.Close()
	session, err := connect()
	if err != nil {
		log.Printf("ERROR: %s", err)
		return
	}
	defer session.Close()
	for _, news := range newsList {
		var body bytes.Buffer
		emailTemplate.Execute(&body, map[string]string{
			"subject":   news.Title,
			"recipient": recipient,
			"sender":    sender,
			"link":      baseURL + news.ID,
		})
		err := mailSender.SendMail(recipient, body.Bytes())
		if err != nil {
			log.Printf("ERROR: %s", err)
			return
		}
		collection := notificationsCollection(session)
		collection.Insert(Notification{NewsID: news.ID, Recipient: recipient, Date: time.Now()})
	}
}

func main() {
	var failures int
	flag.Parse()
	if sender == "" {
		log.Print("Please provide the sender")
		failures++
	}
	if recipient == "" {
		log.Print("Please provide the recipient")
		failures++
	}
	if password == "" {
		log.Print("Please provide the password")
		failures++
	}
	if baseURL == "" {
		log.Print("Please provide the base URL")
		failures++
	}
	if failures > 0 {
		os.Exit(2)
	}
	poolRecords(time.Tick(tickerTime))
}
<|endoftext|>