<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/cloudfoundry-incubator/cf-debug-server"
	"github.com/cloudfoundry-incubator/cf-lager"
	"github.com/cloudfoundry-incubator/cf-tcp-router/config"
	"github.com/cloudfoundry-incubator/cf-tcp-router/configurer"
	"github.com/cloudfoundry-incubator/cf-tcp-router/metrics_reporter"
	"github.com/cloudfoundry-incubator/cf-tcp-router/metrics_reporter/haproxy_client"
	"github.com/cloudfoundry-incubator/cf-tcp-router/models"
	"github.com/cloudfoundry-incubator/cf-tcp-router/routing_table"
	"github.com/cloudfoundry-incubator/cf-tcp-router/syncer"
	"github.com/cloudfoundry-incubator/cf-tcp-router/watcher"
	"github.com/cloudfoundry-incubator/routing-api"
	uaaclient "github.com/cloudfoundry-incubator/uaa-go-client"
	uaaconfig "github.com/cloudfoundry-incubator/uaa-go-client/config"
	"github.com/cloudfoundry/dropsonde"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/sigmon"
)

const (
	defaultTokenFetchRetryInterval = 5 * time.Second
	defaultTokenFetchNumRetries    = uint(3)
)

var tcpLoadBalancer = flag.String(
	"tcpLoadBalancer",
	configurer.HaProxyConfigurer,
	"The tcp load balancer to use.",
)

var tcpLoadBalancerBaseCfg = flag.String(
	"tcpLoadBalancerBaseConfig",
	"",
	"The tcp load balancer base configuration file name. This contains the basic header information.",
)

var tcpLoadBalancerCfg = flag.String(
	"tcpLoadBalancerConfig",
	"",
	"The tcp load balancer configuration file name.",
)

var tcpLoadBalancerStatsUnixSocket = flag.String(
	"tcpLoadBalancerStatsUnixSocket",
	"/var/vcap/jobs/haproxy/config/haproxy.sock",
	"Unix domain socket for tcp load balancer",
)

var subscriptionRetryInterval = flag.Int(
	"subscriptionRetryInterval",
	5,
	"Retry interval between retries to subscribe for tcp events from routing api (in seconds)",
)

var configFile = flag.String(
	"config",
	"/var/vcap/jobs/router_configurer/config/router_configurer.yml",
	"The Router configurer yml config.",
)

var syncInterval = flag.Duration(
	"syncInterval",
	time.Minute,
	"The interval between syncs of the routing table from routing api.",
)

var tokenFetchMaxRetries = flag.Uint(
	"tokenFetchMaxRetries",
	defaultTokenFetchNumRetries,
	"Maximum number of retries the Token Fetcher will use every time FetchToken is called",
)

var tokenFetchRetryInterval = flag.Duration(
	"tokenFetchRetryInterval",
	defaultTokenFetchRetryInterval,
	"interval to wait before TokenFetcher retries to fetch a token",
)

var tokenFetchExpirationBufferTime = flag.Uint64(
	"tokenFetchExpirationBufferTime",
	30,
	"Buffer time in seconds before the actual token expiration time, when TokenFetcher consider a token expired",
)

var statsCollectionInterval = flag.Duration(
	"statsCollectionInterval",
	time.Minute,
	"The interval between collection of stats from tcp load balancer.",
)

var dropsondePort = flag.Int(
	"dropsondePort",
	3457,
	"Port the local metron agent is listening on",
)

const (
	dropsondeOrigin        = "router-configurer"
	statsConnectionTimeout = 10 * time.Second
)

func main() {
	cf_debug_server.AddFlags(flag.CommandLine)
	cf_lager.AddFlags(flag.CommandLine)
	flag.Parse()

	logger, reconfigurableSink := cf_lager.New("router-configurer")
	logger.Info("starting")
	clock := clock.NewClock()

	initializeDropsonde(logger)

	routingTable := models.NewRoutingTable()
	configurer := configurer.NewConfigurer(logger,
		*tcpLoadBalancer, *tcpLoadBalancerBaseCfg, *tcpLoadBalancerCfg)

	cfg, err := config.New(*configFile)
	if err != nil {
		logger.Error("failed-to-unmarshal-config-file", err)
		os.Exit(1)
	}

	uaaClient := newUaaClient(logger, cfg, clock)
	_, err = uaaClient.FetchToken(false)
	if err != nil {
		logger.Error("error-fetching-oauth-token", err)
		os.Exit(1)
	}

	routingAPIAddress := fmt.Sprintf("%s:%d", cfg.RoutingAPI.URI, cfg.RoutingAPI.Port)
	logger.Debug("creating-routing-api-client", lager.Data{"api-location": routingAPIAddress})
	routingAPIClient := routing_api.NewClient(routingAPIAddress)

	updater := routing_table.NewUpdater(logger, &routingTable, configurer, routingAPIClient, uaaClient)
	syncChannel := make(chan struct{})
	syncRunner := syncer.New(clock, *syncInterval, syncChannel, logger)
	watcher := watcher.New(routingAPIClient, updater, uaaClient, *subscriptionRetryInterval, syncChannel, logger)

	haproxyClient := haproxy_client.NewClient(logger, *tcpLoadBalancerStatsUnixSocket, statsConnectionTimeout)
	metricsEmitter := metrics_reporter.NewMetricsEmitter()
	metricsReporter := metrics_reporter.NewMetricsReporter(clock, haproxyClient, metricsEmitter, *statsCollectionInterval)

	members := grouper.Members{
		{"watcher", watcher},
		{"syncer", syncRunner},
		{"metricsReporter", metricsReporter},
	}

	if dbgAddr := cf_debug_server.DebugAddress(flag.CommandLine); dbgAddr != "" {
		members = append(grouper.Members{
			{"debug-server", cf_debug_server.Runner(dbgAddr, reconfigurableSink)},
		}, members...)
	}

	group := grouper.NewOrdered(os.Interrupt, members)

	monitor := ifrit.Invoke(sigmon.New(group))

	logger.Info("started")

	err = <-monitor.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
}

func newUaaClient(logger lager.Logger, c *config.Config, klok clock.Clock) uaaclient.Client {
	if c.RoutingAPI.AuthDisabled {
		logger.Debug("creating-noop-token-fetcher")
		client := uaaclient.NewNoOpUaaClient()
		return client
	}
	logger.Debug("creating-uaa-token-fetcher")

	if c.OAuth.Port == -1 {
		logger.Fatal("tls-not-enabled", errors.New("TcpRouter requires to communicate with UAA over TLS"), lager.Data{"token-endpoint": c.OAuth.TokenEndpoint, "port": c.OAuth.Port})
	}

	scheme := "https"
	tokenURL := fmt.Sprintf("%s://%s:%d", scheme, c.OAuth.TokenEndpoint, c.OAuth.Port)
	logger.Info(fmt.Sprintf("using-%s-scheme-for-uaa", scheme))

	cfg := &uaaconfig.Config{
		UaaEndpoint:           tokenURL,
		SkipVerification:      c.OAuth.SkipOAuthTLSVerification,
		ClientName:            c.OAuth.ClientName,
		ClientSecret:          c.OAuth.ClientSecret,
		MaxNumberOfRetries:    uint32(*tokenFetchMaxRetries),
		RetryInterval:         *tokenFetchRetryInterval,
		ExpirationBufferInSec: int64(*tokenFetchExpirationBufferTime),
	}

	logger.Info("fetching-token-from-uaa")

	uaaClient, err := uaaclient.NewClient(logger, cfg, klok)
	if err != nil {
		logger.Fatal("initialize-token-fetcher-error", err)
	}
	return uaaClient
}

func initializeDropsonde(logger lager.Logger) {
	dropsondeDestination := fmt.Sprintf("localhost:%d", *dropsondePort)
	err := dropsonde.Initialize(dropsondeDestination, dropsondeOrigin)
	if err != nil {
		logger.Error("failed-to-initialize-dropsonde", err)
	}
}
<commit_msg>Remove reference to uaa-token-fetcher<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/cloudfoundry-incubator/cf-debug-server"
	"github.com/cloudfoundry-incubator/cf-lager"
	"github.com/cloudfoundry-incubator/cf-tcp-router/config"
	"github.com/cloudfoundry-incubator/cf-tcp-router/configurer"
	"github.com/cloudfoundry-incubator/cf-tcp-router/metrics_reporter"
	"github.com/cloudfoundry-incubator/cf-tcp-router/metrics_reporter/haproxy_client"
	"github.com/cloudfoundry-incubator/cf-tcp-router/models"
	"github.com/cloudfoundry-incubator/cf-tcp-router/routing_table"
	"github.com/cloudfoundry-incubator/cf-tcp-router/syncer"
	"github.com/cloudfoundry-incubator/cf-tcp-router/watcher"
	"github.com/cloudfoundry-incubator/routing-api"
	uaaclient "github.com/cloudfoundry-incubator/uaa-go-client"
	uaaconfig "github.com/cloudfoundry-incubator/uaa-go-client/config"
	"github.com/cloudfoundry/dropsonde"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/sigmon"
)

const (
	defaultTokenFetchRetryInterval = 5 * time.Second
	defaultTokenFetchNumRetries    = uint(3)
)

var tcpLoadBalancer = flag.String(
	"tcpLoadBalancer",
	configurer.HaProxyConfigurer,
	"The tcp load balancer to use.",
)

var tcpLoadBalancerBaseCfg = flag.String(
	"tcpLoadBalancerBaseConfig",
	"",
	"The tcp load balancer base configuration file name. This contains the basic header information.",
)

var tcpLoadBalancerCfg = flag.String(
	"tcpLoadBalancerConfig",
	"",
	"The tcp load balancer configuration file name.",
)

var tcpLoadBalancerStatsUnixSocket = flag.String(
	"tcpLoadBalancerStatsUnixSocket",
	"/var/vcap/jobs/haproxy/config/haproxy.sock",
	"Unix domain socket for tcp load balancer",
)

var subscriptionRetryInterval = flag.Int(
	"subscriptionRetryInterval",
	5,
	"Retry interval between retries to subscribe for tcp events from routing api (in seconds)",
)

var configFile = flag.String(
	"config",
	"/var/vcap/jobs/router_configurer/config/router_configurer.yml",
	"The Router configurer yml config.",
)

var syncInterval = flag.Duration(
	"syncInterval",
	time.Minute,
	"The interval between syncs of the routing table from routing api.",
)

var tokenFetchMaxRetries = flag.Uint(
	"tokenFetchMaxRetries",
	defaultTokenFetchNumRetries,
	"Maximum number of retries the Token Fetcher will use every time FetchToken is called",
)

var tokenFetchRetryInterval = flag.Duration(
	"tokenFetchRetryInterval",
	defaultTokenFetchRetryInterval,
	"interval to wait before TokenFetcher retries to fetch a token",
)

var tokenFetchExpirationBufferTime = flag.Uint64(
	"tokenFetchExpirationBufferTime",
	30,
	"Buffer time in seconds before the actual token expiration time, when TokenFetcher consider a token expired",
)

var statsCollectionInterval = flag.Duration(
	"statsCollectionInterval",
	time.Minute,
	"The interval between collection of stats from tcp load balancer.",
)

var dropsondePort = flag.Int(
	"dropsondePort",
	3457,
	"Port the local metron agent is listening on",
)

const (
	dropsondeOrigin        = "router-configurer"
	statsConnectionTimeout = 10 * time.Second
)

func main() {
	cf_debug_server.AddFlags(flag.CommandLine)
	cf_lager.AddFlags(flag.CommandLine)
	flag.Parse()

	logger, reconfigurableSink := cf_lager.New("router-configurer")
	logger.Info("starting")
	clock := clock.NewClock()

	initializeDropsonde(logger)

	routingTable := models.NewRoutingTable()
	configurer := configurer.NewConfigurer(logger,
		*tcpLoadBalancer, *tcpLoadBalancerBaseCfg, *tcpLoadBalancerCfg)

	cfg, err := config.New(*configFile)
	if err != nil {
		logger.Error("failed-to-unmarshal-config-file", err)
		os.Exit(1)
	}

	uaaClient := newUaaClient(logger, cfg, clock)
	_, err = uaaClient.FetchToken(false)
	if err != nil {
		logger.Error("error-fetching-oauth-token", err)
		os.Exit(1)
	}

	routingAPIAddress := fmt.Sprintf("%s:%d", cfg.RoutingAPI.URI, cfg.RoutingAPI.Port)
	logger.Debug("creating-routing-api-client", lager.Data{"api-location": routingAPIAddress})
	routingAPIClient := routing_api.NewClient(routingAPIAddress)

	updater := routing_table.NewUpdater(logger, &routingTable, configurer, routingAPIClient, uaaClient)
	syncChannel := make(chan struct{})
	syncRunner := syncer.New(clock, *syncInterval, syncChannel, logger)
	watcher := watcher.New(routingAPIClient, updater, uaaClient, *subscriptionRetryInterval, syncChannel, logger)

	haproxyClient := haproxy_client.NewClient(logger, *tcpLoadBalancerStatsUnixSocket, statsConnectionTimeout)
	metricsEmitter := metrics_reporter.NewMetricsEmitter()
	metricsReporter := metrics_reporter.NewMetricsReporter(clock, haproxyClient, metricsEmitter, *statsCollectionInterval)

	members := grouper.Members{
		{"watcher", watcher},
		{"syncer", syncRunner},
		{"metricsReporter", metricsReporter},
	}

	if dbgAddr := cf_debug_server.DebugAddress(flag.CommandLine); dbgAddr != "" {
		members = append(grouper.Members{
			{"debug-server", cf_debug_server.Runner(dbgAddr, reconfigurableSink)},
		}, members...)
	}

	group := grouper.NewOrdered(os.Interrupt, members)

	monitor := ifrit.Invoke(sigmon.New(group))

	logger.Info("started")

	err = <-monitor.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
}

func newUaaClient(logger lager.Logger, c *config.Config, klok clock.Clock) uaaclient.Client {
	if c.RoutingAPI.AuthDisabled {
		logger.Debug("creating-noop-uaa-client")
		client := uaaclient.NewNoOpUaaClient()
		return client
	}
	logger.Debug("creating-uaa-client")

	if c.OAuth.Port == -1 {
		logger.Fatal("tls-not-enabled", errors.New("TcpRouter requires to communicate with UAA over TLS"), lager.Data{"token-endpoint": c.OAuth.TokenEndpoint, "port": c.OAuth.Port})
	}

	scheme := "https"
	tokenURL := fmt.Sprintf("%s://%s:%d", scheme, c.OAuth.TokenEndpoint, c.OAuth.Port)
	logger.Info(fmt.Sprintf("using-%s-scheme-for-uaa", scheme))

	cfg := &uaaconfig.Config{
		UaaEndpoint:           tokenURL,
		SkipVerification:      c.OAuth.SkipOAuthTLSVerification,
		ClientName:            c.OAuth.ClientName,
		ClientSecret:          c.OAuth.ClientSecret,
		MaxNumberOfRetries:    uint32(*tokenFetchMaxRetries),
		RetryInterval:         *tokenFetchRetryInterval,
		ExpirationBufferInSec: int64(*tokenFetchExpirationBufferTime),
	}

	logger.Info("fetching-token-from-uaa")

	uaaClient, err := uaaclient.NewClient(logger, cfg, klok)
	if err != nil {
		logger.Fatal("initialize-token-fetcher-error", err)
	}
	return uaaClient
}

func initializeDropsonde(logger lager.Logger) {
	dropsondeDestination := fmt.Sprintf("localhost:%d", *dropsondePort)
	err := dropsonde.Initialize(dropsondeDestination, dropsondeOrigin)
	if err != nil {
		logger.Error("failed-to-initialize-dropsonde", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"math"
	"net/http"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/Sirupsen/logrus"
	dto "github.com/prometheus/client_model/go"
	"github.com/prometheus/common/expfmt"
)

var (
	debug       = flag.Bool("d", false, "Enable debug mode")
	metricsHost = flag.String("h", "http://localhost:9090/metrics", "The full URL — like 'http://localhost:9090/metrics' to query for Prometheus metrics.")
	interval    = flag.String("i", "10s", "The interval at which to query. Value must be parseable by time.ParseDuration (https://golang.org/pkg/time/#ParseDuration).")
	statsHost   = flag.String("s", "127.0.0.1:8126", "The host and port — like '127.0.0.1:8126' — to send our metrics to.")
)

func main() {
	flag.Parse()

	c, _ := statsd.New(*statsHost)

	if *debug {
		logrus.SetLevel(logrus.DebugLevel)
	}
	logrus.Debug("HELLO")

	i, err := time.ParseDuration(*interval)
	if err != nil {
		logrus.WithError(err).Fatalf("Failed to parse interval '%s'", *interval)
	}

	ticker := time.NewTicker(i)
	for _ = range ticker.C {
		collect(c)
	}
}

func collect(c *statsd.Client) {
	logrus.WithFields(logrus.Fields{
		"stats_host":   *statsHost,
		"metrics_host": *metricsHost,
	}).Debug("Beginning collection")

	resp, _ := http.Get(*metricsHost)
	d := expfmt.NewDecoder(resp.Body, expfmt.FmtText)
	var mf dto.MetricFamily
	for {
		err := d.Decode(&mf)
		if err == io.EOF {
			// We've hit the end, break out!
			break
		} else if err != nil {
			logrus.WithError(err).Warn("Failed to decode a metric")
			break
		}

		switch mf.GetType() {
		case dto.MetricType_COUNTER:
			for _, counter := range mf.GetMetric() {
				var tags []string
				labels := counter.GetLabel()
				for _, pair := range labels {
					tags = append(tags, fmt.Sprintf("%s:%s", pair.GetName(), pair.GetValue()))
				}
				c.Count(mf.GetName(), int64(counter.GetCounter().GetValue()), tags, 1.0)
			}
		case dto.MetricType_GAUGE:
			for _, gauge := range mf.GetMetric() {
				var tags []string
				labels := gauge.GetLabel()
				for _, pair := range labels {
					tags = append(tags, fmt.Sprintf("%s:%s", pair.GetName(), pair.GetValue()))
				}
				c.Gauge(mf.GetName(), float64(gauge.GetGauge().GetValue()), tags, 1.0)
			}
		case dto.MetricType_HISTOGRAM, dto.MetricType_SUMMARY:
			for _, histo := range mf.GetMetric() {
				var tags []string
				labels := histo.GetLabel()
				for _, pair := range labels {
					tags = append(tags, fmt.Sprintf("%s:%s", pair.GetName(), pair.GetValue()))
				}
				hname := mf.GetName()
				summ := histo.GetSummary()
				c.Gauge(fmt.Sprintf("%s.sum", hname), summ.GetSampleSum(), tags, 1.0)
				c.Gauge(fmt.Sprintf("%s.count", hname), float64(summ.GetSampleCount()), tags, 1.0)
				for _, quantile := range summ.GetQuantile() {
					v := quantile.GetValue()
					if !math.IsNaN(v) {
						c.Gauge(fmt.Sprintf("%s.%dpercentile", hname, int(quantile.GetQuantile()*100)), v, tags, 1.0)
					}
				}
			}
		}
	}
}
<commit_msg>Metrics!<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"math"
	"net/http"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/Sirupsen/logrus"
	dto "github.com/prometheus/client_model/go"
	"github.com/prometheus/common/expfmt"
)

var (
	debug       = flag.Bool("d", false, "Enable debug mode")
	metricsHost = flag.String("h", "http://localhost:9090/metrics", "The full URL — like 'http://localhost:9090/metrics' to query for Prometheus metrics.")
	interval    = flag.String("i", "10s", "The interval at which to query. Value must be parseable by time.ParseDuration (https://golang.org/pkg/time/#ParseDuration).")
	statsHost   = flag.String("s", "127.0.0.1:8126", "The host and port — like '127.0.0.1:8126' — to send our metrics to.")
)

func main() {
	flag.Parse()

	c, _ := statsd.New(*statsHost)

	if *debug {
		logrus.SetLevel(logrus.DebugLevel)
	}
	logrus.Debug("HELLO")

	i, err := time.ParseDuration(*interval)
	if err != nil {
		logrus.WithError(err).Fatalf("Failed to parse interval '%s'", *interval)
	}

	ticker := time.NewTicker(i)
	for _ = range ticker.C {
		collect(c)
	}
}

func collect(c *statsd.Client) {
	logrus.WithFields(logrus.Fields{
		"stats_host":   *statsHost,
		"metrics_host": *metricsHost,
	}).Debug("Beginning collection")

	resp, _ := http.Get(*metricsHost)
	d := expfmt.NewDecoder(resp.Body, expfmt.FmtText)
	var mf dto.MetricFamily
	for {
		err := d.Decode(&mf)
		if err == io.EOF {
			// We've hit the end, break out!
			break
		} else if err != nil {
			c.Count("veneur.prometheus.decode_errors_total", 1, nil, 1.0)
			logrus.WithError(err).Warn("Failed to decode a metric")
			break
		}

		metricCount := int64(0)
		switch mf.GetType() {
		case dto.MetricType_COUNTER:
			for _, counter := range mf.GetMetric() {
				var tags []string
				labels := counter.GetLabel()
				for _, pair := range labels {
					tags = append(tags, fmt.Sprintf("%s:%s", pair.GetName(), pair.GetValue()))
				}
				c.Count(mf.GetName(), int64(counter.GetCounter().GetValue()), tags, 1.0)
				metricCount++
			}
		case dto.MetricType_GAUGE:
			for _, gauge := range mf.GetMetric() {
				var tags []string
				labels := gauge.GetLabel()
				for _, pair := range labels {
					tags = append(tags, fmt.Sprintf("%s:%s", pair.GetName(), pair.GetValue()))
				}
				c.Gauge(mf.GetName(), float64(gauge.GetGauge().GetValue()), tags, 1.0)
				metricCount++
			}
		case dto.MetricType_HISTOGRAM, dto.MetricType_SUMMARY:
			for _, histo := range mf.GetMetric() {
				var tags []string
				labels := histo.GetLabel()
				for _, pair := range labels {
					tags = append(tags, fmt.Sprintf("%s:%s", pair.GetName(), pair.GetValue()))
				}
				hname := mf.GetName()
				summ := histo.GetSummary()
				c.Gauge(fmt.Sprintf("%s.sum", hname), summ.GetSampleSum(), tags, 1.0)
				c.Gauge(fmt.Sprintf("%s.count", hname), float64(summ.GetSampleCount()), tags, 1.0)
				for _, quantile := range summ.GetQuantile() {
					v := quantile.GetValue()
					if !math.IsNaN(v) {
						c.Gauge(fmt.Sprintf("%s.%dpercentile", hname, int(quantile.GetQuantile()*100)), v, tags, 1.0)
						metricCount++
					}
				}
			}
		default:
			c.Count("veneur.prometheus.unknown_metric_type_total", 1, nil, 1.0)
		}
		c.Count("veneur.prometheus.metrics_flushed_total", metricCount, nil, 1.0)
	}
}
<|endoftext|>


<commit_before>package email

import (
	"code.google.com/p/go-uuid/uuid"
	"github.com/nathan-osman/go-cannon/queue"
	"github.com/nathan-osman/go-cannon/util"

	"bytes"
	"fmt"
	"io"
	"mime/multipart"
	"net/textproto"
	"strings"
	"time"
)

// Abstract representation of an email.
type Email struct {
	From        string       `json:"from"`
	To          []string     `json:"to"`
	Cc          []string     `json:"cc"`
	Bcc         []string     `json:"bcc"`
	Subject     string       `json:"subject"`
	Text        string       `json:"text"`
	Html        string       `json:"html"`
	Attachments []Attachment `json:"attachments"`
}

// Create a multipart body with the specified text and HTML and write it to the
// specified writer. A temporary buffer is used to work around a cyclical
// dependency with respect to the writer, header, and part.
func writeMultipartBody(w *multipart.Writer, text, html string) error {
	var (
		buff      = &bytes.Buffer{}
		altWriter = multipart.NewWriter(buff)
		headers   = textproto.MIMEHeader{
			"Content-Type": []string{
				fmt.Sprintf("multipart/alternative; boundary=\"%s\"", altWriter.Boundary()),
			},
		}
		textPart = &Attachment{
			ContentType: "text/plain; charset=\"utf-8\"",
			Content:     text,
		}
		htmlPart = &Attachment{
			ContentType: "text/html; charset=\"utf-8\"",
			Content:     html,
		}
	)
	part, err := w.CreatePart(headers)
	if err != nil {
		return err
	}
	if err := textPart.Write(altWriter); err != nil {
		return err
	}
	if err := htmlPart.Write(altWriter); err != nil {
		return err
	}
	if err := altWriter.Close(); err != nil {
		return err
	}
	_, err = io.Copy(part, buff)
	return err
}

// Convert the email into an array of messages grouped by host suitable for
// delivery to a mail queue.
func (e *Email) Messages(directory string) ([]*queue.Message, error) {
	id := uuid.New()
	if b, w, err := queue.NewBody(directory, id); err == nil {
		var (
			m       = multipart.NewWriter(w)
			headers = EmailHeaders{
				"Message-Id":   fmt.Sprintf("<%s@go-cannon>", id),
				"From":         e.From,
				"To":           strings.Join(e.To, ", "),
				"Subject":      e.Subject,
				"Date":         time.Now().Format("Mon, 02 Jan 2006 15:04:05 -0700"),
				"MIME-Version": "1.0",
				"Content-Type": fmt.Sprintf("multipart/mixed; boundary=%s", m.Boundary()),
			}
			addresses = append(append(e.To, e.Cc...), e.Bcc...)
		)
		if len(e.Cc) > 0 {
			headers["Cc"] = strings.Join(e.Cc, ",")
		}
		if err := headers.Write(w); err != nil {
			return nil, err
		}
		if err := writeMultipartBody(m, e.Text, e.Html); err != nil {
			return nil, err
		}
		for _, a := range e.Attachments {
			if err := a.Write(m); err != nil {
				return nil, err
			}
		}
		if err := m.Close(); err != nil {
			return nil, err
		}
		if err := w.Close(); err != nil {
			return nil, err
		}
		if addrMap, err := util.GroupAddressesByHost(addresses); err == nil {
			messages := make([]*queue.Message, 0, 1)
			for h, to := range addrMap {
				if m, err := queue.NewMessage(directory, e.From, h, to, b); err == nil {
					messages = append(messages, m)
				} else {
					return nil, err
				}
			}
			return messages, nil
		} else {
			return nil, err
		}
	} else {
		return nil, err
	}
}
<commit_msg>Fixed parameter order.<commit_after>package email

import (
	"code.google.com/p/go-uuid/uuid"
	"github.com/nathan-osman/go-cannon/queue"
	"github.com/nathan-osman/go-cannon/util"

	"bytes"
	"fmt"
	"io"
	"mime/multipart"
	"net/textproto"
	"strings"
	"time"
)

// Abstract representation of an email.
type Email struct {
	From        string       `json:"from"`
	To          []string     `json:"to"`
	Cc          []string     `json:"cc"`
	Bcc         []string     `json:"bcc"`
	Subject     string       `json:"subject"`
	Text        string       `json:"text"`
	Html        string       `json:"html"`
	Attachments []Attachment `json:"attachments"`
}

// Create a multipart body with the specified text and HTML and write it to the
// specified writer. A temporary buffer is used to work around a cyclical
// dependency with respect to the writer, header, and part.
func writeMultipartBody(w *multipart.Writer, text, html string) error {
	var (
		buff      = &bytes.Buffer{}
		altWriter = multipart.NewWriter(buff)
		headers   = textproto.MIMEHeader{
			"Content-Type": []string{
				fmt.Sprintf("multipart/alternative; boundary=\"%s\"", altWriter.Boundary()),
			},
		}
		textPart = &Attachment{
			ContentType: "text/plain; charset=\"utf-8\"",
			Content:     text,
		}
		htmlPart = &Attachment{
			ContentType: "text/html; charset=\"utf-8\"",
			Content:     html,
		}
	)
	part, err := w.CreatePart(headers)
	if err != nil {
		return err
	}
	if err := textPart.Write(altWriter); err != nil {
		return err
	}
	if err := htmlPart.Write(altWriter); err != nil {
		return err
	}
	if err := altWriter.Close(); err != nil {
		return err
	}
	_, err = io.Copy(part, buff)
	return err
}

// Convert the email into an array of messages grouped by host suitable for
// delivery to a mail queue.
func (e *Email) Messages(directory string) ([]*queue.Message, error) {
	id := uuid.New()
	if b, w, err := queue.NewBody(directory, id); err == nil {
		var (
			m       = multipart.NewWriter(w)
			headers = EmailHeaders{
				"Message-Id":   fmt.Sprintf("<%s@go-cannon>", id),
				"From":         e.From,
				"To":           strings.Join(e.To, ", "),
				"Subject":      e.Subject,
				"Date":         time.Now().Format("Mon, 02 Jan 2006 15:04:05 -0700"),
				"MIME-Version": "1.0",
				"Content-Type": fmt.Sprintf("multipart/mixed; boundary=%s", m.Boundary()),
			}
			addresses = append(append(e.To, e.Cc...), e.Bcc...)
		)
		if len(e.Cc) > 0 {
			headers["Cc"] = strings.Join(e.Cc, ",")
		}
		if err := headers.Write(w); err != nil {
			return nil, err
		}
		if err := writeMultipartBody(m, e.Text, e.Html); err != nil {
			return nil, err
		}
		for _, a := range e.Attachments {
			if err := a.Write(m); err != nil {
				return nil, err
			}
		}
		if err := m.Close(); err != nil {
			return nil, err
		}
		if err := w.Close(); err != nil {
			return nil, err
		}
		if addrMap, err := util.GroupAddressesByHost(addresses); err == nil {
			messages := make([]*queue.Message, 0, 1)
			for h, to := range addrMap {
				if m, err := queue.NewMessage(directory, h, e.From, to, b); err == nil {
					messages = append(messages, m)
				} else {
					return nil, err
				}
			}
			return messages, nil
		} else {
			return nil, err
		}
	} else {
		return nil, err
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"strings"
	"sync"
	"time"

	"github.com/garyburd/redigo/redis"
)

// Facility represents channel, which users can listen
// and server can broadcast message to all subscribed users
type Facility struct {
	name    string               // name of facility
	channel MessageChan          // broadcast channel
	clients map[MessageChan]bool // client channels
	l       sync.Locker          // lock for clients
}

// NewFacility creates facility, starts redis and broadcast loops
// and returns initialized *Facility
func NewFacility(name string) *Facility {
	f := new(Facility)
	f.channel = make(MessageChan)
	f.clients = make(map[MessageChan]bool)
	f.l = new(sync.Mutex)
	f.name = name
	go f.loop()
	go f.redisLoop()
	return f
}

// broadcast loop
func (f *Facility) loop() {
	// for every message in channel
	for s := range f.channel {
		f.l.Lock()
		log.Println("facility: got message; broadcasting")
		// async broadcast to all clients of facility
		for client := range f.clients {
			client <- s
		}
		log.Println("facility: broadcast ended")
		f.l.Unlock()
	}
}

// listen to facility key in redis and broadcast all data
func (f *Facility) listenRedis() error {
	conn, err := redis.Dial(redisNetwork, redisAddr)
	if err != nil {
		return err
	}
	conn.Do("select", redisDatabase)
	psc := redis.PubSubConn{Conn: conn}
	k := strings.Join([]string{redisPrefix, facilityPrefix, f.name}, keySeparator)
	log.Println("redis: listening to", k)
	psc.Subscribe(k)
	for {
		switch v := psc.Receive().(type) {
		case redis.Message:
			log.Println("redis: got message; broadcasting")
			f.channel <- v.Data
		case error:
			return err
		}
	}
}

// redis reconnection loop
func (f *Facility) redisLoop() {
	for {
		if err := f.listenRedis(); err != nil {
			log.Println("Redis error:", err)
			time.Sleep(attemptWait)
		}
	}
}

// Subscribe creates new subscription channel and returns it
// adding it to facility clients
func (f *Facility) Subscribe() (m MessageChan) {
	m = make(MessageChan)
	f.l.Lock()
	f.clients[m] = true
	f.l.Unlock()
	return m
}

// Unsubscibe removes channel from facility clients and closes it
func (f *Facility) Unsubscibe(m MessageChan) {
	f.l.Lock()
	close(m)
	delete(f.clients, m)
	f.l.Unlock()
}
<commit_msg>optimized locks<commit_after>package main

import (
	"log"
	"strings"
	"sync"
	"time"

	"github.com/garyburd/redigo/redis"
)

// Facility represents channel, which users can listen
// and server can broadcast message to all subscribed users
type Facility struct {
	name    string               // name of facility
	channel MessageChan          // broadcast channel
	clients map[MessageChan]bool // client channels
	l       sync.Locker          // lock for clients
}

// NewFacility creates facility, starts redis and broadcast loops
// and returns initialized *Facility
func NewFacility(name string) *Facility {
	f := new(Facility)
	f.channel = make(MessageChan)
	f.clients = make(map[MessageChan]bool)
	f.l = new(sync.Mutex)
	f.name = name
	go f.loop()
	go f.redisLoop()
	return f
}

// broadcast loop
func (f *Facility) loop() {
	// for every message in channel
	for s := range f.channel {
		log.Println("facility: got message; broadcasting")
		// async broadcast to all clients of facility
		f.l.Lock()
		for client := range f.clients {
			client <- s
		}
		f.l.Unlock()
		log.Println("facility: broadcast ended")
	}
}

// listen to facility key in redis and broadcast all data
func (f *Facility) listenRedis() error {
	conn, err := redis.Dial(redisNetwork, redisAddr)
	if err != nil {
		return err
	}
	conn.Do("select", redisDatabase)
	psc := redis.PubSubConn{Conn: conn}
	k := strings.Join([]string{redisPrefix, facilityPrefix, f.name}, keySeparator)
	log.Println("redis: listening to", k)
	psc.Subscribe(k)
	for {
		switch v := psc.Receive().(type) {
		case redis.Message:
			log.Println("redis: got message; broadcasting")
			f.channel <- v.Data
		case error:
			return err
		}
	}
}

// redis reconnection loop
func (f *Facility) redisLoop() {
	for {
		if err := f.listenRedis(); err != nil {
			log.Println("Redis error:", err)
			time.Sleep(attemptWait)
		}
	}
}

// Subscribe creates new subscription channel and returns it
// adding it to facility clients
func (f *Facility) Subscribe() (m MessageChan) {
	m = make(MessageChan)
	f.l.Lock()
	f.clients[m] = true
	f.l.Unlock()
	return m
}

// Unsubscibe removes channel from facility clients and closes it
func (f *Facility) Unsubscibe(m MessageChan) {
	f.l.Lock()
	delete(f.clients, m)
	f.l.Unlock()
	close(m)
}
<|endoftext|>


<commit_before>// Package fastrand implements a cryptographically secure pseudorandom number
// generator. The generator is seeded using the system's default entropy
// source, and thereafter produces random values via repeated hashing. As a
// result, fastrand can generate randomness much faster than crypto/rand, and
// generation cannot fail.
package fastrand

import (
	"crypto/rand"
	"hash"
	"io"
	"math"
	"math/big"
	"sync"
	"unsafe"

	"github.com/minio/blake2b-simd"
)

// A randReader produces random values via repeated hashing. The entropy field
// is the concatenation of an initial seed and a 128-bit counter. Each time
// the entropy is hashed, the counter is incremented.
type randReader struct {
	entropy []byte
	h       hash.Hash
	buf     []byte
	mu      sync.Mutex
}

// Read fills b with random data. It always returns len(b), nil.
func (r *randReader) Read(b []byte) (int, error) {
	r.mu.Lock()
	n := 0
	for n < len(b) {
		// Increment counter.
		*(*uint64)(unsafe.Pointer(&r.entropy[0]))++
		if *(*uint64)(unsafe.Pointer(&r.entropy[0])) == 0 {
			*(*uint64)(unsafe.Pointer(&r.entropy[8]))++
		}
		// Hash the counter + initial seed.
		r.h.Reset()
		r.h.Write(r.entropy)
		r.buf = r.h.Sum(r.buf[:0])

		// Fill out 'b'.
		n += copy(b[n:], r.buf[:])
	}
	r.mu.Unlock()
	return n, nil
}

// Reader is a global, shared instance of a cryptographically strong pseudo-
// random generator. It uses blake2b as its hashing function. Reader is safe
// for concurrent use by multiple goroutines.
var Reader = func() *randReader {
	r := &randReader{h: blake2b.New256()}
	// Use 64 bytes in case the first 32 aren't completely random.
	_, err := io.CopyN(r.h, rand.Reader, 64)
	if err != nil {
		panic("crypto: no entropy available")
	}
	r.entropy = make([]byte, 16+32) // blake2b produces [32]byte hashes
	r.h.Sum(r.entropy[16:])
	return r
}()

// Read is a helper function that calls Reader.Read on b. It always fills b
// completely.
func Read(b []byte) { Reader.Read(b) }

// Bytes is a helper function that returns n bytes of random data.
func Bytes(n int) []byte {
	b := make([]byte, n)
	Read(b)
	return b
}

// Intn returns a uniform random value in [0,n). It panics if n <= 0.
func Intn(n int) int {
	if n <= 0 {
		panic("fastrand: argument to Intn is <= 0")
	}
	// To eliminate modulo bias, keep selecting at random until we fall within
	// a range that is evenly divisible by n.
	// NOTE: since n is at most math.MaxUint64/2, max is minimized when:
	//    n = math.MaxUint64/4 + 1 -> max = math.MaxUint64 - math.MaxUint64/4
	// This gives an expected 1.333 tries before choosing a value < max.
	max := math.MaxUint64 - math.MaxUint64%uint64(n)
	b := Bytes(8)
	r := *(*uint64)(unsafe.Pointer(&b[0]))
	for r >= max {
		Read(b)
		r = *(*uint64)(unsafe.Pointer(&b[0]))
	}
	return int(r % uint64(n))
}

// BigIntn returns a uniform random value in [0,n). It panics if n <= 0.
func BigIntn(n *big.Int) *big.Int {
	i, _ := rand.Int(Reader, n)
	return i
}

// Perm returns a random permutation of the integers [0,n).
func Perm(n int) []int {
	m := make([]int, n)
	for i := 1; i < n; i++ {
		j := Intn(i + 1)
		m[i] = m[j]
		m[j] = i
	}
	return m
}
<commit_msg>implement entropy pool<commit_after>// Package fastrand implements a cryptographically secure pseudorandom number
// generator. The generator is seeded using the system's default entropy
// source, and thereafter produces random values via repeated hashing. As a
// result, fastrand can generate randomness much faster than crypto/rand, and
// generation cannot fail.
package fastrand

import (
	"crypto/rand"
	"io"
	"math"
	"math/big"
	"runtime"
	"unsafe"

	"github.com/minio/blake2b-simd"
)

// A randReader produces random values via repeated hashing. The entropy field
// is the concatenation of an initial seed and a 128-bit counter. Each time
// the entropy is hashed, the counter is incremented.
type randReader struct {
	entropy chan []byte
}

// Read fills b with random data. It always returns len(b), nil.
func (r *randReader) Read(b []byte) (int, error) {
	n := 0
	for n < len(b) {
		n += copy(b[n:], <-r.entropy)
	}
	return n, nil
}

// fillEntropy continuously fills r.entropy with new entropy.
func (r *randReader) fillEntropy() {
	// Create a hasher and fill it with 64 bytes of entropy. Technically only 16
	// should be needed, but the underlying RNG may not be secure.
	h := blake2b.New256()
	_, err := io.CopyN(h, rand.Reader, 64)
	if err != nil {
		panic("fastrand: no entropy available")
	}
	seed := h.Sum(nil)

	for {
		for i := uint64(0); i < math.MaxUint64; i++ {
			// Update the seed.
			*(*uint64)(unsafe.Pointer(&seed[0])) = i

			// Hash the seed.
			h.Reset()
			h.Write(seed)

			// Send the entropy down the entropy channel.
			r.entropy <- h.Sum(nil)
		}

		// Re-seed the hasher. Use the entropy that existed previously,
		// protecting against a compromised RNG.
		h.Reset()
		h.Write(seed[:])
		io.CopyN(h, rand.Reader, 64)
		seed = h.Sum(seed[:0])
	}
}

// Reader is a global, shared instance of a cryptographically strong pseudo-
// random generator. It uses blake2b as its hashing function. Reader is safe
// for concurrent use by multiple goroutines.
var Reader io.Reader

func init() {
	r := &randReader{
		entropy: make(chan []byte, 1000),
	}
	for i := 0; i < runtime.NumCPU(); i++ {
		go r.fillEntropy()
	}
	Reader = r
}

// Read is a helper function that calls Reader.Read on b. It always fills b
// completely.
func Read(b []byte) { Reader.Read(b) }

// Bytes is a helper function that returns n bytes of random data.
func Bytes(n int) []byte {
	b := make([]byte, n)
	Read(b)
	return b
}

// Intn returns a uniform random value in [0,n). It panics if n <= 0.
func Intn(n int) int {
	if n <= 0 {
		panic("fastrand: argument to Intn is <= 0")
	}
	// To eliminate modulo bias, keep selecting at random until we fall within
	// a range that is evenly divisible by n.
	// NOTE: since n is at most math.MaxUint64/2, max is minimized when:
	//    n = math.MaxUint64/4 + 1 -> max = math.MaxUint64 - math.MaxUint64/4
	// This gives an expected 1.333 tries before choosing a value < max.
	max := math.MaxUint64 - math.MaxUint64%uint64(n)
	b := Bytes(8)
	r := *(*uint64)(unsafe.Pointer(&b[0]))
	for r >= max {
		Read(b)
		r = *(*uint64)(unsafe.Pointer(&b[0]))
	}
	return int(r % uint64(n))
}

// BigIntn returns a uniform random value in [0,n). It panics if n <= 0.
func BigIntn(n *big.Int) *big.Int {
	i, _ := rand.Int(Reader, n)
	return i
}

// Perm returns a random permutation of the integers [0,n).
func Perm(n int) []int {
	m := make([]int, n)
	for i := 1; i < n; i++ {
		j := Intn(i + 1)
		m[i] = m[j]
		m[j] = i
	}
	return m
}
<|endoftext|>


<commit_before>package spotify

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"time"

	"github.com/go-errors/errors"
	"github.com/golang/glog"
)

type PlaylistVisibility int

const (
	PlaylistPrivate PlaylistVisibility = iota
	PlaylistPublic
	PlaylistCollaborative

	spotifyAPIURL = "https://api.spotify.com"
)

type PrivateProfile struct {
	ID          string `json:"id"`
	DisplayName string `json:"display_name,omitempty"`
	Email       string `json:"email,omitempty"`
	URI         string `json:"uri,omitempty"`
}

type Artist struct {
	ID           string            `json:"id"`
	Name         string            `json:"name"`
	ExternalURLs map[string]string `json:"external_urls"`
}

type Album struct {
	ID           string            `json:"id"`
	Name         string            `json:"name"`
	ExternalURLs map[string]string `json:"external_urls"`
}

type Track struct {
	ID           string            `json:"id"`
	Name         string            `json:"name"`
	Arists       []*Artist         `json:"artists"`
	Album        *Album            `json:"album"`
	ExternalURLs map[string]string `json:"external_urls"`
	URI          string            `json:"uri"`
}

type PublicProfile struct {
	ID          string `json:"id"`
	DisplayName string `json:"display_name"`
}

type PlaylistTrack struct {
	Track   *Track         `json:"track"`
	AddedAt time.Time      `json:"added_at"`
	AddedBy *PublicProfile `json:"added_by"`
}

type listPlaylistTracks struct {
	PlaylistTracks []*PlaylistTrack `json:"items"`
	Next           string           `json:"next,omitempty"`
}

type Playlist struct {
	ID             string            `json:"id"`
	Name           string            `json:"name"`
	Owner          *PublicProfile    `json:"owner"`
	SnapshotID     string            `json:"snapshot_id"`
	Collaborative  bool              `json:"collaborative"`
	ExternalURLs   map[string]string `json:"external_urls"`
	PlaylistTracks []*PlaylistTrack
	RawTracks      listPlaylistTracks `json:"tracks"`
}

type listPlaylists struct {
	Playlists []*Playlist `json:"items"`
	Next      string      `json:"next,omitempty"`
}

type NotFoundError struct {
	url *url.URL
}

var _ error = &NotFoundError{}

func (n *NotFoundError) Error() string {
	return fmt.Sprintf("Resource not found: %v", n.url)
}

type SpotifyClient struct {
	accessToken string
}

func NewSpotifyClient(accessToken string) *SpotifyClient {
	return &SpotifyClient{accessToken: accessToken}
}

func (s *SpotifyClient) GetMyProfile() (*PrivateProfile, error) {
	profile := new(PrivateProfile)
	_, err := s.get("/v1/me", nil, false, &profile)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	return profile, nil
}

func (s *SpotifyClient) GetProfile(userID string) (*PublicProfile, error) {
	profile := new(PublicProfile)
	_, err := s.get(fmt.Sprintf("/v1/users/%s", userID), nil, false, &profile)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	return profile, nil
}

func (s *SpotifyClient) ListMyPlaylists() ([]*Playlist, error) {
	wrappedPlaylists := new(listPlaylists)
	_, err := s.get("/v1/me/playlists", map[string]string{"limit": "50"}, false, &wrappedPlaylists)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	// TODO: pagination!

	return wrappedPlaylists.Playlists, nil
}

func (s *SpotifyClient) GetPlaylist(userID, playlistID string) (*Playlist, error) {
	playlist := new(Playlist)
	resp, err := s.get(fmt.Sprintf("/v1/users/%s/playlists/%s", userID, playlistID), nil, true, &playlist)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	} else if resp.StatusCode == http.StatusNotFound {
		return nil, nil
	}

	if playlist != nil {
		// TODO: pagination
		playlist.PlaylistTracks = playlist.RawTracks.PlaylistTracks
	}

	return playlist, nil
}

func (s *SpotifyClient) CreatePlaylist(userID, name string, visibility PlaylistVisibility) (*Playlist, error) {
	req := map[string]interface{}{
		"name":          name,
		"public":        visibility == PlaylistPublic,
		"collaborative": visibility == PlaylistCollaborative,
	}

	playlist := new(Playlist)
	if _, err := s.post(fmt.Sprintf("/v1/users/%s/playlists", userID), req, playlist); err != nil {
		return nil, err
	}

	return playlist, nil
}

func (s *SpotifyClient) FollowPlaylist(ownerID, playlistID string, public bool) (*http.Response, error) {
	path := fmt.Sprintf("/v1/users/%s/playlists/%s/followers", ownerID, playlistID)
	queryParams := map[string]string{
		"public": fmt.Sprintf("%t", public),
	}
	resp, err := s.put(path, queryParams, nil)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	return resp, nil
}

func (s *SpotifyClient) newRequest(method, path string, queryParams map[string]string, reqBody interface{}) (*http.Request, error) {
	u, err := url.Parse(fmt.Sprintf("%s%s", spotifyAPIURL, path))
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	for k, v := range queryParams {
		u.Query().Set(k, v)
	}

	var body io.Reader
	if reqBody != nil {
		var buf bytes.Buffer
		if err := json.NewEncoder(&buf).Encode(reqBody); err != nil {
			return nil, errors.Wrap(err, 0)
		}
		body = &buf
	}

	req, err := http.NewRequest(method, u.String(), body)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", s.accessToken))

	return req, nil
}

func (s *SpotifyClient) get(path string, queryParams map[string]string, optional bool, data interface{}) (*http.Response, error) {
	req, err := s.newRequest(http.MethodGet, path, queryParams, nil)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	glog.Infof("Spotify GET: %v", req.URL.String())

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	if resp.StatusCode == http.StatusNotFound {
		if !optional {
			return nil, &NotFoundError{url: req.URL}
		}
	} else if resp.StatusCode == http.StatusOK {
		if err := json.NewDecoder(resp.Body).Decode(data); err != nil {
			return nil, errors.WrapPrefix(err, "Unable to decode body", 0)
		}
	} else {
		return nil, errors.Errorf("Unexpected status code %d", resp.StatusCode)
	}

	return resp, nil
}

func (s *SpotifyClient) post(path string, reqBody, respData interface{}) (*http.Response, error) {
	req, err := s.newRequest(http.MethodPost, path, nil, reqBody)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	glog.Infof("Spotify POST: %v `%v`", req.URL.String(), reqBody)

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
		return nil, errors.Errorf("Unexpected status code %d", resp.StatusCode)
	}

	defer resp.Body.Close()
	if err := json.NewDecoder(resp.Body).Decode(respData); err != nil {
		return nil, errors.WrapPrefix(err, "Unable to decode response", 0)
	}

	return resp, nil
}

func (s *SpotifyClient) put(path string, queryParams map[string]string, reqBody interface{}) (*http.Response, error) {
	req, err := s.newRequest(http.MethodPut, path, nil, reqBody)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	glog.Infof("Spotify PUT: %v `%v`", req.URL.String(), reqBody)

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
		return nil, errors.Errorf("Unexpected status code %d", resp.StatusCode)
	}

	return resp, nil
}
<commit_msg>Add pagination support<commit_after>package spotify

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"time"

	"github.com/go-errors/errors"
	"github.com/golang/glog"
)

type PlaylistVisibility int

const (
	PlaylistPrivate PlaylistVisibility = iota
	PlaylistPublic
	PlaylistCollaborative

	spotifyAPIURL = "https://api.spotify.com"
)

type PrivateProfile struct {
	ID          string `json:"id"`
	DisplayName string `json:"display_name,omitempty"`
	Email       string `json:"email,omitempty"`
	URI         string `json:"uri,omitempty"`
}

type Artist struct {
	ID           string            `json:"id"`
	Name         string            `json:"name"`
	ExternalURLs map[string]string `json:"external_urls"`
}

type Album struct {
	ID           string            `json:"id"`
	Name         string            `json:"name"`
	ExternalURLs map[string]string `json:"external_urls"`
}

type Track struct {
	ID           string            `json:"id"`
	Name         string            `json:"name"`
	Arists       []*Artist         `json:"artists"`
	Album        *Album            `json:"album"`
	ExternalURLs map[string]string `json:"external_urls"`
	URI          string            `json:"uri"`
}

type PublicProfile struct {
	ID          string `json:"id"`
	DisplayName string `json:"display_name"`
}

type PlaylistTrack struct {
	Track   *Track         `json:"track"`
	AddedAt time.Time      `json:"added_at"`
	AddedBy *PublicProfile `json:"added_by"`
}

type listPlaylistTracks struct {
	PlaylistTracks []*PlaylistTrack `json:"items"`
	Next           string           `json:"next,omitempty"`
}

type Playlist struct {
	ID             string            `json:"id"`
	Name           string            `json:"name"`
	Owner          *PublicProfile    `json:"owner"`
	SnapshotID     string            `json:"snapshot_id"`
	Collaborative  bool              `json:"collaborative"`
	ExternalURLs   map[string]string `json:"external_urls"`
	PlaylistTracks []*PlaylistTrack
	RawTracks      listPlaylistTracks `json:"tracks"`
}

type listPlaylists struct {
	Playlists []*Playlist `json:"items"`
	Next      string      `json:"next,omitempty"`
}

type NotFoundError struct {
	url *url.URL
}

var _ error = &NotFoundError{}

func (n *NotFoundError) Error() string {
	return fmt.Sprintf("Resource not found: %v", n.url)
}

type SpotifyClient struct {
	accessToken string
}

func NewSpotifyClient(accessToken string) *SpotifyClient {
	return &SpotifyClient{accessToken: accessToken}
}

func (s *SpotifyClient) GetMyProfile() (*PrivateProfile, error) {
	profile := new(PrivateProfile)
	_, err := s.get("/v1/me", nil, false, &profile)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	return profile, nil
}

func (s *SpotifyClient) GetProfile(userID string) (*PublicProfile, error) {
	profile := new(PublicProfile)
	_, err := s.get(fmt.Sprintf("/v1/users/%s", userID), nil, false, &profile)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	return profile, nil
}

func (s *SpotifyClient) ListMyPlaylists() ([]*Playlist, error) {
	wrappedPlaylists := new(listPlaylists)
	_, err := s.get("/v1/me/playlists", map[string]string{"limit": "50"}, false, &wrappedPlaylists)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	allPlaylists := wrappedPlaylists.Playlists
	for next := wrappedPlaylists.Next; len(next) > 0; {
		playlists := new(listPlaylists)
		if _, err := s.get(next, nil, false, playlists); err != nil {
			return nil, err
		}

		allPlaylists = append(allPlaylists, playlists.Playlists...)
		next = playlists.Next
	}

	return allPlaylists, nil
}

func (s *SpotifyClient) GetPlaylist(userID, playlistID string) (*Playlist, error) {
	playlist := new(Playlist)
	resp, err := s.get(fmt.Sprintf("/v1/users/%s/playlists/%s", userID, playlistID), nil, true, &playlist)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	} else if resp.StatusCode == http.StatusNotFound {
		return nil, nil
	}

	// Load all additional pages for paginated results.
	allTracks := playlist.RawTracks.PlaylistTracks
	for next := playlist.RawTracks.Next; len(next) > 0; {
		tracks := new(listPlaylistTracks)
		if _, err := s.get(next, nil, false, tracks); err != nil {
			return nil, err
		}

		allTracks = append(allTracks, tracks.PlaylistTracks...)
		next = tracks.Next
	}

	playlist.PlaylistTracks = allTracks

	return playlist, nil
}

func (s *SpotifyClient) CreatePlaylist(userID, name string, visibility PlaylistVisibility) (*Playlist, error) {
	req := map[string]interface{}{
		"name":          name,
		"public":        visibility == PlaylistPublic,
		"collaborative": visibility == PlaylistCollaborative,
	}

	playlist := new(Playlist)
	if _, err := s.post(fmt.Sprintf("/v1/users/%s/playlists", userID), req, playlist); err != nil {
		return nil, err
	}

	return playlist, nil
}

func (s *SpotifyClient) FollowPlaylist(ownerID, playlistID string, public bool) (*http.Response, error) {
	path := fmt.Sprintf("/v1/users/%s/playlists/%s/followers", ownerID, playlistID)
	queryParams := map[string]string{
		"public": fmt.Sprintf("%t", public),
	}
	resp, err := s.put(path, queryParams, nil)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	return resp, nil
}

// path may be either a relative ("/foo/bar") or absoluate ("http://example.com/foo/bar").
// If path is relative then it will be prefixed with spotifyAPIURL.
func (s *SpotifyClient) newRequest(method, path string, queryParams map[string]string, reqBody interface{}) (*http.Request, error) {
	u, err := url.Parse(path)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	} else if len(u.Scheme) == 0 {
		return s.newRequest(method, fmt.Sprintf("%s%s", spotifyAPIURL, path), queryParams, reqBody)
	}

	for k, v := range queryParams {
		u.Query().Set(k, v)
	}

	var body io.Reader
	if reqBody != nil {
		var buf bytes.Buffer
		if err := json.NewEncoder(&buf).Encode(reqBody); err != nil {
			return nil, errors.Wrap(err, 0)
		}
		body = &buf
	}

	req, err := http.NewRequest(method, u.String(), body)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", s.accessToken))

	return req, nil
}

func (s *SpotifyClient) get(path string, queryParams map[string]string, optional bool, data interface{}) (*http.Response, error) {
	req, err := s.newRequest(http.MethodGet, path, queryParams, nil)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	glog.Infof("Spotify GET: %v", req.URL.String())

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	if resp.StatusCode == http.StatusNotFound {
		if !optional {
			return nil, &NotFoundError{url: req.URL}
		}
	} else if resp.StatusCode == http.StatusOK {
		if err := json.NewDecoder(resp.Body).Decode(data); err != nil {
			return nil, errors.WrapPrefix(err, "Unable to decode body", 0)
		}
	} else {
		return nil, errors.Errorf("Unexpected status code %d", resp.StatusCode)
	}

	return resp, nil
}

func (s *SpotifyClient) post(path string, reqBody, respData interface{}) (*http.Response, error) {
	req, err := s.newRequest(http.MethodPost, path, nil, reqBody)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	glog.Infof("Spotify POST: %v `%v`", req.URL.String(), reqBody)

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
		return nil, errors.Errorf("Unexpected status code %d", resp.StatusCode)
	}

	defer resp.Body.Close()
	if err := json.NewDecoder(resp.Body).Decode(respData); err != nil {
		return nil, errors.WrapPrefix(err, "Unable to decode response", 0)
	}

	return resp, nil
}

func (s *SpotifyClient) put(path string, queryParams map[string]string, reqBody interface{}) (*http.Response, error) {
	req, err := s.newRequest(http.MethodPut, path, nil, reqBody)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	glog.Infof("Spotify PUT: %v `%v`", req.URL.String(), reqBody)

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
		return nil, errors.Errorf("Unexpected status code %d", resp.StatusCode)
	}

	return resp, nil
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"

	"github.com/couchbase/gomemcached"
	log "github.com/couchbaselabs/clog"
	"github.com/couchbaselabs/go-couchbase"
)

type DCPFeed struct {
	name       string
	url        string
	poolName   string
	bucketName string
	bucketUUID string
	pf         StreamPartitionFunc
	streams    map[string]Stream
	closeCh    chan bool
	doneCh     chan bool
	doneErr    error
	doneMsg    string
}

func NewDCPFeed(name, url, poolName, bucketName, bucketUUID string,
	pf StreamPartitionFunc, streams map[string]Stream) (*DCPFeed, error) {
	return &DCPFeed{
		name:       name,
		url:        url,
		poolName:   poolName,
		bucketName: bucketName,
		bucketUUID: bucketUUID,
		pf:         pf,
		streams:    streams,
		closeCh:    make(chan bool),
		doneCh:     make(chan bool),
		doneErr:    nil,
		doneMsg:    "",
	}, nil
}

func (t *DCPFeed) Name() string {
	return t.name
}

func (t *DCPFeed) Start() error {
	log.Printf("DCPFeed.Start, name: %s", t.Name())

	go ExponentialBackoffLoop(t.Name(),
		func() int {
			progress, err := t.feed()
			if err != nil {
				log.Printf("DCPFeed name: %s, progress: %d, err: %v",
					t.Name(), progress, err)
			}
			return progress
		},
		FEED_SLEEP_INIT_MS,  // Milliseconds.
		FEED_BACKOFF_FACTOR, // Backoff.
		FEED_SLEEP_MAX_MS)

	return nil
}

func (t *DCPFeed) feed() (int, error) {
	select {
	case <-t.closeCh:
		t.doneErr = nil
		t.doneMsg = "closeCh closed"
		close(t.doneCh)
		return -1, nil
	default:
	}

	bucket, err := couchbase.GetBucket(t.url, t.poolName, t.bucketName)
	if err != nil {
		return 0, err
	}
	defer bucket.Close()

	if t.bucketUUID != "" && t.bucketUUID != bucket.UUID {
		bucket.Close()
		return -1, fmt.Errorf("mismatched bucket uuid,"+
			"bucketName: %s, bucketUUID: %s, bucket.UUID: %s",
			t.bucketName, t.bucketUUID, bucket.UUID)
	}

	// TODO: See if UprFeed name is important.
	feed, err := bucket.StartUprFeed("index" /*name*/, 0)
	if err != nil {
		return 0, err
	}
	defer feed.Close()

	err = feed.UprRequestStream(
		uint16(0),          /*vbno - TODO: use the vbno's*/
		uint16(0),          /*opaque*/
		0,                  /*flag*/
		0,                  /*vbuuid*/
		0,                  /*seqStart - TODO: use the seqno's*/
		0xFFFFFFFFFFFFFFFF, /*seqEnd*/
		0,                  /*snaps*/
		0)
	if err != nil {
		return 0, err
	}

loop:
	for {
		select {
		case <-t.closeCh:
			t.doneErr = nil
			t.doneMsg = "closeCh closed"
			close(t.doneCh)
			return -1, nil

		case uprEvent, alive := <-feed.C:
			if !alive {
				break loop
			}

			partition := fmt.Sprintf("%d", uprEvent.VBucket)
			stream, err := t.pf(uprEvent.Key, partition, t.streams)
			if err != nil {
				return 1, fmt.Errorf("error: DCPFeed:"+
					" partition func error from url: %s,"+
					" poolName: %s, bucketName: %s, uprEvent: %#v, streams: %#v, err: %v",
					t.url, t.poolName, t.bucketName, uprEvent, t.streams, err)
			}

			if uprEvent.Opcode == gomemcached.UPR_MUTATION {
				stream <- &StreamRequest{
					Op:  STREAM_OP_UPDATE,
					Key: uprEvent.Key,
					Val: uprEvent.Value,
				}
			} else if uprEvent.Opcode == gomemcached.UPR_DELETION {
				stream <- &StreamRequest{
					Op:  STREAM_OP_DELETE,
					Key: uprEvent.Key,
				}
			}
		}
	}

	return 1, nil
}

func (t *DCPFeed) Close() error {
	select {
	case <-t.doneCh:
		return t.doneErr
	default:
	}

	close(t.closeCh)
	<-t.doneCh
	return t.doneErr
}

func (t *DCPFeed) Streams() map[string]Stream {
	return t.streams
}
<commit_msg>placeholder integration of cbdatasource<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"
	"sync"

	"github.com/couchbase/gomemcached"
	log "github.com/couchbaselabs/clog"

	"github.com/steveyen/cbdatasource"
)

type DCPMutation struct {
	delete    bool
	vbucketId uint16
	key       []byte
	seq       uint64
}

// Implements both Feed and cbdatasource.Receiver interfaces.
type DCPFeed struct {
	name       string
	url        string
	poolName   string
	bucketName string
	bucketUUID string
	pf         StreamPartitionFunc
	streams    map[string]Stream
	bds        cbdatasource.BucketDataSource

	m    sync.Mutex
	errs []error
	muts []*DCPMutation
	meta map[uint16][]byte

	numSnapshotStarts int
	numSetMetaDatas   int
	numGetMetaDatas   int
	numRollbacks      int
}

func NewDCPFeed(name, url, poolName, bucketName, bucketUUID string,
	pf StreamPartitionFunc, streams map[string]Stream) (*DCPFeed, error) {
	vbucketIds, err := ParsePartitionsToVBucketIds(streams)
	if err != nil {
		return nil, err
	}
	if len(vbucketIds) <= 0 {
		vbucketIds = nil
	}

	var authFunc cbdatasource.AuthFunc
	var options *cbdatasource.BucketDataSourceOptions

	feed := &DCPFeed{
		name:       name,
		url:        url,
		poolName:   poolName,
		bucketName: bucketName,
		bucketUUID: bucketUUID,
		pf:         pf,
		streams:    streams,
	}

	feed.bds, err = cbdatasource.NewBucketDataSource([]string{url},
		poolName, bucketName, bucketUUID,
		vbucketIds, authFunc, feed, options)
	if err != nil {
		return nil, err
	}

	return feed, nil
}

func (t *DCPFeed) Name() string {
	return t.name
}

func (t *DCPFeed) Start() error {
	log.Printf("DCPFeed.Start, name: %s", t.Name())
	return t.bds.Start()
}

func (t *DCPFeed) Close() error {
	log.Printf("DCPFeed.Close, name: %s", t.Name())
	return t.bds.Close()
}

func (t *DCPFeed) Streams() map[string]Stream {
	return t.streams
}

// --------------------------------------------------------

func (r *DCPFeed) OnError(err error) {
	r.m.Lock()
	defer r.m.Unlock()

	// fmt.Printf("  DCPFeed.name: %s: %v\n", r.name, err)
	r.errs = append(r.errs, err)
}

func (r *DCPFeed) DataUpdate(vbucketId uint16, key []byte, seq uint64,
	req *gomemcached.MCRequest) error {
	r.m.Lock()
	defer r.m.Unlock()

	r.muts = append(r.muts, &DCPMutation{
		delete:    false,
		vbucketId: vbucketId,
		key:       key,
		seq:       seq,
	})
	return nil
}

func (r *DCPFeed) DataDelete(vbucketId uint16, key []byte, seq uint64,
	req *gomemcached.MCRequest) error {
	r.m.Lock()
	defer r.m.Unlock()

	r.muts = append(r.muts, &DCPMutation{
		delete:    true,
		vbucketId: vbucketId,
		key:       key,
		seq:       seq,
	})
	return nil
}

func (r *DCPFeed) SnapshotStart(vbucketId uint16,
	snapStart, snapEnd uint64, snapType uint32) error {
	r.numSnapshotStarts += 1
	return nil
}

func (r *DCPFeed) SetMetaData(vbucketId uint16, value []byte) error {
	r.m.Lock()
	defer r.m.Unlock()

	r.numSetMetaDatas += 1
	if r.meta == nil {
		r.meta = make(map[uint16][]byte)
	}
	r.meta[vbucketId] = value
	return nil
}

func (r *DCPFeed) GetMetaData(vbucketId uint16) (value []byte, lastSeq uint64, err error) {
	r.m.Lock()
	defer r.m.Unlock()

	r.numGetMetaDatas += 1
	rv := []byte(nil)
	if r.meta != nil {
		rv = r.meta[vbucketId]
	}
	for i := len(r.muts) - 1; i >= 0; i = i - 1 {
		if r.muts[i].vbucketId == vbucketId {
			return rv, r.muts[i].seq, nil
		}
	}
	return rv, 0, nil
}

func (r *DCPFeed) Rollback(vbucketId uint16, rollbackSeq uint64) error {
	r.numRollbacks += 1
	return fmt.Errorf("bad-rollback")
}
<|endoftext|>


<commit_before>// Package lint provides abstractions on top of go/analysis.
package lint

import (
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/token"
	"strconv"
	"strings"

	"golang.org/x/tools/go/analysis"
)

type Analyzer struct {
	// The analyzer's documentation. Unlike go/analysis.Analyzer.Doc,
	// this field is structured, providing access to severity, options
	// etc.
	Doc      *Documentation
	Analyzer *analysis.Analyzer
}

func (a *Analyzer) initialize() {
	a.Analyzer.Doc = fmt.Sprintf("%s\nOnline documentation\n    https://staticcheck.io/docs/checks#%s", a.Doc.String(), a.Analyzer.Name)
	if a.Analyzer.Flags.Usage == nil {
		fs := flag.NewFlagSet("", flag.PanicOnError)
		fs.Var(newVersionFlag(), "go", "Target Go version")
		a.Analyzer.Flags = *fs
	}
}

func InitializeAnalyzers(docs map[string]*Documentation, analyzers map[string]*analysis.Analyzer) map[string]*Analyzer {
	out := make(map[string]*Analyzer, len(analyzers))
	for k, v := range analyzers {
		v.Name = k
		a := &Analyzer{
			Doc:      docs[k],
			Analyzer: v,
		}
		a.initialize()
		out[k] = a
	}
	return out
}

type Severity int

const (
	SeverityNone Severity = iota
	SeverityError
	SeverityDeprecated
	SeverityWarning
	SeverityInfo
	SeverityHint
)

type Documentation struct {
	Title      string
	Text       string
	Since      string
	NonDefault bool
	Options    []string
	Severity   Severity
}

func Markdownify(m map[string]*Documentation) map[string]*Documentation {
	for _, v := range m {
		v.Title = toMarkdown(v.Title)
		v.Text = toMarkdown(v.Text)
	}
	return m
}

func toMarkdown(s string) string {
	return strings.ReplaceAll(s, `\'`, "`")
}

func (doc *Documentation) String() string {
	b := &strings.Builder{}
	fmt.Fprintf(b, "%s\n\n", doc.Title)
	if doc.Text != "" {
		fmt.Fprintf(b, "%s\n\n", doc.Text)
	}
	fmt.Fprint(b, "Available since\n    ")
	if doc.Since == "" {
		fmt.Fprint(b, "unreleased")
	} else {
		fmt.Fprintf(b, "%s", doc.Since)
	}
	if doc.NonDefault {
		fmt.Fprint(b, ", non-default")
	}
	fmt.Fprint(b, "\n")
	if len(doc.Options) > 0 {
		fmt.Fprintf(b, "\nOptions\n")
		for _, opt := range doc.Options {
			fmt.Fprintf(b, "    %s", opt)
		}
		fmt.Fprint(b, "\n")
	}
	return b.String()
}

func newVersionFlag() flag.Getter {
	tags := build.Default.ReleaseTags
	v := tags[len(tags)-1][2:]
	version := new(VersionFlag)
	if err := version.Set(v); err != nil {
		panic(fmt.Sprintf("internal error: %s", err))
	}
	return version
}

type VersionFlag int

func (v *VersionFlag) String() string {
	return fmt.Sprintf("1.%d", *v)
}

func (v *VersionFlag) Set(s string) error {
	if len(s) < 3 {
		return fmt.Errorf("invalid Go version: %q", s)
	}
	if s[0] != '1' {
		return fmt.Errorf("invalid Go version: %q", s)
	}
	if s[1] != '.' {
		return fmt.Errorf("invalid Go version: %q", s)
	}
	i, err := strconv.Atoi(s[2:])
	*v = VersionFlag(i)
	return err
}

func (v *VersionFlag) Get() interface{} {
	return int(*v)
}

// ExhaustiveTypeSwitch panics when called. It can be used to ensure
// that type switches are exhaustive.
func ExhaustiveTypeSwitch(v interface{}) {
	panic(fmt.Sprintf("internal error: unhandled case %T", v))
}

// A directive is a comment of the form '//lint:<command>
// [arguments...]'. It represents instructions to the static analysis
// tool.
type Directive struct {
	Command   string
	Arguments []string
	Directive *ast.Comment
	Node      ast.Node
}

func parseDirective(s string) (cmd string, args []string) {
	if !strings.HasPrefix(s, "//lint:") {
		return "", nil
	}
	s = strings.TrimPrefix(s, "//lint:")
	fields := strings.Split(s, " ")
	return fields[0], fields[1:]
}

func ParseDirectives(files []*ast.File, fset *token.FileSet) []Directive {
	var dirs []Directive
	for _, f := range files {
		// OPT(dh): in our old code, we skip all the commentmap work if we
		// couldn't find any directives, benchmark if that's actually
		// worth doing
		cm := ast.NewCommentMap(fset, f, f.Comments)
		for node, cgs := range cm {
			for _, cg := range cgs {
				for _, c := range cg.List {
					if !strings.HasPrefix(c.Text, "//lint:") {
						continue
					}
					cmd, args := parseDirective(c.Text)
					d := Directive{
						Command:   cmd,
						Arguments: args,
						Directive: c,
						Node:      node,
					}
					dirs = append(dirs, d)
				}
			}
		}
	}
	return dirs
}
<commit_msg>analysis/lint: don't crash on missing documentation<commit_after>// Package lint provides abstractions on top of go/analysis.
package lint

import (
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/token"
	"strconv"
	"strings"

	"golang.org/x/tools/go/analysis"
)

type Analyzer struct {
	// The analyzer's documentation. Unlike go/analysis.Analyzer.Doc,
	// this field is structured, providing access to severity, options
	// etc.
	Doc      *Documentation
	Analyzer *analysis.Analyzer
}

func (a *Analyzer) initialize() {
	a.Analyzer.Doc = fmt.Sprintf("%s\nOnline documentation\n    https://staticcheck.io/docs/checks#%s", a.Doc.String(), a.Analyzer.Name)
	if a.Analyzer.Flags.Usage == nil {
		fs := flag.NewFlagSet("", flag.PanicOnError)
		fs.Var(newVersionFlag(), "go", "Target Go version")
		a.Analyzer.Flags = *fs
	}
}

func InitializeAnalyzers(docs map[string]*Documentation, analyzers map[string]*analysis.Analyzer) map[string]*Analyzer {
	out := make(map[string]*Analyzer, len(analyzers))
	for k, v := range analyzers {
		v.Name = k
		a := &Analyzer{
			Doc:      docs[k],
			Analyzer: v,
		}
		a.initialize()
		out[k] = a
	}
	return out
}

type Severity int

const (
	SeverityNone Severity = iota
	SeverityError
	SeverityDeprecated
	SeverityWarning
	SeverityInfo
	SeverityHint
)

type Documentation struct {
	Title      string
	Text       string
	Since      string
	NonDefault bool
	Options    []string
	Severity   Severity
}

func Markdownify(m map[string]*Documentation) map[string]*Documentation {
	for _, v := range m {
		v.Title = toMarkdown(v.Title)
		v.Text = toMarkdown(v.Text)
	}
	return m
}

func toMarkdown(s string) string {
	return strings.ReplaceAll(s, `\'`, "`")
}

func (doc *Documentation) String() string {
	if doc == nil {
		return "Error: No documentation."
	}

	b := &strings.Builder{}
	fmt.Fprintf(b, "%s\n\n", doc.Title)
	if doc.Text != "" {
		fmt.Fprintf(b, "%s\n\n", doc.Text)
	}
	fmt.Fprint(b, "Available since\n    ")
	if doc.Since == "" {
		fmt.Fprint(b, "unreleased")
	} else {
		fmt.Fprintf(b, "%s", doc.Since)
	}
	if doc.NonDefault {
		fmt.Fprint(b, ", non-default")
	}
	fmt.Fprint(b, "\n")
	if len(doc.Options) > 0 {
		fmt.Fprintf(b, "\nOptions\n")
		for _, opt := range doc.Options {
			fmt.Fprintf(b, "    %s", opt)
		}
		fmt.Fprint(b, "\n")
	}
	return b.String()
}

func newVersionFlag() flag.Getter {
	tags := build.Default.ReleaseTags
	v := tags[len(tags)-1][2:]
	version := new(VersionFlag)
	if err := version.Set(v); err != nil {
		panic(fmt.Sprintf("internal error: %s", err))
	}
	return version
}

type VersionFlag int

func (v *VersionFlag) String() string {
	return fmt.Sprintf("1.%d", *v)
}

func (v *VersionFlag) Set(s string) error {
	if len(s) < 3 {
		return fmt.Errorf("invalid Go version: %q", s)
	}
	if s[0] != '1' {
		return fmt.Errorf("invalid Go version: %q", s)
	}
	if s[1] != '.' {
		return fmt.Errorf("invalid Go version: %q", s)
	}
	i, err := strconv.Atoi(s[2:])
	*v = VersionFlag(i)
	return err
}

func (v *VersionFlag) Get() interface{} {
	return int(*v)
}

// ExhaustiveTypeSwitch panics when called. It can be used to ensure
// that type switches are exhaustive.
func ExhaustiveTypeSwitch(v interface{}) {
	panic(fmt.Sprintf("internal error: unhandled case %T", v))
}

// A directive is a comment of the form '//lint:<command>
// [arguments...]'. It represents instructions to the static analysis
// tool.
type Directive struct {
	Command   string
	Arguments []string
	Directive *ast.Comment
	Node      ast.Node
}

func parseDirective(s string) (cmd string, args []string) {
	if !strings.HasPrefix(s, "//lint:") {
		return "", nil
	}
	s = strings.TrimPrefix(s, "//lint:")
	fields := strings.Split(s, " ")
	return fields[0], fields[1:]
}

func ParseDirectives(files []*ast.File, fset *token.FileSet) []Directive {
	var dirs []Directive
	for _, f := range files {
		// OPT(dh): in our old code, we skip all the commentmap work if we
		// couldn't find any directives, benchmark if that's actually
		// worth doing
		cm := ast.NewCommentMap(fset, f, f.Comments)
		for node, cgs := range cm {
			for _, cg := range cgs {
				for _, c := range cg.List {
					if !strings.HasPrefix(c.Text, "//lint:") {
						continue
					}
					cmd, args := parseDirective(c.Text)
					d := Directive{
						Command:   cmd,
						Arguments: args,
						Directive: c,
						Node:      node,
					}
					dirs = append(dirs, d)
				}
			}
		}
	}
	return dirs
}
<|endoftext|>


<commit_before>package annotations

import (
	"encoding/json"
	"errors"
	"fmt"
	"time"

	"regexp"

	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

var uuidExtractRegex = regexp.MustCompile(".*/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$")

// Service interface. Compatible with the baserwftapp service EXCEPT for
// 1) the Write function, which has signature Write(thing interface{}) error...
// 2) the DecodeJson function, which has signature DecodeJSON(*json.Decoder) (thing interface{}, identity string, err error)
// The problem is that we have a list of things, and the uuid is for a related OTHER thing
// TODO - move to implement a shared defined Service interface?
type Service interface {
	Write(contentUUID string, thing interface{}) (err error)
	Read(contentUUID string) (thing interface{}, found bool, err error)
	Delete(contentUUID string) (found bool, err error)
	Check() (err error)
	DecodeJSON(*json.Decoder) (thing interface{}, err error)
	Count() (int, error)
	Initialise() error
}

//holds the Neo4j-specific information
type service struct {
	conn            neoutils.NeoConnection
	platformVersion string
}

const (
	v1PlatformVersion         = "v1"
	v2PlatformVersion         = "v2"
	brightcovePlatformVersion = "brightcove"
)

//NewCypherAnnotationsService instantiate driver
func NewCypherAnnotationsService(cypherRunner neoutils.NeoConnection, platformVersion string) service {
	if platformVersion == "" {
		log.Fatalf("PlatformVersion was not specified!")
	}
	return service{cypherRunner, platformVersion}
}

// DecodeJSON decodes to a list of annotations, for ease of use this is a struct itself
func (s service) DecodeJSON(dec *json.Decoder) (interface{}, error) {
	a := annotations{}
	err := dec.Decode(&a)
	return a, err
}

func (s service) Read(contentUUID string) (thing interface{}, found bool, err error) {
	results := []annotation{}

	//TODO shouldn't return Provenances if none of the scores, agentRole or atTime are set
	statementTemplate := `
					MATCH (c:Thing{uuid:{contentUUID}})-[rel{platformVersion:{platformVersion}}]->(cc:Thing)
					WITH c, cc, rel, {id:cc.uuid,prefLabel:cc.prefLabel,types:labels(cc),predicate:type(rel)} as thing,
					collect(
						{scores:[
							{scoringSystem:'%s', value:rel.relevanceScore},
							{scoringSystem:'%s', value:rel.confidenceScore}],
						agentRole:rel.annotatedBy,
						atTime:rel.annotatedDate}) as provenances
					RETURN thing, provenances ORDER BY thing.id
									`
	statement := fmt.Sprintf(statementTemplate, relevanceScoringSystem, confidenceScoringSystem)

	query := &neoism.CypherQuery{
		Statement:  statement,
		Parameters: neoism.Props{"contentUUID": contentUUID, "platformVersion": s.platformVersion},
		Result:     &results,
	}
	err = s.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up uuid %s with query %s from neoism: %+v", contentUUID, query.Statement, err)
		return annotations{}, false, fmt.Errorf("Error accessing Annotations datastore for uuid: %s", contentUUID)
	}
	log.Debugf("CypherResult Read Annotations for uuid: %s was: %+v", contentUUID, results)
	if (len(results)) == 0 {
		return annotations{}, false, nil
	}

	for idx := range results {
		mapToResponseFormat(&results[idx])
	}

	return annotations(results), true, nil
}

//Delete removes all the annotations for this content. Ignore the nodes on either end -
//may leave nodes that are only 'things' inserted by this writer: clean up
//as a result of this will need to happen externally if required
func (s service) Delete(contentUUID string) (bool, error) {

	var deleteStatement string

	switch {
	case s.platformVersion == v2PlatformVersion:
		deleteStatement = `MATCH (c:Thing{uuid: {contentUUID}})-[rel:MENTIONS{platformVersion:{platformVersion}}]->(cc:Thing) DELETE rel`
	case s.platformVersion == brightcovePlatformVersion:
		// TODO this clause should be refactored when all videos in Neo4j have brightcove only as lifecycle and no v1 reference
		deleteStatement = `	OPTIONAL MATCH (c:Thing{uuid: {contentUUID}})-[r]->(cc:Thing)
					WHERE r.lifecycle={lifecycle} OR r.lifecycle={v1Lifecycle}
					DELETE r`
	default:
		deleteStatement = `MATCH (c:Thing{uuid: {contentUUID}})-[rel{platformVersion:{platformVersion}}]->(cc:Thing) DELETE rel`
	}

	query := &neoism.CypherQuery{
		Statement: deleteStatement,
		Parameters: neoism.Props{"contentUUID": contentUUID, "platformVersion": s.platformVersion,
			"lifecycle": lifecycle(s.platformVersion), "v1Lifecycle": lifecycle(v1PlatformVersion)},
		IncludeStats: true,
	}

	err := s.conn.CypherBatch([]*neoism.CypherQuery{query})

	stats, err := query.Stats()
	if err != nil {
		return false, err
	}

	return stats.ContainsUpdates, err
}

//Write a set of annotations associated with a piece of content. Any annotations
//already there will be removed
func (s service) Write(contentUUID string, thing interface{}) (err error) {
	annotationsToWrite := thing.(annotations)

	if contentUUID == "" {
		return errors.New("Content uuid is required")
	}
	if err := validateAnnotations(&annotationsToWrite); err != nil {
		log.Warnf("Validation of supplied annotations failed")
		return err
	}

	if len(annotationsToWrite) == 0 {
		log.Warnf("No new annotations supplied for content uuid: %s", contentUUID)
	}

	queries := append([]*neoism.CypherQuery{}, dropAllAnnotationsQuery(contentUUID, s.platformVersion))

	var statements = []string{}
	for _, annotationToWrite := range annotationsToWrite {
		query, err := createAnnotationQuery(contentUUID, annotationToWrite, s.platformVersion)
		if err != nil {
			return err
		}
		statements = append(statements, query.Statement)
		queries = append(queries, query)
	}
	log.Infof("Updated Annotations for content uuid: %s", contentUUID)
	log.Debugf("For update, ran statements: %+v", statements)

	return s.conn.CypherBatch(queries)
}

// Check tests neo4j by running a simple cypher query
func (s service) Check() error {
	return neoutils.Check(s.conn)
}

func (s service) Count() (int, error) {
	results := []struct {
		Count int `json:"c"`
	}{}

	query := &neoism.CypherQuery{
		Statement: `MATCH ()-[r{platformVersion:{platformVersion}}]->()
                WHERE r.lifecycle = {lifecycle}
                OR r.lifecycle IS NULL
                RETURN count(r) as c`,
		Parameters: neoism.Props{"platformVersion": s.platformVersion, "lifecycle": lifecycle(s.platformVersion)},
		Result:     &results,
	}

	err := s.conn.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return 0, err
	}

	return results[0].Count, nil
}

func (s service) Initialise() error {
	return nil // No constraints need to be set up
}

func createAnnotationRelationship(relation string) (statement string) {
	stmt := `
                MERGE (content:Thing{uuid:{contentID}})
                MERGE (upp:Identifier:UPPIdentifier{value:{conceptID}})
                MERGE (upp)-[:IDENTIFIES]->(concept:Thing) ON CREATE SET concept.uuid = {conceptID}
                MERGE (content)-[pred:%s {platformVersion:{platformVersion}}]->(concept)
                SET pred={annProps}
          `
	statement = fmt.Sprintf(stmt, relation)
	return statement
}

func getRelationshipFromPredicate(predicate string) (relation string) {
	if predicate != "" {
		relation = relations[predicate]
	} else {
		relation = relations["mentions"]
	}
	return relation
}

func createAnnotationQuery(contentUUID string, ann annotation, platformVersion string) (*neoism.CypherQuery, error) {
	query := neoism.CypherQuery{}
	thingID, err := extractUUIDFromURI(ann.Thing.ID)
	if err != nil {
		return nil, err
	}

	//todo temporary change to deal with multiple provenances
	/*if len(ann.Provenances) > 1 {
		return nil, errors.New("Cannot insert a MENTIONS annotation with multiple provenances")
	}*/

	var prov provenance
	params := map[string]interface{}{}
	params["platformVersion"] = platformVersion
	params["lifecycle"] = lifecycle(platformVersion)

	if len(ann.Provenances) >= 1 {
		prov = ann.Provenances[0]
		annotatedBy, annotatedDateEpoch, relevanceScore, confidenceScore, supplied, err := extractDataFromProvenance(&prov)

		if err != nil {
			log.Infof("ERROR=%s", err)
			return nil, err
		}

		if supplied == true {
			if annotatedBy != "" {
				params["annotatedBy"] = annotatedBy
			}
			if prov.AtTime != "" {
				params["annotatedDateEpoch"] = annotatedDateEpoch
				params["annotatedDate"] = prov.AtTime
			}
			params["relevanceScore"] = relevanceScore
			params["confidenceScore"] = confidenceScore
		}
	}

	relation := getRelationshipFromPredicate(ann.Thing.Predicate)
	query.Statement = createAnnotationRelationship(relation)
	query.Parameters = map[string]interface{}{
		"contentID":       contentUUID,
		"conceptID":       thingID,
		"platformVersion": platformVersion,
		"annProps":        params,
	}
	return &query, nil
}

func extractDataFromProvenance(prov *provenance) (string, int64, float64, float64, bool, error) {
	if len(prov.Scores) == 0 {
		return "", -1, -1, -1, false, nil
	}
	var annotatedBy string
	var annotatedDateEpoch int64
	var confidenceScore, relevanceScore float64
	var err error
	if prov.AgentRole != "" {
		annotatedBy, err = extractUUIDFromURI(prov.AgentRole)
	}
	if prov.AtTime != "" {
		annotatedDateEpoch, err = convertAnnotatedDateToEpoch(prov.AtTime)
	}
	relevanceScore, confidenceScore, err = extractScores(prov.Scores)

	if err != nil {
		return "", -1, -1, -1, true, err
	}
	return annotatedBy, annotatedDateEpoch, relevanceScore, confidenceScore, true, nil
}

func extractUUIDFromURI(uri string) (string, error) {
	result := uuidExtractRegex.FindStringSubmatch(uri)
	if len(result) == 2 {
		return result[1], nil
	}
	return "", fmt.Errorf("Couldn't extract uuid from uri %s", uri)
}

func convertAnnotatedDateToEpoch(annotatedDateString string) (int64, error) {
	datetimeEpoch, err := time.Parse(time.RFC3339, annotatedDateString)

	if err != nil {
		return 0, err
	}

	return datetimeEpoch.Unix(), nil
}

func extractScores(scores []score) (float64, float64, error) {
	var relevanceScore, confidenceScore float64
	for _, score := range scores {
		scoringSystem := score.ScoringSystem
		value := score.Value
		switch scoringSystem {
		case relevanceScoringSystem:
			relevanceScore = value
		case confidenceScoringSystem:
			confidenceScore = value
		}
	}
	return relevanceScore, confidenceScore, nil
}

func dropAllAnnotationsQuery(contentUUID string, platformVersion string) *neoism.CypherQuery {

	var matchStmtTemplate string

	//TODO hard-coded verification:
	//WE STILL NEED THIS UNTIL EVERYTHNG HAS A LIFECYCLE PROPERTY!
	// -> necessary for brands - which got written by content-api with isClassifiedBy relationship, and should not be deleted by annotations-rw
	// -> so far brands are the only v2 concepts which have isClassifiedBy relationship; as soon as this changes: implementation needs to be updated
	switch {
	case platformVersion == v2PlatformVersion:
		matchStmtTemplate = `	OPTIONAL MATCH (:Thing{uuid:{contentID}})-[r:MENTIONS{platformVersion:{platformVersion}}]->(t:Thing)
                         		DELETE r`
	case platformVersion == brightcovePlatformVersion:
		// TODO this clause should be refactored when all videos in Neo4j have brightcove only as lifecycle and no v1 reference
		matchStmtTemplate = `	OPTIONAL MATCH (:Thing{uuid:{contentID}})-[r]->(t:Thing)
					WHERE r.lifecycle={lifecycle} OR r.lifecycle={v1Lifecycle}
					DELETE r`
	default:
		matchStmtTemplate = `	OPTIONAL MATCH (:Thing{uuid:{contentID}})-[r]->(t:Thing)
					WHERE r.platformVersion={platformVersion}
                         		DELETE r`
	}

	query := neoism.CypherQuery{}
	query.Statement = matchStmtTemplate
	query.Parameters = neoism.Props{"contentID": contentUUID, "platformVersion": platformVersion,
		"lifecycle": lifecycle(platformVersion), "v1Lifecycle": lifecycle(v1PlatformVersion)}
	return &query
}

func validateAnnotations(annotations *annotations) error {
	//TODO - for consistency, we should probably just not create the annotation?
	for _, annotation := range *annotations {
		if annotation.Thing.ID == "" {
			return ValidationError{fmt.Sprintf("Concept uuid missing for annotation %+v", annotation)}
		}
	}
	return nil
}

//ValidationError is thrown when the annotations are not valid because mandatory information is missing
type ValidationError struct {
	Msg string
}

func (v ValidationError) Error() string {
	return v.Msg
}

func mapToResponseFormat(ann *annotation) {
	ann.Thing.ID = mapper.IDURL(ann.Thing.ID)
	// We expect only ONE provenance - provenance value is considered valid even if the AgentRole is not specified. See: v1 - isClassifiedBy
	for idx := range ann.Provenances {
		if ann.Provenances[idx].AgentRole != "" {
			ann.Provenances[idx].AgentRole = mapper.IDURL(ann.Provenances[idx].AgentRole)
		}
	}
}

func lifecycle(platformVersion string) string {
	return "annotations-" + platformVersion
}
<commit_msg>Refactorig - use a single method for building delete annotation statement.<commit_after>package annotations

import (
	"encoding/json"
	"errors"
	"fmt"
	"time"

	"regexp"

	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

var uuidExtractRegex = regexp.MustCompile(".*/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$")

// Service interface. Compatible with the baserwftapp service EXCEPT for
// 1) the Write function, which has signature Write(thing interface{}) error...
// 2) the DecodeJson function, which has signature DecodeJSON(*json.Decoder) (thing interface{}, identity string, err error)
// The problem is that we have a list of things, and the uuid is for a related OTHER thing
// TODO - move to implement a shared defined Service interface?
type Service interface {
	Write(contentUUID string, thing interface{}) (err error)
	Read(contentUUID string) (thing interface{}, found bool, err error)
	Delete(contentUUID string) (found bool, err error)
	Check() (err error)
	DecodeJSON(*json.Decoder) (thing interface{}, err error)
	Count() (int, error)
	Initialise() error
}

//holds the Neo4j-specific information
type service struct {
	conn            neoutils.NeoConnection
	platformVersion string
}

const (
	v1PlatformVersion         = "v1"
	v2PlatformVersion         = "v2"
	brightcovePlatformVersion = "brightcove"
)

//NewCypherAnnotationsService instantiate driver
func NewCypherAnnotationsService(cypherRunner neoutils.NeoConnection, platformVersion string) service {
	if platformVersion == "" {
		log.Fatalf("PlatformVersion was not specified!")
	}
	return service{cypherRunner, platformVersion}
}

// DecodeJSON decodes to a list of annotations, for ease of use this is a struct itself
func (s service) DecodeJSON(dec *json.Decoder) (interface{}, error) {
	a := annotations{}
	err := dec.Decode(&a)
	return a, err
}

func (s service) Read(contentUUID string) (thing interface{}, found bool, err error) {
	results := []annotation{}

	//TODO shouldn't return Provenances if none of the scores, agentRole or atTime are set
	statementTemplate := `
					MATCH (c:Thing{uuid:{contentUUID}})-[rel{platformVersion:{platformVersion}}]->(cc:Thing)
					WITH c, cc, rel, {id:cc.uuid,prefLabel:cc.prefLabel,types:labels(cc),predicate:type(rel)} as thing,
					collect(
						{scores:[
							{scoringSystem:'%s', value:rel.relevanceScore},
							{scoringSystem:'%s', value:rel.confidenceScore}],
						agentRole:rel.annotatedBy,
						atTime:rel.annotatedDate}) as provenances
					RETURN thing, provenances ORDER BY thing.id
									`
	statement := fmt.Sprintf(statementTemplate, relevanceScoringSystem, confidenceScoringSystem)

	query := &neoism.CypherQuery{
		Statement:  statement,
		Parameters: neoism.Props{"contentUUID": contentUUID, "platformVersion": s.platformVersion},
		Result:     &results,
	}
	err = s.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up uuid %s with query %s from neoism: %+v", contentUUID, query.Statement, err)
		return annotations{}, false, fmt.Errorf("Error accessing Annotations datastore for uuid: %s", contentUUID)
	}
	log.Debugf("CypherResult Read Annotations for uuid: %s was: %+v", contentUUID, results)
	if (len(results)) == 0 {
		return annotations{}, false, nil
	}

	for idx := range results {
		mapToResponseFormat(&results[idx])
	}

	return annotations(results), true, nil
}

//Delete removes all the annotations for this content. Ignore the nodes on either end -
//may leave nodes that are only 'things' inserted by this writer: clean up
//as a result of this will need to happen externally if required
func (s service) Delete(contentUUID string) (bool, error) {

	query := buildDeleteQuery(contentUUID, s.platformVersion, true)

	err := s.conn.CypherBatch([]*neoism.CypherQuery{query})

	stats, err := query.Stats()
	if err != nil {
		return false, err
	}

	return stats.ContainsUpdates, err
}

//Write a set of annotations associated with a piece of content. Any annotations
//already there will be removed
func (s service) Write(contentUUID string, thing interface{}) (err error) {
	annotationsToWrite := thing.(annotations)

	if contentUUID == "" {
		return errors.New("Content uuid is required")
	}
	if err := validateAnnotations(&annotationsToWrite); err != nil {
		log.Warnf("Validation of supplied annotations failed")
		return err
	}

	if len(annotationsToWrite) == 0 {
		log.Warnf("No new annotations supplied for content uuid: %s", contentUUID)
	}

	queries := append([]*neoism.CypherQuery{}, buildDeleteQuery(contentUUID, s.platformVersion, false))

	var statements = []string{}
	for _, annotationToWrite := range annotationsToWrite {
		query, err := createAnnotationQuery(contentUUID, annotationToWrite, s.platformVersion)
		if err != nil {
			return err
		}
		statements = append(statements, query.Statement)
		queries = append(queries, query)
	}
	log.Infof("Updated Annotations for content uuid: %s", contentUUID)
	log.Debugf("For update, ran statements: %+v", statements)

	return s.conn.CypherBatch(queries)
}

// Check tests neo4j by running a simple cypher query
func (s service) Check() error {
	return neoutils.Check(s.conn)
}

func (s service) Count() (int, error) {
	results := []struct {
		Count int `json:"c"`
	}{}

	query := &neoism.CypherQuery{
		Statement: `MATCH ()-[r{platformVersion:{platformVersion}}]->()
                WHERE r.lifecycle = {lifecycle}
                OR r.lifecycle IS NULL
                RETURN count(r) as c`,
		Parameters: neoism.Props{"platformVersion": s.platformVersion, "lifecycle": lifecycle(s.platformVersion)},
		Result:     &results,
	}

	err := s.conn.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return 0, err
	}

	return results[0].Count, nil
}

func (s service) Initialise() error {
	return nil // No constraints need to be set up
}

func createAnnotationRelationship(relation string) (statement string) {
	stmt := `
                MERGE (content:Thing{uuid:{contentID}})
                MERGE (upp:Identifier:UPPIdentifier{value:{conceptID}})
                MERGE (upp)-[:IDENTIFIES]->(concept:Thing) ON CREATE SET concept.uuid = {conceptID}
                MERGE (content)-[pred:%s {platformVersion:{platformVersion}}]->(concept)
                SET pred={annProps}
          `
	statement = fmt.Sprintf(stmt, relation)
	return statement
}

func getRelationshipFromPredicate(predicate string) (relation string) {
	if predicate != "" {
		relation = relations[predicate]
	} else {
		relation = relations["mentions"]
	}
	return relation
}

func createAnnotationQuery(contentUUID string, ann annotation, platformVersion string) (*neoism.CypherQuery, error) {
	query := neoism.CypherQuery{}
	thingID, err := extractUUIDFromURI(ann.Thing.ID)
	if err != nil {
		return nil, err
	}

	//todo temporary change to deal with multiple provenances
	/*if len(ann.Provenances) > 1 {
		return nil, errors.New("Cannot insert a MENTIONS annotation with multiple provenances")
	}*/

	var prov provenance
	params := map[string]interface{}{}
	params["platformVersion"] = platformVersion
	params["lifecycle"] = lifecycle(platformVersion)

	if len(ann.Provenances) >= 1 {
		prov = ann.Provenances[0]
		annotatedBy, annotatedDateEpoch, relevanceScore, confidenceScore, supplied, err := extractDataFromProvenance(&prov)

		if err != nil {
			log.Infof("ERROR=%s", err)
			return nil, err
		}

		if supplied == true {
			if annotatedBy != "" {
				params["annotatedBy"] = annotatedBy
			}
			if prov.AtTime != "" {
				params["annotatedDateEpoch"] = annotatedDateEpoch
				params["annotatedDate"] = prov.AtTime
			}
			params["relevanceScore"] = relevanceScore
			params["confidenceScore"] = confidenceScore
		}
	}

	relation := getRelationshipFromPredicate(ann.Thing.Predicate)
	query.Statement = createAnnotationRelationship(relation)
	query.Parameters = map[string]interface{}{
		"contentID":       contentUUID,
		"conceptID":       thingID,
		"platformVersion": platformVersion,
		"annProps":        params,
	}
	return &query, nil
}

func extractDataFromProvenance(prov *provenance) (string, int64, float64, float64, bool, error) {
	if len(prov.Scores) == 0 {
		return "", -1, -1, -1, false, nil
	}
	var annotatedBy string
	var annotatedDateEpoch int64
	var confidenceScore, relevanceScore float64
	var err error
	if prov.AgentRole != "" {
		annotatedBy, err = extractUUIDFromURI(prov.AgentRole)
	}
	if prov.AtTime != "" {
		annotatedDateEpoch, err = convertAnnotatedDateToEpoch(prov.AtTime)
	}
	relevanceScore, confidenceScore, err = extractScores(prov.Scores)

	if err != nil {
		return "", -1, -1, -1, true, err
	}
	return annotatedBy, annotatedDateEpoch, relevanceScore, confidenceScore, true, nil
}

func extractUUIDFromURI(uri string) (string, error) {
	result := uuidExtractRegex.FindStringSubmatch(uri)
	if len(result) == 2 {
		return result[1], nil
	}
	return "", fmt.Errorf("Couldn't extract uuid from uri %s", uri)
}

func convertAnnotatedDateToEpoch(annotatedDateString string) (int64, error) {
	datetimeEpoch, err := time.Parse(time.RFC3339, annotatedDateString)

	if err != nil {
		return 0, err
	}

	return datetimeEpoch.Unix(), nil
}

func extractScores(scores []score) (float64, float64, error) {
	var relevanceScore, confidenceScore float64
	for _, score := range scores {
		scoringSystem := score.ScoringSystem
		value := score.Value
		switch scoringSystem {
		case relevanceScoringSystem:
			relevanceScore = value
		case confidenceScoringSystem:
			confidenceScore = value
		}
	}
	return relevanceScore, confidenceScore, nil
}

func buildDeleteQuery(contentUUID string, platformVersion string, includeStats bool) *neoism.CypherQuery {
	var statement string

	//TODO hard-coded verification:
	//WE STILL NEED THIS UNTIL EVERYTHNG HAS A LIFECYCLE PROPERTY!
	// -> necessary for brands - which got written by content-api with isClassifiedBy relationship, and should not be deleted by annotations-rw
	// -> so far brands are the only v2 concepts which have isClassifiedBy relationship; as soon as this changes: implementation needs to be updated
	switch {
	case platformVersion == v2PlatformVersion:
		statement = `	OPTIONAL MATCH (:Thing{uuid:{contentID}})-[r:MENTIONS{platformVersion:{platformVersion}}]->(t:Thing)
                         		DELETE r`
	case platformVersion == brightcovePlatformVersion:
		// TODO this clause should be refactored when all videos in Neo4j have brightcove only as lifecycle and no v1 reference
		statement = `	OPTIONAL MATCH (:Thing{uuid:{contentID}})-[r]->(t:Thing)
					WHERE r.lifecycle={lifecycle} OR r.lifecycle={v1Lifecycle}
					DELETE r`
	default:
		statement = `	OPTIONAL MATCH (:Thing{uuid:{contentID}})-[r]->(t:Thing)
					WHERE r.platformVersion={platformVersion}
                         		DELETE r`
	}

	query := neoism.CypherQuery{
		Statement: statement,
		Parameters: neoism.Props{"contentID": contentUUID, "platformVersion": platformVersion,
			"lifecycle": lifecycle(platformVersion), "v1Lifecycle": lifecycle(v1PlatformVersion)},
		IncludeStats: includeStats}
	return &query
}

func validateAnnotations(annotations *annotations) error {
	//TODO - for consistency, we should probably just not create the annotation?
	for _, annotation := range *annotations {
		if annotation.Thing.ID == "" {
			return ValidationError{fmt.Sprintf("Concept uuid missing for annotation %+v", annotation)}
		}
	}
	return nil
}

//ValidationError is thrown when the annotations are not valid because mandatory information is missing
type ValidationError struct {
	Msg string
}

func (v ValidationError) Error() string {
	return v.Msg
}

func mapToResponseFormat(ann *annotation) {
	ann.Thing.ID = mapper.IDURL(ann.Thing.ID)
	// We expect only ONE provenance - provenance value is considered valid even if the AgentRole is not specified. See: v1 - isClassifiedBy
	for idx := range ann.Provenances {
		if ann.Provenances[idx].AgentRole != "" {
			ann.Provenances[idx].AgentRole = mapper.IDURL(ann.Provenances[idx].AgentRole)
		}
	}
}

func lifecycle(platformVersion string) string {
	return "annotations-" + platformVersion
}
<|endoftext|>