<commit_before>package main

import (
	"cred-alert/db"
	"cred-alert/db/migrations"
	"cred-alert/gitclient"
	"cred-alert/metrics"
	"cred-alert/notifications"
	"cred-alert/revok"
	"cred-alert/revok/stats"
	"cred-alert/sniff"
	"fmt"
	"log"
	"net/http"
	"os"
	"time"

	"golang.org/x/oauth2"

	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
	"github.com/google/go-github/github"
	flags "github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
)

type Opts struct {
	LogLevel                    string        `long:"log-level" description:"log level to use"`
	WorkDir                     string        `long:"work-dir" description:"directory to work in" value-name:"PATH" required:"true"`
	RepositoryDiscoveryInterval time.Duration `long:"repository-discovery-interval" description:"how frequently to ask GitHub for all repos to check which ones we need to clone and dirscan" required:"true" value-name:"SCAN_INTERVAL" default:"1h"`
	ChangeDiscoveryInterval     time.Duration `long:"change-discovery-interval" description:"how frequently to fetch changes for repositories on disk and scan the changes" required:"true" value-name:"SCAN_INTERVAL" default:"1h"`

	Whitelist []string `short:"i" long:"ignore-patterns" description:"List of regex patterns to ignore." env:"IGNORED_PATTERNS" env-delim:"," value-name:"REGEX"`

	Port uint16 `short:"p" long:"port" description:"the port to listen on" default:"8080" env:"PORT" value-name:"PORT"`

	GitHub struct {
		AccessToken    string `short:"a" long:"access-token" description:"github api access token" env:"GITHUB_ACCESS_TOKEN" value-name:"TOKEN" required:"true"`
		PrivateKeyPath string `long:"github-private-key-path" description:"private key to use for GitHub auth" required:"true" value-name:"SSH_KEY"`
		PublicKeyPath  string `long:"github-public-key-path" description:"public key to use for GitHub auth" required:"true" value-name:"SSH_KEY"`
	} `group:"GitHub Options"`

	Metrics struct {
		SentryDSN     string `long:"sentry-dsn" description:"DSN to emit to Sentry with" env:"SENTRY_DSN" value-name:"DSN"`
		DatadogAPIKey string `long:"datadog-api-key" description:"key to emit to datadog" env:"DATADOG_API_KEY" value-name:"KEY"`
		Environment   string `long:"environment" description:"environment tag for metrics" env:"ENVIRONMENT" value-name:"NAME" default:"development"`
	} `group:"Metrics Options"`

	Slack struct {
		WebhookUrl string `long:"slack-webhook-url" description:"Slack webhook URL" env:"SLACK_WEBHOOK_URL" value-name:"WEBHOOK"`
	} `group:"Slack Options"`

	MySQL struct {
		Username string `long:"mysql-username" description:"MySQL username" value-name:"USERNAME" required:"true"`
		Password string `long:"mysql-password" description:"MySQL password" value-name:"PASSWORD"`
		Hostname string `long:"mysql-hostname" description:"MySQL hostname" value-name:"HOSTNAME" required:"true"`
		Port     uint16 `long:"mysql-port" description:"MySQL port" value-name:"PORT" required:"true"`
		DBName   string `long:"mysql-dbname" description:"MySQL database name" value-name:"DBNAME" required:"true"`
	}
}

func main() {
	var opts Opts

	logger := lager.NewLogger("revok-worker")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))

	logger.Debug("starting")

	_, err := flags.Parse(&opts)
	if err != nil {
		os.Exit(1)
	}

	if opts.Metrics.SentryDSN != "" {
		logger.RegisterSink(revok.NewSentrySink(opts.Metrics.SentryDSN, opts.Metrics.Environment))
	}

	workdir := opts.WorkDir
	_, err = os.Lstat(workdir)
	if err != nil {
		log.Fatalf("workdir error: %s", err)
	}

	githubHTTPClient := &http.Client{
		Timeout: 30 * time.Second,
		Transport: &oauth2.Transport{
			Source: oauth2.StaticTokenSource(
				&oauth2.Token{AccessToken: opts.GitHub.AccessToken},
			),
			Base: &http.Transport{
				DisableKeepAlives: true,
			},
		},
	}

	dbURI := db.NewDSN(opts.MySQL.Username, opts.MySQL.Password, opts.MySQL.DBName, opts.MySQL.Hostname, int(opts.MySQL.Port))
	database, err := migrations.LockDBAndMigrate(logger, "mysql", dbURI)
	if err != nil {
		log.Fatalf("db error: %s", err)
	}

	database.LogMode(false)

	clock := clock.NewClock()

	cloneMsgCh := make(chan revok.CloneMsg)
	ghClient := revok.NewGitHubClient(github.NewClient(githubHTTPClient))

	scanRepository := db.NewScanRepository(database, clock)
	repositoryRepository := db.NewRepositoryRepository(database)
	fetchRepository := db.NewFetchRepository(database)
	emitter := metrics.BuildEmitter(opts.Metrics.DatadogAPIKey, opts.Metrics.Environment)
	gitClient := gitclient.New(opts.GitHub.PrivateKeyPath, opts.GitHub.PublicKeyPath)
	repoWhitelist := notifications.BuildWhitelist(opts.Whitelist...)
	notifier := notifications.NewSlackNotifier(opts.Slack.WebhookUrl, clock, repoWhitelist)
	sniffer := sniff.NewDefaultSniffer()
	ancestryScanner := revok.NewScanner(
		gitClient,
		repositoryRepository,
		scanRepository,
		sniffer,
		notifier,
		emitter,
	)

	repoDiscoverer := revok.NewRepoDiscoverer(
		logger,
		workdir,
		cloneMsgCh,
		ghClient,
		clock,
		opts.RepositoryDiscoveryInterval,
		repositoryRepository,
	)

	cloner := revok.NewCloner(
		logger,
		workdir,
		cloneMsgCh,
		gitClient,
		repositoryRepository,
		ancestryScanner,
		emitter,
	)

	changeDiscoverer := revok.NewChangeDiscoverer(
		logger,
		gitClient,
		clock,
		opts.ChangeDiscoveryInterval,
		ancestryScanner,
		repositoryRepository,
		fetchRepository,
		emitter,
	)

	dirscanUpdater := revok.NewRescanner(
		logger,
		scanRepository,
		db.NewCredentialRepository(database),
		ancestryScanner,
		notifier,
		emitter,
	)

	statsReporter := stats.NewReporter(
		logger,
		clock,
		60*time.Second,
		db.NewStatsRepository(database),
		emitter,
	)

	handler := revok.NewHandler(
		logger,
		changeDiscoverer,
		repositoryRepository,
	)

	router := http.NewServeMux()
	router.Handle("/webhook", handler)

	runner := sigmon.New(grouper.NewParallel(os.Interrupt, []grouper.Member{
		{"repo-discoverer", repoDiscoverer},
		{"cloner", cloner},
		{"change-discoverer", changeDiscoverer},
		{"dirscan-updater", dirscanUpdater},
		{"stats-reporter", statsReporter},
		{"handler", http_server.New(fmt.Sprintf(":%d", opts.Port), router)},
	}))

	err = <-ifrit.Invoke(runner).Wait()
	if err != nil {
		log.Fatal("failed-to-start: %s", err)
	}
}
<commit_msg>fix flag name<commit_after>package main

import (
	"cred-alert/db"
	"cred-alert/db/migrations"
	"cred-alert/gitclient"
	"cred-alert/metrics"
	"cred-alert/notifications"
	"cred-alert/revok"
	"cred-alert/revok/stats"
	"cred-alert/sniff"
	"fmt"
	"log"
	"net/http"
	"os"
	"time"

	"golang.org/x/oauth2"

	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
	"github.com/google/go-github/github"
	flags "github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
)

type Opts struct {
	LogLevel                    string        `long:"log-level" description:"log level to use"`
	WorkDir                     string        `long:"work-dir" description:"directory to work in" value-name:"PATH" required:"true"`
	RepositoryDiscoveryInterval time.Duration `long:"repository-discovery-interval" description:"how frequently to ask GitHub for all repos to check which ones we need to clone and dirscan" required:"true" value-name:"SCAN_INTERVAL" default:"1h"`
	ChangeDiscoveryInterval     time.Duration `long:"change-discovery-interval" description:"how frequently to fetch changes for repositories on disk and scan the changes" required:"true" value-name:"SCAN_INTERVAL" default:"1h"`

	Whitelist []string `short:"i" long:"ignore-pattern" description:"List of regex patterns to ignore." env:"IGNORED_PATTERNS" env-delim:"," value-name:"REGEX"`

	Port uint16 `short:"p" long:"port" description:"the port to listen on" default:"8080" env:"PORT" value-name:"PORT"`

	GitHub struct {
		AccessToken    string `short:"a" long:"access-token" description:"github api access token" env:"GITHUB_ACCESS_TOKEN" value-name:"TOKEN" required:"true"`
		PrivateKeyPath string `long:"github-private-key-path" description:"private key to use for GitHub auth" required:"true" value-name:"SSH_KEY"`
		PublicKeyPath  string `long:"github-public-key-path" description:"public key to use for GitHub auth" required:"true" value-name:"SSH_KEY"`
	} `group:"GitHub Options"`

	Metrics struct {
		SentryDSN     string `long:"sentry-dsn" description:"DSN to emit to Sentry with" env:"SENTRY_DSN" value-name:"DSN"`
		DatadogAPIKey string `long:"datadog-api-key" description:"key to emit to datadog" env:"DATADOG_API_KEY" value-name:"KEY"`
		Environment   string `long:"environment" description:"environment tag for metrics" env:"ENVIRONMENT" value-name:"NAME" default:"development"`
	} `group:"Metrics Options"`

	Slack struct {
		WebhookUrl string `long:"slack-webhook-url" description:"Slack webhook URL" env:"SLACK_WEBHOOK_URL" value-name:"WEBHOOK"`
	} `group:"Slack Options"`

	MySQL struct {
		Username string `long:"mysql-username" description:"MySQL username" value-name:"USERNAME" required:"true"`
		Password string `long:"mysql-password" description:"MySQL password" value-name:"PASSWORD"`
		Hostname string `long:"mysql-hostname" description:"MySQL hostname" value-name:"HOSTNAME" required:"true"`
		Port     uint16 `long:"mysql-port" description:"MySQL port" value-name:"PORT" required:"true"`
		DBName   string `long:"mysql-dbname" description:"MySQL database name" value-name:"DBNAME" required:"true"`
	}
}

func main() {
	var opts Opts

	logger := lager.NewLogger("revok-worker")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))

	logger.Debug("starting")

	_, err := flags.Parse(&opts)
	if err != nil {
		os.Exit(1)
	}

	if opts.Metrics.SentryDSN != "" {
		logger.RegisterSink(revok.NewSentrySink(opts.Metrics.SentryDSN, opts.Metrics.Environment))
	}

	workdir := opts.WorkDir
	_, err = os.Lstat(workdir)
	if err != nil {
		log.Fatalf("workdir error: %s", err)
	}

	githubHTTPClient := &http.Client{
		Timeout: 30 * time.Second,
		Transport: &oauth2.Transport{
			Source: oauth2.StaticTokenSource(
				&oauth2.Token{AccessToken: opts.GitHub.AccessToken},
			),
			Base: &http.Transport{
				DisableKeepAlives: true,
			},
		},
	}

	dbURI := db.NewDSN(opts.MySQL.Username, opts.MySQL.Password, opts.MySQL.DBName, opts.MySQL.Hostname, int(opts.MySQL.Port))
	database, err := migrations.LockDBAndMigrate(logger, "mysql", dbURI)
	if err != nil {
		log.Fatalf("db error: %s", err)
	}

	database.LogMode(false)

	clock := clock.NewClock()

	cloneMsgCh := make(chan revok.CloneMsg)
	ghClient := revok.NewGitHubClient(github.NewClient(githubHTTPClient))

	scanRepository := db.NewScanRepository(database, clock)
	repositoryRepository := db.NewRepositoryRepository(database)
	fetchRepository := db.NewFetchRepository(database)
	emitter := metrics.BuildEmitter(opts.Metrics.DatadogAPIKey, opts.Metrics.Environment)
	gitClient := gitclient.New(opts.GitHub.PrivateKeyPath, opts.GitHub.PublicKeyPath)
	repoWhitelist := notifications.BuildWhitelist(opts.Whitelist...)
	notifier := notifications.NewSlackNotifier(opts.Slack.WebhookUrl, clock, repoWhitelist)
	sniffer := sniff.NewDefaultSniffer()
	ancestryScanner := revok.NewScanner(
		gitClient,
		repositoryRepository,
		scanRepository,
		sniffer,
		notifier,
		emitter,
	)

	repoDiscoverer := revok.NewRepoDiscoverer(
		logger,
		workdir,
		cloneMsgCh,
		ghClient,
		clock,
		opts.RepositoryDiscoveryInterval,
		repositoryRepository,
	)

	cloner := revok.NewCloner(
		logger,
		workdir,
		cloneMsgCh,
		gitClient,
		repositoryRepository,
		ancestryScanner,
		emitter,
	)

	changeDiscoverer := revok.NewChangeDiscoverer(
		logger,
		gitClient,
		clock,
		opts.ChangeDiscoveryInterval,
		ancestryScanner,
		repositoryRepository,
		fetchRepository,
		emitter,
	)

	dirscanUpdater := revok.NewRescanner(
		logger,
		scanRepository,
		db.NewCredentialRepository(database),
		ancestryScanner,
		notifier,
		emitter,
	)

	statsReporter := stats.NewReporter(
		logger,
		clock,
		60*time.Second,
		db.NewStatsRepository(database),
		emitter,
	)

	handler := revok.NewHandler(
		logger,
		changeDiscoverer,
		repositoryRepository,
	)

	router := http.NewServeMux()
	router.Handle("/webhook", handler)

	runner := sigmon.New(grouper.NewParallel(os.Interrupt, []grouper.Member{
		{"repo-discoverer", repoDiscoverer},
		{"cloner", cloner},
		{"change-discoverer", changeDiscoverer},
		{"dirscan-updater", dirscanUpdater},
		{"stats-reporter", statsReporter},
		{"handler", http_server.New(fmt.Sprintf(":%d", opts.Port), router)},
	}))

	err = <-ifrit.Invoke(runner).Wait()
	if err != nil {
		log.Fatal("failed-to-start: %s", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os/exec"

	"github.com/xiegeo/fensan/bitset"
	"github.com/xiegeo/fensan/hashtree"
	"github.com/xiegeo/fensan/pb"
	"github.com/xiegeo/fensan/pconn"
	"github.com/xiegeo/fensan/store"
)

//make sure go get gets every sub package
var _ = bitset.CHECK_INTEX
var _ = hashtree.HashSize
var _ = &pb.StaticId{}
var _ = pconn.SendBytes
var _ = store.FileNone

func main() {
	buildProtoBuf()
	testCode("bitset")
	testCode("hashtree")
	testCode("pb")
	testCode("pconn")
	testCode("store")
	fmt.Println("\n\ndone all builds and tests")
}

func buildProtoBuf() {
	dir = "pb"
	defer func() { dir = "" }()
	err := doCmd(exec.Command("go", "test", "-v"))
	if err != nil {
		doCmd(exec.Command("protoc", "--gogo_out=.", "*.proto"))
	}
}

func testCode(packageName string) {
	dir = packageName
	defer func() { dir = "" }()
	noErr(doCmd(exec.Command("go", "test", "-v")))
}

func noErr(err error) {
	if err != nil {
		panic(err)
	}
}

var dir = ""

func doCmd(cmd *exec.Cmd) error {
	cmd.Dir = dir
	out, err := cmd.CombinedOutput()
	if err != nil {
		fmt.Println(cmd.Path)
		fmt.Println(cmd.Args)
		fmt.Printf("%s\n", out)
		fmt.Printf("error:%v\n", err)
	}
	return err
}
<commit_msg>clean messages<commit_after>package main

import (
	"fmt"
	"os/exec"

	"github.com/xiegeo/fensan/bitset"
	"github.com/xiegeo/fensan/hashtree"
	"github.com/xiegeo/fensan/pb"
	"github.com/xiegeo/fensan/pconn"
	"github.com/xiegeo/fensan/store"
)

//make sure go get gets every sub package
var _ = bitset.CHECK_INTEX
var _ = hashtree.HashSize
var _ = &pb.StaticId{}
var _ = pconn.SendBytes
var _ = store.FileNone

func main() {
	buildProtoBuf()
	testCode("bitset")
	testCode("hashtree")
	testCode("pb")
	testCode("pconn")
	testCode("store")
	fmt.Println("\n\ndone all builds and tests")
}

func buildProtoBuf() {
	dir = "pb"
	defer func() { dir = "" }()
	err := doHiddenCmd(exec.Command("go", "test", "-v"))
	if err != nil {
		fmt.Println("rebuilding .pb.go files")
		err := doCmd(exec.Command("protoc", "--gogo_out=.", "*.proto"))
		if err == nil {
			fmt.Println("rebuilded .pb.go files ")
		}
	}
}

func testCode(packageName string) {
	dir = packageName
	defer func() { dir = "" }()
	noErr(doCmd(exec.Command("go", "test", "-v")))
}

func noErr(err error) {
	if err != nil {
		panic(err)
	}
}

var dir = ""

func doCmd(cmd *exec.Cmd) error {
	cmd.Dir = dir
	out, err := cmd.CombinedOutput()
	if err != nil {
		fmt.Println(cmd.Path)
		fmt.Println(cmd.Args)
		fmt.Printf("%s\n", out)
		fmt.Printf("error:%v\n", err)
	}
	return err
}

func doHiddenCmd(cmd *exec.Cmd) error {
	cmd.Dir = dir
	_, err := cmd.CombinedOutput()
	return err
}
<|endoftext|>


<commit_before>// Copyright 2019, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package trace

import (
	"fmt"
	"math"
	"strconv"
	"time"
	"unicode/utf8"

	timestamppb "github.com/golang/protobuf/ptypes/timestamp"
	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
	tracepb "google.golang.org/genproto/googleapis/devtools/cloudtrace/v2"
	statuspb "google.golang.org/genproto/googleapis/rpc/status"
	"google.golang.org/grpc/codes"

	"go.opentelemetry.io/otel/api/kv"
	"go.opentelemetry.io/otel/api/kv/value"
	opentelemetry "go.opentelemetry.io/otel/sdk"
	export "go.opentelemetry.io/otel/sdk/export/trace"
)

const (
	maxAnnotationEventsPerSpan = 32
	// TODO(ymotongpoo): uncomment this after gRPC trace get supported.
	// maxMessageEventsPerSpan    = 128
	maxAttributeStringValue = 256
	agentLabel              = "g.co/agent"

	// Attributes recorded on the span for the requests.
	// Only trace exporters will need them.
	HostAttribute       = "http.host"
	MethodAttribute     = "http.method"
	PathAttribute       = "http.path"
	URLAttribute        = "http.url"
	UserAgentAttribute  = "http.user_agent"
	StatusCodeAttribute = "http.status_code"

	labelHTTPHost       = `/http/host`
	labelHTTPMethod     = `/http/method`
	labelHTTPStatusCode = `/http/status_code`
	labelHTTPPath       = `/http/path`
	labelHTTPUserAgent  = `/http/user_agent`

	version = "0.1.0"
)

var userAgent = fmt.Sprintf("opentelemetry-go %s; cloudtrace-exporter %s", opentelemetry.Version(), version)

func protoFromSpanData(s *export.SpanData, projectID string) *tracepb.Span {
	if s == nil {
		return nil
	}

	traceIDString := s.SpanContext.TraceID.String()
	spanIDString := s.SpanContext.SpanID.String()

	name := s.Name
	switch s.SpanKind {
	// TODO(ymotongpoo): add cases for "Send" and "Recv".
	default:
		name = fmt.Sprintf("Span.%s-%s", s.SpanKind, name)
	}

	sp := &tracepb.Span{
		Name:                    "projects/" + projectID + "/traces/" + traceIDString + "/spans/" + spanIDString,
		SpanId:                  spanIDString,
		DisplayName:             trunc(name, 128),
		StartTime:               timestampProto(s.StartTime),
		EndTime:                 timestampProto(s.EndTime),
		SameProcessAsParentSpan: &wrapperspb.BoolValue{Value: !s.HasRemoteParent},
	}
	if s.ParentSpanID != s.SpanContext.SpanID && s.ParentSpanID.IsValid() {
		sp.ParentSpanId = fmt.Sprintf("%.16x", s.ParentSpanID)
	}
	if s.StatusCode != codes.OK {
		sp.Status = &statuspb.Status{Code: int32(s.StatusCode)}
	}

	copyAttributes(&sp.Attributes, s.Attributes)
	// NOTE(ymotongpoo): omitting copyMonitoringReesourceAttributes()

	var annotations, droppedAnnotationsCount int
	es := s.MessageEvents
	for i, e := range es {
		if annotations >= maxAnnotationEventsPerSpan {
			droppedAnnotationsCount = len(es) - i
			break
		}
		annotation := &tracepb.Span_TimeEvent_Annotation{Description: trunc(e.Name, maxAttributeStringValue)}
		copyAttributes(&annotation.Attributes, e.Attributes)
		event := &tracepb.Span_TimeEvent{
			Time:  timestampProto(e.Time),
			Value: &tracepb.Span_TimeEvent_Annotation_{Annotation: annotation},
		}
		annotations++
		if sp.TimeEvents == nil {
			sp.TimeEvents = &tracepb.Span_TimeEvents{}
		}
		sp.TimeEvents.TimeEvent = append(sp.TimeEvents.TimeEvent, event)
	}

	if sp.Attributes == nil {
		sp.Attributes = &tracepb.Span_Attributes{
			AttributeMap: make(map[string]*tracepb.AttributeValue),
		}
	}

	// Only set the agent label if it is not already set. That enables the
	// OpenTelemery service/collector to set the agent label based on the library that
	// sent the span to the service.
	if _, hasAgent := sp.Attributes.AttributeMap[agentLabel]; !hasAgent {
		sp.Attributes.AttributeMap[agentLabel] = &tracepb.AttributeValue{
			Value: &tracepb.AttributeValue_StringValue{
				StringValue: trunc(userAgent, maxAttributeStringValue),
			},
		}
	}

	// TODO(ymotongpoo): add implementations for Span_TimeEvent_MessageEvent_
	// once OTel finish implementations for gRPC.

	if droppedAnnotationsCount != 0 {
		if sp.TimeEvents == nil {
			sp.TimeEvents = &tracepb.Span_TimeEvents{}
		}
		sp.TimeEvents.DroppedAnnotationsCount = clip32(droppedAnnotationsCount)
	}

	// TODO(ymotongpoo): add implementations for Links

	return sp
}

// timestampProto creates a timestamp proto for a time.Time.
func timestampProto(t time.Time) *timestamppb.Timestamp {
	return &timestamppb.Timestamp{
		Seconds: t.Unix(),
		Nanos:   int32(t.Nanosecond()),
	}
}

// copyAttributes copies a map of attributes to a proto map field.
// It creates the map if it is nil.
func copyAttributes(out **tracepb.Span_Attributes, in []kv.KeyValue) {
	if len(in) == 0 {
		return
	}
	if *out == nil {
		*out = &tracepb.Span_Attributes{}
	}
	if (*out).AttributeMap == nil {
		(*out).AttributeMap = make(map[string]*tracepb.AttributeValue)
	}
	var dropped int32
	for _, kv := range in {
		av := attributeValue(kv)
		if av == nil {
			continue
		}
		switch kv.Key {
		case PathAttribute:
			(*out).AttributeMap[labelHTTPPath] = av
		case HostAttribute:
			(*out).AttributeMap[labelHTTPHost] = av
		case MethodAttribute:
			(*out).AttributeMap[labelHTTPMethod] = av
		case UserAgentAttribute:
			(*out).AttributeMap[labelHTTPUserAgent] = av
		case StatusCodeAttribute:
			(*out).AttributeMap[labelHTTPStatusCode] = av
		default:
			if len(kv.Key) > 128 {
				dropped++
				continue
			}
			(*out).AttributeMap[string(kv.Key)] = av
		}
	}
	(*out).DroppedAttributesCount = dropped
}

func attributeValue(kv kv.KeyValue) *tracepb.AttributeValue {
	v := kv.Value
	switch v.Type() {
	case value.BOOL:
		return &tracepb.AttributeValue{
			Value: &tracepb.AttributeValue_BoolValue{BoolValue: v.AsBool()},
		}
	case value.INT64:
		return &tracepb.AttributeValue{
			Value: &tracepb.AttributeValue_IntValue{IntValue: v.AsInt64()},
		}
	case value.FLOAT64:
		// TODO: set double value if Google Cloud Trace support it in the future.
		return &tracepb.AttributeValue{
			Value: &tracepb.AttributeValue_StringValue{
				StringValue: trunc(strconv.FormatFloat(v.AsFloat64(), 'f', -1, 64),
					maxAttributeStringValue)},
		}
	case value.STRING:
		return &tracepb.AttributeValue{
			Value: &tracepb.AttributeValue_StringValue{StringValue: trunc(v.AsString(), maxAttributeStringValue)},
		}
	}
	return nil
}

// trunc returns a TruncatableString truncated to the given limit.
func trunc(s string, limit int) *tracepb.TruncatableString {
	if len(s) > limit {
		b := []byte(s[:limit])
		for {
			r, size := utf8.DecodeLastRune(b)
			if r == utf8.RuneError && size == 1 {
				b = b[:len(b)-1]
			} else {
				break
			}
		}
		return &tracepb.TruncatableString{
			Value:              string(b),
			TruncatedByteCount: clip32(len(s) - len(b)),
		}
	}
	return &tracepb.TruncatableString{
		Value:              s,
		TruncatedByteCount: 0,
	}
}

// clip32 clips an int to the range of an int32.
func clip32(x int) int32 {
	if x < math.MinInt32 {
		return math.MinInt32
	}
	if x > math.MaxInt32 {
		return math.MaxInt32
	}
	return int32(x)
}
<commit_msg>Fix 'rpc error: code = InvalidArgument desc = Invalid parent id!' errors. (#27)<commit_after>// Copyright 2019, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package trace

import (
	"fmt"
	"math"
	"strconv"
	"time"
	"unicode/utf8"

	timestamppb "github.com/golang/protobuf/ptypes/timestamp"
	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
	tracepb "google.golang.org/genproto/googleapis/devtools/cloudtrace/v2"
	statuspb "google.golang.org/genproto/googleapis/rpc/status"
	"google.golang.org/grpc/codes"

	"go.opentelemetry.io/otel/api/kv"
	"go.opentelemetry.io/otel/api/kv/value"
	opentelemetry "go.opentelemetry.io/otel/sdk"
	export "go.opentelemetry.io/otel/sdk/export/trace"
)

const (
	maxAnnotationEventsPerSpan = 32
	// TODO(ymotongpoo): uncomment this after gRPC trace get supported.
	// maxMessageEventsPerSpan    = 128
	maxAttributeStringValue = 256
	agentLabel              = "g.co/agent"

	// Attributes recorded on the span for the requests.
	// Only trace exporters will need them.
	HostAttribute       = "http.host"
	MethodAttribute     = "http.method"
	PathAttribute       = "http.path"
	URLAttribute        = "http.url"
	UserAgentAttribute  = "http.user_agent"
	StatusCodeAttribute = "http.status_code"

	labelHTTPHost       = `/http/host`
	labelHTTPMethod     = `/http/method`
	labelHTTPStatusCode = `/http/status_code`
	labelHTTPPath       = `/http/path`
	labelHTTPUserAgent  = `/http/user_agent`

	version = "0.1.0"
)

var userAgent = fmt.Sprintf("opentelemetry-go %s; cloudtrace-exporter %s", opentelemetry.Version(), version)

func protoFromSpanData(s *export.SpanData, projectID string) *tracepb.Span {
	if s == nil {
		return nil
	}

	traceIDString := s.SpanContext.TraceID.String()
	spanIDString := s.SpanContext.SpanID.String()

	name := s.Name
	switch s.SpanKind {
	// TODO(ymotongpoo): add cases for "Send" and "Recv".
	default:
		name = fmt.Sprintf("Span.%s-%s", s.SpanKind, name)
	}

	sp := &tracepb.Span{
		Name:                    "projects/" + projectID + "/traces/" + traceIDString + "/spans/" + spanIDString,
		SpanId:                  spanIDString,
		DisplayName:             trunc(name, 128),
		StartTime:               timestampProto(s.StartTime),
		EndTime:                 timestampProto(s.EndTime),
		SameProcessAsParentSpan: &wrapperspb.BoolValue{Value: !s.HasRemoteParent},
	}
	if s.ParentSpanID != s.SpanContext.SpanID && s.ParentSpanID.IsValid() {
		sp.ParentSpanId = s.ParentSpanID.String()
	}
	if s.StatusCode != codes.OK {
		sp.Status = &statuspb.Status{Code: int32(s.StatusCode)}
	}

	copyAttributes(&sp.Attributes, s.Attributes)
	// NOTE(ymotongpoo): omitting copyMonitoringReesourceAttributes()

	var annotations, droppedAnnotationsCount int
	es := s.MessageEvents
	for i, e := range es {
		if annotations >= maxAnnotationEventsPerSpan {
			droppedAnnotationsCount = len(es) - i
			break
		}
		annotation := &tracepb.Span_TimeEvent_Annotation{Description: trunc(e.Name, maxAttributeStringValue)}
		copyAttributes(&annotation.Attributes, e.Attributes)
		event := &tracepb.Span_TimeEvent{
			Time:  timestampProto(e.Time),
			Value: &tracepb.Span_TimeEvent_Annotation_{Annotation: annotation},
		}
		annotations++
		if sp.TimeEvents == nil {
			sp.TimeEvents = &tracepb.Span_TimeEvents{}
		}
		sp.TimeEvents.TimeEvent = append(sp.TimeEvents.TimeEvent, event)
	}

	if sp.Attributes == nil {
		sp.Attributes = &tracepb.Span_Attributes{
			AttributeMap: make(map[string]*tracepb.AttributeValue),
		}
	}

	// Only set the agent label if it is not already set. That enables the
	// OpenTelemery service/collector to set the agent label based on the library that
	// sent the span to the service.
	if _, hasAgent := sp.Attributes.AttributeMap[agentLabel]; !hasAgent {
		sp.Attributes.AttributeMap[agentLabel] = &tracepb.AttributeValue{
			Value: &tracepb.AttributeValue_StringValue{
				StringValue: trunc(userAgent, maxAttributeStringValue),
			},
		}
	}

	// TODO(ymotongpoo): add implementations for Span_TimeEvent_MessageEvent_
	// once OTel finish implementations for gRPC.

	if droppedAnnotationsCount != 0 {
		if sp.TimeEvents == nil {
			sp.TimeEvents = &tracepb.Span_TimeEvents{}
		}
		sp.TimeEvents.DroppedAnnotationsCount = clip32(droppedAnnotationsCount)
	}

	// TODO(ymotongpoo): add implementations for Links

	return sp
}

// timestampProto creates a timestamp proto for a time.Time.
func timestampProto(t time.Time) *timestamppb.Timestamp {
	return &timestamppb.Timestamp{
		Seconds: t.Unix(),
		Nanos:   int32(t.Nanosecond()),
	}
}

// copyAttributes copies a map of attributes to a proto map field.
// It creates the map if it is nil.
func copyAttributes(out **tracepb.Span_Attributes, in []kv.KeyValue) {
	if len(in) == 0 {
		return
	}
	if *out == nil {
		*out = &tracepb.Span_Attributes{}
	}
	if (*out).AttributeMap == nil {
		(*out).AttributeMap = make(map[string]*tracepb.AttributeValue)
	}
	var dropped int32
	for _, kv := range in {
		av := attributeValue(kv)
		if av == nil {
			continue
		}
		switch kv.Key {
		case PathAttribute:
			(*out).AttributeMap[labelHTTPPath] = av
		case HostAttribute:
			(*out).AttributeMap[labelHTTPHost] = av
		case MethodAttribute:
			(*out).AttributeMap[labelHTTPMethod] = av
		case UserAgentAttribute:
			(*out).AttributeMap[labelHTTPUserAgent] = av
		case StatusCodeAttribute:
			(*out).AttributeMap[labelHTTPStatusCode] = av
		default:
			if len(kv.Key) > 128 {
				dropped++
				continue
			}
			(*out).AttributeMap[string(kv.Key)] = av
		}
	}
	(*out).DroppedAttributesCount = dropped
}

func attributeValue(kv kv.KeyValue) *tracepb.AttributeValue {
	v := kv.Value
	switch v.Type() {
	case value.BOOL:
		return &tracepb.AttributeValue{
			Value: &tracepb.AttributeValue_BoolValue{BoolValue: v.AsBool()},
		}
	case value.INT64:
		return &tracepb.AttributeValue{
			Value: &tracepb.AttributeValue_IntValue{IntValue: v.AsInt64()},
		}
	case value.FLOAT64:
		// TODO: set double value if Google Cloud Trace support it in the future.
		return &tracepb.AttributeValue{
			Value: &tracepb.AttributeValue_StringValue{
				StringValue: trunc(strconv.FormatFloat(v.AsFloat64(), 'f', -1, 64),
					maxAttributeStringValue)},
		}
	case value.STRING:
		return &tracepb.AttributeValue{
			Value: &tracepb.AttributeValue_StringValue{StringValue: trunc(v.AsString(), maxAttributeStringValue)},
		}
	}
	return nil
}

// trunc returns a TruncatableString truncated to the given limit.
func trunc(s string, limit int) *tracepb.TruncatableString {
	if len(s) > limit {
		b := []byte(s[:limit])
		for {
			r, size := utf8.DecodeLastRune(b)
			if r == utf8.RuneError && size == 1 {
				b = b[:len(b)-1]
			} else {
				break
			}
		}
		return &tracepb.TruncatableString{
			Value:              string(b),
			TruncatedByteCount: clip32(len(s) - len(b)),
		}
	}
	return &tracepb.TruncatableString{
		Value:              s,
		TruncatedByteCount: 0,
	}
}

// clip32 clips an int to the range of an int32.
func clip32(x int) int32 {
	if x < math.MinInt32 {
		return math.MinInt32
	}
	if x > math.MaxInt32 {
		return math.MaxInt32
	}
	return int32(x)
}
<|endoftext|>


<commit_before>// +build matcha

package bridge

// Go support functions for Objective-C. Note that this
// file is copied into and compiled with the generated
// bindings.

/*
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include "go-foreign.h"
*/
import "C"

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"math"
	"reflect"
	"runtime"
	"runtime/debug"
	"time"
)

//export matchaTestFunc
func matchaTestFunc() {
	count := C.MatchaForeignTrackerCount()

	for i := 0; i < 1000; i++ {
		z := Nil()
		a := Bool(true)
		b := Int64(1234)
		c := Float64(1.234)
		d := String("abc")
		e := Bytes([]byte("def123"))
		f := Interface(123 + 234i)

		if !z.IsNil() ||
			a.ToBool() != true ||
			b.ToInt64() != 1234 ||
			c.ToFloat64() != 1.234 ||
			d.ToString() != "abc" ||
			!bytes.Equal(e.ToBytes(), []byte("def123")) ||
			f.ToInterface() != 123+234i {

			panic("Primitive mismatch")
		}

		arr := Array(z, a, b, c, d, e, f)
		arr2 := arr.ToArray()

		z = arr2[0]
		a = arr2[1]
		b = arr2[2]
		c = arr2[3]
		d = arr2[4]
		e = arr2[5]
		f = arr2[6]

		if !z.IsNil() ||
			a.ToBool() != true ||
			b.ToInt64() != 1234 ||
			c.ToFloat64() != 1.234 ||
			d.ToString() != "abc" ||
			!bytes.Equal(e.ToBytes(), []byte("def123")) ||
			f.ToInterface() != 123+234i {

			panic("Array mismatch")
		}

		runtime.GC()
	}

	// bridge := Bridge("a")
	// fmt.Println("matchaTestFunc() - Bridge:", bridge)

	debug.FreeOSMemory()
	time.Sleep(time.Second)

	newCount := C.MatchaForeignTrackerCount()
	fmt.Println("count", count, newCount)
	if math.Abs(float64(count-newCount)) > 1 { // Allow some leeway cause finalizer acts weirdly...
		panic("Count mismatch")
	}
}

var untrackChan = make(chan int64, 20)

func init() {
	go func() {
		runtime.LockOSThread()
		for i := range untrackChan {
			C.MatchaForeignUntrack(C.FgnRef(i))
		}
		runtime.UnlockOSThread()
	}()
}

type Value struct {
	ref int64
}

func newValue(ref C.FgnRef) *Value {
	v := &Value{ref: int64(ref)}
	runtime.SetFinalizer(v, func(a *Value) {
		untrackChan <- a.ref
	})
	return v
}

func (v *Value) _ref() C.FgnRef {
	return C.FgnRef(v.ref)
}

func Bridge(a string) *Value {
	cstr := cString(a)
	return newValue(C.MatchaForeignBridge(cstr))
}

func Nil() *Value {
	return newValue(C.MatchaForeignNil())
}

func (v *Value) IsNil() bool {
	defer runtime.KeepAlive(v)
	return bool(C.MatchaForeignIsNil(v._ref()))
}

func Bool(v bool) *Value {
	return newValue(C.MatchaForeignBool(C.bool(v)))
}

func (v *Value) ToBool() bool {
	defer runtime.KeepAlive(v)
	return bool(C.MatchaForeignToBool(v._ref()))
}

func Int64(v int64) *Value {
	return newValue(C.MatchaForeignInt64(C.int64_t(v)))
}

func (v *Value) ToInt64() int64 {
	defer runtime.KeepAlive(v)
	return int64(C.MatchaForeignToInt64(v._ref()))
}

func Float64(v float64) *Value {
	return newValue(C.MatchaForeignFloat64(C.double(v)))
}

func (v *Value) ToFloat64() float64 {
	defer runtime.KeepAlive(v)
	return float64(C.MatchaForeignToFloat64(v._ref()))
}

func String(v string) *Value {
	cstr := cString(v)
	return newValue(C.MatchaForeignString(cstr))
}

func (v *Value) ToString() string {
	defer runtime.KeepAlive(v)
	buf := C.MatchaForeignToString(v._ref())
	return goString(buf)
}

func Bytes(v []byte) *Value {
	cbytes := cBytes(v)
	return newValue(C.MatchaForeignBytes(cbytes))
}

func (v *Value) ToBytes() []byte {
	defer runtime.KeepAlive(v)
	buf := C.MatchaForeignToBytes(v._ref())
	return goBytes(buf)
}

func Interface(v interface{}) *Value {
	// Start with a go value.
	// Reflect on it.
	rv := reflect.ValueOf(v)
	// Track it, turning it into a goref.
	ref := matchaGoTrack(rv)
	// Wrap the goref in an foreign object, returning a foreign ref.
	return newValue(C.MatchaForeignGoRef(ref))
}

func (v *Value) ToInterface() interface{} {
	defer runtime.KeepAlive(v)
	// Start with a foreign ref, referring to a foreign value wrapping a go ref.
	// Get the goref.
	ref := C.MatchaForeignToGoRef(v._ref())
	// Get the go object, and unreflect.
	return matchaGoGet(ref).Interface()
}

func Array(a ...*Value) *Value {
	defer runtime.KeepAlive(a)
	ref := C.MatchaForeignArray(cArray2(a))
	return newValue(ref)
}

func (v *Value) ToArray() []*Value { // TODO(KD): Untested....
	defer runtime.KeepAlive(v)
	buf := C.MatchaForeignToArray(v._ref())
	return goArray2(buf)
}

// Call accepts `nil` in its variadic arguments
func (v *Value) Call(s string, args ...*Value) *Value {
	defer runtime.KeepAlive(v)
	defer runtime.KeepAlive(args)
	return newValue(C.MatchaForeignCall(v._ref(), cString(s), cArray2(args)))
}

func cArray(v []reflect.Value) C.CGoBuffer {
	var cstr C.CGoBuffer
	if len(v) == 0 {
		cstr = C.CGoBuffer{}
	} else {
		buf := new(bytes.Buffer)
		for _, i := range v {
			goref := matchaGoTrack(i)
			err := binary.Write(buf, binary.LittleEndian, goref)
			if err != nil {
				fmt.Println("binary.Write failed:", err)
			}
		}
		cstr = C.CGoBuffer{
			ptr: C.CBytes(buf.Bytes()),
			len: C.int64_t(len(buf.Bytes())),
		}
	}
	return cstr
}

func cArray2(v []*Value) C.CGoBuffer {
	var cstr C.CGoBuffer
	if len(v) == 0 {
		cstr = C.CGoBuffer{}
	} else {
		buf := new(bytes.Buffer)
		for _, i := range v {
			foreignRef := i._ref()
			err := binary.Write(buf, binary.LittleEndian, foreignRef)
			if err != nil {
				fmt.Println("binary.Write failed:", err)
			}
		}
		cstr = C.CGoBuffer{
			ptr: C.CBytes(buf.Bytes()),
			len: C.int64_t(len(buf.Bytes())),
		}
	}
	return cstr
}

func cBytes(v []byte) C.CGoBuffer {
	var cstr C.CGoBuffer
	if len(v) == 0 {
		cstr = C.CGoBuffer{}
	} else {
		cstr = C.CGoBuffer{
			ptr: C.CBytes(v),
			len: C.int64_t(len(v)),
		}
	}
	return cstr
}

func cString(v string) C.CGoBuffer {
	var cstr C.CGoBuffer
	if len(v) == 0 {
		cstr = C.CGoBuffer{}
	} else {
		cstr = C.CGoBuffer{
			ptr: C.CBytes([]byte(v)),
			len: C.int64_t(len(v)),
		}
	}
	return cstr
}

func goArray(buf C.CGoBuffer) []reflect.Value {
	defer C.free(buf.ptr)

	gorefs := make([]int64, buf.len/8)
	err := binary.Read(bytes.NewBuffer(C.GoBytes(buf.ptr, C.int(buf.len))), binary.LittleEndian, gorefs)
	if err != nil {
		panic(err)
	}

	rvs := []reflect.Value{}
	for _, i := range gorefs {
		rv := matchaGoGet(C.GoRef(i))
		rvs = append(rvs, rv)
	}
	return rvs
}

func goArray2(buf C.CGoBuffer) []*Value {
	defer C.free(buf.ptr)

	fgnRef := make([]int64, buf.len/8)
	err := binary.Read(bytes.NewBuffer(C.GoBytes(buf.ptr, C.int(buf.len))), binary.LittleEndian, fgnRef)
	if err != nil {
		panic(err)
	}

	rvs := []*Value{}
	for _, i := range fgnRef {
		rv := newValue(C.FgnRef(i))
		rvs = append(rvs, rv)
	}
	return rvs
}

func goString(buf C.CGoBuffer) string {
	defer C.free(buf.ptr)
	str := C.GoBytes(buf.ptr, C.int(buf.len))
	return string(str)
}

func goBytes(buf C.CGoBuffer) []byte {
	defer C.free(buf.ptr)
	return C.GoBytes(buf.ptr, C.int(buf.len))
}
<commit_msg>Cache bridge values<commit_after>// +build matcha

package bridge

// Go support functions for Objective-C. Note that this
// file is copied into and compiled with the generated
// bindings.

/*
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include "go-foreign.h"
*/
import "C"

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"math"
	"reflect"
	"runtime"
	"runtime/debug"
	"time"
)

//export matchaTestFunc
func matchaTestFunc() {
	count := C.MatchaForeignTrackerCount()

	for i := 0; i < 1000; i++ {
		z := Nil()
		a := Bool(true)
		b := Int64(1234)
		c := Float64(1.234)
		d := String("abc")
		e := Bytes([]byte("def123"))
		f := Interface(123 + 234i)

		if !z.IsNil() ||
			a.ToBool() != true ||
			b.ToInt64() != 1234 ||
			c.ToFloat64() != 1.234 ||
			d.ToString() != "abc" ||
			!bytes.Equal(e.ToBytes(), []byte("def123")) ||
			f.ToInterface() != 123+234i {

			panic("Primitive mismatch")
		}

		arr := Array(z, a, b, c, d, e, f)
		arr2 := arr.ToArray()

		z = arr2[0]
		a = arr2[1]
		b = arr2[2]
		c = arr2[3]
		d = arr2[4]
		e = arr2[5]
		f = arr2[6]

		if !z.IsNil() ||
			a.ToBool() != true ||
			b.ToInt64() != 1234 ||
			c.ToFloat64() != 1.234 ||
			d.ToString() != "abc" ||
			!bytes.Equal(e.ToBytes(), []byte("def123")) ||
			f.ToInterface() != 123+234i {

			panic("Array mismatch")
		}

		runtime.GC()
	}

	// bridge := Bridge("a")
	// fmt.Println("matchaTestFunc() - Bridge:", bridge)

	debug.FreeOSMemory()
	time.Sleep(time.Second)

	newCount := C.MatchaForeignTrackerCount()
	fmt.Println("count", count, newCount)
	if math.Abs(float64(count-newCount)) > 1 { // Allow some leeway cause finalizer acts weirdly...
		panic("Count mismatch")
	}
}

var bridgeCache = map[string]*Value{}
var untrackChan = make(chan int64, 20)

func init() {
	go func() {
		runtime.LockOSThread()
		for i := range untrackChan {
			C.MatchaForeignUntrack(C.FgnRef(i))
		}
		runtime.UnlockOSThread()
	}()
}

type Value struct {
	ref int64
}

func newValue(ref C.FgnRef) *Value {
	v := &Value{ref: int64(ref)}
	runtime.SetFinalizer(v, func(a *Value) {
		untrackChan <- a.ref
	})
	return v
}

func (v *Value) _ref() C.FgnRef {
	return C.FgnRef(v.ref)
}

func Bridge(a string) *Value {
	if b, ok := bridgeCache[a]; ok {
		return b
	}
	cstr := cString(a)
	b := newValue(C.MatchaForeignBridge(cstr))
	bridgeCache[a] = b
	return b
}

func Nil() *Value {
	return newValue(C.MatchaForeignNil())
}

func (v *Value) IsNil() bool {
	defer runtime.KeepAlive(v)
	return bool(C.MatchaForeignIsNil(v._ref()))
}

func Bool(v bool) *Value {
	return newValue(C.MatchaForeignBool(C.bool(v)))
}

func (v *Value) ToBool() bool {
	defer runtime.KeepAlive(v)
	return bool(C.MatchaForeignToBool(v._ref()))
}

func Int64(v int64) *Value {
	return newValue(C.MatchaForeignInt64(C.int64_t(v)))
}

func (v *Value) ToInt64() int64 {
	defer runtime.KeepAlive(v)
	return int64(C.MatchaForeignToInt64(v._ref()))
}

func Float64(v float64) *Value {
	return newValue(C.MatchaForeignFloat64(C.double(v)))
}

func (v *Value) ToFloat64() float64 {
	defer runtime.KeepAlive(v)
	return float64(C.MatchaForeignToFloat64(v._ref()))
}

func String(v string) *Value {
	cstr := cString(v)
	return newValue(C.MatchaForeignString(cstr))
}

func (v *Value) ToString() string {
	defer runtime.KeepAlive(v)
	buf := C.MatchaForeignToString(v._ref())
	return goString(buf)
}

func Bytes(v []byte) *Value {
	cbytes := cBytes(v)
	return newValue(C.MatchaForeignBytes(cbytes))
}

func (v *Value) ToBytes() []byte {
	defer runtime.KeepAlive(v)
	buf := C.MatchaForeignToBytes(v._ref())
	return goBytes(buf)
}

func Interface(v interface{}) *Value {
	// Start with a go value.
	// Reflect on it.
	rv := reflect.ValueOf(v)
	// Track it, turning it into a goref.
	ref := matchaGoTrack(rv)
	// Wrap the goref in an foreign object, returning a foreign ref.
	return newValue(C.MatchaForeignGoRef(ref))
}

func (v *Value) ToInterface() interface{} {
	defer runtime.KeepAlive(v)
	// Start with a foreign ref, referring to a foreign value wrapping a go ref.
	// Get the goref.
	ref := C.MatchaForeignToGoRef(v._ref())
	// Get the go object, and unreflect.
	return matchaGoGet(ref).Interface()
}

func Array(a ...*Value) *Value {
	defer runtime.KeepAlive(a)
	ref := C.MatchaForeignArray(cArray2(a))
	return newValue(ref)
}

func (v *Value) ToArray() []*Value { // TODO(KD): Untested....
	defer runtime.KeepAlive(v)
	buf := C.MatchaForeignToArray(v._ref())
	return goArray2(buf)
}

// Call accepts `nil` in its variadic arguments
func (v *Value) Call(s string, args ...*Value) *Value {
	defer runtime.KeepAlive(v)
	defer runtime.KeepAlive(args)
	return newValue(C.MatchaForeignCall(v._ref(), cString(s), cArray2(args)))
}

func cArray(v []reflect.Value) C.CGoBuffer {
	var cstr C.CGoBuffer
	if len(v) == 0 {
		cstr = C.CGoBuffer{}
	} else {
		buf := new(bytes.Buffer)
		for _, i := range v {
			goref := matchaGoTrack(i)
			err := binary.Write(buf, binary.LittleEndian, goref)
			if err != nil {
				fmt.Println("binary.Write failed:", err)
			}
		}
		cstr = C.CGoBuffer{
			ptr: C.CBytes(buf.Bytes()),
			len: C.int64_t(len(buf.Bytes())),
		}
	}
	return cstr
}

func cArray2(v []*Value) C.CGoBuffer {
	var cstr C.CGoBuffer
	if len(v) == 0 {
		cstr = C.CGoBuffer{}
	} else {
		buf := new(bytes.Buffer)
		for _, i := range v {
			foreignRef := i._ref()
			err := binary.Write(buf, binary.LittleEndian, foreignRef)
			if err != nil {
				fmt.Println("binary.Write failed:", err)
			}
		}
		cstr = C.CGoBuffer{
			ptr: C.CBytes(buf.Bytes()),
			len: C.int64_t(len(buf.Bytes())),
		}
	}
	return cstr
}

func cBytes(v []byte) C.CGoBuffer {
	var cstr C.CGoBuffer
	if len(v) == 0 {
		cstr = C.CGoBuffer{}
	} else {
		cstr = C.CGoBuffer{
			ptr: C.CBytes(v),
			len: C.int64_t(len(v)),
		}
	}
	return cstr
}

func cString(v string) C.CGoBuffer {
	var cstr C.CGoBuffer
	if len(v) == 0 {
		cstr = C.CGoBuffer{}
	} else {
		cstr = C.CGoBuffer{
			ptr: C.CBytes([]byte(v)),
			len: C.int64_t(len(v)),
		}
	}
	return cstr
}

func goArray(buf C.CGoBuffer) []reflect.Value {
	defer C.free(buf.ptr)

	gorefs := make([]int64, buf.len/8)
	err := binary.Read(bytes.NewBuffer(C.GoBytes(buf.ptr, C.int(buf.len))), binary.LittleEndian, gorefs)
	if err != nil {
		panic(err)
	}

	rvs := []reflect.Value{}
	for _, i := range gorefs {
		rv := matchaGoGet(C.GoRef(i))
		rvs = append(rvs, rv)
	}
	return rvs
}

func goArray2(buf C.CGoBuffer) []*Value {
	defer C.free(buf.ptr)

	fgnRef := make([]int64, buf.len/8)
	err := binary.Read(bytes.NewBuffer(C.GoBytes(buf.ptr, C.int(buf.len))), binary.LittleEndian, fgnRef)
	if err != nil {
		panic(err)
	}

	rvs := []*Value{}
	for _, i := range fgnRef {
		rv := newValue(C.FgnRef(i))
		rvs = append(rvs, rv)
	}
	return rvs
}

func goString(buf C.CGoBuffer) string {
	defer C.free(buf.ptr)
	str := C.GoBytes(buf.ptr, C.int(buf.len))
	return string(str)
}

func goBytes(buf C.CGoBuffer) []byte {
	defer C.free(buf.ptr)
	return C.GoBytes(buf.ptr, C.int(buf.len))
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package testing

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"io/ioutil"
	"net"
	"net/http"
	"strconv"
	"strings"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/charm"
	"launchpad.net/juju-core/testing"
)

// MockStore provides a mock charm store implementation useful when testing.
type MockStore struct {
	mux            *http.ServeMux
	listener       net.Listener
	bundleBytes    []byte
	bundleSha256   string
	Downloads      []*charm.URL
	Authorizations []string

	charms map[string]int
}

// NewMockStore creates a mock charm store containing the specified charms.
func NewMockStore(c *gc.C, charms map[string]int) *MockStore {
	s := &MockStore{charms: charms}
	bytes, err := ioutil.ReadFile(testing.Charms.BundlePath(c.MkDir(), "dummy"))
	c.Assert(err, gc.IsNil)
	s.bundleBytes = bytes
	h := sha256.New()
	h.Write(bytes)
	s.bundleSha256 = hex.EncodeToString(h.Sum(nil))
	s.mux = http.NewServeMux()
	s.mux.HandleFunc("/charm-info", s.serveInfo)
	s.mux.HandleFunc("/charm-event", s.serveEvent)
	s.mux.HandleFunc("/charm/", s.serveCharm)
	lis, err := net.Listen("tcp", "127.0.0.1:0")
	c.Assert(err, gc.IsNil)
	s.listener = lis
	go http.Serve(s.listener, s)
	return s
}

// Close closes the mock store's socket.
func (s *MockStore) Close() {
	s.listener.Close()
}

// Address returns the URL used to make requests to the mock store.
func (s *MockStore) Address() string {
	return "http://" + s.listener.Addr().String()
}

// ServeHTTP implements http.ServeHTTP
func (s *MockStore) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	s.mux.ServeHTTP(w, r)
}

func (s *MockStore) serveInfo(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	response := map[string]*charm.InfoResponse{}
	for _, url := range r.Form["charms"] {
		cr := &charm.InfoResponse{}
		response[url] = cr
		charmURL := charm.MustParseURL(url)
		switch charmURL.Name {
		case "borken":
			cr.Errors = append(cr.Errors, "badness")
		case "terracotta":
			cr.Errors = append(cr.Errors, "cannot get revision")
		case "unwise":
			cr.Warnings = append(cr.Warnings, "foolishness")
			fallthrough
		default:
			if rev, ok := s.charms[charmURL.WithRevision(-1).String()]; ok {
				if charmURL.Revision == -1 {
					cr.Revision = rev
				} else {
					cr.Revision = charmURL.Revision
				}
				cr.Sha256 = s.bundleSha256
			} else {
				cr.Errors = append(cr.Errors, "entry not found")
			}
		}
	}
	data, err := json.Marshal(response)
	if err != nil {
		panic(err)
	}
	w.Header().Set("Content-Type", "application/json")
	_, err = w.Write(data)
	if err != nil {
		panic(err)
	}
}

func (s *MockStore) serveEvent(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	response := map[string]*charm.EventResponse{}
	for _, url := range r.Form["charms"] {
		digest := ""
		if i := strings.Index(url, "@"); i >= 0 {
			digest = url[i+1:]
			url = url[:i]
		}
		er := &charm.EventResponse{}
		response[url] = er
		if digest != "" && digest != "the-digest" {
			er.Kind = "not-found"
			er.Errors = []string{"entry not found"}
			continue
		}
		charmURL := charm.MustParseURL(url)
		switch charmURL.Name {
		case "borken":
			er.Kind = "publish-error"
			er.Errors = append(er.Errors, "badness")
		case "unwise":
			er.Warnings = append(er.Warnings, "foolishness")
			fallthrough
		default:
			if rev, ok := s.charms[charmURL.WithRevision(-1).String()]; ok {
				er.Kind = "published"
				er.Revision = rev
				er.Digest = "the-digest"
			} else {
				er.Kind = "not-found"
				er.Errors = []string{"entry not found"}
			}
		}
	}
	data, err := json.Marshal(response)
	if err != nil {
		panic(err)
	}
	w.Header().Set("Content-Type", "application/json")
	_, err = w.Write(data)
	if err != nil {
		panic(err)
	}
}

func (s *MockStore) serveCharm(w http.ResponseWriter, r *http.Request) {
	charmURL := charm.MustParseURL("cs:" + r.URL.Path[len("/charm/"):])
	s.Downloads = append(s.Downloads, charmURL)

	if auth := r.Header.Get("Authorization"); auth != "" {
		s.Authorizations = append(s.Authorizations, auth)
	}

	w.Header().Set("Connection", "close")
	w.Header().Set("Content-Type", "application/octet-stream")
	w.Header().Set("Content-Length", strconv.Itoa(len(s.bundleBytes)))
	_, err := w.Write(s.bundleBytes)
	if err != nil {
		panic(err)
	}
}
<commit_msg>Rework tests to account for upstream changes<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package testing

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"io/ioutil"
	"net"
	"net/http"
	"strconv"
	"strings"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/charm"
	"launchpad.net/juju-core/testing"
)

// MockStore provides a mock charm store implementation useful when testing.
type MockStore struct {
	mux            *http.ServeMux
	listener       net.Listener
	bundleBytes    []byte
	bundleSha256   string
	Downloads      []*charm.URL
	Authorizations []string

	charms map[string]int
}

// NewMockStore creates a mock charm store containing the specified charms.
func NewMockStore(c *gc.C, charms map[string]int) *MockStore {
	s := &MockStore{charms: charms}
	bytes, err := ioutil.ReadFile(testing.Charms.BundlePath(c.MkDir(), "dummy"))
	c.Assert(err, gc.IsNil)
	s.bundleBytes = bytes
	h := sha256.New()
	h.Write(bytes)
	s.bundleSha256 = hex.EncodeToString(h.Sum(nil))
	s.mux = http.NewServeMux()
	s.mux.HandleFunc("/charm-info", s.serveInfo)
	s.mux.HandleFunc("/charm-event", s.serveEvent)
	s.mux.HandleFunc("/charm/", s.serveCharm)
	lis, err := net.Listen("tcp", "127.0.0.1:0")
	c.Assert(err, gc.IsNil)
	s.listener = lis
	go http.Serve(s.listener, s)
	return s
}

// Close closes the mock store's socket.
func (s *MockStore) Close() {
	s.listener.Close()
}

// Address returns the URL used to make requests to the mock store.
func (s *MockStore) Address() string {
	return "http://" + s.listener.Addr().String()
}

// UpdateStoreRevision sets the revision of the specified charm to rev.
func (s *MockStore) UpdateStoreRevision(ch string, rev int) {
	s.charms[ch] = rev
}

// ServeHTTP implements http.ServeHTTP
func (s *MockStore) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	s.mux.ServeHTTP(w, r)
}

func (s *MockStore) serveInfo(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	response := map[string]*charm.InfoResponse{}
	for _, url := range r.Form["charms"] {
		cr := &charm.InfoResponse{}
		response[url] = cr
		charmURL := charm.MustParseURL(url)
		switch charmURL.Name {
		case "borken":
			cr.Errors = append(cr.Errors, "badness")
		case "terracotta":
			cr.Errors = append(cr.Errors, "cannot get revision")
		case "unwise":
			cr.Warnings = append(cr.Warnings, "foolishness")
			fallthrough
		default:
			if rev, ok := s.charms[charmURL.WithRevision(-1).String()]; ok {
				if charmURL.Revision == -1 {
					cr.Revision = rev
				} else {
					cr.Revision = charmURL.Revision
				}
				cr.Sha256 = s.bundleSha256
			} else {
				cr.Errors = append(cr.Errors, "entry not found")
			}
		}
	}
	data, err := json.Marshal(response)
	if err != nil {
		panic(err)
	}
	w.Header().Set("Content-Type", "application/json")
	_, err = w.Write(data)
	if err != nil {
		panic(err)
	}
}

func (s *MockStore) serveEvent(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	response := map[string]*charm.EventResponse{}
	for _, url := range r.Form["charms"] {
		digest := ""
		if i := strings.Index(url, "@"); i >= 0 {
			digest = url[i+1:]
			url = url[:i]
		}
		er := &charm.EventResponse{}
		response[url] = er
		if digest != "" && digest != "the-digest" {
			er.Kind = "not-found"
			er.Errors = []string{"entry not found"}
			continue
		}
		charmURL := charm.MustParseURL(url)
		switch charmURL.Name {
		case "borken":
			er.Kind = "publish-error"
			er.Errors = append(er.Errors, "badness")
		case "unwise":
			er.Warnings = append(er.Warnings, "foolishness")
			fallthrough
		default:
			if rev, ok := s.charms[charmURL.WithRevision(-1).String()]; ok {
				er.Kind = "published"
				er.Revision = rev
				er.Digest = "the-digest"
			} else {
				er.Kind = "not-found"
				er.Errors = []string{"entry not found"}
			}
		}
	}
	data, err := json.Marshal(response)
	if err != nil {
		panic(err)
	}
	w.Header().Set("Content-Type", "application/json")
	_, err = w.Write(data)
	if err != nil {
		panic(err)
	}
}

func (s *MockStore) serveCharm(w http.ResponseWriter, r *http.Request) {
	charmURL := charm.MustParseURL("cs:" + r.URL.Path[len("/charm/"):])
	s.Downloads = append(s.Downloads, charmURL)

	if auth := r.Header.Get("Authorization"); auth != "" {
		s.Authorizations = append(s.Authorizations, auth)
	}

	w.Header().Set("Connection", "close")
	w.Header().Set("Content-Type", "application/octet-stream")
	w.Header().Set("Content-Length", strconv.Itoa(len(s.bundleBytes)))
	_, err := w.Write(s.bundleBytes)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package MySQLProtocol

import "testing"
import "github.com/stretchr/testify/assert"

func Test_Packet_HandshakeV10(t *testing.T) {
	var values = []struct {
		packet  Proto
		context Context
	}{
		{packet: Proto{data: StringToPacket(`
36 00 00 00 0a 35 2e 35    2e 32 2d 6d 32 00 0b 00    6....5.5.2-m2...
00 00 64 76 48 40 49 2d    43 4a 00 ff f7 08 02 00    ..dvH@I-CJ......
00 00 00 00 00 00 00 00    00 00 00 00 00 2a 34 64    .............*4d
7c 63 5a 77 6b 34 5e 5d    3a 00                      |cZwk4^]:.
`)}, context: Context{}},

		{packet: Proto{data: StringToPacket(`
50 00 00 00 0a 35 2e 36    2e 34 2d 6d 37 2d 6c 6f    P....5.6.4-m7-lo
67 00 56 0a 00 00 52 42    33 76 7a 26 47 72 00 ff    g.V...RB3vz&Gr..
ff 08 02 00 0f c0 15 00    00 00 00 00 00 00 00 00    ................
00 2b 79 44 26 2f 5a 5a    33 30 35 5a 47 00 6d 79    .+yD&/ZZ305ZG.my
73 71 6c 5f 6e 61 74 69    76 65 5f 70 61 73 73 77    sql_native_passw
6f 72 64 00                                           ord
`)}, context: Context{}},

		{packet: Proto{data: StringToPacket(`
36 00 00 00 0a 35 2e 35    2e 32 2d 6d 32 00 52 00    6....5.5.2-m2.R.
00 00 22 3d 4e 50 29 75    39 56 00 ff ff 08 02 00    .."=NP)u9V......
00 00 00 00 00 00 00 00    00 00 00 00 00 29 64 40    .............)d@
52 5c 55 78 7a 7c 21 29    4b 00                      R\Uxz|!)K.
`)}, context: Context{}},
	}
	var pkt Packet_HandshakeV10

	for _, value := range values {
		pkt = Packet_HandshakeV10{}
		pkt.FromPacket(value.context, value.packet)
		assert.Equal(t, pkt.ToPacket(value.context), value.packet.data, "")
	}
}

func Benchmark_Packet_HandshakeV10_FromPacket(b *testing.B) {
	context := Context{capability: CLIENT_PROTOCOL_41}
	var pkt Packet_HandshakeV10
	var packet = Proto{data: StringToPacket(`
36 00 00 00 0a 35 2e 35    2e 32 2d 6d 32 00 0b 00    6....5.5.2-m2...
00 00 64 76 48 40 49 2d    43 4a 00 ff f7 08 02 00    ..dvH@I-CJ......
00 00 00 00 00 00 00 00    00 00 00 00 00 2a 34 64    .............*4d
7c 63 5a 77 6b 34 5e 5d    3a 00                      |cZwk4^]:.
`)}
	for i := 0; i < b.N; i++ {
		pkt = Packet_HandshakeV10{}
		packet.offset = 0
		pkt.FromPacket(context, packet)
	}
}

func Benchmark_Packet_HandshakeV10_GetPacketSize(b *testing.B) {
	context := Context{capability: CLIENT_PROTOCOL_41}
	pkt := Packet_HandshakeV10{}
	var packet = Proto{data: StringToPacket(`
36 00 00 00 0a 35 2e 35    2e 32 2d 6d 32 00 0b 00    6....5.5.2-m2...
00 00 64 76 48 40 49 2d    43 4a 00 ff f7 08 02 00    ..dvH@I-CJ......
00 00 00 00 00 00 00 00    00 00 00 00 00 2a 34 64    .............*4d
7c 63 5a 77 6b 34 5e 5d    3a 00                      |cZwk4^]:.
`)}
	pkt.FromPacket(context, packet)
	for i := 0; i < b.N; i++ {
		pkt.GetPacketSize(context)
	}
}

func Benchmark_Packet_HandshakeV10_ToPacket(b *testing.B) {
	context := Context{capability: CLIENT_PROTOCOL_41}
	pkt := Packet_HandshakeV10{}
	var packet = Proto{data: StringToPacket(`
36 00 00 00 0a 35 2e 35    2e 32 2d 6d 32 00 0b 00    6....5.5.2-m2...
00 00 64 76 48 40 49 2d    43 4a 00 ff f7 08 02 00    ..dvH@I-CJ......
00 00 00 00 00 00 00 00    00 00 00 00 00 2a 34 64    .............*4d
7c 63 5a 77 6b 34 5e 5d    3a 00                      |cZwk4^]:.
`)}
	pkt.FromPacket(context, packet)
	for i := 0; i < b.N; i++ {
		pkt.ToPacket(context)
	}
}
<commit_msg>Short packets<commit_after>package MySQLProtocol

import "testing"
import "github.com/stretchr/testify/assert"

var Packet_HandshakeV10_test_packets = []struct {
	packet  Proto
	context Context
}{
	{packet: Proto{data: StringToPacket(`
36 00 00 00 0a 35 2e 35    2e 32 2d 6d 32 00 0b 00    6....5.5.2-m2...
00 00 64 76 48 40 49 2d    43 4a 00 ff f7 08 02 00    ..dvH@I-CJ......
00 00 00 00 00 00 00 00    00 00 00 00 00 2a 34 64    .............*4d
7c 63 5a 77 6b 34 5e 5d    3a 00                      |cZwk4^]:.
`)}, context: Context{}},

	{packet: Proto{data: StringToPacket(`
50 00 00 00 0a 35 2e 36    2e 34 2d 6d 37 2d 6c 6f    P....5.6.4-m7-lo
67 00 56 0a 00 00 52 42    33 76 7a 26 47 72 00 ff    g.V...RB3vz&Gr..
ff 08 02 00 0f c0 15 00    00 00 00 00 00 00 00 00    ................
00 2b 79 44 26 2f 5a 5a    33 30 35 5a 47 00 6d 79    .+yD&/ZZ305ZG.my
73 71 6c 5f 6e 61 74 69    76 65 5f 70 61 73 73 77    sql_native_passw
6f 72 64 00                                           ord
`)}, context: Context{}},

	{packet: Proto{data: StringToPacket(`
36 00 00 00 0a 35 2e 35    2e 32 2d 6d 32 00 52 00    6....5.5.2-m2.R.
00 00 22 3d 4e 50 29 75    39 56 00 ff ff 08 02 00    .."=NP)u9V......
00 00 00 00 00 00 00 00    00 00 00 00 00 29 64 40    .............)d@
52 5c 55 78 7a 7c 21 29    4b 00                      R\Uxz|!)K.
`)}, context: Context{}},

    {packet: Proto{data: StringToPacket(`
19 00 00 00 0a 35 2e 35    2e 32 2d 6d 32 00 0b 00    6....5.5.2-m2...
00 00 64 76 48 40 49 2d    43 4a 00 ff f7             ..dvH@I-CJ...
`)}, context: Context{}},

    {packet: Proto{data: StringToPacket(`
2a 00 00 00 0a 35 2e 35    2e 32 2d 6d 32 00 0b 00    *....5.5.2-m2...
00 00 64 76 48 40 49 2d    43 4a 00 ff f7 08 02 00    ..dvH@I-CJ......
00 00 00 00 00 00 00 00    00 00 00 00 00 00          ..............
`)}, context: Context{}},

}

func Test_Packet_HandshakeV10(t *testing.T) {
	var pkt Packet_HandshakeV10

	for _, value := range Packet_HandshakeV10_test_packets {
		pkt = Packet_HandshakeV10{}
		pkt.FromPacket(value.context, value.packet)
		if ! assert.Equal(t, pkt.ToPacket(value.context), value.packet.data, "") {
            DumpPacket(value.packet.data)
            DumpPacket(pkt.ToPacket(value.context))
        }
	}
}

func Benchmark_Packet_HandshakeV10_FromPacket(b *testing.B) {
	context := Packet_HandshakeV10_test_packets[0].context
	packet := Packet_HandshakeV10_test_packets[0].packet
	pkt := Packet_HandshakeV10{}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		packet.offset = 0
		pkt.FromPacket(context, packet)
	}
}

func Benchmark_Packet_HandshakeV10_GetPacketSize(b *testing.B) {
	context := Packet_HandshakeV10_test_packets[0].context
	packet := Packet_HandshakeV10_test_packets[0].packet
	pkt := Packet_HandshakeV10{}
	pkt.FromPacket(context, packet)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		pkt.GetPacketSize(context)
	}
}

func Benchmark_Packet_HandshakeV10_ToPacket(b *testing.B) {
	context := Packet_HandshakeV10_test_packets[0].context
	packet := Packet_HandshakeV10_test_packets[0].packet
	pkt := Packet_HandshakeV10{}
	pkt.FromPacket(context, packet)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		pkt.ToPacket(context)
	}
}
<|endoftext|>


<commit_before>package browsersteps

import (
	"encoding/json"
	"fmt"
	"log"
	"net"
	"os"
	"testing"
	"time"

	"net/http"
	"net/http/httptest"
	"net/url"

	"github.com/DATA-DOG/godog"
	"github.com/tebeka/selenium"
)

func iWaitFor(amount int, unit string) error {
	u := time.Second
	fmt.Printf("Waiting for %d %s", amount, unit)
	time.Sleep(u * time.Duration(amount))
	return nil
}

func FeatureContext(s *godog.Suite) {
	s.Step(`^I wait for (\d+) (milliseconds|millisecond|seconds|second)$`, iWaitFor)

	// selenium.SetDebug(true)
	capabilities := selenium.Capabilities{"browserName": "chrome"}
	capEnv := os.Getenv("SELENIUM_CAPABILITIES")
	if capEnv != "" {
		err := json.Unmarshal([]byte(capEnv), &capabilities)
		if err != nil {
			log.Panic(err)
		}
	}

	bs, err := NewBrowserSteps(s, capabilities, os.Getenv("SELENIUM_URL"))
	if err != nil {
		log.Panic(err.Error())
	}

	var server *httptest.Server
	s.BeforeSuite(func() {
		server = httptest.NewUnstartedServer(http.FileServer(http.Dir("./public")))
		listenAddress := os.Getenv("SERVER_LISTEN")
		if listenAddress != "" {
			var err error
			server.Listener, err = net.Listen("tcp4", listenAddress)
			if err != nil {
				log.Fatal(err)
			}
		}
		server.Start()
		u, err := url.Parse(server.URL)
		if err != nil {
			log.Panic(err.Error())
		}
		bs.SetBaseURL(u)
	})

	s.AfterSuite(func() {
		if server != nil {
			server.Close()
			server = nil
		}
	})
}

func TestMain(m *testing.M) {
	status := godog.Run("browsersteps", FeatureContext)
	os.Exit(status)
}
<commit_msg>DEBUG environment variable<commit_after>package browsersteps

import (
	"encoding/json"
	"fmt"
	"log"
	"net"
	"os"
	"strconv"
	"testing"
	"time"

	"net/http"
	"net/http/httptest"
	"net/url"

	"github.com/DATA-DOG/godog"
	"github.com/tebeka/selenium"
)

func iWaitFor(amount int, unit string) error {
	u := time.Second
	fmt.Printf("Waiting for %d %s", amount, unit)
	time.Sleep(u * time.Duration(amount))
	return nil
}

func FeatureContext(s *godog.Suite) {
	s.Step(`^I wait for (\d+) (milliseconds|millisecond|seconds|second)$`, iWaitFor)

	debug := os.Getenv("DEBUG")
	if debug != "" {
		val, err := strconv.ParseBool(debug)
		if err == nil {
			selenium.SetDebug(val)
		}
	}

	capabilities := selenium.Capabilities{"browserName": "chrome"}
	capEnv := os.Getenv("SELENIUM_CAPABILITIES")
	if capEnv != "" {
		err := json.Unmarshal([]byte(capEnv), &capabilities)
		if err != nil {
			log.Panic(err)
		}
	}

	bs, err := NewBrowserSteps(s, capabilities, os.Getenv("SELENIUM_URL"))
	if err != nil {
		log.Panic(err.Error())
	}

	var server *httptest.Server
	s.BeforeSuite(func() {
		server = httptest.NewUnstartedServer(http.FileServer(http.Dir("./public")))
		listenAddress := os.Getenv("SERVER_LISTEN")
		if listenAddress != "" {
			var err error
			server.Listener, err = net.Listen("tcp4", listenAddress)
			if err != nil {
				log.Fatal(err)
			}
		}
		server.Start()
		u, err := url.Parse(server.URL)
		if err != nil {
			log.Panic(err.Error())
		}
		bs.SetBaseURL(u)
	})

	s.AfterSuite(func() {
		if server != nil {
			server.Close()
			server = nil
		}
	})
}

func TestMain(m *testing.M) {
	status := godog.Run("browsersteps", FeatureContext)
	os.Exit(status)
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2016 Maciek Borzecki <maciek.borzecki@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package main

import (
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sync"
	"time"

	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
)

type ReadSeekCloser interface {
	io.ReadSeeker
	io.Closer
}

type CacheStats struct {
	Hit  int
	Miss int
}

type CacheCount struct {
	// Items is the count of all items (files) in the cache
	Items uint64
	// TotalSize is the aggregate size of all items in bytes
	TotalSize uint64
}

type Cache struct {
	Dir       string
	dirLock   sync.Mutex
	stats     CacheStats
	statsLock sync.Mutex
}

func (c *Cache) getCachePath(name string) string {
	cpath := path.Join(c.Dir, name)
	log.Debugf("cache path: %v", cpath)
	return cpath
}

func (c *Cache) Get(name string) (ReadSeekCloser, int64, error) {
	c.dirLock.Lock()
	defer c.dirLock.Unlock()

	f, err := os.Open(c.getCachePath(name))
	if err != nil {
		if os.IsNotExist(err) {
			c.miss()
		}
		log.Errorf("cache get error: %v", err)
		return nil, 0, err
	}

	c.hit()

	fi, err := f.Stat()
	if err != nil {
		log.Errorf("file %s stat failed: %v", f.Name(), err)
		f.Close()
		return nil, 0, err
	}

	return f, fi.Size(), nil
}

func (c *Cache) Put(name string) (*CacheTemporaryObject, error) {
	c.dirLock.Lock()
	defer c.dirLock.Unlock()

	cpath := c.getCachePath(name)

	if err := os.MkdirAll(path.Dir(cpath), 0700); err != nil {
		return nil, err
	}

	f, err := ioutil.TempFile(path.Dir(cpath), path.Base(cpath)+".part.")
	if err != nil {
		log.Errorf("cache put for %v error: %v", cpath, err)
		return nil, err
	}

	ct := CacheTemporaryObject{
		File:       f,
		targetName: cpath,
		curName:    f.Name(),
	}
	return &ct, nil
}

func (c *Cache) Stats() CacheStats {
	c.statsLock.Lock()
	defer c.statsLock.Unlock()
	return c.stats
}

func (c *Cache) hit() {
	c.statsLock.Lock()
	defer c.statsLock.Unlock()
	c.stats.Hit++
}

func (c *Cache) miss() {
	c.statsLock.Lock()
	defer c.statsLock.Unlock()
	c.stats.Miss++
}

func (c *Cache) Count() (CacheCount, error) {
	c.dirLock.Lock()
	defer c.dirLock.Unlock()

	count := CacheCount{}
	walkCount := func(name string, fi os.FileInfo, err error) error {
		if err != nil {
			return errors.Wrapf(err, "cannot process path %v", name)
		}
		if fi.IsDir() {
			return nil
		}
		count.Items++
		count.TotalSize += uint64(fi.Size())
		return nil
	}
	err := filepath.Walk(c.Dir, walkCount)
	return count, err
}

type PurgeSelector struct {
	OlderThan time.Time
}

func (c *Cache) Purge(what PurgeSelector) (removed uint64, err error) {
	c.dirLock.Lock()
	defer c.dirLock.Unlock()

	log.Infof("cache purge: older than %v", what.OlderThan)

	var rmError error
	walkPurgeSelected := func(name string, fi os.FileInfo, err error) error {
		if err != nil {
			return errors.Wrapf(err, "cannot process path %v", name)
		}
		if fi.IsDir() {
			return nil
		}
		remove := true
		if what.OlderThan != 0 && now.Sub(fi.ModTime()) < what.OlderThan {
			remove = false
		}
		if remove {
			log.Infof("removing %v", name)
			err := os.Remove(name)
			if err != nil && rmError == nil {
				rmError = errors.Wrapf(err, "cannot remove entry %v", name)
			}
			if err == nil {
				removed++
			}
		}
		return nil
	}
	err = filepath.Walk(c.Dir, walkPurgeSelected)
	return removed, err
}

type CacheTemporaryObject struct {
	*os.File
	targetName string
	curName    string
	aborted    bool
}

func (ct *CacheTemporaryObject) Commit() error {
	if ct.aborted {
		return nil
	}

	if err := ct.Close(); err != nil {
		return err
	}

	if err := os.Rename(ct.curName, ct.targetName); err != nil {
		log.Errorf("rename %v -> %v failed: %v",
			ct.curName, ct.targetName, err)
		return err
	}
	log.Debugf("commited cache entry %v to %v", ct.curName, ct.targetName)
	return nil
}

func (ct *CacheTemporaryObject) Abort() error {
	log.Debugf("discard entry %v", ct.curName)
	ct.aborted = true

	if err := ct.Close(); err != nil {
		return err
	}

	if err := os.Remove(ct.curName); err != nil {
		return err
	}
	return nil
}
<commit_msg>cache: record last cache purge time<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2016 Maciek Borzecki <maciek.borzecki@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package main

import (
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sync"
	"time"

	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
)

type ReadSeekCloser interface {
	io.ReadSeeker
	io.Closer
}

type CacheStats struct {
	Hit       int
	Miss      int
	LastPurge time.Time
}

type CacheCount struct {
	// Items is the count of all items (files) in the cache
	Items uint64
	// TotalSize is the aggregate size of all items in bytes
	TotalSize uint64
}

type Cache struct {
	Dir       string
	dirLock   sync.Mutex
	stats     CacheStats
	statsLock sync.Mutex
}

func (c *Cache) getCachePath(name string) string {
	cpath := path.Join(c.Dir, name)
	log.Debugf("cache path: %v", cpath)
	return cpath
}

func (c *Cache) Get(name string) (ReadSeekCloser, int64, error) {
	c.dirLock.Lock()
	defer c.dirLock.Unlock()

	f, err := os.Open(c.getCachePath(name))
	if err != nil {
		if os.IsNotExist(err) {
			c.miss()
		}
		log.Errorf("cache get error: %v", err)
		return nil, 0, err
	}

	c.hit()

	fi, err := f.Stat()
	if err != nil {
		log.Errorf("file %s stat failed: %v", f.Name(), err)
		f.Close()
		return nil, 0, err
	}

	return f, fi.Size(), nil
}

func (c *Cache) Put(name string) (*CacheTemporaryObject, error) {
	c.dirLock.Lock()
	defer c.dirLock.Unlock()

	cpath := c.getCachePath(name)

	if err := os.MkdirAll(path.Dir(cpath), 0700); err != nil {
		return nil, err
	}

	f, err := ioutil.TempFile(path.Dir(cpath), path.Base(cpath)+".part.")
	if err != nil {
		log.Errorf("cache put for %v error: %v", cpath, err)
		return nil, err
	}

	ct := CacheTemporaryObject{
		File:       f,
		targetName: cpath,
		curName:    f.Name(),
	}
	return &ct, nil
}

func (c *Cache) Stats() CacheStats {
	c.statsLock.Lock()
	defer c.statsLock.Unlock()
	return c.stats
}

func (c *Cache) hit() {
	c.statsLock.Lock()
	defer c.statsLock.Unlock()
	c.stats.Hit++
}

func (c *Cache) miss() {
	c.statsLock.Lock()
	defer c.statsLock.Unlock()
	c.stats.Miss++
}

func (c *Cache) Count() (CacheCount, error) {
	c.dirLock.Lock()
	defer c.dirLock.Unlock()

	count := CacheCount{}
	walkCount := func(name string, fi os.FileInfo, err error) error {
		if err != nil {
			return errors.Wrapf(err, "cannot process path %v", name)
		}
		if fi.IsDir() {
			return nil
		}
		count.Items++
		count.TotalSize += uint64(fi.Size())
		return nil
	}
	err := filepath.Walk(c.Dir, walkCount)
	return count, err
}

type PurgeSelector struct {
	OlderThan time.Duration
}

func (c *Cache) Purge(what PurgeSelector) (removed uint64, err error) {
	c.dirLock.Lock()
	defer c.dirLock.Unlock()

	now := time.Now()
	c.stats.LastPurge = now

	log.Infof("cache purge: older than %v", what.OlderThan)

	var rmError error
	walkPurgeSelected := func(name string, fi os.FileInfo, err error) error {
		if err != nil {
			return errors.Wrapf(err, "cannot process path %v", name)
		}
		if fi.IsDir() {
			return nil
		}
		remove := true
		if what.OlderThan != 0 && now.Sub(fi.ModTime()) < what.OlderThan {
			remove = false
		}
		if remove {
			log.Infof("removing %v", name)
			err := os.Remove(name)
			if err != nil && rmError == nil {
				rmError = errors.Wrapf(err, "cannot remove entry %v", name)
			}
			if err == nil {
				removed++
			}
		}
		return nil
	}
	err = filepath.Walk(c.Dir, walkPurgeSelected)
	return removed, err
}

type CacheTemporaryObject struct {
	*os.File
	targetName string
	curName    string
	aborted    bool
}

func (ct *CacheTemporaryObject) Commit() error {
	if ct.aborted {
		return nil
	}

	if err := ct.Close(); err != nil {
		return err
	}

	if err := os.Rename(ct.curName, ct.targetName); err != nil {
		log.Errorf("rename %v -> %v failed: %v",
			ct.curName, ct.targetName, err)
		return err
	}
	log.Debugf("commited cache entry %v to %v", ct.curName, ct.targetName)
	return nil
}

func (ct *CacheTemporaryObject) Abort() error {
	log.Debugf("discard entry %v", ct.curName)
	ct.aborted = true

	if err := ct.Close(); err != nil {
		return err
	}

	if err := os.Remove(ct.curName); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package engine

import (
	"fmt"
	log "github.com/jbdalido/smg/Godeps/_workspace/src/github.com/Sirupsen/logrus"
	"github.com/jbdalido/smg/Godeps/_workspace/src/gopkg.in/yaml.v1"
	"github.com/jbdalido/smg/utils"
	"os"
	"os/user"
	"path"
	"strings"
)

type Config struct {
	Repository string  `yaml:"repository"`
	Docker     *Docker `yaml:"docker"`
}

type Repository struct {
	Login    string `yaml:"login"`
	Password string `yaml:"password"`
}

func NewConfig(filePath string, h string) (*Config, error) {
	c := new(Config)
	dockerCfgPath := "~/"
	usr, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	if strings.Contains(filePath, "~") {
		filePath = strings.Replace(filePath, "~", usr.HomeDir, 1)
	}
	dockerCfgPath = strings.Replace(dockerCfgPath, "~", usr.HomeDir, 1)
	datas, err := utils.OpenAndReadFile(filePath)
	if err != nil {
		c := &Config{
			Repository: "",
			Docker: &Docker{
				Host: h,
			},
		}
		// TODO : find a better way to look
		host := os.Getenv("DOCKER_HOST")
		if h == "" && host != "" {
			// Properly handle the boot2Docker host
			if host != "" {
				log.Debugf("Docker detected at %s", host)
				// Get env variables
				hostTLS := os.Getenv("DOCKER_TLS_VERIFY")
				hostCertPath := os.Getenv("DOCKER_CERT_PATH")

				// Check if they exists
				if hostTLS != "" && hostCertPath != "" {
					log.Debugf("Docker Cert Path at %s", hostCertPath)
					log.Debugf("Docker SSL Mode %s", hostTLS)
					if hostTLS == "1" {
						// Setup path for key and certificates
						key := path.Clean(hostCertPath) + "/key.pem"
						cert := path.Clean(hostCertPath) + "/cert.pem"
						ca := path.Clean(hostCertPath) + "/ca.pem"
						// Test the files
						_, err := utils.OpenFile(key)
						if err != nil {
							return nil, fmt.Errorf("Can't read your boot2docker key: %s", key)
						}
						_, err = utils.OpenFile(cert)
						if err != nil {
							return nil, fmt.Errorf("Can't read your boot2docker cert: %s", cert)
						}
						_, err = utils.OpenFile(ca)
						if err != nil {
							return nil, fmt.Errorf("Can't read your boot2docker Ca cert: %s", ca)
						}
						c.Docker.Cert = cert
						c.Docker.CA = ca
						c.Docker.Key = key
						c.Docker.Host = strings.Replace(host, "tcp", "https", 1)
					}
				}

			}
		} else {
			c.Docker.Host = "unix:///var/run/docker.sock"
		}
		return c, nil
	}
	// Parse the file to the application
	if err := yaml.Unmarshal(datas, &c); err != nil {
		return nil, fmt.Errorf("Error processing %s: %s", filePath, err)
	}
	if c.Repository != "" {
		c.Repository = strings.Replace(c.Repository, "/", "", 1)
	}
	return c, nil
}
<commit_msg>fixed #2<commit_after>package engine

import (
	"fmt"
	log "github.com/jbdalido/smg/Godeps/_workspace/src/github.com/Sirupsen/logrus"
	"github.com/jbdalido/smg/Godeps/_workspace/src/gopkg.in/yaml.v1"
	"github.com/jbdalido/smg/utils"
	"os"
	"path"
	"strings"
)

type Config struct {
	Repository string  `yaml:"repository"`
	Docker     *Docker `yaml:"docker"`
}

type Repository struct {
	Login    string `yaml:"login"`
	Password string `yaml:"password"`
}

func NewConfig(filePath string, h string) (*Config, error) {
	c := new(Config)

	if strings.Contains(filePath, "~") {
		home := os.Getenv("HOME")
		if home == "" {
			log.Fatalf("Can't find home env variable")
		}
		filePath = strings.Replace(filePath, "~", home, 1)
	}

	datas, err := utils.OpenAndReadFile(filePath)
	if err != nil {
		c := &Config{
			Repository: "",
			Docker: &Docker{
				Host: h,
			},
		}
		// TODO : find a better way to look
		host := os.Getenv("DOCKER_HOST")
		if h == "" && host != "" {
			// Properly handle the boot2Docker host
			if host != "" {
				log.Debugf("Docker detected at %s", host)
				// Get env variables
				hostTLS := os.Getenv("DOCKER_TLS_VERIFY")
				hostCertPath := os.Getenv("DOCKER_CERT_PATH")

				// Check if they exists
				if hostTLS != "" && hostCertPath != "" {

					log.Debugf("Docker Cert Path at %s", hostCertPath)
					log.Debugf("Docker SSL Mode %s", hostTLS)

					if hostTLS == "1" {

						// Setup path for key and certificates
						key := path.Clean(hostCertPath) + "/key.pem"
						cert := path.Clean(hostCertPath) + "/cert.pem"
						ca := path.Clean(hostCertPath) + "/ca.pem"

						// Test the files
						_, err := utils.OpenFile(key)
						if err != nil {
							return nil, fmt.Errorf("Can't read your boot2docker key: %s", key)
						}

						_, err = utils.OpenFile(cert)
						if err != nil {
							return nil, fmt.Errorf("Can't read your boot2docker cert: %s", cert)
						}

						_, err = utils.OpenFile(ca)
						if err != nil {
							return nil, fmt.Errorf("Can't read your boot2docker Ca cert: %s", ca)
						}

						c.Docker.Cert = cert
						c.Docker.CA = ca
						c.Docker.Key = key
						c.Docker.Host = strings.Replace(host, "tcp", "https", 1)
					}
				}

			}
		} else {
			c.Docker.Host = "unix:///var/run/docker.sock"
		}
		return c, nil
	}
	// Parse the file to the application
	if err := yaml.Unmarshal(datas, &c); err != nil {
		return nil, fmt.Errorf("Error processing %s: %s", filePath, err)
	}
	if c.Repository != "" {
		c.Repository = strings.Replace(c.Repository, "/", "", 1)
	}
	return c, nil
}
<|endoftext|>


<commit_before>package popularpost

import (
	"fmt"
	"socialapi/config"
	"socialapi/models"
	"time"

	"github.com/jinzhu/now"
	"github.com/koding/logging"
	"github.com/koding/redis"
	"github.com/streadway/amqp"
)

var (
	PopularPostKeyName = "popularpost"
	KeyExistsRegistry  = map[string]bool{}
)

type Controller struct {
	log   logging.Logger
	redis *redis.RedisSession
}

func (t *Controller) DefaultErrHandler(delivery amqp.Delivery, err error) bool {
	if delivery.Redelivered {
		t.log.Error("Redelivered message gave error again, putting to maintenance queue", err)
		delivery.Ack(false)
		return true
	}

	t.log.Error("an error occured putting message back to queue", err)
	delivery.Nack(false, true)
	return false
}

func New(log logging.Logger, redis *redis.RedisSession) *Controller {
	return &Controller{
		log:   log,
		redis: redis,
	}
}

func (f *Controller) InteractionSaved(i *models.Interaction) error {
	return f.handleInteraction(1, i)
}

func (f *Controller) InteractionDeleted(i *models.Interaction) error {
	return f.handleInteraction(-1, i)
}

func (f *Controller) handleInteraction(incrementCount int, i *models.Interaction) error {
	cm, err := models.ChannelMessageById(i.MessageId)
	if err != nil {
		return err
	}

	c, err := models.ChannelById(cm.InitialChannelId)
	if err != nil {
		return err
	}

	if notEligibleForPopularPost(c, cm) {
		f.log.Error(fmt.Sprintf("Not eligible Interaction Id:%d", i.Id))
		return nil
	}

	keyname := &KeyName{
		GroupName: c.GroupName, ChannelName: c.Name,
		Time: cm.CreatedAt,
	}

	err = f.saveToDailyBucket(keyname, incrementCount, i.MessageId)
	if err != nil {
		return err
	}

	difference := int(i.CreatedAt.Sub(cm.CreatedAt).Hours()/24) + 1
	weight := 1 / float64(difference) * float64(incrementCount)

	keyname = &KeyName{
		GroupName: c.GroupName, ChannelName: c.Name,
		Time: time.Now().UTC(),
	}

	err = f.saveToSevenDayBucket(keyname, weight, i.MessageId)
	if err != nil {
		return err
	}

	return nil
}

func (f *Controller) saveToDailyBucket(k *KeyName, inc int, id int64) error {
	key := k.Today()

	_, err := f.redis.SortedSetIncrBy(key, inc, id)
	if err != nil {
		return err
	}

	score, err := f.redis.SortedSetScore(key, id)
	if score <= 0 {
		_, err := f.redis.SortedSetRem(key, id)
		if err != nil {
			return err
		}
	}

	return err
}

func (f *Controller) saveToSevenDayBucket(k *KeyName, inc float64, id int64) error {
	key := k.Weekly()

	_, ok := KeyExistsRegistry[key]
	if !ok {
		exists := f.redis.Exists(key)
		if !exists {
			err := f.createSevenDayBucket(k)
			if err != nil {
				return err
			}
		}

		KeyExistsRegistry[key] = true

		return nil
	}

	_, err := f.redis.SortedSetIncrBy(key, inc, id)
	if err != nil {
		return err
	}

	score, err := f.redis.SortedSetScore(key, id)
	if score <= 0 {
		_, err := f.redis.SortedSetRem(key, id)
		if err != nil {
			return err
		}
	}

	return nil
}

func (f *Controller) createSevenDayBucket(k *KeyName) error {
	keys, weights := []interface{}{}, []interface{}{}

	from := getStartOfDay(k.Time)
	aggregate := "SUM"

	for i := 0; i <= 6; i++ {
		currentDate := getDaysAgo(from, i)
		keys = append(keys, k.Before(currentDate))

		// add by 1 to prevent divide by 0 errors
		weight := float64(i + 1)
		weights = append(weights, float64(1/weight))
	}

	_, err := f.redis.SortedSetsUnion(k.Weekly(), keys, weights, aggregate)

	return err
}

func PopularPostKey(groupName, channelName string, current time.Time) string {
	name := KeyName{
		GroupName: groupName, ChannelName: channelName,
		Time: current.UTC(),
	}

	return name.Weekly()
}

//----------------------------------------------------------
// KeyName
//----------------------------------------------------------

type KeyName struct {
	GroupName, ChannelName string
	Time                   time.Time
}

func (k *KeyName) Today() string {
	return k.do(getStartOfDay(k.Time))
}

func (k *KeyName) Before(t time.Time) string {
	return k.do(t)
}

func (k *KeyName) Weekly() string {
	current := getStartOfDay(k.Time.UTC())
	sevenDaysAgo := getDaysAgo(current, 7).UTC().Unix()

	return fmt.Sprintf("%s-%d", k.do(current), sevenDaysAgo)
}

func (k *KeyName) do(t time.Time) string {
	return fmt.Sprintf("%s:%s:%s:%s:%d",
		config.MustGet().Environment, k.GroupName, PopularPostKeyName,
		k.ChannelName, t.UTC().Unix(),
	)
}

//----------------------------------------------------------
// helpers
//----------------------------------------------------------

func notEligibleForPopularPost(c *models.Channel, cm *models.ChannelMessage) bool {
	if c.MetaBits.Is(models.Troll) {
		return true
	}

	if c.PrivacyConstant != models.Channel_PRIVACY_PUBLIC {
		return true
	}

	if cm.MetaBits.Is(models.Troll) {
		return true
	}

	if cm.TypeConstant != models.ChannelMessage_TYPE_POST {
		return true
	}

	if createdMoreThan7DaysAgo(cm.CreatedAt) {
		return true
	}

	return false
}

//----------------------------------------------------------
// Time helpers
//----------------------------------------------------------

func createdMoreThan7DaysAgo(t time.Time) bool {
	t = t.UTC()
	delta := time.Now().Sub(t)

	return delta.Hours()/24 > 7
}

func getStartOfDay(t time.Time) time.Time {
	t = t.UTC()
	return now.New(t).BeginningOfDay()
}

func getDaysAgo(t time.Time, days int) time.Time {
	t = t.UTC()
	daysAgo := -time.Hour * 24 * time.Duration(days)

	return t.Add(daysAgo)
}

//----------------------------------------------------------
func (t *Controller) CreateKeyAtStartOfDay(groupName, channelName string) {
	endOfDay := now.EndOfDay().UTC()
	difference := time.Now().UTC().Sub(endOfDay)

	<-time.After(difference)

	keyname := &KeyName{
		GroupName: groupName, ChannelName: channelName,
		Time: time.Now().UTC(),
	}

	t.createSevenDayBucket(keyname)
}

func (t *Controller) ResetRegistry() {
	KeyExistsRegistry = map[string]bool{}
}
<commit_msg>popularpost: save weight in 1 point decimal<commit_after>package popularpost

import (
	"fmt"
	"socialapi/config"
	"socialapi/models"
	"time"

	"github.com/jinzhu/now"
	"github.com/koding/logging"
	"github.com/koding/redis"
	"github.com/streadway/amqp"
)

var (
	PopularPostKeyName = "popularpost"
	KeyExistsRegistry  = map[string]bool{}
)

type Controller struct {
	log   logging.Logger
	redis *redis.RedisSession
}

func (t *Controller) DefaultErrHandler(delivery amqp.Delivery, err error) bool {
	if delivery.Redelivered {
		t.log.Error("Redelivered message gave error again, putting to maintenance queue", err)
		delivery.Ack(false)
		return true
	}

	t.log.Error("an error occured putting message back to queue", err)
	delivery.Nack(false, true)
	return false
}

func New(log logging.Logger, redis *redis.RedisSession) *Controller {
	return &Controller{
		log:   log,
		redis: redis,
	}
}

func (f *Controller) InteractionSaved(i *models.Interaction) error {
	return f.handleInteraction(1, i)
}

func (f *Controller) InteractionDeleted(i *models.Interaction) error {
	return f.handleInteraction(-1, i)
}

func (f *Controller) handleInteraction(incrementCount int, i *models.Interaction) error {
	cm, err := models.ChannelMessageById(i.MessageId)
	if err != nil {
		return err
	}

	c, err := models.ChannelById(cm.InitialChannelId)
	if err != nil {
		return err
	}

	if notEligibleForPopularPost(c, cm) {
		f.log.Error(fmt.Sprintf("Not eligible Interaction Id:%d", i.Id))
		return nil
	}

	keyname := &KeyName{
		GroupName: c.GroupName, ChannelName: c.Name,
		Time: cm.CreatedAt,
	}

	err = f.saveToDailyBucket(keyname, incrementCount, i.MessageId)
	if err != nil {
		return err
	}

	difference := int(i.CreatedAt.Sub(cm.CreatedAt).Hours()/24) + 1
	weight := 1 / float64(difference) * float64(incrementCount)
	rounded := fmt.Sprintf("%.1f", weight)

	keyname = &KeyName{
		GroupName: c.GroupName, ChannelName: c.Name,
		Time: time.Now().UTC(),
	}

	err = f.saveToSevenDayBucket(keyname, rounded, i.MessageId)
	if err != nil {
		return err
	}

	return nil
}

func (f *Controller) saveToDailyBucket(k *KeyName, inc int, id int64) error {
	key := k.Today()

	_, err := f.redis.SortedSetIncrBy(key, inc, id)
	if err != nil {
		return err
	}

	score, err := f.redis.SortedSetScore(key, id)
	if score <= 0 {
		_, err := f.redis.SortedSetRem(key, id)
		if err != nil {
			return err
		}
	}

	return err
}

func (f *Controller) saveToSevenDayBucket(k *KeyName, inc string, id int64) error {
	key := k.Weekly()

	_, ok := KeyExistsRegistry[key]
	if !ok {
		exists := f.redis.Exists(key)
		if !exists {
			err := f.createSevenDayBucket(k)
			if err != nil {
				return err
			}
		}

		KeyExistsRegistry[key] = true

		return nil
	}

	_, err := f.redis.SortedSetIncrBy(key, inc, id)
	if err != nil {
		return err
	}

	score, err := f.redis.SortedSetScore(key, id)
	if score <= 0 {
		_, err := f.redis.SortedSetRem(key, id)
		if err != nil {
			return err
		}
	}

	return nil
}

func (f *Controller) createSevenDayBucket(k *KeyName) error {
	keys, weights := []interface{}{}, []interface{}{}

	from := getStartOfDay(k.Time)
	aggregate := "SUM"

	for i := 0; i <= 6; i++ {
		currentDate := getDaysAgo(from, i)
		keys = append(keys, k.Before(currentDate))

		// add by 1 to prevent divide by 0 errors
		weight := float64(i + 1)
		weights = append(weights, float64(1/weight))
	}

	_, err := f.redis.SortedSetsUnion(k.Weekly(), keys, weights, aggregate)

	return err
}

func PopularPostKey(groupName, channelName string, current time.Time) string {
	name := KeyName{
		GroupName: groupName, ChannelName: channelName,
		Time: current.UTC(),
	}

	return name.Weekly()
}

//----------------------------------------------------------
// KeyName
//----------------------------------------------------------

type KeyName struct {
	GroupName, ChannelName string
	Time                   time.Time
}

func (k *KeyName) Today() string {
	return k.do(getStartOfDay(k.Time))
}

func (k *KeyName) Before(t time.Time) string {
	return k.do(t)
}

func (k *KeyName) Weekly() string {
	current := getStartOfDay(k.Time.UTC())
	sevenDaysAgo := getDaysAgo(current, 7).UTC().Unix()

	return fmt.Sprintf("%s-%d", k.do(current), sevenDaysAgo)
}

func (k *KeyName) do(t time.Time) string {
	return fmt.Sprintf("%s:%s:%s:%s:%d",
		config.MustGet().Environment, k.GroupName, PopularPostKeyName,
		k.ChannelName, t.UTC().Unix(),
	)
}

//----------------------------------------------------------
// helpers
//----------------------------------------------------------

func notEligibleForPopularPost(c *models.Channel, cm *models.ChannelMessage) bool {
	if c.MetaBits.Is(models.Troll) {
		return true
	}

	if c.PrivacyConstant != models.Channel_PRIVACY_PUBLIC {
		return true
	}

	if cm.MetaBits.Is(models.Troll) {
		return true
	}

	if cm.TypeConstant != models.ChannelMessage_TYPE_POST {
		return true
	}

	if createdMoreThan7DaysAgo(cm.CreatedAt) {
		return true
	}

	return false
}

//----------------------------------------------------------
// Time helpers
//----------------------------------------------------------

func createdMoreThan7DaysAgo(t time.Time) bool {
	t = t.UTC()
	delta := time.Now().Sub(t)

	return delta.Hours()/24 > 7
}

func getStartOfDay(t time.Time) time.Time {
	t = t.UTC()
	return now.New(t).BeginningOfDay()
}

func getDaysAgo(t time.Time, days int) time.Time {
	t = t.UTC()
	daysAgo := -time.Hour * 24 * time.Duration(days)

	return t.Add(daysAgo)
}

//----------------------------------------------------------
func (t *Controller) CreateKeyAtStartOfDay(groupName, channelName string) {
	endOfDay := now.EndOfDay().UTC()
	difference := time.Now().UTC().Sub(endOfDay)

	<-time.After(difference)

	keyname := &KeyName{
		GroupName: groupName, ChannelName: channelName,
		Time: time.Now().UTC(),
	}

	t.createSevenDayBucket(keyname)
}

func (t *Controller) ResetRegistry() {
	KeyExistsRegistry = map[string]bool{}
}
<|endoftext|>


<commit_before>package cache

import (
	"errors"
	"fmt"
	"time"
)

const (
	EXPIRES_DEFAULT = time.Duration(0)
	EXPIRES_FOREVER = time.Duration(-1)
)

var (
	ErrCacheMiss                             = errors.New("Key not found")
	ErrCacheNotStored                        = errors.New("Data not stored")
	ErrCacheNotSupported                     = errors.New("Operation not supported")
	ErrCacheDataCannotBeIncreasedOrDecreased = errors.New(`
		Data isn't an integer/string type, it cannot be increased or decreased`)
)

// a cached piece of data
type CacheItem struct {
	Expired time.Duration
	Data    []byte
}

// Cache interface contains all behaviors for cache adapter.
type Cache interface {
	// get cached value by key.
	Get(key string) ([]byte, error)
	// set cached value with key and expire time.
	Put(key string, data []byte, expire time.Duration) error
	// delete cached value by key.
	Delete(key string) error
	// check if cached value exists or not.
	IsExist(key string) bool
	// clear all cache.
	ClearAll() error
}

// Some caches like redis automatically clear out the cache
// But for the filesystem and in memory, this cannot.
// Caches that have to manually clear out the cached data should implement this method.
// start trash gc routine based on config string settings.
type GarbageCollector interface {
	StartAndTrashGc(config string) error
	//TrashGc(interval time.Duration)
}

// Store is a function create a new Cache Instance
type Store func() Cache

var adapters = make(map[string]Store)

// Register makes a cache adapter available by the adapter name.
// If Register is called twice with the same name or if adapter is nil,
// it panics.
func Register(name string, adapter Store) {
	if adapter == nil {
		panic("cache: Register adapter is nil")
	}

	if _, ok := adapters[name]; !ok {
		panic("cache: Register called twice for adapter " + name)
	}

	adapters[name] = adapter
}

// NewCache Create a new cache driver by adapter name and config string.
// config need to be correct JSON as string: {"interval":360}.
// it will start gc automatically.
func NewCache(adapterName, config string) (cache Cache, err error) {
	storeFunc, ok := adapters[adapterName]
	if !ok {
		err = fmt.Errorf("cache: unknown adapter name %q (forgot to import?)", adapterName)
		return nil, err
	}

	cache = storeFunc()
	return cache, nil
}
<commit_msg>cache item add create time and IsExpired func<commit_after>package cache

import (
	"errors"
	"fmt"
	"time"
)

const (
	EXPIRES_DEFAULT = time.Duration(0)
	EXPIRES_FOREVER = time.Duration(-1)
)

var (
	ErrCacheMiss                             = errors.New("Key not found")
	ErrCacheNotStored                        = errors.New("Data not stored")
	ErrCacheNotSupported                     = errors.New("Operation not supported")
	ErrCacheDataCannotBeIncreasedOrDecreased = errors.New(`
		Data isn't an integer/string type, it cannot be increased or decreased`)
)

// a cached piece of data
type CacheItem struct {
	CreatedTime time.Time
	Data        []byte
	Expired     time.Duration
}

func (ci *CacheItem) IsExpired() bool {
	// 0 means forever
	if ci.Expired == 0 {
		return false
	}

	return time.Now().Sub(ci.CreatedTime) > ci.Expired
}

// Cache interface contains all behaviors for cache adapter.
type Cache interface {
	// get cached value by key.
	Get(key string) ([]byte, error)
	// set cached value with key and expire time.
	Put(key string, data []byte, expire time.Duration) error
	// delete cached value by key.
	Delete(key string) error
	// check if cached value exists or not.
	IsExist(key string) bool
	// clear all cache.
	ClearAll() error
}

// Some caches like redis automatically clear out the cache
// But for the filesystem and in memory, this cannot.
// Caches that have to manually clear out the cached data should implement this method.
// start trash gc routine based on config string settings.
type GarbageCollector interface {
	//StartAndTrashGc(config string) error
	TrashGc(interval time.Duration)
}

// Store is a function create a new Cache Instance
type Store func() Cache

var adapters = make(map[string]Store)

// Register makes a cache adapter available by the adapter name.
// If Register is called twice with the same name or if adapter is nil,
// it panics.
func Register(name string, adapter Store) {
	if adapter == nil {
		panic("cache: Register adapter is nil")
	}

	if _, ok := adapters[name]; !ok {
		panic("cache: Register called twice for adapter " + name)
	}

	adapters[name] = adapter
}

// NewCache Create a new cache driver by adapter name and config string.
// config need to be correct JSON as string: {"interval":360}.
// it will start gc automatically.
func NewCache(adapterName, config string) (cache Cache, err error) {
	storeFunc, ok := adapters[adapterName]
	if !ok {
		err = fmt.Errorf("cache: unknown adapter name %q (forgot to import?)", adapterName)
		return nil, err
	}

	cache = storeFunc()
	return cache, nil
}
<|endoftext|>