<commit_before>package main

import (
	"fmt"
	"net/http"
	"net/http/pprof"
	"os"

	_ "github.com/jinzhu/gorm/dialects/mysql"

	"code.cloudfoundry.org/lager"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sqs"
	"github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"

	"cred-alert/ingestor"
	"cred-alert/metrics"
	"cred-alert/queue"
)

type AWSOpts struct {
	AwsAccessKey       string `long:"aws-access-key" description:"access key for aws SQS service" env:"AWS_ACCESS_KEY" value-name:"ACCESS_KEY"`
	AwsSecretAccessKey string `long:"aws-secret-key" description:"secret access key for aws SQS service" env:"AWS_SECRET_ACCESS_KEY" value-name:"SECRET_KEY"`
	AwsRegion          string `long:"aws-region" description:"aws region for SQS service" env:"AWS_REGION" value-name:"REGION"`
	SqsQueueName       string `long:"sqs-queue-name" description:"queue name to use with SQS" env:"SQS_QUEUE_NAME" value-name:"QUEUE_NAME"`
}

type Opts struct {
	Port uint16 `short:"p" long:"port" description:"the port to listen on" default:"8080" env:"PORT" value-name:"PORT"`

	GitHub struct {
		WebhookTokens []string `short:"w" long:"webhook-token" description:"github webhook secret token" env:"GITHUB_WEBHOOK_SECRET_KEYS" env-delim:"," value-name:"TOKENS" required:"true"`
	} `group:"GitHub Options"`

	Metrics struct {
		DatadogAPIKey string `long:"datadog-api-key" description:"key to emit to datadog" env:"DATADOG_API_KEY" value-name:"KEY"`
		Environment   string `long:"environment" description:"environment tag for metrics" env:"ENVIRONMENT" value-name:"NAME" default:"development"`
	} `group:"Metrics Options"`

	AWS AWSOpts `group:"AWS Options"`
}

func main() {
	var opts Opts

	logger := lager.NewLogger("cred-alert-ingestor")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.INFO))

	logger.Debug("starting")

	_, err := flags.ParseArgs(&opts, os.Args)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}

	taskQueue, err := createQueue(opts, logger)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}

	emitter := metrics.BuildEmitter(opts.Metrics.DatadogAPIKey, opts.Metrics.Environment)
	generator := queue.NewGenerator()

	in := ingestor.NewIngestor(taskQueue, emitter, "cred_alert", generator)

	router := http.NewServeMux()
	router.Handle("/webhook", ingestor.NewHandler(logger, in, opts.GitHub.WebhookTokens))

	members := []grouper.Member{
		{"api", http_server.New(
			fmt.Sprintf(":%d", opts.Port),
			router,
		)},
		{"debug", http_server.New(
			"127.0.0.1:6060",
			debugHandler(),
		)},
	}

	runner := sigmon.New(grouper.NewParallel(os.Interrupt, members))

	serverLogger := logger.Session("server", lager.Data{
		"port": opts.Port,
	})
	serverLogger.Info("starting")

	err = <-ifrit.Invoke(runner).Wait()
	if err != nil {
		serverLogger.Error("failed", err)
	}
}

func createQueue(opts Opts, logger lager.Logger) (queue.Queue, error) {
	logger = logger.Session("create-queue")
	logger.Debug("starting")

	if sqsValuesExist(opts.AWS) {
		logger.Debug("done")
		return createSqsQueue(logger, opts.AWS)
	}

	logger.Debug("done")
	return queue.NewNullQueue(logger), nil
}

func sqsValuesExist(awsOpts AWSOpts) bool {
	if awsOpts.AwsAccessKey != "" &&
		awsOpts.AwsSecretAccessKey != "" &&
		awsOpts.AwsRegion != "" &&
		awsOpts.SqsQueueName != "" {

		return true
	}

	return false
}

func createSqsQueue(logger lager.Logger, awsOpts AWSOpts) (queue.Queue, error) {
	logger = logger.Session("create-sqs-queue")

	creds := credentials.NewStaticCredentials(awsOpts.AwsAccessKey, awsOpts.AwsSecretAccessKey, "")
	config := aws.NewConfig().WithCredentials(creds).WithRegion(awsOpts.AwsRegion)
	service := sqs.New(session.New(config))

	queue, err := queue.BuildSQSQueue(service, awsOpts.SqsQueueName)
	if err != nil {
		logger.Error("failed", err)
		return nil, err
	}

	logger.Debug("done")
	return queue, nil
}

func debugHandler() http.Handler {
	debugRouter := http.NewServeMux()
	debugRouter.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	debugRouter.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
	debugRouter.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	debugRouter.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	debugRouter.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	return debugRouter
}
<commit_msg>Undo multiple webhook support for cred-alert<commit_after>package main

import (
	"fmt"
	"net/http"
	"net/http/pprof"
	"os"

	_ "github.com/jinzhu/gorm/dialects/mysql"

	"code.cloudfoundry.org/lager"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sqs"
	"github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"

	"cred-alert/ingestor"
	"cred-alert/metrics"
	"cred-alert/queue"
)

type AWSOpts struct {
	AwsAccessKey       string `long:"aws-access-key" description:"access key for aws SQS service" env:"AWS_ACCESS_KEY" value-name:"ACCESS_KEY"`
	AwsSecretAccessKey string `long:"aws-secret-key" description:"secret access key for aws SQS service" env:"AWS_SECRET_ACCESS_KEY" value-name:"SECRET_KEY"`
	AwsRegion          string `long:"aws-region" description:"aws region for SQS service" env:"AWS_REGION" value-name:"REGION"`
	SqsQueueName       string `long:"sqs-queue-name" description:"queue name to use with SQS" env:"SQS_QUEUE_NAME" value-name:"QUEUE_NAME"`
}

type Opts struct {
	Port uint16 `short:"p" long:"port" description:"the port to listen on" default:"8080" env:"PORT" value-name:"PORT"`

	GitHub struct {
		WebhookToken string `short:"w" long:"webhook-token" description:"github webhook secret token" env:"GITHUB_WEBHOOK_SECRET_KEY" value-name:"TOKEN" required:"true"`
	} `group:"GitHub Options"`

	Metrics struct {
		DatadogAPIKey string `long:"datadog-api-key" description:"key to emit to datadog" env:"DATADOG_API_KEY" value-name:"KEY"`
		Environment   string `long:"environment" description:"environment tag for metrics" env:"ENVIRONMENT" value-name:"NAME" default:"development"`
	} `group:"Metrics Options"`

	AWS AWSOpts `group:"AWS Options"`
}

func main() {
	var opts Opts

	logger := lager.NewLogger("cred-alert-ingestor")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.INFO))

	logger.Debug("starting")

	_, err := flags.ParseArgs(&opts, os.Args)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}

	taskQueue, err := createQueue(opts, logger)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}

	emitter := metrics.BuildEmitter(opts.Metrics.DatadogAPIKey, opts.Metrics.Environment)
	generator := queue.NewGenerator()

	in := ingestor.NewIngestor(taskQueue, emitter, "cred_alert", generator)

	router := http.NewServeMux()
	router.Handle("/webhook", ingestor.NewHandler(logger, in, []string{opts.GitHub.WebhookToken}))

	members := []grouper.Member{
		{"api", http_server.New(
			fmt.Sprintf(":%d", opts.Port),
			router,
		)},
		{"debug", http_server.New(
			"127.0.0.1:6060",
			debugHandler(),
		)},
	}

	runner := sigmon.New(grouper.NewParallel(os.Interrupt, members))

	serverLogger := logger.Session("server", lager.Data{
		"port": opts.Port,
	})
	serverLogger.Info("starting")

	err = <-ifrit.Invoke(runner).Wait()
	if err != nil {
		serverLogger.Error("failed", err)
	}
}

func createQueue(opts Opts, logger lager.Logger) (queue.Queue, error) {
	logger = logger.Session("create-queue")
	logger.Debug("starting")

	if sqsValuesExist(opts.AWS) {
		logger.Debug("done")
		return createSqsQueue(logger, opts.AWS)
	}

	logger.Debug("done")
	return queue.NewNullQueue(logger), nil
}

func sqsValuesExist(awsOpts AWSOpts) bool {
	if awsOpts.AwsAccessKey != "" &&
		awsOpts.AwsSecretAccessKey != "" &&
		awsOpts.AwsRegion != "" &&
		awsOpts.SqsQueueName != "" {

		return true
	}

	return false
}

func createSqsQueue(logger lager.Logger, awsOpts AWSOpts) (queue.Queue, error) {
	logger = logger.Session("create-sqs-queue")

	creds := credentials.NewStaticCredentials(awsOpts.AwsAccessKey, awsOpts.AwsSecretAccessKey, "")
	config := aws.NewConfig().WithCredentials(creds).WithRegion(awsOpts.AwsRegion)
	service := sqs.New(session.New(config))

	queue, err := queue.BuildSQSQueue(service, awsOpts.SqsQueueName)
	if err != nil {
		logger.Error("failed", err)
		return nil, err
	}

	logger.Debug("done")
	return queue, nil
}

func debugHandler() http.Handler {
	debugRouter := http.NewServeMux()
	debugRouter.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	debugRouter.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
	debugRouter.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	debugRouter.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	debugRouter.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	return debugRouter
}
<|endoftext|>


<commit_before>package usecases

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/webitel/cdr/src/conf"
	"github.com/webitel/cdr/src/entity"
	"github.com/webitel/cdr/src/logger"
)

type CdrInteractor struct {
	SqlCdrARepository      entity.SqlCdrARepository
	SqlCdrBRepository      entity.SqlCdrBRepository
	ElasticCdrARepository  entity.ElasticCdrARepository
	ElasticCdrBRepository  entity.ElasticCdrBRepository
	AmqPublisherRepository entity.AmqPublisherRepository
	AmqReceiverRepositoryA entity.AmqReceiverRepository
	AmqReceiverRepositoryB entity.AmqReceiverRepository
}

type SqlProcess func(deliveries []entity.Delivery) error

func (interactor *CdrInteractor) InitTables() error {
	if err := interactor.SqlCdrARepository.CreateTableIfNotExist(); err != nil {
		return fmt.Errorf("PostgreSQL. Table A creating error: " + err.Error())
	}
	if err := interactor.SqlCdrBRepository.CreateTableIfNotExist(); err != nil {
		return fmt.Errorf("PostgreSQL. Table B creating error: " + err.Error())
	}
	return nil
}

func (interactor *CdrInteractor) Run() {
	if interactor.AmqPublisherRepository == nil || interactor.SqlCdrBRepository == nil || interactor.SqlCdrARepository == nil {
		return
	}

	publisher := conf.GetPublisher()
	size, interval := conf.GetListenerConfig()
	for {
		var done = make(chan error)
		interactor.AmqPublisherRepository.CreateAmqConnection(publisher.ConnectionString, publisher.ExchangeName, publisher.ExchangeType)
		msgsA, err := interactor.AmqPublisherRepository.GetMessages(publisher.ExchangeName, publisher.ExchangeType, publisher.RoutingKeyA)
		if err != nil {
			logger.Error(err.Error())
			continue
		}
		msgsB, err := interactor.AmqPublisherRepository.GetMessages(publisher.ExchangeName, publisher.ExchangeType, publisher.RoutingKeyB)
		if err != nil {
			logger.Error(err.Error())
			continue
		}
		go interactor.ListenEvents(msgsA, size, interval, done, interactor.AddToSqlA, "Leg A")
		go interactor.ListenEvents(msgsB, size, interval, done, interactor.AddToSqlB, "Leg B")
		logger.Notice("RabbitMQ: start listening...")
		err = <-done
		logger.Error(err.Error())
	}
}

func (interactor *CdrInteractor) ListenEvents(msgs <-chan entity.Delivery, size, interval uint32, done chan error, sqlProcess SqlProcess, key string) {
	batch := make([]entity.Delivery, 0, size)
	promise := time.Millisecond * time.Duration(interval)
	tmr := time.NewTimer(promise)
	for {
		select {
		case <-tmr.C:
			{
				if len(batch) > 0 {
					go interactor.DeliveryProcess(batch, sqlProcess, key)
					batch = make([]entity.Delivery, 0, size)
				}
				tmr.Reset(promise)
				//log.Printf("RabbitMQ: listening [%s]...\n", key)
			}
		case d, ok := <-msgs:
			{
				batch = append(batch, d)
				if len(batch) == cap(batch) {
					go interactor.DeliveryProcess(batch, sqlProcess, key)
					batch = make([]entity.Delivery, 0, size)
					tmr.Reset(promise)
				}
				if !ok {
					if len(batch) > 0 && len(batch) != cap(batch) {
						go interactor.DeliveryProcess(batch, sqlProcess, key)
					}
					done <- fmt.Errorf("ERROR: Deliveries channel closed")
					return
				}
			}
		}
	}
}

func (interactor *CdrInteractor) DeliveryProcess(batch []entity.Delivery, sqlProcess SqlProcess, key string) {
	if err := sqlProcess(batch); err != nil {
		logger.Error("ERROR. %s: %s", key, err)
		for i := 0; i < len(batch); i++ {
			batch[i].Nack(false, true)
		}
		logger.Error("PostgreSQL: failed to store items [%s, %v]", key, len(batch))
	} else {
		for i := 0; i < len(batch); i++ {
			batch[i].Ack(false)
		}
		logger.Notice("PostgreSQL: items stored [%s, %v]", key, len(batch))
	}
	//log.Printf("RabbitMQ: listening [%s]...\n", key)
}

func (interactor *CdrInteractor) AddToSqlA(deliveries []entity.Delivery) error {
	var calls []entity.SqlCdr
	var callsB []entity.SqlCdr
	for _, item := range deliveries {
		call, err := readBytes(item.GetBody())
		if err != nil {
			return err
		}
		uuid, ok := call.(map[string]interface{})["variables"].(map[string]interface{})["uuid"].(string)
		parent := getParentUuid(call)
		if ok {
			if parent == "" {
				sql_call, err := parseToSqlA(item.GetBody(), uuid)
				if err != nil {
					return err
				}
				calls = append(calls, sql_call)
			} else {
				sql_call, err := parseToSqlB(item.GetBody(), uuid, parent)
				if err != nil {
					return err
				}
				callsB = append(callsB, sql_call)
			}
		}
	}
	err := interactor.SqlCdrARepository.InsertPack(calls)
	if err != nil {
		return err
	}
	if len(callsB) > 0 {
		err := interactor.SqlCdrBRepository.InsertPack(callsB)
		if err != nil {
			return err
		}
		logger.Notice("Count of LegB in LegA channel [%v]", len(callsB))
	}
	return nil
}

func (interactor *CdrInteractor) AddToSqlB(deliveries []entity.Delivery) error {
	var calls []entity.SqlCdr
	for _, item := range deliveries {
		call, err := readBytes(item.GetBody())
		if err != nil {
			return err
		}
		uuid, ok := call.(map[string]interface{})["variables"].(map[string]interface{})["uuid"].(string)
		parent := getParentUuid(call)
		if ok {
			sql_call, err := parseToSqlB(item.GetBody(), uuid, parent) //e_call.Variables["uuid"])
			if err != nil {
				return err
			}
			calls = append(calls, sql_call)
		}
	}
	err := interactor.SqlCdrBRepository.InsertPack(calls)
	if err != nil {
		return err
	}
	return nil
}

func getParentUuid(call interface{}) string {
	var (
		s  string
		ok bool
	)
	if s, ok = call.(map[string]interface{})["variables"].(map[string]interface{})["ent_originate_aleg_uuid"].(string); !ok {
		s, ok = call.(map[string]interface{})["variables"].(map[string]interface{})["originating_leg_uuid"].(string)
		if !ok {
			if callflow, ok := call.(map[string]interface{})["callflow"].([]interface{}); ok && len(callflow) > 0 {
				if caller_profile, ok := callflow[0].(map[string]interface{})["caller_profile"].(map[string]interface{}); ok {
					if originator, ok := caller_profile["originator"].(map[string]interface{}); ok {
						if arr, ok := originator["originator_caller_profiles"].([]interface{}); ok && len(arr) > 0 {
							s, _ = arr[0].(map[string]interface{})["uuid"].(string)
						}
					}
				}
			}
		}
	}
	return s
}

func parseToSqlA(body []byte, uuid string) (entity.SqlCdr, error) {
	pg_call := entity.SqlCdr{
		Uuid:           uuid,
		Event:          body,
		Size:           uint32(len(body)),
		Created_at:     uint64(time.Now().UnixNano() / 1000000),
		Stored_state:   0,
		Archived_state: 0,
	}
	return pg_call, nil
}

func parseToSqlB(body []byte, uuid string, parent string) (entity.SqlCdr, error) {
	pg_call := entity.SqlCdr{
		Uuid:           uuid,
		Parent_uuid:    parent,
		Event:          body,
		Size:           uint32(len(body)),
		Created_at:     uint64(time.Now().UnixNano() / 1000000),
		Stored_state:   0,
		Archived_state: 0,
	}
	return pg_call, nil
}

func readBytes(body []byte) (interface{}, error) {
	var call interface{}
	if err := json.Unmarshal(body, &call); err != nil {
		return nil, fmt.Errorf("Parse JSON error. %s", err)
	}
	return call, nil
}
<commit_msg>fix legA<commit_after>package usecases

import (
	"encoding/json"
	"fmt"
	"time"

	"webitel.com/cdr_service/conf"
	"webitel.com/cdr_service/entity"
	"webitel.com/cdr_service/logger"
)

type CdrInteractor struct {
	SqlCdrARepository      entity.SqlCdrARepository
	SqlCdrBRepository      entity.SqlCdrBRepository
	ElasticCdrARepository  entity.ElasticCdrARepository
	ElasticCdrBRepository  entity.ElasticCdrBRepository
	AmqPublisherRepository entity.AmqPublisherRepository
	AmqReceiverRepositoryA entity.AmqReceiverRepository
	AmqReceiverRepositoryB entity.AmqReceiverRepository
}

type SqlProcess func(deliveries []entity.Delivery) error

func (interactor *CdrInteractor) InitTables() error {
	if err := interactor.SqlCdrARepository.CreateTableIfNotExist(); err != nil {
		return fmt.Errorf("PostgreSQL. Table A creating error: " + err.Error())
	}
	if err := interactor.SqlCdrBRepository.CreateTableIfNotExist(); err != nil {
		return fmt.Errorf("PostgreSQL. Table B creating error: " + err.Error())
	}
	return nil
}

func (interactor *CdrInteractor) Run() {
	if interactor.AmqPublisherRepository == nil || interactor.SqlCdrBRepository == nil || interactor.SqlCdrARepository == nil {
		return
	}

	publisher := conf.GetPublisher()
	size, interval := conf.GetListenerConfig()
	for {
		var done = make(chan error)
		interactor.AmqPublisherRepository.CreateAmqConnection(publisher.ConnectionString, publisher.ExchangeName, publisher.ExchangeType)
		msgsA, err := interactor.AmqPublisherRepository.GetMessages(publisher.ExchangeName, publisher.ExchangeType, publisher.RoutingKeyA)
		if err != nil {
			logger.Error(err.Error())
			continue
		}
		msgsB, err := interactor.AmqPublisherRepository.GetMessages(publisher.ExchangeName, publisher.ExchangeType, publisher.RoutingKeyB)
		if err != nil {
			logger.Error(err.Error())
			continue
		}
		go interactor.ListenEvents(msgsA, size, interval, done, interactor.AddToSqlA, "Leg A")
		go interactor.ListenEvents(msgsB, size, interval, done, interactor.AddToSqlB, "Leg B")
		logger.Notice("RabbitMQ: start listening...")
		err = <-done
		logger.Error(err.Error())
	}
}

func (interactor *CdrInteractor) ListenEvents(msgs <-chan entity.Delivery, size, interval uint32, done chan error, sqlProcess SqlProcess, key string) {
	batch := make([]entity.Delivery, 0, size)
	promise := time.Millisecond * time.Duration(interval)
	tmr := time.NewTimer(promise)
	for {
		select {
		case <-tmr.C:
			{
				if len(batch) > 0 {
					go interactor.DeliveryProcess(batch, sqlProcess, key)
					batch = make([]entity.Delivery, 0, size)
				}
				tmr.Reset(promise)
				//log.Printf("RabbitMQ: listening [%s]...\n", key)
			}
		case d, ok := <-msgs:
			{
				batch = append(batch, d)
				if len(batch) == cap(batch) {
					go interactor.DeliveryProcess(batch, sqlProcess, key)
					batch = make([]entity.Delivery, 0, size)
					tmr.Reset(promise)
				}
				if !ok {
					if len(batch) > 0 && len(batch) != cap(batch) {
						go interactor.DeliveryProcess(batch, sqlProcess, key)
					}
					done <- fmt.Errorf("ERROR: Deliveries channel closed")
					return
				}
			}
		}
	}
}

func (interactor *CdrInteractor) DeliveryProcess(batch []entity.Delivery, sqlProcess SqlProcess, key string) {
	if err := sqlProcess(batch); err != nil {
		logger.Error("ERROR. %s: %s", key, err)
		for i := 0; i < len(batch); i++ {
			batch[i].Nack(false, true)
		}
		logger.Error("PostgreSQL: failed to store items [%s, %v]", key, len(batch))
	} else {
		for i := 0; i < len(batch); i++ {
			batch[i].Ack(false)
		}
		logger.Notice("PostgreSQL: items stored [%s, %v]", key, len(batch))
	}
	//log.Printf("RabbitMQ: listening [%s]...\n", key)
}

func (interactor *CdrInteractor) AddToSqlA(deliveries []entity.Delivery) error {
	var calls []entity.SqlCdr
	var callsB []entity.SqlCdr
	for _, item := range deliveries {
		call, err := readBytes(item.GetBody())
		if err != nil {
			return err
		}
		uuid, ok := call.(map[string]interface{})["variables"].(map[string]interface{})["uuid"].(string)
		parent := getParentUuid(call)
		if ok {
			if parent == "" {
				sql_call, err := parseToSqlA(item.GetBody(), uuid)
				if err != nil {
					return err
				}
				calls = append(calls, sql_call)
			} else {
				sql_call, err := parseToSqlB(item.GetBody(), uuid, parent)
				if err != nil {
					return err
				}
				callsB = append(callsB, sql_call)
			}
		}
	}
	if len(calls) > 0 {
		if err := interactor.SqlCdrARepository.InsertPack(calls); err != nil {
			return err
		}
	}
	if len(callsB) > 0 {
		if err := interactor.SqlCdrBRepository.InsertPack(callsB); err != nil {
			return err
		}
		logger.Notice("Count of LegB in LegA channel [%v]", len(callsB))
	}
	return nil
}

func (interactor *CdrInteractor) AddToSqlB(deliveries []entity.Delivery) error {
	var calls []entity.SqlCdr
	for _, item := range deliveries {
		call, err := readBytes(item.GetBody())
		if err != nil {
			return err
		}
		uuid, ok := call.(map[string]interface{})["variables"].(map[string]interface{})["uuid"].(string)
		parent := getParentUuid(call)
		if ok {
			sql_call, err := parseToSqlB(item.GetBody(), uuid, parent)
			if err != nil {
				return err
			}
			calls = append(calls, sql_call)
		}
	}
	if len(calls) > 0 {
		if err := interactor.SqlCdrBRepository.InsertPack(calls); err != nil {
			return err
		}
	}
	return nil
}

func getParentUuid(call interface{}) string {
	var (
		s  string
		ok bool
	)
	if s, ok = call.(map[string]interface{})["variables"].(map[string]interface{})["ent_originate_aleg_uuid"].(string); !ok {
		s, ok = call.(map[string]interface{})["variables"].(map[string]interface{})["originating_leg_uuid"].(string)
		if !ok {
			if callflow, ok := call.(map[string]interface{})["callflow"].([]interface{}); ok && len(callflow) > 0 {
				if caller_profile, ok := callflow[0].(map[string]interface{})["caller_profile"].(map[string]interface{}); ok {
					if originator, ok := caller_profile["originator"].(map[string]interface{}); ok {
						if arr, ok := originator["originator_caller_profiles"].([]interface{}); ok && len(arr) > 0 {
							s, _ = arr[0].(map[string]interface{})["uuid"].(string)
						}
					}
				}
			}
		}
	}
	return s
}

func parseToSqlA(body []byte, uuid string) (entity.SqlCdr, error) {
	pg_call := entity.SqlCdr{
		Uuid:           uuid,
		Event:          body,
		Size:           uint32(len(body)),
		Created_at:     uint64(time.Now().UnixNano() / 1000000),
		Stored_state:   0,
		Archived_state: 0,
	}
	return pg_call, nil
}

func parseToSqlB(body []byte, uuid string, parent string) (entity.SqlCdr, error) {
	pg_call := entity.SqlCdr{
		Uuid:           uuid,
		Parent_uuid:    parent,
		Event:          body,
		Size:           uint32(len(body)),
		Created_at:     uint64(time.Now().UnixNano() / 1000000),
		Stored_state:   0,
		Archived_state: 0,
	}
	return pg_call, nil
}

func readBytes(body []byte) (interface{}, error) {
	var call interface{}
	if err := json.Unmarshal(body, &call); err != nil {
		return nil, fmt.Errorf("Parse JSON error. %s", err)
	}
	return call, nil
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"menteslibres.net/gosexy/redis"
)

var client *redis.Client

func dbGet(key string) (value string) {
	s, _ := client.Get(key)
	return s
}

func dbSet(key string, value string) {
	client.Set(key, value)
}

func dbPop(key string) (value string) {
	client.Lpop(key)
}

func dbPush(key string, value string) {
	client.Lpush(key, value)
}

func dbConn() {

	var err error

	client = redis.New()

	err = client.Connect("127.0.0.1", 6379)

	if err != nil {
		log.Fatalf("Connect failed: %s\n", err.Error())
		return
	}

	log.Println("Connected to redis-server.")

}

func dbClose() {

	client.Quit()

}
<commit_msg>fix Lpop func<commit_after>package main

import (
	"log"
	"menteslibres.net/gosexy/redis"
)

var client *redis.Client

func dbGet(key string) (value string) {
	s, _ := client.Get(key)
	return s
}

func dbSet(key string, value string) {
	client.Set(key, value)
}

func dbPop(key string) (value string) {
	s, _ := client.Lpop(key)
	return s
}

func dbPush(key string, value string) {
	client.Lpush(key, value)
}

func dbConn() {

	var err error

	client = redis.New()

	err = client.Connect("127.0.0.1", 6379)

	if err != nil {
		log.Fatalf("Connect failed: %s\n", err.Error())
		return
	}

	log.Println("Connected to redis-server.")

}

func dbClose() {

	client.Quit()

}
<|endoftext|>


<commit_before>package easyss

import (
	"io"
	"net"
	"time"

	"github.com/nange/easypool"
	"github.com/nange/easyss/cipherstream"
	"github.com/nange/easyss/util"
	log "github.com/sirupsen/logrus"
)

var connStateBytes = util.NewBytes(32)

// relay copies between cipher stream and plaintext stream.
// return the number of bytes copies
// from plaintext stream to cipher stream, from cipher stream to plaintext stream, and needClose on server conn
func relay(cipher, plaintxt io.ReadWriteCloser) (n1 int64, n2 int64, needClose bool) {
	type res struct {
		N   int64
		Err error
	}
	ch1 := make(chan res, 1)
	ch2 := make(chan res, 1)

	go func() {
		n, err := io.Copy(plaintxt, cipher)
		ch2 <- res{N: n, Err: err}
	}()
	go func() {
		n, err := io.Copy(cipher, plaintxt)
		ch1 <- res{N: n, Err: err}
	}()

	var state *ConnState
	for i := 0; i < 2; i++ {
		select {
		case res1 := <-ch1:
			expireConn(cipher)
			n1 = res1.N
			err := res1.Err
			if cipherstream.EncryptErr(err) || cipherstream.WriteCipherErr(err) {
				log.Debugf("io.Copy err:%+v, maybe underline connection have been closed", err)
				markCipherStreamUnusable(cipher)
				continue
			}

			if i == 0 {
				log.Debugf("read plaintxt stream error, set start state. details:%v", err)
				buf := connStateBytes.Get(32)
				defer connStateBytes.Put(buf)
				state = NewConnState(FIN_WAIT1, buf)
			} else if err != nil {
				if !cipherstream.TimeoutErr(err) {
					log.Errorf("execpt error is net: io timeout. but get:%v", err)
				}
			}

		case res2 := <-ch2:
			expireConn(plaintxt)
			n2 = res2.N
			err := res2.Err
			if cipherstream.DecryptErr(err) || cipherstream.ReadCipherErr(err) {
				log.Debugf("io.Copy err:%+v, maybe underline connection have been closed", err)
				markCipherStreamUnusable(cipher)
				continue
			}

			if i == 0 {
				if cipherstream.FINRSTStreamErr(err) {
					log.Debugf("read cipher stream ErrFINRSTStream, set start state")
					buf := connStateBytes.Get(32)
					defer connStateBytes.Put(buf)
					state = NewConnState(CLOSE_WAIT, buf)
				} else {
					log.Errorf("execpt error is ErrFINRSTStream, but get:%v", err)
					markCipherStreamUnusable(cipher)
				}
			} else if err != nil {
				if !cipherstream.TimeoutErr(err) {
					log.Errorf("execpt error is net: io timeout. but get:%v", err)
				}
			}

		}
	}

	if cipherStreamUnusable(cipher) {
		needClose = true
		return
	}

	setCipherDeadline(cipher)
	if state == nil {
		log.Warnf("unexcepted state, some unexcepted error occor")
		needClose = true
		return
	}
	for stateFn := state.fn; stateFn != nil; {
		stateFn = stateFn(cipher).fn
	}
	if state.err != nil {
		log.Warnf("state err:%+v, state:%v", state.err, state.state)
		markCipherStreamUnusable(cipher)
		needClose = true
	}

	return
}

// mark the cipher stream unusable, return mark result
func markCipherStreamUnusable(cipher io.ReadWriteCloser) bool {
	if cs, ok := cipher.(*cipherstream.CipherStream); ok {
		if pc, ok := cs.ReadWriteCloser.(*easypool.PoolConn); ok {
			log.Debugf("mark cipher stream unusable")
			pc.MarkUnusable()
			return true
		}
	}
	return false
}

// return true if the cipher stream is unusable
func cipherStreamUnusable(cipher io.ReadWriteCloser) bool {
	if cs, ok := cipher.(*cipherstream.CipherStream); ok {
		if pc, ok := cs.ReadWriteCloser.(*easypool.PoolConn); ok {
			return pc.IsUnusable()
		}
	}
	return false
}

func expireConn(conn io.ReadWriteCloser) {
	if conn, ok := conn.(net.Conn); ok {
		log.Debugf("expire the plaintxt tcp connection to make the reader to be failed immediately")
		conn.SetDeadline(time.Unix(0, 0))
		return
	}

	if cs, ok := conn.(*cipherstream.CipherStream); ok {
		if conn, ok := cs.ReadWriteCloser.(net.Conn); ok {
			log.Debugf("expire the cipher tcp connection to make the reader to be failed immediately")
			conn.SetDeadline(time.Unix(0, 0))
		}
	}
}

func setCipherDeadline(cipher io.ReadWriteCloser) {
	if cs, ok := cipher.(*cipherstream.CipherStream); ok {
		if conn, ok := cs.ReadWriteCloser.(net.Conn); ok {
			log.Debugf("set cipher tcp connection deadline to 30 second later")
			conn.SetDeadline(time.Now().Add(30 * time.Second))
		}
	}
}
<commit_msg>relay: update log msg<commit_after>package easyss

import (
	"io"
	"net"
	"time"

	"github.com/nange/easypool"
	"github.com/nange/easyss/cipherstream"
	"github.com/nange/easyss/util"
	log "github.com/sirupsen/logrus"
)

var connStateBytes = util.NewBytes(32)

// relay copies between cipher stream and plaintext stream.
// return the number of bytes copies
// from plaintext stream to cipher stream, from cipher stream to plaintext stream, and needClose on server conn
func relay(cipher, plaintxt io.ReadWriteCloser) (n1 int64, n2 int64, needClose bool) {
	type res struct {
		N   int64
		Err error
	}
	ch1 := make(chan res, 1)
	ch2 := make(chan res, 1)

	go func() {
		n, err := io.Copy(plaintxt, cipher)
		ch2 <- res{N: n, Err: err}
	}()
	go func() {
		n, err := io.Copy(cipher, plaintxt)
		ch1 <- res{N: n, Err: err}
	}()

	var state *ConnState
	for i := 0; i < 2; i++ {
		select {
		case res1 := <-ch1:
			expireConn(cipher)
			n1 = res1.N
			err := res1.Err
			if cipherstream.EncryptErr(err) || cipherstream.WriteCipherErr(err) {
				log.Debugf("io.Copy err:%+v, maybe underline connection have been closed", err)
				markCipherStreamUnusable(cipher)
				continue
			}

			if i == 0 {
				log.Debugf("read plaintxt stream error, set start state. details:%v", err)
				buf := connStateBytes.Get(32)
				defer connStateBytes.Put(buf)
				state = NewConnState(FIN_WAIT1, buf)
			} else if err != nil {
				if !cipherstream.TimeoutErr(err) {
					log.Errorf("execpt error is net: io timeout. but get:%v", err)
				}
			}

		case res2 := <-ch2:
			expireConn(plaintxt)
			n2 = res2.N
			err := res2.Err
			if cipherstream.DecryptErr(err) || cipherstream.ReadCipherErr(err) {
				log.Debugf("io.Copy err:%+v, maybe underline connection have been closed", err)
				markCipherStreamUnusable(cipher)
				continue
			}

			if i == 0 {
				if cipherstream.FINRSTStreamErr(err) {
					log.Debugf("read cipher stream ErrFINRSTStream, set start state")
					buf := connStateBytes.Get(32)
					defer connStateBytes.Put(buf)
					state = NewConnState(CLOSE_WAIT, buf)
				} else {
					log.Errorf("execpt error is ErrFINRSTStream, but get:%v", err)
					markCipherStreamUnusable(cipher)
				}
			} else if err != nil {
				if !cipherstream.TimeoutErr(err) {
					log.Errorf("execpt error is net: io timeout. but get:%v", err)
				}
			}

		}
	}

	if cipherStreamUnusable(cipher) {
		needClose = true
		return
	}

	setCipherDeadline(cipher)
	if state == nil {
		log.Warnf("unexcepted state, some unexcepted error occor, maybe client connection is closed")
		needClose = true
		return
	}
	for stateFn := state.fn; stateFn != nil; {
		stateFn = stateFn(cipher).fn
	}
	if state.err != nil {
		log.Warnf("state err:%+v, state:%v", state.err, state.state)
		markCipherStreamUnusable(cipher)
		needClose = true
	}

	return
}

// mark the cipher stream unusable, return mark result
func markCipherStreamUnusable(cipher io.ReadWriteCloser) bool {
	if cs, ok := cipher.(*cipherstream.CipherStream); ok {
		if pc, ok := cs.ReadWriteCloser.(*easypool.PoolConn); ok {
			log.Debugf("mark cipher stream unusable")
			pc.MarkUnusable()
			return true
		}
	}
	return false
}

// return true if the cipher stream is unusable
func cipherStreamUnusable(cipher io.ReadWriteCloser) bool {
	if cs, ok := cipher.(*cipherstream.CipherStream); ok {
		if pc, ok := cs.ReadWriteCloser.(*easypool.PoolConn); ok {
			return pc.IsUnusable()
		}
	}
	return false
}

func expireConn(conn io.ReadWriteCloser) {
	if conn, ok := conn.(net.Conn); ok {
		log.Debugf("expire the plaintxt tcp connection to make the reader to be failed immediately")
		conn.SetDeadline(time.Unix(0, 0))
		return
	}

	if cs, ok := conn.(*cipherstream.CipherStream); ok {
		if conn, ok := cs.ReadWriteCloser.(net.Conn); ok {
			log.Debugf("expire the cipher tcp connection to make the reader to be failed immediately")
			conn.SetDeadline(time.Unix(0, 0))
		}
	}
}

func setCipherDeadline(cipher io.ReadWriteCloser) {
	if cs, ok := cipher.(*cipherstream.CipherStream); ok {
		if conn, ok := cs.ReadWriteCloser.(net.Conn); ok {
			log.Debugf("set cipher tcp connection deadline to 30 second later")
			conn.SetDeadline(time.Now().Add(30 * time.Second))
		}
	}
}
<|endoftext|>


<commit_before>package thumbnailer

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
)

const sniffSize = 4 << 10

// Matching code partially adapted from "net/http/sniff.go"

// Mime type prefix magic number matchers and canonical extensions
var matchers = []Matcher{
	// Probably most common types, this library will be used for, first.
	// More expensive checks are also positioned lower.
	&exactSig{"jpg", "image/jpeg", []byte("\xFF\xD8\xFF")},
	&exactSig{"png", "image/png", []byte("\x89\x50\x4E\x47\x0D\x0A\x1A\x0A")},
	&exactSig{"gif", "image/gif", []byte("GIF87a")},
	&exactSig{"gif", "image/gif", []byte("GIF89a")},
	&maskedSig{
		"webp",
		"image/webp",
		[]byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF"),
		[]byte("RIFF\x00\x00\x00\x00WEBPVP"),
	},
	&maskedSig{
		"ogg",
		"application/ogg",
		[]byte("OggS\x00"),
		[]byte("\x4F\x67\x67\x53\x00"),
	},
	MatcherFunc(matchWebmOrMKV),
	&exactSig{"pdf", "application/pdf", []byte("%PDF-")},
	&maskedSig{
		"mp3",
		"audio/mpeg",
		[]byte("\xFF\xFF\xFF"),
		[]byte("ID3"),
	},
	MatcherFunc(matchMP4),
	&exactSig{"aac", "audio/aac", []byte("ÿñ")},
	&exactSig{"aac", "audio/aac", []byte("ÿù")},
	&exactSig{"bmp", "image/bmp", []byte("BM")},
	&maskedSig{
		"wav",
		"audio/wave",
		[]byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"),
		[]byte("RIFF\x00\x00\x00\x00WAVE"),
	},
	&maskedSig{
		"avi",
		"video/avi",
		[]byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"),
		[]byte("RIFF\x00\x00\x00\x00AVI "),
	},
	&exactSig{"psd", "image/photoshop", []byte("8BPS")},
	&exactSig{"flac", "audio/x-flac", []byte("fLaC")},
	&exactSig{"tiff", "image/tiff", []byte("II*\x00")},
	&exactSig{"tiff", "image/tiff", []byte("MM\x00*")},
	&exactSig{"mov", "video/quicktime", []byte("\x00\x00\x00\x14ftyp")},
	&exactSig{
		"wmv",
		"video/x-ms-wmv",
		[]byte{0x30, 0x26, 0xB2, 0x75, 0x8E, 0x66, 0xCF, 0x11, 0xA6, 0xD9},
	},
	&exactSig{"flv", "video/x-flv", []byte("FLV\x01")},
	&exactSig{"ico", "image/x-icon", []byte("\x00\x00\x01\x00")},
	&maskedSig{
		"midi",
		"audio/midi",
		[]byte("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"),
		[]byte("MThd\x00\x00\x00\x06"),
	},
	MatcherFunc(matchMP3),
}

var (
	// User-defined MIME type processors
	overrideProcessors = map[string]Processor{}
)

// Processor is a specialized file processor for a specific file type
type Processor func(Source, Options) (Source, Thumbnail, error)

// Matcher takes up to the first 512 bytes of a file and returns the MIME type
// and canonical extension, that were matched. Empty string indicates no match.
type Matcher interface {
	Match([]byte) (mime string, extension string)
}

// MatcherFunc is an adapter that allows using functions as Matcher
type MatcherFunc func([]byte) (string, string)

// Match implements Matcher
func (fn MatcherFunc) Match(data []byte) (string, string) {
	return fn(data)
}

type exactSig struct {
	ext, mime string
	sig       []byte
}

func (e *exactSig) Match(data []byte) (string, string) {
	if bytes.HasPrefix(data, e.sig) {
		return e.mime, e.ext
	}
	return "", ""
}

type maskedSig struct {
	ext, mime string
	mask, pat []byte
}

func (m *maskedSig) Match(data []byte) (string, string) {
	if len(data) < len(m.mask) {
		return "", ""
	}
	for i, mask := range m.mask {
		db := data[i] & mask
		if db != m.pat[i] {
			return "", ""
		}
	}
	return m.mime, m.ext
}

func matchWebmOrMKV(data []byte) (string, string) {
	switch {
	case len(data) < 8 || !bytes.HasPrefix(data, []byte("\x1A\x45\xDF\xA3")):
		return "", ""
	case bytes.Contains(data[4:], []byte("webm")):
		return "video/webm", "webm"
	case bytes.Contains(data[4:], []byte("matroska")):
		return "video/x-matroska", "mkv"
	default:
		return "", ""
	}
}

func matchMP4(data []byte) (string, string) {
	if len(data) < 12 {
		return "", ""
	}

	boxSize := int(binary.BigEndian.Uint32(data[:4]))
	nope := boxSize%4 != 0 ||
		len(data) < boxSize ||
		!bytes.Equal(data[4:8], []byte("ftyp"))
	if nope {
		return "", ""
	}

	for st := 8; st < boxSize; st += 4 {
		if st == 12 {
			// minor version number
			continue
		}
		if bytes.Equal(data[st:st+3], []byte("mp4")) {
			return "video/mp4", "mp4"
		}
	}
	return "", ""
}

// MP3 is a retarded standard, that will not always even have a magic number.
// Need to detect with FFMPEG as a last resort.
func matchMP3(data []byte) (mime string, ext string) {
	c, err := NewFFContext(bytes.NewReader(data))
	if err != nil {
		return
	}
	defer c.Close()

	codec, err := c.CodecName(FFAudio)
	if err != nil {
		return
	}
	if codec == "mp3" {
		return "audio/mpeg", "mp3"
	}
	return
}

// UnsupportedMIMEError indicates the MIME type of the file could not be
// detected as a supported type or was not in the AcceptedMimeTypes list, if
// defined.
type UnsupportedMIMEError string

func (u UnsupportedMIMEError) Error() string {
	return fmt.Sprintf("unsupported MIME type: %s", string(u))
}

// RegisterMatcher adds an extra magic prefix-based MIME type matcher to the
// default set with an included canonical file extension.
// Not safe to use concurrently with file processing.
func RegisterMatcher(m Matcher) {
	matchers = append(matchers, m)
}

// RegisterProcessor registers a file processor for a specific MIME type.
// Can be used to add support for additional MIME types or as an override.
// Not safe to use concurrently with file processing.
func RegisterProcessor(mime string, fn Processor) {
	overrideProcessors[mime] = fn
}

// DetectMIME  detects the MIME typ of the r. r must be at starting position.
// accepted, if not nil, specifies MIME types to not reject with
// UnsupportedMIMEError.
func DetectMIME(r io.Reader, accepted map[string]bool) (string, string, error) {
	buf := make([]byte, sniffSize)
	read, err := r.Read(buf)
	if err != nil {
		return "", "", err
	}
	if read < sniffSize {
		buf = buf[:read]
	}
	return detectMimeType(buf, accepted)
}

// DetectMIMEBuffer is like DetectMIME, but accepts a []byte slice already
// loaded into memory.
func DetectMIMEBuffer(buf []byte, accepted map[string]bool) (
	string, string, error,
) {
	if len(buf) > sniffSize {
		buf = buf[:sniffSize]
	}
	return detectMimeType(buf, accepted)
}

// Can be passed either the full read file as []byte or io.ReadSeeker
func detectMimeType(buf []byte, accepted map[string]bool) (
	mime, ext string, err error,
) {
	for _, m := range matchers {
		mime, ext = m.Match(buf)
		if mime != "" {
			break
		}
	}
	switch {
	case mime == "":
		err = UnsupportedMIMEError("application/octet-stream")
	// Check if MIME is accepted, if specified
	case accepted != nil && !accepted[mime]:
		err = UnsupportedMIMEError(mime)
	}
	return
}

func processFile(src Source, opts Options) (Source, Thumbnail, error) {
	override := overrideProcessors[src.Mime]
	if override != nil {
		return override(src, opts)
	}

	switch src.Mime {
	case
		"image/jpeg",
		"image/png",
		"image/gif",
		"image/webp",
		"application/pdf",
		"image/bmp",
		"image/photoshop",
		"image/tiff",
		"image/x-icon":
		return processImage(src, opts)
	case
		"audio/mpeg",
		"audio/aac",
		"audio/wave",
		"audio/x-flac",
		"audio/midi":
		return processAudio(src, opts)
	case
		"application/ogg",
		"video/webm",
		"video/x-matroska",
		"video/mp4",
		"video/avi",
		"video/quicktime",
		"video/x-ms-wmv",
		"video/x-flv":
		return processVideo(src, opts)
	default:
		return src, Thumbnail{}, UnsupportedMIMEError(src.Mime)
	}
}
<commit_msg>Don't detect with FFmpeg if mp3 is not accepted<commit_after>package thumbnailer

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
)

const sniffSize = 4 << 10

// Matching code partially adapted from "net/http/sniff.go"

// Mime type prefix magic number matchers and canonical extensions
var matchers = []Matcher{
	// Probably most common types, this library will be used for, first.
	// More expensive checks are also positioned lower.
	&exactSig{"jpg", "image/jpeg", []byte("\xFF\xD8\xFF")},
	&exactSig{"png", "image/png", []byte("\x89\x50\x4E\x47\x0D\x0A\x1A\x0A")},
	&exactSig{"gif", "image/gif", []byte("GIF87a")},
	&exactSig{"gif", "image/gif", []byte("GIF89a")},
	&maskedSig{
		"webp",
		"image/webp",
		[]byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF"),
		[]byte("RIFF\x00\x00\x00\x00WEBPVP"),
	},
	&maskedSig{
		"ogg",
		"application/ogg",
		[]byte("OggS\x00"),
		[]byte("\x4F\x67\x67\x53\x00"),
	},
	MatcherFunc(matchWebmOrMKV),
	&exactSig{"pdf", "application/pdf", []byte("%PDF-")},
	&maskedSig{
		"mp3",
		"audio/mpeg",
		[]byte("\xFF\xFF\xFF"),
		[]byte("ID3"),
	},
	MatcherFunc(matchMP4),
	&exactSig{"aac", "audio/aac", []byte("ÿñ")},
	&exactSig{"aac", "audio/aac", []byte("ÿù")},
	&exactSig{"bmp", "image/bmp", []byte("BM")},
	&maskedSig{
		"wav",
		"audio/wave",
		[]byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"),
		[]byte("RIFF\x00\x00\x00\x00WAVE"),
	},
	&maskedSig{
		"avi",
		"video/avi",
		[]byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"),
		[]byte("RIFF\x00\x00\x00\x00AVI "),
	},
	&exactSig{"psd", "image/photoshop", []byte("8BPS")},
	&exactSig{"flac", "audio/x-flac", []byte("fLaC")},
	&exactSig{"tiff", "image/tiff", []byte("II*\x00")},
	&exactSig{"tiff", "image/tiff", []byte("MM\x00*")},
	&exactSig{"mov", "video/quicktime", []byte("\x00\x00\x00\x14ftyp")},
	&exactSig{
		"wmv",
		"video/x-ms-wmv",
		[]byte{0x30, 0x26, 0xB2, 0x75, 0x8E, 0x66, 0xCF, 0x11, 0xA6, 0xD9},
	},
	&exactSig{"flv", "video/x-flv", []byte("FLV\x01")},
	&exactSig{"ico", "image/x-icon", []byte("\x00\x00\x01\x00")},
	&maskedSig{
		"midi",
		"audio/midi",
		[]byte("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"),
		[]byte("MThd\x00\x00\x00\x06"),
	},
}

var (
	// User-defined MIME type processors
	overrideProcessors = map[string]Processor{}
)

// Processor is a specialized file processor for a specific file type
type Processor func(Source, Options) (Source, Thumbnail, error)

// Matcher takes up to the first 512 bytes of a file and returns the MIME type
// and canonical extension, that were matched. Empty string indicates no match.
type Matcher interface {
	Match([]byte) (mime string, extension string)
}

// MatcherFunc is an adapter that allows using functions as Matcher
type MatcherFunc func([]byte) (string, string)

// Match implements Matcher
func (fn MatcherFunc) Match(data []byte) (string, string) {
	return fn(data)
}

type exactSig struct {
	ext, mime string
	sig       []byte
}

func (e *exactSig) Match(data []byte) (string, string) {
	if bytes.HasPrefix(data, e.sig) {
		return e.mime, e.ext
	}
	return "", ""
}

type maskedSig struct {
	ext, mime string
	mask, pat []byte
}

func (m *maskedSig) Match(data []byte) (string, string) {
	if len(data) < len(m.mask) {
		return "", ""
	}
	for i, mask := range m.mask {
		db := data[i] & mask
		if db != m.pat[i] {
			return "", ""
		}
	}
	return m.mime, m.ext
}

func matchWebmOrMKV(data []byte) (string, string) {
	switch {
	case len(data) < 8 || !bytes.HasPrefix(data, []byte("\x1A\x45\xDF\xA3")):
		return "", ""
	case bytes.Contains(data[4:], []byte("webm")):
		return "video/webm", "webm"
	case bytes.Contains(data[4:], []byte("matroska")):
		return "video/x-matroska", "mkv"
	default:
		return "", ""
	}
}

func matchMP4(data []byte) (string, string) {
	if len(data) < 12 {
		return "", ""
	}

	boxSize := int(binary.BigEndian.Uint32(data[:4]))
	nope := boxSize%4 != 0 ||
		len(data) < boxSize ||
		!bytes.Equal(data[4:8], []byte("ftyp"))
	if nope {
		return "", ""
	}

	for st := 8; st < boxSize; st += 4 {
		if st == 12 {
			// minor version number
			continue
		}
		if bytes.Equal(data[st:st+3], []byte("mp4")) {
			return "video/mp4", "mp4"
		}
	}
	return "", ""
}

// MP3 is a retarded standard, that will not always even have a magic number.
// Need to detect with FFMPEG as a last resort.
func matchMP3(data []byte) (mime string, ext string) {
	c, err := NewFFContext(bytes.NewReader(data))
	if err != nil {
		return
	}
	defer c.Close()

	codec, err := c.CodecName(FFAudio)
	if err != nil {
		return
	}
	if codec == "mp3" {
		return "audio/mpeg", "mp3"
	}
	return
}

// UnsupportedMIMEError indicates the MIME type of the file could not be
// detected as a supported type or was not in the AcceptedMimeTypes list, if
// defined.
type UnsupportedMIMEError string

func (u UnsupportedMIMEError) Error() string {
	return fmt.Sprintf("unsupported MIME type: %s", string(u))
}

// RegisterMatcher adds an extra magic prefix-based MIME type matcher to the
// default set with an included canonical file extension.
// Not safe to use concurrently with file processing.
func RegisterMatcher(m Matcher) {
	matchers = append(matchers, m)
}

// RegisterProcessor registers a file processor for a specific MIME type.
// Can be used to add support for additional MIME types or as an override.
// Not safe to use concurrently with file processing.
func RegisterProcessor(mime string, fn Processor) {
	overrideProcessors[mime] = fn
}

// DetectMIME  detects the MIME typ of the r. r must be at starting position.
// accepted, if not nil, specifies MIME types to not reject with
// UnsupportedMIMEError.
func DetectMIME(r io.Reader, accepted map[string]bool) (string, string, error) {
	buf := make([]byte, sniffSize)
	read, err := r.Read(buf)
	if err != nil {
		return "", "", err
	}
	if read < sniffSize {
		buf = buf[:read]
	}
	return detectMimeType(buf, accepted)
}

// DetectMIMEBuffer is like DetectMIME, but accepts a []byte slice already
// loaded into memory.
func DetectMIMEBuffer(buf []byte, accepted map[string]bool) (
	string, string, error,
) {
	if len(buf) > sniffSize {
		buf = buf[:sniffSize]
	}
	return detectMimeType(buf, accepted)
}

// Can be passed either the full read file as []byte or io.ReadSeeker
func detectMimeType(buf []byte, accepted map[string]bool) (
	mime, ext string, err error,
) {
	for _, m := range matchers {
		mime, ext = m.Match(buf)
		if mime != "" {
			break
		}
	}

	if mime == "" {
		if accepted == nil || accepted["audio/mpeg"] {
			mime, ext = matchMP3(buf)
		}
	}

	switch {
	case mime == "":
		err = UnsupportedMIMEError("application/octet-stream")
	// Check if MIME is accepted, if specified
	case accepted != nil && !accepted[mime]:
		err = UnsupportedMIMEError(mime)
	}
	return
}

func processFile(src Source, opts Options) (Source, Thumbnail, error) {
	override := overrideProcessors[src.Mime]
	if override != nil {
		return override(src, opts)
	}

	switch src.Mime {
	case
		"image/jpeg",
		"image/png",
		"image/gif",
		"image/webp",
		"application/pdf",
		"image/bmp",
		"image/photoshop",
		"image/tiff",
		"image/x-icon":
		return processImage(src, opts)
	case
		"audio/mpeg",
		"audio/aac",
		"audio/wave",
		"audio/x-flac",
		"audio/midi":
		return processAudio(src, opts)
	case
		"application/ogg",
		"video/webm",
		"video/x-matroska",
		"video/mp4",
		"video/avi",
		"video/quicktime",
		"video/x-ms-wmv",
		"video/x-flv":
		return processVideo(src, opts)
	default:
		return src, Thumbnail{}, UnsupportedMIMEError(src.Mime)
	}
}
<|endoftext|>


<commit_before>// ----------------------------------------------------------------------------
//
//     This file is copied here by Magic Modules. The code for building up a
//     storage bucket object is copied from the manually implemented
//     provider file:
//     third_party/terraform/resources/resource_storage_bucket.go
//
// ----------------------------------------------------------------------------
package google

import (
	"fmt"

	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"

	"google.golang.org/api/storage/v1"
)

func GetStorageBucketCaiObject(d TerraformResourceData, config *Config) ([]Asset, error) {
	name, err := assetName(d, config, "//storage.googleapis.com/{{name}}")
	if err != nil {
		return []Asset{}, err
	}
	if obj, err := GetStorageBucketApiObject(d, config); err == nil {
		return []Asset{{
			Name: name,
			Type: "storage.googleapis.com/Bucket",
			Resource: &AssetResource{
				Version:              "v1",
				DiscoveryDocumentURI: "https://www.googleapis.com/discovery/v1/apis/storage/v1/rest",
				DiscoveryName:        "Bucket",
				Data:                 obj,
			},
		}}, nil
	} else {
		return []Asset{}, err
	}
}

func GetStorageBucketApiObject(d TerraformResourceData, config *Config) (map[string]interface{}, error) {
	project, err := getProject(d, config)
	if err != nil {
		return nil, err
	}

	// Get the bucket and location
	bucket := d.Get("name").(string)
	location := d.Get("location").(string)

	// Create a bucket, setting the labels, location and name.
	sb := &storage.Bucket{
		Name:             bucket,
		Labels:           expandLabels(d),
		Location:         location,
		IamConfiguration: expandIamConfiguration(d),
	}

	if v, ok := d.GetOk("storage_class"); ok {
		sb.StorageClass = v.(string)
	}

	if err := resourceGCSBucketLifecycleCreateOrUpdate(d, sb); err != nil {
		return nil, err
	}

	if v, ok := d.GetOk("versioning"); ok {
		sb.Versioning = expandBucketVersioning(v)
	}

	if v, ok := d.GetOk("website"); ok {
		sb.Website = expandBucketWebsite(v.([]interface{}))
	}

	if v, ok := d.GetOk("retention_policy"); ok {
		sb.RetentionPolicy = expandBucketRetentionPolicy(v.([]interface{}))
	}

	if v, ok := d.GetOk("cors"); ok {
		sb.Cors = expandCors(v.([]interface{}))
	}

	if v, ok := d.GetOk("default_event_based_hold"); ok {
		sb.DefaultEventBasedHold = v.(bool)
	}

	if v, ok := d.GetOk("logging"); ok {
		sb.Logging = expandBucketLogging(v.([]interface{}))
	}

	if v, ok := d.GetOk("encryption"); ok {
		sb.Encryption = expandBucketEncryption(v.([]interface{}))
	}

	if v, ok := d.GetOk("requester_pays"); ok {
		sb.Billing = &storage.BucketBilling{
			RequesterPays: v.(bool),
		}
	}

	m, err := jsonMap(sb)
	if err != nil {
		return nil, err
	}
	m["project"] = project

	return m, nil
}

func expandCors(configured []interface{}) []*storage.BucketCors {
	corsRules := make([]*storage.BucketCors, 0, len(configured))
	for _, raw := range configured {
		data := raw.(map[string]interface{})
		corsRule := storage.BucketCors{
			Origin:         convertStringArr(data["origin"].([]interface{})),
			Method:         convertStringArr(data["method"].([]interface{})),
			ResponseHeader: convertStringArr(data["response_header"].([]interface{})),
			MaxAgeSeconds:  int64(data["max_age_seconds"].(int)),
		}

		corsRules = append(corsRules, &corsRule)
	}
	return corsRules
}

func expandBucketEncryption(configured interface{}) *storage.BucketEncryption {
	encs := configured.([]interface{})
	if len(encs) == 0 || encs[0] == nil {
		return nil
	}
	enc := encs[0].(map[string]interface{})
	keyname := enc["default_kms_key_name"]
	if keyname == nil || keyname.(string) == "" {
		return nil
	}
	bucketenc := &storage.BucketEncryption{
		DefaultKmsKeyName: keyname.(string),
	}
	return bucketenc
}

func expandBucketLogging(configured interface{}) *storage.BucketLogging {
	loggings := configured.([]interface{})
	if len(loggings) == 0 {
		return nil
	}

	logging := loggings[0].(map[string]interface{})

	bucketLogging := &storage.BucketLogging{
		LogBucket:       logging["log_bucket"].(string),
		LogObjectPrefix: logging["log_object_prefix"].(string),
	}

	return bucketLogging
}

func expandBucketVersioning(configured interface{}) *storage.BucketVersioning {
	versionings := configured.([]interface{})
	if len(versionings) == 0 {
		return nil
	}

	versioning := versionings[0].(map[string]interface{})

	bucketVersioning := &storage.BucketVersioning{}

	bucketVersioning.Enabled = versioning["enabled"].(bool)
	bucketVersioning.ForceSendFields = append(bucketVersioning.ForceSendFields, "Enabled")

	return bucketVersioning
}

func expandBucketWebsite(v interface{}) *storage.BucketWebsite {
	if v == nil {
		return nil
	}
	vs := v.([]interface{})

	if len(vs) < 1 || vs[0] == nil {
		return nil
	}

	website := vs[0].(map[string]interface{})
	w := &storage.BucketWebsite{}

	if v := website["not_found_page"]; v != "" {
		w.NotFoundPage = v.(string)
	}

	if v := website["main_page_suffix"]; v != "" {
		w.MainPageSuffix = v.(string)
	}

	return w
}

func expandBucketRetentionPolicy(configured interface{}) *storage.BucketRetentionPolicy {
	retentionPolicies := configured.([]interface{})
	if len(retentionPolicies) == 0 {
		return nil
	}
	retentionPolicy := retentionPolicies[0].(map[string]interface{})

	bucketRetentionPolicy := &storage.BucketRetentionPolicy{
		RetentionPeriod: int64(retentionPolicy["retention_period"].(int)),
	}

	return bucketRetentionPolicy
}

func resourceGCSBucketLifecycleCreateOrUpdate(d TerraformResourceData, sb *storage.Bucket) error {
	if v, ok := d.GetOk("lifecycle_rule"); ok {
		lifecycle_rules := v.([]interface{})

		sb.Lifecycle = &storage.BucketLifecycle{}
		sb.Lifecycle.Rule = make([]*storage.BucketLifecycleRule, 0, len(lifecycle_rules))

		for _, raw_lifecycle_rule := range lifecycle_rules {
			lifecycle_rule := raw_lifecycle_rule.(map[string]interface{})

			target_lifecycle_rule := &storage.BucketLifecycleRule{}

			if v, ok := lifecycle_rule["action"]; ok {
				if actions := v.(*schema.Set).List(); len(actions) == 1 {
					action := actions[0].(map[string]interface{})

					target_lifecycle_rule.Action = &storage.BucketLifecycleRuleAction{}

					if v, ok := action["type"]; ok {
						target_lifecycle_rule.Action.Type = v.(string)
					}

					if v, ok := action["storage_class"]; ok {
						target_lifecycle_rule.Action.StorageClass = v.(string)
					}
				} else {
					return fmt.Errorf("Exactly one action is required")
				}
			}

			if v, ok := lifecycle_rule["condition"]; ok {
				if conditions := v.(*schema.Set).List(); len(conditions) == 1 {
					condition := conditions[0].(map[string]interface{})

					target_lifecycle_rule.Condition = &storage.BucketLifecycleRuleCondition{}

					if v, ok := condition["age"]; ok {
						target_lifecycle_rule.Condition.Age = int64(v.(int))
					}

					if v, ok := condition["created_before"]; ok {
						target_lifecycle_rule.Condition.CreatedBefore = v.(string)
					}

					if v, ok := condition["matches_storage_class"]; ok {
						matches_storage_classes := v.([]interface{})

						target_matches_storage_classes := make([]string, 0, len(matches_storage_classes))

						for _, v := range matches_storage_classes {
							target_matches_storage_classes = append(target_matches_storage_classes, v.(string))
						}

						target_lifecycle_rule.Condition.MatchesStorageClass = target_matches_storage_classes
					}

					if v, ok := condition["num_newer_versions"]; ok {
						target_lifecycle_rule.Condition.NumNewerVersions = int64(v.(int))
					}
				} else {
					return fmt.Errorf("Exactly one condition is required")
				}
			}

			sb.Lifecycle.Rule = append(sb.Lifecycle.Rule, target_lifecycle_rule)
		}
	} else {
		sb.Lifecycle = &storage.BucketLifecycle{
			ForceSendFields: []string{"Rule"},
		}
	}

	return nil
}

// remove this on next major release of the provider.
func expandIamConfiguration(d TerraformResourceData) *storage.BucketIamConfiguration {
	// We are checking for a change because the last else block is only executed on Create.
	enabled := false
	if d.HasChange("bucket_policy_only") {
		enabled = d.Get("bucket_policy_only").(bool)
	} else if d.HasChange("uniform_bucket_level_access") {
		enabled = d.Get("uniform_bucket_level_access").(bool)
	} else {
		enabled = d.Get("bucket_policy_only").(bool) || d.Get("uniform_bucket_level_access").(bool)
	}

	return &storage.BucketIamConfiguration{
		ForceSendFields: []string{"UniformBucketLevelAccess"},
		UniformBucketLevelAccess: &storage.BucketIamConfigurationUniformBucketLevelAccess{
			Enabled:         enabled,
			ForceSendFields: []string{"Enabled"},
		},
	}
}

// Uncomment once the previous function is removed.
// func expandIamConfiguration(d TerraformResourceData) *storage.BucketIamConfiguration {
// 	return &storage.BucketIamConfiguration{
// 		ForceSendFields: []string{"UniformBucketLevelAccess"},
// 		UniformBucketLevelAccess: &storage.BucketIamConfigurationUniformBucketLevelAccess{
// 			Enabled:         d.Get("uniform_bucket_level_access").(bool),
// 			ForceSendFields: []string{"Enabled"},
// 		},
// 	}
// }
<commit_msg>add handling of empty versioning and cors in hand written storage bucket (#4608)<commit_after>// ----------------------------------------------------------------------------
//
//     This file is copied here by Magic Modules. The code for building up a
//     storage bucket object is copied from the manually implemented
//     provider file:
//     third_party/terraform/resources/resource_storage_bucket.go
//
// ----------------------------------------------------------------------------
package google

import (
	"fmt"

	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"

	"google.golang.org/api/storage/v1"
)

func GetStorageBucketCaiObject(d TerraformResourceData, config *Config) ([]Asset, error) {
	name, err := assetName(d, config, "//storage.googleapis.com/{{name}}")
	if err != nil {
		return []Asset{}, err
	}
	if obj, err := GetStorageBucketApiObject(d, config); err == nil {
		return []Asset{{
			Name: name,
			Type: "storage.googleapis.com/Bucket",
			Resource: &AssetResource{
				Version:              "v1",
				DiscoveryDocumentURI: "https://www.googleapis.com/discovery/v1/apis/storage/v1/rest",
				DiscoveryName:        "Bucket",
				Data:                 obj,
			},
		}}, nil
	} else {
		return []Asset{}, err
	}
}

func GetStorageBucketApiObject(d TerraformResourceData, config *Config) (map[string]interface{}, error) {
	project, err := getProject(d, config)
	if err != nil {
		return nil, err
	}

	// Get the bucket and location
	bucket := d.Get("name").(string)
	location := d.Get("location").(string)

	// Create a bucket, setting the labels, location and name.
	sb := &storage.Bucket{
		Name:             bucket,
		Labels:           expandLabels(d),
		Location:         location,
		IamConfiguration: expandIamConfiguration(d),
	}

	if v, ok := d.GetOk("storage_class"); ok {
		sb.StorageClass = v.(string)
	}

	if err := resourceGCSBucketLifecycleCreateOrUpdate(d, sb); err != nil {
		return nil, err
	}

	if v, ok := d.GetOk("versioning"); ok {
		sb.Versioning = expandBucketVersioning(v)
	}

	if v, ok := d.GetOk("website"); ok {
		sb.Website = expandBucketWebsite(v.([]interface{}))
	}

	if v, ok := d.GetOk("retention_policy"); ok {
		sb.RetentionPolicy = expandBucketRetentionPolicy(v.([]interface{}))
	}

	if v, ok := d.GetOk("cors"); ok {
		sb.Cors = expandCors(v.([]interface{}))
	}

	if v, ok := d.GetOk("default_event_based_hold"); ok {
		sb.DefaultEventBasedHold = v.(bool)
	}

	if v, ok := d.GetOk("logging"); ok {
		sb.Logging = expandBucketLogging(v.([]interface{}))
	}

	if v, ok := d.GetOk("encryption"); ok {
		sb.Encryption = expandBucketEncryption(v.([]interface{}))
	}

	if v, ok := d.GetOk("requester_pays"); ok {
		sb.Billing = &storage.BucketBilling{
			RequesterPays: v.(bool),
		}
	}

	m, err := jsonMap(sb)
	if err != nil {
		return nil, err
	}
	m["project"] = project

	return m, nil
}

func expandCors(configured []interface{}) []*storage.BucketCors {
	if len(configured) == 0 || configured[0] == nil {
		return nil
	}
	corsRules := make([]*storage.BucketCors, 0, len(configured))
	for _, raw := range configured {
		data := raw.(map[string]interface{})
		corsRule := storage.BucketCors{
			Origin:         convertStringArr(data["origin"].([]interface{})),
			Method:         convertStringArr(data["method"].([]interface{})),
			ResponseHeader: convertStringArr(data["response_header"].([]interface{})),
			MaxAgeSeconds:  int64(data["max_age_seconds"].(int)),
		}

		corsRules = append(corsRules, &corsRule)
	}
	return corsRules
}

func expandBucketEncryption(configured interface{}) *storage.BucketEncryption {
	encs := configured.([]interface{})
	if len(encs) == 0 || encs[0] == nil {
		return nil
	}
	enc := encs[0].(map[string]interface{})
	keyname := enc["default_kms_key_name"]
	if keyname == nil || keyname.(string) == "" {
		return nil
	}
	bucketenc := &storage.BucketEncryption{
		DefaultKmsKeyName: keyname.(string),
	}
	return bucketenc
}

func expandBucketLogging(configured interface{}) *storage.BucketLogging {
	loggings := configured.([]interface{})
	if len(loggings) == 0 {
		return nil
	}

	logging := loggings[0].(map[string]interface{})

	bucketLogging := &storage.BucketLogging{
		LogBucket:       logging["log_bucket"].(string),
		LogObjectPrefix: logging["log_object_prefix"].(string),
	}

	return bucketLogging
}

func expandBucketVersioning(configured interface{}) *storage.BucketVersioning {
	versionings := configured.([]interface{})
	if len(versionings) == 0 || versionings[0] == nil  {
		return nil
	}

	versioning := versionings[0].(map[string]interface{})

	bucketVersioning := &storage.BucketVersioning{}

	bucketVersioning.Enabled = versioning["enabled"].(bool)
	bucketVersioning.ForceSendFields = append(bucketVersioning.ForceSendFields, "Enabled")

	return bucketVersioning
}

func expandBucketWebsite(v interface{}) *storage.BucketWebsite {
	if v == nil {
		return nil
	}
	vs := v.([]interface{})

	if len(vs) < 1 || vs[0] == nil {
		return nil
	}

	website := vs[0].(map[string]interface{})
	w := &storage.BucketWebsite{}

	if v := website["not_found_page"]; v != "" {
		w.NotFoundPage = v.(string)
	}

	if v := website["main_page_suffix"]; v != "" {
		w.MainPageSuffix = v.(string)
	}

	return w
}

func expandBucketRetentionPolicy(configured interface{}) *storage.BucketRetentionPolicy {
	retentionPolicies := configured.([]interface{})
	if len(retentionPolicies) == 0 {
		return nil
	}
	retentionPolicy := retentionPolicies[0].(map[string]interface{})

	bucketRetentionPolicy := &storage.BucketRetentionPolicy{
		RetentionPeriod: int64(retentionPolicy["retention_period"].(int)),
	}

	return bucketRetentionPolicy
}

func resourceGCSBucketLifecycleCreateOrUpdate(d TerraformResourceData, sb *storage.Bucket) error {
	if v, ok := d.GetOk("lifecycle_rule"); ok {
		lifecycle_rules := v.([]interface{})

		sb.Lifecycle = &storage.BucketLifecycle{}
		sb.Lifecycle.Rule = make([]*storage.BucketLifecycleRule, 0, len(lifecycle_rules))

		for _, raw_lifecycle_rule := range lifecycle_rules {
			lifecycle_rule := raw_lifecycle_rule.(map[string]interface{})

			target_lifecycle_rule := &storage.BucketLifecycleRule{}

			if v, ok := lifecycle_rule["action"]; ok {
				if actions := v.(*schema.Set).List(); len(actions) == 1 {
					action := actions[0].(map[string]interface{})

					target_lifecycle_rule.Action = &storage.BucketLifecycleRuleAction{}

					if v, ok := action["type"]; ok {
						target_lifecycle_rule.Action.Type = v.(string)
					}

					if v, ok := action["storage_class"]; ok {
						target_lifecycle_rule.Action.StorageClass = v.(string)
					}
				} else {
					return fmt.Errorf("Exactly one action is required")
				}
			}

			if v, ok := lifecycle_rule["condition"]; ok {
				if conditions := v.(*schema.Set).List(); len(conditions) == 1 {
					condition := conditions[0].(map[string]interface{})

					target_lifecycle_rule.Condition = &storage.BucketLifecycleRuleCondition{}

					if v, ok := condition["age"]; ok {
						target_lifecycle_rule.Condition.Age = int64(v.(int))
					}

					if v, ok := condition["created_before"]; ok {
						target_lifecycle_rule.Condition.CreatedBefore = v.(string)
					}

					if v, ok := condition["matches_storage_class"]; ok {
						matches_storage_classes := v.([]interface{})

						target_matches_storage_classes := make([]string, 0, len(matches_storage_classes))

						for _, v := range matches_storage_classes {
							target_matches_storage_classes = append(target_matches_storage_classes, v.(string))
						}

						target_lifecycle_rule.Condition.MatchesStorageClass = target_matches_storage_classes
					}

					if v, ok := condition["num_newer_versions"]; ok {
						target_lifecycle_rule.Condition.NumNewerVersions = int64(v.(int))
					}
				} else {
					return fmt.Errorf("Exactly one condition is required")
				}
			}

			sb.Lifecycle.Rule = append(sb.Lifecycle.Rule, target_lifecycle_rule)
		}
	} else {
		sb.Lifecycle = &storage.BucketLifecycle{
			ForceSendFields: []string{"Rule"},
		}
	}

	return nil
}

// remove this on next major release of the provider.
func expandIamConfiguration(d TerraformResourceData) *storage.BucketIamConfiguration {
	// We are checking for a change because the last else block is only executed on Create.
	enabled := false
	if d.HasChange("bucket_policy_only") {
		enabled = d.Get("bucket_policy_only").(bool)
	} else if d.HasChange("uniform_bucket_level_access") {
		enabled = d.Get("uniform_bucket_level_access").(bool)
	} else {
		enabled = d.Get("bucket_policy_only").(bool) || d.Get("uniform_bucket_level_access").(bool)
	}

	return &storage.BucketIamConfiguration{
		ForceSendFields: []string{"UniformBucketLevelAccess"},
		UniformBucketLevelAccess: &storage.BucketIamConfigurationUniformBucketLevelAccess{
			Enabled:         enabled,
			ForceSendFields: []string{"Enabled"},
		},
	}
}

// Uncomment once the previous function is removed.
// func expandIamConfiguration(d TerraformResourceData) *storage.BucketIamConfiguration {
// 	return &storage.BucketIamConfiguration{
// 		ForceSendFields: []string{"UniformBucketLevelAccess"},
// 		UniformBucketLevelAccess: &storage.BucketIamConfigurationUniformBucketLevelAccess{
// 			Enabled:         d.Get("uniform_bucket_level_access").(bool),
// 			ForceSendFields: []string{"Enabled"},
// 		},
// 	}
// }
<|endoftext|>


<commit_before>// Copyright 2017 Monax Industries Limited
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package integration

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"sync/atomic"
	"testing"
	"time"

	"github.com/stretchr/testify/require"

	"github.com/hyperledger/burrow/acm"
	"github.com/hyperledger/burrow/acm/validator"
	"github.com/hyperledger/burrow/config"
	"github.com/hyperledger/burrow/consensus/tendermint"
	"github.com/hyperledger/burrow/core"
	"github.com/hyperledger/burrow/execution"
	"github.com/hyperledger/burrow/genesis"
	"github.com/hyperledger/burrow/keys/mock"
	"github.com/hyperledger/burrow/logging"
	"github.com/hyperledger/burrow/logging/logconfig"
	lConfig "github.com/hyperledger/burrow/logging/logconfig"
	"github.com/hyperledger/burrow/permission"
	"github.com/hyperledger/burrow/rpc"
)

const (
	ChainName  = "Integration_Test_Chain"
	scratchDir = "test_scratch"
)

// Enable logger output during tests

var node uint64 = 0

func NoConsensus(conf *config.BurrowConfig) {
	conf.Tendermint.Enabled = false
}

func CommitImmediately(conf *config.BurrowConfig) {
	conf.Execution.TimeoutFactor = 0
}

func RunNode(t testing.TB, genesisDoc *genesis.GenesisDoc, privateAccounts []*acm.PrivateAccount,
	options ...func(*config.BurrowConfig)) (kern *core.Kernel, shutdown func()) {

	var err error
	var loggingConfig *logconfig.LoggingConfig

	testConfig, cleanup := NewTestConfig(genesisDoc, options...)
	// Uncomment for log output from tests
	//loggingConfig = logconfig.New().Root(func(sink *logconfig.SinkConfig) *logconfig.SinkConfig {
	//	return sink.SetOutput(logconfig.StderrOutput())
	//})
	kern, err = TestKernel(privateAccounts[0], privateAccounts, testConfig, loggingConfig)
	require.NoError(t, err)
	err = kern.Boot()
	require.NoError(t, err)
	// Sometimes better to not shutdown as logging errors on shutdown may obscure real issue
	return kern, func() {
		Shutdown(kern)
		cleanup()
	}
}

func NewTestConfig(genesisDoc *genesis.GenesisDoc,
	options ...func(*config.BurrowConfig)) (conf *config.BurrowConfig, cleanup func()) {

	nodeNumber := atomic.AddUint64(&node, 1)
	name := fmt.Sprintf("node_%03d", nodeNumber)
	conf = config.DefaultBurrowConfig()
	testDir, cleanup := EnterTestDirectory()
	conf.BurrowDir = path.Join(testDir, fmt.Sprintf(".burrow_%s", name))
	conf.GenesisDoc = genesisDoc
	conf.Tendermint.Moniker = name
	// Make blocks for purposes of tests
	conf.Tendermint.CreateEmptyBlocks = tendermint.AlwaysCreateEmptyBlocks
	conf.Keys.RemoteAddress = ""
	// Assign run of ports
	const freeport = "0"
	conf.Tendermint.ListenHost = rpc.LocalHost
	conf.Tendermint.ListenPort = freeport
	conf.RPC.GRPC.ListenHost = rpc.LocalHost
	conf.RPC.GRPC.ListenPort = freeport
	conf.RPC.Metrics.ListenHost = rpc.LocalHost
	conf.RPC.Metrics.ListenPort = freeport
	conf.RPC.Info.ListenHost = rpc.LocalHost
	conf.RPC.Info.ListenPort = freeport
	conf.Execution.TimeoutFactor = 0.5
	conf.Execution.VMOptions = []execution.VMOption{execution.DebugOpcodes}
	for _, opt := range options {
		if opt != nil {
			opt(conf)
		}
	}
	return conf, cleanup
}

// We use this to wrap tests
func TestKernel(validatorAccount *acm.PrivateAccount, keysAccounts []*acm.PrivateAccount,
	testConfig *config.BurrowConfig, loggingConfig *lConfig.LoggingConfig) (*core.Kernel, error) {
	fmt.Println("Creating integration test Kernel...")

	kern, err := core.NewKernel(testConfig.BurrowDir)
	if err != nil {
		return nil, err
	}

	kern.SetLogger(logging.NewNoopLogger())
	if loggingConfig != nil {
		err := kern.LoadLoggerFromConfig(loggingConfig)
		if err != nil {
			return nil, err
		}
	}

	kern.SetKeyClient(mock.NewKeyClient(keysAccounts...))

	err = kern.LoadExecutionOptionsFromConfig(testConfig.Execution)
	if err != nil {
		return nil, err
	}

	err = kern.LoadState(testConfig.GenesisDoc)
	if err != nil {
		return nil, err
	}

	privVal := tendermint.NewPrivValidatorMemory(validatorAccount, validatorAccount)

	err = kern.LoadTendermintFromConfig(testConfig, privVal)
	if err != nil {
		return nil, err
	}

	kern.AddProcesses(core.DefaultProcessLaunchers(kern, testConfig.RPC, testConfig.Keys)...)
	return kern, nil
}

func EnterTestDirectory() (testDir string, cleanup func()) {
	var err error
	testDir, err = ioutil.TempDir("", scratchDir)
	if err != nil {
		panic(fmt.Errorf("could not make temp dir for integration tests: %v", err))
	}
	// If you need to inspectdirs
	//testDir := scratchDir
	os.RemoveAll(testDir)
	os.MkdirAll(testDir, 0777)
	os.Chdir(testDir)
	os.MkdirAll("config", 0777)
	return testDir, func() { os.RemoveAll(testDir) }
}

// TestGenesisDoc creates genesis from a set of accounts
// and validators from indices within that slice
func TestGenesisDoc(addressables []*acm.PrivateAccount, vals ...int) *genesis.GenesisDoc {
	accounts := make(map[string]*acm.Account, len(addressables))
	for i, pa := range addressables {
		account := acm.FromAddressable(pa)
		account.Balance += 1 << 32
		account.Permissions = permission.AllAccountPermissions.Clone()
		accounts[fmt.Sprintf("user_%v", i)] = account
	}
	genesisTime, err := time.Parse("02-01-2006", "27-10-2017")
	if err != nil {
		panic("could not parse test genesis time")
	}

	validators := make(map[string]*validator.Validator)
	for _, i := range vals {
		name := fmt.Sprintf("user_%d", i)
		validators[name] = validator.FromAccount(accounts[name], 1<<16)
	}

	return genesis.MakeGenesisDocFromAccounts(ChainName, nil, genesisTime, accounts, validators)
}

// Deterministic account generation helper. Pass number of accounts to make
func MakePrivateAccounts(sec string, n int) []*acm.PrivateAccount {
	accounts := make([]*acm.PrivateAccount, n)
	for i := 0; i < n; i++ {
		accounts[i] = acm.GeneratePrivateAccountFromSecret(sec + strconv.Itoa(i))
	}
	return accounts
}

func Shutdown(kern *core.Kernel) {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	err := kern.Shutdown(ctx)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error shutting down test kernel %v: %v", kern, err)
	}
}
<commit_msg>do not print every evm opcode in integration tests<commit_after>// Copyright 2017 Monax Industries Limited
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package integration

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"sync/atomic"
	"testing"
	"time"

	"github.com/stretchr/testify/require"

	"github.com/hyperledger/burrow/acm"
	"github.com/hyperledger/burrow/acm/validator"
	"github.com/hyperledger/burrow/config"
	"github.com/hyperledger/burrow/consensus/tendermint"
	"github.com/hyperledger/burrow/core"
	"github.com/hyperledger/burrow/execution"
	"github.com/hyperledger/burrow/genesis"
	"github.com/hyperledger/burrow/keys/mock"
	"github.com/hyperledger/burrow/logging"
	"github.com/hyperledger/burrow/logging/logconfig"
	lConfig "github.com/hyperledger/burrow/logging/logconfig"
	"github.com/hyperledger/burrow/permission"
	"github.com/hyperledger/burrow/rpc"
)

const (
	ChainName  = "Integration_Test_Chain"
	scratchDir = "test_scratch"
)

// Enable logger output during tests

var node uint64 = 0

func NoConsensus(conf *config.BurrowConfig) {
	conf.Tendermint.Enabled = false
}

func CommitImmediately(conf *config.BurrowConfig) {
	conf.Execution.TimeoutFactor = 0
}

func RunNode(t testing.TB, genesisDoc *genesis.GenesisDoc, privateAccounts []*acm.PrivateAccount,
	options ...func(*config.BurrowConfig)) (kern *core.Kernel, shutdown func()) {

	var err error
	var loggingConfig *logconfig.LoggingConfig

	testConfig, cleanup := NewTestConfig(genesisDoc, options...)
	// Uncomment for log output from tests
	//loggingConfig = logconfig.New().Root(func(sink *logconfig.SinkConfig) *logconfig.SinkConfig {
	//	return sink.SetOutput(logconfig.StderrOutput())
	//})
	kern, err = TestKernel(privateAccounts[0], privateAccounts, testConfig, loggingConfig)
	require.NoError(t, err)
	err = kern.Boot()
	require.NoError(t, err)
	// Sometimes better to not shutdown as logging errors on shutdown may obscure real issue
	return kern, func() {
		Shutdown(kern)
		cleanup()
	}
}

func NewTestConfig(genesisDoc *genesis.GenesisDoc,
	options ...func(*config.BurrowConfig)) (conf *config.BurrowConfig, cleanup func()) {

	nodeNumber := atomic.AddUint64(&node, 1)
	name := fmt.Sprintf("node_%03d", nodeNumber)
	conf = config.DefaultBurrowConfig()
	testDir, cleanup := EnterTestDirectory()
	conf.BurrowDir = path.Join(testDir, fmt.Sprintf(".burrow_%s", name))
	conf.GenesisDoc = genesisDoc
	conf.Tendermint.Moniker = name
	// Make blocks for purposes of tests
	conf.Tendermint.CreateEmptyBlocks = tendermint.AlwaysCreateEmptyBlocks
	conf.Keys.RemoteAddress = ""
	// Assign run of ports
	const freeport = "0"
	conf.Tendermint.ListenHost = rpc.LocalHost
	conf.Tendermint.ListenPort = freeport
	conf.RPC.GRPC.ListenHost = rpc.LocalHost
	conf.RPC.GRPC.ListenPort = freeport
	conf.RPC.Metrics.ListenHost = rpc.LocalHost
	conf.RPC.Metrics.ListenPort = freeport
	conf.RPC.Info.ListenHost = rpc.LocalHost
	conf.RPC.Info.ListenPort = freeport
	conf.Execution.TimeoutFactor = 0.5
	conf.Execution.VMOptions = []execution.VMOption{}
	for _, opt := range options {
		if opt != nil {
			opt(conf)
		}
	}
	return conf, cleanup
}

// We use this to wrap tests
func TestKernel(validatorAccount *acm.PrivateAccount, keysAccounts []*acm.PrivateAccount,
	testConfig *config.BurrowConfig, loggingConfig *lConfig.LoggingConfig) (*core.Kernel, error) {
	fmt.Println("Creating integration test Kernel...")

	kern, err := core.NewKernel(testConfig.BurrowDir)
	if err != nil {
		return nil, err
	}

	kern.SetLogger(logging.NewNoopLogger())
	if loggingConfig != nil {
		err := kern.LoadLoggerFromConfig(loggingConfig)
		if err != nil {
			return nil, err
		}
	}

	kern.SetKeyClient(mock.NewKeyClient(keysAccounts...))

	err = kern.LoadExecutionOptionsFromConfig(testConfig.Execution)
	if err != nil {
		return nil, err
	}

	err = kern.LoadState(testConfig.GenesisDoc)
	if err != nil {
		return nil, err
	}

	privVal := tendermint.NewPrivValidatorMemory(validatorAccount, validatorAccount)

	err = kern.LoadTendermintFromConfig(testConfig, privVal)
	if err != nil {
		return nil, err
	}

	kern.AddProcesses(core.DefaultProcessLaunchers(kern, testConfig.RPC, testConfig.Keys)...)
	return kern, nil
}

func EnterTestDirectory() (testDir string, cleanup func()) {
	var err error
	testDir, err = ioutil.TempDir("", scratchDir)
	if err != nil {
		panic(fmt.Errorf("could not make temp dir for integration tests: %v", err))
	}
	// If you need to inspectdirs
	//testDir := scratchDir
	os.RemoveAll(testDir)
	os.MkdirAll(testDir, 0777)
	os.Chdir(testDir)
	os.MkdirAll("config", 0777)
	return testDir, func() { os.RemoveAll(testDir) }
}

// TestGenesisDoc creates genesis from a set of accounts
// and validators from indices within that slice
func TestGenesisDoc(addressables []*acm.PrivateAccount, vals ...int) *genesis.GenesisDoc {
	accounts := make(map[string]*acm.Account, len(addressables))
	for i, pa := range addressables {
		account := acm.FromAddressable(pa)
		account.Balance += 1 << 32
		account.Permissions = permission.AllAccountPermissions.Clone()
		accounts[fmt.Sprintf("user_%v", i)] = account
	}
	genesisTime, err := time.Parse("02-01-2006", "27-10-2017")
	if err != nil {
		panic("could not parse test genesis time")
	}

	validators := make(map[string]*validator.Validator)
	for _, i := range vals {
		name := fmt.Sprintf("user_%d", i)
		validators[name] = validator.FromAccount(accounts[name], 1<<16)
	}

	return genesis.MakeGenesisDocFromAccounts(ChainName, nil, genesisTime, accounts, validators)
}

// Deterministic account generation helper. Pass number of accounts to make
func MakePrivateAccounts(sec string, n int) []*acm.PrivateAccount {
	accounts := make([]*acm.PrivateAccount, n)
	for i := 0; i < n; i++ {
		accounts[i] = acm.GeneratePrivateAccountFromSecret(sec + strconv.Itoa(i))
	}
	return accounts
}

func Shutdown(kern *core.Kernel) {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	err := kern.Shutdown(ctx)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error shutting down test kernel %v: %v", kern, err)
	}
}
<|endoftext|>


<commit_before>/*
 *
 * Copyright 2018 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package binarylog

import (
	"bufio"
	"encoding/binary"
	"io"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	pb "google.golang.org/grpc/binarylog/grpc_binarylog_v1"
)

var (
	// DefaultSink is the sink where the logs will be written to. It's exported
	// for the binarylog package to update.
	DefaultSink Sink = &noopSink{} // TODO(blog): change this default (file in /tmp).
)

// Sink writes log entry into the binary log sink.
//
// sink is a copy of the exported binarylog.Sink, to avoid circular dependency.
type Sink interface {
	// Write will be called to write the log entry into the sink.
	//
	// It should be thread-safe so it can be called in parallel.
	Write(*pb.GrpcLogEntry) error
	// Close will be called when the Sink is replaced by a new Sink.
	Close() error
}

type noopSink struct{}

func (ns *noopSink) Write(*pb.GrpcLogEntry) error { return nil }
func (ns *noopSink) Close() error                 { return nil }

// newWriterSink creates a binary log sink with the given writer.
//
// Write() marshals the proto message and writes it to the given writer. Each
// message is prefixed with a 4 byte big endian unsigned integer as the length.
//
// No buffer is done, Close() doesn't try to close the writer.
func newWriterSink(w io.Writer) Sink {
	return &writerSink{out: w}
}

type writerSink struct {
	out io.Writer
}

func (ws *writerSink) Write(e *pb.GrpcLogEntry) error {
	b, err := proto.Marshal(e)
	if err != nil {
		grpclogLogger.Infof("binary logging: failed to marshal proto message: %v", err)
	}
	hdr := make([]byte, 4)
	binary.BigEndian.PutUint32(hdr, uint32(len(b)))
	if _, err := ws.out.Write(hdr); err != nil {
		return err
	}
	if _, err := ws.out.Write(b); err != nil {
		return err
	}
	return nil
}

func (ws *writerSink) Close() error { return nil }

type bufferedSink struct {
	mu     sync.Mutex
	closer io.Closer
	out    Sink          // out is built on buf.
	buf    *bufio.Writer // buf is kept for flush.

	writeStartOnce sync.Once
	writeTicker    *time.Ticker
	done           chan struct{}
}

func (fs *bufferedSink) Write(e *pb.GrpcLogEntry) error {
	// Start the write loop when Write is called.
	fs.writeStartOnce.Do(fs.startFlushGoroutine)
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if err := fs.out.Write(e); err != nil {
		return err
	}
	return nil
}

const (
	bufFlushDuration = 60 * time.Second
)

func (fs *bufferedSink) startFlushGoroutine() {
	fs.writeTicker = time.NewTicker(bufFlushDuration)
	go func() {
		for {
			select {
			case <-fs.done:
				return
			case <-fs.writeTicker.C:
			}
			fs.mu.Lock()
			if err := fs.buf.Flush(); err != nil {
				grpclogLogger.Warningf("failed to flush to Sink: %v", err)
			}
			fs.mu.Unlock()
		}
	}()
}

func (fs *bufferedSink) Close() error {
	if fs.writeTicker != nil {
		fs.writeTicker.Stop()
	}
	close(fs.done)
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if err := fs.buf.Flush(); err != nil {
		grpclogLogger.Warningf("failed to flush to Sink: %v", err)
	}
	if err := fs.closer.Close(); err != nil {
		grpclogLogger.Warningf("failed to close the underlying WriterCloser: %v", err)
	}
	if err := fs.out.Close(); err != nil {
		grpclogLogger.Warningf("failed to close the Sink: %v", err)
	}
	return nil
}

// NewBufferedSink creates a binary log sink with the given WriteCloser.
//
// Write() marshals the proto message and writes it to the given writer. Each
// message is prefixed with a 4 byte big endian unsigned integer as the length.
//
// Content is kept in a buffer, and is flushed every 60 seconds.
//
// Close closes the WriteCloser.
func NewBufferedSink(o io.WriteCloser) Sink {
	bufW := bufio.NewWriter(o)
	return &bufferedSink{
		closer: o,
		out:    newWriterSink(bufW),
		buf:    bufW,
		done:   make(chan struct{}),
	}
}
<commit_msg>binarylog: Don't continue after failing to marshal the proto (#4582)<commit_after>/*
 *
 * Copyright 2018 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package binarylog

import (
	"bufio"
	"encoding/binary"
	"io"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	pb "google.golang.org/grpc/binarylog/grpc_binarylog_v1"
)

var (
	// DefaultSink is the sink where the logs will be written to. It's exported
	// for the binarylog package to update.
	DefaultSink Sink = &noopSink{} // TODO(blog): change this default (file in /tmp).
)

// Sink writes log entry into the binary log sink.
//
// sink is a copy of the exported binarylog.Sink, to avoid circular dependency.
type Sink interface {
	// Write will be called to write the log entry into the sink.
	//
	// It should be thread-safe so it can be called in parallel.
	Write(*pb.GrpcLogEntry) error
	// Close will be called when the Sink is replaced by a new Sink.
	Close() error
}

type noopSink struct{}

func (ns *noopSink) Write(*pb.GrpcLogEntry) error { return nil }
func (ns *noopSink) Close() error                 { return nil }

// newWriterSink creates a binary log sink with the given writer.
//
// Write() marshals the proto message and writes it to the given writer. Each
// message is prefixed with a 4 byte big endian unsigned integer as the length.
//
// No buffer is done, Close() doesn't try to close the writer.
func newWriterSink(w io.Writer) Sink {
	return &writerSink{out: w}
}

type writerSink struct {
	out io.Writer
}

func (ws *writerSink) Write(e *pb.GrpcLogEntry) error {
	b, err := proto.Marshal(e)
	if err != nil {
		grpclogLogger.Errorf("binary logging: failed to marshal proto message: %v", err)
		return err
	}
	hdr := make([]byte, 4)
	binary.BigEndian.PutUint32(hdr, uint32(len(b)))
	if _, err := ws.out.Write(hdr); err != nil {
		return err
	}
	if _, err := ws.out.Write(b); err != nil {
		return err
	}
	return nil
}

func (ws *writerSink) Close() error { return nil }

type bufferedSink struct {
	mu     sync.Mutex
	closer io.Closer
	out    Sink          // out is built on buf.
	buf    *bufio.Writer // buf is kept for flush.

	writeStartOnce sync.Once
	writeTicker    *time.Ticker
	done           chan struct{}
}

func (fs *bufferedSink) Write(e *pb.GrpcLogEntry) error {
	// Start the write loop when Write is called.
	fs.writeStartOnce.Do(fs.startFlushGoroutine)
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if err := fs.out.Write(e); err != nil {
		return err
	}
	return nil
}

const (
	bufFlushDuration = 60 * time.Second
)

func (fs *bufferedSink) startFlushGoroutine() {
	fs.writeTicker = time.NewTicker(bufFlushDuration)
	go func() {
		for {
			select {
			case <-fs.done:
				return
			case <-fs.writeTicker.C:
			}
			fs.mu.Lock()
			if err := fs.buf.Flush(); err != nil {
				grpclogLogger.Warningf("failed to flush to Sink: %v", err)
			}
			fs.mu.Unlock()
		}
	}()
}

func (fs *bufferedSink) Close() error {
	if fs.writeTicker != nil {
		fs.writeTicker.Stop()
	}
	close(fs.done)
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if err := fs.buf.Flush(); err != nil {
		grpclogLogger.Warningf("failed to flush to Sink: %v", err)
	}
	if err := fs.closer.Close(); err != nil {
		grpclogLogger.Warningf("failed to close the underlying WriterCloser: %v", err)
	}
	if err := fs.out.Close(); err != nil {
		grpclogLogger.Warningf("failed to close the Sink: %v", err)
	}
	return nil
}

// NewBufferedSink creates a binary log sink with the given WriteCloser.
//
// Write() marshals the proto message and writes it to the given writer. Each
// message is prefixed with a 4 byte big endian unsigned integer as the length.
//
// Content is kept in a buffer, and is flushed every 60 seconds.
//
// Close closes the WriteCloser.
func NewBufferedSink(o io.WriteCloser) Sink {
	bufW := bufio.NewWriter(o)
	return &bufferedSink{
		closer: o,
		out:    newWriterSink(bufW),
		buf:    bufW,
		done:   make(chan struct{}),
	}
}
<|endoftext|>


<commit_before>package convert

import (
	"bytes"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"strings"
	"syscall"

	"golang.org/x/sync/errgroup"
)

// Process starts a mandoc process to convert manpages to HTML.
type Process struct {
	mandocConn    *net.UnixConn
	mandocProcess *os.Process
	stopWait      chan bool
}

func NewProcess() (*Process, error) {
	p := &Process{}
	return p, p.initMandoc()
}

func (p *Process) Kill() error {
	if p.mandocProcess == nil {
		return nil
	}
	p.stopWait <- true
	return p.mandocProcess.Kill()
}

func (p *Process) initMandoc() error {
	// TODO: get mandoc version, error if mandoc is not installed

	// TODO: remove once mandoc patch landed upstream
	return nil

	l, err := net.ListenUnix("unix", &net.UnixAddr{Net: "unix"})
	if err != nil {
		return err
	}
	f, err := l.File()
	if err != nil {
		return err
	}

	cmd := exec.Command("mandoc", "-Thtml", "-Ofragment", "-u", "/invalid")
	cmd.ExtraFiles = []*os.File{f}
	cmd.Env = []string{"MANDOC_UNIX_SOCKFD=3"} // go dup2()s each file in ExtraFiles
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		return err
	}
	p.stopWait = make(chan bool)
	go func() {
		wait := make(chan error, 1)
		go func() {
			wait <- cmd.Wait()
		}()
		select {
		case <-p.stopWait:
			return
		case err := <-wait:
			log.Fatalf("mandoc unexpectedly exited: %v", err)
		}
	}()

	p.mandocProcess = cmd.Process

	conn, err := net.DialUnix("unix", nil, l.Addr().(*net.UnixAddr))
	if err != nil {
		return err
	}

	p.mandocConn = conn
	return nil
}

func (p *Process) mandoc(r io.Reader) (stdout string, stderr string, err error) {
	if p.mandocConn != nil {
		stdout, stderr, err = p.mandocUnix(r)
	} else {
		stdout, stderr, err = p.mandocFork(r)
	}
	// TODO(later): once a new-enough version of mandoc is in Debian,
	// get rid of this compatibility code by changing our CSS to not
	// rely on the mandoc class at all anymore.
	if err == nil && !strings.HasPrefix(stdout, `<div class="mandoc">`) {
		stdout = `

` + stdout + `

`
	}
	return stdout, stderr, err
}

func (p *Process) mandocFork(r io.Reader) (stdout string, stderr string, err error) {
	var stdoutb, stderrb bytes.Buffer
	cmd := exec.Command("mandoc", "-Ofragment", "-Thtml")
	cmd.Stdin = r
	cmd.Stdout = &stdoutb
	cmd.Stderr = &stderrb
	if err := cmd.Run(); err != nil {
		return "", "", err
	}
	return stdoutb.String(), stderrb.String(), nil
}

func (p *Process) mandocUnix(r io.Reader) (stdout string, stderr string, err error) {
	manr, manw, err := os.Pipe()
	if err != nil {
		return "", "", err
	}
	defer manr.Close()
	defer manw.Close()

	outr, outw, err := os.Pipe()
	if err != nil {
		return "", "", err
	}
	defer outr.Close()
	defer outw.Close()

	errr, errw, err := os.Pipe()
	if err != nil {
		return "", "", err
	}
	defer errr.Close()
	defer errw.Close()

	scm := syscall.UnixRights(int(manr.Fd()), int(outw.Fd()), int(errw.Fd()))
	if _, _, err := p.mandocConn.WriteMsgUnix(nil, scm, nil); err != nil {
		return "", "", err
	}
	manr.Close()
	outw.Close()
	errw.Close()

	var eg errgroup.Group

	eg.Go(func() error {
		if _, err := io.Copy(manw, r); err != nil {
			return err
		}
		return manw.Close()
	})

	var stdoutb, stderrb []byte

	eg.Go(func() error {
		var err error
		stdoutb, err = ioutil.ReadAll(outr)
		return err
	})

	eg.Go(func() error {
		var err error
		stderrb, err = ioutil.ReadAll(errr)
		return err
	})

	return string(stdoutb), string(stderrb), eg.Wait()
}
<commit_msg>mandoc: implement mandocd interface<commit_after>package convert

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"strings"
	"syscall"

	"golang.org/x/sync/errgroup"
)

// Process starts a mandoc process to convert manpages to HTML.
type Process struct {
	mandocConn    *net.UnixConn
	mandocProcess *os.Process
	stopWait      chan bool
}

func NewProcess() (*Process, error) {
	p := &Process{}
	return p, p.initMandoc()
}

func (p *Process) Kill() error {
	if p.mandocProcess == nil {
		return nil
	}
	p.stopWait <- true
	return p.mandocProcess.Kill()
}

func (p *Process) initMandoc() error {
	pair, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)
	if err != nil {
		return err
	}

	// Use pair[0] in the parent process
	syscall.CloseOnExec(pair[0])
	f := os.NewFile(uintptr(pair[0]), "")
	fc, err := net.FileConn(f)
	if err != nil {
		return err
	}
	conn := fc.(*net.UnixConn)

	path, err := exec.LookPath("mandocd")
	if err != nil {
		if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound {
			log.Printf("mandocd not found, falling back to fork+exec for each manpage")
			return nil
		}
		return err
	}

	cmd := exec.Command(path, "-Thtml", "3") // Go dup2()s ExtraFiles to 3 and onwards
	cmd.ExtraFiles = []*os.File{os.NewFile(uintptr(pair[1]), "")}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		return err
	}

	if err := syscall.Close(pair[1]); err != nil {
		return err
	}

	p.stopWait = make(chan bool)
	go func() {
		wait := make(chan error, 1)
		go func() {
			wait <- cmd.Wait()
		}()
		select {
		case <-p.stopWait:
			return
		case err := <-wait:
			log.Fatalf("mandoc unexpectedly exited: %v", err)
		}
	}()

	p.mandocProcess = cmd.Process
	p.mandocConn = conn
	return nil
}

func (p *Process) mandoc(r io.Reader) (stdout string, stderr string, err error) {
	if p.mandocConn != nil {
		stdout, stderr, err = p.mandocUnix(r)
	} else {
		stdout, stderr, err = p.mandocFork(r)
	}
	// TODO(later): once a new-enough version of mandoc is in Debian,
	// get rid of this compatibility code by changing our CSS to not
	// rely on the mandoc class at all anymore.
	if err == nil && !strings.HasPrefix(stdout, `<div class="mandoc">`) {
		stdout = `

` + stdout + `

`
	}
	return stdout, stderr, err
}

func (p *Process) mandocFork(r io.Reader) (stdout string, stderr string, err error) {
	var stdoutb, stderrb bytes.Buffer
	cmd := exec.Command("mandoc", "-Ofragment", "-Thtml")
	cmd.Stdin = r
	cmd.Stdout = &stdoutb
	cmd.Stderr = &stderrb
	if err := cmd.Run(); err != nil {
		return "", "", fmt.Errorf("%v, stderr: %s", err, stderrb.String())
	}
	return stdoutb.String(), stderrb.String(), nil
}

func (p *Process) mandocUnix(r io.Reader) (stdout string, stderr string, err error) {
	manr, manw, err := os.Pipe()
	if err != nil {
		return "", "", err
	}
	defer manr.Close()
	defer manw.Close()

	outr, outw, err := os.Pipe()
	if err != nil {
		return "", "", err
	}
	defer outr.Close()
	defer outw.Close()

	errr, errw, err := os.Pipe()
	if err != nil {
		return "", "", err
	}
	defer errr.Close()
	defer errw.Close()

	scm := syscall.UnixRights(int(manr.Fd()), int(outw.Fd()), int(errw.Fd()))
	if _, _, err := p.mandocConn.WriteMsgUnix(nil, scm, nil); err != nil {
		return "", "", err
	}
	manr.Close()
	outw.Close()
	errw.Close()

	var eg errgroup.Group

	eg.Go(func() error {
		if _, err := io.Copy(manw, r); err != nil {
			return err
		}
		return manw.Close()
	})

	var stdoutb, stderrb []byte

	eg.Go(func() error {
		var err error
		stdoutb, err = ioutil.ReadAll(outr)
		return err
	})

	eg.Go(func() error {
		var err error
		stderrb, err = ioutil.ReadAll(errr)
		return err
	})

	return string(stdoutb), string(stderrb), eg.Wait()
}
<|endoftext|>


<commit_before>package annotator

import (
	"errors"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"time"

	"golang.org/x/net/context"
	//	"google.golang.org/appengine"
)

var geoData []Node

func init() {
	//TODO: if moved to flex, put handlers in main()

	setupPrometheus()
	http.HandleFunc("/", handler)
	http.HandleFunc("/annotate", annotate)
	log.Println("initialize table")
	InitializeTable(nil)
}

func InitializeTable(ctx context.Context) {

	if ctx == nil {
		ctx = context.Background()
	}

	storageReader, err := createReader("test-annotator-sandbox", "annotator-data/GeoIPCountryWhois.csv", ctx)
	if err != nil {
		return
	}

	geoData, err = createList(storageReader)
	if err != nil {
		log.Println("geoData createList failed")
		os.Exit(1)
	}
}

func annotate(w http.ResponseWriter, r *http.Request) {
	ip, _, err := validate(w, r)
	if err != nil {
		return
	}

	//handle errors here also pass JSON out
	lookupAndRespond(geoData, w, ip)

}

// validates request syntax
// parses request and returns parameters
func validate(w http.ResponseWriter, r *http.Request) (s string, num time.Time, err error) {

	//change metrics to package
	timerStart := time.Now()
	defer metrics_requestTimes.Observe(float64(time.Since(timerStart).Nanoseconds()))

	metrics_activeRequests.Inc()
	defer metrics_activeRequests.Dec()

	query := r.URL.Query()

	time_milli, err := strconv.ParseInt(query.Get("since_epoch"), 10, 64)
	if err != nil {
		fmt.Fprint(w, "INVALID TIME!")
		return s, num, errors.New("Invalid Time!")
	}

	ip := query.Get("ip_addr")

	if net.ParseIP(ip) == nil {
		fmt.Fprint(w, "NOT A RECOGNIZED IP FORMAT!")
		return s, num, errors.New("Strings dont match.")
	}

	return ip, time.Unix(time_milli, 0), nil
}

func handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, "Place requests to /annotate with URL parameters ip_addr and since_epoch!")
}
<commit_msg>Delete route.go<commit_after><|endoftext|>