<commit_before>package operation

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"path/filepath"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/security"
)

type UploadResult struct {
	Name  string `json:"name,omitempty"`
	Size  uint32 `json:"size,omitempty"`
	Error string `json:"error,omitempty"`
	ETag  string `json:"error,omitempty"`
}

var (
	client *http.Client
)

func init() {
	client = &http.Client{Transport: &http.Transport{
		MaxIdleConnsPerHost: 1024,
	}}
}

var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")

// Upload sends a POST request to a volume server to upload the content
func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
	return upload_content(uploadUrl, func(w io.Writer) (err error) {
		_, err = io.Copy(w, reader)
		return
	}, filename, isGzipped, mtype, pairMap, jwt)
}
func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
	body_buf := bytes.NewBufferString("")
	body_writer := multipart.NewWriter(body_buf)
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, fileNameEscaper.Replace(filename)))
	if mtype == "" {
		mtype = mime.TypeByExtension(strings.ToLower(filepath.Ext(filename)))
	}
	if mtype != "" {
		h.Set("Content-Type", mtype)
	}
	if isGzipped {
		h.Set("Content-Encoding", "gzip")
	}
	if jwt != "" {
		h.Set("Authorization", "BEARER "+string(jwt))
	}

	file_writer, cp_err := body_writer.CreatePart(h)
	if cp_err != nil {
		glog.V(0).Infoln("error creating form file", cp_err.Error())
		return nil, cp_err
	}
	if err := fillBufferFunction(file_writer); err != nil {
		glog.V(0).Infoln("error copying data", err)
		return nil, err
	}
	content_type := body_writer.FormDataContentType()
	if err := body_writer.Close(); err != nil {
		glog.V(0).Infoln("error closing body", err)
		return nil, err
	}

	req, postErr := http.NewRequest("POST", uploadUrl, body_buf)
	if postErr != nil {
		glog.V(0).Infoln("failing to upload to", uploadUrl, postErr.Error())
		return nil, postErr
	}
	req.Header.Set("Content-Type", content_type)
	for k, v := range pairMap {
		req.Header.Set(k, v)
	}
	resp, post_err := client.Do(req)
	if post_err != nil {
		glog.V(0).Infoln("failing to upload to", uploadUrl, post_err.Error())
		return nil, post_err
	}
	defer resp.Body.Close()
	etag := getEtag(resp)
	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		return nil, ra_err
	}
	var ret UploadResult
	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.V(0).Infoln("failing to read upload response", uploadUrl, string(resp_body))
		return nil, unmarshal_err
	}
	if ret.Error != "" {
		return nil, errors.New(ret.Error)
	}
	ret.ETag = etag
	return &ret, nil
}

func getEtag(r *http.Response) (etag string) {
	etag = r.Header.Get("ETag")
	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
		etag = etag[1 : len(etag)-1]
	}
	return
}
<commit_msg>fix https://github.com/chrislusf/seaweedfs/issues/780<commit_after>package operation

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"path/filepath"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/security"
)

type UploadResult struct {
	Name  string `json:"name,omitempty"`
	Size  uint32 `json:"size,omitempty"`
	Error string `json:"error,omitempty"`
	ETag  string `json:"error,omitempty"`
}

var (
	client *http.Client
)

func init() {
	client = &http.Client{Transport: &http.Transport{
		MaxIdleConnsPerHost: 1024,
	}}
}

var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")

// Upload sends a POST request to a volume server to upload the content
func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
	return upload_content(uploadUrl, func(w io.Writer) (err error) {
		_, err = io.Copy(w, reader)
		return
	}, filename, isGzipped, mtype, pairMap, jwt)
}
func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
	body_buf := bytes.NewBufferString("")
	body_writer := multipart.NewWriter(body_buf)
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, fileNameEscaper.Replace(filename)))
	if mtype == "" {
		mtype = mime.TypeByExtension(strings.ToLower(filepath.Ext(filename)))
	}
	if mtype != "" {
		h.Set("Content-Type", mtype)
	}
	if isGzipped {
		h.Set("Content-Encoding", "gzip")
	}
	if jwt != "" {
		h.Set("Authorization", "BEARER "+string(jwt))
	}

	file_writer, cp_err := body_writer.CreatePart(h)
	if cp_err != nil {
		glog.V(0).Infoln("error creating form file", cp_err.Error())
		return nil, cp_err
	}
	if err := fillBufferFunction(file_writer); err != nil {
		glog.V(0).Infoln("error copying data", err)
		return nil, err
	}
	content_type := body_writer.FormDataContentType()
	if err := body_writer.Close(); err != nil {
		glog.V(0).Infoln("error closing body", err)
		return nil, err
	}

	req, postErr := http.NewRequest("POST", uploadUrl, body_buf)
	if postErr != nil {
		glog.V(0).Infoln("failing to upload to", uploadUrl, postErr.Error())
		return nil, postErr
	}
	req.Header.Set("Content-Type", content_type)
	for k, v := range pairMap {
		req.Header.Set(k, v)
	}
	resp, post_err := client.Do(req)
	if post_err != nil {
		glog.V(0).Infoln("failing to upload to", uploadUrl, post_err.Error())
		return nil, post_err
	}
	defer resp.Body.Close()

	if resp.StatusCode < http.StatusOK ||
		resp.StatusCode > http.StatusIMUsed {
		return nil, errors.New(http.StatusText(resp.StatusCode))
	}

	etag := getEtag(resp)
	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		return nil, ra_err
	}
	var ret UploadResult
	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.V(0).Infoln("failing to read upload response", uploadUrl, string(resp_body))
		return nil, unmarshal_err
	}
	if ret.Error != "" {
		return nil, errors.New(ret.Error)
	}
	ret.ETag = etag
	return &ret, nil
}

func getEtag(r *http.Response) (etag string) {
	etag = r.Header.Get("ETag")
	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
		etag = etag[1 : len(etag)-1]
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package migrationminion

import (
	"github.com/juju/errors"
	"github.com/juju/loggo"

	"github.com/juju/juju/agent"
	"github.com/juju/juju/api"
	"github.com/juju/juju/api/base"
	"github.com/juju/juju/core/migration"
	"github.com/juju/juju/network"
	"github.com/juju/juju/watcher"
	"github.com/juju/juju/worker"
	"github.com/juju/juju/worker/catacomb"
	"github.com/juju/juju/worker/fortress"
)

var logger = loggo.GetLogger("juju.worker.migrationminion")

// Facade exposes controller functionality to a Worker.
type Facade interface {
	Watch() (watcher.MigrationStatusWatcher, error)
	Report(migrationId string, phase migration.Phase, success bool) error
}

// Config defines the operation of a Worker.
type Config struct {
	Agent             agent.Agent
	Facade            Facade
	Guard             fortress.Guard
	APIOpen           func(*api.Info, api.DialOpts) (api.Connection, error)
	ValidateMigration func(base.APICaller) error
}

// Validate returns an error if config cannot drive a Worker.
func (config Config) Validate() error {
	if config.Agent == nil {
		return errors.NotValidf("nil Agent")
	}
	if config.Facade == nil {
		return errors.NotValidf("nil Facade")
	}
	if config.Guard == nil {
		return errors.NotValidf("nil Guard")
	}
	if config.APIOpen == nil {
		return errors.NotValidf("nil APIOpen")
	}
	if config.ValidateMigration == nil {
		return errors.NotValidf("nil ValidateMigration")
	}
	return nil
}

// New returns a Worker backed by config, or an error.
func New(config Config) (worker.Worker, error) {
	if err := config.Validate(); err != nil {
		return nil, errors.Trace(err)
	}
	w := &Worker{config: config}
	err := catacomb.Invoke(catacomb.Plan{
		Site: &w.catacomb,
		Work: w.loop,
	})
	if err != nil {
		return nil, errors.Trace(err)
	}
	return w, nil
}

// Worker waits for a model migration to be active, then locks down the
// configured fortress and implements the migration.
type Worker struct {
	catacomb catacomb.Catacomb
	config   Config
}

// Kill implements worker.Worker.
func (w *Worker) Kill() {
	w.catacomb.Kill(nil)
}

// Wait implements worker.Worker.
func (w *Worker) Wait() error {
	return w.catacomb.Wait()
}

func (w *Worker) loop() error {
	watcher, err := w.config.Facade.Watch()
	if err != nil {
		return errors.Annotate(err, "setting up watcher")
	}
	if err := w.catacomb.Add(watcher); err != nil {
		return errors.Trace(err)
	}

	for {
		select {
		case <-w.catacomb.Dying():
			return w.catacomb.ErrDying()
		case status, ok := <-watcher.Changes():
			if !ok {
				return errors.New("watcher channel closed")
			}
			if err := w.handle(status); err != nil {
				return errors.Trace(err)
			}
		}
	}
}

func (w *Worker) handle(status watcher.MigrationStatus) error {
	logger.Infof("migration phase is now: %s", status.Phase)

	if !status.Phase.IsRunning() {
		return w.config.Guard.Unlock()
	}

	// Ensure that all workers related to migration fortress have
	// stopped and aren't allowed to restart.
	err := w.config.Guard.Lockdown(w.catacomb.Dying())
	if errors.Cause(err) == fortress.ErrAborted {
		return w.catacomb.ErrDying()
	} else if err != nil {
		return errors.Trace(err)
	}

	switch status.Phase {
	case migration.QUIESCE:
		err = w.doQUIESCE(status)
	case migration.VALIDATION:
		err = w.doVALIDATION(status)
	case migration.SUCCESS:
		err = w.doSUCCESS(status)
	default:
		// The minion doesn't need to do anything for other
		// migration phases.
	}
	return errors.Trace(err)
}

func (w *Worker) doQUIESCE(status watcher.MigrationStatus) error {
	// Report that the minion is ready and that all workers that
	// should be shut down have done so.
	return w.report(status, true)
}

func (w *Worker) doVALIDATION(status watcher.MigrationStatus) error {
	err := w.validate(status)
	if err != nil {
		// Don't return this error just log it and report to the
		// migrationmaster that things didn't work out.
		logger.Errorf("validation failed: %v", err)
	}
	return w.report(status, err == nil)
}

func (w *Worker) validate(status watcher.MigrationStatus) error {
	agentConf := w.config.Agent.CurrentConfig()
	apiInfo, ok := agentConf.APIInfo()
	if !ok {
		return errors.New("no API connection details")
	}
	apiInfo.Addrs = status.TargetAPIAddrs
	apiInfo.CACert = status.TargetCACert

	// Use zero DialOpts (no retries) because the worker must stay
	// responsive to Kill requests. We don't want it to be blocked by
	// a long set of retry attempts.
	conn, err := w.config.APIOpen(apiInfo, api.DialOpts{})
	if err != nil {
		// Don't return this error just log it and report to the
		// migrationmaster that things didn't work out.
		return errors.Annotate(err, "failed to open API to target controller")
	}
	defer conn.Close()

	// Ask the agent to confirm that things look ok.
	err = w.config.ValidateMigration(conn)
	return errors.Trace(err)
}

func (w *Worker) doSUCCESS(status watcher.MigrationStatus) error {
	hps, err := apiAddrsToHostPorts(status.TargetAPIAddrs)
	if err != nil {
		return errors.Annotate(err, "converting API addresses")
	}

	// Report first because the config update that's about to happen
	// will cause the API connection to drop. The SUCCESS phase is the
	// point of no return anyway.
	if err := w.report(status, true); err != nil {
		return errors.Trace(err)
	}

	err = w.config.Agent.ChangeConfig(func(conf agent.ConfigSetter) error {
		conf.SetAPIHostPorts(hps)
		conf.SetCACert(status.TargetCACert)
		return nil
	})
	return errors.Annotate(err, "setting agent config")
}

func (w *Worker) report(status watcher.MigrationStatus, success bool) error {
	err := w.config.Facade.Report(status.MigrationId, status.Phase, success)
	return errors.Annotate(err, "failed to report phase progress")
}

func apiAddrsToHostPorts(addrs []string) ([][]network.HostPort, error) {
	hps, err := network.ParseHostPorts(addrs...)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return [][]network.HostPort{hps}, nil
}
<commit_msg>worker/migrationminion: Log when reporting back<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package migrationminion

import (
	"github.com/juju/errors"
	"github.com/juju/loggo"

	"github.com/juju/juju/agent"
	"github.com/juju/juju/api"
	"github.com/juju/juju/api/base"
	"github.com/juju/juju/core/migration"
	"github.com/juju/juju/network"
	"github.com/juju/juju/watcher"
	"github.com/juju/juju/worker"
	"github.com/juju/juju/worker/catacomb"
	"github.com/juju/juju/worker/fortress"
)

var logger = loggo.GetLogger("juju.worker.migrationminion")

// Facade exposes controller functionality to a Worker.
type Facade interface {
	Watch() (watcher.MigrationStatusWatcher, error)
	Report(migrationId string, phase migration.Phase, success bool) error
}

// Config defines the operation of a Worker.
type Config struct {
	Agent             agent.Agent
	Facade            Facade
	Guard             fortress.Guard
	APIOpen           func(*api.Info, api.DialOpts) (api.Connection, error)
	ValidateMigration func(base.APICaller) error
}

// Validate returns an error if config cannot drive a Worker.
func (config Config) Validate() error {
	if config.Agent == nil {
		return errors.NotValidf("nil Agent")
	}
	if config.Facade == nil {
		return errors.NotValidf("nil Facade")
	}
	if config.Guard == nil {
		return errors.NotValidf("nil Guard")
	}
	if config.APIOpen == nil {
		return errors.NotValidf("nil APIOpen")
	}
	if config.ValidateMigration == nil {
		return errors.NotValidf("nil ValidateMigration")
	}
	return nil
}

// New returns a Worker backed by config, or an error.
func New(config Config) (worker.Worker, error) {
	if err := config.Validate(); err != nil {
		return nil, errors.Trace(err)
	}
	w := &Worker{config: config}
	err := catacomb.Invoke(catacomb.Plan{
		Site: &w.catacomb,
		Work: w.loop,
	})
	if err != nil {
		return nil, errors.Trace(err)
	}
	return w, nil
}

// Worker waits for a model migration to be active, then locks down the
// configured fortress and implements the migration.
type Worker struct {
	catacomb catacomb.Catacomb
	config   Config
}

// Kill implements worker.Worker.
func (w *Worker) Kill() {
	w.catacomb.Kill(nil)
}

// Wait implements worker.Worker.
func (w *Worker) Wait() error {
	return w.catacomb.Wait()
}

func (w *Worker) loop() error {
	watcher, err := w.config.Facade.Watch()
	if err != nil {
		return errors.Annotate(err, "setting up watcher")
	}
	if err := w.catacomb.Add(watcher); err != nil {
		return errors.Trace(err)
	}

	for {
		select {
		case <-w.catacomb.Dying():
			return w.catacomb.ErrDying()
		case status, ok := <-watcher.Changes():
			if !ok {
				return errors.New("watcher channel closed")
			}
			if err := w.handle(status); err != nil {
				return errors.Trace(err)
			}
		}
	}
}

func (w *Worker) handle(status watcher.MigrationStatus) error {
	logger.Infof("migration phase is now: %s", status.Phase)

	if !status.Phase.IsRunning() {
		return w.config.Guard.Unlock()
	}

	// Ensure that all workers related to migration fortress have
	// stopped and aren't allowed to restart.
	err := w.config.Guard.Lockdown(w.catacomb.Dying())
	if errors.Cause(err) == fortress.ErrAborted {
		return w.catacomb.ErrDying()
	} else if err != nil {
		return errors.Trace(err)
	}

	switch status.Phase {
	case migration.QUIESCE:
		err = w.doQUIESCE(status)
	case migration.VALIDATION:
		err = w.doVALIDATION(status)
	case migration.SUCCESS:
		err = w.doSUCCESS(status)
	default:
		// The minion doesn't need to do anything for other
		// migration phases.
	}
	return errors.Trace(err)
}

func (w *Worker) doQUIESCE(status watcher.MigrationStatus) error {
	// Report that the minion is ready and that all workers that
	// should be shut down have done so.
	return w.report(status, true)
}

func (w *Worker) doVALIDATION(status watcher.MigrationStatus) error {
	err := w.validate(status)
	if err != nil {
		// Don't return this error just log it and report to the
		// migrationmaster that things didn't work out.
		logger.Errorf("validation failed: %v", err)
	}
	return w.report(status, err == nil)
}

func (w *Worker) validate(status watcher.MigrationStatus) error {
	agentConf := w.config.Agent.CurrentConfig()
	apiInfo, ok := agentConf.APIInfo()
	if !ok {
		return errors.New("no API connection details")
	}
	apiInfo.Addrs = status.TargetAPIAddrs
	apiInfo.CACert = status.TargetCACert

	// Use zero DialOpts (no retries) because the worker must stay
	// responsive to Kill requests. We don't want it to be blocked by
	// a long set of retry attempts.
	conn, err := w.config.APIOpen(apiInfo, api.DialOpts{})
	if err != nil {
		// Don't return this error just log it and report to the
		// migrationmaster that things didn't work out.
		return errors.Annotate(err, "failed to open API to target controller")
	}
	defer conn.Close()

	// Ask the agent to confirm that things look ok.
	err = w.config.ValidateMigration(conn)
	return errors.Trace(err)
}

func (w *Worker) doSUCCESS(status watcher.MigrationStatus) error {
	hps, err := apiAddrsToHostPorts(status.TargetAPIAddrs)
	if err != nil {
		return errors.Annotate(err, "converting API addresses")
	}

	// Report first because the config update that's about to happen
	// will cause the API connection to drop. The SUCCESS phase is the
	// point of no return anyway.
	if err := w.report(status, true); err != nil {
		return errors.Trace(err)
	}

	err = w.config.Agent.ChangeConfig(func(conf agent.ConfigSetter) error {
		conf.SetAPIHostPorts(hps)
		conf.SetCACert(status.TargetCACert)
		return nil
	})
	return errors.Annotate(err, "setting agent config")
}

func (w *Worker) report(status watcher.MigrationStatus, success bool) error {
	logger.Debugf("reporting back for phase %s: %v", status.Phase, success)
	err := w.config.Facade.Report(status.MigrationId, status.Phase, success)
	return errors.Annotate(err, "failed to report phase progress")
}

func apiAddrsToHostPorts(addrs []string) ([][]network.HostPort, error) {
	hps, err := network.ParseHostPorts(addrs...)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return [][]network.HostPort{hps}, nil
}
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package obsreport

import (
	"context"

	"go.opencensus.io/stats"
	"go.opencensus.io/tag"
	"go.opencensus.io/trace"

	"go.opentelemetry.io/collector/config/configmodels"
)

const (
	// Key used to identify receivers in metrics and traces.
	ReceiverKey = "receiver"
	// Key used to identify the transport used to received the data.
	TransportKey = "transport"
	// Key used to identify the format of the data received.
	FormatKey = "format"

	// Key used to identify spans accepted by the Collector.
	AcceptedSpansKey = "accepted_spans"
	// Key used to identify spans refused (ie.: not ingested) by the Collector.
	RefusedSpansKey = "refused_spans"

	// Key used to identify metric points accepted by the Collector.
	AcceptedMetricPointsKey = "accepted_metric_points"
	// Key used to identify metric points refused (ie.: not ingested) by the
	// Collector.
	RefusedMetricPointsKey = "refused_metric_points"

	// Key used to identify log records accepted by the Collector.
	AcceptedLogRecordsKey = "accepted_log_records"
	// Key used to identify log records refused (ie.: not ingested) by the
	// Collector.
	RefusedLogRecordsKey = "refused_log_records"
)

var (
	tagKeyReceiver, _  = tag.NewKey(ReceiverKey)
	tagKeyTransport, _ = tag.NewKey(TransportKey)

	receiverPrefix                  = ReceiverKey + nameSep
	receiveTraceDataOperationSuffix = nameSep + "TraceDataReceived"
	receiverMetricsOperationSuffix  = nameSep + "MetricsReceived"

	// Receiver metrics. Any count of data items below is in the original format
	// that they were received, reasoning: reconciliation is easier if measurements
	// on clients and receiver are expected to be the same. Translation issues
	// that result in a different number of elements should be reported in a
	// separate way.
	mReceiverAcceptedSpans = stats.Int64(
		receiverPrefix+AcceptedSpansKey,
		"Number of spans successfully pushed into the pipeline.",
		stats.UnitDimensionless)
	mReceiverRefusedSpans = stats.Int64(
		receiverPrefix+RefusedSpansKey,
		"Number of spans that could not be pushed into the pipeline.",
		stats.UnitDimensionless)
	mReceiverAcceptedMetricPoints = stats.Int64(
		receiverPrefix+AcceptedMetricPointsKey,
		"Number of metric points successfully pushed into the pipeline.",
		stats.UnitDimensionless)
	mReceiverRefusedMetricPoints = stats.Int64(
		receiverPrefix+RefusedMetricPointsKey,
		"Number of metric points that could not be pushed into the pipeline.",
		stats.UnitDimensionless)
	mReceiverAcceptedLogRecords = stats.Int64(
		receiverPrefix+AcceptedLogRecordsKey,
		"Number of  log records successfully pushed into the pipeline.",
		stats.UnitDimensionless)
	mReceiverRefusedLogRecords = stats.Int64(
		receiverPrefix+RefusedLogRecordsKey,
		"Number of  log records that could not be pushed into the pipeline.",
		stats.UnitDimensionless)
)

// StartReceiveOptions has the options related to starting a receive operation.
type StartReceiveOptions struct {
	// LongLivedCtx when true indicates that the context passed in the call
	// outlives the individual receive operation. See WithLongLivedCtx() for
	// more information.
	LongLivedCtx bool
}

// StartReceiveOption function applues changes to StartReceiveOptions.
type StartReceiveOption func(*StartReceiveOptions)

// WithLongLivedCtx indicates that the context passed in the call outlives the
// receive operation at hand. Typically the long lived context is associated
// to a connection, eg.: a gRPC stream or a TCP connection, for which many
// batches of data are received in individual operations without a corresponding
// new context per operation.
//
// Example:
//
//    func (r *receiver) ClientConnect(ctx context.Context, rcvChan <-chan consumerdata.TraceData) {
//        longLivedCtx := obsreport.ReceiverContext(ctx, r.config.Name(), r.transport, "")
//        for {
//            // Since the context outlives the individual receive operations call obsreport using
//            // WithLongLivedCtx().
//            ctx := obsreport.StartTraceDataReceiveOp(
//                longLivedCtx,
//                r.config.Name(),
//                r.transport,
//                obsreport.WithLongLivedCtx())
//
//            td, ok := <-rcvChan
//            var err error
//            if ok {
//                err = r.nextConsumer.ConsumeTraceData(ctx, td)
//            }
//            obsreport.EndTraceDataReceiveOp(
//                ctx,
//                r.format,
//                len(td.Spans),
//                err)
//            if !ok {
//                break
//            }
//        }
//    }
//
func WithLongLivedCtx() StartReceiveOption {
	return func(opts *StartReceiveOptions) {
		opts.LongLivedCtx = true
	}
}

// StartTraceDataReceiveOp is called when a \request is received from a client.
// The returned context should be used in other calls to the obsreport functions
// dealing with the same receive operation.
func StartTraceDataReceiveOp(
	operationCtx context.Context,
	receiver string,
	transport string,
	opt ...StartReceiveOption,
) context.Context {
	return traceReceiveOp(
		operationCtx,
		receiver,
		transport,
		receiveTraceDataOperationSuffix,
		opt...)
}

// EndTraceDataReceiveOp completes the receive operation that was started with
// StartTraceDataReceiveOp.
func EndTraceDataReceiveOp(
	receiverCtx context.Context,
	format string,
	numReceivedSpans int,
	err error,
) {
	if useLegacy {
		numReceivedLegacy := numReceivedSpans
		numDroppedSpans := 0
		if err != nil {
			numDroppedSpans = numReceivedSpans
			numReceivedLegacy = 0
		}
		stats.Record(receiverCtx, mReceiverReceivedSpans.M(int64(numReceivedLegacy)), mReceiverDroppedSpans.M(int64(numDroppedSpans)))
	}

	endReceiveOp(
		receiverCtx,
		format,
		numReceivedSpans,
		err,
		configmodels.TracesDataType,
	)
}

// StartMetricsReceiveOp is called when a \request is received from a client.
// The returned context should be used in other calls to the obsreport functions
// dealing with the same receive operation.
func StartMetricsReceiveOp(
	operationCtx context.Context,
	receiver string,
	transport string,
	opt ...StartReceiveOption,
) context.Context {
	return traceReceiveOp(
		operationCtx,
		receiver,
		transport,
		receiverMetricsOperationSuffix,
		opt...)
}

// EndMetricsReceiveOp completes the receive operation that was started with
// StartMetricsReceiveOp.
func EndMetricsReceiveOp(
	receiverCtx context.Context,
	format string,
	numReceivedPoints int,
	numReceivedTimeSeries int, // For legacy measurements.
	err error,
) {
	if useLegacy {
		numDroppedTimeSeries := 0
		if err != nil {
			numDroppedTimeSeries = numReceivedTimeSeries
			numReceivedTimeSeries = 0
		}
		stats.Record(receiverCtx, mReceiverReceivedTimeSeries.M(int64(numReceivedTimeSeries)), mReceiverDroppedTimeSeries.M(int64(numDroppedTimeSeries)))
	}

	endReceiveOp(
		receiverCtx,
		format,
		numReceivedPoints,
		err,
		configmodels.MetricsDataType,
	)
}

// ReceiverContext adds the keys used when recording observability metrics to
// the given context returning the newly created context. This context should
// be used in related calls to the obsreport functions so metrics are properly
// recorded.
func ReceiverContext(
	ctx context.Context,
	receiver string,
	transport string,
	legacyName string,
) context.Context {
	if useLegacy {
		name := receiver
		if legacyName != "" {
			name = legacyName
		}
		ctx, _ = tag.New(ctx, tag.Upsert(LegacyTagKeyReceiver, name, tag.WithTTL(tag.TTLNoPropagation)))
	}

	ctx, _ = tag.New(ctx,
		tag.Upsert(tagKeyReceiver, receiver, tag.WithTTL(tag.TTLNoPropagation)),
		tag.Upsert(tagKeyTransport, transport, tag.WithTTL(tag.TTLNoPropagation)))

	return ctx
}

// traceReceiveOp creates the span used to trace the operation. Returning
// the updated context with the created span.
func traceReceiveOp(
	receiverCtx context.Context,
	receiverName string,
	transport string,
	operationSuffix string,
	opt ...StartReceiveOption,
) context.Context {
	var opts StartReceiveOptions
	for _, o := range opt {
		o(&opts)
	}

	var ctx context.Context
	var span *trace.Span
	spanName := receiverPrefix + receiverName + operationSuffix
	if !opts.LongLivedCtx {
		ctx, span = trace.StartSpan(receiverCtx, spanName)
	} else {
		// Since the receiverCtx is long lived do not use it to start the span.
		// This way this trace ends when the EndTraceDataReceiveOp is called.
		// Here is safe to ignore the returned context since it is not used below.
		_, span = trace.StartSpan(context.Background(), spanName)

		// If the long lived context has a parent span, then add it as a parent link.
		setParentLink(receiverCtx, span)

		ctx = trace.NewContext(receiverCtx, span)
	}

	if transport != "" {
		span.AddAttributes(trace.StringAttribute(TransportKey, transport))
	}
	return ctx
}

// endReceiveOp records the observability signals at the end of an operation.
func endReceiveOp(
	receiverCtx context.Context,
	format string,
	numReceivedItems int,
	err error,
	dataType configmodels.DataType,
) {
	numAccepted := numReceivedItems
	numRefused := 0
	if err != nil {
		numAccepted = 0
		numRefused = numReceivedItems
	}

	span := trace.FromContext(receiverCtx)

	if useNew {
		var acceptedMeasure, refusedMeasure *stats.Int64Measure
		switch dataType {
		case configmodels.TracesDataType:
			acceptedMeasure = mReceiverAcceptedSpans
			refusedMeasure = mReceiverRefusedSpans
		case configmodels.MetricsDataType:
			acceptedMeasure = mReceiverAcceptedMetricPoints
			refusedMeasure = mReceiverRefusedMetricPoints
		}

		stats.Record(
			receiverCtx,
			acceptedMeasure.M(int64(numAccepted)),
			refusedMeasure.M(int64(numRefused)))
	}

	// end span according to errors
	if span.IsRecordingEvents() {
		var acceptedItemsKey, refusedItemsKey string
		switch dataType {
		case configmodels.TracesDataType:
			acceptedItemsKey = AcceptedSpansKey
			refusedItemsKey = RefusedSpansKey
		case configmodels.MetricsDataType:
			acceptedItemsKey = AcceptedMetricPointsKey
			refusedItemsKey = RefusedMetricPointsKey
		}

		span.AddAttributes(
			trace.StringAttribute(
				FormatKey, format),
			trace.Int64Attribute(
				acceptedItemsKey, int64(numAccepted)),
			trace.Int64Attribute(
				refusedItemsKey, int64(numRefused)),
		)
		span.SetStatus(errToStatus(err))
	}
	span.End()
}
<commit_msg>Remove backslash from godoc (#1345)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package obsreport

import (
	"context"

	"go.opencensus.io/stats"
	"go.opencensus.io/tag"
	"go.opencensus.io/trace"

	"go.opentelemetry.io/collector/config/configmodels"
)

const (
	// Key used to identify receivers in metrics and traces.
	ReceiverKey = "receiver"
	// Key used to identify the transport used to received the data.
	TransportKey = "transport"
	// Key used to identify the format of the data received.
	FormatKey = "format"

	// Key used to identify spans accepted by the Collector.
	AcceptedSpansKey = "accepted_spans"
	// Key used to identify spans refused (ie.: not ingested) by the Collector.
	RefusedSpansKey = "refused_spans"

	// Key used to identify metric points accepted by the Collector.
	AcceptedMetricPointsKey = "accepted_metric_points"
	// Key used to identify metric points refused (ie.: not ingested) by the
	// Collector.
	RefusedMetricPointsKey = "refused_metric_points"

	// Key used to identify log records accepted by the Collector.
	AcceptedLogRecordsKey = "accepted_log_records"
	// Key used to identify log records refused (ie.: not ingested) by the
	// Collector.
	RefusedLogRecordsKey = "refused_log_records"
)

var (
	tagKeyReceiver, _  = tag.NewKey(ReceiverKey)
	tagKeyTransport, _ = tag.NewKey(TransportKey)

	receiverPrefix                  = ReceiverKey + nameSep
	receiveTraceDataOperationSuffix = nameSep + "TraceDataReceived"
	receiverMetricsOperationSuffix  = nameSep + "MetricsReceived"

	// Receiver metrics. Any count of data items below is in the original format
	// that they were received, reasoning: reconciliation is easier if measurements
	// on clients and receiver are expected to be the same. Translation issues
	// that result in a different number of elements should be reported in a
	// separate way.
	mReceiverAcceptedSpans = stats.Int64(
		receiverPrefix+AcceptedSpansKey,
		"Number of spans successfully pushed into the pipeline.",
		stats.UnitDimensionless)
	mReceiverRefusedSpans = stats.Int64(
		receiverPrefix+RefusedSpansKey,
		"Number of spans that could not be pushed into the pipeline.",
		stats.UnitDimensionless)
	mReceiverAcceptedMetricPoints = stats.Int64(
		receiverPrefix+AcceptedMetricPointsKey,
		"Number of metric points successfully pushed into the pipeline.",
		stats.UnitDimensionless)
	mReceiverRefusedMetricPoints = stats.Int64(
		receiverPrefix+RefusedMetricPointsKey,
		"Number of metric points that could not be pushed into the pipeline.",
		stats.UnitDimensionless)
	mReceiverAcceptedLogRecords = stats.Int64(
		receiverPrefix+AcceptedLogRecordsKey,
		"Number of  log records successfully pushed into the pipeline.",
		stats.UnitDimensionless)
	mReceiverRefusedLogRecords = stats.Int64(
		receiverPrefix+RefusedLogRecordsKey,
		"Number of  log records that could not be pushed into the pipeline.",
		stats.UnitDimensionless)
)

// StartReceiveOptions has the options related to starting a receive operation.
type StartReceiveOptions struct {
	// LongLivedCtx when true indicates that the context passed in the call
	// outlives the individual receive operation. See WithLongLivedCtx() for
	// more information.
	LongLivedCtx bool
}

// StartReceiveOption function applues changes to StartReceiveOptions.
type StartReceiveOption func(*StartReceiveOptions)

// WithLongLivedCtx indicates that the context passed in the call outlives the
// receive operation at hand. Typically the long lived context is associated
// to a connection, eg.: a gRPC stream or a TCP connection, for which many
// batches of data are received in individual operations without a corresponding
// new context per operation.
//
// Example:
//
//    func (r *receiver) ClientConnect(ctx context.Context, rcvChan <-chan consumerdata.TraceData) {
//        longLivedCtx := obsreport.ReceiverContext(ctx, r.config.Name(), r.transport, "")
//        for {
//            // Since the context outlives the individual receive operations call obsreport using
//            // WithLongLivedCtx().
//            ctx := obsreport.StartTraceDataReceiveOp(
//                longLivedCtx,
//                r.config.Name(),
//                r.transport,
//                obsreport.WithLongLivedCtx())
//
//            td, ok := <-rcvChan
//            var err error
//            if ok {
//                err = r.nextConsumer.ConsumeTraceData(ctx, td)
//            }
//            obsreport.EndTraceDataReceiveOp(
//                ctx,
//                r.format,
//                len(td.Spans),
//                err)
//            if !ok {
//                break
//            }
//        }
//    }
//
func WithLongLivedCtx() StartReceiveOption {
	return func(opts *StartReceiveOptions) {
		opts.LongLivedCtx = true
	}
}

// StartTraceDataReceiveOp is called when a request is received from a client.
// The returned context should be used in other calls to the obsreport functions
// dealing with the same receive operation.
func StartTraceDataReceiveOp(
	operationCtx context.Context,
	receiver string,
	transport string,
	opt ...StartReceiveOption,
) context.Context {
	return traceReceiveOp(
		operationCtx,
		receiver,
		transport,
		receiveTraceDataOperationSuffix,
		opt...)
}

// EndTraceDataReceiveOp completes the receive operation that was started with
// StartTraceDataReceiveOp.
func EndTraceDataReceiveOp(
	receiverCtx context.Context,
	format string,
	numReceivedSpans int,
	err error,
) {
	if useLegacy {
		numReceivedLegacy := numReceivedSpans
		numDroppedSpans := 0
		if err != nil {
			numDroppedSpans = numReceivedSpans
			numReceivedLegacy = 0
		}
		stats.Record(receiverCtx, mReceiverReceivedSpans.M(int64(numReceivedLegacy)), mReceiverDroppedSpans.M(int64(numDroppedSpans)))
	}

	endReceiveOp(
		receiverCtx,
		format,
		numReceivedSpans,
		err,
		configmodels.TracesDataType,
	)
}

// StartMetricsReceiveOp is called when a \request is received from a client.
// The returned context should be used in other calls to the obsreport functions
// dealing with the same receive operation.
func StartMetricsReceiveOp(
	operationCtx context.Context,
	receiver string,
	transport string,
	opt ...StartReceiveOption,
) context.Context {
	return traceReceiveOp(
		operationCtx,
		receiver,
		transport,
		receiverMetricsOperationSuffix,
		opt...)
}

// EndMetricsReceiveOp completes the receive operation that was started with
// StartMetricsReceiveOp.
func EndMetricsReceiveOp(
	receiverCtx context.Context,
	format string,
	numReceivedPoints int,
	numReceivedTimeSeries int, // For legacy measurements.
	err error,
) {
	if useLegacy {
		numDroppedTimeSeries := 0
		if err != nil {
			numDroppedTimeSeries = numReceivedTimeSeries
			numReceivedTimeSeries = 0
		}
		stats.Record(receiverCtx, mReceiverReceivedTimeSeries.M(int64(numReceivedTimeSeries)), mReceiverDroppedTimeSeries.M(int64(numDroppedTimeSeries)))
	}

	endReceiveOp(
		receiverCtx,
		format,
		numReceivedPoints,
		err,
		configmodels.MetricsDataType,
	)
}

// ReceiverContext adds the keys used when recording observability metrics to
// the given context returning the newly created context. This context should
// be used in related calls to the obsreport functions so metrics are properly
// recorded.
func ReceiverContext(
	ctx context.Context,
	receiver string,
	transport string,
	legacyName string,
) context.Context {
	if useLegacy {
		name := receiver
		if legacyName != "" {
			name = legacyName
		}
		ctx, _ = tag.New(ctx, tag.Upsert(LegacyTagKeyReceiver, name, tag.WithTTL(tag.TTLNoPropagation)))
	}

	ctx, _ = tag.New(ctx,
		tag.Upsert(tagKeyReceiver, receiver, tag.WithTTL(tag.TTLNoPropagation)),
		tag.Upsert(tagKeyTransport, transport, tag.WithTTL(tag.TTLNoPropagation)))

	return ctx
}

// traceReceiveOp creates the span used to trace the operation. Returning
// the updated context with the created span.
func traceReceiveOp(
	receiverCtx context.Context,
	receiverName string,
	transport string,
	operationSuffix string,
	opt ...StartReceiveOption,
) context.Context {
	var opts StartReceiveOptions
	for _, o := range opt {
		o(&opts)
	}

	var ctx context.Context
	var span *trace.Span
	spanName := receiverPrefix + receiverName + operationSuffix
	if !opts.LongLivedCtx {
		ctx, span = trace.StartSpan(receiverCtx, spanName)
	} else {
		// Since the receiverCtx is long lived do not use it to start the span.
		// This way this trace ends when the EndTraceDataReceiveOp is called.
		// Here is safe to ignore the returned context since it is not used below.
		_, span = trace.StartSpan(context.Background(), spanName)

		// If the long lived context has a parent span, then add it as a parent link.
		setParentLink(receiverCtx, span)

		ctx = trace.NewContext(receiverCtx, span)
	}

	if transport != "" {
		span.AddAttributes(trace.StringAttribute(TransportKey, transport))
	}
	return ctx
}

// endReceiveOp records the observability signals at the end of an operation.
func endReceiveOp(
	receiverCtx context.Context,
	format string,
	numReceivedItems int,
	err error,
	dataType configmodels.DataType,
) {
	numAccepted := numReceivedItems
	numRefused := 0
	if err != nil {
		numAccepted = 0
		numRefused = numReceivedItems
	}

	span := trace.FromContext(receiverCtx)

	if useNew {
		var acceptedMeasure, refusedMeasure *stats.Int64Measure
		switch dataType {
		case configmodels.TracesDataType:
			acceptedMeasure = mReceiverAcceptedSpans
			refusedMeasure = mReceiverRefusedSpans
		case configmodels.MetricsDataType:
			acceptedMeasure = mReceiverAcceptedMetricPoints
			refusedMeasure = mReceiverRefusedMetricPoints
		}

		stats.Record(
			receiverCtx,
			acceptedMeasure.M(int64(numAccepted)),
			refusedMeasure.M(int64(numRefused)))
	}

	// end span according to errors
	if span.IsRecordingEvents() {
		var acceptedItemsKey, refusedItemsKey string
		switch dataType {
		case configmodels.TracesDataType:
			acceptedItemsKey = AcceptedSpansKey
			refusedItemsKey = RefusedSpansKey
		case configmodels.MetricsDataType:
			acceptedItemsKey = AcceptedMetricPointsKey
			refusedItemsKey = RefusedMetricPointsKey
		}

		span.AddAttributes(
			trace.StringAttribute(
				FormatKey, format),
			trace.Int64Attribute(
				acceptedItemsKey, int64(numAccepted)),
			trace.Int64Attribute(
				refusedItemsKey, int64(numRefused)),
		)
		span.SetStatus(errToStatus(err))
	}
	span.End()
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/binary"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
	"time"

	"github.com/peterstace/grayt/colour"
)

type render struct {
	scene   string
	pxWide  int
	pxHigh  int
	created time.Time

	cnd            *sync.Cond
	desiredWorkers int
	actualWorkers  int

	acc *accumulator

	backoffMu sync.Mutex
	backoff   time.Duration
}

func (r *render) orchestrateWork() {
	// TODO: Could just get lock above loop and then never release?
	// TODO: Add some sort of backoff.
	for {
		r.cnd.L.Lock()
		for r.actualWorkers >= r.desiredWorkers {
			r.cnd.Wait()
		}
		r.sleepForBackoff()
		r.actualWorkers++
		go r.work()
		r.cnd.L.Unlock()
	}
}

func (r *render) sleepForBackoff() {
	r.backoffMu.Lock()
	b := r.backoff
	r.backoffMu.Unlock()
	time.Sleep(b)
}

func (r *render) work() {
	defer func() {
		r.cnd.L.Lock()
		r.actualWorkers--
		r.cnd.L.Unlock()
		r.cnd.Signal()
	}()

	// TODO: allow URL base to be configurable
	url := fmt.Sprintf(
		"http://worker:80/trace?scene_name=%s&px_wide=%d&px_high=%d",
		r.scene, r.pxWide, r.pxHigh,
	)
	resp, err := http.Get(url)
	if err != nil {
		log.Printf("error requesting  work: %v", err)
		return
	}

	if resp.StatusCode == http.StatusTooManyRequests {
		r.backoffMu.Lock()
		r.backoff = 2 * (time.Millisecond + r.backoff)
		backoff := r.backoff
		r.backoffMu.Unlock()
		log.Printf("too many requests, backoff: %s", backoff)
		return
	}
	if resp.StatusCode != http.StatusOK {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			body = []byte("couldn't ready body")
		}
		log.Printf("worker result with non-200 status: %s", string(body))
		return
	}

	r.backoffMu.Lock()
	r.backoff = 0
	r.backoffMu.Unlock()

	// TODO: should be able to reuse the pixel grid to save on allocations
	pixels := r.pxWide * r.pxHigh
	unitOfWork := pixelGrid{
		r.pxWide, r.pxHigh,
		make([]colour.Colour, pixels),
	}
	if err := binary.Read(resp.Body, binary.BigEndian, &unitOfWork.pixels); err != nil {
		log.Printf("could not read from worker response body: %v", err)
		return
	}

	r.acc.merge(&unitOfWork)
}
<commit_msg>Remove backoff<commit_after>package api

import (
	"encoding/binary"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
	"time"

	"github.com/peterstace/grayt/colour"
)

type render struct {
	scene   string
	pxWide  int
	pxHigh  int
	created time.Time

	cnd            *sync.Cond
	desiredWorkers int
	actualWorkers  int

	acc *accumulator
}

func (r *render) orchestrateWork() {
	for {
		r.cnd.L.Lock()
		for r.actualWorkers >= r.desiredWorkers {
			r.cnd.Wait()
		}
		r.actualWorkers++
		go r.work()
		r.cnd.L.Unlock()
	}
}

func (r *render) work() {
	defer func() {
		r.cnd.L.Lock()
		r.actualWorkers--
		r.cnd.L.Unlock()
		r.cnd.Signal()
	}()

	// TODO: allow URL base to be configurable
	url := fmt.Sprintf(
		"http://worker:80/trace?scene_name=%s&px_wide=%d&px_high=%d",
		r.scene, r.pxWide, r.pxHigh,
	)
	resp, err := http.Get(url)
	if err != nil {
		log.Printf("error requesting  work: %v", err)
		return
	}

	if resp.StatusCode == http.StatusTooManyRequests {
		return
	}
	if resp.StatusCode != http.StatusOK {
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			body = []byte("couldn't ready body")
		}
		log.Printf("worker result with non-200 status: %s", string(body))
		return
	}

	// TODO: should be able to reuse the pixel grid to save on allocations
	pixels := r.pxWide * r.pxHigh
	unitOfWork := pixelGrid{
		r.pxWide, r.pxHigh,
		make([]colour.Colour, pixels),
	}
	if err := binary.Read(resp.Body, binary.BigEndian, &unitOfWork.pixels); err != nil {
		log.Printf("could not read from worker response body: %v", err)
		return
	}

	r.acc.merge(&unitOfWork)
}
<|endoftext|>


<commit_before>package wrpendpoint

import (
	"context"
	"errors"

	"github.com/Comcast/webpa-common/logging"
	"github.com/Comcast/webpa-common/tracing"
	"github.com/go-kit/kit/log/level"
)

// serviceFanout takes a single WRP request and dispatches it concurrently to zero
// or more go-kit endpoints.
type serviceFanout struct {
	services map[string]Service
	spanner  tracing.Spanner
}

// fanoutResponse is the internal tuple used to communicate the results of an asynchronously
// invoked service
type fanoutResponse struct {
	name     string
	spans    []tracing.Span
	response Response
	err      error
}

// NewServiceFanout produces a WRP service which invokes each of a set of services concurrently for each WRP request.
// The first service which returns a valid response becomes the response of the fanout service.  If the context is
// cancelled, then ctx.Err() is returned.  Finally, if all services fail then a tracing.SpanError is returned with
// the last error set as the causal error.
func NewServiceFanout(services map[string]Service) Service {
	if len(services) == 0 {
		return ServiceFunc(func(context.Context, Request) (Response, error) {
			return nil, errors.New("No configured services")
		})
	}

	copyOf := make(map[string]Service, len(services))
	for k, v := range services {
		copyOf[k] = v
	}

	return &serviceFanout{
		services: copyOf,
		spanner:  tracing.NewSpanner(),
	}
}

func (sf *serviceFanout) ServeWRP(ctx context.Context, request Request) (Response, error) {
	results := make(chan fanoutResponse, len(sf.services))
	for name, s := range sf.services {
		go func(name string, s Service) {
			var (
				finisher      = sf.spanner.Start(name)
				response, err = s.ServeWRP(ctx, request)
				span          = finisher(err)
			)

			if err != nil {
				results <- fanoutResponse{
					name:  name,
					spans: []tracing.Span{span},
					err:   err,
				}
			} else {
				results <- fanoutResponse{
					name:     name,
					spans:    []tracing.Span{span},
					response: response.AddSpans(span),
				}
			}
		}(name, s)
	}

	var (
		lastError error
		spans     []tracing.Span
	)

	for r := 0; r < len(sf.services); r++ {
		select {
		case <-ctx.Done():
			request.Logger().Log(level.Key(), level.WarnValue(), logging.MessageKey(), "timed out")
			return nil, tracing.NewSpanError(ctx.Err(), spans...)
		case fr := <-results:
			if fr.err != nil {
				lastError = fr.err
				spans = append(spans, fr.spans...)
				request.Logger().Log(level.Key(), level.DebugValue(), "service", fr.name, logging.ErrorKey(), fr.err, logging.MessageKey(), "failed")
			} else {
				request.Logger().Log(level.Key(), level.DebugValue(), "service", fr.name, logging.MessageKey(), "success")
				return fr.response.AddSpans(spans...), nil
			}
		}
	}

	// use the last error as the causal error
	request.Logger().Log(level.Key(), level.ErrorValue(), logging.ErrorKey(), lastError, logging.MessageKey(), "all services failed")
	return nil, tracing.NewSpanError(lastError, spans...)
}
<commit_msg>simplified the span management in the boss goroutine<commit_after>package wrpendpoint

import (
	"context"
	"errors"

	"github.com/Comcast/webpa-common/logging"
	"github.com/Comcast/webpa-common/tracing"
	"github.com/go-kit/kit/log/level"
)

// serviceFanout takes a single WRP request and dispatches it concurrently to zero
// or more go-kit endpoints.
type serviceFanout struct {
	services map[string]Service
	spanner  tracing.Spanner
}

// fanoutResponse is the internal tuple used to communicate the results of an asynchronously
// invoked service
type fanoutResponse struct {
	name     string
	span     tracing.Span
	response Response
	err      error
}

// NewServiceFanout produces a WRP service which invokes each of a set of services concurrently for each WRP request.
// The first service which returns a valid response becomes the response of the fanout service.  If the context is
// cancelled, then ctx.Err() is returned.  Finally, if all services fail then a tracing.SpanError is returned with
// the last error set as the causal error.
func NewServiceFanout(services map[string]Service) Service {
	if len(services) == 0 {
		return ServiceFunc(func(context.Context, Request) (Response, error) {
			return nil, errors.New("No configured services")
		})
	}

	copyOf := make(map[string]Service, len(services))
	for k, v := range services {
		copyOf[k] = v
	}

	return &serviceFanout{
		services: copyOf,
		spanner:  tracing.NewSpanner(),
	}
}

func (sf *serviceFanout) ServeWRP(ctx context.Context, request Request) (Response, error) {
	results := make(chan fanoutResponse, len(sf.services))
	for name, s := range sf.services {
		go func(name string, s Service) {
			var (
				finisher      = sf.spanner.Start(name)
				response, err = s.ServeWRP(ctx, request)
			)

			results <- fanoutResponse{
				name:     name,
				span:     finisher(err),
				response: response,
				err:      err,
			}
		}(name, s)
	}

	var (
		lastError error
		spans     []tracing.Span
	)

	for r := 0; r < len(sf.services); r++ {
		select {
		case <-ctx.Done():
			request.Logger().Log(level.Key(), level.WarnValue(), logging.MessageKey(), "timed out")
			return nil, tracing.NewSpanError(ctx.Err(), spans...)
		case fr := <-results:
			spans = append(spans, fr.span)
			if fr.err != nil {
				lastError = fr.err
				request.Logger().Log(level.Key(), level.DebugValue(), "service", fr.name, logging.ErrorKey(), fr.err, logging.MessageKey(), "failed")
			} else {
				request.Logger().Log(level.Key(), level.DebugValue(), "service", fr.name, logging.MessageKey(), "success")
				return fr.response.AddSpans(spans...), nil
			}
		}
	}

	// use the last error as the causal error
	request.Logger().Log(level.Key(), level.ErrorValue(), logging.ErrorKey(), lastError, logging.MessageKey(), "all services failed")
	return nil, tracing.NewSpanError(lastError, spans...)
}
<|endoftext|>


<commit_before>package api

import (
	"errors"
	"strings"

	"github.com/kirinlabs/HttpRequest"
	"github.com/mjarkk/wotnlclans/other"
)

var routePrefix = "https://api.worldoftanks.eu"

// Routes are all the possible routes to make
var Routes = map[string]string{
	"nicknameAndClan":   "/wot/account/info/?application_id={{key}}&account_id={{playerID}}&fields=nickname%2Cclan_id",
	"clanIcon":          "/wgn/clans/info/?application_id={{key}}&clan_id={{clan}}&fields=tag%2Ccolor%2Cemblems.x195",
	"playerInfoLogedIn": "/wot/account/info/?application_id={{key}}&account_id={{playerID}}&access_token={{playerAccessToken}}",
	"playerInfo":        "/wot/account/info/?application_id={{key}}&account_id={{playerID}}",
	"loginLink":         "/wot/auth/login/?display=popup&nofollow=1&application_id={{key}}&redirect_uri={{redirectURL}}",
	"topClans":          "/wgn/clans/list/?application_id={{key}}&limit=100&game=wot&fields=clan_id&page_no={{pageNum}}",
	"clanDiscription":   "/wgn/clans/info/?application_id={{key}}&clan_id={{clanID}}&fields=description%2Ctag",
	"clanTag":           "/wgn/clans/info/?application_id={{key}}&clan_id={{clanID}}&fields=tag",
	"clanData":          "/wgn/clans/info/?application_id={{key}}&clan_id={{clanID}}&game=wot",
	"clanRating":        "/wot/clanratings/clans/?application_id={{key}}&clan_id={{clanID}}",
}

// GetAPIRoute returns a route with the inputs parsed
func GetAPIRoute(what string, inputs map[string]string) (string, error) {
	if _, hasKey := inputs["key"]; !hasKey {
		inputs["key"] = other.Flags.WGKey
	}
	selectedRoute, check := Routes[what]
	if !check {
		return "", errors.New("Api route not found")
	}
	for id, value := range inputs {
		selectedRoute = strings.Replace(selectedRoute, "{{"+id+"}}", value, -1)
	}
	if strings.Contains(selectedRoute, "{{") || strings.Contains(selectedRoute, "}}") {
		return "", errors.New("Not all inputs are replaced")
	}
	return routePrefix + selectedRoute, nil
}

// Get returns the response data (string) of a url
func Get(uri string) (string, error) {
	out, err := RawGet(uri)
	if err != nil {
		return "", err
	}
	return string(out), nil
}

// RawGet retruns the literal response data (byte data) of a response
func RawGet(uri string) ([]byte, error) {
	req := HttpRequest.NewRequest()
	res, err := req.Get(uri, nil)
	if err != nil {
		return []byte{}, err
	}
	out, err := res.Body()
	return out, err
}

// Post returns the data of a post request to a url
func Post(uri string, postData map[string]interface{}) (string, error) {
	req := HttpRequest.NewRequest()
	res, err := req.Post(uri, postData)
	if err != nil {
		return "", err
	}
	out, err := res.Body()
	if err != nil {
		return "", err
	}
	return string(out), nil
}

// CallRoute makes a network request
func CallRoute(what string, inputs map[string]string) (string, error) {
	url, err := GetAPIRoute(what, inputs)
	if err != nil {
		return "", err
	}
	return Get(url)
}
<commit_msg>Fixed error in cd/ci<commit_after>package api

import (
	"errors"
	"strings"

	"github.com/mjarkk/HttpRequest"
	"github.com/mjarkk/wotnlclans/other"
)

var routePrefix = "https://api.worldoftanks.eu"

// Routes are all the possible routes to make
var Routes = map[string]string{
	"nicknameAndClan":   "/wot/account/info/?application_id={{key}}&account_id={{playerID}}&fields=nickname%2Cclan_id",
	"clanIcon":          "/wgn/clans/info/?application_id={{key}}&clan_id={{clan}}&fields=tag%2Ccolor%2Cemblems.x195",
	"playerInfoLogedIn": "/wot/account/info/?application_id={{key}}&account_id={{playerID}}&access_token={{playerAccessToken}}",
	"playerInfo":        "/wot/account/info/?application_id={{key}}&account_id={{playerID}}",
	"loginLink":         "/wot/auth/login/?display=popup&nofollow=1&application_id={{key}}&redirect_uri={{redirectURL}}",
	"topClans":          "/wgn/clans/list/?application_id={{key}}&limit=100&game=wot&fields=clan_id&page_no={{pageNum}}",
	"clanDiscription":   "/wgn/clans/info/?application_id={{key}}&clan_id={{clanID}}&fields=description%2Ctag",
	"clanTag":           "/wgn/clans/info/?application_id={{key}}&clan_id={{clanID}}&fields=tag",
	"clanData":          "/wgn/clans/info/?application_id={{key}}&clan_id={{clanID}}&game=wot",
	"clanRating":        "/wot/clanratings/clans/?application_id={{key}}&clan_id={{clanID}}",
}

// GetAPIRoute returns a route with the inputs parsed
func GetAPIRoute(what string, inputs map[string]string) (string, error) {
	if _, hasKey := inputs["key"]; !hasKey {
		inputs["key"] = other.Flags.WGKey
	}
	selectedRoute, check := Routes[what]
	if !check {
		return "", errors.New("Api route not found")
	}
	for id, value := range inputs {
		selectedRoute = strings.Replace(selectedRoute, "{{"+id+"}}", value, -1)
	}
	if strings.Contains(selectedRoute, "{{") || strings.Contains(selectedRoute, "}}") {
		return "", errors.New("Not all inputs are replaced")
	}
	return routePrefix + selectedRoute, nil
}

// Get returns the response data (string) of a url
func Get(uri string) (string, error) {
	out, err := RawGet(uri)
	if err != nil {
		return "", err
	}
	return string(out), nil
}

// RawGet retruns the literal response data (byte data) of a response
func RawGet(uri string) ([]byte, error) {
	req := HttpRequest.NewRequest()
	res, err := req.Get(uri, nil)
	if err != nil {
		return []byte{}, err
	}
	out, err := res.Body()
	return out, err
}

// Post returns the data of a post request to a url
func Post(uri string, postData map[string]interface{}) (string, error) {
	req := HttpRequest.NewRequest()
	res, err := req.Post(uri, postData)
	if err != nil {
		return "", err
	}
	out, err := res.Body()
	if err != nil {
		return "", err
	}
	return string(out), nil
}

// CallRoute makes a network request
func CallRoute(what string, inputs map[string]string) (string, error) {
	url, err := GetAPIRoute(what, inputs)
	if err != nil {
		return "", err
	}
	return Get(url)
}
<|endoftext|>


<commit_before>//go:generate stringer -type=CommandStatusType,CommandIDType -output header_data_string.go

package data

// CommandStatusType is type of command status
type CommandStatusType int32

// CommandIDType is type of command id.
type CommandIDType int32

//nolint
const (
	// SMPP Command ID Set
	GENERIC_NACK          = CommandIDType(-2147483648)
	BIND_RECEIVER         = CommandIDType(0x00000001)
	BIND_RECEIVER_RESP    = CommandIDType(-2147483647)
	BIND_TRANSMITTER      = CommandIDType(0x00000002)
	BIND_TRANSMITTER_RESP = CommandIDType(-2147483646)
	QUERY_SM              = CommandIDType(0x00000003)
	QUERY_SM_RESP         = CommandIDType(-2147483645)
	SUBMIT_SM             = CommandIDType(0x00000004)
	SUBMIT_SM_RESP        = CommandIDType(-2147483644)
	DELIVER_SM            = CommandIDType(0x00000005)
	DELIVER_SM_RESP       = CommandIDType(-2147483643)
	UNBIND                = CommandIDType(0x00000006)
	UNBIND_RESP           = CommandIDType(-2147483642)
	REPLACE_SM            = CommandIDType(0x00000007)
	REPLACE_SM_RESP       = CommandIDType(-2147483641)
	CANCEL_SM             = CommandIDType(0x00000008)
	CANCEL_SM_RESP        = CommandIDType(-2147483640)
	BIND_TRANSCEIVER      = CommandIDType(0x00000009)
	BIND_TRANSCEIVER_RESP = CommandIDType(-2147483639)
	OUTBIND               = CommandIDType(0x0000000B)
	ENQUIRE_LINK          = CommandIDType(0x00000015)
	ENQUIRE_LINK_RESP     = CommandIDType(-2147483627)
	SUBMIT_MULTI          = CommandIDType(0x00000021)
	SUBMIT_MULTI_RESP     = CommandIDType(-2147483615)
	ALERT_NOTIFICATION    = CommandIDType(0x00000102)
	DATA_SM               = CommandIDType(0x00000103)
	DATA_SM_RESP          = CommandIDType(-2147483389)
)

const (

	// Command_Status Error Codes
	ESME_ROK           = CommandStatusType(0x00000000)
	ESME_RINVMSGLEN    = CommandStatusType(0x00000001)
	ESME_RINVCMDLEN    = CommandStatusType(0x00000002)
	ESME_RINVCMDID     = CommandStatusType(0x00000003)
	ESME_RINVBNDSTS    = CommandStatusType(0x00000004)
	ESME_RALYBND       = CommandStatusType(0x00000005)
	ESME_RINVPRTFLG    = CommandStatusType(0x00000006)
	ESME_RINVREGDLVFLG = CommandStatusType(0x00000007)
	ESME_RSYSERR       = CommandStatusType(0x00000008)
	ESME_RINVSRCADR    = CommandStatusType(0x0000000A)
	ESME_RINVDSTADR    = CommandStatusType(0x0000000B)
	ESME_RINVMSGID     = CommandStatusType(0x0000000C)
	ESME_RBINDFAIL     = CommandStatusType(0x0000000D)
	ESME_RINVPASWD     = CommandStatusType(0x0000000E)
	ESME_RINVSYSID     = CommandStatusType(0x0000000F)
	ESME_RCANCELFAIL   = CommandStatusType(0x00000011)
	ESME_RREPLACEFAIL  = CommandStatusType(0x00000013)
	ESME_RMSGQFUL      = CommandStatusType(0x00000014)
	ESME_RINVSERTYP    = CommandStatusType(0x00000015)

	ESME_RADDCUSTFAIL  = CommandStatusType(0x00000019) // Failed to Add Customer
	ESME_RDELCUSTFAIL  = CommandStatusType(0x0000001A) // Failed to delete Customer
	ESME_RMODCUSTFAIL  = CommandStatusType(0x0000001B) // Failed to modify customer
	ESME_RENQCUSTFAIL  = CommandStatusType(0x0000001C) // Failed to Enquire Customer
	ESME_RINVCUSTID    = CommandStatusType(0x0000001D) // Invalid Customer ID
	ESME_RINVCUSTNAME  = CommandStatusType(0x0000001F) // Invalid Customer Name
	ESME_RINVCUSTADR   = CommandStatusType(0x00000021) // Invalid Customer Address
	ESME_RINVADR       = CommandStatusType(0x00000022) // Invalid Address
	ESME_RCUSTEXIST    = CommandStatusType(0x00000023) // Customer Exists
	ESME_RCUSTNOTEXIST = CommandStatusType(0x00000024) // Customer does not exist
	ESME_RADDDLFAIL    = CommandStatusType(0x00000026) // Failed to Add DL
	ESME_RMODDLFAIL    = CommandStatusType(0x00000027) // Failed to modify DL
	ESME_RDELDLFAIL    = CommandStatusType(0x00000028) // Failed to Delete DL
	ESME_RVIEWDLFAIL   = CommandStatusType(0x00000029) // Failed to View DL
	ESME_RLISTDLSFAIL  = CommandStatusType(0x00000030) // Failed to list DLs
	ESME_RPARAMRETFAIL = CommandStatusType(0x00000031) // Param Retrieve Failed
	ESME_RINVPARAM     = CommandStatusType(0x00000032) // Invalid Param

	ESME_RINVNUMDESTS = CommandStatusType(0x00000033)
	ESME_RINVDLNAME   = CommandStatusType(0x00000034)

	ESME_RINVDLMEMBDESC = CommandStatusType(0x00000035) // Invalid DL Member Description
	ESME_RINVDLMEMBTYP  = CommandStatusType(0x00000038) // Invalid DL Member Type
	ESME_RINVDLMODOPT   = CommandStatusType(0x00000039) // Invalid DL Modify Option

	ESME_RINVDESTFLAG = CommandStatusType(0x00000040)
	ESME_RINVSUBREP   = CommandStatusType(0x00000042)
	ESME_RINVESMCLASS = CommandStatusType(0x00000043)
	ESME_RCNTSUBDL    = CommandStatusType(0x00000044)
	ESME_RSUBMITFAIL  = CommandStatusType(0x00000045)
	ESME_RINVSRCTON   = CommandStatusType(0x00000048)
	ESME_RINVSRCNPI   = CommandStatusType(0x00000049)
	ESME_RINVDSTTON   = CommandStatusType(0x00000050)
	ESME_RINVDSTNPI   = CommandStatusType(0x00000051)
	ESME_RINVSYSTYP   = CommandStatusType(0x00000053)
	ESME_RINVREPFLAG  = CommandStatusType(0x00000054)
	ESME_RINVNUMMSGS  = CommandStatusType(0x00000055)
	ESME_RTHROTTLED   = CommandStatusType(0x00000058)

	ESME_RPROVNOTALLWD = CommandStatusType(0x00000059) // Provisioning Not Allowed

	ESME_RINVSCHED    = CommandStatusType(0x00000061)
	ESME_RINVEXPIRY   = CommandStatusType(0x00000062)
	ESME_RINVDFTMSGID = CommandStatusType(0x00000063)
	ESME_RX_T_APPN    = CommandStatusType(0x00000064)
	ESME_RX_P_APPN    = CommandStatusType(0x00000065)
	ESME_RX_R_APPN    = CommandStatusType(0x00000066)
	ESME_RQUERYFAIL   = CommandStatusType(0x00000067)

	ESME_RINVPGCUSTID      = CommandStatusType(0x00000080) // Paging Customer ID Invalid No such subscriber
	ESME_RINVPGCUSTIDLEN   = CommandStatusType(0x00000081) // Paging Customer ID length Invalid
	ESME_RINVCITYLEN       = CommandStatusType(0x00000082) // City Length Invalid
	ESME_RINVSTATELEN      = CommandStatusType(0x00000083) // State Length Invalid
	ESME_RINVZIPPREFIXLEN  = CommandStatusType(0x00000084) // Zip Prefix Length Invalid
	ESME_RINVZIPPOSTFIXLEN = CommandStatusType(0x00000085) // Zip Postfix Length Invalid
	ESME_RINVMINLEN        = CommandStatusType(0x00000086) // MIN Length Invalid
	ESME_RINVMIN           = CommandStatusType(0x00000087) // MIN Invalid (i.e. No such MIN)
	ESME_RINVPINLEN        = CommandStatusType(0x00000088) // PIN Length Invalid
	ESME_RINVTERMCODELEN   = CommandStatusType(0x00000089) // Terminal Code Length Invalid
	ESME_RINVCHANNELLEN    = CommandStatusType(0x0000008A) // Channel Length Invalid
	ESME_RINVCOVREGIONLEN  = CommandStatusType(0x0000008B) // Coverage Region Length Invalid
	ESME_RINVCAPCODELEN    = CommandStatusType(0x0000008C) // Cap Code Length Invalid
	ESME_RINVMDTLEN        = CommandStatusType(0x0000008D) // Message delivery time Length Invalid
	ESME_RINVPRIORMSGLEN   = CommandStatusType(0x0000008E) // Priority Message Length Invalid
	ESME_RINVPERMSGLEN     = CommandStatusType(0x0000008F) // Periodic Messages Length Invalid
	ESME_RINVPGALERTLEN    = CommandStatusType(0x00000090) // Paging Alerts Length Invalid
	ESME_RINVSMUSERLEN     = CommandStatusType(0x00000091) // int16 Message User Group Length Invalid
	ESME_RINVRTDBLEN       = CommandStatusType(0x00000092) // Real Time Data broadcasts Length Invalid
	ESME_RINVREGDELLEN     = CommandStatusType(0x00000093) // Registered Delivery Length Invalid
	ESME_RINVMSGDISTLEN    = CommandStatusType(0x00000094) // Message Distribution Length Invalid
	ESME_RINVPRIORMSG      = CommandStatusType(0x00000095) // Priority Message Length Invalid
	ESME_RINVMDT           = CommandStatusType(0x00000096) // Message delivery time Invalid
	ESME_RINVPERMSG        = CommandStatusType(0x00000097) // Periodic Messages Invalid
	ESME_RINVMSGDIST       = CommandStatusType(0x00000098) // Message Distribution Invalid
	ESME_RINVPGALERT       = CommandStatusType(0x00000099) // Paging Alerts Invalid
	ESME_RINVSMUSER        = CommandStatusType(0x0000009A) // int16 Message User Group Invalid
	ESME_RINVRTDB          = CommandStatusType(0x0000009B) // Real Time Data broadcasts Invalid
	ESME_RINVREGDEL        = CommandStatusType(0x0000009C) // Registered Delivery Invalid
	// public static final  ESME_RINVOPTPARSTREAM = CommandStatusType(0x0000009D)   // KIF IW Field out of data
	// public static final  ESME_ROPTPARNOTALLWD = CommandStatusType(0x0000009E)   // Optional Parameter not allowed
	ESME_RINVOPTPARLEN = CommandStatusType(0x0000009F) // Invalid Optional Parameter Length

	ESME_RINVOPTPARSTREAM = CommandStatusType(0x000000C0)
	ESME_ROPTPARNOTALLWD  = CommandStatusType(0x000000C1)
	ESME_RINVPARLEN       = CommandStatusType(0x000000C2)
	ESME_RMISSINGOPTPARAM = CommandStatusType(0x000000C3)
	ESME_RINVOPTPARAMVAL  = CommandStatusType(0x000000C4)
	ESME_RDELIVERYFAILURE = CommandStatusType(0x000000FE)
	ESME_RUNKNOWNERR      = CommandStatusType(0x000000FF)

	ESME_LAST_ERROR = CommandStatusType(0x0000012C) // THE VALUE OF THE LAST ERROR CODE
)
<commit_msg>Fix linting<commit_after>//go:generate stringer -type=CommandStatusType,CommandIDType -output header_data_string.go

package data

// CommandStatusType is type of command status
type CommandStatusType int32

// CommandIDType is type of command id.
type CommandIDType int32

//nolint
const (
	// SMPP Command ID Set
	GENERIC_NACK          = CommandIDType(-2147483648)
	BIND_RECEIVER         = CommandIDType(0x00000001)
	BIND_RECEIVER_RESP    = CommandIDType(-2147483647)
	BIND_TRANSMITTER      = CommandIDType(0x00000002)
	BIND_TRANSMITTER_RESP = CommandIDType(-2147483646)
	QUERY_SM              = CommandIDType(0x00000003)
	QUERY_SM_RESP         = CommandIDType(-2147483645)
	SUBMIT_SM             = CommandIDType(0x00000004)
	SUBMIT_SM_RESP        = CommandIDType(-2147483644)
	DELIVER_SM            = CommandIDType(0x00000005)
	DELIVER_SM_RESP       = CommandIDType(-2147483643)
	UNBIND                = CommandIDType(0x00000006)
	UNBIND_RESP           = CommandIDType(-2147483642)
	REPLACE_SM            = CommandIDType(0x00000007)
	REPLACE_SM_RESP       = CommandIDType(-2147483641)
	CANCEL_SM             = CommandIDType(0x00000008)
	CANCEL_SM_RESP        = CommandIDType(-2147483640)
	BIND_TRANSCEIVER      = CommandIDType(0x00000009)
	BIND_TRANSCEIVER_RESP = CommandIDType(-2147483639)
	OUTBIND               = CommandIDType(0x0000000B)
	ENQUIRE_LINK          = CommandIDType(0x00000015)
	ENQUIRE_LINK_RESP     = CommandIDType(-2147483627)
	SUBMIT_MULTI          = CommandIDType(0x00000021)
	SUBMIT_MULTI_RESP     = CommandIDType(-2147483615)
	ALERT_NOTIFICATION    = CommandIDType(0x00000102)
	DATA_SM               = CommandIDType(0x00000103)
	DATA_SM_RESP          = CommandIDType(-2147483389)
)

//nolint
const (
	// Command_Status Error Codes
	ESME_ROK           = CommandStatusType(0x00000000)
	ESME_RINVMSGLEN    = CommandStatusType(0x00000001)
	ESME_RINVCMDLEN    = CommandStatusType(0x00000002)
	ESME_RINVCMDID     = CommandStatusType(0x00000003)
	ESME_RINVBNDSTS    = CommandStatusType(0x00000004)
	ESME_RALYBND       = CommandStatusType(0x00000005)
	ESME_RINVPRTFLG    = CommandStatusType(0x00000006)
	ESME_RINVREGDLVFLG = CommandStatusType(0x00000007)
	ESME_RSYSERR       = CommandStatusType(0x00000008)
	ESME_RINVSRCADR    = CommandStatusType(0x0000000A)
	ESME_RINVDSTADR    = CommandStatusType(0x0000000B)
	ESME_RINVMSGID     = CommandStatusType(0x0000000C)
	ESME_RBINDFAIL     = CommandStatusType(0x0000000D)
	ESME_RINVPASWD     = CommandStatusType(0x0000000E)
	ESME_RINVSYSID     = CommandStatusType(0x0000000F)
	ESME_RCANCELFAIL   = CommandStatusType(0x00000011)
	ESME_RREPLACEFAIL  = CommandStatusType(0x00000013)
	ESME_RMSGQFUL      = CommandStatusType(0x00000014)
	ESME_RINVSERTYP    = CommandStatusType(0x00000015)

	ESME_RADDCUSTFAIL  = CommandStatusType(0x00000019) // Failed to Add Customer
	ESME_RDELCUSTFAIL  = CommandStatusType(0x0000001A) // Failed to delete Customer
	ESME_RMODCUSTFAIL  = CommandStatusType(0x0000001B) // Failed to modify customer
	ESME_RENQCUSTFAIL  = CommandStatusType(0x0000001C) // Failed to Enquire Customer
	ESME_RINVCUSTID    = CommandStatusType(0x0000001D) // Invalid Customer ID
	ESME_RINVCUSTNAME  = CommandStatusType(0x0000001F) // Invalid Customer Name
	ESME_RINVCUSTADR   = CommandStatusType(0x00000021) // Invalid Customer Address
	ESME_RINVADR       = CommandStatusType(0x00000022) // Invalid Address
	ESME_RCUSTEXIST    = CommandStatusType(0x00000023) // Customer Exists
	ESME_RCUSTNOTEXIST = CommandStatusType(0x00000024) // Customer does not exist
	ESME_RADDDLFAIL    = CommandStatusType(0x00000026) // Failed to Add DL
	ESME_RMODDLFAIL    = CommandStatusType(0x00000027) // Failed to modify DL
	ESME_RDELDLFAIL    = CommandStatusType(0x00000028) // Failed to Delete DL
	ESME_RVIEWDLFAIL   = CommandStatusType(0x00000029) // Failed to View DL
	ESME_RLISTDLSFAIL  = CommandStatusType(0x00000030) // Failed to list DLs
	ESME_RPARAMRETFAIL = CommandStatusType(0x00000031) // Param Retrieve Failed
	ESME_RINVPARAM     = CommandStatusType(0x00000032) // Invalid Param

	ESME_RINVNUMDESTS = CommandStatusType(0x00000033)
	ESME_RINVDLNAME   = CommandStatusType(0x00000034)

	ESME_RINVDLMEMBDESC = CommandStatusType(0x00000035) // Invalid DL Member Description
	ESME_RINVDLMEMBTYP  = CommandStatusType(0x00000038) // Invalid DL Member Type
	ESME_RINVDLMODOPT   = CommandStatusType(0x00000039) // Invalid DL Modify Option

	ESME_RINVDESTFLAG = CommandStatusType(0x00000040)
	ESME_RINVSUBREP   = CommandStatusType(0x00000042)
	ESME_RINVESMCLASS = CommandStatusType(0x00000043)
	ESME_RCNTSUBDL    = CommandStatusType(0x00000044)
	ESME_RSUBMITFAIL  = CommandStatusType(0x00000045)
	ESME_RINVSRCTON   = CommandStatusType(0x00000048)
	ESME_RINVSRCNPI   = CommandStatusType(0x00000049)
	ESME_RINVDSTTON   = CommandStatusType(0x00000050)
	ESME_RINVDSTNPI   = CommandStatusType(0x00000051)
	ESME_RINVSYSTYP   = CommandStatusType(0x00000053)
	ESME_RINVREPFLAG  = CommandStatusType(0x00000054)
	ESME_RINVNUMMSGS  = CommandStatusType(0x00000055)
	ESME_RTHROTTLED   = CommandStatusType(0x00000058)

	ESME_RPROVNOTALLWD = CommandStatusType(0x00000059) // Provisioning Not Allowed

	ESME_RINVSCHED    = CommandStatusType(0x00000061)
	ESME_RINVEXPIRY   = CommandStatusType(0x00000062)
	ESME_RINVDFTMSGID = CommandStatusType(0x00000063)
	ESME_RX_T_APPN    = CommandStatusType(0x00000064)
	ESME_RX_P_APPN    = CommandStatusType(0x00000065)
	ESME_RX_R_APPN    = CommandStatusType(0x00000066)
	ESME_RQUERYFAIL   = CommandStatusType(0x00000067)

	ESME_RINVPGCUSTID      = CommandStatusType(0x00000080) // Paging Customer ID Invalid No such subscriber
	ESME_RINVPGCUSTIDLEN   = CommandStatusType(0x00000081) // Paging Customer ID length Invalid
	ESME_RINVCITYLEN       = CommandStatusType(0x00000082) // City Length Invalid
	ESME_RINVSTATELEN      = CommandStatusType(0x00000083) // State Length Invalid
	ESME_RINVZIPPREFIXLEN  = CommandStatusType(0x00000084) // Zip Prefix Length Invalid
	ESME_RINVZIPPOSTFIXLEN = CommandStatusType(0x00000085) // Zip Postfix Length Invalid
	ESME_RINVMINLEN        = CommandStatusType(0x00000086) // MIN Length Invalid
	ESME_RINVMIN           = CommandStatusType(0x00000087) // MIN Invalid (i.e. No such MIN)
	ESME_RINVPINLEN        = CommandStatusType(0x00000088) // PIN Length Invalid
	ESME_RINVTERMCODELEN   = CommandStatusType(0x00000089) // Terminal Code Length Invalid
	ESME_RINVCHANNELLEN    = CommandStatusType(0x0000008A) // Channel Length Invalid
	ESME_RINVCOVREGIONLEN  = CommandStatusType(0x0000008B) // Coverage Region Length Invalid
	ESME_RINVCAPCODELEN    = CommandStatusType(0x0000008C) // Cap Code Length Invalid
	ESME_RINVMDTLEN        = CommandStatusType(0x0000008D) // Message delivery time Length Invalid
	ESME_RINVPRIORMSGLEN   = CommandStatusType(0x0000008E) // Priority Message Length Invalid
	ESME_RINVPERMSGLEN     = CommandStatusType(0x0000008F) // Periodic Messages Length Invalid
	ESME_RINVPGALERTLEN    = CommandStatusType(0x00000090) // Paging Alerts Length Invalid
	ESME_RINVSMUSERLEN     = CommandStatusType(0x00000091) // int16 Message User Group Length Invalid
	ESME_RINVRTDBLEN       = CommandStatusType(0x00000092) // Real Time Data broadcasts Length Invalid
	ESME_RINVREGDELLEN     = CommandStatusType(0x00000093) // Registered Delivery Length Invalid
	ESME_RINVMSGDISTLEN    = CommandStatusType(0x00000094) // Message Distribution Length Invalid
	ESME_RINVPRIORMSG      = CommandStatusType(0x00000095) // Priority Message Length Invalid
	ESME_RINVMDT           = CommandStatusType(0x00000096) // Message delivery time Invalid
	ESME_RINVPERMSG        = CommandStatusType(0x00000097) // Periodic Messages Invalid
	ESME_RINVMSGDIST       = CommandStatusType(0x00000098) // Message Distribution Invalid
	ESME_RINVPGALERT       = CommandStatusType(0x00000099) // Paging Alerts Invalid
	ESME_RINVSMUSER        = CommandStatusType(0x0000009A) // int16 Message User Group Invalid
	ESME_RINVRTDB          = CommandStatusType(0x0000009B) // Real Time Data broadcasts Invalid
	ESME_RINVREGDEL        = CommandStatusType(0x0000009C) // Registered Delivery Invalid
	// public static final  ESME_RINVOPTPARSTREAM = CommandStatusType(0x0000009D)   // KIF IW Field out of data
	// public static final  ESME_ROPTPARNOTALLWD = CommandStatusType(0x0000009E)   // Optional Parameter not allowed
	ESME_RINVOPTPARLEN = CommandStatusType(0x0000009F) // Invalid Optional Parameter Length

	ESME_RINVOPTPARSTREAM = CommandStatusType(0x000000C0)
	ESME_ROPTPARNOTALLWD  = CommandStatusType(0x000000C1)
	ESME_RINVPARLEN       = CommandStatusType(0x000000C2)
	ESME_RMISSINGOPTPARAM = CommandStatusType(0x000000C3)
	ESME_RINVOPTPARAMVAL  = CommandStatusType(0x000000C4)
	ESME_RDELIVERYFAILURE = CommandStatusType(0x000000FE)
	ESME_RUNKNOWNERR      = CommandStatusType(0x000000FF)

	ESME_LAST_ERROR = CommandStatusType(0x0000012C) // THE VALUE OF THE LAST ERROR CODE
)
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"github.com/gorilla/feeds"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/calendar/v3"
	"html"
	"io/ioutil"
	"time"
)

const (
	CalendarId = "qsqrk2emvnnvu45debac9dugr8@group.calendar.google.com"
	LinkUrl    = "https://calendar.google.com/calendar/embed?src=qsqrk2emvnnvu45debac9dugr8@group.calendar.google.com"
)

func GetSanrioEventsCalendar() (*feeds.Feed, error) {
	json, err := ioutil.ReadFile("google_client_credentials.json")
	if err != nil {
		return nil, err
	}

	config, err := google.JWTConfigFromJSON(json, calendar.CalendarReadonlyScope)
	if err != nil {
		return nil, err
	}

	client := config.Client(oauth2.NoContext)

	service, err := calendar.New(client)
	if err != nil {
		return nil, err
	}

	events, err := service.Events.List(CalendarId).OrderBy("updated").Do()
	if err != nil {
		return nil, err
	}

	return GetSanrioEventsCalendarFromEvents(events)
}

func GetSanrioEventsCalendarFromEvents(events *calendar.Events) (*feeds.Feed, error) {
	var items []*feeds.Item
	items = make([]*feeds.Item, len(events.Items))
	for i, event := range events.Items {
		created, err := time.Parse(time.RFC3339, event.Created)
		if err != nil {
			return nil, err
		}
		updated, err := time.Parse(time.RFC3339, event.Updated)
		if err != nil {
			return nil, err
		}

		var duration string

		switch {
		case event.Start.Date != "" && event.End.Date != "":
			startLoc, err := time.LoadLocation(event.Start.TimeZone)
			if err != nil {
				return nil, err
			}
			start, err := time.ParseInLocation("2006-01-02", event.Start.Date, startLoc)
			if err != nil {
				return nil, err
			}
			endLoc, err := time.LoadLocation(event.Start.TimeZone)
			if err != nil {
				return nil, err
			}
			end, err := time.ParseInLocation("2006-01-02", event.End.Date, endLoc)
			if err != nil {
				return nil, err
			}

			if start.Format("2006-01-02") == end.Format("2006-01-02") {
				duration = start.Format("2006-01-02 (Mon)")
			} else {
				duration = start.Format("2006-01-02 (Mon)") + " - " + end.Format("2006-01-02 (Mon)")
			}

		case event.Start.DateTime != "" && event.End.DateTime != "":
			startLoc, err := time.LoadLocation(event.Start.TimeZone)
			if err != nil {
				return nil, err
			}
			start, err := time.ParseInLocation(time.RFC3339, event.Start.DateTime, startLoc)
			if err != nil {
				return nil, err
			}
			endLoc, err := time.LoadLocation(event.Start.TimeZone)
			if err != nil {
				return nil, err
			}
			end, err := time.ParseInLocation(time.RFC3339, event.End.DateTime, endLoc)
			if err != nil {
				return nil, err
			}

			if start.Format("2006-01-02") == end.Format("2006-01-02") {
				duration = start.Format("2006-01-02 (Mon) 15:04") + " - " + end.Format("15:04")
			} else {
				duration = start.Format("2006-01-02 (Mon) 15:04") + " - " + end.Format("2006-01-02 (Mon) 15:04")
			}

		default:
			return nil, errors.New("must not happen")
		}

		description := fmt.Sprintf("Location: %s
Duration: %s
Description: %s", html.EscapeString(event.Location), html.EscapeString(duration), html.EscapeString(event.Description))

		items[i] = &feeds.Item{
			Id:          event.Etag,
			Title:       event.Summary,
			Description: description,
			Link:        &feeds.Link{Href: event.HtmlLink},
			Author:      &feeds.Author{event.Creator.DisplayName, event.Creator.Email},
			Created:     created,
			Updated:     updated,
		}
	}

	updated, err := time.Parse(time.RFC3339, events.Updated)
	if err != nil {
		return nil, err
	}

	feed := &feeds.Feed{
		Id:          events.Etag,
		Title:       events.Summary,
		Description: events.Description,
		Link:        &feeds.Link{Href: LinkUrl},
		Updated:     updated,
		Items:       items,
	}
	return feed, nil
}
<commit_msg>Improve the format of description<commit_after>package main

import (
	"errors"
	"fmt"
	"github.com/gorilla/feeds"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/calendar/v3"
	"html"
	"io/ioutil"
	"strings"
	"time"
)

const (
	CalendarId = "qsqrk2emvnnvu45debac9dugr8@group.calendar.google.com"
	LinkUrl    = "https://calendar.google.com/calendar/embed?src=qsqrk2emvnnvu45debac9dugr8@group.calendar.google.com"
)

func GetSanrioEventsCalendar() (*feeds.Feed, error) {
	json, err := ioutil.ReadFile("google_client_credentials.json")
	if err != nil {
		return nil, err
	}

	config, err := google.JWTConfigFromJSON(json, calendar.CalendarReadonlyScope)
	if err != nil {
		return nil, err
	}

	client := config.Client(oauth2.NoContext)

	service, err := calendar.New(client)
	if err != nil {
		return nil, err
	}

	events, err := service.Events.List(CalendarId).OrderBy("updated").Do()
	if err != nil {
		return nil, err
	}

	return GetSanrioEventsCalendarFromEvents(events)
}

func GetSanrioEventsCalendarFromEvents(events *calendar.Events) (*feeds.Feed, error) {
	descriptionReplacer := strings.NewReplacer("\n", "
")

	var items []*feeds.Item
	items = make([]*feeds.Item, len(events.Items))
	for i, event := range events.Items {
		created, err := time.Parse(time.RFC3339, event.Created)
		if err != nil {
			return nil, err
		}
		updated, err := time.Parse(time.RFC3339, event.Updated)
		if err != nil {
			return nil, err
		}

		var duration string

		switch {
		case event.Start.Date != "" && event.End.Date != "":
			startLoc, err := time.LoadLocation(event.Start.TimeZone)
			if err != nil {
				return nil, err
			}
			start, err := time.ParseInLocation("2006-01-02", event.Start.Date, startLoc)
			if err != nil {
				return nil, err
			}
			endLoc, err := time.LoadLocation(event.Start.TimeZone)
			if err != nil {
				return nil, err
			}
			end, err := time.ParseInLocation("2006-01-02", event.End.Date, endLoc)
			if err != nil {
				return nil, err
			}

			if start.Format("2006-01-02") == end.Format("2006-01-02") {
				duration = start.Format("2006-01-02 (Mon)")
			} else {
				duration = start.Format("2006-01-02 (Mon)") + " - " + end.Format("2006-01-02 (Mon)")
			}

		case event.Start.DateTime != "" && event.End.DateTime != "":
			startLoc, err := time.LoadLocation(event.Start.TimeZone)
			if err != nil {
				return nil, err
			}
			start, err := time.ParseInLocation(time.RFC3339, event.Start.DateTime, startLoc)
			if err != nil {
				return nil, err
			}
			endLoc, err := time.LoadLocation(event.Start.TimeZone)
			if err != nil {
				return nil, err
			}
			end, err := time.ParseInLocation(time.RFC3339, event.End.DateTime, endLoc)
			if err != nil {
				return nil, err
			}

			if start.Format("2006-01-02") == end.Format("2006-01-02") {
				duration = start.Format("2006-01-02 (Mon) 15:04") + " - " + end.Format("15:04")
			} else {
				duration = start.Format("2006-01-02 (Mon) 15:04") + " - " + end.Format("2006-01-02 (Mon) 15:04")
			}

		default:
			return nil, errors.New("must not happen")
		}

		description := fmt.Sprintf("Location: %s
Duration: %s

%s", html.EscapeString(event.Location), html.EscapeString(duration), descriptionReplacer.Replace(html.EscapeString(event.Description)))

		items[i] = &feeds.Item{
			Id:          event.Etag,
			Title:       event.Summary,
			Description: description,
			Link:        &feeds.Link{Href: event.HtmlLink},
			Author:      &feeds.Author{event.Creator.DisplayName, event.Creator.Email},
			Created:     created,
			Updated:     updated,
		}
	}

	updated, err := time.Parse(time.RFC3339, events.Updated)
	if err != nil {
		return nil, err
	}

	feed := &feeds.Feed{
		Id:          events.Etag,
		Title:       events.Summary,
		Description: events.Description,
		Link:        &feeds.Link{Href: LinkUrl},
		Updated:     updated,
		Items:       items,
	}
	return feed, nil
}
<|endoftext|>


<commit_before>package persist

import (
	"fmt"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"github.com/pachyderm/pachyderm/src/client/pfs"
	pfsclient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pfs/db/persist"
	"github.com/pachyderm/pachyderm/src/server/pfs/drive"
	pfsserver "github.com/pachyderm/pachyderm/src/server/pfs/server"

	"github.com/dancannon/gorethink"
	"go.pedge.io/pb/go/google/protobuf"
	"go.pedge.io/proto/server"
	"google.golang.org/grpc"
)

var (
	port int32 = 30651
)

/*
	CommitBranchIndex

	Given a repo and a clock, returns a commit

	Is used in several places to:

	- find a parent commit given the parent's id in the form of an alias (e.g. "master/0")
	- getHeadOfBranch() -- by doing a range query of the form "branchName/0" to "branchName/max" and returning the last result (in this case the head)
	- getIDOfParentcommit() -- by decrementing this commit's clock value, and searching for that new clock
	- getCommitByAmbmiguousID() -- if the commit ID is in the form of an alias, find the commit using the index

*/

func TestCommitBranchIndexBasic(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)

		term := gorethink.DB(dbName).Table(commitTable)
		cursor, err := term.GetAll(commitID).Run(dbClient)
		require.NoError(t, err)
		c := &persist.Commit{}
		err = cursor.One(c)
		require.NoError(t, err)
		require.Equal(t, commitID, c.ID)

		clock := &persist.Clock{Branch: "master", Clock: 0}
		clockID := getClockID(repo.Name, clock).ID
		cursor, err = gorethink.DB(dbName).Table(clockTable).GetAll(clockID).Run(dbClient)
		require.NoError(t, err)
		returnedClock := &persist.Clock{}
		require.NoError(t, cursor.One(returnedClock))
		require.Equal(t, "master", returnedClock.Branch)
		require.Equal(t, uint64(0), returnedClock.Clock)

		key := []interface{}{repo.Name, clock.Branch, clock.Clock}
		cursor, err = term.GetAllByIndex(commitBranchIndex, key).Run(dbClient)
		require.NoError(t, err)
		returnedCommit := &persist.Commit{}
		require.NoError(t, cursor.One(returnedCommit))
		require.Equal(t, commitID, returnedCommit.ID)
	})
}

func TestCommitBranchIndexHeadOfBranch(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		commitID2 := uuid.NewWithoutDashes()
		err = d.StartCommit(
			repo,
			commitID2,
			commitID,
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit2 := &pfs.Commit{Repo: repo, ID: commitID2}
		require.NoError(t, d.FinishCommit(commit2, timestampNow(), false, nil))

		commitID3 := uuid.NewWithoutDashes()
		err = d.StartCommit(
			repo,
			commitID3,
			commitID2,
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit3 := &pfs.Commit{Repo: repo, ID: commitID3}
		require.NoError(t, d.FinishCommit(commit3, timestampNow(), false, nil))

		// Now that the commits are chained together,
		// Grab the head of master branch using the index
		head := &persist.Commit{}
		term := gorethink.DB(dbName).Table(commitTable)
		cursor, err := term.OrderBy(gorethink.OrderByOpts{
			Index: gorethink.Desc(commitBranchIndex),
		}).Between([]interface{}{repo.Name, "master", 0}, []interface{}{repo.Name, "master", gorethink.MaxVal}, gorethink.BetweenOpts{
			RightBound: "open",
		}).Run(dbClient)
		require.NoError(t, err)
		require.NoError(t, cursor.One(head))
		require.Equal(t, commitID3, head.ID)
	})
}

/*
	diffPathIndex

	used nowhere?
	used in ListFile() to list diffs by path
*/

/* diffCommitIndex

Indexed on commitID field in diff row

Used to:

- in FinishCommit() to gather all of the diffs for this commit

*/

func TestDiffCommitIndexBasic(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		file := &pfs.File{
			Commit: &pfs.Commit{
				Repo: repo,
				ID:   commitID,
			},
		}
		d.PutFile(file, "", pfs.Delimiter_LINE, 0, strings.NewReader("foo\n"))

		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))
	})
}

/* commitModifiedPathsIndex

Indexed on: repo / path / branchclock

Used to:

- in GetFile() to gather all commits within a range that modified a given path

*/

/* commitDeletedPathsIndex

Indexed on :

Used to:

- in GetFile() given a range of commits, find the commits that deleted a path, return the leftmost one

*/

/* clockBranchIndex

Indexed on:

- repo && branchIndex

Used to:

- in ListBranch() to query the clocks table and return the branches

*/

func timestampNow() *google_protobuf.Timestamp {
	return &google_protobuf.Timestamp{Seconds: time.Now().Unix()}
}

func testSetup(t *testing.T, testCode func(drive.Driver, string, *gorethink.Session)) {
	dbName := "pachyderm_test_" + uuid.NewWithoutDashes()[0:12]
	if err := InitDB(RethinkAddress, dbName); err != nil {
		require.NoError(t, err)
		return
	}
	dbClient, err := gorethink.Connect(gorethink.ConnectOpts{
		Address: RethinkAddress,
		Timeout: connectTimeoutSeconds * time.Second,
	})
	require.NoError(t, err)
	d, err := NewDriver("localhost:30652", RethinkAddress, dbName)
	require.NoError(t, err)

	startBlockServer(t, dbName, d)

	testCode(d, dbName, dbClient)

	if err := RemoveDB(RethinkAddress, dbName); err != nil {
		require.NoError(t, err)
		return
	}
}

func runServers(t *testing.T, port int32, blockAPIServer pfsclient.BlockAPIServer) {
	ready := make(chan bool)
	go func() {
		err := protoserver.Serve(
			func(s *grpc.Server) {
				pfsclient.RegisterBlockAPIServer(s, blockAPIServer)
				close(ready)
			},
			protoserver.ServeOptions{Version: version.Version},
			protoserver.ServeEnv{GRPCPort: uint16(port)},
		)
		require.NoError(t, err)
	}()
	<-ready
}

func startBlockServer(t *testing.T, dbName string, driver drive.Driver) {

	root := uniqueString("/tmp/pach_test/run")
	var ports []int32
	ports = append(ports, atomic.AddInt32(&port, 1))
	var addresses []string
	for _, port := range ports {
		addresses = append(addresses, fmt.Sprintf("localhost:%d", port))
	}
	for _, port := range ports {
		blockAPIServer, err := pfsserver.NewLocalBlockAPIServer(root)
		require.NoError(t, err)
		runServers(t, port, blockAPIServer)
	}
	return
}

func uniqueString(prefix string) string {
	return prefix + "." + uuid.NewWithoutDashes()[0:12]
}
<commit_msg>Add failing test TestDiffCommitIndexBasic<commit_after>package persist

import (
	"fmt"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	pclient "github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	pfsclient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pfs/db/persist"
	"github.com/pachyderm/pachyderm/src/server/pfs/drive"
	pfsserver "github.com/pachyderm/pachyderm/src/server/pfs/server"

	"github.com/dancannon/gorethink"
	"go.pedge.io/pb/go/google/protobuf"
	"go.pedge.io/proto/server"
	"google.golang.org/grpc"
)

var (
	port int32 = 30651
)

/*
	CommitBranchIndex

	Given a repo and a clock, returns a commit

	Is used in several places to:

	- find a parent commit given the parent's id in the form of an alias (e.g. "master/0")
	- getHeadOfBranch() -- by doing a range query of the form "branchName/0" to "branchName/max" and returning the last result (in this case the head)
	- getIDOfParentcommit() -- by decrementing this commit's clock value, and searching for that new clock
	- getCommitByAmbmiguousID() -- if the commit ID is in the form of an alias, find the commit using the index

*/

func TestCommitBranchIndexBasic(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)

		term := gorethink.DB(dbName).Table(commitTable)
		cursor, err := term.GetAll(commitID).Run(dbClient)
		require.NoError(t, err)
		c := &persist.Commit{}
		err = cursor.One(c)
		require.NoError(t, err)
		require.Equal(t, commitID, c.ID)

		clock := &persist.Clock{Branch: "master", Clock: 0}
		clockID := getClockID(repo.Name, clock).ID
		cursor, err = gorethink.DB(dbName).Table(clockTable).GetAll(clockID).Run(dbClient)
		require.NoError(t, err)
		returnedClock := &persist.Clock{}
		require.NoError(t, cursor.One(returnedClock))
		require.Equal(t, "master", returnedClock.Branch)
		require.Equal(t, uint64(0), returnedClock.Clock)

		key := []interface{}{repo.Name, clock.Branch, clock.Clock}
		cursor, err = term.GetAllByIndex(commitBranchIndex, key).Run(dbClient)
		require.NoError(t, err)
		returnedCommit := &persist.Commit{}
		require.NoError(t, cursor.One(returnedCommit))
		require.Equal(t, commitID, returnedCommit.ID)
	})
}

func TestCommitBranchIndexHeadOfBranch(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		commitID2 := uuid.NewWithoutDashes()
		err = d.StartCommit(
			repo,
			commitID2,
			commitID,
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit2 := &pfs.Commit{Repo: repo, ID: commitID2}
		require.NoError(t, d.FinishCommit(commit2, timestampNow(), false, nil))

		commitID3 := uuid.NewWithoutDashes()
		err = d.StartCommit(
			repo,
			commitID3,
			commitID2,
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit3 := &pfs.Commit{Repo: repo, ID: commitID3}
		require.NoError(t, d.FinishCommit(commit3, timestampNow(), false, nil))

		// Now that the commits are chained together,
		// Grab the head of master branch using the index
		head := &persist.Commit{}
		term := gorethink.DB(dbName).Table(commitTable)
		cursor, err := term.OrderBy(gorethink.OrderByOpts{
			Index: gorethink.Desc(commitBranchIndex),
		}).Between([]interface{}{repo.Name, "master", 0}, []interface{}{repo.Name, "master", gorethink.MaxVal}, gorethink.BetweenOpts{
			RightBound: "open",
		}).Run(dbClient)
		require.NoError(t, err)
		require.NoError(t, cursor.One(head))
		require.Equal(t, commitID3, head.ID)
	})
}

/*
	diffPathIndex

	used nowhere?
	used in ListFile() to list diffs by path
*/

/* diffCommitIndex

Indexed on commitID field in diff row

Used to:

- in FinishCommit() to gather all of the diffs for this commit

*/

func TestDiffCommitIndexBasic(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		file := &pfs.File{
			Commit: &pfs.Commit{
				Repo: repo,
				ID:   commitID,
			},
			Path: "file",
		}
		d.PutFile(file, "", pfs.Delimiter_LINE, 0, strings.NewReader("foo\n"))

		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		cursor, err := gorethink.DB(dbName).Table(diffTable).GetAllByIndex(diffCommitIndex, commitID).Run(dbClient)
		require.NoError(t, err)
		diff := &persist.Diff{}
		require.NoError(t, cursor.One(diff))
		fmt.Printf("got first diff: %v\n", diff)
		require.Equal(t, "file", diff.Path)
		require.Equal(t, 1, len(diff.BlockRefs))

		block := diff.BlockRefs[0]
		fmt.Printf("block: %v\n", block)
		blockSize := block.Upper - block.Lower

		// Was trying to check on a per block level ...
		// But even GetFile() doesn't seem to return the correct results
		// reader, err := d.GetFile(file, nil, 0,
		//	int64(blockSize), &pfs.Commit{Repo: repo, ID: commitID}, 0, false, "")

		reader, err := client.GetBlock(block.Hash, uint64(0), uint64(blockSize))
		require.NoError(t, err)
		var data []byte
		size, err := reader.Read(data)
		fmt.Printf("data=%v, err=%v\n", string(data), err)
		fmt.Printf("size=%v\n", size)
		require.NoError(t, err)
		require.Equal(t, "foo\n", string(data))

	})
}

/* commitModifiedPathsIndex

Indexed on: repo / path / branchclock

Used to:

- in GetFile() to gather all commits within a range that modified a given path

*/

/* commitDeletedPathsIndex

Indexed on :

Used to:

- in GetFile() given a range of commits, find the commits that deleted a path, return the leftmost one

*/

/* clockBranchIndex

Indexed on:

- repo && branchIndex

Used to:

- in ListBranch() to query the clocks table and return the branches

*/

func timestampNow() *google_protobuf.Timestamp {
	return &google_protobuf.Timestamp{Seconds: time.Now().Unix()}
}

func testSetup(t *testing.T, testCode func(drive.Driver, string, *gorethink.Session, pclient.APIClient)) {
	dbName := "pachyderm_test_" + uuid.NewWithoutDashes()[0:12]
	if err := InitDB(RethinkAddress, dbName); err != nil {
		require.NoError(t, err)
		return
	}
	dbClient, err := gorethink.Connect(gorethink.ConnectOpts{
		Address: RethinkAddress,
		Timeout: connectTimeoutSeconds * time.Second,
	})
	require.NoError(t, err)
	_client, d := startBlockServerAndGetClient(t, dbName)

	testCode(d, dbName, dbClient, _client)

	if err := RemoveDB(RethinkAddress, dbName); err != nil {
		require.NoError(t, err)
		return
	}
}

func runServers(t *testing.T, port int32, blockAPIServer pfsclient.BlockAPIServer) {
	ready := make(chan bool)
	go func() {
		err := protoserver.Serve(
			func(s *grpc.Server) {
				pfsclient.RegisterBlockAPIServer(s, blockAPIServer)
				close(ready)
			},
			protoserver.ServeOptions{Version: version.Version},
			protoserver.ServeEnv{GRPCPort: uint16(port)},
		)
		require.NoError(t, err)
	}()
	<-ready
}

func startBlockServerAndGetClient(t *testing.T, dbName string) (pclient.APIClient, drive.Driver) {

	root := uniqueString("/tmp/pach_test/run")
	var ports []int32
	ports = append(ports, atomic.AddInt32(&port, 1))
	var addresses []string
	for _, port := range ports {
		addresses = append(addresses, fmt.Sprintf("localhost:%d", port))
	}
	var driver drive.Driver
	for i, port := range ports {
		address := addresses[i]
		_driver, err := NewDriver(address, RethinkAddress, dbName)
		driver = _driver
		require.NoError(t, err)
		blockAPIServer, err := pfsserver.NewLocalBlockAPIServer(root)
		require.NoError(t, err)
		runServers(t, port, blockAPIServer)
	}
	clientConn, err := grpc.Dial(addresses[0], grpc.WithInsecure())
	require.NoError(t, err)
	return pclient.APIClient{BlockAPIClient: pfsclient.NewBlockAPIClient(clientConn)}, driver
}

func uniqueString(prefix string) string {
	return prefix + "." + uuid.NewWithoutDashes()[0:12]
}
<|endoftext|>