<commit_before>package record

import (
	"context"
	"fmt"
	"strings"
	"time"

	"sync"

	"github.com/CyCoreSystems/ari"
	"github.com/pkg/errors"
	uuid "github.com/satori/go.uuid"
)

var (

	// RecordingStartTimeout is the amount of time to wait for a recording to start
	// before declaring the recording to have failed.
	RecordingStartTimeout = 1 * time.Second

	// DefaultMaximumDuration is the default maximum amount of time a recording
	// should be allowed to continue before being terminated.
	DefaultMaximumDuration = 24 * time.Hour

	// DefaultMaximumSilence is the default maximum amount of time silence may be
	// detected before terminating the recording.
	DefaultMaximumSilence = 5 * time.Minute

	// ShutdownGracePeriod is the amount of time to allow a Stop transaction to
	// complete before shutting down the session anyway.
	ShutdownGracePeriod = 3 * time.Second
)

// Options describes a set of recording options for a recording Session
type Options struct {
	// name is the name for the live recording
	name string

	format string

	maxDuration time.Duration

	maxSilence time.Duration

	ifExists string

	beep bool

	terminateOn string
}

func defaultOptions() *Options {
	return &Options{
		beep:        false,
		format:      "wav",
		ifExists:    "fail",
		maxDuration: DefaultMaximumDuration,
		maxSilence:  DefaultMaximumSilence,
		name:        uuid.NewV1().String(),
		terminateOn: "none",
	}
}

func (o *Options) toRecordingOptions() *ari.RecordingOptions {
	return &ari.RecordingOptions{
		Beep:        o.beep,
		Format:      o.format,
		Exists:      o.ifExists,
		MaxDuration: o.maxDuration,
		MaxSilence:  o.maxSilence,
		Terminate:   o.terminateOn,
	}
}

// Apply applies a set of options for the recording Session
func (o *Options) Apply(opts ...OptionFunc) {
	for _, f := range opts {
		f(o)
	}
}

// OptionFunc is a function which applies changes to an Options set
type OptionFunc func(*Options)

// Beep indicates that a beep should be played to signal the start of recording
func Beep() OptionFunc {
	return func(o *Options) {
		o.beep = true
	}
}

// Format configures the file format to be used to store the recording
func Format(format string) OptionFunc {
	return func(o *Options) {
		o.format = format
	}
}

// IfExists configures the behaviour of the recording if the file to be
// recorded already exists.
//
// Valid options are:  "fail" (default), "overwrite", and "append".
func IfExists(action string) OptionFunc {
	return func(o *Options) {
		o.ifExists = action
	}
}

// MaxDuration sets the maximum duration to allow for the recording.  After
// this amount of time, the recording will be automatically Finished.
//
// A setting of 0 disables the limit.
func MaxDuration(max time.Duration) OptionFunc {
	return func(o *Options) {
		o.maxDuration = max
	}
}

// MaxSilence sets the amount of time a block of silence is allowed to become
// before the recording should be declared Finished.
//
// A setting of 0 disables silence detection.
func MaxSilence(max time.Duration) OptionFunc {
	return func(o *Options) {
		o.maxSilence = max
	}
}

// Name configures the recording to use the provided name
func Name(name string) OptionFunc {
	return func(o *Options) {
		o.name = name
	}
}

// TerminateOn configures the DTMF which, if received, will terminate the
// recording.
//
// Valid values are "none" (default), "any", "*", and "#".
func TerminateOn(dtmf string) OptionFunc {
	return func(o *Options) {
		o.terminateOn = dtmf
	}
}

// Session desribes the interface to a generic recording session
type Session interface {
	// Done returns a channel which is closed when the session is complete
	Done() <-chan struct{}

	// Err waits for the session to complete, then returns any error encountered
	// during its execution
	Err() error

	// Key returns the ari.Key for the LiveRecording of this session, if one exists.
	Key() *ari.Key

	// Pause temporarily stops the recording session without ending the session
	Pause() error

	// Result waits for the session to complete, then returns the Result
	Result() (*Result, error)

	// Resume restarts a paused recording session
	Resume() error

	// Scrap terminates the recording session and throws away the recording.
	Scrap() error

	// Stop stops the recording session
	Stop() *Result
}

// Result represents the result of a recording Session.  It provides an
// interface to disposition the recording.
type Result struct {
	h *ari.StoredRecordingHandle

	// Data holds the final data for the LiveRecording, if it was successful
	Data *ari.LiveRecordingData

	// DTMF holds any DTMF digits which are received during the recording session
	DTMF string

	// Duration indicates the duration of the recording
	Duration time.Duration

	// Error holds any error encountered during the recording session
	Error error

	// Hangup indicates that the Recorder disappeared (due to hangup or
	// destruction) during or after the recording.
	Hangup bool

	overwrite bool
}

// Delete discards the recording
func (r *Result) Delete() error {
	if r.h == nil {
		return errors.New("no stored recording handle available")
	}
	return r.h.Delete()
}

// Key returns the ari.Key of the StoredRecording, if one exists.
func (r *Result) Key() *ari.Key {
	if r == nil || r.h == nil {
		return nil
	}
	return r.h.Key()
}

// Save stores the recording to the given name
func (r *Result) Save(name string) error {
	if name == "" {
		// no name indicates the default, which is where it already is
		return nil
	}

	if r.h == nil {
		return errors.New("no stored recording handle available")
	}

	// Copy the recording to the desired name
	destH, err := r.h.Copy(name)
	if err != nil {
		if !strings.Contains(err.Error(), "409 Conflict") || !r.overwrite {
			return errors.Wrapf(err, "failed to copy recording (%s)", r.h.ID())
		}

		// we are set to overwrite, so delete the previous recording
		Logger.Debug("overwriting previous recording")
		err = destH.Delete()
		if err != nil {
			return errors.Wrap(err, "failed to remove previous destination recording")
		}
		_, err = r.h.Copy(name)
		if err != nil {
			return errors.Wrap(err, "failed to copy recording")
		}
	}

	// Delete the original
	err = r.h.Delete()
	if err != nil {
		return errors.Wrap(err, "failed to remove temporary recording after copy")
	}

	return nil
}

// URI returns the AudioURI to play the recording
func (r *Result) URI() string {
	return "recording:" + r.h.ID()
}

// Record starts a new recording Session
func Record(ctx context.Context, r ari.Recorder, opts ...OptionFunc) Session {
	s := newRecordingSession(opts...)

	var wg sync.WaitGroup
	wg.Add(1)
	go s.record(ctx, r, &wg)
	wg.Wait()

	Logger.Debug("returned from internal recording start")
	return s
}

// New creates a new recording Session
func newRecordingSession(opts ...OptionFunc) *recordingSession {
	o := defaultOptions()
	o.Apply(opts...)

	s := &recordingSession{
		cancel:  func() {},
		options: o,
		doneCh:  make(chan struct{}),
		res:     new(Result),
	}

	// If the recording options declare that we should overwrite,
	// carry that over to the copy destination.
	if o.ifExists == "overwrite" {
		s.res.overwrite = true
	}

	return s
}

type recordingSession struct {
	h *ari.LiveRecordingHandle

	cancel context.CancelFunc

	doneCh chan struct{}

	options *Options

	res *Result
}

func (s *recordingSession) Done() <-chan struct{} {
	return s.doneCh
}

func (s *recordingSession) Err() error {
	<-s.Done()
	return s.res.Error
}

func (s *recordingSession) Key() *ari.Key {
	if s == nil || s.h == nil {
		return nil
	}
	return s.h.Key()
}

func (s *recordingSession) Result() (*Result, error) {
	<-s.Done()
	return s.res, s.res.Error
}

func (s *recordingSession) Pause() error {
	return s.h.Pause()
}

func (s *recordingSession) Resume() error {
	return s.h.Resume()
}

func (s *recordingSession) Scrap() error {
	return s.h.Scrap()
}

func (s *recordingSession) Stop() *Result {
	// Signal stop
	s.res.Error = s.h.Stop()

	// If we successfully signaled a stop, Wait for the stop to complete
	if s.res.Error == nil {
		select {
		case <-s.Done():
		case <-time.After(ShutdownGracePeriod):
		}
	}

	// Shut down the session
	if s.cancel != nil {
		s.cancel()
	}

	// Return the result
	return s.res
}

// nolint: gocyclo
func (s *recordingSession) record(ctx context.Context, r ari.Recorder, wg *sync.WaitGroup) {
	defer close(s.doneCh)

	ctx, cancel := context.WithCancel(ctx)
	s.cancel = cancel
	defer cancel()

	var err error
	s.h, err = r.StageRecord(s.options.name, s.options.toRecordingOptions())
	if err != nil {
		s.res.Error = errors.Wrap(err, "failed to stage recording")
		wg.Done()
		return
	}

	// Store the eventual StoredRecording handle to the Result
	s.res.h = s.h.Stored()

	dtmfSub := r.Subscribe(ari.Events.ChannelDtmfReceived)
	hangupSub := r.Subscribe(ari.Events.ChannelDestroyed, ari.Events.ChannelHangupRequest, ari.Events.BridgeDestroyed)
	startSub := s.h.Subscribe(ari.Events.RecordingStarted)
	failedSub := s.h.Subscribe(ari.Events.RecordingFailed)
	finishedSub := s.h.Subscribe(ari.Events.RecordingFinished)

	defer func() {
		hangupSub.Cancel()
		failedSub.Cancel()
		startSub.Cancel()
		finishedSub.Cancel()
		dtmfSub.Cancel()
	}()

	wg.Done()

	// Record the duration of the recording
	started := time.Now()
	defer func() {
		s.res.Duration = time.Since(started)
		Logger.Debug("recording duration", "duration", s.res.Duration)
	}()

	// Record any DTMF received during the recording
	go s.collectDtmf(ctx, dtmfSub)

	// Record hangup or destruction of our Recorder
	go s.watchHangup(ctx, hangupSub)

	// Start recording
	Logger.Debug("starting recording")
	if err := s.h.Exec(); err != nil {
		s.res.Error = err
		return
	}

	// Time the recording
	startTimer := time.NewTimer(RecordingStartTimeout)

	for {
		select {
		case <-ctx.Done():
			s.Stop()
			return
		case <-startTimer.C:
			Logger.Debug("timeout waiting to start recording")
			s.res.Error = timeoutErr{"Timeout waiting for recording to start"}
			return
		case _, ok := <-startSub.Events():
			if !ok {
				return
			}
			Logger.Debug("recording started")
			startTimer.Stop()
		case e, ok := <-failedSub.Events():
			if !ok {
				return
			}
			Logger.Debug("recording failed")
			r := e.(*ari.RecordingFailed).Recording
			s.res.Data = &r
			s.res.Error = fmt.Errorf("Recording failed: %s", r.Cause)
			return
		case e, ok := <-finishedSub.Events():
			if !ok {
				return
			}
			r := e.(*ari.RecordingFinished).Recording
			Logger.Debug("recording finished")
			s.res.Data = &r
			return
		}
	}
}

func (s *recordingSession) collectDtmf(ctx context.Context, dtmfSub ari.Subscription) {
	for {
		select {
		case e, ok := <-dtmfSub.Events():
			if !ok {
				return
			}
			v := e.(*ari.ChannelDtmfReceived)
			s.res.DTMF += v.Digit
		case <-ctx.Done():
			return
		}
	}
}

func (s *recordingSession) watchHangup(ctx context.Context, hangupSub ari.Subscription) {
	select {
	case <-hangupSub.Events():
		s.res.Hangup = true
	case <-ctx.Done():
	}
}

type timeoutErr struct {
	msg string
}

func (err timeoutErr) Error() string {
	return err.msg
}

func (err timeoutErr) Timeout() bool {
	return true
}
<commit_msg>ext/record: failure to delete during save is not an error<commit_after>package record

import (
	"context"
	"fmt"
	"strings"
	"time"

	"sync"

	"github.com/CyCoreSystems/ari"
	"github.com/pkg/errors"
	uuid "github.com/satori/go.uuid"
)

var (

	// RecordingStartTimeout is the amount of time to wait for a recording to start
	// before declaring the recording to have failed.
	RecordingStartTimeout = 1 * time.Second

	// DefaultMaximumDuration is the default maximum amount of time a recording
	// should be allowed to continue before being terminated.
	DefaultMaximumDuration = 24 * time.Hour

	// DefaultMaximumSilence is the default maximum amount of time silence may be
	// detected before terminating the recording.
	DefaultMaximumSilence = 5 * time.Minute

	// ShutdownGracePeriod is the amount of time to allow a Stop transaction to
	// complete before shutting down the session anyway.
	ShutdownGracePeriod = 3 * time.Second
)

// Options describes a set of recording options for a recording Session
type Options struct {
	// name is the name for the live recording
	name string

	format string

	maxDuration time.Duration

	maxSilence time.Duration

	ifExists string

	beep bool

	terminateOn string
}

func defaultOptions() *Options {
	return &Options{
		beep:        false,
		format:      "wav",
		ifExists:    "fail",
		maxDuration: DefaultMaximumDuration,
		maxSilence:  DefaultMaximumSilence,
		name:        uuid.NewV1().String(),
		terminateOn: "none",
	}
}

func (o *Options) toRecordingOptions() *ari.RecordingOptions {
	return &ari.RecordingOptions{
		Beep:        o.beep,
		Format:      o.format,
		Exists:      o.ifExists,
		MaxDuration: o.maxDuration,
		MaxSilence:  o.maxSilence,
		Terminate:   o.terminateOn,
	}
}

// Apply applies a set of options for the recording Session
func (o *Options) Apply(opts ...OptionFunc) {
	for _, f := range opts {
		f(o)
	}
}

// OptionFunc is a function which applies changes to an Options set
type OptionFunc func(*Options)

// Beep indicates that a beep should be played to signal the start of recording
func Beep() OptionFunc {
	return func(o *Options) {
		o.beep = true
	}
}

// Format configures the file format to be used to store the recording
func Format(format string) OptionFunc {
	return func(o *Options) {
		o.format = format
	}
}

// IfExists configures the behaviour of the recording if the file to be
// recorded already exists.
//
// Valid options are:  "fail" (default), "overwrite", and "append".
func IfExists(action string) OptionFunc {
	return func(o *Options) {
		o.ifExists = action
	}
}

// MaxDuration sets the maximum duration to allow for the recording.  After
// this amount of time, the recording will be automatically Finished.
//
// A setting of 0 disables the limit.
func MaxDuration(max time.Duration) OptionFunc {
	return func(o *Options) {
		o.maxDuration = max
	}
}

// MaxSilence sets the amount of time a block of silence is allowed to become
// before the recording should be declared Finished.
//
// A setting of 0 disables silence detection.
func MaxSilence(max time.Duration) OptionFunc {
	return func(o *Options) {
		o.maxSilence = max
	}
}

// Name configures the recording to use the provided name
func Name(name string) OptionFunc {
	return func(o *Options) {
		o.name = name
	}
}

// TerminateOn configures the DTMF which, if received, will terminate the
// recording.
//
// Valid values are "none" (default), "any", "*", and "#".
func TerminateOn(dtmf string) OptionFunc {
	return func(o *Options) {
		o.terminateOn = dtmf
	}
}

// Session desribes the interface to a generic recording session
type Session interface {
	// Done returns a channel which is closed when the session is complete
	Done() <-chan struct{}

	// Err waits for the session to complete, then returns any error encountered
	// during its execution
	Err() error

	// Key returns the ari.Key for the LiveRecording of this session, if one exists.
	Key() *ari.Key

	// Pause temporarily stops the recording session without ending the session
	Pause() error

	// Result waits for the session to complete, then returns the Result
	Result() (*Result, error)

	// Resume restarts a paused recording session
	Resume() error

	// Scrap terminates the recording session and throws away the recording.
	Scrap() error

	// Stop stops the recording session
	Stop() *Result
}

// Result represents the result of a recording Session.  It provides an
// interface to disposition the recording.
type Result struct {
	h *ari.StoredRecordingHandle

	// Data holds the final data for the LiveRecording, if it was successful
	Data *ari.LiveRecordingData

	// DTMF holds any DTMF digits which are received during the recording session
	DTMF string

	// Duration indicates the duration of the recording
	Duration time.Duration

	// Error holds any error encountered during the recording session
	Error error

	// Hangup indicates that the Recorder disappeared (due to hangup or
	// destruction) during or after the recording.
	Hangup bool

	overwrite bool
}

// Delete discards the recording
func (r *Result) Delete() error {
	if r.h == nil {
		return errors.New("no stored recording handle available")
	}
	return r.h.Delete()
}

// Key returns the ari.Key of the StoredRecording, if one exists.
func (r *Result) Key() *ari.Key {
	if r == nil || r.h == nil {
		return nil
	}
	return r.h.Key()
}

// Save stores the recording to the given name
func (r *Result) Save(name string) error {
	if name == "" {
		// no name indicates the default, which is where it already is
		return nil
	}

	if r.h == nil {
		return errors.New("no stored recording handle available")
	}

	// Copy the recording to the desired name
	destH, err := r.h.Copy(name)
	if err != nil {
		if !strings.Contains(err.Error(), "409 Conflict") || !r.overwrite {
			return errors.Wrapf(err, "failed to copy recording (%s)", r.h.ID())
		}

		// we are set to overwrite, so delete the previous recording
		Logger.Debug("overwriting previous recording")
		err = destH.Delete()
		if err != nil {
			Logger.Warn("failed to remove previous destination recording; it may not have existed", "error", err)
		}
		_, err = r.h.Copy(name)
		if err != nil {
			return errors.Wrap(err, "failed to copy recording")
		}
	}

	// Delete the original
	err = r.h.Delete()
	if err != nil {
		return errors.Wrap(err, "failed to remove temporary recording after copy")
	}

	return nil
}

// URI returns the AudioURI to play the recording
func (r *Result) URI() string {
	return "recording:" + r.h.ID()
}

// Record starts a new recording Session
func Record(ctx context.Context, r ari.Recorder, opts ...OptionFunc) Session {
	s := newRecordingSession(opts...)

	var wg sync.WaitGroup
	wg.Add(1)
	go s.record(ctx, r, &wg)
	wg.Wait()

	Logger.Debug("returned from internal recording start")
	return s
}

// New creates a new recording Session
func newRecordingSession(opts ...OptionFunc) *recordingSession {
	o := defaultOptions()
	o.Apply(opts...)

	s := &recordingSession{
		cancel:  func() {},
		options: o,
		doneCh:  make(chan struct{}),
		res:     new(Result),
	}

	// If the recording options declare that we should overwrite,
	// carry that over to the copy destination.
	if o.ifExists == "overwrite" {
		s.res.overwrite = true
	}

	return s
}

type recordingSession struct {
	h *ari.LiveRecordingHandle

	cancel context.CancelFunc

	doneCh chan struct{}

	options *Options

	res *Result
}

func (s *recordingSession) Done() <-chan struct{} {
	return s.doneCh
}

func (s *recordingSession) Err() error {
	<-s.Done()
	return s.res.Error
}

func (s *recordingSession) Key() *ari.Key {
	if s == nil || s.h == nil {
		return nil
	}
	return s.h.Key()
}

func (s *recordingSession) Result() (*Result, error) {
	<-s.Done()
	return s.res, s.res.Error
}

func (s *recordingSession) Pause() error {
	return s.h.Pause()
}

func (s *recordingSession) Resume() error {
	return s.h.Resume()
}

func (s *recordingSession) Scrap() error {
	return s.h.Scrap()
}

func (s *recordingSession) Stop() *Result {
	// Signal stop
	s.res.Error = s.h.Stop()

	// If we successfully signaled a stop, Wait for the stop to complete
	if s.res.Error == nil {
		select {
		case <-s.Done():
		case <-time.After(ShutdownGracePeriod):
		}
	}

	// Shut down the session
	if s.cancel != nil {
		s.cancel()
	}

	// Return the result
	return s.res
}

// nolint: gocyclo
func (s *recordingSession) record(ctx context.Context, r ari.Recorder, wg *sync.WaitGroup) {
	defer close(s.doneCh)

	ctx, cancel := context.WithCancel(ctx)
	s.cancel = cancel
	defer cancel()

	var err error
	s.h, err = r.StageRecord(s.options.name, s.options.toRecordingOptions())
	if err != nil {
		s.res.Error = errors.Wrap(err, "failed to stage recording")
		wg.Done()
		return
	}

	// Store the eventual StoredRecording handle to the Result
	s.res.h = s.h.Stored()

	dtmfSub := r.Subscribe(ari.Events.ChannelDtmfReceived)
	hangupSub := r.Subscribe(ari.Events.ChannelDestroyed, ari.Events.ChannelHangupRequest, ari.Events.BridgeDestroyed)
	startSub := s.h.Subscribe(ari.Events.RecordingStarted)
	failedSub := s.h.Subscribe(ari.Events.RecordingFailed)
	finishedSub := s.h.Subscribe(ari.Events.RecordingFinished)

	defer func() {
		hangupSub.Cancel()
		failedSub.Cancel()
		startSub.Cancel()
		finishedSub.Cancel()
		dtmfSub.Cancel()
	}()

	wg.Done()

	// Record the duration of the recording
	started := time.Now()
	defer func() {
		s.res.Duration = time.Since(started)
		Logger.Debug("recording duration", "duration", s.res.Duration)
	}()

	// Record any DTMF received during the recording
	go s.collectDtmf(ctx, dtmfSub)

	// Record hangup or destruction of our Recorder
	go s.watchHangup(ctx, hangupSub)

	// Start recording
	Logger.Debug("starting recording")
	if err := s.h.Exec(); err != nil {
		s.res.Error = err
		return
	}

	// Time the recording
	startTimer := time.NewTimer(RecordingStartTimeout)

	for {
		select {
		case <-ctx.Done():
			s.Stop()
			return
		case <-startTimer.C:
			Logger.Debug("timeout waiting to start recording")
			s.res.Error = timeoutErr{"Timeout waiting for recording to start"}
			return
		case _, ok := <-startSub.Events():
			if !ok {
				return
			}
			Logger.Debug("recording started")
			startTimer.Stop()
		case e, ok := <-failedSub.Events():
			if !ok {
				return
			}
			Logger.Debug("recording failed")
			r := e.(*ari.RecordingFailed).Recording
			s.res.Data = &r
			s.res.Error = fmt.Errorf("Recording failed: %s", r.Cause)
			return
		case e, ok := <-finishedSub.Events():
			if !ok {
				return
			}
			r := e.(*ari.RecordingFinished).Recording
			Logger.Debug("recording finished")
			s.res.Data = &r
			return
		}
	}
}

func (s *recordingSession) collectDtmf(ctx context.Context, dtmfSub ari.Subscription) {
	for {
		select {
		case e, ok := <-dtmfSub.Events():
			if !ok {
				return
			}
			v := e.(*ari.ChannelDtmfReceived)
			s.res.DTMF += v.Digit
		case <-ctx.Done():
			return
		}
	}
}

func (s *recordingSession) watchHangup(ctx context.Context, hangupSub ari.Subscription) {
	select {
	case <-hangupSub.Events():
		s.res.Hangup = true
	case <-ctx.Done():
	}
}

type timeoutErr struct {
	msg string
}

func (err timeoutErr) Error() string {
	return err.msg
}

func (err timeoutErr) Timeout() bool {
	return true
}
<|endoftext|>


<commit_before>package fasthttputil

import (
	"errors"
	"io"
	"net"
	"sync"
	"time"
)

// NewPipeConns returns new bi-directonal connection pipe.
func NewPipeConns() *PipeConns {
	ch1 := acquirePipeChan()
	ch2 := acquirePipeChan()

	pc := &PipeConns{}
	pc.c1.r = ch1
	pc.c1.w = ch2
	pc.c2.r = ch2
	pc.c2.w = ch1
	pc.c1.pc = pc
	pc.c2.pc = pc
	return pc
}

// PipeConns provides bi-directional connection pipe,
// which use in-process memory as a transport.
//
// PipeConns must be created by calling NewPipeConns.
//
// PipeConns have the following additional features comparing to connections
// returned from net.Pipe():
//
//   * It is faster.
//   * It buffers Write calls, so there is no need to have concurrent goroutine
//     calling Read in order to unblock each Write call.
type PipeConns struct {
	c1 pipeConn
	c2 pipeConn
}

// Conn1 returns the first end of bi-directional pipe.
//
// Data written to Conn1 may be read from Conn2.
// Data written to Conn2 may be read from Conn1.
func (pc *PipeConns) Conn1() net.Conn {
	return &pc.c1
}

// Conn2 returns the second end of bi-directional pipe.
//
// Data written to Conn2 may be read from Conn1.
// Data written to Conn1 may be read from Conn2.
func (pc *PipeConns) Conn2() net.Conn {
	return &pc.c2
}

func (pc *PipeConns) release() {
	pc.c1.wlock.Lock()
	pc.c2.wlock.Lock()
	mustRelease := pc.c1.wclosed && pc.c2.wclosed
	pc.c1.wlock.Unlock()
	pc.c2.wlock.Unlock()

	if mustRelease {
		pc.c1.release()
		pc.c2.release()
	}
}

type pipeConn struct {
	r  *pipeChan
	w  *pipeChan
	b  *byteBuffer
	bb []byte

	rlock   sync.Mutex
	rclosed bool

	wlock   sync.Mutex
	wclosed bool

	pc *PipeConns
}

func (c *pipeConn) Write(p []byte) (int, error) {
	b := acquireByteBuffer()
	b.b = append(b.b[:0], p...)

	c.wlock.Lock()
	if c.wclosed {
		c.wlock.Unlock()
		releaseByteBuffer(b)
		return 0, errors.New("connection closed for writing")
	}
	c.w.ch <- b
	c.wlock.Unlock()

	return len(p), nil
}

func (c *pipeConn) Read(p []byte) (int, error) {
	mayBlock := true
	nn := 0
	for len(p) > 0 {
		n, err := c.read(p, mayBlock)
		nn += n
		if err != nil {
			if !mayBlock && err == errWouldBlock {
				err = nil
			}
			return nn, err
		}
		p = p[n:]
		mayBlock = false
	}

	return nn, nil
}

func (c *pipeConn) read(p []byte, mayBlock bool) (int, error) {
	if len(c.bb) == 0 {
		releaseByteBuffer(c.b)
		c.b = nil

		c.rlock.Lock()
		if c.rclosed {
			c.rlock.Unlock()
			return 0, io.EOF
		}

		if mayBlock {
			c.b = <-c.r.ch
		} else {
			select {
			case c.b = <-c.r.ch:
			default:
				c.rlock.Unlock()
				return 0, errWouldBlock
			}
		}

		if c.b == nil {
			c.rclosed = true
			c.rlock.Unlock()
			return 0, io.EOF
		}
		c.rlock.Unlock()

		c.bb = c.b.b
	}
	n := copy(p, c.bb)
	c.bb = c.bb[n:]

	return n, nil
}

var errWouldBlock = errors.New("would block")

func (c *pipeConn) Close() error {
	c.wlock.Lock()
	if c.wclosed {
		c.wlock.Unlock()
		return errors.New("connection already closed")
	}

	c.wclosed = true
	c.w.ch <- nil
	c.wlock.Unlock()

	c.pc.release()
	return nil
}

func (c *pipeConn) release() {
	c.rlock.Lock()

	releaseByteBuffer(c.b)
	c.b = nil
	c.bb = nil

	if !c.rclosed {
		c.rclosed = true
		for b := range c.r.ch {
			releaseByteBuffer(b)
			if b == nil {
				break
			}
		}
	}
	if c.r != nil {
		releasePipeChan(c.r)
		c.r = nil
		c.w = nil
	}

	c.rlock.Unlock()
}

func (p *pipeConn) LocalAddr() net.Addr {
	return pipeAddr(0)
}

func (p *pipeConn) RemoteAddr() net.Addr {
	return pipeAddr(0)
}

func (p *pipeConn) SetDeadline(t time.Time) error {
	return errors.New("deadline not supported")
}

func (p *pipeConn) SetReadDeadline(t time.Time) error {
	return p.SetDeadline(t)
}

func (p *pipeConn) SetWriteDeadline(t time.Time) error {
	return p.SetDeadline(t)
}

type pipeAddr int

func (pipeAddr) Network() string {
	return "pipe"
}

func (pipeAddr) String() string {
	return "pipe"
}

type byteBuffer struct {
	b []byte
}

func acquireByteBuffer() *byteBuffer {
	return byteBufferPool.Get().(*byteBuffer)
}

func releaseByteBuffer(b *byteBuffer) {
	if b != nil {
		byteBufferPool.Put(b)
	}
}

var byteBufferPool = &sync.Pool{
	New: func() interface{} {
		return &byteBuffer{}
	},
}

func acquirePipeChan() *pipeChan {
	ch := pipeChanPool.Get().(*pipeChan)
	if len(ch.ch) > 0 {
		panic("BUG: non-empty pipeChan acquired")
	}
	return ch
}

func releasePipeChan(ch *pipeChan) {
	if len(ch.ch) > 0 {
		panic("BUG: non-empty pipeChan released")
	}
	pipeChanPool.Put(ch)
}

var pipeChanPool = &sync.Pool{
	New: func() interface{} {
		return &pipeChan{
			ch: make(chan *byteBuffer, 4),
		}
	},
}

type pipeChan struct {
	ch chan *byteBuffer
}
<commit_msg>fasthttputil: pipe errors cleanup<commit_after>package fasthttputil

import (
	"errors"
	"io"
	"net"
	"sync"
	"time"
)

// NewPipeConns returns new bi-directonal connection pipe.
func NewPipeConns() *PipeConns {
	ch1 := acquirePipeChan()
	ch2 := acquirePipeChan()

	pc := &PipeConns{}
	pc.c1.r = ch1
	pc.c1.w = ch2
	pc.c2.r = ch2
	pc.c2.w = ch1
	pc.c1.pc = pc
	pc.c2.pc = pc
	return pc
}

// PipeConns provides bi-directional connection pipe,
// which use in-process memory as a transport.
//
// PipeConns must be created by calling NewPipeConns.
//
// PipeConns have the following additional features comparing to connections
// returned from net.Pipe():
//
//   * It is faster.
//   * It buffers Write calls, so there is no need to have concurrent goroutine
//     calling Read in order to unblock each Write call.
type PipeConns struct {
	c1 pipeConn
	c2 pipeConn
}

// Conn1 returns the first end of bi-directional pipe.
//
// Data written to Conn1 may be read from Conn2.
// Data written to Conn2 may be read from Conn1.
func (pc *PipeConns) Conn1() net.Conn {
	return &pc.c1
}

// Conn2 returns the second end of bi-directional pipe.
//
// Data written to Conn2 may be read from Conn1.
// Data written to Conn1 may be read from Conn2.
func (pc *PipeConns) Conn2() net.Conn {
	return &pc.c2
}

func (pc *PipeConns) release() {
	pc.c1.wlock.Lock()
	pc.c2.wlock.Lock()
	mustRelease := pc.c1.wclosed && pc.c2.wclosed
	pc.c1.wlock.Unlock()
	pc.c2.wlock.Unlock()

	if mustRelease {
		pc.c1.release()
		pc.c2.release()
	}
}

type pipeConn struct {
	r  *pipeChan
	w  *pipeChan
	b  *byteBuffer
	bb []byte

	rlock   sync.Mutex
	rclosed bool

	wlock   sync.Mutex
	wclosed bool

	pc *PipeConns
}

func (c *pipeConn) Write(p []byte) (int, error) {
	b := acquireByteBuffer()
	b.b = append(b.b[:0], p...)

	c.wlock.Lock()
	if c.wclosed {
		c.wlock.Unlock()
		releaseByteBuffer(b)
		return 0, errConnectionClosed
	}
	c.w.ch <- b
	c.wlock.Unlock()

	return len(p), nil
}

func (c *pipeConn) Read(p []byte) (int, error) {
	mayBlock := true
	nn := 0
	for len(p) > 0 {
		n, err := c.read(p, mayBlock)
		nn += n
		if err != nil {
			if !mayBlock && err == errWouldBlock {
				err = nil
			}
			return nn, err
		}
		p = p[n:]
		mayBlock = false
	}

	return nn, nil
}

func (c *pipeConn) read(p []byte, mayBlock bool) (int, error) {
	if len(c.bb) == 0 {
		releaseByteBuffer(c.b)
		c.b = nil

		c.rlock.Lock()
		if c.rclosed {
			c.rlock.Unlock()
			return 0, io.EOF
		}

		if mayBlock {
			c.b = <-c.r.ch
		} else {
			select {
			case c.b = <-c.r.ch:
			default:
				c.rlock.Unlock()
				return 0, errWouldBlock
			}
		}

		if c.b == nil {
			c.rclosed = true
			c.rlock.Unlock()
			return 0, io.EOF
		}
		c.rlock.Unlock()

		c.bb = c.b.b
	}
	n := copy(p, c.bb)
	c.bb = c.bb[n:]

	return n, nil
}

var (
	errWouldBlock       = errors.New("would block")
	errConnectionClosed = errors.New("connection closed")
)

func (c *pipeConn) Close() error {
	c.wlock.Lock()
	if c.wclosed {
		c.wlock.Unlock()
		return errConnectionClosed
	}

	c.wclosed = true
	c.w.ch <- nil
	c.wlock.Unlock()

	c.pc.release()
	return nil
}

func (c *pipeConn) release() {
	c.rlock.Lock()

	releaseByteBuffer(c.b)
	c.b = nil
	c.bb = nil

	if !c.rclosed {
		c.rclosed = true
		for b := range c.r.ch {
			releaseByteBuffer(b)
			if b == nil {
				break
			}
		}
	}
	if c.r != nil {
		releasePipeChan(c.r)
		c.r = nil
		c.w = nil
	}

	c.rlock.Unlock()
}

func (p *pipeConn) LocalAddr() net.Addr {
	return pipeAddr(0)
}

func (p *pipeConn) RemoteAddr() net.Addr {
	return pipeAddr(0)
}

func (p *pipeConn) SetDeadline(t time.Time) error {
	return errors.New("deadline not supported")
}

func (p *pipeConn) SetReadDeadline(t time.Time) error {
	return p.SetDeadline(t)
}

func (p *pipeConn) SetWriteDeadline(t time.Time) error {
	return p.SetDeadline(t)
}

type pipeAddr int

func (pipeAddr) Network() string {
	return "pipe"
}

func (pipeAddr) String() string {
	return "pipe"
}

type byteBuffer struct {
	b []byte
}

func acquireByteBuffer() *byteBuffer {
	return byteBufferPool.Get().(*byteBuffer)
}

func releaseByteBuffer(b *byteBuffer) {
	if b != nil {
		byteBufferPool.Put(b)
	}
}

var byteBufferPool = &sync.Pool{
	New: func() interface{} {
		return &byteBuffer{}
	},
}

func acquirePipeChan() *pipeChan {
	ch := pipeChanPool.Get().(*pipeChan)
	if len(ch.ch) > 0 {
		panic("BUG: non-empty pipeChan acquired")
	}
	return ch
}

func releasePipeChan(ch *pipeChan) {
	if len(ch.ch) > 0 {
		panic("BUG: non-empty pipeChan released")
	}
	pipeChanPool.Put(ch)
}

var pipeChanPool = &sync.Pool{
	New: func() interface{} {
		return &pipeChan{
			ch: make(chan *byteBuffer, 4),
		}
	},
}

type pipeChan struct {
	ch chan *byteBuffer
}
<|endoftext|>


<commit_before>// Application that does metrics analysis as described in the design doc:
// go/ct_metrics_analysis
//
// Can be tested locally with:
// $ go run go/worker_scripts/metrics_analysis/main.go --start_range=1 --num=3 --run_id=rmistry-test1 --benchmark_extra_args="--output-format=csv" --metric_name="loadingMetric" --logtostderr=true --local --chromium_hash=cab48032c45695c73b202af7d472a52808d551ca
//
package main

import (
	"bytes"
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"go.skia.org/infra/go/sklog"

	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/ct/go/worker_scripts/worker_common"
	"go.skia.org/infra/go/common"
	skutil "go.skia.org/infra/go/util"
)

const (
	// The number of goroutines that will run in parallel to download traces and
	// run metrics analysis.
	WORKER_POOL_SIZE = 20

	METRICS_BENCHMARK_TIMEOUT_SECS = 300

	TRACE_OUTPUT_BUCKET = "chrome-telemetry-output"
)

var (
	startRange         = flag.Int("start_range", 1, "The number this worker will run metrics analysis from.")
	num                = flag.Int("num", 100, "The total number of traces to run metrics analysis from the start_range.")
	runID              = flag.String("run_id", "", "The unique run id (typically requester + timestamp).")
	benchmarkExtraArgs = flag.String("benchmark_extra_args", "", "The extra arguments that are passed to the specified benchmark.")
	metricName         = flag.String("metric_name", "", "The metric to parse the traces with. Eg: loadingMetric")
	chromiumHash       = flag.String("chromium_hash", "", "The chromium commit hash to sync the local checkout to. Done so that all slaves operate on the same revision.")
)

func metricsAnalysis() error {
	defer common.LogPanic()
	worker_common.Init()
	if !*worker_common.Local {
		defer util.CleanTmpDir()
	}
	defer util.TimeTrack(time.Now(), "Metrics Analysis")
	defer sklog.Flush()

	// Validate required arguments.
	if *runID == "" {
		return errors.New("Must specify --run_id")
	}
	if *metricName == "" {
		return errors.New("Must specify --metric_name")
	}
	if *chromiumHash == "" {
		return errors.New("Must specify --chromium_hash")
	}

	ctx := context.Background()

	// Reset the local chromium checkout.
	if err := util.ResetChromiumCheckout(ctx, util.ChromiumSrcDir); err != nil {
		return fmt.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
	}
	// Sync the local chromium checkout.
	if err := util.SyncDir(ctx, util.ChromiumSrcDir, map[string]string{"src": *chromiumHash}, []string{}); err != nil {
		return fmt.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
	}

	// Instantiate GcsUtil object.
	gs, err := util.NewGcsUtil(nil)
	if err != nil {
		return fmt.Errorf("GcsUtil instantiation failed: %s", err)
	}

	// Download the trace URLs for this run from Google storage.
	tracesFilename := *runID + ".traces.csv"
	tmpDir, err := ioutil.TempDir(util.PagesetsDir, "traces")
	if err != nil {
		return fmt.Errorf("Could not create tmpdir: %s", err)
	}
	defer skutil.RemoveAll(tmpDir)
	remotePatchesDir := filepath.Join(util.BenchmarkRunsDir, *runID)

	// Download the catapult patch for this run from Google storage.
	catapultPatchName := *runID + ".catapult.patch"
	if err := util.DownloadAndApplyPatch(ctx, catapultPatchName, tmpDir, remotePatchesDir, util.CatapultSrcDir, gs); err != nil {
		return fmt.Errorf("Could not apply %s: %s", catapultPatchName, err)
	}

	// Download the chromium patch for this run from Google storage.
	chromiumPatchName := *runID + ".chromium.patch"
	if err := util.DownloadAndApplyPatch(ctx, chromiumPatchName, tmpDir, remotePatchesDir, util.ChromiumSrcDir, gs); err != nil {
		return fmt.Errorf("Could not apply %s: %s", chromiumPatchName, err)
	}

	// Download traces.
	if _, err := util.DownloadPatch(filepath.Join(tmpDir, tracesFilename), filepath.Join(remotePatchesDir, tracesFilename), gs); err != nil {
		return fmt.Errorf("Could not download %s: %s", tracesFilename, err)
	}
	traces, err := util.GetCustomPages(filepath.Join(tmpDir, tracesFilename))
	if err != nil {
		return fmt.Errorf("Could not read custom traces file %s: %s", tracesFilename, err)
	}
	if len(traces) == 0 {
		return errors.New(fmt.Sprintf("No traces found in %s", tracesFilename))
	}
	traces = util.GetCustomPagesWithinRange(*startRange, *num, traces)
	sklog.Infof("Using %d traces", len(traces))

	// Establish output paths for pdf downloads and metrics.
	traceDownloadDir := filepath.Join(util.StorageDir, util.TraceDownloadsDir, *runID)
	skutil.RemoveAll(traceDownloadDir)
	skutil.MkdirAll(traceDownloadDir, 0700)
	defer skutil.RemoveAll(traceDownloadDir)

	localOutputDir := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runID)
	skutil.RemoveAll(localOutputDir)
	skutil.MkdirAll(localOutputDir, 0700)
	defer skutil.RemoveAll(localOutputDir)
	remoteDir := filepath.Join(util.BenchmarkRunsDir, *runID)

	sklog.Infof("===== Going to run the task with %d parallel goroutines =====", WORKER_POOL_SIZE)
	// Create channel that contains all trace ULs. This channel will
	// be consumed by the worker pool.
	traceRequests := getClosedChannelOfTraces(traces)

	var wg sync.WaitGroup

	// If not a single benchmark run succeeds then throw at error at the end.
	atleastOneBenchmarkSucceeded := false
	// Gather traceURLs that could not be downloaded.
	erroredTraces := []string{}
	// Mutex to control access to the above slice.
	var erroredTracesMutex sync.Mutex

	// Loop through workers in the worker pool.
	for i := 0; i < WORKER_POOL_SIZE; i++ {
		// Increment the WaitGroup counter.
		wg.Add(1)

		// Create and run a goroutine closure that runs the analysis benchmark.
		go func() {
			// Decrement the WaitGroup counter when the goroutine completes.
			defer wg.Done()

			for t := range traceRequests {
				sklog.Infof("========== Downloading trace %s ==========", t)
				downloadedTrace, err := downloadTrace(t, traceDownloadDir, gs)
				if err != nil {
					sklog.Errorf("Could not download %s: %s", t, err)
					erroredTracesMutex.Lock()
					erroredTraces = append(erroredTraces, t)
					erroredTracesMutex.Unlock()
					continue
				}

				sklog.Infof("========== Processing %s ==========", t)
				if err := runMetricsAnalysisBenchmark(ctx, localOutputDir, downloadedTrace, t); err != nil {
					sklog.Errorf("Error during run_benchmark: %s", err)
				} else {
					atleastOneBenchmarkSucceeded = true
				}
			}
		}()
	}

	// Wait for all spawned goroutines to complete.
	wg.Wait()

	// Summarize errors.
	if len(erroredTraces) > 0 {
		sklog.Error("The following traces could not be downloaded:")
		for _, erroredTrace := range erroredTraces {
			sklog.Errorf("\t%s", erroredTrace)
		}
	}
	if !atleastOneBenchmarkSucceeded {
		return errors.New("Not a single benchmark run was successful. Something is wrong.")
	}

	// If "--output-format=csv" was specified then merge all CSV files and upload.
	if strings.Contains(*benchmarkExtraArgs, "--output-format=csv") {
		// Construct path to CT's python scripts.
		pathToPyFiles := util.GetPathToPyFiles(!*worker_common.Local)
		if err := util.MergeUploadCSVFilesOnWorkers(ctx, localOutputDir, pathToPyFiles, *runID, remoteDir, gs, *startRange, true /* handleStrings */, false /* addRanks */, map[string]map[string]string{} /* pageRankToAdditionalFields */); err != nil {
			return fmt.Errorf("Error while processing withpatch CSV files: %s", err)
		}
	}

	return nil
}

// runMetricsAnalysisBenchmark runs the analysis_metrics_ct benchmark on the provided trace.
func runMetricsAnalysisBenchmark(ctx context.Context, outputPath, downloadedTrace, cloudTraceLink string) error {
	args := []string{
		filepath.Join(util.TelemetryBinariesDir, util.BINARY_RUN_BENCHMARK),
		util.BENCHMARK_METRICS_ANALYSIS,
		"--local-trace-path", fmt.Sprintf("file://%s", downloadedTrace),
		"--cloud-trace-link", cloudTraceLink,
		"--metric-name", *metricName,
		"--browser", "system", // No browser is brought up but unfortunately this is needed by the framework.
		"--output-dir", filepath.Join(outputPath, getTraceName(downloadedTrace)),
	}
	// Split benchmark args if not empty and append to args.
	if *benchmarkExtraArgs != "" {
		args = append(args, strings.Fields(*benchmarkExtraArgs)...)
	}
	// Set the PYTHONPATH to the telemetry dirs.
	env := []string{
		fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHONPATH", util.TelemetryBinariesDir, util.TelemetrySrcDir, util.CatapultSrcDir),
		"DISPLAY=:0",
	}
	// Append the original environment as well.
	for _, e := range os.Environ() {
		env = append(env, e)
	}

	// Create buffer for capturing the stdout and stderr of the benchmark run.
	var b bytes.Buffer
	if _, err := b.WriteString(fmt.Sprintf("========== Stdout and stderr for %s ==========\n", downloadedTrace)); err != nil {
		return fmt.Errorf("Error writing to output buffer: %s", err)
	}
	if err := util.ExecuteCmdWithConfigurableLogging(ctx, "python", args, env, METRICS_BENCHMARK_TIMEOUT_SECS*time.Second, &b, &b, false, false); err != nil {
		output, getErr := util.GetRunBenchmarkOutput(b)
		skutil.LogErr(getErr)
		fmt.Println(output)
		return fmt.Errorf("Run benchmark command failed with: %s", err)
	}
	output, err := util.GetRunBenchmarkOutput(b)
	if err != nil {
		return fmt.Errorf("Could not get run benchmark output: %s", err)
	}
	// Print the output and return.
	fmt.Println(output)
	return nil
}

// downloadTrace downloads the traceURL from google storage into the specified
// destDir.
func downloadTrace(traceURL, destDir string, gs *util.GcsUtil) (string, error) {
	traceName := getTraceName(traceURL)
	traceDest := filepath.Join(destDir, traceName)
	if err := gs.DownloadRemoteFileFromBucket(TRACE_OUTPUT_BUCKET, traceName, traceDest); err != nil {
		return "", fmt.Errorf("Error downloading %s from %s to %s: %s", traceName, TRACE_OUTPUT_BUCKET, traceDest, err)
	}
	return traceDest, nil
}

// getTraceName parses the provided traceURI and returns the name of the trace.
// traceURI could be a file path or a URL.
func getTraceName(traceURI string) string {
	traceTokens := strings.Split(traceURI, "/")
	return traceTokens[len(traceTokens)-1]
}

// getClosedChannelOfTraces returns a channel that contains all trace URLs.
func getClosedChannelOfTraces(traces []string) chan string {
	tracesChannel := make(chan string, len(traces))
	for _, t := range traces {
		tracesChannel <- t
	}
	close(tracesChannel)
	return tracesChannel
}

func main() {
	retCode := 0
	if err := metricsAnalysis(); err != nil {
		sklog.Errorf("Error while running metrics analysis: %s", err)
		retCode = 255
	}
	os.Exit(retCode)
}
<commit_msg>Revert "[CT] Experiment by increasing to 20 parallel goroutines in metrics analysis"<commit_after>// Application that does metrics analysis as described in the design doc:
// go/ct_metrics_analysis
//
// Can be tested locally with:
// $ go run go/worker_scripts/metrics_analysis/main.go --start_range=1 --num=3 --run_id=rmistry-test1 --benchmark_extra_args="--output-format=csv" --metric_name="loadingMetric" --logtostderr=true --local --chromium_hash=cab48032c45695c73b202af7d472a52808d551ca
//
package main

import (
	"bytes"
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"go.skia.org/infra/go/sklog"

	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/ct/go/worker_scripts/worker_common"
	"go.skia.org/infra/go/common"
	skutil "go.skia.org/infra/go/util"
)

const (
	// The number of goroutines that will run in parallel to download traces and
	// run metrics analysis.
	WORKER_POOL_SIZE = 10

	METRICS_BENCHMARK_TIMEOUT_SECS = 300

	TRACE_OUTPUT_BUCKET = "chrome-telemetry-output"
)

var (
	startRange         = flag.Int("start_range", 1, "The number this worker will run metrics analysis from.")
	num                = flag.Int("num", 100, "The total number of traces to run metrics analysis from the start_range.")
	runID              = flag.String("run_id", "", "The unique run id (typically requester + timestamp).")
	benchmarkExtraArgs = flag.String("benchmark_extra_args", "", "The extra arguments that are passed to the specified benchmark.")
	metricName         = flag.String("metric_name", "", "The metric to parse the traces with. Eg: loadingMetric")
	chromiumHash       = flag.String("chromium_hash", "", "The chromium commit hash to sync the local checkout to. Done so that all slaves operate on the same revision.")
)

func metricsAnalysis() error {
	defer common.LogPanic()
	worker_common.Init()
	if !*worker_common.Local {
		defer util.CleanTmpDir()
	}
	defer util.TimeTrack(time.Now(), "Metrics Analysis")
	defer sklog.Flush()

	// Validate required arguments.
	if *runID == "" {
		return errors.New("Must specify --run_id")
	}
	if *metricName == "" {
		return errors.New("Must specify --metric_name")
	}
	if *chromiumHash == "" {
		return errors.New("Must specify --chromium_hash")
	}

	ctx := context.Background()

	// Reset the local chromium checkout.
	if err := util.ResetChromiumCheckout(ctx, util.ChromiumSrcDir); err != nil {
		return fmt.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
	}
	// Sync the local chromium checkout.
	if err := util.SyncDir(ctx, util.ChromiumSrcDir, map[string]string{"src": *chromiumHash}, []string{}); err != nil {
		return fmt.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
	}

	// Instantiate GcsUtil object.
	gs, err := util.NewGcsUtil(nil)
	if err != nil {
		return fmt.Errorf("GcsUtil instantiation failed: %s", err)
	}

	// Download the trace URLs for this run from Google storage.
	tracesFilename := *runID + ".traces.csv"
	tmpDir, err := ioutil.TempDir(util.PagesetsDir, "traces")
	if err != nil {
		return fmt.Errorf("Could not create tmpdir: %s", err)
	}
	defer skutil.RemoveAll(tmpDir)
	remotePatchesDir := filepath.Join(util.BenchmarkRunsDir, *runID)

	// Download the catapult patch for this run from Google storage.
	catapultPatchName := *runID + ".catapult.patch"
	if err := util.DownloadAndApplyPatch(ctx, catapultPatchName, tmpDir, remotePatchesDir, util.CatapultSrcDir, gs); err != nil {
		return fmt.Errorf("Could not apply %s: %s", catapultPatchName, err)
	}

	// Download the chromium patch for this run from Google storage.
	chromiumPatchName := *runID + ".chromium.patch"
	if err := util.DownloadAndApplyPatch(ctx, chromiumPatchName, tmpDir, remotePatchesDir, util.ChromiumSrcDir, gs); err != nil {
		return fmt.Errorf("Could not apply %s: %s", chromiumPatchName, err)
	}

	// Download traces.
	if _, err := util.DownloadPatch(filepath.Join(tmpDir, tracesFilename), filepath.Join(remotePatchesDir, tracesFilename), gs); err != nil {
		return fmt.Errorf("Could not download %s: %s", tracesFilename, err)
	}
	traces, err := util.GetCustomPages(filepath.Join(tmpDir, tracesFilename))
	if err != nil {
		return fmt.Errorf("Could not read custom traces file %s: %s", tracesFilename, err)
	}
	if len(traces) == 0 {
		return errors.New(fmt.Sprintf("No traces found in %s", tracesFilename))
	}
	traces = util.GetCustomPagesWithinRange(*startRange, *num, traces)
	sklog.Infof("Using %d traces", len(traces))

	// Establish output paths for pdf downloads and metrics.
	traceDownloadDir := filepath.Join(util.StorageDir, util.TraceDownloadsDir, *runID)
	skutil.RemoveAll(traceDownloadDir)
	skutil.MkdirAll(traceDownloadDir, 0700)
	defer skutil.RemoveAll(traceDownloadDir)

	localOutputDir := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runID)
	skutil.RemoveAll(localOutputDir)
	skutil.MkdirAll(localOutputDir, 0700)
	defer skutil.RemoveAll(localOutputDir)
	remoteDir := filepath.Join(util.BenchmarkRunsDir, *runID)

	sklog.Infof("===== Going to run the task with %d parallel goroutines =====", WORKER_POOL_SIZE)
	// Create channel that contains all trace ULs. This channel will
	// be consumed by the worker pool.
	traceRequests := getClosedChannelOfTraces(traces)

	var wg sync.WaitGroup

	// If not a single benchmark run succeeds then throw at error at the end.
	atleastOneBenchmarkSucceeded := false
	// Gather traceURLs that could not be downloaded.
	erroredTraces := []string{}
	// Mutex to control access to the above slice.
	var erroredTracesMutex sync.Mutex

	// Loop through workers in the worker pool.
	for i := 0; i < WORKER_POOL_SIZE; i++ {
		// Increment the WaitGroup counter.
		wg.Add(1)

		// Create and run a goroutine closure that runs the analysis benchmark.
		go func() {
			// Decrement the WaitGroup counter when the goroutine completes.
			defer wg.Done()

			for t := range traceRequests {
				sklog.Infof("========== Downloading trace %s ==========", t)
				downloadedTrace, err := downloadTrace(t, traceDownloadDir, gs)
				if err != nil {
					sklog.Errorf("Could not download %s: %s", t, err)
					erroredTracesMutex.Lock()
					erroredTraces = append(erroredTraces, t)
					erroredTracesMutex.Unlock()
					continue
				}

				sklog.Infof("========== Processing %s ==========", t)
				if err := runMetricsAnalysisBenchmark(ctx, localOutputDir, downloadedTrace, t); err != nil {
					sklog.Errorf("Error during run_benchmark: %s", err)
				} else {
					atleastOneBenchmarkSucceeded = true
				}
			}
		}()
	}

	// Wait for all spawned goroutines to complete.
	wg.Wait()

	// Summarize errors.
	if len(erroredTraces) > 0 {
		sklog.Error("The following traces could not be downloaded:")
		for _, erroredTrace := range erroredTraces {
			sklog.Errorf("\t%s", erroredTrace)
		}
	}
	if !atleastOneBenchmarkSucceeded {
		return errors.New("Not a single benchmark run was successful. Something is wrong.")
	}

	// If "--output-format=csv" was specified then merge all CSV files and upload.
	if strings.Contains(*benchmarkExtraArgs, "--output-format=csv") {
		// Construct path to CT's python scripts.
		pathToPyFiles := util.GetPathToPyFiles(!*worker_common.Local)
		if err := util.MergeUploadCSVFilesOnWorkers(ctx, localOutputDir, pathToPyFiles, *runID, remoteDir, gs, *startRange, true /* handleStrings */, false /* addRanks */, map[string]map[string]string{} /* pageRankToAdditionalFields */); err != nil {
			return fmt.Errorf("Error while processing withpatch CSV files: %s", err)
		}
	}

	return nil
}

// runMetricsAnalysisBenchmark runs the analysis_metrics_ct benchmark on the provided trace.
func runMetricsAnalysisBenchmark(ctx context.Context, outputPath, downloadedTrace, cloudTraceLink string) error {
	args := []string{
		filepath.Join(util.TelemetryBinariesDir, util.BINARY_RUN_BENCHMARK),
		util.BENCHMARK_METRICS_ANALYSIS,
		"--local-trace-path", fmt.Sprintf("file://%s", downloadedTrace),
		"--cloud-trace-link", cloudTraceLink,
		"--metric-name", *metricName,
		"--browser", "system", // No browser is brought up but unfortunately this is needed by the framework.
		"--output-dir", filepath.Join(outputPath, getTraceName(downloadedTrace)),
	}
	// Split benchmark args if not empty and append to args.
	if *benchmarkExtraArgs != "" {
		args = append(args, strings.Fields(*benchmarkExtraArgs)...)
	}
	// Set the PYTHONPATH to the telemetry dirs.
	env := []string{
		fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHONPATH", util.TelemetryBinariesDir, util.TelemetrySrcDir, util.CatapultSrcDir),
		"DISPLAY=:0",
	}
	// Append the original environment as well.
	for _, e := range os.Environ() {
		env = append(env, e)
	}

	// Create buffer for capturing the stdout and stderr of the benchmark run.
	var b bytes.Buffer
	if _, err := b.WriteString(fmt.Sprintf("========== Stdout and stderr for %s ==========\n", downloadedTrace)); err != nil {
		return fmt.Errorf("Error writing to output buffer: %s", err)
	}
	if err := util.ExecuteCmdWithConfigurableLogging(ctx, "python", args, env, METRICS_BENCHMARK_TIMEOUT_SECS*time.Second, &b, &b, false, false); err != nil {
		output, getErr := util.GetRunBenchmarkOutput(b)
		skutil.LogErr(getErr)
		fmt.Println(output)
		return fmt.Errorf("Run benchmark command failed with: %s", err)
	}
	output, err := util.GetRunBenchmarkOutput(b)
	if err != nil {
		return fmt.Errorf("Could not get run benchmark output: %s", err)
	}
	// Print the output and return.
	fmt.Println(output)
	return nil
}

// downloadTrace downloads the traceURL from google storage into the specified
// destDir.
func downloadTrace(traceURL, destDir string, gs *util.GcsUtil) (string, error) {
	traceName := getTraceName(traceURL)
	traceDest := filepath.Join(destDir, traceName)
	if err := gs.DownloadRemoteFileFromBucket(TRACE_OUTPUT_BUCKET, traceName, traceDest); err != nil {
		return "", fmt.Errorf("Error downloading %s from %s to %s: %s", traceName, TRACE_OUTPUT_BUCKET, traceDest, err)
	}
	return traceDest, nil
}

// getTraceName parses the provided traceURI and returns the name of the trace.
// traceURI could be a file path or a URL.
func getTraceName(traceURI string) string {
	traceTokens := strings.Split(traceURI, "/")
	return traceTokens[len(traceTokens)-1]
}

// getClosedChannelOfTraces returns a channel that contains all trace URLs.
func getClosedChannelOfTraces(traces []string) chan string {
	tracesChannel := make(chan string, len(traces))
	for _, t := range traces {
		tracesChannel <- t
	}
	close(tracesChannel)
	return tracesChannel
}

func main() {
	retCode := 0
	if err := metricsAnalysis(); err != nil {
		sklog.Errorf("Error while running metrics analysis: %s", err)
		retCode = 255
	}
	os.Exit(retCode)
}
<|endoftext|>


<commit_before>/*
 * Copyright ©1998-2022 by Richard A. Wilkes. All rights reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, version 2.0. If a copy of the MPL was not distributed with
 * this file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * This Source Code Form is "Incompatible With Secondary Licenses", as
 * defined by the Mozilla Public License, version 2.0.
 */

package editors

import (
	"strings"

	"github.com/richardwilkes/gcs/v5/model/fxp"
	"github.com/richardwilkes/gcs/v5/model/gurps"
	"github.com/richardwilkes/gcs/v5/model/gurps/gid"
	"github.com/richardwilkes/gcs/v5/model/gurps/skill"
	"github.com/richardwilkes/gcs/v5/model/gurps/weapon"
	"github.com/richardwilkes/gcs/v5/res"
	"github.com/richardwilkes/gcs/v5/ui/widget"
	"github.com/richardwilkes/toolbox/i18n"
	"github.com/richardwilkes/unison"
)

// EditSpell displays the editor for an spell.
func EditSpell(owner widget.Rebuildable, spell *gurps.Spell) {
	displayEditor[*gurps.Spell, *gurps.SpellEditData](owner, spell, res.GCSSpellsSVG, initSpellEditor)
}

func initSpellEditor(e *editor[*gurps.Spell, *gurps.SpellEditData], content *unison.Panel) func() {
	var dockableKind string
	if one, ok := e.owner.(widget.DockableKind); ok {
		dockableKind = one.DockableKind()
	}
	isRitualMagic := strings.HasPrefix(e.target.Type, gid.RitualMagicSpell)
	addNameLabelAndField(content, &e.editorData.Name)
	if !e.target.Container() {
		addTechLevelRequired(content, &e.editorData.TechLevel, dockableKind == widget.SheetDockableKind)
		addLabelAndListField(content, i18n.Text("College"), i18n.Text("colleges"), (*[]string)(&e.editorData.College))
		addLabelAndStringField(content, i18n.Text("Class"), "", &e.editorData.Class)
		addLabelAndStringField(content, i18n.Text("Power Source"), "", &e.editorData.PowerSource)
		if isRitualMagic {
			wrapper := addFlowWrapper(content, i18n.Text("Difficulty"), 3)
			addPopup(wrapper, skill.AllTechniqueDifficulty, &e.editorData.Difficulty.Difficulty)
			prereqCount := i18n.Text("Prerequisite Count")
			wrapper.AddChild(widget.NewFieldInteriorLeadingLabel(prereqCount))
			addIntegerField(wrapper, nil, "", prereqCount, "", &e.editorData.RitualPrereqCount, 0, 99)
		} else {
			addDifficultyLabelAndFields(content, e.target.Entity, &e.editorData.Difficulty)
		}
		if dockableKind == widget.SheetDockableKind || dockableKind == widget.TemplateDockableKind {
			pointsLabel := i18n.Text("Points")
			wrapper := addFlowWrapper(content, pointsLabel, 3)
			addDecimalField(wrapper, nil, "", pointsLabel, "", &e.editorData.Points, 0, fxp.MaxBasePoints)
			wrapper.AddChild(widget.NewFieldInteriorLeadingLabel(i18n.Text("Level")))
			levelField := widget.NewNonEditableField(func(field *widget.NonEditableField) {
				points := gurps.AdjustedPointsForNonContainerSpell(e.target.Entity, e.editorData.Points,
					e.editorData.Name, e.editorData.PowerSource, e.editorData.College, e.editorData.Tags, nil)
				var level skill.Level
				if isRitualMagic {
					level = gurps.CalculateRitualMagicSpellLevel(e.target.Entity, e.editorData.Name,
						e.editorData.PowerSource, e.editorData.RitualSkillName, e.editorData.RitualPrereqCount,
						e.editorData.College, e.editorData.Tags, e.editorData.Difficulty, points)
				} else {
					level = gurps.CalculateSpellLevel(e.target.Entity, e.editorData.Name, e.editorData.PowerSource,
						e.editorData.College, e.editorData.Tags, e.editorData.Difficulty, points)
				}
				lvl := level.Level.Trunc()
				if lvl <= 0 {
					field.Text = "-"
				} else {
					field.Text = lvl.String() + "/" + gurps.FormatRelativeSkill(e.target.Entity, e.target.Type,
						e.editorData.Difficulty, level.RelativeLevel)
				}
				field.MarkForLayoutAndRedraw()
			})
			insets := levelField.Border().Insets()
			levelField.SetLayoutData(&unison.FlexLayoutData{
				MinSize: unison.NewSize(levelField.Font.SimpleWidth((-fxp.MaxBasePoints*2).String())+insets.Left+insets.Right, 0),
			})
			wrapper.AddChild(levelField)
		}
		addLabelAndStringField(content, i18n.Text("Resistance"), "", &e.editorData.Resist)
		addLabelAndStringField(content, i18n.Text("Casting Cost"), "", &e.editorData.CastingCost)
		addLabelAndStringField(content, i18n.Text("Maintenance Cost"), "", &e.editorData.MaintenanceCost)
		addLabelAndStringField(content, i18n.Text("Casting Time"), "", &e.editorData.CastingTime)
		addLabelAndStringField(content, i18n.Text("Casting Duration"), "", &e.editorData.Duration)
	}
	addNotesLabelAndField(content, &e.editorData.LocalNotes)
	addVTTNotesLabelAndField(content, &e.editorData.VTTNotes)
	addTagsLabelAndField(content, &e.editorData.Tags)
	addPageRefLabelAndField(content, &e.editorData.PageRef)
	if !e.target.Container() {
		content.AddChild(newPrereqPanel(e.target.Entity, &e.editorData.Prereq))
		for _, wt := range weapon.AllType {
			content.AddChild(newWeaponsPanel(e, e.target, wt, &e.editorData.Weapons))
		}
	}
	return nil
}
<commit_msg>Fix #513: Ritual Magic Spell "Base Skill" field is missing<commit_after>/*
 * Copyright ©1998-2022 by Richard A. Wilkes. All rights reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, version 2.0. If a copy of the MPL was not distributed with
 * this file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * This Source Code Form is "Incompatible With Secondary Licenses", as
 * defined by the Mozilla Public License, version 2.0.
 */

package editors

import (
	"strings"

	"github.com/richardwilkes/gcs/v5/model/fxp"
	"github.com/richardwilkes/gcs/v5/model/gurps"
	"github.com/richardwilkes/gcs/v5/model/gurps/gid"
	"github.com/richardwilkes/gcs/v5/model/gurps/skill"
	"github.com/richardwilkes/gcs/v5/model/gurps/weapon"
	"github.com/richardwilkes/gcs/v5/res"
	"github.com/richardwilkes/gcs/v5/ui/widget"
	"github.com/richardwilkes/toolbox/i18n"
	"github.com/richardwilkes/unison"
)

// EditSpell displays the editor for an spell.
func EditSpell(owner widget.Rebuildable, spell *gurps.Spell) {
	displayEditor[*gurps.Spell, *gurps.SpellEditData](owner, spell, res.GCSSpellsSVG, initSpellEditor)
}

func initSpellEditor(e *editor[*gurps.Spell, *gurps.SpellEditData], content *unison.Panel) func() {
	var dockableKind string
	if one, ok := e.owner.(widget.DockableKind); ok {
		dockableKind = one.DockableKind()
	}
	isRitualMagic := strings.HasPrefix(e.target.Type, gid.RitualMagicSpell)
	addNameLabelAndField(content, &e.editorData.Name)
	if !e.target.Container() {
		addTechLevelRequired(content, &e.editorData.TechLevel, dockableKind == widget.SheetDockableKind)
		addLabelAndListField(content, i18n.Text("College"), i18n.Text("colleges"), (*[]string)(&e.editorData.College))
		addLabelAndStringField(content, i18n.Text("Class"), "", &e.editorData.Class)
		addLabelAndStringField(content, i18n.Text("Power Source"), "", &e.editorData.PowerSource)
		if isRitualMagic {
			addLabelAndStringField(content, i18n.Text("Base Skill"), "", &e.editorData.RitualSkillName)
			wrapper := addFlowWrapper(content, i18n.Text("Difficulty"), 3)
			addPopup(wrapper, skill.AllTechniqueDifficulty, &e.editorData.Difficulty.Difficulty)
			prereqCount := i18n.Text("Prerequisite Count")
			wrapper.AddChild(widget.NewFieldInteriorLeadingLabel(prereqCount))
			addIntegerField(wrapper, nil, "", prereqCount, "", &e.editorData.RitualPrereqCount, 0, 99)
		} else {
			addDifficultyLabelAndFields(content, e.target.Entity, &e.editorData.Difficulty)
		}
		if dockableKind == widget.SheetDockableKind || dockableKind == widget.TemplateDockableKind {
			pointsLabel := i18n.Text("Points")
			wrapper := addFlowWrapper(content, pointsLabel, 3)
			addDecimalField(wrapper, nil, "", pointsLabel, "", &e.editorData.Points, 0, fxp.MaxBasePoints)
			wrapper.AddChild(widget.NewFieldInteriorLeadingLabel(i18n.Text("Level")))
			levelField := widget.NewNonEditableField(func(field *widget.NonEditableField) {
				points := gurps.AdjustedPointsForNonContainerSpell(e.target.Entity, e.editorData.Points,
					e.editorData.Name, e.editorData.PowerSource, e.editorData.College, e.editorData.Tags, nil)
				var level skill.Level
				if isRitualMagic {
					level = gurps.CalculateRitualMagicSpellLevel(e.target.Entity, e.editorData.Name,
						e.editorData.PowerSource, e.editorData.RitualSkillName, e.editorData.RitualPrereqCount,
						e.editorData.College, e.editorData.Tags, e.editorData.Difficulty, points)
				} else {
					level = gurps.CalculateSpellLevel(e.target.Entity, e.editorData.Name, e.editorData.PowerSource,
						e.editorData.College, e.editorData.Tags, e.editorData.Difficulty, points)
				}
				lvl := level.Level.Trunc()
				if lvl <= 0 {
					field.Text = "-"
				} else {
					field.Text = lvl.String() + "/" + gurps.FormatRelativeSkill(e.target.Entity, e.target.Type,
						e.editorData.Difficulty, level.RelativeLevel)
				}
				field.MarkForLayoutAndRedraw()
			})
			insets := levelField.Border().Insets()
			levelField.SetLayoutData(&unison.FlexLayoutData{
				MinSize: unison.NewSize(levelField.Font.SimpleWidth((-fxp.MaxBasePoints*2).String())+insets.Left+insets.Right, 0),
			})
			wrapper.AddChild(levelField)
		}
		addLabelAndStringField(content, i18n.Text("Resistance"), "", &e.editorData.Resist)
		addLabelAndStringField(content, i18n.Text("Casting Cost"), "", &e.editorData.CastingCost)
		addLabelAndStringField(content, i18n.Text("Maintenance Cost"), "", &e.editorData.MaintenanceCost)
		addLabelAndStringField(content, i18n.Text("Casting Time"), "", &e.editorData.CastingTime)
		addLabelAndStringField(content, i18n.Text("Casting Duration"), "", &e.editorData.Duration)
	}
	addNotesLabelAndField(content, &e.editorData.LocalNotes)
	addVTTNotesLabelAndField(content, &e.editorData.VTTNotes)
	addTagsLabelAndField(content, &e.editorData.Tags)
	addPageRefLabelAndField(content, &e.editorData.PageRef)
	if !e.target.Container() {
		content.AddChild(newPrereqPanel(e.target.Entity, &e.editorData.Prereq))
		for _, wt := range weapon.AllType {
			content.AddChild(newWeaponsPanel(e, e.target, wt, &e.editorData.Weapons))
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// +build ignore

package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"time"
)

func init() {
	rand.Seed(42)
}

func ReadRand(bs []byte) (int, error) {
	var r uint32
	for i := range bs {
		if i%4 == 0 {
			r = uint32(rand.Int63())
		}
		bs[i] = byte(r >> uint((i%4)*8))
	}
	return len(bs), nil
}

func name() string {
	var b [16]byte
	ReadRand(b[:])
	return fmt.Sprintf("%x", b[:])
}

func main() {
	var files int
	var maxexp int
	var srcname string

	flag.IntVar(&files, "files", 1000, "Number of files")
	flag.IntVar(&maxexp, "maxexp", 20, "Maximum file size (max = 2^n + 128*1024 B)")
	flag.StringVar(&srcname, "src", "/usr/share/dict/words", "Source material")
	flag.Parse()

	fd, err := os.Open(srcname)
	if err != nil {
		log.Fatal(err)
	}

	for i := 0; i < files; i++ {
		n := name()
		p0 := filepath.Join(string(n[0]), n[0:2])
		err = os.MkdirAll(p0, 0755)
		if err != nil {
			log.Fatal(err)
		}

		s := 1 << uint(rand.Intn(maxexp))
		a := 128 * 1024
		if a > s {
			a = s
		}
		s += rand.Intn(a)

		src := io.LimitReader(&inifiteReader{fd}, int64(s))

		p1 := filepath.Join(p0, n)
		dst, err := os.Create(p1)
		if err != nil {
			log.Fatal(err)
		}

		_, err = io.Copy(dst, src)
		if err != nil {
			log.Fatal(err)
		}

		err = dst.Close()
		if err != nil {
			log.Fatal(err)
		}

		err = os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
		if err != nil {
			log.Fatal(err)
		}

		t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
		err = os.Chtimes(p1, t, t)
		if err != nil {
			log.Fatal(err)
		}
	}
}

type inifiteReader struct {
	rd io.ReadSeeker
}

func (i *inifiteReader) Read(bs []byte) (int, error) {
	n, err := i.rd.Read(bs)
	if err == io.EOF {
		err = nil
		i.rd.Seek(0, 0)
	}
	return n, err
}
<commit_msg>Genfiles should be random by default<commit_after>// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// +build ignore

package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"time"
)

func ReadRand(bs []byte) (int, error) {
	var r uint32
	for i := range bs {
		if i%4 == 0 {
			r = uint32(rand.Int63())
		}
		bs[i] = byte(r >> uint((i%4)*8))
	}
	return len(bs), nil
}

func name() string {
	var b [16]byte
	ReadRand(b[:])
	return fmt.Sprintf("%x", b[:])
}

func main() {
	var files int
	var maxexp int
	var srcname string
	var random bool

	flag.IntVar(&files, "files", 1000, "Number of files")
	flag.IntVar(&maxexp, "maxexp", 20, "Maximum file size (max = 2^n + 128*1024 B)")
	flag.StringVar(&srcname, "src", "/usr/share/dict/words", "Source material")
	flag.BoolVar(&random, "random", true, "When false, always generate the same set of file")
	flag.Parse()

	if random {
		rand.Seed(time.Now().UnixNano())
	} else {
		rand.Seed(42)
	}

	fd, err := os.Open(srcname)
	if err != nil {
		log.Fatal(err)
	}

	for i := 0; i < files; i++ {
		n := name()
		p0 := filepath.Join(string(n[0]), n[0:2])
		err = os.MkdirAll(p0, 0755)
		if err != nil {
			log.Fatal(err)
		}

		s := 1 << uint(rand.Intn(maxexp))
		a := 128 * 1024
		if a > s {
			a = s
		}
		s += rand.Intn(a)

		src := io.LimitReader(&inifiteReader{fd}, int64(s))

		p1 := filepath.Join(p0, n)
		dst, err := os.Create(p1)
		if err != nil {
			log.Fatal(err)
		}

		_, err = io.Copy(dst, src)
		if err != nil {
			log.Fatal(err)
		}

		err = dst.Close()
		if err != nil {
			log.Fatal(err)
		}

		err = os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
		if err != nil {
			log.Fatal(err)
		}

		t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
		err = os.Chtimes(p1, t, t)
		if err != nil {
			log.Fatal(err)
		}
	}
}

type inifiteReader struct {
	rd io.ReadSeeker
}

func (i *inifiteReader) Read(bs []byte) (int, error) {
	n, err := i.rd.Read(bs)
	if err == io.EOF {
		err = nil
		i.rd.Seek(0, 0)
	}
	return n, err
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tchannel_test

import (
	"runtime"
	"testing"

	. "github.com/uber/tchannel-go"

	"github.com/uber/tchannel-go/testutils"
)

// WithVerifiedServer runs the given test function with a server channel that is verified
// at the end to make sure there are no leaks (e.g. no exchanges leaked).
func WithVerifiedServer(t *testing.T, opts *testutils.ChannelOpts, f func(serverCh *Channel, hostPort string)) {
	var ch *Channel
	testutils.WithServer(t, opts, func(serverCh *Channel, hostPort string) {
		f(serverCh, hostPort)
		ch = serverCh
	})

	// Wait till the channel is closed
	for i := 0; i < 10; i++ {
		if ch.State() == ChannelClosed {
			break
		}
		runtime.Gosched()
	}

	// Check the message exchanges and make sure they are all empty.
	if exchangesLeft := CheckEmptyExchanges(ch); exchangesLeft != "" {
		t.Errorf("Found uncleared message exchanges:\n%v", exchangesLeft)
	}
}
<commit_msg>Fail test if verified server does not close<commit_after>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tchannel_test

import (
	"runtime"
	"testing"
	"time"

	. "github.com/uber/tchannel-go"

	"github.com/uber/tchannel-go/testutils"
)

func waitForChannelClose(t *testing.T, ch *Channel) bool {
	var state ChannelState
	for i := 0; i < 10; i++ {
		if state = ch.State(); state == ChannelClosed {
			return true
		}

		if i > 5 {
			time.Sleep(time.Millisecond)
		} else {
			runtime.Gosched()
		}
	}

	// Channel is not closing, fail the test.
	t.Errorf("Channel did not close, last state: %v", state)
	return false
}

// WithVerifiedServer runs the given test function with a server channel that is verified
// at the end to make sure there are no leaks (e.g. no exchanges leaked).
func WithVerifiedServer(t *testing.T, opts *testutils.ChannelOpts, f func(serverCh *Channel, hostPort string)) {
	var ch *Channel
	testutils.WithServer(t, opts, func(serverCh *Channel, hostPort string) {
		f(serverCh, hostPort)
		ch = serverCh
	})

	if !waitForChannelClose(t, ch) {
		return
	}

	// Check the message exchanges and make sure they are all empty.
	if exchangesLeft := CheckEmptyExchanges(ch); exchangesLeft != "" {
		t.Errorf("Found uncleared message exchanges:\n%v", exchangesLeft)
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"strings"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/directconnect"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSDCINTRAVIRTUALINTERFACE_basic(t *testing.T) {
	var virtualIF directconnect.VirtualInterface

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckDCIntraVirtualInterfaceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccDCVirtualInterfaceConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDCIntraVirtualInterfaceExists("aws_dc_intra_virtual_interface_confirm.virtualinterface", &virtualIF),
				),
			},
		},
	})
}

func testAccCheckDCIntraVirtualInterfaceDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).dcconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_dc_intra_virtual_interface_confirm" {
			continue
		}

		// Try to find the resource
		resp, err := conn.DescribeVirtualInterfaces(&directconnect.DescribeVirtualInterfacesInput{
			VirtualInterfaceId: aws.String(rs.Primary.ID),
		})
		if err == nil {
			if len(resp.VirtualInterfaces) > 0 && strings.ToLower(*resp.VirtualInterfaces[0].VirtualInterfaceState) != "deleted" {
				return fmt.Errorf("still exists")
			}

			return nil
		}

		_, ok := err.(awserr.Error)
		if !ok {
			return err
		}
	}

	return nil
}

func testAccCheckDCIntraVirtualInterfaceExists(n string, ng *directconnect.VirtualInterface) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).dcconn

		resp, err := conn.DescribeVirtualInterfaces(&directconnect.DescribeVirtualInterfacesInput{
			VirtualInterfaceId: aws.String(rs.Primary.ID),
		})
		if err != nil {
			return err
		}
		if len(resp.VirtualInterfaces) == 0 {
			return fmt.Errorf("DCVirtualInterface not found")
		}

		*ng = *resp.VirtualInterfaces[0]

		return nil
	}
}

const testAccDCIntraVirtualInterfaceConfig = `

resource "aws_vpc" "vpc" {
    cidr_block = "10.0.0.0/16"
}

resource "aws_vpn_gateway" "vpn_gw" {
    vpc_id = "${aws_vpc.vpc.id}"
}

resource "aws_dc_virtual_interface_confirm" "vif" {
  virtual_interface_id = "vif-xyz123"
  virtual_gateway_id = "${aws_vpn_gateway.vpn_gw.id}"
  interface_type = "private"
}
`
<commit_msg>Fix tests.<commit_after>package aws

import (
	"fmt"
	"strings"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/directconnect"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSDCINTRAVIRTUALINTERFACECONFIRM_basic(t *testing.T) {
	var virtualIF directconnect.VirtualInterface

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckDCIntraVirtualInterfaceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccDCVirtualInterfaceConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDCIntraVirtualInterfaceExists("aws_dc_intra_virtual_interface_confirm.virtualinterface", &virtualIF),
				),
			},
		},
	})
}

func testAccCheckDCIntraVirtualInterfaceConfirmDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).dcconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_dc_intra_virtual_interface_confirm" {
			continue
		}

		// Try to find the resource
		resp, err := conn.DescribeVirtualInterfaces(&directconnect.DescribeVirtualInterfacesInput{
			VirtualInterfaceId: aws.String(rs.Primary.ID),
		})
		if err == nil {
			if len(resp.VirtualInterfaces) > 0 && strings.ToLower(*resp.VirtualInterfaces[0].VirtualInterfaceState) != "deleted" {
				return fmt.Errorf("still exists")
			}

			return nil
		}

		_, ok := err.(awserr.Error)
		if !ok {
			return err
		}
	}

	return nil
}

func testAccCheckDCIntraVirtualInterfaceConfirmExists(n string, ng *directconnect.VirtualInterface) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).dcconn

		resp, err := conn.DescribeVirtualInterfaces(&directconnect.DescribeVirtualInterfacesInput{
			VirtualInterfaceId: aws.String(rs.Primary.ID),
		})
		if err != nil {
			return err
		}
		if len(resp.VirtualInterfaces) == 0 {
			return fmt.Errorf("DCVirtualInterface not found")
		}

		*ng = *resp.VirtualInterfaces[0]

		return nil
	}
}

const testAccDCIntraVirtualInterfaceConfirmConfig = `

resource "aws_vpc" "vpc" {
    cidr_block = "10.0.0.0/16"
}

resource "aws_vpn_gateway" "vpn_gw" {
    vpc_id = "${aws_vpc.vpc.id}"
}

resource "aws_dc_virtual_interface_confirm" "vif" {
  virtual_interface_id = "vif-xyz123"
  virtual_gateway_id = "${aws_vpn_gateway.vpn_gw.id}"
  interface_type = "private"
}
`
<|endoftext|>


<commit_before>package orasrv

import (
	"context"
	"crypto/rand"
	"encoding/json"
	"fmt"
	"reflect"
	"sync"
	"time"

	"github.com/LK4D4/joincontext"
	"github.com/gogo/protobuf/proto"
	bp "github.com/tgulacsi/go/bufpool"
	oracall "github.com/tgulacsi/oracall/lib"
	errors "golang.org/x/xerrors"

	"github.com/go-kit/kit/log"
	"github.com/oklog/ulid"

	"github.com/go-stack/stack"
	"github.com/grpc-ecosystem/go-grpc-middleware"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	_ "google.golang.org/grpc/encoding/gzip"
	"google.golang.org/grpc/status"

	goracle "gopkg.in/goracle.v2"
)

var Timeout = DefaultTimeout

const DefaultTimeout = time.Hour

var bufpool = bp.New(4096)

func GRPCServer(globalCtx context.Context, logger log.Logger, verbose bool, checkAuth func(ctx context.Context, path string) error, options ...grpc.ServerOption) *grpc.Server {
	erroredMethods := make(map[string]struct{})
	var erroredMethodsMu sync.RWMutex

	getLogger := func(ctx context.Context, fullMethod string) (log.Logger, func(error), context.Context, context.CancelFunc) {
		var toCancel context.CancelFunc
		if _, ok := ctx.Deadline(); !ok && Timeout > 0 {
			ctx, toCancel = context.WithTimeout(ctx, Timeout) //nolint:govet
		}
		var cancel context.CancelFunc
		ctx, cancel = joincontext.Join(ctx, globalCtx)
		if toCancel != nil {
			origCancel := cancel
			cancel = func() { toCancel(); origCancel() }
		}
		reqID := ContextGetReqID(ctx)
		ctx = ContextWithReqID(ctx, reqID)
		lgr := log.With(logger, "reqID", reqID)
		ctx = ContextWithLogger(ctx, lgr)
		verbose := verbose
		var wasThere bool
		if !verbose {
			erroredMethodsMu.RLock()
			_, verbose = erroredMethods[fullMethod]
			erroredMethodsMu.RUnlock()
			wasThere = verbose
		}
		if verbose {
			ctx = goracle.ContextWithLog(ctx, log.With(lgr, "lib", "goracle").Log)
		}
		commit := func(err error) {
			if wasThere && err == nil {
				erroredMethodsMu.Lock()
				delete(erroredMethods, fullMethod)
				erroredMethodsMu.Unlock()
			} else if err != nil && !wasThere {
				erroredMethodsMu.Lock()
				erroredMethods[fullMethod] = struct{}{}
				erroredMethodsMu.Unlock()
			}
		}
		return lgr, commit, ctx, cancel
	}

	opts := []grpc.ServerOption{
		grpc.StreamInterceptor(
			func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) (err error) {
				defer func() {
					if r := recover(); r != nil {
						trace := stack.Trace().String()
						var ok bool
						if err, ok = r.(error); ok {
							logger.Log("PANIC", err, "trace", trace)
							return
						}
						err = errors.Errorf("%+v", r)
						logger.Log("PANIC", fmt.Sprintf("%+v", err), "trace", trace)
					}
				}()
				lgr, commit, ctx, cancel := getLogger(ss.Context(), info.FullMethod)
				defer cancel()

				buf := bufpool.Get()
				defer bufpool.Put(buf)
				jenc := json.NewEncoder(buf)
				if err = jenc.Encode(srv); err != nil {
					lgr.Log("marshal error", err, "srv", srv)
				}
				lgr.Log("REQ", info.FullMethod, "srv", buf.String())
				if err = checkAuth(ctx, info.FullMethod); err != nil {
					return status.Error(codes.Unauthenticated, err.Error())
				}

				wss := grpc_middleware.WrapServerStream(ss)
				wss.WrappedContext = ctx
				err = handler(srv, wss)

				lgr.Log("RESP", info.FullMethod, "error", err)
				commit(err)
				return StatusError(err)
			}),

		grpc.UnaryInterceptor(
			func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (_ interface{}, err error) {
				defer func() {
					if r := recover(); r != nil {
						trace := stack.Trace().String()
						var ok bool
						if err, ok = r.(error); ok {
							logger.Log("PANIC", err, "trace", trace)
							return
						}
						err = errors.Errorf("%+v", r)
						logger.Log("PANIC", fmt.Sprintf("%+v", err), "trace", trace)
					}
				}()
				logger, commit, ctx, cancel := getLogger(ctx, info.FullMethod)
				defer cancel()

				if err = checkAuth(ctx, info.FullMethod); err != nil {
					return nil, status.Error(codes.Unauthenticated, err.Error())
				}

				buf := bufpool.Get()
				defer bufpool.Put(buf)
				jenc := json.NewEncoder(buf)
				if err = jenc.Encode(req); err != nil {
					logger.Log("marshal error", err, "req", req)
				}
				logger.Log("REQ", info.FullMethod, "req", buf.String())

				// Fill PArgsHidden
				if r := reflect.ValueOf(req).Elem(); r.Kind() != reflect.Struct {
					logger.Log("error", "not struct", "req", fmt.Sprintf("%T %#v", req, req))
				} else {
					if f := r.FieldByName("PArgsHidden"); f.IsValid() {
						f.Set(reflect.ValueOf(buf.String()))
					}
				}

				res, err := handler(ctx, req)

				logger.Log("RESP", info.FullMethod, "error", err)
				commit(err)

				buf.Reset()
				if jErr := jenc.Encode(res); err != nil {
					logger.Log("marshal error", jErr, "res", res)
				}
				logger.Log("RESP", res, "error", err)

				return res, StatusError(err)
			}),
	}
	return grpc.NewServer(opts...)
}

func StatusError(err error) error {
	if err == nil {
		return err
	}
	var code codes.Code
	var sc interface {
		Code() codes.Code
	}
	if errors.Is(err, oracall.ErrInvalidArgument) {
		code = codes.InvalidArgument
	} else if errors.As(err, &sc) {
		code = sc.Code()
	}
	if code == 0 {
		return err
	}
	s := status.New(code, err.Error())
	if sd, sErr := s.WithDetails(&pbMessage{Message: fmt.Sprintf("%+v", err)}); sErr == nil {
		s = sd
	}
	return s.Err()
}

type pbMessage struct {
	Message string
}

func (m pbMessage) ProtoMessage()   {}
func (m *pbMessage) Reset()         { m.Message = "" }
func (m *pbMessage) String() string { return proto.MarshalTextString(m) }

type ctxKey string

const reqIDCtxKey = ctxKey("reqID")
const loggerCtxKey = ctxKey("logger")

func ContextWithLogger(ctx context.Context, logger log.Logger) context.Context {
	return context.WithValue(ctx, loggerCtxKey, logger)
}
func ContextGetLogger(ctx context.Context) log.Logger {
	if lgr, ok := ctx.Value(loggerCtxKey).(log.Logger); ok {
		return lgr
	}
	return nil
}
func ContextWithReqID(ctx context.Context, reqID string) context.Context {
	if reqID == "" {
		reqID = NewULID()
	}
	return context.WithValue(ctx, reqIDCtxKey, reqID)
}
func ContextGetReqID(ctx context.Context) string {
	if reqID, ok := ctx.Value(reqIDCtxKey).(string); ok {
		return reqID
	}
	return NewULID()
}
func NewULID() string {
	return ulid.MustNew(ulid.Now(), rand.Reader).String()
}
<commit_msg>orasrv: log call duration<commit_after>package orasrv

import (
	"context"
	"crypto/rand"
	"encoding/json"
	"fmt"
	"reflect"
	"sync"
	"time"

	"github.com/LK4D4/joincontext"
	"github.com/gogo/protobuf/proto"
	bp "github.com/tgulacsi/go/bufpool"
	oracall "github.com/tgulacsi/oracall/lib"
	errors "golang.org/x/xerrors"

	"github.com/go-kit/kit/log"
	"github.com/oklog/ulid"

	"github.com/go-stack/stack"
	"github.com/grpc-ecosystem/go-grpc-middleware"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	_ "google.golang.org/grpc/encoding/gzip"
	"google.golang.org/grpc/status"

	goracle "gopkg.in/goracle.v2"
)

var Timeout = DefaultTimeout

const DefaultTimeout = time.Hour

var bufpool = bp.New(4096)

func GRPCServer(globalCtx context.Context, logger log.Logger, verbose bool, checkAuth func(ctx context.Context, path string) error, options ...grpc.ServerOption) *grpc.Server {
	erroredMethods := make(map[string]struct{})
	var erroredMethodsMu sync.RWMutex

	getLogger := func(ctx context.Context, fullMethod string) (log.Logger, func(error), context.Context, context.CancelFunc) {
		var toCancel context.CancelFunc
		if _, ok := ctx.Deadline(); !ok && Timeout > 0 {
			ctx, toCancel = context.WithTimeout(ctx, Timeout) //nolint:govet
		}
		var cancel context.CancelFunc
		ctx, cancel = joincontext.Join(ctx, globalCtx)
		if toCancel != nil {
			origCancel := cancel
			cancel = func() { toCancel(); origCancel() }
		}
		reqID := ContextGetReqID(ctx)
		ctx = ContextWithReqID(ctx, reqID)
		lgr := log.With(logger, "reqID", reqID)
		ctx = ContextWithLogger(ctx, lgr)
		verbose := verbose
		var wasThere bool
		if !verbose {
			erroredMethodsMu.RLock()
			_, verbose = erroredMethods[fullMethod]
			erroredMethodsMu.RUnlock()
			wasThere = verbose
		}
		if verbose {
			ctx = goracle.ContextWithLog(ctx, log.With(lgr, "lib", "goracle").Log)
		}
		commit := func(err error) {
			if wasThere && err == nil {
				erroredMethodsMu.Lock()
				delete(erroredMethods, fullMethod)
				erroredMethodsMu.Unlock()
			} else if err != nil && !wasThere {
				erroredMethodsMu.Lock()
				erroredMethods[fullMethod] = struct{}{}
				erroredMethodsMu.Unlock()
			}
		}
		return lgr, commit, ctx, cancel
	}

	opts := []grpc.ServerOption{
		grpc.StreamInterceptor(
			func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) (err error) {
				defer func() {
					if r := recover(); r != nil {
						trace := stack.Trace().String()
						var ok bool
						if err, ok = r.(error); ok {
							logger.Log("PANIC", err, "trace", trace)
							return
						}
						err = errors.Errorf("%+v", r)
						logger.Log("PANIC", fmt.Sprintf("%+v", err), "trace", trace)
					}
				}()
				lgr, commit, ctx, cancel := getLogger(ss.Context(), info.FullMethod)
				defer cancel()

				buf := bufpool.Get()
				defer bufpool.Put(buf)
				jenc := json.NewEncoder(buf)
				if err = jenc.Encode(srv); err != nil {
					lgr.Log("marshal error", err, "srv", srv)
				}
				lgr.Log("REQ", info.FullMethod, "srv", buf.String())
				if err = checkAuth(ctx, info.FullMethod); err != nil {
					return status.Error(codes.Unauthenticated, err.Error())
				}

				wss := grpc_middleware.WrapServerStream(ss)
				wss.WrappedContext = ctx
				start := time.Now()
				err = handler(srv, wss)
				lgr.Log("RESP", info.FullMethod, "dur", time.Since(start), "error", err)
				commit(err)
				return StatusError(err)
			}),

		grpc.UnaryInterceptor(
			func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (_ interface{}, err error) {
				defer func() {
					if r := recover(); r != nil {
						trace := stack.Trace().String()
						var ok bool
						if err, ok = r.(error); ok {
							logger.Log("PANIC", err, "trace", trace)
							return
						}
						err = errors.Errorf("%+v", r)
						logger.Log("PANIC", fmt.Sprintf("%+v", err), "trace", trace)
					}
				}()
				logger, commit, ctx, cancel := getLogger(ctx, info.FullMethod)
				defer cancel()

				if err = checkAuth(ctx, info.FullMethod); err != nil {
					return nil, status.Error(codes.Unauthenticated, err.Error())
				}

				buf := bufpool.Get()
				defer bufpool.Put(buf)
				jenc := json.NewEncoder(buf)
				if err = jenc.Encode(req); err != nil {
					logger.Log("marshal error", err, "req", req)
				}
				logger.Log("REQ", info.FullMethod, "req", buf.String())

				// Fill PArgsHidden
				if r := reflect.ValueOf(req).Elem(); r.Kind() != reflect.Struct {
					logger.Log("error", "not struct", "req", fmt.Sprintf("%T %#v", req, req))
				} else {
					if f := r.FieldByName("PArgsHidden"); f.IsValid() {
						f.Set(reflect.ValueOf(buf.String()))
					}
				}

				start := time.Now()
				res, err := handler(ctx, req)

				logger.Log("RESP", info.FullMethod, "dur", time.Since(start), "error", err)
				commit(err)

				buf.Reset()
				if jErr := jenc.Encode(res); err != nil {
					logger.Log("marshal error", jErr, "res", res)
				}
				logger.Log("RESP", res, "error", err)

				return res, StatusError(err)
			}),
	}
	return grpc.NewServer(opts...)
}

func StatusError(err error) error {
	if err == nil {
		return err
	}
	var code codes.Code
	var sc interface {
		Code() codes.Code
	}
	if errors.Is(err, oracall.ErrInvalidArgument) {
		code = codes.InvalidArgument
	} else if errors.As(err, &sc) {
		code = sc.Code()
	}
	if code == 0 {
		return err
	}
	s := status.New(code, err.Error())
	if sd, sErr := s.WithDetails(&pbMessage{Message: fmt.Sprintf("%+v", err)}); sErr == nil {
		s = sd
	}
	return s.Err()
}

type pbMessage struct {
	Message string
}

func (m pbMessage) ProtoMessage()   {}
func (m *pbMessage) Reset()         { m.Message = "" }
func (m *pbMessage) String() string { return proto.MarshalTextString(m) }

type ctxKey string

const reqIDCtxKey = ctxKey("reqID")
const loggerCtxKey = ctxKey("logger")

func ContextWithLogger(ctx context.Context, logger log.Logger) context.Context {
	return context.WithValue(ctx, loggerCtxKey, logger)
}
func ContextGetLogger(ctx context.Context) log.Logger {
	if lgr, ok := ctx.Value(loggerCtxKey).(log.Logger); ok {
		return lgr
	}
	return nil
}
func ContextWithReqID(ctx context.Context, reqID string) context.Context {
	if reqID == "" {
		reqID = NewULID()
	}
	return context.WithValue(ctx, reqIDCtxKey, reqID)
}
func ContextGetReqID(ctx context.Context) string {
	if reqID, ok := ctx.Value(reqIDCtxKey).(string); ok {
		return reqID
	}
	return NewULID()
}
func NewULID() string {
	return ulid.MustNew(ulid.Now(), rand.Reader).String()
}
<|endoftext|>