<commit_before>package eventservices

import (
	"context"
	"errors"
	"fmt"
	"math/big"
	"time"

	"github.com/cenkalti/backoff"
	"github.com/ethereum/go-ethereum"
	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/golang/glog"
	lpcommon "github.com/livepeer/go-livepeer/common"
	"github.com/livepeer/go-livepeer/core"
	"github.com/livepeer/go-livepeer/eth"
	lpTypes "github.com/livepeer/go-livepeer/eth/types"
)

var (
	ErrJobServiceStarted  = fmt.Errorf("job service already started")
	ErrJobServicedStopped = fmt.Errorf("job service already stopped")
)

type JobService struct {
	eventMonitor eth.EventMonitor
	node         *core.LivepeerNode
	sub          ethereum.Subscription
	logsCh       chan types.Log
}

func NewJobService(eventMonitor eth.EventMonitor, node *core.LivepeerNode) *JobService {
	return &JobService{
		eventMonitor: eventMonitor,
		node:         node,
	}
}

func (s *JobService) Start(ctx context.Context) error {
	if s.sub != nil {
		return ErrJobServiceStarted
	}

	logsCh := make(chan types.Log)
	sub, err := s.eventMonitor.SubscribeNewJob(ctx, "NewJob", logsCh, common.Address{}, func(l types.Log) (bool, error) {
		jid := parseNewJobLog(l)
		if jid == nil {
			// Return true to ignore this log and continue monitoring for new jobs
			return true, errors.New("InvalidJobLog")
		}

		var job *lpTypes.Job
		getJob := func() error {
			j, err := s.node.Eth.GetJob(jid)
			if err != nil || j == nil {
				glog.Errorf("Unable to get job %v, try again. Error: %v", jid, err)
				return err
			}
			if j.StreamId == "" {
				glog.Errorf("Got empty job for id:%v. Should try again.", jid.Int64())
				return errors.New("ErrGetJob")
			}
			job = j
			return err
		}
		if err := backoff.Retry(getJob, backoff.NewConstantBackOff(time.Second*2)); err != nil {
			glog.Errorf("Error getting job info: %v", err)
			return false, err
		}

		assignedAddr, err := s.node.Eth.AssignedTranscoder(job)
		if err != nil {
			glog.Errorf("Error checking for assignment: %v", err)
			return false, err
		}

		if assignedAddr == s.node.Eth.Account().Address {
			dbjob := lpcommon.NewDBJob(
				job.JobId, job.StreamId,
				job.MaxPricePerSegment, job.Profiles,
				job.BroadcasterAddress, s.node.Eth.Account().Address,
				job.CreationBlock, job.EndBlock)
			s.node.Database.InsertJob(dbjob)
			s.firstClaim(job)
		}
		return true, nil
	})

	if err != nil {
		return err
	}

	s.logsCh = logsCh
	s.sub = sub

	return nil
}

func (s *JobService) Stop() error {
	if s.sub == nil {
		return ErrJobServicedStopped
	}

	close(s.logsCh)
	s.sub.Unsubscribe()

	s.logsCh = nil
	s.sub = nil

	return nil
}

func (s *JobService) IsWorking() bool {
	return s.sub != nil
}

func (s *JobService) RestartTranscoder() error {

	return eth.RecoverClaims(s.node.Eth, s.node.Ipfs, s.node.Database)
}

func (s *JobService) firstClaim(job *lpTypes.Job) {
	cm, err := s.node.GetClaimManager(job)
	if err != nil {
		glog.Error("Could not get claim manager: ", err)
		return
	}
	firstClaimBlock := new(big.Int).Add(job.CreationBlock, eth.BlocksUntilFirstClaimDeadline)
	headersCh := make(chan *types.Header)
	sub, err := s.eventMonitor.SubscribeNewBlock(context.Background(), fmt.Sprintf("FirstClaimForJob%v", job.JobId), headersCh, func(h *types.Header) (bool, error) {
		if cm.DidFirstClaim() {
			// If the first claim has already been made then exit
			return false, nil
		}

		// Check if current block is job creation block + 230
		if h.Number.Cmp(firstClaimBlock) != -1 {
			glog.Infof("Making the first claim")

			canClaim, err := cm.CanClaim()
			if err != nil {
				return false, err
			}

			if canClaim {
				err := cm.ClaimVerifyAndDistributeFees()
				if err != nil {
					return false, err
				} else {
					// If this claim was successful then the first claim has been made - exit
					return false, nil
				}
			} else {
				glog.Infof("No segments to claim")
				// If there are no segments to claim at this point just stop watching
				return false, nil
			}
		} else {
			return true, nil
		}
	})
	if err == nil {
		sub.Unsubscribe()
	}
}

func parseNewJobLog(log types.Log) (jid *big.Int) {
	if len(log.Data) < 32 {
		glog.Error("Malformed job event log!")
		return nil
	}
	return new(big.Int).SetBytes(log.Data[0:32])
}
<commit_msg>jobwatcher: Retry the assigned transcoder lookup.<commit_after>package eventservices

import (
	"context"
	"errors"
	"fmt"
	"math/big"
	"time"

	"github.com/cenkalti/backoff"
	"github.com/ethereum/go-ethereum"
	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/golang/glog"
	lpcommon "github.com/livepeer/go-livepeer/common"
	"github.com/livepeer/go-livepeer/core"
	"github.com/livepeer/go-livepeer/eth"
	lpTypes "github.com/livepeer/go-livepeer/eth/types"
)

var (
	ErrJobServiceStarted  = fmt.Errorf("job service already started")
	ErrJobServicedStopped = fmt.Errorf("job service already stopped")
)

type JobService struct {
	eventMonitor eth.EventMonitor
	node         *core.LivepeerNode
	sub          ethereum.Subscription
	logsCh       chan types.Log
}

func NewJobService(eventMonitor eth.EventMonitor, node *core.LivepeerNode) *JobService {
	return &JobService{
		eventMonitor: eventMonitor,
		node:         node,
	}
}

func (s *JobService) Start(ctx context.Context) error {
	if s.sub != nil {
		return ErrJobServiceStarted
	}

	logsCh := make(chan types.Log)
	sub, err := s.eventMonitor.SubscribeNewJob(ctx, "NewJob", logsCh, common.Address{}, func(l types.Log) (bool, error) {
		jid := parseNewJobLog(l)
		if jid == nil {
			// Return true to ignore this log and continue monitoring for new jobs
			return true, errors.New("InvalidJobLog")
		}

		var job *lpTypes.Job
		getJob := func() error {
			j, err := s.node.Eth.GetJob(jid)
			if err != nil || j == nil {
				glog.Errorf("Unable to get job %v, try again. Error: %v", jid, err)
				return err
			}
			if j.StreamId == "" {
				glog.Errorf("Got empty job for id:%v. Should try again.", jid.Int64())
				return errors.New("ErrGetJob")
			}
			assignedAddr, err := s.node.Eth.AssignedTranscoder(j)
			if err != nil {
				glog.Errorf("Error checking for job %v assignment: %v", jid, err)
				return err
			}
			j.TranscoderAddress = assignedAddr
			job = j
			return err
		}
		if err := backoff.Retry(getJob, backoff.NewConstantBackOff(time.Second*2)); err != nil {
			glog.Errorf("Error getting job info: %v", err)
			return false, err
		}

		if job.TranscoderAddress == s.node.Eth.Account().Address {
			dbjob := lpcommon.NewDBJob(
				job.JobId, job.StreamId,
				job.MaxPricePerSegment, job.Profiles,
				job.BroadcasterAddress, s.node.Eth.Account().Address,
				job.CreationBlock, job.EndBlock)
			s.node.Database.InsertJob(dbjob)
			s.firstClaim(job)
		}
		return true, nil
	})

	if err != nil {
		return err
	}

	s.logsCh = logsCh
	s.sub = sub

	return nil
}

func (s *JobService) Stop() error {
	if s.sub == nil {
		return ErrJobServicedStopped
	}

	close(s.logsCh)
	s.sub.Unsubscribe()

	s.logsCh = nil
	s.sub = nil

	return nil
}

func (s *JobService) IsWorking() bool {
	return s.sub != nil
}

func (s *JobService) RestartTranscoder() error {

	return eth.RecoverClaims(s.node.Eth, s.node.Ipfs, s.node.Database)
}

func (s *JobService) firstClaim(job *lpTypes.Job) {
	cm, err := s.node.GetClaimManager(job)
	if err != nil {
		glog.Error("Could not get claim manager: ", err)
		return
	}
	firstClaimBlock := new(big.Int).Add(job.CreationBlock, eth.BlocksUntilFirstClaimDeadline)
	headersCh := make(chan *types.Header)
	sub, err := s.eventMonitor.SubscribeNewBlock(context.Background(), fmt.Sprintf("FirstClaimForJob%v", job.JobId), headersCh, func(h *types.Header) (bool, error) {
		if cm.DidFirstClaim() {
			// If the first claim has already been made then exit
			return false, nil
		}

		// Check if current block is job creation block + 230
		if h.Number.Cmp(firstClaimBlock) != -1 {
			glog.Infof("Making the first claim")

			canClaim, err := cm.CanClaim()
			if err != nil {
				return false, err
			}

			if canClaim {
				err := cm.ClaimVerifyAndDistributeFees()
				if err != nil {
					return false, err
				} else {
					// If this claim was successful then the first claim has been made - exit
					return false, nil
				}
			} else {
				glog.Infof("No segments to claim")
				// If there are no segments to claim at this point just stop watching
				return false, nil
			}
		} else {
			return true, nil
		}
	})
	if err == nil {
		sub.Unsubscribe()
	}
}

func parseNewJobLog(log types.Log) (jid *big.Int) {
	if len(log.Data) < 32 {
		glog.Error("Malformed job event log!")
		return nil
	}
	return new(big.Int).SetBytes(log.Data[0:32])
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
//-----------------------------------------------------------------------------

package mop

import (
	`fmt`
	`bytes`
	`io/ioutil`
	`net/http`
	`regexp`
	`strings`
)

const market_url = `http://finance.yahoo.com/marketupdate/overview`

type Market struct {
	IsClosed   bool
	Dow        map[string]string
	Nasdaq     map[string]string
	Sp500      map[string]string
	Advances   map[string]string
	Declines   map[string]string
	Unchanged  map[string]string
	Highs      map[string]string
	Lows       map[string]string
	regex      *regexp.Regexp
	errors     string
}

//-----------------------------------------------------------------------------
func (self *Market) Initialize() *Market {
	self.IsClosed   = false
	self.Dow        = make(map[string]string)
	self.Nasdaq     = make(map[string]string)
	self.Sp500      = make(map[string]string)
	self.Advances   = make(map[string]string)
	self.Declines   = make(map[string]string)
	self.Unchanged  = make(map[string]string)
	self.Highs      = make(map[string]string)
	self.Lows       = make(map[string]string)
	self.errors     = ``

	const any = `\s*<.+?>`
	const some = `<.+?`
	const space = `\s*`
	const color = `#([08c]{6});">\s*`
	const price = `([\d\.,]+)`
	const percent = `\(([\d\.,%]+)\)`

	rules := []string{
		`(Dow)`, any, price, some, color, price, some, percent, any,
		`(Nasdaq)`, any, price, some, color, price, some, percent, any,
		`(S&P 500)`, any, price, some, color, price, some, percent, any,
		`(Advances)`, any, price, space, percent, any, price, space, percent, any,
		`(Declines)`, any, price, space, percent, any, price, space, percent, any,
		`(Unchanged)`, any, price, space, percent, any, price, space, percent, any,
		`(New Hi's)`, any, price, any, price, any,
		`(New Lo's)`, any, price, any, price, any,
	}

	self.regex = regexp.MustCompile(strings.Join(rules, ``))

	return self
}

//-----------------------------------------------------------------------------
func (self *Market) Fetch() (this *Market) {
	this = self // <-- This ensures we return correct self after recover() from panic() attack.
	defer func() {
		if err := recover(); err != nil {
			self.errors = fmt.Sprintf("Error fetching market data...\n%s", err)
		}
	}()

	response, err := http.Get(market_url)
	if err != nil {
		panic(err)
	}

	defer response.Body.Close()
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		panic(err)
	}

	body = self.check_if_market_open(body)
	return self.extract(self.trim(body))
}

//-----------------------------------------------------------------------------
func (self *Market) Ok() (bool, string) {
	return self.errors == ``, self.errors
}

// private
//-----------------------------------------------------------------------------
func (self *Market) check_if_market_open(body []byte) []byte {
	start := bytes.Index(body, []byte(`id="yfs_market_time"`))
	finish := start + bytes.Index(body[start:], []byte(`</span>`))
	snippet := body[start:finish]
	self.IsClosed = bytes.Contains(snippet, []byte(`closed`)) || bytes.Contains(snippet, []byte(`open in`))

	return body[finish:]
}

//-----------------------------------------------------------------------------
func (self *Market) trim(body []byte) []byte {
	start := bytes.Index(body, []byte(`<table id="yfimktsumm"`))
	finish := bytes.LastIndex(body, []byte(`<table id="yfimktsumm"`))
	snippet := bytes.Replace(body[start:finish], []byte{'\n'}, []byte{}, -1)
	snippet = bytes.Replace(snippet, []byte(`&`), []byte{'&'}, -1)

	return snippet
}

//-----------------------------------------------------------------------------
func (self *Market) extract(snippet []byte) *Market {
	matches := self.regex.FindAllStringSubmatch(string(snippet), -1)
	if len(matches) < 1 || len(matches[0]) < 37 {
		panic(`Unable to parse ` + market_url)
	}

	self.Dow[`name`] = matches[0][1]
	self.Dow[`latest`] = matches[0][2]
	self.Dow[`change`] = matches[0][4]
	switch matches[0][3] {
	case `008800`:
		self.Dow[`change`] = `+` + matches[0][4]
		self.Dow[`percent`] = `+` + matches[0][5]
	case `cc0000`:
		self.Dow[`change`] = `-` + matches[0][4]
		self.Dow[`percent`] = `-` + matches[0][5]
	default:
		self.Dow[`change`] = matches[0][4]
		self.Dow[`percent`] = matches[0][5]
	}

	self.Nasdaq[`name`] = matches[0][6]
	self.Nasdaq[`latest`] = matches[0][7]
	switch matches[0][8] {
	case `008800`:
		self.Nasdaq[`change`] = `+` + matches[0][9]
		self.Nasdaq[`percent`] = `+` + matches[0][10]
	case `cc0000`:
		self.Nasdaq[`change`] = `-` + matches[0][9]
		self.Nasdaq[`percent`] = `-` + matches[0][10]
	default:
		self.Nasdaq[`change`] = matches[0][9]
		self.Nasdaq[`percent`] = matches[0][10]
	}

	self.Sp500[`name`] = matches[0][11]
	self.Sp500[`latest`] = matches[0][12]
	switch matches[0][13] {
	case `008800`:
		self.Sp500[`change`] = `+` + matches[0][14]
		self.Sp500[`percent`] = `+` + matches[0][15]
	case `cc0000`:
		self.Sp500[`change`] = `-` + matches[0][14]
		self.Sp500[`percent`] = `-` + matches[0][15]
	default:
		self.Sp500[`change`] = matches[0][14]
		self.Sp500[`percent`] = matches[0][15]
	}

	self.Advances[`name`] = matches[0][16]
	self.Advances[`nyse`] = matches[0][17]
	self.Advances[`nysep`] = matches[0][18]
	self.Advances[`nasdaq`] = matches[0][19]
	self.Advances[`nasdaqp`] = matches[0][20]

	self.Declines[`name`] = matches[0][21]
	self.Declines[`nyse`] = matches[0][22]
	self.Declines[`nysep`] = matches[0][23]
	self.Declines[`nasdaq`] = matches[0][24]
	self.Declines[`nasdaqp`] = matches[0][25]

	self.Unchanged[`name`] = matches[0][26]
	self.Unchanged[`nyse`] = matches[0][27]
	self.Unchanged[`nysep`] = matches[0][28]
	self.Unchanged[`nasdaq`] = matches[0][29]
	self.Unchanged[`nasdaqp`] = matches[0][30]

	self.Highs[`name`] = matches[0][31]
	self.Highs[`nyse`] = matches[0][32]
	self.Highs[`nasdaq`] = matches[0][33]
	self.Lows[`name`] = matches[0][34]
	self.Lows[`nyse`] = matches[0][35]
	self.Lows[`nasdaq`] = matches[0][36]

	return self
}
<commit_msg>First golint pass for yahoo_market.go<commit_after>// Copyright (c) 2013 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package mop

import (
	`bytes`
	`fmt`
	`io/ioutil`
	`net/http`
	`regexp`
	`strings`
)

const marketURL = `http://finance.yahoo.com/marketupdate/overview`

type Market struct {
	IsClosed   bool		      // True when U.S. markets are closed.
	Dow        map[string]string  // Hash of Dow Jones indicators.
	Nasdaq     map[string]string  // Hash of NASDAQ indicators.
	Sp500      map[string]string  // Hash of S&P 500 indicators.
	Advances   map[string]string  //
	Declines   map[string]string  //
	Unchanged  map[string]string  //
	Highs      map[string]string  //
	Lows       map[string]string  //
	regex      *regexp.Regexp     //
	errors     string	      //
}

//-----------------------------------------------------------------------------
func (market *Market) Initialize() *Market {
	market.IsClosed   = false
	market.Dow        = make(map[string]string)
	market.Nasdaq     = make(map[string]string)
	market.Sp500      = make(map[string]string)
	market.Advances   = make(map[string]string)
	market.Declines   = make(map[string]string)
	market.Unchanged  = make(map[string]string)
	market.Highs      = make(map[string]string)
	market.Lows       = make(map[string]string)
	market.errors     = ``

	const any = `\s*<.+?>`
	const some = `<.+?`
	const space = `\s*`
	const color = `#([08c]{6});">\s*`
	const price = `([\d\.,]+)`
	const percent = `\(([\d\.,%]+)\)`

	rules := []string{
		`(Dow)`, any, price, some, color, price, some, percent, any,
		`(Nasdaq)`, any, price, some, color, price, some, percent, any,
		`(S&P 500)`, any, price, some, color, price, some, percent, any,
		`(Advances)`, any, price, space, percent, any, price, space, percent, any,
		`(Declines)`, any, price, space, percent, any, price, space, percent, any,
		`(Unchanged)`, any, price, space, percent, any, price, space, percent, any,
		`(New Hi's)`, any, price, any, price, any,
		`(New Lo's)`, any, price, any, price, any,
	}

	market.regex = regexp.MustCompile(strings.Join(rules, ``))

	return market
}

//-----------------------------------------------------------------------------
func (market *Market) Fetch() (this *Market) {
	this = market // <-- This ensures we return correct market after recover() from panic() attack.
	defer func() {
		if err := recover(); err != nil {
			market.errors = fmt.Sprintf("Error fetching market data...\n%s", err)
		}
	}()

	response, err := http.Get(marketURL)
	if err != nil {
		panic(err)
	}

	defer response.Body.Close()
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		panic(err)
	}

	body = market.checkIfMarketIsOpen(body)
	return market.extract(market.trim(body))
}

//-----------------------------------------------------------------------------
func (market *Market) Ok() (bool, string) {
	return market.errors == ``, market.errors
}

// private
//-----------------------------------------------------------------------------
func (market *Market) checkIfMarketIsOpen(body []byte) []byte {
	start := bytes.Index(body, []byte(`id="yfs_market_time"`))
	finish := start + bytes.Index(body[start:], []byte(`</span>`))
	snippet := body[start:finish]
	market.IsClosed = bytes.Contains(snippet, []byte(`closed`)) || bytes.Contains(snippet, []byte(`open in`))

	return body[finish:]
}

//-----------------------------------------------------------------------------
func (market *Market) trim(body []byte) []byte {
	start := bytes.Index(body, []byte(`<table id="yfimktsumm"`))
	finish := bytes.LastIndex(body, []byte(`<table id="yfimktsumm"`))
	snippet := bytes.Replace(body[start:finish], []byte{'\n'}, []byte{}, -1)
	snippet = bytes.Replace(snippet, []byte(`&`), []byte{'&'}, -1)

	return snippet
}

//-----------------------------------------------------------------------------
func (market *Market) extract(snippet []byte) *Market {
	matches := market.regex.FindAllStringSubmatch(string(snippet), -1)
	if len(matches) < 1 || len(matches[0]) < 37 {
		panic(`Unable to parse ` + marketURL)
	}

	market.Dow[`name`] = matches[0][1]
	market.Dow[`latest`] = matches[0][2]
	market.Dow[`change`] = matches[0][4]
	switch matches[0][3] {
	case `008800`:
		market.Dow[`change`] = `+` + matches[0][4]
		market.Dow[`percent`] = `+` + matches[0][5]
	case `cc0000`:
		market.Dow[`change`] = `-` + matches[0][4]
		market.Dow[`percent`] = `-` + matches[0][5]
	default:
		market.Dow[`change`] = matches[0][4]
		market.Dow[`percent`] = matches[0][5]
	}

	market.Nasdaq[`name`] = matches[0][6]
	market.Nasdaq[`latest`] = matches[0][7]
	switch matches[0][8] {
	case `008800`:
		market.Nasdaq[`change`] = `+` + matches[0][9]
		market.Nasdaq[`percent`] = `+` + matches[0][10]
	case `cc0000`:
		market.Nasdaq[`change`] = `-` + matches[0][9]
		market.Nasdaq[`percent`] = `-` + matches[0][10]
	default:
		market.Nasdaq[`change`] = matches[0][9]
		market.Nasdaq[`percent`] = matches[0][10]
	}

	market.Sp500[`name`] = matches[0][11]
	market.Sp500[`latest`] = matches[0][12]
	switch matches[0][13] {
	case `008800`:
		market.Sp500[`change`] = `+` + matches[0][14]
		market.Sp500[`percent`] = `+` + matches[0][15]
	case `cc0000`:
		market.Sp500[`change`] = `-` + matches[0][14]
		market.Sp500[`percent`] = `-` + matches[0][15]
	default:
		market.Sp500[`change`] = matches[0][14]
		market.Sp500[`percent`] = matches[0][15]
	}

	market.Advances[`name`] = matches[0][16]
	market.Advances[`nyse`] = matches[0][17]
	market.Advances[`nysep`] = matches[0][18]
	market.Advances[`nasdaq`] = matches[0][19]
	market.Advances[`nasdaqp`] = matches[0][20]

	market.Declines[`name`] = matches[0][21]
	market.Declines[`nyse`] = matches[0][22]
	market.Declines[`nysep`] = matches[0][23]
	market.Declines[`nasdaq`] = matches[0][24]
	market.Declines[`nasdaqp`] = matches[0][25]

	market.Unchanged[`name`] = matches[0][26]
	market.Unchanged[`nyse`] = matches[0][27]
	market.Unchanged[`nysep`] = matches[0][28]
	market.Unchanged[`nasdaq`] = matches[0][29]
	market.Unchanged[`nasdaqp`] = matches[0][30]

	market.Highs[`name`] = matches[0][31]
	market.Highs[`nyse`] = matches[0][32]
	market.Highs[`nasdaq`] = matches[0][33]
	market.Lows[`name`] = matches[0][34]
	market.Lows[`nyse`] = matches[0][35]
	market.Lows[`nasdaq`] = matches[0][36]

	return market
}
<|endoftext|>


<commit_before>package dptools

import (
	"github.com/bit4bit/glivo"
	"bytes"
)

//Execute Bridge and returns the events hangup for the ALeg and BLeg of the call
func (dptools *DPTools) Bridge(endpoint string) (aleg *glivo.Event, bleg *glivo.Event) {
	var send bytes.Buffer

	send.WriteString(endpoint)

	wait_bridge := dptools.call.WaitExecute("bridge", map[string]string{
		"Event-Name": "CHANNEL_EXECUTE",
		"Application": "bridge",
	})
	dptools.call.Execute("bridge", send.String(), true)
	<-wait_bridge

	wait_action := dptools.call.OnceEventHandle("bridge_action", func(res chan interface{}) glivo.HandlerEvent {
		return glivo.NewWaitAnyEventHandle(res,
			[]map[string]string{
				{"Event-Name": "CHANNEL_UNBRIDGE"},
				{"Event-Name": "CHANNEL_HANGUP"},
				{"Event-Name": "CHANNEL_EXECUTE_COMPLETE"},
			})
	})

	action := (<-wait_action).(glivo.Event)

	switch action.Content["Event-Name"] {
		//hangup BLeg
	case "CHANNEL_UNBRIDGE":
		bleg = &action
		//wait hangup ALeg
		action_aleg := (<-wait_bridge).(glivo.Event)
		aleg = &action_aleg
		//hangup trying bridge
		//hangup ALeg
	case "CHANNEL_HANGUP":
		aleg = &action
	case "CHANNEL_EXECUTE_COMPLETE":
		aleg = &action
	}

	return aleg, bleg
}
<commit_msg>bridge un solo evento de retorno<commit_after>package dptools

import (
	"github.com/bit4bit/glivo"
	"bytes"
)

//Execute Bridge and returns the events hangup for the ALeg and BLeg of the call
func (dptools *DPTools) Bridge(endpoint string) *glivo.Event {
	var send bytes.Buffer

	send.WriteString(endpoint)

	wait_bridge := dptools.call.WaitExecute("bridge", map[string]string{
		"Event-Name": "CHANNEL_EXECUTE",
		"Application": "bridge",
	})
	dptools.call.Execute("bridge", send.String(), true)
	<-wait_bridge

	wait_action := dptools.call.OnceEventHandle("bridge_action", func(res chan interface{}) glivo.HandlerEvent {
		return glivo.NewWaitAnyEventHandle(res,
			[]map[string]string{
				{"Event-Name": "CHANNEL_UNBRIDGE", "Application": "bridge" },
				{"Event-Name": "CHANNEL_HANGUP", "Application": "bridge" },
				{"Event-Name": "CHANNEL_EXECUTE_COMPLETE", "Application": "bridge" },
			})
	})

	action := (<-wait_action).(glivo.Event)

	switch action.Content["Event-Name"] {
		//hangup BLeg
	case "CHANNEL_UNBRIDGE":
		return &action
	case "CHANNEL_HANGUP":
		return &action
	case "CHANNEL_EXECUTE_COMPLETE":
		return &action
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"

	"golang.org/x/net/context"
	"google.golang.org/grpc/examples/helloworld/helloworld"

	"github.com/ligato/cn-infra/core"
	"github.com/ligato/cn-infra/flavors/local"
	"github.com/ligato/cn-infra/flavors/rpc"
	"github.com/ligato/cn-infra/rpc/grpc"
)

// *************************************************************************
// This file contains GRPC service exposure example. To register service use
// Server.RegisterService(descriptor, service)
// ************************************************************************/

func main() {
	// Init close channel to stop the example after everything was logged
	exampleFinished := make(chan struct{})

	// Start Agent with ExamplePlugin & FlavorRPC (reused cn-infra plugins).
	agent := rpc.NewAgent(rpc.WithPlugins(func(flavor *rpc.FlavorRPC) []*core.NamedPlugin {
		examplePlug := &ExamplePlugin{
			exampleFinished: exampleFinished,
			Deps: Deps{
				PluginLogDeps: *flavor.LogDeps("example"),
				GRPC:          &flavor.GRPC,
			},
		}
		return []*core.NamedPlugin{{examplePlug.PluginName, examplePlug}}
	}))
	core.EventLoopWithInterrupt(agent, exampleFinished)
}

// ExamplePlugin presents the PluginLogger API.
type ExamplePlugin struct {
	Deps
	exampleFinished chan struct{}
}

// Deps - dependencies for ExamplePlugin
type Deps struct {
	local.PluginLogDeps
	GRPC grpc.Server
}

// Init demonstrates the usage of PluginLogger API.
func (plugin *ExamplePlugin) Init() (err error) {
	plugin.Log.Info("Example Init")

	helloworld.RegisterGreeterServer(plugin.GRPC.GetServer(), &GreeterService{})

	return nil
}

// GreeterService implements GRPC GreeterServer interface (interface generated from protobuf definition file).
// It is a simple implementation for testing/demo only purposes.
type GreeterService struct{}

// SayHello returns error if request.name was not filled otherwise: "hello " + request.Name
func (*GreeterService) SayHello(ctx context.Context, request *helloworld.HelloRequest) (*helloworld.HelloReply, error) {
	if request.Name == "" {
		return nil, errors.New("not filled name in the request")
	}

	return &helloworld.HelloReply{Message: "hello " + request.Name}, nil
}
<commit_msg>Update GRPC example<commit_after>package main

import (
	"errors"
	"log"

	"github.com/ligato/cn-infra/agent"
	"github.com/ligato/cn-infra/logging"
	"github.com/ligato/cn-infra/logging/logrus"
	"golang.org/x/net/context"
	"google.golang.org/grpc/examples/helloworld/helloworld"

	"github.com/ligato/cn-infra/rpc/grpc"
)

// *************************************************************************
// This file contains GRPC service exposure example. To register service use
// Server.RegisterService(descriptor, service)
// ************************************************************************/

const PluginName = "example"

func main() {
	// Init close channel to stop the example after everything was logged
	/*exampleFinished := make(chan struct{})

	// Start Agent with ExamplePlugin & FlavorRPC (reused cn-infra plugins).
	agent := rpc.NewAgent(rpc.WithPlugins(func(flavor *rpc.FlavorRPC) []*core.NamedPlugin {
		examplePlug := &ExamplePlugin{
			exampleFinished: exampleFinished,
			Deps: Deps{
				PluginLogDeps: *flavor.LogDeps("example"),
				GRPC:          &flavor.GRPC,
			},
		}
		return []*core.NamedPlugin{{examplePlug.PluginName, examplePlug}}
	}))
	core.EventLoopWithInterrupt(agent, exampleFinished)*/

	p := &ExamplePlugin{
		Deps: Deps{
			Log:  logging.ForPlugin(PluginName, logrus.DefaultRegistry),
			GRPC: grpc.DefaultPlugin,
		},
	}

	a := agent.NewAgent(agent.AllPlugins(p))
	if err := a.Run(); err != nil {
		log.Fatal(err)
	}
}

// ExamplePlugin presents the PluginLogger API.
type ExamplePlugin struct {
	Deps
	//exampleFinished chan struct{}
}

// Deps - dependencies for ExamplePlugin
type Deps struct {
	Log logging.PluginLogger
	//local.PluginLogDeps
	GRPC grpc.Server
}

// Init demonstrates the usage of PluginLogger API.
func (plugin *ExamplePlugin) Init() error {
	plugin.Log.Info("Example Init")

	helloworld.RegisterGreeterServer(plugin.GRPC.GetServer(), &GreeterService{})

	return nil
}

func (plugin *ExamplePlugin) Close() error {
	return nil
}

// GreeterService implements GRPC GreeterServer interface (interface generated from protobuf definition file).
// It is a simple implementation for testing/demo only purposes.
type GreeterService struct{}

// SayHello returns error if request.name was not filled otherwise: "hello " + request.Name
func (*GreeterService) SayHello(ctx context.Context, request *helloworld.HelloRequest) (*helloworld.HelloReply, error) {
	if request.Name == "" {
		return nil, errors.New("not filled name in the request")
	}

	return &helloworld.HelloReply{Message: "hello " + request.Name}, nil
}
<|endoftext|>


<commit_before>package api

import (
	"testing"

	"github.com/NebulousLabs/Sia/build"
)

// TestVersion checks that /daemon/version is responding with the correct
// version.
func TestVersion(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	st, err := createServerTester("TestVersion")
	if err != nil {
		t.Fatal(err)
	}
	defer st.server.Close()
	var dv DaemonVersion
	st.getAPI("/daemon/version", &dv)
	if dv.Version != build.Version {
		t.Fatalf("/daemon/version reporting bad version: expected %v, got %v", build.Version, dv.Version)
	}
}

/*
// TODO: enable this test again once proper daemon shutdown is implemented (shutting down modules and listener separately).
// TestStop tests the /daemon/stop handler.
func TestStop(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	st, err := createServerTester("TestStop")
	if err != nil {
		t.Fatal(err)
	}
	var success struct{ Success bool }
	err = st.getAPI("/daemon/stop", &success)
	if err != nil {
		t.Fatal(err)
	}
	// Sleep to give time for server to close, as /daemon/stop will return success
	// before Server.Close() is called.
	time.Sleep(200 * time.Millisecond)
	err = st.getAPI("/daemon/stop", &success)
	if err == nil {
		t.Fatal("after /daemon/stop, subsequent calls should fail")
	}
}
*/
<commit_msg>test daemon API update with TestUpdate<commit_after>package api

import (
	"testing"

	"github.com/NebulousLabs/Sia/build"
)

// TestVersion checks that /daemon/version is responding with the correct
// version.
func TestVersion(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	st, err := createServerTester("TestVersion")
	if err != nil {
		t.Fatal(err)
	}
	defer st.server.Close()
	var dv DaemonVersion
	st.getAPI("/daemon/version", &dv)
	if dv.Version != build.Version {
		t.Fatalf("/daemon/version reporting bad version: expected %v, got %v", build.Version, dv.Version)
	}
}

// TestUpdate checks that /daemon/update correctly asserts that an update is
// not available for the daemon (since the test build is always up to date).
func TestUpdate(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	st, err := createServerTester("TestUpdate")
	if err != nil {
		t.Fatal(err)
	}
	defer st.server.Close()

	var update UpdateInfo
	if err = st.getAPI("/daemon/update", &update); err != nil {
		t.Fatal(err)
	}
	if update.Available && build.Version == update.Version {
		t.Fatal("daemon should not have an update available")
	}
}

/*
// TODO: enable this test again once proper daemon shutdown is implemented (shutting down modules and listener separately).
// TestStop tests the /daemon/stop handler.
func TestStop(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	st, err := createServerTester("TestStop")
	if err != nil {
		t.Fatal(err)
	}
	var success struct{ Success bool }
	err = st.getAPI("/daemon/stop", &success)
	if err != nil {
		t.Fatal(err)
	}
	// Sleep to give time for server to close, as /daemon/stop will return success
	// before Server.Close() is called.
	time.Sleep(200 * time.Millisecond)
	err = st.getAPI("/daemon/stop", &success)
	if err == nil {
		t.Fatal("after /daemon/stop, subsequent calls should fail")
	}
}
*/
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/tbruyelle/hipchat-go/hipchat"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
)

type HipChatEvent struct {
	Event string           `json:"event"`
	Item  HipChatEventItem `json:"item"`
}

type HipChatEventItem struct {
	Message HipChatEventMessage `json:"message"`
	Room    HipChatRoom         `json:"room"`
}

type HipChatEventMessage struct {
	Date          string
	File          HipChatFile `json:"file,omitempty"`
	From          interface{}
	Message       string
	Color         string
	Type          string `json:"type,omitempty"`
	Id            string
	MessageFormat string `json:"message_format"`
}

type HipChatUser struct {
	Id      int    `json:"id"`
	Mention string `json:"mention_name"`
	Name    string `json:"name"`
}

type HipChatFile struct {
	Name     string `json:"name"`
	Size     int    `json:"size"`
	ThumbUrl string `json:"thumb_url"`
	Url      string `json:"url"`
}

type HipChatRoom struct {
	Id   int    `json:"id"`
	Name string `json:"name"`
}

var (
	AccessToken = flag.String("token", "", "Access Token")
	RoomID      = flag.String("id", "", "ID of Chart Room")
	MsgColor    = flag.String("color", "green", "Color of Message")
)

func writeToFile(f *os.File, sourceRoom HipChatRoom, sourceMessage HipChatEventMessage) error {

	strFrom, ok := sourceMessage.From.(string)

	if !ok {
		str, err := json.Marshal(sourceMessage.From)
		if err != nil {
			return err
		}
		var user HipChatUser
		err = json.Unmarshal([]byte(str), &user)
		if err != nil {
			return err
		}
		strFrom = user.Name
	}

	Msg_Split := strings.Split(sourceMessage.Message, " ")

	var sendMsg string
	var buffer bytes.Buffer

	for value := 1; value < len(Msg_Split); value++ {
		buffer.WriteString(fmt.Sprintf("%s ", Msg_Split[value]))
	}
	SearchKey := buffer.String()

	if Msg_Split[0] == "/Search" {
		sendMsg = fmt.Sprintf("You want to SEARCH Keyword is %s", SearchKey)
	} else if Msg_Split[0] == "/Asset" {
		sendMsg = fmt.Sprintf("You want to SEARCH ID is %s", SearchKey)
	} else {
		sendMsg = fmt.Sprintf("Usage :
 /Search [Keyword]
 /Asset [Device ID]
 /Help : display help message")
	}

	fmt.Printf("[%s|%s] %s: %s\n", sourceMessage.Date, sourceRoom.Name, strFrom, sourceMessage.Message)
	send_Notify(*AccessToken, *RoomID, sendMsg, *MsgColor)

	msg := fmt.Sprintf("[%s|%s] %s: %s\n", sourceMessage.Date, sourceRoom.Name, strFrom, sourceMessage.Message)
	_, err := f.WriteString(msg)
	return err
}

type myReader struct {
	*bytes.Buffer
}

func handler(w http.ResponseWriter, r *http.Request, outFile *os.File) {
	var notifyEvent HipChatEvent

	//buf := new(bytes.Buffer)
	//buf.ReadFrom(r.Body)
	//s := buf.String()

	//fmt.Println(s)

	var bodyBytes []byte
	if r.Body != nil {
		bodyBytes, _ = ioutil.ReadAll(r.Body)
	}
	r.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
	bodyString := string(bodyBytes)

	fmt.Println("--- Original ---")
	fmt.Println(bodyString)
	fmt.Println("--- End ---")

	json.NewDecoder(r.Body).Decode(&notifyEvent)

	err := writeToFile(outFile, notifyEvent.Item.Room, notifyEvent.Item.Message)
	if err != nil {
		w.WriteHeader(200)
	} else {
		w.WriteHeader(500)
	}
}

func send_Notify(token string, id string, message string, color string) {

	c := hipchat.NewClient(token)

	notifRq := &hipchat.NotificationRequest{Message: message, Color: color}
	resp2, err := c.Room.Notification(id, notifRq)

	if err != nil {
		fmt.Printf("Error during room notification %q\n", err)
		fmt.Printf("Server returns %+v\n", resp2)
	}
}

func main() {
	flag.Parse()

	if *AccessToken == "" || *RoomID == "" {
		fmt.Println("Please Input Access Token ,Room ID")
		os.Exit(-1)
	} else {
		filePath := "hip.log"
		out, err := os.OpenFile(filePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
		if err != nil {
			log.Fatalf("can not open the log file: %s, err: %v", filePath, err)
		}
		defer out.Close()
		http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			handler(w, r, out)
		})
		http.ListenAndServe(":8088", nil)
	}
}
<commit_msg>current error INT to FLOAT<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/tbruyelle/hipchat-go/hipchat"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
)

type HipChatEvent struct {
	Event string           `json:"event"`
	Item  HipChatEventItem `json:"item"`
}

type HipChatEventItem struct {
	Message HipChatEventMessage `json:"message"`
	Room    HipChatRoom         `json:"room"`
}

type HipChatEventMessage struct {
	Date          string
	File          HipChatFile `json:"file,omitempty"`
	From          interface{}
	Message       string
	Color         string
	Type          string `json:"type,omitempty"`
	Id            string
	MessageFormat string `json:"message_format"`
}

type HipChatUser struct {
	Id      float64    `json:"id"`
	Mention string `json:"mention_name"`
	Name    string `json:"name"`
}

type HipChatFile struct {
	Name     string `json:"name"`
	Size     int    `json:"size"`
	ThumbUrl string `json:"thumb_url"`
	Url      string `json:"url"`
}

type HipChatRoom struct {
	Id   int    `json:"id"`
	Name string `json:"name"`
}

var (
	AccessToken = flag.String("token", "", "Access Token")
	RoomID      = flag.String("id", "", "ID of Chart Room")
	MsgColor    = flag.String("color", "green", "Color of Message")
)

func writeToFile(f *os.File, sourceRoom HipChatRoom, sourceMessage HipChatEventMessage) error {

	fmt.Println("--- Bgin writeToFile ---")
	strFrom, ok := sourceMessage.From.(string)

	if !ok {
		str, err := json.Marshal(sourceMessage.From)
		if err != nil {
                        log.Printf("Error: %v\n", err)
			return err
		}
		var user HipChatUser
		err = json.Unmarshal([]byte(str), &user)
		if err != nil {
                        log.Printf("Error: %v\n", err)
			return err
		}
		strFrom = user.Name
	}

	Msg_Split := strings.Split(sourceMessage.Message, " ")

	var sendMsg string
	var buffer bytes.Buffer

	for value := 1; value < len(Msg_Split); value++ {
		buffer.WriteString(fmt.Sprintf("%s ", Msg_Split[value]))
	}
	SearchKey := buffer.String()

	if Msg_Split[0] == "/Search" {
		sendMsg = fmt.Sprintf("You want to SEARCH Keyword is %s", SearchKey)
	} else if Msg_Split[0] == "/Asset" {
		sendMsg = fmt.Sprintf("You want to SEARCH ID is %s", SearchKey)
	} else {
		sendMsg = fmt.Sprintf("Usage :
 /Search [Keyword]
 /Asset [Device ID]
 /Help : display help message")
	}

	fmt.Printf("\n[%s|%s] %s: %s\n", sourceMessage.Date, sourceRoom.Name, strFrom, sourceMessage.Message)
	send_Notify(*AccessToken, *RoomID, sendMsg, *MsgColor)

	fmt.Println("--- End ---")

	msg := fmt.Sprintf("[%s|%s] %s: %s\n", sourceMessage.Date, sourceRoom.Name, strFrom, sourceMessage.Message)
	_, err := f.WriteString(msg)
	return err
}

type myReader struct {
	*bytes.Buffer
}

func handler(w http.ResponseWriter, r *http.Request, outFile *os.File) {
	var notifyEvent HipChatEvent

	//fmt.Println(s)

	var bodyBytes []byte
	if r.Body != nil {
		bodyBytes, _ = ioutil.ReadAll(r.Body)
	}
	r.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
	bodyString := string(bodyBytes)

	fmt.Println("\n\n--- Begin ---")
	fmt.Println(bodyString)

	json.NewDecoder(r.Body).Decode(&notifyEvent)

	err := writeToFile(outFile, notifyEvent.Item.Room, notifyEvent.Item.Message)
	if err != nil {
		log.Printf("Error: %v\n", err)
		w.WriteHeader(200)
	} else {
		w.WriteHeader(500)
	}
}

func send_Notify(token string, id string, message string, color string) {

	c := hipchat.NewClient(token)

	notifRq := &hipchat.NotificationRequest{Message: message, Color: color}
	resp2, err := c.Room.Notification(id, notifRq)

	if err != nil {
		fmt.Printf("Error during room notification %q\n", err)
		fmt.Printf("Server returns %+v\n", resp2)
	}
}

func main() {
	flag.Parse()

	if *AccessToken == "" || *RoomID == "" {
		fmt.Println("Please Input Access Token ,Room ID")
		os.Exit(-1)
	} else {
		filePath := "hip.log"
		out, err := os.OpenFile(filePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
		if err != nil {
			log.Fatalf("can not open the log file: %s, err: %v", filePath, err)
		}
		defer out.Close()
		http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			handler(w, r, out)
		})
		http.ListenAndServe(":8088", nil)
	}
}
<|endoftext|>


<commit_before>package kite

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"strings"
)

// Options is passed to kite.New when creating new instance.
type Options struct {
	// Mandatory fields
	Kitename    string
	Version     string
	Environment string
	Region      string

	// Optional fields
	PublicIP string // default: 0.0.0.0
	Port     string // default: random
	Path     string // default: /<kitename>

	// Do not authenticate incoming requests
	DisableAuthentication bool
}

// validate fields of the options struct. It exits if an error is occured.
func (o *Options) validate() {
	if o.Kitename == "" {
		log.Fatal("ERROR: options.Kitename field is not set")
	}

	if digits := strings.Split(o.Version, "."); len(digits) != 3 {
		log.Fatal("ERROR: please use 3-digits semantic versioning for options.version")
	}

	if o.Region == "" {
		log.Fatal("ERROR: options.Region field is not set")
	}

	if o.Environment == "" {
		log.Fatal("ERROR: options.Environment field is not set")
	}

	if o.PublicIP == "" {
		o.PublicIP = "0.0.0.0"
	}

	if o.Port == "" {
		o.Port = "0" // OS binds to an automatic port
	}

	if o.Path == "" {
		o.Path = "/" + o.Kitename
	}

	if o.Path[0] != '/' {
		o.Path = "/" + o.Path
	}
}

// Read options from a file.
func ReadKiteOptions(configfile string) (*Options, error) {
	file, err := ioutil.ReadFile(configfile)
	if err != nil {
		return nil, err
	}

	options := &Options{}
	return options, json.Unmarshal(file, &options)
}
<commit_msg>add kite prefix in front of kite urls<commit_after>package kite

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"strings"
)

// Options is passed to kite.New when creating new instance.
type Options struct {
	// Mandatory fields
	Kitename    string
	Version     string
	Environment string
	Region      string

	// Optional fields
	PublicIP string // default: 0.0.0.0
	Port     string // default: random
	Path     string // default: /<kitename>

	// Do not authenticate incoming requests
	DisableAuthentication bool
}

// validate fields of the options struct. It exits if an error is occured.
func (o *Options) validate() {
	if o.Kitename == "" {
		log.Fatal("ERROR: options.Kitename field is not set")
	}

	if digits := strings.Split(o.Version, "."); len(digits) != 3 {
		log.Fatal("ERROR: please use 3-digits semantic versioning for options.version")
	}

	if o.Region == "" {
		log.Fatal("ERROR: options.Region field is not set")
	}

	if o.Environment == "" {
		log.Fatal("ERROR: options.Environment field is not set")
	}

	if o.PublicIP == "" {
		o.PublicIP = "0.0.0.0"
	}

	if o.Port == "" {
		o.Port = "0" // OS binds to an automatic port
	}

	if o.Path == "" {
		o.Path = "/kite/" + o.Kitename
	}

	if o.Path[0] != '/' {
		o.Path = "/" + o.Path
	}
}

// Read options from a file.
func ReadKiteOptions(configfile string) (*Options, error) {
	file, err := ioutil.ReadFile(configfile)
	if err != nil {
		return nil, err
	}

	options := &Options{}
	return options, json.Unmarshal(file, &options)
}
<|endoftext|>


<commit_before>// (c) Copyright IBM Corp. 2021
// (c) Copyright Instana Inc. 2016

package instana

import (
	"os"
	"strconv"
)

// Options allows the user to configure the to-be-initialized sensor
type Options struct {
	Service                     string
	AgentHost                   string
	AgentPort                   int
	MaxBufferedSpans            int
	ForceTransmissionStartingAt int
	LogLevel                    int
	EnableAutoProfile           bool
	MaxBufferedProfiles         int
	IncludeProfilerFrames       bool
	Tracer                      TracerOptions

	disableW3CTraceCorrelation bool
}

// DefaultOptions returns the default set of options to configure Instana sensor.
// The service name is set to the name of current executable, the MaxBufferedSpans
// and ForceTransmissionStartingAt are set to instana.DefaultMaxBufferedSpans and
// instana.DefaultForceSpanSendAt correspondigly. The AgentHost and AgentPort are
// taken from the env INSTANA_AGENT_HOST and INSTANA_AGENT_PORT if set, and default
// to localhost and 42699 otherwise.
func DefaultOptions() *Options {
	opts := &Options{
		Tracer: DefaultTracerOptions(),
	}
	opts.setDefaults()

	return opts
}

func (opts *Options) setDefaults() {
	if opts.MaxBufferedSpans == 0 {
		opts.MaxBufferedSpans = DefaultMaxBufferedSpans
	}

	if opts.ForceTransmissionStartingAt == 0 {
		opts.ForceTransmissionStartingAt = DefaultForceSpanSendAt
	}

	if opts.AgentHost == "" {
		opts.AgentHost = agentDefaultHost

		if host := os.Getenv("INSTANA_AGENT_HOST"); host != "" {
			opts.AgentHost = host
		}
	}

	if opts.AgentPort == 0 {
		opts.AgentPort = agentDefaultPort

		if port, err := strconv.Atoi(os.Getenv("INSTANA_AGENT_PORT")); err == nil {
			opts.AgentPort = port
		}
	}

	secretsMatcher, err := parseInstanaSecrets(os.Getenv("INSTANA_SECRETS"))
	if err != nil {
		defaultLogger.Warn("invalid INSTANA_SECRETS= env variable value: ", err, ", ignoring")
		secretsMatcher = opts.Tracer.Secrets
	}

	if secretsMatcher == nil {
		secretsMatcher = DefaultSecretsMatcher()
	}

	opts.Tracer.Secrets = secretsMatcher

	if collectableHeaders, ok := os.LookupEnv("INSTANA_EXTRA_HTTP_HEADERS"); ok {
		opts.Tracer.CollectableHTTPHeaders = parseInstanaExtraHTTPHeaders(collectableHeaders)
	}

	opts.disableW3CTraceCorrelation = os.Getenv("INSTANA_DISABLE_W3C_TRACE_CORRELATION") != ""
}
<commit_msg>Document instana.Options fields<commit_after>// (c) Copyright IBM Corp. 2021
// (c) Copyright Instana Inc. 2016

package instana

import (
	"os"
	"strconv"
)

// Options allows the user to configure the to-be-initialized sensor
type Options struct {
	// Service is the global service name that will be used to identify the program in the Instana backend
	Service string
	// AgentHost is the Instana host agent host name
	//
	// Note: This setting has no effect in serverless environments. To specify the serverless acceptor endpoint,
	// INSTANA_ENDPOINT_URL env var. See https://www.instana.com/docs/reference/environment_variables/#serverless-monitoring
	// for more details.
	AgentHost string
	// AgentPort is the Instana host agent port
	//
	// Note: This setting has no effect in serverless environments. To specify the serverless acceptor endpoint,
	// INSTANA_ENDPOINT_URL env var. See https://www.instana.com/docs/reference/environment_variables/#serverless-monitoring
	// for more details.
	AgentPort int
	// MaxBufferedSpans is the maximum number of spans to buffer
	MaxBufferedSpans int
	// ForceTransmissionStartingAt is the number of spans to collect before flushing the buffer to the agent
	ForceTransmissionStartingAt int
	// LogLevel is the initial logging level for the logger used by Instana tracer. The valid log levels are
	// logger.{Error,Warn,Info,Debug}Level provided by the github.com/instana/go-sensor/logger package.
	//
	// Note: this setting is only used to initialize the default logger and has no effect if a custom logger is set via instana.SetLogger()
	LogLevel int
	// EnableAutoProfile enables automatic continuous process profiling when set to true
	EnableAutoProfile bool
	// MaxBufferedProfiles is the maximum number of profiles to buffer
	MaxBufferedProfiles int
	// IncludeProfilerFrames is whether to include profiler calls into the profile or not
	IncludeProfilerFrames bool
	// Tracer contains tracer-specific configuration used by all tracers
	Tracer TracerOptions

	disableW3CTraceCorrelation bool
}

// DefaultOptions returns the default set of options to configure Instana sensor.
// The service name is set to the name of current executable, the MaxBufferedSpans
// and ForceTransmissionStartingAt are set to instana.DefaultMaxBufferedSpans and
// instana.DefaultForceSpanSendAt correspondigly. The AgentHost and AgentPort are
// taken from the env INSTANA_AGENT_HOST and INSTANA_AGENT_PORT if set, and default
// to localhost and 42699 otherwise.
func DefaultOptions() *Options {
	opts := &Options{
		Tracer: DefaultTracerOptions(),
	}
	opts.setDefaults()

	return opts
}

func (opts *Options) setDefaults() {
	if opts.MaxBufferedSpans == 0 {
		opts.MaxBufferedSpans = DefaultMaxBufferedSpans
	}

	if opts.ForceTransmissionStartingAt == 0 {
		opts.ForceTransmissionStartingAt = DefaultForceSpanSendAt
	}

	if opts.AgentHost == "" {
		opts.AgentHost = agentDefaultHost

		if host := os.Getenv("INSTANA_AGENT_HOST"); host != "" {
			opts.AgentHost = host
		}
	}

	if opts.AgentPort == 0 {
		opts.AgentPort = agentDefaultPort

		if port, err := strconv.Atoi(os.Getenv("INSTANA_AGENT_PORT")); err == nil {
			opts.AgentPort = port
		}
	}

	secretsMatcher, err := parseInstanaSecrets(os.Getenv("INSTANA_SECRETS"))
	if err != nil {
		defaultLogger.Warn("invalid INSTANA_SECRETS= env variable value: ", err, ", ignoring")
		secretsMatcher = opts.Tracer.Secrets
	}

	if secretsMatcher == nil {
		secretsMatcher = DefaultSecretsMatcher()
	}

	opts.Tracer.Secrets = secretsMatcher

	if collectableHeaders, ok := os.LookupEnv("INSTANA_EXTRA_HTTP_HEADERS"); ok {
		opts.Tracer.CollectableHTTPHeaders = parseInstanaExtraHTTPHeaders(collectableHeaders)
	}

	opts.disableW3CTraceCorrelation = os.Getenv("INSTANA_DISABLE_W3C_TRACE_CORRELATION") != ""
}
<|endoftext|>


<commit_before>package be2bill

import (
	"fmt"
	"net/url"
	"sort"
)

const (
	HTMLOptionForm   = "FORM"
	HTMLOptionSubmit = "SUBMIT"

	Param3DSecure            = "3DSECURE"
	Param3DSecureDisplayMode = "3DSECUREDISPLAYMODE"
	ParamAmount              = "AMOUNT"
	ParamAmounts             = "AMOUNTS"
	ParamBillingAddress      = "BILLINGADDRESS"
	ParamBillingCountry      = "BILLINGCOUNTRY"
	ParamBillingFirstName    = "BILLINGFIRSTNAME"
	ParamBillingLastName     = "BILLINGLASTNAME"
	ParamBillingPhone        = "BILLINGPHONE"
	ParamBillingPostalCode   = "BILLINGPOSTALCODE"
	ParamCardCode            = "CARDCODE"
	ParamCardCVV             = "CARDCVV"
	ParamCardFullName        = "CARDFULLNAME"
	ParamCardValidityDate    = "CARDVALIDITYDATE"
	ParamClientAddress       = "CLIENTADDRESS"
	ParamClientDOB           = "CLIENTDOB"
	ParamClientEmail         = "CLIENTEMAIL"
	ParamClientIdent         = "CLIENTIDENT"
	ParamClientIP            = "CLIENTIP"
	ParamClientUserAgent     = "CLIENTUSERAGENT"
	ParamCreateAlias         = "CREATEALIAS"
	ParamDescription         = "DESCRIPTION"
	ParamDisplayCreateAlias  = "DISPLAYCREATEALIAS"
	ParamExtraData           = "EXTRADATA"
	ParamHash                = "HASH"
	ParamHideCardFullName    = "HIDECARDFULLNAME"
	ParamHideClientEmail     = "HIDECLIENTEMAIL"
	ParamIdentifier          = "IDENTIFIER"
	ParamLanguage            = "LANGUAGE"
	ParamOperationType       = "OPERATIONTYPE"
	ParamOrderID             = "ORDERID"
	ParamShipToAddress       = "SHIPTOADDRESS"
	ParamShipToFirstName     = "SHIPTOFIRSTNAME"
	ParamShipToLastName      = "SHIPTOLASTNAME"
	ParamShipToPhone         = "SHIPTOPHONE"
	ParamShipToPostalCode    = "SHIPTOPOSTALCODE"
	ParamSubmit              = "SUBMIT"
	ParamTransactionID       = "TRANSACTIONID"
	ParamVersion             = "VERSION"
	ParamVME                 = "VME"
)

type Options map[string]interface{}

func (p Options) sortedKeys() []string {
	keys := make([]string, len(p))
	i := 0
	for k := range p {
		keys[i] = k
		i++
	}
	sort.Strings(keys)
	return keys
}

func (p Options) copy() Options {
	c := make(Options)
	for k, v := range p {
		c[k] = v
	}
	return c
}

func (p Options) urlValues() url.Values {
	values := url.Values{}
	for k, v := range p {
		if opts, ok := v.(Options); ok {
			for subkey, subval := range opts {
				values.Set(fmt.Sprint(k, "[", subkey, "]"), fmt.Sprint(subval))
			}
		} else {
			values.Set(k, fmt.Sprint(v))
		}
	}

	return values
}

var DefaultOptions = make(Options)
<commit_msg>Add parameters.<commit_after>package be2bill

import (
	"fmt"
	"net/url"
	"sort"
)

const (
	HTMLOptionForm   = "FORM"
	HTMLOptionSubmit = "SUBMIT"

	Param3DSecure            = "3DSECURE"
	Param3DSecureDisplayMode = "3DSECUREDISPLAYMODE"
	ParamAlias               = "ALIAS"
	ParamAliasMode           = "ALIASMODE"
	ParamAmount              = "AMOUNT"
	ParamAmounts             = "AMOUNTS"
	ParamBillingAddress      = "BILLINGADDRESS"
	ParamBillingCountry      = "BILLINGCOUNTRY"
	ParamBillingFirstName    = "BILLINGFIRSTNAME"
	ParamBillingLastName     = "BILLINGLASTNAME"
	ParamBillingPhone        = "BILLINGPHONE"
	ParamBillingPostalCode   = "BILLINGPOSTALCODE"
	ParamCallbackURL         = "CALLBACKURL"
	ParamCardCode            = "CARDCODE"
	ParamCardCVV             = "CARDCVV"
	ParamCardFullName        = "CARDFULLNAME"
	ParamCardValidityDate    = "CARDVALIDITYDATE"
	ParamClientAddress       = "CLIENTADDRESS"
	ParamClientDOB           = "CLIENTDOB"
	ParamClientEmail         = "CLIENTEMAIL"
	ParamClientIdent         = "CLIENTIDENT"
	ParamClientIP            = "CLIENTIP"
	ParamClientReferrer      = "CLIENTREFERRER"
	ParamClientUserAgent     = "CLIENTUSERAGENT"
	ParamCompression         = "COMPRESSION"
	ParamCreateAlias         = "CREATEALIAS"
	ParamDate                = "DATE"
	ParamDay                 = "DAY"
	ParamDescription         = "DESCRIPTION"
	ParamDisplayCreateAlias  = "DISPLAYCREATEALIAS"
	ParamExtraData           = "EXTRADATA"
	ParamHash                = "HASH"
	ParamHideCardFullName    = "HIDECARDFULLNAME"
	ParamHideClientEmail     = "HIDECLIENTEMAIL"
	ParamIdentifier          = "IDENTIFIER"
	ParamLanguage            = "LANGUAGE"
	ParamMailTo              = "MAILTO"
	ParamMetadata            = "METADATA"
	ParamOperationType       = "OPERATIONTYPE"
	ParamOrderID             = "ORDERID"
	ParamScheduleID          = "SCHEDULEID"
	ParamShipToAddress       = "SHIPTOADDRESS"
	ParamShipToCountry       = "SHIPTOCOUNTRY"
	ParamShipToFirstName     = "SHIPTOFIRSTNAME"
	ParamShipToLastName      = "SHIPTOLASTNAME"
	ParamShipToPhone         = "SHIPTOPHONE"
	ParamShipToPostalCode    = "SHIPTOPOSTALCODE"
	ParamTimeZone            = "TIMEZONE"
	ParamTransactionID       = "TRANSACTIONID"
	ParamVersion             = "VERSION"
	ParamVME                 = "VME"
)

type Options map[string]interface{}

func (p Options) sortedKeys() []string {
	keys := make([]string, len(p))
	i := 0
	for k := range p {
		keys[i] = k
		i++
	}
	sort.Strings(keys)
	return keys
}

func (p Options) copy() Options {
	c := make(Options)
	for k, v := range p {
		c[k] = v
	}
	return c
}

func (p Options) urlValues() url.Values {
	values := url.Values{}
	for k, v := range p {
		if opts, ok := v.(Options); ok {
			for subkey, subval := range opts {
				values.Set(fmt.Sprint(k, "[", subkey, "]"), fmt.Sprint(subval))
			}
		} else {
			values.Set(k, fmt.Sprint(v))
		}
	}

	return values
}

var DefaultOptions = make(Options)
<|endoftext|>


<commit_before><commit_msg>Make file names unique.<commit_after><|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// +build linux

package bytes_test

import (
	. "bytes"
	"syscall"
	"testing"
	"unsafe"
)

// This file tests the situation where memeq is checking
// data very near to a page boundary.  We want to make sure
// equal does not read across the boundary and cause a page
// fault where it shouldn't.

// This test runs only on linux.  The code being tested is
// not OS-specific, so it does not need to be tested on all
// operating systems.

func TestEqualNearPageBoundary(t *testing.T) {
	pagesize := syscall.Getpagesize()
	b := make([]byte, 4*pagesize)
	i := pagesize
	for ; uintptr(unsafe.Pointer(&b[i]))%uintptr(pagesize) != 0; i++ {
	}
	syscall.Mprotect(b[i-pagesize:i], 0)
	syscall.Mprotect(b[i+pagesize:i+2*pagesize], 0)

	// both of these should fault
	//pagesize += int(b[i-1])
	//pagesize += int(b[i+pagesize])

	for j := 0; j < pagesize; j++ {
		b[i+j] = 'A'
	}
	for j := 0; j <= pagesize; j++ {
		Equal(b[i:i+j], b[i+pagesize-j:i+pagesize])
		Equal(b[i+pagesize-j:i+pagesize], b[i:i+j])
	}
}
<commit_msg>bytes: don't leave mprotect-ed pages after unsafe test.<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// +build linux

package bytes_test

import (
	. "bytes"
	"syscall"
	"testing"
	"unsafe"
)

// This file tests the situation where memeq is checking
// data very near to a page boundary.  We want to make sure
// equal does not read across the boundary and cause a page
// fault where it shouldn't.

// This test runs only on linux.  The code being tested is
// not OS-specific, so it does not need to be tested on all
// operating systems.

func TestEqualNearPageBoundary(t *testing.T) {
	pagesize := syscall.Getpagesize()
	b := make([]byte, 4*pagesize)
	i := pagesize
	for ; uintptr(unsafe.Pointer(&b[i]))%uintptr(pagesize) != 0; i++ {
	}
	syscall.Mprotect(b[i-pagesize:i], 0)
	syscall.Mprotect(b[i+pagesize:i+2*pagesize], 0)
	defer syscall.Mprotect(b[i-pagesize:i], syscall.PROT_READ|syscall.PROT_WRITE)
	defer syscall.Mprotect(b[i+pagesize:i+2*pagesize], syscall.PROT_READ|syscall.PROT_WRITE)

	// both of these should fault
	//pagesize += int(b[i-1])
	//pagesize += int(b[i+pagesize])

	for j := 0; j < pagesize; j++ {
		b[i+j] = 'A'
	}
	for j := 0; j <= pagesize; j++ {
		Equal(b[i:i+j], b[i+pagesize-j:i+pagesize])
		Equal(b[i+pagesize-j:i+pagesize], b[i:i+j])
	}
}
<|endoftext|>


<commit_before>package daemon

import (
	"fmt"
	"log"
	"os"
	"http"
	"flag"
)

type DaemonLogger struct {
	stdoutLog *log.Logger
	fileLog   *log.Logger
	error     int
}

var debug bool

func init() {
	flag.BoolVar(&debug, "d", false, "Log debug information")
}

func NewDaemonLogger(logFilePath string, daemonName string) *DaemonLogger {
	daemonLogger := new(DaemonLogger)
	hostname, _ := os.Hostname()
	pid := os.Getpid()
	header := fmt.Sprintf("%s %s[%d]: ", hostname, daemonName, pid)
	daemonLogger.stdoutLog = log.New(os.Stdout, header, log.LstdFlags)
	logFile, _ := os.OpenFile(logFilePath+daemonName+".log", os.O_WRONLY|os.O_CREATE, 0666)
	logFile.Seek(0, 2)
	daemonLogger.fileLog = log.New(logFile, header, log.LstdFlags)
	daemonLogger.error = 0
	return daemonLogger
}

func (dLog DaemonLogger) PrintAll(message string) {
	dLog.stdoutLog.Print(message)
	dLog.fileLog.Print(message)
}

func (dLog *DaemonLogger) Log(message string) {
	dLog.PrintAll(message)
}

func (dLog *DaemonLogger) LogError(message string, error os.Error) {
	if error != nil {
		dLog.PrintAll(message)
		dLog.error++
	}
}

func (dLog *DaemonLogger) LogHttp(request *http.Request) {
	msg := fmt.Sprintf("%s %s Bytes Recieved: %d", request.Method, request.RawURL, request.ContentLength)
	dLog.PrintAll(msg)
}

func (dLog *DaemonLogger) DebugHttp(request *http.Request) {
	if debug {
		msg := fmt.Sprintf("%s %s Bytes Recieved: %d", request.Method, request.RawURL, request.ContentLength)
		dLog.PrintAll(msg)
	}
}

func (dLog *DaemonLogger) LogDebug(message string) {
	if debug {
		dLog.PrintAll(message)
	}
}

func (daemonLogger *DaemonLogger) ReturnError() int {
	return daemonLogger.error
}
<commit_msg>log: handle permission errors gracefully<commit_after>package daemon

import (
	"fmt"
	"log"
	"os"
	"http"
	"flag"
)

type DaemonLogger struct {
	stdoutLog *log.Logger
	fileLog   *log.Logger
	error     int
}

var debug bool

func init() {
	flag.BoolVar(&debug, "d", false, "Log debug information")
}

func NewDaemonLogger(logFilePath string, daemonName string) *DaemonLogger {
	daemonLogger := new(DaemonLogger)
	hostname, _ := os.Hostname()
	pid := os.Getpid()
	header := fmt.Sprintf("%s %s[%d]: ", hostname, daemonName, pid)
	daemonLogger.stdoutLog = log.New(os.Stdout, header, log.LstdFlags)
	logFile, err := os.OpenFile(logFilePath+daemonName+".log", os.O_WRONLY|os.O_CREATE, 0666)
        if (err != nil) {
           fmt.Println("Failed to open logfile: ", err)
        } else {
	   logFile.Seek(0, 2)
	   daemonLogger.fileLog = log.New(logFile, header, log.LstdFlags)
        }
	daemonLogger.error = 0
	return daemonLogger
}

func (dLog DaemonLogger) PrintAll(message string) {
	dLog.stdoutLog.Print(message)
        if dLog.fileLog != nil {
	   dLog.fileLog.Print(message)
        }
}

func (dLog *DaemonLogger) Log(message string) {
	dLog.PrintAll(message)
}

func (dLog *DaemonLogger) LogError(message string, error os.Error) {
	if error != nil {
		dLog.PrintAll(message)
		dLog.error++
	}
}

func (dLog *DaemonLogger) LogHttp(request *http.Request) {
	msg := fmt.Sprintf("%s %s Bytes Recieved: %d", request.Method, request.RawURL, request.ContentLength)
	dLog.PrintAll(msg)
}

func (dLog *DaemonLogger) DebugHttp(request *http.Request) {
	if debug {
		msg := fmt.Sprintf("%s %s Bytes Recieved: %d", request.Method, request.RawURL, request.ContentLength)
		dLog.PrintAll(msg)
	}
}

func (dLog *DaemonLogger) LogDebug(message string) {
	if debug {
		dLog.PrintAll(message)
	}
}

func (daemonLogger *DaemonLogger) ReturnError() int {
	return daemonLogger.error
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements the 64-bit cyclic redundancy check, or CRC-64, checksum.
// See http://en.wikipedia.org/wiki/Cyclic_redundancy_check for information.
package crc64

import (
	"hash"
	"os"
)

// The size of a CRC-64 checksum in bytes.
const Size = 8

// Predefined polynomials.
const (
	// The ISO polynomial, defined in ISO 3309 and used in HDLC.
	ISO = 0xD800000000000000

	// The ECMA polynomial, defined in ECMA 182.
	ECMA = 0xC96C5795D7870F42
)

// Table is a 256-word table representing the polynomial for efficient processing.
type Table [256]uint64

// MakeTable returns the Table constructed from the specified polynomial.
func MakeTable(poly uint64) *Table {
	t := new(Table)
	for i := 0; i < 256; i++ {
		crc := uint64(i)
		for j := 0; j < 8; j++ {
			if crc&1 == 1 {
				crc = (crc >> 1) ^ poly
			} else {
				crc >>= 1
			}
		}
		t[i] = crc
	}
	return t
}

// digest represents the partial evaluation of a checksum.
type digest struct {
	crc uint64
	tab *Table
}

// New creates a new hash.Hash64 computing the CRC-64 checksum
// using the polynomial represented by the Table.
func New(tab *Table) hash.Hash64 { return &digest{0, tab} }

func (d *digest) Size() int { return Size }

func (d *digest) Reset() { d.crc = 0 }

func update(crc uint64, tab *Table, p []byte) uint64 {
	crc = ^crc
	for _, v := range p {
		crc = tab[byte(crc)^v] ^ (crc >> 8)
	}
	return ^crc
}

// Update returns the result of adding the bytes in p to the crc.
func Update(crc uint64, tab *Table, p []byte) uint64 {
	return update(crc, tab, p)
}

func (d *digest) Write(p []byte) (n int, err os.Error) {
	d.crc = update(d.crc, d.tab, p)
	return len(p), nil
}

func (d *digest) Sum64() uint64 { return d.crc }

func (d *digest) Sum() []byte {
	p := make([]byte, 8)
	s := d.Sum64()
	p[0] = byte(s >> 54)
	p[1] = byte(s >> 48)
	p[2] = byte(s >> 40)
	p[3] = byte(s >> 32)
	p[4] = byte(s >> 24)
	p[5] = byte(s >> 16)
	p[6] = byte(s >> 8)
	p[7] = byte(s)
	return p
}

// Checksum returns the CRC-64 checksum of data
// using the polynomial represented by the Table.
func Checksum(data []byte, tab *Table) uint64 { return update(0, tab, data) }
<commit_msg>hash/crc64: fix typo in Sum.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements the 64-bit cyclic redundancy check, or CRC-64, checksum.
// See http://en.wikipedia.org/wiki/Cyclic_redundancy_check for information.
package crc64

import (
	"hash"
	"os"
)

// The size of a CRC-64 checksum in bytes.
const Size = 8

// Predefined polynomials.
const (
	// The ISO polynomial, defined in ISO 3309 and used in HDLC.
	ISO = 0xD800000000000000

	// The ECMA polynomial, defined in ECMA 182.
	ECMA = 0xC96C5795D7870F42
)

// Table is a 256-word table representing the polynomial for efficient processing.
type Table [256]uint64

// MakeTable returns the Table constructed from the specified polynomial.
func MakeTable(poly uint64) *Table {
	t := new(Table)
	for i := 0; i < 256; i++ {
		crc := uint64(i)
		for j := 0; j < 8; j++ {
			if crc&1 == 1 {
				crc = (crc >> 1) ^ poly
			} else {
				crc >>= 1
			}
		}
		t[i] = crc
	}
	return t
}

// digest represents the partial evaluation of a checksum.
type digest struct {
	crc uint64
	tab *Table
}

// New creates a new hash.Hash64 computing the CRC-64 checksum
// using the polynomial represented by the Table.
func New(tab *Table) hash.Hash64 { return &digest{0, tab} }

func (d *digest) Size() int { return Size }

func (d *digest) Reset() { d.crc = 0 }

func update(crc uint64, tab *Table, p []byte) uint64 {
	crc = ^crc
	for _, v := range p {
		crc = tab[byte(crc)^v] ^ (crc >> 8)
	}
	return ^crc
}

// Update returns the result of adding the bytes in p to the crc.
func Update(crc uint64, tab *Table, p []byte) uint64 {
	return update(crc, tab, p)
}

func (d *digest) Write(p []byte) (n int, err os.Error) {
	d.crc = update(d.crc, d.tab, p)
	return len(p), nil
}

func (d *digest) Sum64() uint64 { return d.crc }

func (d *digest) Sum() []byte {
	p := make([]byte, 8)
	s := d.Sum64()
	p[0] = byte(s >> 56)
	p[1] = byte(s >> 48)
	p[2] = byte(s >> 40)
	p[3] = byte(s >> 32)
	p[4] = byte(s >> 24)
	p[5] = byte(s >> 16)
	p[6] = byte(s >> 8)
	p[7] = byte(s)
	return p
}

// Checksum returns the CRC-64 checksum of data
// using the polynomial represented by the Table.
func Checksum(data []byte, tab *Table) uint64 { return update(0, tab, data) }
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"errors"
	"os"
)

func query(filename, query string, bufSize int) (res []string, err error) {
	file, err := os.OpenFile(filename, os.O_RDWR, 0)
	if err != nil {
		return
	}
	defer file.Close()

	_, err = file.Seek(0, 0)
	if err != nil {
		return
	}
	_, err = file.WriteString(query)
	if err != nil {
		return
	}
	_, err = file.Seek(0, 0)
	if err != nil {
		return
	}
	buf := make([]byte, bufSize)
	for {
		n, _ := file.Read(buf)
		if n <= 0 {
			break
		}
		res = append(res, string(buf[:n]))
	}
	return
}

func queryCS(net, host, service string) (res []string, err error) {
	switch net {
	case "tcp4", "tcp6":
		net = "tcp"
	case "udp4", "udp6":
		net = "udp"
	}
	if host == "" {
		host = "*"
	}
	return query(netdir+"/cs", net+"!"+host+"!"+service, 128)
}

func queryCS1(net string, ip IP, port int) (clone, dest string, err error) {
	ips := "*"
	if len(ip) != 0 && !ip.IsUnspecified() {
		ips = ip.String()
	}
	lines, err := queryCS(net, ips, itoa(port))
	if err != nil {
		return
	}
	f := getFields(lines[0])
	if len(f) < 2 {
		return "", "", errors.New("net: bad response from ndb/cs")
	}
	clone, dest = f[0], f[1]
	return
}

func queryDNS(addr string, typ string) (res []string, err error) {
	return query(netdir+"/dns", addr+" "+typ, 1024)
}

// toLower returns a lower-case version of in. Restricting us to
// ASCII is sufficient to handle the IP protocol names and allow
// us to not depend on the strings and unicode packages.
func toLower(in string) string {
	for _, c := range in {
		if 'A' <= c && c <= 'Z' {
			// Has upper case; need to fix.
			out := []byte(in)
			for i := 0; i < len(in); i++ {
				c := in[i]
				if 'A' <= c && c <= 'Z' {
					c += 'a' - 'A'
				}
				out[i] = c
			}
			return string(out)
		}
	}
	return in
}

// lookupProtocol looks up IP protocol name and returns
// the corresponding protocol number.
func lookupProtocol(name string) (proto int, err error) {
	lines, err := query(netdir+"/cs", "!protocol="+toLower(name), 128)
	if err != nil {
		return 0, err
	}
	unknownProtoError := errors.New("unknown IP protocol specified: " + name)
	if len(lines) == 0 {
		return 0, unknownProtoError
	}
	f := getFields(lines[0])
	if len(f) < 2 {
		return 0, unknownProtoError
	}
	s := f[1]
	if n, _, ok := dtoi(s, byteIndex(s, '=')+1); ok {
		return n, nil
	}
	return 0, unknownProtoError
}

func lookupHost(host string) (addrs []string, err error) {
	// Use netdir/cs instead of netdir/dns because cs knows about
	// host names in local network (e.g. from /lib/ndb/local)
	lines, err := queryCS("net", host, "1")
	if err != nil {
		return
	}
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 2 {
			continue
		}
		addr := f[1]
		if i := byteIndex(addr, '!'); i >= 0 {
			addr = addr[:i] // remove port
		}
		if ParseIP(addr) == nil {
			continue
		}
		addrs = append(addrs, addr)
	}
	return
}

func lookupIP(host string) (ips []IP, err error) {
	addrs, err := LookupHost(host)
	if err != nil {
		return
	}
	for _, addr := range addrs {
		if ip := ParseIP(addr); ip != nil {
			ips = append(ips, ip)
		}
	}
	return
}

func lookupPort(network, service string) (port int, err error) {
	switch network {
	case "tcp4", "tcp6":
		network = "tcp"
	case "udp4", "udp6":
		network = "udp"
	}
	lines, err := queryCS(network, "127.0.0.1", service)
	if err != nil {
		return
	}
	unknownPortError := &AddrError{"unknown port", network + "/" + service}
	if len(lines) == 0 {
		return 0, unknownPortError
	}
	f := getFields(lines[0])
	if len(f) < 2 {
		return 0, unknownPortError
	}
	s := f[1]
	if i := byteIndex(s, '!'); i >= 0 {
		s = s[i+1:] // remove address
	}
	if n, _, ok := dtoi(s, 0); ok {
		return n, nil
	}
	return 0, unknownPortError
}

func lookupCNAME(name string) (cname string, err error) {
	lines, err := queryDNS(name, "cname")
	if err != nil {
		return
	}
	if len(lines) > 0 {
		if f := getFields(lines[0]); len(f) >= 3 {
			return f[2] + ".", nil
		}
	}
	return "", errors.New("net: bad response from ndb/dns")
}

func lookupSRV(service, proto, name string) (cname string, addrs []*SRV, err error) {
	var target string
	if service == "" && proto == "" {
		target = name
	} else {
		target = "_" + service + "._" + proto + "." + name
	}
	lines, err := queryDNS(target, "srv")
	if err != nil {
		return
	}
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 6 {
			continue
		}
		port, _, portOk := dtoi(f[4], 0)
		priority, _, priorityOk := dtoi(f[3], 0)
		weight, _, weightOk := dtoi(f[2], 0)
		if !(portOk && priorityOk && weightOk) {
			continue
		}
		addrs = append(addrs, &SRV{f[5], uint16(port), uint16(priority), uint16(weight)})
		cname = f[0]
	}
	byPriorityWeight(addrs).sort()
	return
}

func lookupMX(name string) (mx []*MX, err error) {
	lines, err := queryDNS(name, "mx")
	if err != nil {
		return
	}
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 4 {
			continue
		}
		if pref, _, ok := dtoi(f[2], 0); ok {
			mx = append(mx, &MX{f[3], uint16(pref)})
		}
	}
	byPref(mx).sort()
	return
}

func lookupNS(name string) (ns []*NS, err error) {
	lines, err := queryDNS(name, "ns")
	if err != nil {
		return
	}
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 3 {
			continue
		}
		ns = append(ns, &NS{f[2]})
	}
	return
}

func lookupTXT(name string) (txt []string, err error) {
	lines, err := queryDNS(name, "txt")
	if err != nil {
		return
	}
	for _, line := range lines {
		if i := byteIndex(line, '\t'); i >= 0 {
			txt = append(txt, line[i+1:])
		}
	}
	return
}

func lookupAddr(addr string) (name []string, err error) {
	arpa, err := reverseaddr(addr)
	if err != nil {
		return
	}
	lines, err := queryDNS(arpa, "ptr")
	if err != nil {
		return
	}
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 3 {
			continue
		}
		name = append(name, f[2])
	}
	return
}
<commit_msg>net: only return unique hosts during hostname lookup on plan 9<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"errors"
	"os"
)

func query(filename, query string, bufSize int) (res []string, err error) {
	file, err := os.OpenFile(filename, os.O_RDWR, 0)
	if err != nil {
		return
	}
	defer file.Close()

	_, err = file.Seek(0, 0)
	if err != nil {
		return
	}
	_, err = file.WriteString(query)
	if err != nil {
		return
	}
	_, err = file.Seek(0, 0)
	if err != nil {
		return
	}
	buf := make([]byte, bufSize)
	for {
		n, _ := file.Read(buf)
		if n <= 0 {
			break
		}
		res = append(res, string(buf[:n]))
	}
	return
}

func queryCS(net, host, service string) (res []string, err error) {
	switch net {
	case "tcp4", "tcp6":
		net = "tcp"
	case "udp4", "udp6":
		net = "udp"
	}
	if host == "" {
		host = "*"
	}
	return query(netdir+"/cs", net+"!"+host+"!"+service, 128)
}

func queryCS1(net string, ip IP, port int) (clone, dest string, err error) {
	ips := "*"
	if len(ip) != 0 && !ip.IsUnspecified() {
		ips = ip.String()
	}
	lines, err := queryCS(net, ips, itoa(port))
	if err != nil {
		return
	}
	f := getFields(lines[0])
	if len(f) < 2 {
		return "", "", errors.New("net: bad response from ndb/cs")
	}
	clone, dest = f[0], f[1]
	return
}

func queryDNS(addr string, typ string) (res []string, err error) {
	return query(netdir+"/dns", addr+" "+typ, 1024)
}

// toLower returns a lower-case version of in. Restricting us to
// ASCII is sufficient to handle the IP protocol names and allow
// us to not depend on the strings and unicode packages.
func toLower(in string) string {
	for _, c := range in {
		if 'A' <= c && c <= 'Z' {
			// Has upper case; need to fix.
			out := []byte(in)
			for i := 0; i < len(in); i++ {
				c := in[i]
				if 'A' <= c && c <= 'Z' {
					c += 'a' - 'A'
				}
				out[i] = c
			}
			return string(out)
		}
	}
	return in
}

// lookupProtocol looks up IP protocol name and returns
// the corresponding protocol number.
func lookupProtocol(name string) (proto int, err error) {
	lines, err := query(netdir+"/cs", "!protocol="+toLower(name), 128)
	if err != nil {
		return 0, err
	}
	unknownProtoError := errors.New("unknown IP protocol specified: " + name)
	if len(lines) == 0 {
		return 0, unknownProtoError
	}
	f := getFields(lines[0])
	if len(f) < 2 {
		return 0, unknownProtoError
	}
	s := f[1]
	if n, _, ok := dtoi(s, byteIndex(s, '=')+1); ok {
		return n, nil
	}
	return 0, unknownProtoError
}

func lookupHost(host string) (addrs []string, err error) {
	// Use netdir/cs instead of netdir/dns because cs knows about
	// host names in local network (e.g. from /lib/ndb/local)
	lines, err := queryCS("net", host, "1")
	if err != nil {
		return
	}
loop:
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 2 {
			continue
		}
		addr := f[1]
		if i := byteIndex(addr, '!'); i >= 0 {
			addr = addr[:i] // remove port
		}
		if ParseIP(addr) == nil {
			continue
		}
		// only return unique addresses
		for _, a := range addrs {
			if a == addr {
				continue loop
			}
		}
		addrs = append(addrs, addr)
	}
	return
}

func lookupIP(host string) (ips []IP, err error) {
	addrs, err := LookupHost(host)
	if err != nil {
		return
	}
	for _, addr := range addrs {
		if ip := ParseIP(addr); ip != nil {
			ips = append(ips, ip)
		}
	}
	return
}

func lookupPort(network, service string) (port int, err error) {
	switch network {
	case "tcp4", "tcp6":
		network = "tcp"
	case "udp4", "udp6":
		network = "udp"
	}
	lines, err := queryCS(network, "127.0.0.1", service)
	if err != nil {
		return
	}
	unknownPortError := &AddrError{"unknown port", network + "/" + service}
	if len(lines) == 0 {
		return 0, unknownPortError
	}
	f := getFields(lines[0])
	if len(f) < 2 {
		return 0, unknownPortError
	}
	s := f[1]
	if i := byteIndex(s, '!'); i >= 0 {
		s = s[i+1:] // remove address
	}
	if n, _, ok := dtoi(s, 0); ok {
		return n, nil
	}
	return 0, unknownPortError
}

func lookupCNAME(name string) (cname string, err error) {
	lines, err := queryDNS(name, "cname")
	if err != nil {
		return
	}
	if len(lines) > 0 {
		if f := getFields(lines[0]); len(f) >= 3 {
			return f[2] + ".", nil
		}
	}
	return "", errors.New("net: bad response from ndb/dns")
}

func lookupSRV(service, proto, name string) (cname string, addrs []*SRV, err error) {
	var target string
	if service == "" && proto == "" {
		target = name
	} else {
		target = "_" + service + "._" + proto + "." + name
	}
	lines, err := queryDNS(target, "srv")
	if err != nil {
		return
	}
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 6 {
			continue
		}
		port, _, portOk := dtoi(f[4], 0)
		priority, _, priorityOk := dtoi(f[3], 0)
		weight, _, weightOk := dtoi(f[2], 0)
		if !(portOk && priorityOk && weightOk) {
			continue
		}
		addrs = append(addrs, &SRV{f[5], uint16(port), uint16(priority), uint16(weight)})
		cname = f[0]
	}
	byPriorityWeight(addrs).sort()
	return
}

func lookupMX(name string) (mx []*MX, err error) {
	lines, err := queryDNS(name, "mx")
	if err != nil {
		return
	}
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 4 {
			continue
		}
		if pref, _, ok := dtoi(f[2], 0); ok {
			mx = append(mx, &MX{f[3], uint16(pref)})
		}
	}
	byPref(mx).sort()
	return
}

func lookupNS(name string) (ns []*NS, err error) {
	lines, err := queryDNS(name, "ns")
	if err != nil {
		return
	}
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 3 {
			continue
		}
		ns = append(ns, &NS{f[2]})
	}
	return
}

func lookupTXT(name string) (txt []string, err error) {
	lines, err := queryDNS(name, "txt")
	if err != nil {
		return
	}
	for _, line := range lines {
		if i := byteIndex(line, '\t'); i >= 0 {
			txt = append(txt, line[i+1:])
		}
	}
	return
}

func lookupAddr(addr string) (name []string, err error) {
	arpa, err := reverseaddr(addr)
	if err != nil {
		return
	}
	lines, err := queryDNS(arpa, "ptr")
	if err != nil {
		return
	}
	for _, line := range lines {
		f := getFields(line)
		if len(f) < 3 {
			continue
		}
		name = append(name, f[2])
	}
	return
}
<|endoftext|>


<commit_before>package s3

import (
	"bytes"
	"io"
	"path"
	"restic"
	"strings"
	"sync"

	"restic/backend"
	"restic/errors"

	"github.com/minio/minio-go"

	"restic/debug"
)

const connLimit = 10

// s3 is a backend which stores the data on an S3 endpoint.
type s3 struct {
	client       *minio.Client
	connChan     chan struct{}
	bucketname   string
	prefix       string
	cacheMutex   sync.RWMutex
	cacheObjSize map[string]int64
	backend.Layout
}

// Open opens the S3 backend at bucket and region. The bucket is created if it
// does not exist yet.
func Open(cfg Config) (restic.Backend, error) {
	debug.Log("open, config %#v", cfg)

	client, err := minio.New(cfg.Endpoint, cfg.KeyID, cfg.Secret, !cfg.UseHTTP)
	if err != nil {
		return nil, errors.Wrap(err, "minio.New")
	}

	be := &s3{
		client:       client,
		bucketname:   cfg.Bucket,
		prefix:       cfg.Prefix,
		cacheObjSize: make(map[string]int64),
		Layout:       &backend.S3Layout{Path: cfg.Prefix, Join: path.Join},
	}

	client.SetCustomTransport(backend.Transport())

	be.createConnections()

	found, err := client.BucketExists(cfg.Bucket)
	if err != nil {
		debug.Log("BucketExists(%v) returned err %v", cfg.Bucket, err)
		return nil, errors.Wrap(err, "client.BucketExists")
	}

	if !found {
		// create new bucket with default ACL in default region
		err = client.MakeBucket(cfg.Bucket, "")
		if err != nil {
			return nil, errors.Wrap(err, "client.MakeBucket")
		}
	}

	return be, nil
}

func (be *s3) createConnections() {
	be.connChan = make(chan struct{}, connLimit)
	for i := 0; i < connLimit; i++ {
		be.connChan <- struct{}{}
	}
}

// Location returns this backend's location (the bucket name).
func (be *s3) Location() string {
	return be.bucketname
}

// Save stores data in the backend at the handle.
func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) {
	if err := h.Valid(); err != nil {
		return err
	}

	objName := be.Filename(h)

	debug.Log("Save %v at %v", h, objName)

	// Check key does not already exist
	_, err = be.client.StatObject(be.bucketname, objName)
	if err == nil {
		debug.Log("%v already exists", h)
		return errors.New("key already exists")
	}

	<-be.connChan
	defer func() {
		be.connChan <- struct{}{}
	}()

	debug.Log("PutObject(%v, %v)",
		be.bucketname, objName)
	n, err := be.client.PutObject(be.bucketname, objName, rd, "binary/octet-stream")
	debug.Log("%v -> %v bytes, err %#v", objName, n, err)

	return errors.Wrap(err, "client.PutObject")
}

// wrapReader wraps an io.ReadCloser to run an additional function on Close.
type wrapReader struct {
	io.ReadCloser
	f func()
}

func (wr wrapReader) Close() error {
	err := wr.ReadCloser.Close()
	wr.f()
	return err
}

// Load returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
	debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h))
	if err := h.Valid(); err != nil {
		return nil, err
	}

	if offset < 0 {
		return nil, errors.New("offset is negative")
	}

	if length < 0 {
		return nil, errors.Errorf("invalid length %d", length)
	}

	var obj *minio.Object
	var size int64

	objName := be.Filename(h)

	// get token for connection
	<-be.connChan

	obj, err := be.client.GetObject(be.bucketname, objName)
	if err != nil {
		debug.Log("  err %v", err)

		// return token
		be.connChan <- struct{}{}

		return nil, errors.Wrap(err, "client.GetObject")
	}

	// if we're going to read the whole object, just pass it on.
	if length == 0 {
		debug.Log("Load %v: pass on object", h)

		_, err = obj.Seek(offset, 0)
		if err != nil {
			_ = obj.Close()

			// return token
			be.connChan <- struct{}{}

			return nil, errors.Wrap(err, "obj.Seek")
		}

		rd := wrapReader{
			ReadCloser: obj,
			f: func() {
				debug.Log("Close()")
				// return token
				be.connChan <- struct{}{}
			},
		}
		return rd, nil
	}

	defer func() {
		// return token
		be.connChan <- struct{}{}
	}()

	// otherwise use a buffer with ReadAt
	be.cacheMutex.RLock()
	size, cacheHit := be.cacheObjSize[objName]
	be.cacheMutex.RUnlock()

	if !cacheHit {
		info, err := obj.Stat()
		if err != nil {
			_ = obj.Close()
			return nil, errors.Wrap(err, "obj.Stat")
		}
		size = info.Size
		be.cacheMutex.Lock()
		be.cacheObjSize[objName] = size
		be.cacheMutex.Unlock()
	}

	if offset > size {
		_ = obj.Close()
		return nil, errors.New("offset larger than file size")
	}

	l := int64(length)
	if offset+l > size {
		l = size - offset
	}

	buf := make([]byte, l)
	n, err := obj.ReadAt(buf, offset)
	debug.Log("Load %v: use buffer with ReadAt: %v, %v", h, n, err)
	if err == io.EOF {
		debug.Log("Load %v: shorten buffer %v -> %v", h, len(buf), n)
		buf = buf[:n]
		err = nil
	}

	if err != nil {
		_ = obj.Close()
		return nil, errors.Wrap(err, "obj.ReadAt")
	}

	return backend.Closer{Reader: bytes.NewReader(buf)}, nil
}

// Stat returns information about a blob.
func (be *s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
	debug.Log("%v", h)

	objName := be.Filename(h)
	var obj *minio.Object

	obj, err = be.client.GetObject(be.bucketname, objName)
	if err != nil {
		debug.Log("GetObject() err %v", err)
		return restic.FileInfo{}, errors.Wrap(err, "client.GetObject")
	}

	// make sure that the object is closed properly.
	defer func() {
		e := obj.Close()
		if err == nil {
			err = errors.Wrap(e, "Close")
		}
	}()

	fi, err := obj.Stat()
	if err != nil {
		debug.Log("Stat() err %v", err)
		return restic.FileInfo{}, errors.Wrap(err, "Stat")
	}

	return restic.FileInfo{Size: fi.Size}, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (be *s3) Test(h restic.Handle) (bool, error) {
	found := false
	objName := be.Filename(h)
	_, err := be.client.StatObject(be.bucketname, objName)
	if err == nil {
		found = true
	}

	// If error, then not found
	return found, nil
}

// Remove removes the blob with the given name and type.
func (be *s3) Remove(h restic.Handle) error {
	objName := be.Filename(h)
	err := be.client.RemoveObject(be.bucketname, objName)
	debug.Log("Remove(%v) at %v -> err %v", h, objName, err)
	return errors.Wrap(err, "client.RemoveObject")
}

// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (be *s3) List(t restic.FileType, done <-chan struct{}) <-chan string {
	debug.Log("listing %v", t)
	ch := make(chan string)

	prefix := be.Dirname(restic.Handle{Type: t})

	listresp := be.client.ListObjects(be.bucketname, prefix, true, done)

	go func() {
		defer close(ch)
		for obj := range listresp {
			m := strings.TrimPrefix(obj.Key, prefix)
			if m == "" {
				continue
			}

			select {
			case ch <- m:
			case <-done:
				return
			}
		}
	}()

	return ch
}

// Remove keys for a specified backend type.
func (be *s3) removeKeys(t restic.FileType) error {
	done := make(chan struct{})
	defer close(done)
	for key := range be.List(restic.DataFile, done) {
		err := be.Remove(restic.Handle{Type: restic.DataFile, Name: key})
		if err != nil {
			return err
		}
	}

	return nil
}

// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *s3) Delete() error {
	alltypes := []restic.FileType{
		restic.DataFile,
		restic.KeyFile,
		restic.LockFile,
		restic.SnapshotFile,
		restic.IndexFile}

	for _, t := range alltypes {
		err := be.removeKeys(t)
		if err != nil {
			return nil
		}
	}

	return be.Remove(restic.Handle{Type: restic.ConfigFile})
}

// Close does nothing
func (be *s3) Close() error { return nil }
<commit_msg>s3: Use low-level API with a Range header for Load<commit_after>package s3

import (
	"fmt"
	"io"
	"path"
	"restic"
	"strings"
	"sync"

	"restic/backend"
	"restic/errors"

	"github.com/minio/minio-go"

	"restic/debug"
)

const connLimit = 10

// s3 is a backend which stores the data on an S3 endpoint.
type s3 struct {
	client       *minio.Client
	connChan     chan struct{}
	bucketname   string
	prefix       string
	cacheMutex   sync.RWMutex
	cacheObjSize map[string]int64
	backend.Layout
}

// Open opens the S3 backend at bucket and region. The bucket is created if it
// does not exist yet.
func Open(cfg Config) (restic.Backend, error) {
	debug.Log("open, config %#v", cfg)

	client, err := minio.New(cfg.Endpoint, cfg.KeyID, cfg.Secret, !cfg.UseHTTP)
	if err != nil {
		return nil, errors.Wrap(err, "minio.New")
	}

	be := &s3{
		client:       client,
		bucketname:   cfg.Bucket,
		prefix:       cfg.Prefix,
		cacheObjSize: make(map[string]int64),
		Layout:       &backend.S3Layout{Path: cfg.Prefix, Join: path.Join},
	}

	client.SetCustomTransport(backend.Transport())

	be.createConnections()

	found, err := client.BucketExists(cfg.Bucket)
	if err != nil {
		debug.Log("BucketExists(%v) returned err %v", cfg.Bucket, err)
		return nil, errors.Wrap(err, "client.BucketExists")
	}

	if !found {
		// create new bucket with default ACL in default region
		err = client.MakeBucket(cfg.Bucket, "")
		if err != nil {
			return nil, errors.Wrap(err, "client.MakeBucket")
		}
	}

	return be, nil
}

func (be *s3) createConnections() {
	be.connChan = make(chan struct{}, connLimit)
	for i := 0; i < connLimit; i++ {
		be.connChan <- struct{}{}
	}
}

// Location returns this backend's location (the bucket name).
func (be *s3) Location() string {
	return be.bucketname
}

// Save stores data in the backend at the handle.
func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) {
	if err := h.Valid(); err != nil {
		return err
	}

	objName := be.Filename(h)

	debug.Log("Save %v at %v", h, objName)

	// Check key does not already exist
	_, err = be.client.StatObject(be.bucketname, objName)
	if err == nil {
		debug.Log("%v already exists", h)
		return errors.New("key already exists")
	}

	<-be.connChan
	defer func() {
		be.connChan <- struct{}{}
	}()

	debug.Log("PutObject(%v, %v)",
		be.bucketname, objName)
	n, err := be.client.PutObject(be.bucketname, objName, rd, "binary/octet-stream")
	debug.Log("%v -> %v bytes, err %#v", objName, n, err)

	return errors.Wrap(err, "client.PutObject")
}

// wrapReader wraps an io.ReadCloser to run an additional function on Close.
type wrapReader struct {
	io.ReadCloser
	f func()
}

func (wr wrapReader) Close() error {
	err := wr.ReadCloser.Close()
	wr.f()
	return err
}

// Load returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
	debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h))
	if err := h.Valid(); err != nil {
		return nil, err
	}

	if offset < 0 {
		return nil, errors.New("offset is negative")
	}

	if length < 0 {
		return nil, errors.Errorf("invalid length %d", length)
	}

	objName := be.Filename(h)

	// get token for connection
	<-be.connChan

	byteRange := fmt.Sprintf("bytes=%d-", offset)
	if length > 0 {
		byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
	}
	headers := minio.NewGetReqHeaders()
	headers.Add("Range", byteRange)
	debug.Log("Load(%v) send range %v", h, byteRange)

	coreClient := minio.Core{be.client}
	rd, _, err := coreClient.GetObject(be.bucketname, objName, headers)

	// return token
	be.connChan <- struct{}{}

	return rd, err
}

// Stat returns information about a blob.
func (be *s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
	debug.Log("%v", h)

	objName := be.Filename(h)
	var obj *minio.Object

	obj, err = be.client.GetObject(be.bucketname, objName)
	if err != nil {
		debug.Log("GetObject() err %v", err)
		return restic.FileInfo{}, errors.Wrap(err, "client.GetObject")
	}

	// make sure that the object is closed properly.
	defer func() {
		e := obj.Close()
		if err == nil {
			err = errors.Wrap(e, "Close")
		}
	}()

	fi, err := obj.Stat()
	if err != nil {
		debug.Log("Stat() err %v", err)
		return restic.FileInfo{}, errors.Wrap(err, "Stat")
	}

	return restic.FileInfo{Size: fi.Size}, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (be *s3) Test(h restic.Handle) (bool, error) {
	found := false
	objName := be.Filename(h)
	_, err := be.client.StatObject(be.bucketname, objName)
	if err == nil {
		found = true
	}

	// If error, then not found
	return found, nil
}

// Remove removes the blob with the given name and type.
func (be *s3) Remove(h restic.Handle) error {
	objName := be.Filename(h)
	err := be.client.RemoveObject(be.bucketname, objName)
	debug.Log("Remove(%v) at %v -> err %v", h, objName, err)
	return errors.Wrap(err, "client.RemoveObject")
}

// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (be *s3) List(t restic.FileType, done <-chan struct{}) <-chan string {
	debug.Log("listing %v", t)
	ch := make(chan string)

	prefix := be.Dirname(restic.Handle{Type: t})

	listresp := be.client.ListObjects(be.bucketname, prefix, true, done)

	go func() {
		defer close(ch)
		for obj := range listresp {
			m := strings.TrimPrefix(obj.Key, prefix)
			if m == "" {
				continue
			}

			select {
			case ch <- m:
			case <-done:
				return
			}
		}
	}()

	return ch
}

// Remove keys for a specified backend type.
func (be *s3) removeKeys(t restic.FileType) error {
	done := make(chan struct{})
	defer close(done)
	for key := range be.List(restic.DataFile, done) {
		err := be.Remove(restic.Handle{Type: restic.DataFile, Name: key})
		if err != nil {
			return err
		}
	}

	return nil
}

// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *s3) Delete() error {
	alltypes := []restic.FileType{
		restic.DataFile,
		restic.KeyFile,
		restic.LockFile,
		restic.SnapshotFile,
		restic.IndexFile}

	for _, t := range alltypes {
		err := be.removeKeys(t)
		if err != nil {
			return nil
		}
	}

	return be.Remove(restic.Handle{Type: restic.ConfigFile})
}

// Close does nothing
func (be *s3) Close() error { return nil }
<|endoftext|>