<commit_before>package fox

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"util"
)

func getFileName(uuid string) string {
	return util.GetConfig().Storage.Filepath + uuid
}

// StoreFox persists the fox instance to somewhere
func StoreFox(fox Fox, uuid string) UUID {

	err := os.Chmod(getFileName(""), 0744)

	if err != nil {
		panic(err)
	}

	f, err := os.Create(getFileName(uuid))

	if err != nil {
		panic(err)
	}

	defer f.Close()

	// Make sure the file name matches the uuid in the structure
	fox.Uuid = uuid
	if err := json.NewEncoder(f).Encode(fox); err != nil {
		panic(err)
	}

	return UUID{Uuid: uuid}
}

func ReadFox(uuid string) (Fox, error) {
	var fox Fox
	data, err := ioutil.ReadFile(getFileName(uuid))

	if err != nil {
		return fox, err
	}

	if err := json.Unmarshal(data, &fox); err != nil {
		return fox, err
	}

	return fox, nil
}

func GetFoxes() ([]Fox, error) {
	var foxes []Fox

	foxes = make([]Fox, 0)
	fname := util.GetConfig().Storage.Filepath
	files, _ := ioutil.ReadDir(fname)
	for _, f := range files {
		fox, err := ReadFox(f.Name())
		if err != nil {
			return foxes, err
		}
		foxes = append(foxes, fox)
	}

	return foxes, nil
}

func FoxExists(uuid string) bool {
	if _, err := os.Stat(getFileName(uuid)); os.IsNotExist(err) {
		return false
	}
	return true
}

// Deletes the fox, if it exists. Does nothing if it does not
func DeleteFoxFromStorage(uuid string) {

	if !FoxExists(uuid) {
		// Exit quietly, if the fox is not there
		return
	}

	// Attempt to remove the file if it is there. Panic if it fails
	if err := os.Remove(getFileName(uuid)); err != nil {
		panic(err)
	}
}
<commit_msg>added mkdir for StoreFox(), since /tmp/foxdb/ does not seem to exist<commit_after>package fox

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"util"
)

func getFileName(uuid string) string {
	return util.GetConfig().Storage.Filepath + uuid
}

// StoreFox persists the fox instance to somewhere
func StoreFox(fox Fox, uuid string) UUID {

	err := os.MkdirAll(getFileName(""), 0744)

	if err != nil {
		panic(err)
	}

	err = os.Chmod(getFileName(""), 0744)

	if err != nil {
		panic(err)
	}

	f, err := os.Create(getFileName(uuid))

	if err != nil {
		panic(err)
	}

	defer f.Close()

	// Make sure the file name matches the uuid in the structure
	fox.Uuid = uuid
	if err := json.NewEncoder(f).Encode(fox); err != nil {
		panic(err)
	}

	return UUID{Uuid: uuid}
}

func ReadFox(uuid string) (Fox, error) {
	var fox Fox
	data, err := ioutil.ReadFile(getFileName(uuid))

	if err != nil {
		return fox, err
	}

	if err := json.Unmarshal(data, &fox); err != nil {
		return fox, err
	}

	return fox, nil
}

func GetFoxes() ([]Fox, error) {
	var foxes []Fox

	foxes = make([]Fox, 0)
	fname := util.GetConfig().Storage.Filepath
	files, _ := ioutil.ReadDir(fname)
	for _, f := range files {
		fox, err := ReadFox(f.Name())
		if err != nil {
			return foxes, err
		}
		foxes = append(foxes, fox)
	}

	return foxes, nil
}

func FoxExists(uuid string) bool {
	if _, err := os.Stat(getFileName(uuid)); os.IsNotExist(err) {
		return false
	}
	return true
}

// Deletes the fox, if it exists. Does nothing if it does not
func DeleteFoxFromStorage(uuid string) {

	if !FoxExists(uuid) {
		// Exit quietly, if the fox is not there
		return
	}

	// Attempt to remove the file if it is there. Panic if it fails
	if err := os.Remove(getFileName(uuid)); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package brokers

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"sync"

	"github.com/RichardKnop/machinery/Godeps/_workspace/src/github.com/streadway/amqp"
	"github.com/RichardKnop/machinery/v1/config"
	"github.com/RichardKnop/machinery/v1/signatures"
	"github.com/RichardKnop/machinery/v1/utils"
)

var once sync.Once
var conn *amqp.Connection

// AMQPBroker represents an AMQP broker
type AMQPBroker struct {
	config    *config.Config
	retryFunc func()
	stopChan  chan int
}

// NewAMQPBroker creates new AMQPBroker instance
func NewAMQPBroker(cnf *config.Config) Broker {
	return Broker(&AMQPBroker{
		config: cnf,
	})
}

// StartConsuming enters a loop and waits for incoming messages
func (amqpBroker *AMQPBroker) StartConsuming(consumerTag string, taskProcessor TaskProcessor) (bool, error) {
	if amqpBroker.retryFunc == nil {
		amqpBroker.retryFunc = utils.RetryClosure()
	}

	_, channel, queue, _, err := amqpBroker.open()
	if err != nil {
		amqpBroker.retryFunc()
		return true, err // retry true
	}

	amqpBroker.retryFunc = utils.RetryClosure()

	amqpBroker.stopChan = make(chan int)

	if err := channel.Qos(
		3,     // prefetch count
		0,     // prefetch size
		false, // global
	); err != nil {
		return false, fmt.Errorf("Channel Qos: %s", err)
	}

	deliveries, err := channel.Consume(
		queue.Name,  // queue
		consumerTag, // consumer tag
		false,       // auto-ack
		false,       // exclusive
		false,       // no-local
		false,       // no-wait
		nil,         // arguments
	)
	if err != nil {
		return false, fmt.Errorf("Queue Consume: %s", err)
	}

	log.Print("[*] Waiting for messages. To exit press CTRL+C")

	if err := amqpBroker.consume(deliveries, taskProcessor); err != nil {
		return true, err // retry true
	}

	return false, nil
}

// StopConsuming quits the loop
func (amqpBroker *AMQPBroker) StopConsuming() {
	// Notifying the stop channel stops consuming of messages
	amqpBroker.stopChan <- 1
}

// Publish places a new message on the default queue
func (amqpBroker *AMQPBroker) Publish(signature *signatures.TaskSignature) error {
	_, channel, _, confirmsChan, err := amqpBroker.open()
	defer channel.Close()
	if err != nil {
		return err
	}

	message, err := json.Marshal(signature)
	if err != nil {
		return fmt.Errorf("JSON Encode Message: %v", err)
	}

	signature.AdjustRoutingKey(
		amqpBroker.config.ExchangeType,
		amqpBroker.config.BindingKey,
		amqpBroker.config.DefaultQueue,
	)
	if err := channel.Publish(
		amqpBroker.config.Exchange, // exchange
		signature.RoutingKey,       // routing key
		false,                      // mandatory
		false,                      // immediate
		amqp.Publishing{
			ContentType:  "application/json",
			Body:         message,
			DeliveryMode: amqp.Persistent,
		},
	); err != nil {
		return err
	}

	confirmed := <-confirmsChan

	if confirmed.Ack {
		return nil
	}

	return fmt.Errorf("Failed delivery of delivery tag: %v", confirmed.DeliveryTag)
}

// Consume a single message
func (amqpBroker *AMQPBroker) consumeOne(d amqp.Delivery, taskProcessor TaskProcessor, errorsChan chan error) {
	if len(d.Body) == 0 {
		d.Nack(false, false)                                   // multiple, requeue
		errorsChan <- errors.New("Received an empty message.") // RabbitMQ down?
		return
	}

	log.Printf("Received new message: %s", d.Body)

	signature := signatures.TaskSignature{}
	if err := json.Unmarshal(d.Body, &signature); err != nil {
		d.Nack(false, false) // multiple, requeue
		errorsChan <- err
		return
	}

	d.Ack(false) // multiple

	if err := taskProcessor.Process(&signature); err != nil {
		errorsChan <- err
	}
}

// Consumes messages...
func (amqpBroker *AMQPBroker) consume(deliveries <-chan amqp.Delivery, taskProcessor TaskProcessor) error {
	errorsChan := make(chan error)
	for {
		select {
		case err := <-errorsChan:
			return err
		case d := <-deliveries:
			// Consume the task inside a gotourine so multiple tasks
			// can be processed concurrently
			go func() {
				amqpBroker.consumeOne(d, taskProcessor, errorsChan)
			}()
		case <-amqpBroker.stopChan:
			return nil
		}
	}
}

// Connects to the message queue, opens a channel, declares a queue
func (amqpBroker *AMQPBroker) connect() {

	var err error
	fmt.Println("connecting...")
	conn, err = amqp.Dial(amqpBroker.config.Broker)
	if err != nil {
		fmt.Printf("Dial: %s\n", err)
	}

}

// Connects to the message queue, opens a channel, declares a queue
func (amqpBroker *AMQPBroker) open() (*amqp.Connection, *amqp.Channel, amqp.Queue, <-chan amqp.Confirmation, error) {
	var err error
	var channel *amqp.Channel
	var queue amqp.Queue
	if conn == nil {
		connected := make(chan bool)
		go func() {
			once.Do(amqpBroker.connect)
			connected <- true
		}()
		<-connected
	}

	if conn == nil {
		return conn, channel, queue, nil, fmt.Errorf("Can't connect to the server")
	}

	channel, err = conn.Channel()
	if err != nil {
		fmt.Printf("Channel: %s\n", err)
	}
	if err := channel.ExchangeDeclare(
		amqpBroker.config.Exchange,     // name of the exchange
		amqpBroker.config.ExchangeType, // type
		true,  // durable
		false, // delete when complete
		false, // internal
		false, // noWait
		nil,   // arguments
	); err != nil {
		fmt.Printf("Exchange: %s\n", err)
	}

	queue, err = channel.QueueDeclare(
		amqpBroker.config.DefaultQueue, // name
		true,  // durable
		false, // delete when unused
		false, // exclusive
		false, // no-wait
		nil,   // arguments
	)
	if err != nil {
		fmt.Printf("Queue Declare: %s\n", err)
	}

	if err := channel.QueueBind(
		queue.Name,                   // name of the queue
		amqpBroker.config.BindingKey, // binding key
		amqpBroker.config.Exchange,   // source exchange
		false, // noWait
		nil,   // arguments
	); err != nil {
		fmt.Printf("Queue Bind: %s\n", err)
	}

	confirmsChan := make(chan amqp.Confirmation, 1)
	// Enable publish confirmations
	if err := channel.Confirm(false); err != nil {
		close(confirmsChan)
		fmt.Printf("Channel could not be put into confirm mode: %s\n", err)
	}

	return conn, channel, queue, channel.NotifyPublish(confirmsChan), nil
}
<commit_msg>close the channel after consume<commit_after>package brokers

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"sync"

	"github.com/RichardKnop/machinery/Godeps/_workspace/src/github.com/streadway/amqp"
	"github.com/RichardKnop/machinery/v1/config"
	"github.com/RichardKnop/machinery/v1/signatures"
	"github.com/RichardKnop/machinery/v1/utils"
)

var once sync.Once
var conn *amqp.Connection

// AMQPBroker represents an AMQP broker
type AMQPBroker struct {
	config    *config.Config
	retryFunc func()
	stopChan  chan int
}

// NewAMQPBroker creates new AMQPBroker instance
func NewAMQPBroker(cnf *config.Config) Broker {
	return Broker(&AMQPBroker{
		config: cnf,
	})
}

// StartConsuming enters a loop and waits for incoming messages
func (amqpBroker *AMQPBroker) StartConsuming(consumerTag string, taskProcessor TaskProcessor) (bool, error) {
	if amqpBroker.retryFunc == nil {
		amqpBroker.retryFunc = utils.RetryClosure()
	}

	_, channel, queue, _, err := amqpBroker.open()
	defer channel.Close()
	if err != nil {
		amqpBroker.retryFunc()
		return true, err // retry true
	}

	amqpBroker.retryFunc = utils.RetryClosure()

	amqpBroker.stopChan = make(chan int)

	if err := channel.Qos(
		3,     // prefetch count
		0,     // prefetch size
		false, // global
	); err != nil {
		return false, fmt.Errorf("Channel Qos: %s", err)
	}

	deliveries, err := channel.Consume(
		queue.Name,  // queue
		consumerTag, // consumer tag
		false,       // auto-ack
		false,       // exclusive
		false,       // no-local
		false,       // no-wait
		nil,         // arguments
	)
	if err != nil {
		return false, fmt.Errorf("Queue Consume: %s", err)
	}

	log.Print("[*] Waiting for messages. To exit press CTRL+C")

	if err := amqpBroker.consume(deliveries, taskProcessor); err != nil {
		return true, err // retry true
	}

	return false, nil
}

// StopConsuming quits the loop
func (amqpBroker *AMQPBroker) StopConsuming() {
	// Notifying the stop channel stops consuming of messages
	amqpBroker.stopChan <- 1
}

// Publish places a new message on the default queue
func (amqpBroker *AMQPBroker) Publish(signature *signatures.TaskSignature) error {
	_, channel, _, confirmsChan, err := amqpBroker.open()
	defer channel.Close()
	if err != nil {
		return err
	}

	message, err := json.Marshal(signature)
	if err != nil {
		return fmt.Errorf("JSON Encode Message: %v", err)
	}

	signature.AdjustRoutingKey(
		amqpBroker.config.ExchangeType,
		amqpBroker.config.BindingKey,
		amqpBroker.config.DefaultQueue,
	)
	if err := channel.Publish(
		amqpBroker.config.Exchange, // exchange
		signature.RoutingKey,       // routing key
		false,                      // mandatory
		false,                      // immediate
		amqp.Publishing{
			ContentType:  "application/json",
			Body:         message,
			DeliveryMode: amqp.Persistent,
		},
	); err != nil {
		return err
	}

	confirmed := <-confirmsChan

	if confirmed.Ack {
		return nil
	}

	return fmt.Errorf("Failed delivery of delivery tag: %v", confirmed.DeliveryTag)
}

// Consume a single message
func (amqpBroker *AMQPBroker) consumeOne(d amqp.Delivery, taskProcessor TaskProcessor, errorsChan chan error) {
	if len(d.Body) == 0 {
		d.Nack(false, false)                                   // multiple, requeue
		errorsChan <- errors.New("Received an empty message.") // RabbitMQ down?
		return
	}

	log.Printf("Received new message: %s", d.Body)

	signature := signatures.TaskSignature{}
	if err := json.Unmarshal(d.Body, &signature); err != nil {
		d.Nack(false, false) // multiple, requeue
		errorsChan <- err
		return
	}

	d.Ack(false) // multiple

	if err := taskProcessor.Process(&signature); err != nil {
		errorsChan <- err
	}
}

// Consumes messages...
func (amqpBroker *AMQPBroker) consume(deliveries <-chan amqp.Delivery, taskProcessor TaskProcessor) error {
	errorsChan := make(chan error)
	for {
		select {
		case err := <-errorsChan:
			return err
		case d := <-deliveries:
			// Consume the task inside a gotourine so multiple tasks
			// can be processed concurrently
			go func() {
				amqpBroker.consumeOne(d, taskProcessor, errorsChan)
			}()
		case <-amqpBroker.stopChan:
			return nil
		}
	}
}

// Connects to the message queue
func (amqpBroker *AMQPBroker) connect() {

	var err error
	fmt.Println("connecting...")
	conn, err = amqp.Dial(amqpBroker.config.Broker)
	if err != nil {
		fmt.Printf("Dial: %s\n", err)
	}

}

// Connects to the message queue, opens a channel, declares a queue
func (amqpBroker *AMQPBroker) open() (*amqp.Connection, *amqp.Channel, amqp.Queue, <-chan amqp.Confirmation, error) {
	var err error
	var channel *amqp.Channel
	var queue amqp.Queue
	if conn == nil {
		once.Do(amqpBroker.connect)
	}

	if conn == nil {
		return conn, channel, queue, nil, fmt.Errorf("Can't connect to the server")
	}

	channel, err = conn.Channel()
	if err != nil {
		fmt.Printf("Channel: %s\n", err)
	}
	if err := channel.ExchangeDeclare(
		amqpBroker.config.Exchange,     // name of the exchange
		amqpBroker.config.ExchangeType, // type
		true,  // durable
		false, // delete when complete
		false, // internal
		false, // noWait
		nil,   // arguments
	); err != nil {
		fmt.Printf("Exchange: %s\n", err)
	}

	queue, err = channel.QueueDeclare(
		amqpBroker.config.DefaultQueue, // name
		true,  // durable
		false, // delete when unused
		false, // exclusive
		false, // no-wait
		nil,   // arguments
	)
	if err != nil {
		fmt.Printf("Queue Declare: %s\n", err)
	}

	if err := channel.QueueBind(
		queue.Name,                   // name of the queue
		amqpBroker.config.BindingKey, // binding key
		amqpBroker.config.Exchange,   // source exchange
		false, // noWait
		nil,   // arguments
	); err != nil {
		fmt.Printf("Queue Bind: %s\n", err)
	}

	confirmsChan := make(chan amqp.Confirmation, 1)
	// Enable publish confirmations
	if err := channel.Confirm(false); err != nil {
		close(confirmsChan)
		fmt.Printf("Channel could not be put into confirm mode: %s\n", err)
	}

	return conn, channel, queue, channel.NotifyPublish(confirmsChan), nil
}
<|endoftext|>


<commit_before>package rest

import (
	"encoding/json"
	"fmt"
	"path"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/fundingloan"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/fundingtrade"
	"github.com/bitfinexcom/bitfinex-api-go/v2"
)

// KeepFundingRequest - data structure for constructing keep funding request payload
type KeepFundingRequest struct {
	Type string `json:"type"`
	ID   int    `json:"id"`
}

// FundingService manages the Funding endpoint.
type FundingService struct {
	requestFactory
	Synchronous
}

// Retreive all of the active fundign offers
// see https://docs.bitfinex.com/reference#rest-auth-funding-offers for more info
func (fs *FundingService) Offers(symbol string) (*bitfinex.FundingOfferSnapshot, error) {
	req, err := fs.requestFactory.NewAuthenticatedRequest(bitfinex.PermissionRead, path.Join("funding/offers", symbol))
	if err != nil {
		return nil, err
	}
	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}
	offers, err := bitfinex.NewFundingOfferSnapshotFromRaw(raw)
	if err != nil {
		return nil, err
	}
	return offers, nil
}

// Retreive all of the past in-active funding offers
// see https://docs.bitfinex.com/reference#rest-auth-funding-offers-hist for more info
func (fs *FundingService) OfferHistory(symbol string) (*bitfinex.FundingOfferSnapshot, error) {
	req, err := fs.requestFactory.NewAuthenticatedRequest(bitfinex.PermissionRead, path.Join("funding/offers", symbol, "hist"))
	if err != nil {
		return nil, err
	}
	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}
	offers, err := bitfinex.NewFundingOfferSnapshotFromRaw(raw)
	if err != nil {
		return nil, err
	}
	return offers, nil
}

// Retreive all of the active funding loans
// see https://docs.bitfinex.com/reference#rest-auth-funding-loans for more info
func (fs *FundingService) Loans(symbol string) (*fundingloan.Snapshot, error) {
	req, err := fs.requestFactory.NewAuthenticatedRequest(bitfinex.PermissionRead, path.Join("funding/loans", symbol))
	if err != nil {
		return nil, err
	}
	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}
	loans, err := fundingloan.SnapshotFromRaw(raw)
	if err != nil {
		return nil, err
	}
	return loans, nil
}

// Retreive all of the past in-active funding loans
// see https://docs.bitfinex.com/reference#rest-auth-funding-loans-hist for more info
func (fs *FundingService) LoansHistory(symbol string) (*fundingloan.Snapshot, error) {
	req, err := fs.requestFactory.NewAuthenticatedRequest(bitfinex.PermissionRead, path.Join("funding/loans", symbol, "hist"))
	if err != nil {
		return nil, err
	}
	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}
	loans, err := fundingloan.SnapshotFromRaw(raw)
	if err != nil {
		return nil, err
	}
	return loans, nil
}

// Retreive all of the active credits used in positions
// see https://docs.bitfinex.com/reference#rest-auth-funding-credits for more info
func (fs *FundingService) Credits(symbol string) (*bitfinex.FundingCreditSnapshot, error) {
	req, err := fs.requestFactory.NewAuthenticatedRequest(bitfinex.PermissionRead, path.Join("funding/credits", symbol))
	if err != nil {
		return nil, err
	}
	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}
	loans, err := bitfinex.NewFundingCreditSnapshotFromRaw(raw)
	if err != nil {
		return nil, err
	}
	return loans, nil
}

// Retreive all of the past in-active credits used in positions
// see https://docs.bitfinex.com/reference#rest-auth-funding-credits-hist for more info
func (fs *FundingService) CreditsHistory(symbol string) (*bitfinex.FundingCreditSnapshot, error) {
	req, err := fs.requestFactory.NewAuthenticatedRequest(bitfinex.PermissionRead, path.Join("funding/credits", symbol, "hist"))
	if err != nil {
		return nil, err
	}
	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}
	loans, err := bitfinex.NewFundingCreditSnapshotFromRaw(raw)
	if err != nil {
		return nil, err
	}
	return loans, nil
}

// Retreive all of the matched funding trades
// see https://docs.bitfinex.com/reference#rest-auth-funding-trades-hist for more info
func (fs *FundingService) Trades(symbol string) (*fundingtrade.Snapshot, error) {
	req, err := fs.requestFactory.NewAuthenticatedRequest(bitfinex.PermissionRead, path.Join("funding/trades", symbol, "hist"))
	if err != nil {
		return nil, err
	}

	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}

	fts, err := fundingtrade.SnapshotFromRaw(raw)
	if err != nil {
		return nil, err
	}

	return fts, nil
}

// Submits a request to create a new funding offer
// see https://docs.bitfinex.com/reference#submit-funding-offer for more info
func (fs *FundingService) SubmitOffer(fo *bitfinex.FundingOfferRequest) (*bitfinex.Notification, error) {
	bytes, err := fo.ToJSON()
	if err != nil {
		return nil, err
	}
	req, err := fs.requestFactory.NewAuthenticatedRequestWithBytes(bitfinex.PermissionWrite, path.Join("funding/offer/submit"), bytes)
	if err != nil {
		return nil, err
	}
	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}
	return bitfinex.NewNotificationFromRaw(raw)
}

// Submits a request to cancel the given offer
// see https://docs.bitfinex.com/reference#cancel-funding-offer for more info
func (fs *FundingService) CancelOffer(fc *bitfinex.FundingOfferCancelRequest) (*bitfinex.Notification, error) {
	bytes, err := fc.ToJSON()
	if err != nil {
		return nil, err
	}
	req, err := fs.requestFactory.NewAuthenticatedRequestWithBytes(bitfinex.PermissionWrite, "funding/offer/cancel", bytes)
	if err != nil {
		return nil, err
	}
	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}
	return bitfinex.NewNotificationFromRaw(raw)
}

// KeepFunding - toggle to keep funding taken. Specify loan for unused funding and credit for used funding.
// see https://docs.bitfinex.com/reference#rest-auth-keep-funding for more info
func (fs *FundingService) KeepFunding(args KeepFundingRequest) (*bitfinex.Notification, error) {
	if args.Type != "credit" && args.Type != "loan" {
		return nil, fmt.Errorf("Expected type: credit or loan, got: %s", args.Type)
	}

	bytes, err := json.Marshal(args)
	if err != nil {
		return nil, err
	}

	req, err := fs.requestFactory.NewAuthenticatedRequestWithBytes(
		bitfinex.PermissionWrite,
		path.Join("funding", "keep"),
		bytes,
	)
	if err != nil {
		return nil, err
	}

	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}

	return bitfinex.NewNotificationFromRaw(raw)
}
<commit_msg>v2/rest/funding.go putting new fundingoffer package to work<commit_after>package rest

import (
	"encoding/json"
	"fmt"
	"path"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/fundingloan"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/fundingoffer"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/fundingtrade"
	"github.com/bitfinexcom/bitfinex-api-go/v2"
)

// KeepFundingRequest - data structure for constructing keep funding request payload
type KeepFundingRequest struct {
	Type string `json:"type"`
	ID   int    `json:"id"`
}

// FundingService manages the Funding endpoint.
type FundingService struct {
	requestFactory
	Synchronous
}

// Retreive all of the active fundign offers
// see https://docs.bitfinex.com/reference#rest-auth-funding-offers for more info
func (fs *FundingService) Offers(symbol string) (*fundingoffer.Snapshot, error) {
	req, err := fs.requestFactory.NewAuthenticatedRequest(bitfinex.PermissionRead, path.Join("funding/offers", symbol))
	if err != nil {
		return nil, err
	}
	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}
	offers, err := fundingoffer.SnapshotFromRaw(raw)
	if err != nil {
		return nil, err
	}
	return offers, nil
}

// Retreive all of the past in-active funding offers
// see https://docs.bitfinex.com/reference#rest-auth-funding-offers-hist for more info
func (fs *FundingService) OfferHistory(symbol string) (*fundingoffer.Snapshot, error) {
	req, err := fs.requestFactory.NewAuthenticatedRequest(bitfinex.PermissionRead, path.Join("funding/offers", symbol, "hist"))
	if err != nil {
		return nil, err
	}
	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}
	offers, err := fundingoffer.SnapshotFromRaw(raw)
	if err != nil {
		return nil, err
	}
	return offers, nil
}

// Retreive all of the active funding loans
// see https://docs.bitfinex.com/reference#rest-auth-funding-loans for more info
func (fs *FundingService) Loans(symbol string) (*fundingloan.Snapshot, error) {
	req, err := fs.requestFactory.NewAuthenticatedRequest(bitfinex.PermissionRead, path.Join("funding/loans", symbol))
	if err != nil {
		return nil, err
	}
	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}
	loans, err := fundingloan.SnapshotFromRaw(raw)
	if err != nil {
		return nil, err
	}
	return loans, nil
}

// Retreive all of the past in-active funding loans
// see https://docs.bitfinex.com/reference#rest-auth-funding-loans-hist for more info
func (fs *FundingService) LoansHistory(symbol string) (*fundingloan.Snapshot, error) {
	req, err := fs.requestFactory.NewAuthenticatedRequest(bitfinex.PermissionRead, path.Join("funding/loans", symbol, "hist"))
	if err != nil {
		return nil, err
	}
	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}
	loans, err := fundingloan.SnapshotFromRaw(raw)
	if err != nil {
		return nil, err
	}
	return loans, nil
}

// Retreive all of the active credits used in positions
// see https://docs.bitfinex.com/reference#rest-auth-funding-credits for more info
func (fs *FundingService) Credits(symbol string) (*bitfinex.FundingCreditSnapshot, error) {
	req, err := fs.requestFactory.NewAuthenticatedRequest(bitfinex.PermissionRead, path.Join("funding/credits", symbol))
	if err != nil {
		return nil, err
	}
	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}
	loans, err := bitfinex.NewFundingCreditSnapshotFromRaw(raw)
	if err != nil {
		return nil, err
	}
	return loans, nil
}

// Retreive all of the past in-active credits used in positions
// see https://docs.bitfinex.com/reference#rest-auth-funding-credits-hist for more info
func (fs *FundingService) CreditsHistory(symbol string) (*bitfinex.FundingCreditSnapshot, error) {
	req, err := fs.requestFactory.NewAuthenticatedRequest(bitfinex.PermissionRead, path.Join("funding/credits", symbol, "hist"))
	if err != nil {
		return nil, err
	}
	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}
	loans, err := bitfinex.NewFundingCreditSnapshotFromRaw(raw)
	if err != nil {
		return nil, err
	}
	return loans, nil
}

// Retreive all of the matched funding trades
// see https://docs.bitfinex.com/reference#rest-auth-funding-trades-hist for more info
func (fs *FundingService) Trades(symbol string) (*fundingtrade.Snapshot, error) {
	req, err := fs.requestFactory.NewAuthenticatedRequest(bitfinex.PermissionRead, path.Join("funding/trades", symbol, "hist"))
	if err != nil {
		return nil, err
	}

	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}

	fts, err := fundingtrade.SnapshotFromRaw(raw)
	if err != nil {
		return nil, err
	}

	return fts, nil
}

// Submits a request to create a new funding offer
// see https://docs.bitfinex.com/reference#submit-funding-offer for more info
func (fs *FundingService) SubmitOffer(fo *fundingoffer.SubmitRequest) (*bitfinex.Notification, error) {
	bytes, err := fo.ToJSON()
	if err != nil {
		return nil, err
	}
	req, err := fs.requestFactory.NewAuthenticatedRequestWithBytes(bitfinex.PermissionWrite, path.Join("funding/offer/submit"), bytes)
	if err != nil {
		return nil, err
	}
	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}
	return bitfinex.NewNotificationFromRaw(raw)
}

// Submits a request to cancel the given offer
// see https://docs.bitfinex.com/reference#cancel-funding-offer for more info
func (fs *FundingService) CancelOffer(fc *fundingoffer.CancelRequest) (*bitfinex.Notification, error) {
	bytes, err := fc.ToJSON()
	if err != nil {
		return nil, err
	}
	req, err := fs.requestFactory.NewAuthenticatedRequestWithBytes(bitfinex.PermissionWrite, "funding/offer/cancel", bytes)
	if err != nil {
		return nil, err
	}
	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}
	return bitfinex.NewNotificationFromRaw(raw)
}

// KeepFunding - toggle to keep funding taken. Specify loan for unused funding and credit for used funding.
// see https://docs.bitfinex.com/reference#rest-auth-keep-funding for more info
func (fs *FundingService) KeepFunding(args KeepFundingRequest) (*bitfinex.Notification, error) {
	if args.Type != "credit" && args.Type != "loan" {
		return nil, fmt.Errorf("Expected type: credit or loan, got: %s", args.Type)
	}

	bytes, err := json.Marshal(args)
	if err != nil {
		return nil, err
	}

	req, err := fs.requestFactory.NewAuthenticatedRequestWithBytes(
		bitfinex.PermissionWrite,
		path.Join("funding", "keep"),
		bytes,
	)
	if err != nil {
		return nil, err
	}

	raw, err := fs.Request(req)
	if err != nil {
		return nil, err
	}

	return bitfinex.NewNotificationFromRaw(raw)
}
<|endoftext|>


<commit_before>package golib

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"reflect"
	"sync"
	"time"
)

// ========= Task interface
type StopChan <-chan error

// Semantics: Start() may only be called once, but Stop() should be idempotent.
// One error must be sent on StopChan upon stopping. The error can be nil.
type Task interface {
	Start(wg *sync.WaitGroup) StopChan
	Stop()
	String() string // Tasks are frequently printed
}

type NoopTask struct {
	Chan        chan error
	Description string
}

func (task *NoopTask) Start(*sync.WaitGroup) StopChan {
	return task.Chan
}
func (task *NoopTask) Stop() {
	task.Chan <- nil
}
func (task *NoopTask) String() string {
	return fmt.Sprintf("Task(%s)", task.Description)
}

type CleanupTask struct {
	Cleanup     func()
	Description string
	once        sync.Once
}

func (task *CleanupTask) Start(*sync.WaitGroup) StopChan {
	return nil
}
func (task *CleanupTask) Stop() {
	task.once.Do(func() {
		if cleanup := task.Cleanup; cleanup != nil {
			cleanup()
		}
	})
}
func (task *CleanupTask) String() string {
	return fmt.Sprintf("Cleanup(%s)", task.Description)
}

type LoopTask struct {
	*OneshotCondition
	loop        func(stop StopChan)
	Description string
}

func (task *LoopTask) Start(wg *sync.WaitGroup) StopChan {
	cond := task.OneshotCondition
	if loop := task.loop; loop != nil {
		stop := cond.Start(wg)
		if wg != nil {
			wg.Add(1)
		}
		go func() {
			if wg != nil {
				defer wg.Done()
			}
			for !cond.Enabled() {
				loop(stop)
			}
		}()
	}
	return cond.Start(wg)
}

func (task *LoopTask) String() string {
	return fmt.Sprintf("LoopTask(%s)", task.Description)
}

func NewLoopTask(description string, loop func(stop StopChan)) *LoopTask {
	return &LoopTask{
		OneshotCondition: NewOneshotCondition(),
		loop:             loop,
		Description:      description,
	}
}

// ========= Helpers to implement Task interface

func TaskFinished() StopChan {
	return TaskFinishedError(nil)
}

func TaskFinishedError(err error) StopChan {
	res := make(chan error, 1)
	res <- err
	return res
}

func WaitErrFunc(wg *sync.WaitGroup, wait func() error) StopChan {
	if wg != nil {
		wg.Add(1)
	}
	finished := make(chan error, 1)
	go func() {
		if wg != nil {
			defer wg.Done()
		}
		var err error
		if wait != nil {
			err = wait()
		}
		finished <- err
		close(finished)
	}()
	return finished
}

func WaitFunc(wg *sync.WaitGroup, wait func()) StopChan {
	return WaitErrFunc(wg, func() error {
		wait()
		return nil
	})
}

func WaitCondition(wg *sync.WaitGroup, cond *OneshotCondition) StopChan {
	if cond == nil {
		return nil
	}
	return WaitFunc(wg, func() {
		cond.Wait()
	})
}

func WaitForAny(channels []StopChan) (int, error) {
	if len(channels) < 1 {
		return -1, nil
	}
	// Use reflect package to wait for any of the given channels
	var cases []reflect.SelectCase
	for _, ch := range channels {
		if ch != nil {
			refCase := reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(ch)}
			cases = append(cases, refCase)
		}
	}
	choice, result, _ := reflect.Select(cases)
	channels[choice] = nil // Already received
	if err, ok := result.Interface().(error); ok {
		return choice, err
	} else {
		return choice, nil
	}
}

func WaitForAnyTask(wg *sync.WaitGroup, tasks []Task) (Task, error, []Task, []StopChan) {
	channels := make([]StopChan, 0, len(tasks))
	waitingTasks := make([]Task, 0, len(tasks))
	for _, task := range tasks {
		if channel := task.Start(wg); channel != nil {
			channels = append(channels, channel)
			waitingTasks = append(waitingTasks, task)
		}
	}
	choice, err := WaitForAny(channels)
	return waitingTasks[choice], err, waitingTasks, channels
}

func WaitForSetup(wg *sync.WaitGroup, setup func() error) StopChan {
	if wg != nil {
		wg.Add(1)
	}
	failed := make(chan error, 1)
	go func() {
		if wg != nil {
			defer wg.Done()
		}
		if setup != nil {
			if err := setup(); err != nil {
				failed <- err
				close(failed)
			}
		}
	}()
	return failed
}

// ========= Task Group

type TaskGroup struct {
	names  []string          // Track order of added new groups
	groups map[string][]Task // Groups will be stopped sequentially, but Tasks in one group in parallel
	all    []Task
}

func NewTaskGroup(tasks ...Task) *TaskGroup {
	group := &TaskGroup{
		groups: make(map[string][]Task),
	}
	for _, o := range tasks {
		group.Add(o)
	}
	return group
}

func (group *TaskGroup) Add(tasks ...Task) {
	group.AddNamed("default", tasks...)
}

func (group *TaskGroup) AddNamed(name string, tasks ...Task) {
	var list []Task
	if existingList, ok := group.groups[name]; ok {
		list = existingList
	} else {
		group.names = append(group.names, name)
	}
	for _, task := range tasks {
		if task != nil {
			group.all = append(group.all, tasks...)
			list = append(list, task)
		}
	}
	group.groups[name] = list
}

func (group *TaskGroup) WaitForAny(wg *sync.WaitGroup) (Task, error, []Task, []StopChan) {
	return WaitForAnyTask(wg, group.all)
}

func (group *TaskGroup) ReverseStop(printTasks bool) {
	for i := len(group.names) - 1; i >= 0; i-- {
		// Stop groups in reverse order
		var wg sync.WaitGroup
		tasks := group.groups[group.names[i]]
		for _, task := range tasks {
			// Stop tasks in one group in parallel
			wg.Add(1)
			go func(task Task) {
				defer wg.Done()
				if printTasks {
					log.Printf("Stopping %v\n", task)
				}
				task.Stop()
			}(task)
		}
		wg.Wait()
	}
}

func collectErrors(inputs []StopChan, tasks []Task, printWait bool) []error {
	result := make([]error, 0, len(inputs))
	for i, input := range inputs {
		if input != nil {
			if printWait {
				task := tasks[i]
				log.Printf("Waiting for %v\n", task)
			}
			if err := <-input; err != nil {
				result = append(result, err)
			}
		}
	}
	return result
}

func (group *TaskGroup) WaitAndStop(timeout time.Duration, printWait bool) (Task, []error) {
	var wg sync.WaitGroup
	choice, err, tasks, channels := group.WaitForAny(&wg)
	if timeout > 0 {
		time.AfterFunc(timeout, func() {
			panic("Waiting for stopping goroutines timed out")
		})
	}
	group.ReverseStop(printWait)
	wg.Wait()
	errors := collectErrors(channels, tasks, printWait)
	errors = append(errors, err)
	return choice, errors
}

var (
	DefaultTaskStopTimeout   = time.Duration(0)
	DefaultPrintTaskStopWait = false
)

func init() {
	flag.BoolVar(&DefaultPrintTaskStopWait, "task_stop_print", DefaultPrintTaskStopWait, "Print tasks waited for when stopping (for debugging)")
	flag.DurationVar(&DefaultTaskStopTimeout, "task_stop_timeout", DefaultTaskStopTimeout, "Timeout duration wen stopping and waiting for tasks to finish")
}

func (group *TaskGroup) WaitAndExit() {
	os.Exit(group.PrintWaitAndStop())
}

func (group *TaskGroup) PrintWaitAndStop() int {
	return group.TimeoutPrintWaitAndStop(DefaultTaskStopTimeout, DefaultPrintTaskStopWait)
}

func (group *TaskGroup) TimeoutPrintWaitAndStop(timeout time.Duration, printWait bool) (numErrors int) {
	reason, errors := group.WaitAndStop(timeout, printWait)
	log.Printf("Stopped because of %v\n", reason)
	for _, err := range errors {
		if err != nil {
			log.Println("Error:", err)
			numErrors++
		}
	}
	return
}

// ========= Sources of interrupts by the user

func ExternalInterrupt() chan error {
	// This must be done after starting any subprocess that depends
	// the ignore-handler for SIGNIT provided by ./noint
	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt)
	stop := make(chan error, 2)
	go func() {
		defer signal.Stop(interrupt)
		select {
		case <-interrupt:
		case <-stop:
		}
		stop <- nil
	}()
	return stop
}

func UserInput() chan error {
	userinput := make(chan error, 2)
	go func() {
		reader := bufio.NewReader(os.Stdin)
		_, err := reader.ReadString('\n')
		if err != nil {
			err = fmt.Errorf("Error reading user input: %v", err)
		}
		userinput <- err
	}()
	return userinput
}

func StdinClosed() chan error {
	closed := make(chan error, 2)
	go func() {
		_, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			err = fmt.Errorf("Error reading stdin: %v", err)
		}
		closed <- err
	}()
	return closed
}
<commit_msg>fixed typo<commit_after>package golib

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"reflect"
	"sync"
	"time"
)

// ========= Task interface
type StopChan <-chan error

// Semantics: Start() may only be called once, but Stop() should be idempotent.
// One error must be sent on StopChan upon stopping. The error can be nil.
type Task interface {
	Start(wg *sync.WaitGroup) StopChan
	Stop()
	String() string // Tasks are frequently printed
}

type NoopTask struct {
	Chan        chan error
	Description string
}

func (task *NoopTask) Start(*sync.WaitGroup) StopChan {
	return task.Chan
}
func (task *NoopTask) Stop() {
	task.Chan <- nil
}
func (task *NoopTask) String() string {
	return fmt.Sprintf("Task(%s)", task.Description)
}

type CleanupTask struct {
	Cleanup     func()
	Description string
	once        sync.Once
}

func (task *CleanupTask) Start(*sync.WaitGroup) StopChan {
	return nil
}
func (task *CleanupTask) Stop() {
	task.once.Do(func() {
		if cleanup := task.Cleanup; cleanup != nil {
			cleanup()
		}
	})
}
func (task *CleanupTask) String() string {
	return fmt.Sprintf("Cleanup(%s)", task.Description)
}

type LoopTask struct {
	*OneshotCondition
	loop        func(stop StopChan)
	Description string
}

func (task *LoopTask) Start(wg *sync.WaitGroup) StopChan {
	cond := task.OneshotCondition
	if loop := task.loop; loop != nil {
		stop := cond.Start(wg)
		if wg != nil {
			wg.Add(1)
		}
		go func() {
			if wg != nil {
				defer wg.Done()
			}
			for !cond.Enabled() {
				loop(stop)
			}
		}()
	}
	return cond.Start(wg)
}

func (task *LoopTask) String() string {
	return fmt.Sprintf("LoopTask(%s)", task.Description)
}

func NewLoopTask(description string, loop func(stop StopChan)) *LoopTask {
	return &LoopTask{
		OneshotCondition: NewOneshotCondition(),
		loop:             loop,
		Description:      description,
	}
}

// ========= Helpers to implement Task interface

func TaskFinished() StopChan {
	return TaskFinishedError(nil)
}

func TaskFinishedError(err error) StopChan {
	res := make(chan error, 1)
	res <- err
	return res
}

func WaitErrFunc(wg *sync.WaitGroup, wait func() error) StopChan {
	if wg != nil {
		wg.Add(1)
	}
	finished := make(chan error, 1)
	go func() {
		if wg != nil {
			defer wg.Done()
		}
		var err error
		if wait != nil {
			err = wait()
		}
		finished <- err
		close(finished)
	}()
	return finished
}

func WaitFunc(wg *sync.WaitGroup, wait func()) StopChan {
	return WaitErrFunc(wg, func() error {
		wait()
		return nil
	})
}

func WaitCondition(wg *sync.WaitGroup, cond *OneshotCondition) StopChan {
	if cond == nil {
		return nil
	}
	return WaitFunc(wg, func() {
		cond.Wait()
	})
}

func WaitForAny(channels []StopChan) (int, error) {
	if len(channels) < 1 {
		return -1, nil
	}
	// Use reflect package to wait for any of the given channels
	var cases []reflect.SelectCase
	for _, ch := range channels {
		if ch != nil {
			refCase := reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(ch)}
			cases = append(cases, refCase)
		}
	}
	choice, result, _ := reflect.Select(cases)
	channels[choice] = nil // Already received
	if err, ok := result.Interface().(error); ok {
		return choice, err
	} else {
		return choice, nil
	}
}

func WaitForAnyTask(wg *sync.WaitGroup, tasks []Task) (Task, error, []Task, []StopChan) {
	channels := make([]StopChan, 0, len(tasks))
	waitingTasks := make([]Task, 0, len(tasks))
	for _, task := range tasks {
		if channel := task.Start(wg); channel != nil {
			channels = append(channels, channel)
			waitingTasks = append(waitingTasks, task)
		}
	}
	choice, err := WaitForAny(channels)
	return waitingTasks[choice], err, waitingTasks, channels
}

func WaitForSetup(wg *sync.WaitGroup, setup func() error) StopChan {
	if wg != nil {
		wg.Add(1)
	}
	failed := make(chan error, 1)
	go func() {
		if wg != nil {
			defer wg.Done()
		}
		if setup != nil {
			if err := setup(); err != nil {
				failed <- err
				close(failed)
			}
		}
	}()
	return failed
}

// ========= Task Group

type TaskGroup struct {
	names  []string          // Track order of added new groups
	groups map[string][]Task // Groups will be stopped sequentially, but Tasks in one group in parallel
	all    []Task
}

func NewTaskGroup(tasks ...Task) *TaskGroup {
	group := &TaskGroup{
		groups: make(map[string][]Task),
	}
	for _, o := range tasks {
		group.Add(o)
	}
	return group
}

func (group *TaskGroup) Add(tasks ...Task) {
	group.AddNamed("default", tasks...)
}

func (group *TaskGroup) AddNamed(name string, tasks ...Task) {
	var list []Task
	if existingList, ok := group.groups[name]; ok {
		list = existingList
	} else {
		group.names = append(group.names, name)
	}
	for _, task := range tasks {
		if task != nil {
			group.all = append(group.all, tasks...)
			list = append(list, task)
		}
	}
	group.groups[name] = list
}

func (group *TaskGroup) WaitForAny(wg *sync.WaitGroup) (Task, error, []Task, []StopChan) {
	return WaitForAnyTask(wg, group.all)
}

func (group *TaskGroup) ReverseStop(printTasks bool) {
	for i := len(group.names) - 1; i >= 0; i-- {
		// Stop groups in reverse order
		var wg sync.WaitGroup
		tasks := group.groups[group.names[i]]
		for _, task := range tasks {
			// Stop tasks in one group in parallel
			wg.Add(1)
			go func(task Task) {
				defer wg.Done()
				if printTasks {
					log.Printf("Stopping %v\n", task)
				}
				task.Stop()
			}(task)
		}
		wg.Wait()
	}
}

func collectErrors(inputs []StopChan, tasks []Task, printWait bool) []error {
	result := make([]error, 0, len(inputs))
	for i, input := range inputs {
		if input != nil {
			if printWait {
				task := tasks[i]
				log.Printf("Waiting for %v\n", task)
			}
			if err := <-input; err != nil {
				result = append(result, err)
			}
		}
	}
	return result
}

func (group *TaskGroup) WaitAndStop(timeout time.Duration, printWait bool) (Task, []error) {
	var wg sync.WaitGroup
	choice, err, tasks, channels := group.WaitForAny(&wg)
	if timeout > 0 {
		time.AfterFunc(timeout, func() {
			panic("Waiting for stopping goroutines timed out")
		})
	}
	group.ReverseStop(printWait)
	wg.Wait()
	errors := collectErrors(channels, tasks, printWait)
	errors = append(errors, err)
	return choice, errors
}

var (
	DefaultTaskStopTimeout   = time.Duration(0)
	DefaultPrintTaskStopWait = false
)

func init() {
	flag.BoolVar(&DefaultPrintTaskStopWait, "task_stop_print", DefaultPrintTaskStopWait, "Print tasks waited for when stopping (for debugging)")
	flag.DurationVar(&DefaultTaskStopTimeout, "task_stop_timeout", DefaultTaskStopTimeout, "Timeout duration when stopping and waiting for tasks to finish")
}

func (group *TaskGroup) WaitAndExit() {
	os.Exit(group.PrintWaitAndStop())
}

func (group *TaskGroup) PrintWaitAndStop() int {
	return group.TimeoutPrintWaitAndStop(DefaultTaskStopTimeout, DefaultPrintTaskStopWait)
}

func (group *TaskGroup) TimeoutPrintWaitAndStop(timeout time.Duration, printWait bool) (numErrors int) {
	reason, errors := group.WaitAndStop(timeout, printWait)
	log.Printf("Stopped because of %v\n", reason)
	for _, err := range errors {
		if err != nil {
			log.Println("Error:", err)
			numErrors++
		}
	}
	return
}

// ========= Sources of interrupts by the user

func ExternalInterrupt() chan error {
	// This must be done after starting any subprocess that depends
	// the ignore-handler for SIGNIT provided by ./noint
	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt)
	stop := make(chan error, 2)
	go func() {
		defer signal.Stop(interrupt)
		select {
		case <-interrupt:
		case <-stop:
		}
		stop <- nil
	}()
	return stop
}

func UserInput() chan error {
	userinput := make(chan error, 2)
	go func() {
		reader := bufio.NewReader(os.Stdin)
		_, err := reader.ReadString('\n')
		if err != nil {
			err = fmt.Errorf("Error reading user input: %v", err)
		}
		userinput <- err
	}()
	return userinput
}

func StdinClosed() chan error {
	closed := make(chan error, 2)
	go func() {
		_, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			err = fmt.Errorf("Error reading stdin: %v", err)
		}
		closed <- err
	}()
	return closed
}
<|endoftext|>


<commit_before>/* Key-value data structure with non-blocking resize */
package godict

import (
	"fmt"
	mmh "murmur3"
	"sync"
)

const hash_seed uint32 = 6012
const perturb_shift = 5
const activeMul uint32 = 3
const sizeMul uint32 = 2

func GenHash(key string) uint32 {
	return mmh.MurMur3_32([]byte(key), hash_seed)
}

func New() *Dict {
	d := new(Dict)
	d.dict = make([]entry, 8, 8)
	d.mask = 7
	return d
}

type hashTable []entry

type Dict struct {
	sync.RWMutex
	active    uint32
	dict      hashTable
	sparedict hashTable
	mask      uint32 // mask = size - 1
	sparemask uint32
	rehashing bool
}

func (d *Dict) Active() uint32 {
	return d.active
}

//Set sets string value to key, spawn rehashing if needed
func (d *Dict) Set(key, value string) error {

	hash := GenHash(key)

	d.Lock()
	slot, err := d.lookUpEntry(key, hash)

	if err != nil {
		return err
	}

	slot.Init(key, value, hash)
	d.active++
	d.Unlock()

	go d.resizeIfNeeded()

	return nil
}

//Get retrieve slot from dict, spawn error if no key in dict
func (d *Dict) Get(key string) (slot *entry, err error) {
	hash := GenHash(key)

	d.RLock()
	defer d.RUnlock()
	slot, err = d.lookUpEntry(key, hash)

	if err == nil && slot.data == nil {
		err = fmt.Errorf("Key %v missing in the dictionary", key)
	}

	return
}

//Delete mark slot as deleted and wipe it`s data, spawn error if no key in dict
func (d *Dict) Delete(key string) error {
	hash := GenHash(key)

	d.Lock()
	defer d.Unlock()
	slot, err := d.lookUpEntry(key, hash)

	if err != nil {
		return err
	}

	if slot.data == nil {
		return fmt.Errorf("Key %v missing in the dictionary", key)
	}

	slot.delete()
	d.active--

	return nil
}

// Look for entry by key and hash in hashtable, returns pointer to entry
func (ht hashTable) findSlot(key string, hash, mask uint32) *entry {

	var freeSlot *entry

	index := hash & mask
	slot := &ht[index]

	if slot.deleted {
		freeSlot = slot
	} else {
		if slot.data == nil || slot.key == key {
			return slot
		}
	}

	for perturb := hash; ; perturb >>= perturb_shift {
		index = ((index << 2) + index + perturb + 1) & mask
		slot = &ht[index]

		if slot.deleted {
			if freeSlot == nil {
				freeSlot = slot
			}
			continue
		}

		if slot.data == nil {
			if freeSlot != nil {
				slot = freeSlot
			}
			return slot
		}

		if slot.key == key {
			return slot
		}
	}
}

func (d *Dict) lookUpEntry(key string, hash uint32) (*entry, error) {
	slot := d.dict.findSlot(key, hash, d.mask)

	if slot == nil {
		return nil, fmt.Errorf("Not slot found for key %v, hash %v", key, hash)
	}

	if slot.rehashed {
		slot = d.sparedict.findSlot(key, hash, d.sparemask)
	}

	return slot, nil
}

// rehash make incremental rehashing to sparedict
func (d *Dict) rehash(newsize uint32) {

	d.sparedict = make([]entry, newsize, newsize)
	d.sparemask = newsize - 1

	for _, e := range d.dict {
		d.Lock()
		if e.data != nil {
			slot := d.sparedict.findSlot(e.key, e.hash, d.sparemask)
			slot.data = e.data
		}
		e.rehashed = true
		d.Unlock()
	}
	d.Lock()
	defer d.Unlock()
	d.mask = d.sparemask
	d.dict = d.sparedict
	d.rehashing = false
	d.sparedict = nil
	d.sparemask = 0
}

// isReadyForResize atomically check if we can resize and set rehashing true
// in this case
//
// returns true if we must begin resize or false if we must not
func (d *Dict) isReadyForResize() bool {
	d.Lock()
	defer d.Unlock()
	if ((d.mask+1)*sizeMul >= d.active*activeMul) || d.rehashing {
		return false
	}
	d.rehashing = true
	return true
}

func (d *Dict) resizeIfNeeded() {

	if !d.isReadyForResize() {
		return
	}

	newsize := d.mask + 1
	active := d.active

	var mul uint32

	switch {
	case newsize < 50000:
		mul = 2
	default:
		mul = 4
	}

	for ; newsize <= mul*active; newsize <<= 1 {
	}
	d.rehash(newsize)
}
<commit_msg>Fix bug when keys not marks rehashed at all<commit_after>/* Key-value data structure with non-blocking resize */
package godict

import (
	"fmt"
	log "logging"
	mmh "murmur3"
	"sync"
)

const hash_seed uint32 = 6012
const perturb_shift = 5
const activeMul uint32 = 3
const sizeMul uint32 = 2

func GenHash(key string) uint32 {
	return mmh.MurMur3_32([]byte(key), hash_seed)
}

func New() *Dict {
	d := new(Dict)
	d.dict = make([]entry, 8, 8)
	d.mask = 7
	return d
}

type hashTable []entry

type Dict struct {
	sync.RWMutex
	active    uint32
	dict      hashTable
	sparedict hashTable
	mask      uint32 // mask = size - 1
	sparemask uint32
	rehashing bool
}

func (d *Dict) Active() uint32 {
	return d.active
}

//Set sets string value to key, spawn rehashing if needed
func (d *Dict) Set(key, value string) error {

	hash := GenHash(key)

	log.Debug("Rehashing status %v", d.rehashing)

	d.Lock()
	slot, err := d.lookUpEntry(key, hash)

	if err != nil {
		return err
	}

	slot.Init(key, value, hash)
	d.active++
	d.Unlock()

	go d.resizeIfNeeded()

	return nil
}

//Get retrieve slot from dict, spawn error if no key in dict
func (d *Dict) Get(key string) (slot *entry, err error) {
	hash := GenHash(key)

	d.RLock()
	defer d.RUnlock()
	slot, err = d.lookUpEntry(key, hash)

	if err == nil && slot.data == nil {
		err = fmt.Errorf("Key %v missing in the dictionary", key)
	}

	return
}

//Delete mark slot as deleted and wipe it`s data, spawn error if no key in dict
func (d *Dict) Delete(key string) error {
	hash := GenHash(key)

	d.Lock()
	defer d.Unlock()
	slot, err := d.lookUpEntry(key, hash)

	if err != nil {
		return err
	}

	if slot.data == nil {
		return fmt.Errorf("Key %v missing in the dictionary", key)
	}

	slot.delete()
	d.active--

	return nil
}

// Look for entry by key and hash in hashtable, returns pointer to entry
func (ht hashTable) findSlot(key string, hash, mask uint32) *entry {

	var freeSlot *entry

	index := hash & mask
	slot := &ht[index]

	if slot.deleted {
		freeSlot = slot
	} else {
		if slot.data == nil || slot.key == key {
			return slot
		}
	}

	for perturb := hash; ; perturb >>= perturb_shift {
		index = ((index << 2) + index + perturb + 1) & mask
		slot = &ht[index]

		if slot.deleted {
			if freeSlot == nil {
				freeSlot = slot
			}
			continue
		}

		if slot.data == nil {
			if freeSlot != nil {
				slot = freeSlot
			}
			return slot
		}

		if slot.key == key {
			return slot
		}
	}
}

func (d *Dict) lookUpEntry(key string, hash uint32) (*entry, error) {
	slot := d.dict.findSlot(key, hash, d.mask)

	if slot == nil {
		return nil, fmt.Errorf("Not slot found for key %v, hash %v", key, hash)
	}

	if slot.rehashed {
		log.Debug("Slot for %q found, but it`s rehashed", key)
		slot = d.sparedict.findSlot(key, hash, d.sparemask)
	}

	return slot, nil
}

// rehash make incremental rehashing to sparedict
func (d *Dict) rehash(newsize uint32) {
	log.Debug("Rehashing started")
	defer log.Debug("Rehashing finished")

	d.sparedict = make([]entry, newsize, newsize)
	d.sparemask = newsize - 1

	for i := range d.dict {
		d.Lock()
		e := &d.dict[i]
		if e.data != nil {
			log.Debug("Rehashing key %q", e.key)
			slot := d.sparedict.findSlot(e.key, e.hash, d.sparemask)
			slot.data = e.data
		}
		e.rehashed = true
		d.Unlock()
	}
	d.Lock()
	defer d.Unlock()
	d.mask = d.sparemask
	d.dict = d.sparedict
	d.rehashing = false
	d.sparedict = nil
	d.sparemask = 0
}

// isReadyForResize atomically check if we can resize and set rehashing true
// in this case
//
// returns true if we must begin resize or false if we must not
func (d *Dict) isReadyForResize() bool {
	d.Lock()
	defer d.Unlock()
	if ((d.mask+1)*sizeMul >= d.active*activeMul) || d.rehashing {
		return false
	}
	d.rehashing = true
	return true
}

func (d *Dict) resizeIfNeeded() {

	if !d.isReadyForResize() {
		return
	}

	newsize := d.mask + 1
	active := d.active

	var mul uint32

	switch {
	case newsize < 50000:
		mul = 2
	default:
		mul = 4
	}

	for ; newsize <= mul*active; newsize <<= 1 {
	}
	d.rehash(newsize)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package log implements a simple logging package. It defines a type, Logger,
// with methods for formatting output. It also has a predefined 'standard'
// Logger accessible through helper functions Print[f|ln], Fatal[f|ln], and
// Panic[f|ln], which are easier to use than creating a Logger manually.
// That logger writes to standard error and prints the date and time
// of each logged message.
// Every log message is output on a separate line: if the message being
// printed does not end in a newline, the logger will add one.
// The Fatal functions call os.Exit(1) after writing the log message.
// The Panic functions call panic after writing the log message.
package log

import (
	"fmt"
	"io"
	"os"
	"runtime"
	"sync"
	"time"
)

// These flags define which text to prefix to each log entry generated by the Logger.
const (
	// Bits or'ed together to control what's printed.
	// There is no control over the order they appear (the order listed
	// here) or the format they present (as described in the comments).
	// The prefix is followed by a colon only when Llongfile or Lshortfile
	// is specified.
	// For example, flags Ldate | Ltime (or LstdFlags) produce,
	//	2009/01/23 01:23:23 message
	// while flags Ldate | Ltime | Lmicroseconds | Llongfile produce,
	//	2009/01/23 01:23:23.123123 /a/b/c/d.go:23: message
	Ldate         = 1 << iota     // the date in the local time zone: 2009/01/23
	Ltime                         // the time in the local time zone: 01:23:23
	Lmicroseconds                 // microsecond resolution: 01:23:23.123123.  assumes Ltime.
	Llongfile                     // full file name and line number: /a/b/c/d.go:23
	Lshortfile                    // final file name element and line number: d.go:23. overrides Llongfile
	LUTC                          // if Ldate or Ltime is set, use UTC rather than the local time zone
	LstdFlags     = Ldate | Ltime // initial values for the standard logger
)

// A Logger represents an active logging object that generates lines of
// output to an io.Writer. Each logging operation makes a single call to
// the Writer's Write method. A Logger can be used simultaneously from
// multiple goroutines; it guarantees to serialize access to the Writer.
type Logger struct {
	mu     sync.Mutex // ensures atomic writes; protects the following fields
	prefix string     // prefix to write at beginning of each line
	flag   int        // properties
	out    io.Writer  // destination for output
	buf    []byte     // for accumulating text to write
}

// New creates a new Logger. The out variable sets the
// destination to which log data will be written.
// The prefix appears at the beginning of each generated log line.
// The flag argument defines the logging properties.
func New(out io.Writer, prefix string, flag int) *Logger {
	return &Logger{out: out, prefix: prefix, flag: flag}
}

// SetOutput sets the output destination for the logger.
func (l *Logger) SetOutput(w io.Writer) {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.out = w
}

var std = New(os.Stderr, "", LstdFlags)

// Cheap integer to fixed-width decimal ASCII. Give a negative width to avoid zero-padding.
func itoa(buf *[]byte, i int, wid int) {
	// Assemble decimal in reverse order.
	var b [20]byte
	bp := len(b) - 1
	for i >= 10 || wid > 1 {
		wid--
		q := i / 10
		b[bp] = byte('0' + i - q*10)
		bp--
		i = q
	}
	// i < 10
	b[bp] = byte('0' + i)
	*buf = append(*buf, b[bp:]...)
}

// formatHeader writes log header to buf in following order:
//   * l.prefix (if it's not blank),
//   * date and/or time (if corresponding flags are provided),
//   * file and line number (if corresponding flags are provided).
func (l *Logger) formatHeader(buf *[]byte, t time.Time, file string, line int) {
	*buf = append(*buf, l.prefix...)
	if l.flag&(Ldate|Ltime|Lmicroseconds) != 0 {
		if l.flag&LUTC != 0 {
			t = t.UTC()
		}
		if l.flag&Ldate != 0 {
			year, month, day := t.Date()
			itoa(buf, year, 4)
			*buf = append(*buf, '/')
			itoa(buf, int(month), 2)
			*buf = append(*buf, '/')
			itoa(buf, day, 2)
			*buf = append(*buf, ' ')
		}
		if l.flag&(Ltime|Lmicroseconds) != 0 {
			hour, min, sec := t.Clock()
			itoa(buf, hour, 2)
			*buf = append(*buf, ':')
			itoa(buf, min, 2)
			*buf = append(*buf, ':')
			itoa(buf, sec, 2)
			if l.flag&Lmicroseconds != 0 {
				*buf = append(*buf, '.')
				itoa(buf, t.Nanosecond()/1e3, 6)
			}
			*buf = append(*buf, ' ')
		}
	}
	if l.flag&(Lshortfile|Llongfile) != 0 {
		if l.flag&Lshortfile != 0 {
			short := file
			for i := len(file) - 1; i > 0; i-- {
				if file[i] == '/' {
					short = file[i+1:]
					break
				}
			}
			file = short
		}
		*buf = append(*buf, file...)
		*buf = append(*buf, ':')
		itoa(buf, line, -1)
		*buf = append(*buf, ": "...)
	}
}

// Output writes the output for a logging event. The string s contains
// the text to print after the prefix specified by the flags of the
// Logger. A newline is appended if the last character of s is not
// already a newline. Calldepth is used to recover the PC and is
// provided for generality, although at the moment on all pre-defined
// paths it will be 2.
func (l *Logger) Output(calldepth int, s string) error {
	// Get time early if we need it.
	var now time.Time
	if l.flag&(Ldate|Ltime|Lmicroseconds) != 0 {
		now = time.Now()
	}
	var file string
	var line int
	l.mu.Lock()
	defer l.mu.Unlock()
	if l.flag&(Lshortfile|Llongfile) != 0 {
		// Release lock while getting caller info - it's expensive.
		l.mu.Unlock()
		var ok bool
		_, file, line, ok = runtime.Caller(calldepth)
		if !ok {
			file = "???"
			line = 0
		}
		l.mu.Lock()
	}
	l.buf = l.buf[:0]
	l.formatHeader(&l.buf, now, file, line)
	l.buf = append(l.buf, s...)
	if len(s) == 0 || s[len(s)-1] != '\n' {
		l.buf = append(l.buf, '\n')
	}
	_, err := l.out.Write(l.buf)
	return err
}

// Printf calls l.Output to print to the logger.
// Arguments are handled in the manner of fmt.Printf.
func (l *Logger) Printf(format string, v ...interface{}) {
	l.Output(2, fmt.Sprintf(format, v...))
}

// Print calls l.Output to print to the logger.
// Arguments are handled in the manner of fmt.Print.
func (l *Logger) Print(v ...interface{}) { l.Output(2, fmt.Sprint(v...)) }

// Println calls l.Output to print to the logger.
// Arguments are handled in the manner of fmt.Println.
func (l *Logger) Println(v ...interface{}) { l.Output(2, fmt.Sprintln(v...)) }

// Fatal is equivalent to l.Print() followed by a call to os.Exit(1).
func (l *Logger) Fatal(v ...interface{}) {
	l.Output(2, fmt.Sprint(v...))
	os.Exit(1)
}

// Fatalf is equivalent to l.Printf() followed by a call to os.Exit(1).
func (l *Logger) Fatalf(format string, v ...interface{}) {
	l.Output(2, fmt.Sprintf(format, v...))
	os.Exit(1)
}

// Fatalln is equivalent to l.Println() followed by a call to os.Exit(1).
func (l *Logger) Fatalln(v ...interface{}) {
	l.Output(2, fmt.Sprintln(v...))
	os.Exit(1)
}

// Panic is equivalent to l.Print() followed by a call to panic().
func (l *Logger) Panic(v ...interface{}) {
	s := fmt.Sprint(v...)
	l.Output(2, s)
	panic(s)
}

// Panicf is equivalent to l.Printf() followed by a call to panic().
func (l *Logger) Panicf(format string, v ...interface{}) {
	s := fmt.Sprintf(format, v...)
	l.Output(2, s)
	panic(s)
}

// Panicln is equivalent to l.Println() followed by a call to panic().
func (l *Logger) Panicln(v ...interface{}) {
	s := fmt.Sprintln(v...)
	l.Output(2, s)
	panic(s)
}

// Flags returns the output flags for the logger.
func (l *Logger) Flags() int {
	l.mu.Lock()
	defer l.mu.Unlock()
	return l.flag
}

// SetFlags sets the output flags for the logger.
func (l *Logger) SetFlags(flag int) {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.flag = flag
}

// Prefix returns the output prefix for the logger.
func (l *Logger) Prefix() string {
	l.mu.Lock()
	defer l.mu.Unlock()
	return l.prefix
}

// SetPrefix sets the output prefix for the logger.
func (l *Logger) SetPrefix(prefix string) {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.prefix = prefix
}

// SetOutput sets the output destination for the standard logger.
func SetOutput(w io.Writer) {
	std.mu.Lock()
	defer std.mu.Unlock()
	std.out = w
}

// Flags returns the output flags for the standard logger.
func Flags() int {
	return std.Flags()
}

// SetFlags sets the output flags for the standard logger.
func SetFlags(flag int) {
	std.SetFlags(flag)
}

// Prefix returns the output prefix for the standard logger.
func Prefix() string {
	return std.Prefix()
}

// SetPrefix sets the output prefix for the standard logger.
func SetPrefix(prefix string) {
	std.SetPrefix(prefix)
}

// These functions write to the standard logger.

// Print calls Output to print to the standard logger.
// Arguments are handled in the manner of fmt.Print.
func Print(v ...interface{}) {
	std.Output(2, fmt.Sprint(v...))
}

// Printf calls Output to print to the standard logger.
// Arguments are handled in the manner of fmt.Printf.
func Printf(format string, v ...interface{}) {
	std.Output(2, fmt.Sprintf(format, v...))
}

// Println calls Output to print to the standard logger.
// Arguments are handled in the manner of fmt.Println.
func Println(v ...interface{}) {
	std.Output(2, fmt.Sprintln(v...))
}

// Fatal is equivalent to Print() followed by a call to os.Exit(1).
func Fatal(v ...interface{}) {
	std.Output(2, fmt.Sprint(v...))
	os.Exit(1)
}

// Fatalf is equivalent to Printf() followed by a call to os.Exit(1).
func Fatalf(format string, v ...interface{}) {
	std.Output(2, fmt.Sprintf(format, v...))
	os.Exit(1)
}

// Fatalln is equivalent to Println() followed by a call to os.Exit(1).
func Fatalln(v ...interface{}) {
	std.Output(2, fmt.Sprintln(v...))
	os.Exit(1)
}

// Panic is equivalent to Print() followed by a call to panic().
func Panic(v ...interface{}) {
	s := fmt.Sprint(v...)
	std.Output(2, s)
	panic(s)
}

// Panicf is equivalent to Printf() followed by a call to panic().
func Panicf(format string, v ...interface{}) {
	s := fmt.Sprintf(format, v...)
	std.Output(2, s)
	panic(s)
}

// Panicln is equivalent to Println() followed by a call to panic().
func Panicln(v ...interface{}) {
	s := fmt.Sprintln(v...)
	std.Output(2, s)
	panic(s)
}

// Output writes the output for a logging event. The string s contains
// the text to print after the prefix specified by the flags of the
// Logger. A newline is appended if the last character of s is not
// already a newline. Calldepth is the count of the number of
// frames to skip when computing the file name and line number
// if Llongfile or Lshortfile is set; a value of 1 will print the details
// for the caller of Output.
func Output(calldepth int, s string) error {
	return std.Output(calldepth+1, s) // +1 for this frame.
}
<commit_msg>log: move Ldate doc to general declaration<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package log implements a simple logging package. It defines a type, Logger,
// with methods for formatting output. It also has a predefined 'standard'
// Logger accessible through helper functions Print[f|ln], Fatal[f|ln], and
// Panic[f|ln], which are easier to use than creating a Logger manually.
// That logger writes to standard error and prints the date and time
// of each logged message.
// Every log message is output on a separate line: if the message being
// printed does not end in a newline, the logger will add one.
// The Fatal functions call os.Exit(1) after writing the log message.
// The Panic functions call panic after writing the log message.
package log

import (
	"fmt"
	"io"
	"os"
	"runtime"
	"sync"
	"time"
)

// These flags define which text to prefix to each log entry generated by the Logger.
// Bits are or'ed together to control what's printed.
// There is no control over the order they appear (the order listed
// here) or the format they present (as described in the comments).
// The prefix is followed by a colon only when Llongfile or Lshortfile
// is specified.
// For example, flags Ldate | Ltime (or LstdFlags) produce,
//	2009/01/23 01:23:23 message
// while flags Ldate | Ltime | Lmicroseconds | Llongfile produce,
//	2009/01/23 01:23:23.123123 /a/b/c/d.go:23: message
const (
	Ldate         = 1 << iota     // the date in the local time zone: 2009/01/23
	Ltime                         // the time in the local time zone: 01:23:23
	Lmicroseconds                 // microsecond resolution: 01:23:23.123123.  assumes Ltime.
	Llongfile                     // full file name and line number: /a/b/c/d.go:23
	Lshortfile                    // final file name element and line number: d.go:23. overrides Llongfile
	LUTC                          // if Ldate or Ltime is set, use UTC rather than the local time zone
	LstdFlags     = Ldate | Ltime // initial values for the standard logger
)

// A Logger represents an active logging object that generates lines of
// output to an io.Writer. Each logging operation makes a single call to
// the Writer's Write method. A Logger can be used simultaneously from
// multiple goroutines; it guarantees to serialize access to the Writer.
type Logger struct {
	mu     sync.Mutex // ensures atomic writes; protects the following fields
	prefix string     // prefix to write at beginning of each line
	flag   int        // properties
	out    io.Writer  // destination for output
	buf    []byte     // for accumulating text to write
}

// New creates a new Logger. The out variable sets the
// destination to which log data will be written.
// The prefix appears at the beginning of each generated log line.
// The flag argument defines the logging properties.
func New(out io.Writer, prefix string, flag int) *Logger {
	return &Logger{out: out, prefix: prefix, flag: flag}
}

// SetOutput sets the output destination for the logger.
func (l *Logger) SetOutput(w io.Writer) {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.out = w
}

var std = New(os.Stderr, "", LstdFlags)

// Cheap integer to fixed-width decimal ASCII. Give a negative width to avoid zero-padding.
func itoa(buf *[]byte, i int, wid int) {
	// Assemble decimal in reverse order.
	var b [20]byte
	bp := len(b) - 1
	for i >= 10 || wid > 1 {
		wid--
		q := i / 10
		b[bp] = byte('0' + i - q*10)
		bp--
		i = q
	}
	// i < 10
	b[bp] = byte('0' + i)
	*buf = append(*buf, b[bp:]...)
}

// formatHeader writes log header to buf in following order:
//   * l.prefix (if it's not blank),
//   * date and/or time (if corresponding flags are provided),
//   * file and line number (if corresponding flags are provided).
func (l *Logger) formatHeader(buf *[]byte, t time.Time, file string, line int) {
	*buf = append(*buf, l.prefix...)
	if l.flag&(Ldate|Ltime|Lmicroseconds) != 0 {
		if l.flag&LUTC != 0 {
			t = t.UTC()
		}
		if l.flag&Ldate != 0 {
			year, month, day := t.Date()
			itoa(buf, year, 4)
			*buf = append(*buf, '/')
			itoa(buf, int(month), 2)
			*buf = append(*buf, '/')
			itoa(buf, day, 2)
			*buf = append(*buf, ' ')
		}
		if l.flag&(Ltime|Lmicroseconds) != 0 {
			hour, min, sec := t.Clock()
			itoa(buf, hour, 2)
			*buf = append(*buf, ':')
			itoa(buf, min, 2)
			*buf = append(*buf, ':')
			itoa(buf, sec, 2)
			if l.flag&Lmicroseconds != 0 {
				*buf = append(*buf, '.')
				itoa(buf, t.Nanosecond()/1e3, 6)
			}
			*buf = append(*buf, ' ')
		}
	}
	if l.flag&(Lshortfile|Llongfile) != 0 {
		if l.flag&Lshortfile != 0 {
			short := file
			for i := len(file) - 1; i > 0; i-- {
				if file[i] == '/' {
					short = file[i+1:]
					break
				}
			}
			file = short
		}
		*buf = append(*buf, file...)
		*buf = append(*buf, ':')
		itoa(buf, line, -1)
		*buf = append(*buf, ": "...)
	}
}

// Output writes the output for a logging event. The string s contains
// the text to print after the prefix specified by the flags of the
// Logger. A newline is appended if the last character of s is not
// already a newline. Calldepth is used to recover the PC and is
// provided for generality, although at the moment on all pre-defined
// paths it will be 2.
func (l *Logger) Output(calldepth int, s string) error {
	// Get time early if we need it.
	var now time.Time
	if l.flag&(Ldate|Ltime|Lmicroseconds) != 0 {
		now = time.Now()
	}
	var file string
	var line int
	l.mu.Lock()
	defer l.mu.Unlock()
	if l.flag&(Lshortfile|Llongfile) != 0 {
		// Release lock while getting caller info - it's expensive.
		l.mu.Unlock()
		var ok bool
		_, file, line, ok = runtime.Caller(calldepth)
		if !ok {
			file = "???"
			line = 0
		}
		l.mu.Lock()
	}
	l.buf = l.buf[:0]
	l.formatHeader(&l.buf, now, file, line)
	l.buf = append(l.buf, s...)
	if len(s) == 0 || s[len(s)-1] != '\n' {
		l.buf = append(l.buf, '\n')
	}
	_, err := l.out.Write(l.buf)
	return err
}

// Printf calls l.Output to print to the logger.
// Arguments are handled in the manner of fmt.Printf.
func (l *Logger) Printf(format string, v ...interface{}) {
	l.Output(2, fmt.Sprintf(format, v...))
}

// Print calls l.Output to print to the logger.
// Arguments are handled in the manner of fmt.Print.
func (l *Logger) Print(v ...interface{}) { l.Output(2, fmt.Sprint(v...)) }

// Println calls l.Output to print to the logger.
// Arguments are handled in the manner of fmt.Println.
func (l *Logger) Println(v ...interface{}) { l.Output(2, fmt.Sprintln(v...)) }

// Fatal is equivalent to l.Print() followed by a call to os.Exit(1).
func (l *Logger) Fatal(v ...interface{}) {
	l.Output(2, fmt.Sprint(v...))
	os.Exit(1)
}

// Fatalf is equivalent to l.Printf() followed by a call to os.Exit(1).
func (l *Logger) Fatalf(format string, v ...interface{}) {
	l.Output(2, fmt.Sprintf(format, v...))
	os.Exit(1)
}

// Fatalln is equivalent to l.Println() followed by a call to os.Exit(1).
func (l *Logger) Fatalln(v ...interface{}) {
	l.Output(2, fmt.Sprintln(v...))
	os.Exit(1)
}

// Panic is equivalent to l.Print() followed by a call to panic().
func (l *Logger) Panic(v ...interface{}) {
	s := fmt.Sprint(v...)
	l.Output(2, s)
	panic(s)
}

// Panicf is equivalent to l.Printf() followed by a call to panic().
func (l *Logger) Panicf(format string, v ...interface{}) {
	s := fmt.Sprintf(format, v...)
	l.Output(2, s)
	panic(s)
}

// Panicln is equivalent to l.Println() followed by a call to panic().
func (l *Logger) Panicln(v ...interface{}) {
	s := fmt.Sprintln(v...)
	l.Output(2, s)
	panic(s)
}

// Flags returns the output flags for the logger.
func (l *Logger) Flags() int {
	l.mu.Lock()
	defer l.mu.Unlock()
	return l.flag
}

// SetFlags sets the output flags for the logger.
func (l *Logger) SetFlags(flag int) {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.flag = flag
}

// Prefix returns the output prefix for the logger.
func (l *Logger) Prefix() string {
	l.mu.Lock()
	defer l.mu.Unlock()
	return l.prefix
}

// SetPrefix sets the output prefix for the logger.
func (l *Logger) SetPrefix(prefix string) {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.prefix = prefix
}

// SetOutput sets the output destination for the standard logger.
func SetOutput(w io.Writer) {
	std.mu.Lock()
	defer std.mu.Unlock()
	std.out = w
}

// Flags returns the output flags for the standard logger.
func Flags() int {
	return std.Flags()
}

// SetFlags sets the output flags for the standard logger.
func SetFlags(flag int) {
	std.SetFlags(flag)
}

// Prefix returns the output prefix for the standard logger.
func Prefix() string {
	return std.Prefix()
}

// SetPrefix sets the output prefix for the standard logger.
func SetPrefix(prefix string) {
	std.SetPrefix(prefix)
}

// These functions write to the standard logger.

// Print calls Output to print to the standard logger.
// Arguments are handled in the manner of fmt.Print.
func Print(v ...interface{}) {
	std.Output(2, fmt.Sprint(v...))
}

// Printf calls Output to print to the standard logger.
// Arguments are handled in the manner of fmt.Printf.
func Printf(format string, v ...interface{}) {
	std.Output(2, fmt.Sprintf(format, v...))
}

// Println calls Output to print to the standard logger.
// Arguments are handled in the manner of fmt.Println.
func Println(v ...interface{}) {
	std.Output(2, fmt.Sprintln(v...))
}

// Fatal is equivalent to Print() followed by a call to os.Exit(1).
func Fatal(v ...interface{}) {
	std.Output(2, fmt.Sprint(v...))
	os.Exit(1)
}

// Fatalf is equivalent to Printf() followed by a call to os.Exit(1).
func Fatalf(format string, v ...interface{}) {
	std.Output(2, fmt.Sprintf(format, v...))
	os.Exit(1)
}

// Fatalln is equivalent to Println() followed by a call to os.Exit(1).
func Fatalln(v ...interface{}) {
	std.Output(2, fmt.Sprintln(v...))
	os.Exit(1)
}

// Panic is equivalent to Print() followed by a call to panic().
func Panic(v ...interface{}) {
	s := fmt.Sprint(v...)
	std.Output(2, s)
	panic(s)
}

// Panicf is equivalent to Printf() followed by a call to panic().
func Panicf(format string, v ...interface{}) {
	s := fmt.Sprintf(format, v...)
	std.Output(2, s)
	panic(s)
}

// Panicln is equivalent to Println() followed by a call to panic().
func Panicln(v ...interface{}) {
	s := fmt.Sprintln(v...)
	std.Output(2, s)
	panic(s)
}

// Output writes the output for a logging event. The string s contains
// the text to print after the prefix specified by the flags of the
// Logger. A newline is appended if the last character of s is not
// already a newline. Calldepth is the count of the number of
// frames to skip when computing the file name and line number
// if Llongfile or Lshortfile is set; a value of 1 will print the details
// for the caller of Output.
func Output(calldepth int, s string) error {
	return std.Output(calldepth+1, s) // +1 for this frame.
}
<|endoftext|>


<commit_before>package migrations

import (
	"fmt"
	"gopkg.in/go-pg/migrations.v4"
)

var Initialization = migrations.Migration{
	Version: 1,
	Up: func(db migrations.DB) error {
		fmt.Println("Upgrading Initialization migration")
		queries := []string{
			`CREATE TABLE public.organizations (
			 	id SERIAL NOT NULL,
				name CHARACTER VARYING(200) NOT NULL
			)`,
			`CREATE TABLE public.users (
				id SERIAL NOT NULL,
				username CHARACTER VARYING(200) NOT NULL,
				email CHARACTER VARYING(200) NOT NULL,
				password CHARACTER VARYING(200) NOT NULL
			)`,
			`CREATE TABLE public.customers (
				id SERIAL NOT NULL,
				company_name CHARACTER VARYING(200) NOT NULL,
				first_name CHARACTER VARYING(200) NOT NULL,
				last_name CHARACTER VARYING(200) NOT NULL,
				email CHARACTER VARYING(200) NOT NULL,
				phone_number CHARACTER VARYING(200) NOT NULL,
				vat_number CHARACTER VARYING(200) NOT NULL
			)`,
		}
		for _, q := range queries {
			_, err := db.Exec(q)
			if err != nil {
				return err
			}
		}
		return nil
	},
	Down: func(db migrations.DB) error {
		fmt.Println("Downgrading Initialization migration")
		queries := []string{
			`DROP TABLE users`,
			`DROP TABLE organizations`,
			`DROP TABLE customers`,
		}
		for _, q := range queries {
			_, err := db.Exec(q)
			if err != nil {
				return err
			}
		}
		return nil
	},
}
<commit_msg>Added addresses migration<commit_after>package migrations

import (
	"fmt"
	"gopkg.in/go-pg/migrations.v4"
)

var Initialization = migrations.Migration{
	Version: 1,
	Up: func(db migrations.DB) error {
		fmt.Println("Upgrading Initialization migration")
		queries := []string{
			`CREATE TABLE public.organizations (
			 	id SERIAL NOT NULL,
				name CHARACTER VARYING(200) NOT NULL
			)`,
			`CREATE TABLE public.users (
				id SERIAL NOT NULL,
				username CHARACTER VARYING(200) NOT NULL,
				email CHARACTER VARYING(200) NOT NULL,
				password CHARACTER VARYING(200) NOT NULL
			)`,
			`CREATE TABLE public.customers (
				id SERIAL NOT NULL,
				company_name CHARACTER VARYING(200) NOT NULL,
				first_name CHARACTER VARYING(200) NOT NULL,
				last_name CHARACTER VARYING(200) NOT NULL,
				email CHARACTER VARYING(200) NOT NULL,
				phone_number CHARACTER VARYING(200) NOT NULL,
				vat_number CHARACTER VARYING(200) NOT NULL
			)`,
			`CREATE TABLE public.addresses (
				id SERIAL NOT NULL,
				address CHARACTER VARYING(255) NOT NULL,
				postal_code CHARACTER VARYING(10) NOT NULL,
				city CHARACTER VARYING(200) NOT NULL,
				country CHARACTER VARYING(200) NOT NULL,
			)`,
		}
		for _, q := range queries {
			_, err := db.Exec(q)
			if err != nil {
				return err
			}
		}
		return nil
	},
	Down: func(db migrations.DB) error {
		fmt.Println("Downgrading Initialization migration")
		queries := []string{
			`DROP TABLE users`,
			`DROP TABLE organizations`,
			`DROP TABLE customers`,
			`DROP TABLE addresses`,
		}
		for _, q := range queries {
			_, err := db.Exec(q)
			if err != nil {
				return err
			}
		}
		return nil
	},
}
<|endoftext|>


<commit_before>// Copyright 2013 The Chihaya Authors. All rights reserved.
// Use of this source code is governed by the BSD 2-Clause license,
// which can be found in the LICENSE file.

package config

import (
	"time"
)

// MockConfig is a pre-initialized config that can be used for testing purposes.
var MockConfig = Config{
	Addr: ":80",
	Tracker: DataStore{
		Driver: "mock",
	},
	Backend: DataStore{
		Driver: "mock",
	},
	Private:        true,
	Freeleech:      false,
	Announce:       Duration{30 * time.Minute},
	MinAnnounce:    Duration{15 * time.Minute},
	ReadTimeout:    Duration{20 % time.Second},
	DefaultNumWant: 50,
}
<commit_msg>mockConfig.Addr :80 -> :34000<commit_after>// Copyright 2013 The Chihaya Authors. All rights reserved.
// Use of this source code is governed by the BSD 2-Clause license,
// which can be found in the LICENSE file.

package config

import (
	"time"
)

// MockConfig is a pre-initialized config that can be used for testing purposes.
var MockConfig = Config{
	Addr: ":34000",
	Tracker: DataStore{
		Driver: "mock",
	},
	Backend: DataStore{
		Driver: "mock",
	},
	Private:        true,
	Freeleech:      false,
	Announce:       Duration{30 * time.Minute},
	MinAnnounce:    Duration{15 * time.Minute},
	ReadTimeout:    Duration{20 % time.Second},
	DefaultNumWant: 50,
}
<|endoftext|>


<commit_before>package parser

import (
	"fmt"
	"net/http"

	"github.com/go-playground/validator"
	"github.com/labstack/echo"
	"github.com/leebenson/conform"
	"github.com/microcosm-cc/bluemonday"
)

// Request holds the mapped fields from the request's JSON body

type field struct {
	Name     string      `json:"name" validate:"required,gte=3,lte=30,contains=.,excludesall= " conform:"trim,lower"` // TODO: Add custom validator
	Operator *string     `json:"operator" validate:"omitempty,gte=1,lte=3,alpha" conform:"trim,upper"`
	Value    interface{} `json:"value" validate:"required"`
	Next     *operation  `json:"next" validate:"omitempty"`
}

type operation struct {
	Condition string `json:"condition" validate:"required,gte=1,lte=3,alpha" conform:"trim,upper"` // TODO: Add custom validator
	Field     *field `json:"field" validate:"omitempty"`
}

type variables struct {
	Field field `json:"field" validate:"required"`
}

type Request struct {
	Query     string    `json:"query" validate:"required,gte=10,lte=5000" conform:"trim"`
	Variables variables `json:"variables" validate:"required"`
}

// TODO: Consider extracting the common parts into its own package

// Parse parses, scrubs and escapes a request's JSON body and maps it to a struct
func Parse(context echo.Context) (*Request, error) {
	request := new(Request)

	conform.Strings(request)
	escape(request)

	if err := bind(request, context); err != nil {
		return request, err
	}

	if err := validate(request, context); err != nil {
		return request, err
	}

	return request, nil
}

func bind(request *Request, context echo.Context) error {
	if err := context.Bind(request); err != nil {
		errorDescription := err.Error()
		errorMessage := fmt.Sprintf("Error parsing request: %s", errorDescription)
		errorCode := http.StatusBadRequest

		if httpError, ok := err.(*echo.HTTPError); ok {
			if value, isString := httpError.Message.(string); isString {
				errorMessage = value
				errorCode = httpError.Code
			}
		}

		context.Logger().Error("Error binding request: ", errorDescription)

		return echo.NewHTTPError(errorCode, []string{errorMessage})
	}

	return nil
}

func validate(request *Request, context echo.Context) error {
	if errs := context.Validate(request); errs != nil {
		var errorList []string
		var errorMessage = "Error validating request: "

		if _, ok := errs.(*validator.InvalidValidationError); ok {
			context.Logger().Error(errorMessage, errs.Error())

			return echo.NewHTTPError(http.StatusUnprocessableEntity, []string{errs.Error()})
		}

		for _, err := range errs.(validator.ValidationErrors) {
			errorDescription := err.(error).Error()
			errorList = append(errorList, errorDescription)

			context.Logger().Error(errorMessage, errorDescription)
		}

		return echo.NewHTTPError(http.StatusUnprocessableEntity, errorList)
	}

	return nil
}

func escape(request *Request) {
	sanitizer := bluemonday.StrictPolicy()

	request.Query = sanitizer.Sanitize(request.Query)
	request.Variables.Field.Name = sanitizer.Sanitize(request.Variables.Field.Name)

	switch value := request.Variables.Field.Value.(type) {
	case string:
		request.Variables.Field.Value = sanitizer.Sanitize(value)
	default:
		break
	}
}
<commit_msg>Added TODO comment<commit_after>package parser

import (
	"fmt"
	"net/http"

	"github.com/go-playground/validator"
	"github.com/labstack/echo"
	"github.com/leebenson/conform"
	"github.com/microcosm-cc/bluemonday"
)

// Request holds the mapped fields from the request's JSON body

type field struct {
	Name     string      `json:"name" validate:"required,gte=3,lte=30,contains=.,excludesall= " conform:"trim,lower"` // TODO: Add custom validator
	Operator *string     `json:"operator" validate:"omitempty,gte=1,lte=3,alpha" conform:"trim,upper"`                // TODO: Add custom validator
	Value    interface{} `json:"value" validate:"required"`
	Next     *operation  `json:"next" validate:"omitempty"`
}

type operation struct {
	Condition string `json:"condition" validate:"required,gte=1,lte=3,alpha" conform:"trim,upper"` // TODO: Add custom validator
	Field     *field `json:"field" validate:"omitempty"`
}

type variables struct {
	Field field `json:"field" validate:"required"`
}

type Request struct {
	Query     string    `json:"query" validate:"required,gte=10,lte=5000" conform:"trim"`
	Variables variables `json:"variables" validate:"required"`
}

// TODO: Consider extracting the common parts into its own package

// Parse parses, scrubs and escapes a request's JSON body and maps it to a struct
func Parse(context echo.Context) (*Request, error) {
	request := new(Request)

	conform.Strings(request)
	escape(request)

	if err := bind(request, context); err != nil {
		return request, err
	}

	if err := validate(request, context); err != nil {
		return request, err
	}

	return request, nil
}

func bind(request *Request, context echo.Context) error {
	if err := context.Bind(request); err != nil {
		errorDescription := err.Error()
		errorMessage := fmt.Sprintf("Error parsing request: %s", errorDescription)
		errorCode := http.StatusBadRequest

		if httpError, ok := err.(*echo.HTTPError); ok {
			if value, isString := httpError.Message.(string); isString {
				errorMessage = value
				errorCode = httpError.Code
			}
		}

		context.Logger().Error("Error binding request: ", errorDescription)

		return echo.NewHTTPError(errorCode, []string{errorMessage})
	}

	return nil
}

func validate(request *Request, context echo.Context) error {
	if errs := context.Validate(request); errs != nil {
		var errorList []string
		var errorMessage = "Error validating request: "

		if _, ok := errs.(*validator.InvalidValidationError); ok {
			context.Logger().Error(errorMessage, errs.Error())

			return echo.NewHTTPError(http.StatusUnprocessableEntity, []string{errs.Error()})
		}

		for _, err := range errs.(validator.ValidationErrors) {
			errorDescription := err.(error).Error()
			errorList = append(errorList, errorDescription)

			context.Logger().Error(errorMessage, errorDescription)
		}

		return echo.NewHTTPError(http.StatusUnprocessableEntity, errorList)
	}

	return nil
}

func escape(request *Request) {
	sanitizer := bluemonday.StrictPolicy()

	request.Query = sanitizer.Sanitize(request.Query)
	request.Variables.Field.Name = sanitizer.Sanitize(request.Variables.Field.Name)

	switch value := request.Variables.Field.Value.(type) {
	case string:
		request.Variables.Field.Value = sanitizer.Sanitize(value)
	default:
		break
	}
}
<|endoftext|>


<commit_before>// Support for containerising tests. Currently Docker only.

package test

import "fmt"
import "io/ioutil"
import "os/exec"
import "path"
import "strings"
import "time"

import "build"
import "core"

func runContainerisedTest(state *core.BuildState, target *core.BuildTarget) ([]byte, error) {
	testDir := path.Join(core.RepoRoot, target.TestDir())
	replacedCmd := build.ReplaceTestSequences(target, target.TestCommand)
	replacedCmd += " " + strings.Join(state.TestArgs, " ")
	containerName := state.Config.Docker.DefaultImage
	if target.ContainerSettings != nil && target.ContainerSettings.DockerImage != "" {
		containerName = target.ContainerSettings.DockerImage
	}
	// Gentle hack: remove the absolute path from the command
	replacedCmd = strings.Replace(replacedCmd, testDir, "/tmp/test", -1)
	// Fiddly hack follows to handle docker run --rm failing saying "Cannot destroy container..."
	// "Driver aufs failed to remove root filesystem... device or resource busy"
	cidfile := path.Join(testDir, ".container_id")
	// Using C.UTF-8 for LC_ALL because it works. Not sure it's strictly
	// correct to mix that with LANG=en_GB.UTF-8
	command := []string{"run", "--cidfile", cidfile, "-e", "LC_ALL=C.UTF-8"}
	if target.ContainerSettings != nil {
		if target.ContainerSettings.DockerRunArgs != "" {
			command = append(command, strings.Split(target.ContainerSettings.DockerRunArgs, " ")...)
		}
		if target.ContainerSettings.DockerUser != "" {
			command = append(command, "-u", target.ContainerSettings.DockerUser)
		}
	} else {
		command = append(command, state.Config.Docker.RunArgs...)
	}
	for _, env := range core.BuildEnvironment(state, target, true) {
		command = append(command, "-e", strings.Replace(env, testDir, "/tmp/test", -1))
	}
	replacedCmd = "mkdir -p /tmp/test && cp -r /tmp/test_in/* /tmp/test && cd /tmp/test && " + replacedCmd
	command = append(command, "-v", testDir+":/tmp/test_in", "-w", "/tmp/test_in", containerName, "bash", "-o", "pipefail", "-c", replacedCmd)
	if state.PrintCommands {
		log.Notice("Running containerised test %s: docker %s", target.Label, strings.Join(command, " "))
	} else {
		log.Debug("Running containerised test %s: docker %s", target.Label, strings.Join(command, " "))
	}
	cmd := exec.Command("docker", command...)
	cmd.Dir = target.TestDir()
	out, err := core.ExecWithTimeout(cmd, target.TestTimeout, state.Config.Test.Timeout)
	_, isTimeout := err.(core.TimeoutError)
	retrieveResultsAndRemoveContainer(target, cidfile, !isTimeout)
	return out, err
}

func runPossiblyContainerisedTest(state *core.BuildState, target *core.BuildTarget) (out []byte, err error) {
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("%s", r)
		}
	}()

	if target.Containerise {
		out, err = runContainerisedTest(state, target)
		if err != nil && state.Config.Docker.AllowLocalFallback {
			log.Warning("Failed to run %s containerised: %s %s. Falling back to local version.",
				target.Label, out, err)
			return runTest(state, target, state.Config.Test.Timeout)
		}
		return out, err
	}
	return runTest(state, target, state.Config.Test.Timeout)
}

// retrieveResultsAndRemoveContainer copies the test.results file out of the Docker container and into
// the expected location. It then removes the container.
func retrieveResultsAndRemoveContainer(target *core.BuildTarget, containerFile string, warn bool) {
	cid, err := ioutil.ReadFile(containerFile)
	if err != nil {
		log.Warning("Failed to read Docker container file %s", containerFile)
		return
	}
	if !target.NoTestOutput {
		retrieveFile(target, cid, "test.results", warn)
	}
	if core.State.NeedCoverage {
		retrieveFile(target, cid, "test.coverage", false)
	}
	for _, output := range target.TestOutputs {
		retrieveFile(target, cid, output, false)
	}
	// Give this some time to complete. Processes inside the container might not be ready
	// to shut down immediately.
	timeout := core.State.Config.Docker.RemoveTimeout
	for i := 0; i < 5; i++ {
		cmd := exec.Command("docker", "rm", string(cid))
		if _, err := core.ExecWithTimeout(cmd, timeout, timeout); err == nil {
			return
		}
		time.Sleep(100 * time.Millisecond)
	}
}

// retrieveFile retrieves a single file (or directory) from a Docker container.
func retrieveFile(target *core.BuildTarget, cid []byte, filename string, warn bool) {
	log.Debug("Attempting to retrieve file %s for %s...", filename, target.Label)
	timeout := core.State.Config.Docker.ResultsTimeout
	cmd := exec.Command("docker", "cp", string(cid)+":/tmp/test/"+filename, target.TestDir())
	if out, err := core.ExecWithTimeout(cmd, timeout, timeout); err != nil {
		if warn {
			log.Warning("Failed to retrieve results for %s: %s [%s]", target.Label, err, out)
		} else {
			log.Debug("Failed to retrieve results for %s: %s [%s]", target.Label, err, out)
		}
	}
}
<commit_msg>Force docker container removal. (#116)<commit_after>// Support for containerising tests. Currently Docker only.

package test

import "fmt"
import "io/ioutil"
import "os/exec"
import "path"
import "strings"
import "time"

import "build"
import "core"

func runContainerisedTest(state *core.BuildState, target *core.BuildTarget) ([]byte, error) {
	testDir := path.Join(core.RepoRoot, target.TestDir())
	replacedCmd := build.ReplaceTestSequences(target, target.TestCommand)
	replacedCmd += " " + strings.Join(state.TestArgs, " ")
	containerName := state.Config.Docker.DefaultImage
	if target.ContainerSettings != nil && target.ContainerSettings.DockerImage != "" {
		containerName = target.ContainerSettings.DockerImage
	}
	// Gentle hack: remove the absolute path from the command
	replacedCmd = strings.Replace(replacedCmd, testDir, "/tmp/test", -1)
	// Fiddly hack follows to handle docker run --rm failing saying "Cannot destroy container..."
	// "Driver aufs failed to remove root filesystem... device or resource busy"
	cidfile := path.Join(testDir, ".container_id")
	// Using C.UTF-8 for LC_ALL because it works. Not sure it's strictly
	// correct to mix that with LANG=en_GB.UTF-8
	command := []string{"run", "--cidfile", cidfile, "-e", "LC_ALL=C.UTF-8"}
	if target.ContainerSettings != nil {
		if target.ContainerSettings.DockerRunArgs != "" {
			command = append(command, strings.Split(target.ContainerSettings.DockerRunArgs, " ")...)
		}
		if target.ContainerSettings.DockerUser != "" {
			command = append(command, "-u", target.ContainerSettings.DockerUser)
		}
	} else {
		command = append(command, state.Config.Docker.RunArgs...)
	}
	for _, env := range core.BuildEnvironment(state, target, true) {
		command = append(command, "-e", strings.Replace(env, testDir, "/tmp/test", -1))
	}
	replacedCmd = "mkdir -p /tmp/test && cp -r /tmp/test_in/* /tmp/test && cd /tmp/test && " + replacedCmd
	command = append(command, "-v", testDir+":/tmp/test_in", "-w", "/tmp/test_in", containerName, "bash", "-o", "pipefail", "-c", replacedCmd)
	if state.PrintCommands {
		log.Notice("Running containerised test %s: docker %s", target.Label, strings.Join(command, " "))
	} else {
		log.Debug("Running containerised test %s: docker %s", target.Label, strings.Join(command, " "))
	}
	cmd := exec.Command("docker", command...)
	cmd.Dir = target.TestDir()
	out, err := core.ExecWithTimeout(cmd, target.TestTimeout, state.Config.Test.Timeout)
	_, isTimeout := err.(core.TimeoutError)
	retrieveResultsAndRemoveContainer(target, cidfile, !isTimeout)
	return out, err
}

func runPossiblyContainerisedTest(state *core.BuildState, target *core.BuildTarget) (out []byte, err error) {
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("%s", r)
		}
	}()

	if target.Containerise {
		out, err = runContainerisedTest(state, target)
		if err != nil && state.Config.Docker.AllowLocalFallback {
			log.Warning("Failed to run %s containerised: %s %s. Falling back to local version.",
				target.Label, out, err)
			return runTest(state, target, state.Config.Test.Timeout)
		}
		return out, err
	}
	return runTest(state, target, state.Config.Test.Timeout)
}

// retrieveResultsAndRemoveContainer copies the test.results file out of the Docker container and into
// the expected location. It then removes the container.
func retrieveResultsAndRemoveContainer(target *core.BuildTarget, containerFile string, warn bool) {
	cid, err := ioutil.ReadFile(containerFile)
	if err != nil {
		log.Warning("Failed to read Docker container file %s", containerFile)
		return
	}
	if !target.NoTestOutput {
		retrieveFile(target, cid, "test.results", warn)
	}
	if core.State.NeedCoverage {
		retrieveFile(target, cid, "test.coverage", false)
	}
	for _, output := range target.TestOutputs {
		retrieveFile(target, cid, output, false)
	}
	// Give this some time to complete. Processes inside the container might not be ready
	// to shut down immediately.
	timeout := core.State.Config.Docker.RemoveTimeout
	for i := 0; i < 5; i++ {
		cmd := exec.Command("docker", "rm", "-f", string(cid))
		if _, err := core.ExecWithTimeout(cmd, timeout, timeout); err == nil {
			return
		}
		time.Sleep(100 * time.Millisecond)
	}
}

// retrieveFile retrieves a single file (or directory) from a Docker container.
func retrieveFile(target *core.BuildTarget, cid []byte, filename string, warn bool) {
	log.Debug("Attempting to retrieve file %s for %s...", filename, target.Label)
	timeout := core.State.Config.Docker.ResultsTimeout
	cmd := exec.Command("docker", "cp", string(cid)+":/tmp/test/"+filename, target.TestDir())
	if out, err := core.ExecWithTimeout(cmd, timeout, timeout); err != nil {
		if warn {
			log.Warning("Failed to retrieve results for %s: %s [%s]", target.Label, err, out)
		} else {
			log.Debug("Failed to retrieve results for %s: %s [%s]", target.Label, err, out)
		}
	}
}
<|endoftext|>