<commit_before>/*
 * Copyright (c) 2016, TEECOM
 *
 * This code is provided for free, as is, under the MIT license
 * (see LICENSE.md).
 */

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
)

// Strucures; the actual information is separated from updates
type line struct {
	Name  string
	ID    string
	Times []int
	Color string
}

type coordinates struct {
	Lat float64
	Lon float64
}

type station struct {
	Name  string
	ID    string
	Coord coordinates

	Directions [2]string

	Lines [2]map[string]*line
}

type system struct {
	Name    string
	Tagline string
	Stops   []station
	TimeMax int
	stopMap map[string]*station
	rwLock  *sync.RWMutex
}

// Update structures (externally generated)
type lineUpdate struct {
	LineID string
	Index  int
	Times  []int
}

type stationUpdate struct {
	StationID string
	Lines     []lineUpdate
}

type update struct {
	Stops []stationUpdate
}

// This is the main system information; fill this as appropriate
var mainSystem system = system{
	Name:    "TEECOM Shuttle Service",
	Tagline: "San Francisco Bay Area",
	Stops: []station{
		station{
			Name: "TEECOM",
			ID:   "tee",
			Coord: coordinates{
				Lat: 10.5,
				Lon: 10.5,
			},
			Directions: [2]string{"Northbound", "Southbound"},
			Lines: [2]map[string]*line{
				map[string]*line{
					"sh": &line{
						Name:  "Shuttle",
						ID:    "sh",
						Color: "#ff0000",
					},
				},
			},
		},
	},

	TimeMax: 45,

	stopMap: make(map[string]*station),
	rwLock:  &sync.RWMutex{},
}

// Where static files will be found
const staticDirectory string = "static"

func prepareSystem() {
	// Cache system IDs for future lookup.
	// No need to do any locking as the server hasn't
	// started up yet.
	for i := 0; i < len(mainSystem.Stops); i++ {
		stop := &mainSystem.Stops[i]
		mainSystem.stopMap[stop.ID] = stop
	}
}

func main() {
	log.Println("Starting server")
	prepareSystem()

	// Setup routing
	http.HandleFunc("/info", handleInfo)
	http.HandleFunc("/update", handleUpdate)
	http.HandleFunc("/stop", handleStopInfo)

	// Run server on port 8080
	if err := http.ListenAndServe(":8080", nil); err != nil {
		log.Fatal(err)
	}
}

// JSON encode all of the information
func handleInfo(w http.ResponseWriter, r *http.Request) {
	// Obtain a read lock for the system
	mainSystem.rwLock.RLock()
	defer mainSystem.rwLock.RUnlock()

	if err := json.NewEncoder(w).Encode(mainSystem); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintln(w, "Internal Server Error")
	}
}

func handleStopInfo(w http.ResponseWriter, r *http.Request) {
	// Check for valid GET parameters
	stopID := r.URL.Query()["id"]
	if stopID == nil || len(stopID) != 1 {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintln(w, "400 Bad Request: Missing stop ID")
		return
	}

	// Obtain a read lock for the system
	mainSystem.rwLock.RLock()
	defer mainSystem.rwLock.RUnlock()

	// Try to find the correct stop
	stop := mainSystem.stopMap[stopID[0]]
	if stop == nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "400 Bad Request: Invalid stop id (%s)\n", stopID[0])
		return
	}

	// Send the response
	if err := json.NewEncoder(w).Encode(stop); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintln(w, "Internal Server Error")
	}
}

// Handle update request
func handleUpdate(w http.ResponseWriter, r *http.Request) {
	// Ensure we are dealing with a POST request
	if r.Method != "POST" {
		serve(w, "update.html", http.StatusBadRequest)
		return
	}

	// Decode the JSON
	var new update
	if err := json.NewDecoder(r.Body).Decode(&new); err != nil {
		serve(w, "badupdate.html", http.StatusBadRequest)
		return
	}

	// Try to apply the updates
	if err := processUpdates(&new); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "400 Bad Request: %s\n", err.Error())
		return
	}
}

func processUpdates(u *update) error {
	// Obtain a writer lock
	mainSystem.rwLock.Lock()
	defer mainSystem.rwLock.Unlock()

	for _, su := range u.Stops {
		stop := mainSystem.stopMap[su.StationID]
		if stop == nil {
			return errors.New("Invalid station ID")
		}

		for _, lu := range su.Lines {
			if lu.Index > 1 {
				return errors.New("Line index out of bounds")
			}

			ln := stop.Lines[lu.Index][lu.LineID]
			if ln == nil {
				return errors.New("Invalid line ID")
			}

			ln.Times = lu.Times
		}
	}

	return nil
}

func serve(w http.ResponseWriter, f string, code int) {
	text, err := ioutil.ReadFile(staticDirectory + "/" + f)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintln(w, "500 Internal Server Error")
		return
	}

	w.WriteHeader(code)
	fmt.Fprintf(w, "%s", text)
}
<commit_msg>Use embedded RWMutex in system.<commit_after>/*
 * Copyright (c) 2016, TEECOM
 *
 * This code is provided for free, as is, under the MIT license
 * (see LICENSE.md).
 */

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
)

// Strucures; the actual information is separated from updates
type line struct {
	Name  string
	ID    string
	Times []int
	Color string
}

type coordinates struct {
	Lat float64
	Lon float64
}

type station struct {
	Name  string
	ID    string
	Coord coordinates

	Directions [2]string

	Lines [2]map[string]*line
}

type system struct {
	sync.RWMutex // Protects everything below
	Name         string
	Tagline      string
	Stops        []station
	TimeMax      int
	stopMap      map[string]*station
}

// Update structures (externally generated)
type lineUpdate struct {
	LineID string
	Index  int
	Times  []int
}

type stationUpdate struct {
	StationID string
	Lines     []lineUpdate
}

type update struct {
	Stops []stationUpdate
}

// This is the main system information; fill this as appropriate
var mainSystem system = system{
	Name:    "TEECOM Shuttle Service",
	Tagline: "San Francisco Bay Area",
	Stops: []station{
		station{
			Name: "TEECOM",
			ID:   "tee",
			Coord: coordinates{
				Lat: 10.5,
				Lon: 10.5,
			},
			Directions: [2]string{"Northbound", "Southbound"},
			Lines: [2]map[string]*line{
				map[string]*line{
					"sh": &line{
						Name:  "Shuttle",
						ID:    "sh",
						Color: "#ff0000",
					},
				},
			},
		},
	},

	TimeMax: 45,

	stopMap: make(map[string]*station),
}

// Where static files will be found
const staticDirectory string = "static"

func prepareSystem() {
	// Cache system IDs for future lookup.
	// No need to do any locking as the server hasn't
	// started up yet.
	for i := 0; i < len(mainSystem.Stops); i++ {
		stop := &mainSystem.Stops[i]
		mainSystem.stopMap[stop.ID] = stop
	}
}

func main() {
	log.Println("Starting server")
	prepareSystem()

	// Setup routing
	http.HandleFunc("/info", handleInfo)
	http.HandleFunc("/update", handleUpdate)
	http.HandleFunc("/stop", handleStopInfo)

	// Run server on port 8080
	if err := http.ListenAndServe(":8080", nil); err != nil {
		log.Fatal(err)
	}
}

// JSON encode all of the information
func handleInfo(w http.ResponseWriter, r *http.Request) {
	// Obtain a read lock for the system
	mainSystem.RLock()
	defer mainSystem.RUnlock()

	if err := json.NewEncoder(w).Encode(mainSystem); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintln(w, "Internal Server Error")
	}
}

func handleStopInfo(w http.ResponseWriter, r *http.Request) {
	// Check for valid GET parameters
	stopID := r.URL.Query()["id"]
	if stopID == nil || len(stopID) != 1 {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintln(w, "400 Bad Request: Missing stop ID")
		return
	}

	// Obtain a read lock for the system
	mainSystem.RLock()
	defer mainSystem.RUnlock()

	// Try to find the correct stop
	stop := mainSystem.stopMap[stopID[0]]
	if stop == nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "400 Bad Request: Invalid stop id (%s)\n", stopID[0])
		return
	}

	// Send the response
	if err := json.NewEncoder(w).Encode(stop); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintln(w, "Internal Server Error")
	}
}

// Handle update request
func handleUpdate(w http.ResponseWriter, r *http.Request) {
	// Ensure we are dealing with a POST request
	if r.Method != "POST" {
		serve(w, "update.html", http.StatusBadRequest)
		return
	}

	// Decode the JSON
	var new update
	if err := json.NewDecoder(r.Body).Decode(&new); err != nil {
		serve(w, "badupdate.html", http.StatusBadRequest)
		return
	}

	// Try to apply the updates
	if err := processUpdates(&new); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "400 Bad Request: %s\n", err.Error())
		return
	}
}

func processUpdates(u *update) error {
	// Obtain a writer lock
	mainSystem.Lock()
	defer mainSystem.Unlock()

	for _, su := range u.Stops {
		stop := mainSystem.stopMap[su.StationID]
		if stop == nil {
			return errors.New("Invalid station ID")
		}

		for _, lu := range su.Lines {
			if lu.Index > 1 {
				return errors.New("Line index out of bounds")
			}

			ln := stop.Lines[lu.Index][lu.LineID]
			if ln == nil {
				return errors.New("Invalid line ID")
			}

			ln.Times = lu.Times
		}
	}

	return nil
}

func serve(w http.ResponseWriter, f string, code int) {
	text, err := ioutil.ReadFile(staticDirectory + "/" + f)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintln(w, "500 Internal Server Error")
		return
	}

	w.WriteHeader(code)
	fmt.Fprintf(w, "%s", text)
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"regexp"
	"strings"

	"github.com/ymotongpoo/goltsv"
)

var KEYS_LIST = regexp.MustCompile(`(?:[^,\\]|\\.)*`)

func ParseKeysList(list string) []string {
	keys := KEYS_LIST.FindAllString(list, -1)
	for i := 0; i < len(keys); i++ {
		keys[i] = strings.Replace(keys[i], `\,`, `,`, -1)
		keys[i] = strings.Replace(keys[i], `\\`, `\`, -1)
	}
	return keys
}

type LTSVScanner struct {
	keys   []string
	line   string
	err    error
	reader *goltsv.LTSVReader
}

func NewLTSVScanner(keys []string, r io.Reader) *LTSVScanner {
	return &LTSVScanner{
		keys:   keys,
		reader: goltsv.NewReader(r),
	}
}

func (l *LTSVScanner) Scan() bool {
	if l.err != nil {
		return false
	}

	recode, err := l.reader.Read()
	if err != nil {
		l.err = err
		return false
	}

	var values []string
	for _, key := range l.keys {
		values = append(values, recode[key])
	}
	l.line = strings.Join(values, "\t")

	return true
}

func (l *LTSVScanner) Err() error {
	if l.err == io.EOF {
		return nil
	}
	return l.err
}

func (l *LTSVScanner) Text() string {
	return l.line
}
<commit_msg>Make LTSVScanner.Text returns "" after the end<commit_after>package main

import (
	"io"
	"regexp"
	"strings"

	"github.com/ymotongpoo/goltsv"
)

var KEYS_LIST = regexp.MustCompile(`(?:[^,\\]|\\.)*`)

func ParseKeysList(list string) []string {
	keys := KEYS_LIST.FindAllString(list, -1)
	for i := 0; i < len(keys); i++ {
		keys[i] = strings.Replace(keys[i], `\,`, `,`, -1)
		keys[i] = strings.Replace(keys[i], `\\`, `\`, -1)
	}
	return keys
}

type LTSVScanner struct {
	keys   []string
	line   string
	err    error
	reader *goltsv.LTSVReader
}

func NewLTSVScanner(keys []string, r io.Reader) *LTSVScanner {
	return &LTSVScanner{
		keys:   keys,
		reader: goltsv.NewReader(r),
	}
}

func (l *LTSVScanner) Scan() bool {
	if l.err != nil {
		return false
	}

	recode, err := l.reader.Read()
	if err != nil {
		l.err = err
		l.line = ""
		return false
	}

	var values []string
	for _, key := range l.keys {
		values = append(values, recode[key])
	}
	l.line = strings.Join(values, "\t")

	return true
}

func (l *LTSVScanner) Err() error {
	if l.err == io.EOF {
		return nil
	}
	return l.err
}

func (l *LTSVScanner) Text() string {
	return l.line
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package commitlog

import (
	"errors"
	"sync"
	"sync/atomic"
	"time"

	"github.com/m3db/m3db/clock"
	"github.com/m3db/m3db/context"
	"github.com/m3db/m3db/ts"
	"github.com/m3db/m3x/log"
	"github.com/m3db/m3x/time"

	"github.com/uber-go/tally"
)

var (
	errCommitLogClosed    = errors.New("commit log is closed")
	errCommitLogQueueFull = errors.New("commit log queue is full")

	timeZero = time.Time{}
)

type newCommitLogWriterFn func(flushFn flushFn, opts Options) commitLogWriter

type commitLogFailFn func(err error)

type completionFn func(err error)

type commitLog struct {
	sync.RWMutex
	opts    Options
	nowFn   clock.NowFn
	scope   tally.Scope
	metrics commitLogMetrics

	log xlog.Logger

	newCommitLogWriterFn newCommitLogWriterFn
	commitLogFailFn      commitLogFailFn
	writer               commitLogWriter

	// TODO(r): replace buffered channel with concurrent striped
	// circular buffer to avoid central write lock contention
	writes chan commitLogWrite

	flushMutex      sync.RWMutex
	lastFlushAt     time.Time
	pendingFlushFns []completionFn

	bitset         bitset
	writerExpireAt time.Time
	closed         bool
	closeErr       chan error
}

type commitLogMetrics struct {
	queued      tally.Gauge
	success     tally.Counter
	errors      tally.Counter
	openErrors  tally.Counter
	closeErrors tally.Counter
	flushErrors tally.Counter
	flushDone   tally.Counter
}

type valueType int

const (
	writeValueType valueType = iota
	flushValueType
)

type commitLogWrite struct {
	valueType valueType

	series       Series
	datapoint    ts.Datapoint
	unit         xtime.Unit
	annotation   ts.Annotation
	completionFn completionFn
}

// NewCommitLog creates a new commit log
func NewCommitLog(opts Options) CommitLog {
	iopts := opts.InstrumentOptions().SetMetricsScope(
		opts.InstrumentOptions().MetricsScope().SubScope("commitlog"))
	scope := iopts.MetricsScope()

	commitLog := &commitLog{
		opts:  opts,
		nowFn: opts.ClockOptions().NowFn(),
		scope: scope,
		metrics: commitLogMetrics{
			queued:      scope.Gauge("writes.queued"),
			success:     scope.Counter("writes.success"),
			errors:      scope.Counter("writes.errors"),
			openErrors:  scope.Counter("writes.open-errors"),
			closeErrors: scope.Counter("writes.close-errors"),
			flushErrors: scope.Counter("writes.flush-errors"),
			flushDone:   scope.Counter("writes.flush-done"),
		},
		log:                  iopts.Logger(),
		newCommitLogWriterFn: newCommitLogWriter,
		writes:               make(chan commitLogWrite, opts.BacklogQueueSize()),
		closeErr:             make(chan error),
	}

	return commitLog
}

func (l *commitLog) Open() error {
	// Open the buffered commit log writer
	if err := l.openWriter(l.nowFn()); err != nil {
		return err
	}

	// Flush the info header to ensure we can write to disk
	if err := l.writer.Flush(); err != nil {
		return err
	}

	// NB(r): In the future we can introduce a commit log failure policy
	// similar to Cassandra's "stop", for example see:
	// https://github.com/apache/cassandra/blob/6dfc1e7eeba539774784dfd650d3e1de6785c938/conf/cassandra.yaml#L232
	// Right now it is a large amount of coordination to implement something similiar.
	var fails int64

	l.commitLogFailFn = func(err error) {
		currFails := atomic.AddInt64(&fails, 1)

		if currFails < 3 {
			go func() {
				time.Sleep(time.Second)
				atomic.AddInt64(&fails, -1)
			}()

			return
		}

		l.log.Fatalf("fatal commit log error: %v", err)
	}

	// Asynchronously write
	go l.write()

	if flushInterval := l.opts.FlushInterval(); flushInterval > 0 {
		// Continually flush the commit log at given interval if set
		go l.flushEvery(flushInterval)
	}

	return nil
}

func (l *commitLog) flushEvery(interval time.Duration) {
	// Periodically flush the underlying commit log writer to cover
	// the case when writes stall for a considerable time
	var sleepForOverride time.Duration

	for {
		l.metrics.queued.Update(int64(len(l.writes)))

		sleepFor := interval

		if sleepForOverride > 0 {
			sleepFor = sleepForOverride
			sleepForOverride = 0
		}

		time.Sleep(sleepFor)

		l.flushMutex.RLock()
		lastFlushAt := l.lastFlushAt
		l.flushMutex.RUnlock()

		if sinceFlush := l.nowFn().Sub(lastFlushAt); sinceFlush < interval {
			// Flushed already recently, sleep until we would next consider flushing
			sleepForOverride = interval - sinceFlush
			continue
		}

		// Request a flush
		l.RLock()
		if l.closed {
			l.RUnlock()
			return
		}

		l.writes <- commitLogWrite{valueType: flushValueType}
		l.RUnlock()
	}
}

func (l *commitLog) write() {
	for write := range l.writes {
		// For writes requiring acks add to pending acks
		if write.completionFn != nil {
			l.pendingFlushFns = append(l.pendingFlushFns, write.completionFn)
		}

		if write.valueType == flushValueType {
			l.writer.Flush()
			continue
		}

		if now := l.nowFn(); !now.Before(l.writerExpireAt) {
			if err := l.openWriter(now); err != nil {
				l.metrics.errors.Inc(1)
				l.metrics.openErrors.Inc(1)
				l.log.Errorf("failed to open commit log: %v", err)

				if l.commitLogFailFn != nil {
					l.commitLogFailFn(err)
				}

				continue
			}
		}

		err := l.writer.Write(
			write.series, write.datapoint, write.unit, write.annotation)
		write.series.ID.OnClose()

		if err != nil {
			l.metrics.errors.Inc(1)
			l.log.Errorf("failed to write to commit log: %v", err)

			if l.commitLogFailFn != nil {
				l.commitLogFailFn(err)
			}

			continue
		}

		l.metrics.success.Inc(1)
	}

	l.Lock()
	defer l.Unlock()

	writer := l.writer
	l.writer = nil
	l.closeErr <- writer.Close()
}

func (l *commitLog) onFlush(err error) {
	l.flushMutex.Lock()
	l.lastFlushAt = l.nowFn()
	l.flushMutex.Unlock()

	if err != nil {
		l.metrics.errors.Inc(1)
		l.metrics.flushErrors.Inc(1)
		l.log.Errorf("failed to flush commit log: %v", err)

		if l.commitLogFailFn != nil {
			l.commitLogFailFn(err)
		}
	}

	// onFlush only ever called by "write()" and "openWriter" or
	// before "write()" begins on "Open()" and there are no other
	// accessors of "pendingFlushFns" so it is safe to read and mutate
	// without a lock here
	if len(l.pendingFlushFns) == 0 {
		l.metrics.flushDone.Inc(1)
		return
	}

	for i := range l.pendingFlushFns {
		l.pendingFlushFns[i](err)
	}

	l.pendingFlushFns = l.pendingFlushFns[:0]
	l.metrics.flushDone.Inc(1)
}

func (l *commitLog) openWriter(now time.Time) error {
	if l.writer != nil {
		if err := l.writer.Close(); err != nil {
			l.metrics.closeErrors.Inc(1)
			l.log.Errorf("failed to close commit log: %v", err)

			// If we failed to close then create a new commit log writer
			l.writer = nil
		}
	}

	if l.writer == nil {
		l.writer = l.newCommitLogWriterFn(l.onFlush, l.opts)
	}

	blockSize := l.opts.RetentionOptions().BlockSize()
	start := now.Truncate(blockSize)

	if err := l.writer.Open(start, blockSize); err != nil {
		return err
	}

	l.writerExpireAt = start.Add(blockSize)

	return nil
}

func (l *commitLog) Write(
	ctx context.Context,
	series Series,
	datapoint ts.Datapoint,
	unit xtime.Unit,
	annotation ts.Annotation,
) error {
	l.RLock()
	if l.closed {
		l.RUnlock()
		return errCommitLogClosed
	}

	var (
		wg     sync.WaitGroup
		result error
	)

	wg.Add(1)

	completion := func(err error) {
		result = err
		wg.Done()
	}

	write := commitLogWrite{
		series:       series,
		datapoint:    datapoint,
		unit:         unit,
		annotation:   annotation,
		completionFn: completion,
	}

	enqueued := false

	select {
	case l.writes <- write:
		enqueued = true
	default:
	}

	l.RUnlock()

	if !enqueued {
		return errCommitLogQueueFull
	}

	wg.Wait()

	return result
}

func (l *commitLog) WriteBehind(
	ctx context.Context,
	series Series,
	datapoint ts.Datapoint,
	unit xtime.Unit,
	annotation ts.Annotation,
) error {
	l.RLock()
	if l.closed {
		l.RUnlock()
		return errCommitLogClosed
	}

	write := commitLogWrite{
		series:     series,
		datapoint:  datapoint,
		unit:       unit,
		annotation: annotation,
	}

	enqueued := false

	select {
	case l.writes <- write:
		enqueued = true
	default:
	}

	l.RUnlock()

	if !enqueued {
		return errCommitLogQueueFull
	}

	return nil
}

func (l *commitLog) Iter() (Iterator, error) {
	return NewIterator(l.opts)
}

func (l *commitLog) Close() error {
	l.Lock()
	if l.closed {
		l.Unlock()
		return nil
	}

	l.closed = true
	close(l.writes)
	l.Unlock()

	// Receive the result of closing the writer from asynchronous writer
	return <-l.closeErr
}
<commit_msg>Halt on commit log failure instead of using sliding window of 3 failures (#131)<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package commitlog

import (
	"errors"
	"sync"
	"time"

	"github.com/m3db/m3db/clock"
	"github.com/m3db/m3db/context"
	"github.com/m3db/m3db/ts"
	"github.com/m3db/m3x/log"
	"github.com/m3db/m3x/time"

	"github.com/uber-go/tally"
)

var (
	errCommitLogClosed    = errors.New("commit log is closed")
	errCommitLogQueueFull = errors.New("commit log queue is full")

	timeZero = time.Time{}
)

type newCommitLogWriterFn func(flushFn flushFn, opts Options) commitLogWriter

type commitLogFailFn func(err error)

type completionFn func(err error)

type commitLog struct {
	sync.RWMutex
	opts    Options
	nowFn   clock.NowFn
	scope   tally.Scope
	metrics commitLogMetrics

	log xlog.Logger

	newCommitLogWriterFn newCommitLogWriterFn
	commitLogFailFn      commitLogFailFn
	writer               commitLogWriter

	// TODO(r): replace buffered channel with concurrent striped
	// circular buffer to avoid central write lock contention
	writes chan commitLogWrite

	flushMutex      sync.RWMutex
	lastFlushAt     time.Time
	pendingFlushFns []completionFn

	bitset         bitset
	writerExpireAt time.Time
	closed         bool
	closeErr       chan error
}

type commitLogMetrics struct {
	queued      tally.Gauge
	success     tally.Counter
	errors      tally.Counter
	openErrors  tally.Counter
	closeErrors tally.Counter
	flushErrors tally.Counter
	flushDone   tally.Counter
}

type valueType int

const (
	writeValueType valueType = iota
	flushValueType
)

type commitLogWrite struct {
	valueType valueType

	series       Series
	datapoint    ts.Datapoint
	unit         xtime.Unit
	annotation   ts.Annotation
	completionFn completionFn
}

// NewCommitLog creates a new commit log
func NewCommitLog(opts Options) CommitLog {
	iopts := opts.InstrumentOptions().SetMetricsScope(
		opts.InstrumentOptions().MetricsScope().SubScope("commitlog"))
	scope := iopts.MetricsScope()

	commitLog := &commitLog{
		opts:  opts,
		nowFn: opts.ClockOptions().NowFn(),
		scope: scope,
		metrics: commitLogMetrics{
			queued:      scope.Gauge("writes.queued"),
			success:     scope.Counter("writes.success"),
			errors:      scope.Counter("writes.errors"),
			openErrors:  scope.Counter("writes.open-errors"),
			closeErrors: scope.Counter("writes.close-errors"),
			flushErrors: scope.Counter("writes.flush-errors"),
			flushDone:   scope.Counter("writes.flush-done"),
		},
		log:                  iopts.Logger(),
		newCommitLogWriterFn: newCommitLogWriter,
		writes:               make(chan commitLogWrite, opts.BacklogQueueSize()),
		closeErr:             make(chan error),
	}

	return commitLog
}

func (l *commitLog) Open() error {
	// Open the buffered commit log writer
	if err := l.openWriter(l.nowFn()); err != nil {
		return err
	}

	// Flush the info header to ensure we can write to disk
	if err := l.writer.Flush(); err != nil {
		return err
	}

	// NB(r): In the future we can introduce a commit log failure policy
	// similar to Cassandra's "stop", for example see:
	// https://github.com/apache/cassandra/blob/6dfc1e7eeba539774784dfd650d3e1de6785c938/conf/cassandra.yaml#L232
	// Right now it is a large amount of coordination to implement something similiar.
	l.commitLogFailFn = func(err error) {
		l.log.Fatalf("fatal commit log error: %v", err)
	}

	// Asynchronously write
	go l.write()

	if flushInterval := l.opts.FlushInterval(); flushInterval > 0 {
		// Continually flush the commit log at given interval if set
		go l.flushEvery(flushInterval)
	}

	return nil
}

func (l *commitLog) flushEvery(interval time.Duration) {
	// Periodically flush the underlying commit log writer to cover
	// the case when writes stall for a considerable time
	var sleepForOverride time.Duration

	for {
		l.metrics.queued.Update(int64(len(l.writes)))

		sleepFor := interval

		if sleepForOverride > 0 {
			sleepFor = sleepForOverride
			sleepForOverride = 0
		}

		time.Sleep(sleepFor)

		l.flushMutex.RLock()
		lastFlushAt := l.lastFlushAt
		l.flushMutex.RUnlock()

		if sinceFlush := l.nowFn().Sub(lastFlushAt); sinceFlush < interval {
			// Flushed already recently, sleep until we would next consider flushing
			sleepForOverride = interval - sinceFlush
			continue
		}

		// Request a flush
		l.RLock()
		if l.closed {
			l.RUnlock()
			return
		}

		l.writes <- commitLogWrite{valueType: flushValueType}
		l.RUnlock()
	}
}

func (l *commitLog) write() {
	for write := range l.writes {
		// For writes requiring acks add to pending acks
		if write.completionFn != nil {
			l.pendingFlushFns = append(l.pendingFlushFns, write.completionFn)
		}

		if write.valueType == flushValueType {
			l.writer.Flush()
			continue
		}

		if now := l.nowFn(); !now.Before(l.writerExpireAt) {
			if err := l.openWriter(now); err != nil {
				l.metrics.errors.Inc(1)
				l.metrics.openErrors.Inc(1)
				l.log.Errorf("failed to open commit log: %v", err)

				if l.commitLogFailFn != nil {
					l.commitLogFailFn(err)
				}

				continue
			}
		}

		err := l.writer.Write(
			write.series, write.datapoint, write.unit, write.annotation)
		write.series.ID.OnClose()

		if err != nil {
			l.metrics.errors.Inc(1)
			l.log.Errorf("failed to write to commit log: %v", err)

			if l.commitLogFailFn != nil {
				l.commitLogFailFn(err)
			}

			continue
		}

		l.metrics.success.Inc(1)
	}

	l.Lock()
	defer l.Unlock()

	writer := l.writer
	l.writer = nil
	l.closeErr <- writer.Close()
}

func (l *commitLog) onFlush(err error) {
	l.flushMutex.Lock()
	l.lastFlushAt = l.nowFn()
	l.flushMutex.Unlock()

	if err != nil {
		l.metrics.errors.Inc(1)
		l.metrics.flushErrors.Inc(1)
		l.log.Errorf("failed to flush commit log: %v", err)

		if l.commitLogFailFn != nil {
			l.commitLogFailFn(err)
		}
	}

	// onFlush only ever called by "write()" and "openWriter" or
	// before "write()" begins on "Open()" and there are no other
	// accessors of "pendingFlushFns" so it is safe to read and mutate
	// without a lock here
	if len(l.pendingFlushFns) == 0 {
		l.metrics.flushDone.Inc(1)
		return
	}

	for i := range l.pendingFlushFns {
		l.pendingFlushFns[i](err)
	}

	l.pendingFlushFns = l.pendingFlushFns[:0]
	l.metrics.flushDone.Inc(1)
}

func (l *commitLog) openWriter(now time.Time) error {
	if l.writer != nil {
		if err := l.writer.Close(); err != nil {
			l.metrics.closeErrors.Inc(1)
			l.log.Errorf("failed to close commit log: %v", err)

			// If we failed to close then create a new commit log writer
			l.writer = nil
		}
	}

	if l.writer == nil {
		l.writer = l.newCommitLogWriterFn(l.onFlush, l.opts)
	}

	blockSize := l.opts.RetentionOptions().BlockSize()
	start := now.Truncate(blockSize)

	if err := l.writer.Open(start, blockSize); err != nil {
		return err
	}

	l.writerExpireAt = start.Add(blockSize)

	return nil
}

func (l *commitLog) Write(
	ctx context.Context,
	series Series,
	datapoint ts.Datapoint,
	unit xtime.Unit,
	annotation ts.Annotation,
) error {
	l.RLock()
	if l.closed {
		l.RUnlock()
		return errCommitLogClosed
	}

	var (
		wg     sync.WaitGroup
		result error
	)

	wg.Add(1)

	completion := func(err error) {
		result = err
		wg.Done()
	}

	write := commitLogWrite{
		series:       series,
		datapoint:    datapoint,
		unit:         unit,
		annotation:   annotation,
		completionFn: completion,
	}

	enqueued := false

	select {
	case l.writes <- write:
		enqueued = true
	default:
	}

	l.RUnlock()

	if !enqueued {
		return errCommitLogQueueFull
	}

	wg.Wait()

	return result
}

func (l *commitLog) WriteBehind(
	ctx context.Context,
	series Series,
	datapoint ts.Datapoint,
	unit xtime.Unit,
	annotation ts.Annotation,
) error {
	l.RLock()
	if l.closed {
		l.RUnlock()
		return errCommitLogClosed
	}

	write := commitLogWrite{
		series:     series,
		datapoint:  datapoint,
		unit:       unit,
		annotation: annotation,
	}

	enqueued := false

	select {
	case l.writes <- write:
		enqueued = true
	default:
	}

	l.RUnlock()

	if !enqueued {
		return errCommitLogQueueFull
	}

	return nil
}

func (l *commitLog) Iter() (Iterator, error) {
	return NewIterator(l.opts)
}

func (l *commitLog) Close() error {
	l.Lock()
	if l.closed {
		l.Unlock()
		return nil
	}

	l.closed = true
	close(l.writes)
	l.Unlock()

	// Receive the result of closing the writer from asynchronous writer
	return <-l.closeErr
}
<|endoftext|>


<commit_before>package iter

// First is useful when you only expect one (1) item in the iterator.
//
// Note that it will return an error if there is more than one (1) item
// in the iterator. (It checks for this.)
//
// Example Usage
//
//	iterator := ...
//
//	err := iter.First{Iterator: iterator}.Decode(v)
//	if nil != err {
//		switch {
//		case iter.EmptyIteratorComplainer:
//			//@TODO
//		default:
//			return err
//		}
//	}
type First struct {
	Iterator Iterator
}

func (receiver First) Decode(v interface{}) error {
	return receiver.decode(v)
}

func (receiver First) decode(v interface{}) (err error) {

	iterator := receiver.Iterator
	if nil == iterator {
		return errNilIterator
	}
	defer func(i Iterator){
		if e := i.Close(); nil != e {
			if nil == err {
				err = e
			}
		}
	}(iterator)

	if ! iterator.Next() {
		return errEmptyIterator
	}
	if err := iterator.Decode(v); nil != err {
		return err
	}
	if iterator.Next() {
		return errTooManyIterations
	}
	if err := iterator.Err(); nil != err {
		return err
	}

	return nil
}
<commit_msg>removed the "too many iterations" check.<commit_after>package iter

// First is useful when you only expect one (1) item in the iterator.
//
// Note that it will return an error if there is more than one (1) item
// in the iterator. (It checks for this.)
//
// Example Usage
//
//	iterator := ...
//
//	err := iter.First{Iterator: iterator}.Decode(v)
//	if nil != err {
//		switch {
//		case iter.EmptyIteratorComplainer:
//			//@TODO
//		default:
//			return err
//		}
//	}
type First struct {
	Iterator Iterator
}

func (receiver First) Decode(v interface{}) error {
	return receiver.decode(v)
}

func (receiver First) decode(v interface{}) (err error) {

	iterator := receiver.Iterator
	if nil == iterator {
		return errNilIterator
	}
	defer func(i Iterator){
		if e := i.Close(); nil != e {
			if nil == err {
				err = e
			}
		}
	}(iterator)

	if ! iterator.Next() {
		return errEmptyIterator
	}
	if err := iterator.Decode(v); nil != err {
		return err
	}
	if err := iterator.Err(); nil != err {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/rand"
	"encoding/base64"
	"flag"
	"io"
	"log"
	"os"
	"strconv"

	"github.com/flynn/go-discover/discover"
	"github.com/flynn/lorne/types"
	"github.com/flynn/rpcplus"
	"github.com/flynn/sampi/types"
	"github.com/titanous/go-dockerclient"
)

var state = NewState()
var Docker *docker.Client

func main() {
	externalAddr := flag.String("external", "", "external IP of host")
	configFile := flag.String("config", "", "configuration file")
	hostID := flag.String("id", randomID(), "host id")
	flag.Parse()

	go server()
	go allocatePorts()

	disc, err := discover.NewClient()
	if err != nil {
		log.Fatal(err)
	}
	if err := disc.Register("flynn-lorne."+*hostID, "1113", nil); err != nil {
		log.Fatal(err)
	}

	services, err := disc.Services("flynn-sampi")
	if err != nil {
		log.Fatal(err)
	}
	schedulers := services.Online()
	if len(schedulers) == 0 {
		log.Fatal("No sampi instances found")
	}

	scheduler, err := rpcplus.DialHTTP("tcp", schedulers[0].Addr)
	if err != nil {
		log.Fatal(err)
	}
	log.Print("Connected to scheduler")

	Docker, err = docker.NewClient("http://localhost:4243")
	if err != nil {
		log.Fatal(err)
	}

	go streamEvents(Docker)
	go syncScheduler(scheduler)

	var host *sampi.Host
	if *configFile != "" {
		host, err = parseConfig(*configFile)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		host = &sampi.Host{Resources: make(map[string]sampi.ResourceValue)}
	}
	if _, ok := host.Resources["memory"]; !ok {
		host.Resources["memory"] = sampi.ResourceValue{Value: 1024}
	}
	host.ID = *hostID

	jobs := make(chan *sampi.Job)
	scheduler.StreamGo("Scheduler.RegisterHost", host, jobs)
	log.Print("Host registered")
	for job := range jobs {
		log.Printf("%#v", job.Config)
		var hostConfig *docker.HostConfig
		if job.TCPPorts > 0 {
			port := strconv.Itoa(<-portAllocator)
			job.Config.Env = append(job.Config.Env, "PORT="+port)
			job.Config.ExposedPorts = map[string]struct{}{port + "/tcp": struct{}{}}
			job.Config.Name = "flynn-" + job.ID
			hostConfig = &docker.HostConfig{
				PortBindings:    map[string][]docker.PortBinding{port + "/tcp": {{HostPort: port}}},
				PublishAllPorts: true,
			}
		}
		if *externalAddr != "" {
			job.Config.Env = append(job.Config.Env, "EXTERNAL_IP="+*externalAddr, "SD_HOST="+*externalAddr, "DISCOVERD="+*externalAddr+":1111")
		}
		state.AddJob(job)
		container, err := Docker.CreateContainer(job.Config)
		if err == docker.ErrNoSuchImage {
			err = Docker.PullImage(docker.PullImageOptions{Repository: job.Config.Image}, os.Stdout)
			if err != nil {
				log.Fatal(err)
			}
			container, err = Docker.CreateContainer(job.Config)
		}
		if err != nil {
			log.Fatal(err)
		}
		state.SetContainerID(job.ID, container.ID)
		state.WaitAttach(job.ID)
		if err := Docker.StartContainer(container.ID, hostConfig); err != nil {
			log.Fatal(err)
		}
		state.SetStatusRunning(job.ID)
	}
}

func syncScheduler(client *rpcplus.Client) {
	events := make(chan lorne.Event)
	state.AddListener("all", events)
	for event := range events {
		if event.Event != "stop" {
			continue
		}
		log.Println("remove job", event.JobID)
		if err := client.Call("Scheduler.RemoveJobs", []string{event.JobID}, &struct{}{}); err != nil {
			log.Println("remove job", event.JobID, "error:", err)
			// TODO: try to reconnect?
		}
	}
}

func streamEvents(client *docker.Client) {
	stream, err := client.Events()
	if err != nil {
		log.Fatal(err)
	}
	for event := range stream.Events {
		log.Printf("%#v", event)
		if event.Status != "die" {
			continue
		}
		container, err := client.InspectContainer(event.ID)
		if err != nil {
			log.Println("inspect container", event.ID, "error:", err)
			// TODO: set job status anyway?
			continue
		}
		state.SetStatusDone(event.ID, container.State.ExitCode)
	}
	log.Println("events done", stream.Error)
}

func randomID() string {
	b := make([]byte, 16)
	enc := make([]byte, 24)
	_, err := io.ReadFull(rand.Reader, b)
	if err != nil {
		panic(err) // This shouldn't ever happen, right?
	}
	base64.URLEncoding.Encode(enc, b)
	return string(bytes.TrimRight(enc, "="))
}

var portAllocator = make(chan int)

// TODO: fix this, horribly broken
const startPort = 55000
const endPort = 65535

func allocatePorts() {
	for i := startPort; i < endPort; i++ {
		portAllocator <- i
	}
	// TODO: handle wrap-around
}
<commit_msg>host: Use sampi client<commit_after>package main

import (
	"bytes"
	"crypto/rand"
	"encoding/base64"
	"flag"
	"io"
	"log"
	"os"
	"strconv"

	"github.com/flynn/go-discover/discover"
	"github.com/flynn/lorne/types"
	sampic "github.com/flynn/sampi/client"
	"github.com/flynn/sampi/types"
	"github.com/titanous/go-dockerclient"
)

var state = NewState()
var Docker *docker.Client

func main() {
	externalAddr := flag.String("external", "", "external IP of host")
	configFile := flag.String("config", "", "configuration file")
	hostID := flag.String("id", randomID(), "host id")
	flag.Parse()

	go server()
	go allocatePorts()

	disc, err := discover.NewClient()
	if err != nil {
		log.Fatal(err)
	}
	if err := disc.Register("flynn-lorne."+*hostID, "1113", nil); err != nil {
		log.Fatal(err)
	}

	scheduler, err := sampic.New()
	if err != nil {
		log.Fatal(err)
	}
	log.Print("Connected to scheduler")

	Docker, err = docker.NewClient("http://localhost:4243")
	if err != nil {
		log.Fatal(err)
	}

	go streamEvents(Docker)
	go syncScheduler(scheduler)

	var host *sampi.Host
	if *configFile != "" {
		host, err = parseConfig(*configFile)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		host = &sampi.Host{Resources: make(map[string]sampi.ResourceValue)}
	}
	if _, ok := host.Resources["memory"]; !ok {
		host.Resources["memory"] = sampi.ResourceValue{Value: 1024}
	}
	host.ID = *hostID

	jobs := make(chan *sampi.Job)
	scheduler.RegisterHost(host, jobs)
	log.Print("Host registered")
	for job := range jobs {
		log.Printf("%#v", job.Config)
		var hostConfig *docker.HostConfig
		if job.TCPPorts > 0 {
			port := strconv.Itoa(<-portAllocator)
			job.Config.Env = append(job.Config.Env, "PORT="+port)
			job.Config.ExposedPorts = map[string]struct{}{port + "/tcp": struct{}{}}
			job.Config.Name = "flynn-" + job.ID
			hostConfig = &docker.HostConfig{
				PortBindings:    map[string][]docker.PortBinding{port + "/tcp": {{HostPort: port}}},
				PublishAllPorts: true,
			}
		}
		if *externalAddr != "" {
			job.Config.Env = append(job.Config.Env, "EXTERNAL_IP="+*externalAddr, "SD_HOST="+*externalAddr, "DISCOVERD="+*externalAddr+":1111")
		}
		state.AddJob(job)
		container, err := Docker.CreateContainer(job.Config)
		if err == docker.ErrNoSuchImage {
			err = Docker.PullImage(docker.PullImageOptions{Repository: job.Config.Image}, os.Stdout)
			if err != nil {
				log.Fatal(err)
			}
			container, err = Docker.CreateContainer(job.Config)
		}
		if err != nil {
			log.Fatal(err)
		}
		state.SetContainerID(job.ID, container.ID)
		state.WaitAttach(job.ID)
		if err := Docker.StartContainer(container.ID, hostConfig); err != nil {
			log.Fatal(err)
		}
		state.SetStatusRunning(job.ID)
	}
}

func syncScheduler(scheduler *sampic.Client) {
	events := make(chan lorne.Event)
	state.AddListener("all", events)
	for event := range events {
		if event.Event != "stop" {
			continue
		}
		log.Println("remove job", event.JobID)
		if err := scheduler.RemoveJobs([]string{event.JobID}); err != nil {
			log.Println("remove job", event.JobID, "error:", err)
			// TODO: try to reconnect?
		}
	}
}

func streamEvents(client *docker.Client) {
	stream, err := client.Events()
	if err != nil {
		log.Fatal(err)
	}
	for event := range stream.Events {
		log.Printf("%#v", event)
		if event.Status != "die" {
			continue
		}
		container, err := client.InspectContainer(event.ID)
		if err != nil {
			log.Println("inspect container", event.ID, "error:", err)
			// TODO: set job status anyway?
			continue
		}
		state.SetStatusDone(event.ID, container.State.ExitCode)
	}
	log.Println("events done", stream.Error)
}

func randomID() string {
	b := make([]byte, 16)
	enc := make([]byte, 24)
	_, err := io.ReadFull(rand.Reader, b)
	if err != nil {
		panic(err) // This shouldn't ever happen, right?
	}
	base64.URLEncoding.Encode(enc, b)
	return string(bytes.TrimRight(enc, "="))
}

var portAllocator = make(chan int)

// TODO: fix this, horribly broken
const startPort = 55000
const endPort = 65535

func allocatePorts() {
	for i := startPort; i < endPort; i++ {
		portAllocator <- i
	}
	// TODO: handle wrap-around
}
<|endoftext|>


<commit_before>// Copyright 2012 Jesse van den Kieboom. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package flags provides an extensive command line option parser.
The flags package is similar in functionality to the go built-in flag package
but provides more options and uses reflection to provide a convenient and
succinct way of specifying command line options.

Supported features

The following features are supported in go-flags:

    Options with short names (-v)
    Options with long names (--verbose)
    Options with and without arguments (bool v.s. other type)
    Options with optional arguments and default values
    Option default values from ENVIRONMENT_VARIABLES, including slice and map values
    Multiple option groups each containing a set of options
    Generate and print well-formatted help message
    Passing remaining command line arguments after -- (optional)
    Ignoring unknown command line options (optional)
    Supports -I/usr/include -I=/usr/include -I /usr/include option argument specification
    Supports multiple short options -aux
    Supports all primitive go types (string, int{8..64}, uint{8..64}, float)
    Supports same option multiple times (can store in slice or last option counts)
    Supports maps
    Supports function callbacks
    Supports namespaces for (nested) option groups

Additional features specific to Windows:
    Options with short names (/v)
    Options with long names (/verbose)
    Windows-style options with arguments use a colon as the delimiter
    Modify generated help message with Windows-style / options
    Windows style options can be disabled at build time using the "forceposix"
    build tag

Basic usage

The flags package uses structs, reflection and struct field tags
to allow users to specify command line options. This results in very simple
and concise specification of your application options. For example:

    type Options struct {
        Verbose []bool `short:"v" long:"verbose" description:"Show verbose debug information"`
    }

This specifies one option with a short name -v and a long name --verbose.
When either -v or --verbose is found on the command line, a 'true' value
will be appended to the Verbose field. e.g. when specifying -vvv, the
resulting value of Verbose will be {[true, true, true]}.

Slice options work exactly the same as primitive type options, except that
whenever the option is encountered, a value is appended to the slice.

Map options from string to primitive type are also supported. On the command
line, you specify the value for such an option as key:value. For example

    type Options struct {
        AuthorInfo string[string] `short:"a"`
    }

Then, the AuthorInfo map can be filled with something like
-a name:Jesse -a "surname:van den Kieboom".

Finally, for full control over the conversion between command line argument
values and options, user defined types can choose to implement the Marshaler
and Unmarshaler interfaces.

Available field tags

The following is a list of tags for struct fields supported by go-flags:

    short:            the short name of the option (single character)
    long:             the long name of the option
    required:         if non empty, makes the option required to appear on the command
                      line. If a required option is not present, the parser will
                      return ErrRequired (optional)
    description:      the description of the option (optional)
    long-description: the long description of the option. Currently only
                      displayed in generated man pages (optional)
    no-flag:          if non-empty, this field is ignored as an option (optional)

    optional:       if non-empty, makes the argument of the option optional. When an
                    argument is optional it can only be specified using
                    --option=argument (optional)
    optional-value: the value of an optional option when the option occurs
                    without an argument. This tag can be specified multiple
                    times in the case of maps or slices (optional)
    default:        the default value of an option. This tag can be specified
                    multiple times in the case of slices or maps (optional)
    default-mask:   when specified, this value will be displayed in the help
                    instead of the actual default value. This is useful
                    mostly for hiding otherwise sensitive information from
                    showing up in the help. If default-mask takes the special
                    value "-", then no default value will be shown at all
                    (optional)
    env:            the default value of the option is overridden from the
                    specified environment variable, if one has been defined.
                    (optional)
    env-delim:      the 'env' default value from environment is split into
                    multiple values with the given delimiter string, use with
                    slices and maps (optional)
    value-name:     the name of the argument value (to be shown in the help)
                    (optional)
    choice:         limits the values for an option to a set of values.
                    This tag can be specified multiple times (optional)
    hidden:         if non-empty, the option is not visible in the help or man page.

    base: a base (radix) used to convert strings to integer values, the
          default base is 10 (i.e. decimal) (optional)

    ini-name:       the explicit ini option name (optional)
    no-ini:         if non-empty this field is ignored as an ini option
                    (optional)

    group:                when specified on a struct field, makes the struct
                          field a separate group with the given name (optional)
    namespace:            when specified on a group struct field, the namespace
                          gets prepended to every option's long name and
                          subgroup's namespace of this group, separated by
                          the parser's namespace delimiter (optional)
    env-namespace:        when specified on a group struct field, the env-namespace
                          gets prepended to every option's env key and
                          subgroup's env-namespace of this group, separated by
                          the parser's env-namespace delimiter (optional)
    command:              when specified on a struct field, makes the struct
                          field a (sub)command with the given name (optional)
    subcommands-optional: when specified on a command struct field, makes
                          any subcommands of that command optional (optional)
    alias:                when specified on a command struct field, adds the
                          specified name as an alias for the command. Can be
                          be specified multiple times to add more than one
                          alias (optional)
    positional-args:      when specified on a field with a struct type,
                          uses the fields of that struct to parse remaining
                          positional command line arguments into (in order
                          of the fields). If a field has a slice type,
                          then all remaining arguments will be added to it.
                          Positional arguments are optional by default,
                          unless the "required" tag is specified together
                          with the "positional-args" tag. The "required" tag
                          can also be set on the individual rest argument
                          fields, to require only the first N positional
                          arguments. If the "required" tag is set on the
                          rest arguments slice, then its value determines
                          the minimum amount of rest arguments that needs to
                          be provided (e.g. `required:"2"`) (optional)
    positional-arg-name:  used on a field in a positional argument struct; name
                          of the positional argument placeholder to be shown in
                          the help (optional)

Either the `short:` tag or the `long:` must be specified to make the field eligible as an
option.

Option groups

Option groups are a simple way to semantically separate your options. All
options in a particular group are shown together in the help under the name
of the group. Namespaces can be used to specify option long names more
precisely and emphasize the options affiliation to their group.

There are currently three ways to specify option groups.

    1. Use NewNamedParser specifying the various option groups.
    2. Use AddGroup to add a group to an existing parser.
    3. Add a struct field to the top-level options annotated with the
       group:"group-name" tag.

Commands

The flags package also has basic support for commands. Commands are often
used in monolithic applications that support various commands or actions.
Take git for example, all of the add, commit, checkout, etc. are called
commands. Using commands you can easily separate multiple functions of your
application.

There are currently two ways to specify a command.

    1. Use AddCommand on an existing parser.
    2. Add a struct field to your options struct annotated with the
       command:"command-name" tag.

The most common, idiomatic way to implement commands is to define a global
parser instance and implement each command in a separate file. These
command files should define a go init function which calls AddCommand on
the global parser.

When parsing ends and there is an active command and that command implements
the Commander interface, then its Execute method will be run with the
remaining command line arguments.

Command structs can have options which become valid to parse after the
command has been specified on the command line, in addition to the options
of all the parent commands. I.e. considering a -v flag on the parser and an
add command, the following are equivalent:

    ./app -v add
    ./app add -v

However, if the -v flag is defined on the add command, then the first of
the two examples above would fail since the -v flag is not defined before
the add command.

Completion

go-flags has builtin support to provide bash completion of flags, commands
and argument values. To use completion, the binary which uses go-flags
can be invoked in a special environment to list completion of the current
command line argument. It should be noted that this `executes` your application,
and it is up to the user to make sure there are no negative side effects (for
example from init functions).

Setting the environment variable `GO_FLAGS_COMPLETION=1` enables completion
by replacing the argument parsing routine with the completion routine which
outputs completions for the passed arguments. The basic invocation to
complete a set of arguments is therefore:

    GO_FLAGS_COMPLETION=1 ./completion-example arg1 arg2 arg3

where `completion-example` is the binary, `arg1` and `arg2` are
the current arguments, and `arg3` (the last argument) is the argument
to be completed. If the GO_FLAGS_COMPLETION is set to "verbose", then
descriptions of possible completion items will also be shown, if there
are more than 1 completion items.

To use this with bash completion, a simple file can be written which
calls the binary which supports go-flags completion:

    _completion_example() {
        # All arguments except the first one
        args=("${COMP_WORDS[@]:1:$COMP_CWORD}")

        # Only split on newlines
        local IFS=$'\n'

        # Call completion (note that the first element of COMP_WORDS is
        # the executable itself)
        COMPREPLY=($(GO_FLAGS_COMPLETION=1 ${COMP_WORDS[0]} "${args[@]}"))
        return 0
    }

    complete -F _completion_example completion-example

Completion requires the parser option PassDoubleDash and is therefore enforced if the environment variable GO_FLAGS_COMPLETION is set.

Customized completion for argument values is supported by implementing
the flags.Completer interface for the argument value type. An example
of a type which does so is the flags.Filename type, an alias of string
allowing simple filename completion. A slice or array argument value
whose element type implements flags.Completer will also be completed.
*/
package flags
<commit_msg>Explain how to use choices<commit_after>// Copyright 2012 Jesse van den Kieboom. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package flags provides an extensive command line option parser.
The flags package is similar in functionality to the go built-in flag package
but provides more options and uses reflection to provide a convenient and
succinct way of specifying command line options.

Supported features

The following features are supported in go-flags:

    Options with short names (-v)
    Options with long names (--verbose)
    Options with and without arguments (bool v.s. other type)
    Options with optional arguments and default values
    Option default values from ENVIRONMENT_VARIABLES, including slice and map values
    Multiple option groups each containing a set of options
    Generate and print well-formatted help message
    Passing remaining command line arguments after -- (optional)
    Ignoring unknown command line options (optional)
    Supports -I/usr/include -I=/usr/include -I /usr/include option argument specification
    Supports multiple short options -aux
    Supports all primitive go types (string, int{8..64}, uint{8..64}, float)
    Supports same option multiple times (can store in slice or last option counts)
    Supports maps
    Supports function callbacks
    Supports namespaces for (nested) option groups

Additional features specific to Windows:
    Options with short names (/v)
    Options with long names (/verbose)
    Windows-style options with arguments use a colon as the delimiter
    Modify generated help message with Windows-style / options
    Windows style options can be disabled at build time using the "forceposix"
    build tag

Basic usage

The flags package uses structs, reflection and struct field tags
to allow users to specify command line options. This results in very simple
and concise specification of your application options. For example:

    type Options struct {
        Verbose []bool `short:"v" long:"verbose" description:"Show verbose debug information"`
    }

This specifies one option with a short name -v and a long name --verbose.
When either -v or --verbose is found on the command line, a 'true' value
will be appended to the Verbose field. e.g. when specifying -vvv, the
resulting value of Verbose will be {[true, true, true]}.

Slice options work exactly the same as primitive type options, except that
whenever the option is encountered, a value is appended to the slice.

Map options from string to primitive type are also supported. On the command
line, you specify the value for such an option as key:value. For example

    type Options struct {
        AuthorInfo string[string] `short:"a"`
    }

Then, the AuthorInfo map can be filled with something like
-a name:Jesse -a "surname:van den Kieboom".

Finally, for full control over the conversion between command line argument
values and options, user defined types can choose to implement the Marshaler
and Unmarshaler interfaces.

Available field tags

The following is a list of tags for struct fields supported by go-flags:

    short:            the short name of the option (single character)
    long:             the long name of the option
    required:         if non empty, makes the option required to appear on the command
                      line. If a required option is not present, the parser will
                      return ErrRequired (optional)
    description:      the description of the option (optional)
    long-description: the long description of the option. Currently only
                      displayed in generated man pages (optional)
    no-flag:          if non-empty, this field is ignored as an option (optional)

    optional:       if non-empty, makes the argument of the option optional. When an
                    argument is optional it can only be specified using
                    --option=argument (optional)
    optional-value: the value of an optional option when the option occurs
                    without an argument. This tag can be specified multiple
                    times in the case of maps or slices (optional)
    default:        the default value of an option. This tag can be specified
                    multiple times in the case of slices or maps (optional)
    default-mask:   when specified, this value will be displayed in the help
                    instead of the actual default value. This is useful
                    mostly for hiding otherwise sensitive information from
                    showing up in the help. If default-mask takes the special
                    value "-", then no default value will be shown at all
                    (optional)
    env:            the default value of the option is overridden from the
                    specified environment variable, if one has been defined.
                    (optional)
    env-delim:      the 'env' default value from environment is split into
                    multiple values with the given delimiter string, use with
                    slices and maps (optional)
    value-name:     the name of the argument value (to be shown in the help)
                    (optional)
    choice:         limits the values for an option to a set of values.
                    Repeat this tag once for each allowable value.
                    e.g. `long:"animal" choice:"cat" choice:"dog"`
    hidden:         if non-empty, the option is not visible in the help or man page.

    base: a base (radix) used to convert strings to integer values, the
          default base is 10 (i.e. decimal) (optional)

    ini-name:       the explicit ini option name (optional)
    no-ini:         if non-empty this field is ignored as an ini option
                    (optional)

    group:                when specified on a struct field, makes the struct
                          field a separate group with the given name (optional)
    namespace:            when specified on a group struct field, the namespace
                          gets prepended to every option's long name and
                          subgroup's namespace of this group, separated by
                          the parser's namespace delimiter (optional)
    env-namespace:        when specified on a group struct field, the env-namespace
                          gets prepended to every option's env key and
                          subgroup's env-namespace of this group, separated by
                          the parser's env-namespace delimiter (optional)
    command:              when specified on a struct field, makes the struct
                          field a (sub)command with the given name (optional)
    subcommands-optional: when specified on a command struct field, makes
                          any subcommands of that command optional (optional)
    alias:                when specified on a command struct field, adds the
                          specified name as an alias for the command. Can be
                          be specified multiple times to add more than one
                          alias (optional)
    positional-args:      when specified on a field with a struct type,
                          uses the fields of that struct to parse remaining
                          positional command line arguments into (in order
                          of the fields). If a field has a slice type,
                          then all remaining arguments will be added to it.
                          Positional arguments are optional by default,
                          unless the "required" tag is specified together
                          with the "positional-args" tag. The "required" tag
                          can also be set on the individual rest argument
                          fields, to require only the first N positional
                          arguments. If the "required" tag is set on the
                          rest arguments slice, then its value determines
                          the minimum amount of rest arguments that needs to
                          be provided (e.g. `required:"2"`) (optional)
    positional-arg-name:  used on a field in a positional argument struct; name
                          of the positional argument placeholder to be shown in
                          the help (optional)

Either the `short:` tag or the `long:` must be specified to make the field eligible as an
option.

Option groups

Option groups are a simple way to semantically separate your options. All
options in a particular group are shown together in the help under the name
of the group. Namespaces can be used to specify option long names more
precisely and emphasize the options affiliation to their group.

There are currently three ways to specify option groups.

    1. Use NewNamedParser specifying the various option groups.
    2. Use AddGroup to add a group to an existing parser.
    3. Add a struct field to the top-level options annotated with the
       group:"group-name" tag.

Commands

The flags package also has basic support for commands. Commands are often
used in monolithic applications that support various commands or actions.
Take git for example, all of the add, commit, checkout, etc. are called
commands. Using commands you can easily separate multiple functions of your
application.

There are currently two ways to specify a command.

    1. Use AddCommand on an existing parser.
    2. Add a struct field to your options struct annotated with the
       command:"command-name" tag.

The most common, idiomatic way to implement commands is to define a global
parser instance and implement each command in a separate file. These
command files should define a go init function which calls AddCommand on
the global parser.

When parsing ends and there is an active command and that command implements
the Commander interface, then its Execute method will be run with the
remaining command line arguments.

Command structs can have options which become valid to parse after the
command has been specified on the command line, in addition to the options
of all the parent commands. I.e. considering a -v flag on the parser and an
add command, the following are equivalent:

    ./app -v add
    ./app add -v

However, if the -v flag is defined on the add command, then the first of
the two examples above would fail since the -v flag is not defined before
the add command.

Completion

go-flags has builtin support to provide bash completion of flags, commands
and argument values. To use completion, the binary which uses go-flags
can be invoked in a special environment to list completion of the current
command line argument. It should be noted that this `executes` your application,
and it is up to the user to make sure there are no negative side effects (for
example from init functions).

Setting the environment variable `GO_FLAGS_COMPLETION=1` enables completion
by replacing the argument parsing routine with the completion routine which
outputs completions for the passed arguments. The basic invocation to
complete a set of arguments is therefore:

    GO_FLAGS_COMPLETION=1 ./completion-example arg1 arg2 arg3

where `completion-example` is the binary, `arg1` and `arg2` are
the current arguments, and `arg3` (the last argument) is the argument
to be completed. If the GO_FLAGS_COMPLETION is set to "verbose", then
descriptions of possible completion items will also be shown, if there
are more than 1 completion items.

To use this with bash completion, a simple file can be written which
calls the binary which supports go-flags completion:

    _completion_example() {
        # All arguments except the first one
        args=("${COMP_WORDS[@]:1:$COMP_CWORD}")

        # Only split on newlines
        local IFS=$'\n'

        # Call completion (note that the first element of COMP_WORDS is
        # the executable itself)
        COMPREPLY=($(GO_FLAGS_COMPLETION=1 ${COMP_WORDS[0]} "${args[@]}"))
        return 0
    }

    complete -F _completion_example completion-example

Completion requires the parser option PassDoubleDash and is therefore enforced if the environment variable GO_FLAGS_COMPLETION is set.

Customized completion for argument values is supported by implementing
the flags.Completer interface for the argument value type. An example
of a type which does so is the flags.Filename type, an alias of string
allowing simple filename completion. A slice or array argument value
whose element type implements flags.Completer will also be completed.
*/
package flags
<|endoftext|>


<commit_before>package main

import "github.com/codegangsta/cli"

var (
	amqpURL            string
	amqpExchange       string
	amqpCARoot         string
	customDNSResolvers string
	host               string
	originRegexp       string
	port               int
	development        bool
	hostFlag           = cli.StringFlag{
		Name:        "bind, b",
		Value:       "",
		Usage:       "host to bind the ws server",
		EnvVar:      "WS_BIND_HOST",
		Destination: &host,
	}
	portFlag = cli.IntFlag{
		Name:        "port, p",
		Value:       9090,
		Usage:       "port to listen on",
		EnvVar:      "WS_BIND_PORT",
		Destination: &port,
	}
	originRegexpFlag = cli.StringFlag{
		Name:        "allow-origin-regexp, r",
		Value:       "",
		Usage:       "a regexp string used to validate request origin header",
		EnvVar:      "WS_ALLOW_ORIGIN_REGEXP",
		Destination: &originRegexp,
	}
	amqpURLFlag = cli.StringFlag{
		Name:        "amqp-url, u",
		Usage:       "full amqp URL",
		EnvVar:      "AMQP_URL",
		Destination: &amqpURL,
	}
	amqpExchangeFlag = cli.StringFlag{
		Name:        "amqp-exchange, x",
		Value:       "",
		Usage:       "topic exchange name",
		EnvVar:      "AMQP_EXCHANGE",
		Destination: &amqpExchange,
	}
	amqpCARootFlag = cli.StringFlag{
		Name:        "amqp-ca-root, c",
		Value:       "",
		Usage:       "AMQP CA root, if applicable",
		EnvVar:      "AMQP_CA_CERT",
		Destination: &amqpCARoot,
	}
	customDNSResolversFlag = cli.StringFlag{
		Name:        "custom-dns-resolvers, d",
		Value:       "",
		Usage:       "custom DNS resolvers, if applicable",
		EnvVar:      "CUSTOM_DNS_RESOLVERS",
		Destination: &customDNSResolvers,
	}
	developmentFlag = cli.BoolFlag{
		Name:        "development, d",
		Usage:       "run in development mode, without CORS and TLS",
		Destination: &development,
	}
)
<commit_msg>Fix flag override<commit_after>package main

import "github.com/codegangsta/cli"

var (
	amqpURL            string
	amqpExchange       string
	amqpCARoot         string
	customDNSResolvers string
	host               string
	originRegexp       string
	port               int
	development        bool
	hostFlag           = cli.StringFlag{
		Name:        "bind, b",
		Value:       "",
		Usage:       "host to bind the ws server",
		EnvVar:      "WS_BIND_HOST",
		Destination: &host,
	}
	portFlag = cli.IntFlag{
		Name:        "port, p",
		Value:       9090,
		Usage:       "port to listen on",
		EnvVar:      "WS_BIND_PORT",
		Destination: &port,
	}
	originRegexpFlag = cli.StringFlag{
		Name:        "allow-origin-regexp, r",
		Value:       "",
		Usage:       "a regexp string used to validate request origin header",
		EnvVar:      "WS_ALLOW_ORIGIN_REGEXP",
		Destination: &originRegexp,
	}
	amqpURLFlag = cli.StringFlag{
		Name:        "amqp-url, u",
		Usage:       "full amqp URL",
		EnvVar:      "AMQP_URL",
		Destination: &amqpURL,
	}
	amqpExchangeFlag = cli.StringFlag{
		Name:        "amqp-exchange, x",
		Value:       "",
		Usage:       "topic exchange name",
		EnvVar:      "AMQP_EXCHANGE",
		Destination: &amqpExchange,
	}
	amqpCARootFlag = cli.StringFlag{
		Name:        "amqp-ca-root, c",
		Value:       "",
		Usage:       "AMQP CA root, if applicable",
		EnvVar:      "AMQP_CA_CERT",
		Destination: &amqpCARoot,
	}
	customDNSResolversFlag = cli.StringFlag{
		Name:        "custom-dns-resolvers",
		Value:       "",
		Usage:       "custom DNS resolvers, if applicable",
		EnvVar:      "CUSTOM_DNS_RESOLVERS",
		Destination: &customDNSResolvers,
	}
	developmentFlag = cli.BoolFlag{
		Name:        "development, d",
		Usage:       "run in development mode, without CORS and TLS",
		Destination: &development,
	}
)
<|endoftext|>


<commit_before>package box

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"
)

// TODO(ttacon): reconcile this with Folder for one common struct?
type File struct {
	ID                string          `json:"id,omitempty"`
	FolderUploadEmail *AccessEmail    `json:"folder_upload_email,omitempty"`
	Parent            *Item           `json:"parent,omitempty"`
	ItemStatus        string          `json:"item_status"`
	ItemCollection    *ItemCollection `json:"item_collection"`
	Type              string          `json:"type"` // TODO(ttacon): enum
	Description       string          `json:"description"`
	Size              int             `json:"size"`
	CreateBy          *Item           `json:"created_by"`
	ModifiedBy        *Item           `json:"modified_by"`
	TrashedAt         *string         `json:"trashed_at"`          // TODO(ttacon): change to time.Time
	ContentModifiedAt *string         `json:"content_modified_at"` // TODO(ttacon): change to time.Time
	PurgedAt          *string         `json:"purged_at"`           // TODO(ttacon): change to time.Time, this field isn't documented but I keep getting it back...
	SharedLinkg       *string         `json:"shared_link"`
	SequenceId        string          `json:"sequence_id"`
	ETag              *string         `json:"etag"`
	Name              string          `json:"name"`
	CreatedAt         *string         `json:"created_at"` // TODO(ttacon): change to time.Time
	OwnedBy           *Item           `json:"owned_by"`
	ModifiedAt        *string         `json:"modified_at"`        // TODO(ttacon): change to time.Time
	ContentCreatedAt  *string         `json:"content_created_at"` // TODO(ttacon): change to time.Time
	PathCollection    *PathCollection `json:"path_collection"`    // TODO(ttacon): make sure this is the correct kind of struct(ure)
	SharedLink        *Link           `json:"shared_link"`

	SHA1 string `json:"sha1"`
}

type FileCollection struct {
	TotalCount int     `json:"total_count"`
	Entries    []*File `json:"entries"`
}

// Documentation: https://developer.box.com/docs/#files-get
func (c *Client) GetFile(fileId string) (*http.Response, *File, error) {
	req, err := c.NewRequest(
		"GET",
		fmt.Sprintf("/files/%s", fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *File
	resp, err := c.Do(req, data)
	return resp, data, err
}

// Documentation https://developer.box.com/docs/#files-upload-a-file
// TODO(ttacon): deal with handling SHA1 headers
func (c *Client) UploadFile(filePath, parentId string) (*http.Response, *FileCollection, error) {
	file, err := os.Open(filePath)
	if err != nil {
		return nil, nil, err
	}
	defer file.Close()

	fileContents, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, nil, err
	}

	fi, err := file.Stat()
	if err != nil {
		return nil, nil, err
	}

	var (
		body   = &bytes.Buffer{}
		writer = multipart.NewWriter(body)
	)

	// write the file
	part, err := writer.CreateFormFile("file", fi.Name())
	if err != nil {
		return nil, nil, err
	}
	part.Write(fileContents)

	// write the other form fields we need
	err = writer.WriteField("filename", fi.Name())
	if err != nil {
		return nil, nil, err
	}

	err = writer.WriteField("parent_id", parentId)
	if err != nil {
		return nil, nil, err
	}

	// TODO(ttacon): add in content_created_at, content_modified_at

	err = writer.Close()
	if err != nil {
		return nil, nil, err
	}

	req, err := http.NewRequest(
		"POST",
		fmt.Sprintf("https://upload.box.com/api/2.0/files/content"),
		body,
	)
	req.Header.Add("Content-Type", writer.FormDataContentType())
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data FileCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-delete-a-file
func (c *Client) DeleteFile(fileId string) (*http.Response, error) {
	req, err := c.NewRequest(
		"DELETE",
		fmt.Sprintf("/files/%s", fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Do(req, nil)
}

// Documentation: https://developers.box.com/docs/#files-copy-a-file
func (c *Client) CopyFile(fileId, parent, name string) (*http.Response, *File, error) {
	var bodyData = map[string]interface{}{
		"parent": map[string]string{
			"id": parent,
		},
		"name": name,
	}

	req, err := http.NewRequest(
		"POST",
		fmt.Sprintf("/files/%s/copy", fileId),
		dataMap,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *File
	resp, err := c.Do(req, data)
	return resp, data, err
}

// NOTE: we return the http.Response as Box may return a 202 if there is not
// yet a download link, or a 302 with the link - this allows the user to
// decide what to do.
// Documentation: https://developers.box.com/docs/#files-download-a-file
func (c *Client) DownloadFile(fileId string) (*http.Response, error) {
	req, err := c.NewRequest(
		"GET",
		fmt.Sprintf("/files/%s/content", fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Do(req, nil)
}

// Documentation: https://developers.box.com/docs/#files-view-versions-of-a-file
// TODO(ttacon): don't use file collection, make actual structs specific to file versions
func (c *Client) ViewVersionsOfFile(fileId string) (*http.Response, *FileCollection, error) {
	req, err := c.NewRequest(
		"GET",
		fmt.Sprintf("/files/%s/versions", fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *FileCollection
	resp, err := c.Do(req, data)
	return resp, data, err
}

// NOTE: we only return the response as there are many possible responses that we
// feel the user should have control over
// Documentation: https://developers.box.com/docs/#files-get-a-thumbnail-for-a-file
func (c *Client) GetThumbnail(fileId string) (*http.Response, error) {
	req, err := c.NewRequest(
		"GET",
		fmt.Sprintf("/files/%s/thumbnail.extension", fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Do(req)
}

// Documentation: https://developers.box.com/docs/#files-create-a-shared-link-for-a-file
func (c *Client) CreateSharedLinkForFile(fileId, access, unsharedAt string, canDownload, canPreview bool) (*http.Response, *File, error) {
	var dataMap = make(map[string]interface{})
	if len(access) > 0 {
		dataMap["access"] = access
	}
	// TODO(ttacon): support unshared_at as time.Time
	// TODO(ttacon): validate access is open or company before add permissions
	if canDownload {
		dataMap["permissions"] = map[string]bool{
			"can_download": canDownload,
		}
	}
	if canPreview {
		if m, ok := dataMap["permissions"]; ok {
			mVal, _ := m.(map[string]bool)
			mVal["can_preview"] = canPreview
		} else {
			dataMap["permissions"] = map[string]bool{
				"can_preview": canPreview,
			}
		}
	}

	req, err := c.NewRequest(
		"PUT",
		fmt.Sprintf("/files/%s", fileId),
		dataMap,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *File
	resp, err := c.Do(req, data)
	return resp, data, err
}

// Documentation: https://developers.box.com/docs/#files-get-a-trashed-file
func (c *Client) GetTrashedFile(fileId string) (*http.Response, *File, error) {
	req, err := c.NewRequest(
		"GET",
		fmt.Sprintf("/files/%s/trash", fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *File
	resp, err := c.Do(req, data)
	return resp, data, err
}

// Documentation: https://developers.box.com/docs/#files-restore-a-trashed-item
func (c *Client) RestoreTrashedItem(name, parentId string) (*http.Response, *File, error) {
	var dataMap = make(map[string]interface{})
	if len(name) > 0 {
		dataMap["name"] = name
	}
	if len(parentId) > 0 {
		dataMap["parent"] = map[string]string{
			"id": parentId,
		}
	}
	dataBytes, err := json.Marshal(&dataMap)
	if err != nil {
		return nil, nil, err
	}

	req, err := http.NewRequest(
		"POST",
		fmt.Sprintf(),
		bytes.NewReader(dataBytes),
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data File
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-permanently-delete-a-trashed-file
func (c *Client) PermanentlyDeleteTrashedFile(fileId string) (*http.Response, error) {
	req, err := http.NewRequest(
		"DELETE",
		fmt.Sprintf("%s/files/%s/trash", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Trans.Client().Do(req)
}

// Documentation: https://developers.box.com/docs/#files-view-the-comments-on-a-file
func (c *Client) ViewCommentsOnFile(fileId string) (*http.Response, *CommentCollection, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/comments", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data CommentCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-get-the-tasks-for-a-file
func (c *Client) GetTasksForFile(fileId string) (*http.Response, *TaskCollection, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/tasks", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data TaskCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}
<commit_msg>refactor restore trashed item<commit_after>package box

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"
)

// TODO(ttacon): reconcile this with Folder for one common struct?
type File struct {
	ID                string          `json:"id,omitempty"`
	FolderUploadEmail *AccessEmail    `json:"folder_upload_email,omitempty"`
	Parent            *Item           `json:"parent,omitempty"`
	ItemStatus        string          `json:"item_status"`
	ItemCollection    *ItemCollection `json:"item_collection"`
	Type              string          `json:"type"` // TODO(ttacon): enum
	Description       string          `json:"description"`
	Size              int             `json:"size"`
	CreateBy          *Item           `json:"created_by"`
	ModifiedBy        *Item           `json:"modified_by"`
	TrashedAt         *string         `json:"trashed_at"`          // TODO(ttacon): change to time.Time
	ContentModifiedAt *string         `json:"content_modified_at"` // TODO(ttacon): change to time.Time
	PurgedAt          *string         `json:"purged_at"`           // TODO(ttacon): change to time.Time, this field isn't documented but I keep getting it back...
	SharedLinkg       *string         `json:"shared_link"`
	SequenceId        string          `json:"sequence_id"`
	ETag              *string         `json:"etag"`
	Name              string          `json:"name"`
	CreatedAt         *string         `json:"created_at"` // TODO(ttacon): change to time.Time
	OwnedBy           *Item           `json:"owned_by"`
	ModifiedAt        *string         `json:"modified_at"`        // TODO(ttacon): change to time.Time
	ContentCreatedAt  *string         `json:"content_created_at"` // TODO(ttacon): change to time.Time
	PathCollection    *PathCollection `json:"path_collection"`    // TODO(ttacon): make sure this is the correct kind of struct(ure)
	SharedLink        *Link           `json:"shared_link"`

	SHA1 string `json:"sha1"`
}

type FileCollection struct {
	TotalCount int     `json:"total_count"`
	Entries    []*File `json:"entries"`
}

// Documentation: https://developer.box.com/docs/#files-get
func (c *Client) GetFile(fileId string) (*http.Response, *File, error) {
	req, err := c.NewRequest(
		"GET",
		fmt.Sprintf("/files/%s", fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *File
	resp, err := c.Do(req, data)
	return resp, data, err
}

// Documentation https://developer.box.com/docs/#files-upload-a-file
// TODO(ttacon): deal with handling SHA1 headers
func (c *Client) UploadFile(filePath, parentId string) (*http.Response, *FileCollection, error) {
	file, err := os.Open(filePath)
	if err != nil {
		return nil, nil, err
	}
	defer file.Close()

	fileContents, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, nil, err
	}

	fi, err := file.Stat()
	if err != nil {
		return nil, nil, err
	}

	var (
		body   = &bytes.Buffer{}
		writer = multipart.NewWriter(body)
	)

	// write the file
	part, err := writer.CreateFormFile("file", fi.Name())
	if err != nil {
		return nil, nil, err
	}
	part.Write(fileContents)

	// write the other form fields we need
	err = writer.WriteField("filename", fi.Name())
	if err != nil {
		return nil, nil, err
	}

	err = writer.WriteField("parent_id", parentId)
	if err != nil {
		return nil, nil, err
	}

	// TODO(ttacon): add in content_created_at, content_modified_at

	err = writer.Close()
	if err != nil {
		return nil, nil, err
	}

	req, err := http.NewRequest(
		"POST",
		fmt.Sprintf("https://upload.box.com/api/2.0/files/content"),
		body,
	)
	req.Header.Add("Content-Type", writer.FormDataContentType())
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data FileCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-delete-a-file
func (c *Client) DeleteFile(fileId string) (*http.Response, error) {
	req, err := c.NewRequest(
		"DELETE",
		fmt.Sprintf("/files/%s", fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Do(req, nil)
}

// Documentation: https://developers.box.com/docs/#files-copy-a-file
func (c *Client) CopyFile(fileId, parent, name string) (*http.Response, *File, error) {
	var bodyData = map[string]interface{}{
		"parent": map[string]string{
			"id": parent,
		},
		"name": name,
	}

	req, err := http.NewRequest(
		"POST",
		fmt.Sprintf("/files/%s/copy", fileId),
		dataMap,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *File
	resp, err := c.Do(req, data)
	return resp, data, err
}

// NOTE: we return the http.Response as Box may return a 202 if there is not
// yet a download link, or a 302 with the link - this allows the user to
// decide what to do.
// Documentation: https://developers.box.com/docs/#files-download-a-file
func (c *Client) DownloadFile(fileId string) (*http.Response, error) {
	req, err := c.NewRequest(
		"GET",
		fmt.Sprintf("/files/%s/content", fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Do(req, nil)
}

// Documentation: https://developers.box.com/docs/#files-view-versions-of-a-file
// TODO(ttacon): don't use file collection, make actual structs specific to file versions
func (c *Client) ViewVersionsOfFile(fileId string) (*http.Response, *FileCollection, error) {
	req, err := c.NewRequest(
		"GET",
		fmt.Sprintf("/files/%s/versions", fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *FileCollection
	resp, err := c.Do(req, data)
	return resp, data, err
}

// NOTE: we only return the response as there are many possible responses that we
// feel the user should have control over
// Documentation: https://developers.box.com/docs/#files-get-a-thumbnail-for-a-file
func (c *Client) GetThumbnail(fileId string) (*http.Response, error) {
	req, err := c.NewRequest(
		"GET",
		fmt.Sprintf("/files/%s/thumbnail.extension", fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Do(req)
}

// Documentation: https://developers.box.com/docs/#files-create-a-shared-link-for-a-file
func (c *Client) CreateSharedLinkForFile(fileId, access, unsharedAt string, canDownload, canPreview bool) (*http.Response, *File, error) {
	var dataMap = make(map[string]interface{})
	if len(access) > 0 {
		dataMap["access"] = access
	}
	// TODO(ttacon): support unshared_at as time.Time
	// TODO(ttacon): validate access is open or company before add permissions
	if canDownload {
		dataMap["permissions"] = map[string]bool{
			"can_download": canDownload,
		}
	}
	if canPreview {
		if m, ok := dataMap["permissions"]; ok {
			mVal, _ := m.(map[string]bool)
			mVal["can_preview"] = canPreview
		} else {
			dataMap["permissions"] = map[string]bool{
				"can_preview": canPreview,
			}
		}
	}

	req, err := c.NewRequest(
		"PUT",
		fmt.Sprintf("/files/%s", fileId),
		dataMap,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *File
	resp, err := c.Do(req, data)
	return resp, data, err
}

// Documentation: https://developers.box.com/docs/#files-get-a-trashed-file
func (c *Client) GetTrashedFile(fileId string) (*http.Response, *File, error) {
	req, err := c.NewRequest(
		"GET",
		fmt.Sprintf("/files/%s/trash", fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *File
	resp, err := c.Do(req, data)
	return resp, data, err
}

// Documentation: https://developers.box.com/docs/#files-restore-a-trashed-item
func (c *Client) RestoreTrashedItem(fileId, name, parentId string) (*http.Response, *File, error) {
	var dataMap = make(map[string]interface{})
	if len(name) > 0 {
		dataMap["name"] = name
	}
	if len(parentId) > 0 {
		dataMap["parent"] = map[string]string{
			"id": parentId,
		}
	}

	req, err := c.NewRequest(
		"POST",
		fmt.Sprintf("/files/%s", fileId),
		dataMap,
	)
	if err != nil {
		return nil, nil, err
	}

	var data *File
	resp, err := c.Do(req, data)
	return resp, data, err
}

// Documentation: https://developers.box.com/docs/#files-permanently-delete-a-trashed-file
func (c *Client) PermanentlyDeleteTrashedFile(fileId string) (*http.Response, error) {
	req, err := http.NewRequest(
		"DELETE",
		fmt.Sprintf("%s/files/%s/trash", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, err
	}

	return c.Trans.Client().Do(req)
}

// Documentation: https://developers.box.com/docs/#files-view-the-comments-on-a-file
func (c *Client) ViewCommentsOnFile(fileId string) (*http.Response, *CommentCollection, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/comments", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data CommentCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}

// Documentation: https://developers.box.com/docs/#files-get-the-tasks-for-a-file
func (c *Client) GetTasksForFile(fileId string) (*http.Response, *TaskCollection, error) {
	req, err := http.NewRequest(
		"GET",
		fmt.Sprintf("%s/files/%s/tasks", BASE_URL, fileId),
		nil,
	)
	if err != nil {
		return nil, nil, err
	}

	resp, err := c.Trans.Client().Do(req)
	if err != nil {
		return resp, nil, err
	}

	var data TaskCollection
	err = json.NewDecoder(resp.Body).Decode(&data)
	return resp, &data, err
}
<|endoftext|>