<commit_before>package topic

import (
	"container/list"
	"fmt"
	"io"

	"gopkg.in/logex.v1"

	"time"

	"github.com/chzyer/mmq/internal/bitmap"
	"github.com/chzyer/mmq/internal/utils"
	"github.com/chzyer/mmq/mmq"
)

const (
	MaxBenchSize = 100
)

var (
	ErrBenchSizeTooLarge = logex.Define("bench size is too large")
)

type Config struct {
	Root      string
	IndexName string
	ChunkBit  uint
}

func (c *Config) Path(name string) string {
	return fmt.Sprintf("%s/%s", c.Root, name)
}

type Instance struct {
	config *Config
	Name   string
	index  int
	file   *bitmap.File
	writer *utils.Writer

	// linked list for Waiters
	waiterList *list.List
	newMsgChan chan struct{}
	putChan    chan *putArgs
	getChan    chan *getArgs
}

func New(name string, config *Config) (t *Instance, err error) {
	t = &Instance{
		config:     config,
		Name:       name,
		newMsgChan: make(chan struct{}, 1),
		putChan:    make(chan *putArgs, 1<<3),
	}
	path := config.Path(t.TopicPath())
	t.file, err = bitmap.NewFileEx(path, config.ChunkBit)
	if err != nil {
		return nil, logex.Trace(err)
	}
	// TODO: must read some metafile to determine the offset
	t.writer = &utils.Writer{t.file, 0}
	go t.ioLoop()
	return t, nil
}

func (t *Instance) TopicPath() string {
	return utils.PathEncode(t.Name)
}

func (t *Instance) notifyNewMsg() {
	select {
	case t.newMsgChan <- struct{}{}:
	default:
	}
}

func (t *Instance) ioLoop() {
	var (
		put *putArgs
		get *getArgs
		ok  bool

		timer = time.NewTimer(0)
	)
	for {
		select {
		case put, ok = <-t.putChan:
			if !ok {
				goto exit
			}
			t.put(put, timer)
		case get, ok = <-t.getChan:
			if !ok {
				goto exit
			}
			t.getAsync(get, timer)
		}
	}
exit:
}

type putArgs struct {
	msgs  []*mmq.Message
	reply chan<- []error
}

func (t *Instance) Put(msgs []*mmq.Message, reply chan []error) {
	t.putChan <- &putArgs{msgs, reply}
}

func (t *Instance) put(arg *putArgs, timer *time.Timer) {
	errs := make([]error, len(arg.msgs))
	for i := 0; i < len(arg.msgs); i++ {
		_, errs[i] = arg.msgs[i].WriteTo(t.writer)
	}
	t.notifyNewMsg()
	timer.Reset(time.Second)
	select {
	case arg.reply <- errs:
	case <-timer.C:
		logex.Error("write reply timeout")
	}
}

type getArgs struct {
	offset int64
	size   int
	reply  chan<- []*mmq.Message
	err    chan<- error
}

func (t *Instance) Get(offset int64, size int, reply chan<- []*mmq.Message, err chan<- error) {
	t.getChan <- &getArgs{offset, size, reply, err}
}

func (t *Instance) getAsync(arg *getArgs, timer *time.Timer) {
	err := t.get(arg)
	timer.Reset(time.Second)
	select {
	case arg.err <- err:
	case <-timer.C:
		logex.Error("reply to get timeout")
	}
}

func (t *Instance) get(arg *getArgs) error {
	if arg.size > MaxBenchSize {
		return ErrBenchSizeTooLarge.Trace()
	}

	msgs := make([]*mmq.Message, arg.size)
	var (
		msg *mmq.Message
		err error
	)

	var header mmq.HeaderBin

	// check offset
	r := &utils.Reader{t.file, arg.offset}
	p := 0
	for i := 0; i < arg.size; i++ {
		msg, err = mmq.ReadMessage(&header, r, mmq.RF_RESEEK_ON_FAULT)
		if logex.Equal(err, io.ErrUnexpectedEOF) || logex.Equal(err, io.EOF) {
			// use chan, to subscribe
			break
		}
		if err != nil {
			break
		}
		msgs[p] = msg
		p++
	}

	select {
	case arg.reply <- msgs[:p]:
	}
	return logex.Trace(err)
}
<commit_msg>init get Chan<commit_after>package topic

import (
	"container/list"
	"fmt"
	"io"

	"gopkg.in/logex.v1"

	"time"

	"github.com/chzyer/mmq/internal/bitmap"
	"github.com/chzyer/mmq/internal/utils"
	"github.com/chzyer/mmq/mmq"
)

const (
	MaxBenchSize = 100
)

var (
	ErrBenchSizeTooLarge = logex.Define("bench size is too large")
)

type Config struct {
	Root      string
	IndexName string
	ChunkBit  uint
}

func (c *Config) Path(name string) string {
	return fmt.Sprintf("%s/%s", c.Root, name)
}

type Instance struct {
	config *Config
	Name   string
	index  int
	file   *bitmap.File
	writer *utils.Writer

	// linked list for Waiters
	waiterList *list.List
	newMsgChan chan struct{}
	putChan    chan *putArgs
	getChan    chan *getArgs
}

func New(name string, config *Config) (t *Instance, err error) {
	t = &Instance{
		config:     config,
		Name:       name,
		newMsgChan: make(chan struct{}, 1),
		putChan:    make(chan *putArgs, 1<<3),
		getChan:    make(chan *getArgs, 1<<3),
	}
	path := config.Path(t.TopicPath())
	t.file, err = bitmap.NewFileEx(path, config.ChunkBit)
	if err != nil {
		return nil, logex.Trace(err)
	}
	// TODO: must read some metafile to determine the offset
	t.writer = &utils.Writer{t.file, 0}
	go t.ioLoop()
	return t, nil
}

func (t *Instance) TopicPath() string {
	return utils.PathEncode(t.Name)
}

func (t *Instance) notifyNewMsg() {
	select {
	case t.newMsgChan <- struct{}{}:
	default:
	}
}

func (t *Instance) ioLoop() {
	var (
		put *putArgs
		get *getArgs
		ok  bool

		timer = time.NewTimer(0)
	)
	for {
		select {
		case put, ok = <-t.putChan:
			if !ok {
				goto exit
			}
			t.put(put, timer)
		case get, ok = <-t.getChan:
			if !ok {
				goto exit
			}
			t.getAsync(get, timer)
		}
	}
exit:
}

type putArgs struct {
	msgs  []*mmq.Message
	reply chan<- []error
}

func (t *Instance) Put(msgs []*mmq.Message, reply chan []error) {
	t.putChan <- &putArgs{msgs, reply}
}

func (t *Instance) put(arg *putArgs, timer *time.Timer) {
	errs := make([]error, len(arg.msgs))
	for i := 0; i < len(arg.msgs); i++ {
		_, errs[i] = arg.msgs[i].WriteTo(t.writer)
	}
	t.notifyNewMsg()
	timer.Reset(time.Second)
	select {
	case arg.reply <- errs:
	case <-timer.C:
		logex.Error("write reply timeout")
	}
}

type getArgs struct {
	offset int64
	size   int
	reply  chan<- []*mmq.Message
	err    chan<- error
}

func (t *Instance) Get(offset int64, size int, reply chan<- []*mmq.Message, err chan<- error) {
	t.getChan <- &getArgs{offset, size, reply, err}
}

func (t *Instance) getAsync(arg *getArgs, timer *time.Timer) {
	err := t.get(arg)
	timer.Reset(time.Second)
	select {
	case arg.err <- err:
	case <-timer.C:
		logex.Error("reply to get timeout")
	}
}

func (t *Instance) get(arg *getArgs) error {
	if arg.size > MaxBenchSize {
		return ErrBenchSizeTooLarge.Trace()
	}

	msgs := make([]*mmq.Message, arg.size)
	var (
		msg *mmq.Message
		err error
	)

	var header mmq.HeaderBin

	// check offset
	r := &utils.Reader{t.file, arg.offset}
	p := 0
	for i := 0; i < arg.size; i++ {
		msg, err = mmq.ReadMessage(&header, r, mmq.RF_RESEEK_ON_FAULT)
		if logex.Equal(err, io.ErrUnexpectedEOF) || logex.Equal(err, io.EOF) {
			// use chan, to subscribe
			break
		}
		if err != nil {
			break
		}
		msgs[p] = msg
		p++
	}

	select {
	case arg.reply <- msgs[:p]:
	}
	return logex.Trace(err)
}
<|endoftext|>


<commit_before>// Copyright 2015 ThoughtWorks, Inc.

// This file is part of Gauge.

// Gauge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Gauge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Gauge.  If not, see <http://www.gnu.org/licenses/>.

package track

import (
	"net/http"
	"net/http/httputil"
	"strings"

	"fmt"

	"github.com/getgauge/gauge/config"
	"github.com/getgauge/gauge/logger"
	"github.com/getgauge/gauge/manifest"
	"github.com/getgauge/gauge/version"
	"github.com/jpillora/go-ogle-analytics"
)

const (
	gaTrackingID  = "UA-100778536-1"
	appName       = "Gauge Core"
	consoleMedium = "console"
	apiMedium     = "api"
)

func send(category, action, label, medium string) {
	if !config.TelemetryEnabled() {
		return
	}
	client, err := ga.NewClient(gaTrackingID)
	client.HttpClient = &http.Client{}
	client.ClientID(config.UniqueID())
	client.AnonymizeIP(true)
	client.ApplicationName(appName)
	client.ApplicationVersion(version.FullVersion())
	client.CampaignMedium(medium)
	client.CampaignSource(appName)

	if config.TelemetryLogEnabled() {
		client.HttpClient = newlogEnabledHTTPClient()
	}

	if err != nil {
		logger.Warning("Unable to create ga client, %s", err)
	}

	ev := ga.NewEvent(category, action)
	if label != "" {
		ev.Label(label)
	}

	err = client.Send(ev)
	if err != nil {
		logger.Warning("Unable to send telemetry data, %s", err)
	}
}

func trackConsole(category, action, label string) {
	go send(category, action, label, consoleMedium)
}

func trackAPI(category, action, label string) {
	go send(category, action, label, apiMedium)
}

func trackManifest() {
	m, err := manifest.ProjectManifest()
	if err == nil {
		trackConsole("manifest", "language", fmt.Sprintf("%s,%s", m.Language, strings.Join(m.Plugins, ",")))
	}
}

func Execution(parallel, tagged, sorted, simpleConsole, verbose bool, parallelExecutionStrategy string) {
	action := "serial"
	if parallel {
		action = "parallel"
	}
	flags := []string{action}

	if tagged {
		flags = append(flags, "tagged")
	}

	if sorted {
		flags = append(flags, "sorted")
	}

	if simpleConsole {
		flags = append(flags, "simple-console")
	}

	if verbose {
		flags = append(flags, "verbose")
	}

	trackManifest()
	trackConsole("execution", action, strings.Join(flags, ","))
}

func Validation() {
	trackConsole("validation", "validate", "")
}

func Docs(docs string) {
	trackConsole("docs", "generate", docs)
}

func Format() {
	trackConsole("formatting", "format", "")
}

func Refactor() {
	trackConsole("refactoring", "rephrase", "")
}

func ListTemplates() {
	trackConsole("templates", "list", "")
}

func AddPlugins(plugin string) {
	trackConsole("plugins", "add", plugin)
}

func UninstallPlugin(plugin string) {
	trackConsole("plugins", "uninstall", plugin)
}

func ProjectInit() {
	trackConsole("project", "init", "")
}

func Install(plugin string, zip bool) {
	if zip {
		trackConsole("plugins", "install-zip", plugin)
	} else {
		trackConsole("plugins", "install", plugin)
	}
}

func Update(plugin string) {
	trackConsole("plugins", "update", plugin)
}

func UpdateAll() {
	trackConsole("plugins", "update-all", "")
}

func InstallAll() {
	trackConsole("plugins", "install-all", "")
}

func CheckUpdates() {
	trackConsole("updates", "check", "")
}

func Daemon() {
	trackConsole("execution", "daemon", "")
}

func APIRefactoring() {
	trackAPI("refactoring", "rephrase", "")
}

func APIExtractConcept() {
	trackAPI("refactoring", "extract-concept", "")
}

func APIFormat() {
	trackAPI("formatting", "format", "")
}

func newlogEnabledHTTPClient() *http.Client {
	return &http.Client{
		Transport: logEnabledRoundTripper{},
	}
}

type logEnabledRoundTripper struct {
}

func (r logEnabledRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	dump, err := httputil.DumpRequestOut(req, true)
	if err != nil {
		logger.Warning("Unable to dump telemetry request, %s", err)
	}

	logger.Debug(fmt.Sprintf("%q", dump))
	return http.DefaultTransport.RoundTrip(req)
}
<commit_msg>set campaignSource = ci if it is CI, #702<commit_after>// Copyright 2015 ThoughtWorks, Inc.

// This file is part of Gauge.

// Gauge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Gauge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Gauge.  If not, see <http://www.gnu.org/licenses/>.

package track

import (
	"net/http"
	"net/http/httputil"
	"strings"

	"fmt"

	"os"

	"github.com/getgauge/gauge/config"
	"github.com/getgauge/gauge/logger"
	"github.com/getgauge/gauge/manifest"
	"github.com/getgauge/gauge/version"
	"github.com/jpillora/go-ogle-analytics"
)

const (
	gaTrackingID  = "UA-100778536-1"
	appName       = "Gauge Core"
	consoleMedium = "console"
	apiMedium     = "api"
	ciMedium      = "CI"
)

func send(category, action, label, medium string) {
	if !config.TelemetryEnabled() {
		return
	}
	client, err := ga.NewClient(gaTrackingID)
	client.HttpClient = &http.Client{}
	client.ClientID(config.UniqueID())
	client.AnonymizeIP(true)
	client.ApplicationName(appName)
	client.ApplicationVersion(version.FullVersion())
	client.CampaignMedium(medium)
	client.CampaignSource(appName)

	if config.TelemetryLogEnabled() {
		client.HttpClient = newlogEnabledHTTPClient()
	}

	if err != nil {
		logger.Warning("Unable to create ga client, %s", err)
	}

	ev := ga.NewEvent(category, action)
	if label != "" {
		ev.Label(label)
	}

	err = client.Send(ev)
	if err != nil {
		logger.Warning("Unable to send telemetry data, %s", err)
	}
}

func trackConsole(category, action, label string) {
	var medium = consoleMedium
	if isCI() {
		medium = ciMedium
	}
	go send(category, action, label, medium)
}

func trackAPI(category, action, label string) {
	var medium = apiMedium
	if isCI() {
		medium = ciMedium
	}
	go send(category, action, label, medium)
}

func isCI() bool {
	// Travis, AppVeyor, CircleCI, Wercket, drone.io, gitlab-ci
	if os.Getenv("CI") == "true" {
		return true
	}

	// GoCD
	if os.Getenv("GO_SERVER_URL") != "" {
		return true
	}

	// Teamcity
	if os.Getenv("TEAMCITY_VERSION") != "" {
		return true
	}

	// TFS
	if os.Getenv("TFS_BUILD") == "true" {
		return true
	}
	return false
}

func trackManifest() {
	m, err := manifest.ProjectManifest()
	if err == nil {
		trackConsole("manifest", "language", fmt.Sprintf("%s,%s", m.Language, strings.Join(m.Plugins, ",")))
	}
}

func Execution(parallel, tagged, sorted, simpleConsole, verbose bool, parallelExecutionStrategy string) {
	action := "serial"
	if parallel {
		action = "parallel"
	}
	flags := []string{action}

	if tagged {
		flags = append(flags, "tagged")
	}

	if sorted {
		flags = append(flags, "sorted")
	}

	if simpleConsole {
		flags = append(flags, "simple-console")
	}

	if verbose {
		flags = append(flags, "verbose")
	}

	trackManifest()
	trackConsole("execution", action, strings.Join(flags, ","))
}

func Validation() {
	trackConsole("validation", "validate", "")
}

func Docs(docs string) {
	trackConsole("docs", "generate", docs)
}

func Format() {
	trackConsole("formatting", "format", "")
}

func Refactor() {
	trackConsole("refactoring", "rephrase", "")
}

func ListTemplates() {
	trackConsole("templates", "list", "")
}

func AddPlugins(plugin string) {
	trackConsole("plugins", "add", plugin)
}

func UninstallPlugin(plugin string) {
	trackConsole("plugins", "uninstall", plugin)
}

func ProjectInit() {
	trackConsole("project", "init", "")
}

func Install(plugin string, zip bool) {
	if zip {
		trackConsole("plugins", "install-zip", plugin)
	} else {
		trackConsole("plugins", "install", plugin)
	}
}

func Update(plugin string) {
	trackConsole("plugins", "update", plugin)
}

func UpdateAll() {
	trackConsole("plugins", "update-all", "")
}

func InstallAll() {
	trackConsole("plugins", "install-all", "")
}

func CheckUpdates() {
	trackConsole("updates", "check", "")
}

func Daemon() {
	trackConsole("execution", "daemon", "")
}

func APIRefactoring() {
	trackAPI("refactoring", "rephrase", "")
}

func APIExtractConcept() {
	trackAPI("refactoring", "extract-concept", "")
}

func APIFormat() {
	trackAPI("formatting", "format", "")
}

func newlogEnabledHTTPClient() *http.Client {
	return &http.Client{
		Transport: logEnabledRoundTripper{},
	}
}

type logEnabledRoundTripper struct {
}

func (r logEnabledRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	dump, err := httputil.DumpRequestOut(req, true)
	if err != nil {
		logger.Warning("Unable to dump telemetry request, %s", err)
	}

	logger.Debug(fmt.Sprintf("%q", dump))
	return http.DefaultTransport.RoundTrip(req)
}
<|endoftext|>


<commit_before>// Copyright 2015 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	fct "github.com/FactomProject/factoid"
	"github.com/FactomProject/factom"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
	"strconv"
)

var _ = hex.EncodeToString
var serverFct = "localhost:8089"

var badChar, _ = regexp.Compile("[^A-Za-z0-9_-]")

type Response struct {
	Response string
	Success  bool
}

func ValidateKey(key string) (msg string, valid bool) {
	if len(key) > fct.ADDRESS_LENGTH {
		return "Key is too long.  Keys must be less than 32 characters", false
	}
	if badChar.FindStringIndex(key) != nil {
		str := fmt.Sprintf("The key or name '%s' contains invalid characters.\n"+
			"Keys and names are restricted to alphanumeric characters,\n"+
			"minuses (dashes), and underscores", key)
		return str, false
	}
	return "", true
}

func getCmd(cmd string, cmderror string) {
	resp, err := http.Get(cmd)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	resp.Body.Close()

	b := new(Response)
	if err := json.Unmarshal(body, b); err != nil || !b.Success {
		fmt.Println(cmderror)
		fmt.Println("Command Failed: ", string(body))
		os.Exit(1)
	}
	fmt.Println(b.Response)
	return
}

func postCmd(cmd string) {
	resp, err := http.PostForm(cmd, nil)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	resp.Body.Close()

	b := new(Response)
	if err := json.Unmarshal(body, b); err != nil {
		fmt.Printf("Failed to parse the response from factomd: %s\n", body)
		os.Exit(1)
	}

	fmt.Println(b.Response)

	if !b.Success {
		os.Exit(1)
	}

	return
}

// Generates a new Address
func generateaddress(args []string) {

	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 2 {
		man("generatefactoidaddress")
		os.Exit(1)
	}

	var err error
	var Addr string
	switch args[0] {
	case "ec":
		Addr, err = factom.GenerateEntryCreditAddress(args[1])
	case "fct":
		Addr, err = factom.GenerateFactoidAddress(args[1])
	default:
		panic("Expected ec|fct name")
	}

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	fmt.Println(args[0], " = ", Addr)
	return

}

func getaddresses(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) > 0 {
		man("getaddresses")
		os.Exit(1)
	}

	str := fmt.Sprintf("http://%s/v1/factoid-get-addresses/", serverFct)
	getCmd(str, "Error printing addresses")

	return
}

func gettransactions(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) > 0 {
		man("transactions")
		os.Exit(1)
	}

	str := fmt.Sprintf("http://%s/v1/factoid-get-transactions/", serverFct)
	getCmd(str, "Error Getting Transactions")

	return
}

func fctnewtrans(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 1 {
		fmt.Println("Missing Key")
		man("newtransaction")
		os.Exit(1)
	}
	msg, valid := ValidateKey(args[0])
	if !valid {
		fmt.Println(msg)
		os.Exit(1)
	}
	str := fmt.Sprintf("http://%s/v1/factoid-new-transaction/%s", serverFct, args[0])
	postCmd(str)

	return

}

func fctdeletetrans(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 1 {
		fmt.Println("Missing Key")
		man("deletetransaction")
		os.Exit(1)
	}
	msg, valid := ValidateKey(args[0])
	if !valid {
		fmt.Println(msg)
		os.Exit(1)
	}
	str := fmt.Sprintf("http://%s/v1/factoid-delete-transaction/%s", serverFct, args[0])
	postCmd(str)

	return

}

func fctaddfee(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 2 {
		fmt.Println("Was expecting a transaction key, and an address used as an input to that transaction.")
		os.Exit(1)
	}

	msg, valid := ValidateKey(args[0])
	if !valid {
		fmt.Println(msg)
		os.Exit(1)
	}

	str := fmt.Sprintf("http://%s/v1/factoid-add-fee/?key=%s&name=%s",
		serverFct, args[0], args[1])
	postCmd(str)

	return
}

func fctaddinput(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 3 {
		fmt.Println("Expecting a 1) transaction key, 2) an Address or Address name, and 3) an amount.")
		os.Exit(1)
	}

	msg, valid := ValidateKey(args[0])
	if !valid {
		fmt.Println(msg)
		os.Exit(1)
	}

	amt, err := fct.ConvertFixedPoint(args[2])
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	ramt, err := strconv.ParseInt(amt, 10, 64)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	_, err = fct.ValidateAmounts(uint64(ramt))
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	str := fmt.Sprintf("http://%s/v1/factoid-add-input/?key=%s&name=%s&amount=%s",
		serverFct, args[0], args[1], amt)
	postCmd(str)

	return
}

func fctaddoutput(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 3 {
		fmt.Println("Expecting a 1) transaction key, 2) an Address or Address name, and 3) an amount.")
		os.Exit(1)
	}
	// localhost:8089/v1/factoid-add-input/?key=<key>&name=<name or address>&amount=<amount>

	msg, valid := ValidateKey(args[0])
	if !valid {
		fmt.Println(msg)
		os.Exit(1)
	}

	amt, err := fct.ConvertFixedPoint(args[2])
	if err != nil {
		fmt.Println("Invalid format for a number: ", args[2])
		man("addoutput")
		os.Exit(1)
	}

	ramt, err := strconv.ParseInt(amt, 10, 64)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	_, err = fct.ValidateAmounts(uint64(ramt))
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	str := fmt.Sprintf("http://%s/v1/factoid-add-output/?key=%s&name=%s&amount=%s",
		serverFct, args[0], args[1], amt)
	postCmd(str)

	return
}

func fctaddecoutput(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 3 {
		fmt.Println("Expecting a 1) transaction key, 2) an Address or Address name, and 3) an amount.")
		os.Exit(1)
	}
	// localhost:8089/v1/factoid-add-input/?key=<key>&name=<name or address>&amount=<amount>

	msg, valid := ValidateKey(args[0])
	if !valid {
		fmt.Println(msg)
		os.Exit(1)
	}

	amt, err := fct.ConvertFixedPoint(args[2])
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	ramt, err := strconv.ParseInt(amt, 10, 64)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	_, err = fct.ValidateAmounts(uint64(ramt))
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	str := fmt.Sprintf("http://%s/v1/factoid-add-ecoutput/?key=%s&name=%s&amount=%s",
		serverFct, args[0], args[1], amt)
	postCmd(str)

	return
}

func fctgetfee(args []string) {
	var getfeereq string
	if len(args)>1 {
		getfeereq = fmt.Sprintf("http://%s/v1/factoid-get-fee/?key=%s", serverFct,args[1])
	}else{
		getfeereq = fmt.Sprintf("http://%s/v1/factoid-get-fee/", serverFct)
	}

	resp, err := http.Get(getfeereq)

	if err != nil {
		fmt.Println("Command Failed Get")
		os.Exit(1)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println("Failed to understand response from fctwallet")
		os.Exit(1)
	}
	resp.Body.Close()

	// We pull the fee.  If the fee isn't positive, or if we fail to marshal, then there is a failure
	type x struct{
		Response string
		Success  bool
	}

	b := new(x)
	if err := json.Unmarshal(body, b); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}else if !b.Success {
		fmt.Println(b.Response)
		os.Exit(1)
	}
	if len(args)<2 {
		fmt.Printf("Currently, Entry Credits are %s Factoids each\n",b.Response)
	}else{
		fmt.Printf("The fee due for this transaction is %s Factoids\n", b.Response)
	}
}

func fctsign(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 1 {
		fmt.Println("Missing Key")
		os.Exit(1)
	}

	msg, valid := ValidateKey(args[0])
	if !valid {
		fmt.Println(msg)
		os.Exit(1)
	}

	str := fmt.Sprintf("http://%s/v1/factoid-sign-transaction/%s", serverFct, args[0])
	postCmd(str)

}

func fctsubmit(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 1 {
		fmt.Println("Missing Key")
		os.Exit(1)
	}

	msg, valid := ValidateKey(args[0])
	if !valid {
		fmt.Println(msg)
		os.Exit(1)
	}

	s := struct{ Transaction string }{args[0]}

	jdata, err := json.Marshal(s)
	if err != nil {
		fmt.Println("Submitt failed")
		os.Exit(1)
	}

	str := fmt.Sprintf("http://%s/v1/factoid-submit/%s", serverFct, bytes.NewBuffer(jdata))
	postCmd(str)
}

func fctsetup(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 1 {
		msg := "You must supply some random seed. For example (don't use this!)\n" +
			"  factom-cli setup 'woe!#in31!%234ng)%^&$%oeg%^&*^jp45694a;gmr@#t4 q34y'\n" +
			"would make a nice seed.  The more random the better.\n\n" +
			"Note that if you create an address before you call Setup, you must\n" +
			"use those address(s) as you access the fountians."

		fmt.Println(msg)
		os.Exit(1)
	}

	s := struct{ Transaction string }{args[0]}

	jdata, err := json.Marshal(s)
	if err != nil {
		fmt.Println("Submitt failed")
		os.Exit(1)
	}

	str := fmt.Sprintf("http://%s/v1/factoid-setup/%s", serverFct, bytes.NewBuffer(jdata))
	postCmd(str)
}
<commit_msg>fix for #673<commit_after>// Copyright 2015 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	fct "github.com/FactomProject/factoid"
	"github.com/FactomProject/factom"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
	"strconv"
)

var _ = hex.EncodeToString
var serverFct = "localhost:8089"

var badChar, _ = regexp.Compile("[^A-Za-z0-9_-]")

type Response struct {
	Response string
	Success  bool
}

func ValidateKey(key string) (msg string, valid bool) {
	if len(key) > fct.ADDRESS_LENGTH {
		return "Key is too long.  Keys must be less than or equal to 32 characters", false
	}
	if badChar.FindStringIndex(key) != nil {
		str := fmt.Sprintf("The key or name '%s' contains invalid characters.\n"+
			"Keys and names are restricted to alphanumeric characters,\n"+
			"minuses (dashes), and underscores", key)
		return str, false
	}
	return "", true
}

func getCmd(cmd string, cmderror string) {
	resp, err := http.Get(cmd)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	resp.Body.Close()

	b := new(Response)
	if err := json.Unmarshal(body, b); err != nil || !b.Success {
		fmt.Println(cmderror)
		fmt.Println("Command Failed: ", string(body))
		os.Exit(1)
	}
	fmt.Println(b.Response)
	return
}

func postCmd(cmd string) {
	resp, err := http.PostForm(cmd, nil)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	resp.Body.Close()

	b := new(Response)
	if err := json.Unmarshal(body, b); err != nil {
		fmt.Printf("Failed to parse the response from factomd: %s\n", body)
		os.Exit(1)
	}

	fmt.Println(b.Response)

	if !b.Success {
		os.Exit(1)
	}

	return
}

// Generates a new Address
func generateaddress(args []string) {

	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 2 {
		man("generatefactoidaddress")
		os.Exit(1)
	}

	msg, valid := ValidateKey(args[1])
	if !valid {
		fmt.Println(msg)
		os.Exit(1)
	}

	var err error
	var Addr string
	switch args[0] {
	case "ec":
		Addr, err = factom.GenerateEntryCreditAddress(args[1])
	case "fct":
		Addr, err = factom.GenerateFactoidAddress(args[1])
	default:
		panic("Expected ec|fct name")
	}

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	fmt.Println(args[0], " = ", Addr)
	return

}

func getaddresses(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) > 0 {
		man("getaddresses")
		os.Exit(1)
	}

	str := fmt.Sprintf("http://%s/v1/factoid-get-addresses/", serverFct)
	getCmd(str, "Error printing addresses")

	return
}

func gettransactions(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) > 0 {
		man("transactions")
		os.Exit(1)
	}

	str := fmt.Sprintf("http://%s/v1/factoid-get-transactions/", serverFct)
	getCmd(str, "Error Getting Transactions")

	return
}

func fctnewtrans(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 1 {
		fmt.Println("Missing Key")
		man("newtransaction")
		os.Exit(1)
	}
	msg, valid := ValidateKey(args[0])
	if !valid {
		fmt.Println(msg)
		os.Exit(1)
	}
	str := fmt.Sprintf("http://%s/v1/factoid-new-transaction/%s", serverFct, args[0])
	postCmd(str)

	return

}

func fctdeletetrans(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 1 {
		fmt.Println("Missing Key")
		man("deletetransaction")
		os.Exit(1)
	}
	msg, valid := ValidateKey(args[0])
	if !valid {
		fmt.Println(msg)
		os.Exit(1)
	}
	str := fmt.Sprintf("http://%s/v1/factoid-delete-transaction/%s", serverFct, args[0])
	postCmd(str)

	return

}

func fctaddfee(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 2 {
		fmt.Println("Was expecting a transaction key, and an address used as an input to that transaction.")
		os.Exit(1)
	}

	msg, valid := ValidateKey(args[0])
	if !valid {
		fmt.Println(msg)
		os.Exit(1)
	}

	str := fmt.Sprintf("http://%s/v1/factoid-add-fee/?key=%s&name=%s",
		serverFct, args[0], args[1])
	postCmd(str)

	return
}

func fctaddinput(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 3 {
		fmt.Println("Expecting a 1) transaction key, 2) an Address or Address name, and 3) an amount.")
		os.Exit(1)
	}

	msg, valid := ValidateKey(args[0])
	if !valid {
		fmt.Println(msg)
		os.Exit(1)
	}

	amt, err := fct.ConvertFixedPoint(args[2])
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	ramt, err := strconv.ParseInt(amt, 10, 64)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	_, err = fct.ValidateAmounts(uint64(ramt))
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	str := fmt.Sprintf("http://%s/v1/factoid-add-input/?key=%s&name=%s&amount=%s",
		serverFct, args[0], args[1], amt)
	postCmd(str)

	return
}

func fctaddoutput(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 3 {
		fmt.Println("Expecting a 1) transaction key, 2) an Address or Address name, and 3) an amount.")
		os.Exit(1)
	}
	// localhost:8089/v1/factoid-add-input/?key=<key>&name=<name or address>&amount=<amount>

	msg, valid := ValidateKey(args[0])
	if !valid {
		fmt.Println(msg)
		os.Exit(1)
	}

	amt, err := fct.ConvertFixedPoint(args[2])
	if err != nil {
		fmt.Println("Invalid format for a number: ", args[2])
		man("addoutput")
		os.Exit(1)
	}

	ramt, err := strconv.ParseInt(amt, 10, 64)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	_, err = fct.ValidateAmounts(uint64(ramt))
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	str := fmt.Sprintf("http://%s/v1/factoid-add-output/?key=%s&name=%s&amount=%s",
		serverFct, args[0], args[1], amt)
	postCmd(str)

	return
}

func fctaddecoutput(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 3 {
		fmt.Println("Expecting a 1) transaction key, 2) an Address or Address name, and 3) an amount.")
		os.Exit(1)
	}
	// localhost:8089/v1/factoid-add-input/?key=<key>&name=<name or address>&amount=<amount>

	msg, valid := ValidateKey(args[0])
	if !valid {
		fmt.Println(msg)
		os.Exit(1)
	}

	amt, err := fct.ConvertFixedPoint(args[2])
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	ramt, err := strconv.ParseInt(amt, 10, 64)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	_, err = fct.ValidateAmounts(uint64(ramt))
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	str := fmt.Sprintf("http://%s/v1/factoid-add-ecoutput/?key=%s&name=%s&amount=%s",
		serverFct, args[0], args[1], amt)
	postCmd(str)

	return
}

func fctgetfee(args []string) {
	var getfeereq string
	if len(args)>1 {
		getfeereq = fmt.Sprintf("http://%s/v1/factoid-get-fee/?key=%s", serverFct,args[1])
	}else{
		getfeereq = fmt.Sprintf("http://%s/v1/factoid-get-fee/", serverFct)
	}

	resp, err := http.Get(getfeereq)

	if err != nil {
		fmt.Println("Command Failed Get")
		os.Exit(1)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println("Failed to understand response from fctwallet")
		os.Exit(1)
	}
	resp.Body.Close()

	// We pull the fee.  If the fee isn't positive, or if we fail to marshal, then there is a failure
	type x struct{
		Response string
		Success  bool
	}

	b := new(x)
	if err := json.Unmarshal(body, b); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}else if !b.Success {
		fmt.Println(b.Response)
		os.Exit(1)
	}
	if len(args)<2 {
		fmt.Printf("Currently, Entry Credits are %s Factoids each\n",b.Response)
	}else{
		fmt.Printf("The fee due for this transaction is %s Factoids\n", b.Response)
	}
}

func fctsign(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 1 {
		fmt.Println("Missing Key")
		os.Exit(1)
	}

	msg, valid := ValidateKey(args[0])
	if !valid {
		fmt.Println(msg)
		os.Exit(1)
	}

	str := fmt.Sprintf("http://%s/v1/factoid-sign-transaction/%s", serverFct, args[0])
	postCmd(str)

}

func fctsubmit(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 1 {
		fmt.Println("Missing Key")
		os.Exit(1)
	}

	msg, valid := ValidateKey(args[0])
	if !valid {
		fmt.Println(msg)
		os.Exit(1)
	}

	s := struct{ Transaction string }{args[0]}

	jdata, err := json.Marshal(s)
	if err != nil {
		fmt.Println("Submitt failed")
		os.Exit(1)
	}

	str := fmt.Sprintf("http://%s/v1/factoid-submit/%s", serverFct, bytes.NewBuffer(jdata))
	postCmd(str)
}

func fctsetup(args []string) {
	os.Args = args
	flag.Parse()
	args = flag.Args()
	if len(args) < 1 {
		msg := "You must supply some random seed. For example (don't use this!)\n" +
			"  factom-cli setup 'woe!#in31!%234ng)%^&$%oeg%^&*^jp45694a;gmr@#t4 q34y'\n" +
			"would make a nice seed.  The more random the better.\n\n" +
			"Note that if you create an address before you call Setup, you must\n" +
			"use those address(s) as you access the fountians."

		fmt.Println(msg)
		os.Exit(1)
	}

	s := struct{ Transaction string }{args[0]}

	jdata, err := json.Marshal(s)
	if err != nil {
		fmt.Println("Submitt failed")
		os.Exit(1)
	}

	str := fmt.Sprintf("http://%s/v1/factoid-setup/%s", serverFct, bytes.NewBuffer(jdata))
	postCmd(str)
}
<|endoftext|>


<commit_before>package transloadit

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"
)

type Config struct {
	AuthKey    string
	AuthSecret string
	Region     string
	Endpoint   string
}

var DefaultConfig = Config{
	Region:   "us-east-1",
	Endpoint: "http://api2.transloadit.com",
}

type Client struct {
	config     Config
	httpClient *http.Client
}

// Options when retrieving a list.
// Look at the documentation which properties are accepted
// and to see their meaining, e.g. https://transloadit.com/docs/api-docs#retrieve-assembly-list
// for listing assemblies.
type ListOptions struct {
	Page       int        `json:"page,omitempty"`
	PageSize   int        `json:"pagesize,omitempty"`
	Sort       string     `json:"sort,omitempty"`
	Order      string     `json:"order,omitempty"`
	Fields     []string   `json:"fields,omitempty"`
	Type       string     `json:"type,omitempty"`
	Keywords   []string   `json:"keyword,omitempty"`
	AssemblyId string     `json:"assembly_id,omitempty"`
	FromDate   *time.Time `json:"fromdate,omitempty"`
	ToDate     *time.Time `json:"todate,omitempty"`
}

type authListOptions struct {
	*ListOptions

	// For internal use only!
	Auth struct {
		Key     string `json:"key"`
		Expires string `json:"expires"`
	} `json:"auth"`
}

type RequestError struct {
	Code    string `json:"error"`
	Message string `json:"message"`
}

func (err RequestError) Error() string {
	return fmt.Sprintf("request failed due to %s: %s", err.Code, err.Message)
}

// Create a new client using the provided configuration object.
// An error will be returned if no AuthKey or AuthSecret is found in config.
func NewClient(config Config) (*Client, error) {
	if config.AuthKey == "" {
		return nil, errors.New("failed to create client: missing AuthKey")
	}

	if config.AuthSecret == "" {
		return nil, errors.New("failed to create client: missing AuthSecret")
	}

	client := &Client{
		config:     config,
		httpClient: &http.Client{},
	}

	return client, nil
}

func (client *Client) sign(params map[string]interface{}) (string, string, error) {
	params["auth"] = map[string]string{
		"key":     client.config.AuthKey,
		"expires": getExpireString(),
	}

	b, err := json.Marshal(params)
	if err != nil {
		return "", "", fmt.Errorf("unable to create signature: %s", err)
	}

	hash := hmac.New(sha1.New, []byte(client.config.AuthSecret))
	hash.Write(b)
	return string(b), hex.EncodeToString(hash.Sum(nil)), nil
}

func (client *Client) doRequest(req *http.Request, result interface{}) error {
	req.Header.Set("User-Agent", "Transloadit Go SDK "+Version)

	res, err := client.httpClient.Do(req)
	if err != nil {
		return fmt.Errorf("failed execute http request: %s", err)
	}
	defer res.Body.Close()

	// Limit response to 128MB
	reader := io.LimitReader(res.Body, 128*1024*1024)
	decoder := json.NewDecoder(reader)

	if !(res.StatusCode >= 200 && res.StatusCode < 300) {
		var reqErr RequestError
		if err := decoder.Decode(&reqErr); err != nil {
			return fmt.Errorf("failed unmarshal http request: %s", err)
		}

		return reqErr
	}

	if result != nil {
		if err := decoder.Decode(result); err != nil {
			return fmt.Errorf("failed unmarshal http request: %s", err)
		}
	}

	return nil
}

func (client *Client) request(method string, path string, content map[string]interface{}, result interface{}) error {
	uri := path
	// Don't add host for absolute urls
	if u, err := url.Parse(path); err == nil && u.Scheme == "" {
		uri = client.config.Endpoint + "/" + path
	}

	// Ensure content is a map
	if content == nil {
		content = make(map[string]interface{})
	}

	// Create signature
	params, signature, err := client.sign(content)
	if err != nil {
		return fmt.Errorf("request: %s", err)
	}

	v := url.Values{}
	v.Set("params", params)
	v.Set("signature", signature)

	var body io.Reader
	if method == "GET" {
		uri += "?" + v.Encode()
	} else {
		body = strings.NewReader(v.Encode())
	}
	req, err := http.NewRequest(method, uri, body)
	if err != nil {
		return fmt.Errorf("request: %s", err)
	}

	if method != "GET" {
		// Add content type header
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	}

	return client.doRequest(req, result)
}

func (client *Client) listRequest(path string, listOptions *ListOptions, result interface{}) error {
	uri := client.config.Endpoint + "/" + path

	options := authListOptions{
		ListOptions: listOptions,
		Auth: struct {
			Key     string `json:"key"`
			Expires string `json:"expires"`
		}{
			Key:     client.config.AuthKey,
			Expires: getExpireString(),
		},
	}

	b, err := json.Marshal(options)
	if err != nil {
		return fmt.Errorf("unable to create signature: %s", err)
	}

	hash := hmac.New(sha1.New, []byte(client.config.AuthSecret))
	hash.Write(b)

	params := string(b)
	signature := hex.EncodeToString(hash.Sum(nil))

	v := url.Values{}
	v.Set("params", params)
	v.Set("signature", signature)

	uri += "?" + v.Encode()

	req, err := http.NewRequest("GET", uri, nil)
	if err != nil {
		return fmt.Errorf("request: %s", err)
	}

	return client.doRequest(req, result)
}

func getExpireString() string {
	// Expires in 1 hour
	expires := time.Now().UTC().Add(time.Hour)
	expiresStr := fmt.Sprintf("%04d/%02d/%02d %02d:%02d:%02d+00:00", expires.Year(), expires.Month(), expires.Day(), expires.Hour(), expires.Minute(), expires.Second())
	return string(expiresStr)
}
<commit_msg>Use structure for authentication parameters<commit_after>package transloadit

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"
)

type Config struct {
	AuthKey    string
	AuthSecret string
	Region     string
	Endpoint   string
}

var DefaultConfig = Config{
	Region:   "us-east-1",
	Endpoint: "http://api2.transloadit.com",
}

type Client struct {
	config     Config
	httpClient *http.Client
}

// Options when retrieving a list.
// Look at the documentation which properties are accepted
// and to see their meaining, e.g. https://transloadit.com/docs/api-docs#retrieve-assembly-list
// for listing assemblies.
type ListOptions struct {
	Page       int        `json:"page,omitempty"`
	PageSize   int        `json:"pagesize,omitempty"`
	Sort       string     `json:"sort,omitempty"`
	Order      string     `json:"order,omitempty"`
	Fields     []string   `json:"fields,omitempty"`
	Type       string     `json:"type,omitempty"`
	Keywords   []string   `json:"keyword,omitempty"`
	AssemblyId string     `json:"assembly_id,omitempty"`
	FromDate   *time.Time `json:"fromdate,omitempty"`
	ToDate     *time.Time `json:"todate,omitempty"`
}

type authParams struct {
	Key     string `json:"key"`
	Expires string `json:"expires"`
}

type authListOptions struct {
	*ListOptions

	// For internal use only!
	Auth authParams `json:"auth"`
}

type RequestError struct {
	Code    string `json:"error"`
	Message string `json:"message"`
}

func (err RequestError) Error() string {
	return fmt.Sprintf("request failed due to %s: %s", err.Code, err.Message)
}

// Create a new client using the provided configuration object.
// An error will be returned if no AuthKey or AuthSecret is found in config.
func NewClient(config Config) (*Client, error) {
	if config.AuthKey == "" {
		return nil, errors.New("failed to create client: missing AuthKey")
	}

	if config.AuthSecret == "" {
		return nil, errors.New("failed to create client: missing AuthSecret")
	}

	client := &Client{
		config:     config,
		httpClient: &http.Client{},
	}

	return client, nil
}

func (client *Client) sign(params map[string]interface{}) (string, string, error) {
	params["auth"] = authParams{
		Key:     client.config.AuthKey,
		Expires: getExpireString(),
	}

	b, err := json.Marshal(params)
	if err != nil {
		return "", "", fmt.Errorf("unable to create signature: %s", err)
	}

	hash := hmac.New(sha1.New, []byte(client.config.AuthSecret))
	hash.Write(b)
	return string(b), hex.EncodeToString(hash.Sum(nil)), nil
}

func (client *Client) doRequest(req *http.Request, result interface{}) error {
	req.Header.Set("User-Agent", "Transloadit Go SDK "+Version)

	res, err := client.httpClient.Do(req)
	if err != nil {
		return fmt.Errorf("failed execute http request: %s", err)
	}
	defer res.Body.Close()

	// Limit response to 128MB
	reader := io.LimitReader(res.Body, 128*1024*1024)
	decoder := json.NewDecoder(reader)

	if !(res.StatusCode >= 200 && res.StatusCode < 300) {
		var reqErr RequestError
		if err := decoder.Decode(&reqErr); err != nil {
			return fmt.Errorf("failed unmarshal http request: %s", err)
		}

		return reqErr
	}

	if result != nil {
		if err := decoder.Decode(result); err != nil {
			return fmt.Errorf("failed unmarshal http request: %s", err)
		}
	}

	return nil
}

func (client *Client) request(method string, path string, content map[string]interface{}, result interface{}) error {
	uri := path
	// Don't add host for absolute urls
	if u, err := url.Parse(path); err == nil && u.Scheme == "" {
		uri = client.config.Endpoint + "/" + path
	}

	// Ensure content is a map
	if content == nil {
		content = make(map[string]interface{})
	}

	// Create signature
	params, signature, err := client.sign(content)
	if err != nil {
		return fmt.Errorf("request: %s", err)
	}

	v := url.Values{}
	v.Set("params", params)
	v.Set("signature", signature)

	var body io.Reader
	if method == "GET" {
		uri += "?" + v.Encode()
	} else {
		body = strings.NewReader(v.Encode())
	}
	req, err := http.NewRequest(method, uri, body)
	if err != nil {
		return fmt.Errorf("request: %s", err)
	}

	if method != "GET" {
		// Add content type header
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	}

	return client.doRequest(req, result)
}

func (client *Client) listRequest(path string, listOptions *ListOptions, result interface{}) error {
	uri := client.config.Endpoint + "/" + path

	options := authListOptions{
		ListOptions: listOptions,
		Auth: authParams{
			Key:     client.config.AuthKey,
			Expires: getExpireString(),
		},
	}

	b, err := json.Marshal(options)
	if err != nil {
		return fmt.Errorf("unable to create signature: %s", err)
	}

	hash := hmac.New(sha1.New, []byte(client.config.AuthSecret))
	hash.Write(b)

	params := string(b)
	signature := hex.EncodeToString(hash.Sum(nil))

	v := url.Values{}
	v.Set("params", params)
	v.Set("signature", signature)

	uri += "?" + v.Encode()

	req, err := http.NewRequest("GET", uri, nil)
	if err != nil {
		return fmt.Errorf("request: %s", err)
	}

	return client.doRequest(req, result)
}

func getExpireString() string {
	// Expires in 1 hour
	expires := time.Now().UTC().Add(time.Hour)
	expiresStr := fmt.Sprintf("%04d/%02d/%02d %02d:%02d:%02d+00:00", expires.Year(), expires.Month(), expires.Day(), expires.Hour(), expires.Minute(), expires.Second())
	return string(expiresStr)
}
<|endoftext|>


<commit_before>// Copyright 2015 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types

import (
	"fmt"
	"regexp"

	"github.com/prometheus/common/model"
)

// Matcher defines a matching rule for the value of a given label.
type Matcher struct {
	Name    string `json:"name"`
	Value   string `json:"value"`
	IsRegex bool   `json:"isRegex"`

	regex *regexp.Regexp
}

// Init internals of the Matcher. Must be called before using Match.
func (m *Matcher) Init() error {
	if !m.IsRegex {
		return nil
	}
	re, err := regexp.Compile(m.Value)
	if err == nil {
		m.regex = re
	}
	return err
}

func (m *Matcher) String() string {
	if m.IsRegex {
		return fmt.Sprintf("<RegexMatcher %s:%q>", m.Name, m.Value)
	}
	return fmt.Sprintf("<Matcher %s:%q>", m.Name, m.Value)
}

// Match checks whether the label of the matcher has the specified
// matching value.
func (m *Matcher) Match(lset model.LabelSet) bool {
	// Unset labels are treated as unset labels globally. Thus, if a
	// label is not set we retrieve the empty label which is correct
	// for the comparison below.
	v := lset[model.LabelName(m.Name)]

	if m.IsRegex {
		return m.regex.MatchString(string(v))
	}
	return string(v) == m.Value
}

// NewMatcher returns a new matcher that compares against equality of
// the given value.
func NewMatcher(name model.LabelName, value string) *Matcher {
	return &Matcher{
		Name:    string(name),
		Value:   value,
		IsRegex: false,
	}
}

// NewRegexMatcher returns a new matcher that compares values against
// a regular expression. The matcher is already initialized.
//
// TODO(fabxc): refactor usage.
func NewRegexMatcher(name model.LabelName, re *regexp.Regexp) *Matcher {
	return &Matcher{
		Name:    string(name),
		Value:   re.String(),
		IsRegex: true,
		regex:   re,
	}
}

// Matchers provides the Match and Fingerprint methods for a slice of Matchers.
type Matchers []*Matcher

// Match checks whether all matchers are fulfilled against the given label set.
func (ms Matchers) Match(lset model.LabelSet) bool {
	for _, m := range ms {
		if !m.Match(lset) {
			return false
		}
	}
	return true
}

// Validate returns true iff all fields of the matcher have valid values.
func (m *Matcher) Validate() error {
	if !model.LabelName(m.Name).IsValid() {
		return fmt.Errorf("invalid name %q", m.Name)
	}
	if m.IsRegex {
		if _, err := regexp.Compile(m.Value); err != nil {
			return fmt.Errorf("invalid regular expression %q", m.Value)
		}
	} else if !model.LabelValue(m.Value).IsValid() || len(m.Value) == 0 {
		return fmt.Errorf("invalid value %q", m.Value)
	}
	return nil
}

// Fingerprint returns a quasi-unique fingerprint for the matchers.
func (ms Matchers) Fingerprint() model.Fingerprint {
	lset := make(model.LabelSet, 3*len(ms))

	for _, m := range ms {
		lset[model.LabelName(fmt.Sprintf("%s-%s-%v", m.Name, m.Value, m.IsRegex))] = ""
	}

	return lset.Fingerprint()
}
<commit_msg>types: make Matchers sorted and compareable<commit_after>// Copyright 2015 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types

import (
	"fmt"
	"regexp"
	"sort"

	"github.com/prometheus/common/model"
)

// Matcher defines a matching rule for the value of a given label.
type Matcher struct {
	Name    string `json:"name"`
	Value   string `json:"value"`
	IsRegex bool   `json:"isRegex"`

	regex *regexp.Regexp
}

// Init internals of the Matcher. Must be called before using Match.
func (m *Matcher) Init() error {
	if !m.IsRegex {
		return nil
	}
	re, err := regexp.Compile(m.Value)
	if err == nil {
		m.regex = re
	}
	return err
}

func (m *Matcher) String() string {
	if m.IsRegex {
		return fmt.Sprintf("<RegexMatcher %s:%q>", m.Name, m.Value)
	}
	return fmt.Sprintf("<Matcher %s:%q>", m.Name, m.Value)
}

// Match checks whether the label of the matcher has the specified
// matching value.
func (m *Matcher) Match(lset model.LabelSet) bool {
	// Unset labels are treated as unset labels globally. Thus, if a
	// label is not set we retrieve the empty label which is correct
	// for the comparison below.
	v := lset[model.LabelName(m.Name)]

	if m.IsRegex {
		return m.regex.MatchString(string(v))
	}
	return string(v) == m.Value
}

// NewMatcher returns a new matcher that compares against equality of
// the given value.
func NewMatcher(name model.LabelName, value string) *Matcher {
	return &Matcher{
		Name:    string(name),
		Value:   value,
		IsRegex: false,
	}
}

// NewRegexMatcher returns a new matcher that compares values against
// a regular expression. The matcher is already initialized.
//
// TODO(fabxc): refactor usage.
func NewRegexMatcher(name model.LabelName, re *regexp.Regexp) *Matcher {
	return &Matcher{
		Name:    string(name),
		Value:   re.String(),
		IsRegex: true,
		regex:   re,
	}
}

// Matchers provides the Match and Fingerprint methods for a slice of Matchers.
// Matchers must always be sorted.
type Matchers []*Matcher

// NewMatchers returns the given Matchers sorted.
func NewMatchers(ms ...*Matcher) Matchers {
	m := Matchers(ms)
	sort.Sort(m)
	return m
}

func (ms Matchers) Len() int      { return len(ms) }
func (ms Matchers) Swap(i, j int) { ms[i], ms[j] = ms[j], ms[i] }

func (ms Matchers) Less(i, j int) bool {
	if ms[i].Name > ms[j].Name {
		return false
	}
	if ms[i].Name < ms[j].Name {
		return true
	}
	if ms[i].Value > ms[j].Value {
		return false
	}
	if ms[i].Value < ms[j].Value {
		return true
	}
	return !ms[i].IsRegex && ms[j].IsRegex
}

// Equal returns whether both Matchers are equal.
func (ms Matchers) Equal(o Matchers) bool {
	if len(ms) != len(o) {
		return false
	}
	for i, a := range ms {
		if *a != *o[i] {
			return false
		}
	}
	return true
}

// Match checks whether all matchers are fulfilled against the given label set.
func (ms Matchers) Match(lset model.LabelSet) bool {
	for _, m := range ms {
		if !m.Match(lset) {
			return false
		}
	}
	return true
}

// Validate returns true iff all fields of the matcher have valid values.
func (m *Matcher) Validate() error {
	if !model.LabelName(m.Name).IsValid() {
		return fmt.Errorf("invalid name %q", m.Name)
	}
	if m.IsRegex {
		if _, err := regexp.Compile(m.Value); err != nil {
			return fmt.Errorf("invalid regular expression %q", m.Value)
		}
	} else if !model.LabelValue(m.Value).IsValid() || len(m.Value) == 0 {
		return fmt.Errorf("invalid value %q", m.Value)
	}
	return nil
}

// Fingerprint returns a quasi-unique fingerprint for the matchers.
func (ms Matchers) Fingerprint() model.Fingerprint {
	lset := make(model.LabelSet, 3*len(ms))

	for _, m := range ms {
		lset[model.LabelName(fmt.Sprintf("%s-%s-%v", m.Name, m.Value, m.IsRegex))] = ""
	}

	return lset.Fingerprint()
}
<|endoftext|>


<commit_before>package types

import "github.com/docker/docker/pkg/version"

// ContainerCreateResponse contains the information returned to a client on the
// creation of a new container.
type ContainerCreateResponse struct {
	// ID is the ID of the created container.
	ID string `json:"Id"`

	// Warnings are any warnings encountered during the creation of the container.
	Warnings []string `json:"Warnings"`
}

// POST /containers/{name:.*}/exec
type ContainerExecCreateResponse struct {
	// ID is the exec ID.
	ID string `json:"Id"`
}

// POST /auth
type AuthResponse struct {
	// Status is the authentication status
	Status string `json:"Status"`
}

// POST "/containers/"+containerID+"/wait"
type ContainerWaitResponse struct {
	// StatusCode is the status code of the wait job
	StatusCode int `json:"StatusCode"`
}

// POST "/commit?container="+containerID
type ContainerCommitResponse struct {
	ID string `json:"Id"`
}

// GET "/containers/{name:.*}/changes"
type ContainerChange struct {
	Kind int
	Path string
}

// GET "/images/{name:.*}/history"
type ImageHistory struct {
	ID        string `json:"Id"`
	Created   int64
	CreatedBy string
	Tags      []string
	Size      int64
	Comment   string
}

// DELETE "/images/{name:.*}"
type ImageDelete struct {
	Untagged string `json:",omitempty"`
	Deleted  string `json:",omitempty"`
}

// GET "/images/json"
type Image struct {
	ID          string `json:"Id"`
	ParentId    string
	RepoTags    []string
	RepoDigests []string
	Created     int
	Size        int
	VirtualSize int
	Labels      map[string]string
}

type LegacyImage struct {
	ID          string `json:"Id"`
	Repository  string
	Tag         string
	Created     int
	Size        int
	VirtualSize int
}

// GET  "/containers/json"
type Port struct {
	IP          string
	PrivatePort int
	PublicPort  int
	Type        string
}

type Container struct {
	ID         string            `json:"Id"`
	Names      []string          `json:",omitempty"`
	Image      string            `json:",omitempty"`
	Command    string            `json:",omitempty"`
	Created    int               `json:",omitempty"`
	Ports      []Port            `json:",omitempty"`
	SizeRw     int               `json:",omitempty"`
	SizeRootFs int               `json:",omitempty"`
	Labels     map[string]string `json:",omitempty"`
	Status     string            `json:",omitempty"`
}

// POST "/containers/"+containerID+"/copy"
type CopyConfig struct {
	Resource string
}

// GET "/containers/{name:.*}/top"
type ContainerProcessList struct {
	Processes [][]string
	Titles    []string
}

type Version struct {
	Version       string
	ApiVersion    version.Version
	GitCommit     string
	GoVersion     string
	Os            string
	Arch          string
	KernelVersion string `json:",omitempty"`
}

// GET "/info"
type Info struct {
	ID                 string
	Containers         int
	Images             int
	Driver             string
	DriverStatus       [][2]string
	MemoryLimit        bool
	SwapLimit          bool
	CpuCfsQuota        bool
	IPv4Forwarding     bool
	Debug              bool
	NFd                int
	NGoroutines        int
	SystemTime         string
	ExecutionDriver    string
	LoggingDriver      string
	NEventsListener    int
	KernelVersion      string
	OperatingSystem    string
	IndexServerAddress string
	RegistryConfig     interface{}
	InitSha1           string
	InitPath           string
	NCPU               int
	MemTotal           int64
	DockerRootDir      string
	HttpProxy          string
	HttpsProxy         string
	NoProxy            string
	Name               string
	Labels             []string
}

// This struct is a temp struct used by execStart
// Config fields is part of ExecConfig in runconfig package
type ExecStartCheck struct {
	// ExecStart will first check if it's detached
	Detach bool
	// Check if there's a tty
	Tty bool
}
<commit_msg>Remove job from container_inspect<commit_after>package types

import (
	"time"

	"github.com/docker/docker/daemon/network"
	"github.com/docker/docker/pkg/version"
	"github.com/docker/docker/runconfig"
)

// ContainerCreateResponse contains the information returned to a client on the
// creation of a new container.
type ContainerCreateResponse struct {
	// ID is the ID of the created container.
	ID string `json:"Id"`

	// Warnings are any warnings encountered during the creation of the container.
	Warnings []string `json:"Warnings"`
}

// POST /containers/{name:.*}/exec
type ContainerExecCreateResponse struct {
	// ID is the exec ID.
	ID string `json:"Id"`
}

// POST /auth
type AuthResponse struct {
	// Status is the authentication status
	Status string `json:"Status"`
}

// POST "/containers/"+containerID+"/wait"
type ContainerWaitResponse struct {
	// StatusCode is the status code of the wait job
	StatusCode int `json:"StatusCode"`
}

// POST "/commit?container="+containerID
type ContainerCommitResponse struct {
	ID string `json:"Id"`
}

// GET "/containers/{name:.*}/changes"
type ContainerChange struct {
	Kind int
	Path string
}

// GET "/images/{name:.*}/history"
type ImageHistory struct {
	ID        string `json:"Id"`
	Created   int64
	CreatedBy string
	Tags      []string
	Size      int64
	Comment   string
}

// DELETE "/images/{name:.*}"
type ImageDelete struct {
	Untagged string `json:",omitempty"`
	Deleted  string `json:",omitempty"`
}

// GET "/images/json"
type Image struct {
	ID          string `json:"Id"`
	ParentId    string
	RepoTags    []string
	RepoDigests []string
	Created     int
	Size        int
	VirtualSize int
	Labels      map[string]string
}

type LegacyImage struct {
	ID          string `json:"Id"`
	Repository  string
	Tag         string
	Created     int
	Size        int
	VirtualSize int
}

// GET  "/containers/json"
type Port struct {
	IP          string
	PrivatePort int
	PublicPort  int
	Type        string
}

type Container struct {
	ID         string            `json:"Id"`
	Names      []string          `json:",omitempty"`
	Image      string            `json:",omitempty"`
	Command    string            `json:",omitempty"`
	Created    int               `json:",omitempty"`
	Ports      []Port            `json:",omitempty"`
	SizeRw     int               `json:",omitempty"`
	SizeRootFs int               `json:",omitempty"`
	Labels     map[string]string `json:",omitempty"`
	Status     string            `json:",omitempty"`
}

// POST "/containers/"+containerID+"/copy"
type CopyConfig struct {
	Resource string
}

// GET "/containers/{name:.*}/top"
type ContainerProcessList struct {
	Processes [][]string
	Titles    []string
}

type Version struct {
	Version       string
	ApiVersion    version.Version
	GitCommit     string
	GoVersion     string
	Os            string
	Arch          string
	KernelVersion string `json:",omitempty"`
}

// GET "/info"
type Info struct {
	ID                 string
	Containers         int
	Images             int
	Driver             string
	DriverStatus       [][2]string
	MemoryLimit        bool
	SwapLimit          bool
	CpuCfsQuota        bool
	IPv4Forwarding     bool
	Debug              bool
	NFd                int
	NGoroutines        int
	SystemTime         string
	ExecutionDriver    string
	LoggingDriver      string
	NEventsListener    int
	KernelVersion      string
	OperatingSystem    string
	IndexServerAddress string
	RegistryConfig     interface{}
	InitSha1           string
	InitPath           string
	NCPU               int
	MemTotal           int64
	DockerRootDir      string
	HttpProxy          string
	HttpsProxy         string
	NoProxy            string
	Name               string
	Labels             []string
}

// This struct is a temp struct used by execStart
// Config fields is part of ExecConfig in runconfig package
type ExecStartCheck struct {
	// ExecStart will first check if it's detached
	Detach bool
	// Check if there's a tty
	Tty bool
}

type ContainerState struct {
	Running    bool
	Paused     bool
	Restarting bool
	OOMKilled  bool
	Dead       bool
	Pid        int
	ExitCode   int
	Error      string
	StartedAt  time.Time
	FinishedAt time.Time
}

// GET "/containers/{name:.*}/json"
type ContainerJSON struct {
	Id              string
	Created         time.Time
	Path            string
	Args            []string
	Config          *runconfig.Config
	State           *ContainerState
	Image           string
	NetworkSettings *network.Settings
	ResolvConfPath  string
	HostnamePath    string
	HostsPath       string
	LogPath         string
	Name            string
	RestartCount    int
	Driver          string
	ExecDriver      string
	MountLabel      string
	ProcessLabel    string
	Volumes         map[string]string
	VolumesRW       map[string]bool
	AppArmorProfile string
	ExecIDs         []string
	HostConfig      *runconfig.HostConfig
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/git-lfs/git-lfs/lfs"
	"github.com/git-lfs/git-lfs/localstorage"
	"github.com/git-lfs/git-lfs/subprocess"

	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/spf13/cobra"
)

var (
	cloneFlags git.CloneFlags

	cloneSkipRepoInstall bool

	// cloneIsDeprecated marks whether or not the clone command is
	// deprecated. It is false until Git v2.15.0 is released, including the
	// 'delay' capability.
	cloneIsDeprecated = false
)

func cloneCommand(cmd *cobra.Command, args []string) {
	requireGitVersion()

	if cloneIsDeprecated {
		msg := []string{
			"WARNING: 'git lfs clone' is deprecated and will not be updated",
			"          with new flags from 'git clone'",
			"",
			"'git clone' has been updated in upstream Git to have comparable",
			"speeds to 'git lfs clone'.",
		}

		fmt.Fprintln(os.Stderr, strings.Join(msg, "\n")+"\n")
	}

	// We pass all args to git clone
	err := git.CloneWithoutFilters(cloneFlags, args)
	if err != nil {
		Exit("Error(s) during clone:\n%v", err)
	}

	// now execute pull (need to be inside dir)
	cwd, err := tools.Getwd()
	if err != nil {
		Exit("Unable to derive current working dir: %v", err)
	}

	// Either the last argument was a relative or local dir, or we have to
	// derive it from the clone URL
	clonedir, err := filepath.Abs(args[len(args)-1])
	if err != nil || !tools.DirExists(clonedir) {
		// Derive from clone URL instead
		base := path.Base(args[len(args)-1])
		if strings.HasSuffix(base, ".git") {
			base = base[:len(base)-4]
		}
		clonedir, _ = filepath.Abs(base)
		if !tools.DirExists(clonedir) {
			Exit("Unable to find clone dir at %q", clonedir)
		}
	}

	err = os.Chdir(clonedir)
	if err != nil {
		Exit("Unable to change directory to clone dir %q: %v", clonedir, err)
	}

	// Make sure we pop back to dir we started in at the end
	defer os.Chdir(cwd)

	// Also need to derive dirs now
	localstorage.ResolveDirs()
	requireInRepo()

	// Now just call pull with default args
	// Support --origin option to clone
	var remote string
	if len(cloneFlags.Origin) > 0 {
		remote = cloneFlags.Origin
	} else {
		remote = "origin"
	}

	includeArg, excludeArg := getIncludeExcludeArgs(cmd)
	filter := buildFilepathFilter(cfg, includeArg, excludeArg)
	if cloneFlags.NoCheckout || cloneFlags.Bare {
		// If --no-checkout or --bare then we shouldn't check out, just fetch instead
		cfg.CurrentRemote = remote
		fetchRef("HEAD", filter)
	} else {
		pull(remote, filter)
		err := postCloneSubmodules(args)
		if err != nil {
			Exit("Error performing 'git lfs pull' for submodules: %v", err)
		}
	}

	if !cloneSkipRepoInstall {
		// If --skip-repo wasn't given, install repo-level hooks while
		// we're still in the checkout directory.

		if err := lfs.InstallHooks(false); err != nil {
			ExitWithError(err)
		}
	}
}

func postCloneSubmodules(args []string) error {
	// In git 2.9+ the filter option will have been passed through to submodules
	// So we need to lfs pull inside each
	if !git.Config.IsGitVersionAtLeast("2.9.0") {
		// In earlier versions submodules would have used smudge filter
		return nil
	}
	// Also we only do this if --recursive or --recurse-submodules was provided
	if !cloneFlags.Recursive && !cloneFlags.RecurseSubmodules {
		return nil
	}

	// Use `git submodule foreach --recursive` to cascade into nested submodules
	// Also good to call a new instance of git-lfs rather than do things
	// inside this instance, since that way we get a clean env in that subrepo
	cmd := subprocess.ExecCommand("git", "submodule", "foreach", "--recursive",
		"git lfs pull")
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func init() {
	RegisterCommand("clone", cloneCommand, func(cmd *cobra.Command) {
		cmd.PreRun = nil

		// Mirror all git clone flags
		cmd.Flags().StringVarP(&cloneFlags.TemplateDirectory, "template", "", "", "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Local, "local", "l", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Shared, "shared", "s", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.NoHardlinks, "no-hardlinks", "", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Quiet, "quiet", "q", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.NoCheckout, "no-checkout", "n", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Progress, "progress", "", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Bare, "bare", "", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Mirror, "mirror", "", false, "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.Origin, "origin", "o", "", "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.Branch, "branch", "b", "", "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.Upload, "upload-pack", "u", "", "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.Reference, "reference", "", "", "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.ReferenceIfAble, "reference-if-able", "", "", "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Dissociate, "dissociate", "", false, "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.SeparateGit, "separate-git-dir", "", "", "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.Depth, "depth", "", "", "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Recursive, "recursive", "", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.RecurseSubmodules, "recurse-submodules", "", false, "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.Config, "config", "c", "", "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.SingleBranch, "single-branch", "", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.NoSingleBranch, "no-single-branch", "", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Verbose, "verbose", "v", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Ipv4, "ipv4", "", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Ipv6, "ipv6", "", false, "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.ShallowSince, "shallow-since", "", "", "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.ShallowExclude, "shallow-exclude", "", "", "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.ShallowSubmodules, "shallow-submodules", "", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.NoShallowSubmodules, "no-shallow-submodules", "", false, "See 'git clone --help'")
		cmd.Flags().Int64VarP(&cloneFlags.Jobs, "jobs", "j", -1, "See 'git clone --help'")

		cmd.Flags().StringVarP(&includeArg, "include", "I", "", "Include a list of paths")
		cmd.Flags().StringVarP(&excludeArg, "exclude", "X", "", "Exclude a list of paths")

		cmd.Flags().BoolVar(&cloneSkipRepoInstall, "skip-repo", false, "Skip LFS repo setup")
	})
}
<commit_msg>commands: remove extraneous trailing LF<commit_after>package commands

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/git-lfs/git-lfs/lfs"
	"github.com/git-lfs/git-lfs/localstorage"
	"github.com/git-lfs/git-lfs/subprocess"

	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/spf13/cobra"
)

var (
	cloneFlags git.CloneFlags

	cloneSkipRepoInstall bool

	// cloneIsDeprecated marks whether or not the clone command is
	// deprecated. It is false until Git v2.15.0 is released, including the
	// 'delay' capability.
	cloneIsDeprecated = false
)

func cloneCommand(cmd *cobra.Command, args []string) {
	requireGitVersion()

	if cloneIsDeprecated {
		msg := []string{
			"WARNING: 'git lfs clone' is deprecated and will not be updated",
			"          with new flags from 'git clone'",
			"",
			"'git clone' has been updated in upstream Git to have comparable",
			"speeds to 'git lfs clone'.",
		}

		fmt.Fprintln(os.Stderr, strings.Join(msg, "\n"))
	}

	// We pass all args to git clone
	err := git.CloneWithoutFilters(cloneFlags, args)
	if err != nil {
		Exit("Error(s) during clone:\n%v", err)
	}

	// now execute pull (need to be inside dir)
	cwd, err := tools.Getwd()
	if err != nil {
		Exit("Unable to derive current working dir: %v", err)
	}

	// Either the last argument was a relative or local dir, or we have to
	// derive it from the clone URL
	clonedir, err := filepath.Abs(args[len(args)-1])
	if err != nil || !tools.DirExists(clonedir) {
		// Derive from clone URL instead
		base := path.Base(args[len(args)-1])
		if strings.HasSuffix(base, ".git") {
			base = base[:len(base)-4]
		}
		clonedir, _ = filepath.Abs(base)
		if !tools.DirExists(clonedir) {
			Exit("Unable to find clone dir at %q", clonedir)
		}
	}

	err = os.Chdir(clonedir)
	if err != nil {
		Exit("Unable to change directory to clone dir %q: %v", clonedir, err)
	}

	// Make sure we pop back to dir we started in at the end
	defer os.Chdir(cwd)

	// Also need to derive dirs now
	localstorage.ResolveDirs()
	requireInRepo()

	// Now just call pull with default args
	// Support --origin option to clone
	var remote string
	if len(cloneFlags.Origin) > 0 {
		remote = cloneFlags.Origin
	} else {
		remote = "origin"
	}

	includeArg, excludeArg := getIncludeExcludeArgs(cmd)
	filter := buildFilepathFilter(cfg, includeArg, excludeArg)
	if cloneFlags.NoCheckout || cloneFlags.Bare {
		// If --no-checkout or --bare then we shouldn't check out, just fetch instead
		cfg.CurrentRemote = remote
		fetchRef("HEAD", filter)
	} else {
		pull(remote, filter)
		err := postCloneSubmodules(args)
		if err != nil {
			Exit("Error performing 'git lfs pull' for submodules: %v", err)
		}
	}

	if !cloneSkipRepoInstall {
		// If --skip-repo wasn't given, install repo-level hooks while
		// we're still in the checkout directory.

		if err := lfs.InstallHooks(false); err != nil {
			ExitWithError(err)
		}
	}
}

func postCloneSubmodules(args []string) error {
	// In git 2.9+ the filter option will have been passed through to submodules
	// So we need to lfs pull inside each
	if !git.Config.IsGitVersionAtLeast("2.9.0") {
		// In earlier versions submodules would have used smudge filter
		return nil
	}
	// Also we only do this if --recursive or --recurse-submodules was provided
	if !cloneFlags.Recursive && !cloneFlags.RecurseSubmodules {
		return nil
	}

	// Use `git submodule foreach --recursive` to cascade into nested submodules
	// Also good to call a new instance of git-lfs rather than do things
	// inside this instance, since that way we get a clean env in that subrepo
	cmd := subprocess.ExecCommand("git", "submodule", "foreach", "--recursive",
		"git lfs pull")
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func init() {
	RegisterCommand("clone", cloneCommand, func(cmd *cobra.Command) {
		cmd.PreRun = nil

		// Mirror all git clone flags
		cmd.Flags().StringVarP(&cloneFlags.TemplateDirectory, "template", "", "", "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Local, "local", "l", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Shared, "shared", "s", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.NoHardlinks, "no-hardlinks", "", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Quiet, "quiet", "q", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.NoCheckout, "no-checkout", "n", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Progress, "progress", "", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Bare, "bare", "", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Mirror, "mirror", "", false, "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.Origin, "origin", "o", "", "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.Branch, "branch", "b", "", "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.Upload, "upload-pack", "u", "", "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.Reference, "reference", "", "", "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.ReferenceIfAble, "reference-if-able", "", "", "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Dissociate, "dissociate", "", false, "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.SeparateGit, "separate-git-dir", "", "", "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.Depth, "depth", "", "", "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Recursive, "recursive", "", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.RecurseSubmodules, "recurse-submodules", "", false, "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.Config, "config", "c", "", "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.SingleBranch, "single-branch", "", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.NoSingleBranch, "no-single-branch", "", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Verbose, "verbose", "v", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Ipv4, "ipv4", "", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.Ipv6, "ipv6", "", false, "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.ShallowSince, "shallow-since", "", "", "See 'git clone --help'")
		cmd.Flags().StringVarP(&cloneFlags.ShallowExclude, "shallow-exclude", "", "", "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.ShallowSubmodules, "shallow-submodules", "", false, "See 'git clone --help'")
		cmd.Flags().BoolVarP(&cloneFlags.NoShallowSubmodules, "no-shallow-submodules", "", false, "See 'git clone --help'")
		cmd.Flags().Int64VarP(&cloneFlags.Jobs, "jobs", "j", -1, "See 'git clone --help'")

		cmd.Flags().StringVarP(&includeArg, "include", "I", "", "Include a list of paths")
		cmd.Flags().StringVarP(&excludeArg, "exclude", "X", "", "Exclude a list of paths")

		cmd.Flags().BoolVar(&cloneSkipRepoInstall, "skip-repo", false, "Skip LFS repo setup")
	})
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"time"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)

var (
	fetchCmd = &cobra.Command{
		Use:   "fetch",
		Short: "Downloads LFS files",
		Run:   fetchCommand,
	}
	fetchIncludeArg string
	fetchExcludeArg string
	fetchRecentArg  bool
	fetchAllArg     bool
)

func fetchCommand(cmd *cobra.Command, args []string) {
	var refs []*git.Ref

	if len(args) > 0 {
		// Remote is first arg
		lfs.Config.CurrentRemote = args[0]
	} else {
		trackedRemote, err := git.CurrentRemote()
		if err == nil {
			lfs.Config.CurrentRemote = trackedRemote
		} // otherwise leave as default (origin)
	}
	if len(args) > 1 {
		for _, r := range args[1:] {
			ref, err := git.ResolveRef(r)
			if err != nil {
				Panic(err, "Invalid ref argument")
			}
			refs = append(refs, ref)
		}
	} else {
		ref, err := git.CurrentRef()
		if err != nil {
			Panic(err, "Could not fetch")
		}
		refs = []*git.Ref{ref}
	}

	if fetchAllArg && (fetchRecentArg || len(refs) > 0) {
		Exit("Cannot combined --all with ref arguments or --recent")
	}

	if fetchAllArg {
		if fetchRecentArg || len(refs) > 0 {
			Exit("Cannot combine --all with ref arguments or --recent")
		}
		if fetchIncludeArg != "" || fetchExcludeArg != "" {
			Exit("Cannot combine --all with --include or --exclude")
		}
		if len(lfs.Config.FetchIncludePaths()) > 0 || len(lfs.Config.FetchExcludePaths()) > 0 {
			Print("Ignoring global include / exclude paths to fulfil --all")
		}
		fetchAll()

	} else { // !all

		includePaths, excludePaths := determineIncludeExcludePaths(fetchIncludeArg, fetchExcludeArg)

		// Fetch refs sequentially per arg order; duplicates in later refs will be ignored
		for _, ref := range refs {
			Print("Fetching %v", ref.Name)
			fetchRef(ref.Sha, includePaths, excludePaths)
		}

		if fetchRecentArg || lfs.Config.FetchPruneConfig().FetchRecentAlways {
			fetchRecent(refs, includePaths, excludePaths)
		}
	}
}

func init() {
	fetchCmd.Flags().StringVarP(&fetchIncludeArg, "include", "I", "", "Include a list of paths")
	fetchCmd.Flags().StringVarP(&fetchExcludeArg, "exclude", "X", "", "Exclude a list of paths")
	fetchCmd.Flags().BoolVarP(&fetchRecentArg, "recent", "r", false, "Fetch recent refs & commits")
	fetchCmd.Flags().BoolVarP(&fetchAllArg, "all", "a", false, "Fetch all LFS files ever referenced")
	RootCmd.AddCommand(fetchCmd)
}

func pointersToFetchForRef(ref string) ([]*lfs.WrappedPointer, error) {
	// Use SkipDeletedBlobs to avoid fetching ALL previous versions of modified files
	opts := &lfs.ScanRefsOptions{ScanMode: lfs.ScanRefsMode, SkipDeletedBlobs: true}
	return lfs.ScanRefs(ref, "", opts)
}

func fetchRefToChan(ref string, include, exclude []string) chan *lfs.WrappedPointer {
	c := make(chan *lfs.WrappedPointer)
	pointers, err := pointersToFetchForRef(ref)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}

	go fetchAndReportToChan(pointers, include, exclude, c)

	return c
}

// Fetch all binaries for a given ref (that we don't have already)
func fetchRef(ref string, include, exclude []string) {
	pointers, err := pointersToFetchForRef(ref)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}
	fetchPointers(pointers, include, exclude)
}

// Fetch all previous versions of objects from since to ref (not including final state at ref)
// So this will fetch all the '-' sides of the diff from since to ref
func fetchPreviousVersions(ref string, since time.Time, include, exclude []string) {
	pointers, err := lfs.ScanPreviousVersions(ref, since)
	if err != nil {
		Panic(err, "Could not scan for Git LFS previous versions")
	}
	fetchPointers(pointers, include, exclude)
}

// Fetch recent objects based on config
func fetchRecent(alreadyFetchedRefs []*git.Ref, include, exclude []string) {
	fetchconf := lfs.Config.FetchPruneConfig()

	if fetchconf.FetchRecentRefsDays == 0 && fetchconf.FetchRecentCommitsDays == 0 {
		return
	}

	// Make a list of what unique commits we've already fetched for to avoid duplicating work
	uniqueRefShas := make(map[string]string, len(alreadyFetchedRefs))
	for _, ref := range alreadyFetchedRefs {
		uniqueRefShas[ref.Sha] = ref.Name
	}
	// First find any other recent refs
	if fetchconf.FetchRecentRefsDays > 0 {
		Print("Fetching recent branches within %v days", fetchconf.FetchRecentRefsDays)
		refsSince := time.Now().AddDate(0, 0, -fetchconf.FetchRecentRefsDays)
		refs, err := git.RecentBranches(refsSince, fetchconf.FetchRecentRefsIncludeRemotes, lfs.Config.CurrentRemote)
		if err != nil {
			Panic(err, "Could not scan for recent refs")
		}
		for _, ref := range refs {
			// Don't fetch for the same SHA twice
			if prevRefName, ok := uniqueRefShas[ref.Sha]; ok {
				if ref.Name != prevRefName {
					tracerx.Printf("Skipping fetch for %v, already fetched via %v", ref.Name, prevRefName)
				}
			} else {
				uniqueRefShas[ref.Sha] = ref.Name
				Print("Fetching %v", ref.Name)
				fetchRef(ref.Sha, include, exclude)
			}
		}
	}
	// For every unique commit we've fetched, check recent commits too
	if fetchconf.FetchRecentCommitsDays > 0 {
		for commit, refName := range uniqueRefShas {
			// We measure from the last commit at the ref
			summ, err := git.GetCommitSummary(commit)
			if err != nil {
				Error("Couldn't scan commits at %v: %v", refName, err)
				continue
			}
			Print("Fetching changes within %v days of %v", fetchconf.FetchRecentCommitsDays, refName)
			commitsSince := summ.CommitDate.AddDate(0, 0, -fetchconf.FetchRecentCommitsDays)
			fetchPreviousVersions(commit, commitsSince, include, exclude)
		}

	}
}

func fetchAll() {
	// converts to `git rev-list --all`
	// We only pick up objects in real commits and not the reflog
	opts := &lfs.ScanRefsOptions{ScanMode: lfs.ScanAllMode, SkipDeletedBlobs: false}
	// This could be a long process so use the chan version & report progress
	Print("Scanning for objects")
	spinner := lfs.NewSpinner()
	var numObjs int64
	pointerchan, err := lfs.ScanRefsToChan("", "", opts)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}
	pointers := make([]*lfs.WrappedPointer, 0)
	for p := range pointerchan {
		numObjs++
		spinner.Print(OutputWriter, fmt.Sprintf("%d objects found", numObjs))
		pointers = append(pointers, p)
	}
	spinner.Finish(OutputWriter, fmt.Sprintf("%d objects found", numObjs))
	Print("Downloading objects")
	fetchPointers(pointers, nil, nil)
}

func fetchPointers(pointers []*lfs.WrappedPointer, include, exclude []string) {
	fetchAndReportToChan(pointers, include, exclude, nil)
}

// Fetch and report completion of each OID to a channel (optional, pass nil to skip)
func fetchAndReportToChan(pointers []*lfs.WrappedPointer, include, exclude []string, out chan<- *lfs.WrappedPointer) {

	totalSize := int64(0)
	for _, p := range pointers {
		totalSize += p.Size
	}
	q := lfs.NewDownloadQueue(len(pointers), totalSize, false)

	for _, p := range pointers {
		// Only add to download queue if local file is not the right size already
		// This avoids previous case of over-reporting a requirement for files we already have
		// which would only be skipped by PointerSmudgeObject later
		passFilter := lfs.FilenamePassesIncludeExcludeFilter(p.Name, include, exclude)
		if !lfs.ObjectExistsOfSize(p.Oid, p.Size) && passFilter {
			tracerx.Printf("fetch %v [%v]", p.Name, p.Oid)
			q.Add(lfs.NewDownloadable(p))
		} else {
			if !passFilter {
				tracerx.Printf("Skipping %v [%v], include/exclude filters applied", p.Name, p.Oid)
			} else {
				tracerx.Printf("Skipping %v [%v], already exists", p.Name, p.Oid)
			}

			// If we already have it, or it won't be fetched
			// report it to chan immediately to support pull/checkout
			if out != nil {
				out <- p
			}

		}
	}

	if out != nil {
		dlwatch := q.Watch()

		go func() {
			// fetch only reports single OID, but OID *might* be referenced by multiple
			// WrappedPointers if same content is at multiple paths, so map oid->slice
			oidToPointers := make(map[string][]*lfs.WrappedPointer, len(pointers))
			for _, pointer := range pointers {
				plist := oidToPointers[pointer.Oid]
				oidToPointers[pointer.Oid] = append(plist, pointer)
			}

			for oid := range dlwatch {
				plist, ok := oidToPointers[oid]
				if !ok {
					continue
				}
				for _, p := range plist {
					out <- p
				}
			}
			close(out)
		}()

	}
	processQueue := time.Now()
	q.Wait()
	tracerx.PerformanceSince("process queue", processQueue)

	for _, err := range q.Errors() {
		if Debugging || lfs.IsFatalError(err) {
			LoggedError(err, err.Error())
		} else {
			Error(err.Error())
		}
	}
}
<commit_msg>アアー アアアア アー<commit_after>package commands

import (
	"fmt"
	"time"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)

var (
	fetchCmd = &cobra.Command{
		Use:   "fetch",
		Short: "Downloads LFS files",
		Run:   fetchCommand,
	}
	fetchIncludeArg string
	fetchExcludeArg string
	fetchRecentArg  bool
	fetchAllArg     bool
)

func fetchCommand(cmd *cobra.Command, args []string) {
	var refs []*git.Ref

	if len(args) > 0 {
		// Remote is first arg
		lfs.Config.CurrentRemote = args[0]
	} else {
		trackedRemote, err := git.CurrentRemote()
		if err == nil {
			lfs.Config.CurrentRemote = trackedRemote
		} // otherwise leave as default (origin)
	}
	if len(args) > 1 {
		for _, r := range args[1:] {
			ref, err := git.ResolveRef(r)
			if err != nil {
				Panic(err, "Invalid ref argument")
			}
			refs = append(refs, ref)
		}
	} else {
		ref, err := git.CurrentRef()
		if err != nil {
			Panic(err, "Could not fetch")
		}
		refs = []*git.Ref{ref}
	}

	if fetchAllArg {
		if fetchRecentArg || len(refs) > 0 {
			Exit("Cannot combine --all with ref arguments or --recent")
		}
		if fetchIncludeArg != "" || fetchExcludeArg != "" {
			Exit("Cannot combine --all with --include or --exclude")
		}
		if len(lfs.Config.FetchIncludePaths()) > 0 || len(lfs.Config.FetchExcludePaths()) > 0 {
			Print("Ignoring global include / exclude paths to fulfil --all")
		}
		fetchAll()

	} else { // !all

		includePaths, excludePaths := determineIncludeExcludePaths(fetchIncludeArg, fetchExcludeArg)

		// Fetch refs sequentially per arg order; duplicates in later refs will be ignored
		for _, ref := range refs {
			Print("Fetching %v", ref.Name)
			fetchRef(ref.Sha, includePaths, excludePaths)
		}

		if fetchRecentArg || lfs.Config.FetchPruneConfig().FetchRecentAlways {
			fetchRecent(refs, includePaths, excludePaths)
		}
	}
}

func init() {
	fetchCmd.Flags().StringVarP(&fetchIncludeArg, "include", "I", "", "Include a list of paths")
	fetchCmd.Flags().StringVarP(&fetchExcludeArg, "exclude", "X", "", "Exclude a list of paths")
	fetchCmd.Flags().BoolVarP(&fetchRecentArg, "recent", "r", false, "Fetch recent refs & commits")
	fetchCmd.Flags().BoolVarP(&fetchAllArg, "all", "a", false, "Fetch all LFS files ever referenced")
	RootCmd.AddCommand(fetchCmd)
}

func pointersToFetchForRef(ref string) ([]*lfs.WrappedPointer, error) {
	// Use SkipDeletedBlobs to avoid fetching ALL previous versions of modified files
	opts := &lfs.ScanRefsOptions{ScanMode: lfs.ScanRefsMode, SkipDeletedBlobs: true}
	return lfs.ScanRefs(ref, "", opts)
}

func fetchRefToChan(ref string, include, exclude []string) chan *lfs.WrappedPointer {
	c := make(chan *lfs.WrappedPointer)
	pointers, err := pointersToFetchForRef(ref)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}

	go fetchAndReportToChan(pointers, include, exclude, c)

	return c
}

// Fetch all binaries for a given ref (that we don't have already)
func fetchRef(ref string, include, exclude []string) {
	pointers, err := pointersToFetchForRef(ref)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}
	fetchPointers(pointers, include, exclude)
}

// Fetch all previous versions of objects from since to ref (not including final state at ref)
// So this will fetch all the '-' sides of the diff from since to ref
func fetchPreviousVersions(ref string, since time.Time, include, exclude []string) {
	pointers, err := lfs.ScanPreviousVersions(ref, since)
	if err != nil {
		Panic(err, "Could not scan for Git LFS previous versions")
	}
	fetchPointers(pointers, include, exclude)
}

// Fetch recent objects based on config
func fetchRecent(alreadyFetchedRefs []*git.Ref, include, exclude []string) {
	fetchconf := lfs.Config.FetchPruneConfig()

	if fetchconf.FetchRecentRefsDays == 0 && fetchconf.FetchRecentCommitsDays == 0 {
		return
	}

	// Make a list of what unique commits we've already fetched for to avoid duplicating work
	uniqueRefShas := make(map[string]string, len(alreadyFetchedRefs))
	for _, ref := range alreadyFetchedRefs {
		uniqueRefShas[ref.Sha] = ref.Name
	}
	// First find any other recent refs
	if fetchconf.FetchRecentRefsDays > 0 {
		Print("Fetching recent branches within %v days", fetchconf.FetchRecentRefsDays)
		refsSince := time.Now().AddDate(0, 0, -fetchconf.FetchRecentRefsDays)
		refs, err := git.RecentBranches(refsSince, fetchconf.FetchRecentRefsIncludeRemotes, lfs.Config.CurrentRemote)
		if err != nil {
			Panic(err, "Could not scan for recent refs")
		}
		for _, ref := range refs {
			// Don't fetch for the same SHA twice
			if prevRefName, ok := uniqueRefShas[ref.Sha]; ok {
				if ref.Name != prevRefName {
					tracerx.Printf("Skipping fetch for %v, already fetched via %v", ref.Name, prevRefName)
				}
			} else {
				uniqueRefShas[ref.Sha] = ref.Name
				Print("Fetching %v", ref.Name)
				fetchRef(ref.Sha, include, exclude)
			}
		}
	}
	// For every unique commit we've fetched, check recent commits too
	if fetchconf.FetchRecentCommitsDays > 0 {
		for commit, refName := range uniqueRefShas {
			// We measure from the last commit at the ref
			summ, err := git.GetCommitSummary(commit)
			if err != nil {
				Error("Couldn't scan commits at %v: %v", refName, err)
				continue
			}
			Print("Fetching changes within %v days of %v", fetchconf.FetchRecentCommitsDays, refName)
			commitsSince := summ.CommitDate.AddDate(0, 0, -fetchconf.FetchRecentCommitsDays)
			fetchPreviousVersions(commit, commitsSince, include, exclude)
		}

	}
}

func fetchAll() {
	// converts to `git rev-list --all`
	// We only pick up objects in real commits and not the reflog
	opts := &lfs.ScanRefsOptions{ScanMode: lfs.ScanAllMode, SkipDeletedBlobs: false}
	// This could be a long process so use the chan version & report progress
	Print("Scanning for objects")
	spinner := lfs.NewSpinner()
	var numObjs int64
	pointerchan, err := lfs.ScanRefsToChan("", "", opts)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}
	pointers := make([]*lfs.WrappedPointer, 0)
	for p := range pointerchan {
		numObjs++
		spinner.Print(OutputWriter, fmt.Sprintf("%d objects found", numObjs))
		pointers = append(pointers, p)
	}
	spinner.Finish(OutputWriter, fmt.Sprintf("%d objects found", numObjs))
	Print("Downloading objects")
	fetchPointers(pointers, nil, nil)
}

func fetchPointers(pointers []*lfs.WrappedPointer, include, exclude []string) {
	fetchAndReportToChan(pointers, include, exclude, nil)
}

// Fetch and report completion of each OID to a channel (optional, pass nil to skip)
func fetchAndReportToChan(pointers []*lfs.WrappedPointer, include, exclude []string, out chan<- *lfs.WrappedPointer) {

	totalSize := int64(0)
	for _, p := range pointers {
		totalSize += p.Size
	}
	q := lfs.NewDownloadQueue(len(pointers), totalSize, false)

	for _, p := range pointers {
		// Only add to download queue if local file is not the right size already
		// This avoids previous case of over-reporting a requirement for files we already have
		// which would only be skipped by PointerSmudgeObject later
		passFilter := lfs.FilenamePassesIncludeExcludeFilter(p.Name, include, exclude)
		if !lfs.ObjectExistsOfSize(p.Oid, p.Size) && passFilter {
			tracerx.Printf("fetch %v [%v]", p.Name, p.Oid)
			q.Add(lfs.NewDownloadable(p))
		} else {
			if !passFilter {
				tracerx.Printf("Skipping %v [%v], include/exclude filters applied", p.Name, p.Oid)
			} else {
				tracerx.Printf("Skipping %v [%v], already exists", p.Name, p.Oid)
			}

			// If we already have it, or it won't be fetched
			// report it to chan immediately to support pull/checkout
			if out != nil {
				out <- p
			}

		}
	}

	if out != nil {
		dlwatch := q.Watch()

		go func() {
			// fetch only reports single OID, but OID *might* be referenced by multiple
			// WrappedPointers if same content is at multiple paths, so map oid->slice
			oidToPointers := make(map[string][]*lfs.WrappedPointer, len(pointers))
			for _, pointer := range pointers {
				plist := oidToPointers[pointer.Oid]
				oidToPointers[pointer.Oid] = append(plist, pointer)
			}

			for oid := range dlwatch {
				plist, ok := oidToPointers[oid]
				if !ok {
					continue
				}
				for _, p := range plist {
					out <- p
				}
			}
			close(out)
		}()

	}
	processQueue := time.Now()
	q.Wait()
	tracerx.PerformanceSince("process queue", processQueue)

	for _, err := range q.Errors() {
		if Debugging || lfs.IsFatalError(err) {
			LoggedError(err, err.Error())
		} else {
			Error(err.Error())
		}
	}
}
<|endoftext|>