<commit_before>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package gin

import (
	"bufio"
	"errors"
	"log"
	"net"
	"net/http"
)

const (
	NoWritten = -1
)

type (
	ResponseWriter interface {
		http.ResponseWriter
		http.Hijacker
		http.Flusher
		http.CloseNotifier

		Status() int
		Size() int
		Written() bool
		WriteHeaderNow()
	}

	responseWriter struct {
		http.ResponseWriter
		status int
		size   int
	}
)

func (w *responseWriter) reset(writer http.ResponseWriter) {
	w.ResponseWriter = writer
	w.status = 200
	w.size = NoWritten
}

func (w *responseWriter) WriteHeader(code int) {
	if code > 0 {
		w.status = code
		if w.Written() {
			log.Println("[GIN] WARNING. Headers were already written!")
		}
	}
}

func (w *responseWriter) WriteHeaderNow() {
	if !w.Written() {
		w.size = 0
		w.ResponseWriter.WriteHeader(w.status)
	}
}

func (w *responseWriter) Write(data []byte) (n int, err error) {
	w.WriteHeaderNow()
	n, err = w.ResponseWriter.Write(data)
	w.size += n
	return
}

func (w *responseWriter) Status() int {
	return w.status
}

func (w *responseWriter) Size() int {
	return w.size
}

func (w *responseWriter) Written() bool {
	return w.size != NoWritten
}

// Implements the http.Hijacker interface
func (w *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	hijacker, ok := w.ResponseWriter.(http.Hijacker)
	if !ok {
		return nil, nil, errors.New("the ResponseWriter doesn't support the Hijacker interface")
	}
	return hijacker.Hijack()
}

// Implements the http.CloseNotify interface
func (w *responseWriter) CloseNotify() <-chan bool {
	return w.ResponseWriter.(http.CloseNotifier).CloseNotify()
}

// Implements the http.Flush interface
func (w *responseWriter) Flush() {
	flusher, ok := w.ResponseWriter.(http.Flusher)
	if ok {
		flusher.Flush()
	}
}
<commit_msg>Fixes #239 bug<commit_after>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package gin

import (
	"bufio"
	"errors"
	"log"
	"net"
	"net/http"
)

const (
	NoWritten = -1
)

type (
	ResponseWriter interface {
		http.ResponseWriter
		http.Hijacker
		http.Flusher
		http.CloseNotifier

		Status() int
		Size() int
		Written() bool
		WriteHeaderNow()
	}

	responseWriter struct {
		http.ResponseWriter
		status int
		size   int
	}
)

func (w *responseWriter) reset(writer http.ResponseWriter) {
	w.ResponseWriter = writer
	w.status = 200
	w.size = NoWritten
}

func (w *responseWriter) WriteHeader(code int) {
	if code > 0 {
		w.status = code
		if w.Written() {
			log.Println("[GIN] WARNING. Headers were already written!")
		}
	}
}

func (w *responseWriter) WriteHeaderNow() {
	if !w.Written() {
		w.size = 0
		w.ResponseWriter.WriteHeader(w.status)
	}
}

func (w *responseWriter) Write(data []byte) (n int, err error) {
	w.WriteHeaderNow()
	n, err = w.ResponseWriter.Write(data)
	w.size += n
	return
}

func (w *responseWriter) Status() int {
	return w.status
}

func (w *responseWriter) Size() int {
	return w.size
}

func (w *responseWriter) Written() bool {
	return w.size != NoWritten
}

// Implements the http.Hijacker interface
func (w *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	w.size = 0 // this prevents Gin to write the HTTP headers
	return w.ResponseWriter.(http.Hijacker).Hijack()
}

// Implements the http.CloseNotify interface
func (w *responseWriter) CloseNotify() <-chan bool {
	return w.ResponseWriter.(http.CloseNotifier).CloseNotify()
}

// Implements the http.Flush interface
func (w *responseWriter) Flush() {
	flusher, ok := w.ResponseWriter.(http.Flusher)
	if ok {
		flusher.Flush()
	}
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"github.com/dotcloud/docker/api"
	"github.com/dotcloud/docker/engine"
	"github.com/dotcloud/docker/utils"
	"io"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"
)

func TestGetBoolParam(t *testing.T) {
	if ret, err := getBoolParam("true"); err != nil || !ret {
		t.Fatalf("true -> true, nil | got %t %s", ret, err)
	}
	if ret, err := getBoolParam("True"); err != nil || !ret {
		t.Fatalf("True -> true, nil | got %t %s", ret, err)
	}
	if ret, err := getBoolParam("1"); err != nil || !ret {
		t.Fatalf("1 -> true, nil | got %t %s", ret, err)
	}
	if ret, err := getBoolParam(""); err != nil || ret {
		t.Fatalf("\"\" -> false, nil | got %t %s", ret, err)
	}
	if ret, err := getBoolParam("false"); err != nil || ret {
		t.Fatalf("false -> false, nil | got %t %s", ret, err)
	}
	if ret, err := getBoolParam("0"); err != nil || ret {
		t.Fatalf("0 -> false, nil | got %t %s", ret, err)
	}
	if ret, err := getBoolParam("faux"); err == nil || ret {
		t.Fatalf("faux -> false, err | got %t %s", ret, err)

	}
}

func TesthttpError(t *testing.T) {
	r := httptest.NewRecorder()

	httpError(r, fmt.Errorf("No such method"))
	if r.Code != http.StatusNotFound {
		t.Fatalf("Expected %d, got %d", http.StatusNotFound, r.Code)
	}

	httpError(r, fmt.Errorf("This accound hasn't been activated"))
	if r.Code != http.StatusForbidden {
		t.Fatalf("Expected %d, got %d", http.StatusForbidden, r.Code)
	}

	httpError(r, fmt.Errorf("Some error"))
	if r.Code != http.StatusInternalServerError {
		t.Fatalf("Expected %d, got %d", http.StatusInternalServerError, r.Code)
	}
}

func TestGetVersion(t *testing.T) {
	tmp, err := utils.TestDirectory("")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmp)
	eng, err := engine.New(tmp)
	if err != nil {
		t.Fatal(err)
	}
	var called bool
	eng.Register("version", func(job *engine.Job) engine.Status {
		called = true
		v := &engine.Env{}
		v.SetJson("Version", "42.1")
		v.Set("ApiVersion", "1.1.1.1.1")
		v.Set("GoVersion", "2.42")
		v.Set("Os", "Linux")
		v.Set("Arch", "x86_64")
		if _, err := v.WriteTo(job.Stdout); err != nil {
			return job.Error(err)
		}
		return engine.StatusOK
	})

	r := httptest.NewRecorder()
	req, err := http.NewRequest("GET", "/version", nil)
	if err != nil {
		t.Fatal(err)
	}
	// FIXME getting the version should require an actual running Server
	if err := ServeRequest(eng, api.APIVERSION, r, req); err != nil {
		t.Fatal(err)
	}
	if !called {
		t.Fatalf("handler was not called")
	}
	out := engine.NewOutput()
	v, err := out.AddEnv()
	if err != nil {
		t.Fatal(err)
	}
	if _, err := io.Copy(out, r.Body); err != nil {
		t.Fatal(err)
	}
	out.Close()
	expected := "42.1"
	if result := v.Get("Version"); result != expected {
		t.Errorf("Expected version %s, %s found", expected, result)
	}
	expected = "application/json"
	if result := r.HeaderMap.Get("Content-Type"); result != expected {
		t.Errorf("Expected Content-Type %s, %s found", expected, result)
	}
}

func TestGetInfo(t *testing.T) {
	tmp, err := utils.TestDirectory("")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmp)
	eng, err := engine.New(tmp)
	if err != nil {
		t.Fatal(err)
	}

	var called bool
	eng.Register("info", func(job *engine.Job) engine.Status {
		called = true
		v := &engine.Env{}
		v.SetInt("Containers", 1)
		v.SetInt("Images", 42000)
		if _, err := v.WriteTo(job.Stdout); err != nil {
			return job.Error(err)
		}
		return engine.StatusOK
	})

	r := httptest.NewRecorder()
	req, err := http.NewRequest("GET", "/info", nil)
	if err != nil {
		t.Fatal(err)
	}
	// FIXME getting the version should require an actual running Server
	if err := ServeRequest(eng, api.APIVERSION, r, req); err != nil {
		t.Fatal(err)
	}
	if !called {
		t.Fatalf("handler was not called")
	}

	out := engine.NewOutput()
	i, err := out.AddEnv()
	if err != nil {
		t.Fatal(err)
	}
	if _, err := io.Copy(out, r.Body); err != nil {
		t.Fatal(err)
	}
	out.Close()
	{
		expected := 42000
		result := i.GetInt("Images")
		if expected != result {
			t.Fatalf("%#v\n", result)
		}
	}
	{
		expected := 1
		result := i.GetInt("Containers")
		if expected != result {
			t.Fatalf("%#v\n", result)
		}
	}
	{
		expected := "application/json"
		if result := r.HeaderMap.Get("Content-Type"); result != expected {
			t.Fatalf("%#v\n", result)
		}
	}
}
<commit_msg>Make remote API unit tests easier to read and write<commit_after>package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/dotcloud/docker/api"
	"github.com/dotcloud/docker/engine"
	"github.com/dotcloud/docker/utils"
	"io"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"
)

func TestGetBoolParam(t *testing.T) {
	if ret, err := getBoolParam("true"); err != nil || !ret {
		t.Fatalf("true -> true, nil | got %t %s", ret, err)
	}
	if ret, err := getBoolParam("True"); err != nil || !ret {
		t.Fatalf("True -> true, nil | got %t %s", ret, err)
	}
	if ret, err := getBoolParam("1"); err != nil || !ret {
		t.Fatalf("1 -> true, nil | got %t %s", ret, err)
	}
	if ret, err := getBoolParam(""); err != nil || ret {
		t.Fatalf("\"\" -> false, nil | got %t %s", ret, err)
	}
	if ret, err := getBoolParam("false"); err != nil || ret {
		t.Fatalf("false -> false, nil | got %t %s", ret, err)
	}
	if ret, err := getBoolParam("0"); err != nil || ret {
		t.Fatalf("0 -> false, nil | got %t %s", ret, err)
	}
	if ret, err := getBoolParam("faux"); err == nil || ret {
		t.Fatalf("faux -> false, err | got %t %s", ret, err)

	}
}

func TesthttpError(t *testing.T) {
	r := httptest.NewRecorder()

	httpError(r, fmt.Errorf("No such method"))
	if r.Code != http.StatusNotFound {
		t.Fatalf("Expected %d, got %d", http.StatusNotFound, r.Code)
	}

	httpError(r, fmt.Errorf("This accound hasn't been activated"))
	if r.Code != http.StatusForbidden {
		t.Fatalf("Expected %d, got %d", http.StatusForbidden, r.Code)
	}

	httpError(r, fmt.Errorf("Some error"))
	if r.Code != http.StatusInternalServerError {
		t.Fatalf("Expected %d, got %d", http.StatusInternalServerError, r.Code)
	}
}

func TestGetVersion(t *testing.T) {
	eng := tmpEngine(t)
	defer rmEngine(eng)
	var called bool
	eng.Register("version", func(job *engine.Job) engine.Status {
		called = true
		v := &engine.Env{}
		v.SetJson("Version", "42.1")
		v.Set("ApiVersion", "1.1.1.1.1")
		v.Set("GoVersion", "2.42")
		v.Set("Os", "Linux")
		v.Set("Arch", "x86_64")
		if _, err := v.WriteTo(job.Stdout); err != nil {
			return job.Error(err)
		}
		return engine.StatusOK
	})
	r := serveRequest("GET", "/version", nil, eng, t)
	if !called {
		t.Fatalf("handler was not called")
	}
	v := readEnv(r.Body, t)
	if v.Get("Version") != "42.1" {
		t.Fatalf("%#v\n", v)
	}
	if r.HeaderMap.Get("Content-Type") != "application/json" {
		t.Fatalf("%#v\n", r)
	}
}

func TestGetInfo(t *testing.T) {
	eng := tmpEngine(t)
	defer rmEngine(eng)
	var called bool
	eng.Register("info", func(job *engine.Job) engine.Status {
		called = true
		v := &engine.Env{}
		v.SetInt("Containers", 1)
		v.SetInt("Images", 42000)
		if _, err := v.WriteTo(job.Stdout); err != nil {
			return job.Error(err)
		}
		return engine.StatusOK
	})
	r := serveRequest("GET", "/info", nil, eng, t)
	if !called {
		t.Fatalf("handler was not called")
	}
	v := readEnv(r.Body, t)
	if v.GetInt("Images") != 42000 {
		t.Fatalf("%#v\n", v)
	}
	if v.GetInt("Containers") != 1 {
		t.Fatalf("%#v\n", v)
	}
	if r.HeaderMap.Get("Content-Type") != "application/json" {
		t.Fatalf("%#v\n", r)
	}
}

func serveRequest(method, target string, body io.Reader, eng *engine.Engine, t *testing.T) *httptest.ResponseRecorder {
	r := httptest.NewRecorder()
	req, err := http.NewRequest(method, target, body)
	if err != nil {
		t.Fatal(err)
	}
	if err := ServeRequest(eng, api.APIVERSION, r, req); err != nil {
		t.Fatal(err)
	}
	return r
}

func tmpEngine(t *testing.T) *engine.Engine {
	tmp, err := utils.TestDirectory("")
	if err != nil {
		t.Fatal(err)
	}
	eng, err := engine.New(tmp)
	if err != nil {
		t.Fatal(err)
	}
	return eng
}

func rmEngine(eng *engine.Engine) {
	os.RemoveAll(eng.Root())
}

func readEnv(src io.Reader, t *testing.T) *engine.Env {
	out := engine.NewOutput()
	v, err := out.AddEnv()
	if err != nil {
		t.Fatal(err)
	}
	if _, err := io.Copy(out, src); err != nil {
		t.Fatal(err)
	}
	out.Close()
	return v
}

func toJson(data interface{}, t *testing.T) io.Reader {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(data); err != nil {
		t.Fatal(err)
	}
	return &buf
}
<|endoftext|>


<commit_before>package cfsigner

import (
	"crypto"
	"crypto/rsa"
	"crypto/sha1"
	"encoding/base64"
	"encoding/json"
	"errors"
	"strings"
	"time"
)

type CloudFront struct {
	domain    string
	KeyPairId string
	key       *rsa.PrivateKey
}

type Policy struct {
	policy []byte
	cf     *CloudFront
}

type policy struct {
	Statement []statement
}

type statement struct {
	Resource  string
	Condition conditions
}

type conditions struct {
	DateLessThan    epochTime
	DateGreaterThan *epochTime `json:",omitempty"`
	IpAddress       *ipAddress `json:",omitempty"`
}

type epochTime struct {
	Timestamp int64 `json:"AWS:EpochTime"`
}

type ipAddress struct {
	Addr string `json:"AWS:SourceIp"`
}

// Will convert values from base64 encoded string to be URL safe
var invalidReplacer = strings.NewReplacer("+", "-", "=", "_", "/", "~")

var ErrMissingRequiredParam = errors.New("Missing required parameter")

func New(key *rsa.PrivateKey, keyPairId string) *CloudFront {
	return &CloudFront{
		KeyPairId: keyPairId,
		key:       key,
	}
}

func (p Policy) Encode() string {
	return invalidReplacer.Replace(base64.StdEncoding.EncodeToString(p.policy))
}

func (p Policy) Sign() (string, error) {
	hash := hashSha(p.policy)
	signed, err := rsa.SignPKCS1v15(nil, p.cf.key, crypto.SHA1, hash)
	if err != nil {
		return "", err
	}

	return invalidReplacer.Replace(base64.StdEncoding.EncodeToString(signed)), nil
}

func (cf *CloudFront) CreatePolicy(resource string, expiry time.Time, validAt *time.Time, ip *string) (Policy, error) {
	if expiry.IsZero() {
		return nil, ErrMissingRequiredParam
	}

	conds := conditions{
		DateLessThan: epochTime{
			Timestamp: expiry.Truncate(time.Millisecond).Unix(),
		},
	}

	if validAt != nil {
		conds.DateGreaterThan = &epochTime{
			Timestamp: *validAt.Truncate(time.Millisecond).Unix(),
		}
	}

	if ip != nil {
		conds.IpAddress = &ipAddress{
			Addr: *ip,
		}
	}

	policy, err := buildPolicy(resource, conds)
	if err != nil {
		return nil, err
	}

	return Policy{cf: cf, policy: policy}, nil
}

func hashSha(policy []byte) []byte {
	hash := sha1.New()
	hash.Write(p.policy)
	hash.Sum(nil)
}

func buildPolicy(resource string, conditions conditions) ([]byte, error) {
	p := &policy{
		Statement: []statement{
			statement{
				Resource:  resource,
				Condition: conditions,
			},
		},
	}

	return json.Marshal(p)
}
<commit_msg>Couple of fixes.<commit_after>package cfsigner

import (
	"crypto"
	"crypto/rsa"
	"crypto/sha1"
	"encoding/base64"
	"encoding/json"
	"errors"
	"strings"
	"time"
)

type CloudFront struct {
	domain    string
	KeyPairId string
	key       *rsa.PrivateKey
}

type Policy struct {
	policy []byte
	cf     *CloudFront
}

type policy struct {
	Statement []statement
}

type statement struct {
	Resource  string
	Condition conditions
}

type conditions struct {
	DateLessThan    epochTime
	DateGreaterThan *epochTime `json:",omitempty"`
	IpAddress       *ipAddress `json:",omitempty"`
}

type epochTime struct {
	Timestamp int64 `json:"AWS:EpochTime"`
}

type ipAddress struct {
	Addr string `json:"AWS:SourceIp"`
}

// Will convert values from base64 encoded string to be URL safe
var invalidReplacer = strings.NewReplacer("+", "-", "=", "_", "/", "~")

var ErrMissingRequiredParam = errors.New("Missing required parameter")

func New(key *rsa.PrivateKey, keyPairId string) *CloudFront {
	return &CloudFront{
		KeyPairId: keyPairId,
		key:       key,
	}
}

func (p *Policy) Encode() string {
	return invalidReplacer.Replace(base64.StdEncoding.EncodeToString(p.policy))
}

func (p *Policy) Sign() (string, error) {
	hash := hashSha(p.policy)
	signed, err := rsa.SignPKCS1v15(nil, p.cf.key, crypto.SHA1, hash)
	if err != nil {
		return "", err
	}

	return invalidReplacer.Replace(base64.StdEncoding.EncodeToString(signed)), nil
}

func (cf *CloudFront) CreatePolicy(resource string, expiry time.Time, validAt *time.Time, ip *string) (*Policy, error) {
	if expiry.IsZero() {
		return nil, ErrMissingRequiredParam
	}

	conds := conditions{
		DateLessThan: epochTime{
			Timestamp: expiry.Truncate(time.Millisecond).Unix(),
		},
	}

	if validAt != nil {
		conds.DateGreaterThan = &epochTime{
			Timestamp: validAt.Truncate(time.Millisecond).Unix(),
		}
	}

	if ip != nil {
		conds.IpAddress = &ipAddress{
			Addr: *ip,
		}
	}

	policy, err := buildPolicy(resource, conds)
	if err != nil {
		return nil, err
	}

	return &Policy{cf: cf, policy: policy}, nil
}

func hashSha(policy []byte) []byte {
	hash := sha1.New()
	hash.Write(policy)
	return hash.Sum(nil)
}

func buildPolicy(resource string, conditions conditions) ([]byte, error) {
	p := &policy{
		Statement: []statement{
			statement{
				Resource:  resource,
				Condition: conditions,
			},
		},
	}

	return json.Marshal(p)
}
<|endoftext|>


<commit_before>package process

import (
	"bufio"
	"encoding/csv"
	"fmt"
	"io"
	"log"
	"os"
	"strconv"
	"strings"
)

const filePrefix = "./resources/"
const fileSufix = "_vectorized.png"

// LoadGameData load data from pokedex file
func LoadGameData(nameFile string) (gameDataArray []GameData, ok bool) {
	gameDataArray = make([]GameData, 0)

	f, err := os.Open(nameFile)
	if err != nil {
		log.Print(err)
		return nil, false
	}

	r := csv.NewReader(bufio.NewReader(f))
	for {
		record, err := r.Read()
		// Stop at EOF.
		if err == io.EOF {
			break
		}
		if len(record) < 6 {
			continue
		}

		number, _ := strconv.Atoi(record[0])
		name := record[1]
		type1 := record[2]
		type2 := record[3]
		nickname := strings.Split(record[4], "_")
		evolve := strings.Split(record[5], "_")
		fmt.Printf("evolve %v", evolve)
		if evolve[0] == "-" {
			evolve = evolve[:0]
		}
		avatarFile := fmt.Sprintf("%s%03d%s", filePrefix, number, fileSufix)

		data := GameData{number, name, type1, type2, nickname, evolve, avatarFile}
		gameDataArray = append(gameDataArray, data)

		ok = true
	}

	return
}

// FindPokemon compare data in array
func FindPokemon(gameData []GameData, msg string) (pokemon GameData, ok bool) {
	ok = false
	if len(msg) == 0 {
		return
	}
	num, _ := strconv.Atoi(msg)
	if num > 0 {
		fmt.Printf("finding for pokemon #%d\n", num)
		for _, data := range gameData {
			if data.Number == num {
				pokemon = data
				ok = true
				return
			}
		}
		return
	}
	fmt.Printf("finding for pokemon name `%s`\n", msg)
	msg = strings.ToLower(msg)
	for _, data := range gameData {
		if strings.Contains(strings.ToLower(data.Name), msg) {
			pokemon = data
			ok = true
			return
		}
	}
	fmt.Printf("finding for pokemon nickname `%s`\n", msg)
	for _, data := range gameData {
		for _, nickname := range data.Nickname {
			if strings.Contains(nickname, msg) {
				pokemon = data
				ok = true
				return
			}
		}
	}

	return
}
<commit_msg>use whitespace and underline as seperator<commit_after>package process

import (
	"bufio"
	"encoding/csv"
	"fmt"
	"io"
	"log"
	"os"
	"strconv"
	"strings"
	"unicode"
)

const filePrefix = "./resources/"
const fileSufix = "_vectorized.png"

// LoadGameData load data from pokedex file
func LoadGameData(nameFile string) (gameDataArray []GameData, ok bool) {
	gameDataArray = make([]GameData, 0)

	f, err := os.Open(nameFile)
	if err != nil {
		log.Print(err)
		return nil, false
	}

	r := csv.NewReader(bufio.NewReader(f))
	for {
		record, err := r.Read()
		// Stop at EOF.
		if err == io.EOF {
			break
		}
		if len(record) < 6 {
			continue
		}

		number, _ := strconv.Atoi(record[0])
		name := record[1]
		type1 := record[2]
		type2 := record[3]
		nickname := strings.Split(record[4], "_")
		// support whitespace and underline as field seperator
		f := func(c rune) bool {
			return unicode.IsSpace(c) || unicode.IsControl(c) || c == '_'
		}
		evolve := strings.FieldsFunc(record[5], f)
		if evolve[0] == "-" {
			evolve = evolve[:0]
		}
		avatarFile := fmt.Sprintf("%s%03d%s", filePrefix, number, fileSufix)

		data := GameData{number, name, type1, type2, nickname, evolve, avatarFile}
		gameDataArray = append(gameDataArray, data)

		ok = true
	}

	return
}

// FindPokemon compare data in array
func FindPokemon(gameData []GameData, msg string) (pokemon GameData, ok bool) {
	ok = false
	if len(msg) == 0 {
		return
	}
	num, _ := strconv.Atoi(msg)
	if num > 0 {
		fmt.Printf("finding for pokemon #%d\n", num)
		for _, data := range gameData {
			if data.Number == num {
				pokemon = data
				ok = true
				return
			}
		}
		return
	}
	fmt.Printf("finding for pokemon name `%s`\n", msg)
	msg = strings.ToLower(msg)
	for _, data := range gameData {
		if strings.Contains(strings.ToLower(data.Name), msg) {
			pokemon = data
			ok = true
			return
		}
	}
	fmt.Printf("finding for pokemon nickname `%s`\n", msg)
	for _, data := range gameData {
		for _, nickname := range data.Nickname {
			if strings.Contains(nickname, msg) {
				pokemon = data
				ok = true
				return
			}
		}
	}

	return
}
<|endoftext|>


<commit_before>package process

// Logic for this file is largely based on:
// https://github.com/jarib/childprocess/blob/783f7a00a1678b5d929062564ef5ae76822dfd62/lib/childprocess/unix/process.rb

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strconv"
	"sync"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/buildkite/agent/logger"
)

type Process struct {
	Pid        int
	PTY        bool
	Timestamp  bool
	Script     []string
	Env        []string
	ExitStatus string

	buffer  outputBuffer
	command *exec.Cmd

	// This callback is called when the process offically starts
	StartCallback func()

	// For every line in the process output, this callback will be called
	// with the contents of the line if its filter returns true.
	LineCallback       func(string)
	LinePreProcessor   func(string) string
	LineCallbackFilter func(string) bool

	// Running is stored as an int32 so we can use atomic operations to
	// set/get it (it's accessed by multiple goroutines)
	running int32
}

// If you change header parsing here make sure to change it in the
// buildkite.com frontend logic, too

var headerExpansionRegex = regexp.MustCompile("^(?:\\^\\^\\^\\s+\\+\\+\\+)\\s*$")

func (p *Process) Start() error {
	p.command = exec.Command(p.Script[0], p.Script[1:]...)

	// Copy the current processes ENV and merge in the new ones. We do this
	// so the sub process gets PATH and stuff. We merge our path in over
	// the top of the current one so the ENV from Buildkite and the agent
	// take precedence over the agent
	currentEnv := os.Environ()
	p.command.Env = append(currentEnv, p.Env...)

	var waitGroup sync.WaitGroup

	lineReaderPipe, lineWriterPipe := io.Pipe()

	var multiWriter io.Writer
	if p.Timestamp {
		multiWriter = io.MultiWriter(lineWriterPipe)
	} else {
		multiWriter = io.MultiWriter(&p.buffer, lineWriterPipe)
	}

	// Toggle between running in a pty
	if p.PTY {
		pty, err := StartPTY(p.command)
		if err != nil {
			p.ExitStatus = "1"
			return err
		}

		p.Pid = p.command.Process.Pid
		p.setRunning(true)

		waitGroup.Add(1)

		go func() {
			logger.Debug("[Process] Starting to copy PTY to the buffer")

			// Copy the pty to our buffer. This will block until it
			// EOF's or something breaks.
			_, err = io.Copy(multiWriter, pty)
			if e, ok := err.(*os.PathError); ok && e.Err == syscall.EIO {
				// We can safely ignore this error, because
				// it's just the PTY telling us that it closed
				// successfully.  See:
				// https://github.com/buildkite/agent/pull/34#issuecomment-46080419
				err = nil
			}

			if err != nil {
				logger.Error("[Process] PTY output copy failed with error: %T: %v", err, err)
			} else {
				logger.Debug("[Process] PTY has finished being copied to the buffer")
			}

			waitGroup.Done()
		}()
	} else {
		p.command.Stdout = multiWriter
		p.command.Stderr = multiWriter
		p.command.Stdin = nil

		err := p.command.Start()
		if err != nil {
			p.ExitStatus = "1"
			return err
		}

		p.Pid = p.command.Process.Pid
		p.setRunning(true)
	}

	logger.Info("[Process] Process is running with PID: %d", p.Pid)

	// Add the line callback routine to the waitGroup
	waitGroup.Add(1)

	go func() {
		logger.Debug("[LineScanner] Starting to read lines")

		reader := bufio.NewReader(lineReaderPipe)

		var appending []byte
		var lineCallbackWaitGroup sync.WaitGroup

		for {
			line, isPrefix, err := reader.ReadLine()
			if err != nil {
				if err == io.EOF {
					logger.Debug("[LineScanner] Encountered EOF")
					break
				}

				logger.Error("[LineScanner] Failed to read: (%T: %v)", err, err)
			}

			// If isPrefix is true, that means we've got a really
			// long line incoming, and we'll keep appending to it
			// until isPrefix is false (which means the long line
			// has ended.
			if isPrefix && appending == nil {
				logger.Debug("[LineScanner] Line is too long to read, going to buffer it until it finishes")
				// bufio.ReadLine returns a slice which is only valid until the next invocation
				// since it points to its own internal buffer array. To accumulate the entire
				// result we make a copy of the first prefix, and insure there is spare capacity
				// for future appends to minimize the need for resizing on append.
				appending = make([]byte, len(line), (cap(line))*2)
				copy(appending, line)

				continue
			}

			// Should we be appending?
			if appending != nil {
				appending = append(appending, line...)

				// No more isPrefix! Line is finished!
				if !isPrefix {
					logger.Debug("[LineScanner] Finished buffering long line")
					line = appending

					// Reset appending back to nil
					appending = nil
				} else {
					continue
				}
			}

			// If we're timestamping this main thread will take
			// the hit of running the regex so we can build up
			// the timestamped buffer without breaking headers,
			// otherwise we let the goroutines take the perf hit.

			checkedForCallback := false
			lineHasCallback := false
			lineString := p.LinePreProcessor(string(line))

			// Create the prefixed buffer
			if p.Timestamp {
				lineHasCallback = p.LineCallbackFilter(lineString)
				checkedForCallback = true
				if lineHasCallback || headerExpansionRegex.MatchString(lineString) {
					// Don't timestamp special lines (e.g. header)
					p.buffer.WriteString(fmt.Sprintf("%s\n", line))
				} else {
					currentTime := time.Now().UTC().Format(time.RFC3339)
					p.buffer.WriteString(fmt.Sprintf("[%s] %s\n", currentTime, line))
				}
			}

			if lineHasCallback || !checkedForCallback {
				lineCallbackWaitGroup.Add(1)
				go func(line string) {
					defer lineCallbackWaitGroup.Done()
					if (checkedForCallback && lineHasCallback) || p.LineCallbackFilter(lineString) {
						p.LineCallback(line)
					}
				}(lineString)
			}
		}

		// We need to make sure all the line callbacks have finish before
		// finish up the process
		logger.Debug("[LineScanner] Waiting for callbacks to finish")
		lineCallbackWaitGroup.Wait()

		logger.Debug("[LineScanner] Finished")
		waitGroup.Done()
	}()

	// Call the StartCallback
	go p.StartCallback()

	// Wait until the process has finished. The returned error is nil if the command runs,
	// has no problems copying stdin, stdout, and stderr, and exits with a zero exit status.
	waitResult := p.command.Wait()

	// Close the line writer pipe
	lineWriterPipe.Close()

	// The process is no longer running at this point
	p.setRunning(false)

	// Find the exit status of the script
	p.ExitStatus = getExitStatus(waitResult)

	logger.Info("Process with PID: %d finished with Exit Status: %s", p.Pid, p.ExitStatus)

	// Sometimes (in docker containers) io.Copy never seems to finish. This is a mega
	// hack around it. If it doesn't finish after 1 second, just continue.
	logger.Debug("[Process] Waiting for routines to finish")
	err := timeoutWait(&waitGroup)
	if err != nil {
		logger.Debug("[Process] Timed out waiting for wait group: (%T: %v)", err, err)
	}

	// No error occurred so we can return nil
	return nil
}

func (p *Process) Output() string {
	return p.buffer.String()
}

func (p *Process) Kill() error {
	var err error
	if runtime.GOOS == "windows" {
		// Sending Interrupt on Windows is not implemented.
		// https://golang.org/src/os/exec.go?s=3842:3884#L110
		err = exec.Command("CMD", "/C", "TASKKILL", "/F", "/PID", strconv.Itoa(p.Pid)).Run()
	} else {
		// Send a sigterm
		err = p.signal(syscall.SIGTERM)
	}
	if err != nil {
		return err
	}

	// Make a channel that we'll use as a timeout
	c := make(chan int, 1)
	checking := true

	// Start a routine that checks to see if the process
	// is still alive.
	go func() {
		for checking {
			logger.Debug("[Process] Checking to see if PID: %d is still alive", p.Pid)

			foundProcess, err := os.FindProcess(p.Pid)

			// Can't find the process at all
			if err != nil {
				logger.Debug("[Process] Could not find process with PID: %d", p.Pid)

				break
			}

			// We have some information about the process
			if foundProcess != nil {
				processState, err := foundProcess.Wait()

				if err != nil || processState.Exited() {
					logger.Debug("[Process] Process with PID: %d has exited.", p.Pid)

					break
				}
			}

			// Retry in a moment
			sleepTime := time.Duration(1 * time.Second)
			time.Sleep(sleepTime)
		}

		c <- 1
	}()

	// Timeout this process after 3 seconds
	select {
	case _ = <-c:
		// Was successfully terminated
	case <-time.After(10 * time.Second):
		// Stop checking in the routine above
		checking = false

		// Forcefully kill the thing
		err = p.signal(syscall.SIGKILL)

		if err != nil {
			return err
		}
	}

	return nil
}

func (p *Process) signal(sig os.Signal) error {
	if p.command != nil && p.command.Process != nil {
		logger.Debug("[Process] Sending signal: %s to PID: %d", sig.String(), p.Pid)

		err := p.command.Process.Signal(sig)
		if err != nil {
			logger.Error("[Process] Failed to send signal: %s to PID: %d (%T: %v)", sig.String(), p.Pid, err, err)
			return err
		}
	} else {
		logger.Debug("[Process] No process to signal yet")
	}

	return nil
}

// Returns whether or not the process is running
func (p *Process) IsRunning() bool {
	return atomic.LoadInt32(&p.running) != 0
}

// Sets the running flag of the process
func (p *Process) setRunning(r bool) {
	// Use the atomic package to avoid race conditions when setting the
	// `running` value from multiple routines
	if r {
		atomic.StoreInt32(&p.running, 1)
	} else {
		atomic.StoreInt32(&p.running, 0)
	}
}

// https://github.com/hnakamur/commango/blob/fe42b1cf82bf536ce7e24dceaef6656002e03743/os/executil/executil.go#L29
// TODO: Can this be better?
func getExitStatus(waitResult error) string {
	exitStatus := -1

	if waitResult != nil {
		if err, ok := waitResult.(*exec.ExitError); ok {
			if s, ok := err.Sys().(syscall.WaitStatus); ok {
				exitStatus = s.ExitStatus()
			} else {
				logger.Error("[Process] Unimplemented for system where exec.ExitError.Sys() is not syscall.WaitStatus.")
			}
		} else {
			logger.Error("[Process] Unexpected error type in getExitStatus: %#v", waitResult)
		}
	} else {
		exitStatus = 0
	}

	return fmt.Sprintf("%d", exitStatus)
}

func timeoutWait(waitGroup *sync.WaitGroup) error {
	// Make a chanel that we'll use as a timeout
	c := make(chan int, 1)

	// Start waiting for the routines to finish
	go func() {
		waitGroup.Wait()
		c <- 1
	}()

	select {
	case _ = <-c:
		return nil
	case <-time.After(10 * time.Second):
		return errors.New("Timeout")
	}

	return nil
}

// outputBuffer is a goroutine safe bytes.Buffer
type outputBuffer struct {
	sync.RWMutex
	buf bytes.Buffer
}

// Write appends the contents of p to the buffer, growing the buffer as needed. It returns
// the number of bytes written.
func (ob *outputBuffer) Write(p []byte) (n int, err error) {
	ob.Lock()
	defer ob.Unlock()
	return ob.buf.Write(p)
}

// WriteString appends the contents of s to the buffer, growing the buffer as needed. It returns
// the number of bytes written.
func (ob *outputBuffer) WriteString(s string) (n int, err error) {
	return ob.Write([]byte(s))
}

// String returns the contents of the unread portion of the buffer
// as a string.  If the Buffer is a nil pointer, it returns "<nil>".
func (ob *outputBuffer) String() string {
	ob.RLock()
	defer ob.RUnlock()
	return ob.buf.String()
}
<commit_msg>Cancellation on Windows: Pass `/T` to `taskkill`<commit_after>package process

// Logic for this file is largely based on:
// https://github.com/jarib/childprocess/blob/783f7a00a1678b5d929062564ef5ae76822dfd62/lib/childprocess/unix/process.rb

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strconv"
	"sync"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/buildkite/agent/logger"
)

type Process struct {
	Pid        int
	PTY        bool
	Timestamp  bool
	Script     []string
	Env        []string
	ExitStatus string

	buffer  outputBuffer
	command *exec.Cmd

	// This callback is called when the process offically starts
	StartCallback func()

	// For every line in the process output, this callback will be called
	// with the contents of the line if its filter returns true.
	LineCallback       func(string)
	LinePreProcessor   func(string) string
	LineCallbackFilter func(string) bool

	// Running is stored as an int32 so we can use atomic operations to
	// set/get it (it's accessed by multiple goroutines)
	running int32
}

// If you change header parsing here make sure to change it in the
// buildkite.com frontend logic, too

var headerExpansionRegex = regexp.MustCompile("^(?:\\^\\^\\^\\s+\\+\\+\\+)\\s*$")

func (p *Process) Start() error {
	p.command = exec.Command(p.Script[0], p.Script[1:]...)

	// Copy the current processes ENV and merge in the new ones. We do this
	// so the sub process gets PATH and stuff. We merge our path in over
	// the top of the current one so the ENV from Buildkite and the agent
	// take precedence over the agent
	currentEnv := os.Environ()
	p.command.Env = append(currentEnv, p.Env...)

	var waitGroup sync.WaitGroup

	lineReaderPipe, lineWriterPipe := io.Pipe()

	var multiWriter io.Writer
	if p.Timestamp {
		multiWriter = io.MultiWriter(lineWriterPipe)
	} else {
		multiWriter = io.MultiWriter(&p.buffer, lineWriterPipe)
	}

	// Toggle between running in a pty
	if p.PTY {
		pty, err := StartPTY(p.command)
		if err != nil {
			p.ExitStatus = "1"
			return err
		}

		p.Pid = p.command.Process.Pid
		p.setRunning(true)

		waitGroup.Add(1)

		go func() {
			logger.Debug("[Process] Starting to copy PTY to the buffer")

			// Copy the pty to our buffer. This will block until it
			// EOF's or something breaks.
			_, err = io.Copy(multiWriter, pty)
			if e, ok := err.(*os.PathError); ok && e.Err == syscall.EIO {
				// We can safely ignore this error, because
				// it's just the PTY telling us that it closed
				// successfully.  See:
				// https://github.com/buildkite/agent/pull/34#issuecomment-46080419
				err = nil
			}

			if err != nil {
				logger.Error("[Process] PTY output copy failed with error: %T: %v", err, err)
			} else {
				logger.Debug("[Process] PTY has finished being copied to the buffer")
			}

			waitGroup.Done()
		}()
	} else {
		p.command.Stdout = multiWriter
		p.command.Stderr = multiWriter
		p.command.Stdin = nil

		err := p.command.Start()
		if err != nil {
			p.ExitStatus = "1"
			return err
		}

		p.Pid = p.command.Process.Pid
		p.setRunning(true)
	}

	logger.Info("[Process] Process is running with PID: %d", p.Pid)

	// Add the line callback routine to the waitGroup
	waitGroup.Add(1)

	go func() {
		logger.Debug("[LineScanner] Starting to read lines")

		reader := bufio.NewReader(lineReaderPipe)

		var appending []byte
		var lineCallbackWaitGroup sync.WaitGroup

		for {
			line, isPrefix, err := reader.ReadLine()
			if err != nil {
				if err == io.EOF {
					logger.Debug("[LineScanner] Encountered EOF")
					break
				}

				logger.Error("[LineScanner] Failed to read: (%T: %v)", err, err)
			}

			// If isPrefix is true, that means we've got a really
			// long line incoming, and we'll keep appending to it
			// until isPrefix is false (which means the long line
			// has ended.
			if isPrefix && appending == nil {
				logger.Debug("[LineScanner] Line is too long to read, going to buffer it until it finishes")
				// bufio.ReadLine returns a slice which is only valid until the next invocation
				// since it points to its own internal buffer array. To accumulate the entire
				// result we make a copy of the first prefix, and insure there is spare capacity
				// for future appends to minimize the need for resizing on append.
				appending = make([]byte, len(line), (cap(line))*2)
				copy(appending, line)

				continue
			}

			// Should we be appending?
			if appending != nil {
				appending = append(appending, line...)

				// No more isPrefix! Line is finished!
				if !isPrefix {
					logger.Debug("[LineScanner] Finished buffering long line")
					line = appending

					// Reset appending back to nil
					appending = nil
				} else {
					continue
				}
			}

			// If we're timestamping this main thread will take
			// the hit of running the regex so we can build up
			// the timestamped buffer without breaking headers,
			// otherwise we let the goroutines take the perf hit.

			checkedForCallback := false
			lineHasCallback := false
			lineString := p.LinePreProcessor(string(line))

			// Create the prefixed buffer
			if p.Timestamp {
				lineHasCallback = p.LineCallbackFilter(lineString)
				checkedForCallback = true
				if lineHasCallback || headerExpansionRegex.MatchString(lineString) {
					// Don't timestamp special lines (e.g. header)
					p.buffer.WriteString(fmt.Sprintf("%s\n", line))
				} else {
					currentTime := time.Now().UTC().Format(time.RFC3339)
					p.buffer.WriteString(fmt.Sprintf("[%s] %s\n", currentTime, line))
				}
			}

			if lineHasCallback || !checkedForCallback {
				lineCallbackWaitGroup.Add(1)
				go func(line string) {
					defer lineCallbackWaitGroup.Done()
					if (checkedForCallback && lineHasCallback) || p.LineCallbackFilter(lineString) {
						p.LineCallback(line)
					}
				}(lineString)
			}
		}

		// We need to make sure all the line callbacks have finish before
		// finish up the process
		logger.Debug("[LineScanner] Waiting for callbacks to finish")
		lineCallbackWaitGroup.Wait()

		logger.Debug("[LineScanner] Finished")
		waitGroup.Done()
	}()

	// Call the StartCallback
	go p.StartCallback()

	// Wait until the process has finished. The returned error is nil if the command runs,
	// has no problems copying stdin, stdout, and stderr, and exits with a zero exit status.
	waitResult := p.command.Wait()

	// Close the line writer pipe
	lineWriterPipe.Close()

	// The process is no longer running at this point
	p.setRunning(false)

	// Find the exit status of the script
	p.ExitStatus = getExitStatus(waitResult)

	logger.Info("Process with PID: %d finished with Exit Status: %s", p.Pid, p.ExitStatus)

	// Sometimes (in docker containers) io.Copy never seems to finish. This is a mega
	// hack around it. If it doesn't finish after 1 second, just continue.
	logger.Debug("[Process] Waiting for routines to finish")
	err := timeoutWait(&waitGroup)
	if err != nil {
		logger.Debug("[Process] Timed out waiting for wait group: (%T: %v)", err, err)
	}

	// No error occurred so we can return nil
	return nil
}

func (p *Process) Output() string {
	return p.buffer.String()
}

func (p *Process) Kill() error {
	var err error
	if runtime.GOOS == "windows" {
		// Sending Interrupt on Windows is not implemented.
		// https://golang.org/src/os/exec.go?s=3842:3884#L110
		err = exec.Command("CMD", "/C", "TASKKILL", "/F", "/T", "/PID", strconv.Itoa(p.Pid)).Run()
	} else {
		// Send a sigterm
		err = p.signal(syscall.SIGTERM)
	}
	if err != nil {
		return err
	}

	// Make a channel that we'll use as a timeout
	c := make(chan int, 1)
	checking := true

	// Start a routine that checks to see if the process
	// is still alive.
	go func() {
		for checking {
			logger.Debug("[Process] Checking to see if PID: %d is still alive", p.Pid)

			foundProcess, err := os.FindProcess(p.Pid)

			// Can't find the process at all
			if err != nil {
				logger.Debug("[Process] Could not find process with PID: %d", p.Pid)

				break
			}

			// We have some information about the process
			if foundProcess != nil {
				processState, err := foundProcess.Wait()

				if err != nil || processState.Exited() {
					logger.Debug("[Process] Process with PID: %d has exited.", p.Pid)

					break
				}
			}

			// Retry in a moment
			sleepTime := time.Duration(1 * time.Second)
			time.Sleep(sleepTime)
		}

		c <- 1
	}()

	// Timeout this process after 3 seconds
	select {
	case _ = <-c:
		// Was successfully terminated
	case <-time.After(10 * time.Second):
		// Stop checking in the routine above
		checking = false

		// Forcefully kill the thing
		err = p.signal(syscall.SIGKILL)

		if err != nil {
			return err
		}
	}

	return nil
}

func (p *Process) signal(sig os.Signal) error {
	if p.command != nil && p.command.Process != nil {
		logger.Debug("[Process] Sending signal: %s to PID: %d", sig.String(), p.Pid)

		err := p.command.Process.Signal(sig)
		if err != nil {
			logger.Error("[Process] Failed to send signal: %s to PID: %d (%T: %v)", sig.String(), p.Pid, err, err)
			return err
		}
	} else {
		logger.Debug("[Process] No process to signal yet")
	}

	return nil
}

// Returns whether or not the process is running
func (p *Process) IsRunning() bool {
	return atomic.LoadInt32(&p.running) != 0
}

// Sets the running flag of the process
func (p *Process) setRunning(r bool) {
	// Use the atomic package to avoid race conditions when setting the
	// `running` value from multiple routines
	if r {
		atomic.StoreInt32(&p.running, 1)
	} else {
		atomic.StoreInt32(&p.running, 0)
	}
}

// https://github.com/hnakamur/commango/blob/fe42b1cf82bf536ce7e24dceaef6656002e03743/os/executil/executil.go#L29
// TODO: Can this be better?
func getExitStatus(waitResult error) string {
	exitStatus := -1

	if waitResult != nil {
		if err, ok := waitResult.(*exec.ExitError); ok {
			if s, ok := err.Sys().(syscall.WaitStatus); ok {
				exitStatus = s.ExitStatus()
			} else {
				logger.Error("[Process] Unimplemented for system where exec.ExitError.Sys() is not syscall.WaitStatus.")
			}
		} else {
			logger.Error("[Process] Unexpected error type in getExitStatus: %#v", waitResult)
		}
	} else {
		exitStatus = 0
	}

	return fmt.Sprintf("%d", exitStatus)
}

func timeoutWait(waitGroup *sync.WaitGroup) error {
	// Make a chanel that we'll use as a timeout
	c := make(chan int, 1)

	// Start waiting for the routines to finish
	go func() {
		waitGroup.Wait()
		c <- 1
	}()

	select {
	case _ = <-c:
		return nil
	case <-time.After(10 * time.Second):
		return errors.New("Timeout")
	}

	return nil
}

// outputBuffer is a goroutine safe bytes.Buffer
type outputBuffer struct {
	sync.RWMutex
	buf bytes.Buffer
}

// Write appends the contents of p to the buffer, growing the buffer as needed. It returns
// the number of bytes written.
func (ob *outputBuffer) Write(p []byte) (n int, err error) {
	ob.Lock()
	defer ob.Unlock()
	return ob.buf.Write(p)
}

// WriteString appends the contents of s to the buffer, growing the buffer as needed. It returns
// the number of bytes written.
func (ob *outputBuffer) WriteString(s string) (n int, err error) {
	return ob.Write([]byte(s))
}

// String returns the contents of the unread portion of the buffer
// as a string.  If the Buffer is a nil pointer, it returns "<nil>".
func (ob *outputBuffer) String() string {
	ob.RLock()
	defer ob.RUnlock()
	return ob.buf.String()
}
<|endoftext|>


<commit_before>// +build linux darwin freebsd

package cmd

import (
	"fmt"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"strings"
)

var configCmd = &cobra.Command{
	Use:   "config",
	Short: "Get, set, delete and show configuration in Consul for Docker containers.",
	Long:  `Get, set, delete and show configuration in Consul for Docker containers.`,
	Run:   startConfig,
}

func startConfig(cmd *cobra.Command, args []string) {
	fmt.Println("octo config -h")
}

var (
	// Container is the Docker container we are loading config for.
	Container string

	// ConfigKey is the key for the ENV variable.
	ConfigKey string

	// ConfigValue is the value for the ENV variable.
	ConfigValue string
)

func init() {
	RootCmd.AddCommand(configCmd)
	configCmd.PersistentFlags().StringVarP(&Container, "container", "c", "", "Docker Container")
	configCmd.PersistentFlags().StringVarP(&ConfigKey, "key", "", "", "Key for environmental variable.")
	configCmd.PersistentFlags().StringVarP(&ConfigValue, "value", "", "", "Value for environmental variable.")
}

// ConfigEnv is the struct for an environmental variable for a container.
type ConfigEnv struct {
	Container string
	Key       string
	Value     string
}

// Prefix returns the Consul path for the Container.
func (c *ConfigEnv) Prefix() string {
	prefix := ""
	if prefix = viper.GetString("prefix"); prefix == "" {
		prefix = ConsulPrefix
	}
	containerPath := fmt.Sprintf("%s/%s", strings.TrimPrefix(prefix, "/"), c.Container)
	return containerPath
}

// Path returns the entire Consul path for a Consul config variable.
func (c *ConfigEnv) Path() string {
	fullPath := fmt.Sprintf("%s/%s", c.Prefix(), strings.ToUpper(c.Key))
	return fullPath
}

// Get returns the value of the key passed.
func (c *ConfigEnv) Get() string {
	consul, err := ConsulSetup()
	if err != nil {
		Log("Fatal Consul setup problem.", "info")
	}
	value := ConsulGet(consul, c.Path())
	return value
}

// Set sets a key to a value for a container.
func (c *ConfigEnv) Set() bool {
	consul, err := ConsulSetup()
	if err != nil {
		Log("Fatal Consul setup problem.", "info")
	}
	if ConsulSet(consul, c.Path(), c.Value) {
		Log(fmt.Sprintf("ConfigSet key='%s'", c.Path()), "info")
		return true
	}
	return false
}

// Del deletes a key from Consul.
func (c *ConfigEnv) Del() bool {
	consul, err := ConsulSetup()
	if err != nil {
		Log("Fatal Consul setup problem.", "info")
	}
	value := ConsulDel(consul, c.Path())
	return value
}

// Keys shows the keys for a particular Container.
func (c *ConfigEnv) Keys() []string {
	consul, err := ConsulSetup()
	if err != nil {
		Log("Fatal Consul setup problem.", "info")
	}
	value := ConsulKeys(consul, c.Prefix())
	return value
}

// Variables returns all ConfigEnv structs for particular container.
func (c *ConfigEnv) Variables() []ConfigEnv {
	var vars []ConfigEnv
	consul, err := ConsulSetup()
	if err != nil {
		Log("Fatal Consul setup problem.", "info")
	}
	keys := c.Keys()
	for _, value := range keys {
		keyValue := ConsulGet(consul, value)
		split := strings.Split(value, "/")
		cvar := ConfigEnv{Container: split[1], Key: split[2], Value: keyValue}
		vars = append(vars, cvar)
	}
	return vars
}

func (c *ConfigEnv) Show() {
	if strings.Contains(c.Value, " ") {
		c.Value = fmt.Sprintf("\"%s\"", c.Value)
	}
	fmt.Printf("/%s/%s:%s\n", c.Container, c.Key, c.Value)
}
<commit_msg>Add comment.<commit_after>// +build linux darwin freebsd

package cmd

import (
	"fmt"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"strings"
)

var configCmd = &cobra.Command{
	Use:   "config",
	Short: "Get, set, delete and show configuration in Consul for Docker containers.",
	Long:  `Get, set, delete and show configuration in Consul for Docker containers.`,
	Run:   startConfig,
}

func startConfig(cmd *cobra.Command, args []string) {
	fmt.Println("octo config -h")
}

var (
	// Container is the Docker container we are loading config for.
	Container string

	// ConfigKey is the key for the ENV variable.
	ConfigKey string

	// ConfigValue is the value for the ENV variable.
	ConfigValue string
)

func init() {
	RootCmd.AddCommand(configCmd)
	configCmd.PersistentFlags().StringVarP(&Container, "container", "c", "", "Docker Container")
	configCmd.PersistentFlags().StringVarP(&ConfigKey, "key", "", "", "Key for environmental variable.")
	configCmd.PersistentFlags().StringVarP(&ConfigValue, "value", "", "", "Value for environmental variable.")
}

// ConfigEnv is the struct for an environmental variable for a container.
type ConfigEnv struct {
	Container string
	Key       string
	Value     string
}

// Prefix returns the Consul path for the Container.
func (c *ConfigEnv) Prefix() string {
	prefix := ""
	if prefix = viper.GetString("prefix"); prefix == "" {
		prefix = ConsulPrefix
	}
	containerPath := fmt.Sprintf("%s/%s", strings.TrimPrefix(prefix, "/"), c.Container)
	return containerPath
}

// Path returns the entire Consul path for a Consul config variable.
func (c *ConfigEnv) Path() string {
	fullPath := fmt.Sprintf("%s/%s", c.Prefix(), strings.ToUpper(c.Key))
	return fullPath
}

// Get returns the value of the key passed.
func (c *ConfigEnv) Get() string {
	consul, err := ConsulSetup()
	if err != nil {
		Log("Fatal Consul setup problem.", "info")
	}
	value := ConsulGet(consul, c.Path())
	return value
}

// Set sets a key to a value for a container.
func (c *ConfigEnv) Set() bool {
	consul, err := ConsulSetup()
	if err != nil {
		Log("Fatal Consul setup problem.", "info")
	}
	if ConsulSet(consul, c.Path(), c.Value) {
		Log(fmt.Sprintf("ConfigSet key='%s'", c.Path()), "info")
		return true
	}
	return false
}

// Del deletes a key from Consul.
func (c *ConfigEnv) Del() bool {
	consul, err := ConsulSetup()
	if err != nil {
		Log("Fatal Consul setup problem.", "info")
	}
	value := ConsulDel(consul, c.Path())
	return value
}

// Keys shows the keys for a particular Container.
func (c *ConfigEnv) Keys() []string {
	consul, err := ConsulSetup()
	if err != nil {
		Log("Fatal Consul setup problem.", "info")
	}
	value := ConsulKeys(consul, c.Prefix())
	return value
}

// Variables returns all ConfigEnv structs for particular container.
func (c *ConfigEnv) Variables() []ConfigEnv {
	var vars []ConfigEnv
	consul, err := ConsulSetup()
	if err != nil {
		Log("Fatal Consul setup problem.", "info")
	}
	keys := c.Keys()
	for _, value := range keys {
		keyValue := ConsulGet(consul, value)
		split := strings.Split(value, "/")
		cvar := ConfigEnv{Container: split[1], Key: split[2], Value: keyValue}
		vars = append(vars, cvar)
	}
	return vars
}

// Show lists all config variables for a particular container.
func (c *ConfigEnv) Show() {
	if strings.Contains(c.Value, " ") {
		c.Value = fmt.Sprintf("\"%s\"", c.Value)
	}
	fmt.Printf("/%s/%s:%s\n", c.Container, c.Key, c.Value)
}
<|endoftext|>


<commit_before>/*
 * MinIO Client (C) 2015 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"path/filepath"
	"strings"

	"github.com/minio/mc/pkg/probe"
)

type copyURLsType uint8

//   NOTE: All the parse rules should reduced to A: Copy(Source, Target).
//
//   * VALID RULES
//   =======================
//   A: copy(f, f) -> copy(f, f)
//   B: copy(f, d) -> copy(f, d/f) -> []A
//   C: copy(d1..., d2) -> []copy(f, d2/d1/f) -> []A
//   D: copy([]f, d) -> []B

//   * INVALID RULES
//   =========================
//   copy(d, f)
//   copy(d..., f)
//   copy([](f|d)..., f)

const (
	copyURLsTypeInvalid copyURLsType = iota
	copyURLsTypeA
	copyURLsTypeB
	copyURLsTypeC
	copyURLsTypeD
)

// guessCopyURLType guesses the type of clientURL. This approach all allows prepareURL
// functions to accurately report failure causes.
func guessCopyURLType(sourceURLs []string, targetURL string, isRecursive bool, keys map[string][]prefixSSEPair) (copyURLsType, *probe.Error) {
	if len(sourceURLs) == 1 { // 1 Source, 1 Target
		sourceURL := sourceURLs[0]
		_, sourceContent, err := url2Stat(sourceURL, false, keys)
		if err != nil {
			return copyURLsTypeInvalid, err
		}

		// If recursion is ON, it is type C.
		// If source is a folder, it is Type C.
		if sourceContent.Type.IsDir() || isRecursive {
			return copyURLsTypeC, nil
		}

		// If target is a folder, it is Type B.
		if isAliasURLDir(targetURL, keys) {
			return copyURLsTypeB, nil
		}
		// else Type A.
		return copyURLsTypeA, nil
	}

	// Multiple source args and target is a folder. It is Type D.
	if isAliasURLDir(targetURL, keys) {
		return copyURLsTypeD, nil
	}

	return copyURLsTypeInvalid, errInvalidArgument().Trace()
}

// SINGLE SOURCE - Type A: copy(f, f) -> copy(f, f)
// prepareCopyURLsTypeA - prepares target and source clientURLs for copying.
func prepareCopyURLsTypeA(sourceURL string, targetURL string, encKeyDB map[string][]prefixSSEPair) URLs {
	// Extract alias before fiddling with the clientURL.
	sourceAlias, _, _ := mustExpandAlias(sourceURL)
	// Find alias and expanded clientURL.
	targetAlias, targetURL, _ := mustExpandAlias(targetURL)

	_, sourceContent, err := url2Stat(sourceURL, false, encKeyDB)
	if err != nil {
		// Source does not exist or insufficient privileges.
		return URLs{Error: err.Trace(sourceURL)}
	}
	if !sourceContent.Type.IsRegular() {
		// Source is not a regular file
		return URLs{Error: errInvalidSource(sourceURL).Trace(sourceURL)}
	}

	// All OK.. We can proceed. Type A
	return makeCopyContentTypeA(sourceAlias, sourceContent, targetAlias, targetURL, encKeyDB)
}

// prepareCopyContentTypeA - makes CopyURLs content for copying.
func makeCopyContentTypeA(sourceAlias string, sourceContent *ClientContent, targetAlias string, targetURL string, encKeyDB map[string][]prefixSSEPair) URLs {
	targetContent := ClientContent{URL: *newClientURL(targetURL)}
	return URLs{
		SourceAlias:   sourceAlias,
		SourceContent: sourceContent,
		TargetAlias:   targetAlias,
		TargetContent: &targetContent,
	}
}

// SINGLE SOURCE - Type B: copy(f, d) -> copy(f, d/f) -> A
// prepareCopyURLsTypeB - prepares target and source clientURLs for copying.
func prepareCopyURLsTypeB(sourceURL string, targetURL string, encKeyDB map[string][]prefixSSEPair) URLs {
	// Extract alias before fiddling with the clientURL.
	sourceAlias, _, _ := mustExpandAlias(sourceURL)
	// Find alias and expanded clientURL.
	targetAlias, targetURL, _ := mustExpandAlias(targetURL)

	_, sourceContent, err := url2Stat(sourceURL, false, encKeyDB)
	if err != nil {
		// Source does not exist or insufficient privileges.
		return URLs{Error: err.Trace(sourceURL)}
	}

	if !sourceContent.Type.IsRegular() {
		if sourceContent.Type.IsDir() {
			return URLs{Error: errSourceIsDir(sourceURL).Trace(sourceURL)}
		}
		// Source is not a regular file.
		return URLs{Error: errInvalidSource(sourceURL).Trace(sourceURL)}
	}

	// All OK.. We can proceed. Type B: source is a file, target is a folder and exists.
	return makeCopyContentTypeB(sourceAlias, sourceContent, targetAlias, targetURL, encKeyDB)
}

// makeCopyContentTypeB - CopyURLs content for copying.
func makeCopyContentTypeB(sourceAlias string, sourceContent *ClientContent, targetAlias string, targetURL string, encKeyDB map[string][]prefixSSEPair) URLs {
	// All OK.. We can proceed. Type B: source is a file, target is a folder and exists.
	targetURLParse := newClientURL(targetURL)
	targetURLParse.Path = filepath.ToSlash(filepath.Join(targetURLParse.Path, filepath.Base(sourceContent.URL.Path)))
	return makeCopyContentTypeA(sourceAlias, sourceContent, targetAlias, targetURLParse.String(), encKeyDB)
}

// SINGLE SOURCE - Type C: copy(d1..., d2) -> []copy(d1/f, d1/d2/f) -> []A
// prepareCopyRecursiveURLTypeC - prepares target and source clientURLs for copying.
func prepareCopyURLsTypeC(sourceURL, targetURL string, isRecursive bool, encKeyDB map[string][]prefixSSEPair) <-chan URLs {
	// Extract alias before fiddling with the clientURL.
	sourceAlias, _, _ := mustExpandAlias(sourceURL)
	// Find alias and expanded clientURL.
	targetAlias, targetURL, _ := mustExpandAlias(targetURL)
	copyURLsCh := make(chan URLs)
	go func(sourceURL, targetURL string, copyURLsCh chan URLs) {
		defer close(copyURLsCh)
		sourceClient, err := newClient(sourceURL)
		if err != nil {
			// Source initialization failed.
			copyURLsCh <- URLs{Error: err.Trace(sourceURL)}
			return
		}

		isIncomplete := false
		for sourceContent := range sourceClient.List(isRecursive, isIncomplete, false, DirNone) {
			if sourceContent.Err != nil {
				// Listing failed.
				copyURLsCh <- URLs{Error: sourceContent.Err.Trace(sourceClient.GetURL().String())}
				continue
			}

			if !sourceContent.Type.IsRegular() {
				// Source is not a regular file. Skip it for copy.
				continue
			}

			// All OK.. We can proceed. Type B: source is a file, target is a folder and exists.
			copyURLsCh <- makeCopyContentTypeC(sourceAlias, sourceClient.GetURL(), sourceContent, targetAlias, targetURL, encKeyDB)
		}
	}(sourceURL, targetURL, copyURLsCh)
	return copyURLsCh
}

// makeCopyContentTypeC - CopyURLs content for copying.
func makeCopyContentTypeC(sourceAlias string, sourceURL ClientURL, sourceContent *ClientContent, targetAlias string, targetURL string, encKeyDB map[string][]prefixSSEPair) URLs {
	newSourceURL := sourceContent.URL
	pathSeparatorIndex := strings.LastIndex(sourceURL.Path, string(sourceURL.Separator))
	newSourceSuffix := filepath.ToSlash(newSourceURL.Path)
	if pathSeparatorIndex > 1 {
		sourcePrefix := filepath.ToSlash(sourceURL.Path[:pathSeparatorIndex])
		// do not preserve unix cp behavior when copying from filesytem to
		// objectstore.
		if sourceAlias == "" && targetAlias != "" {
			sourcePrefix = sourceURL.Path
		}
		newSourceSuffix = strings.TrimPrefix(newSourceSuffix, sourcePrefix)
	}
	newTargetURL := urlJoinPath(targetURL, newSourceSuffix)
	return makeCopyContentTypeA(sourceAlias, sourceContent, targetAlias, newTargetURL, encKeyDB)
}

// MULTI-SOURCE - Type D: copy([](f|d...), d) -> []B
// prepareCopyURLsTypeE - prepares target and source clientURLs for copying.
func prepareCopyURLsTypeD(sourceURLs []string, targetURL string, isRecursive bool, encKeyDB map[string][]prefixSSEPair) <-chan URLs {
	copyURLsCh := make(chan URLs)
	go func(sourceURLs []string, targetURL string, copyURLsCh chan URLs) {
		defer close(copyURLsCh)
		for _, sourceURL := range sourceURLs {
			for cpURLs := range prepareCopyURLsTypeC(sourceURL, targetURL, isRecursive, encKeyDB) {
				copyURLsCh <- cpURLs
			}
		}
	}(sourceURLs, targetURL, copyURLsCh)
	return copyURLsCh
}

// prepareCopyURLs - prepares target and source clientURLs for copying.
func prepareCopyURLs(sourceURLs []string, targetURL string, isRecursive bool, encKeyDB map[string][]prefixSSEPair, olderThan, newerThan string) chan URLs {
	copyURLsCh := make(chan URLs)
	go func(sourceURLs []string, targetURL string, copyURLsCh chan URLs, encKeyDB map[string][]prefixSSEPair) {
		defer close(copyURLsCh)
		cpType, err := guessCopyURLType(sourceURLs, targetURL, isRecursive, encKeyDB)
		fatalIf(err.Trace(), "Unable to guess the type of copy operation.")

		switch cpType {
		case copyURLsTypeA:
			copyURLsCh <- prepareCopyURLsTypeA(sourceURLs[0], targetURL, encKeyDB)
		case copyURLsTypeB:
			copyURLsCh <- prepareCopyURLsTypeB(sourceURLs[0], targetURL, encKeyDB)
		case copyURLsTypeC:
			for cURLs := range prepareCopyURLsTypeC(sourceURLs[0], targetURL, isRecursive, encKeyDB) {
				copyURLsCh <- cURLs
			}
		case copyURLsTypeD:
			for cURLs := range prepareCopyURLsTypeD(sourceURLs, targetURL, isRecursive, encKeyDB) {
				copyURLsCh <- cURLs
			}
		default:
			copyURLsCh <- URLs{Error: errInvalidArgument().Trace(sourceURLs...)}
		}
	}(sourceURLs, targetURL, copyURLsCh, encKeyDB)

	finalCopyURLsCh := make(chan URLs)
	go func() {
		defer close(finalCopyURLsCh)
		for cpURLs := range copyURLsCh {
			// Skip objects older than --older-than parameter if specified
			if olderThan != "" && isOlder(cpURLs.SourceContent.Time, olderThan) {
				continue
			}

			// Skip objects newer than --newer-than parameter if specified
			if newerThan != "" && isNewer(cpURLs.SourceContent.Time, newerThan) {
				continue
			}

			finalCopyURLsCh <- cpURLs
		}
	}()

	return finalCopyURLsCh
}
<commit_msg>fix: copy filesytem to objectstore regression (#3183)<commit_after>/*
 * MinIO Client (C) 2015 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"os"
	"path/filepath"
	"strings"

	"github.com/minio/mc/pkg/probe"
)

type copyURLsType uint8

//   NOTE: All the parse rules should reduced to A: Copy(Source, Target).
//
//   * VALID RULES
//   =======================
//   A: copy(f, f) -> copy(f, f)
//   B: copy(f, d) -> copy(f, d/f) -> []A
//   C: copy(d1..., d2) -> []copy(f, d2/d1/f) -> []A
//   D: copy([]f, d) -> []B

//   * INVALID RULES
//   =========================
//   copy(d, f)
//   copy(d..., f)
//   copy([](f|d)..., f)

const (
	copyURLsTypeInvalid copyURLsType = iota
	copyURLsTypeA
	copyURLsTypeB
	copyURLsTypeC
	copyURLsTypeD
)

// guessCopyURLType guesses the type of clientURL. This approach all allows prepareURL
// functions to accurately report failure causes.
func guessCopyURLType(sourceURLs []string, targetURL string, isRecursive bool, keys map[string][]prefixSSEPair) (copyURLsType, *probe.Error) {
	if len(sourceURLs) == 1 { // 1 Source, 1 Target
		sourceURL := sourceURLs[0]
		_, sourceContent, err := url2Stat(sourceURL, false, keys)
		if err != nil {
			return copyURLsTypeInvalid, err
		}

		// If recursion is ON, it is type C.
		// If source is a folder, it is Type C.
		if sourceContent.Type.IsDir() || isRecursive {
			return copyURLsTypeC, nil
		}

		// If target is a folder, it is Type B.
		if isAliasURLDir(targetURL, keys) {
			return copyURLsTypeB, nil
		}
		// else Type A.
		return copyURLsTypeA, nil
	}

	// Multiple source args and target is a folder. It is Type D.
	if isAliasURLDir(targetURL, keys) {
		return copyURLsTypeD, nil
	}

	return copyURLsTypeInvalid, errInvalidArgument().Trace()
}

// SINGLE SOURCE - Type A: copy(f, f) -> copy(f, f)
// prepareCopyURLsTypeA - prepares target and source clientURLs for copying.
func prepareCopyURLsTypeA(sourceURL string, targetURL string, encKeyDB map[string][]prefixSSEPair) URLs {
	// Extract alias before fiddling with the clientURL.
	sourceAlias, _, _ := mustExpandAlias(sourceURL)
	// Find alias and expanded clientURL.
	targetAlias, targetURL, _ := mustExpandAlias(targetURL)

	_, sourceContent, err := url2Stat(sourceURL, false, encKeyDB)
	if err != nil {
		// Source does not exist or insufficient privileges.
		return URLs{Error: err.Trace(sourceURL)}
	}
	if !sourceContent.Type.IsRegular() {
		// Source is not a regular file
		return URLs{Error: errInvalidSource(sourceURL).Trace(sourceURL)}
	}

	// All OK.. We can proceed. Type A
	return makeCopyContentTypeA(sourceAlias, sourceContent, targetAlias, targetURL, encKeyDB)
}

// prepareCopyContentTypeA - makes CopyURLs content for copying.
func makeCopyContentTypeA(sourceAlias string, sourceContent *ClientContent, targetAlias string, targetURL string, encKeyDB map[string][]prefixSSEPair) URLs {
	targetContent := ClientContent{URL: *newClientURL(targetURL)}
	return URLs{
		SourceAlias:   sourceAlias,
		SourceContent: sourceContent,
		TargetAlias:   targetAlias,
		TargetContent: &targetContent,
	}
}

// SINGLE SOURCE - Type B: copy(f, d) -> copy(f, d/f) -> A
// prepareCopyURLsTypeB - prepares target and source clientURLs for copying.
func prepareCopyURLsTypeB(sourceURL string, targetURL string, encKeyDB map[string][]prefixSSEPair) URLs {
	// Extract alias before fiddling with the clientURL.
	sourceAlias, _, _ := mustExpandAlias(sourceURL)
	// Find alias and expanded clientURL.
	targetAlias, targetURL, _ := mustExpandAlias(targetURL)

	_, sourceContent, err := url2Stat(sourceURL, false, encKeyDB)
	if err != nil {
		// Source does not exist or insufficient privileges.
		return URLs{Error: err.Trace(sourceURL)}
	}

	if !sourceContent.Type.IsRegular() {
		if sourceContent.Type.IsDir() {
			return URLs{Error: errSourceIsDir(sourceURL).Trace(sourceURL)}
		}
		// Source is not a regular file.
		return URLs{Error: errInvalidSource(sourceURL).Trace(sourceURL)}
	}

	// All OK.. We can proceed. Type B: source is a file, target is a folder and exists.
	return makeCopyContentTypeB(sourceAlias, sourceContent, targetAlias, targetURL, encKeyDB)
}

// makeCopyContentTypeB - CopyURLs content for copying.
func makeCopyContentTypeB(sourceAlias string, sourceContent *ClientContent, targetAlias string, targetURL string, encKeyDB map[string][]prefixSSEPair) URLs {
	// All OK.. We can proceed. Type B: source is a file, target is a folder and exists.
	targetURLParse := newClientURL(targetURL)
	targetURLParse.Path = filepath.ToSlash(filepath.Join(targetURLParse.Path, filepath.Base(sourceContent.URL.Path)))
	return makeCopyContentTypeA(sourceAlias, sourceContent, targetAlias, targetURLParse.String(), encKeyDB)
}

// SINGLE SOURCE - Type C: copy(d1..., d2) -> []copy(d1/f, d1/d2/f) -> []A
// prepareCopyRecursiveURLTypeC - prepares target and source clientURLs for copying.
func prepareCopyURLsTypeC(sourceURL, targetURL string, isRecursive bool, encKeyDB map[string][]prefixSSEPair) <-chan URLs {
	// Extract alias before fiddling with the clientURL.
	sourceAlias, _, _ := mustExpandAlias(sourceURL)
	// Find alias and expanded clientURL.
	targetAlias, targetURL, _ := mustExpandAlias(targetURL)
	copyURLsCh := make(chan URLs)
	go func(sourceURL, targetURL string, copyURLsCh chan URLs) {
		defer close(copyURLsCh)
		sourceClient, err := newClient(sourceURL)
		if err != nil {
			// Source initialization failed.
			copyURLsCh <- URLs{Error: err.Trace(sourceURL)}
			return
		}

		isIncomplete := false
		for sourceContent := range sourceClient.List(isRecursive, isIncomplete, false, DirNone) {
			if sourceContent.Err != nil {
				// Listing failed.
				copyURLsCh <- URLs{Error: sourceContent.Err.Trace(sourceClient.GetURL().String())}
				continue
			}

			if !sourceContent.Type.IsRegular() {
				// Source is not a regular file. Skip it for copy.
				continue
			}

			// All OK.. We can proceed. Type B: source is a file, target is a folder and exists.
			copyURLsCh <- makeCopyContentTypeC(sourceAlias, sourceClient.GetURL(), sourceContent, targetAlias, targetURL, encKeyDB)
		}
	}(sourceURL, targetURL, copyURLsCh)
	return copyURLsCh
}

// makeCopyContentTypeC - CopyURLs content for copying.
func makeCopyContentTypeC(sourceAlias string, sourceURL ClientURL, sourceContent *ClientContent, targetAlias string, targetURL string, encKeyDB map[string][]prefixSSEPair) URLs {
	newSourceURL := sourceContent.URL
	pathSeparatorIndex := strings.LastIndex(sourceURL.Path, string(sourceURL.Separator))
	newSourceSuffix := filepath.ToSlash(newSourceURL.Path)
	if pathSeparatorIndex > 1 {
		sourcePrefix := filepath.ToSlash(sourceURL.Path[:pathSeparatorIndex])
		// do not preserve unix cp behavior when copying a filesytem dir to
		// objectstore.
		if sourceAlias == "" && targetAlias != "" {
			// Check if sourceURL.Path is a directory or not
			fileInfo, err := os.Stat(sourceURL.Path)
			if err != nil {
				return URLs{Error: probe.NewError(err)}
			}
			if fileInfo.IsDir() {
				sourcePrefix = sourceURL.Path
			}
		}
		newSourceSuffix = strings.TrimPrefix(newSourceSuffix, sourcePrefix)
	}
	newTargetURL := urlJoinPath(targetURL, newSourceSuffix)
	return makeCopyContentTypeA(sourceAlias, sourceContent, targetAlias, newTargetURL, encKeyDB)
}

// MULTI-SOURCE - Type D: copy([](f|d...), d) -> []B
// prepareCopyURLsTypeE - prepares target and source clientURLs for copying.
func prepareCopyURLsTypeD(sourceURLs []string, targetURL string, isRecursive bool, encKeyDB map[string][]prefixSSEPair) <-chan URLs {
	copyURLsCh := make(chan URLs)
	go func(sourceURLs []string, targetURL string, copyURLsCh chan URLs) {
		defer close(copyURLsCh)
		for _, sourceURL := range sourceURLs {
			for cpURLs := range prepareCopyURLsTypeC(sourceURL, targetURL, isRecursive, encKeyDB) {
				copyURLsCh <- cpURLs
			}
		}
	}(sourceURLs, targetURL, copyURLsCh)
	return copyURLsCh
}

// prepareCopyURLs - prepares target and source clientURLs for copying.
func prepareCopyURLs(sourceURLs []string, targetURL string, isRecursive bool, encKeyDB map[string][]prefixSSEPair, olderThan, newerThan string) chan URLs {
	copyURLsCh := make(chan URLs)
	go func(sourceURLs []string, targetURL string, copyURLsCh chan URLs, encKeyDB map[string][]prefixSSEPair) {
		defer close(copyURLsCh)
		cpType, err := guessCopyURLType(sourceURLs, targetURL, isRecursive, encKeyDB)
		fatalIf(err.Trace(), "Unable to guess the type of copy operation.")

		switch cpType {
		case copyURLsTypeA:
			copyURLsCh <- prepareCopyURLsTypeA(sourceURLs[0], targetURL, encKeyDB)
		case copyURLsTypeB:
			copyURLsCh <- prepareCopyURLsTypeB(sourceURLs[0], targetURL, encKeyDB)
		case copyURLsTypeC:
			for cURLs := range prepareCopyURLsTypeC(sourceURLs[0], targetURL, isRecursive, encKeyDB) {
				copyURLsCh <- cURLs
			}
		case copyURLsTypeD:
			for cURLs := range prepareCopyURLsTypeD(sourceURLs, targetURL, isRecursive, encKeyDB) {
				copyURLsCh <- cURLs
			}
		default:
			copyURLsCh <- URLs{Error: errInvalidArgument().Trace(sourceURLs...)}
		}
	}(sourceURLs, targetURL, copyURLsCh, encKeyDB)

	finalCopyURLsCh := make(chan URLs)
	go func() {
		defer close(finalCopyURLsCh)
		for cpURLs := range copyURLsCh {
			// Skip objects older than --older-than parameter if specified
			if olderThan != "" && isOlder(cpURLs.SourceContent.Time, olderThan) {
				continue
			}

			// Skip objects newer than --newer-than parameter if specified
			if newerThan != "" && isNewer(cpURLs.SourceContent.Time, newerThan) {
				continue
			}

			finalCopyURLsCh <- cpURLs
		}
	}()

	return finalCopyURLsCh
}
<|endoftext|>


<commit_before>package cmd

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/fatih/color"
	"github.com/spf13/cobra"
	"github.com/wataru0225/sreq/config"
	"github.com/wataru0225/sreq/snippet"
)

var editor string
var browse bool

var searchCmd = &cobra.Command{
	Use:     "search",
	Aliases: []string{"s"},
	Short:   "Search on Qiita (short-cut alias: \"s\")",
	Long:    "Search on Qiita (short-cut alias: \"s\")",
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) == 0 {
			fmt.Println("Failed to not argument of search keyword.")
			os.Exit(2)
		}

		argument := strings.Join(args, ",")
		pagenation := 1

		for {
			end := execute(argument, pagenation)
			if end {
				break
			}
			pagenation++
		}
	},
}

func init() {
	RootCmd.AddCommand(searchCmd)
	searchCmd.Flags().StringVar(&editor, "editor", "vim", "Open editor")
	searchCmd.Flags().Bool("browse", false, "Open browse")
}

func execute(argument string, pagenation int) bool {
	resp, err := http.Get(config.BaseURL(strconv.Itoa(pagenation), argument))
	end := true
	if err == nil {
		defer resp.Body.Close()
		if b, err := ioutil.ReadAll(resp.Body); err == nil {
			contents := rendering(b)
			end = scan(contents, argument)
		}
	}

	return end
}

func rendering(b []byte) []*config.Qiita {
	var contents []*config.Qiita
	json.Unmarshal(b, &contents)
	for i, c := range contents {
		fmt.Print(color.YellowString(strconv.Itoa(i) + " -> "))
		fmt.Println(c.Title)
		if count := len(c.Body); count > 256 {
			fmt.Println(color.GreenString(strings.Replace(c.Body, "\n", "", -1)[0:256]))
		} else {
			fmt.Println(color.GreenString(strings.Replace(c.Body, "\n", "", -1)))
		}
		fmt.Print("\n")
	}
	if len(contents) == 10 {
		fmt.Println(color.YellowString("n -> ") + "next page")
	}
	fmt.Print("SELECT > ")
	return contents
}

func scan(content []*config.Qiita, argument string) bool {
	var num string
	if _, err := fmt.Scanf("%s", &num); err == nil {
		if num == "n" {
			return false
		}
		numb, _ := strconv.Atoi(num)
		url, body := writeHistory(content[numb], argument)

		var cfg config.Config
		cfg.Load()

		if cfg.General.OutputType == "browse" || browse == true {
			OpenBrowse(url)
			return true
		}

		if editor == "" {
			editor = cfg.General.Editor
		}
		OpenEditor(body, editor)
	} else {
		fmt.Println(err)
	}
	return true
}

func writeHistory(content *config.Qiita, argument string) (string, string) {
	var snippets snippet.Snippets
	file := config.HistoryFile()
	snippets.Load(file)
	url := content.URL
	newSnippet := snippet.SnippetInfo{
		SearchKeyword: argument,
		Url:           url,
		Title:         content.Title,
	}
	snippets.Snippets = append(snippets.Snippets, newSnippet)
	if err := snippets.Save(file); err != nil {
		fmt.Printf("Failed. %v", err)
		os.Exit(2)
	}
	return url, content.Body
}
<commit_msg>Modified to only use less for markdown viewer<commit_after>package cmd

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/fatih/color"
	"github.com/spf13/cobra"
	"github.com/wataru0225/sreq/config"
	"github.com/wataru0225/sreq/snippet"
)

var editor string
var browse bool

var searchCmd = &cobra.Command{
	Use:     "search",
	Aliases: []string{"s"},
	Short:   "Search on Qiita (short-cut alias: \"s\")",
	Long:    "Search on Qiita (short-cut alias: \"s\")",
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) == 0 {
			fmt.Println("Failed to not argument of search keyword.")
			os.Exit(2)
		}

		argument := strings.Join(args, ",")
		pagenation := 1

		for {
			end := execute(argument, pagenation)
			if end {
				break
			}
			pagenation++
		}
	},
}

func init() {
	RootCmd.AddCommand(searchCmd)
}

func execute(argument string, pagenation int) bool {
	resp, err := http.Get(config.BaseURL(strconv.Itoa(pagenation), argument))
	end := true
	if err == nil {
		defer resp.Body.Close()
		if b, err := ioutil.ReadAll(resp.Body); err == nil {
			contents := rendering(b)
			end = scan(contents, argument)
		}
	}

	return end
}

func rendering(b []byte) []*config.Qiita {
	var contents []*config.Qiita
	json.Unmarshal(b, &contents)
	for i, c := range contents {
		fmt.Print(color.YellowString(strconv.Itoa(i) + " -> "))
		fmt.Println(c.Title)
		if count := len(c.Body); count > 256 {
			fmt.Println(color.GreenString(strings.Replace(c.Body, "\n", "", -1)[0:256]))
		} else {
			fmt.Println(color.GreenString(strings.Replace(c.Body, "\n", "", -1)))
		}
		fmt.Print("\n")
	}
	if len(contents) == 10 {
		fmt.Println(color.YellowString("n -> ") + "next page")
	}
	fmt.Print("SELECT > ")
	return contents
}

func scan(content []*config.Qiita, argument string) bool {
	var num string
	if _, err := fmt.Scanf("%s", &num); err == nil {
		if num == "n" {
			return false
		}
		numb, _ := strconv.Atoi(num)

		target := content[numb]

		go func() {
			writeHistory(target, argument)
		}()

		OpenEditor(target.Body, "less")
	} else {
		fmt.Println(err)
	}
	return true
}

func writeHistory(content *config.Qiita, argument string) {
	var snippets snippet.Snippets
	file := config.HistoryFile()
	snippets.Load(file)
	url := content.URL
	newSnippet := snippet.SnippetInfo{
		SearchKeyword: argument,
		Url:           url,
		Title:         content.Title,
	}
	snippets.Snippets = append(snippets.Snippets, newSnippet)
	if err := snippets.Save(file); err != nil {
		fmt.Printf("Failed. %v", err)
		os.Exit(2)
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os/exec"
	"strconv"
	"strings"

	"github.com/fatih/color"
	"github.com/spf13/cobra"
	"github.com/wataru0225/sreq/snippet"
)

var pagenation int
var argument string

type Qiita struct {
	Title string `json: "title"`
	Url   string `json: "url"`
}

var searchCmd = &cobra.Command{
	Use:   "search",
	Short: "Call Api of Qiita",
	Long:  "Call Api of Qiita",
	Run: func(cmd *cobra.Command, args []string) {
		pagenation = 1
		argument = strings.Join(args, ",")
		var snippets snippet.Snippets
		err := snippets.Load()
		if err != nil {
			fmt.Errorf("Failed. %v", err)
		}
		execute()
	},
}

func init() {
	RootCmd.AddCommand(searchCmd)
}

func execute() {
	resp, err := http.Get("http://qiita.com/api/v2/items?page=" + strconv.Itoa(pagenation) + "&per_page=10&query=" + argument)
	if err == nil {
		defer resp.Body.Close()
		b, errr := ioutil.ReadAll(resp.Body)
		if errr == nil {
			rendering(b)
		}
	}
}

func rendering(b []byte) {
	var content interface{}
	json.Unmarshal(b, &content)
	for i := 0; i < 10; i++ {
		fmt.Print(color.YellowString(strconv.Itoa(i) + " -> "))
		fmt.Println(content.([]interface{})[i].(map[string]interface{})["title"].(string))
	}
	fmt.Println(color.YellowString("n -> ") + "next page")
	fmt.Print("SELECT > ")
	scan(content)
}

func scan(content interface{}) {
	var num string
	_, err := fmt.Scanf("%s", &num)
	if err == nil {
		if num == "n" {
			pagenation++
			execute()
		} else {
			var snippets snippet.Snippets
			numb, _ := strconv.Atoi(num)
			url := content.([]interface{})[numb].(map[string]interface{})["url"].(string)
			newSnippet := snippet.SnippetInfo{
				SearchKeyword: argument,
				Url:           url,
			}
			snippets.Snippets = append(snippets.Snippets, newSnippet)
			errr := snippets.Save()
			if errr != nil {
				fmt.Errorf("Failed. %v", errr)
			}
			exec.Command("open", url).Run()
		}
	} else {
		fmt.Println(err)
	}
}
<commit_msg>Fix snippet append save<commit_after>package cmd

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os/exec"
	"strconv"
	"strings"

	"github.com/fatih/color"
	"github.com/spf13/cobra"
	"github.com/wataru0225/sreq/snippet"
)

var pagenation int
var argument string

type Qiita struct {
	Title string `json: "title"`
	Url   string `json: "url"`
}

var searchCmd = &cobra.Command{
	Use:   "search",
	Short: "Call Api of Qiita",
	Long:  "Call Api of Qiita",
	Run: func(cmd *cobra.Command, args []string) {
		pagenation = 1
		argument = strings.Join(args, ",")
		execute()
	},
}

func init() {
	RootCmd.AddCommand(searchCmd)
}

func execute() {
	resp, err := http.Get("http://qiita.com/api/v2/items?page=" + strconv.Itoa(pagenation) + "&per_page=10&query=" + argument)
	if err == nil {
		defer resp.Body.Close()
		b, errr := ioutil.ReadAll(resp.Body)
		if errr == nil {
			rendering(b)
		}
	}
}

func rendering(b []byte) {
	var content interface{}
	json.Unmarshal(b, &content)
	for i := 0; i < 10; i++ {
		fmt.Print(color.YellowString(strconv.Itoa(i) + " -> "))
		fmt.Println(content.([]interface{})[i].(map[string]interface{})["title"].(string))
	}
	fmt.Println(color.YellowString("n -> ") + "next page")
	fmt.Print("SELECT > ")
	scan(content)
}

func scan(content interface{}) {
	var num string
	_, err := fmt.Scanf("%s", &num)
	if err == nil {
		if num == "n" {
			pagenation++
			execute()
		} else {
			var snippets snippet.Snippets
			snippets.Load()
			numb, _ := strconv.Atoi(num)
			url := content.([]interface{})[numb].(map[string]interface{})["url"].(string)
			newSnippet := snippet.SnippetInfo{
				SearchKeyword: argument,
				Url:           url,
			}
			snippets.Snippets = append(snippets.Snippets, newSnippet)
			errr := snippets.Save()
			if errr != nil {
				fmt.Errorf("Failed. %v", errr)
			}
			exec.Command("open", url).Run()
		}
	} else {
		fmt.Println(err)
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"github.com/codegangsta/cli"
	"github.com/go-macaron/switcher"
	"github.com/jacobhands/multiweb/cmd/flag"
	"github.com/jacobhands/multiweb/router"
	"gopkg.in/macaron.v1"
)

// CmdServer will serve the website
var CmdServer = cli.Command{
	Name:        "server",
	ShortName:   "s",
	Description: "Serves up websites",
	Action:      runCmdServer,
	Flags:       cmdServerFlags,
}

// Flags
var cmdServerFlags = []cli.Flag{
	cli.StringFlag{
		Name:  flag.Folder,
		Value: "./sites",
		Usage: "Folder containing websites. Eg. foo.com and bar.com",
	},
	cli.StringFlag{
		Name:  flag.BaseURL,
		Value: "sites.example.com",
		Usage: "The base domain to route with.",
	},
}

func runCmdServer(ctx *cli.Context) {
	println("Serving files!", ctx.String(flag.Folder), ctx.IsSet(flag.Folder))

	baseURL := ctx.String(flag.BaseURL)
	r := router.New(ctx)
	m := macaron.Classic()
	hs := switcher.NewHostSwitcher()

	// Set instance corresponding to host address.
	hs.Set("*."+baseURL, m)

	m.Get("/", r.GET)
	hs.Run()
}
<commit_msg>Match against any URI<commit_after>package cmd

import (
	"github.com/codegangsta/cli"
	"github.com/go-macaron/switcher"
	"github.com/jacobhands/multiweb/cmd/flag"
	"github.com/jacobhands/multiweb/router"
	"gopkg.in/macaron.v1"
)

// CmdServer will serve the website
var CmdServer = cli.Command{
	Name:        "server",
	ShortName:   "s",
	Description: "Serves up websites",
	Action:      runCmdServer,
	Flags:       cmdServerFlags,
}

// Flags
var cmdServerFlags = []cli.Flag{
	cli.StringFlag{
		Name:  flag.Folder,
		Value: "./sites",
		Usage: "Folder containing websites. Eg. foo.com and bar.com",
	},
	cli.StringFlag{
		Name:  flag.BaseURL,
		Value: "sites.example.com",
		Usage: "The base domain to route with.",
	},
}

func runCmdServer(ctx *cli.Context) {
	println("Serving files!", ctx.String(flag.Folder), ctx.IsSet(flag.Folder))

	baseURL := ctx.String(flag.BaseURL)
	r := router.New(ctx)
	m := macaron.Classic()
	hs := switcher.NewHostSwitcher()

	// Set instance corresponding to host address.
	hs.Set("*."+baseURL, m)

	m.Get("/*", r.GET)
	hs.Run()
}
<|endoftext|>


<commit_before>// Copyright © 2015 NAME HERE <EMAIL ADDRESS>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"bytes"
	"encoding/csv"
	"fmt"
	"github.com/cheggaaa/pb"
	"github.com/spf13/cobra"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"path"
	"time"
)

var countFlag int
var bsFlag int
var checksumFlag string
var cernDistributionFlag bool

var uploadCmd = &cobra.Command{
	Use:   "upload",
	Short: "Benchmarks the uploading process using different object sizes",
	RunE:  upload,
	Long: `This benchmark test will measure the upload performance.

The object size is the result of block size x count. This is the same
approach used by dd.`,
}

// createFile is a substitute for dd
// char is the character to insert
// count is the number of blocks
// bs is the block size: how many bytes are we going to write flush every round.
func createFile(fn, char string, count, bs int) (*os.File, error) {
	var fd *os.File
	if fn == "" {
		tf, err := ioutil.TempFile("", "CLAWIOBENCH-")
		if err != nil {
			return nil, err
		}
		fd = tf
	} else {
		tf, err := os.Create(path.Join(os.TempDir(), fn))
		if err != nil {
			log.Error(err)
			return nil, err
		}
		fd = tf
	}

	// if char is 1 byte then the buffer size will be equal to bs
	buffer := bytes.Repeat([]byte(char), bs)

	for i := 0; i < count; i++ {
		_, err := fd.Write(buffer)
		if err != nil {
			return nil, err
		}
	}

	return fd, nil
}

// This is the distribution of files at CERN
func createCERNDistribution() ([]string, error) {
	fds := []*os.File{}
	fns := []string{}
	fd50MB, err := createFile("testfile-50MB", "1", 1024, 1024*50)
	if err != nil {
		return fns, err
	}
	fds = append(fds, fd50MB)

	fd15MB, err := createFile("testfile-15MB", "1", 1024, 1024*15)
	if err != nil {
		return fns, err
	}
	fds = append(fds, fd15MB)

	fd8MB, err := createFile("testfile-8MB", "1", 1024, 1024*8)
	if err != nil {
		return fns, err
	}
	fds = append(fds, fd8MB)

	fd10MB, err := createFile("testfile-8MB", "1", 1024, 1024*10)
	if err != nil {
		return fns, err
	}
	fds = append(fds, fd10MB)

	fd5MB, err := createFile("testfile-5MB", "1", 1024, 1024*5)
	if err != nil {
		return fns, err
	}
	fds = append(fds, fd5MB)

	fd4MB, err := createFile("testfile-4MB", "1", 1024, 1024*4)
	if err != nil {
		return fns, err
	}
	fds = append(fds, fd4MB)

	fd3MB, err := createFile("testfile-3MB", "1", 1024, 1024*3)
	if err != nil {
		return fns, err
	}
	fds = append(fds, fd3MB)

	fd2MB, err := createFile("testfile-2MB", "1", 1024, 1024*2)
	if err != nil {
		return fns, err
	}
	fds = append(fds, fd2MB)

	fd1MB, err := createFile("testfile-1MB", "1", 1024, 1024)
	if err != nil {
		return fns, err
	}
	fds = append(fds, fd1MB)

	for i := 0; i < 11; i++ {
		fn := fmt.Sprintf("testfile-500KB-%d", i)
		fd, err := createFile(fn, "1", 1024, 500)
		if err != nil {
			return fns, err
		}
		fds = append(fds, fd)
	}

	for i := 0; i < 32; i++ {
		fn := fmt.Sprintf("testfile-50KB-%d", i)
		fd, err := createFile(fn, "1", 1024, 50)
		if err != nil {
			return fns, err
		}
		fds = append(fds, fd)
	}

	for i := 0; i < 28; i++ {
		fn := fmt.Sprintf("testfile-5KB-%d", i)
		fd, err := createFile(fn, "1", 1024, 5)
		if err != nil {
			return fns, err
		}
		fds = append(fds, fd)
	}

	for i := 0; i < 15; i++ {
		fn := fmt.Sprintf("testfile-1KB-%d", i)
		fd, err := createFile(fn, "1", 1024, 1)
		if err != nil {
			return fns, err
		}
		fds = append(fds, fd)
	}

	for i := 0; i < 5; i++ {
		fn := fmt.Sprintf("testfile-100B-%d", i)
		fd, err := createFile(fn, "1", 1, 100)
		if err != nil {
			return fns, err
		}
		fds = append(fds, fd)
	}

	for _, v := range fds {
		fns = append(fns, v.Name())
		v.Close()
	}

	return fns, nil
}

func upload(cmd *cobra.Command, args []string) error {
	if len(args) != 1 {
		cmd.Help()
		return nil
	}

	if concurrencyFlag > probesFlag {
		concurrencyFlag = probesFlag
	}
	if concurrencyFlag == 0 {
		concurrencyFlag++
	}

	token, err := getToken()
	if err != nil {
		log.Error(err)
		return err
	}

	var fns []string
	if cernDistributionFlag {
		vals, err := createCERNDistribution()
		if err != nil {
			return err
		}
		fns = vals
	} else {
		fd, err := createFile(fmt.Sprintf("testfile-manual-count-%d-bs-%d", countFlag, bsFlag), "1", countFlag, bsFlag)
		if err != nil {
			return err
		}
		fns = append(fns, fd.Name())
		fd.Close()
	}

	defer func() {
		for _, v := range fns {
			os.RemoveAll(v)
		}
	}()

	benchStart := time.Now()

	total := 0
	errorProbes := 0

	errChan := make(chan error)
	resChan := make(chan string)
	doneChan := make(chan bool)
	limitChan := make(chan int, concurrencyFlag)

	for i := 0; i < concurrencyFlag; i++ {
		limitChan <- 1
	}

	var bar *pb.ProgressBar
	if progressBar {
		fmt.Printf("There are %d filenames: %+v\n", len(fns), fns)
		bar = pb.StartNew(probesFlag)
	}

	for i := 0; i < probesFlag; i++ {
		go func(fn string) {
			<-limitChan
			defer func() {
				limitChan <- 1
			}()

			// open again the file
			lfd, err := os.Open(fn)
			if err != nil {
				errChan <- err
			}
			defer lfd.Close()

			c := &http.Client{} // connections are reused if we reuse the client
			// PUT will close the fd
			// is it possible that the HTTP client is reusing connections so is being blocked?
			req, err := http.NewRequest("PUT", dataAddr+args[0], lfd)
			if err != nil {
				errChan <- err
				return
			}

			req.Header.Add("Content-Type", "application/octet-stream")
			req.Header.Add("Authorization", "Bearer "+token)
			req.Header.Add("CIO-Checksum", checksumFlag)

			res, err := c.Do(req)
			if err != nil {
				errChan <- err
				return
			}

			err = res.Body.Close()
			if err != nil {
				errChan <- err
				return
			}

			if res.StatusCode != 201 {
				err := fmt.Errorf("Request failed with status code %d", res.StatusCode)
				errChan <- err
				return
			}

			doneChan <- true
			resChan <- ""
			return
		}(fns[rand.Intn(len(fns))])
	}

	for {
		select {
		case _ = <-doneChan:
			total++
			if progressBar {
				bar.Increment()
			}
		case _ = <-resChan:
		case err := <-errChan:
			log.Error(err)
			errorProbes++
			total++
			if progressBar {
				bar.Increment()
			}
		}

		if total == probesFlag {
			break
		}
	}

	if progressBar {
		bar.Finish()
	}

	numberRequests := probesFlag
	concurrency := concurrencyFlag
	totalTime := time.Since(benchStart).Seconds()
	failedRequests := errorProbes
	frequency := float64(numberRequests-failedRequests) / totalTime
	period := float64(1 / frequency)
	volume := numberRequests * countFlag * bsFlag / 1024 / 1024
	throughput := float64(volume) / totalTime
	data := [][]string{
		{"#NUMBER", "CONCURRENCY", "TIME", "FAILED", "FREQ", "PERIOD", "VOLUME", "THROUGHPUT"},
		{fmt.Sprintf("%d", numberRequests), fmt.Sprintf("%d", concurrency), fmt.Sprintf("%f", totalTime), fmt.Sprintf("%d", failedRequests), fmt.Sprintf("%f", frequency), fmt.Sprintf("%f", period), fmt.Sprintf("%d", volume), fmt.Sprintf("%f", throughput)},
	}
	w := csv.NewWriter(output)
	w.Comma = ' '
	for _, d := range data {
		if err := w.Write(d); err != nil {
			return err
		}
	}
	w.Flush()

	if err := w.Error(); err != nil {
		return err
	}
	return nil
}

func init() {
	RootCmd.AddCommand(uploadCmd)

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// uploadCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// uploadCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")

	uploadCmd.Flags().IntVar(&countFlag, "count", 1024, "The number of blocks of the file")
	uploadCmd.Flags().IntVar(&bsFlag, "bs", 1024, "The number of bytes of each block")
	uploadCmd.Flags().StringVar(&checksumFlag, "checksum", "", "The checksum for the file")
	uploadCmd.Flags().BoolVar(&cernDistributionFlag, "cern-distribution", false, "Use file sizes that follow the distribution found on CERNBox")

}
<commit_msg>Added random source<commit_after>// Copyright © 2015 NAME HERE <EMAIL ADDRESS>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"bytes"
	"encoding/csv"
	"fmt"
	"github.com/cheggaaa/pb"
	"github.com/spf13/cobra"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"path"
	"time"
)

var countFlag int
var bsFlag int
var checksumFlag string
var cernDistributionFlag bool

var uploadCmd = &cobra.Command{
	Use:   "upload",
	Short: "Benchmarks the uploading process using different object sizes",
	RunE:  upload,
	Long: `This benchmark test will measure the upload performance.

The object size is the result of block size x count. This is the same
approach used by dd.`,
}

// createFile is a substitute for dd
// char is the character to insert
// count is the number of blocks
// bs is the block size: how many bytes are we going to write flush every round.
func createFile(fn, char string, count, bs int) (*os.File, error) {
	var fd *os.File
	if fn == "" {
		tf, err := ioutil.TempFile("", "CLAWIOBENCH-")
		if err != nil {
			return nil, err
		}
		fd = tf
	} else {
		tf, err := os.Create(path.Join(os.TempDir(), fn))
		if err != nil {
			log.Error(err)
			return nil, err
		}
		fd = tf
	}

	// if char is 1 byte then the buffer size will be equal to bs
	buffer := bytes.Repeat([]byte(char), bs)

	for i := 0; i < count; i++ {
		_, err := fd.Write(buffer)
		if err != nil {
			return nil, err
		}
	}

	return fd, nil
}

// This is the distribution of files at CERN
func createCERNDistribution() ([]string, error) {
	fds := []*os.File{}
	fns := []string{}
	fd50MB, err := createFile("testfile-50MB", "1", 1024, 1024*50)
	if err != nil {
		return fns, err
	}
	fds = append(fds, fd50MB)

	fd15MB, err := createFile("testfile-15MB", "1", 1024, 1024*15)
	if err != nil {
		return fns, err
	}
	fds = append(fds, fd15MB)

	fd8MB, err := createFile("testfile-8MB", "1", 1024, 1024*8)
	if err != nil {
		return fns, err
	}
	fds = append(fds, fd8MB)

	fd10MB, err := createFile("testfile-8MB", "1", 1024, 1024*10)
	if err != nil {
		return fns, err
	}
	fds = append(fds, fd10MB)

	fd5MB, err := createFile("testfile-5MB", "1", 1024, 1024*5)
	if err != nil {
		return fns, err
	}
	fds = append(fds, fd5MB)

	fd4MB, err := createFile("testfile-4MB", "1", 1024, 1024*4)
	if err != nil {
		return fns, err
	}
	fds = append(fds, fd4MB)

	fd3MB, err := createFile("testfile-3MB", "1", 1024, 1024*3)
	if err != nil {
		return fns, err
	}
	fds = append(fds, fd3MB)

	fd2MB, err := createFile("testfile-2MB", "1", 1024, 1024*2)
	if err != nil {
		return fns, err
	}
	fds = append(fds, fd2MB)

	fd1MB, err := createFile("testfile-1MB", "1", 1024, 1024)
	if err != nil {
		return fns, err
	}
	fds = append(fds, fd1MB)

	for i := 0; i < 11; i++ {
		fn := fmt.Sprintf("testfile-500KB-%d", i)
		fd, err := createFile(fn, "1", 1024, 500)
		if err != nil {
			return fns, err
		}
		fds = append(fds, fd)
	}

	for i := 0; i < 32; i++ {
		fn := fmt.Sprintf("testfile-50KB-%d", i)
		fd, err := createFile(fn, "1", 1024, 50)
		if err != nil {
			return fns, err
		}
		fds = append(fds, fd)
	}

	for i := 0; i < 28; i++ {
		fn := fmt.Sprintf("testfile-5KB-%d", i)
		fd, err := createFile(fn, "1", 1024, 5)
		if err != nil {
			return fns, err
		}
		fds = append(fds, fd)
	}

	for i := 0; i < 15; i++ {
		fn := fmt.Sprintf("testfile-1KB-%d", i)
		fd, err := createFile(fn, "1", 1024, 1)
		if err != nil {
			return fns, err
		}
		fds = append(fds, fd)
	}

	for i := 0; i < 5; i++ {
		fn := fmt.Sprintf("testfile-100B-%d", i)
		fd, err := createFile(fn, "1", 1, 100)
		if err != nil {
			return fns, err
		}
		fds = append(fds, fd)
	}

	for _, v := range fds {
		fns = append(fns, v.Name())
		v.Close()
	}

	return fns, nil
}

func upload(cmd *cobra.Command, args []string) error {
	if len(args) != 1 {
		cmd.Help()
		return nil
	}

	if concurrencyFlag > probesFlag {
		concurrencyFlag = probesFlag
	}
	if concurrencyFlag == 0 {
		concurrencyFlag++
	}

	token, err := getToken()
	if err != nil {
		log.Error(err)
		return err
	}

	var fns []string
	if cernDistributionFlag {
		vals, err := createCERNDistribution()
		if err != nil {
			return err
		}
		fns = vals
	} else {
		fd, err := createFile(fmt.Sprintf("testfile-manual-count-%d-bs-%d", countFlag, bsFlag), "1", countFlag, bsFlag)
		if err != nil {
			return err
		}
		fns = []string{fd.Name()}
		fd.Close()
	}

	defer func() {
		for _, v := range fns {
			os.RemoveAll(v)
		}
	}()

	benchStart := time.Now()

	total := 0
	errorProbes := 0

	errChan := make(chan error)
	resChan := make(chan string)
	doneChan := make(chan bool)
	limitChan := make(chan int, concurrencyFlag)

	for i := 0; i < concurrencyFlag; i++ {
		limitChan <- 1
	}

	var bar *pb.ProgressBar
	if progressBar {
		fmt.Printf("There are %d possible files to upload\n", len(fns))
		bar = pb.StartNew(probesFlag)
	}

	for i := 0; i < probesFlag; i++ {
		rand.Seed(time.Now().UnixNano())
		filename := fns[rand.Intn(len(fns))]
		go func(fn string) {
			<-limitChan
			defer func() {
				limitChan <- 1
			}()

			// open again the file
			lfd, err := os.Open(fn)
			if err != nil {
				errChan <- err
			}
			defer lfd.Close()

			c := &http.Client{} // connections are reused if we reuse the client
			// PUT will close the fd
			// is it possible that the HTTP client is reusing connections so is being blocked?
			req, err := http.NewRequest("PUT", dataAddr+args[0], lfd)
			if err != nil {
				errChan <- err
				return
			}

			req.Header.Add("Content-Type", "application/octet-stream")
			req.Header.Add("Authorization", "Bearer "+token)
			req.Header.Add("CIO-Checksum", checksumFlag)

			res, err := c.Do(req)
			if err != nil {
				errChan <- err
				return
			}

			err = res.Body.Close()
			if err != nil {
				errChan <- err
				return
			}

			if res.StatusCode != 201 {
				err := fmt.Errorf("Request failed with status code %d", res.StatusCode)
				errChan <- err
				return
			}

			doneChan <- true
			resChan <- ""
			return
		}(filename)
	}

	for {
		select {
		case _ = <-doneChan:
			total++
			if progressBar {
				bar.Increment()
			}
		case _ = <-resChan:
		case err := <-errChan:
			log.Error(err)
			errorProbes++
			total++
			if progressBar {
				bar.Increment()
			}
		}

		if total == probesFlag {
			break
		}
	}

	if progressBar {
		bar.Finish()
	}

	numberRequests := probesFlag
	concurrency := concurrencyFlag
	totalTime := time.Since(benchStart).Seconds()
	failedRequests := errorProbes
	frequency := float64(numberRequests-failedRequests) / totalTime
	period := float64(1 / frequency)
	volume := numberRequests * countFlag * bsFlag / 1024 / 1024
	throughput := float64(volume) / totalTime
	data := [][]string{
		{"#NUMBER", "CONCURRENCY", "TIME", "FAILED", "FREQ", "PERIOD", "VOLUME", "THROUGHPUT"},
		{fmt.Sprintf("%d", numberRequests), fmt.Sprintf("%d", concurrency), fmt.Sprintf("%f", totalTime), fmt.Sprintf("%d", failedRequests), fmt.Sprintf("%f", frequency), fmt.Sprintf("%f", period), fmt.Sprintf("%d", volume), fmt.Sprintf("%f", throughput)},
	}
	w := csv.NewWriter(output)
	w.Comma = ' '
	for _, d := range data {
		if err := w.Write(d); err != nil {
			return err
		}
	}
	w.Flush()

	if err := w.Error(); err != nil {
		return err
	}
	return nil
}

func init() {
	RootCmd.AddCommand(uploadCmd)

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// uploadCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// uploadCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")

	uploadCmd.Flags().IntVar(&countFlag, "count", 1024, "The number of blocks of the file")
	uploadCmd.Flags().IntVar(&bsFlag, "bs", 1024, "The number of bytes of each block")
	uploadCmd.Flags().StringVar(&checksumFlag, "checksum", "", "The checksum for the file")
	uploadCmd.Flags().BoolVar(&cernDistributionFlag, "cern-distribution", false, "Use file sizes that follow the distribution found on CERNBox")

}
<|endoftext|>