<commit_before>package common

import (
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"strings"

	sh "github.com/codeskyblue/go-sh"
)

// ShellCmd represents a shell command to be run for dokku
type ShellCmd struct {
	Env           map[string]string
	Command       *exec.Cmd
	CommandString string
	Args          []string
	ShowOutput    bool
}

// NewShellCmd returns a new ShellCmd struct
func NewShellCmd(command string) *ShellCmd {
	items := strings.Split(command, " ")
	cmd := items[0]
	args := items[1:]
	return &ShellCmd{
		Command:       exec.Command(cmd, args...),
		CommandString: command,
		Args:          args,
		ShowOutput:    true,
	}
}

// Execute is a lightweight wrapper around exec.Command
func (dc *ShellCmd) Execute() bool {
	env := os.Environ()
	for k, v := range dc.Env {
		env = append(env, fmt.Sprintf("%s=%s", k, v))
	}
	dc.Command.Env = env
	if dc.ShowOutput {
		dc.Command.Stdout = os.Stdout
		dc.Command.Stderr = os.Stderr
	}
	err := dc.Command.Run()
	if err != nil {
		return false
	}
	return true
}

// VerifyAppName verifies app name format and app existence"
func VerifyAppName(appName string) (err error) {
	dokkuRoot := MustGetEnv("DOKKU_ROOT")
	appRoot := strings.Join([]string{dokkuRoot, appName}, "/")
	_, err = os.Stat(appRoot)
	if os.IsNotExist(err) {
		return fmt.Errorf("App %s does not exist: %v\n", appName, err)
	}
	r, _ := regexp.Compile("^[a-z].*")
	if !r.MatchString(appName) {
		return fmt.Errorf("App name (%s) must begin with lowercase alphanumeric character\n", appName)
	}
	return err
}

// MustGetEnv returns env variable or fails if it's not set
func MustGetEnv(key string) string {
	dokkuRoot := os.Getenv(key)
	if dokkuRoot == "" {
		LogFail(fmt.Sprintf("%s not set!", key))
	}
	return dokkuRoot
}

// LogFail is the failure log formatter
// prints text to stderr and exits with status 1
func LogFail(text string) {
	fmt.Fprintln(os.Stderr, fmt.Sprintf("FAILED: %s", text))
	os.Exit(1)
}

// GetDeployingAppImageName returns deploying image identifier for a given app, tag tuple. validate if tag is presented
func GetDeployingAppImageName(appName, imageTag, imageRepo string) (imageName string) {
	if appName == "" {
		LogFail("(GetDeployingAppImageName) APP must not be empty")
	}

	b, err := sh.Command("plugn", "trigger", "deployed-app-repository", appName).Output()
	if err != nil {
		LogFail(err.Error())
	}
	imageRemoteRepository := string(b[:])

	b, err = sh.Command("plugn", "trigger", "deployed-app-image-tag", appName).Output()
	if err != nil {
		LogFail(err.Error())
	}
	newImageTag := string(b[:])

	b, err = sh.Command("plugn", "trigger", "deployed-app-image-repo", appName).Output()
	if err != nil {
		LogFail(err.Error())
	}
	newImageRepo := string(b[:])

	if newImageRepo != "" {
		imageRepo = newImageRepo
	}
	if newImageTag != "" {
		imageTag = newImageTag
	}
	if imageRepo == "" {
		imageRepo = GetAppImageRepo(appName)
	}
	if imageTag == "" {
		imageTag = "latest"
	}

	imageName = fmt.Sprintf("%s%s:%s", imageRemoteRepository, imageRepo, imageTag)
	if !VerifyImage(imageName) {
		LogFail(fmt.Sprintf("app image (%s) not found", imageName))
	}
	return
}

// GetAppImageRepo is the central definition of a dokku image repo pattern
func GetAppImageRepo(appName string) string {
	return strings.Join([]string{"dokku", appName}, "/")
}

// VerifyImage returns true if docker image exists in local repo
func VerifyImage(image string) bool {
	imageCmd := NewShellCmd(strings.Join([]string{"docker inspect", image}, " "))
	imageCmd.ShowOutput = false
	if imageCmd.Execute() {
		return true
	}
	return false
}
<commit_msg>fix overlooked rename of struct method receiver var<commit_after>package common

import (
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"strings"

	sh "github.com/codeskyblue/go-sh"
)

// ShellCmd represents a shell command to be run for dokku
type ShellCmd struct {
	Env           map[string]string
	Command       *exec.Cmd
	CommandString string
	Args          []string
	ShowOutput    bool
}

// NewShellCmd returns a new ShellCmd struct
func NewShellCmd(command string) *ShellCmd {
	items := strings.Split(command, " ")
	cmd := items[0]
	args := items[1:]
	return &ShellCmd{
		Command:       exec.Command(cmd, args...),
		CommandString: command,
		Args:          args,
		ShowOutput:    true,
	}
}

// Execute is a lightweight wrapper around exec.Command
func (sc *ShellCmd) Execute() bool {
	env := os.Environ()
	for k, v := range sc.Env {
		env = append(env, fmt.Sprintf("%s=%s", k, v))
	}
	sc.Command.Env = env
	if sc.ShowOutput {
		sc.Command.Stdout = os.Stdout
		sc.Command.Stderr = os.Stderr
	}
	err := sc.Command.Run()
	if err != nil {
		return false
	}
	return true
}

// VerifyAppName verifies app name format and app existence"
func VerifyAppName(appName string) (err error) {
	dokkuRoot := MustGetEnv("DOKKU_ROOT")
	appRoot := strings.Join([]string{dokkuRoot, appName}, "/")
	_, err = os.Stat(appRoot)
	if os.IsNotExist(err) {
		return fmt.Errorf("App %s does not exist: %v\n", appName, err)
	}
	r, _ := regexp.Compile("^[a-z].*")
	if !r.MatchString(appName) {
		return fmt.Errorf("App name (%s) must begin with lowercase alphanumeric character\n", appName)
	}
	return err
}

// MustGetEnv returns env variable or fails if it's not set
func MustGetEnv(key string) string {
	dokkuRoot := os.Getenv(key)
	if dokkuRoot == "" {
		LogFail(fmt.Sprintf("%s not set!", key))
	}
	return dokkuRoot
}

// LogFail is the failure log formatter
// prints text to stderr and exits with status 1
func LogFail(text string) {
	fmt.Fprintln(os.Stderr, fmt.Sprintf("FAILED: %s", text))
	os.Exit(1)
}

// GetDeployingAppImageName returns deploying image identifier for a given app, tag tuple. validate if tag is presented
func GetDeployingAppImageName(appName, imageTag, imageRepo string) (imageName string) {
	if appName == "" {
		LogFail("(GetDeployingAppImageName) APP must not be empty")
	}

	b, err := sh.Command("plugn", "trigger", "deployed-app-repository", appName).Output()
	if err != nil {
		LogFail(err.Error())
	}
	imageRemoteRepository := string(b[:])

	b, err = sh.Command("plugn", "trigger", "deployed-app-image-tag", appName).Output()
	if err != nil {
		LogFail(err.Error())
	}
	newImageTag := string(b[:])

	b, err = sh.Command("plugn", "trigger", "deployed-app-image-repo", appName).Output()
	if err != nil {
		LogFail(err.Error())
	}
	newImageRepo := string(b[:])

	if newImageRepo != "" {
		imageRepo = newImageRepo
	}
	if newImageTag != "" {
		imageTag = newImageTag
	}
	if imageRepo == "" {
		imageRepo = GetAppImageRepo(appName)
	}
	if imageTag == "" {
		imageTag = "latest"
	}

	imageName = fmt.Sprintf("%s%s:%s", imageRemoteRepository, imageRepo, imageTag)
	if !VerifyImage(imageName) {
		LogFail(fmt.Sprintf("app image (%s) not found", imageName))
	}
	return
}

// GetAppImageRepo is the central definition of a dokku image repo pattern
func GetAppImageRepo(appName string) string {
	return strings.Join([]string{"dokku", appName}, "/")
}

// VerifyImage returns true if docker image exists in local repo
func VerifyImage(image string) bool {
	imageCmd := NewShellCmd(strings.Join([]string{"docker inspect", image}, " "))
	imageCmd.ShowOutput = false
	if imageCmd.Execute() {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package queue

import (
	"encoding/json"
	"errors"
	"fmt"
	"time"

	log "github.com/SpirentOrion/logrus"
	"github.com/SpirentOrion/luddite/stats"
	"github.com/goamz/goamz/aws"
	"github.com/goamz/goamz/sqs"
)

const (
	statSqsSendSuffix  = ".send"
	statSqsErrorSuffix = ".error."
)

// Verify that SqsQueue implements Queue.
var _ Queue = &SqsQueue{}

// SqsParams holds properties for SQS-based queues.
type SqsParams struct {
	Region    string
	QueueName string
	AccessKey string
	SecretKey string
}

// NewSqsParams extracts SQS provider parameters from a
// generic string map and returns a SqsParams structure.
func NewSqsParams(params map[string]string) (*SqsParams, error) {
	p := &SqsParams{
		Region:    params["region"],
		QueueName: params["queue_name"],
		AccessKey: params["access_key"],
		SecretKey: params["secret_key"],
	}

	if p.Region == "" {
		return nil, errors.New("SQS providers require a 'region' parameter")
	}
	if p.QueueName == "" {
		return nil, errors.New("SQS providers require a 'queue_name' parameter")
	}

	return p, nil
}

type SqsQueue struct {
	logger      *log.Entry
	stats       stats.Stats
	statsPrefix string
	*sqs.Queue
}

func NewSqsQueue(params *SqsParams, logger *log.Entry, stats stats.Stats) (*SqsQueue, error) {
	auth, err := aws.GetAuth(params.AccessKey, params.SecretKey, "", time.Time{})
	if err != nil {
		return nil, err
	}
	sqs := sqs.New(auth, aws.Regions[params.Region])
	queue, err := sqs.GetQueue(params.QueueName)
	if err != nil {
		return nil, err
	}
	return &SqsQueue{
		logger:      logger,
		stats:       stats,
		statsPrefix: fmt.Sprintf("queue.%s.%s.%s.", SQS_PROVIDER, params.Region, params.QueueName),
		Queue:       queue,
	}, nil
}

func (q *SqsQueue) Send(msg *Message) error {
	const op = "SendMessage"

	buf, err := json.Marshal(msg)
	if err != nil {
		q.logger.WithFields(log.Fields{
			"provider":   SQS_PROVIDER,
			"region":     q.Queue.Region,
			"queue_name": q.Queue.Name,
			"op":         op,
			"error":      err,
		}).Error()
		q.stats.Incr(q.statsPrefix+statSqsErrorSuffix, 1)
		return err
	}

	resp, err := q.SendMessage(string(buf))
	if err != nil {
		q.logger.WithFields(log.Fields{
			"provider":   SQS_PROVIDER,
			"region":     q.Queue.Region,
			"queue_name": q.Queue.Name,
			"op":         op,
			"error":      err,
		}).Error()
		q.stats.Incr(q.statsPrefix+statSqsErrorSuffix, 1)
		return err
	}

	q.logger.WithFields(log.Fields{
		"provider":   SQS_PROVIDER,
		"region":     q.Queue.Region,
		"queue_name": q.Queue.Name,
		"op":         op,
		"message_id": resp.Id,
	}).Info()
	q.stats.Incr(q.statsPrefix+statSqsSendSuffix, 1)
	return nil
}
<commit_msg>Added support for receiving SQS messages (to be used by orion-worker processes).<commit_after>package queue

import (
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
	"time"

	log "github.com/SpirentOrion/logrus"
	"github.com/SpirentOrion/luddite/stats"
	"github.com/goamz/goamz/aws"
	"github.com/goamz/goamz/sqs"
)

const (
	statSqsSendSuffix    = ".send"
	statSqsReceiveSuffix = ".receive"
	statSqsDeleteSuffix  = ".delete"
	statSqsErrorSuffix   = ".error"
)

// Verify that SqsQueue implements Queue.
var _ Queue = &SqsQueue{}

// SqsParams holds properties for SQS-based queues.
type SqsParams struct {
	Region    string
	QueueName string
	AccessKey string
	SecretKey string
}

// NewSqsParams extracts SQS provider parameters from a
// generic string map and returns a SqsParams structure.
func NewSqsParams(params map[string]string) (*SqsParams, error) {
	p := &SqsParams{
		Region:    params["region"],
		QueueName: params["queue_name"],
		AccessKey: params["access_key"],
		SecretKey: params["secret_key"],
	}

	if p.Region == "" {
		return nil, errors.New("SQS providers require a 'region' parameter")
	}
	if p.QueueName == "" {
		return nil, errors.New("SQS providers require a 'queue_name' parameter")
	}

	return p, nil
}

type SqsQueue struct {
	logger      *log.Entry
	stats       stats.Stats
	statsPrefix string
	queue       *sqs.Queue
}

func NewSqsQueue(params *SqsParams, logger *log.Entry, stats stats.Stats) (*SqsQueue, error) {
	auth, err := aws.GetAuth(params.AccessKey, params.SecretKey, "", time.Time{})
	if err != nil {
		return nil, err
	}
	sqs := sqs.New(auth, aws.Regions[params.Region])
	queue, err := sqs.GetQueue(params.QueueName)
	if err != nil {
		return nil, err
	}
	return &SqsQueue{
		logger:      logger,
		stats:       stats,
		statsPrefix: fmt.Sprintf("queue.%s.%s.%s.", SQS_PROVIDER, params.Region, params.QueueName),
		queue:       queue,
	}, nil
}

func (q *SqsQueue) String() string {
	return q.queue.Name
}

func (q *SqsQueue) Send(msg *Message) error {
	const op = "SendMessage"

	buf, err := json.Marshal(msg)
	if err != nil {
		q.logger.WithFields(log.Fields{
			"provider":   SQS_PROVIDER,
			"region":     q.queue.Region,
			"queue_name": q.queue.Name,
			"op":         op,
			"error":      err,
		}).Error(err.Error())
		q.stats.Incr(q.statsPrefix+statSqsErrorSuffix, 1)
		return err
	}

	resp, err := q.queue.SendMessage(string(buf))
	if err != nil {
		q.logger.WithFields(log.Fields{
			"provider":   SQS_PROVIDER,
			"region":     q.queue.Region,
			"queue_name": q.queue.Name,
			"op":         op,
			"error":      err,
		}).Error(err.Error())
		q.stats.Incr(q.statsPrefix+statSqsErrorSuffix, 1)
		return err
	}

	q.logger.WithFields(log.Fields{
		"provider":   SQS_PROVIDER,
		"region":     q.queue.Region,
		"queue_name": q.queue.Name,
		"op":         op,
		"message_id": resp.Id,
	}).Info()
	q.stats.Incr(q.statsPrefix+statSqsSendSuffix, 1)
	return nil
}

// SqsMessage holds Messages and extra properties for messages
// received from an SQS queue.
type SqsMessage struct {
	MessageId     string
	ReceiptHandle string
	Message
}

func (q *SqsQueue) Receive(maxMessages int, waitTime time.Duration) ([]SqsMessage, error) {
	const op = "ReceiveMessage"

	params := map[string]string{
		"MaxNumberOfMessages": strconv.Itoa(maxMessages),
		"WaitTimeSeconds":     strconv.Itoa(int(waitTime.Seconds())),
	}

	resp, err := q.queue.ReceiveMessageWithParameters(params)
	if err != nil {
		q.logger.WithFields(log.Fields{
			"provider":   SQS_PROVIDER,
			"region":     q.queue.Region,
			"queue_name": q.queue.Name,
			"op":         op,
			"error":      err,
		}).Error(err.Error())
		q.stats.Incr(q.statsPrefix+statSqsErrorSuffix, 1)
		return nil, err
	}
	if len(resp.Messages) == 0 {
		return nil, nil
	}

	msgs := make([]SqsMessage, 0, len(resp.Messages))
	for _, respMsg := range resp.Messages {
		msg := new(SqsMessage)
		if err := json.Unmarshal([]byte(respMsg.Body), &msg.Message); err != nil {
			q.logger.WithFields(log.Fields{
				"provider":   SQS_PROVIDER,
				"region":     q.queue.Region,
				"queue_name": q.queue.Name,
				"op":         op,
				"message_id": respMsg.MessageId,
				"error":      err,
			}).Error(err.Error())
			q.stats.Incr(q.statsPrefix+statSqsErrorSuffix, 1)
			defer q.queue.DeleteMessageUsingReceiptHandle(respMsg.ReceiptHandle)
		}

		msg.MessageId = respMsg.MessageId
		msg.ReceiptHandle = respMsg.ReceiptHandle
		msgs = append(msgs, *msg)

		q.logger.WithFields(log.Fields{
			"provider":       SQS_PROVIDER,
			"region":         q.queue.Region,
			"queue_name":     q.queue.Name,
			"op":             op,
			"message_id":     msg.MessageId,
			"receipt_handle": msg.ReceiptHandle,
		}).Info()
		q.stats.Incr(q.statsPrefix+statSqsReceiveSuffix, 1)
	}

	return msgs, nil
}

func (q *SqsQueue) Delete(receiptHandle string) error {
	const op = "DeleteMessage"

	if _, err := q.queue.DeleteMessageUsingReceiptHandle(receiptHandle); err != nil {
		q.logger.WithFields(log.Fields{
			"provider":   SQS_PROVIDER,
			"region":     q.queue.Region,
			"queue_name": q.queue.Name,
			"op":         op,
			"error":      err,
		}).Error()
		q.stats.Incr(q.statsPrefix+statSqsErrorSuffix, 1)
		return err
	}

	q.logger.WithFields(log.Fields{
		"provider":       SQS_PROVIDER,
		"region":         q.queue.Region,
		"queue_name":     q.queue.Name,
		"op":             op,
		"receipt_handle": receiptHandle,
	}).Info()
	q.stats.Incr(q.statsPrefix+statSqsDeleteSuffix, 1)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"github.com/ernado/selectel/storage"
	"github.com/jwaldrip/odin/cli"
	"github.com/olekukonko/tablewriter"
	"io"
	"log"
	"os"
)

const (
	envKey       = storage.EnvKey
	envUser      = storage.EnvUser
	envContainer = "SELECTEL_CONTAINER"
)

var (
	client         = cli.New("0.1", "Selectel storage command line client", connect)
	user, key      string
	container      string
	api            storage.API
	debug          bool
	errorNotEnough = errors.New("Not enought arguments")
)

func init() {
	client.DefineBoolFlagVar(&debug, "debug", false, "debug mode")
	client.DefineStringFlag("key", "", fmt.Sprintf("selectel storage key (%s)", envKey))
	client.DefineStringFlag("user", "", fmt.Sprintf("selectel storage user (%s)", envUser))
	client.DefineStringFlag("container", "", fmt.Sprintf("default container (%s)", envContainer))
	infoCommand := client.DefineSubCommand("info", "print information about storage/container/object", wrap(info))
	infoCommand.DefineStringFlag("type", "storage", "storage, container or object")
	client.DefineSubCommand("upload", "upload object to container", wrap(upload))
	listCommand := client.DefineSubCommand("list", "list objects in container/storage", wrap(list))
	listCommand.DefineStringFlag("type", "storage", "storage or container")
	downloadCommand := client.DefineSubCommand("download", "download object from container", wrap(download))
	downloadCommand.DefineStringFlag("path", "", "destination path")
	removeCommand := client.DefineSubCommand("remove", "remove object or container", wrap(remove))
	removeCommand.DefineStringFlag("type", "object", "container or object")
	client.DefineSubCommand("create", "create container", wrap(create))
}

func readFlag(c cli.Command, name, env string) string {
	if len(os.Getenv(env)) > 0 {
		return os.Getenv(env)
	}
	return c.Flag(name).String()
}

func blank(s string) bool {
	return len(s) == 0
}

// connect reads credentials and performs auth
func connect(c cli.Command) {
	key = readFlag(c, "key", envKey)
	user = readFlag(c, "user", envUser)
	container = readFlag(c, "container", envContainer)

	// checking for blank credentials
	if blank(key) || blank(user) {
		log.Fatal(storage.ErrorBadCredentials)
	}

	// connencting to api
	var err error
	api, err = storage.New(user, key)
	if err != nil {
		log.Fatal(err)
	}
	api.Debug(debug)
}

func wrap(callback func(cli.Command)) func(cli.Command) {
	return func(c cli.Command) {
		connect(c.Parent())
		callback(c)
	}
}

// info prints information about storage
func info(c cli.Command) {
	var (
		containerName = container
		objectName    string
		data          interface{}
		err           error
		arglen        = len(c.Args())
		command       = c.Flag("type").String()
	)

	defer func() {
		if err != nil {
			log.Fatal(err)
		}
		if blank(containerName) || command == "storage" {
			data = api.Info()
		} else {
			containerApi := api.Container(containerName)
			if blank(objectName) {
				data, err = containerApi.Info()
			} else {
				data, err = containerApi.Object(objectName).Info()
			}
		}
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("%+v\n", data)
	}()

	if arglen > 0 {
		if command == "container" {
			containerName = c.Arg(0).String()
			return
		}
		command = "object"
		if !blank(containerName) && arglen == 1 {
			objectName = c.Arg(0).String()
			return
		}
		if arglen == 2 {
			containerName = c.Arg(0).String()
			objectName = c.Arg(1).String()
			return
		}
	}
	if command == "container" && !blank(containerName) {
		return
	}
	if command == "storage" {
		return
	}
	err = errorNotEnough
}

func remove(c cli.Command) {
	var (
		arglen  = len(c.Args())
		object  string
		err     error
		message string
	)
	if arglen == 2 {
		container = c.Arg(0).String()
		object = c.Arg(1).String()
	}
	if arglen == 1 {
		if c.Flag("type").String() == "container" {
			container = c.Arg(0).String()
		} else {
			object = c.Arg(0).String()
		}
	}
	if blank(container) {
		log.Fatal(errorNotEnough)
	}
	if blank(object) {
		err = api.Container(container).Remove()
		message = fmt.Sprintf("container %s removed", container)
	} else {
		err = api.Container(container).Object(object).Remove()
		message = fmt.Sprintf("object %s removed in container %s", object, container)
	}
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(message)
}

func create(c cli.Command) {
	if len(c.Args()) == 0 {
		log.Fatal(errorNotEnough)
	}
	var name = c.Arg(0).String()
	if _, err := api.CreateContainer(name, false); err != nil {
		log.Fatal(err)
	}
	fmt.Printf("created container %s\n", name)
}

func upload(c cli.Command) {
	var path string
	switch len(c.Args()) {
	case 1:
		path = c.Arg(0).String()
	case 2:
		container = c.Arg(0).String()
		path = c.Arg(1).String()
	}
	if blank(container) || blank(path) {
		log.Fatal(errorNotEnough)
	}
	if err := api.Container(container).UploadFile(path); err != nil {
		log.Fatal(err)
	}
	fmt.Printf("uploaded to %s\n", container)
}

func list(c cli.Command) {
	var (
		arglen = len(c.Args())
		table  = tablewriter.NewWriter(os.Stdout)
	)
	if arglen == 0 && (blank(container) || c.Flag("type").String() == "storage") {
		containers, err := api.ContainersInfo()
		if err != nil {
			log.Fatal(err)
		}
		table.SetHeader([]string{"Name", "Objects", "Type"})
		for _, cont := range containers {
			v := []string{cont.Name, fmt.Sprint(cont.ObjectCount), cont.Type}
			table.Append(v)
		}
		table.Render()
		return
	}
	if arglen == 1 {
		container = c.Arg(0).String()
	}
	if blank(container) {
		log.Fatal(errorNotEnough)
	}
	objects, err := api.Container(container).ObjectsInfo()
	if err != nil {
		log.Fatal(err)
	}
	table.SetHeader([]string{"Name", "Size", "Downloaded"})
	for _, object := range objects {
		v := []string{object.Name, fmt.Sprint(object.Size), fmt.Sprint(object.Downloaded)}
		table.Append(v)
	}
	table.Render()
}

func download(c cli.Command) {
	var (
		arglen     = len(c.Args())
		objectName string
		path       = c.Flag("path").String()
	)
	switch arglen {
	case 1:
		objectName = c.Arg(0).String()
	case 2:
		objectName = c.Arg(1).String()
		container = c.Arg(0).String()
	}
	if blank(container) || blank(objectName) {
		log.Fatal(errorNotEnough)
	}
	if blank(path) {
		path = objectName
	}
	reader, err := api.Container(container).Object(objectName).GetReader()
	if err != nil {
		log.Fatal(err)
	}
	defer reader.Close()
	fmt.Printf("downloading %s->%s from %s\n", objectName, path, container)
	f, err := os.Create(path)
	if err != nil {
		log.Fatal(err)
	}
	n, err := io.Copy(f, reader)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("downloaded %s, %d bytes\n", objectName, n)
}

func main() {
	defer func() {
		if r := recover(); r != nil {
			fmt.Println("Recovered", r)
		}
	}()
	client.Start()
}
<commit_msg>implemented force removal of containers, refactor<commit_after>package main

import (
	"errors"
	"fmt"
	"github.com/ernado/selectel/storage"
	"github.com/jwaldrip/odin/cli"
	"github.com/olekukonko/tablewriter"
	"io"
	"log"
	"os"
)

const (
	envKey       = storage.EnvKey
	envUser      = storage.EnvUser
	envContainer = "SELECTEL_CONTAINER"
)

var (
	client         = cli.New("0.1", "Selectel storage command line client", connect)
	user, key      string
	container      string
	api            storage.API
	debug          bool
	errorNotEnough = errors.New("Not enought arguments")
)

func init() {
	client.DefineBoolFlagVar(&debug, "debug", false, "debug mode")
	client.DefineStringFlag("key", "", fmt.Sprintf("selectel storage key (%s)", envKey))
	client.AliasFlag('k', "key")
	client.DefineStringFlag("user", "", fmt.Sprintf("selectel storage user (%s)", envUser))
	client.AliasFlag('u', "user")
	client.DefineStringFlag("container", "", fmt.Sprintf("default container (%s)", envContainer))
	client.AliasFlag('c', "container")

	infoCommand := client.DefineSubCommand("info", "print information about storage/container/object", wrap(info))
	infoCommand.DefineStringFlag("type", "storage", "storage, container or object")
	infoCommand.AliasFlag('t', "type")

	listCommand := client.DefineSubCommand("list", "list objects in container/storage", wrap(list))
	listCommand.DefineStringFlag("type", "storage", "storage or container")
	listCommand.AliasFlag('t', "type")

	client.DefineSubCommand("upload", "upload object to container", wrap(upload))
	downloadCommand := client.DefineSubCommand("download", "download object from container", wrap(download))
	downloadCommand.DefineStringFlag("path", "", "destination path")
	downloadCommand.AliasFlag('p', "path")

	client.DefineSubCommand("create", "create container", wrap(create))

	removeCommand := client.DefineSubCommand("remove", "remove object or container", wrap(remove))
	removeCommand.DefineStringFlag("type", "object", "container or object")
	removeCommand.DefineBoolFlag("force", false, "remove container with files")
	removeCommand.AliasFlag('f', "force")
	removeCommand.AliasFlag('t', "type")
}

func readFlag(c cli.Command, name, env string) string {
	if len(os.Getenv(env)) > 0 {
		return os.Getenv(env)
	}
	return c.Flag(name).String()
}

func blank(s string) bool {
	return len(s) == 0
}

// connect reads credentials and performs auth
func connect(c cli.Command) {
	key = readFlag(c, "key", envKey)
	user = readFlag(c, "user", envUser)
	container = readFlag(c, "container", envContainer)

	// checking for blank credentials
	if blank(key) || blank(user) {
		log.Fatal(storage.ErrorBadCredentials)
	}

	// connencting to api
	var err error
	api, err = storage.New(user, key)
	if err != nil {
		log.Fatal(err)
	}
	api.Debug(debug)
}

func wrap(callback func(cli.Command)) func(cli.Command) {
	return func(c cli.Command) {
		connect(c.Parent())
		callback(c)
	}
}

// info prints information about storage
func info(c cli.Command) {
	var (
		containerName = container
		objectName    string
		data          interface{}
		err           error
		arglen        = len(c.Args())
		command       = c.Flag("type").String()
	)

	defer func() {
		if err != nil {
			log.Fatal(err)
		}
		if blank(containerName) || command == "storage" {
			data = api.Info()
		} else {
			containerApi := api.Container(containerName)
			if blank(objectName) {
				data, err = containerApi.Info()
			} else {
				data, err = containerApi.Object(objectName).Info()
			}
		}
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("%+v\n", data)
	}()

	if arglen > 0 {
		if command == "container" {
			containerName = c.Arg(0).String()
			return
		}
		command = "object"
		if !blank(containerName) && arglen == 1 {
			objectName = c.Arg(0).String()
			return
		}
		if arglen == 2 {
			containerName = c.Arg(0).String()
			objectName = c.Arg(1).String()
			return
		}
	}
	if command == "container" && !blank(containerName) {
		return
	}
	if command == "storage" {
		return
	}
	err = errorNotEnough
}

func remove(c cli.Command) {
	var (
		arglen  = len(c.Args())
		object  string
		err     error
		message string
		objects []storage.ObjectAPI
	)
	if arglen == 2 {
		container = c.Arg(0).String()
		object = c.Arg(1).String()
	}
	if arglen == 1 {
		if c.Flag("type").String() == "container" {
			container = c.Arg(0).String()
		} else {
			object = c.Arg(0).String()
		}
	}
	if blank(container) {
		log.Fatal(errorNotEnough)
	}
	if blank(object) {
		containerApi := api.Container(container)
		err = containerApi.Remove()

		// forced removal of container
		if err == storage.ErrorConianerNotEmpty && c.Flag("force").Get().(bool) {
			fmt.Println("removing all objects of", container)
			objects, err = containerApi.Objects()
			if err != nil {
				log.Fatal(err)
			}
			for _, object := range objects {
				err = object.Remove()
				// skipping NotFound errors as non-critical
				if err != nil && err != storage.ErrorObjectNotFound {
					log.Fatal(err)
				}
			}
			err = containerApi.Remove()
		}
		message = fmt.Sprintf("container %s removed", container)
	} else {
		err = api.Container(container).Object(object).Remove()
		message = fmt.Sprintf("object %s removed in container %s", object, container)
	}
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(message)
}

func create(c cli.Command) {
	if len(c.Args()) == 0 {
		log.Fatal(errorNotEnough)
	}
	var name = c.Arg(0).String()
	if _, err := api.CreateContainer(name, false); err != nil {
		log.Fatal(err)
	}
	fmt.Printf("created container %s\n", name)
}

func upload(c cli.Command) {
	var path string
	switch len(c.Args()) {
	case 1:
		path = c.Arg(0).String()
	case 2:
		container = c.Arg(0).String()
		path = c.Arg(1).String()
	}
	if blank(container) || blank(path) {
		log.Fatal(errorNotEnough)
	}
	if err := api.Container(container).UploadFile(path); err != nil {
		log.Fatal(err)
	}
	fmt.Printf("uploaded to %s\n", container)
}

func list(c cli.Command) {
	var (
		arglen = len(c.Args())
		table  = tablewriter.NewWriter(os.Stdout)
	)
	if arglen == 0 && (blank(container) || c.Flag("type").String() == "storage") {
		containers, err := api.ContainersInfo()
		if err != nil {
			log.Fatal(err)
		}
		table.SetHeader([]string{"Name", "Objects", "Type"})
		for _, cont := range containers {
			v := []string{cont.Name, fmt.Sprint(cont.ObjectCount), cont.Type}
			table.Append(v)
		}
		table.Render()
		return
	}
	if arglen == 1 {
		container = c.Arg(0).String()
	}
	if blank(container) {
		log.Fatal(errorNotEnough)
	}
	objects, err := api.Container(container).ObjectsInfo()
	if err != nil {
		log.Fatal(err)
	}
	table.SetHeader([]string{"Name", "Size", "Downloaded"})
	for _, object := range objects {
		v := []string{object.Name, fmt.Sprint(object.Size), fmt.Sprint(object.Downloaded)}
		table.Append(v)
	}
	table.Render()
}

func download(c cli.Command) {
	var (
		arglen     = len(c.Args())
		objectName string
		path       = c.Flag("path").String()
	)
	switch arglen {
	case 1:
		objectName = c.Arg(0).String()
	case 2:
		objectName = c.Arg(1).String()
		container = c.Arg(0).String()
	}
	if blank(container) || blank(objectName) {
		log.Fatal(errorNotEnough)
	}
	if blank(path) {
		path = objectName
	}
	reader, err := api.Container(container).Object(objectName).GetReader()
	if err != nil {
		log.Fatal(err)
	}
	defer reader.Close()
	fmt.Printf("downloading %s->%s from %s\n", objectName, path, container)
	f, err := os.Create(path)
	if err != nil {
		log.Fatal(err)
	}
	n, err := io.Copy(f, reader)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("downloaded %s, %d bytes\n", objectName, n)
}

func main() {
	defer func() {
		if r := recover(); r != nil {
			fmt.Println("Recovered", r)
		}
	}()
	client.Start()
}
<|endoftext|>


<commit_before>package store

import (
	"fmt"
	"path"
	"strings"
	"sync"

	etcdErr "github.com/coreos/etcd/error"
)

type EventHistory struct {
	Queue      eventQueue
	StartIndex uint64
	LastIndex  uint64
	rwl        sync.RWMutex
}

func newEventHistory(capacity int) *EventHistory {
	return &EventHistory{
		Queue: eventQueue{
			Capacity: capacity,
			Events:   make([]*Event, capacity),
		},
	}
}

// addEvent function adds event into the eventHistory
func (eh *EventHistory) addEvent(e *Event) *Event {
	eh.rwl.Lock()
	defer eh.rwl.Unlock()

	eh.Queue.insert(e)

	eh.LastIndex = e.Index()

	eh.StartIndex = eh.Queue.Events[eh.Queue.Front].Index()

	return e
}

// scan function is enumerating events from the index in history and
// stops till the first point where the key has identified key
func (eh *EventHistory) scan(key string, recursive bool, index uint64) (*Event, *etcdErr.Error) {
	eh.rwl.RLock()
	defer eh.rwl.RUnlock()

	// the index should locate after the event history's StartIndex
	if index < eh.StartIndex {
		return nil,
			etcdErr.NewError(etcdErr.EcodeEventIndexCleared,
				fmt.Sprintf("the requested history has been cleared [%v/%v]",
					eh.StartIndex, index), 0)
	}

	// the index should locate before the size of the queue minus the duplicate count
	if index > eh.LastIndex { // future index
		return nil, nil
	}

	i := eh.Queue.Front

	for {
		e := eh.Queue.Events[i]

		ok := (e.Node.Key == key)

		if recursive {
			// add tailing slash
			key := path.Clean(key)
			if key[len(key)-1] != '/' {
				key = key + "/"
			}

			ok = ok || strings.HasPrefix(e.Node.Key, key)
		}

		if ok && index <= e.Index() { // make sure we bypass the smaller one
			return e, nil
		}

		i = (i + 1) % eh.Queue.Capacity

		if i == eh.Queue.Back {
			return nil, nil
		}
	}
}

// clone will be protected by a stop-world lock
// do not need to obtain internal lock
func (eh *EventHistory) clone() *EventHistory {
	clonedQueue := eventQueue{
		Capacity: eh.Queue.Capacity,
		Events:   make([]*Event, eh.Queue.Capacity),
		Size:     eh.Queue.Size,
		Front:    eh.Queue.Front,
		Back:     eh.Queue.Back,
	}

	for i, e := range eh.Queue.Events {
		clonedQueue.Events[i] = e
	}

	return &EventHistory{
		StartIndex: eh.StartIndex,
		Queue:      clonedQueue,
		LastIndex:  eh.LastIndex,
	}

}
<commit_msg>refactor(event_history.go) remove the extra logic<commit_after>package store

import (
	"fmt"
	"path"
	"strings"
	"sync"

	etcdErr "github.com/coreos/etcd/error"
)

type EventHistory struct {
	Queue      eventQueue
	StartIndex uint64
	LastIndex  uint64
	rwl        sync.RWMutex
}

func newEventHistory(capacity int) *EventHistory {
	return &EventHistory{
		Queue: eventQueue{
			Capacity: capacity,
			Events:   make([]*Event, capacity),
		},
	}
}

// addEvent function adds event into the eventHistory
func (eh *EventHistory) addEvent(e *Event) *Event {
	eh.rwl.Lock()
	defer eh.rwl.Unlock()

	eh.Queue.insert(e)

	eh.LastIndex = e.Index()

	eh.StartIndex = eh.Queue.Events[eh.Queue.Front].Index()

	return e
}

// scan function is enumerating events from the index in history and
// stops till the first point where the key has identified key
func (eh *EventHistory) scan(key string, recursive bool, index uint64) (*Event, *etcdErr.Error) {
	eh.rwl.RLock()
	defer eh.rwl.RUnlock()

	// the index should locate after the event history's StartIndex
	if index < eh.StartIndex {
		return nil,
			etcdErr.NewError(etcdErr.EcodeEventIndexCleared,
				fmt.Sprintf("the requested history has been cleared [%v/%v]",
					eh.StartIndex, index), 0)
	}

	// the index should locate before the size of the queue minus the duplicate count
	if index > eh.LastIndex { // future index
		return nil, nil
	}

	offset := index - eh.StartIndex
	i := (eh.Queue.Front + int(offset)) % eh.Queue.Capacity

	for {
		e := eh.Queue.Events[i]

		ok := (e.Node.Key == key)

		if recursive {
			// add tailing slash
			key := path.Clean(key)
			if key[len(key)-1] != '/' {
				key = key + "/"
			}

			ok = ok || strings.HasPrefix(e.Node.Key, key)
		}

		if ok {
			return e, nil
		}

		i = (i + 1) % eh.Queue.Capacity

		if i == eh.Queue.Back {
			return nil, nil
		}
	}
}

// clone will be protected by a stop-world lock
// do not need to obtain internal lock
func (eh *EventHistory) clone() *EventHistory {
	clonedQueue := eventQueue{
		Capacity: eh.Queue.Capacity,
		Events:   make([]*Event, eh.Queue.Capacity),
		Size:     eh.Queue.Size,
		Front:    eh.Queue.Front,
		Back:     eh.Queue.Back,
	}

	for i, e := range eh.Queue.Events {
		clonedQueue.Events[i] = e
	}

	return &EventHistory{
		StartIndex: eh.StartIndex,
		Queue:      clonedQueue,
		LastIndex:  eh.LastIndex,
	}

}
<|endoftext|>


<commit_before>package mapi

import (
	"errors"
	"fmt"
	"github.com/Unknwon/com"
	"github.com/fuxiaohei/purine/model"
	"github.com/fuxiaohei/purine/utils"
	"github.com/lunny/tango"
	"mime/multipart"
	"os"
	"path"
	"strings"
	"time"
)

var (
	ERR_MEDIA_LARGE      = errors.New("media-too-large")
	ERR_MEDIA_WRONG_TYPE = errors.New("media-error-type")

	Media = new(MediaApi) // media api group
)

// media api group struct
type MediaApi struct{}

// media upload option
type MediaUploadOption struct {
	Ctx      tango.Ctx
	User     *model.User // media's owner
	FormName string      // form field name
	IsImage  bool        // is image type
}

// upload media
//
//  in  : *MediaUploadOption
//  out : {
//          "media":*Media
//        }
//
func (_ *MediaApi) Upload(v interface{}) *Res {
	meta, ok := v.(*MediaUploadOption)
	if !ok {
		return Fail(paramTypeError(meta))
	}
	res := Setting.ReadMedia(nil)
	if !res.Status {
		return res
	}
	setting := res.Data["media"].(*SettingMedia)
	f, h, err := meta.Ctx.Req().FormFile(meta.FormName)
	if err != nil {
		return Fail(err)
	}

	// check file size
	size, err := getUploadFileSize(f)
	if err != nil {
		return Fail(err)
	}
	if size > setting.MaxSize {
		return Fail(ERR_MEDIA_LARGE)
	}

	// check ext
	ext := path.Ext(h.Filename)
	extRule := setting.FileExt
	if meta.IsImage {
		extRule = setting.ImageExt
	}
	if !strings.Contains(extRule, ext) {
		return Fail(ERR_MEDIA_WRONG_TYPE)
	}

	// hash file name, make dir
	now := time.Now()
	hashName := utils.Md5String(fmt.Sprintf("%d%s%d", meta.User.Id, h.Filename, now.UnixNano())) + ext
	fileName := path.Join("static/upload", hashName)
	fileDir := path.Dir(fileName)
	if !com.IsDir(fileDir) {
		if err = os.MkdirAll(fileDir, os.ModePerm); err != nil {
			return Fail(err)
		}
	}
	if err = meta.Ctx.SaveToFile(meta.FormName, fileName); err != nil {
		return Fail(err)
	}

	// save file media info
	m := &model.Media{
		Name:     h.Filename,
		FileName: hashName,
		FilePath: fileName,
		FileSize: size,
		FileType: h.Header.Get("Content-Type"),
		OwnerId:  meta.User.Id,
	}
	if err = model.SaveMedia(m); err != nil {
		return Fail(err)
	}

	return Success(map[string]interface{}{
		"media": m,
	})
}

// an interface to check Size() method
type fileSizer interface {
	Size() int64
}

// get file size
func getUploadFileSize(f multipart.File) (int64, error) {
	// if return *http.sectionReader, it is alias to *io.SectionReader
	if s, ok := f.(fileSizer); ok {
		return s.Size(), nil
	}
	// or *os.File
	if fp, ok := f.(*os.File); ok {
		fi, err := fp.Stat()
		if err != nil {
			return 0, err
		}
		return fi.Size(), nil
	}
	return 0, nil
}

// media list option
type MediaListOption struct {
	Page, Size int64
}

// list media data
//
//  in  : *MediaListOption
//  out : {
//          "media":[]*Media,
//          "pager":*utils.Pager
//        }
//
func (_ *MediaApi) List(v interface{}) *Res {
	opt, ok := v.(*MediaListOption)
	if !ok {
		return Fail(paramTypeError(opt))
	}
	media, err := model.ListMedia(opt.Page, opt.Size)
	if err != nil {
		return Fail(err)
	}
	count, err := model.CountMedia()
	if err != nil {
		return Fail(err)
	}
	return Success(map[string]interface{}{
		"media": media,
		"pager": utils.CreatePager(opt.Page, opt.Size, count),
	})
}

// remove media file
//
//  in  : int64
//  out : nil
//
func (_ *MediaApi) Remove(v interface{}) *Res {
	id, ok := v.(int64)
	if !ok {
		return Fail(paramTypeError(id))
	}
	media, err := model.GetMediaBy("id", id)
	if err != nil {
		return Fail(err)
	}
	// remove media file
	if err = os.Remove(media.FilePath); err != nil {
		return Fail(err)
	}
	// remove database record
	if err = model.RemoveMedia(id); err != nil {
		return Fail(err)
	}
	return Success(nil)
}
<commit_msg>fix upload file handler not close<commit_after>package mapi

import (
	"errors"
	"fmt"
	"github.com/Unknwon/com"
	"github.com/fuxiaohei/purine/model"
	"github.com/fuxiaohei/purine/utils"
	"github.com/lunny/tango"
	"mime/multipart"
	"os"
	"path"
	"strings"
	"time"
)

var (
	ERR_MEDIA_LARGE      = errors.New("media-too-large")
	ERR_MEDIA_WRONG_TYPE = errors.New("media-error-type")

	Media = new(MediaApi) // media api group
)

// media api group struct
type MediaApi struct{}

// media upload option
type MediaUploadOption struct {
	Ctx      tango.Ctx
	User     *model.User // media's owner
	FormName string      // form field name
	IsImage  bool        // is image type
}

// upload media
//
//  in  : *MediaUploadOption
//  out : {
//          "media":*Media
//        }
//
func (_ *MediaApi) Upload(v interface{}) *Res {
	meta, ok := v.(*MediaUploadOption)
	if !ok {
		return Fail(paramTypeError(meta))
	}
	res := Setting.ReadMedia(nil)
	if !res.Status {
		return res
	}
	setting := res.Data["media"].(*SettingMedia)
	f, h, err := meta.Ctx.Req().FormFile(meta.FormName)
	if err != nil {
		return Fail(err)
	}
    defer f.Close()

	// check file size
	size, err := getUploadFileSize(f)
	if err != nil {
		return Fail(err)
	}
	if size > setting.MaxSize {
		return Fail(ERR_MEDIA_LARGE)
	}

	// check ext
	ext := path.Ext(h.Filename)
	extRule := setting.FileExt
	if meta.IsImage {
		extRule = setting.ImageExt
	}
	if !strings.Contains(extRule, ext) {
		return Fail(ERR_MEDIA_WRONG_TYPE)
	}

	// hash file name, make dir
	now := time.Now()
	hashName := utils.Md5String(fmt.Sprintf("%d%s%d", meta.User.Id, h.Filename, now.UnixNano())) + ext
	fileName := path.Join("static/upload", hashName)
	fileDir := path.Dir(fileName)
	if !com.IsDir(fileDir) {
		if err = os.MkdirAll(fileDir, os.ModePerm); err != nil {
			return Fail(err)
		}
	}
	if err = meta.Ctx.SaveToFile(meta.FormName, fileName); err != nil {
		return Fail(err)
	}

	// save file media info
	m := &model.Media{
		Name:     h.Filename,
		FileName: hashName,
		FilePath: fileName,
		FileSize: size,
		FileType: h.Header.Get("Content-Type"),
		OwnerId:  meta.User.Id,
	}
	if err = model.SaveMedia(m); err != nil {
		return Fail(err)
	}

	return Success(map[string]interface{}{
		"media": m,
	})
}

// an interface to check Size() method
type fileSizer interface {
	Size() int64
}

// get file size
func getUploadFileSize(f multipart.File) (int64, error) {
	// if return *http.sectionReader, it is alias to *io.SectionReader
	if s, ok := f.(fileSizer); ok {
		return s.Size(), nil
	}
	// or *os.File
	if fp, ok := f.(*os.File); ok {
		fi, err := fp.Stat()
		if err != nil {
			return 0, err
		}
		return fi.Size(), nil
	}
	return 0, nil
}

// media list option
type MediaListOption struct {
	Page, Size int64
}

// list media data
//
//  in  : *MediaListOption
//  out : {
//          "media":[]*Media,
//          "pager":*utils.Pager
//        }
//
func (_ *MediaApi) List(v interface{}) *Res {
	opt, ok := v.(*MediaListOption)
	if !ok {
		return Fail(paramTypeError(opt))
	}
	media, err := model.ListMedia(opt.Page, opt.Size)
	if err != nil {
		return Fail(err)
	}
	count, err := model.CountMedia()
	if err != nil {
		return Fail(err)
	}
	return Success(map[string]interface{}{
		"media": media,
		"pager": utils.CreatePager(opt.Page, opt.Size, count),
	})
}

// remove media file
//
//  in  : int64
//  out : nil
//
func (_ *MediaApi) Remove(v interface{}) *Res {
	id, ok := v.(int64)
	if !ok {
		return Fail(paramTypeError(id))
	}
	media, err := model.GetMediaBy("id", id)
	if err != nil {
		return Fail(err)
	}
	// remove media file
	if err = os.Remove(media.FilePath); err != nil {
		return Fail(err)
	}
	// remove database record
	if err = model.RemoveMedia(id); err != nil {
		return Fail(err)
	}
	return Success(nil)
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"os"
)

var version = "v1.4.3"

var optionDocs = map[string]string{
	"-d": "Show debug message",
	"-v": "Show version",
	"-h": "Show help",
}

var cmds = []string{
	"account",
	"dircache",
	"listbucket",
	"alilistbucket",
	"prefop",
	"fput",
	"rput",
	"qupload",
	"qdownload",
	"stat",
	"delete",
	"move",
	"copy",
	"chgm",
	"fetch",
	"prefetch",
	"batchstat",
	"batchdelete",
	"batchchgm",
	"batchcopy",
	"batchmove",
	"batchrename",
	"checkqrsync",
	"b64encode",
	"b64decode",
	"urlencode",
	"urldecode",
	"ts2d",
	"tms2d",
	"tns2d",
	"d2ts",
	"ip",
	"qetag",
	"unzip",
	"privateurl",
	"saveas",
	"reqid",
	"m3u8delete",
}
var cmdDocs = map[string][]string{
	"account":       []string{"qshell [-d] account [<AccessKey> <SecretKey>]", "Get/Set AccessKey and SecretKey"},
	"dircache":      []string{"qshell [-d] dircache <DirCacheRootPath> <DirCacheResultFile>", "Cache the directory structure of a file path"},
	"listbucket":    []string{"qshell [-d] listbucket <Bucket> [<Prefix>] <ListBucketResultFile>", "List all the file in the bucket by prefix"},
	"alilistbucket": []string{"qshell [-d] alilistbucket <DataCenter> <Bucket> <AccessKeyId> <AccesskeySecret> [Prefix] <ListBucketResultFile>", "List all the file in the bucket of aliyun oss by prefix"},
	"prefop":        []string{"qshell [-d] prefop <PersistentId>", "Query the fop status"},
	"fput":          []string{"qshell [-d] fput <Bucket> <Key> <LocalFile> [MimeType] [UpHost]", "Form upload a local file"},
	"rput":          []string{"qshell [-d] rput <Bucket> <Key> <LocalFile> [MimeType] [UpHost]", "Resumable upload a local file"},
	"qupload":       []string{"qshell [-d] qupload [<ThreadCount>] <LocalUploadConfig>", "Batch upload files to the qiniu bucket"},
	"qdownload":     []string{"qshell [-d] qdownload [<ThreadCount>] <LocalDownloadConfig>", "Batch download files from the qiniu bucket"},
	"stat":          []string{"qshell [-d] stat <Bucket> <Key>", "Get the basic info of a remote file"},
	"delete":        []string{"qshell [-d] delete <Bucket> <Key>", "Delete a remote file in the bucket"},
	"move":          []string{"qshell [-d] move <SrcBucket> <SrcKey> <DestBucket> <DestKey>", "Move/Rename a file and save in bucket"},
	"copy":          []string{"qshell [-d] copy <SrcBucket> <SrcKey> <DestBucket> [<DestKey>]", "Make a copy of a file and save in bucket"},
	"chgm":          []string{"qshell [-d] chgm <Bucket> <Key> <NewMimeType>", "Change the mimeType of a file"},
	"fetch":         []string{"qshell [-d] fetch <RemoteResourceUrl> <Bucket> [<Key>]", "Fetch a remote resource by url and save in bucket"},
	"prefetch":      []string{"qshell [-d] prefetch <Bucket> <Key>", "Fetch and update the file in bucket using mirror storage"},
	"batchstat":     []string{"qshell [-d] batchstat <Bucket> <KeyListFile>", "Batch stat files in bucket"},
	"batchdelete":   []string{"qshell [-d] batchdelete <Bucket> <KeyListFile>", "Batch delete files in bucket"},
	"batchchgm":     []string{"qshell [-d] batchchgm <Bucket> <KeyMimeMapFile>", "Batch chgm files in bucket"},
	"batchcopy":     []string{"qshell [-d] batchcopy <SrcBucket> <DestBucket> <SrcDestKeyMapFile>", "Batch copy files from bucket to bucket"},
	"batchmove":     []string{"qshell [-d] batchmove <SrcBucket> <DestBucket> <SrcDestKeyMapFile>", "Batch move files from bucket to bucket"},
	"batchrename":   []string{"qshell [-d] batchrename <Bucket> <OldNewKeyMapFile>", "Batch rename files in the bucket"},
	"checkqrsync":   []string{"qshell [-d] checkqrsync <DirCacheResultFile> <ListBucketResultFile> <IgnoreLocalDir> [Prefix]", "Check the qrsync result"},
	"b64encode":     []string{"qshell [-d] b64encode [<UrlSafe>] <DataToEncode>", "Base64 Encode"},
	"b64decode":     []string{"qshell [-d] b64decode [<UrlSafe>] <DataToDecode>", "Base64 Decode"},
	"urlencode":     []string{"qshell [-d] urlencode <DataToEncode>", "Url encode"},
	"urldecode":     []string{"qshell [-d] urldecode <DataToDecode>", "Url decode"},
	"ts2d":          []string{"qshell [-d] ts2d <TimestampInSeconds>", "Convert timestamp in seconds to a date (TZ: Local)"},
	"tms2d":         []string{"qshell [-d] tms2d <TimestampInMilliSeconds>", "Convert timestamp in milli-seconds to a date (TZ: Local)"},
	"tns2d":         []string{"qshell [-d] tns2d <TimestampIn100NanoSeconds>", "Convert timestamp in 100 nano-seconds to a date (TZ: Local)"},
	"d2ts":          []string{"qshell [-d] d2ts <SecondsToNow>", "Create a timestamp in seconds using seconds to now"},
	"ip":            []string{"qshell [-d] ip <Ip1> [<Ip2> [<Ip3> ...]]]", "Query the ip information"},
	"qetag":         []string{"qshell [-d] qetag <LocalFilePath>", "Calculate the hash of local file using the algorithm of qiniu qetag"},
	"unzip":         []string{"qshell [-d] unzip <QiniuZipFilePath> [<UnzipToDir>]", "Unzip the archive file created by the qiniu mkzip API"},
	"privateurl":    []string{"qshell [-d] privateurl <PublicUrl> [<Deadline>]", "Create private resource access url"},
	"saveas":        []string{"qshell [-d] saveas <PublicUrlWithFop> <SaveBucket> <SaveKey>", "Create a resource access url with fop and saveas"},
	"reqid":         []string{"qshell [-d] reqid <ReqIdToDecode>", "Decode a qiniu reqid"},
	"m3u8delete":    []string{"qshell [-d] m3u8delete <Bucket> <M3u8Key> [<IsPrivate>]", "Delete m3u8 playlist and the slices it references"},
}

func Version() {
	fmt.Println("qshell", version)
}

func Help(cmd string, params ...string) {
	if len(params) == 0 {
		fmt.Println(CmdList())
	} else {
		CmdHelps(params...)
	}
}

func CmdList() string {
	helpAll := fmt.Sprintf("QShell %s\r\n\r\n", version)
	helpAll += "Options:\r\n"
	for k, v := range optionDocs {
		helpAll += fmt.Sprintf("\t%-20s%-20s\r\n", k, v)
	}
	helpAll += "\r\n"
	helpAll += "Commands:\r\n"
	for _, cmd := range cmds {
		if help, ok := cmdDocs[cmd]; ok {
			cmdDesc := help[1]
			helpAll += fmt.Sprintf("\t%-20s%-20s\r\n", cmd, cmdDesc)
		}
	}
	return helpAll
}

func CmdHelps(cmds ...string) {
	defer os.Exit(1)
	if len(cmds) == 0 {
		fmt.Println(CmdList())
	} else {
		for _, cmd := range cmds {
			CmdHelp(cmd)
		}
	}
}

func CmdHelp(cmd string) {
	docStr := fmt.Sprintf("Unknow cmd `%s'", cmd)
	if cmdDoc, ok := cmdDocs[cmd]; ok {
		docStr = fmt.Sprintf("Usage: %s\r\n  %s\r\n", cmdDoc[0], cmdDoc[1])
	}
	fmt.Println(docStr)
}
<commit_msg>Add version 1.4.4<commit_after>package cli

import (
	"fmt"
	"os"
)

var version = "v1.4.4"

var optionDocs = map[string]string{
	"-d": "Show debug message",
	"-v": "Show version",
	"-h": "Show help",
}

var cmds = []string{
	"account",
	"dircache",
	"listbucket",
	"alilistbucket",
	"prefop",
	"fput",
	"rput",
	"qupload",
	"qdownload",
	"stat",
	"delete",
	"move",
	"copy",
	"chgm",
	"fetch",
	"prefetch",
	"batchstat",
	"batchdelete",
	"batchchgm",
	"batchcopy",
	"batchmove",
	"batchrename",
	"checkqrsync",
	"b64encode",
	"b64decode",
	"urlencode",
	"urldecode",
	"ts2d",
	"tms2d",
	"tns2d",
	"d2ts",
	"ip",
	"qetag",
	"unzip",
	"privateurl",
	"saveas",
	"reqid",
	"m3u8delete",
}
var cmdDocs = map[string][]string{
	"account":       []string{"qshell [-d] account [<AccessKey> <SecretKey>]", "Get/Set AccessKey and SecretKey"},
	"dircache":      []string{"qshell [-d] dircache <DirCacheRootPath> <DirCacheResultFile>", "Cache the directory structure of a file path"},
	"listbucket":    []string{"qshell [-d] listbucket <Bucket> [<Prefix>] <ListBucketResultFile>", "List all the file in the bucket by prefix"},
	"alilistbucket": []string{"qshell [-d] alilistbucket <DataCenter> <Bucket> <AccessKeyId> <AccesskeySecret> [Prefix] <ListBucketResultFile>", "List all the file in the bucket of aliyun oss by prefix"},
	"prefop":        []string{"qshell [-d] prefop <PersistentId>", "Query the fop status"},
	"fput":          []string{"qshell [-d] fput <Bucket> <Key> <LocalFile> [MimeType] [UpHost]", "Form upload a local file"},
	"rput":          []string{"qshell [-d] rput <Bucket> <Key> <LocalFile> [MimeType] [UpHost]", "Resumable upload a local file"},
	"qupload":       []string{"qshell [-d] qupload [<ThreadCount>] <LocalUploadConfig>", "Batch upload files to the qiniu bucket"},
	"qdownload":     []string{"qshell [-d] qdownload [<ThreadCount>] <LocalDownloadConfig>", "Batch download files from the qiniu bucket"},
	"stat":          []string{"qshell [-d] stat <Bucket> <Key>", "Get the basic info of a remote file"},
	"delete":        []string{"qshell [-d] delete <Bucket> <Key>", "Delete a remote file in the bucket"},
	"move":          []string{"qshell [-d] move <SrcBucket> <SrcKey> <DestBucket> <DestKey>", "Move/Rename a file and save in bucket"},
	"copy":          []string{"qshell [-d] copy <SrcBucket> <SrcKey> <DestBucket> [<DestKey>]", "Make a copy of a file and save in bucket"},
	"chgm":          []string{"qshell [-d] chgm <Bucket> <Key> <NewMimeType>", "Change the mimeType of a file"},
	"fetch":         []string{"qshell [-d] fetch <RemoteResourceUrl> <Bucket> [<Key>]", "Fetch a remote resource by url and save in bucket"},
	"prefetch":      []string{"qshell [-d] prefetch <Bucket> <Key>", "Fetch and update the file in bucket using mirror storage"},
	"batchstat":     []string{"qshell [-d] batchstat <Bucket> <KeyListFile>", "Batch stat files in bucket"},
	"batchdelete":   []string{"qshell [-d] batchdelete <Bucket> <KeyListFile>", "Batch delete files in bucket"},
	"batchchgm":     []string{"qshell [-d] batchchgm <Bucket> <KeyMimeMapFile>", "Batch chgm files in bucket"},
	"batchcopy":     []string{"qshell [-d] batchcopy <SrcBucket> <DestBucket> <SrcDestKeyMapFile>", "Batch copy files from bucket to bucket"},
	"batchmove":     []string{"qshell [-d] batchmove <SrcBucket> <DestBucket> <SrcDestKeyMapFile>", "Batch move files from bucket to bucket"},
	"batchrename":   []string{"qshell [-d] batchrename <Bucket> <OldNewKeyMapFile>", "Batch rename files in the bucket"},
	"checkqrsync":   []string{"qshell [-d] checkqrsync <DirCacheResultFile> <ListBucketResultFile> <IgnoreLocalDir> [Prefix]", "Check the qrsync result"},
	"b64encode":     []string{"qshell [-d] b64encode [<UrlSafe>] <DataToEncode>", "Base64 Encode"},
	"b64decode":     []string{"qshell [-d] b64decode [<UrlSafe>] <DataToDecode>", "Base64 Decode"},
	"urlencode":     []string{"qshell [-d] urlencode <DataToEncode>", "Url encode"},
	"urldecode":     []string{"qshell [-d] urldecode <DataToDecode>", "Url decode"},
	"ts2d":          []string{"qshell [-d] ts2d <TimestampInSeconds>", "Convert timestamp in seconds to a date (TZ: Local)"},
	"tms2d":         []string{"qshell [-d] tms2d <TimestampInMilliSeconds>", "Convert timestamp in milli-seconds to a date (TZ: Local)"},
	"tns2d":         []string{"qshell [-d] tns2d <TimestampIn100NanoSeconds>", "Convert timestamp in 100 nano-seconds to a date (TZ: Local)"},
	"d2ts":          []string{"qshell [-d] d2ts <SecondsToNow>", "Create a timestamp in seconds using seconds to now"},
	"ip":            []string{"qshell [-d] ip <Ip1> [<Ip2> [<Ip3> ...]]]", "Query the ip information"},
	"qetag":         []string{"qshell [-d] qetag <LocalFilePath>", "Calculate the hash of local file using the algorithm of qiniu qetag"},
	"unzip":         []string{"qshell [-d] unzip <QiniuZipFilePath> [<UnzipToDir>]", "Unzip the archive file created by the qiniu mkzip API"},
	"privateurl":    []string{"qshell [-d] privateurl <PublicUrl> [<Deadline>]", "Create private resource access url"},
	"saveas":        []string{"qshell [-d] saveas <PublicUrlWithFop> <SaveBucket> <SaveKey>", "Create a resource access url with fop and saveas"},
	"reqid":         []string{"qshell [-d] reqid <ReqIdToDecode>", "Decode a qiniu reqid"},
	"m3u8delete":    []string{"qshell [-d] m3u8delete <Bucket> <M3u8Key> [<IsPrivate>]", "Delete m3u8 playlist and the slices it references"},
}

func Version() {
	fmt.Println("qshell", version)
}

func Help(cmd string, params ...string) {
	if len(params) == 0 {
		fmt.Println(CmdList())
	} else {
		CmdHelps(params...)
	}
}

func CmdList() string {
	helpAll := fmt.Sprintf("QShell %s\r\n\r\n", version)
	helpAll += "Options:\r\n"
	for k, v := range optionDocs {
		helpAll += fmt.Sprintf("\t%-20s%-20s\r\n", k, v)
	}
	helpAll += "\r\n"
	helpAll += "Commands:\r\n"
	for _, cmd := range cmds {
		if help, ok := cmdDocs[cmd]; ok {
			cmdDesc := help[1]
			helpAll += fmt.Sprintf("\t%-20s%-20s\r\n", cmd, cmdDesc)
		}
	}
	return helpAll
}

func CmdHelps(cmds ...string) {
	defer os.Exit(1)
	if len(cmds) == 0 {
		fmt.Println(CmdList())
	} else {
		for _, cmd := range cmds {
			CmdHelp(cmd)
		}
	}
}

func CmdHelp(cmd string) {
	docStr := fmt.Sprintf("Unknow cmd `%s'", cmd)
	if cmdDoc, ok := cmdDocs[cmd]; ok {
		docStr = fmt.Sprintf("Usage: %s\r\n  %s\r\n", cmdDoc[0], cmdDoc[1])
	}
	fmt.Println(docStr)
}
<|endoftext|>


<commit_before>package discordgo_test

import (
	"os"
	"runtime"
	"testing"
	"time"

	. "github.com/bwmarrin/discordgo"
)

//////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////// VARS NEEDED FOR TESTING
var (
	dg *Session // Stores global discordgo session

	envToken    string = os.Getenv("DG_TOKEN")    // Token to use when authenticating
	envUsername string = os.Getenv("DG_USERNAME") // Username to use when authenticating
	envPassword string = os.Getenv("DG_PASSWORD") // Password to use when authenticating
	envGuild    string = os.Getenv("DG_GUILD")    // Guild ID to use for tests
	envChannel  string = os.Getenv("DG_CHANNEL")  // Channel ID to use for tests
	envUser     string = os.Getenv("DG_USER")     // User ID to use for tests
	envAdmin    string = os.Getenv("DG_ADMIN")    // User ID of admin user to use for tests
)

//////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////// HELPER FUNCTIONS USED FOR TESTING

// This waits x time for the check bool to be the want bool
func waitBoolEqual(timeout time.Duration, check *bool, want bool) bool {

	start := time.Now()
	for {
		if *check == want {
			return true
		}

		if time.Since(start) > timeout {
			return false
		}

		runtime.Gosched()
	}
}

//////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////// START OF TESTS

// TestNew tests the New() function without any arguments.  This should return
// a valid Session{} struct and no errors.
func TestNew(t *testing.T) {

	_, err := New()
	if err != nil {
		t.Errorf("New() returned error: %+v", err)
	}
}

// TestNewUserPass tests the New() function with a username and password.
// This should return a valid Session{}, a valid Session.Token, and open
// a websocket connection to Discord.
func TestNewUserPass(t *testing.T) {
	if envUsername == "" || envPassword == "" {
		t.Skip("Skipping New(username,password), DG_USERNAME or DG_PASSWORD not set")
		return
	}
	// Not testing yet.
}

// TestNewToken tests the New() function with a Token.  This should return
// the same as the TestNewUserPass function.
func TestNewToken(t *testing.T) {
	if envToken == "" {
		t.Skip("Skipping New(token), DG_TOKEN not set")
		return
	}

	d, err := New(envToken)
	if err != nil {
		t.Fatalf("New(envToken) returned error: %+v", err)
	}

	if d == nil {
		t.Fatal("New(envToken), d is nil, should be Session{}")
	}

	if d.Token == "" {
		t.Fatal("New(envToken), d.Token is empty, should be a valid Token.")
	}

	if !waitBoolEqual(10*time.Second, &d.DataReady, true) {
		t.Fatal("New(envToken), d.DataReady is false after 10 seconds.  Should be true.")
	}

	t.Log("Successfully connected to Discord.")
	dg = d

}
<commit_msg>Added isConnected helper func<commit_after>package discordgo_test

import (
	"os"
	"runtime"
	"testing"
	"time"

	. "github.com/bwmarrin/discordgo"
)

//////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////// VARS NEEDED FOR TESTING
var (
	dg *Session // Stores global discordgo session

	envToken    string = os.Getenv("DG_TOKEN")    // Token to use when authenticating
	envUsername string = os.Getenv("DG_USERNAME") // Username to use when authenticating
	envPassword string = os.Getenv("DG_PASSWORD") // Password to use when authenticating
	envGuild    string = os.Getenv("DG_GUILD")    // Guild ID to use for tests
	envChannel  string = os.Getenv("DG_CHANNEL")  // Channel ID to use for tests
	envUser     string = os.Getenv("DG_USER")     // User ID to use for tests
	envAdmin    string = os.Getenv("DG_ADMIN")    // User ID of admin user to use for tests
)

//////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////// HELPER FUNCTIONS USED FOR TESTING

// This waits x time for the check bool to be the want bool
func waitBoolEqual(timeout time.Duration, check *bool, want bool) bool {

	start := time.Now()
	for {
		if *check == want {
			return true
		}

		if time.Since(start) > timeout {
			return false
		}

		runtime.Gosched()
	}
}

// Checks if we're connected to Discord
func isConnected() bool {

	if dg == nil {
		return false
	}

	if dg.Token == "" {
		return false
	}

	// Need a way to see if the ws connection is nil

	if !waitBoolEqual(10*time.Second, &dg.DataReady, true) {
		return false
	}

	return true
}

//////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////// START OF TESTS

// TestNew tests the New() function without any arguments.  This should return
// a valid Session{} struct and no errors.
func TestNew(t *testing.T) {

	_, err := New()
	if err != nil {
		t.Errorf("New() returned error: %+v", err)
	}
}

// TestNewUserPass tests the New() function with a username and password.
// This should return a valid Session{}, a valid Session.Token, and open
// a websocket connection to Discord.
func TestNewUserPass(t *testing.T) {

	if isConnected() {
		t.Skip("Skipping New(username,password), already connected.")
	}

	if envUsername == "" || envPassword == "" {
		t.Skip("Skipping New(username,password), DG_USERNAME or DG_PASSWORD not set")
		return
	}
	// Not testing yet.
}

// TestNewToken tests the New() function with a Token.  This should return
// the same as the TestNewUserPass function.
func TestNewToken(t *testing.T) {

	if isConnected() {
		t.Skip("Skipping New(token), already connected.")
	}

	if envToken == "" {
		t.Skip("Skipping New(token), DG_TOKEN not set")
	}

	d, err := New(envToken)
	if err != nil {
		t.Fatalf("New(envToken) returned error: %+v", err)
	}

	if d == nil {
		t.Fatal("New(envToken), d is nil, should be Session{}")
	}

	if d.Token == "" {
		t.Fatal("New(envToken), d.Token is empty, should be a valid Token.")
	}

	if !waitBoolEqual(10*time.Second, &d.DataReady, true) {
		t.Fatal("New(envToken), d.DataReady is false after 10 seconds.  Should be true.")
	}

	t.Log("Successfully connected to Discord.")
	dg = d

}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Generic JSON representation.

package json

import (
	"array";
	"fmt";
	"math";
	"json";
	"strconv";
	"strings";
)

export const (
	StringKind = iota;
	NumberKind;
	MapKind;		// JSON term is "Object", but in Go, it's a map
	ArrayKind;
	BoolKind;
	NullKind;
)

export type Json interface {
	Kind() int;
	String() string;
	Number() float64;
	Bool() bool;
	Get(s string) Json;
	Elem(i int) Json;
	Len() int;
}

export func JsonToString(j Json) string {
	if j == nil {
		return "null"
	}
	if j.Kind() == StringKind {
		return Quote(j.String())
	}
	return j.String()
}

type Null struct { }
export var null Json = &Null{}
func (*Null) Kind() int { return NullKind }
func (*Null) String() string { return "null" }
func (*Null) Number() float64 { return 0 }
func (*Null) Bool() bool { return false }
func (*Null) Get(s string) Json { return null }
func (*Null) Elem(int) Json { return null }
func (*Null) Len() int { return 0 }

type String struct { s string; Null }
func (j *String) Kind() int { return StringKind }
func (j *String) String() string { return j.s }

type Number struct { f float64; Null }
func (j *Number) Kind() int { return NumberKind }
func (j *Number) Number() float64 { return j.f }
func (j *Number) String() string {
	if math.Floor(j.f) == j.f {
		return fmt.sprintf("%.0f", j.f);
	}
	return fmt.sprintf("%g", j.f);
}

type Array struct { a *array.Array; Null }
func (j *Array) Kind() int { return ArrayKind }
func (j *Array) Len() int { return j.a.Len() }
func (j *Array) Elem(i int) Json {
	if i < 0 || i >= j.a.Len() {
		return null
	}
	return j.a.At(i)
}
func (j *Array) String() string {
	s := "[";
	for i := 0; i < j.a.Len(); i++ {
		if i > 0 {
			s += ",";
		}
		s += JsonToString(j.a.At(i).(Json));
	}
	s += "]";
	return s;
}

type Bool struct { b bool; Null }
func (j *Bool) Kind() int { return BoolKind }
func (j *Bool) Bool() bool { return j.b }
func (j *Bool) String() string {
	if j.b {
		return "true"
	}
	return "false"
}

type Map struct { m *map[string]Json; Null }
func (j *Map) Kind() int { return MapKind }
func (j *Map) Get(s string) Json {
	if j.m == nil {
		return null
	}
	v, ok := j.m[s];
	if !ok {
		return null
	}
	return v;
}
func (j *Map) String() string {
	s := "{";
	first := true;
	for k,v range j.m {
		if first {
			first = false;
		} else {
			s += ",";
		}
		s += Quote(k);
		s += ":";
		s += JsonToString(v);
	}
	s += "}";
	return s;
}

export func Walk(j Json, path string) Json {
	for len(path) > 0 {
		var elem string;
		if i := strings.index(path, '/'); i >= 0 {
			elem = path[0:i];
			path = path[i+1:len(path)];
		} else {
			elem = path;
			path = "";
		}
		switch j.Kind() {
		case ArrayKind:
			indx, err := strconv.atoi(elem);
			if err != nil {
				return null
			}
			j = j.Elem(indx);
		case MapKind:
			j = j.Get(elem);
		default:
			return null
		}
	}
	return j
}

export func Equal(a, b Json) bool {
	switch {
	case a == nil && b == nil:
		return true;
	case a == nil || b == nil:
		return false;
	case a.Kind() != b.Kind():
		return false;
	}

	switch a.Kind() {
	case NullKind:
		return true;
	case StringKind:
		return a.String() == b.String();
	case NumberKind:
		return a.Number() == b.Number();
	case BoolKind:
		return a.Bool() == b.Bool();
	case ArrayKind:
		if a.Len() != b.Len() {
			return false;
		}
		for i := 0; i < a.Len(); i++ {
			if !Equal(a.Elem(i), b.Elem(i)) {
				return false;
			}
		}
		return true;
	case MapKind:
		m := a.(*Map).m;
		if len(m) != len(b.(*Map).m) {
			return false;
		}
		for k,v range m {
			if !Equal(v, b.Get(k)) {
				return false;
			}
		}
		return true;
	}

	// invalid kind
	return false;
}

// Parse builder for Json objects.

type JsonBuilder struct {
	// either writing to *ptr
	ptr *Json;

	// or to a[i] (can't set ptr = &a[i])
	a *array.Array;
	i int;

	// or to m[k] (can't set ptr = &m[k])
	m *map[string] Json;
	k string;
}

func (b *JsonBuilder) Put(j Json) {
	switch {
	case b.ptr != nil:
		*b.ptr = j;
	case b.a != nil:
		b.a.Set(b.i, j);
	case b.m != nil:
		b.m[b.k] = j;
	}
}

func (b *JsonBuilder) Get() Json {
	switch {
	case b.ptr != nil:
		return *b.ptr;
	case b.a != nil:
		return b.a.At(b.i);
	case b.m != nil:
		return b.m[b.k];
	}
	return nil
}

func (b *JsonBuilder) Float64(f float64) {
	b.Put(&Number{f, Null{}})
}

func (b *JsonBuilder) Int64(i int64) {
	b.Float64(float64(i))
}

func (b *JsonBuilder) Uint64(i uint64) {
	b.Float64(float64(i))
}

func (b *JsonBuilder) Bool(tf bool) {
	b.Put(&Bool{tf, Null{}})
}

func (b *JsonBuilder) Null() {
	b.Put(null)
}

func (b *JsonBuilder) String(s string) {
	b.Put(&String{s, Null{}})
}

func (b *JsonBuilder) Array() {
	b.Put(&Array{array.New(0), Null{}})
}

func (b *JsonBuilder) Map() {
	b.Put(&Map{new(map[string]Json), Null{}})
}

func (b *JsonBuilder) Elem(i int) Builder {
	bb := new(JsonBuilder);
	bb.a = b.Get().(*Array).a;
	bb.i = i;
	for i >= bb.a.Len() {
		bb.a.Push(null)
	}
	return bb
}

func (b *JsonBuilder) Key(k string) Builder {
	bb := new(JsonBuilder);
	bb.m = b.Get().(*Map).m;
	bb.k = k;
	bb.m[k] = null;
	return bb
}

export func StringToJson(s string) (json Json, ok bool, errtok string) {
	var errindx int;
	var j Json;
	b := new(JsonBuilder);
	b.ptr = &j;
	ok, errindx, errtok = Parse(s, b);
	if !ok {
		return nil, false, errtok
	}
	return j, true, ""
}
<commit_msg>remove implicit int -> string<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Generic JSON representation.

package json

import (
	"array";
	"fmt";
	"math";
	"json";
	"strconv";
	"strings";
)

export const (
	StringKind = iota;
	NumberKind;
	MapKind;		// JSON term is "Object", but in Go, it's a map
	ArrayKind;
	BoolKind;
	NullKind;
)

export type Json interface {
	Kind() int;
	String() string;
	Number() float64;
	Bool() bool;
	Get(s string) Json;
	Elem(i int) Json;
	Len() int;
}

export func JsonToString(j Json) string {
	if j == nil {
		return "null"
	}
	if j.Kind() == StringKind {
		return Quote(j.String())
	}
	return j.String()
}

type Null struct { }
export var null Json = &Null{}
func (*Null) Kind() int { return NullKind }
func (*Null) String() string { return "null" }
func (*Null) Number() float64 { return 0 }
func (*Null) Bool() bool { return false }
func (*Null) Get(s string) Json { return null }
func (*Null) Elem(int) Json { return null }
func (*Null) Len() int { return 0 }

type String struct { s string; Null }
func (j *String) Kind() int { return StringKind }
func (j *String) String() string { return j.s }

type Number struct { f float64; Null }
func (j *Number) Kind() int { return NumberKind }
func (j *Number) Number() float64 { return j.f }
func (j *Number) String() string {
	if math.Floor(j.f) == j.f {
		return fmt.sprintf("%.0f", j.f);
	}
	return fmt.sprintf("%g", j.f);
}

type Array struct { a *array.Array; Null }
func (j *Array) Kind() int { return ArrayKind }
func (j *Array) Len() int { return j.a.Len() }
func (j *Array) Elem(i int) Json {
	if i < 0 || i >= j.a.Len() {
		return null
	}
	return j.a.At(i)
}
func (j *Array) String() string {
	s := "[";
	for i := 0; i < j.a.Len(); i++ {
		if i > 0 {
			s += ",";
		}
		s += JsonToString(j.a.At(i).(Json));
	}
	s += "]";
	return s;
}

type Bool struct { b bool; Null }
func (j *Bool) Kind() int { return BoolKind }
func (j *Bool) Bool() bool { return j.b }
func (j *Bool) String() string {
	if j.b {
		return "true"
	}
	return "false"
}

type Map struct { m *map[string]Json; Null }
func (j *Map) Kind() int { return MapKind }
func (j *Map) Get(s string) Json {
	if j.m == nil {
		return null
	}
	v, ok := j.m[s];
	if !ok {
		return null
	}
	return v;
}
func (j *Map) String() string {
	s := "{";
	first := true;
	for k,v range j.m {
		if first {
			first = false;
		} else {
			s += ",";
		}
		s += Quote(k);
		s += ":";
		s += JsonToString(v);
	}
	s += "}";
	return s;
}

export func Walk(j Json, path string) Json {
	for len(path) > 0 {
		var elem string;
		if i := strings.index(path, "/"); i >= 0 {
			elem = path[0:i];
			path = path[i+1:len(path)];
		} else {
			elem = path;
			path = "";
		}
		switch j.Kind() {
		case ArrayKind:
			indx, err := strconv.atoi(elem);
			if err != nil {
				return null
			}
			j = j.Elem(indx);
		case MapKind:
			j = j.Get(elem);
		default:
			return null
		}
	}
	return j
}

export func Equal(a, b Json) bool {
	switch {
	case a == nil && b == nil:
		return true;
	case a == nil || b == nil:
		return false;
	case a.Kind() != b.Kind():
		return false;
	}

	switch a.Kind() {
	case NullKind:
		return true;
	case StringKind:
		return a.String() == b.String();
	case NumberKind:
		return a.Number() == b.Number();
	case BoolKind:
		return a.Bool() == b.Bool();
	case ArrayKind:
		if a.Len() != b.Len() {
			return false;
		}
		for i := 0; i < a.Len(); i++ {
			if !Equal(a.Elem(i), b.Elem(i)) {
				return false;
			}
		}
		return true;
	case MapKind:
		m := a.(*Map).m;
		if len(m) != len(b.(*Map).m) {
			return false;
		}
		for k,v range m {
			if !Equal(v, b.Get(k)) {
				return false;
			}
		}
		return true;
	}

	// invalid kind
	return false;
}

// Parse builder for Json objects.

type JsonBuilder struct {
	// either writing to *ptr
	ptr *Json;

	// or to a[i] (can't set ptr = &a[i])
	a *array.Array;
	i int;

	// or to m[k] (can't set ptr = &m[k])
	m *map[string] Json;
	k string;
}

func (b *JsonBuilder) Put(j Json) {
	switch {
	case b.ptr != nil:
		*b.ptr = j;
	case b.a != nil:
		b.a.Set(b.i, j);
	case b.m != nil:
		b.m[b.k] = j;
	}
}

func (b *JsonBuilder) Get() Json {
	switch {
	case b.ptr != nil:
		return *b.ptr;
	case b.a != nil:
		return b.a.At(b.i);
	case b.m != nil:
		return b.m[b.k];
	}
	return nil
}

func (b *JsonBuilder) Float64(f float64) {
	b.Put(&Number{f, Null{}})
}

func (b *JsonBuilder) Int64(i int64) {
	b.Float64(float64(i))
}

func (b *JsonBuilder) Uint64(i uint64) {
	b.Float64(float64(i))
}

func (b *JsonBuilder) Bool(tf bool) {
	b.Put(&Bool{tf, Null{}})
}

func (b *JsonBuilder) Null() {
	b.Put(null)
}

func (b *JsonBuilder) String(s string) {
	b.Put(&String{s, Null{}})
}

func (b *JsonBuilder) Array() {
	b.Put(&Array{array.New(0), Null{}})
}

func (b *JsonBuilder) Map() {
	b.Put(&Map{new(map[string]Json), Null{}})
}

func (b *JsonBuilder) Elem(i int) Builder {
	bb := new(JsonBuilder);
	bb.a = b.Get().(*Array).a;
	bb.i = i;
	for i >= bb.a.Len() {
		bb.a.Push(null)
	}
	return bb
}

func (b *JsonBuilder) Key(k string) Builder {
	bb := new(JsonBuilder);
	bb.m = b.Get().(*Map).m;
	bb.k = k;
	bb.m[k] = null;
	return bb
}

export func StringToJson(s string) (json Json, ok bool, errtok string) {
	var errindx int;
	var j Json;
	b := new(JsonBuilder);
	b.ptr = &j;
	ok, errindx, errtok = Parse(s, b);
	if !ok {
		return nil, false, errtok
	}
	return j, true, ""
}
<|endoftext|>


<commit_before>package logBackup

import (
	"net"
	"fmt"
	"time"
	"errors"
	"bufio"
	"os"
	"path/filepath"
	"strings"
	"strconv"
	"io"
	"path"
	"regexp"
	"crypto/md5"
	"encoding/hex"
)

type Server struct {
	addr       string
	socket     *net.TCPListener
	cm         *ConnManager
}

func NewServer() (*Server, error) {
	Logger.Printf("server will running at %s", GloablConfig.Addr)
	return &Server{
		addr       : GloablConfig.Addr,
		socket     : nil,
		cm         : NewConnManager(),
	}, nil
}

func (srv *Server) Start() error {
	addr, err := net.ResolveTCPAddr("tcp", srv.addr)
	if err != nil {
		return fmt.Errorf("fail to resolve addr: %v", err)
	}

	sock, err := net.ListenTCP("tcp", addr)
	if err != nil {
		Logger.Printf("server run %s failed %s", srv.addr, err)
		return fmt.Errorf("fail to listen tcp: %v", err)
	}

	srv.socket = sock

	return srv.acceptConn()
}

func (srv *Server) Stop() error {
	srv.socket.SetDeadline(time.Now())

	tt := time.NewTimer(time.Second * time.Duration(30))
	wait := make(chan struct{})
	go func() {
		srv.cm.Wait()
		wait <- struct{}{}
	}()

	select {
	case <-tt.C:
		return errors.New("stop server wait timeout")
	case <-wait:
		return nil
	}
}

func (srv *Server)acceptConn() error {
	defer func() {
		srv.socket.Close()
	}()

	for {
		conn, err := srv.socket.Accept()
		if err != nil {
			if nerr, ok := err.(net.Error); ok && nerr.Timeout() {
				break;
			}
			continue
		}

		go func(conn net.Conn) {
			srv.cm.Add(1)
			srv.handleConn(&FConn{conn:conn})
			srv.cm.Done()
		}(conn)
	}

	return nil
}

func (srv *Server) handleConn(conn *FConn) {
	var clientAddr = conn.ClientAddress()
	Logger.Printf("%s connected", clientAddr)

	defer func() {
		Logger.Printf("%s disconnected", clientAddr)
		err := conn.Close()
		if err != nil {
			Logger.Printf("%s disconnected error %s", clientAddr, err)
		}
	}()

	data := make([]byte, 1024)
	r := bufio.NewReader(conn)

	//file size path\r\n
	//data
	for {
		conn.SetReadDeadline(time.Now().Add(time.Second * time.Duration(30)))

		content, err := r.ReadString('\n')
		if err != nil {
			if err != io.EOF {
				Logger.Printf("%s parse transfer header failed %v", clientAddr, err)
				conn.Write([]byte("ERROR\r\n"))
			}
			break
		}

		//ping     PING@
		//transfer FILE_NAME@FILE_SIZE@md5sum@PATH
		summaryInfo := strings.Split(strings.Trim(content, "\r\n"), "@")
		summaryLen := len(summaryInfo)
		if summaryLen < 4 {
			if summaryInfo[0] == "PING" {
				conn.Write([]byte("PONG\r\n"))
				continue
			} else {
				Logger.Printf("%s parse transfer header failed %s", clientAddr, content)
				conn.Write([]byte("parse transfer header failed\r\n"))
				break
			}
		}

		//文件相对路径
		fpath := summaryInfo[summaryLen-1]
		if len(fpath) > 0 {
			if matched, err := regexp.Match(`^[0-9a-zA-Z\-_/]{1,}$`, []byte(fpath)); err != nil || !matched {
				Logger.Printf("Sorry, transfer file path is invalid\n")
				conn.Write([]byte("ERROR\r\n"))
				break
			}
		}

		//文件md5校验码
		fsum := summaryInfo[summaryLen-2]

		//文件大小
		fsize := 0
		if len(summaryInfo[summaryLen-3]) > 0 {
			num, err := strconv.Atoi(summaryInfo[summaryLen-3])
			if err != nil {
				Logger.Printf("%s parse transfer header failed %s %s", clientAddr, content, err)
				conn.Write([]byte("parse transfer header failed error size\r\n"))
				break
			}
			fsize = num
		}

		if fsize == 0 {
			Logger.Printf("%s parse transfer header failed %s error size", clientAddr, content)
			conn.Write([]byte("parse transfer header failed error size\r\n"))
			break
		}

		//避免文件名包含@
		fname := strings.Trim(strings.Join(summaryInfo[:summaryLen-3], "@"), "@")

		//文件保存路径
		fileName := path.Join(GloablConfig.BackupPath, fpath, fname)

		Logger.Printf("%s backup file %s size %d md5sum %s", clientAddr, fileName, fsize, fsum)

		parentDir := filepath.Dir(fileName)
		if _, err := os.Stat(parentDir); os.IsNotExist(err) {
			if err := os.MkdirAll(parentDir, GloablConfig.Perm); err != nil {
				Logger.Printf("%s create folder %s failed %v", clientAddr, parentDir, err)
				conn.Write([]byte("ERROR\r\n"))
				break
			}
		}

		fi, err := os.Stat(fileName)
		if err == nil && fi.Size() > 0 {
			t, err := os.Open(fileName)
			if err == nil {
				h := md5.New()
				if _, err := io.Copy(h, t); err != nil {
					t.Close()
					Logger.Printf("Get file %s md5sum failed %v", fileName, err)
					conn.Write([]byte("ERROR\r\n"))
					break
				}

				ms := hex.EncodeToString(h.Sum(nil))
				Logger.Printf("Get file %s md5sum %s", fileName, ms)

				t.Close()
				if ms == fsum {
					Logger.Printf("Transfer file %s has same md5sum %s", fileName, ms)
					conn.Write([]byte("ALL_SAME\r\n"))
					continue;
				}
			} else {
				Logger.Printf("Stat file %s failed %s", fileName, err)
			}
		} else {
			conn.Write([]byte("CONTINUE\r\n"))
		}

		tmpFileName := fmt.Sprintf("%s.tmp", fileName)
		f, err := os.OpenFile(tmpFileName, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, GloablConfig.Perm)
		if err != nil {
			Logger.Printf("%s open file %s failed %v", clientAddr, tmpFileName, err)
			conn.Write([]byte("ERROR\r\n"))
			break
		}

		h := md5.New()
		m := 0
		for {
			conn.SetReadDeadline(time.Now().Add(time.Second * time.Duration(180)))
			n, err := r.Read(data)
			if err != nil {
				break;
			}

			h.Write(data[:n])
			f.Write(data[:n])
			m += n
			if m >= fsize {
				break
			}
		}

		ms := hex.EncodeToString(h.Sum(nil))

		f.Close()

		err = os.Rename(tmpFileName, fileName)
		if err != nil {
			Logger.Printf("Rename file %s failed %v", tmpFileName, err)
			conn.Write([]byte("ERROR\r\n"))
			break
		}

		if GloablConfig.ToChown {
			go func() {
				ChownR(GloablConfig.BackupPath, GloablConfig.Uid, GloablConfig.Gid)
			}()
		}

		if m == fsize && ms == fsum {
			Logger.Printf("Transfer file %s success md5sum %s size %d", fileName, ms, m)
			conn.Write([]byte("OK\r\n"))
		} else {
			Logger.Printf("Transfer file %s failed md5sum %s size %d", fileName, ms, m)
			conn.Write([]byte("ERROR\r\n"))
			break
		}
	}
}<commit_msg>fixed bug<commit_after>package logBackup

import (
	"net"
	"fmt"
	"time"
	"errors"
	"bufio"
	"os"
	"path/filepath"
	"strings"
	"strconv"
	"io"
	"path"
	"regexp"
	"crypto/md5"
	"encoding/hex"
)

type Server struct {
	addr       string
	socket     *net.TCPListener
	cm         *ConnManager
}

func NewServer() (*Server, error) {
	Logger.Printf("server will running at %s", GloablConfig.Addr)
	return &Server{
		addr       : GloablConfig.Addr,
		socket     : nil,
		cm         : NewConnManager(),
	}, nil
}

func (srv *Server) Start() error {
	addr, err := net.ResolveTCPAddr("tcp", srv.addr)
	if err != nil {
		return fmt.Errorf("fail to resolve addr: %v", err)
	}

	sock, err := net.ListenTCP("tcp", addr)
	if err != nil {
		Logger.Printf("server run %s failed %s", srv.addr, err)
		return fmt.Errorf("fail to listen tcp: %v", err)
	}

	srv.socket = sock

	return srv.acceptConn()
}

func (srv *Server) Stop() error {
	srv.socket.SetDeadline(time.Now())

	tt := time.NewTimer(time.Second * time.Duration(30))
	wait := make(chan struct{})
	go func() {
		srv.cm.Wait()
		wait <- struct{}{}
	}()

	select {
	case <-tt.C:
		return errors.New("stop server wait timeout")
	case <-wait:
		return nil
	}
}

func (srv *Server)acceptConn() error {
	defer func() {
		srv.socket.Close()
	}()

	for {
		conn, err := srv.socket.Accept()
		if err != nil {
			if nerr, ok := err.(net.Error); ok && nerr.Timeout() {
				break;
			}
			continue
		}

		go func(conn net.Conn) {
			srv.cm.Add(1)
			srv.handleConn(&FConn{conn:conn})
			srv.cm.Done()
		}(conn)
	}

	return nil
}

func (srv *Server) handleConn(conn *FConn) {
	var clientAddr = conn.ClientAddress()
	Logger.Printf("%s connected", clientAddr)

	defer func() {
		Logger.Printf("%s disconnected", clientAddr)
		err := conn.Close()
		if err != nil {
			Logger.Printf("%s disconnected error %s", clientAddr, err)
		}
	}()

	data := make([]byte, 1024)
	r := bufio.NewReader(conn)

	//file size path\r\n
	//data
	for {
		conn.SetReadDeadline(time.Now().Add(time.Second * time.Duration(30)))

		content, err := r.ReadString('\n')
		if err != nil {
			if err != io.EOF {
				Logger.Printf("%s parse transfer header failed %v", clientAddr, err)
				conn.Write([]byte("ERROR\r\n"))
			}
			break
		}

		//ping     PING@
		//transfer FILE_NAME@FILE_SIZE@md5sum@PATH
		summaryInfo := strings.Split(strings.Trim(content, "\r\n"), "@")
		summaryLen := len(summaryInfo)
		if summaryLen < 4 {
			if summaryInfo[0] == "PING" {
				conn.Write([]byte("PONG\r\n"))
				continue
			} else {
				Logger.Printf("%s parse transfer header failed %s", clientAddr, content)
				conn.Write([]byte("parse transfer header failed\r\n"))
				break
			}
		}

		//文件相对路径
		fpath := summaryInfo[summaryLen-1]
		if len(fpath) > 0 {
			if matched, err := regexp.Match(`^[0-9a-zA-Z\-_/]{1,}$`, []byte(fpath)); err != nil || !matched {
				Logger.Printf("Sorry, transfer file path is invalid\n")
				conn.Write([]byte("ERROR\r\n"))
				break
			}
		}

		//文件md5校验码
		fsum := summaryInfo[summaryLen-2]

		//文件大小
		fsize := 0
		if len(summaryInfo[summaryLen-3]) > 0 {
			num, err := strconv.Atoi(summaryInfo[summaryLen-3])
			if err != nil {
				Logger.Printf("%s parse transfer header failed %s %s", clientAddr, content, err)
				conn.Write([]byte("parse transfer header failed error size\r\n"))
				break
			}
			fsize = num
		}

		if fsize == 0 {
			Logger.Printf("%s parse transfer header failed %s error size", clientAddr, content)
			conn.Write([]byte("parse transfer header failed error size\r\n"))
			break
		}

		//避免文件名包含@
		fname := strings.Trim(strings.Join(summaryInfo[:summaryLen-3], "@"), "@")

		//文件保存路径
		fileName := path.Join(GloablConfig.BackupPath, fpath, fname)

		Logger.Printf("%s backup file %s size %d md5sum %s", clientAddr, fileName, fsize, fsum)

		parentDir := filepath.Dir(fileName)
		if _, err := os.Stat(parentDir); os.IsNotExist(err) {
			if err := os.MkdirAll(parentDir, GloablConfig.Perm); err != nil {
				Logger.Printf("%s create folder %s failed %v", clientAddr, parentDir, err)
				conn.Write([]byte("ERROR\r\n"))
				break
			}
		}

		fi, err := os.Stat(fileName)
		if err == nil && fi.Size() > 0 {
			t, err := os.Open(fileName)
			if err == nil {
				h := md5.New()
				if _, err := io.Copy(h, t); err != nil {
					t.Close()
					Logger.Printf("Get file %s md5sum failed %v", fileName, err)
					conn.Write([]byte("ERROR\r\n"))
					break
				}

				ms := hex.EncodeToString(h.Sum(nil))
				Logger.Printf("Get file %s md5sum %s", fileName, ms)

				t.Close()
				if ms == fsum {
					Logger.Printf("Transfer file %s has same md5sum %s", fileName, ms)
					conn.Write([]byte("ALL_SAME\r\n"))
					continue;
				} else {
					Logger.Printf("Stat file %s md5sum not same and will transfer file data", fileName)
				}
			} else {
				Logger.Printf("Stat file %s failed %v and will transfer file data", fileName, err)
			}
		} else {
			Logger.Printf("Stat file %s failed %v and will transfer file data", fileName, err)
		}

		conn.Write([]byte("CONTINUE\r\n"))

		tmpFileName := fmt.Sprintf("%s.tmp", fileName)
		f, err := os.OpenFile(tmpFileName, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, GloablConfig.Perm)
		if err != nil {
			Logger.Printf("%s open file %s failed %v", clientAddr, tmpFileName, err)
			conn.Write([]byte("ERROR\r\n"))
			break
		}

		h := md5.New()
		m := 0
		for {
			conn.SetReadDeadline(time.Now().Add(time.Second * time.Duration(180)))
			n, err := r.Read(data)
			if err != nil {
				break;
			}

			h.Write(data[:n])
			f.Write(data[:n])
			m += n
			if m >= fsize {
				break
			}
		}

		ms := hex.EncodeToString(h.Sum(nil))

		f.Close()

		err = os.Rename(tmpFileName, fileName)
		if err != nil {
			Logger.Printf("Rename file %s failed %v", tmpFileName, err)
			conn.Write([]byte("ERROR\r\n"))
			break
		}

		if GloablConfig.ToChown {
			go func() {
				ChownR(GloablConfig.BackupPath, GloablConfig.Uid, GloablConfig.Gid)
			}()
		}

		if m == fsize && ms == fsum {
			Logger.Printf("Transfer file %s success md5sum %s size %d", fileName, ms, m)
			conn.Write([]byte("OK\r\n"))
		} else {
			Logger.Printf("Transfer file %s failed md5sum %s size %d", fileName, ms, m)
			conn.Write([]byte("ERROR\r\n"))
			break
		}
	}
}<|endoftext|>


<commit_before>// Package five00px provides ...
package five00px

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"sort"

	"github.com/Sirupsen/logrus"
)

type httpError struct {
	ErrorCode int
	Response  string
}

func (h httpError) Error() string {
	return fmt.Sprintf("HTTP error %d. Message: %s", h.ErrorCode, h.Response)
}

func buildQuery(v url.Values) string {
	if v == nil {
		return ""
	}
	var buf bytes.Buffer
	_, _ := buf.WriteString("?")
	keys := make([]string, 0, len(v))
	for k := range v {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		vs := v[k]
		prefix := k + "="
		for _, v := range vs {
			if buf.Len() > 1 {
				buf.WriteByte('&')
			}
			buf.WriteString(urlEncode(prefix))
			buf.WriteString(urlEncode(v))
		}
	}
	return buf.String()
}

func doCommand(c *http.Client, dstPoint, method string, vals url.Values) ([]byte, error) {

	dstPoint += buildQuery(vals)

	log := logrus.WithFields(logrus.Fields{
		"context": "HTTP " + method,
		"host":    mainAPIUrl,
		"path":    dstPoint,
		"values":  vals,
	})

	log.Info("Initiating request")

	req, err := http.NewRequest(method, mainAPIUrl+dstPoint, nil)
	r, err := c.Do(req)
	defer func() {
		_ = r.Body.Close()
	}()
	if err != nil {
		log.WithError(err).Error("Request failed")
		return nil, err
	} else if r.StatusCode != 200 {
		b, err := ioutil.ReadAll(r.Body)
		d := ""
		if err == nil {
			d = string(b)
		}

		log.WithFields(logrus.Fields{
			"StatusCode": r.StatusCode,
			"data":       d,
		}).Warn("Server returns error")
		return b, httpError{r.StatusCode, d}
	}

	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.WithError(err).Error("Reading error")
		return b, err
	}
	log.Info("Done")
	return b, nil
}
<commit_msg>compilation fix<commit_after>// Package five00px provides ...
package five00px

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"sort"

	"github.com/Sirupsen/logrus"
)

type httpError struct {
	ErrorCode int
	Response  string
}

func (h httpError) Error() string {
	return fmt.Sprintf("HTTP error %d. Message: %s", h.ErrorCode, h.Response)
}

func buildQuery(v url.Values) string {
	if v == nil {
		return ""
	}
	var buf bytes.Buffer
	_, _ = buf.WriteString("?")
	keys := make([]string, 0, len(v))
	for k := range v {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		vs := v[k]
		prefix := k + "="
		for _, v := range vs {
			if buf.Len() > 1 {
				buf.WriteByte('&')
			}
			buf.WriteString(urlEncode(prefix))
			buf.WriteString(urlEncode(v))
		}
	}
	return buf.String()
}

func doCommand(c *http.Client, dstPoint, method string, vals url.Values) ([]byte, error) {

	dstPoint += buildQuery(vals)

	log := logrus.WithFields(logrus.Fields{
		"context": "HTTP " + method,
		"host":    mainAPIUrl,
		"path":    dstPoint,
		"values":  vals,
	})

	log.Info("Initiating request")

	req, err := http.NewRequest(method, mainAPIUrl+dstPoint, nil)
	r, err := c.Do(req)
	defer func() {
		_ = r.Body.Close()
	}()
	if err != nil {
		log.WithError(err).Error("Request failed")
		return nil, err
	} else if r.StatusCode != 200 {
		b, err := ioutil.ReadAll(r.Body)
		d := ""
		if err == nil {
			d = string(b)
		}

		log.WithFields(logrus.Fields{
			"StatusCode": r.StatusCode,
			"data":       d,
		}).Warn("Server returns error")
		return b, httpError{r.StatusCode, d}
	}

	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.WithError(err).Error("Reading error")
		return b, err
	}
	log.Info("Done")
	return b, nil
}
<|endoftext|>


<commit_before>// Copyright 2020 The Moov Authors
// Use of this source code is governed by an Apache License
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"os"
	"text/tabwriter"

	"github.com/moov-io/ach"
)

func dumpFiles(paths []string) error {
	for i := range paths {
		if i > 0 {
			fmt.Println("") // extra newline between multiple ACH files
		}
		fmt.Printf("Describing ACH file '%s'\n\n", paths[i])
		file, err := readACHFile(paths[i])
		if err != nil {
			fmt.Printf("WARN: problem reading %s:\n %v\n\n", paths[i], err)
		}
		dumpFile(file)
	}
	return nil
}

func dumpFile(file *ach.File) {
	if file == nil {
		return
	}

	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
	defer w.Flush()

	fh, fc := file.Header, file.Control

	// FileHeader
	fmt.Fprintln(w, "  Origin\tOriginName\tDestination\tDestinationName\tFileCreationDate\tFileCreationTime")
	fmt.Fprintln(w, fmt.Sprintf("  %s\t%s\t%s\t%s\t%s\t%s", fh.ImmediateOrigin, fh.ImmediateOriginName, fh.ImmediateDestination, fh.ImmediateDestinationName, fh.FileCreationDate, fh.FileCreationTime))

	// FileControl
	fmt.Fprintln(w, "\n  BatchCount\tBlockCount\tEntryAddendaCount\tTotalDebitAmount\tTotalCreditAmount")
	fmt.Fprintln(w, fmt.Sprintf("  %d\t%d\t%d\t%d\t%d", fc.BatchCount, fc.BlockCount, fc.EntryAddendaCount, fc.TotalDebitEntryDollarAmountInFile, fc.TotalCreditEntryDollarAmountInFile))

	// Batches
	for i := range file.Batches {
		fmt.Fprintln(w, "\n  BatchNumber\tType\tServiceClass\tAddendas\tTotalDebits\tTotalCredits")

		bh := file.Batches[i].GetHeader()
		bc := file.Batches[i].GetControl()
		fmt.Fprintln(w, fmt.Sprintf("  %d\t%s\t%d\t%d\t%d\t%d", bh.BatchNumber, bh.StandardEntryClassCode, bh.ServiceClassCode, bc.EntryAddendaCount, bc.TotalDebitEntryDollarAmount, bc.TotalCreditEntryDollarAmount))

		entries := file.Batches[i].GetEntries()
		if len(entries) > 0 {
			fmt.Fprintln(w, "\n    TxCode\tAccountNumber\tAmount\tName\tTraceNumber")
		}
		for j := range entries {
			e := entries[j]
			fmt.Fprintln(w, fmt.Sprintf("    %d\t%s\t%d\t%s\t%s", e.TransactionCode, e.DFIAccountNumber, e.Amount, e.IndividualName, e.TraceNumber))
		}
	}
	// TODO(adam): Do different stuff with -v enabeld
}

func readACHFile(path string) (*ach.File, error) {
	fd, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("problem opening %s: %v", path, err)
	}
	defer fd.Close()

	f, err := ach.NewReader(fd).Read()
	return &f, err
}
<commit_msg>cmd/ach: linter fixup<commit_after>// Copyright 2020 The Moov Authors
// Use of this source code is governed by an Apache License
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"os"
	"text/tabwriter"

	"github.com/moov-io/ach"
)

func dumpFiles(paths []string) error {
	for i := range paths {
		if i > 0 {
			fmt.Println("") // extra newline between multiple ACH files
		}
		fmt.Printf("Describing ACH file '%s'\n\n", paths[i])
		file, err := readACHFile(paths[i])
		if err != nil {
			fmt.Printf("WARN: problem reading %s:\n %v\n\n", paths[i], err)
		}
		dumpFile(file)
	}
	return nil
}

func dumpFile(file *ach.File) {
	if file == nil {
		return
	}

	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
	defer w.Flush()

	fh, fc := file.Header, file.Control

	// FileHeader
	fmt.Fprintln(w, "  Origin\tOriginName\tDestination\tDestinationName\tFileCreationDate\tFileCreationTime")
	fmt.Fprintf(w, "  %s\t%s\t%s\t%s\t%s\t%s\n", fh.ImmediateOrigin, fh.ImmediateOriginName, fh.ImmediateDestination, fh.ImmediateDestinationName, fh.FileCreationDate, fh.FileCreationTime)

	// FileControl
	fmt.Fprintln(w, "\n  BatchCount\tBlockCount\tEntryAddendaCount\tTotalDebitAmount\tTotalCreditAmount")
	fmt.Fprintf(w, "  %d\t%d\t%d\t%d\t%d\n", fc.BatchCount, fc.BlockCount, fc.EntryAddendaCount, fc.TotalDebitEntryDollarAmountInFile, fc.TotalCreditEntryDollarAmountInFile)

	// Batches
	for i := range file.Batches {
		fmt.Fprintln(w, "\n  BatchNumber\tType\tServiceClass\tAddendas\tTotalDebits\tTotalCredits")

		bh := file.Batches[i].GetHeader()
		bc := file.Batches[i].GetControl()
		fmt.Fprintf(w, "  %d\t%s\t%d\t%d\t%d\t%d\n", bh.BatchNumber, bh.StandardEntryClassCode, bh.ServiceClassCode, bc.EntryAddendaCount, bc.TotalDebitEntryDollarAmount, bc.TotalCreditEntryDollarAmount)

		entries := file.Batches[i].GetEntries()
		if len(entries) > 0 {
			fmt.Fprintln(w, "\n    TxCode\tAccountNumber\tAmount\tName\tTraceNumber")
		}
		for j := range entries {
			e := entries[j]
			fmt.Fprintf(w, "    %d\t%s\t%d\t%s\t%s\n", e.TransactionCode, e.DFIAccountNumber, e.Amount, e.IndividualName, e.TraceNumber)
		}
	}
	// TODO(adam): Do different stuff with -v enabeld
}

func readACHFile(path string) (*ach.File, error) {
	fd, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("problem opening %s: %v", path, err)
	}
	defer fd.Close()

	f, err := ach.NewReader(fd).Read()
	return &f, err
}
<|endoftext|>


<commit_before>// gb-list lists the packages named by the import paths, one per line.
//
//     usage: gb list [-s] [-f format] [-json] [packages]
//
// The default output shows the package import path:
//
//     % gb list github.com/constabulary/...
//     github.com/constabulary/gb
//     github.com/constabulary/gb/cmd
//     github.com/constabulary/gb/cmd/gb
//     github.com/constabulary/gb/cmd/gb-env
//     github.com/constabulary/gb/cmd/gb-list
//
// The -f flag specifies an alternate format for the list, using the
// syntax of package template.  The default output is equivalent to -f
// '{{.ImportPath}}'. The struct being passed to the template is currently
// an instance of gb.Package. This structure is under active development
// and it's contents are not guarenteed to be stable.
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"text/template"

	"go/build"

	"github.com/constabulary/gb"
	"github.com/constabulary/gb/cmd"
)

func main() {
	var (
		projectroot string
		format      string
		formatStdin bool
		jsonOutput  bool
	)
	flag.StringVar(&projectroot, "R", os.Getenv("GB_PROJECT_DIR"), "set the project root")
	flag.StringVar(&format, "f", "{{.ImportPath}}", "format template")
	flag.BoolVar(&formatStdin, "s", false, "read format from stdin")
	flag.BoolVar(&gb.Verbose, "v", gb.Verbose, "enable log levels below INFO level")
	flag.BoolVar(&jsonOutput, "json", false, "outputs json")

	flag.Parse()

	if formatStdin {
		var formatBuffer bytes.Buffer
		io.Copy(&formatBuffer, os.Stdin)
		format = formatBuffer.String()
	}

	gopath := filepath.SplitList(os.Getenv("GOPATH"))
	root, err := cmd.FindProjectroot(projectroot, gopath)
	if err != nil {
		gb.Fatalf("could not locate project root: %v", err)
	}
	project := gb.NewProject(root)

	ctx, err := project.NewContext(
		gb.GcToolchain(),
	)
	if err != nil {
		gb.Fatalf("unable to construct context: %v", err)
	}

	args := cmd.ImportPaths(ctx, projectroot, flag.Args())
	pkgs, err := resolvePackages(ctx, projectroot, args...)
	if err != nil {
		gb.Fatalf("unable to resolve: %v", err)
	}

	if jsonOutput {
		views := make([]*PackageView, 0, len(pkgs))
		for _, pkg := range pkgs {
			views = append(views, NewPackageView(pkg))
		}
		encoded, err := json.MarshalIndent(views, " ", "  ")
		if err != nil {
			gb.Fatalf("Error occurred during json encoding: %v", err)
		}
		fmt.Println(string(encoded))
	} else {
		tmpl, err := template.New("list").Parse(format)
		if err != nil {
			gb.Fatalf("unable to parse template %q: %v", format, err)
		}

		for _, pkg := range pkgs {
			if err := tmpl.Execute(os.Stdout, pkg); err != nil {
				gb.Fatalf("unable to execute template: %v", err)
			}
			fmt.Fprintln(os.Stdout)
		}
	}
}

func resolvePackages(ctx *gb.Context, projectroot string, args ...string) ([]*gb.Package, error) {
	var pkgs []*gb.Package
	for _, arg := range args {
		if arg == "." {
			var err error
			arg, err = filepath.Rel(ctx.Srcdirs()[0], projectroot)
			if err != nil {
				return pkgs, err
			}
		}
		pkg, err := ctx.ResolvePackage(arg)
		if err != nil {
			if _, ok := err.(*build.NoGoError); ok {
				gb.Debugf("skipping %q", arg)
				continue
			}
			return pkgs, fmt.Errorf("failed to resolve package %q: %v", arg, err)
		}
		pkgs = append(pkgs, pkg)
	}
	return pkgs, nil
}
<commit_msg>cmd/gb-list: use cmd.ResolvePackages<commit_after>// gb-list lists the packages named by the import paths, one per line.
//
//     usage: gb list [-s] [-f format] [-json] [packages]
//
// The default output shows the package import path:
//
//     % gb list github.com/constabulary/...
//     github.com/constabulary/gb
//     github.com/constabulary/gb/cmd
//     github.com/constabulary/gb/cmd/gb
//     github.com/constabulary/gb/cmd/gb-env
//     github.com/constabulary/gb/cmd/gb-list
//
// The -f flag specifies an alternate format for the list, using the
// syntax of package template.  The default output is equivalent to -f
// '{{.ImportPath}}'. The struct being passed to the template is currently
// an instance of gb.Package. This structure is under active development
// and it's contents are not guarenteed to be stable.
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"text/template"

	"github.com/constabulary/gb"
	"github.com/constabulary/gb/cmd"
)

func main() {
	var (
		projectroot string
		format      string
		formatStdin bool
		jsonOutput  bool
	)
	flag.StringVar(&projectroot, "R", os.Getenv("GB_PROJECT_DIR"), "set the project root")
	flag.StringVar(&format, "f", "{{.ImportPath}}", "format template")
	flag.BoolVar(&formatStdin, "s", false, "read format from stdin")
	flag.BoolVar(&gb.Verbose, "v", gb.Verbose, "enable log levels below INFO level")
	flag.BoolVar(&jsonOutput, "json", false, "outputs json")

	flag.Parse()

	if formatStdin {
		var formatBuffer bytes.Buffer
		io.Copy(&formatBuffer, os.Stdin)
		format = formatBuffer.String()
	}

	gopath := filepath.SplitList(os.Getenv("GOPATH"))
	root, err := cmd.FindProjectroot(projectroot, gopath)
	if err != nil {
		gb.Fatalf("could not locate project root: %v", err)
	}
	project := gb.NewProject(root)

	ctx, err := project.NewContext(
		gb.GcToolchain(),
	)
	if err != nil {
		gb.Fatalf("unable to construct context: %v", err)
	}

	args := cmd.ImportPaths(ctx, projectroot, flag.Args())
	pkgs, err := cmd.ResolvePackages(ctx, projectroot, args...)
	if err != nil {
		gb.Fatalf("unable to resolve: %v", err)
	}

	if jsonOutput {
		views := make([]*PackageView, 0, len(pkgs))
		for _, pkg := range pkgs {
			views = append(views, NewPackageView(pkg))
		}
		encoded, err := json.MarshalIndent(views, " ", "  ")
		if err != nil {
			gb.Fatalf("Error occurred during json encoding: %v", err)
		}
		fmt.Println(string(encoded))
	} else {
		tmpl, err := template.New("list").Parse(format)
		if err != nil {
			gb.Fatalf("unable to parse template %q: %v", format, err)
		}

		for _, pkg := range pkgs {
			if err := tmpl.Execute(os.Stdout, pkg); err != nil {
				gb.Fatalf("unable to execute template: %v", err)
			}
			fmt.Fprintln(os.Stdout)
		}
	}
}
<|endoftext|>