<commit_before>package model

import (
	"fmt"
	"os"
	"path/filepath"
	"testing"

	"github.com/go-xorm/core"
	"github.com/go-xorm/xorm"
	"github.com/stretchr/testify/assert"
	"gopkg.in/testfixtures.v2"
)

// ggzRoot a path to the ggz root
var ggzRoot string

func fatalTestError(fmtStr string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, fmtStr, args...)
	os.Exit(1)
}

// MainTest a reusable TestMain(..) function for unit tests that need to use a
// test database. Creates the test database, and sets necessary settings.
func MainTest(m *testing.M, pathToGgzRoot string) {
	var err error
	ggzRoot = pathToGgzRoot
	fixturesDir := filepath.Join(pathToGgzRoot, "model", "fixtures")
	if err = createTestEngine(fixturesDir); err != nil {
		fatalTestError("Error creating test engine: %v\n", err)
	}
	exitStatus := m.Run()
	os.Exit(exitStatus)
}

func createTestEngine(fixturesDir string) error {
	var err error
	x, err = xorm.NewEngine("sqlite3", "file::memory:?cache=shared")
	if err != nil {
		return err
	}
	x.SetMapper(core.GonicMapper{})
	if err = x.StoreEngine("InnoDB").Sync2(tables...); err != nil {
		return err
	}
	switch os.Getenv("GGZ_UNIT_TESTS_VERBOSE") {
	case "true", "1":
		x.ShowSQL(true)
	}

	return InitFixtures(&testfixtures.SQLite{}, fixturesDir)
}

// PrepareTestDatabase load test fixtures into test database
func PrepareTestDatabase() error {
	return LoadFixtures()
}

// PrepareTestEnv prepares the environment for unit tests. Can only be called
// by tests that use the above MainTest(..) function.
func PrepareTestEnv(t testing.TB) {
	assert.NoError(t, PrepareTestDatabase())
}

type testCond struct {
	query interface{}
	args  []interface{}
}

// Cond create a condition with arguments for a test
func Cond(query interface{}, args ...interface{}) interface{} {
	return &testCond{query: query, args: args}
}

func whereConditions(sess *xorm.Session, conditions []interface{}) {
	for _, condition := range conditions {
		switch cond := condition.(type) {
		case *testCond:
			sess.Where(cond.query, cond.args...)
		default:
			sess.Where(cond)
		}
	}
}

func loadBeanIfExists(bean interface{}, conditions ...interface{}) (bool, error) {
	sess := x.NewSession()
	defer sess.Close()
	whereConditions(sess, conditions)
	return sess.Get(bean)
}

// BeanExists for testing, check if a bean exists
func BeanExists(t testing.TB, bean interface{}, conditions ...interface{}) bool {
	exists, err := loadBeanIfExists(bean, conditions...)
	assert.NoError(t, err)
	return exists
}

// AssertExistsAndLoadBean assert that a bean exists and load it from the test
// database
func AssertExistsAndLoadBean(t testing.TB, bean interface{}, conditions ...interface{}) interface{} {
	exists, err := loadBeanIfExists(bean, conditions...)
	assert.NoError(t, err)
	assert.True(t, exists,
		"Expected to find %+v (of type %T, with conditions %+v), but did not",
		bean, bean, conditions)
	return bean
}

// GetCount get the count of a bean
func GetCount(t testing.TB, bean interface{}, conditions ...interface{}) int {
	sess := x.NewSession()
	defer sess.Close()
	whereConditions(sess, conditions)
	count, err := sess.Count(bean)
	assert.NoError(t, err)
	return int(count)
}

// AssertNotExistsBean assert that a bean does not exist in the test database
func AssertNotExistsBean(t testing.TB, bean interface{}, conditions ...interface{}) {
	exists, err := loadBeanIfExists(bean, conditions...)
	assert.NoError(t, err)
	assert.False(t, exists)
}

// AssertExistsIf asserts that a bean exists or does not exist, depending on
// what is expected.
func AssertExistsIf(t *testing.T, expected bool, bean interface{}, conditions ...interface{}) {
	exists, err := loadBeanIfExists(bean, conditions...)
	assert.NoError(t, err)
	assert.Equal(t, expected, exists)
}

// AssertSuccessfulInsert assert that beans is successfully inserted
func AssertSuccessfulInsert(t testing.TB, beans ...interface{}) {
	_, err := x.Insert(beans...)
	assert.NoError(t, err)
}

// AssertCount assert the count of a bean
func AssertCount(t testing.TB, bean interface{}, expected interface{}) {
	assert.EqualValues(t, expected, GetCount(t, bean))
}

// AssertInt64InRange assert value is in range [low, high]
func AssertInt64InRange(t testing.TB, low, high, value int64) {
	assert.True(t, value >= low && value <= high,
		"Expected value in range [%d, %d], found %d", low, high, value)
}
<commit_msg>show sql in dev mode.<commit_after>package model

import (
	"fmt"
	"os"
	"path/filepath"
	"testing"

	"github.com/go-ggz/ggz/config"

	"github.com/go-xorm/core"
	"github.com/go-xorm/xorm"
	"github.com/stretchr/testify/assert"
	"gopkg.in/testfixtures.v2"
)

// ggzRoot a path to the ggz root
var ggzRoot string

func fatalTestError(fmtStr string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, fmtStr, args...)
	os.Exit(1)
}

// MainTest a reusable TestMain(..) function for unit tests that need to use a
// test database. Creates the test database, and sets necessary settings.
func MainTest(m *testing.M, pathToGgzRoot string) {
	var err error
	ggzRoot = pathToGgzRoot
	fixturesDir := filepath.Join(pathToGgzRoot, "model", "fixtures")
	if err = createTestEngine(fixturesDir); err != nil {
		fatalTestError("Error creating test engine: %v\n", err)
	}
	exitStatus := m.Run()
	os.Exit(exitStatus)
}

func createTestEngine(fixturesDir string) error {
	var err error
	x, err = xorm.NewEngine("sqlite3", "file::memory:?cache=shared")
	if err != nil {
		return err
	}
	x.SetMapper(core.GonicMapper{})
	if err = x.StoreEngine("InnoDB").Sync2(tables...); err != nil {
		return err
	}
	switch os.Getenv("GGZ_UNIT_TESTS_VERBOSE") {
	case "true", "1":
		x.ShowSQL(config.Debug)
	}

	return InitFixtures(&testfixtures.SQLite{}, fixturesDir)
}

// PrepareTestDatabase load test fixtures into test database
func PrepareTestDatabase() error {
	return LoadFixtures()
}

// PrepareTestEnv prepares the environment for unit tests. Can only be called
// by tests that use the above MainTest(..) function.
func PrepareTestEnv(t testing.TB) {
	assert.NoError(t, PrepareTestDatabase())
}

type testCond struct {
	query interface{}
	args  []interface{}
}

// Cond create a condition with arguments for a test
func Cond(query interface{}, args ...interface{}) interface{} {
	return &testCond{query: query, args: args}
}

func whereConditions(sess *xorm.Session, conditions []interface{}) {
	for _, condition := range conditions {
		switch cond := condition.(type) {
		case *testCond:
			sess.Where(cond.query, cond.args...)
		default:
			sess.Where(cond)
		}
	}
}

func loadBeanIfExists(bean interface{}, conditions ...interface{}) (bool, error) {
	sess := x.NewSession()
	defer sess.Close()
	whereConditions(sess, conditions)
	return sess.Get(bean)
}

// BeanExists for testing, check if a bean exists
func BeanExists(t testing.TB, bean interface{}, conditions ...interface{}) bool {
	exists, err := loadBeanIfExists(bean, conditions...)
	assert.NoError(t, err)
	return exists
}

// AssertExistsAndLoadBean assert that a bean exists and load it from the test
// database
func AssertExistsAndLoadBean(t testing.TB, bean interface{}, conditions ...interface{}) interface{} {
	exists, err := loadBeanIfExists(bean, conditions...)
	assert.NoError(t, err)
	assert.True(t, exists,
		"Expected to find %+v (of type %T, with conditions %+v), but did not",
		bean, bean, conditions)
	return bean
}

// GetCount get the count of a bean
func GetCount(t testing.TB, bean interface{}, conditions ...interface{}) int {
	sess := x.NewSession()
	defer sess.Close()
	whereConditions(sess, conditions)
	count, err := sess.Count(bean)
	assert.NoError(t, err)
	return int(count)
}

// AssertNotExistsBean assert that a bean does not exist in the test database
func AssertNotExistsBean(t testing.TB, bean interface{}, conditions ...interface{}) {
	exists, err := loadBeanIfExists(bean, conditions...)
	assert.NoError(t, err)
	assert.False(t, exists)
}

// AssertExistsIf asserts that a bean exists or does not exist, depending on
// what is expected.
func AssertExistsIf(t *testing.T, expected bool, bean interface{}, conditions ...interface{}) {
	exists, err := loadBeanIfExists(bean, conditions...)
	assert.NoError(t, err)
	assert.Equal(t, expected, exists)
}

// AssertSuccessfulInsert assert that beans is successfully inserted
func AssertSuccessfulInsert(t testing.TB, beans ...interface{}) {
	_, err := x.Insert(beans...)
	assert.NoError(t, err)
}

// AssertCount assert the count of a bean
func AssertCount(t testing.TB, bean interface{}, expected interface{}) {
	assert.EqualValues(t, expected, GetCount(t, bean))
}

// AssertInt64InRange assert value is in range [low, high]
func AssertInt64InRange(t testing.TB, low, high, value int64) {
	assert.True(t, value >= low && value <= high,
		"Expected value in range [%d, %d], found %d", low, high, value)
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package models

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/Unknwon/com"

	"github.com/gogits/gogs/modules/log"
)

const (
	// "### autogenerated by gitgos, DO NOT EDIT\n"
	TPL_PUBLICK_KEY = `command="%s serv key-%d",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty %s`
)

var (
	ErrKeyAlreadyExist = errors.New("Public key already exist")
)

var sshOpLocker = sync.Mutex{}

var (
	sshPath string
	appPath string
)

// exePath returns the executable path.
func exePath() (string, error) {
	file, err := exec.LookPath(os.Args[0])
	if err != nil {
		return "", err
	}
	return filepath.Abs(file)
}

// homeDir returns the home directory of current user.
func homeDir() string {
	home, err := com.HomeDir()
	if err != nil {
		return "/"
	}
	return home
}

func init() {
	var err error

	appPath, err = exePath()
	if err != nil {
		fmt.Printf("publickey.init(fail to get app path): %v\n", err)
		os.Exit(2)
	}

	// Determine and create .ssh path.
	sshPath = filepath.Join(homeDir(), ".ssh")
	if err = os.MkdirAll(sshPath, os.ModePerm); err != nil {
		fmt.Printf("publickey.init(fail to create sshPath(%s)): %v\n", sshPath, err)
		os.Exit(2)
	}
}

// PublicKey represents a SSH key of user.
type PublicKey struct {
	Id          int64
	OwnerId     int64  `xorm:"index"`
	Name        string `xorm:"unique not null"`
	Fingerprint string
	Content     string    `xorm:"TEXT not null"`
	Created     time.Time `xorm:"created"`
	Updated     time.Time `xorm:"updated"`
}

// GenAuthorizedKey returns formatted public key string.
func GenAuthorizedKey(keyId int64, key string) string {
	return fmt.Sprintf(TPL_PUBLICK_KEY+"\n", appPath, keyId, key)
}

// AddPublicKey adds new public key to database and SSH key file.
func AddPublicKey(key *PublicKey) (err error) {
	// Check if public key name has been used.
	has, err := orm.Get(key)
	if err != nil {
		return err
	} else if has {
		return ErrKeyAlreadyExist
	}

	// Calculate fingerprint.
	tmpPath := strings.Replace(filepath.Join(os.TempDir(), fmt.Sprintf("%d", time.Now().Nanosecond()),
		"id_rsa.pub"), "\\", "/", -1)
	os.MkdirAll(path.Dir(tmpPath), os.ModePerm)
	if err = ioutil.WriteFile(tmpPath, []byte(key.Content), os.ModePerm); err != nil {
		return err
	}
	stdout, _, err := com.ExecCmd("ssh-keygen", "-l", "-f", tmpPath)
	if err != nil {
		return err
	} else if len(stdout) < 2 {
		return errors.New("Not enough output for calculating fingerprint")
	}
	key.Fingerprint = strings.Split(stdout, " ")[1]

	// Save SSH key.
	if _, err = orm.Insert(key); err != nil {
		return err
	}
	if err = SaveAuthorizedKeyFile(key); err != nil {
		if _, err2 := orm.Delete(key); err2 != nil {
			return err2
		}
		return err
	}

	return nil
}

func rewriteAuthorizedKeys(key *PublicKey, p, tmpP string) error {
	// Delete SSH key in SSH key file.
	sshOpLocker.Lock()
	defer sshOpLocker.Unlock()

	fr, err := os.Open(p)
	if err != nil {
		return err
	}
	defer fr.Close()

	fw, err := os.Create(tmpP)
	if err != nil {
		return err
	}
	defer fw.Close()

	buf := bufio.NewReader(fr)
	for {
		line, errRead := buf.ReadString('\n')
		line = strings.TrimSpace(line)

		if errRead != nil {
			if errRead != io.EOF {
				return errRead
			}

			// Reached end of file, if nothing to read then break,
			// otherwise handle the last line.
			if len(line) == 0 {
				break
			}
		}

		// Found the line and copy rest of file.
		if strings.Contains(line, fmt.Sprintf("key-%d", key.Id)) && strings.Contains(line, key.Content) {
			continue
		}
		// Still finding the line, copy the line that currently read.
		if _, err = fw.WriteString(line + "\n"); err != nil {
			return err
		}

		if errRead == io.EOF {
			break
		}
	}
	return nil
}

// DeletePublicKey deletes SSH key information both in database and authorized_keys file.
func DeletePublicKey(key *PublicKey) (err error) {
	// Delete SSH key in database.
	has, err := orm.Id(key.Id).Get(key)
	if err != nil {
		return err
	} else if !has {
		return errors.New("Public key does not exist")
	}
	if _, err = orm.Delete(key); err != nil {
		return err
	}

	p := filepath.Join(sshPath, "authorized_keys")
	tmpP := filepath.Join(sshPath, "authorized_keys.tmp")
	log.Trace("ssh.DeletePublicKey(authorized_keys): %s", p)

	if err = rewriteAuthorizedKeys(key, p, tmpP); err != nil {
		return err
	} else if err = os.Remove(p); err != nil {
		return err
	}
	return os.Rename(tmpP, p)
}

// ListPublicKey returns a list of public keys that user has.
func ListPublicKey(userId int64) ([]PublicKey, error) {
	keys := make([]PublicKey, 0)
	err := orm.Find(&keys, &PublicKey{OwnerId: userId})
	return keys, err
}

// SaveAuthorizedKeyFile writes SSH key content to SSH key file.
func SaveAuthorizedKeyFile(key *PublicKey) error {
	sshOpLocker.Lock()
	defer sshOpLocker.Unlock()

	p := filepath.Join(sshPath, "authorized_keys")
	f, err := os.OpenFile(p, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = f.WriteString(GenAuthorizedKey(key.Id, key.Content))
	return err
}
<commit_msg>fix unique<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package models

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/Unknwon/com"

	"github.com/gogits/gogs/modules/log"
)

const (
	// "### autogenerated by gitgos, DO NOT EDIT\n"
	TPL_PUBLICK_KEY = `command="%s serv key-%d",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty %s`
)

var (
	ErrKeyAlreadyExist = errors.New("Public key already exist")
)

var sshOpLocker = sync.Mutex{}

var (
	sshPath string
	appPath string
)

// exePath returns the executable path.
func exePath() (string, error) {
	file, err := exec.LookPath(os.Args[0])
	if err != nil {
		return "", err
	}
	return filepath.Abs(file)
}

// homeDir returns the home directory of current user.
func homeDir() string {
	home, err := com.HomeDir()
	if err != nil {
		return "/"
	}
	return home
}

func init() {
	var err error

	appPath, err = exePath()
	if err != nil {
		fmt.Printf("publickey.init(fail to get app path): %v\n", err)
		os.Exit(2)
	}

	// Determine and create .ssh path.
	sshPath = filepath.Join(homeDir(), ".ssh")
	if err = os.MkdirAll(sshPath, os.ModePerm); err != nil {
		fmt.Printf("publickey.init(fail to create sshPath(%s)): %v\n", sshPath, err)
		os.Exit(2)
	}
}

// PublicKey represents a SSH key of user.
type PublicKey struct {
	Id          int64
	OwnerId     int64  `xorm:"unique(s) index not null"`
	Name        string `xorm:"unique(s) not null"`
	Fingerprint string
	Content     string    `xorm:"TEXT not null"`
	Created     time.Time `xorm:"created"`
	Updated     time.Time `xorm:"updated"`
}

// GenAuthorizedKey returns formatted public key string.
func GenAuthorizedKey(keyId int64, key string) string {
	return fmt.Sprintf(TPL_PUBLICK_KEY+"\n", appPath, keyId, key)
}

// AddPublicKey adds new public key to database and SSH key file.
func AddPublicKey(key *PublicKey) (err error) {
	// Check if public key name has been used.
	has, err := orm.Get(key)
	if err != nil {
		return err
	} else if has {
		return ErrKeyAlreadyExist
	}

	// Calculate fingerprint.
	tmpPath := strings.Replace(filepath.Join(os.TempDir(), fmt.Sprintf("%d", time.Now().Nanosecond()),
		"id_rsa.pub"), "\\", "/", -1)
	os.MkdirAll(path.Dir(tmpPath), os.ModePerm)
	if err = ioutil.WriteFile(tmpPath, []byte(key.Content), os.ModePerm); err != nil {
		return err
	}
	stdout, _, err := com.ExecCmd("ssh-keygen", "-l", "-f", tmpPath)
	if err != nil {
		return err
	} else if len(stdout) < 2 {
		return errors.New("Not enough output for calculating fingerprint")
	}
	key.Fingerprint = strings.Split(stdout, " ")[1]

	// Save SSH key.
	if _, err = orm.Insert(key); err != nil {
		return err
	}
	if err = SaveAuthorizedKeyFile(key); err != nil {
		if _, err2 := orm.Delete(key); err2 != nil {
			return err2
		}
		return err
	}

	return nil
}

func rewriteAuthorizedKeys(key *PublicKey, p, tmpP string) error {
	// Delete SSH key in SSH key file.
	sshOpLocker.Lock()
	defer sshOpLocker.Unlock()

	fr, err := os.Open(p)
	if err != nil {
		return err
	}
	defer fr.Close()

	fw, err := os.Create(tmpP)
	if err != nil {
		return err
	}
	defer fw.Close()

	buf := bufio.NewReader(fr)
	for {
		line, errRead := buf.ReadString('\n')
		line = strings.TrimSpace(line)

		if errRead != nil {
			if errRead != io.EOF {
				return errRead
			}

			// Reached end of file, if nothing to read then break,
			// otherwise handle the last line.
			if len(line) == 0 {
				break
			}
		}

		// Found the line and copy rest of file.
		if strings.Contains(line, fmt.Sprintf("key-%d", key.Id)) && strings.Contains(line, key.Content) {
			continue
		}
		// Still finding the line, copy the line that currently read.
		if _, err = fw.WriteString(line + "\n"); err != nil {
			return err
		}

		if errRead == io.EOF {
			break
		}
	}
	return nil
}

// DeletePublicKey deletes SSH key information both in database and authorized_keys file.
func DeletePublicKey(key *PublicKey) (err error) {
	// Delete SSH key in database.
	has, err := orm.Id(key.Id).Get(key)
	if err != nil {
		return err
	} else if !has {
		return errors.New("Public key does not exist")
	}
	if _, err = orm.Delete(key); err != nil {
		return err
	}

	p := filepath.Join(sshPath, "authorized_keys")
	tmpP := filepath.Join(sshPath, "authorized_keys.tmp")
	log.Trace("ssh.DeletePublicKey(authorized_keys): %s", p)

	if err = rewriteAuthorizedKeys(key, p, tmpP); err != nil {
		return err
	} else if err = os.Remove(p); err != nil {
		return err
	}
	return os.Rename(tmpP, p)
}

// ListPublicKey returns a list of public keys that user has.
func ListPublicKey(userId int64) ([]PublicKey, error) {
	keys := make([]PublicKey, 0)
	err := orm.Find(&keys, &PublicKey{OwnerId: userId})
	return keys, err
}

// SaveAuthorizedKeyFile writes SSH key content to SSH key file.
func SaveAuthorizedKeyFile(key *PublicKey) error {
	sshOpLocker.Lock()
	defer sshOpLocker.Unlock()

	p := filepath.Join(sshPath, "authorized_keys")
	f, err := os.OpenFile(p, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = f.WriteString(GenAuthorizedKey(key.Id, key.Content))
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"syscall"

	"github.com/BurntSushi/toml"
	"github.com/judwhite/go-svc/svc"
	"github.com/prism-river/killy/internal/version"
	"github.com/prism-river/killy/killyd"
)

type program struct {
	killyd *killyd.KILLYD
}

func killydFlagSet(opts *killyd.Options) *flag.FlagSet {
	flagSet := flag.NewFlagSet("killyd", flag.ExitOnError)
	flagSet.Bool("version", false, "print version string")
	flagSet.String("config", "", "path to config file")
	return flagSet
}

func main() {
	prg := &program{}
	if err := svc.Run(prg, syscall.SIGINT, syscall.SIGTERM); err != nil {
		log.Fatal(err)
	}
}

func (p *program) Init(env svc.Environment) error {
	if env.IsWindowsService() {
		dir := filepath.Dir(os.Args[0])
		return os.Chdir(dir)
	}
	return nil
}

func loadmeta(configFile string) (meta killyd.Meta, err error) {
	if configFile != "" {
		_, err = toml.DecodeFile(configFile, &meta)
		if err != nil {
			return
		}
	}
	return
}

func (p *program) Start() error {
	opts := killyd.NewOptions()
	flagSet := killydFlagSet(opts)
	flagSet.Parse(os.Args[1:])

	if flagSet.Lookup("version").Value.(flag.Getter).Get().(bool) {
		fmt.Println(version.String())
		os.Exit(0)
	}

	killyd, err := killyd.New(opts)
	if err != nil {
		fmt.Println(err.Error())
		os.Exit(0)
	}
	configFile := flagSet.Lookup("config").Value.String()
	meta, err := loadmeta(configFile)
	if err != nil {
		log.Fatalf("ERROR: failed to load config file %s - %s", configFile, err.Error())
	}
	//for k,v := range meta.Topics {
	//	fmt.Println("--")
	//	fmt.Println(k)
	//	for kk,vv := range v {
	//		fmt.Println("-")
	//		fmt.Println(kk)
	//		fmt.Println(vv.Address)
	//		fmt.Println(vv.Interval)
	//	}
	//}

	err = killyd.Loadmeta(meta)
	if err != nil {
		log.Fatalf("ERROR: %s", err.Error())
	}
	killyd.Main()
	p.killyd = killyd

	return nil
}

func (p *program) Stop() error {
	if p.killyd != nil {
		p.killyd.Exit()
		log.Fatalf("INFO: stop the killy")
	}
	return nil
}
<commit_msg>fix: Config cannot be empty<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"syscall"

	"github.com/BurntSushi/toml"
	"github.com/judwhite/go-svc/svc"
	"github.com/prism-river/killy/internal/version"
	"github.com/prism-river/killy/killyd"
)

type program struct {
	killyd *killyd.KILLYD
}

func killydFlagSet(opts *killyd.Options) *flag.FlagSet {
	flagSet := flag.NewFlagSet("killyd", flag.ExitOnError)
	flagSet.Bool("version", false, "print version string")
	flagSet.String("config", "", "path to config file")
	return flagSet
}

func main() {
	prg := &program{}
	if err := svc.Run(prg, syscall.SIGINT, syscall.SIGTERM); err != nil {
		log.Fatal(err)
	}
}

func (p *program) Init(env svc.Environment) error {
	if env.IsWindowsService() {
		dir := filepath.Dir(os.Args[0])
		return os.Chdir(dir)
	}
	return nil
}

func loadmeta(configFile string) (meta killyd.Meta, err error) {
	if configFile != "" {
		_, err = toml.DecodeFile(configFile, &meta)
		if err != nil {
			return
		}
	}
	return meta, errors.New("Config is empty")
}

func (p *program) Start() error {
	opts := killyd.NewOptions()
	flagSet := killydFlagSet(opts)
	flagSet.Parse(os.Args[1:])

	if flagSet.Lookup("version").Value.(flag.Getter).Get().(bool) {
		fmt.Println(version.String())
		os.Exit(0)
	}
	configFile := flagSet.Lookup("config").Value.String()
	meta, err := loadmeta(configFile)
	if err != nil {
		log.Fatalf("ERROR: failed to load config file %s - %s", configFile, err.Error())
	}
	killyd, err := killyd.New(opts)
	if err != nil {
		fmt.Println(err.Error())
		os.Exit(0)
	}
	err = killyd.Loadmeta(meta)
	if err != nil {
		log.Fatalf("ERROR: %s", err.Error())
	}
	killyd.Main()
	p.killyd = killyd
	return nil
}

func (p *program) Stop() error {
	if p.killyd != nil {
		p.killyd.Exit()
		log.Fatalf("INFO: stop the killy")
	}
	return nil
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/bwmarrin/discordgo"
	"github.com/oct2pus/bocto"
	"github.com/oct2pus/jadebot/art"
	"github.com/oct2pus/jadebot/db"
)

// Confused is the respond JadeBot provides when the user inputs an incorrect command.
func Confused(bot bocto.Bot,
	message *discordgo.MessageCreate,
	input []string) {
	respond(bot, message,
		"i don't understand, maybe you should ask for `"+bot.Prefix+"help` ;P")
}

// Mentioned is the command that activates when someone @'s JadeBot.
func Mentioned(bot bocto.Bot, message *discordgo.MessageCreate, input []string) {
	respond(bot, message, "hello!\n by the way, my prefix is `"+bot.Prefix+"`! just incase you wanted to know.", art.Emoji["owo"])
}

// Reminder reminds people to not add spaces between the prefix and command.
func Reminder(bot bocto.Bot,
	message *discordgo.MessageCreate,
	input []string) {
	if len(input) > 0 {
		output := fmt.Sprintf("please don't add a space between commands! try `%v%v` :)", bot.Prefix, strings.Join(input, " "))
		bot.Session.ChannelMessageSend(message.ChannelID, output)
	} else {
		bot.Confused(bot, message, input)
	}
}

// OwO whats this?
func OwO(bot bocto.Bot, message *discordgo.MessageCreate, input []string) {
	respond(bot, message, "oh woah whats this?", art.Emoji["owo"])
}

// GoodDog best friend.
func GoodDog(bot bocto.Bot, message *discordgo.MessageCreate, input []string) {
	respond(bot, message, "best friend", art.Emoji["headpat"])
}

// Teef teefs.
func Teef(bot bocto.Bot, message *discordgo.MessageCreate, input []string) {
	respond(bot, message, art.Emoji["teefs"])
}

// KissJade fulfills an **IMPORTANT** business rule for JadeBot.
func KissJade(bot bocto.Bot, message *discordgo.MessageCreate, input []string) {
	respond(bot, message, art.Emoji["embarassed"]+art.Emoji["heart"])
}

// Headpat feels good. Please give headpats.
func Headpat(bot bocto.Bot, message *discordgo.MessageCreate, input []string) {
	respond(bot, message, art.Emoji["headpat"])
}

// Thinking 🤔.
func Thinking(bot bocto.Bot, message *discordgo.MessageCreate, input []string) {
	respond(bot, message, art.Emoji["thinking"])
}

// ReminderMarkov reminds people to not add spaces between the prefix and command.
// Uses Markov instead of bot.Confused when no input is detected.
func ReminderMarkov(bot bocto.Bot,
	message *discordgo.MessageCreate,
	input []string) {
	if len(input) > 0 {
		Reminder(bot, message, input)
	} else {
		Markov(bot, message, input)
	}
}

func respond(bot bocto.Bot, message *discordgo.MessageCreate, responses ...string) {
	i, err := strconv.Atoi(message.GuildID)
	if err != nil {
		fmt.Printf("cannot convert guildID number, something really bad happened.\n")
		return
	}
	o, err := db.LookupEntry(i, "responses")
	if err != nil {
		return
	}
	if o == "true" || o == "" {
		for _, response := range responses {
			bot.Session.ChannelMessageSend(message.ChannelID, response)
		}
	}
}
<commit_msg>modified love to be a bit more cheeky<commit_after>package command

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/bwmarrin/discordgo"
	"github.com/oct2pus/bocto"
	"github.com/oct2pus/jadebot/art"
	"github.com/oct2pus/jadebot/db"
)

// Confused is the respond JadeBot provides when the user inputs an incorrect command.
func Confused(bot bocto.Bot,
	message *discordgo.MessageCreate,
	input []string) {
	respond(bot, message,
		"i don't understand, maybe you should ask for `"+bot.Prefix+"help` ;P")
}

// Mentioned is the command that activates when someone @'s JadeBot.
func Mentioned(bot bocto.Bot, message *discordgo.MessageCreate, input []string) {
	respond(bot, message, "hello!\n by the way, my prefix is `"+bot.Prefix+"`! just incase you wanted to know.", art.Emoji["owo"])
}

// Reminder reminds people to not add spaces between the prefix and command.
func Reminder(bot bocto.Bot,
	message *discordgo.MessageCreate,
	input []string) {
	if len(input) > 0 {
		output := fmt.Sprintf("please don't add a space between commands! try `%v%v` :)", bot.Prefix, strings.Join(input, " "))
		bot.Session.ChannelMessageSend(message.ChannelID, output)
	} else {
		bot.Confused(bot, message, input)
	}
}

// OwO whats this?
func OwO(bot bocto.Bot, message *discordgo.MessageCreate, input []string) {
	respond(bot, message, "oh woah whats this?", art.Emoji["owo"])
}

// Love is a human emotion.
func Love(bot bocto.Bot, message *discordgo.MessageCreate, input []string) {
	respond(bot, message, "i love you too."+art.Emoji["teefs"])
}

// GoodDog best friend.
func GoodDog(bot bocto.Bot, message *discordgo.MessageCreate, input []string) {
	respond(bot, message, "best friend", art.Emoji["headpat"])
}

// Teef teefs.
func Teef(bot bocto.Bot, message *discordgo.MessageCreate, input []string) {
	respond(bot, message, art.Emoji["teefs"])
}

// Headpat feels good. Please give headpats.
func Headpat(bot bocto.Bot, message *discordgo.MessageCreate, input []string) {
	respond(bot, message, art.Emoji["headpat"])
}

// Thinking 🤔.
func Thinking(bot bocto.Bot, message *discordgo.MessageCreate, input []string) {
	respond(bot, message, art.Emoji["thinking"])
}

// ReminderMarkov reminds people to not add spaces between the prefix and command.
// Uses Markov instead of bot.Confused when no input is detected.
func ReminderMarkov(bot bocto.Bot,
	message *discordgo.MessageCreate,
	input []string) {
	if len(input) > 0 {
		Reminder(bot, message, input)
	} else {
		Markov(bot, message, input)
	}
}

func respond(bot bocto.Bot, message *discordgo.MessageCreate, responses ...string) {
	i, err := strconv.Atoi(message.GuildID)
	if err != nil {
		fmt.Printf("cannot convert guildID number, something really bad happened.\n")
		return
	}
	o, err := db.LookupEntry(i, "responses")
	if err != nil {
		return
	}
	if o == "true" || o == "" {
		for _, response := range responses {
			bot.Session.ChannelMessageSend(message.ChannelID, response)
		}
	}
}
<|endoftext|>


<commit_before>package discordgo_polly

import (
	"encoding/binary"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	session2 "github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/polly"
	"github.com/bitly/go-simplejson"
	"github.com/bwmarrin/discordgo"
	"io"
	"io/ioutil"
	"layeh.com/gopus"
	"log"
	"sync"
)

func relax(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

// To keep this module if you don't want to use the config structure I use, this just returns a static credential
func LoadConfig() aws.Config {
	//load our bot config
	file, err := ioutil.ReadFile("./config.json")
	if err != nil {
		log.Fatal(err)
	}
	fileLoad, err := simplejson.NewJson(file)
	if err != nil {
		log.Fatal(err)
	}

	//AWS credentials for polly
	awsid := fileLoad.Get("awsapiid")
	if awsid == nil {
		log.Println("No AWS APIID")
	}
	awsapiid, err := awsid.String()
	relax(err)
	awssecret := fileLoad.Get("awsapisecret")
	if awssecret == nil {
		log.Println("no AWS APISECRET")
	}
	awsapisecret, err := awssecret.String()
	relax(err)
	awsregion := fileLoad.Get("awsapiregion")
	if awssecret == nil {
		log.Println("no AWS APISREGION")
	}
	awsapiregion, err := awsregion.String()
	relax(err)

	return aws.Config{
		Region:      aws.String(awsapiregion),
		Credentials: credentials.NewStaticCredentials(awsapiid, awsapisecret, ""),
	}
}

//ultimate goal is to have a function that we can just say
type DgoPolly struct {
	pollySession polly.Polly
	pollyConfig  polly.SynthesizeSpeechInput
}

func newPolly(creds credentials.Credentials, aws aws.Config) (ourPolly DgoPolly) {
	session, err := session2.NewSession(&aws)
	relax(err)
	ourPolly.pollySession = *polly.New(session)

	testToSynth := "Default Message"
	outputFormat := polly.OutputFormatPcm

	pollyconfig := polly.SynthesizeSpeechInput{
		Text:         &testToSynth,
		OutputFormat: &outputFormat,
	}

	ourPolly.pollyConfig = pollyconfig
	return
}

func (p *DgoPolly) say(vc *discordgo.VoiceConnection, text string) {
	opusChannel := p.getSynthSpeech(text)
	for {
		opus, ok := <-*opusChannel
		if !ok {
			return
		}

		vc.OpusSend <- opus
	}
}

//Returns an Opus channel for play at 48000Hz, what discordGo needs for vc.OpusSend
func (p *DgoPolly) getSynthSpeech(text string) *chan []byte {
	p.pollyConfig.Text = &text

	request, response := p.pollySession.SynthesizeSpeechRequest(&p.pollyConfig)
	err := request.Send()
	relax(err)

	outputChan := make(chan []byte, 1000)
	encodeChan := make(chan []int16, 1000)

	opusEncoder, err := gopus.NewEncoder(16000, 1, gopus.Voip)
	relax(err)
	opusEncoder.SetBitrate(64 * 1000)

	wg := sync.WaitGroup{}

	loadBuffer := func(input io.Reader) {

		defer func() {
			fmt.Println("done loading frames")
			wg.Done()
			close(encodeChan)
		}()

		for {
			//960 is the framesize, 1 is the number of channels the input has
			inBuf := make([]int16, 960*1)
			err := binary.Read(input, binary.LittleEndian, &inBuf)
			if err == io.EOF || err == io.ErrUnexpectedEOF {
				return
			}
			//Upsamping.. apparently
			// polly does audio in 16KHz, discord needs 48KHz, so we add every frame 3 times.
			encodeChan <- inBuf
			encodeChan <- inBuf
			encodeChan <- inBuf

		}
	}

	encodeBuffer := func() {
		defer func() {
			fmt.Println("Done encoding new frames")
			wg.Done()
			close(outputChan)
		}()

		for {
			pcm, ok := <-encodeChan
			if !ok {
				return
			}
			//we want audio in two channels. So the normal 960*2, plus the additional *2 because the source has more
			// channels
			opus, err := opusEncoder.Encode(pcm, 960, 960*2*2)
			relax(err)

			outputChan <- opus
		}
	}

	wg.Add(2)
	go loadBuffer(response.AudioStream)
	go encodeBuffer()
	wg.Wait()

	return &outputChan

}
<commit_msg>moved melfina to using this as a test. Cleaned up usage.<commit_after>package discordgo_polly

import (
	"encoding/binary"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	session2 "github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/polly"
	"github.com/bitly/go-simplejson"
	"github.com/bwmarrin/discordgo"
	"io"
	"io/ioutil"
	"layeh.com/gopus"
	"log"
	"sync"
)

func relax(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

// To keep this module if you don't want to use the config structure I use, this just returns a static credential
func LoadConfig() aws.Config {
	//load our bot config
	file, err := ioutil.ReadFile("./config.json")
	if err != nil {
		log.Fatal(err)
	}
	fileLoad, err := simplejson.NewJson(file)
	if err != nil {
		log.Fatal(err)
	}

	//AWS credentials for polly
	awsid := fileLoad.Get("awsapiid")
	if awsid == nil {
		log.Println("No AWS APIID")
	}
	awsapiid, err := awsid.String()
	relax(err)
	awssecret := fileLoad.Get("awsapisecret")
	if awssecret == nil {
		log.Println("no AWS APISECRET")
	}
	awsapisecret, err := awssecret.String()
	relax(err)
	awsregion := fileLoad.Get("awsapiregion")
	if awssecret == nil {
		log.Println("no AWS APISREGION")
	}
	awsapiregion, err := awsregion.String()
	relax(err)

	return aws.Config{
		Region:      aws.String(awsapiregion),
		Credentials: credentials.NewStaticCredentials(awsapiid, awsapisecret, ""),
	}
}

//ultimate goal is to have a function that we can just say
type DgoPolly struct {
	pollySession polly.Polly
	PollyConfig  polly.SynthesizeSpeechInput
}

func NewPolly(aws aws.Config) (ourPolly DgoPolly) {
	session, err := session2.NewSession(&aws)
	relax(err)
	ourPolly.pollySession = *polly.New(session)

	testToSynth := "Default Message"
	outputFormat := polly.OutputFormatPcm
	voiceid := polly.VoiceIdSalli

	pollyconfig := polly.SynthesizeSpeechInput{
		Text:         &testToSynth,
		OutputFormat: &outputFormat,
		VoiceId:      &voiceid,
	}

	ourPolly.PollyConfig = pollyconfig
	return
}

func (p *DgoPolly) Say(vc *discordgo.VoiceConnection, text string) {
	opusChannel := p.getSynthSpeech(text)
	for {
		opus, ok := <-*opusChannel
		if !ok {
			return
		}

		vc.OpusSend <- opus
	}
}

//Returns an Opus channel for play at 48000Hz, what discordGo needs for vc.OpusSend
func (p *DgoPolly) getSynthSpeech(text string) *chan []byte {
	p.PollyConfig.Text = &text

	request, response := p.pollySession.SynthesizeSpeechRequest(&p.PollyConfig)
	err := request.Send()
	relax(err)

	outputChan := make(chan []byte, 1000)
	encodeChan := make(chan []int16, 1000)

	opusEncoder, err := gopus.NewEncoder(16000, 1, gopus.Voip)
	relax(err)
	opusEncoder.SetBitrate(64 * 1000)

	wg := sync.WaitGroup{}

	loadBuffer := func(input io.Reader) {

		defer func() {
			wg.Done()
			close(encodeChan)
		}()

		for {
			//960 is the framesize, 1 is the number of channels the input has
			inBuf := make([]int16, 960*1)
			err := binary.Read(input, binary.LittleEndian, &inBuf)
			if err == io.EOF || err == io.ErrUnexpectedEOF {
				return
			}
			//Upsamping.. apparently
			// polly does audio in 16KHz, discord needs 48KHz, so we add every frame 3 times.
			encodeChan <- inBuf
			encodeChan <- inBuf
			encodeChan <- inBuf

		}
	}

	encodeBuffer := func() {
		defer func() {
			wg.Done()
			close(outputChan)
		}()

		for {
			pcm, ok := <-encodeChan
			if !ok {
				return
			}
			//we want audio in two channels. So the normal 960*2, plus the additional *2 because the source has more
			// channels
			opus, err := opusEncoder.Encode(pcm, 960, 960*2*2)
			relax(err)

			outputChan <- opus
		}
	}

	wg.Add(2)
	go loadBuffer(response.AudioStream)
	go encodeBuffer()
	wg.Wait()

	return &outputChan

}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	api "github.com/yungsang/dockerclient"
	"github.com/yungsang/tablewriter"
	"github.com/yungsang/talk2docker/client"
)

func CommandImages(ctx *cli.Context) {
	docker, err := client.GetDockerClient(ctx)
	if err != nil {
		log.Fatal(err)
	}

	images, err := docker.ListImages(ctx.Bool("all"))
	if err != nil {
		log.Fatal(err)
	}

	var matchName = ""
	if len(ctx.Args()) > 0 {
		matchName = ctx.Args()[0]
	}

	matchImageByName := func(tags []string, name string) bool {
		_name := strings.Split(name, ":")

		for _, tag := range tags {
			_tag := strings.Split(tag, ":")
			if _tag[0] == _name[0] {
				if (len(_name) < 2) || (_tag[1] == _name[1]) {
					return true
				}
			}
		}

		return false
	}

	if ctx.Bool("quiet") {
		for _, image := range images {
			if (matchName == "") || matchImageByName(image.RepoTags, matchName) {
				fmt.Println(Truncate(image.Id, 12))
			}
		}
		return
	}

	var items [][]string

	if ctx.Bool("all") {
		roots := make([]*api.Image, 0)
		parents := make(map[string][]*api.Image)
		for _, image := range images {
			if image.ParentId == "" {
				roots = append(roots, image)
			} else {
				if children, exists := parents[image.ParentId]; exists {
					parents[image.ParentId] = append(children, image)
				} else {
					children := make([]*api.Image, 0)
					parents[image.ParentId] = append(children, image)
				}
			}
		}

		items = walkTree(roots, parents, "\u2063", items)
	} else {
		for _, image := range images {
			if (matchName == "") || matchImageByName(image.RepoTags, matchName) {
				name := strings.Join(image.RepoTags, ",\u00a0")
				if name == "<none>:<none>" {
					name = "<none>"
				}
				out := []string{
					Truncate(image.Id, 12),
					name,
					fmt.Sprintf("%.3f", float64(image.VirtualSize)/1000000.0),
					FormatDateTime(time.Unix(image.Created, 0)),
				}
				items = append(items, out)
			}
		}
	}

	var header = []string{
		"ID",
		"Name:Tags",
		"Size(MB)",
	}
	if !ctx.Bool("all") {
		header = append(header, "Created at")
	}

	table := tablewriter.NewWriter(os.Stdout)
	if !ctx.Bool("no-header") {
		table.SetHeader(header)
	}
	table.SetBorder(false)
	table.AppendBulk(items)
	table.Render()
}

func walkTree(images []*api.Image, parents map[string][]*api.Image, prefix string, items [][]string) [][]string {
	printImage := func(prefix string, image *api.Image, isLeaf bool) {
		name := strings.Join(image.RepoTags, ",\u00a0")
		if name == "<none>:<none>" {
			if isLeaf {
				name = "<none>"
			} else {
				name = ""
			}
		}
		out := []string{
			fmt.Sprintf("%s%s%s", prefix, "\u00a0", Truncate(image.Id, 12)),
			name,
			fmt.Sprintf("%.3f", float64(image.VirtualSize)/1000000.0),
		}
		items = append(items, out)
	}

	length := len(images)
	if length > 1 {
		for index, image := range images {
			if (index + 1) == length {
				subimages, exists := parents[image.Id]
				printImage(prefix+"└", image, !exists)
				if exists {
					items = walkTree(subimages, parents, prefix+"\u00a0", items)
				}
			} else {
				subimages, exists := parents[image.Id]
				printImage(prefix+"├", image, !exists)
				if exists {
					items = walkTree(subimages, parents, prefix+"│", items)
				}
			}
		}
	} else {
		for _, image := range images {
			subimages, exists := parents[image.Id]
			printImage(prefix+"└", image, !exists)
			if exists {
				items = walkTree(subimages, parents, prefix+"\u00a0", items)
			}
		}
	}
	return items
}
<commit_msg>Fix padding for the image tree<commit_after>package commands

import (
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	api "github.com/yungsang/dockerclient"
	"github.com/yungsang/tablewriter"
	"github.com/yungsang/talk2docker/client"
)

func CommandImages(ctx *cli.Context) {
	docker, err := client.GetDockerClient(ctx)
	if err != nil {
		log.Fatal(err)
	}

	images, err := docker.ListImages(ctx.Bool("all"))
	if err != nil {
		log.Fatal(err)
	}

	var matchName = ""
	if len(ctx.Args()) > 0 {
		matchName = ctx.Args()[0]
	}

	matchImageByName := func(tags []string, name string) bool {
		_name := strings.Split(name, ":")

		for _, tag := range tags {
			_tag := strings.Split(tag, ":")
			if _tag[0] == _name[0] {
				if (len(_name) < 2) || (_tag[1] == _name[1]) {
					return true
				}
			}
		}

		return false
	}

	if ctx.Bool("quiet") {
		for _, image := range images {
			if (matchName == "") || matchImageByName(image.RepoTags, matchName) {
				fmt.Println(Truncate(image.Id, 12))
			}
		}
		return
	}

	var items [][]string

	if ctx.Bool("all") {
		roots := make([]*api.Image, 0)
		parents := make(map[string][]*api.Image)
		for _, image := range images {
			if image.ParentId == "" {
				roots = append(roots, image)
			} else {
				if children, exists := parents[image.ParentId]; exists {
					parents[image.ParentId] = append(children, image)
				} else {
					children := make([]*api.Image, 0)
					parents[image.ParentId] = append(children, image)
				}
			}
		}

		items = walkTree(roots, parents, "", items)
	} else {
		for _, image := range images {
			if (matchName == "") || matchImageByName(image.RepoTags, matchName) {
				name := strings.Join(image.RepoTags, ",\u00a0")
				if name == "<none>:<none>" {
					name = "<none>"
				}
				out := []string{
					Truncate(image.Id, 12),
					name,
					fmt.Sprintf("%.3f", float64(image.VirtualSize)/1000000.0),
					FormatDateTime(time.Unix(image.Created, 0)),
				}
				items = append(items, out)
			}
		}
	}

	var header = []string{
		"ID",
		"Name:Tags",
		"Size(MB)",
	}
	if !ctx.Bool("all") {
		header = append(header, "Created at")
	}

	table := tablewriter.NewWriter(os.Stdout)
	if !ctx.Bool("no-header") {
		table.SetHeader(header)
	}
	table.SetBorder(false)
	table.AppendBulk(items)
	table.Render()
}

func walkTree(images []*api.Image, parents map[string][]*api.Image, prefix string, items [][]string) [][]string {
	printImage := func(prefix string, image *api.Image, isLeaf bool) {
		name := strings.Join(image.RepoTags, ",\u00a0")
		if name == "<none>:<none>" {
			if isLeaf {
				name = "<none>"
			} else {
				name = ""
			}
		}
		out := []string{
			fmt.Sprintf("%s%s%s", prefix, "\u00a0", Truncate(image.Id, 12)),
			name,
			fmt.Sprintf("%.3f", float64(image.VirtualSize)/1000000.0),
		}
		items = append(items, out)
	}

	length := len(images)
	if length > 1 {
		for index, image := range images {
			if (index + 1) == length {
				subimages, exists := parents[image.Id]
				printImage(prefix+"└", image, !exists)
				if exists {
					items = walkTree(subimages, parents, prefix+"\u00a0", items)
				}
			} else {
				subimages, exists := parents[image.Id]
				printImage(prefix+"├", image, !exists)
				if exists {
					items = walkTree(subimages, parents, prefix+"│", items)
				}
			}
		}
	} else {
		for _, image := range images {
			subimages, exists := parents[image.Id]
			printImage(prefix+"└", image, !exists)
			if exists {
				items = walkTree(subimages, parents, prefix+"\u2063", items)
			}
		}
	}
	return items
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import (
	"internal/syscall/windows"
	"syscall"
	"unsafe"
)

// Stat returns the FileInfo structure describing file.
// If there is an error, it will be of type *PathError.
func (file *File) Stat() (FileInfo, error) {
	if file == nil {
		return nil, ErrInvalid
	}
	if file == nil {
		return nil, syscall.EINVAL
	}
	if file.isdir() {
		// I don't know any better way to do that for directory
		return Stat(file.dirinfo.path)
	}
	if file.name == DevNull {
		return &devNullStat, nil
	}

	ft, err := file.pfd.GetFileType()
	if err != nil {
		return nil, &PathError{"GetFileType", file.name, err}
	}
	switch ft {
	case syscall.FILE_TYPE_PIPE, syscall.FILE_TYPE_CHAR:
		return &fileStat{name: basename(file.name), filetype: ft}, nil
	}

	var d syscall.ByHandleFileInformation
	err = file.pfd.GetFileInformationByHandle(&d)
	if err != nil {
		return nil, &PathError{"GetFileInformationByHandle", file.name, err}
	}
	return &fileStat{
		name: basename(file.name),
		sys: syscall.Win32FileAttributeData{
			FileAttributes: d.FileAttributes,
			CreationTime:   d.CreationTime,
			LastAccessTime: d.LastAccessTime,
			LastWriteTime:  d.LastWriteTime,
			FileSizeHigh:   d.FileSizeHigh,
			FileSizeLow:    d.FileSizeLow,
		},
		filetype: ft,
		vol:      d.VolumeSerialNumber,
		idxhi:    d.FileIndexHigh,
		idxlo:    d.FileIndexLow,
	}, nil
}

// Stat returns a FileInfo structure describing the named file.
// If there is an error, it will be of type *PathError.
func Stat(name string) (FileInfo, error) {
	if len(name) == 0 {
		return nil, &PathError{"Stat", name, syscall.Errno(syscall.ERROR_PATH_NOT_FOUND)}
	}
	if name == DevNull {
		return &devNullStat, nil
	}
	namep, err := syscall.UTF16PtrFromString(fixLongPath(name))
	if err != nil {
		return nil, &PathError{"Stat", name, err}
	}
	// Apparently (see https://github.com/golang/go/issues/19922#issuecomment-300031421)
	// GetFileAttributesEx is fastest approach to get file info.
	// It does not work for symlinks. But symlinks are rare,
	// so try GetFileAttributesEx first.
	var fs fileStat
	err = syscall.GetFileAttributesEx(namep, syscall.GetFileExInfoStandard, (*byte)(unsafe.Pointer(&fs.sys)))
	if err == nil && fs.sys.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT == 0 {
		fs.path = name
		if !isAbs(fs.path) {
			fs.path, err = syscall.FullPath(fs.path)
			if err != nil {
				return nil, &PathError{"FullPath", name, err}
			}
		}
		fs.name = basename(name)
		return &fs, nil
	}
	// Use Windows I/O manager to dereference the symbolic link, as per
	// https://blogs.msdn.microsoft.com/oldnewthing/20100212-00/?p=14963/
	h, err := syscall.CreateFile(namep, 0, 0, nil,
		syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0)
	if err != nil {
		if err == windows.ERROR_SHARING_VIOLATION {
			// try FindFirstFile now that CreateFile failed
			return statWithFindFirstFile(name, namep)
		}
		return nil, &PathError{"CreateFile", name, err}
	}
	defer syscall.CloseHandle(h)

	var d syscall.ByHandleFileInformation
	err = syscall.GetFileInformationByHandle(h, &d)
	if err != nil {
		return nil, &PathError{"GetFileInformationByHandle", name, err}
	}
	return &fileStat{
		name: basename(name),
		sys: syscall.Win32FileAttributeData{
			FileAttributes: d.FileAttributes,
			CreationTime:   d.CreationTime,
			LastAccessTime: d.LastAccessTime,
			LastWriteTime:  d.LastWriteTime,
			FileSizeHigh:   d.FileSizeHigh,
			FileSizeLow:    d.FileSizeLow,
		},
		vol:   d.VolumeSerialNumber,
		idxhi: d.FileIndexHigh,
		idxlo: d.FileIndexLow,
		// fileStat.path is used by os.SameFile to decide if it needs
		// to fetch vol, idxhi and idxlo. But these are already set,
		// so set fileStat.path to "" to prevent os.SameFile doing it again.
		// Also do not set fileStat.filetype, because it is only used for
		// console and stdin/stdout. But you cannot call os.Stat for these.
	}, nil
}

// statWithFindFirstFile is used by Stat to handle special case of statting
// c:\pagefile.sys. We might discover that other files need similar treatment.
func statWithFindFirstFile(name string, namep *uint16) (FileInfo, error) {
	var fd syscall.Win32finddata
	h, err := syscall.FindFirstFile(namep, &fd)
	if err != nil {
		return nil, &PathError{"FindFirstFile", name, err}
	}
	syscall.FindClose(h)

	fullpath := name
	if !isAbs(fullpath) {
		fullpath, err = syscall.FullPath(fullpath)
		if err != nil {
			return nil, &PathError{"FullPath", name, err}
		}
	}
	return &fileStat{
		name: basename(name),
		path: fullpath,
		sys: syscall.Win32FileAttributeData{
			FileAttributes: fd.FileAttributes,
			CreationTime:   fd.CreationTime,
			LastAccessTime: fd.LastAccessTime,
			LastWriteTime:  fd.LastWriteTime,
			FileSizeHigh:   fd.FileSizeHigh,
			FileSizeLow:    fd.FileSizeLow,
		},
	}, nil
}

// Lstat returns the FileInfo structure describing the named file.
// If the file is a symbolic link, the returned FileInfo
// describes the symbolic link. Lstat makes no attempt to follow the link.
// If there is an error, it will be of type *PathError.
func Lstat(name string) (FileInfo, error) {
	if len(name) == 0 {
		return nil, &PathError{"Lstat", name, syscall.Errno(syscall.ERROR_PATH_NOT_FOUND)}
	}
	if name == DevNull {
		return &devNullStat, nil
	}
	fs := &fileStat{name: basename(name)}
	namep, e := syscall.UTF16PtrFromString(fixLongPath(name))
	if e != nil {
		return nil, &PathError{"Lstat", name, e}
	}
	e = syscall.GetFileAttributesEx(namep, syscall.GetFileExInfoStandard, (*byte)(unsafe.Pointer(&fs.sys)))
	if e != nil {
		if e != windows.ERROR_SHARING_VIOLATION {
			return nil, &PathError{"GetFileAttributesEx", name, e}
		}
		// try FindFirstFile now that GetFileAttributesEx failed
		return statWithFindFirstFile(name, namep)
	}
	fs.path = name
	if !isAbs(fs.path) {
		fs.path, e = syscall.FullPath(fs.path)
		if e != nil {
			return nil, &PathError{"FullPath", name, e}
		}
	}
	return fs, nil
}
<commit_msg>os: remove duplicate check from windows os.Stat<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import (
	"internal/syscall/windows"
	"syscall"
	"unsafe"
)

// Stat returns the FileInfo structure describing file.
// If there is an error, it will be of type *PathError.
func (file *File) Stat() (FileInfo, error) {
	if file == nil {
		return nil, ErrInvalid
	}

	if file.isdir() {
		// I don't know any better way to do that for directory
		return Stat(file.dirinfo.path)
	}
	if file.name == DevNull {
		return &devNullStat, nil
	}

	ft, err := file.pfd.GetFileType()
	if err != nil {
		return nil, &PathError{"GetFileType", file.name, err}
	}
	switch ft {
	case syscall.FILE_TYPE_PIPE, syscall.FILE_TYPE_CHAR:
		return &fileStat{name: basename(file.name), filetype: ft}, nil
	}

	var d syscall.ByHandleFileInformation
	err = file.pfd.GetFileInformationByHandle(&d)
	if err != nil {
		return nil, &PathError{"GetFileInformationByHandle", file.name, err}
	}
	return &fileStat{
		name: basename(file.name),
		sys: syscall.Win32FileAttributeData{
			FileAttributes: d.FileAttributes,
			CreationTime:   d.CreationTime,
			LastAccessTime: d.LastAccessTime,
			LastWriteTime:  d.LastWriteTime,
			FileSizeHigh:   d.FileSizeHigh,
			FileSizeLow:    d.FileSizeLow,
		},
		filetype: ft,
		vol:      d.VolumeSerialNumber,
		idxhi:    d.FileIndexHigh,
		idxlo:    d.FileIndexLow,
	}, nil
}

// Stat returns a FileInfo structure describing the named file.
// If there is an error, it will be of type *PathError.
func Stat(name string) (FileInfo, error) {
	if len(name) == 0 {
		return nil, &PathError{"Stat", name, syscall.Errno(syscall.ERROR_PATH_NOT_FOUND)}
	}
	if name == DevNull {
		return &devNullStat, nil
	}
	namep, err := syscall.UTF16PtrFromString(fixLongPath(name))
	if err != nil {
		return nil, &PathError{"Stat", name, err}
	}
	// Apparently (see https://github.com/golang/go/issues/19922#issuecomment-300031421)
	// GetFileAttributesEx is fastest approach to get file info.
	// It does not work for symlinks. But symlinks are rare,
	// so try GetFileAttributesEx first.
	var fs fileStat
	err = syscall.GetFileAttributesEx(namep, syscall.GetFileExInfoStandard, (*byte)(unsafe.Pointer(&fs.sys)))
	if err == nil && fs.sys.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT == 0 {
		fs.path = name
		if !isAbs(fs.path) {
			fs.path, err = syscall.FullPath(fs.path)
			if err != nil {
				return nil, &PathError{"FullPath", name, err}
			}
		}
		fs.name = basename(name)
		return &fs, nil
	}
	// Use Windows I/O manager to dereference the symbolic link, as per
	// https://blogs.msdn.microsoft.com/oldnewthing/20100212-00/?p=14963/
	h, err := syscall.CreateFile(namep, 0, 0, nil,
		syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0)
	if err != nil {
		if err == windows.ERROR_SHARING_VIOLATION {
			// try FindFirstFile now that CreateFile failed
			return statWithFindFirstFile(name, namep)
		}
		return nil, &PathError{"CreateFile", name, err}
	}
	defer syscall.CloseHandle(h)

	var d syscall.ByHandleFileInformation
	err = syscall.GetFileInformationByHandle(h, &d)
	if err != nil {
		return nil, &PathError{"GetFileInformationByHandle", name, err}
	}
	return &fileStat{
		name: basename(name),
		sys: syscall.Win32FileAttributeData{
			FileAttributes: d.FileAttributes,
			CreationTime:   d.CreationTime,
			LastAccessTime: d.LastAccessTime,
			LastWriteTime:  d.LastWriteTime,
			FileSizeHigh:   d.FileSizeHigh,
			FileSizeLow:    d.FileSizeLow,
		},
		vol:   d.VolumeSerialNumber,
		idxhi: d.FileIndexHigh,
		idxlo: d.FileIndexLow,
		// fileStat.path is used by os.SameFile to decide if it needs
		// to fetch vol, idxhi and idxlo. But these are already set,
		// so set fileStat.path to "" to prevent os.SameFile doing it again.
		// Also do not set fileStat.filetype, because it is only used for
		// console and stdin/stdout. But you cannot call os.Stat for these.
	}, nil
}

// statWithFindFirstFile is used by Stat to handle special case of statting
// c:\pagefile.sys. We might discover that other files need similar treatment.
func statWithFindFirstFile(name string, namep *uint16) (FileInfo, error) {
	var fd syscall.Win32finddata
	h, err := syscall.FindFirstFile(namep, &fd)
	if err != nil {
		return nil, &PathError{"FindFirstFile", name, err}
	}
	syscall.FindClose(h)

	fullpath := name
	if !isAbs(fullpath) {
		fullpath, err = syscall.FullPath(fullpath)
		if err != nil {
			return nil, &PathError{"FullPath", name, err}
		}
	}
	return &fileStat{
		name: basename(name),
		path: fullpath,
		sys: syscall.Win32FileAttributeData{
			FileAttributes: fd.FileAttributes,
			CreationTime:   fd.CreationTime,
			LastAccessTime: fd.LastAccessTime,
			LastWriteTime:  fd.LastWriteTime,
			FileSizeHigh:   fd.FileSizeHigh,
			FileSizeLow:    fd.FileSizeLow,
		},
	}, nil
}

// Lstat returns the FileInfo structure describing the named file.
// If the file is a symbolic link, the returned FileInfo
// describes the symbolic link. Lstat makes no attempt to follow the link.
// If there is an error, it will be of type *PathError.
func Lstat(name string) (FileInfo, error) {
	if len(name) == 0 {
		return nil, &PathError{"Lstat", name, syscall.Errno(syscall.ERROR_PATH_NOT_FOUND)}
	}
	if name == DevNull {
		return &devNullStat, nil
	}
	fs := &fileStat{name: basename(name)}
	namep, e := syscall.UTF16PtrFromString(fixLongPath(name))
	if e != nil {
		return nil, &PathError{"Lstat", name, e}
	}
	e = syscall.GetFileAttributesEx(namep, syscall.GetFileExInfoStandard, (*byte)(unsafe.Pointer(&fs.sys)))
	if e != nil {
		if e != windows.ERROR_SHARING_VIOLATION {
			return nil, &PathError{"GetFileAttributesEx", name, e}
		}
		// try FindFirstFile now that GetFileAttributesEx failed
		return statWithFindFirstFile(name, namep)
	}
	fs.path = name
	if !isAbs(fs.path) {
		fs.path, e = syscall.FullPath(fs.path)
		if e != nil {
			return nil, &PathError{"FullPath", name, e}
		}
	}
	return fs, nil
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"

	"github.com/cristianoliveira/ergo/proxy"
)

// RemoveServiceCommand removes a service from the configuration
// and tells the proxy to remove it from the config file.
// USAGE:
// ergo remove myservicename
type RemoveServiceCommand struct {
	Service proxy.Service
}

func findService(service proxy.Service, services map[string]proxy.Service) (*proxy.Service, bool) {
	for _, srv := range services {
		if srv.URL == service.URL || srv.Name == service.Name {
			return &srv, true
		}
	}

	return nil, false
}

// Execute apply the RemoveServiceCommand
func (c RemoveServiceCommand) Execute(config *proxy.Config) (string, error) {
	srv, isPresent := findService(c.Service, config.Services)

	if isPresent {
		err := proxy.RemoveService(config.ConfigFile, *srv)
		if err != nil {
			return "", fmt.Errorf("Failed to remove service cause %s", err)
		}
	} else {
		return "", fmt.Errorf("Service %s not found", c.Service.Name)
	}

	return "Service Removed", nil
}
<commit_msg>refactor: use guard clause<commit_after>package commands

import (
	"fmt"

	"github.com/cristianoliveira/ergo/proxy"
)

// RemoveServiceCommand removes a service from the configuration
// and tells the proxy to remove it from the config file.
// USAGE:
// ergo remove myservicename
type RemoveServiceCommand struct {
	Service proxy.Service
}

func findService(service proxy.Service, services map[string]proxy.Service) (*proxy.Service, bool) {
	for _, srv := range services {
		if srv.URL == service.URL || srv.Name == service.Name {
			return &srv, true
		}
	}

	return nil, false
}

// Execute apply the RemoveServiceCommand
func (c RemoveServiceCommand) Execute(config *proxy.Config) (string, error) {
	srv, isPresent := findService(c.Service, config.Services)

	if !isPresent {
		return "", fmt.Errorf("Service %s not found", c.Service.Name)
	}

	err := proxy.RemoveService(config.ConfigFile, *srv)
	if err != nil {
		return "", fmt.Errorf("Failed to remove service cause %s", err)
	}

	return "Service Removed", nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 - Michael Woolnough <michael.woolnough@gmail.com>
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
//    list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// MemIO implements Read, Write, Seek, Close and other io methods for a byte slice.
package memio

import "io"

// Closed is an error returned when trying to perform an operation after using Close().
type Closed struct{}

func (Closed) Error() string {
	return "operation not permitted when closed"
}

type readMem struct {
	data []byte
	pos  int
}

// Use a byte slice for reading. Implements io.Reader, io.Seeker, io.Closer, io.ReaderAt, io.ByteReader and io.WriterTo.
func Open(data []byte) *readMem {
	return &readMem{data, 0}
}

func (b *readMem) Read(p []byte) (int, error) {
	if b.data == nil {
		return 0, &Closed{}
	} else if b.pos >= len(b.data) {
		return 0, io.EOF
	}
	n := copy(p, b.data[b.pos:])
	b.pos += n
	return n, nil
}

func (b *readMem) ReadByte() (byte, error) {
	if b.data == nil {
		return 0, &Closed{}
	} else if b.pos >= len(b.data) {
		return 0, io.EOF
	}
	c := b.data[b.pos]
	b.pos++
	return c, nil
}

func (b *readMem) Seek(offset int64, whence int) (int64, error) {
	if b.data == nil {
		return 0, &Closed{}
	}
	switch whence {
	case 0:
		b.pos = int(offset)
	case 1:
		b.pos += int(offset)
	case 2:
		b.pos = len(b.data) - int(offset)
	}
	if b.pos < 0 {
		b.pos = 0
	}
	return int64(b.pos), nil
}

func (b *readMem) Close() error {
	b.data = nil
	return nil
}

func (b *readMem) ReadAt(p []byte, off int64) (int, error) {
	if b.data == nil {
		return 0, &Closed{}
	} else if off >= int64(len(b.data)) {
		return 0, io.EOF
	}
	return copy(p, b.data[off:]), nil
}

func (b *readMem) WriteTo(f io.Writer) (int64, error) {
	if b.data == nil {
		return 0, &Closed{}
	} else if b.pos >= len(b.data) {
		return 0, io.EOF
	}
	n, err := f.Write(b.data[b.pos:])
	return int64(n), err
}

type writeMem struct {
	data *[]byte
	pos  int
}

// Use a byte slice for writing. Implements io.Writer, io.Seeker, io.Closer, io.WriterAt, io.ByteWriter and io.ReaderFrom.
func Create(data *[]byte) *writeMem {
	return &writeMem{data, 0}
}

func (b *writeMem) Write(p []byte) (int, error) {
	if b.data == nil {
		return 0, &Closed{}
	}
	b.setSize(b.pos + len(p))
	n := copy((*b.data)[b.pos:], p)
	b.pos += n
	return n, nil
}

func (b *writeMem) WriteAt(p []byte, off int64) (int, error) {
	if b.data == nil {
		return 0, &Closed{}
	}
	b.setSize(int(off) + len(p))
	return copy((*b.data)[off:], p), nil
}

func (b *writeMem) WriteByte(c byte) error {
	if b.data == nil {
		return &Closed{}
	}
	b.setSize(b.pos + 1)
	(*b.data)[b.pos] = c
	b.pos++
	return nil
}

func (b *writeMem) ReadFrom(f io.Reader) (int64, error) {
	if b.data == nil {
		return 0, &Closed{}
	}
	var (
		c   int64
		n   int
		err error
	)
	buf := make([]byte, 1024)
	for {
		n, err = f.Read(buf)
		if n > 0 {
			c += int64(n)
			b.setSize(b.pos + n)
			copy((*b.data)[b.pos:], buf[:n])
		}
		if err != nil {
			if err == io.EOF {
				err = nil
			}
			break
		}
	}
	return c, err
}

func (b *writeMem) Seek(offset int64, whence int) (int64, error) {
	if b.data == nil {
		return 0, &Closed{}
	}
	switch whence {
	case 0:
		b.pos = int(offset)
	case 1:
		b.pos += int(offset)
	case 2:
		b.pos = len(*b.data) - int(offset)
	}
	if b.pos < 0 {
		b.pos = 0
	}
	return int64(b.pos), nil
}

func (b *writeMem) Close() error {
	b.data = nil
	return nil
}

func (b *writeMem) setSize(end int) {
	if end > len(*b.data) {
		if end < cap(*b.data) {
			*b.data = (*b.data)[:end]
		} else {
			var newData []byte
			if len(*b.data) < 512 {
				newData = make([]byte, end, end<<1)
			} else {
				newData = make([]byte, end, end+(end>>2))
			}
			copy(newData, *b.data)
			*b.data = newData
		}
	}
}
<commit_msg>Replaced magic numbers with constants<commit_after>// Copyright (c) 2013 - Michael Woolnough <michael.woolnough@gmail.com>
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
//    list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// MemIO implements Read, Write, Seek, Close and other io methods for a byte slice.
package memio

import "io"

const (
	SEEK_SET int = iota
	SEEK_CURR
	SEEK_END
)

// Closed is an error returned when trying to perform an operation after using Close().
type Closed struct{}

func (Closed) Error() string {
	return "operation not permitted when closed"
}

type readMem struct {
	data []byte
	pos  int
}

// Use a byte slice for reading. Implements io.Reader, io.Seeker, io.Closer, io.ReaderAt, io.ByteReader and io.WriterTo.
func Open(data []byte) *readMem {
	return &readMem{data, 0}
}

func (b *readMem) Read(p []byte) (int, error) {
	if b.data == nil {
		return 0, &Closed{}
	} else if b.pos >= len(b.data) {
		return 0, io.EOF
	}
	n := copy(p, b.data[b.pos:])
	b.pos += n
	return n, nil
}

func (b *readMem) ReadByte() (byte, error) {
	if b.data == nil {
		return 0, &Closed{}
	} else if b.pos >= len(b.data) {
		return 0, io.EOF
	}
	c := b.data[b.pos]
	b.pos++
	return c, nil
}

func (b *readMem) Seek(offset int64, whence int) (int64, error) {
	if b.data == nil {
		return 0, &Closed{}
	}
	switch whence {
	case SEEK_SET:
		b.pos = int(offset)
	case SEEK_CURR:
		b.pos += int(offset)
	case SEEK_END:
		b.pos = len(b.data) - int(offset)
	}
	if b.pos < 0 {
		b.pos = 0
	}
	return int64(b.pos), nil
}

func (b *readMem) Close() error {
	b.data = nil
	return nil
}

func (b *readMem) ReadAt(p []byte, off int64) (int, error) {
	if b.data == nil {
		return 0, &Closed{}
	} else if off >= int64(len(b.data)) {
		return 0, io.EOF
	}
	return copy(p, b.data[off:]), nil
}

func (b *readMem) WriteTo(f io.Writer) (int64, error) {
	if b.data == nil {
		return 0, &Closed{}
	} else if b.pos >= len(b.data) {
		return 0, io.EOF
	}
	n, err := f.Write(b.data[b.pos:])
	return int64(n), err
}

type writeMem struct {
	data *[]byte
	pos  int
}

// Use a byte slice for writing. Implements io.Writer, io.Seeker, io.Closer, io.WriterAt, io.ByteWriter and io.ReaderFrom.
func Create(data *[]byte) *writeMem {
	return &writeMem{data, 0}
}

func (b *writeMem) Write(p []byte) (int, error) {
	if b.data == nil {
		return 0, &Closed{}
	}
	b.setSize(b.pos + len(p))
	n := copy((*b.data)[b.pos:], p)
	b.pos += n
	return n, nil
}

func (b *writeMem) WriteAt(p []byte, off int64) (int, error) {
	if b.data == nil {
		return 0, &Closed{}
	}
	b.setSize(int(off) + len(p))
	return copy((*b.data)[off:], p), nil
}

func (b *writeMem) WriteByte(c byte) error {
	if b.data == nil {
		return &Closed{}
	}
	b.setSize(b.pos + 1)
	(*b.data)[b.pos] = c
	b.pos++
	return nil
}

func (b *writeMem) ReadFrom(f io.Reader) (int64, error) {
	if b.data == nil {
		return 0, &Closed{}
	}
	var (
		c   int64
		n   int
		err error
	)
	buf := make([]byte, 1024)
	for {
		n, err = f.Read(buf)
		if n > 0 {
			c += int64(n)
			b.setSize(b.pos + n)
			copy((*b.data)[b.pos:], buf[:n])
		}
		if err != nil {
			if err == io.EOF {
				err = nil
			}
			break
		}
	}
	return c, err
}

func (b *writeMem) Seek(offset int64, whence int) (int64, error) {
	if b.data == nil {
		return 0, &Closed{}
	}
	switch whence {
	case SEEK_SET:
		b.pos = int(offset)
	case SEEK_CURR:
		b.pos += int(offset)
	case SEEK_END:
		b.pos = len(*b.data) - int(offset)
	}
	if b.pos < 0 {
		b.pos = 0
	}
	return int64(b.pos), nil
}

func (b *writeMem) Close() error {
	b.data = nil
	return nil
}

func (b *writeMem) setSize(end int) {
	if end > len(*b.data) {
		if end < cap(*b.data) {
			*b.data = (*b.data)[:end]
		} else {
			var newData []byte
			if len(*b.data) < 512 {
				newData = make([]byte, end, end<<1)
			} else {
				newData = make([]byte, end, end+(end>>2))
			}
			copy(newData, *b.data)
			*b.data = newData
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Tamás Gulácsi
//
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package grpcer

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"os"
	"reflect"
)

func mergeStreams(w io.Writer, first interface{}, recv interface {
	Recv() (interface{}, error)
},
	Log func(...interface{}) error,
) {
	if Log == nil {
		Log = func(...interface{}) error { return nil }
	}

	slice, notSlice := sliceFields(first)
	if len(slice) == 0 {
		var err error
		part := first
		enc := json.NewEncoder(w)
		for {
			if err := enc.Encode(part); err != nil {
				Log("encode", part, "error", err)
				return
			}

			part, err = recv.Recv()
			if err != nil {
				if err != io.EOF {
					Log("msg", "recv", "error", err)
				}
				break
			}
		}
		return
	}

	Log("slices", slice)
	w.Write([]byte("{"))
	for _, f := range notSlice {
		tw := newTrimWriter(w, "", "\n")
		json.NewEncoder(tw).Encode(f.Name)
		tw.Close()
		w.Write([]byte{':'})
		tw = newTrimWriter(w, "", "\n")
		json.NewEncoder(tw).Encode(f.Value)
		tw.Close()
		w.Write([]byte{','})
	}
	tw := newTrimWriter(w, "", "\n")
	json.NewEncoder(tw).Encode(slice[0].Name)
	tw.Close()
	w.Write([]byte(":"))
	tw = newTrimWriter(w, "", "]\n")
	json.NewEncoder(tw).Encode(slice[0].Value)
	tw.Close()

	files := make(map[string]*os.File, len(slice)-1)
	for _, f := range slice[1:] {
		fh, err := ioutil.TempFile("", "merge-"+f.Name+"-")
		if err != nil {
			Log("tempFile", f.Name, "error", err)
			return
		}
		os.Remove(fh.Name())
		defer fh.Close()
		files[f.Name] = fh
		tw := newTrimWriter(fh, "", "\n")
		json.NewEncoder(tw).Encode(f.Name)
		tw.Close()
		io.WriteString(fh, ":[")
		tw = newTrimWriter(fh, "[", "]\n")
		json.NewEncoder(tw).Encode(f.Value)
		tw.Close()
	}

	var part interface{}
	var err error
	for {
		part, err = recv.Recv()
		if err != nil {
			if err != io.EOF {
				Log("msg", "recv", "error", err)
			}
			break
		}

		S, _ := sliceFields(part)
		for i, j := 0, 0; i < len(S) && j < len(slice); {
			if S[i].Name == slice[j].Name {
				i++
				j++
				continue
			}
			j++
		}
		if S[0].Name == slice[0].Name {
			w.Write([]byte{','})
			tw := newTrimWriter(w, "[", "]\n")
			json.NewEncoder(tw).Encode(S[0].Value)
			tw.Close()
			S = S[1:]
		}
		for _, f := range S {
			fh := files[f.Name]
			if _, err := fh.Write([]byte{','}); err != nil {
				Log("write", fh.Name(), "error", err)
			}
			tw := newTrimWriter(fh, "[", "]\n")
			json.NewEncoder(tw).Encode(f.Value)
			tw.Close()
		}
	}
	w.Write([]byte("]"))

	for _, fh := range files {
		if _, err := fh.Seek(0, 0); err != nil {
			Log("Seek", fh.Name(), "error", err)
			continue
		}
		w.Write([]byte{','})
		io.Copy(w, fh)
		w.Write([]byte{']'})
	}
	w.Write([]byte{'}', '\n'})
}

type field struct {
	Name  string
	Value interface{}
}

func sliceFields(part interface{}) (slice, notSlice []field) {
	rv := reflect.ValueOf(part)
	t := rv.Type()
	n := t.NumField()
	for i := 0; i < n; i++ {
		f := rv.Field(i)
		if f.Type().Kind() != reflect.Slice {
			notSlice = append(notSlice, field{Name: t.Field(i).Name, Value: f.Interface()})
			continue
		}
		if f.IsNil() {
			continue
		}
		slice = append(slice, field{Name: t.Field(i).Name, Value: f.Interface()})
	}
	return slice, notSlice
}

type trimWriter struct {
	w              io.Writer
	prefix, suffix string
	buf            []byte
}

func newTrimWriter(w io.Writer, prefix, suffix string) *trimWriter {
	return &trimWriter{w: w, prefix: prefix, suffix: suffix}
}
func (tw *trimWriter) Write(p []byte) (int, error) {
	n := len(p)
	if tw.prefix != "" {
		if len(tw.prefix) >= len(p) {
			tw.prefix = tw.prefix[len(p):]
			return n, nil
		}
		p = p[len(tw.prefix):]
		tw.prefix = ""
	}

	if len(tw.buf) > 0 && len(tw.buf) >= len(tw.suffix) {
		if _, err := tw.w.Write(tw.buf); err != nil {
			return 0, err
		}
		tw.buf = tw.buf[:0]
	}
	if len(p) <= len(tw.suffix) {
		tw.buf = append(tw.buf, p...)
		return n, nil
	}
	i := len(p) - len(tw.suffix) + len(tw.buf)
	tw.buf = append(tw.buf, p[i:]...)
	_, err := tw.w.Write(p[:i])
	return n, err
}
func (tw *trimWriter) Close() error {
	if tw.suffix == string(tw.buf) {
		return nil
	}
	_, err := tw.w.Write(tw.buf)
	return err
}
<commit_msg>dereference pointer to struct<commit_after>// Copyright 2017 Tamás Gulácsi
//
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package grpcer

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"reflect"
)

func mergeStreams(w io.Writer, first interface{}, recv interface {
	Recv() (interface{}, error)
},
	Log func(...interface{}) error,
) {
	if Log == nil {
		Log = func(...interface{}) error { return nil }
	}

	slice, notSlice := sliceFields(first)
	if len(slice) == 0 {
		var err error
		part := first
		enc := json.NewEncoder(w)
		for {
			if err := enc.Encode(part); err != nil {
				Log("encode", part, "error", err)
				return
			}

			part, err = recv.Recv()
			if err != nil {
				if err != io.EOF {
					Log("msg", "recv", "error", err)
				}
				break
			}
		}
		return
	}

	Log("slices", slice)
	w.Write([]byte("{"))
	for _, f := range notSlice {
		tw := newTrimWriter(w, "", "\n")
		json.NewEncoder(tw).Encode(f.Name)
		tw.Close()
		w.Write([]byte{':'})
		tw = newTrimWriter(w, "", "\n")
		json.NewEncoder(tw).Encode(f.Value)
		tw.Close()
		w.Write([]byte{','})
	}
	tw := newTrimWriter(w, "", "\n")
	json.NewEncoder(tw).Encode(slice[0].Name)
	tw.Close()
	w.Write([]byte(":"))
	tw = newTrimWriter(w, "", "]\n")
	json.NewEncoder(tw).Encode(slice[0].Value)
	tw.Close()

	files := make(map[string]*os.File, len(slice)-1)
	for _, f := range slice[1:] {
		fh, err := ioutil.TempFile("", "merge-"+f.Name+"-")
		if err != nil {
			Log("tempFile", f.Name, "error", err)
			return
		}
		os.Remove(fh.Name())
		defer fh.Close()
		files[f.Name] = fh
		tw := newTrimWriter(fh, "", "\n")
		json.NewEncoder(tw).Encode(f.Name)
		tw.Close()
		io.WriteString(fh, ":[")
		tw = newTrimWriter(fh, "[", "]\n")
		json.NewEncoder(tw).Encode(f.Value)
		tw.Close()
	}

	var part interface{}
	var err error
	for {
		part, err = recv.Recv()
		if err != nil {
			if err != io.EOF {
				Log("msg", "recv", "error", err)
			}
			break
		}

		S, _ := sliceFields(part)
		for i, j := 0, 0; i < len(S) && j < len(slice); {
			if S[i].Name == slice[j].Name {
				i++
				j++
				continue
			}
			j++
		}
		if S[0].Name == slice[0].Name {
			w.Write([]byte{','})
			tw := newTrimWriter(w, "[", "]\n")
			json.NewEncoder(tw).Encode(S[0].Value)
			tw.Close()
			S = S[1:]
		}
		for _, f := range S {
			fh := files[f.Name]
			if _, err := fh.Write([]byte{','}); err != nil {
				Log("write", fh.Name(), "error", err)
			}
			tw := newTrimWriter(fh, "[", "]\n")
			json.NewEncoder(tw).Encode(f.Value)
			tw.Close()
		}
	}
	w.Write([]byte("]"))

	for _, fh := range files {
		if _, err := fh.Seek(0, 0); err != nil {
			Log("Seek", fh.Name(), "error", err)
			continue
		}
		w.Write([]byte{','})
		io.Copy(w, fh)
		w.Write([]byte{']'})
	}
	w.Write([]byte{'}', '\n'})
}

type field struct {
	Name  string
	Value interface{}
}

func sliceFields(part interface{}) (slice, notSlice []field) {
	rv := reflect.ValueOf(part)
	t := rv.Type()
	if t.Kind() == reflect.Ptr {
		rv = rv.Elem()
		t = rv.Type()
	}
	n := t.NumField()
	for i := 0; i < n; i++ {
		f := rv.Field(i)
		if f.Type().Kind() != reflect.Slice {
			notSlice = append(notSlice, field{Name: t.Field(i).Name, Value: f.Interface()})
			continue
		}
		if f.IsNil() {
			continue
		}
		slice = append(slice, field{Name: t.Field(i).Name, Value: f.Interface()})
	}
	return slice, notSlice
}

type trimWriter struct {
	w              io.Writer
	prefix, suffix string
	buf            []byte
}

func newTrimWriter(w io.Writer, prefix, suffix string) *trimWriter {
	return &trimWriter{w: w, prefix: prefix, suffix: suffix}
}
func (tw *trimWriter) Write(p []byte) (int, error) {
	n := len(p)
	if tw.prefix != "" {
		if len(tw.prefix) >= len(p) {
			tw.prefix = tw.prefix[len(p):]
			return n, nil
		}
		p = p[len(tw.prefix):]
		tw.prefix = ""
	}

	if len(tw.buf) > 0 && len(tw.buf) >= len(tw.suffix) {
		if _, err := tw.w.Write(tw.buf); err != nil {
			return 0, err
		}
		tw.buf = tw.buf[:0]
	}
	if len(p) <= len(tw.suffix) {
		tw.buf = append(tw.buf, p...)
		return n, nil
	}
	i := len(p) - len(tw.suffix) + len(tw.buf)
	tw.buf = append(tw.buf, p[i:]...)
	_, err := tw.w.Write(p[:i])
	return n, err
}
func (tw *trimWriter) Close() error {
	if tw.suffix == string(tw.buf) {
		return nil
	}
	_, err := tw.w.Write(tw.buf)
	return err
}
<|endoftext|>


<commit_before>package weed_server

import (
	"bytes"
	"crypto/md5"
	"hash"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var bufPool = sync.Pool{
	New: func() interface{} {
		return new(bytes.Buffer)
	},
}

func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, fileName, contentType string, contentLength int64, so *operation.StorageOption) ([]*filer_pb.FileChunk, hash.Hash, int64, error, []byte) {
	var fileChunks []*filer_pb.FileChunk

	md5Hash := md5.New()
	var partReader = ioutil.NopCloser(io.TeeReader(reader, md5Hash))

	chunkOffset := int64(0)
	var smallContent []byte

	bytesBuffer := bufPool.Get().(*bytes.Buffer)
	defer bufPool.Put(bytesBuffer)
	for {
		limitedReader := io.LimitReader(partReader, int64(chunkSize))

		bytesBuffer.Reset()

		dataSize, err := bytesBuffer.ReadFrom(limitedReader)

		// data, err := ioutil.ReadAll(limitedReader)
		if err != nil {
			return nil, nil, 0, err, nil
		}
		if chunkOffset == 0 && !isAppend(r) {
			if dataSize < fs.option.SaveToFilerLimit || strings.HasPrefix(r.URL.Path, filer.DirectoryEtcRoot) && dataSize < 4*1024 {
				chunkOffset += dataSize
				smallContent = make([]byte, dataSize)
				bytesBuffer.Write(smallContent)
				break
			}
		}

		chunk, uploadErr := fs.dataToChunk(fileName, contentType, bytesBuffer.Bytes(), chunkOffset, so, md5Hash)

		if uploadErr != nil {
			return nil, nil, 0, uploadErr, nil
		}

		// if last chunk exhausted the reader exactly at the border
		if chunk == nil {
			break
		}
		// Save to chunk manifest structure
		fileChunks = append(fileChunks, chunk)

		glog.V(4).Infof("uploaded %s chunk %d to %s [%d,%d)", fileName, len(fileChunks), chunk.FileId, chunkOffset, chunkOffset+int64(chunk.Size))

		// reset variables for the next chunk
		chunkOffset = chunkOffset + int64(chunk.Size)

		// if last chunk was not at full chunk size, but already exhausted the reader
		if int64(chunk.Size) < int64(chunkSize) {
			break
		}
	}

	return fileChunks, md5Hash, chunkOffset, nil, smallContent
}

func (fs *FilerServer) doUpload(urlLocation string, limitedReader io.Reader, fileName string, contentType string, pairMap map[string]string, auth security.EncodedJwt) (*operation.UploadResult, error, []byte) {

	stats.FilerRequestCounter.WithLabelValues("chunkUpload").Inc()
	start := time.Now()
	defer func() {
		stats.FilerRequestHistogram.WithLabelValues("chunkUpload").Observe(time.Since(start).Seconds())
	}()

	uploadResult, err, data := operation.Upload(urlLocation, fileName, fs.option.Cipher, limitedReader, false, contentType, pairMap, auth)
	if uploadResult != nil && uploadResult.RetryCount > 0 {
		stats.FilerRequestCounter.WithLabelValues("chunkUploadRetry").Add(float64(uploadResult.RetryCount))
	}
	return uploadResult, err, data
}

func (fs *FilerServer) dataToChunk(fileName, contentType string, data []byte, chunkOffset int64, so *operation.StorageOption, md5Hash hash.Hash) (*filer_pb.FileChunk, error) {
	dataReader := util.NewBytesReader(data)

	// retry to assign a different file id
	var fileId, urlLocation string
	var auth security.EncodedJwt
	var uploadErr error
	var uploadResult *operation.UploadResult
	for i := 0; i < 3; i++ {
		// assign one file id for one chunk
		fileId, urlLocation, auth, uploadErr = fs.assignNewFileInfo(so)
		if uploadErr != nil {
			glog.V(4).Infof("retry later due to assign error: %v", uploadErr)
			time.Sleep(time.Duration(i+1) * 251 * time.Millisecond)
			continue
		}

		// upload the chunk to the volume server
		uploadResult, uploadErr, _ = fs.doUpload(urlLocation, dataReader, fileName, contentType, nil, auth)
		if uploadErr != nil {
			glog.V(4).Infof("retry later due to upload error: %v", uploadErr)
			time.Sleep(time.Duration(i+1) * 251 * time.Millisecond)
			continue
		}
		break
	}
	if uploadErr != nil {
		glog.Errorf("upload error: %v", uploadErr)
		return nil, uploadErr
	}

	// if last chunk exhausted the reader exactly at the border
	if uploadResult.Size == 0 {
		return nil, nil
	}
	if chunkOffset == 0 {
		uploadedMd5 := util.Base64Md5ToBytes(uploadResult.ContentMd5)
		readedMd5 := md5Hash.Sum(nil)
		if !bytes.Equal(uploadedMd5, readedMd5) {
			glog.Errorf("md5 %x does not match %x uploaded chunk %s to the volume server", readedMd5, uploadedMd5, uploadResult.Name)
		}
	}

	return uploadResult.ToPbFileChunk(fileId, chunkOffset), nil
}
<commit_msg>filer: parallel data upload<commit_after>package weed_server

import (
	"bytes"
	"crypto/md5"
	"hash"
	"io"
	"io/ioutil"
	"net/http"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var bufPool = sync.Pool{
	New: func() interface{} {
		return new(bytes.Buffer)
	},
}

func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, fileName, contentType string, contentLength int64, so *operation.StorageOption) ([]*filer_pb.FileChunk, hash.Hash, int64, error, []byte) {
	var fileChunks []*filer_pb.FileChunk

	md5Hash := md5.New()
	var partReader = ioutil.NopCloser(io.TeeReader(reader, md5Hash))

	chunkOffset := int64(0)
	var smallContent []byte
	var uploadErr error

	var wg sync.WaitGroup
	for {

		// need to throttle this for large files
		bytesBuffer := bufPool.Get().(*bytes.Buffer)
		defer bufPool.Put(bytesBuffer)

		limitedReader := io.LimitReader(partReader, int64(chunkSize))

		bytesBuffer.Reset()

		dataSize, err := bytesBuffer.ReadFrom(limitedReader)

		// data, err := ioutil.ReadAll(limitedReader)
		if err != nil || dataSize == 0 {
			return nil, md5Hash, 0, err, nil
		}
		if chunkOffset == 0 && !isAppend(r) {
			if dataSize < fs.option.SaveToFilerLimit || strings.HasPrefix(r.URL.Path, filer.DirectoryEtcRoot) && dataSize < 4*1024 {
				chunkOffset += dataSize
				smallContent = make([]byte, dataSize)
				bytesBuffer.Write(smallContent)
				break
			}
		}

		wg.Add(1)
		go func(offset int64) {
			defer wg.Done()

			chunk, toChunkErr := fs.dataToChunk(fileName, contentType, bytesBuffer.Bytes(), offset, so, md5Hash)
			if toChunkErr != nil {
				uploadErr = toChunkErr
			}
			if chunk != nil {
				fileChunks = append(fileChunks, chunk)
				glog.V(4).Infof("uploaded %s chunk %d to %s [%d,%d)", fileName, len(fileChunks), chunk.FileId, offset, offset+int64(chunk.Size))
			}
		}(chunkOffset)

		// reset variables for the next chunk
		chunkOffset = chunkOffset + dataSize

		// if last chunk was not at full chunk size, but already exhausted the reader
		if dataSize < int64(chunkSize) {
			break
		}
	}

	wg.Wait()

	if uploadErr != nil {
		return nil, md5Hash, 0, uploadErr, nil
	}

	sort.Slice(fileChunks, func(i, j int) bool {
		return fileChunks[i].Offset < fileChunks[j].Offset
	})

	return fileChunks, md5Hash, chunkOffset, nil, smallContent
}

func (fs *FilerServer) doUpload(urlLocation string, limitedReader io.Reader, fileName string, contentType string, pairMap map[string]string, auth security.EncodedJwt) (*operation.UploadResult, error, []byte) {

	stats.FilerRequestCounter.WithLabelValues("chunkUpload").Inc()
	start := time.Now()
	defer func() {
		stats.FilerRequestHistogram.WithLabelValues("chunkUpload").Observe(time.Since(start).Seconds())
	}()

	uploadResult, err, data := operation.Upload(urlLocation, fileName, fs.option.Cipher, limitedReader, false, contentType, pairMap, auth)
	if uploadResult != nil && uploadResult.RetryCount > 0 {
		stats.FilerRequestCounter.WithLabelValues("chunkUploadRetry").Add(float64(uploadResult.RetryCount))
	}
	return uploadResult, err, data
}

func (fs *FilerServer) dataToChunk(fileName, contentType string, data []byte, chunkOffset int64, so *operation.StorageOption, md5Hash hash.Hash) (*filer_pb.FileChunk, error) {
	dataReader := util.NewBytesReader(data)

	// retry to assign a different file id
	var fileId, urlLocation string
	var auth security.EncodedJwt
	var uploadErr error
	var uploadResult *operation.UploadResult
	for i := 0; i < 3; i++ {
		// assign one file id for one chunk
		fileId, urlLocation, auth, uploadErr = fs.assignNewFileInfo(so)
		if uploadErr != nil {
			glog.V(4).Infof("retry later due to assign error: %v", uploadErr)
			time.Sleep(time.Duration(i+1) * 251 * time.Millisecond)
			continue
		}

		// upload the chunk to the volume server
		uploadResult, uploadErr, _ = fs.doUpload(urlLocation, dataReader, fileName, contentType, nil, auth)
		if uploadErr != nil {
			glog.V(4).Infof("retry later due to upload error: %v", uploadErr)
			time.Sleep(time.Duration(i+1) * 251 * time.Millisecond)
			continue
		}
		break
	}
	if uploadErr != nil {
		glog.Errorf("upload error: %v", uploadErr)
		return nil, uploadErr
	}

	// if last chunk exhausted the reader exactly at the border
	if uploadResult.Size == 0 {
		return nil, nil
	}
	if chunkOffset == 0 {
		uploadedMd5 := util.Base64Md5ToBytes(uploadResult.ContentMd5)
		readedMd5 := md5Hash.Sum(nil)
		if !bytes.Equal(uploadedMd5, readedMd5) {
			glog.Errorf("md5 %x does not match %x uploaded chunk %s to the volume server", readedMd5, uploadedMd5, uploadResult.Name)
		}
	}

	return uploadResult.ToPbFileChunk(fileId, chunkOffset), nil
}
<|endoftext|>


<commit_before>package readline

import (
	"bytes"
	"strings"

	"github.com/chzyer/readline/runes"
)

type PrefixCompleter struct {
	Name     []rune
	Children []*PrefixCompleter
}

func (p *PrefixCompleter) Tree(prefix string) string {
	buf := bytes.NewBuffer(nil)
	p.Print(prefix, 0, buf)
	return buf.String()
}

func (p *PrefixCompleter) Print(prefix string, level int, buf *bytes.Buffer) {
	if strings.TrimSpace(string(p.Name)) != "" {
		buf.WriteString(prefix)
		if level > 0 {
			buf.WriteString("├")
			buf.WriteString(strings.Repeat("─", (level*4)-2))
			buf.WriteString(" ")
		}
		buf.WriteString(string(p.Name) + "\n")
		level++
	}
	for _, ch := range p.Children {
		ch.Print(prefix, level, buf)
	}
}

func NewPrefixCompleter(pc ...*PrefixCompleter) *PrefixCompleter {
	return PcItem("", pc...)
}

func PcItem(name string, pc ...*PrefixCompleter) *PrefixCompleter {
	name += " "
	return &PrefixCompleter{
		Name:     []rune(name),
		Children: pc,
	}
}

func (p *PrefixCompleter) Do(line []rune, pos int) (newLine [][]rune, offset int) {
	line = line[:pos]
	goNext := false
	var lineCompleter *PrefixCompleter
	for _, child := range p.Children {
		if len(line) >= len(child.Name) {
			if runes.HasPrefix(line, child.Name) {
				if len(line) == len(child.Name) {
					newLine = append(newLine, []rune{' '})
				} else {
					newLine = append(newLine, child.Name)
				}
				offset = len(child.Name)
				lineCompleter = child
				goNext = true
			}
		} else {
			if runes.HasPrefix(child.Name, line) {
				newLine = append(newLine, child.Name[len(line):])
				offset = len(line)
				lineCompleter = child
			}
		}
	}

	if len(newLine) != 1 {
		return
	}

	tmpLine := make([]rune, 0, len(line))
	for i := offset; i < len(line); i++ {
		if line[i] == ' ' {
			continue
		}

		tmpLine = append(tmpLine, line[i:]...)
		return lineCompleter.Do(tmpLine, len(tmpLine))
	}

	if goNext {
		return lineCompleter.Do(nil, 0)
	}
	return
}
<commit_msg>Hack to allow dynamic content in autocompletion.<commit_after>package readline

import (
	"bytes"
	"strings"

	"github.com/chzyer/readline/runes"
)

type DynChildrenFunc func() [][]rune

type PrefixCompleter struct {
	Name        []rune
	Children    []*PrefixCompleter
	DynChildren DynChildrenFunc
}

func empty() [][]rune {
	return [][]rune{}
}

func (p *PrefixCompleter) Tree(prefix string) string {
	buf := bytes.NewBuffer(nil)
	p.Print(prefix, 0, buf)
	return buf.String()
}

func (p *PrefixCompleter) Print(prefix string, level int, buf *bytes.Buffer) {
	if strings.TrimSpace(string(p.Name)) != "" {
		buf.WriteString(prefix)
		if level > 0 {
			buf.WriteString("├")
			buf.WriteString(strings.Repeat("─", (level*4)-2))
			buf.WriteString(" ")
		}
		buf.WriteString(string(p.Name) + "\n")
		level++
	}
	for _, ch := range p.Children {
		ch.Print(prefix, level, buf)
	}
}

func NewPrefixCompleter(pc ...*PrefixCompleter) *PrefixCompleter {
	return PcItem("", pc...)
}

func PcDyn(name string, term DynChildrenFunc) *PrefixCompleter {
	name += " "
	return &PrefixCompleter{
		Name:        []rune(name),
		DynChildren: term,
	}
}

func PcItem(name string, pc ...*PrefixCompleter) *PrefixCompleter {
	name += " "
	return &PrefixCompleter{
		Name:        []rune(name),
		Children:    pc,
		DynChildren: empty,
	}
}

func (p *PrefixCompleter) Do(line []rune, pos int) (newLine [][]rune, offset int) {
	end := PcItem("")
	line = line[:pos]
	goNext := false
	var lineCompleter *PrefixCompleter
	for _, child := range p.Children {
		if len(line) >= len(child.Name) {
			if runes.HasPrefix(line, child.Name) {
				if len(line) == len(child.Name) {
					newLine = append(newLine, []rune{' '})
				} else {
					newLine = append(newLine, child.Name)
				}
				offset = len(child.Name)
				lineCompleter = child
				goNext = true
			}
		} else {
			if runes.HasPrefix(child.Name, line) {
				newLine = append(newLine, child.Name[len(line):])
				offset = len(line)
				lineCompleter = child
			}
		}
	}
	for _, dynChild := range p.DynChildren() {
		if len(line) >= len(dynChild) {
			if runes.HasPrefix(line, dynChild) {
				if len(line) == len(dynChild) {
					newLine = append(newLine, []rune{' '})
				} else {
					newLine = append(newLine, dynChild)
				}
				offset = len(dynChild)
				lineCompleter = end
			}
		} else {
			if runes.HasPrefix(dynChild, line) {
				newLine = append(newLine, dynChild[len(line):])
				offset = len(line)
				lineCompleter = end
			}
		}
	}

	if len(newLine) != 1 {
		return
	}

	tmpLine := make([]rune, 0, len(line))
	for i := offset; i < len(line); i++ {
		if line[i] == ' ' {
			continue
		}

		tmpLine = append(tmpLine, line[i:]...)
		return lineCompleter.Do(tmpLine, len(tmpLine))
	}

	if goNext {
		return lineCompleter.Do(nil, 0)
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/go-ole/go-ole"
	"github.com/go-ole/go-ole/oleutil"
)

// copy & paste from
// http://stackoverflow.com/questions/20365286/query-wmi-from-go
func listComPorts() {
	// init COM
	ole.CoInitialize(0)
	defer ole.CoUninitialize()

	unknown, _ := oleutil.CreateObject("WbemScripting.SWbemLocator")
	defer unknown.Release()

	wmi, _ := unknown.QueryInterface(ole.IID_IDispatch)
	defer wmi.Release()

	// service is a SWbemServices
	serviceRaw, _ := oleutil.CallMethod(wmi, "ConnectServer")
	service := serviceRaw.ToIDispatch()
	defer service.Release()

	// result is a SWBemObjectSet
	resultRaw, _ := oleutil.CallMethod(service, "ExecQuery", "SELECT * FROM WIN32_SerialPort")
	result := resultRaw.ToIDispatch()
	defer result.Release()

	countVar, _ := oleutil.GetProperty(result, "Count")
	count := int(countVar.Val)

	var ports [][]string
	re := regexp.MustCompile(`(.*) \((.*?)\)$`)
	for i := 0; i < count; i++ {
		// item is a SWbemObject, but really a Win32_Process
		itemRaw, _ := oleutil.CallMethod(result, "ItemIndex", i)
		item := itemRaw.ToIDispatch()
		defer item.Release()

		property, _ := oleutil.GetProperty(item, "Name")

		port := re.FindAllStringSubmatch(property.ToString(), -1)
		ports = append(ports, port[0][1:3])
	}

	// " " pading
	maxlen := 0
	for i := 0; i < len(ports); i++ {
		if maxlen < len(ports[i][1]) {
			maxlen = len(ports[i][1])
		}
	}
	for i := 0; i < len(ports); i++ {
		fmt.Println(ports[i][1] + strings.Repeat(" ", maxlen-len(ports[i][1])+1) + ": " + ports[i][0])
	}
}
<commit_msg>Improve device listing<commit_after>package main

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/go-ole/go-ole"
	"github.com/go-ole/go-ole/oleutil"
)

// copy & paste from
// http://stackoverflow.com/questions/20365286/query-wmi-from-go
func listComPorts() {
	// init COM
	ole.CoInitialize(0)
	defer ole.CoUninitialize()

	unknown, _ := oleutil.CreateObject("WbemScripting.SWbemLocator")
	defer unknown.Release()

	wmi, _ := unknown.QueryInterface(ole.IID_IDispatch)
	defer wmi.Release()

	// service is a SWbemServices
	serviceRaw, _ := oleutil.CallMethod(wmi, "ConnectServer")
	service := serviceRaw.ToIDispatch()
	defer service.Release()

	// result is a SWBemObjectSet
	resultRaw, _ := oleutil.CallMethod(service, "ExecQuery", "SELECT * FROM Win32_PnPEntity WHERE Name LIKE '%(COM%)'")
	result := resultRaw.ToIDispatch()
	defer result.Release()

	countVar, _ := oleutil.GetProperty(result, "Count")
	count := int(countVar.Val)

	var ports [][]string
	re := regexp.MustCompile(`(.*) \((.*?)\)$`)
	for i := 0; i < count; i++ {
		// item is a SWbemObject, but really a Win32_Process
		itemRaw, _ := oleutil.CallMethod(result, "ItemIndex", i)
		item := itemRaw.ToIDispatch()
		defer item.Release()

		property, _ := oleutil.GetProperty(item, "Name")
		port := re.FindAllStringSubmatch(property.ToString(), -1)
		ports = append(ports, port[0][1:3])
	}

	// " " pading
	maxlen := 0
	for i := 0; i < len(ports); i++ {
		if maxlen < len(ports[i][1]) {
			maxlen = len(ports[i][1])
		}
	}
	for i := 0; i < len(ports); i++ {
		fmt.Println(ports[i][1] + strings.Repeat(" ", maxlen-len(ports[i][1])+1) + ": " + ports[i][0])
	}
}
<|endoftext|>