<commit_before>package topic

import (
	"bytes"
	"errors"
	"fmt"
	"path/filepath"

	"github.com/tylertreat/BoomFilters"

	"github.com/celrenheit/sandflake"
	"github.com/celrenheit/sandglass/sgproto"
	"github.com/celrenheit/sandglass/sgutils"
	"github.com/celrenheit/sandglass/storage"
	"github.com/celrenheit/sandglass/storage/badger"
	"github.com/celrenheit/sandglass/storage/rocksdb"
	"github.com/celrenheit/sandglass/storage/scommons"
	"github.com/gogo/protobuf/proto"
	"github.com/willf/bloom"
)

var (
	ErrNoKeySet = errors.New("ErrNoKeySet")
)

type Partition struct {
	db       storage.Storage
	Id       string
	Replicas []string
	idgen    sandflake.Generator
	basepath string
	topic    *Topic

	bf  *bloom.BloomFilter // TODO: persist bloom filter
	ibf boom.Filter
}

func (t *Partition) InitStore(basePath string) error {
	t.bf = bloom.NewWithEstimates(1e3, 1e-2)
	t.ibf = boom.NewInverseBloomFilter(1e3)
	msgdir := filepath.Join(basePath, t.Id)
	if err := sgutils.MkdirIfNotExist(msgdir); err != nil {
		return err
	}

	var err error
	switch t.topic.StorageDriver {
	case sgproto.StorageDriver_Badger:
		t.db, err = badger.NewStorage(msgdir)
	case sgproto.StorageDriver_RocksDB:
		t.db, err = rocksdb.NewStorage(msgdir)
	default:
		panic(fmt.Sprintf("unknown storage driver: %v for topic: %v", t.topic.StorageDriver, t.topic.Name))
	}
	if err != nil {
		return err
	}

	t.basepath = basePath
	return nil
}

func (t *Partition) String() string {
	return t.Id
}

func (t *Partition) getStorageKey(msg *sgproto.Message) []byte {
	var storekey []byte
	switch t.topic.Kind {
	case sgproto.TopicKind_TimerKind:
		storekey = msg.Offset[:]
	case sgproto.TopicKind_CompactedKind:
		storekey = joinKeys(msg.Key, msg.ClusteringKey)
	default:
		panic("INVALID STORAGE KIND: " + t.topic.Kind.String())
	}
	return scommons.PrependPrefix(scommons.MsgPrefix, storekey)
}

func (s *Partition) GetMessage(offset sandflake.ID, k, suffix []byte) (*sgproto.Message, error) {
	switch s.topic.Kind {
	case sgproto.TopicKind_TimerKind:
		val, err := s.db.Get(scommons.PrependPrefix(scommons.MsgPrefix, offset[:]))
		if err != nil {
			return nil, err
		}

		var msg sgproto.Message
		err = proto.Unmarshal(val, &msg)
		if err != nil {
			return nil, err
		}

		return &msg, nil
	case sgproto.TopicKind_CompactedKind:
		val := s.db.LastKVForPrefix(scommons.PrependPrefix(scommons.MsgPrefix, k), suffix)
		if val == nil {
			return nil, nil
		}

		var msg sgproto.Message
		err := proto.Unmarshal(val, &msg)
		if err != nil {
			return nil, err
		}

		return &msg, nil
	default:
		panic("INVALID STORAGE KIND: " + s.topic.Kind.String())
	}
}

func (t *Partition) PutMessage(msg *sgproto.Message) error {
	storagekey := t.getStorageKey(msg)
	if msg.Index == sandflake.Nil {
		msg.Index = t.NextID()
	}

	val, err := proto.Marshal(msg)
	if err != nil {
		return err
	}

	entries := []*storage.Entry{
		{Key: storagekey, Value: val},                                 // msg
		{Key: t.newWALKey(msg.Index, storagekey), Value: []byte("X")}, // wal
	}

	err = t.db.BatchPut(entries)
	if err != nil {
		return err
	}

	t.bf.Add(storagekey)
	t.ibf.Add(storagekey)

	return nil
}

func (t *Partition) HasKey(key, clusterKey []byte) (bool, error) {
	switch t.topic.Kind {
	case sgproto.TopicKind_CompactedKind:
	default:
		panic("not compacted topic")
	}

	pk := joinKeys(key, clusterKey)
	existKey := scommons.PrependPrefix(scommons.MsgPrefix, pk)

	if t.ibf.Test(existKey) {
		return true, nil
	}

	if !t.bf.Test(existKey) {
		return false, nil
	}

	msg, err := t.GetMessage(sandflake.Nil, pk, nil)
	if err != nil {
		return false, err
	}

	if msg == nil {
		return false, nil
	}

	return msg.Offset != sandflake.Nil, nil
}

func (t *Partition) newWALKey(index sandflake.ID, key []byte) []byte {
	return bytes.Join([][]byte{scommons.WalPrefix, index.Bytes(), key}, []byte("/"))
}

func (t *Partition) BatchPutMessages(msgs []*sgproto.Message) error {
	entries := make([]*storage.Entry, len(msgs))
	for i, msg := range msgs {
		if msg.Offset == sandflake.Nil {
			return ErrNoKeySet
		}
		if msg.Index == sandflake.Nil {
			msg.Index = t.NextID()
		}
		storagekey := t.getStorageKey(msg)
		val, err := proto.Marshal(msg)
		if err != nil {
			return err
		}
		t.bf.Add(storagekey)
		t.ibf.Add(storagekey)
		entries[i] = &storage.Entry{
			Key:   storagekey,
			Value: val,
		}
	}

	if len(entries) == 0 {
		return nil
	}

	err := t.db.BatchPut(entries)
	if err != nil {
		return err
	}

	for i, e := range entries {
		e.Key = t.newWALKey(msgs[i].Index, e.Key)
		e.Value = nil
	}

	return t.db.BatchPut(entries)
}

func (t *Partition) NextID() sandflake.ID {
	return t.idgen.Next()
}

func (p *Partition) ForRange(min, max sandflake.ID, fn func(msg *sgproto.Message) error) error {
	var lastKey []byte
	switch p.topic.Kind {
	case sgproto.TopicKind_TimerKind:
		return p.db.ForRange(min, max, func(msg *sgproto.Message) error {
			err := fn(msg)
			return err
		})
	case sgproto.TopicKind_CompactedKind:
		it := scommons.NewMessageIterator(p.db, &storage.IterOptions{
			FetchValues: true,
			Reverse:     true,
		})
		defer it.Close()
		for m := it.Rewind(); it.Valid(); m = it.Next() {
			if lastKey == nil || !bytes.Equal(m.Key, lastKey) {
				err := fn(m)
				if err != nil {
					return err
				}
				lastKey = m.Key
			}
		}
	default:
		panic("unknown topic kind: " + p.topic.Kind.String())
	}
	return nil
}

func (p *Partition) Close() error {
	return p.db.Close()
}

func (p *Partition) Iter() storage.MessageIterator {
	return scommons.NewMessageIterator(p.db, &storage.IterOptions{
		FetchValues: true,
		Reverse:     false,
	})
}

func (p *Partition) RangeFromWAL(min []byte, fn func(*sgproto.Message) error) error {
	return p.db.ForEachKey(min, func(k []byte) error {
		if len(k) == 0 {
			panic("empty wal key")
		}

		k = k[len(scommons.WalPrefix)+1+sandflake.Size+1:]
		msg, err := p.getMessageByStorageKey(k)
		if err != nil {
			return err
		}

		return fn(msg)
	})
}

func (p *Partition) LastWALEntry() []byte {
	return p.db.LastKeyForPrefix(scommons.WalPrefix)
}

func (p *Partition) LastMessage() (*sgproto.Message, error) {
	key := p.db.LastKeyForPrefix(scommons.WalPrefix)
	if key == nil {
		return nil, nil
	}

	if len(key) == 0 {
		panic("empty wal key")
	}

	key = key[len(scommons.WalPrefix)+1+sandflake.Size+1:]
	return p.getMessageByStorageKey(key)
}

func (p *Partition) getMessageByStorageKey(k []byte) (*sgproto.Message, error) {
	b, err := p.db.Get(k)
	if err != nil {
		return nil, err
	}

	if b == nil {
		panic("should not happend, key in wal should always be also in msgs")
	}

	var msg sgproto.Message
	err = proto.Unmarshal(b, &msg)
	if err != nil {
		return nil, err
	}

	return &msg, nil
}

func joinKeys(key, clusterKey []byte) []byte {
	return bytes.Join([][]byte{
		key,
		clusterKey,
	}, []byte{'/'})
}
<commit_msg>fix partition_test<commit_after>package topic

import (
	"bytes"
	"errors"
	"fmt"
	"path/filepath"

	"github.com/tylertreat/BoomFilters"

	"github.com/celrenheit/sandflake"
	"github.com/celrenheit/sandglass/sgproto"
	"github.com/celrenheit/sandglass/sgutils"
	"github.com/celrenheit/sandglass/storage"
	"github.com/celrenheit/sandglass/storage/badger"
	"github.com/celrenheit/sandglass/storage/rocksdb"
	"github.com/celrenheit/sandglass/storage/scommons"
	"github.com/gogo/protobuf/proto"
	"github.com/willf/bloom"
)

var (
	ErrNoKeySet = errors.New("ErrNoKeySet")
)

type Partition struct {
	db       storage.Storage
	Id       string
	Replicas []string
	idgen    sandflake.Generator
	basepath string
	topic    *Topic

	bf  *bloom.BloomFilter // TODO: persist bloom filter
	ibf boom.Filter
}

func (t *Partition) InitStore(basePath string) error {
	t.bf = bloom.NewWithEstimates(1e3, 1e-2)
	t.ibf = boom.NewInverseBloomFilter(1e3)
	msgdir := filepath.Join(basePath, t.Id)
	if err := sgutils.MkdirIfNotExist(msgdir); err != nil {
		return err
	}

	var err error
	switch t.topic.StorageDriver {
	case sgproto.StorageDriver_Badger:
		t.db, err = badger.NewStorage(msgdir)
	case sgproto.StorageDriver_RocksDB:
		t.db, err = rocksdb.NewStorage(msgdir)
	default:
		panic(fmt.Sprintf("unknown storage driver: %v for topic: %v", t.topic.StorageDriver, t.topic.Name))
	}
	if err != nil {
		return err
	}

	t.basepath = basePath
	return nil
}

func (t *Partition) String() string {
	return t.Id
}

func (t *Partition) getStorageKey(msg *sgproto.Message) []byte {
	var storekey []byte
	switch t.topic.Kind {
	case sgproto.TopicKind_TimerKind:
		storekey = msg.Offset[:]
	case sgproto.TopicKind_CompactedKind:
		storekey = msg.Key
		if len(msg.ClusteringKey) > 0 {
			storekey = joinKeys(msg.Key, msg.ClusteringKey)
		}
	default:
		panic("INVALID STORAGE KIND: " + t.topic.Kind.String())
	}
	return scommons.PrependPrefix(scommons.MsgPrefix, storekey)
}

func (s *Partition) GetMessage(offset sandflake.ID, k, suffix []byte) (*sgproto.Message, error) {
	switch s.topic.Kind {
	case sgproto.TopicKind_TimerKind:
		val, err := s.db.Get(scommons.PrependPrefix(scommons.MsgPrefix, offset[:]))
		if err != nil {
			return nil, err
		}

		var msg sgproto.Message
		err = proto.Unmarshal(val, &msg)
		if err != nil {
			return nil, err
		}

		return &msg, nil
	case sgproto.TopicKind_CompactedKind:
		val := s.db.LastKVForPrefix(scommons.PrependPrefix(scommons.MsgPrefix, k), suffix)
		if val == nil {
			return nil, nil
		}

		var msg sgproto.Message
		err := proto.Unmarshal(val, &msg)
		if err != nil {
			return nil, err
		}

		return &msg, nil
	default:
		panic("INVALID STORAGE KIND: " + s.topic.Kind.String())
	}
}

func (t *Partition) PutMessage(msg *sgproto.Message) error {
	storagekey := t.getStorageKey(msg)
	if msg.Index == sandflake.Nil {
		msg.Index = t.NextID()
	}

	val, err := proto.Marshal(msg)
	if err != nil {
		return err
	}

	entries := []*storage.Entry{
		{Key: storagekey, Value: val},                                 // msg
		{Key: t.newWALKey(msg.Index, storagekey), Value: []byte("X")}, // wal
	}

	err = t.db.BatchPut(entries)
	if err != nil {
		return err
	}

	t.bf.Add(storagekey)
	t.ibf.Add(storagekey)

	return nil
}

func (t *Partition) HasKey(key, clusterKey []byte) (bool, error) {
	switch t.topic.Kind {
	case sgproto.TopicKind_CompactedKind:
	default:
		panic("not compacted topic")
	}

	pk := joinKeys(key, clusterKey)
	existKey := scommons.PrependPrefix(scommons.MsgPrefix, pk)

	if t.ibf.Test(existKey) {
		return true, nil
	}

	if !t.bf.Test(existKey) {
		return false, nil
	}

	msg, err := t.GetMessage(sandflake.Nil, pk, nil)
	if err != nil {
		return false, err
	}

	if msg == nil {
		return false, nil
	}

	return msg.Offset != sandflake.Nil, nil
}

func (t *Partition) newWALKey(index sandflake.ID, key []byte) []byte {
	return bytes.Join([][]byte{scommons.WalPrefix, index.Bytes(), key}, []byte("/"))
}

func (t *Partition) BatchPutMessages(msgs []*sgproto.Message) error {
	entries := make([]*storage.Entry, len(msgs))
	for i, msg := range msgs {
		if msg.Offset == sandflake.Nil {
			return ErrNoKeySet
		}
		if msg.Index == sandflake.Nil {
			msg.Index = t.NextID()
		}
		storagekey := t.getStorageKey(msg)
		val, err := proto.Marshal(msg)
		if err != nil {
			return err
		}
		t.bf.Add(storagekey)
		t.ibf.Add(storagekey)
		entries[i] = &storage.Entry{
			Key:   storagekey,
			Value: val,
		}
	}

	if len(entries) == 0 {
		return nil
	}

	err := t.db.BatchPut(entries)
	if err != nil {
		return err
	}

	for i, e := range entries {
		e.Key = t.newWALKey(msgs[i].Index, e.Key)
		e.Value = nil
	}

	return t.db.BatchPut(entries)
}

func (t *Partition) NextID() sandflake.ID {
	return t.idgen.Next()
}

func (p *Partition) ForRange(min, max sandflake.ID, fn func(msg *sgproto.Message) error) error {
	var lastKey []byte
	switch p.topic.Kind {
	case sgproto.TopicKind_TimerKind:
		return p.db.ForRange(min, max, func(msg *sgproto.Message) error {
			err := fn(msg)
			return err
		})
	case sgproto.TopicKind_CompactedKind:
		it := scommons.NewMessageIterator(p.db, &storage.IterOptions{
			FetchValues: true,
			Reverse:     true,
		})
		defer it.Close()
		for m := it.Rewind(); it.Valid(); m = it.Next() {
			if lastKey == nil || !bytes.Equal(m.Key, lastKey) {
				err := fn(m)
				if err != nil {
					return err
				}
				lastKey = m.Key
			}
		}
	default:
		panic("unknown topic kind: " + p.topic.Kind.String())
	}
	return nil
}

func (p *Partition) Close() error {
	return p.db.Close()
}

func (p *Partition) Iter() storage.MessageIterator {
	return scommons.NewMessageIterator(p.db, &storage.IterOptions{
		FetchValues: true,
		Reverse:     false,
	})
}

func (p *Partition) RangeFromWAL(min []byte, fn func(*sgproto.Message) error) error {
	return p.db.ForEachKey(min, func(k []byte) error {
		if len(k) == 0 {
			panic("empty wal key")
		}

		k = k[len(scommons.WalPrefix)+1+sandflake.Size+1:]
		msg, err := p.getMessageByStorageKey(k)
		if err != nil {
			return err
		}

		return fn(msg)
	})
}

func (p *Partition) LastWALEntry() []byte {
	return p.db.LastKeyForPrefix(scommons.WalPrefix)
}

func (p *Partition) LastMessage() (*sgproto.Message, error) {
	key := p.db.LastKeyForPrefix(scommons.WalPrefix)
	if key == nil {
		return nil, nil
	}

	if len(key) == 0 {
		panic("empty wal key")
	}

	key = key[len(scommons.WalPrefix)+1+sandflake.Size+1:]
	return p.getMessageByStorageKey(key)
}

func (p *Partition) getMessageByStorageKey(k []byte) (*sgproto.Message, error) {
	b, err := p.db.Get(k)
	if err != nil {
		return nil, err
	}

	if b == nil {
		panic("should not happend, key in wal should always be also in msgs")
	}

	var msg sgproto.Message
	err = proto.Unmarshal(b, &msg)
	if err != nil {
		return nil, err
	}

	return &msg, nil
}

func joinKeys(key, clusterKey []byte) []byte {
	return bytes.Join([][]byte{
		key,
		clusterKey,
	}, []byte{'/'})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/fsouza/go-dockerclient"
	"github.com/progrium/go-shell"
	"github.com/spf13/cobra"
)

func init() {
	Glu.AddCommand(buildCmd)
}

var buildCmd = &cobra.Command{
	Use:   "build <os-list> [<pkgs>] [<name>]",
	Short: "Builds a Go project of Glider Labs",
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) < 1 {
			cmd.Usage()
			os.Exit(1)
		}
		defer shell.ErrExit()
		shell.Trace = true
		shell.Tee = os.Stdout

		if tryContainer(cmd, args) {
			return
		}

		var (
			info   = NewProjectInfo()
			osList = strings.Split(args[0], ",")
			pkgs   = optArg(args, 1, ".")
			name   = optArg(args, 2, info.Name)

			ldFlag string
		)

		if insideContainer() {
			os.Setenv("GOPATH", "/go")
			path := fmt.Sprintf("/go/src/%s", info.Repo)
			sh("mkdir -p", filepath.Dir(path))
			sh("cp -r /project", path)
			sh("cd", path) // for show
			os.Chdir(path)
			sh("go get")
		}

		if info.Version != "" {
			ldFlag = fmt.Sprintf("-ldflags \"-X main.Version %s\"", info.Version)
		}

		os.Setenv("CGO_ENABLED", "0")
		for i := range osList {
			os.Setenv("GOOS", strings.ToLower(osList[i]))
			path := shell.Path("build", strings.Title(osList[i]))
			sh("mkdir -p", path)
			sh("go build -a -installsuffix cgo", ldFlag, "-o", shell.Path(path, name), pkgs)
		}

		if insideContainer() {
			sh("rm -rf /project/build")
			sh("mv build /project")
			for i := range osList {
				sh(fmt.Sprintf("tar -czvf /artifacts/%s-%s.tgz -C /project/build/%s %s",
					name, strings.ToLower(osList[i]), strings.Title(osList[i]), name))
			}
			sh("tar -czf /artifacts/go-workspace.tgz -C /go .")
			sh("rm -rf /go")
		}
	},
}

func tryContainer(cmd *cobra.Command, args []string) bool {
	if insideContainer() {
		return false
	}
	if !dockerExistsByName("glu") {
		return false
	}
	fmt.Fprintln(os.Stderr, "* Using glu container")
	args = append(strings.Split(cmd.CommandPath(), " "), args...)
	var newCmd []string
	var binary string
	var err error
	if os.Getenv("CIRCLECI") == "true" {
		if binary, err = exec.LookPath("sudo"); err != nil {
			return false
		}
		os.Setenv("GLU_CONTAINER", "true")
		newCmd = []string{"sudo", "-E", "lxc-attach", "-n", dockerID("glu"), "--", "/bin/glu"}
		newCmd = append(newCmd, args[1:]...)
	} else {
		if binary, err = exec.LookPath("docker"); err != nil {
			return false
		}
		newCmd = []string{"docker", "exec", "glu"}
		newCmd = append(newCmd, args...)
	}
	syscall.Exec(binary, newCmd, os.Environ())
	return true
}

func insideContainer() bool {
	return os.Getenv("GLU_CONTAINER") == "true"
}

func dockerID(container string) string {
	client, err := docker.NewClientFromEnv()
	fatal(err)
	containers, err := client.ListContainers(docker.ListContainersOptions{})
	fatal(err)
	for _, cntr := range containers {
		for _, name := range cntr.Names {
			if name[1:] == container {
				return cntr.ID
			}
		}
	}
	return ""
}
<commit_msg>go get the right packages<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/fsouza/go-dockerclient"
	"github.com/progrium/go-shell"
	"github.com/spf13/cobra"
)

func init() {
	Glu.AddCommand(buildCmd)
}

var buildCmd = &cobra.Command{
	Use:   "build <os-list> [<pkgs>] [<name>]",
	Short: "Builds a Go project of Glider Labs",
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) < 1 {
			cmd.Usage()
			os.Exit(1)
		}
		defer shell.ErrExit()
		shell.Trace = true
		shell.Tee = os.Stdout

		if tryContainer(cmd, args) {
			return
		}

		var (
			info   = NewProjectInfo()
			osList = strings.Split(args[0], ",")
			pkgs   = optArg(args, 1, ".")
			name   = optArg(args, 2, info.Name)

			ldFlag string
		)
		if info.Version != "" {
			ldFlag = fmt.Sprintf("-ldflags \"-X main.Version %s\"", info.Version)
		}

		if insideContainer() {
			os.Setenv("GOPATH", "/go")
			path := fmt.Sprintf("/go/src/%s", info.Repo)
			sh("mkdir -p", filepath.Dir(path))
			sh("cp -r /project", path)
			sh("cd", path) // for show
			os.Chdir(path)
		}

		sh("go get", pkgs)

		os.Setenv("CGO_ENABLED", "0")
		for i := range osList {
			os.Setenv("GOOS", strings.ToLower(osList[i]))
			path := shell.Path("build", strings.Title(osList[i]))
			sh("mkdir -p", path)
			sh("go build -a -installsuffix cgo", ldFlag, "-o", shell.Path(path, name), pkgs)
		}

		if insideContainer() {
			sh("rm -rf /project/build")
			sh("mv build /project")
			for i := range osList {
				sh(fmt.Sprintf("tar -czvf /artifacts/%s-%s.tgz -C /project/build/%s %s",
					name, strings.ToLower(osList[i]), strings.Title(osList[i]), name))
			}
			sh("tar -czf /artifacts/go-workspace.tgz -C /go .")
			sh("rm -rf /go")
		}
	},
}

func tryContainer(cmd *cobra.Command, args []string) bool {
	if insideContainer() {
		return false
	}
	if !dockerExistsByName("glu") {
		return false
	}
	fmt.Fprintln(os.Stderr, "* Using glu container")
	args = append(strings.Split(cmd.CommandPath(), " "), args...)
	var newCmd []string
	var binary string
	var err error
	if os.Getenv("CIRCLECI") == "true" {
		if binary, err = exec.LookPath("sudo"); err != nil {
			return false
		}
		os.Setenv("GLU_CONTAINER", "true")
		newCmd = []string{"sudo", "-E", "lxc-attach", "-n", dockerID("glu"), "--", "/bin/glu"}
		newCmd = append(newCmd, args[1:]...)
	} else {
		if binary, err = exec.LookPath("docker"); err != nil {
			return false
		}
		newCmd = []string{"docker", "exec", "glu"}
		newCmd = append(newCmd, args...)
	}
	syscall.Exec(binary, newCmd, os.Environ())
	return true
}

func insideContainer() bool {
	return os.Getenv("GLU_CONTAINER") == "true"
}

func dockerID(container string) string {
	client, err := docker.NewClientFromEnv()
	fatal(err)
	containers, err := client.ListContainers(docker.ListContainersOptions{})
	fatal(err)
	for _, cntr := range containers {
		for _, name := range cntr.Names {
			if name[1:] == container {
				return cntr.ID
			}
		}
	}
	return ""
}
<|endoftext|>


<commit_before>// Main package wraps sprite_sass tool for use with the command line
// See -h for list of available options
package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"runtime/pprof"
	"strconv"
	"strings"
	"time"

	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	libsass "github.com/wellington/go-libsass"
	"github.com/wellington/wellington/cfg"
	"github.com/wellington/wellington/version"

	wt "github.com/wellington/wellington"
	_ "github.com/wellington/wellington/handlers"
)

var (
	font, dir, gen, includes      string
	mainFile, style               string
	comments, watch               bool
	cpuprofile, buildDir          string
	jsDir                         string
	ishttp, showHelp, showVersion bool
	httpPath                      string
	timeB                         bool
	config                        string
)

/*
   --app APP                    Tell compass what kind of application it is integrating with. E.g. rails
   --fonts-dir FONTS_DIR        The directory where you keep your fonts.
*/
func init() {

	// Interoperability args
}

func flags(set *pflag.FlagSet) {
	set.BoolVarP(&showVersion, "version", "v", false, "Show the app version")
	//wtCmd.PersistentFlags().BoolVarP(&showHelp, "help", "h", false, "this help")
	set.StringVar(&dir, "images-dir", "", "Compass Image Directory")
	set.StringVarP(&dir, "dir", "d", "", "Compass Image Directory")
	set.StringVar(&jsDir, "javascripts-dir", "", "Compass JS Directory")
	set.BoolVar(&timeB, "time", false, "Retrieve timing information")

	set.StringVarP(&buildDir, "build", "b", "", "Target directory for generated CSS, relative paths from sass-dir are preserved")

	// set.StringVar(&gen, "css-dir", "", "Location of CSS files")
	set.StringVar(&gen, "gen", ".", "Generated images directory")

	set.StringVar(&includes, "sass-dir", "", "Compass Sass Directory")
	set.StringVarP(&includes, "proj", "p", "", "Project directory")

	set.StringVar(&font, "font", ".", "Font Directory")
	set.StringVarP(&style, "style", "s", "nested", "CSS nested style")

	set.StringVarP(&config, "config", "c", "", "Location of the config file")

	set.BoolVarP(&comments, "comment", "", true, "Turn on source comments")

	set.BoolVarP(&watch, "watch", "w", false, "File watcher that will rebuild css on file changes")

	set.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to file")

}

var compileCmd = &cobra.Command{
	Use:   "compile",
	Short: "Compile Sass stylesheets to CSS",
	Long: `Fast compilation of Sass stylesheets to CSS. For usage consult
the documentation at https://github.com/wellington/wellington#wellington`,
	Run: Run,
}

var watchCmd = &cobra.Command{
	Use:   "watch",
	Short: "Watch Sass files for changes and rebuild CSS",
	Long:  ``,
	Run: func(cmd *cobra.Command, args []string) {
		watch = true
		Run(cmd, args)
	},
}

var httpCmd = &cobra.Command{
	Use:   "serve",
	Short: "Starts a http server that will convert Sass to CSS",
	Long:  ``,
	Run: func(cmd *cobra.Command, args []string) {
		ishttp = true
		Run(cmd, args)
	},
}

func init() {
	hostname := os.Getenv("HOSTNAME")
	if len(hostname) > 0 {
		if !strings.HasPrefix(hostname, "http") {
			hostname = "http://" + hostname
		}
	} else if host, err := os.Hostname(); err == nil {
		hostname = "http://" + host
	}
	httpCmd.Flags().StringVar(&httpPath, "httppath", hostname,
		"Only for HTTP, overrides generated sprite paths to support http")

}

func root() {
	flags(wtCmd.PersistentFlags())
}

func AddCommands() {
	wtCmd.AddCommand(httpCmd)
	wtCmd.AddCommand(compileCmd)
	wtCmd.AddCommand(watchCmd)
}

var wtCmd = &cobra.Command{
	Use:   "wt",
	Short: "wt builds Sass",
	Run:   Run,
}

func main() {
	AddCommands()
	root()

	wtCmd.Execute()
}

func Run(cmd *cobra.Command, files []string) {

	start := time.Now()

	if showVersion {
		fmt.Printf("   libsass: %s\n", libsass.Version())
		fmt.Printf("Wellington: %s\n", version.Version)
		os.Exit(0)
	}

	if showHelp {
		fmt.Println("Please specify input filepath.")
		fmt.Println("\nAvailable options:")
		//flag.PrintDefaults()
		os.Exit(0)
	}

	defer func() {
		diff := float64(time.Since(start).Nanoseconds()) / float64(time.Millisecond)
		log.Printf("Compilation took: %sms\n",
			strconv.FormatFloat(diff, 'f', 3, 32))
	}()

	// Profiling code
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		log.Println("Starting profiler")
		pprof.StartCPUProfile(f)
		defer func() {
			pprof.StopCPUProfile()
			err := f.Close()
			if err != nil {
				log.Fatal(err)
			}
			log.Println("Stopping Profiller")
		}()
	}

	for _, v := range files {
		if strings.HasPrefix(v, "-") {
			log.Fatalf("Please specify flags before other arguments: %s", v)
		}
	}

	if gen != "" {
		err := os.MkdirAll(gen, 0755)
		if err != nil {
			log.Fatal(err)
		}
	}

	style, ok := libsass.Style[style]

	if !ok {
		style = libsass.NESTED_STYLE
	}

	if len(config) > 0 {
		cfg, err := cfg.Parse(config)
		if err != nil {
			log.Fatal(err)
		}
		// Manually walk through known variables looking for matches
		// These do not override the cli flags
		if p, ok := cfg["css_dir"]; ok && len(buildDir) == 0 {
			buildDir = p
		}

		if p, ok := cfg["images_dir"]; ok && len(dir) == 0 {
			dir = p
		}

		if p, ok := cfg["sass_dir"]; ok && len(includes) == 0 {
			includes = p
		}

		if p, ok := cfg["generated_images_dir"]; ok && len(gen) == 0 {
			gen = p
		}

		// As of yet, unsupported
		if p, ok := cfg["http_path"]; ok {
			_ = p
		}

		if p, ok := cfg["http_generated_images_path"]; ok {
			_ = p
		}

		if p, ok := cfg["fonts_dir"]; ok {
			font = p
		}
	}

	gba := wt.NewBuildArgs()

	gba.Dir = dir
	gba.BuildDir = buildDir
	gba.Includes = includes
	gba.Font = font
	gba.Style = style
	gba.Gen = gen
	gba.Comments = comments

	pMap := wt.NewPartialMap()
	// FIXME: Copy pasta with LoadAndBuild
	ctx := &libsass.Context{
		Payload:      gba.Payload,
		OutputStyle:  gba.Style,
		BuildDir:     gba.BuildDir,
		ImageDir:     gba.Dir,
		FontDir:      gba.Font,
		GenImgDir:    gba.Gen,
		Comments:     gba.Comments,
		HTTPPath:     httpPath,
		IncludePaths: []string{gba.Includes},
	}
	wt.InitializeContext(ctx)
	ctx.Imports.Init()

	if ishttp {
		if len(gba.Gen) == 0 {
			log.Fatal("Must pass an image build directory to use HTTP")
		}
		http.Handle("/build/", wt.FileHandler(gba.Gen))
		log.Println("Web server started on :12345")
		http.HandleFunc("/", wt.HTTPHandler(ctx))
		err := http.ListenAndServe(":12345", nil)
		if err != nil {
			log.Fatal("ListenAndServe: ", err)
		}

		return
	}

	// Only inject files when a config is passed. Otherwise,
	// assume we are waiting for input from stdin
	if len(includes) > 0 && len(config) > 0 {
		rot := filepath.Join(includes, "*.scss")
		pat := filepath.Join(includes, "**/*.scss")
		rotFiles, _ := filepath.Glob(rot)
		patFiles, _ := filepath.Glob(pat)
		files = append(rotFiles, patFiles...)
		// Probably a better way to do this, but I'm impatient

		clean := make([]string, 0, len(files))

		for _, file := range files {
			if !strings.HasPrefix(filepath.Base(file), "_") {
				clean = append(clean, file)
			}
		}
		files = clean
	}

	if len(files) == 0 && len(config) == 0 {

		// Read from stdin
		fmt.Println("Reading from stdin, -h for help")
		out := os.Stdout
		in := os.Stdin

		var pout bytes.Buffer
		_, err := wt.StartParser(ctx, in, &pout, wt.NewPartialMap())
		if err != nil {
			log.Println(err)
		}
		err = ctx.Compile(&pout, out)

		if err != nil {
			log.Println(err)
		}
		return
	}

	sassPaths := make([]string, len(files))
	for i, f := range files {
		sassPaths[i] = filepath.Dir(f)
		err := wt.LoadAndBuild(f, gba, pMap)
		if err != nil {
			log.Println(err)
			os.Exit(1)
		}
	}

	if watch {
		w := wt.NewWatcher()
		w.PartialMap = pMap
		w.Dirs = sassPaths
		w.BArgs = gba
		w.Watch()

		fmt.Println("File watcher started use `ctrl+d` to exit")
		in := bufio.NewReader(os.Stdin)
		for {
			_, err := in.ReadString(' ')
			if err != nil {
				if err == io.EOF {
					os.Exit(0)
				}
				fmt.Println("error", err)
			}
		}
	}
}
<commit_msg>dump config when debug flag is passed fixes #110<commit_after>// Main package wraps sprite_sass tool for use with the command line
// See -h for list of available options
package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"runtime/pprof"
	"strconv"
	"strings"
	"time"

	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	libsass "github.com/wellington/go-libsass"
	"github.com/wellington/wellington/cfg"
	"github.com/wellington/wellington/version"

	wt "github.com/wellington/wellington"
	_ "github.com/wellington/wellington/handlers"
)

var (
	font, dir, gen, includes      string
	mainFile, style               string
	comments, watch               bool
	cpuprofile, buildDir          string
	jsDir                         string
	ishttp, showHelp, showVersion bool
	httpPath                      string
	timeB                         bool
	config                        string
	debug                         bool
)

/*
   --app APP                    Tell compass what kind of application it is integrating with. E.g. rails
   --fonts-dir FONTS_DIR        The directory where you keep your fonts.
*/
func init() {

	// Interoperability args
}

func flags(set *pflag.FlagSet) {
	set.BoolVarP(&showVersion, "version", "v", false, "Show the app version")
	//wtCmd.PersistentFlags().BoolVarP(&showHelp, "help", "h", false, "this help")
	set.BoolVar(&debug, "debug", false, "Show detailed debug information")
	set.StringVar(&dir, "images-dir", "", "Compass Image Directory")
	set.StringVarP(&dir, "dir", "d", "", "Compass Image Directory")
	set.StringVar(&jsDir, "javascripts-dir", "", "Compass JS Directory")
	set.BoolVar(&timeB, "time", false, "Retrieve timing information")

	set.StringVarP(&buildDir, "build", "b", "", "Target directory for generated CSS, relative paths from sass-dir are preserved")

	// set.StringVar(&gen, "css-dir", "", "Location of CSS files")
	set.StringVar(&gen, "gen", ".", "Generated images directory")

	set.StringVar(&includes, "sass-dir", "", "Compass Sass Directory")
	set.StringVarP(&includes, "proj", "p", "", "Project directory")

	set.StringVar(&font, "font", ".", "Font Directory")
	set.StringVarP(&style, "style", "s", "nested", "CSS nested style")

	set.StringVarP(&config, "config", "c", "", "Location of the config file")

	set.BoolVarP(&comments, "comment", "", true, "Turn on source comments")

	set.BoolVarP(&watch, "watch", "w", false, "File watcher that will rebuild css on file changes")

	set.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to file")

}

var compileCmd = &cobra.Command{
	Use:   "compile",
	Short: "Compile Sass stylesheets to CSS",
	Long: `Fast compilation of Sass stylesheets to CSS. For usage consult
the documentation at https://github.com/wellington/wellington#wellington`,
	Run: Run,
}

var watchCmd = &cobra.Command{
	Use:   "watch",
	Short: "Watch Sass files for changes and rebuild CSS",
	Long:  ``,
	Run: func(cmd *cobra.Command, args []string) {
		watch = true
		Run(cmd, args)
	},
}

var httpCmd = &cobra.Command{
	Use:   "serve",
	Short: "Starts a http server that will convert Sass to CSS",
	Long:  ``,
	Run: func(cmd *cobra.Command, args []string) {
		ishttp = true
		Run(cmd, args)
	},
}

func init() {
	hostname := os.Getenv("HOSTNAME")
	if len(hostname) > 0 {
		if !strings.HasPrefix(hostname, "http") {
			hostname = "http://" + hostname
		}
	} else if host, err := os.Hostname(); err == nil {
		hostname = "http://" + host
	}
	httpCmd.Flags().StringVar(&httpPath, "httppath", hostname,
		"Only for HTTP, overrides generated sprite paths to support http")

}

func root() {
	flags(wtCmd.PersistentFlags())
}

func AddCommands() {
	wtCmd.AddCommand(httpCmd)
	wtCmd.AddCommand(compileCmd)
	wtCmd.AddCommand(watchCmd)
}

var wtCmd = &cobra.Command{
	Use:   "wt",
	Short: "wt builds Sass",
	Run:   Run,
}

func main() {
	AddCommands()
	root()

	wtCmd.Execute()
}

func Run(cmd *cobra.Command, files []string) {

	start := time.Now()

	if showVersion {
		fmt.Printf("   libsass: %s\n", libsass.Version())
		fmt.Printf("Wellington: %s\n", version.Version)
		os.Exit(0)
	}

	if showHelp {
		fmt.Println("Please specify input filepath.")
		fmt.Println("\nAvailable options:")
		//flag.PrintDefaults()
		os.Exit(0)
	}

	defer func() {
		diff := float64(time.Since(start).Nanoseconds()) / float64(time.Millisecond)
		log.Printf("Compilation took: %sms\n",
			strconv.FormatFloat(diff, 'f', 3, 32))
	}()

	// Profiling code
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		log.Println("Starting profiler")
		pprof.StartCPUProfile(f)
		defer func() {
			pprof.StopCPUProfile()
			err := f.Close()
			if err != nil {
				log.Fatal(err)
			}
			log.Println("Stopping Profiller")
		}()
	}

	for _, v := range files {
		if strings.HasPrefix(v, "-") {
			log.Fatalf("Please specify flags before other arguments: %s", v)
		}
	}

	if gen != "" {
		err := os.MkdirAll(gen, 0755)
		if err != nil {
			log.Fatal(err)
		}
	}

	style, ok := libsass.Style[style]

	if !ok {
		style = libsass.NESTED_STYLE
	}

	if len(config) > 0 {
		cfg, err := cfg.Parse(config)
		if err != nil {
			log.Fatal(err)
		}
		// Manually walk through known variables looking for matches
		// These do not override the cli flags
		if p, ok := cfg["css_dir"]; ok && len(buildDir) == 0 {
			buildDir = p
		}

		if p, ok := cfg["images_dir"]; ok && len(dir) == 0 {
			dir = p
		}

		if p, ok := cfg["sass_dir"]; ok && len(includes) == 0 {
			includes = p
		}

		if p, ok := cfg["generated_images_dir"]; ok && len(gen) == 0 {
			gen = p
		}

		// As of yet, unsupported
		if p, ok := cfg["http_path"]; ok {
			_ = p
		}

		if p, ok := cfg["http_generated_images_path"]; ok {
			_ = p
		}

		if p, ok := cfg["fonts_dir"]; ok {
			font = p
		}
	}

	gba := wt.NewBuildArgs()

	gba.Dir = dir
	gba.BuildDir = buildDir
	gba.Includes = includes
	gba.Font = font
	gba.Style = style
	gba.Gen = gen
	gba.Comments = comments

	pMap := wt.NewPartialMap()
	// FIXME: Copy pasta with LoadAndBuild
	ctx := &libsass.Context{
		Payload:      gba.Payload,
		OutputStyle:  gba.Style,
		BuildDir:     gba.BuildDir,
		ImageDir:     gba.Dir,
		FontDir:      gba.Font,
		GenImgDir:    gba.Gen,
		Comments:     gba.Comments,
		HTTPPath:     httpPath,
		IncludePaths: []string{gba.Includes},
	}
	if debug {
		fmt.Printf("      Font  Dir: %s\n", gba.Font)
		fmt.Printf("      Image Dir: %s\n", gba.Dir)
		fmt.Printf("      Build Dir: %s\n", gba.BuildDir)
		fmt.Printf("Build Image Dir: %s\n", gba.Gen)
		fmt.Printf(" Include Dir(s): %s\n", gba.Includes)
		fmt.Println("===================================")
	}
	wt.InitializeContext(ctx)
	ctx.Imports.Init()

	if ishttp {
		if len(gba.Gen) == 0 {
			log.Fatal("Must pass an image build directory to use HTTP")
		}
		http.Handle("/build/", wt.FileHandler(gba.Gen))
		log.Println("Web server started on :12345")
		http.HandleFunc("/", wt.HTTPHandler(ctx))
		err := http.ListenAndServe(":12345", nil)
		if err != nil {
			log.Fatal("ListenAndServe: ", err)
		}

		return
	}

	// Only inject files when a config is passed. Otherwise,
	// assume we are waiting for input from stdin
	if len(includes) > 0 && len(config) > 0 {
		rot := filepath.Join(includes, "*.scss")
		pat := filepath.Join(includes, "**/*.scss")
		rotFiles, _ := filepath.Glob(rot)
		patFiles, _ := filepath.Glob(pat)
		files = append(rotFiles, patFiles...)
		// Probably a better way to do this, but I'm impatient

		clean := make([]string, 0, len(files))

		for _, file := range files {
			if !strings.HasPrefix(filepath.Base(file), "_") {
				clean = append(clean, file)
			}
		}
		files = clean
	}

	if len(files) == 0 && len(config) == 0 {

		// Read from stdin
		fmt.Println("Reading from stdin, -h for help")
		out := os.Stdout
		in := os.Stdin

		var pout bytes.Buffer
		_, err := wt.StartParser(ctx, in, &pout, wt.NewPartialMap())
		if err != nil {
			log.Println(err)
		}
		err = ctx.Compile(&pout, out)

		if err != nil {
			log.Println(err)
		}
		return
	}

	sassPaths := make([]string, len(files))
	for i, f := range files {
		sassPaths[i] = filepath.Dir(f)
		err := wt.LoadAndBuild(f, gba, pMap)
		if err != nil {
			log.Println(err)
			os.Exit(1)
		}
	}

	if watch {
		w := wt.NewWatcher()
		w.PartialMap = pMap
		w.Dirs = sassPaths
		w.BArgs = gba
		w.Watch()

		fmt.Println("File watcher started use `ctrl+d` to exit")
		in := bufio.NewReader(os.Stdin)
		for {
			_, err := in.ReadString(' ')
			if err != nil {
				if err == io.EOF {
					os.Exit(0)
				}
				fmt.Println("error", err)
			}
		}
	}
}
<|endoftext|>


<commit_before>package dht

import "net"

type DHT interface {
	Announce()
	Peers() chan []*net.TCPAddr
}
<commit_msg>add comment to dht<commit_after>package dht

import "net"

type DHT interface {
	// Announce must request new peers from DHT.
	// Announce is called by torrent periodically or when more peers are needed.
	Announce()
	// Peers must return a channel for peer addresses returned in response to Announce call.
	Peers() chan []*net.TCPAddr
}
<|endoftext|>


<commit_before>//package main //for test
package hpack

import (
	"encoding/hex"
	"fmt"
	"huffman"
)

func PackIntRepresentation(I uint32, N byte) (buf *[]byte) {
	if I < uint32(1<<N)-1 {
		return &[]byte{byte(I)}
	} else {
		buf = &[]byte{byte(1<<N) - 1}
		I -= uint32(1<<N) - 1
		for I >= 0x80 {
			*buf = append(*buf, byte(I)&0x7f|0x80)
			I = (I >> 7)
		}
		*buf = append(*buf, byte(I))
		return buf
	}
}

func PackContent(content string, toHuffman bool) string {
	if len(content) == 0 {
		if toHuffman {
			return "80"
		} else {
			return "00"
		}
	}

	Wire := ""
	if toHuffman {
		encoded, length := huffman.Root.Encode(content)
		intRep := PackIntRepresentation(uint32(length), 7)
		(*intRep)[0] |= 0x80

		//Wire += hex.EncodeToString(*intRep) + strings.Trim(hex.EncodeToString(b), "00") // + encoded
		Wire += hex.EncodeToString(*intRep) + hex.EncodeToString(encoded)
	} else {
		intRep := PackIntRepresentation(uint32(len(content)), 7)
		Wire += hex.EncodeToString(*intRep) + hex.EncodeToString([]byte(content))
	}
	return Wire
}

func Encode(Headers []Header, fromStaticTable, fromHeaderTable, toHuffman bool, table *Table, headerTableSize int) (Wire string) {
	if headerTableSize != -1 {
		intRep := PackIntRepresentation(uint32(headerTableSize), 5)
		(*intRep)[0] |= 0x20
		Wire += hex.EncodeToString(*intRep)
	}

	for _, header := range Headers {
		match, index := table.FindHeader(header)
		if fromStaticTable && match {
			var indexLen, mask byte
			var content string
			if fromHeaderTable {
				indexLen = 7
				mask = 0x80
				content = ""
			} else {
				indexLen = 4
				mask = 0x00
				content = PackContent(header.Value, toHuffman)
			}
			intRep := PackIntRepresentation(uint32(index), indexLen)
			(*intRep)[0] |= mask
			Wire += hex.EncodeToString(*intRep) + content
		} else if fromStaticTable && !match && index > 0 {
			var indexLen, mask byte
			if fromHeaderTable {
				indexLen = 6
				mask = 0x40
				table.AddHeader(header)
			} else {
				indexLen = 4
				mask = 0x00
			}
			intRep := PackIntRepresentation(uint32(index), indexLen)
			(*intRep)[0] |= mask
			Wire += hex.EncodeToString(*intRep) + PackContent(header.Value, toHuffman)
		} else {
			var prefix string
			if fromHeaderTable {
				prefix = "40"
				table.AddHeader(header)
			} else {
				prefix = "00"
			}
			content := PackContent(header.Name, toHuffman) + PackContent(header.Value, toHuffman)
			Wire += prefix + content
		}
	}

	return
}

func ParseIntRepresentation(buf []byte, N byte) (I, cursor uint32) {
	I = uint32(buf[0] & ((1 << N) - 1)) // byte could be used as byte
	cursor = 1
	if I < ((1 << N) - 1) {
		return I, cursor
	} else {
		var M byte = 0
		for (buf[cursor] & 0x80) > 0 {
			I += uint32(buf[cursor]&0x7f) * (1 << M)
			M += 7
			cursor += 1
		}
		I += uint32(buf[cursor]&0x7f) * (1 << M)
		return I, cursor + 1
	}
}

func ExtractContent(buf []byte, length uint32, isHuffman bool) string {
	if isHuffman {
		return huffman.Root.Decode(buf, length)
	} else {
		return string(buf[:length])
	}
}

func ParseFromByte(buf []byte) (content string, cursor uint32) {
	isHuffman := false
	if buf[0]&0x80 > 0 {
		isHuffman = true
	}
	length, cursor := ParseIntRepresentation(buf, 7)
	content = ExtractContent(buf[cursor:], length, isHuffman)
	cursor += length
	return
}

func ParseHeader(index uint32, buf []byte, isIndexed bool, table *Table) (name, value string, cursor uint32) {
	if c := uint32(0); !isIndexed {
		if index == 0 {
			name, c = ParseFromByte(buf[cursor:])
			cursor += c
		}
		value, c = ParseFromByte(buf[cursor:])
		cursor += c
	}

	if index > 0 {
		header := table.GetHeader(index)

		name = header.Name
		if len(value) == 0 {
			value = header.Value
		}
	}
	return
}

func Decode(wire string, table *Table) (Headers []Header) {
	var buf *[]byte
	nums, err := hex.DecodeString(string(wire))
	if err != nil {
		panic(err)
	}
	buf = &nums

	var cursor uint32 = 0
	for cursor < uint32(len(nums)) {
		isIndexed := false
		isIncremental := false
		var index, c uint32
		if (*buf)[cursor]&0xe0 == 0x20 {
			// 7.3 Header Table Size Update
			size, c := ParseIntRepresentation((*buf)[cursor:], 5)
			table.SetMaxHeaderTableSize(size)
			cursor += c
		}

		if ((*buf)[cursor] & 0x80) > 0 {
			// 7.1 Indexed Header Field
			if ((*buf)[cursor] & 0x7f) == 0 {
				panic('a')
			}
			index, c = ParseIntRepresentation((*buf)[cursor:], 7)
			isIndexed = true
		} else {
			if (*buf)[cursor]&0xc0 == 0x40 {
				// 7.2.1 Literal Header Field with Incremental Indexing
				index, c = ParseIntRepresentation((*buf)[cursor:], 6)
				isIncremental = true
			} else if (*buf)[cursor]&0xf0 == 0xf0 {
				// 7.2.3 Literal Header Field never Indexed
				index, c = ParseIntRepresentation((*buf)[cursor:], 4)
			} else {
				// 7.2.2 Literal Header Field without Indexing
				index, c = ParseIntRepresentation((*buf)[cursor:], 4)
			}
		}
		cursor += c

		name, value, c := ParseHeader(index, (*buf)[cursor:], isIndexed, table)
		cursor += c

		header := Header{name, value}
		if isIncremental {
			table.AddHeader(header)
		}
		Headers = append(Headers, header)
	}

	return
}

func main() {
	//nums, _ := hex.DecodeString(string("1FA18DB701"))
	//fmt.Println(nums)
	//fmt.Println(ParseIntRepresentation(nums, 5))
	//decode("ff80000111")
	//fmt.Println(Decode("00073a6d6574686f640347455400073a736368656d650468747470000a3a617574686f726974790f7777772e7961686f6f2e636f2e6a7000053a70617468012f"))
	fmt.Println(huffman.HUFFMAN_TABLE)
}
<commit_msg>remove ExtractContent and refactoring<commit_after>//package main //for test
package hpack

import (
	"encoding/hex"
	"fmt"
	"huffman"
)

func PackIntRepresentation(I uint32, N byte) (buf *[]byte) {
	if I < uint32(1<<N)-1 {
		return &[]byte{byte(I)}
	} else {
		buf = &[]byte{byte(1<<N) - 1}
		I -= uint32(1<<N) - 1
		for I >= 0x80 {
			*buf = append(*buf, byte(I)&0x7f|0x80)
			I = (I >> 7)
		}
		*buf = append(*buf, byte(I))
		return buf
	}
}

func PackContent(content string, toHuffman bool) string {
	if len(content) == 0 {
		if toHuffman {
			return "80"
		} else {
			return "00"
		}
	}

	Wire := ""
	if toHuffman {
		encoded, length := huffman.Root.Encode(content)
		intRep := PackIntRepresentation(uint32(length), 7)
		(*intRep)[0] |= 0x80

		//Wire += hex.EncodeToString(*intRep) + strings.Trim(hex.EncodeToString(b), "00") // + encoded
		Wire += hex.EncodeToString(*intRep) + hex.EncodeToString(encoded)
	} else {
		intRep := PackIntRepresentation(uint32(len(content)), 7)
		Wire += hex.EncodeToString(*intRep) + hex.EncodeToString([]byte(content))
	}
	return Wire
}

func Encode(Headers []Header, fromStaticTable, fromHeaderTable, toHuffman bool, table *Table, headerTableSize int) (Wire string) {
	if headerTableSize != -1 {
		intRep := PackIntRepresentation(uint32(headerTableSize), 5)
		(*intRep)[0] |= 0x20
		Wire += hex.EncodeToString(*intRep)
	}

	for _, header := range Headers {
		match, index := table.FindHeader(header)
		if fromStaticTable && match {
			var indexLen, mask byte
			var content string
			if fromHeaderTable {
				indexLen = 7
				mask = 0x80
				content = ""
			} else {
				indexLen = 4
				mask = 0x00
				content = PackContent(header.Value, toHuffman)
			}
			intRep := PackIntRepresentation(uint32(index), indexLen)
			(*intRep)[0] |= mask
			Wire += hex.EncodeToString(*intRep) + content
		} else if fromStaticTable && !match && index > 0 {
			var indexLen, mask byte
			if fromHeaderTable {
				indexLen = 6
				mask = 0x40
				table.AddHeader(header)
			} else {
				indexLen = 4
				mask = 0x00
			}
			intRep := PackIntRepresentation(uint32(index), indexLen)
			(*intRep)[0] |= mask
			Wire += hex.EncodeToString(*intRep) + PackContent(header.Value, toHuffman)
		} else {
			var prefix string
			if fromHeaderTable {
				prefix = "40"
				table.AddHeader(header)
			} else {
				prefix = "00"
			}
			content := PackContent(header.Name, toHuffman) + PackContent(header.Value, toHuffman)
			Wire += prefix + content
		}
	}

	return
}

func ParseIntRepresentation(buf []byte, N byte) (I, cursor uint32) {
	I = uint32(buf[0] & ((1 << N) - 1)) // byte could be used as byte
	cursor = 1
	if I < ((1 << N) - 1) {
		return I, cursor
	} else {
		var M byte = 0
		for (buf[cursor] & 0x80) > 0 {
			I += uint32(buf[cursor]&0x7f) * (1 << M)
			M += 7
			cursor += 1
		}
		I += uint32(buf[cursor]&0x7f) * (1 << M)
		return I, cursor + 1
	}
}

func ParseFromByte(buf []byte) (content string, cursor uint32) {
	length, cursor := ParseIntRepresentation(buf, 7)

	if buf[0]&0x80 > 0 {
		content = huffman.Root.Decode(buf[cursor:], length)
	} else {
		content = string(buf[cursor : cursor+length])
	}

	cursor += length
	return
}

func ParseHeader(index uint32, buf []byte, isIndexed bool, table *Table) (name, value string, cursor uint32) {
	if c := uint32(0); !isIndexed {
		if index == 0 {
			name, c = ParseFromByte(buf[cursor:])
			cursor += c
		}
		value, c = ParseFromByte(buf[cursor:])
		cursor += c
	}

	if index > 0 {
		header := table.GetHeader(index)

		name = header.Name
		if len(value) == 0 {
			value = header.Value
		}
	}
	return
}

func Decode(wire string, table *Table) (Headers []Header) {
	var buf *[]byte
	nums, err := hex.DecodeString(string(wire))
	if err != nil {
		panic(err)
	}
	buf = &nums

	var cursor uint32 = 0
	for cursor < uint32(len(nums)) {
		isIndexed := false
		isIncremental := false
		var index, c uint32
		if (*buf)[cursor]&0xe0 == 0x20 {
			// 7.3 Header Table Size Update
			size, c := ParseIntRepresentation((*buf)[cursor:], 5)
			table.SetMaxHeaderTableSize(size)
			cursor += c
		}

		if ((*buf)[cursor] & 0x80) > 0 {
			// 7.1 Indexed Header Field
			if ((*buf)[cursor] & 0x7f) == 0 {
				panic('a')
			}
			index, c = ParseIntRepresentation((*buf)[cursor:], 7)
			isIndexed = true
		} else {
			if (*buf)[cursor]&0xc0 == 0x40 {
				// 7.2.1 Literal Header Field with Incremental Indexing
				index, c = ParseIntRepresentation((*buf)[cursor:], 6)
				isIncremental = true
			} else if (*buf)[cursor]&0xf0 == 0xf0 {
				// 7.2.3 Literal Header Field never Indexed
				index, c = ParseIntRepresentation((*buf)[cursor:], 4)
			} else {
				// 7.2.2 Literal Header Field without Indexing
				index, c = ParseIntRepresentation((*buf)[cursor:], 4)
			}
		}
		cursor += c

		name, value, c := ParseHeader(index, (*buf)[cursor:], isIndexed, table)
		cursor += c

		header := Header{name, value}
		if isIncremental {
			table.AddHeader(header)
		}
		Headers = append(Headers, header)
	}

	return
}

func main() {
	//nums, _ := hex.DecodeString(string("1FA18DB701"))
	//fmt.Println(nums)
	//fmt.Println(ParseIntRepresentation(nums, 5))
	//decode("ff80000111")
	//fmt.Println(Decode("00073a6d6574686f640347455400073a736368656d650468747470000a3a617574686f726974790f7777772e7961686f6f2e636f2e6a7000053a70617468012f"))
	fmt.Println(huffman.HUFFMAN_TABLE)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"

	log "github.com/Sirupsen/logrus"
	"github.com/elazarl/goproxy"
	"github.com/garyburd/redigo/redis"
)

type DBClient struct {
	cache Cache
	http  *http.Client
}

// request holds structure for request
type request struct {
	details requestDetails
}

var emptyResp = &http.Response{}

// requestDetails stores information about request, it's used for creating unique hash and also as a payload structure
type requestDetails struct {
	Path        string `json:"path"`
	Method      string `json:"method"`
	Destination string `json:"destination"`
	Query       string `json:"query"`
}

// hash returns unique hash key for request
func (r *request) hash() string {
	h := md5.New()
	io.WriteString(h, fmt.Sprintf("%s%s%s%s", r.details.Destination, r.details.Path, r.details.Method, r.details.Query))
	return fmt.Sprintf("%x", h.Sum(nil))
}

// res structure hold response body from external service, body is not decoded and is supposed
// to be bytes, however headers should provide all required information for later decoding
// by the client.
type response struct {
	Status  int               `json:"status"`
	Body    []byte            `json:"body"`
	Headers map[string]string `json:"headers"`
}

// Payload structure holds request and response structure
type Payload struct {
	Response response       `json:"response"`
	Request  requestDetails `json:"request"`
	ID       string         `json:"id"`
}

// recordRequest saves request for later playback
func (d *DBClient) recordRequest(req *http.Request) (*http.Response, error) {

	// forwarding request
	resp, err := d.doRequest(req)

	if err == nil {

		// getting response body
		body, err := httputil.DumpResponse(resp, true)
		if err != nil {
			// copying the response body did not work
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
				}).Error("Failed to copy response body.")
			}
		}

		// saving response body with request/response meta to cache
		go d.save(req, resp, body)
	}

	// return new response or error here
	return resp, err
}

// doRequest performs original request and returns response that should be returned to client and error (if there is one)
func (d *DBClient) doRequest(request *http.Request) (*http.Response, error) {
	// We can't have this set. And it only contains "/pkg/net/http/" anyway
	request.RequestURI = ""

	resp, err := d.http.Do(request)

	if err != nil {
		log.WithFields(log.Fields{
			"error":  err.Error(),
			"host":   request.Host,
			"method": request.Method,
			"path":   request.URL.Path,
		}).Error("Could not forward request.")
		return nil, err
	}

	log.WithFields(log.Fields{
		"host":   request.Host,
		"method": request.Method,
		"path":   request.URL.Path,
	}).Info("Response got successfuly!")

	resp.Header.Set("Gen-proxy", "Was-Here")
	return resp, nil

}

// save gets request fingerprint, extracts request body, status code and headers, then saves it to cache
func (d *DBClient) save(req *http.Request, resp *http.Response, respBody []byte) {
	// record request here
	key := getRequestFingerprint(req)

	if resp == nil {
		resp = emptyResp
	} else {
		responseObj := response{
			Status:  resp.StatusCode,
			Body:    respBody,
			Headers: getHeadersMap(resp.Header),
		}

		log.WithFields(log.Fields{
			"path":          req.URL.Path,
			"rawQuery":      req.URL.RawQuery,
			"requestMethod": req.Method,
			"destination":   req.Host,
			"hashKey":       key,
		}).Info("Recording")

		requestObj := requestDetails{
			Path:        req.URL.Path,
			Method:      req.Method,
			Destination: req.Host,
			Query:       req.URL.RawQuery,
		}

		payload := Payload{
			Response: responseObj,
			Request:  requestObj,
			ID:       key,
		}
		// converting it to json bytes
		bts, err := json.Marshal(payload)

		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to marshal json")
		} else {
			d.cache.set(key, bts)
		}
	}

}

// getAllRecordsRaw returns raw (json string) for all records
func (d *DBClient) getAllRecordsRaw() ([]string, error) {
	keys, err := d.cache.getAllKeys()

	if err == nil {

		jsonStrs, err := d.cache.getAllValues(keys)

		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to get all values (raw)")
			return nil, err
		} else {
			return jsonStrs, nil
		}

	} else {
		return nil, err
	}
}

// getAllRecords returns all stored
func (d *DBClient) getAllRecords() ([]Payload, error) {
	var payloads []Payload

	jsonStrs, err := d.getAllRecordsRaw()

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to get all values")
	} else {

		for _, v := range jsonStrs {
			var pl Payload
			err = json.Unmarshal([]byte(v), &pl)
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
					"json":  v,
				}).Warning("Failed to deserialize json")
			} else {
				payloads = append(payloads, pl)
			}
		}
	}

	return payloads, err

}

// deleteAllRecords deletes all recorded requests
func (d *DBClient) deleteAllRecords() error {
	keys, err := d.cache.getAllKeys()

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Warning("Failed to get keys, cannot delete all records")
		return err
	} else {
		for _, v := range keys {
			d.cache.delete(v)
		}
		return nil
	}
}

// getRequestFingerprint returns request hash
func getRequestFingerprint(req *http.Request) string {
	details := requestDetails{Path: req.URL.Path, Method: req.Method, Destination: req.Host, Query: req.URL.RawQuery}
	r := request{details: details}
	return r.hash()
}

// getHeadersMap converts map[string][]string to map[string]string structure
func getHeadersMap(hds map[string][]string) map[string]string {
	headers := make(map[string]string)
	for key, value := range hds {
		headers[key] = value[0]
	}
	return headers
}

// getResponse returns stored response from cache
func (d *DBClient) getResponse(req *http.Request) *http.Response {
	log.Info("Returning response")

	key := getRequestFingerprint(req)
	var payload Payload

	payloadBts, err := redis.Bytes(d.cache.get(key))

	if err == nil {
		log.Info("Decoding bytes")
		// getting cache response
		err = json.Unmarshal(payloadBts, &payload)
		if err != nil {
			log.Error(err)
			// what now?
		}

		newResponse := &http.Response{}
		newResponse.Request = req
		// adding headers
		newResponse.Header = make(http.Header)
		if len(payload.Response.Headers) > 0 {
			for k, v := range payload.Response.Headers {
				newResponse.Header.Set(k, v)
			}
		}
		newResponse.Header.Set("Gen-Proxy", "Playback")
		// adding body
		buf := bytes.NewBuffer(payload.Response.Body)
		newResponse.ContentLength = int64(buf.Len())
		newResponse.Body = ioutil.NopCloser(buf)

		newResponse.StatusCode = payload.Response.Status

		log.WithFields(log.Fields{
			"key":        key,
			"status":     payload.Response.Status,
			"bodyLength": newResponse.ContentLength,
		}).Info("Response found, returning")

		return newResponse

	} else {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to retrieve response from cache")
		// return error? if we return nil - proxy forwards request to original destination
		return goproxy.NewResponse(req,
			goproxy.ContentTypeText, http.StatusPreconditionFailed,
			"Coudldn't find recorded request, please record it first!")
	}

}
<commit_msg>added checks when there are no records<commit_after>package main

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"

	log "github.com/Sirupsen/logrus"
	"github.com/elazarl/goproxy"
	"github.com/garyburd/redigo/redis"
)

type DBClient struct {
	cache Cache
	http  *http.Client
}

// request holds structure for request
type request struct {
	details requestDetails
}

var emptyResp = &http.Response{}

// requestDetails stores information about request, it's used for creating unique hash and also as a payload structure
type requestDetails struct {
	Path        string `json:"path"`
	Method      string `json:"method"`
	Destination string `json:"destination"`
	Query       string `json:"query"`
}

// hash returns unique hash key for request
func (r *request) hash() string {
	h := md5.New()
	io.WriteString(h, fmt.Sprintf("%s%s%s%s", r.details.Destination, r.details.Path, r.details.Method, r.details.Query))
	return fmt.Sprintf("%x", h.Sum(nil))
}

// res structure hold response body from external service, body is not decoded and is supposed
// to be bytes, however headers should provide all required information for later decoding
// by the client.
type response struct {
	Status  int               `json:"status"`
	Body    []byte            `json:"body"`
	Headers map[string]string `json:"headers"`
}

// Payload structure holds request and response structure
type Payload struct {
	Response response       `json:"response"`
	Request  requestDetails `json:"request"`
	ID       string         `json:"id"`
}

// recordRequest saves request for later playback
func (d *DBClient) recordRequest(req *http.Request) (*http.Response, error) {

	// forwarding request
	resp, err := d.doRequest(req)

	if err == nil {

		// getting response body
		body, err := httputil.DumpResponse(resp, true)
		if err != nil {
			// copying the response body did not work
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
				}).Error("Failed to copy response body.")
			}
		}

		// saving response body with request/response meta to cache
		go d.save(req, resp, body)
	}

	// return new response or error here
	return resp, err
}

// doRequest performs original request and returns response that should be returned to client and error (if there is one)
func (d *DBClient) doRequest(request *http.Request) (*http.Response, error) {
	// We can't have this set. And it only contains "/pkg/net/http/" anyway
	request.RequestURI = ""

	resp, err := d.http.Do(request)

	if err != nil {
		log.WithFields(log.Fields{
			"error":  err.Error(),
			"host":   request.Host,
			"method": request.Method,
			"path":   request.URL.Path,
		}).Error("Could not forward request.")
		return nil, err
	}

	log.WithFields(log.Fields{
		"host":   request.Host,
		"method": request.Method,
		"path":   request.URL.Path,
	}).Info("Response got successfuly!")

	resp.Header.Set("Gen-proxy", "Was-Here")
	return resp, nil

}

// save gets request fingerprint, extracts request body, status code and headers, then saves it to cache
func (d *DBClient) save(req *http.Request, resp *http.Response, respBody []byte) {
	// record request here
	key := getRequestFingerprint(req)

	if resp == nil {
		resp = emptyResp
	} else {
		responseObj := response{
			Status:  resp.StatusCode,
			Body:    respBody,
			Headers: getHeadersMap(resp.Header),
		}

		log.WithFields(log.Fields{
			"path":          req.URL.Path,
			"rawQuery":      req.URL.RawQuery,
			"requestMethod": req.Method,
			"destination":   req.Host,
			"hashKey":       key,
		}).Info("Recording")

		requestObj := requestDetails{
			Path:        req.URL.Path,
			Method:      req.Method,
			Destination: req.Host,
			Query:       req.URL.RawQuery,
		}

		payload := Payload{
			Response: responseObj,
			Request:  requestObj,
			ID:       key,
		}
		// converting it to json bytes
		bts, err := json.Marshal(payload)

		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to marshal json")
		} else {
			d.cache.set(key, bts)
		}
	}

}

// getAllRecordsRaw returns raw (json string) for all records
func (d *DBClient) getAllRecordsRaw() ([]string, error) {
	keys, err := d.cache.getAllKeys()

	if err == nil {

		// checking if there are any records
		if len(keys) == 0 {
			return nil, nil
		}

		jsonStrs, err := d.cache.getAllValues(keys)

		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to get all values (raw)")
			return nil, err
		} else {
			return jsonStrs, nil
		}

	} else {
		return nil, err
	}
}

// getAllRecords returns all stored
func (d *DBClient) getAllRecords() ([]Payload, error) {
	var payloads []Payload

	jsonStrs, err := d.getAllRecordsRaw()

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to get all values")
	} else {

		if jsonStrs != nil {
			for _, v := range jsonStrs {
				var pl Payload
				err = json.Unmarshal([]byte(v), &pl)
				if err != nil {
					log.WithFields(log.Fields{
						"error": err.Error(),
						"json":  v,
					}).Warning("Failed to deserialize json")
				} else {
					payloads = append(payloads, pl)
				}
			}
		}
	}

	return payloads, err

}

// deleteAllRecords deletes all recorded requests
func (d *DBClient) deleteAllRecords() error {
	keys, err := d.cache.getAllKeys()

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Warning("Failed to get keys, cannot delete all records")
		return err
	} else {
		for _, v := range keys {
			d.cache.delete(v)
		}
		return nil
	}
}

// getRequestFingerprint returns request hash
func getRequestFingerprint(req *http.Request) string {
	details := requestDetails{Path: req.URL.Path, Method: req.Method, Destination: req.Host, Query: req.URL.RawQuery}
	r := request{details: details}
	return r.hash()
}

// getHeadersMap converts map[string][]string to map[string]string structure
func getHeadersMap(hds map[string][]string) map[string]string {
	headers := make(map[string]string)
	for key, value := range hds {
		headers[key] = value[0]
	}
	return headers
}

// getResponse returns stored response from cache
func (d *DBClient) getResponse(req *http.Request) *http.Response {
	log.Info("Returning response")

	key := getRequestFingerprint(req)
	var payload Payload

	payloadBts, err := redis.Bytes(d.cache.get(key))

	if err == nil {
		log.Info("Decoding bytes")
		// getting cache response
		err = json.Unmarshal(payloadBts, &payload)
		if err != nil {
			log.Error(err)
			// what now?
		}

		newResponse := &http.Response{}
		newResponse.Request = req
		// adding headers
		newResponse.Header = make(http.Header)
		if len(payload.Response.Headers) > 0 {
			for k, v := range payload.Response.Headers {
				newResponse.Header.Set(k, v)
			}
		}
		newResponse.Header.Set("Gen-Proxy", "Playback")
		// adding body
		buf := bytes.NewBuffer(payload.Response.Body)
		newResponse.ContentLength = int64(buf.Len())
		newResponse.Body = ioutil.NopCloser(buf)

		newResponse.StatusCode = payload.Response.Status

		log.WithFields(log.Fields{
			"key":        key,
			"status":     payload.Response.Status,
			"bodyLength": newResponse.ContentLength,
		}).Info("Response found, returning")

		return newResponse

	} else {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to retrieve response from cache")
		// return error? if we return nil - proxy forwards request to original destination
		return goproxy.NewResponse(req,
			goproxy.ContentTypeText, http.StatusPreconditionFailed,
			"Coudldn't find recorded request, please record it first!")
	}

}
<|endoftext|>


<commit_before>package cbfsclient

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"sync"
	"time"
)

type FetchCallback func(oid string, r io.Reader) error

type blobInfo struct {
	Nodes map[string]time.Time
}

func (c Client) getBlobInfos(oids ...string) (map[string]blobInfo, error) {
	inputUrl, err := url.Parse(string(c))
	if err != nil {
		return nil, err
	}

	inputUrl.Path = "/.cbfs/blob/info/"
	form := url.Values{"blob": oids}
	res, err := http.PostForm(inputUrl.String(), form)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("HTTP error fetching blob info: %v",
			res.Status)
	}

	d := json.NewDecoder(res.Body)
	rv := map[string]blobInfo{}
	err = d.Decode(&rv)
	return rv, err
}

type fetchWork struct {
	oid string
	bi  blobInfo
}

type brokenReader struct{ err error }

func (b brokenReader) Read([]byte) (int, error) {
	return 0, b.err
}

func fetchOne(oid string, si StorageNode, cb FetchCallback) error {
	res, err := http.Get(si.BlobURL(oid))
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return fmt.Errorf("HTTP error: %v", res.Status)
	}
	return cb(oid, res.Body)
}

func fetchWorker(cb FetchCallback, nodes map[string]StorageNode,
	ch chan fetchWork, wg *sync.WaitGroup) {

	defer wg.Done()
	for w := range ch {
		var err error
		for n := range w.bi.Nodes {
			err = fetchOne(w.oid, nodes[n], cb)
			if err == nil {
				break
			}
		}
		if err != nil {
			cb(w.oid,
				brokenReader{fmt.Errorf("couldn't find %v", w.oid)})
		}
	}
}

// Fetch many blobs in bulk.
func (c Client) GetBlobs(concurrency int,
	cb FetchCallback, oids ...string) error {

	nodes, err := c.Nodes()
	if err != nil {
		return err
	}

	infos, err := c.getBlobInfos(oids...)
	if err != nil {
		return err
	}

	workch := make(chan fetchWork)

	wg := &sync.WaitGroup{}
	for i := 0; i < concurrency; i++ {
		wg.Add(1)
		go fetchWorker(cb, nodes, workch, wg)
	}

	for oid, info := range infos {
		workch <- fetchWork{oid, info}
	}
	close(workch)

	wg.Done()
	return nil
}
<commit_msg>Fixed up error handling/wg behavior in download<commit_after>package cbfsclient

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"sync"
	"time"
)

type FetchCallback func(oid string, r io.Reader) error

type blobInfo struct {
	Nodes map[string]time.Time
}

func (c Client) getBlobInfos(oids ...string) (map[string]blobInfo, error) {
	inputUrl, err := url.Parse(string(c))
	if err != nil {
		return nil, err
	}

	inputUrl.Path = "/.cbfs/blob/info/"
	form := url.Values{"blob": oids}
	res, err := http.PostForm(inputUrl.String(), form)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("HTTP error fetching blob info: %v",
			res.Status)
	}

	d := json.NewDecoder(res.Body)
	rv := map[string]blobInfo{}
	err = d.Decode(&rv)
	return rv, err
}

type fetchWork struct {
	oid string
	bi  blobInfo
}

type brokenReader struct{ err error }

func (b brokenReader) Read([]byte) (int, error) {
	return 0, b.err
}

func fetchOne(oid string, si StorageNode, cb FetchCallback) error {
	res, err := http.Get(si.BlobURL(oid))
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return fmt.Errorf("HTTP error: %v", res.Status)
	}
	return cb(oid, res.Body)
}

func fetchWorker(cb FetchCallback, nodes map[string]StorageNode,
	ch chan fetchWork, errch chan<- error, wg *sync.WaitGroup) {

	defer wg.Done()
	for w := range ch {
		var err error
		for n := range w.bi.Nodes {
			err = fetchOne(w.oid, nodes[n], cb)
			if err == nil {
				break
			}
		}
		if err != nil {
			select {
			case errch <- err:
			default:
			}
			cb(w.oid,
				brokenReader{fmt.Errorf("couldn't find %v", w.oid)})
		}
	}
}

// Fetch many blobs in bulk.
func (c Client) GetBlobs(concurrency int,
	cb FetchCallback, oids ...string) error {

	nodes, err := c.Nodes()
	if err != nil {
		return err
	}

	infos, err := c.getBlobInfos(oids...)
	if err != nil {
		return err
	}

	workch := make(chan fetchWork)
	errch := make(chan error, 1)

	wg := &sync.WaitGroup{}
	for i := 0; i < concurrency; i++ {
		wg.Add(1)
		go fetchWorker(cb, nodes, workch, errch, wg)
	}

	for oid, info := range infos {
		workch <- fetchWork{oid, info}
	}
	close(workch)

	go func() {
		wg.Wait()
		close(errch)
	}()

	return <-errch
}
<|endoftext|>


<commit_before>package dependency

import (
	"errors"
	"fmt"
	"log"
	"regexp"
	"sort"
)

type NodeDetail struct {
	Node     *Node
	Services NodeServiceList
}

type NodeService struct {
	ID string
	Service string
	Tags    ServiceTags
	Port    int
}

type CatalogNode struct {
	rawKey     string
	dataCenter string
}

// Fetch queries the Consul API defined by the given client and returns a
// of NodeDetail object
func (d *CatalogNode) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
	if opts == nil {
		opts = &QueryOptions{}
	}

	consulOpts := opts.consulQueryOptions()
	if d.dataCenter != "" {
		consulOpts.Datacenter = d.dataCenter
	}

	log.Printf("[DEBUG] (%s) querying Consul with %+v", d.Display(), consulOpts)

	consul, err := clients.Consul()
	if err != nil {
		return nil, nil, fmt.Errorf("catalog node: error getting client: %s", err)
	}

	nodeName := d.rawKey
	if nodeName == "" {
		log.Printf("[DEBUG] (%s) getting local agent name", d.Display())
		nodeName, err = consul.Agent().NodeName()
		if err != nil {
			return nil, nil, fmt.Errorf("catalog node: error getting local agent: %s", err)
		}
	}

	catalog := consul.Catalog()
	n, qm, err := catalog.Node(nodeName, consulOpts)
	if err != nil {
		return nil, nil, fmt.Errorf("catalog node: error fetching: %s", err)
	}

	rm := &ResponseMetadata{
		LastIndex:   qm.LastIndex,
		LastContact: qm.LastContact,
	}

	if n == nil {
		log.Printf("[WARN] (%s) could not find node by that name", d.Display())
		var node *NodeDetail
		return node, rm, nil
	}

	services := make(NodeServiceList, 0, len(n.Services))
	for _, v := range n.Services {
		services = append(services, &NodeService{
			ID:      v.ID,
			Service: v.Service,
			Tags:    ServiceTags(deepCopyAndSortTags(v.Tags)),
			Port:    v.Port,
		})
	}
	sort.Stable(services)

	node := &NodeDetail{
		Node: &Node{
			Node:    n.Node.Node,
			Address: n.Node.Address,
		},
		Services: services,
	}

	return node, rm, nil
}

func (d *CatalogNode) HashCode() string {
	if d.dataCenter != "" {
		return fmt.Sprintf("NodeDetail|%s@%s", d.rawKey, d.dataCenter)
	}
	return fmt.Sprintf("NodeDetail|%s", d.rawKey)
}

func (d *CatalogNode) Display() string {
	if d.dataCenter != "" {
		return fmt.Sprintf("node(%s@%s)", d.rawKey, d.dataCenter)
	}
	return fmt.Sprintf(`"node(%s)"`, d.rawKey)
}

// ParseCatalogNode parses a name name and optional datacenter value.
// If the name is empty or not provided then the current agent is used.
func ParseCatalogNode(s ...string) (*CatalogNode, error) {
	switch len(s) {
	case 0:
		return &CatalogNode{}, nil
	case 1:
		return &CatalogNode{rawKey: s[0]}, nil
	case 2:
		dc := s[1]

		re := regexp.MustCompile(`\A` +
			`(@(?P<datacenter>[[:word:]\.\-]+))?` +
			`\z`)
		names := re.SubexpNames()
		match := re.FindAllStringSubmatch(dc, -1)

		if len(match) == 0 {
			return nil, errors.New("invalid node dependency format")
		}

		r := match[0]

		m := map[string]string{}
		for i, n := range r {
			if names[i] != "" {
				m[names[i]] = n
			}
		}

		nd := &CatalogNode{
			rawKey:     s[0],
			dataCenter: m["datacenter"],
		}

		return nd, nil
	default:
		return nil, fmt.Errorf("expected 0, 1, or 2 arguments, got %d", len(s))
	}
}

// Sorting

type NodeServiceList []*NodeService

func (s NodeServiceList) Len() int      { return len(s) }
func (s NodeServiceList) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s NodeServiceList) Less(i, j int) bool {
	return s[i].ID <= s[j].ID
}
<commit_msg>Update NodeServiceList Less func<commit_after>package dependency

import (
	"errors"
	"fmt"
	"log"
	"regexp"
	"sort"
)

type NodeDetail struct {
	Node     *Node
	Services NodeServiceList
}

type NodeService struct {
	ID string
	Service string
	Tags    ServiceTags
	Port    int
}

type CatalogNode struct {
	rawKey     string
	dataCenter string
}

// Fetch queries the Consul API defined by the given client and returns a
// of NodeDetail object
func (d *CatalogNode) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
	if opts == nil {
		opts = &QueryOptions{}
	}

	consulOpts := opts.consulQueryOptions()
	if d.dataCenter != "" {
		consulOpts.Datacenter = d.dataCenter
	}

	log.Printf("[DEBUG] (%s) querying Consul with %+v", d.Display(), consulOpts)

	consul, err := clients.Consul()
	if err != nil {
		return nil, nil, fmt.Errorf("catalog node: error getting client: %s", err)
	}

	nodeName := d.rawKey
	if nodeName == "" {
		log.Printf("[DEBUG] (%s) getting local agent name", d.Display())
		nodeName, err = consul.Agent().NodeName()
		if err != nil {
			return nil, nil, fmt.Errorf("catalog node: error getting local agent: %s", err)
		}
	}

	catalog := consul.Catalog()
	n, qm, err := catalog.Node(nodeName, consulOpts)
	if err != nil {
		return nil, nil, fmt.Errorf("catalog node: error fetching: %s", err)
	}

	rm := &ResponseMetadata{
		LastIndex:   qm.LastIndex,
		LastContact: qm.LastContact,
	}

	if n == nil {
		log.Printf("[WARN] (%s) could not find node by that name", d.Display())
		var node *NodeDetail
		return node, rm, nil
	}

	services := make(NodeServiceList, 0, len(n.Services))
	for _, v := range n.Services {
		services = append(services, &NodeService{
			ID:      v.ID,
			Service: v.Service,
			Tags:    ServiceTags(deepCopyAndSortTags(v.Tags)),
			Port:    v.Port,
		})
	}
	sort.Stable(services)

	node := &NodeDetail{
		Node: &Node{
			Node:    n.Node.Node,
			Address: n.Node.Address,
		},
		Services: services,
	}

	return node, rm, nil
}

func (d *CatalogNode) HashCode() string {
	if d.dataCenter != "" {
		return fmt.Sprintf("NodeDetail|%s@%s", d.rawKey, d.dataCenter)
	}
	return fmt.Sprintf("NodeDetail|%s", d.rawKey)
}

func (d *CatalogNode) Display() string {
	if d.dataCenter != "" {
		return fmt.Sprintf("node(%s@%s)", d.rawKey, d.dataCenter)
	}
	return fmt.Sprintf(`"node(%s)"`, d.rawKey)
}

// ParseCatalogNode parses a name name and optional datacenter value.
// If the name is empty or not provided then the current agent is used.
func ParseCatalogNode(s ...string) (*CatalogNode, error) {
	switch len(s) {
	case 0:
		return &CatalogNode{}, nil
	case 1:
		return &CatalogNode{rawKey: s[0]}, nil
	case 2:
		dc := s[1]

		re := regexp.MustCompile(`\A` +
			`(@(?P<datacenter>[[:word:]\.\-]+))?` +
			`\z`)
		names := re.SubexpNames()
		match := re.FindAllStringSubmatch(dc, -1)

		if len(match) == 0 {
			return nil, errors.New("invalid node dependency format")
		}

		r := match[0]

		m := map[string]string{}
		for i, n := range r {
			if names[i] != "" {
				m[names[i]] = n
			}
		}

		nd := &CatalogNode{
			rawKey:     s[0],
			dataCenter: m["datacenter"],
		}

		return nd, nil
	default:
		return nil, fmt.Errorf("expected 0, 1, or 2 arguments, got %d", len(s))
	}
}

// Sorting

type NodeServiceList []*NodeService

func (s NodeServiceList) Len() int      { return len(s) }
func (s NodeServiceList) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s NodeServiceList) Less(i, j int) bool {
	if s[i].Service == s[j].Service {
		return s[i].ID <= s[j].ID
	}
	return s[i].Service <= s[j].Service
}
<|endoftext|>


<commit_before>package zip4win

import (
	"archive/zip"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/pkg/errors"
	"golang.org/x/text/unicode/norm"
)

const dsStoreName = ".ds_store"

// Writer implements a zip file writer.
type Writer struct {
	zw               *zip.Writer
	Normalizing      bool
	ExcludeDSStore   bool
	ExcludeDotfiles  bool
	UseUTC           bool
	CompressionLevel int

	fwPool sync.Pool
}

// New returns a new Writer wrting a zip file to w with converting file name encoding.
func New(w io.Writer) *Writer {
	writer := &Writer{
		zw:               zip.NewWriter(w),
		Normalizing:      true,
		ExcludeDSStore:   true,
		ExcludeDotfiles:  false,
		UseUTC:           false,
		CompressionLevel: 6,
	}
	writer.init()

	return writer
}

func (w *Writer) init() {
	w.zw.RegisterCompressor(zip.Deflate, zip.Compressor(w.newFlateWriter))
}

// Close finishes writing the zip file by writing the central directory. It does not (and cannot) close the underlying writer.
func (w *Writer) Close() error {
	return w.zw.Close()
}

// create adds a file to zip file using the provided name.
func (w *Writer) create(fi os.FileInfo, name string) (io.Writer, error) {
	h, err := zip.FileInfoHeader(fi)
	if err != nil {
		return nil, err
	}

	if filepath.IsAbs(name) {
		// If path is absolute, a entry name is a relative path from root.
		name, err = filepath.Rel(filepath.Clean("/"), name)
		if err != nil {
			return nil, errors.Wrapf(err, "Could not get a relative path from root : %s", name)
		}
	}
	name = filepath.ToSlash(filepath.Clean(name))

	if w.Normalizing {
		name = norm.NFC.String(name)
	}

	if fi.IsDir() {
		name = name + "/"
	}

	h.Name = name

	if !w.UseUTC {
		h.Modified = fi.ModTime()
	}

	return w.zw.CreateHeader(h)
}

// WriteEntry add a new entry to zip archive.
func (w *Writer) WriteEntry(path string) error {
	wd, err := os.Getwd()
	if err != nil {
		return errors.Wrap(err, "Cound not get the working directory.")
	}
	fiWd, err := os.Lstat(wd)
	if err != nil {
		return errors.Wrap(err, "Cound not get the working directory.")
	}

	err = filepath.Walk(path, func(p string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if os.SameFile(fi, fiWd) {
			return nil
		}
		if w.ExcludeDSStore && strings.ToLower(fi.Name()) == dsStoreName {
			return nil
		}
		if w.ExcludeDotfiles && strings.HasPrefix(fi.Name(), ".") {
			return nil
		}

		return w.writeFile(p, fi)
	})
	if err != nil {
		if pathErr, ok := err.(*os.PathError); ok {
			return errors.Wrapf(err, "No such file or directory : %s", pathErr.Path)
		}

		return err
	}

	return nil
}

// writeFile creates a entry to the zip archive.
func (w *Writer) writeFile(path string, fi os.FileInfo) error {
	fw, err := w.create(fi, path)
	if err != nil {
		return errors.Wrap(err, "Could not create a new file in zip archive.")
	}

	fmt.Printf("%s\n", path)

	if fi.IsDir() {
		return nil
	}

	fp, err := os.Open(path)
	if err != nil {
		return errors.Wrapf(err, "Could not open the file [%s].", path)
	}
	defer fp.Close()

	_, err = io.Copy(fw, fp)
	if err != nil {
		return errors.Wrap(err, "Could not write to zip archive.")
	}

	return nil
}
<commit_msg>Fix a bug that file entry is not compressed<commit_after>package zip4win

import (
	"archive/zip"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/pkg/errors"
	"golang.org/x/text/unicode/norm"
)

const dsStoreName = ".ds_store"

// Writer implements a zip file writer.
type Writer struct {
	zw               *zip.Writer
	Normalizing      bool
	ExcludeDSStore   bool
	ExcludeDotfiles  bool
	UseUTC           bool
	CompressionLevel int

	fwPool sync.Pool
}

// New returns a new Writer wrting a zip file to w with converting file name encoding.
func New(w io.Writer) *Writer {
	writer := &Writer{
		zw:               zip.NewWriter(w),
		Normalizing:      true,
		ExcludeDSStore:   true,
		ExcludeDotfiles:  false,
		UseUTC:           false,
		CompressionLevel: 6,
	}
	writer.init()

	return writer
}

func (w *Writer) init() {
	w.zw.RegisterCompressor(zip.Deflate, zip.Compressor(w.newFlateWriter))
}

// Close finishes writing the zip file by writing the central directory. It does not (and cannot) close the underlying writer.
func (w *Writer) Close() error {
	return w.zw.Close()
}

// create adds a file to zip file using the provided name.
func (w *Writer) create(fi os.FileInfo, name string) (io.Writer, error) {
	h, err := zip.FileInfoHeader(fi)
	if err != nil {
		return nil, err
	}

	h.Method = zip.Deflate

	if filepath.IsAbs(name) {
		// If path is absolute, a entry name is a relative path from root.
		name, err = filepath.Rel(filepath.Clean("/"), name)
		if err != nil {
			return nil, errors.Wrapf(err, "Could not get a relative path from root : %s", name)
		}
	}
	name = filepath.ToSlash(filepath.Clean(name))

	if w.Normalizing {
		name = norm.NFC.String(name)
	}

	if fi.IsDir() {
		name = name + "/"
	}

	h.Name = name

	if !w.UseUTC {
		h.Modified = fi.ModTime()
	}

	return w.zw.CreateHeader(h)
}

// WriteEntry add a new entry to zip archive.
func (w *Writer) WriteEntry(path string) error {
	wd, err := os.Getwd()
	if err != nil {
		return errors.Wrap(err, "Cound not get the working directory.")
	}
	fiWd, err := os.Lstat(wd)
	if err != nil {
		return errors.Wrap(err, "Cound not get the working directory.")
	}

	err = filepath.Walk(path, func(p string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if os.SameFile(fi, fiWd) {
			return nil
		}
		if w.ExcludeDSStore && strings.ToLower(fi.Name()) == dsStoreName {
			return nil
		}
		if w.ExcludeDotfiles && strings.HasPrefix(fi.Name(), ".") {
			return nil
		}

		return w.writeFile(p, fi)
	})
	if err != nil {
		if pathErr, ok := err.(*os.PathError); ok {
			return errors.Wrapf(err, "No such file or directory : %s", pathErr.Path)
		}

		return err
	}

	return nil
}

// writeFile creates a entry to the zip archive.
func (w *Writer) writeFile(path string, fi os.FileInfo) error {
	fw, err := w.create(fi, path)
	if err != nil {
		return errors.Wrap(err, "Could not create a new file in zip archive.")
	}

	fmt.Printf("%s\n", path)

	if fi.IsDir() {
		return nil
	}

	fp, err := os.Open(path)
	if err != nil {
		return errors.Wrapf(err, "Could not open the file [%s].", path)
	}
	defer fp.Close()

	_, err = io.Copy(fw, fp)
	if err != nil {
		return errors.Wrap(err, "Could not write to zip archive.")
	}

	return nil
}
<|endoftext|>


<commit_before>package client

import (
	"context"

	"google.golang.org/grpc"

	pbns "github.com/slok/ragnarok/grpc/nodestatus"
	"github.com/slok/ragnarok/log"
	"github.com/slok/ragnarok/types"
)

// Status interface will implement the required methods to be able to communicate
// with a node status server
type Status interface {
	// RegisterNode registers a node as available on the server
	RegisterNode(id string, tags map[string]string) error
	// NodeHeartbeat sends a node heartbeat to the master
	NodeHeartbeat(id string, status types.NodeState)
}

// StatusGRPC satisfies Status interface with GRPC communication
type StatusGRPC struct {
	c           pbns.NodeStatusClient
	stateParser types.NodeStateParser
	logger      log.Logger
}

// NewStatusGRPCFromConnection returns a new Status GRPC client based on the grpc connection
func NewStatusGRPCFromConnection(connection *grpc.ClientConn, stateParser types.NodeStateParser, logger log.Logger) (*StatusGRPC, error) {
	c := pbns.NewNodeStatusClient(connection)
	return NewStatusGRPC(c, stateParser, logger)
}

// NewStatusGRPC returns a new Status GRPC client
func NewStatusGRPC(client pbns.NodeStatusClient, stateParser types.NodeStateParser, logger log.Logger) (*StatusGRPC, error) {
	logger = logger.WithField("service", "node-status").WithField("service-kind", "grpc")
	return &StatusGRPC{
		c:           client,
		stateParser: stateParser,
		logger:      logger,
	}, nil
}

// RegisterNode satisfies Status interface
func (s *StatusGRPC) RegisterNode(id string, tags map[string]string) error {
	logger := s.logger.WithField("call", "register-node").WithField("id", id)
	logger.Debug("making GRPC service call")

	// Create the request objects
	n := &pbns.Node{
		Id:   id,
		Tags: tags,
	}

	// Make the request synchronously
	resp, err := s.c.Register(context.Background(), n)

	// Call error
	if err != nil {
		return err
	}
	// If we have a response then (we should)
	if resp != nil {
		logger.Debugf("call response: %s", resp.Message)
	}

	return nil
}

// NodeHeartbeat satisfies Status interface
func (s *StatusGRPC) NodeHeartbeat(id string, state types.NodeState) error {
	logger := s.logger.WithField("call", "node-heartbeat").WithField("id", id)
	logger.Debug("making GRPC service call")

	st, err := s.stateParser.NodeStateToPB(state)
	if err != nil {
		return err
	}

	ns := &pbns.NodeState{
		Id:    id,
		State: st,
	}

	if _, err := s.c.Heartbeat(context.Background(), ns); err != nil {
		return err
	}
	logger.Debugf("heartbeat succeeded")
	return nil
}
<commit_msg>Fix return type on Status node client interface<commit_after>package client

import (
	"context"

	"google.golang.org/grpc"

	pbns "github.com/slok/ragnarok/grpc/nodestatus"
	"github.com/slok/ragnarok/log"
	"github.com/slok/ragnarok/types"
)

// Status interface will implement the required methods to be able to communicate
// with a node status server
type Status interface {
	// RegisterNode registers a node as available on the server
	RegisterNode(id string, tags map[string]string) error
	// NodeHeartbeat sends a node heartbeat to the master
	NodeHeartbeat(id string, status types.NodeState) error
}

// StatusGRPC satisfies Status interface with GRPC communication
type StatusGRPC struct {
	c           pbns.NodeStatusClient
	stateParser types.NodeStateParser
	logger      log.Logger
}

// NewStatusGRPCFromConnection returns a new Status GRPC client based on the grpc connection
func NewStatusGRPCFromConnection(connection *grpc.ClientConn, stateParser types.NodeStateParser, logger log.Logger) (*StatusGRPC, error) {
	c := pbns.NewNodeStatusClient(connection)
	return NewStatusGRPC(c, stateParser, logger)
}

// NewStatusGRPC returns a new Status GRPC client
func NewStatusGRPC(client pbns.NodeStatusClient, stateParser types.NodeStateParser, logger log.Logger) (*StatusGRPC, error) {
	logger = logger.WithField("service", "node-status").WithField("service-kind", "grpc")
	return &StatusGRPC{
		c:           client,
		stateParser: stateParser,
		logger:      logger,
	}, nil
}

// RegisterNode satisfies Status interface
func (s *StatusGRPC) RegisterNode(id string, tags map[string]string) error {
	logger := s.logger.WithField("call", "register-node").WithField("id", id)
	logger.Debug("making GRPC service call")

	// Create the request objects
	n := &pbns.Node{
		Id:   id,
		Tags: tags,
	}

	// Make the request synchronously
	resp, err := s.c.Register(context.Background(), n)

	// Call error
	if err != nil {
		return err
	}
	// If we have a response then (we should)
	if resp != nil {
		logger.Debugf("call response: %s", resp.Message)
	}

	return nil
}

// NodeHeartbeat satisfies Status interface
func (s *StatusGRPC) NodeHeartbeat(id string, state types.NodeState) error {
	logger := s.logger.WithField("call", "node-heartbeat").WithField("id", id)
	logger.Debug("making GRPC service call")

	st, err := s.stateParser.NodeStateToPB(state)
	if err != nil {
		return err
	}

	ns := &pbns.NodeState{
		Id:    id,
		State: st,
	}

	if _, err := s.c.Heartbeat(context.Background(), ns); err != nil {
		return err
	}
	logger.Debugf("heartbeat succeeded")
	return nil
}
<|endoftext|>


<commit_before>package ipfs

import (
	"io"
	"io/ioutil"
	"os"

	logWriter "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log/writer"

	ipfsconfig "github.com/ipfs/go-ipfs/repo/config"
	"github.com/ipfs/go-ipfs/repo/fsrepo"
)

// Init creates an initialized .ipfs directory in the directory `path`.
// The generated RSA key will have `keySize` bits.
func Init(path string, keySize int) error {
	if err := os.MkdirAll(path, 0700); err != nil {
		return err
	}

	// init, but discard the log messages about generating a key.
	cfg, err := ipfsconfig.Init(ioutil.Discard, keySize)
	if err != nil {
		return err
	}

	// Init the actual data store.
	if err := fsrepo.Init(path, cfg); err != nil {
		return err
	}

	return nil
}

// ForwardLog routes all ipfs logs to a file provided by brig.
func ForwardLog(w io.Writer) {
	logWriter.Configure(
		logWriter.Output(w),
		logWriter.LevelError,
	)
}
<commit_msg>ipfs: try to make the log level of ipfs less spammy (no success)<commit_after>package ipfs

import (
	"io"
	"io/ioutil"
	"os"

	ipfsLogging "gx/ipfs/QmQvJiADDe7JR4m968MwXobTCCzUqQkP87aRHe29MEBGHV/go-logging"
	ipfsLog "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log"
	logWriter "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log/writer"

	brigLog "github.com/Sirupsen/logrus"

	ipfsconfig "github.com/ipfs/go-ipfs/repo/config"
	"github.com/ipfs/go-ipfs/repo/fsrepo"
)

// Init creates an initialized .ipfs directory in the directory `path`.
// The generated RSA key will have `keySize` bits.
func Init(path string, keySize int) error {
	if err := os.MkdirAll(path, 0700); err != nil {
		return err
	}

	// init, but discard the log messages about generating a key.
	cfg, err := ipfsconfig.Init(ioutil.Discard, keySize)
	if err != nil {
		return err
	}

	// Init the actual data store.
	if err := fsrepo.Init(path, cfg); err != nil {
		return err
	}

	return nil
}

// ForwardLog routes all ipfs logs to a file provided by brig.
func ForwardLog(w io.Writer) {
	// TODO: The log level setting of ipfs does not work yet.
	// It is still setting the log level to INFO.
	logWriter.Configure(logWriter.Output(w))
	ipfsLogging.SetLevel(ipfsLogging.WARNING, "*")

	if err := ipfsLog.SetLogLevel("*", "warning"); err != nil {
		brigLog.Errorf("failed to set ipfs log level: %v", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	gologme "github.com/erasche/gologme/util"
	_ "github.com/mattn/go-sqlite3"
	"log"
	"net/rpc"
	"os/user"
	"path"
)

func send(wl []gologme.WindowLogs, wi int, kl []gologme.KeyLogs, standalone bool) {
	if standalone {
		send_local(wl, kl, wi)
	} else {
		send_remote(wl, kl, wi)
	}
}

func send_local(wl []gologme.WindowLogs, kl []gologme.KeyLogs, wi int) {
	user, err := user.Current()
	if err != nil {
		log.Fatal(err)

	}
	db, err := sql.Open("sqlite3", path.Join(user.HomeDir, ".gologme.db"))
	// TODO: Ensure admin user?
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	golog := new(gologme.Golog)
	golog.SetupDb(db)
	golog.LogToDb(1, wl, kl, wi)
}

func send_remote(wl []gologme.WindowLogs, kl []gologme.KeyLogs, wi int) {
	client, err := rpc.DialHTTP("tcp", ":10000")
	if err != nil {
		fmt.Printf("Error in dialing, droping logs, %s\n", err)
		return
		// TODO: requeue
	}
	args := &gologme.RpcArgs{
		User:             "hxr",
		ApiKey:           "deadbeefcafe",
		Windows:          wl,
		KeyLogs:          kl,
		WindowLogsLength: wi,
	}
	var result int
	err = client.Call("Golog.Log", args, &result)
	if err != nil {
		fmt.Printf("Error in calling RPC method, droping logs, %s\n", err)
		return
		// TODO: retry
	}
}
<commit_msg>Log storage path<commit_after>package main

import (
	"database/sql"
	"fmt"
	gologme "github.com/erasche/gologme/util"
	_ "github.com/mattn/go-sqlite3"
	"log"
	"net/rpc"
	"os/user"
	"path"
)

func send(wl []gologme.WindowLogs, wi int, kl []gologme.KeyLogs, standalone bool) {
	if standalone {
		send_local(wl, kl, wi)
	} else {
		send_remote(wl, kl, wi)
	}
}

func send_local(wl []gologme.WindowLogs, kl []gologme.KeyLogs, wi int) {
	user, err := user.Current()
	if err != nil {
		log.Fatal(err)

	}
    fn := path.Join(user.HomeDir, ".gologme.db")
    fmt.Printf("Storing to %s\n", fn)
	db, err := sql.Open("sqlite3", fn)
	// TODO: Ensure admin user?
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	golog := new(gologme.Golog)
	golog.SetupDb(db)
	golog.LogToDb(1, wl, kl, wi)
}

func send_remote(wl []gologme.WindowLogs, kl []gologme.KeyLogs, wi int) {
	client, err := rpc.DialHTTP("tcp", ":10000")
	if err != nil {
		fmt.Printf("Error in dialing, droping logs, %s\n", err)
		return
		// TODO: requeue
	}
	args := &gologme.RpcArgs{
		User:             "hxr",
		ApiKey:           "deadbeefcafe",
		Windows:          wl,
		KeyLogs:          kl,
		WindowLogsLength: wi,
	}
	var result int
	err = client.Call("Golog.Log", args, &result)
	if err != nil {
		fmt.Printf("Error in calling RPC method, droping logs, %s\n", err)
		return
		// TODO: retry
	}
}
<|endoftext|>