<commit_before>package apt

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"io"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
	"github.com/subutai-io/gorjun/download"
	"github.com/subutai-io/gorjun/upload"

	"github.com/mkrautz/goar"
	"github.com/subutai-io/agent/log"
	"os/exec"
)

func readDeb(hash string) (control bytes.Buffer, err error) {
	file, err := os.Open(config.Storage.Path + hash)
	log.Check(log.WarnLevel, "Opening deb package", err)

	defer file.Close()

	library := ar.NewReader(file)
	for header, err := library.Next(); err != io.EOF; header, err = library.Next() {
		if err != nil {
			return control, err
		}
		if header.Name == "control.tar.gz" {
			ungzip, err := gzip.NewReader(library)
			if err != nil {
				return control, err
			}

			defer ungzip.Close()

			tr := tar.NewReader(ungzip)
			for tarHeader, err := tr.Next(); err != io.EOF; tarHeader, err = tr.Next() {
				if err != nil {
					return control, err
				}
				if tarHeader.Name == "./control" {
					if _, err := io.Copy(&control, tr); err != nil {
						return control, err
					}
					break
				}
			}
		}
	}
	return
}

func getControl(control bytes.Buffer) map[string]string {
	d := make(map[string]string)
	for _, v := range strings.Split(control.String(), "\n") {
		line := strings.Split(v, ":")
		if len(line) > 1 {
			d[line[0]] = strings.TrimPrefix(line[1], " ")
		}
	}
	return d
}

func getSize(file string) (size int) {
	f, err := os.Open(file)
	if !log.Check(log.WarnLevel, "Opening file "+file, err) {
		stat, _ := f.Stat()
		f.Close()
		size = int(stat.Size())
	}
	return size
}

func Upload(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		_, header, _ := r.FormFile("file")
		md5, sha256, owner := upload.Handler(w, r)
		if len(md5) == 0 || len(sha256) == 0 {
			return
		}
		control, err := readDeb(md5)
		if err != nil {
			log.Warn(err.Error())
			w.WriteHeader(http.StatusUnsupportedMediaType)
			w.Write([]byte(err.Error()))
			if db.Delete(owner, "apt", md5) == 0 {
				os.Remove(config.Storage.Path + md5)
			}
			return
		}
		meta := getControl(control)
		meta["Filename"] = header.Filename
		meta["Size"] = strconv.Itoa(getSize(config.Storage.Path + md5))
		meta["SHA512"] = upload.Hash(config.Storage.Path+md5, "sha512")
		meta["SHA256"] = upload.Hash(config.Storage.Path+md5, "sha256")
		meta["SHA1"] = upload.Hash(config.Storage.Path+md5, "sha1")
		meta["MD5sum"] = md5
		meta["type"] = "apt"
		db.Write(owner, md5, header.Filename, meta)
		w.Write([]byte(md5))
		log.Info(meta["Filename"] + " saved to apt repo by " + owner)
		os.Rename(config.Storage.Path+md5, config.Storage.Path+header.Filename)
		renameOldDebFiles()
	}
}

func Download(w http.ResponseWriter, r *http.Request) {
	file := r.URL.Query().Get("hash")
	if len(file) == 0 {
		file = strings.TrimPrefix(r.RequestURI, "/kurjun/rest/apt/")
	}
	size := getSize(config.Storage.Path + "Packages")
	if file == "Packages" && size == 0 {
		GenerateReleaseFile()
	}
	if f, err := os.Open(config.Storage.Path + file); err == nil && file != "" {
		defer f.Close()
		io.Copy(w, f)
	} else {
		w.WriteHeader(http.StatusNotFound)
	}
}

func Delete(w http.ResponseWriter, r *http.Request) {
	if r.Method == "DELETE" {
		if hash := upload.Delete(w, r); len(hash) != 0 {
			w.Write([]byte("Removed"))
			return
		}
	} else {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Incorrect method"))
	}
}

func Info(w http.ResponseWriter, r *http.Request) {
	if info := download.Info("apt", r); len(info) != 0 {
		w.Write(info)
		return
	}
	w.Write([]byte("Not found"))
}

func renameOldDebFiles() {
	list := db.Search("")
	for _, k := range list {
		if db.CheckRepo("", "apt", k) == 0 {
			continue
		}
		item := download.FormatItem(db.Info(k), "apt", "")
		os.Rename(config.Storage.Path+item.Hash.Md5, config.Storage.Path+item.Name)
	}
}

func GenerateReleaseFile() {
	cmd := exec.Command("bash", "-c", "dpkg-scanpackages . /dev/null | tee Packages | gzip > Packages.gz")
	cmd.Dir = config.Storage.Path
	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
		log.Info("Can't run dpkg-scanpackages")
	}

	cmd = exec.Command("bash", "-c", "apt-ftparchive release . > Release")
	cmd.Dir = config.Storage.Path
	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
		log.Info("Can't run apt-ftparchive")
	}
	cmd = exec.Command("bash", "-c", "gpg --batch --yes --armor -u subutai-release@subutai.io -abs -o Release.gpg Release")
	cmd.Dir = config.Storage.Path
	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
		log.Info("Can't sign Realease file")
	}
}

func Generate(w http.ResponseWriter, r *http.Request) {
	token := r.Header.Get("token")
	owner := strings.ToLower(db.CheckToken(token))
	if len(token) == 0 || len(owner) == 0 {
		w.WriteHeader(http.StatusUnauthorized)
		w.Write([]byte("Not authorized"))
		log.Warn(r.RemoteAddr + " - rejecting generate request")
		return
	}
	if owner != "subutai" && owner != "jenkins" {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Only allowed users can generate release file"))
		log.Warn(r.RemoteAddr + " - rejecting generate request")
		return
	}
	GenerateReleaseFile()
}
<commit_msg>Add response for apt generate REST<commit_after>package apt

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"io"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
	"github.com/subutai-io/gorjun/download"
	"github.com/subutai-io/gorjun/upload"

	"github.com/mkrautz/goar"
	"github.com/subutai-io/agent/log"
	"os/exec"
)

func readDeb(hash string) (control bytes.Buffer, err error) {
	file, err := os.Open(config.Storage.Path + hash)
	log.Check(log.WarnLevel, "Opening deb package", err)

	defer file.Close()

	library := ar.NewReader(file)
	for header, err := library.Next(); err != io.EOF; header, err = library.Next() {
		if err != nil {
			return control, err
		}
		if header.Name == "control.tar.gz" {
			ungzip, err := gzip.NewReader(library)
			if err != nil {
				return control, err
			}

			defer ungzip.Close()

			tr := tar.NewReader(ungzip)
			for tarHeader, err := tr.Next(); err != io.EOF; tarHeader, err = tr.Next() {
				if err != nil {
					return control, err
				}
				if tarHeader.Name == "./control" {
					if _, err := io.Copy(&control, tr); err != nil {
						return control, err
					}
					break
				}
			}
		}
	}
	return
}

func getControl(control bytes.Buffer) map[string]string {
	d := make(map[string]string)
	for _, v := range strings.Split(control.String(), "\n") {
		line := strings.Split(v, ":")
		if len(line) > 1 {
			d[line[0]] = strings.TrimPrefix(line[1], " ")
		}
	}
	return d
}

func getSize(file string) (size int) {
	f, err := os.Open(file)
	if !log.Check(log.WarnLevel, "Opening file "+file, err) {
		stat, _ := f.Stat()
		f.Close()
		size = int(stat.Size())
	}
	return size
}

func Upload(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		_, header, _ := r.FormFile("file")
		md5, sha256, owner := upload.Handler(w, r)
		if len(md5) == 0 || len(sha256) == 0 {
			return
		}
		control, err := readDeb(md5)
		if err != nil {
			log.Warn(err.Error())
			w.WriteHeader(http.StatusUnsupportedMediaType)
			w.Write([]byte(err.Error()))
			if db.Delete(owner, "apt", md5) == 0 {
				os.Remove(config.Storage.Path + md5)
			}
			return
		}
		meta := getControl(control)
		meta["Filename"] = header.Filename
		meta["Size"] = strconv.Itoa(getSize(config.Storage.Path + md5))
		meta["SHA512"] = upload.Hash(config.Storage.Path+md5, "sha512")
		meta["SHA256"] = upload.Hash(config.Storage.Path+md5, "sha256")
		meta["SHA1"] = upload.Hash(config.Storage.Path+md5, "sha1")
		meta["MD5sum"] = md5
		meta["type"] = "apt"
		db.Write(owner, md5, header.Filename, meta)
		w.Write([]byte(md5))
		log.Info(meta["Filename"] + " saved to apt repo by " + owner)
		os.Rename(config.Storage.Path+md5, config.Storage.Path+header.Filename)
		renameOldDebFiles()
	}
}

func Download(w http.ResponseWriter, r *http.Request) {
	file := r.URL.Query().Get("hash")
	if len(file) == 0 {
		file = strings.TrimPrefix(r.RequestURI, "/kurjun/rest/apt/")
	}
	size := getSize(config.Storage.Path + "Packages")
	if file == "Packages" && size == 0 {
		GenerateReleaseFile()
	}
	if f, err := os.Open(config.Storage.Path + file); err == nil && file != "" {
		defer f.Close()
		io.Copy(w, f)
	} else {
		w.WriteHeader(http.StatusNotFound)
	}
}

func Delete(w http.ResponseWriter, r *http.Request) {
	if r.Method == "DELETE" {
		if hash := upload.Delete(w, r); len(hash) != 0 {
			w.Write([]byte("Removed"))
			return
		}
	} else {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Incorrect method"))
	}
}

func Info(w http.ResponseWriter, r *http.Request) {
	if info := download.Info("apt", r); len(info) != 0 {
		w.Write(info)
		return
	}
	w.Write([]byte("Not found"))
}

func renameOldDebFiles() {
	list := db.Search("")
	for _, k := range list {
		if db.CheckRepo("", "apt", k) == 0 {
			continue
		}
		item := download.FormatItem(db.Info(k), "apt", "")
		os.Rename(config.Storage.Path+item.Hash.Md5, config.Storage.Path+item.Name)
	}
}

func GenerateReleaseFile() {
	cmd := exec.Command("bash", "-c", "dpkg-scanpackages . /dev/null | tee Packages | gzip > Packages.gz")
	cmd.Dir = config.Storage.Path
	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
		log.Info("Can't run dpkg-scanpackages")
	}

	cmd = exec.Command("bash", "-c", "apt-ftparchive release . > Release")
	cmd.Dir = config.Storage.Path
	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
		log.Info("Can't run apt-ftparchive")
	}
	cmd = exec.Command("bash", "-c", "gpg --batch --yes --armor -u subutai-release@subutai.io -abs -o Release.gpg Release")
	cmd.Dir = config.Storage.Path
	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
		log.Info("Can't sign Realease file")
	}
}

func Generate(w http.ResponseWriter, r *http.Request) {
	token := r.Header.Get("token")
	owner := strings.ToLower(db.CheckToken(token))
	if len(token) == 0 || len(owner) == 0 {
		w.WriteHeader(http.StatusUnauthorized)
		w.Write([]byte("Not authorized"))
		log.Warn(r.RemoteAddr + " - rejecting generate request")
		return
	}
	if owner != "subutai" && owner != "jenkins" {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Only allowed users can generate release file"))
		log.Warn(r.RemoteAddr + " - rejecting generate request")
		return
	}
	GenerateReleaseFile()
	w.Write([]byte("New Packages file generated and Release file signed"))
}
<|endoftext|>


<commit_before>package etcd

import (
	"encoding/json"
	"fmt"
	"math"
	"strings"
	"time"

	e "github.com/coreos/go-etcd/etcd"
	"github.com/portworx/kvdb"
)

const (
	Name    = "etcd-kv"
	defHost = "http://127.0.0.1:4001"
)

type EtcdKV struct {
	client *e.Client
	domain string
}

func EtcdInit(domain string,
	machines []string,
	options map[string]string) (kvdb.Kvdb, error) {

	if len(machines) == 0 {
		machines = []string{defHost}
	}
	if domain != "" && !strings.HasSuffix(domain, "/") {
		domain = domain + "/"
	}
	kv := &EtcdKV{
		client: e.NewClient(machines),
		domain: domain,
	}
	return kv, nil
}

func (kv *EtcdKV) String() string {
	return Name
}

func (kv *EtcdKV) nodeToKv(node *e.Node) *kvdb.KVPair {

	kvp := &kvdb.KVPair{
		Value:         []byte(node.Value),
		TTL:           node.TTL,
		ModifiedIndex: node.ModifiedIndex,
		CreatedIndex:  node.CreatedIndex,
	}

	// Strip out the leading '/'
	if len(node.Key) != 0 {
		kvp.Key = node.Key[1:]
	} else {
		kvp.Key = node.Key
	}
	kvp.Key = strings.TrimPrefix(kvp.Key, kv.domain)
	return kvp
}

func (kv *EtcdKV) resultToKv(result *e.Response) *kvdb.KVPair {

	kvp := kv.nodeToKv(result.Node)
	switch result.Action {
	case "create":
		kvp.Action = kvdb.KVCreate
	case "set", "update":
		kvp.Action = kvdb.KVSet
	case "delete":
		kvp.Action = kvdb.KVDelete
	case "get":
		kvp.Action = kvdb.KVGet
	default:
		kvp.Action = kvdb.KVUknown
	}
	kvp.KVDBIndex = result.EtcdIndex
	return kvp
}

func (kv *EtcdKV) resultToKvs(result *e.Response) kvdb.KVPairs {

	kvs := make([]*kvdb.KVPair, len(result.Node.Nodes))
	for i := range result.Node.Nodes {
		kvs[i] = kv.nodeToKv(result.Node.Nodes[i])
		kvs[i].KVDBIndex = result.EtcdIndex
	}
	return kvs
}

func (kv *EtcdKV) Get(key string) (*kvdb.KVPair, error) {
	key = kv.domain + key
	result, err := kv.client.Get(key, false, false)
	if err != nil {
		return nil, err
	}
	return kv.resultToKv(result), err
}

func (kv *EtcdKV) GetVal(key string, val interface{}) (*kvdb.KVPair, error) {
	kvp, err := kv.Get(key)
	if err != nil {
		return nil, err
	}
	err = json.Unmarshal(kvp.Value, val)
	return kvp, err
}

func (kv *EtcdKV) toBytes(val interface{}) ([]byte, error) {
	var (
		b   []byte
		err error
	)

	switch val.(type) {
	case string:
		b = []byte(val.(string))
	case []byte:
		b = val.([]byte)
	default:
		b, err = json.Marshal(val)
		if err != nil {
			return nil, err
		}
	}

	return b, nil
}

func (kv *EtcdKV) Put(
	key string,
	val interface{},
	ttl uint64) (*kvdb.KVPair, error) {

	key = kv.domain + key
	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}
	result, err := kv.client.Set(key, string(b), ttl)
	if err != nil {
		return nil, err
	}

	return kv.resultToKv(result), err
}

func (kv *EtcdKV) Create(key string, val interface{}, ttl uint64) (*kvdb.KVPair, error) {
	key = kv.domain + key

	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}
	result, err := kv.client.Create(key, string(b), ttl)
	if err != nil {
		return nil, err
	}
	return kv.resultToKv(result), err
}

func (kv *EtcdKV) Update(key string, val interface{}, ttl uint64) (*kvdb.KVPair, error) {
	key = kv.domain + key

	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}
	result, err := kv.client.Update(key, string(b), ttl)
	if err != nil {
		return nil, err
	}
	return kv.resultToKv(result), err
}

func (kv *EtcdKV) Enumerate(prefix string) (kvdb.KVPairs, error) {
	prefix = kv.domain + prefix

	result, err := kv.client.Get(prefix, true, true)

	if err != nil {
		return nil, err
	}
	return kv.resultToKvs(result), err
}

func (kv *EtcdKV) Delete(key string) (*kvdb.KVPair, error) {
	key = kv.domain + key

	result, err := kv.client.Delete(key, false)
	if err == nil {
		return kv.resultToKv(result), err
	}
	return nil, err
}

func (kv *EtcdKV) DeleteTree(prefix string) error {
	prefix = kv.domain + prefix

	_, err := kv.client.Delete(prefix, true)
	return err
}

func (kv *EtcdKV) Keys(prefix, key string) ([]string, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *EtcdKV) CompareAndSet(
	kvp *kvdb.KVPair,
	flags kvdb.KVFlags,
	prevValue []byte) (*kvdb.KVPair, error) {

	prevIndex := uint64(0)
	if (flags & kvdb.KVModifiedIndex) != 0 {
		prevIndex = kvp.ModifiedIndex
	}
	result, err := kv.client.CompareAndSwap(
		kv.domain+kvp.Key,
		string(kvp.Value),
		0,
		string(prevValue),
		prevIndex)
	if err != nil {
		return nil, err
	}
	return kv.resultToKv(result), err
}

func (kv *EtcdKV) CompareAndDelete(
	kvp *kvdb.KVPair,
	flags kvdb.KVFlags) (*kvdb.KVPair, error) {
	result, err := kv.client.CompareAndDelete(kv.domain+kvp.Key,
		string(kvp.Value),
		kvp.ModifiedIndex)

	if err != nil {
		return nil, err
	}
	return kv.resultToKv(result), err
}

func (kv *EtcdKV) WatchKey(
	key string,
	waitIndex uint64,
	opaque interface{},
	cb kvdb.WatchCB) error {

	key = kv.domain + key
	go kv.watchStart(key, false, waitIndex, opaque, cb)
	return nil
}

func (kv *EtcdKV) WatchTree(
	prefix string,
	waitIndex uint64,
	opaque interface{},
	cb kvdb.WatchCB) error {

	prefix = kv.domain + prefix
	go kv.watchStart(prefix, true, waitIndex, opaque, cb)
	return nil
}

func (kv *EtcdKV) Lock(key string, ttl uint64) (*kvdb.KVPair, error) {
	key = kv.domain + key

	count := 1
	duration := time.Duration(math.Min(float64(time.Second),
		float64((time.Duration(ttl)*time.Second)/10)))

	result, err := kv.client.Create(key, "locked", ttl)
	for err != nil {
		time.Sleep(duration)
		count++
		result, err = kv.client.Create(key, "locked", ttl)
	}

	if err != nil {
		return nil, err
	}
	if count > 3 {
		fmt.Printf("ETCD: spent %v iteations locking %v", count, key)
	}
	return kv.resultToKv(result), err
}

func (kv *EtcdKV) Unlock(kvp *kvdb.KVPair) error {
	// Don't modify kvp here, CompareAndDelete does that.
	_, err := kv.CompareAndDelete(kvp, kvdb.KVFlags(0))
	return err
}

func (kv *EtcdKV) watchReceive(
	key string,
	opaque interface{},
	c chan *e.Response,
	stop chan bool,
	cb kvdb.WatchCB) {

	var err error
	for r, more := <-c; err == nil && more; {
		if more {
			err = cb(key, opaque, kv.resultToKv(r), nil)
			if err == nil {
				r, more = <-c
			}
		}
	}
	stop <- true
	close(stop)
}

func (kv *EtcdKV) watchStart(key string,
	recursive bool,
	waitIndex uint64,
	opaque interface{},
	cb kvdb.WatchCB) {

	ch := make(chan *e.Response, 10)
	stop := make(chan bool, 1)

	go kv.watchReceive(key, opaque, ch, stop, cb)

	_, err := kv.client.Watch(key, waitIndex, recursive, ch, stop)
	if err != e.ErrWatchStoppedByUser {
		e, ok := err.(e.EtcdError)
		if ok {
			fmt.Printf("Etcd error code %d, message %s cause %s Index %ju\n",
				e.ErrorCode, e.Message, e.Cause, e.Index)
		}
		cb(key, opaque, nil, err)
		fmt.Errorf("Watch returned unexpected error %s\n", err.Error())
	} else {
		cb(key, opaque, nil, kvdb.ErrWatchStopped)
	}
}

func (kv *EtcdKV) TxNew() (kvdb.Tx, error) {
	return nil, kvdb.ErrNotSupported
}

func init() {
	kvdb.Register(Name, EtcdInit)
}
<commit_msg>Wait for goroutines to startup before returning from watch<commit_after>package etcd

import (
	"encoding/json"
	"fmt"
	"math"
	"strings"
	"time"

	e "github.com/coreos/go-etcd/etcd"
	"github.com/portworx/kvdb"
)

const (
	Name    = "etcd-kv"
	defHost = "http://127.0.0.1:4001"
)

type EtcdKV struct {
	client *e.Client
	domain string
}

func EtcdInit(domain string,
	machines []string,
	options map[string]string) (kvdb.Kvdb, error) {

	if len(machines) == 0 {
		machines = []string{defHost}
	}
	if domain != "" && !strings.HasSuffix(domain, "/") {
		domain = domain + "/"
	}
	kv := &EtcdKV{
		client: e.NewClient(machines),
		domain: domain,
	}
	return kv, nil
}

func (kv *EtcdKV) String() string {
	return Name
}

func (kv *EtcdKV) nodeToKv(node *e.Node) *kvdb.KVPair {

	kvp := &kvdb.KVPair{
		Value:         []byte(node.Value),
		TTL:           node.TTL,
		ModifiedIndex: node.ModifiedIndex,
		CreatedIndex:  node.CreatedIndex,
	}

	// Strip out the leading '/'
	if len(node.Key) != 0 {
		kvp.Key = node.Key[1:]
	} else {
		kvp.Key = node.Key
	}
	kvp.Key = strings.TrimPrefix(kvp.Key, kv.domain)
	return kvp
}

func (kv *EtcdKV) resultToKv(result *e.Response) *kvdb.KVPair {

	kvp := kv.nodeToKv(result.Node)
	switch result.Action {
	case "create":
		kvp.Action = kvdb.KVCreate
	case "set", "update":
		kvp.Action = kvdb.KVSet
	case "delete":
		kvp.Action = kvdb.KVDelete
	case "get":
		kvp.Action = kvdb.KVGet
	default:
		kvp.Action = kvdb.KVUknown
	}
	kvp.KVDBIndex = result.EtcdIndex
	return kvp
}

func (kv *EtcdKV) resultToKvs(result *e.Response) kvdb.KVPairs {

	kvs := make([]*kvdb.KVPair, len(result.Node.Nodes))
	for i := range result.Node.Nodes {
		kvs[i] = kv.nodeToKv(result.Node.Nodes[i])
		kvs[i].KVDBIndex = result.EtcdIndex
	}
	return kvs
}

func (kv *EtcdKV) Get(key string) (*kvdb.KVPair, error) {
	key = kv.domain + key
	result, err := kv.client.Get(key, false, false)
	if err != nil {
		return nil, err
	}
	return kv.resultToKv(result), err
}

func (kv *EtcdKV) GetVal(key string, val interface{}) (*kvdb.KVPair, error) {
	kvp, err := kv.Get(key)
	if err != nil {
		return nil, err
	}
	err = json.Unmarshal(kvp.Value, val)
	return kvp, err
}

func (kv *EtcdKV) toBytes(val interface{}) ([]byte, error) {
	var (
		b   []byte
		err error
	)

	switch val.(type) {
	case string:
		b = []byte(val.(string))
	case []byte:
		b = val.([]byte)
	default:
		b, err = json.Marshal(val)
		if err != nil {
			return nil, err
		}
	}

	return b, nil
}

func (kv *EtcdKV) Put(
	key string,
	val interface{},
	ttl uint64) (*kvdb.KVPair, error) {

	key = kv.domain + key
	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}
	result, err := kv.client.Set(key, string(b), ttl)
	if err != nil {
		return nil, err
	}

	return kv.resultToKv(result), err
}

func (kv *EtcdKV) Create(key string, val interface{}, ttl uint64) (*kvdb.KVPair, error) {
	key = kv.domain + key

	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}
	result, err := kv.client.Create(key, string(b), ttl)
	if err != nil {
		return nil, err
	}
	return kv.resultToKv(result), err
}

func (kv *EtcdKV) Update(key string, val interface{}, ttl uint64) (*kvdb.KVPair, error) {
	key = kv.domain + key

	b, err := kv.toBytes(val)
	if err != nil {
		return nil, err
	}
	result, err := kv.client.Update(key, string(b), ttl)
	if err != nil {
		return nil, err
	}
	return kv.resultToKv(result), err
}

func (kv *EtcdKV) Enumerate(prefix string) (kvdb.KVPairs, error) {
	prefix = kv.domain + prefix

	result, err := kv.client.Get(prefix, true, true)

	if err != nil {
		return nil, err
	}
	return kv.resultToKvs(result), err
}

func (kv *EtcdKV) Delete(key string) (*kvdb.KVPair, error) {
	key = kv.domain + key

	result, err := kv.client.Delete(key, false)
	if err == nil {
		return kv.resultToKv(result), err
	}
	return nil, err
}

func (kv *EtcdKV) DeleteTree(prefix string) error {
	prefix = kv.domain + prefix

	_, err := kv.client.Delete(prefix, true)
	return err
}

func (kv *EtcdKV) Keys(prefix, key string) ([]string, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *EtcdKV) CompareAndSet(
	kvp *kvdb.KVPair,
	flags kvdb.KVFlags,
	prevValue []byte) (*kvdb.KVPair, error) {

	prevIndex := uint64(0)
	if (flags & kvdb.KVModifiedIndex) != 0 {
		prevIndex = kvp.ModifiedIndex
	}
	result, err := kv.client.CompareAndSwap(
		kv.domain+kvp.Key,
		string(kvp.Value),
		0,
		string(prevValue),
		prevIndex)
	if err != nil {
		return nil, err
	}
	return kv.resultToKv(result), err
}

func (kv *EtcdKV) CompareAndDelete(
	kvp *kvdb.KVPair,
	flags kvdb.KVFlags) (*kvdb.KVPair, error) {
	result, err := kv.client.CompareAndDelete(kv.domain+kvp.Key,
		string(kvp.Value),
		kvp.ModifiedIndex)

	if err != nil {
		return nil, err
	}
	return kv.resultToKv(result), err
}

func (kv *EtcdKV) WatchKey(
	key string,
	waitIndex uint64,
	opaque interface{},
	cb kvdb.WatchCB) error {

	key = kv.domain + key
	started := make(chan bool, 1)
	go kv.watchStart(key, false, waitIndex, opaque, cb, started)
	<-started
	close(started)
	return nil
}

func (kv *EtcdKV) WatchTree(
	prefix string,
	waitIndex uint64,
	opaque interface{},
	cb kvdb.WatchCB) error {

	prefix = kv.domain + prefix
	started := make(chan bool, 1)
	go kv.watchStart(prefix, true, waitIndex, opaque, cb, started)
	<-started
	close(started)
	return nil
}

func (kv *EtcdKV) Lock(key string, ttl uint64) (*kvdb.KVPair, error) {
	key = kv.domain + key

	count := 1
	duration := time.Duration(math.Min(float64(time.Second),
		float64((time.Duration(ttl)*time.Second)/10)))

	result, err := kv.client.Create(key, "locked", ttl)
	for err != nil {
		time.Sleep(duration)
		count++
		result, err = kv.client.Create(key, "locked", ttl)
	}

	if err != nil {
		return nil, err
	}
	if count > 3 {
		fmt.Printf("ETCD: spent %v iteations locking %v", count, key)
	}
	return kv.resultToKv(result), err
}

func (kv *EtcdKV) Unlock(kvp *kvdb.KVPair) error {
	// Don't modify kvp here, CompareAndDelete does that.
	_, err := kv.CompareAndDelete(kvp, kvdb.KVFlags(0))
	return err
}

func (kv *EtcdKV) watchReceive(
	key string,
	opaque interface{},
	c chan *e.Response,
	stop chan bool,
	cb kvdb.WatchCB) {

	var err error
	for r, more := <-c; err == nil && more; {
		if more {
			err = cb(key, opaque, kv.resultToKv(r), nil)
			if err == nil {
				r, more = <-c
			}
		}
	}
	stop <- true
	close(stop)
}

func (kv *EtcdKV) watchStart(key string,
	recursive bool,
	waitIndex uint64,
	opaque interface{},
	cb kvdb.WatchCB,
	started chan bool,
) {

	ch := make(chan *e.Response, 10)
	stop := make(chan bool, 1)

	go kv.watchReceive(key, opaque, ch, stop, cb)

	started <- true
	_, err := kv.client.Watch(key, waitIndex, recursive, ch, stop)
	if err != e.ErrWatchStoppedByUser {
		e, ok := err.(e.EtcdError)
		if ok {
			fmt.Printf("Etcd error code %d, message %s cause %s Index %ju\n",
				e.ErrorCode, e.Message, e.Cause, e.Index)
		}
		cb(key, opaque, nil, err)
		fmt.Errorf("Watch returned unexpected error %s\n", err.Error())
	} else {
		cb(key, opaque, nil, kvdb.ErrWatchStopped)
	}
}

func (kv *EtcdKV) TxNew() (kvdb.Tx, error) {
	return nil, kvdb.ErrNotSupported
}

func init() {
	kvdb.Register(Name, EtcdInit)
}
<|endoftext|>


<commit_before>package etcd

import (
	"fmt"
	"net/url"
	"reflect"
)

type Options map[string]interface{}

// An internally-used data structure that represents a mapping
// between valid options and their kinds
type validOptions map[string]reflect.Kind

// Valid options for GET, PUT, POST, DELETE
// Using CAPITALIZED_UNDERSCORE to emphasize that these
// values are meant to be used as constants.
var (
	VALID_GET_OPTIONS = validOptions{
		"recursive":  reflect.Bool,
		"consistent": reflect.Bool,
		"sorted":     reflect.Bool,
		"wait":       reflect.Bool,
		"waitIndex":  reflect.Uint64,
	}

	VALID_PUT_OPTIONS = validOptions{
		"prevValue": reflect.String,
		"prevIndex": reflect.Uint64,
		"prevExist": reflect.Bool,
		"dir":       reflect.Bool,
	}

	VALID_POST_OPTIONS = validOptions{}

	VALID_DELETE_OPTIONS = validOptions{
		"recursive": reflect.Bool,
		"dir":       reflect.Bool,
		"prevValue": reflect.String,
		"prevIndex": reflect.Uint64,
	}
)

// Convert options to a string of HTML parameters
func (ops Options) toParameters(validOps validOptions) (string, error) {
	p := "?"
	values := url.Values{}

	if ops == nil {
		return "", nil
	}

	for k, v := range ops {
		// Check if the given option is valid (that it exists)
		kind := validOps[k]
		if kind == reflect.Invalid {
			return "", fmt.Errorf("Invalid option: %v", k)
		}

		// Check if the given option is of the valid type
		t := reflect.TypeOf(v)
		if kind != t.Kind() {
			return "", fmt.Errorf("Option %s should be of %v kind, not of %v kind.",
				k, kind, t.Kind())
		}

		values.Set(k, fmt.Sprintf("%v", v))
	}

	p += values.Encode()
	return p, nil
}
<commit_msg>fixed invalid option error<commit_after>package etcd

import (
	"fmt"
	"net/url"
	"reflect"
)

type Options map[string]interface{}

// An internally-used data structure that represents a mapping
// between valid options and their kinds
type validOptions map[string]reflect.Kind

// Valid options for GET, PUT, POST, DELETE
// Using CAPITALIZED_UNDERSCORE to emphasize that these
// values are meant to be used as constants.
var (
	VALID_GET_OPTIONS = validOptions{
		"recursive":  reflect.Bool,
		"consistent": reflect.Bool,
		"quorum":     reflect.Bool,
		"sorted":     reflect.Bool,
		"wait":       reflect.Bool,
		"waitIndex":  reflect.Uint64,
	}

	VALID_PUT_OPTIONS = validOptions{
		"prevValue": reflect.String,
		"prevIndex": reflect.Uint64,
		"prevExist": reflect.Bool,
		"dir":       reflect.Bool,
	}

	VALID_POST_OPTIONS = validOptions{}

	VALID_DELETE_OPTIONS = validOptions{
		"recursive": reflect.Bool,
		"dir":       reflect.Bool,
		"prevValue": reflect.String,
		"prevIndex": reflect.Uint64,
	}
)

// Convert options to a string of HTML parameters
func (ops Options) toParameters(validOps validOptions) (string, error) {
	p := "?"
	values := url.Values{}

	if ops == nil {
		return "", nil
	}

	for k, v := range ops {
		// Check if the given option is valid (that it exists)
		kind := validOps[k]
		if kind == reflect.Invalid {
			return "", fmt.Errorf("Invalid option: %v", k)
		}

		// Check if the given option is of the valid type
		t := reflect.TypeOf(v)
		if kind != t.Kind() {
			return "", fmt.Errorf("Option %s should be of %v kind, not of %v kind.",
				k, kind, t.Kind())
		}

		values.Set(k, fmt.Sprintf("%v", v))
	}

	p += values.Encode()
	return p, nil
}
<|endoftext|>


<commit_before>package conf

import (
	"encoding/json"
	"io"
	"strings"
	"v2ray.com/core"
	"v2ray.com/core/common/errors"
	v2net "v2ray.com/core/common/net"
	"v2ray.com/core/common/serial"
	json_reader "v2ray.com/core/tools/conf/json"
)

var (
	inboundConfigLoader = NewJSONConfigLoader(ConfigCreatorCache{
		"dokodemo-door": func() interface{} { return new(DokodemoConfig) },
		"http":          func() interface{} { return new(HttpServerConfig) },
		"shadowsocks":   func() interface{} { return new(ShadowsocksServerConfig) },
		"socks":         func() interface{} { return new(SocksServerConfig) },
		"vmess":         func() interface{} { return new(VMessInboundConfig) },
	}, "protocol", "settings")

	outboundConfigLoader = NewJSONConfigLoader(ConfigCreatorCache{
		"blackhole":   func() interface{} { return new(BlackholeConfig) },
		"freedom":     func() interface{} { return new(FreedomConfig) },
		"shadowsocks": func() interface{} { return new(ShadowsocksClientConfig) },
		"vmess":       func() interface{} { return new(VMessOutboundConfig) },
	}, "protocol", "settings")
)

type InboundConnectionConfig struct {
	Port          uint16          `json:"port"`
	Listen        *Address        `json:"listen"`
	Protocol      string          `json:"protocol"`
	StreamSetting *StreamConfig   `json:"streamSettings"`
	Settings      json.RawMessage `json:"settings"`
	AllowPassive  bool            `json:"allowPassive"`
	Tag           string          `json:"tag"`
}

func (v *InboundConnectionConfig) Build() (*core.InboundConnectionConfig, error) {
	config := new(core.InboundConnectionConfig)
	config.PortRange = &v2net.PortRange{
		From: uint32(v.Port),
		To:   uint32(v.Port),
	}
	if v.Listen != nil {
		if v.Listen.Family().IsDomain() {
			return nil, errors.New("Point: Unable to listen on domain address: " + v.Listen.Domain())
		}
		config.ListenOn = v.Listen.Build()
	}
	if v.StreamSetting != nil {
		ts, err := v.StreamSetting.Build()
		if err != nil {
			return nil, err
		}
		config.StreamSettings = ts
	}
	config.AllowPassiveConnection = v.AllowPassive

	jsonConfig, err := inboundConfigLoader.LoadWithID(v.Settings, v.Protocol)
	if err != nil {
		return nil, errors.Base(err).Message("Failed to load inbound config.")
	}
	ts, err := jsonConfig.(Buildable).Build()
	if err != nil {
		return nil, err
	}
	config.Settings = ts
	if len(v.Tag) > 0 {
		config.Tag = v.Tag
	}
	return config, nil
}

type OutboundConnectionConfig struct {
	Protocol      string          `json:"protocol"`
	SendThrough   *Address        `json:"sendThrough"`
	StreamSetting *StreamConfig   `json:"streamSettings"`
	ProxySettings *ProxyConfig    `json:"proxySettings"`
	Settings      json.RawMessage `json:"settings"`
	Tag           string          `json:"tag"`
}

func (v *OutboundConnectionConfig) Build() (*core.OutboundConnectionConfig, error) {
	config := new(core.OutboundConnectionConfig)
	rawConfig, err := outboundConfigLoader.LoadWithID(v.Settings, v.Protocol)
	if err != nil {
		return nil, errors.Base(err).Message("Failed to parse outbound config.")
	}
	ts, err := rawConfig.(Buildable).Build()
	if err != nil {
		return nil, err
	}
	config.Settings = ts

	if v.SendThrough != nil {
		address := v.SendThrough
		if address.Family().IsDomain() {
			return nil, errors.New("Invalid sendThrough address: " + address.String())
		}
		config.SendThrough = address.Build()
	}
	if v.StreamSetting != nil {
		ss, err := v.StreamSetting.Build()
		if err != nil {
			return nil, err
		}
		config.StreamSettings = ss
	}
	if v.ProxySettings != nil {
		ps, err := v.ProxySettings.Build()
		if err != nil {
			return nil, errors.Base(err).Message("Invalid outbound proxy settings.")
		}
		config.ProxySettings = ps
	}
	if len(v.Tag) > 0 {
		config.Tag = v.Tag
	}
	return config, nil
}

type InboundDetourAllocationConfig struct {
	Strategy    string  `json:"strategy"`
	Concurrency *uint32 `json:"concurrency"`
	RefreshMin  *uint32 `json:"refresh"`
}

func (v *InboundDetourAllocationConfig) Build() (*core.AllocationStrategy, error) {
	config := new(core.AllocationStrategy)
	switch strings.ToLower(v.Strategy) {
	case "always":
		config.Type = core.AllocationStrategy_Always
	case "random":
		config.Type = core.AllocationStrategy_Random
	case "external":
		config.Type = core.AllocationStrategy_External
	default:
		return nil, errors.New("Unknown allocation strategy: ", v.Strategy)
	}
	if v.Concurrency != nil {
		config.Concurrency = &core.AllocationStrategyConcurrency{
			Value: *v.Concurrency,
		}
	}

	if v.RefreshMin != nil {
		config.Refresh = &core.AllocationStrategyRefresh{
			Value: *v.RefreshMin,
		}
	}

	return config, nil
}

type InboundDetourConfig struct {
	Protocol      string                         `json:"protocol"`
	PortRange     *PortRange                     `json:"port"`
	ListenOn      *Address                       `json:"listen"`
	Settings      json.RawMessage                `json:"settings"`
	Tag           string                         `json:"tag"`
	Allocation    *InboundDetourAllocationConfig `json:"allocate"`
	StreamSetting *StreamConfig                  `json:"streamSettings"`
	AllowPassive  bool                           `json:"allowPassive"`
}

func (v *InboundDetourConfig) Build() (*core.InboundConnectionConfig, error) {
	config := new(core.InboundConnectionConfig)
	if v.PortRange == nil {
		return nil, errors.New("Port range not specified in InboundDetour.")
	}
	config.PortRange = v.PortRange.Build()

	if v.ListenOn != nil {
		if v.ListenOn.Family().IsDomain() {
			return nil, errors.New("Unable to listen on domain address: ", v.ListenOn.Domain())
		}
		config.ListenOn = v.ListenOn.Build()
	}
	config.Tag = v.Tag
	if v.Allocation != nil {
		as, err := v.Allocation.Build()
		if err != nil {
			return nil, err
		}
		config.AllocationStrategy = as
	}
	if v.StreamSetting != nil {
		ss, err := v.StreamSetting.Build()
		if err != nil {
			return nil, err
		}
		config.StreamSettings = ss
	}
	config.AllowPassiveConnection = v.AllowPassive

	rawConfig, err := inboundConfigLoader.LoadWithID(v.Settings, v.Protocol)
	if err != nil {
		return nil, errors.Base(err).Message("Failed to load inbound detour config.")
	}
	ts, err := rawConfig.(Buildable).Build()
	if err != nil {
		return nil, err
	}
	config.Settings = ts
	return config, nil
}

type OutboundDetourConfig struct {
	Protocol      string          `json:"protocol"`
	SendThrough   *Address        `json:"sendThrough"`
	Tag           string          `json:"tag"`
	Settings      json.RawMessage `json:"settings"`
	StreamSetting *StreamConfig   `json:"streamSettings"`
	ProxySettings *ProxyConfig    `json:"proxySettings"`
}

func (v *OutboundDetourConfig) Build() (*core.OutboundConnectionConfig, error) {
	config := new(core.OutboundConnectionConfig)
	config.Tag = v.Tag

	if v.SendThrough != nil {
		address := v.SendThrough
		if address.Family().IsDomain() {
			return nil, errors.New("Point: Unable to send through: " + address.String())
		}
		config.SendThrough = address.Build()
	}

	if v.StreamSetting != nil {
		ss, err := v.StreamSetting.Build()
		if err != nil {
			return nil, err
		}
		config.StreamSettings = ss
	}

	rawConfig, err := outboundConfigLoader.LoadWithID(v.Settings, v.Protocol)
	if err != nil {
		return nil, errors.Base(err).Message("Failed to parse to outbound detour config.")
	}
	ts, err := rawConfig.(Buildable).Build()
	if err != nil {
		return nil, err
	}

	if v.ProxySettings != nil {
		ps, err := v.ProxySettings.Build()
		if err != nil {
			return nil, errors.Base(err).Message("Invalid outbound detour proxy settings.")
		}
		config.ProxySettings = ps
	}
	config.Settings = ts
	return config, nil
}

type Config struct {
	Port            uint16                    `json:"port"` // Port of this Point server.
	LogConfig       *LogConfig                `json:"log"`
	RouterConfig    *RouterConfig             `json:"routing"`
	DNSConfig       *DnsConfig                `json:"dns"`
	InboundConfig   *InboundConnectionConfig  `json:"inbound"`
	OutboundConfig  *OutboundConnectionConfig `json:"outbound"`
	InboundDetours  []InboundDetourConfig     `json:"inboundDetour"`
	OutboundDetours []OutboundDetourConfig    `json:"outboundDetour"`
	Transport       *TransportConfig          `json:"transport"`
}

func (v *Config) Build() (*core.Config, error) {
	config := new(core.Config)

	if v.LogConfig != nil {
		config.Log = v.LogConfig.Build()
	}

	if v.Transport != nil {
		ts, err := v.Transport.Build()
		if err != nil {
			return nil, err
		}
		config.Transport = ts
	}

	if v.RouterConfig != nil {
		routerConfig, err := v.RouterConfig.Build()
		if err != nil {
			return nil, err
		}
		config.App = append(config.App, serial.ToTypedMessage(routerConfig))
	}

	if v.DNSConfig != nil {
		config.App = append(config.App, serial.ToTypedMessage(v.DNSConfig.Build()))
	}

	if v.InboundConfig == nil {
		return nil, errors.New("No inbound config specified.")
	}

	if v.InboundConfig.Port == 0 && v.Port > 0 {
		v.InboundConfig.Port = v.Port
	}

	ic, err := v.InboundConfig.Build()
	if err != nil {
		return nil, err
	}
	config.Inbound = append(config.Inbound, ic)

	for _, rawInboundConfig := range v.InboundDetours {
		ic, err := rawInboundConfig.Build()
		if err != nil {
			return nil, err
		}
		config.Inbound = append(config.Inbound, ic)
	}

	oc, err := v.OutboundConfig.Build()
	if err != nil {
		return nil, err
	}
	config.Outbound = append(config.Outbound, oc)

	for _, rawOutboundConfig := range v.OutboundDetours {
		oc, err := rawOutboundConfig.Build()
		if err != nil {
			return nil, err
		}
		config.Outbound = append(config.Outbound, oc)
	}

	return config, nil
}

func init() {
	core.RegisterConfigLoader(core.ConfigFormat_JSON, func(input io.Reader) (*core.Config, error) {
		jsonConfig := &Config{}
		decoder := json.NewDecoder(&json_reader.Reader{
			Reader: input,
		})
		err := decoder.Decode(jsonConfig)
		if err != nil {
			return nil, errors.Base(err).Message("Invalid V2Ray config.")
		}

		return jsonConfig.Build()
	})
}
<commit_msg>enable socks outbound<commit_after>package conf

import (
	"encoding/json"
	"io"
	"strings"

	"v2ray.com/core"
	"v2ray.com/core/common/errors"
	v2net "v2ray.com/core/common/net"
	"v2ray.com/core/common/serial"
	json_reader "v2ray.com/core/tools/conf/json"
)

var (
	inboundConfigLoader = NewJSONConfigLoader(ConfigCreatorCache{
		"dokodemo-door": func() interface{} { return new(DokodemoConfig) },
		"http":          func() interface{} { return new(HttpServerConfig) },
		"shadowsocks":   func() interface{} { return new(ShadowsocksServerConfig) },
		"socks":         func() interface{} { return new(SocksServerConfig) },
		"vmess":         func() interface{} { return new(VMessInboundConfig) },
	}, "protocol", "settings")

	outboundConfigLoader = NewJSONConfigLoader(ConfigCreatorCache{
		"blackhole":   func() interface{} { return new(BlackholeConfig) },
		"freedom":     func() interface{} { return new(FreedomConfig) },
		"shadowsocks": func() interface{} { return new(ShadowsocksClientConfig) },
		"vmess":       func() interface{} { return new(VMessOutboundConfig) },
		"socks":       func() interface{} { return new(SocksClientConfig) },
	}, "protocol", "settings")
)

type InboundConnectionConfig struct {
	Port          uint16          `json:"port"`
	Listen        *Address        `json:"listen"`
	Protocol      string          `json:"protocol"`
	StreamSetting *StreamConfig   `json:"streamSettings"`
	Settings      json.RawMessage `json:"settings"`
	AllowPassive  bool            `json:"allowPassive"`
	Tag           string          `json:"tag"`
}

func (v *InboundConnectionConfig) Build() (*core.InboundConnectionConfig, error) {
	config := new(core.InboundConnectionConfig)
	config.PortRange = &v2net.PortRange{
		From: uint32(v.Port),
		To:   uint32(v.Port),
	}
	if v.Listen != nil {
		if v.Listen.Family().IsDomain() {
			return nil, errors.New("Point: Unable to listen on domain address: " + v.Listen.Domain())
		}
		config.ListenOn = v.Listen.Build()
	}
	if v.StreamSetting != nil {
		ts, err := v.StreamSetting.Build()
		if err != nil {
			return nil, err
		}
		config.StreamSettings = ts
	}
	config.AllowPassiveConnection = v.AllowPassive

	jsonConfig, err := inboundConfigLoader.LoadWithID(v.Settings, v.Protocol)
	if err != nil {
		return nil, errors.Base(err).Message("Failed to load inbound config.")
	}
	ts, err := jsonConfig.(Buildable).Build()
	if err != nil {
		return nil, err
	}
	config.Settings = ts
	if len(v.Tag) > 0 {
		config.Tag = v.Tag
	}
	return config, nil
}

type OutboundConnectionConfig struct {
	Protocol      string          `json:"protocol"`
	SendThrough   *Address        `json:"sendThrough"`
	StreamSetting *StreamConfig   `json:"streamSettings"`
	ProxySettings *ProxyConfig    `json:"proxySettings"`
	Settings      json.RawMessage `json:"settings"`
	Tag           string          `json:"tag"`
}

func (v *OutboundConnectionConfig) Build() (*core.OutboundConnectionConfig, error) {
	config := new(core.OutboundConnectionConfig)
	rawConfig, err := outboundConfigLoader.LoadWithID(v.Settings, v.Protocol)
	if err != nil {
		return nil, errors.Base(err).Message("Failed to parse outbound config.")
	}
	ts, err := rawConfig.(Buildable).Build()
	if err != nil {
		return nil, err
	}
	config.Settings = ts

	if v.SendThrough != nil {
		address := v.SendThrough
		if address.Family().IsDomain() {
			return nil, errors.New("Invalid sendThrough address: " + address.String())
		}
		config.SendThrough = address.Build()
	}
	if v.StreamSetting != nil {
		ss, err := v.StreamSetting.Build()
		if err != nil {
			return nil, err
		}
		config.StreamSettings = ss
	}
	if v.ProxySettings != nil {
		ps, err := v.ProxySettings.Build()
		if err != nil {
			return nil, errors.Base(err).Message("Invalid outbound proxy settings.")
		}
		config.ProxySettings = ps
	}
	if len(v.Tag) > 0 {
		config.Tag = v.Tag
	}
	return config, nil
}

type InboundDetourAllocationConfig struct {
	Strategy    string  `json:"strategy"`
	Concurrency *uint32 `json:"concurrency"`
	RefreshMin  *uint32 `json:"refresh"`
}

func (v *InboundDetourAllocationConfig) Build() (*core.AllocationStrategy, error) {
	config := new(core.AllocationStrategy)
	switch strings.ToLower(v.Strategy) {
	case "always":
		config.Type = core.AllocationStrategy_Always
	case "random":
		config.Type = core.AllocationStrategy_Random
	case "external":
		config.Type = core.AllocationStrategy_External
	default:
		return nil, errors.New("Unknown allocation strategy: ", v.Strategy)
	}
	if v.Concurrency != nil {
		config.Concurrency = &core.AllocationStrategyConcurrency{
			Value: *v.Concurrency,
		}
	}

	if v.RefreshMin != nil {
		config.Refresh = &core.AllocationStrategyRefresh{
			Value: *v.RefreshMin,
		}
	}

	return config, nil
}

type InboundDetourConfig struct {
	Protocol      string                         `json:"protocol"`
	PortRange     *PortRange                     `json:"port"`
	ListenOn      *Address                       `json:"listen"`
	Settings      json.RawMessage                `json:"settings"`
	Tag           string                         `json:"tag"`
	Allocation    *InboundDetourAllocationConfig `json:"allocate"`
	StreamSetting *StreamConfig                  `json:"streamSettings"`
	AllowPassive  bool                           `json:"allowPassive"`
}

func (v *InboundDetourConfig) Build() (*core.InboundConnectionConfig, error) {
	config := new(core.InboundConnectionConfig)
	if v.PortRange == nil {
		return nil, errors.New("Port range not specified in InboundDetour.")
	}
	config.PortRange = v.PortRange.Build()

	if v.ListenOn != nil {
		if v.ListenOn.Family().IsDomain() {
			return nil, errors.New("Unable to listen on domain address: ", v.ListenOn.Domain())
		}
		config.ListenOn = v.ListenOn.Build()
	}
	config.Tag = v.Tag
	if v.Allocation != nil {
		as, err := v.Allocation.Build()
		if err != nil {
			return nil, err
		}
		config.AllocationStrategy = as
	}
	if v.StreamSetting != nil {
		ss, err := v.StreamSetting.Build()
		if err != nil {
			return nil, err
		}
		config.StreamSettings = ss
	}
	config.AllowPassiveConnection = v.AllowPassive

	rawConfig, err := inboundConfigLoader.LoadWithID(v.Settings, v.Protocol)
	if err != nil {
		return nil, errors.Base(err).Message("Failed to load inbound detour config.")
	}
	ts, err := rawConfig.(Buildable).Build()
	if err != nil {
		return nil, err
	}
	config.Settings = ts
	return config, nil
}

type OutboundDetourConfig struct {
	Protocol      string          `json:"protocol"`
	SendThrough   *Address        `json:"sendThrough"`
	Tag           string          `json:"tag"`
	Settings      json.RawMessage `json:"settings"`
	StreamSetting *StreamConfig   `json:"streamSettings"`
	ProxySettings *ProxyConfig    `json:"proxySettings"`
}

func (v *OutboundDetourConfig) Build() (*core.OutboundConnectionConfig, error) {
	config := new(core.OutboundConnectionConfig)
	config.Tag = v.Tag

	if v.SendThrough != nil {
		address := v.SendThrough
		if address.Family().IsDomain() {
			return nil, errors.New("Point: Unable to send through: " + address.String())
		}
		config.SendThrough = address.Build()
	}

	if v.StreamSetting != nil {
		ss, err := v.StreamSetting.Build()
		if err != nil {
			return nil, err
		}
		config.StreamSettings = ss
	}

	rawConfig, err := outboundConfigLoader.LoadWithID(v.Settings, v.Protocol)
	if err != nil {
		return nil, errors.Base(err).Message("Failed to parse to outbound detour config.")
	}
	ts, err := rawConfig.(Buildable).Build()
	if err != nil {
		return nil, err
	}

	if v.ProxySettings != nil {
		ps, err := v.ProxySettings.Build()
		if err != nil {
			return nil, errors.Base(err).Message("Invalid outbound detour proxy settings.")
		}
		config.ProxySettings = ps
	}
	config.Settings = ts
	return config, nil
}

type Config struct {
	Port            uint16                    `json:"port"` // Port of this Point server.
	LogConfig       *LogConfig                `json:"log"`
	RouterConfig    *RouterConfig             `json:"routing"`
	DNSConfig       *DnsConfig                `json:"dns"`
	InboundConfig   *InboundConnectionConfig  `json:"inbound"`
	OutboundConfig  *OutboundConnectionConfig `json:"outbound"`
	InboundDetours  []InboundDetourConfig     `json:"inboundDetour"`
	OutboundDetours []OutboundDetourConfig    `json:"outboundDetour"`
	Transport       *TransportConfig          `json:"transport"`
}

func (v *Config) Build() (*core.Config, error) {
	config := new(core.Config)

	if v.LogConfig != nil {
		config.Log = v.LogConfig.Build()
	}

	if v.Transport != nil {
		ts, err := v.Transport.Build()
		if err != nil {
			return nil, err
		}
		config.Transport = ts
	}

	if v.RouterConfig != nil {
		routerConfig, err := v.RouterConfig.Build()
		if err != nil {
			return nil, err
		}
		config.App = append(config.App, serial.ToTypedMessage(routerConfig))
	}

	if v.DNSConfig != nil {
		config.App = append(config.App, serial.ToTypedMessage(v.DNSConfig.Build()))
	}

	if v.InboundConfig == nil {
		return nil, errors.New("No inbound config specified.")
	}

	if v.InboundConfig.Port == 0 && v.Port > 0 {
		v.InboundConfig.Port = v.Port
	}

	ic, err := v.InboundConfig.Build()
	if err != nil {
		return nil, err
	}
	config.Inbound = append(config.Inbound, ic)

	for _, rawInboundConfig := range v.InboundDetours {
		ic, err := rawInboundConfig.Build()
		if err != nil {
			return nil, err
		}
		config.Inbound = append(config.Inbound, ic)
	}

	oc, err := v.OutboundConfig.Build()
	if err != nil {
		return nil, err
	}
	config.Outbound = append(config.Outbound, oc)

	for _, rawOutboundConfig := range v.OutboundDetours {
		oc, err := rawOutboundConfig.Build()
		if err != nil {
			return nil, err
		}
		config.Outbound = append(config.Outbound, oc)
	}

	return config, nil
}

func init() {
	core.RegisterConfigLoader(core.ConfigFormat_JSON, func(input io.Reader) (*core.Config, error) {
		jsonConfig := &Config{}
		decoder := json.NewDecoder(&json_reader.Reader{
			Reader: input,
		})
		err := decoder.Decode(jsonConfig)
		if err != nil {
			return nil, errors.Base(err).Message("Invalid V2Ray config.")
		}

		return jsonConfig.Build()
	})
}
<|endoftext|>


<commit_before>package ethutil

import (
	"fmt"
	"reflect"
)

// TODO
// A StateObject is an object that has a state root
// This is goig to be the object for the second level caching (the caching of object which have a state such as contracts)
type StateObject interface {
	State() *Trie
	Sync()
	Undo()
}

type Node struct {
	Key   []byte
	Value *Value
	Dirty bool
}

func NewNode(key []byte, val *Value, dirty bool) *Node {
	return &Node{Key: key, Value: val, Dirty: dirty}
}

func (n *Node) Copy() *Node {
	return NewNode(n.Key, n.Value, n.Dirty)
}

type Cache struct {
	nodes   map[string]*Node
	db      Database
	IsDirty bool
}

func NewCache(db Database) *Cache {
	return &Cache{db: db, nodes: make(map[string]*Node)}
}

func (cache *Cache) Put(v interface{}) interface{} {
	value := NewValue(v)

	enc := value.Encode()
	if len(enc) >= 32 {
		sha := Sha3Bin(enc)

		cache.nodes[string(sha)] = NewNode(sha, value, true)
		cache.IsDirty = true

		return sha
	}

	return v
}

func (cache *Cache) Get(key []byte) *Value {
	// First check if the key is the cache
	if cache.nodes[string(key)] != nil {
		return cache.nodes[string(key)].Value
	}

	// Get the key of the database instead and cache it
	data, _ := cache.db.Get(key)
	// Create the cached value
	value := NewValueFromBytes(data)
	// Create caching node
	cache.nodes[string(key)] = NewNode(key, value, false)

	return value
}

func (cache *Cache) Delete(key []byte) {
	delete(cache.nodes, string(key))

	cache.db.Delete(key)
}

func (cache *Cache) Commit() {
	// Don't try to commit if it isn't dirty
	if !cache.IsDirty {
		return
	}

	for key, node := range cache.nodes {
		if node.Dirty {
			cache.db.Put([]byte(key), node.Value.Encode())
			node.Dirty = false
		}
	}
	cache.IsDirty = false

	// If the nodes grows beyond the 200 entries we simple empty it
	// FIXME come up with something better
	if len(cache.nodes) > 200 {
		cache.nodes = make(map[string]*Node)
	}
}

func (cache *Cache) Undo() {
	for key, node := range cache.nodes {
		if node.Dirty {
			delete(cache.nodes, key)
		}
	}
	cache.IsDirty = false
}

// A (modified) Radix Trie implementation. The Trie implements
// a caching mechanism and will used cached values if they are
// present. If a node is not present in the cache it will try to
// fetch it from the database and store the cached value.
// Please note that the data isn't persisted unless `Sync` is
// explicitly called.
type Trie struct {
	prevRoot interface{}
	Root     interface{}
	//db   Database
	cache *Cache
}

func NewTrie(db Database, Root interface{}) *Trie {
	return &Trie{cache: NewCache(db), Root: Root, prevRoot: Root}
}

// Save the cached value to the database.
func (t *Trie) Sync() {
	t.cache.Commit()
	t.prevRoot = t.Root
}

func (t *Trie) Undo() {
	t.cache.Undo()
	t.Root = t.prevRoot
}

func (t *Trie) Cache() *Cache {
	return t.cache
}

/*
 * Public (query) interface functions
 */
func (t *Trie) Update(key string, value string) {
	k := CompactHexDecode(key)

	t.Root = t.UpdateState(t.Root, k, value)
}

func (t *Trie) Get(key string) string {
	k := CompactHexDecode(key)
	c := NewValue(t.GetState(t.Root, k))

	return c.Str()
}

func (t *Trie) Delete(key string) {
	t.Update(key, "")
}

func (t *Trie) GetState(node interface{}, key []int) interface{} {
	n := NewValue(node)
	// Return the node if key is empty (= found)
	if len(key) == 0 || n.IsNil() || n.Len() == 0 {
		return node
	}

	currentNode := t.GetNode(node)
	length := currentNode.Len()

	if length == 0 {
		return ""
	} else if length == 2 {
		// Decode the key
		k := CompactDecode(currentNode.Get(0).Str())
		v := currentNode.Get(1).Raw()

		if len(key) >= len(k) && CompareIntSlice(k, key[:len(k)]) {
			return t.GetState(v, key[len(k):])
		} else {
			return ""
		}
	} else if length == 17 {
		return t.GetState(currentNode.Get(key[0]).Raw(), key[1:])
	}

	// It shouldn't come this far
	fmt.Println("GetState unexpected return")
	return ""
}

func (t *Trie) GetNode(node interface{}) *Value {
	n := NewValue(node)

	if !n.Get(0).IsNil() {
		return n
	}

	str := n.Str()
	if len(str) == 0 {
		return n
	} else if len(str) < 32 {
		return NewValueFromBytes([]byte(str))
	}

	return t.cache.Get(n.Bytes())
}

func (t *Trie) UpdateState(node interface{}, key []int, value string) interface{} {

	if value != "" {
		return t.InsertState(node, key, value)
	} else {
		// delete it
		return t.DeleteState(node, key)
	}

	return t.Root
}

func (t *Trie) Put(node interface{}) interface{} {
	/*
		enc := Encode(node)
		if len(enc) >= 32 {
			var sha []byte
			sha = Sha3Bin(enc)
			//t.db.Put([]byte(sha), enc)

			return sha
		}
		return node
	*/

	/*
		TODO?
			c := Conv(t.Root)
			fmt.Println(c.Type(), c.Length())
			if c.Type() == reflect.String && c.AsString() == "" {
				return enc
			}
	*/

	return t.cache.Put(node)

}

func EmptyStringSlice(l int) []interface{} {
	slice := make([]interface{}, l)
	for i := 0; i < l; i++ {
		slice[i] = ""
	}
	return slice
}

func (t *Trie) InsertState(node interface{}, key []int, value interface{}) interface{} {
	if len(key) == 0 {
		return value
	}

	// New node
	n := NewValue(node)
	if node == nil || (n.Type() == reflect.String && (n.Str() == "" || n.Get(0).IsNil())) || n.Len() == 0 {
		newNode := []interface{}{CompactEncode(key), value}

		return t.Put(newNode)
	}

	currentNode := t.GetNode(node)
	// Check for "special" 2 slice type node
	if currentNode.Len() == 2 {
		// Decode the key

		k := CompactDecode(currentNode.Get(0).Str())
		v := currentNode.Get(1).Raw()

		// Matching key pair (ie. there's already an object with this key)
		if CompareIntSlice(k, key) {
			newNode := []interface{}{CompactEncode(key), value}
			return t.Put(newNode)
		}

		var newHash interface{}
		matchingLength := MatchingNibbleLength(key, k)
		if matchingLength == len(k) {
			// Insert the hash, creating a new node
			newHash = t.InsertState(v, key[matchingLength:], value)
		} else {
			// Expand the 2 length slice to a 17 length slice
			oldNode := t.InsertState("", k[matchingLength+1:], v)
			newNode := t.InsertState("", key[matchingLength+1:], value)
			// Create an expanded slice
			scaledSlice := EmptyStringSlice(17)
			// Set the copied and new node
			scaledSlice[k[matchingLength]] = oldNode
			scaledSlice[key[matchingLength]] = newNode

			newHash = t.Put(scaledSlice)
		}

		if matchingLength == 0 {
			// End of the chain, return
			return newHash
		} else {
			newNode := []interface{}{CompactEncode(key[:matchingLength]), newHash}
			return t.Put(newNode)
		}
	} else {

		// Copy the current node over to the new node and replace the first nibble in the key
		newNode := EmptyStringSlice(17)

		for i := 0; i < 17; i++ {
			cpy := currentNode.Get(i).Raw()
			if cpy != nil {
				newNode[i] = cpy
			}
		}

		newNode[key[0]] = t.InsertState(currentNode.Get(key[0]).Raw(), key[1:], value)

		return t.Put(newNode)
	}

	return ""
}

func (t *Trie) DeleteState(node interface{}, key []int) interface{} {
	if len(key) == 0 {
		return ""
	}

	// New node
	n := NewValue(node)
	if node == nil || (n.Type() == reflect.String && (n.Str() == "" || n.Get(0).IsNil())) || n.Len() == 0 {
		return ""
	}

	currentNode := t.GetNode(node)
	// Check for "special" 2 slice type node
	if currentNode.Len() == 2 {
		// Decode the key
		k := CompactDecode(currentNode.Get(0).Str())
		v := currentNode.Get(1).Raw()

		// Matching key pair (ie. there's already an object with this key)
		if CompareIntSlice(k, key) {
			return ""
		} else if CompareIntSlice(key[:len(k)], k) {
			hash := t.DeleteState(v, key[len(k):])
			child := t.GetNode(hash)

			var newNode []interface{}
			if child.Len() == 2 {
				newKey := append(k, CompactDecode(child.Get(0).Str())...)
				newNode = []interface{}{CompactEncode(newKey), child.Get(1).Raw()}
			} else {
				newNode = []interface{}{currentNode.Get(0).Str(), hash}
			}

			return t.Put(newNode)
		} else {
			return node
		}
	} else {
		// Copy the current node over to the new node and replace the first nibble in the key
		n := EmptyStringSlice(17)
		var newNode []interface{}

		for i := 0; i < 17; i++ {
			cpy := currentNode.Get(i).Raw()
			if cpy != nil {
				n[i] = cpy
			}
		}

		n[key[0]] = t.DeleteState(n[key[0]], key[1:])
		amount := -1
		for i := 0; i < 17; i++ {
			if n[i] != "" {
				if amount == -1 {
					amount = i
				} else {
					amount = -2
				}
			}
		}
		if amount == 16 {
			newNode = []interface{}{CompactEncode([]int{16}), n[amount]}
		} else if amount >= 0 {
			child := t.GetNode(n[amount])
			if child.Len() == 17 {
				newNode = []interface{}{CompactEncode([]int{amount}), n[amount]}
			} else if child.Len() == 2 {
				key := append([]int{amount}, CompactDecode(child.Get(0).Str())...)
				newNode = []interface{}{CompactEncode(key), child.Get(1).Str()}
			}

		} else {
			newNode = n
		}

		return t.Put(newNode)
	}

	return ""
}

// Simple compare function which creates a rlp value out of the evaluated objects
func (t *Trie) Cmp(trie *Trie) bool {
	return NewValue(t.Root).Cmp(NewValue(trie.Root))
}

// Returns a copy of this trie
func (t *Trie) Copy() *Trie {
	trie := NewTrie(t.cache.db, t.Root)
	for key, node := range t.cache.nodes {
		trie.cache.nodes[key] = node.Copy()
	}

	return trie
}

type TrieIterator struct {
	trie  *Trie
	key   string
	value string

	shas   [][]byte
	values []string
}

func (t *Trie) NewIterator() *TrieIterator {
	return &TrieIterator{trie: t}
}

// Some time in the near future this will need refactoring :-)
// XXX Note to self, IsSlice == inline node. Str == sha3 to node
func (it *TrieIterator) workNode(currentNode *Value) {
	if currentNode.Len() == 2 {
		k := CompactDecode(currentNode.Get(0).Str())

		if currentNode.Get(1).Str() == "" {
			it.workNode(currentNode.Get(1))
		} else {
			if k[len(k)-1] == 16 {
				it.values = append(it.values, currentNode.Get(1).Str())
			} else {
				it.shas = append(it.shas, currentNode.Get(1).Bytes())
				it.getNode(currentNode.Get(1).Bytes())
			}
		}
	} else {
		for i := 0; i < currentNode.Len(); i++ {
			if i == 16 && currentNode.Get(i).Len() != 0 {
				it.values = append(it.values, currentNode.Get(i).Str())
			} else {
				if currentNode.Get(i).Str() == "" {
					it.workNode(currentNode.Get(i))
				} else {
					val := currentNode.Get(i).Str()
					if val != "" {
						it.shas = append(it.shas, currentNode.Get(1).Bytes())
						it.getNode([]byte(val))
					}
				}
			}
		}
	}
}

func (it *TrieIterator) getNode(node []byte) {
	currentNode := it.trie.cache.Get(node)
	it.workNode(currentNode)
}

func (it *TrieIterator) Collect() [][]byte {
	if it.trie.Root == "" {
		return nil
	}

	it.getNode(NewValue(it.trie.Root).Bytes())

	return it.shas
}

func (it *TrieIterator) Purge() int {
	shas := it.Collect()
	for _, sha := range shas {
		it.trie.cache.Delete(sha)
	}
	return len(it.values)
}

func (it *TrieIterator) Key() string {
	return ""
}

func (it *TrieIterator) Value() string {
	return ""
}
<commit_msg>Removed comments<commit_after>package ethutil

import (
	"fmt"
	"reflect"
)

// TODO
// A StateObject is an object that has a state root
// This is goig to be the object for the second level caching (the caching of object which have a state such as contracts)
type StateObject interface {
	State() *Trie
	Sync()
	Undo()
}

type Node struct {
	Key   []byte
	Value *Value
	Dirty bool
}

func NewNode(key []byte, val *Value, dirty bool) *Node {
	return &Node{Key: key, Value: val, Dirty: dirty}
}

func (n *Node) Copy() *Node {
	return NewNode(n.Key, n.Value, n.Dirty)
}

type Cache struct {
	nodes   map[string]*Node
	db      Database
	IsDirty bool
}

func NewCache(db Database) *Cache {
	return &Cache{db: db, nodes: make(map[string]*Node)}
}

func (cache *Cache) Put(v interface{}) interface{} {
	value := NewValue(v)

	enc := value.Encode()
	if len(enc) >= 32 {
		sha := Sha3Bin(enc)

		cache.nodes[string(sha)] = NewNode(sha, value, true)
		cache.IsDirty = true

		return sha
	}

	return v
}

func (cache *Cache) Get(key []byte) *Value {
	// First check if the key is the cache
	if cache.nodes[string(key)] != nil {
		return cache.nodes[string(key)].Value
	}

	// Get the key of the database instead and cache it
	data, _ := cache.db.Get(key)
	// Create the cached value
	value := NewValueFromBytes(data)
	// Create caching node
	cache.nodes[string(key)] = NewNode(key, value, false)

	return value
}

func (cache *Cache) Delete(key []byte) {
	delete(cache.nodes, string(key))

	cache.db.Delete(key)
}

func (cache *Cache) Commit() {
	// Don't try to commit if it isn't dirty
	if !cache.IsDirty {
		return
	}

	for key, node := range cache.nodes {
		if node.Dirty {
			cache.db.Put([]byte(key), node.Value.Encode())
			node.Dirty = false
		}
	}
	cache.IsDirty = false

	// If the nodes grows beyond the 200 entries we simple empty it
	// FIXME come up with something better
	if len(cache.nodes) > 200 {
		cache.nodes = make(map[string]*Node)
	}
}

func (cache *Cache) Undo() {
	for key, node := range cache.nodes {
		if node.Dirty {
			delete(cache.nodes, key)
		}
	}
	cache.IsDirty = false
}

// A (modified) Radix Trie implementation. The Trie implements
// a caching mechanism and will used cached values if they are
// present. If a node is not present in the cache it will try to
// fetch it from the database and store the cached value.
// Please note that the data isn't persisted unless `Sync` is
// explicitly called.
type Trie struct {
	prevRoot interface{}
	Root     interface{}
	//db   Database
	cache *Cache
}

func NewTrie(db Database, Root interface{}) *Trie {
	return &Trie{cache: NewCache(db), Root: Root, prevRoot: Root}
}

// Save the cached value to the database.
func (t *Trie) Sync() {
	t.cache.Commit()
	t.prevRoot = t.Root
}

func (t *Trie) Undo() {
	t.cache.Undo()
	t.Root = t.prevRoot
}

func (t *Trie) Cache() *Cache {
	return t.cache
}

/*
 * Public (query) interface functions
 */
func (t *Trie) Update(key string, value string) {
	k := CompactHexDecode(key)

	t.Root = t.UpdateState(t.Root, k, value)
}

func (t *Trie) Get(key string) string {
	k := CompactHexDecode(key)
	c := NewValue(t.GetState(t.Root, k))

	return c.Str()
}

func (t *Trie) Delete(key string) {
	t.Update(key, "")
}

func (t *Trie) GetState(node interface{}, key []int) interface{} {
	n := NewValue(node)
	// Return the node if key is empty (= found)
	if len(key) == 0 || n.IsNil() || n.Len() == 0 {
		return node
	}

	currentNode := t.GetNode(node)
	length := currentNode.Len()

	if length == 0 {
		return ""
	} else if length == 2 {
		// Decode the key
		k := CompactDecode(currentNode.Get(0).Str())
		v := currentNode.Get(1).Raw()

		if len(key) >= len(k) && CompareIntSlice(k, key[:len(k)]) {
			return t.GetState(v, key[len(k):])
		} else {
			return ""
		}
	} else if length == 17 {
		return t.GetState(currentNode.Get(key[0]).Raw(), key[1:])
	}

	// It shouldn't come this far
	fmt.Println("GetState unexpected return")
	return ""
}

func (t *Trie) GetNode(node interface{}) *Value {
	n := NewValue(node)

	if !n.Get(0).IsNil() {
		return n
	}

	str := n.Str()
	if len(str) == 0 {
		return n
	} else if len(str) < 32 {
		return NewValueFromBytes([]byte(str))
	}

	return t.cache.Get(n.Bytes())
}

func (t *Trie) UpdateState(node interface{}, key []int, value string) interface{} {

	if value != "" {
		return t.InsertState(node, key, value)
	} else {
		// delete it
		return t.DeleteState(node, key)
	}

	return t.Root
}

func (t *Trie) Put(node interface{}) interface{} {
	/*
		TODO?
			c := Conv(t.Root)
			fmt.Println(c.Type(), c.Length())
			if c.Type() == reflect.String && c.AsString() == "" {
				return enc
			}
	*/

	return t.cache.Put(node)

}

func EmptyStringSlice(l int) []interface{} {
	slice := make([]interface{}, l)
	for i := 0; i < l; i++ {
		slice[i] = ""
	}
	return slice
}

func (t *Trie) InsertState(node interface{}, key []int, value interface{}) interface{} {
	if len(key) == 0 {
		return value
	}

	// New node
	n := NewValue(node)
	if node == nil || (n.Type() == reflect.String && (n.Str() == "" || n.Get(0).IsNil())) || n.Len() == 0 {
		newNode := []interface{}{CompactEncode(key), value}

		return t.Put(newNode)
	}

	currentNode := t.GetNode(node)
	// Check for "special" 2 slice type node
	if currentNode.Len() == 2 {
		// Decode the key

		k := CompactDecode(currentNode.Get(0).Str())
		v := currentNode.Get(1).Raw()

		// Matching key pair (ie. there's already an object with this key)
		if CompareIntSlice(k, key) {
			newNode := []interface{}{CompactEncode(key), value}
			return t.Put(newNode)
		}

		var newHash interface{}
		matchingLength := MatchingNibbleLength(key, k)
		if matchingLength == len(k) {
			// Insert the hash, creating a new node
			newHash = t.InsertState(v, key[matchingLength:], value)
		} else {
			// Expand the 2 length slice to a 17 length slice
			oldNode := t.InsertState("", k[matchingLength+1:], v)
			newNode := t.InsertState("", key[matchingLength+1:], value)
			// Create an expanded slice
			scaledSlice := EmptyStringSlice(17)
			// Set the copied and new node
			scaledSlice[k[matchingLength]] = oldNode
			scaledSlice[key[matchingLength]] = newNode

			newHash = t.Put(scaledSlice)
		}

		if matchingLength == 0 {
			// End of the chain, return
			return newHash
		} else {
			newNode := []interface{}{CompactEncode(key[:matchingLength]), newHash}
			return t.Put(newNode)
		}
	} else {

		// Copy the current node over to the new node and replace the first nibble in the key
		newNode := EmptyStringSlice(17)

		for i := 0; i < 17; i++ {
			cpy := currentNode.Get(i).Raw()
			if cpy != nil {
				newNode[i] = cpy
			}
		}

		newNode[key[0]] = t.InsertState(currentNode.Get(key[0]).Raw(), key[1:], value)

		return t.Put(newNode)
	}

	return ""
}

func (t *Trie) DeleteState(node interface{}, key []int) interface{} {
	if len(key) == 0 {
		return ""
	}

	// New node
	n := NewValue(node)
	if node == nil || (n.Type() == reflect.String && (n.Str() == "" || n.Get(0).IsNil())) || n.Len() == 0 {
		return ""
	}

	currentNode := t.GetNode(node)
	// Check for "special" 2 slice type node
	if currentNode.Len() == 2 {
		// Decode the key
		k := CompactDecode(currentNode.Get(0).Str())
		v := currentNode.Get(1).Raw()

		// Matching key pair (ie. there's already an object with this key)
		if CompareIntSlice(k, key) {
			return ""
		} else if CompareIntSlice(key[:len(k)], k) {
			hash := t.DeleteState(v, key[len(k):])
			child := t.GetNode(hash)

			var newNode []interface{}
			if child.Len() == 2 {
				newKey := append(k, CompactDecode(child.Get(0).Str())...)
				newNode = []interface{}{CompactEncode(newKey), child.Get(1).Raw()}
			} else {
				newNode = []interface{}{currentNode.Get(0).Str(), hash}
			}

			return t.Put(newNode)
		} else {
			return node
		}
	} else {
		// Copy the current node over to the new node and replace the first nibble in the key
		n := EmptyStringSlice(17)
		var newNode []interface{}

		for i := 0; i < 17; i++ {
			cpy := currentNode.Get(i).Raw()
			if cpy != nil {
				n[i] = cpy
			}
		}

		n[key[0]] = t.DeleteState(n[key[0]], key[1:])
		amount := -1
		for i := 0; i < 17; i++ {
			if n[i] != "" {
				if amount == -1 {
					amount = i
				} else {
					amount = -2
				}
			}
		}
		if amount == 16 {
			newNode = []interface{}{CompactEncode([]int{16}), n[amount]}
		} else if amount >= 0 {
			child := t.GetNode(n[amount])
			if child.Len() == 17 {
				newNode = []interface{}{CompactEncode([]int{amount}), n[amount]}
			} else if child.Len() == 2 {
				key := append([]int{amount}, CompactDecode(child.Get(0).Str())...)
				newNode = []interface{}{CompactEncode(key), child.Get(1).Str()}
			}

		} else {
			newNode = n
		}

		return t.Put(newNode)
	}

	return ""
}

// Simple compare function which creates a rlp value out of the evaluated objects
func (t *Trie) Cmp(trie *Trie) bool {
	return NewValue(t.Root).Cmp(NewValue(trie.Root))
}

// Returns a copy of this trie
func (t *Trie) Copy() *Trie {
	trie := NewTrie(t.cache.db, t.Root)
	for key, node := range t.cache.nodes {
		trie.cache.nodes[key] = node.Copy()
	}

	return trie
}

type TrieIterator struct {
	trie  *Trie
	key   string
	value string

	shas   [][]byte
	values []string
}

func (t *Trie) NewIterator() *TrieIterator {
	return &TrieIterator{trie: t}
}

// Some time in the near future this will need refactoring :-)
// XXX Note to self, IsSlice == inline node. Str == sha3 to node
func (it *TrieIterator) workNode(currentNode *Value) {
	if currentNode.Len() == 2 {
		k := CompactDecode(currentNode.Get(0).Str())

		if currentNode.Get(1).Str() == "" {
			it.workNode(currentNode.Get(1))
		} else {
			if k[len(k)-1] == 16 {
				it.values = append(it.values, currentNode.Get(1).Str())
			} else {
				it.shas = append(it.shas, currentNode.Get(1).Bytes())
				it.getNode(currentNode.Get(1).Bytes())
			}
		}
	} else {
		for i := 0; i < currentNode.Len(); i++ {
			if i == 16 && currentNode.Get(i).Len() != 0 {
				it.values = append(it.values, currentNode.Get(i).Str())
			} else {
				if currentNode.Get(i).Str() == "" {
					it.workNode(currentNode.Get(i))
				} else {
					val := currentNode.Get(i).Str()
					if val != "" {
						it.shas = append(it.shas, currentNode.Get(1).Bytes())
						it.getNode([]byte(val))
					}
				}
			}
		}
	}
}

func (it *TrieIterator) getNode(node []byte) {
	currentNode := it.trie.cache.Get(node)
	it.workNode(currentNode)
}

func (it *TrieIterator) Collect() [][]byte {
	if it.trie.Root == "" {
		return nil
	}

	it.getNode(NewValue(it.trie.Root).Bytes())

	return it.shas
}

func (it *TrieIterator) Purge() int {
	shas := it.Collect()
	for _, sha := range shas {
		it.trie.cache.Delete(sha)
	}
	return len(it.values)
}

func (it *TrieIterator) Key() string {
	return ""
}

func (it *TrieIterator) Value() string {
	return ""
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"errors"
)

type TykEvent string	// A type so we can ENUM event types easily, e.g. EVENT_QuotaExceeded
type TykEventHandlerName string // A type for handler codes in API definitions

// The name for event handlers as defined in the API Definition JSON/BSON format
const (
	EH_LogHandler TykEventHandlerName = "eh_log_handler"
)

// Register new event types here
const (
	EVENT_QuotaExceeded TykEvent = "QuotaExceeded"
	EVENT_RateLimitExceeded TykEvent = "RatelimitExceeded"
	EVENT_AuthFailure TykEvent = "AuthFailure"
	EVENT_KeyExpired TykEvent = "KeyExpired"
	EVENT_VersionFailure TykEvent = "VersionFailure"
)

// EventMetaDefault is a standard embedded struct to be used with custom event metadata types, gives an interface for
// easily extending event metadata objects
type EventMetaDefault struct {
	Message string
}

// EVENT_QuotaExceededMeta is the metadata structure for a quota exceeded event (EVENT_QuotaExceeded)
type EVENT_QuotaExceededMeta struct {
	EventMetaDefault
	Path string
	Origin string
	Key string
}

// EVENT_RateLimitExceededMeta is the metadata structure for a rate limit exceeded event (EVENT_RateLimitExceeded)
type EVENT_RateLimitExceededMeta struct {
	EventMetaDefault
	Path string
	Origin string
	Key string
}

// EVENT_AuthFailureMeta is the metadata structure for an auth failure (EVENT_AuthFailure)
type EVENT_AuthFailureMeta struct {
	EventMetaDefault
	Path string
	Origin string
	Key string
}

// EVENT_KeyExpiredMeta is the metadata structure for an auth failure (EVENT_KeyExpired)
type EVENT_KeyExpiredMeta struct {
	EventMetaDefault
	Path string
	Origin string
	Key string
}

// EVENT_VersionFailureMeta is the metadata structure for an auth failure (EVENT_KeyExpired)
type EVENT_VersionFailureMeta struct {
	EventMetaDefault
	Path string
	Origin string
	Key string
	Reason string
}

// EventMessage is a standard form to send event data to handlers
type EventMessage struct {
	EventType TykEvent
	EventMetaData interface{}
}

// TykEventHandler defines an event handler, e.g. LogMessageEventHandler will handle an event by logging it to stdout.
type TykEventHandler interface {
	New(interface{}) TykEventHandler
	HandleEvent(EventMessage)
}

// GetEventHandlerByName is a convenience function to get event handler instances from an API Definition
func GetEventHandlerByName(handlerConf EventHandlerTriggerConfig) (TykEventHandler, error) {
	switch handlerConf.Handler {
		case EH_LogHandler: return LogMessageEventHandler{}.New(handlerConf.HandlerMeta), nil
	}

	return nil, errors.New("Handler not found")
}

// FireEvent is added to the tykMiddleware object so it is available across the entire stack
func (t TykMiddleware) FireEvent(eventName TykEvent, eventMetaData interface{}) {

	log.Debug("EVENT FIRED")

	handlers, handlerExists := t.Spec.EventPaths[eventName]

	if handlerExists {
		log.Debug("FOUND EVENT HANDLERS")
		eventMessage := EventMessage{}
		eventMessage.EventMetaData = eventMetaData
		eventMessage.EventType = eventName

		for _, handler := range(handlers) {
			log.Debug("FIRING HANDLER")
			go handler.HandleEvent(eventMessage)
		}
	}
}

// LogMessageEventHandler is a sample Event Handler
type LogMessageEventHandler struct {
	conf map[string]interface{}
}

// New enables the intitialisation of event handler instances when they are created on ApiSpec creation
func (l LogMessageEventHandler) New(handlerConf interface{}) TykEventHandler {
	thisHandler := LogMessageEventHandler{}
	thisHandler.conf = handlerConf.(map[string]interface{})

	return thisHandler
}
// HandleEvent will be fired when the event handler instance is found in an APISpec EventPaths object during a request chain
func (l LogMessageEventHandler) HandleEvent(em EventMessage) {
	var msgConf EVENT_QuotaExceededMeta
	// type assert the metadata and then use it however you like
	msgConf = em.EventMetaData.(EVENT_QuotaExceededMeta)

	var formattedMsgString string
	formattedMsgString = fmt.Sprintf("%s:%s:%s", l.conf["prefix"].(string), em.EventType, msgConf.Message)

	// We can handle specific event types easily
	if em.EventType == EVENT_QuotaExceeded {
		formattedMsgString = fmt.Sprintf("%s:%s:%s:%s", formattedMsgString, msgConf.Key, msgConf.Origin, msgConf.Path)
	}

	log.Warning(formattedMsgString)
}

<commit_msg>minor doc fix<commit_after>package main

import (
	"fmt"
	"errors"
)

type TykEvent string	// A type so we can ENUM event types easily, e.g. EVENT_QuotaExceeded
type TykEventHandlerName string // A type for handler codes in API definitions

// The name for event handlers as defined in the API Definition JSON/BSON format
const (
	EH_LogHandler TykEventHandlerName = "eh_log_handler"
)

// Register new event types here, the string is the code used to hook at the Api Deifnititon JSON/BSON level
const (
	EVENT_QuotaExceeded TykEvent = "QuotaExceeded"
	EVENT_RateLimitExceeded TykEvent = "RatelimitExceeded"
	EVENT_AuthFailure TykEvent = "AuthFailure"
	EVENT_KeyExpired TykEvent = "KeyExpired"
	EVENT_VersionFailure TykEvent = "VersionFailure"
)

// EventMetaDefault is a standard embedded struct to be used with custom event metadata types, gives an interface for
// easily extending event metadata objects
type EventMetaDefault struct {
	Message string
}

// EVENT_QuotaExceededMeta is the metadata structure for a quota exceeded event (EVENT_QuotaExceeded)
type EVENT_QuotaExceededMeta struct {
	EventMetaDefault
	Path string
	Origin string
	Key string
}

// EVENT_RateLimitExceededMeta is the metadata structure for a rate limit exceeded event (EVENT_RateLimitExceeded)
type EVENT_RateLimitExceededMeta struct {
	EventMetaDefault
	Path string
	Origin string
	Key string
}

// EVENT_AuthFailureMeta is the metadata structure for an auth failure (EVENT_AuthFailure)
type EVENT_AuthFailureMeta struct {
	EventMetaDefault
	Path string
	Origin string
	Key string
}

// EVENT_KeyExpiredMeta is the metadata structure for an auth failure (EVENT_KeyExpired)
type EVENT_KeyExpiredMeta struct {
	EventMetaDefault
	Path string
	Origin string
	Key string
}

// EVENT_VersionFailureMeta is the metadata structure for an auth failure (EVENT_KeyExpired)
type EVENT_VersionFailureMeta struct {
	EventMetaDefault
	Path string
	Origin string
	Key string
	Reason string
}

// EventMessage is a standard form to send event data to handlers
type EventMessage struct {
	EventType TykEvent
	EventMetaData interface{}
}

// TykEventHandler defines an event handler, e.g. LogMessageEventHandler will handle an event by logging it to stdout.
type TykEventHandler interface {
	New(interface{}) TykEventHandler
	HandleEvent(EventMessage)
}

// GetEventHandlerByName is a convenience function to get event handler instances from an API Definition
func GetEventHandlerByName(handlerConf EventHandlerTriggerConfig) (TykEventHandler, error) {
	switch handlerConf.Handler {
		case EH_LogHandler: return LogMessageEventHandler{}.New(handlerConf.HandlerMeta), nil
	}

	return nil, errors.New("Handler not found")
}

// FireEvent is added to the tykMiddleware object so it is available across the entire stack
func (t TykMiddleware) FireEvent(eventName TykEvent, eventMetaData interface{}) {

	log.Debug("EVENT FIRED")

	handlers, handlerExists := t.Spec.EventPaths[eventName]

	if handlerExists {
		log.Debug("FOUND EVENT HANDLERS")
		eventMessage := EventMessage{}
		eventMessage.EventMetaData = eventMetaData
		eventMessage.EventType = eventName

		for _, handler := range(handlers) {
			log.Debug("FIRING HANDLER")
			go handler.HandleEvent(eventMessage)
		}
	}
}

// LogMessageEventHandler is a sample Event Handler
type LogMessageEventHandler struct {
	conf map[string]interface{}
}

// New enables the intitialisation of event handler instances when they are created on ApiSpec creation
func (l LogMessageEventHandler) New(handlerConf interface{}) TykEventHandler {
	thisHandler := LogMessageEventHandler{}
	thisHandler.conf = handlerConf.(map[string]interface{})

	return thisHandler
}
// HandleEvent will be fired when the event handler instance is found in an APISpec EventPaths object during a request chain
func (l LogMessageEventHandler) HandleEvent(em EventMessage) {
	var msgConf EVENT_QuotaExceededMeta
	// type assert the metadata and then use it however you like
	msgConf = em.EventMetaData.(EVENT_QuotaExceededMeta)

	var formattedMsgString string
	formattedMsgString = fmt.Sprintf("%s:%s:%s", l.conf["prefix"].(string), em.EventType, msgConf.Message)

	// We can handle specific event types easily
	if em.EventType == EVENT_QuotaExceeded {
		formattedMsgString = fmt.Sprintf("%s:%s:%s:%s", formattedMsgString, msgConf.Key, msgConf.Origin, msgConf.Path)
	}

	log.Warning(formattedMsgString)
}

<|endoftext|>


<commit_before>package docker

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/dotcloud/docker/utils"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
)

type Archive io.Reader

type Compression uint32

const (
	Uncompressed Compression = iota
	Bzip2
	Gzip
	Xz
)

func DetectCompression(source []byte) Compression {
	for _, c := range source[:10] {
		utils.Debugf("%x", c)
	}

	sourceLen := len(source)
	for compression, m := range map[Compression][]byte{
		Bzip2: {0x42, 0x5A, 0x68},
		Gzip:  {0x1F, 0x8B, 0x08},
		Xz:    {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
	} {
		fail := false
		if len(m) > sourceLen {
			utils.Debugf("Len too short")
			continue
		}
		i := 0
		for _, b := range m {
			if b != source[i] {
				fail = true
				break
			}
			i++
		}
		if !fail {
			return compression
		}
	}
	return Uncompressed
}

func (compression *Compression) Flag() string {
	switch *compression {
	case Bzip2:
		return "j"
	case Gzip:
		return "z"
	case Xz:
		return "J"
	}
	return ""
}

func (compression *Compression) Extension() string {
	switch *compression {
	case Uncompressed:
		return "tar"
	case Bzip2:
		return "tar.bz2"
	case Gzip:
		return "tar.gz"
	case Xz:
		return "tar.xz"
	}
	return ""
}

func Tar(path string, compression Compression) (io.Reader, error) {
	return CmdStream(exec.Command("tar", "-f", "-", "-C", path, "-c"+compression.Flag(), "."))
}

func Untar(archive io.Reader, path string) error {

	buf := make([]byte, 10)
	if _, err := archive.Read(buf); err != nil {
		return err
	}
	compression := DetectCompression(buf)
	archive = io.MultiReader(bytes.NewReader(buf), archive)

	utils.Debugf("Archive compression detected: %s", compression.Extension())

	cmd := exec.Command("tar", "-f", "-", "-C", path, "-x"+compression.Flag())
	cmd.Stdin = archive
	// Hardcode locale environment for predictable outcome regardless of host configuration.
	//   (see https://github.com/dotcloud/docker/issues/355)
	cmd.Env = []string{"LANG=en_US.utf-8", "LC_ALL=en_US.utf-8"}
	output, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("%s: %s", err, output)
	}
	return nil
}

// CmdStream executes a command, and returns its stdout as a stream.
// If the command fails to run or doesn't complete successfully, an error
// will be returned, including anything written on stderr.
func CmdStream(cmd *exec.Cmd) (io.Reader, error) {
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return nil, err
	}
	pipeR, pipeW := io.Pipe()
	errChan := make(chan []byte)
	// Collect stderr, we will use it in case of an error
	go func() {
		errText, e := ioutil.ReadAll(stderr)
		if e != nil {
			errText = []byte("(...couldn't fetch stderr: " + e.Error() + ")")
		}
		errChan <- errText
	}()
	// Copy stdout to the returned pipe
	go func() {
		_, err := io.Copy(pipeW, stdout)
		if err != nil {
			pipeW.CloseWithError(err)
		}
		errText := <-errChan
		if err := cmd.Wait(); err != nil {
			pipeW.CloseWithError(errors.New(err.Error() + ": " + string(errText)))
		} else {
			pipeW.Close()
		}
	}()
	// Run the command and return the pipe
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	return pipeR, nil
}

// NewTempArchive reads the content of src into a temporary file, and returns the contents
// of that file as an archive. The archive can only be read once - as soon as reading completes,
// the file will be deleted.
func NewTempArchive(src Archive, dir string) (*TempArchive, error) {
	f, err := ioutil.TempFile(dir, "")
	if err != nil {
		return nil, err
	}
	if _, err := io.Copy(f, src); err != nil {
		return nil, err
	}
	if _, err := f.Seek(0, 0); err != nil {
		return nil, err
	}
	st, err := f.Stat()
	if err != nil {
		return nil, err
	}
	size := st.Size()
	return &TempArchive{f, size}, nil
}

type TempArchive struct {
	*os.File
	Size int64 // Pre-computed from Stat().Size() as a convenience
}

func (archive *TempArchive) Read(data []byte) (int, error) {
	n, err := archive.File.Read(data)
	if err != nil {
		os.Remove(archive.File.Name())
	}
	return n, err
}
<commit_msg>Fix bug on compression detection when chunck < 10bytes<commit_after>package docker

import (
	"bufio"
	"errors"
	"fmt"
	"github.com/dotcloud/docker/utils"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
)

type Archive io.Reader

type Compression uint32

const (
	Uncompressed Compression = iota
	Bzip2
	Gzip
	Xz
)

func DetectCompression(source []byte) Compression {
	for _, c := range source[:10] {
		utils.Debugf("%x", c)
	}

	sourceLen := len(source)
	for compression, m := range map[Compression][]byte{
		Bzip2: {0x42, 0x5A, 0x68},
		Gzip:  {0x1F, 0x8B, 0x08},
		Xz:    {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
	} {
		fail := false
		if len(m) > sourceLen {
			utils.Debugf("Len too short")
			continue
		}
		i := 0
		for _, b := range m {
			if b != source[i] {
				fail = true
				break
			}
			i++
		}
		if !fail {
			return compression
		}
	}
	return Uncompressed
}

func (compression *Compression) Flag() string {
	switch *compression {
	case Bzip2:
		return "j"
	case Gzip:
		return "z"
	case Xz:
		return "J"
	}
	return ""
}

func (compression *Compression) Extension() string {
	switch *compression {
	case Uncompressed:
		return "tar"
	case Bzip2:
		return "tar.bz2"
	case Gzip:
		return "tar.gz"
	case Xz:
		return "tar.xz"
	}
	return ""
}

func Tar(path string, compression Compression) (io.Reader, error) {
	return CmdStream(exec.Command("tar", "-f", "-", "-C", path, "-c"+compression.Flag(), "."))
}

func Untar(archive io.Reader, path string) error {

	bufferedArchive := bufio.NewReaderSize(archive, 10)
	buf, err := bufferedArchive.Peek(10)
	if err != nil {
		return err
	}
	compression := DetectCompression(buf)

	utils.Debugf("Archive compression detected: %s", compression.Extension())

	cmd := exec.Command("tar", "-f", "-", "-C", path, "-x"+compression.Flag())
	cmd.Stdin = bufferedArchive
	// Hardcode locale environment for predictable outcome regardless of host configuration.
	//   (see https://github.com/dotcloud/docker/issues/355)
	cmd.Env = []string{"LANG=en_US.utf-8", "LC_ALL=en_US.utf-8"}
	output, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("%s: %s", err, output)
	}
	return nil
}

// CmdStream executes a command, and returns its stdout as a stream.
// If the command fails to run or doesn't complete successfully, an error
// will be returned, including anything written on stderr.
func CmdStream(cmd *exec.Cmd) (io.Reader, error) {
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return nil, err
	}
	pipeR, pipeW := io.Pipe()
	errChan := make(chan []byte)
	// Collect stderr, we will use it in case of an error
	go func() {
		errText, e := ioutil.ReadAll(stderr)
		if e != nil {
			errText = []byte("(...couldn't fetch stderr: " + e.Error() + ")")
		}
		errChan <- errText
	}()
	// Copy stdout to the returned pipe
	go func() {
		_, err := io.Copy(pipeW, stdout)
		if err != nil {
			pipeW.CloseWithError(err)
		}
		errText := <-errChan
		if err := cmd.Wait(); err != nil {
			pipeW.CloseWithError(errors.New(err.Error() + ": " + string(errText)))
		} else {
			pipeW.Close()
		}
	}()
	// Run the command and return the pipe
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	return pipeR, nil
}

// NewTempArchive reads the content of src into a temporary file, and returns the contents
// of that file as an archive. The archive can only be read once - as soon as reading completes,
// the file will be deleted.
func NewTempArchive(src Archive, dir string) (*TempArchive, error) {
	f, err := ioutil.TempFile(dir, "")
	if err != nil {
		return nil, err
	}
	if _, err := io.Copy(f, src); err != nil {
		return nil, err
	}
	if _, err := f.Seek(0, 0); err != nil {
		return nil, err
	}
	st, err := f.Stat()
	if err != nil {
		return nil, err
	}
	size := st.Size()
	return &TempArchive{f, size}, nil
}

type TempArchive struct {
	*os.File
	Size int64 // Pre-computed from Stat().Size() as a convenience
}

func (archive *TempArchive) Read(data []byte) (int, error) {
	n, err := archive.File.Read(data)
	if err != nil {
		os.Remove(archive.File.Name())
	}
	return n, err
}
<|endoftext|>


<commit_before>package resourcecollector

import (
	"fmt"

	"github.com/libopenstorage/stork/drivers/volume"
	stork_api "github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
)

func (r *ResourceCollector) pvToBeCollected(
	includeObjects map[stork_api.ObjectInfo]bool,
	labelSelectors map[string]string,
	object runtime.Unstructured,
	namespace string,
	allDrivers bool,
) (bool, error) {
	var pv v1.PersistentVolume
	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), &pv); err != nil {
		return false, fmt.Errorf("error converting to persistent volume: %v", err)
	}

	// Only collect Bound PVs
	if pv.Status.Phase != v1.VolumeBound {
		return false, nil
	}

	if pv.Spec.ClaimRef != nil {
		pvcName := pv.Spec.ClaimRef.Name
		// Collect only PVs which have a reference to a PVC in the namespace requested
		if pvcName == "" {
			return false, nil
		}
		pvcNamespace := pv.Spec.ClaimRef.Namespace
		if pvcNamespace != namespace {
			return false, nil
		}

		pvc, err := r.coreOps.GetPersistentVolumeClaim(pvcName, pvcNamespace)
		if err != nil {
			return false, err
		}

		if len(includeObjects) > 0 {
			info := stork_api.ObjectInfo{
				GroupVersionKind: metav1.GroupVersionKind{
					Group:   "core",
					Version: "v1",
					Kind:    "PersistentVolumeClaim",
				},
				Name:      pvc.Name,
				Namespace: pvc.Namespace,
			}
			if val, present := includeObjects[info]; !present || !val {
				return false, nil
			}
		}

		// Don't collect PVCs not owned by the driver if collecting for a specific
		// driver
		if !allDrivers && !r.Driver.OwnsPVC(r.coreOps, pvc) {
			return false, nil
		}
		// Else collect PVCs for all supported drivers
		_, err = volume.GetPVCDriver(r.coreOps, pvc)
		if err != nil {
			return false, nil
		}

		// Also check the labels on the PVC since the PV doesn't inherit the labels
		if len(pvc.Labels) == 0 && len(labelSelectors) > 0 {
			return false, nil
		}

		// labels.AreLabelsInWhiteList removed in k8s 1.20. It has been replaced with isSubset:
		// https://github.com/kubernetes/kubernetes/commit/c9051308befb12f66c3e222de6df9d29f3f2f77d
		if !isSubset(labels.Set(labelSelectors),
			labels.Set(pvc.Labels)) {
			return false, nil
		}
		return true, nil
	}
	return false, nil
}

func (r *ResourceCollector) preparePVResourceForCollection(
	object runtime.Unstructured,
) error {
	err := unstructured.SetNestedField(object.UnstructuredContent(), nil, "spec", "claimRef")
	if err != nil {
		return err
	}
	return unstructured.SetNestedField(object.UnstructuredContent(), "", "spec", "storageClassName")
}

// Updates the PV by pointing to the new volume. Also updated the name of the PV
// itself. The restored PVC will point to this new PV name.
func (r *ResourceCollector) preparePVResourceForApply(
	object runtime.Unstructured,
	pvNameMappings map[string]string,
	vInfo []*stork_api.ApplicationRestoreVolumeInfo,
) (bool, error) {
	var updatedName string
	var present bool

	var pv v1.PersistentVolume
	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), &pv); err != nil {
		return false, fmt.Errorf("error converting to persistent volume: %v", err)
	}

	// Skip the PV if it isn't bound to a PVC that needs to be restored
	if len(pvNameMappings) == 0 {
		return true, nil
	}
	if updatedName, present = pvNameMappings[pv.Name]; !present {
		return true, nil
	}

	pv.Name = updatedName
	var driverName string
	var volumeInfo *stork_api.ApplicationRestoreVolumeInfo
	for _, vol := range vInfo {
		if vol.RestoreVolume == pv.Name {
			volumeInfo = vol
			driverName = vol.DriverName
			break
		}
	}

	driver, err := volume.Get(driverName)
	if err != nil {
		return false, err
	}
	_, err = driver.UpdateMigratedPersistentVolumeSpec(&pv, volumeInfo)
	if err != nil {
		return false, err
	}
	o, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pv)
	if err != nil {
		return false, err
	}
	object.SetUnstructuredContent(o)

	return false, err
}

func isSubset(subSet, superSet labels.Set) bool {
	if len(superSet) == 0 {
		return true
	}

	for k, v := range subSet {
		value, ok := superSet[k]
		if !ok {
			return false
		}
		if value != v {
			return false
		}
	}
	return true
}
<commit_msg>PWX-22971: application clone failing at resource stage<commit_after>package resourcecollector

import (
	"fmt"

	"github.com/libopenstorage/stork/drivers/volume"
	stork_api "github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
)

func (r *ResourceCollector) pvToBeCollected(
	includeObjects map[stork_api.ObjectInfo]bool,
	labelSelectors map[string]string,
	object runtime.Unstructured,
	namespace string,
	allDrivers bool,
) (bool, error) {
	var pv v1.PersistentVolume
	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), &pv); err != nil {
		return false, fmt.Errorf("error converting to persistent volume: %v", err)
	}

	// Only collect Bound PVs
	if pv.Status.Phase != v1.VolumeBound {
		return false, nil
	}

	if pv.Spec.ClaimRef != nil {
		pvcName := pv.Spec.ClaimRef.Name
		// Collect only PVs which have a reference to a PVC in the namespace requested
		if pvcName == "" {
			return false, nil
		}
		pvcNamespace := pv.Spec.ClaimRef.Namespace
		if pvcNamespace != namespace {
			return false, nil
		}

		pvc, err := r.coreOps.GetPersistentVolumeClaim(pvcName, pvcNamespace)
		if err != nil {
			return false, err
		}

		if len(includeObjects) > 0 {
			info := stork_api.ObjectInfo{
				GroupVersionKind: metav1.GroupVersionKind{
					Group:   "core",
					Version: "v1",
					Kind:    "PersistentVolumeClaim",
				},
				Name:      pvc.Name,
				Namespace: pvc.Namespace,
			}
			if val, present := includeObjects[info]; !present || !val {
				return false, nil
			}
		}

		// Don't collect PVCs not owned by the driver if collecting for a specific
		// driver
		if !allDrivers && !r.Driver.OwnsPVC(r.coreOps, pvc) {
			return false, nil
		}
		// Else collect PVCs for all supported drivers
		_, err = volume.GetPVCDriver(r.coreOps, pvc)
		if err != nil {
			return false, nil
		}

		// Also check the labels on the PVC since the PV doesn't inherit the labels
		if len(pvc.Labels) == 0 && len(labelSelectors) > 0 {
			return false, nil
		}

		// labels.AreLabelsInWhiteList removed in k8s 1.20. It has been replaced with isSubset:
		// https://github.com/kubernetes/kubernetes/commit/c9051308befb12f66c3e222de6df9d29f3f2f77d
		if !isSubset(labels.Set(labelSelectors),
			labels.Set(pvc.Labels)) {
			return false, nil
		}
		return true, nil
	}
	return false, nil
}

func (r *ResourceCollector) preparePVResourceForCollection(
	object runtime.Unstructured,
) error {
	err := unstructured.SetNestedField(object.UnstructuredContent(), nil, "spec", "claimRef")
	if err != nil {
		return err
	}
	return unstructured.SetNestedField(object.UnstructuredContent(), "", "spec", "storageClassName")
}

// Updates the PV by pointing to the new volume. Also updated the name of the PV
// itself. The restored PVC will point to this new PV name.
func (r *ResourceCollector) preparePVResourceForApply(
	object runtime.Unstructured,
	pvNameMappings map[string]string,
	vInfo []*stork_api.ApplicationRestoreVolumeInfo,
) (bool, error) {
	var updatedName string
	var present bool

	var pv v1.PersistentVolume
	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), &pv); err != nil {
		return false, fmt.Errorf("error converting to persistent volume: %v", err)
	}

	// Skip the PV if it isn't bound to a PVC that needs to be restored
	if len(pvNameMappings) == 0 {
		return true, nil
	}
	if updatedName, present = pvNameMappings[pv.Name]; !present {
		return true, nil
	}

	pv.Name = updatedName
	var driverName string
	var volumeInfo *stork_api.ApplicationRestoreVolumeInfo
	for _, vol := range vInfo {
		if vol.RestoreVolume == pv.Name {
			volumeInfo = vol
			driverName = vol.DriverName
			break
		}
	}
	// in case of non-restore call make sure resourcecollector
	// checks proper driver by looking at pv name
	if driverName == "" {
		var err error
		driverName, err = volume.GetPVDriver(&pv)
		if err != nil {
			return false, err
		}
	}

	driver, err := volume.Get(driverName)
	if err != nil {
		return false, err
	}
	_, err = driver.UpdateMigratedPersistentVolumeSpec(&pv, volumeInfo)
	if err != nil {
		return false, err
	}
	o, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pv)
	if err != nil {
		return false, err
	}
	object.SetUnstructuredContent(o)

	return false, err
}

func isSubset(subSet, superSet labels.Set) bool {
	if len(superSet) == 0 {
		return true
	}

	for k, v := range subSet {
		value, ok := superSet[k]
		if !ok {
			return false
		}
		if value != v {
			return false
		}
	}
	return true
}
<|endoftext|>