<commit_before>package download

import (
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/subutai-io/agent/log"
	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
)

type AptItem struct {
	ID           string            `json:"id"`
	Size         string            `json:"size"`
	Name         string            `json:"name,omitempty"`
	Owner        []string          `json:"owner,omitempty"`
	Version      string            `json:"version,omitempty"`
	Filename     string            `json:"filename,omitempty"`
	Signature    map[string]string `json:"signature,omitempty"`
	Description  string            `json:"description,omitempty"`
	Architecture string            `json:"architecture,omitempty"`
}

type RawItem struct {
	ID        string            `json:"id"`
	Size      int64             `json:"size"`
	Name      string            `json:"name,omitempty"`
	Owner     []string          `json:"owner,omitempty"`
	Version   string            `json:"version,omitempty"`
	Signature map[string]string `json:"signature,omitempty"`
}

type ListItem struct {
	ID           string            `json:"id"`
	Size         int64             `json:"size"`
	Name         string            `json:"name"`
	Owner        []string          `json:"owner"`
	Parent       string            `json:"parent"`
	Version      string            `json:"version"`
	Filename     string            `json:"filename"`
	Prefsize     string            `json:"prefsize"`
	Architecture string            `json:"architecture"`
	Signature    map[string]string `json:"signature,omitempty"`
}

func Handler(repo string, w http.ResponseWriter, r *http.Request) {
	hash := r.URL.Query().Get("hash")
	name := r.URL.Query().Get("name")
	if len(r.URL.Query().Get("id")) > 0 {
		hash = r.URL.Query().Get("id")
		if tmp := strings.Split(hash, "."); len(tmp) > 1 {
			hash = tmp[1]
		}
	}
	if len(hash) == 0 && len(name) == 0 {
		io.WriteString(w, "Please specify hash or name")
		return
	} else if len(name) != 0 {
		hash = db.LastHash(name, repo)
	}

	if len(db.Read(hash)) > 0 && !db.Public(hash) && !db.CheckShare(hash, db.CheckToken(r.URL.Query().Get("token"))) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("Not found"))
		return
	}

	// if len(db.Read(hash)) == 0 && repo == "template" && !torrent.IsDownloaded(hash) {
	// 	torrent.AddTorrent(hash)
	// 	w.WriteHeader(http.StatusAccepted)
	// 	w.Write([]byte(torrent.Info(hash)))
	// 	return
	// }

	f, err := os.Open(config.Storage.Path + hash)
	defer f.Close()

	if log.Check(log.WarnLevel, "Opening file "+config.Storage.Path+hash, err) || len(hash) == 0 {
		if len(config.CDN.Node) > 0 {
			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
			resp, err := client.Get(config.CDN.Node + r.URL.RequestURI())
			if !log.Check(log.WarnLevel, "Getting file from CDN", err) {
				w.Header().Set("Content-Length", resp.Header.Get("Content-Length"))
				w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
				w.Header().Set("Last-Modified", resp.Header.Get("Last-Modified"))
				w.Header().Set("Content-Disposition", resp.Header.Get("Content-Disposition"))

				io.Copy(w, resp.Body)
				resp.Body.Close()
				return
			}
		}

		w.WriteHeader(http.StatusNotFound)
		io.WriteString(w, "File not found")
		return
	}
	fi, _ := f.Stat()

	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && fi.ModTime().Unix() <= t.Unix() {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	w.Header().Set("Content-Length", fmt.Sprint(fi.Size()))
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))

	if name = db.Read(hash); len(name) == 0 && len(config.CDN.Node) > 0 {
		httpclient := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
		resp, err := httpclient.Get(config.CDN.Node + "/kurjun/rest/template/info?id=" + hash)
		if !log.Check(log.WarnLevel, "Getting info from CDN", err) {
			var info ListItem
			rsp, err := ioutil.ReadAll(resp.Body)
			if log.Check(log.WarnLevel, "Reading from CDN response", err) {
				w.WriteHeader(http.StatusNotFound)
				io.WriteString(w, "File not found")
				return
			}
			if !log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &info)) {
				w.Header().Set("Content-Disposition", "attachment; filename=\""+info.Filename+"\"")
			}
			resp.Body.Close()
		}
	} else {
		w.Header().Set("Content-Disposition", "attachment; filename=\""+db.Read(hash)+"\"")
	}

	io.Copy(w, f)
}

func Info(repo string, r *http.Request) []byte {
	var js []byte
	var info map[string]string
	var counter int
	p := []int{0, 1000}

	id := r.URL.Query().Get("id")
	name := r.URL.Query().Get("name")
	page := r.URL.Query().Get("page")
	owner := r.URL.Query().Get("owner")
	token := r.URL.Query().Get("token")
	version := r.URL.Query().Get("version")
	verified := r.URL.Query().Get("verified")

	list := db.Search(name)
	if len(id) > 0 {
		list = append(list[:0], id)
	} else if verified == "true" {
		return getVerified(list, name, repo)
	}

	pstr := strings.Split(page, ",")
	p[0], _ = strconv.Atoi(pstr[0])
	if len(pstr) == 2 {
		p[1], _ = strconv.Atoi(pstr[1])
	}

	for _, k := range list {
		if (!db.Public(k) && !db.CheckShare(k, db.CheckToken(token))) ||
			(len(owner) > 0 && db.CheckRepo(owner, repo, k) == 0) ||
			db.CheckRepo("", repo, k) == 0 {
			continue
		}

		if counter++; counter < p[0] {
			continue
		}

		if name == "management" && repo == "template" {
			info = db.LatestTmpl(name, version)
			if len(info["name"]) == 0 {
				continue
			}
		} else {
			info = db.Info(k)
		}

		item, _ := formatItem(info, repo, name)

		if strings.HasPrefix(info["name"], name+"-subutai-template") || name == info["name"] {
			if (len(version) == 0 || strings.Contains(info["version"], version)) && k == db.LastHash(info["name"], repo) {
				return item
			}
			continue
		}

		if counter == p[0]+p[1] {
			break
		} else if len(js) > 0 {
			js = append(js, []byte(",")...)
		}

		js = append(js, item...)
	}
	if counter > 1 {
		js = append([]byte("["), js...)
		js = append(js, []byte("]")...)
	}
	return js
}

// ProxyList retrieves list of artifacts from main CDN nodes if no data found in local database
// It creates simple JSON list of artifacts to provide it to Subutai Social.
func ProxyList(t string) []byte {
	if len(config.CDN.Node) == 0 {
		return nil
	}
	list := make([]ListItem, 0)

	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
	resp, err := client.Get(config.CDN.Node + "/kurjun/rest/" + t + "/list")
	defer resp.Body.Close()
	if log.Check(log.WarnLevel, "Getting list from CDN", err) {
		return nil
	}

	rsp, err := ioutil.ReadAll(resp.Body)
	if log.Check(log.WarnLevel, "Reading from CDN response", err) {
		return nil
	}

	if log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &list)) {
		return nil
	}

	output, err := json.Marshal(list)
	if log.Check(log.WarnLevel, "Marshaling list", err) {
		return nil
	}
	return output
}

// ProxyInfo retrieves information from main CDN nodes if no data found in local database
// It creates simple info JSON to provide it to Subutai Social.
func ProxyInfo(uri string) []byte {
	if len(config.CDN.Node) == 0 {
		return nil
	}
	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
	resp, err := client.Get(config.CDN.Node + uri)
	defer resp.Body.Close()
	if log.Check(log.WarnLevel, "Getting list of templates from CDN", err) {
		return nil
	}

	rsp, err := ioutil.ReadAll(resp.Body)
	if log.Check(log.WarnLevel, "Reading from CDN response", err) {
		return nil
	}
	return rsp
}

func in(str string, list []string) bool {
	for _, s := range list {
		if s == str {
			return true
		}
	}
	return false
}

func getVerified(list []string, name, repo string) []byte {
	for _, k := range list {
		if info := db.Info(k); db.CheckRepo("", repo, k) > 0 {
			if info["name"] == name || (strings.HasPrefix(info["name"], name+"-subutai-template") && repo == "template") {
				for _, owner := range db.FileOwner(info["id"]) {
					if in(owner, []string{"subutai", "jenkins", "docker"}) {
						item, _ := formatItem(info, repo, name)
						return item
					}
				}
			}
		}
	}
	return nil
}

func formatItem(info map[string]string, repo, name string) ([]byte, error) {
	size, err := strconv.ParseInt(info["size"], 10, 64)
	if err != nil {
		return nil, err
	}

	switch repo {

	case "template":
		if len(info["prefsize"]) == 0 {
			info["prefsize"] = "tiny"
		}
		item, err := json.Marshal(ListItem{
			ID:           info["id"],
			Name:         strings.Split(info["name"], "-subutai-template")[0],
			Size:         size,
			Owner:        db.FileOwner(info["id"]),
			Version:      info["version"],
			Filename:     info["name"],
			Parent:       info["parent"],
			Prefsize:     info["prefsize"],
			Architecture: strings.ToUpper(info["arch"]),
			Signature:    db.FileSignatures(info["id"], name),
		})
		return item, err

	case "apt":
		item, err := json.Marshal(AptItem{
			ID:           info["id"],
			Name:         info["name"],
			Size:         info["size"],
			Owner:        db.FileOwner(info["id"]),
			Version:      info["Version"],
			Description:  info["Description"],
			Architecture: info["Architecture"],
			Signature:    db.FileSignatures(info["id"], name),
		})
		return item, err

	case "raw":
		item, err := json.Marshal(RawItem{
			ID:        info["id"],
			Name:      info["name"],
			Size:      size,
			Owner:     db.FileOwner(info["id"]),
			Version:   info["version"],
			Signature: db.FileSignatures(info["id"], name),
		})
		return item, err
	}

	return nil, errors.New("Failed to process item.")
}
<commit_msg>Now info returns only array of elements. #56<commit_after>package download

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/subutai-io/agent/log"
	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
)

type ListItem struct {
	ID           string            `json:"id"`
	Size         string            `json:"size,omitempty"`
	Name         string            `json:"name,omitempty"`
	Owner        []string          `json:"owner,omitempty"`
	Parent       string            `json:"parent,omitempty"`
	Version      string            `json:"version,omitempty"`
	Filename     string            `json:"filename,omitempty"`
	Prefsize     string            `json:"prefsize,omitempty"`
	Signature    map[string]string `json:"signature,omitempty"`
	Description  string            `json:"description,omitempty"`
	Architecture string            `json:"architecture,omitempty"`
}

func Handler(repo string, w http.ResponseWriter, r *http.Request) {
	hash := r.URL.Query().Get("hash")
	name := r.URL.Query().Get("name")
	if len(r.URL.Query().Get("id")) > 0 {
		hash = r.URL.Query().Get("id")
		if tmp := strings.Split(hash, "."); len(tmp) > 1 {
			hash = tmp[1]
		}
	}
	if len(hash) == 0 && len(name) == 0 {
		io.WriteString(w, "Please specify hash or name")
		return
	} else if len(name) != 0 {
		hash = db.LastHash(name, repo)
	}

	if len(db.Read(hash)) > 0 && !db.Public(hash) && !db.CheckShare(hash, db.CheckToken(r.URL.Query().Get("token"))) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("Not found"))
		return
	}

	// if len(db.Read(hash)) == 0 && repo == "template" && !torrent.IsDownloaded(hash) {
	// 	torrent.AddTorrent(hash)
	// 	w.WriteHeader(http.StatusAccepted)
	// 	w.Write([]byte(torrent.Info(hash)))
	// 	return
	// }

	f, err := os.Open(config.Storage.Path + hash)
	defer f.Close()

	if log.Check(log.WarnLevel, "Opening file "+config.Storage.Path+hash, err) || len(hash) == 0 {
		if len(config.CDN.Node) > 0 {
			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
			resp, err := client.Get(config.CDN.Node + r.URL.RequestURI())
			if !log.Check(log.WarnLevel, "Getting file from CDN", err) {
				w.Header().Set("Content-Length", resp.Header.Get("Content-Length"))
				w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
				w.Header().Set("Last-Modified", resp.Header.Get("Last-Modified"))
				w.Header().Set("Content-Disposition", resp.Header.Get("Content-Disposition"))

				io.Copy(w, resp.Body)
				resp.Body.Close()
				return
			}
		}

		w.WriteHeader(http.StatusNotFound)
		io.WriteString(w, "File not found")
		return
	}
	fi, _ := f.Stat()

	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && fi.ModTime().Unix() <= t.Unix() {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	w.Header().Set("Content-Length", fmt.Sprint(fi.Size()))
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))

	if name = db.Read(hash); len(name) == 0 && len(config.CDN.Node) > 0 {
		httpclient := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
		resp, err := httpclient.Get(config.CDN.Node + "/kurjun/rest/template/info?id=" + hash)
		if !log.Check(log.WarnLevel, "Getting info from CDN", err) {
			var info ListItem
			rsp, err := ioutil.ReadAll(resp.Body)
			if log.Check(log.WarnLevel, "Reading from CDN response", err) {
				w.WriteHeader(http.StatusNotFound)
				io.WriteString(w, "File not found")
				return
			}
			if !log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &info)) {
				w.Header().Set("Content-Disposition", "attachment; filename=\""+info.Filename+"\"")
			}
			resp.Body.Close()
		}
	} else {
		w.Header().Set("Content-Disposition", "attachment; filename=\""+db.Read(hash)+"\"")
	}

	io.Copy(w, f)
}

func Info(repo string, r *http.Request) []byte {
	var items []ListItem
	var info map[string]string
	p := []int{0, 1000}

	id := r.URL.Query().Get("id")
	name := r.URL.Query().Get("name")
	page := r.URL.Query().Get("page")
	owner := r.URL.Query().Get("owner")
	token := r.URL.Query().Get("token")
	version := r.URL.Query().Get("version")
	verified := r.URL.Query().Get("verified")

	list := db.Search(name)
	if len(id) > 0 {
		list = append(list[:0], id)
	} else if verified == "true" {
		items := append(items, getVerified(list, name, repo))
		output, err := json.Marshal(items)
		if err != nil || string(output) == "null" {
			return nil
		}
		return output
	}

	pstr := strings.Split(page, ",")
	p[0], _ = strconv.Atoi(pstr[0])
	if len(pstr) == 2 {
		p[1], _ = strconv.Atoi(pstr[1])
	}

	for _, k := range list {
		if (!db.Public(k) && !db.CheckShare(k, db.CheckToken(token))) ||
			(len(owner) > 0 && db.CheckRepo(owner, repo, k) == 0) ||
			db.CheckRepo("", repo, k) == 0 {
			continue
		}

		if len(items) < p[0] {
			continue
		}

		if name == "management" && repo == "template" {
			info = db.LatestTmpl(name, version)
			if len(info["name"]) == 0 {
				continue
			}
		} else {
			info = db.Info(k)
		}

		item := formatItem(info, repo, name)

		if strings.HasPrefix(info["name"], name+"-subutai-template") || name == info["name"] {
			if (len(version) == 0 || strings.Contains(info["version"], version)) && k == db.LastHash(info["name"], repo) {
				items = []ListItem{item}
				break
			}
			continue
		}

		if len(items) == p[0]+p[1] {
			break
		}
		items = append(items, item)
	}
	output, err := json.Marshal(items)
	if err != nil || string(output) == "null" {
		return nil
	}
	return output
}

// ProxyList retrieves list of artifacts from main CDN nodes if no data found in local database
// It creates simple JSON list of artifacts to provide it to Subutai Social.
func ProxyList(t string) []byte {
	if len(config.CDN.Node) == 0 {
		return nil
	}
	list := make([]ListItem, 0)

	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
	resp, err := client.Get(config.CDN.Node + "/kurjun/rest/" + t + "/list")
	defer resp.Body.Close()
	if log.Check(log.WarnLevel, "Getting list from CDN", err) {
		return nil
	}

	rsp, err := ioutil.ReadAll(resp.Body)
	if log.Check(log.WarnLevel, "Reading from CDN response", err) {
		return nil
	}

	if log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &list)) {
		return nil
	}

	output, err := json.Marshal(list)
	if log.Check(log.WarnLevel, "Marshaling list", err) {
		return nil
	}
	return output
}

// ProxyInfo retrieves information from main CDN nodes if no data found in local database
// It creates simple info JSON to provide it to Subutai Social.
func ProxyInfo(uri string) []byte {
	if len(config.CDN.Node) == 0 {
		return nil
	}
	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
	resp, err := client.Get(config.CDN.Node + uri)
	defer resp.Body.Close()
	if log.Check(log.WarnLevel, "Getting list of templates from CDN", err) {
		return nil
	}

	rsp, err := ioutil.ReadAll(resp.Body)
	if log.Check(log.WarnLevel, "Reading from CDN response", err) {
		return nil
	}
	return rsp
}

func in(str string, list []string) bool {
	for _, s := range list {
		if s == str {
			return true
		}
	}
	return false
}

func getVerified(list []string, name, repo string) ListItem {
	for _, k := range list {
		if info := db.Info(k); db.CheckRepo("", repo, k) > 0 {
			if info["name"] == name || (strings.HasPrefix(info["name"], name+"-subutai-template") && repo == "template") {
				for _, owner := range db.FileOwner(info["id"]) {
					if in(owner, []string{"subutai", "jenkins", "docker"}) {
						return formatItem(info, repo, name)
					}
				}
			}
		}
	}
	return ListItem{}
}

func formatItem(info map[string]string, repo, name string) ListItem {
	if len(info["prefsize"]) == 0 && repo == "template" {
		info["prefsize"] = "tiny"
	}
	item := ListItem{
		ID:           info["id"],
		Name:         strings.Split(info["name"], "-subutai-template")[0],
		Size:         info["size"],
		Owner:        db.FileOwner(info["id"]),
		Version:      info["version"],
		Filename:     info["name"],
		Parent:       info["parent"],
		Prefsize:     info["prefsize"],
		Architecture: strings.ToUpper(info["arch"]),
		Signature:    db.FileSignatures(info["id"], name),
		Description:  info["Description"],
	}

	if repo == "apt" {
		item.Architecture = info["Architecture"]
	}

	return item
}
<|endoftext|>


<commit_before>package proxy

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"log"
	"math/rand"
	"sync"
	"time"

	"github.com/yinqiwen/gsnova/common/event"
)

const (
	stateCloseToSendReq  = 1
	stateCloseWaitingACK = 2
)

var ErrChannelReadTimeout = errors.New("Remote channel read timeout")
var ErrChannelAuthFailed = errors.New("Remote channel auth failed")

type ProxyChannel interface {
	Write(event.Event) (event.Event, error)
}

type RemoteProxyChannel interface {
	Open(iv uint64) error
	Closed() bool
	Request([]byte) ([]byte, error)
	ReadTimeout() time.Duration
	io.ReadWriteCloser
}

type RemoteChannel struct {
	Addr            string
	Index           int
	DirectIO        bool
	WriteJoinAuth   bool
	OpenJoinAuth    bool
	HeartBeat       bool
	ReconnectPeriod int
	C               RemoteProxyChannel

	connSendedEvents uint32
	authResult       int
	iv               uint64
	wch              chan event.Event
	running          bool

	connectTime time.Time
	closeState  int
	// activeSids     map[uint32]bool
	// activeSidMutex sync.Mutex
}

// func (rc *RemoteChannel) updateActiveSid(id uint32, insertOrRemove bool) {
// 	rc.activeSidMutex.Lock()
// 	if insertOrRemove {
// 		rc.activeSids[id] = true
// 	} else {
// 		delete(rc.activeSids, id)
// 	}
// 	rc.activeSidMutex.Unlock()
// }
// func (rc *RemoteChannel) activeSidSize() int {
// 	rc.activeSidMutex.Lock()
// 	s := len(rc.activeSids)
// 	rc.activeSidMutex.Unlock()
// 	return s
// }

func (rc *RemoteChannel) authed() bool {
	return rc.authResult != 0
}
func (rc *RemoteChannel) generateIV() uint64 {
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	tmp := uint64(r.Int63())
	rc.iv = tmp
	return tmp
}

func (rc *RemoteChannel) Init() error {
	rc.running = true
	rc.authResult = 0
	//rc.activeSids = make(map[uint32]bool)

	//authSession := newRandomSession()
	if !rc.DirectIO {
		rc.wch = make(chan event.Event, 5)
		go rc.processWrite()
		go rc.processRead()
	}
	if rc.HeartBeat {
		go rc.heartbeat()
	}

	start := time.Now()
	authTimeout := rc.C.ReadTimeout()
	for rc.authResult == 0 {
		if time.Now().After(start.Add(authTimeout)) {
			rc.Stop()
			rc.authResult = -1 //timeout
			return fmt.Errorf("Server:%s auth timeout after %v", rc.Addr, time.Now().Sub(start))
		}
		time.Sleep(1 * time.Millisecond)
	}
	if rc.authResult == event.ErrAuthFailed {
		rc.Stop()
		return fmt.Errorf("Server:%s auth failed.", rc.Addr)
	} else if rc.authResult == event.SuccessAuthed {
		log.Printf("Server:%s authed success.", rc.Addr)
	} else {
		return fmt.Errorf("Server:%s auth recv unexpected code:%d.", rc.Addr, rc.authResult)
	}
	//closeProxySession(authSession.id)
	return nil
}
func (rc *RemoteChannel) Close() {
	c := rc.C
	if nil != c {
		c.Close()
	}
}
func (rc *RemoteChannel) Stop() {
	rc.running = false
	rc.Close()
}

func (rc *RemoteChannel) heartbeat() {
	ticker := time.NewTicker(5 * time.Second)
	for rc.running {
		select {
		case <-ticker.C:
			if !rc.C.Closed() && getProxySessionSize() > 0 {
				rc.Write(&event.HeartBeatEvent{})
			}
		}
	}
}

func (rc *RemoteChannel) processWrite() {
	readBufferEv := func(evs []event.Event) []event.Event {
		sev := <-rc.wch
		if nil != sev {
			evs = append(evs, sev)
		}
		return evs
	}
	var sendEvents []event.Event
	for rc.running {
		conn := rc.C
		//disable write if waiting for close CK
		if rc.closeState == stateCloseWaitingACK {
			time.Sleep(1 * time.Millisecond)
			continue
		}

		if len(sendEvents) == 0 {
			if len(rc.wch) > 0 {
				for len(rc.wch) > 0 {
					sendEvents = readBufferEv(sendEvents)
				}
			} else {
				sendEvents = readBufferEv(sendEvents)
			}
		}

		if !rc.running && len(sendEvents) == 0 {
			return
		}
		if conn.Closed() {
			time.Sleep(5 * time.Millisecond)
			continue
		}

		var buf bytes.Buffer
		civ := rc.iv
		if rc.WriteJoinAuth || (rc.connSendedEvents == 0 && rc.OpenJoinAuth) {
			auth := NewAuthEvent()
			auth.Index = int64(rc.Index)
			auth.IV = civ
			event.EncryptEvent(&buf, auth, 0)
			rc.connSendedEvents++
		}

		for _, sev := range sendEvents {
			if auth, ok := sev.(*event.AuthEvent); ok {
				if auth.IV != civ {
					log.Printf("####Got %d %d", civ, auth.IV)
				}
				auth.IV = civ
				event.EncryptEvent(&buf, sev, 0)
			} else {
				event.EncryptEvent(&buf, sev, civ)
			}
		}
		if rc.closeState == stateCloseToSendReq {
			closeReq := &event.ChannelCloseReqEvent{}
			event.EncryptEvent(&buf, closeReq, civ)
		}
		rc.connSendedEvents += uint32(len(sendEvents))

		if buf.Len() > 0 {
			//start := time.Now()
			_, err := conn.Write(buf.Bytes())
			if nil != err {
				conn.Close()
				log.Printf("Failed to write tcp messgage:%v", err)
				//resend `sendEvents` in next process
			} else {
				//log.Printf("[%d]%s cost %v to write %d events.", rc.Index, rc.Addr, time.Now().Sub(start), len(sendEvents))
				sendEvents = nil
				//set state if write success
				if rc.closeState == stateCloseToSendReq {
					rc.closeState = stateCloseWaitingACK
				}
			}
		}
	}
}

func (rc *RemoteChannel) processRead() {
	for rc.running {
		conn := rc.C
		if conn.Closed() {
			rc.closeState = 0
			if rc.authed() && getProxySessionSize() == 0 {
				time.Sleep(10 * time.Millisecond)
				continue
			}
			rc.generateIV()
			rc.connSendedEvents = 0
			err := conn.Open(rc.iv)
			if nil != err {
				log.Printf("Channel[%d] connect %s failed:%v.", rc.Index, rc.Addr, err)
				time.Sleep(1 * time.Second)
				continue
			}
			rc.connectTime = time.Now()
			log.Printf("Channel[%d] connect %s success.", rc.Index, rc.Addr)
			if rc.OpenJoinAuth {
				rc.Write(nil)
			}
		}
		data := make([]byte, 8192)
		var buf bytes.Buffer
		for {
			n, cerr := conn.Read(data)
			buf.Write(data[0:n])
			if rc.ReconnectPeriod > 0 && rc.closeState == 0 {
				if rc.connectTime.Add(time.Duration(rc.ReconnectPeriod) * time.Second).Before(time.Now()) {
					rc.closeState = stateCloseToSendReq
					rc.Write(nil) //trigger to write ChannelCloseReqEvent
					log.Printf("Channel[%d] prepare to close %s to reconnect.", rc.Index, rc.Addr)
				}
			}
			for buf.Len() > 0 {
				err, ev := event.DecryptEvent(&buf, rc.iv)
				if nil != err {
					if err == event.EBNR {
						err = nil
					} else {
						log.Printf("Failed to decode event for reason:%v", err)
						conn.Close()
					}
					break
				}
				switch ev.(type) {
				case *event.NotifyEvent:
					if !rc.authed() {
						auth := ev.(*event.NotifyEvent)
						rc.authResult = int(auth.Code)
						continue
					}
				case *event.ChannelCloseACKEvent:
					conn.Close()
					log.Printf("Channel[%d] close %s after recved close ACK.", rc.Index, rc.Addr)
					continue
				}
				if !rc.authed() {
					log.Printf("[ERROR]Expected auth result event for auth all connection, but got %T.", ev)
					conn.Close()
					continue
				}
				HandleEvent(ev)
			}
			if nil != cerr {
				if cerr != io.EOF && cerr != ErrChannelReadTimeout {
					log.Printf("Failed to read channel for reason:%v", cerr)
				}
				conn.Close()
				break
			}
		}
	}
}

func (rc *RemoteChannel) Request(ev event.Event) (event.Event, error) {
	var buf bytes.Buffer
	auth := NewAuthEvent()
	auth.Index = int64(rc.Index)
	auth.IV = rc.generateIV()
	event.EncryptEvent(&buf, auth, 0)
	event.EncryptEvent(&buf, ev, auth.IV)
	//event.EncodeEvent(&buf, ev)
	res, err := rc.C.Request(buf.Bytes())
	if nil != err {
		return nil, err
	}
	rbuf := bytes.NewBuffer(res)
	var rev event.Event
	err, rev = event.DecryptEvent(rbuf, auth.IV)
	if nil != err {
		return nil, err
	}
	return rev, nil
}

func (rc *RemoteChannel) Write(ev event.Event) error {
	// if nil != ev {
	// 	rc.updateActiveSid(ev.GetId(), true)
	// }
	rc.wch <- ev
	return nil
}

func (rc *RemoteChannel) WriteRaw(p []byte) (int, error) {
	return rc.C.Write(p)
}

type RemoteChannelTable struct {
	cs     []*RemoteChannel
	cursor int
	mutex  sync.Mutex
}

func (p *RemoteChannelTable) Add(c *RemoteChannel) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	p.cs = append(p.cs, c)
}

func (p *RemoteChannelTable) StopAll() {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	for _, c := range p.cs {
		c.Stop()
	}
	p.cs = make([]*RemoteChannel, 0)
}

func (p *RemoteChannelTable) Select() *RemoteChannel {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	if len(p.cs) == 0 {
		return nil
	}
	startCursor := p.cursor
	for {
		if p.cursor >= len(p.cs) {
			p.cursor = 0
		}
		c := p.cs[p.cursor]
		p.cursor++
		if nil != c {
			return c
		}
		if p.cursor == startCursor {
			break
		}
	}
	return nil
}

func NewRemoteChannelTable() *RemoteChannelTable {
	p := new(RemoteChannelTable)
	p.cs = make([]*RemoteChannel, 0)
	return p
}
<commit_msg>reduce local memory usage by reuse read/write buffer<commit_after>package proxy

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"log"
	"math/rand"
	"sync"
	"time"

	"github.com/yinqiwen/gsnova/common/event"
)

const (
	stateCloseToSendReq  = 1
	stateCloseWaitingACK = 2
)

var ErrChannelReadTimeout = errors.New("Remote channel read timeout")
var ErrChannelAuthFailed = errors.New("Remote channel auth failed")

type ProxyChannel interface {
	Write(event.Event) (event.Event, error)
}

type RemoteProxyChannel interface {
	Open(iv uint64) error
	Closed() bool
	Request([]byte) ([]byte, error)
	ReadTimeout() time.Duration
	io.ReadWriteCloser
}

type RemoteChannel struct {
	Addr            string
	Index           int
	DirectIO        bool
	WriteJoinAuth   bool
	OpenJoinAuth    bool
	HeartBeat       bool
	ReconnectPeriod int
	C               RemoteProxyChannel

	connSendedEvents uint32
	authResult       int
	iv               uint64
	wch              chan event.Event
	running          bool

	connectTime time.Time
	closeState  int
	// activeSids     map[uint32]bool
	// activeSidMutex sync.Mutex
}

// func (rc *RemoteChannel) updateActiveSid(id uint32, insertOrRemove bool) {
// 	rc.activeSidMutex.Lock()
// 	if insertOrRemove {
// 		rc.activeSids[id] = true
// 	} else {
// 		delete(rc.activeSids, id)
// 	}
// 	rc.activeSidMutex.Unlock()
// }
// func (rc *RemoteChannel) activeSidSize() int {
// 	rc.activeSidMutex.Lock()
// 	s := len(rc.activeSids)
// 	rc.activeSidMutex.Unlock()
// 	return s
// }

func (rc *RemoteChannel) authed() bool {
	return rc.authResult != 0
}
func (rc *RemoteChannel) generateIV() uint64 {
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	tmp := uint64(r.Int63())
	rc.iv = tmp
	return tmp
}

func (rc *RemoteChannel) Init() error {
	rc.running = true
	rc.authResult = 0
	//rc.activeSids = make(map[uint32]bool)

	//authSession := newRandomSession()
	if !rc.DirectIO {
		rc.wch = make(chan event.Event, 5)
		go rc.processWrite()
		go rc.processRead()
	}
	if rc.HeartBeat {
		go rc.heartbeat()
	}

	start := time.Now()
	authTimeout := rc.C.ReadTimeout()
	for rc.authResult == 0 {
		if time.Now().After(start.Add(authTimeout)) {
			rc.Stop()
			rc.authResult = -1 //timeout
			return fmt.Errorf("Server:%s auth timeout after %v", rc.Addr, time.Now().Sub(start))
		}
		time.Sleep(1 * time.Millisecond)
	}
	if rc.authResult == event.ErrAuthFailed {
		rc.Stop()
		return fmt.Errorf("Server:%s auth failed.", rc.Addr)
	} else if rc.authResult == event.SuccessAuthed {
		log.Printf("Server:%s authed success.", rc.Addr)
	} else {
		return fmt.Errorf("Server:%s auth recv unexpected code:%d.", rc.Addr, rc.authResult)
	}
	//closeProxySession(authSession.id)
	return nil
}
func (rc *RemoteChannel) Close() {
	c := rc.C
	if nil != c {
		c.Close()
	}
}
func (rc *RemoteChannel) Stop() {
	rc.running = false
	rc.Close()
}

func (rc *RemoteChannel) heartbeat() {
	ticker := time.NewTicker(5 * time.Second)
	for rc.running {
		select {
		case <-ticker.C:
			if !rc.C.Closed() && getProxySessionSize() > 0 {
				rc.Write(&event.HeartBeatEvent{})
			}
		}
	}
}

func (rc *RemoteChannel) processWrite() {
	readBufferEv := func(evs []event.Event) []event.Event {
		sev := <-rc.wch
		if nil != sev {
			evs = append(evs, sev)
		}
		return evs
	}
	var sendEvents []event.Event
	var wbuf bytes.Buffer
	for rc.running {
		conn := rc.C
		//disable write if waiting for close CK
		if rc.closeState == stateCloseWaitingACK {
			time.Sleep(1 * time.Millisecond)
			continue
		}

		if len(sendEvents) == 0 {
			if len(rc.wch) > 0 {
				for len(rc.wch) > 0 {
					sendEvents = readBufferEv(sendEvents)
				}
			} else {
				sendEvents = readBufferEv(sendEvents)
			}
		}

		if !rc.running && len(sendEvents) == 0 {
			return
		}
		if conn.Closed() {
			time.Sleep(5 * time.Millisecond)
			continue
		}
		wbuf.Reset()
		civ := rc.iv
		if rc.WriteJoinAuth || (rc.connSendedEvents == 0 && rc.OpenJoinAuth) {
			auth := NewAuthEvent()
			auth.Index = int64(rc.Index)
			auth.IV = civ
			event.EncryptEvent(&wbuf, auth, 0)
			rc.connSendedEvents++
		}

		for _, sev := range sendEvents {
			if auth, ok := sev.(*event.AuthEvent); ok {
				if auth.IV != civ {
					log.Printf("####Got %d %d", civ, auth.IV)
				}
				auth.IV = civ
				event.EncryptEvent(&wbuf, sev, 0)
			} else {
				event.EncryptEvent(&wbuf, sev, civ)
			}
		}
		if rc.closeState == stateCloseToSendReq {
			closeReq := &event.ChannelCloseReqEvent{}
			event.EncryptEvent(&wbuf, closeReq, civ)
		}
		rc.connSendedEvents += uint32(len(sendEvents))

		if wbuf.Len() > 0 {
			//start := time.Now()
			_, err := conn.Write(wbuf.Bytes())
			if nil != err {
				conn.Close()
				log.Printf("Failed to write tcp messgage:%v", err)
				//resend `sendEvents` in next process
			} else {
				//log.Printf("[%d]%s cost %v to write %d events.", rc.Index, rc.Addr, time.Now().Sub(start), len(sendEvents))
				sendEvents = nil
				//set state if write success
				if rc.closeState == stateCloseToSendReq {
					rc.closeState = stateCloseWaitingACK
				}
			}
		}
	}
}

type bufferEOFReader struct {
	r   io.Reader
	err error
}

func (r *bufferEOFReader) Read(p []byte) (int, error) {
	n, err := r.r.Read(p)
	r.err = err
	if nil != err {
		return n, err
	}
	return n, io.EOF
}

func (rc *RemoteChannel) processRead() {
	for rc.running {
		conn := rc.C
		if conn.Closed() {
			rc.closeState = 0
			if rc.authed() && getProxySessionSize() == 0 {
				time.Sleep(10 * time.Millisecond)
				continue
			}
			rc.generateIV()
			rc.connSendedEvents = 0
			err := conn.Open(rc.iv)
			if nil != err {
				log.Printf("Channel[%d] connect %s failed:%v.", rc.Index, rc.Addr, err)
				time.Sleep(1 * time.Second)
				continue
			}
			rc.connectTime = time.Now()
			log.Printf("Channel[%d] connect %s success.", rc.Index, rc.Addr)
			if rc.OpenJoinAuth {
				rc.Write(nil)
			}
		}
		//data := make([]byte, 8192)
		var buf bytes.Buffer
		reader := &bufferEOFReader{conn, nil}
		for {
			//buf.Truncate(buf.Len())
			buf.Grow(8192)
			buf.ReadFrom(reader)
			cerr := reader.err
			//n, cerr := conn.Read(data)
			//buf.Write(data[0:n])
			if rc.ReconnectPeriod > 0 && rc.closeState == 0 {
				if rc.connectTime.Add(time.Duration(rc.ReconnectPeriod) * time.Second).Before(time.Now()) {
					rc.closeState = stateCloseToSendReq
					rc.Write(nil) //trigger to write ChannelCloseReqEvent
					log.Printf("Channel[%d] prepare to close %s to reconnect.", rc.Index, rc.Addr)
				}
			}
			for buf.Len() > 0 {
				err, ev := event.DecryptEvent(&buf, rc.iv)
				if nil != err {
					if err == event.EBNR {
						err = nil
					} else {
						log.Printf("Failed to decode event for reason:%v", err)
						conn.Close()
					}
					break
				}
				switch ev.(type) {
				case *event.NotifyEvent:
					if !rc.authed() {
						auth := ev.(*event.NotifyEvent)
						rc.authResult = int(auth.Code)
						continue
					}
				case *event.ChannelCloseACKEvent:
					conn.Close()
					log.Printf("Channel[%d] close %s after recved close ACK.", rc.Index, rc.Addr)
					continue
				}
				if !rc.authed() {
					log.Printf("[ERROR]Expected auth result event for auth all connection, but got %T.", ev)
					conn.Close()
					continue
				}
				HandleEvent(ev)
			}
			if nil != cerr {
				if cerr != io.EOF && cerr != ErrChannelReadTimeout {
					log.Printf("Failed to read channel for reason:%v", cerr)
				}
				conn.Close()
				break
			}
		}
	}
}

func (rc *RemoteChannel) Request(ev event.Event) (event.Event, error) {
	var buf bytes.Buffer
	auth := NewAuthEvent()
	auth.Index = int64(rc.Index)
	auth.IV = rc.generateIV()
	event.EncryptEvent(&buf, auth, 0)
	event.EncryptEvent(&buf, ev, auth.IV)
	//event.EncodeEvent(&buf, ev)
	res, err := rc.C.Request(buf.Bytes())
	if nil != err {
		return nil, err
	}
	rbuf := bytes.NewBuffer(res)
	var rev event.Event
	err, rev = event.DecryptEvent(rbuf, auth.IV)
	if nil != err {
		return nil, err
	}
	return rev, nil
}

func (rc *RemoteChannel) Write(ev event.Event) error {
	// if nil != ev {
	// 	rc.updateActiveSid(ev.GetId(), true)
	// }
	rc.wch <- ev
	return nil
}

func (rc *RemoteChannel) WriteRaw(p []byte) (int, error) {
	return rc.C.Write(p)
}

type RemoteChannelTable struct {
	cs     []*RemoteChannel
	cursor int
	mutex  sync.Mutex
}

func (p *RemoteChannelTable) Add(c *RemoteChannel) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	p.cs = append(p.cs, c)
}

func (p *RemoteChannelTable) StopAll() {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	for _, c := range p.cs {
		c.Stop()
	}
	p.cs = make([]*RemoteChannel, 0)
}

func (p *RemoteChannelTable) Select() *RemoteChannel {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	if len(p.cs) == 0 {
		return nil
	}
	startCursor := p.cursor
	for {
		if p.cursor >= len(p.cs) {
			p.cursor = 0
		}
		c := p.cs[p.cursor]
		p.cursor++
		if nil != c {
			return c
		}
		if p.cursor == startCursor {
			break
		}
	}
	return nil
}

func NewRemoteChannelTable() *RemoteChannelTable {
	p := new(RemoteChannelTable)
	p.cs = make([]*RemoteChannel, 0)
	return p
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/urfave/cli"
)

func TestVersion(t *testing.T) {
	const testAppName = "foo"
	const testAppVersion = "0.1.0"

	app := cli.NewApp()
	ctx := cli.NewContext(app, nil, nil)
	app.Name = testAppName
	app.Version = testAppVersion

	fn, ok := versionCLICommand.Action.(func(context *cli.Context) error)
	assert.True(t, ok)

	tmpfile, err := ioutil.TempFile("", "")
	assert.NoError(t, err)
	defer os.Remove(tmpfile.Name())

	ctx.App.Writer = tmpfile

	err = fn(ctx)
	assert.NoError(t, err)

	pattern := fmt.Sprintf("%s.*version.*%s", testAppName, testAppVersion)
	err = grep(pattern, tmpfile.Name())
	assert.NoError(t, err)
}
<commit_msg>tests: Added missing file header.<commit_after>//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/urfave/cli"
)

func TestVersion(t *testing.T) {
	const testAppName = "foo"
	const testAppVersion = "0.1.0"

	app := cli.NewApp()
	ctx := cli.NewContext(app, nil, nil)
	app.Name = testAppName
	app.Version = testAppVersion

	fn, ok := versionCLICommand.Action.(func(context *cli.Context) error)
	assert.True(t, ok)

	tmpfile, err := ioutil.TempFile("", "")
	assert.NoError(t, err)
	defer os.Remove(tmpfile.Name())

	ctx.App.Writer = tmpfile

	err = fn(ctx)
	assert.NoError(t, err)

	pattern := fmt.Sprintf("%s.*version.*%s", testAppName, testAppVersion)
	err = grep(pattern, tmpfile.Name())
	assert.NoError(t, err)
}
<|endoftext|>


<commit_before>//go:build linux && cgo && !agent
// +build linux,cgo,!agent

package db

import (
	"database/sql"
	"encoding/json"
	"fmt"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/shared/api"
)

// GetNetworkACLs returns the names of existing Network ACLs.
func (c *Cluster) GetNetworkACLs(project string) ([]string, error) {
	q := `SELECT name FROM networks_acls
		WHERE project_id = (SELECT id FROM projects WHERE name = ? LIMIT 1)
		ORDER BY id
	`

	var aclNames []string

	err := c.Transaction(func(tx *ClusterTx) error {
		return tx.QueryScan(q, func(scan func(dest ...interface{}) error) error {
			var aclName string

			err := scan(&aclName)
			if err != nil {
				return err
			}

			aclNames = append(aclNames, aclName)

			return nil
		}, project)
	})
	if err != nil {
		return nil, err
	}

	return aclNames, nil
}

// GetNetworkACLIDsByNames returns a map of names to IDs of existing Network ACLs.
func (c *Cluster) GetNetworkACLIDsByNames(project string) (map[string]int64, error) {
	q := `SELECT id, name FROM networks_acls
		WHERE project_id = (SELECT id FROM projects WHERE name = ? LIMIT 1)
		ORDER BY id
	`

	acls := make(map[string]int64)

	err := c.Transaction(func(tx *ClusterTx) error {
		return tx.QueryScan(q, func(scan func(dest ...interface{}) error) error {
			var aclID int64
			var aclName string

			err := scan(&aclID, &aclName)
			if err != nil {
				return err
			}

			acls[aclName] = aclID

			return nil
		}, project)
	})
	if err != nil {
		return nil, err
	}

	return acls, nil
}

// GetNetworkACL returns the Network ACL with the given name in the given project.
func (c *Cluster) GetNetworkACL(projectName string, name string) (int64, *api.NetworkACL, error) {
	var id int64 = int64(-1)
	var ingressJSON string
	var egressJSON string

	acl := api.NetworkACL{
		NetworkACLPost: api.NetworkACLPost{
			Name: name,
		},
	}

	q := `
		SELECT id, description, ingress, egress
		FROM networks_acls
		WHERE project_id = (SELECT id FROM projects WHERE name = ? LIMIT 1) AND name=?
		LIMIT 1
	`

	err := c.Transaction(func(tx *ClusterTx) error {
		err := tx.tx.QueryRow(q, projectName, name).Scan(&id, &acl.Description, &ingressJSON, &egressJSON)
		if err != nil {
			return err
		}

		err = c.networkACLConfig(tx, id, &acl)
		if err != nil {
			return errors.Wrapf(err, "Failed loading config")
		}

		return nil
	})
	if err != nil {
		if err == sql.ErrNoRows {
			return -1, nil, ErrNoSuchObject
		}

		return -1, nil, err
	}

	acl.Ingress = []api.NetworkACLRule{}
	if ingressJSON != "" {
		err = json.Unmarshal([]byte(ingressJSON), &acl.Ingress)
		if err != nil {
			return -1, nil, errors.Wrapf(err, "Failed unmarshalling ingress rules")
		}
	}

	acl.Egress = []api.NetworkACLRule{}
	if egressJSON != "" {
		err = json.Unmarshal([]byte(egressJSON), &acl.Egress)
		if err != nil {
			return -1, nil, errors.Wrapf(err, "Failed unmarshalling egress rules")
		}
	}

	return id, &acl, nil
}

// GetNetworkACLNameAndProjectWithID returns the network ACL name and project name for the given ID.
func (c *Cluster) GetNetworkACLNameAndProjectWithID(networkACLID int) (string, string, error) {
	var networkACLName string
	var projectName string

	q := `SELECT networks_acls.name, projects.name FROM networks_acls JOIN projects ON projects.id=networks.project_id WHERE networks_acls.id=?`

	err := c.Transaction(func(tx *ClusterTx) error {
		return tx.tx.QueryRow(q, networkACLID).Scan(&networkACLName, &projectName)
	})
	if err != nil {
		if err == sql.ErrNoRows {
			return "", "", ErrNoSuchObject
		}

		return "", "", err
	}

	return networkACLName, projectName, nil
}

// networkACLConfig populates the config map of the Network ACL with the given ID.
func (c *Cluster) networkACLConfig(tx *ClusterTx, id int64, acl *api.NetworkACL) error {
	q := `
		SELECT key, value
		FROM networks_acls_config
		WHERE network_acl_id=?
	`

	acl.Config = make(map[string]string)
	return tx.QueryScan(q, func(scan func(dest ...interface{}) error) error {
		var key, value string

		err := scan(&key, &value)
		if err != nil {
			return err
		}

		_, found := acl.Config[key]
		if found {
			return fmt.Errorf("Duplicate config row found for key %q for network ACL ID %d", key, id)
		}

		acl.Config[key] = value

		return nil
	}, id)
}

// CreateNetworkACL creates a new Network ACL.
func (c *Cluster) CreateNetworkACL(projectName string, info *api.NetworkACLsPost) (int64, error) {
	var id int64
	var err error
	var ingressJSON, egressJSON []byte

	if info.Ingress != nil {
		ingressJSON, err = json.Marshal(info.Ingress)
		if err != nil {
			return -1, errors.Wrapf(err, "Failed marshalling ingress rules")
		}
	}

	if info.Egress != nil {
		egressJSON, err = json.Marshal(info.Egress)
		if err != nil {
			return -1, errors.Wrapf(err, "Failed marshalling egress rules")
		}
	}

	err = c.Transaction(func(tx *ClusterTx) error {
		// Insert a new Network ACL record.
		result, err := tx.tx.Exec(`
			INSERT INTO networks_acls (project_id, name, description, ingress, egress)
			VALUES ((SELECT id FROM projects WHERE name = ? LIMIT 1), ?, ?, ?, ?)
		`, projectName, info.Name, info.Description, string(ingressJSON), string(egressJSON))
		if err != nil {
			return err
		}

		id, err := result.LastInsertId()
		if err != nil {
			return err
		}

		err = networkACLConfigAdd(tx.tx, id, info.Config)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		id = -1
	}

	return id, err
}

// networkACLConfigAdd inserts Network ACL config keys.
func networkACLConfigAdd(tx *sql.Tx, id int64, config map[string]string) error {
	sql := "INSERT INTO networks_acls_config (network_acl_id, key, value) VALUES(?, ?, ?)"
	stmt, err := tx.Prepare(sql)
	if err != nil {
		return err
	}
	defer stmt.Close()

	for k, v := range config {
		if v == "" {
			continue
		}

		_, err = stmt.Exec(id, k, v)
		if err != nil {
			return errors.Wrapf(err, "Failed inserting config")
		}
	}

	return nil
}

// UpdateNetworkACL updates the Network ACL with the given ID.
func (c *Cluster) UpdateNetworkACL(id int64, config *api.NetworkACLPut) error {
	var err error
	var ingressJSON, egressJSON []byte

	if config.Ingress != nil {
		ingressJSON, err = json.Marshal(config.Ingress)
		if err != nil {
			return errors.Wrapf(err, "Failed marshalling ingress rules")
		}
	}

	if config.Egress != nil {
		egressJSON, err = json.Marshal(config.Egress)
		if err != nil {
			return errors.Wrapf(err, "Failed marshalling egress rules")
		}
	}

	return c.Transaction(func(tx *ClusterTx) error {
		_, err := tx.tx.Exec(`
			UPDATE networks_acls
			SET description=?, ingress = ?, egress = ?
			WHERE id=?
		`, config.Description, ingressJSON, egressJSON, id)
		if err != nil {
			return err
		}

		_, err = tx.tx.Exec("DELETE FROM networks_acls_config WHERE network_acl_id=?", id)
		if err != nil {
			return err
		}

		err = networkACLConfigAdd(tx.tx, id, config.Config)
		if err != nil {
			return err
		}

		return nil
	})
}

// RenameNetworkACL renames a Network ACL.
func (c *Cluster) RenameNetworkACL(id int64, newName string) error {
	return c.Transaction(func(tx *ClusterTx) error {
		_, err := tx.tx.Exec("UPDATE networks_acls SET name=? WHERE id=?", newName, id)
		return err
	})
}

// DeleteNetworkACL deletes the Network ACL.
func (c *Cluster) DeleteNetworkACL(id int64) error {
	return c.Transaction(func(tx *ClusterTx) error {
		_, err := tx.tx.Exec("DELETE FROM networks_acls WHERE id=?", id)
		return err
	})
}
<commit_msg>lxd/db/network/acls: Removes networkACLConfig from Cluster type for consistency with networkACLConfigAdd<commit_after>//go:build linux && cgo && !agent
// +build linux,cgo,!agent

package db

import (
	"database/sql"
	"encoding/json"
	"fmt"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/shared/api"
)

// GetNetworkACLs returns the names of existing Network ACLs.
func (c *Cluster) GetNetworkACLs(project string) ([]string, error) {
	q := `SELECT name FROM networks_acls
		WHERE project_id = (SELECT id FROM projects WHERE name = ? LIMIT 1)
		ORDER BY id
	`

	var aclNames []string

	err := c.Transaction(func(tx *ClusterTx) error {
		return tx.QueryScan(q, func(scan func(dest ...interface{}) error) error {
			var aclName string

			err := scan(&aclName)
			if err != nil {
				return err
			}

			aclNames = append(aclNames, aclName)

			return nil
		}, project)
	})
	if err != nil {
		return nil, err
	}

	return aclNames, nil
}

// GetNetworkACLIDsByNames returns a map of names to IDs of existing Network ACLs.
func (c *Cluster) GetNetworkACLIDsByNames(project string) (map[string]int64, error) {
	q := `SELECT id, name FROM networks_acls
		WHERE project_id = (SELECT id FROM projects WHERE name = ? LIMIT 1)
		ORDER BY id
	`

	acls := make(map[string]int64)

	err := c.Transaction(func(tx *ClusterTx) error {
		return tx.QueryScan(q, func(scan func(dest ...interface{}) error) error {
			var aclID int64
			var aclName string

			err := scan(&aclID, &aclName)
			if err != nil {
				return err
			}

			acls[aclName] = aclID

			return nil
		}, project)
	})
	if err != nil {
		return nil, err
	}

	return acls, nil
}

// GetNetworkACL returns the Network ACL with the given name in the given project.
func (c *Cluster) GetNetworkACL(projectName string, name string) (int64, *api.NetworkACL, error) {
	var id int64 = int64(-1)
	var ingressJSON string
	var egressJSON string

	acl := api.NetworkACL{
		NetworkACLPost: api.NetworkACLPost{
			Name: name,
		},
	}

	q := `
		SELECT id, description, ingress, egress
		FROM networks_acls
		WHERE project_id = (SELECT id FROM projects WHERE name = ? LIMIT 1) AND name=?
		LIMIT 1
	`

	err := c.Transaction(func(tx *ClusterTx) error {
		err := tx.tx.QueryRow(q, projectName, name).Scan(&id, &acl.Description, &ingressJSON, &egressJSON)
		if err != nil {
			return err
		}

		err = networkACLConfig(tx, id, &acl)
		if err != nil {
			return errors.Wrapf(err, "Failed loading config")
		}

		return nil
	})
	if err != nil {
		if err == sql.ErrNoRows {
			return -1, nil, ErrNoSuchObject
		}

		return -1, nil, err
	}

	acl.Ingress = []api.NetworkACLRule{}
	if ingressJSON != "" {
		err = json.Unmarshal([]byte(ingressJSON), &acl.Ingress)
		if err != nil {
			return -1, nil, errors.Wrapf(err, "Failed unmarshalling ingress rules")
		}
	}

	acl.Egress = []api.NetworkACLRule{}
	if egressJSON != "" {
		err = json.Unmarshal([]byte(egressJSON), &acl.Egress)
		if err != nil {
			return -1, nil, errors.Wrapf(err, "Failed unmarshalling egress rules")
		}
	}

	return id, &acl, nil
}

// GetNetworkACLNameAndProjectWithID returns the network ACL name and project name for the given ID.
func (c *Cluster) GetNetworkACLNameAndProjectWithID(networkACLID int) (string, string, error) {
	var networkACLName string
	var projectName string

	q := `SELECT networks_acls.name, projects.name FROM networks_acls JOIN projects ON projects.id=networks.project_id WHERE networks_acls.id=?`

	err := c.Transaction(func(tx *ClusterTx) error {
		return tx.tx.QueryRow(q, networkACLID).Scan(&networkACLName, &projectName)
	})
	if err != nil {
		if err == sql.ErrNoRows {
			return "", "", ErrNoSuchObject
		}

		return "", "", err
	}

	return networkACLName, projectName, nil
}

// networkACLConfig populates the config map of the Network ACL with the given ID.
func networkACLConfig(tx *ClusterTx, id int64, acl *api.NetworkACL) error {
	q := `
		SELECT key, value
		FROM networks_acls_config
		WHERE network_acl_id=?
	`

	acl.Config = make(map[string]string)
	return tx.QueryScan(q, func(scan func(dest ...interface{}) error) error {
		var key, value string

		err := scan(&key, &value)
		if err != nil {
			return err
		}

		_, found := acl.Config[key]
		if found {
			return fmt.Errorf("Duplicate config row found for key %q for network ACL ID %d", key, id)
		}

		acl.Config[key] = value

		return nil
	}, id)
}

// CreateNetworkACL creates a new Network ACL.
func (c *Cluster) CreateNetworkACL(projectName string, info *api.NetworkACLsPost) (int64, error) {
	var id int64
	var err error
	var ingressJSON, egressJSON []byte

	if info.Ingress != nil {
		ingressJSON, err = json.Marshal(info.Ingress)
		if err != nil {
			return -1, errors.Wrapf(err, "Failed marshalling ingress rules")
		}
	}

	if info.Egress != nil {
		egressJSON, err = json.Marshal(info.Egress)
		if err != nil {
			return -1, errors.Wrapf(err, "Failed marshalling egress rules")
		}
	}

	err = c.Transaction(func(tx *ClusterTx) error {
		// Insert a new Network ACL record.
		result, err := tx.tx.Exec(`
			INSERT INTO networks_acls (project_id, name, description, ingress, egress)
			VALUES ((SELECT id FROM projects WHERE name = ? LIMIT 1), ?, ?, ?, ?)
		`, projectName, info.Name, info.Description, string(ingressJSON), string(egressJSON))
		if err != nil {
			return err
		}

		id, err := result.LastInsertId()
		if err != nil {
			return err
		}

		err = networkACLConfigAdd(tx.tx, id, info.Config)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		id = -1
	}

	return id, err
}

// networkACLConfigAdd inserts Network ACL config keys.
func networkACLConfigAdd(tx *sql.Tx, id int64, config map[string]string) error {
	sql := "INSERT INTO networks_acls_config (network_acl_id, key, value) VALUES(?, ?, ?)"
	stmt, err := tx.Prepare(sql)
	if err != nil {
		return err
	}
	defer stmt.Close()

	for k, v := range config {
		if v == "" {
			continue
		}

		_, err = stmt.Exec(id, k, v)
		if err != nil {
			return errors.Wrapf(err, "Failed inserting config")
		}
	}

	return nil
}

// UpdateNetworkACL updates the Network ACL with the given ID.
func (c *Cluster) UpdateNetworkACL(id int64, config *api.NetworkACLPut) error {
	var err error
	var ingressJSON, egressJSON []byte

	if config.Ingress != nil {
		ingressJSON, err = json.Marshal(config.Ingress)
		if err != nil {
			return errors.Wrapf(err, "Failed marshalling ingress rules")
		}
	}

	if config.Egress != nil {
		egressJSON, err = json.Marshal(config.Egress)
		if err != nil {
			return errors.Wrapf(err, "Failed marshalling egress rules")
		}
	}

	return c.Transaction(func(tx *ClusterTx) error {
		_, err := tx.tx.Exec(`
			UPDATE networks_acls
			SET description=?, ingress = ?, egress = ?
			WHERE id=?
		`, config.Description, ingressJSON, egressJSON, id)
		if err != nil {
			return err
		}

		_, err = tx.tx.Exec("DELETE FROM networks_acls_config WHERE network_acl_id=?", id)
		if err != nil {
			return err
		}

		err = networkACLConfigAdd(tx.tx, id, config.Config)
		if err != nil {
			return err
		}

		return nil
	})
}

// RenameNetworkACL renames a Network ACL.
func (c *Cluster) RenameNetworkACL(id int64, newName string) error {
	return c.Transaction(func(tx *ClusterTx) error {
		_, err := tx.tx.Exec("UPDATE networks_acls SET name=? WHERE id=?", newName, id)
		return err
	})
}

// DeleteNetworkACL deletes the Network ACL.
func (c *Cluster) DeleteNetworkACL(id int64) error {
	return c.Transaction(func(tx *ClusterTx) error {
		_, err := tx.tx.Exec("DELETE FROM networks_acls WHERE id=?", id)
		return err
	})
}
<|endoftext|>


<commit_before>package handlers

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"

	"github.com/gorilla/mux"
	"github.com/play-with-docker/play-with-docker/provisioner"
	"github.com/play-with-docker/play-with-docker/pwd"
	"github.com/play-with-docker/play-with-docker/pwd/types"
)

func NewInstance(rw http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	sessionId := vars["sessionId"]

	body := types.InstanceConfig{PlaygroundFQDN: req.Host}

	json.NewDecoder(req.Body).Decode(&body)

	s, err := core.SessionGet(sessionId)
	if err != nil {
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	playground := core.PlaygroundGet(s.PlaygroundId)
	if playground == nil {
		log.Printf("Playground with id %s for session %s was not found!", s.PlaygroundId, s.Id)
		rw.WriteHeader(http.StatusBadRequest)
		return
	}

	if body.Type == "windows" && !playground.AllowWindowsInstances {
		rw.WriteHeader(http.StatusUnauthorized)
		return
	}

	instances, err := core.InstanceFindBySession(s)

	if err != nil {
		log.Println(err)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	if playground.InstancesMax > 0 && len(instances) > playground.MaxInstances {
		log.Println(err)
		rw.WriteHeader(http.StatusConflict)
		return
	}

	i, err := core.InstanceNew(s, body)
	if err != nil {
		if pwd.SessionComplete(err) {
			rw.WriteHeader(http.StatusConflict)
			return
		} else if provisioner.OutOfCapacity(err) {
			rw.WriteHeader(http.StatusServiceUnavailable)
			fmt.Fprintln(rw, `{"error": "out_of_capacity"}`)
			return
		}
		log.Println(err)
		rw.WriteHeader(http.StatusInternalServerError)
		return
		//TODO: Set a status error
	} else {
		json.NewEncoder(rw).Encode(i)
	}
}
<commit_msg>Fix InstanceMax typo<commit_after>package handlers

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"

	"github.com/gorilla/mux"
	"github.com/play-with-docker/play-with-docker/provisioner"
	"github.com/play-with-docker/play-with-docker/pwd"
	"github.com/play-with-docker/play-with-docker/pwd/types"
)

func NewInstance(rw http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	sessionId := vars["sessionId"]

	body := types.InstanceConfig{PlaygroundFQDN: req.Host}

	json.NewDecoder(req.Body).Decode(&body)

	s, err := core.SessionGet(sessionId)
	if err != nil {
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	playground := core.PlaygroundGet(s.PlaygroundId)
	if playground == nil {
		log.Printf("Playground with id %s for session %s was not found!", s.PlaygroundId, s.Id)
		rw.WriteHeader(http.StatusBadRequest)
		return
	}

	if body.Type == "windows" && !playground.AllowWindowsInstances {
		rw.WriteHeader(http.StatusUnauthorized)
		return
	}

	instances, err := core.InstanceFindBySession(s)

	if err != nil {
		log.Println(err)
		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	if playground.MaxInstances > 0 && len(instances) > playground.MaxInstances {
		log.Println(err)
		rw.WriteHeader(http.StatusConflict)
		return
	}

	i, err := core.InstanceNew(s, body)
	if err != nil {
		if pwd.SessionComplete(err) {
			rw.WriteHeader(http.StatusConflict)
			return
		} else if provisioner.OutOfCapacity(err) {
			rw.WriteHeader(http.StatusServiceUnavailable)
			fmt.Fprintln(rw, `{"error": "out_of_capacity"}`)
			return
		}
		log.Println(err)
		rw.WriteHeader(http.StatusInternalServerError)
		return
		//TODO: Set a status error
	} else {
		json.NewEncoder(rw).Encode(i)
	}
}
<|endoftext|>


<commit_before>package ecs

import (
	"testing"
)

func TestAllocatePublicIpAddress(t *testing.T) {

	client := NewClient(TestAccessKeyId, TestAccessKeySecret)
	instance, err := client.DescribeInstanceAttribute(TestInstanceId)
	if err != nil {
		t.Fatalf("Failed to describe instance %s: %v", TestInstanceId, err)
	}
	t.Logf("Instance: %++v  %v", instance, err)
	ipAddr, err := client.AllocatePublicIpAddress(TestInstanceId)
	if err != nil {
		t.Fatalf("Failed to allocate public IP address for instance %s: %v", TestInstanceId, err)
	}
	t.Logf("Public IP address of instance %s: %s", TestInstanceId, ipAddr)

}

func testEipAddress(t *testing.T, client *Client, regionId Region, instanceId string) error {

	args := AllocateEipAddressArgs{
		RegionId:           regionId,
		Bandwidth:          5,
		InternetChargeType: PayByTraffic,
		ClientToken:        client.GenerateClientToken(),
	}
	ipAddr, allocationId, err := client.AllocateEipAddress(&args)
	if err != nil {
		t.Error("Failed to allocate EIP address: %v", err)
		return err
	}
	t.Logf("EIP address: %s, AllocationId: %s", ipAddr, allocationId)

	err = client.WaitForEip(regionId, allocationId, EipStatusAvailable, 0)
	if err != nil {
		t.Errorf("Failed to wait EIP %s: %v", allocationId, err)
	}

	err = client.AssociateEipAddress(allocationId, instanceId)
	if err != nil {
		t.Errorf("Failed to associate EIP address: %v", err)
	}
	err = client.WaitForEip(regionId, allocationId, EipStatusInUse, 0)
	if err != nil {
		t.Errorf("Failed to wait EIP %s: %v", allocationId, err)
	}
	err = client.UnassociateEipAddress(allocationId, instanceId)
	if err != nil {
		t.Errorf("Failed to unassociate EIP address: %v", err)
	}
	err = client.WaitForEip(regionId, allocationId, EipStatusAvailable, 0)
	if err != nil {
		t.Errorf("Failed to wait EIP %s: %v", allocationId, err)
	}
	err = client.ReleaseEipAddress(allocationId)
	if err != nil {
		t.Errorf("Failed to release EIP address: %v", err)
	}
	return err
}
<commit_msg>Fix typo<commit_after>package ecs

import (
	"testing"
)

func TestAllocatePublicIpAddress(t *testing.T) {

	client := NewClient(TestAccessKeyId, TestAccessKeySecret)
	instance, err := client.DescribeInstanceAttribute(TestInstanceId)
	if err != nil {
		t.Fatalf("Failed to describe instance %s: %v", TestInstanceId, err)
	}
	t.Logf("Instance: %++v  %v", instance, err)
	ipAddr, err := client.AllocatePublicIpAddress(TestInstanceId)
	if err != nil {
		t.Fatalf("Failed to allocate public IP address for instance %s: %v", TestInstanceId, err)
	}
	t.Logf("Public IP address of instance %s: %s", TestInstanceId, ipAddr)

}

func testEipAddress(t *testing.T, client *Client, regionId Region, instanceId string) error {

	args := AllocateEipAddressArgs{
		RegionId:           regionId,
		Bandwidth:          5,
		InternetChargeType: PayByTraffic,
		ClientToken:        client.GenerateClientToken(),
	}
	ipAddr, allocationId, err := client.AllocateEipAddress(&args)
	if err != nil {
		t.Errorf("Failed to allocate EIP address: %v", err)
		return err
	}
	t.Logf("EIP address: %s, AllocationId: %s", ipAddr, allocationId)

	err = client.WaitForEip(regionId, allocationId, EipStatusAvailable, 0)
	if err != nil {
		t.Errorf("Failed to wait EIP %s: %v", allocationId, err)
	}

	err = client.AssociateEipAddress(allocationId, instanceId)
	if err != nil {
		t.Errorf("Failed to associate EIP address: %v", err)
	}
	err = client.WaitForEip(regionId, allocationId, EipStatusInUse, 0)
	if err != nil {
		t.Errorf("Failed to wait EIP %s: %v", allocationId, err)
	}
	err = client.UnassociateEipAddress(allocationId, instanceId)
	if err != nil {
		t.Errorf("Failed to unassociate EIP address: %v", err)
	}
	err = client.WaitForEip(regionId, allocationId, EipStatusAvailable, 0)
	if err != nil {
		t.Errorf("Failed to wait EIP %s: %v", allocationId, err)
	}
	err = client.ReleaseEipAddress(allocationId)
	if err != nil {
		t.Errorf("Failed to release EIP address: %v", err)
	}
	return err
}
<|endoftext|>


<commit_before>package master

import (
	"github.com/KIT-MAMID/mamid/model"
	"github.com/KIT-MAMID/mamid/msp"
	"github.com/jinzhu/gorm"
	"github.com/stretchr/testify/assert"
	"sync"
	"testing"
)

func createDB(t *testing.T) (db *gorm.DB, err error) {
	// Setup database
	db, err = model.InitializeTestDB()

	dbSlave := model.Slave{
		ID:                   1,
		Hostname:             "host1",
		Port:                 1,
		MongodPortRangeBegin: 2,
		MongodPortRangeEnd:   3,
		PersistentStorage:    true,
		Mongods:              []*model.Mongod{},
		ConfiguredState:      model.SlaveStateActive,
	}
	assert.NoError(t, db.Create(&dbSlave).Error)
	m1 := model.Mongod{
		Port:           2000,
		ReplSetName:    "repl1",
		ParentSlaveID:  1,
		DesiredStateID: 1,
	}
	des1 := model.MongodState{
		ID: 1,
		IsShardingConfigServer: false,
		ExecutionState:         model.MongodExecutionStateRunning,
	}
	assert.NoError(t, db.Create(&des1).Error)
	assert.NoError(t, db.Create(&m1).Error)

	return
}

type FakeMSPClient struct {
	msp.MSPClient
	Status []msp.Mongod
	Error  *msp.Error
}

func (m FakeMSPClient) RequestStatus(Target msp.HostPort) ([]msp.Mongod, *msp.Error) {
	return m.Status, m.Error
}

func TestMonitor_observeSlave(t *testing.T) {
	db, err := createDB(t)
	assert.NoError(t, err)

	mspClient := FakeMSPClient{
		Status: []msp.Mongod{
			msp.Mongod{
				Port:                    2000,
				ReplicaSetName:          "repl1",
				ReplicaSetMembers:       []msp.HostPort{},
				ShardingConfigServer:    false,
				StatusError:             nil,
				LastEstablishStateError: nil,
				State: msp.MongodStateRunning,
			},
		},
		Error: nil,
	}

	wg := new(sync.WaitGroup)
	bus := NewBus()
	readChannel := bus.GetNewReadChannel()
	monitor := Monitor{
		DB:              db,
		BusWriteChannel: bus.GetNewWriteChannel(),
		MSPClient:       mspClient,
	}

	wg.Add(1)
	go func() {
		bus.Run()
		wg.Done()
	}()

	//Observe Slave
	var slave model.Slave
	db.First(&slave, 1)

	monitor.observeSlave(slave)

	var mongod model.Mongod
	db.First(&mongod, 1)
	assert.Nil(t, db.Model(&mongod).Related(&mongod.ObservedState, "ObservedState").Error, "after observation, the observed state should be != nil")
	assert.Equal(t, model.MongodExecutionStateRunning, mongod.ObservedState.ExecutionState)

	connStatusX := <-readChannel
	connStatus, ok := connStatusX.(model.ConnectionStatus)
	assert.True(t, ok)
	assert.False(t, connStatus.Unreachable)

	mismatchX := <-readChannel
	mismatch, ok := mismatchX.(model.MongodMatchStatus)
	assert.False(t, mismatch.Mismatch)

	//-----------------
	//Slave cannot observe mongod
	//-----------------
	monitor.MSPClient = FakeMSPClient{
		Status: []msp.Mongod{
			msp.Mongod{
				Port:           2000,
				ReplicaSetName: "repl1",
				StatusError: &msp.Error{
					Identifier:  "foo",
					Description: "bar",
				},
			},
		},
		Error: nil,
	}

	db.First(&slave, 1)

	monitor.observeSlave(slave)

	db.First(&mongod, 1)

	//Mongod should have an observation error
	db.Model(&mongod).Related(&mongod.ObservationError, "ObservationError")
	assert.NotZero(t, mongod.ObservationErrorID)

	connStatusX = <-readChannel
	connStatus, ok = connStatusX.(model.ConnectionStatus)
	assert.True(t, ok)
	assert.False(t, connStatus.Unreachable)

	<-readChannel //mismatch

	//-----------------
	//Mongod gone
	//-----------------
	monitor.MSPClient = FakeMSPClient{
		Status: []msp.Mongod{},
		Error:  nil,
	}

	db.First(&slave, 1)

	monitor.observeSlave(slave)

	db.First(&mongod, 1)

	//Mongod should not have observed state anymore
	assert.True(t, db.Model(&mongod).Related(&mongod.ObservedState, "ObservedState").RecordNotFound())

	connStatusX = <-readChannel
	connStatus, ok = connStatusX.(model.ConnectionStatus)
	assert.True(t, ok)
	assert.False(t, connStatus.Unreachable)

	<-readChannel //mismatch

	//-----------------
	//Slave becomes unreachable
	//-----------------
	monitor.MSPClient = FakeMSPClient{
		Status: []msp.Mongod{},
		Error:  &msp.Error{Identifier: msp.CommunicationError},
	}

	db.First(&slave, 1)

	monitor.observeSlave(slave)

	connStatusX = <-readChannel
	connStatus, ok = connStatusX.(model.ConnectionStatus)
	assert.True(t, ok)
	assert.True(t, connStatus.Unreachable)

	bus.Kill()
	wg.Wait()
}
<commit_msg>Add additional assertion on observation error.<commit_after>package master

import (
	"github.com/KIT-MAMID/mamid/model"
	"github.com/KIT-MAMID/mamid/msp"
	"github.com/jinzhu/gorm"
	"github.com/stretchr/testify/assert"
	"sync"
	"testing"
)

func createDB(t *testing.T) (db *gorm.DB, err error) {
	// Setup database
	db, err = model.InitializeTestDB()

	dbSlave := model.Slave{
		ID:                   1,
		Hostname:             "host1",
		Port:                 1,
		MongodPortRangeBegin: 2,
		MongodPortRangeEnd:   3,
		PersistentStorage:    true,
		Mongods:              []*model.Mongod{},
		ConfiguredState:      model.SlaveStateActive,
	}
	assert.NoError(t, db.Create(&dbSlave).Error)
	m1 := model.Mongod{
		Port:           2000,
		ReplSetName:    "repl1",
		ParentSlaveID:  1,
		DesiredStateID: 1,
	}
	des1 := model.MongodState{
		ID: 1,
		IsShardingConfigServer: false,
		ExecutionState:         model.MongodExecutionStateRunning,
	}
	assert.NoError(t, db.Create(&des1).Error)
	assert.NoError(t, db.Create(&m1).Error)

	return
}

type FakeMSPClient struct {
	msp.MSPClient
	Status []msp.Mongod
	Error  *msp.Error
}

func (m FakeMSPClient) RequestStatus(Target msp.HostPort) ([]msp.Mongod, *msp.Error) {
	return m.Status, m.Error
}

func TestMonitor_observeSlave(t *testing.T) {
	db, err := createDB(t)
	assert.NoError(t, err)

	mspClient := FakeMSPClient{
		Status: []msp.Mongod{
			msp.Mongod{
				Port:                    2000,
				ReplicaSetName:          "repl1",
				ReplicaSetMembers:       []msp.HostPort{},
				ShardingConfigServer:    false,
				StatusError:             nil,
				LastEstablishStateError: nil,
				State: msp.MongodStateRunning,
			},
		},
		Error: nil,
	}

	wg := new(sync.WaitGroup)
	bus := NewBus()
	readChannel := bus.GetNewReadChannel()
	monitor := Monitor{
		DB:              db,
		BusWriteChannel: bus.GetNewWriteChannel(),
		MSPClient:       mspClient,
	}

	wg.Add(1)
	go func() {
		bus.Run()
		wg.Done()
	}()

	//Observe Slave
	var slave model.Slave
	db.First(&slave, 1)

	monitor.observeSlave(slave)

	var mongod model.Mongod
	db.First(&mongod, 1)
	assert.Nil(t, db.Model(&mongod).Related(&mongod.ObservedState, "ObservedState").Error, "after observation, the observed state should be != nil")
	assert.Equal(t, model.MongodExecutionStateRunning, mongod.ObservedState.ExecutionState)

	connStatusX := <-readChannel
	connStatus, ok := connStatusX.(model.ConnectionStatus)
	assert.True(t, ok)
	assert.False(t, connStatus.Unreachable)

	mismatchX := <-readChannel
	mismatch, ok := mismatchX.(model.MongodMatchStatus)
	assert.False(t, mismatch.Mismatch)

	//-----------------
	//Slave cannot observe mongod
	//-----------------
	monitor.MSPClient = FakeMSPClient{
		Status: []msp.Mongod{
			msp.Mongod{
				Port:           2000,
				ReplicaSetName: "repl1",
				StatusError: &msp.Error{
					Identifier:  "foo",
					Description: "cannot observe mongod",
				},
			},
		},
		Error: nil,
	}

	db.First(&slave, 1)

	monitor.observeSlave(slave)

	db.First(&mongod, 1)

	//Mongod should have an observation error
	db.Model(&mongod).Related(&mongod.ObservationError, "ObservationError")
	assert.EqualValues(t, "cannot observe mongod", mongod.ObservationError.Description)
	assert.NotZero(t, mongod.ObservationErrorID)

	connStatusX = <-readChannel
	connStatus, ok = connStatusX.(model.ConnectionStatus)
	assert.True(t, ok)
	assert.False(t, connStatus.Unreachable)

	<-readChannel //mismatch

	//-----------------
	//Mongod gone
	//-----------------
	monitor.MSPClient = FakeMSPClient{
		Status: []msp.Mongod{},
		Error:  nil,
	}

	db.First(&slave, 1)

	monitor.observeSlave(slave)

	db.First(&mongod, 1)

	//Mongod should not have observed state anymore
	assert.True(t, db.Model(&mongod).Related(&mongod.ObservedState, "ObservedState").RecordNotFound())

	connStatusX = <-readChannel
	connStatus, ok = connStatusX.(model.ConnectionStatus)
	assert.True(t, ok)
	assert.False(t, connStatus.Unreachable)

	<-readChannel //mismatch

	//-----------------
	//Slave becomes unreachable
	//-----------------
	monitor.MSPClient = FakeMSPClient{
		Status: []msp.Mongod{},
		Error:  &msp.Error{Identifier: msp.CommunicationError},
	}

	db.First(&slave, 1)

	monitor.observeSlave(slave)

	connStatusX = <-readChannel
	connStatus, ok = connStatusX.(model.ConnectionStatus)
	assert.True(t, ok)
	assert.True(t, connStatus.Unreachable)

	bus.Kill()
	wg.Wait()
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2014 Conformal Systems LLC <info@conformal.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

/*
Package waddrmgr provides a secure hierarchical deterministic wallet address
manager.

Overview

One of the fundamental jobs of a wallet is to manage addresses, private keys,
and script data associated with them.  At a high level, this package provides
the facilities to perform this task with a focus on security and also allows
recovery through the use of hierarchical deterministic keys (BIP0032) generated
from a caller provided seed.  The specific structure used is as described in
BIP0044.  This setup means as long as the user writes the seed down (even better
is to use a mnemonic for the seed), all their addresses and private keys can be
regenerated from the seed.

There are two master keys which are protected by two independent passphrases.
One is intended for public facing data, while the other is intended for private
data.  The public password can be hardcoded for callers who don't want the
additional public data protection or the same password can be used if a single
password is desired.  These choices provide a usability versus security
tradeoff.  However, keep in mind that extended hd keys, as called out in BIP0032
need to be handled more carefully than normal EC public keys because they can be
used to generate all future addresses.  While this is part of what makes them
attractive, it also means an attacker getting access to your extended public key
for an account will allow them to know all addresses you will use and hence
reduces privacy.  For this reason, it is highly recommended that you do not hard
code a password which allows any attacker who gets a copy of your address
manager database to access your effectively plain text extended public keys.

Each master key in turn protects the three real encryption keys (called crypto
keys) for public, private, and script data.  Some examples include payment
addresses, extended hd keys, and scripts associated with pay-to-script-hash
addresses.  This scheme makes changing passphrases more efficient since only the
crypto keys need to be re-encrypted versus every single piece of information
(which is what is needed for *rekeying*).  This results in a fully encrypted
database where access to it does not compromise address, key, or script privacy.
This differs from the handling by other wallets at the time of this writing in
that they divulge your addresses, and worse, some even expose the chain code
which can be used by the attacker to know all future addresses that will be
used.

The address manager is also hardened against memory scrapers.  This is
accomplished by typically having the address manager locked meaning no private
keys or scripts are in memory.  Unlocking the address manager causes the crypto
private and script keys to be decrypted and loaded in memory which in turn are
used to decrypt private keys and scripts on demand.  Relocking the address
manager actively zeros all private material from memory.  In addition, temp
private key material used internally is zeroed as soon as it's used.

Locking and Unlocking

As previously mentioned, this package provide facilities for locking and
unlocking the address manager to protect access to private material and remove
it from memory when locked.  The Lock, Unlock, and IsLocked functions are used
for this purpose.

Creating a New Address Manager

A new address manager is created via the Create function.  This function accepts
the path to a database file to create, passphrases, network, and perhaps most
importantly, a cryptographically random seed which is used to generate the
master node of the hierarchical deterministic keychain which allows all
addresses and private keys to be recovered with only the seed.  The GenerateSeed
function in the hdkeychain package can be used as a convenient way to create a
random seed for use with this function.  The address manager is locked
immediately upon being created.

Opening an Existing Address Manager

An existing address manager is opened via the Open function.  This function
accepts the path to the existing database file, the public passphrase, and
network.  The address manager is opened locked as expected since the open
function does not take the private passphrase to unlock it.

Closing the Address Manager

The Close method should be called on the address manager when the caller is done
with it.  While it is not required, it is recommended because it sanely shuts
down the database and ensures all private and public key material is purged from
memory.

Managed Addresses

Each address returned by the address manager satisifies the ManagedAddress
interface as well as either the ManagedPubKeyAddress or ManagedScriptAddress
interfaces.  These interfaces provide the means to obtain relevant information
about the addresses such as their private keys and scripts.

Chained Addresses

Most callers will make use of the chained addresses for normal operations.
Internal addresses are intended for internal wallet uses such as change outputs,
while external addresses are intended for uses such payment addresses that are
shared.  The NextInternalAddresses and NextExternalAddresses functions provide
the means to acquire one or more of the next addresses that have not already
been provided.  In addition, the LastInternalAddress and LastExternalAddress
functions can be used to get the most recently provided internal and external
address, respectively.

Requesting Existing Addresses

In addition to generating new addresses, access to old addresses is often
required.  Most notably, to sign transactions in order to redeem them.  The
Address function provides this capability and returns a ManagedAddress.

Importing Addresses

While the recommended approach is to use the chained addresses discussed above
because they can be deterministically regenerated to avoid losing funds as long
as the user has the master seed, there are many addresses that already exist,
and as a result, this package provides the ability to import existing private
keys in Wallet Import Format (WIF) and hence the associated public key and
address.

Importing Scripts

In order to support pay-to-script-hash transactions, the script must be securely
stored as it is needed to redeem the transaction.  This can be useful for a
variety of scenarios, however the most common use is currently multi-signature
transactions.

Syncing

The address manager also supports storing and retrieving a block hash and height
which the manager is known to have all addresses synced through.  The manager
itself does not have any notion of which addresses are synced or not.  It only
provides the storage as a convenience for the caller.

Network

The address manager must be associated with a given network in order to provide
appropriate addresses and reject imported addresses and scripts which don't
apply to the associated network.

Errors

All errors returned from this package are of type waddrmgr.ManagerError.  This
allows the caller to programmatically ascertain the specific reasons for failure
by examining the ErrorCode field of the type asserted ManagerError.  For certain
error codes, as documented the specific error codes, the underlying error will
be contained in the Err field.

Bitcoin Improvement Proposals

This package includes concepts outlined by the following BIPs:

		BIP0032 (https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki)
		BIP0043 (https://github.com/bitcoin/bips/blob/master/bip-0043.mediawiki)
		BIP0044 (https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki)
*/
package waddrmgr
<commit_msg>waddrmgr: Update documentation for walletdb.<commit_after>/*
 * Copyright (c) 2014 Conformal Systems LLC <info@conformal.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

/*
Package waddrmgr provides a secure hierarchical deterministic wallet address
manager.

Overview

One of the fundamental jobs of a wallet is to manage addresses, private keys,
and script data associated with them.  At a high level, this package provides
the facilities to perform this task with a focus on security and also allows
recovery through the use of hierarchical deterministic keys (BIP0032) generated
from a caller provided seed.  The specific structure used is as described in
BIP0044.  This setup means as long as the user writes the seed down (even better
is to use a mnemonic for the seed), all their addresses and private keys can be
regenerated from the seed.

There are two master keys which are protected by two independent passphrases.
One is intended for public facing data, while the other is intended for private
data.  The public password can be hardcoded for callers who don't want the
additional public data protection or the same password can be used if a single
password is desired.  These choices provide a usability versus security
tradeoff.  However, keep in mind that extended hd keys, as called out in BIP0032
need to be handled more carefully than normal EC public keys because they can be
used to generate all future addresses.  While this is part of what makes them
attractive, it also means an attacker getting access to your extended public key
for an account will allow them to know all addresses you will use and hence
reduces privacy.  For this reason, it is highly recommended that you do not hard
code a password which allows any attacker who gets a copy of your address
manager database to access your effectively plain text extended public keys.

Each master key in turn protects the three real encryption keys (called crypto
keys) for public, private, and script data.  Some examples include payment
addresses, extended hd keys, and scripts associated with pay-to-script-hash
addresses.  This scheme makes changing passphrases more efficient since only the
crypto keys need to be re-encrypted versus every single piece of information
(which is what is needed for *rekeying*).  This results in a fully encrypted
database where access to it does not compromise address, key, or script privacy.
This differs from the handling by other wallets at the time of this writing in
that they divulge your addresses, and worse, some even expose the chain code
which can be used by the attacker to know all future addresses that will be
used.

The address manager is also hardened against memory scrapers.  This is
accomplished by typically having the address manager locked meaning no private
keys or scripts are in memory.  Unlocking the address manager causes the crypto
private and script keys to be decrypted and loaded in memory which in turn are
used to decrypt private keys and scripts on demand.  Relocking the address
manager actively zeros all private material from memory.  In addition, temp
private key material used internally is zeroed as soon as it's used.

Locking and Unlocking

As previously mentioned, this package provide facilities for locking and
unlocking the address manager to protect access to private material and remove
it from memory when locked.  The Lock, Unlock, and IsLocked functions are used
for this purpose.

Creating a New Address Manager

A new address manager is created via the Create function.  This function accepts
a wallet database namespace, passphrases, network, and perhaps most importantly,
a cryptographically random seed which is used to generate the master node of the
hierarchical deterministic keychain which allows all addresses and private keys
to be recovered with only the seed.  The GenerateSeed function in the hdkeychain
package can be used as a convenient way to create a random seed for use with
this function.  The address manager is locked immediately upon being created.

Opening an Existing Address Manager

An existing address manager is opened via the Open function.  This function
accepts an existing wallet database namespace, the public passphrase, and
network.  The address manager is opened locked as expected since the open
function does not take the private passphrase to unlock it.

Closing the Address Manager

The Close method should be called on the address manager when the caller is done
with it.  While it is not required, it is recommended because it sanely shuts
down the database and ensures all private and public key material is purged from
memory.

Managed Addresses

Each address returned by the address manager satisifies the ManagedAddress
interface as well as either the ManagedPubKeyAddress or ManagedScriptAddress
interfaces.  These interfaces provide the means to obtain relevant information
about the addresses such as their private keys and scripts.

Chained Addresses

Most callers will make use of the chained addresses for normal operations.
Internal addresses are intended for internal wallet uses such as change outputs,
while external addresses are intended for uses such payment addresses that are
shared.  The NextInternalAddresses and NextExternalAddresses functions provide
the means to acquire one or more of the next addresses that have not already
been provided.  In addition, the LastInternalAddress and LastExternalAddress
functions can be used to get the most recently provided internal and external
address, respectively.

Requesting Existing Addresses

In addition to generating new addresses, access to old addresses is often
required.  Most notably, to sign transactions in order to redeem them.  The
Address function provides this capability and returns a ManagedAddress.

Importing Addresses

While the recommended approach is to use the chained addresses discussed above
because they can be deterministically regenerated to avoid losing funds as long
as the user has the master seed, there are many addresses that already exist,
and as a result, this package provides the ability to import existing private
keys in Wallet Import Format (WIF) and hence the associated public key and
address.

Importing Scripts

In order to support pay-to-script-hash transactions, the script must be securely
stored as it is needed to redeem the transaction.  This can be useful for a
variety of scenarios, however the most common use is currently multi-signature
transactions.

Syncing

The address manager also supports storing and retrieving a block hash and height
which the manager is known to have all addresses synced through.  The manager
itself does not have any notion of which addresses are synced or not.  It only
provides the storage as a convenience for the caller.

Network

The address manager must be associated with a given network in order to provide
appropriate addresses and reject imported addresses and scripts which don't
apply to the associated network.

Errors

All errors returned from this package are of type ManagerError.  This allows the
caller to programmatically ascertain the specific reasons for failure by
examining the ErrorCode field of the type asserted ManagerError.  For certain
error codes, as documented by the specific error codes, the underlying error
will be contained in the Err field.

Bitcoin Improvement Proposals

This package includes concepts outlined by the following BIPs:

		BIP0032 (https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki)
		BIP0043 (https://github.com/bitcoin/bips/blob/master/bip-0043.mediawiki)
		BIP0044 (https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki)
*/
package waddrmgr
<|endoftext|>