<commit_before>package util

import (
	"errors"
	"fmt"
	"github.com/davyxu/cellnet"
	"net"
	"strconv"
	"strings"
)

// 将地址拆分为ip和端口
func SpliteAddress(addr string) (host string, port int, err error) {

	var portStr string

	host, portStr, err = net.SplitHostPort(addr)

	if err != nil {
		return "", 0, err
	}

	port, err = strconv.Atoi(portStr)

	if err != nil {
		return "", 0, err
	}

	return
}

// 将ip和端口合并为地址
func JoinAddress(host string, port int) string {
	return fmt.Sprintf("%s:%d", host, port)
}

// 获取session远程的地址
func GetRemoteAddrss(ses cellnet.Session) (string, bool) {
	if c, ok := ses.Raw().(net.Conn); ok {
		return c.RemoteAddr().String(), true
	}

	return "", false
}

var (
	ErrInvalidPortRange = errors.New("invalid port range")
)

// 在给定的端口范围内找到一个能用的端口 格式: localhost:5000~6000
func DetectPort(addr string, fn func(string) (net.Listener, error)) (net.Listener, error) {
	// host:port 或 host:min~max
	parts := strings.Split(addr, ":")

	// host:port格式
	if len(parts) < 2 {
		return fn(addr)
	}

	// 间隔分割
	ports := strings.Split(parts[len(parts)-1], "~")

	// 单独的端口
	if len(ports) < 2 {
		return fn(addr)
	}

	// extract min port
	min, err := strconv.Atoi(ports[0])
	if err != nil {
		return nil, ErrInvalidPortRange
	}

	// extract max port
	max, err := strconv.Atoi(ports[1])
	if err != nil {
		return nil, ErrInvalidPortRange
	}

	host := parts[0]

	for port := min; port <= max; port++ {

		// 使用回调侦听
		ln, err := fn(fmt.Sprintf("%s:%d", host, port))
		if err == nil {
			return ln, nil
		}

		// hit max port
		if port == max {
			return nil, err
		}
	}

	return nil, fmt.Errorf("unable to bind to %s", addr)
}
<commit_msg>add 整合本地ip接口<commit_after>package util

import (
	"errors"
	"fmt"
	"github.com/davyxu/cellnet"
	"net"
	"strconv"
	"strings"
)

// 将地址拆分为ip和端口
func SpliteAddress(addr string) (host string, port int, err error) {

	var portStr string

	host, portStr, err = net.SplitHostPort(addr)

	if err != nil {
		return "", 0, err
	}

	port, err = strconv.Atoi(portStr)

	if err != nil {
		return "", 0, err
	}

	return
}

// 将ip和端口合并为地址
func JoinAddress(host string, port int) string {
	return fmt.Sprintf("%s:%d", host, port)
}

// 获取session远程的地址
func GetRemoteAddrss(ses cellnet.Session) (string, bool) {
	if c, ok := ses.Raw().(net.Conn); ok {
		return c.RemoteAddr().String(), true
	}

	return "", false
}

var (
	ErrInvalidPortRange = errors.New("invalid port range")
)

// 在给定的端口范围内找到一个能用的端口 格式: localhost:5000~6000
func DetectPort(addr string, fn func(string) (net.Listener, error)) (net.Listener, error) {
	// host:port 或 host:min~max
	parts := strings.Split(addr, ":")

	// host:port格式
	if len(parts) < 2 {
		return fn(addr)
	}

	// 间隔分割
	ports := strings.Split(parts[len(parts)-1], "~")

	// 单独的端口
	if len(ports) < 2 {
		return fn(addr)
	}

	// extract min port
	min, err := strconv.Atoi(ports[0])
	if err != nil {
		return nil, ErrInvalidPortRange
	}

	// extract max port
	max, err := strconv.Atoi(ports[1])
	if err != nil {
		return nil, ErrInvalidPortRange
	}

	host := parts[0]

	for port := min; port <= max; port++ {

		// 使用回调侦听
		ln, err := fn(fmt.Sprintf("%s:%d", host, port))
		if err == nil {
			return ln, nil
		}

		// hit max port
		if port == max {
			return nil, err
		}
	}

	return nil, fmt.Errorf("unable to bind to %s", addr)
}

// 获取本地IP地址，有多重IP时，默认取第一个
func GetLocalIP() string {

	list, err := GetPrivateIPv4()
	if err != nil {
		return ""
	}

	if len(list) == 0 {
		return ""
	}

	return list[0].String()
}

// from consul

// GetPrivateIPv4 returns the list of private network IPv4 addresses on
// all active interfaces.
func GetPrivateIPv4() ([]*net.IPAddr, error) {
	addresses, err := activeInterfaceAddresses()
	if err != nil {
		return nil, fmt.Errorf("Failed to get interface addresses: %v", err)
	}

	var addrs []*net.IPAddr
	for _, rawAddr := range addresses {
		var ip net.IP
		switch addr := rawAddr.(type) {
		case *net.IPAddr:
			ip = addr.IP
		case *net.IPNet:
			ip = addr.IP
		default:
			continue
		}
		if ip.To4() == nil {
			continue
		}
		if !isPrivate(ip) {
			continue
		}
		addrs = append(addrs, &net.IPAddr{IP: ip})
	}
	return addrs, nil
}

// GetPublicIPv6 returns the list of all public IPv6 addresses
// on all active interfaces.
func GetPublicIPv6() ([]*net.IPAddr, error) {
	addresses, err := net.InterfaceAddrs()
	if err != nil {
		return nil, fmt.Errorf("Failed to get interface addresses: %v", err)
	}

	var addrs []*net.IPAddr
	for _, rawAddr := range addresses {
		var ip net.IP
		switch addr := rawAddr.(type) {
		case *net.IPAddr:
			ip = addr.IP
		case *net.IPNet:
			ip = addr.IP
		default:
			continue
		}
		if ip.To4() != nil {
			continue
		}
		if isPrivate(ip) {
			continue
		}
		addrs = append(addrs, &net.IPAddr{IP: ip})
	}
	return addrs, nil
}

// privateBlocks contains non-forwardable address blocks which are used
// for private networks. RFC 6890 provides an overview of special
// address blocks.
var privateBlocks = []*net.IPNet{
	parseCIDR("10.0.0.0/8"),     // RFC 1918 IPv4 private network address
	parseCIDR("100.64.0.0/10"),  // RFC 6598 IPv4 shared address space
	parseCIDR("127.0.0.0/8"),    // RFC 1122 IPv4 loopback address
	parseCIDR("169.254.0.0/16"), // RFC 3927 IPv4 link local address
	parseCIDR("172.16.0.0/12"),  // RFC 1918 IPv4 private network address
	parseCIDR("192.0.0.0/24"),   // RFC 6890 IPv4 IANA address
	parseCIDR("192.0.2.0/24"),   // RFC 5737 IPv4 documentation address
	parseCIDR("192.168.0.0/16"), // RFC 1918 IPv4 private network address
	parseCIDR("::1/128"),        // RFC 1884 IPv6 loopback address
	parseCIDR("fe80::/10"),      // RFC 4291 IPv6 link local addresses
	parseCIDR("fc00::/7"),       // RFC 4193 IPv6 unique local addresses
	parseCIDR("fec0::/10"),      // RFC 1884 IPv6 site-local addresses
	parseCIDR("2001:db8::/32"),  // RFC 3849 IPv6 documentation address
}

func parseCIDR(s string) *net.IPNet {
	_, block, err := net.ParseCIDR(s)
	if err != nil {
		panic(fmt.Sprintf("Bad CIDR %s: %s", s, err))
	}
	return block
}

func isPrivate(ip net.IP) bool {
	for _, priv := range privateBlocks {
		if priv.Contains(ip) {
			return true
		}
	}
	return false
}

// Returns addresses from interfaces that is up
func activeInterfaceAddresses() ([]net.Addr, error) {
	var upAddrs []net.Addr
	var loAddrs []net.Addr

	interfaces, err := net.Interfaces()
	if err != nil {
		return nil, fmt.Errorf("Failed to get interfaces: %v", err)
	}

	for _, iface := range interfaces {
		// Require interface to be up
		if iface.Flags&net.FlagUp == 0 {
			continue
		}

		addresses, err := iface.Addrs()
		if err != nil {
			return nil, fmt.Errorf("Failed to get interface addresses: %v", err)
		}

		if iface.Flags&net.FlagLoopback != 0 {
			loAddrs = append(loAddrs, addresses...)
			continue
		}

		upAddrs = append(upAddrs, addresses...)
	}

	if len(upAddrs) == 0 {
		return loAddrs, nil
	}

	return upAddrs, nil
}
<|endoftext|>


<commit_before>package util

import (
	"fmt"
	"github.com/go-martini/martini"
	"github.com/leptonyu/goeast/db"
	"github.com/leptonyu/goeast/handler"
	"github.com/leptonyu/goeast/wechat"
	"html/template"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
)

func Template(key string, m interface{}) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		t := template.New(key)
		t, _ = t.ParseFiles("templates/" + key)
		t.Execute(w, m)
	}
}

func Interval(wait time.Duration, keys ...string) {
	if len(keys) > 0 {
		go func() {
			for !c.close {
				time.Sleep(wait)
				for _, v := range keys {
					c.update(v)
				}
			}
		}()
	}
}

func StartWeb(port int, dbname string, api string) {
	m := martini.Classic()
	m.NotFound(Template("404.tpl", m))
	m.Use(martini.Static("static", martini.StaticOptions{Prefix: "static"}))
	m.Get("/", Template("index.tpl", m))
	config := db.NewDBConfig(api)
	wc, err := config.CreateWeChat(dbname, api)
	if err != nil {
		panic(err)
	}
	if port == 8080 {
		go func() {
			for {
				a, err := wc.UpdateAccessToken()
				time.Sleep((-1 * time.Since(a.ExpireTime).Seconds()) + time.Second)
			}
		}()
		f := func(wait time.Duration, keys ...string) {
			if len(keys) > 0 {
				for !c.close {
					time.Sleep(wait)
					for _, v := range keys {
						config.Update(v)
					}
				}
			}
		}
		go f(30*time.Minute, data.Blog, data.Events)
		go f(24*time.Hour,
			data.Home,
			data.Campus,
			data.Contact,
			data.Galleries,
			data.One2one,
			data.Online,
			data.Onsite,
			data.Teachers,
			data.Testimonials)
	}
	wc.HandleFunc(wechat.MsgTypeText, func(w wechat.ResponseWriter, r *wechat.Request) {
		txt := r.Content
		sig := strings.ToLower(txt)
		v, ok := mr[sig]
		if ok {
			v(w, c)
		} else {
			w.ReplyText(txt)
		}
	})
	//Create api route
	ff := wc.CreateHandlerFunc()
	m.Get("/"+c.Api, ff)
	m.Post("/"+c.Api, ff)
	err = http.ListenAndServe(":"+strconv.Itoa(port), m)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	fmt.Println("stop!")
	c.Session.Close()
	os.Exit(0)
}
<commit_msg>wechat<commit_after>package util

import (
	"fmt"
	"github.com/go-martini/martini"
	"github.com/leptonyu/goeast/db"
	"github.com/leptonyu/goeast/wechat"
	"html/template"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
)

func Template(key string, m interface{}) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		t := template.New(key)
		t, _ = t.ParseFiles("templates/" + key)
		t.Execute(w, m)
	}
}

func Interval(wait time.Duration, keys ...string) {
	if len(keys) > 0 {
		go func() {
			for !c.close {
				time.Sleep(wait)
				for _, v := range keys {
					c.update(v)
				}
			}
		}()
	}
}

func StartWeb(port int, dbname string, api string) {
	m := martini.Classic()
	m.NotFound(Template("404.tpl", m))
	m.Use(martini.Static("static", martini.StaticOptions{Prefix: "static"}))
	m.Get("/", Template("index.tpl", m))
	config := db.NewDBConfig(api)
	wc, err := config.CreateWeChat(dbname, api)
	if err != nil {
		panic(err)
	}
	if port == 8080 {
		go func() {
			for {
				a, err := wc.UpdateAccessToken()
				time.Sleep((-1 * time.Since(a.ExpireTime).Seconds()) + time.Second)
			}
		}()
		f := func(wait time.Duration, keys ...string) {
			if len(keys) > 0 {
				for {
					time.Sleep(wait)
					for _, v := range keys {
						config.Update(v)
					}
				}
			}
		}
		go f(30*time.Minute, db.Blog, db.Events)
		go f(24*time.Hour,
			db.Home,
			db.Campus,
			db.Contact,
			db.Galleries,
			db.One2one,
			db.Online,
			db.Onsite,
			db.Teachers,
			db.Testimonials)
	}
	wc.HandleFunc(wechat.MsgTypeText, func(w wechat.ResponseWriter, r *wechat.Request) {
		txt := r.Content
		sig := strings.ToLower(txt)
		v, ok := mr[sig]
		if ok {
			v(w, c)
		} else {
			w.ReplyText(txt)
		}
	})
	//Create api route
	ff := wc.CreateHandlerFunc()
	m.Get("/"+c.Api, ff)
	m.Post("/"+c.Api, ff)
	err = http.ListenAndServe(":"+strconv.Itoa(port), m)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	fmt.Println("stop!")
	c.Session.Close()
	os.Exit(0)
}
<|endoftext|>


<commit_before>//Package util provides utility functions used in my this web project.
package util

import (
	"appengine"
	"appengine/user"
	"github.com/gorilla/mux"
	"html/template"
	"math"
	"net/http"
	"time"
)

const (
	TimeFormat = "20060102150405" //Time format used in update queries and reponses and in filenames.
	t          = "templates/"     //Directory with templates
)

var templates *template.Template
var Tz, _ = time.LoadLocation("UTC")

func init() {
	temp := template.New("").Funcs(template.FuncMap{
		"equal": func(x, y int) bool {
			return x == y
		},
		"subtract": func(x, y int) int {
			return x - y
		},
		"add": func(x, y int) int {
			return x + y
		}})
	// List of template files. When creating new template, add it here.
	templates = template.Must(temp.ParseFiles(t+"upload.html", t+"layout/header.html", t+"layout/footer.html", t+"archive.html", t+"presentation.html", t+"config.html", t+"layout/configMenu.html", t+"timeConfig.html", t+"timeConfigEdit.html", t+"index.html"))
}

//Type used for passing data to handlers
type Context struct {
	Ac   appengine.Context
	W    http.ResponseWriter
	R    *http.Request
	Vars map[string]string
}

//Maps standard net/http handlers to handlers accepting Context
func Handler(hand func(Context)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		ac := appengine.NewContext(r)
		vars := mux.Vars(r)
		hand(Context{Ac: ac, W: w, R: r, Vars: vars})
	}
}

//Sends an Internal Server Error to user with error message from the error.
func Log500(err error, c Context) {
	c.Ac.Warningf("Error 500. %v", err)
	http.Error(c.W, err.Error(), http.StatusInternalServerError)
}

//Sends a Not Found Error to user with error message from the error.
func Log404(err error, c Context) {
	c.Ac.Infof("Error 404. %v", err)
	http.Error(c.W, err.Error(), http.StatusNotFound)
}

//Inserts template with given name into the layout and sets the title and pipeline.
//The template should be loaded inside templates variable
//If any arguments are provided after the context, they will be treated like links
//to JavaScript scripts to load in the header of the template.
func RenderLayout(tmpl string, title string, data interface{}, c Context, jsIncludes ...string) {
	renderTemplate("header.html", struct {
		Title      string
		JsIncludes []string
		Admin      bool
	}{title, jsIncludes, user.IsAdmin(c.Ac)}, c)
	renderTemplate(tmpl, data, c)
	renderTemplate("footer.html", nil, c)
}

//Renders a single template
func renderTemplate(tmpl string, data interface{}, c Context) {
	if err := templates.ExecuteTemplate(c.W, tmpl, data); err != nil {
		Log500(err, c)
	}
}

func Average(nums ...float64) float64 {
	var total float64
	for _, x := range nums {
		total += x
	}
	return total / float64(len(nums))
}

// Round return rounded version of x with prec precision.
//
// Special cases are:
//	Round(±0) = ±0
//	Round(±Inf) = ±Inf
//	Round(NaN) = NaN
func Round(x float64, prec int) float64 {
	var rounder float64
	pow := math.Pow(10, float64(prec))
	intermed := x * pow

	if intermed < 0.0 {
		intermed -= 0.5
	} else {
		intermed += 0.5
	}
	rounder = float64(int64(intermed))

	return rounder / float64(pow)
}

func NormalizeDate(t time.Time) time.Time {
	return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, Tz)
}

func NormalizeTime(t time.Time) time.Time {
	return time.Date(1, 1, 1, t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), Tz)
}
<commit_msg>Improvements in util documentation<commit_after>//Package util provides utility functions used in my this web project.
package util

import (
	"appengine"
	"appengine/user"
	"github.com/gorilla/mux"
	"html/template"
	"math"
	"net/http"
	"time"
)

const (
	TimeFormat = "20060102150405" //Time format used in update queries and reponses and in filenames.
	t          = "templates/"     //Directory with templates
)

var templates *template.Template
var Tz, _ = time.LoadLocation("UTC")

//In init we inject few utility functions into templates we're using
func init() {
	temp := template.New("").Funcs(template.FuncMap{
		"equal": func(x, y int) bool {
			return x == y
		},
		"subtract": func(x, y int) int {
			return x - y
		},
		"add": func(x, y int) int {
			return x + y
		}})
	// List of template files. When creating new template, add it here.
	templates = template.Must(temp.ParseFiles(t+"upload.html", t+"layout/header.html", t+"layout/footer.html", t+"archive.html", t+"presentation.html", t+"config.html", t+"layout/configMenu.html", t+"timeConfig.html", t+"timeConfigEdit.html", t+"index.html"))
}

//Context is the type used for passing data to handlers
type Context struct {
	Ac   appengine.Context
	W    http.ResponseWriter
	R    *http.Request
	Vars map[string]string
}

//Handler maps standard net/http handlers to handlers accepting Context
func Handler(hand func(Context)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		ac := appengine.NewContext(r)
		vars := mux.Vars(r)
		hand(Context{Ac: ac, W: w, R: r, Vars: vars})
	}
}

//Log500 sends an Internal Server Error to user with error message from the error.
func Log500(err error, c Context) {
	c.Ac.Warningf("Error 500. %v", err)
	http.Error(c.W, err.Error(), http.StatusInternalServerError)
}

//Log404 sends a Not Found Error to user with error message from the error.
func Log404(err error, c Context) {
	c.Ac.Infof("Error 404. %v", err)
	http.Error(c.W, err.Error(), http.StatusNotFound)
}

//RenderLayout inserts template with given name into the layout and sets the title and pipeline.
//The template should be loaded inside templates variable
//If any arguments are provided after the context, they will be treated like links
//to JavaScript scripts to load in the header of the template.
func RenderLayout(tmpl string, title string, data interface{}, c Context, jsIncludes ...string) {
	renderTemplate("header.html", struct {
		Title      string
		JsIncludes []string
		Admin      bool
	}{title, jsIncludes, user.IsAdmin(c.Ac)}, c)
	renderTemplate(tmpl, data, c)
	renderTemplate("footer.html", nil, c)
}

//renderTemplate renders a single template
func renderTemplate(tmpl string, data interface{}, c Context) {
	if err := templates.ExecuteTemplate(c.W, tmpl, data); err != nil {
		Log500(err, c)
	}
}

//Average returns an Average of its arguments
func Average(nums ...float64) float64 {
	var total float64
	for _, x := range nums {
		total += x
	}
	return total / float64(len(nums))
}

// Round return rounded version of x with prec precision.
//
// Special cases are:
//	Round(±0) = ±0
//	Round(±Inf) = ±Inf
//	Round(NaN) = NaN
func Round(x float64, prec int) float64 {
	var rounder float64
	pow := math.Pow(10, float64(prec))
	intermed := x * pow

	if intermed < 0.0 {
		intermed -= 0.5
	} else {
		intermed += 0.5
	}
	rounder = float64(int64(intermed))

	return rounder / float64(pow)
}

//NormalizeDate strips the time part from time.Date leaving only
//year, month and day.
func NormalizeDate(t time.Time) time.Time {
	return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, Tz)
}

//NormalizeTime strips the date part from time.Date leaving only
//hours, minutes, seconds and nanoseconds.
func NormalizeTime(t time.Time) time.Time {
	return time.Date(1, 1, 1, t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), Tz)
}
<|endoftext|>


<commit_before>package bagman_test

import (
    "testing"
    "os"
    "path/filepath"
    "github.com/APTrust/bagman"
)

func TestBagmanHome(t *testing.T) {
    bagmanHome := os.Getenv("BAGMAN_HOME")
    goHome := os.Getenv("GOPATH")
    defer os.Setenv("BAGMAN_HOME", bagmanHome)
    defer os.Setenv("GOPATH", goHome)

    // Should use BAGMAN_HOME, if it's set...
    os.Setenv("BAGMAN_HOME", "/bagman_home")
    bagmanHome, err := bagman.BagmanHome()
    if err != nil {
        t.Error(err)
    }
    if bagmanHome != "/bagman_home" {
        t.Errorf("BagmanHome returned '%s', expected '%s'",
            bagmanHome,
            "/bagman_home")
    }
    os.Setenv("BAGMAN_HOME", "")

    // Otherwise, should use GOPATH
    os.Setenv("GOPATH", "/go_home")
    bagmanHome, err = bagman.BagmanHome()
    if err != nil {
        t.Error(err)
    }
    if bagmanHome != "/go_home/src/github.com/APTrust/bagman" {
        t.Errorf("BagmanHome returned '%s', expected '%s'",
            bagmanHome,
            "/go_home")
    }
    os.Setenv("GOPATH", "")

    // Without BAGMAN_HOME and GOPATH, we should get an error
    bagmanHome, err = bagman.BagmanHome()
    if err == nil {
        t.Error("BagmanHome should have an thrown exception.")
    }
}

func TestLoadRelativeFile(t *testing.T) {
    path := filepath.Join("testdata", "result_good.json")
    data, err := bagman.LoadRelativeFile(path)
    if err != nil {
        t.Error(err)
    }
    if data == nil || len(data) == 0 {
        t.Errorf("Read no data out of file '%s'", path)
    }
}

func TestSyncMap(t *testing.T) {
	syncMap := bagman.NewSynchronizedMap()
	keys1 := [...]string { "1", "2", "3", "4", "5" }
	keys2 := [...]string { "6", "7", "8", "9", "10" }
	go testSyncMap(t, syncMap, keys1)
	go testSyncMap(t, syncMap, keys2)
}

func testSyncMap(t *testing.T, syncMap *bagman.SynchronizedMap, keys [5]string) {
	for i, key := range keys {
		syncMap.Add(key, key)
		if syncMap.HasKey(key) == false {
			t.Errorf("SyncMap should have key %s", key)
		}
		if syncMap.Get(key) != key {
			t.Errorf("SyncMap key %s has value %s, expected %s", key, syncMap.Get(key), key)
		}
		if len(syncMap.Keys()) < i {
			t.Errorf("SyncMap should have at least %d keys, but it has %d", i, len(syncMap.Keys()))
		}
		if len(syncMap.Values()) < i {
			t.Errorf("SyncMap should have at least %d values, but it has %d", i, len(syncMap.Values()))
		}
	}
}

func TestSyncMapDelete(t *testing.T) {
	syncMap := bagman.NewSynchronizedMap()
	keys := [...]string { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10"}

	for _, key := range keys {
		syncMap.Add(key, key)
	}
	if len(syncMap.Keys()) != len(keys) {
		t.Errorf("SyncMap should have %d keys, but it has %d", len(keys), len(syncMap.Keys()))
	}
	if len(syncMap.Values()) != len(keys) {
		t.Errorf("SyncMap should have %d values, but it has %d", len(keys), len(syncMap.Values()))
	}
	for _, key := range keys {
		syncMap.Delete(key)
		if syncMap.HasKey(key) == true {
			t.Errorf("SyncMap should not have key %s", key)
		}
	}
	if len(syncMap.Keys()) != 0 {
		t.Errorf("SyncMap should have 0 keys, but it has %d", len(syncMap.Keys()))
	}
	if len(syncMap.Values()) != 0 {
		t.Errorf("SyncMap should have 0 values, but it has %d", len(syncMap.Values()))
	}
}
<commit_msg>Fixed []string type<commit_after>package bagman_test

import (
    "testing"
    "os"
    "path/filepath"
    "github.com/APTrust/bagman"
)

func TestBagmanHome(t *testing.T) {
    bagmanHome := os.Getenv("BAGMAN_HOME")
    goHome := os.Getenv("GOPATH")
    defer os.Setenv("BAGMAN_HOME", bagmanHome)
    defer os.Setenv("GOPATH", goHome)

    // Should use BAGMAN_HOME, if it's set...
    os.Setenv("BAGMAN_HOME", "/bagman_home")
    bagmanHome, err := bagman.BagmanHome()
    if err != nil {
        t.Error(err)
    }
    if bagmanHome != "/bagman_home" {
        t.Errorf("BagmanHome returned '%s', expected '%s'",
            bagmanHome,
            "/bagman_home")
    }
    os.Setenv("BAGMAN_HOME", "")

    // Otherwise, should use GOPATH
    os.Setenv("GOPATH", "/go_home")
    bagmanHome, err = bagman.BagmanHome()
    if err != nil {
        t.Error(err)
    }
    if bagmanHome != "/go_home/src/github.com/APTrust/bagman" {
        t.Errorf("BagmanHome returned '%s', expected '%s'",
            bagmanHome,
            "/go_home")
    }
    os.Setenv("GOPATH", "")

    // Without BAGMAN_HOME and GOPATH, we should get an error
    bagmanHome, err = bagman.BagmanHome()
    if err == nil {
        t.Error("BagmanHome should have an thrown exception.")
    }
}

func TestLoadRelativeFile(t *testing.T) {
    path := filepath.Join("testdata", "result_good.json")
    data, err := bagman.LoadRelativeFile(path)
    if err != nil {
        t.Error(err)
    }
    if data == nil || len(data) == 0 {
        t.Errorf("Read no data out of file '%s'", path)
    }
}

func TestSyncMap(t *testing.T) {
	syncMap := bagman.NewSynchronizedMap()
	keys1 := []string { "1", "2", "3", "4", "5" }
	keys2 := []string { "6", "7", "8", "9", "10" }
	go testSyncMap(t, syncMap, keys1)
	go testSyncMap(t, syncMap, keys2)
}

func testSyncMap(t *testing.T, syncMap *bagman.SynchronizedMap, keys []string) {
	for i, key := range keys {
		syncMap.Add(key, key)
		if syncMap.HasKey(key) == false {
			t.Errorf("SyncMap should have key %s", key)
		}
		if syncMap.Get(key) != key {
			t.Errorf("SyncMap key %s has value %s, expected %s", key, syncMap.Get(key), key)
		}
		if len(syncMap.Keys()) < i {
			t.Errorf("SyncMap should have at least %d keys, but it has %d", i, len(syncMap.Keys()))
		}
		if len(syncMap.Values()) < i {
			t.Errorf("SyncMap should have at least %d values, but it has %d", i, len(syncMap.Values()))
		}
	}
}

func TestSyncMapDelete(t *testing.T) {
	syncMap := bagman.NewSynchronizedMap()
	keys := []string { "1", "2", "3", "4", "5", "6", "7", "8", "9", "10"}

	for _, key := range keys {
		syncMap.Add(key, key)
	}
	if len(syncMap.Keys()) != len(keys) {
		t.Errorf("SyncMap should have %d keys, but it has %d", len(keys), len(syncMap.Keys()))
	}
	if len(syncMap.Values()) != len(keys) {
		t.Errorf("SyncMap should have %d values, but it has %d", len(keys), len(syncMap.Values()))
	}
	for _, key := range keys {
		syncMap.Delete(key)
		if syncMap.HasKey(key) == true {
			t.Errorf("SyncMap should not have key %s", key)
		}
	}
	if len(syncMap.Keys()) != 0 {
		t.Errorf("SyncMap should have 0 keys, but it has %d", len(syncMap.Keys()))
	}
	if len(syncMap.Values()) != 0 {
		t.Errorf("SyncMap should have 0 values, but it has %d", len(syncMap.Values()))
	}
}
<|endoftext|>


<commit_before>package smsclub

import (
	"bytes"
	"context"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
)

type methodAPI int

const (
	mSend methodAPI = iota
	mStatus
	mBalance
)

const (
	endPoint   = "https://gate.smsclub.mobi/token/"
	urlSend    = endPoint
	urlStatus  = endPoint + "state.php"
	urlBalance = endPoint + "getbalance.php"
)

var mapURL = map[methodAPI]string{
	mSend:    urlSend,
	mStatus:  urlStatus,
	mBalance: urlBalance,
}

// SMSCluber is interface for https://smsclub.mobi/en/pages/show/api
type SMSCluber interface {
	// Balance returns values for balance and credit.
	Balance() (float64, float64, error)

	// Send sends SMS text message to recipients.
	Send(text string, to ...string) ([]string, error)

	// Status gets list of SMS identifiers and returns statuses for ones.
	Status(id ...string) ([]string, error)
}

// New returns SMSCluber interface. Minimal *must* options are User(), Token().
func New(options ...func(*option) error) (SMSCluber, error) {
	c := &client{
		&option{},
	}

	var err error
	for i := range options {
		err = options[i](c.option)
		if err != nil {
			return nil, err
		}
	}

	return c, nil
}

type option struct {
	user     string
	token    string
	sender   string // alphaname
	lifetime time.Duration
	timeout  time.Duration
}

type client struct {
	*option
}

func (c *client) String() string {
	return fmt.Sprintf("%s %s %s %d", c.user, c.token, c.sender, c.lifetime)
}

func (c *client) makeForm(f url.Values) url.Values {
	if f == nil {
		f = url.Values{}
	}

	f.Set("username", c.user)
	f.Set("token", c.token)
	if c.lifetime > 0 {
		f.Set("lifetime", strconv.Itoa(int(c.lifetime.Minutes())))
	}

	return f
}

func (c *client) withRequest(m methodAPI, v url.Values) (*http.Request, error) {
	f := c.makeForm(v)
	r, err := http.NewRequest("POST", mapURL[m], strings.NewReader(f.Encode()))
	if err != nil {
		return nil, err
	}
	r.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	return r, nil
}

func (c *client) parseResponse(r io.Reader) ([]string, error) {
	b, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}

	var out []string
	for _, s := range strings.Split(string(b), "
") {
		if strings.Contains(s, "=") || strings.TrimSpace(s) == "" {
			continue
		}
		out = append(out, s)
	}

	if out == nil {
		return nil, io.EOF
	}

	return out, nil
}

func (c *client) callAPI(r *http.Request) ([]string, error) {
	if c.timeout > 0 {
		ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
		defer cancel()
		r = r.WithContext(ctx)
	}

	res, err := http.DefaultClient.Do(r)
	if err != nil || res == nil {
		return nil, err
	}
	if res != nil {
		defer res.Body.Close()
	}

	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("smsclub: server returns %d %s", res.StatusCode, res.Status)
	}

	return c.parseResponse(res.Body)
}

// Balance returns values for balance and credit.
func (c *client) Balance() (float64, float64, error) {
	req, err := c.withRequest(mBalance, nil)
	if err != nil {
		return 0.0, 0.0, err
	}

	res, err := c.callAPI(req)
	if err != nil {
		return 0.0, 0.0, err
	}

	var bal, cre float64
	if len(res) == 2 {
		bal, _ = strconv.ParseFloat(res[0], 64)
		cre, _ = strconv.ParseFloat(res[1], 64)
	}

	return bal, cre, nil
}

// Send sends SMS text message to recipients.
func (c *client) Send(text string, to ...string) ([]string, error) {
	toBase64 := func(s string) string {
		return base64.StdEncoding.EncodeToString([]byte(s))
	}
	toWin1251 := func(s string) string {
		b := new(bytes.Buffer)
		w := transform.NewWriter(b, charmap.Windows1251.NewEncoder())
		_, _ = w.Write([]byte(s))
		return b.String()
	}

	form := url.Values{
		"from": []string{c.sender},
		"text": []string{toBase64(toWin1251(text))},
		"to":   []string{strings.Join(to, ";")},
	}

	req, err := c.withRequest(mSend, form)
	if err != nil {
		return nil, err
	}
	return c.callAPI(req)
}

// Status gets list of SMS identifiers and returns statuses for ones.
func (c *client) Status(id ...string) ([]string, error) {
	form := url.Values{
		"smscid": []string{strings.Join(id, ";")},
	}

	req, err := c.withRequest(mStatus, form)
	if err != nil {
		return nil, err
	}

	res, err := c.callAPI(req)
	if err != nil {
		return nil, err
	}

	for i := range res {
		spl := strings.Split(res[i], ":")
		if len(spl) == 2 {
			res[i] = strings.TrimSpace(spl[1])
		}
	}

	return res, nil
}

// User is login of user’s account
func User(name string) func(*option) error {
	return func(o *option) error {
		if name == "" {
			return fmt.Errorf("smsclub: username is empty")
		}
		o.user = name
		return nil
	}
}

// Token is token of user’s account (you can find it in profile).
func Token(val string) func(*option) error {
	return func(o *option) error {
		if val == "" {
			return fmt.Errorf("smsclub: token is empty")
		}
		o.token = val
		return nil
	}
}

// Sender is Sender ID, from which mail-out is perfomed (11 English letters, numbers, spaces).
// See https://my.smsclub.mobi/en/alphanames/index.
func Sender(val string) func(*option) error {
	return func(o *option) error {
		if val == "" {
			return fmt.Errorf("smsclub: sender (alphaName) is empty")
		}
		o.sender = val
		return nil
	}
}

// LifeTime sets life time of SMS, which is specified in minutes.
func LifeTime(d time.Duration) func(*option) error {
	return func(o *option) error {
		if d < 0 {
			return fmt.Errorf("smsclub: invalid duration value %d", d)
		}
		o.lifetime = d
		return nil
	}
}

// Timeout sets timeout for calls Balance, Send and Status.
func Timeout(d time.Duration) func(*option) error {
	return func(o *option) error {
		if d < 0 {
			return fmt.Errorf("smsclub: invalid duration value %d", d)
		}
		o.timeout = d
		return nil
	}
}
<commit_msg>add workaround for nginx HTTP/2 bug<commit_after>package smsclub

import (
	"bytes"
	"context"
	"crypto/tls"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
)

type methodAPI int

const (
	mSend methodAPI = iota
	mStatus
	mBalance
)

const (
	endPoint   = "https://gate.smsclub.mobi/token/"
	urlSend    = endPoint
	urlStatus  = endPoint + "state.php"
	urlBalance = endPoint + "getbalance.php"
)

var mapURL = map[methodAPI]string{
	mSend:    urlSend,
	mStatus:  urlStatus,
	mBalance: urlBalance,
}

// SMSCluber is interface for https://smsclub.mobi/en/pages/show/api
type SMSCluber interface {
	// Balance returns values for balance and credit.
	Balance() (float64, float64, error)

	// Send sends SMS text message to recipients.
	Send(text string, to ...string) ([]string, error)

	// Status gets list of SMS identifiers and returns statuses for ones.
	Status(id ...string) ([]string, error)
}

// New returns SMSCluber interface. Minimal *must* options are User(), Token().
func New(options ...func(*option) error) (SMSCluber, error) {
	c := &client{
		&option{},
	}

	var err error
	for i := range options {
		err = options[i](c.option)
		if err != nil {
			return nil, err
		}
	}

	return c, nil
}

type option struct {
	user     string
	token    string
	sender   string // alphaname
	lifetime time.Duration
	timeout  time.Duration
}

type client struct {
	*option
}

func (c *client) String() string {
	return fmt.Sprintf("%s %s %s %d", c.user, c.token, c.sender, c.lifetime)
}

func (c *client) makeForm(v url.Values) url.Values {
	if v == nil {
		v = url.Values{}
	}

	v.Set("username", c.user)
	v.Set("token", c.token)
	if c.lifetime > 0 {
		v.Set("lifetime", strconv.Itoa(int(c.lifetime.Minutes())))
	}

	return v
}

func (c *client) withRequest(m methodAPI, v url.Values) (*http.Request, error) {
	v = c.makeForm(v)
	r, err := http.NewRequest("GET", fmt.Sprintf("%s?%s", mapURL[m], v.Encode()), nil)
	if err != nil {
		return nil, err
	}
	return r, nil
}

func (c *client) parseResponse(r io.Reader) ([]string, error) {
	b, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}

	var out []string
	for _, s := range strings.Split(string(b), "
") {
		if strings.Contains(s, "=") || strings.TrimSpace(s) == "" {
			continue
		}
		out = append(out, s)
	}

	if out == nil {
		return nil, io.EOF
	}

	return out, nil
}

func (c *client) callAPI(r *http.Request) ([]string, error) {
	if c.timeout > 0 {
		ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
		defer cancel()
		r = r.WithContext(ctx)
	}

	// this is workaround for
	// https://trac.nginx.org/nginx/ticket/959
	tr := &http.Transport{
		TLSNextProto: make(map[string]func(string, *tls.Conn) http.RoundTripper),
	}
	cli := &http.Client{Transport: tr}

	res, err := cli.Do(r)
	if err != nil || res == nil {
		return nil, err
	}
	if res != nil {
		defer res.Body.Close()
	}

	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("smsclub: server returns %d %s", res.StatusCode, res.Status)
	}

	return c.parseResponse(res.Body)
}

// Balance returns values for balance and credit.
func (c *client) Balance() (float64, float64, error) {
	req, err := c.withRequest(mBalance, nil)
	if err != nil {
		return 0.0, 0.0, err
	}

	res, err := c.callAPI(req)
	if err != nil {
		return 0.0, 0.0, err
	}

	var bal, cre float64
	if len(res) == 2 {
		bal, _ = strconv.ParseFloat(res[0], 64)
		cre, _ = strconv.ParseFloat(res[1], 64)
	}

	return bal, cre, nil
}

// Send sends SMS text message to recipients.
func (c *client) Send(text string, to ...string) ([]string, error) {
	toBase64 := func(s string) string {
		return base64.StdEncoding.EncodeToString([]byte(s))
	}
	toWin1251 := func(s string) string {
		b := new(bytes.Buffer)
		w := transform.NewWriter(b, charmap.Windows1251.NewEncoder())
		_, _ = w.Write([]byte(s))
		return b.String()
	}

	form := url.Values{
		"from": []string{c.sender},
		"text": []string{toBase64(toWin1251(text))},
		"to":   []string{strings.Join(to, ";")},
	}

	req, err := c.withRequest(mSend, form)
	if err != nil {
		return nil, err
	}
	return c.callAPI(req)
}

// Status gets list of SMS identifiers and returns statuses for ones.
func (c *client) Status(id ...string) ([]string, error) {
	form := url.Values{
		"smscid": []string{strings.Join(id, ";")},
	}

	req, err := c.withRequest(mStatus, form)
	if err != nil {
		return nil, err
	}

	res, err := c.callAPI(req)
	if err != nil {
		return nil, err
	}

	for i := range res {
		spl := strings.Split(res[i], ":")
		if len(spl) == 2 {
			res[i] = strings.TrimSpace(spl[1])
		}
	}

	return res, nil
}

// User is login of user’s account
func User(name string) func(*option) error {
	return func(o *option) error {
		if name == "" {
			return fmt.Errorf("smsclub: username is empty")
		}
		o.user = name
		return nil
	}
}

// Token is token of user’s account (you can find it in profile).
func Token(val string) func(*option) error {
	return func(o *option) error {
		if val == "" {
			return fmt.Errorf("smsclub: token is empty")
		}
		o.token = val
		return nil
	}
}

// Sender is Sender ID, from which mail-out is perfomed (11 English letters, numbers, spaces).
// See https://my.smsclub.mobi/en/alphanames/index.
func Sender(val string) func(*option) error {
	return func(o *option) error {
		if val == "" {
			return fmt.Errorf("smsclub: sender (alphaName) is empty")
		}
		o.sender = val
		return nil
	}
}

// LifeTime sets life time of SMS, which is specified in minutes.
func LifeTime(d time.Duration) func(*option) error {
	return func(o *option) error {
		if d < 0 {
			return fmt.Errorf("smsclub: invalid duration value %d", d)
		}
		o.lifetime = d
		return nil
	}
}

// Timeout sets timeout for calls Balance, Send and Status.
func Timeout(d time.Duration) func(*option) error {
	return func(o *option) error {
		if d < 0 {
			return fmt.Errorf("smsclub: invalid duration value %d", d)
		}
		o.timeout = d
		return nil
	}
}
<|endoftext|>


<commit_before>package content

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"time"

	"github.com/containerd/containerd/log"
	digest "github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
)

// Store is digest-keyed store for content. All data written into the store is
// stored under a verifiable digest.
//
// Store can generally support multi-reader, single-writer ingest of data,
// including resumable ingest.
type store struct {
	root string
}

func NewStore(root string) (Store, error) {
	if err := os.MkdirAll(filepath.Join(root, "ingest"), 0777); err != nil && !os.IsExist(err) {
		return nil, err
	}

	return &store{
		root: root,
	}, nil
}

func (s *store) Info(ctx context.Context, dgst digest.Digest) (Info, error) {
	p := s.blobPath(dgst)
	fi, err := os.Stat(p)
	if err != nil {
		if os.IsNotExist(err) {
			err = ErrNotFound("")
		}

		return Info{}, err
	}

	return s.info(dgst, fi), nil
}

func (s *store) info(dgst digest.Digest, fi os.FileInfo) Info {
	return Info{
		Digest:      dgst,
		Size:        fi.Size(),
		CommittedAt: fi.ModTime(),
	}
}

// Reader returns an io.ReadCloser for the blob.
func (s *store) Reader(ctx context.Context, dgst digest.Digest) (io.ReadCloser, error) {
	fp, err := os.Open(s.blobPath(dgst))
	if err != nil {
		if os.IsNotExist(err) {
			err = ErrNotFound("")
		}
		return nil, err
	}

	return fp, nil
}

// ReaderAt returns an io.ReaderAt for the blob.
func (s *store) ReaderAt(ctx context.Context, dgst digest.Digest) (io.ReaderAt, error) {
	return readerAt{f: s.blobPath(dgst)}, nil
}

// Delete removes a blob by its digest.
//
// While this is safe to do concurrently, safe exist-removal logic must hold
// some global lock on the store.
func (cs *store) Delete(ctx context.Context, dgst digest.Digest) error {
	if err := os.RemoveAll(cs.blobPath(dgst)); err != nil {
		if !os.IsNotExist(err) {
			return err
		}

		return ErrNotFound("")
	}

	return nil
}

// TODO(stevvooe): Allow querying the set of blobs in the blob store.

func (cs *store) Walk(ctx context.Context, fn WalkFunc) error {
	root := filepath.Join(cs.root, "blobs")
	var alg digest.Algorithm
	return filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !fi.IsDir() && !alg.Available() {
			return nil
		}

		// TODO(stevvooe): There are few more cases with subdirs that should be
		// handled in case the layout gets corrupted. This isn't strict enough
		// an may spew bad data.

		if path == root {
			return nil
		}
		if filepath.Dir(path) == root {
			alg = digest.Algorithm(filepath.Base(path))

			if !alg.Available() {
				alg = ""
				return filepath.SkipDir
			}

			// descending into a hash directory
			return nil
		}

		dgst := digest.NewDigestFromHex(alg.String(), filepath.Base(path))
		if err := dgst.Validate(); err != nil {
			// log error but don't report
			log.L.WithError(err).WithField("path", path).Error("invalid digest for blob path")
			// if we see this, it could mean some sort of corruption of the
			// store or extra paths not expected previously.
		}

		return fn(cs.info(dgst, fi))
	})
}

func (s *store) Status(ctx context.Context, re string) ([]Status, error) {
	fp, err := os.Open(filepath.Join(s.root, "ingest"))
	if err != nil {
		return nil, err
	}

	defer fp.Close()

	fis, err := fp.Readdir(-1)
	if err != nil {
		return nil, err
	}

	rec, err := regexp.Compile(re)
	if err != nil {
		return nil, err
	}

	var active []Status
	for _, fi := range fis {
		p := filepath.Join(s.root, "ingest", fi.Name())
		stat, err := s.status(p)
		if err != nil {
			if !os.IsNotExist(err) {
				return nil, err
			}

			// TODO(stevvooe): This is a common error if uploads are being
			// completed while making this listing. Need to consider taking a
			// lock on the whole store to coordinate this aspect.
			//
			// Another option is to cleanup downloads asynchronously and
			// coordinate this method with the cleanup process.
			//
			// For now, we just skip them, as they really don't exist.
			continue
		}

		if !rec.MatchString(stat.Ref) {
			continue
		}

		active = append(active, stat)
	}

	return active, nil
}

// status works like stat above except uses the path to the ingest.
func (s *store) status(ingestPath string) (Status, error) {
	dp := filepath.Join(ingestPath, "data")
	fi, err := os.Stat(dp)
	if err != nil {
		return Status{}, err
	}

	ref, err := readFileString(filepath.Join(ingestPath, "ref"))
	if err != nil {
		return Status{}, err
	}

	return Status{
		Ref:       ref,
		Offset:    fi.Size(),
		Total:     s.total(ingestPath),
		UpdatedAt: fi.ModTime(),
		StartedAt: getStartTime(fi),
	}, nil
}

// total attempts to resolve the total expected size for the write.
func (s *store) total(ingestPath string) int64 {
	totalS, err := readFileString(filepath.Join(ingestPath, "total"))
	if err != nil {
		return 0
	}

	total, err := strconv.ParseInt(totalS, 10, 64)
	if err != nil {
		// represents a corrupted file, should probably remove.
		return 0
	}

	return total
}

// Writer begins or resumes the active writer identified by ref. If the writer
// is already in use, an error is returned. Only one writer may be in use per
// ref at a time.
//
// The argument `ref` is used to uniquely identify a long-lived writer transaction.
func (s *store) Writer(ctx context.Context, ref string, total int64, expected digest.Digest) (Writer, error) {
	// TODO(stevvooe): Need to actually store expected here. We have
	// code in the service that shouldn't be dealing with this.
	if expected != "" {
		p := s.blobPath(expected)
		if _, err := os.Stat(p); err == nil {
			return nil, ErrExists
		}
	}

	path, refp, data := s.ingestPaths(ref)

	if err := tryLock(ref); err != nil {
		return nil, errors.Wrapf(err, "locking %v failed", ref)
	}

	var (
		digester  = digest.Canonical.Digester()
		offset    int64
		startedAt time.Time
		updatedAt time.Time
	)

	// ensure that the ingest path has been created.
	if err := os.Mkdir(path, 0755); err != nil {
		if !os.IsExist(err) {
			return nil, err
		}

		status, err := s.status(path)
		if err != nil {
			return nil, errors.Wrap(err, "failed reading status of resume write")
		}

		if ref != status.Ref {
			// NOTE(stevvooe): This is fairly catastrophic. Either we have some
			// layout corruption or a hash collision for the ref key.
			return nil, errors.Wrapf(err, "ref key does not match: %v != %v", ref, status.Ref)
		}

		if total > 0 && status.Total > 0 && total != status.Total {
			return nil, errors.Errorf("provided total differs from status: %v != %v", total, status.Total)
		}

		// slow slow slow!!, send to goroutine or use resumable hashes
		fp, err := os.Open(data)
		if err != nil {
			return nil, err
		}
		defer fp.Close()

		p := bufPool.Get().([]byte)
		defer bufPool.Put(p)

		offset, err = io.CopyBuffer(digester.Hash(), fp, p)
		if err != nil {
			return nil, err
		}

		updatedAt = status.UpdatedAt
		startedAt = status.StartedAt
		total = status.Total
	} else {
		// the ingest is new, we need to setup the target location.
		// write the ref to a file for later use
		if err := ioutil.WriteFile(refp, []byte(ref), 0666); err != nil {
			return nil, err
		}

		if total > 0 {
			if err := ioutil.WriteFile(filepath.Join(path, "total"), []byte(fmt.Sprint(total)), 0666); err != nil {
				return nil, err
			}
		}

		startedAt = time.Now()
		updatedAt = startedAt
	}

	fp, err := os.OpenFile(data, os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		return nil, errors.Wrap(err, "failed to open data file")
	}

	return &writer{
		s:         s,
		fp:        fp,
		ref:       ref,
		path:      path,
		offset:    offset,
		total:     total,
		digester:  digester,
		startedAt: startedAt,
		updatedAt: updatedAt,
	}, nil
}

// Abort an active transaction keyed by ref. If the ingest is active, it will
// be cancelled. Any resources associated with the ingest will be cleaned.
func (s *store) Abort(ctx context.Context, ref string) error {
	root := s.ingestRoot(ref)
	if err := os.RemoveAll(root); err != nil {
		if os.IsNotExist(err) {
			return ErrNotFound("")
		}

		return err
	}

	return nil
}

func (cs *store) blobPath(dgst digest.Digest) string {
	return filepath.Join(cs.root, "blobs", dgst.Algorithm().String(), dgst.Hex())
}

func (s *store) ingestRoot(ref string) string {
	dgst := digest.FromString(ref)
	return filepath.Join(s.root, "ingest", dgst.Hex())
}

// ingestPaths are returned. The paths are the following:
//
// - root: entire ingest directory
// - ref: name of the starting ref, must be unique
// - data: file where data is written
//
func (s *store) ingestPaths(ref string) (string, string, string) {
	var (
		fp = s.ingestRoot(ref)
		rp = filepath.Join(fp, "ref")
		dp = filepath.Join(fp, "data")
	)

	return fp, rp, dp
}
<commit_msg>Fix store errors build conflict<commit_after>package content

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"time"

	"github.com/containerd/containerd/log"
	digest "github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
)

// Store is digest-keyed store for content. All data written into the store is
// stored under a verifiable digest.
//
// Store can generally support multi-reader, single-writer ingest of data,
// including resumable ingest.
type store struct {
	root string
}

func NewStore(root string) (Store, error) {
	if err := os.MkdirAll(filepath.Join(root, "ingest"), 0777); err != nil && !os.IsExist(err) {
		return nil, err
	}

	return &store{
		root: root,
	}, nil
}

func (s *store) Info(ctx context.Context, dgst digest.Digest) (Info, error) {
	p := s.blobPath(dgst)
	fi, err := os.Stat(p)
	if err != nil {
		if os.IsNotExist(err) {
			err = ErrNotFound("")
		}

		return Info{}, err
	}

	return s.info(dgst, fi), nil
}

func (s *store) info(dgst digest.Digest, fi os.FileInfo) Info {
	return Info{
		Digest:      dgst,
		Size:        fi.Size(),
		CommittedAt: fi.ModTime(),
	}
}

// Reader returns an io.ReadCloser for the blob.
func (s *store) Reader(ctx context.Context, dgst digest.Digest) (io.ReadCloser, error) {
	fp, err := os.Open(s.blobPath(dgst))
	if err != nil {
		if os.IsNotExist(err) {
			err = ErrNotFound("")
		}
		return nil, err
	}

	return fp, nil
}

// ReaderAt returns an io.ReaderAt for the blob.
func (s *store) ReaderAt(ctx context.Context, dgst digest.Digest) (io.ReaderAt, error) {
	return readerAt{f: s.blobPath(dgst)}, nil
}

// Delete removes a blob by its digest.
//
// While this is safe to do concurrently, safe exist-removal logic must hold
// some global lock on the store.
func (cs *store) Delete(ctx context.Context, dgst digest.Digest) error {
	if err := os.RemoveAll(cs.blobPath(dgst)); err != nil {
		if !os.IsNotExist(err) {
			return err
		}

		return ErrNotFound("")
	}

	return nil
}

// TODO(stevvooe): Allow querying the set of blobs in the blob store.

func (cs *store) Walk(ctx context.Context, fn WalkFunc) error {
	root := filepath.Join(cs.root, "blobs")
	var alg digest.Algorithm
	return filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !fi.IsDir() && !alg.Available() {
			return nil
		}

		// TODO(stevvooe): There are few more cases with subdirs that should be
		// handled in case the layout gets corrupted. This isn't strict enough
		// an may spew bad data.

		if path == root {
			return nil
		}
		if filepath.Dir(path) == root {
			alg = digest.Algorithm(filepath.Base(path))

			if !alg.Available() {
				alg = ""
				return filepath.SkipDir
			}

			// descending into a hash directory
			return nil
		}

		dgst := digest.NewDigestFromHex(alg.String(), filepath.Base(path))
		if err := dgst.Validate(); err != nil {
			// log error but don't report
			log.L.WithError(err).WithField("path", path).Error("invalid digest for blob path")
			// if we see this, it could mean some sort of corruption of the
			// store or extra paths not expected previously.
		}

		return fn(cs.info(dgst, fi))
	})
}

func (s *store) Status(ctx context.Context, re string) ([]Status, error) {
	fp, err := os.Open(filepath.Join(s.root, "ingest"))
	if err != nil {
		return nil, err
	}

	defer fp.Close()

	fis, err := fp.Readdir(-1)
	if err != nil {
		return nil, err
	}

	rec, err := regexp.Compile(re)
	if err != nil {
		return nil, err
	}

	var active []Status
	for _, fi := range fis {
		p := filepath.Join(s.root, "ingest", fi.Name())
		stat, err := s.status(p)
		if err != nil {
			if !os.IsNotExist(err) {
				return nil, err
			}

			// TODO(stevvooe): This is a common error if uploads are being
			// completed while making this listing. Need to consider taking a
			// lock on the whole store to coordinate this aspect.
			//
			// Another option is to cleanup downloads asynchronously and
			// coordinate this method with the cleanup process.
			//
			// For now, we just skip them, as they really don't exist.
			continue
		}

		if !rec.MatchString(stat.Ref) {
			continue
		}

		active = append(active, stat)
	}

	return active, nil
}

// status works like stat above except uses the path to the ingest.
func (s *store) status(ingestPath string) (Status, error) {
	dp := filepath.Join(ingestPath, "data")
	fi, err := os.Stat(dp)
	if err != nil {
		return Status{}, err
	}

	ref, err := readFileString(filepath.Join(ingestPath, "ref"))
	if err != nil {
		return Status{}, err
	}

	return Status{
		Ref:       ref,
		Offset:    fi.Size(),
		Total:     s.total(ingestPath),
		UpdatedAt: fi.ModTime(),
		StartedAt: getStartTime(fi),
	}, nil
}

// total attempts to resolve the total expected size for the write.
func (s *store) total(ingestPath string) int64 {
	totalS, err := readFileString(filepath.Join(ingestPath, "total"))
	if err != nil {
		return 0
	}

	total, err := strconv.ParseInt(totalS, 10, 64)
	if err != nil {
		// represents a corrupted file, should probably remove.
		return 0
	}

	return total
}

// Writer begins or resumes the active writer identified by ref. If the writer
// is already in use, an error is returned. Only one writer may be in use per
// ref at a time.
//
// The argument `ref` is used to uniquely identify a long-lived writer transaction.
func (s *store) Writer(ctx context.Context, ref string, total int64, expected digest.Digest) (Writer, error) {
	// TODO(stevvooe): Need to actually store expected here. We have
	// code in the service that shouldn't be dealing with this.
	if expected != "" {
		p := s.blobPath(expected)
		if _, err := os.Stat(p); err == nil {
			return nil, ErrExists("")
		}
	}

	path, refp, data := s.ingestPaths(ref)

	if err := tryLock(ref); err != nil {
		return nil, errors.Wrapf(err, "locking %v failed", ref)
	}

	var (
		digester  = digest.Canonical.Digester()
		offset    int64
		startedAt time.Time
		updatedAt time.Time
	)

	// ensure that the ingest path has been created.
	if err := os.Mkdir(path, 0755); err != nil {
		if !os.IsExist(err) {
			return nil, err
		}

		status, err := s.status(path)
		if err != nil {
			return nil, errors.Wrap(err, "failed reading status of resume write")
		}

		if ref != status.Ref {
			// NOTE(stevvooe): This is fairly catastrophic. Either we have some
			// layout corruption or a hash collision for the ref key.
			return nil, errors.Wrapf(err, "ref key does not match: %v != %v", ref, status.Ref)
		}

		if total > 0 && status.Total > 0 && total != status.Total {
			return nil, errors.Errorf("provided total differs from status: %v != %v", total, status.Total)
		}

		// slow slow slow!!, send to goroutine or use resumable hashes
		fp, err := os.Open(data)
		if err != nil {
			return nil, err
		}
		defer fp.Close()

		p := bufPool.Get().([]byte)
		defer bufPool.Put(p)

		offset, err = io.CopyBuffer(digester.Hash(), fp, p)
		if err != nil {
			return nil, err
		}

		updatedAt = status.UpdatedAt
		startedAt = status.StartedAt
		total = status.Total
	} else {
		// the ingest is new, we need to setup the target location.
		// write the ref to a file for later use
		if err := ioutil.WriteFile(refp, []byte(ref), 0666); err != nil {
			return nil, err
		}

		if total > 0 {
			if err := ioutil.WriteFile(filepath.Join(path, "total"), []byte(fmt.Sprint(total)), 0666); err != nil {
				return nil, err
			}
		}

		startedAt = time.Now()
		updatedAt = startedAt
	}

	fp, err := os.OpenFile(data, os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		return nil, errors.Wrap(err, "failed to open data file")
	}

	return &writer{
		s:         s,
		fp:        fp,
		ref:       ref,
		path:      path,
		offset:    offset,
		total:     total,
		digester:  digester,
		startedAt: startedAt,
		updatedAt: updatedAt,
	}, nil
}

// Abort an active transaction keyed by ref. If the ingest is active, it will
// be cancelled. Any resources associated with the ingest will be cleaned.
func (s *store) Abort(ctx context.Context, ref string) error {
	root := s.ingestRoot(ref)
	if err := os.RemoveAll(root); err != nil {
		if os.IsNotExist(err) {
			return ErrNotFound("")
		}

		return err
	}

	return nil
}

func (cs *store) blobPath(dgst digest.Digest) string {
	return filepath.Join(cs.root, "blobs", dgst.Algorithm().String(), dgst.Hex())
}

func (s *store) ingestRoot(ref string) string {
	dgst := digest.FromString(ref)
	return filepath.Join(s.root, "ingest", dgst.Hex())
}

// ingestPaths are returned. The paths are the following:
//
// - root: entire ingest directory
// - ref: name of the starting ref, must be unique
// - data: file where data is written
//
func (s *store) ingestPaths(ref string) (string, string, string) {
	var (
		fp = s.ingestRoot(ref)
		rp = filepath.Join(fp, "ref")
		dp = filepath.Join(fp, "data")
	)

	return fp, rp, dp
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013 Conformal Systems LLC <info@conformal.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package main

import (
	"code.google.com/p/go.net/websocket"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/conformal/btcjson"
	"github.com/conformal/btcwire"
	"net/http"
	"sync"
)

var (
	ConnRefused = errors.New("Connection refused")

	// Channel to close to notify that connection to btcd has been lost.
	btcdDisconnected = make(chan int)

	// Channel to send messages btcwallet does not understand to btcd.
	btcdMsgs = make(chan []byte, 100)

	// Adds a frontend listener channel
	addFrontendListener = make(chan (chan []byte))

	// Removes a frontend listener channel
	deleteFrontendListener = make(chan (chan []byte))

	// Messages sent to this channel are sent to each connected frontend.
	frontendNotificationMaster = make(chan []byte, 100)

	replyHandlers = struct {
		sync.Mutex
		m map[uint64]func(interface{}) bool
	}{
		m: make(map[uint64]func(interface{}) bool),
	}
)

// frontendListenerDuplicator listens for new wallet listener channels
// and duplicates messages sent to frontendNotificationMaster to all
// connected listeners.
func frontendListenerDuplicator() {
	// frontendListeners is a map holding each currently connected frontend
	// listener as the key.  The value is ignored, as this is only used as
	// a set.
	frontendListeners := make(map[chan []byte]bool)

	// Don't want to add or delete a wallet listener while iterating
	// through each to propigate to every attached wallet.  Use a mutex to
	// prevent this.
	mtx := new(sync.Mutex)

	// Check for listener channels to add or remove from set.
	go func() {
		for {
			select {
			case c := <-addFrontendListener:
				mtx.Lock()
				frontendListeners[c] = true
				mtx.Unlock()
			case c := <-deleteFrontendListener:
				mtx.Lock()
				delete(frontendListeners, c)
				mtx.Unlock()
			}
		}
	}()

	// Duplicate all messages sent across frontendNotificationMaster to each
	// listening wallet.
	for {
		ntfn := <-frontendNotificationMaster
		mtx.Lock()
		for c, _ := range frontendListeners {
			c <- ntfn
		}
		mtx.Unlock()
	}
}

// frontendReqsNotifications is the handler function for websocket
// connections from a btcwallet instance.  It reads messages from wallet and
// sends back replies, as well as notififying wallets of chain updates.
// There can possibly be many of these running, one for each currently
// connected frontend.
func frontendReqsNotifications(ws *websocket.Conn) {
	// Add frontend notification channel to set so this handler receives
	// updates.
	frontendNotification := make(chan []byte)
	addFrontendListener <- frontendNotification
	defer func() {
		deleteFrontendListener <- frontendNotification
	}()

	// jsonMsgs receives JSON messages from the currently connected frontend.
	jsonMsgs := make(chan []byte)

	// Receive messages from websocket and send across jsonMsgs until
	// connection is lost
	go func() {
		for {
			var m []byte
			if err := websocket.Message.Receive(ws, &m); err != nil {
				close(jsonMsgs)
				return
			}
			jsonMsgs <- m
		}
	}()

	for {
		select {
		case <-btcdDisconnected:
			var idStr interface{} = "btcwallet:btcddisconnected"
			r := btcjson.Reply{
				Id: &idStr,
			}
			m, _ := json.Marshal(r)
			websocket.Message.Send(ws, m)
			return
		case m, ok := <-jsonMsgs:
			if !ok {
				// frontend disconnected.
				return
			}
			// Handle JSON message here.
			go ProcessFrontendMsg(frontendNotification, m)
		case ntfn, _ := <-frontendNotification:
			if err := websocket.Message.Send(ws, ntfn); err != nil {
				// Frontend disconnected.
				return
			}
		}
	}
}

// BtcdHandler listens for replies and notifications from btcd over a
// websocket and sends messages that btcwallet does not understand to
// btcd.  Unlike FrontendHandler, exactly one BtcdHandler goroutine runs.
func BtcdHandler(ws *websocket.Conn) {
	disconnected := make(chan int)

	defer func() {
		close(disconnected)
		close(btcdDisconnected)
	}()

	// Listen for replies/notifications from btcd, and decide how to handle them.
	replies := make(chan []byte)
	go func() {
		defer close(replies)
		for {
			select {
			case <-disconnected:
				return
			default:
				var m []byte
				if err := websocket.Message.Receive(ws, &m); err != nil {
					return
				}
				replies <- m
			}
		}
	}()

	// TODO(jrick): hook this up with addresses in wallet.
	// reqTxsForAddress("addr")

	for {
		select {
		case rply, ok := <-replies:
			if !ok {
				// btcd disconnected
				return
			}
			// Handle message here.
			go ProcessBtcdNotificationReply(rply)
		case r := <-btcdMsgs:
			if err := websocket.Message.Send(ws, r); err != nil {
				// btcd disconnected.
				return
			}
		}
	}
}

// ProcessBtcdNotificationReply unmarshalls the JSON notification or
// reply received from btcd and decides how to handle it.  Replies are
// routed back to the frontend who sent the message, and wallet
// notifications are processed by btcwallet, and frontend notifications
// are sent to every connected frontend.
func ProcessBtcdNotificationReply(b []byte) {
	// Check if the json id field was set by btcwallet.
	var routeId uint64
	var origId string

	var m map[string]interface{}
	json.Unmarshal(b, &m)
	idStr, ok := m["id"].(string)
	if !ok {
		// btcd should only ever be sending JSON messages with a string in
		// the id field.  Log the error and drop the message.
		log.Error("Unable to process btcd notification or reply.")
		return
	}

	n, _ := fmt.Sscanf(idStr, "btcwallet(%d)-%s", &routeId, &origId)
	if n == 1 {
		// Request originated from btcwallet. Run and remove correct
		// handler.
		replyHandlers.Lock()
		f := replyHandlers.m[routeId]
		replyHandlers.Unlock()
		if f != nil {
			go func() {
				if f(m["result"]) {
					replyHandlers.Lock()
					delete(replyHandlers.m, routeId)
					replyHandlers.Unlock()
				}
			}()
		}
	} else if n == 2 {
		// Attempt to route btcd reply to correct frontend.
		replyRouter.Lock()
		c := replyRouter.m[routeId]
		if c != nil {
			delete(replyRouter.m, routeId)
		} else {
			// Can't route to a frontend, drop reply.
			log.Info("Unable to route btcd reply to frontend. Dropping.")
			return
		}
		replyRouter.Unlock()

		// Convert string back to number if possible.
		var origIdNum float64
		n, _ := fmt.Sscanf(origId, "%f", &origIdNum)
		if n == 1 {
			m["id"] = origIdNum
		} else {
			m["id"] = origId
		}

		b, err := json.Marshal(m)
		if err != nil {
			log.Error("Error marshalling btcd reply. Dropping.")
			return
		}
		c <- b
	} else {
		// btcd notification must either be handled by btcwallet or sent
		// to all frontends if btcwallet can not handle it.
		switch idStr {
		case "btcd:blockconnected":
			result := m["result"].(map[string]interface{})
			hashResult := result["hash"].([]interface{})
			hash := new(btcwire.ShaHash)
			for i, _ := range hash[:] {
				hash[i] = byte(hashResult[i].(float64))
			}
			height := int64(result["height"].(float64))

			// TODO(jrick): update TxStore and UtxoStore with new hash
			var id interface{} = "btcwallet:newblockchainheight"
			m := &btcjson.Reply{
				Result: height,
				Id: &id,
			}
			msg, _ := json.Marshal(m)
			frontendNotificationMaster <- msg

		case "btcd:blockdisconnected":
			// TODO(jrick): rollback txs and utxos from removed block.

		default:
			frontendNotificationMaster <- b
		}
	}
}

// ListenAndServe connects to a running btcd instance over a websocket
// for sending and receiving chain-related messages, failing if the
// connection can not be established.  An additional HTTP server is then
// started to provide websocket connections for any number of btcwallet
// frontends.
func ListenAndServe() error {
	// Attempt to connect to running btcd instance. Bail if it fails.
	btcdws, err := websocket.Dial(
		fmt.Sprintf("ws://localhost:%d/wallet", cfg.BtcdPort),
		"",
		"http://localhost/")
	if err != nil {
		return ConnRefused
	}
	go BtcdHandler(btcdws)

	log.Info("Established connection to btcd.")

	// We'll need to duplicate replies to frontends to each frontend.
	// Replies are sent to frontendReplyMaster, and duplicated to each valid
	// channel in frontendReplySet.  This runs a goroutine to duplicate
	// requests for each channel in the set.
	go frontendListenerDuplicator()

	// XXX(jrick): We need some sort of authentication before websocket
	// connections are allowed, and perhaps TLS on the server as well.
	http.Handle("/frontend", websocket.Handler(frontendReqsNotifications))
	if err := http.ListenAndServe(fmt.Sprintf(":%d", cfg.SvrPort), nil); err != nil {
		return err
	}

	return nil
}

func reqTxsForAddress(addr string) {
	for i := 0; i < 10; i++ {
		seq.Lock()
		n := seq.n
		seq.n++
		seq.Unlock()

		id := fmt.Sprintf("btcwallet(%v)", n)
		msg, err := btcjson.CreateMessageWithId("getblockhash", id, i)
		if err != nil {
			fmt.Println(msg)
			panic(err)
		}

		replyHandlers.Lock()
		replyHandlers.m[n] = func(result interface{}) bool {
			fmt.Println(result)
			return true
		}
		replyHandlers.Unlock()

		btcdMsgs <- msg
	}

	seq.Lock()
	n := seq.n
	seq.n++
	seq.Unlock()

	m := &btcjson.Message{
		Jsonrpc: "",
		Id:      fmt.Sprintf("btcwallet(%v)", n),
		Method:  "rescanforutxo",
		Params: []interface{}{
			"17XhEvq9Nahdj7Xe1nv6oRe1tEmaHUuynH",
		},
	}
	msg, err := json.Marshal(m)
	if err != nil {
		panic(err)
	}

	replyHandlers.Lock()
	replyHandlers.m[n] = func(result interface{}) bool {
		fmt.Println("result:", result)
		return result == nil
	}
	replyHandlers.Unlock()

	btcdMsgs <- msg
}
<commit_msg>I even did the go fmt this time but forgot to stage it.<commit_after>/*
 * Copyright (c) 2013 Conformal Systems LLC <info@conformal.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package main

import (
	"code.google.com/p/go.net/websocket"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/conformal/btcjson"
	"github.com/conformal/btcwire"
	"net/http"
	"sync"
)

var (
	ConnRefused = errors.New("Connection refused")

	// Channel to close to notify that connection to btcd has been lost.
	btcdDisconnected = make(chan int)

	// Channel to send messages btcwallet does not understand to btcd.
	btcdMsgs = make(chan []byte, 100)

	// Adds a frontend listener channel
	addFrontendListener = make(chan (chan []byte))

	// Removes a frontend listener channel
	deleteFrontendListener = make(chan (chan []byte))

	// Messages sent to this channel are sent to each connected frontend.
	frontendNotificationMaster = make(chan []byte, 100)

	replyHandlers = struct {
		sync.Mutex
		m map[uint64]func(interface{}) bool
	}{
		m: make(map[uint64]func(interface{}) bool),
	}
)

// frontendListenerDuplicator listens for new wallet listener channels
// and duplicates messages sent to frontendNotificationMaster to all
// connected listeners.
func frontendListenerDuplicator() {
	// frontendListeners is a map holding each currently connected frontend
	// listener as the key.  The value is ignored, as this is only used as
	// a set.
	frontendListeners := make(map[chan []byte]bool)

	// Don't want to add or delete a wallet listener while iterating
	// through each to propigate to every attached wallet.  Use a mutex to
	// prevent this.
	mtx := new(sync.Mutex)

	// Check for listener channels to add or remove from set.
	go func() {
		for {
			select {
			case c := <-addFrontendListener:
				mtx.Lock()
				frontendListeners[c] = true
				mtx.Unlock()
			case c := <-deleteFrontendListener:
				mtx.Lock()
				delete(frontendListeners, c)
				mtx.Unlock()
			}
		}
	}()

	// Duplicate all messages sent across frontendNotificationMaster to each
	// listening wallet.
	for {
		ntfn := <-frontendNotificationMaster
		mtx.Lock()
		for c, _ := range frontendListeners {
			c <- ntfn
		}
		mtx.Unlock()
	}
}

// frontendReqsNotifications is the handler function for websocket
// connections from a btcwallet instance.  It reads messages from wallet and
// sends back replies, as well as notififying wallets of chain updates.
// There can possibly be many of these running, one for each currently
// connected frontend.
func frontendReqsNotifications(ws *websocket.Conn) {
	// Add frontend notification channel to set so this handler receives
	// updates.
	frontendNotification := make(chan []byte)
	addFrontendListener <- frontendNotification
	defer func() {
		deleteFrontendListener <- frontendNotification
	}()

	// jsonMsgs receives JSON messages from the currently connected frontend.
	jsonMsgs := make(chan []byte)

	// Receive messages from websocket and send across jsonMsgs until
	// connection is lost
	go func() {
		for {
			var m []byte
			if err := websocket.Message.Receive(ws, &m); err != nil {
				close(jsonMsgs)
				return
			}
			jsonMsgs <- m
		}
	}()

	for {
		select {
		case <-btcdDisconnected:
			var idStr interface{} = "btcwallet:btcddisconnected"
			r := btcjson.Reply{
				Id: &idStr,
			}
			m, _ := json.Marshal(r)
			websocket.Message.Send(ws, m)
			return
		case m, ok := <-jsonMsgs:
			if !ok {
				// frontend disconnected.
				return
			}
			// Handle JSON message here.
			go ProcessFrontendMsg(frontendNotification, m)
		case ntfn, _ := <-frontendNotification:
			if err := websocket.Message.Send(ws, ntfn); err != nil {
				// Frontend disconnected.
				return
			}
		}
	}
}

// BtcdHandler listens for replies and notifications from btcd over a
// websocket and sends messages that btcwallet does not understand to
// btcd.  Unlike FrontendHandler, exactly one BtcdHandler goroutine runs.
func BtcdHandler(ws *websocket.Conn) {
	disconnected := make(chan int)

	defer func() {
		close(disconnected)
		close(btcdDisconnected)
	}()

	// Listen for replies/notifications from btcd, and decide how to handle them.
	replies := make(chan []byte)
	go func() {
		defer close(replies)
		for {
			select {
			case <-disconnected:
				return
			default:
				var m []byte
				if err := websocket.Message.Receive(ws, &m); err != nil {
					return
				}
				replies <- m
			}
		}
	}()

	// TODO(jrick): hook this up with addresses in wallet.
	// reqTxsForAddress("addr")

	for {
		select {
		case rply, ok := <-replies:
			if !ok {
				// btcd disconnected
				return
			}
			// Handle message here.
			go ProcessBtcdNotificationReply(rply)
		case r := <-btcdMsgs:
			if err := websocket.Message.Send(ws, r); err != nil {
				// btcd disconnected.
				return
			}
		}
	}
}

// ProcessBtcdNotificationReply unmarshalls the JSON notification or
// reply received from btcd and decides how to handle it.  Replies are
// routed back to the frontend who sent the message, and wallet
// notifications are processed by btcwallet, and frontend notifications
// are sent to every connected frontend.
func ProcessBtcdNotificationReply(b []byte) {
	// Check if the json id field was set by btcwallet.
	var routeId uint64
	var origId string

	var m map[string]interface{}
	json.Unmarshal(b, &m)
	idStr, ok := m["id"].(string)
	if !ok {
		// btcd should only ever be sending JSON messages with a string in
		// the id field.  Log the error and drop the message.
		log.Error("Unable to process btcd notification or reply.")
		return
	}

	n, _ := fmt.Sscanf(idStr, "btcwallet(%d)-%s", &routeId, &origId)
	if n == 1 {
		// Request originated from btcwallet. Run and remove correct
		// handler.
		replyHandlers.Lock()
		f := replyHandlers.m[routeId]
		replyHandlers.Unlock()
		if f != nil {
			go func() {
				if f(m["result"]) {
					replyHandlers.Lock()
					delete(replyHandlers.m, routeId)
					replyHandlers.Unlock()
				}
			}()
		}
	} else if n == 2 {
		// Attempt to route btcd reply to correct frontend.
		replyRouter.Lock()
		c := replyRouter.m[routeId]
		if c != nil {
			delete(replyRouter.m, routeId)
		} else {
			// Can't route to a frontend, drop reply.
			log.Info("Unable to route btcd reply to frontend. Dropping.")
			return
		}
		replyRouter.Unlock()

		// Convert string back to number if possible.
		var origIdNum float64
		n, _ := fmt.Sscanf(origId, "%f", &origIdNum)
		if n == 1 {
			m["id"] = origIdNum
		} else {
			m["id"] = origId
		}

		b, err := json.Marshal(m)
		if err != nil {
			log.Error("Error marshalling btcd reply. Dropping.")
			return
		}
		c <- b
	} else {
		// btcd notification must either be handled by btcwallet or sent
		// to all frontends if btcwallet can not handle it.
		switch idStr {
		case "btcd:blockconnected":
			result := m["result"].(map[string]interface{})
			hashResult := result["hash"].([]interface{})
			hash := new(btcwire.ShaHash)
			for i, _ := range hash[:] {
				hash[i] = byte(hashResult[i].(float64))
			}
			height := int64(result["height"].(float64))

			// TODO(jrick): update TxStore and UtxoStore with new hash
			var id interface{} = "btcwallet:newblockchainheight"
			m := &btcjson.Reply{
				Result: height,
				Id:     &id,
			}
			msg, _ := json.Marshal(m)
			frontendNotificationMaster <- msg

		case "btcd:blockdisconnected":
			// TODO(jrick): rollback txs and utxos from removed block.

		default:
			frontendNotificationMaster <- b
		}
	}
}

// ListenAndServe connects to a running btcd instance over a websocket
// for sending and receiving chain-related messages, failing if the
// connection can not be established.  An additional HTTP server is then
// started to provide websocket connections for any number of btcwallet
// frontends.
func ListenAndServe() error {
	// Attempt to connect to running btcd instance. Bail if it fails.
	btcdws, err := websocket.Dial(
		fmt.Sprintf("ws://localhost:%d/wallet", cfg.BtcdPort),
		"",
		"http://localhost/")
	if err != nil {
		return ConnRefused
	}
	go BtcdHandler(btcdws)

	log.Info("Established connection to btcd.")

	// We'll need to duplicate replies to frontends to each frontend.
	// Replies are sent to frontendReplyMaster, and duplicated to each valid
	// channel in frontendReplySet.  This runs a goroutine to duplicate
	// requests for each channel in the set.
	go frontendListenerDuplicator()

	// XXX(jrick): We need some sort of authentication before websocket
	// connections are allowed, and perhaps TLS on the server as well.
	http.Handle("/frontend", websocket.Handler(frontendReqsNotifications))
	if err := http.ListenAndServe(fmt.Sprintf(":%d", cfg.SvrPort), nil); err != nil {
		return err
	}

	return nil
}

func reqTxsForAddress(addr string) {
	for i := 0; i < 10; i++ {
		seq.Lock()
		n := seq.n
		seq.n++
		seq.Unlock()

		id := fmt.Sprintf("btcwallet(%v)", n)
		msg, err := btcjson.CreateMessageWithId("getblockhash", id, i)
		if err != nil {
			fmt.Println(msg)
			panic(err)
		}

		replyHandlers.Lock()
		replyHandlers.m[n] = func(result interface{}) bool {
			fmt.Println(result)
			return true
		}
		replyHandlers.Unlock()

		btcdMsgs <- msg
	}

	seq.Lock()
	n := seq.n
	seq.n++
	seq.Unlock()

	m := &btcjson.Message{
		Jsonrpc: "",
		Id:      fmt.Sprintf("btcwallet(%v)", n),
		Method:  "rescanforutxo",
		Params: []interface{}{
			"17XhEvq9Nahdj7Xe1nv6oRe1tEmaHUuynH",
		},
	}
	msg, err := json.Marshal(m)
	if err != nil {
		panic(err)
	}

	replyHandlers.Lock()
	replyHandlers.m[n] = func(result interface{}) bool {
		fmt.Println("result:", result)
		return result == nil
	}
	replyHandlers.Unlock()

	btcdMsgs <- msg
}
<|endoftext|>


<commit_before>// Package docker provides a Pipe that creates and pushes a Docker image
package docker

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/alecthomas/template"
	"github.com/apex/log"
	"github.com/goreleaser/goreleaser/config"
	"github.com/goreleaser/goreleaser/context"
	"github.com/goreleaser/goreleaser/pipeline"
	"github.com/pkg/errors"
)

// ErrNoDocker is shown when docker cannot be found in $PATH
var ErrNoDocker = errors.New("docker not present in $PATH")

// Pipe for docker
type Pipe struct{}

func (Pipe) String() string {
	return "creating Docker images"
}

// Run the pipe
func (Pipe) Run(ctx *context.Context) error {
	if len(ctx.Config.Dockers) == 0 || ctx.Config.Dockers[0].Image == "" {
		return pipeline.Skip("docker section is not configured")
	}
	_, err := exec.LookPath("docker")
	if err != nil {
		return ErrNoDocker
	}
	return doRun(ctx)
}

// Default sets the pipe defaults
func (Pipe) Default(ctx *context.Context) error {
	for i := range ctx.Config.Dockers {
		if ctx.Config.Dockers[i].TagTemplate == "" {
			ctx.Config.Dockers[i].TagTemplate = "{{ .Version }}"
		}
	}
	// only set defaults if there is exacly 1 docker setup in the config file.
	if len(ctx.Config.Dockers) != 1 {
		return nil
	}
	if ctx.Config.Dockers[0].Goos == "" {
		ctx.Config.Dockers[0].Goos = "linux"
	}
	if ctx.Config.Dockers[0].Goarch == "" {
		ctx.Config.Dockers[0].Goarch = "amd64"
	}
	if ctx.Config.Dockers[0].Binary == "" {
		ctx.Config.Dockers[0].Binary = ctx.Config.Builds[0].Binary
	}
	if ctx.Config.Dockers[0].Dockerfile == "" {
		ctx.Config.Dockers[0].Dockerfile = "Dockerfile"
	}
	return nil
}

func doRun(ctx *context.Context) error {
	for _, docker := range ctx.Config.Dockers {
		var imagePlatform = docker.Goos + docker.Goarch + docker.Goarm
		for platform, groups := range ctx.Binaries {
			if platform != imagePlatform {
				continue
			}
			for folder, binaries := range groups {
				for _, binary := range binaries {
					if binary.Name != docker.Binary {
						continue
					}
					var err = process(ctx, folder, docker, binary)
					if err != nil && !pipeline.IsSkip(err) {
						return err
					}
				}
			}
		}
	}
	return nil
}

func tagName(ctx *context.Context, docker config.Docker) (string, error) {
	var out bytes.Buffer
	t, err := template.New("tag").Parse(docker.TagTemplate)
	if err != nil {
		return "", err
	}
	data := struct {
		Version, Tag string
	}{
		Version: ctx.Version,
		Tag:     ctx.Git.CurrentTag,
	}
	err = t.Execute(&out, data)
	return out.String(), err
}

func process(ctx *context.Context, folder string, docker config.Docker, binary context.Binary) error {
	var root = filepath.Join(ctx.Config.Dist, folder)
	var dockerfile = filepath.Join(root, filepath.Base(docker.Dockerfile))
	tag, err := tagName(ctx, docker)
	if err != nil {
		return err
	}
	var image = fmt.Sprintf("%s:%s", docker.Image, tag)
	var latest = fmt.Sprintf("%s:latest", docker.Image)

	if err := os.Link(docker.Dockerfile, dockerfile); err != nil {
		return errors.Wrap(err, "failed to link dockerfile")
	}
	for _, file := range docker.Files {
		if err := os.Link(file, filepath.Join(root, filepath.Base(file))); err != nil {
			return errors.Wrapf(err, "failed to link extra file '%s'", file)
		}
	}
	if err := dockerBuild(root, dockerfile, image); err != nil {
		return err
	}
	if docker.Latest {
		if err := dockerTag(image, latest); err != nil {
			return err
		}
	}

	return publish(ctx, docker, image, latest)
}

func publish(ctx *context.Context, docker config.Docker, image, latest string) error {
	// TODO: improve this so it can log it to stdout
	if !ctx.Publish {
		return pipeline.Skip("--skip-publish is set")
	}
	if ctx.Config.Release.Draft {
		return pipeline.Skip("release is marked as draft")
	}
	if err := dockerPush(image); err != nil {
		return err
	}
	ctx.AddDocker(image)
	if !docker.Latest {
		return nil
	}
	if err := dockerTag(image, latest); err != nil {
		return err
	}
	return dockerPush(latest)
}

func dockerBuild(root, dockerfile, image string) error {
	log.WithField("image", image).Info("building docker image")
	/* #nosec */
	var cmd = exec.Command("docker", "build", "-f", dockerfile, "-t", image, root)
	log.WithField("cmd", cmd).Debug("executing")
	out, err := cmd.CombinedOutput()
	if err != nil {
		return errors.Wrapf(err, "failed to build docker image: \n%s", string(out))
	}
	log.Debugf("docker build output: \n%s", string(out))
	return nil
}

func dockerTag(image, tag string) error {
	log.WithField("image", image).WithField("tag", tag).Info("tagging docker image")
	/* #nosec */
	var cmd = exec.Command("docker", "tag", image, tag)
	log.WithField("cmd", cmd).Debug("executing")
	out, err := cmd.CombinedOutput()
	if err != nil {
		return errors.Wrapf(err, "failed to tag docker image: \n%s", string(out))
	}
	log.Debugf("docker tag output: \n%s", string(out))
	return nil
}

func dockerPush(image string) error {
	log.WithField("image", image).Info("pushing docker image")
	/* #nosec */
	var cmd = exec.Command("docker", "push", image)
	log.WithField("cmd", cmd).Debug("executing")
	out, err := cmd.CombinedOutput()
	if err != nil {
		return errors.Wrapf(err, "failed to push docker image: \n%s", string(out))
	}
	log.Debugf("docker push output: \n%s", string(out))
	return nil
}
<commit_msg>fix: using the right import<commit_after>// Package docker provides a Pipe that creates and pushes a Docker image
package docker

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"text/template"

	"github.com/apex/log"
	"github.com/goreleaser/goreleaser/config"
	"github.com/goreleaser/goreleaser/context"
	"github.com/goreleaser/goreleaser/pipeline"
	"github.com/pkg/errors"
)

// ErrNoDocker is shown when docker cannot be found in $PATH
var ErrNoDocker = errors.New("docker not present in $PATH")

// Pipe for docker
type Pipe struct{}

func (Pipe) String() string {
	return "creating Docker images"
}

// Run the pipe
func (Pipe) Run(ctx *context.Context) error {
	if len(ctx.Config.Dockers) == 0 || ctx.Config.Dockers[0].Image == "" {
		return pipeline.Skip("docker section is not configured")
	}
	_, err := exec.LookPath("docker")
	if err != nil {
		return ErrNoDocker
	}
	return doRun(ctx)
}

// Default sets the pipe defaults
func (Pipe) Default(ctx *context.Context) error {
	for i := range ctx.Config.Dockers {
		if ctx.Config.Dockers[i].TagTemplate == "" {
			ctx.Config.Dockers[i].TagTemplate = "{{ .Version }}"
		}
	}
	// only set defaults if there is exacly 1 docker setup in the config file.
	if len(ctx.Config.Dockers) != 1 {
		return nil
	}
	if ctx.Config.Dockers[0].Goos == "" {
		ctx.Config.Dockers[0].Goos = "linux"
	}
	if ctx.Config.Dockers[0].Goarch == "" {
		ctx.Config.Dockers[0].Goarch = "amd64"
	}
	if ctx.Config.Dockers[0].Binary == "" {
		ctx.Config.Dockers[0].Binary = ctx.Config.Builds[0].Binary
	}
	if ctx.Config.Dockers[0].Dockerfile == "" {
		ctx.Config.Dockers[0].Dockerfile = "Dockerfile"
	}
	return nil
}

func doRun(ctx *context.Context) error {
	for _, docker := range ctx.Config.Dockers {
		var imagePlatform = docker.Goos + docker.Goarch + docker.Goarm
		for platform, groups := range ctx.Binaries {
			if platform != imagePlatform {
				continue
			}
			for folder, binaries := range groups {
				for _, binary := range binaries {
					if binary.Name != docker.Binary {
						continue
					}
					var err = process(ctx, folder, docker, binary)
					if err != nil && !pipeline.IsSkip(err) {
						return err
					}
				}
			}
		}
	}
	return nil
}

func tagName(ctx *context.Context, docker config.Docker) (string, error) {
	var out bytes.Buffer
	t, err := template.New("tag").Parse(docker.TagTemplate)
	if err != nil {
		return "", err
	}
	data := struct {
		Version, Tag string
	}{
		Version: ctx.Version,
		Tag:     ctx.Git.CurrentTag,
	}
	err = t.Execute(&out, data)
	return out.String(), err
}

func process(ctx *context.Context, folder string, docker config.Docker, binary context.Binary) error {
	var root = filepath.Join(ctx.Config.Dist, folder)
	var dockerfile = filepath.Join(root, filepath.Base(docker.Dockerfile))
	tag, err := tagName(ctx, docker)
	if err != nil {
		return err
	}
	var image = fmt.Sprintf("%s:%s", docker.Image, tag)
	var latest = fmt.Sprintf("%s:latest", docker.Image)

	if err := os.Link(docker.Dockerfile, dockerfile); err != nil {
		return errors.Wrap(err, "failed to link dockerfile")
	}
	for _, file := range docker.Files {
		if err := os.Link(file, filepath.Join(root, filepath.Base(file))); err != nil {
			return errors.Wrapf(err, "failed to link extra file '%s'", file)
		}
	}
	if err := dockerBuild(root, dockerfile, image); err != nil {
		return err
	}
	if docker.Latest {
		if err := dockerTag(image, latest); err != nil {
			return err
		}
	}

	return publish(ctx, docker, image, latest)
}

func publish(ctx *context.Context, docker config.Docker, image, latest string) error {
	// TODO: improve this so it can log it to stdout
	if !ctx.Publish {
		return pipeline.Skip("--skip-publish is set")
	}
	if ctx.Config.Release.Draft {
		return pipeline.Skip("release is marked as draft")
	}
	if err := dockerPush(image); err != nil {
		return err
	}
	ctx.AddDocker(image)
	if !docker.Latest {
		return nil
	}
	if err := dockerTag(image, latest); err != nil {
		return err
	}
	return dockerPush(latest)
}

func dockerBuild(root, dockerfile, image string) error {
	log.WithField("image", image).Info("building docker image")
	/* #nosec */
	var cmd = exec.Command("docker", "build", "-f", dockerfile, "-t", image, root)
	log.WithField("cmd", cmd).Debug("executing")
	out, err := cmd.CombinedOutput()
	if err != nil {
		return errors.Wrapf(err, "failed to build docker image: \n%s", string(out))
	}
	log.Debugf("docker build output: \n%s", string(out))
	return nil
}

func dockerTag(image, tag string) error {
	log.WithField("image", image).WithField("tag", tag).Info("tagging docker image")
	/* #nosec */
	var cmd = exec.Command("docker", "tag", image, tag)
	log.WithField("cmd", cmd).Debug("executing")
	out, err := cmd.CombinedOutput()
	if err != nil {
		return errors.Wrapf(err, "failed to tag docker image: \n%s", string(out))
	}
	log.Debugf("docker tag output: \n%s", string(out))
	return nil
}

func dockerPush(image string) error {
	log.WithField("image", image).Info("pushing docker image")
	/* #nosec */
	var cmd = exec.Command("docker", "push", image)
	log.WithField("cmd", cmd).Debug("executing")
	out, err := cmd.CombinedOutput()
	if err != nil {
		return errors.Wrapf(err, "failed to push docker image: \n%s", string(out))
	}
	log.Debugf("docker push output: \n%s", string(out))
	return nil
}
<|endoftext|>


<commit_before>package pipeline

import (
	"github.com/PreetamJinka/sflow"

	"github.com/PreetamJinka/cistern/state/metrics"
)

type HostProcessor struct {
	reg      *metrics.HostRegistry
	inbound  chan Message
	outbound chan Message
}

func NewHostProcessor(reg *metrics.HostRegistry) *HostProcessor {
	return &HostProcessor{
		reg:      reg,
		outbound: make(chan Message, 4),
	}
}

func (h *HostProcessor) SetInbound(inbound chan Message) {
	h.inbound = inbound
}

func (h *HostProcessor) Outbound() chan Message {
	return h.outbound
}

func (h *HostProcessor) Process() {
	for message := range h.inbound {
		record := message.Record
		registryKey := message.Source

		switch record.(type) {
		case sflow.HostCpuCounters:
			c := record.(sflow.HostCpuCounters)

			h.reg.Insert(registryKey, "cpu.user", metrics.TypeDerivative, c.CpuUser)
			h.reg.Insert(registryKey, "cpu.nice", metrics.TypeDerivative, c.CpuNice)
			h.reg.Insert(registryKey, "cpu.sys", metrics.TypeDerivative, c.CpuSys)
			h.reg.Insert(registryKey, "cpu.idle", metrics.TypeDerivative, c.CpuIdle)
			h.reg.Insert(registryKey, "cpu.wio", metrics.TypeDerivative, c.CpuWio)
			h.reg.Insert(registryKey, "cpu.intr", metrics.TypeDerivative, c.CpuIntr)
			h.reg.Insert(registryKey, "cpu.softintr", metrics.TypeDerivative, c.CpuSoftIntr)

		case sflow.HostMemoryCounters:
			m := record.(sflow.HostMemoryCounters)

			h.reg.Insert(registryKey, "mem.total", metrics.TypeGauge, m.Total)
			h.reg.Insert(registryKey, "mem.free", metrics.TypeGauge, m.Free)
			h.reg.Insert(registryKey, "mem.shared", metrics.TypeGauge, m.Shared)
			h.reg.Insert(registryKey, "mem.buffers", metrics.TypeGauge, m.Buffers)
			h.reg.Insert(registryKey, "mem.cached", metrics.TypeGauge, m.Cached)
			h.reg.Insert(registryKey, "mem.swap_total", metrics.TypeGauge, m.SwapTotal)
			h.reg.Insert(registryKey, "mem.swap_free", metrics.TypeGauge, m.SwapFree)

			h.reg.Insert(registryKey, "mem.page_in", metrics.TypeDerivative, m.PageIn)
			h.reg.Insert(registryKey, "mem.page_out", metrics.TypeDerivative, m.PageOut)
			h.reg.Insert(registryKey, "mem.swap_in", metrics.TypeDerivative, m.SwapIn)
			h.reg.Insert(registryKey, "mem.swap_out", metrics.TypeDerivative, m.SwapOut)

		case sflow.HostDiskCounters:
			d := record.(sflow.HostDiskCounters)

			h.reg.Insert(registryKey, "disk.total", metrics.TypeGauge, d.Total)
			h.reg.Insert(registryKey, "disk.free", metrics.TypeGauge, d.Free)
			h.reg.Insert(registryKey, "disk.max_used", metrics.TypeGauge, d.MaxUsedPercent)

			h.reg.Insert(registryKey, "disk.reads", metrics.TypeDerivative, d.Reads)
			h.reg.Insert(registryKey, "disk.bytes_read", metrics.TypeDerivative, d.BytesRead)
			h.reg.Insert(registryKey, "disk.read_time", metrics.TypeDerivative, d.ReadTime)

			h.reg.Insert(registryKey, "disk.writes", metrics.TypeDerivative, d.Writes)
			h.reg.Insert(registryKey, "disk.bytes_written", metrics.TypeDerivative, d.BytesWritten)
			h.reg.Insert(registryKey, "disk.write_time", metrics.TypeDerivative, d.WriteTime)

		case sflow.HostNetCounters:
			n := record.(sflow.HostNetCounters)

			h.reg.Insert(registryKey, "net.bytes_in", metrics.TypeDerivative, n.BytesIn)
			h.reg.Insert(registryKey, "net.packets_in", metrics.TypeDerivative, n.PacketsIn)
			h.reg.Insert(registryKey, "net.errs_in", metrics.TypeDerivative, n.ErrsIn)
			h.reg.Insert(registryKey, "net.drops_in", metrics.TypeDerivative, n.DropsIn)

			h.reg.Insert(registryKey, "net.bytes_out", metrics.TypeDerivative, n.BytesOut)
			h.reg.Insert(registryKey, "net.packets_out", metrics.TypeDerivative, n.PacketsOut)
			h.reg.Insert(registryKey, "net.errs_out", metrics.TypeDerivative, n.ErrsOut)
			h.reg.Insert(registryKey, "net.drops_out", metrics.TypeDerivative, n.DropsOut)

		default:
			h.outbound <- message
		}
	}
}
<commit_msg>update package path, non-blocking channel send<commit_after>package pipeline

import (
	"github.com/PreetamJinka/cistern/net/sflow"
	"github.com/PreetamJinka/cistern/state/metrics"
)

type HostProcessor struct {
	reg      *metrics.HostRegistry
	inbound  chan Message
	outbound chan Message
}

func NewHostProcessor(reg *metrics.HostRegistry) *HostProcessor {
	return &HostProcessor{
		reg:      reg,
		outbound: make(chan Message, 4),
	}
}

func (h *HostProcessor) SetInbound(inbound chan Message) {
	h.inbound = inbound
}

func (h *HostProcessor) Outbound() chan Message {
	return h.outbound
}

func (h *HostProcessor) Process() {
	for message := range h.inbound {
		record := message.Record
		registryKey := message.Source

		switch record.(type) {
		case sflow.HostCpuCounters:
			c := record.(sflow.HostCpuCounters)

			h.reg.Insert(registryKey, "cpu.user", metrics.TypeDerivative, c.CpuUser)
			h.reg.Insert(registryKey, "cpu.nice", metrics.TypeDerivative, c.CpuNice)
			h.reg.Insert(registryKey, "cpu.sys", metrics.TypeDerivative, c.CpuSys)
			h.reg.Insert(registryKey, "cpu.idle", metrics.TypeDerivative, c.CpuIdle)
			h.reg.Insert(registryKey, "cpu.wio", metrics.TypeDerivative, c.CpuWio)
			h.reg.Insert(registryKey, "cpu.intr", metrics.TypeDerivative, c.CpuIntr)
			h.reg.Insert(registryKey, "cpu.softintr", metrics.TypeDerivative, c.CpuSoftIntr)

		case sflow.HostMemoryCounters:
			m := record.(sflow.HostMemoryCounters)

			h.reg.Insert(registryKey, "mem.total", metrics.TypeGauge, m.Total)
			h.reg.Insert(registryKey, "mem.free", metrics.TypeGauge, m.Free)
			h.reg.Insert(registryKey, "mem.shared", metrics.TypeGauge, m.Shared)
			h.reg.Insert(registryKey, "mem.buffers", metrics.TypeGauge, m.Buffers)
			h.reg.Insert(registryKey, "mem.cached", metrics.TypeGauge, m.Cached)
			h.reg.Insert(registryKey, "mem.swap_total", metrics.TypeGauge, m.SwapTotal)
			h.reg.Insert(registryKey, "mem.swap_free", metrics.TypeGauge, m.SwapFree)

			h.reg.Insert(registryKey, "mem.page_in", metrics.TypeDerivative, m.PageIn)
			h.reg.Insert(registryKey, "mem.page_out", metrics.TypeDerivative, m.PageOut)
			h.reg.Insert(registryKey, "mem.swap_in", metrics.TypeDerivative, m.SwapIn)
			h.reg.Insert(registryKey, "mem.swap_out", metrics.TypeDerivative, m.SwapOut)

		case sflow.HostDiskCounters:
			d := record.(sflow.HostDiskCounters)

			h.reg.Insert(registryKey, "disk.total", metrics.TypeGauge, d.Total)
			h.reg.Insert(registryKey, "disk.free", metrics.TypeGauge, d.Free)
			h.reg.Insert(registryKey, "disk.max_used", metrics.TypeGauge, d.MaxUsedPercent)

			h.reg.Insert(registryKey, "disk.reads", metrics.TypeDerivative, d.Reads)
			h.reg.Insert(registryKey, "disk.bytes_read", metrics.TypeDerivative, d.BytesRead)
			h.reg.Insert(registryKey, "disk.read_time", metrics.TypeDerivative, d.ReadTime)

			h.reg.Insert(registryKey, "disk.writes", metrics.TypeDerivative, d.Writes)
			h.reg.Insert(registryKey, "disk.bytes_written", metrics.TypeDerivative, d.BytesWritten)
			h.reg.Insert(registryKey, "disk.write_time", metrics.TypeDerivative, d.WriteTime)

		case sflow.HostNetCounters:
			n := record.(sflow.HostNetCounters)

			h.reg.Insert(registryKey, "net.bytes_in", metrics.TypeDerivative, n.BytesIn)
			h.reg.Insert(registryKey, "net.packets_in", metrics.TypeDerivative, n.PacketsIn)
			h.reg.Insert(registryKey, "net.errs_in", metrics.TypeDerivative, n.ErrsIn)
			h.reg.Insert(registryKey, "net.drops_in", metrics.TypeDerivative, n.DropsIn)

			h.reg.Insert(registryKey, "net.bytes_out", metrics.TypeDerivative, n.BytesOut)
			h.reg.Insert(registryKey, "net.packets_out", metrics.TypeDerivative, n.PacketsOut)
			h.reg.Insert(registryKey, "net.errs_out", metrics.TypeDerivative, n.ErrsOut)
			h.reg.Insert(registryKey, "net.drops_out", metrics.TypeDerivative, n.DropsOut)

		default:
			select {
			case h.outbound <- message:
			default:
			}
		}
	}
}
<|endoftext|>


<commit_before>// Package som allows to build and train Self-organizing Maps (SOM) in Go
//
// You can create and train SOMs of arbitrary sizes using the provided API.
// The package implements two main SOM training algorithms: sequential and batch.
// Som package allows you to choose different map and training configuration
// paramters that can help you tune the output to discover undelrying data
// features. You can also visualize the trained SOM using umatrix function.

// Package som also provides a handful of  useful functions which can
// be use spearately outside the SOM realm.
package som
<commit_msg>Updated godoc info.<commit_after>// Package som allows to build and train Self-organizing Maps (SOM) in Go
//
// You can create and train SOMs of arbitrary sizes using the provided API.
// The package implements two main SOM training algorithms: sequential and batch.
// Som package allows you to choose different map and training configuration
// paramters that can help you tune the output to discover undelrying data
// features. You can also visualize the trained SOM using umatrix function.
// The package also provides a handful of useful functions which can
// be use spearately outside the SOM realm.
package som
<|endoftext|>


<commit_before>package events

import (
	"testing"
	"time"

	tu "github.com/Cristofori/kmud/testutils"
	"github.com/Cristofori/kmud/types"
)

func Test_EventLoop(t *testing.T) {
	StartEvents()
	StartCombatLoop()

	char := types.NewMockPC()

	eventListener := Register("Test_EventLoop()")

	message := "hey how are yah"
	Broadcast(TellEvent{char, char, message})

	timeout := tu.Timeout(3 * time.Second)

	select {
	case event := <-eventListener.Channel:
		tu.Assert(event.Type() == TellEventType, t, "Didn't get a Tell event back")
		tellEvent := event.(TellEvent)
		tu.Assert(tellEvent.Message == message, t, "Didn't get the right message back:", tellEvent.Message, message)
	case <-timeout:
		tu.Assert(false, t, "Timed out waiting for tell event")
	}
}

func Test_CombatLoop(t *testing.T) {
	char1 := types.NewMockPC()
	char2 := types.NewMockPC()
	char1.RoomId = char2.RoomId

	eventListener1 := Register("Test_CombatLoop() - char1")

	StartFight(char1, char2)

	verifyEvents := func(listener EventListener) {
		timeout := tu.Timeout(4 * time.Second)
		expectedTypes := make(map[EventType]bool)
		expectedTypes[CombatEventType] = true
		expectedTypes[CombatStartEventType] = true

	Loop:
		for {
			select {
			case event := <-listener.Channel:
				if event.Type() != TickEventType {
					tu.Assert(expectedTypes[event.Type()] == true, t, "Unexpected event type:", event.Type())
					delete(expectedTypes, event.Type())
				}
			case <-timeout:
				tu.Assert(false, t, "Timed out waiting for combat event")
				break Loop
			}

			if len(expectedTypes) == 0 {
				break
			}
		}
	}
	verifyEvents(eventListener1)
}
<commit_msg>Fixed the unit test<commit_after>package events

import (
	"testing"
	"time"

	tu "github.com/Cristofori/kmud/testutils"
	"github.com/Cristofori/kmud/types"
)

func Test_EventLoop(t *testing.T) {
	StartEvents()
	StartCombatLoop()

	char := types.NewMockPC()

	eventListener := Register(char)

	message := "hey how are yah"
	Broadcast(TellEvent{char, char, message})

	timeout := tu.Timeout(3 * time.Second)

	select {
	case event := <-eventListener.Channel:
		tu.Assert(event.Type() == TellEventType, t, "Didn't get a Tell event back")
		tellEvent := event.(TellEvent)
		tu.Assert(tellEvent.Message == message, t, "Didn't get the right message back:", tellEvent.Message, message)
	case <-timeout:
		tu.Assert(false, t, "Timed out waiting for tell event")
	}
}

func Test_CombatLoop(t *testing.T) {
	char1 := types.NewMockPC()
	char2 := types.NewMockPC()
	char1.RoomId = char2.RoomId

	eventListener1 := Register(char1)

	StartFight(char1, char2)

	verifyEvents := func(listener *EventListener) {
		timeout := tu.Timeout(4 * time.Second)
		expectedTypes := make(map[EventType]bool)
		expectedTypes[CombatEventType] = true
		expectedTypes[CombatStartEventType] = true

	Loop:
		for {
			select {
			case event := <-listener.Channel:
				if event.Type() != TickEventType {
					tu.Assert(expectedTypes[event.Type()] == true, t, "Unexpected event type:", event.Type())
					delete(expectedTypes, event.Type())
				}
			case <-timeout:
				tu.Assert(false, t, "Timed out waiting for combat event")
				break Loop
			}

			if len(expectedTypes) == 0 {
				break
			}
		}
	}
	verifyEvents(eventListener1)
}
<|endoftext|>