<commit_before>// Copyright 2014 Wandoujia Inc. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package main

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/juju/errors"
	"github.com/ngaut/go-zookeeper/zk"
	"github.com/ngaut/zkhelper"
	"github.com/wandoulabs/codis/pkg/models"

	"sync/atomic"

	stdlog "log"

	"github.com/codegangsta/martini-contrib/binding"
	"github.com/codegangsta/martini-contrib/render"
	"github.com/docopt/docopt-go"
	"github.com/go-martini/martini"
	"github.com/martini-contrib/cors"

	log "github.com/ngaut/logging"
	"github.com/wandoulabs/codis/pkg/utils"
)

func cmdDashboard(argv []string) (err error) {
	usage := `usage: codis-config dashboard [--addr=<address>] [--http-log=<log_file>]

options:
	--addr	listen ip:port, e.g. localhost:12345, :8086, [default: :8086]
	--http-log	http request log [default: request.log ]
`

	args, err := docopt.Parse(usage, argv, true, "", false)
	if err != nil {
		log.Error(err)
		return err
	}
	log.Debug(args)

	logFileName := "request.log"
	if args["--http-log"] != nil {
		logFileName = args["--http-log"].(string)
	}

	addr := ":8086"
	if args["--addr"] != nil {
		addr = args["--addr"].(string)
	}

	runDashboard(addr, logFileName)
	return nil
}

var (
	proxiesSpeed int64
	safeZkConn   zkhelper.Conn
	unsafeZkConn zkhelper.Conn
)

func jsonRet(output map[string]interface{}) (int, string) {
	b, err := json.Marshal(output)
	if err != nil {
		log.Warning(err)
	}
	return 200, string(b)
}

func jsonRetFail(errCode int, msg string) (int, string) {
	return jsonRet(map[string]interface{}{
		"ret": errCode,
		"msg": msg,
	})
}

func jsonRetSucc() (int, string) {
	return jsonRet(map[string]interface{}{
		"ret": 0,
		"msg": "OK",
	})
}

func getAllProxyOps() int64 {
	proxies, err := models.ProxyList(unsafeZkConn, globalEnv.ProductName(), nil)
	if err != nil {
		log.Warning(err)
		return -1
	}

	var total int64
	for _, p := range proxies {
		i, err := p.Ops()
		if err != nil {
			log.Warning(err)
		}
		total += i
	}
	return total
}

// for debug
func getAllProxyDebugVars() map[string]map[string]interface{} {
	proxies, err := models.ProxyList(unsafeZkConn, globalEnv.ProductName(), nil)
	if err != nil {
		log.Warning(err)
		return nil
	}

	ret := make(map[string]map[string]interface{})
	for _, p := range proxies {
		m, err := p.DebugVars()
		if err != nil {
			log.Warning(err)
		}
		ret[p.Id] = m
	}
	return ret
}

func getProxySpeedChan() <-chan int64 {
	c := make(chan int64)
	go func() {
		var lastCnt int64
		for {
			cnt := getAllProxyOps()
			if lastCnt > 0 {
				c <- cnt - lastCnt
			}
			lastCnt = cnt
			time.Sleep(1 * time.Second)
		}
	}()
	return c
}

func pageSlots(r render.Render) {
	r.HTML(200, "slots", nil)
}

func createDashboardNode() error {

	// make sure root dir is exists
	rootDir := fmt.Sprintf("/zk/codis/db_%s", globalEnv.ProductName())
	zkhelper.CreateRecursive(safeZkConn, rootDir, "", 0, zkhelper.DefaultDirACLs())

	zkPath := fmt.Sprintf("%s/dashboard", rootDir)
	// make sure we're the only one dashboard
	if exists, _, _ := safeZkConn.Exists(zkPath); exists {
		data, _, _ := safeZkConn.Get(zkPath)
		return errors.New("dashboard already exists: " + string(data))
	}

	content := fmt.Sprintf(`{"addr": "%v", "pid": %v}`, globalEnv.DashboardAddr(), os.Getpid())
	pathCreated, err := safeZkConn.Create(zkPath, []byte(content),
		zk.FlagEphemeral, zkhelper.DefaultFileACLs())

	log.Info("dashboard node created:", pathCreated, string(content))

	return errors.Trace(err)
}

func releaseDashboardNode() {

	zkPath := fmt.Sprintf("/zk/codis/db_%s/dashboard", globalEnv.ProductName())
	if exists, _, _ := safeZkConn.Exists(zkPath); exists {
		log.Info("removing dashboard node")
		safeZkConn.Delete(zkPath, 0)
	}
}

func runDashboard(addr string, httpLogFile string) {
	log.Info("dashboard listening on addr: ", addr)
	m := martini.Classic()
	f, err := os.OpenFile(httpLogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		Fatal(err)
	}
	defer f.Close()

	m.Map(stdlog.New(f, "[martini]", stdlog.LstdFlags))
	binRoot, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err != nil {
		Fatal(err)
	}

	m.Use(martini.Static(filepath.Join(binRoot, "assets/statics")))
	m.Use(render.Renderer(render.Options{
		Directory:  filepath.Join(binRoot, "assets/template"),
		Extensions: []string{".tmpl", ".html"},
		Charset:    "UTF-8",
		IndentJSON: true,
	}))

	m.Use(cors.Allow(&cors.Options{
		AllowOrigins:     []string{"*"},
		AllowMethods:     []string{"POST", "GET", "DELETE", "PUT"},
		AllowHeaders:     []string{"Origin", "x-requested-with", "Content-Type", "Content-Range", "Content-Disposition", "Content-Description"},
		ExposeHeaders:    []string{"Content-Length"},
		AllowCredentials: false,
	}))

	m.Get("/api/server_groups", apiGetServerGroupList)
	m.Get("/api/overview", apiOverview)

	m.Get("/api/redis/:addr/stat", apiRedisStat)
	m.Get("/api/redis/:addr/:id/slotinfo", apiGetRedisSlotInfo)
	m.Get("/api/redis/group/:group_id/:slot_id/slotinfo", apiGetRedisSlotInfoFromGroupId)

	m.Put("/api/server_groups", binding.Json(models.ServerGroup{}), apiAddServerGroup)
	m.Put("/api/server_group/(?P<id>[0-9]+)/addServer", binding.Json(models.Server{}), apiAddServerToGroup)
	m.Delete("/api/server_group/(?P<id>[0-9]+)", apiRemoveServerGroup)

	m.Put("/api/server_group/(?P<id>[0-9]+)/removeServer", binding.Json(models.Server{}), apiRemoveServerFromGroup)
	m.Get("/api/server_group/(?P<id>[0-9]+)", apiGetServerGroup)
	m.Post("/api/server_group/(?P<id>[0-9]+)/promote", binding.Json(models.Server{}), apiPromoteServer)

	m.Get("/api/migrate/status", apiMigrateStatus)
	m.Get("/api/migrate/tasks", apiGetMigrateTasks)
	m.Delete("/api/migrate/pending_task/:id/remove", apiRemovePendingMigrateTask)
	m.Delete("/api/migrate/task/:id/stop", apiStopMigratingTask)
	m.Post("/api/migrate", binding.Json(MigrateTaskInfo{}), apiDoMigrate)

	m.Post("/api/rebalance", apiRebalance)
	m.Get("/api/rebalance/status", apiRebalanceStatus)

	m.Get("/api/slot/list", apiGetSlots)
	m.Get("/api/slot/:id", apiGetSingleSlot)
	m.Post("/api/slots/init", apiInitSlots)
	m.Get("/api/slots", apiGetSlots)
	m.Post("/api/slot", binding.Json(RangeSetTask{}), apiSlotRangeSet)
	m.Get("/api/proxy/list", apiGetProxyList)
	m.Get("/api/proxy/debug/vars", apiGetProxyDebugVars)
	m.Post("/api/proxy", binding.Json(models.ProxyInfo{}), apiSetProxyStatus)

	m.Get("/api/action/gc", apiActionGC)
	m.Get("/api/force_remove_locks", apiForceRemoveLocks)
	m.Get("/api/remove_fence", apiRemoveFence)

	m.Get("/slots", pageSlots)
	m.Get("/", func(r render.Render) {
		r.Redirect("/admin")
	})
	zkBuilder := utils.NewConnBuilder(globalEnv.NewZkConn)
	safeZkConn = zkBuilder.GetSafeConn()
	unsafeZkConn = zkBuilder.GetUnsafeConn()

	// create temp node in ZK
	if err := createDashboardNode(); err != nil {
		Fatal(err)
	}
	defer releaseDashboardNode()

	// create long live migrate manager
	globalMigrateManager = NewMigrateManager(safeZkConn, globalEnv.ProductName(), preMigrateCheck)
	defer globalMigrateManager.removeNode()

	go func() {
		c := getProxySpeedChan()
		for {
			atomic.StoreInt64(&proxiesSpeed, <-c)
		}
	}()

	m.RunOnAddr(addr)
}
<commit_msg>make dashboard on zk permanent<commit_after>// Copyright 2014 Wandoujia Inc. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package main

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/juju/errors"
	"github.com/ngaut/go-zookeeper/zk"
	"github.com/ngaut/zkhelper"
	"github.com/wandoulabs/codis/pkg/models"

	"sync/atomic"

	stdlog "log"

	"github.com/codegangsta/martini-contrib/binding"
	"github.com/codegangsta/martini-contrib/render"
	"github.com/docopt/docopt-go"
	"github.com/go-martini/martini"
	"github.com/martini-contrib/cors"

	log "github.com/ngaut/logging"
	"github.com/wandoulabs/codis/pkg/utils"
)

func cmdDashboard(argv []string) (err error) {
	usage := `usage: codis-config dashboard [--addr=<address>] [--http-log=<log_file>]

options:
	--addr	listen ip:port, e.g. localhost:12345, :8086, [default: :8086]
	--http-log	http request log [default: request.log ]
`

	args, err := docopt.Parse(usage, argv, true, "", false)
	if err != nil {
		log.Error(err)
		return err
	}
	log.Debug(args)

	logFileName := "request.log"
	if args["--http-log"] != nil {
		logFileName = args["--http-log"].(string)
	}

	addr := ":8086"
	if args["--addr"] != nil {
		addr = args["--addr"].(string)
	}

	runDashboard(addr, logFileName)
	return nil
}

var (
	proxiesSpeed int64
	safeZkConn   zkhelper.Conn
	unsafeZkConn zkhelper.Conn
)

func jsonRet(output map[string]interface{}) (int, string) {
	b, err := json.Marshal(output)
	if err != nil {
		log.Warning(err)
	}
	return 200, string(b)
}

func jsonRetFail(errCode int, msg string) (int, string) {
	return jsonRet(map[string]interface{}{
		"ret": errCode,
		"msg": msg,
	})
}

func jsonRetSucc() (int, string) {
	return jsonRet(map[string]interface{}{
		"ret": 0,
		"msg": "OK",
	})
}

func getAllProxyOps() int64 {
	proxies, err := models.ProxyList(unsafeZkConn, globalEnv.ProductName(), nil)
	if err != nil {
		log.Warning(err)
		return -1
	}

	var total int64
	for _, p := range proxies {
		i, err := p.Ops()
		if err != nil {
			log.Warning(err)
		}
		total += i
	}
	return total
}

// for debug
func getAllProxyDebugVars() map[string]map[string]interface{} {
	proxies, err := models.ProxyList(unsafeZkConn, globalEnv.ProductName(), nil)
	if err != nil {
		log.Warning(err)
		return nil
	}

	ret := make(map[string]map[string]interface{})
	for _, p := range proxies {
		m, err := p.DebugVars()
		if err != nil {
			log.Warning(err)
		}
		ret[p.Id] = m
	}
	return ret
}

func getProxySpeedChan() <-chan int64 {
	c := make(chan int64)
	go func() {
		var lastCnt int64
		for {
			cnt := getAllProxyOps()
			if lastCnt > 0 {
				c <- cnt - lastCnt
			}
			lastCnt = cnt
			time.Sleep(1 * time.Second)
		}
	}()
	return c
}

func pageSlots(r render.Render) {
	r.HTML(200, "slots", nil)
}

func createDashboardNode() error {

	// make sure root dir is exists
	rootDir := fmt.Sprintf("/zk/codis/db_%s", globalEnv.ProductName())
	zkhelper.CreateRecursive(safeZkConn, rootDir, "", 0, zkhelper.DefaultDirACLs())

	zkPath := fmt.Sprintf("%s/dashboard", rootDir)
	// make sure we're the only one dashboard
	if exists, _, _ := safeZkConn.Exists(zkPath); exists {
		data, _, _ := safeZkConn.Get(zkPath)
		return errors.New("dashboard already exists: " + string(data))
	}

	content := fmt.Sprintf(`{"addr": "%v", "pid": %v}`, globalEnv.DashboardAddr(), os.Getpid())
	pathCreated, err := safeZkConn.Create(zkPath, []byte(content), 0, zkhelper.DefaultFileACLs())

	log.Info("dashboard node created:", pathCreated, string(content))

	return errors.Trace(err)
}

func releaseDashboardNode() {

	zkPath := fmt.Sprintf("/zk/codis/db_%s/dashboard", globalEnv.ProductName())
	if exists, _, _ := safeZkConn.Exists(zkPath); exists {
		log.Info("removing dashboard node")
		safeZkConn.Delete(zkPath, 0)
	}
}

func runDashboard(addr string, httpLogFile string) {
	log.Info("dashboard listening on addr: ", addr)
	m := martini.Classic()
	f, err := os.OpenFile(httpLogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		Fatal(err)
	}
	defer f.Close()

	m.Map(stdlog.New(f, "[martini]", stdlog.LstdFlags))
	binRoot, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err != nil {
		Fatal(err)
	}

	m.Use(martini.Static(filepath.Join(binRoot, "assets/statics")))
	m.Use(render.Renderer(render.Options{
		Directory:  filepath.Join(binRoot, "assets/template"),
		Extensions: []string{".tmpl", ".html"},
		Charset:    "UTF-8",
		IndentJSON: true,
	}))

	m.Use(cors.Allow(&cors.Options{
		AllowOrigins:     []string{"*"},
		AllowMethods:     []string{"POST", "GET", "DELETE", "PUT"},
		AllowHeaders:     []string{"Origin", "x-requested-with", "Content-Type", "Content-Range", "Content-Disposition", "Content-Description"},
		ExposeHeaders:    []string{"Content-Length"},
		AllowCredentials: false,
	}))

	m.Get("/api/server_groups", apiGetServerGroupList)
	m.Get("/api/overview", apiOverview)

	m.Get("/api/redis/:addr/stat", apiRedisStat)
	m.Get("/api/redis/:addr/:id/slotinfo", apiGetRedisSlotInfo)
	m.Get("/api/redis/group/:group_id/:slot_id/slotinfo", apiGetRedisSlotInfoFromGroupId)

	m.Put("/api/server_groups", binding.Json(models.ServerGroup{}), apiAddServerGroup)
	m.Put("/api/server_group/(?P<id>[0-9]+)/addServer", binding.Json(models.Server{}), apiAddServerToGroup)
	m.Delete("/api/server_group/(?P<id>[0-9]+)", apiRemoveServerGroup)

	m.Put("/api/server_group/(?P<id>[0-9]+)/removeServer", binding.Json(models.Server{}), apiRemoveServerFromGroup)
	m.Get("/api/server_group/(?P<id>[0-9]+)", apiGetServerGroup)
	m.Post("/api/server_group/(?P<id>[0-9]+)/promote", binding.Json(models.Server{}), apiPromoteServer)

	m.Get("/api/migrate/status", apiMigrateStatus)
	m.Get("/api/migrate/tasks", apiGetMigrateTasks)
	m.Delete("/api/migrate/pending_task/:id/remove", apiRemovePendingMigrateTask)
	m.Delete("/api/migrate/task/:id/stop", apiStopMigratingTask)
	m.Post("/api/migrate", binding.Json(MigrateTaskInfo{}), apiDoMigrate)

	m.Post("/api/rebalance", apiRebalance)
	m.Get("/api/rebalance/status", apiRebalanceStatus)

	m.Get("/api/slot/list", apiGetSlots)
	m.Get("/api/slot/:id", apiGetSingleSlot)
	m.Post("/api/slots/init", apiInitSlots)
	m.Get("/api/slots", apiGetSlots)
	m.Post("/api/slot", binding.Json(RangeSetTask{}), apiSlotRangeSet)
	m.Get("/api/proxy/list", apiGetProxyList)
	m.Get("/api/proxy/debug/vars", apiGetProxyDebugVars)
	m.Post("/api/proxy", binding.Json(models.ProxyInfo{}), apiSetProxyStatus)

	m.Get("/api/action/gc", apiActionGC)
	m.Get("/api/force_remove_locks", apiForceRemoveLocks)
	m.Get("/api/remove_fence", apiRemoveFence)

	m.Get("/slots", pageSlots)
	m.Get("/", func(r render.Render) {
		r.Redirect("/admin")
	})
	zkBuilder := utils.NewConnBuilder(globalEnv.NewZkConn)
	safeZkConn = zkBuilder.GetSafeConn()
	unsafeZkConn = zkBuilder.GetUnsafeConn()

	// create temp node in ZK
	if err := createDashboardNode(); err != nil {
		log.Fatal(err) // do not release dashborad node here
	}
	defer releaseDashboardNode()

	// create long live migrate manager
	globalMigrateManager = NewMigrateManager(safeZkConn, globalEnv.ProductName(), preMigrateCheck)
	defer globalMigrateManager.removeNode()

	go func() {
		c := getProxySpeedChan()
		for {
			atomic.StoreInt64(&proxiesSpeed, <-c)
		}
	}()

	m.RunOnAddr(addr)
}
<|endoftext|>


<commit_before>// ccprices prints current currency prices in ledger format.
package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"time"
)

const (
	euroAPI  = "http://data.fixer.io/api/latest"
	xauAPI   = "https://www.quandl.com/api/v3/datasets/LBMA/GOLD.json?limit=1"
	xagAPI   = "https://www.quandl.com/api/v3/datasets/LBMA/SILVER.json?limit=1"
	coinsAPI = "https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest"
)

var (
	// CoinMarketCap API key can be set via environment variable COINMARKETCAP_API_KEY
	coinmarketcap = os.Getenv("COINMARKETCAP_API_KEY")
	// Fixer API key can be set via environment variable FIXER_API_KEY
	fixer = os.Getenv("FIXER_API_KEY")
	// Quandl API key can be set via environment variable QUANDL_API_KEY
	quandl = os.Getenv("QUANDL_API_KEY")
	coins  = []string{
		"Bitcoin",
		"Bitcoin Cash",
		"Bitcoin Gold",
		"Bitcoin SV",
		"Dash",
		"Decred",
		"Ethereum",
		"Grin",
		"Litecoin",
		"NEAR Protocol",
		"Monero",
		"Particl",
		"Tezos",
		"Zcash",
	}
)

type result struct {
	symbol string
	price  float64
}

func httpGetWithWarning(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		warning(fmt.Sprintf("GET %s: %s", url, resp.Status))
		return nil, nil
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return b, err
}

func getEuroExchangeRates(api string) (map[string]interface{}, error) {
	if fixer != "" {
		api += "?access_key=" + fixer
	}
	b, err := httpGetWithWarning(api)
	if err != nil {
		return nil, err
	}
	if b == nil {
		return nil, nil
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn["rates"].(map[string]interface{}), nil
}

func getLBMAPrice(api string, dataIndex int) (float64, error) {
	if quandl != "" {
		api += "&api_key=" + quandl
	}
	b, err := httpGetWithWarning(api)
	if err != nil {
		return 0, err
	}
	if b == nil {
		return 0, nil
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return 0, err
	}
	data := jsn["dataset"].(map[string]interface{})["data"].([]interface{})
	var price float64
	if data[0].([]interface{})[dataIndex] != nil {
		// p.m. price is available
		price = data[0].([]interface{})[dataIndex].(float64)
	} else {
		// p.m. price is not available, use a.m. price instead
		price = data[0].([]interface{})[dataIndex-1].(float64)
	}
	return price, nil
}

func getCoinPrices() ([]interface{}, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", coinsAPI, nil)
	if err != nil {
		return nil, err
	}

	q := url.Values{}
	q.Add("start", "1")
	q.Add("limit", "2000")
	q.Add("convert", "EUR")

	req.Header.Set("Accepts", "application/json")
	req.Header.Add("X-CMC_PRO_API_KEY", coinmarketcap)
	req.URL.RawQuery = q.Encode()

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		warning(fmt.Sprintf("GET %s: %s", coinmarketcap, resp.Status))
		return nil, nil
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn["data"].([]interface{}), nil
}

func warning(warn string) {
	fmt.Fprintf(os.Stderr, "%s: warning: %s\n", os.Args[0], warn)
}

func fatal(err error) {
	fmt.Fprintf(os.Stderr, "%s: error: %s\n", os.Args[0], err)
	os.Exit(1)
}

func main() {
	// get euro exchange rates
	rates, err := getEuroExchangeRates(euroAPI)
	if err != nil {
		fatal(err)
	}
	// get gold price
	xau, err := getLBMAPrice(xauAPI, 6)
	if err != nil {
		fatal(err)
	}
	// get silver price
	xag, err := getLBMAPrice(xagAPI, 3)
	if err != nil {
		fatal(err)
	}
	// get all coin prices
	all, err := getCoinPrices()
	if err != nil {
		fatal(err)
	}
	// construct map of coin names we are interested in
	var (
		names  map[string]struct{}
		prices map[string]*result
	)
	if all != nil {
		names = make(map[string]struct{})
		for _, name := range coins {
			names[name] = struct{}{}
		}
		prices = make(map[string]*result)
		// iterate over all coin informations
		var btc, bch float64
		for _, info := range all {
			coin := info.(map[string]interface{})
			name := coin["name"].(string)
			_, ok := names[name]
			if ok {
				// we are interested in this coin -> store price and symbol
				quote := coin["quote"].(map[string]interface{})
				eur := quote["EUR"].(map[string]interface{})
				p := eur["price"].(float64)
				prices[name] = &result{symbol: coin["symbol"].(string), price: p}
				if coin["symbol"] == "BTC" {
					btc = p
				}
				if coin["symbol"] == "BCH" {
					bch = p
				}
			}
		}
		fmt.Fprintf(os.Stderr, "BCH/BTC ratio: %.2f%%\n", bch*100.0/btc)
	}
	// output all prices
	t := time.Now().Format("2006/01/02 15:04:05")
	if rates != nil {
		fmt.Printf("P %s USD %11.6f EUR\n", t, 1/rates["USD"].(float64))
		fmt.Printf("P %s GBP %11.6f EUR\n", t, 1/rates["GBP"].(float64))
		fmt.Printf("P %s CHF %11.6f EUR\n", t, 1/rates["CHF"].(float64))
		fmt.Printf("P %s CZK %11.6f EUR\n", t, 1/rates["CZK"].(float64))
		fmt.Printf("P %s THB %11.6f EUR\n", t, 1/rates["THB"].(float64))
	}
	if xau != 0 {
		fmt.Printf("P %s XAU %11.6f EUR\n", t, xau)
	}
	if xag != 0 {
		fmt.Printf("P %s XAG %11.6f EUR\n", t, xag)
	}
	if all != nil {
		for _, name := range coins {
			price, ok := prices[name]
			if ok {
				fmt.Printf("P %s %s %11.6f EUR\n", t, price.symbol, price.price)
			} else {
				fmt.Fprintf(os.Stderr, "price for \"%s\" does not exist\n", name)
			}
		}
	}
}
<commit_msg>ccprices: add ESD<commit_after>// ccprices prints current currency prices in ledger format.
package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"time"
)

const (
	euroAPI     = "http://data.fixer.io/api/latest"
	xauAPI      = "https://www.quandl.com/api/v3/datasets/LBMA/GOLD.json?limit=1"
	xagAPI      = "https://www.quandl.com/api/v3/datasets/LBMA/SILVER.json?limit=1"
	coinsAPI    = "https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest"
	esdAPI      = "https://api.coingecko.com/api/v3/simple/token_price/ethereum?contract_addresses=0x36f3fd68e7325a35eb768f1aedaae9ea0689d723&vs_currencies=EUR"
	esdContract = "0x36f3fd68e7325a35eb768f1aedaae9ea0689d723"
)

var (
	// CoinMarketCap API key can be set via environment variable COINMARKETCAP_API_KEY
	coinmarketcap = os.Getenv("COINMARKETCAP_API_KEY")
	// Fixer API key can be set via environment variable FIXER_API_KEY
	fixer = os.Getenv("FIXER_API_KEY")
	// Quandl API key can be set via environment variable QUANDL_API_KEY
	quandl = os.Getenv("QUANDL_API_KEY")
	coins  = []string{
		"Bitcoin",
		"Bitcoin Cash",
		"Bitcoin Gold",
		"Bitcoin SV",
		"Dash",
		"Decred",
		"Ethereum",
		"Grin",
		"Litecoin",
		"NEAR Protocol",
		"Monero",
		"Particl",
		"Tezos",
		"Zcash",
	}
)

type result struct {
	symbol string
	price  float64
}

func httpGetWithWarning(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		warning(fmt.Sprintf("GET %s: %s", url, resp.Status))
		return nil, nil
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	return b, err
}

func getEuroExchangeRates(api string) (map[string]interface{}, error) {
	if fixer != "" {
		api += "?access_key=" + fixer
	}
	b, err := httpGetWithWarning(api)
	if err != nil {
		return nil, err
	}
	if b == nil {
		return nil, nil
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn["rates"].(map[string]interface{}), nil
}

func getLBMAPrice(api string, dataIndex int) (float64, error) {
	if quandl != "" {
		api += "&api_key=" + quandl
	}
	b, err := httpGetWithWarning(api)
	if err != nil {
		return 0, err
	}
	if b == nil {
		return 0, nil
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return 0, err
	}
	data := jsn["dataset"].(map[string]interface{})["data"].([]interface{})
	var price float64
	if data[0].([]interface{})[dataIndex] != nil {
		// p.m. price is available
		price = data[0].([]interface{})[dataIndex].(float64)
	} else {
		// p.m. price is not available, use a.m. price instead
		price = data[0].([]interface{})[dataIndex-1].(float64)
	}
	return price, nil
}

func getESTPrice(api string) (map[string]interface{}, error) {
	b, err := httpGetWithWarning(api)
	if err != nil {
		return nil, err
	}
	if b == nil {
		return nil, nil
	}
	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn[esdContract].(map[string]interface{}), nil
}

func getCoinPrices() ([]interface{}, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", coinsAPI, nil)
	if err != nil {
		return nil, err
	}

	q := url.Values{}
	q.Add("start", "1")
	q.Add("limit", "2000")
	q.Add("convert", "EUR")

	req.Header.Set("Accepts", "application/json")
	req.Header.Add("X-CMC_PRO_API_KEY", coinmarketcap)
	req.URL.RawQuery = q.Encode()

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		warning(fmt.Sprintf("GET %s: %s", coinmarketcap, resp.Status))
		return nil, nil
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	jsn := make(map[string]interface{})
	if err := json.Unmarshal(b, &jsn); err != nil {
		return nil, err
	}
	return jsn["data"].([]interface{}), nil
}

func warning(warn string) {
	fmt.Fprintf(os.Stderr, "%s: warning: %s\n", os.Args[0], warn)
}

func fatal(err error) {
	fmt.Fprintf(os.Stderr, "%s: error: %s\n", os.Args[0], err)
	os.Exit(1)
}

func main() {
	// get euro exchange rates
	rates, err := getEuroExchangeRates(euroAPI)
	if err != nil {
		fatal(err)
	}
	// get gold price
	xau, err := getLBMAPrice(xauAPI, 6)
	if err != nil {
		fatal(err)
	}
	// get silver price
	xag, err := getLBMAPrice(xagAPI, 3)
	if err != nil {
		fatal(err)
	}
	// get all coin prices
	all, err := getCoinPrices()
	if err != nil {
		fatal(err)
	}
	// get ESD price
	esd, err := getESTPrice(esdAPI)
	if err != nil {
		fatal(err)
	}
	// construct map of coin names we are interested in
	var (
		names  map[string]struct{}
		prices map[string]*result
	)
	if all != nil {
		names = make(map[string]struct{})
		for _, name := range coins {
			names[name] = struct{}{}
		}
		prices = make(map[string]*result)
		// iterate over all coin informations
		var btc, bch float64
		for _, info := range all {
			coin := info.(map[string]interface{})
			name := coin["name"].(string)
			_, ok := names[name]
			if ok {
				// we are interested in this coin -> store price and symbol
				quote := coin["quote"].(map[string]interface{})
				eur := quote["EUR"].(map[string]interface{})
				p := eur["price"].(float64)
				prices[name] = &result{symbol: coin["symbol"].(string), price: p}
				if coin["symbol"] == "BTC" {
					btc = p
				}
				if coin["symbol"] == "BCH" {
					bch = p
				}
			}
		}
		fmt.Fprintf(os.Stderr, "BCH/BTC ratio: %.2f%%\n", bch*100.0/btc)
	}
	// output all prices
	t := time.Now().Format("2006/01/02 15:04:05")
	if rates != nil {
		fmt.Printf("P %s USD %11.6f EUR\n", t, 1/rates["USD"].(float64))
		fmt.Printf("P %s GBP %11.6f EUR\n", t, 1/rates["GBP"].(float64))
		fmt.Printf("P %s CHF %11.6f EUR\n", t, 1/rates["CHF"].(float64))
		fmt.Printf("P %s CZK %11.6f EUR\n", t, 1/rates["CZK"].(float64))
		fmt.Printf("P %s THB %11.6f EUR\n", t, 1/rates["THB"].(float64))
	}
	if xau != 0 {
		fmt.Printf("P %s XAU %11.6f EUR\n", t, xau)
	}
	if xag != 0 {
		fmt.Printf("P %s XAG %11.6f EUR\n", t, xag)
	}
	if all != nil {
		for _, name := range coins {
			price, ok := prices[name]
			if ok {
				fmt.Printf("P %s %s %11.6f EUR\n", t, price.symbol, price.price)
			} else {
				fmt.Fprintf(os.Stderr, "price for \"%s\" does not exist\n", name)
			}
		}
	}
	fmt.Printf("P %s ESD %11.6f EUR\n", t, esd["eur"].(float64))
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"time"

	chttpd "github.com/nochso/colourl/http"
	log "github.com/sirupsen/logrus"
	prefixed "github.com/x-cray/logrus-prefixed-formatter"
)

var (
	port    int
	verbose bool
)

var (
	Version   string
	BuildDate string
)

func init() {
	flag.IntVar(&port, "p", 9191, "HTTP listening port")
	flag.BoolVar(&verbose, "v", false, "Enable verbose / debug output")
}

func main() {
	flag.Parse()
	formatter := new(prefixed.TextFormatter)
	formatter.TimestampFormat = "2006-01-02 15:04:05 Z07:00"
	log.SetFormatter(formatter)
	if verbose {
		log.SetLevel(log.DebugLevel)
	}
	log.WithFields(log.Fields{
		"version":    Version,
		"build_date": BuildDate,
	}).Info("colourl-http")
	log.WithFields(log.Fields{
		"port":    port,
		"verbose": verbose,
	}).Info("Starting HTTP server")
	http.Handle("/", logHandler(chttpd.IndexMux().ServeHTTP))
	http.HandleFunc("/svg", logHandler(chttpd.SVGHandler))
	http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
}

func logHandler(fn http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		start := time.Now()
		fn(w, r)
		log.WithFields(log.Fields{
			"duration": time.Now().Sub(start),
			"url":      r.URL,
			"method":   r.Method,
			"remote":   r.RemoteAddr,
		}).Debug("HTTP request")
	}
}
<commit_msg>Use logrus default formatter<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"time"

	chttpd "github.com/nochso/colourl/http"
	log "github.com/sirupsen/logrus"
)

var (
	port    int
	verbose bool
)

var (
	Version   string
	BuildDate string
)

func init() {
	flag.IntVar(&port, "p", 9191, "HTTP listening port")
	flag.BoolVar(&verbose, "v", false, "Enable verbose / debug output")
}

func main() {
	flag.Parse()
	log.SetFormatter(&log.TextFormatter{FullTimestamp: true})
	if verbose {
		log.SetLevel(log.DebugLevel)
	}
	log.WithFields(log.Fields{
		"version":    Version,
		"build_date": BuildDate,
	}).Info("colourl-http")
	log.WithFields(log.Fields{
		"port":    port,
		"verbose": verbose,
	}).Info("Starting HTTP server")
	http.Handle("/", logHandler(chttpd.IndexMux().ServeHTTP))
	http.HandleFunc("/svg", logHandler(chttpd.SVGHandler))
	http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
}

func logHandler(fn http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		start := time.Now()
		fn(w, r)
		log.WithFields(log.Fields{
			"duration": time.Now().Sub(start),
			"url":      r.URL,
			"method":   r.Method,
			"remote":   r.RemoteAddr,
		}).Debug("HTTP request")
	}
}
<|endoftext|>


<commit_before>// Downloads torrents from the command-line.
package main

import (
	"fmt"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"io"
	"io/ioutil"
	"strings"
	"time"
	"bufio"

	_ "github.com/anacrolix/envpprof"
	"github.com/dustin/go-humanize"
	"github.com/jessevdk/go-flags"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/metainfo"
)

// fmt.Fprintf(os.Stderr, "Usage: %s \n", os.Args[0])

func resolvedPeerAddrs(ss []string) (ret []torrent.Peer, err error) {
	for _, s := range ss {
		var addr *net.TCPAddr
		addr, err = net.ResolveTCPAddr("tcp", s)
		if err != nil {
			return
		}
		ret = append(ret, torrent.Peer{
			IP:   addr.IP,
			Port: addr.Port,
		})
	}
	return
}

func bytesCompleted(tc *torrent.Client) (ret int64) {
	for _, t := range tc.Torrents() {
		if t.Info != nil {
			ret += t.BytesCompleted()
		}
	}
	return
}

// Returns an estimate of the total bytes for all torrents.
func totalBytesEstimate(tc *torrent.Client) (ret int64) {
	var noInfo, hadInfo int64
	for _, t := range tc.Torrents() {
		info := t.Info()
		if info == nil {
			noInfo++
			continue
		}
		ret += info.TotalLength()
		hadInfo++
	}
	if hadInfo != 0 {
		// Treat each torrent without info as the average of those with,
		// rounded up.
		ret += (noInfo*ret + hadInfo - 1) / hadInfo
	}
	return
}

func progressLine(tc *torrent.Client) string {
	return fmt.Sprintf("\033[K%s / %s\r", humanize.Bytes(uint64(bytesCompleted(tc))), humanize.Bytes(uint64(totalBytesEstimate(tc))))
}

func dstFileName(picked string) string {
	parts := strings.Split(picked, "/")
	return parts[len(parts)-1]
}

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	var rootGroup struct {
		Client    torrent.Config `group:"Client Options"`
		Seed      bool           `long:"seed" description:"continue seeding torrents after completed"`
		TestPeers []string       `long:"test-peer" description:"address of peer to inject to every torrent"`
		Pick      string         `long:"pick" description:"filename to pick"`
	}
	// Don't pass flags.PrintError because it's inconsistent with printing.
	// https://github.com/jessevdk/go-flags/issues/132
	parser := flags.NewParser(&rootGroup, flags.HelpFlag|flags.PassDoubleDash)
	parser.Usage = "[OPTIONS] (magnet URI or .torrent file path)..."
	posArgs, err := parser.Parse()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Download from the BitTorrent network.\n")
		fmt.Println(err)
		os.Exit(2)
	}
	log.Printf("File to pick: %s", rootGroup.Pick)

	testPeers, err := resolvedPeerAddrs(rootGroup.TestPeers)
	if err != nil {
		log.Fatal(err)
	}

	if len(posArgs) == 0 {
		fmt.Fprintln(os.Stderr, "no torrents specified")
		return
	}

	tmpdir, err := ioutil.TempDir("", "torrent-pick-")
	if err != nil {
		log.Fatal(err)
	}

	defer os.RemoveAll(tmpdir)

	rootGroup.Client.DataDir = tmpdir

	client, err := torrent.NewClient(&rootGroup.Client)
	if err != nil {
		log.Fatalf("error creating client: %s", err)
	}
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		client.WriteStatus(w)
	})
	defer client.Close()

	dstName := dstFileName(rootGroup.Pick)

	f, err := os.Create(dstName)
	if err != nil {
		log.Fatal(err)
	}
	dstWriter := bufio.NewWriter(f)

	done := make(chan struct{})
	for _, arg := range posArgs {
		t := func() torrent.Torrent {
			if strings.HasPrefix(arg, "magnet:") {
				t, err := client.AddMagnet(arg)
				if err != nil {
					log.Fatalf("error adding magnet: %s", err)
				}
				return t
			} else {
				metaInfo, err := metainfo.LoadFromFile(arg)
				if err != nil {
					log.Fatal(err)
				}
				t, err := client.AddTorrent(metaInfo)
				if err != nil {
					log.Fatal(err)
				}
				return t
			}
		}()
		err := t.AddPeers(testPeers)
		if err != nil {
			log.Fatal(err)
		}

		go func() {
			<-t.GotInfo()
			files := t.Files()
			for _, file := range files {
				if file.Path() ==  rootGroup.Pick {

					log.Printf("Downloading file: %s", file.Path())

					srcReader := io.NewSectionReader(t.NewReader(), file.Offset(), file.Length())
					io.Copy(dstWriter, srcReader)
					close(done)
					break
				}
			}
		}()
	}

	ticker := time.NewTicker(time.Second)
waitDone:
	for {
		select {
		case <-done:
			break waitDone
		case <-ticker.C:
			os.Stdout.WriteString(progressLine(client))
		}
	}
	if rootGroup.Seed {
		select {}
	}
}
<commit_msg>cmd/torrent-pick: gofmt and sortimports<commit_after>// Downloads torrents from the command-line.
package main

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"strings"
	"time"

	_ "github.com/anacrolix/envpprof"
	"github.com/dustin/go-humanize"
	"github.com/jessevdk/go-flags"

	"github.com/anacrolix/torrent"
	"github.com/anacrolix/torrent/metainfo"
)

// fmt.Fprintf(os.Stderr, "Usage: %s \n", os.Args[0])

func resolvedPeerAddrs(ss []string) (ret []torrent.Peer, err error) {
	for _, s := range ss {
		var addr *net.TCPAddr
		addr, err = net.ResolveTCPAddr("tcp", s)
		if err != nil {
			return
		}
		ret = append(ret, torrent.Peer{
			IP:   addr.IP,
			Port: addr.Port,
		})
	}
	return
}

func bytesCompleted(tc *torrent.Client) (ret int64) {
	for _, t := range tc.Torrents() {
		if t.Info != nil {
			ret += t.BytesCompleted()
		}
	}
	return
}

// Returns an estimate of the total bytes for all torrents.
func totalBytesEstimate(tc *torrent.Client) (ret int64) {
	var noInfo, hadInfo int64
	for _, t := range tc.Torrents() {
		info := t.Info()
		if info == nil {
			noInfo++
			continue
		}
		ret += info.TotalLength()
		hadInfo++
	}
	if hadInfo != 0 {
		// Treat each torrent without info as the average of those with,
		// rounded up.
		ret += (noInfo*ret + hadInfo - 1) / hadInfo
	}
	return
}

func progressLine(tc *torrent.Client) string {
	return fmt.Sprintf("\033[K%s / %s\r", humanize.Bytes(uint64(bytesCompleted(tc))), humanize.Bytes(uint64(totalBytesEstimate(tc))))
}

func dstFileName(picked string) string {
	parts := strings.Split(picked, "/")
	return parts[len(parts)-1]
}

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	var rootGroup struct {
		Client    torrent.Config `group:"Client Options"`
		Seed      bool           `long:"seed" description:"continue seeding torrents after completed"`
		TestPeers []string       `long:"test-peer" description:"address of peer to inject to every torrent"`
		Pick      string         `long:"pick" description:"filename to pick"`
	}
	// Don't pass flags.PrintError because it's inconsistent with printing.
	// https://github.com/jessevdk/go-flags/issues/132
	parser := flags.NewParser(&rootGroup, flags.HelpFlag|flags.PassDoubleDash)
	parser.Usage = "[OPTIONS] (magnet URI or .torrent file path)..."
	posArgs, err := parser.Parse()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Download from the BitTorrent network.\n")
		fmt.Println(err)
		os.Exit(2)
	}
	log.Printf("File to pick: %s", rootGroup.Pick)

	testPeers, err := resolvedPeerAddrs(rootGroup.TestPeers)
	if err != nil {
		log.Fatal(err)
	}

	if len(posArgs) == 0 {
		fmt.Fprintln(os.Stderr, "no torrents specified")
		return
	}

	tmpdir, err := ioutil.TempDir("", "torrent-pick-")
	if err != nil {
		log.Fatal(err)
	}

	defer os.RemoveAll(tmpdir)

	rootGroup.Client.DataDir = tmpdir

	client, err := torrent.NewClient(&rootGroup.Client)
	if err != nil {
		log.Fatalf("error creating client: %s", err)
	}
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		client.WriteStatus(w)
	})
	defer client.Close()

	dstName := dstFileName(rootGroup.Pick)

	f, err := os.Create(dstName)
	if err != nil {
		log.Fatal(err)
	}
	dstWriter := bufio.NewWriter(f)

	done := make(chan struct{})
	for _, arg := range posArgs {
		t := func() torrent.Torrent {
			if strings.HasPrefix(arg, "magnet:") {
				t, err := client.AddMagnet(arg)
				if err != nil {
					log.Fatalf("error adding magnet: %s", err)
				}
				return t
			} else {
				metaInfo, err := metainfo.LoadFromFile(arg)
				if err != nil {
					log.Fatal(err)
				}
				t, err := client.AddTorrent(metaInfo)
				if err != nil {
					log.Fatal(err)
				}
				return t
			}
		}()
		err := t.AddPeers(testPeers)
		if err != nil {
			log.Fatal(err)
		}

		go func() {
			<-t.GotInfo()
			files := t.Files()
			for _, file := range files {
				if file.Path() == rootGroup.Pick {

					log.Printf("Downloading file: %s", file.Path())

					srcReader := io.NewSectionReader(t.NewReader(), file.Offset(), file.Length())
					io.Copy(dstWriter, srcReader)
					close(done)
					break
				}
			}
		}()
	}

	ticker := time.NewTicker(time.Second)
waitDone:
	for {
		select {
		case <-done:
			break waitDone
		case <-ticker.C:
			os.Stdout.WriteString(progressLine(client))
		}
	}
	if rootGroup.Seed {
		select {}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Upspin-audit provides subcommands for auditing storage consumption.
// It has several subcommands that should be used in a way yet to be
// determined.
package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"

	"upspin.io/config"
	"upspin.io/flags"
	"upspin.io/subcmd"
	"upspin.io/transports"
	"upspin.io/upspin"
	"upspin.io/version"
)

type State struct {
	*subcmd.State
}

const help = `Upspin-audit provides subcommands for auditing storage consumption.
It has subcommands scandir and scanstore to scan the directory and storage servers
and report the storage consumed by those servers.
The set of tools will grow.
`

func main() {
	const name = "audit"

	log.SetFlags(0)
	log.SetPrefix("upspin-audit: ")
	flag.Usage = usage
	flags.ParseArgsInto(flag.CommandLine, os.Args[1:], flags.Client, "version")

	if flags.Version {
		fmt.Fprint(os.Stdout, version.Version())
		os.Exit(2)
	}

	if flag.NArg() < 1 {
		usage()
	}
	s := &State{
		State: subcmd.NewState(name),
	}

	cfg, err := config.FromFile(flags.Config)
	if err != nil {
		s.Exit(err)
	}
	transports.Init(cfg)
	s.State.Init(cfg)

	switch flag.Arg(0) {
	case "scandir":
		s.scanDirectories(flag.Args()[1:])
	case "scanstore":
		s.scanStore(flag.Args()[1:])
	default:
		usage()
	}

	s.ExitNow()
}

func usage() {
	fmt.Fprintln(os.Stderr, help)
	fmt.Fprintln(os.Stderr, "Usage of upspin audit:")
	fmt.Fprintln(os.Stderr, "\tupspin [globalflags] audit <command> [flags] ...")
	fmt.Fprintln(os.Stderr, "\twhere <command> is one of scandir, scanstore")
	flag.PrintDefaults()
	os.Exit(2)
}

// dataDirFlag returns a string pointer bound to a new flag that specifies the data directory.
// Done here so the definition can be common among the commands.
func dataDirFlag(fs *flag.FlagSet) *string {
	var dataDir string
	fs.StringVar(&dataDir, "data", filepath.Join(os.Getenv("HOME"), "upspin", "store"), "`directory` storing scan data")
	return &dataDir
}

// ByteSize provides a way to make numbers format in nice compact form.
// Convert a number to ByteSize and print it using its String method to see
// 2392685154 print as 2.23GB.
type ByteSize float64

const (
	_           = iota // ignore first value by assigning to blank identifier
	KB ByteSize = 1 << (10 * iota)
	MB
	GB
	TB
	PB
	EB
	ZB
	YB
)

func (b ByteSize) String() string {
	switch {
	case b >= YB:
		return fmt.Sprintf("%.2fYB", b/YB)
	case b >= ZB:
		return fmt.Sprintf("%.2fZB", b/ZB)
	case b >= EB:
		return fmt.Sprintf("%.2fEB", b/EB)
	case b >= PB:
		return fmt.Sprintf("%.2fPB", b/PB)
	case b >= TB:
		return fmt.Sprintf("%.2fTB", b/TB)
	case b >= GB:
		return fmt.Sprintf("%.2fGB", b/GB)
	case b >= MB:
		return fmt.Sprintf("%.2fMB", b/MB)
	case b >= KB:
		return fmt.Sprintf("%.2fKB", b/KB)
	}
	return fmt.Sprintf("%.2fB", b)
}

// writeItems sorts and writes a list of reference/size pairs to file.
func (s *State) writeItems(file string, items []upspin.ListRefsItem) {
	sort.Slice(items, func(i, j int) bool { return items[i].Ref < items[j].Ref })

	f, err := os.Create(file)
	if err != nil {
		s.Exit(err)
	}
	defer func() {
		if err := f.Close(); err != nil {
			s.Exit(err)
		}
	}()
	w := bufio.NewWriter(f)
	for _, ri := range items {
		if _, err := fmt.Fprintf(w, "%q %d\n", ri.Ref, ri.Size); err != nil {
			s.Exit(err)
		}
	}
	if err := w.Flush(); err != nil {
		s.Exit(err)
	}
}
<commit_msg>cmd/upspin-audit: change default datadir to $HOME/upspin/audit<commit_after>// Copyright 2017 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Upspin-audit provides subcommands for auditing storage consumption.
// It has several subcommands that should be used in a way yet to be
// determined.
package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"

	"upspin.io/config"
	"upspin.io/flags"
	"upspin.io/subcmd"
	"upspin.io/transports"
	"upspin.io/upspin"
	"upspin.io/version"
)

type State struct {
	*subcmd.State
}

const help = `Upspin-audit provides subcommands for auditing storage consumption.
It has subcommands scandir and scanstore to scan the directory and storage servers
and report the storage consumed by those servers.
The set of tools will grow.
`

func main() {
	const name = "audit"

	log.SetFlags(0)
	log.SetPrefix("upspin-audit: ")
	flag.Usage = usage
	flags.ParseArgsInto(flag.CommandLine, os.Args[1:], flags.Client, "version")

	if flags.Version {
		fmt.Fprint(os.Stdout, version.Version())
		os.Exit(2)
	}

	if flag.NArg() < 1 {
		usage()
	}
	s := &State{
		State: subcmd.NewState(name),
	}

	cfg, err := config.FromFile(flags.Config)
	if err != nil {
		s.Exit(err)
	}
	transports.Init(cfg)
	s.State.Init(cfg)

	switch flag.Arg(0) {
	case "scandir":
		s.scanDirectories(flag.Args()[1:])
	case "scanstore":
		s.scanStore(flag.Args()[1:])
	default:
		usage()
	}

	s.ExitNow()
}

func usage() {
	fmt.Fprintln(os.Stderr, help)
	fmt.Fprintln(os.Stderr, "Usage of upspin audit:")
	fmt.Fprintln(os.Stderr, "\tupspin [globalflags] audit <command> [flags] ...")
	fmt.Fprintln(os.Stderr, "\twhere <command> is one of scandir, scanstore")
	flag.PrintDefaults()
	os.Exit(2)
}

// dataDirFlag returns a string pointer bound to a new flag that specifies the data directory.
// Done here so the definition can be common among the commands.
func dataDirFlag(fs *flag.FlagSet) *string {
	var dataDir string
	fs.StringVar(&dataDir, "data", filepath.Join(os.Getenv("HOME"), "upspin", "audit"), "`directory` storing scan data")
	return &dataDir
}

// ByteSize provides a way to make numbers format in nice compact form.
// Convert a number to ByteSize and print it using its String method to see
// 2392685154 print as 2.23GB.
type ByteSize float64

const (
	_           = iota // ignore first value by assigning to blank identifier
	KB ByteSize = 1 << (10 * iota)
	MB
	GB
	TB
	PB
	EB
	ZB
	YB
)

func (b ByteSize) String() string {
	switch {
	case b >= YB:
		return fmt.Sprintf("%.2fYB", b/YB)
	case b >= ZB:
		return fmt.Sprintf("%.2fZB", b/ZB)
	case b >= EB:
		return fmt.Sprintf("%.2fEB", b/EB)
	case b >= PB:
		return fmt.Sprintf("%.2fPB", b/PB)
	case b >= TB:
		return fmt.Sprintf("%.2fTB", b/TB)
	case b >= GB:
		return fmt.Sprintf("%.2fGB", b/GB)
	case b >= MB:
		return fmt.Sprintf("%.2fMB", b/MB)
	case b >= KB:
		return fmt.Sprintf("%.2fKB", b/KB)
	}
	return fmt.Sprintf("%.2fB", b)
}

// writeItems sorts and writes a list of reference/size pairs to file.
func (s *State) writeItems(file string, items []upspin.ListRefsItem) {
	sort.Slice(items, func(i, j int) bool { return items[i].Ref < items[j].Ref })

	f, err := os.Create(file)
	if err != nil {
		s.Exit(err)
	}
	defer func() {
		if err := f.Close(); err != nil {
			s.Exit(err)
		}
	}()
	w := bufio.NewWriter(f)
	for _, ri := range items {
		if _, err := fmt.Fprintf(w, "%q %d\n", ri.Ref, ri.Size); err != nil {
			s.Exit(err)
		}
	}
	if err := w.Flush(); err != nil {
		s.Exit(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"image"
	"image/jpeg"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"text/template"

	"github.com/yml/whiteboardcleaner"
)

var (
	maxMemory int64 = 1 * 1024 * 1024 // 1MB

	layoutTmpl string = `{{ define "base" }}<!DOCTYPE html>

		<meta charset="UTF-8"/>
		<title>{{ template "title" .}}</title>

	{{ template "content" . }}

{{ end }}
`

	resultTmpl string = `{{ define "title" }}Whiteboord cleaner | result{{ end }}
{{ define "content" }}{{ range . }}

{{ end }}{{ end}}
`

	indexTmpl string = `{{ define "title" }}Whiteboard cleaner{{ end }}
{{ define "content" }}
	<form action="/upload/" method="POST" enctype="multipart/form-data">
		<fieldset>
		<legend>Edge detection</legend>
		{{ if .Errors.EdgeDetectionKernelSize }}
{{ .Errors.EdgeDetectionKernelSize }}
{{ end }}
		EdgeDetectionKernelSize
		<input name="EdgeDetectionKernelSize" type="text" value="{{ .Opts.EdgeDetectionKernelSize }}"></input>

		{{ if .Errors.ConvolutionMultiplicator }}
{{ .Errors.ConvolutionMultiplicator }}
{{ end }}
		ConvolutionMultiplicator
		<input name="ConvolutionMultiplicator" type="text" value="{{ .Opts.ConvolutionMultiplicator }}"></input>
		</fieldset>

		<fieldset>
		<legend>cleanup the image to get a white backgound</legend>

		{{ if .Errors.GaussianBlurSigma }}
{{ .Errors.GaussianBlurSigma }}
{{ end }}
		GaussianBlurSigma
		<input name="GaussianBlurSigma" type="text" value="{{ .Opts.GaussianBlurSigma }}"></input>

		{{ if .Errors.SigmoidMidpoint }}
{{ .Errors.SigmoidMidpoint }}
{{ end }}
		SigmoidMidpoint
		<input name="SigmoidMidpoint" type="text" value="{{ .Opts.SigmoidMidpoint }}"></input>

		{{ if .Errors.MedianKsize }}
{{ .Errors.MedianKsize }}
{{ end }}
		MedianKsize
		<input name="MedianKsize" type="text" value="{{ .Opts.MedianKsize }}"></input>
		</fieldset>

		<fieldset>
		<legend>Image</legend>
		{{ if .Errors.file }}
{{ .Errors.file }}
{{ end }}
		File:
		<input name="file" type="file"></input>
		</fieldset>

		<input type="submit"></input>
	</form>
{{ end }}
`
)

type appContext struct {
	TmpDir                          string
	PrefixTmpDir                    string
	UploadURL, ResultURL, StaticURL string
	Templates                       map[string]*template.Template
}

func uploadHandler(ctx *appContext) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if err := r.ParseMultipartForm(maxMemory); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		filterOpts := whiteboardcleaner.NewOptions()
		errors := make(map[string]string)
		// Update filterOpts with the values from the form
		for k, v := range r.MultipartForm.Value {
			switch k {
			case "EdgeDetectionKernelSize":
				val, err := strconv.Atoi(v[0])
				if err != nil {
					errors["EdgeDetectionKernelSize"] = err.Error()
				}
				filterOpts.EdgeDetectionKernelSize = val
			case "ConvolutionMultiplicator":
				val, err := strconv.ParseFloat(v[0], 32)
				if err != nil {
					errors["ConvolutionMultiplicator"] = err.Error()
				}
				filterOpts.ConvolutionMultiplicator = float32(val)
			case "GaussianBlurSigma":
				val, err := strconv.ParseFloat(v[0], 32)
				if err != nil {
					errors["GaussianBlurSigma"] = err.Error()
				}
				filterOpts.GaussianBlurSigma = float32(val)
			case "SigmoidMidpoint":
				val, err := strconv.ParseFloat(v[0], 32)
				if err != nil {
					errors["SigmoidMidpoint"] = err.Error()
				}
				filterOpts.SigmoidMidpoint = float32(val)
			case "SigmoidFactor":
				val, err := strconv.ParseFloat(v[0], 32)
				if err != nil {
					errors["SigmoidFactor"] = err.Error()
				}
				filterOpts.SigmoidFactor = float32(val)
			case "MedianKsize":
				val, err := strconv.Atoi(v[0])
				if err != nil {
					errors["MedianKsize"] = err.Error()
				}
				filterOpts.MedianKsize = val

			}
		}
		if len(errors) > 0 {
			tmpl := ctx.Templates["index"]
			tmpl.ExecuteTemplate(
				w,
				"base",
				struct {
					Opts   *whiteboardcleaner.Options
					Errors map[string]string
				}{Opts: filterOpts, Errors: errors})

			return

		}

		dirPath, err := ioutil.TempDir(ctx.TmpDir, ctx.PrefixTmpDir)
		_, dirName := filepath.Split(dirPath)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}

		for _, fileHeaders := range r.MultipartForm.File {
			for _, fileHeader := range fileHeaders {
				file, err := fileHeader.Open()
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
				}
				tf, err := ioutil.TempFile(dirPath, fmt.Sprintf("%s_", fileHeader.Filename))

				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
				}
				io.Copy(tf, file)
				// rewind the file to the  begining
				tf.Seek(0, 0)
				// Decode the image
				img, err := jpeg.Decode(tf)
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
				}
				g := whiteboardcleaner.NewFilter(filterOpts)
				dst := image.NewRGBA(g.Bounds(img.Bounds()))
				g.Draw(dst, img)
				// Create the dstTemporaryFile
				dstTemporaryFile, err := ioutil.TempFile(dirPath, fmt.Sprintf("cleaned_%s_", fileHeader.Filename))
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
				}
				jpeg.Encode(dstTemporaryFile, dst, &jpeg.Options{Quality: 99})
				http.Redirect(
					w, r, fmt.Sprintf("%s%s", ctx.ResultURL, dirName), http.StatusMovedPermanently)
			}
		}
	}
}

func resultHandler(ctx *appContext) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		path := r.URL.Path
		dirName, err := filepath.Rel(ctx.ResultURL, path)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		files, err := filepath.Glob(filepath.Join(ctx.TmpDir, dirName, "*"))
		for i, file := range files {
			rel, err := filepath.Rel(os.TempDir(), file)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
			}
			files[i] = filepath.Join(ctx.StaticURL, rel)
		}
		tmpl := ctx.Templates["result"]
		tmpl.ExecuteTemplate(w, "base", files)
	}
}

func indexHandler(ctx *appContext) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		filterOpts := whiteboardcleaner.NewOptions()
		errors := make(map[string]string)
		tmpl := ctx.Templates["index"]
		tmpl.ExecuteTemplate(
			w,
			"base",
			struct {
				Opts   *whiteboardcleaner.Options
				Errors map[string]string
			}{Opts: filterOpts, Errors: errors})
	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	addr := flag.String("addr", ":8080", "path to the source image")
	tmpls := make(map[string]*template.Template)
	layout := template.Must(template.New("Layout").Parse(layoutTmpl))
	tmpl := template.Must(layout.Clone())
	tmpls["index"] = template.Must(tmpl.New("index").Parse(indexTmpl))
	tmpl = template.Must(layout.Clone())
	tmpls["result"] = template.Must(tmpl.New("result").Parse(resultTmpl))

	ctx := &appContext{
		TmpDir:       os.TempDir(),
		PrefixTmpDir: "whiteboardcleaner_",
		UploadURL:    "/upload/",
		ResultURL:    "/cleaned/",
		StaticURL:    "/static/",
		Templates:    tmpls,
	}

	fmt.Println("Starting whiteboard cleaner server listening on addr", *addr)

	mux := http.NewServeMux()
	mux.HandleFunc(ctx.UploadURL, uploadHandler(ctx))
	mux.HandleFunc(ctx.ResultURL, resultHandler(ctx))
	mux.Handle(ctx.StaticURL,
		http.StripPrefix(ctx.StaticURL, http.FileServer(http.Dir(os.TempDir()))))
	mux.HandleFunc("/", indexHandler(ctx))
	http.ListenAndServe(*addr, mux)
}
<commit_msg>Add a Custom loggedResponseWriter to log the request and the status code<commit_after>package main

import (
	"flag"
	"fmt"
	"image"
	"image/jpeg"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"text/template"
	"time"

	"github.com/yml/whiteboardcleaner"
)

var (
	maxMemory int64 = 1 * 1024 * 1024 // 1MB

	layoutTmpl string = `{{ define "base" }}<!DOCTYPE html>

		<meta charset="UTF-8"/>
		<title>{{ template "title" .}}</title>

	{{ template "content" . }}

{{ end }}
`

	resultTmpl string = `{{ define "title" }}Whiteboord cleaner | result{{ end }}
{{ define "content" }}{{ range . }}

{{ end }}{{ end}}
`

	indexTmpl string = `{{ define "title" }}Whiteboard cleaner{{ end }}
{{ define "content" }}
	<form action="/upload/" method="POST" enctype="multipart/form-data">
		<fieldset>
		<legend>Edge detection</legend>
		{{ if .Errors.EdgeDetectionKernelSize }}
{{ .Errors.EdgeDetectionKernelSize }}
{{ end }}
		EdgeDetectionKernelSize
		<input name="EdgeDetectionKernelSize" type="text" value="{{ .Opts.EdgeDetectionKernelSize }}"></input>

		{{ if .Errors.ConvolutionMultiplicator }}
{{ .Errors.ConvolutionMultiplicator }}
{{ end }}
		ConvolutionMultiplicator
		<input name="ConvolutionMultiplicator" type="text" value="{{ .Opts.ConvolutionMultiplicator }}"></input>
		</fieldset>

		<fieldset>
		<legend>cleanup the image to get a white backgound</legend>

		{{ if .Errors.GaussianBlurSigma }}
{{ .Errors.GaussianBlurSigma }}
{{ end }}
		GaussianBlurSigma
		<input name="GaussianBlurSigma" type="text" value="{{ .Opts.GaussianBlurSigma }}"></input>

		{{ if .Errors.SigmoidMidpoint }}
{{ .Errors.SigmoidMidpoint }}
{{ end }}
		SigmoidMidpoint
		<input name="SigmoidMidpoint" type="text" value="{{ .Opts.SigmoidMidpoint }}"></input>

		{{ if .Errors.MedianKsize }}
{{ .Errors.MedianKsize }}
{{ end }}
		MedianKsize
		<input name="MedianKsize" type="text" value="{{ .Opts.MedianKsize }}"></input>
		</fieldset>

		<fieldset>
		<legend>Image</legend>
		{{ if .Errors.file }}
{{ .Errors.file }}
{{ end }}
		File:
		<input name="file" type="file"></input>
		</fieldset>

		<input type="submit"></input>
	</form>
{{ end }}
`
)

type appContext struct {
	TmpDir                          string
	PrefixTmpDir                    string
	UploadURL, ResultURL, StaticURL string
	Templates                       map[string]*template.Template
}

func uploadHandler(ctx *appContext) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if err := r.ParseMultipartForm(maxMemory); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		filterOpts := whiteboardcleaner.NewOptions()
		errors := make(map[string]string)
		// Update filterOpts with the values from the form
		for k, v := range r.MultipartForm.Value {
			switch k {
			case "EdgeDetectionKernelSize":
				val, err := strconv.Atoi(v[0])
				if err != nil {
					errors["EdgeDetectionKernelSize"] = err.Error()
				}
				filterOpts.EdgeDetectionKernelSize = val
			case "ConvolutionMultiplicator":
				val, err := strconv.ParseFloat(v[0], 32)
				if err != nil {
					errors["ConvolutionMultiplicator"] = err.Error()
				}
				filterOpts.ConvolutionMultiplicator = float32(val)
			case "GaussianBlurSigma":
				val, err := strconv.ParseFloat(v[0], 32)
				if err != nil {
					errors["GaussianBlurSigma"] = err.Error()
				}
				filterOpts.GaussianBlurSigma = float32(val)
			case "SigmoidMidpoint":
				val, err := strconv.ParseFloat(v[0], 32)
				if err != nil {
					errors["SigmoidMidpoint"] = err.Error()
				}
				filterOpts.SigmoidMidpoint = float32(val)
			case "SigmoidFactor":
				val, err := strconv.ParseFloat(v[0], 32)
				if err != nil {
					errors["SigmoidFactor"] = err.Error()
				}
				filterOpts.SigmoidFactor = float32(val)
			case "MedianKsize":
				val, err := strconv.Atoi(v[0])
				if err != nil {
					errors["MedianKsize"] = err.Error()
				}
				filterOpts.MedianKsize = val

			}
		}
		if len(errors) > 0 {
			tmpl := ctx.Templates["index"]
			tmpl.ExecuteTemplate(
				w,
				"base",
				struct {
					Opts   *whiteboardcleaner.Options
					Errors map[string]string
				}{Opts: filterOpts, Errors: errors})

			return

		}

		dirPath, err := ioutil.TempDir(ctx.TmpDir, ctx.PrefixTmpDir)
		_, dirName := filepath.Split(dirPath)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}

		for _, fileHeaders := range r.MultipartForm.File {
			for _, fileHeader := range fileHeaders {
				file, err := fileHeader.Open()
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
				}
				tf, err := ioutil.TempFile(dirPath, fmt.Sprintf("%s_", fileHeader.Filename))

				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
				}
				io.Copy(tf, file)
				// rewind the file to the  begining
				tf.Seek(0, 0)
				// Decode the image
				img, err := jpeg.Decode(tf)
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
				}
				g := whiteboardcleaner.NewFilter(filterOpts)
				dst := image.NewRGBA(g.Bounds(img.Bounds()))
				g.Draw(dst, img)
				// Create the dstTemporaryFile
				dstTemporaryFile, err := ioutil.TempFile(dirPath, fmt.Sprintf("cleaned_%s_", fileHeader.Filename))
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
				}
				jpeg.Encode(dstTemporaryFile, dst, &jpeg.Options{Quality: 99})
				http.Redirect(
					w, r, fmt.Sprintf("%s%s", ctx.ResultURL, dirName), http.StatusMovedPermanently)
			}
		}
	}
}

func resultHandler(ctx *appContext) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		path := r.URL.Path
		dirName, err := filepath.Rel(ctx.ResultURL, path)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		files, err := filepath.Glob(filepath.Join(ctx.TmpDir, dirName, "*"))
		for i, file := range files {
			rel, err := filepath.Rel(os.TempDir(), file)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
			}
			files[i] = filepath.Join(ctx.StaticURL, rel)
		}
		tmpl := ctx.Templates["result"]
		tmpl.ExecuteTemplate(w, "base", files)
	}
}

func indexHandler(ctx *appContext) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/" {
			http.NotFound(w, r)
			return
		}
		filterOpts := whiteboardcleaner.NewOptions()
		errors := make(map[string]string)
		tmpl := ctx.Templates["index"]
		tmpl.ExecuteTemplate(
			w,
			"base",
			struct {
				Opts   *whiteboardcleaner.Options
				Errors map[string]string
			}{Opts: filterOpts, Errors: errors})
	}
}

type loggedResponseWriter struct {
	http.ResponseWriter
	status int
}

func (l *loggedResponseWriter) WriteHeader(status int) {
	l.status = status
	l.ResponseWriter.WriteHeader(status)
}

func wrap(h http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		start := time.Now()
		lw := &loggedResponseWriter{ResponseWriter: w, status: http.StatusOK}
		l := log.New(os.Stdout, "[whiteboardcleaner]", 0)
		h.ServeHTTP(lw, r)
		l.Printf("%s %s %d %s\n", r.Method, r.URL, lw.status, time.Since(start))
	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	addr := flag.String("addr", ":8080", "path to the source image")
	tmpls := make(map[string]*template.Template)
	layout := template.Must(template.New("Layout").Parse(layoutTmpl))
	tmpl := template.Must(layout.Clone())
	tmpls["index"] = template.Must(tmpl.New("index").Parse(indexTmpl))
	tmpl = template.Must(layout.Clone())
	tmpls["result"] = template.Must(tmpl.New("result").Parse(resultTmpl))

	ctx := &appContext{
		TmpDir:       os.TempDir(),
		PrefixTmpDir: "whiteboardcleaner_",
		UploadURL:    "/upload/",
		ResultURL:    "/cleaned/",
		StaticURL:    "/static/",
		Templates:    tmpls,
	}

	fmt.Println("Starting whiteboard cleaner server listening on addr", *addr)

	mux := http.NewServeMux()
	mux.HandleFunc(ctx.UploadURL, wrap(uploadHandler(ctx)))
	mux.HandleFunc(ctx.ResultURL, wrap(resultHandler(ctx)))
	mux.Handle(ctx.StaticURL,
		http.StripPrefix(ctx.StaticURL, http.FileServer(http.Dir(os.TempDir()))))
	mux.HandleFunc("/", wrap(indexHandler(ctx)))
	http.ListenAndServe(*addr, mux)
}
<|endoftext|>


<commit_before>// deter is the deterlab process that should run on the boss node
//
// It spawns multiple timestampers and clients, while constructing
// the topology defined on cfg.json. It assumes that hosts.txt has
// the entire list of hosts to run timestampers on and that the final
// host is the designated logging server.
//
// The overall topology that is created is defined by cfg.json.
// The port layout for each node, however, is specified here.
// cfg.json will assign each node a port p. This is the port
// that each singing node is listening on. The timestamp server
// to which clients connect is listneing on port p+1. And the
// pprof server for each node is listening on port p+2. This
// means that in order to debug each client, you can forward
// the p+2 port of each node to your localhost.
//
// In the future the loggingserver will be connecting to the
// servers on the pprof port in order to gather extra data.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/dedis/prifi/coco/test/cliutils"
	"github.com/dedis/prifi/coco/test/config"
	"github.com/dedis/prifi/coco/test/graphs"
)

var rootname string

func GenExecCmd(rFail, fFail, failures int, phys string, names []string, loggerport, rootwait string, testConnect bool) string {
	total := ""
	connectOn := -1
	if testConnect == true {
		connectOn = rand.Intn(len(names))
	}
	for i, n := range names {
		connect := false
		if connectOn == i {
			connect = true
		}
		amroot := " -amroot=false"
		if n == rootname {
			amroot = " -amroot=true"
		}
		total += "(cd remote; sudo ./forkexec -rootwait=" + rootwait +
			" -rfail=" + strconv.Itoa(rFail) +
			" -ffail=" + strconv.Itoa(fFail) +
			" -failures=" + strconv.Itoa(failures) +
			" -physaddr=" + phys +
			" -hostname=" + n +
			" -logger=" + loggerport +
			" -debug=" + debug +
			" -rounds=" + strconv.Itoa(rounds) +
			" -test_connect=" + strconv.FormatBool(connect) +
			amroot +
			" ); "
		//" </dev/null 2>/dev/null 1>/dev/null &); "
	}
	return total
}

var nmsgs string
var hpn string
var bf string
var debug string
var rate int
var failures int
var rFail int
var fFail int
var rounds int
var kill bool
var testConnect bool

func init() {
	flag.StringVar(&nmsgs, "nmsgs", "100", "the number of messages per round")
	flag.StringVar(&hpn, "hpn", "", "number of hosts per node")
	flag.StringVar(&bf, "bf", "", "branching factor")
	flag.StringVar(&debug, "debug", "false", "set debug mode")
	flag.IntVar(&rate, "rate", -1, "number of milliseconds between messages")
	flag.IntVar(&failures, "failures", 0, "percent showing per node probability of failure")
	flag.IntVar(&rFail, "rfail", 0, "number of consecutive rounds each root runs before it fails")
	flag.IntVar(&fFail, "ffail", 0, "number of consecutive rounds each follower runs before it fails")
	flag.IntVar(&rounds, "rounds", 100, "number of rounds to timestamp")
	flag.BoolVar(&kill, "kill", false, "kill everything (and don't start anything)")
	flag.BoolVar(&testConnect, "test_connect", false, "test connecting and disconnecting")
}

func main() {
	flag.Parse()
	log.SetFlags(log.Lshortfile)
	fmt.Println("running deter with nmsgs:", nmsgs, rate, rounds)

	virt, err := cliutils.ReadLines("remote/virt.txt")
	if err != nil {
		log.Fatal(err)
	}
	phys, err := cliutils.ReadLines("remote/phys.txt")
	if err != nil {
		log.Fatal(err)
	}
	vpmap := make(map[string]string)
	for i := range virt {
		vpmap[virt[i]] = phys[i]
	}
	// kill old processes
	var wg sync.WaitGroup
	for _, h := range phys {
		wg.Add(1)
		go func(h string) {
			defer wg.Done()
			cliutils.SshRun("", h, "sudo killall exec logserver timeclient scp ssh 2>/dev/null >/dev/null")
			time.Sleep(1 * time.Second)
			cliutils.SshRun("", h, "sudo killall forkexec 2>/dev/null >/dev/null")
		}(h)
	}
	wg.Wait()

	if kill {
		return
	}

	for _, h := range phys {
		wg.Add(1)
		go func(h string) {
			defer wg.Done()
			cliutils.Rsync("", h, "remote", "")
		}(h)
	}
	wg.Wait()

	nloggers := 3
	masterLogger := phys[0]
	slaveLogger1 := phys[1]
	slaveLogger2 := phys[2]
	loggers := []string{masterLogger, slaveLogger1, slaveLogger2}

	phys = phys[nloggers:]
	virt = virt[nloggers:]

	// Read in and parse the configuration file
	file, err := ioutil.ReadFile("remote/cfg.json")
	if err != nil {
		log.Fatal("deter.go: error reading configuration file: %v\n", err)
	}
	log.Println("cfg file:", string(file))
	var cf config.ConfigFile
	err = json.Unmarshal(file, &cf)
	if err != nil {
		log.Fatal("unable to unmarshal config.ConfigFile:", err)
	}

	hostnames := cf.Hosts

	depth := graphs.Depth(cf.Tree)

	rootname = hostnames[0]

	log.Println("depth of tree:", depth)

	// mapping from physical node name to the timestamp servers that are running there
	// essentially a reverse mapping of vpmap except ports are also used
	physToServer := make(map[string][]string)
	for _, virt := range hostnames {
		v, _, _ := net.SplitHostPort(virt)
		p := vpmap[v]
		ss := physToServer[p]
		ss = append(ss, virt)
		physToServer[p] = ss
	}

	// start up the logging server on the final host at port 10000
	fmt.Println("starting up logserver")
	// start up the master logger
	loggerports := make([]string, len(loggers))
	for i, logger := range loggers {
		loggerport := logger + ":10000"
		loggerports[i] = loggerport
		// redirect to the master logger
		master := masterLogger + ":10000"
		// if this is the master logger than don't set the master to anything
		if loggerport == masterLogger+":10000" {
			master = ""
		}

		go cliutils.SshRunStdout("", logger, "cd remote/logserver; sudo ./logserver -addr="+loggerport+
			" -hosts="+strconv.Itoa(len(hostnames))+
			" -depth="+strconv.Itoa(depth)+
			" -bf="+bf+
			" -hpn="+hpn+
			" -nmsgs="+nmsgs+
			" -rate="+strconv.Itoa(rate)+
			" -master="+master)
	}

	// wait a little bit for the logserver to start up
	time.Sleep(5 * time.Second)
	fmt.Println("starting time clients")

	// start up one timeclient per physical machine
	// it requests timestamps from all the servers on that machine
	i := 0
	for p, ss := range physToServer {
		if len(ss) == 0 {
			continue
		}
		servers := strings.Join(ss, ",")
		go func(i int, p string) {
			_, err := cliutils.SshRun("", p, "cd remote; sudo ./timeclient -nmsgs="+nmsgs+
				" -name=client@"+p+
				" -server="+servers+
				" -logger="+loggerports[i]+
				" -debug="+debug+
				" -rate="+strconv.Itoa(rate))
			if err != nil {
				log.Println(err)
			}
		}(i, p)
		i = (i + 1) % len(loggerports)
	}
	rootwait := strconv.Itoa(10)
	var connectOn = -1
	if testConnect {
		connectOn = rand.Intn(len(physToServer))
	}
	physi := 0
	for phys, virts := range physToServer {
		if len(virts) == 0 {
			continue
		}
		connect := false
		if connectOn == physi {
			connect = true
		}
		log.Println("starting timestamper")
		cmd := GenExecCmd(rFail, fFail, failures, phys, virts, loggerports[i], rootwait, connect)
		i = (i + 1) % len(loggerports)
		physi++
		wg.Add(1)
		//time.Sleep(500 * time.Millisecond)
		go func(phys, cmd string) {
			//log.Println("running on ", phys, cmd)
			defer wg.Done()
			err := cliutils.SshRunStdout("", phys, cmd)
			if err != nil {
				log.Fatal("ERROR STARTING TIMESTAMPER:", err)
			}
		}(phys, cmd)

	}
	// wait for the servers to finish before stopping
	wg.Wait()
	time.Sleep(10 * time.Minute)
}
<commit_msg>deter.go if testing connect and disconnect should only set leaves to test_connect mode.<commit_after>// deter is the deterlab process that should run on the boss node
//
// It spawns multiple timestampers and clients, while constructing
// the topology defined on cfg.json. It assumes that hosts.txt has
// the entire list of hosts to run timestampers on and that the final
// host is the designated logging server.
//
// The overall topology that is created is defined by cfg.json.
// The port layout for each node, however, is specified here.
// cfg.json will assign each node a port p. This is the port
// that each singing node is listening on. The timestamp server
// to which clients connect is listneing on port p+1. And the
// pprof server for each node is listening on port p+2. This
// means that in order to debug each client, you can forward
// the p+2 port of each node to your localhost.
//
// In the future the loggingserver will be connecting to the
// servers on the pprof port in order to gather extra data.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/dedis/prifi/coco/test/cliutils"
	"github.com/dedis/prifi/coco/test/config"
	"github.com/dedis/prifi/coco/test/graphs"
)

var rootname string

func GenExecCmd(rFail, fFail, failures int, phys string, names []string, loggerport, rootwait string, random_leaf string) string {
	total := ""
	for _, n := range names {
		connect := false
		if n == random_leaf && testConnect {
			connect = true
		}
		amroot := " -amroot=false"
		if n == rootname {
			amroot = " -amroot=true"
		}
		total += "(cd remote; sudo ./forkexec -rootwait=" + rootwait +
			" -rfail=" + strconv.Itoa(rFail) +
			" -ffail=" + strconv.Itoa(fFail) +
			" -failures=" + strconv.Itoa(failures) +
			" -physaddr=" + phys +
			" -hostname=" + n +
			" -logger=" + loggerport +
			" -debug=" + debug +
			" -rounds=" + strconv.Itoa(rounds) +
			" -test_connect=" + strconv.FormatBool(connect) +
			amroot +
			" ); "
		//" </dev/null 2>/dev/null 1>/dev/null &); "
	}
	return total
}

var nmsgs string
var hpn string
var bf string
var debug string
var rate int
var failures int
var rFail int
var fFail int
var rounds int
var kill bool
var testConnect bool

func init() {
	flag.StringVar(&nmsgs, "nmsgs", "100", "the number of messages per round")
	flag.StringVar(&hpn, "hpn", "", "number of hosts per node")
	flag.StringVar(&bf, "bf", "", "branching factor")
	flag.StringVar(&debug, "debug", "false", "set debug mode")
	flag.IntVar(&rate, "rate", -1, "number of milliseconds between messages")
	flag.IntVar(&failures, "failures", 0, "percent showing per node probability of failure")
	flag.IntVar(&rFail, "rfail", 0, "number of consecutive rounds each root runs before it fails")
	flag.IntVar(&fFail, "ffail", 0, "number of consecutive rounds each follower runs before it fails")
	flag.IntVar(&rounds, "rounds", 100, "number of rounds to timestamp")
	flag.BoolVar(&kill, "kill", false, "kill everything (and don't start anything)")
	flag.BoolVar(&testConnect, "test_connect", false, "test connecting and disconnecting")
}

func main() {
	flag.Parse()
	log.SetFlags(log.Lshortfile)
	fmt.Println("running deter with nmsgs:", nmsgs, rate, rounds)

	virt, err := cliutils.ReadLines("remote/virt.txt")
	if err != nil {
		log.Fatal(err)
	}
	phys, err := cliutils.ReadLines("remote/phys.txt")
	if err != nil {
		log.Fatal(err)
	}
	vpmap := make(map[string]string)
	for i := range virt {
		vpmap[virt[i]] = phys[i]
	}
	// kill old processes
	var wg sync.WaitGroup
	for _, h := range phys {
		wg.Add(1)
		go func(h string) {
			defer wg.Done()
			cliutils.SshRun("", h, "sudo killall exec logserver timeclient scp ssh 2>/dev/null >/dev/null")
			time.Sleep(1 * time.Second)
			cliutils.SshRun("", h, "sudo killall forkexec 2>/dev/null >/dev/null")
		}(h)
	}
	wg.Wait()

	if kill {
		return
	}

	for _, h := range phys {
		wg.Add(1)
		go func(h string) {
			defer wg.Done()
			cliutils.Rsync("", h, "remote", "")
		}(h)
	}
	wg.Wait()

	nloggers := 3
	masterLogger := phys[0]
	slaveLogger1 := phys[1]
	slaveLogger2 := phys[2]
	loggers := []string{masterLogger, slaveLogger1, slaveLogger2}

	phys = phys[nloggers:]
	virt = virt[nloggers:]

	// Read in and parse the configuration file
	file, err := ioutil.ReadFile("remote/cfg.json")
	if err != nil {
		log.Fatal("deter.go: error reading configuration file: %v\n", err)
	}
	log.Println("cfg file:", string(file))
	var cf config.ConfigFile
	err = json.Unmarshal(file, &cf)
	if err != nil {
		log.Fatal("unable to unmarshal config.ConfigFile:", err)
	}

	hostnames := cf.Hosts

	depth := graphs.Depth(cf.Tree)
	var random_leaf string
	cf.Tree.TraverseTree(func(t *graphs.Tree) {
		if random_leaf == "" {
			return
		}
		if len(t.Children) == 0 {
			random_leaf = t.Name
		}
	})

	rootname = hostnames[0]

	log.Println("depth of tree:", depth)

	// mapping from physical node name to the timestamp servers that are running there
	// essentially a reverse mapping of vpmap except ports are also used
	physToServer := make(map[string][]string)
	for _, virt := range hostnames {
		v, _, _ := net.SplitHostPort(virt)
		p := vpmap[v]
		ss := physToServer[p]
		ss = append(ss, virt)
		physToServer[p] = ss
	}

	// start up the logging server on the final host at port 10000
	fmt.Println("starting up logserver")
	// start up the master logger
	loggerports := make([]string, len(loggers))
	for i, logger := range loggers {
		loggerport := logger + ":10000"
		loggerports[i] = loggerport
		// redirect to the master logger
		master := masterLogger + ":10000"
		// if this is the master logger than don't set the master to anything
		if loggerport == masterLogger+":10000" {
			master = ""
		}

		go cliutils.SshRunStdout("", logger, "cd remote/logserver; sudo ./logserver -addr="+loggerport+
			" -hosts="+strconv.Itoa(len(hostnames))+
			" -depth="+strconv.Itoa(depth)+
			" -bf="+bf+
			" -hpn="+hpn+
			" -nmsgs="+nmsgs+
			" -rate="+strconv.Itoa(rate)+
			" -master="+master)
	}

	// wait a little bit for the logserver to start up
	time.Sleep(5 * time.Second)
	fmt.Println("starting time clients")

	// start up one timeclient per physical machine
	// it requests timestamps from all the servers on that machine
	i := 0
	for p, ss := range physToServer {
		if len(ss) == 0 {
			continue
		}
		servers := strings.Join(ss, ",")
		go func(i int, p string) {
			_, err := cliutils.SshRun("", p, "cd remote; sudo ./timeclient -nmsgs="+nmsgs+
				" -name=client@"+p+
				" -server="+servers+
				" -logger="+loggerports[i]+
				" -debug="+debug+
				" -rate="+strconv.Itoa(rate))
			if err != nil {
				log.Println(err)
			}
		}(i, p)
		i = (i + 1) % len(loggerports)
	}
	rootwait := strconv.Itoa(10)
	for phys, virts := range physToServer {
		if len(virts) == 0 {
			continue
		}
		log.Println("starting timestamper")
		cmd := GenExecCmd(rFail, fFail, failures, phys, virts, loggerports[i], rootwait, random_leaf)
		i = (i + 1) % len(loggerports)
		wg.Add(1)
		//time.Sleep(500 * time.Millisecond)
		go func(phys, cmd string) {
			//log.Println("running on ", phys, cmd)
			defer wg.Done()
			err := cliutils.SshRunStdout("", phys, cmd)
			if err != nil {
				log.Fatal("ERROR STARTING TIMESTAMPER:", err)
			}
		}(phys, cmd)

	}
	// wait for the servers to finish before stopping
	wg.Wait()
	time.Sleep(10 * time.Minute)
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Jaime Lopez. All rights reserved.
// Use of this source code is governed by a MIT license
// that can be found in the LICENSE file.

// The datatypes/collection package provides new structures and
// behaviours to the iteration of non-sorted unique element and homogeneous
// lists accepting primitives types and complex user structs as well.

// This part of package contains the core behaviour

package collection

import (
	"reflect"

	"github.com/jaimelopez/datatypes/generic"
)

// Generic element
type Element interface{}

// A list of elements
// elements := ElementList{"element1", "element2"}
type ElementList []Element

// Generic elements collection
// Used as parameter type in order to allow encapsulate any
// kind of iterable object including ElementList as well
type CollectionElements interface{}

// Non-sorted unique element and homogeneous lists
type Collection struct {
	definition reflect.Type
	elements   []Element
}

// Adds a single element to the collection
// The collection must to be homogeneous so the specified element
// should be the same type such the other elements already stored in the collection.
// If the collection is empty and has no elements, it will take the type of
// that element as type definition for the collection
func (col *Collection) Add(element Element) {
	if col.IsEmpty() {
		col.definition = reflect.TypeOf(element)
	}

	if !col.isHomogeneousWith(element) {
		NewInvalidElementTypeError(col.definition.Name())
	}

	if col.Contains(element) {
		NewDuplicatedElementError()
	}

	col.elements = append(col.elements, element)
}

// Inserts a range (slice) inside the collection
// If the parameter can't be converted to a iterable data type it's return an error
func (col *Collection) AddRange(elements CollectionElements) {
	for _, element := range generic.ToSlice(elements) {
		col.Add(element)
	}
}

// Adds the elements contained in the parameter collection inside the instanced collection
// If the parameter can't be converted to a iterable data type it's return an error
func (col *Collection) AddCollection(collection *Collection) {
	col.AddRange(collection.elements)
}

// Returns the first element without removing it from the collection
func (col *Collection) First() Element {
	return col.elements[0]
}

// Returns the last element without removing it from the collection
func (col *Collection) Last() Element {
	return col.elements[len(col.elements)-1]
}

// Although a collection is an unsorted data structure list and the position
// of the elements could be changed, this method allows to return an specific index position.
// Be aware that the order of elements could be changed constantly such it's described before
func (col *Collection) ElementAt(position int) Element {
	return col.elements[position]
}

// Returns the stored collection elements as slice of this elements
// This is the proper way to iterate over all the elements of the collection
// treating them as a normal range
func (col *Collection) Elements() []Element {
	return col.elements
}

// Extract the first element and return it
// Keep in mind that this method will modify the collection elements substracting that element
func (col *Collection) Extract() Element {
	element := col.First()
	col.elements = col.elements[1:]

	return element
}

// Sets a new value for a specified index element
func (col *Collection) Set(position int, element Element) {
	if !col.isHomogeneousWith(element) {
		NewInvalidElementTypeError(col.definition.Name())
	}

	col.elements[position] = element
}

// Removes an specified already stored element
// If it's not found the method will return an error
func (col *Collection) Delete(element Element) {
	if !col.isHomogeneousWith(element) {
		NewInvalidElementTypeError(col.definition.Name())
	}

	for index, current := range col.elements {
		if reflect.DeepEqual(current, element) {
			col.elements[index] = col.elements[col.Count()-1]
			col.elements = col.elements[:col.Count()-1]

			return
		}
	}

	NewElementNotFoundError()
}

// Removes all the found elements contained in the specified range (slice)
// If the parameter can't be converted to a iterable data type it's return an error
func (col *Collection) DeleteRange(elements CollectionElements) {
	for _, element := range generic.ToSlice(elements) {
		col.Delete(element)
	}
}

// Removes all the found elements contained in the specified collection from the instaced collection
// If the parameter can't be converted to a iterable data type it's return an error
func (col *Collection) DeleteCollection(collection *Collection) {
	col.DeleteRange(collection.elements)
}

// Checks if the specified element is already existing in the collection
func (col *Collection) Contains(element Element) bool {
	for _, iterator := range col.elements {
		if reflect.DeepEqual(iterator, element) {
			return true
		}
	}

	return false
}

// Checks if any of the parameter elements there are already contained in the collection
func (col *Collection) ContainsAny(elements CollectionElements) (result bool) {
	defer func() {
		if recover() != nil {
			result = false
		}
	}()

	for _, element := range generic.ToSlice(elements) {
		if col.Contains(element) {
			return true
		}
	}

	return
}

// Returns the number of elements inside the collection
func (col *Collection) Count() int {
	return len(col.elements)
}

// Checks if the collection is empty or not
func (col *Collection) IsEmpty() bool {
	return col.Count() == 0
}

func (col *Collection) isHomogeneousWith(element Element) bool {
	return col.definition == reflect.TypeOf(element)
}

// Instances a new empty collection
func NewEmptyCollection() *Collection {
	return new(Collection)
}

// This method allows to instance a new Collection with a group of elements
// It accepts an enumerable
func NewCollection(elements CollectionElements) (collection *Collection) {
	collection = new(Collection)

	defer func(collection *Collection) {
		if recover() != nil {
			collection.Add(elements)
		}
	}(collection)

	collection.AddRange(elements)

	return
}
<commit_msg>Revert<commit_after>// Copyright (c) 2017 Jaime Lopez. All rights reserved.
// Use of this source code is governed by a MIT license
// that can be found in the LICENSE file.

// The datatypes/collection package provides new structures and
// behaviours to the iteration of non-sorted unique element and homogeneous
// lists accepting primitives types and complex user structs as well.

// This part of package contains the core behaviour

package collection

import (
	"datatypes/generic"
	"reflect"
)

// Generic element
type Element interface{}

// A list of elements
// elements := ElementList{"element1", "element2"}
type ElementList []Element

// Generic elements collection
// Used as parameter type in order to allow encapsulate any
// kind of iterable object including ElementList as well
type CollectionElements interface{}

// Non-sorted unique element and homogeneous lists
type Collection struct {
	definition reflect.Type
	elements   []Element
}

// Adds a single element to the collection
// The collection must to be homogeneous so the specified element
// should be the same type such the other elements already stored in the collection.
// If the collection is empty and has no elements, it will take the type of
// that element as type definition for the collection
func (col *Collection) Add(element Element) {
	if col.IsEmpty() {
		col.definition = reflect.TypeOf(element)
	}

	if !col.isHomogeneousWith(element) {
		NewInvalidElementTypeError(col.definition.Name())
	}

	if col.Contains(element) {
		NewDuplicatedElementError()
	}

	col.elements = append(col.elements, element)
}

// Inserts a range (slice) inside the collection
// If the parameter can't be converted to a iterable data type it's return an error
func (col *Collection) AddRange(elements CollectionElements) {
	for _, element := range generic.ToSlice(elements) {
		col.Add(element)
	}
}

// Adds the elements contained in the parameter collection inside the instanced collection
// If the parameter can't be converted to a iterable data type it's return an error
func (col *Collection) AddCollection(collection *Collection) {
	col.AddRange(collection.elements)
}

// Returns the first element without removing it from the collection
func (col *Collection) First() Element {
	return col.elements[0]
}

// Returns the last element without removing it from the collection
func (col *Collection) Last() Element {
	return col.elements[len(col.elements)-1]
}

// Although a collection is an unsorted data structure list and the position
// of the elements could be changed, this method allows to return an specific index position.
// Be aware that the order of elements could be changed constantly such it's described before
func (col *Collection) ElementAt(position int) Element {
	return col.elements[position]
}

// Returns the stored collection elements as slice of this elements
// This is the proper way to iterate over all the elements of the collection
// treating them as a normal range
func (col *Collection) Elements() []Element {
	return col.elements
}

// Extract the first element and return it
// Keep in mind that this method will modify the collection elements substracting that element
func (col *Collection) Extract() Element {
	element := col.First()
	col.elements = col.elements[1:]

	return element
}

// Sets a new value for a specified index element
func (col *Collection) Set(position int, element Element) {
	if !col.isHomogeneousWith(element) {
		NewInvalidElementTypeError(col.definition.Name())
	}

	col.elements[position] = element
}

// Removes an specified already stored element
// If it's not found the method will return an error
func (col *Collection) Delete(element Element) {
	if !col.isHomogeneousWith(element) {
		NewInvalidElementTypeError(col.definition.Name())
	}

	for index, current := range col.elements {
		if reflect.DeepEqual(current, element) {
			col.elements[index] = col.elements[col.Count()-1]
			col.elements = col.elements[:col.Count()-1]

			return
		}
	}

	NewElementNotFoundError()
}

// Removes all the found elements contained in the specified range (slice)
// If the parameter can't be converted to a iterable data type it's return an error
func (col *Collection) DeleteRange(elements CollectionElements) {
	for _, element := range generic.ToSlice(elements) {
		col.Delete(element)
	}
}

// Removes all the found elements contained in the specified collection from the instaced collection
// If the parameter can't be converted to a iterable data type it's return an error
func (col *Collection) DeleteCollection(collection *Collection) {
	col.DeleteRange(collection.elements)
}

// Checks if the specified element is already existing in the collection
func (col *Collection) Contains(element Element) bool {
	for _, iterator := range col.elements {
		if reflect.DeepEqual(iterator, element) {
			return true
		}
	}

	return false
}

// Checks if any of the parameter elements there are already contained in the collection
func (col *Collection) ContainsAny(elements CollectionElements) (result bool) {
	defer func() {
		if recover() != nil {
			result = false
		}
	}()

	for _, element := range generic.ToSlice(elements) {
		if col.Contains(element) {
			return true
		}
	}

	return
}

// Returns the number of elements inside the collection
func (col *Collection) Count() int {
	return len(col.elements)
}

// Checks if the collection is empty or not
func (col *Collection) IsEmpty() bool {
	return col.Count() == 0
}

func (col *Collection) isHomogeneousWith(element Element) bool {
	return col.definition == reflect.TypeOf(element)
}

// Instances a new empty collection
func NewEmptyCollection() *Collection {
	return new(Collection)
}

// This method allows to instance a new Collection with a group of elements
// It accepts an enumerable
func NewCollection(elements CollectionElements) (collection *Collection) {
	collection = new(Collection)

	defer func(collection *Collection) {
		if recover() != nil {
			collection.Add(elements)
		}
	}(collection)

	collection.AddRange(elements)

	return
}
<|endoftext|>


<commit_before>package collection

import (
  "github.com/rkbodenner/parallel_universe/game"
)

func NewTicTacToe() *game.Game {
  var setup = []*game.SetupRule{
    game.NewSetupRule("Draw 3x3 grid", "Once"),
    game.NewSetupRule("Choose X or O", "Each player"),
  }

  return game.NewGame("Tic-Tac-Toe", setup, 2, 2)
}

func NewForbiddenIsland() *game.Game {
  var setup = []*game.SetupRule{
    game.NewSetupRule("Create Forbidden Island", "Once"),     //0
    game.NewSetupRule("Place the treasures", "Once"),
    game.NewSetupRule("Divide the cards", "Once"),            //2
    game.NewSetupRule("The island starts to sink", "Once"),   //3
    game.NewSetupRule("Deal Adventurer cards", "Once"),       //4
    game.NewSetupRule("Place Adventurer pawn", "Each player"),//5
    game.NewSetupRule("Hand out Treasure deck cards", "Once"),//6
    game.NewSetupRule("Set the water level", "Once"),
  }
  setup[3].Dependencies = []*game.SetupRule{setup[0], setup[2]}
  setup[4].Dependencies = []*game.SetupRule{setup[2]}
  setup[5].Dependencies = []*game.SetupRule{setup[4]}
  setup[6].Dependencies = []*game.SetupRule{setup[2]}

  return game.NewGame("Forbidden Island", setup, 2, 4)
}

func NewOraEtLaboraShortMultiplayer() *game.Game {
  var setup = []*game.SetupRule{
    game.NewSetupRule("Choose game board for short 3-4 player game", "Once"),

    game.NewSetupRule("Attach production wheel to game board", "Once"),
    game.NewSetupRule("Place 7 wooden goods indicators on game board", "Once"),
    game.NewSetupRule("Sort the building cards", "Once"),
    game.NewSetupRule("Place the start buildings", "Once"),
    game.NewSetupRule("Place the A, B, C, D buildings", "Once"),

    game.NewSetupRule("Place the black stone goods indicator", "Once"),
    game.NewSetupRule("Place the purple grapes goods indicator", "Once"),
    game.NewSetupRule("Take a heartland landscape board", "Each player"),
    game.NewSetupRule("Place moor and forest cards on landscape board", "Each player"),
    game.NewSetupRule("Choose a color", "Each player"),

    game.NewSetupRule("Take 1 prior and 1 lay brother of your color", "Each player"),
    game.NewSetupRule("Take 8 settlement cards of your color", "Each player"),
    game.NewSetupRule("Take 1 of each of the 6 starting goods", "Each player"),
    game.NewSetupRule("Remove unused tiles", "Once"),
    game.NewSetupRule("Sort districts and plots by cost", "Once"),
  }

  setup[0].Details = "The correct board will have an icon with two players, in the center on the reverse side. Place the board in the middle of the table."

  setup[1].Details = "Side showing 0/2/3/4/... should face up. Orient the wheel so that the beam points to the bible symbol. You can unscrew the wheel from the board with a fingernail."
  setup[1].Dependencies = []*game.SetupRule{setup[0]}
  setup[2].Details = "Place onto the board where the production wheel indicates 0 (clay, coins, grain, livestock, wood, peat, joker)"
  setup[2].Dependencies = []*game.SetupRule{setup[1]}
  // FIXME: Player number variation
  setup[3].Details = "3-player game: Remove the cards with a 4 or a 3+ in the lower right corner. 4-player game: Remove the cards with a 4 in the lower right corner. Turn each card so that the chosen country variant (France or Ireland) faces up. Sort the buildings into stacks by the letter or bible symbol in the middle left of the card."
  setup[4].Details = "Start buildings have a bible symbol in the middle left of the card. Place the stack anywhere all players can see them."
  setup[4].Dependencies = []*game.SetupRule{setup[3]}
  setup[5].Details = "Place each stack next to the matching blue A, B, C, D symbol on the edge of the game board."
  setup[5].Dependencies = []*game.SetupRule{setup[1], setup[3]}

  setup[6].Details = "Place it at the position indicated by the matching symbol on the edge of the game board."
  setup[6].Dependencies = []*game.SetupRule{setup[1]}
  // FIXME: Variant
  setup[7].Details = "Only if playing the France variant. Place it at the position indicated by the matching symbol on the edge of the game board."
  setup[7].Dependencies = []*game.SetupRule{setup[1]}
  setup[9].Details = "Place 1 moor and 2 forest. Leave the left-most two spaces empty on the upper row of the landscape board."
  setup[9].Dependencies = []*game.SetupRule{setup[8]}

  setup[11].Dependencies = []*game.SetupRule{setup[10]}
  setup[12].Details = "Stack buildings marked A, B, C, D under the respective piles of building cards next to the board."
  setup[12].Dependencies = []*game.SetupRule{setup[10]}
  setup[13].Details = "Clay, coin, grain, livestock, wood, peat. Place them right-side up."
  // FIXME: Variant
  setup[14].Details = "France variant: Remove malt/beer. Ireland variant: Remove flour/bread and grapes/wine."
  setup[15].Details = "Lowest cost on top."

  return game.NewGame("Ora et Labora", setup, 3, 4)
}

type Collection struct {
  Games []*game.Game
}

func NewCollection() *Collection {
  return &Collection{
    []*game.Game{
      NewTicTacToe(),
      NewForbiddenIsland(),
      NewOraEtLaboraShortMultiplayer(),
    },
  }
}
<commit_msg>Setup details for Forbidden Island<commit_after>package collection

import (
  "github.com/rkbodenner/parallel_universe/game"
)

func NewTicTacToe() *game.Game {
  var setup = []*game.SetupRule{
    game.NewSetupRule("Draw 3x3 grid", "Once"),
    game.NewSetupRule("Choose X or O", "Each player"),
  }

  return game.NewGame("Tic-Tac-Toe", setup, 2, 2)
}

func NewForbiddenIsland() *game.Game {
  var setup = []*game.SetupRule{
    game.NewSetupRule("Create Forbidden Island", "Once"),     //0
    game.NewSetupRule("Place the treasures", "Once"),
    game.NewSetupRule("Divide the cards", "Once"),            //2
    game.NewSetupRule("The island starts to sink", "Once"),   //3
    game.NewSetupRule("The Adventurers appear", "Once"),      //4
    game.NewSetupRule("Place Adventurer pawn", "Each player"),//5
    game.NewSetupRule("Hand out Treasure deck cards", "Once"),//6
    game.NewSetupRule("Set the water level", "Once"),
  }
  setup[0].Details = "Shuffle the 24 Island tiles and randomly place them colorful-side-up into a 4x4 grid, then place 2 tiles next to each of the two middle tiles on every side of the square. Leave a small gap between the tiles."
  setup[1].Details = "Place the 4 treasure figurines--The Earth Stone, The Statue of the Wind, The Crystal of Fire, and The Ocean's Chalice--anywhere off to the side of the island"
  setup[2].Details = "Separate the cards into three decks according to the card backs: Flood deck (blue back), Treasure deck (red), and Adventurer cards (6 cards)"
  setup[3].Details = "Shuffle the Flood deck and place it face down on one side of the island, forming the Flood draw pile. Draw the top 6 cards (1 at a time) and place them face up next to the draw pile, forming the Flood discard pile. For each card drawn, flip the corresponding Island tile over to its flooded (blue & white) side."
  setup[4].Details = "Shuffle the Adventurer cards and randomly deal 1 to each player. Put undealt cards and their matching pawns back in the box."
  setup[5].Details = "Take the pawn matching the color of your Adventurer card and place it on the corresponding Island tile. Look for the matching pawn icon in the lower right corner of the Gates and Fools' Landing tiles. It's OK to start on a flooded tile."
  setup[6].Details = "Shuffle the Treasure deck and deal 2 cards to each player. Place your cards face up in front of you. If anyone gets a Waters Rise! card, give them a replacement and shuffle Waters Rise! back into the deck. Place the Treasure deck face down by one side of the island."
  setup[7].Details = "Place the Water Level slider on the left side of the Water Meter board and set it to the difficulty level of your choice"

  setup[3].Dependencies = []*game.SetupRule{setup[0], setup[2]}
  setup[4].Dependencies = []*game.SetupRule{setup[2]}
  setup[5].Dependencies = []*game.SetupRule{setup[4]}
  setup[6].Dependencies = []*game.SetupRule{setup[2]}

  return game.NewGame("Forbidden Island", setup, 2, 4)
}

func NewOraEtLaboraShortMultiplayer() *game.Game {
  var setup = []*game.SetupRule{
    game.NewSetupRule("Choose game board for short 3-4 player game", "Once"),

    game.NewSetupRule("Attach production wheel to game board", "Once"),
    game.NewSetupRule("Place 7 wooden goods indicators on game board", "Once"),
    game.NewSetupRule("Sort the building cards", "Once"),
    game.NewSetupRule("Place the start buildings", "Once"),
    game.NewSetupRule("Place the A, B, C, D buildings", "Once"),

    game.NewSetupRule("Place the black stone goods indicator", "Once"),
    game.NewSetupRule("Place the purple grapes goods indicator", "Once"),
    game.NewSetupRule("Take a heartland landscape board", "Each player"),
    game.NewSetupRule("Place moor and forest cards on landscape board", "Each player"),
    game.NewSetupRule("Choose a color", "Each player"),

    game.NewSetupRule("Take 1 prior and 1 lay brother of your color", "Each player"),
    game.NewSetupRule("Take 8 settlement cards of your color", "Each player"),
    game.NewSetupRule("Take 1 of each of the 6 starting goods", "Each player"),
    game.NewSetupRule("Remove unused tiles", "Once"),
    game.NewSetupRule("Sort districts and plots by cost", "Once"),
  }

  setup[0].Details = "The correct board will have an icon with two players, in the center on the reverse side. Place the board in the middle of the table."

  setup[1].Details = "Side showing 0/2/3/4/... should face up. Orient the wheel so that the beam points to the bible symbol. You can unscrew the wheel from the board with a fingernail."
  setup[1].Dependencies = []*game.SetupRule{setup[0]}
  setup[2].Details = "Place onto the board where the production wheel indicates 0 (clay, coins, grain, livestock, wood, peat, joker)"
  setup[2].Dependencies = []*game.SetupRule{setup[1]}
  // TODO: Player number variation
  setup[3].Details = "3-player game: Remove the cards with a 4 or a 3+ in the lower right corner. 4-player game: Remove the cards with a 4 in the lower right corner. Turn each card so that the chosen country variant (France or Ireland) faces up. Sort the buildings into stacks by the letter or bible symbol in the middle left of the card."
  setup[4].Details = "Start buildings have a bible symbol in the middle left of the card. Place the stack anywhere all players can see them."
  setup[4].Dependencies = []*game.SetupRule{setup[3]}
  setup[5].Details = "Place each stack next to the matching blue A, B, C, D symbol on the edge of the game board."
  setup[5].Dependencies = []*game.SetupRule{setup[1], setup[3]}

  setup[6].Details = "Place it at the position indicated by the matching symbol on the edge of the game board."
  setup[6].Dependencies = []*game.SetupRule{setup[1]}
  // TODO: Variant
  setup[7].Details = "Only if playing the France variant. Place it at the position indicated by the matching symbol on the edge of the game board."
  setup[7].Dependencies = []*game.SetupRule{setup[1]}
  setup[9].Details = "Place 1 moor and 2 forest. Leave the left-most two spaces empty on the upper row of the landscape board."
  setup[9].Dependencies = []*game.SetupRule{setup[8]}

  setup[11].Dependencies = []*game.SetupRule{setup[10]}
  setup[12].Details = "Stack buildings marked A, B, C, D under the respective piles of building cards next to the board."
  setup[12].Dependencies = []*game.SetupRule{setup[10]}
  setup[13].Details = "Clay, coin, grain, livestock, wood, peat. Place them right-side up."
  // TODO: Variant
  setup[14].Details = "France variant: Remove malt/beer. Ireland variant: Remove flour/bread and grapes/wine."
  setup[15].Details = "Lowest cost on top."

  return game.NewGame("Ora et Labora", setup, 3, 4)
}

type Collection struct {
  Games []*game.Game
}

func NewCollection() *Collection {
  return &Collection{
    []*game.Game{
      NewTicTacToe(),
      NewForbiddenIsland(),
      NewOraEtLaboraShortMultiplayer(),
    },
  }
}
<|endoftext|>