<commit_before>package main

import (
	"errors"
	"fmt"
	"mime"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
)

var extraMimeTypes = map[string]string{
	".icon": "image-x-icon",
	".ttf":  "application/x-font-ttf",
	".woff": "application/x-font-woff",
	".eot":  "application/vnd.ms-fontobject",
	".svg":  "image/svg+xml",
}

type Error struct {
	Message string `json:"error"`
}

func NewError(err error) Error {
	return Error{err.Error()}
}

func assetContentType(name string) string {
	ext := filepath.Ext(name)
	result := mime.TypeByExtension(ext)

	if result == "" {
		result = extraMimeTypes[ext]
	}

	if result == "" {
		result = "text/plain; charset=utf-8"
	}

	return result
}

func setupRoutes(router *gin.Engine) {
	router.GET("/", API_Home)
	router.GET("/static/:type/:name", API_ServeAsset)

	api := router.Group("/api")
	{
		api.Use(ApiMiddleware())

		api.POST("/connect", API_Connect)
		api.GET("/databases", API_GetDatabases)
		api.GET("/connection", API_ConnectionInfo)
		api.GET("/tables", API_GetTables)
		api.GET("/tables/:table", API_GetTable)
		api.GET("/tables/:table/rows", API_GetTableRows)
		api.GET("/tables/:table/info", API_GetTableInfo)
		api.GET("/tables/:table/indexes", API_TableIndexes)
		api.GET("/query", API_RunQuery)
		api.POST("/query", API_RunQuery)
		api.GET("/explain", API_ExplainQuery)
		api.POST("/explain", API_ExplainQuery)
		api.GET("/history", API_History)
		api.GET("/bookmarks", API_Bookmarks)
	}
}

// Middleware function to check database connection status before running queries
func ApiMiddleware() gin.HandlerFunc {
	allowedPaths := []string{
		"/api/connect",
		"/api/bookmarks",
		"/api/history",
	}

	return func(c *gin.Context) {
		if dbClient != nil {
			c.Next()
			return
		}

		currentPath := c.Request.URL.Path
		allowed := false

		for _, path := range allowedPaths {
			if path == currentPath {
				allowed = true
				break
			}
		}

		if allowed {
			c.Next()
		} else {
			c.JSON(400, Error{"Not connected"})
			c.Abort(400)
		}

		return
	}
}

func API_Home(c *gin.Context) {
	data, err := Asset("static/index.html")

	if err != nil {
		c.String(400, err.Error())
		return
	}

	c.Data(200, "text/html; charset=utf-8", data)
}

func API_Connect(c *gin.Context) {
	url := c.Request.FormValue("url")

	if url == "" {
		c.JSON(400, Error{"Url parameter is required"})
		return
	}

	opts := Options{Url: url}
	url, err := formatConnectionUrl(opts)

	if err != nil {
		c.JSON(400, Error{err.Error()})
		return
	}

	client, err := NewClientFromUrl(url)
	if err != nil {
		c.JSON(400, Error{err.Error()})
		return
	}

	err = client.Test()
	if err != nil {
		c.JSON(400, Error{err.Error()})
		return
	}

	info, err := client.Info()

	if err == nil {
		if dbClient != nil {
			dbClient.db.Close()
		}

		dbClient = client
	}

	c.JSON(200, info.Format()[0])
}

func API_GetDatabases(c *gin.Context) {
	names, err := dbClient.Databases()

	if err != nil {
		c.JSON(400, NewError(err))
		return
	}

	c.JSON(200, names)
}

func API_RunQuery(c *gin.Context) {
	query := strings.TrimSpace(c.Request.FormValue("query"))

	if query == "" {
		c.JSON(400, errors.New("Query parameter is missing"))
		return
	}

	API_HandleQuery(query, c)
}

func API_ExplainQuery(c *gin.Context) {
	query := strings.TrimSpace(c.Request.FormValue("query"))

	if query == "" {
		c.JSON(400, errors.New("Query parameter is missing"))
		return
	}

	API_HandleQuery(fmt.Sprintf("EXPLAIN ANALYZE %s", query), c)
}

func API_GetTables(c *gin.Context) {
	names, err := dbClient.Tables()

	if err != nil {
		c.JSON(400, NewError(err))
		return
	}

	c.JSON(200, names)
}

func API_GetTable(c *gin.Context) {
	res, err := dbClient.Table(c.Params.ByName("table"))

	if err != nil {
		c.JSON(400, NewError(err))
		return
	}

	c.JSON(200, res)
}

func API_GetTableRows(c *gin.Context) {
	limit := 1000 // Number of rows to fetch
	limitVal := c.Request.FormValue("limit")

	if limitVal != "" {
		num, err := strconv.Atoi(limitVal)

		if err != nil {
			c.JSON(400, Error{"Invalid limit value"})
			return
		}

		if num <= 0 {
			c.JSON(400, Error{"Limit should be greater than 0"})
			return
		}

		limit = num
	}

	opts := RowsOptions{
		Limit:      limit,
		SortColumn: c.Request.FormValue("sort_column"),
		SortOrder:  c.Request.FormValue("sort_order"),
	}

	res, err := dbClient.TableRows(c.Params.ByName("table"), opts)

	if err != nil {
		c.JSON(400, NewError(err))
		return
	}

	c.JSON(200, res)
}

func API_GetTableInfo(c *gin.Context) {
	res, err := dbClient.TableInfo(c.Params.ByName("table"))

	if err != nil {
		c.JSON(400, NewError(err))
		return
	}

	c.JSON(200, res.Format()[0])
}

func API_History(c *gin.Context) {
	c.JSON(200, dbClient.history)
}

func API_ConnectionInfo(c *gin.Context) {
	res, err := dbClient.Info()

	if err != nil {
		c.JSON(400, NewError(err))
		return
	}

	c.JSON(200, res.Format()[0])
}

func API_TableIndexes(c *gin.Context) {
	res, err := dbClient.TableIndexes(c.Params.ByName("table"))

	if err != nil {
		c.JSON(400, NewError(err))
		return
	}

	c.JSON(200, res)
}

func API_HandleQuery(query string, c *gin.Context) {
	result, err := dbClient.Query(query)

	if err != nil {
		c.JSON(400, NewError(err))
		return
	}

	q := c.Request.URL.Query()

	if len(q["format"]) > 0 && q["format"][0] == "csv" {
		filename := fmt.Sprintf("pgweb-%v.csv", time.Now().Unix())
		c.Writer.Header().Set("Content-disposition", "attachment;filename="+filename)
		c.Data(200, "text/csv", result.CSV())
		return
	}

	c.JSON(200, result)
}

func API_Bookmarks(c *gin.Context) {
	bookmarks, err := readAllBookmarks()

	if err != nil {
		c.JSON(400, NewError(err))
		return
	}

	c.JSON(200, bookmarks)
}

func API_ServeAsset(c *gin.Context) {
	file := fmt.Sprintf(
		"static/%s/%s",
		c.Params.ByName("type"),
		c.Params.ByName("name"),
	)

	data, err := Asset(file)

	if err != nil {
		c.String(400, err.Error())
		return
	}

	if len(data) == 0 {
		c.String(404, "Asset is empty")
		return
	}

	c.Data(200, assetContentType(file), data)
}
<commit_msg>Use wildcard in asset path<commit_after>package main

import (
	"errors"
	"fmt"
	"mime"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
)

var extraMimeTypes = map[string]string{
	".icon": "image-x-icon",
	".ttf":  "application/x-font-ttf",
	".woff": "application/x-font-woff",
	".eot":  "application/vnd.ms-fontobject",
	".svg":  "image/svg+xml",
}

type Error struct {
	Message string `json:"error"`
}

func NewError(err error) Error {
	return Error{err.Error()}
}

func assetContentType(name string) string {
	ext := filepath.Ext(name)
	result := mime.TypeByExtension(ext)

	if result == "" {
		result = extraMimeTypes[ext]
	}

	if result == "" {
		result = "text/plain; charset=utf-8"
	}

	return result
}

func setupRoutes(router *gin.Engine) {
	router.GET("/", API_Home)
	router.GET("/static/*path", API_ServeAsset)

	api := router.Group("/api")
	{
		api.Use(ApiMiddleware())

		api.POST("/connect", API_Connect)
		api.GET("/databases", API_GetDatabases)
		api.GET("/connection", API_ConnectionInfo)
		api.GET("/tables", API_GetTables)
		api.GET("/tables/:table", API_GetTable)
		api.GET("/tables/:table/rows", API_GetTableRows)
		api.GET("/tables/:table/info", API_GetTableInfo)
		api.GET("/tables/:table/indexes", API_TableIndexes)
		api.GET("/query", API_RunQuery)
		api.POST("/query", API_RunQuery)
		api.GET("/explain", API_ExplainQuery)
		api.POST("/explain", API_ExplainQuery)
		api.GET("/history", API_History)
		api.GET("/bookmarks", API_Bookmarks)
	}
}

// Middleware function to check database connection status before running queries
func ApiMiddleware() gin.HandlerFunc {
	allowedPaths := []string{
		"/api/connect",
		"/api/bookmarks",
		"/api/history",
	}

	return func(c *gin.Context) {
		if dbClient != nil {
			c.Next()
			return
		}

		currentPath := c.Request.URL.Path
		allowed := false

		for _, path := range allowedPaths {
			if path == currentPath {
				allowed = true
				break
			}
		}

		if allowed {
			c.Next()
		} else {
			c.JSON(400, Error{"Not connected"})
			c.Abort(400)
		}

		return
	}
}

func API_Home(c *gin.Context) {
	data, err := Asset("static/index.html")

	if err != nil {
		c.String(400, err.Error())
		return
	}

	c.Data(200, "text/html; charset=utf-8", data)
}

func API_Connect(c *gin.Context) {
	url := c.Request.FormValue("url")

	if url == "" {
		c.JSON(400, Error{"Url parameter is required"})
		return
	}

	opts := Options{Url: url}
	url, err := formatConnectionUrl(opts)

	if err != nil {
		c.JSON(400, Error{err.Error()})
		return
	}

	client, err := NewClientFromUrl(url)
	if err != nil {
		c.JSON(400, Error{err.Error()})
		return
	}

	err = client.Test()
	if err != nil {
		c.JSON(400, Error{err.Error()})
		return
	}

	info, err := client.Info()

	if err == nil {
		if dbClient != nil {
			dbClient.db.Close()
		}

		dbClient = client
	}

	c.JSON(200, info.Format()[0])
}

func API_GetDatabases(c *gin.Context) {
	names, err := dbClient.Databases()

	if err != nil {
		c.JSON(400, NewError(err))
		return
	}

	c.JSON(200, names)
}

func API_RunQuery(c *gin.Context) {
	query := strings.TrimSpace(c.Request.FormValue("query"))

	if query == "" {
		c.JSON(400, errors.New("Query parameter is missing"))
		return
	}

	API_HandleQuery(query, c)
}

func API_ExplainQuery(c *gin.Context) {
	query := strings.TrimSpace(c.Request.FormValue("query"))

	if query == "" {
		c.JSON(400, errors.New("Query parameter is missing"))
		return
	}

	API_HandleQuery(fmt.Sprintf("EXPLAIN ANALYZE %s", query), c)
}

func API_GetTables(c *gin.Context) {
	names, err := dbClient.Tables()

	if err != nil {
		c.JSON(400, NewError(err))
		return
	}

	c.JSON(200, names)
}

func API_GetTable(c *gin.Context) {
	res, err := dbClient.Table(c.Params.ByName("table"))

	if err != nil {
		c.JSON(400, NewError(err))
		return
	}

	c.JSON(200, res)
}

func API_GetTableRows(c *gin.Context) {
	limit := 1000 // Number of rows to fetch
	limitVal := c.Request.FormValue("limit")

	if limitVal != "" {
		num, err := strconv.Atoi(limitVal)

		if err != nil {
			c.JSON(400, Error{"Invalid limit value"})
			return
		}

		if num <= 0 {
			c.JSON(400, Error{"Limit should be greater than 0"})
			return
		}

		limit = num
	}

	opts := RowsOptions{
		Limit:      limit,
		SortColumn: c.Request.FormValue("sort_column"),
		SortOrder:  c.Request.FormValue("sort_order"),
	}

	res, err := dbClient.TableRows(c.Params.ByName("table"), opts)

	if err != nil {
		c.JSON(400, NewError(err))
		return
	}

	c.JSON(200, res)
}

func API_GetTableInfo(c *gin.Context) {
	res, err := dbClient.TableInfo(c.Params.ByName("table"))

	if err != nil {
		c.JSON(400, NewError(err))
		return
	}

	c.JSON(200, res.Format()[0])
}

func API_History(c *gin.Context) {
	c.JSON(200, dbClient.history)
}

func API_ConnectionInfo(c *gin.Context) {
	res, err := dbClient.Info()

	if err != nil {
		c.JSON(400, NewError(err))
		return
	}

	c.JSON(200, res.Format()[0])
}

func API_TableIndexes(c *gin.Context) {
	res, err := dbClient.TableIndexes(c.Params.ByName("table"))

	if err != nil {
		c.JSON(400, NewError(err))
		return
	}

	c.JSON(200, res)
}

func API_HandleQuery(query string, c *gin.Context) {
	result, err := dbClient.Query(query)

	if err != nil {
		c.JSON(400, NewError(err))
		return
	}

	q := c.Request.URL.Query()

	if len(q["format"]) > 0 && q["format"][0] == "csv" {
		filename := fmt.Sprintf("pgweb-%v.csv", time.Now().Unix())
		c.Writer.Header().Set("Content-disposition", "attachment;filename="+filename)
		c.Data(200, "text/csv", result.CSV())
		return
	}

	c.JSON(200, result)
}

func API_Bookmarks(c *gin.Context) {
	bookmarks, err := readAllBookmarks()

	if err != nil {
		c.JSON(400, NewError(err))
		return
	}

	c.JSON(200, bookmarks)
}

func API_ServeAsset(c *gin.Context) {
	path := "static" + c.Params.ByName("path")
	data, err := Asset(path)

	if err != nil {
		c.String(400, err.Error())
		return
	}

	if len(data) == 0 {
		c.String(404, "Asset is empty")
		return
	}

	c.Data(200, assetContentType(path), data)
}
<|endoftext|>


<commit_before>// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package control

import (
	"fmt"
	"math"
	"strings"
	"sync"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/coreos/dbtester/pkg/report"
	"github.com/gyuho/dataframe"
	"golang.org/x/net/context"
)

type values struct {
	bytes      [][]byte
	strings    []string
	sampleSize int
}

func newValues(cfg Config) (v values, rerr error) {
	v.bytes = [][]byte{randBytes(cfg.Step2.ValueSize)}
	v.strings = []string{string(v.bytes[0])}
	v.sampleSize = 1
	return
}

type benchmark struct {
	bar        *pb.ProgressBar
	report     report.Report
	reportDone <-chan report.Stats
	stats      report.Stats

	reqHandlers []ReqHandler
	reqGen      func(chan<- request)
	reqDone     func()
	wg          sync.WaitGroup

	mu           sync.RWMutex
	inflightReqs chan request
}

// pass totalN in case that 'cfg' is manipulated
func newBenchmark(totalN int, clientsN int, reqHandlers []ReqHandler, reqDone func(), reqGen func(chan<- request)) (b *benchmark) {
	b = &benchmark{
		bar:         pb.New(totalN),
		reqHandlers: reqHandlers,
		reqGen:      reqGen,
		reqDone:     reqDone,
		wg:          sync.WaitGroup{},
	}
	b.inflightReqs = make(chan request, clientsN)

	b.bar.Format("Bom !")
	b.bar.Start()
	b.report = report.NewReportSample("%4.4f")
	return
}

// only useful when multiple ranges of requests are run with one report
func (b *benchmark) reset(clientsN int, reqHandlers []ReqHandler, reqDone func(), reqGen func(chan<- request)) {
	if len(reqHandlers) == 0 {
		panic(fmt.Errorf("got 0 reqHandlers"))
	}
	b.reqHandlers = reqHandlers
	b.reqDone = reqDone
	b.reqGen = reqGen

	// inflight requests will be dropped!
	b.mu.Lock()
	b.inflightReqs = make(chan request, clientsN)
	b.mu.Unlock()
}

func (b *benchmark) getInflightsReqs() (ch chan request) {
	b.mu.RLock()
	ch = b.inflightReqs
	b.mu.RUnlock()
	return
}

func (b *benchmark) startRequests() {
	for i := range b.reqHandlers {
		b.wg.Add(1)
		go func(rh ReqHandler) {
			defer b.wg.Done()
			for req := range b.getInflightsReqs() {
				if rh == nil {
					panic(fmt.Errorf("got nil rh"))
				}
				st := time.Now()
				err := rh(context.Background(), &req)
				b.report.Results() <- report.Result{Err: err, Start: st, End: time.Now()}
				b.bar.Increment()
			}
		}(b.reqHandlers[i])
	}
	go b.reqGen(b.getInflightsReqs())
	b.reportDone = b.report.Stats()
}

func (b *benchmark) waitRequestsEnd() {
	b.wg.Wait()
	if b.reqDone != nil {
		b.reqDone() // cancel connections
	}
}

func (b *benchmark) finishReports() {
	close(b.report.Results())
	b.bar.Finish()
	st := <-b.reportDone
	b.stats = st
}

func (b *benchmark) waitAll() {
	b.waitRequestsEnd()
	b.finishReports()
}

func printStats(st report.Stats) {
	// to be piped to cfg.Log via stdout when dbtester executed
	if len(st.Lats) > 0 {
		fmt.Printf("Total: %v\n", st.Total)
		fmt.Printf("Slowest: %f secs\n", st.Slowest)
		fmt.Printf("Fastest: %f secs\n", st.Fastest)
		fmt.Printf("Average: %f secs\n", st.Average)
		fmt.Printf("Requests/sec: %4.4f\n", st.RPS)
	}
	if len(st.ErrorDist) > 0 {
		for k, v := range st.ErrorDist {
			fmt.Printf("ERROR %q : %d\n", k, v)
		}
	} else {
		fmt.Println("ERRRO: 0")
	}
}

func saveDataLatencyDistributionSummary(cfg Config, st report.Stats) {
	fr := dataframe.New()

	c1 := dataframe.NewColumn("TOTAL-SECONDS")
	c1.PushBack(dataframe.NewStringValue(fmt.Sprintf("%4.4f", st.Total.Seconds())))
	if err := fr.AddColumn(c1); err != nil {
		plog.Fatal(err)
	}

	c2 := dataframe.NewColumn("REQUESTS-PER-SECOND")
	c2.PushBack(dataframe.NewStringValue(fmt.Sprintf("%4.4f", st.RPS)))
	if err := fr.AddColumn(c2); err != nil {
		plog.Fatal(err)
	}

	c3 := dataframe.NewColumn("SLOWEST-LATENCY-MS")
	c3.PushBack(dataframe.NewStringValue(fmt.Sprintf("%4.4f", 1000*st.Slowest)))
	if err := fr.AddColumn(c3); err != nil {
		plog.Fatal(err)
	}

	c4 := dataframe.NewColumn("FASTEST-LATENCY-MS")
	c4.PushBack(dataframe.NewStringValue(fmt.Sprintf("%4.4f", 1000*st.Fastest)))
	if err := fr.AddColumn(c4); err != nil {
		plog.Fatal(err)
	}

	c5 := dataframe.NewColumn("AVERAGE-LATENCY-MS")
	c5.PushBack(dataframe.NewStringValue(fmt.Sprintf("%4.4f", 1000*st.Average)))
	if err := fr.AddColumn(c5); err != nil {
		plog.Fatal(err)
	}

	c6 := dataframe.NewColumn("STDDEV-LATENCY-MS")
	c6.PushBack(dataframe.NewStringValue(fmt.Sprintf("%4.4f", 1000*st.Stddev)))
	if err := fr.AddColumn(c6); err != nil {
		plog.Fatal(err)
	}

	if len(st.ErrorDist) > 0 {
		for errName, errN := range st.ErrorDist {
			errcol := dataframe.NewColumn(fmt.Sprintf("ERROR: %q", errName))
			errcol.PushBack(dataframe.NewStringValue(errN))
			if err := fr.AddColumn(errcol); err != nil {
				plog.Fatal(err)
			}
		}
	} else {
		errcol := dataframe.NewColumn("ERROR")
		errcol.PushBack(dataframe.NewStringValue("0"))
		if err := fr.AddColumn(errcol); err != nil {
			plog.Fatal(err)
		}
	}

	if err := fr.CSVHorizontal(cfg.DataLatencyDistributionSummary); err != nil {
		plog.Fatal(err)
	}
}

func saveDataLatencyDistributionPercentile(cfg Config, st report.Stats) {
	pctls, seconds := report.Percentiles(st.Lats)
	c1 := dataframe.NewColumn("LATENCY-PERCENTILE")
	c2 := dataframe.NewColumn("LATENCY-MS")
	for i := range pctls {
		pct := fmt.Sprintf("p%.1f", pctls[i])
		if strings.HasSuffix(pct, ".0") {
			pct = strings.Replace(pct, ".0", "", -1)
		}

		c1.PushBack(dataframe.NewStringValue(pct))
		c2.PushBack(dataframe.NewStringValue(fmt.Sprintf("%f", 1000*seconds[i])))
	}

	fr := dataframe.New()
	if err := fr.AddColumn(c1); err != nil {
		plog.Fatal(err)
	}
	if err := fr.AddColumn(c2); err != nil {
		plog.Fatal(err)
	}
	if err := fr.CSV(cfg.DataLatencyDistributionPercentile); err != nil {
		plog.Fatal(err)
	}
}

func saveDataLatencyDistributionAll(cfg Config, st report.Stats) {
	min := int64(math.MaxInt64)
	max := int64(-100000)
	rm := make(map[int64]int64)
	for _, lt := range st.Lats {
		// convert second(float64) to millisecond
		ms := lt * 1000

		// truncate all digits below 10ms
		// (e.g. 125.11ms becomes 120ms)
		v := int64(math.Trunc(ms/10) * 10)
		if _, ok := rm[v]; !ok {
			rm[v] = 1
		} else {
			rm[v]++
		}

		if min > v {
			min = v
		}
		if max < v {
			max = v
		}
	}

	c1 := dataframe.NewColumn("LATENCY-MS")
	c2 := dataframe.NewColumn("COUNT")
	cur := min
	for {
		c1.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", int64(cur))))
		v, ok := rm[cur]
		if ok {
			c2.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", v)))
		} else {
			c2.PushBack(dataframe.NewStringValue("0"))
		}
		cur += 10
		if cur-10 == max { // was last point
			break
		}
	}
	fr := dataframe.New()
	if err := fr.AddColumn(c1); err != nil {
		plog.Fatal(err)
	}
	if err := fr.AddColumn(c2); err != nil {
		plog.Fatal(err)
	}
	if err := fr.CSV(cfg.DataLatencyDistributionAll); err != nil {
		plog.Fatal(err)
	}
}

func saveDataLatencyThroughputTimeseries(cfg Config, st report.Stats, tsToClientN map[int64]int) {
	c1 := dataframe.NewColumn("UNIX-SECOND")
	c2 := dataframe.NewColumn("CONTROL-CLIENT-NUM")
	c3 := dataframe.NewColumn("AVG-LATENCY-MS")
	c4 := dataframe.NewColumn("AVG-THROUGHPUT")
	for i := range st.TimeSeries {
		// this Timestamp is unix seconds
		c1.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", st.TimeSeries[i].Timestamp)))
		if len(tsToClientN) == 0 {
			c2.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", cfg.Step2.Clients)))
		} else {
			c2.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", tsToClientN[st.TimeSeries[i].Timestamp])))
		}
		c3.PushBack(dataframe.NewStringValue(fmt.Sprintf("%f", toMillisecond(st.TimeSeries[i].AvgLatency))))
		c4.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", st.TimeSeries[i].ThroughPut)))
	}

	fr := dataframe.New()
	if err := fr.AddColumn(c1); err != nil {
		plog.Fatal(err)
	}
	if err := fr.AddColumn(c2); err != nil {
		plog.Fatal(err)
	}
	if err := fr.AddColumn(c3); err != nil {
		plog.Fatal(err)
	}
	if err := fr.AddColumn(c4); err != nil {
		plog.Fatal(err)
	}
	if err := fr.CSV(cfg.DataLatencyThroughputTimeseries); err != nil {
		plog.Fatal(err)
	}

	// aggregate latency by the number of keys
	tss := processTimeSeries(st.TimeSeries, 1000, cfg.Step2.TotalRequests)
	ctt1 := dataframe.NewColumn("KEYS")
	// dataframe.NewColumn("MIN-LATENCY-MS")
	ctt2 := dataframe.NewColumn("AVG-LATENCY-MS")
	// dataframe.NewColumn("MAX-LATENCY-MS")
	for i := range tss {
		ctt1.PushBack(dataframe.NewStringValue(tss[i].keyNum))
		ctt2.PushBack(dataframe.NewStringValue(fmt.Sprintf("%f", toMillisecond(tss[i].avgLat))))
	}
	frr := dataframe.New()
	if err := frr.AddColumn(ctt1); err != nil {
		plog.Fatal(err)
	}
	if err := frr.AddColumn(ctt2); err != nil {
		plog.Fatal(err)
	}
	// if err := frr.AddColumn(ctt3); err != nil {
	// 	plog.Fatal(err)
	// }
	// if err := frr.AddColumn(ctt4); err != nil {
	// 	plog.Fatal(err)
	// }
	if err := frr.CSV(cfg.DataLatencyByKeyNumber); err != nil {
		plog.Fatal(err)
	}
}

func generateReport(cfg Config, h []ReqHandler, reqDone func(), reqGen func(chan<- request)) {
	b := newBenchmark(cfg.Step2.TotalRequests, cfg.Step2.Clients, h, reqDone, reqGen)
	b.startRequests()
	b.waitAll()

	printStats(b.stats)
	saveAllStats(cfg, b.stats, nil)
}

func saveAllStats(cfg Config, stats report.Stats, tsToClientN map[int64]int) {
	// cfg.DataLatencyDistributionSummary
	saveDataLatencyDistributionSummary(cfg, stats)

	// cfg.DataLatencyDistributionPercentile
	saveDataLatencyDistributionPercentile(cfg, stats)

	// cfg.DataLatencyDistributionAll
	saveDataLatencyDistributionAll(cfg, stats)

	// cfg.DataLatencyThroughputTimeseries
	saveDataLatencyThroughputTimeseries(cfg, stats, tsToClientN)
}
<commit_msg>control: save MIN,MAX latency<commit_after>// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package control

import (
	"fmt"
	"math"
	"strings"
	"sync"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/coreos/dbtester/pkg/report"
	"github.com/gyuho/dataframe"
	"golang.org/x/net/context"
)

type values struct {
	bytes      [][]byte
	strings    []string
	sampleSize int
}

func newValues(cfg Config) (v values, rerr error) {
	v.bytes = [][]byte{randBytes(cfg.Step2.ValueSize)}
	v.strings = []string{string(v.bytes[0])}
	v.sampleSize = 1
	return
}

type benchmark struct {
	bar        *pb.ProgressBar
	report     report.Report
	reportDone <-chan report.Stats
	stats      report.Stats

	reqHandlers []ReqHandler
	reqGen      func(chan<- request)
	reqDone     func()
	wg          sync.WaitGroup

	mu           sync.RWMutex
	inflightReqs chan request
}

// pass totalN in case that 'cfg' is manipulated
func newBenchmark(totalN int, clientsN int, reqHandlers []ReqHandler, reqDone func(), reqGen func(chan<- request)) (b *benchmark) {
	b = &benchmark{
		bar:         pb.New(totalN),
		reqHandlers: reqHandlers,
		reqGen:      reqGen,
		reqDone:     reqDone,
		wg:          sync.WaitGroup{},
	}
	b.inflightReqs = make(chan request, clientsN)

	b.bar.Format("Bom !")
	b.bar.Start()
	b.report = report.NewReportSample("%4.4f")
	return
}

// only useful when multiple ranges of requests are run with one report
func (b *benchmark) reset(clientsN int, reqHandlers []ReqHandler, reqDone func(), reqGen func(chan<- request)) {
	if len(reqHandlers) == 0 {
		panic(fmt.Errorf("got 0 reqHandlers"))
	}
	b.reqHandlers = reqHandlers
	b.reqDone = reqDone
	b.reqGen = reqGen

	// inflight requests will be dropped!
	b.mu.Lock()
	b.inflightReqs = make(chan request, clientsN)
	b.mu.Unlock()
}

func (b *benchmark) getInflightsReqs() (ch chan request) {
	b.mu.RLock()
	ch = b.inflightReqs
	b.mu.RUnlock()
	return
}

func (b *benchmark) startRequests() {
	for i := range b.reqHandlers {
		b.wg.Add(1)
		go func(rh ReqHandler) {
			defer b.wg.Done()
			for req := range b.getInflightsReqs() {
				if rh == nil {
					panic(fmt.Errorf("got nil rh"))
				}
				st := time.Now()
				err := rh(context.Background(), &req)
				b.report.Results() <- report.Result{Err: err, Start: st, End: time.Now()}
				b.bar.Increment()
			}
		}(b.reqHandlers[i])
	}
	go b.reqGen(b.getInflightsReqs())
	b.reportDone = b.report.Stats()
}

func (b *benchmark) waitRequestsEnd() {
	b.wg.Wait()
	if b.reqDone != nil {
		b.reqDone() // cancel connections
	}
}

func (b *benchmark) finishReports() {
	close(b.report.Results())
	b.bar.Finish()
	st := <-b.reportDone
	b.stats = st
}

func (b *benchmark) waitAll() {
	b.waitRequestsEnd()
	b.finishReports()
}

func printStats(st report.Stats) {
	// to be piped to cfg.Log via stdout when dbtester executed
	if len(st.Lats) > 0 {
		fmt.Printf("Total: %v\n", st.Total)
		fmt.Printf("Slowest: %f secs\n", st.Slowest)
		fmt.Printf("Fastest: %f secs\n", st.Fastest)
		fmt.Printf("Average: %f secs\n", st.Average)
		fmt.Printf("Requests/sec: %4.4f\n", st.RPS)
	}
	if len(st.ErrorDist) > 0 {
		for k, v := range st.ErrorDist {
			fmt.Printf("ERROR %q : %d\n", k, v)
		}
	} else {
		fmt.Println("ERRRO: 0")
	}
}

func saveDataLatencyDistributionSummary(cfg Config, st report.Stats) {
	fr := dataframe.New()

	c1 := dataframe.NewColumn("TOTAL-SECONDS")
	c1.PushBack(dataframe.NewStringValue(fmt.Sprintf("%4.4f", st.Total.Seconds())))
	if err := fr.AddColumn(c1); err != nil {
		plog.Fatal(err)
	}

	c2 := dataframe.NewColumn("REQUESTS-PER-SECOND")
	c2.PushBack(dataframe.NewStringValue(fmt.Sprintf("%4.4f", st.RPS)))
	if err := fr.AddColumn(c2); err != nil {
		plog.Fatal(err)
	}

	c3 := dataframe.NewColumn("SLOWEST-LATENCY-MS")
	c3.PushBack(dataframe.NewStringValue(fmt.Sprintf("%4.4f", 1000*st.Slowest)))
	if err := fr.AddColumn(c3); err != nil {
		plog.Fatal(err)
	}

	c4 := dataframe.NewColumn("FASTEST-LATENCY-MS")
	c4.PushBack(dataframe.NewStringValue(fmt.Sprintf("%4.4f", 1000*st.Fastest)))
	if err := fr.AddColumn(c4); err != nil {
		plog.Fatal(err)
	}

	c5 := dataframe.NewColumn("AVERAGE-LATENCY-MS")
	c5.PushBack(dataframe.NewStringValue(fmt.Sprintf("%4.4f", 1000*st.Average)))
	if err := fr.AddColumn(c5); err != nil {
		plog.Fatal(err)
	}

	c6 := dataframe.NewColumn("STDDEV-LATENCY-MS")
	c6.PushBack(dataframe.NewStringValue(fmt.Sprintf("%4.4f", 1000*st.Stddev)))
	if err := fr.AddColumn(c6); err != nil {
		plog.Fatal(err)
	}

	if len(st.ErrorDist) > 0 {
		for errName, errN := range st.ErrorDist {
			errcol := dataframe.NewColumn(fmt.Sprintf("ERROR: %q", errName))
			errcol.PushBack(dataframe.NewStringValue(errN))
			if err := fr.AddColumn(errcol); err != nil {
				plog.Fatal(err)
			}
		}
	} else {
		errcol := dataframe.NewColumn("ERROR")
		errcol.PushBack(dataframe.NewStringValue("0"))
		if err := fr.AddColumn(errcol); err != nil {
			plog.Fatal(err)
		}
	}

	if err := fr.CSVHorizontal(cfg.DataLatencyDistributionSummary); err != nil {
		plog.Fatal(err)
	}
}

func saveDataLatencyDistributionPercentile(cfg Config, st report.Stats) {
	pctls, seconds := report.Percentiles(st.Lats)
	c1 := dataframe.NewColumn("LATENCY-PERCENTILE")
	c2 := dataframe.NewColumn("LATENCY-MS")
	for i := range pctls {
		pct := fmt.Sprintf("p%.1f", pctls[i])
		if strings.HasSuffix(pct, ".0") {
			pct = strings.Replace(pct, ".0", "", -1)
		}

		c1.PushBack(dataframe.NewStringValue(pct))
		c2.PushBack(dataframe.NewStringValue(fmt.Sprintf("%f", 1000*seconds[i])))
	}

	fr := dataframe.New()
	if err := fr.AddColumn(c1); err != nil {
		plog.Fatal(err)
	}
	if err := fr.AddColumn(c2); err != nil {
		plog.Fatal(err)
	}
	if err := fr.CSV(cfg.DataLatencyDistributionPercentile); err != nil {
		plog.Fatal(err)
	}
}

func saveDataLatencyDistributionAll(cfg Config, st report.Stats) {
	min := int64(math.MaxInt64)
	max := int64(-100000)
	rm := make(map[int64]int64)
	for _, lt := range st.Lats {
		// convert second(float64) to millisecond
		ms := lt * 1000

		// truncate all digits below 10ms
		// (e.g. 125.11ms becomes 120ms)
		v := int64(math.Trunc(ms/10) * 10)
		if _, ok := rm[v]; !ok {
			rm[v] = 1
		} else {
			rm[v]++
		}

		if min > v {
			min = v
		}
		if max < v {
			max = v
		}
	}

	c1 := dataframe.NewColumn("LATENCY-MS")
	c2 := dataframe.NewColumn("COUNT")
	cur := min
	for {
		c1.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", int64(cur))))
		v, ok := rm[cur]
		if ok {
			c2.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", v)))
		} else {
			c2.PushBack(dataframe.NewStringValue("0"))
		}
		cur += 10
		if cur-10 == max { // was last point
			break
		}
	}
	fr := dataframe.New()
	if err := fr.AddColumn(c1); err != nil {
		plog.Fatal(err)
	}
	if err := fr.AddColumn(c2); err != nil {
		plog.Fatal(err)
	}
	if err := fr.CSV(cfg.DataLatencyDistributionAll); err != nil {
		plog.Fatal(err)
	}
}

func saveDataLatencyThroughputTimeseries(cfg Config, st report.Stats, tsToClientN map[int64]int) {
	c1 := dataframe.NewColumn("UNIX-SECOND")
	c2 := dataframe.NewColumn("CONTROL-CLIENT-NUM")
	c3 := dataframe.NewColumn("MIN-LATENCY-MS")
	c4 := dataframe.NewColumn("AVG-LATENCY-MS")
	c5 := dataframe.NewColumn("MAX-LATENCY-MS")
	c6 := dataframe.NewColumn("AVG-THROUGHPUT")
	for i := range st.TimeSeries {
		// this Timestamp is unix seconds
		c1.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", st.TimeSeries[i].Timestamp)))

		if len(tsToClientN) == 0 {
			c2.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", cfg.Step2.Clients)))
		} else {
			c2.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", tsToClientN[st.TimeSeries[i].Timestamp])))
		}

		c3.PushBack(dataframe.NewStringValue(fmt.Sprintf("%f", toMillisecond(st.TimeSeries[i].MinLatency))))
		c4.PushBack(dataframe.NewStringValue(fmt.Sprintf("%f", toMillisecond(st.TimeSeries[i].AvgLatency))))
		c5.PushBack(dataframe.NewStringValue(fmt.Sprintf("%f", toMillisecond(st.TimeSeries[i].MaxLatency))))

		c6.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", st.TimeSeries[i].ThroughPut)))
	}

	fr := dataframe.New()
	if err := fr.AddColumn(c1); err != nil {
		plog.Fatal(err)
	}
	if err := fr.AddColumn(c2); err != nil {
		plog.Fatal(err)
	}
	if err := fr.AddColumn(c3); err != nil {
		plog.Fatal(err)
	}
	if err := fr.AddColumn(c4); err != nil {
		plog.Fatal(err)
	}
	if err := fr.AddColumn(c5); err != nil {
		plog.Fatal(err)
	}
	if err := fr.AddColumn(c6); err != nil {
		plog.Fatal(err)
	}

	if err := fr.CSV(cfg.DataLatencyThroughputTimeseries); err != nil {
		plog.Fatal(err)
	}

	// aggregate latency by the number of keys
	tss := processTimeSeries(st.TimeSeries, 1000, cfg.Step2.TotalRequests)
	ctt1 := dataframe.NewColumn("KEYS")
	ctt2 := dataframe.NewColumn("MIN-LATENCY-MS")
	ctt3 := dataframe.NewColumn("AVG-LATENCY-MS")
	ctt4 := dataframe.NewColumn("MAX-LATENCY-MS")
	for i := range tss {
		ctt1.PushBack(dataframe.NewStringValue(tss[i].keyNum))
		ctt2.PushBack(dataframe.NewStringValue(fmt.Sprintf("%f", toMillisecond(tss[i].minLat))))
		ctt3.PushBack(dataframe.NewStringValue(fmt.Sprintf("%f", toMillisecond(tss[i].avgLat))))
		ctt4.PushBack(dataframe.NewStringValue(fmt.Sprintf("%f", toMillisecond(tss[i].maxLat))))
	}

	frr := dataframe.New()
	if err := frr.AddColumn(ctt1); err != nil {
		plog.Fatal(err)
	}
	if err := frr.AddColumn(ctt2); err != nil {
		plog.Fatal(err)
	}
	if err := frr.AddColumn(ctt3); err != nil {
		plog.Fatal(err)
	}
	if err := frr.AddColumn(ctt4); err != nil {
		plog.Fatal(err)
	}

	if err := frr.CSV(cfg.DataLatencyByKeyNumber); err != nil {
		plog.Fatal(err)
	}
}

func generateReport(cfg Config, h []ReqHandler, reqDone func(), reqGen func(chan<- request)) {
	b := newBenchmark(cfg.Step2.TotalRequests, cfg.Step2.Clients, h, reqDone, reqGen)
	b.startRequests()
	b.waitAll()

	printStats(b.stats)
	saveAllStats(cfg, b.stats, nil)
}

func saveAllStats(cfg Config, stats report.Stats, tsToClientN map[int64]int) {
	// cfg.DataLatencyDistributionSummary
	saveDataLatencyDistributionSummary(cfg, stats)

	// cfg.DataLatencyDistributionPercentile
	saveDataLatencyDistributionPercentile(cfg, stats)

	// cfg.DataLatencyDistributionAll
	saveDataLatencyDistributionAll(cfg, stats)

	// cfg.DataLatencyThroughputTimeseries
	saveDataLatencyThroughputTimeseries(cfg, stats, tsToClientN)
}
<|endoftext|>


<commit_before>package plugin

import (
	"strings"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestMetric(t *testing.T) {
	Convey("error on invalid pulse content type", t, func() {
		m := []PluginMetricType{
			*NewPluginMetricType([]string{"foo", "bar"}, 1),
			*NewPluginMetricType([]string{"foo", "baz"}, 2),
		}
		a, c, e := MarshallPluginMetricTypes("foo", m)
		So(e, ShouldNotBeNil)
		So(e.Error(), ShouldEqual, "invlaid pulse content type: foo")
		So(a, ShouldBeNil)
		So(c, ShouldEqual, "")
	})

	Convey("error on empty metric slice", t, func() {
		m := []PluginMetricType{}
		a, c, e := MarshallPluginMetricTypes("foo", m)
		So(e, ShouldNotBeNil)
		So(e.Error(), ShouldEqual, "attempt to marshall empty slice of metrics: foo")
		So(a, ShouldBeNil)
		So(c, ShouldEqual, "")
	})

	Convey("marshall using pulse.* default to pulse.gob", t, func() {
		m := []PluginMetricType{
			*NewPluginMetricType([]string{"foo", "bar"}, 1),
			*NewPluginMetricType([]string{"foo", "baz"}, "2"),
		}
		a, c, e := MarshallPluginMetricTypes("pulse.*", m)
		So(e, ShouldBeNil)
		So(a, ShouldNotBeNil)
		So(len(a), ShouldBeGreaterThan, 0)
		So(c, ShouldEqual, "pulse.gob")

		Convey("unmarshal pulse.gob", func() {
			m, e = UnmarshallPluginMetricTypes("pulse.gob", a)
			So(e, ShouldBeNil)
			So(strings.Join(m[0].Namespace(), "/"), ShouldResemble, "foo/bar")
			So(m[0].Data(), ShouldResemble, 1)
			So(strings.Join(m[1].Namespace(), "/"), ShouldResemble, "foo/baz")
			So(m[1].Data(), ShouldResemble, "2")
		})

	})

	Convey("marshall using pulse.gob", t, func() {
		m := []PluginMetricType{
			*NewPluginMetricType([]string{"foo", "bar"}, 1),
			*NewPluginMetricType([]string{"foo", "baz"}, "2"),
		}
		a, c, e := MarshallPluginMetricTypes("pulse.gob", m)
		So(e, ShouldBeNil)
		So(a, ShouldNotBeNil)
		So(len(a), ShouldBeGreaterThan, 0)
		So(c, ShouldEqual, "pulse.gob")

		Convey("unmarshal pulse.gob", func() {
			m, e = UnmarshallPluginMetricTypes("pulse.gob", a)
			So(e, ShouldBeNil)
			So(strings.Join(m[0].Namespace(), "/"), ShouldResemble, "foo/bar")
			So(m[0].Data(), ShouldResemble, 1)
			So(strings.Join(m[1].Namespace(), "/"), ShouldResemble, "foo/baz")
			So(m[1].Data(), ShouldResemble, "2")
		})

		Convey("error on bad corrupt data", func() {
			a = []byte{1, 0, 1, 1, 1, 1, 1, 0, 0, 1}
			m, e = UnmarshallPluginMetricTypes("pulse.gob", a)
			So(e, ShouldNotBeNil)
			So(e.Error(), ShouldResemble, "gob: decoding into local type *[]plugin.PluginMetricType, received remote type unknown type")
		})
	})

	Convey("marshall using pulse.json", t, func() {
		m := []PluginMetricType{
			*NewPluginMetricType([]string{"foo", "bar"}, 1),
			*NewPluginMetricType([]string{"foo", "baz"}, "2"),
		}
		a, c, e := MarshallPluginMetricTypes("pulse.json", m)
		So(e, ShouldBeNil)
		So(a, ShouldNotBeNil)
		So(len(a), ShouldBeGreaterThan, 0)
		So(c, ShouldEqual, "pulse.json")

		Convey("unmarshal pulse.json", func() {
			m, e = UnmarshallPluginMetricTypes("pulse.json", a)
			So(e, ShouldBeNil)
			So(strings.Join(m[0].Namespace(), "/"), ShouldResemble, "foo/bar")
			So(m[0].Data(), ShouldResemble, float64(1))
			So(strings.Join(m[1].Namespace(), "/"), ShouldResemble, "foo/baz")
			So(m[1].Data(), ShouldResemble, "2")
		})

		Convey("error on bad corrupt data", func() {
			a = []byte{1, 0, 1, 1, 1, 1, 1, 0, 0, 1}
			m, e = UnmarshallPluginMetricTypes("pulse.json", a)
			So(e, ShouldNotBeNil)
			So(e.Error(), ShouldResemble, "invalid character '\\x01' looking for beginning of value")
		})
	})

	Convey("error on unmarshall using bad content type", t, func() {
		m := []PluginMetricType{
			*NewPluginMetricType([]string{"foo", "bar"}, 1),
			*NewPluginMetricType([]string{"foo", "baz"}, "2"),
		}
		a, c, e := MarshallPluginMetricTypes("pulse.json", m)
		So(e, ShouldBeNil)
		So(a, ShouldNotBeNil)
		So(len(a), ShouldBeGreaterThan, 0)
		So(c, ShouldEqual, "pulse.json")

		m, e = UnmarshallPluginMetricTypes("pulse.wat", a)
		So(e, ShouldNotBeNil)
		So(e.Error(), ShouldEqual, "invlaid pulse content type for unmarshalling: pulse.wat")
		So(m, ShouldBeNil)
	})

	Convey("swap from pulse.gob to pulse.json", t, func() {
		m := []PluginMetricType{
			*NewPluginMetricType([]string{"foo", "bar"}, 1),
			*NewPluginMetricType([]string{"foo", "baz"}, "2"),
		}
		a, c, e := MarshallPluginMetricTypes("pulse.gob", m)
		So(e, ShouldBeNil)
		So(a, ShouldNotBeNil)
		So(len(a), ShouldBeGreaterThan, 0)
		So(c, ShouldEqual, "pulse.gob")

		b, c, e := SwapPluginMetricContentType(c, "pulse.json", a)
		So(e, ShouldBeNil)
		So(c, ShouldResemble, "pulse.json")
		So(b, ShouldNotBeNil)

		m, e = UnmarshallPluginMetricTypes(c, b)
		So(e, ShouldBeNil)
		So(strings.Join(m[0].Namespace(), "/"), ShouldResemble, "foo/bar")
		So(m[0].Data(), ShouldResemble, float64(1))
		So(strings.Join(m[1].Namespace(), "/"), ShouldResemble, "foo/baz")
		So(m[1].Data(), ShouldResemble, "2")
	})

	Convey("swap from pulse.json to pulse.*", t, func() {
		m := []PluginMetricType{
			*NewPluginMetricType([]string{"foo", "bar"}, 1),
			*NewPluginMetricType([]string{"foo", "baz"}, "2"),
		}
		a, c, e := MarshallPluginMetricTypes("pulse.json", m)
		So(e, ShouldBeNil)
		So(a, ShouldNotBeNil)
		So(len(a), ShouldBeGreaterThan, 0)
		So(c, ShouldEqual, "pulse.json")

		b, c, e := SwapPluginMetricContentType(c, "pulse.*", a)
		So(e, ShouldBeNil)
		So(c, ShouldResemble, "pulse.gob")
		So(b, ShouldNotBeNil)

		m, e = UnmarshallPluginMetricTypes(c, b)
		So(e, ShouldBeNil)
		So(strings.Join(m[0].Namespace(), "/"), ShouldResemble, "foo/bar")
		So(m[0].Data(), ShouldResemble, float64(1))
		So(strings.Join(m[1].Namespace(), "/"), ShouldResemble, "foo/baz")
		So(m[1].Data(), ShouldResemble, "2")
	})

	Convey("swap from pulse.json to pulse.gob", t, func() {
		m := []PluginMetricType{
			*NewPluginMetricType([]string{"foo", "bar"}, 1),
			*NewPluginMetricType([]string{"foo", "baz"}, "2"),
		}
		a, c, e := MarshallPluginMetricTypes("pulse.json", m)
		So(e, ShouldBeNil)
		So(a, ShouldNotBeNil)
		So(len(a), ShouldBeGreaterThan, 0)
		So(c, ShouldEqual, "pulse.json")

		b, c, e := SwapPluginMetricContentType(c, "pulse.gob", a)
		So(e, ShouldBeNil)
		So(c, ShouldResemble, "pulse.gob")
		So(b, ShouldNotBeNil)

		m, e = UnmarshallPluginMetricTypes(c, b)
		So(e, ShouldBeNil)
		So(strings.Join(m[0].Namespace(), "/"), ShouldResemble, "foo/bar")
		So(m[0].Data(), ShouldResemble, float64(1))
		So(strings.Join(m[1].Namespace(), "/"), ShouldResemble, "foo/baz")
		So(m[1].Data(), ShouldResemble, "2")
	})

	Convey("error on bad content type to swap", t, func() {
		m := []PluginMetricType{
			*NewPluginMetricType([]string{"foo", "bar"}, 1),
			*NewPluginMetricType([]string{"foo", "baz"}, "2"),
		}
		a, c, e := MarshallPluginMetricTypes("pulse.json", m)
		So(e, ShouldBeNil)
		So(a, ShouldNotBeNil)
		So(len(a), ShouldBeGreaterThan, 0)
		So(c, ShouldEqual, "pulse.json")

		b, c, e := SwapPluginMetricContentType("pulse.wat", "pulse.gob", a)
		So(e, ShouldNotBeNil)
		So(e.Error(), ShouldResemble, "invlaid pulse content type for unmarshalling: pulse.wat")
		So(b, ShouldBeNil)
	})
}
<commit_msg>Slightly increasing the coverage for metric.go<commit_after>package plugin

import (
	"github.com/intelsdi-x/pulse/core/cdata"
	"github.com/intelsdi-x/pulse/core/ctypes"
	"strings"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestMetric(t *testing.T) {
	Convey("error on invalid pulse content type", t, func() {
		m := []PluginMetricType{
			*NewPluginMetricType([]string{"foo", "bar"}, 1),
			*NewPluginMetricType([]string{"foo", "baz"}, 2),
		}
		a, c, e := MarshallPluginMetricTypes("foo", m)
		m[0].Version_ = 1
		m[0].AddData(3)
		configNewNode := cdata.NewNode()
		configNewNode.AddItem("user", ctypes.ConfigValueStr{Value: "foo"})
		m[0].Config_ = configNewNode
		So(e, ShouldNotBeNil)
		So(e.Error(), ShouldEqual, "invlaid pulse content type: foo")
		So(a, ShouldBeNil)
		So(c, ShouldEqual, "")
		So(m[0].Version(), ShouldResemble, 1)
		So(m[0].Data(), ShouldResemble, 3)
		So(m[0].Config(), ShouldNotBeNil)
	})

	Convey("error on empty metric slice", t, func() {
		m := []PluginMetricType{}
		a, c, e := MarshallPluginMetricTypes("foo", m)
		So(e, ShouldNotBeNil)
		So(e.Error(), ShouldEqual, "attempt to marshall empty slice of metrics: foo")
		So(a, ShouldBeNil)
		So(c, ShouldEqual, "")
	})

	Convey("marshall using pulse.* default to pulse.gob", t, func() {
		m := []PluginMetricType{
			*NewPluginMetricType([]string{"foo", "bar"}, 1),
			*NewPluginMetricType([]string{"foo", "baz"}, "2"),
		}
		a, c, e := MarshallPluginMetricTypes("pulse.*", m)
		So(e, ShouldBeNil)
		So(a, ShouldNotBeNil)
		So(len(a), ShouldBeGreaterThan, 0)
		So(c, ShouldEqual, "pulse.gob")

		Convey("unmarshal pulse.gob", func() {
			m, e = UnmarshallPluginMetricTypes("pulse.gob", a)
			So(e, ShouldBeNil)
			So(strings.Join(m[0].Namespace(), "/"), ShouldResemble, "foo/bar")
			So(m[0].Data(), ShouldResemble, 1)
			So(strings.Join(m[1].Namespace(), "/"), ShouldResemble, "foo/baz")
			So(m[1].Data(), ShouldResemble, "2")
		})

	})

	Convey("marshall using pulse.gob", t, func() {
		m := []PluginMetricType{
			*NewPluginMetricType([]string{"foo", "bar"}, 1),
			*NewPluginMetricType([]string{"foo", "baz"}, "2"),
		}
		a, c, e := MarshallPluginMetricTypes("pulse.gob", m)
		So(e, ShouldBeNil)
		So(a, ShouldNotBeNil)
		So(len(a), ShouldBeGreaterThan, 0)
		So(c, ShouldEqual, "pulse.gob")

		Convey("unmarshal pulse.gob", func() {
			m, e = UnmarshallPluginMetricTypes("pulse.gob", a)
			So(e, ShouldBeNil)
			So(strings.Join(m[0].Namespace(), "/"), ShouldResemble, "foo/bar")
			So(m[0].Data(), ShouldResemble, 1)
			So(strings.Join(m[1].Namespace(), "/"), ShouldResemble, "foo/baz")
			So(m[1].Data(), ShouldResemble, "2")
		})

		Convey("error on bad corrupt data", func() {
			a = []byte{1, 0, 1, 1, 1, 1, 1, 0, 0, 1}
			m, e = UnmarshallPluginMetricTypes("pulse.gob", a)
			So(e, ShouldNotBeNil)
			So(e.Error(), ShouldResemble, "gob: decoding into local type *[]plugin.PluginMetricType, received remote type unknown type")
		})
	})

	Convey("marshall using pulse.json", t, func() {
		m := []PluginMetricType{
			*NewPluginMetricType([]string{"foo", "bar"}, 1),
			*NewPluginMetricType([]string{"foo", "baz"}, "2"),
		}
		a, c, e := MarshallPluginMetricTypes("pulse.json", m)
		So(e, ShouldBeNil)
		So(a, ShouldNotBeNil)
		So(len(a), ShouldBeGreaterThan, 0)
		So(c, ShouldEqual, "pulse.json")

		Convey("unmarshal pulse.json", func() {
			m, e = UnmarshallPluginMetricTypes("pulse.json", a)
			So(e, ShouldBeNil)
			So(strings.Join(m[0].Namespace(), "/"), ShouldResemble, "foo/bar")
			So(m[0].Data(), ShouldResemble, float64(1))
			So(strings.Join(m[1].Namespace(), "/"), ShouldResemble, "foo/baz")
			So(m[1].Data(), ShouldResemble, "2")
		})

		Convey("error on bad corrupt data", func() {
			a = []byte{1, 0, 1, 1, 1, 1, 1, 0, 0, 1}
			m, e = UnmarshallPluginMetricTypes("pulse.json", a)
			So(e, ShouldNotBeNil)
			So(e.Error(), ShouldResemble, "invalid character '\\x01' looking for beginning of value")
		})
	})

	Convey("error on unmarshall using bad content type", t, func() {
		m := []PluginMetricType{
			*NewPluginMetricType([]string{"foo", "bar"}, 1),
			*NewPluginMetricType([]string{"foo", "baz"}, "2"),
		}
		a, c, e := MarshallPluginMetricTypes("pulse.json", m)
		So(e, ShouldBeNil)
		So(a, ShouldNotBeNil)
		So(len(a), ShouldBeGreaterThan, 0)
		So(c, ShouldEqual, "pulse.json")

		m, e = UnmarshallPluginMetricTypes("pulse.wat", a)
		So(e, ShouldNotBeNil)
		So(e.Error(), ShouldEqual, "invlaid pulse content type for unmarshalling: pulse.wat")
		So(m, ShouldBeNil)
	})

	Convey("swap from pulse.gob to pulse.json", t, func() {
		m := []PluginMetricType{
			*NewPluginMetricType([]string{"foo", "bar"}, 1),
			*NewPluginMetricType([]string{"foo", "baz"}, "2"),
		}
		a, c, e := MarshallPluginMetricTypes("pulse.gob", m)
		So(e, ShouldBeNil)
		So(a, ShouldNotBeNil)
		So(len(a), ShouldBeGreaterThan, 0)
		So(c, ShouldEqual, "pulse.gob")

		b, c, e := SwapPluginMetricContentType(c, "pulse.json", a)
		So(e, ShouldBeNil)
		So(c, ShouldResemble, "pulse.json")
		So(b, ShouldNotBeNil)

		m, e = UnmarshallPluginMetricTypes(c, b)
		So(e, ShouldBeNil)
		So(strings.Join(m[0].Namespace(), "/"), ShouldResemble, "foo/bar")
		So(m[0].Data(), ShouldResemble, float64(1))
		So(strings.Join(m[1].Namespace(), "/"), ShouldResemble, "foo/baz")
		So(m[1].Data(), ShouldResemble, "2")
	})

	Convey("swap from pulse.json to pulse.*", t, func() {
		m := []PluginMetricType{
			*NewPluginMetricType([]string{"foo", "bar"}, 1),
			*NewPluginMetricType([]string{"foo", "baz"}, "2"),
		}
		a, c, e := MarshallPluginMetricTypes("pulse.json", m)
		So(e, ShouldBeNil)
		So(a, ShouldNotBeNil)
		So(len(a), ShouldBeGreaterThan, 0)
		So(c, ShouldEqual, "pulse.json")

		b, c, e := SwapPluginMetricContentType(c, "pulse.*", a)
		So(e, ShouldBeNil)
		So(c, ShouldResemble, "pulse.gob")
		So(b, ShouldNotBeNil)

		m, e = UnmarshallPluginMetricTypes(c, b)
		So(e, ShouldBeNil)
		So(strings.Join(m[0].Namespace(), "/"), ShouldResemble, "foo/bar")
		So(m[0].Data(), ShouldResemble, float64(1))
		So(strings.Join(m[1].Namespace(), "/"), ShouldResemble, "foo/baz")
		So(m[1].Data(), ShouldResemble, "2")
	})

	Convey("swap from pulse.json to pulse.gob", t, func() {
		m := []PluginMetricType{
			*NewPluginMetricType([]string{"foo", "bar"}, 1),
			*NewPluginMetricType([]string{"foo", "baz"}, "2"),
		}
		a, c, e := MarshallPluginMetricTypes("pulse.json", m)
		So(e, ShouldBeNil)
		So(a, ShouldNotBeNil)
		So(len(a), ShouldBeGreaterThan, 0)
		So(c, ShouldEqual, "pulse.json")

		b, c, e := SwapPluginMetricContentType(c, "pulse.gob", a)
		So(e, ShouldBeNil)
		So(c, ShouldResemble, "pulse.gob")
		So(b, ShouldNotBeNil)

		m, e = UnmarshallPluginMetricTypes(c, b)
		So(e, ShouldBeNil)
		So(strings.Join(m[0].Namespace(), "/"), ShouldResemble, "foo/bar")
		So(m[0].Data(), ShouldResemble, float64(1))
		So(strings.Join(m[1].Namespace(), "/"), ShouldResemble, "foo/baz")
		So(m[1].Data(), ShouldResemble, "2")
	})

	Convey("error on bad content type to swap", t, func() {
		m := []PluginMetricType{
			*NewPluginMetricType([]string{"foo", "bar"}, 1),
			*NewPluginMetricType([]string{"foo", "baz"}, "2"),
		}
		a, c, e := MarshallPluginMetricTypes("pulse.json", m)
		So(e, ShouldBeNil)
		So(a, ShouldNotBeNil)
		So(len(a), ShouldBeGreaterThan, 0)
		So(c, ShouldEqual, "pulse.json")

		b, c, e := SwapPluginMetricContentType("pulse.wat", "pulse.gob", a)
		So(e, ShouldNotBeNil)
		So(e.Error(), ShouldResemble, "invlaid pulse content type for unmarshalling: pulse.wat")
		So(b, ShouldBeNil)
	})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/Financial-Times/base-ft-rw-app-go/baseftrwapp"
	"github.com/Financial-Times/go-fthealth/v1a"
	"github.com/Financial-Times/http-handlers-go/httphandlers"
	"github.com/Financial-Times/public-brands-api/brands"
	status "github.com/Financial-Times/service-status-go/httphandlers"
	log "github.com/Sirupsen/logrus"
	"github.com/gorilla/mux"
	"github.com/jawher/mow.cli"
	"github.com/jmcvetta/neoism"
	"github.com/rcrowley/go-metrics"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
)

func main() {
	app := cli.App("public-brands-api", "A public RESTful API for accessing Brands in neo4j")
	neoURL := app.StringOpt("neo-url", "http://localhost:7474/db/data", "neo4j endpoint URL")
	port := app.StringOpt("port", "8080", "Port to listen on")
	logLevel := app.StringOpt("log-level", "INFO", "Logging level (DEBUG, INFO, WARN, ERROR)")
	env := app.StringOpt("env", "local", "environment this app is running in")
	graphiteTCPAddress := app.StringOpt("graphiteTCPAddress", "",
		"Graphite TCP address, e.g. graphite.ft.com:2003. Leave as default if you do NOT want to output to graphite (e.g. if running locally)")
	graphitePrefix := app.StringOpt("graphitePrefix", "",
		"Prefix to use. Should start with content, include the environment, and the host name. e.g. content.test.public.brands.api.ftaps59382-law1a-eu-t")
	logMetrics := app.BoolOpt("logMetrics", false, "Whether to log metrics. Set to true if running locally and you want metrics output")
	cacheDuration := app.StringOpt("cache-duration", "1h", "Duration Get requests should be cached for. e.g. 2h45m would set the max-age value to '7440' seconds")

	app.Action = func() {

		baseftrwapp.OutputMetricsIfRequired(*graphiteTCPAddress, *graphitePrefix, *logMetrics)
		if *env != "local" {
			f, err := os.OpenFile("/var/log/apps/public-brands-api-go-app.log", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755)
			if err == nil {
				log.SetOutput(f)
				log.SetFormatter(&log.TextFormatter{DisableColors: true})
			} else {
				log.Fatalf("Failed to initialise log file, %v", err)
			}

			defer f.Close()
		}

		log.Infof("public-brands-api will listen on port: %s, connecting to: %s", *port, *neoURL)
		runServer(*neoURL, *port, *cacheDuration, *env)

	}
	setLogLevel(strings.ToUpper(*logLevel))
	log.Infof("Application started with args %s", os.Args)
	app.Run(os.Args)
}

func runServer(neoURL string, port string, cacheDuration string, env string) {

	if duration, durationErr := time.ParseDuration(cacheDuration); durationErr != nil {
		log.Fatalf("Failed to parse cache duration string, %v", durationErr)
	} else {
		brands.CacheControlHeader = fmt.Sprintf("max-age=%s, public", strconv.FormatFloat(duration.Seconds(), 'f', 0, 64))
	}

	db, err := neoism.Connect(neoURL)
	db.Session.Client = &http.Client{Transport: &http.Transport{MaxIdleConnsPerHost: 100}}
	if err != nil {
		log.Fatalf("Error connecting to neo4j %s", err)
	}
	brands.BrandsDriver = brands.NewCypherDriver(db, env)

	servicesRouter := mux.NewRouter()

	servicesRouter.HandleFunc("/brands/{uuid}", brands.GetBrand).Methods("GET")
	servicesRouter.HandleFunc("/brands/{uuid}", brands.MethodNotAllowedHandler)

	var monitoringRouter http.Handler = servicesRouter
	monitoringRouter = httphandlers.TransactionAwareRequestLoggingHandler(log.StandardLogger(), monitoringRouter)
	monitoringRouter = httphandlers.HTTPMetricsHandler(metrics.DefaultRegistry, monitoringRouter)

	http.HandleFunc("/__health", v1a.Handler("PublicBrandsRead Healthchecks",
		"Checks for accessing neo4j", brands.HealthCheck()))
	http.HandleFunc("/health", v1a.Handler("PublicBrandsRead Healthchecks",
		"Checks for accessing neo4j", brands.HealthCheck()))
	http.HandleFunc(status.PingPath, status.PingHandler)
	http.HandleFunc(status.PingPathDW, status.PingHandler)
	http.HandleFunc(status.BuildInfoPath, status.BuildInfoHandler)
	http.HandleFunc(status.BuildInfoPathDW, status.BuildInfoHandler)
	http.Handle("/", monitoringRouter)

	if err := http.ListenAndServe(":"+port, nil); err != nil {
		log.Fatalf("Unable to start server: %v", err)
	}

	if err := http.ListenAndServe(":"+port,
		httphandlers.HTTPMetricsHandler(metrics.DefaultRegistry,
			httphandlers.TransactionAwareRequestLoggingHandler(log.StandardLogger(), monitoringRouter))); err != nil {
		log.Fatalf("Unable to start server: %v", err)
	}
}

func setLogLevel(level string) {
	switch level {
	case "DEBUG":
		log.SetLevel(log.DebugLevel)
	case "INFO":
		log.SetLevel(log.InfoLevel)
	case "WARN":
		log.SetLevel(log.WarnLevel)
	case "ERROR":
		log.SetLevel(log.ErrorLevel)
	default:
		log.Errorf("Requested log level %s is not supported, will default to INFO level", level)
		log.SetLevel(log.InfoLevel)
	}
	log.Debugf("Logging level set to %s", level)
}
<commit_msg>Reconfigured params to accept environment variables<commit_after>package main

import (
	"fmt"
	"github.com/Financial-Times/base-ft-rw-app-go/baseftrwapp"
	"github.com/Financial-Times/go-fthealth/v1a"
	"github.com/Financial-Times/http-handlers-go/httphandlers"
	"github.com/Financial-Times/public-brands-api/brands"
	status "github.com/Financial-Times/service-status-go/httphandlers"
	log "github.com/Sirupsen/logrus"
	"github.com/gorilla/mux"
	"github.com/jawher/mow.cli"
	"github.com/jmcvetta/neoism"
	"github.com/rcrowley/go-metrics"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
)

func main() {
	app := cli.App("public-brands-api", "A public RESTful API for accessing Brands in neo4j")

	neoURL := app.String(cli.StringOpt{
		Name:   "neo-url",
		Value:  "http://localhost:7474/db/data",
		Desc:   "neo4j endpoint URL",
		EnvVar: "NEO_URL",
	})
	port := app.String(cli.StringOpt{
		Name:   "port",
		Value:  "8080",
		Desc:   "Port to listen on",
		EnvVar: "PORT",
	})
	logLevel := app.String(cli.StringOpt{
		Name:   "log-level",
		Value:  "8080",
		Desc:   "Logging level (DEBUG, INFO, WARN, ERROR)",
		EnvVar: "LOG_LEVEL",
	})
	env := app.String(cli.StringOpt{
		Name: "env",
		Value: "local",
		Desc:  "environment this app is running in",
		EnvVar: "ENV",
	})
	graphiteTCPAddress := app.String(cli.StringOpt{
		Name: "graphiteTCPAddress",
		Value: "",
		Desc:  "Graphite TCP address, e.g. graphite.ft.com:2003. Leave as default if you do NOT want to output to graphite (e.g. if running locally)",
		EnvVar: "GRAPHITE_ADDRESS",
	})
	graphitePrefix := app.String(cli.StringOpt{
		Name: "graphitePrefix",
		Value: "",
		Desc: "Prefix to use. Should start with content, include the environment, and the host name. e.g. content.test.public.brands.api.ftaps59382-law1a-eu-t",
		EnvVar: "GRAPHITE_PREFIX",
	})
	logMetrics := app.Bool(cli.BoolOpt{
		Name: "logMetrics",
		Value: false,
		Desc: "Whether to log metrics. Set to true if running locally and you want metrics output",
		EnvVar: "LOG_METRICS",
	})
	cacheDuration := app.String(cli.StringOpt{
		Name: "cache-duration",
		Value: "1h",
		Desc: "Duration Get requests should be cached for. e.g. 2h45m would set the max-age value to '7440' seconds",
		EnvVar: "CACHE_DURATION",
	})

	app.Action = func() {

		baseftrwapp.OutputMetricsIfRequired(*graphiteTCPAddress, *graphitePrefix, *logMetrics)
		if *env != "local" {
			f, err := os.OpenFile("/var/log/apps/public-brands-api-go-app.log", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755)
			if err == nil {
				log.SetOutput(f)
				log.SetFormatter(&log.TextFormatter{DisableColors: true})
			} else {
				log.Fatalf("Failed to initialise log file, %v", err)
			}

			defer f.Close()
		}

		log.Infof("public-brands-api will listen on port: %s, connecting to: %s", *port, *neoURL)
		runServer(*neoURL, *port, *cacheDuration, *env)

	}
	setLogLevel(strings.ToUpper(*logLevel))
	log.Infof("Application started with args %s", os.Args)
	app.Run(os.Args)
}

func runServer(neoURL string, port string, cacheDuration string, env string) {

	if duration, durationErr := time.ParseDuration(cacheDuration); durationErr != nil {
		log.Fatalf("Failed to parse cache duration string, %v", durationErr)
	} else {
		brands.CacheControlHeader = fmt.Sprintf("max-age=%s, public", strconv.FormatFloat(duration.Seconds(), 'f', 0, 64))
	}

	db, err := neoism.Connect(neoURL)
	db.Session.Client = &http.Client{Transport: &http.Transport{MaxIdleConnsPerHost: 100}}
	if err != nil {
		log.Fatalf("Error connecting to neo4j %s", err)
	}
	brands.BrandsDriver = brands.NewCypherDriver(db, env)

	servicesRouter := mux.NewRouter()

	servicesRouter.HandleFunc("/brands/{uuid}", brands.GetBrand).Methods("GET")
	servicesRouter.HandleFunc("/brands/{uuid}", brands.MethodNotAllowedHandler)

	var monitoringRouter http.Handler = servicesRouter
	monitoringRouter = httphandlers.TransactionAwareRequestLoggingHandler(log.StandardLogger(), monitoringRouter)
	monitoringRouter = httphandlers.HTTPMetricsHandler(metrics.DefaultRegistry, monitoringRouter)

	http.HandleFunc("/__health", v1a.Handler("PublicBrandsRead Healthchecks",
		"Checks for accessing neo4j", brands.HealthCheck()))
	http.HandleFunc("/health", v1a.Handler("PublicBrandsRead Healthchecks",
		"Checks for accessing neo4j", brands.HealthCheck()))
	http.HandleFunc(status.PingPath, status.PingHandler)
	http.HandleFunc(status.PingPathDW, status.PingHandler)
	http.HandleFunc(status.BuildInfoPath, status.BuildInfoHandler)
	http.HandleFunc(status.BuildInfoPathDW, status.BuildInfoHandler)
	http.Handle("/", monitoringRouter)

	if err := http.ListenAndServe(":"+port, nil); err != nil {
		log.Fatalf("Unable to start server: %v", err)
	}

	if err := http.ListenAndServe(":"+port,
		httphandlers.HTTPMetricsHandler(metrics.DefaultRegistry,
			httphandlers.TransactionAwareRequestLoggingHandler(log.StandardLogger(), monitoringRouter))); err != nil {
		log.Fatalf("Unable to start server: %v", err)
	}
}

func setLogLevel(level string) {
	switch level {
	case "DEBUG":
		log.SetLevel(log.DebugLevel)
	case "INFO":
		log.SetLevel(log.InfoLevel)
	case "WARN":
		log.SetLevel(log.WarnLevel)
	case "ERROR":
		log.SetLevel(log.ErrorLevel)
	default:
		log.Errorf("Requested log level %s is not supported, will default to INFO level", level)
		log.SetLevel(log.InfoLevel)
	}
	log.Debugf("Logging level set to %s", level)
}
<|endoftext|>


<commit_before>package lapi

import (
	"fmt"
	"net/http"
	"sync"

	"github.com/goline/errors"
)

// App is a central application
type App interface {
	AppLoader
	AppRunner
	AppRouter
	AppRescuer
	AppConfigger
	ContainerAware
	http.Handler
}

// AppLoader handles application's loader
type AppLoader interface {
	// WithLoader allows to register application's loader
	WithLoader(loader Loader) App
}

type AppConfigger interface {
	// Config returns application's configuration
	Config() interface{}

	// WithConfig sets application's config
	WithConfig(config interface{}) App
}

// AppRouter handles router
type AppRouter interface {
	// Router returns an instance of Router
	Router() Router

	// WithRouter sets router
	WithRouter(router Router) App
}

// AppRescuer manages error handler
type AppRescuer interface {
	// Rescuer returns an instance of Rescuer
	Rescuer() Rescuer

	// WithRescuer sets error handler
	WithRescuer(handler Rescuer) App
}

// AppRunner runs application
type AppRunner interface {
	// Run brings application up
	// Any errors should manage inside this method
	Run()
}

func NewApp() App {
	return &FactoryApp{
		loaders: make([]Loader, 0),
		router:  NewRouter(),
	}
}

type FactoryApp struct {
	config    interface{}
	container Container
	loaders   []Loader
	router    Router
	rescuer   Rescuer
}

func (a *FactoryApp) WithLoader(loader Loader) App {
	a.loaders = append(a.loaders, loader)
	return a
}

func (a *FactoryApp) Config() interface{} {
	return a.config
}

func (a *FactoryApp) WithConfig(config interface{}) App {
	a.config = config
	return a
}

func (a *FactoryApp) Router() Router {
	return a.router
}

func (a *FactoryApp) WithRouter(router Router) App {
	a.router = router
	return a
}

func (a *FactoryApp) Rescuer() Rescuer {
	if a.rescuer == nil {
		a.rescuer = NewRescuer()
	}

	return a.rescuer
}

func (a *FactoryApp) WithRescuer(handler Rescuer) App {
	a.rescuer = handler
	return a
}

func (a *FactoryApp) Container() Container {
	return a.container
}

func (a *FactoryApp) WithContainer(container Container) ContainerAware {
	a.container = container
	return a
}

func (a *FactoryApp) Run() {
	a.SetUp().Handle()
}

func (a *FactoryApp) SetUp() *FactoryApp {
	if a.container == nil {
		panic(errors.New(ERR_CONTAINER_NOT_DEFINED, "App requires a container to run"))
	}

	var wg sync.WaitGroup
	for _, loader := range a.loaders {
		wg.Add(1)
		go func(loader Loader) {
			defer wg.Done()
			loader.Load(a)
		}(loader)
	}
	wg.Wait()

	if a.rescuer == nil {
		a.WithRescuer(NewRescuer())
	}
	PanicOnError(a.container.Inject(a.rescuer))
	return a
}

func (a *FactoryApp) Handle() *FactoryApp {
	if a.router == nil {
		panic(errors.New(ERR_ROUTER_NOT_DEFINED, fmt.Sprint("Router is not defined yet.")))
	}

	http.Handle("/", a)
	if c, ok := a.config.(ServerConfig); ok == true {
		PanicOnError(http.ListenAndServe(c.Address(), nil))
	} else {
		panic(errors.New(ERR_SERVER_CONFIG_MISSING, fmt.Sprint("Server configuration is missing")))
	}
	return a
}

func (a *FactoryApp) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	connection := a.setUpConnection(w, r)
	defer a.forceSendResponse(connection)

	PanicOnError(a.router.Route(connection.Request()))
	for _, hook := range connection.Request().Route().Hooks() {
		PanicOnError(hook.SetUp(connection))
	}
	if connection.Response().IsSent() == true {
		return
	}

	handler := connection.Request().Route().Handler()
	if handler == nil {
		panic(errors.New(ERR_NO_HANDLER_FOUND, "No handler found"))
	}

	PanicOnError(a.container.Inject(handler))
	if h, ok := handler.(ContainerAware); ok == true {
		h.WithContainer(a.container)
	}
	result, err := handler.Handle(connection)
	for _, hook := range connection.Request().Route().Hooks() {
		PanicOnError(hook.TearDown(connection, result, err))
	}
}

func (a *FactoryApp) forceSendResponse(connection Connection) {
	if r := recover(); r != nil {
		PanicOnError(a.rescuer.Rescue(connection, r))
	}
	if connection.Response().IsSent() == false {
		connection.Response().Send()
	}
}

func (a *FactoryApp) setUpConnection(w http.ResponseWriter, r *http.Request) Connection {
	request := NewRequest(r)
	response := NewResponse(w)

	// let make request and response have same content-type and charset as default
	response.WithContentType(request.ContentType()).WithCharset(request.Charset())

	return NewConnection(request, response)
}
<commit_msg>Add default rescuer, container<commit_after>package lapi

import (
	"fmt"
	"net/http"
	"sync"

	"github.com/goline/errors"
)

// App is a central application
type App interface {
	AppLoader
	AppRunner
	AppRouter
	AppRescuer
	AppConfigger
	ContainerAware
	http.Handler
}

// AppLoader handles application's loader
type AppLoader interface {
	// WithLoader allows to register application's loader
	WithLoader(loader Loader) App
}

type AppConfigger interface {
	// Config returns application's configuration
	Config() interface{}

	// WithConfig sets application's config
	WithConfig(config interface{}) App
}

// AppRouter handles router
type AppRouter interface {
	// Router returns an instance of Router
	Router() Router

	// WithRouter sets router
	WithRouter(router Router) App
}

// AppRescuer manages error handler
type AppRescuer interface {
	// Rescuer returns an instance of Rescuer
	Rescuer() Rescuer

	// WithRescuer sets error handler
	WithRescuer(handler Rescuer) App
}

// AppRunner runs application
type AppRunner interface {
	// Run brings application up
	// Any errors should manage inside this method
	Run()
}

func NewApp() App {
	return &FactoryApp{
		loaders:   make([]Loader, 0),
		router:    NewRouter(),
		container: NewContainer(),
		rescuer:   NewRescuer(),
	}
}

type FactoryApp struct {
	config    interface{}
	container Container
	loaders   []Loader
	router    Router
	rescuer   Rescuer
}

func (a *FactoryApp) WithLoader(loader Loader) App {
	a.loaders = append(a.loaders, loader)
	return a
}

func (a *FactoryApp) Config() interface{} {
	return a.config
}

func (a *FactoryApp) WithConfig(config interface{}) App {
	a.config = config
	return a
}

func (a *FactoryApp) Router() Router {
	return a.router
}

func (a *FactoryApp) WithRouter(router Router) App {
	a.router = router
	return a
}

func (a *FactoryApp) Rescuer() Rescuer {
	if a.rescuer == nil {
		a.rescuer = NewRescuer()
	}

	return a.rescuer
}

func (a *FactoryApp) WithRescuer(handler Rescuer) App {
	a.rescuer = handler
	return a
}

func (a *FactoryApp) Container() Container {
	return a.container
}

func (a *FactoryApp) WithContainer(container Container) ContainerAware {
	a.container = container
	return a
}

func (a *FactoryApp) Run() {
	a.SetUp().Handle()
}

func (a *FactoryApp) SetUp() *FactoryApp {
	if a.container == nil {
		panic(errors.New(ERR_CONTAINER_NOT_DEFINED, "App requires a container to run"))
	}

	var wg sync.WaitGroup
	for _, loader := range a.loaders {
		wg.Add(1)
		go func(loader Loader) {
			defer wg.Done()
			loader.Load(a)
		}(loader)
	}
	wg.Wait()

	if a.rescuer == nil {
		a.WithRescuer(NewRescuer())
	}
	PanicOnError(a.container.Inject(a.rescuer))
	return a
}

func (a *FactoryApp) Handle() *FactoryApp {
	if a.router == nil {
		panic(errors.New(ERR_ROUTER_NOT_DEFINED, fmt.Sprint("Router is not defined yet.")))
	}

	http.Handle("/", a)
	if c, ok := a.config.(ServerConfig); ok == true {
		PanicOnError(http.ListenAndServe(c.Address(), nil))
	} else {
		panic(errors.New(ERR_SERVER_CONFIG_MISSING, fmt.Sprint("Server configuration is missing")))
	}
	return a
}

func (a *FactoryApp) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	connection := a.setUpConnection(w, r)
	defer a.forceSendResponse(connection)

	PanicOnError(a.router.Route(connection.Request()))
	for _, hook := range connection.Request().Route().Hooks() {
		PanicOnError(hook.SetUp(connection))
	}
	if connection.Response().IsSent() == true {
		return
	}

	handler := connection.Request().Route().Handler()
	if handler == nil {
		panic(errors.New(ERR_NO_HANDLER_FOUND, "No handler found"))
	}

	PanicOnError(a.container.Inject(handler))
	if h, ok := handler.(ContainerAware); ok == true {
		h.WithContainer(a.container)
	}
	result, err := handler.Handle(connection)
	for _, hook := range connection.Request().Route().Hooks() {
		PanicOnError(hook.TearDown(connection, result, err))
	}
}

func (a *FactoryApp) forceSendResponse(connection Connection) {
	if r := recover(); r != nil {
		PanicOnError(a.rescuer.Rescue(connection, r))
	}
	if connection.Response().IsSent() == false {
		connection.Response().Send()
	}
}

func (a *FactoryApp) setUpConnection(w http.ResponseWriter, r *http.Request) Connection {
	request := NewRequest(r)
	response := NewResponse(w)

	// let make request and response have same content-type and charset as default
	response.WithContentType(request.ContentType()).WithCharset(request.Charset())

	return NewConnection(request, response)
}
<|endoftext|>


<commit_before>package app

import (
	"bytes"
	"image"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"

	_ "image/jpeg"
	_ "image/png"

	fb "github.com/huandu/facebook"

	"appengine"
	"appengine/datastore"
	"appengine/urlfetch"
)

var FbApp = fb.New("526791527487217", "e314e5fc761425d59ea9e2666c63e108")
var aboutParams = fb.Params{
	"method":       fb.GET,
	"relative_url": "me",
	"fields":       "name,email,gender,age_range,hometown",
}

var photoParams = fb.Params{
	"method":       fb.GET,
	"relative_url": "me/picture?width=320&height=320&redirect=false",
}

func init() {
	http.HandleFunc("/static/", StaticHandler)
	http.HandleFunc("/", MainHandler)

	fb.Debug = fb.DEBUG_ALL
	FbApp.EnableAppsecretProof = true
}

func StaticHandler(w http.ResponseWriter, r *http.Request) {
	path := "." + r.URL.Path

	if f, err := os.Stat(path); err == nil && !f.IsDir() {
		http.ServeFile(w, r, path)
		return
	}

	http.NotFound(w, r)
}

func MainHandler(w http.ResponseWriter, r *http.Request) {
	switch {
	case r.URL.Path != "/":
		http.NotFound(w, r)
	case r.Method == "GET":
		handleGet(w, r)
	case r.Method == "POST":
		handlePost(w, r)
	}

	return
}

func handleGet(w http.ResponseWriter, r *http.Request) {
	context := appengine.NewContext(r)
	data, err := ioutil.ReadFile("index.html")
	check(err, context)
	w.Write(data)
}

type Log struct {
	Name     string
	Gender   string
	Party    string
	Email    string
	AgeRange string
	Hometown string
}

func handlePost(w http.ResponseWriter, r *http.Request) {
	context := appengine.NewContext(r)

	r.ParseForm()
	access_token := r.FormValue("access_token")
	party := r.FormValue("party")
	context.Infof("party = %s", party)

	session := FbApp.Session(access_token)
	session.HttpClient = urlfetch.Client(context)
	err := session.Validate()
	check(err, context)

	results, err := session.BatchApi(aboutParams, photoParams)
	check(err, context)

	aboutBatch, err := results[0].Batch()
	check(err, context)
	photoBatch, err := results[1].Batch()
	check(err, context)

	aboutResp := aboutBatch.Result
	photoResp := photoBatch.Result

	SaveAboutUser(&aboutResp, context)
	profilePicture := GetUserPhoto(&photoResp, context)

	imagebytes := addLogo(profilePicture, party, context)
	form, mime := CreateImageForm(imagebytes, context)

	url := "https://graph.facebook.com/me/photos" +
		"?access_token=" + access_token +
		"&appsecret_proof=" + session.AppsecretProof() +
		"&no_story=true"

	uploadResquest, _ := http.NewRequest("POST", url, &form)
	uploadResquest.Header.Set("Content-Type", mime)
	uploadResp, _ := session.Request(uploadResquest)
	check(err, context)

	var photoID string
	uploadResp.DecodeField("id", &photoID)
	redirectUrl := "https://facebook.com/photo.php?fbid=" + photoID + "&makeprofile=1"
	http.Redirect(w, r, redirectUrl, 303)
}

func SaveAboutUser(aboutResp *fb.Result, context appengine.Context) {
	var log Log
	aboutResp.Decode(&log)

	var ageRange map[string]string
	aboutResp.DecodeField("age_range", &ageRange)
	log.AgeRange = ageRange["min"]

	_, err := datastore.Put(context,
		datastore.NewIncompleteKey(context, "log", nil),
		&log)
	check(err, context)
}

func GetUserPhoto(photoResp *fb.Result, context appengine.Context) *image.Image {
	var dataField fb.Result
	photoResp.DecodeField("data", &dataField)

	var url string
	dataField.DecodeField("url", &url)

	client := urlfetch.Client(context)
	resp, err := client.Get(url)
	data, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	check(err, context)

	reader := bytes.NewReader(data)
	profilePicture, _, err := image.Decode(reader)
	check(err, context)

	return &profilePicture
}

func CreateImageForm(imageBytes []byte, context appengine.Context) (bytes.Buffer, string) {
	var formBuffer bytes.Buffer
	multiWriter := multipart.NewWriter(&formBuffer)

	imageField, err := multiWriter.CreateFormFile("source", "election.png")
	check(err, context)

	imageBuffer := bytes.NewBuffer(imageBytes)
	_, err = io.Copy(imageField, imageBuffer)
	check(err, context)

	messageField, err := multiWriter.CreateFormField("caption")
	check(err, context)
	_, err = messageField.Write([]byte("Created with http://www.google.com."))
	check(err, context)

	multiWriter.Close()

	return formBuffer, multiWriter.FormDataContentType()
}
<commit_msg>dynamic caption<commit_after>package app

import (
	"bytes"
	"image"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"

	_ "image/jpeg"
	_ "image/png"

	fb "github.com/huandu/facebook"

	"appengine"
	"appengine/datastore"
	"appengine/urlfetch"
)

var FbApp = fb.New("526791527487217", "e314e5fc761425d59ea9e2666c63e108")
var aboutParams = fb.Params{
	"method":       fb.GET,
	"relative_url": "me",
	"fields":       "name,email,gender,age_range,hometown",
}

var photoParams = fb.Params{
	"method":       fb.GET,
	"relative_url": "me/picture?width=320&height=320&redirect=false",
}

func init() {
	http.HandleFunc("/static/", StaticHandler)
	http.HandleFunc("/", MainHandler)

	fb.Debug = fb.DEBUG_ALL
	FbApp.EnableAppsecretProof = true
}

func StaticHandler(w http.ResponseWriter, r *http.Request) {
	path := "." + r.URL.Path

	if f, err := os.Stat(path); err == nil && !f.IsDir() {
		http.ServeFile(w, r, path)
		return
	}

	http.NotFound(w, r)
}

func MainHandler(w http.ResponseWriter, r *http.Request) {
	switch {
	case r.URL.Path != "/":
		http.NotFound(w, r)
	case r.Method == "GET":
		handleGet(w, r)
	case r.Method == "POST":
		handlePost(w, r)
	}

	return
}

func handleGet(w http.ResponseWriter, r *http.Request) {
	context := appengine.NewContext(r)
	data, err := ioutil.ReadFile("index.html")
	check(err, context)
	w.Write(data)
}

type Log struct {
	Name     string
	Gender   string
	Party    string
	Email    string
	AgeRange string
	Hometown string
}

func handlePost(w http.ResponseWriter, r *http.Request) {
	context := appengine.NewContext(r)

	r.ParseForm()
	access_token := r.FormValue("access_token")
	party := r.FormValue("party")
	context.Infof("party = %s", party)

	session := FbApp.Session(access_token)
	session.HttpClient = urlfetch.Client(context)
	err := session.Validate()
	check(err, context)

	results, err := session.BatchApi(aboutParams, photoParams)
	check(err, context)

	aboutBatch, err := results[0].Batch()
	check(err, context)
	photoBatch, err := results[1].Batch()
	check(err, context)

	aboutResp := aboutBatch.Result
	photoResp := photoBatch.Result

	SaveAboutUser(&aboutResp, context)
	profilePicture := GetUserPhoto(&photoResp, context)

	imagebytes := addLogo(profilePicture, party, context)
	form, mime := CreateImageForm(imagebytes, context, r.Host)

	url := "https://graph.facebook.com/me/photos" +
		"?access_token=" + access_token +
		"&appsecret_proof=" + session.AppsecretProof()
	//+ "&no_story=true"

	uploadResquest, _ := http.NewRequest("POST", url, &form)
	uploadResquest.Header.Set("Content-Type", mime)
	uploadResp, _ := session.Request(uploadResquest)
	check(err, context)

	var photoID string
	uploadResp.DecodeField("id", &photoID)
	redirectUrl := "https://facebook.com/photo.php?fbid=" + photoID + "&makeprofile=1&prof"
	http.Redirect(w, r, redirectUrl, 303)
}

func SaveAboutUser(aboutResp *fb.Result, context appengine.Context) {
	var log Log
	aboutResp.Decode(&log)

	var ageRange map[string]string
	aboutResp.DecodeField("age_range", &ageRange)
	log.AgeRange = ageRange["min"]

	_, err := datastore.Put(context,
		datastore.NewIncompleteKey(context, "log", nil),
		&log)
	check(err, context)
}

func GetUserPhoto(photoResp *fb.Result, context appengine.Context) *image.Image {
	var dataField fb.Result
	photoResp.DecodeField("data", &dataField)

	var url string
	dataField.DecodeField("url", &url)

	client := urlfetch.Client(context)
	resp, err := client.Get(url)
	data, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	check(err, context)

	reader := bytes.NewReader(data)
	profilePicture, _, err := image.Decode(reader)
	check(err, context)

	return &profilePicture
}

func CreateImageForm(imageBytes []byte, context appengine.Context, host string) (bytes.Buffer, string) {
	var formBuffer bytes.Buffer
	multiWriter := multipart.NewWriter(&formBuffer)

	imageField, err := multiWriter.CreateFormFile("source", "election.png")
	check(err, context)

	imageBuffer := bytes.NewBuffer(imageBytes)
	_, err = io.Copy(imageField, imageBuffer)
	check(err, context)

	messageField, err := multiWriter.CreateFormField("caption")
	check(err, context)
	_, err = messageField.Write([]byte("Created at http://" + host))
	check(err, context)

	multiWriter.Close()

	return formBuffer, multiWriter.FormDataContentType()
}
<|endoftext|>


<commit_before>package extensions

import (
	"bytes"
	cryptoRand "crypto/rand"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"io"
	"math/rand"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
	"unicode"
)

type Version struct {
	Major          int
	Minor          int
	Revision       int
	MajorString    string
	MinorString    string
	RevisionString string
	Value          string
}

type FilePath struct {
	Name string `json:"Name"`
	Path string `json:"Path"`
	Type string `json:"Type"`
}

func (obj *FilePath) ToString() (str string) {
	return obj.Path + string(os.PathSeparator) + obj.Name + " | " + obj.Type
}

func GetDecimalAndStringFromHex(twoDigitHexCode string) (decimalValue int, asciiString string, err error) {
	decoded, err := hex.DecodeString(twoDigitHexCode)
	if err != nil {
		return
	}
	decimalValue = int(decoded[0])
	asciiString = string(string(decoded))
	return
}

/*
* leftPad and rightPad just repoeat the padStr the indicated
* number of times
*
 */
func LeftPad(s string, padStr string, pLen int) string {
	return strings.Repeat(padStr, pLen) + s
}
func RightPad(s string, padStr string, pLen int) string {
	return s + strings.Repeat(padStr, pLen)
}

/* the Pad2Len functions are generally assumed to be padded with short sequences of strings
* in many cases with a single character sequence
*
* so we assume we can build the string out as if the char seq is 1 char and then
* just substr the string if it is longer than needed
*
* this means we are wasting some cpu and memory work
* but this always get us to want we want it to be
*
* in short not optimized to for massive string work
*
* If the overallLen is shorter than the original string length
* the string will be shortened to this length (substr)
*
 */
func RightPad2Len(s string, padStr string, overallLen int) string {
	var padCountInt int
	padCountInt = 1 + ((overallLen - len(padStr)) / len(padStr))
	var retStr = s + strings.Repeat(padStr, padCountInt)
	return retStr[:overallLen]
}

func LeftPad2Len(s string, padStr string, overallLen int) string {
	var padCountInt int
	padCountInt = 1 + ((overallLen - len(padStr)) / len(padStr))
	var retStr = strings.Repeat(padStr, padCountInt) + s
	return retStr[(len(retStr) - overallLen):]
}

func TrimSuffix(s, suffix string) string {
	if strings.HasSuffix(s, suffix) {
		s = s[:len(s)-len(suffix)]
	}
	return s
}

func PrintKiloBytes(bytes int64) string {

	var kilobytes float64
	kilobytes = float64(bytes / 1024)

	return fmt.Sprint(FloatToString(kilobytes, 2), " kB")
}

func PrintMegaBytes(bytes int64) string {

	var kilobytes float64
	kilobytes = float64(bytes / 1024)

	var megabytes float64
	megabytes = kilobytes / 1024 // cast to type float64

	return fmt.Sprint(FloatToString(megabytes, 2), " MB")
}

func IsPrintable(s string) bool {
	for _, c := range s {
		if (c < 32 || c > 126) && c != 10 && c != 13 && c != 9 {
			return false
		}
	}
	return true
}

func PrintZettaBytes(bytes int64) string {

	var kilobytes float64
	kilobytes = float64(bytes / 1024)

	var megabytes float64
	megabytes = (kilobytes / 1024) // cast to type float64

	var gigabytes float64
	gigabytes = (megabytes / 1024)

	var terabytes float64
	terabytes = (gigabytes / 1024)

	var petabytes float64
	petabytes = (terabytes / 1024)

	var exabytes float64
	exabytes = (petabytes / 1024)

	var zettabytes float64
	zettabytes = (exabytes / 1024)

	return fmt.Sprint(FloatToString(zettabytes, 2), " ZB")
}

func FloatToString(input_num float64, decimals int) string {
	// to convert a float number to a string
	return strconv.FormatFloat(input_num, 'f', decimals, 64)
}

func Round(x, unit float64) float64 {
	return float64(int64(x/unit+0.5)) * unit
}

func StringToInt(val string) int {

	r, err := strconv.Atoi(val)
	if err != nil {
		return 0
	}
	return r
}

func StringToFloat(val string, bitSize int) (r float64) {
	r, err := strconv.ParseFloat(val, bitSize)
	if err != nil {
		return 0
	}
	return r
}

func StringToUInt64(val string) uint64 {
	i, err := strconv.ParseUint(val, 10, 64)
	if err != nil {
		return 0
	}
	return i
}

func IntToString(val int) string {
	return strconv.Itoa(val)
}

func IntToBool(val int) bool {
	return val != 0
}

func Int32ToString(val int32) string {
	return strconv.Itoa(Int32ToInt(val))
}

func Int64ToString(val int64) string {
	return strconv.FormatInt(val, 10)
}

func Int64ToInt32(val int64) (ret int) {
	tempLong := ((val >> 32) << 32) //shift it right then left 32 bits, which zeroes the lower half of the long
	ret = (int)(val - tempLong)
	return ret
}

func Int32ToInt(val int32) (ret int) {
	tempLong := ((val >> 32) << 32) //shift it right then left 32 bits, which zeroes the lower half of the long
	ret = (int)(val - tempLong)
	return ret
}

// Int32ToDWORD converts an int32 to DWORD byte array.
func Int32ToDWORD(val int32) (value []byte) {
	buff := new(bytes.Buffer)
	err := binary.Write(buff, binary.BigEndian, val)
	if err != nil {
		return
	}
	value = buff.Bytes()
	return
}

// DWordToInt converts an 4 byte to a int32
func DWordToInt(data []byte) (num int32) {
	buff := bytes.NewReader(data)
	err := binary.Read(buff, binary.BigEndian, &num)
	if err != nil {
		return
	}

	return
}

func BoolToString(val bool) string {
	return strconv.FormatBool(val)
}

func StringToBool(val string) bool {
	r, err := strconv.ParseBool(val)
	if err != nil {
		return false
	}
	return r
}

func (obj *Version) Init(value string) {
	versionInfo := strings.Split(value, ".")

	obj.MajorString = versionInfo[0]
	obj.MinorString = versionInfo[1]
	obj.RevisionString = versionInfo[2]
	obj.Value = value

	if val, err := strconv.Atoi(versionInfo[0]); err == nil {
		obj.Major = val
	}

	if val, err := strconv.Atoi(versionInfo[1]); err == nil {
		obj.Minor = val
	}

	if val, err := strconv.Atoi(versionInfo[2]); err == nil {
		obj.Revision = val
	}
}

func GenPackageImport(name string, imports []string) string {

	val := "package " + name + "\n\n"
	val += "import(\n"
	for _, imp := range imports {
		if imp == "" {
			continue
		}
		val += "\t\"" + imp + "\"\n"
	}
	val += ")\n\n"

	return val
}

func MakeFirstLowerCase(s string) string {

	if len(s) < 2 {
		return strings.ToLower(s)
	}

	bts := []byte(s)

	lc := bytes.ToLower([]byte{bts[0]})
	rest := bts[1:]

	return string(bytes.Join([][]byte{lc, rest}, nil))
}

func ExtractArgsWithinBrackets(str string) (res []string) {

	brackets := &unicode.RangeTable{
		R16: []unicode.Range16{
			// {0x0028, 0x0029, 1}, // ( )
			// {0x005b, 0x005d, 1}, // [ ]
			{0x007b, 0x007d, 1}, // { }
		},
	}

	isBracket := func(r rune) bool {
		if unicode.In(r, brackets) {
			return true
		}
		return false
	}

	res = strings.FieldsFunc(str, isBracket)
	return
}

func Random(min, max int) int {
	rand.Seed(time.Now().UTC().UnixNano())
	return rand.Intn(max-min) + min
}

//Between will return substring between two strings
func Between(value string, a string, b string) string {
	// Get substring between two strings.
	posFirst := strings.Index(value, a)
	if posFirst == -1 {
		return ""
	}
	posLast := strings.Index(value, b)
	if posLast == -1 {
		return ""
	}
	posFirstAdjusted := posFirst + len(a)
	if posFirstAdjusted >= posLast {
		return ""
	}
	return value[posFirstAdjusted:posLast]
}

//SyncMapAny will return true if there are any items in the sync.Map
func SyncMapAny(x *sync.Map) (ok bool) {
	x.Range(func(key interface{}, value interface{}) bool {
		ok = true
		return false
	})
	return
}

//SyncMapLength will return true the length of items in the sync.Map
func SyncMapLength(x *sync.Map) (length int) {
	x.Range(func(key interface{}, value interface{}) bool {
		length = length + 1
		return true
	})
	return
}

// NewUUID generates a random UUID according to RFC 4122
func NewUUID() (string, error) {
	uuid := make([]byte, 16)
	n, err := io.ReadFull(cryptoRand.Reader, uuid)
	if n != len(uuid) || err != nil {
		return "", err
	}
	// variant bits; see section 4.1.1
	uuid[8] = uuid[8]&^0xc0 | 0x80
	// version 4 (pseudo-random); see section 4.1.3
	uuid[6] = uuid[6]&^0xf0 | 0x40
	return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil
}

//RandomString returns a random string of length
func RandomString(strlen int) string {
	rand.Seed(time.Now().UTC().UnixNano())
	const chars = "abcdefghijklmnopqrstuvwxyz0123456789"
	result := make([]byte, strlen)
	for i := 0; i < strlen; i++ {
		result[i] = chars[rand.Intn(len(chars))]
	}
	return string(result)
}

//BigEndianUInt16 will convert a uint16 to 2 bytes
func BigEndianUInt16(value uint16) (value1 byte, value2 byte) {
	inject := make([]byte, 2)
	binary.BigEndian.PutUint16(inject, value)
	value1 = inject[0]
	value2 = inject[1]
	return
}

//BigEndianUInt32 will convert a uint32 to 4 bytes
func BigEndianUInt32(value uint32) (value1 byte, value2 byte, value3 byte, value4 byte) {
	inject := make([]byte, 4)
	binary.BigEndian.PutUint32(inject, value)
	value1 = inject[0]
	value2 = inject[1]
	value3 = inject[2]
	value4 = inject[3]
	return
}

//IpAddressToUint32 will convert a string ip address to UInt32
func IpAddressToUint32(ip string) (value uint32) {
	binary.Read(bytes.NewBuffer(net.ParseIP(ip).To4()), binary.BigEndian, &value)
	return
}
<commit_msg>new var<commit_after>package extensions

import (
	"bytes"
	cryptoRand "crypto/rand"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"io"
	"math/rand"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
	"unicode"
)

type Version struct {
	Major          int
	Minor          int
	Revision       int
	MajorString    string
	MinorString    string
	RevisionString string
	Value          string
}

type FilePath struct {
	Name string `json:"Name"`
	Path string `json:"Path"`
	Type string `json:"Type"`
}

func (obj *FilePath) ToString() (str string) {
	return obj.Path + string(os.PathSeparator) + obj.Name + " | " + obj.Type
}

func GetDecimalAndStringFromHex(twoDigitHexCode string) (decimalValue int, asciiString string, err error) {
	decoded, err := hex.DecodeString(twoDigitHexCode)
	if err != nil {
		return
	}
	decimalValue = int(decoded[0])
	asciiString = string(string(decoded))
	return
}

/*
* leftPad and rightPad just repoeat the padStr the indicated
* number of times
*
 */
func LeftPad(s string, padStr string, pLen int) string {
	return strings.Repeat(padStr, pLen) + s
}
func RightPad(s string, padStr string, pLen int) string {
	return s + strings.Repeat(padStr, pLen)
}

/* the Pad2Len functions are generally assumed to be padded with short sequences of strings
* in many cases with a single character sequence
*
* so we assume we can build the string out as if the char seq is 1 char and then
* just substr the string if it is longer than needed
*
* this means we are wasting some cpu and memory work
* but this always get us to want we want it to be
*
* in short not optimized to for massive string work
*
* If the overallLen is shorter than the original string length
* the string will be shortened to this length (substr)
*
 */
func RightPad2Len(s string, padStr string, overallLen int) string {
	var padCountInt int
	padCountInt = 1 + ((overallLen - len(padStr)) / len(padStr))
	var retStr = s + strings.Repeat(padStr, padCountInt)
	return retStr[:overallLen]
}

func LeftPad2Len(s string, padStr string, overallLen int) string {
	var padCountInt int
	padCountInt = 1 + ((overallLen - len(padStr)) / len(padStr))
	var retStr = strings.Repeat(padStr, padCountInt) + s
	return retStr[(len(retStr) - overallLen):]
}

func TrimSuffix(s, suffix string) string {
	if strings.HasSuffix(s, suffix) {
		s = s[:len(s)-len(suffix)]
	}
	return s
}

func PrintKiloBytes(bytes int64) string {

	var kilobytes float64
	kilobytes = float64(bytes / 1024)

	return fmt.Sprint(FloatToString(kilobytes, 2), " kB")
}

func PrintMegaBytes(bytes int64) string {

	var kilobytes float64
	kilobytes = float64(bytes / 1024)

	var megabytes float64
	megabytes = kilobytes / 1024 // cast to type float64

	return fmt.Sprint(FloatToString(megabytes, 2), " MB")
}

func IsPrintable(s string) bool {
	for _, c := range s {
		if (c < 32 || c > 126) && c != 10 && c != 13 && c != 9 {
			return false
		}
	}
	return true
}

func PrintZettaBytes(bytes int64) string {

	var kilobytes float64
	kilobytes = float64(bytes / 1024)

	var megabytes float64
	megabytes = (kilobytes / 1024) // cast to type float64

	var gigabytes float64
	gigabytes = (megabytes / 1024)

	var terabytes float64
	terabytes = (gigabytes / 1024)

	var petabytes float64
	petabytes = (terabytes / 1024)

	var exabytes float64
	exabytes = (petabytes / 1024)

	var zettabytes float64
	zettabytes = (exabytes / 1024)

	return fmt.Sprint(FloatToString(zettabytes, 2), " ZB")
}

func FloatToString(input_num float64, precision int) string {
	// to convert a float number to a string
	return strconv.FormatFloat(input_num, 'f', precision, 64)
}

func Round(x, unit float64) float64 {
	return float64(int64(x/unit+0.5)) * unit
}

func StringToInt(val string) int {

	r, err := strconv.Atoi(val)
	if err != nil {
		return 0
	}
	return r
}

func StringToFloat(val string, precision int) (r float64) {
	r, err := strconv.ParseFloat(val, precision)
	if err != nil {
		return 0
	}
	return r
}

func StringToUInt64(val string) uint64 {
	i, err := strconv.ParseUint(val, 10, 64)
	if err != nil {
		return 0
	}
	return i
}

func IntToString(val int) string {
	return strconv.Itoa(val)
}

func IntToBool(val int) bool {
	return val != 0
}

func Int32ToString(val int32) string {
	return strconv.Itoa(Int32ToInt(val))
}

func Int64ToString(val int64) string {
	return strconv.FormatInt(val, 10)
}

func Int64ToInt32(val int64) (ret int) {
	tempLong := ((val >> 32) << 32) //shift it right then left 32 bits, which zeroes the lower half of the long
	ret = (int)(val - tempLong)
	return ret
}

func Int32ToInt(val int32) (ret int) {
	tempLong := ((val >> 32) << 32) //shift it right then left 32 bits, which zeroes the lower half of the long
	ret = (int)(val - tempLong)
	return ret
}

// Int32ToDWORD converts an int32 to DWORD byte array.
func Int32ToDWORD(val int32) (value []byte) {
	buff := new(bytes.Buffer)
	err := binary.Write(buff, binary.BigEndian, val)
	if err != nil {
		return
	}
	value = buff.Bytes()
	return
}

// DWordToInt converts an 4 byte to a int32
func DWordToInt(data []byte) (num int32) {
	buff := bytes.NewReader(data)
	err := binary.Read(buff, binary.BigEndian, &num)
	if err != nil {
		return
	}

	return
}

func BoolToString(val bool) string {
	return strconv.FormatBool(val)
}

func StringToBool(val string) bool {
	r, err := strconv.ParseBool(val)
	if err != nil {
		return false
	}
	return r
}

func (obj *Version) Init(value string) {
	versionInfo := strings.Split(value, ".")

	obj.MajorString = versionInfo[0]
	obj.MinorString = versionInfo[1]
	obj.RevisionString = versionInfo[2]
	obj.Value = value

	if val, err := strconv.Atoi(versionInfo[0]); err == nil {
		obj.Major = val
	}

	if val, err := strconv.Atoi(versionInfo[1]); err == nil {
		obj.Minor = val
	}

	if val, err := strconv.Atoi(versionInfo[2]); err == nil {
		obj.Revision = val
	}
}

func GenPackageImport(name string, imports []string) string {

	val := "package " + name + "\n\n"
	val += "import(\n"
	for _, imp := range imports {
		if imp == "" {
			continue
		}
		val += "\t\"" + imp + "\"\n"
	}
	val += ")\n\n"

	return val
}

func MakeFirstLowerCase(s string) string {

	if len(s) < 2 {
		return strings.ToLower(s)
	}

	bts := []byte(s)

	lc := bytes.ToLower([]byte{bts[0]})
	rest := bts[1:]

	return string(bytes.Join([][]byte{lc, rest}, nil))
}

func ExtractArgsWithinBrackets(str string) (res []string) {

	brackets := &unicode.RangeTable{
		R16: []unicode.Range16{
			// {0x0028, 0x0029, 1}, // ( )
			// {0x005b, 0x005d, 1}, // [ ]
			{0x007b, 0x007d, 1}, // { }
		},
	}

	isBracket := func(r rune) bool {
		if unicode.In(r, brackets) {
			return true
		}
		return false
	}

	res = strings.FieldsFunc(str, isBracket)
	return
}

func Random(min, max int) int {
	rand.Seed(time.Now().UTC().UnixNano())
	return rand.Intn(max-min) + min
}

//Between will return substring between two strings
func Between(value string, a string, b string) string {
	// Get substring between two strings.
	posFirst := strings.Index(value, a)
	if posFirst == -1 {
		return ""
	}
	posLast := strings.Index(value, b)
	if posLast == -1 {
		return ""
	}
	posFirstAdjusted := posFirst + len(a)
	if posFirstAdjusted >= posLast {
		return ""
	}
	return value[posFirstAdjusted:posLast]
}

//SyncMapAny will return true if there are any items in the sync.Map
func SyncMapAny(x *sync.Map) (ok bool) {
	x.Range(func(key interface{}, value interface{}) bool {
		ok = true
		return false
	})
	return
}

//SyncMapLength will return true the length of items in the sync.Map
func SyncMapLength(x *sync.Map) (length int) {
	x.Range(func(key interface{}, value interface{}) bool {
		length = length + 1
		return true
	})
	return
}

// NewUUID generates a random UUID according to RFC 4122
func NewUUID() (string, error) {
	uuid := make([]byte, 16)
	n, err := io.ReadFull(cryptoRand.Reader, uuid)
	if n != len(uuid) || err != nil {
		return "", err
	}
	// variant bits; see section 4.1.1
	uuid[8] = uuid[8]&^0xc0 | 0x80
	// version 4 (pseudo-random); see section 4.1.3
	uuid[6] = uuid[6]&^0xf0 | 0x40
	return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil
}

//RandomString returns a random string of length
func RandomString(strlen int) string {
	rand.Seed(time.Now().UTC().UnixNano())
	const chars = "abcdefghijklmnopqrstuvwxyz0123456789"
	result := make([]byte, strlen)
	for i := 0; i < strlen; i++ {
		result[i] = chars[rand.Intn(len(chars))]
	}
	return string(result)
}

//BigEndianUInt16 will convert a uint16 to 2 bytes
func BigEndianUInt16(value uint16) (value1 byte, value2 byte) {
	inject := make([]byte, 2)
	binary.BigEndian.PutUint16(inject, value)
	value1 = inject[0]
	value2 = inject[1]
	return
}

//BigEndianUInt32 will convert a uint32 to 4 bytes
func BigEndianUInt32(value uint32) (value1 byte, value2 byte, value3 byte, value4 byte) {
	inject := make([]byte, 4)
	binary.BigEndian.PutUint32(inject, value)
	value1 = inject[0]
	value2 = inject[1]
	value3 = inject[2]
	value4 = inject[3]
	return
}

//IpAddressToUint32 will convert a string ip address to UInt32
func IpAddressToUint32(ip string) (value uint32) {
	binary.Read(bytes.NewBuffer(net.ParseIP(ip).To4()), binary.BigEndian, &value)
	return
}
<|endoftext|>