<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"regexp"
	"strings"

	"gopkg.in/jackc/pgx.v2"

	"github.com/golang/groupcache"

	"github.com/LeKovr/dbrpc/workman"
	"github.com/LeKovr/go-base/logger"
)

// TableRow holds query result row
type TableRow map[string]interface{}

// TableRows holds slice of query result rows
type TableRows []TableRow

// -----------------------------------------------------------------------------

// Processor gets value from cache and converts it into Result struct
func cacheFetcher(log *logger.Log, cacheGroup *groupcache.Group) workman.WorkerFunc {
	// https://github.com/capotej/groupcache-db-experiment
	return func(payload string) workman.Result {
		var data []byte
		log.Debugf("asked for %s from groupcache", payload)
		err := cacheGroup.Get(nil, payload,
			groupcache.AllocatingByteSliceSink(&data))
		var res workman.Result
		if err != nil {
			res = workman.Result{Success: false, Error: err.Error()}
		} else if len(data) == 0 {
			res = workman.Result{Success: false, Error: "This internal error must be catched earlier. Please contact vendor"}
		} else {
			d := data[1:]
			raw := json.RawMessage(d)
			ok := data[0] == 1
			if ok { // First byte stores success state (1: true, 0: false)
				res = workman.Result{Success: ok, Result: &raw}
			} else {
				res = workman.Result{Success: ok, Error: &raw}
			}
		}
		return res
	}
}

// -----------------------------------------------------------------------------

func dbFetcher(cfg *AplFlags, log *logger.Log, db *pgx.ConnPool) groupcache.GetterFunc {
	return func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
		log.Printf("asking for %s from dbserver", key)

		var isOk byte = 1 // status: success

		data, err := dbQuery(cfg, log, db, key)
		if err != nil {
			log.Warnf("Query for key %s error: %+v", key, err)
			// TODO: send to error logging channel
			data, err = parseError(err)
			if err != nil {
				data, _ = json.Marshal(err.Error())
			}
			isOk = 0
		}

		result := []byte{isOk}
		result = append(result, data...)

		dd := result[1:]
		log.Debugf("Save data: %s", dd)
		dest.SetBytes([]byte(result))
		return nil
	}
}

// -----------------------------------------------------------------------------

// FuncDef holds function definition
type FuncDef struct {
	NspName string // function namespace
	ProName string // function name
	//	Permit  *string // permit code
	MaxAge int  // cache max age
	IsRO   bool // function is read-only (not volatile)
}

// FuncMap holds map of function definitions
type FuncMap map[string]FuncDef

// -----------------------------------------------------------------------------

func indexFetcher(cfg *AplFlags, log *logger.Log, db *pgx.ConnPool) (index *FuncMap, err error) {
	var rows *pgx.Rows

	// q := fmt.Sprintf("select code, nspname, proname, permit_code, max_age, is_ro from %s()", cfg.ArgIndexFunc)
	q := fmt.Sprintf("select code, nspname, proname, max_age, is_ro from %s()", cfg.ArgIndexFunc)
	log.Debugf("Query: %s", q)

	rows, err = db.Query(q)
	if err != nil {
		return
	}

	ind := FuncMap{}
	for rows.Next() {
		var fmr FuncDef
		var code string
		// err = rows.Scan(&code, &fmr.NspName, &fmr.ProName, &fmr.Permit, &fmr.MaxAge, &fmr.IsRO)
		err = rows.Scan(&code, &fmr.NspName, &fmr.ProName, &fmr.MaxAge, &fmr.IsRO)
		if err != nil {
			log.Warnf("Value fetch error: %s", err.Error())
			return
		}
		ind[code] = fmr
	}
	// Any errors encountered by rows.Next or rows.Scan will be returned here
	if rows.Err() != nil {
		err = rows.Err()
		return
	}
	log.Debugf("Index: %+v", ind)
	index = &ind
	return
}

// -----------------------------------------------------------------------------

func dbQuery(cfg *AplFlags, log *logger.Log, db *pgx.ConnPool, key string) (data []byte, err error) {
	var rows *pgx.Rows
	var q string
	var vals []interface{}
	var table *TableRows
	inTx := cfg.BeginFunc != "" // call in transaction
	var lang *string
	var tz *string
	if strings.HasPrefix(key, "[") {
		var args []interface{}
		err = json.Unmarshal([]byte(key), &args)
		if err != nil {
			return
		}
		inTx = false // no transaction for internal calls
		q, vals = PrepareFuncSQL(cfg, args)
	} else {
		var args CallDef
		err = json.Unmarshal([]byte(key), &args)
		if err != nil {
			return
		}
		q, vals, lang, tz = PrepareFuncSQLmap(cfg, args)
	}

	log.Debugf("Query: %s args: %+v", q, vals)

	if inTx {
		// call func after begin
		log.Debug("Run in transaction mode")
		var tx *pgx.Tx
		tx, err = db.Begin()
		if err != nil {
			return
		}
		// Rollback is safe to call even if the tx is already closed, so if
		// the tx commits successfully, this is a no-op
		defer tx.Rollback()

		rows, err = tx.Query("select "+cfg.BeginFunc+"($1,$2)", lang, tz)
		if err != nil {
			return
		}
		rows.Next() // just fetch unneded data or will be "conn is busy"
		if rows.Err() != nil {
			err = rows.Err()
			return
		}
		rows.Close()

		log.Debug("Run main call")
		rows, err = tx.Query(q, vals...)
		if err != nil {
			return
		}
		table, err = FetchSQLResult(rows, log)
		if err != nil {
			return
		}

		err = tx.Commit()
		log.Debug("Call committed")

	} else {
		rows, err = db.Query(q, vals...)
		if err != nil {
			return
		}
		table, err = FetchSQLResult(rows, log)
		defer rows.Close()
	}

	if err != nil {
		return
	}

	data, err = json.Marshal(*table)
	if err != nil {
		return
	}

	return
}

// -----------------------------------------------------------------------------

// PrepareFuncSQL prepares sql query with args placeholders
func PrepareFuncSQL(cfg *AplFlags, args []interface{}) (string, []interface{}) {

	proc := args[2].(string) // nil, cache_id, method, args..
	argVals := args[3:]

	argValPrep := make([]interface{}, len(argVals))
	argIDs := make([]string, len(argVals))

	for i, v := range argVals {
		argIDs[i] = fmt.Sprintf("$%d", i+1)
		argValPrep[i] = v
	}

	argIDStr := strings.Join(argIDs, ",")

	var q string
	if args[0] != nil {
		nsp := args[0].(string)
		q = fmt.Sprintf("select * from %s.%s(%s)", nsp, proc, argIDStr)
	} else {
		// use search_path
		q = fmt.Sprintf("select * from %s(%s)", proc, argIDStr)
	}

	return q, argValPrep
}

// -----------------------------------------------------------------------------

// PrepareFuncSQLmap prepares sql query with named args placeholders
func PrepareFuncSQLmap(cfg *AplFlags, args CallDef) (string, []interface{}, *string, *string) {

	var proc string
	ref := args.Proc
	if ref != nil {
		proc = *ref
	} else {
		// fatal - incorrect map structure
	}

	argValPrep := make([]interface{}, len(args.Args))
	argIDs := make([]string, len(args.Args))

	i := 0
	for k, v := range args.Args {
		argIDs[i] = fmt.Sprintf("%s %s $%d", k, cfg.ArgSyntax, i+1)
		argValPrep[i] = v
		i++
	}

	argIDStr := strings.Join(argIDs, ",")

	var q string
	ref = args.Name
	if ref != nil {
		nsp := *ref
		q = fmt.Sprintf("select * from %s.%s(%s)", nsp, proc, argIDStr)
	} else {
		// use search_path
		q = fmt.Sprintf("select * from %s(%s)", proc, argIDStr)
	}

	return q, argValPrep, args.Lang, args.TZ
}

// -----------------------------------------------------------------------------

// FetchSQLResult fetches sql result and marshalls it into json
func FetchSQLResult(rows *pgx.Rows, log *logger.Log) (data *TableRows, err error) {
	// http://stackoverflow.com/a/29164115
	columnDefs := rows.FieldDescriptions()
	//log.Debugf("=========== %+v", columnDefs)
	columns := []string{}
	types := []string{}
	for _, c := range columnDefs {
		columns = append(columns, c.Name)
		types = append(types, c.DataTypeName)
	}

	var tableData TableRows
	for rows.Next() {
		var values []interface{}
		values, err = rows.Values()
		if err != nil {
			log.Warnf("Value fetch error: %s", err.Error())
			return
		}
		log.Debugf("Values: %+v", values)

		entry := make(map[string]interface{})
		for i, col := range columns {
			var v interface{}
			val := values[i]
			if types[i] == "json" || types[i] == "jsonb" {
				if val != nil {
					raw := fmt.Sprintf("%s", val)
					ref := json.RawMessage(raw)
					entry[col] = &ref
				}
			} else {
				v = val
				entry[col] = v
			}
		}
		tableData = append(tableData, entry)

	}
	// Any errors encountered by rows.Next or rows.Scan will be returned here
	if rows.Err() != nil {
		err = rows.Err()
		return
	}
	if tableData == nil {
		log.Warn("Empty result")
		tableData = TableRows{} // empty result is empty array
	}
	data = &tableData
	return
}

var reError = regexp.MustCompile(`^ERROR: (.+) \(SQLSTATE (\w+)\)$`)
var reHash = regexp.MustCompile(`^\{.+\}$`)

// Vars holds error fields hash
type Vars map[string]interface{}

// ErrorVars holds returned error struct
type ErrorVars struct {
	Code    string `json:"code"`
	Message string `json:"message,omitempty"`
	Data    Vars   `json:"data,omitempty"`
}

func parseError(e error) (data []byte, err error) {

	// ERROR: {\"code\" : \"YA014\", \"data\" : {\"login\": \"john\"}} (SQLSTATE P0001)"
	// EMFE - Error Message Format Error
	match := reError.FindStringSubmatch(e.Error())
	if len(match) == 3 {
		if match[2] == "P0001" {
			if reHash.MatchString(match[1]) {
				ev := ErrorVars{}
				err = json.Unmarshal([]byte(match[1]), &ev)
				if err != nil {
					err = fmt.Errorf("%s (EMFE: parse json error: %s)", match[1], err.Error())
				} else {
					data, _ = json.Marshal(ev) // Got application error
				}
			} else {
				err = fmt.Errorf("%s (EMFE: not json hash in message)", match[1])
			}
		} else {
			ev := ErrorVars{Code: match[2], Message: match[1]}
			data, _ = json.Marshal(ev) // Got postgresql error
			return
		}
	} else {
		err = fmt.Errorf("%s (EMFE: Unknown error format)", match)
	}
	return
}
<commit_msg>fix: json* result parsing<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"regexp"
	"strings"

	"gopkg.in/jackc/pgx.v2"

	"github.com/golang/groupcache"

	"github.com/LeKovr/dbrpc/workman"
	"github.com/LeKovr/go-base/logger"
)

// TableRow holds query result row
type TableRow map[string]interface{}

// TableRows holds slice of query result rows
type TableRows []TableRow

// -----------------------------------------------------------------------------

// Processor gets value from cache and converts it into Result struct
func cacheFetcher(log *logger.Log, cacheGroup *groupcache.Group) workman.WorkerFunc {
	// https://github.com/capotej/groupcache-db-experiment
	return func(payload string) workman.Result {
		var data []byte
		log.Debug("asking groupcache")
		err := cacheGroup.Get(nil, payload,
			groupcache.AllocatingByteSliceSink(&data))
		var res workman.Result
		if err != nil {
			res = workman.Result{Success: false, Error: err.Error()}
		} else if len(data) == 0 {
			res = workman.Result{Success: false, Error: "This internal error must be catched earlier. Please contact vendor"}
		} else {
			d := data[1:]
			raw := json.RawMessage(d)
			ok := data[0] == 1
			if ok { // First byte stores success state (1: true, 0: false)
				res = workman.Result{Success: ok, Result: &raw}
			} else {
				res = workman.Result{Success: ok, Error: &raw}
			}
		}
		return res
	}
}

// -----------------------------------------------------------------------------

func dbFetcher(cfg *AplFlags, log *logger.Log, db *pgx.ConnPool) groupcache.GetterFunc {
	return func(ctx groupcache.Context, key string, dest groupcache.Sink) error {
		log.Info("asking dbserver")

		var isOk byte = 1 // status: success

		data, err := dbQuery(cfg, log, db, key)
		if err != nil {
			log.Warnf("Query for key %s error: %+v", key, err)
			// TODO: send to error logging channel
			data, err = parseError(err)
			if err != nil {
				data, _ = json.Marshal(err.Error())
			}
			isOk = 0
		}

		result := []byte{isOk}
		result = append(result, data...)

		dd := result[1:]
		log.Debugf("Save data: %s", dd)
		dest.SetBytes([]byte(result))
		return nil
	}
}

// -----------------------------------------------------------------------------

// FuncDef holds function definition
type FuncDef struct {
	NspName string // function namespace
	ProName string // function name
	//	Permit  *string // permit code
	MaxAge int  // cache max age
	IsRO   bool // function is read-only (not volatile)
}

// FuncMap holds map of function definitions
type FuncMap map[string]FuncDef

// -----------------------------------------------------------------------------

func indexFetcher(cfg *AplFlags, log *logger.Log, db *pgx.ConnPool) (index *FuncMap, err error) {
	var rows *pgx.Rows

	// q := fmt.Sprintf("select code, nspname, proname, permit_code, max_age, is_ro from %s()", cfg.ArgIndexFunc)
	q := fmt.Sprintf("select code, nspname, proname, max_age, is_ro from %s()", cfg.ArgIndexFunc)
	log.Debugf("Query: %s", q)

	rows, err = db.Query(q)
	if err != nil {
		return
	}

	ind := FuncMap{}
	for rows.Next() {
		var fmr FuncDef
		var code string
		// err = rows.Scan(&code, &fmr.NspName, &fmr.ProName, &fmr.Permit, &fmr.MaxAge, &fmr.IsRO)
		err = rows.Scan(&code, &fmr.NspName, &fmr.ProName, &fmr.MaxAge, &fmr.IsRO)
		if err != nil {
			log.Warnf("Value fetch error: %s", err.Error())
			return
		}
		ind[code] = fmr
	}
	// Any errors encountered by rows.Next or rows.Scan will be returned here
	if rows.Err() != nil {
		err = rows.Err()
		return
	}
	log.Debugf("Index: %+v", ind)
	index = &ind
	return
}

// -----------------------------------------------------------------------------

func dbQuery(cfg *AplFlags, log *logger.Log, db *pgx.ConnPool, key string) (data []byte, err error) {
	var rows *pgx.Rows
	var q string
	var vals []interface{}
	var table *TableRows
	inTx := cfg.BeginFunc != "" // call in transaction
	var lang *string
	var tz *string
	if strings.HasPrefix(key, "[") {
		var args []interface{}
		err = json.Unmarshal([]byte(key), &args)
		if err != nil {
			return
		}
		inTx = false // no transaction for internal calls
		q, vals = PrepareFuncSQL(cfg, args)
	} else {
		var args CallDef
		err = json.Unmarshal([]byte(key), &args)
		if err != nil {
			return
		}
		q, vals, lang, tz = PrepareFuncSQLmap(cfg, args)
	}

	log.Debugf("Query: %s args: %+v", q, vals)

	if inTx {
		// call func after begin
		log.Debug("Run in transaction mode")
		var tx *pgx.Tx
		tx, err = db.Begin()
		if err != nil {
			return
		}
		// Rollback is safe to call even if the tx is already closed, so if
		// the tx commits successfully, this is a no-op
		defer tx.Rollback()

		rows, err = tx.Query("select "+cfg.BeginFunc+"($1,$2)", lang, tz)
		if err != nil {
			return
		}
		rows.Next() // just fetch unneded data or will be "conn is busy"
		if rows.Err() != nil {
			err = rows.Err()
			return
		}
		rows.Close()

		log.Debug("Run main call")
		rows, err = tx.Query(q, vals...)
		if err != nil {
			return
		}
		table, err = FetchSQLResult(rows, log)
		if err != nil {
			return
		}

		err = tx.Commit()
		log.Debug("Call committed")

	} else {
		rows, err = db.Query(q, vals...)
		if err != nil {
			return
		}
		table, err = FetchSQLResult(rows, log)
		defer rows.Close()
	}

	if err != nil {
		return
	}

	data, err = json.Marshal(*table)
	if err != nil {
		return
	}

	return
}

// -----------------------------------------------------------------------------

// PrepareFuncSQL prepares sql query with args placeholders
func PrepareFuncSQL(cfg *AplFlags, args []interface{}) (string, []interface{}) {

	proc := args[2].(string) // nil, cache_id, method, args..
	argVals := args[3:]

	argValPrep := make([]interface{}, len(argVals))
	argIDs := make([]string, len(argVals))

	for i, v := range argVals {
		argIDs[i] = fmt.Sprintf("$%d", i+1)
		argValPrep[i] = v
	}

	argIDStr := strings.Join(argIDs, ",")

	var q string
	if args[0] != nil {
		nsp := args[0].(string)
		q = fmt.Sprintf("select * from %s.%s(%s)", nsp, proc, argIDStr)
	} else {
		// use search_path
		q = fmt.Sprintf("select * from %s(%s)", proc, argIDStr)
	}

	return q, argValPrep
}

// -----------------------------------------------------------------------------

// PrepareFuncSQLmap prepares sql query with named args placeholders
func PrepareFuncSQLmap(cfg *AplFlags, args CallDef) (string, []interface{}, *string, *string) {

	var proc string
	ref := args.Proc
	if ref != nil {
		proc = *ref
	} else {
		// fatal - incorrect map structure
	}

	argValPrep := make([]interface{}, len(args.Args))
	argIDs := make([]string, len(args.Args))

	i := 0
	for k, v := range args.Args {
		argIDs[i] = fmt.Sprintf("%s %s $%d", k, cfg.ArgSyntax, i+1)
		argValPrep[i] = v
		i++
	}

	argIDStr := strings.Join(argIDs, ",")

	var q string
	ref = args.Name
	if ref != nil {
		nsp := *ref
		q = fmt.Sprintf("select * from %s.%s(%s)", nsp, proc, argIDStr)
	} else {
		// use search_path
		q = fmt.Sprintf("select * from %s(%s)", proc, argIDStr)
	}

	return q, argValPrep, args.Lang, args.TZ
}

// -----------------------------------------------------------------------------

// FetchSQLResult fetches sql result and marshalls it into json
func FetchSQLResult(rows *pgx.Rows, log *logger.Log) (data *TableRows, err error) {
	// http://stackoverflow.com/a/29164115
	columnDefs := rows.FieldDescriptions()
	//log.Debugf("=========== %+v", columnDefs)
	columns := []string{}
	types := []string{}
	for _, c := range columnDefs {
		columns = append(columns, c.Name)
		types = append(types, c.DataTypeName)
	}

	var tableData TableRows
	for rows.Next() {
		var values []interface{}
		values, err = rows.Values()
		if err != nil {
			log.Warnf("Value fetch error: %s", err.Error())
			return
		}
		log.Debugf("Values: %+v", values)

		entry := make(map[string]interface{})
		for i, col := range columns {
			var v interface{}
			val := values[i]
			if types[i] == "json" || types[i] == "jsonb" {
				if val != nil {
					var raw []byte
					raw, err = json.Marshal(val)
					if err != nil {
						log.Warnf("Value marshal error: %s", err.Error())
						return
					}
					ref := json.RawMessage(raw)
					entry[col] = &ref
				}
			} else {
				v = val
				entry[col] = v
			}
		}
		tableData = append(tableData, entry)

	}
	// Any errors encountered by rows.Next or rows.Scan will be returned here
	if rows.Err() != nil {
		err = rows.Err()
		return
	}
	if tableData == nil {
		log.Warn("Empty result")
		tableData = TableRows{} // empty result is empty array
	}
	data = &tableData
	return
}

var reError = regexp.MustCompile(`^ERROR: (.+) \(SQLSTATE (\w+)\)$`)
var reHash = regexp.MustCompile(`^\{.+\}$`)

// Vars holds error fields hash
type Vars map[string]interface{}

// ErrorVars holds returned error struct
type ErrorVars struct {
	Code    string `json:"code"`
	Message string `json:"message,omitempty"`
	Data    Vars   `json:"data,omitempty"`
}

func parseError(e error) (data []byte, err error) {

	// ERROR: {\"code\" : \"YA014\", \"data\" : {\"login\": \"john\"}} (SQLSTATE P0001)"
	// EMFE - Error Message Format Error
	match := reError.FindStringSubmatch(e.Error())
	if len(match) == 3 {
		if match[2] == "P0001" {
			if reHash.MatchString(match[1]) {
				ev := ErrorVars{}
				err = json.Unmarshal([]byte(match[1]), &ev)
				if err != nil {
					err = fmt.Errorf("%s (EMFE: parse json error: %s)", match[1], err.Error())
				} else {
					data, _ = json.Marshal(ev) // Got application error
				}
			} else {
				err = fmt.Errorf("%s (EMFE: not json hash in message)", match[1])
			}
		} else {
			ev := ErrorVars{Code: match[2], Message: match[1]}
			data, _ = json.Marshal(ev) // Got postgresql error
			return
		}
	} else {
		err = fmt.Errorf("%s (EMFE: Unknown error format)", match)
	}
	return
}
<|endoftext|>


<commit_before>package pop

import (
	"fmt"
	"reflect"
	"strconv"

	"github.com/satori/go.uuid"
)

// Find the first record of the model in the database with a particular id.
//
//	c.Find(&User{}, 1)
func (c *Connection) Find(model interface{}, id interface{}) error {
	return Q(c).Find(model, id)
}

// Find the first record of the model in the database with a particular id.
//
//	q.Find(&User{}, 1)
func (q *Query) Find(model interface{}, id interface{}) error {
	m := &Model{Value: model}
	idq := fmt.Sprintf("%s.id = ?", m.TableName())
	switch t := id.(type) {
	case uuid.UUID:
		return q.Where(idq, t.String()).First(model)
	case string:
		var err error
		id, err = strconv.Atoi(t)
		if err != nil {
			return q.Where(idq, t).First(model)
		}
	}
	return q.Where(idq, id).First(model)
}

// First record of the model in the database that matches the query.
//
//	c.First(&User{})
func (c *Connection) First(model interface{}) error {
	return Q(c).First(model)
}

// First record of the model in the database that matches the query.
//
//	q.Where("name = ?", "mark").First(&User{})
func (q *Query) First(model interface{}) error {
	return q.Connection.timeFunc("First", func() error {
		q.Limit(1)
		m := &Model{Value: model}
		if err := q.Connection.Dialect.SelectOne(q.Connection.Store, m, *q); err != nil {
			return err
		}
		return m.afterFind(q.Connection)
	})
}

// Last record of the model in the database that matches the query.
//
//	c.Last(&User{})
func (c *Connection) Last(model interface{}) error {
	return Q(c).Last(model)
}

// Last record of the model in the database that matches the query.
//
//	q.Where("name = ?", "mark").Last(&User{})
func (q *Query) Last(model interface{}) error {
	return q.Connection.timeFunc("Last", func() error {
		q.Limit(1)
		q.Order("id desc")
		m := &Model{Value: model}
		if err := q.Connection.Dialect.SelectOne(q.Connection.Store, m, *q); err != nil {
			return err
		}
		return m.afterFind(q.Connection)
	})
}

// All retrieves all of the records in the database that match the query.
//
//	c.All(&[]User{})
func (c *Connection) All(models interface{}) error {
	return Q(c).All(models)
}

// All retrieves all of the records in the database that match the query.
//
//	q.Where("name = ?", "mark").All(&[]User{})
func (q *Query) All(models interface{}) error {
	return q.Connection.timeFunc("All", func() error {
		m := &Model{Value: models}
		err := q.Connection.Dialect.SelectMany(q.Connection.Store, m, *q)
		if err == nil && q.Paginator != nil {
			ct, err := q.Count(models)
			if err == nil {
				q.Paginator.TotalEntriesSize = ct
				st := reflect.ValueOf(models).Elem()
				q.Paginator.CurrentEntriesSize = st.Len()
				q.Paginator.TotalPages = (q.Paginator.TotalEntriesSize / q.Paginator.PerPage)
				if q.Paginator.TotalEntriesSize%q.Paginator.PerPage > 0 {
					q.Paginator.TotalPages = q.Paginator.TotalPages + 1
				}
			}
		}
		if err != nil {
			return err
		}
		return m.afterFind(q.Connection)
	})
}

// Exists returns true/false if a record exists in the database that matches
// the query.
//
// 	q.Where("name = ?", "mark").Exists(&User{})
func (q *Query) Exists(model interface{}) (bool, error) {
	i, err := q.Count(model)
	return i != 0, err
}

//only works with postgresql
func (q Query) WrapCount(model interface{}, field string) (int, error) {
	tmpQuery := Q(q.Connection)
	q.Clone(tmpQuery) //avoid mendling with original query

	res := &rowCount{}
	err := tmpQuery.Connection.timeFunc("Count", func() error {
		tmpQuery.Paginator = nil
		tmpQuery.orderClauses = clauses{}
		query, args := tmpQuery.ToSQL(&Model{Value: model})

		countQuery := fmt.Sprintf("select count(%s) as row_count from (%s) a", field, query)

		Log(countQuery, args...)
		return q.Connection.Store.Get(res, countQuery, args...)
	})
	return res.Count, err
}

// Count the number of records in the database.
//
//	c.Count(&User{})
func (c *Connection) Count(model interface{}) (int, error) {
	return Q(c).Count(model)
}

// Count the number of records in the database.
//
//	q.Where("name = ?", "mark").Count(&User{})
func (q Query) Count(model interface{}) (int, error) {
	return q.CountByField(model, "*")
}

func (q Query) CountByField(model interface{}, field string) (int, error) {
	tmpQuery := Q(q.Connection)
	q.Clone(tmpQuery) //avoid mendling with original query

	res := &rowCount{}
	err := tmpQuery.Connection.timeFunc("Count", func() error {
		tmpQuery.Paginator = nil
		tmpQuery.orderClauses = clauses{}
		query, args := tmpQuery.ToSQL(&Model{Value: model})

		countQuery := fmt.Sprintf("select count(%s) as row_count from (%s) a", field, query)

		Log(countQuery, args...)
		return q.Connection.Store.Get(res, countQuery, args...)
	})
	return res.Count, err
}

type rowCount struct {
	Count int `db:"row_count"`
}
<commit_msg>count works with group by on sqlite, mysql n postgresql<commit_after>package pop

import (
	"fmt"
	"reflect"
	"strconv"

	"github.com/satori/go.uuid"
)

// Find the first record of the model in the database with a particular id.
//
//	c.Find(&User{}, 1)
func (c *Connection) Find(model interface{}, id interface{}) error {
	return Q(c).Find(model, id)
}

// Find the first record of the model in the database with a particular id.
//
//	q.Find(&User{}, 1)
func (q *Query) Find(model interface{}, id interface{}) error {
	m := &Model{Value: model}
	idq := fmt.Sprintf("%s.id = ?", m.TableName())
	switch t := id.(type) {
	case uuid.UUID:
		return q.Where(idq, t.String()).First(model)
	case string:
		var err error
		id, err = strconv.Atoi(t)
		if err != nil {
			return q.Where(idq, t).First(model)
		}
	}
	return q.Where(idq, id).First(model)
}

// First record of the model in the database that matches the query.
//
//	c.First(&User{})
func (c *Connection) First(model interface{}) error {
	return Q(c).First(model)
}

// First record of the model in the database that matches the query.
//
//	q.Where("name = ?", "mark").First(&User{})
func (q *Query) First(model interface{}) error {
	return q.Connection.timeFunc("First", func() error {
		q.Limit(1)
		m := &Model{Value: model}
		if err := q.Connection.Dialect.SelectOne(q.Connection.Store, m, *q); err != nil {
			return err
		}
		return m.afterFind(q.Connection)
	})
}

// Last record of the model in the database that matches the query.
//
//	c.Last(&User{})
func (c *Connection) Last(model interface{}) error {
	return Q(c).Last(model)
}

// Last record of the model in the database that matches the query.
//
//	q.Where("name = ?", "mark").Last(&User{})
func (q *Query) Last(model interface{}) error {
	return q.Connection.timeFunc("Last", func() error {
		q.Limit(1)
		q.Order("id desc")
		m := &Model{Value: model}
		if err := q.Connection.Dialect.SelectOne(q.Connection.Store, m, *q); err != nil {
			return err
		}
		return m.afterFind(q.Connection)
	})
}

// All retrieves all of the records in the database that match the query.
//
//	c.All(&[]User{})
func (c *Connection) All(models interface{}) error {
	return Q(c).All(models)
}

// All retrieves all of the records in the database that match the query.
//
//	q.Where("name = ?", "mark").All(&[]User{})
func (q *Query) All(models interface{}) error {
	return q.Connection.timeFunc("All", func() error {
		m := &Model{Value: models}
		err := q.Connection.Dialect.SelectMany(q.Connection.Store, m, *q)
		if err == nil && q.Paginator != nil {
			ct, err := q.Count(models)
			if err == nil {
				q.Paginator.TotalEntriesSize = ct
				st := reflect.ValueOf(models).Elem()
				q.Paginator.CurrentEntriesSize = st.Len()
				q.Paginator.TotalPages = (q.Paginator.TotalEntriesSize / q.Paginator.PerPage)
				if q.Paginator.TotalEntriesSize%q.Paginator.PerPage > 0 {
					q.Paginator.TotalPages = q.Paginator.TotalPages + 1
				}
			}
		}
		if err != nil {
			return err
		}
		return m.afterFind(q.Connection)
	})
}

// Exists returns true/false if a record exists in the database that matches
// the query.
//
// 	q.Where("name = ?", "mark").Exists(&User{})
func (q *Query) Exists(model interface{}) (bool, error) {
	i, err := q.Count(model)
	return i != 0, err
}

// Count the number of records in the database.
//
//	c.Count(&User{})
func (c *Connection) Count(model interface{}) (int, error) {
	return Q(c).Count(model)
}

// Count the number of records in the database.
//
//	q.Where("name = ?", "mark").Count(&User{})
func (q Query) Count(model interface{}) (int, error) {
	return q.CountByField(model, "*")
}

func (q Query) CountByField(model interface{}, field string) (int, error) {
	tmpQuery := Q(q.Connection)
	q.Clone(tmpQuery) //avoid mendling with original query

	res := &rowCount{}
	err := tmpQuery.Connection.timeFunc("Count", func() error {
		tmpQuery.Paginator = nil
		tmpQuery.orderClauses = clauses{}
		query, args := tmpQuery.ToSQL(&Model{Value: model})

		countQuery := fmt.Sprintf("select count(%s) as row_count from (%s) a", field, query)

		Log(countQuery, args...)
		return q.Connection.Store.Get(res, countQuery, args...)
	})
	return res.Count, err
}

type rowCount struct {
	Count int `db:"row_count"`
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Opsidian Ltd.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package json_test

import (
	"io/ioutil"
	"testing"

	encoding_json "encoding/json"

	"github.com/opsidian/parsley/combinator"
	"github.com/opsidian/parsley/examples/json/json"
	"github.com/opsidian/parsley/parsley"
	"github.com/opsidian/parsley/text"
)

func benchmarkParsleyJSON(b *testing.B, jsonFilePath string) {
	f, err := text.ReadFile(jsonFilePath)
	if err != nil {
		b.Fatal(err)
	}

	s := combinator.Sentence(json.NewParser())
	r := text.NewReader(f)
	ctx := parsley.NewContext(parsley.NewFileSet(), r)
	if _, err = parsley.Evaluate(ctx, s); err != nil {
		b.Fatal(err)
	}

	for n := 0; n < b.N; n++ {
		ctx := parsley.NewContext(parsley.NewFileSet(), r)
		_, _ = parsley.Evaluate(ctx, s)
	}
}

func BenchmarkParsleyJSON1k(b *testing.B)   { benchmarkParsleyJSON(b, "../example_1k.json") }
func BenchmarkParsleyJSON10k(b *testing.B)  { benchmarkParsleyJSON(b, "../example_10k.json") }
func BenchmarkParsleyJSON100k(b *testing.B) { benchmarkParsleyJSON(b, "../example_100k.json") }

func benchmarkEncodingJSON(b *testing.B, jsonFilePath string) {
	input, err := ioutil.ReadFile(jsonFilePath)
	if err != nil {
		b.Fatal(err)
	}

	var val interface{}
	if err := encoding_json.Unmarshal(input, &val); err != nil {
		b.Fatal(err)
	}

	for n := 0; n < b.N; n++ {
		var val interface{}
		_ = encoding_json.Unmarshal(input, &val)
	}
}

func BenchmarkEncodingJSON1k(b *testing.B)   { benchmarkEncodingJSON(b, "../example_1k.json") }
func BenchmarkEncodingJSON10k(b *testing.B)  { benchmarkEncodingJSON(b, "../example_10k.json") }
func BenchmarkEncodingJSON100k(b *testing.B) { benchmarkEncodingJSON(b, "../example_100k.json") }
<commit_msg>Enable static checking and transformation in JSON benchmarks<commit_after>// Copyright (c) 2017 Opsidian Ltd.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package json_test

import (
	"io/ioutil"
	"testing"

	encoding_json "encoding/json"

	"github.com/opsidian/parsley/combinator"
	"github.com/opsidian/parsley/examples/json/json"
	"github.com/opsidian/parsley/parsley"
	"github.com/opsidian/parsley/text"
)

func benchmarkParsleyJSON(b *testing.B, jsonFilePath string) {
	f, err := text.ReadFile(jsonFilePath)
	if err != nil {
		b.Fatal(err)
	}

	s := combinator.Sentence(json.NewParser())
	r := text.NewReader(f)
	ctx := parsley.NewContext(parsley.NewFileSet(), r)
	ctx.EnableStaticCheck()
	ctx.EnableTransformation()
	if _, err = parsley.Evaluate(ctx, s); err != nil {
		b.Fatal(err)
	}

	for n := 0; n < b.N; n++ {
		_, _ = parsley.Evaluate(ctx, s)
	}
}

func BenchmarkParsleyJSON1k(b *testing.B)   { benchmarkParsleyJSON(b, "../example_1k.json") }
func BenchmarkParsleyJSON10k(b *testing.B)  { benchmarkParsleyJSON(b, "../example_10k.json") }
func BenchmarkParsleyJSON100k(b *testing.B) { benchmarkParsleyJSON(b, "../example_100k.json") }

func benchmarkEncodingJSON(b *testing.B, jsonFilePath string) {
	input, err := ioutil.ReadFile(jsonFilePath)
	if err != nil {
		b.Fatal(err)
	}

	var val interface{}
	if err := encoding_json.Unmarshal(input, &val); err != nil {
		b.Fatal(err)
	}

	for n := 0; n < b.N; n++ {
		var val interface{}
		_ = encoding_json.Unmarshal(input, &val)
	}
}

func BenchmarkEncodingJSON1k(b *testing.B)   { benchmarkEncodingJSON(b, "../example_1k.json") }
func BenchmarkEncodingJSON10k(b *testing.B)  { benchmarkEncodingJSON(b, "../example_10k.json") }
func BenchmarkEncodingJSON100k(b *testing.B) { benchmarkEncodingJSON(b, "../example_100k.json") }
<|endoftext|>


<commit_before>package main

/*
#include <unistd.h>
#include <stdlib.h>
#include <arpa/inet.h>
#include <linux/netlink.h>

#include <linux/netfilter/nfnetlink.h>
#include <linux/netfilter/nfnetlink_conntrack.h>
*/
import "C"

import (
	"fmt"
	"net"
	"os"
	"strconv"
	"syscall"
	"time"
	"unsafe"
	mnl "cgolmnl"
	"cgolmnl/inet"
)

type Nstats struct {
	Addr		net.IP
	Pkts, Bytes	uint64
}

var nstats_map = make(map[string]*Nstats)

func parse_counters_cb(attr *mnl.Nlattr, data interface{}) (int, syscall.Errno) {
	tb := data.(map[uint16]*mnl.Nlattr)
	attr_type := attr.GetType()

	if ret, _ := attr.TypeValid(C.CTA_COUNTERS_MAX); ret < 0 {
		return mnl.MNL_CB_OK, 0
	}

	switch attr_type {
	case C.CTA_COUNTERS_PACKETS: fallthrough
	case C.CTA_COUNTERS_BYTES:
		if ret, err := attr.Validate(mnl.MNL_TYPE_U64); ret < 0 {
			fmt.Fprintf(os.Stderr, "mnl_attr_validate: %s\n", err)
			return mnl.MNL_CB_ERROR, err.(syscall.Errno)
		}
	}
	tb[attr_type] = attr
	return mnl.MNL_CB_OK, 0
}

func parse_counters(nest *mnl.Nlattr, ns *Nstats) {
	tb := make(map[uint16]*mnl.Nlattr, C.CTA_COUNTERS_MAX + 1)

	nest.ParseNested(parse_counters_cb, tb)
	if tb[C.CTA_COUNTERS_PACKETS] != nil {
		ns.Pkts += inet.Be64toh(tb[C.CTA_COUNTERS_PACKETS].U64())
	}
	if tb[C.CTA_COUNTERS_BYTES] != nil {
		ns.Bytes += inet.Be64toh(tb[C.CTA_COUNTERS_BYTES].U64())
	}
}

func parse_ip_cb(attr *mnl.Nlattr, data interface{}) (int, syscall.Errno) {
	tb := data.(map[uint16]*mnl.Nlattr)
	attr_type := attr.GetType()

	if ret, _ := attr.TypeValid(C.CTA_IP_MAX); ret < 0 {
		return mnl.MNL_CB_OK, 0
	}

	switch attr_type {
	case C.CTA_IP_V4_SRC: fallthrough
	case C.CTA_IP_V4_DST:
		if ret, err := attr.Validate(mnl.MNL_TYPE_U32); ret < 0 {
			fmt.Fprintf(os.Stderr, "mnl_attr_validate: %s\n", err)
			return mnl.MNL_CB_ERROR, err.(syscall.Errno)
		}
	case C.CTA_IP_V6_SRC: fallthrough
	case C.CTA_IP_V6_DST:
		if ret, err := attr.Validate2(mnl.MNL_TYPE_BINARY, net.IPv6len); ret < 0 {
			fmt.Fprintf(os.Stderr, "mnl_attr_validate2: %s\n", err)
			return mnl.MNL_CB_ERROR, err.(syscall.Errno)
		}
	}
	tb[attr_type] = attr
	return mnl.MNL_CB_OK, 0
}

func parse_ip(nest *mnl.Nlattr, ns *Nstats) {
	tb := make(map[uint16]*mnl.Nlattr, C.CTA_IP_MAX + 1)

	nest.ParseNested(parse_ip_cb, tb)
	if tb[C.CTA_IP_V4_SRC] != nil {
		ns.Addr = net.IP(tb[C.CTA_IP_V4_SRC].PayloadBytes())
	}
	if tb[C.CTA_IP_V6_SRC] != nil {
		ns.Addr = net.IP(tb[C.CTA_IP_V6_SRC].PayloadBytes())
	}
}

func parse_tuple_cb(attr *mnl.Nlattr, data interface{}) (int, syscall.Errno) {
	tb := data.(map[uint16]*mnl.Nlattr)
	attr_type := attr.GetType()

	if ret, _ := attr.TypeValid(C.CTA_TUPLE_MAX); ret < 0 {
		return mnl.MNL_CB_OK, 0
	}

	switch attr_type {
	case C.CTA_TUPLE_IP:
		if ret, err := attr.Validate(mnl.MNL_TYPE_NESTED); ret < 0 {
			fmt.Fprintf(os.Stderr, "mnl_attr_validate: %s\n", err)
			return mnl.MNL_CB_ERROR, err.(syscall.Errno)
		}
	}
	tb[attr_type] = attr
	return mnl.MNL_CB_OK, 0
}

func parse_tuple(nest *mnl.Nlattr, ns *Nstats) {
	tb := make(map[uint16]*mnl.Nlattr, C.CTA_TUPLE_MAX + 1)

	nest.ParseNested(parse_tuple_cb, tb)
	if tb[C.CTA_TUPLE_IP] != nil {
		parse_ip(tb[C.CTA_TUPLE_IP], ns)
	}
}

func data_attr_cb(attr *mnl.Nlattr, data interface{}) (int, syscall.Errno) {
	tb := data.(map[uint16]*mnl.Nlattr)
	attr_type := attr.GetType()

	if ret, _ := attr.TypeValid(C.CTA_MAX); ret < 0 {
		return mnl.MNL_CB_OK, 0
	}

	switch attr_type {
	case C.CTA_TUPLE_ORIG: fallthrough
	case C.CTA_COUNTERS_ORIG: fallthrough
	case C.CTA_COUNTERS_REPLY:
		if ret, err := attr.Validate(mnl.MNL_TYPE_NESTED); ret < 0 {
			fmt.Fprintf(os.Stderr, "mnl_attr_validate: %s\n", err)
			return mnl.MNL_CB_ERROR, err.(syscall.Errno)
		}
	}
	tb[attr_type] = attr
	return mnl.MNL_CB_OK, 0
}

func data_cb(nlh *mnl.Nlmsghdr, data interface{}) (int, syscall.Errno) {
	tb := make(map[uint16]*mnl.Nlattr, C.CTA_MAX + 1)
	ns := &Nstats{}

	nlh.Parse(SizeofNfgenmsg, data_attr_cb, tb)
	if tb[C.CTA_TUPLE_ORIG] != nil {
		parse_tuple(tb[C.CTA_TUPLE_ORIG], ns)
	}

	if tb[C.CTA_COUNTERS_ORIG] != nil {
		parse_counters(tb[C.CTA_COUNTERS_ORIG], ns)
	}

	if tb[C.CTA_COUNTERS_REPLY] != nil {
		parse_counters(tb[C.CTA_COUNTERS_REPLY], ns)
	}

	cur := nstats_map[ns.Addr.String()]
	if cur == nil {
		cur = ns
		nstats_map[ns.Addr.String()] = ns
	}
	cur.Pkts += ns.Pkts
	cur.Bytes += ns.Bytes

	return mnl.MNL_CB_OK, 0
}

func handle(nl *mnl.Socket) int {
	buf := make([]byte, mnl.MNL_SOCKET_BUFFER_SIZE)

	nrcv, err := nl.Recvfrom(buf)
	if err != nil {
		// It only happens if NETLINK_NO_ENOBUFS is not set, it means
		// we are leaking statistics.
		if err == syscall.ENOBUFS {
			fmt.Fprintf(os.Stderr, "The daemon has hit ENOBUFS, you can " +
				"increase the size of your receiver " +
				"buffer to mitigate this or enable " +
				"reliable delivery.\n")
			// http://stackoverflow.com/questions/7933460/how-do-you-write-multiline-strings-in-go
			// `line1
			// line2
			// line3`
		} else {
			fmt.Fprintf(os.Stderr, "mnl_socket_recvfrom: %s\n", err)
		}
		return -1
	}

	if ret, err := mnl.CbRun(buf[:nrcv], 0, 0, data_cb, nil); ret <= mnl.MNL_CB_ERROR {
		fmt.Fprintf(os.Stderr, "mnl_cb_run: %s", err)
		return -1
	} else if ret <= mnl.MNL_CB_STOP {
		return 0
	}

	return 0
}

func main() {
	if len(os.Args) != 2 {
		fmt.Printf("Usage: %s <poll-secs>\n", os.Args[0])
		os.Exit(C.EXIT_FAILURE)
	}
	secs, _ := strconv.Atoi(os.Args[1])

	fmt.Printf("Polling every %d seconds from kernel...\n", secs)

	// Set high priority for this process, less chances to overrun
	// the netlink receiver buffer since the scheduler gives this process
	// more chances to run.
	C.nice(C.int(-20))

	// Open netlink socket to operate with netfilter
	nl, err := mnl.SocketOpen(C.NETLINK_NETFILTER)
	if err != nil {
		fmt.Fprintf(os.Stderr, "mnl_socket_open: %s\n", err)
		os.Exit(C.EXIT_FAILURE)
	}

	// Subscribe to destroy events to avoid leaking counters. The same
	// socket is used to periodically atomically dump and reset counters.
	if err := nl.Bind(C.NF_NETLINK_CONNTRACK_DESTROY, mnl.MNL_SOCKET_AUTOPID); err != nil {
		fmt.Fprintf(os.Stderr, "mnl_socket_bind: %s\n", err)
		os.Exit(C.EXIT_FAILURE)
	}

	// Set netlink receiver buffer to 16 MBytes, to avoid packet drops */
	buffersize := (1 << 22)
	C.setsockopt(C.int(nl.Fd()), C.SOL_SOCKET, C.SO_RCVBUFFORCE,
		unsafe.Pointer(&buffersize), SizeofSocklen_t)

	// The two tweaks below enable reliable event delivery, packets may
	// be dropped if the netlink receiver buffer overruns. This happens ...
	//
	// a) if the kernel spams this user-space process until the receiver
	//    is filled.
	//
	// or:
	//
	// b) if the user-space process does not pull messages from the
	//    receiver buffer so often.
	nl.SetsockoptCint(C.NETLINK_BROADCAST_ERROR, 1)
	nl.SetsockoptCint(C.NETLINK_NO_ENOBUFS, 1)

	buf := make([]byte, mnl.MNL_SOCKET_BUFFER_SIZE)
	nlh, _ := mnl.NlmsgPutHeaderBytes(buf)
	// Counters are atomically zeroed in each dump
	nlh.Type = (C.NFNL_SUBSYS_CTNETLINK << 8) | C.IPCTNL_MSG_CT_GET_CTRZERO
	nlh.Flags = C.NLM_F_REQUEST | C.NLM_F_DUMP

	nfh := (*Nfgenmsg)(nlh.PutExtraHeader(SizeofNfgenmsg))
	nfh.Nfgen_family = C.AF_INET
	nfh.Version = C.NFNETLINK_V0
	nfh.Res_id = 0

	// Filter by mark: We only want to dump entries whose mark is zero
	nlh.PutU32(C.CTA_MARK, inet.Htonl(0))
	nlh.PutU32(C.CTA_MARK_MASK, inet.Htonl(0xffffffff))

	// prepare for epoll
	epfd, err := syscall.EpollCreate1(0)
	if err != nil {
		fmt.Fprintf(os.Stderr, "EpollCreate1: %s", err)
		os.Exit(C.EXIT_FAILURE)
	}
	defer syscall.Close(epfd)

	var event syscall.EpollEvent
	events := make([]syscall.EpollEvent, 64) // XXX: magic number
	event.Events = syscall.EPOLLIN
	event.Fd = int32(nl.Fd())
	if err := syscall.EpollCtl(epfd, syscall.EPOLL_CTL_ADD, int(event.Fd), &event); err != nil {
		fmt.Fprintf(os.Stderr, "EpollCtl: %s", err)
		os.Exit(C.EXIT_FAILURE)
	}

	// Every N seconds ...
	ticker := time.NewTicker(time.Second * time.Duration(secs))
	go func() {
		for _ = range ticker.C {
			if _, err := nl.SendNlmsg(nlh);  err != nil {
				fmt.Fprintf(os.Stderr, "mnl_socket_sendto: %s\n", err)
				os.Exit(C.EXIT_FAILURE)
			}
		}
	}()

	for true {
		nevents, err := syscall.EpollWait(epfd, events, -1)
		if err != nil {
			if err == syscall.EINTR {
				continue
			}
			fmt.Fprintf(os.Stderr, "EpollWait: %s\n", err)
			os.Exit(C.EXIT_FAILURE)
		}

		// Handled event and periodic atomic-dump-and-reset messages
		for i := 0; i < nevents; i++ {
			if events[i].Fd != event.Fd {
				continue
			}
			if ret := handle(nl); ret < 0 {
				fmt.Fprintf(os.Stderr, "handle failed: %d\n", ret)
				os.Exit(C.EXIT_FAILURE)
			}

			// print the content of the list
			for k, v := range nstats_map {
				fmt.Printf("src=%s ", k)
				fmt.Printf("counters %d %d\n", v.Pkts, v.Bytes)
			}
		}
	}
}
<commit_msg>examples: update<commit_after>package main

/*
#include <unistd.h>
#include <stdlib.h>
#include <arpa/inet.h>
#include <linux/netlink.h>

#include <linux/netfilter/nfnetlink.h>
#include <linux/netfilter/nfnetlink_conntrack.h>
*/
import "C"

import (
	"fmt"
	"net"
	"os"
	"strconv"
	"syscall"
	"time"
	"unsafe"
	mnl "cgolmnl"
	"cgolmnl/inet"
)

type Nstats struct {
	Addr		net.IP
	Pkts, Bytes	uint64
}

var nstats_map = make(map[string]*Nstats)

func parse_counters_cb(attr *mnl.Nlattr, data interface{}) (int, syscall.Errno) {
	tb := data.(map[uint16]*mnl.Nlattr)
	attr_type := attr.GetType()

	if ret, _ := attr.TypeValid(C.CTA_COUNTERS_MAX); ret < 0 {
		return mnl.MNL_CB_OK, 0
	}

	switch attr_type {
	case C.CTA_COUNTERS_PACKETS: fallthrough
	case C.CTA_COUNTERS_BYTES:
		if ret, err := attr.Validate(mnl.MNL_TYPE_U64); ret < 0 {
			fmt.Fprintf(os.Stderr, "mnl_attr_validate: %s\n", err)
			return mnl.MNL_CB_ERROR, err.(syscall.Errno)
		}
	}
	tb[attr_type] = attr
	return mnl.MNL_CB_OK, 0
}

func parse_counters(nest *mnl.Nlattr, ns *Nstats) {
	tb := make(map[uint16]*mnl.Nlattr, C.CTA_COUNTERS_MAX + 1)

	nest.ParseNested(parse_counters_cb, tb)
	if tb[C.CTA_COUNTERS_PACKETS] != nil {
		ns.Pkts += inet.Be64toh(tb[C.CTA_COUNTERS_PACKETS].U64())
	}
	if tb[C.CTA_COUNTERS_BYTES] != nil {
		ns.Bytes += inet.Be64toh(tb[C.CTA_COUNTERS_BYTES].U64())
	}
}

func parse_ip_cb(attr *mnl.Nlattr, data interface{}) (int, syscall.Errno) {
	tb := data.(map[uint16]*mnl.Nlattr)
	attr_type := attr.GetType()

	if ret, _ := attr.TypeValid(C.CTA_IP_MAX); ret < 0 {
		return mnl.MNL_CB_OK, 0
	}

	switch attr_type {
	case C.CTA_IP_V4_SRC: fallthrough
	case C.CTA_IP_V4_DST:
		if ret, err := attr.Validate(mnl.MNL_TYPE_U32); ret < 0 {
			fmt.Fprintf(os.Stderr, "mnl_attr_validate: %s\n", err)
			return mnl.MNL_CB_ERROR, err.(syscall.Errno)
		}
	case C.CTA_IP_V6_SRC: fallthrough
	case C.CTA_IP_V6_DST:
		if ret, err := attr.Validate2(mnl.MNL_TYPE_BINARY, net.IPv6len); ret < 0 {
			fmt.Fprintf(os.Stderr, "mnl_attr_validate2: %s\n", err)
			return mnl.MNL_CB_ERROR, err.(syscall.Errno)
		}
	}
	tb[attr_type] = attr
	return mnl.MNL_CB_OK, 0
}

func parse_ip(nest *mnl.Nlattr, ns *Nstats) {
	tb := make(map[uint16]*mnl.Nlattr, C.CTA_IP_MAX + 1)

	nest.ParseNested(parse_ip_cb, tb)
	if tb[C.CTA_IP_V4_SRC] != nil {
		ns.Addr = net.IP(tb[C.CTA_IP_V4_SRC].PayloadBytes())
	}
	if tb[C.CTA_IP_V6_SRC] != nil {
		ns.Addr = net.IP(tb[C.CTA_IP_V6_SRC].PayloadBytes())
	}
}

func parse_tuple_cb(attr *mnl.Nlattr, data interface{}) (int, syscall.Errno) {
	tb := data.(map[uint16]*mnl.Nlattr)
	attr_type := attr.GetType()

	if ret, _ := attr.TypeValid(C.CTA_TUPLE_MAX); ret < 0 {
		return mnl.MNL_CB_OK, 0
	}

	switch attr_type {
	case C.CTA_TUPLE_IP:
		if ret, err := attr.Validate(mnl.MNL_TYPE_NESTED); ret < 0 {
			fmt.Fprintf(os.Stderr, "mnl_attr_validate: %s\n", err)
			return mnl.MNL_CB_ERROR, err.(syscall.Errno)
		}
	}
	tb[attr_type] = attr
	return mnl.MNL_CB_OK, 0
}

func parse_tuple(nest *mnl.Nlattr, ns *Nstats) {
	tb := make(map[uint16]*mnl.Nlattr, C.CTA_TUPLE_MAX + 1)

	nest.ParseNested(parse_tuple_cb, tb)
	if tb[C.CTA_TUPLE_IP] != nil {
		parse_ip(tb[C.CTA_TUPLE_IP], ns)
	}
}

func data_attr_cb(attr *mnl.Nlattr, data interface{}) (int, syscall.Errno) {
	tb := data.(map[uint16]*mnl.Nlattr)
	attr_type := attr.GetType()

	if ret, _ := attr.TypeValid(C.CTA_MAX); ret < 0 {
		return mnl.MNL_CB_OK, 0
	}

	switch attr_type {
	case C.CTA_TUPLE_ORIG: fallthrough
	case C.CTA_COUNTERS_ORIG: fallthrough
	case C.CTA_COUNTERS_REPLY:
		if ret, err := attr.Validate(mnl.MNL_TYPE_NESTED); ret < 0 {
			fmt.Fprintf(os.Stderr, "mnl_attr_validate: %s\n", err)
			return mnl.MNL_CB_ERROR, err.(syscall.Errno)
		}
	}
	tb[attr_type] = attr
	return mnl.MNL_CB_OK, 0
}

func data_cb(nlh *mnl.Nlmsghdr, data interface{}) (int, syscall.Errno) {
	tb := make(map[uint16]*mnl.Nlattr, C.CTA_MAX + 1)
	ns := &Nstats{}

	nlh.Parse(SizeofNfgenmsg, data_attr_cb, tb)
	if tb[C.CTA_TUPLE_ORIG] != nil {
		parse_tuple(tb[C.CTA_TUPLE_ORIG], ns)
	}

	if tb[C.CTA_COUNTERS_ORIG] != nil {
		parse_counters(tb[C.CTA_COUNTERS_ORIG], ns)
	}

	if tb[C.CTA_COUNTERS_REPLY] != nil {
		parse_counters(tb[C.CTA_COUNTERS_REPLY], ns)
	}

	cur := nstats_map[ns.Addr.String()]
	if cur == nil {
		cur = ns
		nstats_map[ns.Addr.String()] = ns
	}
	cur.Pkts += ns.Pkts
	cur.Bytes += ns.Bytes

	return mnl.MNL_CB_OK, 0
}

func handle(nl *mnl.Socket) int {
	buf := make([]byte, mnl.MNL_SOCKET_BUFFER_SIZE)

	nrcv, err := nl.Recvfrom(buf)
	if err != nil {
		// It only happens if NETLINK_NO_ENOBUFS is not set, it means
		// we are leaking statistics.
		if err == syscall.ENOBUFS {
			fmt.Fprintf(os.Stderr, "The daemon has hit ENOBUFS, you can " +
				"increase the size of your receiver " +
				"buffer to mitigate this or enable " +
				"reliable delivery.\n")
			// http://stackoverflow.com/questions/7933460/how-do-you-write-multiline-strings-in-go
			// `line1
			// line2
			// line3`
		} else {
			fmt.Fprintf(os.Stderr, "mnl_socket_recvfrom: %s\n", err)
		}
		return -1
	}

	if ret, err := mnl.CbRun(buf[:nrcv], 0, 0, data_cb, nil); ret <= mnl.MNL_CB_ERROR {
		fmt.Fprintf(os.Stderr, "mnl_cb_run: %s", err)
		return -1
	} else if ret <= mnl.MNL_CB_STOP {
		return 0
	}

	return 0
}

func main() {
	if len(os.Args) != 2 {
		fmt.Printf("Usage: %s <poll-secs>\n", os.Args[0])
		os.Exit(C.EXIT_FAILURE)
	}
	secs, _ := strconv.Atoi(os.Args[1])

	fmt.Printf("Polling every %d seconds from kernel...\n", secs)

	// Set high priority for this process, less chances to overrun
	// the netlink receiver buffer since the scheduler gives this process
	// more chances to run.
	C.nice(C.int(-20))

	// Open netlink socket to operate with netfilter
	nl, err := mnl.SocketOpen(C.NETLINK_NETFILTER)
	if err != nil {
		fmt.Fprintf(os.Stderr, "mnl_socket_open: %s\n", err)
		os.Exit(C.EXIT_FAILURE)
	}
	defer nl.Close()

	// Subscribe to destroy events to avoid leaking counters. The same
	// socket is used to periodically atomically dump and reset counters.
	if err := nl.Bind(C.NF_NETLINK_CONNTRACK_DESTROY, mnl.MNL_SOCKET_AUTOPID); err != nil {
		fmt.Fprintf(os.Stderr, "mnl_socket_bind: %s\n", err)
		os.Exit(C.EXIT_FAILURE)
	}

	// Set netlink receiver buffer to 16 MBytes, to avoid packet drops */
	buffersize := (1 << 22)
	C.setsockopt(C.int(nl.Fd()), C.SOL_SOCKET, C.SO_RCVBUFFORCE,
		unsafe.Pointer(&buffersize), SizeofSocklen_t)

	// The two tweaks below enable reliable event delivery, packets may
	// be dropped if the netlink receiver buffer overruns. This happens ...
	//
	// a) if the kernel spams this user-space process until the receiver
	//    is filled.
	//
	// or:
	//
	// b) if the user-space process does not pull messages from the
	//    receiver buffer so often.
	nl.SetsockoptCint(C.NETLINK_BROADCAST_ERROR, 1)
	nl.SetsockoptCint(C.NETLINK_NO_ENOBUFS, 1)

	buf := make([]byte, mnl.MNL_SOCKET_BUFFER_SIZE)
	nlh, _ := mnl.NlmsgPutHeaderBytes(buf)
	// Counters are atomically zeroed in each dump
	nlh.Type = (C.NFNL_SUBSYS_CTNETLINK << 8) | C.IPCTNL_MSG_CT_GET_CTRZERO
	nlh.Flags = C.NLM_F_REQUEST | C.NLM_F_DUMP

	nfh := (*Nfgenmsg)(nlh.PutExtraHeader(SizeofNfgenmsg))
	nfh.Nfgen_family = C.AF_INET
	nfh.Version = C.NFNETLINK_V0
	nfh.Res_id = 0

	// Filter by mark: We only want to dump entries whose mark is zero
	nlh.PutU32(C.CTA_MARK, inet.Htonl(0))
	nlh.PutU32(C.CTA_MARK_MASK, inet.Htonl(0xffffffff))

	// Every N seconds ...
	ticker := time.NewTicker(time.Second * time.Duration(secs))
	go func() {
		for _ = range ticker.C {
			// periodic atomic-dump-and-reset messages
			if _, err := nl.SendNlmsg(nlh);  err != nil {
				fmt.Fprintf(os.Stderr, "mnl_socket_sendto: %s\n", err)
				os.Exit(C.EXIT_FAILURE)
			}

			// print the content of the list
			for k, v := range nstats_map {
				fmt.Printf("src=%s ", k)
				fmt.Printf("counters %d %d\n", v.Pkts, v.Bytes)
			}
		}
	}()

	// prepare for epoll
	epfd, err := syscall.EpollCreate1(0)
	if err != nil {
		fmt.Fprintf(os.Stderr, "EpollCreate1: %s", err)
		os.Exit(C.EXIT_FAILURE)
	}
	defer syscall.Close(epfd)

	var event syscall.EpollEvent
	events := make([]syscall.EpollEvent, 64) // XXX: magic number
	event.Events = syscall.EPOLLIN
	event.Fd = int32(nl.Fd())
	if err := syscall.EpollCtl(epfd, syscall.EPOLL_CTL_ADD, int(event.Fd), &event); err != nil {
		fmt.Fprintf(os.Stderr, "EpollCtl: %s", err)
		os.Exit(C.EXIT_FAILURE)
	}
	for true {
		nevents, err := syscall.EpollWait(epfd, events, -1)
		if err != nil {
			if err == syscall.EINTR {
				continue
			}
			fmt.Fprintf(os.Stderr, "EpollWait: %s\n", err)
			os.Exit(C.EXIT_FAILURE)
		}

		// Handled event
		for i := 0; i < nevents; i++ {
			if events[i].Fd != event.Fd {
				continue
			}
			if ret := handle(nl); ret < 0 {
				fmt.Fprintf(os.Stderr, "handle failed: %d\n", ret)
				os.Exit(C.EXIT_FAILURE)
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package index

import (
	"reflect"
	"testing"
	"time"

	"tchaik.com/index/attr"
)

type testTrack struct {
	Name, Album, Artist, Composer           string
	TrackNumber, DiscNumber, Duration, Year int
	stringsMap                              map[string][]string
}

func (f testTrack) GetString(k string) string {
	switch k {
	case "Name":
		return f.Name
	case "Album":
		return f.Album
	case "Artist":
		return f.Artist
	case "Composer":
		return f.Composer
	}
	return ""
}

func (f testTrack) GetStrings(k string) []string {
	v, ok := f.stringsMap[k]
	if ok {
		return v
	}

	switch k {
	case "Artist", "AlbumArtist", "Composer":
		return DefaultGetStrings(f, k)
	}
	return nil
}

func (f testTrack) GetInt(k string) int {
	switch k {
	case "TrackNumber":
		return f.TrackNumber
	case "DiscNumber":
		return f.DiscNumber
	case "Duration":
		return f.Duration
	case "Year":
		return f.Year
	}
	return 0
}

func (f testTrack) GetBool(k string) bool {
	return false
}

func (f testTrack) GetTime(string) time.Time {
	return time.Time{}
}

type testTracker []testTrack

func (d testTracker) Tracks() []Track {
	result := make([]Track, len(d))
	for i, x := range d {
		result[i] = x
	}
	return result
}

func TestByAttr(t *testing.T) {
	trackListing := []testTrack{
		{Name: "A", Album: "Album A"},
		{Name: "B", Album: "Album A"},
		{Name: "C", Album: "Album B"},
		{Name: "D", Album: "Album B"},
	}

	expected := map[string][]Track{
		"Album A": {trackListing[0], trackListing[1]},
		"Album B": {trackListing[2], trackListing[3]},
	}

	attrGroup := By(attr.String("Album")).Collect(testTracker(trackListing[:]))
	SortKeysByGroupName(attrGroup)

	nkm := nameKeyMap(attrGroup)
	for n, v := range expected {
		k, ok := nkm[n]
		if !ok {
			t.Errorf("%v is not a key of nkm", n)
		}

		gr := attrGroup.Get(Key(k))
		if gr == nil {
			t.Errorf("attrGroup.Get(%v) = nil, expected non-nil!", k)
		}

		got := gr.Tracks()
		if !reflect.DeepEqual(v, got) {
			t.Errorf("gr.Tracks() = %#v, expected: %#v", got, v)
		}
	}
}

func TestSubCollect(t *testing.T) {
	album1 := "Mahler Symphonies"
	album2 := "Shostakovich Symphonies"

	prefix1 := "Symphony No. 1 in D"
	prefix2 := "A B C Y"

	trackListing := []testTrack{
		{Name: prefix1 + ": I. Langsam, schleppend - Immer sehr gemächlich", Album: album1},
		{Name: prefix1 + ": II. Kräftig bewegt, doch nicht zu schnell - Recht gemächlich", Album: album1},
		{Name: prefix1 + ": III. Feierlich und gemessen, ohne zu schleppen", Album: album1},
		{Name: prefix2 + " 1", Album: album2},
		{Name: prefix2 + " 2", Album: album2},
		{Name: prefix2 + " 2", Album: album2},
	}
	expectedAlbums := []string{album1, album2}

	albums := By(attr.String("Album")).Collect(testTracker(trackListing[:]))
	SortKeysByGroupName(albums)
	albNames := names(albums)

	if !reflect.DeepEqual(albNames, expectedAlbums) {
		t.Errorf("albums.Names() = %v, expected %#v", names, expectedAlbums)
	}

	albPfx := SubCollect(albums, ByPrefix("Name"))
	albPfxNames := names(albPfx)

	if !reflect.DeepEqual(albPfxNames, expectedAlbums[:]) {
		t.Errorf("albPfx.Names() = %v, expected %#v", albPfxNames, expectedAlbums)
	}

	nkm := nameKeyMap(albPfx)
	prefixGroup := albPfx.Get(nkm[album1])
	pfxCol, ok := prefixGroup.(Collection)
	if !ok {
		t.Errorf("expected a Collection, but got %T", prefixGroup)
	}

	pfxColNames := names(pfxCol)
	expectedPrefixGroupNames := []string{prefix1}

	if !reflect.DeepEqual(pfxColNames, expectedPrefixGroupNames) {
		t.Errorf("prefixCollection.Names() = %#v, expected %#v", pfxColNames, expectedPrefixGroupNames)
	}
}
<commit_msg>Fix incorrect var in printf.<commit_after>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package index

import (
	"reflect"
	"testing"
	"time"

	"tchaik.com/index/attr"
)

type testTrack struct {
	Name, Album, Artist, Composer           string
	TrackNumber, DiscNumber, Duration, Year int
	stringsMap                              map[string][]string
}

func (f testTrack) GetString(k string) string {
	switch k {
	case "Name":
		return f.Name
	case "Album":
		return f.Album
	case "Artist":
		return f.Artist
	case "Composer":
		return f.Composer
	}
	return ""
}

func (f testTrack) GetStrings(k string) []string {
	v, ok := f.stringsMap[k]
	if ok {
		return v
	}

	switch k {
	case "Artist", "AlbumArtist", "Composer":
		return DefaultGetStrings(f, k)
	}
	return nil
}

func (f testTrack) GetInt(k string) int {
	switch k {
	case "TrackNumber":
		return f.TrackNumber
	case "DiscNumber":
		return f.DiscNumber
	case "Duration":
		return f.Duration
	case "Year":
		return f.Year
	}
	return 0
}

func (f testTrack) GetBool(k string) bool {
	return false
}

func (f testTrack) GetTime(string) time.Time {
	return time.Time{}
}

type testTracker []testTrack

func (d testTracker) Tracks() []Track {
	result := make([]Track, len(d))
	for i, x := range d {
		result[i] = x
	}
	return result
}

func TestByAttr(t *testing.T) {
	trackListing := []testTrack{
		{Name: "A", Album: "Album A"},
		{Name: "B", Album: "Album A"},
		{Name: "C", Album: "Album B"},
		{Name: "D", Album: "Album B"},
	}

	expected := map[string][]Track{
		"Album A": {trackListing[0], trackListing[1]},
		"Album B": {trackListing[2], trackListing[3]},
	}

	attrGroup := By(attr.String("Album")).Collect(testTracker(trackListing[:]))
	SortKeysByGroupName(attrGroup)

	nkm := nameKeyMap(attrGroup)
	for n, v := range expected {
		k, ok := nkm[n]
		if !ok {
			t.Errorf("%v is not a key of nkm", n)
		}

		gr := attrGroup.Get(Key(k))
		if gr == nil {
			t.Errorf("attrGroup.Get(%v) = nil, expected non-nil!", k)
		}

		got := gr.Tracks()
		if !reflect.DeepEqual(v, got) {
			t.Errorf("gr.Tracks() = %#v, expected: %#v", got, v)
		}
	}
}

func TestSubCollect(t *testing.T) {
	album1 := "Mahler Symphonies"
	album2 := "Shostakovich Symphonies"

	prefix1 := "Symphony No. 1 in D"
	prefix2 := "A B C Y"

	trackListing := []testTrack{
		{Name: prefix1 + ": I. Langsam, schleppend - Immer sehr gemächlich", Album: album1},
		{Name: prefix1 + ": II. Kräftig bewegt, doch nicht zu schnell - Recht gemächlich", Album: album1},
		{Name: prefix1 + ": III. Feierlich und gemessen, ohne zu schleppen", Album: album1},
		{Name: prefix2 + " 1", Album: album2},
		{Name: prefix2 + " 2", Album: album2},
		{Name: prefix2 + " 2", Album: album2},
	}
	expectedAlbums := []string{album1, album2}

	albums := By(attr.String("Album")).Collect(testTracker(trackListing[:]))
	SortKeysByGroupName(albums)
	albNames := names(albums)

	if !reflect.DeepEqual(albNames, expectedAlbums) {
		t.Errorf("albums.Names() = %v, expected %#v", albNames, expectedAlbums)
	}

	albPfx := SubCollect(albums, ByPrefix("Name"))
	albPfxNames := names(albPfx)

	if !reflect.DeepEqual(albPfxNames, expectedAlbums[:]) {
		t.Errorf("albPfx.Names() = %v, expected %#v", albPfxNames, expectedAlbums)
	}

	nkm := nameKeyMap(albPfx)
	prefixGroup := albPfx.Get(nkm[album1])
	pfxCol, ok := prefixGroup.(Collection)
	if !ok {
		t.Errorf("expected a Collection, but got %T", prefixGroup)
	}

	pfxColNames := names(pfxCol)
	expectedPrefixGroupNames := []string{prefix1}

	if !reflect.DeepEqual(pfxColNames, expectedPrefixGroupNames) {
		t.Errorf("prefixCollection.Names() = %#v, expected %#v", pfxColNames, expectedPrefixGroupNames)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/control-center/serviced/volume"
	"github.com/docker/docker/pkg/units"
	"github.com/jessevdk/go-flags"
)

// VolumeCreate is the subcommand for creating a new volume on a driver
type VolumeCreate struct {
	Path flags.Filename `long:"driver" short:"d" description:"Path of the driver"`
	Args struct {
		Name string `description:"Name of the volume to create"`
	} `positional-args:"yes" required:"yes"`
}

// VolumeMount is the subcommand for mounting an existing volume from a driver
type VolumeMount struct {
	Path flags.Filename `long:"driver" short:"d" description:"Path of the driver"`
	Args struct {
		Name string `description:"Name of the volume to mount"`
	} `positional-args:"yes" required:"yes"`
}

// VolumeRemove is the subcommand for deleting an existing volume from a driver
type VolumeRemove struct {
	Path flags.Filename `long:"driver" short:"d" description:"Path of the driver"`
	Args struct {
		Name string `description:"Name of the volume to remove"`
	} `positional-args:"yes" required:"yes"`
}

// VolumeResize is the subcommand for enlarging an existing devicemapper volume
type VolumeResize struct {
	Path flags.Filename `long:"driver" short:"d" description:"Path of the driver"`
	Args struct {
		Name string `description:"Name of the volume to mount"`
		Size string `description:"New size of the volume"`
	} `positional-args:"yes" required:"yes"`
}

// Execute creates a new volume on a driver
func (c *VolumeCreate) Execute(args []string) error {
	App.initializeLogging()
	directory := GetDefaultDriver(string(c.Path))
	driver, err := InitDriverIfExists(directory)
	if err != nil {
		log.Fatal(err)
	}
	logger := log.WithFields(log.Fields{
		"directory": driver.Root(),
		"type":      driver.DriverType(),
		"volume":    c.Args.Name,
	})
	logger.Info("Creating volume")
	vol, err := driver.Create(c.Args.Name)
	if err != nil {
		logger.Fatal(err)
	}
	logger.WithFields(log.Fields{
		"mount": vol.Path(),
	}).Info("Volume mounted")
	return nil
}

// Execute mounts an existing volume from a driver
func (c *VolumeMount) Execute(args []string) error {
	App.initializeLogging()
	directory := GetDefaultDriver(string(c.Path))
	driver, err := InitDriverIfExists(directory)
	if err != nil {
		log.Fatal(err)
	}
	logger := log.WithFields(log.Fields{
		"directory": driver.Root(),
		"type":      driver.DriverType(),
		"volume":    c.Args.Name,
	})
	logger.Info("Mounting volume")
	vol, err := driver.Get(c.Args.Name)
	if err != nil {
		logger.Fatal(err)
	}
	logger.WithFields(log.Fields{
		"mount": vol.Path(),
	}).Info("Volume mounted")
	return nil
}

// Execute removes an existing volume from a driver
func (c *VolumeRemove) Execute(args []string) error {
	App.initializeLogging()
	directory := GetDefaultDriver(string(c.Path))
	driver, err := InitDriverIfExists(directory)
	if err != nil {
		log.Fatal(err)
	}
	logger := log.WithFields(log.Fields{
		"directory": driver.Root(),
		"type":      driver.DriverType(),
		"volume":    c.Args.Name,
	})
	if !driver.Exists(c.Args.Name) {
		logger.Fatal("Volume does not exist")
	}
	logger.Info("Removing volume")
	if err := driver.Remove(c.Args.Name); err != nil {
		logger.Fatal(err)
	}
	logger.Info("Volume deleted")
	return nil
}

// Resize increases the space available to an existing volume
func (c *VolumeResize) Execute(args []string) error {
	App.initializeLogging()
	directory := GetDefaultDriver(string(c.Path))
	driver, err := InitDriverIfExists(directory)
	if err != nil {
		log.Fatal(err)
	}
	logger := log.WithFields(log.Fields{
		"directory": driver.Root(),
		"volume":    c.Args.Name,
		"type":      driver.DriverType(),
	})
	if driver.DriverType() != volume.DriverTypeDeviceMapper {
		logger.Fatal("Only devicemapper volumes can be resized")
	}
	if !driver.Exists(c.Args.Name) {
		logger.Fatal("Volume does not exist")
	}
	size, err := units.RAMInBytes(c.Args.Size)
	if err != nil {
		return err
	}
	if err := driver.Resize(c.Args.Name, uint64(size)); err != nil {
		logger.Fatal(err)
	}
	logger.Info("Volume resized")
	return nil
}
<commit_msg>ported volume sync to develop<commit_after>// Copyright 2015 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/control-center/serviced/volume"
	"github.com/docker/docker/pkg/units"
	"github.com/jessevdk/go-flags"
)

// VolumeCreate is the subcommand for creating a new volume on a driver
type VolumeCreate struct {
	Path flags.Filename `long:"driver" short:"d" description:"Path of the driver"`
	Args struct {
		Name string `description:"Name of the volume to create"`
	} `positional-args:"yes" required:"yes"`
}

// VolumeMount is the subcommand for mounting an existing volume from a driver
type VolumeMount struct {
	Path flags.Filename `long:"driver" short:"d" description:"Path of the driver"`
	Args struct {
		Name string `description:"Name of the volume to mount"`
	} `positional-args:"yes" required:"yes"`
}

// VolumeRemove is the subcommand for deleting an existing volume from a driver
type VolumeRemove struct {
	Path flags.Filename `long:"driver" short:"d" description:"Path of the driver"`
	Args struct {
		Name string `description:"Name of the volume to remove"`
	} `positional-args:"yes" required:"yes"`
}

// DriverSync is the subcommand for syncing two volumes
type DriverSync struct {
	Create bool   `description:"Indicates that the destination driver should be created" long:"create" short:"c"`
	Type   string `description:"Type of the destination driver (btrfs|devicemapper|rsync)" long:"type" short:"t"`
	Args   struct {
		SourcePath      flags.Filename `description:"Path of the source driver"`
		DestinationPath flags.Filename `description:"Path of the destionation"`
	} `positional-args:"yes" required:"yes"`
}

//Execute syncs to volume
func (c *DriverSync) Execute(args []string) error {
	App.initializeLogging()
	destinationPath := string(c.Args.DestinationPath)
	sourcePath := string(c.Args.SourcePath)
	logger := log.WithFields(log.Fields{
		"destination":destinationPath,
		"source":sourcePath})
	if c.Create {
		logger = logger.WithFields(log.Fields{
			"type":c.Type,
		})
		logger.Info("Determining driver type for destination")
		destinationDriverType, err := volume.StringToDriverType(c.Type)
		if err != nil {
			logger.Fatal(err)
		}
		logger.Info("Creating driver for destination")
		initStatus := volume.InitDriver(destinationDriverType, destinationPath, App.Options.Options)
		if initStatus != nil {
			logger.Fatal(initStatus)
		}
	}
	destinationDirectory := GetDefaultDriver(destinationPath)
	logger.Info("Getting driver for destination")
	destinationDriver, err := InitDriverIfExists(destinationDirectory)
	if err != nil {
		logger.Fatal(err)
	}
	sourceDirectory := GetDefaultDriver(string(sourcePath))
	logger.Info("Getting driver for source")
	sourceDriver, err := InitDriverIfExists(sourceDirectory)
	if err != nil {
		logger.Fatal(err)
	}
	sourceVolumes := sourceDriver.List()
	logger = logger.WithFields(log.Fields{
		"numberOfVolumes": len(sourceVolumes),
	})
	for i := 0; i < len(sourceVolumes); i++ {
		volumeName := sourceVolumes[i]
		volumeLogger := logger.WithFields(log.Fields{
			"volumeName": volumeName,
		})
		volumeLogger.Info("Syncing data from source volume")
		sourceVolume, err := sourceDriver.Get(volumeName)
		if err != nil {
			volumeLogger.Fatal(err)
		}
		if !destinationDriver.Exists(volumeName) {
			logger.Info("Creating destination volume")
			createVolume(string(destinationPath), volumeName)
		}
		volumeLogger = volumeLogger.WithFields(log.Fields{
			"sourcePath":sourceVolume.Path(),
		})
		volumeLogger.Info("using rsync to sync source to destination")
		rsync(sourceVolume.Path(), string(c.Args.DestinationPath))
	}
	return nil
}

func rsync(sourcePath string, destinationPath string) {
	rsyncBin, err := exec.LookPath("rsync")
	if err != nil {
		log.Fatal(err)
	}
	rsyncArgv := []string{"-a", "--progress", "--stats", "--human-readable", sourcePath, destinationPath}
	rsync := exec.Command(rsyncBin, rsyncArgv...)
	log.Info("Starting rsync command")
	rsync.Stdout = os.Stdout
	rsync.Stderr = os.Stderr
	rsync.Run()
}

// VolumeResize is the subcommand for enlarging an existing devicemapper volume
type VolumeResize struct {
	Path flags.Filename `long:"driver" short:"d" description:"Path of the driver"`
	Args struct {
		Name string `description:"Name of the volume to mount"`
		Size string `description:"New size of the volume"`
	} `positional-args:"yes" required:"yes"`
}

// Execute creates a new volume on a driver
func (c *VolumeCreate) Execute(args []string) error {
	App.initializeLogging()
	directory := GetDefaultDriver(string(c.Path))
	driver, err := InitDriverIfExists(directory)
	if err != nil {
		log.Fatal(err)
	}
	logger := log.WithFields(log.Fields{
		"directory": driver.Root(),
		"type":      driver.DriverType(),
		"volume":    c.Args.Name,
	})
	logger.Info("Creating volume")
	vol, err := driver.Create(c.Args.Name)
	if err != nil {
		logger.Fatal(err)
	}
	logger.WithFields(log.Fields{
		"mount": vol.Path(),
	}).Info("Volume mounted")
	return nil
}

// Execute mounts an existing volume from a driver
func (c *VolumeMount) Execute(args []string) error {
	App.initializeLogging()
	directory := GetDefaultDriver(string(c.Path))
	driver, err := InitDriverIfExists(directory)
	if err != nil {
		log.Fatal(err)
	}
	logger := log.WithFields(log.Fields{
		"directory": driver.Root(),
		"type":      driver.DriverType(),
		"volume":    c.Args.Name,
	})
	logger.Info("Mounting volume")
	vol, err := driver.Get(c.Args.Name)
	if err != nil {
		logger.Fatal(err)
	}
	logger.WithFields(log.Fields{
		"mount": vol.Path(),
	}).Info("Volume mounted")
	return nil
}

// Execute removes an existing volume from a driver
func (c *VolumeRemove) Execute(args []string) error {
	App.initializeLogging()
	directory := GetDefaultDriver(string(c.Path))
	driver, err := InitDriverIfExists(directory)
	if err != nil {
		log.Fatal(err)
	}
	logger := log.WithFields(log.Fields{
		"directory": driver.Root(),
		"type":      driver.DriverType(),
		"volume":    c.Args.Name,
	})
	if !driver.Exists(c.Args.Name) {
		logger.Fatal("Volume does not exist")
	}
	logger.Info("Removing volume")
	if err := driver.Remove(c.Args.Name); err != nil {
		logger.Fatal(err)
	}
	logger.Info("Volume deleted")
	return nil
}

// Resize increases the space available to an existing volume
func (c *VolumeResize) Execute(args []string) error {
	App.initializeLogging()
	directory := GetDefaultDriver(string(c.Path))
	driver, err := InitDriverIfExists(directory)
	if err != nil {
		log.Fatal(err)
	}
	logger := log.WithFields(log.Fields{
		"directory": driver.Root(),
		"volume":    c.Args.Name,
		"type":      driver.DriverType(),
	})
	if driver.DriverType() != volume.DriverTypeDeviceMapper {
		logger.Fatal("Only devicemapper volumes can be resized")
	}
	if !driver.Exists(c.Args.Name) {
		logger.Fatal("Volume does not exist")
	}
	size, err := units.RAMInBytes(c.Args.Size)
	if err != nil {
		return err
	}
	if err := driver.Resize(c.Args.Name, uint64(size)); err != nil {
		logger.Fatal(err)
	}
	logger.Info("Volume resized")
	return nil
}
<|endoftext|>


<commit_before>package mixpanel

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"time"
)

// The official base URL
const MixpanelBaseURL = "http://mixpanel.com/api"

// Mixpanel struct represents a set of credentials used to access the Mixpanel
// API for a particular product.
type Mixpanel struct {
	Product string
	Key     string
	Secret  string
	BaseURL string
}

// New creates a Mixpanel object with the given API credentials and uses the
// official API URL.
func New(product, key, secret string) *Mixpanel {
	return NewWithUrl(product, key, secret, MixpanelBaseURL)
}

// NewWithURL creates a Mixpanel object with the given API credentials and a
// custom Mixpanel API URL. (I doubt this will ever be useful but there you go)
func NewWithURL(product, key, secret, baseURL string) *Mixpanel {
	m := new(Mixpanel)
	m.Product = product
	m.Key = key
	m.Secret = secret
	m.BaseURL = baseURL
	return m
}

// Add the cryptographic signature that Mixpanel API requests require.
func (m *Mixpanel) addSignature(args *url.Values) {
	hash := md5.New()
	io.WriteString(hash, args.Encode())
	io.WriteString(hash, m.Secret)

	args.Set("sig", string(hash.Sum(nil)))
}

// Generate the initial, base arguments that all Mixpanel API requests use.
func (m *Mixpanel) makeArgs() url.Values {
	args := url.Values{}

	args.Set("format", "json")
	args.Set("api_key", m.Key)
	args.Set("expire", string(time.Now().Unix()+10000))

	return args
}

// ExportDate downloads event data for the given day and streams the resulting
// transformed JSON blobs as byte strings over the send-only channel passed
// to the function.
//
// The optional `moreArgs` parameter can be given to add additional URL
// parameters to the API request.
func (m *Mixpanel) ExportDate(date time.Time, outChan chan<- []byte, moreArgs *url.Values) {
	args := m.MakeArgs()

	if moreArgs != nil {
		for k, vs := range *moreArgs {
			for _, v := range vs {
				args.Add(k, v)
			}
		}
	}

	day := date.Format("2006-01-02")
	args.Set("start", day)
	args.Set("end", day)

	m.AddSignature(&args)

	eventChans := make(map[string]chan map[string]interface{})

	resp, err := http.Get(fmt.Sprintf("%s/2.0/export?%s", m.BaseURL, args.Encode()))
	if err != nil {
		panic("XXX handle this. FAILED")
	}

	type JSONEvent struct {
		event      string
		properties map[string]interface{}
	}

	decoder := json.NewDecoder(resp.Body)
	for {
		var ev JSONEvent
		if err := decoder.Decode(&ev); err == io.EOF {
			break
		} else if err != nil {
			panic(err)
		}

		// Send the data to the proper event channel, or create it if it
		// doesn't already exist.
		if eventChan, ok := eventChans[ev.event]; ok {
			eventChan <- ev.properties
		} else {
			eventChans[ev.event] = make(chan map[string]interface{})
			go m.eventHandler(ev.event, eventChans[ev.event], outChan)

			eventChans[ev.event] <- ev.properties
		}
	}

	// Finish off all the handlers
	for _, ch := range eventChans {
		close(ch)
	}
}

func (m *Mixpanel) eventHandler(event string, jsonChan chan map[string]interface{}, output chan<- []byte) {
	// XXX: This function is possibly irrelevant, can be done in single thread in `ExportDate`
	// TODO: ensure distinct_id is present
	for {
		props, ok := <-jsonChan

		if !ok {
			break
		}

		props["product"] = m.Product
		props["event"] = event

		var buf bytes.Buffer
		encoder := json.NewEncoder(&buf)
		encoder.Encode(props)

		output <- buf.Bytes()
	}
}
<commit_msg>Don't run each event in a separate goroutine.<commit_after>package mixpanel

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"time"
)

// The official base URL
const MixpanelBaseURL = "http://mixpanel.com/api"

// Mixpanel struct represents a set of credentials used to access the Mixpanel
// API for a particular product.
type Mixpanel struct {
	Product string
	Key     string
	Secret  string
	BaseURL string
}

// New creates a Mixpanel object with the given API credentials and uses the
// official API URL.
func New(product, key, secret string) *Mixpanel {
	return NewWithUrl(product, key, secret, MixpanelBaseURL)
}

// NewWithURL creates a Mixpanel object with the given API credentials and a
// custom Mixpanel API URL. (I doubt this will ever be useful but there you go)
func NewWithURL(product, key, secret, baseURL string) *Mixpanel {
	m := new(Mixpanel)
	m.Product = product
	m.Key = key
	m.Secret = secret
	m.BaseURL = baseURL
	return m
}

// Add the cryptographic signature that Mixpanel API requests require.
func (m *Mixpanel) addSignature(args *url.Values) {
	hash := md5.New()
	io.WriteString(hash, args.Encode())
	io.WriteString(hash, m.Secret)

	args.Set("sig", string(hash.Sum(nil)))
}

// Generate the initial, base arguments that all Mixpanel API requests use.
func (m *Mixpanel) makeArgs() url.Values {
	args := url.Values{}

	args.Set("format", "json")
	args.Set("api_key", m.Key)
	args.Set("expire", string(time.Now().Unix()+10000))

	return args
}

// ExportDate downloads event data for the given day and streams the resulting
// transformed JSON blobs as byte strings over the send-only channel passed
// to the function.
//
// The optional `moreArgs` parameter can be given to add additional URL
// parameters to the API request.
func (m *Mixpanel) ExportDate(date time.Time, outChan chan<- []byte, moreArgs *url.Values) {
	args := m.MakeArgs()

	if moreArgs != nil {
		for k, vs := range *moreArgs {
			for _, v := range vs {
				args.Add(k, v)
			}
		}
	}

	day := date.Format("2006-01-02")
	args.Set("start", day)
	args.Set("end", day)

	m.AddSignature(&args)

	resp, err := http.Get(fmt.Sprintf("%s/2.0/export?%s", m.BaseURL, args.Encode()))
	if err != nil {
		panic("XXX handle this. FAILED")
	}

	type JSONEvent struct {
		event      string
		properties map[string]interface{}
	}

	decoder := json.NewDecoder(resp.Body)
	for {
		var ev JSONEvent
		if err := decoder.Decode(&ev); err == io.EOF {
			break
		} else if err != nil {
			// TODO: handle {"error": "..."} responses more gracefully?
			panic(err)
		}

		// TODO: ensure that distinct_id is present (even though it should be)
		props := ev.properties
		props["product"] = m.Product
		props["event"] = ev.event

		var buf bytes.Buffer
		encoder := json.NewEncoder(&buf)
		encoder.Encode(props)

		output <- buf.Bytes()
	}
}
<|endoftext|>


<commit_before>package middleware

import (
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"
)

func TestNewResponseRecorder(t *testing.T) {
	w := httptest.NewRecorder()
	recordRequest := NewResponseRecorder(w)
	if !reflect.DeepEqual(recordRequest.ResponseWriter, w) {
		t.Fatalf("Expected Response writer in the Recording to be same as the one sent")
	}
	if recordRequest.status != http.StatusOK {
		t.Fatalf("Expected recorded status  to be http.StatusOK")
	}
}
<commit_msg>Complete test coverage for middleware/recorder.go<commit_after>package middleware

import (
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestNewResponseRecorder(t *testing.T) {
	w := httptest.NewRecorder()
	recordRequest := NewResponseRecorder(w)
	if !(recordRequest.ResponseWriter == w) {
		t.Fatalf("Expected Response writer in the Recording to be same as the one sent\n")
	}
	if recordRequest.status != http.StatusOK {
		t.Fatalf("Expected recorded status  to be http.StatusOK (%d) , but found %d\n ", recordRequest.status)
	}
}
func TestWriteHeader(t *testing.T) {
	w := httptest.NewRecorder()
	recordRequest := NewResponseRecorder(w)
	recordRequest.WriteHeader(401)
	if w.Code != 401 || recordRequest.status != 401 {
		t.Fatalf("Expected Response status to be set to 401, but found %d\n", recordRequest.status)
	}
}

func TestWrite(t *testing.T) {
	w := httptest.NewRecorder()
	responseTestString := "test"
	recordRequest := NewResponseRecorder(w)
	buf := []byte(responseTestString)
	recordRequest.Write(buf)
	if recordRequest.size != len(buf) {
		t.Fatalf("Expected the bytes written counter to be %d, but instead found %d\n", len(buf), recordRequest.size)
	}
	if w.Body.String() != responseTestString {
		t.Fatalf("Expected Response Body to be %s , but found %s\n", w.Body.String())
	}
}
<|endoftext|>


<commit_before>package midihandler_test

import (
	"bytes"
	"fmt"
	"io"
	"time"

	"github.com/gomidi/midi/midihandler"
	"github.com/gomidi/midi/midimessage/channel"
	"github.com/gomidi/midi/midimessage/meta"
	"github.com/gomidi/midi/midiwriter"
	"github.com/gomidi/midi/smf"
	"github.com/gomidi/midi/smf/smfwriter"
)

func mkSMF() io.Reader {
	var bf bytes.Buffer

	wr := smfwriter.New(&bf)
	wr.Write(meta.Tempo(160))
	wr.Write(channel.Ch2.NoteOn(65, 90))
	wr.SetDelta(40)
	wr.Write(channel.Ch2.NoteOff(65))
	wr.Write(meta.EndOfTrack)

	return bytes.NewReader(bf.Bytes())
}

func Example() {
	// This example illustrates how the same handler can be used for live and SMF MIDI messages

	hd := midihandler.New(midihandler.NoLogger())

	// needed for the SMF timing
	var ticks smf.MetricTicks
	var bpm uint32 = 120 // default according to SMF spec

	// needed for the live timing
	var start = time.Now()

	// a helper to round the duration to milliseconds
	var roundMS = func(d time.Duration) time.Duration {
		return time.Millisecond * time.Duration((d.Nanoseconds() / 1000000))
	}

	// a helper to calculate the duration for both live and SMF messages
	var calcDuration = func(p *midihandler.SMFPosition) (dur time.Duration) {
		if p == nil {
			// we are in a live setting
			dur = roundMS(time.Now().Sub(start))
			return
		}

		// SMF data, calculate the time from the timeformat of the SMF file
		// we ignore the possibility that tempo information may come in a track following the one of
		// the current message as the spec does not recommend this
		return roundMS(ticks.Duration(bpm, uint32(p.AbsTime)))
	}

	hd.SMFHeader = func(head smf.Header) {
		// here we ignore that the timeformat could also be SMPTE
		ticks = head.TimeFormat.(smf.MetricTicks)
	}

	// we will override the tempo by the one given in the SMF
	hd.Message.Meta.Tempo = func(p midihandler.SMFPosition, valBPM uint32) {
		bpm = valBPM
	}

	// set the functions for the messages you are interested in
	hd.Message.Channel.NoteOn = func(p *midihandler.SMFPosition, channel, key, vel uint8) {
		fmt.Printf("[%v] NoteOn at channel %v: key %v velocity: %v\n", calcDuration(p), channel, key, vel)
	}

	hd.Message.Channel.NoteOff = func(p *midihandler.SMFPosition, channel, key, vel uint8) {
		fmt.Printf("[%v] NoteOff at channel %v: key %v velocity: %v\n", calcDuration(p), channel, key, vel)
	}

	// handle the smf
	fmt.Println("-- SMF data --")
	hd.ReadSMF(mkSMF())

	// handle the live data
	fmt.Println("-- live data --")
	lrd, lwr := io.Pipe()

	// WARNING this example does not deal with races and synchronization, it is just for illustration
	go func() {
		hd.ReadLive(lrd)
	}()

	mwr := midiwriter.New(lwr)
	start = time.Now()

	// now write some live data
	mwr.Write(channel.Ch11.NoteOn(120, 50))
	time.Sleep(time.Millisecond * 20)
	mwr.Write(channel.Ch11.NoteOff(120))

	// Output: -- SMF data --
	// [0s] NoteOn at channel 2: key 65 velocity: 90
	// [16ms] NoteOff at channel 2: key 65 velocity: 0
	// -- live data --
	// [0s] NoteOn at channel 11: key 120 velocity: 50
	// [20ms] NoteOff at channel 11: key 120 velocity: 0
}
<commit_msg>change example test to seconds instead of milliseconds to make travis happy<commit_after>package midihandler_test

import (
	"bytes"
	"fmt"
	"io"
	"time"

	"github.com/gomidi/midi/midihandler"
	"github.com/gomidi/midi/midimessage/channel"
	"github.com/gomidi/midi/midimessage/meta"
	"github.com/gomidi/midi/midiwriter"
	"github.com/gomidi/midi/smf"
	"github.com/gomidi/midi/smf/smfwriter"
)

func mkSMF() io.Reader {
	var bf bytes.Buffer

	wr := smfwriter.New(&bf)
	wr.Write(meta.Tempo(160))
	wr.Write(channel.Ch2.NoteOn(65, 90))
	wr.SetDelta(4000)
	wr.Write(channel.Ch2.NoteOff(65))
	wr.Write(meta.EndOfTrack)

	return bytes.NewReader(bf.Bytes())
}

func Example() {
	// This example illustrates how the same handler can be used for live and SMF MIDI messages

	hd := midihandler.New(midihandler.NoLogger())

	// needed for the SMF timing
	var ticks smf.MetricTicks
	var bpm uint32 = 120 // default according to SMF spec

	// needed for the live timing
	var start = time.Now()

	// a helper to round the duration to seconds
	var roundSec = func(d time.Duration) time.Duration {
		return time.Second * time.Duration((d.Nanoseconds() / 1000000000))
	}

	// a helper to calculate the duration for both live and SMF messages
	var calcDuration = func(p *midihandler.SMFPosition) (dur time.Duration) {
		if p == nil {
			// we are in a live setting
			dur = roundSec(time.Now().Sub(start))
			return
		}

		// SMF data, calculate the time from the timeformat of the SMF file
		// we ignore the possibility that tempo information may come in a track following the one of
		// the current message as the spec does not recommend this
		return roundSec(ticks.Duration(bpm, uint32(p.AbsTime)))
	}

	hd.SMFHeader = func(head smf.Header) {
		// here we ignore that the timeformat could also be SMPTE
		ticks = head.TimeFormat.(smf.MetricTicks)
	}

	// we will override the tempo by the one given in the SMF
	hd.Message.Meta.Tempo = func(p midihandler.SMFPosition, valBPM uint32) {
		bpm = valBPM
	}

	// set the functions for the messages you are interested in
	hd.Message.Channel.NoteOn = func(p *midihandler.SMFPosition, channel, key, vel uint8) {
		fmt.Printf("[%v] NoteOn at channel %v: key %v velocity: %v\n", calcDuration(p), channel, key, vel)
	}

	hd.Message.Channel.NoteOff = func(p *midihandler.SMFPosition, channel, key, vel uint8) {
		fmt.Printf("[%v] NoteOff at channel %v: key %v velocity: %v\n", calcDuration(p), channel, key, vel)
	}

	// handle the smf
	fmt.Println("-- SMF data --")
	hd.ReadSMF(mkSMF())

	// handle the live data
	fmt.Println("-- live data --")
	lrd, lwr := io.Pipe()

	// WARNING this example does not deal with races and synchronization, it is just for illustration
	go func() {
		hd.ReadLive(lrd)
	}()

	mwr := midiwriter.New(lwr)
	start = time.Now()

	// now write some live data
	mwr.Write(channel.Ch11.NoteOn(120, 50))
	time.Sleep(time.Second * 2)
	mwr.Write(channel.Ch11.NoteOff(120))

	// Output: -- SMF data --
	// [0s] NoteOn at channel 2: key 65 velocity: 90
	// [1s] NoteOff at channel 2: key 65 velocity: 0
	// -- live data --
	// [0s] NoteOn at channel 11: key 120 velocity: 50
	// [2s] NoteOff at channel 11: key 120 velocity: 0
}
<|endoftext|>


<commit_before>package stripe

import (
	"fmt"
	"net/url"
	"strconv"
	"strings"
)

// Credit Card Types accepted by the Stripe API.
const (
	AmericanExpress = "American Express"
	DinersClub      = "Diners Club"
	Discover        = "Discover"
	JCB             = "JCB"
	MasterCard      = "MasterCard"
	Visa            = "Visa"
	UnknownCard     = "Unknown"
)

// Card represents details about a Credit Card entered into Stripe.
type Card struct {
	ID                string `json:"id"`
	Name              string `json:"name,omitempty"`
	Type              string `json:"type"`
	ExpMonth          int    `json:"exp_month"`
	ExpYear           int    `json:"exp_year"`
	Last4             string `json:"last4"`
	Fingerprint       string `json:"fingerprint"`
	Country           string `json:"country,omitempty"`
	Address1          string `json:"address_line1,omitempty"`
	Address2          string `json:"address_line2,omitempty"`
	AddressCountry    string `json:"address_country,omitempty"`
	AddressState      string `json:"address_state,omitempty"`
	AddressZip        string `json:"address_zip,omitempty"`
	AddressLine1Check string `json:"address_line1_check,omitempty"`
	AddressZipCheck   string `json:"address_zip_check,omitempty"`
	CVCCheck          string `json:"cvc_check,omitempty"`
	Customer          string `json:"customer,omitempty"`
}

// CardParams encapsulates options for Creating or Updating Credit Cards.
type CardParams struct {
	// (Optional) Cardholder's full name.
	Name string

	// The card number, as a string without any separators.
	Number string

	// The card's expiration month.
	ExpMonth int

	// The card's expiration year.
	ExpYear int

	// Card security code
	CVC string

	// (Optional) Billing address line 1
	Address1 string

	// (Optional) Billing address line 2
	Address2 string

	// (Optional) Billing address country
	AddressCountry string

	// (Optional) Billing address state
	AddressState string

	// (Optional) Billing address zip code
	AddressZip string
}

type CardClient struct{}

func (CardClient) Create(customerID, token string, card *CardParams) (*Card, error) {
	params := make(url.Values)
	if token != "" {
		params.Add("card", token)
	} else {
		appendCardParams(params, card)
	}
	res := &Card{}
	return res, query("POST", fmt.Sprintf("/customers/%s/cards", url.QueryEscape(customerID)), params, res)
}

func (CardClient) Update(customerID, cardID string, card *CardParams) (*Card, error) {
	params := make(url.Values)
	appendCardParams(params, card)
	res := &Card{}
	return res, query("POST", fmt.Sprintf("/customers/%s/cards/%s", url.QueryEscape(customerID), url.QueryEscape(cardID)), params, res)
}

func (CardClient) Delete(customerID, cardID string) (bool, error) {
	res := &DeleteResp{}
	err := query("DELETE", fmt.Sprintf("/customers/%s/cards/%s", url.QueryEscape(customerID), url.QueryEscape(cardID)), nil, res)
	return res.Deleted, err
}

func (CardClient) Retrieve(customerID, cardID string) (*Card, error) {
	res := &Card{}
	return res, query("GET", fmt.Sprintf("/customers/%s/cards/%s", url.QueryEscape(customerID), url.QueryEscape(cardID)), nil, res)
}

func (CardClient) List(customerID string, limit int, before, after string) ([]*Card, bool, error) {
	res := struct {
		ListObject
		Data []*Card
	}{}
	err := query("GET", fmt.Sprintf("/customers/%s/cards", url.QueryEscape(customerID)), listParams(limit, before, after), &res)
	return res.Data, res.More, err
}

// IsLuhnValid uses the Luhn Algorithm (also known as the Mod 10 algorithm) to
// verify a credit cards checksum, which helps flag accidental data entry
// errors.
//
// see http://en.wikipedia.org/wiki/Luhn_algorithm
func IsLuhnValid(card string) (bool, error) {
	var sum = 0
	var digits = strings.Split(card, "")

	// iterate through the digits in reverse order
	for i, even := len(digits)-1, false; i >= 0; i, even = i-1, !even {
		// convert the digit to an integer
		digit, err := strconv.Atoi(digits[i])
		if err != nil {
			return false, err
		}

		// we multiply every other digit by 2, adding the product to the sum.
		// note: if the product is double digits (i.e. 14) we add the two digits
		//       to the sum (14 -> 1+4 = 5). A simple shortcut is to subtract 9
		//       from a double digit product (14 -> 14 - 9 = 5).
		switch {
		case even && digit > 4:
			sum += (digit * 2) - 9
		case even:
			sum += digit * 2
		case !even:
			sum += digit
		}
	}

	// if the sum is divisible by 10, it passes the check
	return sum%10 == 0, nil
}

// GetCardType is a simple algorithm to determine the Card Type (ie Visa,
// Discover) based on the Credit Card Number. If the Number is not recognized, a
// value of "Unknown" will be returned.
func GetCardType(card string) string {
	switch card[0:1] {
	case "4":
		return Visa
	case "2", "1":
		switch card[0:4] {
		case "2131", "1800":
			return JCB
		}
	case "6":
		switch card[0:4] {
		case "6011":
			return Discover
		}
	case "5":
		switch card[0:2] {
		case "51", "52", "53", "54", "55":
			return MasterCard
		}
	case "3":
		switch card[0:2] {
		case "34", "37":
			return AmericanExpress
		case "36":
			return DinersClub
		case "30":
			switch card[0:3] {
			case "300", "301", "302", "303", "304", "305":
				return DinersClub
			}
		default:
			return JCB
		}
	}

	return UnknownCard
}
<commit_msg>Add card path builder<commit_after>package stripe

import (
	"fmt"
	"net/url"
	"strconv"
	"strings"
)

// Credit Card Types accepted by the Stripe API.
const (
	AmericanExpress = "American Express"
	DinersClub      = "Diners Club"
	Discover        = "Discover"
	JCB             = "JCB"
	MasterCard      = "MasterCard"
	Visa            = "Visa"
	UnknownCard     = "Unknown"
)

// Card represents details about a Credit Card entered into Stripe.
type Card struct {
	ID                string `json:"id"`
	Name              string `json:"name,omitempty"`
	Type              string `json:"type"`
	ExpMonth          int    `json:"exp_month"`
	ExpYear           int    `json:"exp_year"`
	Last4             string `json:"last4"`
	Fingerprint       string `json:"fingerprint"`
	Country           string `json:"country,omitempty"`
	Address1          string `json:"address_line1,omitempty"`
	Address2          string `json:"address_line2,omitempty"`
	AddressCountry    string `json:"address_country,omitempty"`
	AddressState      string `json:"address_state,omitempty"`
	AddressZip        string `json:"address_zip,omitempty"`
	AddressLine1Check string `json:"address_line1_check,omitempty"`
	AddressZipCheck   string `json:"address_zip_check,omitempty"`
	CVCCheck          string `json:"cvc_check,omitempty"`
	Customer          string `json:"customer,omitempty"`
}

// CardParams encapsulates options for Creating or Updating Credit Cards.
type CardParams struct {
	// (Optional) Cardholder's full name.
	Name string

	// The card number, as a string without any separators.
	Number string

	// The card's expiration month.
	ExpMonth int

	// The card's expiration year.
	ExpYear int

	// Card security code
	CVC string

	// (Optional) Billing address line 1
	Address1 string

	// (Optional) Billing address line 2
	Address2 string

	// (Optional) Billing address country
	AddressCountry string

	// (Optional) Billing address state
	AddressState string

	// (Optional) Billing address zip code
	AddressZip string
}

type CardClient struct{}

func (c CardClient) path(customerID, cardID string) string {
	p := fmt.Sprintf("/customers/%s/cards", url.QueryEscape(customerID))
	if cardID != "" {
		p += "/" + url.QueryEscape(cardID)
	}
	return p
}

func (c CardClient) Create(customerID, token string, card *CardParams) (*Card, error) {
	params := make(url.Values)
	if token != "" {
		params.Add("card", token)
	} else {
		appendCardParams(params, card)
	}
	res := &Card{}
	return res, query("POST", c.path(customerID, ""), params, res)
}

func (c CardClient) Update(customerID, cardID string, card *CardParams) (*Card, error) {
	params := make(url.Values)
	appendCardParams(params, card)
	res := &Card{}
	return res, query("POST", c.path(customerID, cardID), params, res)
}

func (c CardClient) Delete(customerID, cardID string) (bool, error) {
	res := &DeleteResp{}
	err := query("DELETE", c.path(customerID, cardID), nil, res)
	return res.Deleted, err
}

func (c CardClient) Retrieve(customerID, cardID string) (*Card, error) {
	res := &Card{}
	return res, query("GET", c.path(customerID, cardID), nil, res)
}

func (c CardClient) List(customerID string, limit int, before, after string) ([]*Card, bool, error) {
	res := struct {
		ListObject
		Data []*Card
	}{}
	err := query("GET", c.path(customerID, ""), listParams(limit, before, after), &res)
	return res.Data, res.More, err
}

// IsLuhnValid uses the Luhn Algorithm (also known as the Mod 10 algorithm) to
// verify a credit cards checksum, which helps flag accidental data entry
// errors.
//
// see http://en.wikipedia.org/wiki/Luhn_algorithm
func IsLuhnValid(card string) (bool, error) {
	var sum = 0
	var digits = strings.Split(card, "")

	// iterate through the digits in reverse order
	for i, even := len(digits)-1, false; i >= 0; i, even = i-1, !even {
		// convert the digit to an integer
		digit, err := strconv.Atoi(digits[i])
		if err != nil {
			return false, err
		}

		// we multiply every other digit by 2, adding the product to the sum.
		// note: if the product is double digits (i.e. 14) we add the two digits
		//       to the sum (14 -> 1+4 = 5). A simple shortcut is to subtract 9
		//       from a double digit product (14 -> 14 - 9 = 5).
		switch {
		case even && digit > 4:
			sum += (digit * 2) - 9
		case even:
			sum += digit * 2
		case !even:
			sum += digit
		}
	}

	// if the sum is divisible by 10, it passes the check
	return sum%10 == 0, nil
}

// GetCardType is a simple algorithm to determine the Card Type (ie Visa,
// Discover) based on the Credit Card Number. If the Number is not recognized, a
// value of "Unknown" will be returned.
func GetCardType(card string) string {
	switch card[0:1] {
	case "4":
		return Visa
	case "2", "1":
		switch card[0:4] {
		case "2131", "1800":
			return JCB
		}
	case "6":
		switch card[0:4] {
		case "6011":
			return Discover
		}
	case "5":
		switch card[0:2] {
		case "51", "52", "53", "54", "55":
			return MasterCard
		}
	case "3":
		switch card[0:2] {
		case "34", "37":
			return AmericanExpress
		case "36":
			return DinersClub
		case "30":
			switch card[0:3] {
			case "300", "301", "302", "303", "304", "305":
				return DinersClub
			}
		default:
			return JCB
		}
	}

	return UnknownCard
}
<|endoftext|>


<commit_before>package cbqd

import (
	"flag"
	"github.com/op/go-logging"
	"io/ioutil"
	"os"
	"path/filepath"
)

type AccessCreds struct {
	Dkey  string `json:"username"`
	Dpass string `json:"password"`
}

type Database struct {
	Ukey       AccessCreds
	Host, Port string
}

var (
	dbflag = flag.String("db", "mysql", "Database type to dump.")
	csflag = flag.String("cs", "aws", "S3 storage repository to use.")
	kvflag = flag.Bool("kv", false, "Access vault to acquire secrets.")
	dhflag = flag.String("dh", "127.0.0.1", "Host IP for the database to be backuped up.")
	dpflag = flag.String("dp", "3306", "Database port for access.")
	vpflag = flag.Bool("version", false, "Prints out the version number.")
	veflag = formattedVersion()
	lgform = logging.MustStringFormatter(`%{color}%{time:15:04:05.000} %{shortpkg} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`)
)

func (a AccessCreds) GetCreds(vbackend string, inout string, kvault bool) (AccessCreds, error) {
	un := inout + "_KEY"
	up := inout + "_PASS"
	ac := AccessCreds{}

	if kvault == false {
		ac.Dkey = os.Getenv(un)
		ac.Dpass = os.Getenv(up)
		if ac.Dkey == "" && ac.Dpass == "" {
			log.Fatalln(OS_ENVIRONMENT_UNSET)
		}
		return ac, nil
	}
	return ac, VAULT_CREDENTIAL_ERROR
}

func usage() {
	//
}

func init() {
	flag.Parse()
	bl1 := logging.NewLogBackend(os.Stderr, "", 0)
	blf := logging.NewBackendFormatter(bl1, lgform)
}

func Cbqd() {
	logging.SetBackend(bl1, blf)
	increds, err := new(AccessCreds).GetCreds(*dbflag, "CBQD_IN", *kvflag)
	if err != nil {
		log.Fatal(err)
	}

	db := Database{increds, *dhflag, *dpflag}
	outcreds, err := new(AccessCreds).GetCreds(*csflag, "CBQD_OUT", *kvflag)
	if err != nil {
		log.Fatal(err)
	}

	topdir, err := ioutil.TempDir("", "cbqd_state")
	if err != nil {
		log.Fatal(err)
	}

	defer os.RemoveAll(topdir)
	tmpfhandle := filepath.Join(topdir, "tmpfile")

	bname, err := MYSQL{}.DBdump(db, tmpfhandle)
	if err != nil {
		log.Fatal(err)
	}

	err0 := AWS{}.CloudSend(outcreds, bname, tmpfhandle)
	if err0 != nil {
		log.Fatal(BACKUP_UPLOAD_ERROR)
	}

}
<commit_msg>set logger to log with<commit_after>package cbqd

import (
	"flag"
	"github.com/op/go-logging"
	"io/ioutil"
	"os"
	"path/filepath"
)

type AccessCreds struct {
	Dkey  string `json:"username"`
	Dpass string `json:"password"`
}

type Database struct {
	Ukey       AccessCreds
	Host, Port string
}

var (
	dbflag = flag.String("db", "mysql", "Database type to dump.")
	csflag = flag.String("cs", "aws", "S3 storage repository to use.")
	kvflag = flag.Bool("kv", false, "Access vault to acquire secrets.")
	dhflag = flag.String("dh", "127.0.0.1", "Host IP for the database to be backuped up.")
	dpflag = flag.String("dp", "3306", "Database port for access.")
	vpflag = flag.Bool("version", false, "Prints out the version number.")
	veflag = formattedVersion()
	log    = logging.MustGetLogger("cbqd")
	lgform = logging.MustStringFormatter(`%{color}%{time:15:04:05.000} %{shortpkg} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`)
)

func (a AccessCreds) GetCreds(vbackend string, inout string, kvault bool) (AccessCreds, error) {
	un := inout + "_KEY"
	up := inout + "_PASS"
	ac := AccessCreds{}

	if kvault == false {
		ac.Dkey = os.Getenv(un)
		ac.Dpass = os.Getenv(up)
		if ac.Dkey == "" && ac.Dpass == "" {
			log.Fatalln(OS_ENVIRONMENT_UNSET)
		}
		return ac, nil
	}
	return ac, VAULT_CREDENTIAL_ERROR
}

func usage() {
	//
}

func init() {
	flag.Parse()
	bl1 := logging.NewLogBackend(os.Stderr, "", 0)
	blf := logging.NewBackendFormatter(bl1, lgform)
}

func Cbqd() {
	logging.SetBackend(bl1, blf)
	increds, err := new(AccessCreds).GetCreds(*dbflag, "CBQD_IN", *kvflag)
	if err != nil {
		log.Fatal(err)
	}

	db := Database{increds, *dhflag, *dpflag}
	outcreds, err := new(AccessCreds).GetCreds(*csflag, "CBQD_OUT", *kvflag)
	if err != nil {
		log.Fatal(err)
	}

	topdir, err := ioutil.TempDir("", "cbqd_state")
	if err != nil {
		log.Fatal(err)
	}

	defer os.RemoveAll(topdir)
	tmpfhandle := filepath.Join(topdir, "tmpfile")

	bname, err := MYSQL{}.DBdump(db, tmpfhandle)
	if err != nil {
		log.Fatal(err)
	}

	err0 := AWS{}.CloudSend(outcreds, bname, tmpfhandle)
	if err0 != nil {
		log.Fatal(BACKUP_UPLOAD_ERROR)
	}

}
<|endoftext|>