<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import (
	"runtime/internal/sys"
	"unsafe"
)

const (
	c0 = uintptr((8-sys.PtrSize)/4*2860486313 + (sys.PtrSize-4)/4*33054211828000289)
	c1 = uintptr((8-sys.PtrSize)/4*3267000013 + (sys.PtrSize-4)/4*23344194077549503)
)

// type algorithms - known to compiler
const (
	alg_MEM = iota
	alg_MEM0
	alg_MEM8
	alg_MEM16
	alg_MEM32
	alg_MEM64
	alg_MEM128
	alg_NOEQ
	alg_NOEQ0
	alg_NOEQ8
	alg_NOEQ16
	alg_NOEQ32
	alg_NOEQ64
	alg_NOEQ128
	alg_STRING
	alg_INTER
	alg_NILINTER
	alg_SLICE
	alg_FLOAT32
	alg_FLOAT64
	alg_CPLX64
	alg_CPLX128
	alg_max
)

type typeAlg struct {
	// function for hashing objects of this type
	// (ptr to object, seed) -> hash
	hash func(unsafe.Pointer, uintptr) uintptr
	// function for comparing objects of this type
	// (ptr to object A, ptr to object B) -> ==?
	equal func(unsafe.Pointer, unsafe.Pointer) bool
}

func memhash0(p unsafe.Pointer, h uintptr) uintptr {
	return h
}
func memhash8(p unsafe.Pointer, h uintptr) uintptr {
	return memhash(p, h, 1)
}
func memhash16(p unsafe.Pointer, h uintptr) uintptr {
	return memhash(p, h, 2)
}
func memhash32(p unsafe.Pointer, h uintptr) uintptr {
	return memhash(p, h, 4)
}
func memhash64(p unsafe.Pointer, h uintptr) uintptr {
	return memhash(p, h, 8)
}
func memhash128(p unsafe.Pointer, h uintptr) uintptr {
	return memhash(p, h, 16)
}

// memhash_varlen is defined in assembly because it needs access
// to the closure.  It appears here to provide an argument
// signature for the assembly routine.
func memhash_varlen(p unsafe.Pointer, h uintptr) uintptr

var algarray = [alg_max]typeAlg{
	alg_MEM:      {nil, nil}, // not used
	alg_MEM0:     {nil, nil}, //{memhash0, memequal0},
	alg_MEM8:     {nil, nil}, //{memhash8, memequal8},
	alg_MEM16:    {nil, nil}, //{memhash16, memequal16},
	alg_MEM32:    {nil, nil}, //{memhash32, memequal32},
	alg_MEM64:    {nil, nil}, //{memhash64, memequal64},
	alg_MEM128:   {nil, nil}, //{memhash128, memequal128},
	alg_NOEQ:     {nil, nil},
	alg_NOEQ0:    {nil, nil},
	alg_NOEQ8:    {nil, nil},
	alg_NOEQ16:   {nil, nil},
	alg_NOEQ32:   {nil, nil},
	alg_NOEQ64:   {nil, nil},
	alg_NOEQ128:  {nil, nil},
	alg_STRING:   {nil, nil}, //{strhash, strequal},
	alg_INTER:    {nil, nil}, //{interhash, interequal},
	alg_NILINTER: {nil, nil}, //{nilinterhash, nilinterequal},
	alg_SLICE:    {nil, nil},
	alg_FLOAT32:  {nil, nil}, //{f32hash, f32equal},
	alg_FLOAT64:  {nil, nil}, //{f64hash, f64equal},
	alg_CPLX64:   {nil, nil}, //{c64hash, c64equal},
	alg_CPLX128:  {nil, nil}, //{c128hash, c128equal},
}

func f32hash(p unsafe.Pointer, h uintptr) uintptr {
	f := *(*float32)(p)
	switch {
	case f == 0:
		return c1 * (c0 ^ h) // +0, -0
	case f != f:
		return c1 * (c0 ^ h ^ uintptr(fastrand1())) // any kind of NaN
	default:
		return memhash(p, h, 4)
	}
}

func f64hash(p unsafe.Pointer, h uintptr) uintptr {
	f := *(*float64)(p)
	switch {
	case f == 0:
		return c1 * (c0 ^ h) // +0, -0
	case f != f:
		return c1 * (c0 ^ h ^ uintptr(fastrand1())) // any kind of NaN
	default:
		return memhash(p, h, 8)
	}
}
<commit_msg>tiny: more stubs<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import (
	"runtime/internal/sys"
	"unsafe"
)

const (
	c0 = uintptr((8-sys.PtrSize)/4*2860486313 + (sys.PtrSize-4)/4*33054211828000289)
	c1 = uintptr((8-sys.PtrSize)/4*3267000013 + (sys.PtrSize-4)/4*23344194077549503)
)

// type algorithms - known to compiler
const (
	alg_MEM = iota
	alg_MEM0
	alg_MEM8
	alg_MEM16
	alg_MEM32
	alg_MEM64
	alg_MEM128
	alg_NOEQ
	alg_NOEQ0
	alg_NOEQ8
	alg_NOEQ16
	alg_NOEQ32
	alg_NOEQ64
	alg_NOEQ128
	alg_STRING
	alg_INTER
	alg_NILINTER
	alg_SLICE
	alg_FLOAT32
	alg_FLOAT64
	alg_CPLX64
	alg_CPLX128
	alg_max
)

type typeAlg struct {
	// function for hashing objects of this type
	// (ptr to object, seed) -> hash
	hash func(unsafe.Pointer, uintptr) uintptr
	// function for comparing objects of this type
	// (ptr to object A, ptr to object B) -> ==?
	equal func(unsafe.Pointer, unsafe.Pointer) bool
}

func memhash0(p unsafe.Pointer, h uintptr) uintptr {
	return h
}
func memhash8(p unsafe.Pointer, h uintptr) uintptr {
	return memhash(p, h, 1)
}
func memhash16(p unsafe.Pointer, h uintptr) uintptr {
	return memhash(p, h, 2)
}
func memhash32(p unsafe.Pointer, h uintptr) uintptr {
	return memhash(p, h, 4)
}
func memhash64(p unsafe.Pointer, h uintptr) uintptr {
	return memhash(p, h, 8)
}
func memhash128(p unsafe.Pointer, h uintptr) uintptr {
	return memhash(p, h, 16)
}

// memhash_varlen is defined in assembly because it needs access
// to the closure.  It appears here to provide an argument
// signature for the assembly routine.
func memhash_varlen(p unsafe.Pointer, h uintptr) uintptr

var algarray = [alg_max]typeAlg{
	alg_MEM:      {nil, nil}, // not used
	alg_MEM0:     {nil, nil}, //{memhash0, memequal0},
	alg_MEM8:     {nil, nil}, //{memhash8, memequal8},
	alg_MEM16:    {nil, nil}, //{memhash16, memequal16},
	alg_MEM32:    {nil, nil}, //{memhash32, memequal32},
	alg_MEM64:    {nil, nil}, //{memhash64, memequal64},
	alg_MEM128:   {nil, nil}, //{memhash128, memequal128},
	alg_NOEQ:     {nil, nil},
	alg_NOEQ0:    {nil, nil},
	alg_NOEQ8:    {nil, nil},
	alg_NOEQ16:   {nil, nil},
	alg_NOEQ32:   {nil, nil},
	alg_NOEQ64:   {nil, nil},
	alg_NOEQ128:  {nil, nil},
	alg_STRING:   {nil, nil}, //{strhash, strequal},
	alg_INTER:    {nil, nil}, //{interhash, interequal},
	alg_NILINTER: {nil, nil}, //{nilinterhash, nilinterequal},
	alg_SLICE:    {nil, nil},
	alg_FLOAT32:  {nil, nil}, //{f32hash, f32equal},
	alg_FLOAT64:  {nil, nil}, //{f64hash, f64equal},
	alg_CPLX64:   {nil, nil}, //{c64hash, c64equal},
	alg_CPLX128:  {nil, nil}, //{c128hash, c128equal},
}

func f32hash(p unsafe.Pointer, h uintptr) uintptr {
	f := *(*float32)(p)
	switch {
	case f == 0:
		return c1 * (c0 ^ h) // +0, -0
	case f != f:
		return c1 * (c0 ^ h ^ uintptr(fastrand1())) // any kind of NaN
	default:
		return memhash(p, h, 4)
	}
}

func f64hash(p unsafe.Pointer, h uintptr) uintptr {
	f := *(*float64)(p)
	switch {
	case f == 0:
		return c1 * (c0 ^ h) // +0, -0
	case f != f:
		return c1 * (c0 ^ h ^ uintptr(fastrand1())) // any kind of NaN
	default:
		return memhash(p, h, 8)
	}
}

type stringStruct struct {
	str unsafe.Pointer
	len int
}

func strhash(a unsafe.Pointer, h uintptr) uintptr {
	x := (*stringStruct)(a)
	return memhash(x.str, h, uintptr(x.len))
}
<|endoftext|>


<commit_before>package datastore

import (
	"net/http"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/kolide/kolide-ose/server/errors"
	"github.com/kolide/kolide-ose/server/kolide"
)

func (orm gormDB) EnrollHost(uuid, hostname, ip, platform string, nodeKeySize int) (*kolide.Host, error) {
	if uuid == "" {
		return nil, errors.New("missing uuid for host enrollment", "programmer error?")
	}
	host := kolide.Host{UUID: uuid}
	err := orm.DB.Where(&host).First(&host).Error
	if err != nil {
		switch err {
		case gorm.ErrRecordNotFound:
			// Create new Host
			host = kolide.Host{
				UUID:             uuid,
				HostName:         hostname,
				PrimaryIP:        ip,
				Platform:         platform,
				DetailUpdateTime: time.Unix(0, 0).Add(24 * time.Hour),
			}

		default:
			return nil, err
		}
	}

	// Generate a new key each enrollment
	host.NodeKey, err = generateRandomText(nodeKeySize)
	if err != nil {
		return nil, err
	}

	// Update these fields if provided
	if hostname != "" {
		host.HostName = hostname
	}
	if ip != "" {
		host.PrimaryIP = ip
	}
	if platform != "" {
		host.Platform = platform
	}

	if err := orm.DB.Save(&host).Error; err != nil {
		return nil, err
	}

	return &host, nil
}

func (orm gormDB) AuthenticateHost(nodeKey string) (*kolide.Host, error) {
	host := kolide.Host{NodeKey: nodeKey}
	err := orm.DB.Where("node_key = ?", host.NodeKey).First(&host).Error
	if err != nil {
		switch err {
		case gorm.ErrRecordNotFound:
			e := errors.NewFromError(
				err,
				http.StatusUnauthorized,
				"invalid node key",
			)
			// osqueryd expects the literal string "true" here
			e.Extra = map[string]interface{}{"node_invalid": "true"}
			return nil, e
		default:
			return nil, errors.DatabaseError(err)
		}
	}

	return &host, nil
}

func (orm gormDB) SaveHost(host *kolide.Host) error {
	if err := orm.DB.Save(host).Error; err != nil {
		return errors.DatabaseError(err)
	}
	return nil
}

func (orm gormDB) DeleteHost(host *kolide.Host) error {
	return orm.DB.Delete(host).Error
}

func (orm gormDB) Host(id uint) (*kolide.Host, error) {
	host := &kolide.Host{
		ID: id,
	}
	err := orm.DB.Where(host).First(host).Error
	if err != nil {
		return nil, err
	}
	return host, nil
}

func (orm gormDB) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) {
	var hosts []*kolide.Host
	err := orm.applyListOptions(opt).Find(&hosts).Error
	if err != nil {
		return nil, err
	}
	return hosts, nil
}

func (orm gormDB) NewHost(host *kolide.Host) (*kolide.Host, error) {
	if host == nil {
		return nil, errors.New(
			"error creating host",
			"nil pointer passed to NewHost",
		)
	}
	err := orm.DB.Create(host).Error
	if err != nil {
		return nil, err
	}
	return host, err
}

func (orm gormDB) MarkHostSeen(host *kolide.Host, t time.Time) error {
	err := orm.DB.Exec("UPDATE hosts SET updated_at=? WHERE node_key=?", t, host.NodeKey).Error
	if err != nil {
		return errors.DatabaseError(err)
	}
	host.UpdatedAt = t
	return nil
}

func (orm gormDB) SearchHosts(query string, omit []uint) ([]kolide.Host, error) {
	sql := `
SELECT *
FROM hosts
WHERE MATCH(host_name, primary_ip)
AGAINST(? IN BOOLEAN MODE)
`
	results := []kolide.Host{}

	var db *gorm.DB
	if len(omit) > 0 {
		sql += "AND id NOT IN (?) LIMIT 10;"
		db = orm.DB.Raw(sql, query+"*", omit)
	} else {
		sql += "LIMIT 10;"
		db = orm.DB.Raw(sql, query+"*")
	}

	err := db.Scan(&results).Error
	if err != nil && err != gorm.ErrRecordNotFound {
		return nil, errors.DatabaseError(err)
	}
	return results, nil
}

func (orm gormDB) DistributedQueriesForHost(host *kolide.Host) (map[uint]string, error) {
	sql := `
SELECT DISTINCT dqc.id, q.query
FROM distributed_query_campaigns dqc
JOIN distributed_query_campaign_targets dqct
    ON (dqc.id = dqct.distributed_query_campaign_id)
LEFT JOIN label_query_executions lqe
    ON (dqct.type = ? AND dqct.target_id = lqe.label_id AND lqe.matches)
LEFT JOIN hosts h
    ON ((dqct.type = ? AND lqe.host_id = h.id) OR (dqct.type = ? AND dqct.target_id = h.id))
LEFT JOIN distributed_query_executions dqe
    ON (h.id = dqe.host_id AND dqc.id = dqe.distributed_query_id)
JOIN queries q
    ON (dqc.query_id = q.id)
WHERE dqe.status IS NULL AND dqc.status = ? AND h.id = ?;
`
	rows, err := orm.DB.Raw(sql, kolide.TargetLabel, kolide.TargetLabel, kolide.TargetHost, kolide.QueryRunning, host.ID).Rows()
	if err != nil && err != gorm.ErrRecordNotFound {
		return nil, errors.DatabaseError(err)
	}
	defer rows.Close()

	results := map[uint]string{}
	for rows.Next() {
		var id uint
		var query string
		err = rows.Scan(&id, &query)
		if err != nil {
			return nil, errors.DatabaseError(err)
		}
		results[id] = query
	}

	return results, nil

}
<commit_msg>Fix changed ID column name in DistributedQueriesForHost query (#473)<commit_after>package datastore

import (
	"net/http"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/kolide/kolide-ose/server/errors"
	"github.com/kolide/kolide-ose/server/kolide"
)

func (orm gormDB) EnrollHost(uuid, hostname, ip, platform string, nodeKeySize int) (*kolide.Host, error) {
	if uuid == "" {
		return nil, errors.New("missing uuid for host enrollment", "programmer error?")
	}
	host := kolide.Host{UUID: uuid}
	err := orm.DB.Where(&host).First(&host).Error
	if err != nil {
		switch err {
		case gorm.ErrRecordNotFound:
			// Create new Host
			host = kolide.Host{
				UUID:             uuid,
				HostName:         hostname,
				PrimaryIP:        ip,
				Platform:         platform,
				DetailUpdateTime: time.Unix(0, 0).Add(24 * time.Hour),
			}

		default:
			return nil, err
		}
	}

	// Generate a new key each enrollment
	host.NodeKey, err = generateRandomText(nodeKeySize)
	if err != nil {
		return nil, err
	}

	// Update these fields if provided
	if hostname != "" {
		host.HostName = hostname
	}
	if ip != "" {
		host.PrimaryIP = ip
	}
	if platform != "" {
		host.Platform = platform
	}

	if err := orm.DB.Save(&host).Error; err != nil {
		return nil, err
	}

	return &host, nil
}

func (orm gormDB) AuthenticateHost(nodeKey string) (*kolide.Host, error) {
	host := kolide.Host{NodeKey: nodeKey}
	err := orm.DB.Where("node_key = ?", host.NodeKey).First(&host).Error
	if err != nil {
		switch err {
		case gorm.ErrRecordNotFound:
			e := errors.NewFromError(
				err,
				http.StatusUnauthorized,
				"invalid node key",
			)
			// osqueryd expects the literal string "true" here
			e.Extra = map[string]interface{}{"node_invalid": "true"}
			return nil, e
		default:
			return nil, errors.DatabaseError(err)
		}
	}

	return &host, nil
}

func (orm gormDB) SaveHost(host *kolide.Host) error {
	if err := orm.DB.Save(host).Error; err != nil {
		return errors.DatabaseError(err)
	}
	return nil
}

func (orm gormDB) DeleteHost(host *kolide.Host) error {
	return orm.DB.Delete(host).Error
}

func (orm gormDB) Host(id uint) (*kolide.Host, error) {
	host := &kolide.Host{
		ID: id,
	}
	err := orm.DB.Where(host).First(host).Error
	if err != nil {
		return nil, err
	}
	return host, nil
}

func (orm gormDB) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) {
	var hosts []*kolide.Host
	err := orm.applyListOptions(opt).Find(&hosts).Error
	if err != nil {
		return nil, err
	}
	return hosts, nil
}

func (orm gormDB) NewHost(host *kolide.Host) (*kolide.Host, error) {
	if host == nil {
		return nil, errors.New(
			"error creating host",
			"nil pointer passed to NewHost",
		)
	}
	err := orm.DB.Create(host).Error
	if err != nil {
		return nil, err
	}
	return host, err
}

func (orm gormDB) MarkHostSeen(host *kolide.Host, t time.Time) error {
	err := orm.DB.Exec("UPDATE hosts SET updated_at=? WHERE node_key=?", t, host.NodeKey).Error
	if err != nil {
		return errors.DatabaseError(err)
	}
	host.UpdatedAt = t
	return nil
}

func (orm gormDB) SearchHosts(query string, omit []uint) ([]kolide.Host, error) {
	sql := `
SELECT *
FROM hosts
WHERE MATCH(host_name, primary_ip)
AGAINST(? IN BOOLEAN MODE)
`
	results := []kolide.Host{}

	var db *gorm.DB
	if len(omit) > 0 {
		sql += "AND id NOT IN (?) LIMIT 10;"
		db = orm.DB.Raw(sql, query+"*", omit)
	} else {
		sql += "LIMIT 10;"
		db = orm.DB.Raw(sql, query+"*")
	}

	err := db.Scan(&results).Error
	if err != nil && err != gorm.ErrRecordNotFound {
		return nil, errors.DatabaseError(err)
	}
	return results, nil
}

func (orm gormDB) DistributedQueriesForHost(host *kolide.Host) (map[uint]string, error) {
	sql := `
SELECT DISTINCT dqc.id, q.query
FROM distributed_query_campaigns dqc
JOIN distributed_query_campaign_targets dqct
    ON (dqc.id = dqct.distributed_query_campaign_id)
LEFT JOIN label_query_executions lqe
    ON (dqct.type = ? AND dqct.target_id = lqe.label_id AND lqe.matches)
LEFT JOIN hosts h
    ON ((dqct.type = ? AND lqe.host_id = h.id) OR (dqct.type = ? AND dqct.target_id = h.id))
LEFT JOIN distributed_query_executions dqe
    ON (h.id = dqe.host_id AND dqc.id = dqe.distributed_query_campaign_id)
JOIN queries q
    ON (dqc.query_id = q.id)
WHERE dqe.status IS NULL AND dqc.status = ? AND h.id = ?;
`
	rows, err := orm.DB.Raw(sql, kolide.TargetLabel, kolide.TargetLabel, kolide.TargetHost, kolide.QueryRunning, host.ID).Rows()
	if err != nil && err != gorm.ErrRecordNotFound {
		return nil, errors.DatabaseError(err)
	}
	defer rows.Close()

	results := map[uint]string{}
	for rows.Next() {
		var id uint
		var query string
		err = rows.Scan(&id, &query)
		if err != nil {
			return nil, errors.DatabaseError(err)
		}
		results[id] = query
	}

	return results, nil

}
<|endoftext|>


<commit_before>package schemabuilder

import (
	"fmt"
	"reflect"

	"github.com/samsarahq/thunder/graphql"
)

type Schema struct {
	objects   map[string]*Object
	enumTypes map[reflect.Type]*EnumMapping
}

func NewSchema() *Schema {
	schema := &Schema{
		objects: make(map[string]*Object),
	}

	// Default registrations.
	schema.Enum(SortOrder(0), map[string]SortOrder{
		"asc":  SortOrder_Ascending,
		"desc": SortOrder_Descending,
	})

	return schema
}

// Enum registers an enumType in the schema. The val should be any arbitrary value
// of the enumType to be used for reflection, and the enumMap should be
// the corresponding map of the enums.
//
// For example a enum could be declared as follows:
//   type enumType int32
//   const (
//	  one   enumType = 1
//	  two   enumType = 2
//	  three enumType = 3
//   )
//
// Then the Enum can be registered as:
//   s.Enum(enumType(1), map[string]interface{}{
//     "one":   enumType(1),
//     "two":   enumType(2),
//     "three": enumType(3),
//   })
func (s *Schema) Enum(val interface{}, enumMap interface{}) {
	typ := reflect.TypeOf(val)
	if s.enumTypes == nil {
		s.enumTypes = make(map[reflect.Type]*EnumMapping)
	}

	eMap, rMap := getEnumMap(enumMap, typ)
	s.enumTypes[typ] = &EnumMapping{Map: eMap, ReverseMap: rMap}
}

func getEnumMap(enumMap interface{}, typ reflect.Type) (map[string]interface{}, map[interface{}]string) {
	rMap := make(map[interface{}]string)
	eMap := make(map[string]interface{})
	v := reflect.ValueOf(enumMap)
	if v.Kind() == reflect.Map {
		for _, key := range v.MapKeys() {
			val := v.MapIndex(key)
			valInterface := val.Interface()
			if reflect.TypeOf(valInterface).Kind() != typ.Kind() {
				panic("types are not equal")
			}
			if key.Kind() == reflect.String {
				mapVal := reflect.ValueOf(valInterface).Convert(typ)
				eMap[key.String()] = mapVal.Interface()
			} else {
				panic("keys are not strings")
			}
		}
	} else {
		panic("enum function not passed a map")
	}

	for key, val := range eMap {
		rMap[val] = key
	}
	return eMap, rMap

}

func (s *Schema) Object(name string, typ interface{}) *Object {
	if object, ok := s.objects[name]; ok {
		if reflect.TypeOf(object.Type) != reflect.TypeOf(typ) {
			panic("re-registered object with different type")
		}
		return object
	}
	object := &Object{
		Name: name,
		Type: typ,
	}
	s.objects[name] = object
	return object
}

type query struct{}

func (s *Schema) Query() *Object {
	return s.Object("Query", query{})
}

type mutation struct{}

func (s *Schema) Mutation() *Object {
	return s.Object("Mutation", mutation{})
}

func (s *Schema) Build() (*graphql.Schema, error) {
	sb := &schemaBuilder{
		types:        make(map[reflect.Type]graphql.Type),
		objects:      make(map[reflect.Type]*Object),
		enumMappings: s.enumTypes,
		typeCache:    make(map[reflect.Type]cachedType, 0),
	}

	for _, object := range s.objects {
		typ := reflect.TypeOf(object.Type)
		if typ.Kind() != reflect.Struct {
			return nil, fmt.Errorf("object.Type should be a struct, not %s", typ.String())
		}

		if _, ok := sb.objects[typ]; ok {
			return nil, fmt.Errorf("duplicate object for %s", typ.String())
		}

		sb.objects[typ] = object
	}

	queryTyp, err := sb.getType(reflect.TypeOf(&query{}))
	if err != nil {
		return nil, err
	}
	mutationTyp, err := sb.getType(reflect.TypeOf(&mutation{}))
	if err != nil {
		return nil, err
	}
	return &graphql.Schema{
		Query:    queryTyp,
		Mutation: mutationTyp,
	}, nil
}

// MustBuildSchema builds a schema and panics if an error occurs
func (s *Schema) MustBuild() *graphql.Schema {
	built, err := s.Build()
	if err != nil {
		panic(err)
	}
	return built
}
<commit_msg>go schemabuilder: Add comments for Schema exported functions<commit_after>package schemabuilder

import (
	"fmt"
	"reflect"

	"github.com/samsarahq/thunder/graphql"
)

// Schema is a struct that can be used to build out a GraphQL schema.  Functions
// can be registered against the "Mutation" and "Query" objects in order to
// build out a full GraphQL schema.
type Schema struct {
	objects   map[string]*Object
	enumTypes map[reflect.Type]*EnumMapping
}

// NewSchema creates a new schema.
func NewSchema() *Schema {
	schema := &Schema{
		objects: make(map[string]*Object),
	}

	// Default registrations.
	schema.Enum(SortOrder(0), map[string]SortOrder{
		"asc":  SortOrder_Ascending,
		"desc": SortOrder_Descending,
	})

	return schema
}

// Enum registers an enumType in the schema. The val should be any arbitrary value
// of the enumType to be used for reflection, and the enumMap should be
// the corresponding map of the enums.
//
// For example a enum could be declared as follows:
//   type enumType int32
//   const (
//	  one   enumType = 1
//	  two   enumType = 2
//	  three enumType = 3
//   )
//
// Then the Enum can be registered as:
//   s.Enum(enumType(1), map[string]interface{}{
//     "one":   enumType(1),
//     "two":   enumType(2),
//     "three": enumType(3),
//   })
func (s *Schema) Enum(val interface{}, enumMap interface{}) {
	typ := reflect.TypeOf(val)
	if s.enumTypes == nil {
		s.enumTypes = make(map[reflect.Type]*EnumMapping)
	}

	eMap, rMap := getEnumMap(enumMap, typ)
	s.enumTypes[typ] = &EnumMapping{Map: eMap, ReverseMap: rMap}
}

func getEnumMap(enumMap interface{}, typ reflect.Type) (map[string]interface{}, map[interface{}]string) {
	rMap := make(map[interface{}]string)
	eMap := make(map[string]interface{})
	v := reflect.ValueOf(enumMap)
	if v.Kind() == reflect.Map {
		for _, key := range v.MapKeys() {
			val := v.MapIndex(key)
			valInterface := val.Interface()
			if reflect.TypeOf(valInterface).Kind() != typ.Kind() {
				panic("types are not equal")
			}
			if key.Kind() == reflect.String {
				mapVal := reflect.ValueOf(valInterface).Convert(typ)
				eMap[key.String()] = mapVal.Interface()
			} else {
				panic("keys are not strings")
			}
		}
	} else {
		panic("enum function not passed a map")
	}

	for key, val := range eMap {
		rMap[val] = key
	}
	return eMap, rMap

}

// Object registers a struct as a GraphQL Object in our Schema.
// (https://facebook.github.io/graphql/June2018/#sec-Objects)
// We'll read the fields of the struct to determine it's basic "Fields" and
// we'll return an Object struct that we can use to register custom
// relationships and fields on the object.
func (s *Schema) Object(name string, typ interface{}) *Object {
	if object, ok := s.objects[name]; ok {
		if reflect.TypeOf(object.Type) != reflect.TypeOf(typ) {
			panic("re-registered object with different type")
		}
		return object
	}
	object := &Object{
		Name: name,
		Type: typ,
	}
	s.objects[name] = object
	return object
}

type query struct{}

// Query returns an Object struct that we can use to register all the top level
// graphql query functions we'd like to expose.
func (s *Schema) Query() *Object {
	return s.Object("Query", query{})
}

type mutation struct{}

// Mutation returns an Object struct that we can use to register all the top level
// graphql mutations functions we'd like to expose.
func (s *Schema) Mutation() *Object {
	return s.Object("Mutation", mutation{})
}

// Build takes the schema we have built on our Query and Mutation starting
// points and builds a full graphql.Schema we can use to execute and run
// queries.  Essentially we read through all the methods we've attached to our
// Query and Mutation Objects and ensure that those functions are returning
// other Objects that we can resolve in our GraphQL graph.
func (s *Schema) Build() (*graphql.Schema, error) {
	sb := &schemaBuilder{
		types:        make(map[reflect.Type]graphql.Type),
		objects:      make(map[reflect.Type]*Object),
		enumMappings: s.enumTypes,
		typeCache:    make(map[reflect.Type]cachedType, 0),
	}

	for _, object := range s.objects {
		typ := reflect.TypeOf(object.Type)
		if typ.Kind() != reflect.Struct {
			return nil, fmt.Errorf("object.Type should be a struct, not %s", typ.String())
		}

		if _, ok := sb.objects[typ]; ok {
			return nil, fmt.Errorf("duplicate object for %s", typ.String())
		}

		sb.objects[typ] = object
	}

	queryTyp, err := sb.getType(reflect.TypeOf(&query{}))
	if err != nil {
		return nil, err
	}
	mutationTyp, err := sb.getType(reflect.TypeOf(&mutation{}))
	if err != nil {
		return nil, err
	}
	return &graphql.Schema{
		Query:    queryTyp,
		Mutation: mutationTyp,
	}, nil
}

// MustBuildSchema builds a schema and panics if an error occurs.
func (s *Schema) MustBuild() *graphql.Schema {
	built, err := s.Build()
	if err != nil {
		panic(err)
	}
	return built
}
<|endoftext|>


<commit_before>package docker

import (
	"fmt"
	"github.com/dotcloud/docker/utils"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/user"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"
)

const unitTestImageName string = "docker-ut"
const unitTestImageId string = "e9aa60c60128cad1"
const unitTestStoreBase string = "/var/lib/docker/unit-tests"

func nuke(runtime *Runtime) error {
	var wg sync.WaitGroup
	for _, container := range runtime.List() {
		wg.Add(1)
		go func(c *Container) {
			c.Kill()
			wg.Done()
		}(container)
	}
	wg.Wait()
	return os.RemoveAll(runtime.root)
}

func layerArchive(tarfile string) (io.Reader, error) {
	// FIXME: need to close f somewhere
	f, err := os.Open(tarfile)
	if err != nil {
		return nil, err
	}
	return f, nil
}

func init() {
	// Hack to run sys init during unit testing
	if utils.SelfPath() == "/sbin/init" {
		SysInit()
		return
	}

	if usr, err := user.Current(); err != nil {
		panic(err)
	} else if usr.Uid != "0" {
		panic("docker tests needs to be run as root")
	}

	NetworkBridgeIface = "testdockbr0"

	// Make it our Store root
	runtime, err := NewRuntimeFromDirectory(unitTestStoreBase, false)
	if err != nil {
		panic(err)
	}

	// Create the "Server"
	srv := &Server{
		runtime:     runtime,
		enableCors:  false,
		lock:        &sync.Mutex{},
		pullingPool: make(map[string]struct{}),
		pushingPool: make(map[string]struct{}),
	}
	// Retrieve the Image
	if err := srv.ImagePull(unitTestImageName, "", "", os.Stdout, utils.NewStreamFormatter(false), nil); err != nil {
		panic(err)
	}
}

// FIXME: test that ImagePull(json=true) send correct json output

func newTestRuntime() (*Runtime, error) {
	root, err := ioutil.TempDir("", "docker-test")
	if err != nil {
		return nil, err
	}
	if err := os.Remove(root); err != nil {
		return nil, err
	}
	if err := utils.CopyDirectory(unitTestStoreBase, root); err != nil {
		return nil, err
	}

	runtime, err := NewRuntimeFromDirectory(root, false)
	if err != nil {
		return nil, err
	}
	runtime.UpdateCapabilities(true)
	return runtime, nil
}

func GetTestImage(runtime *Runtime) *Image {
	imgs, err := runtime.graph.All()
	if err != nil {
		panic(err)
	} else if len(imgs) < 1 {
		panic("GASP")
	}
	return imgs[0]
}

func TestRuntimeCreate(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)

	// Make sure we start we 0 containers
	if len(runtime.List()) != 0 {
		t.Errorf("Expected 0 containers, %v found", len(runtime.List()))
	}

	builder := NewBuilder(runtime)

	container, err := builder.Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := runtime.Destroy(container); err != nil {
			t.Error(err)
		}
	}()

	// Make sure we can find the newly created container with List()
	if len(runtime.List()) != 1 {
		t.Errorf("Expected 1 container, %v found", len(runtime.List()))
	}

	// Make sure the container List() returns is the right one
	if runtime.List()[0].ID != container.ID {
		t.Errorf("Unexpected container %v returned by List", runtime.List()[0])
	}

	// Make sure we can get the container with Get()
	if runtime.Get(container.ID) == nil {
		t.Errorf("Unable to get newly created container")
	}

	// Make sure it is the right container
	if runtime.Get(container.ID) != container {
		t.Errorf("Get() returned the wrong container")
	}

	// Make sure Exists returns it as existing
	if !runtime.Exists(container.ID) {
		t.Errorf("Exists() returned false for a newly created container")
	}

	// Make sure crete with bad parameters returns an error
	_, err = builder.Create(
		&Config{
			Image: GetTestImage(runtime).ID,
		},
	)
	if err == nil {
		t.Fatal("Builder.Create should throw an error when Cmd is missing")
	}

	_, err = builder.Create(
		&Config{
			Image: GetTestImage(runtime).ID,
			Cmd:   []string{},
		},
	)
	if err == nil {
		t.Fatal("Builder.Create should throw an error when Cmd is empty")
	}
}

func TestDestroy(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)
	container, err := NewBuilder(runtime).Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	// Destroy
	if err := runtime.Destroy(container); err != nil {
		t.Error(err)
	}

	// Make sure runtime.Exists() behaves correctly
	if runtime.Exists("test_destroy") {
		t.Errorf("Exists() returned true")
	}

	// Make sure runtime.List() doesn't list the destroyed container
	if len(runtime.List()) != 0 {
		t.Errorf("Expected 0 container, %v found", len(runtime.List()))
	}

	// Make sure runtime.Get() refuses to return the unexisting container
	if runtime.Get(container.ID) != nil {
		t.Errorf("Unable to get newly created container")
	}

	// Make sure the container root directory does not exist anymore
	_, err = os.Stat(container.root)
	if err == nil || !os.IsNotExist(err) {
		t.Errorf("Container root directory still exists after destroy")
	}

	// Test double destroy
	if err := runtime.Destroy(container); err == nil {
		// It should have failed
		t.Errorf("Double destroy did not fail")
	}
}

func TestGet(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)

	builder := NewBuilder(runtime)

	container1, err := builder.Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime.Destroy(container1)

	container2, err := builder.Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime.Destroy(container2)

	container3, err := builder.Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime.Destroy(container3)

	if runtime.Get(container1.ID) != container1 {
		t.Errorf("Get(test1) returned %v while expecting %v", runtime.Get(container1.ID), container1)
	}

	if runtime.Get(container2.ID) != container2 {
		t.Errorf("Get(test2) returned %v while expecting %v", runtime.Get(container2.ID), container2)
	}

	if runtime.Get(container3.ID) != container3 {
		t.Errorf("Get(test3) returned %v while expecting %v", runtime.Get(container3.ID), container3)
	}

}

func findAvailalblePort(runtime *Runtime, port int) (*Container, error) {
	strPort := strconv.Itoa(port)
	container, err := NewBuilder(runtime).Create(&Config{
		Image:     GetTestImage(runtime).ID,
		Cmd:       []string{"sh", "-c", "echo well hello there | nc -l -p " + strPort},
		PortSpecs: []string{strPort},
	},
	)
	if err != nil {
		return nil, err
	}
	if err := container.Start(); err != nil {
		if strings.Contains(err.Error(), "address already in use") {
			return nil, nil
		}
		return nil, err
	}
	return container, nil
}

// Run a container with a TCP port allocated, and test that it can receive connections on localhost
func TestAllocatePortLocalhost(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	port := 5554

	var container *Container
	for {
		port += 1
		log.Println("Trying port", port)
		t.Log("Trying port", port)
		container, err = findAvailalblePort(runtime, port)
		if container != nil {
			break
		}
		if err != nil {
			t.Fatal(err)
		}
		log.Println("Port", port, "already in use")
		t.Log("Port", port, "already in use")
	}

	defer container.Kill()

	setTimeout(t, "Waiting for the container to be started timed out", 2*time.Second, func() {
		for {
			if container.State.Running {
				break
			}
			time.Sleep(10 * time.Millisecond)
		}
	})

	conn, err := net.Dial("tcp",
		fmt.Sprintf(
			"localhost:%s", container.NetworkSettings.PortMapping[strconv.Itoa(port)],
		),
	)
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()
	output, err := ioutil.ReadAll(conn)
	if err != nil {
		t.Fatal(err)
	}
	if string(output) != "well hello there\n" {
		t.Fatalf("Received wrong output from network connection: should be '%s', not '%s'",
			"well hello there\n",
			string(output),
		)
	}
	container.Wait()
}

func TestRestore(t *testing.T) {

	root, err := ioutil.TempDir("", "docker-test")
	if err != nil {
		t.Fatal(err)
	}
	if err := os.Remove(root); err != nil {
		t.Fatal(err)
	}
	if err := utils.CopyDirectory(unitTestStoreBase, root); err != nil {
		t.Fatal(err)
	}

	runtime1, err := NewRuntimeFromDirectory(root, false)
	if err != nil {
		t.Fatal(err)
	}

	builder := NewBuilder(runtime1)

	// Create a container with one instance of docker
	container1, err := builder.Create(&Config{
		Image: GetTestImage(runtime1).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime1.Destroy(container1)

	// Create a second container meant to be killed
	container2, err := builder.Create(&Config{
		Image:     GetTestImage(runtime1).ID,
		Cmd:       []string{"/bin/cat"},
		OpenStdin: true,
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime1.Destroy(container2)

	// Start the container non blocking
	if err := container2.Start(); err != nil {
		t.Fatal(err)
	}

	if !container2.State.Running {
		t.Fatalf("Container %v should appear as running but isn't", container2.ID)
	}

	// Simulate a crash/manual quit of dockerd: process dies, states stays 'Running'
	cStdin, _ := container2.StdinPipe()
	cStdin.Close()
	if err := container2.WaitTimeout(2 * time.Second); err != nil {
		t.Fatal(err)
	}
	container2.State.Running = true
	container2.ToDisk()

	if len(runtime1.List()) != 2 {
		t.Errorf("Expected 2 container, %v found", len(runtime1.List()))
	}
	if err := container1.Run(); err != nil {
		t.Fatal(err)
	}

	if !container2.State.Running {
		t.Fatalf("Container %v should appear as running but isn't", container2.ID)
	}

	// Here are are simulating a docker restart - that is, reloading all containers
	// from scratch
	runtime2, err := NewRuntimeFromDirectory(root, false)
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime2)
	if len(runtime2.List()) != 2 {
		t.Errorf("Expected 2 container, %v found", len(runtime2.List()))
	}
	runningCount := 0
	for _, c := range runtime2.List() {
		if c.State.Running {
			t.Errorf("Running container found: %v (%v)", c.ID, c.Path)
			runningCount++
		}
	}
	if runningCount != 0 {
		t.Fatalf("Expected 0 container alive, %d found", runningCount)
	}
	container3 := runtime2.Get(container1.ID)
	if container3 == nil {
		t.Fatal("Unable to Get container")
	}
	if err := container3.Run(); err != nil {
		t.Fatal(err)
	}
	container2.State.Running = false
}
<commit_msg>Remove dependency on 'os/user', which cannot be used with CGO_ENABLED=0. This allows running the tests without CGO.<commit_after>package docker

import (
	"fmt"
	"github.com/dotcloud/docker/utils"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"testing"
	"time"
)

const unitTestImageName string = "docker-ut"
const unitTestImageId string = "e9aa60c60128cad1"
const unitTestStoreBase string = "/var/lib/docker/unit-tests"

func nuke(runtime *Runtime) error {
	var wg sync.WaitGroup
	for _, container := range runtime.List() {
		wg.Add(1)
		go func(c *Container) {
			c.Kill()
			wg.Done()
		}(container)
	}
	wg.Wait()
	return os.RemoveAll(runtime.root)
}

func layerArchive(tarfile string) (io.Reader, error) {
	// FIXME: need to close f somewhere
	f, err := os.Open(tarfile)
	if err != nil {
		return nil, err
	}
	return f, nil
}

func init() {
	// Hack to run sys init during unit testing
	if utils.SelfPath() == "/sbin/init" {
		SysInit()
		return
	}

	if uid := syscall.Geteuid(); uid != 0 {
		log.Fatal("docker tests needs to be run as root")
	}

	NetworkBridgeIface = "testdockbr0"

	// Make it our Store root
	runtime, err := NewRuntimeFromDirectory(unitTestStoreBase, false)
	if err != nil {
		panic(err)
	}

	// Create the "Server"
	srv := &Server{
		runtime:     runtime,
		enableCors:  false,
		lock:        &sync.Mutex{},
		pullingPool: make(map[string]struct{}),
		pushingPool: make(map[string]struct{}),
	}
	// Retrieve the Image
	if err := srv.ImagePull(unitTestImageName, "", "", os.Stdout, utils.NewStreamFormatter(false), nil); err != nil {
		panic(err)
	}
}

// FIXME: test that ImagePull(json=true) send correct json output

func newTestRuntime() (*Runtime, error) {
	root, err := ioutil.TempDir("", "docker-test")
	if err != nil {
		return nil, err
	}
	if err := os.Remove(root); err != nil {
		return nil, err
	}
	if err := utils.CopyDirectory(unitTestStoreBase, root); err != nil {
		return nil, err
	}

	runtime, err := NewRuntimeFromDirectory(root, false)
	if err != nil {
		return nil, err
	}
	runtime.UpdateCapabilities(true)
	return runtime, nil
}

func GetTestImage(runtime *Runtime) *Image {
	imgs, err := runtime.graph.All()
	if err != nil {
		panic(err)
	} else if len(imgs) < 1 {
		panic("GASP")
	}
	return imgs[0]
}

func TestRuntimeCreate(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)

	// Make sure we start we 0 containers
	if len(runtime.List()) != 0 {
		t.Errorf("Expected 0 containers, %v found", len(runtime.List()))
	}

	builder := NewBuilder(runtime)

	container, err := builder.Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := runtime.Destroy(container); err != nil {
			t.Error(err)
		}
	}()

	// Make sure we can find the newly created container with List()
	if len(runtime.List()) != 1 {
		t.Errorf("Expected 1 container, %v found", len(runtime.List()))
	}

	// Make sure the container List() returns is the right one
	if runtime.List()[0].ID != container.ID {
		t.Errorf("Unexpected container %v returned by List", runtime.List()[0])
	}

	// Make sure we can get the container with Get()
	if runtime.Get(container.ID) == nil {
		t.Errorf("Unable to get newly created container")
	}

	// Make sure it is the right container
	if runtime.Get(container.ID) != container {
		t.Errorf("Get() returned the wrong container")
	}

	// Make sure Exists returns it as existing
	if !runtime.Exists(container.ID) {
		t.Errorf("Exists() returned false for a newly created container")
	}

	// Make sure crete with bad parameters returns an error
	_, err = builder.Create(
		&Config{
			Image: GetTestImage(runtime).ID,
		},
	)
	if err == nil {
		t.Fatal("Builder.Create should throw an error when Cmd is missing")
	}

	_, err = builder.Create(
		&Config{
			Image: GetTestImage(runtime).ID,
			Cmd:   []string{},
		},
	)
	if err == nil {
		t.Fatal("Builder.Create should throw an error when Cmd is empty")
	}
}

func TestDestroy(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)
	container, err := NewBuilder(runtime).Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	// Destroy
	if err := runtime.Destroy(container); err != nil {
		t.Error(err)
	}

	// Make sure runtime.Exists() behaves correctly
	if runtime.Exists("test_destroy") {
		t.Errorf("Exists() returned true")
	}

	// Make sure runtime.List() doesn't list the destroyed container
	if len(runtime.List()) != 0 {
		t.Errorf("Expected 0 container, %v found", len(runtime.List()))
	}

	// Make sure runtime.Get() refuses to return the unexisting container
	if runtime.Get(container.ID) != nil {
		t.Errorf("Unable to get newly created container")
	}

	// Make sure the container root directory does not exist anymore
	_, err = os.Stat(container.root)
	if err == nil || !os.IsNotExist(err) {
		t.Errorf("Container root directory still exists after destroy")
	}

	// Test double destroy
	if err := runtime.Destroy(container); err == nil {
		// It should have failed
		t.Errorf("Double destroy did not fail")
	}
}

func TestGet(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime)

	builder := NewBuilder(runtime)

	container1, err := builder.Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime.Destroy(container1)

	container2, err := builder.Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime.Destroy(container2)

	container3, err := builder.Create(&Config{
		Image: GetTestImage(runtime).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime.Destroy(container3)

	if runtime.Get(container1.ID) != container1 {
		t.Errorf("Get(test1) returned %v while expecting %v", runtime.Get(container1.ID), container1)
	}

	if runtime.Get(container2.ID) != container2 {
		t.Errorf("Get(test2) returned %v while expecting %v", runtime.Get(container2.ID), container2)
	}

	if runtime.Get(container3.ID) != container3 {
		t.Errorf("Get(test3) returned %v while expecting %v", runtime.Get(container3.ID), container3)
	}

}

func findAvailalblePort(runtime *Runtime, port int) (*Container, error) {
	strPort := strconv.Itoa(port)
	container, err := NewBuilder(runtime).Create(&Config{
		Image:     GetTestImage(runtime).ID,
		Cmd:       []string{"sh", "-c", "echo well hello there | nc -l -p " + strPort},
		PortSpecs: []string{strPort},
	},
	)
	if err != nil {
		return nil, err
	}
	if err := container.Start(); err != nil {
		if strings.Contains(err.Error(), "address already in use") {
			return nil, nil
		}
		return nil, err
	}
	return container, nil
}

// Run a container with a TCP port allocated, and test that it can receive connections on localhost
func TestAllocatePortLocalhost(t *testing.T) {
	runtime, err := newTestRuntime()
	if err != nil {
		t.Fatal(err)
	}
	port := 5554

	var container *Container
	for {
		port += 1
		log.Println("Trying port", port)
		t.Log("Trying port", port)
		container, err = findAvailalblePort(runtime, port)
		if container != nil {
			break
		}
		if err != nil {
			t.Fatal(err)
		}
		log.Println("Port", port, "already in use")
		t.Log("Port", port, "already in use")
	}

	defer container.Kill()

	setTimeout(t, "Waiting for the container to be started timed out", 2*time.Second, func() {
		for {
			if container.State.Running {
				break
			}
			time.Sleep(10 * time.Millisecond)
		}
	})

	conn, err := net.Dial("tcp",
		fmt.Sprintf(
			"localhost:%s", container.NetworkSettings.PortMapping[strconv.Itoa(port)],
		),
	)
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()
	output, err := ioutil.ReadAll(conn)
	if err != nil {
		t.Fatal(err)
	}
	if string(output) != "well hello there\n" {
		t.Fatalf("Received wrong output from network connection: should be '%s', not '%s'",
			"well hello there\n",
			string(output),
		)
	}
	container.Wait()
}

func TestRestore(t *testing.T) {

	root, err := ioutil.TempDir("", "docker-test")
	if err != nil {
		t.Fatal(err)
	}
	if err := os.Remove(root); err != nil {
		t.Fatal(err)
	}
	if err := utils.CopyDirectory(unitTestStoreBase, root); err != nil {
		t.Fatal(err)
	}

	runtime1, err := NewRuntimeFromDirectory(root, false)
	if err != nil {
		t.Fatal(err)
	}

	builder := NewBuilder(runtime1)

	// Create a container with one instance of docker
	container1, err := builder.Create(&Config{
		Image: GetTestImage(runtime1).ID,
		Cmd:   []string{"ls", "-al"},
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime1.Destroy(container1)

	// Create a second container meant to be killed
	container2, err := builder.Create(&Config{
		Image:     GetTestImage(runtime1).ID,
		Cmd:       []string{"/bin/cat"},
		OpenStdin: true,
	},
	)
	if err != nil {
		t.Fatal(err)
	}
	defer runtime1.Destroy(container2)

	// Start the container non blocking
	if err := container2.Start(); err != nil {
		t.Fatal(err)
	}

	if !container2.State.Running {
		t.Fatalf("Container %v should appear as running but isn't", container2.ID)
	}

	// Simulate a crash/manual quit of dockerd: process dies, states stays 'Running'
	cStdin, _ := container2.StdinPipe()
	cStdin.Close()
	if err := container2.WaitTimeout(2 * time.Second); err != nil {
		t.Fatal(err)
	}
	container2.State.Running = true
	container2.ToDisk()

	if len(runtime1.List()) != 2 {
		t.Errorf("Expected 2 container, %v found", len(runtime1.List()))
	}
	if err := container1.Run(); err != nil {
		t.Fatal(err)
	}

	if !container2.State.Running {
		t.Fatalf("Container %v should appear as running but isn't", container2.ID)
	}

	// Here are are simulating a docker restart - that is, reloading all containers
	// from scratch
	runtime2, err := NewRuntimeFromDirectory(root, false)
	if err != nil {
		t.Fatal(err)
	}
	defer nuke(runtime2)
	if len(runtime2.List()) != 2 {
		t.Errorf("Expected 2 container, %v found", len(runtime2.List()))
	}
	runningCount := 0
	for _, c := range runtime2.List() {
		if c.State.Running {
			t.Errorf("Running container found: %v (%v)", c.ID, c.Path)
			runningCount++
		}
	}
	if runningCount != 0 {
		t.Fatalf("Expected 0 container alive, %d found", runningCount)
	}
	container3 := runtime2.Get(container1.ID)
	if container3 == nil {
		t.Fatal("Unable to Get container")
	}
	if err := container3.Run(); err != nil {
		t.Fatal(err)
	}
	container2.State.Running = false
}
<|endoftext|>


<commit_before>// Copyright 2021 PingCAP, Inc. Licensed under Apache-2.0.

package logutil_test

import (
	"context"
	"fmt"
	"math"
	"strings"
	"testing"
	"time"

	. "github.com/pingcap/check"
	"github.com/pingcap/errors"
	backuppb "github.com/pingcap/kvproto/pkg/brpb"
	"github.com/pingcap/kvproto/pkg/import_sstpb"
	"github.com/pingcap/kvproto/pkg/metapb"
	berrors "github.com/pingcap/tidb/br/pkg/errors"
	"github.com/pingcap/tidb/br/pkg/logutil"
	"github.com/prometheus/client_golang/prometheus"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
	"go.uber.org/zap/zaptest/observer"
)

func Test(t *testing.T) {
	TestingT(t)
}

var _ = Suite(&testLoggingSuite{})

type testLoggingSuite struct{}

func assertTrimEqual(c *C, f zapcore.Field, expect string) {
	encoder := zapcore.NewConsoleEncoder(zapcore.EncoderConfig{})
	out, err := encoder.EncodeEntry(zapcore.Entry{}, []zap.Field{f})
	c.Assert(err, IsNil)
	c.Assert(strings.TrimRight(out.String(), "\n"), Equals, expect)
}

func newFile(j int) *backuppb.File {
	return &backuppb.File{
		Name:         fmt.Sprint(j),
		StartKey:     []byte(fmt.Sprint(j)),
		EndKey:       []byte(fmt.Sprint(j + 1)),
		TotalKvs:     uint64(j),
		TotalBytes:   uint64(j),
		StartVersion: uint64(j),
		EndVersion:   uint64(j + 1),
		Crc64Xor:     uint64(j),
		Sha256:       []byte(fmt.Sprint(j)),
		Cf:           "write",
		Size_:        uint64(j),
	}
}

type isAbout struct{}

func (isAbout) Info() *CheckerInfo {
	return &CheckerInfo{
		Name: "isAbout",
		Params: []string{
			"actual",
			"expect",
		},
	}
}

func (isAbout) Check(params []interface{}, names []string) (result bool, error string) {
	actual := params[0].(float64)
	expect := params[1].(float64)

	if diff := math.Abs(1 - (actual / expect)); diff > 0.1 {
		return false, fmt.Sprintf("The diff(%.2f) between actual(%.2f) and expect(%.2f) is too huge.", diff, actual, expect)
	}
	return true, ""
}

func (s *testLoggingSuite) TestRater(c *C) {
	m := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "testing",
		Name:      "rater",
		Help:      "A testing counter for the rater",
	})
	m.Add(42)

	rater := logutil.TraceRateOver(m)
	timePass := time.Now()
	rater.Inc()
	c.Assert(rater.RateAt(timePass.Add(100*time.Millisecond)), isAbout{}, 10.0)
	rater.Inc()
	c.Assert(rater.RateAt(timePass.Add(150*time.Millisecond)), isAbout{}, 13.0)
	rater.Add(18)
	c.Assert(rater.RateAt(timePass.Add(200*time.Millisecond)), isAbout{}, 100.0)
}

func (s *testLoggingSuite) TestFile(c *C) {
	assertTrimEqual(c, logutil.File(newFile(1)),
		`{"file": {"name": "1", "CF": "write", "sha256": "31", "startKey": "31", "endKey": "32", "startVersion": 1, "endVersion": 2, "totalKvs": 1, "totalBytes": 1, "CRC64Xor": 1}}`)
}

func (s *testLoggingSuite) TestFiles(c *C) {
	cases := []struct {
		count  int
		expect string
	}{
		{0, `{"files": {"total": 0, "files": [], "totalKVs": 0, "totalBytes": 0, "totalSize": 0}}`},
		{1, `{"files": {"total": 1, "files": ["0"], "totalKVs": 0, "totalBytes": 0, "totalSize": 0}}`},
		{2, `{"files": {"total": 2, "files": ["0", "1"], "totalKVs": 1, "totalBytes": 1, "totalSize": 1}}`},
		{3, `{"files": {"total": 3, "files": ["0", "1", "2"], "totalKVs": 3, "totalBytes": 3, "totalSize": 3}}`},
		{4, `{"files": {"total": 4, "files": ["0", "1", "2", "3"], "totalKVs": 6, "totalBytes": 6, "totalSize": 6}}`},
		{5, `{"files": {"total": 5, "files": ["0", "(skip 3)", "4"], "totalKVs": 10, "totalBytes": 10, "totalSize": 10}}`},
		{6, `{"files": {"total": 6, "files": ["0", "(skip 4)", "5"], "totalKVs": 15, "totalBytes": 15, "totalSize": 15}}`},
		{1024, `{"files": {"total": 1024, "files": ["0", "(skip 1022)", "1023"], "totalKVs": 523776, "totalBytes": 523776, "totalSize": 523776}}`},
	}

	for _, cs := range cases {
		ranges := make([]*backuppb.File, cs.count)
		for j := 0; j < cs.count; j++ {
			ranges[j] = newFile(j)
		}
		assertTrimEqual(c, logutil.Files(ranges), cs.expect)
	}
}

func (s *testLoggingSuite) TestKey(c *C) {
	encoder := zapcore.NewConsoleEncoder(zapcore.EncoderConfig{})
	out, err := encoder.EncodeEntry(zapcore.Entry{}, []zap.Field{logutil.Key("test", []byte{0, 1, 2, 3})})
	c.Assert(err, IsNil)
	c.Assert(strings.Trim(out.String(), "\n"), Equals, `{"test": "00010203"}`)
}

func (s *testLoggingSuite) TestKeys(c *C) {
	cases := []struct {
		count  int
		expect string
	}{
		{0, `{"keys": {"total": 0, "keys": []}}`},
		{1, `{"keys": {"total": 1, "keys": ["30303030"]}}`},
		{2, `{"keys": {"total": 2, "keys": ["30303030", "30303031"]}}`},
		{3, `{"keys": {"total": 3, "keys": ["30303030", "30303031", "30303032"]}}`},
		{4, `{"keys": {"total": 4, "keys": ["30303030", "30303031", "30303032", "30303033"]}}`},
		{5, `{"keys": {"total": 5, "keys": ["30303030", "(skip 3)", "30303034"]}}`},
		{6, `{"keys": {"total": 6, "keys": ["30303030", "(skip 4)", "30303035"]}}`},
		{1024, `{"keys": {"total": 1024, "keys": ["30303030", "(skip 1022)", "31303233"]}}`},
	}

	for _, cs := range cases {
		keys := make([][]byte, cs.count)
		for j := 0; j < cs.count; j++ {
			keys[j] = []byte(fmt.Sprintf("%04d", j))
		}
		assertTrimEqual(c, logutil.Keys(keys), cs.expect)
	}
}

func (s *testLoggingSuite) TestRewriteRule(c *C) {
	rule := &import_sstpb.RewriteRule{
		OldKeyPrefix: []byte("old"),
		NewKeyPrefix: []byte("new"),
		NewTimestamp: 0x555555,
	}

	encoder := zapcore.NewConsoleEncoder(zapcore.EncoderConfig{})
	out, err := encoder.EncodeEntry(zapcore.Entry{}, []zap.Field{logutil.RewriteRule(rule)})
	c.Assert(err, IsNil)
	c.Assert(strings.Trim(out.String(), "\n"), Equals, `{"rewriteRule": {"oldKeyPrefix": "6f6c64", "newKeyPrefix": "6e6577", "newTimestamp": 5592405}}`)
}

func (s *testLoggingSuite) TestRegion(c *C) {
	region := &metapb.Region{
		Id:          1,
		StartKey:    []byte{0x00, 0x01},
		EndKey:      []byte{0x00, 0x02},
		RegionEpoch: &metapb.RegionEpoch{ConfVer: 1, Version: 1},
		Peers:       []*metapb.Peer{{Id: 2, StoreId: 3}, {Id: 4, StoreId: 5}},
	}

	assertTrimEqual(c, logutil.Region(region),
		`{"region": {"ID": 1, "startKey": "0001", "endKey": "0002", "epoch": "conf_ver:1 version:1 ", "peers": "id:2 store_id:3 ,id:4 store_id:5 "}}`)
}

func (s *testLoggingSuite) TestLeader(c *C) {
	leader := &metapb.Peer{Id: 2, StoreId: 3}

	assertTrimEqual(c, logutil.Leader(leader), `{"leader": "id:2 store_id:3 "}`)
}

func (s *testLoggingSuite) TestSSTMeta(c *C) {
	meta := &import_sstpb.SSTMeta{
		Uuid: []byte("mock uuid"),
		Range: &import_sstpb.Range{
			Start: []byte{0x00, 0x01},
			End:   []byte{0x00, 0x02},
		},
		Crc32:       uint32(0x555555),
		Length:      1,
		CfName:      "default",
		RegionId:    1,
		RegionEpoch: &metapb.RegionEpoch{ConfVer: 1, Version: 1},
	}

	assertTrimEqual(c, logutil.SSTMeta(meta),
		`{"sstMeta": {"CF": "default", "endKeyExclusive": false, "CRC32": 5592405, "length": 1, "regionID": 1, "regionEpoch": "conf_ver:1 version:1 ", "startKey": "0001", "endKey": "0002", "UUID": "invalid UUID 6d6f636b2075756964"}}`)
}

func (s *testLoggingSuite) TestShortError(c *C) {
	err := errors.Annotate(berrors.ErrInvalidArgument, "test")

	assertTrimEqual(c, logutil.ShortError(err), `{"error": "test: [BR:Common:ErrInvalidArgument]invalid argument"}`)
}

type FieldEquals struct{}

func (f FieldEquals) Info() *CheckerInfo {
	return &CheckerInfo{
		Name: "FieldEquals",
		Params: []string{
			"expected",
			"actual",
		},
	}
}

func (f FieldEquals) Check(params []interface{}, names []string) (result bool, err string) {
	expected := params[0].(zap.Field)
	actual := params[1].(zap.Field)

	if !expected.Equals(actual) {
		return false, "Field not match."
	}
	return true, ""
}

func (s *testLoggingSuite) TestContextual(c *C) {
	testCore, logs := observer.New(zap.InfoLevel)
	logutil.ResetGlobalLogger(zap.New(testCore))

	ctx := context.Background()
	l0 := logutil.LoggerFromContext(ctx)
	l0.Info("going to take an adventure?", zap.Int("HP", 50), zap.Int("HP-MAX", 50), zap.String("character", "solte"))
	lctx := logutil.ContextWithField(ctx, zap.Strings("friends", []string{"firo", "seren", "black"}))
	l := logutil.LoggerFromContext(lctx)
	l.Info("let's go!", zap.String("character", "solte"))

	observedLogs := logs.TakeAll()
	checkLog(c, observedLogs[0],
		"going to take an adventure?", zap.Int("HP", 50), zap.Int("HP-MAX", 50), zap.String("character", "solte"))
	checkLog(c, observedLogs[1],
		"let's go!", zap.Strings("friends", []string{"firo", "seren", "black"}), zap.String("character", "solte"))
}

func checkLog(c *C, actual observer.LoggedEntry, message string, fields ...zap.Field) {
	c.Assert(message, Equals, actual.Message)
	for i, f := range fields {
		c.Assert(f, FieldEquals{}, actual.Context[i])
	}
}
<commit_msg>Refactor: migrate test infra for br/pkg/logutil/loggin_test.go to testify (#28557)<commit_after>// Copyright 2021 PingCAP, Inc. Licensed under Apache-2.0.

package logutil_test

import (
	"context"
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/pingcap/errors"
	backuppb "github.com/pingcap/kvproto/pkg/brpb"
	"github.com/pingcap/kvproto/pkg/import_sstpb"
	"github.com/pingcap/kvproto/pkg/metapb"
	berrors "github.com/pingcap/tidb/br/pkg/errors"
	"github.com/pingcap/tidb/br/pkg/logutil"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/stretchr/testify/require"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
	"go.uber.org/zap/zaptest/observer"
)

func assertTrimEqual(t *testing.T, f zapcore.Field, expect string) {
	encoder := zapcore.NewConsoleEncoder(zapcore.EncoderConfig{})
	out, err := encoder.EncodeEntry(zapcore.Entry{}, []zap.Field{f})
	require.NoError(t, err)
	require.JSONEq(t, expect, strings.TrimRight(out.String(), "\n"))
}

func newFile(j int) *backuppb.File {
	return &backuppb.File{
		Name:         fmt.Sprint(j),
		StartKey:     []byte(fmt.Sprint(j)),
		EndKey:       []byte(fmt.Sprint(j + 1)),
		TotalKvs:     uint64(j),
		TotalBytes:   uint64(j),
		StartVersion: uint64(j),
		EndVersion:   uint64(j + 1),
		Crc64Xor:     uint64(j),
		Sha256:       []byte(fmt.Sprint(j)),
		Cf:           "write",
		Size_:        uint64(j),
	}
}

func TestRater(t *testing.T) {
	t.Parallel()
	m := prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "testing",
		Name:      "rater",
		Help:      "A testing counter for the rater",
	})
	m.Add(42)

	rater := logutil.TraceRateOver(m)
	timePass := time.Now()
	rater.Inc()
	require.InEpsilon(t, 10.0, rater.RateAt(timePass.Add(100*time.Millisecond)), 0.1)
	rater.Inc()
	require.InEpsilon(t, 13.0, rater.RateAt(timePass.Add(150*time.Millisecond)), 0.1)
	rater.Add(18)
	require.InEpsilon(t, 100.0, rater.RateAt(timePass.Add(200*time.Millisecond)), 0.1)
}

func TestFile(t *testing.T) {
	t.Parallel()
	assertTrimEqual(t, logutil.File(newFile(1)),
		`{"file": {"name": "1", "CF": "write", "sha256": "31", "startKey": "31", "endKey": "32", "startVersion": 1, "endVersion": 2, "totalKvs": 1, "totalBytes": 1, "CRC64Xor": 1}}`)
}

func TestFiles(t *testing.T) {
	t.Parallel()
	cases := []struct {
		count  int
		expect string
	}{
		{0, `{"files": {"total": 0, "files": [], "totalKVs": 0, "totalBytes": 0, "totalSize": 0}}`},
		{1, `{"files": {"total": 1, "files": ["0"], "totalKVs": 0, "totalBytes": 0, "totalSize": 0}}`},
		{2, `{"files": {"total": 2, "files": ["0", "1"], "totalKVs": 1, "totalBytes": 1, "totalSize": 1}}`},
		{3, `{"files": {"total": 3, "files": ["0", "1", "2"], "totalKVs": 3, "totalBytes": 3, "totalSize": 3}}`},
		{4, `{"files": {"total": 4, "files": ["0", "1", "2", "3"], "totalKVs": 6, "totalBytes": 6, "totalSize": 6}}`},
		{5, `{"files": {"total": 5, "files": ["0", "(skip 3)", "4"], "totalKVs": 10, "totalBytes": 10, "totalSize": 10}}`},
		{6, `{"files": {"total": 6, "files": ["0", "(skip 4)", "5"], "totalKVs": 15, "totalBytes": 15, "totalSize": 15}}`},
		{1024, `{"files": {"total": 1024, "files": ["0", "(skip 1022)", "1023"], "totalKVs": 523776, "totalBytes": 523776, "totalSize": 523776}}`},
	}

	for _, cs := range cases {
		ranges := make([]*backuppb.File, cs.count)
		for j := 0; j < cs.count; j++ {
			ranges[j] = newFile(j)
		}
		assertTrimEqual(t, logutil.Files(ranges), cs.expect)
	}
}

func TestKey(t *testing.T) {
	t.Parallel()
	encoder := zapcore.NewConsoleEncoder(zapcore.EncoderConfig{})
	out, err := encoder.EncodeEntry(zapcore.Entry{}, []zap.Field{logutil.Key("test", []byte{0, 1, 2, 3})})
	require.NoError(t, err)
	require.JSONEq(t, `{"test": "00010203"}`, strings.Trim(out.String(), "\n"))
}

func TestKeys(t *testing.T) {
	t.Parallel()
	cases := []struct {
		count  int
		expect string
	}{
		{0, `{"keys": {"total": 0, "keys": []}}`},
		{1, `{"keys": {"total": 1, "keys": ["30303030"]}}`},
		{2, `{"keys": {"total": 2, "keys": ["30303030", "30303031"]}}`},
		{3, `{"keys": {"total": 3, "keys": ["30303030", "30303031", "30303032"]}}`},
		{4, `{"keys": {"total": 4, "keys": ["30303030", "30303031", "30303032", "30303033"]}}`},
		{5, `{"keys": {"total": 5, "keys": ["30303030", "(skip 3)", "30303034"]}}`},
		{6, `{"keys": {"total": 6, "keys": ["30303030", "(skip 4)", "30303035"]}}`},
		{1024, `{"keys": {"total": 1024, "keys": ["30303030", "(skip 1022)", "31303233"]}}`},
	}

	for _, cs := range cases {
		keys := make([][]byte, cs.count)
		for j := 0; j < cs.count; j++ {
			keys[j] = []byte(fmt.Sprintf("%04d", j))
		}
		assertTrimEqual(t, logutil.Keys(keys), cs.expect)
	}
}

func TestRewriteRule(t *testing.T) {
	t.Parallel()
	rule := &import_sstpb.RewriteRule{
		OldKeyPrefix: []byte("old"),
		NewKeyPrefix: []byte("new"),
		NewTimestamp: 0x555555,
	}

	encoder := zapcore.NewConsoleEncoder(zapcore.EncoderConfig{})
	out, err := encoder.EncodeEntry(zapcore.Entry{}, []zap.Field{logutil.RewriteRule(rule)})
	require.NoError(t, err)
	require.JSONEq(t, `{"rewriteRule": {"oldKeyPrefix": "6f6c64", "newKeyPrefix": "6e6577", "newTimestamp": 5592405}}`, strings.Trim(out.String(), "\n"))
}

func TestRegion(t *testing.T) {
	t.Parallel()
	region := &metapb.Region{
		Id:          1,
		StartKey:    []byte{0x00, 0x01},
		EndKey:      []byte{0x00, 0x02},
		RegionEpoch: &metapb.RegionEpoch{ConfVer: 1, Version: 1},
		Peers:       []*metapb.Peer{{Id: 2, StoreId: 3}, {Id: 4, StoreId: 5}},
	}

	assertTrimEqual(t, logutil.Region(region),
		`{"region": {"ID": 1, "startKey": "0001", "endKey": "0002", "epoch": "conf_ver:1 version:1 ", "peers": "id:2 store_id:3 ,id:4 store_id:5 "}}`)
}

func TestLeader(t *testing.T) {
	t.Parallel()
	leader := &metapb.Peer{Id: 2, StoreId: 3}

	assertTrimEqual(t, logutil.Leader(leader), `{"leader": "id:2 store_id:3 "}`)
}

func TestSSTMeta(t *testing.T) {
	t.Parallel()
	meta := &import_sstpb.SSTMeta{
		Uuid: []byte("mock uuid"),
		Range: &import_sstpb.Range{
			Start: []byte{0x00, 0x01},
			End:   []byte{0x00, 0x02},
		},
		Crc32:       uint32(0x555555),
		Length:      1,
		CfName:      "default",
		RegionId:    1,
		RegionEpoch: &metapb.RegionEpoch{ConfVer: 1, Version: 1},
	}

	assertTrimEqual(t, logutil.SSTMeta(meta),
		`{"sstMeta": {"CF": "default", "endKeyExclusive": false, "CRC32": 5592405, "length": 1, "regionID": 1, "regionEpoch": "conf_ver:1 version:1 ", "startKey": "0001", "endKey": "0002", "UUID": "invalid UUID 6d6f636b2075756964"}}`)
}

func TestShortError(t *testing.T) {
	t.Parallel()
	err := errors.Annotate(berrors.ErrInvalidArgument, "test")

	assertTrimEqual(t, logutil.ShortError(err), `{"error": "test: [BR:Common:ErrInvalidArgument]invalid argument"}`)
}

func TestContextual(t *testing.T) {
	t.Parallel()
	testCore, logs := observer.New(zap.InfoLevel)
	logutil.ResetGlobalLogger(zap.New(testCore))

	ctx := context.Background()
	l0 := logutil.LoggerFromContext(ctx)
	l0.Info("going to take an adventure?", zap.Int("HP", 50), zap.Int("HP-MAX", 50), zap.String("character", "solte"))
	lctx := logutil.ContextWithField(ctx, zap.Strings("friends", []string{"firo", "seren", "black"}))
	l := logutil.LoggerFromContext(lctx)
	l.Info("let's go!", zap.String("character", "solte"))

	observedLogs := logs.TakeAll()
	checkLog(t, observedLogs[0],
		"going to take an adventure?", zap.Int("HP", 50), zap.Int("HP-MAX", 50), zap.String("character", "solte"))
	checkLog(t, observedLogs[1],
		"let's go!", zap.Strings("friends", []string{"firo", "seren", "black"}), zap.String("character", "solte"))
}

func checkLog(t *testing.T, actual observer.LoggedEntry, message string, fields ...zap.Field) {
	require.Equal(t, message, actual.Message)
	for i, f := range fields {
		require.Truef(t, f.Equals(actual.Context[i]), "Expected field(%+v) does not equal to actual one(%+v).", f, actual.Context[i])
	}
}
<|endoftext|>


<commit_before>package main
import (
 "fmt"
 "net/http"
 "os"
)
func handler(w http.ResponseWriter, r *http.Request) {
h, _ := os.Hostname()
fmt.Fprintf(w, "Hi there, I'm served from %s!", h)
}
func main() {
http.HandleFunc("/", handler)
http.ListenAndServe(":8484", nil)
}
<commit_msg>formating<commit_after>package main
import (
    "fmt"
    "net/http"
    "os"
)
func handler(w http.ResponseWriter, r *http.Request) {
    h, _ := os.Hostname()
    fmt.Fprintf(w, "Hi there, I'm served from %s!", h)
}
func main() {
    http.HandleFunc("/", handler)
    http.ListenAndServe(":8484", nil)
}
<|endoftext|>


<commit_before>package gads

import "encoding/xml"

// Ad type as defined
// https://developers.google.com/adwords/api/docs/reference/v201506/AdGroupAdService.Ad
type CommonAd struct {
	Type                string            `xml:"xsi:type,attr,omitempty"`
	ID                  int64             `xml:"id,omitempty"`
	URL                 string            `xml:"url,omitempty"`
	DisplayURL          string            `xml:"displayUrl,omitempty"`
	FinalURLs           []string          `xml:"finalUrls,omitempty"`
	FinalMobileURLs     []string          `xml:"finalMobileUrls,omitempty"`
	FinalAppURLs        []AppUrl          `xml:"finalAppUrls,omitempty"`
	TrackingURLTemplate *string           `xml:"trackingUrlTemplate,omitempty"`
	URLCustomParameters *CustomParameters `xml:"urlCustomParameters,omitempty"`
	DevicePreference    int64             `xml:"devicePreference,omitempty"`
}

type Ad interface {
	GetID() int64
	GetURL() string
	GetTrackingURLTemplate() *string
	GetFinalURLs() []string

	CloneForTemplate([]string, *string) Ad
}

func (c CommonAd) GetID() int64 {
	return c.ID
}

func (c CommonAd) GetURL() string {
	return c.URL
}

func (c CommonAd) GetTrackingURLTemplate() *string {
	return c.TrackingURLTemplate
}

func (c CommonAd) GetFinalURLs() []string {
	return c.FinalURLs
}

func (c CommonAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0 // value used by go for omitempty
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

func (c TextAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0 // value used by go for omitempty
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate

	return c
}

func (c ImageAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0 // value used by go for omitempty
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

func (c TemplateAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0 // value used by go for omitempty
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

func (c MobileAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0 // value used by go for omitempty
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

func (c DynamicSearchAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0 // value used by go for omitempty
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

type TextAd struct {
	CommonAd
	Headline     string `xml:"headline"`
	Description1 string `xml:"description1"`
	Description2 string `xml:"description2"`
}

type DynamicSearchAd struct {
	CommonAd
	Description1 string `xml:"description1"`
	Description2 string `xml:"description2"`
}

type ImageAd struct {
	CommonAd
	Image             int64  `xml:"imageId"` //TODO should actually be Image object, not just an int
	Name              string `xml:"name"`
	AdToCopyImageFrom int64  `xml:"adToCopyImageFrom"`
}

type MobileAd struct {
	CommonAd
	Headline        string   `xml:"headline"`
	Description     string   `xml:"description"`
	MarkupLanguages []string `xml:"markupLanguages"`
	MobileCarriers  []string `xml:"mobileCarriers"`
	BusinessName    string   `xml:"businessName"`
	CountryCode     string   `xml:"countryCode"`
	PhoneNumber     string   `xml:"phoneNumber"`
}

type TemplateAd struct {
	CommonAd
	TemplateId       int64             `xml:"templateId"`
	AdUnionId        int64             `xml:"adUnionId"`
	TemplateElements []TemplateElement `xml:"templateElements"`
	Dimensions       []Dimensions      `xml:"dimensions"`
	Name             string            `xml:"name"`
	Duration         int64             `xml:"duration"`
	originAdId       *int64            `xml:"originAdId"`
}

func (i ImageAd) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	return ERROR_NOT_YET_IMPLEMENTED
}

func (t TemplateAd) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	return ERROR_NOT_YET_IMPLEMENTED
}
<commit_msg>added some doc<commit_after>package gads

import "encoding/xml"

// Ad type as defined
// https://developers.google.com/adwords/api/docs/reference/v201506/AdGroupAdService.Ad
type CommonAd struct {
	Type                string            `xml:"xsi:type,attr,omitempty"`
	ID                  int64             `xml:"id,omitempty"`
	URL                 string            `xml:"url,omitempty"`
	DisplayURL          string            `xml:"displayUrl,omitempty"`
	FinalURLs           []string          `xml:"finalUrls,omitempty"`
	FinalMobileURLs     []string          `xml:"finalMobileUrls,omitempty"`
	FinalAppURLs        []AppUrl          `xml:"finalAppUrls,omitempty"`
	TrackingURLTemplate *string           `xml:"trackingUrlTemplate,omitempty"`
	URLCustomParameters *CustomParameters `xml:"urlCustomParameters,omitempty"`
	DevicePreference    int64             `xml:"devicePreference,omitempty"`
}

type Ad interface {
	GetID() int64
	GetURL() string
	GetTrackingURLTemplate() *string
	GetFinalURLs() []string

	CloneForTemplate([]string, *string) Ad
}

// GetID returns the Ad's id
func (c CommonAd) GetID() int64 {
	return c.ID
}

// GetURL returns the Ad's url (a.k.a destination url)
func (c CommonAd) GetURL() string {
	return c.URL
}

// GetTrackingURLTemplate returns the  tracking url template
func (c CommonAd) GetTrackingURLTemplate() *string {
	return c.TrackingURLTemplate
}

func (c CommonAd) GetFinalURLs() []string {
	return c.FinalURLs
}

// CloneForTemplate create a clone of an Ad, to recreate it for changing the tracking Url Template (as Ad are immutable)
func (c CommonAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0 // value used by go for omitempty
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

// CloneForTemplate create a clone of an Ad, to recreate it for changing the tracking Url Template
func (c TextAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0 // value used by go for omitempty
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate

	return c
}

// CloneForTemplate create a clone of an Ad, to recreate it for changing the tracking Url Template
func (c ImageAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0 // value used by go for omitempty
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

// CloneForTemplate create a clone of an Ad, to recreate it for changing the tracking Url Template
func (c TemplateAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0 // value used by go for omitempty
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

// CloneForTemplate create a clone of an Ad, to recreate it for changing the tracking Url Template
func (c MobileAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0 // value used by go for omitempty
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

// CloneForTemplate create a clone of an Ad, to recreate it for changing the tracking Url Template
func (c DynamicSearchAd) CloneForTemplate(finalURLs []string, trackingURLTemplate *string) Ad {
	c.ID = 0 // value used by go for omitempty
	c.FinalURLs = finalURLs
	c.TrackingURLTemplate = trackingURLTemplate
	return c
}

type TextAd struct {
	CommonAd
	Headline     string `xml:"headline"`
	Description1 string `xml:"description1"`
	Description2 string `xml:"description2"`
}

type DynamicSearchAd struct {
	CommonAd
	Description1 string `xml:"description1"`
	Description2 string `xml:"description2"`
}

type ImageAd struct {
	CommonAd
	Image             int64  `xml:"imageId"` //TODO should actually be Image object, not just an int
	Name              string `xml:"name"`
	AdToCopyImageFrom int64  `xml:"adToCopyImageFrom"`
}

type MobileAd struct {
	CommonAd
	Headline        string   `xml:"headline"`
	Description     string   `xml:"description"`
	MarkupLanguages []string `xml:"markupLanguages"`
	MobileCarriers  []string `xml:"mobileCarriers"`
	BusinessName    string   `xml:"businessName"`
	CountryCode     string   `xml:"countryCode"`
	PhoneNumber     string   `xml:"phoneNumber"`
}

type TemplateAd struct {
	CommonAd
	TemplateId       int64             `xml:"templateId"`
	AdUnionId        int64             `xml:"adUnionId"`
	TemplateElements []TemplateElement `xml:"templateElements"`
	Dimensions       []Dimensions      `xml:"dimensions"`
	Name             string            `xml:"name"`
	Duration         int64             `xml:"duration"`
	originAdId       *int64            `xml:"originAdId"`
}

func (i ImageAd) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	return ERROR_NOT_YET_IMPLEMENTED
}

func (t TemplateAd) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	return ERROR_NOT_YET_IMPLEMENTED
}
<|endoftext|>


<commit_before>package common

import (
	"io/ioutil"
	"os"
	"testing"

	"github.com/mitchellh/multistep"
)

func TestStepCleanVMX_impl(t *testing.T) {
	var _ multistep.Step = new(StepCleanVMX)
}

func TestStepCleanVMX(t *testing.T) {
	state := testState(t)
	step := new(StepCleanVMX)

	vmxPath := testVMXFile(t)
	defer os.Remove(vmxPath)
	state.Put("vmx_path", vmxPath)

	// Test the run
	if action := step.Run(state); action != multistep.ActionContinue {
		t.Fatalf("bad action: %#v", action)
	}
	if _, ok := state.GetOk("error"); ok {
		t.Fatal("should NOT have error")
	}
}

func TestStepCleanVMX_floppyPath(t *testing.T) {
	state := testState(t)
	step := new(StepCleanVMX)

	vmxPath := testVMXFile(t)
	defer os.Remove(vmxPath)
	if err := ioutil.WriteFile(vmxPath, []byte(testVMXFloppyPath), 0644); err != nil {
		t.Fatalf("err: %s", err)
	}

	state.Put("vmx_path", vmxPath)

	// Test the run
	if action := step.Run(state); action != multistep.ActionContinue {
		t.Fatalf("bad action: %#v", action)
	}
	if _, ok := state.GetOk("error"); ok {
		t.Fatal("should NOT have error")
	}

	// Test the resulting data
	vmxContents, err := ioutil.ReadFile(vmxPath)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	vmxData := ParseVMX(string(vmxContents))

	cases := []struct {
		Key   string
		Value string
	}{
		{"floppy0.present", "FALSE"},
		{"floppy0.filetype", ""},
		{"floppy0.filename", ""},
	}

	for _, tc := range cases {
		if tc.Value == "" {
			if _, ok := vmxData[tc.Key]; ok {
				t.Fatalf("should not have key: %s", tc.Key)
			}
		} else {
			if vmxData[tc.Key] != tc.Value {
				t.Fatalf("bad: %s %#v", tc.Key, vmxData[tc.Key])
			}
		}
	}
}

func TestStepCleanVMX_isoPath(t *testing.T) {
	state := testState(t)
	step := new(StepCleanVMX)

	vmxPath := testVMXFile(t)
	defer os.Remove(vmxPath)
	if err := ioutil.WriteFile(vmxPath, []byte(testVMXISOPath), 0644); err != nil {
		t.Fatalf("err: %s", err)
	}

	state.Put("vmx_path", vmxPath)

	// Test the run
	if action := step.Run(state); action != multistep.ActionContinue {
		t.Fatalf("bad action: %#v", action)
	}
	if _, ok := state.GetOk("error"); ok {
		t.Fatal("should NOT have error")
	}

	// Test the resulting data
	vmxContents, err := ioutil.ReadFile(vmxPath)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	vmxData := ParseVMX(string(vmxContents))

	cases := []struct {
		Key   string
		Value string
	}{
		{"ide0:0.filename", "auto detect"},
		{"ide0:0.devicetype", "cdrom-raw"},
		{"ide0:1.filename", "bar"},
		{"foo", "bar"},
	}

	for _, tc := range cases {
		if tc.Value == "" {
			if _, ok := vmxData[tc.Key]; ok {
				t.Fatalf("should not have key: %s", tc.Key)
			}
		} else {
			if vmxData[tc.Key] != tc.Value {
				t.Fatalf("bad: %s %#v", tc.Key, vmxData[tc.Key])
			}
		}
	}
}

const testVMXFloppyPath = `
floppy0.present = "TRUE"
floppy0.filetype = "file"
`

const testVMXISOPath = `
ide0:0.devicetype = "cdrom-image"
ide0:0.filename = "foo"
ide0:1.filename = "bar"
foo = "bar"
`
<commit_msg>fixed the test as well<commit_after>package common

import (
	"io/ioutil"
	"os"
	"testing"

	"github.com/mitchellh/multistep"
)

func TestStepCleanVMX_impl(t *testing.T) {
	var _ multistep.Step = new(StepCleanVMX)
}

func TestStepCleanVMX(t *testing.T) {
	state := testState(t)
	step := new(StepCleanVMX)

	vmxPath := testVMXFile(t)
	defer os.Remove(vmxPath)
	state.Put("vmx_path", vmxPath)

	// Test the run
	if action := step.Run(state); action != multistep.ActionContinue {
		t.Fatalf("bad action: %#v", action)
	}
	if _, ok := state.GetOk("error"); ok {
		t.Fatal("should NOT have error")
	}
}

func TestStepCleanVMX_floppyPath(t *testing.T) {
	state := testState(t)
	step := new(StepCleanVMX)

	vmxPath := testVMXFile(t)
	defer os.Remove(vmxPath)
	if err := ioutil.WriteFile(vmxPath, []byte(testVMXFloppyPath), 0644); err != nil {
		t.Fatalf("err: %s", err)
	}

	state.Put("vmx_path", vmxPath)

	// Test the run
	if action := step.Run(state); action != multistep.ActionContinue {
		t.Fatalf("bad action: %#v", action)
	}
	if _, ok := state.GetOk("error"); ok {
		t.Fatal("should NOT have error")
	}

	// Test the resulting data
	vmxContents, err := ioutil.ReadFile(vmxPath)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	vmxData := ParseVMX(string(vmxContents))

	cases := []struct {
		Key   string
		Value string
	}{
		{"floppy0.present", "FALSE"},
		{"floppy0.fileType", ""},
		{"floppy0.fileName", ""},
	}

	for _, tc := range cases {
		if tc.Value == "" {
			if _, ok := vmxData[tc.Key]; ok {
				t.Fatalf("should not have key: %s", tc.Key)
			}
		} else {
			if vmxData[tc.Key] != tc.Value {
				t.Fatalf("bad: %s %#v", tc.Key, vmxData[tc.Key])
			}
		}
	}
}

func TestStepCleanVMX_isoPath(t *testing.T) {
	state := testState(t)
	step := new(StepCleanVMX)

	vmxPath := testVMXFile(t)
	defer os.Remove(vmxPath)
	if err := ioutil.WriteFile(vmxPath, []byte(testVMXISOPath), 0644); err != nil {
		t.Fatalf("err: %s", err)
	}

	state.Put("vmx_path", vmxPath)

	// Test the run
	if action := step.Run(state); action != multistep.ActionContinue {
		t.Fatalf("bad action: %#v", action)
	}
	if _, ok := state.GetOk("error"); ok {
		t.Fatal("should NOT have error")
	}

	// Test the resulting data
	vmxContents, err := ioutil.ReadFile(vmxPath)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	vmxData := ParseVMX(string(vmxContents))

	cases := []struct {
		Key   string
		Value string
	}{
		{"ide0:0.fileName", "auto detect"},
		{"ide0:0.deviceType", "cdrom-raw"},
		{"ide0:1.fileName", "bar"},
		{"foo", "bar"},
	}

	for _, tc := range cases {
		if tc.Value == "" {
			if _, ok := vmxData[tc.Key]; ok {
				t.Fatalf("should not have key: %s", tc.Key)
			}
		} else {
			if vmxData[tc.Key] != tc.Value {
				t.Fatalf("bad: %s %#v", tc.Key, vmxData[tc.Key])
			}
		}
	}
}

const testVMXFloppyPath = `
floppy0.present = "TRUE"
floppy0.fileType = "file"
`

const testVMXISOPath = `
ide0:0.deviceType = "cdrom-image"
ide0:0.fileName = "foo"
ide0:1.fileName = "bar"
foo = "bar"
`
<|endoftext|>


<commit_before>// Boringstreamer looks for mp3 files and broadcasts via http (live streaming.)
//
// $ boringstreamer
//
// or
//
// c:\>boringstreamer.exe
//
// recursively looks for .mp3 files starting from current working directory and broadcasts on port 4444 for at most 42 concurrent http clients.
//
// See -h for details.
//
// Browse to listen (e.g. http://localhost:4444/)
package main

import (
	"bytes"
	"bufio"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/tcolgate/mp3"
)

var (
	addr           = flag.String("addr", ":4444", "listen on address (format: :port or host:port)")
	maxConnections = flag.Int("max", 42, "set maximum number of streaming connections")
	recursively    = flag.Bool("r", true, "recursively look for music starting from path")
	verbose        = flag.Bool("v", false, "display verbose messages")
)

var debugging bool	// controlled by hidden command line argument -debug

// like /dev/null
type nullWriter struct {
}

func (nw nullWriter) Write(p []byte) (n int, err error) {
	return len(p), nil
}

type streamFrame []byte

// client's event
type broadcastResult struct {
	qid int
	err error
}

// After a start() mux broadcasts audio stream to subscribed clients (ie. to http servers).
// Clients subscribe() and unsubscribe by writing to result chanel.
type mux struct {
	sync.Mutex

	clients map[int]chan streamFrame // set of listener clients to be notified
	result  chan broadcastResult     // clients share broadcast success-failure here
}

// subscribe(ch) adds ch to the set of channels to be received on by the clients when a new audio frame is available.
// Returns uniq client id (qid) for ch and a broadcast result channel for the client.
// Returns -1, nil if too many clients are already listening.
// clients: qid, br := m.subscribe(ch)
func (m *mux) subscribe(ch chan streamFrame) (int, chan broadcastResult) {
	m.Lock()
	// search for available qid
	qid := 0
	_, ok := m.clients[qid]
	for ; ok; _, ok = m.clients[qid] {
		if qid >= *maxConnections-1 {
			m.Unlock()
			return -1, nil
		}
		qid++
	}
	m.clients[qid] = ch
	m.Unlock()
	if *verbose {
		fmt.Printf("New connection (qid: %v), streaming to %v connections, at %v\n", qid, len(m.clients), time.Now().Format(time.Stamp))
	}

	return qid, m.result
}

// start() initializes a multiplexer for raw audio streams
// e.g: m := new(mux).start(path)
func (m *mux) start(path string) *mux {
	m.result = make(chan broadcastResult)
	m.clients = make(map[int]chan streamFrame)

	// flow structure: fs -> nextFile -> nextStream -> nextFrame -> subscribed http servers -> browsers
	nextFile := make(chan string)			// next file to be broadcast
	nextStream := make(chan io.Reader)		// next raw audio stream
	nextFrame := make(chan streamFrame)	// next audio frame

	// generate randomized list of files available from path
	rand.Seed(time.Now().Unix()) // minimal randomness
	rescan := make(chan chan string)
	go func() {
		if path == "-" {
			return
		}

		for {
			files := <-rescan
			filepath.Walk(path, func(wpath string, info os.FileInfo, err error) error {
				if err != nil {
					return nil
				}
				if !info.Mode().IsRegular() {
					return nil
				}
				probably := strings.HasSuffix(strings.ToLower(info.Name()), ".mp3") // probably mp3
				if !info.IsDir() && !probably {
					return nil
				}
				files <- wpath // found file

				return nil
			})
			close(files)
			time.Sleep(1 * time.Second) // if no files are found, poll at least with 1Hz
		}
	}()

	// buffer and shuffle
	go func() {
		if path == "-" {
			return
		}

		for {
			files := make(chan string)
			rescan <- files

			shuffled := make([]string, 0) // randomized set of files

			for f := range files {
				select {
				case <- time.After(100*time.Millisecond):	// start playing as soon as possible, but wait at least 0.1 second for shuffling
					nextFile <- f
					if *verbose {
						fmt.Printf("Next: %v\n", f)
					}
				default:
					// shuffle files for random playback
					// (random permutation)
					if len(shuffled) == 0 {
						shuffled = append(shuffled, f)
					} else {
						i := rand.Intn(len(shuffled))
						shuffled = append(shuffled, shuffled[i])
						shuffled[i] = f
					}
				}
			}

			// queue shuffled files
			for _, f := range shuffled {
				nextFile <- f
				if *verbose {
					fmt.Printf("Next: %v\n", f)
				}
			}
		}
	}()

	// open file
	go func() {
		if path == "-" {
			nextStream <- os.Stdin
			return
		}

		for {
			filename := <-nextFile
			f, err := os.Open(filename)
			if err != nil {
				if debugging {
					log.Printf("Skipped \"%v\", err=%v", filename, err)
				}
				continue
			}
			nextStream <- bufio.NewReaderSize(f, 1024*1024)
			if *verbose {
				fmt.Printf("Now playing: %v\n", filename)
			}
		}
	}()

	// decode stream to frames and delay for frame duration
	go func() {
		nullwriter := new(nullWriter)
		var cumwait time.Duration
		for {
			streamReader := <- nextStream
			d := mp3.NewDecoder(streamReader)
			var f mp3.Frame
			for {
				t0 := time.Now()
				tmp := log.Prefix()
				if !debugging {
					log.SetOutput(nullwriter) // hack to silence mp3 debug/log output
				} else {
					log.SetPrefix("info: mp3 decode msg: ")
				}
				err := d.Decode(&f)
				log.SetPrefix(tmp)
				if !debugging {
					log.SetOutput(os.Stderr)
				}
				if err == io.EOF {
					break
				}
				if err != nil {
					if debugging {
						log.Printf("Skipping frame, d.Decode() err=%v", err)
					}
					continue
				}
				buf, err := ioutil.ReadAll(f.Reader())
				if err != nil {
					if debugging {
						log.Printf("Skipping frame, ioutil.ReadAll() err=%v", err)
					}
					continue
				}
				nextFrame <- buf

				towait := f.Duration() - time.Now().Sub(t0)
				cumwait += towait // towait can be negative -> cumwait
				if cumwait > 1*time.Second {
					time.Sleep(cumwait)
					cumwait = 0
				}
			}
		}
	}()

	// broadcast frame to clients
	go func() {
		for {
			f := <-nextFrame
			// notify clients of new audio frame or let them quit
			m.Lock()
			for _, ch := range m.clients {
				m.Unlock()
				ch <- f
				br := <-m.result // handle quitting clients
				if br.err != nil {
					m.Lock()
					close(m.clients[br.qid])
					delete(m.clients, br.qid)
					nclients := len(m.clients)
					m.Unlock()
					if debugging {
						log.Printf("Connection exited, qid: %v, error %v. Now streaming to %v connections.", br.qid, br.err, nclients)
					} else if *verbose {
						fmt.Printf("Connection exited, qid: %v. Now streaming to %v connections, at %v\n", br.qid, nclients, time.Now().Format(time.Stamp))
					}
				}
				m.Lock()
			}
			m.Unlock()
		}
	}()

	return m
}

type streamHandler struct {
	stream *mux
}

// chrome and firefox play mp3 audio stream directly
func (sh streamHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	now := time.Now().UTC()
	frames := make(chan streamFrame)
	qid, br := sh.stream.subscribe(frames)
	if qid < 0 {
		log.Printf("Error: new connection request denied, already serving %v connections. See -h for details.", *maxConnections)
		w.WriteHeader(http.StatusTooManyRequests)
		return
	}

	w.WriteHeader(http.StatusOK)
	w.Header().Set("Date", now.Format(http.TimeFormat))
	w.Header().Set("Connection", "Keep-Alive")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Content-Type", "audio/mpeg")
	w.Header().Set("Server", "BoringStreamer/4.0")

	// some browsers need ID3 tag to identify first frame as audio media to be played
	// minimal ID3 header to designate audio stream
	b := []byte{0x49, 0x44, 0x33, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
	_, err := io.Copy(w, bytes.NewReader(b))
	if err == nil {
		// broadcast mp3 stream to w
		broadcastTimeout := 4 * time.Second // timeout for slow clients
		result := make(chan error)
		for {
			buf := <-frames

			go func(r chan error, b []byte) {
				_, err = io.Copy(w, bytes.NewReader(b))
				r <- err
			}(result, buf)

			select {
			case err = <-result:
				if err != nil {
					break
				}
				br <- broadcastResult{qid, nil} // frame streamed, no error, send ack
			case <-time.After(broadcastTimeout): // it's an error if io.Copy() is not finished within broadcastTimeout, ServeHTTP should exit
				err = errors.New(fmt.Sprintf("timeout: %v", broadcastTimeout))
			}

			if err != nil {
				break
			}
		}
	}
	br <- broadcastResult{qid, err} // error, send nack
}

func main() {
	flag.Usage = func() {
		fmt.Printf("Usage: %s [flags] [path]\n", os.Args[0])
		fmt.Printf("Streaming from standard input: %v -\n", os.Args[0])
		fmt.Printf("Browse to listen (e.g. http://localhost:4444/)\n\nflags:\n")
		flag.PrintDefaults()
	}
	flag.Parse()
	if len(flag.Args()) > 1 && flag.Args()[1] != "-debug" {
		flag.Usage()
		os.Exit(1)
	}

	path := "."
	switch len(flag.Args()) {
	case 0:
		if *verbose {
			fmt.Printf("Using path %#v, see -h for details.\n", path)
		}
	case 1:
		path = flag.Args()[0]
	case 2:
		path = flag.Args()[0]
		debugging = true
	}

	// check if path is available
	if path != "-" {
		if *verbose {
			fmt.Printf("Looking for files available from \"%v\" ...\n", path)
		}

		matches, err := filepath.Glob(path)
		if err != nil || len(matches) != 1 {
			fmt.Fprintf(os.Stderr, "Error: \"%v\" unavailable, nothing to play.\n", path)
			os.Exit(1)
		}
	}

	// initialize and start mp3 streamer
	http.Handle("/", streamHandler{new(mux).start(path)})
	if *verbose {
		fmt.Printf("Waiting for connections on %v\n", *addr)
	}

	err := http.ListenAndServe(*addr, nil)
	if err != nil {
		log.Fatalf("Exiting, error: %v", err)
	}
}
<commit_msg>fix bug, log.Fatalf() race with log.SetPrefix()<commit_after>// Boringstreamer looks for mp3 files and broadcasts via http (live streaming.)
//
// $ boringstreamer
//
// or
//
// c:\>boringstreamer.exe
//
// recursively looks for .mp3 files starting from current working directory and broadcasts on port 4444 for at most 42 concurrent http clients.
//
// See -h for details.
//
// Browse to listen (e.g. http://localhost:4444/)
package main

import (
	"bytes"
	"bufio"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/tcolgate/mp3"
)

var (
	addr           = flag.String("addr", ":4444", "listen on address (format: :port or host:port)")
	maxConnections = flag.Int("max", 42, "set maximum number of streaming connections")
	recursively    = flag.Bool("r", true, "recursively look for music starting from path")
	verbose        = flag.Bool("v", false, "display verbose messages")
)

var debugging bool	// controlled by hidden command line argument -debug

// like /dev/null
type nullWriter struct {
}

func (nw nullWriter) Write(p []byte) (n int, err error) {
	return len(p), nil
}

type streamFrame []byte

// client's event
type broadcastResult struct {
	qid int
	err error
}

// After a start() mux broadcasts audio stream to subscribed clients (ie. to http servers).
// Clients subscribe() and unsubscribe by writing to result chanel.
type mux struct {
	sync.Mutex

	clients map[int]chan streamFrame // set of listener clients to be notified
	result  chan broadcastResult     // clients share broadcast success-failure here
}

// subscribe(ch) adds ch to the set of channels to be received on by the clients when a new audio frame is available.
// Returns uniq client id (qid) for ch and a broadcast result channel for the client.
// Returns -1, nil if too many clients are already listening.
// clients: qid, br := m.subscribe(ch)
func (m *mux) subscribe(ch chan streamFrame) (int, chan broadcastResult) {
	m.Lock()
	// search for available qid
	qid := 0
	_, ok := m.clients[qid]
	for ; ok; _, ok = m.clients[qid] {
		if qid >= *maxConnections-1 {
			m.Unlock()
			return -1, nil
		}
		qid++
	}
	m.clients[qid] = ch
	m.Unlock()
	if *verbose {
		fmt.Printf("New connection (qid: %v), streaming to %v connections, at %v\n", qid, len(m.clients), time.Now().Format(time.Stamp))
	}

	return qid, m.result
}

// start() initializes a multiplexer for raw audio streams
// e.g: m := new(mux).start(path)
func (m *mux) start(path string) *mux {
	m.result = make(chan broadcastResult)
	m.clients = make(map[int]chan streamFrame)

	// flow structure: fs -> nextFile -> nextStream -> nextFrame -> subscribed http servers -> browsers
	nextFile := make(chan string)			// next file to be broadcast
	nextStream := make(chan io.Reader)		// next raw audio stream
	nextFrame := make(chan streamFrame)	// next audio frame

	// generate randomized list of files available from path
	rand.Seed(time.Now().Unix()) // minimal randomness
	rescan := make(chan chan string)
	go func() {
		if path == "-" {
			return
		}

		for {
			files := <-rescan
			filepath.Walk(path, func(wpath string, info os.FileInfo, err error) error {
				if err != nil {
					return nil
				}
				if !info.Mode().IsRegular() {
					return nil
				}
				probably := strings.HasSuffix(strings.ToLower(info.Name()), ".mp3") // probably mp3
				if !info.IsDir() && !probably {
					return nil
				}
				files <- wpath // found file

				return nil
			})
			close(files)
			time.Sleep(1 * time.Second) // if no files are found, poll at least with 1Hz
		}
	}()

	// buffer and shuffle
	go func() {
		if path == "-" {
			return
		}

		for {
			files := make(chan string)
			rescan <- files

			shuffled := make([]string, 0) // randomized set of files

			for f := range files {
				select {
				case <- time.After(100*time.Millisecond):	// start playing as soon as possible, but wait at least 0.1 second for shuffling
					nextFile <- f
					if *verbose {
						fmt.Printf("Next: %v\n", f)
					}
				default:
					// shuffle files for random playback
					// (random permutation)
					if len(shuffled) == 0 {
						shuffled = append(shuffled, f)
					} else {
						i := rand.Intn(len(shuffled))
						shuffled = append(shuffled, shuffled[i])
						shuffled[i] = f
					}
				}
			}

			// queue shuffled files
			for _, f := range shuffled {
				nextFile <- f
				if *verbose {
					fmt.Printf("Next: %v\n", f)
				}
			}
		}
	}()

	// open file
	go func() {
		if path == "-" {
			nextStream <- os.Stdin
			return
		}

		for {
			filename := <-nextFile
			f, err := os.Open(filename)
			if err != nil {
				if debugging {
					log.Printf("Skipped \"%v\", err=%v", filename, err)
				}
				continue
			}
			nextStream <- bufio.NewReaderSize(f, 1024*1024)
			if *verbose {
				fmt.Printf("Now playing: %v\n", filename)
			}
		}
	}()

	// decode stream to frames and delay for frame duration
	go func() {
		nullwriter := new(nullWriter)
		var cumwait time.Duration
		for {
			streamReader := <- nextStream
			d := mp3.NewDecoder(streamReader)
			var f mp3.Frame
			for {
				t0 := time.Now()
				tmp := log.Prefix()
				if !debugging {
					log.SetOutput(nullwriter) // hack to silence mp3 debug/log output
				} else {
					log.SetPrefix("info: mp3 decode msg: ")
				}
				err := d.Decode(&f)
				log.SetPrefix(tmp)
				if !debugging {
					log.SetOutput(os.Stderr)
				}
				if err == io.EOF {
					break
				}
				if err != nil {
					if debugging {
						log.Printf("Skipping frame, d.Decode() err=%v", err)
					}
					continue
				}
				buf, err := ioutil.ReadAll(f.Reader())
				if err != nil {
					if debugging {
						log.Printf("Skipping frame, ioutil.ReadAll() err=%v", err)
					}
					continue
				}
				nextFrame <- buf

				towait := f.Duration() - time.Now().Sub(t0)
				cumwait += towait // towait can be negative -> cumwait
				if cumwait > 1*time.Second {
					time.Sleep(cumwait)
					cumwait = 0
				}
			}
		}
	}()

	// broadcast frame to clients
	go func() {
		for {
			f := <-nextFrame
			// notify clients of new audio frame or let them quit
			m.Lock()
			for _, ch := range m.clients {
				m.Unlock()
				ch <- f
				br := <-m.result // handle quitting clients
				if br.err != nil {
					m.Lock()
					close(m.clients[br.qid])
					delete(m.clients, br.qid)
					nclients := len(m.clients)
					m.Unlock()
					if debugging {
						log.Printf("Connection exited, qid: %v, error %v. Now streaming to %v connections.", br.qid, br.err, nclients)
					} else if *verbose {
						fmt.Printf("Connection exited, qid: %v. Now streaming to %v connections, at %v\n", br.qid, nclients, time.Now().Format(time.Stamp))
					}
				}
				m.Lock()
			}
			m.Unlock()
		}
	}()

	return m
}

type streamHandler struct {
	stream *mux
}

// chrome and firefox play mp3 audio stream directly
func (sh streamHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	now := time.Now().UTC()
	frames := make(chan streamFrame)
	qid, br := sh.stream.subscribe(frames)
	if qid < 0 {
		log.Printf("Error: new connection request denied, already serving %v connections. See -h for details.", *maxConnections)
		w.WriteHeader(http.StatusTooManyRequests)
		return
	}

	w.WriteHeader(http.StatusOK)
	w.Header().Set("Date", now.Format(http.TimeFormat))
	w.Header().Set("Connection", "Keep-Alive")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Content-Type", "audio/mpeg")
	w.Header().Set("Server", "BoringStreamer/4.0")

	// some browsers need ID3 tag to identify first frame as audio media to be played
	// minimal ID3 header to designate audio stream
	b := []byte{0x49, 0x44, 0x33, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
	_, err := io.Copy(w, bytes.NewReader(b))
	if err == nil {
		// broadcast mp3 stream to w
		broadcastTimeout := 4 * time.Second // timeout for slow clients
		result := make(chan error)
		for {
			buf := <-frames

			go func(r chan error, b []byte) {
				_, err = io.Copy(w, bytes.NewReader(b))
				r <- err
			}(result, buf)

			select {
			case err = <-result:
				if err != nil {
					break
				}
				br <- broadcastResult{qid, nil} // frame streamed, no error, send ack
			case <-time.After(broadcastTimeout): // it's an error if io.Copy() is not finished within broadcastTimeout, ServeHTTP should exit
				err = errors.New(fmt.Sprintf("timeout: %v", broadcastTimeout))
			}

			if err != nil {
				break
			}
		}
	}
	br <- broadcastResult{qid, err} // error, send nack
}

func main() {
	flag.Usage = func() {
		fmt.Printf("Usage: %s [flags] [path]\n", os.Args[0])
		fmt.Printf("Streaming from standard input: %v -\n", os.Args[0])
		fmt.Printf("Browse to listen (e.g. http://localhost:4444/)\n\nflags:\n")
		flag.PrintDefaults()
	}
	flag.Parse()
	if len(flag.Args()) > 1 && flag.Args()[1] != "-debug" {
		flag.Usage()
		os.Exit(1)
	}

	path := "."
	switch len(flag.Args()) {
	case 0:
		if *verbose {
			fmt.Printf("Using path %#v, see -h for details.\n", path)
		}
	case 1:
		path = flag.Args()[0]
	case 2:
		path = flag.Args()[0]
		debugging = true
	}

	// check if path is available
	if path != "-" {
		if *verbose {
			fmt.Printf("Looking for files available from \"%v\" ...\n", path)
		}

		matches, err := filepath.Glob(path)
		if err != nil || len(matches) != 1 {
			fmt.Fprintf(os.Stderr, "Error: \"%v\" unavailable, nothing to play.\n", path)
			os.Exit(1)
		}
	}

	// initialize and start mp3 streamer
	http.Handle("/", streamHandler{new(mux).start(path)})
	if *verbose {
		fmt.Printf("Waiting for connections on %v\n", *addr)
	}

	err := http.ListenAndServe(*addr, nil)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Exiting, error: %v", err)	// log.Fatalf() race with log.SetPrefix()
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package metadata

import (
	"bytes"

	"github.com/BurntSushi/toml"
)

// TOMLParser is the Parser for TOML
type TOMLParser struct {
	metadata Metadata
	markdown *bytes.Buffer
}

// Type returns the kind of parser this struct is.
func (t *TOMLParser) Type() string {
	return "TOML"
}

// Init prepares and parses the metadata and markdown file itself
func (t *TOMLParser) Init(b *bytes.Buffer) bool {
	meta, data := splitBuffer(b, "+++")
	if meta == nil || data == nil {
		return false
	}
	t.markdown = data

	m := make(map[string]interface{})
	if err := toml.Unmarshal(meta.Bytes(), &m); err != nil {
		return false
	}
	t.metadata = NewMetadata(m)

	return true
}

// Metadata returns parsed metadata.  It should be called
// only after a call to Parse returns without error.
func (t *TOMLParser) Metadata() Metadata {
	return t.metadata
}

// Markdown returns parser markdown.  It should be called only after a call to Parse returns without error.
func (t *TOMLParser) Markdown() []byte {
	return t.markdown.Bytes()
}
<commit_msg>Use naoina/toml instead of BurntSushi/toml<commit_after>package metadata

import (
	"bytes"

	"github.com/naoina/toml"
)

// TOMLParser is the Parser for TOML
type TOMLParser struct {
	metadata Metadata
	markdown *bytes.Buffer
}

// Type returns the kind of parser this struct is.
func (t *TOMLParser) Type() string {
	return "TOML"
}

// Init prepares and parses the metadata and markdown file itself
func (t *TOMLParser) Init(b *bytes.Buffer) bool {
	meta, data := splitBuffer(b, "+++")
	if meta == nil || data == nil {
		return false
	}
	t.markdown = data

	m := make(map[string]interface{})
	if err := toml.Unmarshal(meta.Bytes(), &m); err != nil {
		return false
	}
	t.metadata = NewMetadata(m)

	return true
}

// Metadata returns parsed metadata.  It should be called
// only after a call to Parse returns without error.
func (t *TOMLParser) Metadata() Metadata {
	return t.metadata
}

// Markdown returns parser markdown.  It should be called only after a call to Parse returns without error.
func (t *TOMLParser) Markdown() []byte {
	return t.markdown.Bytes()
}
<|endoftext|>