<commit_before>package dynamo

import (
	"encoding/binary"
	"fmt"
	"math"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface"
	"github.com/pkg/errors"

	"github.com/yuuki/diamondb/lib/config"
	"github.com/yuuki/diamondb/lib/mathutil"
	"github.com/yuuki/diamondb/lib/series"
	"github.com/yuuki/diamondb/lib/util"
)

// DynamoDB provides a dynamodb client.
type DynamoDB struct {
	svc         dynamodbiface.DynamoDBAPI
	tablePrefix string
}

type timeSlot struct {
	tableName string
	itemEpoch int64
}

type query struct {
	names []string
	start time.Time
	end   time.Time
	slot  *timeSlot
	step  int
	// context
}

const (
	oneYear time.Duration = time.Duration(24*360) * time.Hour
	oneWeek time.Duration = time.Duration(24*7) * time.Hour
	oneDay  time.Duration = time.Duration(24*1) * time.Hour
)

var (
	dynamodbBatchLimit = 100

	oneYearSeconds = int(oneYear.Seconds())
	oneWeekSeconds = int(oneWeek.Seconds())
	oneDaySeconds  = int(oneDay.Seconds())
)

// NewDynamoDB creates a new DynamoDB.
func NewDynamoDB() *DynamoDB {
	return &DynamoDB{
		svc: dynamodb.New(
			session.New(),
			aws.NewConfig().WithRegion(config.Config.DynamoDBRegion),
		),
		tablePrefix: config.Config.DynamoDBTablePrefix,
	}
}

// Ping pings DynamoDB endpoint.
func (d *DynamoDB) Ping() error {
	var params *dynamodb.DescribeLimitsInput
	_, err := d.svc.DescribeLimits(params)
	if err != nil {
		return errors.WithStack(err)
	}
	return nil
}

// Fetch fetches datapoints by name from start until end.
func (d *DynamoDB) Fetch(name string, start, end time.Time) (series.SeriesMap, error) {
	slots, step := selectTimeSlots(start, end, d.tablePrefix)
	nameGroups := util.GroupNames(util.SplitName(name), dynamodbBatchLimit)
	numQueries := len(slots) * len(nameGroups)

	type result struct {
		value series.SeriesMap
		err   error
	}
	c := make(chan *result, numQueries)
	for _, slot := range slots {
		for _, names := range nameGroups {
			q := &query{
				names: names,
				start: start,
				end:   end,
				slot:  slot,
				step:  step,
			}
			go func() {
				sm, err := d.batchGet(q)
				c <- &result{value: sm, err: err}
			}()
		}
	}
	sm := make(series.SeriesMap, len(nameGroups))
	for i := 0; i < numQueries; i++ {
		ret := <-c
		if ret.err != nil {
			return nil, errors.WithStack(ret.err)
		}
		sm.MergePointsToMap(ret.value)
	}
	return sm, nil
}

func batchGetResultToMap(resp *dynamodb.BatchGetItemOutput, q *query) series.SeriesMap {
	sm := make(series.SeriesMap, len(resp.Responses))
	for _, xs := range resp.Responses {
		for _, x := range xs {
			name := (*x["MetricName"].S)
			points := make(series.DataPoints, 0, len(x["Values"].BS))
			for _, y := range x["Values"].BS {
				t := int64(binary.BigEndian.Uint64(y[0:8]))
				v := math.Float64frombits(binary.BigEndian.Uint64(y[8:]))
				points = append(points, series.NewDataPoint(t, v))
			}
			sm[name] = series.NewSeriesPoint(name, points, q.step)
		}
	}
	return sm
}

func (d *DynamoDB) batchGet(q *query) (series.SeriesMap, error) {
	var keys []map[string]*dynamodb.AttributeValue
	for _, name := range q.names {
		keys = append(keys, map[string]*dynamodb.AttributeValue{
			"MetricName": {S: aws.String(name)},
			"Timestamp":  {N: aws.String(fmt.Sprintf("%d", q.slot.itemEpoch))},
		})
	}
	items := make(map[string]*dynamodb.KeysAndAttributes)
	items[q.slot.tableName] = &dynamodb.KeysAndAttributes{Keys: keys}
	params := &dynamodb.BatchGetItemInput{
		RequestItems:           items,
		ReturnConsumedCapacity: aws.String("NONE"),
	}
	resp, err := d.svc.BatchGetItem(params)
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceNotFoundException" {
				// Don't handle ResourceNotFoundException as error
				// bacause diamondb web return length 0 series as 200.
				return series.SeriesMap{}, nil
			}
		}
		return nil, errors.Wrapf(err, "Failed to BatchGetItem %s %d %s %d",
			q.slot.tableName, q.slot.itemEpoch, strings.Join(q.names, ","), q.step,
		)
	}
	return batchGetResultToMap(resp, q), nil
}

func selectTimeSlots(startTime, endTime time.Time, tablePrefix string) ([]*timeSlot, int) {
	var (
		tableName      string
		step           int
		tableEpochStep int
		itemEpochStep  int
	)
	diffTime := endTime.Sub(startTime)
	if oneYear <= diffTime {
		tableName = tablePrefix + "-1d360d"
		tableEpochStep = oneYearSeconds
		itemEpochStep = tableEpochStep
		step = 60 * 60 * 24
	} else if oneWeek <= diffTime {
		tableName = tablePrefix + "-1h7d"
		tableEpochStep = 60 * 60 * 24 * 7
		itemEpochStep = tableEpochStep
		step = 60 * 60
	} else if oneDay <= diffTime {
		tableName = tablePrefix + "-5m1d"
		tableEpochStep = 60 * 60 * 24
		itemEpochStep = tableEpochStep
		step = 5 * 60
	} else {
		tableName = tablePrefix + "-1m1h"
		tableEpochStep = 60 * 60 * 24
		itemEpochStep = 60 * 60
		step = 60
	}

	slots := make([]*timeSlot, 0, 5)
	startTableEpoch := startTime.Unix() - startTime.Unix()%int64(tableEpochStep)
	endTableEpoch := endTime.Unix()
	for tableEpoch := startTableEpoch; tableEpoch < endTableEpoch; tableEpoch += int64(tableEpochStep) {
		startItemEpoch := mathutil.MaxInt64(tableEpoch, startTime.Unix()-startTime.Unix()%int64(itemEpochStep))
		endItemEpoch := mathutil.MinInt64(tableEpoch+int64(tableEpochStep), endTime.Unix())
		for itemEpoch := startItemEpoch; itemEpoch < endItemEpoch; itemEpoch += int64(itemEpochStep) {
			slot := timeSlot{
				tableName: fmt.Sprintf("%s-%d", tableName, tableEpoch),
				itemEpoch: itemEpoch,
			}
			slots = append(slots, &slot)
		}
	}

	return slots, step
}
<commit_msg>Avoid the common goroutine mistake just in case<commit_after>package dynamo

import (
	"encoding/binary"
	"fmt"
	"math"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface"
	"github.com/pkg/errors"

	"github.com/yuuki/diamondb/lib/config"
	"github.com/yuuki/diamondb/lib/mathutil"
	"github.com/yuuki/diamondb/lib/series"
	"github.com/yuuki/diamondb/lib/util"
)

// DynamoDB provides a dynamodb client.
type DynamoDB struct {
	svc         dynamodbiface.DynamoDBAPI
	tablePrefix string
}

type timeSlot struct {
	tableName string
	itemEpoch int64
}

type query struct {
	names []string
	start time.Time
	end   time.Time
	slot  *timeSlot
	step  int
	// context
}

const (
	oneYear time.Duration = time.Duration(24*360) * time.Hour
	oneWeek time.Duration = time.Duration(24*7) * time.Hour
	oneDay  time.Duration = time.Duration(24*1) * time.Hour
)

var (
	dynamodbBatchLimit = 100

	oneYearSeconds = int(oneYear.Seconds())
	oneWeekSeconds = int(oneWeek.Seconds())
	oneDaySeconds  = int(oneDay.Seconds())
)

// NewDynamoDB creates a new DynamoDB.
func NewDynamoDB() *DynamoDB {
	return &DynamoDB{
		svc: dynamodb.New(
			session.New(),
			aws.NewConfig().WithRegion(config.Config.DynamoDBRegion),
		),
		tablePrefix: config.Config.DynamoDBTablePrefix,
	}
}

// Ping pings DynamoDB endpoint.
func (d *DynamoDB) Ping() error {
	var params *dynamodb.DescribeLimitsInput
	_, err := d.svc.DescribeLimits(params)
	if err != nil {
		return errors.WithStack(err)
	}
	return nil
}

// Fetch fetches datapoints by name from start until end.
func (d *DynamoDB) Fetch(name string, start, end time.Time) (series.SeriesMap, error) {
	slots, step := selectTimeSlots(start, end, d.tablePrefix)
	nameGroups := util.GroupNames(util.SplitName(name), dynamodbBatchLimit)
	numQueries := len(slots) * len(nameGroups)

	type result struct {
		value series.SeriesMap
		err   error
	}
	c := make(chan *result, numQueries)
	for _, slot := range slots {
		for _, names := range nameGroups {
			q := &query{
				names: names,
				start: start,
				end:   end,
				slot:  slot,
				step:  step,
			}
			go func(q *query) {
				sm, err := d.batchGet(q)
				c <- &result{value: sm, err: err}
			}(q)
		}
	}
	sm := make(series.SeriesMap, len(nameGroups))
	for i := 0; i < numQueries; i++ {
		ret := <-c
		if ret.err != nil {
			return nil, errors.WithStack(ret.err)
		}
		sm.MergePointsToMap(ret.value)
	}
	return sm, nil
}

func batchGetResultToMap(resp *dynamodb.BatchGetItemOutput, q *query) series.SeriesMap {
	sm := make(series.SeriesMap, len(resp.Responses))
	for _, xs := range resp.Responses {
		for _, x := range xs {
			name := (*x["MetricName"].S)
			points := make(series.DataPoints, 0, len(x["Values"].BS))
			for _, y := range x["Values"].BS {
				t := int64(binary.BigEndian.Uint64(y[0:8]))
				v := math.Float64frombits(binary.BigEndian.Uint64(y[8:]))
				points = append(points, series.NewDataPoint(t, v))
			}
			sm[name] = series.NewSeriesPoint(name, points, q.step)
		}
	}
	return sm
}

func (d *DynamoDB) batchGet(q *query) (series.SeriesMap, error) {
	var keys []map[string]*dynamodb.AttributeValue
	for _, name := range q.names {
		keys = append(keys, map[string]*dynamodb.AttributeValue{
			"MetricName": {S: aws.String(name)},
			"Timestamp":  {N: aws.String(fmt.Sprintf("%d", q.slot.itemEpoch))},
		})
	}
	items := make(map[string]*dynamodb.KeysAndAttributes)
	items[q.slot.tableName] = &dynamodb.KeysAndAttributes{Keys: keys}
	params := &dynamodb.BatchGetItemInput{
		RequestItems:           items,
		ReturnConsumedCapacity: aws.String("NONE"),
	}
	resp, err := d.svc.BatchGetItem(params)
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceNotFoundException" {
				// Don't handle ResourceNotFoundException as error
				// bacause diamondb web return length 0 series as 200.
				return series.SeriesMap{}, nil
			}
		}
		return nil, errors.Wrapf(err, "Failed to BatchGetItem %s %d %s %d",
			q.slot.tableName, q.slot.itemEpoch, strings.Join(q.names, ","), q.step,
		)
	}
	return batchGetResultToMap(resp, q), nil
}

func selectTimeSlots(startTime, endTime time.Time, tablePrefix string) ([]*timeSlot, int) {
	var (
		tableName      string
		step           int
		tableEpochStep int
		itemEpochStep  int
	)
	diffTime := endTime.Sub(startTime)
	if oneYear <= diffTime {
		tableName = tablePrefix + "-1d360d"
		tableEpochStep = oneYearSeconds
		itemEpochStep = tableEpochStep
		step = 60 * 60 * 24
	} else if oneWeek <= diffTime {
		tableName = tablePrefix + "-1h7d"
		tableEpochStep = 60 * 60 * 24 * 7
		itemEpochStep = tableEpochStep
		step = 60 * 60
	} else if oneDay <= diffTime {
		tableName = tablePrefix + "-5m1d"
		tableEpochStep = 60 * 60 * 24
		itemEpochStep = tableEpochStep
		step = 5 * 60
	} else {
		tableName = tablePrefix + "-1m1h"
		tableEpochStep = 60 * 60 * 24
		itemEpochStep = 60 * 60
		step = 60
	}

	slots := make([]*timeSlot, 0, 5)
	startTableEpoch := startTime.Unix() - startTime.Unix()%int64(tableEpochStep)
	endTableEpoch := endTime.Unix()
	for tableEpoch := startTableEpoch; tableEpoch < endTableEpoch; tableEpoch += int64(tableEpochStep) {
		startItemEpoch := mathutil.MaxInt64(tableEpoch, startTime.Unix()-startTime.Unix()%int64(itemEpochStep))
		endItemEpoch := mathutil.MinInt64(tableEpoch+int64(tableEpochStep), endTime.Unix())
		for itemEpoch := startItemEpoch; itemEpoch < endItemEpoch; itemEpoch += int64(itemEpochStep) {
			slot := timeSlot{
				tableName: fmt.Sprintf("%s-%d", tableName, tableEpoch),
				itemEpoch: itemEpoch,
			}
			slots = append(slots, &slot)
		}
	}

	return slots, step
}
<|endoftext|>


<commit_before><commit_msg>Element now accepts interface{} as a payload<commit_after><|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"github.com/BurntSushi/toml"
	"github.com/osrg/gobgp/config"
	"net"
)

func main() {
	b := config.Bgp{
		Global: config.Global{
			As:       12332,
			RouterId: net.ParseIP("10.0.0.1"),
		},
		NeighborList: []config.Neighbor{
			config.Neighbor{
				PeerAs:          12333,
				NeighborAddress: net.ParseIP("192.168.177.32"),
				AuthPassword:    "apple",
				AfiSafiList:     []config.AfiSafi{config.AfiSafi{AfiSafiName: "ipv4-unicast"}, config.AfiSafi{AfiSafiName: "ipv6-unicast"}},
				ApplyPolicy: config.ApplyPolicy{
					ImportPolicies:      []string{"pd1"},
					DefaultImportPolicy: config.DEFAULT_POLICY_TYPE_ACCEPT_ROUTE,
				},
			},
			config.Neighbor{
				PeerAs:          12334,
				NeighborAddress: net.ParseIP("192.168.177.33"),
				AuthPassword:    "orange",
			},
			config.Neighbor{
				PeerAs:          12335,
				NeighborAddress: net.ParseIP("192.168.177.34"),
				AuthPassword:    "grape",
			},
		},
	}

	var buffer bytes.Buffer
	encoder := toml.NewEncoder(&buffer)
	err := encoder.Encode(b)
	if err != nil {
		panic(err)
	}

	err = encoder.Encode(policy())
	if err != nil {
		panic(err)
	}
	fmt.Printf("%v\n", buffer.String())
}

func policy() config.RoutingPolicy {

	ps := config.PrefixSet{
		PrefixSetName: "ps1",
		PrefixList: []config.Prefix{
			config.Prefix{
				Address:         net.ParseIP("10.3.192.0"),
				Masklength:      21,
				MasklengthRange: "21..24",
			}},
	}

	ns := config.NeighborSet{
		NeighborSetName: "ns1",
		NeighborInfoList: []config.NeighborInfo{
			config.NeighborInfo{
				Address: net.ParseIP("10.0.0.2"),
			}},
	}

	cs := config.CommunitySet{
		CommunitySetName: "community1",
		CommunityMembers: []string{"65100:10"},
	}

	as := config.AsPathSet{
		AsPathSetName: "aspath1",
		AsPathSetMembers: []string{"^65100"},
	}

	bds := config.BgpDefinedSets{
		CommunitySetList: []config.CommunitySet{cs},
		AsPathSetList:	[]config.AsPathSet{as},
	}

	ds := config.DefinedSets{
		PrefixSetList:   []config.PrefixSet{ps},
		NeighborSetList: []config.NeighborSet{ns},
		BgpDefinedSets: bds,
	}

	al := config.AsPathLength{
		Operator: "eq",
		Value: 2,
	}

	s := config.Statement{
		Name: "statement1",
		Conditions: config.Conditions{
			MatchPrefixSet:   "ps1",
			MatchNeighborSet: "ns1",
			MatchSetOptions:  config.MATCH_SET_OPTIONS_TYPE_ALL,
			BgpConditions: config.BgpConditions{
				MatchCommunitySet: "community1",
				MatchAsPathSet: "aspath1",
				AsPathLength: al,
			},
		},
		Actions: config.Actions{
			AcceptRoute: false,
			RejectRoute: true,
			BgpActions: config.BgpActions{
				SetCommunity: config.SetCommunity{
					Communities: []string{"65100:20"},
					Options: "ADD",
				},
				SetMed: "-200",
			},
		},
	}

	pd := config.PolicyDefinition{
		Name:          "pd1",
		StatementList: []config.Statement{s},
	}

	p := config.RoutingPolicy{
		DefinedSets:          ds,
		PolicyDefinitionList: []config.PolicyDefinition{pd},
	}

	return p
}
<commit_msg>tool: update example_toml.go<commit_after>package main

import (
	"bytes"
	"fmt"
	"github.com/BurntSushi/toml"
	"github.com/osrg/gobgp/config"
	"net"
)

func main() {
	b := config.Bgp{
		Global: config.Global{
			As:       12332,
			RouterId: net.ParseIP("10.0.0.1"),
		},
		NeighborList: []config.Neighbor{
			config.Neighbor{
				PeerAs:          12333,
				NeighborAddress: net.ParseIP("192.168.177.32"),
				AuthPassword:    "apple",
				AfiSafiList:     []config.AfiSafi{config.AfiSafi{AfiSafiName: "ipv4-unicast"}, config.AfiSafi{AfiSafiName: "ipv6-unicast"}},
				ApplyPolicy: config.ApplyPolicy{
					ImportPolicies:      []string{"pd1"},
					DefaultImportPolicy: config.DEFAULT_POLICY_TYPE_ACCEPT_ROUTE,
				},
			},
			config.Neighbor{
				PeerAs:          12334,
				NeighborAddress: net.ParseIP("192.168.177.33"),
				AuthPassword:    "orange",
			},
			config.Neighbor{
				PeerAs:          12335,
				NeighborAddress: net.ParseIP("192.168.177.34"),
				AuthPassword:    "grape",
			},
		},
	}

	var buffer bytes.Buffer
	encoder := toml.NewEncoder(&buffer)
	err := encoder.Encode(b)
	if err != nil {
		panic(err)
	}

	err = encoder.Encode(policy())
	if err != nil {
		panic(err)
	}
	fmt.Printf("%v\n", buffer.String())
}

func policy() config.RoutingPolicy {

	ps := config.PrefixSet{
		PrefixSetName: "ps1",
		PrefixList: []config.Prefix{
			config.Prefix{
				Address:         net.ParseIP("10.3.192.0"),
				Masklength:      21,
				MasklengthRange: "21..24",
			}},
	}

	ns := config.NeighborSet{
		NeighborSetName: "ns1",
		NeighborInfoList: []config.NeighborInfo{
			config.NeighborInfo{
				Address: net.ParseIP("10.0.0.2"),
			}},
	}

	cs := config.CommunitySet{
		CommunitySetName: "community1",
		CommunityMembers: []string{"65100:10"},
	}

	ecs := config.ExtCommunitySet{
		ExtCommunitySetName: "ecommunity1",
		ExtCommunityMembers: []string{"RT:65001:200"},
	}

	as := config.AsPathSet{
		AsPathSetName: "aspath1",
		AsPathSetMembers: []string{"^65100"},
	}

	bds := config.BgpDefinedSets{
		CommunitySetList: []config.CommunitySet{cs},
		ExtCommunitySetList: []config.ExtCommunitySet{ecs},
		AsPathSetList:	[]config.AsPathSet{as},
	}

	ds := config.DefinedSets{
		PrefixSetList:   []config.PrefixSet{ps},
		NeighborSetList: []config.NeighborSet{ns},
		BgpDefinedSets: bds,
	}

	al := config.AsPathLength{
		Operator: "eq",
		Value: 2,
	}

	s := config.Statement{
		Name: "statement1",
		Conditions: config.Conditions{
			MatchPrefixSet:   "ps1",
			MatchNeighborSet: "ns1",
			MatchSetOptions:  config.MATCH_SET_OPTIONS_TYPE_ALL,
			BgpConditions: config.BgpConditions{
				MatchCommunitySet: "community1",
				MatchExtCommunitySet: "ecommunity1",
				MatchAsPathSet: "aspath1",
				AsPathLength: al,
			},
		},
		Actions: config.Actions{
			AcceptRoute: false,
			RejectRoute: true,
			BgpActions: config.BgpActions{
				SetCommunity: config.SetCommunity{
					Communities: []string{"65100:20"},
					Options: "ADD",
				},
				SetMed: "-200",
			},
		},
	}

	pd := config.PolicyDefinition{
		Name:          "pd1",
		StatementList: []config.Statement{s},
	}

	p := config.RoutingPolicy{
		DefinedSets:          ds,
		PolicyDefinitionList: []config.PolicyDefinition{pd},
	}

	return p
}
<|endoftext|>


<commit_before>// @author       sigu-399
// @description  An implementation of JSON Schema, draft v4 - Go language
// @created      26-02-2013

package gojsonschema

import (
	"errors"
	"fmt"
	"gojsonreference"
	"reflect"
)

func NewJsonSchemaDocument(documentReferenceString string) (*JsonSchemaDocument, error) {

	var err error

	d := JsonSchemaDocument{}
	d.documentReference, err = gojsonreference.NewJsonReference(documentReferenceString)
	d.pool = NewSchemaPool()

	spd, err := d.pool.GetPoolDocument(d.documentReference)
	if err != nil {
		return nil, err
	}

	err = d.parse(spd.Document)
	return &d, err
}

type JsonSchemaDocument struct {
	documentReference gojsonreference.JsonReference
	rootSchema        *JsonSchema
	pool              *SchemaPool
}

func (d *JsonSchemaDocument) parse(document interface{}) error {
	d.rootSchema = &JsonSchema{}
	return d.parseSchema(document, d.rootSchema)
}

func (d *JsonSchemaDocument) parseSchema(documentNode interface{}, currentSchema *JsonSchema) error {

	if !isKind(documentNode, reflect.Map) {
		return errors.New("Schema must be an object")
	}

	m := documentNode.(map[string]interface{})

	if currentSchema == d.rootSchema {
		if !existsMapKey(m, "$schema") {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_IS_REQUIRED, "$schema"))
		}
		if !isKind(m["$schema"], reflect.String) {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "$schema", "string"))
		}
		schemaRef := m["$schema"].(string)
		schemaReference, err := gojsonreference.NewJsonReference(schemaRef)
		currentSchema.schema = &schemaReference
		if err != nil {
			return err
		}

		currentSchema.ref = &d.documentReference
	}

	// ref
	if existsMapKey(m, "$ref") && !isKind(m["$ref"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "$ref", "string"))
	}
	if k, ok := m["$ref"].(string); ok {
		jsonReference, err := gojsonreference.NewJsonReference(k)
		if err != nil {
			return err
		}

		if jsonReference.HasFullUrl {
			currentSchema.ref = &jsonReference
		} else {
			inheritedReference, err := gojsonreference.Inherits(*currentSchema.ref, jsonReference)
			if err != nil {
				return err
			}
			currentSchema.ref = inheritedReference
		}

		dsp, err := d.pool.GetPoolDocument(*currentSchema.ref)
		if err != nil {
			return err
		}

		jsonPointer := currentSchema.ref.GetPointer()

		httpDocumentNode := jsonPointer.Get( dsp.Document )
		fmt.Printf("%s\n", httpDocumentNode)

		if !isKind(httpDocumentNode, reflect.Map) {
			return errors.New("Schema must be an object")
		}
		m = httpDocumentNode.(map[string]interface{})
	}

	// id
	if existsMapKey(m, "id") && !isKind(m["id"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "id", "string"))
	}
	if k, ok := m["id"].(string); ok {
		currentSchema.id = &k
	}

	// title
	if existsMapKey(m, "title") && !isKind(m["title"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "title", "string"))
	}
	if k, ok := m["title"].(string); ok {
		currentSchema.title = &k
	}

	// description
	if existsMapKey(m, "description") && !isKind(m["description"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "description", "string"))
	}
	if k, ok := m["description"].(string); ok {
		currentSchema.description = &k
	}

	// properties
	/*	if !existsMapKey(m, "properties") {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_IS_REQUIRED, "properties"))
		}
	*/
	for k := range m {
		if k == "properties" {
			err := d.parseProperties(m[k], currentSchema)
			if err != nil {
				return err
			}
		}
	}

	// items
	/*	if !existsMapKey(m, "items") {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_IS_REQUIRED, "items"))
		}
	*/
	for k := range m {
		if k == "items" {
			newSchema := &JsonSchema{parent: currentSchema}
			currentSchema.AddPropertiesChild(newSchema)
			err := d.parseSchema(m[k], newSchema)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func (d *JsonSchemaDocument) parseProperties(documentNode interface{}, currentSchema *JsonSchema) error {

	if !isKind(documentNode, reflect.Map) {
		return errors.New("Properties must be an object")
	}

	m := documentNode.(map[string]interface{})
	for k := range m {
		schemaProperty := k
		newSchema := &JsonSchema{property: &schemaProperty, parent: currentSchema, ref: currentSchema.ref}
		currentSchema.AddPropertiesChild(newSchema)
		err := d.parseSchema(m[k], newSchema)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>added type parsing, and pointer error handling<commit_after>// @author       sigu-399
// @description  An implementation of JSON Schema, draft v4 - Go language
// @created      26-02-2013

package gojsonschema

import (
	"errors"
	"fmt"
	"gojsonreference"
	"reflect"
)

func NewJsonSchemaDocument(documentReferenceString string) (*JsonSchemaDocument, error) {

	var err error

	d := JsonSchemaDocument{}
	d.documentReference, err = gojsonreference.NewJsonReference(documentReferenceString)
	d.pool = NewSchemaPool()

	spd, err := d.pool.GetPoolDocument(d.documentReference)
	if err != nil {
		return nil, err
	}

	err = d.parse(spd.Document)
	return &d, err
}

type JsonSchemaDocument struct {
	documentReference gojsonreference.JsonReference
	rootSchema        *JsonSchema
	pool              *SchemaPool
}

func (d *JsonSchemaDocument) parse(document interface{}) error {
	d.rootSchema = &JsonSchema{}
	return d.parseSchema(document, d.rootSchema)
}

func (d *JsonSchemaDocument) parseSchema(documentNode interface{}, currentSchema *JsonSchema) error {

	if !isKind(documentNode, reflect.Map) {
		return errors.New("Schema must be an object")
	}

	m := documentNode.(map[string]interface{})

	if currentSchema == d.rootSchema {
		if !existsMapKey(m, "$schema") {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_IS_REQUIRED, "$schema"))
		}
		if !isKind(m["$schema"], reflect.String) {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "$schema", "string"))
		}
		schemaRef := m["$schema"].(string)
		schemaReference, err := gojsonreference.NewJsonReference(schemaRef)
		currentSchema.schema = &schemaReference
		if err != nil {
			return err
		}

		currentSchema.ref = &d.documentReference
	}

	// ref
	if existsMapKey(m, "$ref") && !isKind(m["$ref"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "$ref", "string"))
	}
	if k, ok := m["$ref"].(string); ok {
		jsonReference, err := gojsonreference.NewJsonReference(k)
		if err != nil {
			return err
		}

		if jsonReference.HasFullUrl {
			currentSchema.ref = &jsonReference
		} else {
			inheritedReference, err := gojsonreference.Inherits(*currentSchema.ref, jsonReference)
			if err != nil {
				return err
			}
			currentSchema.ref = inheritedReference
		}

		dsp, err := d.pool.GetPoolDocument(*currentSchema.ref)
		if err != nil {
			return err
		}

		jsonPointer := currentSchema.ref.GetPointer()

		httpDocumentNode, err := jsonPointer.Get(dsp.Document)
		if err != nil {
			return err
		}

		fmt.Printf("%s\n", httpDocumentNode)

		if !isKind(httpDocumentNode, reflect.Map) {
			return errors.New("Schema must be an object")
		}
		m = httpDocumentNode.(map[string]interface{})
	}

	// id
	if existsMapKey(m, "id") && !isKind(m["id"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "id", "string"))
	}
	if k, ok := m["id"].(string); ok {
		currentSchema.id = &k
	}

	// title
	if existsMapKey(m, "title") && !isKind(m["title"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "title", "string"))
	}
	if k, ok := m["title"].(string); ok {
		currentSchema.title = &k
	}

	// description
	if existsMapKey(m, "description") && !isKind(m["description"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "description", "string"))
	}
	if k, ok := m["description"].(string); ok {
		currentSchema.description = &k
	}

	// type
	if existsMapKey(m, "type") && !isKind(m["type"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "type", "string"))
	}
	if k, ok := m["type"].(string); ok {
		currentSchema.etype = &k
	}

	// properties
	/*	if !existsMapKey(m, "properties") {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_IS_REQUIRED, "properties"))
		}
	*/
	for k := range m {
		if k == "properties" {
			err := d.parseProperties(m[k], currentSchema)
			if err != nil {
				return err
			}
		}
	}

	// items
	/*	if !existsMapKey(m, "items") {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_IS_REQUIRED, "items"))
		}
	*/
	for k := range m {
		if k == "items" {
			newSchema := &JsonSchema{parent: currentSchema}
			currentSchema.AddPropertiesChild(newSchema)
			err := d.parseSchema(m[k], newSchema)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func (d *JsonSchemaDocument) parseProperties(documentNode interface{}, currentSchema *JsonSchema) error {

	if !isKind(documentNode, reflect.Map) {
		return errors.New("Properties must be an object")
	}

	m := documentNode.(map[string]interface{})
	for k := range m {
		schemaProperty := k
		newSchema := &JsonSchema{property: &schemaProperty, parent: currentSchema, ref: currentSchema.ref}
		currentSchema.AddPropertiesChild(newSchema)
		err := d.parseSchema(m[k], newSchema)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// @author       sigu-399
// @description  An implementation of JSON Schema, draft v4 - Go language
// @created      26-02-2013

package gojsonschema

import (
	"errors"
	"fmt"
	"gojsonreference"
	"reflect"
)

func NewJsonSchemaDocument(documentReferenceString string) (*JsonSchemaDocument, error) {

	var err error

	d := JsonSchemaDocument{}
	d.documentReference, err = gojsonreference.NewJsonReference(documentReferenceString)

	document, err := GetHttpJson(documentReferenceString)
	if err != nil {
		return nil, err
	}

	err = d.parse(document)
	return &d, err
}

type JsonSchemaDocument struct {
	documentReference gojsonreference.JsonReference
	rootSchema        *JsonSchema
}

func (d *JsonSchemaDocument) parse(document interface{}) error {
	d.rootSchema = &JsonSchema{}
	return d.parseSchema(document, d.rootSchema)
}

func (d *JsonSchemaDocument) parseSchema(documentNode interface{}, currentSchema *JsonSchema) error {

	if !isKind(documentNode, reflect.Map) {
		return errors.New("Schema must be an object")
	}

	m := documentNode.(map[string]interface{})

	if currentSchema == d.rootSchema {
		if !existsMapKey(m, "$schema") {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_IS_REQUIRED, "$schema"))
		}
		if !isKind(m["$schema"], reflect.String) {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "$schema", "string"))
		}
		schemaRef := m["$schema"].(string)
		schemaReference, err := gojsonreference.NewJsonReference(schemaRef)
		currentSchema.schema = &schemaReference
		if err != nil {
			return err
		}

		currentSchema.ref = &d.documentReference
	}

	// ref
	if existsMapKey(m, "$ref") && !isKind(m["$ref"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "$ref", "string"))
	}
	if k, ok := m["$ref"].(string); ok {
		jsonReference, err := gojsonreference.NewJsonReference(k)
		if err != nil {
			return err
		}
		inheritedReference, err := gojsonreference.Inherits(*currentSchema.ref,jsonReference)
		if err != nil {
			return err
		}
		currentSchema.ref = inheritedReference
		httpDocumentNode, err := GetHttpJson(currentSchema.ref.String())
		if err != nil {
			return err
		}
		if !isKind(httpDocumentNode, reflect.Map) {
			return errors.New("Schema must be an object")
		}
		m = httpDocumentNode.(map[string]interface{})
	}

	// id
	if existsMapKey(m, "id") && !isKind(m["id"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "id", "string"))
	}
	if k, ok := m["id"].(string); ok {
		currentSchema.id = &k
	}

	// title
	if existsMapKey(m, "title") && !isKind(m["title"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "title", "string"))
	}
	if k, ok := m["title"].(string); ok {
		currentSchema.title = &k
	}

	// description
	if existsMapKey(m, "description") && !isKind(m["description"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "description", "string"))
	}
	if k, ok := m["description"].(string); ok {
		currentSchema.description = &k
	}

	// properties
	/*	if !existsMapKey(m, "properties") {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_IS_REQUIRED, "properties"))
		}
	*/
	for k := range m {
		if k == "properties" {
			err := d.parseProperties(m[k], currentSchema)
			if err != nil {
				return err
			}
		}
	}

	// items
	/*	if !existsMapKey(m, "items") {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_IS_REQUIRED, "items"))
		}
	*/
	for k := range m {
		if k == "items" {
			newSchema := &JsonSchema{ parent : currentSchema }
			currentSchema.AddPropertiesChild(newSchema)
			err := d.parseSchema(m[k], newSchema)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func (d *JsonSchemaDocument) parseProperties(documentNode interface{}, currentSchema *JsonSchema) error {

	if !isKind(documentNode, reflect.Map) {
		return errors.New("Properties must be an object")
	}

	m := documentNode.(map[string]interface{})
	for k := range m {
		schemaProperty := k
		newSchema := &JsonSchema{property: &schemaProperty, parent : currentSchema, ref: currentSchema.ref}
		currentSchema.AddPropertiesChild(newSchema)
		err := d.parseSchema(m[k], newSchema)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>load first schema through pool<commit_after>// @author       sigu-399
// @description  An implementation of JSON Schema, draft v4 - Go language
// @created      26-02-2013

package gojsonschema

import (
	"errors"
	"fmt"
	"gojsonreference"
	"reflect"
)

func NewJsonSchemaDocument(documentReferenceString string) (*JsonSchemaDocument, error) {

	var err error

	d := JsonSchemaDocument{}
	d.documentReference, err = gojsonreference.NewJsonReference(documentReferenceString)
	d.pool = NewSchemaPool()

	spd, err := d.pool.GetPoolDocument(d.documentReference)
	if err != nil {
		return nil, err
	}

	err = d.parse(spd.Document)
	return &d, err
}

type JsonSchemaDocument struct {
	documentReference gojsonreference.JsonReference
	rootSchema        *JsonSchema
	pool              *SchemaPool
}

func (d *JsonSchemaDocument) parse(document interface{}) error {
	d.rootSchema = &JsonSchema{}
	return d.parseSchema(document, d.rootSchema)
}

func (d *JsonSchemaDocument) parseSchema(documentNode interface{}, currentSchema *JsonSchema) error {

	if !isKind(documentNode, reflect.Map) {
		return errors.New("Schema must be an object")
	}

	m := documentNode.(map[string]interface{})

	if currentSchema == d.rootSchema {
		if !existsMapKey(m, "$schema") {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_IS_REQUIRED, "$schema"))
		}
		if !isKind(m["$schema"], reflect.String) {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "$schema", "string"))
		}
		schemaRef := m["$schema"].(string)
		schemaReference, err := gojsonreference.NewJsonReference(schemaRef)
		currentSchema.schema = &schemaReference
		if err != nil {
			return err
		}

		currentSchema.ref = &d.documentReference
	}

	// ref
	if existsMapKey(m, "$ref") && !isKind(m["$ref"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "$ref", "string"))
	}
	if k, ok := m["$ref"].(string); ok {
		jsonReference, err := gojsonreference.NewJsonReference(k)
		if err != nil {
			return err
		}
		inheritedReference, err := gojsonreference.Inherits(*currentSchema.ref, jsonReference)
		if err != nil {
			return err
		}
		currentSchema.ref = inheritedReference
		httpDocumentNode, err := GetHttpJson(currentSchema.ref.String())
		if err != nil {
			return err
		}
		if !isKind(httpDocumentNode, reflect.Map) {
			return errors.New("Schema must be an object")
		}
		m = httpDocumentNode.(map[string]interface{})
	}

	// id
	if existsMapKey(m, "id") && !isKind(m["id"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "id", "string"))
	}
	if k, ok := m["id"].(string); ok {
		currentSchema.id = &k
	}

	// title
	if existsMapKey(m, "title") && !isKind(m["title"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "title", "string"))
	}
	if k, ok := m["title"].(string); ok {
		currentSchema.title = &k
	}

	// description
	if existsMapKey(m, "description") && !isKind(m["description"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "description", "string"))
	}
	if k, ok := m["description"].(string); ok {
		currentSchema.description = &k
	}

	// properties
	/*	if !existsMapKey(m, "properties") {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_IS_REQUIRED, "properties"))
		}
	*/
	for k := range m {
		if k == "properties" {
			err := d.parseProperties(m[k], currentSchema)
			if err != nil {
				return err
			}
		}
	}

	// items
	/*	if !existsMapKey(m, "items") {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_IS_REQUIRED, "items"))
		}
	*/
	for k := range m {
		if k == "items" {
			newSchema := &JsonSchema{parent: currentSchema}
			currentSchema.AddPropertiesChild(newSchema)
			err := d.parseSchema(m[k], newSchema)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func (d *JsonSchemaDocument) parseProperties(documentNode interface{}, currentSchema *JsonSchema) error {

	if !isKind(documentNode, reflect.Map) {
		return errors.New("Properties must be an object")
	}

	m := documentNode.(map[string]interface{})
	for k := range m {
		schemaProperty := k
		newSchema := &JsonSchema{property: &schemaProperty, parent: currentSchema, ref: currentSchema.ref}
		currentSchema.AddPropertiesChild(newSchema)
		err := d.parseSchema(m[k], newSchema)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
//    "net/ipv4"
    "golang.org/x/net/ipv4"
    "os"
)

func main() {
    fmt.Println("Hello, World!")

    c, err := openUDP(5049, 5050)
    if err != nil {
        fmt.Println("Failed")
        os.Exit(1)
    }
    read(c, 1024)
}

//type netConn interface {
//    open(src, dest int) (S UDP, error)
//}

type UDP struct {
	conn *ipv4.RawConn
}
func openUDP(src, dest int) (*UDP, error) {
    return nil, nil
}
func read(S *UDP, size int) ([]byte, error) {
    return make([]byte, 0), nil
}<commit_msg>Finished writing the framework for the project.<commit_after>package main

import (
    "fmt"
//    "net/ipv4"
    "golang.org/x/net/ipv4"
    "os"
)

func main() {
    fmt.Println("Hello, World!")

    c, err := openUDP(5049, 5050)
    if err != nil {
        fmt.Println("Failed")
        os.Exit(1)
    }
    readUDP(c, 1024)
    writeUDP(c, make([]byte, 0))
    closeUDP(c)
}

type UDP struct {
    open bool
	conn *ipv4.RawConn
}
func openUDP(src, dest int) (*UDP, error) {
    return nil, nil
}
func readUDP(c *UDP, size int) ([]byte, error) {
    return make([]byte, 0), nil
}
func writeUDP(c *UDP, x []byte) error {
    return nil
}
func closeUDP(c *UDP) error {
    return nil
}<|endoftext|>


<commit_before><commit_msg>memdb: Use DeleteNode() to reduce an addtional findNode() for deletes<commit_after><|endoftext|>


<commit_before>package transport

import (
	"errors"
	"math/rand"

	nsq "github.com/bitly/go-nsq"
	log "github.com/cihub/seelog"

	"github.com/mattheath/phosphor/util"
)

var ErrPublishFailure = errors.New("Failed to publish to NSQD")

// NewNSQTransport initialises a Transport over NSQ
func NewNSQTransport(topic string, nsqdTCPAddrs util.StringArray) (Transport, error) {

	// Currently using default config
	cfg := nsq.NewConfig()

	// Create a producer for each nsqd node provided
	producers := make(map[string]*nsq.Producer)
	for _, addr := range nsqdTCPAddrs {
		producer, err := nsq.NewProducer(addr, cfg)
		if err != nil {
			log.Warnf("failed to create nsq.Producer - %s", err)
		}
		producers[addr] = producer
	}

	return &NSQPublisher{
		topic:     topic,
		producers: producers,
	}, nil
}

type NSQPublisher struct {
	topic     string
	producers map[string]*nsq.Producer
}

func (p *NSQPublisher) MultiPublish(body [][]byte) error {

	// Round robin, from a random starting position
	i := rand.Intn(len(p.producers)) - 1

	// Attempt up to our number of configured nodes
	for attempt := 0; attempt < len(p.producers); attempt++ {
		pd := p.producers[i]
		if err := pd.MultiPublish(p.topic, body); err == nil {
			// success!
			return nil
		}

		// Move to next host, or cycle back around
		i++
		if i >= len(p.producers) {
			i = 0
		}
	}

	// We've run out of nodes, and not managed to publish
	return ErrPublishFailure
}
<commit_msg>Add lookup index for nsq producers, so we can iterate nodes by numerical index<commit_after>package transport

import (
	"errors"
	"math/rand"

	nsq "github.com/bitly/go-nsq"
	log "github.com/cihub/seelog"

	"github.com/mattheath/phosphor/util"
)

var ErrPublishFailure = errors.New("Failed to publish to NSQD")

// NewNSQTransport initialises a Transport over NSQ
func NewNSQTransport(topic string, nsqdTCPAddrs util.StringArray) (Transport, error) {

	// Currently using default config
	cfg := nsq.NewConfig()

	// Create a producer for each nsqd node provided
	producers := make(map[string]*nsq.Producer)
	producersIndex := make([]*nsq.Producer, 0, len(nsqdTCPAddrs))
	for _, addr := range nsqdTCPAddrs {
		producer, err := nsq.NewProducer(addr, cfg)
		if err != nil {
			log.Warnf("failed to create nsq.Producer - %s", err)
		}
		producers[addr] = producer
		producersIndex = append(producersIndex, producers[addr])
	}

	return &NSQPublisher{
		topic:          topic,
		producers:      producers,
		producersIndex: producersIndex,
	}, nil
}

type NSQPublisher struct {
	topic          string
	producers      map[string]*nsq.Producer
	producersIndex []*nsq.Producer
}

func (p *NSQPublisher) MultiPublish(body [][]byte) error {

	// Round robin, from a random starting position
	i := rand.Intn(len(p.producers)) - 1

	// Attempt up to our number of configured nodes
	for attempt := 0; attempt < len(p.producers); attempt++ {
		pd := p.producersIndex[i]
		if err := pd.MultiPublish(p.topic, body); err == nil {
			// success!
			return nil
		}

		// Move to next host, or cycle back around
		i++
		if i >= len(p.producers) {
			i = 0
		}
	}

	// We've run out of nodes, and not managed to publish
	return ErrPublishFailure
}
<|endoftext|>


<commit_before>package tlc

import (
	"archive/zip"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
)

const (
	// ModeMask is or'd with files being diffed
	ModeMask = 0644

	// NullPath can be specified instead of a directory to yield an empty container
	NullPath = "/dev/null"
)

// A FilterFunc allows ignoring certain files or directories when walking the filesystem
// When a directory is ignored by a FilterFunc, all its children are, too!
type FilterFunc func(fileInfo os.FileInfo) bool

// DefaultFilter is a passthrough that filters out no files at all
var DefaultFilter FilterFunc = func(fileInfo os.FileInfo) bool {
	return true
}

// Walk goes through every file in a director
func Walk(BasePath string, filter FilterFunc) (*Container, error) {
	if filter == nil {
		filter = DefaultFilter
	}

	var Dirs []*Dir
	var Symlinks []*Symlink
	var Files []*File

	TotalOffset := int64(0)

	onEntry := func(FullPath string, fileInfo os.FileInfo, err error) error {
		// we shouldn't encounter any error crawling the repo
		if err != nil {
			if os.IsPermission(err) {
				// ...except permission errors, those are fine
				log.Printf("Permission error: %s\n", err.Error())
			} else {
				return err
			}
		}

		Path, err := filepath.Rel(BasePath, FullPath)
		if err != nil {
			return err
		}

		Path = filepath.ToSlash(Path)

		// don't end up with files we (the patcher) can't modify
		Mode := fileInfo.Mode() | ModeMask

		if !filter(fileInfo) {
			if Mode.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		if Mode.IsDir() {
			Dirs = append(Dirs, &Dir{Path: Path, Mode: uint32(Mode)})
		} else if Mode.IsRegular() {
			Size := fileInfo.Size()
			Offset := TotalOffset
			OffsetEnd := Offset + Size

			Files = append(Files, &File{Path: Path, Mode: uint32(Mode), Size: Size, Offset: Offset})
			TotalOffset = OffsetEnd
		} else if Mode&os.ModeSymlink > 0 {
			Dest, err := os.Readlink(FullPath)
			if err != nil {
				return err
			}

			Dest = filepath.ToSlash(Dest)
			Symlinks = append(Symlinks, &Symlink{Path: Path, Mode: uint32(Mode), Dest: Dest})
		}

		return nil
	}

	if BasePath == NullPath {
		// empty container is fine - /dev/null is legal even on Win32 where it doesn't exist
	} else {
		fi, err := os.Lstat(BasePath)
		if err != nil {
			return nil, err
		}

		if !fi.IsDir() {
			return nil, fmt.Errorf("tlc: can't walk non-directory %s", BasePath)
		}

		err = filepath.Walk(BasePath, onEntry)
		if err != nil {
			return nil, err
		}
	}

	container := &Container{Size: TotalOffset, Dirs: Dirs, Symlinks: Symlinks, Files: Files}
	return container, nil
}

// WalkZip walks all file in a zip archive and returns a container
func WalkZip(zr *zip.Reader, filter FilterFunc) (*Container, error) {
	if filter == nil {
		// default filter is a passthrough
		filter = func(fileInfo os.FileInfo) bool {
			return true
		}
	}

	var Dirs []*Dir
	var Symlinks []*Symlink
	var Files []*File

	dirMap := make(map[string]os.FileMode)

	TotalOffset := int64(0)

	for _, file := range zr.File {
		// don't trust zip files to have directory entries for
		// all directories. it's a miracle anything works.
		dir := path.Dir(file.Name)
		if dir != "" && dirMap[dir] == 0 {
			dirMap[dir] = os.FileMode(0755)
		}

		info := file.FileInfo()
		mode := info.Mode() | ModeMask

		if info.IsDir() {
			dirMap[dir] = mode
		} else if mode&os.ModeSymlink > 0 {
			var linkname []byte

			err := func() error {
				reader, err := file.Open()
				if err != nil {
					return err
				}
				defer reader.Close()

				linkname, err = ioutil.ReadAll(reader)
				if err != nil {
					return err
				}
				return nil
			}()

			if err != nil {
				return nil, err
			}

			Symlinks = append(Symlinks, &Symlink{
				Path: file.Name,
				Dest: string(linkname),
				Mode: uint32(mode),
			})
		} else {
			Size := info.Size()

			Files = append(Files, &File{
				Path:   file.Name,
				Mode:   uint32(mode),
				Size:   Size,
				Offset: TotalOffset,
			})

			TotalOffset += Size
		}
	}

	for dirPath, dirMode := range dirMap {
		Dirs = append(Dirs, &Dir{
			Path: dirPath,
			Mode: uint32(dirMode),
		})
	}

	container := &Container{Size: TotalOffset, Dirs: Dirs, Symlinks: Symlinks, Files: Files}
	return container, nil
}

// Stats return a human-readable summary of the contents of a container
func (container *Container) Stats() string {
	return fmt.Sprintf("%d files, %d dirs, %d symlinks",
		len(container.Files), len(container.Dirs), len(container.Symlinks))
}
<commit_msg>Don't add dot folder. Closes https://github.com/itchio/butler/issues/60<commit_after>package tlc

import (
	"archive/zip"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
)

const (
	// ModeMask is or'd with files being diffed
	ModeMask = 0644

	// NullPath can be specified instead of a directory to yield an empty container
	NullPath = "/dev/null"
)

// A FilterFunc allows ignoring certain files or directories when walking the filesystem
// When a directory is ignored by a FilterFunc, all its children are, too!
type FilterFunc func(fileInfo os.FileInfo) bool

// DefaultFilter is a passthrough that filters out no files at all
var DefaultFilter FilterFunc = func(fileInfo os.FileInfo) bool {
	return true
}

// Walk goes through every file in a director
func Walk(BasePath string, filter FilterFunc) (*Container, error) {
	if filter == nil {
		filter = DefaultFilter
	}

	var Dirs []*Dir
	var Symlinks []*Symlink
	var Files []*File

	TotalOffset := int64(0)

	onEntry := func(FullPath string, fileInfo os.FileInfo, err error) error {
		// we shouldn't encounter any error crawling the repo
		if err != nil {
			if os.IsPermission(err) {
				// ...except permission errors, those are fine
				log.Printf("Permission error: %s\n", err.Error())
			} else {
				return err
			}
		}

		Path, err := filepath.Rel(BasePath, FullPath)
		if err != nil {
			return err
		}

		Path = filepath.ToSlash(Path)
		if Path == "." {
			// Don't store a single folder named "."
			return nil
		}

		// don't end up with files we (the patcher) can't modify
		Mode := fileInfo.Mode() | ModeMask

		if !filter(fileInfo) {
			if Mode.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		if Mode.IsDir() {
			Dirs = append(Dirs, &Dir{Path: Path, Mode: uint32(Mode)})
		} else if Mode.IsRegular() {
			Size := fileInfo.Size()
			Offset := TotalOffset
			OffsetEnd := Offset + Size

			Files = append(Files, &File{Path: Path, Mode: uint32(Mode), Size: Size, Offset: Offset})
			TotalOffset = OffsetEnd
		} else if Mode&os.ModeSymlink > 0 {
			Dest, err := os.Readlink(FullPath)
			if err != nil {
				return err
			}

			Dest = filepath.ToSlash(Dest)
			Symlinks = append(Symlinks, &Symlink{Path: Path, Mode: uint32(Mode), Dest: Dest})
		}

		return nil
	}

	if BasePath == NullPath {
		// empty container is fine - /dev/null is legal even on Win32 where it doesn't exist
	} else {
		fi, err := os.Lstat(BasePath)
		if err != nil {
			return nil, err
		}

		if !fi.IsDir() {
			return nil, fmt.Errorf("tlc: can't walk non-directory %s", BasePath)
		}

		err = filepath.Walk(BasePath, onEntry)
		if err != nil {
			return nil, err
		}
	}

	container := &Container{Size: TotalOffset, Dirs: Dirs, Symlinks: Symlinks, Files: Files}
	return container, nil
}

// WalkZip walks all file in a zip archive and returns a container
func WalkZip(zr *zip.Reader, filter FilterFunc) (*Container, error) {
	if filter == nil {
		// default filter is a passthrough
		filter = func(fileInfo os.FileInfo) bool {
			return true
		}
	}

	var Dirs []*Dir
	var Symlinks []*Symlink
	var Files []*File

	dirMap := make(map[string]os.FileMode)

	TotalOffset := int64(0)

	for _, file := range zr.File {
		// don't trust zip files to have directory entries for
		// all directories. it's a miracle anything works.
		dir := path.Dir(file.Name)
		if dir != "" && dirMap[dir] == 0 {
			dirMap[dir] = os.FileMode(0755)
		}

		info := file.FileInfo()
		mode := info.Mode() | ModeMask

		if info.IsDir() {
			dirMap[dir] = mode
		} else if mode&os.ModeSymlink > 0 {
			var linkname []byte

			err := func() error {
				reader, err := file.Open()
				if err != nil {
					return err
				}
				defer reader.Close()

				linkname, err = ioutil.ReadAll(reader)
				if err != nil {
					return err
				}
				return nil
			}()

			if err != nil {
				return nil, err
			}

			Symlinks = append(Symlinks, &Symlink{
				Path: file.Name,
				Dest: string(linkname),
				Mode: uint32(mode),
			})
		} else {
			Size := info.Size()

			Files = append(Files, &File{
				Path:   file.Name,
				Mode:   uint32(mode),
				Size:   Size,
				Offset: TotalOffset,
			})

			TotalOffset += Size
		}
	}

	for dirPath, dirMode := range dirMap {
		Dirs = append(Dirs, &Dir{
			Path: dirPath,
			Mode: uint32(dirMode),
		})
	}

	container := &Container{Size: TotalOffset, Dirs: Dirs, Symlinks: Symlinks, Files: Files}
	return container, nil
}

// Stats return a human-readable summary of the contents of a container
func (container *Container) Stats() string {
	return fmt.Sprintf("%d files, %d dirs, %d symlinks",
		len(container.Files), len(container.Dirs), len(container.Symlinks))
}
<|endoftext|>


<commit_before>package osin

import (
	"encoding/base64"
	"strings"

	"github.com/satori/go.uuid"
)

// AuthorizeTokenGenDefault is the default authorization token generator
type AuthorizeTokenGenDefault struct {
}

func removePadding(token string) string {
	return strings.TrimRight(token, "=")
}

// GenerateAuthorizeToken generates a base64-encoded UUID code
func (a *AuthorizeTokenGenDefault) GenerateAuthorizeToken(data *AuthorizeData) (ret string, err error) {
	token := uuid.NewV4()
	return removePadding(base64.URLEncoding.EncodeToString(token.Bytes())), nil
}

// AccessTokenGenDefault is the default authorization token generator
type AccessTokenGenDefault struct {
}

// GenerateAccessToken generates base64-encoded UUID access and refresh tokens
func (a *AccessTokenGenDefault) GenerateAccessToken(data *AccessData, generaterefresh bool) (accesstoken string, refreshtoken string, err error) {
	token := uuid.NewV4()
	accesstoken = removePadding(base64.URLEncoding.EncodeToString(token.Bytes()))

	if generaterefresh {
		rtoken := uuid.NewV4()
		refreshtoken = removePadding(base64.URLEncoding.EncodeToString(rtoken.Bytes()))
	}
	return
}
<commit_msg>Switch uuid to original implementation<commit_after>package osin

import (
	"encoding/base64"
	"strings"

	"github.com/pborman/uuid"
)

// AuthorizeTokenGenDefault is the default authorization token generator
type AuthorizeTokenGenDefault struct {
}

func removePadding(token string) string {
	return strings.TrimRight(token, "=")
}

// GenerateAuthorizeToken generates a base64-encoded UUID code
func (a *AuthorizeTokenGenDefault) GenerateAuthorizeToken(data *AuthorizeData) (ret string, err error) {
	token := uuid.NewRandom()
	return removePadding(base64.URLEncoding.EncodeToString([]byte(token))), nil
}

// AccessTokenGenDefault is the default authorization token generator
type AccessTokenGenDefault struct {
}

// GenerateAccessToken generates base64-encoded UUID access and refresh tokens
func (a *AccessTokenGenDefault) GenerateAccessToken(data *AccessData, generaterefresh bool) (accesstoken string, refreshtoken string, err error) {
	token := uuid.NewRandom()
	return removePadding(base64.URLEncoding.EncodeToString([]byte(token))), nil

	if generaterefresh {
		rtoken := uuid.NewRandom()
		refreshtoken = removePadding(base64.URLEncoding.EncodeToString([]byte(rtoken)))
	}
	return
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package internalbootstrap

import (
	fcv1a1 "k8s.io/api/flowcontrol/v1alpha1"
	"k8s.io/apimachinery/pkg/conversion"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap"
	"k8s.io/kubernetes/pkg/apis/flowcontrol"
	"k8s.io/kubernetes/pkg/apis/flowcontrol/install"
)

// MandatoryFlowSchemas holds the untyped renditions of the mandatory
// flow schemas.  In this map the key is the schema's name and the
// value is the `*FlowSchema`.  Nobody should mutate anything
// reachable from this map.
var MandatoryFlowSchemas = internalizeFSes(bootstrap.MandatoryFlowSchemas)

// MandatoryPriorityLevels holds the untyped renditions of the
// mandatory priority level configuration objects.  In this map the
// key is the object's name and the value is the
// `*PriorityLevelConfiguration`.  Nobody should mutate anything
// reachable from this map.
var MandatoryPriorityLevelConfigurations = internalizePLs(bootstrap.MandatoryPriorityLevelConfigurations)

func NewAPFScheme() *runtime.Scheme {
	scheme := runtime.NewScheme()
	install.Install(scheme)
	return scheme
}

func internalizeFSes(exts []*fcv1a1.FlowSchema) map[string]*flowcontrol.FlowSchema {
	ans := make(map[string]*flowcontrol.FlowSchema, len(exts))
	converter := NewAPFScheme().Converter()
	for _, ext := range exts {
		var untyped flowcontrol.FlowSchema
		err := converter.Convert(ext, &untyped, 0, &conversion.Meta{})
		if err != nil {
			panic(err)
		}
		ans[ext.Name] = &untyped
	}
	return ans
}

func internalizePLs(exts []*fcv1a1.PriorityLevelConfiguration) map[string]*flowcontrol.PriorityLevelConfiguration {
	ans := make(map[string]*flowcontrol.PriorityLevelConfiguration, len(exts))
	converter := NewAPFScheme().Converter()
	for _, ext := range exts {
		var untyped flowcontrol.PriorityLevelConfiguration
		err := converter.Convert(ext, &untyped, 0, &conversion.Meta{})
		if err != nil {
			panic(err)
		}
		ans[ext.Name] = &untyped
	}
	return ans
}
<commit_msg>Fixup comments in internalbootstrap<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package internalbootstrap

import (
	fcv1a1 "k8s.io/api/flowcontrol/v1alpha1"
	"k8s.io/apimachinery/pkg/conversion"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap"
	"k8s.io/kubernetes/pkg/apis/flowcontrol"
	"k8s.io/kubernetes/pkg/apis/flowcontrol/install"
)

// MandatoryFlowSchemas holds the untyped renditions of the mandatory
// flow schemas.  In this map the key is the schema's name and the
// value is the `*FlowSchema`.  Nobody should mutate anything
// reachable from this map.
var MandatoryFlowSchemas = internalizeFSes(bootstrap.MandatoryFlowSchemas)

// MandatoryPriorityLevelConfigurations holds the untyped renditions of the
// mandatory priority level configuration objects.  In this map the
// key is the object's name and the value is the
// `*PriorityLevelConfiguration`.  Nobody should mutate anything
// reachable from this map.
var MandatoryPriorityLevelConfigurations = internalizePLs(bootstrap.MandatoryPriorityLevelConfigurations)

// NewAPFScheme constructs and returns a Scheme configured to handle
// the API object types that are used to configure API Priority and
// Fairness
func NewAPFScheme() *runtime.Scheme {
	scheme := runtime.NewScheme()
	install.Install(scheme)
	return scheme
}

func internalizeFSes(exts []*fcv1a1.FlowSchema) map[string]*flowcontrol.FlowSchema {
	ans := make(map[string]*flowcontrol.FlowSchema, len(exts))
	converter := NewAPFScheme().Converter()
	for _, ext := range exts {
		var untyped flowcontrol.FlowSchema
		err := converter.Convert(ext, &untyped, 0, &conversion.Meta{})
		if err != nil {
			panic(err)
		}
		ans[ext.Name] = &untyped
	}
	return ans
}

func internalizePLs(exts []*fcv1a1.PriorityLevelConfiguration) map[string]*flowcontrol.PriorityLevelConfiguration {
	ans := make(map[string]*flowcontrol.PriorityLevelConfiguration, len(exts))
	converter := NewAPFScheme().Converter()
	for _, ext := range exts {
		var untyped flowcontrol.PriorityLevelConfiguration
		err := converter.Convert(ext, &untyped, 0, &conversion.Meta{})
		if err != nil {
			panic(err)
		}
		ans[ext.Name] = &untyped
	}
	return ans
}
<|endoftext|>


<commit_before>// Package streaminput provides a dbus Input plugin that can integrate with external system
package streaminput

import (
	"bufio"

	"github.com/funkygao/dbus/engine"
	"github.com/funkygao/dbus/pkg/model"
	"github.com/funkygao/golib/pipestream"
	conf "github.com/funkygao/jsconf"
)

var (
	_ engine.Input = &StreamInput{}
)

// StreamInput is a dbus Input plugin that can integrate with external system.
type StreamInput struct {
	cmdAndArgs []string
}

func (this *StreamInput) Init(config *conf.Conf) {
	this.cmdAndArgs = config.StringList("cmd", nil)
	if len(this.cmdAndArgs) == 0 {
		panic("empty cmd in config")
	}
}

func (this *StreamInput) OnAck(pack *engine.Packet) error {
	return nil
}

func (this *StreamInput) Stop(r engine.InputRunner) {}

func (this *StreamInput) Run(r engine.InputRunner, h engine.PluginHelper) error {
	cmd := pipestream.New(this.cmdAndArgs[0], this.cmdAndArgs[1:]...)
	if err := cmd.Open(); err != nil {
		return err
	}
	defer cmd.Close()

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Bytes()

		pack, ok := <-r.InChan()
		if !ok {
			// dbus shutdown
			break
		}

		pack.Payload = model.Bytes(line)
		r.Inject(pack)
	}

	return nil
}

func init() {
	engine.RegisterPlugin("StreamInput", func() engine.Plugin {
		return new(StreamInput)
	})
}
<commit_msg>add exchange support<commit_after>// Package streaminput provides a dbus Input plugin that can integrate with external system
package streaminput

import (
	"bufio"

	"github.com/funkygao/dbus/engine"
	"github.com/funkygao/dbus/pkg/model"
	"github.com/funkygao/golib/pipestream"
	conf "github.com/funkygao/jsconf"
)

var (
	_ engine.Input = &StreamInput{}
)

// StreamInput is a dbus Input plugin that can integrate with external system.
type StreamInput struct {
	cmdAndArgs []string
}

func (this *StreamInput) Init(config *conf.Conf) {
	this.cmdAndArgs = config.StringList("cmd", nil)
	if len(this.cmdAndArgs) == 0 {
		panic("empty cmd in config")
	}
}

func (this *StreamInput) OnAck(pack *engine.Packet) error {
	return nil
}

func (this *StreamInput) Stop(r engine.InputRunner) {}

func (this *StreamInput) Run(r engine.InputRunner, h engine.PluginHelper) error {
	cmd := pipestream.New(this.cmdAndArgs[0], this.cmdAndArgs[1:]...)
	if err := cmd.Open(); err != nil {
		return err
	}
	defer cmd.Close()

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Bytes()

		pack, ok := <-r.Exchange().InChan()
		if !ok {
			// dbus shutdown
			break
		}

		pack.Payload = model.Bytes(line)
		r.Exchange().Inject(pack)
	}

	return nil
}

func init() {
	engine.RegisterPlugin("StreamInput", func() engine.Plugin {
		return new(StreamInput)
	})
}
<|endoftext|>


<commit_before>// Code generated by github.com/ungerik/pkgreflect DO NOT EDIT.

package acl

import "reflect"

var Types = map[string]reflect.Type{
	"ACLAddReplace":                reflect.TypeOf((*ACLAddReplace)(nil)).Elem(),
	"ACLAddReplaceReply":           reflect.TypeOf((*ACLAddReplaceReply)(nil)).Elem(),
	"ACLDel":                       reflect.TypeOf((*ACLDel)(nil)).Elem(),
	"ACLDelReply":                  reflect.TypeOf((*ACLDelReply)(nil)).Elem(),
	"ACLDetails":                   reflect.TypeOf((*ACLDetails)(nil)).Elem(),
	"ACLDump":                      reflect.TypeOf((*ACLDump)(nil)).Elem(),
	"ACLInterfaceAddDel":           reflect.TypeOf((*ACLInterfaceAddDel)(nil)).Elem(),
	"ACLInterfaceAddDelReply":      reflect.TypeOf((*ACLInterfaceAddDelReply)(nil)).Elem(),
	"ACLInterfaceListDetails":      reflect.TypeOf((*ACLInterfaceListDetails)(nil)).Elem(),
	"ACLInterfaceListDump":         reflect.TypeOf((*ACLInterfaceListDump)(nil)).Elem(),
	"ACLInterfaceSetACLList":       reflect.TypeOf((*ACLInterfaceSetACLList)(nil)).Elem(),
	"ACLInterfaceSetACLListReply":  reflect.TypeOf((*ACLInterfaceSetACLListReply)(nil)).Elem(),
	"ACLPluginControlPing":         reflect.TypeOf((*ACLPluginControlPing)(nil)).Elem(),
	"ACLPluginControlPingReply":    reflect.TypeOf((*ACLPluginControlPingReply)(nil)).Elem(),
	"ACLPluginGetVersion":          reflect.TypeOf((*ACLPluginGetVersion)(nil)).Elem(),
	"ACLPluginGetVersionReply":     reflect.TypeOf((*ACLPluginGetVersionReply)(nil)).Elem(),
	"ACLRule":                      reflect.TypeOf((*ACLRule)(nil)).Elem(),
	"MacipACLAdd":                  reflect.TypeOf((*MacipACLAdd)(nil)).Elem(),
	"MacipACLAddReplace":           reflect.TypeOf((*MacipACLAddReplace)(nil)).Elem(),
	"MacipACLAddReplaceReply":      reflect.TypeOf((*MacipACLAddReplaceReply)(nil)).Elem(),
	"MacipACLAddReply":             reflect.TypeOf((*MacipACLAddReply)(nil)).Elem(),
	"MacipACLDel":                  reflect.TypeOf((*MacipACLDel)(nil)).Elem(),
	"MacipACLDelReply":             reflect.TypeOf((*MacipACLDelReply)(nil)).Elem(),
	"MacipACLDetails":              reflect.TypeOf((*MacipACLDetails)(nil)).Elem(),
	"MacipACLDump":                 reflect.TypeOf((*MacipACLDump)(nil)).Elem(),
	"MacipACLInterfaceAddDel":      reflect.TypeOf((*MacipACLInterfaceAddDel)(nil)).Elem(),
	"MacipACLInterfaceAddDelReply": reflect.TypeOf((*MacipACLInterfaceAddDelReply)(nil)).Elem(),
	"MacipACLInterfaceGet":         reflect.TypeOf((*MacipACLInterfaceGet)(nil)).Elem(),
	"MacipACLInterfaceGetReply":    reflect.TypeOf((*MacipACLInterfaceGetReply)(nil)).Elem(),
	"MacipACLInterfaceListDetails": reflect.TypeOf((*MacipACLInterfaceListDetails)(nil)).Elem(),
	"MacipACLInterfaceListDump":    reflect.TypeOf((*MacipACLInterfaceListDump)(nil)).Elem(),
	"MacipACLRule":                 reflect.TypeOf((*MacipACLRule)(nil)).Elem(),
}

var Functions = map[string]reflect.Value{
	"ConfigACL":                       reflect.ValueOf(ConfigACL),
	"ConfigIfACL":                     reflect.ValueOf(ConfigIfACL),
	"DelACL":                          reflect.ValueOf(DelACL),
	"DelIfACL":                        reflect.ValueOf(DelIfACL),
	"DumpACL":                         reflect.ValueOf(DumpACL),
	"NewACLAddReplace":                reflect.ValueOf(NewACLAddReplace),
	"NewACLAddReplaceReply":           reflect.ValueOf(NewACLAddReplaceReply),
	"NewACLDel":                       reflect.ValueOf(NewACLDel),
	"NewACLDelReply":                  reflect.ValueOf(NewACLDelReply),
	"NewACLDetails":                   reflect.ValueOf(NewACLDetails),
	"NewACLDump":                      reflect.ValueOf(NewACLDump),
	"NewACLInterfaceAddDel":           reflect.ValueOf(NewACLInterfaceAddDel),
	"NewACLInterfaceAddDelReply":      reflect.ValueOf(NewACLInterfaceAddDelReply),
	"NewACLInterfaceListDetails":      reflect.ValueOf(NewACLInterfaceListDetails),
	"NewACLInterfaceListDump":         reflect.ValueOf(NewACLInterfaceListDump),
	"NewACLInterfaceSetACLList":       reflect.ValueOf(NewACLInterfaceSetACLList),
	"NewACLInterfaceSetACLListReply":  reflect.ValueOf(NewACLInterfaceSetACLListReply),
	"NewACLPluginControlPing":         reflect.ValueOf(NewACLPluginControlPing),
	"NewACLPluginControlPingReply":    reflect.ValueOf(NewACLPluginControlPingReply),
	"NewACLPluginGetVersion":          reflect.ValueOf(NewACLPluginGetVersion),
	"NewACLPluginGetVersionReply":     reflect.ValueOf(NewACLPluginGetVersionReply),
	"NewMacipACLAdd":                  reflect.ValueOf(NewMacipACLAdd),
	"NewMacipACLAddReplace":           reflect.ValueOf(NewMacipACLAddReplace),
	"NewMacipACLAddReplaceReply":      reflect.ValueOf(NewMacipACLAddReplaceReply),
	"NewMacipACLAddReply":             reflect.ValueOf(NewMacipACLAddReply),
	"NewMacipACLDel":                  reflect.ValueOf(NewMacipACLDel),
	"NewMacipACLDelReply":             reflect.ValueOf(NewMacipACLDelReply),
	"NewMacipACLDetails":              reflect.ValueOf(NewMacipACLDetails),
	"NewMacipACLDump":                 reflect.ValueOf(NewMacipACLDump),
	"NewMacipACLInterfaceAddDel":      reflect.ValueOf(NewMacipACLInterfaceAddDel),
	"NewMacipACLInterfaceAddDelReply": reflect.ValueOf(NewMacipACLInterfaceAddDelReply),
	"NewMacipACLInterfaceGet":         reflect.ValueOf(NewMacipACLInterfaceGet),
	"NewMacipACLInterfaceGetReply":    reflect.ValueOf(NewMacipACLInterfaceGetReply),
	"NewMacipACLInterfaceListDetails": reflect.ValueOf(NewMacipACLInterfaceListDetails),
	"NewMacipACLInterfaceListDump":    reflect.ValueOf(NewMacipACLInterfaceListDump),
}

var Variables = map[string]reflect.Value{}

var Consts = map[string]reflect.Value{
	"VlAPIVersion": reflect.ValueOf(VlAPIVersion),
}
<commit_msg>Regenerate pkgreflect<commit_after>// Code generated by github.com/ungerik/pkgreflect DO NOT EDIT.

package acl

import "reflect"

var Types = map[string]reflect.Type{
	"ACLAddReplace":                reflect.TypeOf((*ACLAddReplace)(nil)).Elem(),
	"ACLAddReplaceReply":           reflect.TypeOf((*ACLAddReplaceReply)(nil)).Elem(),
	"ACLDel":                       reflect.TypeOf((*ACLDel)(nil)).Elem(),
	"ACLDelReply":                  reflect.TypeOf((*ACLDelReply)(nil)).Elem(),
	"ACLDetails":                   reflect.TypeOf((*ACLDetails)(nil)).Elem(),
	"ACLDump":                      reflect.TypeOf((*ACLDump)(nil)).Elem(),
	"ACLInterfaceAddDel":           reflect.TypeOf((*ACLInterfaceAddDel)(nil)).Elem(),
	"ACLInterfaceAddDelReply":      reflect.TypeOf((*ACLInterfaceAddDelReply)(nil)).Elem(),
	"ACLInterfaceListDetails":      reflect.TypeOf((*ACLInterfaceListDetails)(nil)).Elem(),
	"ACLInterfaceListDump":         reflect.TypeOf((*ACLInterfaceListDump)(nil)).Elem(),
	"ACLInterfaceSetACLList":       reflect.TypeOf((*ACLInterfaceSetACLList)(nil)).Elem(),
	"ACLInterfaceSetACLListReply":  reflect.TypeOf((*ACLInterfaceSetACLListReply)(nil)).Elem(),
	"ACLPluginControlPing":         reflect.TypeOf((*ACLPluginControlPing)(nil)).Elem(),
	"ACLPluginControlPingReply":    reflect.TypeOf((*ACLPluginControlPingReply)(nil)).Elem(),
	"ACLPluginGetVersion":          reflect.TypeOf((*ACLPluginGetVersion)(nil)).Elem(),
	"ACLPluginGetVersionReply":     reflect.TypeOf((*ACLPluginGetVersionReply)(nil)).Elem(),
	"ACLRule":                      reflect.TypeOf((*ACLRule)(nil)).Elem(),
	"MacipACLAdd":                  reflect.TypeOf((*MacipACLAdd)(nil)).Elem(),
	"MacipACLAddReplace":           reflect.TypeOf((*MacipACLAddReplace)(nil)).Elem(),
	"MacipACLAddReplaceReply":      reflect.TypeOf((*MacipACLAddReplaceReply)(nil)).Elem(),
	"MacipACLAddReply":             reflect.TypeOf((*MacipACLAddReply)(nil)).Elem(),
	"MacipACLDel":                  reflect.TypeOf((*MacipACLDel)(nil)).Elem(),
	"MacipACLDelReply":             reflect.TypeOf((*MacipACLDelReply)(nil)).Elem(),
	"MacipACLDetails":              reflect.TypeOf((*MacipACLDetails)(nil)).Elem(),
	"MacipACLDump":                 reflect.TypeOf((*MacipACLDump)(nil)).Elem(),
	"MacipACLInterfaceAddDel":      reflect.TypeOf((*MacipACLInterfaceAddDel)(nil)).Elem(),
	"MacipACLInterfaceAddDelReply": reflect.TypeOf((*MacipACLInterfaceAddDelReply)(nil)).Elem(),
	"MacipACLInterfaceGet":         reflect.TypeOf((*MacipACLInterfaceGet)(nil)).Elem(),
	"MacipACLInterfaceGetReply":    reflect.TypeOf((*MacipACLInterfaceGetReply)(nil)).Elem(),
	"MacipACLInterfaceListDetails": reflect.TypeOf((*MacipACLInterfaceListDetails)(nil)).Elem(),
	"MacipACLInterfaceListDump":    reflect.TypeOf((*MacipACLInterfaceListDump)(nil)).Elem(),
	"MacipACLRule":                 reflect.TypeOf((*MacipACLRule)(nil)).Elem(),
}

var Functions = map[string]reflect.Value{
	"NewACLAddReplace":                reflect.ValueOf(NewACLAddReplace),
	"NewACLAddReplaceReply":           reflect.ValueOf(NewACLAddReplaceReply),
	"NewACLDel":                       reflect.ValueOf(NewACLDel),
	"NewACLDelReply":                  reflect.ValueOf(NewACLDelReply),
	"NewACLDetails":                   reflect.ValueOf(NewACLDetails),
	"NewACLDump":                      reflect.ValueOf(NewACLDump),
	"NewACLInterfaceAddDel":           reflect.ValueOf(NewACLInterfaceAddDel),
	"NewACLInterfaceAddDelReply":      reflect.ValueOf(NewACLInterfaceAddDelReply),
	"NewACLInterfaceListDetails":      reflect.ValueOf(NewACLInterfaceListDetails),
	"NewACLInterfaceListDump":         reflect.ValueOf(NewACLInterfaceListDump),
	"NewACLInterfaceSetACLList":       reflect.ValueOf(NewACLInterfaceSetACLList),
	"NewACLInterfaceSetACLListReply":  reflect.ValueOf(NewACLInterfaceSetACLListReply),
	"NewACLPluginControlPing":         reflect.ValueOf(NewACLPluginControlPing),
	"NewACLPluginControlPingReply":    reflect.ValueOf(NewACLPluginControlPingReply),
	"NewACLPluginGetVersion":          reflect.ValueOf(NewACLPluginGetVersion),
	"NewACLPluginGetVersionReply":     reflect.ValueOf(NewACLPluginGetVersionReply),
	"NewMacipACLAdd":                  reflect.ValueOf(NewMacipACLAdd),
	"NewMacipACLAddReplace":           reflect.ValueOf(NewMacipACLAddReplace),
	"NewMacipACLAddReplaceReply":      reflect.ValueOf(NewMacipACLAddReplaceReply),
	"NewMacipACLAddReply":             reflect.ValueOf(NewMacipACLAddReply),
	"NewMacipACLDel":                  reflect.ValueOf(NewMacipACLDel),
	"NewMacipACLDelReply":             reflect.ValueOf(NewMacipACLDelReply),
	"NewMacipACLDetails":              reflect.ValueOf(NewMacipACLDetails),
	"NewMacipACLDump":                 reflect.ValueOf(NewMacipACLDump),
	"NewMacipACLInterfaceAddDel":      reflect.ValueOf(NewMacipACLInterfaceAddDel),
	"NewMacipACLInterfaceAddDelReply": reflect.ValueOf(NewMacipACLInterfaceAddDelReply),
	"NewMacipACLInterfaceGet":         reflect.ValueOf(NewMacipACLInterfaceGet),
	"NewMacipACLInterfaceGetReply":    reflect.ValueOf(NewMacipACLInterfaceGetReply),
	"NewMacipACLInterfaceListDetails": reflect.ValueOf(NewMacipACLInterfaceListDetails),
	"NewMacipACLInterfaceListDump":    reflect.ValueOf(NewMacipACLInterfaceListDump),
}

var Variables = map[string]reflect.Value{}

var Consts = map[string]reflect.Value{
	"VlAPIVersion": reflect.ValueOf(VlAPIVersion),
}
<|endoftext|>


<commit_before>/* Coordination between a collection of chunks. */
package db

import (
	"errors"
	"fmt"
	"github.com/HouzuoGuo/tiedot/chunk"
	"github.com/HouzuoGuo/tiedot/chunkfile"
	"github.com/HouzuoGuo/tiedot/tdlog"
	"math/rand"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"sync"
)

type Col struct {
	BaseDir       string // Collection dir path
	Chunks        []*chunk.ChunkCol
	ChunkMutexes  []*sync.Mutex // Synchronize access to chunks
	NewChunkMutex sync.Mutex    // Synchronize creation of new chunk
	NumChunks     uint64        // Total number of chunks
}

// Open a collection (made of chunks).
func OpenCol(baseDir string) (col *Col, err error) {
	// Create the directory if it does not yet exist
	if err = os.MkdirAll(baseDir, 0700); err != nil {
		return
	}
	col = &Col{BaseDir: baseDir, NewChunkMutex: sync.Mutex{},
		Chunks: make([]*chunk.ChunkCol, 0), ChunkMutexes: make([]*sync.Mutex, 0)}
	// Walk the collection directory and look for how many chunks there are
	maxChunk := 0
	walker := func(currPath string, info os.FileInfo, err2 error) error {
		if err2 != nil {
			// log and skip the error
			tdlog.Error(err)
			return nil
		}
		if info.IsDir() {
			// According to the directory name - chunk number, look for how many chunks there are
			if chunkNum, err := strconv.Atoi(info.Name()); err == nil {
				if chunkNum > maxChunk {
					maxChunk = chunkNum
				}
			} else {
				tdlog.Errorf("Directory %s does not seem to belong to tiedot, it is ignored", info.Name())
			}
			return nil
		}
		return nil
	}
	if err = filepath.Walk(baseDir, walker); err != nil {
		return
	}
	col.NumChunks = uint64(maxChunk) + 1
	// Open all chunks - there is at least one chunk
	tdlog.Printf("Opening %d chunks in collection %s", maxChunk+1, baseDir)
	for i := 0; i <= maxChunk; i++ {
		var oneChunk *chunk.ChunkCol
		oneChunk, err = chunk.OpenChunk(uint64(i), path.Join(baseDir, strconv.Itoa(i)))
		col.Chunks = append(col.Chunks, oneChunk)
		col.ChunkMutexes = append(col.ChunkMutexes, new(sync.Mutex))
	}
	return
}

// Insert a new document, return new document's ID.
func (col *Col) Insert(doc interface{}) (id uint64, err error) {
	// Try to insert the doc into a random chunk
	randChunkNum := uint64(rand.Int63n(int64(col.NumChunks)))
	randChunk := col.Chunks[randChunkNum]
	randChunkMutexes := col.ChunkMutexes[randChunkNum]
	randChunkMutexes.Lock()
	idInChunk, outOfSpace, err := randChunk.Insert(doc)
	if !outOfSpace {
		randChunkMutexes.Unlock()
		id = randChunkNum*chunkfile.COL_FILE_SIZE + idInChunk
		return
	}
	randChunkMutexes.Unlock()
	// If the random chunk was full, try again with the last chunk
	lastChunk := col.Chunks[col.NumChunks-1]
	lastChunkMutexes := col.ChunkMutexes[col.NumChunks-1]
	lastChunkMutexes.Lock()
	idInChunk, outOfSpace, err = lastChunk.Insert(doc)
	if !outOfSpace {
		id = (col.NumChunks-1)*chunkfile.COL_FILE_SIZE + idInChunk
		lastChunkMutexes.Unlock()
		return
	}
	lastChunkMutexes.Unlock()
	// If the last chunk is full, make a new chunk
	col.NewChunkMutex.Lock()
	tdlog.Printf("Going to create a new chunk (number %d) in collection %s", col.NumChunks, col.BaseDir)
	newChunk, err := chunk.OpenChunk(col.NumChunks, path.Join(col.BaseDir, strconv.Itoa(int(col.NumChunks))))
	col.NumChunks += 1
	if err != nil {
		col.NewChunkMutex.Unlock()
		return
	}
	// Put the new chunk into col structures
	col.Chunks = append(col.Chunks, newChunk)
	col.ChunkMutexes = append(col.ChunkMutexes, new(sync.Mutex))
	col.NewChunkMutex.Unlock()
	// Now there is a new chunk, let us try again
	return col.Insert(doc)
}

// Read document at the given ID.
func (col *Col) Read(id uint64, doc interface{}) (err error) {
	chunkNum := id / chunkfile.COL_FILE_SIZE
	if chunkNum >= col.NumChunks {
		return errors.New(fmt.Sprintf("Document %d does not exist in %s - out of bound chunk", id, col.BaseDir))
	}
	chunk := col.Chunks[chunkNum]
	chunkMutex := col.ChunkMutexes[chunkNum]
	chunkMutex.Lock()
	err = chunk.Read(id, doc)
	chunkMutex.Unlock()
	return
}
<commit_msg>basic CRUD in db/col.go<commit_after>/* Coordination between a collection of chunks. */
package db

import (
	"errors"
	"fmt"
	"github.com/HouzuoGuo/tiedot/chunk"
	"github.com/HouzuoGuo/tiedot/chunkfile"
	"github.com/HouzuoGuo/tiedot/tdlog"
	"math/rand"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"sync"
)

type Col struct {
	BaseDir       string // Collection dir path
	Chunks        []*chunk.ChunkCol
	ChunkMutexes  []*sync.Mutex // Synchronize access to chunks
	NewChunkMutex sync.Mutex    // Synchronize creation of new chunk
	NumChunks     uint64        // Total number of chunks
}

// Open a collection (made of chunks).
func OpenCol(baseDir string) (col *Col, err error) {
	// Create the directory if it does not yet exist
	if err = os.MkdirAll(baseDir, 0700); err != nil {
		return
	}
	col = &Col{BaseDir: baseDir, NewChunkMutex: sync.Mutex{},
		Chunks: make([]*chunk.ChunkCol, 0), ChunkMutexes: make([]*sync.Mutex, 0)}
	// Walk the collection directory and look for how many chunks there are
	maxChunk := 0
	walker := func(currPath string, info os.FileInfo, err2 error) error {
		if err2 != nil {
			// log and skip the error
			tdlog.Error(err)
			return nil
		}
		if info.IsDir() {
			// According to the directory name - chunk number, look for how many chunks there are
			if chunkNum, err := strconv.Atoi(info.Name()); err == nil {
				if chunkNum > maxChunk {
					maxChunk = chunkNum
				}
			} else {
				tdlog.Errorf("Directory %s does not seem to belong to tiedot, it is ignored", info.Name())
			}
			return nil
		}
		return nil
	}
	if err = filepath.Walk(baseDir, walker); err != nil {
		return
	}
	col.NumChunks = uint64(maxChunk) + 1
	// Open all chunks - there is at least one chunk
	tdlog.Printf("Opening %d chunks in collection %s", maxChunk+1, baseDir)
	for i := 0; i <= maxChunk; i++ {
		var oneChunk *chunk.ChunkCol
		oneChunk, err = chunk.OpenChunk(uint64(i), path.Join(baseDir, strconv.Itoa(i)))
		col.Chunks = append(col.Chunks, oneChunk)
		col.ChunkMutexes = append(col.ChunkMutexes, new(sync.Mutex))
	}
	return
}

// Insert a new document, return new document's ID.
func (col *Col) Insert(doc interface{}) (id uint64, err error) {
	// Try to insert the doc into a random chunk
	randChunkNum := uint64(rand.Int63n(int64(col.NumChunks)))
	randChunk := col.Chunks[randChunkNum]
	randChunkMutexes := col.ChunkMutexes[randChunkNum]
	randChunkMutexes.Lock()
	idInChunk, outOfSpace, err := randChunk.Insert(doc)
	if !outOfSpace {
		randChunkMutexes.Unlock()
		id = randChunkNum*chunkfile.COL_FILE_SIZE + idInChunk
		return
	}
	randChunkMutexes.Unlock()
	// If the random chunk was full, try again with the last chunk
	lastChunk := col.Chunks[col.NumChunks-1]
	lastChunkMutexes := col.ChunkMutexes[col.NumChunks-1]
	lastChunkMutexes.Lock()
	idInChunk, outOfSpace, err = lastChunk.Insert(doc)
	if !outOfSpace {
		id = (col.NumChunks-1)*chunkfile.COL_FILE_SIZE + idInChunk
		lastChunkMutexes.Unlock()
		return
	}
	lastChunkMutexes.Unlock()
	// If the last chunk is full, make a new chunk
	col.NewChunkMutex.Lock()
	tdlog.Printf("Going to create a new chunk (number %d) in collection %s", col.NumChunks, col.BaseDir)
	newChunk, err := chunk.OpenChunk(col.NumChunks, path.Join(col.BaseDir, strconv.Itoa(int(col.NumChunks))))
	col.NumChunks += 1
	if err != nil {
		col.NewChunkMutex.Unlock()
		return
	}
	// Put the new chunk into col structures
	col.Chunks = append(col.Chunks, newChunk)
	col.ChunkMutexes = append(col.ChunkMutexes, new(sync.Mutex))
	col.NewChunkMutex.Unlock()
	// Now there is a new chunk, let us try again
	return col.Insert(doc)
}

// Read document at the given ID.
func (col *Col) Read(id uint64, doc interface{}) (err error) {
	chunkNum := id / chunkfile.COL_FILE_SIZE
	if chunkNum >= col.NumChunks {
		return errors.New(fmt.Sprintf("Document %d does not exist in %s - out of bound chunk", id, col.BaseDir))
	}
	chunkDocID := id % chunkfile.COL_FILE_SIZE
	chunk := col.Chunks[chunkNum]
	chunkMutex := col.ChunkMutexes[chunkNum]
	chunkMutex.Lock()
	err = chunk.Read(chunkDocID, doc)
	chunkMutex.Unlock()
	return
}

// Update a document, return its new ID.
func (col *Col) Update(id uint64, doc interface{}) (newID uint64, err error) {
	chunkNum := id / chunkfile.COL_FILE_SIZE
	if chunkNum >= col.NumChunks {
		err = errors.New(fmt.Sprintf("Document %d does not exist in %s - out of bound chunk", id, col.BaseDir))
		return
	}
	chunkDocID := id % chunkfile.COL_FILE_SIZE
	chunk := col.Chunks[chunkNum]
	chunkMutex := col.ChunkMutexes[chunkNum]
	chunkMutex.Lock()
	newID, outOfSpace, err := chunk.Update(chunkDocID, doc)
	chunkMutex.Unlock()
	if !outOfSpace {
		return
	}
	// The chunk does not have enough space for the updated document, let us put it somewhere else
	// The document has already been removed from its original chunk
	return col.Insert(doc)
}

// Delete a document by ID.
func (col *Col) Delete(id uint64) {
	chunkNum := id / chunkfile.COL_FILE_SIZE
	if chunkNum >= col.NumChunks {
		return
	}
	chunkDocID := id % chunkfile.COL_FILE_SIZE
	chunk := col.Chunks[chunkNum]
	chunkMutex := col.ChunkMutexes[chunkNum]
	chunkMutex.Lock()
	chunk.Delete(chunkDocID)
	chunkMutex.Unlock()
}
<|endoftext|>


<commit_before>// Copyright 2013 The gopp AUTHORS. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gopp

import (
	"errors"
	"fmt"
	"github.com/skelterjohn/debugtags"
	"io"
	"io/ioutil"
	"reflect"
	"strconv"
	"strings"
)

type DecoderFactory struct {
	g     Grammar
	start string
	types map[string]reflect.Type
}

func NewDecoderFactory(gopp string, start string) (df *DecoderFactory, err error) {
	df = &DecoderFactory{
		start: start,
		types: map[string]reflect.Type{},
	}
	ast, err := Parse(ByHandGrammar, "Grammar", []byte(gopp))
	if err != nil {
		return
	}
	sa := NewStructuredAST(ast)
	sa.RegisterType(RepeatZeroTerm{})
	sa.RegisterType(RepeatOneTerm{})
	sa.RegisterType(OptionalTerm{})
	sa.RegisterType(GroupTerm{})
	sa.RegisterType(RuleTerm{})
	sa.RegisterType(InlineRuleTerm{})
	sa.RegisterType(TagTerm{})
	sa.RegisterType(LiteralTerm{})
	err = sa.Decode(&df.g)
	if err != nil {
		return
	}
	return
}

func (df *DecoderFactory) RegisterType(x interface{}) {
	typ := reflect.TypeOf(x)
	df.types[typ.Name()] = typ
}

func (df *DecoderFactory) NewDecoder(r io.Reader) (d Decoder) {
	d = Decoder{
		DecoderFactory: df,
		Reader:         r,
	}
	return
}

type Decoder struct {
	*DecoderFactory
	io.Reader
}

func (d *Decoder) Decode(obj interface{}) (err error) {
	document, err := ioutil.ReadAll(d.Reader)
	if err != nil {
		return
	}
	ast, err := Parse(d.g, d.start, document)
	if err != nil {
		return
	}
	sa := NewStructuredAST(ast)
	sa.types = d.types
	err = sa.Decode(obj)
	if err != nil {
		return
	}
	return
}

func getTagValue(typ string, t Tag) (value string, ok bool) {
	prefix := typ + "="
	if strings.HasPrefix(string(t), prefix) {
		value = string(t[len(prefix):])
		ok = true
	}
	return
}

var _ = fmt.Println

type StructuredAST struct {
	ast   AST
	types map[string]reflect.Type
}

func NewStructuredAST(ast AST) (sa StructuredAST) {
	sa = StructuredAST{
		ast:   ast,
		types: map[string]reflect.Type{},
	}
	return
}

func (sa StructuredAST) RegisterType(x interface{}) {
	t := reflect.TypeOf(x)
	sa.types[t.Name()] = t
}

func (sa StructuredAST) Decode(obj interface{}) (err error) {
	return sa.decode([]Node(sa.ast), reflect.ValueOf(obj))
}

var dtr = debugtags.Tracer{Enabled: false}

func (sa StructuredAST) decode(node Node, v reflect.Value) (err error) {
	name := fmt.Sprintf("%T", v.Interface())
	dtr.In(name, node)
	defer func() {
		dtr.Out(name, v.Interface())
	}()

	typ := v.Type()

	// deref a pointer
	if typ.Kind() == reflect.Ptr {
		// but first check if it's nil and, if so, allocate
		if v.IsNil() {
			v.Elem().Set(reflect.New(typ.Elem()))
		}
		v = v.Elem()
		typ = typ.Elem()
	}

	switch typ.Kind() {
	// populate struct fields
	case reflect.Struct:
		// we've got a struct pointer - iterate through node looking for field= tags
		nodes, ok := node.([]Node)
		if !ok {
			err = errors.New("Need to populate struct via []Node with tags.")
			return
		}
		for i := range nodes {
			if tag, ok := nodes[i].(Tag); ok {
				if typName, isType := getTagValue("type", tag); isType {
					if typName != typ.Name() {
						err = fmt.Errorf("AST wants type %q, being decoded to type %q.", typName, typ.Name())
					}
				}

				if name, isField := getTagValue("field", tag); isField {
					// if we have a field tag, that indicates that the next node should be decided into the field with the given name.
					var fv reflect.Value
					fv, err = getField(v, name)
					if err != nil {
						return
					}

					if fv.Type().Kind() == reflect.Interface {
						//dtr.Println("field of interface")
						var pv reflect.Value
						pv, err = sa.makePointerWithType(nodes[i+1])
						if err != nil {
							return
						}
						err = sa.decode(nodes[i+1], pv.Elem())
						fv.Set(pv.Elem())
						if err != nil {
							return
						}
					} else {
						sa.decode(nodes[i+1], fv)
					}
				}
			}
		}

	// map things into slices
	case reflect.Slice:
		//fmt.Printf("Going into %s is\n", typ.Elem().Name())
		//printNode(node, 0)
		isInterfaceSlice := typ.Elem().Kind() == reflect.Interface
		// if isInterfaceSlice {
		// 	dtr.Println("slice of interface")
		// }
		nodes, ok := node.([]Node)
		if !ok {
			err = errors.New("Need to populate slice via []Node.")
			return
		}
		for _, n := range nodes {
			// create an addressable value to put in the slice
			ev := reflect.New(typ.Elem()).Elem()
			if isInterfaceSlice {
				var pv reflect.Value
				pv, err = sa.makePointerWithType(n)
				if err != nil {
					return
				}
				err = sa.decode(n, pv.Elem())
				ev.Set(pv.Elem())
				if err != nil {
					return
				}
			} else {
				err = sa.decode(n, ev)
				if err != nil {
					return
				}
			}
			// this is how append looks w/ reflect
			v.Set(reflect.Append(v, ev))
		}

	// symbols, literals, and tags go into strings
	case reflect.String:
		s := ""
		if s, err = getString(node); err == nil {
			v.SetString(s)
		} else {
			err = errors.New("Trying to store invalid type into string field.")
		}

	// and into ints
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		s := ""
		if s, err = getString(node); err == nil {
			var x int64
			if x, err = strconv.ParseInt(string(s), 0, 64); err == nil {
				v.SetInt(x)
			}
		} else {
			err = errors.New("Trying to store invalid type into integer field.")
		}

	// and also into uints
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		s := ""
		if s, err = getString(node); err == nil {
			var x uint64
			if x, err = strconv.ParseUint(string(s), 0, 64); err == nil {
				v.SetUint(x)
			}
		} else {
			err = errors.New("Trying to store invalid type into unsigned integer field.")
		}

	default:
		err = fmt.Errorf("Unanticipated type: %s.", typ.Name())
	}

	return
}

func (sa StructuredAST) makePointerWithType(node Node) (pointer reflect.Value, err error) {
	var ntag Tag
	nodes, ok := node.([]Node)
	if ok && len(nodes) != 0 {
		ntag, ok = nodes[0].(Tag)
		if ok {
			ok = strings.HasPrefix(string(ntag), "type=")
		}
	}
	if !ok {
		err = errors.New("Can only infer type from []Node with a type= tag.")
		return
	}
	typeName := ntag[len("type="):]
	typ := sa.types[string(typeName)]
	pointer = reflect.New(typ)
	return
}

func getField(v reflect.Value, field string) (fv reflect.Value, err error) {
	defer func() {
		if recover() != nil {
			err = fmt.Errorf("Type %s has no field named %q.", v.Type().Name(), field)
		}
	}()
	if field == "." {
		// . means to store the next level deeper in the same value
		fv = v
	} else {
		fv = v.FieldByName(field)
	}
	return
}

func getString(node Node) (s string, err error) {
	switch nn := node.(type) {
	case SymbolText:
		s = nn.Text
	case Tag:
		s = string(nn)
	case Literal:
		s = string(nn)
	default:
		return "", fmt.Errorf("Expected symbol, tag, or literal, but got %T", node)
	}
	ds, derr := descapeString(s)
	if derr == nil {
		s = ds
	}
	return
}
<commit_msg>check for error existence and abort, rather than error non-existence and continue<commit_after>// Copyright 2013 The gopp AUTHORS. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gopp

import (
	"errors"
	"fmt"
	"github.com/skelterjohn/debugtags"
	"io"
	"io/ioutil"
	"reflect"
	"strconv"
	"strings"
)

type DecoderFactory struct {
	g     Grammar
	start string
	types map[string]reflect.Type
}

func NewDecoderFactory(gopp string, start string) (df *DecoderFactory, err error) {
	df = &DecoderFactory{
		start: start,
		types: map[string]reflect.Type{},
	}
	ast, err := Parse(ByHandGrammar, "Grammar", []byte(gopp))
	if err != nil {
		return
	}
	sa := NewStructuredAST(ast)
	sa.RegisterType(RepeatZeroTerm{})
	sa.RegisterType(RepeatOneTerm{})
	sa.RegisterType(OptionalTerm{})
	sa.RegisterType(GroupTerm{})
	sa.RegisterType(RuleTerm{})
	sa.RegisterType(InlineRuleTerm{})
	sa.RegisterType(TagTerm{})
	sa.RegisterType(LiteralTerm{})
	err = sa.Decode(&df.g)
	if err != nil {
		return
	}
	return
}

func (df *DecoderFactory) RegisterType(x interface{}) {
	typ := reflect.TypeOf(x)
	df.types[typ.Name()] = typ
}

func (df *DecoderFactory) NewDecoder(r io.Reader) (d Decoder) {
	d = Decoder{
		DecoderFactory: df,
		Reader:         r,
	}
	return
}

type Decoder struct {
	*DecoderFactory
	io.Reader
}

func (d *Decoder) Decode(obj interface{}) (err error) {
	document, err := ioutil.ReadAll(d.Reader)
	if err != nil {
		return
	}
	ast, err := Parse(d.g, d.start, document)
	if err != nil {
		return
	}
	sa := NewStructuredAST(ast)
	sa.types = d.types
	err = sa.Decode(obj)
	if err != nil {
		return
	}
	return
}

func getTagValue(typ string, t Tag) (value string, ok bool) {
	prefix := typ + "="
	if strings.HasPrefix(string(t), prefix) {
		value = string(t[len(prefix):])
		ok = true
	}
	return
}

var _ = fmt.Println

type StructuredAST struct {
	ast   AST
	types map[string]reflect.Type
}

func NewStructuredAST(ast AST) (sa StructuredAST) {
	sa = StructuredAST{
		ast:   ast,
		types: map[string]reflect.Type{},
	}
	return
}

func (sa StructuredAST) RegisterType(x interface{}) {
	t := reflect.TypeOf(x)
	sa.types[t.Name()] = t
}

func (sa StructuredAST) Decode(obj interface{}) (err error) {
	return sa.decode([]Node(sa.ast), reflect.ValueOf(obj))
}

var dtr = debugtags.Tracer{Enabled: false}

func (sa StructuredAST) decode(node Node, v reflect.Value) (err error) {
	name := fmt.Sprintf("%T", v.Interface())
	dtr.In(name, node)
	defer func() {
		dtr.Out(name, v.Interface())
	}()

	typ := v.Type()

	// deref a pointer
	if typ.Kind() == reflect.Ptr {
		// but first check if it's nil and, if so, allocate
		if v.IsNil() {
			v.Elem().Set(reflect.New(typ.Elem()))
		}
		v = v.Elem()
		typ = typ.Elem()
	}

	switch typ.Kind() {
	// populate struct fields
	case reflect.Struct:
		// we've got a struct pointer - iterate through node looking for field= tags
		nodes, ok := node.([]Node)
		if !ok {
			err = errors.New("Need to populate struct via []Node with tags.")
			return
		}
		for i := range nodes {
			if tag, ok := nodes[i].(Tag); ok {
				if typName, isType := getTagValue("type", tag); isType {
					if typName != typ.Name() {
						err = fmt.Errorf("AST wants type %q, being decoded to type %q.", typName, typ.Name())
					}
				}

				if name, isField := getTagValue("field", tag); isField {
					// if we have a field tag, that indicates that the next node should be decided into the field with the given name.
					var fv reflect.Value
					fv, err = getField(v, name)
					if err != nil {
						return
					}

					if fv.Type().Kind() == reflect.Interface {
						//dtr.Println("field of interface")
						var pv reflect.Value
						pv, err = sa.makePointerWithType(nodes[i+1])
						if err != nil {
							return
						}
						err = sa.decode(nodes[i+1], pv.Elem())
						fv.Set(pv.Elem())
						if err != nil {
							return
						}
					} else {
						sa.decode(nodes[i+1], fv)
					}
				}
			}
		}

	// map things into slices
	case reflect.Slice:
		//fmt.Printf("Going into %s is\n", typ.Elem().Name())
		//printNode(node, 0)
		isInterfaceSlice := typ.Elem().Kind() == reflect.Interface
		// if isInterfaceSlice {
		// 	dtr.Println("slice of interface")
		// }
		nodes, ok := node.([]Node)
		if !ok {
			err = errors.New("Need to populate slice via []Node.")
			return
		}
		for _, n := range nodes {
			// create an addressable value to put in the slice
			ev := reflect.New(typ.Elem()).Elem()
			if isInterfaceSlice {
				var pv reflect.Value
				pv, err = sa.makePointerWithType(n)
				if err != nil {
					return
				}
				err = sa.decode(n, pv.Elem())
				ev.Set(pv.Elem())
				if err != nil {
					return
				}
			} else {
				err = sa.decode(n, ev)
				if err != nil {
					return
				}
			}
			// this is how append looks w/ reflect
			v.Set(reflect.Append(v, ev))
		}

	// symbols, literals, and tags go into strings
	case reflect.String:
		s := ""
		if s, err = getString(node); err != nil {
			err = errors.New("Trying to store invalid type into string field.")
			return
		}
		v.SetString(s)

	// and into ints
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		s := ""
		if s, err = getString(node); err != nil {
			err = errors.New("Trying to store invalid type into integer field.")
			return
		}
		var x int64
		if x, err = strconv.ParseInt(string(s), 0, 64); err != nil {
			return
		}
		v.SetInt(x)

	// and also into uints
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		s := ""
		if s, err = getString(node); err != nil {
			err = errors.New("Trying to store invalid type into unsigned integer field.")
			return
		}
		var x uint64
		if x, err = strconv.ParseUint(string(s), 0, 64); err != nil {
			return
		}
		v.SetUint(x)

	default:
		err = fmt.Errorf("Unanticipated type: %s.", typ.Name())
	}

	return
}

func (sa StructuredAST) makePointerWithType(node Node) (pointer reflect.Value, err error) {
	var ntag Tag
	nodes, ok := node.([]Node)
	if ok && len(nodes) != 0 {
		ntag, ok = nodes[0].(Tag)
		if ok {
			ok = strings.HasPrefix(string(ntag), "type=")
		}
	}
	if !ok {
		err = errors.New("Can only infer type from []Node with a type= tag.")
		return
	}
	typeName := ntag[len("type="):]
	typ := sa.types[string(typeName)]
	pointer = reflect.New(typ)
	return
}

func getField(v reflect.Value, field string) (fv reflect.Value, err error) {
	defer func() {
		if recover() != nil {
			err = fmt.Errorf("Type %s has no field named %q.", v.Type().Name(), field)
		}
	}()
	if field == "." {
		// . means to store the next level deeper in the same value
		fv = v
	} else {
		fv = v.FieldByName(field)
	}
	return
}

func getString(node Node) (s string, err error) {
	switch nn := node.(type) {
	case SymbolText:
		s = nn.Text
	case Tag:
		s = string(nn)
	case Literal:
		s = string(nn)
	default:
		return "", fmt.Errorf("Expected symbol, tag, or literal, but got %T", node)
	}
	ds, derr := descapeString(s)
	if derr == nil {
		s = ds
	}
	return
}
<|endoftext|>


<commit_before><commit_msg>add lacking functions<commit_after><|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"flag"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"github.com/paulkramme/toml"
	"log"
	"os"
)

const (
	StopCode    = 1000
	ConfirmCode = 1001
	ErrorCode   = 1002
)

func main() {
	fmt.Println("BTSOOT - Copyright (c) 2017 Paul Kramme All Rights Reserved.")

	ConfigLocation := flag.String("c", "", "Specifies configfile location")
	flag.Parse()

	var Config Configuration

	_, err := toml.DecodeFile(*ConfigLocation, &Config)
	if err != nil {
		panic(err)
	}

	var f *os.File
	if Config.LogFileLocation != "" {
		f, err = os.OpenFile(Config.LogFileLocation, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			log.Fatalln(err)
		}
		defer f.Close()
		log.SetOutput(f)
	}

	// NOTE: Create a spacer in the log
	log.Println("\n\n")

	db, err := sql.Open("sqlite3", Config.DBFileLocation)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	err = db.Ping()
	if err != nil {
		panic(err)
	}

	err = DatabaseSetup(db)
	if err != nil {
		panic(err)
	}

	_ = ScanFiles(Config.Source, Config.MaxWorkerThreads)
}

func DatabaseSetup(db *sql.DB) (err error) {
	_, err = db.Exec(`
		CREATE TABLE IF NOT EXISTS blocks(
			name text PRIMARY KEY NOT NULL,
			interval VARCHAR,
			path text NOT NULL,
			destination text NOT NULL,
			absolutepath BOOL)`)
	if err != nil {
		return err
	}
	_, err = db.Exec(`
		CREATE TABLE IF NOT EXISTS files(
			block text NOT NULL,
			checksum TEXT NOT NULL,
			filename text,
			absolutepath text,
			FOREIGN KEY (block) REFERENCES blocks(name))`)
	if err != nil {
		return err
	}
	_, err = db.Exec(
		`CREATE TABLE IF NOT EXISTS users(
			username text,
			password hash)`)
	if err != nil {
		return err
	}
	// _, err = db.Exec("CREATE TABLE IF NOT EXISTS version")
	// if err != nil {
	// 	return err
	// }
	return
}
<commit_msg>Replace log spacer with "started" message<commit_after>package main

import (
	"database/sql"
	"flag"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"github.com/paulkramme/toml"
	"log"
	"os"
)

const (
	StopCode    = 1000
	ConfirmCode = 1001
	ErrorCode   = 1002
)

func main() {
	fmt.Println("BTSOOT - Copyright (c) 2017 Paul Kramme All Rights Reserved.")

	ConfigLocation := flag.String("c", "", "Specifies configfile location")
	flag.Parse()

	var Config Configuration

	_, err := toml.DecodeFile(*ConfigLocation, &Config)
	if err != nil {
		panic(err)
	}

	var f *os.File
	if Config.LogFileLocation != "" {
		f, err = os.OpenFile(Config.LogFileLocation, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			log.Fatalln(err)
		}
		defer f.Close()
		log.SetOutput(f)
	}

	// NOTE: Create a spacer in the log
	log.Println("BTSOOT started")

	db, err := sql.Open("sqlite3", Config.DBFileLocation)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	err = db.Ping()
	if err != nil {
		panic(err)
	}

	err = DatabaseSetup(db)
	if err != nil {
		panic(err)
	}

	_ = ScanFiles(Config.Source, Config.MaxWorkerThreads)
}

func DatabaseSetup(db *sql.DB) (err error) {
	_, err = db.Exec(`
		CREATE TABLE IF NOT EXISTS blocks(
			name text PRIMARY KEY NOT NULL,
			interval VARCHAR,
			path text NOT NULL,
			destination text NOT NULL,
			absolutepath BOOL)`)
	if err != nil {
		return err
	}
	_, err = db.Exec(`
		CREATE TABLE IF NOT EXISTS files(
			block text NOT NULL,
			checksum TEXT NOT NULL,
			filename text,
			absolutepath text,
			FOREIGN KEY (block) REFERENCES blocks(name))`)
	if err != nil {
		return err
	}
	_, err = db.Exec(
		`CREATE TABLE IF NOT EXISTS users(
			username text,
			password hash)`)
	if err != nil {
		return err
	}
	// _, err = db.Exec("CREATE TABLE IF NOT EXISTS version")
	// if err != nil {
	// 	return err
	// }
	return
}
<|endoftext|>