<commit_before>package schema

import (
	"encoding/json"
	"reflect"

	"github.com/cayleygraph/cayley/query/linkedql"
	"github.com/cayleygraph/quad"
	"github.com/cayleygraph/quad/voc/owl"
	"github.com/cayleygraph/quad/voc/rdf"
	"github.com/cayleygraph/quad/voc/rdfs"
	"github.com/cayleygraph/quad/voc/xsd"
)

var (
	pathStep         = reflect.TypeOf((*linkedql.PathStep)(nil)).Elem()
	iteratorStep     = reflect.TypeOf((*linkedql.IteratorStep)(nil)).Elem()
	entityIdentifier = reflect.TypeOf((*linkedql.EntityIdentifier)(nil)).Elem()
	value            = reflect.TypeOf((*quad.Value)(nil)).Elem()
	operator         = reflect.TypeOf((*linkedql.Operator)(nil)).Elem()
	propertyPath     = reflect.TypeOf((*linkedql.PropertyPath)(nil)).Elem()
)

func typeToRange(t reflect.Type) string {
	if t.Kind() == reflect.Slice {
		return typeToRange(t.Elem())
	}
	if t.Kind() == reflect.String {
		return xsd.String
	}
	if t.Kind() == reflect.Bool {
		return xsd.Boolean
	}
	if kind := t.Kind(); kind == reflect.Int64 || kind == reflect.Int {
		return xsd.Int
	}
	if t.Implements(pathStep) {
		return linkedql.Prefix + "PathStep"
	}
	if t.Implements(operator) {
		return linkedql.Prefix + "Operator"
	}
	if t.Implements(value) {
		return rdfs.Resource
	}
	if t.Implements(entityIdentifier) {
		return owl.Thing
	}
	if t == propertyPath {
		return linkedql.Prefix + "PropertyPath"
	}
	panic("Unexpected type " + t.String())
}

// identified is used for referencing a type
type identified struct {
	ID string `json:"@id"`
}

// newIdentified creates new identified struct
func newIdentified(id string) identified {
	return identified{ID: id}
}

// cardinalityRestriction is used to indicate a how many values can a property get
type cardinalityRestriction struct {
	ID          string     `json:"@id"`
	Type        string     `json:"@type"`
	Cardinality int        `json:"owl:cardinality"`
	Property    identified `json:"owl:onProperty"`
}

func newBlankNodeID() string {
	return quad.RandomBlankNode().String()
}

// newSingleCardinalityRestriction creates a cardinality of 1 restriction for given property
func newSingleCardinalityRestriction(prop string) cardinalityRestriction {
	return cardinalityRestriction{
		ID:          newBlankNodeID(),
		Type:        owl.Restriction,
		Cardinality: 1,
		Property:    identified{ID: prop},
	}
}

// getOWLPropertyType for given kind of value type returns property OWL type
func getOWLPropertyType(kind reflect.Kind) string {
	if kind == reflect.String || kind == reflect.Bool || kind == reflect.Int64 || kind == reflect.Int {
		return owl.DatatypeProperty
	}
	return owl.ObjectProperty
}

// property is used to declare a property
type property struct {
	ID     string      `json:"@id"`
	Type   string      `json:"@type"`
	Domain interface{} `json:"rdfs:domain"`
	Range  interface{} `json:"rdfs:range"`
}

// class is used to declare a class
type class struct {
	ID           string        `json:"@id"`
	Type         string        `json:"@type"`
	Comment      string        `json:"rdfs:comment"`
	SuperClasses []interface{} `json:"rdfs:subClassOf"`
}

// newClass creates a new class struct
func newClass(id string, superClasses []interface{}, comment string) class {
	return class{
		ID:           id,
		Type:         rdfs.Class,
		SuperClasses: superClasses,
		Comment:      comment,
	}
}

// getStepTypeClasses for given step type returns the matching class identifiers
func getStepTypeClasses(t reflect.Type) []string {
	var typeClasses []string
	if t.Implements(pathStep) {
		typeClasses = append(typeClasses, linkedql.Prefix+"PathStep")
	}
	if t.Implements(iteratorStep) {
		typeClasses = append(typeClasses, linkedql.Prefix+"IteratorStep")
	}
	return typeClasses
}

type list struct {
	Members []interface{} `json:"@list"`
}

func newList(members []interface{}) list {
	return list{
		Members: members,
	}
}

type unionOf struct {
	ID   string `json:"@id"`
	Type string `json:"@type"`
	List list   `json:"owl:unionOf"`
}

func newUnionOf(classes []string) unionOf {
	var members []interface{}
	for _, class := range classes {
		members = append(members, newIdentified(class))
	}
	return unionOf{
		ID:   newBlankNodeID(),
		Type: owl.Class,
		List: newList(members),
	}
}

func newGenerator() *generator {
	return &generator{
		propToTypes:   make(map[string]map[string]struct{}),
		propToDomains: make(map[string]map[string]struct{}),
		propToRanges:  make(map[string]map[string]struct{}),
	}
}

type generator struct {
	out           []interface{}
	propToTypes   map[string]map[string]struct{}
	propToDomains map[string]map[string]struct{}
	propToRanges  map[string]map[string]struct{}
}

// returns super types
func (g *generator) addTypeFields(name string, t reflect.Type, indirect bool) []interface{} {
	var super []interface{}
	for j := 0; j < t.NumField(); j++ {
		f := t.Field(j)
		if f.Anonymous {
			if f.Type.Kind() != reflect.Struct || !indirect {
				continue
			}
			super = append(super, g.addTypeFields(name, f.Type, false)...)
			continue
		}
		prop := linkedql.Prefix + f.Tag.Get("json")
		if f.Type.Kind() != reflect.Slice {
			super = append(super, newSingleCardinalityRestriction(prop))
		}
		typ := getOWLPropertyType(f.Type.Kind())

		if g.propToTypes[prop] == nil {
			g.propToTypes[prop] = make(map[string]struct{})
		}
		g.propToTypes[prop][typ] = struct{}{}

		if g.propToDomains[prop] == nil {
			g.propToDomains[prop] = make(map[string]struct{})
		}
		g.propToDomains[prop][name] = struct{}{}

		if g.propToRanges[prop] == nil {
			g.propToRanges[prop] = make(map[string]struct{})
		}
		g.propToRanges[prop][typeToRange(f.Type)] = struct{}{}
	}
	return super
}

func (g *generator) AddType(name string, t reflect.Type) {
	step, ok := reflect.New(t).Interface().(linkedql.Step)
	if !ok {
		return
	}
	var super []interface{}
	stepTypeClasses := getStepTypeClasses(reflect.PtrTo(t))
	for _, typeClass := range stepTypeClasses {
		super = append(super, newIdentified(typeClass))
	}
	super = append(super, g.addTypeFields(name, t, true)...)
	g.out = append(g.out, newClass(name, super, step.Description()))
}

func (g *generator) Generate() []byte {
	for prop, types := range g.propToTypes {
		if len(types) != 1 {
			panic("Properties must be either object properties or datatype properties. " + prop + " has both.")
		}
		var typ string
		for t := range types {
			typ = t
			break
		}
		var domains []string
		for d := range g.propToDomains[prop] {
			domains = append(domains, d)
		}
		var ranges []string
		for r := range g.propToRanges[prop] {
			ranges = append(ranges, r)
		}
		var dom interface{}
		if len(domains) == 1 {
			dom = identified{domains[0]}
		} else {
			dom = newUnionOf(domains)
		}
		var rng interface{}
		if len(ranges) == 1 {
			rng = newIdentified(ranges[0])
		} else {
			rng = newUnionOf(ranges)
		}
		g.out = append(g.out, property{
			ID:     prop,
			Type:   typ,
			Domain: dom,
			Range:  rng,
		})
	}
	graph := []interface{}{
		map[string]string{
			"@id":   linkedql.Prefix + "Step",
			"@type": owl.Class,
		},
		map[string]interface{}{
			"@id":           linkedql.Prefix + "PathStep",
			"@type":         owl.Class,
			rdfs.SubClassOf: identified{ID: linkedql.Prefix + "Step"},
		},
		map[string]interface{}{
			"@id":           linkedql.Prefix + "IteratorStep",
			"@type":         owl.Class,
			rdfs.SubClassOf: identified{ID: linkedql.Prefix + "Step"},
		},
	}
	graph = append(graph, g.out...)
	data, err := json.Marshal(map[string]interface{}{
		"@context": map[string]interface{}{
			"rdf":      rdf.NS,
			"rdfs":     rdfs.NS,
			"owl":      owl.NS,
			"xsd":      xsd.NS,
			"linkedql": linkedql.Namespace,
		},
		"@graph": graph,
	})
	if err != nil {
		panic(err)
	}
	return data
}

// Generate a schema in JSON-LD format that contains all registered LinkedQL types and properties.
func Generate() []byte {
	g := newGenerator()
	for _, name := range linkedql.RegisteredTypes() {
		t, ok := linkedql.TypeByName(name)
		if !ok {
			panic("type is registered, but the lookup fails")
		}
		g.AddType(name, t)
	}
	return g.Generate()
}
<commit_msg>Handle string map in schema<commit_after>package schema

import (
	"encoding/json"
	"reflect"

	"github.com/cayleygraph/cayley/query/linkedql"
	_ "github.com/cayleygraph/cayley/query/linkedql/steps"
	"github.com/cayleygraph/quad"
	"github.com/cayleygraph/quad/voc/owl"
	"github.com/cayleygraph/quad/voc/rdf"
	"github.com/cayleygraph/quad/voc/rdfs"
	"github.com/cayleygraph/quad/voc/xsd"
)

var (
	pathStep         = reflect.TypeOf((*linkedql.PathStep)(nil)).Elem()
	iteratorStep     = reflect.TypeOf((*linkedql.IteratorStep)(nil)).Elem()
	entityIdentifier = reflect.TypeOf((*linkedql.EntityIdentifier)(nil)).Elem()
	value            = reflect.TypeOf((*quad.Value)(nil)).Elem()
	operator         = reflect.TypeOf((*linkedql.Operator)(nil)).Elem()
	propertyPath     = reflect.TypeOf((*linkedql.PropertyPath)(nil)).Elem()
	stringMap        = reflect.TypeOf(map[string]string{})
)

func typeToRange(t reflect.Type) string {
	if t == stringMap {
		return "rdf:JSON"
	}
	if t.Kind() == reflect.Slice {
		return typeToRange(t.Elem())
	}
	if t.Kind() == reflect.String {
		return xsd.String
	}
	if t.Kind() == reflect.Bool {
		return xsd.Boolean
	}
	if kind := t.Kind(); kind == reflect.Int64 || kind == reflect.Int {
		return xsd.Int
	}
	if t.Implements(pathStep) {
		return linkedql.Prefix + "PathStep"
	}
	if t.Implements(operator) {
		return linkedql.Prefix + "Operator"
	}
	if t.Implements(value) {
		return rdfs.Resource
	}
	if t.Implements(entityIdentifier) {
		return owl.Thing
	}
	if t == propertyPath {
		return linkedql.Prefix + "PropertyPath"
	}
	panic("Unexpected type " + t.String())
}

// identified is used for referencing a type
type identified struct {
	ID string `json:"@id"`
}

// newIdentified creates new identified struct
func newIdentified(id string) identified {
	return identified{ID: id}
}

// cardinalityRestriction is used to indicate a how many values can a property get
type cardinalityRestriction struct {
	ID          string     `json:"@id"`
	Type        string     `json:"@type"`
	Cardinality int        `json:"owl:cardinality"`
	Property    identified `json:"owl:onProperty"`
}

func newBlankNodeID() string {
	return quad.RandomBlankNode().String()
}

// newSingleCardinalityRestriction creates a cardinality of 1 restriction for given property
func newSingleCardinalityRestriction(prop string) cardinalityRestriction {
	return cardinalityRestriction{
		ID:          newBlankNodeID(),
		Type:        owl.Restriction,
		Cardinality: 1,
		Property:    identified{ID: prop},
	}
}

// getOWLPropertyType for given kind of value type returns property OWL type
func getOWLPropertyType(kind reflect.Kind) string {
	if kind == reflect.String || kind == reflect.Bool || kind == reflect.Int64 || kind == reflect.Int {
		return owl.DatatypeProperty
	}
	return owl.ObjectProperty
}

// property is used to declare a property
type property struct {
	ID     string      `json:"@id"`
	Type   string      `json:"@type"`
	Domain interface{} `json:"rdfs:domain"`
	Range  interface{} `json:"rdfs:range"`
}

// class is used to declare a class
type class struct {
	ID           string        `json:"@id"`
	Type         string        `json:"@type"`
	Comment      string        `json:"rdfs:comment"`
	SuperClasses []interface{} `json:"rdfs:subClassOf"`
}

// newClass creates a new class struct
func newClass(id string, superClasses []interface{}, comment string) class {
	return class{
		ID:           id,
		Type:         rdfs.Class,
		SuperClasses: superClasses,
		Comment:      comment,
	}
}

// getStepTypeClasses for given step type returns the matching class identifiers
func getStepTypeClasses(t reflect.Type) []string {
	var typeClasses []string
	if t.Implements(pathStep) {
		typeClasses = append(typeClasses, linkedql.Prefix+"PathStep")
	}
	if t.Implements(iteratorStep) {
		typeClasses = append(typeClasses, linkedql.Prefix+"IteratorStep")
	}
	return typeClasses
}

type list struct {
	Members []interface{} `json:"@list"`
}

func newList(members []interface{}) list {
	return list{
		Members: members,
	}
}

type unionOf struct {
	ID   string `json:"@id"`
	Type string `json:"@type"`
	List list   `json:"owl:unionOf"`
}

func newUnionOf(classes []string) unionOf {
	var members []interface{}
	for _, class := range classes {
		members = append(members, newIdentified(class))
	}
	return unionOf{
		ID:   newBlankNodeID(),
		Type: owl.Class,
		List: newList(members),
	}
}

func newGenerator() *generator {
	return &generator{
		propToTypes:   make(map[string]map[string]struct{}),
		propToDomains: make(map[string]map[string]struct{}),
		propToRanges:  make(map[string]map[string]struct{}),
	}
}

type generator struct {
	out           []interface{}
	propToTypes   map[string]map[string]struct{}
	propToDomains map[string]map[string]struct{}
	propToRanges  map[string]map[string]struct{}
}

// returns super types
func (g *generator) addTypeFields(name string, t reflect.Type, indirect bool) []interface{} {
	var super []interface{}
	for j := 0; j < t.NumField(); j++ {
		f := t.Field(j)
		if f.Anonymous {
			if f.Type.Kind() != reflect.Struct || !indirect {
				continue
			}
			super = append(super, g.addTypeFields(name, f.Type, false)...)
			continue
		}
		prop := linkedql.Prefix + f.Tag.Get("json")
		if f.Type.Kind() != reflect.Slice {
			super = append(super, newSingleCardinalityRestriction(prop))
		}
		typ := getOWLPropertyType(f.Type.Kind())

		if g.propToTypes[prop] == nil {
			g.propToTypes[prop] = make(map[string]struct{})
		}
		g.propToTypes[prop][typ] = struct{}{}

		if g.propToDomains[prop] == nil {
			g.propToDomains[prop] = make(map[string]struct{})
		}
		g.propToDomains[prop][name] = struct{}{}

		if g.propToRanges[prop] == nil {
			g.propToRanges[prop] = make(map[string]struct{})
		}
		g.propToRanges[prop][typeToRange(f.Type)] = struct{}{}
	}
	return super
}

func (g *generator) AddType(name string, t reflect.Type) {
	step, ok := reflect.New(t).Interface().(linkedql.Step)
	if !ok {
		return
	}
	var super []interface{}
	stepTypeClasses := getStepTypeClasses(reflect.PtrTo(t))
	for _, typeClass := range stepTypeClasses {
		super = append(super, newIdentified(typeClass))
	}
	super = append(super, g.addTypeFields(name, t, true)...)
	g.out = append(g.out, newClass(name, super, step.Description()))
}

func (g *generator) Generate() []byte {
	for prop, types := range g.propToTypes {
		if len(types) != 1 {
			panic("Properties must be either object properties or datatype properties. " + prop + " has both.")
		}
		var typ string
		for t := range types {
			typ = t
			break
		}
		var domains []string
		for d := range g.propToDomains[prop] {
			domains = append(domains, d)
		}
		var ranges []string
		for r := range g.propToRanges[prop] {
			ranges = append(ranges, r)
		}
		var dom interface{}
		if len(domains) == 1 {
			dom = identified{domains[0]}
		} else {
			dom = newUnionOf(domains)
		}
		var rng interface{}
		if len(ranges) == 1 {
			rng = newIdentified(ranges[0])
		} else {
			rng = newUnionOf(ranges)
		}
		g.out = append(g.out, property{
			ID:     prop,
			Type:   typ,
			Domain: dom,
			Range:  rng,
		})
	}
	graph := []interface{}{
		map[string]string{
			"@id":   linkedql.Prefix + "Step",
			"@type": owl.Class,
		},
		map[string]interface{}{
			"@id":           linkedql.Prefix + "PathStep",
			"@type":         owl.Class,
			rdfs.SubClassOf: identified{ID: linkedql.Prefix + "Step"},
		},
		map[string]interface{}{
			"@id":           linkedql.Prefix + "IteratorStep",
			"@type":         owl.Class,
			rdfs.SubClassOf: identified{ID: linkedql.Prefix + "Step"},
		},
	}
	graph = append(graph, g.out...)
	data, err := json.Marshal(map[string]interface{}{
		"@context": map[string]interface{}{
			"rdf":      rdf.NS,
			"rdfs":     rdfs.NS,
			"owl":      owl.NS,
			"xsd":      xsd.NS,
			"linkedql": linkedql.Namespace,
		},
		"@graph": graph,
	})
	if err != nil {
		panic(err)
	}
	return data
}

// Generate a schema in JSON-LD format that contains all registered LinkedQL types and properties.
func Generate() []byte {
	g := newGenerator()
	for _, name := range linkedql.RegisteredTypes() {
		t, ok := linkedql.TypeByName(name)
		if !ok {
			panic("type is registered, but the lookup fails")
		}
		g.AddType(name, t)
	}
	return g.Generate()
}
<|endoftext|>


<commit_before>package internal

import (
	"github.com/lawrencewoodman/dlit_go"
	"testing"
)

func TestPercentGetResult(t *testing.T) {
	records := []map[string]*dlit.Literal{
		map[string]*dlit.Literal{
			"income": dlit.MustNew(3),
			"cost":   dlit.MustNew(4.5),
			"band":   dlit.MustNew(4),
		},
		map[string]*dlit.Literal{
			"income": dlit.MustNew(3),
			"cost":   dlit.MustNew(3.2),
			"band":   dlit.MustNew(7),
		},
		map[string]*dlit.Literal{
			"income": dlit.MustNew(2),
			"cost":   dlit.MustNew(1.2),
			"band":   dlit.MustNew(4),
		},
		map[string]*dlit.Literal{
			"income": dlit.MustNew(2),
			"cost":   dlit.MustNew(5.6),
			"band":   dlit.MustNew(4),
		},
		map[string]*dlit.Literal{
			"income": dlit.MustNew(2),
			"cost":   dlit.MustNew(0.6),
			"band":   dlit.MustNew(4),
		},
		map[string]*dlit.Literal{
			"income": dlit.MustNew(2),
			"cost":   dlit.MustNew(0.8),
			"band":   dlit.MustNew(4),
		},
		map[string]*dlit.Literal{
			"income": dlit.MustNew(9),
			"cost":   dlit.MustNew(2),
			"band":   dlit.MustNew(9),
		},
	}
	percentCostGt2, err := NewPercentAggregator("percentCostGt2", "cost > 2")
	if err != nil {
		t.Errorf("NewPercentAggregator(\"percentCostGt2\", \"cost > 2\") err == %s",
			err)
	}
	aggregators := []Aggregator{percentCostGt2}

	for i, record := range records {
		percentCostGt2.NextRecord(record, i != 1)
	}
	want := 33.33
	numRecords := int64(len(records))
	got := percentCostGt2.GetResult(aggregators, numRecords)
	gotFloat, gotIsFloat := got.Float()
	if !gotIsFloat || gotFloat != want {
		t.Errorf("GetResult() got: %s, want: %s", got, want)
	}
}

func TestPercentCloneNew(t *testing.T) {
	record := map[string]*dlit.Literal{
		"income": dlit.MustNew(3),
		"band":   dlit.MustNew(4),
		"cost":   dlit.MustNew(4),
	}
	numRecords := int64(1)
	percentCostGt2, err := NewPercentAggregator("percentCostGt2", "cost > 2")
	if err != nil {
		t.Errorf("NewPercentAggregator(\"percentCostGt2\", \"cost > 2\") err == %s",
			err)
	}
	percentCostGt2_2 := percentCostGt2.CloneNew()
	aggregators := []Aggregator{}
	want := int64(100)
	percentCostGt2.NextRecord(record, true)
	got1 := percentCostGt2.GetResult(aggregators, numRecords)
	got2 := percentCostGt2_2.GetResult(aggregators, numRecords)

	gotInt1, gotIsInt1 := got1.Int()
	if !gotIsInt1 || gotInt1 != want {
		t.Errorf("GetResult() got: %s, want: %s", gotInt1, want)
	}
	gotInt2, gotIsInt2 := got2.Int()
	if !gotIsInt2 || gotInt2 != 0 {
		t.Errorf("GetResult() got: %s, want: %s", gotInt1, 0)
	}
}
<commit_msg>Test when no records for PercentAggregator<commit_after>package internal

import (
	"github.com/lawrencewoodman/dlit_go"
	"testing"
)

func TestPercentGetResult(t *testing.T) {
	records := []map[string]*dlit.Literal{
		map[string]*dlit.Literal{
			"income": dlit.MustNew(3),
			"cost":   dlit.MustNew(4.5),
			"band":   dlit.MustNew(4),
		},
		map[string]*dlit.Literal{
			"income": dlit.MustNew(3),
			"cost":   dlit.MustNew(3.2),
			"band":   dlit.MustNew(7),
		},
		map[string]*dlit.Literal{
			"income": dlit.MustNew(2),
			"cost":   dlit.MustNew(1.2),
			"band":   dlit.MustNew(4),
		},
		map[string]*dlit.Literal{
			"income": dlit.MustNew(2),
			"cost":   dlit.MustNew(5.6),
			"band":   dlit.MustNew(4),
		},
		map[string]*dlit.Literal{
			"income": dlit.MustNew(2),
			"cost":   dlit.MustNew(0.6),
			"band":   dlit.MustNew(4),
		},
		map[string]*dlit.Literal{
			"income": dlit.MustNew(2),
			"cost":   dlit.MustNew(0.8),
			"band":   dlit.MustNew(4),
		},
		map[string]*dlit.Literal{
			"income": dlit.MustNew(9),
			"cost":   dlit.MustNew(2),
			"band":   dlit.MustNew(9),
		},
	}
	cases := []struct {
		records []map[string]*dlit.Literal
		want    float64
	}{
		{records, 33.33},
		{[]map[string]*dlit.Literal{}, 0},
	}
	for _, c := range cases {
		percentCostGt2, err := NewPercentAggregator("percentCostGt2", "cost > 2")
		if err != nil {
			t.Errorf("NewPercentAggregator(\"percentCostGt2\", \"cost > 2\") err == %s",
				err)
		}
		aggregators := []Aggregator{percentCostGt2}

		for i, record := range c.records {
			percentCostGt2.NextRecord(record, i != 1)
		}
		numRecords := int64(len(c.records))
		got := percentCostGt2.GetResult(aggregators, numRecords)
		gotFloat, gotIsFloat := got.Float()
		if !gotIsFloat || gotFloat != c.want {
			t.Errorf("GetResult() got: %s, want: %s", got, c.want)
		}
	}
}

func TestPercentCloneNew(t *testing.T) {
	record := map[string]*dlit.Literal{
		"income": dlit.MustNew(3),
		"band":   dlit.MustNew(4),
		"cost":   dlit.MustNew(4),
	}
	numRecords := int64(1)
	percentCostGt2, err := NewPercentAggregator("percentCostGt2", "cost > 2")
	if err != nil {
		t.Errorf("NewPercentAggregator(\"percentCostGt2\", \"cost > 2\") err == %s",
			err)
	}
	percentCostGt2_2 := percentCostGt2.CloneNew()
	aggregators := []Aggregator{}
	want := int64(100)
	percentCostGt2.NextRecord(record, true)
	got1 := percentCostGt2.GetResult(aggregators, numRecords)
	got2 := percentCostGt2_2.GetResult(aggregators, numRecords)

	gotInt1, gotIsInt1 := got1.Int()
	if !gotIsInt1 || gotInt1 != want {
		t.Errorf("GetResult() got: %s, want: %s", gotInt1, want)
	}
	gotInt2, gotIsInt2 := got2.Int()
	if !gotIsInt2 || gotInt2 != 0 {
		t.Errorf("GetResult() got: %s, want: %s", gotInt1, 0)
	}
}
<|endoftext|>


<commit_before>// Package defaults implements the Pipe interface providing default values
// for missing configuration.
package defaults

import (
	"fmt"
	"strings"

	"github.com/goreleaser/goreleaser/internal/client"
	"github.com/goreleaser/goreleaser/internal/middleware/errhandler"
	"github.com/goreleaser/goreleaser/internal/middleware/logging"
	"github.com/goreleaser/goreleaser/internal/tmpl"
	"github.com/goreleaser/goreleaser/pkg/context"
	"github.com/goreleaser/goreleaser/pkg/defaults"
)

// Pipe that sets the defaults.
type Pipe struct{}

func (Pipe) String() string { return "setting defaults" }

// Run the pipe.
func (Pipe) Run(ctx *context.Context) error {
	if ctx.Config.Dist == "" {
		ctx.Config.Dist = "dist"
	}
	if ctx.Config.GitHubURLs.Download == "" {
		ctx.Config.GitHubURLs.Download = client.DefaultGitHubDownloadURL
	}
	if ctx.Config.GitLabURLs.Download == "" {
		ctx.Config.GitLabURLs.Download = client.DefaultGitLabDownloadURL
	}
	if ctx.Config.GiteaURLs.Download == "" {
		apiURL, err := tmpl.New(ctx).Apply(ctx.Config.GiteaURLs.API)
		if err != nil {
			return fmt.Errorf("templating Gitea API URL: %w", err)
		}

		ctx.Config.GiteaURLs.Download = strings.ReplaceAll(apiURL, "/api/v1", "")
	}
	for _, defaulter := range defaults.Defaulters {
		if err := logging.Log(
			defaulter.String(),
			errhandler.Handle(defaulter.Default),
			logging.ExtraPadding,
		)(ctx); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>feat: less logs for defaults (#2795)<commit_after>// Package defaults implements the Pipe interface providing default values
// for missing configuration.
package defaults

import (
	"fmt"
	"strings"

	"github.com/goreleaser/goreleaser/internal/client"
	"github.com/goreleaser/goreleaser/internal/middleware/errhandler"
	"github.com/goreleaser/goreleaser/internal/tmpl"
	"github.com/goreleaser/goreleaser/pkg/context"
	"github.com/goreleaser/goreleaser/pkg/defaults"
)

// Pipe that sets the defaults.
type Pipe struct{}

func (Pipe) String() string { return "setting defaults" }

// Run the pipe.
func (Pipe) Run(ctx *context.Context) error {
	if ctx.Config.Dist == "" {
		ctx.Config.Dist = "dist"
	}
	if ctx.Config.GitHubURLs.Download == "" {
		ctx.Config.GitHubURLs.Download = client.DefaultGitHubDownloadURL
	}
	if ctx.Config.GitLabURLs.Download == "" {
		ctx.Config.GitLabURLs.Download = client.DefaultGitLabDownloadURL
	}
	if ctx.Config.GiteaURLs.Download == "" {
		apiURL, err := tmpl.New(ctx).Apply(ctx.Config.GiteaURLs.API)
		if err != nil {
			return fmt.Errorf("templating Gitea API URL: %w", err)
		}

		ctx.Config.GiteaURLs.Download = strings.ReplaceAll(apiURL, "/api/v1", "")
	}
	for _, defaulter := range defaults.Defaulters {
		if err := errhandler.Handle(defaulter.Default)(ctx); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package e2e

import (
	"bytes"
	"fmt"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/oinume/lekcije/server/config"
	"github.com/oinume/lekcije/server/logger"
	"github.com/oinume/lekcije/server/model"
	"github.com/oinume/lekcije/server/route"
	"github.com/sclevine/agouti"
	"go.uber.org/zap/zapcore"
)

var server *httptest.Server
var client = http.DefaultClient
var db *gorm.DB
var helper = model.NewTestHelper()

func TestMain(m *testing.M) {
	config.MustProcessDefault()
	//bootstrap.CheckCLIEnvVars()
	db = helper.DB()
	if err := os.Setenv("MYSQL_DATABASE", "lekcije_test"); err != nil {
		// TODO: Not use panic
		panic(err)
	}
	//bootstrap.CheckServerEnvVars()

	var accessLogBuffer, appLogBuffer bytes.Buffer
	logger.InitializeAccessLogger(&accessLogBuffer)
	appLogLevel := zapcore.InfoLevel
	if level := os.Getenv("LOG_LEVEL"); level != "" {
		appLogLevel = logger.NewLevel(level)
	}
	logger.InitializeAppLogger(&appLogBuffer, appLogLevel)

	helper.TruncateAllTables(helper.DB())

	port := config.DefaultVars.HTTPPort
	routes := route.Create(nil) // TODO: grpc-gateway
	port += 1
	server = newTestServer(routes, port)
	fmt.Printf("Test HTTP server created: port=%d, url=%s\n", port, server.URL)
	defer server.Close()

	client.Timeout = 5 * time.Second
	os.Chdir("../")
	status := m.Run()
	defer os.Exit(status)
}

// newTestServer returns a new test Server with fixed port number.
func newTestServer(handler http.Handler, port int) *httptest.Server {
	listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
	if err != nil {
		if listener, err = net.Listen("tcp6", fmt.Sprintf("[::1]:%d", port)); err != nil {
			panic(fmt.Sprintf("httptest: failed to listen on a port: %v", err))
		}
	}
	ts := &httptest.Server{
		Listener: listener,
		Config:   &http.Server{Handler: handler},
	}
	ts.Start()
	return ts
}

func newWebDriver() *agouti.WebDriver {
	e2eWebDriver := os.Getenv("E2E_WEB_DRIVER")
	var driver *agouti.WebDriver
	switch strings.ToLower(e2eWebDriver) {
	case "chromedriver_headless":
		driver = agouti.ChromeDriver(
			agouti.ChromeOptions("args", []string{
				"--headless",             // headlessモードの指定
				"--window-size=1280,800", // ウィンドウサイズの指定
			}),
			agouti.Debug,
		)
	default:
		driver = agouti.ChromeDriver()
	}
	driver.HTTPClient = client
	return driver
}
<commit_msg>Remove dead code<commit_after>package e2e

import (
	"bytes"
	"fmt"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/oinume/lekcije/server/config"
	"github.com/oinume/lekcije/server/logger"
	"github.com/oinume/lekcije/server/model"
	"github.com/oinume/lekcije/server/route"
	"github.com/sclevine/agouti"
	"go.uber.org/zap/zapcore"
)

var server *httptest.Server
var client = http.DefaultClient
var db *gorm.DB
var helper = model.NewTestHelper()

func TestMain(m *testing.M) {
	config.MustProcessDefault()
	db = helper.DB()
	if err := os.Setenv("MYSQL_DATABASE", "lekcije_test"); err != nil {
		// TODO: Not use panic
		panic(err)
	}

	var accessLogBuffer, appLogBuffer bytes.Buffer
	logger.InitializeAccessLogger(&accessLogBuffer)
	appLogLevel := zapcore.InfoLevel
	if level := os.Getenv("LOG_LEVEL"); level != "" {
		appLogLevel = logger.NewLevel(level)
	}
	logger.InitializeAppLogger(&appLogBuffer, appLogLevel)

	helper.TruncateAllTables(helper.DB())

	port := config.DefaultVars.HTTPPort
	routes := route.Create(nil) // TODO: grpc-gateway
	port += 1
	server = newTestServer(routes, port)
	fmt.Printf("Test HTTP server created: port=%d, url=%s\n", port, server.URL)
	defer server.Close()

	client.Timeout = 5 * time.Second
	os.Chdir("../")
	status := m.Run()
	defer os.Exit(status)
}

// newTestServer returns a new test Server with fixed port number.
func newTestServer(handler http.Handler, port int) *httptest.Server {
	listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
	if err != nil {
		if listener, err = net.Listen("tcp6", fmt.Sprintf("[::1]:%d", port)); err != nil {
			panic(fmt.Sprintf("httptest: failed to listen on a port: %v", err))
		}
	}
	ts := &httptest.Server{
		Listener: listener,
		Config:   &http.Server{Handler: handler},
	}
	ts.Start()
	return ts
}

func newWebDriver() *agouti.WebDriver {
	e2eWebDriver := os.Getenv("E2E_WEB_DRIVER")
	var driver *agouti.WebDriver
	switch strings.ToLower(e2eWebDriver) {
	case "chromedriver_headless":
		driver = agouti.ChromeDriver(
			agouti.ChromeOptions("args", []string{
				"--headless",             // headlessモードの指定
				"--window-size=1280,800", // ウィンドウサイズの指定
			}),
			agouti.Debug,
		)
	default:
		driver = agouti.ChromeDriver()
	}
	driver.HTTPClient = client
	return driver
}
<|endoftext|>


<commit_before>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package oci

import (
	"context"
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"time"

	kptfile "github.com/GoogleContainerTools/kpt/pkg/api/kptfile/v1"
	"github.com/GoogleContainerTools/kpt/pkg/oci"
	"github.com/GoogleContainerTools/kpt/porch/api/porch/v1alpha1"
	configapi "github.com/GoogleContainerTools/kpt/porch/api/porchconfig/v1alpha1"
	"github.com/GoogleContainerTools/kpt/porch/pkg/repository"
	"github.com/google/go-containerregistry/pkg/gcrane"
	"github.com/google/go-containerregistry/pkg/name"
	v1 "github.com/google/go-containerregistry/pkg/v1"
	"github.com/google/go-containerregistry/pkg/v1/google"
	"github.com/google/go-containerregistry/pkg/v1/remote"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/klog/v2"
)

func OpenRepository(name string, namespace string, content configapi.RepositoryContent, spec *configapi.OciRepository, cacheDir string) (repository.Repository, error) {
	storage, err := oci.NewStorage(cacheDir)
	if err != nil {
		return nil, err
	}

	return &ociRepository{
		name:      name,
		namespace: namespace,
		content:   content,
		spec:      *spec.DeepCopy(),
		storage:   storage,
	}, nil

}

type ociRepository struct {
	name      string
	namespace string
	content   configapi.RepositoryContent
	spec      configapi.OciRepository

	storage *oci.Storage
}

var _ repository.Repository = &ociRepository{}
var _ repository.FunctionRepository = &ociRepository{}

func (r *ociRepository) ListPackageRevisions(ctx context.Context, filter repository.ListPackageRevisionFilter) ([]repository.PackageRevision, error) {
	if r.content != configapi.RepositoryContentPackage {
		return []repository.PackageRevision{}, nil
	}

	ctx, span := tracer.Start(ctx, "ociRepository::ListPackageRevisions")
	defer span.End()

	ociRepo, err := name.NewRepository(r.spec.Registry)
	if err != nil {
		return nil, err
	}

	options := r.storage.CreateOptions(ctx)

	tags, err := google.List(ociRepo, options...)
	if err != nil {
		return nil, err
	}

	klog.Infof("tags: %#v", tags)

	var result []repository.PackageRevision
	for _, childName := range tags.Children {
		path := fmt.Sprintf("%s/%s", r.spec.Registry, childName)
		child, err := name.NewRepository(path, name.StrictValidation)
		if err != nil {
			klog.Warningf("Cannot create nested repository %q: %v", path, err)
			continue
		}

		childTags, err := google.List(child, options...)
		if err != nil {
			klog.Warningf("Cannot list nested repository %q: %v", path, err)
			continue
		}

		// klog.Infof("childTags: %#v", childTags)

		for digest, m := range childTags.Manifests {
			for _, tag := range m.Tags {
				created := m.Created
				if created.IsZero() {
					created = m.Uploaded
				}

				// ref := child.Tag(tag)
				// ref := child.Digest(digest)

				p := &ociPackageRevision{
					// tagName: ImageTagName{
					// 	Image: child.Name(),
					// 	Tag:   tag,
					// },
					digestName: oci.ImageDigestName{
						Image:  child.Name(),
						Digest: digest,
					},
					packageName:     childName,
					revision:        tag,
					created:         created,
					parent:          r,
					resourceVersion: constructResourceVersion(m.Created),
				}
				p.uid = constructUID(p.packageName + ":" + p.revision)

				lifecycle, err := r.getLifecycle(ctx, p.digestName)
				if err != nil {
					return nil, err
				}
				p.lifecycle = lifecycle

				tasks, err := r.loadTasks(ctx, p.digestName)
				if err != nil {
					return nil, err
				}
				p.tasks = tasks

				if filter.Matches(p) {
					result = append(result, p)
				}
			}
		}
	}

	return result, nil
}

func (r *ociRepository) ListPackages(ctx context.Context, filter repository.ListPackageFilter) ([]repository.Package, error) {
	return nil, fmt.Errorf("ListPackages not supported for OCI packages")
}

func (r *ociRepository) buildPackageRevision(ctx context.Context, name oci.ImageDigestName, packageName string, revision string, created time.Time) (repository.PackageRevision, error) {
	if r.content != configapi.RepositoryContentPackage {
		return nil, fmt.Errorf("repository is not a package repo, type is %v", r.content)
	}

	ctx, span := tracer.Start(ctx, "ociRepository::buildPackageRevision")
	defer span.End()

	p := &ociPackageRevision{
		digestName:      name,
		packageName:     packageName,
		revision:        revision,
		created:         created,
		parent:          r,
		resourceVersion: constructResourceVersion(created),
	}
	p.uid = constructUID(p.packageName + ":" + p.revision)

	lifecycle, err := r.getLifecycle(ctx, p.digestName)
	if err != nil {
		return nil, err
	}
	p.lifecycle = lifecycle

	tasks, err := r.loadTasks(ctx, p.digestName)
	if err != nil {
		return nil, err
	}
	p.tasks = tasks

	return p, nil
}

func GetFunctionMeta(reference string, ctx context.Context) (*functionMeta, error) {
	ref, err := name.ParseReference(reference)
	if err != nil {
		return nil, fmt.Errorf("parse image reference %v: %v", reference, err)
	}
	image, err := remote.Image(ref, remote.WithAuthFromKeychain(gcrane.Keychain), remote.WithContext(ctx))
	if err != nil {
		return nil, fmt.Errorf("pull remote image %v: %v", reference, err)
	}
	manifest, err := image.Manifest()
	if err != nil {
		return nil, fmt.Errorf("get manifest from image %v: %v", reference, err)
	}
	return &functionMeta{
		FunctionTypes:    GetSliceFromAnnotation(FunctionTypesKey, manifest),
		Description:      GetSingleFromAnnotation(DescriptionKey, manifest),
		DocumentationUrl: GetSingleFromAnnotation(DocumentationURLKey, manifest),
		Keywords:         GetSliceFromAnnotation(keywordsKey, manifest),
		FunctionConfigs:  GetDefaultFunctionConfig(manifest),
	}, nil
}

func GetDefaultFunctionConfig(manifest *v1.Manifest) []functionConfig {
	val, ok := manifest.Annotations[ConfigMapFnKey]
	if !ok {
		return nil
	}
	return []functionConfig{
		{
			TypeMeta: metav1.TypeMeta{
				Kind:       "ConfigMap",
				APIVersion: "v1",
			},
			RequiredFields: AnnotationToSlice(val),
		},
	}
}

func GetSliceFromAnnotation(key string, manifest *v1.Manifest) []string {
	slice, ok := manifest.Annotations[key]
	if !ok {
		return nil
	}
	return AnnotationToSlice(slice)
}

func GetSingleFromAnnotation(key string, manifest *v1.Manifest) string {
	if val, ok := manifest.Annotations[key]; ok {
		return val
	}
	return fmt.Sprintf("annotation %v unset", key)
}

func (r *ociRepository) ListFunctions(ctx context.Context) ([]repository.Function, error) {
	// Repository whose content type is not Function contains no Function resources.
	if r.content != configapi.RepositoryContentFunction {
		klog.Infof("Repository %q doesn't contain functions; contains %s", r.name, r.content)
		return []repository.Function{}, nil
	}

	ctx, span := tracer.Start(ctx, "ociRepository::ListFunctions")
	defer span.End()

	ociRepo, err := name.NewRepository(r.spec.Registry)
	if err != nil {
		return nil, err
	}

	options := r.storage.CreateOptions(ctx)

	result := []repository.Function{}

	err = google.Walk(ociRepo, func(repo name.Repository, tags *google.Tags, err error) error {
		if err != nil {
			klog.Warningf(" Walk %s encountered error: %w", repo, err)
			return err
		}

		if tags == nil {
			return nil
		}

		if cl := len(tags.Children); cl > 0 {
			// Expect no manifests or tags
			if ml, tl := len(tags.Manifests), len(tags.Tags); ml != 0 || tl != 0 {
				return fmt.Errorf("OCI repository with children (%d) as well as Manifests (%d) or Tags (%d)", cl, ml, tl)
			}
			return nil
		}

		functionName := parseFunctionName(repo.RepositoryStr())

		for digest, manifest := range tags.Manifests {
			// Only consider tagged images.
			for _, tag := range manifest.Tags {

				created := manifest.Created
				if created.IsZero() {
					created = manifest.Uploaded
				}
				meta, err := GetFunctionMeta(repo.Digest(digest).Name(), ctx)
				if err != nil {
					klog.Warningf(" pull function %v error: %w", functionName, err)
					continue
				}
				result = append(result, &ociFunction{
					ref:     repo.Digest(digest),
					tag:     repo.Tag(tag),
					name:    functionName,
					version: tag,
					meta:    meta,
					created: created,
					parent:  r,
				})
			}
		}

		return nil
	}, options...)
	if err != nil {
		return nil, err
	}

	return result, nil
}

type ociPackageRevision struct {
	digestName      oci.ImageDigestName
	packageName     string
	revision        string
	created         time.Time
	resourceVersion string
	uid             types.UID

	parent *ociRepository

	tasks []v1alpha1.Task

	lifecycle v1alpha1.PackageRevisionLifecycle
}

var _ repository.PackageRevision = &ociPackageRevision{}

func (p *ociPackageRevision) GetResources(ctx context.Context) (*v1alpha1.PackageRevisionResources, error) {
	resources, err := LoadResources(ctx, p.parent.storage, &p.digestName)
	if err != nil {
		return nil, err
	}

	key := p.Key()

	return &v1alpha1.PackageRevisionResources{
		TypeMeta: metav1.TypeMeta{
			Kind:       "PackageRevisionResources",
			APIVersion: v1alpha1.SchemeGroupVersion.Identifier(),
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      p.KubeObjectName(),
			Namespace: p.parent.namespace,
			CreationTimestamp: metav1.Time{
				Time: p.created,
			},
			ResourceVersion: p.resourceVersion,
			UID:             p.uid,
		},
		Spec: v1alpha1.PackageRevisionResourcesSpec{
			PackageName:    key.Package,
			Revision:       key.Revision,
			RepositoryName: key.Repository,

			Resources: resources.Contents,
		},
	}, nil
}

func (p *ociPackageRevision) KubeObjectName() string {
	hash := sha1.Sum([]byte(fmt.Sprintf("%s:%s:%s", p.parent.name, p.packageName, p.revision)))
	return p.parent.name + "-" + hex.EncodeToString(hash[:])
}

func (p *ociPackageRevision) Key() repository.PackageRevisionKey {
	return repository.PackageRevisionKey{
		Repository: p.parent.name,
		Package:    p.packageName,
		Revision:   p.revision,
	}
}

func (p *ociPackageRevision) GetPackageRevision() *v1alpha1.PackageRevision {
	key := p.Key()

	return &v1alpha1.PackageRevision{
		TypeMeta: metav1.TypeMeta{
			Kind:       "PackageRevision",
			APIVersion: v1alpha1.SchemeGroupVersion.Identifier(),
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      p.KubeObjectName(),
			Namespace: p.parent.namespace,
			CreationTimestamp: metav1.Time{
				Time: p.created,
			},
			ResourceVersion: p.resourceVersion,
			UID:             p.uid,
		},
		Spec: v1alpha1.PackageRevisionSpec{
			PackageName:    key.Package,
			Revision:       key.Revision,
			RepositoryName: key.Repository,

			Lifecycle: p.Lifecycle(),
			Tasks:     p.tasks,
		},
	}
}

func (p *ociPackageRevision) GetUpstreamLock() (kptfile.Upstream, kptfile.UpstreamLock, error) {
	return kptfile.Upstream{}, kptfile.UpstreamLock{}, fmt.Errorf("UpstreamLock is not supported for OCI packages (%s)", p.KubeObjectName())
}

func (p *ociPackageRevision) GetLock() (kptfile.Upstream, kptfile.UpstreamLock, error) {
	return kptfile.Upstream{}, kptfile.UpstreamLock{}, fmt.Errorf("Lock is not supported for OCI packages (%s)", p.KubeObjectName())
}

func (p *ociPackageRevision) Lifecycle() v1alpha1.PackageRevisionLifecycle {
	return p.lifecycle
}
<commit_msg>Fix format-error when warning during package walk (#3574)<commit_after>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package oci

import (
	"context"
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"time"

	kptfile "github.com/GoogleContainerTools/kpt/pkg/api/kptfile/v1"
	"github.com/GoogleContainerTools/kpt/pkg/oci"
	"github.com/GoogleContainerTools/kpt/porch/api/porch/v1alpha1"
	configapi "github.com/GoogleContainerTools/kpt/porch/api/porchconfig/v1alpha1"
	"github.com/GoogleContainerTools/kpt/porch/pkg/repository"
	"github.com/google/go-containerregistry/pkg/gcrane"
	"github.com/google/go-containerregistry/pkg/name"
	v1 "github.com/google/go-containerregistry/pkg/v1"
	"github.com/google/go-containerregistry/pkg/v1/google"
	"github.com/google/go-containerregistry/pkg/v1/remote"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/klog/v2"
)

func OpenRepository(name string, namespace string, content configapi.RepositoryContent, spec *configapi.OciRepository, cacheDir string) (repository.Repository, error) {
	storage, err := oci.NewStorage(cacheDir)
	if err != nil {
		return nil, err
	}

	return &ociRepository{
		name:      name,
		namespace: namespace,
		content:   content,
		spec:      *spec.DeepCopy(),
		storage:   storage,
	}, nil

}

type ociRepository struct {
	name      string
	namespace string
	content   configapi.RepositoryContent
	spec      configapi.OciRepository

	storage *oci.Storage
}

var _ repository.Repository = &ociRepository{}
var _ repository.FunctionRepository = &ociRepository{}

func (r *ociRepository) ListPackageRevisions(ctx context.Context, filter repository.ListPackageRevisionFilter) ([]repository.PackageRevision, error) {
	if r.content != configapi.RepositoryContentPackage {
		return []repository.PackageRevision{}, nil
	}

	ctx, span := tracer.Start(ctx, "ociRepository::ListPackageRevisions")
	defer span.End()

	ociRepo, err := name.NewRepository(r.spec.Registry)
	if err != nil {
		return nil, err
	}

	options := r.storage.CreateOptions(ctx)

	tags, err := google.List(ociRepo, options...)
	if err != nil {
		return nil, err
	}

	klog.Infof("tags: %#v", tags)

	var result []repository.PackageRevision
	for _, childName := range tags.Children {
		path := fmt.Sprintf("%s/%s", r.spec.Registry, childName)
		child, err := name.NewRepository(path, name.StrictValidation)
		if err != nil {
			klog.Warningf("Cannot create nested repository %q: %v", path, err)
			continue
		}

		childTags, err := google.List(child, options...)
		if err != nil {
			klog.Warningf("Cannot list nested repository %q: %v", path, err)
			continue
		}

		// klog.Infof("childTags: %#v", childTags)

		for digest, m := range childTags.Manifests {
			for _, tag := range m.Tags {
				created := m.Created
				if created.IsZero() {
					created = m.Uploaded
				}

				// ref := child.Tag(tag)
				// ref := child.Digest(digest)

				p := &ociPackageRevision{
					// tagName: ImageTagName{
					// 	Image: child.Name(),
					// 	Tag:   tag,
					// },
					digestName: oci.ImageDigestName{
						Image:  child.Name(),
						Digest: digest,
					},
					packageName:     childName,
					revision:        tag,
					created:         created,
					parent:          r,
					resourceVersion: constructResourceVersion(m.Created),
				}
				p.uid = constructUID(p.packageName + ":" + p.revision)

				lifecycle, err := r.getLifecycle(ctx, p.digestName)
				if err != nil {
					return nil, err
				}
				p.lifecycle = lifecycle

				tasks, err := r.loadTasks(ctx, p.digestName)
				if err != nil {
					return nil, err
				}
				p.tasks = tasks

				if filter.Matches(p) {
					result = append(result, p)
				}
			}
		}
	}

	return result, nil
}

func (r *ociRepository) ListPackages(ctx context.Context, filter repository.ListPackageFilter) ([]repository.Package, error) {
	return nil, fmt.Errorf("ListPackages not supported for OCI packages")
}

func (r *ociRepository) buildPackageRevision(ctx context.Context, name oci.ImageDigestName, packageName string, revision string, created time.Time) (repository.PackageRevision, error) {
	if r.content != configapi.RepositoryContentPackage {
		return nil, fmt.Errorf("repository is not a package repo, type is %v", r.content)
	}

	ctx, span := tracer.Start(ctx, "ociRepository::buildPackageRevision")
	defer span.End()

	p := &ociPackageRevision{
		digestName:      name,
		packageName:     packageName,
		revision:        revision,
		created:         created,
		parent:          r,
		resourceVersion: constructResourceVersion(created),
	}
	p.uid = constructUID(p.packageName + ":" + p.revision)

	lifecycle, err := r.getLifecycle(ctx, p.digestName)
	if err != nil {
		return nil, err
	}
	p.lifecycle = lifecycle

	tasks, err := r.loadTasks(ctx, p.digestName)
	if err != nil {
		return nil, err
	}
	p.tasks = tasks

	return p, nil
}

func GetFunctionMeta(reference string, ctx context.Context) (*functionMeta, error) {
	ref, err := name.ParseReference(reference)
	if err != nil {
		return nil, fmt.Errorf("parse image reference %v: %v", reference, err)
	}
	image, err := remote.Image(ref, remote.WithAuthFromKeychain(gcrane.Keychain), remote.WithContext(ctx))
	if err != nil {
		return nil, fmt.Errorf("pull remote image %v: %v", reference, err)
	}
	manifest, err := image.Manifest()
	if err != nil {
		return nil, fmt.Errorf("get manifest from image %v: %v", reference, err)
	}
	return &functionMeta{
		FunctionTypes:    GetSliceFromAnnotation(FunctionTypesKey, manifest),
		Description:      GetSingleFromAnnotation(DescriptionKey, manifest),
		DocumentationUrl: GetSingleFromAnnotation(DocumentationURLKey, manifest),
		Keywords:         GetSliceFromAnnotation(keywordsKey, manifest),
		FunctionConfigs:  GetDefaultFunctionConfig(manifest),
	}, nil
}

func GetDefaultFunctionConfig(manifest *v1.Manifest) []functionConfig {
	val, ok := manifest.Annotations[ConfigMapFnKey]
	if !ok {
		return nil
	}
	return []functionConfig{
		{
			TypeMeta: metav1.TypeMeta{
				Kind:       "ConfigMap",
				APIVersion: "v1",
			},
			RequiredFields: AnnotationToSlice(val),
		},
	}
}

func GetSliceFromAnnotation(key string, manifest *v1.Manifest) []string {
	slice, ok := manifest.Annotations[key]
	if !ok {
		return nil
	}
	return AnnotationToSlice(slice)
}

func GetSingleFromAnnotation(key string, manifest *v1.Manifest) string {
	if val, ok := manifest.Annotations[key]; ok {
		return val
	}
	return fmt.Sprintf("annotation %v unset", key)
}

func (r *ociRepository) ListFunctions(ctx context.Context) ([]repository.Function, error) {
	// Repository whose content type is not Function contains no Function resources.
	if r.content != configapi.RepositoryContentFunction {
		klog.Infof("Repository %q doesn't contain functions; contains %s", r.name, r.content)
		return []repository.Function{}, nil
	}

	ctx, span := tracer.Start(ctx, "ociRepository::ListFunctions")
	defer span.End()

	ociRepo, err := name.NewRepository(r.spec.Registry)
	if err != nil {
		return nil, err
	}

	options := r.storage.CreateOptions(ctx)

	result := []repository.Function{}

	err = google.Walk(ociRepo, func(repo name.Repository, tags *google.Tags, err error) error {
		if err != nil {
			klog.Warningf(" Walk %s encountered error: %v", repo, err)
			return err
		}

		if tags == nil {
			return nil
		}

		if cl := len(tags.Children); cl > 0 {
			// Expect no manifests or tags
			if ml, tl := len(tags.Manifests), len(tags.Tags); ml != 0 || tl != 0 {
				return fmt.Errorf("OCI repository with children (%d) as well as Manifests (%d) or Tags (%d)", cl, ml, tl)
			}
			return nil
		}

		functionName := parseFunctionName(repo.RepositoryStr())

		for digest, manifest := range tags.Manifests {
			// Only consider tagged images.
			for _, tag := range manifest.Tags {

				created := manifest.Created
				if created.IsZero() {
					created = manifest.Uploaded
				}
				meta, err := GetFunctionMeta(repo.Digest(digest).Name(), ctx)
				if err != nil {
					klog.Warningf(" pull function %v error: %w", functionName, err)
					continue
				}
				result = append(result, &ociFunction{
					ref:     repo.Digest(digest),
					tag:     repo.Tag(tag),
					name:    functionName,
					version: tag,
					meta:    meta,
					created: created,
					parent:  r,
				})
			}
		}

		return nil
	}, options...)
	if err != nil {
		return nil, err
	}

	return result, nil
}

type ociPackageRevision struct {
	digestName      oci.ImageDigestName
	packageName     string
	revision        string
	created         time.Time
	resourceVersion string
	uid             types.UID

	parent *ociRepository

	tasks []v1alpha1.Task

	lifecycle v1alpha1.PackageRevisionLifecycle
}

var _ repository.PackageRevision = &ociPackageRevision{}

func (p *ociPackageRevision) GetResources(ctx context.Context) (*v1alpha1.PackageRevisionResources, error) {
	resources, err := LoadResources(ctx, p.parent.storage, &p.digestName)
	if err != nil {
		return nil, err
	}

	key := p.Key()

	return &v1alpha1.PackageRevisionResources{
		TypeMeta: metav1.TypeMeta{
			Kind:       "PackageRevisionResources",
			APIVersion: v1alpha1.SchemeGroupVersion.Identifier(),
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      p.KubeObjectName(),
			Namespace: p.parent.namespace,
			CreationTimestamp: metav1.Time{
				Time: p.created,
			},
			ResourceVersion: p.resourceVersion,
			UID:             p.uid,
		},
		Spec: v1alpha1.PackageRevisionResourcesSpec{
			PackageName:    key.Package,
			Revision:       key.Revision,
			RepositoryName: key.Repository,

			Resources: resources.Contents,
		},
	}, nil
}

func (p *ociPackageRevision) KubeObjectName() string {
	hash := sha1.Sum([]byte(fmt.Sprintf("%s:%s:%s", p.parent.name, p.packageName, p.revision)))
	return p.parent.name + "-" + hex.EncodeToString(hash[:])
}

func (p *ociPackageRevision) Key() repository.PackageRevisionKey {
	return repository.PackageRevisionKey{
		Repository: p.parent.name,
		Package:    p.packageName,
		Revision:   p.revision,
	}
}

func (p *ociPackageRevision) GetPackageRevision() *v1alpha1.PackageRevision {
	key := p.Key()

	return &v1alpha1.PackageRevision{
		TypeMeta: metav1.TypeMeta{
			Kind:       "PackageRevision",
			APIVersion: v1alpha1.SchemeGroupVersion.Identifier(),
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      p.KubeObjectName(),
			Namespace: p.parent.namespace,
			CreationTimestamp: metav1.Time{
				Time: p.created,
			},
			ResourceVersion: p.resourceVersion,
			UID:             p.uid,
		},
		Spec: v1alpha1.PackageRevisionSpec{
			PackageName:    key.Package,
			Revision:       key.Revision,
			RepositoryName: key.Repository,

			Lifecycle: p.Lifecycle(),
			Tasks:     p.tasks,
		},
	}
}

func (p *ociPackageRevision) GetUpstreamLock() (kptfile.Upstream, kptfile.UpstreamLock, error) {
	return kptfile.Upstream{}, kptfile.UpstreamLock{}, fmt.Errorf("UpstreamLock is not supported for OCI packages (%s)", p.KubeObjectName())
}

func (p *ociPackageRevision) GetLock() (kptfile.Upstream, kptfile.UpstreamLock, error) {
	return kptfile.Upstream{}, kptfile.UpstreamLock{}, fmt.Errorf("Lock is not supported for OCI packages (%s)", p.KubeObjectName())
}

func (p *ociPackageRevision) Lifecycle() v1alpha1.PackageRevisionLifecycle {
	return p.lifecycle
}
<|endoftext|>


<commit_before>package postgres

import (
	"errors"
	"fmt"
	"os"

	"github.com/flynn/go-discoverd"
	"github.com/flynn/go-sql"
	_ "github.com/flynn/pq"
)

func Open(service, dsn string) (*DB, error) {
	if service == "" {
		service = os.Getenv("FLYNN_POSTGRES")
	}
	set, err := discoverd.NewServiceSet(service)
	if err != nil {
		return nil, err
	}
	db := &DB{set: set, dsn: dsn}
	firstErr := make(chan error)
	go db.followLeader(firstErr)
	return db, <-firstErr
}

type DB struct {
	*sql.DB

	set discoverd.ServiceSet
	dsn string
}

var ErrNoServers = errors.New("postgres: no servers found")

func (db *DB) followLeader(firstErr chan<- error) {
	for update := range db.set.Watch(true) {
		leader := db.set.Leader()
		if leader == nil || leader.Attrs["up"] != "true" {
			if firstErr != nil {
				firstErr <- ErrNoServers
				return
			}
			continue
		}
		if !update.Online || update.Addr != leader.Addr {
			continue
		}
		dsn := fmt.Sprintf("host=%s port=%s %s", leader.Host, leader.Port, db.dsn)
		if db.DB == nil {
			var err error
			db.DB, err = sql.Open("postgres", dsn)
			firstErr <- err
			if err != nil {
				return
			}
		} else {
			db.DB.SetDSN(dsn)
		}
	}
	// TODO: reconnect to discoverd here
}

func (db *DB) Close() error {
	db.set.Close()
	return db.DB.Close()
}
<commit_msg>pkg/postgres: Add DSN getter<commit_after>package postgres

import (
	"errors"
	"fmt"
	"os"
	"sync"

	"github.com/flynn/go-discoverd"
	"github.com/flynn/go-sql"
	_ "github.com/flynn/pq"
)

func Open(service, dsn string) (*DB, error) {
	if service == "" {
		service = os.Getenv("FLYNN_POSTGRES")
	}
	set, err := discoverd.NewServiceSet(service)
	if err != nil {
		return nil, err
	}
	db := &DB{set: set, dsnSuffix: dsn}
	firstErr := make(chan error)
	go db.followLeader(firstErr)
	return db, <-firstErr
}

type DB struct {
	*sql.DB

	set discoverd.ServiceSet

	dsnSuffix string

	mtx sync.RWMutex
	dsn string
}

var ErrNoServers = errors.New("postgres: no servers found")

func (db *DB) followLeader(firstErr chan<- error) {
	for update := range db.set.Watch(true) {
		leader := db.set.Leader()
		if leader == nil || leader.Attrs["up"] != "true" {
			if firstErr != nil {
				firstErr <- ErrNoServers
				return
			}
			continue
		}
		if !update.Online || update.Addr != leader.Addr {
			continue
		}

		dsn := fmt.Sprintf("host=%s port=%s %s", leader.Host, leader.Port, db.dsnSuffix)
		db.mtx.Lock()
		db.dsn = dsn
		db.mtx.Unlock()

		if db.DB == nil {
			var err error
			db.DB, err = sql.Open("postgres", dsn)
			firstErr <- err
			if err != nil {
				return
			}
		} else {
			db.DB.SetDSN(dsn)
		}
	}
	// TODO: reconnect to discoverd here
}

func (db *DB) DSN() string {
	db.mtx.RLock()
	defer db.mtx.RUnlock()
	return db.dsn
}

func (db *DB) Close() error {
	db.set.Close()
	return db.DB.Close()
}
<|endoftext|>


<commit_before>package minion

import (
	"bytes"
	"encoding/json"
	"log"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/dnaeon/gru/classifier"
	"github.com/dnaeon/gru/task"
	"github.com/dnaeon/gru/utils"

	"code.google.com/p/go-uuid/uuid"
	etcdclient "github.com/coreos/etcd/client"
	"golang.org/x/net/context"
)

// Minions keyspace in etcd
const EtcdMinionSpace = "/gru/minion"

// Etcd Minion
type etcdMinion struct {
	// Name of this minion
	name string

	// Minion root directory in etcd
	rootDir string

	// Minion queue directory in etcd
	queueDir string

	// Log directory to keep previously executed tasks
	logDir string

	// Classifier directory in etcd
	classifierDir string

	// Minion unique identifier
	id uuid.UUID

	// KeysAPI client to etcd
	kapi etcdclient.KeysAPI

	// Task queue to which tasks are sent for processing
	taskQueue chan *task.Task
}

// Creates a new etcd minion
func NewEtcdMinion(name string, cfg etcdclient.Config) Minion {
	c, err := etcdclient.New(cfg)
	if err != nil {
		log.Fatal(err)
	}

	kapi := etcdclient.NewKeysAPI(c)
	id := utils.GenerateUUID(name)
	rootDir := filepath.Join(EtcdMinionSpace, id.String())
	queueDir := filepath.Join(rootDir, "queue")
	classifierDir := filepath.Join(rootDir, "classifier")
	logDir := filepath.Join(rootDir, "log")

	m := &etcdMinion{
		name:          name,
		rootDir:       rootDir,
		queueDir:      queueDir,
		classifierDir: classifierDir,
		logDir:        logDir,
		id:            id,
		kapi:          kapi,
	}

	return m
}

// Set the human-readable name of the minion in etcd
func (m *etcdMinion) setName() error {
	nameKey := filepath.Join(m.rootDir, "name")
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), nameKey, m.Name(), opts)

	return err
}

// Set the time the minion was last seen in seconds since the Epoch
func (m *etcdMinion) setLastseen(s int64) error {
	lastseenKey := filepath.Join(m.rootDir, "lastseen")
	lastseenValue := strconv.FormatInt(s, 10)
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), lastseenKey, lastseenValue, opts)

	return err
}

// Checks for any pending tasks in queue
func (m *etcdMinion) checkQueue(c chan<- *task.Task) error {
	opts := &etcdclient.GetOptions{
		Recursive: true,
		Sort:      true,
	}

	// Get backlog tasks if any
	resp, err := m.kapi.Get(context.Background(), m.queueDir, opts)
	if err != nil {
		return nil
	}

	backlog := resp.Node.Nodes
	if len(backlog) == 0 {
		// No backlog tasks found
		return nil
	}

	log.Printf("Found %d tasks in queue", len(backlog))
	for _, node := range backlog {
		task, err := EtcdUnmarshalTask(node)
		m.kapi.Delete(context.Background(), node.Key, nil)

		if err != nil {
			continue
		}

		c <- task
	}

	return nil
}

// Runs periodic jobs such as refreshing classifiers and
// updating the lastseen time
func (m *etcdMinion) periodicRunner(ticker *time.Ticker) error {
	for _ = range ticker.C {
		// Update minion classifiers
		m.Classify()

		// Update lastseen time
		now := time.Now().Unix()
		err := m.setLastseen(now)
		if err != nil {
			log.Printf("Failed to update lastseen time: %s\n", err)
		}
	}

	return nil
}

// Processes new tasks
func (m *etcdMinion) processTask(t *task.Task) error {
	var buf bytes.Buffer

	// Update state of task that we are now processing it
	t.State = task.TaskStateProcessing
	m.SaveTaskResult(t)

	cmd := exec.Command(t.Command, t.Args...)
	cmd.Stdout = &buf
	cmd.Stderr = &buf

	log.Printf("Processing task %s\n", t.TaskID)

	cmdError := cmd.Run()
	t.TimeProcessed = time.Now().Unix()
	t.Result = buf.String()

	if cmdError != nil {
		log.Printf("Failed to process task %s\n", t.TaskID)
		t.Error = cmdError.Error()
		t.State = task.TaskStateFailed
	} else {
		log.Printf("Finished processing task %s\n", t.TaskID)
		t.State = task.TaskStateSuccess
	}

	m.SaveTaskResult(t)

	return cmdError
}

// Saves the task result
func (m *etcdMinion) SaveTaskResult(t *task.Task) error {
	// Task key in etcd
	taskKey := filepath.Join(m.logDir, t.TaskID.String())

	// Serialize task to JSON
	data, err := json.Marshal(t)
	if err != nil {
		log.Printf("Failed to serialize task %s: %s\n", t.TaskID, err)
		return err
	}

	// Save the task result in etcd
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err = m.kapi.Set(context.Background(), taskKey, string(data), opts)
	if err != nil {
		log.Printf("Failed to save task %s: %s\n", t.TaskID, err)
	}

	return err
}

// Unmarshals task from etcd
func EtcdUnmarshalTask(node *etcdclient.Node) (*task.Task, error) {
	task := new(task.Task)
	err := json.Unmarshal([]byte(node.Value), &task)

	return task, err
}

// Returns the minion unique identifier
func (m *etcdMinion) ID() uuid.UUID {
	return m.id
}

// Returns the assigned name of the minion
func (m *etcdMinion) Name() string {
	return m.name
}

// Classifies the minion
func (m *etcdMinion) Classify() error {
	// Classifiers in etcd expire after an hour
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
		TTL:       time.Hour,
	}

	// Update classifiers
	for key, _ := range classifier.Registry {
		klassifier, err := classifier.Get(key)

		if err != nil {
			continue
		}

		// Serialize classifier to JSON and save it in etcd
		data, err := json.Marshal(klassifier)
		if err != nil {
			log.Printf("Failed to serialize classifier: %s\n", key)
			continue
		}

		// Classifier key in etcd
		klassifierKey := filepath.Join(m.classifierDir, key)
		_, err = m.kapi.Set(context.Background(), klassifierKey, string(data), opts)

		if err != nil {
			log.Printf("Failed to set classifier %s: %s\n", key, err)
		}
	}

	return nil
}

// Monitors etcd for new tasks
func (m *etcdMinion) TaskListener(c chan<- *task.Task) error {
	watcherOpts := &etcdclient.WatcherOptions{
		Recursive: true,
	}
	watcher := m.kapi.Watcher(m.queueDir, watcherOpts)

	for {
		resp, err := watcher.Next(context.Background())
		if err != nil {
			log.Printf("Failed to read task: %s\n", err)
			continue
		}

		// Ignore "delete" events when removing a task from the queue
		action := strings.ToLower(resp.Action)
		if strings.EqualFold(action, "delete") {
			continue
		}

		// Remove task from the queue
		t, err := EtcdUnmarshalTask(resp.Node)
		m.kapi.Delete(context.Background(), resp.Node.Key, nil)

		if err != nil {
			log.Printf("Invalid task %s: %s\n", resp.Node.Key, err)
			continue
		}

		// Update task state and save it
		t.State = task.TaskStateQueued
		t.TimeReceived = time.Now().Unix()
		m.SaveTaskResult(t)

		log.Printf("Received task %s\n", t.TaskID)

		c <- t
	}

	return nil
}

// Processes new tasks
func (m *etcdMinion) TaskRunner(c <-chan *task.Task) error {
	for t := range c {
		if t.IsConcurrent {
			go m.processTask(t)
		} else {
			m.processTask(t)
		}
	}

	return nil
}

// Main entry point of the minion
func (m *etcdMinion) Serve() error {
	err := m.setName()
	if err != nil {
		return err
	}

	// Run periodic scheduler every fifteen minutes
	schedule := time.Minute * 15
	ticker := time.NewTicker(schedule)
	log.Printf("Periodic runner schedule set to run every %s\n", schedule)
	go m.periodicRunner(ticker)

	log.Printf("Minion %s is ready to serve", m.ID())

	// Check for pending tasks in the
	// queue and process them first
 	m.taskQueue = make(chan *task.Task)
	go m.TaskRunner(m.taskQueue)
	m.checkQueue(m.taskQueue)

	// Start listening for new tasks
	go m.TaskListener(m.taskQueue)

	return nil
}

// Stops the minion and performs any cleanup tasks
func (m *etcdMinion) Stop() error {
	log.Println("Minion is shutting down")
	close(m.taskQueue)

	return nil
}
<commit_msg>Enable more logging and error checking in etcd minion<commit_after>package minion

import (
	"bytes"
	"encoding/json"
	"log"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/dnaeon/gru/classifier"
	"github.com/dnaeon/gru/task"
	"github.com/dnaeon/gru/utils"

	"code.google.com/p/go-uuid/uuid"
	etcdclient "github.com/coreos/etcd/client"
	"golang.org/x/net/context"
)

// Minions keyspace in etcd
const EtcdMinionSpace = "/gru/minion"

// Etcd Minion
type etcdMinion struct {
	// Name of the minion
	name string

	// Minion root directory in etcd
	rootDir string

	// Minion queue directory in etcd
	queueDir string

	// Log directory of previously executed tasks
	logDir string

	// Classifier directory in etcd
	classifierDir string

	// Minion unique identifier
	id uuid.UUID

	// KeysAPI client to etcd
	kapi etcdclient.KeysAPI

	// Channel over which tasks are sent for processing
	taskQueue chan *task.Task
}

// Creates a new etcd minion
func NewEtcdMinion(name string, cfg etcdclient.Config) Minion {
	c, err := etcdclient.New(cfg)
	if err != nil {
		log.Fatal(err)
	}

	kapi := etcdclient.NewKeysAPI(c)
	id := utils.GenerateUUID(name)
	rootDir := filepath.Join(EtcdMinionSpace, id.String())
	queueDir := filepath.Join(rootDir, "queue")
	classifierDir := filepath.Join(rootDir, "classifier")
	logDir := filepath.Join(rootDir, "log")
	taskQueue := make(chan *task.Task)

	m := &etcdMinion{
		name:          name,
		rootDir:       rootDir,
		queueDir:      queueDir,
		classifierDir: classifierDir,
		logDir:        logDir,
		id:            id,
		kapi:          kapi,
		taskQueue:     taskQueue,
	}

	return m
}

// Set the human-readable name of the minion in etcd
func (m *etcdMinion) setName() error {
	nameKey := filepath.Join(m.rootDir, "name")
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), nameKey, m.Name(), opts)
	if err != nil {
		log.Printf("Failed to set name of minion: %s\n", err)
	}

	return err
}

// Set the time the minion was last seen in seconds since the Epoch
func (m *etcdMinion) setLastseen(s int64) error {
	lastseenKey := filepath.Join(m.rootDir, "lastseen")
	lastseenValue := strconv.FormatInt(s, 10)
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), lastseenKey, lastseenValue, opts)
	if err != nil {
		log.Printf("Failed to set lastseen time: %s\n", err)
	}

	return err
}

// Checks for any pending tasks and sends them
// for processing if there are any
func (m *etcdMinion) checkQueue() error {
	opts := &etcdclient.GetOptions{
		Recursive: true,
		Sort:      true,
	}

	// Get backlog tasks if any
	resp, err := m.kapi.Get(context.Background(), m.queueDir, opts)
	if err != nil {
		log.Printf("Failed to get backlog tasks: %s\n", err)
		return err
	}

	backlog := resp.Node.Nodes
	if len(backlog) == 0 {
		// No backlog tasks found
		return nil
	}

	log.Printf("Found %d pending tasks in queue", len(backlog))
	for _, node := range backlog {
		t, err := EtcdUnmarshalTask(node)
		m.kapi.Delete(context.Background(), node.Key, nil)

		if err != nil {
			continue
		}

		m.taskQueue <- t
	}

	return nil
}

// Runs periodic jobs such as refreshing classifiers and
// updating the lastseen time every fifteen minutes
func (m *etcdMinion) periodicRunner() {
	schedule := time.Minute * 15
	ticker := time.NewTicker(schedule)
	log.Printf("Periodic scheduler set to run every %s\n", schedule)

	for now := range ticker.C {
		// Run any periodic jobs
		m.Classify()
		m.setLastseen(now.Unix())
	}
}

// Processes new tasks
func (m *etcdMinion) processTask(t *task.Task) error {
	var buf bytes.Buffer

	// Update state of task to indicate that we are now processing it
	t.State = task.TaskStateProcessing
	m.SaveTaskResult(t)

	cmd := exec.Command(t.Command, t.Args...)
	cmd.Stdout = &buf
	cmd.Stderr = &buf

	log.Printf("Processing task %s\n", t.TaskID)

	cmdError := cmd.Run()
	t.TimeProcessed = time.Now().Unix()
	t.Result = buf.String()

	if cmdError != nil {
		log.Printf("Failed to process task %s\n", t.TaskID)
		t.Error = cmdError.Error()
		t.State = task.TaskStateFailed
	} else {
		log.Printf("Finished processing task %s\n", t.TaskID)
		t.State = task.TaskStateSuccess
	}

	m.SaveTaskResult(t)

	return cmdError
}

// Saves a task in etcd
func (m *etcdMinion) SaveTaskResult(t *task.Task) error {
	taskKey := filepath.Join(m.logDir, t.TaskID.String())

	data, err := json.Marshal(t)
	if err != nil {
		log.Printf("Failed to serialize task %s: %s\n", t.TaskID, err)
		return err
	}

	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err = m.kapi.Set(context.Background(), taskKey, string(data), opts)
	if err != nil {
		log.Printf("Failed to save task %s: %s\n", t.TaskID, err)
	}

	return err
}

// Unmarshals task from etcd
func EtcdUnmarshalTask(node *etcdclient.Node) (*task.Task, error) {
	task := new(task.Task)
	err := json.Unmarshal([]byte(node.Value), &task)

	return task, err
}

// Returns the minion unique identifier
func (m *etcdMinion) ID() uuid.UUID {
	return m.id
}

// Returns the assigned name of the minion
func (m *etcdMinion) Name() string {
	return m.name
}

// Classifies the minion
func (m *etcdMinion) Classify() error {
	// Classifiers in etcd expire after an hour
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
		TTL:       time.Hour,
	}

	// Set/update classifiers in etcd
	for key, _ := range classifier.Registry {
		klassifier, err := classifier.Get(key)

		if err != nil {
			log.Printf("Failed to get classifier %s: %s\n", key, err)
			continue
		}

		// Serialize classifier to JSON and save it in etcd
		data, err := json.Marshal(klassifier)
		if err != nil {
			log.Printf("Failed to serialize classifier: %s\n", key)
			continue
		}

		// Classifier key in etcd
		klassifierKey := filepath.Join(m.classifierDir, key)
		_, err = m.kapi.Set(context.Background(), klassifierKey, string(data), opts)

		if err != nil {
			log.Printf("Failed to set classifier %s: %s\n", key, err)
		}
	}

	return nil
}

// Monitors etcd for new tasks
func (m *etcdMinion) TaskListener(c chan<- *task.Task) error {
	log.Printf("Task listener is watching %s\n", m.queueDir)

	watcherOpts := &etcdclient.WatcherOptions{
		Recursive: true,
	}
	watcher := m.kapi.Watcher(m.queueDir, watcherOpts)

	for {
		resp, err := watcher.Next(context.Background())
		if err != nil {
			log.Printf("Failed to receive task: %s\n", err)
			continue
		}

		// Ignore "delete" events when removing a task from the queue
		action := strings.ToLower(resp.Action)
		if strings.EqualFold(action, "delete") {
			continue
		}

		// Unmarshal and remove task from the queue
		t, err := EtcdUnmarshalTask(resp.Node)
		m.kapi.Delete(context.Background(), resp.Node.Key, nil)

		if err != nil {
			log.Printf("Received invalid task %s: %s\n", resp.Node.Key, err)
			continue
		}

		// Update task state and send it for processing
		log.Printf("Received task %s\n", t.TaskID)
		t.State = task.TaskStateQueued
		t.TimeReceived = time.Now().Unix()
		m.SaveTaskResult(t)
		c <- t
	}

	return nil
}

// Processes new tasks
func (m *etcdMinion) TaskRunner(c <-chan *task.Task) error {
	log.Println("Starting task runner")

	for t := range c {
		if t.IsConcurrent {
			go m.processTask(t)
		} else {
			m.processTask(t)
		}
	}

	return nil
}

// Main entry point of the minion
func (m *etcdMinion) Serve() error {
	err := m.setName()
	if err != nil {
		return err
	}

	// Start minion services
	go m.periodicRunner()
	go m.checkQueue()
	go m.TaskRunner(m.taskQueue)
	go m.TaskListener(m.taskQueue)

	log.Printf("Minion %s is ready to serve", m.ID())

	return nil
}

// Stops the minion and performs any cleanup tasks
func (m *etcdMinion) Stop() error {
	log.Println("Minion is shutting down")

	close(m.taskQueue)

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"path"
	"strconv"
	"strings"

	"github.com/ethereum/eth-go"
	"github.com/ethereum/eth-go/ethchain"
	"github.com/ethereum/eth-go/ethcrypto"
	"github.com/ethereum/eth-go/ethpipe"
	"github.com/ethereum/eth-go/ethstate"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/ethereum/go-ethereum/javascript"
	"gopkg.in/qml.v1"
)

type memAddr struct {
	Num   string
	Value string
}

// UI Library that has some basic functionality exposed
type UiLib struct {
	*ethpipe.JSPipe
	engine    *qml.Engine
	eth       *eth.Ethereum
	connected bool
	assetPath string
	// The main application window
	win      *qml.Window
	Db       *Debugger
	DbWindow *DebuggerWindow

	jsEngine *javascript.JSRE

	filterCallbacks map[int][]int
}

func NewUiLib(engine *qml.Engine, eth *eth.Ethereum, assetPath string) *UiLib {
	return &UiLib{JSPipe: ethpipe.NewJSPipe(eth), engine: engine, eth: eth, assetPath: assetPath, jsEngine: javascript.NewJSRE(eth), filterCallbacks: make(map[int][]int)} //, filters: make(map[int]*ethpipe.JSFilter)}
}

func (self *UiLib) Notef(args []interface{}) {
	logger.Infoln(args...)
}

func (self *UiLib) LookupDomain(domain string) string {
	world := self.World()

	if len(domain) > 32 {
		domain = string(ethcrypto.Sha3Bin([]byte(domain)))
	}
	data := world.Config().Get("DnsReg").StorageString(domain).Bytes()

	// Left padded = A record, Right padded = CNAME
	if len(data) > 0 && data[0] == 0 {
		data = bytes.TrimLeft(data, "\x00")
		var ipSlice []string
		for _, d := range data {
			ipSlice = append(ipSlice, strconv.Itoa(int(d)))
		}

		return strings.Join(ipSlice, ".")
	} else {
		data = bytes.TrimRight(data, "\x00")

		return string(data)
	}
}

func (self *UiLib) LookupName(addr string) string {
	var (
		nameReg = self.World().Config().Get("NameReg")
		lookup  = nameReg.Storage(ethutil.Hex2Bytes(addr))
	)

	if lookup.Len() != 0 {
		return strings.Trim(lookup.Str(), "\x00")
	}

	return addr
}

func (self *UiLib) LookupAddress(name string) string {
	var (
		nameReg = self.World().Config().Get("NameReg")
		lookup  = nameReg.Storage(ethutil.RightPadBytes([]byte(name), 32))
	)

	if lookup.Len() != 0 {
		return ethutil.Bytes2Hex(lookup.Bytes())
	}

	return ""
}

func (self *UiLib) PastPeers() *ethutil.List {
	return ethutil.NewList(eth.PastPeers())
}

func (self *UiLib) ImportTx(rlpTx string) {
	tx := ethchain.NewTransactionFromBytes(ethutil.Hex2Bytes(rlpTx))
	self.eth.TxPool().QueueTransaction(tx)
}

func (self *UiLib) EvalJavascriptFile(path string) {
	self.jsEngine.LoadExtFile(path[7:])
}

func (self *UiLib) EvalJavascriptString(str string) string {
	value, err := self.jsEngine.Run(str)
	if err != nil {
		return err.Error()
	}

	return fmt.Sprintf("%v", value)
}

func (ui *UiLib) OpenQml(path string) {
	container := NewQmlApplication(path[7:], ui)
	app := NewExtApplication(container, ui)

	go app.run()
}

func (ui *UiLib) OpenHtml(path string) {
	container := NewHtmlApplication(path, ui)
	app := NewExtApplication(container, ui)

	go app.run()
}

func (ui *UiLib) OpenBrowser() {
	ui.OpenHtml("file://" + ui.AssetPath("ext/home.html"))
}

func (ui *UiLib) Muted(content string) {
	component, err := ui.engine.LoadFile(ui.AssetPath("qml/muted.qml"))
	if err != nil {
		logger.Debugln(err)

		return
	}
	win := component.CreateWindow(nil)
	go func() {
		path := "file://" + ui.AssetPath("muted/index.html")
		win.Set("url", path)

		win.Show()
		win.Wait()
	}()
}

func (ui *UiLib) Connect(button qml.Object) {
	if !ui.connected {
		ui.eth.Start(true)
		ui.connected = true
		button.Set("enabled", false)
	}
}

func (ui *UiLib) ConnectToPeer(addr string) {
	ui.eth.ConnectToPeer(addr)
}

func (ui *UiLib) AssetPath(p string) string {
	return path.Join(ui.assetPath, p)
}

func (self *UiLib) StartDbWithContractAndData(contractHash, data string) {
	dbWindow := NewDebuggerWindow(self)
	object := self.eth.StateManager().CurrentState().GetStateObject(ethutil.Hex2Bytes(contractHash))
	if len(object.Code) > 0 {
		dbWindow.SetCode("0x" + ethutil.Bytes2Hex(object.Code))
	}
	dbWindow.SetData("0x" + data)

	dbWindow.Show()
}

func (self *UiLib) StartDbWithCode(code string) {
	dbWindow := NewDebuggerWindow(self)
	dbWindow.SetCode("0x" + code)
	dbWindow.Show()
}

func (self *UiLib) StartDebugger() {
	dbWindow := NewDebuggerWindow(self)

	dbWindow.Show()
}

func (self *UiLib) NewFilter(object map[string]interface{}) int {
	filter, id := self.eth.InstallFilter(object)
	filter.MessageCallback = func(messages ethstate.Messages) {
		self.win.Root().Call("invokeFilterCallback", ethpipe.ToJSMessages(messages), id)
	}

	return id
}

func (self *UiLib) NewFilterString(typ string) int {
	filter, id := self.eth.InstallFilter(nil)
	filter.BlockCallback = func(block *ethchain.Block) {
		self.win.Root().Call("invokeFilterCallback", "{}", id)
	}

	return id
}

func (self *UiLib) Messages(id int) *ethutil.List {
	filter := self.eth.GetFilter(id)
	if filter != nil {
		messages := ethpipe.ToJSMessages(filter.Find())

		return messages
	}

	return ethutil.EmptyList()
}

func (self *UiLib) UninstallFilter(id int) {
	self.eth.UninstallFilter(id)
}

func mapToTxParams(object map[string]interface{}) map[string]string {
	// Default values
	if object["from"] == nil {
		object["from"] = ""
	}
	if object["to"] == nil {
		object["to"] = ""
	}
	if object["value"] == nil {
		object["value"] = ""
	}
	if object["gas"] == nil {
		object["gas"] = ""
	}
	if object["gasPrice"] == nil {
		object["gasPrice"] = ""
	}

	var dataStr string
	var data []string
	if list, ok := object["data"].(*qml.List); ok {
		list.Convert(&data)
	} else if str, ok := object["data"].(string); ok {
		data = []string{str}
	}

	for _, str := range data {
		if ethutil.IsHex(str) {
			str = str[2:]

			if len(str) != 64 {
				str = ethutil.LeftPadString(str, 64)
			}
		} else {
			str = ethutil.Bytes2Hex(ethutil.LeftPadBytes(ethutil.Big(str).Bytes(), 32))
		}

		dataStr += str
	}
	object["data"] = dataStr

	conv := make(map[string]string)
	for key, value := range object {
		if v, ok := value.(string); ok {
			conv[key] = v
		}
	}

	return conv
}

func (self *UiLib) Transact(params map[string]interface{}) (*ethpipe.JSReceipt, error) {
	object := mapToTxParams(params)

	return self.JSPipe.Transact(
		object["from"],
		object["to"],
		object["value"],
		object["gas"],
		object["gasPrice"],
		object["data"],
	)
}

func (self *UiLib) Compile(code string) (string, error) {
	bcode, err := ethutil.Compile(code, false)
	if err != nil {
		return err.Error(), err
	}

	return ethutil.Bytes2Hex(bcode), err
}

func (self *UiLib) Call(params map[string]interface{}) (string, error) {
	object := mapToTxParams(params)

	return self.JSPipe.Execute(
		object["to"],
		object["value"],
		object["gas"],
		object["gasPrice"],
		object["data"],
	)
}
<commit_msg>Renamed Sha3Bin to Sha3<commit_after>package main

import (
	"bytes"
	"fmt"
	"path"
	"strconv"
	"strings"

	"github.com/ethereum/eth-go"
	"github.com/ethereum/eth-go/ethchain"
	"github.com/ethereum/eth-go/ethcrypto"
	"github.com/ethereum/eth-go/ethpipe"
	"github.com/ethereum/eth-go/ethstate"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/ethereum/go-ethereum/javascript"
	"gopkg.in/qml.v1"
)

type memAddr struct {
	Num   string
	Value string
}

// UI Library that has some basic functionality exposed
type UiLib struct {
	*ethpipe.JSPipe
	engine    *qml.Engine
	eth       *eth.Ethereum
	connected bool
	assetPath string
	// The main application window
	win      *qml.Window
	Db       *Debugger
	DbWindow *DebuggerWindow

	jsEngine *javascript.JSRE

	filterCallbacks map[int][]int
}

func NewUiLib(engine *qml.Engine, eth *eth.Ethereum, assetPath string) *UiLib {
	return &UiLib{JSPipe: ethpipe.NewJSPipe(eth), engine: engine, eth: eth, assetPath: assetPath, jsEngine: javascript.NewJSRE(eth), filterCallbacks: make(map[int][]int)} //, filters: make(map[int]*ethpipe.JSFilter)}
}

func (self *UiLib) Notef(args []interface{}) {
	logger.Infoln(args...)
}

func (self *UiLib) LookupDomain(domain string) string {
	world := self.World()

	if len(domain) > 32 {
		domain = string(ethcrypto.Sha3([]byte(domain)))
	}
	data := world.Config().Get("DnsReg").StorageString(domain).Bytes()

	// Left padded = A record, Right padded = CNAME
	if len(data) > 0 && data[0] == 0 {
		data = bytes.TrimLeft(data, "\x00")
		var ipSlice []string
		for _, d := range data {
			ipSlice = append(ipSlice, strconv.Itoa(int(d)))
		}

		return strings.Join(ipSlice, ".")
	} else {
		data = bytes.TrimRight(data, "\x00")

		return string(data)
	}
}

func (self *UiLib) LookupName(addr string) string {
	var (
		nameReg = self.World().Config().Get("NameReg")
		lookup  = nameReg.Storage(ethutil.Hex2Bytes(addr))
	)

	if lookup.Len() != 0 {
		return strings.Trim(lookup.Str(), "\x00")
	}

	return addr
}

func (self *UiLib) LookupAddress(name string) string {
	var (
		nameReg = self.World().Config().Get("NameReg")
		lookup  = nameReg.Storage(ethutil.RightPadBytes([]byte(name), 32))
	)

	if lookup.Len() != 0 {
		return ethutil.Bytes2Hex(lookup.Bytes())
	}

	return ""
}

func (self *UiLib) PastPeers() *ethutil.List {
	return ethutil.NewList(eth.PastPeers())
}

func (self *UiLib) ImportTx(rlpTx string) {
	tx := ethchain.NewTransactionFromBytes(ethutil.Hex2Bytes(rlpTx))
	self.eth.TxPool().QueueTransaction(tx)
}

func (self *UiLib) EvalJavascriptFile(path string) {
	self.jsEngine.LoadExtFile(path[7:])
}

func (self *UiLib) EvalJavascriptString(str string) string {
	value, err := self.jsEngine.Run(str)
	if err != nil {
		return err.Error()
	}

	return fmt.Sprintf("%v", value)
}

func (ui *UiLib) OpenQml(path string) {
	container := NewQmlApplication(path[7:], ui)
	app := NewExtApplication(container, ui)

	go app.run()
}

func (ui *UiLib) OpenHtml(path string) {
	container := NewHtmlApplication(path, ui)
	app := NewExtApplication(container, ui)

	go app.run()
}

func (ui *UiLib) OpenBrowser() {
	ui.OpenHtml("file://" + ui.AssetPath("ext/home.html"))
}

func (ui *UiLib) Muted(content string) {
	component, err := ui.engine.LoadFile(ui.AssetPath("qml/muted.qml"))
	if err != nil {
		logger.Debugln(err)

		return
	}
	win := component.CreateWindow(nil)
	go func() {
		path := "file://" + ui.AssetPath("muted/index.html")
		win.Set("url", path)

		win.Show()
		win.Wait()
	}()
}

func (ui *UiLib) Connect(button qml.Object) {
	if !ui.connected {
		ui.eth.Start(true)
		ui.connected = true
		button.Set("enabled", false)
	}
}

func (ui *UiLib) ConnectToPeer(addr string) {
	ui.eth.ConnectToPeer(addr)
}

func (ui *UiLib) AssetPath(p string) string {
	return path.Join(ui.assetPath, p)
}

func (self *UiLib) StartDbWithContractAndData(contractHash, data string) {
	dbWindow := NewDebuggerWindow(self)
	object := self.eth.StateManager().CurrentState().GetStateObject(ethutil.Hex2Bytes(contractHash))
	if len(object.Code) > 0 {
		dbWindow.SetCode("0x" + ethutil.Bytes2Hex(object.Code))
	}
	dbWindow.SetData("0x" + data)

	dbWindow.Show()
}

func (self *UiLib) StartDbWithCode(code string) {
	dbWindow := NewDebuggerWindow(self)
	dbWindow.SetCode("0x" + code)
	dbWindow.Show()
}

func (self *UiLib) StartDebugger() {
	dbWindow := NewDebuggerWindow(self)

	dbWindow.Show()
}

func (self *UiLib) NewFilter(object map[string]interface{}) int {
	filter, id := self.eth.InstallFilter(object)
	filter.MessageCallback = func(messages ethstate.Messages) {
		self.win.Root().Call("invokeFilterCallback", ethpipe.ToJSMessages(messages), id)
	}

	return id
}

func (self *UiLib) NewFilterString(typ string) int {
	filter, id := self.eth.InstallFilter(nil)
	filter.BlockCallback = func(block *ethchain.Block) {
		self.win.Root().Call("invokeFilterCallback", "{}", id)
	}

	return id
}

func (self *UiLib) Messages(id int) *ethutil.List {
	filter := self.eth.GetFilter(id)
	if filter != nil {
		messages := ethpipe.ToJSMessages(filter.Find())

		return messages
	}

	return ethutil.EmptyList()
}

func (self *UiLib) UninstallFilter(id int) {
	self.eth.UninstallFilter(id)
}

func mapToTxParams(object map[string]interface{}) map[string]string {
	// Default values
	if object["from"] == nil {
		object["from"] = ""
	}
	if object["to"] == nil {
		object["to"] = ""
	}
	if object["value"] == nil {
		object["value"] = ""
	}
	if object["gas"] == nil {
		object["gas"] = ""
	}
	if object["gasPrice"] == nil {
		object["gasPrice"] = ""
	}

	var dataStr string
	var data []string
	if list, ok := object["data"].(*qml.List); ok {
		list.Convert(&data)
	} else if str, ok := object["data"].(string); ok {
		data = []string{str}
	}

	for _, str := range data {
		if ethutil.IsHex(str) {
			str = str[2:]

			if len(str) != 64 {
				str = ethutil.LeftPadString(str, 64)
			}
		} else {
			str = ethutil.Bytes2Hex(ethutil.LeftPadBytes(ethutil.Big(str).Bytes(), 32))
		}

		dataStr += str
	}
	object["data"] = dataStr

	conv := make(map[string]string)
	for key, value := range object {
		if v, ok := value.(string); ok {
			conv[key] = v
		}
	}

	return conv
}

func (self *UiLib) Transact(params map[string]interface{}) (*ethpipe.JSReceipt, error) {
	object := mapToTxParams(params)

	return self.JSPipe.Transact(
		object["from"],
		object["to"],
		object["value"],
		object["gas"],
		object["gasPrice"],
		object["data"],
	)
}

func (self *UiLib) Compile(code string) (string, error) {
	bcode, err := ethutil.Compile(code, false)
	if err != nil {
		return err.Error(), err
	}

	return ethutil.Bytes2Hex(bcode), err
}

func (self *UiLib) Call(params map[string]interface{}) (string, error) {
	object := mapToTxParams(params)

	return self.JSPipe.Execute(
		object["to"],
		object["value"],
		object["gas"],
		object["gasPrice"],
		object["data"],
	)
}
<|endoftext|>


<commit_before>// Package model provides the complete representation of the model for a given GEP problem.
package model

import (
	"fmt"
	"log"
	"math/rand"
	"runtime"

	"github.com/gmlewis/gep/functions"
	"github.com/gmlewis/gep/gene"
	"github.com/gmlewis/gep/genome"
)

// Generation represents one complete generation of the model.
type Generation struct {
	Genomes     []*genome.Genome
	Funcs       []gene.FuncWeight
	ScoringFunc genome.ScoringFunc
}

// New creates a new random generation of the model.
// fs is a slice of function weights.
// fm is the map of available functions to use for creating the generation of the model.
// numGenomes is the number of genomes to use to populate this generation of the model.
// headSize is the number of head symbols to use in a genome.
// numGenesPerGenome is the number of genes to use per genome.
// numTerminals is the number of terminals (inputs) to use within each gene.
// linkFunc is the linking function used to combine the genes within a genome.
// sf is the scoring (or fitness) function.
func New(fs []gene.FuncWeight, fm functions.FuncMap, numGenomes, headSize, numGenesPerGenome, numTerminals int, linkFunc string, sf genome.ScoringFunc) *Generation {
	r := &Generation{
		Genomes:     make([]*genome.Genome, numGenomes, numGenomes),
		Funcs:       fs,
		ScoringFunc: sf,
	}
	n := maxArity(fs, fm)
	tailSize := headSize*(n-1) + 1
	for i := range r.Genomes {
		genes := make([]*gene.Gene, numGenesPerGenome, numGenesPerGenome)
		for j := range genes {
			genes[j] = gene.RandomNew(headSize, tailSize, numTerminals, fs)
		}
		r.Genomes[i] = genome.New(genes, linkFunc)
	}
	return r
}

// Evolve runs the GEP algorithm for the given number of iterations, or until a score of 1000 (or more) is reached.
func (g *Generation) Evolve(iterations int) *genome.Genome {
	runtime.GOMAXPROCS(runtime.NumCPU()) // Use all CPUs
	// Algorithm flow diagram, figure 3.1, book page 56
	for i := 0; i < iterations; i++ {
		// fmt.Printf("Iteration #%v...\n", i)
		bestGenome := g.getBest() // Preserve the best genome
		if bestGenome.Score >= 1000.0 {
			fmt.Printf("Stopping after generation #%v\n", i)
			return bestGenome
		}
		// fmt.Printf("Best genome (score %v): %v\n", bestGenome.Score, *bestGenome)
		saveCopy := bestGenome.Dup()
		g.replication() // Section 3.3.1, book page 75
		g.mutation()    // Section 3.3.2, book page 77
		// g.isTransposition()
		// g.risTransposition()
		// g.geneTransposition()
		// g.onePointRecombination()
		// g.twoPointRecombination()
		// g.geneRecombination()
		// Now that replication is done, restore the best genome (aka "elitism")
		g.Genomes[0] = saveCopy
	}
	fmt.Printf("Stopping after generation #%v\n", iterations)
	return g.getBest()
}

func (g *Generation) replication() {
	// roulette wheel selection - see www.youtube.com/watch?v=aHLslaWO-AQ
	maxWeight := 0.0
	for _, v := range g.Genomes {
		if v.Score > maxWeight {
			maxWeight = v.Score
		}
	}
	result := make([]*genome.Genome, 0, len(g.Genomes))
	index := rand.Intn(len(g.Genomes))
	beta := 0.0
	for i := 0; i < len(g.Genomes); i++ {
		beta += rand.Float64() * 2.0 * maxWeight
		for beta > g.Genomes[index].Score {
			beta -= g.Genomes[index].Score
			index = (index + 1) % len(g.Genomes)
		}
		result = append(result, g.Genomes[index].Dup())
	}
	g.Genomes = result
}

func (g *Generation) mutation() {
	// Determine the total number of genomes to mutate
	numGenomes := 1 + rand.Intn(len(g.Genomes)-1)
	for i := 0; i < numGenomes; i++ {
		// Pick a random genome
		genomeNum := rand.Intn(len(g.Genomes))
		genome := &g.Genomes[genomeNum]
		// Determine the total number of mutations to perform within the genome
		numMutations := 1 + rand.Intn(2)
		// fmt.Printf("\nMutating genome #%v %v times, before:\n%v\n", genomeNum, numMutations, genome)
		genome.Mutate(numMutations)
		// fmt.Printf("after:\n%v\n", genome)
	}
}

// getBest evaluates all genomes and returns a pointer to the best one.
func (g *Generation) getBest() *genome.Genome {
	bestScore := 0.0
	bestGenome := g.Genomes[0]
	c := make(chan *genome.Genome)
	for i := 0; i < len(g.Genomes); i++ { // Evaluate genomes concurrently
		go g.Genomes[i].Evaluate(g.ScoringFunc, c)
	}
	for i := 0; i < len(g.Genomes); i++ { // Collect and return the highest scoring Genome
		gn := <-c
		if gn.Score > bestScore {
			bestGenome = gn
			bestScore = gn.Score
		}
	}
	return bestGenome
}

// maxArity determines the maximum number of input terminals for the given set of symbols.
func maxArity(fs []gene.FuncWeight, fm functions.FuncMap) int {
	r := 0
	for _, f := range fs {
		if fn, ok := fm[f.Symbol]; ok {
			if fn.Terminals() > r {
				r = fn.Terminals()
			}
		} else {
			log.Printf("unable to find symbol %v in function map\n", f.Symbol)
		}
	}
	return r
}
<commit_msg>gep: fix mutation problem<commit_after>// Package model provides the complete representation of the model for a given GEP problem.
package model

import (
	"fmt"
	"log"
	"math/rand"
	"runtime"

	"github.com/gmlewis/gep/functions"
	"github.com/gmlewis/gep/gene"
	"github.com/gmlewis/gep/genome"
)

// Generation represents one complete generation of the model.
type Generation struct {
	Genomes     []*genome.Genome
	Funcs       []gene.FuncWeight
	ScoringFunc genome.ScoringFunc
}

// New creates a new random generation of the model.
// fs is a slice of function weights.
// fm is the map of available functions to use for creating the generation of the model.
// numGenomes is the number of genomes to use to populate this generation of the model.
// headSize is the number of head symbols to use in a genome.
// numGenesPerGenome is the number of genes to use per genome.
// numTerminals is the number of terminals (inputs) to use within each gene.
// linkFunc is the linking function used to combine the genes within a genome.
// sf is the scoring (or fitness) function.
func New(fs []gene.FuncWeight, fm functions.FuncMap, numGenomes, headSize, numGenesPerGenome, numTerminals int, linkFunc string, sf genome.ScoringFunc) *Generation {
	r := &Generation{
		Genomes:     make([]*genome.Genome, numGenomes, numGenomes),
		Funcs:       fs,
		ScoringFunc: sf,
	}
	n := maxArity(fs, fm)
	tailSize := headSize*(n-1) + 1
	for i := range r.Genomes {
		genes := make([]*gene.Gene, numGenesPerGenome, numGenesPerGenome)
		for j := range genes {
			genes[j] = gene.RandomNew(headSize, tailSize, numTerminals, fs)
		}
		r.Genomes[i] = genome.New(genes, linkFunc)
	}
	return r
}

// Evolve runs the GEP algorithm for the given number of iterations, or until a score of 1000 (or more) is reached.
func (g *Generation) Evolve(iterations int) *genome.Genome {
	runtime.GOMAXPROCS(runtime.NumCPU()) // Use all CPUs
	// Algorithm flow diagram, figure 3.1, book page 56
	for i := 0; i < iterations; i++ {
		// fmt.Printf("Iteration #%v...\n", i)
		bestGenome := g.getBest() // Preserve the best genome
		if bestGenome.Score >= 1000.0 {
			fmt.Printf("Stopping after generation #%v\n", i)
			return bestGenome
		}
		// fmt.Printf("Best genome (score %v): %v\n", bestGenome.Score, *bestGenome)
		saveCopy := bestGenome.Dup()
		g.replication() // Section 3.3.1, book page 75
		g.mutation()    // Section 3.3.2, book page 77
		// g.isTransposition()
		// g.risTransposition()
		// g.geneTransposition()
		// g.onePointRecombination()
		// g.twoPointRecombination()
		// g.geneRecombination()
		// Now that replication is done, restore the best genome (aka "elitism")
		g.Genomes[0] = saveCopy
	}
	fmt.Printf("Stopping after generation #%v\n", iterations)
	return g.getBest()
}

func (g *Generation) replication() {
	// roulette wheel selection - see www.youtube.com/watch?v=aHLslaWO-AQ
	maxWeight := 0.0
	for _, v := range g.Genomes {
		if v.Score > maxWeight {
			maxWeight = v.Score
		}
	}
	result := make([]*genome.Genome, 0, len(g.Genomes))
	index := rand.Intn(len(g.Genomes))
	beta := 0.0
	for i := 0; i < len(g.Genomes); i++ {
		beta += rand.Float64() * 2.0 * maxWeight
		for beta > g.Genomes[index].Score {
			beta -= g.Genomes[index].Score
			index = (index + 1) % len(g.Genomes)
		}
		result = append(result, g.Genomes[index].Dup())
	}
	g.Genomes = result
}

func (g *Generation) mutation() {
	// Determine the total number of genomes to mutate
	numGenomes := 1 + rand.Intn(len(g.Genomes)-1)
	for i := 0; i < numGenomes; i++ {
		// Pick a random genome
		genomeNum := rand.Intn(len(g.Genomes))
		genome := g.Genomes[genomeNum]
		// Determine the total number of mutations to perform within the genome
		numMutations := 1 + rand.Intn(2)
		// fmt.Printf("\nMutating genome #%v %v times, before:\n%v\n", genomeNum, numMutations, genome)
		genome.Mutate(numMutations)
		// fmt.Printf("after:\n%v\n", genome)
	}
}

// getBest evaluates all genomes and returns a pointer to the best one.
func (g *Generation) getBest() *genome.Genome {
	bestScore := 0.0
	bestGenome := g.Genomes[0]
	c := make(chan *genome.Genome)
	for i := 0; i < len(g.Genomes); i++ { // Evaluate genomes concurrently
		go g.Genomes[i].Evaluate(g.ScoringFunc, c)
	}
	for i := 0; i < len(g.Genomes); i++ { // Collect and return the highest scoring Genome
		gn := <-c
		if gn.Score > bestScore {
			bestGenome = gn
			bestScore = gn.Score
		}
	}
	return bestGenome
}

// maxArity determines the maximum number of input terminals for the given set of symbols.
func maxArity(fs []gene.FuncWeight, fm functions.FuncMap) int {
	r := 0
	for _, f := range fs {
		if fn, ok := fm[f.Symbol]; ok {
			if fn.Terminals() > r {
				r = fn.Terminals()
			}
		} else {
			log.Printf("unable to find symbol %v in function map\n", f.Symbol)
		}
	}
	return r
}
<|endoftext|>


<commit_before>package model

import (
	"bytes"
	"database/sql"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"time"

	"github.com/ViBiOh/funds/crawler"
	"github.com/ViBiOh/funds/db"
	"github.com/ViBiOh/funds/jsonHttp"
	"github.com/ViBiOh/funds/tools"
)

const refreshDelay = 8 * time.Hour

var listRequest = regexp.MustCompile(`^/list$`)
var fundURL string
var fundsMap *tools.ConcurrentMap

type results struct {
	Results interface{} `json:"results"`
}

// Init start concurrent map and init it from crawling
func Init(url string) error {
	fundURL = url
	fundsMap = tools.CreateConcurrentMap(len(fundsIds), crawler.MaxConcurrentFetcher)

	go func() {
		if err := refresh(); err != nil {
			log.Print(err)
		}
		c := time.Tick(refreshDelay)
		for range c {
			if err := refresh(); err != nil {
				log.Print(err)
			}
		}
	}()

	return nil
}

func refresh() error {
	if err := refreshData(); err != nil {
		return fmt.Errorf(`Error while refreshing: %v`, err)
	}

	if db.Ping() {
		if err := saveData(); err != nil {
			return fmt.Errorf(`Error while saving: %v`, err)
		}
	}

	return nil
}

func refreshData() error {
	results, errors := crawler.Crawl(fundsIds, func(ID []byte) (interface{}, error) {
		return fetchFund(ID)
	})

	idsLength := len(fundsIds)
	idsErrors := make([][]byte, 0)

	for i := 0; i < idsLength; i++ {
	select {
		case id := <-errors:
		        idsErrors = append(idsErrors, id)
		        break
		case fund := <-results:
		        fundsMap.Push(fund.(tools.MapContent))
			break
	}
	}

	if len(idsErrors) > 0 {
		return fmt.Errorf(`Errors with ids %s`, bytes.Join(idsErrors, []byte(`,`)))
	}
	return nil
}

func saveData() (err error) {
	var tx *sql.Tx
	if tx, err = db.GetTx(nil); err != nil {
		return err
	}

	defer func() {
		err = db.EndTx(tx, err)
	}()

	for fund := range fundsMap.List() {
		if err == nil {
			err = SaveFund(fund.(Fund), tx)
		}
	}

	return
}

// ListFunds return content of funds' map
func ListFunds() []Fund {
	funds := make([]Fund, 0, len(fundsIds))
	for fund := range fundsMap.List() {
		funds = append(funds, fund.(Fund))
	}

	return funds
}

func listHandler(w http.ResponseWriter, r *http.Request) {
	jsonHttp.ResponseJSON(w, results{ListFunds()})
}

// Handler for model request. Should be use with net/http
type Handler struct {
}

func (handler Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add(`Access-Control-Allow-Origin`, `*`)
	w.Header().Add(`Access-Control-Allow-Headers`, `Content-Type`)
	w.Header().Add(`Access-Control-Allow-Methods`, `GET`)
	w.Header().Add(`X-Content-Type-Options`, `nosniff`)

	if r.Method == http.MethodOptions {
		w.Write(nil)
		return
	}

	urlPath := []byte(r.URL.Path)

	if listRequest.Match(urlPath) {
		if r.Method == http.MethodGet {
			listHandler(w, r)
		} else {
			w.WriteHeader(http.StatusMethodNotAllowed)
		}
	} else {
		w.WriteHeader(http.StatusNotFound)
	}
}
<commit_msg>Fixing go fmt<commit_after>package model

import (
	"bytes"
	"database/sql"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"time"

	"github.com/ViBiOh/funds/crawler"
	"github.com/ViBiOh/funds/db"
	"github.com/ViBiOh/funds/jsonHttp"
	"github.com/ViBiOh/funds/tools"
)

const refreshDelay = 8 * time.Hour

var listRequest = regexp.MustCompile(`^/list$`)
var fundURL string
var fundsMap *tools.ConcurrentMap

type results struct {
	Results interface{} `json:"results"`
}

// Init start concurrent map and init it from crawling
func Init(url string) error {
	fundURL = url
	fundsMap = tools.CreateConcurrentMap(len(fundsIds), crawler.MaxConcurrentFetcher)

	go func() {
		if err := refresh(); err != nil {
			log.Print(err)
		}
		c := time.Tick(refreshDelay)
		for range c {
			if err := refresh(); err != nil {
				log.Print(err)
			}
		}
	}()

	return nil
}

func refresh() error {
	if err := refreshData(); err != nil {
		return fmt.Errorf(`Error while refreshing: %v`, err)
	}

	if db.Ping() {
		if err := saveData(); err != nil {
			return fmt.Errorf(`Error while saving: %v`, err)
		}
	}

	return nil
}

func refreshData() error {
	results, errors := crawler.Crawl(fundsIds, func(ID []byte) (interface{}, error) {
		return fetchFund(ID)
	})

	idsLength := len(fundsIds)
	idsErrors := make([][]byte, 0)

	for i := 0; i < idsLength; i++ {
		select {
		case id := <-errors:
			idsErrors = append(idsErrors, id)
			break
		case fund := <-results:
			fundsMap.Push(fund.(tools.MapContent))
			break
		}
	}

	if len(idsErrors) > 0 {
		return fmt.Errorf(`Errors with ids %s`, bytes.Join(idsErrors, []byte(`,`)))
	}
	return nil
}

func saveData() (err error) {
	var tx *sql.Tx
	if tx, err = db.GetTx(nil); err != nil {
		return err
	}

	defer func() {
		err = db.EndTx(tx, err)
	}()

	for fund := range fundsMap.List() {
		if err == nil {
			err = SaveFund(fund.(Fund), tx)
		}
	}

	return
}

// ListFunds return content of funds' map
func ListFunds() []Fund {
	funds := make([]Fund, 0, len(fundsIds))
	for fund := range fundsMap.List() {
		funds = append(funds, fund.(Fund))
	}

	return funds
}

func listHandler(w http.ResponseWriter, r *http.Request) {
	jsonHttp.ResponseJSON(w, results{ListFunds()})
}

// Handler for model request. Should be use with net/http
type Handler struct {
}

func (handler Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add(`Access-Control-Allow-Origin`, `*`)
	w.Header().Add(`Access-Control-Allow-Headers`, `Content-Type`)
	w.Header().Add(`Access-Control-Allow-Methods`, `GET`)
	w.Header().Add(`X-Content-Type-Options`, `nosniff`)

	if r.Method == http.MethodOptions {
		w.Write(nil)
		return
	}

	urlPath := []byte(r.URL.Path)

	if listRequest.Match(urlPath) {
		if r.Method == http.MethodGet {
			listHandler(w, r)
		} else {
			w.WriteHeader(http.StatusMethodNotAllowed)
		}
	} else {
		w.WriteHeader(http.StatusNotFound)
	}
}
<|endoftext|>