<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal"

import (
	"fmt"
	"regexp"
	"sort"
	"strconv"

	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/model/labels"
	"github.com/prometheus/prometheus/model/textparse"
	"github.com/prometheus/prometheus/model/value"
	"go.opentelemetry.io/collector/pdata/pmetric"
	"go.uber.org/zap"
)

var (
	notUsefulLabelsHistogram = sortString([]string{model.MetricNameLabel, model.InstanceLabel, model.SchemeLabel, model.MetricsPathLabel, model.JobLabel, model.BucketLabel})
	notUsefulLabelsSummary   = sortString([]string{model.MetricNameLabel, model.InstanceLabel, model.SchemeLabel, model.MetricsPathLabel, model.JobLabel, model.QuantileLabel})
	notUsefulLabelsOther     = sortString([]string{model.MetricNameLabel, model.InstanceLabel, model.SchemeLabel, model.MetricsPathLabel, model.JobLabel})
)

func sortString(strs []string) []string {
	sort.Strings(strs)
	return strs
}

func getSortedNotUsefulLabels(mType pmetric.MetricDataType) []string {
	switch mType {
	case pmetric.MetricDataTypeHistogram:
		return notUsefulLabelsHistogram
	case pmetric.MetricDataTypeSummary:
		return notUsefulLabelsSummary
	default:
		return notUsefulLabelsOther
	}
}

func getBoundary(metricType pmetric.MetricDataType, labels labels.Labels) (float64, error) {
	val := ""
	switch metricType {
	case pmetric.MetricDataTypeHistogram:
		val = labels.Get(model.BucketLabel)
		if val == "" {
			return 0, errEmptyLeLabel
		}
	case pmetric.MetricDataTypeSummary:
		val = labels.Get(model.QuantileLabel)
		if val == "" {
			return 0, errEmptyQuantileLabel
		}
	default:
		return 0, errNoBoundaryLabel
	}

	return strconv.ParseFloat(val, 64)
}

// convToMetricType returns the data type and if it is monotonic
func convToMetricType(metricType textparse.MetricType) (pmetric.MetricDataType, bool) {
	switch metricType {
	case textparse.MetricTypeCounter:
		// always use float64, as it's the internal data type used in prometheus
		return pmetric.MetricDataTypeSum, true
	// textparse.MetricTypeUnknown is converted to gauge by default to prevent Prometheus untyped metrics from being dropped
	case textparse.MetricTypeGauge, textparse.MetricTypeUnknown:
		return pmetric.MetricDataTypeGauge, false
	case textparse.MetricTypeHistogram:
		return pmetric.MetricDataTypeHistogram, true
	// dropping support for gaugehistogram for now until we have an official spec of its implementation
	// a draft can be found in: https://docs.google.com/document/d/1KwV0mAXwwbvvifBvDKH_LU1YjyXE_wxCkHNoCGq1GX0/edit#heading=h.1cvzqd4ksd23
	// case textparse.MetricTypeGaugeHistogram:
	//	return <pdata gauge histogram type>
	case textparse.MetricTypeSummary:
		return pmetric.MetricDataTypeSummary, true
	case textparse.MetricTypeInfo, textparse.MetricTypeStateset:
		return pmetric.MetricDataTypeSum, false
	default:
		// including: textparse.MetricTypeGaugeHistogram
		return pmetric.MetricDataTypeNone, false
	}
}

type metricBuilder struct {
	families             map[string]*metricFamily
	hasData              bool
	hasInternalMetric    bool
	mc                   MetadataCache
	useStartTimeMetric   bool
	startTimeMetricRegex *regexp.Regexp
	startTime            float64
	logger               *zap.Logger
}

// newMetricBuilder creates a MetricBuilder which is allowed to feed all the datapoints from a single prometheus
// scraped page by calling its AddDataPoint function, and turn them into a pmetric.Metrics object.
// by calling its Build function
func newMetricBuilder(mc MetadataCache, useStartTimeMetric bool, startTimeMetricRegex string, logger *zap.Logger) *metricBuilder {
	var regex *regexp.Regexp
	if startTimeMetricRegex != "" {
		regex, _ = regexp.Compile(startTimeMetricRegex)
	}
	return &metricBuilder{
		families:             map[string]*metricFamily{},
		mc:                   mc,
		logger:               logger,
		useStartTimeMetric:   useStartTimeMetric,
		startTimeMetricRegex: regex,
	}
}

func (b *metricBuilder) matchStartTimeMetric(metricName string) bool {
	if b.startTimeMetricRegex != nil {
		return b.startTimeMetricRegex.MatchString(metricName)
	}

	return metricName == startTimeMetricName
}

// AddDataPoint is for feeding prometheus data values in its processing order
func (b *metricBuilder) AddDataPoint(ls labels.Labels, t int64, v float64) error {
	// Any datapoint with duplicate labels MUST be rejected per:
	// * https://github.com/open-telemetry/wg-prometheus/issues/44
	// * https://github.com/open-telemetry/opentelemetry-collector/issues/3407
	// as Prometheus rejects such too as of version 2.16.0, released on 2020-02-13.
	var dupLabels []string
	for i := 0; i < len(ls)-1; i++ {
		if ls[i].Name == ls[i+1].Name {
			dupLabels = append(dupLabels, ls[i].Name)
		}
	}
	if len(dupLabels) != 0 {
		sort.Strings(dupLabels)
		return fmt.Errorf("invalid sample: non-unique label names: %q", dupLabels)
	}

	metricName := ls.Get(model.MetricNameLabel)
	switch {
	case metricName == "":
		return errMetricNameNotFound
	case isInternalMetric(metricName):
		b.hasInternalMetric = true
		// See https://www.prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series
		// up: 1 if the instance is healthy, i.e. reachable, or 0 if the scrape failed.
		// But it can also be a staleNaN, which is inserted when the target goes away.
		if metricName == scrapeUpMetricName && v != 1.0 && !value.IsStaleNaN(v) {
			if v == 0.0 {
				b.logger.Warn("Failed to scrape Prometheus endpoint",
					zap.Int64("scrape_timestamp", t),
					zap.Stringer("target_labels", ls))
			} else {
				b.logger.Warn("The 'up' metric contains invalid value",
					zap.Float64("value", v),
					zap.Int64("scrape_timestamp", t),
					zap.Stringer("target_labels", ls))
			}
		}
	case b.useStartTimeMetric && b.matchStartTimeMetric(metricName):
		b.startTime = v
	}

	b.hasData = true

	curMF, ok := b.families[metricName]
	if !ok {
		familyName := normalizeMetricName(metricName)
		if mf, ok := b.families[familyName]; ok && mf.includesMetric(metricName) {
			curMF = mf
		} else {
			curMF = newMetricFamily(metricName, b.mc, b.logger)
			b.families[curMF.name] = curMF
		}
	}

	return curMF.Add(metricName, ls, t, v)
}

// appendMetrics appends all metrics to the given slice.
// The only error returned by this function is errNoDataToBuild.
func (b *metricBuilder) appendMetrics(metrics pmetric.MetricSlice) error {
	if !b.hasData {
		if b.hasInternalMetric {
			return nil
		}
		return errNoDataToBuild
	}

	for _, mf := range b.families {
		mf.appendMetric(metrics)
	}

	return nil
}
<commit_msg>[chore] prometheusreceiver: remove impossible condition, internal and no data (#13769)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal"

import (
	"fmt"
	"regexp"
	"sort"
	"strconv"

	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/model/labels"
	"github.com/prometheus/prometheus/model/textparse"
	"github.com/prometheus/prometheus/model/value"
	"go.opentelemetry.io/collector/pdata/pmetric"
	"go.uber.org/zap"
)

var (
	notUsefulLabelsHistogram = sortString([]string{model.MetricNameLabel, model.InstanceLabel, model.SchemeLabel, model.MetricsPathLabel, model.JobLabel, model.BucketLabel})
	notUsefulLabelsSummary   = sortString([]string{model.MetricNameLabel, model.InstanceLabel, model.SchemeLabel, model.MetricsPathLabel, model.JobLabel, model.QuantileLabel})
	notUsefulLabelsOther     = sortString([]string{model.MetricNameLabel, model.InstanceLabel, model.SchemeLabel, model.MetricsPathLabel, model.JobLabel})
)

func sortString(strs []string) []string {
	sort.Strings(strs)
	return strs
}

func getSortedNotUsefulLabels(mType pmetric.MetricDataType) []string {
	switch mType {
	case pmetric.MetricDataTypeHistogram:
		return notUsefulLabelsHistogram
	case pmetric.MetricDataTypeSummary:
		return notUsefulLabelsSummary
	default:
		return notUsefulLabelsOther
	}
}

func getBoundary(metricType pmetric.MetricDataType, labels labels.Labels) (float64, error) {
	val := ""
	switch metricType {
	case pmetric.MetricDataTypeHistogram:
		val = labels.Get(model.BucketLabel)
		if val == "" {
			return 0, errEmptyLeLabel
		}
	case pmetric.MetricDataTypeSummary:
		val = labels.Get(model.QuantileLabel)
		if val == "" {
			return 0, errEmptyQuantileLabel
		}
	default:
		return 0, errNoBoundaryLabel
	}

	return strconv.ParseFloat(val, 64)
}

// convToMetricType returns the data type and if it is monotonic
func convToMetricType(metricType textparse.MetricType) (pmetric.MetricDataType, bool) {
	switch metricType {
	case textparse.MetricTypeCounter:
		// always use float64, as it's the internal data type used in prometheus
		return pmetric.MetricDataTypeSum, true
	// textparse.MetricTypeUnknown is converted to gauge by default to prevent Prometheus untyped metrics from being dropped
	case textparse.MetricTypeGauge, textparse.MetricTypeUnknown:
		return pmetric.MetricDataTypeGauge, false
	case textparse.MetricTypeHistogram:
		return pmetric.MetricDataTypeHistogram, true
	// dropping support for gaugehistogram for now until we have an official spec of its implementation
	// a draft can be found in: https://docs.google.com/document/d/1KwV0mAXwwbvvifBvDKH_LU1YjyXE_wxCkHNoCGq1GX0/edit#heading=h.1cvzqd4ksd23
	// case textparse.MetricTypeGaugeHistogram:
	//	return <pdata gauge histogram type>
	case textparse.MetricTypeSummary:
		return pmetric.MetricDataTypeSummary, true
	case textparse.MetricTypeInfo, textparse.MetricTypeStateset:
		return pmetric.MetricDataTypeSum, false
	default:
		// including: textparse.MetricTypeGaugeHistogram
		return pmetric.MetricDataTypeNone, false
	}
}

type metricBuilder struct {
	families             map[string]*metricFamily
	hasData              bool
	mc                   MetadataCache
	useStartTimeMetric   bool
	startTimeMetricRegex *regexp.Regexp
	startTime            float64
	logger               *zap.Logger
}

// newMetricBuilder creates a MetricBuilder which is allowed to feed all the datapoints from a single prometheus
// scraped page by calling its AddDataPoint function, and turn them into a pmetric.Metrics object.
// by calling its Build function
func newMetricBuilder(mc MetadataCache, useStartTimeMetric bool, startTimeMetricRegex string, logger *zap.Logger) *metricBuilder {
	var regex *regexp.Regexp
	if startTimeMetricRegex != "" {
		regex, _ = regexp.Compile(startTimeMetricRegex)
	}
	return &metricBuilder{
		families:             map[string]*metricFamily{},
		mc:                   mc,
		logger:               logger,
		useStartTimeMetric:   useStartTimeMetric,
		startTimeMetricRegex: regex,
	}
}

func (b *metricBuilder) matchStartTimeMetric(metricName string) bool {
	if b.startTimeMetricRegex != nil {
		return b.startTimeMetricRegex.MatchString(metricName)
	}

	return metricName == startTimeMetricName
}

// AddDataPoint is for feeding prometheus data values in its processing order
func (b *metricBuilder) AddDataPoint(ls labels.Labels, t int64, v float64) error {
	// Any datapoint with duplicate labels MUST be rejected per:
	// * https://github.com/open-telemetry/wg-prometheus/issues/44
	// * https://github.com/open-telemetry/opentelemetry-collector/issues/3407
	// as Prometheus rejects such too as of version 2.16.0, released on 2020-02-13.
	var dupLabels []string
	for i := 0; i < len(ls)-1; i++ {
		if ls[i].Name == ls[i+1].Name {
			dupLabels = append(dupLabels, ls[i].Name)
		}
	}
	if len(dupLabels) != 0 {
		sort.Strings(dupLabels)
		return fmt.Errorf("invalid sample: non-unique label names: %q", dupLabels)
	}

	metricName := ls.Get(model.MetricNameLabel)
	switch {
	case metricName == "":
		return errMetricNameNotFound
	case isInternalMetric(metricName):
		// See https://www.prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series
		// up: 1 if the instance is healthy, i.e. reachable, or 0 if the scrape failed.
		// But it can also be a staleNaN, which is inserted when the target goes away.
		if metricName == scrapeUpMetricName && v != 1.0 && !value.IsStaleNaN(v) {
			if v == 0.0 {
				b.logger.Warn("Failed to scrape Prometheus endpoint",
					zap.Int64("scrape_timestamp", t),
					zap.Stringer("target_labels", ls))
			} else {
				b.logger.Warn("The 'up' metric contains invalid value",
					zap.Float64("value", v),
					zap.Int64("scrape_timestamp", t),
					zap.Stringer("target_labels", ls))
			}
		}
	case b.useStartTimeMetric && b.matchStartTimeMetric(metricName):
		b.startTime = v
	}

	b.hasData = true

	curMF, ok := b.families[metricName]
	if !ok {
		familyName := normalizeMetricName(metricName)
		if mf, ok := b.families[familyName]; ok && mf.includesMetric(metricName) {
			curMF = mf
		} else {
			curMF = newMetricFamily(metricName, b.mc, b.logger)
			b.families[curMF.name] = curMF
		}
	}

	return curMF.Add(metricName, ls, t, v)
}

// appendMetrics appends all metrics to the given slice.
// The only error returned by this function is errNoDataToBuild.
func (b *metricBuilder) appendMetrics(metrics pmetric.MetricSlice) error {
	if !b.hasData {
		return errNoDataToBuild
	}

	for _, mf := range b.families {
		mf.appendMetric(metrics)
	}

	return nil
}
<|endoftext|>


<commit_before>package db

import (
	"container/list"
	"encoding/binary"
	"fmt"
	"strings"
	"time"

	"github.com/google/btree"
	query "github.com/gtfierro/hod/query"
	"github.com/syndtr/goleveldb/leveldb"
)

// make a set of structs that capture what these queries want to do

// So, how do queries work?
// We have a list of filters, each of which has a subject, list of predicate things, and
// an object. Any of these might be variables, which we can distinguish by having a "?"
// in front of the value.
//
// First we "clean" these by making sure that they have their full
// namespaces rather than the prefix

type Item [4]byte

func (i Item) Less(than btree.Item) bool {
	t := than.(Item)
	return binary.LittleEndian.Uint32(i[:]) < binary.LittleEndian.Uint32(t[:])
}

func (db *DB) RunQuery(q query.Query) {
	// "clean" the query by expanding out the prefixes
	for idx, filter := range q.Where {
		if !strings.HasPrefix(filter.Subject.Value, "?") {
			if full, found := db.namespaces[filter.Subject.Namespace]; found {
				filter.Subject.Namespace = full
			}
			q.Where[idx] = filter
		}
		if !strings.HasPrefix(filter.Object.Value, "?") {
			if full, found := db.namespaces[filter.Object.Namespace]; found {
				filter.Object.Namespace = full
			}
			q.Where[idx] = filter
		}
		for idx2, pred := range filter.Path {
			if !strings.HasPrefix(pred.Predicate.Value, "?") {
				if full, found := db.namespaces[pred.Predicate.Namespace]; found {
					pred.Predicate.Namespace = full
				}
				filter.Path[idx2] = pred
			}
		}
		q.Where[idx] = filter
	}

	fmt.Println("-------------- start query plan -------------")
	planStart := time.Now()
	dg := db.formDependencyGraph(q)
	qp := db.formQueryPlan(dg)
	for _, op := range qp.operations {
		log.Debug("op", op)
	}
	log.Infof("Formed execution plan in %s", time.Since(planStart))
	fmt.Println("-------------- end query plan -------------")

	runStart := time.Now()
	rm := db.executeQueryPlan(qp)
	log.Infof("Ran query in %s", time.Since(runStart))

	runStart = time.Now()
	results := db.expandTuples(rm, qp.selectVars)
	log.Infof("Expanded tuples in %s", time.Since(runStart))
	if q.Select.Count {
		fmt.Println(len(results))
	} else {
		for _, r := range results {
			fmt.Println(r)
		}
	}
	return
}

// We need an execution plan for the list of filters contained in a query. How do we do this?
func (db *DB) formDependencyGraph(q query.Query) *dependencyGraph {
	dg := makeDependencyGraph(q)
	terms := make([]*queryTerm, len(q.Where))
	for i, f := range q.Where {
		terms[i] = dg.makeQueryTerm(f)
	}

	numUnresolved := func(qt *queryTerm) int {
		num := 0
		for _, v := range qt.variables {
			if !dg.variables[v] {
				num++
			}
		}
		return num
	}

	for len(terms) > 0 {
		// first find all the terms with 0 or 1 unresolved variable terms
		var added = []*queryTerm{}
		for _, term := range terms {
			if numUnresolved(term) < 2 {
				dg.addRootTerm(term)
				added = append(added, term)
			}
		}
		// remove the terms that we added to the root set
		terms = filterTermList(terms, added)
		added = []*queryTerm{}
		for _, term := range terms {
			if dg.addChild(term) {
				added = append(added, term)
			}
		}
		terms = filterTermList(terms, added)
	}
	dg.dump()
	return dg
}

func (db *DB) executeQueryPlan(qp *queryPlan) *resultMap {
	rm := newResultMap()
	rm.varOrder = qp.varOrder
	var err error
	for _, op := range qp.operations {
		rm, err = op.run(db, qp.varOrder, rm)
		if err != nil {
			log.Fatal(err)
		}
	}
	//for vname := range qp.varOrder.vars {
	//	fmt.Println(vname, rm.getVariableChain(vname))
	//	for re := range rm.iterVariable(vname) {
	//		fmt.Println(vname, re)
	//	}
	//}
	for vname, tree := range rm.vars {
		fmt.Println(vname, tree.Len())
	}
	return rm
}

// takes the inverse of every relationship. If no inverse exists, returns nil
func (db *DB) reversePathPattern(path []query.PathPattern) []query.PathPattern {
	var reverse = make([]query.PathPattern, len(path))
	for idx, pred := range path {
		if inverse, found := db.relationships[pred.Predicate]; found {
			pred.Predicate = inverse
			reverse[idx] = pred
		} else {
			return nil
		}
	}
	return reverse
}

// follow the pattern from the given object's InEdges, placing the results in the btree
func (db *DB) followPathFromObject(object *Entity, results *btree.BTree, searchstack *list.List, pattern query.PathPattern) {
	stack := list.New()
	stack.PushFront(object)

	predHash, err := db.GetHash(pattern.Predicate)
	if err != nil && err == leveldb.ErrNotFound {
		log.Infof("Adding unseen predicate %s", pattern.Predicate)
		var hashdest [4]byte
		if err := db.insertEntity(pattern.Predicate, hashdest[:]); err != nil {
			panic(fmt.Errorf("Could not insert entity %s (%v)", pattern.Predicate, err))
		}
	} else if err != nil {
		panic(fmt.Errorf("Not found: %v (%s)", pattern.Predicate, err))
	}

	for stack.Len() > 0 {
		entity := stack.Remove(stack.Front()).(*Entity)
		switch pattern.Pattern {
		case query.PATTERN_SINGLE:
			// [found] indicates whether or not we have any edges with the given pattern
			edges, found := entity.InEdges[string(predHash[:])]
			// this requires the pattern to exist, so we skip if we have no edges of that name
			if !found {
				continue
			}
			// here, these entities are all connected by the required predicate
			for _, entityHash := range edges {
				nextEntity := db.MustGetEntityFromHash(entityHash)
				results.ReplaceOrInsert(Item(nextEntity.PK))
				searchstack.PushBack(nextEntity)
			}
			// because this is one hop, we don't add any new entities to the stack
		case query.PATTERN_ZERO_ONE:
			log.Notice("PATH ?", pattern)
			// this does not require the pattern to exist, so we add ALL entities connected
			// by ALL edges
			for _, endpointHashList := range entity.InEdges {
				for _, entityHash := range endpointHashList {
					nextEntity := db.MustGetEntityFromHash(entityHash)
					results.ReplaceOrInsert(Item(nextEntity.PK))
					searchstack.PushBack(nextEntity)
				}
			}
			// because this is one hop, we don't add any new entities to the stack
		case query.PATTERN_ZERO_PLUS:
			log.Notice("PATH *", pattern)
		case query.PATTERN_ONE_PLUS:
			edges, found := entity.InEdges[string(predHash[:])]
			// this requires the pattern to exist, so we skip if we have no edges of that name
			if !found {
				continue
			}
			// here, these entities are all connected by the required predicate
			for _, entityHash := range edges {
				nextEntity := db.MustGetEntityFromHash(entityHash)
				results.ReplaceOrInsert(Item(nextEntity.PK))
				searchstack.PushBack(nextEntity)
				// also make sure to add this to the stack so we can search
				stack.PushBack(nextEntity)
			}
		}
	}
}

// follow the pattern from the given subject's OutEdges, placing the results in the btree
func (db *DB) followPathFromSubject(subject *Entity, results *btree.BTree, searchstack *list.List, pattern query.PathPattern) {
	stack := list.New()
	stack.PushFront(subject)

	predHash, err := db.GetHash(pattern.Predicate)
	if err != nil {
		panic(err)
	}

	for stack.Len() > 0 {
		entity := stack.Remove(stack.Front()).(*Entity)
		switch pattern.Pattern {
		case query.PATTERN_SINGLE:
			// [found] indicates whether or not we have any edges with the given pattern
			edges, found := entity.OutEdges[string(predHash[:])]
			// this requires the pattern to exist, so we skip if we have no edges of that name
			if !found {
				continue
			}
			// here, these entities are all connected by the required predicate
			for _, entityHash := range edges {
				nextEntity := db.MustGetEntityFromHash(entityHash)
				results.ReplaceOrInsert(Item(nextEntity.PK))
				searchstack.PushBack(nextEntity)
			}
			// because this is one hop, we don't add any new entities to the stack
		case query.PATTERN_ZERO_ONE:
			// this does not require the pattern to exist, so we add ALL entities connected
			// by ALL edges
			for _, endpointHashList := range entity.OutEdges {
				for _, entityHash := range endpointHashList {
					nextEntity := db.MustGetEntityFromHash(entityHash)
					results.ReplaceOrInsert(Item(nextEntity.PK))
					searchstack.PushBack(nextEntity)
				}
			}
			// because this is one hop, we don't add any new entities to the stack
		case query.PATTERN_ZERO_PLUS:
			panic("TODO TODO TODO")
		case query.PATTERN_ONE_PLUS:
			edges, found := entity.OutEdges[string(predHash[:])]
			// this requires the pattern to exist, so we skip if we have no edges of that name
			if !found {
				continue
			}
			// here, these entities are all connected by the required predicate
			for _, entityHash := range edges {
				nextEntity := db.MustGetEntityFromHash(entityHash)
				results.ReplaceOrInsert(Item(nextEntity.PK))
				searchstack.PushBack(nextEntity)
				// also make sure to add this to the stack so we can search
				stack.PushBack(nextEntity)
			}
		}
	}
}

func (db *DB) getSubjectFromPredObject(objectHash [4]byte, path []query.PathPattern) *btree.BTree {
	// first get the initial object entity from the db
	// then we're going to conduct a BFS search starting from this entity looking for all entities
	// that have the required path sequence. We place the results in a BTree to maintain uniqueness

	// So how does this traversal actually work?
	// At each 'step', we are looking at an entity and some offset into the path.

	// get the object, look in its "in" edges for the path pattern
	objEntity, err := db.GetEntityFromHash(objectHash)
	if err != nil {
		panic(err)
	}

	results := btree.New(2)

	stack := list.New()
	stack.PushFront(objEntity)

	for stack.Len() > 0 {
		entity := stack.Remove(stack.Front()).(*Entity)
		for _, pat := range path {
			db.followPathFromObject(entity, results, stack, pat)
		}
	}

	return results
}

// Given object and predicate, get all subjects
func (db *DB) getObjectFromSubjectPred(subjectHash [4]byte, path []query.PathPattern) *btree.BTree {
	subEntity, err := db.GetEntityFromHash(subjectHash)
	if err != nil {
		panic(err)
	}

	results := btree.New(2)

	stack := list.New()
	stack.PushFront(subEntity)

	for stack.Len() > 0 {
		entity := stack.Remove(stack.Front()).(*Entity)
		for _, pat := range path {
			db.followPathFromSubject(entity, results, stack, pat)
		}
	}

	return results
}

// Given a predicate, it returns pairs of (subject, object) that are connected by that relationship
func (db *DB) getSubjectObjectFromPred(path []query.PathPattern) (soPair [][][4]byte) {
	//pe, found := db.predIndex[pattern.Predicate]
	//if !found {
	//	panic(fmt.Sprintf("Cannot find predicate %s", pattern.Predicate))
	//}
	//for subject, objectMap := range pe.Subjects {
	//	for object := range objectMap {
	//		var sh, oh [4]byte
	//		copy(sh[:], subject)
	//		copy(oh[:], object)
	//		soPair = append(soPair, [][4]byte{sh, oh})
	//	}
	//}
	return soPair
}
<commit_msg>detecting when we need to skip to 2-var terms in dependency graph<commit_after>package db

import (
	"container/list"
	"encoding/binary"
	"fmt"
	"strings"
	"time"

	"github.com/google/btree"
	query "github.com/gtfierro/hod/query"
	"github.com/syndtr/goleveldb/leveldb"
)

// make a set of structs that capture what these queries want to do

// So, how do queries work?
// We have a list of filters, each of which has a subject, list of predicate things, and
// an object. Any of these might be variables, which we can distinguish by having a "?"
// in front of the value.
//
// First we "clean" these by making sure that they have their full
// namespaces rather than the prefix

type Item [4]byte

func (i Item) Less(than btree.Item) bool {
	t := than.(Item)
	return binary.LittleEndian.Uint32(i[:]) < binary.LittleEndian.Uint32(t[:])
}

func (db *DB) RunQuery(q query.Query) {
	// "clean" the query by expanding out the prefixes
	for idx, filter := range q.Where {
		if !strings.HasPrefix(filter.Subject.Value, "?") {
			if full, found := db.namespaces[filter.Subject.Namespace]; found {
				filter.Subject.Namespace = full
			}
			q.Where[idx] = filter
		}
		if !strings.HasPrefix(filter.Object.Value, "?") {
			if full, found := db.namespaces[filter.Object.Namespace]; found {
				filter.Object.Namespace = full
			}
			q.Where[idx] = filter
		}
		for idx2, pred := range filter.Path {
			if !strings.HasPrefix(pred.Predicate.Value, "?") {
				if full, found := db.namespaces[pred.Predicate.Namespace]; found {
					pred.Predicate.Namespace = full
				}
				filter.Path[idx2] = pred
			}
		}
		q.Where[idx] = filter
	}

	fmt.Println("-------------- start query plan -------------")
	planStart := time.Now()
	dg := db.formDependencyGraph(q)
	qp := db.formQueryPlan(dg)
	for _, op := range qp.operations {
		log.Debug("op", op)
	}
	log.Infof("Formed execution plan in %s", time.Since(planStart))
	fmt.Println("-------------- end query plan -------------")

	runStart := time.Now()
	rm := db.executeQueryPlan(qp)
	log.Infof("Ran query in %s", time.Since(runStart))

	runStart = time.Now()
	results := db.expandTuples(rm, qp.selectVars)
	log.Infof("Expanded tuples in %s", time.Since(runStart))
	if q.Select.Count {
		fmt.Println(len(results))
	} else {
		for _, r := range results {
			fmt.Println(r)
		}
	}
	return
}

// We need an execution plan for the list of filters contained in a query. How do we do this?
func (db *DB) formDependencyGraph(q query.Query) *dependencyGraph {
	dg := makeDependencyGraph(q)
	terms := make([]*queryTerm, len(q.Where))
	for i, f := range q.Where {
		terms[i] = dg.makeQueryTerm(f)
	}

	numUnresolved := func(qt *queryTerm) int {
		num := 0
		for _, v := range qt.variables {
			if !dg.variables[v] {
				num++
			}
		}
		return num
	}

	originalLength := len(terms)
	for len(terms) > 0 {
		// first find all the terms with 0 or 1 unresolved variable terms
		var added = []*queryTerm{}
		for _, term := range terms {
			if numUnresolved(term) < 2 {
				dg.addRootTerm(term)
				added = append(added, term)
			}
		}
		// remove the terms that we added to the root set
		terms = filterTermList(terms, added)
		added = []*queryTerm{}
		for _, term := range terms {
			if dg.addChild(term) {
				added = append(added, term)
			}
		}
		terms = filterTermList(terms, added)
		if len(terms) == originalLength {
			// we don't have any root elements. Need to consider 2-variable terms
			added = []*queryTerm{}
			for _, term := range terms {
				if numUnresolved(term) == 2 {
					dg.addRootTerm(term)
					added = append(added, term)
					break
				}
			}
			terms = filterTermList(terms, added)
		}
	}
	dg.dump()
	return dg
}

func (db *DB) executeQueryPlan(qp *queryPlan) *resultMap {
	rm := newResultMap()
	rm.varOrder = qp.varOrder
	var err error
	for _, op := range qp.operations {
		rm, err = op.run(db, qp.varOrder, rm)
		if err != nil {
			log.Fatal(err)
		}
	}
	//for vname := range qp.varOrder.vars {
	//	fmt.Println(vname, rm.getVariableChain(vname))
	//	for re := range rm.iterVariable(vname) {
	//		fmt.Println(vname, re)
	//	}
	//}
	for vname, tree := range rm.vars {
		fmt.Println(vname, tree.Len())
	}
	return rm
}

// takes the inverse of every relationship. If no inverse exists, returns nil
func (db *DB) reversePathPattern(path []query.PathPattern) []query.PathPattern {
	var reverse = make([]query.PathPattern, len(path))
	for idx, pred := range path {
		if inverse, found := db.relationships[pred.Predicate]; found {
			pred.Predicate = inverse
			reverse[idx] = pred
		} else {
			return nil
		}
	}
	return reverse
}

// follow the pattern from the given object's InEdges, placing the results in the btree
func (db *DB) followPathFromObject(object *Entity, results *btree.BTree, searchstack *list.List, pattern query.PathPattern) {
	stack := list.New()
	stack.PushFront(object)

	predHash, err := db.GetHash(pattern.Predicate)
	if err != nil && err == leveldb.ErrNotFound {
		log.Infof("Adding unseen predicate %s", pattern.Predicate)
		var hashdest [4]byte
		if err := db.insertEntity(pattern.Predicate, hashdest[:]); err != nil {
			panic(fmt.Errorf("Could not insert entity %s (%v)", pattern.Predicate, err))
		}
	} else if err != nil {
		panic(fmt.Errorf("Not found: %v (%s)", pattern.Predicate, err))
	}

	for stack.Len() > 0 {
		entity := stack.Remove(stack.Front()).(*Entity)
		switch pattern.Pattern {
		case query.PATTERN_SINGLE:
			// [found] indicates whether or not we have any edges with the given pattern
			edges, found := entity.InEdges[string(predHash[:])]
			// this requires the pattern to exist, so we skip if we have no edges of that name
			if !found {
				continue
			}
			// here, these entities are all connected by the required predicate
			for _, entityHash := range edges {
				nextEntity := db.MustGetEntityFromHash(entityHash)
				results.ReplaceOrInsert(Item(nextEntity.PK))
				searchstack.PushBack(nextEntity)
			}
			// because this is one hop, we don't add any new entities to the stack
		case query.PATTERN_ZERO_ONE:
			log.Notice("PATH ?", pattern)
			// this does not require the pattern to exist, so we add ALL entities connected
			// by ALL edges
			for _, endpointHashList := range entity.InEdges {
				for _, entityHash := range endpointHashList {
					nextEntity := db.MustGetEntityFromHash(entityHash)
					results.ReplaceOrInsert(Item(nextEntity.PK))
					searchstack.PushBack(nextEntity)
				}
			}
			// because this is one hop, we don't add any new entities to the stack
		case query.PATTERN_ZERO_PLUS:
			log.Notice("PATH *", pattern)
		case query.PATTERN_ONE_PLUS:
			edges, found := entity.InEdges[string(predHash[:])]
			// this requires the pattern to exist, so we skip if we have no edges of that name
			if !found {
				continue
			}
			// here, these entities are all connected by the required predicate
			for _, entityHash := range edges {
				nextEntity := db.MustGetEntityFromHash(entityHash)
				results.ReplaceOrInsert(Item(nextEntity.PK))
				searchstack.PushBack(nextEntity)
				// also make sure to add this to the stack so we can search
				stack.PushBack(nextEntity)
			}
		}
	}
}

// follow the pattern from the given subject's OutEdges, placing the results in the btree
func (db *DB) followPathFromSubject(subject *Entity, results *btree.BTree, searchstack *list.List, pattern query.PathPattern) {
	stack := list.New()
	stack.PushFront(subject)

	predHash, err := db.GetHash(pattern.Predicate)
	if err != nil {
		panic(err)
	}

	for stack.Len() > 0 {
		entity := stack.Remove(stack.Front()).(*Entity)
		switch pattern.Pattern {
		case query.PATTERN_SINGLE:
			// [found] indicates whether or not we have any edges with the given pattern
			edges, found := entity.OutEdges[string(predHash[:])]
			// this requires the pattern to exist, so we skip if we have no edges of that name
			if !found {
				continue
			}
			// here, these entities are all connected by the required predicate
			for _, entityHash := range edges {
				nextEntity := db.MustGetEntityFromHash(entityHash)
				results.ReplaceOrInsert(Item(nextEntity.PK))
				searchstack.PushBack(nextEntity)
			}
			// because this is one hop, we don't add any new entities to the stack
		case query.PATTERN_ZERO_ONE:
			// this does not require the pattern to exist, so we add ALL entities connected
			// by ALL edges
			for _, endpointHashList := range entity.OutEdges {
				for _, entityHash := range endpointHashList {
					nextEntity := db.MustGetEntityFromHash(entityHash)
					results.ReplaceOrInsert(Item(nextEntity.PK))
					searchstack.PushBack(nextEntity)
				}
			}
			// because this is one hop, we don't add any new entities to the stack
		case query.PATTERN_ZERO_PLUS:
			panic("TODO TODO TODO")
		case query.PATTERN_ONE_PLUS:
			edges, found := entity.OutEdges[string(predHash[:])]
			// this requires the pattern to exist, so we skip if we have no edges of that name
			if !found {
				continue
			}
			// here, these entities are all connected by the required predicate
			for _, entityHash := range edges {
				nextEntity := db.MustGetEntityFromHash(entityHash)
				results.ReplaceOrInsert(Item(nextEntity.PK))
				searchstack.PushBack(nextEntity)
				// also make sure to add this to the stack so we can search
				stack.PushBack(nextEntity)
			}
		}
	}
}

func (db *DB) getSubjectFromPredObject(objectHash [4]byte, path []query.PathPattern) *btree.BTree {
	// first get the initial object entity from the db
	// then we're going to conduct a BFS search starting from this entity looking for all entities
	// that have the required path sequence. We place the results in a BTree to maintain uniqueness

	// So how does this traversal actually work?
	// At each 'step', we are looking at an entity and some offset into the path.

	// get the object, look in its "in" edges for the path pattern
	objEntity, err := db.GetEntityFromHash(objectHash)
	if err != nil {
		panic(err)
	}

	results := btree.New(2)

	stack := list.New()
	stack.PushFront(objEntity)

	for stack.Len() > 0 {
		entity := stack.Remove(stack.Front()).(*Entity)
		for _, pat := range path {
			db.followPathFromObject(entity, results, stack, pat)
		}
	}

	return results
}

// Given object and predicate, get all subjects
func (db *DB) getObjectFromSubjectPred(subjectHash [4]byte, path []query.PathPattern) *btree.BTree {
	subEntity, err := db.GetEntityFromHash(subjectHash)
	if err != nil {
		panic(err)
	}

	results := btree.New(2)

	stack := list.New()
	stack.PushFront(subEntity)

	for stack.Len() > 0 {
		entity := stack.Remove(stack.Front()).(*Entity)
		for _, pat := range path {
			db.followPathFromSubject(entity, results, stack, pat)
		}
	}

	return results
}

// Given a predicate, it returns pairs of (subject, object) that are connected by that relationship
func (db *DB) getSubjectObjectFromPred(path []query.PathPattern) (soPair [][][4]byte) {
	//pe, found := db.predIndex[pattern.Predicate]
	//if !found {
	//	panic(fmt.Sprintf("Cannot find predicate %s", pattern.Predicate))
	//}
	//for subject, objectMap := range pe.Subjects {
	//	for object := range objectMap {
	//		var sh, oh [4]byte
	//		copy(sh[:], subject)
	//		copy(oh[:], object)
	//		soPair = append(soPair, [][4]byte{sh, oh})
	//	}
	//}
	return soPair
}
<|endoftext|>


<commit_before>package HumorChecker

import "testing"

var negAndPosTests = []struct {
	Function string
	Type     string
	In       string
	Expected float64
}{
	{"Negativity", "Score", "bastard", 5.0},
	{"Negativity", "Score", "scumbag", 4.0},
	{"Negativity", "Score", "evil", 3.0},
	{"Negativity", "Score", "ache", 2.0},
	{"Negativity", "Score", "anti", 1.0},

	{"Negativity", "Comparative", "Hey scumbag", 2.0},
	{"Negativity", "Comparative", "I'll be here till 5", 0},

	{"Negativity", "WordCount", "This is two anti evil words", 2.0},

	{"Positivity", "Score", "superb", 5.0},
	{"Positivity", "Score", "amazing", 4.0},
	{"Positivity", "Score", "admire", 3.0},
	{"Positivity", "Score", "amaze", 2.0},
	{"Positivity", "Score", "cool", 1.0},

	{"Positivity", "Comparative", "Hey amazing", 2.0},
	{"Positivity", "Comparative", "I'll be here till 5", 0.0},

	{"Positivity", "WordCount", "This is two amazing cool words", 2.0},
}

func TestNegAndPos(t *testing.T) {
	var f func(string) Score
	for _, test := range negAndPosTests {
		if test.Function == "Negativity" {
			f = Negativity
		} else if test.Function == "Positivity" {
			f = Positivity
		} else {
			t.Fatal("Impossible humor type check")
		}

		result := f(test.In)
		if test.Type == "Score" {
			if result.Score != test.Expected {
				t.Errorf("got wrong score for %s in %s. got: %v. expected: %v", test.Function, test.In, result.Score, test.Expected)
			}
		} else if test.Type == "Comparative" {
			if result.Comparative != test.Expected {
				t.Errorf("got wrong comparative for %s in %s. got: %v. expected: %v", test.Function, test.In, result.Comparative, test.Expected)
			}
		} else if test.Type == "WordCount" {
			if len(result.Words) != int(test.Expected) {
				t.Errorf("got wrong comparative for %s in %s. got: %v. expected: %v", test.Function, test.In, len(result.Words), int(test.Expected))
			}
		} else {
			t.Fatal("Impossible test")
		}
	}
}

func TestAnalyze(t *testing.T) {
	// if v := Analyze("Hey Amazing Scumbag").Score; v != 0 {
	// 	t.Errorf("error analyzing score of sentence. got: %v", v)
	// }
	// if v := Analyze("Cool beans").Score; v != 1 {
	// 	t.Errorf("error analyzing score of sentence: should be positive for only positives. got: %v", v)
	// }
	// if v := Analyze("Hey scumbag").Score; v != -4 {
	// 	t.Errorf("error analyzing score of sentence: should be negative for only negatives. got: %v", v)
	// }
	if v := Analyze("Fearless!").Score; v != 2 {
		t.Errorf("error analyzing score of sentence: punctuation must be ignored (positive). got: %v", v)
	}
	if v := Analyze("Crash!").Score; v != -2 {
		t.Errorf("error analyzing score of sentence: punctuation must be ignored (negative). got: %v", v)
	}
	if v := Analyze("#fearless").Score; v != 2 {
		t.Errorf("error analyzing score of sentence: hashtags must be ignored (positive). got: %v", v)
	}
	if v := Analyze("#crash").Score; v != -2 {
		t.Errorf("error analyzing score of sentence: hashtags must be ignored (negative). got: %v", v)
	}
	// if v := Analyze("An amazing anti").Comparative; v != 1 {
	// 	t.Errorf("error analyzing comparative score of sentence. got: %v", v)
	// }
}
<commit_msg>Addeed benchmark<commit_after>package HumorChecker

import "testing"

var negAndPosTests = []struct {
	Function string
	Type     string
	In       string
	Expected float64
}{
	{"Negativity", "Score", "bastard", 5.0},
	{"Negativity", "Score", "scumbag", 4.0},
	{"Negativity", "Score", "evil", 3.0},
	{"Negativity", "Score", "ache", 2.0},
	{"Negativity", "Score", "anti", 1.0},

	{"Negativity", "Comparative", "Hey scumbag", 2.0},
	{"Negativity", "Comparative", "I'll be here till 5", 0},

	{"Negativity", "WordCount", "This is two anti evil words", 2.0},

	{"Positivity", "Score", "superb", 5.0},
	{"Positivity", "Score", "amazing", 4.0},
	{"Positivity", "Score", "admire", 3.0},
	{"Positivity", "Score", "amaze", 2.0},
	{"Positivity", "Score", "cool", 1.0},

	{"Positivity", "Comparative", "Hey amazing", 2.0},
	{"Positivity", "Comparative", "I'll be here till 5", 0.0},

	{"Positivity", "WordCount", "This is two amazing cool words", 2.0},
}

func TestNegAndPos(t *testing.T) {
	var f func(string) Score
	for _, test := range negAndPosTests {
		if test.Function == "Negativity" {
			f = Negativity
		} else if test.Function == "Positivity" {
			f = Positivity
		} else {
			t.Fatal("Impossible humor type check")
		}

		result := f(test.In)
		if test.Type == "Score" {
			if result.Score != test.Expected {
				t.Errorf("got wrong score for %s in %s. got: %v. expected: %v", test.Function, test.In, result.Score, test.Expected)
			}
		} else if test.Type == "Comparative" {
			if result.Comparative != test.Expected {
				t.Errorf("got wrong comparative for %s in %s. got: %v. expected: %v", test.Function, test.In, result.Comparative, test.Expected)
			}
		} else if test.Type == "WordCount" {
			if len(result.Words) != int(test.Expected) {
				t.Errorf("got wrong comparative for %s in %s. got: %v. expected: %v", test.Function, test.In, len(result.Words), int(test.Expected))
			}
		} else {
			t.Fatal("Impossible test")
		}
	}
}

func TestAnalyze(t *testing.T) {
	// if v := Analyze("Hey Amazing Scumbag").Score; v != 0 {
	// 	t.Errorf("error analyzing score of sentence. got: %v", v)
	// }
	// if v := Analyze("Cool beans").Score; v != 1 {
	// 	t.Errorf("error analyzing score of sentence: should be positive for only positives. got: %v", v)
	// }
	// if v := Analyze("Hey scumbag").Score; v != -4 {
	// 	t.Errorf("error analyzing score of sentence: should be negative for only negatives. got: %v", v)
	// }
	if v := Analyze("Fearless!").Score; v != 2 {
		t.Errorf("error analyzing score of sentence: punctuation must be ignored (positive). got: %v", v)
	}
	if v := Analyze("Crash!").Score; v != -2 {
		t.Errorf("error analyzing score of sentence: punctuation must be ignored (negative). got: %v", v)
	}
	if v := Analyze("#fearless").Score; v != 2 {
		t.Errorf("error analyzing score of sentence: hashtags must be ignored (positive). got: %v", v)
	}
	if v := Analyze("#crash").Score; v != -2 {
		t.Errorf("error analyzing score of sentence: hashtags must be ignored (negative). got: %v", v)
	}
	// if v := Analyze("An amazing anti").Comparative; v != 1 {
	// 	t.Errorf("error analyzing comparative score of sentence. got: %v", v)
	// }
}

func BenchmarkAnalyze(b *testing.B) {
	text := `Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent ut ligula semper, posuere nisl nec, sagittis tortor. Quisque tincidunt ante a auctor facilisis. Aenean dui lectus, tincidunt semper nisl non, convallis aliquet tortor. Vivamus hendrerit vehicula tempus. Suspendisse tellus elit, venenatis mollis dapibus nec, faucibus ut est. Proin a dolor et dolor gravida viverra in sed est. Pellentesque vulputate turpis vitae velit finibus mattis.

Integer ultrices ac nisi non sodales. Sed velit felis, efficitur nec dapibus non, malesuada eu sem. Curabitur laoreet libero lacinia varius tristique. Ut dictum quis ex sit amet pulvinar. Sed tincidunt, ante non efficitur ornare, erat ligula porta risus, sit amet porttitor ligula odio vulputate ante. Phasellus porttitor faucibus sagittis. Ut malesuada consectetur venenatis. Pellentesque eu magna at dolor pretium maximus. Pellentesque porta velit odio, sit amet efficitur tellus commodo sed. Nam vitae pulvinar metus. Integer vitae risus ac est mattis rutrum vitae a lacus.

In suscipit, risus at malesuada fringilla, dolor lorem interdum orci, at ullamcorper leo nunc sed purus. Vivamus molestie ultrices velit, et auctor dolor accumsan eu. Phasellus tincidunt tempus cursus. Fusce sit amet mattis justo, tincidunt congue neque. Aenean rutrum dolor vel tristique lacinia. Proin aliquet libero a enim iaculis maximus. Nulla est nisi, tincidunt ut feugiat ac, dictum sed tellus. Vestibulum elementum erat nec sem hendrerit sagittis. Etiam non sapien et libero fringilla elementum vel vitae augue. Aenean volutpat fermentum lorem. Nunc imperdiet ex in erat egestas, accumsan cursus diam iaculis. Donec non vehicula nisl, quis fermentum dolor. Pellentesque ultricies ex sit amet volutpat maximus. Vestibulum nec ligula sit amet sem semper tincidunt ut at magna. Integer nec pretium magna.

Phasellus pulvinar orci et dictum fermentum. Ut vestibulum, turpis eget blandit molestie, magna mauris consequat nulla, vitae varius nisi turpis vel nisi. Integer ac eros enim. Integer eu pharetra arcu. Quisque id luctus dui. Curabitur fringilla, nulla quis hendrerit facilisis, justo dui rutrum libero, at auctor nisi nisl viverra sem. Praesent ornare efficitur tincidunt. Vestibulum venenatis bibendum tortor, vel laoreet lacus sagittis eu. Maecenas accumsan, tortor eu euismod dignissim, dui nisl mollis nisl, ac varius nibh lorem sed velit. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Donec sit amet congue massa. Donec eget massa sed enim imperdiet efficitur. Ut tempus porta nulla, ac sollicitudin dui egestas vitae. Nullam euismod erat aliquet mattis molestie. Phasellus placerat, turpis euismod iaculis iaculis, massa quam euismod quam, eu aliquet ligula lectus quis justo. Suspendisse placerat ligula in egestas ullamcorper.

Fusce at eleifend tellus, ac luctus purus. Praesent eu eros sit amet odio bibendum scelerisque. Vestibulum in dui nec ex vulputate semper a vitae lorem. Donec rutrum ex sit amet mauris iaculis consectetur. Fusce ornare luctus augue ut laoreet. Proin dui diam, laoreet eget laoreet et, ultrices eget leo. Pellentesque ac mauris in elit placerat tincidunt id vitae urna. Donec id mauris in velit volutpat convallis. Mauris pulvinar orci sapien, sed pellentesque arcu elementum ac. Nulla facilisi. Cras aliquet mauris et sapien aliquam convallis. Curabitur eu tristique diam.`
	for n := 0; n < b.N; n++ {
		Analyze(text)
	}
}
<|endoftext|>


<commit_before>// (c) Copyright 2015-2017 JONNALAGADDA Srinivas
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package flow

import (
	"database/sql"
	"testing"

	_ "github.com/go-sql-driver/mysql"
)

var gt *testing.T

// error0 expects only an error value as its argument.
func error0(err error) {
	if err != nil {
		gt.Errorf("%v", err)
	}
}

// error1 expects a value and an error value as its arguments.
func error1(val1 interface{}, err error) interface{} {
	if err != nil {
		gt.Errorf("%v", err)
	}
	return val1
}

// fatal0 expects only an error value as its argument.
func fatal0(err error) {
	if err != nil {
		gt.Fatalf("%v", err)
	}
}

// fatal1 expects a value and an error value as its arguments.
func fatal1(val1 interface{}, err error) interface{} {
	if err != nil {
		gt.Fatalf("%v", err)
	}
	return val1
}

// assertEqual compares the two given values for equality.  In case of
// a difference, it errors with the given message.
func assertEqual(expected, observed interface{}, msg string) {
	if expected == observed {
		return
	}

	gt.Errorf("expected : '%v', observed : '%v'\t%s", expected, observed, msg)
}

// Driver test function.
func TestFlow01(t *testing.T) {
	gt = t

	// Connect to the database.
	driver, connStr := "mysql", "travis@/flow"
	tdb := fatal1(sql.Open(driver, connStr))
	db := tdb.(*sql.DB)
	defer db.Close()
	RegisterDB(db)

	// Test-local state.
	var dtID1, dtID2 DocTypeID
	var dsID1, dsID2, dsID3, dsID4, dsID5 DocStateID
	var daID1, daID2, daID3, daID4, daID5, daID6, daID7, daID8, daID9 DocActionID

	var roleID1, roleID2 RoleID
	var uID1, uID2, uID3, uID4 UserID
	var gID1, gID2, gID3, gID4, gID5, gID6 GroupID

	// Create operations.
	t.Run("Create", func(t *testing.T) {
		t.Run("DocTypes", func(t *testing.T) {
			ttx := fatal1(db.Begin())
			tx := ttx.(*sql.Tx)
			defer tx.Rollback()

			tdt := fatal1(DocTypes.New(tx, "STOR_REQ"))
			dtID1 = tdt.(DocTypeID)
			tdt = fatal1(DocTypes.New(tx, "COMPUTE_REQ"))
			dtID2 = tdt.(DocTypeID)

			fatal0(tx.Commit())
		})

		t.Run("DocStates", func(t *testing.T) {
			ttx := fatal1(db.Begin())
			tx := ttx.(*sql.Tx)
			defer tx.Rollback()

			tds := fatal1(DocStates.New(tx, dtID1, "INITIAL"))
			dsID1 = tds.(DocStateID)
			tds = fatal1(DocStates.New(tx, dtID1, "PENDING_APPROVAL"))
			dsID2 = tds.(DocStateID)
			tds = fatal1(DocStates.New(tx, dtID1, "APPROVED"))
			dsID3 = tds.(DocStateID)
			tds = fatal1(DocStates.New(tx, dtID1, "REJECTED"))
			dsID4 = tds.(DocStateID)
			tds = fatal1(DocStates.New(tx, dtID1, "DISCARDED"))
			dsID5 = tds.(DocStateID)

			fatal0(tx.Commit())
		})

		t.Run("DocActions", func(t *testing.T) {
			ttx := fatal1(db.Begin())
			tx := ttx.(*sql.Tx)
			defer tx.Rollback()

			tda := fatal1(DocActions.New(tx, "INITIALISE"))
			daID1 = tda.(DocActionID)
			tda = fatal1(DocActions.New(tx, "NEW"))
			daID2 = tda.(DocActionID)
			tda = fatal1(DocActions.New(tx, "GET"))
			daID3 = tda.(DocActionID)
			tda = fatal1(DocActions.New(tx, "UPDATE"))
			daID4 = tda.(DocActionID)
			tda = fatal1(DocActions.New(tx, "DELETE"))
			daID5 = tda.(DocActionID)
			tda = fatal1(DocActions.New(tx, "APPROVE"))
			daID6 = tda.(DocActionID)
			tda = fatal1(DocActions.New(tx, "REJECT"))
			daID7 = tda.(DocActionID)
			tda = fatal1(DocActions.New(tx, "RETURN"))
			daID8 = tda.(DocActionID)
			tda = fatal1(DocActions.New(tx, "DISCARD"))
			daID9 = tda.(DocActionID)

			fatal0(tx.Commit())
		})

		t.Run("Roles", func(t *testing.T) {
			ttx := fatal1(db.Begin())
			tx := ttx.(*sql.Tx)
			defer tx.Rollback()

			tr := fatal1(Roles.New(tx, "ADMIN"))
			roleID1 = tr.(RoleID)
			tr = fatal1(Roles.New(tx, "RESEARCH_ANALYST"))
			roleID2 = tr.(RoleID)

			fatal0(tx.Commit())
		})

		t.Run("Users", func(t *testing.T) {
			ttx := fatal1(db.Begin())
			tx := ttx.(*sql.Tx)
			defer tx.Rollback()

			res, err := tx.Exec(`INSERT INTO users_master(first_name, last_name, email, active)
			VALUES('FN 1', 'LN 1', 'email1@example.com', 1)`)
			if err != nil {
				t.Fatalf("%v\n", err)
			}
			uid, _ := res.LastInsertId()
			uID1 = UserID(uid)
			tg := fatal1(Groups.NewSingleton(tx, uID1))
			gID1 = tg.(GroupID)

			res, err = tx.Exec(`INSERT INTO users_master(first_name, last_name, email, active)
			VALUES('FN 2', 'LN 2', 'email2@example.com', 1)`)
			if err != nil {
				t.Fatalf("%v\n", err)
			}
			uid, _ = res.LastInsertId()
			uID2 = UserID(uid)
			tg = fatal1(Groups.NewSingleton(tx, uID2))
			gID2 = tg.(GroupID)

			res, err = tx.Exec(`INSERT INTO users_master(first_name, last_name, email, active)
			VALUES('FN 3', 'LN 3', 'email3@example.com', 1)`)
			if err != nil {
				t.Errorf("%v\n", err)
			}
			uid, _ = res.LastInsertId()
			uID3 = UserID(uid)
			tg = fatal1(Groups.NewSingleton(tx, uID3))
			gID3 = tg.(GroupID)

			res, err = tx.Exec(`INSERT INTO users_master(first_name, last_name, email, active)
			VALUES('FN 4', 'LN 4', 'email4@example.com', 1)`)
			if err != nil {
				t.Fatalf("%v\n", err)
			}
			uid, _ = res.LastInsertId()
			uID4 = UserID(uid)
			tg = fatal1(Groups.NewSingleton(tx, uID4))
			gID4 = tg.(GroupID)

			fatal0(tx.Commit())
		})

		t.Run("Groups", func(t *testing.T) {
			ttx := fatal1(db.Begin())
			tx := ttx.(*sql.Tx)
			defer tx.Rollback()

			td := fatal1(Groups.New(tx, "RAs", "G"))
			gID5 = td.(GroupID)

			td = fatal1(Groups.New(tx, "PIs", "G"))
			gID6 = td.(GroupID)

			fatal0(tx.Commit())
		})

		t.Run("GroupsAddUsers", func(t *testing.T) {
			ttx := fatal1(db.Begin())
			tx := ttx.(*sql.Tx)
			defer tx.Rollback()

			fatal0(Groups.AddUser(tx, gID5, uID1))
			fatal0(Groups.AddUser(tx, gID5, uID2))
			fatal0(Groups.AddUser(tx, gID5, uID3))

			fatal0(Groups.AddUser(tx, gID6, uID2))
			fatal0(Groups.AddUser(tx, gID6, uID3))
			fatal0(Groups.AddUser(tx, gID6, uID4))

			fatal0(tx.Commit())
		})
	})

	// Entity update operations.
	t.Run("Update", func(t *testing.T) {
		t.Run("DocTypeRename", func(t *testing.T) {
			ttx := fatal1(db.Begin())
			tx := ttx.(*sql.Tx)
			defer tx.Rollback()

			error0(DocTypes.Rename(tx, dtID1, "STORAGE_REQ"))

			fatal0(tx.Commit())

			tobj := error1(DocTypes.Get(dtID1))
			obj := tobj.(*DocType)
			assertEqual("STORAGE_REQ", obj.Name, "")
		})

		t.Run("DocStateRename", func(t *testing.T) {
			ttx := fatal1(db.Begin())
			tx := ttx.(*sql.Tx)
			defer tx.Rollback()

			error0(DocStates.Rename(tx, dsID1, "DRAFT"))

			fatal0(tx.Commit())

			tobj := error1(DocStates.Get(dsID1))
			obj := tobj.(*DocState)
			assertEqual("DRAFT", obj.Name, "")
		})

		t.Run("DocActionRename", func(t *testing.T) {
			ttx := fatal1(db.Begin())
			tx := ttx.(*sql.Tx)
			defer tx.Rollback()

			error0(DocActions.Rename(tx, daID1, "LIST"))

			fatal0(tx.Commit())

			tobj := error1(DocActions.Get(daID1))
			obj := tobj.(*DocAction)
			assertEqual("LIST", obj.Name, "")
		})

		t.Run("GroupRename", func(t *testing.T) {
			ttx := fatal1(db.Begin())
			tx := ttx.(*sql.Tx)
			defer tx.Rollback()

			error0(Groups.Rename(tx, gID5, "Research Associates"))

			fatal0(tx.Commit())

			tobj := error1(Groups.Get(gID5))
			obj := tobj.(*Group)
			assertEqual("Research Associates", obj.Name, "")
		})
	})

	// Entity deletion operations.
	t.Run("Delete", func(t *testing.T) {
		t.Run("GroupsDeleteUsers", func(t *testing.T) {
			ttx := fatal1(db.Begin())
			tx := ttx.(*sql.Tx)
			defer tx.Rollback()

			error0(Groups.RemoveUser(tx, gID6, uID2))

			fatal0(tx.Commit())
		})
	})

	// Tear down.
	t.Run("TearDown", func(t *testing.T) {
		ttx := fatal1(db.Begin())
		tx := ttx.(*sql.Tx)
		defer tx.Rollback()

		error1(tx.Exec(`DELETE FROM wf_ac_group_roles`))
		error1(tx.Exec(`DELETE FROM wf_ac_group_hierarchy`))
		error1(tx.Exec(`DELETE FROM wf_access_contexts`))

		error1(tx.Exec(`DELETE FROM wf_group_users`))
		error1(tx.Exec(`DELETE FROM wf_groups_master`))
		error1(tx.Exec(`DELETE FROM users_master`))
		error1(tx.Exec(`DELETE FROM wf_roles_master`))

		error1(tx.Exec(`DELETE FROM wf_docactions_master`))
		error1(tx.Exec(`DELETE FROM wf_docstates_master`))
		error1(tx.Exec(`DELETE FROM wf_doctypes_master`))

		fatal0(tx.Commit())
	})
}
<commit_msg>Fold result type assertions into function call lines in `flow_test.go`<commit_after>// (c) Copyright 2015-2017 JONNALAGADDA Srinivas
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package flow

import (
	"database/sql"
	"testing"

	_ "github.com/go-sql-driver/mysql"
)

var gt *testing.T

// error0 expects only an error value as its argument.
func error0(err error) {
	if err != nil {
		gt.Errorf("%v", err)
	}
}

// error1 expects a value and an error value as its arguments.
func error1(val1 interface{}, err error) interface{} {
	if err != nil {
		gt.Errorf("%v", err)
	}
	return val1
}

// fatal0 expects only an error value as its argument.
func fatal0(err error) {
	if err != nil {
		gt.Fatalf("%v", err)
	}
}

// fatal1 expects a value and an error value as its arguments.
func fatal1(val1 interface{}, err error) interface{} {
	if err != nil {
		gt.Fatalf("%v", err)
	}
	return val1
}

// assertEqual compares the two given values for equality.  In case of
// a difference, it errors with the given message.
func assertEqual(expected, observed interface{}, msg string) {
	if expected == observed {
		return
	}

	gt.Errorf("expected : '%v', observed : '%v'\t%s", expected, observed, msg)
}

// Driver test function.
func TestFlow01(t *testing.T) {
	gt = t

	// Connect to the database.
	driver, connStr := "mysql", "travis@/flow"
	db := fatal1(sql.Open(driver, connStr)).(*sql.DB)
	defer db.Close()
	RegisterDB(db)

	// Test-local state.
	var dtID1, dtID2 DocTypeID
	var dsID1, dsID2, dsID3, dsID4, dsID5 DocStateID
	var daID1, daID2, daID3, daID4, daID5, daID6, daID7, daID8, daID9 DocActionID

	var roleID1, roleID2 RoleID
	var uID1, uID2, uID3, uID4 UserID
	var gID1, gID2, gID3, gID4, gID5, gID6 GroupID

	// Create operations.
	t.Run("Create", func(t *testing.T) {
		t.Run("DocTypes", func(t *testing.T) {
			tx := fatal1(db.Begin()).(*sql.Tx)
			defer tx.Rollback()

			dtID1 = fatal1(DocTypes.New(tx, "STOR_REQ")).(DocTypeID)
			dtID2 = fatal1(DocTypes.New(tx, "COMPUTE_REQ")).(DocTypeID)

			fatal0(tx.Commit())
		})

		t.Run("DocStates", func(t *testing.T) {
			tx := fatal1(db.Begin()).(*sql.Tx)
			defer tx.Rollback()

			dsID1 = fatal1(DocStates.New(tx, dtID1, "INITIAL")).(DocStateID)
			dsID2 = fatal1(DocStates.New(tx, dtID1, "PENDING_APPROVAL")).(DocStateID)
			dsID3 = fatal1(DocStates.New(tx, dtID1, "APPROVED")).(DocStateID)
			dsID4 = fatal1(DocStates.New(tx, dtID1, "REJECTED")).(DocStateID)
			dsID5 = fatal1(DocStates.New(tx, dtID1, "DISCARDED")).(DocStateID)

			fatal0(tx.Commit())
		})

		t.Run("DocActions", func(t *testing.T) {
			tx := fatal1(db.Begin()).(*sql.Tx)
			defer tx.Rollback()

			daID1 = fatal1(DocActions.New(tx, "INITIALISE")).(DocActionID)
			daID2 = fatal1(DocActions.New(tx, "NEW")).(DocActionID)
			daID3 = fatal1(DocActions.New(tx, "GET")).(DocActionID)
			daID4 = fatal1(DocActions.New(tx, "UPDATE")).(DocActionID)
			daID5 = fatal1(DocActions.New(tx, "DELETE")).(DocActionID)
			daID6 = fatal1(DocActions.New(tx, "APPROVE")).(DocActionID)
			daID7 = fatal1(DocActions.New(tx, "REJECT")).(DocActionID)
			daID8 = fatal1(DocActions.New(tx, "RETURN")).(DocActionID)
			daID9 = fatal1(DocActions.New(tx, "DISCARD")).(DocActionID)

			fatal0(tx.Commit())
		})

		t.Run("Roles", func(t *testing.T) {
			tx := fatal1(db.Begin()).(*sql.Tx)
			defer tx.Rollback()

			roleID1 = fatal1(Roles.New(tx, "ADMIN")).(RoleID)
			roleID2 = fatal1(Roles.New(tx, "RESEARCH_ANALYST")).(RoleID)

			fatal0(tx.Commit())
		})

		t.Run("Users", func(t *testing.T) {
			tx := fatal1(db.Begin()).(*sql.Tx)
			defer tx.Rollback()

			res, err := tx.Exec(`INSERT INTO users_master(first_name, last_name, email, active)
			VALUES('FN 1', 'LN 1', 'email1@example.com', 1)`)
			if err != nil {
				t.Fatalf("%v\n", err)
			}
			uid, _ := res.LastInsertId()
			uID1 = UserID(uid)
			gID1 = fatal1(Groups.NewSingleton(tx, uID1)).(GroupID)

			res, err = tx.Exec(`INSERT INTO users_master(first_name, last_name, email, active)
			VALUES('FN 2', 'LN 2', 'email2@example.com', 1)`)
			if err != nil {
				t.Fatalf("%v\n", err)
			}
			uid, _ = res.LastInsertId()
			uID2 = UserID(uid)
			gID2 = fatal1(Groups.NewSingleton(tx, uID2)).(GroupID)

			res, err = tx.Exec(`INSERT INTO users_master(first_name, last_name, email, active)
			VALUES('FN 3', 'LN 3', 'email3@example.com', 1)`)
			if err != nil {
				t.Errorf("%v\n", err)
			}
			uid, _ = res.LastInsertId()
			uID3 = UserID(uid)
			gID3 = fatal1(Groups.NewSingleton(tx, uID3)).(GroupID)

			res, err = tx.Exec(`INSERT INTO users_master(first_name, last_name, email, active)
			VALUES('FN 4', 'LN 4', 'email4@example.com', 1)`)
			if err != nil {
				t.Fatalf("%v\n", err)
			}
			uid, _ = res.LastInsertId()
			uID4 = UserID(uid)
			gID4 = fatal1(Groups.NewSingleton(tx, uID4)).(GroupID)

			fatal0(tx.Commit())
		})

		t.Run("Groups", func(t *testing.T) {
			tx := fatal1(db.Begin()).(*sql.Tx)
			defer tx.Rollback()

			gID5 = fatal1(Groups.New(tx, "RAs", "G")).(GroupID)
			gID6 = fatal1(Groups.New(tx, "PIs", "G")).(GroupID)

			fatal0(tx.Commit())
		})

		t.Run("GroupsAddUsers", func(t *testing.T) {
			tx := fatal1(db.Begin()).(*sql.Tx)
			defer tx.Rollback()

			fatal0(Groups.AddUser(tx, gID5, uID1))
			fatal0(Groups.AddUser(tx, gID5, uID2))
			fatal0(Groups.AddUser(tx, gID5, uID3))

			fatal0(Groups.AddUser(tx, gID6, uID2))
			fatal0(Groups.AddUser(tx, gID6, uID3))
			fatal0(Groups.AddUser(tx, gID6, uID4))

			fatal0(tx.Commit())
		})
	})

	// Entity update operations.
	t.Run("Update", func(t *testing.T) {
		t.Run("DocTypeRename", func(t *testing.T) {
			tx := fatal1(db.Begin()).(*sql.Tx)
			defer tx.Rollback()

			error0(DocTypes.Rename(tx, dtID1, "STORAGE_REQ"))

			fatal0(tx.Commit())

			obj := error1(DocTypes.Get(dtID1)).(*DocType)
			assertEqual("STORAGE_REQ", obj.Name, "")
		})

		t.Run("DocStateRename", func(t *testing.T) {
			tx := fatal1(db.Begin()).(*sql.Tx)
			defer tx.Rollback()

			error0(DocStates.Rename(tx, dsID1, "DRAFT"))

			fatal0(tx.Commit())

			obj := error1(DocStates.Get(dsID1)).(*DocState)
			assertEqual("DRAFT", obj.Name, "")
		})

		t.Run("DocActionRename", func(t *testing.T) {
			tx := fatal1(db.Begin()).(*sql.Tx)
			defer tx.Rollback()

			error0(DocActions.Rename(tx, daID1, "LIST"))

			fatal0(tx.Commit())

			obj := error1(DocActions.Get(daID1)).(*DocAction)
			assertEqual("LIST", obj.Name, "")
		})

		t.Run("GroupRename", func(t *testing.T) {
			tx := fatal1(db.Begin()).(*sql.Tx)
			defer tx.Rollback()

			error0(Groups.Rename(tx, gID5, "Research Associates"))

			fatal0(tx.Commit())

			obj := error1(Groups.Get(gID5)).(*Group)
			assertEqual("Research Associates", obj.Name, "")
		})
	})

	// Entity deletion operations.
	t.Run("Delete", func(t *testing.T) {
		t.Run("GroupsDeleteUsers", func(t *testing.T) {
			tx := fatal1(db.Begin()).(*sql.Tx)
			defer tx.Rollback()

			error0(Groups.RemoveUser(tx, gID6, uID2))

			fatal0(tx.Commit())
		})
	})

	// Tear down.
	t.Run("TearDown", func(t *testing.T) {
		tx := fatal1(db.Begin()).(*sql.Tx)
		defer tx.Rollback()

		error1(tx.Exec(`DELETE FROM wf_ac_group_roles`))
		error1(tx.Exec(`DELETE FROM wf_ac_group_hierarchy`))
		error1(tx.Exec(`DELETE FROM wf_access_contexts`))

		error1(tx.Exec(`DELETE FROM wf_group_users`))
		error1(tx.Exec(`DELETE FROM wf_groups_master`))
		error1(tx.Exec(`DELETE FROM users_master`))
		error1(tx.Exec(`DELETE FROM wf_roles_master`))

		error1(tx.Exec(`DELETE FROM wf_docactions_master`))
		error1(tx.Exec(`DELETE FROM wf_docstates_master`))
		error1(tx.Exec(`DELETE FROM wf_doctypes_master`))

		fatal0(tx.Commit())
	})
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.71"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.38", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	g, err := lb.NewAllGrouper(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	h := lb.NewProxy(k, g, r, conf)
	h = g.Wrap(h) // add/del/list endpoints
	h = r.Wrap(h) // stats / dash endpoint

	err = serve(conf.Listen, h)
	if err != nil {
		logrus.WithError(err).Fatal("server error")
	}
}

func serve(addr string, handler http.Handler) error {
	server := &http.Server{Addr: addr, Handler: handler}

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)
	go func() {
		for sig := range ch {
			logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")
			server.Shutdown(context.Background()) // safe shutdown
			return
		}
	}()
	return server.ListenAndServe()
}
<commit_msg>fnlb: 0.0.72 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.72"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.39", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	g, err := lb.NewAllGrouper(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	h := lb.NewProxy(k, g, r, conf)
	h = g.Wrap(h) // add/del/list endpoints
	h = r.Wrap(h) // stats / dash endpoint

	err = serve(conf.Listen, h)
	if err != nil {
		logrus.WithError(err).Fatal("server error")
	}
}

func serve(addr string, handler http.Handler) error {
	server := &http.Server{Addr: addr, Handler: handler}

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)
	go func() {
		for sig := range ch {
			logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")
			server.Shutdown(context.Background()) // safe shutdown
			return
		}
	}()
	return server.ListenAndServe()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.137"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.104", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	g, err := lb.NewAllGrouper(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	h := lb.NewProxy(k, g, r, conf)
	h = g.Wrap(h) // add/del/list endpoints
	h = r.Wrap(h) // stats / dash endpoint

	err = serve(conf.Listen, h)
	if err != nil {
		logrus.WithError(err).Fatal("server error")
	}
}

func serve(addr string, handler http.Handler) error {
	server := &http.Server{Addr: addr, Handler: handler}

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)
	go func() {
		for sig := range ch {
			logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")
			server.Shutdown(context.Background()) // safe shutdown
			return
		}
	}()
	return server.ListenAndServe()
}
<commit_msg>fnlb: 0.0.138 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.138"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.105", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	g, err := lb.NewAllGrouper(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	h := lb.NewProxy(k, g, r, conf)
	h = g.Wrap(h) // add/del/list endpoints
	h = r.Wrap(h) // stats / dash endpoint

	err = serve(conf.Listen, h)
	if err != nil {
		logrus.WithError(err).Fatal("server error")
	}
}

func serve(addr string, handler http.Handler) error {
	server := &http.Server{Addr: addr, Handler: handler}

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)
	go func() {
		for sig := range ch {
			logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")
			server.Shutdown(context.Background()) // safe shutdown
			return
		}
	}()
	return server.ListenAndServe()
}
<|endoftext|>


<commit_before>package kvstore

import (
	"fmt"
	"strconv"

	"github.com/garyburd/redigo/redis"
)

// Ints is a helper that converts a multi-bulk command reply to a []int.
// If err is not equal to nil, then Ints returns nil, err.  If one if the
// multi-bulk items is not a bulk value or nil, then Ints returns an error.
func Ints(reply interface{}, err error) ([]int, error) {
	if err != nil {
		return nil, err
	}
	switch reply := reply.(type) {
	case []interface{}:
		result := make([]int, len(reply))
		for i := range reply {
			if reply[i] == nil {
				continue
			}
			p, ok := reply[i].([]byte)
			if !ok {
				return nil, fmt.Errorf("kvstore: unexpected element type for Ints, got type %T", reply[i])
			}

			n, err := strconv.ParseInt(string(p), 10, 0)

			if err != nil {
				return nil, err
			}

			result[i] = int(n)
		}
		return result, nil
	case nil:
		return nil, redis.ErrNil
	case redis.Error:
		return nil, reply
	}
	return nil, fmt.Errorf("kvstore: unexpected type for Ints, got type %T", reply)
}
<commit_msg>kvstore: add bytes helper, ignore nils<commit_after>package kvstore

import (
	"fmt"
	"strconv"

	"github.com/garyburd/redigo/redis"
)

// Ints is a helper that converts a multi-bulk command reply to a []int.
// If err is not equal to nil, then Ints returns nil, err.  If one if the
// multi-bulk items is not a bulk value or nil, then Ints returns an error.
func Ints(reply interface{}, err error) ([]int, error) {
	if err != nil {
		return nil, err
	}
	switch reply := reply.(type) {
	case []interface{}:
		result := make([]int, 0, len(reply))
		n := 0
		for _, v := range reply {
			if v == nil {
				continue
			}
			p, ok := v.([]byte)
			if !ok {
				return nil, fmt.Errorf("kvstore: unexpected element type for Ints, got type %T", v)
			}

			nr, err := strconv.ParseInt(string(p), 10, 0)

			if err != nil {
				return nil, err
			}

			n++
			result = append(result, int(nr))
		}
		return result[:n], nil
	case nil:
		return nil, redis.ErrNil
	case redis.Error:
		return nil, reply
	}
	return nil, fmt.Errorf("kvstore: unexpected type for Ints, got type %T", reply)
}

// Bytes is a helper that converts a multi-bulk command reply to a [][]byte.
// If err is not equal to nil, then Ints returns nil, err.  If one if the
// multi-bulk items is not a bulk value or nil, then Ints returns an error.
func Bytes(reply interface{}, err error) ([][]byte, error) {
	if err != nil {
		return nil, err
	}
	switch reply := reply.(type) {
	case []interface{}:
		result := make([][]byte, 0, len(reply))
		n := 0
		for _, v := range reply {
			var b []byte
			switch raw := v.(type) {
			case []byte:
				b = raw
			case string:
				b = []byte(raw)
			case nil:
				continue
			default:
				return nil, fmt.Errorf("kvstore: unexpected type for Bytes, got type %T", v)
			}

			n++
			result = append(result, b)
		}
		return result[:n], nil
	case nil:
		return nil, redis.ErrNil
	case redis.Error:
		return nil, reply
	}
	return nil, fmt.Errorf("kvstore: unexpected type for Bytes, got type %T", reply)
}
<|endoftext|>


<commit_before>package followingfeed

import (
	"socialapi/models"

	"github.com/koding/logging"
	"github.com/streadway/amqp"
)

type Action func(*Controller, *models.ChannelMessage) error

type Controller struct {
	log logging.Logger
}

func (f *Controller) DefaultErrHandler(delivery amqp.Delivery, err error) bool {
	if delivery.Redelivered {
		f.log.Error("Redelivered message gave error again, putting to maintenance queue", err)
		delivery.Ack(false)
		return true
	}

	f.log.Error("an error occured putting message back to queue", err)
	delivery.Nack(false, true)
	return false
}

func New(log logging.Logger) *Controller {
	return &Controller{
		log: log,
	}
}

func (f *Controller) MessageSaved(data *models.ChannelMessage) error {
	if res, _ := isEligible(data); !res {
		return nil
	}

	a := models.NewAccount()
	a.Id = data.AccountId
	channelIds, err := a.FetchFollowerChannelIds()
	if err != nil {
		return err
	}

	return nil
}

func (f *Controller) MessageUpdated(data *models.ChannelMessage) error {
	if res, _ := isEligible(data); !res {
		return nil
	}

	return nil

}

func (f *Controller) MessageDeleted(data *models.ChannelMessage) error {
	if res, _ := isEligible(data); !res {
		return nil
	}

	return nil
}

func isEligible(cm *models.ChannelMessage) (bool, error) {
	if cm.InitialChannelId == 0 {
		return false, nil
	}

	if cm.TypeConstant != models.ChannelMessage_TYPE_POST {
		return false, nil
	}

	return isChannelEligible(cm)
}

func isChannelEligible(cm *models.ChannelMessage) (bool, error) {
	c, err := fetchChannel(cm.InitialChannelId)
	if err != nil {
		return false, err
	}

	if c.Name != models.Channel_KODING_NAME {
		return false, nil
	}

	return true, nil
}

// todo add caching here
func fetchChannel(channelId int64) (*models.Channel, error) {
	c := models.NewChannel()
	// todo - fetch only name here
	if err := c.ById(channelId); err != nil {
		return nil, err
	}

	return c, nil
}
<commit_msg>social: remove unused variable<commit_after>package followingfeed

import (
	"socialapi/models"

	"github.com/koding/logging"
	"github.com/streadway/amqp"
)

type Action func(*Controller, *models.ChannelMessage) error

type Controller struct {
	log logging.Logger
}

func (f *Controller) DefaultErrHandler(delivery amqp.Delivery, err error) bool {
	if delivery.Redelivered {
		f.log.Error("Redelivered message gave error again, putting to maintenance queue", err)
		delivery.Ack(false)
		return true
	}

	f.log.Error("an error occured putting message back to queue", err)
	delivery.Nack(false, true)
	return false
}

func New(log logging.Logger) *Controller {
	return &Controller{
		log: log,
	}
}

func (f *Controller) MessageSaved(data *models.ChannelMessage) error {
	if res, _ := isEligible(data); !res {
		return nil
	}

	a := models.NewAccount()
	a.Id = data.AccountId
	_, err := a.FetchFollowerChannelIds()
	if err != nil {
		return err
	}

	return nil
}

func (f *Controller) MessageUpdated(data *models.ChannelMessage) error {
	if res, _ := isEligible(data); !res {
		return nil
	}

	return nil

}

func (f *Controller) MessageDeleted(data *models.ChannelMessage) error {
	if res, _ := isEligible(data); !res {
		return nil
	}

	return nil
}

func isEligible(cm *models.ChannelMessage) (bool, error) {
	if cm.InitialChannelId == 0 {
		return false, nil
	}

	if cm.TypeConstant != models.ChannelMessage_TYPE_POST {
		return false, nil
	}

	return isChannelEligible(cm)
}

func isChannelEligible(cm *models.ChannelMessage) (bool, error) {
	c, err := fetchChannel(cm.InitialChannelId)
	if err != nil {
		return false, err
	}

	if c.Name != models.Channel_KODING_NAME {
		return false, nil
	}

	return true, nil
}

// todo add caching here
func fetchChannel(channelId int64) (*models.Channel, error) {
	c := models.NewChannel()
	// todo - fetch only name here
	if err := c.ById(channelId); err != nil {
		return nil, err
	}

	return c, nil
}
<|endoftext|>


<commit_before>// A command line tool for fragmenta which can be used to build and run websites
// this tool calls subcommands for most of the work, usually one command per file in this pkg
// See docs at http://godoc.org/github.com/fragmenta/fragmenta

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
)

const (
	// The version of this tool
	fragmentaVersion = "1.5.7"

	// Used for outputting console messages
	fragmentaDivider = "\n------\n"
)

// Modes used for setting the config used
const (
	ModeProduction  = "production"
	ModeDevelopment = "development"
	ModeTest        = "test"
)

var (
	// ConfigDevelopment holds the development config from fragmenta.json
	ConfigDevelopment map[string]string

	// ConfigProduction holds development config from fragmenta.json
	ConfigProduction map[string]string

	// ConfigTest holds the app test config from fragmenta.json
	ConfigTest map[string]string
)

// main - parse the command line arguments and respond
func main() {

	// Log time as well as date
	log.SetFlags(log.Ltime)

	// Parse commands
	args := os.Args
	command := ""

	if len(args) > 1 {
		command = args[1]
	}

	// We assume the project path is the current directory (for now)
	projectPath, err := filepath.Abs(".")
	if err != nil {
		log.Printf("Error getting path %s", err)
		return
	}

	// If this is a valid project, read the config, else continue
	if isValidProject(projectPath) {
		readConfig(projectPath)
	}

	switch command {

	case "new", "n":
		RunNew(args)

	case "version", "v":
		ShowVersion()

	case "help", "h", "wat", "?":
		ShowHelp(args)

	case "server", "s":
		if requireValidProject(projectPath) {
			RunServer(projectPath)
		}

	case "test", "t":
		if requireValidProject(projectPath) {
			// Remove fragmenta test from args list
			args = args[2:]
			RunTests(args)
		}

	case "build", "B":
		if requireValidProject(projectPath) {
			RunBuild(args)
		}

	case "generate", "g":
		if requireValidProject(projectPath) {
			RunGenerate(args)
		}

	case "migrate", "m":
		if requireValidProject(projectPath) {
			RunMigrate(args)
		}

	case "backup", "b":
		if requireValidProject(projectPath) {
			RunBackup(args)
		}

	case "restore", "r":
		if requireValidProject(projectPath) {
			RunRestore(args)
		}

	case "deploy", "d":
		if requireValidProject(projectPath) {
			RunDeploy(args)
		}
	case "":
		// Special case no commands to build and run the server
		if requireValidProject(projectPath) {
			RunServer(projectPath)
		}
	default:
		// Command not recognised so show the help
		ShowHelp(args)
	}

}

// ShowVersion shows the version of this tool
func ShowVersion() {
	helpString := fragmentaDivider
	helpString += fmt.Sprintf("Fragmenta version: %s", fragmentaVersion)
	helpString += fragmentaDivider
	log.Print(helpString)
}

// ShowHelp shows the help for this tool
func ShowHelp(args []string) {
	helpString := fragmentaDivider
	helpString += fmt.Sprintf("Fragmenta version: %s", fragmentaVersion)
	helpString += "\n  fragmenta version -> display version"
	helpString += "\n  fragmenta help -> display help"
	helpString += "\n  fragmenta new [app|cms|URL] path/to/app -> creates a new app from the repository at URL at the path supplied"
	helpString += "\n  fragmenta -> builds and runs a fragmenta app"
	helpString += "\n  fragmenta server -> builds and runs a fragmenta app"
	helpString += "\n  fragmenta test  -> run tests"
	helpString += "\n  fragmenta migrate -> runs new sql migrations in db/migrate"
	helpString += "\n  fragmenta backup [development|production|test] -> backup the database to db/backup"
	helpString += "\n  fragmenta restore [development|production|test] -> backup the database from latest file in db/backup"
	helpString += "\n  fragmenta deploy [development|production|test] -> build and deploy using bin/deploy"
	helpString += "\n  fragmenta generate resource [name] [fieldname]:[fieldtype]* -> creates resource CRUD actions and views"
	helpString += "\n  fragmenta generate migration [name] -> creates a new named sql migration in db/migrate"

	helpString += fragmentaDivider
	log.Print(helpString)
}

// Ideally all these paths could be configured,
// rather than baking assumptions about project structure into the tool

// serverName returns the path of the cross-compiled target server binary
// this does not end in .exe as we assume a target of linux
func serverName() string {
	return "fragmenta-server"
}

// localServerName returns a server name for the local server binary (prefixed with local)
func localServerName() string {
	if isWindows() {
		return serverName() + "-local.exe"
	}
	return serverName() + "-local"
}

// localServerPath returns the local server binary for running on the dev machine locally
func localServerPath(projectPath string) string {
	return filepath.Join(projectPath, "bin", localServerName())
}

// serverPath returns the cross-compiled server binary
func serverPath(projectPath string) string {
	return filepath.Join(projectPath, "bin", serverName())
}

// serverCompilePath returns the server entrypoint
func serverCompilePath(projectPath string) string {
	return filepath.Join(projectPath, "server.go")
}

// srcPath returns the path for Go code within the project
func srcPath(projectPath string) string {
	return filepath.Join(projectPath, "src")
}

// publicPath returns the path for the public directory of the web application
func publicPath(projectPath string) string {
	return filepath.Join(projectPath, "public")
}

// configPath returns the path for the fragment config file (required)
func configPath(projectPath string) string {
	return filepath.Join(secretsPath(projectPath), "fragmenta.json")
}

// secretsPath returns the path for secrets
func secretsPath(projectPath string) string {
	return filepath.Join(projectPath, "secrets")
}

// templatesPath returns the path for templates
func templatesPath() string {
	path := filepath.Join(goPath(), "src", "github.com", "fragmenta", "fragmenta", "templates")
	return os.ExpandEnv(path)
}

// dbMigratePath returns a path to store database migrations
func dbMigratePath(projectPath string) string {
	return filepath.Join(projectPath, "db", "migrate")
}

// dbBackupPath returns a path to store database backups
func dbBackupPath(projectPath string) string {
	return filepath.Join(projectPath, "db", "backup")
}

// projectPathRelative returns the relative path
func projectPathRelative(projectPath string) string {
	goSrc := filepath.Join(goPath(), "src")
	return strings.Replace(projectPath, goSrc, "", 1)
}

// goPath returns the setting of env variable $GOPATH
// or $HOME/go if no $GOPATH is set.
func goPath() string {
	p := os.ExpandEnv("$GOPATH")
	if len(p) > 0 {
		return p
	}

	return filepath.Join(os.ExpandEnv("$HOME"), "go")
}

// RunServer runs the server
func RunServer(projectPath string) {
	ShowVersion()

	log.Println("Building server...")
	err := buildServer(localServerPath(projectPath), nil)

	if err != nil {
		log.Printf("Error building server: %s", err)
		return
	}

	log.Println("Launching server...")
	cmd := exec.Command(localServerPath(projectPath))
	stdout, _ := cmd.StdoutPipe()
	stderr, _ := cmd.StderrPipe()
	err = cmd.Start()
	if err != nil {
		log.Println(err)
	}
	go io.Copy(os.Stdout, stdout)
	go io.Copy(os.Stderr, stderr)
	cmd.Wait()

}

// runCommand runs a command with exec.Command
func runCommand(command string, args ...string) ([]byte, error) {

	cmd := exec.Command(command, args...)
	output, err := cmd.CombinedOutput()
	if err != nil {
		return output, err
	}

	return output, nil
}

// requireValidProject returns true if we have a valid project at projectPath
func requireValidProject(projectPath string) bool {
	if isValidProject(projectPath) {
		return true
	}

	log.Printf("No fragmenta project found at this path\n")
	return false
}

// isValidProject returns true if this is a valid fragmenta project (checks for server.go file and config file)
func isValidProject(projectPath string) bool {

	// Make sure we have server.go at root of this dir
	_, err := os.Stat(serverCompilePath(projectPath))
	if err != nil {
		return false
	}

	return true
}

// fileExists returns true if this file exists
func fileExists(p string) bool {
	_, err := os.Stat(p)
	if err != nil && os.IsNotExist(err) {
		return false
	}

	return true
}

// readConfig reads our config file and set up the server accordingly
func readConfig(projectPath string) error {
	configPath := configPath(projectPath)

	// Read the config json file
	file, err := ioutil.ReadFile(configPath)
	if err != nil {
		log.Printf("Error opening config at %s\n%s", configPath, err)
		return err
	}

	var data map[string]map[string]string
	err = json.Unmarshal(file, &data)
	if err != nil {
		log.Printf("Error parsing config %s %v", configPath, err)
		return err
	}

	ConfigDevelopment = data["development"]
	ConfigProduction = data["production"]
	ConfigTest = data["test"]

	return nil
}

// isWindows returns true if the Go architecture target (GOOS) is windows
func isWindows() bool {
	if runtime.GOOS == "windows" {
		return true
	}
	return false
}
<commit_msg>Fix ref to /Users/kenny on windows<commit_after>// A command line tool for fragmenta which can be used to build and run websites
// this tool calls subcommands for most of the work, usually one command per file in this pkg
// See docs at http://godoc.org/github.com/fragmenta/fragmenta

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
)

const (
	// The version of this tool
	fragmentaVersion = "1.5.7"

	// Used for outputting console messages
	fragmentaDivider = "\n------\n"
)

// Modes used for setting the config used
const (
	ModeProduction  = "production"
	ModeDevelopment = "development"
	ModeTest        = "test"
)

var (
	// ConfigDevelopment holds the development config from fragmenta.json
	ConfigDevelopment map[string]string

	// ConfigProduction holds development config from fragmenta.json
	ConfigProduction map[string]string

	// ConfigTest holds the app test config from fragmenta.json
	ConfigTest map[string]string
)

// main - parse the command line arguments and respond
func main() {

	// Log time as well as date
	log.SetFlags(log.Ltime)

	// Parse commands
	args := os.Args
	command := ""

	if len(args) > 1 {
		command = args[1]
	}

	// We assume the project path is the current directory (for now)
	projectPath, err := filepath.Abs(".")
	if err != nil {
		log.Printf("Error getting path %s", err)
		return
	}

	// If this is a valid project, read the config, else continue
	if isValidProject(projectPath) {
		readConfig(projectPath)
	}

	switch command {

	case "new", "n":
		RunNew(args)

	case "version", "v":
		ShowVersion()

	case "help", "h", "wat", "?":
		ShowHelp(args)

	case "server", "s":
		if requireValidProject(projectPath) {
			RunServer(projectPath)
		}

	case "test", "t":
		if requireValidProject(projectPath) {
			// Remove fragmenta test from args list
			args = args[2:]
			RunTests(args)
		}

	case "build", "B":
		if requireValidProject(projectPath) {
			RunBuild(args)
		}

	case "generate", "g":
		if requireValidProject(projectPath) {
			RunGenerate(args)
		}

	case "migrate", "m":
		if requireValidProject(projectPath) {
			RunMigrate(args)
		}

	case "backup", "b":
		if requireValidProject(projectPath) {
			RunBackup(args)
		}

	case "restore", "r":
		if requireValidProject(projectPath) {
			RunRestore(args)
		}

	case "deploy", "d":
		if requireValidProject(projectPath) {
			RunDeploy(args)
		}
	case "":
		// Special case no commands to build and run the server
		if requireValidProject(projectPath) {
			RunServer(projectPath)
		}
	default:
		// Command not recognised so show the help
		ShowHelp(args)
	}

}

// ShowVersion shows the version of this tool
func ShowVersion() {
	helpString := fragmentaDivider
	helpString += fmt.Sprintf("Fragmenta version: %s", fragmentaVersion)
	helpString += fragmentaDivider
	log.Print(helpString)
}

// ShowHelp shows the help for this tool
func ShowHelp(args []string) {
	helpString := fragmentaDivider
	helpString += fmt.Sprintf("Fragmenta version: %s", fragmentaVersion)
	helpString += "\n  fragmenta version -> display version"
	helpString += "\n  fragmenta help -> display help"
	helpString += "\n  fragmenta new [app|cms|URL] path/to/app -> creates a new app from the repository at URL at the path supplied"
	helpString += "\n  fragmenta -> builds and runs a fragmenta app"
	helpString += "\n  fragmenta server -> builds and runs a fragmenta app"
	helpString += "\n  fragmenta test  -> run tests"
	helpString += "\n  fragmenta migrate -> runs new sql migrations in db/migrate"
	helpString += "\n  fragmenta backup [development|production|test] -> backup the database to db/backup"
	helpString += "\n  fragmenta restore [development|production|test] -> backup the database from latest file in db/backup"
	helpString += "\n  fragmenta deploy [development|production|test] -> build and deploy using bin/deploy"
	helpString += "\n  fragmenta generate resource [name] [fieldname]:[fieldtype]* -> creates resource CRUD actions and views"
	helpString += "\n  fragmenta generate migration [name] -> creates a new named sql migration in db/migrate"

	helpString += fragmentaDivider
	log.Print(helpString)
}

// Ideally all these paths could be configured,
// rather than baking assumptions about project structure into the tool

// serverName returns the path of the cross-compiled target server binary
// this does not end in .exe as we assume a target of linux
func serverName() string {
	return "fragmenta-server"
}

// localServerName returns a server name for the local server binary (prefixed with local)
func localServerName() string {
	if isWindows() {
		return serverName() + "-local.exe"
	}
	return serverName() + "-local"
}

// localServerPath returns the local server binary for running on the dev machine locally
func localServerPath(projectPath string) string {
	return filepath.Join(projectPath, "bin", localServerName())
}

// serverPath returns the cross-compiled server binary
func serverPath(projectPath string) string {
	return filepath.Join(projectPath, "bin", serverName())
}

// serverCompilePath returns the server entrypoint
func serverCompilePath(projectPath string) string {
	return filepath.Join(projectPath, "server.go")
}

// srcPath returns the path for Go code within the project
func srcPath(projectPath string) string {
	return filepath.Join(projectPath, "src")
}

// publicPath returns the path for the public directory of the web application
func publicPath(projectPath string) string {
	return filepath.Join(projectPath, "public")
}

// configPath returns the path for the fragment config file (required)
func configPath(projectPath string) string {
	return filepath.Join(secretsPath(projectPath), "fragmenta.json")
}

// secretsPath returns the path for secrets
func secretsPath(projectPath string) string {
	return filepath.Join(projectPath, "secrets")
}

// templatesPath returns the path for templates
func templatesPath() string {
	path := filepath.Join(goPath(), "src", "github.com", "fragmenta", "fragmenta", "templates")
	return os.ExpandEnv(path)
}

// dbMigratePath returns a path to store database migrations
func dbMigratePath(projectPath string) string {
	return filepath.Join(projectPath, "db", "migrate")
}

// dbBackupPath returns a path to store database backups
func dbBackupPath(projectPath string) string {
	return filepath.Join(projectPath, "db", "backup")
}

// projectPathRelative returns the relative path
func projectPathRelative(projectPath string) string {
	goSrc := filepath.Join(goPath(), "src")
	return strings.Replace(projectPath, goSrc, "", 1)
}

// goPath returns the setting of env variable $GOPATH
// or $HOME/go if no $GOPATH is set.
func goPath() string {
	p := os.ExpandEnv("$GOPATH")
	if len(p) > 0 {
		return p
	}
	return filepath.Join(homePath(), "go")
}

// homePath returns the user's home directory
func homePath() string {
	if isWindows() {
		return os.ExpandEnv("$userprofile")
	}
	return os.ExpandEnv("$HOME")
}

// RunServer runs the server
func RunServer(projectPath string) {
	ShowVersion()

	log.Println("Building server...")
	err := buildServer(localServerPath(projectPath), nil)

	if err != nil {
		log.Printf("Error building server: %s", err)
		return
	}

	log.Println("Launching server...")
	cmd := exec.Command(localServerPath(projectPath))
	stdout, _ := cmd.StdoutPipe()
	stderr, _ := cmd.StderrPipe()
	err = cmd.Start()
	if err != nil {
		log.Println(err)
	}
	go io.Copy(os.Stdout, stdout)
	go io.Copy(os.Stderr, stderr)
	cmd.Wait()

}

// runCommand runs a command with exec.Command
func runCommand(command string, args ...string) ([]byte, error) {

	cmd := exec.Command(command, args...)
	output, err := cmd.CombinedOutput()
	if err != nil {
		return output, err
	}

	return output, nil
}

// requireValidProject returns true if we have a valid project at projectPath
func requireValidProject(projectPath string) bool {
	if isValidProject(projectPath) {
		return true
	}

	log.Printf("No fragmenta project found at this path\n")
	return false
}

// isValidProject returns true if this is a valid fragmenta project (checks for server.go file and config file)
func isValidProject(projectPath string) bool {

	// Make sure we have server.go at root of this dir
	_, err := os.Stat(serverCompilePath(projectPath))
	if err != nil {
		return false
	}

	return true
}

// fileExists returns true if this file exists
func fileExists(p string) bool {
	_, err := os.Stat(p)
	if err != nil && os.IsNotExist(err) {
		return false
	}

	return true
}

// readConfig reads our config file and set up the server accordingly
func readConfig(projectPath string) error {
	configPath := configPath(projectPath)

	// Read the config json file
	file, err := ioutil.ReadFile(configPath)
	if err != nil {
		log.Printf("Error opening config at %s\n%s", configPath, err)
		return err
	}

	var data map[string]map[string]string
	err = json.Unmarshal(file, &data)
	if err != nil {
		log.Printf("Error parsing config %s %v", configPath, err)
		return err
	}

	ConfigDevelopment = data["development"]
	ConfigProduction = data["production"]
	ConfigTest = data["test"]

	return nil
}

// isWindows returns true if the Go architecture target (GOOS) is windows
func isWindows() bool {
	if runtime.GOOS == "windows" {
		return true
	}
	return false
}
<|endoftext|>