<commit_before>package scheduler

import (
	"math"
	"time"

	"github.com/hashicorp/nomad/nomad/structs"
)

const (
	// serviceJobAntiAffinityPenalty is the penalty applied
	// to the score for placing an alloc on a node that
	// already has an alloc for this job.
	serviceJobAntiAffinityPenalty = 10.0

	// batchJobAntiAffinityPenalty is the same as the
	// serviceJobAntiAffinityPenalty but for batch type jobs.
	batchJobAntiAffinityPenalty = 5.0
)

// Stack is a chained collection of iterators. The stack is used to
// make placement decisions. Different schedulers may customize the
// stack they use to vary the way placements are made.
type Stack interface {
	// SetNodes is used to set the base set of potential nodes
	SetNodes([]*structs.Node)

	// SetTaskGroup is used to set the job for selection
	SetJob(job *structs.Job)

	// Select is used to select a node for the task group
	Select(tg *structs.TaskGroup) (*RankedNode, *structs.Resources)
}

// GenericStack is the Stack used for the Generic scheduler. It is
// designed to make better placement decisions at the cost of performance.
type GenericStack struct {
	batch  bool
	ctx    Context
	source *StaticIterator

	wrappedChecks       *FeasibilityWrapper
	jobConstraint       *ConstraintChecker
	taskGroupDrivers    *DriverChecker
	taskGroupConstraint *ConstraintChecker

	proposedAllocConstraint *ProposedAllocConstraintIterator
	binPack                 *BinPackIterator
	jobAntiAff              *JobAntiAffinityIterator
	limit                   *LimitIterator
	maxScore                *MaxScoreIterator
}

// NewGenericStack constructs a stack used for selecting service placements
func NewGenericStack(batch bool, ctx Context) *GenericStack {
	// Create a new stack
	s := &GenericStack{
		batch: batch,
		ctx:   ctx,
	}

	// Create the source iterator. We randomize the order we visit nodes
	// to reduce collisions between schedulers and to do a basic load
	// balancing across eligible nodes.
	s.source = NewRandomIterator(ctx, nil)

	// Attach the job constraints. The job is filled in later.
	s.jobConstraint = NewConstraintChecker(ctx, nil)

	// Filter on task group drivers first as they are faster
	s.taskGroupDrivers = NewDriverChecker(ctx, nil)

	// Filter on task group constraints second
	s.taskGroupConstraint = NewConstraintChecker(ctx, nil)

	// Create the feasibility wrapper which wraps all feasibility checks in
	// which feasibility checking can be skipped if the computed node class has
	// previously been marked as eligible or ineligible. Generally this will be
	// checks that only needs to examine the single node to determine feasibility.
	jobs := []FeasibilityChecker{s.jobConstraint}
	tgs := []FeasibilityChecker{s.taskGroupDrivers, s.taskGroupConstraint}
	s.wrappedChecks = NewFeasibilityWrapper(ctx, s.source, jobs, tgs)

	// Filter on constraints that are affected by propsed allocations.
	s.proposedAllocConstraint = NewProposedAllocConstraintIterator(ctx, s.wrappedChecks)

	// Upgrade from feasible to rank iterator
	rankSource := NewFeasibleRankIterator(ctx, s.proposedAllocConstraint)

	// Apply the bin packing, this depends on the resources needed
	// by a particular task group. Only enable eviction for the service
	// scheduler as that logic is expensive.
	evict := !batch
	s.binPack = NewBinPackIterator(ctx, rankSource, evict, 0)

	// Apply the job anti-affinity iterator. This is to avoid placing
	// multiple allocations on the same node for this job. The penalty
	// is less for batch jobs as it matters less.
	penalty := serviceJobAntiAffinityPenalty
	if batch {
		penalty = batchJobAntiAffinityPenalty
	}
	s.jobAntiAff = NewJobAntiAffinityIterator(ctx, s.binPack, penalty, "")

	// Apply a limit function. This is to avoid scanning *every* possible node.
	s.limit = NewLimitIterator(ctx, s.jobAntiAff, 2)

	// Select the node with the maximum score for placement
	s.maxScore = NewMaxScoreIterator(ctx, s.limit)
	return s
}

func (s *GenericStack) SetNodes(baseNodes []*structs.Node) {
	// Shuffle base nodes
	shuffleNodes(baseNodes)

	// Update the set of base nodes
	s.source.SetNodes(baseNodes)

	// Apply a limit function. This is to avoid scanning *every* possible node.
	// For batch jobs we only need to evaluate 2 options and depend on the
	// power of two choices. For services jobs we need to visit "enough".
	// Using a log of the total number of nodes is a good restriction, with
	// at least 2 as the floor
	limit := 2
	if n := len(baseNodes); !s.batch && n > 0 {
		logLimit := int(math.Ceil(math.Log2(float64(n))))
		if logLimit > limit {
			limit = logLimit
		}
	}
	s.limit.SetLimit(limit)
}

func (s *GenericStack) SetJob(job *structs.Job) {
	s.jobConstraint.SetConstraints(job.Constraints)
	s.proposedAllocConstraint.SetJob(job)
	s.binPack.SetPriority(job.Priority)
	s.jobAntiAff.SetJob(job.ID)
	s.ctx.Eligibility().SetJob(job)
}

func (s *GenericStack) Select(tg *structs.TaskGroup) (*RankedNode, *structs.Resources) {
	// Reset the max selector and context
	s.maxScore.Reset()
	s.ctx.Reset()
	start := time.Now()

	// Get the task groups constraints.
	tgConstr := taskGroupConstraints(tg)

	// Update the parameters of iterators
	s.taskGroupDrivers.SetDrivers(tgConstr.drivers)
	s.taskGroupConstraint.SetConstraints(tgConstr.constraints)
	s.proposedAllocConstraint.SetTaskGroup(tg)
	s.wrappedChecks.SetTaskGroup(tg.Name)
	s.binPack.SetTaskGroup(tg)

	// Find the node with the max score
	option := s.maxScore.Next()

	// Ensure that the task resources were specified
	if option != nil && len(option.TaskResources) != len(tg.Tasks) {
		for _, task := range tg.Tasks {
			option.SetTaskResources(task, task.Resources)
		}
	}

	// Store the compute time
	s.ctx.Metrics().AllocationTime = time.Since(start)
	return option, tgConstr.size
}

// SelectPreferredNode returns a node where an allocation of the task group can
// be placed, the node passed to it is preferred over the other available nodes
func (s *GenericStack) SelectPreferringNodes(tg *structs.TaskGroup, nodes []*structs.Node) (*RankedNode, *structs.Resources) {
	originalNodes := s.source.nodes
	s.source.SetNodes(nodes)
	if option, resources := s.Select(tg); option != nil {
		s.source.SetNodes(originalNodes)
		return option, resources
	}
	s.source.SetNodes(originalNodes)
	return s.Select(tg)
}

// SystemStack is the Stack used for the System scheduler. It is designed to
// attempt to make placements on all nodes.
type SystemStack struct {
	ctx                 Context
	source              *StaticIterator
	wrappedChecks       *FeasibilityWrapper
	jobConstraint       *ConstraintChecker
	taskGroupDrivers    *DriverChecker
	taskGroupConstraint *ConstraintChecker
	binPack             *BinPackIterator
}

// NewSystemStack constructs a stack used for selecting service placements
func NewSystemStack(ctx Context) *SystemStack {
	// Create a new stack
	s := &SystemStack{ctx: ctx}

	// Create the source iterator. We visit nodes in a linear order because we
	// have to evaluate on all nodes.
	s.source = NewStaticIterator(ctx, nil)

	// Attach the job constraints. The job is filled in later.
	s.jobConstraint = NewConstraintChecker(ctx, nil)

	// Filter on task group drivers first as they are faster
	s.taskGroupDrivers = NewDriverChecker(ctx, nil)

	// Filter on task group constraints second
	s.taskGroupConstraint = NewConstraintChecker(ctx, nil)

	// Create the feasibility wrapper which wraps all feasibility checks in
	// which feasibility checking can be skipped if the computed node class has
	// previously been marked as eligible or ineligible. Generally this will be
	// checks that only needs to examine the single node to determine feasibility.
	jobs := []FeasibilityChecker{s.jobConstraint}
	tgs := []FeasibilityChecker{s.taskGroupDrivers, s.taskGroupConstraint}
	s.wrappedChecks = NewFeasibilityWrapper(ctx, s.source, jobs, tgs)

	// Upgrade from feasible to rank iterator
	rankSource := NewFeasibleRankIterator(ctx, s.wrappedChecks)

	// Apply the bin packing, this depends on the resources needed
	// by a particular task group. Enable eviction as system jobs are high
	// priority.
	s.binPack = NewBinPackIterator(ctx, rankSource, true, 0)
	return s
}

func (s *SystemStack) SetNodes(baseNodes []*structs.Node) {
	// Update the set of base nodes
	s.source.SetNodes(baseNodes)
}

func (s *SystemStack) SetJob(job *structs.Job) {
	s.jobConstraint.SetConstraints(job.Constraints)
	s.binPack.SetPriority(job.Priority)
	s.ctx.Eligibility().SetJob(job)
}

func (s *SystemStack) Select(tg *structs.TaskGroup) (*RankedNode, *structs.Resources) {
	// Reset the binpack selector and context
	s.binPack.Reset()
	s.ctx.Reset()
	start := time.Now()

	// Get the task groups constraints.
	tgConstr := taskGroupConstraints(tg)

	// Update the parameters of iterators
	s.taskGroupDrivers.SetDrivers(tgConstr.drivers)
	s.taskGroupConstraint.SetConstraints(tgConstr.constraints)
	s.binPack.SetTaskGroup(tg)
	s.wrappedChecks.SetTaskGroup(tg.Name)

	// Get the next option that satisfies the constraints.
	option := s.binPack.Next()

	// Ensure that the task resources were specified
	if option != nil && len(option.TaskResources) != len(tg.Tasks) {
		for _, task := range tg.Tasks {
			option.SetTaskResources(task, task.Resources)
		}
	}

	// Store the compute time
	s.ctx.Metrics().AllocationTime = time.Since(start)
	return option, tgConstr.size
}
<commit_msg>Double the anti-affinity for placing same task group on node<commit_after>package scheduler

import (
	"math"
	"time"

	"github.com/hashicorp/nomad/nomad/structs"
)

const (
	// serviceJobAntiAffinityPenalty is the penalty applied
	// to the score for placing an alloc on a node that
	// already has an alloc for this job.
	serviceJobAntiAffinityPenalty = 20.0

	// batchJobAntiAffinityPenalty is the same as the
	// serviceJobAntiAffinityPenalty but for batch type jobs.
	batchJobAntiAffinityPenalty = 10.0
)

// Stack is a chained collection of iterators. The stack is used to
// make placement decisions. Different schedulers may customize the
// stack they use to vary the way placements are made.
type Stack interface {
	// SetNodes is used to set the base set of potential nodes
	SetNodes([]*structs.Node)

	// SetTaskGroup is used to set the job for selection
	SetJob(job *structs.Job)

	// Select is used to select a node for the task group
	Select(tg *structs.TaskGroup) (*RankedNode, *structs.Resources)
}

// GenericStack is the Stack used for the Generic scheduler. It is
// designed to make better placement decisions at the cost of performance.
type GenericStack struct {
	batch  bool
	ctx    Context
	source *StaticIterator

	wrappedChecks       *FeasibilityWrapper
	jobConstraint       *ConstraintChecker
	taskGroupDrivers    *DriverChecker
	taskGroupConstraint *ConstraintChecker

	proposedAllocConstraint *ProposedAllocConstraintIterator
	binPack                 *BinPackIterator
	jobAntiAff              *JobAntiAffinityIterator
	limit                   *LimitIterator
	maxScore                *MaxScoreIterator
}

// NewGenericStack constructs a stack used for selecting service placements
func NewGenericStack(batch bool, ctx Context) *GenericStack {
	// Create a new stack
	s := &GenericStack{
		batch: batch,
		ctx:   ctx,
	}

	// Create the source iterator. We randomize the order we visit nodes
	// to reduce collisions between schedulers and to do a basic load
	// balancing across eligible nodes.
	s.source = NewRandomIterator(ctx, nil)

	// Attach the job constraints. The job is filled in later.
	s.jobConstraint = NewConstraintChecker(ctx, nil)

	// Filter on task group drivers first as they are faster
	s.taskGroupDrivers = NewDriverChecker(ctx, nil)

	// Filter on task group constraints second
	s.taskGroupConstraint = NewConstraintChecker(ctx, nil)

	// Create the feasibility wrapper which wraps all feasibility checks in
	// which feasibility checking can be skipped if the computed node class has
	// previously been marked as eligible or ineligible. Generally this will be
	// checks that only needs to examine the single node to determine feasibility.
	jobs := []FeasibilityChecker{s.jobConstraint}
	tgs := []FeasibilityChecker{s.taskGroupDrivers, s.taskGroupConstraint}
	s.wrappedChecks = NewFeasibilityWrapper(ctx, s.source, jobs, tgs)

	// Filter on constraints that are affected by propsed allocations.
	s.proposedAllocConstraint = NewProposedAllocConstraintIterator(ctx, s.wrappedChecks)

	// Upgrade from feasible to rank iterator
	rankSource := NewFeasibleRankIterator(ctx, s.proposedAllocConstraint)

	// Apply the bin packing, this depends on the resources needed
	// by a particular task group. Only enable eviction for the service
	// scheduler as that logic is expensive.
	evict := !batch
	s.binPack = NewBinPackIterator(ctx, rankSource, evict, 0)

	// Apply the job anti-affinity iterator. This is to avoid placing
	// multiple allocations on the same node for this job. The penalty
	// is less for batch jobs as it matters less.
	penalty := serviceJobAntiAffinityPenalty
	if batch {
		penalty = batchJobAntiAffinityPenalty
	}
	s.jobAntiAff = NewJobAntiAffinityIterator(ctx, s.binPack, penalty, "")

	// Apply a limit function. This is to avoid scanning *every* possible node.
	s.limit = NewLimitIterator(ctx, s.jobAntiAff, 2)

	// Select the node with the maximum score for placement
	s.maxScore = NewMaxScoreIterator(ctx, s.limit)
	return s
}

func (s *GenericStack) SetNodes(baseNodes []*structs.Node) {
	// Shuffle base nodes
	shuffleNodes(baseNodes)

	// Update the set of base nodes
	s.source.SetNodes(baseNodes)

	// Apply a limit function. This is to avoid scanning *every* possible node.
	// For batch jobs we only need to evaluate 2 options and depend on the
	// power of two choices. For services jobs we need to visit "enough".
	// Using a log of the total number of nodes is a good restriction, with
	// at least 2 as the floor
	limit := 2
	if n := len(baseNodes); !s.batch && n > 0 {
		logLimit := int(math.Ceil(math.Log2(float64(n))))
		if logLimit > limit {
			limit = logLimit
		}
	}
	s.limit.SetLimit(limit)
}

func (s *GenericStack) SetJob(job *structs.Job) {
	s.jobConstraint.SetConstraints(job.Constraints)
	s.proposedAllocConstraint.SetJob(job)
	s.binPack.SetPriority(job.Priority)
	s.jobAntiAff.SetJob(job.ID)
	s.ctx.Eligibility().SetJob(job)
}

func (s *GenericStack) Select(tg *structs.TaskGroup) (*RankedNode, *structs.Resources) {
	// Reset the max selector and context
	s.maxScore.Reset()
	s.ctx.Reset()
	start := time.Now()

	// Get the task groups constraints.
	tgConstr := taskGroupConstraints(tg)

	// Update the parameters of iterators
	s.taskGroupDrivers.SetDrivers(tgConstr.drivers)
	s.taskGroupConstraint.SetConstraints(tgConstr.constraints)
	s.proposedAllocConstraint.SetTaskGroup(tg)
	s.wrappedChecks.SetTaskGroup(tg.Name)
	s.binPack.SetTaskGroup(tg)

	// Find the node with the max score
	option := s.maxScore.Next()

	// Ensure that the task resources were specified
	if option != nil && len(option.TaskResources) != len(tg.Tasks) {
		for _, task := range tg.Tasks {
			option.SetTaskResources(task, task.Resources)
		}
	}

	// Store the compute time
	s.ctx.Metrics().AllocationTime = time.Since(start)
	return option, tgConstr.size
}

// SelectPreferredNode returns a node where an allocation of the task group can
// be placed, the node passed to it is preferred over the other available nodes
func (s *GenericStack) SelectPreferringNodes(tg *structs.TaskGroup, nodes []*structs.Node) (*RankedNode, *structs.Resources) {
	originalNodes := s.source.nodes
	s.source.SetNodes(nodes)
	if option, resources := s.Select(tg); option != nil {
		s.source.SetNodes(originalNodes)
		return option, resources
	}
	s.source.SetNodes(originalNodes)
	return s.Select(tg)
}

// SystemStack is the Stack used for the System scheduler. It is designed to
// attempt to make placements on all nodes.
type SystemStack struct {
	ctx                 Context
	source              *StaticIterator
	wrappedChecks       *FeasibilityWrapper
	jobConstraint       *ConstraintChecker
	taskGroupDrivers    *DriverChecker
	taskGroupConstraint *ConstraintChecker
	binPack             *BinPackIterator
}

// NewSystemStack constructs a stack used for selecting service placements
func NewSystemStack(ctx Context) *SystemStack {
	// Create a new stack
	s := &SystemStack{ctx: ctx}

	// Create the source iterator. We visit nodes in a linear order because we
	// have to evaluate on all nodes.
	s.source = NewStaticIterator(ctx, nil)

	// Attach the job constraints. The job is filled in later.
	s.jobConstraint = NewConstraintChecker(ctx, nil)

	// Filter on task group drivers first as they are faster
	s.taskGroupDrivers = NewDriverChecker(ctx, nil)

	// Filter on task group constraints second
	s.taskGroupConstraint = NewConstraintChecker(ctx, nil)

	// Create the feasibility wrapper which wraps all feasibility checks in
	// which feasibility checking can be skipped if the computed node class has
	// previously been marked as eligible or ineligible. Generally this will be
	// checks that only needs to examine the single node to determine feasibility.
	jobs := []FeasibilityChecker{s.jobConstraint}
	tgs := []FeasibilityChecker{s.taskGroupDrivers, s.taskGroupConstraint}
	s.wrappedChecks = NewFeasibilityWrapper(ctx, s.source, jobs, tgs)

	// Upgrade from feasible to rank iterator
	rankSource := NewFeasibleRankIterator(ctx, s.wrappedChecks)

	// Apply the bin packing, this depends on the resources needed
	// by a particular task group. Enable eviction as system jobs are high
	// priority.
	s.binPack = NewBinPackIterator(ctx, rankSource, true, 0)
	return s
}

func (s *SystemStack) SetNodes(baseNodes []*structs.Node) {
	// Update the set of base nodes
	s.source.SetNodes(baseNodes)
}

func (s *SystemStack) SetJob(job *structs.Job) {
	s.jobConstraint.SetConstraints(job.Constraints)
	s.binPack.SetPriority(job.Priority)
	s.ctx.Eligibility().SetJob(job)
}

func (s *SystemStack) Select(tg *structs.TaskGroup) (*RankedNode, *structs.Resources) {
	// Reset the binpack selector and context
	s.binPack.Reset()
	s.ctx.Reset()
	start := time.Now()

	// Get the task groups constraints.
	tgConstr := taskGroupConstraints(tg)

	// Update the parameters of iterators
	s.taskGroupDrivers.SetDrivers(tgConstr.drivers)
	s.taskGroupConstraint.SetConstraints(tgConstr.constraints)
	s.binPack.SetTaskGroup(tg)
	s.wrappedChecks.SetTaskGroup(tg.Name)

	// Get the next option that satisfies the constraints.
	option := s.binPack.Next()

	// Ensure that the task resources were specified
	if option != nil && len(option.TaskResources) != len(tg.Tasks) {
		for _, task := range tg.Tasks {
			option.SetTaskResources(task, task.Resources)
		}
	}

	// Store the compute time
	s.ctx.Metrics().AllocationTime = time.Since(start)
	return option, tgConstr.size
}
<|endoftext|>


<commit_before>package meter

import "time"

type Filter struct {
	maxage time.Duration
	res    *Resolution
	dims   [][]string // Dimensions for this filter

	maxdimsize int
	attrmask   map[string]bool // Needed attributes
}

// NewFilter creates and initializes a new Filter
func NewFilter(res *Resolution, maxage time.Duration, dims ...[]string) *Filter {
	f := &Filter{
		maxage:     maxage,
		attrmask:   make(map[string]bool),
		res:        res,
		dims:       dims,
		maxdimsize: 0,
	}
	for _, dim := range f.dims {
		if n := len(dim); n > f.maxdimsize {
			f.maxdimsize = n
		}
		for _, d := range dim {
			f.attrmask[d] = true
		}
	}

	return f
}

func (f *Filter) MaxDimSize() int {
	return f.maxdimsize
}
func (f *Filter) MaxAge() time.Duration {
	return f.maxage
}

func (f *Filter) Dimensions() [][]string {
	return f.dims
}
func (f *Filter) Resolution() *Resolution {
	return f.res
}

func (t *Filter) NeedsAttr(a string) bool {
	return t.attrmask[a]
}
<commit_msg>Remove unused filter code<commit_after>package meter

import "time"

type Filter struct {
	maxage time.Duration
	res    *Resolution
	dims   [][]string // Dimensions for this filter

}

// NewFilter creates and initializes a new Filter
func NewFilter(res *Resolution, maxage time.Duration, dims ...[]string) *Filter {
	f := &Filter{
		maxage: maxage,
		res:    res,
		dims:   make([][]string, 0, len(dims)),
	}
	for _, dim := range dims {
		if len(dim) > 0 {
			fdim := make([]string, 0, len(dim))
			for _, d := range dim {
				if d != "" {
					fdim = append(fdim, d)
				}
			}
			if len(fdim) > 0 {
				f.dims = append(f.dims, fdim)
			}
		}
	}

	return f
}

func (f *Filter) MaxAge() time.Duration {
	return f.maxage
}

func (f *Filter) Dimensions() [][]string {
	return f.dims
}
func (f *Filter) Resolution() *Resolution {
	return f.res
}
<|endoftext|>


<commit_before>package db

import (
	"fmt"
	"strings"
	"sync"
	"time"

	turtle "github.com/gtfierro/hod/goraptor"
	query "github.com/gtfierro/hod/lang"
	sparql "github.com/gtfierro/hod/lang/ast"

	"github.com/blevesearch/bleve"
	"github.com/coocood/freecache"
	"github.com/kr/pretty"
	"github.com/mitghi/btree"
	"github.com/pkg/errors"
)

func prettyprint(v interface{}) {
	fmt.Printf("%# v", pretty.Formatter(v))
}

func (db *DB) RunQuery(q *sparql.Query) (QueryResult, error) {
	fullQueryStart := time.Now()

	// "clean" the query by expanding out the prefixes
	// make sure to first do the Filters, then the Or clauses
	q.IterTriples(func(triple sparql.Triple) sparql.Triple {
		if !strings.HasPrefix(triple.Subject.Value, "?") {
			if full, found := db.namespaces[triple.Subject.Namespace]; found {
				triple.Subject.Namespace = full
			}
		}
		if !strings.HasPrefix(triple.Object.Value, "?") {
			if full, found := db.namespaces[triple.Object.Namespace]; found {
				triple.Object.Namespace = full
			}
		}
		for idx2, pred := range triple.Predicates {
			if !strings.HasPrefix(pred.Predicate.Value, "?") {
				if full, found := db.namespaces[pred.Predicate.Namespace]; found {
					pred.Predicate.Namespace = full
				}
				triple.Predicates[idx2] = pred
			}
		}
		return triple
	})

	// expand the graphgroup unions
	var ors [][]sparql.Triple
	if q.Where.GraphGroup != nil {
		for _, group := range q.Where.GraphGroup.Expand() {
			newterms := make([]sparql.Triple, len(q.Where.Terms))
			copy(newterms, q.Where.Terms)
			ors = append(ors, append(newterms, group...))
		}
	}

	// check query hash
	var queryhash []byte
	if db.queryCacheEnabled {
		queryhash = hashQuery(q)
		if ans, err := db.queryCache.Get(queryhash); err == nil {
			var res QueryResult
			if _, err := res.UnmarshalMsg(ans); err != nil {
				log.Error(errors.Wrap(err, "Could not fetch query from cache. Running..."))
			} else {
				// successful!
				res.Elapsed = time.Since(fullQueryStart)
				return res, nil
			}
		} else if err != nil && err == freecache.ErrNotFound {
			log.Notice("Could not fetch query from cache")
		} else if err != nil {
			log.Error(errors.Wrap(err, "Could not access query cache"))
		}
	}

	unionedRows := btree.New(BTREE_DEGREE, "")
	defer cleanResultRows(unionedRows)

	// if we have terms that are part of a set of OR statements, then we run
	// parallel queries for each fully-elaborated "branch" or the OR statement,
	// and then merge the results together at the end
	if len(ors) > 0 {
		var rowLock sync.Mutex
		var wg sync.WaitGroup
		var queryErr error
		wg.Add(len(ors))
		for _, group := range ors {
			tmpQuery := q.CopyWithNewTerms(group)
			tmpQuery.PopulateVars()
			go func(q *sparql.Query) {
				results, err := db.getQueryResults(&tmpQuery)
				rowLock.Lock()
				if err != nil {
					queryErr = err
				} else {
					log.Debug("got", len(results))
					for _, row := range results {
						unionedRows.ReplaceOrInsert(row)
					}
				}
				rowLock.Unlock()
				wg.Done()
			}(&tmpQuery)
		}
		wg.Wait()
		if queryErr != nil {
			return QueryResult{}, queryErr
		}
	} else {
		q.PopulateVars()
		results, err := db.getQueryResults(q)
		if err != nil {
			return QueryResult{}, err
		}
		for _, row := range results {
			unionedRows.ReplaceOrInsert(row)
		}
	}
	if db.showQueryLatencies {
		log.Noticef("Full Query took %s", time.Since(fullQueryStart))
	}

	var result = newQueryResult()
	result.selectVars = q.Select.Vars
	result.Elapsed = time.Since(fullQueryStart)

	// TODO: count!
	// return the rows
	log.Debug(unionedRows.Len())

	result.Count = unionedRows.Len()
	if !q.Count {
		i := unionedRows.DeleteMax()
		for i != nil {
			row := i.(*ResultRow)
			m := make(ResultMap)
			for idx, vname := range q.Select.Vars {
				m[vname] = row.row[idx]
			}
			result.Rows = append(result.Rows, m)
			finishResultRow(row)
			i = unionedRows.DeleteMax()
		}
	}

	if db.queryCacheEnabled {
		// set this in the cache
		marshalled, err := result.MarshalMsg(nil)
		if err != nil {
			log.Error(errors.Wrap(err, "Could not marshal results"))
		}
		if err := db.queryCache.Set(queryhash, marshalled, -1); err != nil {
			log.Error(errors.Wrap(err, "Could not cache results"))
		}
	}

	return result, nil
}

// takes a query and returns a DOT representation to visualize
// the construction of the query
func (db *DB) QueryToDOT(querystring string) (string, error) {
	q, err := query.Parse(querystring)
	if err != nil {
		return "", err
	}

	dot := ""
	dot += "digraph G {\n"
	dot += "ratio=\"auto\"\n"
	dot += "rankdir=\"LR\"\n"
	dot += "size=\"7.5,10\"\n"

	//if len(q.Where.Ors) > 0 {
	//	orTerms := query.FlattenOrClauseList(q.Where.Ors)
	//	oldFilters := q.Where.Filters
	//	for _, orTerm := range orTerms {
	//		filters := append(oldFilters, orTerm...)
	//		for _, filter := range filters {
	//			var parts []string
	//			for _, p := range filter.Path {
	//				parts = append(parts, fmt.Sprintf("%s%s", p.Predicate, p.Pattern))
	//			}
	//			line := fmt.Sprintf("\"%s\" -> \"%s\" [label=\"%s\"];\n", filter.Subject, filter.Object, strings.Join(parts, "/"))
	//			if !strings.Contains(dot, line) {
	//				dot += line
	//			}

	//		}
	//	}
	//} else {
	for _, filter := range q.Where.Terms {
		var parts []string
		for _, p := range filter.Predicates {
			parts = append(parts, fmt.Sprintf("%s%s", p.Predicate, p.Pattern))
		}
		line := fmt.Sprintf("\"%s\" -> \"%s\" [label=\"%s\"];\n", filter.Subject, filter.Object, strings.Join(parts, "/"))
		if !strings.Contains(dot, line) {
			dot += line
		}
	}
	//}
	for _, sv := range q.Select.Vars {
		dot += fmt.Sprintf("\"%s\" [fillcolor=#e57373]\n", sv)
	}
	dot += "}"
	return dot, nil
}

// executes a query and returns a DOT string of the classes involved
func (db *DB) QueryToClassDOT(querystring string) (string, error) {
	q, err := query.Parse(querystring)
	if err != nil {
		return "", err
	}
	// create DOT template string
	dot := ""
	dot += "digraph G {\n"

	// get rdf:type predicate hash as a string
	typeURI := turtle.ParseURI("rdf:type")
	typeURI.Namespace = db.namespaces[typeURI.Namespace]
	typeKey, err := db.GetHash(typeURI)
	if err != nil {
		return "", err
	}
	typeKeyString := typeKey.String()

	getClass := func(ent *Entity) (classes []turtle.URI, err error) {
		_classes := ent.OutEdges[typeKeyString]
		for _, class := range _classes {
			classes = append(classes, db.MustGetURI(class))
		}
		return
	}

	getEdges := func(ent *Entity) (predicates, objects []turtle.URI, reterr error) {
		var predKey Key
		for predKeyString, objectList := range ent.OutEdges {
			predKey.FromSlice([]byte(predKeyString))
			predURI, err := db.GetURI(predKey)
			if err != nil {
				reterr = err
				return
			}
			for _, objectKey := range objectList {
				objectEnt, err := db.GetEntityFromHash(objectKey)
				if err != nil {
					reterr = err
					return
				}
				objectClasses, err := getClass(objectEnt)
				if err != nil {
					reterr = err
					return
				}
				for _, class := range objectClasses {
					if predURI.Value != "type" && class.Value != "Class" {
						predicates = append(predicates, predURI)
						objects = append(objects, class)
					}
				}

			}
		}
		return
	}

	result, err := db.RunQuery(q)
	if err != nil {
		return "", err
	}
	for _, row := range result.Rows {
		for _, uri := range row {
			ent, err := db.GetEntity(uri)
			if err != nil {
				return "", err
			}
			classList, err := getClass(ent)
			if err != nil {
				return "", err
			}
			preds, objs, err := getEdges(ent)
			if err != nil {
				return "", err
			}
			// add class as node to graph
			for _, class := range classList {
				line := fmt.Sprintf("\"%s\" [fillcolor=\"#4caf50\"];\n", db.Abbreviate(class))
				if !strings.Contains(dot, line) {
					dot += line
				}
				for i := 0; i < len(preds); i++ {
					line := fmt.Sprintf("\"%s\" -> \"%s\" [label=\"%s\"];\n", db.Abbreviate(class), db.Abbreviate(objs[i]), db.Abbreviate(preds[i]))
					if !strings.Contains(dot, line) {
						dot += line
					}
				}
			}

		}
	}

	dot += "}"

	return dot, nil
}

func (db *DB) Abbreviate(uri turtle.URI) string {
	for abbv, ns := range db.namespaces {
		if abbv != "" && ns == uri.Namespace {
			return abbv + ":" + uri.Value
		}
	}
	return uri.Value
}

// Searches all of the values in the database; basic wildcard search
func (db *DB) Search(q string, n int) ([]string, error) {
	var res []string

	fmt.Println("Displaying", n, "results")
	query := bleve.NewMatchQuery(q)
	search := bleve.NewSearchRequestOptions(query, n, 0, false)
	searchResults, err := db.textidx.Search(search)
	if err != nil {
		fmt.Println(err)
		return res, err
	}
	for _, doc := range searchResults.Hits {
		res = append(res, db.Abbreviate(turtle.ParseURI(doc.ID)))
	}
	return res, nil
}
<commit_msg>add runquerystring method to api for embedding<commit_after>package db

import (
	"fmt"
	"strings"
	"sync"
	"time"

	turtle "github.com/gtfierro/hod/goraptor"
	query "github.com/gtfierro/hod/lang"
	sparql "github.com/gtfierro/hod/lang/ast"

	"github.com/blevesearch/bleve"
	"github.com/coocood/freecache"
	"github.com/kr/pretty"
	"github.com/mitghi/btree"
	"github.com/pkg/errors"
)

func prettyprint(v interface{}) {
	fmt.Printf("%# v", pretty.Formatter(v))
}

func (db *DB) RunQueryString(q string) (QueryResult, error) {
	var emptyres QueryResult
	if q, err := query.Parse(q); err != nil {
		e := errors.Wrap(err, "Could not parse hod query")
		log.Error(e)
		return emptyres, e
	} else if result, err := db.RunQuery(q); err != nil {
		e := errors.Wrap(err, "Could not complete hod query")
		log.Error(e)
		return emptyres, e
	} else {
		return result, nil
	}
}

func (db *DB) RunQuery(q *sparql.Query) (QueryResult, error) {
	fullQueryStart := time.Now()

	// "clean" the query by expanding out the prefixes
	// make sure to first do the Filters, then the Or clauses
	q.IterTriples(func(triple sparql.Triple) sparql.Triple {
		if !strings.HasPrefix(triple.Subject.Value, "?") {
			if full, found := db.namespaces[triple.Subject.Namespace]; found {
				triple.Subject.Namespace = full
			}
		}
		if !strings.HasPrefix(triple.Object.Value, "?") {
			if full, found := db.namespaces[triple.Object.Namespace]; found {
				triple.Object.Namespace = full
			}
		}
		for idx2, pred := range triple.Predicates {
			if !strings.HasPrefix(pred.Predicate.Value, "?") {
				if full, found := db.namespaces[pred.Predicate.Namespace]; found {
					pred.Predicate.Namespace = full
				}
				triple.Predicates[idx2] = pred
			}
		}
		return triple
	})

	// expand the graphgroup unions
	var ors [][]sparql.Triple
	if q.Where.GraphGroup != nil {
		for _, group := range q.Where.GraphGroup.Expand() {
			newterms := make([]sparql.Triple, len(q.Where.Terms))
			copy(newterms, q.Where.Terms)
			ors = append(ors, append(newterms, group...))
		}
	}

	// check query hash
	var queryhash []byte
	if db.queryCacheEnabled {
		queryhash = hashQuery(q)
		if ans, err := db.queryCache.Get(queryhash); err == nil {
			var res QueryResult
			if _, err := res.UnmarshalMsg(ans); err != nil {
				log.Error(errors.Wrap(err, "Could not fetch query from cache. Running..."))
			} else {
				// successful!
				res.Elapsed = time.Since(fullQueryStart)
				return res, nil
			}
		} else if err != nil && err == freecache.ErrNotFound {
			log.Notice("Could not fetch query from cache")
		} else if err != nil {
			log.Error(errors.Wrap(err, "Could not access query cache"))
		}
	}

	unionedRows := btree.New(BTREE_DEGREE, "")
	defer cleanResultRows(unionedRows)

	// if we have terms that are part of a set of OR statements, then we run
	// parallel queries for each fully-elaborated "branch" or the OR statement,
	// and then merge the results together at the end
	if len(ors) > 0 {
		var rowLock sync.Mutex
		var wg sync.WaitGroup
		var queryErr error
		wg.Add(len(ors))
		for _, group := range ors {
			tmpQuery := q.CopyWithNewTerms(group)
			tmpQuery.PopulateVars()
			go func(q *sparql.Query) {
				results, err := db.getQueryResults(&tmpQuery)
				rowLock.Lock()
				if err != nil {
					queryErr = err
				} else {
					log.Debug("got", len(results))
					for _, row := range results {
						unionedRows.ReplaceOrInsert(row)
					}
				}
				rowLock.Unlock()
				wg.Done()
			}(&tmpQuery)
		}
		wg.Wait()
		if queryErr != nil {
			return QueryResult{}, queryErr
		}
	} else {
		q.PopulateVars()
		results, err := db.getQueryResults(q)
		if err != nil {
			return QueryResult{}, err
		}
		for _, row := range results {
			unionedRows.ReplaceOrInsert(row)
		}
	}
	if db.showQueryLatencies {
		log.Noticef("Full Query took %s", time.Since(fullQueryStart))
	}

	var result = newQueryResult()
	result.selectVars = q.Select.Vars
	result.Elapsed = time.Since(fullQueryStart)

	// TODO: count!
	// return the rows
	log.Debug(unionedRows.Len())

	result.Count = unionedRows.Len()
	if !q.Count {
		i := unionedRows.DeleteMax()
		for i != nil {
			row := i.(*ResultRow)
			m := make(ResultMap)
			for idx, vname := range q.Select.Vars {
				m[vname] = row.row[idx]
			}
			result.Rows = append(result.Rows, m)
			finishResultRow(row)
			i = unionedRows.DeleteMax()
		}
	}

	if db.queryCacheEnabled {
		// set this in the cache
		marshalled, err := result.MarshalMsg(nil)
		if err != nil {
			log.Error(errors.Wrap(err, "Could not marshal results"))
		}
		if err := db.queryCache.Set(queryhash, marshalled, -1); err != nil {
			log.Error(errors.Wrap(err, "Could not cache results"))
		}
	}

	return result, nil
}

// takes a query and returns a DOT representation to visualize
// the construction of the query
func (db *DB) QueryToDOT(querystring string) (string, error) {
	q, err := query.Parse(querystring)
	if err != nil {
		return "", err
	}

	dot := ""
	dot += "digraph G {\n"
	dot += "ratio=\"auto\"\n"
	dot += "rankdir=\"LR\"\n"
	dot += "size=\"7.5,10\"\n"

	//if len(q.Where.Ors) > 0 {
	//	orTerms := query.FlattenOrClauseList(q.Where.Ors)
	//	oldFilters := q.Where.Filters
	//	for _, orTerm := range orTerms {
	//		filters := append(oldFilters, orTerm...)
	//		for _, filter := range filters {
	//			var parts []string
	//			for _, p := range filter.Path {
	//				parts = append(parts, fmt.Sprintf("%s%s", p.Predicate, p.Pattern))
	//			}
	//			line := fmt.Sprintf("\"%s\" -> \"%s\" [label=\"%s\"];\n", filter.Subject, filter.Object, strings.Join(parts, "/"))
	//			if !strings.Contains(dot, line) {
	//				dot += line
	//			}

	//		}
	//	}
	//} else {
	for _, filter := range q.Where.Terms {
		var parts []string
		for _, p := range filter.Predicates {
			parts = append(parts, fmt.Sprintf("%s%s", p.Predicate, p.Pattern))
		}
		line := fmt.Sprintf("\"%s\" -> \"%s\" [label=\"%s\"];\n", filter.Subject, filter.Object, strings.Join(parts, "/"))
		if !strings.Contains(dot, line) {
			dot += line
		}
	}
	//}
	for _, sv := range q.Select.Vars {
		dot += fmt.Sprintf("\"%s\" [fillcolor=#e57373]\n", sv)
	}
	dot += "}"
	return dot, nil
}

// executes a query and returns a DOT string of the classes involved
func (db *DB) QueryToClassDOT(querystring string) (string, error) {
	q, err := query.Parse(querystring)
	if err != nil {
		return "", err
	}
	// create DOT template string
	dot := ""
	dot += "digraph G {\n"

	// get rdf:type predicate hash as a string
	typeURI := turtle.ParseURI("rdf:type")
	typeURI.Namespace = db.namespaces[typeURI.Namespace]
	typeKey, err := db.GetHash(typeURI)
	if err != nil {
		return "", err
	}
	typeKeyString := typeKey.String()

	getClass := func(ent *Entity) (classes []turtle.URI, err error) {
		_classes := ent.OutEdges[typeKeyString]
		for _, class := range _classes {
			classes = append(classes, db.MustGetURI(class))
		}
		return
	}

	getEdges := func(ent *Entity) (predicates, objects []turtle.URI, reterr error) {
		var predKey Key
		for predKeyString, objectList := range ent.OutEdges {
			predKey.FromSlice([]byte(predKeyString))
			predURI, err := db.GetURI(predKey)
			if err != nil {
				reterr = err
				return
			}
			for _, objectKey := range objectList {
				objectEnt, err := db.GetEntityFromHash(objectKey)
				if err != nil {
					reterr = err
					return
				}
				objectClasses, err := getClass(objectEnt)
				if err != nil {
					reterr = err
					return
				}
				for _, class := range objectClasses {
					if predURI.Value != "type" && class.Value != "Class" {
						predicates = append(predicates, predURI)
						objects = append(objects, class)
					}
				}

			}
		}
		return
	}

	result, err := db.RunQuery(q)
	if err != nil {
		return "", err
	}
	for _, row := range result.Rows {
		for _, uri := range row {
			ent, err := db.GetEntity(uri)
			if err != nil {
				return "", err
			}
			classList, err := getClass(ent)
			if err != nil {
				return "", err
			}
			preds, objs, err := getEdges(ent)
			if err != nil {
				return "", err
			}
			// add class as node to graph
			for _, class := range classList {
				line := fmt.Sprintf("\"%s\" [fillcolor=\"#4caf50\"];\n", db.Abbreviate(class))
				if !strings.Contains(dot, line) {
					dot += line
				}
				for i := 0; i < len(preds); i++ {
					line := fmt.Sprintf("\"%s\" -> \"%s\" [label=\"%s\"];\n", db.Abbreviate(class), db.Abbreviate(objs[i]), db.Abbreviate(preds[i]))
					if !strings.Contains(dot, line) {
						dot += line
					}
				}
			}

		}
	}

	dot += "}"

	return dot, nil
}

func (db *DB) Abbreviate(uri turtle.URI) string {
	for abbv, ns := range db.namespaces {
		if abbv != "" && ns == uri.Namespace {
			return abbv + ":" + uri.Value
		}
	}
	return uri.Value
}

// Searches all of the values in the database; basic wildcard search
func (db *DB) Search(q string, n int) ([]string, error) {
	var res []string

	fmt.Println("Displaying", n, "results")
	query := bleve.NewMatchQuery(q)
	search := bleve.NewSearchRequestOptions(query, n, 0, false)
	searchResults, err := db.textidx.Search(search)
	if err != nil {
		fmt.Println(err)
		return res, err
	}
	for _, doc := range searchResults.Hits {
		res = append(res, db.Abbreviate(turtle.ParseURI(doc.ID)))
	}
	return res, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fswatch

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"
)

type FolderChange struct {
	timeStamp     time.Time
	newItems      []string
	movedItems    []string
	modifiedItems []string
}

func newFolderChange(newItems, movedItems, modifiedItems []string) *FolderChange {
	return &FolderChange{
		timeStamp:     time.Now(),
		newItems:      newItems,
		movedItems:    movedItems,
		modifiedItems: modifiedItems,
	}
}

func (folderChange *FolderChange) String() string {
	return fmt.Sprintf("Folderchange (timestamp: %s, new: %d, moved: %d)", folderChange.timeStamp, len(folderChange.New()), len(folderChange.Moved()))
}

func (folderChange *FolderChange) TimeStamp() time.Time {
	return folderChange.timeStamp
}

func (folderChange *FolderChange) New() []string {
	return folderChange.newItems
}

func (folderChange *FolderChange) Moved() []string {
	return folderChange.movedItems
}

type FolderWatcher struct {
	Change  chan *FolderChange
	Stopped chan bool

	recurse  bool
	skipFile func(path string) bool

	debug   bool
	folder  string
	running bool
}

func NewFolderWatcher(folderPath string, recurse bool, skipFile func(path string) bool) *FolderWatcher {
	return &FolderWatcher{
		Change:  make(chan *FolderChange),
		Stopped: make(chan bool),

		recurse:  recurse,
		skipFile: skipFile,

		debug:  false,
		folder: folderPath,
	}
}

func (folderWatcher *FolderWatcher) String() string {
	return fmt.Sprintf("Folderwatcher %q", folderWatcher.folder)
}

func (folderWatcher *FolderWatcher) Start() *FolderWatcher {
	folderWatcher.running = true
	sleepInterval := time.Second * 2

	go func() {

		// get existing entries
		directory := folderWatcher.folder
		entryList := getFolderEntries(directory, folderWatcher.recurse, folderWatcher.skipFile)

		for folderWatcher.IsRunning() {

			// get new entries
			updatedEntryList := getFolderEntries(directory, folderWatcher.recurse, folderWatcher.skipFile)

			// check for new items
			newItems := make([]string, 0)
			modifiedItems := make([]string, 0)

			for _, entry := range updatedEntryList {

				if isNewItem := !sliceContainsElement(entryList, entry); isNewItem {
					// entry is new
					newItems = append(newItems, entry)
					continue
				}

				// check if the file changed
				if fileInfo, err := os.Stat(entry); err == nil {

					// check if file has been modified
					timeOfLastCheck := time.Now().Add(sleepInterval * -1)
					if fileHasChanged(fileInfo, timeOfLastCheck) {

						// existing entry has been modified
						modifiedItems = append(modifiedItems, entry)
					}

				}
			}

			// check for moved items
			movedItems := make([]string, 0)
			for _, entry := range entryList {
				isMoved := !sliceContainsElement(updatedEntryList, entry)
				if isMoved {
					movedItems = append(movedItems, entry)
				}
			}

			// assign the new list
			entryList = updatedEntryList

			// sleep
			time.Sleep(sleepInterval)

			// check if something happened
			if len(newItems) > 0 || len(movedItems) > 0 || len(modifiedItems) > 0 {

				// send out change
				go func() {
					folderWatcher.Change <- newFolderChange(newItems, movedItems, modifiedItems)
				}()
			}
		}

		go func() {
			folderWatcher.Stopped <- true
		}()

		folderWatcher.log("Stopped")
	}()

	return folderWatcher
}

func (folderWatcher *FolderWatcher) Stop() *FolderWatcher {
	folderWatcher.log("Stopping")
	folderWatcher.running = false
	return folderWatcher
}

func (folderWatcher *FolderWatcher) IsRunning() bool {
	return folderWatcher.running
}

func (folderWatcher *FolderWatcher) log(message string) *FolderWatcher {
	if folderWatcher.debug {
		fmt.Printf("%s - %s\n", folderWatcher, message)
	}

	return folderWatcher
}

func getFolderEntries(directory string, recurse bool, skipFile func(path string) bool) []string {

	// the return array
	entries := make([]string, 0)

	// read the entries of the specified directory
	directoryEntries, err := ioutil.ReadDir(directory)
	if err != nil {
		return entries
	}

	for _, entry := range directoryEntries {

		// get the full path
		subEntryPath := filepath.Join(directory, entry.Name())

		// check if the enty shall be ignored
		if skipFile(subEntryPath) {
			continue
		}

		// recurse or append
		if recurse && entry.IsDir() {

			// recurse
			subFolderEntries := getFolderEntries(subEntryPath, recurse, skipFile)
			entries = append(entries, subFolderEntries...)

		} else {

			// append entry
			entries = append(entries, subEntryPath)
		}

	}

	return entries
}
<commit_msg>Bug fix for the folder watcher: The folder watcher did not recurse properly because the skipFile expression was evaluated for folders.<commit_after>// Copyright 2013 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fswatch

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"
)

type FolderChange struct {
	timeStamp     time.Time
	newItems      []string
	movedItems    []string
	modifiedItems []string
}

func newFolderChange(newItems, movedItems, modifiedItems []string) *FolderChange {
	return &FolderChange{
		timeStamp:     time.Now(),
		newItems:      newItems,
		movedItems:    movedItems,
		modifiedItems: modifiedItems,
	}
}

func (folderChange *FolderChange) String() string {
	return fmt.Sprintf("Folderchange (timestamp: %s, new: %d, moved: %d)", folderChange.timeStamp, len(folderChange.New()), len(folderChange.Moved()))
}

func (folderChange *FolderChange) TimeStamp() time.Time {
	return folderChange.timeStamp
}

func (folderChange *FolderChange) New() []string {
	return folderChange.newItems
}

func (folderChange *FolderChange) Moved() []string {
	return folderChange.movedItems
}

type FolderWatcher struct {
	Change  chan *FolderChange
	Stopped chan bool

	recurse  bool
	skipFile func(path string) bool

	debug   bool
	folder  string
	running bool
}

func NewFolderWatcher(folderPath string, recurse bool, skipFile func(path string) bool) *FolderWatcher {
	return &FolderWatcher{
		Change:  make(chan *FolderChange),
		Stopped: make(chan bool),

		recurse:  recurse,
		skipFile: skipFile,

		debug:  false,
		folder: folderPath,
	}
}

func (folderWatcher *FolderWatcher) String() string {
	return fmt.Sprintf("Folderwatcher %q", folderWatcher.folder)
}

func (folderWatcher *FolderWatcher) Start() *FolderWatcher {
	folderWatcher.running = true
	sleepInterval := time.Second * 2

	go func() {

		// get existing entries
		directory := folderWatcher.folder
		entryList := getFolderEntries(directory, folderWatcher.recurse, folderWatcher.skipFile)

		for folderWatcher.IsRunning() {

			// get new entries
			updatedEntryList := getFolderEntries(directory, folderWatcher.recurse, folderWatcher.skipFile)

			// check for new items
			newItems := make([]string, 0)
			modifiedItems := make([]string, 0)

			for _, entry := range updatedEntryList {

				if isNewItem := !sliceContainsElement(entryList, entry); isNewItem {
					// entry is new
					newItems = append(newItems, entry)
					continue
				}

				// check if the file changed
				if fileInfo, err := os.Stat(entry); err == nil {

					// check if file has been modified
					timeOfLastCheck := time.Now().Add(sleepInterval * -1)
					if fileHasChanged(fileInfo, timeOfLastCheck) {

						// existing entry has been modified
						modifiedItems = append(modifiedItems, entry)
					}

				}
			}

			// check for moved items
			movedItems := make([]string, 0)
			for _, entry := range entryList {
				isMoved := !sliceContainsElement(updatedEntryList, entry)
				if isMoved {
					movedItems = append(movedItems, entry)
				}
			}

			// assign the new list
			entryList = updatedEntryList

			// sleep
			time.Sleep(sleepInterval)

			// check if something happened
			if len(newItems) > 0 || len(movedItems) > 0 || len(modifiedItems) > 0 {

				// send out change
				go func() {
					folderWatcher.Change <- newFolderChange(newItems, movedItems, modifiedItems)
				}()
			}
		}

		go func() {
			folderWatcher.Stopped <- true
		}()

		folderWatcher.log("Stopped")
	}()

	return folderWatcher
}

func (folderWatcher *FolderWatcher) Stop() *FolderWatcher {
	folderWatcher.log("Stopping")
	folderWatcher.running = false
	return folderWatcher
}

func (folderWatcher *FolderWatcher) IsRunning() bool {
	return folderWatcher.running
}

func (folderWatcher *FolderWatcher) log(message string) *FolderWatcher {
	if folderWatcher.debug {
		fmt.Printf("%s - %s\n", folderWatcher, message)
	}

	return folderWatcher
}

func getFolderEntries(directory string, recurse bool, skipFile func(path string) bool) []string {

	// the return array
	entries := make([]string, 0)

	// read the entries of the specified directory
	directoryEntries, err := ioutil.ReadDir(directory)
	if err != nil {
		return entries
	}

	for _, entry := range directoryEntries {

		// get the full path
		subEntryPath := filepath.Join(directory, entry.Name())

		// recurse or append
		if recurse && entry.IsDir() {

			// recurse
			subFolderEntries := getFolderEntries(subEntryPath, recurse, skipFile)
			entries = append(entries, subFolderEntries...)

		} else {

			// check if the enty shall be ignored
			if skipFile(subEntryPath) {
				continue
			}

			// append entry
			entries = append(entries, subEntryPath)
		}

	}

	return entries
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/katnegermis/pocketmine-rcon"
)

func main() {
	if len(os.Args) < 2 {
		fmt.Printf("Usage: ./rcon address password")
		return
	}
	addr := os.Args[1]
	pass := os.Args[2]

	conn, err := rcon.NewConnection(addr, pass)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Printf("Successfully logged in at %s!\n", addr)

	prompt()
	stdin := bufio.NewReader(os.Stdin)
	input := ""
	for {
		if input, err = stdin.ReadString('\n'); err != nil {
			fmt.Println(err)
			return
		}
		input = strings.Trim(input[:len(input)-1], " ")
		if input == ".exit" {
			break
		}
		if len(input) == 0 {
			prompt()
			continue
		}

		r, err := conn.SendCommand(input)
		if err != nil {
			fmt.Printf("Error: %s\n", err)
			prompt()
			continue
		}

		fmt.Printf("Server:\n%s\n", r)
		prompt()
	}
}

func prompt() {
	fmt.Print("Enter command:\n>")
}
<commit_msg>Add trimsuffix to remove windows line endings<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/katnegermis/pocketmine-rcon"
)

func main() {
	if len(os.Args) < 2 {
		fmt.Printf("Usage: ./rcon address password")
		return
	}
	addr := os.Args[1]
	pass := os.Args[2]

	conn, err := rcon.NewConnection(addr, pass)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Printf("Successfully logged in at %s!\n", addr)

	prompt()
	stdin := bufio.NewReader(os.Stdin)
	input := ""
	for {
		if input, err = stdin.ReadString('\n'); err != nil {
			fmt.Println(err)
			return
		}
		input = strings.TrimSuffix(input, "\r\n")
		input = strings.Trim(input[:len(input)-1], " ")
		if input == ".exit" {
			break
		}
		if len(input) == 0 {
			prompt()
			continue
		}

		r, err := conn.SendCommand(input)
		if err != nil {
			fmt.Printf("Error: %s\n", err)
			prompt()
			continue
		}

		fmt.Printf("Server:\n%s\n", r)
		prompt()
	}
}

func prompt() {
	fmt.Print("Enter command:\n>")
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// Additions and modifications under the MIT License.

package dateformatter

import (
	"errors"
	"strconv"
	"time"

	"github.com/altipla-consulting/i18n-dateformatter/symbols"
)

const (
	_                        = iota
	stdLongMonth             = iota + stdNeedDate  // "January"
	stdMonth                                       // "Jan"
	stdNumMonth                                    // "1"
	stdZeroMonth                                   // "01"
	stdLongWeekDay                                 // "Monday"
	stdWeekDay                                     // "Mon"
	stdDay                                         // "2"
	stdUnderDay                                    // "_2"
	stdZeroDay                                     // "02"
	stdHour                  = iota + stdNeedClock // "15"
	stdHour12                                      // "3"
	stdZeroHour12                                  // "03"
	stdMinute                                      // "4"
	stdZeroMinute                                  // "04"
	stdSecond                                      // "5"
	stdZeroSecond                                  // "05"
	stdLongYear              = iota + stdNeedDate  // "2006"
	stdYear                                        // "06"
	stdPM                    = iota + stdNeedClock // "PM"
	stdpm                                          // "pm"
	stdTZ                    = iota                // "MST"
	stdISO8601TZ                                   // "Z0700"  // prints Z for UTC
	stdISO8601SecondsTZ                            // "Z070000"
	stdISO8601ColonTZ                              // "Z07:00" // prints Z for UTC
	stdISO8601ColonSecondsTZ                       // "Z07:00:00"
	stdNumTZ                                       // "-0700"  // always numeric
	stdNumSecondsTz                                // "-070000"
	stdNumShortTZ                                  // "-07"    // always numeric
	stdNumColonTZ                                  // "-07:00" // always numeric
	stdNumColonSecondsTZ                           // "-07:00:00"
	stdFracSecond0                                 // ".0", ".00", ... , trailing zeros included
	stdFracSecond9                                 // ".9", ".99", ..., trailing zeros omitted

	stdNeedDate  = 1 << 8             // need month, day, year
	stdNeedClock = 2 << 8             // need hour, minute, second
	stdArgShift  = 16                 // extra argument in high bits, above low stdArgShift
	stdMask      = 1<<stdArgShift - 1 // mask out argument
)

// std0x records the std values for "01", "02", ..., "06".
var std0x = [...]int{stdZeroMonth, stdZeroDay, stdZeroHour12, stdZeroMinute, stdZeroSecond, stdYear}

// startsWithLowerCase reports whether the string has a lower-case letter at the beginning.
// Its purpose is to prevent matching strings like "Month" when looking for "Mon".
func startsWithLowerCase(str string) bool {
	if len(str) == 0 {
		return false
	}
	c := str[0]
	return 'a' <= c && c <= 'z'
}

// nextStdChunk finds the first occurrence of a std string in
// layout and returns the text before, the std string, and the text after.
func nextStdChunk(layout string) (prefix string, std int, suffix string) {
	for i := 0; i < len(layout); i++ {
		switch c := int(layout[i]); c {
		case 'J': // January, Jan
			if len(layout) >= i+3 && layout[i:i+3] == "Jan" {
				if len(layout) >= i+7 && layout[i:i+7] == "January" {
					return layout[0:i], stdLongMonth, layout[i+7:]
				}
				if !startsWithLowerCase(layout[i+3:]) {
					return layout[0:i], stdMonth, layout[i+3:]
				}
			}

		case 'M': // Monday, Mon, MST
			if len(layout) >= i+3 {
				if layout[i:i+3] == "Mon" {
					if len(layout) >= i+6 && layout[i:i+6] == "Monday" {
						return layout[0:i], stdLongWeekDay, layout[i+6:]
					}
					if !startsWithLowerCase(layout[i+3:]) {
						return layout[0:i], stdWeekDay, layout[i+3:]
					}
				}
				if layout[i:i+3] == "MST" {
					return layout[0:i], stdTZ, layout[i+3:]
				}
			}

		case '0': // 01, 02, 03, 04, 05, 06
			if len(layout) >= i+2 && '1' <= layout[i+1] && layout[i+1] <= '6' {
				return layout[0:i], std0x[layout[i+1]-'1'], layout[i+2:]
			}

		case '1': // 15, 1
			if len(layout) >= i+2 && layout[i+1] == '5' {
				return layout[0:i], stdHour, layout[i+2:]
			}
			return layout[0:i], stdNumMonth, layout[i+1:]

		case '2': // 2006, 2
			if len(layout) >= i+4 && layout[i:i+4] == "2006" {
				return layout[0:i], stdLongYear, layout[i+4:]
			}
			return layout[0:i], stdDay, layout[i+1:]

		case '_': // _2
			if len(layout) >= i+2 && layout[i+1] == '2' {
				return layout[0:i], stdUnderDay, layout[i+2:]
			}

		case '3':
			return layout[0:i], stdHour12, layout[i+1:]

		case '4':
			return layout[0:i], stdMinute, layout[i+1:]

		case '5':
			return layout[0:i], stdSecond, layout[i+1:]

		case 'P': // PM
			if len(layout) >= i+2 && layout[i+1] == 'M' {
				return layout[0:i], stdPM, layout[i+2:]
			}

		case 'p': // pm
			if len(layout) >= i+2 && layout[i+1] == 'm' {
				return layout[0:i], stdpm, layout[i+2:]
			}
		}
	}
	return layout, 0, ""
}

// match reports whether s1 and s2 match ignoring case.
// It is assumed s1 and s2 are the same length.
func match(s1, s2 string) bool {
	for i := 0; i < len(s1); i++ {
		c1 := s1[i]
		c2 := s2[i]
		if c1 != c2 {
			// Switch to lower-case; 'a'-'A' is known to be a single bit.
			c1 |= 'a' - 'A'
			c2 |= 'a' - 'A'
			if c1 != c2 || c1 < 'a' || c1 > 'z' {
				return false
			}
		}
	}
	return true
}

func lookup(tab []string, val string) (int, string, error) {
	for i, v := range tab {
		if len(val) >= len(v) && match(val[0:len(v)], v) {
			return i, val[len(v):], nil
		}
	}
	return -1, val, errBad
}

// appendInt appends the decimal form of x to b and returns the result.
// If the decimal form (excluding sign) is shorter than width, the result is padded with leading 0's.
// Duplicates functionality in strconv, but avoids dependency.
func appendInt(b []byte, x int, width int) []byte {
	formatted := strconv.AppendInt(b, int64(x), 10)

	for i := 0; i < width-len(formatted); i++ {
		b = append(b, '0')
	}
	b = append(b, formatted...)

	return b
}

// Format returns a textual representation of the time value formatted
// according to layout, which defines the format by showing how the reference
// time, defined to be
//  Mon Jan 2 15:04:05 -0700 MST 2006
// would be displayed if it were the value; it serves as an example of the
// desired output. The same display rules will then be applied to the time
// value.
//
// A fractional second is represented by adding a period and zeros
// to the end of the seconds section of layout string, as in "15:04:05.000"
// to format a time stamp with millisecond precision.
//
// Predefined layouts ANSIC, UnixDate, RFC3339 and others describe standard
// and convenient representations of the reference time. For more information
// about the formats and the definition of the reference time, see the
// documentation for ANSIC and the other constants defined by this package.
func Format(t time.Time, locale, layout string) string {
	var b []byte

	var (
		year  int = -1
		month time.Month
		day   int
		hour  int = -1
		min   int
		sec   int
	)
	// Each iteration generates one std value.
	for layout != "" {
		prefix, std, suffix := nextStdChunk(layout)
		if prefix != "" {
			b = append(b, prefix...)
		}
		if std == 0 {
			break
		}
		layout = suffix

		// Compute year, month, day if needed.
		if year < 0 && std&stdNeedDate != 0 {
			year, month, day = t.Date()
		}

		// Compute hour, minute, second if needed.
		if hour < 0 && std&stdNeedClock != 0 {
			hour, min, sec = t.Clock()
		}

		switch std & stdMask {
		case stdYear:
			y := year
			if y < 0 {
				y = -y
			}
			b = appendInt(b, y%100, 2)
		case stdLongYear:
			b = appendInt(b, year, 4)
		case stdMonth:
			m := symbols.ShortMonthNames[locale][month]
			b = append(b, m...)
		case stdLongMonth:
			m := symbols.LongMonthNames[locale][month]
			b = append(b, m...)
		case stdNumMonth:
			b = appendInt(b, int(month), 0)
		case stdZeroMonth:
			b = appendInt(b, int(month), 2)
		case stdWeekDay:
			b = append(b, t.Weekday().String()[:3]...)
		case stdLongWeekDay:
			s := t.Weekday().String()
			b = append(b, s...)
		case stdDay:
			b = appendInt(b, day, 0)
		case stdUnderDay:
			if day < 10 {
				b = append(b, ' ')
			}
			b = appendInt(b, day, 0)
		case stdZeroDay:
			b = appendInt(b, day, 2)
		case stdHour:
			b = appendInt(b, hour, 2)
		case stdHour12:
			// Noon is 12PM, midnight is 12AM.
			hr := hour % 12
			if hr == 0 {
				hr = 12
			}
			b = appendInt(b, hr, 0)
		case stdZeroHour12:
			// Noon is 12PM, midnight is 12AM.
			hr := hour % 12
			if hr == 0 {
				hr = 12
			}
			b = appendInt(b, hr, 2)
		case stdMinute:
			b = appendInt(b, min, 0)
		case stdZeroMinute:
			b = appendInt(b, min, 2)
		case stdSecond:
			b = appendInt(b, sec, 2)
		case stdZeroSecond:
			b = appendInt(b, sec, 2)
		case stdPM:
			if hour >= 12 {
				b = append(b, "PM"...)
			} else {
				b = append(b, "AM"...)
			}
		case stdpm:
			if hour >= 12 {
				b = append(b, "pm"...)
			} else {
				b = append(b, "am"...)
			}
		}
	}

	return string(b)
}

var errBad = errors.New("bad value for field") // placeholder not passed to user

// isDigit reports whether s[i] is in range and is a decimal digit.
func isDigit(s string, i int) bool {
	if len(s) <= i {
		return false
	}
	c := s[i]
	return '0' <= c && c <= '9'
}
<commit_msg>Use fmt standard lib directly to avoid repetitions when formatting.<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// Additions and modifications under the MIT License.

package dateformatter

import (
	"errors"
	"fmt"
	"time"

	"github.com/altipla-consulting/i18n-dateformatter/symbols"
)

const (
	_                        = iota
	stdLongMonth             = iota + stdNeedDate  // "January"
	stdMonth                                       // "Jan"
	stdNumMonth                                    // "1"
	stdZeroMonth                                   // "01"
	stdLongWeekDay                                 // "Monday"
	stdWeekDay                                     // "Mon"
	stdDay                                         // "2"
	stdUnderDay                                    // "_2"
	stdZeroDay                                     // "02"
	stdHour                  = iota + stdNeedClock // "15"
	stdHour12                                      // "3"
	stdZeroHour12                                  // "03"
	stdMinute                                      // "4"
	stdZeroMinute                                  // "04"
	stdSecond                                      // "5"
	stdZeroSecond                                  // "05"
	stdLongYear              = iota + stdNeedDate  // "2006"
	stdYear                                        // "06"
	stdPM                    = iota + stdNeedClock // "PM"
	stdpm                                          // "pm"
	stdTZ                    = iota                // "MST"
	stdISO8601TZ                                   // "Z0700"  // prints Z for UTC
	stdISO8601SecondsTZ                            // "Z070000"
	stdISO8601ColonTZ                              // "Z07:00" // prints Z for UTC
	stdISO8601ColonSecondsTZ                       // "Z07:00:00"
	stdNumTZ                                       // "-0700"  // always numeric
	stdNumSecondsTz                                // "-070000"
	stdNumShortTZ                                  // "-07"    // always numeric
	stdNumColonTZ                                  // "-07:00" // always numeric
	stdNumColonSecondsTZ                           // "-07:00:00"
	stdFracSecond0                                 // ".0", ".00", ... , trailing zeros included
	stdFracSecond9                                 // ".9", ".99", ..., trailing zeros omitted

	stdNeedDate  = 1 << 8             // need month, day, year
	stdNeedClock = 2 << 8             // need hour, minute, second
	stdArgShift  = 16                 // extra argument in high bits, above low stdArgShift
	stdMask      = 1<<stdArgShift - 1 // mask out argument
)

// std0x records the std values for "01", "02", ..., "06".
var std0x = [...]int{stdZeroMonth, stdZeroDay, stdZeroHour12, stdZeroMinute, stdZeroSecond, stdYear}

// startsWithLowerCase reports whether the string has a lower-case letter at the beginning.
// Its purpose is to prevent matching strings like "Month" when looking for "Mon".
func startsWithLowerCase(str string) bool {
	if len(str) == 0 {
		return false
	}
	c := str[0]
	return 'a' <= c && c <= 'z'
}

// nextStdChunk finds the first occurrence of a std string in
// layout and returns the text before, the std string, and the text after.
func nextStdChunk(layout string) (prefix string, std int, suffix string) {
	for i := 0; i < len(layout); i++ {
		switch c := int(layout[i]); c {
		case 'J': // January, Jan
			if len(layout) >= i+3 && layout[i:i+3] == "Jan" {
				if len(layout) >= i+7 && layout[i:i+7] == "January" {
					return layout[0:i], stdLongMonth, layout[i+7:]
				}
				if !startsWithLowerCase(layout[i+3:]) {
					return layout[0:i], stdMonth, layout[i+3:]
				}
			}

		case 'M': // Monday, Mon, MST
			if len(layout) >= i+3 {
				if layout[i:i+3] == "Mon" {
					if len(layout) >= i+6 && layout[i:i+6] == "Monday" {
						return layout[0:i], stdLongWeekDay, layout[i+6:]
					}
					if !startsWithLowerCase(layout[i+3:]) {
						return layout[0:i], stdWeekDay, layout[i+3:]
					}
				}
				if layout[i:i+3] == "MST" {
					return layout[0:i], stdTZ, layout[i+3:]
				}
			}

		case '0': // 01, 02, 03, 04, 05, 06
			if len(layout) >= i+2 && '1' <= layout[i+1] && layout[i+1] <= '6' {
				return layout[0:i], std0x[layout[i+1]-'1'], layout[i+2:]
			}

		case '1': // 15, 1
			if len(layout) >= i+2 && layout[i+1] == '5' {
				return layout[0:i], stdHour, layout[i+2:]
			}
			return layout[0:i], stdNumMonth, layout[i+1:]

		case '2': // 2006, 2
			if len(layout) >= i+4 && layout[i:i+4] == "2006" {
				return layout[0:i], stdLongYear, layout[i+4:]
			}
			return layout[0:i], stdDay, layout[i+1:]

		case '_': // _2
			if len(layout) >= i+2 && layout[i+1] == '2' {
				return layout[0:i], stdUnderDay, layout[i+2:]
			}

		case '3':
			return layout[0:i], stdHour12, layout[i+1:]

		case '4':
			return layout[0:i], stdMinute, layout[i+1:]

		case '5':
			return layout[0:i], stdSecond, layout[i+1:]

		case 'P': // PM
			if len(layout) >= i+2 && layout[i+1] == 'M' {
				return layout[0:i], stdPM, layout[i+2:]
			}

		case 'p': // pm
			if len(layout) >= i+2 && layout[i+1] == 'm' {
				return layout[0:i], stdpm, layout[i+2:]
			}
		}
	}
	return layout, 0, ""
}

// match reports whether s1 and s2 match ignoring case.
// It is assumed s1 and s2 are the same length.
func match(s1, s2 string) bool {
	for i := 0; i < len(s1); i++ {
		c1 := s1[i]
		c2 := s2[i]
		if c1 != c2 {
			// Switch to lower-case; 'a'-'A' is known to be a single bit.
			c1 |= 'a' - 'A'
			c2 |= 'a' - 'A'
			if c1 != c2 || c1 < 'a' || c1 > 'z' {
				return false
			}
		}
	}
	return true
}

func lookup(tab []string, val string) (int, string, error) {
	for i, v := range tab {
		if len(val) >= len(v) && match(val[0:len(v)], v) {
			return i, val[len(v):], nil
		}
	}
	return -1, val, errBad
}

// appendInt appends the decimal form of x to b and returns the result.
// If the decimal form (excluding sign) is shorter than width, the result is padded with leading 0's.
// Duplicates functionality in strconv, but avoids dependency.
func appendInt(b []byte, x int, width int) []byte {
	format := fmt.Sprintf("%%0%dd", width)
	return append(b, []byte(fmt.Sprintf(format, x))...)
}

// Format returns a textual representation of the time value formatted
// according to layout, which defines the format by showing how the reference
// time, defined to be
//  Mon Jan 2 15:04:05 2006
// would be displayed if it were the value; it serves as an example of the
// desired output. The same display rules will then be applied to the time
// value.
//
// Predefined layouts ANSIC, UnixDate, RFC3339 and others describe standard
// and convenient representations of the reference time. For more information
// about the formats and the definition of the reference time, see the
// documentation for ANSIC and the other constants defined by this package.
func Format(t time.Time, locale, layout string) string {
	var b []byte

	var (
		year  int = -1
		month time.Month
		day   int
		hour  int = -1
		min   int
		sec   int
	)
	// Each iteration generates one std value.
	for layout != "" {
		prefix, std, suffix := nextStdChunk(layout)
		if prefix != "" {
			b = append(b, prefix...)
		}
		if std == 0 {
			break
		}
		layout = suffix

		// Compute year, month, day if needed.
		if year < 0 && std&stdNeedDate != 0 {
			year, month, day = t.Date()
		}

		// Compute hour, minute, second if needed.
		if hour < 0 && std&stdNeedClock != 0 {
			hour, min, sec = t.Clock()
		}

		switch std & stdMask {
		case stdYear:
			y := year
			if y < 0 {
				y = -y
			}
			b = appendInt(b, y%100, 2)
		case stdLongYear:
			b = appendInt(b, year, 4)
		case stdMonth:
			m := symbols.ShortMonthNames[locale][month]
			b = append(b, m...)
		case stdLongMonth:
			m := symbols.LongMonthNames[locale][month]
			b = append(b, m...)
		case stdNumMonth:
			b = appendInt(b, int(month), 0)
		case stdZeroMonth:
			b = appendInt(b, int(month), 2)
		case stdWeekDay:
			b = append(b, t.Weekday().String()[:3]...)
		case stdLongWeekDay:
			s := t.Weekday().String()
			b = append(b, s...)
		case stdDay:
			b = appendInt(b, day, 0)
		case stdUnderDay:
			if day < 10 {
				b = append(b, ' ')
			}
			b = appendInt(b, day, 0)
		case stdZeroDay:
			b = appendInt(b, day, 2)
		case stdHour:
			b = appendInt(b, hour, 2)
		case stdHour12:
			// Noon is 12PM, midnight is 12AM.
			hr := hour % 12
			if hr == 0 {
				hr = 12
			}
			b = appendInt(b, hr, 0)
		case stdZeroHour12:
			// Noon is 12PM, midnight is 12AM.
			hr := hour % 12
			if hr == 0 {
				hr = 12
			}
			b = appendInt(b, hr, 2)
		case stdMinute:
			b = appendInt(b, min, 0)
		case stdZeroMinute:
			b = appendInt(b, min, 2)
		case stdSecond:
			b = appendInt(b, sec, 2)
		case stdZeroSecond:
			b = appendInt(b, sec, 2)
		case stdPM:
			if hour >= 12 {
				b = append(b, "PM"...)
			} else {
				b = append(b, "AM"...)
			}
		case stdpm:
			if hour >= 12 {
				b = append(b, "pm"...)
			} else {
				b = append(b, "am"...)
			}
		}
	}

	return string(b)
}

var errBad = errors.New("bad value for field") // placeholder not passed to user

// isDigit reports whether s[i] is in range and is a decimal digit.
func isDigit(s string, i int) bool {
	if len(s) <= i {
		return false
	}
	c := s[i]
	return '0' <= c && c <= '9'
}
<|endoftext|>


<commit_before>// Copyright 2018 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cli

import (
	"net/url"
	"os"
	"path"
	"time"

	"github.com/go-openapi/strfmt"
	"github.com/prometheus/common/version"
	kingpin "gopkg.in/alecthomas/kingpin.v2"

	"github.com/prometheus/alertmanager/api/v2/client"
	"github.com/prometheus/alertmanager/cli/config"
	"github.com/prometheus/alertmanager/cli/format"

	clientruntime "github.com/go-openapi/runtime/client"
)

var (
	verbose         bool
	alertmanagerURL *url.URL
	output          string
	timeout         time.Duration

	configFiles = []string{os.ExpandEnv("$HOME/.config/amtool/config.yml"), "/etc/amtool/config.yml"}
	legacyFlags = map[string]string{"comment_required": "require-comment"}
)

func requireAlertManagerURL(pc *kingpin.ParseContext) error {
	// Return without error if any help flag is set.
	for _, elem := range pc.Elements {
		f, ok := elem.Clause.(*kingpin.FlagClause)
		if !ok {
			continue
		}
		name := f.Model().Name
		if name == "help" || name == "help-long" || name == "help-man" {
			return nil
		}
	}
	if alertmanagerURL == nil {
		kingpin.Fatalf("required flag --alertmanager.url not provided")
	}
	return nil
}

const (
	defaultAmHost      = "localhost"
	defaultAmPort      = "9093"
	defaultAmApiv2path = "/api/v2"
)

// NewAlertmanagerClient initializes an alertmanager client with the given URL
func NewAlertmanagerClient(amURL *url.URL) *client.Alertmanager {
	address := defaultAmHost + ":" + defaultAmPort
	schemes := []string{"http"}

	if amURL.Host != "" {
		address = amURL.Host // URL documents host as host or host:port
	}
	if amURL.Scheme != "" {
		schemes = []string{amURL.Scheme}
	}

	cr := clientruntime.New(address, path.Join(amURL.Path, defaultAmApiv2path), schemes)

	if amURL.User != nil {
		password, _ := amURL.User.Password()
		cr.DefaultAuthentication = clientruntime.BasicAuth(amURL.User.Username(), password)
	}

	return client.New(cr, strfmt.Default)
}

// Execute is the main function for the amtool command
func Execute() {
	var (
		app = kingpin.New("amtool", helpRoot).UsageWriter(os.Stdout)
	)

	format.InitFormatFlags(app)

	app.Flag("verbose", "Verbose running information").Short('v').BoolVar(&verbose)
	app.Flag("alertmanager.url", "Alertmanager to talk to").URLVar(&alertmanagerURL)
	app.Flag("output", "Output formatter (simple, extended, json)").Short('o').Default("simple").EnumVar(&output, "simple", "extended", "json")
	app.Flag("timeout", "Timeout for the executed command").Default("30s").DurationVar(&timeout)

	app.Version(version.Print("amtool"))
	app.GetFlag("help").Short('h')
	app.UsageTemplate(kingpin.CompactUsageTemplate)

	resolver, err := config.NewResolver(configFiles, legacyFlags)
	if err != nil {
		kingpin.Fatalf("could not load config file: %v\n", err)
	}

	configureAlertCmd(app)
	configureSilenceCmd(app)
	configureCheckConfigCmd(app)
	configureClusterCmd(app)
	configureConfigCmd(app)
	configureTemplateCmd(app)

	err = resolver.Bind(app, os.Args[1:])
	if err != nil {
		kingpin.Fatalf("%v\n", err)
	}

	_, err = app.Parse(os.Args[1:])
	if err != nil {
		kingpin.Fatalf("%v\n", err)
	}
}

const (
	helpRoot = `View and modify the current Alertmanager state.

Config File:
The alertmanager tool will read a config file in YAML format from one of two
default config locations: $HOME/.config/amtool/config.yml or
/etc/amtool/config.yml

All flags can be given in the config file, but the following are the suited for
static configuration:

	alertmanager.url
		Set a default alertmanager url for each request

	author
		Set a default author value for new silences. If this argument is not
		specified then the username will be used

	require-comment
		Bool, whether to require a comment on silence creation. Defaults to true

	output
		Set a default output type. Options are (simple, extended, json)

	date.format
		Sets the output format for dates. Defaults to "2006-01-02 15:04:05 MST"
`
)
<commit_msg>Add ability to skip TLS verification for amtool (#2663)<commit_after>// Copyright 2018 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cli

import (
	"crypto/tls"
	"net/http"
	"net/url"
	"os"
	"path"
	"time"

	"github.com/go-openapi/strfmt"
	"github.com/prometheus/common/version"
	kingpin "gopkg.in/alecthomas/kingpin.v2"

	"github.com/prometheus/alertmanager/api/v2/client"
	"github.com/prometheus/alertmanager/cli/config"
	"github.com/prometheus/alertmanager/cli/format"

	clientruntime "github.com/go-openapi/runtime/client"
)

var (
	verbose               bool
	alertmanagerURL       *url.URL
	output                string
	timeout               time.Duration
	tlsInsecureSkipVerify bool

	configFiles = []string{os.ExpandEnv("$HOME/.config/amtool/config.yml"), "/etc/amtool/config.yml"}
	legacyFlags = map[string]string{"comment_required": "require-comment"}
)

func requireAlertManagerURL(pc *kingpin.ParseContext) error {
	// Return without error if any help flag is set.
	for _, elem := range pc.Elements {
		f, ok := elem.Clause.(*kingpin.FlagClause)
		if !ok {
			continue
		}
		name := f.Model().Name
		if name == "help" || name == "help-long" || name == "help-man" {
			return nil
		}
	}
	if alertmanagerURL == nil {
		kingpin.Fatalf("required flag --alertmanager.url not provided")
	}
	return nil
}

const (
	defaultAmHost      = "localhost"
	defaultAmPort      = "9093"
	defaultAmApiv2path = "/api/v2"
)

// NewAlertmanagerClient initializes an alertmanager client with the given URL
func NewAlertmanagerClient(amURL *url.URL) *client.Alertmanager {
	address := defaultAmHost + ":" + defaultAmPort
	schemes := []string{"http"}

	if amURL.Host != "" {
		address = amURL.Host // URL documents host as host or host:port
	}
	if amURL.Scheme != "" {
		schemes = []string{amURL.Scheme}
	}

	cr := clientruntime.New(address, path.Join(amURL.Path, defaultAmApiv2path), schemes)

	if tlsInsecureSkipVerify {
		transport := http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
		cr.Transport = &transport
	}

	if amURL.User != nil {
		password, _ := amURL.User.Password()
		cr.DefaultAuthentication = clientruntime.BasicAuth(amURL.User.Username(), password)
	}

	return client.New(cr, strfmt.Default)
}

// Execute is the main function for the amtool command
func Execute() {
	var (
		app = kingpin.New("amtool", helpRoot).UsageWriter(os.Stdout)
	)

	format.InitFormatFlags(app)

	app.Flag("verbose", "Verbose running information").Short('v').BoolVar(&verbose)
	app.Flag("alertmanager.url", "Alertmanager to talk to").URLVar(&alertmanagerURL)
	app.Flag("output", "Output formatter (simple, extended, json)").Short('o').Default("simple").EnumVar(&output, "simple", "extended", "json")
	app.Flag("timeout", "Timeout for the executed command").Default("30s").DurationVar(&timeout)
	app.Flag("tls.insecure.skip.verify", "Skip TLS certificate verification").BoolVar(&tlsInsecureSkipVerify)

	app.Version(version.Print("amtool"))
	app.GetFlag("help").Short('h')
	app.UsageTemplate(kingpin.CompactUsageTemplate)

	resolver, err := config.NewResolver(configFiles, legacyFlags)
	if err != nil {
		kingpin.Fatalf("could not load config file: %v\n", err)
	}

	configureAlertCmd(app)
	configureSilenceCmd(app)
	configureCheckConfigCmd(app)
	configureClusterCmd(app)
	configureConfigCmd(app)
	configureTemplateCmd(app)

	err = resolver.Bind(app, os.Args[1:])
	if err != nil {
		kingpin.Fatalf("%v\n", err)
	}

	_, err = app.Parse(os.Args[1:])
	if err != nil {
		kingpin.Fatalf("%v\n", err)
	}
}

const (
	helpRoot = `View and modify the current Alertmanager state.

Config File:
The alertmanager tool will read a config file in YAML format from one of two
default config locations: $HOME/.config/amtool/config.yml or
/etc/amtool/config.yml

All flags can be given in the config file, but the following are the suited for
static configuration:

	alertmanager.url
		Set a default alertmanager url for each request

	author
		Set a default author value for new silences. If this argument is not
		specified then the username will be used

	require-comment
		Bool, whether to require a comment on silence creation. Defaults to true

	output
		Set a default output type. Options are (simple, extended, json)

	date.format
		Sets the output format for dates. Defaults to "2006-01-02 15:04:05 MST"

	tls.insecure.skip.verify
		Skips TLS certificate verification for all HTTPS requests.
		Defaults to false.
`
)
<|endoftext|>


<commit_before>// Package rpcutil provides various methods for working with gorilla's JSON RPC
// 2 interface (http://www.gorillatoolkit.org/pkg/rpc/v2/json2)
package rpcutil

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"

	"gopkg.in/validator.v2"

	"github.com/gorilla/rpc/v2"
	"github.com/gorilla/rpc/v2/json2"
	"github.com/levenlabs/go-llog"
)

// RequestKV returns a basic KV for passing into llog, filled with entries
// related to the passed in http.Request
func RequestKV(r *http.Request) llog.KV {
	return llog.KV{
		"ip": RequestIP(r),
	}
}

// we don't ever really pass this into encoding/json, so having it implement
// json.Marshaler isn't really necessary, but it's helpful to think of it in
// this way
type jsonInliner struct {
	orig  interface{}
	extra map[string]interface{}
}

func (j jsonInliner) MarshalJSON() ([]byte, error) {
	bOrig, err := json.Marshal(j.orig)
	if err != nil {
		return nil, err
	}
	if len(bOrig) < 2 || bOrig[len(bOrig)-1] != '}' {
		return nil, errors.New("jsonInliner original value not an object")
	}
	if len(j.extra) == 0 {
		return bOrig, nil
	}

	bExtra, err := json.Marshal(j.extra)
	if err != nil {
		return nil, err
	}

	bOrig = bOrig[:len(bOrig)-1]
	bOrig = append(bOrig, ',')
	bOrig = append(bOrig, bExtra[1:]...)
	return bOrig, nil
}

// LLCodec wraps around gorilla's json2.Codec, adding logging to all requests
type LLCodec struct {
	c rpc.Codec

	// If true any errors which are not user caused (error code < 1) will not
	// actually be returned to the client, only a generic error message in their
	// place
	HideServerErrors bool

	// If true the gopkg.in/validator.v2 package will be used to automatically
	// validate inputs to calls
	ValidateInput bool

	// If set, once a non-error response is returned by an rpc endpoint this
	// will be called and the result (if non-nil) will be inlined with the
	// original response. The original response must encode to a json object for
	// this to work.
	//
	// For example, if the original response encodes to `{"success":true}`, and
	// ResponseInliner returns `{"currentTime":123456}`, the final response sent
	// to the client will be `{"success":true,"currentTime":123456}`
	ResponseInliner func(*http.Request) map[string]interface{}
}

// NewLLCodec returns an LLCodec, which is an implementation of rpc.Codec around
// json2.Codec. All public fields on LLCodec can be modified up intil passing
// this into rpc.RegisterCodec
func NewLLCodec() LLCodec {
	return LLCodec{c: json2.NewCodec()}
}

// NewRequest implements the NewRequest method for the rpc.Codec interface
func (c LLCodec) NewRequest(r *http.Request) rpc.CodecRequest {
	return llCodecRequest{
		c:            &c,
		CodecRequest: c.c.NewRequest(r),
		r:            r,
		kv:           RequestKV(r),
	}
}

type llCodecRequest struct {
	c *LLCodec
	rpc.CodecRequest
	r  *http.Request
	kv llog.KV
}

func (cr llCodecRequest) ReadRequest(args interface{}) error {
	// After calling the underlying ReadRequest the args will be filled in
	if err := cr.CodecRequest.ReadRequest(args); err != nil {
		// err will already be a json2.Error in this specific case, we don't
		// have to wrap it again
		return err
	}

	cr.kv["method"], _ = cr.CodecRequest.Method()
	var fn llog.LogFunc
	if llog.GetLevel() == llog.DebugLevel {
		cr.kv["args"] = fmt.Sprintf("%+v", args)
		fn = llog.Debug
	} else {
		fn = llog.Info
	}

	if cr.c.ValidateInput {
		if err := validator.Validate(args); err != nil {
			return &json2.Error{
				Code:    json2.E_BAD_PARAMS,
				Message: err.Error(),
			}
		}
	}

	fn("jsonrpc incoming request", cr.kv)

	return nil
}

func (cr llCodecRequest) maybeInlineExtra(r interface{}) (interface{}, error) {
	if cr.c.ResponseInliner == nil {
		return r, nil
	}
	extra := cr.c.ResponseInliner(cr.r)
	if extra == nil {
		return r, nil
	}

	j := jsonInliner{orig: r, extra: extra}
	b, err := j.MarshalJSON()
	if err != nil {
		return nil, err
	}
	jr := json.RawMessage(b)
	return &jr, nil
}

func (cr llCodecRequest) WriteResponse(w http.ResponseWriter, r interface{}) {
	if llog.GetLevel() == llog.DebugLevel {
		cr.kv["response"] = fmt.Sprintf("%+v", r)
		llog.Debug("jsonrpc responding", cr.kv)
	}

	newR, err := cr.maybeInlineExtra(r)
	if err != nil {
		cr.kv["err"] = err
		cr.kv["orig"], _ = json.Marshal(r)
		llog.Error("jsonrpc could not inline extra", cr.kv)
	} else {
		r = newR
	}

	cr.CodecRequest.WriteResponse(w, r)
}

func (cr llCodecRequest) WriteError(w http.ResponseWriter, status int, err error) {
	// status is ignored by gorilla

	cr.kv["err"] = err

	jsonErr, ok := err.(*json2.Error)
	if !ok {
		jsonErr = &json2.Error{
			Code:    json2.E_SERVER,
			Message: fmt.Sprintf("unexpected internal server error: %s", err),
		}
	}
	if kv, ok := jsonErr.Data.(llog.KV); ok {
		for k, v := range kv {
			cr.kv[k] = v
		}
	}

	// The only predefined error that is considered a server error really is
	// E_SERVER, all the others which are less than it are basically client
	// errors. So all within this range are considered internal server errors,
	// and need to be possibly hidden and definitely output as errors
	if jsonErr.Code < 0 && jsonErr.Code >= json2.E_SERVER {
		if cr.c.HideServerErrors {
			jsonErr = &json2.Error{
				Code:    json2.E_SERVER,
				Message: "internal server error",
			}
		}
		llog.Error("jsonrpc internal server error", cr.kv)
	} else {
		llog.Warn("jsonrpc client error", cr.kv)
	}

	cr.CodecRequest.WriteError(w, status, jsonErr)
}
<commit_msg>allow for excluding certain endpoint from being logged by LLCodec<commit_after>// Package rpcutil provides various methods for working with gorilla's JSON RPC
// 2 interface (http://www.gorillatoolkit.org/pkg/rpc/v2/json2)
package rpcutil

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"

	"gopkg.in/validator.v2"

	"github.com/gorilla/rpc/v2"
	"github.com/gorilla/rpc/v2/json2"
	"github.com/levenlabs/go-llog"
)

// RequestKV returns a basic KV for passing into llog, filled with entries
// related to the passed in http.Request
func RequestKV(r *http.Request) llog.KV {
	return llog.KV{
		"ip": RequestIP(r),
	}
}

// we don't ever really pass this into encoding/json, so having it implement
// json.Marshaler isn't really necessary, but it's helpful to think of it in
// this way
type jsonInliner struct {
	orig  interface{}
	extra map[string]interface{}
}

func (j jsonInliner) MarshalJSON() ([]byte, error) {
	bOrig, err := json.Marshal(j.orig)
	if err != nil {
		return nil, err
	}
	if len(bOrig) < 2 || bOrig[len(bOrig)-1] != '}' {
		return nil, errors.New("jsonInliner original value not an object")
	}
	if len(j.extra) == 0 {
		return bOrig, nil
	}

	bExtra, err := json.Marshal(j.extra)
	if err != nil {
		return nil, err
	}

	bOrig = bOrig[:len(bOrig)-1]
	bOrig = append(bOrig, ',')
	bOrig = append(bOrig, bExtra[1:]...)
	return bOrig, nil
}

// LLCodec wraps around gorilla's json2.Codec, adding logging to all requests
type LLCodec struct {
	c rpc.Codec

	// If true any errors which are not user caused (error code < 1) will not
	// actually be returned to the client, only a generic error message in their
	// place
	HideServerErrors bool

	// If true the gopkg.in/validator.v2 package will be used to automatically
	// validate inputs to calls
	ValidateInput bool

	// If set, once a non-error response is returned by an rpc endpoint this
	// will be called and the result (if non-nil) will be inlined with the
	// original response. The original response must encode to a json object for
	// this to work.
	//
	// For example, if the original response encodes to `{"success":true}`, and
	// ResponseInliner returns `{"currentTime":123456}`, the final response sent
	// to the client will be `{"success":true,"currentTime":123456}`
	ResponseInliner func(*http.Request) map[string]interface{}

	// All endpoints (fullname, i.e. "Service.Method") set as keys in this map
	// will not have an INFO log printed out when they are hit
	ExcludeRequestLog map[string]bool
}

// NewLLCodec returns an LLCodec, which is an implementation of rpc.Codec around
// json2.Codec. All public fields on LLCodec can be modified up intil passing
// this into rpc.RegisterCodec
func NewLLCodec() LLCodec {
	return LLCodec{c: json2.NewCodec(), ExcludeRequestLog: map[string]bool{}}
}

// NewRequest implements the NewRequest method for the rpc.Codec interface
func (c LLCodec) NewRequest(r *http.Request) rpc.CodecRequest {
	return llCodecRequest{
		c:            &c,
		CodecRequest: c.c.NewRequest(r),
		r:            r,
		kv:           RequestKV(r),
	}
}

type llCodecRequest struct {
	c *LLCodec
	rpc.CodecRequest
	r  *http.Request
	kv llog.KV
}

func (cr llCodecRequest) ReadRequest(args interface{}) error {
	// After calling the underlying ReadRequest the args will be filled in
	if err := cr.CodecRequest.ReadRequest(args); err != nil {
		// err will already be a json2.Error in this specific case, we don't
		// have to wrap it again
		return err
	}

	method, _ := cr.CodecRequest.Method()
	cr.kv["method"] = method
	var fn llog.LogFunc
	if cr.c.ExcludeRequestLog[method] {
		// don't log anything
	} else if llog.GetLevel() == llog.DebugLevel {
		cr.kv["args"] = fmt.Sprintf("%+v", args)
		fn = llog.Debug
	} else {
		fn = llog.Info
	}

	if cr.c.ValidateInput {
		if err := validator.Validate(args); err != nil {
			return &json2.Error{
				Code:    json2.E_BAD_PARAMS,
				Message: err.Error(),
			}
		}
	}

	if fn != nil {
		fn("jsonrpc incoming request", cr.kv)
	}

	return nil
}

func (cr llCodecRequest) maybeInlineExtra(r interface{}) (interface{}, error) {
	if cr.c.ResponseInliner == nil {
		return r, nil
	}
	extra := cr.c.ResponseInliner(cr.r)
	if extra == nil {
		return r, nil
	}

	j := jsonInliner{orig: r, extra: extra}
	b, err := j.MarshalJSON()
	if err != nil {
		return nil, err
	}
	jr := json.RawMessage(b)
	return &jr, nil
}

func (cr llCodecRequest) WriteResponse(w http.ResponseWriter, r interface{}) {
	if llog.GetLevel() == llog.DebugLevel {
		cr.kv["response"] = fmt.Sprintf("%+v", r)
		llog.Debug("jsonrpc responding", cr.kv)
	}

	newR, err := cr.maybeInlineExtra(r)
	if err != nil {
		cr.kv["err"] = err
		cr.kv["orig"], _ = json.Marshal(r)
		llog.Error("jsonrpc could not inline extra", cr.kv)
	} else {
		r = newR
	}

	cr.CodecRequest.WriteResponse(w, r)
}

func (cr llCodecRequest) WriteError(w http.ResponseWriter, status int, err error) {
	// status is ignored by gorilla

	cr.kv["err"] = err

	jsonErr, ok := err.(*json2.Error)
	if !ok {
		jsonErr = &json2.Error{
			Code:    json2.E_SERVER,
			Message: fmt.Sprintf("unexpected internal server error: %s", err),
		}
	}
	if kv, ok := jsonErr.Data.(llog.KV); ok {
		for k, v := range kv {
			cr.kv[k] = v
		}
	}

	// The only predefined error that is considered a server error really is
	// E_SERVER, all the others which are less than it are basically client
	// errors. So all within this range are considered internal server errors,
	// and need to be possibly hidden and definitely output as errors
	if jsonErr.Code < 0 && jsonErr.Code >= json2.E_SERVER {
		if cr.c.HideServerErrors {
			jsonErr = &json2.Error{
				Code:    json2.E_SERVER,
				Message: "internal server error",
			}
		}
		llog.Error("jsonrpc internal server error", cr.kv)
	} else {
		llog.Warn("jsonrpc client error", cr.kv)
	}

	cr.CodecRequest.WriteError(w, status, jsonErr)
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/jakubknejzlik/kontena-git-cli/kontena"
	"github.com/jakubknejzlik/kontena-git-cli/model"
	"github.com/jakubknejzlik/kontena-git-cli/utils"

	"github.com/urfave/cli"
)

// GridCommand ...
func GridCommand() cli.Command {
	return cli.Command{
		Name: "grid",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:   "grid",
				EnvVar: "GRID",
				Usage:  "grid used for installing",
			},
		},
		Subcommands: []cli.Command{
			installCommand(),
		},
	}
}

func installCommand() cli.Command {
	return cli.Command{
		Name: "install",
		Action: func(c *cli.Context) error {
			client := kontena.Client{}

			if err := client.EnsureMasterLogin(); err != nil {
				return err
			}

			grid := c.Parent().String("grid")
			if client.CurrentGrid().Name == "" || grid != "" {
				if err := client.GridUse(grid); err != nil {
					return err
				}
			}

			if err := installCoreCommand().Run(c); err != nil {
				return err
			}

			if err := installRegistriesCommand().Run(c); err != nil {
				return err
			}

			if err := pruneStacksCommand().Run(c); err != nil {
				return err
			}

			if err := installStacksCommand().Run(c); err != nil {
				return err
			}

			return nil
		},
		Subcommands: []cli.Command{
			installCoreCommand(),
			installRegistriesCommand(),
			pruneStacksCommand(),
			installStacksCommand(),
		},
	}
}

func installRegistriesCommand() cli.Command {
	return cli.Command{
		Name: "registries",
		Action: func(c *cli.Context) error {
			client := kontena.Client{}

			currentRegistries, err := client.CurrentRegistries()
			if err != nil {
				return cli.NewExitError(err, 1)
			}
			for _, regName := range currentRegistries {
				if client.RegistryExists(regName) {
					client.RegistryRemove(regName)
				}
			}

			registries, err := model.RegistriesLoad("registries.yml")
			if err != nil {
				return cli.NewExitError(err, 1)
			}

			for _, registry := range registries {
				if !client.RegistryExists(registry.Name) {
					if err := client.RegistryAdd(registry); err != nil {
						return cli.NewExitError(err, 1)
					}
				}
			}

			return nil
		},
	}
}

func installCoreCommand() cli.Command {
	return cli.Command{
		Name: "core",
		Action: func(c *cli.Context) error {
			client := kontena.Client{}

			dc, err := model.KontenaLoad("kontena.yml")
			if err != nil {
				return cli.NewExitError(err, 1)
			}

			if err := client.StackInstallOrUpgrade(dc); err != nil {
				return cli.NewExitError(err, 1)
			}

			return nil
		},
	}
}

func pruneStacksCommand() cli.Command {
	return cli.Command{
		Name: "prune",
		Action: func(c *cli.Context) error {
			client := kontena.Client{}

			stacks, err := client.StackList()
			if err != nil {
				return err
			}

			for _, stack := range stacks {
				if stack == "core" {
					continue
				}
				if _, err := os.Stat(fmt.Sprintf("./stacks/%s", stack)); os.IsNotExist(err) {
					if err := client.StackRemove(stack); err != nil {
						return err
					}
				}
			}
			return nil
		},
	}
}

func installStacksCommand() cli.Command {
	return cli.Command{
		Name: "stacks",
		Action: func(c *cli.Context) error {
			client := kontena.Client{}

			stacks, _ := ioutil.ReadDir("./stacks")
			for _, stack := range stacks {
				stackName := stack.Name()
				if err := client.SecretsImport(stackName, fmt.Sprintf("./stacks/%s/secrets.yml", stackName)); err != nil {
					log.Println(err)
					return err
				}
				if !client.StackExists(stackName) {
					utils.Log("installing stack", stackName)
					dc := defaultStack(stackName)
					if err := client.StackInstall(dc); err != nil {
						return cli.NewExitError(err, 1)
					}
				} else {
					utils.Log("deploying stack", stackName)
					if err := client.StackDeploy(stackName); err != nil {
						return cli.NewExitError(err, 1)
					}
				}
				time.Sleep(time.Second * 3)
			}
			return nil
		},
	}
}

func defaultStack(name string) model.Kontena {
	stackConfigPath := fmt.Sprintf("./stacks/%s/kontena.yml", name)
	if _, err := os.Stat(stackConfigPath); err == nil {
		stack, err := model.KontenaLoad(stackConfigPath)
		if err == nil {
			return stack
		}
	}
	return model.Kontena{
		Stack:   name,
		Version: "0.0.1",
		Services: map[string]model.KontenaService{
			"web": model.KontenaService{
				Image: "ksdn117/test-page",
				Links: []string{
					"core/internet_lb",
				},
				Secrets: []model.KontenaSecret{
					model.KontenaSecret{
						Secret: "VIRTUAL_HOSTS",
						Name:   "KONTENA_LB_VIRTUAL_HOSTS",
						Type:   "env",
					},
				},
			},
		},
	}
}
<commit_msg>Fix stack upgrading<commit_after>package cmd

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/jakubknejzlik/kontena-git-cli/kontena"
	"github.com/jakubknejzlik/kontena-git-cli/model"
	"github.com/jakubknejzlik/kontena-git-cli/utils"

	"github.com/urfave/cli"
)

// GridCommand ...
func GridCommand() cli.Command {
	return cli.Command{
		Name: "grid",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:   "grid",
				EnvVar: "GRID",
				Usage:  "grid used for installing",
			},
		},
		Subcommands: []cli.Command{
			installCommand(),
		},
	}
}

func installCommand() cli.Command {
	return cli.Command{
		Name: "install",
		Action: func(c *cli.Context) error {
			client := kontena.Client{}

			if err := client.EnsureMasterLogin(); err != nil {
				return err
			}

			grid := c.Parent().String("grid")
			if client.CurrentGrid().Name == "" || grid != "" {
				if err := client.GridUse(grid); err != nil {
					return err
				}
			}

			if err := installCoreCommand().Run(c); err != nil {
				return err
			}

			if err := installRegistriesCommand().Run(c); err != nil {
				return err
			}

			if err := pruneStacksCommand().Run(c); err != nil {
				return err
			}

			if err := installStacksCommand().Run(c); err != nil {
				return err
			}

			return nil
		},
		Subcommands: []cli.Command{
			installCoreCommand(),
			installRegistriesCommand(),
			pruneStacksCommand(),
			installStacksCommand(),
		},
	}
}

func installRegistriesCommand() cli.Command {
	return cli.Command{
		Name: "registries",
		Action: func(c *cli.Context) error {
			client := kontena.Client{}

			currentRegistries, err := client.CurrentRegistries()
			if err != nil {
				return cli.NewExitError(err, 1)
			}
			for _, regName := range currentRegistries {
				if client.RegistryExists(regName) {
					client.RegistryRemove(regName)
				}
			}

			registries, err := model.RegistriesLoad("registries.yml")
			if err != nil {
				return cli.NewExitError(err, 1)
			}

			for _, registry := range registries {
				if !client.RegistryExists(registry.Name) {
					if err := client.RegistryAdd(registry); err != nil {
						return cli.NewExitError(err, 1)
					}
				}
			}

			return nil
		},
	}
}

func installCoreCommand() cli.Command {
	return cli.Command{
		Name: "core",
		Action: func(c *cli.Context) error {
			client := kontena.Client{}

			dc, err := model.KontenaLoad("kontena.yml")
			if err != nil {
				return cli.NewExitError(err, 1)
			}

			if err := client.StackInstallOrUpgrade(dc); err != nil {
				return cli.NewExitError(err, 1)
			}

			return nil
		},
	}
}

func pruneStacksCommand() cli.Command {
	return cli.Command{
		Name: "prune",
		Action: func(c *cli.Context) error {
			client := kontena.Client{}

			stacks, err := client.StackList()
			if err != nil {
				return err
			}

			for _, stack := range stacks {
				if stack == "core" {
					continue
				}
				if _, err := os.Stat(fmt.Sprintf("./stacks/%s", stack)); os.IsNotExist(err) {
					if err := client.StackRemove(stack); err != nil {
						return err
					}
				}
			}
			return nil
		},
	}
}

func installStacksCommand() cli.Command {
	return cli.Command{
		Name: "stacks",
		Action: func(c *cli.Context) error {
			client := kontena.Client{}

			stacks, _ := ioutil.ReadDir("./stacks")
			for _, stack := range stacks {
				stackName := stack.Name()
				if err := client.SecretsImport(stackName, fmt.Sprintf("./stacks/%s/secrets.yml", stackName)); err != nil {
					log.Println(err)
					return err
				}
				if !client.StackExists(stackName) {
					utils.Log("installing stack", stackName)
					dc := getDefaultStack(stackName)
					if err := client.StackInstall(dc); err != nil {
						return cli.NewExitError(err, 1)
					}
				} else {
					if stack, err := getStack(stackName); err == nil {
						utils.Log("upgrading stack", stackName)
						if err := client.StackUpgrade(stack); err != nil {
							return cli.NewExitError(err, 1)
						}
					}
					utils.Log("deploying stack", stackName)
					if err := client.StackDeploy(stackName); err != nil {
						return cli.NewExitError(err, 1)
					}
				}
				time.Sleep(time.Second * 3)
			}
			return nil
		},
	}
}

func getStack(name string) (model.Kontena, error) {
	var k model.Kontena
	stackConfigPath := fmt.Sprintf("./stacks/%s/kontena.yml", name)
	if _, err := os.Stat(stackConfigPath); err != nil {
		return k, err
	}
	return model.KontenaLoad(stackConfigPath)
}

func getDefaultStack(name string) model.Kontena {
	if stack, err := getStack(name); err == nil {
		return stack
	}
	return model.Kontena{
		Stack:   name,
		Version: "0.0.1",
		Services: map[string]model.KontenaService{
			"web": model.KontenaService{
				Image: "ksdn117/test-page",
				Links: []string{
					"core/internet_lb",
				},
				Secrets: []model.KontenaSecret{
					model.KontenaSecret{
						Secret: "VIRTUAL_HOSTS",
						Name:   "KONTENA_LB_VIRTUAL_HOSTS",
						Type:   "env",
					},
				},
			},
		},
	}
}
<|endoftext|>