<commit_before>// vmwatcher is an enforcer that gets various metrics (currently only
// `NetworkOut` from cloudwatch api is implemented) and stops the vm if it's
// over the limit for that metric; in addition it also exposes a http endpoint
// for various workers (kloud for now) to check if user is overlimit before
// taking an action (ie starting a vm).
//
// The goal of this worker is to prevent users from abusing the system, not be
// a secondary storage for metrics data.
package main

import (
	"koding/artifact"
	"net/http"

	"github.com/robfig/cron"
)

var (
	WorkerName    = "vmwatcher"
	WorkerVersion = "0.0.1"

	NetworkOut              = "NetworkOut"
	NetworkOutLimit float64 = 7000 // 7GB/week

	// defines list of metrics, all queue/fetch/save operations
	// must iterate this list and not use metric directly
	metricsToSave = []Metric{
		&Cloudwatch{Name: NetworkOut, Limit: NetworkOutLimit},
	}
)

func main() {
	c := cron.New()

	// queue to get metrics at top of every hour; uses redis set to queue
	// the usernames so multiple workers don't queue the same usernames.
	// this needs to be done at top of hour, so running multiple workers
	// won't cause a problem.
	c.AddFunc("0 0 * * * *", func() {
		err := queueUsernamesForMetricGet()
		if err != nil {
			Log.Fatal(err.Error())
		}
	})

	// get and save metrics at 15th minute of every hour
	c.AddFunc("0 15 * * * *", func() {
		err := getAndSaveQueueMachineMetrics()
		if err != nil {
			Log.Fatal(err.Error())
		}
	})

	// stop machines overlimit at 20th & 40th of every hour; there's no reason
	// for running it at a certain point except not having overlap in logs
	c.AddFunc("0 20,40 * * * *", func() {
		err := stopMachinesOverLimit()
		if err != nil {
			Log.Fatal(err.Error())
		}
	})

	c.Start()

	// expose api for workers like kloud to check if users is over limit
	http.HandleFunc("/", checkerHTTP)

	http.HandleFunc("/version", artifact.VersionHandler())
	http.HandleFunc("/healthCheck", artifact.HealthCheckHandler(WorkerName))

	Log.Info("Listening on port: %s", port)

	http.ListenAndServe(":"+port, nil)
}
<commit_msg>vmwatcher: don't approximate 7gb, use real number<commit_after>// vmwatcher is an enforcer that gets various metrics (currently only
// `NetworkOut` from cloudwatch api is implemented) and stops the vm if it's
// over the limit for that metric; in addition it also exposes a http endpoint
// for various workers (kloud for now) to check if user is overlimit before
// taking an action (ie starting a vm).
//
// The goal of this worker is to prevent users from abusing the system, not be
// a secondary storage for metrics data.
package main

import (
	"koding/artifact"
	"net/http"

	"github.com/robfig/cron"
)

var (
	WorkerName    = "vmwatcher"
	WorkerVersion = "0.0.1"

	NetworkOut              = "NetworkOut"
	NetworkOutLimit float64 = 7 * 1024 // 7GB/week

	// defines list of metrics, all queue/fetch/save operations
	// must iterate this list and not use metric directly
	metricsToSave = []Metric{
		&Cloudwatch{Name: NetworkOut, Limit: NetworkOutLimit},
	}
)

func main() {
	c := cron.New()

	// queue to get metrics at top of every hour; uses redis set to queue
	// the usernames so multiple workers don't queue the same usernames.
	// this needs to be done at top of hour, so running multiple workers
	// won't cause a problem.
	c.AddFunc("0 0 * * * *", func() {
		err := queueUsernamesForMetricGet()
		if err != nil {
			Log.Fatal(err.Error())
		}
	})

	// get and save metrics at 15th minute of every hour
	c.AddFunc("0 15 * * * *", func() {
		err := getAndSaveQueueMachineMetrics()
		if err != nil {
			Log.Fatal(err.Error())
		}
	})

	// stop machines overlimit at 20th & 40th of every hour; there's no reason
	// for running it at a certain point except not having overlap in logs
	c.AddFunc("0 20,40 * * * *", func() {
		err := stopMachinesOverLimit()
		if err != nil {
			Log.Fatal(err.Error())
		}
	})

	c.Start()

	// expose api for workers like kloud to check if users is over limit
	http.HandleFunc("/", checkerHTTP)

	http.HandleFunc("/version", artifact.VersionHandler())
	http.HandleFunc("/healthCheck", artifact.HealthCheckHandler(WorkerName))

	Log.Info("Listening on port: %s", port)

	http.ListenAndServe(":"+port, nil)
}
<|endoftext|>


<commit_before>package rbt

import (
	"fmt"
	"math/rand"
	"testing"

	"github.com/stretchr/testify/assert"
)

func less(x1, x2 interface{}) bool {
	i1, i2 := x1.(int), x2.(int)
	return i1 < i2
}

func TestNode(t *testing.T) {
	k := 3

	n := (*node)(nil)
	t.Log(n)
	n = n.insert(k, less)
	t.Log(n)

	kk, ok := n.search(k, less)
	assert.True(t, ok)
	assert.Equal(t, kk, k)
}

func TestNodeBalance(t *testing.T) {
	ks := []int{1, 2, 3, 4, 5, 6, 7, 8}
	n := (*node)(nil)

	for _, k := range ks {
		n.dump()
		n = n.insert(k, less)
	}

	n.dump()
}

func TestNodeTakeMax(t *testing.T) {
	ks := []int{1, 2, 3, 4, 5, 6, 7, 8}
	n := (*node)(nil)

	for _, k1 := range ks {
		n = n.insert(k1, less)
		n.dump()

		k2, m, _ := n.takeMax()
		assert.Equal(t, k1, k2.(int))
		m.dump()
	}
}

func TestNodeRemove(t *testing.T) {
	ks := []int{1, 2, 3, 4, 5, 6, 7, 8}
	n := (*node)(nil)

	for _, k := range ks {
		n = n.insert(k, less)
	}

	n.dump()

	for _, k := range ks {
		n = n.remove(k, less)
		n.dump()
	}
}

const (
	MaxIters = 2000
	MaxKey   = MaxIters / 2
)

func generateKey() int {
	return rand.Int() % MaxKey
}

func TestNodeInsertRandomly(t *testing.T) {
	n := (*node)(nil)

	for i := 0; i < MaxIters; i++ {
		k := generateKey()
		old := n

		n = n.insert(k, less)

		n.rank() // check ranks

		if !n.checkColors() {
			failWithDump(t, true, k, old, n)
		}
	}
}

func TestNodeRemoveRandomly(t *testing.T) {
	n := (*node)(nil)

	for i := 0; i < MaxIters; i++ {
		k := generateKey()
		old := n

		var insert bool
		n, insert = n.insertOrRemove(t, k)

		n.dump()
		n.rank() // check ranks

		if !n.checkColors() {
			failWithDump(t, insert, k, old, n)
		}
	}
}

func TestInsertRemovePersistency(t *testing.T) {
	n := (*node)(nil)

	for i := 0; i < MaxIters; i++ {
		k := generateKey()
		old := n
		oldCopy := n.deepCopy()

		var insert bool
		n, insert = n.insertOrRemove(t, k)

		n.dump()
		n.rank() // check ranks

		if !(old.equal(oldCopy) && n.checkColors()) {
			failWithDump(t, insert, k, old, n)
		}
	}
}

func (n *node) insertOrRemove(t *testing.T, x interface{}) (*node, bool) {
	insert := rand.Int()%2 == 0

	if insert {
		n = n.insert(x, less)
	} else {
		n = n.remove(x, less)
	}

	_, ok := n.search(x, less)
	assert.True(t, insert && ok || !insert && !ok)

	return n, insert
}

func failWithDump(t *testing.T, insert bool, k int, old, new *node) {
	if insert {
		fmt.Println("INSERT")
	} else {
		fmt.Println("REMOVE")
	}

	fmt.Println("KEY:", k)
	fmt.Println("OLD:")
	old.dump()
	fmt.Println("NEW:")
	new.dump()
	t.FailNow()
}

func TestNodeEqual(t *testing.T) {
	n0 := (*node)(nil)
	n1 := n0.insert(1, less)
	n2 := n0.insert(2, less)
	n3 := n1.insert(2, less)
	n4 := n3.insert(3, less)

	for _, test := range []struct {
		n1, n2 *node
		equal  bool
	}{
		{n0, n0, true},
		{n1, n1, true},
		{n2, n2, true},
		{n3, n3, true},
		{n4, n4, true},
		{n0, n1, false},
		{n0, n2, false},
		{n0, n3, false},
		{n0, n4, false},
		{n1, n2, false},
		{n1, n3, false},
		{n2, n3, false},
		{n2, n4, false},
		{n3, n4, false},
	} {
		assert.Equal(t, test.equal, test.n1.equal(test.n2))
	}
}

func TestNodeRankError(t *testing.T) {
	defer func() {
		if r := recover(); r == nil {
			t.Fail()
		}
	}()

	n := (*node)(nil).insert(0, less).insert(1, less).insert(2, less)
	n.dump()
	n.left.color = red
	n.dump()

	n.rank()
}
<commit_msg>Test node.checkColors()<commit_after>package rbt

import (
	"fmt"
	"math/rand"
	"testing"

	"github.com/stretchr/testify/assert"
)

func less(x1, x2 interface{}) bool {
	i1, i2 := x1.(int), x2.(int)
	return i1 < i2
}

func TestNode(t *testing.T) {
	k := 3

	n := (*node)(nil)
	t.Log(n)
	n = n.insert(k, less)
	t.Log(n)

	kk, ok := n.search(k, less)
	assert.True(t, ok)
	assert.Equal(t, kk, k)
}

func TestNodeBalance(t *testing.T) {
	ks := []int{1, 2, 3, 4, 5, 6, 7, 8}
	n := (*node)(nil)

	for _, k := range ks {
		n.dump()
		n = n.insert(k, less)
	}

	n.dump()
}

func TestNodeTakeMax(t *testing.T) {
	ks := []int{1, 2, 3, 4, 5, 6, 7, 8}
	n := (*node)(nil)

	for _, k1 := range ks {
		n = n.insert(k1, less)
		n.dump()

		k2, m, _ := n.takeMax()
		assert.Equal(t, k1, k2.(int))
		m.dump()
	}
}

func TestNodeRemove(t *testing.T) {
	ks := []int{1, 2, 3, 4, 5, 6, 7, 8}
	n := (*node)(nil)

	for _, k := range ks {
		n = n.insert(k, less)
	}

	n.dump()

	for _, k := range ks {
		n = n.remove(k, less)
		n.dump()
	}
}

const (
	MaxIters = 2000
	MaxKey   = MaxIters / 2
)

func generateKey() int {
	return rand.Int() % MaxKey
}

func TestNodeInsertRandomly(t *testing.T) {
	n := (*node)(nil)

	for i := 0; i < MaxIters; i++ {
		k := generateKey()
		old := n

		n = n.insert(k, less)

		n.rank() // check ranks

		if !n.checkColors() {
			failWithDump(t, true, k, old, n)
		}
	}
}

func TestNodeRemoveRandomly(t *testing.T) {
	n := (*node)(nil)

	for i := 0; i < MaxIters; i++ {
		k := generateKey()
		old := n

		var insert bool
		n, insert = n.insertOrRemove(t, k)

		n.dump()
		n.rank() // check ranks

		if !n.checkColors() {
			failWithDump(t, insert, k, old, n)
		}
	}
}

func TestInsertRemovePersistency(t *testing.T) {
	n := (*node)(nil)

	for i := 0; i < MaxIters; i++ {
		k := generateKey()
		old := n
		oldCopy := n.deepCopy()

		var insert bool
		n, insert = n.insertOrRemove(t, k)

		n.dump()
		n.rank() // check ranks

		if !(old.equal(oldCopy) && n.checkColors()) {
			failWithDump(t, insert, k, old, n)
		}
	}
}

func (n *node) insertOrRemove(t *testing.T, x interface{}) (*node, bool) {
	insert := rand.Int()%2 == 0

	if insert {
		n = n.insert(x, less)
	} else {
		n = n.remove(x, less)
	}

	_, ok := n.search(x, less)
	assert.True(t, insert && ok || !insert && !ok)

	return n, insert
}

func failWithDump(t *testing.T, insert bool, k int, old, new *node) {
	if insert {
		fmt.Println("INSERT")
	} else {
		fmt.Println("REMOVE")
	}

	fmt.Println("KEY:", k)
	fmt.Println("OLD:")
	old.dump()
	fmt.Println("NEW:")
	new.dump()
	t.FailNow()
}

func TestNodeEqual(t *testing.T) {
	n0 := (*node)(nil)
	n1 := n0.insert(1, less)
	n2 := n0.insert(2, less)
	n3 := n1.insert(2, less)
	n4 := n3.insert(3, less)

	for _, test := range []struct {
		n1, n2 *node
		equal  bool
	}{
		{n0, n0, true},
		{n1, n1, true},
		{n2, n2, true},
		{n3, n3, true},
		{n4, n4, true},
		{n0, n1, false},
		{n0, n2, false},
		{n0, n3, false},
		{n0, n4, false},
		{n1, n2, false},
		{n1, n3, false},
		{n2, n3, false},
		{n2, n4, false},
		{n3, n4, false},
	} {
		assert.Equal(t, test.equal, test.n1.equal(test.n2))
	}
}

func TestNodeRankError(t *testing.T) {
	defer func() {
		if r := recover(); r == nil {
			t.Fail()
		}
	}()

	n := (*node)(nil).insert(0, less).insert(1, less).insert(2, less)
	n.dump()
	n.left.color = red
	n.dump()

	n.rank()
}

func TestNodeCheckColorsError(t *testing.T) {
	n := (*node)(nil).insert(0, less).insert(1, less).insert(2, less).insert(3, less)
	n.dump()
	n.right.color = red
	n.dump()
	assert.True(t, !n.checkColors())
}
<|endoftext|>


<commit_before>/*
Licensed to the Apache Software Foundation (ASF) Under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

package gremlingo

const (
	baseNamespace         = "org.apache.tinkerpop.gremlin.process.traversal.strategy."
	decorationNamespace   = baseNamespace + "decoration."
	finalizationNamespace = baseNamespace + "finalization."
	optimizationNamespace = baseNamespace + "optimization."
	verificationNamespace = baseNamespace + "verification."
)

func PartitionStrategy(partitionKey, writePartition, readPartitions, includeMetaProperties string) *traversalStrategy {
	config := make(map[string]string)
	if partitionKey != "" {
		config["partitionKey"] = partitionKey
	}
	if writePartition != "" {
		config["writePartition"] = writePartition
	}
	if readPartitions != "" {
		config["readPartitions"] = readPartitions
	}
	if includeMetaProperties != "" {
		config["includeMetaProperties"] = includeMetaProperties
	}
	return &traversalStrategy{name: decorationNamespace + "PartitionStrategy", configuration: config}
}

func ReadOnlyStrategy() *traversalStrategy {
	return &traversalStrategy{name: verificationNamespace + "ReadOnlyStrategy"}
}

func RemoteStrategy(connection DriverRemoteConnection) *traversalStrategy {
	a := func(g GraphTraversal) {
		result, err := g.getResults()
		if err != nil || result != nil {
			return
		}

		rs, err := connection.submitBytecode(g.bytecode)
		if err != nil {
			return
		}
		g.results = rs
	}

	return &traversalStrategy{name: "RemoteStrategy", apply: a}
}
<commit_msg>AN-1066 Optimization strategies definition<commit_after>/*
Licensed to the Apache Software Foundation (ASF) Under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

package gremlingo

const (
	baseNamespace         = "org.apache.tinkerpop.gremlin.process.traversal.strategy."
	decorationNamespace   = baseNamespace + "decoration."
	finalizationNamespace = baseNamespace + "finalization."
	optimizationNamespace = baseNamespace + "optimization."
	verificationNamespace = baseNamespace + "verification."
)

// Decoration strategies

func PartitionStrategy(partitionKey, writePartition, readPartitions, includeMetaProperties string) *traversalStrategy {
	config := make(map[string]string)
	if partitionKey != "" {
		config["partitionKey"] = partitionKey
	}
	if writePartition != "" {
		config["writePartition"] = writePartition
	}
	if readPartitions != "" {
		config["readPartitions"] = readPartitions
	}
	if includeMetaProperties != "" {
		config["includeMetaProperties"] = includeMetaProperties
	}
	return &traversalStrategy{name: decorationNamespace + "PartitionStrategy", configuration: config}
}

// Verification strategies

func ReadOnlyStrategy() *traversalStrategy {
	return &traversalStrategy{name: verificationNamespace + "ReadOnlyStrategy"}
}

// Optimization strategies

// AdjacentToIncidentStrategy looks for vertex- and value-emitting steps followed by a CountGlobalStep and replaces
// the pattern with an edge- or property-emitting step followed by a CountGlobalStep. Furthermore, if a vertex-
// or value-emitting step is the last step in a .has(traversal), .and(traversal, ...) or .or(traversal, ...)
// child traversal, it is replaced by an appropriate edge- or property-emitting step.
// Performing this replacement removes situations where the more expensive trip to an adjacent graph element (e.g.
// the vertex on the other side of an edge) can be satisfied by trips to incident graph elements (e.g. just the edge
// itself).
func AdjacentToIncidentStrategy() *traversalStrategy {
	return &traversalStrategy{name: optimizationNamespace + "AdjacentToIncidentStrategy"}
}

// ByModulatorOptimizationStrategy looks for standard traversals in by-modulators and replaces them with more
// optimized traversals (e.g. TokenTraversal) if possible.
func ByModulatorOptimizationStrategy() *traversalStrategy {
	return &traversalStrategy{name: optimizationNamespace + "ByModulatorOptimizationStrategy"}
}

// CountStrategy optimizes any occurrence of CountGlobalStep followed by an IsStep The idea is to limit
// the number of incoming elements in a way that it's enough for the IsStep to decide whether it evaluates
// true or false. If the traversal already contains a user supplied limit, the strategy won't modify it.
func CountStrategy() *traversalStrategy {
	return &traversalStrategy{name: optimizationNamespace + "CountStrategy"}
}

// EarlyLimitStrategy looks for RangeGlobalSteps that can be moved further left in the traversal and thus be applied
// earlier. It will also try to merge multiple RangeGlobalSteps into one.
// If the logical consequence of one or multiple RangeGlobalSteps is an empty result, the strategy will remove
// as many steps as possible and add a NoneStep instead.
func EarlyLimitStrategy() *traversalStrategy {
	return &traversalStrategy{name: optimizationNamespace + "EarlyLimitStrategy"}
}

// FilterRankingStrategy reorders filter- and order-steps according to their rank. Step ranks are defined within
// the strategy and indicate when it is reasonable for a step to move in front of another. It will also do its best to
// push step labels as far "right" as possible in order to keep traversers as small and bulkable as possible prior to
// the absolute need for path-labeling.
func FilterRankingStrategy() *traversalStrategy {
	return &traversalStrategy{name: optimizationNamespace + "FilterRankingStrategy"}
}

// IdentityRemovalStrategy looks for IdentityStep instances and removes them.
// If the identity step is labeled, its labels are added to the previous step.
// If the identity step is labeled and it's the first step in the traversal, it stays.
func IdentityRemovalStrategy() *traversalStrategy {
	return &traversalStrategy{name: optimizationNamespace + "IdentityRemovalStrategy"}
}

// IncidentToAdjacentStrategy looks for .outE().inV(), .inE().outV() and .bothE().otherV()
// and replaces these step sequences with .out(), .in() or .both() respectively.
// The strategy won't modify the traversal if:
//   the edge step is labeled
//   the traversal contains a path step
//   the traversal contains a lambda step
func IncidentToAdjacentStrategy() *traversalStrategy {
	return &traversalStrategy{name: optimizationNamespace + "IncidentToAdjacentStrategy"}
}

// InlineFilterStrategy analyzes filter-steps with child traversals that themselves are pure filters. If
// the child traversals are pure filters then the wrapping parent filter is not needed and thus, the children
// can be "inlined." Normalizing  pure filters with inlining reduces the number of variations of a filter that
// a graph provider may need to reason about when writing their own strategies. As a result, this strategy helps
// increase the likelihood that a provider's filtering optimization will succeed at re-writing the traversal.
func InlineFilterStrategy() *traversalStrategy {
	return &traversalStrategy{name: optimizationNamespace + "InlineFilterStrategy"}
}

// LazyBarrierStrategy is an OLTP-only strategy that automatically inserts a NoOpBarrierStep after every
// FlatMapStep if neither path-tracking nor partial path-tracking is required, and the next step is not the
// traversal's last step or a Barrier. NoOpBarrierSteps allow traversers to be bulked, thus this strategy
// is meant to reduce memory requirements and improve the overall query performance.
func LazyBarrierStrategy() *traversalStrategy {
	return &traversalStrategy{name: optimizationNamespace + "LazyBarrierStrategy"}
}

// MatchPredicateStrategy will fold any post-where() step that maintains a traversal constraint into
// match(). MatchStep is intelligent with traversal constraint applications and thus, can more
// efficiently use the constraint of WhereTraversalStep or WherePredicateStep.
func MatchPredicateStrategy() *traversalStrategy {
	return &traversalStrategy{name: optimizationNamespace + "MatchPredicateStrategy"}
}

// OrderLimitStrategy is an OLAP strategy that folds a RangeGlobalStep into a preceding
// OrderGlobalStep. This helps to eliminate traversers early in the traversal and can
// significantly reduce the amount of memory required by the OLAP execution engine.
func OrderLimitStrategy() *traversalStrategy {
	return &traversalStrategy{name: optimizationNamespace + "OrderLimitStrategy"}
}

// PathProcessStrategy  is an OLAP strategy that does its best to turn non-local children in where()
// and select() into local children by inlining components of the non-local child. In this way,
// PathProcessStrategy helps to ensure that more traversals meet the local child constraint imposed
// on OLAP traversals.
func PathProcessStrategy() *traversalStrategy {
	return &traversalStrategy{name: optimizationNamespace + "PathProcessStrategy"}
}

// PathRetractionStrategy TODO.
func PathRetractionStrategy() *traversalStrategy {
	return &traversalStrategy{name: optimizationNamespace + "PathRetractionStrategy"}
}

// ProductiveByStrategy takes an argument of by() and wraps it CoalesceStep so that the result is either
// the initial Traversal argument or null. In this way, the by() is always "productive". This strategy
// is an "optimization" but it is perhaps more of a "decoration", but it should follow
// ByModulatorOptimizationStrategy which features optimizations relevant to this one.
func ProductiveByStrategy() *traversalStrategy {
	return &traversalStrategy{name: optimizationNamespace + "ProductiveByStrategy"}
}

// RepeatUnrollStrategy is an OLTP-only strategy that unrolls any RepeatStep if it uses a constant
// number of loops (times(x)) and doesn't emit intermittent elements. If any of the following 3 steps appears
// within the repeat-traversal, the strategy will not be applied:
//   DedupGlobalStep
//   LoopsStep
//   LambdaHolder
func RepeatUnrollStrategy() *traversalStrategy {
	return &traversalStrategy{name: optimizationNamespace + "RepeatUnrollStrategy"}
}

func RemoteStrategy(connection DriverRemoteConnection) *traversalStrategy {
	a := func(g GraphTraversal) {
		result, err := g.getResults()
		if err != nil || result != nil {
			return
		}

		rs, err := connection.submitBytecode(g.bytecode)
		if err != nil {
			return
		}
		g.results = rs
	}

	return &traversalStrategy{name: "RemoteStrategy", apply: a}
}
<|endoftext|>


<commit_before>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package api holds common data types and public interface exposed by the indexer library.
// Refer to the doc
// https://docs.google.com/a/squareup.com/document/d/1k0Wgi2wnJPQoyDyReb9dyIqRrD8-v0u8hz37S282ii4/edit
// for the terminology.

package api

// SeriesType is a different aspect of data.
// For example, Blueflood may stores (min / max / average / count) during rollups,
// and these data are exposed via columns
type SeriesType string

// SampleMethod determines how the given time series should be sampled.
type SampleMethod int

const (
	// SamplingMax chooses the maximum value.
	SampleMax SampleMethod = iota + 1
	// SamplingMin chooses the minimum value.
	SampleMin
	// SamplingMean chooses the average value.
	SampleMean
)

func (sm SampleMethod) String() string {
	switch sm {
	case SampleMax:
		return "SampleMax"
	case SampleMin:
		return "SampleMin"
	case SampleMean:
		return "SampleMean"
	}

	return "unknown"
}
<commit_msg>Remove google doc link<commit_after>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

// SeriesType is a different aspect of data.
// For example, Blueflood may stores (min / max / average / count) during rollups,
// and these data are exposed via columns
type SeriesType string

// SampleMethod determines how the given time series should be sampled.
type SampleMethod int

const (
	// SamplingMax chooses the maximum value.
	SampleMax SampleMethod = iota + 1
	// SamplingMin chooses the minimum value.
	SampleMin
	// SamplingMean chooses the average value.
	SampleMean
)

func (sm SampleMethod) String() string {
	switch sm {
	case SampleMax:
		return "SampleMax"
	case SampleMin:
		return "SampleMin"
	case SampleMean:
		return "SampleMean"
	}

	return "unknown"
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"github.com/fatih/color"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

type Heading struct {
	Text           string
	RightJustified bool
	MinWidth       int

	// Calculated/working values
	width  int
	format string
	min    int
	max    int
}

type Cell struct {
	Text      string
	CellColor *color.Color
}

type Row []*Cell
type Rows []Row

type PrintConfig struct {
	NumSpaceSep   int
	UnderlineChar string
}

type PrintData struct {
	Headings []*Heading
	Rows     Rows
	Config   PrintConfig
}

////////////////////////////////////////////////////////////////////////////////

var defaultPrintConf = PrintConfig{
	NumSpaceSep:   3,
	UnderlineChar: "-",
}

// Get the number of columns available on the terminal. We'll try to squeeze
// the results into this width.
func numTTYColumns() int {
	cmd := exec.Command("stty", "size")
	cmd.Stdin = os.Stdin
	res, err := cmd.Output()
	if err == nil {
		tokens := strings.Split(strings.TrimSpace(string(res)), " ")
		if len(tokens) == 2 {
			cols, _ := strconv.Atoi(tokens[1])
			return cols
		}
	}

	return -1
}

// Get the length of the longest word in a line
func longestWordLen(line string) int {
	start, longest, pos := 0, 0, 0
	for pos <= len(line) {
		// Break char or EOL
		if pos == len(line) || line[pos] == ' ' {
			// (pos-1) - start + 1
			wordlen := pos - start
			start = pos + 1
			pos = start

			if wordlen > longest {
				longest = wordlen
			}
		} else {
			pos += 1
		}
	}
	return longest
}

// Calculate the final widths of each column.
func (pd *PrintData) calcWidths() {
	// First pass, get the max and min widths of each column. Best case scenerio,
	// we fit within the real estate we have. Worst case scenerio, the min width
	// is smaller than what we have to work with.

	// Start with the headings, and don't break them up
	for _, h := range pd.Headings {
		h.min = h.MinWidth
		if h.min < len(h.Text) {
			h.min = len(h.Text)
		}
		h.max = h.min
	}

	// Now, the rows. The min column width should be the shortest word we find.
	// The max width is the width of the longest cell.
	for _, row := range pd.Rows {
		for col, cell := range row {
			lw := longestWordLen(cell.Text)
			if pd.Headings[col].min < lw {
				pd.Headings[col].min = lw
			}
			if len(cell.Text) > pd.Headings[col].max {
				pd.Headings[col].max = len(cell.Text)
			}
		}
	}

	// Figure out how many columns we have to work with. But, at least give
	// us something to work with.
	remaining := numTTYColumns()
	if remaining < 80 {
		remaining = 80
	}

	// Deduct the column spacers
	remaining -= (len(pd.Headings) - 1) * pd.Config.NumSpaceSep

	// Start out by setting the widths to the min widths.
	// (OK if remaining goes negative)
	for _, h := range pd.Headings {
		h.width = h.min
		remaining -= h.width
	}

	// Divide up the remaining columns equally
	for remaining > 0 {
		didOne := false
		for _, h := range pd.Headings {
			if h.width < h.max && remaining > 0 {
				remaining -= 1
				h.width += 1
				didOne = true
			}
		}
		if !didOne {
			break
		}
	}
}

// Split a single-line cell into (possibly) mutiple cells, with one line per cell.
func (cell *Cell) split(width int) []*Cell {
	// We currently use the simple (and common) greedy algorithm for
	// filling each row.
	// TODO: Change this to Knuth's algorithm for minumum raggedness

	res := make([]*Cell, 0)
	remaining := width
	line := ""
	words := strings.Split(cell.Text, " ")

	for _, word := range words {
		if len(line) == 0 {
			// First word
			line = word
			remaining = width - len(word)
		} else if remaining >= len(word)+1 {
			// The word + space fits in the current line
			remaining -= (len(word) + 1)
			line += " " + word
		} else {
			// The word doesn't fit; finish the old line and start a new one.
			res = append(res, &Cell{line, cell.CellColor})
			remaining = width - len(word)
			line = word
		}
	}

	// Make sure the last line gets added
	if len(line) > 0 {
		res = append(res, &Cell{line, cell.CellColor})
	}

	return res
}

// Split this row if it that has any cells that are too long for their column.
func (row Row) split(headings []*Heading) []Row {
	newRows := make([]Row, 0)

	splits := make([][]*Cell, len(row))
	numLines := 0
	for i, cell := range row {
		splits[i] = cell.split(headings[i].width)
		if len(splits[i]) > numLines {
			numLines = len(splits[i])
		}
	}

	// At this point we have something like this:
	// [    ]  [      ]  [        ]
	// [    ]            [        ]
	//                   [        ]
	//                   [        ]
	//
	// Each cell has been broken up into columnar slice, and we now have
	// to piece back together rows. We need to make sure each new row
	// has the right number of columns, even if we have blank cells.
	// We iterate over the rows, and if the cell split has that row,
	// we add it, otherwise we just add a placeholder.
	for i := 0; i < numLines; i++ {
		// The new row has to have the same number of columns
		newRow := make(Row, len(row))

		for col, _ := range row {
			if i < len(splits[col]) {
				newRow[col] = splits[col][i]
			} else {
				newRow[col] = &Cell{"", nil}
			}
		}
		newRows = append(newRows, newRow)
	}

	return newRows
}

// Split any rows that have cells that are too long for their column.
func (pd *PrintData) splitRows() {
	newRows := make(Rows, 0)

	for _, row := range pd.Rows {
		rows := row.split(pd.Headings)
		newRows = append(newRows, rows...)
	}

	pd.Rows = newRows
}

func (pd *PrintData) Print() error {
	// Do we have the right number of columns in the data? This is a
	// programming error if we don't.
	for _, row := range pd.Rows {
		if len(row) != len(pd.Headings) {
			return errors.New("Wrong number of columns")
		}
	}

	// Calculate min/max column widths and split up cells if needed
	pd.calcWidths()
	pd.splitRows()

	// Next compute the format string for each cell
	fmtStrings := make([]string, 0, len(pd.Headings))
	for i, h := range pd.Headings {
		fmtStr := "%"
		if !h.RightJustified {
			fmtStr += "-"
		}
		fmtStr += fmt.Sprintf("%v", h.width)
		fmtStr += "v"
		if i+1 < len(pd.Headings) && pd.Config.NumSpaceSep > 0 {
			fmtStr += strings.Repeat(" ", pd.Config.NumSpaceSep)
		} else {
			fmtStr += "\n"
		}
		fmtStrings = append(fmtStrings, fmtStr)
	}

	// Print heading
	for i, h := range pd.Headings {
		fmt.Printf(fmtStrings[i], h.Text)
	}

	// Underlines
	if pd.Config.UnderlineChar != "" {
		for i, h := range pd.Headings {
			fmt.Printf(fmtStrings[i], strings.Repeat(pd.Config.UnderlineChar, h.width))
		}
	}

	for _, row := range pd.Rows {
		for col, cell := range row {
			if cell.CellColor != nil {
				cell.CellColor.Printf(fmtStrings[col], cell.Text)
			} else {
				fmt.Printf(fmtStrings[col], cell.Text)
			}
		}
	}

	return nil
}
<commit_msg>Bold column headings for tabular printing. This is the default now.<commit_after>package main

import (
	"errors"
	"fmt"
	"github.com/fatih/color"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

type Heading struct {
	Text           string
	RightJustified bool
	MinWidth       int

	// Calculated/working values
	width  int
	format string
	min    int
	max    int
}

type Cell struct {
	Text      string
	CellColor *color.Color
}

type Row []*Cell
type Rows []Row

type PrintConfig struct {
	NumSpaceSep   int
	UnderlineChar string
	BoldHeadings  bool
}

type PrintData struct {
	Headings []*Heading
	Rows     Rows
	Config   PrintConfig
}

////////////////////////////////////////////////////////////////////////////////

var defaultPrintConf = PrintConfig{
	NumSpaceSep:   3,
	UnderlineChar: "-",
	BoldHeadings:  true,
}

// Get the number of columns available on the terminal. We'll try to squeeze
// the results into this width.
func numTTYColumns() int {
	cmd := exec.Command("stty", "size")
	cmd.Stdin = os.Stdin
	res, err := cmd.Output()
	if err == nil {
		tokens := strings.Split(strings.TrimSpace(string(res)), " ")
		if len(tokens) == 2 {
			cols, _ := strconv.Atoi(tokens[1])
			return cols
		}
	}

	return -1
}

// Get the length of the longest word in a line
func longestWordLen(line string) int {
	start, longest, pos := 0, 0, 0
	for pos <= len(line) {
		// Break char or EOL
		if pos == len(line) || line[pos] == ' ' {
			// (pos-1) - start + 1
			wordlen := pos - start
			start = pos + 1
			pos = start

			if wordlen > longest {
				longest = wordlen
			}
		} else {
			pos += 1
		}
	}
	return longest
}

// Calculate the final widths of each column.
func (pd *PrintData) calcWidths() {
	// First pass, get the max and min widths of each column. Best case scenerio,
	// we fit within the real estate we have. Worst case scenerio, the min width
	// is smaller than what we have to work with.

	// Start with the headings, and don't break them up
	for _, h := range pd.Headings {
		h.min = h.MinWidth
		if h.min < len(h.Text) {
			h.min = len(h.Text)
		}
		h.max = h.min
	}

	// Now, the rows. The min column width should be the shortest word we find.
	// The max width is the width of the longest cell.
	for _, row := range pd.Rows {
		for col, cell := range row {
			lw := longestWordLen(cell.Text)
			if pd.Headings[col].min < lw {
				pd.Headings[col].min = lw
			}
			if len(cell.Text) > pd.Headings[col].max {
				pd.Headings[col].max = len(cell.Text)
			}
		}
	}

	// Figure out how many columns we have to work with. But, at least give
	// us something to work with.
	remaining := numTTYColumns()
	if remaining < 80 {
		remaining = 80
	}

	// Deduct the column spacers
	remaining -= (len(pd.Headings) - 1) * pd.Config.NumSpaceSep

	// Start out by setting the widths to the min widths.
	// (OK if remaining goes negative)
	for _, h := range pd.Headings {
		h.width = h.min
		remaining -= h.width
	}

	// Divide up the remaining columns equally
	for remaining > 0 {
		didOne := false
		for _, h := range pd.Headings {
			if h.width < h.max && remaining > 0 {
				remaining -= 1
				h.width += 1
				didOne = true
			}
		}
		if !didOne {
			break
		}
	}
}

// Split a single-line cell into (possibly) mutiple cells, with one line per cell.
func (cell *Cell) split(width int) []*Cell {
	// We currently use the simple (and common) greedy algorithm for
	// filling each row.
	// TODO: Change this to Knuth's algorithm for minumum raggedness

	res := make([]*Cell, 0)
	remaining := width
	line := ""
	words := strings.Split(cell.Text, " ")

	for _, word := range words {
		if len(line) == 0 {
			// First word
			line = word
			remaining = width - len(word)
		} else if remaining >= len(word)+1 {
			// The word + space fits in the current line
			remaining -= (len(word) + 1)
			line += " " + word
		} else {
			// The word doesn't fit; finish the old line and start a new one.
			res = append(res, &Cell{line, cell.CellColor})
			remaining = width - len(word)
			line = word
		}
	}

	// Make sure the last line gets added
	if len(line) > 0 {
		res = append(res, &Cell{line, cell.CellColor})
	}

	return res
}

// Split this row if it that has any cells that are too long for their column.
func (row Row) split(headings []*Heading) []Row {
	newRows := make([]Row, 0)

	splits := make([][]*Cell, len(row))
	numLines := 0
	for i, cell := range row {
		splits[i] = cell.split(headings[i].width)
		if len(splits[i]) > numLines {
			numLines = len(splits[i])
		}
	}

	// At this point we have something like this:
	// [    ]  [      ]  [        ]
	// [    ]            [        ]
	//                   [        ]
	//                   [        ]
	//
	// Each cell has been broken up into columnar slice, and we now have
	// to piece back together rows. We need to make sure each new row
	// has the right number of columns, even if we have blank cells.
	// We iterate over the rows, and if the cell split has that row,
	// we add it, otherwise we just add a placeholder.
	for i := 0; i < numLines; i++ {
		// The new row has to have the same number of columns
		newRow := make(Row, len(row))

		for col, _ := range row {
			if i < len(splits[col]) {
				newRow[col] = splits[col][i]
			} else {
				newRow[col] = &Cell{"", nil}
			}
		}
		newRows = append(newRows, newRow)
	}

	return newRows
}

// Split any rows that have cells that are too long for their column.
func (pd *PrintData) splitRows() {
	newRows := make(Rows, 0)

	for _, row := range pd.Rows {
		rows := row.split(pd.Headings)
		newRows = append(newRows, rows...)
	}

	pd.Rows = newRows
}

func (pd *PrintData) Print() error {
	// Do we have the right number of columns in the data? This is a
	// programming error if we don't.
	for _, row := range pd.Rows {
		if len(row) != len(pd.Headings) {
			return errors.New("Wrong number of columns")
		}
	}

	// Calculate min/max column widths and split up cells if needed
	pd.calcWidths()
	pd.splitRows()

	// Next compute the format string for each cell
	fmtStrings := make([]string, 0, len(pd.Headings))
	for i, h := range pd.Headings {
		fmtStr := "%"
		if !h.RightJustified {
			fmtStr += "-"
		}
		fmtStr += fmt.Sprintf("%v", h.width)
		fmtStr += "v"
		if i+1 < len(pd.Headings) && pd.Config.NumSpaceSep > 0 {
			fmtStr += strings.Repeat(" ", pd.Config.NumSpaceSep)
		} else {
			fmtStr += "\n"
		}
		fmtStrings = append(fmtStrings, fmtStr)
	}

	// Print heading
	bold := color.New(color.Bold)

	for i, h := range pd.Headings {
		if pd.Config.BoldHeadings {
			bold.Printf(fmtStrings[i], h.Text)
		} else {
			fmt.Printf(fmtStrings[i], h.Text)
		}
	}

	// Underlines
	if pd.Config.UnderlineChar != "" {
		for i, h := range pd.Headings {
			fmt.Printf(fmtStrings[i], strings.Repeat(pd.Config.UnderlineChar, h.width))
		}
	}

	for _, row := range pd.Rows {
		for col, cell := range row {
			if cell.CellColor != nil {
				cell.CellColor.Printf(fmtStrings[col], cell.Text)
			} else {
				fmt.Printf(fmtStrings[col], cell.Text)
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package download

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/subutai-io/agent/log"
	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
	"github.com/blang/semver"
	"net/url"
)

// ListItem describes Gorjun entity. It can be APT package, Subutai template or Raw file.
type ListItem struct {
	ID           string            `json:"id"`
	Hash         hashsums          `json:"hash"`
	Size         int               `json:"size"`
	Date         time.Time         `json:"upload-date-formatted"`
	Timestamp    string            `json:"upload-date-timestamp,omitempty"`
	Name         string            `json:"name,omitempty"`
	Tags         []string          `json:"tags,omitempty"`
	Owner        []string          `json:"owner,omitempty"`
	Parent       string            `json:"parent,omitempty"`
	Version      string            `json:"version,omitempty"`
	Filename     string            `json:"filename,omitempty"`
	Prefsize     string            `json:"prefsize,omitempty"`
	Signature    map[string]string `json:"signature,omitempty"`
	Description  string            `json:"description,omitempty"`
	Architecture string            `json:"architecture,omitempty"`
}

type hashsums struct {
	Md5    string `json:"md5,omitempty"`
	Sha256 string `json:"sha256,omitempty"`
}

// Handler provides download functionality for all artifacts.
func Handler(repo string, w http.ResponseWriter, r *http.Request) {
	id := r.URL.Query().Get("id")
	name := r.URL.Query().Get("name")
	if len(id) == 0 && len(name) == 0 {
		io.WriteString(w, "Please specify id or name")
		return
	} else if len(name) != 0 {
		id = db.LastHash(name, repo)
	}

	if len(db.Read(id)) > 0 && !db.Public(id) && !db.CheckShare(id, db.CheckToken(r.URL.Query().Get("token"))) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("Not found"))
		return
	}

	path := config.Storage.Path + id
	if md5, _ := db.Hash(id); len(md5) != 0 {
		path = config.Storage.Path + md5
	}

	f, err := os.Open(path)
	defer f.Close()

	if log.Check(log.WarnLevel, "Opening file "+config.Storage.Path+id, err) || len(id) == 0 {
		if len(config.CDN.Node) > 0 {
			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
			resp, err := client.Get(config.CDN.Node + r.URL.RequestURI())
			if !log.Check(log.WarnLevel, "Getting file from CDN", err) {
				w.Header().Set("Content-Length", resp.Header.Get("Content-Length"))
				w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
				w.Header().Set("Last-Modified", resp.Header.Get("Last-Modified"))
				w.Header().Set("Content-Disposition", resp.Header.Get("Content-Disposition"))

				io.Copy(w, resp.Body)
				resp.Body.Close()
				return
			}
		}

		w.WriteHeader(http.StatusNotFound)
		io.WriteString(w, "File not found")
		return
	}
	fi, _ := f.Stat()

	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && fi.ModTime().Unix() <= t.Unix() {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	w.Header().Set("Content-Length", fmt.Sprint(fi.Size()))
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))

	if name = db.Read(id); len(name) == 0 && len(config.CDN.Node) > 0 {
		httpclient := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
		resp, err := httpclient.Get(config.CDN.Node + "/kurjun/rest/template/info?id=" + id)
		if !log.Check(log.WarnLevel, "Getting info from CDN", err) {
			var info ListItem
			rsp, err := ioutil.ReadAll(resp.Body)
			if log.Check(log.WarnLevel, "Reading from CDN response", err) {
				w.WriteHeader(http.StatusNotFound)
				io.WriteString(w, "File not found")
				return
			}
			if !log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &info)) {
				w.Header().Set("Content-Disposition", "attachment; filename=\""+info.Filename+"\"")
			}
			resp.Body.Close()
		}
	} else {
		w.Header().Set("Content-Disposition", "attachment; filename=\""+db.Read(id)+"\"")
	}

	io.Copy(w, f)
}

// Info returns JSON formatted list of elements. It allows to apply some filters to Search.
func Info(repo string, r *http.Request) []byte {
	var items []ListItem
	var fullname bool
	var itemLatestVersion ListItem
	p := []int{0, 1000}
	id := r.URL.Query().Get("id")
	tag := r.URL.Query().Get("tag")
	name := r.URL.Query().Get("name")
	page := r.URL.Query().Get("page")
	owner := r.URL.Query().Get("owner")
	token := r.URL.Query().Get("token")
	subname := r.URL.Query().Get("subname")
	version := r.URL.Query().Get("version")
	verified := r.URL.Query().Get("verified")
	if len(subname) != 0 {
		name = subname
	}

	list := db.Search(name)
	if len(tag) > 0 {
		listByTag, err := db.Tag(tag)
		log.Check(log.DebugLevel, "Looking for artifacts with tag "+tag, err)
		list = intersect(list, listByTag)
	}
	if onlyOneParameterProvided("name",r) {
		verified = "true"
	}
	if len(id) > 0 {
		list = append(list[:0], id)
	} else if verified == "true" {
		items = append(items, getVerified(list, name, repo))
		output, err := json.Marshal(items)
		if err == nil && len(items) > 0 && items[0].ID != "" {
			return output
		}
		return nil
	}

	pstr := strings.Split(page, ",")
	p[0], _ = strconv.Atoi(pstr[0])
	if len(pstr) == 2 {
		p[1], _ = strconv.Atoi(pstr[1])
	}
	latestVersion, _ := semver.Make("0.0.0")
	for _, k := range list {
		if (!db.Public(k) && !db.CheckShare(k, db.CheckToken(token))) ||
			(len(owner) > 0 && db.CheckRepo(owner, repo, k) == 0) ||
			db.CheckRepo("", repo, k) == 0 {
			continue
		}

		if p[0]--; p[0] > 0 {
			continue
		}

		item := FormatItem(db.Info(k), repo, name)
		if len(subname) == 0 && name == item.Name {
			if strings.HasSuffix(item.Version, version) || len(version) == 0 {
				items = []ListItem{item}
				fullname = true
				itemVersion, _:= semver.Make(item.Version)
				if itemVersion.Compare(latestVersion) == 1 {
					latestVersion = itemVersion
					itemLatestVersion = item
				}
			}
		} else if !fullname && (len(version) == 0 || item.Version == version) {
			items = append(items, item)
		}

		if len(items) >= p[1] {
			break
		}
	}
	if len(items) == 1 {
		if version == "" && repo == "template" && itemLatestVersion.ID != "" {
			items[0] = itemLatestVersion
		}
		items[0].Signature = db.FileSignatures(items[0].ID)
	}
	output, err := json.Marshal(items)
	if err != nil || string(output) == "null" {
		return nil
	}
	return output
}

func in(str string, list []string) bool {
	for _, s := range list {
		if s == str {
			return true
		}
	}
	return false
}

func getVerified(list []string, name, repo string) ListItem {
	latestVersion, _ := semver.Make("0.0.0")
	var itemLatestVersion ListItem
	for _, k := range list {
		if info := db.Info(k); db.CheckRepo("", repo, k) > 0 {
			if info["name"] == name || (strings.HasPrefix(info["name"], name+"-subutai-template") && repo == "template") {
				for _, owner := range db.FileField(info["id"], "owner") {
					itemVersion, _:= semver.Make(info["version"])
					if in(owner, []string{"subutai", "jenkins", "docker","tester"}) &&
						itemVersion.Compare(latestVersion) == 1 {
						latestVersion = itemVersion
						itemLatestVersion = FormatItem(db.Info(k), repo, name)
					}
				}
			}
		}
	}
	return itemLatestVersion
}

func FormatItem(info map[string]string, repo, name string) ListItem {
	if len(info["prefsize"]) == 0 && repo == "template" {
		info["prefsize"] = "tiny"
	}

	date, _ := time.Parse(time.RFC3339Nano, info["date"])
	timestamp := strconv.FormatInt(date.Unix(), 10)
	item := ListItem{
		ID:           info["id"],
		Date:         date,
		Hash:         hashsums{Md5: info["md5"], Sha256: info["sha256"]},
		Name:         strings.Split(info["name"], "-subutai-template")[0],
		Tags:         db.FileField(info["id"], "tags"),
		Owner:        db.FileField(info["id"], "owner"),
		Version:      info["version"],
		Filename:     info["name"],
		Parent:       info["parent"],
		Prefsize:     info["prefsize"],
		Architecture: strings.ToUpper(info["arch"]),
		Description:  info["Description"],
		Timestamp:    timestamp,
	}
	item.Size, _ = strconv.Atoi(info["size"])

	if repo == "apt" {
		item.Version = info["Version"]
		item.Architecture = info["Architecture"]
		item.Size, _ = strconv.Atoi(info["Size"])
	}
	if len(item.Hash.Md5) == 0 {
		item.Hash.Md5 = item.ID
	}
	return item
}

func intersect(listA, listB []string) (list []string) {
	mapA := map[string]bool{}
	for _, item := range listA {
		mapA[item] = true
	}
	for _, item := range listB {
		if mapA[item] {
			list = append(list, item)
		}
	}
	return list
}

func onlyOneParameterProvided(parameter string,r *http.Request) bool {
	u, _ := url.Parse(r.RequestURI)
	parameters, _ := url.ParseQuery(u.RawQuery)
	for key, _ := range parameters {
		if key != parameter {
			return false
		}
	}
	return len(parameters) > 0
}
<commit_msg>Hot-fix deleted useless verified owner's<commit_after>package download

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/subutai-io/agent/log"
	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
	"github.com/blang/semver"
	"net/url"
)

// ListItem describes Gorjun entity. It can be APT package, Subutai template or Raw file.
type ListItem struct {
	ID           string            `json:"id"`
	Hash         hashsums          `json:"hash"`
	Size         int               `json:"size"`
	Date         time.Time         `json:"upload-date-formatted"`
	Timestamp    string            `json:"upload-date-timestamp,omitempty"`
	Name         string            `json:"name,omitempty"`
	Tags         []string          `json:"tags,omitempty"`
	Owner        []string          `json:"owner,omitempty"`
	Parent       string            `json:"parent,omitempty"`
	Version      string            `json:"version,omitempty"`
	Filename     string            `json:"filename,omitempty"`
	Prefsize     string            `json:"prefsize,omitempty"`
	Signature    map[string]string `json:"signature,omitempty"`
	Description  string            `json:"description,omitempty"`
	Architecture string            `json:"architecture,omitempty"`
}

type hashsums struct {
	Md5    string `json:"md5,omitempty"`
	Sha256 string `json:"sha256,omitempty"`
}

// Handler provides download functionality for all artifacts.
func Handler(repo string, w http.ResponseWriter, r *http.Request) {
	id := r.URL.Query().Get("id")
	name := r.URL.Query().Get("name")
	if len(id) == 0 && len(name) == 0 {
		io.WriteString(w, "Please specify id or name")
		return
	} else if len(name) != 0 {
		id = db.LastHash(name, repo)
	}

	if len(db.Read(id)) > 0 && !db.Public(id) && !db.CheckShare(id, db.CheckToken(r.URL.Query().Get("token"))) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("Not found"))
		return
	}

	path := config.Storage.Path + id
	if md5, _ := db.Hash(id); len(md5) != 0 {
		path = config.Storage.Path + md5
	}

	f, err := os.Open(path)
	defer f.Close()

	if log.Check(log.WarnLevel, "Opening file "+config.Storage.Path+id, err) || len(id) == 0 {
		if len(config.CDN.Node) > 0 {
			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
			resp, err := client.Get(config.CDN.Node + r.URL.RequestURI())
			if !log.Check(log.WarnLevel, "Getting file from CDN", err) {
				w.Header().Set("Content-Length", resp.Header.Get("Content-Length"))
				w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
				w.Header().Set("Last-Modified", resp.Header.Get("Last-Modified"))
				w.Header().Set("Content-Disposition", resp.Header.Get("Content-Disposition"))

				io.Copy(w, resp.Body)
				resp.Body.Close()
				return
			}
		}

		w.WriteHeader(http.StatusNotFound)
		io.WriteString(w, "File not found")
		return
	}
	fi, _ := f.Stat()

	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && fi.ModTime().Unix() <= t.Unix() {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	w.Header().Set("Content-Length", fmt.Sprint(fi.Size()))
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))

	if name = db.Read(id); len(name) == 0 && len(config.CDN.Node) > 0 {
		httpclient := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
		resp, err := httpclient.Get(config.CDN.Node + "/kurjun/rest/template/info?id=" + id)
		if !log.Check(log.WarnLevel, "Getting info from CDN", err) {
			var info ListItem
			rsp, err := ioutil.ReadAll(resp.Body)
			if log.Check(log.WarnLevel, "Reading from CDN response", err) {
				w.WriteHeader(http.StatusNotFound)
				io.WriteString(w, "File not found")
				return
			}
			if !log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &info)) {
				w.Header().Set("Content-Disposition", "attachment; filename=\""+info.Filename+"\"")
			}
			resp.Body.Close()
		}
	} else {
		w.Header().Set("Content-Disposition", "attachment; filename=\""+db.Read(id)+"\"")
	}

	io.Copy(w, f)
}

// Info returns JSON formatted list of elements. It allows to apply some filters to Search.
func Info(repo string, r *http.Request) []byte {
	var items []ListItem
	var fullname bool
	var itemLatestVersion ListItem
	p := []int{0, 1000}
	id := r.URL.Query().Get("id")
	tag := r.URL.Query().Get("tag")
	name := r.URL.Query().Get("name")
	page := r.URL.Query().Get("page")
	owner := r.URL.Query().Get("owner")
	token := r.URL.Query().Get("token")
	subname := r.URL.Query().Get("subname")
	version := r.URL.Query().Get("version")
	verified := r.URL.Query().Get("verified")
	if len(subname) != 0 {
		name = subname
	}

	list := db.Search(name)
	if len(tag) > 0 {
		listByTag, err := db.Tag(tag)
		log.Check(log.DebugLevel, "Looking for artifacts with tag "+tag, err)
		list = intersect(list, listByTag)
	}
	if onlyOneParameterProvided("name",r) {
		verified = "true"
	}
	if len(id) > 0 {
		list = append(list[:0], id)
	} else if verified == "true" {
		items = append(items, getVerified(list, name, repo))
		output, err := json.Marshal(items)
		if err == nil && len(items) > 0 && items[0].ID != "" {
			return output
		}
		return nil
	}

	pstr := strings.Split(page, ",")
	p[0], _ = strconv.Atoi(pstr[0])
	if len(pstr) == 2 {
		p[1], _ = strconv.Atoi(pstr[1])
	}
	latestVersion, _ := semver.Make("0.0.0")
	for _, k := range list {
		if (!db.Public(k) && !db.CheckShare(k, db.CheckToken(token))) ||
			(len(owner) > 0 && db.CheckRepo(owner, repo, k) == 0) ||
			db.CheckRepo("", repo, k) == 0 {
			continue
		}

		if p[0]--; p[0] > 0 {
			continue
		}

		item := FormatItem(db.Info(k), repo, name)
		if len(subname) == 0 && name == item.Name {
			if strings.HasSuffix(item.Version, version) || len(version) == 0 {
				items = []ListItem{item}
				fullname = true
				itemVersion, _:= semver.Make(item.Version)
				if itemVersion.Compare(latestVersion) == 1 {
					latestVersion = itemVersion
					itemLatestVersion = item
				}
			}
		} else if !fullname && (len(version) == 0 || item.Version == version) {
			items = append(items, item)
		}

		if len(items) >= p[1] {
			break
		}
	}
	if len(items) == 1 {
		if version == "" && repo == "template" && itemLatestVersion.ID != "" {
			items[0] = itemLatestVersion
		}
		items[0].Signature = db.FileSignatures(items[0].ID)
	}
	output, err := json.Marshal(items)
	if err != nil || string(output) == "null" {
		return nil
	}
	return output
}

func in(str string, list []string) bool {
	for _, s := range list {
		if s == str {
			return true
		}
	}
	return false
}

func getVerified(list []string, name, repo string) ListItem {
	latestVersion, _ := semver.Make("0.0.0")
	var itemLatestVersion ListItem
	for _, k := range list {
		if info := db.Info(k); db.CheckRepo("", repo, k) > 0 {
			if info["name"] == name || (strings.HasPrefix(info["name"], name+"-subutai-template") && repo == "template") {
				for _, owner := range db.FileField(info["id"], "owner") {
					itemVersion, _:= semver.Make(info["version"])
					if in(owner, []string{"subutai", "jenkins", "docker"}) &&
						itemVersion.Compare(latestVersion) == 1 {
						latestVersion = itemVersion
						itemLatestVersion = FormatItem(db.Info(k), repo, name)
					}
				}
			}
		}
	}
	return itemLatestVersion
}

func FormatItem(info map[string]string, repo, name string) ListItem {
	if len(info["prefsize"]) == 0 && repo == "template" {
		info["prefsize"] = "tiny"
	}

	date, _ := time.Parse(time.RFC3339Nano, info["date"])
	timestamp := strconv.FormatInt(date.Unix(), 10)
	item := ListItem{
		ID:           info["id"],
		Date:         date,
		Hash:         hashsums{Md5: info["md5"], Sha256: info["sha256"]},
		Name:         strings.Split(info["name"], "-subutai-template")[0],
		Tags:         db.FileField(info["id"], "tags"),
		Owner:        db.FileField(info["id"], "owner"),
		Version:      info["version"],
		Filename:     info["name"],
		Parent:       info["parent"],
		Prefsize:     info["prefsize"],
		Architecture: strings.ToUpper(info["arch"]),
		Description:  info["Description"],
		Timestamp:    timestamp,
	}
	item.Size, _ = strconv.Atoi(info["size"])

	if repo == "apt" {
		item.Version = info["Version"]
		item.Architecture = info["Architecture"]
		item.Size, _ = strconv.Atoi(info["Size"])
	}
	if len(item.Hash.Md5) == 0 {
		item.Hash.Md5 = item.ID
	}
	return item
}

func intersect(listA, listB []string) (list []string) {
	mapA := map[string]bool{}
	for _, item := range listA {
		mapA[item] = true
	}
	for _, item := range listB {
		if mapA[item] {
			list = append(list, item)
		}
	}
	return list
}

func onlyOneParameterProvided(parameter string,r *http.Request) bool {
	u, _ := url.Parse(r.RequestURI)
	parameters, _ := url.ParseQuery(u.RawQuery)
	for key, _ := range parameters {
		if key != parameter {
			return false
		}
	}
	return len(parameters) > 0
}
<|endoftext|>


<commit_before>package analyze

import (
	"encoding/json"
	"net/http"
	"regexp"

	"github.com/deckarep/golang-set"
	"github.com/jzelinskie/geddit"
)

func AnalyzeComments(comments []*geddit.Comment) error {
	r, err := regexp.Compile(`github\.com\/\w+\/\w+`)
	if err != nil {
		return err
	}
	for _, comment := range comments {
		if comment.Author == "github-stats-bot" {
			continue
		}
		if r.MatchString(comment.Body) {
			linkSet := mapset.NewSet()
			for _, link := range r.FindAllString(comment.Body, -1) {
				linkSet.Add(link)
			}
			var links []string
			for _, link := range linkSet.ToSlice() {
				links = append(links, link.(string))
			}
			go postReply(comment, links)
		}
	}
	return nil
}

func postReply(comment *geddit.Comment, links []string) {
	for _, link := range links {
		var data map[string]interface{}
		if err := getStats(link, &data); err != nil {
			return
		}
		if data["message"] == "Not Found" {
			return
		}
	}
}

func getStats(link string, data *map[string]interface{}) error {
	res, err := http.Get("https://api.github.com/repos/" + link[11:])
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if err = json.NewDecoder(res.Body).Decode(&data); err != nil {
		return err
	}
	return nil
}
<commit_msg>finished formatting reply and footer<commit_after>package analyze

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"regexp"

	"github.com/deckarep/golang-set"
	"github.com/jzelinskie/geddit"
)

func AnalyzeComments(comments []*geddit.Comment) error {
	r, err := regexp.Compile(`github\.com\/\w+\/\w+`)
	if err != nil {
		return err
	}
	for _, comment := range comments {
		if comment.Author == "github-stats-bot" {
			continue
		}
		if r.MatchString(comment.Body) {
			linkSet := mapset.NewSet()
			for _, link := range r.FindAllString(comment.Body, -1) {
				linkSet.Add(link)
			}
			var links []string
			for _, link := range linkSet.ToSlice() {
				links = append(links, link.(string))
			}
			if err = postReply(comment, links); err != nil {
				return err
			}
		}
	}
	return nil
}

func postReply(comment *geddit.Comment, links []string) error {
	footer := "***\n^(This is Earth radio, and now here's human music ♫)\n\n^[Source](https://github.com/anaskhan96/github-stats-bot) ^| ^[PMme](https://np.reddit.com/message/compose?to=github-stats-bot)"
	var reply string
	for _, link := range links {
		var data map[string]interface{}
		if err := getStats(link, &data); err != nil {
			return err
		}
		if data["message"] == "Not Found" {
			return errors.New("Wrong GitHub API endpoint")
		}
		description := data["description"].(string)
		stargazers := int(data["stargazers_count"].(float64))
		forks := int(data["forks_count"].(float64))
		issuesURL := "https://" + link + "/issues"
		pullsURL := "https://" + link + "/pulls"
		reply += fmt.Sprintf("\n[%s](https://%s)\n\n> *Description*: %s\n\n> *Stars*: %d\n\n> *Forks*: %d\n\n> [Issues](%s) | [Pull Requests](%s)\n\n",
			link, link, description, stargazers, forks, issuesURL, pullsURL)
	}
	reply += footer
	return nil
}

func getStats(link string, data *map[string]interface{}) error {
	res, err := http.Get("https://api.github.com/repos/" + link[11:])
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if err = json.NewDecoder(res.Body).Decode(&data); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package types

import (
	"fmt"
	"unicode/utf8"
)

//
// Recipe Creation Types
//

type Recipe struct {
	Title        string       `json:"title" validate:"recipetitle"`
	Notes        string       `json:"notes" validate:"recipenotes"`
	Ingredients  []Ingredient `json:"ingredients" validate:"existence"`
	CookTime     int          `json:"cooktime" validate:"time"`
	CookTimeUnit string       `json:"cooktimeunit" validate:"timeunit"`
	PrepTime     int          `json:"preptime" validate:"time"`
	PrepTimeUnit string       `json:"preptimeunit" validate:"timeunit"`
	Steps        []Step       `json:"steps" validate:"existence"`
	Tags         []Tag        `json:"tags"`
	Public       bool         `json:"public"`
}

//
// Recipe Component Types
//

type Ingredient struct {
	Name       string `json:"name" validate:"ingredient"`
	Amount     int    `json:"amount"`
	AmountUnit string `json:"amountunit"`
	URL        string `json:"url" validate:"url"`
}

type Step struct {
	Instruction string `json:"instruction" validate:"step"`
	Time        int    `json:"time" validate:"time"`
	TimeUnit    string `json:"timeunit" validate:"timeunit"`
}

//
// Recipe Tag Types
//

type Tag struct {
	Name string `json:"name" validate:"tag"`
}

//
// Recipe Validator Functions
//

func (v RicettaValidator) validateTimeUnit(i interface{}) error {
	timeUnit := i.(string)
	if timeUnit == "" {
		return fmt.Errorf("Required field")
	} else if !v.Constants.TIME_UNIT_REGEX.MatchString(timeUnit) {
		return fmt.Errorf(timeUnit + " is not a valid unit of time - [sec(s), min(s), hr(s), day(s), week(s)]")
	} else {
		return nil
	}
}

func (v RicettaValidator) validateTime(i interface{}) error {
	time, ok := i.(int)
	if !ok {
		return fmt.Errorf("Required field")
	} else if time <= 0 || time > 1000 {
		return fmt.Errorf("Invalid time unit: %d", time)
	} else {
		return nil
	}
}

func (v RicettaValidator) validateRecipeTitle(i interface{}) error {
	title := i.(string)
	titlelen := utf8.RuneCountInString(title)
	if title == "" {
		return fmt.Errorf("Required field")
	} else if titlelen > v.Constants.MAX_RECIPE_TITLE_LENGTH {
		return fmt.Errorf("Exceeds max title length of %d", v.Constants.MAX_RECIPE_TITLE_LENGTH)
	} else {
		return nil
	}
}

func (v RicettaValidator) validateRecipeNotes(i interface{}) error {
	notes := i.(string)
	noteslen := utf8.RuneCountInString(notes)
	if notes == "" {
		return fmt.Errorf("Required field")
	} else if noteslen > v.Constants.MAX_RECIPE_NOTES_LENGTH {
		return fmt.Errorf("Exceeds max notes length of %d", v.Constants.MAX_RECIPE_NOTES_LENGTH)
	} else {
		return nil
	}
}

func (v RicettaValidator) validateIngredient(i interface{}) error {
	ingredient := i.(string)
	ingredientlen := utf8.RuneCountInString(ingredient)
	if ingredient == "" {
		return fmt.Errorf("Required field")
	} else if ingredientlen > v.Constants.MAX_INGREDIENT_LENGTH {
		return fmt.Errorf("Exceeds max ingredient length of %d", v.Constants.MAX_INGREDIENT_LENGTH)
	} else {
		return nil
	}
}

func (v RicettaValidator) validateStep(i interface{}) error {
	step := i.(string)
	steplen := utf8.RuneCountInString(step)
	if step == "" {
		return fmt.Errorf("Required field")
	} else if steplen > v.Constants.MAX_STEP_LENGTH {
		return fmt.Errorf("Exceeds max step length of %d", v.Constants.MAX_STEP_LENGTH)
	} else {
		return nil
	}
}

func (v RicettaValidator) validateURL(i interface{}) error {
	url := i.(string)
	if url == "" {
		// This demonstrates how to validate an optional field
		return nil
	} else if !v.Constants.URL_REGEX.MatchString(url) {
		return fmt.Errorf("Not a valid URL string: %s", url)
	} else {
		return nil
	}
}

func (v RicettaValidator) validateTag(i interface{}) error {
	tag := i.(string)
	taglen := utf8.RuneCountInString(tag)
	if tag == "" {
		return fmt.Errorf("Required field")
	} else if taglen > v.Constants.MAX_TAG_LENGTH {
		return fmt.Errorf("Exceeds max tag length of %d", v.Constants.MAX_TAG_LENGTH)
	} else {
		return nil
	}
}
<commit_msg>Changed public bool to private<commit_after>package types

import (
	"fmt"
	"unicode/utf8"
)

//
// Recipe Creation Types
//

type Recipe struct {
	Title        string       `json:"title" validate:"recipetitle"`
	Notes        string       `json:"notes" validate:"recipenotes"`
	Ingredients  []Ingredient `json:"ingredients" validate:"existence"`
	CookTime     int          `json:"cooktime" validate:"time"`
	CookTimeUnit string       `json:"cooktimeunit" validate:"timeunit"`
	PrepTime     int          `json:"preptime" validate:"time"`
	PrepTimeUnit string       `json:"preptimeunit" validate:"timeunit"`
	Steps        []Step       `json:"steps" validate:"existence"`
	Tags         []Tag        `json:"tags"`
	Private      bool         `json:"private"`
}

//
// Recipe Component Types
//

type Ingredient struct {
	Name       string `json:"name" validate:"ingredient"`
	Amount     int    `json:"amount"`
	AmountUnit string `json:"amountunit"`
	URL        string `json:"url" validate:"url"`
}

type Step struct {
	Instruction string `json:"instruction" validate:"step"`
	Time        int    `json:"time" validate:"time"`
	TimeUnit    string `json:"timeunit" validate:"timeunit"`
}

//
// Recipe Tag Types
//

type Tag struct {
	Name string `json:"name" validate:"tag"`
}

//
// Recipe Validator Functions
//

func (v RicettaValidator) validateTimeUnit(i interface{}) error {
	timeUnit := i.(string)
	if timeUnit == "" {
		return fmt.Errorf("Required field")
	} else if !v.Constants.TIME_UNIT_REGEX.MatchString(timeUnit) {
		return fmt.Errorf(timeUnit + " is not a valid unit of time - [sec(s), min(s), hr(s), day(s), week(s)]")
	} else {
		return nil
	}
}

func (v RicettaValidator) validateTime(i interface{}) error {
	time, ok := i.(int)
	if !ok {
		return fmt.Errorf("Required field")
	} else if time <= 0 || time > 1000 {
		return fmt.Errorf("Invalid time unit: %d", time)
	} else {
		return nil
	}
}

func (v RicettaValidator) validateRecipeTitle(i interface{}) error {
	title := i.(string)
	titlelen := utf8.RuneCountInString(title)
	if title == "" {
		return fmt.Errorf("Required field")
	} else if titlelen > v.Constants.MAX_RECIPE_TITLE_LENGTH {
		return fmt.Errorf("Exceeds max title length of %d", v.Constants.MAX_RECIPE_TITLE_LENGTH)
	} else {
		return nil
	}
}

func (v RicettaValidator) validateRecipeNotes(i interface{}) error {
	notes := i.(string)
	noteslen := utf8.RuneCountInString(notes)
	if notes == "" {
		return fmt.Errorf("Required field")
	} else if noteslen > v.Constants.MAX_RECIPE_NOTES_LENGTH {
		return fmt.Errorf("Exceeds max notes length of %d", v.Constants.MAX_RECIPE_NOTES_LENGTH)
	} else {
		return nil
	}
}

func (v RicettaValidator) validateIngredient(i interface{}) error {
	ingredient := i.(string)
	ingredientlen := utf8.RuneCountInString(ingredient)
	if ingredient == "" {
		return fmt.Errorf("Required field")
	} else if ingredientlen > v.Constants.MAX_INGREDIENT_LENGTH {
		return fmt.Errorf("Exceeds max ingredient length of %d", v.Constants.MAX_INGREDIENT_LENGTH)
	} else {
		return nil
	}
}

func (v RicettaValidator) validateStep(i interface{}) error {
	step := i.(string)
	steplen := utf8.RuneCountInString(step)
	if step == "" {
		return fmt.Errorf("Required field")
	} else if steplen > v.Constants.MAX_STEP_LENGTH {
		return fmt.Errorf("Exceeds max step length of %d", v.Constants.MAX_STEP_LENGTH)
	} else {
		return nil
	}
}

func (v RicettaValidator) validateURL(i interface{}) error {
	url := i.(string)
	if url == "" {
		// This demonstrates how to validate an optional field
		return nil
	} else if !v.Constants.URL_REGEX.MatchString(url) {
		return fmt.Errorf("Not a valid URL string: %s", url)
	} else {
		return nil
	}
}

func (v RicettaValidator) validateTag(i interface{}) error {
	tag := i.(string)
	taglen := utf8.RuneCountInString(tag)
	if tag == "" {
		return fmt.Errorf("Required field")
	} else if taglen > v.Constants.MAX_TAG_LENGTH {
		return fmt.Errorf("Exceeds max tag length of %d", v.Constants.MAX_TAG_LENGTH)
	} else {
		return nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"io/ioutil"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"sync"
	"time"
)

var (
	reDimensions = regexp.MustCompile(" ([0-9]+)x([0-9]+)")
	reImage = regexp.MustCompile("(?i)^(.+)\\.(gif|jpeg|jpg|png)$")
)

// Image information, gasp
type ImageInfo struct {
	FileSize    int64  `json:"s"`
	ModTime     int64  `json:"m"`
	ImagePath   string `json:"i"`
	ImageWidth  int    `json:"w"`
	ImageHeight int    `json:"h"`
	ThumbPath   string `json:"t"`
}

type Thumbnailer struct {
	*sync.Mutex
}

func NewThumbnailer() *Thumbnailer {
	return &Thumbnailer{&sync.Mutex{}}
}

func (t *Thumbnailer) ScanFolder(gallery *GalleryConfig, basePath string) ([]string, []ImageInfo, error) {
	// start := time.Now()
	// defer func() {
	// 	log.Info("ScanFolder(%s) took %s", basePath, time.Since(start))
	// }()

	// Acquire lock
	t.Lock()
	defer t.Unlock()

	// Check cache
	cacheDirs, cacheImages, cacheOk := cache.Get(basePath)
	if cacheOk {
		return cacheDirs, cacheImages, nil
	}

	// Get a Redis connection
	conn := redisPool.Get()
	defer conn.Close()

	// Vars
	var dirs []string
	var images []ImageInfo

	// Get the files
	fileNames, err := ioutil.ReadDir(basePath)
	if err != nil {
		return nil, nil, err
	}

	// Subfolders need a fake .. directory
	if basePath != gallery.ImagePath {
		dirs = append(dirs, "..")
	}

	// Try fetching data from Redis
	// t1 := time.Now()
	jsonData, err := redis.String(conn.Do("HGET", "images", basePath))
	if err != redis.ErrNil && err != nil {
		return nil, nil, err
	}
	// log.Debug("HGET took %s", time.Since(t1))

	// Try unmarshalling
	// t2 := time.Now()
	fileMap := make(map[string]ImageInfo)
	if jsonData != "" {
		if err = json.Unmarshal([]byte(jsonData), &fileMap); err != nil {
			return nil, nil, err
		}
	}
	// log.Debug("Unmarshal took %s", time.Since(t2))

	// Some things
	resizeStr := fmt.Sprintf("%dx%d^", gallery.ThumbWidth, gallery.ThumbHeight)
	extentStr := fmt.Sprintf("%dx%d", gallery.ThumbWidth, gallery.ThumbHeight)

	// Iterateee
	// t3 := time.Now()
	for _, fileInfo := range fileNames {
		tl := time.Now()

		fileName := fileInfo.Name()

		// Don't care about directories
		if fileInfo.IsDir() {
			dirs = append(dirs, fileName)
			continue
		}

		// Don't care about weird filetypes
		if !reImage.MatchString(fileName) {
			continue
		}

		// Check to see if the image has changed
		fileModTime := fileInfo.ModTime().Unix()
		fileSize := fileInfo.Size()

		imageInfo, ok := fileMap[fileName]
		if ok && imageInfo.FileSize == fileSize && imageInfo.ModTime == fileModTime && imageInfo.ThumbPath != "" {
			images = append(images, imageInfo)
			continue
		}

		filePath := path.Join(basePath, fileName)

		// Generate the thumbnail filename and path
		b, err := ioutil.ReadFile(filePath)
		if err != nil {
			return nil, nil, err
		}
		thumbName := fmt.Sprintf("%x.jpg", md5.Sum(b))
		thumbPart := path.Join(string(thumbName[0]), thumbName)
		thumbPath := path.Join(gallery.ThumbPath, thumbPart)

		// Generate the thumbnail image and save it
		// t := time.Now()

		cmd := exec.Command("convert", fmt.Sprintf("%s[0]", filePath), "-thumbnail", resizeStr, "-gravity", "center", "-quality", "90", "-extent", extentStr, "-verbose", thumbPath)
		out, err := cmd.CombinedOutput()
		if err != nil {
			return nil, nil, err
		}

		// Get image dimensions from the output
		matches := reDimensions.FindAllStringSubmatch(string(out), -1)
		if len(matches) == 0 {
			log.Warning("matches failed: %q", out)
			return nil, nil, err
		}

		imageWidth, err := strconv.ParseInt(matches[0][1], 10, 32)
		if err != nil {
			return nil, nil, err
		}
		imageHeight, err := strconv.ParseInt(matches[0][2], 10, 32)
		if err != nil {
			return nil, nil, err
		}

		// log.Debug("thumbnail for %s took %s", filePath, time.Since(t))

		// Finish junk
		imagePart, _ := filepath.Rel(gallery.ImagePath, filePath)

		imageInfo = ImageInfo{
			FileSize:    fileSize,
			ModTime:     fileModTime,
			ImagePath:   imagePart,
			ImageWidth:  int(imageWidth),
			ImageHeight: int(imageHeight),
			ThumbPath:   thumbPart,
		}
		images = append(images, imageInfo)
		fileMap[fileName] = imageInfo

		log.Debug("loop for %s took %s", filePath, time.Since(tl))
	}
	// log.Debug("Loop took %s", time.Since(t3))

	// Update cache
	cache.Set(basePath, dirs, images)

	// Update Redis
	b, err := json.Marshal(fileMap)
	if err != nil {
		return nil, nil, err
	}
	conn.Do("HSET", "images", basePath, string(b))

	return dirs, images, nil
}
<commit_msg>Stupid go fmt hook not working<commit_after>package main

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"io/ioutil"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"sync"
	"time"
)

var (
	reDimensions = regexp.MustCompile(" ([0-9]+)x([0-9]+)")
	reImage      = regexp.MustCompile("(?i)^(.+)\\.(gif|jpeg|jpg|png)$")
)

// Image information, gasp
type ImageInfo struct {
	FileSize    int64  `json:"s"`
	ModTime     int64  `json:"m"`
	ImagePath   string `json:"i"`
	ImageWidth  int    `json:"w"`
	ImageHeight int    `json:"h"`
	ThumbPath   string `json:"t"`
}

type Thumbnailer struct {
	*sync.Mutex
}

func NewThumbnailer() *Thumbnailer {
	return &Thumbnailer{&sync.Mutex{}}
}

func (t *Thumbnailer) ScanFolder(gallery *GalleryConfig, basePath string) ([]string, []ImageInfo, error) {
	// start := time.Now()
	// defer func() {
	// 	log.Info("ScanFolder(%s) took %s", basePath, time.Since(start))
	// }()

	// Acquire lock
	t.Lock()
	defer t.Unlock()

	// Check cache
	cacheDirs, cacheImages, cacheOk := cache.Get(basePath)
	if cacheOk {
		return cacheDirs, cacheImages, nil
	}

	// Get a Redis connection
	conn := redisPool.Get()
	defer conn.Close()

	// Vars
	var dirs []string
	var images []ImageInfo

	// Get the files
	fileNames, err := ioutil.ReadDir(basePath)
	if err != nil {
		return nil, nil, err
	}

	// Subfolders need a fake .. directory
	if basePath != gallery.ImagePath {
		dirs = append(dirs, "..")
	}

	// Try fetching data from Redis
	// t1 := time.Now()
	jsonData, err := redis.String(conn.Do("HGET", "images", basePath))
	if err != redis.ErrNil && err != nil {
		return nil, nil, err
	}
	// log.Debug("HGET took %s", time.Since(t1))

	// Try unmarshalling
	// t2 := time.Now()
	fileMap := make(map[string]ImageInfo)
	if jsonData != "" {
		if err = json.Unmarshal([]byte(jsonData), &fileMap); err != nil {
			return nil, nil, err
		}
	}
	// log.Debug("Unmarshal took %s", time.Since(t2))

	// Some things
	resizeStr := fmt.Sprintf("%dx%d^", gallery.ThumbWidth, gallery.ThumbHeight)
	extentStr := fmt.Sprintf("%dx%d", gallery.ThumbWidth, gallery.ThumbHeight)

	// Iterateee
	// t3 := time.Now()
	for _, fileInfo := range fileNames {
		tl := time.Now()

		fileName := fileInfo.Name()

		// Don't care about directories
		if fileInfo.IsDir() {
			dirs = append(dirs, fileName)
			continue
		}

		// Don't care about weird filetypes
		if !reImage.MatchString(fileName) {
			continue
		}

		// Check to see if the image has changed
		fileModTime := fileInfo.ModTime().Unix()
		fileSize := fileInfo.Size()

		imageInfo, ok := fileMap[fileName]
		if ok && imageInfo.FileSize == fileSize && imageInfo.ModTime == fileModTime && imageInfo.ThumbPath != "" {
			images = append(images, imageInfo)
			continue
		}

		filePath := path.Join(basePath, fileName)

		// Generate the thumbnail filename and path
		b, err := ioutil.ReadFile(filePath)
		if err != nil {
			return nil, nil, err
		}
		thumbName := fmt.Sprintf("%x.jpg", md5.Sum(b))
		thumbPart := path.Join(string(thumbName[0]), thumbName)
		thumbPath := path.Join(gallery.ThumbPath, thumbPart)

		// Generate the thumbnail image and save it
		// t := time.Now()

		cmd := exec.Command("convert", fmt.Sprintf("%s[0]", filePath), "-thumbnail", resizeStr, "-gravity", "center", "-quality", "90", "-extent", extentStr, "-verbose", thumbPath)
		out, err := cmd.CombinedOutput()
		if err != nil {
			return nil, nil, err
		}

		// Get image dimensions from the output
		matches := reDimensions.FindAllStringSubmatch(string(out), -1)
		if len(matches) == 0 {
			log.Warning("matches failed: %q", out)
			return nil, nil, err
		}

		imageWidth, err := strconv.ParseInt(matches[0][1], 10, 32)
		if err != nil {
			return nil, nil, err
		}
		imageHeight, err := strconv.ParseInt(matches[0][2], 10, 32)
		if err != nil {
			return nil, nil, err
		}

		// log.Debug("thumbnail for %s took %s", filePath, time.Since(t))

		// Finish junk
		imagePart, _ := filepath.Rel(gallery.ImagePath, filePath)

		imageInfo = ImageInfo{
			FileSize:    fileSize,
			ModTime:     fileModTime,
			ImagePath:   imagePart,
			ImageWidth:  int(imageWidth),
			ImageHeight: int(imageHeight),
			ThumbPath:   thumbPart,
		}
		images = append(images, imageInfo)
		fileMap[fileName] = imageInfo

		log.Debug("loop for %s took %s", filePath, time.Since(tl))
	}
	// log.Debug("Loop took %s", time.Since(t3))

	// Update cache
	cache.Set(basePath, dirs, images)

	// Update Redis
	b, err := json.Marshal(fileMap)
	if err != nil {
		return nil, nil, err
	}
	conn.Do("HSET", "images", basePath, string(b))

	return dirs, images, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"regexp"
	"strings"
	"sync/atomic"
	"time"
)

var conf config

// config represents the application configuration
type config struct {
	url           string
	interval      time.Duration
	minCompletion float64
	host          string
	port          int
}

// Mirror is a description of an Arch Linux Mirror
type Mirror struct {
	Protocol    string     `json:"protocol"`
	URL         string     `json:"url"`
	Country     string     `json:"country"`
	LastSync    *time.Time `json:"last_sync"`
	Delay       int        `json:"delay"`
	Score       float64    `json:"score"`
	Completion  float64    `json:"completion_pct"`
	CountryCode string     `json:"country_code"`
	DurStdDev   float64    `json:"duration_stddev"`
	DurAvg      float64    `json:"duration_avg"`
}

// MirrorStatus is global status information for mirror checks
type MirrorStatus struct {
	Cutoff         int       `json:"cutoff"`
	CheckFrequency int       `json:"check_frequency"`
	NumChecks      int       `json:"num_checks"`
	LastCheck      time.Time `json:"last_check"`
	Version        int       `json:"version"`
	URLs           []*Mirror `json:"urls"`
}

func init() {
	flag.StringVar(&conf.url, "url", "https://www.archlinux.org/mirrors/status/json/", "upstream mirror information URL")
	flag.DurationVar(&conf.interval, "interval", time.Hour, "auto update time in minutes")
	flag.Float64Var(&conf.minCompletion, "completion", 1.0, "minimum mirror completion threshold")
	flag.StringVar(&conf.host, "host", "0.0.0.0", "host to listen for connections on")
	flag.IntVar(&conf.port, "port", 9090, "port to listen for on")
}

func main() {
	// parse the command line flags
	flag.Parse()

	log.Print("Starting...")

	// we want atomic writes to the global mirror status data
	var status atomic.Value

	// run the autoupdater forever
	go update(&status, &conf)

	// handle the endpoints
	mux := http.NewServeMux()

	countryHandler := methodOnly("GET", makeCountryHandler(&status))
	globalHandler := methodOnly("GET", makeGlobalHandler(&status))
	mux.HandleFunc("/country/", countryHandler)
	mux.HandleFunc("/global", globalHandler)

	addr := fmt.Sprintf("%s:%d", conf.host, conf.port)

	log.Printf("Listening on %s", addr)

	// serve forever
	http.ListenAndServe(addr, mux)
}

// update performs a mirror status update whenever the ticker ticks, i.e.
// once per configured interval.
func update(status *atomic.Value, c *config) {
	log.Printf("Auto updater started (interval %s).", c.interval)
	ticker := time.NewTicker(c.interval)
	// perform an update operation once per tick forever
	for {
		log.Print("Performing auto update...")
		newM, err := getData(c)
		// we might recover next tick, so log the error and move on.
		if err != nil {
			log.Print(err)
			continue
		}
		// store the new configuration in a globally atomic operation
		status.Store(newM)
		log.Print("Auto update complete.")
		<-ticker.C
	}
}

// getData downloads and parses the mirror data from the configured URL. It also
// filters mirrors for completion percentage and
func getData(c *config) (*MirrorStatus, error) {
	log.Printf("Downloading mirror list from '%s'...", c.url)
	resp, err := http.Get(c.url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var m MirrorStatus

	// unmarshal the retrieved JSON data
	decoder := json.NewDecoder(resp.Body)
	err = decoder.Decode(&m)
	if err != nil {
		return nil, err
	}

	// nice reporting statistics

	log.Printf("Filtering mirrors with HTTP and completion>=%f...", c.minCompletion)
	totalCount, httpCount, completeCount := len(m.URLs), 0, 0
	newURLs := make([]*Mirror, 0, totalCount)

	// filter mirrors based on completion and protocol
	for _, url := range m.URLs {
		var isHTTP, isComplete bool

		if url.Protocol == "http" {
			httpCount++
			isHTTP = true
		}

		if url.Completion >= c.minCompletion {
			completeCount++
			isComplete = true
		}

		if isHTTP && isComplete {
			newURLs = append(newURLs, url)
		}
	}

	m.URLs = newURLs

	log.Printf("Mirror stats: Total: %d, HTTP: %d, Complete: %d. HTTP and Complete: %d", totalCount, httpCount, completeCount, len(newURLs))

	return &m, nil
}

// methodOnly restricts HTTP handlers to receiving a single HTTP method only.
func methodOnly(method string, next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.Method != method {
			http.Error(w, http.StatusText(405), 405)
			return
		}
		next.ServeHTTP(w, r)
	}
}

// makeGlobalHandler handles worldwide mirrors, and naïvely returns any valid
// mirror.
func makeGlobalHandler(val *atomic.Value) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		status := val.Load().(*MirrorStatus)

		// an empty mirror list is an error
		if len(status.URLs) == 0 {
			http.Error(w, http.StatusText(500), 500)
		}

		pick := rand.Intn(len(status.URLs))
		http.Redirect(w, r, status.URLs[pick].URL, http.StatusFound)
	}
}

// makeCountryHandler handlers mirrors within a two-letter country code and will
// fail gracefully (i.e. with appropriate HTTP error) when the provided country
// code is invalid or no valid mirrors are available for the country.
func makeCountryHandler(val *atomic.Value) http.HandlerFunc {
	// match against two digit country codes
	re := regexp.MustCompile(`/([A-Za-z]{2})$`)
	return func(w http.ResponseWriter, r *http.Request) {
		matches := re.FindStringSubmatch(r.URL.Path)

		// no match means a bogus country name
		if len(matches) == 0 {
			http.NotFound(w, r)
			return
		}

		// all mirror information contains uppercase country codes
		countryCode := strings.ToUpper(matches[1])

		status := val.Load().(*MirrorStatus)

		// an empty mirror list is an error
		if len(status.URLs) == 0 {
			http.Error(w, http.StatusText(500), 500)
		}

		// filter mirrors based on their country code
		res := make([]*Mirror, 0, len(status.URLs))

		for i, mirror := range status.URLs {
			if mirror.CountryCode == countryCode && mirror.Completion >= 1.0 {
				res = append(res, status.URLs[i])
			}
		}

		// an empty list means no mirrors were found
		if len(res) == 0 {
			http.NotFound(w, r)
			return
		}

		// choose a random mirror from the list and send the user to it
		pick := rand.Intn(len(res))
		http.Redirect(w, r, res[pick].URL, http.StatusFound)
	}
}
<commit_msg>refactored into single redirector handler<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"path"
	"regexp"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

var conf config

type selector func(*MirrorStatus, *http.Request) (*Mirror, error)

// config represents the application configuration
type config struct {
	url           string
	interval      time.Duration
	minCompletion float64
	host          string
	port          int
}

// Mirror is a description of an Arch Linux Mirror
type Mirror struct {
	Protocol    string     `json:"protocol"`
	URL         string     `json:"url"`
	Country     string     `json:"country"`
	LastSync    *time.Time `json:"last_sync"`
	Delay       int        `json:"delay"`
	Score       float64    `json:"score"`
	Completion  float64    `json:"completion_pct"`
	CountryCode string     `json:"country_code"`
	DurStdDev   float64    `json:"duration_stddev"`
	DurAvg      float64    `json:"duration_avg"`
}

// MirrorStatus is global status information for mirror checks
type MirrorStatus struct {
	Cutoff         int       `json:"cutoff"`
	CheckFrequency int       `json:"check_frequency"`
	NumChecks      int       `json:"num_checks"`
	LastCheck      time.Time `json:"last_check"`
	Version        int       `json:"version"`
	Global         []*Mirror `json:"urls"`
	Countries      map[string][]*Mirror
}

// ByScore implements sort.Interface for []*Mirror based on the Score field.
type ByScore []*Mirror

func (s ByScore) Len() int           { return len(s) }
func (s ByScore) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s ByScore) Less(i, j int) bool { return s[i].Score < s[j].Score }

func init() {
	flag.StringVar(&conf.url, "url", "https://www.archlinux.org/mirrors/status/json/", "upstream mirror information URL")
	flag.DurationVar(&conf.interval, "interval", time.Hour, "auto update time in minutes")
	flag.Float64Var(&conf.minCompletion, "completion", 1.0, "minimum mirror completion threshold")
	flag.StringVar(&conf.host, "host", "0.0.0.0", "host to listen for connections on")
	flag.IntVar(&conf.port, "port", 9090, "port to listen for on")
}

func main() {
	// parse the command line flags
	flag.Parse()

	log.Print("Starting...")

	// we want atomic writes to the global mirror status data
	var status atomic.Value
	done := make(chan bool)

	// run the autoupdater forever
	go update(&status, &conf, done)

	// handle the endpoints
	mux := http.NewServeMux()

	countryHandler := http.StripPrefix("/country", redirector(&status, countrySelector()))
	globalHandler := http.StripPrefix("/global", redirector(&status, globalSelector))
	mux.HandleFunc("/country/", countryHandler.(http.HandlerFunc))
	mux.HandleFunc("/global/", globalHandler.(http.HandlerFunc))

	addr := fmt.Sprintf("%s:%d", conf.host, conf.port)

	// serve forever
	<-done
	log.Printf("Init finished. Listening on %s", addr)
	http.ListenAndServe(addr, mux)
}

// update performs a mirror status update whenever the ticker ticks, i.e.
// once per configured interval.
func update(status *atomic.Value, c *config, done chan<- bool) {
	log.Printf("Auto updater started (interval %s).", c.interval)
	ticker := time.NewTicker(c.interval)
	var once sync.Once
	// perform an update operation once per tick forever
	for {
		log.Print("Performing auto update...")
		newM, err := getMirrorInfo(c)
		// we might recover next tick, so log the error and move on.
		if err != nil {
			log.Print(err)
			continue
		}
		// store the new configuration in a globally atomic operation
		status.Store(newM)
		log.Print("Auto update complete.")
		once.Do(func() {
			done <- true
			close(done)
		})
		<-ticker.C
	}
}

// getMirrorInfo downloads and parses the mirror data from the configured URL.
// It also filters mirrors for completion percentage and HTTP protocol.
func getMirrorInfo(c *config) (*MirrorStatus, error) {
	log.Printf("Downloading mirror list from '%s'...", c.url)
	resp, err := http.Get(c.url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var m MirrorStatus

	// unmarshal the retrieved JSON data
	decoder := json.NewDecoder(resp.Body)
	err = decoder.Decode(&m)
	if err != nil {
		return nil, err
	}

	// nice reporting statistics

	log.Printf("Filtering mirrors with HTTP and completion>=%f...", c.minCompletion)
	totalCount, httpCount, completeCount := len(m.Global), 0, 0
	newMirrors := make([]*Mirror, 0, totalCount)
	m.Countries = make(map[string][]*Mirror)

	sort.Stable(sort.Reverse(ByScore(m.Global)))

	// filter mirrors based on completion and protocol
	for _, mirror := range m.Global {
		var isHTTP, isComplete bool

		if mirror.Protocol == "http" {
			httpCount++
			isHTTP = true
		}

		if mirror.Completion >= c.minCompletion {
			completeCount++
			isComplete = true
		}

		if isHTTP && isComplete {
			newMirrors = append(newMirrors, mirror)
			country, ok := m.Countries[mirror.CountryCode]
			if !ok {
				country = make([]*Mirror, 0)
			}
			m.Countries[mirror.CountryCode] = append(country, mirror)
		}
	}

	m.Global = newMirrors

	log.Printf("Mirror stats: Total: %d, HTTP: %d, Complete: %d. HTTP and Complete: %d", totalCount, httpCount, completeCount, len(m.Global))

	return &m, nil
}

func redirector(status *atomic.Value, s selector) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "GET" {
			http.Error(w, http.StatusText(405), 405)
			return
		}

		c := status.Load().(*MirrorStatus)
		if len(c.Global) == 0 {
			http.Error(w, http.StatusText(500), 500)
			return
		}
		mirror, err := s(c, r)
		if err != nil {
			http.Error(w, err.Error(), 404)
			return
		}

		url, err := url.Parse(mirror.URL)
		if err != nil {
			http.Error(w, http.StatusText(500), 500)
			return
		}

		url.Path = path.Join(url.Path, r.URL.Path)

		http.Redirect(w, r, url.String(), 302)
	}
}

func globalSelector(status *MirrorStatus, r *http.Request) (*Mirror, error) {
	return status.Global[0], nil
}

func countrySelector() selector {
	re := regexp.MustCompile(`^/([a-z]{2})(?:/|$)`)
	return func(status *MirrorStatus, r *http.Request) (*Mirror, error) {
		res := re.FindStringSubmatch(r.URL.Path)
		if len(res) != 2 {
			return nil, fmt.Errorf("invalid country code %+v", res)
		}

		country, ok := status.Countries[strings.ToUpper(res[1])]
		if !ok {
			return nil, errors.New("country not found")
		}

		r.URL.Path = r.URL.Path[len(res[0]):]
		return country[0], nil
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/fatih/color"
	"github.com/spf13/cobra"
	"github.com/wataru0225/sreq/config"
	"github.com/wataru0225/sreq/snippet"
)

var pagenation int
var argument string
var editor string

type Qiita struct {
	Title string `json: "title"`
	Url   string `json: "url"`
	Body  string `json: "body"`
}

var searchCmd = &cobra.Command{
	Use:   "search",
	Short: "Search on Qiita",
	Long:  "Search on Qiita",
	Run: func(cmd *cobra.Command, args []string) {
		pagenation = 1
		argument = strings.Join(args, ",")
		execute()
	},
}

func init() {
	RootCmd.AddCommand(searchCmd)
	searchCmd.Flags().StringVar(&editor, "editor", "vim", "You choise editor")
	searchCmd.Flags().Bool("browse", false, "Use open browse")
}

func execute() {
	resp, err := http.Get("http://qiita.com/api/v2/items?page=" + strconv.Itoa(pagenation) + "&per_page=10&query=" + argument)
	if err == nil {
		defer resp.Body.Close()
		if b, err := ioutil.ReadAll(resp.Body); err == nil {
			rendering(b)
		}
	}
}

func rendering(b []byte) {
	var contents []Qiita
	json.Unmarshal(b, &contents)
	for i, c := range contents {
		fmt.Print(color.YellowString(strconv.Itoa(i) + " -> "))
		fmt.Println(c.Title)
		if count := len(c.Body); count > 256 {
			fmt.Println(color.GreenString(strings.Replace(c.Body, "\n", "", -1)[0:256]))
		} else {
			fmt.Println(color.GreenString(strings.Replace(c.Body, "\n", "", -1)))
		}
		fmt.Print("\n")
	}
	if len(contents) == 10 {
		fmt.Println(color.YellowString("n -> ") + "next page")
	}
	fmt.Print("SELECT > ")
	scan(contents)
}

func scan(content []Qiita) {
	var num string
	_, err := fmt.Scanf("%s", &num)
	if err == nil {
		if num == "n" {
			pagenation++
			execute()
		} else {
			numb, _ := strconv.Atoi(num)
			url, body := writeHistory(content[numb])

			var cfg config.Config
			cfg.Load()
			if cfg.General.OutputType == "editor" {
				text := []byte(body)
				ioutil.WriteFile("/tmp/sreq.txt", text, os.ModePerm)
				cmd := exec.Command(editor, "/tmp/sreq.txt")
				cmd.Stdin = os.Stdin
				cmd.Stdout = os.Stdout
				cmd.Run()
			} else if cfg.General.OutputType == "browse" {
				exec.Command("open", url).Run()
			}
			fmt.Println(err)
		}
	} else {
		fmt.Println(err)
	}
}

func writeHistory(content Qiita) (string, string) {
	var snippets snippet.Snippets
	file := config.HistoryFile()
	snippets.Load(file)
	url := content.Url
	newSnippet := snippet.SnippetInfo{
		SearchKeyword: argument,
		Url:           url,
		Title:         content.Title,
	}
	snippets.Snippets = append(snippets.Snippets, newSnippet)
	if err := snippets.Save(file); err != nil {
		fmt.Errorf("Failed. %v", err)
	}
	return url, content.Body
}
<commit_msg>Add search option chooising editor or browse<commit_after>package cmd

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/fatih/color"
	"github.com/spf13/cobra"
	"github.com/wataru0225/sreq/config"
	"github.com/wataru0225/sreq/snippet"
)

var pagenation int
var argument string
var editor string
var browse bool

type Qiita struct {
	Title string `json: "title"`
	Url   string `json: "url"`
	Body  string `json: "body"`
}

var searchCmd = &cobra.Command{
	Use:   "search",
	Short: "Search on Qiita",
	Long:  "Search on Qiita",
	Run: func(cmd *cobra.Command, args []string) {
		pagenation = 1
		argument = strings.Join(args, ",")
		execute()
	},
}

func init() {
	RootCmd.AddCommand(searchCmd)
	searchCmd.Flags().StringVar(&editor, "editor", "vim", "You choise editor")
	searchCmd.Flags().Bool("browse", false, "Use open browse")
}

func execute() {
	resp, err := http.Get("http://qiita.com/api/v2/items?page=" + strconv.Itoa(pagenation) + "&per_page=10&query=" + argument)
	if err == nil {
		defer resp.Body.Close()
		if b, err := ioutil.ReadAll(resp.Body); err == nil {
			rendering(b)
		}
	}
}

func rendering(b []byte) {
	var contents []Qiita
	json.Unmarshal(b, &contents)
	for i, c := range contents {
		fmt.Print(color.YellowString(strconv.Itoa(i) + " -> "))
		fmt.Println(c.Title)
		if count := len(c.Body); count > 256 {
			fmt.Println(color.GreenString(strings.Replace(c.Body, "\n", "", -1)[0:256]))
		} else {
			fmt.Println(color.GreenString(strings.Replace(c.Body, "\n", "", -1)))
		}
		fmt.Print("\n")
	}
	if len(contents) == 10 {
		fmt.Println(color.YellowString("n -> ") + "next page")
	}
	fmt.Print("SELECT > ")
	scan(contents)
}

func scan(content []Qiita) {
	var num string
	_, err := fmt.Scanf("%s", &num)
	if err == nil {
		if num == "n" {
			pagenation++
			execute()
		} else {
			numb, _ := strconv.Atoi(num)
			url, body := writeHistory(content[numb])

			var cfg config.Config
			cfg.Load()
			if editor == "" && browse == false {
				if cfg.General.OutputType == "editor" {
					editor = cfg.General.Editor
					execEditor(body, editor)
				} else if cfg.General.OutputType == "browse" {
					execBrowse(url)
				}
			} else if editor != "" {
				execEditor(body, editor)
			} else if browse == true {
				execBrowse(url)
			} else {
				fmt.Println("You should choice editor or browse")
			}
		}
	} else {
		fmt.Println(err)
	}
}

func execBrowse(url string) {
	exec.Command("open", url).Run()
}

func execEditor(body string, edotor string) {
	text := []byte(body)
	ioutil.WriteFile("/tmp/sreq.txt", text, os.ModePerm)
	cmd := exec.Command(editor, "/tmp/sreq.txt")
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Run()
}

func writeHistory(content Qiita) (string, string) {
	var snippets snippet.Snippets
	file := config.HistoryFile()
	snippets.Load(file)
	url := content.Url
	newSnippet := snippet.SnippetInfo{
		SearchKeyword: argument,
		Url:           url,
		Title:         content.Title,
	}
	snippets.Snippets = append(snippets.Snippets, newSnippet)
	if err := snippets.Save(file); err != nil {
		fmt.Errorf("Failed. %v", err)
	}
	return url, content.Body
}
<|endoftext|>