<commit_before>// Copyright (c) 2015, Peter Mrekaj. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE.txt file.

package graphs

type state byte

const (
	undiscovered state = iota
	discovered
	processed
)

// Vertex represents a Graph vertex with its connection edges to another vertices.
type Vertex struct {
	label string
	state state // Default is undiscovered.
	edges []*Vertex
}

func hasCycle(c, p *Vertex) bool {
	if c.state == discovered { // Base case.
		return true
	}

	c.state = discovered // In process.
	for _, n := range c.edges {
		if n != p && n.state != processed && hasCycle(n, c) {
			return true
		}
	}
	c.state = processed // Done.
	return false
}

// IsMinimallyConnected returns true if graph is minimally connected.
// The time complexity is O(v+e) where v is the number of vertices and e is the
// number of edges. However, if given graph is an undirected graph with no cycles
// then the time complexity is O(v). The O(v) additional space is needed.
func IsMinimallyConnected(graph []*Vertex) bool {
	if len(graph) == 0 {
		return true
	}
	return !hasCycle(graph[0], nil)
}
<commit_msg>Rename variables inside the graphs.hasCycle function in order to improve readability<commit_after>// Copyright (c) 2015, Peter Mrekaj. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE.txt file.

package graphs

type state byte

const (
	undiscovered state = iota
	discovered
	processed
)

// Vertex represents a Graph vertex with its connection edges to another vertices.
type Vertex struct {
	label string
	state state // Default is undiscovered.
	edges []*Vertex
}

func hasCycle(curr, prev *Vertex) bool {
	if curr.state == discovered { // Base case.
		return true
	}

	curr.state = discovered // In process.
	for _, next := range curr.edges {
		if next != prev && next.state != processed && hasCycle(next, curr) {
			return true
		}
	}
	curr.state = processed // Done.
	return false
}

// IsMinimallyConnected returns true if graph is minimally connected.
// The time complexity is O(v+e) where v is the number of vertices and e is the
// number of edges. However, if given graph is an undirected graph with no cycles
// then the time complexity is O(v). The O(v) additional space is needed.
func IsMinimallyConnected(graph []*Vertex) bool {
	if len(graph) == 0 {
		return true
	}
	return !hasCycle(graph[0], nil)
}
<|endoftext|>


<commit_before>package spscq

import (
	"errors"
	"fmt"
	"github.com/fmstephe/flib/fmath"
	"math/rand"
	"testing"
)

// Test that we can call newCommonQ(...) for every power of 2 in an int64
func TestNewCommonQPowerOf2(t *testing.T) {
	for size := int64(1); size <= maxSize; size *= 2 {
		_, err := newCommonQ(size)
		if err != nil {
			t.Errorf("Error found for size %d", size)
		}
	}
}

// Test that we can't call newCommonQ(...) with a non-power of 2 size
func TestNewCommonQNotPowerOf2(t *testing.T) {
	for size := int64(1); size < 10*1000; size++ {
		if !fmath.PowerOfTwo(size) {
			makeBadQ(size, t)
		}
	}
}

func makeBadQ(size int64, t *testing.T) {
	_, err := newCommonQ(size)
	if err == nil {
		t.Errorf("No error detected for size %d", size)
	}
}

func testAcquireWrite(writeBufferSize, from, to int64, cq, snap commonQ) error {
	actualWriteSize := to - from
	if actualWriteSize == 0 && cq.failedWrites.Value != snap.failedWrites.Value+1 {
		msg := "failedWrites not incremented. Expected %d, found %d"
		return errors.New(fmt.Sprintf(msg, snap.failedWrites.Value+1, cq.failedWrites.Value))
	}
	if actualWriteSize > writeBufferSize {
		msg := "Actual write size (%d) larger than requested buffer size (%d)"
		return errors.New(fmt.Sprintf(msg, actualWriteSize, writeBufferSize))
	}
	if (actualWriteSize < writeBufferSize) && (cq.write.Value+actualWriteSize) != (cq.read.Value+cq.size) {
		msg := "Actual write size (%d) could have been bigger.\nsnap %s\ncq  %s"
		return errors.New(fmt.Sprintf(msg, actualWriteSize, snap.String(), cq.String()))
	}
	if (cq.write.Value + actualWriteSize) > (cq.read.Value + cq.size) {
		msg := "Actual write size (%d) overwrites unread data.\ncq %s"
		return errors.New(fmt.Sprintf(msg, actualWriteSize, cq.String()))
	}
	if cq.writeSize.Value != actualWriteSize {
		msg := "cq.writeSize does not equal actual write size"
		return errors.New(fmt.Sprintf(msg, cq.writeSize, actualWriteSize))
	}
	if from > to {
		msg := "from (%d) is greater than to (%d)"
		return errors.New(fmt.Sprintf(msg, from, to))
	}
	if from >= cq.size || from < 0 {
		msg := "from (%d) must be a valid index for an array of size %d"
		return errors.New(fmt.Sprintf(msg, from, cq.size))
	}
	if to > cq.size || to < 0 {
		msg := "to (%d) must be a valid index for an array of size %d"
		return errors.New(fmt.Sprintf(msg, to, cq.size))
	}
	return nil
}

func testReleaseWrite(cq, snap commonQ) error {
	if cq.writeSize.Value != 0 {
		return errors.New(fmt.Sprintf("cq.writeSize was not reset to 0, %d found instead", cq.writeSize))
	}
	if cq.write.Value != snap.write.Value+snap.writeSize.Value {
		return errors.New(fmt.Sprintf("write has not been advanced by the correct amount.\nsnap %s\ncq  %s", snap.String(), cq.String()))
	}
	return nil
}

func testAcquireRead(readBufferSize, from, to int64, cq, snap commonQ) error {
	actualReadSize := to - from
	if actualReadSize == 0 && cq.failedReads.Value != snap.failedReads.Value+1 {
		msg := "failedReads not incremented. Expected %d, found %d"
		return errors.New(fmt.Sprintf(msg, snap.failedReads.Value+1, cq.failedReads.Value))
	}
	if actualReadSize > readBufferSize {
		msg := "Actual read size (%d) larger than requested buffer size (%d)"
		return errors.New(fmt.Sprintf(msg, actualReadSize, readBufferSize))
	}
	if (actualReadSize < readBufferSize) && (cq.read.Value+actualReadSize) != (cq.write.Value) {
		msg := "Actual read size (%d) could have been bigger.\nsnap %s\ncq  %s"
		return errors.New(fmt.Sprintf(msg, actualReadSize, snap.String(), cq.String()))
	}
	if (cq.read.Value + actualReadSize) > cq.write.Value {
		msg := "Actual read size (%d) reads past write position (%d).\ncq %s"
		return errors.New(fmt.Sprintf(msg, actualReadSize, cq.write.Value, cq.String()))
	}
	if cq.readSize.Value != actualReadSize {
		msg := "cq.readSize does not equal actual read size"
		return errors.New(fmt.Sprintf(msg, cq.readSize, actualReadSize))
	}
	if from > to {
		msg := "from (%d) is greater than to (%d)"
		return errors.New(fmt.Sprintf(msg, from, to))
	}
	if from >= cq.size || from < 0 {
		msg := "from (%d) must be a valid index for an array of size %d"
		return errors.New(fmt.Sprintf(msg, from, cq.size))
	}
	if to > cq.size || to < 0 {
		msg := "to (%d) must be a valid index for an array of size %d"
		return errors.New(fmt.Sprintf(msg, to, cq.size))
	}
	return nil
}

func testReleaseRead(cq, snap commonQ) error {
	if cq.readSize.Value != 0 {
		return errors.New(fmt.Sprintf("cq.readSize was not reset to 0, %d found instead", cq.readSize))
	}
	if cq.read.Value != snap.read.Value+snap.readSize.Value {
		return errors.New(fmt.Sprintf("read has not been advanced by the correct amount.\nsnap %s\ncq   %s", snap.String(), cq.String()))
	}
	return nil
}

func TestEvenReadWrites(t *testing.T) {
	rand.Seed(1)
	for i := uint64(0); i <= 41; i++ {
		size := int64(1 << i)
		cq, err := newCommonQ(size)
		if err != nil {
			t.Error(err.Error())
			continue
		}
		bufferSize := fmath.Max(size/128, 1)
		for j := int64(0); j < 1024; j++ {
			// write
			snap := cq
			wfrom, wto := cq.acquireWrite(bufferSize)
			if err := testAcquireWrite(bufferSize, wfrom, wto, cq, snap); err != nil {
				t.Error(err.Error())
				return
			}
			snap = cq
			cq.ReleaseWrite()
			if err := testReleaseWrite(cq, snap); err != nil {
				t.Error(err.Error())
				return
			}
			// read
			snap = cq
			rfrom, rto := cq.acquireRead(bufferSize)
			if err := testAcquireRead(bufferSize, rfrom, rto, cq, snap); err != nil {
				t.Error(err.Error())
				return
			}
			snap = cq
			cq.ReleaseRead()
			if err := testReleaseRead(cq, snap); err != nil {
				t.Error(err.Error())
				return
			}
		}
	}
}
<commit_msg>sqscq.commonQ tests for three scenarios<commit_after>package spscq

import (
	"errors"
	"fmt"
	"github.com/fmstephe/flib/fmath"
	"math/rand"
	"testing"
)

// Test that we can call newCommonQ(...) for every power of 2 in an int64
func TestNewCommonQPowerOf2(t *testing.T) {
	for size := int64(1); size <= maxSize; size *= 2 {
		_, err := newCommonQ(size)
		if err != nil {
			t.Errorf("Error found for size %d", size)
		}
	}
}

// Test that we can't call newCommonQ(...) with a non-power of 2 size
func TestNewCommonQNotPowerOf2(t *testing.T) {
	for size := int64(1); size < 10*1000; size++ {
		if !fmath.PowerOfTwo(size) {
			makeBadQ(size, t)
		}
	}
}

func makeBadQ(size int64, t *testing.T) {
	_, err := newCommonQ(size)
	if err == nil {
		t.Errorf("No error detected for size %d", size)
	}
}

func testAcquireWrite(writeBufferSize, from, to int64, cq, snap commonQ) error {
	actualWriteSize := to - from
	if actualWriteSize == 0 && cq.failedWrites.Value != snap.failedWrites.Value+1 {
		msg := "failedWrites not incremented. Expected %d, found %d"
		return errors.New(fmt.Sprintf(msg, snap.failedWrites.Value+1, cq.failedWrites.Value))
	}
	if actualWriteSize > writeBufferSize {
		msg := "Actual write size (%d) larger than requested buffer size (%d)"
		return errors.New(fmt.Sprintf(msg, actualWriteSize, writeBufferSize))
	}
	if (actualWriteSize < writeBufferSize) && (cq.write.Value+actualWriteSize) != (cq.read.Value+cq.size) {
		msg := "Actual write size (%d) could have been bigger.\nsnap %s\ncq  %s"
		return errors.New(fmt.Sprintf(msg, actualWriteSize, snap.String(), cq.String()))
	}
	if (cq.write.Value + actualWriteSize) > (cq.read.Value + cq.size) {
		msg := "Actual write size (%d) overwrites unread data.\ncq %s"
		return errors.New(fmt.Sprintf(msg, actualWriteSize, cq.String()))
	}
	if cq.writeSize.Value != actualWriteSize {
		msg := "cq.writeSize does not equal actual write size"
		return errors.New(fmt.Sprintf(msg, cq.writeSize, actualWriteSize))
	}
	if from > to {
		msg := "from (%d) is greater than to (%d)"
		return errors.New(fmt.Sprintf(msg, from, to))
	}
	if from >= cq.size || from < 0 {
		msg := "from (%d) must be a valid index for an array of size %d"
		return errors.New(fmt.Sprintf(msg, from, cq.size))
	}
	if to > cq.size || to < 0 {
		msg := "to (%d) must be a valid index for an array of size %d"
		return errors.New(fmt.Sprintf(msg, to, cq.size))
	}
	return nil
}

func testReleaseWrite(cq, snap commonQ) error {
	if cq.writeSize.Value != 0 {
		return errors.New(fmt.Sprintf("cq.writeSize was not reset to 0, %d found instead", cq.writeSize))
	}
	if cq.write.Value != snap.write.Value+snap.writeSize.Value {
		return errors.New(fmt.Sprintf("write has not been advanced by the correct amount.\nsnap %s\ncq  %s", snap.String(), cq.String()))
	}
	return nil
}

func testAcquireRead(readBufferSize, from, to int64, cq, snap commonQ) error {
	actualReadSize := to - from
	if actualReadSize == 0 && cq.failedReads.Value != snap.failedReads.Value+1 {
		msg := "failedReads not incremented. Expected %d, found %d"
		return errors.New(fmt.Sprintf(msg, snap.failedReads.Value+1, cq.failedReads.Value))
	}
	if actualReadSize > readBufferSize {
		msg := "Actual read size (%d) larger than requested buffer size (%d)"
		return errors.New(fmt.Sprintf(msg, actualReadSize, readBufferSize))
	}
	if (actualReadSize < readBufferSize) && (cq.read.Value+actualReadSize) != (cq.write.Value) {
		msg := "Actual read size (%d) could have been bigger.\nsnap %s\ncq  %s"
		return errors.New(fmt.Sprintf(msg, actualReadSize, snap.String(), cq.String()))
	}
	if (cq.read.Value + actualReadSize) > cq.write.Value {
		msg := "Actual read size (%d) reads past write position (%d).\ncq %s"
		return errors.New(fmt.Sprintf(msg, actualReadSize, cq.write.Value, cq.String()))
	}
	if cq.readSize.Value != actualReadSize {
		msg := "cq.readSize does not equal actual read size"
		return errors.New(fmt.Sprintf(msg, cq.readSize, actualReadSize))
	}
	if from > to {
		msg := "from (%d) is greater than to (%d)"
		return errors.New(fmt.Sprintf(msg, from, to))
	}
	if from >= cq.size || from < 0 {
		msg := "from (%d) must be a valid index for an array of size %d"
		return errors.New(fmt.Sprintf(msg, from, cq.size))
	}
	if to > cq.size || to < 0 {
		msg := "to (%d) must be a valid index for an array of size %d"
		return errors.New(fmt.Sprintf(msg, to, cq.size))
	}
	return nil
}

func testReleaseRead(cq, snap commonQ) error {
	if cq.readSize.Value != 0 {
		return errors.New(fmt.Sprintf("cq.readSize was not reset to 0, %d found instead", cq.readSize))
	}
	if cq.read.Value != snap.read.Value+snap.readSize.Value {
		return errors.New(fmt.Sprintf("read has not been advanced by the correct amount.\nsnap %s\ncq   %s", snap.String(), cq.String()))
	}
	return nil
}

func TestEvenWriteRead(t *testing.T) {
	rand.Seed(1)
	for i := uint(0); i <= 41; i++ {
		size := int64(1 << i)
		bufferSize := fmath.Max(size/128, 1)
		testSymmetricReadWrites(t, size, bufferSize, bufferSize, 1024)
	}
}

func TestLightWriteHeavyRead(t *testing.T) {
	rand.Seed(1)
	for i := uint(0); i <= 41; i++ {
		size := int64(1 << i)
		bufferSize := fmath.Max(size/128, 1)
		testSymmetricReadWrites(t, size, bufferSize, bufferSize*2, 1024)
	}
}

func TestHeavyWriteLightRead(t *testing.T) {
	rand.Seed(1)
	for i := uint(0); i <= 41; i++ {
		size := int64(1 << i)
		bufferSize := fmath.Max(size/128, 1)
		testSymmetricReadWrites(t, size, bufferSize*2, bufferSize, 1024)
	}
}

func testSymmetricReadWrites(t *testing.T, size int64, writeSize, readSize, iterations int64) {
	cq, err := newCommonQ(size)
	if err != nil {
		t.Error(err.Error())
		return
	}
	for j := int64(0); j < iterations; j++ {
		// write
		snap := cq
		wfrom, wto := cq.acquireWrite(writeSize)
		if err := testAcquireWrite(writeSize, wfrom, wto, cq, snap); err != nil {
			t.Error(err.Error())
			return
		}
		snap = cq
		cq.ReleaseWrite()
		if err := testReleaseWrite(cq, snap); err != nil {
			t.Error(err.Error())
			return
		}
		// read
		snap = cq
		rfrom, rto := cq.acquireRead(readSize)
		if err := testAcquireRead(readSize, rfrom, rto, cq, snap); err != nil {
			t.Error(err.Error())
			return
		}
		snap = cq
		cq.ReleaseRead()
		if err := testReleaseRead(cq, snap); err != nil {
			t.Error(err.Error())
			return
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package query

import "github.com/square/metrics/api"

// Command is the final result of the parsing.
// A command contains all the information to execute the
// given query against the API.
type Command interface {
	// Execute the given command. Returns JSON-encodable result or an error.
	Execute(b api.Backend) (interface{}, error)
}

// DescribeCommand describes the tag set managed by the given metric indexer.
type DescribeCommand struct {
	metricName api.MetricKey
	predicate  api.Predicate
}

// DescribeAllCommand returns all the metrics available in the system.
type DescribeAllCommand struct {
}

// SelectCommand is the bread and butter of the metrics query engine.
// It actually performs the query against the underlying metrics system.
type SelectCommand struct {
	predicate   api.Predicate
	expressions []Expression
	context     *evaluationContextNode
}

// Execute returns the list of tags satisfying the provided predicate.
func (cmd *DescribeCommand) Execute(b api.Backend) (interface{}, error) {
	tags, _ := b.Api().GetAllTags(cmd.metricName)
	output := make([]string, 0, len(tags))
	for _, tag := range tags {
		if cmd.predicate.Apply(tag) {
			output = append(output, tag.Serialize())
		}
	}
	return output, nil
}

// Execute of a DescribeAllCommand returns the list of all metrics.
func (cmd *DescribeAllCommand) Execute(b api.Backend) (interface{}, error) {
	return b.Api().GetAllMetrics()
}

// Execute performs the query represented by the given query string, and returs the result.
func (cmd *SelectCommand) Execute(b api.Backend) (interface{}, error) {
	timerange, err := api.NewTimerange(cmd.context.Start, cmd.context.End, cmd.context.Resolution)
	if err != nil {
		return nil, err
	}
	return evaluateExpressions(EvaluationContext{
		Backend:      b,
		Timerange:    *timerange,
		SampleMethod: cmd.context.SampleMethod,
		Predicate:    cmd.predicate,
	}, cmd.expressions)
}
<commit_msg>sorting the results for describe all<commit_after>// Copyright 2015 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package query

import (
	"github.com/square/metrics/api"
	"sort"
)

// Command is the final result of the parsing.
// A command contains all the information to execute the
// given query against the API.
type Command interface {
	// Execute the given command. Returns JSON-encodable result or an error.
	Execute(b api.Backend) (interface{}, error)
}

// DescribeCommand describes the tag set managed by the given metric indexer.
type DescribeCommand struct {
	metricName api.MetricKey
	predicate  api.Predicate
}

// DescribeAllCommand returns all the metrics available in the system.
type DescribeAllCommand struct {
}

// SelectCommand is the bread and butter of the metrics query engine.
// It actually performs the query against the underlying metrics system.
type SelectCommand struct {
	predicate   api.Predicate
	expressions []Expression
	context     *evaluationContextNode
}

// Execute returns the list of tags satisfying the provided predicate.
func (cmd *DescribeCommand) Execute(b api.Backend) (interface{}, error) {
	tags, _ := b.Api().GetAllTags(cmd.metricName)
	output := make([]string, 0, len(tags))
	for _, tag := range tags {
		if cmd.predicate.Apply(tag) {
			output = append(output, tag.Serialize())
		}
	}
	sort.Strings(output)
	return output, nil
}

// Execute of a DescribeAllCommand returns the list of all metrics.
func (cmd *DescribeAllCommand) Execute(b api.Backend) (interface{}, error) {
	result, err := b.Api().GetAllMetrics()
	if err == nil {
		sort.Sort(api.MetricKeys(result))
	}
	return result, err
}

// Execute performs the query represented by the given query string, and returs the result.
func (cmd *SelectCommand) Execute(b api.Backend) (interface{}, error) {
	timerange, err := api.NewTimerange(cmd.context.Start, cmd.context.End, cmd.context.Resolution)
	if err != nil {
		return nil, err
	}
	return evaluateExpressions(EvaluationContext{
		Backend:      b,
		Timerange:    *timerange,
		SampleMethod: cmd.context.SampleMethod,
		Predicate:    cmd.predicate,
	}, cmd.expressions)
}
<|endoftext|>


<commit_before>package carto

import (
	"reflect"
	"testing"

	"github.com/mathuin/gdal"
)

var buildMap_tests = []struct {
	name   string
	elname string
	lcname string
	ll     FloatExtents
	histos []RasterHInfo
}{
	{
		"BlockIsland", "elevation.tif", "landcover.tif",
		FloatExtents{-71.575, -71.576, 41.189, 41.191},
		[]RasterHInfo{
			RasterHInfo{"Int16", map[int]int{31: 2083, 90: 6713, 43: 2322, 22: 757, 42: 1061, 11: 49617, 95: 358, 41: 316, 21: 2005, 23: 304}},
			RasterHInfo{"Int16", map[int]int{62: 63631, 63: 1860, 64: 45}},
			RasterHInfo{"Int16", map[int]int{7: 644, 6: 732, 5: 688, 15: 591, 27: 568, 13: 729, 12: 603, 10: 608, 3: 699, 0: 15919, 19: 641, 21: 728, 22: 637, 24: 580, 29: 567, 16: 741, 11: 720, 9: 791, 18: 660, 20: 583, 4: 896, 17: 675, 28: 655, 14: 666, 8: 678, 2: 743, 26: 607, 30: 29983, 1: 904, 23: 651, 25: 649}},
			RasterHInfo{"Int16", map[int]int{1: 13163, 2: 39217, 3: 12367, 4: 789}},
		},
	},
}

func Test_buildMap(t *testing.T) {
	for _, tt := range buildMap_tests {
		r := MakeRegion(tt.name, tt.ll, tt.elname, tt.lcname)
		r.tilesize = 16
		// Debug = true
		r.BuildMap()
		// Debug = false

		// check the raster minmaxes
		ds, err := gdal.Open(r.mapfile, gdal.ReadOnly)
		if err != nil {
			t.Fail()
		}
		if Debug {
			datasetInfo(ds, "Test")
		}

		histos := datasetHistograms(ds)
		if len(histos) != len(tt.histos) {
			t.Fatalf("len(histos) %d != len(tt.histos) %d", len(histos), len(tt.histos))
		}
		for i, v := range histos {
			if tt.histos[i].datatype != v.datatype {
				t.Errorf("Raster #%d: expected datatype \"%s\", got \"%s\"", i+1, tt.histos[i].datatype, v.datatype)
			}
			if !reflect.DeepEqual(tt.histos[i].buckets, v.buckets) {
				// JMT: crust raster is expected to vary
				if i != 3 {
					t.Errorf("Raster #%d: expected buckets \"%+#v\", got \"%+#v\"", i+1, tt.histos[i].buckets, v.buckets)
				}
			}
		}
	}
}
<commit_msg>Cleaned up test results.<commit_after>package carto

import (
	"reflect"
	"testing"

	"github.com/mathuin/gdal"
)

var buildMap_tests = []struct {
	name   string
	elname string
	lcname string
	ll     FloatExtents
	histos []RasterHInfo
}{
	{
		"BlockIsland", "elevation.tif", "landcover.tif",
		FloatExtents{-71.575, -71.576, 41.189, 41.191},
		[]RasterHInfo{
			// landcover
			RasterHInfo{"Int16", map[int]int{
				11: 49727,
				21: 1987,
				22: 734,
				23: 278,
				31: 2019,
				41: 298,
				43: 2317,
				42: 1067,
				90: 6767,
				95: 342,
			}},
			// elevation
			RasterHInfo{"Int16", map[int]int{
				62: 63631,
				63: 1860,
				64: 45,
			}},
			// bathy
			RasterHInfo{"Int16", map[int]int{
				0:  15809,
				1:  916,
				2:  729,
				3:  694,
				4:  893,
				5:  679,
				6:  747,
				7:  641,
				8:  689,
				9:  766,
				10: 595,
				11: 728,
				12: 595,
				13: 737,
				14: 654,
				15: 588,
				16: 750,
				17: 667,
				18: 671,
				19: 635,
				20: 580,
				21: 732,
				22: 638,
				23: 663,
				24: 578,
				25: 655,
				26: 599,
				27: 569,
				28: 651,
				29: 570,
				30: 30118,
			}},
			// crust -- may change
			RasterHInfo{"Int16", map[int]int{
				1: 13163,
				2: 39217,
				3: 12367,
				4: 789,
			}},
		},
	},
}

func Test_buildMap(t *testing.T) {
	for _, tt := range buildMap_tests {
		r := MakeRegion(tt.name, tt.ll, tt.elname, tt.lcname)
		r.tilesize = 16
		// Debug = true
		r.BuildMap()
		// Debug = false

		// check the raster minmaxes
		ds, err := gdal.Open(r.mapfile, gdal.ReadOnly)
		if err != nil {
			t.Fail()
		}
		if Debug {
			datasetInfo(ds, "Test")
		}

		histos := datasetHistograms(ds)
		if len(histos) != len(tt.histos) {
			t.Fatalf("len(histos) %d != len(tt.histos) %d", len(histos), len(tt.histos))
		}
		for i, v := range histos {
			if tt.histos[i].datatype != v.datatype {
				t.Errorf("Raster #%d: expected datatype \"%s\", got \"%s\"", i+1, tt.histos[i].datatype, v.datatype)
			}
			if !reflect.DeepEqual(tt.histos[i].buckets, v.buckets) {
				// JMT: crust raster is expected to vary
				if i != 3 {
					t.Errorf("Raster #%d: expected buckets \"%+#v\", got \"%+#v\"", i+1, tt.histos[i].buckets, v.buckets)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html"
	"log"
	"net/http"
)

func main() {
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "Hello, %q", html.EscapeString(r.URL.Path))
	})

	log.Fatal(http.ListenAndServe(":8080", nil))

}
<commit_msg>post fix<commit_after>package main

import (
	"fmt"
	"html"
	"log"
	"net/http"
)

func main() {
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "Hello, %q", html.EscapeString(r.URL.Path))
	})

	log.Fatal(http.ListenAndServe(":80", nil))

}
<|endoftext|>


<commit_before>// All this package really does is add a little structure and a little color
// to the standard log package

package logsip

import (
	"fmt"
	"io"
	"log"
	"os"

	"github.com/fatih/color"
)

type Logger struct {
	WarnPrefix  string
	FatalPrefix string
	InfoPrefix  string
	PanicPrefix string
	*log.Logger
}

func New(out io.Writer) *Logger {
	return &Logger{Logger: log.New(out, "", 0)}
}

func Default() *Logger {
	return &Logger{
		WarnPrefix:  color.New(color.FgYellow).SprintFunc()("==> Warn: "),
		InfoPrefix:  color.New(color.FgCyan).SprintFunc()("==> Info: "),
		FatalPrefix: color.New(color.FgRed).SprintFunc()("==> Fatal: "),
		PanicPrefix: color.New(color.FgRed).SprintFunc()("==> Panic: "),
		Logger:      log.New(os.Stdout, "", 0),
	}
}

func (l *Logger) Info(v ...interface{}) {
	l.Print(l.InfoPrefix + fmt.Sprint(v...))
}
func (l *Logger) Warn(v ...interface{}) {
	l.Print(l.WarnPrefix + fmt.Sprint(v...))
}
func (l *Logger) Fatal(v ...interface{}) {
	l.Print(l.FatalPrefix + fmt.Sprint(v...))
	os.Exit(1)
}

func (l *Logger) Panic(v ...interface{}) {
	l.Print(l.PanicPrefix + fmt.Sprint(v...))
	panic(l)
}

func (l *Logger) Infof(format string, v ...interface{}) {
	l.Print(l.InfoPrefix + fmt.Sprintf(format, v...))
}

func (l *Logger) Warnf(format string, v ...interface{}) {
	l.Print(l.WarnPrefix + fmt.Sprintf(format, v...))
}
func (l *Logger) Fatalf(format string, v ...interface{}) {
	l.Print(l.FatalPrefix + fmt.Sprintf(format, v...))
	os.Exit(1)
}
func (l *Logger) Panicf(format string, v ...interface{}) {
	l.Print(l.PanicPrefix + fmt.Sprintf(format, v...))
	panic(l)
}

func (l *Logger) Infoln(v ...interface{}) {
	l.Println(l.InfoPrefix + fmt.Sprint(v...))
}

func (l *Logger) Warnln(v ...interface{}) {
	l.Println(l.WarnPrefix + fmt.Sprint(v...))
}

func (l *Logger) Fatalln(v ...interface{}) {
	l.Println(l.FatalPrefix + fmt.Sprint(v...))
	os.Exit(1)
}

func (l *Logger) Panicln(v ...interface{}) {
	l.Println(l.PanicPrefix + fmt.Sprint(v...))
	panic(l)
}
<commit_msg>Fixed some spacing issues<commit_after>// All this package really does is add a little structure and a little color
// to the standard log package

package logsip

import (
	"fmt"
	"io"
	"log"
	"os"

	"github.com/fatih/color"
)

type Logger struct {
	WarnPrefix  string
	FatalPrefix string
	InfoPrefix  string
	PanicPrefix string
	*log.Logger
}

func New(out io.Writer) *Logger {
	return &Logger{Logger: log.New(out, "", 0)}
}

func Default() *Logger {
	return &Logger{
		WarnPrefix:  color.New(color.FgYellow).SprintFunc()("==> Warn: "),
		InfoPrefix:  color.New(color.FgCyan).SprintFunc()("==> Info: "),
		FatalPrefix: color.New(color.FgRed).SprintFunc()("==> Fatal: "),
		PanicPrefix: color.New(color.FgRed).SprintFunc()("==> Panic: "),
		Logger:      log.New(os.Stdout, "", 0),
	}
}

func (l *Logger) Info(v ...interface{}) {
	l.Print(l.InfoPrefix + fmt.Sprint(v...))
}
func (l *Logger) Warn(v ...interface{}) {
	l.Print(l.WarnPrefix + fmt.Sprint(v...))
}
func (l *Logger) Fatal(v ...interface{}) {
	l.Print(l.FatalPrefix + fmt.Sprint(v...))
	os.Exit(1)
}
func (l *Logger) Panic(v ...interface{}) {
	l.Print(l.PanicPrefix + fmt.Sprint(v...))
	panic(l)
}
func (l *Logger) Infof(format string, v ...interface{}) {
	l.Print(l.InfoPrefix + fmt.Sprintf(format, v...))
}
func (l *Logger) Warnf(format string, v ...interface{}) {
	l.Print(l.WarnPrefix + fmt.Sprintf(format, v...))
}
func (l *Logger) Fatalf(format string, v ...interface{}) {
	l.Print(l.FatalPrefix + fmt.Sprintf(format, v...))
	os.Exit(1)
}
func (l *Logger) Panicf(format string, v ...interface{}) {
	l.Print(l.PanicPrefix + fmt.Sprintf(format, v...))
	panic(l)
}
func (l *Logger) Infoln(v ...interface{}) {
	l.Println(l.InfoPrefix + fmt.Sprint(v...))
}
func (l *Logger) Warnln(v ...interface{}) {
	l.Println(l.WarnPrefix + fmt.Sprint(v...))
}
func (l *Logger) Fatalln(v ...interface{}) {
	l.Println(l.FatalPrefix + fmt.Sprint(v...))
	os.Exit(1)
}
func (l *Logger) Panicln(v ...interface{}) {
	l.Println(l.PanicPrefix + fmt.Sprint(v...))
	panic(l)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"

	"knative.dev/pkg/apis"
	"knative.dev/pkg/apis/duck"
	"knative.dev/pkg/tracker"
)

// +genduck
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// Binding is a duck type that specifies the partial schema to which all
// Binding implementations should adhere.
type Binding struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`

	Spec BindingSpec `json:"status"`
}

// Verify that Binding implements the appropriate interfaces.
var (
	_ duck.Implementable = (*Binding)(nil)
	_ duck.Populatable   = (*Binding)(nil)
	_ apis.Listable      = (*Binding)(nil)
)

// BindingSpec specifies the spec portion of the Binding partial-schema.
type BindingSpec struct {
	// Subject references the resource(s) whose "runtime contract" should be
	// augmented by Binding implementations.
	Subject tracker.Reference `json:"subject"`
}

// GetFullType implements duck.Implementable
func (*Binding) GetFullType() duck.Populatable {
	return &Binding{}
}

// Populate implements duck.Populatable
func (t *Binding) Populate() {
	t.Spec = BindingSpec{
		Subject: tracker.Reference{
			APIVersion: "apps/v1",
			Kind:       "Deployment",
			Namespace:  "default",
			// Name and Selector are mutually exclusive,
			// but we fill them both in for this test.
			Name: "bazinga",
			Selector: &metav1.LabelSelector{
				MatchLabels: map[string]string{
					"foo": "bar",
					"baz": "blah",
				},
			},
		},
	}
}

// GetListType implements apis.Listable
func (*Binding) GetListType() runtime.Object {
	return &BindingList{}
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// BindingList is a list of Binding resources
type BindingList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata"`

	Items []Binding `json:"items"`
}
<commit_msg>Fix the serialized name of the spec field (oops) (#899)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"

	"knative.dev/pkg/apis"
	"knative.dev/pkg/apis/duck"
	"knative.dev/pkg/tracker"
)

// +genduck
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// Binding is a duck type that specifies the partial schema to which all
// Binding implementations should adhere.
type Binding struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`

	Spec BindingSpec `json:"spec"`
}

// Verify that Binding implements the appropriate interfaces.
var (
	_ duck.Implementable = (*Binding)(nil)
	_ duck.Populatable   = (*Binding)(nil)
	_ apis.Listable      = (*Binding)(nil)
)

// BindingSpec specifies the spec portion of the Binding partial-schema.
type BindingSpec struct {
	// Subject references the resource(s) whose "runtime contract" should be
	// augmented by Binding implementations.
	Subject tracker.Reference `json:"subject"`
}

// GetFullType implements duck.Implementable
func (*Binding) GetFullType() duck.Populatable {
	return &Binding{}
}

// Populate implements duck.Populatable
func (t *Binding) Populate() {
	t.Spec = BindingSpec{
		Subject: tracker.Reference{
			APIVersion: "apps/v1",
			Kind:       "Deployment",
			Namespace:  "default",
			// Name and Selector are mutually exclusive,
			// but we fill them both in for this test.
			Name: "bazinga",
			Selector: &metav1.LabelSelector{
				MatchLabels: map[string]string{
					"foo": "bar",
					"baz": "blah",
				},
			},
		},
	}
}

// GetListType implements apis.Listable
func (*Binding) GetListType() runtime.Object {
	return &BindingList{}
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// BindingList is a list of Binding resources
type BindingList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata"`

	Items []Binding `json:"items"`
}
<|endoftext|>


<commit_before>package base

/* CHECKLIST
 * [ ] Uses interfaces as appropriate
 * [ ] Private package variables use underscore prefix
 * [ ] All parameters validated
 * [ ] All errors handled
 * [ ] Reviewed for concurrency safety
 * [ ] Code complete
 * [ ] Full test coverage
 */

import (
	"github.com/tidepool-org/platform/app"
	"github.com/tidepool-org/platform/data"
)

const SchemaVersionCurrent = 3

type Base struct {
	Active         bool   `json:"-" bson:"_active"`
	CreatedTime    string `json:"createdTime,omitempty" bson:"createdTime,omitempty"`
	CreatedUserID  string `json:"createdUserId,omitempty" bson:"createdUserId,omitempty"`
	DeletedTime    string `json:"deletedTime,omitempty" bson:"deletedTime,omitempty"`
	DeletedUserID  string `json:"deletedUserId,omitempty" bson:"deletedUserId,omitempty"`
	GroupID        string `json:"-" bson:"_groupId,omitempty"`
	GUID           string `json:"guid,omitempty" bson:"guid,omitempty"`
	ID             string `json:"id,omitempty" bson:"id,omitempty"`
	ModifiedTime   string `json:"modifiedTime,omitempty" bson:"modifiedTime,omitempty"`
	ModifiedUserID string `json:"modifiedUserId,omitempty" bson:"modifiedUserId,omitempty"`
	SchemaVersion  int    `json:"-" bson:"_schemaVersion,omitempty"`
	Type           string `json:"type,omitempty" bson:"type,omitempty"`
	UploadID       string `json:"uploadId,omitempty" bson:"uploadId,omitempty"`
	UserID         string `json:"-" bson:"_userId,omitempty"`
	Version        int    `json:"-" bson:"_version,omitempty"`

	Annotations      *[]interface{} `json:"annotations,omitempty" bson:"annotations,omitempty"`
	ClockDriftOffset *int           `json:"clockDriftOffset,omitempty" bson:"clockDriftOffset,omitempty"`
	ConversionOffset *int           `json:"conversionOffset,omitempty" bson:"conversionOffset,omitempty"`
	DeviceID         *string        `json:"deviceId,omitempty" bson:"deviceId,omitempty"`
	DeviceTime       *string        `json:"deviceTime,omitempty" bson:"deviceTime,omitempty"`
	Payload          *interface{}   `json:"payload,omitempty" bson:"payload,omitempty"`
	Time             *string        `json:"time,omitempty" bson:"time,omitempty"`
	TimezoneOffset   *int           `json:"timezoneOffset,omitempty" bson:"timezoneOffset,omitempty"`
}

type Meta struct {
	Type string `json:"type,omitempty"`
}

func (b *Base) Init() {
	b.Active = false
	b.CreatedTime = ""
	b.CreatedUserID = ""
	b.DeletedTime = ""
	b.DeletedUserID = ""
	b.GroupID = ""
	b.GUID = app.NewUUID()
	b.ID = app.NewID() // TODO: Move calculation to Normalize to follow Jellyfish algorithm
	b.ModifiedTime = ""
	b.ModifiedUserID = ""
	b.SchemaVersion = SchemaVersionCurrent
	b.Type = ""
	b.UploadID = ""
	b.UserID = ""
	b.Version = 0

	b.Annotations = nil
	b.ClockDriftOffset = nil
	b.ConversionOffset = nil
	b.DeviceID = nil
	b.DeviceTime = nil
	b.Payload = nil
	b.Time = nil
	b.TimezoneOffset = nil
}

func (b *Base) Meta() interface{} {
	return &Meta{
		Type: b.Type,
	}
}

func (b *Base) Parse(parser data.ObjectParser) error {
	b.Annotations = parser.ParseInterfaceArray("annotations")
	b.ClockDriftOffset = parser.ParseInteger("clockDriftOffset")
	b.ConversionOffset = parser.ParseInteger("conversionOffset")
	b.DeviceID = parser.ParseString("deviceId")
	b.DeviceTime = parser.ParseString("deviceTime")
	b.Payload = parser.ParseInterface("payload")
	b.Time = parser.ParseString("time")
	b.TimezoneOffset = parser.ParseInteger("timezoneOffset")

	return nil
}

func (b *Base) Validate(validator data.Validator) error {
	validator.ValidateString("type", &b.Type).Exists().NotEmpty()

	// validator.ValidateInterfaceArray("annotations", b.Annotations)    // TODO: Any validations? Optional? Size?
	// validator.ValidateInteger("clockDriftOffset", b.ClockDriftOffset) // TODO: Any validations? Optional? Range?
	// validator.ValidateInteger("conversionOffset", b.ConversionOffset) // TODO: Any validations? Optional? Range?
	validator.ValidateString("deviceId", b.DeviceID).Exists().NotEmpty()
	validator.ValidateStringAsTime("deviceTime", b.DeviceTime, "2006-01-02T15:04:05") // TODO: Not in upload!  -> .Exists()
	// validator.ValidateInterface("payload", b.Payload) // TODO: Any validations? Optional? Size?
	validator.ValidateStringAsTime("time", b.Time, "2006-01-02T15:04:05Z07:00").Exists()
	// validator.ValidateInteger("timezoneOffset", b.TimezoneOffset) // TODO: Any validations? Optional? Range?

	// TODO: NOT IN UPLOAD: annotations, clockDriftOffset, deviceTime, payload

	return nil
}

func (b *Base) Normalize(normalizer data.Normalizer) error {
	return nil
}

func (b *Base) SetUserID(userID string) {
	b.UserID = userID
}

func (b *Base) SetGroupID(groupID string) {
	b.GroupID = groupID
}

func (b *Base) SetDatasetID(datasetID string) {
	b.UploadID = datasetID
}

func (b *Base) SetActive(active bool) {
	b.Active = active
}

func (b *Base) SetCreatedTime(createdTime string) {
	b.CreatedTime = createdTime
}

func (b *Base) SetCreatedUserID(createdUserID string) {
	b.CreatedUserID = createdUserID
}

func (b *Base) SetModifiedTime(modifiedTime string) {
	b.ModifiedTime = modifiedTime
}

func (b *Base) SetModifiedUserID(modifiedUserID string) {
	b.ModifiedUserID = modifiedUserID
}

func (b *Base) SetDeletedTime(deletedTime string) {
	b.DeletedTime = deletedTime
}

func (b *Base) SetDeletedUserID(deletedUserID string) {
	b.DeletedUserID = deletedUserID
}
<commit_msg>Add base Source for backwards compatibility with Carelink<commit_after>package base

/* CHECKLIST
 * [ ] Uses interfaces as appropriate
 * [ ] Private package variables use underscore prefix
 * [ ] All parameters validated
 * [ ] All errors handled
 * [ ] Reviewed for concurrency safety
 * [ ] Code complete
 * [ ] Full test coverage
 */

import (
	"github.com/tidepool-org/platform/app"
	"github.com/tidepool-org/platform/data"
)

const SchemaVersionCurrent = 3

type Base struct {
	Active         bool   `json:"-" bson:"_active"`
	CreatedTime    string `json:"createdTime,omitempty" bson:"createdTime,omitempty"`
	CreatedUserID  string `json:"createdUserId,omitempty" bson:"createdUserId,omitempty"`
	DeletedTime    string `json:"deletedTime,omitempty" bson:"deletedTime,omitempty"`
	DeletedUserID  string `json:"deletedUserId,omitempty" bson:"deletedUserId,omitempty"`
	GroupID        string `json:"-" bson:"_groupId,omitempty"`
	GUID           string `json:"guid,omitempty" bson:"guid,omitempty"`
	ID             string `json:"id,omitempty" bson:"id,omitempty"`
	ModifiedTime   string `json:"modifiedTime,omitempty" bson:"modifiedTime,omitempty"`
	ModifiedUserID string `json:"modifiedUserId,omitempty" bson:"modifiedUserId,omitempty"`
	SchemaVersion  int    `json:"-" bson:"_schemaVersion,omitempty"`
	Type           string `json:"type,omitempty" bson:"type,omitempty"`
	UploadID       string `json:"uploadId,omitempty" bson:"uploadId,omitempty"`
	UserID         string `json:"-" bson:"_userId,omitempty"`
	Version        int    `json:"-" bson:"_version,omitempty"`

	Annotations      *[]interface{} `json:"annotations,omitempty" bson:"annotations,omitempty"`
	ClockDriftOffset *int           `json:"clockDriftOffset,omitempty" bson:"clockDriftOffset,omitempty"`
	ConversionOffset *int           `json:"conversionOffset,omitempty" bson:"conversionOffset,omitempty"`
	DeviceID         *string        `json:"deviceId,omitempty" bson:"deviceId,omitempty"`
	DeviceTime       *string        `json:"deviceTime,omitempty" bson:"deviceTime,omitempty"`
	Payload          *interface{}   `json:"payload,omitempty" bson:"payload,omitempty"`
	Source           *string        `json:"source,omitempty" bson:"source,omitempty"`
	Time             *string        `json:"time,omitempty" bson:"time,omitempty"`
	TimezoneOffset   *int           `json:"timezoneOffset,omitempty" bson:"timezoneOffset,omitempty"`
}

type Meta struct {
	Type string `json:"type,omitempty"`
}

func (b *Base) Init() {
	b.Active = false
	b.CreatedTime = ""
	b.CreatedUserID = ""
	b.DeletedTime = ""
	b.DeletedUserID = ""
	b.GroupID = ""
	b.GUID = app.NewUUID()
	b.ID = app.NewID() // TODO: Move calculation to Normalize to follow Jellyfish algorithm
	b.ModifiedTime = ""
	b.ModifiedUserID = ""
	b.SchemaVersion = SchemaVersionCurrent
	b.Type = ""
	b.UploadID = ""
	b.UserID = ""
	b.Version = 0

	b.Annotations = nil
	b.ClockDriftOffset = nil
	b.ConversionOffset = nil
	b.DeviceID = nil
	b.DeviceTime = nil
	b.Payload = nil
	b.Source = nil
	b.Time = nil
	b.TimezoneOffset = nil
}

func (b *Base) Meta() interface{} {
	return &Meta{
		Type: b.Type,
	}
}

func (b *Base) Parse(parser data.ObjectParser) error {
	b.Annotations = parser.ParseInterfaceArray("annotations")
	b.ClockDriftOffset = parser.ParseInteger("clockDriftOffset")
	b.ConversionOffset = parser.ParseInteger("conversionOffset")
	b.DeviceID = parser.ParseString("deviceId")
	b.DeviceTime = parser.ParseString("deviceTime")
	b.Payload = parser.ParseInterface("payload")
	b.Source = parser.ParseString("source")
	b.Time = parser.ParseString("time")
	b.TimezoneOffset = parser.ParseInteger("timezoneOffset")

	return nil
}

func (b *Base) Validate(validator data.Validator) error {
	validator.ValidateString("type", &b.Type).Exists().NotEmpty()

	// validator.ValidateInterfaceArray("annotations", b.Annotations)    // TODO: Any validations? Optional? Size?
	// validator.ValidateInteger("clockDriftOffset", b.ClockDriftOffset) // TODO: Any validations? Optional? Range?
	// validator.ValidateInteger("conversionOffset", b.ConversionOffset) // TODO: Any validations? Optional? Range?
	validator.ValidateString("deviceId", b.DeviceID).Exists().NotEmpty()
	validator.ValidateStringAsTime("deviceTime", b.DeviceTime, "2006-01-02T15:04:05") // TODO: Not in upload!  -> .Exists()
	// validator.ValidateInterface("payload", b.Payload) // TODO: Any validations? Optional? Size?
	validator.ValidateString("source", b.Source).NotEmpty()
	validator.ValidateStringAsTime("time", b.Time, "2006-01-02T15:04:05Z07:00").Exists()
	// validator.ValidateInteger("timezoneOffset", b.TimezoneOffset) // TODO: Any validations? Optional? Range?

	// TODO: NOT IN UPLOAD: annotations, clockDriftOffset, deviceTime, payload

	return nil
}

func (b *Base) Normalize(normalizer data.Normalizer) error {
	return nil
}

func (b *Base) SetUserID(userID string) {
	b.UserID = userID
}

func (b *Base) SetGroupID(groupID string) {
	b.GroupID = groupID
}

func (b *Base) SetDatasetID(datasetID string) {
	b.UploadID = datasetID
}

func (b *Base) SetActive(active bool) {
	b.Active = active
}

func (b *Base) SetCreatedTime(createdTime string) {
	b.CreatedTime = createdTime
}

func (b *Base) SetCreatedUserID(createdUserID string) {
	b.CreatedUserID = createdUserID
}

func (b *Base) SetModifiedTime(modifiedTime string) {
	b.ModifiedTime = modifiedTime
}

func (b *Base) SetModifiedUserID(modifiedUserID string) {
	b.ModifiedUserID = modifiedUserID
}

func (b *Base) SetDeletedTime(deletedTime string) {
	b.DeletedTime = deletedTime
}

func (b *Base) SetDeletedUserID(deletedUserID string) {
	b.DeletedUserID = deletedUserID
}
<|endoftext|>


<commit_before>package tests_test

import (
	"flag"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	tests "kubevirt.io/kubevirt-ansible/tests/framework"
	"kubevirt.io/kubevirt/pkg/kubecli"
	ktests "kubevirt.io/kubevirt/tests"
)

var _ = Describe("High performance vm test", func() {
	/*
	* This test includes the features:
	* 1. Headless
	* 2. Support memory over commitment
	 */

	const (
		virtRawVMFilePath          = "tests/manifests/virt-testing-vm.yml"
		graphicDeviceOffStr        = "Autoattach Graphics Device:  false"
		vncErr                     = "Can't connect to websocket (400): No graphics devices are present"
		overcommitGuestOverheadStr = "Overcommit Guest Overhead:  true"
		memoryOvercommit           = "true"
		headless                   = "false"
		vmAPIVersion               = "kubevirt.io/v1alpha3"
	)

	flag.Parse()
	virtClient, err := kubecli.GetKubevirtClient()
	containerDisk := ktests.ContainerDiskFor(ktests.ContainerDiskCirros)
	ktests.PanicOnError(err)

	ktests.BeforeAll(func() {
		ktests.BeforeTestCleanup()
	})

	Context("Headless vm test", func() {
		headlessDstVMFilePath := "/tmp/headlesstest-vm.json"
		headlesstestVMName := "headlesstest"

		It("Create headless VM", func() {
			tests.ProcessTemplateWithParameters(virtRawVMFilePath, headlessDstVMFilePath, "VM_NAME="+headlesstestVMName, "AUTO_GRAPHIC_DEVICE="+headless, "IMAGE_NAME="+containerDisk, "VM_APIVERSION="+vmAPIVersion)
			tests.CreateResourceWithFilePathTestNamespace(headlessDstVMFilePath)
			tests.WaitUntilResourceReadyByNameTestNamespace("vmi", headlesstestVMName, "-o=jsonpath='{.status.phase}'", "Running")
		})

		It("Check VM settings with 'oc describe'", func() {
			res := tests.RunOcDescribeCommand("vmis", headlesstestVMName)
			Expect(strings.Contains(res, graphicDeviceOffStr)).To(BeTrue())
		})
		It("[Negative] Check console VNC is disable", func() {
			_, _, err := tests.OpenConsole(virtClient, headlesstestVMName, tests.NamespaceTestDefault, 20*time.Second, "vnc")
			Expect(strings.Contains(string(err.Error()), vncErr)).To(BeTrue())
		})
	})

	Context("Support memory over commitment test", func() {
		memoryOvercommitDstVMFilePath := "/tmp/memoryOvercommit-vm.json"
		memoryOvercommitVMName := "memoryovercommit"

		It("Create memoryOvercommit VM", func() {
			tests.ProcessTemplateWithParameters(virtRawVMFilePath, memoryOvercommitDstVMFilePath, "VM_NAME="+memoryOvercommitVMName, "OVER_COMMIT_GUEST_OVERLOAD="+memoryOvercommit, "IMAGE_NAME="+containerDisk, "VM_APIVERSION="+vmAPIVersion)
			tests.CreateResourceWithFilePathTestNamespace(memoryOvercommitDstVMFilePath)
			tests.WaitUntilResourceReadyByNameTestNamespace("vmi", memoryOvercommitVMName, "-o=jsonpath='{.status.phase}'", "Running")
		})
		It("Check VM settings with 'oc describe'", func() {
			res := tests.RunOcDescribeCommand("vmis", memoryOvercommitVMName)
			Expect(strings.Contains(res, overcommitGuestOverheadStr)).To(BeTrue())
		})
	})

	Context("Headless and Support memory over commitment VM test", func() {
		memoryOvercommitDstVMFilePath := "/tmp/headlessAndMemoryOvercommit-vm.json"
		memoryOvercommitVMName := "headlessandmemoryovercommit"

		It("Create headless and memory over commit VM", func() {
			tests.ProcessTemplateWithParameters(virtRawVMFilePath, memoryOvercommitDstVMFilePath, "VM_NAME="+memoryOvercommitVMName, "OVER_COMMIT_GUEST_OVERLOAD="+memoryOvercommit, "AUTO_GRAPHIC_DEVICE="+headless, "IMAGE_NAME="+containerDisk, "VM_APIVERSION="+vmAPIVersion)
			tests.CreateResourceWithFilePathTestNamespace(memoryOvercommitDstVMFilePath)
			tests.WaitUntilResourceReadyByNameTestNamespace("vmi", memoryOvercommitVMName, "-o=jsonpath='{.status.phase}'", "Running")
		})
		It("Check VM settings with 'oc describe'", func() {
			res := tests.RunOcDescribeCommand("vmis", memoryOvercommitVMName)
			Expect(strings.Contains(res, overcommitGuestOverheadStr)).To(BeTrue())
			Expect(strings.Contains(res, graphicDeviceOffStr)).To(BeTrue())
		})
		It("[Negative] Check console VNC is disable", func() {
			_, _, err := tests.OpenConsole(virtClient, memoryOvercommitVMName, tests.NamespaceTestDefault, 20*time.Second, "vnc")
			Expect(strings.Contains(string(err.Error()), vncErr)).To(BeTrue())
		})
	})
})
<commit_msg>Added attributes and test_ids to the high_performance_vm tests<commit_after>package tests_test

import (
	"flag"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	tests "kubevirt.io/kubevirt-ansible/tests/framework"
	"kubevirt.io/kubevirt/pkg/kubecli"
	ktests "kubevirt.io/kubevirt/tests"
)

var _ = Describe("[rfe_id:609][crit:medium][vendor:cnv-qe@redhat.com][level:component]High performance vm test", func() {
	/*
	* This test includes the features:
	* 1. Headless
	* 2. Support memory over commitment
	 */

	const (
		virtRawVMFilePath          = "tests/manifests/virt-testing-vm.yml"
		graphicDeviceOffStr        = "Autoattach Graphics Device:  false"
		vncErr                     = "Can't connect to websocket (400): No graphics devices are present"
		overcommitGuestOverheadStr = "Overcommit Guest Overhead:  true"
		memoryOvercommit           = "true"
		headless                   = "false"
		vmAPIVersion               = "kubevirt.io/v1alpha3"
	)

	flag.Parse()
	virtClient, err := kubecli.GetKubevirtClient()
	containerDisk := ktests.ContainerDiskFor(ktests.ContainerDiskCirros)
	ktests.PanicOnError(err)

	ktests.BeforeAll(func() {
		ktests.BeforeTestCleanup()
	})

	Context("Headless vm test", func() {
		headlessDstVMFilePath := "/tmp/headlesstest-vm.json"
		headlesstestVMName := "headlesstest"

		It("[test_id:707]Create headless VM", func() {
			tests.ProcessTemplateWithParameters(virtRawVMFilePath, headlessDstVMFilePath, "VM_NAME="+headlesstestVMName, "AUTO_GRAPHIC_DEVICE="+headless, "IMAGE_NAME="+containerDisk, "VM_APIVERSION="+vmAPIVersion)
			tests.CreateResourceWithFilePathTestNamespace(headlessDstVMFilePath)
			tests.WaitUntilResourceReadyByNameTestNamespace("vmi", headlesstestVMName, "-o=jsonpath='{.status.phase}'", "Running")
		})

		It("[test_id:708]Check VM settings with 'oc describe'", func() {
			res := tests.RunOcDescribeCommand("vmis", headlesstestVMName)
			Expect(strings.Contains(res, graphicDeviceOffStr)).To(BeTrue())
		})
		It("[test_id:712][posneg:negative]Check console VNC is disable", func() {
			_, _, err := tests.OpenConsole(virtClient, headlesstestVMName, tests.NamespaceTestDefault, 20*time.Second, "vnc")
			Expect(strings.Contains(string(err.Error()), vncErr)).To(BeTrue())
		})
	})

	Context("Support memory over commitment test", func() {
		memoryOvercommitDstVMFilePath := "/tmp/memoryOvercommit-vm.json"
		memoryOvercommitVMName := "memoryovercommit"

		It("[test_id:730]Create memoryOvercommit VM", func() {
			tests.ProcessTemplateWithParameters(virtRawVMFilePath, memoryOvercommitDstVMFilePath, "VM_NAME="+memoryOvercommitVMName, "OVER_COMMIT_GUEST_OVERLOAD="+memoryOvercommit, "IMAGE_NAME="+containerDisk, "VM_APIVERSION="+vmAPIVersion)
			tests.CreateResourceWithFilePathTestNamespace(memoryOvercommitDstVMFilePath)
			tests.WaitUntilResourceReadyByNameTestNamespace("vmi", memoryOvercommitVMName, "-o=jsonpath='{.status.phase}'", "Running")
		})
		It("[test_id:731]Check VM settings with 'oc describe'", func() {
			res := tests.RunOcDescribeCommand("vmis", memoryOvercommitVMName)
			Expect(strings.Contains(res, overcommitGuestOverheadStr)).To(BeTrue())
		})
	})

	Context("Headless and Support memory over commitment VM test", func() {
		memoryOvercommitDstVMFilePath := "/tmp/headlessAndMemoryOvercommit-vm.json"
		memoryOvercommitVMName := "headlessandmemoryovercommit"

		It("Create headless and memory over commit VM", func() {
			tests.ProcessTemplateWithParameters(virtRawVMFilePath, memoryOvercommitDstVMFilePath, "VM_NAME="+memoryOvercommitVMName, "OVER_COMMIT_GUEST_OVERLOAD="+memoryOvercommit, "AUTO_GRAPHIC_DEVICE="+headless, "IMAGE_NAME="+containerDisk, "VM_APIVERSION="+vmAPIVersion)
			tests.CreateResourceWithFilePathTestNamespace(memoryOvercommitDstVMFilePath)
			tests.WaitUntilResourceReadyByNameTestNamespace("vmi", memoryOvercommitVMName, "-o=jsonpath='{.status.phase}'", "Running")
		})
		It("[test_id:737]Check VM settings with 'oc describe'", func() {
			res := tests.RunOcDescribeCommand("vmis", memoryOvercommitVMName)
			Expect(strings.Contains(res, overcommitGuestOverheadStr)).To(BeTrue())
			Expect(strings.Contains(res, graphicDeviceOffStr)).To(BeTrue())
		})
		It("[test_id:738][posneg:negative]Check console VNC is disable", func() {
			_, _, err := tests.OpenConsole(virtClient, memoryOvercommitVMName, tests.NamespaceTestDefault, 20*time.Second, "vnc")
			Expect(strings.Contains(string(err.Error()), vncErr)).To(BeTrue())
		})
	})
})
<|endoftext|>


<commit_before>package cloud

import (
	"bytes"
	"strings"
	"text/template"

	api "github.com/appscode/pharmer/apis/v1alpha1"
	"github.com/ghodss/yaml"
	"github.com/hashicorp/go-version"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
)

type TemplateData struct {
	BinaryVersion      string
	KubeadmToken       string
	CAKey              string
	FrontProxyKey      string
	APIServerAddress   string
	ExtraDomains       string
	NetworkProvider    string
	CloudConfig        string
	Provider           string
	ExternalProvider   bool
	KubeadmTokenLoader string

	MasterConfiguration *kubeadmapi.MasterConfiguration
	KubeletExtraArgs    map[string]string
}

func (td TemplateData) MasterConfigurationYAML() (string, error) {
	if td.MasterConfiguration == nil {
		return "", nil
	}
	cb, err := yaml.Marshal(td.MasterConfiguration)
	return string(cb), err
}

func (td TemplateData) IsPreReleaseVersion() bool {
	if v, err := version.NewVersion(td.BinaryVersion); err == nil && v.Prerelease() != "" {
		return true
	}
	return false
}

func (td TemplateData) KubeletExtraArgsStr() string {
	var buf bytes.Buffer
	for k, v := range td.KubeletExtraArgs {
		buf.WriteString("--")
		buf.WriteString(k)
		buf.WriteRune('=')
		buf.WriteString(v)
		buf.WriteRune(' ')
	}
	return buf.String()
}

func (td TemplateData) KubeletExtraArgsEmptyCloudProviderStr() string {
	var buf bytes.Buffer
	for k, v := range td.KubeletExtraArgs {
		if k == "cloud-config" {
			continue
		}
		if k == "cloud-provider" {
			v = ""
		}
		buf.WriteString("--")
		buf.WriteString(k)
		buf.WriteRune('=')
		buf.WriteString(v)
		buf.WriteRune(' ')
	}
	return buf.String()
}

func (td TemplateData) PackageList() string {
	pkgs := []string{
		"cron",
		"docker.io",
		"ebtables",
		"git",
		"glusterfs-client",
		"haveged",
		"nfs-common",
		"socat",
	}
	if !td.IsPreReleaseVersion() {
		if td.BinaryVersion == "" {
			pkgs = append(pkgs, "kubeadm", "kubelet", "kubectl")
		} else {
			pkgs = append(pkgs, "kubeadm="+td.BinaryVersion, "kubelet="+td.BinaryVersion, "kubectl="+td.BinaryVersion)
		}
	}
	if td.Provider != "gce" && td.Provider != "gke" {
		pkgs = append(pkgs, "ntp")
	}
	return strings.Join(pkgs, " ")
}

var (
	StartupScriptTemplate = template.Must(template.New(api.RoleMaster).Parse(`#!/bin/bash
set -x
set -o errexit
set -o nounset
set -o pipefail

# log to /var/log/startup-script.log
exec > >(tee -a /var/log/startup-script.log)
exec 2>&1

# kill apt processes (E: Unable to lock directory /var/lib/apt/lists/)
kill $(ps aux | grep '[a]pt' | awk '{print $2}') || true

{{ template "prepare-host" . }}

apt-get update -y
apt-get install -y apt-transport-https curl ca-certificates

curl -fSsL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
echo 'deb http://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list

add-apt-repository -y ppa:gluster/glusterfs-3.10

apt-get update -y
apt-get install -y {{ .PackageList }} || true
{{ if .IsPreReleaseVersion }}
curl -Lo kubeadm https://dl.k8s.io/release/{{ .KubeadmVersion }}/bin/linux/amd64/kubeadm \
    && chmod +x kubeadm \
	&& mv kubeadm /usr/bin/
{{ end }}

curl -Lo pre-k https://cdn.appscode.com/binaries/pre-k/0.1.0-alpha.7/pre-k-linux-amd64 \
	&& chmod +x pre-k \
	&& mv pre-k /usr/bin/

cat > /etc/systemd/system/kubelet.service.d/20-pharmer.conf <<EOF
[Service]
Environment="KUBELET_EXTRA_ARGS={{ if .ExternalProvider }}{{ .KubeletExtraArgsEmptyCloudProviderStr }}{{ else }}{{ .KubeletExtraArgsStr }}{{ end }}"
EOF

kubeadm reset

{{ template "setup-certs" . }}

{{ if .CloudConfig }}
cat > /etc/kubernetes/cloud-config <<EOF
{{ .CloudConfig }}
EOF
{{ end }}

mkdir -p /etc/kubernetes/kubeadm

{{ if .MasterConfiguration }}
cat > /etc/kubernetes/kubeadm/base.yaml <<EOF
{{ .MasterConfigurationYAML }}
EOF
{{ end }}

pre-k merge master-config \
	--config=/etc/kubernetes/kubeadm/base.yaml \
	--apiserver-advertise-address=$(pre-k get public-ips --all=false) \
	--apiserver-cert-extra-sans=$(pre-k get public-ips --routable) \
	--apiserver-cert-extra-sans=$(pre-k get private-ips) \
	--apiserver-cert-extra-sans={{ .ExtraDomains }} \
	> /etc/kubernetes/kubeadm/config.yaml
kubeadm init --config=/etc/kubernetes/kubeadm/config.yaml --skip-token-print

{{ if eq .NetworkProvider "flannel" }}
{{ template "flannel" . }}
{{ else if eq .NetworkProvider "calico" }}
{{ template "calico" . }}
{{ end }}

kubectl apply \
  -f https://raw.githubusercontent.com/appscode/pharmer/master/addons/kubeadm-probe/ds.yaml \
  --kubeconfig /etc/kubernetes/admin.conf

mkdir -p ~/.kube
sudo cp -i /etc/kubernetes/admin.conf ~/.kube/config
sudo chown $(id -u):$(id -g) ~/.kube/config

{{ if .ExternalProvider }}
{{ template "ccm" . }}
{{end}}

`))

	_ = template.Must(StartupScriptTemplate.New(api.RoleNode).Parse(`#!/bin/bash
set -x
set -o errexit
set -o nounset
set -o pipefail

# log to /var/log/startup-script.log
exec > >(tee -a /var/log/startup-script.log)
exec 2>&1

# kill apt processes (E: Unable to lock directory /var/lib/apt/lists/)
kill $(ps aux | grep '[a]pt' | awk '{print $2}') || true

{{ template "prepare-host" . }}

apt-get update -y
apt-get install -y apt-transport-https curl ca-certificates

curl -fSsL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
echo 'deb http://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list

add-apt-repository -y ppa:gluster/glusterfs-3.10

apt-get update -y
apt-get install -y {{ .PackageList }} || true
{{ if .IsPreReleaseVersion }}
curl -Lo kubeadm https://dl.k8s.io/release/{{ .KubeadmVersion }}/bin/linux/amd64/kubeadm \
    && chmod +x kubeadm \
	&& mv kubeadm /usr/bin/
{{ end }}

systemctl enable docker
systemctl start docker

{{ if .CloudConfig }}
cat > /etc/kubernetes/cloud-config <<EOF
{{ .CloudConfig }}
EOF
{{ end }}

cat > /etc/systemd/system/kubelet.service.d/20-pharmer.conf <<EOF
[Service]
Environment="KUBELET_EXTRA_ARGS={{ .KubeletExtraArgsStr }}"
EOF

systemctl daemon-reload
systemctl restart kubelet

kubeadm reset
{{ .KubeadmTokenLoader  }}
KUBEADM_TOKEN=${KUBEADM_TOKEN:-{{ .KubeadmToken }}}
kubeadm join --token=${KUBEADM_TOKEN} {{ .APIServerAddress }}
`))

	_ = template.Must(StartupScriptTemplate.New("prepare-host").Parse(``))

	_ = template.Must(StartupScriptTemplate.New("setup-certs").Parse(`
mkdir -p /etc/kubernetes/pki

cat > /etc/kubernetes/pki/ca.key <<EOF
{{ .CAKey }}
EOF
pre-k get cacert --common-name=ca < /etc/kubernetes/pki/ca.key > /etc/kubernetes/pki/ca.crt

cat > /etc/kubernetes/pki/front-proxy-ca.key <<EOF
{{ .FrontProxyKey }}
EOF
pre-k get cacert --common-name=front-proxy-ca < /etc/kubernetes/pki/front-proxy-ca.key > /etc/kubernetes/pki/front-proxy-ca.crt
chmod 600 /etc/kubernetes/pki/ca.key /etc/kubernetes/pki/front-proxy-ca.key
`))

	_ = template.Must(StartupScriptTemplate.New("ccm").Parse(`
until [ $(kubectl get pods -n kube-system -l k8s-app=kube-dns -o jsonpath='{.items[0].status.phase}' --kubeconfig /etc/kubernetes/admin.conf) == "Running" ]
do
   echo '.'
   sleep 5
done

kubectl apply \
  -f https://raw.githubusercontent.com/appscode/pharmer/master/cloud/providers/digitalocean/cloud-control-manager.yaml \
  --kubeconfig /etc/kubernetes/admin.conf

until [ $(kubectl get pods -n kube-system -l app=cloud-controller-manager -o jsonpath='{.items[0].status.phase}' --kubeconfig /etc/kubernetes/admin.conf) == "Running" ]
do
   echo '.'
   sleep 5
done

kubectl taint nodes $(uname -n) node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule --kubeconfig /etc/kubernetes/admin.conf

cat > /etc/systemd/system/kubelet.service.d/20-pharmer.conf <<EOF
[Service]
Environment="KUBELET_EXTRA_ARGS={{ .KubeletExtraArgsStr }}"
EOF
systemctl daemon-reload
systemctl restart kubelet

# systemctl enable docker
# systemctl start docker

# until [ $(kubectl get pods -n kube-system -l k8s-app=kube-dns -o jsonpath='{.items[0].status.phase}' --kubeconfig /etc/kubernetes/admin.conf) == "Running" ]
# do
#    echo '.'
#    sleep 5
# done
#
# kubectl apply -f "https://raw.githubusercontent.com/appscode/pharmer/master/cloud/providers/{{ .Provider }}/cloud-control-manager.yaml" --kubeconfig /etc/kubernetes/admin.conf
#
# until [ $(kubectl get pods -n kube-system -l app=cloud-controller-manager -o jsonpath='{.items[0].status.phase}' --kubeconfig /etc/kubernetes/admin.conf) == "Running" ]
# do
#    echo '.'
#    sleep 5
# done
#
# cat > /etc/systemd/system/kubelet.service.d/20-pharmer.conf <<EOF
# [Service]
# Environment="KUBELET_EXTRA_ARGS={{ .KubeletExtraArgsStr }}"
# EOF
#
# NODE_NAME=$(uname -n)
# kubectl taint nodes $(uname -n) node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule --kubeconfig /etc/kubernetes/admin.conf
#
# systemctl daemon-reload
# systemctl restart kubelet

# sleep 10
# reboot
`))

	_ = template.Must(StartupScriptTemplate.New("calico").Parse(`
kubectl apply \
  -f https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml \
  --kubeconfig /etc/kubernetes/admin.conf
`))

	_ = template.Must(StartupScriptTemplate.New("flannel").Parse(`
kubectl apply \
  -f https://raw.githubusercontent.com/coreos/flannel/v0.8.0/Documentation/kube-flannel.yml \
  --kubeconfig /etc/kubernetes/admin.conf
kubectl apply \
  -f https://raw.githubusercontent.com/coreos/flannel/v0.8.0/Documentation/kube-flannel-rbac.yml \
  --kubeconfig /etc/kubernetes/admin.conf
`))
)
<commit_msg>Use calico installer from addon folder<commit_after>package cloud

import (
	"bytes"
	"strings"
	"text/template"

	api "github.com/appscode/pharmer/apis/v1alpha1"
	"github.com/ghodss/yaml"
	"github.com/hashicorp/go-version"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
)

type TemplateData struct {
	BinaryVersion      string
	KubeadmToken       string
	CAKey              string
	FrontProxyKey      string
	APIServerAddress   string
	ExtraDomains       string
	NetworkProvider    string
	CloudConfig        string
	Provider           string
	ExternalProvider   bool
	KubeadmTokenLoader string

	MasterConfiguration *kubeadmapi.MasterConfiguration
	KubeletExtraArgs    map[string]string
}

func (td TemplateData) MasterConfigurationYAML() (string, error) {
	if td.MasterConfiguration == nil {
		return "", nil
	}
	cb, err := yaml.Marshal(td.MasterConfiguration)
	return string(cb), err
}

func (td TemplateData) IsPreReleaseVersion() bool {
	if v, err := version.NewVersion(td.BinaryVersion); err == nil && v.Prerelease() != "" {
		return true
	}
	return false
}

func (td TemplateData) KubeletExtraArgsStr() string {
	var buf bytes.Buffer
	for k, v := range td.KubeletExtraArgs {
		buf.WriteString("--")
		buf.WriteString(k)
		buf.WriteRune('=')
		buf.WriteString(v)
		buf.WriteRune(' ')
	}
	return buf.String()
}

func (td TemplateData) KubeletExtraArgsEmptyCloudProviderStr() string {
	var buf bytes.Buffer
	for k, v := range td.KubeletExtraArgs {
		if k == "cloud-config" {
			continue
		}
		if k == "cloud-provider" {
			v = ""
		}
		buf.WriteString("--")
		buf.WriteString(k)
		buf.WriteRune('=')
		buf.WriteString(v)
		buf.WriteRune(' ')
	}
	return buf.String()
}

func (td TemplateData) PackageList() string {
	pkgs := []string{
		"cron",
		"docker.io",
		"ebtables",
		"git",
		"glusterfs-client",
		"haveged",
		"nfs-common",
		"socat",
	}
	if !td.IsPreReleaseVersion() {
		if td.BinaryVersion == "" {
			pkgs = append(pkgs, "kubeadm", "kubelet", "kubectl")
		} else {
			pkgs = append(pkgs, "kubeadm="+td.BinaryVersion, "kubelet="+td.BinaryVersion, "kubectl="+td.BinaryVersion)
		}
	}
	if td.Provider != "gce" && td.Provider != "gke" {
		pkgs = append(pkgs, "ntp")
	}
	return strings.Join(pkgs, " ")
}

var (
	StartupScriptTemplate = template.Must(template.New(api.RoleMaster).Parse(`#!/bin/bash
set -x
set -o errexit
set -o nounset
set -o pipefail

# log to /var/log/startup-script.log
exec > >(tee -a /var/log/startup-script.log)
exec 2>&1

# kill apt processes (E: Unable to lock directory /var/lib/apt/lists/)
kill $(ps aux | grep '[a]pt' | awk '{print $2}') || true

{{ template "prepare-host" . }}

apt-get update -y
apt-get install -y apt-transport-https curl ca-certificates

curl -fSsL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
echo 'deb http://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list

add-apt-repository -y ppa:gluster/glusterfs-3.10

apt-get update -y
apt-get install -y {{ .PackageList }} || true
{{ if .IsPreReleaseVersion }}
curl -Lo kubeadm https://dl.k8s.io/release/{{ .KubeadmVersion }}/bin/linux/amd64/kubeadm \
    && chmod +x kubeadm \
	&& mv kubeadm /usr/bin/
{{ end }}

curl -Lo pre-k https://cdn.appscode.com/binaries/pre-k/0.1.0-alpha.7/pre-k-linux-amd64 \
	&& chmod +x pre-k \
	&& mv pre-k /usr/bin/

cat > /etc/systemd/system/kubelet.service.d/20-pharmer.conf <<EOF
[Service]
Environment="KUBELET_EXTRA_ARGS={{ if .ExternalProvider }}{{ .KubeletExtraArgsEmptyCloudProviderStr }}{{ else }}{{ .KubeletExtraArgsStr }}{{ end }}"
EOF

kubeadm reset

{{ template "setup-certs" . }}

{{ if .CloudConfig }}
cat > /etc/kubernetes/cloud-config <<EOF
{{ .CloudConfig }}
EOF
{{ end }}

mkdir -p /etc/kubernetes/kubeadm

{{ if .MasterConfiguration }}
cat > /etc/kubernetes/kubeadm/base.yaml <<EOF
{{ .MasterConfigurationYAML }}
EOF
{{ end }}

pre-k merge master-config \
	--config=/etc/kubernetes/kubeadm/base.yaml \
	--apiserver-advertise-address=$(pre-k get public-ips --all=false) \
	--apiserver-cert-extra-sans=$(pre-k get public-ips --routable) \
	--apiserver-cert-extra-sans=$(pre-k get private-ips) \
	--apiserver-cert-extra-sans={{ .ExtraDomains }} \
	> /etc/kubernetes/kubeadm/config.yaml
kubeadm init --config=/etc/kubernetes/kubeadm/config.yaml --skip-token-print

{{ if eq .NetworkProvider "flannel" }}
{{ template "flannel" . }}
{{ else if eq .NetworkProvider "calico" }}
{{ template "calico" . }}
{{ end }}

kubectl apply \
  -f https://raw.githubusercontent.com/appscode/pharmer/master/addons/kubeadm-probe/ds.yaml \
  --kubeconfig /etc/kubernetes/admin.conf

mkdir -p ~/.kube
sudo cp -i /etc/kubernetes/admin.conf ~/.kube/config
sudo chown $(id -u):$(id -g) ~/.kube/config

{{ if .ExternalProvider }}
{{ template "ccm" . }}
{{end}}

`))

	_ = template.Must(StartupScriptTemplate.New(api.RoleNode).Parse(`#!/bin/bash
set -x
set -o errexit
set -o nounset
set -o pipefail

# log to /var/log/startup-script.log
exec > >(tee -a /var/log/startup-script.log)
exec 2>&1

# kill apt processes (E: Unable to lock directory /var/lib/apt/lists/)
kill $(ps aux | grep '[a]pt' | awk '{print $2}') || true

{{ template "prepare-host" . }}

apt-get update -y
apt-get install -y apt-transport-https curl ca-certificates

curl -fSsL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
echo 'deb http://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list

add-apt-repository -y ppa:gluster/glusterfs-3.10

apt-get update -y
apt-get install -y {{ .PackageList }} || true
{{ if .IsPreReleaseVersion }}
curl -Lo kubeadm https://dl.k8s.io/release/{{ .KubeadmVersion }}/bin/linux/amd64/kubeadm \
    && chmod +x kubeadm \
	&& mv kubeadm /usr/bin/
{{ end }}

systemctl enable docker
systemctl start docker

{{ if .CloudConfig }}
cat > /etc/kubernetes/cloud-config <<EOF
{{ .CloudConfig }}
EOF
{{ end }}

cat > /etc/systemd/system/kubelet.service.d/20-pharmer.conf <<EOF
[Service]
Environment="KUBELET_EXTRA_ARGS={{ .KubeletExtraArgsStr }}"
EOF

systemctl daemon-reload
systemctl restart kubelet

kubeadm reset
{{ .KubeadmTokenLoader  }}
KUBEADM_TOKEN=${KUBEADM_TOKEN:-{{ .KubeadmToken }}}
kubeadm join --token=${KUBEADM_TOKEN} {{ .APIServerAddress }}
`))

	_ = template.Must(StartupScriptTemplate.New("prepare-host").Parse(``))

	_ = template.Must(StartupScriptTemplate.New("setup-certs").Parse(`
mkdir -p /etc/kubernetes/pki

cat > /etc/kubernetes/pki/ca.key <<EOF
{{ .CAKey }}
EOF
pre-k get cacert --common-name=ca < /etc/kubernetes/pki/ca.key > /etc/kubernetes/pki/ca.crt

cat > /etc/kubernetes/pki/front-proxy-ca.key <<EOF
{{ .FrontProxyKey }}
EOF
pre-k get cacert --common-name=front-proxy-ca < /etc/kubernetes/pki/front-proxy-ca.key > /etc/kubernetes/pki/front-proxy-ca.crt
chmod 600 /etc/kubernetes/pki/ca.key /etc/kubernetes/pki/front-proxy-ca.key
`))

	_ = template.Must(StartupScriptTemplate.New("ccm").Parse(`
until [ $(kubectl get pods -n kube-system -l k8s-app=kube-dns -o jsonpath='{.items[0].status.phase}' --kubeconfig /etc/kubernetes/admin.conf) == "Running" ]
do
   echo '.'
   sleep 5
done

kubectl apply \
  -f https://raw.githubusercontent.com/appscode/pharmer/master/cloud/providers/digitalocean/cloud-control-manager.yaml \
  --kubeconfig /etc/kubernetes/admin.conf

until [ $(kubectl get pods -n kube-system -l app=cloud-controller-manager -o jsonpath='{.items[0].status.phase}' --kubeconfig /etc/kubernetes/admin.conf) == "Running" ]
do
   echo '.'
   sleep 5
done

kubectl taint nodes $(uname -n) node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule --kubeconfig /etc/kubernetes/admin.conf

cat > /etc/systemd/system/kubelet.service.d/20-pharmer.conf <<EOF
[Service]
Environment="KUBELET_EXTRA_ARGS={{ .KubeletExtraArgsStr }}"
EOF
systemctl daemon-reload
systemctl restart kubelet

# systemctl enable docker
# systemctl start docker

# until [ $(kubectl get pods -n kube-system -l k8s-app=kube-dns -o jsonpath='{.items[0].status.phase}' --kubeconfig /etc/kubernetes/admin.conf) == "Running" ]
# do
#    echo '.'
#    sleep 5
# done
#
# kubectl apply -f "https://raw.githubusercontent.com/appscode/pharmer/master/cloud/providers/{{ .Provider }}/cloud-control-manager.yaml" --kubeconfig /etc/kubernetes/admin.conf
#
# until [ $(kubectl get pods -n kube-system -l app=cloud-controller-manager -o jsonpath='{.items[0].status.phase}' --kubeconfig /etc/kubernetes/admin.conf) == "Running" ]
# do
#    echo '.'
#    sleep 5
# done
#
# cat > /etc/systemd/system/kubelet.service.d/20-pharmer.conf <<EOF
# [Service]
# Environment="KUBELET_EXTRA_ARGS={{ .KubeletExtraArgsStr }}"
# EOF
#
# NODE_NAME=$(uname -n)
# kubectl taint nodes $(uname -n) node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule --kubeconfig /etc/kubernetes/admin.conf
#
# systemctl daemon-reload
# systemctl restart kubelet

# sleep 10
# reboot
`))

	_ = template.Must(StartupScriptTemplate.New("calico").Parse(`
kubectl apply \
  -f https://raw.githubusercontent.com/appscode/pharmer/master/addons/calico/2.6/calico.yaml \
  --kubeconfig /etc/kubernetes/admin.conf
`))

	_ = template.Must(StartupScriptTemplate.New("flannel").Parse(`
kubectl apply \
  -f https://raw.githubusercontent.com/coreos/flannel/v0.8.0/Documentation/kube-flannel.yml \
  --kubeconfig /etc/kubernetes/admin.conf
kubectl apply \
  -f https://raw.githubusercontent.com/coreos/flannel/v0.8.0/Documentation/kube-flannel-rbac.yml \
  --kubeconfig /etc/kubernetes/admin.conf
`))
)
<|endoftext|>


<commit_before>package awsutil_test

import (
	"testing"
	"github.com/tleyden/awsutil"
	"github.com/tleyden/aws-sdk-mock/mockcloudformation"
	"github.com/tleyden/aws-sdk-mock/mockec2"
	"github.com/aws/aws-sdk-go/service/cloudformation"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/stretchr/testify/mock"
	"github.com/stretchr/testify/assert"
)

func TestStopEC2Instances(t *testing.T) {

	mockCfn := NewMockCloudformationAPI()
	mockEc2 := mockec2.NewEC2APIMock()

	// Mock cloudformation returns stack with some ec2 instances and some non-ec2 instances
	mockCfn.On("DescribeStackResources", mock.Anything).Return(
		&cloudformation.DescribeStackResourcesOutput{
			StackResources: []*cloudformation.StackResource{
				&cloudformation.StackResource{
					ResourceType: awsutil.StringPointer(awsutil.AWS_EC2_INSTANCE),
				},
				&cloudformation.StackResource{
					ResourceType: awsutil.StringPointer(awsutil.AWS_EC2_HOST),
				},

			},
		},
		nil,
	).Once()

	// Expect a call to ec2 StopInstances
	mockEc2.On("StopInstances", mock.Anything).Return(
		&ec2.StopInstancesOutput{},
		nil,
	).Once()

	// Create CloudformationUtil which is the struct being tested
	cfnUtil, err := awsutil.NewCloudformationUtil(mockCfn, mockEc2)
	assert.NoError(t, err, "Error creating NewCloudformationUtil")

	// Tell it to stop all EC2 instances in the fake Cloudformation Stack
	cfnUtil.StopEC2Instances("fake_stack")

	// assert that all expectations met
	mockCfn.AssertExpectations(t)
	mockEc2.AssertExpectations(t)

}

func TestStopEc2InstanceStackResource(t *testing.T)  {

	mockInstanceId := "i-mock"

	mockCfn := NewMockCloudformationAPI()
	mockEc2 := mockec2.NewEC2APIMock()

	// The mock ec2 API is expecting to get this as the parameter to
	// the ec2Api.StopInstances invocation
	expectedStopInstancesInput := ec2.StopInstancesInput{
		InstanceIds: []*string{
			&mockInstanceId,
		},
	}

	mockEc2.On("StopInstances", &expectedStopInstancesInput).Return(
		&ec2.StopInstancesOutput{},
		nil,
	).Once()

	cfnUtil, err := awsutil.NewCloudformationUtil(mockCfn, mockEc2)
	assert.NoError(t, err, "Error calling NewCloudformationUtil")

	stackResource := cloudformation.StackResource{
		ResourceType: awsutil.StringPointer(awsutil.AWS_EC2_INSTANCE),
		PhysicalResourceId: &mockInstanceId,
	}

	err = cfnUtil.StopEc2InstanceStackResource(stackResource)
	assert.NoError(t, err, "Error calling StopEc2InstanceStackResource")

	mockEc2.AssertExpectations(t)

}

// Creates a cloudformation API
func NewMockCloudformationAPI() *mockcloudformation.CloudFormationAPIMock {

	return mockcloudformation.NewCloudFormationAPIMock()

}

<commit_msg>Enhance TestStopEC2Instances<commit_after>package awsutil_test

import (
	"testing"
	"github.com/tleyden/awsutil"
	"github.com/tleyden/aws-sdk-mock/mockcloudformation"
	"github.com/tleyden/aws-sdk-mock/mockec2"
	"github.com/aws/aws-sdk-go/service/cloudformation"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/stretchr/testify/mock"
	"github.com/stretchr/testify/assert"
)

func TestStopEC2Instances(t *testing.T) {

	mockInstanceId := "i-mock"

	mockCfn := NewMockCloudformationAPI()
	mockEc2 := mockec2.NewEC2APIMock()

	// Mock cloudformation returns stack with some ec2 instances and some non-ec2 instances
	mockCfn.On("DescribeStackResources", mock.Anything).Return(
		&cloudformation.DescribeStackResourcesOutput{
			StackResources: []*cloudformation.StackResource{
				&cloudformation.StackResource{
					ResourceType: awsutil.StringPointer(awsutil.AWS_EC2_INSTANCE),
					PhysicalResourceId: &mockInstanceId,
				},
				&cloudformation.StackResource{
					ResourceType: awsutil.StringPointer(awsutil.AWS_EC2_HOST),
				},

			},
		},
		nil,
	).Once()

	// The mock ec2 API is expecting to get this as the parameter to
	// the ec2Api.StopInstances invocation
	expectedStopInstancesInput := &ec2.StopInstancesInput{
		InstanceIds: []*string{
			&mockInstanceId,
		},
	}

	// Expect a call to ec2 StopInstances
	mockEc2.On("StopInstances", expectedStopInstancesInput).Return(
		&ec2.StopInstancesOutput{},
		nil,
	).Once()

	// Create CloudformationUtil which is the struct being tested
	cfnUtil, err := awsutil.NewCloudformationUtil(mockCfn, mockEc2)
	assert.NoError(t, err, "Error creating NewCloudformationUtil")

	// Tell it to stop all EC2 instances in the fake Cloudformation Stack
	cfnUtil.StopEC2Instances("fake_stack")

	// assert that all expectations met
	mockCfn.AssertExpectations(t)
	mockEc2.AssertExpectations(t)

}

func TestStopEc2InstanceStackResource(t *testing.T)  {

	mockInstanceId := "i-mock"

	mockCfn := NewMockCloudformationAPI()
	mockEc2 := mockec2.NewEC2APIMock()

	// The mock ec2 API is expecting to get this as the parameter to
	// the ec2Api.StopInstances invocation
	expectedStopInstancesInput := &ec2.StopInstancesInput{
		InstanceIds: []*string{
			&mockInstanceId,
		},
	}

	mockEc2.On("StopInstances", expectedStopInstancesInput).Return(
		&ec2.StopInstancesOutput{},
		nil,
	).Once()

	cfnUtil, err := awsutil.NewCloudformationUtil(mockCfn, mockEc2)
	assert.NoError(t, err, "Error calling NewCloudformationUtil")

	stackResource := cloudformation.StackResource{
		ResourceType: awsutil.StringPointer(awsutil.AWS_EC2_INSTANCE),
		PhysicalResourceId: &mockInstanceId,
	}

	err = cfnUtil.StopEc2InstanceStackResource(stackResource)
	assert.NoError(t, err, "Error calling StopEc2InstanceStackResource")

	mockEc2.AssertExpectations(t)

}

// Creates a cloudformation API
func NewMockCloudformationAPI() *mockcloudformation.CloudFormationAPIMock {

	return mockcloudformation.NewCloudFormationAPIMock()

}

<|endoftext|>


<commit_before>package libcentrifugo

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/shilkin/centrifugo/libcentrifugo/extender"
	"github.com/shilkin/centrifugo/libcentrifugo/logger"
	"github.com/tarantool/go-tarantool"
	"strconv"
	"strings"
)

func (p *TarantoolPool) get() (conn *tarantool.Connection, err error) {
	if len(p.pool) == 0 {
		return nil, errors.New("Empty tarantool pool")
	}
	conn = p.pool[p.current]
	p.current++
	p.current = (p.current) % len(p.pool)
	return
}

type TarantoolEngine struct {
	app      *Application
	pool     *TarantoolPool
	extender extender.Extender
	endpoint string
}

type TarantoolEngineConfig struct {
	PoolConfig  TarantoolPoolConfig
	Endpoint    string
	TTConnector extender.Config
}

type TarantoolPool struct {
	pool    []*tarantool.Connection
	config  TarantoolPoolConfig
	current int
}

type TarantoolPoolConfig struct {
	Address  string
	PoolSize int
	Opts     tarantool.Opts
}

/* MessageType
{
	"body": {
		"uid":"026c380d-13e1-47d9-42d2-e2dc0e41e8d5",
		"timestamp":"1440434259",
		"info":{
			"user":"3",
			"client":"83309b33-deb7-48ff-76c6-04b10e6a6523",
			"default_info":null,
			"channel_info": {
				"channel_extra_info_example":"you can add additional JSON data when authorizing"
			}
		},
		"channel":"$3_0",
		"data": {
				"Action":"mark",
				"Data":["00000000000000395684"]
			},
		"client":"83309b33-deb7-48ff-76c6-04b10e6a6523"
	},
	"error":null,
	"method":"message"
}
*/

type MessageType struct {
	Body   Message `json:"body"`
	Error  string  `json:"error"`
	Method string  `json:"method"`
}

type ServiceMessage struct {
	Action string
	Data   []string
}

type IDs []string

func NewTarantoolEngine(app *Application, conf TarantoolEngineConfig) *TarantoolEngine {
	logger.INFO.Printf("Initializing tarantool connection pool...")
	pool, err := newTarantoolPool(conf.PoolConfig)
	if err != nil {
		logger.FATAL.Fatalln(err)
	}

	extender, err := extender.New(conf.TTConnector)
	if err != nil {
		logger.FATAL.Fatalln(err)
	}

	e := &TarantoolEngine{
		app:      app,
		pool:     pool,
		extender: extender,
		endpoint: conf.Endpoint,
	}

	return e
}

func newTarantoolPool(config TarantoolPoolConfig) (p *TarantoolPool, err error) {
	if config.PoolSize == 0 {
		err = errors.New("Size of tarantool pool is zero")
		return
	}

	p = &TarantoolPool{
		pool:   make([]*tarantool.Connection, config.PoolSize),
		config: config,
	}

	for i := 0; i < config.PoolSize; i++ {
		logger.INFO.Printf("[%d] Connecting to tarantool on %s... [%d]", i, config.Address, config.Opts.MaxReconnects)
		p.pool[i], err = tarantool.Connect(config.Address, config.Opts)
		if err != nil && config.Opts.Reconnect > 0 {
			logger.ERROR.Printf("[%d] connection to tarantool on %s failed with '%s'", i, config.Address, err)
			err = nil // just log and reset error: reconnection inside tarantool.Connect
		}
		if err == nil {
			logger.INFO.Printf("[%d] Connected to tarantool on %s", i, config.Address)
		}
	}

	return
}

// getName returns a name of concrete engine implementation
func (e *TarantoolEngine) name() string {
	return "Tarantool"
}

// publish allows to send message into channel
func (e *TarantoolEngine) publish(chID ChannelID, message []byte) error {
	// Process service messages
	if chID != e.app.config.ControlChannel && chID != e.app.config.AdminChannel {
		newMessage, err := e.processMessage(chID, message)
		if err != nil {
			return err // if no need further processing
		}
		message = newMessage
	}
	// All other messages
	return e.app.handleMsg(chID, message)
}

// subscribe on channel
func (e *TarantoolEngine) subscribe(chID ChannelID) (err error) {
	logger.INFO.Printf("subscribe %s", chID)
	endpoint, err := e.makeEndpointFromChannelID(chID)
	if err != nil {
		logger.ERROR.Printf("subscribe make endpoint string error: %v\n", err.Error())
		return
	}

	conn, err := e.pool.get()
	if err != nil {
		logger.ERROR.Printf("subscribe tarantool pool error: %v\n", err.Error())
		return
	}

	_, err = conn.Call("notification_channel_subscribe", []interface{}{chID, endpoint})

	return
}

// unsubscribe from channel
func (e *TarantoolEngine) unsubscribe(chID ChannelID) (err error) {
	endpoint, err := e.makeEndpointFromChannelID(chID)
	if err != nil {
		logger.ERROR.Printf("unsubscribe make endpoint string error: %v\n", err.Error())
		return
	}

	conn, err := e.pool.get()
	if err != nil {
		logger.ERROR.Printf("unsubscribe tarantool pool error: %v\n", err.Error())
		return
	}

	_, err = conn.Call("notification_channel_unsubscribe", []interface{}{chID, endpoint})

	return
}

// addPresence sets or updates presence info for connection with uid
func (e *TarantoolEngine) addPresence(chID ChannelID, uid ConnID, info ClientInfo) (err error) {
	// not implemented
	return
}

// removePresence removes presence information for connection with uid
func (e *TarantoolEngine) removePresence(chID ChannelID, uid ConnID) (err error) {
	// not implemented
	return
}

// getPresence returns actual presence information for channel
func (e *TarantoolEngine) presence(chID ChannelID) (result map[ConnID]ClientInfo, err error) {
	// not implemented
	return
}

// addHistory adds message into channel history and takes care about history size
func (e *TarantoolEngine) addHistory(chID ChannelID, message Message, size, lifetime int64) (err error) {
	// not implemented
	return
}

// getHistory returns history messages for channel
// return empty slice
// all history pushed via publish
func (e *TarantoolEngine) history(chID ChannelID) (msgs []Message, err error) {
	conn, err := e.pool.get()
	if err != nil {
		logger.ERROR.Printf("history tarantool pool error: %v\n", err.Error())
		return nil, err
	}

	history, err := conn.Call("notification_notification_history", []interface{}{chID})
	if err != nil {
		logger.ERROR.Printf("history error: %v\n", err.Error())
		return nil, err
	}

	return processHistory(history)
}

// helpers

type tarantoolHistoryItem struct {
	Count  interface{} `json:"count"`
	Status string      `json:"status"`
	ID     string      `json:"id"`
}

func processHistory(history *tarantool.Response) (msgs []Message, err error) {
	if len(history.Data) == 0 {
		return // history is empty
	}

	data := history.Data[0].([]interface{})
	if len(data) != 2 {
		return // history is empty
	}

	count := data[0]                       // ring counter
	buffer := data[1].(string)             // string buffer
	ring := strings.Split(buffer[1:], ",") // array of IDs

	if len(ring) == 0 {
		return // history buffer is empty [useless?]
	}

	for _, id := range ring {
		encoded, err := json.Marshal(tarantoolHistoryItem{
			Count:  count, // redundancy in each item to pass number of unread notifications
			Status: string(id[0]),
			ID:     string(id[1:]),
		})
		if err != nil {
			logger.ERROR.Println(err)
			continue
		}
		rawMessage := json.RawMessage([]byte(encoded))
		msgs = append(msgs, Message{Data: &rawMessage})
	}

	return
}

func (e *TarantoolEngine) extendMessage(chID ChannelID, message []byte) (newMessage []byte, err error) {
	logger.DEBUG.Printf("try to extend message chID = %s, message = %s", chID, string(message))

	//uid, _, _, err := parseChannelID(chID)
	//if err != nil {
	//	return
	//}

	var m MessageType
	err = json.Unmarshal(message, &m)
	if err != nil {
		return
	}

	extended, err := e.extender.Extend(m.Body.Data, string(chID))
	if extended != nil {
		m.Body.Data = extended
		newMessage, err = json.Marshal(&m)
		logger.DEBUG.Printf("data extended to: %s", string(*m.Body.Data))
	}

	return
}

func (e *TarantoolEngine) processMessage(chID ChannelID, message []byte) (newMessage []byte, err error) {
	newMessage = message // by default, but may be changed

	var msg MessageType
	err = json.Unmarshal(message, &msg)
	if err != nil {
		return
	}

	var srv ServiceMessage
	err = json.Unmarshal(*msg.Body.Data, &srv)
	if err != nil {
		return
	}

	newMessage, err = e.extendMessage(chID, message)
	if err != nil {
		logger.ERROR.Printf("extend message failed with '%s'", err)
	}

	return
}

func (e *TarantoolEngine) makeEndpointFromChannelID(chID ChannelID) (endpoint string, err error) {
	// split chID <centrifugo>.<project>.[$]<uid>_<ringno>
	str := string(chID)
	logger.INFO.Printf("makeEndpointFromChannelID %s", str)
	result := strings.Split(str, ".")
	if len(result) != 3 {
		err = fmt.Errorf("unexpected ChannelID %s", str)
		return
	}
	endpoint = e.endpoint + "/api/" + result[1]
	return
}
<commit_msg>MAILPAAS-3148| fix unused import<commit_after>package libcentrifugo

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/shilkin/centrifugo/libcentrifugo/extender"
	"github.com/shilkin/centrifugo/libcentrifugo/logger"
	"github.com/tarantool/go-tarantool"
	"strings"
)

func (p *TarantoolPool) get() (conn *tarantool.Connection, err error) {
	if len(p.pool) == 0 {
		return nil, errors.New("Empty tarantool pool")
	}
	conn = p.pool[p.current]
	p.current++
	p.current = (p.current) % len(p.pool)
	return
}

type TarantoolEngine struct {
	app      *Application
	pool     *TarantoolPool
	extender extender.Extender
	endpoint string
}

type TarantoolEngineConfig struct {
	PoolConfig  TarantoolPoolConfig
	Endpoint    string
	TTConnector extender.Config
}

type TarantoolPool struct {
	pool    []*tarantool.Connection
	config  TarantoolPoolConfig
	current int
}

type TarantoolPoolConfig struct {
	Address  string
	PoolSize int
	Opts     tarantool.Opts
}

/* MessageType
{
	"body": {
		"uid":"026c380d-13e1-47d9-42d2-e2dc0e41e8d5",
		"timestamp":"1440434259",
		"info":{
			"user":"3",
			"client":"83309b33-deb7-48ff-76c6-04b10e6a6523",
			"default_info":null,
			"channel_info": {
				"channel_extra_info_example":"you can add additional JSON data when authorizing"
			}
		},
		"channel":"$3_0",
		"data": {
				"Action":"mark",
				"Data":["00000000000000395684"]
			},
		"client":"83309b33-deb7-48ff-76c6-04b10e6a6523"
	},
	"error":null,
	"method":"message"
}
*/

type MessageType struct {
	Body   Message `json:"body"`
	Error  string  `json:"error"`
	Method string  `json:"method"`
}

type ServiceMessage struct {
	Action string
	Data   []string
}

type IDs []string

func NewTarantoolEngine(app *Application, conf TarantoolEngineConfig) *TarantoolEngine {
	logger.INFO.Printf("Initializing tarantool connection pool...")
	pool, err := newTarantoolPool(conf.PoolConfig)
	if err != nil {
		logger.FATAL.Fatalln(err)
	}

	extender, err := extender.New(conf.TTConnector)
	if err != nil {
		logger.FATAL.Fatalln(err)
	}

	e := &TarantoolEngine{
		app:      app,
		pool:     pool,
		extender: extender,
		endpoint: conf.Endpoint,
	}

	return e
}

func newTarantoolPool(config TarantoolPoolConfig) (p *TarantoolPool, err error) {
	if config.PoolSize == 0 {
		err = errors.New("Size of tarantool pool is zero")
		return
	}

	p = &TarantoolPool{
		pool:   make([]*tarantool.Connection, config.PoolSize),
		config: config,
	}

	for i := 0; i < config.PoolSize; i++ {
		logger.INFO.Printf("[%d] Connecting to tarantool on %s... [%d]", i, config.Address, config.Opts.MaxReconnects)
		p.pool[i], err = tarantool.Connect(config.Address, config.Opts)
		if err != nil && config.Opts.Reconnect > 0 {
			logger.ERROR.Printf("[%d] connection to tarantool on %s failed with '%s'", i, config.Address, err)
			err = nil // just log and reset error: reconnection inside tarantool.Connect
		}
		if err == nil {
			logger.INFO.Printf("[%d] Connected to tarantool on %s", i, config.Address)
		}
	}

	return
}

// getName returns a name of concrete engine implementation
func (e *TarantoolEngine) name() string {
	return "Tarantool"
}

// publish allows to send message into channel
func (e *TarantoolEngine) publish(chID ChannelID, message []byte) error {
	// Process service messages
	if chID != e.app.config.ControlChannel && chID != e.app.config.AdminChannel {
		newMessage, err := e.processMessage(chID, message)
		if err != nil {
			return err // if no need further processing
		}
		message = newMessage
	}
	// All other messages
	return e.app.handleMsg(chID, message)
}

// subscribe on channel
func (e *TarantoolEngine) subscribe(chID ChannelID) (err error) {
	logger.INFO.Printf("subscribe %s", chID)
	endpoint, err := e.makeEndpointFromChannelID(chID)
	if err != nil {
		logger.ERROR.Printf("subscribe make endpoint string error: %v\n", err.Error())
		return
	}

	conn, err := e.pool.get()
	if err != nil {
		logger.ERROR.Printf("subscribe tarantool pool error: %v\n", err.Error())
		return
	}

	_, err = conn.Call("notification_channel_subscribe", []interface{}{chID, endpoint})

	return
}

// unsubscribe from channel
func (e *TarantoolEngine) unsubscribe(chID ChannelID) (err error) {
	endpoint, err := e.makeEndpointFromChannelID(chID)
	if err != nil {
		logger.ERROR.Printf("unsubscribe make endpoint string error: %v\n", err.Error())
		return
	}

	conn, err := e.pool.get()
	if err != nil {
		logger.ERROR.Printf("unsubscribe tarantool pool error: %v\n", err.Error())
		return
	}

	_, err = conn.Call("notification_channel_unsubscribe", []interface{}{chID, endpoint})

	return
}

// addPresence sets or updates presence info for connection with uid
func (e *TarantoolEngine) addPresence(chID ChannelID, uid ConnID, info ClientInfo) (err error) {
	// not implemented
	return
}

// removePresence removes presence information for connection with uid
func (e *TarantoolEngine) removePresence(chID ChannelID, uid ConnID) (err error) {
	// not implemented
	return
}

// getPresence returns actual presence information for channel
func (e *TarantoolEngine) presence(chID ChannelID) (result map[ConnID]ClientInfo, err error) {
	// not implemented
	return
}

// addHistory adds message into channel history and takes care about history size
func (e *TarantoolEngine) addHistory(chID ChannelID, message Message, size, lifetime int64) (err error) {
	// not implemented
	return
}

// getHistory returns history messages for channel
// return empty slice
// all history pushed via publish
func (e *TarantoolEngine) history(chID ChannelID) (msgs []Message, err error) {
	conn, err := e.pool.get()
	if err != nil {
		logger.ERROR.Printf("history tarantool pool error: %v\n", err.Error())
		return nil, err
	}

	history, err := conn.Call("notification_notification_history", []interface{}{chID})
	if err != nil {
		logger.ERROR.Printf("history error: %v\n", err.Error())
		return nil, err
	}

	return processHistory(history)
}

// helpers

type tarantoolHistoryItem struct {
	Count  interface{} `json:"count"`
	Status string      `json:"status"`
	ID     string      `json:"id"`
}

func processHistory(history *tarantool.Response) (msgs []Message, err error) {
	if len(history.Data) == 0 {
		return // history is empty
	}

	data := history.Data[0].([]interface{})
	if len(data) != 2 {
		return // history is empty
	}

	count := data[0]                       // ring counter
	buffer := data[1].(string)             // string buffer
	ring := strings.Split(buffer[1:], ",") // array of IDs

	if len(ring) == 0 {
		return // history buffer is empty [useless?]
	}

	for _, id := range ring {
		encoded, err := json.Marshal(tarantoolHistoryItem{
			Count:  count, // redundancy in each item to pass number of unread notifications
			Status: string(id[0]),
			ID:     string(id[1:]),
		})
		if err != nil {
			logger.ERROR.Println(err)
			continue
		}
		rawMessage := json.RawMessage([]byte(encoded))
		msgs = append(msgs, Message{Data: &rawMessage})
	}

	return
}

func (e *TarantoolEngine) extendMessage(chID ChannelID, message []byte) (newMessage []byte, err error) {
	logger.DEBUG.Printf("try to extend message chID = %s, message = %s", chID, string(message))

	//uid, _, _, err := parseChannelID(chID)
	//if err != nil {
	//	return
	//}

	var m MessageType
	err = json.Unmarshal(message, &m)
	if err != nil {
		return
	}

	extended, err := e.extender.Extend(m.Body.Data, string(chID))
	if extended != nil {
		m.Body.Data = extended
		newMessage, err = json.Marshal(&m)
		logger.DEBUG.Printf("data extended to: %s", string(*m.Body.Data))
	}

	return
}

func (e *TarantoolEngine) processMessage(chID ChannelID, message []byte) (newMessage []byte, err error) {
	newMessage = message // by default, but may be changed

	var msg MessageType
	err = json.Unmarshal(message, &msg)
	if err != nil {
		return
	}

	var srv ServiceMessage
	err = json.Unmarshal(*msg.Body.Data, &srv)
	if err != nil {
		return
	}

	newMessage, err = e.extendMessage(chID, message)
	if err != nil {
		logger.ERROR.Printf("extend message failed with '%s'", err)
	}

	return
}

func (e *TarantoolEngine) makeEndpointFromChannelID(chID ChannelID) (endpoint string, err error) {
	// split chID <centrifugo>.<project>.[$]<uid>_<ringno>
	str := string(chID)
	logger.INFO.Printf("makeEndpointFromChannelID %s", str)
	result := strings.Split(str, ".")
	if len(result) != 3 {
		err = fmt.Errorf("unexpected ChannelID %s", str)
		return
	}
	endpoint = e.endpoint + "/api/" + result[1]
	return
}
<|endoftext|>