<commit_before>// Copyright 2013 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package commands

import (
	"fmt"
	. "github.com/limetext/lime/backend"
	"github.com/limetext/lime/backend/util"
	"github.com/limetext/text"
)

const (
	// Beginning of line
	BOL MoveToType = iota
	// End of line
	EOL
	// Beginning of file
	BOF
	// End of file
	EOF
	// Current level close bracket
	Brackets
)

const (
	// Move by Characters
	Characters MoveByType = iota
	// Move by Stops (TODO(.): what exactly is a stop?)
	Stops
	// Move by Lines
	Lines
	// Move by Words
	Words
	// Move by Word Ends
	WordEnds
	// Move by Sub Words
	SubWords
	// Move by Sub Word Ends
	SubWordEnds
	// Move by Page
	Pages
)

type (
	// The MoveCommand moves the current selection
	MoveCommand struct {
		DefaultCommand
		// Specifies the type of "move" operation
		By MoveByType
		// Whether the current selection should be extended or not
		Extend bool
		// Whether to move forward or backwards
		Forward bool
		// Used together with By=Stops, and ??? (TODO(.): document better)
		WordBegin bool
		// Used together with By=Stops, and ??? (TODO(.): document better)
		PunctBegin bool
		Separators bool
	}

	// Specifies the type of "move" operation
	MoveByType int

	// Specifies the type of "move_to" operation to perform
	MoveToType int

	// The MoveToCommand moves or extends the current selection to the specified location
	MoveToCommand struct {
		DefaultCommand
		// The type of "move_to" operation to perform
		To MoveToType
		// Whether the current selection should be extended or not
		Extend bool
	}

	// The ScrollLinesCommand moves the viewpoint "Amount" lines from the current location
	ScrollLinesCommand struct {
		BypassUndoCommand
		// The number of lines to scroll (positive or negative direction)
		Amount int
	}
)

func move_action(v *View, extend bool, transform func(r text.Region) int) {
	sel := v.Sel()
	r := sel.Regions()
	bs := v.Buffer().Size()
	for i := range r {
		r[i].B = transform(r[i])
		if r[i].B < 0 {
			r[i].B = 0
		} else if r[i].B > bs {
			// Yes > the size, and not size-1 because the cursor being at "size"
			// is the position it will be at when we are appending
			// to the buffer
			r[i].B = bs
		}

		if !extend {
			r[i].A = r[i].B
		}
	}
	sel.Clear()
	sel.AddAll(r)
}

func (mt *MoveToType) Set(v interface{}) error {
	switch to := v.(string); to {
	case "eol":
		*mt = EOL
	case "bol":
		*mt = BOL
	case "bof":
		*mt = BOF
	case "eof":
		*mt = EOF
	case "brackets":
		*mt = Brackets
	default:
		return fmt.Errorf("move_to: Unimplemented 'to' type: %s", to)
	}
	return nil
}

func (c *MoveToCommand) Run(v *View, e *Edit) error {
	switch c.To {
	case EOL:
		move_action(v, c.Extend, func(r text.Region) int {
			line := v.Buffer().Line(r.B)
			return line.B
		})
	case BOL:
		move_action(v, c.Extend, func(r text.Region) int {
			line := v.Buffer().Line(r.B)
			return line.A
		})
	case BOF:
		move_action(v, c.Extend, func(r text.Region) int {
			return 0
		})
	case EOF:
		move_action(v, c.Extend, func(r text.Region) int {
			return v.Buffer().Size()
		})
	case Brackets:
		move_action(v, c.Extend, func(r text.Region) int {
			text := v.Buffer().Substr(tex.Region{r.B, v.Buffer().Size()})
			count := 1
			for i, c := range text {
				switch c {
				case '(':
					c++
				case '[':
					c++
				case '{':
					c++
				case ')':
					c--
				case ']':
					c--
				case '}':
					c--
				}
				if c == 0 {
					return i + r.B
				}
			}
			return r.B
		})
	default:
		return fmt.Errorf("move_to: Unimplemented 'to' action: %d", c.To)
	}
	return nil
}

func (m *MoveByType) Set(v interface{}) error {
	switch by := v.(string); by {
	case "lines":
		*m = Lines
	case "characters":
		*m = Characters
	case "stops":
		*m = Stops
	case "words":
		*m = Words
	case "word_ends":
		*m = WordEnds
	case "subwords":
		*m = SubWords
	case "subword_ends":
		*m = SubWordEnds
	case "pages":
		*m = Pages
	default:
		return fmt.Errorf("move: Unimplemented 'by' action: %s", by)
	}
	return nil
}

func (c *MoveCommand) Run(v *View, e *Edit) error {
	p := util.Prof.Enter("move.run.init")

	p.Exit()
	p = util.Prof.Enter("move.run.action")
	defer p.Exit()

	switch c.By {
	case Characters:
		dir := 1
		if !c.Forward {
			dir = -1
		}
		move_action(v, c.Extend, func(r text.Region) int {
			return r.B + dir
		})
	case Stops:
		move_action(v, c.Extend, func(r text.Region) int {
			var next text.Region
			word := v.Buffer().Word(r.B)
			if c.WordEnd && c.Forward && r.B < word.End() {
				next = word
			} else if c.WordBegin && !c.Forward && r.B > word.Begin() {
				next = word
			} else if c.Forward {
				next = v.Buffer().Word(word.B + 1)
			} else {
				next = v.Buffer().Word(word.A - 1)
				next = v.Buffer().Word(next.A - 1)
			}

			if c.WordBegin {
				return next.A
			} else if c.WordEnd {
				return next.B
			}
			return r.B
		})
	case Lines:
		move_action(v, c.Extend, func(in text.Region) int {
			r, col := v.Buffer().RowCol(in.B)
			_ = r
			if !c.Forward {
				r--
			} else {
				r++
			}
			return v.Buffer().TextPoint(r, col)
		})
	}
	return nil
}

func (c *ScrollLinesCommand) Run(v *View, e *Edit) error {
	fe := GetEditor().Frontend()
	vr := fe.VisibleRegion(v)
	var r int
	if c.Amount >= 0 {
		r, _ = v.Buffer().RowCol(vr.Begin())
		r -= c.Amount
	} else {
		r, _ = v.Buffer().RowCol(vr.End() - 1)
		r -= c.Amount
	}
	r = v.Buffer().TextPoint(r, 0)
	fe.Show(v, text.Region{r, r})
	return nil
}

func init() {
	register([]Command{
		&MoveCommand{},
		&MoveToCommand{},
		&ScrollLinesCommand{},
	})
}
<commit_msg>Completed MoveCommand type variables + documentation<commit_after>// Copyright 2013 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package commands

import (
	"fmt"
	. "github.com/limetext/lime/backend"
	"github.com/limetext/lime/backend/util"
	"github.com/limetext/text"
)

const (
	// Beginning of line
	BOL MoveToType = iota
	// End of line
	EOL
	// Beginning of file
	BOF
	// End of file
	EOF
	// Current level close bracket
	Brackets
)

const (
	// Move by Characters
	Characters MoveByType = iota
	// Move by Stops (TODO(.): what exactly is a stop?)
	Stops
	// Move by Lines
	Lines
	// Move by Words
	Words
	// Move by Word Ends
	WordEnds
	// Move by Sub Words
	SubWords
	// Move by Sub Word Ends
	SubWordEnds
	// Move by Page
	Pages
)

type (
	// The MoveCommand moves the current selection
	MoveCommand struct {
		DefaultCommand
		// Specifies the type of "move" operation
		By MoveByType
		// Whether the current selection should be extended or not
		Extend bool
		// Whether to move forward or backwards
		Forward bool
		// Used together with By=Stops, go to word begin
		WordBegin bool
		// Used together with By=Stops, go to word end
		WordEnd bool
		// Used together with By=Stops, go to punctuation begin
		PunctBegin bool
		// Used together with By=Stops, go to punctuation end
		PunctEnd bool
		// Used together with By=Stops, extends "word_separators" defined by settings
		Separators string
		// Used together with By=Stops, go to an empty line
		EmptyLine bool
		// Used together with By=Stops, TODO: ???
		ClipToLine bool
	}

	// Specifies the type of "move" operation
	MoveByType int

	// Specifies the type of "move_to" operation to perform
	MoveToType int

	// The MoveToCommand moves or extends the current selection to the specified location
	MoveToCommand struct {
		DefaultCommand
		// The type of "move_to" operation to perform
		To MoveToType
		// Whether the current selection should be extended or not
		Extend bool
	}

	// The ScrollLinesCommand moves the viewpoint "Amount" lines from the current location
	ScrollLinesCommand struct {
		BypassUndoCommand
		// The number of lines to scroll (positive or negative direction)
		Amount int
	}
)

func move_action(v *View, extend bool, transform func(r text.Region) int) {
	sel := v.Sel()
	r := sel.Regions()
	bs := v.Buffer().Size()
	for i := range r {
		r[i].B = transform(r[i])
		if r[i].B < 0 {
			r[i].B = 0
		} else if r[i].B > bs {
			// Yes > the size, and not size-1 because the cursor being at "size"
			// is the position it will be at when we are appending
			// to the buffer
			r[i].B = bs
		}

		if !extend {
			r[i].A = r[i].B
		}
	}
	sel.Clear()
	sel.AddAll(r)
}

func (mt *MoveToType) Set(v interface{}) error {
	switch to := v.(string); to {
	case "eol":
		*mt = EOL
	case "bol":
		*mt = BOL
	case "bof":
		*mt = BOF
	case "eof":
		*mt = EOF
	case "brackets":
		*mt = Brackets
	default:
		return fmt.Errorf("move_to: Unimplemented 'to' type: %s", to)
	}
	return nil
}

func (c *MoveToCommand) Run(v *View, e *Edit) error {
	switch c.To {
	case EOL:
		move_action(v, c.Extend, func(r text.Region) int {
			line := v.Buffer().Line(r.B)
			return line.B
		})
	case BOL:
		move_action(v, c.Extend, func(r text.Region) int {
			line := v.Buffer().Line(r.B)
			return line.A
		})
	case BOF:
		move_action(v, c.Extend, func(r text.Region) int {
			return 0
		})
	case EOF:
		move_action(v, c.Extend, func(r text.Region) int {
			return v.Buffer().Size()
		})
	case Brackets:
		move_action(v, c.Extend, func(r text.Region) int {
			text := v.Buffer().Substr(tex.Region{r.B, v.Buffer().Size()})
			count := 1
			for i, c := range text {
				switch c {
				case '(':
					c++
				case '[':
					c++
				case '{':
					c++
				case ')':
					c--
				case ']':
					c--
				case '}':
					c--
				}
				if c == 0 {
					return i + r.B
				}
			}
			return r.B
		})
	default:
		return fmt.Errorf("move_to: Unimplemented 'to' action: %d", c.To)
	}
	return nil
}

func (m *MoveByType) Set(v interface{}) error {
	switch by := v.(string); by {
	case "lines":
		*m = Lines
	case "characters":
		*m = Characters
	case "stops":
		*m = Stops
	case "words":
		*m = Words
	case "word_ends":
		*m = WordEnds
	case "subwords":
		*m = SubWords
	case "subword_ends":
		*m = SubWordEnds
	case "pages":
		*m = Pages
	default:
		return fmt.Errorf("move: Unimplemented 'by' action: %s", by)
	}
	return nil
}

func (c *MoveCommand) Run(v *View, e *Edit) error {
	p := util.Prof.Enter("move.run.init")

	p.Exit()
	p = util.Prof.Enter("move.run.action")
	defer p.Exit()

	switch c.By {
	case Characters:
		dir := 1
		if !c.Forward {
			dir = -1
		}
		move_action(v, c.Extend, func(r text.Region) int {
			return r.B + dir
		})
	case Stops:
		move_action(v, c.Extend, func(r text.Region) int {
			var next text.Region
			word := v.Buffer().Word(r.B)
			if c.WordEnd && c.Forward && r.B < word.End() {
				next = word
			} else if c.WordBegin && !c.Forward && r.B > word.Begin() {
				next = word
			} else if c.Forward {
				next = v.Buffer().Word(word.B + 1)
			} else {
				next = v.Buffer().Word(word.A - 1)
				next = v.Buffer().Word(next.A - 1)
			}

			if c.WordBegin {
				return next.A
			} else if c.WordEnd {
				return next.B
			}
			return r.B
		})
	case Lines:
		move_action(v, c.Extend, func(in text.Region) int {
			r, col := v.Buffer().RowCol(in.B)
			_ = r
			if !c.Forward {
				r--
			} else {
				r++
			}
			return v.Buffer().TextPoint(r, col)
		})
	}
	return nil
}

func (c *ScrollLinesCommand) Run(v *View, e *Edit) error {
	fe := GetEditor().Frontend()
	vr := fe.VisibleRegion(v)
	var r int
	if c.Amount >= 0 {
		r, _ = v.Buffer().RowCol(vr.Begin())
		r -= c.Amount
	} else {
		r, _ = v.Buffer().RowCol(vr.End() - 1)
		r -= c.Amount
	}
	r = v.Buffer().TextPoint(r, 0)
	fe.Show(v, text.Region{r, r})
	return nil
}

func init() {
	register([]Command{
		&MoveCommand{},
		&MoveToCommand{},
		&ScrollLinesCommand{},
	})
}
<|endoftext|>


<commit_before>// Copyright 2016 Red Hat, Inc. and/or its affiliates
// and other contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package backend
package memory

import (
	"net/url"
	"regexp"
	"time"

	"github.com/yaacov/mohawk/backend"
)

type TimeValuePair struct {
	timeStamp int64
	value     float64
}

type TimeSeries struct {
	tags map[string]string
	data []TimeValuePair
}

type Tenant struct {
	ts map[string]*TimeSeries
}

type Backend struct {
	firstPosTimestamp  int64
	timeGranularitySec int64
	timeRetentionSec   int64

	tenant map[string]*Tenant
}

// Backend functions
// Required by backend interface

func (r Backend) Name() string {
	return "Backend-Memory"
}

func (r *Backend) Open(options url.Values) {
	// set time granularity to 30 sec
	r.timeGranularitySec = 30
	// set time retention to 7 days
	r.timeRetentionSec = 7 * 24 * 60 * 60
	// set first time stamp to now
	r.firstPosTimestamp = int64(time.Now().UTC().Unix() / r.timeGranularitySec)

	// open db connection
	r.tenant = make(map[string]*Tenant, 0)
}

func (r Backend) GetTenants() []backend.Tenant {
	res := make([]backend.Tenant, 0)

	// return a list of tenants
	for key, _ := range r.tenant {
		res = append(res, backend.Tenant{Id: key})
	}

	return res
}

func (r Backend) GetItemList(tenant string, tags map[string]string) []backend.Item {
	res := make([]backend.Item, 0)
	t, ok := r.tenant[tenant]

	// check tenant
	if !ok {
		return res
	}

	for key, ts := range t.ts {
		res = append(res, backend.Item{
			Id:   key,
			Type: "gauge",
			Tags: ts.tags,
		})
	}

	// filter using tags
	// 	if we have a list of _all_ items, we need to filter them by tags
	// 	if the list is already filtered, we do not need to re-filter it
	if len(tags) > 0 {
		for key, value := range tags {
			res = backend.FilterItems(res, func(i backend.Item) bool {
				r, _ := regexp.Compile("^" + value + "$")
				return r.MatchString(i.Tags[key])
			})
		}
	}

	return res
}

func (r *Backend) GetRawData(tenant string, id string, end int64, start int64, limit int64, order string) []backend.DataItem {
	res := make([]backend.DataItem, 0)

	arraySize := r.timeRetentionSec / r.timeGranularitySec
	pStart := r.getPosForTimestamp(start)
	pEnd := r.getPosForTimestamp(end)

	// sanity check pEnd
	if pEnd < pStart {
		pEnd += arraySize
	}

	// check if tenant and id exists, create them if necessary
	r.checkID(tenant, id)

	// fill data out array
	count := int64(0)
	for i := pStart; count < limit && i <= pEnd; i++ {
		d := r.tenant[tenant].ts[id].data[i%arraySize]

		// if this is a valid point
		if d.timeStamp <= end && d.timeStamp > start {
			count++
			res = append(res, backend.DataItem{
				Timestamp: d.timeStamp,
				Value:     d.value,
			})
		}
	}

	return res
}

func (r Backend) GetStatData(tenant string, id string, end int64, start int64, limit int64, order string, bucketDuration int64) []backend.StatItem {
	var res []backend.StatItem

	arraySize := r.timeRetentionSec / r.timeGranularitySec
	pStep := bucketDuration / r.timeGranularitySec
	pStart := r.getPosForTimestamp(start)
	pEnd := r.getPosForTimestamp(end)

	// sanity check pEnd
	if pEnd < pStart {
		pEnd += arraySize
	}

	// sanity check step
	if pStep < 1 {
		pStep = 1
	}
	if pStep > (pEnd - pStart) {
		pStep = pEnd - pStart
	}

	startTimestamp := end
	stepMillisec := pStep * r.timeGranularitySec * 1000

	// check if tenant and id exists, create them if necessary
	r.checkID(tenant, id)

	// fill data out array
	count := int64(0)
	for b := pEnd; count < limit && b > pStart && startTimestamp > stepMillisec; b -= pStep {
		samples := int64(0)
		sum := float64(0)
		last := float64(0)

		// loop on all points in bucket
		for i := (b - pStep); i < b; i++ {
			d := r.tenant[tenant].ts[id].data[i%arraySize]
			if d.timeStamp <= end && d.timeStamp > start {
				samples++
				last = d.value
				sum += d.value
			}
		}

		// all points are valid
		startTimestamp -= stepMillisec
		count++

		// all points are valid
		if samples > 0 {
			res = append(res, backend.StatItem{
				Start:   startTimestamp,
				End:     startTimestamp + stepMillisec,
				Empty:   false,
				Samples: samples,
				Min:     0,
				Max:     last,
				Avg:     sum / float64(samples),
				Median:  0,
				Sum:     sum,
			})
		} else {
			count++
			res = append(res, backend.StatItem{
				Start:   startTimestamp,
				End:     startTimestamp + stepMillisec,
				Empty:   true,
				Samples: 0,
				Min:     0,
				Max:     0,
				Avg:     0,
				Median:  0,
				Sum:     0,
			})
		}
	}

	return res
}

func (r *Backend) PostRawData(tenant string, id string, t int64, v float64) bool {
	// check if tenant and id exists, create them if necessary
	r.checkID(tenant, id)

	// update time value pair to the time serias
	p := r.getPosForTimestamp(t)
	r.tenant[tenant].ts[id].data[p] = TimeValuePair{timeStamp: t, value: v}

	return true
}

func (r *Backend) PutTags(tenant string, id string, tags map[string]string) bool {
	// check if tenant and id exists, create them if necessary
	r.checkID(tenant, id)

	// update time serias tags
	if len(tags) > 0 {
		for key, value := range tags {
			r.tenant[tenant].ts[id].tags[key] = value
		}
	}

	return true
}

func (r *Backend) DeleteData(tenant string, id string, end int64, start int64) bool {
	return true
}

func (r *Backend) DeleteTags(tenant string, id string, tags []string) bool {
	return true
}

// Helper functions
// Not required by backend interface

func (r *Backend) getPosForTimestamp(timestamp int64) int64 {
	arraySize := r.timeRetentionSec / r.timeGranularitySec
	arrayPos := timestamp / 1000 / r.timeGranularitySec

	return arrayPos % arraySize
}

func (r *Backend) checkID(tenant string, id string) {
	var ok bool

	// check for tenant
	if _, ok = r.tenant[tenant]; !ok {
		r.tenant[tenant] = &Tenant{ts: make(map[string]*TimeSeries)}
	}

	// check for TimeSeries
	if _, ok = r.tenant[tenant].ts[id]; !ok {
		r.tenant[tenant].ts[id] = &TimeSeries{
			tags: make(map[string]string),
			data: make([]TimeValuePair, r.timeRetentionSec/r.timeGranularitySec),
		}
	}
}
<commit_msg>remove first timestamp<commit_after>// Copyright 2016 Red Hat, Inc. and/or its affiliates
// and other contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package backend
package memory

import (
	"net/url"
	"regexp"

	"github.com/yaacov/mohawk/backend"
)

type TimeValuePair struct {
	timeStamp int64
	value     float64
}

type TimeSeries struct {
	tags map[string]string
	data []TimeValuePair
}

type Tenant struct {
	ts map[string]*TimeSeries
}

type Backend struct {
	timeGranularitySec int64
	timeRetentionSec   int64

	tenant map[string]*Tenant
}

// Backend functions
// Required by backend interface

func (r Backend) Name() string {
	return "Backend-Memory"
}

func (r *Backend) Open(options url.Values) {
	// set time granularity to 30 sec
	r.timeGranularitySec = 30
	// set time retention to 7 days
	r.timeRetentionSec = 7 * 24 * 60 * 60

	// open db connection
	r.tenant = make(map[string]*Tenant, 0)
}

func (r Backend) GetTenants() []backend.Tenant {
	res := make([]backend.Tenant, 0)

	// return a list of tenants
	for key, _ := range r.tenant {
		res = append(res, backend.Tenant{Id: key})
	}

	return res
}

func (r Backend) GetItemList(tenant string, tags map[string]string) []backend.Item {
	res := make([]backend.Item, 0)
	t, ok := r.tenant[tenant]

	// check tenant
	if !ok {
		return res
	}

	for key, ts := range t.ts {
		res = append(res, backend.Item{
			Id:   key,
			Type: "gauge",
			Tags: ts.tags,
		})
	}

	// filter using tags
	// 	if we have a list of _all_ items, we need to filter them by tags
	// 	if the list is already filtered, we do not need to re-filter it
	if len(tags) > 0 {
		for key, value := range tags {
			res = backend.FilterItems(res, func(i backend.Item) bool {
				r, _ := regexp.Compile("^" + value + "$")
				return r.MatchString(i.Tags[key])
			})
		}
	}

	return res
}

func (r *Backend) GetRawData(tenant string, id string, end int64, start int64, limit int64, order string) []backend.DataItem {
	res := make([]backend.DataItem, 0)

	arraySize := r.timeRetentionSec / r.timeGranularitySec
	pStart := r.getPosForTimestamp(start)
	pEnd := r.getPosForTimestamp(end)

	// sanity check pEnd
	if pEnd < pStart {
		pEnd += arraySize
	}

	// check if tenant and id exists, create them if necessary
	r.checkID(tenant, id)

	// fill data out array
	count := int64(0)
	for i := pStart; count < limit && i <= pEnd; i++ {
		d := r.tenant[tenant].ts[id].data[i%arraySize]

		// if this is a valid point
		if d.timeStamp <= end && d.timeStamp > start {
			count++
			res = append(res, backend.DataItem{
				Timestamp: d.timeStamp,
				Value:     d.value,
			})
		}
	}

	return res
}

func (r Backend) GetStatData(tenant string, id string, end int64, start int64, limit int64, order string, bucketDuration int64) []backend.StatItem {
	var res []backend.StatItem

	arraySize := r.timeRetentionSec / r.timeGranularitySec
	pStep := bucketDuration / r.timeGranularitySec
	pStart := r.getPosForTimestamp(start)
	pEnd := r.getPosForTimestamp(end)

	// sanity check pEnd
	if pEnd < pStart {
		pEnd += arraySize
	}

	// sanity check step
	if pStep < 1 {
		pStep = 1
	}
	if pStep > (pEnd - pStart) {
		pStep = pEnd - pStart
	}

	startTimestamp := end
	stepMillisec := pStep * r.timeGranularitySec * 1000

	// check if tenant and id exists, create them if necessary
	r.checkID(tenant, id)

	// fill data out array
	count := int64(0)
	for b := pEnd; count < limit && b > pStart && startTimestamp > stepMillisec; b -= pStep {
		samples := int64(0)
		sum := float64(0)
		last := float64(0)

		// loop on all points in bucket
		for i := (b - pStep); i < b; i++ {
			d := r.tenant[tenant].ts[id].data[i%arraySize]
			if d.timeStamp <= end && d.timeStamp > start {
				samples++
				last = d.value
				sum += d.value
			}
		}

		// all points are valid
		startTimestamp -= stepMillisec
		count++

		// all points are valid
		if samples > 0 {
			res = append(res, backend.StatItem{
				Start:   startTimestamp,
				End:     startTimestamp + stepMillisec,
				Empty:   false,
				Samples: samples,
				Min:     0,
				Max:     last,
				Avg:     sum / float64(samples),
				Median:  0,
				Sum:     sum,
			})
		} else {
			count++
			res = append(res, backend.StatItem{
				Start:   startTimestamp,
				End:     startTimestamp + stepMillisec,
				Empty:   true,
				Samples: 0,
				Min:     0,
				Max:     0,
				Avg:     0,
				Median:  0,
				Sum:     0,
			})
		}
	}

	return res
}

func (r *Backend) PostRawData(tenant string, id string, t int64, v float64) bool {
	// check if tenant and id exists, create them if necessary
	r.checkID(tenant, id)

	// update time value pair to the time serias
	p := r.getPosForTimestamp(t)
	r.tenant[tenant].ts[id].data[p] = TimeValuePair{timeStamp: t, value: v}

	return true
}

func (r *Backend) PutTags(tenant string, id string, tags map[string]string) bool {
	// check if tenant and id exists, create them if necessary
	r.checkID(tenant, id)

	// update time serias tags
	if len(tags) > 0 {
		for key, value := range tags {
			r.tenant[tenant].ts[id].tags[key] = value
		}
	}

	return true
}

func (r *Backend) DeleteData(tenant string, id string, end int64, start int64) bool {
	return true
}

func (r *Backend) DeleteTags(tenant string, id string, tags []string) bool {
	return true
}

// Helper functions
// Not required by backend interface

func (r *Backend) getPosForTimestamp(timestamp int64) int64 {
	arraySize := r.timeRetentionSec / r.timeGranularitySec
	arrayPos := timestamp / 1000 / r.timeGranularitySec

	return arrayPos % arraySize
}

func (r *Backend) checkID(tenant string, id string) {
	var ok bool

	// check for tenant
	if _, ok = r.tenant[tenant]; !ok {
		r.tenant[tenant] = &Tenant{ts: make(map[string]*TimeSeries)}
	}

	// check for TimeSeries
	if _, ok = r.tenant[tenant].ts[id]; !ok {
		r.tenant[tenant].ts[id] = &TimeSeries{
			tags: make(map[string]string),
			data: make([]TimeValuePair, r.timeRetentionSec/r.timeGranularitySec),
		}
	}
}
<|endoftext|>


<commit_before>// Copyright © 2017 The Kubicorn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubeconfig

import (
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"syscall"
	"time"

	"github.com/kris-nova/kubicorn/apis/cluster"
	"github.com/kris-nova/kubicorn/cutil/local"
	"github.com/kris-nova/kubicorn/cutil/logger"
	"github.com/pkg/sftp"
	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
	"golang.org/x/crypto/ssh/terminal"
)

func GetConfig(existing *cluster.Cluster) error {
	user := existing.SSH.User
	pubKeyPath := local.Expand(existing.SSH.PublicKeyPath)
	privKeyPath := strings.Replace(pubKeyPath, ".pub", "", 1)
	address := fmt.Sprintf("%s:%s", existing.KubernetesAPI.Endpoint, "22")
	localDir := fmt.Sprintf("%s/.kube", local.Home())
	localPath, err := getKubeConfigPath(localDir)
	if err != nil {
		return err
	}
	sshConfig := &ssh.ClientConfig{
		User:            user,
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
	}
	remotePath := ""
	if user == "root" {
		remotePath = "/root/.kube/config"
	} else {
		remotePath = fmt.Sprintf("/home/%s/.kube/config", user)
	}

	//fmt.Println(pubKeyPath)
	//fmt.Println(privKeyPath)
	//fmt.Println(address)
	//fmt.Println(user)
	//fmt.Println(remotePath)
	//fmt.Println(localPath)

	pemBytes, err := ioutil.ReadFile(privKeyPath)
	if err != nil {
		return err
	}
	signer, err := getSigner(pemBytes)
	if err != nil {
		return err
	}
	sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signer))
	agent := sshAgent()
	if agent != nil {
		sshConfig.Auth = append(sshConfig.Auth, agent)
	}
	sshConfig.SetDefaults()
	conn, err := ssh.Dial("tcp", address, sshConfig)
	if err != nil {
		return err
	}
	defer conn.Close()
	c, err := sftp.NewClient(conn)
	if err != nil {
		return err
	}
	defer c.Close()
	r, err := c.Open(remotePath)
	if err != nil {
		return err
	}
	defer r.Close()
	bytes, err := ioutil.ReadAll(r)
	if err != nil {
		return err
	}

	if _, err := os.Stat(localPath); os.IsNotExist(err) {
		empty := []byte("")
		err := ioutil.WriteFile(localPath, empty, 0755)
		if err != nil {
			return err
		}
	}

	f, err := os.OpenFile(localPath, os.O_APPEND|os.O_WRONLY, os.ModeAppend)
	if err != nil {
		return err
	}
	_, err = f.WriteString(string(bytes))
	if err != nil {
		return err
	}
	defer f.Close()
	logger.Always("Wrote kubeconfig to [%s]", localPath)
	return nil
}

const (
	RetryAttempts     = 150
	RetrySleepSeconds = 5
)

func RetryGetConfig(existing *cluster.Cluster) error {
	for i := 0; i <= RetryAttempts; i++ {
		err := GetConfig(existing)
		if err != nil {
			if strings.Contains(err.Error(), "file does not exist") || strings.Contains(err.Error(), "getsockopt: connection refused") || strings.Contains(err.Error(), "unable to authenticate") {
				logger.Debug("Waiting for Kubernetes to come up.. [%v]", err)
				time.Sleep(time.Duration(RetrySleepSeconds) * time.Second)
				continue
			}
			return err
		}
		return nil
	}
	return fmt.Errorf("Timedout writing kubeconfig")
}

func getSigner(pemBytes []byte) (ssh.Signer, error) {
	signerwithoutpassphrase, err := ssh.ParsePrivateKey(pemBytes)
	if err != nil {
		logger.Debug(err.Error())
		fmt.Print("SSH Key Passphrase [none]: ")
		passPhrase, err := terminal.ReadPassword(int(syscall.Stdin))
		fmt.Println("")
		if err != nil {
			return nil, err
		}
		signerwithpassphrase, err := ssh.ParsePrivateKeyWithPassphrase(pemBytes, passPhrase)
		if err != nil {
			return nil, err
		}

		return signerwithpassphrase, err
	}

	return signerwithoutpassphrase, err
}

func sshAgent() ssh.AuthMethod {
	sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
	if err == nil {
		return ssh.PublicKeysCallback(agent.NewClient(sshAgent).Signers)
	}
	return nil
}

func getKubeConfigPath(path string) (string, error) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		if err := os.Mkdir(path, 0777); err != nil {
			return "", err
		}
	}
	return fmt.Sprintf("%s/config", path), nil
}
<commit_msg>Fixes kubeconfig.go ability to use ssh agent<commit_after>// Copyright © 2017 The Kubicorn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubeconfig

import (
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"syscall"
	"time"

	"github.com/kris-nova/kubicorn/apis/cluster"
	"github.com/kris-nova/kubicorn/cutil/local"
	"github.com/kris-nova/kubicorn/cutil/logger"
	"github.com/pkg/sftp"
	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
	"golang.org/x/crypto/ssh/terminal"
)

func GetConfig(existing *cluster.Cluster) error {
	user := existing.SSH.User
	pubKeyPath := local.Expand(existing.SSH.PublicKeyPath)
	privKeyPath := strings.Replace(pubKeyPath, ".pub", "", 1)
	address := fmt.Sprintf("%s:%s", existing.KubernetesAPI.Endpoint, "22")
	localDir := fmt.Sprintf("%s/.kube", local.Home())
	localPath, err := getKubeConfigPath(localDir)
	if err != nil {
		return err
	}
	sshConfig := &ssh.ClientConfig{
		User:            user,
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
	}
	remotePath := ""
	if user == "root" {
		remotePath = "/root/.kube/config"
	} else {
		remotePath = fmt.Sprintf("/home/%s/.kube/config", user)
	}

	//fmt.Println(pubKeyPath)
	//fmt.Println(privKeyPath)
	//fmt.Println(address)
	//fmt.Println(user)
	//fmt.Println(remotePath)
	//fmt.Println(localPath)

	agent := sshAgent()
	if agent != nil {
		sshConfig.Auth = append(sshConfig.Auth, agent)
	} else {
		pemBytes, err := ioutil.ReadFile(privKeyPath)
		if err != nil {
			return err
		}
		signer, err := getSigner(pemBytes)
		if err != nil {
			return err
		}
		sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signer))
	}

	sshConfig.SetDefaults()
	conn, err := ssh.Dial("tcp", address, sshConfig)
	if err != nil {
		return err
	}
	defer conn.Close()
	c, err := sftp.NewClient(conn)
	if err != nil {
		return err
	}
	defer c.Close()
	r, err := c.Open(remotePath)
	if err != nil {
		return err
	}
	defer r.Close()
	bytes, err := ioutil.ReadAll(r)
	if err != nil {
		return err
	}

	if _, err := os.Stat(localPath); os.IsNotExist(err) {
		empty := []byte("")
		err := ioutil.WriteFile(localPath, empty, 0755)
		if err != nil {
			return err
		}
	}

	f, err := os.OpenFile(localPath, os.O_APPEND|os.O_WRONLY, os.ModeAppend)
	if err != nil {
		return err
	}
	_, err = f.WriteString(string(bytes))
	if err != nil {
		return err
	}
	defer f.Close()
	logger.Always("Wrote kubeconfig to [%s]", localPath)
	return nil
}

const (
	RetryAttempts     = 150
	RetrySleepSeconds = 5
)

func RetryGetConfig(existing *cluster.Cluster) error {
	for i := 0; i <= RetryAttempts; i++ {
		err := GetConfig(existing)
		if err != nil {
			if strings.Contains(err.Error(), "file does not exist") || strings.Contains(err.Error(), "getsockopt: connection refused") || strings.Contains(err.Error(), "unable to authenticate") {
				logger.Debug("Waiting for Kubernetes to come up.. [%v]", err)
				time.Sleep(time.Duration(RetrySleepSeconds) * time.Second)
				continue
			}
			return err
		}
		return nil
	}
	return fmt.Errorf("Timedout writing kubeconfig")
}

func getSigner(pemBytes []byte) (ssh.Signer, error) {
	signerwithoutpassphrase, err := ssh.ParsePrivateKey(pemBytes)
	if err != nil {
		logger.Debug(err.Error())
		fmt.Print("SSH Key Passphrase [none]: ")
		passPhrase, err := terminal.ReadPassword(int(syscall.Stdin))
		fmt.Println("")
		if err != nil {
			return nil, err
		}
		signerwithpassphrase, err := ssh.ParsePrivateKeyWithPassphrase(pemBytes, passPhrase)
		if err != nil {
			return nil, err
		}

		return signerwithpassphrase, err
	}

	return signerwithoutpassphrase, err
}

func sshAgent() ssh.AuthMethod {
	sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
	if err == nil {
		return ssh.PublicKeysCallback(agent.NewClient(sshAgent).Signers)
	}
	return nil
}

func getKubeConfigPath(path string) (string, error) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		if err := os.Mkdir(path, 0777); err != nil {
			return "", err
		}
	}
	return fmt.Sprintf("%s/config", path), nil
}
<|endoftext|>


<commit_before>// +build !nopkcs11

package pkcs11key

import (
	"bytes"
	"crypto"
	"crypto/ecdsa"
	"crypto/rand"
	"crypto/rsa"
	"github.com/miekg/pkcs11"
	"reflect"
	"testing"
)

type mockCtx struct {
	currentSearch []*pkcs11.Attribute
}

const sessionHandle = pkcs11.SessionHandle(17)
const rsaPrivateKeyHandle = pkcs11.ObjectHandle(23)

// Correct EC private key
const ecPrivateKeyHandle = pkcs11.ObjectHandle(32)
const ecPublicKeyHandle = pkcs11.ObjectHandle(33)
const ecCorrectKeyId = byte(0x03)

// EC private key with no matching public key
const ecPrivNoPubHandle = pkcs11.ObjectHandle(34)
const ecPrivNoPubId = byte(0x4)

// EC private and public key with invalid EC point
const ecInvEcPointPrivHandle = pkcs11.ObjectHandle(35)
const ecInvEcPointPubHandle = pkcs11.ObjectHandle(36)
const ecInvEcPointId = byte(0x5)

var slots = []uint{7, 8, 9}
var tokenInfo = pkcs11.TokenInfo{
	Label: "token label",
}

func (c *mockCtx) CloseSession(sh pkcs11.SessionHandle) error {
	return nil
}

func (c *mockCtx) FindObjectsFinal(sh pkcs11.SessionHandle) error {
	c.currentSearch = []*pkcs11.Attribute{}
	return nil
}

func (c *mockCtx) FindObjectsInit(sh pkcs11.SessionHandle, temp []*pkcs11.Attribute) error {
	c.currentSearch = temp
	return nil
}

func (c *mockCtx) FindObjects(sh pkcs11.SessionHandle, max int) ([]pkcs11.ObjectHandle, bool, error) {
	for _, a := range c.currentSearch {
		// We search private keys using CKA_LABEL
		if a.Type == pkcs11.CKA_LABEL {
			switch string(a.Value) {
			case "rsa":
				return []pkcs11.ObjectHandle{rsaPrivateKeyHandle}, true, nil
			case "ec":
				return []pkcs11.ObjectHandle{ecPrivateKeyHandle}, true, nil
			case "no_public_key_ec":
				return []pkcs11.ObjectHandle{ecPrivNoPubHandle}, true, nil
			case "invalid_ec_point":
				return []pkcs11.ObjectHandle{ecInvEcPointPrivHandle}, true, nil
			}
		}
		// We search th EC public key using CKA_ID
		if a.Type == pkcs11.CKA_ID {
			switch a.Value[0] {
			case ecCorrectKeyId:
				return []pkcs11.ObjectHandle{ecPublicKeyHandle}, true, nil
			case ecInvEcPointId:
				return []pkcs11.ObjectHandle{ecInvEcPointPubHandle}, true, nil
			default:
				return []pkcs11.ObjectHandle{}, true, nil
			}
		}
	}
	return nil, false, nil
}

func p11Attribute(Type uint, Value []byte) *pkcs11.Attribute {
	return &pkcs11.Attribute{
		Type:  Type,
		Value: Value,
	}
}

func rsaPrivateAttributes(template []*pkcs11.Attribute) ([]*pkcs11.Attribute, error) {
	var output []*pkcs11.Attribute
	for _, a := range template {
		// Return simple values for these attributes. Note that a value of `1` for
		// both modulus an public exponent would be a very bad public key, but it's
		// sufficient to satisfy the current code.
		if a.Type == pkcs11.CKA_MODULUS ||
			a.Type == pkcs11.CKA_PUBLIC_EXPONENT ||
			a.Type == pkcs11.CKA_ALWAYS_AUTHENTICATE {
			output = append(output, p11Attribute(a.Type, []byte{byte(1)}))
		}
		if a.Type == pkcs11.CKA_KEY_TYPE {
			output = append(output, p11Attribute(a.Type, []byte{byte(pkcs11.CKK_RSA)}))
		}
	}
	return output, nil
}

var ecOid = []byte{0x06, 0x08, 0x2A, 0x86, 0x48, 0xCE, 0x3D, 0x03, 0x01, 0x07}

var ecPoint = []byte{0x04, 0x41, 0x04, 0x4C, 0xD7, 0x7B, 0x7B, 0x2E,
	0x3D, 0x57, 0x98, 0xB8, 0x2F, 0x99, 0xB4, 0x83,
	0x99, 0xE6, 0xD4, 0x4C, 0x4F, 0xBC, 0x2D, 0x60,
	0xCD, 0x08, 0x8E, 0x93, 0x65, 0x6F, 0x20, 0x51,
	0x1C, 0xE7, 0xFD, 0x59, 0x34, 0xAA, 0xA9, 0x36,
	0x26, 0xCE, 0x4A, 0xC5, 0xA2, 0x4A, 0x85, 0x6C,
	0xB3, 0x95, 0xFF, 0x92, 0x0F, 0x56, 0x76, 0x34,
	0x1F, 0x69, 0x52, 0x5F, 0x20, 0x83, 0x13, 0x50,
	0xA3, 0xDE, 0xBE}

func ecCorrectKeyAttributes(template []*pkcs11.Attribute) ([]*pkcs11.Attribute, error) {
	var output []*pkcs11.Attribute
	for _, a := range template {
		switch a.Type {
		case pkcs11.CKA_EC_PARAMS:
			output = append(output, p11Attribute(a.Type, ecOid))
		case pkcs11.CKA_EC_POINT:
			output = append(output, p11Attribute(a.Type, ecPoint))
		case pkcs11.CKA_KEY_TYPE:
			output = append(output, p11Attribute(a.Type, []byte{byte(pkcs11.CKK_EC)}))
		case pkcs11.CKA_ID:
			output = append(output, p11Attribute(a.Type, []byte{byte(ecCorrectKeyId)}))
		}
	}
	return output, nil
}

func ecPrivNoPubAttributes(template []*pkcs11.Attribute) ([]*pkcs11.Attribute, error) {
	var output []*pkcs11.Attribute
	for _, a := range template {
		switch a.Type {
		case pkcs11.CKA_EC_PARAMS:
			output = append(output, p11Attribute(a.Type, ecOid))
		case pkcs11.CKA_KEY_TYPE:
			output = append(output, p11Attribute(a.Type, []byte{byte(pkcs11.CKK_EC)}))
		case pkcs11.CKA_ID:
			output = append(output, p11Attribute(a.Type, []byte{byte(ecPrivNoPubId)}))
		}
	}
	return output, nil
}

func ecInvalidEcPointAttributes(template []*pkcs11.Attribute) ([]*pkcs11.Attribute, error) {
	var output []*pkcs11.Attribute
	for _, a := range template {
		switch a.Type {
		case pkcs11.CKA_EC_PARAMS:
			output = append(output, p11Attribute(a.Type, ecOid))
		case pkcs11.CKA_EC_POINT:
			output = append(output, p11Attribute(a.Type, []byte{byte(0)}))
		case pkcs11.CKA_KEY_TYPE:
			output = append(output, p11Attribute(a.Type, []byte{byte(pkcs11.CKK_EC)}))
		case pkcs11.CKA_ID:
			output = append(output, p11Attribute(a.Type, []byte{byte(ecInvEcPointId)}))
		}
	}
	return output, nil
}

func (c *mockCtx) GetAttributeValue(sh pkcs11.SessionHandle, o pkcs11.ObjectHandle, template []*pkcs11.Attribute) ([]*pkcs11.Attribute, error) {
	if o == rsaPrivateKeyHandle {
		return rsaPrivateAttributes(template)
	} else if o == ecPrivateKeyHandle {
		return ecCorrectKeyAttributes(template)
	} else if o == ecPublicKeyHandle {
		return ecCorrectKeyAttributes(template)
	} else if o == ecPrivNoPubHandle {
		return ecPrivNoPubAttributes(template)
	} else if o == ecInvEcPointPrivHandle {
		return ecInvalidEcPointAttributes(template)
	} else if o == ecInvEcPointPubHandle {
		return ecInvalidEcPointAttributes(template)
	}
	return nil, nil
}

func (c *mockCtx) GetSlotList(tokenPresent bool) ([]uint, error) {
	return slots, nil
}

func (c *mockCtx) GetTokenInfo(slotID uint) (pkcs11.TokenInfo, error) {
	return tokenInfo, nil
}

func (c *mockCtx) Initialize() error {
	return nil
}

func (c *mockCtx) Login(sh pkcs11.SessionHandle, userType uint, pin string) error {
	return nil
}

func (c *mockCtx) Logout(sh pkcs11.SessionHandle) error {
	return nil
}

func (c *mockCtx) OpenSession(slotID uint, flags uint) (pkcs11.SessionHandle, error) {
	return sessionHandle, nil
}

func (c *mockCtx) SignInit(sh pkcs11.SessionHandle, m []*pkcs11.Mechanism, o pkcs11.ObjectHandle) error {
	return nil
}

func (c *mockCtx) Sign(sh pkcs11.SessionHandle, message []byte) ([]byte, error) {
	return message, nil
}

func setup(t *testing.T, label string) *Key {
	ps := Key{
		module:     &mockCtx{},
		tokenLabel: "token label",
		pin:        "unused",
	}
	err := ps.setup(label)
	if err != nil {
		t.Fatalf("Failed to set up Key: %s", err)
	}
	return &ps
}

var signInput = []byte("1234567890 1234567890 1234567890")

func sign(t *testing.T, ps *Key) []byte {
	// Sign input must be exactly 32 bytes to match SHA256 size. In normally
	// usage, Sign would be called by e.g. x509.CreateCertificate, which would
	// handle padding to the necessary size.
	output, err := ps.Sign(rand.Reader, signInput, crypto.SHA256)
	if err != nil {
		t.Fatalf("Failed to sign: %s", err)
	}

	if len(output) < len(signInput) {
		t.Fatalf("Invalid signature size")
	}

	i := len(output) - len(signInput)
	if !bytes.Equal(output[i:], signInput) {
		t.Fatal("Incorrect sign output")
	}
	return output
}

func TestSign(t *testing.T) {
	ps := setup(t, "rsa")
	sig := sign(t, ps)

	// Check that the RSA signature starts with the SHA256 hash prefix
	var sha256Pre = []byte{0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86,
		0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20}
	if !(bytes.Equal(sha256Pre, sig[0:19])) {
		t.Fatal("RSA signature doesn't start with prefix")
	}

	pub := ps.Public()
	// Check public key is of right type
	_, ok := pub.(*rsa.PublicKey)
	if !ok {
		t.Errorf("Attempted to load RSA key from module, got key of type %s. Expected *rsa.PublicKey", reflect.TypeOf(pub))
	}

	ps = setup(t, "ec")
	sig = sign(t, ps)

	if !(bytes.Equal(signInput, sig)) {
		t.Fatal("ECDSA signature error")
	}

	pub = ps.Public()
	// Check public key is of right type
	ecPub, ok := pub.(*ecdsa.PublicKey)
	if !ok {
		t.Errorf("Attempted to load ECDSA key from module, got key of type %s. Expected *ecdsa.PublicKey", reflect.TypeOf(pub))
	}
	if ecPub == nil ||
		!(bytes.Equal(ecPub.X.Bytes(), ecPoint[3:35]) &&
			bytes.Equal(ecPub.Y.Bytes(), ecPoint[35:])) {
		t.Fatal("Incorrect decoding of EC Point")
	}

	// Disable this test because it can only work in go 1.5 and later
	// if strings.Compare(ecPub.Curve.Params().Name, "P-256") != 0 {
	// 	t.Fatal("Invalid curve decoded")
	// }

	k := Key{
		module:     &mockCtx{},
		tokenLabel: "token label",
		pin:        "unused",
	}

	// Trying to load private EC key with no public key
	err := k.setup("no_publick_key_ec")
	if err == nil {
		t.Errorf("Unexpected success")
	}

	// Trying to load private EC key with invalid EC point
	err = k.setup("invalid_ec_point")
	if err == nil {
		t.Errorf("Unexpected success")
	}
}

// This is a version of the mock that gives CKR_ATTRIBUTE_TYPE_INVALID when
// asked about the CKA_ALWAYS_AUTHENTICATE attribute.
type mockCtxFailsAlwaysAuthenticate struct {
	mockCtx
}

func (c *mockCtxFailsAlwaysAuthenticate) GetAttributeValue(sh pkcs11.SessionHandle, o pkcs11.ObjectHandle, template []*pkcs11.Attribute) ([]*pkcs11.Attribute, error) {
	for _, a := range template {
		if a.Type == pkcs11.CKA_ALWAYS_AUTHENTICATE {
			return nil, pkcs11.Error(pkcs11.CKR_ATTRIBUTE_TYPE_INVALID)
		}
	}
	return c.mockCtx.GetAttributeValue(sh, o, template)
}

func TestAttributeTypeInvalid(t *testing.T) {
	ps := &Key{
		module:     &mockCtxFailsAlwaysAuthenticate{},
		tokenLabel: "token label",
		pin:        "unused",
	}
	err := ps.setup("rsa")
	if err != nil {
		t.Errorf("Failed to set up with a token that returns CKR_ATTRIBUTE_TYPE_INVALID: %s", err)
	}
}
<commit_msg>Fix variable names<commit_after>// +build !nopkcs11

package pkcs11key

import (
	"bytes"
	"crypto"
	"crypto/ecdsa"
	"crypto/rand"
	"crypto/rsa"
	"github.com/miekg/pkcs11"
	"reflect"
	"testing"
)

type mockCtx struct {
	currentSearch []*pkcs11.Attribute
}

const sessionHandle = pkcs11.SessionHandle(17)
const rsaPrivateKeyHandle = pkcs11.ObjectHandle(23)

// Correct EC private key
const ecPrivateKeyHandle = pkcs11.ObjectHandle(32)
const ecPublicKeyHandle = pkcs11.ObjectHandle(33)
const ecCorrectKeyID = byte(0x03)

// EC private key with no matching public key
const ecPrivNoPubHandle = pkcs11.ObjectHandle(34)
const ecPrivNoPubID = byte(0x4)

// EC private and public key with invalid EC point
const ecInvEcPointPrivHandle = pkcs11.ObjectHandle(35)
const ecInvEcPointPubHandle = pkcs11.ObjectHandle(36)
const ecInvEcPointID = byte(0x5)

var slots = []uint{7, 8, 9}
var tokenInfo = pkcs11.TokenInfo{
	Label: "token label",
}

func (c *mockCtx) CloseSession(sh pkcs11.SessionHandle) error {
	return nil
}

func (c *mockCtx) FindObjectsFinal(sh pkcs11.SessionHandle) error {
	c.currentSearch = []*pkcs11.Attribute{}
	return nil
}

func (c *mockCtx) FindObjectsInit(sh pkcs11.SessionHandle, temp []*pkcs11.Attribute) error {
	c.currentSearch = temp
	return nil
}

func (c *mockCtx) FindObjects(sh pkcs11.SessionHandle, max int) ([]pkcs11.ObjectHandle, bool, error) {
	for _, a := range c.currentSearch {
		// We search private keys using CKA_LABEL
		if a.Type == pkcs11.CKA_LABEL {
			switch string(a.Value) {
			case "rsa":
				return []pkcs11.ObjectHandle{rsaPrivateKeyHandle}, true, nil
			case "ec":
				return []pkcs11.ObjectHandle{ecPrivateKeyHandle}, true, nil
			case "no_public_key_ec":
				return []pkcs11.ObjectHandle{ecPrivNoPubHandle}, true, nil
			case "invalid_ec_point":
				return []pkcs11.ObjectHandle{ecInvEcPointPrivHandle}, true, nil
			}
		}
		// We search th EC public key using CKA_ID
		if a.Type == pkcs11.CKA_ID {
			switch a.Value[0] {
			case ecCorrectKeyID:
				return []pkcs11.ObjectHandle{ecPublicKeyHandle}, true, nil
			case ecInvEcPointID:
				return []pkcs11.ObjectHandle{ecInvEcPointPubHandle}, true, nil
			default:
				return []pkcs11.ObjectHandle{}, true, nil
			}
		}
	}
	return nil, false, nil
}

func p11Attribute(Type uint, Value []byte) *pkcs11.Attribute {
	return &pkcs11.Attribute{
		Type:  Type,
		Value: Value,
	}
}

func rsaPrivateAttributes(template []*pkcs11.Attribute) ([]*pkcs11.Attribute, error) {
	var output []*pkcs11.Attribute
	for _, a := range template {
		// Return simple values for these attributes. Note that a value of `1` for
		// both modulus an public exponent would be a very bad public key, but it's
		// sufficient to satisfy the current code.
		if a.Type == pkcs11.CKA_MODULUS ||
			a.Type == pkcs11.CKA_PUBLIC_EXPONENT ||
			a.Type == pkcs11.CKA_ALWAYS_AUTHENTICATE {
			output = append(output, p11Attribute(a.Type, []byte{byte(1)}))
		}
		if a.Type == pkcs11.CKA_KEY_TYPE {
			output = append(output, p11Attribute(a.Type, []byte{byte(pkcs11.CKK_RSA)}))
		}
	}
	return output, nil
}

var ecOid = []byte{0x06, 0x08, 0x2A, 0x86, 0x48, 0xCE, 0x3D, 0x03, 0x01, 0x07}

var ecPoint = []byte{0x04, 0x41, 0x04, 0x4C, 0xD7, 0x7B, 0x7B, 0x2E,
	0x3D, 0x57, 0x98, 0xB8, 0x2F, 0x99, 0xB4, 0x83,
	0x99, 0xE6, 0xD4, 0x4C, 0x4F, 0xBC, 0x2D, 0x60,
	0xCD, 0x08, 0x8E, 0x93, 0x65, 0x6F, 0x20, 0x51,
	0x1C, 0xE7, 0xFD, 0x59, 0x34, 0xAA, 0xA9, 0x36,
	0x26, 0xCE, 0x4A, 0xC5, 0xA2, 0x4A, 0x85, 0x6C,
	0xB3, 0x95, 0xFF, 0x92, 0x0F, 0x56, 0x76, 0x34,
	0x1F, 0x69, 0x52, 0x5F, 0x20, 0x83, 0x13, 0x50,
	0xA3, 0xDE, 0xBE}

func ecCorrectKeyAttributes(template []*pkcs11.Attribute) ([]*pkcs11.Attribute, error) {
	var output []*pkcs11.Attribute
	for _, a := range template {
		switch a.Type {
		case pkcs11.CKA_EC_PARAMS:
			output = append(output, p11Attribute(a.Type, ecOid))
		case pkcs11.CKA_EC_POINT:
			output = append(output, p11Attribute(a.Type, ecPoint))
		case pkcs11.CKA_KEY_TYPE:
			output = append(output, p11Attribute(a.Type, []byte{byte(pkcs11.CKK_EC)}))
		case pkcs11.CKA_ID:
			output = append(output, p11Attribute(a.Type, []byte{byte(ecCorrectKeyID)}))
		}
	}
	return output, nil
}

func ecPrivNoPubAttributes(template []*pkcs11.Attribute) ([]*pkcs11.Attribute, error) {
	var output []*pkcs11.Attribute
	for _, a := range template {
		switch a.Type {
		case pkcs11.CKA_EC_PARAMS:
			output = append(output, p11Attribute(a.Type, ecOid))
		case pkcs11.CKA_KEY_TYPE:
			output = append(output, p11Attribute(a.Type, []byte{byte(pkcs11.CKK_EC)}))
		case pkcs11.CKA_ID:
			output = append(output, p11Attribute(a.Type, []byte{byte(ecPrivNoPubID)}))
		}
	}
	return output, nil
}

func ecInvalidEcPointAttributes(template []*pkcs11.Attribute) ([]*pkcs11.Attribute, error) {
	var output []*pkcs11.Attribute
	for _, a := range template {
		switch a.Type {
		case pkcs11.CKA_EC_PARAMS:
			output = append(output, p11Attribute(a.Type, ecOid))
		case pkcs11.CKA_EC_POINT:
			output = append(output, p11Attribute(a.Type, []byte{byte(0)}))
		case pkcs11.CKA_KEY_TYPE:
			output = append(output, p11Attribute(a.Type, []byte{byte(pkcs11.CKK_EC)}))
		case pkcs11.CKA_ID:
			output = append(output, p11Attribute(a.Type, []byte{byte(ecInvEcPointID)}))
		}
	}
	return output, nil
}

func (c *mockCtx) GetAttributeValue(sh pkcs11.SessionHandle, o pkcs11.ObjectHandle, template []*pkcs11.Attribute) ([]*pkcs11.Attribute, error) {
	if o == rsaPrivateKeyHandle {
		return rsaPrivateAttributes(template)
	} else if o == ecPrivateKeyHandle {
		return ecCorrectKeyAttributes(template)
	} else if o == ecPublicKeyHandle {
		return ecCorrectKeyAttributes(template)
	} else if o == ecPrivNoPubHandle {
		return ecPrivNoPubAttributes(template)
	} else if o == ecInvEcPointPrivHandle {
		return ecInvalidEcPointAttributes(template)
	} else if o == ecInvEcPointPubHandle {
		return ecInvalidEcPointAttributes(template)
	}
	return nil, nil
}

func (c *mockCtx) GetSlotList(tokenPresent bool) ([]uint, error) {
	return slots, nil
}

func (c *mockCtx) GetTokenInfo(slotID uint) (pkcs11.TokenInfo, error) {
	return tokenInfo, nil
}

func (c *mockCtx) Initialize() error {
	return nil
}

func (c *mockCtx) Login(sh pkcs11.SessionHandle, userType uint, pin string) error {
	return nil
}

func (c *mockCtx) Logout(sh pkcs11.SessionHandle) error {
	return nil
}

func (c *mockCtx) OpenSession(slotID uint, flags uint) (pkcs11.SessionHandle, error) {
	return sessionHandle, nil
}

func (c *mockCtx) SignInit(sh pkcs11.SessionHandle, m []*pkcs11.Mechanism, o pkcs11.ObjectHandle) error {
	return nil
}

func (c *mockCtx) Sign(sh pkcs11.SessionHandle, message []byte) ([]byte, error) {
	return message, nil
}

func setup(t *testing.T, label string) *Key {
	ps := Key{
		module:     &mockCtx{},
		tokenLabel: "token label",
		pin:        "unused",
	}
	err := ps.setup(label)
	if err != nil {
		t.Fatalf("Failed to set up Key: %s", err)
	}
	return &ps
}

var signInput = []byte("1234567890 1234567890 1234567890")

func sign(t *testing.T, ps *Key) []byte {
	// Sign input must be exactly 32 bytes to match SHA256 size. In normally
	// usage, Sign would be called by e.g. x509.CreateCertificate, which would
	// handle padding to the necessary size.
	output, err := ps.Sign(rand.Reader, signInput, crypto.SHA256)
	if err != nil {
		t.Fatalf("Failed to sign: %s", err)
	}

	if len(output) < len(signInput) {
		t.Fatalf("Invalid signature size")
	}

	i := len(output) - len(signInput)
	if !bytes.Equal(output[i:], signInput) {
		t.Fatal("Incorrect sign output")
	}
	return output
}

func TestSign(t *testing.T) {
	ps := setup(t, "rsa")
	sig := sign(t, ps)

	// Check that the RSA signature starts with the SHA256 hash prefix
	var sha256Pre = []byte{0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86,
		0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20}
	if !(bytes.Equal(sha256Pre, sig[0:19])) {
		t.Fatal("RSA signature doesn't start with prefix")
	}

	pub := ps.Public()
	// Check public key is of right type
	_, ok := pub.(*rsa.PublicKey)
	if !ok {
		t.Errorf("Attempted to load RSA key from module, got key of type %s. Expected *rsa.PublicKey", reflect.TypeOf(pub))
	}

	ps = setup(t, "ec")
	sig = sign(t, ps)

	if !(bytes.Equal(signInput, sig)) {
		t.Fatal("ECDSA signature error")
	}

	pub = ps.Public()
	// Check public key is of right type
	ecPub, ok := pub.(*ecdsa.PublicKey)
	if !ok {
		t.Errorf("Attempted to load ECDSA key from module, got key of type %s. Expected *ecdsa.PublicKey", reflect.TypeOf(pub))
	}
	if ecPub == nil ||
		!(bytes.Equal(ecPub.X.Bytes(), ecPoint[3:35]) &&
			bytes.Equal(ecPub.Y.Bytes(), ecPoint[35:])) {
		t.Fatal("Incorrect decoding of EC Point")
	}

	// Disable this test because it can only work in go 1.5 and later
	// if strings.Compare(ecPub.Curve.Params().Name, "P-256") != 0 {
	// 	t.Fatal("Invalid curve decoded")
	// }

	k := Key{
		module:     &mockCtx{},
		tokenLabel: "token label",
		pin:        "unused",
	}

	// Trying to load private EC key with no public key
	err := k.setup("no_publick_key_ec")
	if err == nil {
		t.Errorf("Unexpected success")
	}

	// Trying to load private EC key with invalid EC point
	err = k.setup("invalid_ec_point")
	if err == nil {
		t.Errorf("Unexpected success")
	}
}

// This is a version of the mock that gives CKR_ATTRIBUTE_TYPE_INVALID when
// asked about the CKA_ALWAYS_AUTHENTICATE attribute.
type mockCtxFailsAlwaysAuthenticate struct {
	mockCtx
}

func (c *mockCtxFailsAlwaysAuthenticate) GetAttributeValue(sh pkcs11.SessionHandle, o pkcs11.ObjectHandle, template []*pkcs11.Attribute) ([]*pkcs11.Attribute, error) {
	for _, a := range template {
		if a.Type == pkcs11.CKA_ALWAYS_AUTHENTICATE {
			return nil, pkcs11.Error(pkcs11.CKR_ATTRIBUTE_TYPE_INVALID)
		}
	}
	return c.mockCtx.GetAttributeValue(sh, o, template)
}

func TestAttributeTypeInvalid(t *testing.T) {
	ps := &Key{
		module:     &mockCtxFailsAlwaysAuthenticate{},
		tokenLabel: "token label",
		pin:        "unused",
	}
	err := ps.setup("rsa")
	if err != nil {
		t.Errorf("Failed to set up with a token that returns CKR_ATTRIBUTE_TYPE_INVALID: %s", err)
	}
}
<|endoftext|>


<commit_before>package agent

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/ActiveState/tail"
	"github.com/tutumcloud/tutum-agent/utils"
)

type TunnelPatchForm struct {
	Tunnel  string `json:"tunnel"`
	Version string `json:"agent_version"`
}

func NatTunnel(url, ngrokPath, ngrokLogPath, ngrokConfPath string) {
	if !utils.FileExist(ngrokPath) {
		Logger.Printf("Cannot find ngrok binary(%s), skipping NAT tunnel", ngrokPath)
		return
	}

	if !isNodeNated() {
		return
	}

	updateNgrokHost(url)
	createNgrokConfFile(ngrokConfPath)

	var cmd *exec.Cmd
	if *FlagNgrokToken != "" {
		Logger.Println("About to tunnel to public ngrok service")
		cmd = exec.Command(ngrokPath,
			"-log", "stdout",
			"-authtoken", *FlagNgrokToken,
			"-proto", "tcp",
			DockerHostPort)
	} else {
		Logger.Println("About to tunnel to private ngrok service")
		if !utils.FileExist(ngrokConfPath) {
			Logger.Println("Cannot find ngrok conf, skipping NAT tunnel")
			return
		}
		cmd = exec.Command(ngrokPath,
			"-config", ngrokConfPath,
			"-log", "stdout",
			"-proto", "tcp",
			DockerHostPort)
	}

	os.RemoveAll(ngrokLogPath)
	logFile, err := os.OpenFile(ngrokLogPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		Logger.Println(err)
	} else {
		defer logFile.Close()
		cmd.Stdout = logFile
	}

	Logger.Println("Starting montoring tunnel:", cmd.Args)
	go monitorTunnels(url, ngrokLogPath)
	Logger.Println("Starting NAT tunnel:", cmd.Args)

	for {
		runGronk(cmd)
		time.Sleep(10 * time.Second)
		Logger.Println("Restarting NAT tunnel:", cmd.Args)
	}
}

func runGronk(cmd *exec.Cmd) bool {
	if err := cmd.Start(); err != nil {
		return true
	}
	cmd.Wait()
	return true
}

func monitorTunnels(url, ngrokLogPath string) {
	update, _ := tail.TailFile(ngrokLogPath, tail.Config{
		Follow: true,
		ReOpen: true})
	for line := range update.Lines {
		if strings.Contains(line.Text, "[INFO] [client] Tunnel established at") {
			terms := strings.Split(line.Text, " ")
			tunnel := terms[len(terms)-1]
			Logger.Printf("Found new tunnel:%s", tunnel)
			patchTunnelToTutum(url, tunnel)
		}
	}
}

func patchTunnelToTutum(url, tunnel string) {
	Logger.Println("Patching tunnel address to Tutum")
	form := TunnelPatchForm{}
	form.Version = VERSION
	form.Tunnel = tunnel
	data, err := json.Marshal(form)
	if err != nil {
		Logger.Printf("Cannot marshal the TunnelPatch form:%s\f", err)
	}

	headers := []string{"Authorization TutumAgentToken " + Conf.TutumToken,
		"Content-Type", "application/json"}
	_, err = SendRequest("PATCH", utils.JoinURL(url, Conf.TutumUUID), data, headers)
	if err != nil {
		Logger.Println("Failed to patch tunnel address to Tutum,", err)
	} else {
		Logger.Println("Successfully Patched tunnel address to Tutum")
	}
}

func DownloadNgrok(url, ngrokBinPath string) {
	if utils.FileExist(ngrokBinPath) {
		Logger.Printf("Found ngrok locally(%s), skip downloading", ngrokBinPath)
	} else {
		Logger.Println("No ngrok binary is found locally. Starting to download ngrok...")
		downloadFile(url, ngrokBinPath, "ngrok")
	}
}

func createNgrokConfFile(ngrokConfPath string) {
	ngrokConfStr := fmt.Sprintf("server_addr: %s\ntrust_host_root_certs: false", NgrokHost)
	Logger.Printf("Creating ngrok config file in %s ...", ngrokConfPath)
	if err := ioutil.WriteFile(ngrokConfPath, []byte(ngrokConfStr), 0666); err != nil {
		Logger.Println("Cannot create ngrok config file:", err)
	}
}

func updateNgrokHost(url string) {
	if NgrokHost != "" {
		return
	}

	headers := []string{"Authorization TutumAgentToken " + Conf.TutumToken,
		"Content-Type application/json"}
	body, err := SendRequest("GET", utils.JoinURL(url, Conf.TutumUUID), nil, headers)
	if err != nil {
		Logger.Printf("Get registration info error, %s", err)
	} else {
		var form RegGetForm
		if err = json.Unmarshal(body, &form); err != nil {
			Logger.Println("Cannot unmarshal the response, ", err)
		} else {
			if form.NgrokHost != "" {
				NgrokHost = form.NgrokHost
				Logger.Println("Set ngrok server address to", NgrokHost)
			}
		}
	}
}

func isNodeNated() bool {
	Logger.Printf("Testing if port %s is publicly reachable ...", DockerHostPort)
	Logger.Println("Waiting for the starup of docker ...")
	for {
		cmdstring := fmt.Sprintf("nc -w 10 127.0.0.1 %s < /dev/null", DockerHostPort)
		command := exec.Command("/bin/sh", "-c", cmdstring)
		command.Start()
		if err := command.Wait(); err == nil {
			Logger.Println("Docker daemon is started, testing if it is behind firewall")
			break
		} else {
			Logger.Println("Docker daemon is not started yet. Retry in 2 seconds")
			time.Sleep(2 * time.Second)
		}
	}

	cmdstring := fmt.Sprintf("nc -w 10 %s %s < /dev/null", Conf.CertCommonName, DockerHostPort)
	command := exec.Command("/bin/sh", "-c", cmdstring)
	command.Start()
	if err := command.Wait(); err != nil {
		Logger.Printf("Port %s is not publicly reachable", DockerHostPort)
		return true
	} else {
		Logger.Printf("Port %s is publicly reachable, skipping NAT tunnle", DockerHostPort)
		return false
	}
}
<commit_msg>minor logging fixes<commit_after>package agent

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/ActiveState/tail"
	"github.com/tutumcloud/tutum-agent/utils"
)

type TunnelPatchForm struct {
	Tunnel  string `json:"tunnel"`
	Version string `json:"agent_version"`
}

func NatTunnel(url, ngrokPath, ngrokLogPath, ngrokConfPath string) {
	if !utils.FileExist(ngrokPath) {
		Logger.Printf("Cannot find ngrok binary(%s), skipping NAT tunnel", ngrokPath)
		return
	}

	if !isNodeNated() {
		return
	}

	updateNgrokHost(url)
	createNgrokConfFile(ngrokConfPath)

	var cmd *exec.Cmd
	if *FlagNgrokToken != "" {
		Logger.Println("About to tunnel to public ngrok service")
		cmd = exec.Command(ngrokPath,
			"-log", "stdout",
			"-authtoken", *FlagNgrokToken,
			"-proto", "tcp",
			DockerHostPort)
	} else {
		Logger.Println("About to tunnel to private ngrok service")
		if !utils.FileExist(ngrokConfPath) {
			Logger.Println("Cannot find ngrok conf, skipping NAT tunnel")
			return
		}
		cmd = exec.Command(ngrokPath,
			"-config", ngrokConfPath,
			"-log", "stdout",
			"-proto", "tcp",
			DockerHostPort)
	}

	os.RemoveAll(ngrokLogPath)
	logFile, err := os.OpenFile(ngrokLogPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		Logger.Println(err)
	} else {
		defer logFile.Close()
		cmd.Stdout = logFile
	}

	Logger.Println("Starting montoring tunnel:", cmd.Args)
	go monitorTunnels(url, ngrokLogPath)
	Logger.Println("Starting NAT tunnel:", cmd.Args)

	for {
		runGronk(cmd)
		time.Sleep(10 * time.Second)
		Logger.Println("Restarting NAT tunnel:", cmd.Args)
	}
}

func runGronk(cmd *exec.Cmd) bool {
	if err := cmd.Start(); err != nil {
		return true
	}
	cmd.Wait()
	return true
}

func monitorTunnels(url, ngrokLogPath string) {
	update, _ := tail.TailFile(ngrokLogPath, tail.Config{
		Follow: true,
		ReOpen: true})
	for line := range update.Lines {
		if strings.Contains(line.Text, "[INFO] [client] Tunnel established at") {
			terms := strings.Split(line.Text, " ")
			tunnel := terms[len(terms)-1]
			Logger.Printf("Found new tunnel:%s", tunnel)
			patchTunnelToTutum(url, tunnel)
		}
	}
}

func patchTunnelToTutum(url, tunnel string) {
	Logger.Println("Patching tunnel address to Tutum")
	form := TunnelPatchForm{}
	form.Version = VERSION
	form.Tunnel = tunnel
	data, err := json.Marshal(form)
	if err != nil {
		Logger.Printf("Cannot marshal the TunnelPatch form:%s\f", err)
	}

	headers := []string{"Authorization TutumAgentToken " + Conf.TutumToken,
		"Content-Type", "application/json"}
	_, err = SendRequest("PATCH", utils.JoinURL(url, Conf.TutumUUID), data, headers)
	if err != nil {
		Logger.Println("Failed to patch tunnel address to Tutum,", err)
	} else {
		Logger.Println("Successfully Patched tunnel address to Tutum")
	}
}

func DownloadNgrok(url, ngrokBinPath string) {
	if utils.FileExist(ngrokBinPath) {
		Logger.Printf("Found ngrok locally(%s), skip downloading", ngrokBinPath)
	} else {
		Logger.Println("No ngrok binary is found locally. Starting to download ngrok...")
		downloadFile(url, ngrokBinPath, "ngrok")
	}
}

func createNgrokConfFile(ngrokConfPath string) {
	ngrokConfStr := fmt.Sprintf("server_addr: %s\ntrust_host_root_certs: false", NgrokHost)
	Logger.Printf("Creating ngrok config file in %s ...", ngrokConfPath)
	if err := ioutil.WriteFile(ngrokConfPath, []byte(ngrokConfStr), 0666); err != nil {
		Logger.Println("Cannot create ngrok config file:", err)
	}
}

func updateNgrokHost(url string) {
	if NgrokHost != "" {
		return
	}

	headers := []string{"Authorization TutumAgentToken " + Conf.TutumToken,
		"Content-Type application/json"}
	body, err := SendRequest("GET", utils.JoinURL(url, Conf.TutumUUID), nil, headers)
	if err != nil {
		Logger.Printf("Get registration info error, %s", err)
	} else {
		var form RegGetForm
		if err = json.Unmarshal(body, &form); err != nil {
			Logger.Println("Cannot unmarshal the response, ", err)
		} else {
			if form.NgrokHost != "" {
				NgrokHost = form.NgrokHost
				Logger.Println("Set ngrok server address to", NgrokHost)
			}
		}
	}
}

func isNodeNated() bool {
	Logger.Printf("Testing if port %s is publicly reachable ...", DockerHostPort)
	Logger.Println("Waiting for the startup of docker ...")
	for {
		cmdstring := fmt.Sprintf("nc -w 10 127.0.0.1 %s < /dev/null", DockerHostPort)
		command := exec.Command("/bin/sh", "-c", cmdstring)
		command.Start()
		if err := command.Wait(); err == nil {
			Logger.Println("Docker daemon has started, testing if it's publicly reachable")
			break
		} else {
			Logger.Println("Docker daemon has not started yet. Retrying in 2 seconds")
			time.Sleep(2 * time.Second)
		}
	}

	cmdstring := fmt.Sprintf("nc -w 10 %s %s < /dev/null", Conf.CertCommonName, DockerHostPort)
	command := exec.Command("/bin/sh", "-c", cmdstring)
	command.Start()
	if err := command.Wait(); err != nil {
		Logger.Printf("Port %s is not publicly reachable", DockerHostPort)
		return true
	} else {
		Logger.Printf("Port %s is publicly reachable, skipping NAT tunnel", DockerHostPort)
		return false
	}
}
<|endoftext|>


<commit_before>package agent

import (
	"net/http"
	"strings"
)

func (s *HTTPServer) AgentServices(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	services := s.agent.state.Services()
	return services, nil
}

func (s *HTTPServer) AgentChecks(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	checks := s.agent.state.Checks()
	return checks, nil
}

func (s *HTTPServer) AgentMembers(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	// Check if the WAN is being queried
	wan := false
	if other := req.URL.Query().Get("wan"); other != "" {
		wan = true
	}
	if wan {
		return s.agent.WANMembers(), nil
	} else {
		return s.agent.LANMembers(), nil
	}
}

func (s *HTTPServer) AgentJoin(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	// Check if the WAN is being queried
	wan := false
	if other := req.URL.Query().Get("wan"); other != "" {
		wan = true
	}

	// Get the address
	addr := strings.TrimPrefix(req.URL.Path, "/v1/agent/join/")
	if wan {
		_, err := s.agent.JoinWAN([]string{addr})
		return err, nil
	} else {
		_, err := s.agent.JoinLAN([]string{addr})
		return err, nil
	}
}

func (s *HTTPServer) AgentForceLeave(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	addr := strings.TrimPrefix(req.URL.Path, "/v1/agent/force-leave/")
	return s.agent.ForceLeave(addr), nil
}

func (s *HTTPServer) AgentRegisterCheck(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	return nil, nil
}

func (s *HTTPServer) AgentDeregisterCheck(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	return nil, nil
}

func (s *HTTPServer) AgentCheckPass(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	return nil, nil
}

func (s *HTTPServer) AgentCheckWarn(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	return nil, nil
}

func (s *HTTPServer) AgentCheckFail(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	return nil, nil
}

func (s *HTTPServer) AgentRegisterService(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	return nil, nil
}

func (s *HTTPServer) AgentDeregisterService(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	return nil, nil
}
<commit_msg>agent: First pass at the easy endpoints<commit_after>package agent

import (
	"github.com/hashicorp/consul/consul/structs"
	"net/http"
	"strings"
)

func (s *HTTPServer) AgentServices(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	services := s.agent.state.Services()
	return services, nil
}

func (s *HTTPServer) AgentChecks(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	checks := s.agent.state.Checks()
	return checks, nil
}

func (s *HTTPServer) AgentMembers(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	// Check if the WAN is being queried
	wan := false
	if other := req.URL.Query().Get("wan"); other != "" {
		wan = true
	}
	if wan {
		return s.agent.WANMembers(), nil
	} else {
		return s.agent.LANMembers(), nil
	}
}

func (s *HTTPServer) AgentJoin(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	// Check if the WAN is being queried
	wan := false
	if other := req.URL.Query().Get("wan"); other != "" {
		wan = true
	}

	// Get the address
	addr := strings.TrimPrefix(req.URL.Path, "/v1/agent/join/")
	if wan {
		_, err := s.agent.JoinWAN([]string{addr})
		return err, nil
	} else {
		_, err := s.agent.JoinLAN([]string{addr})
		return err, nil
	}
}

func (s *HTTPServer) AgentForceLeave(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	addr := strings.TrimPrefix(req.URL.Path, "/v1/agent/force-leave/")
	return s.agent.ForceLeave(addr), nil
}

func (s *HTTPServer) AgentRegisterCheck(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	return nil, nil
}

func (s *HTTPServer) AgentDeregisterCheck(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	checkID := strings.TrimPrefix(req.URL.Path, "/v1/agent/check/deregister/")
	return s.agent.RemoveCheck(checkID), nil
}

func (s *HTTPServer) AgentCheckPass(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	checkID := strings.TrimPrefix(req.URL.Path, "/v1/agent/check/pass/")
	note := req.URL.Query().Get("note")
	return s.agent.UpdateCheck(checkID, structs.HealthPassing, note), nil
}

func (s *HTTPServer) AgentCheckWarn(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	checkID := strings.TrimPrefix(req.URL.Path, "/v1/agent/check/warn/")
	note := req.URL.Query().Get("note")
	return s.agent.UpdateCheck(checkID, structs.HealthWarning, note), nil
}

func (s *HTTPServer) AgentCheckFail(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	checkID := strings.TrimPrefix(req.URL.Path, "/v1/agent/check/fail/")
	note := req.URL.Query().Get("note")
	return s.agent.UpdateCheck(checkID, structs.HealthCritical, note), nil
}

func (s *HTTPServer) AgentRegisterService(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	return nil, nil
}

func (s *HTTPServer) AgentDeregisterService(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
	serviceID := strings.TrimPrefix(req.URL.Path, "/v1/agent/service/deregister/")
	return s.agent.RemoveService(serviceID), nil
}
<|endoftext|>


<commit_before>package test

import (
	"testing"

	"github.com/golang/glog"
	"github.com/kubeflow/pipelines/backend/api/go_http_client/pipeline_model"
	"github.com/kubeflow/pipelines/backend/src/cmd/ml/cmd"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/suite"
	"src/k8s.io/apimachinery/pkg/util/json"
)

const (
	defaultPageSize    = int32(10)
	myUploadedPipeline = "my-uploaded-pipeline"
	// If true, run as a unit test using a fake client.
	// If false, run as an integration test calling the service.
	// This is useful to test locally before running an e2e test (which takes a while).
	runAsUnitTest = false
)

func GetRealRootCommand() (*cmd.RootCommand, cmd.ClientFactoryInterface) {
	if runAsUnitTest {
		return cmd.GetFakeRootCommand()
	} else {
		clientFactory := cmd.NewClientFactoryWithByteBuffer()
		rootCmd := cmd.NewRootCmd(clientFactory)
		rootCmd = cmd.CreateSubCommands(rootCmd, defaultPageSize)
		return rootCmd, clientFactory
	}
}

type CLIIntegrationTest struct {
	suite.Suite
	namespace string
}

// Check the cluster namespace has Kubeflow pipelines installed and ready.
func (c *CLIIntegrationTest) SetupTest() {
	c.namespace = *namespace

	if !runAsUnitTest {
		// Wait for the system to be ready.
		err := waitForReady(c.namespace, *initializeTimeout)
		if err != nil {
			glog.Exitf("Cluster namespace '%s' is still not ready after timeout. Error: %s", c.namespace,
				err.Error())
		}
	}
}

func (c *CLIIntegrationTest) TearDownTest() {
	err := c.deletePipelines()
	if err != nil {
		glog.Exitf("Failed to delete the pipelines: %v", err)
	}
}

func (c *CLIIntegrationTest) TestPipelineListSuccess() {
	t := c.T()
	rootCmd, _ := GetRealRootCommand()
	args := []string{"pipeline", "list"}
	args = addCommonArgs(args, c.namespace)
	rootCmd.Command().SetArgs(args)
	_, err := rootCmd.Command().ExecuteC()
	assert.Nil(t, err)
}

func (c *CLIIntegrationTest) TestPipelineListFailureInvalidArgument() {
	t := c.T()
	rootCmd, _ := GetRealRootCommand()
	args := []string{"pipeline", "list", "askjdfskldjf"}
	args = addCommonArgs(args, c.namespace)
	rootCmd.Command().SetArgs(args)
	_, err := rootCmd.Command().ExecuteC()
	assert.NotNil(t, err)
}

func (c *CLIIntegrationTest) TestPipelineUploadSuccess() {
	t := c.T()
	rootCmd, _ := GetRealRootCommand()
	args := []string{"pipeline", "upload", "--name", myUploadedPipeline,
		"./resources/hello-world.yaml"}
	args = addCommonArgs(args, c.namespace)
	rootCmd.Command().SetArgs(args)
	_, err := rootCmd.Command().ExecuteC()
	assert.Nil(t, err)
}

func (c *CLIIntegrationTest) TestPipelineCreateGetDeleteSuccess() {
	t := c.T()
	rootCmd, factory := GetRealRootCommand()

	// Create pipeline
	args := []string{"pipeline", "create",
		"https://storage.googleapis.com/ml-pipeline-dataset/sequential.yaml"}
	args = addCommonArgs(args, c.namespace)
	rootCmd.Command().SetArgs(args)
	_, err := rootCmd.Command().ExecuteC()
	assert.Nil(t, err)

	// Get ID
	pipelineID, err := toPipelineID(factory.Result())
	assert.Nil(t, err)

	// Get pipeline
	args = []string{"pipeline", "get", pipelineID}
	args = addCommonArgs(args, c.namespace)
	rootCmd.Command().SetArgs(args)
	_, err = rootCmd.Command().ExecuteC()
	assert.Nil(t, err)

	// Get manifest
	args = []string{"pipeline", "get-manifest", pipelineID}
	args = addCommonArgs(args, c.namespace)
	rootCmd.Command().SetArgs(args)
	_, err = rootCmd.Command().ExecuteC()
	assert.Nil(t, err)

	// Delete pipeline
	args = []string{"pipeline", "delete", pipelineID}
	args = addCommonArgs(args, c.namespace)
	rootCmd.Command().SetArgs(args)
	_, err = rootCmd.Command().ExecuteC()
	assert.Nil(t, err)
}

func TestPipelineAPI(t *testing.T) {
	suite.Run(t, new(CLIIntegrationTest))
}

func addCommonArgs(args []string, namespace string) []string {
	args = append(args, "--debug", "--namespace", namespace, "-o", "json")
	return args
}

func toPipelineID(jsonPipeline string) (string, error) {
	var pipeline pipeline_model.APIPipeline
	err := json.Unmarshal([]byte(jsonPipeline), &pipeline)
	if err != nil {
		return "", err
	}
	return pipeline.ID, nil
}

func toPipelines(jsonPipelines string) ([]pipeline_model.APIPipeline, error) {
	var pipelines []pipeline_model.APIPipeline
	err := json.Unmarshal([]byte(jsonPipelines), &pipelines)
	if err != nil {
		return nil, err
	}
	return pipelines, nil
}

func (c *CLIIntegrationTest) deletePipelines() error {

	pipelines, err := c.listPipelines()
	if err != nil {
		return err
	}
	for _, pipeline := range pipelines {
		err = c.deletePipeline(pipeline)
		if err != nil {
			return err
		}
	}
	return nil
}

func (c *CLIIntegrationTest) deletePipeline(pipeline pipeline_model.APIPipeline) error {
	rootCmd, _ := GetRealRootCommand()
	args := []string{"pipeline", "delete", pipeline.ID}
	args = addCommonArgs(args, c.namespace)
	rootCmd.Command().SetArgs(args)
	_, err := rootCmd.Command().ExecuteC()
	return err
}

func (c *CLIIntegrationTest) listPipelines() ([]pipeline_model.APIPipeline, error) {
	rootCmd, factory := GetRealRootCommand()
	args := []string{"pipeline", "list"}
	args = addCommonArgs(args, c.namespace)
	rootCmd.Command().SetArgs(args)
	_, err := rootCmd.Command().ExecuteC()
	if err != nil {
		return nil, err
	}
	return toPipelines(factory.Result())
}
<commit_msg>Fix typo<commit_after>package test

import (
	"testing"

	"github.com/golang/glog"
	"github.com/kubeflow/pipelines/backend/api/go_http_client/pipeline_model"
	"github.com/kubeflow/pipelines/backend/src/cmd/ml/cmd"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/suite"
	"src/k8s.io/apimachinery/pkg/util/json"
)

const (
	defaultPageSize    = int32(10)
	myUploadedPipeline = "my-uploaded-pipeline"
	// If true, run as a unit test using a fake client.
	// If false, run as an integration test calling the service.
	// This is useful to test locally before running an e2e test (which takes a while).
	runAsUnitTest = false
)

func GetRealRootCommand() (*cmd.RootCommand, cmd.ClientFactoryInterface) {
	if runAsUnitTest {
		return cmd.GetFakeRootCommand()
	} else {
		clientFactory := cmd.NewClientFactoryWithByteBuffer()
		rootCmd := cmd.NewRootCmd(clientFactory)
		rootCmd = cmd.CreateSubCommands(rootCmd, defaultPageSize)
		return rootCmd, clientFactory
	}
}

type CLIIntegrationTest struct {
	suite.Suite
	namespace string
}

// Check the cluster namespace has Kubeflow pipelines installed and ready.
func (c *CLIIntegrationTest) SetupTest() {
	c.namespace = *namespace

	if !runAsUnitTest {
		// Wait for the system to be ready.
		err := waitForReady(c.namespace, *initializeTimeout)
		if err != nil {
			glog.Exitf("Cluster namespace '%s' is still not ready after timeout. Error: %s", c.namespace,
				err.Error())
		}
	}
}

func (c *CLIIntegrationTest) TearDownTest() {
	err := c.deletePipelines()
	if err != nil {
		glog.Exitf("Failed to delete pipelines: %v", err)
	}
}

func (c *CLIIntegrationTest) TestPipelineListSuccess() {
	t := c.T()
	rootCmd, _ := GetRealRootCommand()
	args := []string{"pipeline", "list"}
	args = addCommonArgs(args, c.namespace)
	rootCmd.Command().SetArgs(args)
	_, err := rootCmd.Command().ExecuteC()
	assert.Nil(t, err)
}

func (c *CLIIntegrationTest) TestPipelineListFailureInvalidArgument() {
	t := c.T()
	rootCmd, _ := GetRealRootCommand()
	args := []string{"pipeline", "list", "askjdfskldjf"}
	args = addCommonArgs(args, c.namespace)
	rootCmd.Command().SetArgs(args)
	_, err := rootCmd.Command().ExecuteC()
	assert.NotNil(t, err)
}

func (c *CLIIntegrationTest) TestPipelineUploadSuccess() {
	t := c.T()
	rootCmd, _ := GetRealRootCommand()
	args := []string{"pipeline", "upload", "--name", myUploadedPipeline,
		"./resources/hello-world.yaml"}
	args = addCommonArgs(args, c.namespace)
	rootCmd.Command().SetArgs(args)
	_, err := rootCmd.Command().ExecuteC()
	assert.Nil(t, err)
}

func (c *CLIIntegrationTest) TestPipelineCreateGetDeleteSuccess() {
	t := c.T()
	rootCmd, factory := GetRealRootCommand()

	// Create pipeline
	args := []string{"pipeline", "create",
		"https://storage.googleapis.com/ml-pipeline-dataset/sequential.yaml"}
	args = addCommonArgs(args, c.namespace)
	rootCmd.Command().SetArgs(args)
	_, err := rootCmd.Command().ExecuteC()
	assert.Nil(t, err)

	// Get ID
	pipelineID, err := toPipelineID(factory.Result())
	assert.Nil(t, err)

	// Get pipeline
	args = []string{"pipeline", "get", pipelineID}
	args = addCommonArgs(args, c.namespace)
	rootCmd.Command().SetArgs(args)
	_, err = rootCmd.Command().ExecuteC()
	assert.Nil(t, err)

	// Get manifest
	args = []string{"pipeline", "get-manifest", pipelineID}
	args = addCommonArgs(args, c.namespace)
	rootCmd.Command().SetArgs(args)
	_, err = rootCmd.Command().ExecuteC()
	assert.Nil(t, err)

	// Delete pipeline
	args = []string{"pipeline", "delete", pipelineID}
	args = addCommonArgs(args, c.namespace)
	rootCmd.Command().SetArgs(args)
	_, err = rootCmd.Command().ExecuteC()
	assert.Nil(t, err)
}

func TestPipelineAPI(t *testing.T) {
	suite.Run(t, new(CLIIntegrationTest))
}

func addCommonArgs(args []string, namespace string) []string {
	args = append(args, "--debug", "--namespace", namespace, "-o", "json")
	return args
}

func toPipelineID(jsonPipeline string) (string, error) {
	var pipeline pipeline_model.APIPipeline
	err := json.Unmarshal([]byte(jsonPipeline), &pipeline)
	if err != nil {
		return "", err
	}
	return pipeline.ID, nil
}

func toPipelines(jsonPipelines string) ([]pipeline_model.APIPipeline, error) {
	var pipelines []pipeline_model.APIPipeline
	err := json.Unmarshal([]byte(jsonPipelines), &pipelines)
	if err != nil {
		return nil, err
	}
	return pipelines, nil
}

func (c *CLIIntegrationTest) deletePipelines() error {

	pipelines, err := c.listPipelines()
	if err != nil {
		return err
	}
	for _, pipeline := range pipelines {
		err = c.deletePipeline(pipeline)
		if err != nil {
			return err
		}
	}
	return nil
}

func (c *CLIIntegrationTest) deletePipeline(pipeline pipeline_model.APIPipeline) error {
	rootCmd, _ := GetRealRootCommand()
	args := []string{"pipeline", "delete", pipeline.ID}
	args = addCommonArgs(args, c.namespace)
	rootCmd.Command().SetArgs(args)
	_, err := rootCmd.Command().ExecuteC()
	return err
}

func (c *CLIIntegrationTest) listPipelines() ([]pipeline_model.APIPipeline, error) {
	rootCmd, factory := GetRealRootCommand()
	args := []string{"pipeline", "list"}
	args = addCommonArgs(args, c.namespace)
	rootCmd.Command().SetArgs(args)
	_, err := rootCmd.Command().ExecuteC()
	if err != nil {
		return nil, err
	}
	return toPipelines(factory.Result())
}
<|endoftext|>


<commit_before>package trousseau

import (
	"os"
	"path/filepath"
)

// Global data store file path
var gStorePath string
func SetStorePath(storePath string) { gStorePath = storePath }
func GetStorePath() string          { return gStorePath }

// Gnupg trousseau master gpg key id
var gMasterGpgId string = os.Getenv(ENV_MASTER_GPG_ID_KEY)
var gPasshphrase string

// Ssh default identity file path
var gPrivateRsaKeyPath string = filepath.Join(os.Getenv("HOME"), ".ssh", "id_rsa")

// Keyring manager service and username to use in order to
// retrieve trousseau main gpg key passphrase from system
// keyring
var gKeyringService string = os.Getenv(ENV_KEYRING_SERVICE_KEY)
var gKeyringUser string = os.Getenv(ENV_KEYRING_USER_KEY)
<commit_msg>Remove gPassphrase global<commit_after>package trousseau

import (
	"os"
	"path/filepath"
)

// Global data store file path
var gStorePath string
func SetStorePath(storePath string) { gStorePath = storePath }
func GetStorePath() string          { return gStorePath }

// Gnupg trousseau master gpg key id
var gMasterGpgId string = os.Getenv(ENV_MASTER_GPG_ID_KEY)

// Ssh default identity file path
var gPrivateRsaKeyPath string = filepath.Join(os.Getenv("HOME"), ".ssh", "id_rsa")

// Keyring manager service and username to use in order to
// retrieve trousseau main gpg key passphrase from system
// keyring
var gKeyringService string = os.Getenv(ENV_KEYRING_SERVICE_KEY)
var gKeyringUser string = os.Getenv(ENV_KEYRING_USER_KEY)
<|endoftext|>


<commit_before>package vantage

import (
	"fmt"
	"io"
	"log"
	//"sort"
	"time"
)

const PAGE_COUNT = 513
const PAGE_SIZE = 267
const RECORDS_PER_PAGE = 5
const DATA_RECORD_LENGTH = 52

// Rev B
type ArchiveRecord struct {
	ArchivePage       int
	ArchivePageRecord int
	ArchiveTime       time.Time
	OutsideTemp       float32
	HighOutsideTemp   float32
	LowOutsideTemp    float32
	Rainfall          int
	HighRainRate      int
	Barometer         float32
	SolarRad          int
	WindSamples       int
	InsideTemp        float32
	InsideHumidity    int
	OutsideHumidity   int
	WindAvg           int
	WindMax           int
	WindMaxDir        int
	WindDir           int
	UVIndexAvg        float32
	ET                float32
	HighSolarRad      int
	UVIndexMax        int
	ForecastRule      int
	LeafTemp          []int //2
	LeafWetness       []int //2
	SoilTemp          []int //4
	RecordType        int
	ExtraHumidities   []int //2
	ExtraTemps        []int //3
	SoilMoistures     []int //4
}

type sortedArchive []*ArchiveRecord

func (sa sortedArchive) Len() int           { return len(sa) }
func (sa sortedArchive) Swap(i, j int)      { sa[i], sa[j] = sa[j], sa[i] }
func (sa sortedArchive) Less(i, j int) bool { return sa[i].ArchiveTime.Before(sa[j].ArchiveTime) }

func (vc *Conn) GetArchiveRecords() ([]*ArchiveRecord, error) {
	ars := make(sortedArchive, 0, PAGE_COUNT*RECORDS_PER_PAGE)
	archiveChan := make(chan *ArchiveRecord, 10)
	errChan := make(chan error, 1)

	err := vc.GetArchiveStream(archiveChan, errChan)
	if err != nil {
		return nil, err
	}
	for {
		select {
		case ar := <-archiveChan:
			if ar == nil {
				// Channel closed
				//sort.Sort(ars)
				return ars, nil
			}
			ars = append(ars, ar)
		case err = <-errChan:
			return ars, err
		}
	}
}

func (vc *Conn) GetArchiveStream(archiveChan chan *ArchiveRecord, errChan chan error) error {
	err := vc.sendAckCommand("DMP\n")
	if err != nil {
		return fmt.Errorf("DMP command failed: %v", err)
	}
	go vc.dmpArchive(archiveChan, errChan)
	return nil
}

func (vc *Conn) dmpArchive(archiveChan chan *ArchiveRecord, errChan chan error) {
	pkt := make([]byte, PAGE_SIZE)
	_, err := vc.conn.Write([]byte{ACK})
	if err != nil {
		errChan <- fmt.Errorf("Error first DMP ACK: %v\n", err)
		return
	}
	/*
		j := 0
		for {
			vc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
			n, err := vc.conn.Read(pkt[j:])
			if err != nil {
				errChan <- fmt.Errorf("Err: %v\nDMP data: %v\n", err, pkt[:j+n])
				return
			}
			j += n
		}
	*/
	for i := 0; i < PAGE_COUNT; i++ {
		vc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
		c, err := io.ReadFull(vc.buf, pkt)
		if err != nil {
			if c > 0 {
				log.Printf("Got bytes: %v", pkt[:c])
			}
			errChan <- fmt.Errorf("Error during DMP read: %v\n", err)
			return
		}
		ars, err := parseArchive(pkt)
		if err != nil {
			//TODO
		}
		for _, ar := range ars {
			archiveChan <- ar
		}
		var toSend byte = ACK
		if i > 5 {
			toSend = ESC
		}
		_, err = vc.conn.Write([]byte{toSend})
		if err != nil {
			errChan <- fmt.Errorf("Error during DMP ACK: %v\n", err)
			return
		}
		if toSend == ESC {
			break
		}
	}
	close(archiveChan)
}

func parseArchive(pkt []byte) ([]*ArchiveRecord, error) {
	ret := make([]*ArchiveRecord, 0, 5)
	for i := 0; i < 5; i++ {
		dr := pkt[i*DATA_RECORD_LENGTH+1 : (i+1)*DATA_RECORD_LENGTH]
		tm := parseArchiveTime(toInt(dr[0], dr[1]), toInt(dr[2], dr[3]))
		if tm == (time.Time{}) {
			continue
		}
		// TODO CRC
		ar := &ArchiveRecord{
			ArchivePage:       int(pkt[0]),
			ArchivePageRecord: i,
			ArchiveTime:       tm,
			OutsideTemp:       float32(toInt(dr[4], dr[5])) / 10,
			HighOutsideTemp:   float32(toInt(dr[6], dr[7])) / 10,
			LowOutsideTemp:    float32(toInt(dr[8], dr[9])) / 10,
			Rainfall:          toInt(dr[10], dr[11]),
			HighRainRate:      toInt(dr[12], dr[13]),
			Barometer:         float32(toInt(dr[14], dr[15])) / 1000,
			SolarRad:          toInt(dr[16], dr[17]),
			WindSamples:       toInt(dr[18], dr[19]),
			InsideTemp:        float32(toInt(dr[20], dr[21])) / 10,
			InsideHumidity:    int(dr[22]),
			OutsideHumidity:   int(dr[23]),
			WindAvg:           int(dr[24]),
			WindMax:           int(dr[25]),
			WindMaxDir:        archiveDirectionLookup[int(26)],
			WindDir:           archiveDirectionLookup[int(27)],
			UVIndexAvg:        float32(int(dr[28])) / 10,
			ET:                float32(int(dr[29])) / 1000,
			HighSolarRad:      toInt(dr[30], dr[31]),
			UVIndexMax:        int(dr[32]),
			ForecastRule:      int(dr[33]),
			LeafTemp:          nil,
			LeafWetness:       nil,
			SoilTemp:          nil,
			RecordType:        int(dr[42]),
			ExtraHumidities:   nil,
			ExtraTemps:        nil,
			SoilMoistures:     nil,
		}
		ret = append(ret, ar)
	}
	return ret, nil
}

var archiveDirectionLookup map[int]int = map[int]int{
	0:   0,   // N
	1:   22,  // NNE
	2:   45,  // NE
	3:   67,  // ENE
	4:   90,  // E
	5:   112, // ESE
	6:   135, // SE
	7:   157, // SSE
	8:   180, // S
	9:   202, // SSW
	10:  225, // SW
	11:  247, // WSW
	12:  270, // W
	13:  292, // WNW
	14:  315, // NW
	15:  337, // NNW
	255: 0,
}

func parseArchiveTime(dt, tm int) time.Time {
	if dt == 0 {
		return time.Time{}
	}
	day := dt & 0x1f                     // lower 5 bits
	month := time.Month((dt >> 5) & 0xF) // 4 bits
	year := (dt >> 9) + 2000             // 7 bits
	hour := tm / 100
	min := tm - hour

	return time.Date(year, month, day, hour, min, 0, 0, time.Local)
}
<commit_msg>more parsing debug<commit_after>package vantage

import (
	"fmt"
	"io"
	"log"
	//"sort"
	"time"
)

const PAGE_COUNT = 513
const PAGE_SIZE = 267
const RECORDS_PER_PAGE = 5
const DATA_RECORD_LENGTH = 52

// Rev B
type ArchiveRecord struct {
	ArchivePage       int
	ArchivePageRecord int
	ArchiveTime       time.Time
	OutsideTemp       float32
	HighOutsideTemp   float32
	LowOutsideTemp    float32
	Rainfall          int
	HighRainRate      int
	Barometer         float32
	SolarRad          int
	WindSamples       int
	InsideTemp        float32
	InsideHumidity    int
	OutsideHumidity   int
	WindAvg           int
	WindMax           int
	WindMaxDir        int
	WindDir           int
	UVIndexAvg        float32
	ET                float32
	HighSolarRad      int
	UVIndexMax        int
	ForecastRule      int
	LeafTemp          []int //2
	LeafWetness       []int //2
	SoilTemp          []int //4
	RecordType        int
	ExtraHumidities   []int //2
	ExtraTemps        []int //3
	SoilMoistures     []int //4
}

type sortedArchive []*ArchiveRecord

func (sa sortedArchive) Len() int           { return len(sa) }
func (sa sortedArchive) Swap(i, j int)      { sa[i], sa[j] = sa[j], sa[i] }
func (sa sortedArchive) Less(i, j int) bool { return sa[i].ArchiveTime.Before(sa[j].ArchiveTime) }

func (vc *Conn) GetArchiveRecords() ([]*ArchiveRecord, error) {
	ars := make(sortedArchive, 0, PAGE_COUNT*RECORDS_PER_PAGE)
	archiveChan := make(chan *ArchiveRecord, 10)
	errChan := make(chan error, 1)

	err := vc.GetArchiveStream(archiveChan, errChan)
	if err != nil {
		return nil, err
	}
	for {
		select {
		case ar := <-archiveChan:
			if ar == nil {
				// Channel closed
				//sort.Sort(ars)
				return ars, nil
			}
			ars = append(ars, ar)
		case err = <-errChan:
			return ars, err
		}
	}
}

func (vc *Conn) GetArchiveStream(archiveChan chan *ArchiveRecord, errChan chan error) error {
	err := vc.sendAckCommand("DMP\n")
	if err != nil {
		return fmt.Errorf("DMP command failed: %v", err)
	}
	go vc.dmpArchive(archiveChan, errChan)
	return nil
}

func (vc *Conn) dmpArchive(archiveChan chan *ArchiveRecord, errChan chan error) {
	_, err := vc.conn.Write([]byte{ACK})
	if err != nil {
		errChan <- fmt.Errorf("Error first DMP ACK: %v\n", err)
		return
	}
	/*
		j := 0
		for {
			vc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
			n, err := vc.conn.Read(pkt[j:])
			if err != nil {
				errChan <- fmt.Errorf("Err: %v\nDMP data: %v\n", err, pkt[:j+n])
				return
			}
			j += n
		}
	*/
	for i := 0; i < PAGE_COUNT; i++ {
		pkt := make([]byte, PAGE_SIZE)
		vc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
		c, err := io.ReadFull(vc.buf, pkt)
		if err != nil {
			if c > 0 {
				log.Printf("Got bytes: %v", pkt[:c])
			}
			errChan <- fmt.Errorf("Error during DMP read: %v\n", err)
			return
		}
		ars, err := parseArchive(pkt)
		if err != nil {
			//TODO
		}
		for _, ar := range ars {
			archiveChan <- ar
		}
		var toSend byte = ACK
		if i > 5 {
			toSend = ESC
		}
		_, err = vc.conn.Write([]byte{toSend})
		if err != nil {
			errChan <- fmt.Errorf("Error during DMP ACK: %v\n", err)
			return
		}
		if toSend == ESC {
			break
		}
	}
	close(archiveChan)
}

func parseArchive(pkt []byte) ([]*ArchiveRecord, error) {
	ret := make([]*ArchiveRecord, 0, 5)
	for i := 0; i < 5; i++ {
		dr := pkt[i*DATA_RECORD_LENGTH+1 : (i+1)*DATA_RECORD_LENGTH]
		tm := parseArchiveTime(toInt(dr[0], dr[1]), toInt(dr[2], dr[3]))
		if tm == (time.Time{}) {
			continue
		}
		// TODO CRC
		ar := &ArchiveRecord{
			ArchivePage:       int(pkt[0]),
			ArchivePageRecord: i,
			ArchiveTime:       tm,
			OutsideTemp:       float32(toInt(dr[4], dr[5])) / 10,
			HighOutsideTemp:   float32(toInt(dr[6], dr[7])) / 10,
			LowOutsideTemp:    float32(toInt(dr[8], dr[9])) / 10,
			Rainfall:          toInt(dr[10], dr[11]),
			HighRainRate:      toInt(dr[12], dr[13]),
			Barometer:         float32(toInt(dr[14], dr[15])) / 1000,
			SolarRad:          toInt(dr[16], dr[17]),
			WindSamples:       toInt(dr[18], dr[19]),
			InsideTemp:        float32(toInt(dr[20], dr[21])) / 10,
			InsideHumidity:    int(dr[22]),
			OutsideHumidity:   int(dr[23]),
			WindAvg:           int(dr[24]),
			WindMax:           int(dr[25]),
			WindMaxDir:        archiveDirectionLookup[int(26)],
			WindDir:           archiveDirectionLookup[int(27)],
			UVIndexAvg:        float32(int(dr[28])) / 10,
			ET:                float32(int(dr[29])) / 1000,
			HighSolarRad:      toInt(dr[30], dr[31]),
			UVIndexMax:        int(dr[32]),
			ForecastRule:      int(dr[33]),
			LeafTemp:          nil,
			LeafWetness:       nil,
			SoilTemp:          nil,
			RecordType:        int(dr[42]),
			ExtraHumidities:   nil,
			ExtraTemps:        nil,
			SoilMoistures:     nil,
		}
		ret = append(ret, ar)
	}
	return ret, nil
}

var archiveDirectionLookup map[int]int = map[int]int{
	0:   0,   // N
	1:   22,  // NNE
	2:   45,  // NE
	3:   67,  // ENE
	4:   90,  // E
	5:   112, // ESE
	6:   135, // SE
	7:   157, // SSE
	8:   180, // S
	9:   202, // SSW
	10:  225, // SW
	11:  247, // WSW
	12:  270, // W
	13:  292, // WNW
	14:  315, // NW
	15:  337, // NNW
	255: 0,
}

func parseArchiveTime(dt, tm int) time.Time {
	if dt == 0 {
		return time.Time{}
	}
	day := dt & 0x1f                     // lower 5 bits
	month := time.Month((dt >> 5) & 0xF) // 4 bits
	year := (dt >> 9) + 2000             // 7 bits
	hour := tm / 100
	min := tm - hour

	return time.Date(year, month, day, hour, min, 0, 0, time.Local)
}
<|endoftext|>