<commit_before>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package alicloud

import (
	"strconv"

	"github.com/GoogleCloudPlatform/terraformer/terraform_utils"
	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
	"github.com/aliyun/alibaba-cloud-sdk-go/services/pvtz"
	"github.com/terraform-providers/terraform-provider-alicloud/alicloud/connectivity"
)

// PvtzGenerator Struct for generating AliCloud private zone
type PvtzGenerator struct {
	AliCloudService
}

func resourceFromZoneResponse(zone pvtz.Zone) terraform_utils.Resource {
	return terraform_utils.NewResource(
		zone.ZoneId,                    // id
		zone.ZoneId+"__"+zone.ZoneName, // name
		"alicloud_pvtz_zone",
		"alicloud",
		map[string]string{},
		[]string{},
		map[string]interface{}{},
	)
}

func resourceFromZoneRecordResponse(record pvtz.Record, ZoneID string) terraform_utils.Resource {
	return terraform_utils.NewResource(
		strconv.Itoa(record.RecordId)+":"+ZoneID,     // id
		strconv.Itoa(record.RecordId)+"__"+record.Rr, // name
		"alicloud_pvtz_zone_record",
		"alicloud",
		map[string]string{},
		[]string{},
		map[string]interface{}{},
	)
}

func initZones(client *connectivity.AliyunClient) ([]pvtz.Zone, error) {
	remaining := 1
	pageNumber := 1
	pageSize := 10

	allZones := make([]pvtz.Zone, 1)

	for remaining > 0 {
		raw, err := client.WithPvtzClient(func(pvtzClient *pvtz.Client) (interface{}, error) {
			request := pvtz.CreateDescribeZonesRequest()
			request.RegionId = client.RegionId
			request.PageSize = requests.NewInteger(pageSize)
			request.PageNumber = requests.NewInteger(pageNumber)
			return pvtzClient.DescribeZones(request)
		})
		if err != nil {
			return nil, err
		}

		response := raw.(*pvtz.DescribeZonesResponse)
		for _, Zone := range response.Zones.Zone {
			allZones = append(allZones, Zone)
		}
		remaining = response.TotalItems - pageNumber*pageSize
		pageNumber++
	}
	return allZones, nil
}

func initZoneRecords(client *connectivity.AliyunClient, allZones []pvtz.Zone) ([]pvtz.Record, []string, error) {
	allZoneRecords := make([]pvtz.Record, 1)
	zoneIds := make([]string, 1)

	for _, zone := range allZones {
		remaining := 1
		pageNumber := 1
		pageSize := 10
		if zone.ZoneId == "" {
			continue
		}
		for remaining > 0 {
			raw, err := client.WithPvtzClient(func(pvtzClient *pvtz.Client) (interface{}, error) {
				request := pvtz.CreateDescribeZoneRecordsRequest()
				request.RegionId = client.RegionId
				request.ZoneId = zone.ZoneId
				request.PageSize = requests.NewInteger(pageSize)
				request.PageNumber = requests.NewInteger(pageNumber)
				return pvtzClient.DescribeZoneRecords(request)
			})
			if err != nil {
				return nil, nil, err
			}

			response := raw.(*pvtz.DescribeZoneRecordsResponse)
			for _, zoneRecord := range response.Records.Record {
				allZoneRecords = append(allZoneRecords, zoneRecord)
				zoneIds = append(zoneIds, zone.ZoneId)
			}
			remaining = response.TotalItems - pageNumber*pageSize
			pageNumber++
		}
	}
	return allZoneRecords, zoneIds, nil
}

// InitResources Gets the list of all pvtz Zone ids and generates resources
func (g *PvtzGenerator) InitResources() error {
	client, err := LoadClientFromProfile()
	if err != nil {
		return err
	}

	allZones, err := initZones(client)
	if err != nil {
		return err
	}

	allRecords, zoneIds, err := initZoneRecords(client, allZones)
	if err != nil {
		return err
	}

	for _, Zone := range allZones {
		resource := resourceFromZoneResponse(Zone)
		g.Resources = append(g.Resources, resource)
	}

	for i, record := range allRecords {
		resource := resourceFromZoneRecordResponse(record, zoneIds[i])
		g.Resources = append(g.Resources, resource)
	}

	return nil
}
<commit_msg>Added support for alicloud zone attachment<commit_after>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package alicloud

import (
	"strconv"

	"github.com/GoogleCloudPlatform/terraformer/terraform_utils"
	"github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
	"github.com/aliyun/alibaba-cloud-sdk-go/services/pvtz"
	"github.com/terraform-providers/terraform-provider-alicloud/alicloud/connectivity"
)

// PvtzGenerator Struct for generating AliCloud private zone
type PvtzGenerator struct {
	AliCloudService
}

func resourceFromZoneResponse(zone pvtz.Zone) terraform_utils.Resource {
	return terraform_utils.NewResource(
		zone.ZoneId,                    // id
		zone.ZoneId+"__"+zone.ZoneName, // name
		"alicloud_pvtz_zone",
		"alicloud",
		map[string]string{},
		[]string{},
		map[string]interface{}{},
	)
}

func resourceFromZoneAttachmentResponse(zone pvtz.Zone) terraform_utils.Resource {
	return terraform_utils.NewResource(
		zone.ZoneId, // id
		zone.ZoneId+"__"+zone.ZoneName+"_attachment", // name
		"alicloud_pvtz_zone_attachment",
		"alicloud",
		map[string]string{},
		[]string{},
		map[string]interface{}{},
	)
}

func resourceFromZoneRecordResponse(record pvtz.Record, ZoneID string) terraform_utils.Resource {
	return terraform_utils.NewResource(
		strconv.Itoa(record.RecordId)+":"+ZoneID,     // id
		strconv.Itoa(record.RecordId)+"__"+record.Rr, // name
		"alicloud_pvtz_zone_record",
		"alicloud",
		map[string]string{},
		[]string{},
		map[string]interface{}{},
	)
}

func initZones(client *connectivity.AliyunClient) ([]pvtz.Zone, error) {
	remaining := 1
	pageNumber := 1
	pageSize := 10

	allZones := make([]pvtz.Zone, 1)

	for remaining > 0 {
		raw, err := client.WithPvtzClient(func(pvtzClient *pvtz.Client) (interface{}, error) {
			request := pvtz.CreateDescribeZonesRequest()
			request.RegionId = client.RegionId
			request.PageSize = requests.NewInteger(pageSize)
			request.PageNumber = requests.NewInteger(pageNumber)
			return pvtzClient.DescribeZones(request)
		})
		if err != nil {
			return nil, err
		}

		response := raw.(*pvtz.DescribeZonesResponse)
		for _, Zone := range response.Zones.Zone {
			allZones = append(allZones, Zone)
		}
		remaining = response.TotalItems - pageNumber*pageSize
		pageNumber++
	}
	return allZones, nil
}

func initZoneRecords(client *connectivity.AliyunClient, allZones []pvtz.Zone) ([]pvtz.Record, []string, error) {
	allZoneRecords := make([]pvtz.Record, 1)
	zoneIds := make([]string, 1)

	for _, zone := range allZones {
		remaining := 1
		pageNumber := 1
		pageSize := 10
		if zone.ZoneId == "" {
			continue
		}
		for remaining > 0 {
			raw, err := client.WithPvtzClient(func(pvtzClient *pvtz.Client) (interface{}, error) {
				request := pvtz.CreateDescribeZoneRecordsRequest()
				request.RegionId = client.RegionId
				request.ZoneId = zone.ZoneId
				request.PageSize = requests.NewInteger(pageSize)
				request.PageNumber = requests.NewInteger(pageNumber)
				return pvtzClient.DescribeZoneRecords(request)
			})
			if err != nil {
				return nil, nil, err
			}

			response := raw.(*pvtz.DescribeZoneRecordsResponse)
			for _, zoneRecord := range response.Records.Record {
				allZoneRecords = append(allZoneRecords, zoneRecord)
				zoneIds = append(zoneIds, zone.ZoneId)
			}
			remaining = response.TotalItems - pageNumber*pageSize
			pageNumber++
		}
	}
	return allZoneRecords, zoneIds, nil
}

// InitResources Gets the list of all pvtz Zone ids and generates resources
func (g *PvtzGenerator) InitResources() error {
	client, err := LoadClientFromProfile()
	if err != nil {
		return err
	}

	allZones, err := initZones(client)
	if err != nil {
		return err
	}

	allRecords, zoneIds, err := initZoneRecords(client, allZones)
	if err != nil {
		return err
	}

	for _, zone := range allZones {
		resource := resourceFromZoneResponse(zone)
		g.Resources = append(g.Resources, resource)
	}

	for _, zone := range allZones {
		resource := resourceFromZoneAttachmentResponse(zone)
		g.Resources = append(g.Resources, resource)
	}

	for i, record := range allRecords {
		resource := resourceFromZoneRecordResponse(record, zoneIds[i])
		g.Resources = append(g.Resources, resource)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Pagoda Box Inc
//
// This Source Code Form is subject to the terms of the Mozilla Public License, v.
// 2.0. If a copy of the MPL was not distributed with this file, You can obtain one
// at http://mozilla.org/MPL/2.0/.
//

//
package print

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/howeyc/gopass"
	"github.com/mitchellh/colorstring"

	"github.com/nanobox-io/nanobox-cli/config"
)

// Verbose only prints a message if the 'verbose' flag is passed
func Verbose(msg string) {
	if config.Verbose {
		fmt.Printf(msg)
	}
}

// Silent prints output unless in silent mode
func Silent(msg string) {
	if !config.Silent {
		fmt.Printf(msg)
	}
}

// Color wraps a print message in 'colorstring' and passes it to fmt.Println
func Color(msg string, v ...interface{}) {
	fmt.Println(colorstring.Color(fmt.Sprintf(msg, v...)))
}

// Prompt will prompt for input from the shell and return a trimmed response
func Prompt(p string, v ...interface{}) string {
	reader := bufio.NewReader(os.Stdin)

	//
	fmt.Print(colorstring.Color(fmt.Sprintf(p, v...)))

	input, err := reader.ReadString('\n')
	if err != nil {
		config.Fatal("[utils/ui] reader.ReadString() failed", err.Error())
	}

	return strings.TrimSpace(input)
}

// Password prompts for a password but keeps the typed response hidden
func Password(p string) string {
	fmt.Printf(p)
	return string(gopass.GetPasswd())
}
<commit_msg>Silent -> Silence in print package to better indicate what it does<commit_after>// Copyright (c) 2015 Pagoda Box Inc
//
// This Source Code Form is subject to the terms of the Mozilla Public License, v.
// 2.0. If a copy of the MPL was not distributed with this file, You can obtain one
// at http://mozilla.org/MPL/2.0/.
//

//
package print

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/howeyc/gopass"
	"github.com/mitchellh/colorstring"

	"github.com/nanobox-io/nanobox-cli/config"
)

// Verbose only prints a message if the 'verbose' flag is passed
func Verbose(msg string) {
	if config.Verbose {
		fmt.Printf(msg)
	}
}

// Silence prints output unless in silent mode
func Silence(msg string) {
	if !config.Silent {
		fmt.Printf(msg)
	}
}

// Color wraps a print message in 'colorstring' and passes it to fmt.Println
func Color(msg string, v ...interface{}) {
	fmt.Println(colorstring.Color(fmt.Sprintf(msg, v...)))
}

// Prompt will prompt for input from the shell and return a trimmed response
func Prompt(p string, v ...interface{}) string {
	reader := bufio.NewReader(os.Stdin)

	//
	fmt.Print(colorstring.Color(fmt.Sprintf(p, v...)))

	input, err := reader.ReadString('\n')
	if err != nil {
		config.Fatal("[utils/ui] reader.ReadString() failed", err.Error())
	}

	return strings.TrimSpace(input)
}

// Password prompts for a password but keeps the typed response hidden
func Password(p string) string {
	fmt.Printf(p)
	return string(gopass.GetPasswd())
}
<|endoftext|>


<commit_before>package report

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"net/url"
	"sync"

	"crypto/tls"
	"github.com/valyala/fasthttp"
)

type valueKind byte

const (
	invalidValueKind valueKind = iota
	float64ValueKind
	int64ValueKind
	boolValueKind
)

// Tag represents an InfluxDB tag.
// Users should prefer to keep the strings as long-lived items.
type Tag struct {
	Key, Value string
}

func (t *Tag) Serialize(w io.Writer) {
	fmt.Fprintf(w, "%s=%s", t.Key, t.Value)
}

// Field represents an InfluxDB field.
// Users should prefer to keep the strings as long-lived items.
//
// Implementor's note: this type uses more memory than is ideal, but it avoids
// unsafe and it avoids reflection.
type Field struct {
	key          string
	int64Value   int64
	float64Value float64
	boolValue    bool
	mode         valueKind
}

func (f *Field) SetFloat64(key string, x float64) {
	if f.mode != invalidValueKind {
		panic("logic error: Field already has a value")
	}
	f.key = key
	f.float64Value = x
	f.mode = float64ValueKind
}
func (f *Field) SetInt64(key string, x int64) {
	if f.mode != invalidValueKind {
		panic("logic error: Field already has a value")
	}
	f.key = key
	f.int64Value = x
	f.mode = int64ValueKind
}
func (f *Field) SetBool(key string, x bool) {
	if f.mode != invalidValueKind {
		panic("logic error: Field already has a value")
	}
	f.key = key
	f.boolValue = x
	f.mode = boolValueKind
}

func (f *Field) Serialize(w io.Writer) {
	if f.mode == int64ValueKind {
		fmt.Fprintf(w, "%s=%di", f.key, f.int64Value)
	} else if f.mode == float64ValueKind {
		fmt.Fprintf(w, "%s=%f", f.key, f.float64Value)
	} else {
		fmt.Fprintf(w, "%s=%v", f.key, f.boolValue)
	}
}

// Point wraps an InfluxDB point data.
// Its primary purpose is to be serialized out to a []byte.
// Users should prefer to keep the strings as long-lived items.
type Point struct {
	Measurement   string
	Tags          []Tag
	Fields        []Field
	TimestampNano int64
}

func (p *Point) Init(m string, ts int64) {
	p.Measurement = m
	p.TimestampNano = ts
}

func (p *Point) AddTag(k, v string) {
	p.Tags = append(p.Tags, Tag{Key: k, Value: v})
}

func (p *Point) AddInt64Field(k string, x int64) {
	f := Field{}
	f.SetInt64(k, x)
	p.Fields = append(p.Fields, f)
}

func (p *Point) AddIntField(k string, x int) {
	f := Field{}
	f.SetInt64(k, int64(x))
	p.Fields = append(p.Fields, f)
}

func (p *Point) AddBoolField(k string, x bool) {
	f := Field{}
	f.SetBool(k, x)
	p.Fields = append(p.Fields, f)
}

func (p *Point) AddFloat64Field(k string, x float64) {
	f := Field{}
	f.SetFloat64(k, x)
	p.Fields = append(p.Fields, f)
}

func (p *Point) Serialize(w io.Writer) {
	fmt.Fprintf(w, "%s", p.Measurement)
	for i, tag := range p.Tags {
		if i == 0 {
			fmt.Fprint(w, ",")
		}

		tag.Serialize(w)
		if i < len(p.Tags)-1 {
			fmt.Fprint(w, ",")
		}
	}
	for i, field := range p.Fields {
		if i == 0 {
			fmt.Fprint(w, " ")
		}

		field.Serialize(w)
		if i < len(p.Fields)-1 {
			fmt.Fprint(w, ",")
		}
	}
	if p.TimestampNano > 0 {
		fmt.Fprintf(w, " %d", p.TimestampNano)
	}
}

func (p *Point) Reset() {
	p.Measurement = ""
	p.Tags = p.Tags[:0]
	p.Fields = p.Fields[:0]
	p.TimestampNano = 0
}

var GlobalPointPool *sync.Pool = &sync.Pool{New: func() interface{} { return &Point{} }}

func GetPointFromGlobalPool() *Point {
	return GlobalPointPool.Get().(*Point)
}
func PutPointIntoGlobalPool(p *Point) {
	p.Reset()
	GlobalPointPool.Put(p)
}

type Collector struct {
	Points []*Point

	client           *fasthttp.Client
	writeUri         string
	baseUri          string
	encodedBasicAuth string
	dbName           string

	buf *bytes.Buffer
}

func NewCollector(influxhost, dbname, userName, password string) *Collector {
	encodedBasicAuth := ""
	if userName != "" {
		encodedBasicAuth = "Basic " + base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", userName, password)))
	}
	return &Collector{
		buf:    new(bytes.Buffer),
		Points: make([]*Point, 0, 0),
		client: &fasthttp.Client{
			Name: "collector",
			TLSConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
		baseUri:          influxhost,
		writeUri:         influxhost + "/write?db=" + url.QueryEscape(dbname),
		encodedBasicAuth: encodedBasicAuth,
		dbName:           dbname,
	}
}

func (c *Collector) Put(p *Point) {
	c.Points = append(c.Points, p)
}

func (c *Collector) Reset() {
	c.Points = c.Points[:0]
	c.buf.Reset()
}

func (c *Collector) PrepBatch() {
	for _, p := range c.Points {
		p.Serialize(c.buf)
		fmt.Fprint(c.buf, "\n")
	}
}

func (c *Collector) CreateDatabase() error {
	req := fasthttp.AcquireRequest()
	req.Header.SetMethod("POST")
	req.Header.SetRequestURI(c.baseUri + "/query?q=create%20database%20" + url.QueryEscape(c.dbName))
	if c.encodedBasicAuth != "" {
		req.Header.Set("Authorization", c.encodedBasicAuth)
	}
	req.SetBody(c.buf.Bytes())

	// Perform the request while tracking latency:
	resp := fasthttp.AcquireResponse()
	err := c.client.Do(req, resp)

	if resp.StatusCode() != fasthttp.StatusOK {
		return fmt.Errorf("collector error: unexpected status code %d", resp.StatusCode())
	}

	fasthttp.ReleaseResponse(resp)
	fasthttp.ReleaseRequest(req)

	return err
}

func (c *Collector) SendBatch() error {
	req := fasthttp.AcquireRequest()
	req.Header.SetMethod("POST")
	req.Header.SetRequestURI(c.writeUri)
	if c.encodedBasicAuth != "" {
		req.Header.Set("Authorization", c.encodedBasicAuth)
	}
	req.SetBody(c.buf.Bytes())

	// Perform the request while tracking latency:
	resp := fasthttp.AcquireResponse()
	err := c.client.Do(req, resp)

	if resp.StatusCode() != fasthttp.StatusNoContent {
		return fmt.Errorf("collector error: unexpected status code %d: %s", resp.StatusCode(), resp.Body())
	}

	fasthttp.ReleaseResponse(resp)
	fasthttp.ReleaseRequest(req)

	return err
}
<commit_msg>Expecting also rare 200 HTTP code in respose for write<commit_after>package report

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"net/url"
	"sync"

	"crypto/tls"
	"github.com/valyala/fasthttp"
)

type valueKind byte

const (
	invalidValueKind valueKind = iota
	float64ValueKind
	int64ValueKind
	boolValueKind
)

// Tag represents an InfluxDB tag.
// Users should prefer to keep the strings as long-lived items.
type Tag struct {
	Key, Value string
}

func (t *Tag) Serialize(w io.Writer) {
	fmt.Fprintf(w, "%s=%s", t.Key, t.Value)
}

// Field represents an InfluxDB field.
// Users should prefer to keep the strings as long-lived items.
//
// Implementor's note: this type uses more memory than is ideal, but it avoids
// unsafe and it avoids reflection.
type Field struct {
	key          string
	int64Value   int64
	float64Value float64
	boolValue    bool
	mode         valueKind
}

func (f *Field) SetFloat64(key string, x float64) {
	if f.mode != invalidValueKind {
		panic("logic error: Field already has a value")
	}
	f.key = key
	f.float64Value = x
	f.mode = float64ValueKind
}
func (f *Field) SetInt64(key string, x int64) {
	if f.mode != invalidValueKind {
		panic("logic error: Field already has a value")
	}
	f.key = key
	f.int64Value = x
	f.mode = int64ValueKind
}
func (f *Field) SetBool(key string, x bool) {
	if f.mode != invalidValueKind {
		panic("logic error: Field already has a value")
	}
	f.key = key
	f.boolValue = x
	f.mode = boolValueKind
}

func (f *Field) Serialize(w io.Writer) {
	if f.mode == int64ValueKind {
		fmt.Fprintf(w, "%s=%di", f.key, f.int64Value)
	} else if f.mode == float64ValueKind {
		fmt.Fprintf(w, "%s=%f", f.key, f.float64Value)
	} else {
		fmt.Fprintf(w, "%s=%v", f.key, f.boolValue)
	}
}

// Point wraps an InfluxDB point data.
// Its primary purpose is to be serialized out to a []byte.
// Users should prefer to keep the strings as long-lived items.
type Point struct {
	Measurement   string
	Tags          []Tag
	Fields        []Field
	TimestampNano int64
}

func (p *Point) Init(m string, ts int64) {
	p.Measurement = m
	p.TimestampNano = ts
}

func (p *Point) AddTag(k, v string) {
	p.Tags = append(p.Tags, Tag{Key: k, Value: v})
}

func (p *Point) AddInt64Field(k string, x int64) {
	f := Field{}
	f.SetInt64(k, x)
	p.Fields = append(p.Fields, f)
}

func (p *Point) AddIntField(k string, x int) {
	f := Field{}
	f.SetInt64(k, int64(x))
	p.Fields = append(p.Fields, f)
}

func (p *Point) AddBoolField(k string, x bool) {
	f := Field{}
	f.SetBool(k, x)
	p.Fields = append(p.Fields, f)
}

func (p *Point) AddFloat64Field(k string, x float64) {
	f := Field{}
	f.SetFloat64(k, x)
	p.Fields = append(p.Fields, f)
}

func (p *Point) Serialize(w io.Writer) {
	fmt.Fprintf(w, "%s", p.Measurement)
	for i, tag := range p.Tags {
		if i == 0 {
			fmt.Fprint(w, ",")
		}

		tag.Serialize(w)
		if i < len(p.Tags)-1 {
			fmt.Fprint(w, ",")
		}
	}
	for i, field := range p.Fields {
		if i == 0 {
			fmt.Fprint(w, " ")
		}

		field.Serialize(w)
		if i < len(p.Fields)-1 {
			fmt.Fprint(w, ",")
		}
	}
	if p.TimestampNano > 0 {
		fmt.Fprintf(w, " %d", p.TimestampNano)
	}
}

func (p *Point) Reset() {
	p.Measurement = ""
	p.Tags = p.Tags[:0]
	p.Fields = p.Fields[:0]
	p.TimestampNano = 0
}

var GlobalPointPool *sync.Pool = &sync.Pool{New: func() interface{} { return &Point{} }}

func GetPointFromGlobalPool() *Point {
	return GlobalPointPool.Get().(*Point)
}
func PutPointIntoGlobalPool(p *Point) {
	p.Reset()
	GlobalPointPool.Put(p)
}

type Collector struct {
	Points []*Point

	client           *fasthttp.Client
	writeUri         string
	baseUri          string
	encodedBasicAuth string
	dbName           string

	buf *bytes.Buffer
}

func NewCollector(influxhost, dbname, userName, password string) *Collector {
	encodedBasicAuth := ""
	if userName != "" {
		encodedBasicAuth = "Basic " + base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", userName, password)))
	}
	return &Collector{
		buf:    new(bytes.Buffer),
		Points: make([]*Point, 0, 0),
		client: &fasthttp.Client{
			Name: "collector",
			TLSConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
		baseUri:          influxhost,
		writeUri:         influxhost + "/write?db=" + url.QueryEscape(dbname),
		encodedBasicAuth: encodedBasicAuth,
		dbName:           dbname,
	}
}

func (c *Collector) Put(p *Point) {
	c.Points = append(c.Points, p)
}

func (c *Collector) Reset() {
	c.Points = c.Points[:0]
	c.buf.Reset()
}

func (c *Collector) PrepBatch() {
	for _, p := range c.Points {
		p.Serialize(c.buf)
		fmt.Fprint(c.buf, "\n")
	}
}

func (c *Collector) CreateDatabase() error {
	req := fasthttp.AcquireRequest()
	req.Header.SetMethod("POST")
	req.Header.SetRequestURI(c.baseUri + "/query?q=create%20database%20" + url.QueryEscape(c.dbName))
	if c.encodedBasicAuth != "" {
		req.Header.Set("Authorization", c.encodedBasicAuth)
	}
	req.SetBody(c.buf.Bytes())

	// Perform the request while tracking latency:
	resp := fasthttp.AcquireResponse()
	err := c.client.Do(req, resp)

	if resp.StatusCode() != fasthttp.StatusOK {
		return fmt.Errorf("collector error: unexpected status code %d", resp.StatusCode())
	}

	fasthttp.ReleaseResponse(resp)
	fasthttp.ReleaseRequest(req)

	return err
}

func (c *Collector) SendBatch() error {
	req := fasthttp.AcquireRequest()
	req.Header.SetMethod("POST")
	req.Header.SetRequestURI(c.writeUri)
	if c.encodedBasicAuth != "" {
		req.Header.Set("Authorization", c.encodedBasicAuth)
	}
	req.SetBody(c.buf.Bytes())

	// Perform the request while tracking latency:
	resp := fasthttp.AcquireResponse()
	err := c.client.Do(req, resp)

	if resp.StatusCode() != fasthttp.StatusNoContent && resp.StatusCode() != fasthttp.StatusOK {
		return fmt.Errorf("collector error: unexpected status code %d: %s", resp.StatusCode(), resp.Body())
	}

	fasthttp.ReleaseResponse(resp)
	fasthttp.ReleaseRequest(req)

	return err
}
<|endoftext|>


<commit_before>package nats

import (
	"testing"
	"fmt"
//	"os"
	"os/exec"
	"net"
	"time"
	"math"
	"runtime"
	"regexp"
	"bytes"
	"strings"
)

const natsServer = "nats-server"

type server struct {
	args []string
	cmd  *exec.Cmd
}

var s *server

func startNatsServer(t *testing.T, port uint, other string) *server {
	var s server
	args := fmt.Sprintf("-p %d %s", port, other)
	s.args = strings.Split(args, " ")
	s.cmd = exec.Command(natsServer, s.args...)
	err := s.cmd.Start()
	if err != nil {
		s.cmd = nil
		t.Errorf("Could not start %s, is NATS installed and in path?", natsServer)
		return &s
	}
	// Give it time to start up
	start := time.Now()
	for {
		addr := fmt.Sprintf("localhost:%d", port)
		c, err := net.Dial("tcp", addr)
		if err != nil {
			time.Sleep(50 * time.Millisecond)
			if time.Since(start) > (5 * time.Second) {
				t.Fatalf("Timed out trying to connect to %s", natsServer)
				return nil
			}
		} else {
			c.Close()
			break
		}
	}
	return &s
}

func (s *server) stopServer() {
	if s.cmd != nil && s.cmd.Process != nil {
		s.cmd.Process.Kill()
	}
}

func newConnection(t *testing.T) Connection {
	nc, err := DefaultConnection()
	if err != nil {
		t.Fatal("Failed to create default connection", err)
		return nil
	}
	return nc
}

func TestDefaultConnection(t *testing.T) {
	s = startNatsServer(t, DefaultPort, "")
	nc := newConnection(t)
	nc.Close()
}

func TestClose(t *testing.T) {
	base := runtime.NumGoroutine()
	nc := newConnection(t)
	time.Sleep(10 * time.Millisecond)
	nc.Close()
	time.Sleep(10 * time.Millisecond)
	delta := (runtime.NumGoroutine() - base)
	if delta > 0 {
		t.Fatalf("%d Go routines still exists post Close()", delta)
	}
	// Make sure we can call Close() multiple times
	nc.Close()
}

func TestSimplePublish(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	err := nc.Publish("foo", []byte("Hello World"))
	if err != nil {
		t.Fatal("Failed to publish string message: ", err)
	}
}

func TestAsyncSubscribe(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	omsg := []byte("Hello World")
	received := 0
	_, err := nc.Subscribe("foo", func(subj, _ string, data []byte, sub *Subscription) {
		received += 1
		if !bytes.Equal(data, omsg) {
			t.Fatal("Message received does not match")
		}
		if (sub == nil) {
			t.Fatal("Callback does not have a valid Subsription")
		}
	})
	if err != nil {
		t.Fatal("Failed to subscribe: ", err)
	}
	nc.Publish("foo", omsg)
	nc.Flush()
	if (received != 1) {
		t.Fatal("Message not received for subscription")
	}
}

func TestSyncSubscribe(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	s, err := nc.SubscribeSync("foo")
	if err != nil {
		t.Fatal("Failed to subscribe: ", err)
	}
	omsg := []byte("Hello World")
	nc.Publish("foo", omsg)
	msg, err := s.NextMsg(10 * time.Second)
	if err != nil || !bytes.Equal(msg.Data, omsg) {
		t.Fatal("Message received does not match")
	}
}

func TestPubSubWithReply(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	s, err := nc.SubscribeSync("foo")
	if err != nil {
		t.Fatal("Failed to subscribe: ", err)
	}
	omsg := []byte("Hello World")
	nc.PublishMsg(&Msg{Subject:"foo", Reply:"bar", Data:omsg})
	msg, err := s.NextMsg(10 * time.Second)
	if err != nil || !bytes.Equal(msg.Data, omsg) {
		t.Fatal("Message received does not match")
	}
}

func TestFlush(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	omsg := []byte("Hello World")
	received := 0
	nc.Subscribe("flush", func(_, _ string, _ []byte, _ *Subscription) {
		received += 1
	})
	total := 10000
	for i := 0; i < total; i++ {
		nc.Publish("flush", omsg)
	}
	nc.Flush()
	if (received != total) {
		t.Fatalf("All messages not received: %d != %d\n", received, total)
	}
}

func TestQueueSubscriber(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	r1 := 0
	r2 := 0
	nc.QueueSubscribe("foo", "bar", func(_, _ string, _ []byte, _ *Subscription) {
		r1 += 1
	})
	nc.QueueSubscribe("foo", "bar", func(_, _ string, _ []byte, _ *Subscription) {
		r2 += 1
	})
	omsg := []byte("Hello World")
	nc.Publish("foo", omsg)
	nc.Flush()
	if (r1 + r2) != 1 {
		t.Fatal("Received too many messages for multiple queue subscribers")
	}
	r1, r2 = 0, 0
	total := 100
	for i := 0; i < total; i++ {
		nc.Publish("foo", omsg)
	}
	nc.Flush()
	v := uint(float32(total) * 0.15)
	if r1 + r2 != total {
		t.Fatalf("Incorrect number of messages: %d vs %d", (r1 + r2), total)
	}
	expected := total / 2
	d1 := uint(math.Abs(float64(expected - r1)))
	d2 := uint(math.Abs(float64(expected - r2)))
	if (d1 > v || d2 > v) {
		t.Fatalf("Too much variance in totals: %d, %d > %d", d1, d2, v)
	}
}

func TestReplyArg(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	replyExpected := "bar"
	cbReceived := false
	nc.Subscribe("foo", func(_, reply string, data []byte, sub *Subscription) {
		cbReceived = true
		if (reply != replyExpected) {
			t.Fatalf("Did not receive correct reply arg in callback: " +
				     "('%s' vs '%s')", reply, replyExpected)
		}
	})
	nc.PublishMsg(&Msg{Subject:"foo", Reply:replyExpected, Data:[]byte("Hello")})
	nc.Flush()
	if !cbReceived {
		t.Fatal("Did not receive callback")
	}
}

func TestSyncReplyArg(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	replyExpected := "bar"
	s, _ := nc.SubscribeSync("foo")
	nc.PublishMsg(&Msg{Subject:"foo", Reply:replyExpected, Data:[]byte("Hello")})
	nc.Flush()
	msg, err := s.NextMsg(1 * time.Second)
	if err != nil {
		t.Fatal("Received and err on NextMsg()")
	}
	if (msg.Reply != replyExpected) {
		t.Fatalf("Did not receive correct reply arg in callback: " +
			     "('%s' vs '%s')", msg.Reply, replyExpected)
	}
}

func TestUnsubscribe(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	received := 0
	max := 10
	nc.Subscribe("foo", func(_, _ string, _ []byte, sub *Subscription) {
		received += 1
		if received == max {
			err := sub.Unsubscribe()
			if err != nil {
				t.Fatal("Unsubscribe failed with err:", err)
			}
		}
	})
	send := 20
	for i := 0; i < send; i++ {
		nc.Publish("foo", []byte("hello"))
	}
	nc.Flush()
	if received != max {
		t.Fatalf("Received wrong # of messages after unsubscribe: %d vs %d",
			     received, max)
	}
}

func TestRequestTimeout(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	_, err := nc.Request("foo", []byte("help"), 10*time.Millisecond)
	if err == nil {
		t.Fatalf("Expected to receive a timeout error")
	}
}

func TestRequest(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	response := []byte("I will help you")
	nc.Subscribe("foo", func(_, reply string, _ []byte, sub *Subscription) {
		nc.Publish(reply, response)
	})
	msg, err := nc.Request("foo", []byte("help"), 50*time.Millisecond)
	if err != nil {
		t.Fatalf("Received an error on Request test: %s", err)
	}
	if !bytes.Equal(msg.Data, response) {
		t.Fatalf("Received invalid response")
	}
}

func TestFlushInCB(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	nc.Subscribe("foo", func(_, _ string, _ []byte, _ *Subscription) {
		nc.Flush()
	})
	nc.Publish("foo", []byte("Hello"))
}

func TestReleaseFlush(t *testing.T) {
	nc := newConnection(t)
	for i := 0; i < 1000; i++ {
		nc.Publish("foo", []byte("Hello"))
	}
	go nc.Close()
	nc.Flush()
}

func TestInbox(t *testing.T) {
	inbox := NewInbox()
	matched, _ := regexp.Match(`_INBOX.\S`, []byte(inbox))
	if !matched {
		t.Fatal("Bad INBOX format")
	}
}
<commit_msg>Added empty msg publish test<commit_after>package nats

import (
	"testing"
	"fmt"
//	"os"
	"os/exec"
	"net"
	"time"
	"math"
	"runtime"
	"regexp"
	"bytes"
	"strings"
)

const natsServer = "nats-server"

type server struct {
	args []string
	cmd  *exec.Cmd
}

var s *server

func startNatsServer(t *testing.T, port uint, other string) *server {
	var s server
	args := fmt.Sprintf("-p %d %s", port, other)
	s.args = strings.Split(args, " ")
	s.cmd = exec.Command(natsServer, s.args...)
	err := s.cmd.Start()
	if err != nil {
		s.cmd = nil
		t.Errorf("Could not start %s, is NATS installed and in path?", natsServer)
		return &s
	}
	// Give it time to start up
	start := time.Now()
	for {
		addr := fmt.Sprintf("localhost:%d", port)
		c, err := net.Dial("tcp", addr)
		if err != nil {
			time.Sleep(50 * time.Millisecond)
			if time.Since(start) > (5 * time.Second) {
				t.Fatalf("Timed out trying to connect to %s", natsServer)
				return nil
			}
		} else {
			c.Close()
			break
		}
	}
	return &s
}

func (s *server) stopServer() {
	if s.cmd != nil && s.cmd.Process != nil {
		s.cmd.Process.Kill()
	}
}

func newConnection(t *testing.T) Connection {
	nc, err := DefaultConnection()
	if err != nil {
		t.Fatal("Failed to create default connection", err)
		return nil
	}
	return nc
}

func TestDefaultConnection(t *testing.T) {
	s = startNatsServer(t, DefaultPort, "")
	nc := newConnection(t)
	nc.Close()
}

func TestClose(t *testing.T) {
	base := runtime.NumGoroutine()
	nc := newConnection(t)
	time.Sleep(10 * time.Millisecond)
	nc.Close()
	time.Sleep(10 * time.Millisecond)
	delta := (runtime.NumGoroutine() - base)
	if delta > 0 {
		t.Fatalf("%d Go routines still exists post Close()", delta)
	}
	// Make sure we can call Close() multiple times
	nc.Close()
}

func TestSimplePublish(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	err := nc.Publish("foo", []byte("Hello World"))
	if err != nil {
		t.Fatal("Failed to publish string message: ", err)
	}
}

func TestSimplePublishNoData(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	err := nc.Publish("foo", nil)
	if err != nil {
		t.Fatal("Failed to publish empty message: ", err)
	}
}

func TestAsyncSubscribe(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	omsg := []byte("Hello World")
	received := 0
	_, err := nc.Subscribe("foo", func(subj, _ string, data []byte, sub *Subscription) {
		received += 1
		if !bytes.Equal(data, omsg) {
			t.Fatal("Message received does not match")
		}
		if (sub == nil) {
			t.Fatal("Callback does not have a valid Subsription")
		}
	})
	if err != nil {
		t.Fatal("Failed to subscribe: ", err)
	}
	nc.Publish("foo", omsg)
	nc.Flush()
	if (received != 1) {
		t.Fatal("Message not received for subscription")
	}
}

func TestSyncSubscribe(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	s, err := nc.SubscribeSync("foo")
	if err != nil {
		t.Fatal("Failed to subscribe: ", err)
	}
	omsg := []byte("Hello World")
	nc.Publish("foo", omsg)
	msg, err := s.NextMsg(10 * time.Second)
	if err != nil || !bytes.Equal(msg.Data, omsg) {
		t.Fatal("Message received does not match")
	}
}

func TestPubSubWithReply(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	s, err := nc.SubscribeSync("foo")
	if err != nil {
		t.Fatal("Failed to subscribe: ", err)
	}
	omsg := []byte("Hello World")
	nc.PublishMsg(&Msg{Subject:"foo", Reply:"bar", Data:omsg})
	msg, err := s.NextMsg(10 * time.Second)
	if err != nil || !bytes.Equal(msg.Data, omsg) {
		t.Fatal("Message received does not match")
	}
}

func TestFlush(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	omsg := []byte("Hello World")
	received := 0
	nc.Subscribe("flush", func(_, _ string, _ []byte, _ *Subscription) {
		received += 1
	})
	total := 10000
	for i := 0; i < total; i++ {
		nc.Publish("flush", omsg)
	}
	nc.Flush()
	if (received != total) {
		t.Fatalf("All messages not received: %d != %d\n", received, total)
	}
}

func TestQueueSubscriber(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	r1 := 0
	r2 := 0
	nc.QueueSubscribe("foo", "bar", func(_, _ string, _ []byte, _ *Subscription) {
		r1 += 1
	})
	nc.QueueSubscribe("foo", "bar", func(_, _ string, _ []byte, _ *Subscription) {
		r2 += 1
	})
	omsg := []byte("Hello World")
	nc.Publish("foo", omsg)
	nc.Flush()
	if (r1 + r2) != 1 {
		t.Fatal("Received too many messages for multiple queue subscribers")
	}
	r1, r2 = 0, 0
	total := 100
	for i := 0; i < total; i++ {
		nc.Publish("foo", omsg)
	}
	nc.Flush()
	v := uint(float32(total) * 0.15)
	if r1 + r2 != total {
		t.Fatalf("Incorrect number of messages: %d vs %d", (r1 + r2), total)
	}
	expected := total / 2
	d1 := uint(math.Abs(float64(expected - r1)))
	d2 := uint(math.Abs(float64(expected - r2)))
	if (d1 > v || d2 > v) {
		t.Fatalf("Too much variance in totals: %d, %d > %d", d1, d2, v)
	}
}

func TestReplyArg(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	replyExpected := "bar"
	cbReceived := false
	nc.Subscribe("foo", func(_, reply string, data []byte, sub *Subscription) {
		cbReceived = true
		if (reply != replyExpected) {
			t.Fatalf("Did not receive correct reply arg in callback: " +
				     "('%s' vs '%s')", reply, replyExpected)
		}
	})
	nc.PublishMsg(&Msg{Subject:"foo", Reply:replyExpected, Data:[]byte("Hello")})
	nc.Flush()
	if !cbReceived {
		t.Fatal("Did not receive callback")
	}
}

func TestSyncReplyArg(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	replyExpected := "bar"
	s, _ := nc.SubscribeSync("foo")
	nc.PublishMsg(&Msg{Subject:"foo", Reply:replyExpected, Data:[]byte("Hello")})
	nc.Flush()
	msg, err := s.NextMsg(1 * time.Second)
	if err != nil {
		t.Fatal("Received and err on NextMsg()")
	}
	if (msg.Reply != replyExpected) {
		t.Fatalf("Did not receive correct reply arg in callback: " +
			     "('%s' vs '%s')", msg.Reply, replyExpected)
	}
}

func TestUnsubscribe(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	received := 0
	max := 10
	nc.Subscribe("foo", func(_, _ string, _ []byte, sub *Subscription) {
		received += 1
		if received == max {
			err := sub.Unsubscribe()
			if err != nil {
				t.Fatal("Unsubscribe failed with err:", err)
			}
		}
	})
	send := 20
	for i := 0; i < send; i++ {
		nc.Publish("foo", []byte("hello"))
	}
	nc.Flush()
	if received != max {
		t.Fatalf("Received wrong # of messages after unsubscribe: %d vs %d",
			     received, max)
	}
}

func TestRequestTimeout(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	_, err := nc.Request("foo", []byte("help"), 10*time.Millisecond)
	if err == nil {
		t.Fatalf("Expected to receive a timeout error")
	}
}

func TestRequest(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	response := []byte("I will help you")
	nc.Subscribe("foo", func(_, reply string, _ []byte, sub *Subscription) {
		nc.Publish(reply, response)
	})
	msg, err := nc.Request("foo", []byte("help"), 50*time.Millisecond)
	if err != nil {
		t.Fatalf("Received an error on Request test: %s", err)
	}
	if !bytes.Equal(msg.Data, response) {
		t.Fatalf("Received invalid response")
	}
}

func TestFlushInCB(t *testing.T) {
	nc := newConnection(t)
	defer nc.Close()
	nc.Subscribe("foo", func(_, _ string, _ []byte, _ *Subscription) {
		nc.Flush()
	})
	nc.Publish("foo", []byte("Hello"))
}

func TestReleaseFlush(t *testing.T) {
	nc := newConnection(t)
	for i := 0; i < 1000; i++ {
		nc.Publish("foo", []byte("Hello"))
	}
	go nc.Close()
	nc.Flush()
}

func TestInbox(t *testing.T) {
	inbox := NewInbox()
	matched, _ := regexp.Match(`_INBOX.\S`, []byte(inbox))
	if !matched {
		t.Fatal("Bad INBOX format")
	}
}
<|endoftext|>


<commit_before>package testutils

import (
	"firempq/common"

	. "firempq/api"

	. "github.com/smartystreets/goconvey/convey"
)

func VerifyItemsResponse(r IResponse, length int) ([]IItem, bool) {
	ir, ok := r.(*common.ItemsResponse)
	So(ok, ShouldBeTrue)
	if ok {
		items := ir.GetItems()
		So(len(items), ShouldEqual, length)
		return items, len(items) == length
	}
	return nil, false
}

func VerifyItem(r IResponse, itemId, payload string) bool {

	if items, ok := VerifyItemsResponse(r, 1); ok {
		So(items[0].GetId(), ShouldEqual, itemId)
		So(items[0].GetPayload(), ShouldEqual, payload)
		return items[0].GetId() == itemId && items[0].GetPayload() == payload
	}
	return false
}

func VerifyOk(r IResponse) {
	So(r, ShouldEqual, common.OK_RESPONSE)
}

func VerifyError(r IResponse) bool {
	err := r.IsError()
	So(err, ShouldBeTrue)
	return err
}

func VerifySize(s ISvc, size int) bool {
	So(s.Size(), ShouldEqual, size)
	return s.Size() == size
}
<commit_msg>Added function to compare multiple items at once.<commit_after>package testutils

import (
	"firempq/common"

	. "firempq/api"

	. "github.com/smartystreets/goconvey/convey"
)

func VerifyItemsResponse(r IResponse, size int) ([]IItem, bool) {
	ir, ok := r.(*common.ItemsResponse)
	So(ok, ShouldBeTrue)
	if ok {
		items := ir.GetItems()
		So(len(items), ShouldEqual, size)
		return items, len(items) == size
	}
	return nil, false
}

func VerifySingleItem(r IResponse, itemId, payload string) bool {

	if items, ok := VerifyItemsResponse(r, 1); ok {
		So(items[0].GetId(), ShouldEqual, itemId)
		So(items[0].GetPayload(), ShouldEqual, payload)
		return items[0].GetId() == itemId && items[0].GetPayload() == payload
	}
	return false
}

func VerifyItems(r IResponse, size int, itemSpecs ...string) bool {
	So(size*2, ShouldEqual, len(itemSpecs))
	items, ok := VerifyItemsResponse(r, size)
	if size*2 == len(itemSpecs) && ok {
		for i := 0; i < len(itemSpecs); i += 2 {
			itemPos := i / 2
			itemId := itemSpecs[i]
			itemPayload := itemSpecs[i+1]
			So(items[itemPos].GetId(), ShouldEqual, itemId)
			So(items[itemPos].GetPayload(), ShouldEqual, itemPayload)
		}
		return true
	}
	return false
}

func VerifyOkResponse(r IResponse) {
	So(r, ShouldEqual, common.OK_RESPONSE)
}

func VerifyErrResponse(r IResponse) bool {
	err := r.IsError()
	So(err, ShouldBeTrue)
	return err
}

func VerifyServiceSize(s ISvc, size int) bool {
	So(s.Size(), ShouldEqual, size)
	return s.Size() == size
}
<|endoftext|>


<commit_before>package provisioner

import (
	"fmt"
	"log"

	"github.com/digitalocean/godo"
	"golang.org/x/oauth2"
)

type Provisioner struct {
	client    *godo.Client
	endpoints map[string]*Endpoint
}

var (
	ErrNoToken  = fmt.Errorf("No token provided!")
	ErrNotFound = fmt.Errorf("Endpoint not found!")
)

func checkToken(client *godo.Client) bool {
	log.Println("Checking token...")
	account, _, err := client.Account.Get()
	if err != nil {
		return false
	}

	return account.Status == "active"
}

func NewProvisioner(token string) (*Provisioner, error) {
	if len(token) == 0 {
		return nil, ErrNoToken
	}

	tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
	oauthClient := oauth2.NewClient(oauth2.NoContext, tokenSource)
	client := godo.NewClient(oauthClient)
	if !checkToken(client) {
		return nil, fmt.Errorf("Token is not valid!")
	}

	provisioner := &Provisioner{
		client:    client,
		endpoints: make(map[string]*Endpoint),
	}
	go provisioner.restoreEndpoints()

	return provisioner, nil
}

func (p Provisioner) provisionEndpoint(endpoint *Endpoint, region string) {
	log.Println("Creating SSH key...")
	sshKey, err := createPrivateKey()
	if err != nil {
		return
	}
	sshKeyFingerprint := publicFingerprint(sshKey)
	log.Printf("Fingerprint of key: %s\n", sshKeyFingerprint)

	log.Println("Uploading key...")
	doKey, err := uploadPublicKey(p.client, sshKey.PublicKey(), endpoint.ID)
	if err != nil {
		return
	}
	log.Println("Uploaded key.")

	if doKey.Fingerprint != sshKeyFingerprint {
		err = fmt.Errorf("Key fingerprints do not match: %s, %s", sshKeyFingerprint, doKey.Fingerprint)
		return
	}
	log.Printf("Using key with fingerprint %s", doKey.Fingerprint)

	log.Println("Creating droplet...")
	droplet, err := createDroplet(p.client, doKey, region, endpoint.ID)
	if err != nil {
		return
	}
	endpoint.DropletID = droplet.ID
	log.Printf("Created droplet %s (%d)", droplet.Name, droplet.ID)

	log.Println("Waiting for droplet to be ready...")
	endpoint.IP, err = waitForNetwork(p.client, droplet.ID)
	if err != nil {
		return
	}
	log.Printf("Droplet is ready: %s", endpoint.IP)

	log.Println("Waiting for setup script to complete...")
	sshClient := waitForSetup(sshKey, endpoint.IP)
	defer sshClient.Close()
	log.Println("Setup complete.")

	log.Println("Reading secret...")
	secret, err := readSecret(sshClient)
	if err != nil {
		return
	}
	log.Printf("Successfully read secret.")

	log.Println("Writing configuration...")
	endpoint.Config, err = createConfigFile(endpoint.IP, secret)
	if err != nil {
		return
	}
	log.Println("Created configuration.")

	log.Println("Remove SSH key from DigitalOcean...")
	deletePublicKey(p.client, doKey)
}

func (p Provisioner) unprovisionEndpoint(endpoint *Endpoint) {
	err := deleteDroplet(p.client, endpoint.DropletID)
	if err == nil {
		endpoint.Status = Destroyed
	}
}
<commit_msg>Update status after completing setup.<commit_after>package provisioner

import (
	"fmt"
	"log"

	"github.com/digitalocean/godo"
	"golang.org/x/oauth2"
)

type Provisioner struct {
	client    *godo.Client
	endpoints map[string]*Endpoint
}

var (
	ErrNoToken  = fmt.Errorf("No token provided!")
	ErrNotFound = fmt.Errorf("Endpoint not found!")
)

func checkToken(client *godo.Client) bool {
	log.Println("Checking token...")
	account, _, err := client.Account.Get()
	if err != nil {
		return false
	}

	return account.Status == "active"
}

func NewProvisioner(token string) (*Provisioner, error) {
	if len(token) == 0 {
		return nil, ErrNoToken
	}

	tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
	oauthClient := oauth2.NewClient(oauth2.NoContext, tokenSource)
	client := godo.NewClient(oauthClient)
	if !checkToken(client) {
		return nil, fmt.Errorf("Token is not valid!")
	}

	provisioner := &Provisioner{
		client:    client,
		endpoints: make(map[string]*Endpoint),
	}
	go provisioner.restoreEndpoints()

	return provisioner, nil
}

func (p Provisioner) provisionEndpoint(endpoint *Endpoint, region string) {
	log.Println("Creating SSH key...")
	sshKey, err := createPrivateKey()
	if err != nil {
		return
	}
	sshKeyFingerprint := publicFingerprint(sshKey)
	log.Printf("Fingerprint of key: %s\n", sshKeyFingerprint)

	log.Println("Uploading key...")
	doKey, err := uploadPublicKey(p.client, sshKey.PublicKey(), endpoint.ID)
	if err != nil {
		return
	}
	log.Println("Uploaded key.")

	if doKey.Fingerprint != sshKeyFingerprint {
		err = fmt.Errorf("Key fingerprints do not match: %s, %s", sshKeyFingerprint, doKey.Fingerprint)
		return
	}
	log.Printf("Using key with fingerprint %s", doKey.Fingerprint)

	log.Println("Creating droplet...")
	droplet, err := createDroplet(p.client, doKey, region, endpoint.ID)
	if err != nil {
		return
	}
	endpoint.DropletID = droplet.ID
	log.Printf("Created droplet %s (%d)", droplet.Name, droplet.ID)

	log.Println("Waiting for droplet to be ready...")
	endpoint.IP, err = waitForNetwork(p.client, droplet.ID)
	if err != nil {
		return
	}
	log.Printf("Droplet is ready: %s", endpoint.IP)

	log.Println("Waiting for setup script to complete...")
	sshClient := waitForSetup(sshKey, endpoint.IP)
	defer sshClient.Close()
	log.Println("Setup complete.")

	log.Println("Reading secret...")
	secret, err := readSecret(sshClient)
	if err != nil {
		return
	}
	log.Printf("Successfully read secret.")

	log.Println("Writing configuration...")
	endpoint.Config, err = createConfigFile(endpoint.IP, secret)
	if err != nil {
		return
	}
	log.Println("Created configuration.")

	log.Println("Remove SSH key from DigitalOcean...")
	deletePublicKey(p.client, doKey)

	endpoint.Status = Running
}

func (p Provisioner) unprovisionEndpoint(endpoint *Endpoint) {
	err := deleteDroplet(p.client, endpoint.DropletID)
	if err == nil {
		endpoint.Status = Destroyed
	}
}
<|endoftext|>


<commit_before>package net

import (
	"fmt"
	"math/rand"
	"net"
	"os"
	"strings"
	"time"
)

const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const (
	letterIdxBits = 6                    // 6 bits to represent a letter index
	letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
	letterIdxMax  = 63 / letterIdxBits   // # of letter indices fitting in 63 bits
)

var src = rand.NewSource(time.Now().UnixNano())

// RandStringBytesMaskImprSrc returns a random string given a length
func RandStringBytesMaskImprSrc(n int) string {
	b := make([]byte, n)
	// A src.Int63() generates 63 random bits, enough for letterIdxMax characters!
	for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
		if remain == 0 {
			cache, remain = src.Int63(), letterIdxMax
		}
		if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
			b[i] = letterBytes[idx]
			i--
		}
		cache >>= letterIdxBits
		remain--
	}

	return string(b)
}

// GetAddresses returns the addresses the transport is listening to
func GetAddresses(l net.Listener) []string {
	port := l.Addr().(*net.TCPAddr).Port
	// TODO log errors
	network := strings.ToLower(l.Addr().Network())
	addrs, _ := GetLocalPeerAddresses(port)
	for i, addr := range addrs {
		addrs[i] = fmt.Sprintf("%s:%s", network, addr)
	}
	return addrs
}

func fmtAddress(address string, port int) string {
	return fmt.Sprintf("tcps:%s:%d", address, port)
}

// GetLocalPeerAddresses returns the addresses TCP can listen to on the local machine
func GetLocalPeerAddresses(port int) ([]string, error) {
	// go through all ifs
	ifaces, err := net.Interfaces()
	if err != nil {
		return nil, err
	}

	// gather addresses of all ifs
	ips := []net.Addr{}
	for _, iface := range ifaces {
		ifIPs, err := iface.Addrs()
		if err != nil {
			continue
		}
		ips = append(ips, ifIPs...)
	}

	// gather valid addresses
	addrs := []string{}
	for _, ip := range ips {
		cleanIP, valid := isValidIP(ip)
		if valid {
			hostPort := fmt.Sprintf("%s:%d", cleanIP, port)
			addrs = append(addrs, hostPort)
		}
	}
	return addrs, nil
}

func isValidIP(addr net.Addr) (string, bool) {
	var ip net.IP
	switch v := addr.(type) {
	case *net.IPNet:
		ip = v.IP
	case *net.IPAddr:
		ip = v.IP
	}
	if ip == nil {
		return "", false
	}
	if os.Getenv("BIND_LOCAL") == "" && (ip.IsLoopback() || isPrivate(ip)) {
		return "", false
	}
	if os.Getenv("BIND_IPV6") == "" && isIPv6(ip.String()) {
		return "", false
	}
	return ip.String(), true
}

func isIPv4(address string) bool {
	return strings.Count(address, ":") < 2
}

func isIPv6(address string) bool {
	return strings.Count(address, ":") >= 2
}

func isPrivate(ip net.IP) bool {
	_, block24, _ := net.ParseCIDR("10.0.0.0/8")
	_, block20, _ := net.ParseCIDR("172.16.0.0/12")
	_, block16, _ := net.ParseCIDR("192.168.0.0/16")
	return block16.Contains(ip) || block20.Contains(ip) || block24.Contains(ip)
}
<commit_msg>Addr should use tcps<commit_after>package net

import (
	"fmt"
	"math/rand"
	"net"
	"os"
	"strings"
	"time"
)

const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const (
	letterIdxBits = 6                    // 6 bits to represent a letter index
	letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
	letterIdxMax  = 63 / letterIdxBits   // # of letter indices fitting in 63 bits
)

var src = rand.NewSource(time.Now().UnixNano())

// RandStringBytesMaskImprSrc returns a random string given a length
func RandStringBytesMaskImprSrc(n int) string {
	b := make([]byte, n)
	// A src.Int63() generates 63 random bits, enough for letterIdxMax characters!
	for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
		if remain == 0 {
			cache, remain = src.Int63(), letterIdxMax
		}
		if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
			b[i] = letterBytes[idx]
			i--
		}
		cache >>= letterIdxBits
		remain--
	}

	return string(b)
}

// GetAddresses returns the addresses the transport is listening to
func GetAddresses(l net.Listener) []string {
	port := l.Addr().(*net.TCPAddr).Port
	// TODO log errors
	network := strings.ToLower(l.Addr().Network())
	if network == "tcp" {
		network = "tcps"
	}
	addrs, _ := GetLocalPeerAddresses(port)
	for i, addr := range addrs {
		addrs[i] = fmt.Sprintf("%s:%s", network, addr)
	}
	return addrs
}

func fmtAddress(address string, port int) string {
	return fmt.Sprintf("tcp:%s:%d", address, port)
}

// GetLocalPeerAddresses returns the addresses TCP can listen to on the local machine
func GetLocalPeerAddresses(port int) ([]string, error) {
	// go through all ifs
	ifaces, err := net.Interfaces()
	if err != nil {
		return nil, err
	}

	// gather addresses of all ifs
	ips := []net.Addr{}
	for _, iface := range ifaces {
		ifIPs, err := iface.Addrs()
		if err != nil {
			continue
		}
		ips = append(ips, ifIPs...)
	}

	// gather valid addresses
	addrs := []string{}
	for _, ip := range ips {
		cleanIP, valid := isValidIP(ip)
		if valid {
			hostPort := fmt.Sprintf("%s:%d", cleanIP, port)
			addrs = append(addrs, hostPort)
		}
	}
	return addrs, nil
}

func isValidIP(addr net.Addr) (string, bool) {
	var ip net.IP
	switch v := addr.(type) {
	case *net.IPNet:
		ip = v.IP
	case *net.IPAddr:
		ip = v.IP
	}
	if ip == nil {
		return "", false
	}
	if os.Getenv("BIND_LOCAL") == "" && (ip.IsLoopback() || isPrivate(ip)) {
		return "", false
	}
	if os.Getenv("BIND_IPV6") == "" && isIPv6(ip.String()) {
		return "", false
	}
	return ip.String(), true
}

func isIPv4(address string) bool {
	return strings.Count(address, ":") < 2
}

func isIPv6(address string) bool {
	return strings.Count(address, ":") >= 2
}

func isPrivate(ip net.IP) bool {
	_, block24, _ := net.ParseCIDR("10.0.0.0/8")
	_, block20, _ := net.ParseCIDR("172.16.0.0/12")
	_, block16, _ := net.ParseCIDR("192.168.0.0/16")
	return block16.Contains(ip) || block20.Contains(ip) || block24.Contains(ip)
}
<|endoftext|>


<commit_before>package orm

import (
	"fmt"
	"reflect"

	"github.com/go-pg/pg/types"
)

func compositeScanner(typ reflect.Type) types.ScannerFunc {
	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
	}

	var table *Table
	return func(v reflect.Value, rd types.Reader, n int) error {
		if !v.CanSet() {
			return fmt.Errorf("pg: Scan(nonsettable %s)", v.Type())
		}

		if n == -1 {
			v.Set(reflect.Zero(v.Type()))
			return nil
		}

		if table == nil {
			table = GetTable(typ)
		}
		if v.Kind() == reflect.Ptr {
			if v.IsNil() {
				v.Set(reflect.New(v.Type().Elem()))
			}
			v = v.Elem()
		}

		p := newCompositeParser(rd)
		var elemReader *types.BytesReader

		var firstErr error
		for i := 0; ; i++ {
			elem, err := p.NextElem()
			if err != nil {
				if err == endOfComposite {
					break
				}
				return err
			}

			if i >= len(table.allFields) {
				if firstErr == nil {
					firstErr = fmt.Errorf(
						"%s has %d fields, but composite at least %d values",
						table, len(table.allFields), i)
				}
				continue
			}

			if elemReader == nil {
				elemReader = types.NewBytesReader(elem)
			} else {
				elemReader.Reset(elem)
			}

			field := table.allFields[i]
			err = field.ScanValue(v, elemReader, len(elem))
			if err != nil && firstErr == nil {
				firstErr = err
			}
		}

		return firstErr
	}
}

func compositeAppender(typ reflect.Type) types.AppenderFunc {
	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
	}

	var table *Table
	return func(b []byte, v reflect.Value, quote int) []byte {
		if table == nil {
			table = GetTable(typ)
		}
		if v.Kind() == reflect.Ptr {
			v = v.Elem()
		}

		b = append(b, '(')
		for i, f := range table.Fields {
			if i > 0 {
				b = append(b, ',')
			}
			b = f.AppendValue(b, v, quote)
		}
		b = append(b, ')')
		return b
	}
}
<commit_msg>orm: use field in composite<commit_after>package orm

import (
	"fmt"
	"reflect"

	"github.com/go-pg/pg/types"
)

func compositeScanner(typ reflect.Type) types.ScannerFunc {
	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
	}

	var table *Table
	return func(v reflect.Value, rd types.Reader, n int) error {
		if !v.CanSet() {
			return fmt.Errorf("pg: Scan(nonsettable %s)", v.Type())
		}

		if n == -1 {
			v.Set(reflect.Zero(v.Type()))
			return nil
		}

		if table == nil {
			table = GetTable(typ)
		}
		if v.Kind() == reflect.Ptr {
			if v.IsNil() {
				v.Set(reflect.New(v.Type().Elem()))
			}
			v = v.Elem()
		}

		p := newCompositeParser(rd)
		var elemReader *types.BytesReader

		var firstErr error
		for i := 0; ; i++ {
			elem, err := p.NextElem()
			if err != nil {
				if err == endOfComposite {
					break
				}
				return err
			}

			if i >= len(table.Fields) {
				if firstErr == nil {
					firstErr = fmt.Errorf(
						"%s has %d fields, but composite at least %d values",
						table, len(table.Fields), i)
				}
				continue
			}

			if elemReader == nil {
				elemReader = types.NewBytesReader(elem)
			} else {
				elemReader.Reset(elem)
			}

			field := table.Fields[i]
			err = field.ScanValue(v, elemReader, len(elem))
			if err != nil && firstErr == nil {
				firstErr = err
			}
		}

		return firstErr
	}
}

func compositeAppender(typ reflect.Type) types.AppenderFunc {
	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
	}

	var table *Table
	return func(b []byte, v reflect.Value, quote int) []byte {
		if table == nil {
			table = GetTable(typ)
		}
		if v.Kind() == reflect.Ptr {
			v = v.Elem()
		}

		b = append(b, '(')
		for i, f := range table.Fields {
			if i > 0 {
				b = append(b, ',')
			}
			b = f.AppendValue(b, v, quote)
		}
		b = append(b, ')')
		return b
	}
}
<|endoftext|>


<commit_before>package ds

import (
	"fmt"
	"path/filepath"

	"github.com/sselph/scraper/gdb"
)

// Daphne is a data source using GDB for Daphne games.
type Daphne struct {
	HM *HashMap
}

func (d *Daphne) GetID(p string) (string, error) {
	if filepath.Ext(p) != ".daphne" {
		return "", NotFoundErr
	}
	gameID := filepath.Base(p)
	id, ok := d.HM.GetID(gameID)
	if !ok {
		return "", NotFoundErr
	}
	return id, nil
}

func (d *Daphne) GetName(p string) string {
	gameID := filepath.Base(p)
	n, ok := d.HM.GetName(gameID)
	if !ok {
		return ""
	}
	return n
}

func (d *Daphne) GetGame(id string) (*Game, error) {
	req := gdb.GGReq{ID: id}
	resp, err := gdb.GetGame(req)
	if err != nil {
		return nil, err
	}
	if len(resp.Game) == 0 {
		return nil, fmt.Errorf("game with id (%s) not found", id)
	}
	game := resp.Game[0]
	ret := ParseGDBGame(game, resp.ImageURL)
	ret.ID = id
	return ret, nil
}
<commit_msg>Add daphne wildcards<commit_after>package ds

import (
	"fmt"
	"path/filepath"

	"github.com/sselph/scraper/gdb"
)

// Daphne is a data source using GDB for Daphne games.
type Daphne struct {
	HM *HashMap
}

func (d *Daphne) GetID(p string) (string, error) {
	if filepath.Ext(p) != ".daphne" {
		return "", NotFoundErr
	}
	gameID := filepath.Base(p)
	switch {
	case strings.HasPrefix(gameID, "lair2_"):
		gameID = "lair2_*.daphne"
	case strings.HasPrefix(gameID, "sdq"):
		gameID = "sdq*.daphne"
	case strings.HasPrefix(gameID, "tq"):
		gameID = "tq*.daphne"
	}
	id, ok := d.HM.GetID(gameID)
	if !ok {
		return "", NotFoundErr
	}
	return id, nil
}

func (d *Daphne) GetName(p string) string {
	gameID := filepath.Base(p)
	n, ok := d.HM.GetName(gameID)
	if !ok {
		return ""
	}
	return n
}

func (d *Daphne) GetGame(id string) (*Game, error) {
	req := gdb.GGReq{ID: id}
	resp, err := gdb.GetGame(req)
	if err != nil {
		return nil, err
	}
	if len(resp.Game) == 0 {
		return nil, fmt.Errorf("game with id (%s) not found", id)
	}
	game := resp.Game[0]
	ret := ParseGDBGame(game, resp.ImageURL)
	ret.ID = id
	return ret, nil
}
<|endoftext|>


<commit_before>// dsrt.go -- directoy sort in reverse date order.  IE, newest is first.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
)

const lastCompiled = "24 Apr 17"

/*
Revision History
----------------
20 Apr 17 -- Started writing dsize rtn, based on dirlist.go
21 Apr 17 -- Now tweaking the output format.  And used flag package.  One as a pointer and one as a value, just to learn them.
22 Apr 17 -- Coded the use of the first non flag commandline param,  which is all I need.  Note that the flag must appear before the non-flag param, else the flag is ignored.
22 Apr 17 -- Now writing dsrt, to function similarly to dsort.
24 Apr 17 -- Now adding file matching, like dir or ls does.
*/

// FIS is a FileInfo slice, as in os.FileInfo
type FISlice []os.FileInfo

func (f FISlice) Less(i, j int) bool {
	return f[i].ModTime().UnixNano() > f[j].ModTime().UnixNano() // I want a reverse sort, newest first
}

func (f FISlice) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (f FISlice) Len() int {
	return len(f)
}

func main() {
	const numlines = 50
	var files FISlice
	var err error
	var count int

	var revflag = flag.Bool("r", false, "reverse the sort, ie, oldest is first") // Ptr

	var RevFlag bool
	flag.BoolVar(&RevFlag, "R", false, "Reverse the sort, ie, oldest is first") // Value

	var nlines = flag.Int("n", numlines, "number of lines to display") // Ptr

	var NLines int
	flag.IntVar(&NLines, "N", numlines, "number of lines to display") // Value

	fmt.Println(" dsrt will display a directory by date.  Written in Go.  lastCompiled ", lastCompiled)
	fmt.Println()

	flag.PrintDefaults()
	flag.Parse()
	Reverse := *revflag || RevFlag

	NumLines := numlines
	if *nlines != numlines {
		NumLines = *nlines
	} else if NLines != numlines {
		NumLines = NLines
	}
	askforinput := true

	CleanDirName := "." + string(filepath.Separator)
	CleanFileName := ""
	commandline := flag.Arg(0) // this only gets the first non flag argument.  That's all I want
	if len(commandline) > 0 {
		CleanDirName = filepath.Clean(commandline)
		askforinput = false
	}

	if askforinput {
		// Asking for input so don't have to worry about command line globbing
		fmt.Print(" Enter input for globbing: ")
		scanner := bufio.NewScanner(os.Stdin)
		scanner.Scan()
		newtext := scanner.Text()
		if err = scanner.Err(); err != nil {
			fmt.Fprintln(os.Stderr, " reading std input: ", err)
			os.Exit(1)
		}
		if len(newtext) > 0 {
			// time to do the stuff I'm writing this pgm for
			CleanDirName, CleanFileName = filepath.Split(newtext)
			if len(CleanDirName) == 0 {
				CleanDirName = "." + string(filepath.Separator)
			}
		}

	}

	if len(CleanFileName) == 0 {
		CleanFileName = "*"
	}

	files, err = ioutil.ReadDir(CleanDirName)
	if err != nil {
		log.Fatal(err)
	}

	if Reverse {
		sort.Sort(sort.Reverse(files))
	} else {
		sort.Sort(files)
	}

	fmt.Println(" Dirname is", CleanDirName)

	for _, f := range files {
		if BOOL, _ := filepath.Match(CleanFileName, f.Name()); BOOL {
			s := f.ModTime().Format("Jan-02-2006 15:04:05")
			fmt.Printf("%10v %11d %s %s\n", f.Mode(), f.Size(), s, f.Name())
			count++
			if count > NumLines {
				break
			}
		}
	}

} // end main dsrt

/*
package path
func Match

func Match(pattern, name string) (matched bool, err error)

Match reports whether name matches the shell file name pattern.  The pattern syntax is:

pattern:
	{ term }
term:
	'*'         matches any sequence of non-/ characters
	'?'         matches any single non-/ character
	'[' [ '^' ] { character-range } ']'
	            character class (must be non-empty)
	c           matches character c (c != '*', '?', '\\', '[')
	'\\' c      matches character c

character-range:
	c           matches character c (c != '\\', '-', ']')
	'\\' c      matches character c
	lo '-' hi   matches character c for lo <= c <= hi

Match requires pattern to match all of name, not just a substring.  The only possible returned error is ErrBadPattern, when pattern is malformed.

package os
type FileInfo

type FileInfo interface {
        Name() string       // base name of the file
        Size() int64        // length in bytes for regular files; system-dependent for others
        Mode() FileMode     // file mode bits
        ModTime() time.Time // modification time
        IsDir() bool        // abbreviation for Mode().IsDir()
        Sys() interface{}   // underlying data source (can return nil)
}

A FileInfo describes a file and is returned by Stat and Lstat.

func Lstat

func Lstat(name string) (FileInfo, error)

Lstat returns a FileInfo describing the named file.  If the file is a symbolic link, the returned FileInfo describes the symbolic link.  Lstat makes no attempt to follow the link.
If there is an error, it will be of type *PathError.

func Stat

func Stat(name string) (FileInfo, error)

Stat returns a FileInfo describing the named file.  If there is an error, it will be of type *PathError.

The insight I had with my append troubles that the 1 slice entries were empty, is that when I used append, it would do just that to the end of the slice, and ignore the empty slices.
I needed to make the slice as empty for this to work.  So I am directly assigning the DirEntries slice, and appending the FileNames slice, to make sure that these both are doing what I want.
This code is now doing exactly what I want.  I guess there is no substitute for playing with myself.  Wait, that didn't come out right.  Or did it.

*/
<commit_msg>modified:   dsrt/dsrt.go -- adding dsize functionality<commit_after>// dsrt.go -- directoy sort in reverse date order.  IE, newest is first.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
)

const lastCompiled = "25 Apr 17"

/*
Revision History
----------------
20 Apr 17 -- Started writing dsize rtn, based on dirlist.go
21 Apr 17 -- Now tweaking the output format.  And used flag package.  One as a pointer and one as a value, just to learn them.
22 Apr 17 -- Coded the use of the first non flag commandline param,  which is all I need.  Note that the flag must appear before the non-flag param, else the flag is ignored.
22 Apr 17 -- Now writing dsrt, to function similarly to dsort.
24 Apr 17 -- Now adding file matching, like "dir" or "ls" does.
25 Apr 17 -- Now adding sort by size as an option, like -s
*/

// FIS is a FileInfo slice, as in os.FileInfo
type FISlice []os.FileInfo
type FISliceDate []os.FileInfo
type FISliceSize []os.FileInfo

func (f FISliceDate) Less(i, j int) bool {
	return f[i].ModTime().UnixNano() > f[j].ModTime().UnixNano() // I want a reverse sort, newest first
}

func (f FISliceDate) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (f FISliceDate) Len() int {
	return len(f)
}

func (f FISliceSize) Less(i, j int) bool {
	return f[i].Size() > f[j].Size() // I want a reverse sort, largest first
}

func (f FISliceSize) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (f FISliceSize) Len() int {
	return len(f)
}

func main() {
	const numlines = 50
	var files FISlice
	var filesDate FISliceDate
	var filesSize FISliceSize
	var err error
	var count int

	var revflag = flag.Bool("r", false, "reverse the sort, ie, oldest or smallest is first") // Ptr

	var RevFlag bool
	flag.BoolVar(&RevFlag, "R", false, "Reverse the sort, ie, oldest or smallest is first") // Value

	var nlines = flag.Int("n", numlines, "number of lines to display") // Ptr

	var NLines int
	flag.IntVar(&NLines, "N", numlines, "number of lines to display") // Value

	var helpflag = flag.Bool("h", false, "print help message") // pointer
	var HelpFlag bool
	flag.BoolVar(&HelpFlag, "H", false, "print help message")

	var sizeflag = flag.Bool("s", false, "sort by size instead of by date") // pointer
	var SizeFlag bool
	flag.BoolVar(&SizeFlag, "S", false, "sort by size instead of by date")
	flag.Parse()

	fmt.Println(" dsrt will display a directory by date or size.  Written in Go.  LastCompiled ", lastCompiled)
	fmt.Println()

	if *helpflag || HelpFlag {
		flag.PrintDefaults()
	}

	Reverse := *revflag || RevFlag
	SizeSort := *sizeflag || SizeFlag

	NumLines := numlines
	if *nlines != numlines {
		NumLines = *nlines
	} else if NLines != numlines {
		NumLines = NLines
	}
	askforinput := true

	CleanDirName := "." + string(filepath.Separator)
	CleanFileName := ""
	commandline := flag.Arg(0) // this only gets the first non flag argument.  That's all I want
	if len(commandline) > 0 {
		CleanDirName = filepath.Clean(commandline)
		askforinput = false
	}

	if askforinput {
		// Asking for input so don't have to worry about command line globbing
		fmt.Print(" Enter input for globbing: ")
		scanner := bufio.NewScanner(os.Stdin)
		scanner.Scan()
		newtext := scanner.Text()
		if err = scanner.Err(); err != nil {
			fmt.Fprintln(os.Stderr, " reading std input: ", err)
			os.Exit(1)
		}
		if len(newtext) > 0 {
			// time to do the stuff I'm writing this pgm for
			CleanDirName, CleanFileName = filepath.Split(newtext)
			if len(CleanDirName) == 0 {
				CleanDirName = "." + string(filepath.Separator)
			}
		}

	}

	if len(CleanFileName) == 0 {
		CleanFileName = "*"
	}

	if SizeSort {
		filesSize, err = ioutil.ReadDir(CleanDirName)
		if err != nil {
			log.Fatal(err)
		}
		if Reverse {
			sort.Sort(sort.Reverse(filesSize))
		} else {
			sort.Sort(filesSize)
		}
		files = FISlice(filesSize)
	} else {
		filesDate, err = ioutil.ReadDir(CleanDirName)
		if err != nil {
			log.Fatal(err)
		}
		if Reverse {
			sort.Sort(sort.Reverse(filesDate))
		} else {
			sort.Sort(filesDate)
		}
		files = FISlice(filesDate)
	}

	fmt.Println(" Dirname is", CleanDirName)

	for _, f := range files {
		if BOOL, _ := filepath.Match(CleanFileName, f.Name()); BOOL {
			s := f.ModTime().Format("Jan-02-2006 15:04:05")
			fmt.Printf("%10v %11d %s %s\n", f.Mode(), f.Size(), s, f.Name())
			count++
			if count > NumLines {
				break
			}
		}
	}

} // end main dsrt

/*
package path
func Match

func Match(pattern, name string) (matched bool, err error)

Match reports whether name matches the shell file name pattern.  The pattern syntax is:

pattern:
	{ term }
term:
	'*'         matches any sequence of non-/ characters
	'?'         matches any single non-/ character
	'[' [ '^' ] { character-range } ']'
	            character class (must be non-empty)
	c           matches character c (c != '*', '?', '\\', '[')
	'\\' c      matches character c

character-range:
	c           matches character c (c != '\\', '-', ']')
	'\\' c      matches character c
	lo '-' hi   matches character c for lo <= c <= hi

Match requires pattern to match all of name, not just a substring.  The only possible returned error is ErrBadPattern, when pattern is malformed.

package os
type FileInfo

type FileInfo interface {
        Name() string       // base name of the file
        Size() int64        // length in bytes for regular files; system-dependent for others
        Mode() FileMode     // file mode bits
        ModTime() time.Time // modification time
        IsDir() bool        // abbreviation for Mode().IsDir()
        Sys() interface{}   // underlying data source (can return nil)
}

A FileInfo describes a file and is returned by Stat and Lstat.

func Lstat

func Lstat(name string) (FileInfo, error)

Lstat returns a FileInfo describing the named file.  If the file is a symbolic link, the returned FileInfo describes the symbolic link.  Lstat makes no attempt to follow the link.
If there is an error, it will be of type *PathError.

func Stat

func Stat(name string) (FileInfo, error)

Stat returns a FileInfo describing the named file.  If there is an error, it will be of type *PathError.

The insight I had with my append troubles that the 1 slice entries were empty, is that when I used append, it would do just that to the end of the slice, and ignore the empty slices.
I needed to make the slice as empty for this to work.  So I am directly assigning the DirEntries slice, and appending the FileNames slice, to make sure that these both are doing what I want.
This code is now doing exactly what I want.  I guess there is no substitute for playing with myself.  Wait, that didn't come out right.  Or did it.

*/
<|endoftext|>


<commit_before>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"log"
	"os"
	"strconv"
	"time"

	"net/http"
	"net/url"

	"github.com/coreos/etcd/etcdserver/stats"
	"github.com/coreos/etcd/pkg/fileutil"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/rafthttp"
	"github.com/coreos/etcd/snap"
	"github.com/coreos/etcd/wal"
	"github.com/coreos/etcd/wal/walpb"
	"golang.org/x/net/context"
)

// A key-value stream backed by raft
type raftNode struct {
	proposeC    <-chan string            // proposed messages (k,v)
	confChangeC <-chan raftpb.ConfChange // proposed cluster config changes
	commitC     chan<- *string           // entries committed to log (k,v)
	errorC      chan<- error             // errors from raft session

	id        int      // client ID for raft session
	peers     []string // raft peer URLs
	join      bool     // node is joining an existing cluster
	waldir    string   // path to WAL directory
	snapdir   string   // path to snapshot directory
	lastIndex uint64   // index of log at start

	// raft backing for the commit/error channel
	node        raft.Node
	raftStorage *raft.MemoryStorage
	wal         *wal.WAL
	snapshotter *snap.Snapshotter
	transport   *rafthttp.Transport
	stopc       chan struct{} // signals proposal channel closed
	httpstopc   chan struct{} // signals http server to shutdown
	httpdonec   chan struct{} // signals http server shutdown complete
}

// newRaftNode initiates a raft instance and returns a committed log entry
// channel and error channel. Proposals for log updates are sent over the
// provided the proposal channel. All log entries are replayed over the
// commit channel, followed by a nil message (to indicate the channel is
// current), then new log entries. To shutdown, close proposeC and read errorC.
func newRaftNode(id int, peers []string, join bool, proposeC <-chan string,
	confChangeC <-chan raftpb.ConfChange) (<-chan *string, <-chan error) {

	commitC := make(chan *string)
	errorC := make(chan error)

	rc := &raftNode{
		proposeC:    proposeC,
		confChangeC: confChangeC,
		commitC:     commitC,
		errorC:      errorC,
		id:          id,
		peers:       peers,
		join:        join,
		waldir:      fmt.Sprintf("raftexample-%d", id),
		snapdir:     fmt.Sprintf("raftexample-%d-snap", id),
		raftStorage: raft.NewMemoryStorage(),
		stopc:       make(chan struct{}),
		httpstopc:   make(chan struct{}),
		httpdonec:   make(chan struct{}),
		// rest of structure populated after WAL replay
	}
	go rc.startRaft()
	return commitC, errorC
}

func (rc *raftNode) saveSnap(snap raftpb.Snapshot) error {
	if err := rc.snapshotter.SaveSnap(snap); err != nil {
		return err
	}
	walSnap := walpb.Snapshot{
		Index: snap.Metadata.Index,
		Term:  snap.Metadata.Term,
	}
	if err := rc.wal.SaveSnapshot(walSnap); err != nil {
		return err
	}
	return rc.wal.ReleaseLockTo(snap.Metadata.Index)
}

// publishEntries writes committed log entries to commit channel and returns
// whether all entries could be published.
func (rc *raftNode) publishEntries(ents []raftpb.Entry) bool {
	for i := range ents {
		switch ents[i].Type {
		case raftpb.EntryNormal:
			if len(ents[i].Data) == 0 {
				// ignore empty messages
				break
			}
			s := string(ents[i].Data)
			select {
			case rc.commitC <- &s:
			case <-rc.stopc:
				return false
			}

		case raftpb.EntryConfChange:
			var cc raftpb.ConfChange
			cc.Unmarshal(ents[i].Data)
			rc.node.ApplyConfChange(cc)
			switch cc.Type {
			case raftpb.ConfChangeAddNode:
				if len(cc.Context) > 0 {
					rc.transport.AddPeer(types.ID(cc.NodeID), []string{string(cc.Context)})
				}
			case raftpb.ConfChangeRemoveNode:
				if cc.NodeID == uint64(rc.id) {
					log.Println("I've been removed from the cluster! Shutting down.")
					return false
				}
				rc.transport.RemovePeer(types.ID(cc.NodeID))
			}
		}

		// special nil commit to signal replay has finished
		if ents[i].Index == rc.lastIndex {
			select {
			case rc.commitC <- nil:
			case <-rc.stopc:
				return false
			}
		}
	}
	return true
}

// openWAL returns a WAL ready for reading.
func (rc *raftNode) openWAL() *wal.WAL {
	if !wal.Exist(rc.waldir) {
		if err := os.Mkdir(rc.waldir, 0750); err != nil {
			log.Fatalf("raftexample: cannot create dir for wal (%v)", err)
		}

		w, err := wal.Create(rc.waldir, nil)
		if err != nil {
			log.Fatalf("raftexample: create wal error (%v)", err)
		}
		w.Close()
	}

	w, err := wal.Open(rc.waldir, walpb.Snapshot{})
	if err != nil {
		log.Fatalf("raftexample: error loading wal (%v)", err)
	}

	return w
}

// replayWAL replays WAL entries into the raft instance.
func (rc *raftNode) replayWAL() *wal.WAL {
	w := rc.openWAL()
	_, st, ents, err := w.ReadAll()
	if err != nil {
		log.Fatalf("raftexample: failed to read WAL (%v)", err)
	}
	// append to storage so raft starts at the right place in log
	rc.raftStorage.Append(ents)
	// send nil once lastIndex is published so client knows commit channel is current
	if len(ents) > 0 {
		rc.lastIndex = ents[len(ents)-1].Index
	} else {
		rc.commitC <- nil
	}
	rc.raftStorage.SetHardState(st)
	return w
}

func (rc *raftNode) writeError(err error) {
	rc.stopHTTP()
	close(rc.commitC)
	rc.errorC <- err
	close(rc.errorC)
	rc.node.Stop()
}

func (rc *raftNode) startRaft() {
	if !fileutil.Exist(rc.snapdir) {
		if err := os.Mkdir(rc.snapdir, 0750); err != nil {
			log.Fatalf("raftexample: cannot create dir for snapshot (%v)", err)
		}
	}
	oldwal := wal.Exist(rc.waldir)
	rc.wal = rc.replayWAL()
	rc.snapshotter = snap.New(rc.snapdir)

	rpeers := make([]raft.Peer, len(rc.peers))
	for i := range rpeers {
		rpeers[i] = raft.Peer{ID: uint64(i + 1)}
	}
	c := &raft.Config{
		ID:              uint64(rc.id),
		ElectionTick:    10,
		HeartbeatTick:   1,
		Storage:         rc.raftStorage,
		MaxSizePerMsg:   1024 * 1024,
		MaxInflightMsgs: 256,
	}

	if oldwal {
		rc.node = raft.RestartNode(c)
	} else {
		startPeers := rpeers
		if rc.join {
			startPeers = nil
		}
		rc.node = raft.StartNode(c, startPeers)
	}

	ss := &stats.ServerStats{}
	ss.Initialize()

	rc.transport = &rafthttp.Transport{
		ID:          types.ID(rc.id),
		ClusterID:   0x1000,
		Raft:        rc,
		ServerStats: ss,
		LeaderStats: stats.NewLeaderStats(strconv.Itoa(rc.id)),
		ErrorC:      make(chan error),
	}

	rc.transport.Start()
	for i := range rc.peers {
		if i+1 != rc.id {
			rc.transport.AddPeer(types.ID(i+1), []string{rc.peers[i]})
		}
	}

	go rc.serveRaft()
	go rc.serveChannels()
}

// stop closes http, closes all channels, and stops raft.
func (rc *raftNode) stop() {
	rc.stopHTTP()
	close(rc.commitC)
	close(rc.errorC)
	rc.node.Stop()
}

func (rc *raftNode) stopHTTP() {
	rc.transport.Stop()
	close(rc.httpstopc)
	<-rc.httpdonec
}

func (rc *raftNode) serveChannels() {
	defer rc.wal.Close()

	ticker := time.NewTicker(100 * time.Millisecond)
	defer ticker.Stop()

	// send proposals over raft
	go func() {
		var confChangeCount uint64 = 0

		for rc.proposeC != nil && rc.confChangeC != nil {
			select {
			case prop, ok := <-rc.proposeC:
				if !ok {
					rc.proposeC = nil
				} else {
					// blocks until accepted by raft state machine
					rc.node.Propose(context.TODO(), []byte(prop))
				}

			case cc, ok := <-rc.confChangeC:
				if !ok {
					rc.confChangeC = nil
				} else {
					confChangeCount += 1
					cc.ID = confChangeCount
					rc.node.ProposeConfChange(context.TODO(), cc)
				}
			}
		}
		// client closed channel; shutdown raft if not already
		close(rc.stopc)
	}()

	// event loop on raft state machine updates
	for {
		select {
		case <-ticker.C:
			rc.node.Tick()

		// store raft entries to wal, then publish over commit channel
		case rd := <-rc.node.Ready():
			rc.wal.Save(rd.HardState, rd.Entries)
			if !raft.IsEmptySnap(rd.Snapshot) {
				rc.saveSnap(rd.Snapshot)
				rc.raftStorage.ApplySnapshot(rd.Snapshot)
			}
			rc.raftStorage.Append(rd.Entries)
			rc.transport.Send(rd.Messages)
			if ok := rc.publishEntries(rd.CommittedEntries); !ok {
				rc.stop()
				return
			}
			rc.node.Advance()

		case err := <-rc.transport.ErrorC:
			rc.writeError(err)
			return

		case <-rc.stopc:
			rc.stop()
			return
		}
	}
}

func (rc *raftNode) serveRaft() {
	url, err := url.Parse(rc.peers[rc.id-1])
	if err != nil {
		log.Fatalf("raftexample: Failed parsing URL (%v)", err)
	}

	ln, err := newStoppableListener(url.Host, rc.httpstopc)
	if err != nil {
		log.Fatalf("raftexample: Failed to listen rafthttp (%v)", err)
	}

	err = (&http.Server{Handler: rc.transport.Handler()}).Serve(ln)
	select {
	case <-rc.httpstopc:
	default:
		log.Fatalf("raftexample: Failed to serve rafthttp (%v)", err)
	}
	close(rc.httpdonec)
}

func (rc *raftNode) Process(ctx context.Context, m raftpb.Message) error {
	return rc.node.Step(ctx, m)
}
func (rc *raftNode) IsIDRemoved(id uint64) bool                           { return false }
func (rc *raftNode) ReportUnreachable(id uint64)                          {}
func (rc *raftNode) ReportSnapshot(id uint64, status raft.SnapshotStatus) {}
<commit_msg>raftexample: add index fields to filter entries<commit_after>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"log"
	"os"
	"strconv"
	"time"

	"net/http"
	"net/url"

	"github.com/coreos/etcd/etcdserver/stats"
	"github.com/coreos/etcd/pkg/fileutil"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/rafthttp"
	"github.com/coreos/etcd/snap"
	"github.com/coreos/etcd/wal"
	"github.com/coreos/etcd/wal/walpb"
	"golang.org/x/net/context"
)

// A key-value stream backed by raft
type raftNode struct {
	proposeC    <-chan string            // proposed messages (k,v)
	confChangeC <-chan raftpb.ConfChange // proposed cluster config changes
	commitC     chan<- *string           // entries committed to log (k,v)
	errorC      chan<- error             // errors from raft session

	id        int      // client ID for raft session
	peers     []string // raft peer URLs
	join      bool     // node is joining an existing cluster
	waldir    string   // path to WAL directory
	snapdir   string   // path to snapshot directory
	lastIndex uint64   // index of log at start

	confState     raftpb.ConfState
	snapshotIndex uint64
	appliedIndex  uint64

	// raft backing for the commit/error channel
	node        raft.Node
	raftStorage *raft.MemoryStorage
	wal         *wal.WAL
	snapshotter *snap.Snapshotter
	transport   *rafthttp.Transport
	stopc       chan struct{} // signals proposal channel closed
	httpstopc   chan struct{} // signals http server to shutdown
	httpdonec   chan struct{} // signals http server shutdown complete
}

// newRaftNode initiates a raft instance and returns a committed log entry
// channel and error channel. Proposals for log updates are sent over the
// provided the proposal channel. All log entries are replayed over the
// commit channel, followed by a nil message (to indicate the channel is
// current), then new log entries. To shutdown, close proposeC and read errorC.
func newRaftNode(id int, peers []string, join bool, proposeC <-chan string,
	confChangeC <-chan raftpb.ConfChange) (<-chan *string, <-chan error) {

	commitC := make(chan *string)
	errorC := make(chan error)

	rc := &raftNode{
		proposeC:    proposeC,
		confChangeC: confChangeC,
		commitC:     commitC,
		errorC:      errorC,
		id:          id,
		peers:       peers,
		join:        join,
		waldir:      fmt.Sprintf("raftexample-%d", id),
		snapdir:     fmt.Sprintf("raftexample-%d-snap", id),
		raftStorage: raft.NewMemoryStorage(),
		stopc:       make(chan struct{}),
		httpstopc:   make(chan struct{}),
		httpdonec:   make(chan struct{}),
		// rest of structure populated after WAL replay
	}
	go rc.startRaft()
	return commitC, errorC
}

func (rc *raftNode) saveSnap(snap raftpb.Snapshot) error {
	if err := rc.snapshotter.SaveSnap(snap); err != nil {
		return err
	}
	walSnap := walpb.Snapshot{
		Index: snap.Metadata.Index,
		Term:  snap.Metadata.Term,
	}
	if err := rc.wal.SaveSnapshot(walSnap); err != nil {
		return err
	}
	return rc.wal.ReleaseLockTo(snap.Metadata.Index)
}

func (rc *raftNode) entriesToApply(ents []raftpb.Entry) (nents []raftpb.Entry) {
	if len(ents) == 0 {
		return
	}
	firstIdx := ents[0].Index
	if firstIdx > rc.appliedIndex+1 {
		log.Fatalf("first index of committed entry[%d] should <= progress.appliedIndex[%d] 1", firstIdx, rc.appliedIndex)
	}
	if rc.appliedIndex-firstIdx+1 < uint64(len(ents)) {
		nents = ents[rc.appliedIndex-firstIdx+1:]
	}
	return
}

// publishEntries writes committed log entries to commit channel and returns
// whether all entries could be published.
func (rc *raftNode) publishEntries(ents []raftpb.Entry) bool {
	for i := range ents {
		switch ents[i].Type {
		case raftpb.EntryNormal:
			if len(ents[i].Data) == 0 {
				// ignore empty messages
				break
			}
			s := string(ents[i].Data)
			select {
			case rc.commitC <- &s:
			case <-rc.stopc:
				return false
			}

		case raftpb.EntryConfChange:
			var cc raftpb.ConfChange
			cc.Unmarshal(ents[i].Data)
			rc.node.ApplyConfChange(cc)
			switch cc.Type {
			case raftpb.ConfChangeAddNode:
				if len(cc.Context) > 0 {
					rc.transport.AddPeer(types.ID(cc.NodeID), []string{string(cc.Context)})
				}
			case raftpb.ConfChangeRemoveNode:
				if cc.NodeID == uint64(rc.id) {
					log.Println("I've been removed from the cluster! Shutting down.")
					return false
				}
				rc.transport.RemovePeer(types.ID(cc.NodeID))
			}
		}

		// after commit, update appliedIndex
		rc.appliedIndex = ents[i].Index

		// special nil commit to signal replay has finished
		if ents[i].Index == rc.lastIndex {
			select {
			case rc.commitC <- nil:
			case <-rc.stopc:
				return false
			}
		}
	}
	return true
}

// openWAL returns a WAL ready for reading.
func (rc *raftNode) openWAL() *wal.WAL {
	if !wal.Exist(rc.waldir) {
		if err := os.Mkdir(rc.waldir, 0750); err != nil {
			log.Fatalf("raftexample: cannot create dir for wal (%v)", err)
		}

		w, err := wal.Create(rc.waldir, nil)
		if err != nil {
			log.Fatalf("raftexample: create wal error (%v)", err)
		}
		w.Close()
	}

	w, err := wal.Open(rc.waldir, walpb.Snapshot{})
	if err != nil {
		log.Fatalf("raftexample: error loading wal (%v)", err)
	}

	return w
}

// replayWAL replays WAL entries into the raft instance.
func (rc *raftNode) replayWAL() *wal.WAL {
	w := rc.openWAL()
	_, st, ents, err := w.ReadAll()
	if err != nil {
		log.Fatalf("raftexample: failed to read WAL (%v)", err)
	}
	// append to storage so raft starts at the right place in log
	rc.raftStorage.Append(ents)
	// send nil once lastIndex is published so client knows commit channel is current
	if len(ents) > 0 {
		rc.lastIndex = ents[len(ents)-1].Index
	} else {
		rc.commitC <- nil
	}
	rc.raftStorage.SetHardState(st)
	return w
}

func (rc *raftNode) writeError(err error) {
	rc.stopHTTP()
	close(rc.commitC)
	rc.errorC <- err
	close(rc.errorC)
	rc.node.Stop()
}

func (rc *raftNode) startRaft() {
	if !fileutil.Exist(rc.snapdir) {
		if err := os.Mkdir(rc.snapdir, 0750); err != nil {
			log.Fatalf("raftexample: cannot create dir for snapshot (%v)", err)
		}
	}
	oldwal := wal.Exist(rc.waldir)
	rc.wal = rc.replayWAL()
	rc.snapshotter = snap.New(rc.snapdir)

	rpeers := make([]raft.Peer, len(rc.peers))
	for i := range rpeers {
		rpeers[i] = raft.Peer{ID: uint64(i + 1)}
	}
	c := &raft.Config{
		ID:              uint64(rc.id),
		ElectionTick:    10,
		HeartbeatTick:   1,
		Storage:         rc.raftStorage,
		MaxSizePerMsg:   1024 * 1024,
		MaxInflightMsgs: 256,
	}

	if oldwal {
		rc.node = raft.RestartNode(c)
	} else {
		startPeers := rpeers
		if rc.join {
			startPeers = nil
		}
		rc.node = raft.StartNode(c, startPeers)
	}

	ss := &stats.ServerStats{}
	ss.Initialize()

	rc.transport = &rafthttp.Transport{
		ID:          types.ID(rc.id),
		ClusterID:   0x1000,
		Raft:        rc,
		ServerStats: ss,
		LeaderStats: stats.NewLeaderStats(strconv.Itoa(rc.id)),
		ErrorC:      make(chan error),
	}

	rc.transport.Start()
	for i := range rc.peers {
		if i+1 != rc.id {
			rc.transport.AddPeer(types.ID(i+1), []string{rc.peers[i]})
		}
	}

	go rc.serveRaft()
	go rc.serveChannels()
}

// stop closes http, closes all channels, and stops raft.
func (rc *raftNode) stop() {
	rc.stopHTTP()
	close(rc.commitC)
	close(rc.errorC)
	rc.node.Stop()
}

func (rc *raftNode) stopHTTP() {
	rc.transport.Stop()
	close(rc.httpstopc)
	<-rc.httpdonec
}

func (rc *raftNode) serveChannels() {
	snap, err := rc.raftStorage.Snapshot()
	if err != nil {
		panic(err)
	}
	rc.confState = snap.Metadata.ConfState
	rc.snapshotIndex = snap.Metadata.Index
	rc.appliedIndex = snap.Metadata.Index

	defer rc.wal.Close()

	ticker := time.NewTicker(100 * time.Millisecond)
	defer ticker.Stop()

	// send proposals over raft
	go func() {
		var confChangeCount uint64 = 0

		for rc.proposeC != nil && rc.confChangeC != nil {
			select {
			case prop, ok := <-rc.proposeC:
				if !ok {
					rc.proposeC = nil
				} else {
					// blocks until accepted by raft state machine
					rc.node.Propose(context.TODO(), []byte(prop))
				}

			case cc, ok := <-rc.confChangeC:
				if !ok {
					rc.confChangeC = nil
				} else {
					confChangeCount += 1
					cc.ID = confChangeCount
					rc.node.ProposeConfChange(context.TODO(), cc)
				}
			}
		}
		// client closed channel; shutdown raft if not already
		close(rc.stopc)
	}()

	// event loop on raft state machine updates
	for {
		select {
		case <-ticker.C:
			rc.node.Tick()

		// store raft entries to wal, then publish over commit channel
		case rd := <-rc.node.Ready():
			rc.wal.Save(rd.HardState, rd.Entries)
			if !raft.IsEmptySnap(rd.Snapshot) {
				rc.saveSnap(rd.Snapshot)
				rc.raftStorage.ApplySnapshot(rd.Snapshot)
			}
			rc.raftStorage.Append(rd.Entries)
			rc.transport.Send(rd.Messages)
			// TODO: apply snapshot
			if ok := rc.publishEntries(rc.entriesToApply(rd.CommittedEntries)); !ok {
				rc.stop()
				return
			}
			// TODO: trigger snapshot
			rc.node.Advance()

		case err := <-rc.transport.ErrorC:
			rc.writeError(err)
			return

		case <-rc.stopc:
			rc.stop()
			return
		}
	}
}

func (rc *raftNode) serveRaft() {
	url, err := url.Parse(rc.peers[rc.id-1])
	if err != nil {
		log.Fatalf("raftexample: Failed parsing URL (%v)", err)
	}

	ln, err := newStoppableListener(url.Host, rc.httpstopc)
	if err != nil {
		log.Fatalf("raftexample: Failed to listen rafthttp (%v)", err)
	}

	err = (&http.Server{Handler: rc.transport.Handler()}).Serve(ln)
	select {
	case <-rc.httpstopc:
	default:
		log.Fatalf("raftexample: Failed to serve rafthttp (%v)", err)
	}
	close(rc.httpdonec)
}

func (rc *raftNode) Process(ctx context.Context, m raftpb.Message) error {
	return rc.node.Step(ctx, m)
}
func (rc *raftNode) IsIDRemoved(id uint64) bool                           { return false }
func (rc *raftNode) ReportUnreachable(id uint64)                          {}
func (rc *raftNode) ReportSnapshot(id uint64, status raft.SnapshotStatus) {}
<|endoftext|>


<commit_before>package gomposer

import (
    "os"

    "encoding/json"
)

type Reader interface {
	Read(filename string) *Version
}

type PackageReader struct {
}

func (pr PackageReader) Read(filename string) (*Version, error) {

    buf, err := os.Open(filename)

    if err != nil {
        return nil, err
    }

    output := &Version{}

    json.NewDecoder(buf).Decode(output)

	return output, nil
}
<commit_msg>Refactor dependency resolver + add test for later on<commit_after>package gomposer

import (
	"os"

	"encoding/json"
)

type Reader interface {
	Read(filename string) *Version
}

type PackageReader struct {
}

func (pr PackageReader) Read(filename string) (*Version, error) {

	buf, err := os.Open(filename)

	if err != nil {
		return nil, err
	}

	output := &Version{}

	json.NewDecoder(buf).Decode(output)

	return output, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"regexp"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
)

var (
	dataFile = flag.String("file", "store.json", "data store file name")
	status   = flag.String("status", "stopped|pending|terminated", "the status you would like to poll")
)

var store *StatusStore

type Conn struct {
	aw2 *ec2.EC2
}

// create a struct to map toml config file
// TODO: add this as an switch in init()
type AwsConfig struct {
	AwsSecretKey string `toml:"AWS_ACCESS_KEY_ID"`
	AwsAccessKey string `toml:"AWS_SECRET_ACCESS_KEY"`
	Region       string `toml:"AWS_REGION"`
}

func recieveStatus(dataMap map[string]string) <-chan string {
	c := make(chan string)
	go func() {
		for _, v := range dataMap {
			c <- fmt.Sprintf("%s", v)
			//			time.Sleep(time.Duration(rand.Intn(1e3)) * time.Millisecond)
		}
	}()
	return c

}

func (c *Conn) iterateResToMap(resp *ec2.DescribeInstancesOutput) map[string]string {
	insMap := make(map[string]string)
	for idx, _ := range resp.Reservations {
		for _, inst := range resp.Reservations[idx].Instances {
			// fmt.Printf("   Instance State: %v InstanceID: %v \n", *inst.State.Name, *inst.InstanceID)
			// dereference pointer
			var id, state string
			id = *inst.PrivateDNSName
			state = *inst.State.Name
			insMap[id] = state
		}
	}
	return insMap
}

func (c *Conn) GetEc2Data() map[string]string {

	resp, err := c.aw2.DescribeInstances(nil)
	if err != nil {
		log.Fatal(err)
	}

	newMap := c.iterateResToMap(resp)
	return newMap
}

func (c *Conn) startLoop(status string) bool {
	// Call the DescribeInstances Operation
	resp, err := c.aw2.DescribeInstances(nil)
	if err != nil {
		log.Fatal(err)
	}
	// re-format to method call
	newMap := c.iterateResToMap(resp)

	r := recieveStatus(newMap)

	// set a timeout for the channel
	timeout := time.After(5 * time.Second)

	// begin channel operations
	// TODO: this should have interfaces and structs and a poller
	for {
		select {
		case result := <-r:
			matched, _ := regexp.MatchString(status, result)
			if matched {
				res := strings.Split(result, ":")
				status, ip := res[0], res[1]
				fmt.Printf("Status:  %v PrivateIP:  %v  \n", status, ip)
			}
		case <-timeout:
			return false
		}
	}

}

// TODO: right now this pulls creds from env but has the
// capability of using toml the struct is here just need
// to check on if we are using env or fallback to toml
func NewEc2() *Conn {
	c := new(Conn)
	c.aw2 = ec2.New(&aws.Config{Region: "us-west-2"})

	return c
}

func (d *StatusStore) AddDataToFile(status string) {

	for k, v := range d.status {

		if v == status {
			if d.Get(k) == status {
				fmt.Println("this is already set")
			} else {
				err := d.save(k, v)
				if err != nil {
					log.Printf("some shit went wrong save %s", k)
				}
			}
		}
	}
}

func main() {
	// Create an EC2 service object in the "us-west-2" region
	// Note that you can also configure your region globally by
	// exporting the AWS_REGION environment variable
	flag.Parse()

	// instantiate new ec2 "object"
	c := NewEc2()

	// Get new Status store
	d := NewStatusStore(*dataFile)

	// Get a data set to work with
	dataSet := c.GetEc2Data()

	// set the status map
	d.status = dataSet

	// lets save some data
	d.AddDataToFile(*status)

}

func Add(value string) {
	key := store.Put(value)
	fmt.Println(key)
}
<commit_msg>{WIP} taking curses out of debug lines<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"regexp"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
)

var (
	dataFile = flag.String("file", "store.json", "data store file name")
	status   = flag.String("status", "stopped|pending|terminated", "the status you would like to poll")
)

var store *StatusStore

type Conn struct {
	aw2 *ec2.EC2
}

// create a struct to map toml config file
// TODO: add this as an switch in init()
type AwsConfig struct {
	AwsSecretKey string `toml:"AWS_ACCESS_KEY_ID"`
	AwsAccessKey string `toml:"AWS_SECRET_ACCESS_KEY"`
	Region       string `toml:"AWS_REGION"`
}

func recieveStatus(dataMap map[string]string) <-chan string {
	c := make(chan string)
	go func() {
		for _, v := range dataMap {
			c <- fmt.Sprintf("%s", v)
			//			time.Sleep(time.Duration(rand.Intn(1e3)) * time.Millisecond)
		}
	}()
	return c

}

func (c *Conn) iterateResToMap(resp *ec2.DescribeInstancesOutput) map[string]string {
	insMap := make(map[string]string)
	for idx, _ := range resp.Reservations {
		for _, inst := range resp.Reservations[idx].Instances {
			// fmt.Printf("   Instance State: %v InstanceID: %v \n", *inst.State.Name, *inst.InstanceID)
			// dereference pointer
			var id, state string
			id = *inst.PrivateDNSName
			state = *inst.State.Name
			insMap[id] = state
		}
	}
	return insMap
}

func (c *Conn) GetEc2Data() map[string]string {

	resp, err := c.aw2.DescribeInstances(nil)
	if err != nil {
		log.Fatal(err)
	}

	newMap := c.iterateResToMap(resp)
	return newMap
}

func (c *Conn) startLoop(status string) bool {
	// Call the DescribeInstances Operation
	resp, err := c.aw2.DescribeInstances(nil)
	if err != nil {
		log.Fatal(err)
	}
	// re-format to method call
	newMap := c.iterateResToMap(resp)

	r := recieveStatus(newMap)

	// set a timeout for the channel
	timeout := time.After(5 * time.Second)

	// begin channel operations
	// TODO: this should have interfaces and structs and a poller
	for {
		select {
		case result := <-r:
			matched, _ := regexp.MatchString(status, result)
			if matched {
				res := strings.Split(result, ":")
				status, ip := res[0], res[1]
				fmt.Printf("Status:  %v PrivateIP:  %v  \n", status, ip)
			}
		case <-timeout:
			return false
		}
	}

}

// TODO: right now this pulls creds from env but has the
// capability of using toml the struct is here just need
// to check on if we are using env or fallback to toml
func NewEc2() *Conn {
	c := new(Conn)
	c.aw2 = ec2.New(&aws.Config{Region: "us-west-2"})

	return c
}

func (d *StatusStore) AddDataToFile(status string) {

	for k, v := range d.status {

		if v == status {
			if d.Get(k) == status {
				fmt.Println("this is already set")
			} else {
				err := d.save(k, v)
				if err != nil {
					log.Printf("something went wrong save %s", k)
				}
			}
		}
	}
}

func main() {
	// Create an EC2 service object in the "us-west-2" region
	// Note that you can also configure your region globally by
	// exporting the AWS_REGION environment variable
	flag.Parse()

	// instantiate new ec2 "object"
	c := NewEc2()

	// Get new Status store
	d := NewStatusStore(*dataFile)

	// Get a data set to work with
	dataSet := c.GetEc2Data()

	// set the status map
	d.status = dataSet

	// lets save some data
	d.AddDataToFile(*status)

}

func Add(value string) {
	key := store.Put(value)
	fmt.Println(key)
}
<|endoftext|>