<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package awstasks

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/golang/glog"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
	"k8s.io/kops/upup/pkg/fi/cloudup/cloudformation"
	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
)

//go:generate fitask -type=EBSVolume
type EBSVolume struct {
	Name      *string
	Lifecycle *fi.Lifecycle

	ID               *string
	AvailabilityZone *string
	VolumeType       *string
	SizeGB           *int64
	VolumeIops       *int64
	KmsKeyId         *string
	Encrypted        *bool
	Tags             map[string]string
}

var _ fi.CompareWithID = &EBSVolume{}

func (e *EBSVolume) CompareWithID() *string {
	return e.ID
}

type TaggableResource interface {
	FindResourceID(c fi.Cloud) (*string, error)
}

var _ TaggableResource = &EBSVolume{}

func (e *EBSVolume) FindResourceID(c fi.Cloud) (*string, error) {
	actual, err := e.find(c.(awsup.AWSCloud))
	if err != nil {
		return nil, fmt.Errorf("error querying for EBSVolume: %v", err)
	}
	if actual == nil {
		return nil, nil
	}
	return actual.ID, nil
}

func (e *EBSVolume) Find(context *fi.Context) (*EBSVolume, error) {
	actual, err := e.find(context.Cloud.(awsup.AWSCloud))
	if actual != nil && err == nil {
		e.ID = actual.ID
	}
	return actual, err
}

func (e *EBSVolume) find(cloud awsup.AWSCloud) (*EBSVolume, error) {
	filters := cloud.BuildFilters(e.Name)
	request := &ec2.DescribeVolumesInput{
		Filters: filters,
	}

	response, err := cloud.EC2().DescribeVolumes(request)
	if err != nil {
		return nil, fmt.Errorf("error listing volumes: %v", err)
	}

	if response == nil || len(response.Volumes) == 0 {
		return nil, nil
	}

	if len(response.Volumes) != 1 {
		return nil, fmt.Errorf("found multiple Volumes with name: %s", *e.Name)
	}
	glog.V(2).Info("found existing volume")
	v := response.Volumes[0]
	actual := &EBSVolume{
		ID:               v.VolumeId,
		AvailabilityZone: v.AvailabilityZone,
		VolumeType:       v.VolumeType,
		SizeGB:           v.Size,
		KmsKeyId:         v.KmsKeyId,
		Encrypted:        v.Encrypted,
		Name:             e.Name,
		VolumeIops:       v.Iops,
	}

	actual.Tags = mapEC2TagsToMap(v.Tags)

	// Avoid spurious changes
	actual.Lifecycle = e.Lifecycle

	return actual, nil
}

func (e *EBSVolume) Run(c *fi.Context) error {
	c.Cloud.(awsup.AWSCloud).AddTags(e.Name, e.Tags)
	return fi.DefaultDeltaRunMethod(e, c)
}

func (_ *EBSVolume) CheckChanges(a, e, changes *EBSVolume) error {
	if a == nil {
		if e.Name == nil {
			return fi.RequiredField("Name")
		}
	}
	if a != nil {
		if changes.ID != nil {
			return fi.CannotChangeField("ID")
		}
	}
	return nil
}

func (_ *EBSVolume) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *EBSVolume) error {
	if a == nil {
		glog.V(2).Infof("Creating PersistentVolume with Name:%q", *e.Name)

		request := &ec2.CreateVolumeInput{
			Size:             e.SizeGB,
			AvailabilityZone: e.AvailabilityZone,
			VolumeType:       e.VolumeType,
			KmsKeyId:         e.KmsKeyId,
			Encrypted:        e.Encrypted,
			Iops:             e.VolumeIops,
		}

		if len(e.Tags) != 0 {
			request.TagSpecifications = []*ec2.TagSpecification{
				{ResourceType: aws.String(ec2.ResourceTypeVolume)},
			}

			for k, v := range e.Tags {
				request.TagSpecifications[0].Tags = append(request.TagSpecifications[0].Tags, &ec2.Tag{
					Key:   aws.String(k),
					Value: aws.String(v),
				})
			}
		}

		response, err := t.Cloud.EC2().CreateVolume(request)
		if err != nil {
			return fmt.Errorf("error creating PersistentVolume: %v", err)
		}

		e.ID = response.VolumeId
	}

	if err := t.AddAWSTags(*e.ID, e.Tags); err != nil {
		return fmt.Errorf("error adding AWS Tags to EBS Volume: %v", err)
	}

	if a != nil && len(a.Tags) > 0 {
		tagsToDelete := e.getEBSVolumeTagsToDelete(a.Tags)
		if len(tagsToDelete) > 0 {
			return t.DeleteTags(*e.ID, tagsToDelete)
		}
	}

	return nil
}

// getEBSVolumeTagsToDelete loops through the currently set tags and builds
// a list of tags to be deleted from the EBS Volume
func (e *EBSVolume) getEBSVolumeTagsToDelete(currentTags map[string]string) map[string]string {
	tagsToDelete := map[string]string{}
	for k, v := range currentTags {
		if _, ok := e.Tags[k]; !ok {
			tagsToDelete[k] = v
		}
	}

	return tagsToDelete
}

type terraformVolume struct {
	AvailabilityZone *string           `json:"availability_zone,omitempty"`
	Size             *int64            `json:"size,omitempty"`
	Type             *string           `json:"type,omitempty"`
	KmsKeyId         *string           `json:"kms_key_id,omitempty"`
	Encrypted        *bool             `json:"encrypted,omitempty"`
	Tags             map[string]string `json:"tags,omitempty"`
}

func (_ *EBSVolume) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *EBSVolume) error {
	tf := &terraformVolume{
		AvailabilityZone: e.AvailabilityZone,
		Size:             e.SizeGB,
		Type:             e.VolumeType,
		KmsKeyId:         e.KmsKeyId,
		Encrypted:        e.Encrypted,
		Tags:             e.Tags,
	}

	return t.RenderResource("aws_ebs_volume", *e.Name, tf)
}

func (e *EBSVolume) TerraformLink() *terraform.Literal {
	return terraform.LiteralSelfLink("aws_ebs_volume", *e.Name)
}

type cloudformationVolume struct {
	AvailabilityZone *string             `json:"AvailabilityZone,omitempty"`
	Size             *int64              `json:"Size,omitempty"`
	Type             *string             `json:"VolumeType,omitempty"`
	KmsKeyId         *string             `json:"KmsKeyId,omitempty"`
	Encrypted        *bool               `json:"Encrypted,omitempty"`
	Tags             []cloudformationTag `json:"Tags,omitempty"`
}

func (_ *EBSVolume) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *EBSVolume) error {
	cf := &cloudformationVolume{
		AvailabilityZone: e.AvailabilityZone,
		Size:             e.SizeGB,
		Type:             e.VolumeType,
		KmsKeyId:         e.KmsKeyId,
		Encrypted:        e.Encrypted,
		Tags:             buildCloudformationTags(e.Tags),
	}

	return t.RenderResource("AWS::EC2::Volume", *e.Name, cf)
}

func (e *EBSVolume) CloudformationLink() *cloudformation.Literal {
	return cloudformation.Ref("AWS::EC2::Volume", *e.Name)
}
<commit_msg>add output support to terraform & cloudformation for iops volume<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package awstasks

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/golang/glog"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
	"k8s.io/kops/upup/pkg/fi/cloudup/cloudformation"
	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
)

//go:generate fitask -type=EBSVolume
type EBSVolume struct {
	Name      *string
	Lifecycle *fi.Lifecycle

	ID               *string
	AvailabilityZone *string
	VolumeType       *string
	SizeGB           *int64
	VolumeIops       *int64
	KmsKeyId         *string
	Encrypted        *bool
	Tags             map[string]string
}

var _ fi.CompareWithID = &EBSVolume{}

func (e *EBSVolume) CompareWithID() *string {
	return e.ID
}

type TaggableResource interface {
	FindResourceID(c fi.Cloud) (*string, error)
}

var _ TaggableResource = &EBSVolume{}

func (e *EBSVolume) FindResourceID(c fi.Cloud) (*string, error) {
	actual, err := e.find(c.(awsup.AWSCloud))
	if err != nil {
		return nil, fmt.Errorf("error querying for EBSVolume: %v", err)
	}
	if actual == nil {
		return nil, nil
	}
	return actual.ID, nil
}

func (e *EBSVolume) Find(context *fi.Context) (*EBSVolume, error) {
	actual, err := e.find(context.Cloud.(awsup.AWSCloud))
	if actual != nil && err == nil {
		e.ID = actual.ID
	}
	return actual, err
}

func (e *EBSVolume) find(cloud awsup.AWSCloud) (*EBSVolume, error) {
	filters := cloud.BuildFilters(e.Name)
	request := &ec2.DescribeVolumesInput{
		Filters: filters,
	}

	response, err := cloud.EC2().DescribeVolumes(request)
	if err != nil {
		return nil, fmt.Errorf("error listing volumes: %v", err)
	}

	if response == nil || len(response.Volumes) == 0 {
		return nil, nil
	}

	if len(response.Volumes) != 1 {
		return nil, fmt.Errorf("found multiple Volumes with name: %s", *e.Name)
	}
	glog.V(2).Info("found existing volume")
	v := response.Volumes[0]
	actual := &EBSVolume{
		ID:               v.VolumeId,
		AvailabilityZone: v.AvailabilityZone,
		VolumeType:       v.VolumeType,
		SizeGB:           v.Size,
		KmsKeyId:         v.KmsKeyId,
		Encrypted:        v.Encrypted,
		Name:             e.Name,
		VolumeIops:       v.Iops,
	}

	actual.Tags = mapEC2TagsToMap(v.Tags)

	// Avoid spurious changes
	actual.Lifecycle = e.Lifecycle

	return actual, nil
}

func (e *EBSVolume) Run(c *fi.Context) error {
	c.Cloud.(awsup.AWSCloud).AddTags(e.Name, e.Tags)
	return fi.DefaultDeltaRunMethod(e, c)
}

func (_ *EBSVolume) CheckChanges(a, e, changes *EBSVolume) error {
	if a == nil {
		if e.Name == nil {
			return fi.RequiredField("Name")
		}
	}
	if a != nil {
		if changes.ID != nil {
			return fi.CannotChangeField("ID")
		}
	}
	return nil
}

func (_ *EBSVolume) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *EBSVolume) error {
	if a == nil {
		glog.V(2).Infof("Creating PersistentVolume with Name:%q", *e.Name)

		request := &ec2.CreateVolumeInput{
			Size:             e.SizeGB,
			AvailabilityZone: e.AvailabilityZone,
			VolumeType:       e.VolumeType,
			KmsKeyId:         e.KmsKeyId,
			Encrypted:        e.Encrypted,
			Iops:             e.VolumeIops,
		}

		if len(e.Tags) != 0 {
			request.TagSpecifications = []*ec2.TagSpecification{
				{ResourceType: aws.String(ec2.ResourceTypeVolume)},
			}

			for k, v := range e.Tags {
				request.TagSpecifications[0].Tags = append(request.TagSpecifications[0].Tags, &ec2.Tag{
					Key:   aws.String(k),
					Value: aws.String(v),
				})
			}
		}

		response, err := t.Cloud.EC2().CreateVolume(request)
		if err != nil {
			return fmt.Errorf("error creating PersistentVolume: %v", err)
		}

		e.ID = response.VolumeId
	}

	if err := t.AddAWSTags(*e.ID, e.Tags); err != nil {
		return fmt.Errorf("error adding AWS Tags to EBS Volume: %v", err)
	}

	if a != nil && len(a.Tags) > 0 {
		tagsToDelete := e.getEBSVolumeTagsToDelete(a.Tags)
		if len(tagsToDelete) > 0 {
			return t.DeleteTags(*e.ID, tagsToDelete)
		}
	}

	return nil
}

// getEBSVolumeTagsToDelete loops through the currently set tags and builds
// a list of tags to be deleted from the EBS Volume
func (e *EBSVolume) getEBSVolumeTagsToDelete(currentTags map[string]string) map[string]string {
	tagsToDelete := map[string]string{}
	for k, v := range currentTags {
		if _, ok := e.Tags[k]; !ok {
			tagsToDelete[k] = v
		}
	}

	return tagsToDelete
}

type terraformVolume struct {
	AvailabilityZone *string           `json:"availability_zone,omitempty"`
	Size             *int64            `json:"size,omitempty"`
	Type             *string           `json:"type,omitempty"`
	Iops             *int64            `json:"iops,omitempty"`
	KmsKeyId         *string           `json:"kms_key_id,omitempty"`
	Encrypted        *bool             `json:"encrypted,omitempty"`
	Tags             map[string]string `json:"tags,omitempty"`
}

func (_ *EBSVolume) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *EBSVolume) error {
	tf := &terraformVolume{
		AvailabilityZone: e.AvailabilityZone,
		Size:             e.SizeGB,
		Type:             e.VolumeType,
		Iops:             e.VolumeIops,
		KmsKeyId:         e.KmsKeyId,
		Encrypted:        e.Encrypted,
		Tags:             e.Tags,
	}

	return t.RenderResource("aws_ebs_volume", *e.Name, tf)
}

func (e *EBSVolume) TerraformLink() *terraform.Literal {
	return terraform.LiteralSelfLink("aws_ebs_volume", *e.Name)
}

type cloudformationVolume struct {
	AvailabilityZone *string             `json:"AvailabilityZone,omitempty"`
	Size             *int64              `json:"Size,omitempty"`
	Type             *string             `json:"VolumeType,omitempty"`
	Iops             *int64              `json:"Iops,omitempty"`
	KmsKeyId         *string             `json:"KmsKeyId,omitempty"`
	Encrypted        *bool               `json:"Encrypted,omitempty"`
	Tags             []cloudformationTag `json:"Tags,omitempty"`
}

func (_ *EBSVolume) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *EBSVolume) error {
	cf := &cloudformationVolume{
		AvailabilityZone: e.AvailabilityZone,
		Size:             e.SizeGB,
		Type:             e.VolumeType,
		Iops:             e.VolumeIops,
		KmsKeyId:         e.KmsKeyId,
		Encrypted:        e.Encrypted,
		Tags:             buildCloudformationTags(e.Tags),
	}

	return t.RenderResource("AWS::EC2::Volume", *e.Name, cf)
}

func (e *EBSVolume) CloudformationLink() *cloudformation.Literal {
	return cloudformation.Ref("AWS::EC2::Volume", *e.Name)
}
<|endoftext|>


<commit_before>package picarus
import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"bytes"
	"mime/multipart"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"
)

type Conn struct {
	Email string
	ApiKey string
	LoginKey string
	Server string
}

func (conn *Conn) call(method string, path []string, params url.Values, files map[string][]byte, key string) ([]byte, error) {
	var err error
	var req *http.Request
	serverPrefix := conn.Server + "/" + strings.Join(path, "/")
	if method == "GET" {
		req, err = http.NewRequest(method, serverPrefix + "?" + params.Encode(), nil)
	} else {
		buf := new(bytes.Buffer)
		w := multipart.NewWriter(buf)
		for fileName, fileData := range files {
			_, err = w.CreateFormFile(fileName, fileName) // fw
			if err != nil {
				return nil, err
			}
			buf.Write([]byte(fileData))
		}
		for name, values := range params {
			for _, value := range values {
				err = w.WriteField(name, value)
				if err != nil {
					return nil, err
				}
			}
		}
		w.Close()
		req, err = http.NewRequest(method, serverPrefix, buf)
		if err != nil {
			return nil, err
		}
		req.Header.Set("Content-Type", w.FormDataContentType())
	}
	if err != nil {
		return nil, err
	}
	req.SetBasicAuth(conn.Email, key)
	response, err := http.DefaultClient.Do(req) // res
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	return ioutil.ReadAll(response.Body)
}

func decodeLod(values []map[string]string) ([]map[string]string, error) {
	out := []map[string]string{}
	for _, value := range values {
		curOut, err := decodeDict(value)
		if err != nil {
			return nil, err
		}
		out = append(out, curOut)
	}
	return out, nil
}

func decodeDict(value map[string]string) (map[string]string, error) {
	out := map[string]string{}
	for k, v := range value {
		// TODO: Use a checked version of b64dec
		if k == "row" {
			out[k] = B64Dec(v)
		} else {
			out[B64Dec(k)] = B64Dec(v)
		}
	}
	return out, nil
}

func decodeValues(value map[string]string) (map[string]string, error) {
	out := map[string]string{}
	for k, v := range value {
		// TODO: Use a checked version of b64dec
		out[k] = B64Dec(v)
	}
	return out, nil
}

func encodeDict(value map[string]string) map[string]string {
	out := map[string]string{}
	for k, v := range value {
		out[B64Enc(k)] = B64Enc(v)
	}
	return out
}

func encodeValues(value map[string]string) map[string]string {
	out := map[string]string{}
	for k, v := range value {
		out[k] = B64Enc(v)
	}
	return out
}

func encodeFiles(files map[string][]byte) map[string][]byte {
	out := map[string][]byte{}
	for k, v := range files {
		out[B64Enc(k)] = v
	}
	return out
}

func (conn *Conn) GetRow(table string, row string, columns []string) (map[string]string, error) {
	params := url.Values{}
	if len(columns) > 0 {
		params.Set("columns", encodeColumns(columns))
	}
	data, err := conn.call("GET", []string{"v0", "data", table, UB64Enc(row)}, params, map[string][]byte{}, conn.ApiKey)
	if err != nil {
		return nil, err
	}
	dataParsed := map[string]string{}
	json.Unmarshal(data, &dataParsed)
	return decodeDict(dataParsed)
}

func (conn *Conn) GetTable(table string, columns []string) ([]map[string]string, error) {
	params := url.Values{}
	if len(columns) > 0 {
		params.Set("columns", encodeColumns(columns))
	}
	data, err := conn.call("GET", []string{"v0", "data", table}, params, map[string][]byte{}, conn.ApiKey)
	if err != nil {
		return nil, err
	}
	dataParsed := []map[string]string{}
	json.Unmarshal(data, &dataParsed)
	return decodeLod(dataParsed)
}

func encodeColumns(columns []string) string {
	columnsB64 := []string{}
	for _, c := range columns {
		columnsB64 = append(columnsB64, B64Enc(c))
	}
	return strings.Join(columnsB64, ",")
}

func (conn *Conn) GetSlice(table string, startRow string, stopRow string, columns []string) ([]map[string]string, error) {
	params := url.Values{}
	if len(columns) > 0 {
		params.Set("columns", encodeColumns(columns))
	}
	data, err := conn.call("GET", []string{"v0", "slice", table, UB64Enc(startRow), UB64Enc(stopRow)}, params, map[string][]byte{}, conn.ApiKey)
	if err != nil {
		return nil, err
	}
	dataParsed := []map[string]string{}
	json.Unmarshal(data, &dataParsed)
	return decodeLod(dataParsed)
}

func mapToUrlValues(data map[string]string) url.Values {
	out := url.Values{}
	for k, v := range data {
		out.Set(k, v)
	}
	return out
}

func (conn *Conn) PostSlice(table string, startRow string, stopRow string, params map[string]string) (map[string]string, error) {
	data, err := conn.call("POST", []string{"v0", "slice", table, UB64Enc(startRow), UB64Enc(stopRow)}, mapToUrlValues(encodeValues(params)), map[string][]byte{}, conn.ApiKey)
	if err != nil {
		return nil, err
	}
	dataParsed := map[string]string{}
	json.Unmarshal(data, &dataParsed)
	return decodeDict(dataParsed)
}

type Slice struct {
    StartRow, StopRow string
}

func encodeSlices(slices []Slice) string {
	out := []string{}
	for _, v := range slices {
		out = append(out, B64Enc(v.StartRow) + "," + B64Enc(v.StopRow))
	}
	return strings.Join(out, ";")
}

func (conn *Conn) PostTable(table string, params map[string]string, files map[string][]byte, slices []Slice) (map[string]string, error) {
	if len(slices) > 0 {
		params["slices"] = encodeSlices(slices)
	}
	data, err := conn.call("POST", []string{"v0", "data", table}, mapToUrlValues(encodeDict(params)), encodeFiles(files), conn.ApiKey)
	if err != nil {
		return nil, err
	}
	dataParsed := map[string]string{}
	json.Unmarshal(data, &dataParsed)
	return decodeValues(dataParsed)
}

func (conn *Conn) PostRow(table string, row string, params map[string]string) (map[string]string, error) {
	data, err := conn.call("POST", []string{"v0", "data", table, UB64Enc(row)}, mapToUrlValues(encodeValues(params)), encodeFiles(files), conn.ApiKey)
	if err != nil {
		return nil, err
	}
	dataParsed := map[string]string{}
	json.Unmarshal(data, &dataParsed)
	return decodeDict(dataParsed)
}

func (conn *Conn) PatchRow(table string, row string, params map[string]string, files map[string][]byte) (map[string]string, error) {
	data, err := conn.call("PATCH", []string{"v0", "data", table, UB64Enc(row)}, mapToUrlValues(encodeDict(params)), encodeFiles(files), conn.ApiKey)
	if err != nil {
		return nil, err
	}
	dataParsed := map[string]string{}
	json.Unmarshal(data, &dataParsed)
	return decodeDict(dataParsed)
}

func UB64Dec(s string) string {
	decoded, err := base64.URLEncoding.DecodeString(s)
	if err != nil {
		panic(err)
	}
	return string(decoded)
}

func B64Dec(s string) string {
	decoded, err := base64.StdEncoding.DecodeString(s)
	if err != nil {
		fmt.Println(s)
		panic(err)
	}
	return string(decoded)
}

func B64DecBytes(s string) []byte {
	decoded, err := base64.StdEncoding.DecodeString(s)
	if err != nil {
		fmt.Println(s)
		panic(err)
	}
	return decoded
}

func UB64Enc(s string) string {
	return base64.URLEncoding.EncodeToString([]byte(s))
}

func B64Enc(s string) string {
	return base64.StdEncoding.EncodeToString([]byte(s))
}

func (conn *Conn) WatchJob(row string) error {
	for {
		data, err := conn.GetRow("jobs", row, []string{})
		if err != nil {
			return err
		}
		fmt.Println(data)
		if data["status"] == "completed" {
			return nil
		}
		time.Sleep(1000 * time.Millisecond)
	}
}<commit_msg>Fixed typo<commit_after>package picarus
import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"bytes"
	"mime/multipart"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"
)

type Conn struct {
	Email string
	ApiKey string
	LoginKey string
	Server string
}

func (conn *Conn) call(method string, path []string, params url.Values, files map[string][]byte, key string) ([]byte, error) {
	var err error
	var req *http.Request
	serverPrefix := conn.Server + "/" + strings.Join(path, "/")
	if method == "GET" {
		req, err = http.NewRequest(method, serverPrefix + "?" + params.Encode(), nil)
	} else {
		buf := new(bytes.Buffer)
		w := multipart.NewWriter(buf)
		for fileName, fileData := range files {
			_, err = w.CreateFormFile(fileName, fileName) // fw
			if err != nil {
				return nil, err
			}
			buf.Write([]byte(fileData))
		}
		for name, values := range params {
			for _, value := range values {
				err = w.WriteField(name, value)
				if err != nil {
					return nil, err
				}
			}
		}
		w.Close()
		req, err = http.NewRequest(method, serverPrefix, buf)
		if err != nil {
			return nil, err
		}
		req.Header.Set("Content-Type", w.FormDataContentType())
	}
	if err != nil {
		return nil, err
	}
	req.SetBasicAuth(conn.Email, key)
	response, err := http.DefaultClient.Do(req) // res
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	return ioutil.ReadAll(response.Body)
}

func decodeLod(values []map[string]string) ([]map[string]string, error) {
	out := []map[string]string{}
	for _, value := range values {
		curOut, err := decodeDict(value)
		if err != nil {
			return nil, err
		}
		out = append(out, curOut)
	}
	return out, nil
}

func decodeDict(value map[string]string) (map[string]string, error) {
	out := map[string]string{}
	for k, v := range value {
		// TODO: Use a checked version of b64dec
		if k == "row" {
			out[k] = B64Dec(v)
		} else {
			out[B64Dec(k)] = B64Dec(v)
		}
	}
	return out, nil
}

func decodeValues(value map[string]string) (map[string]string, error) {
	out := map[string]string{}
	for k, v := range value {
		// TODO: Use a checked version of b64dec
		out[k] = B64Dec(v)
	}
	return out, nil
}

func encodeDict(value map[string]string) map[string]string {
	out := map[string]string{}
	for k, v := range value {
		out[B64Enc(k)] = B64Enc(v)
	}
	return out
}

func encodeValues(value map[string]string) map[string]string {
	out := map[string]string{}
	for k, v := range value {
		out[k] = B64Enc(v)
	}
	return out
}

func encodeFiles(files map[string][]byte) map[string][]byte {
	out := map[string][]byte{}
	for k, v := range files {
		out[B64Enc(k)] = v
	}
	return out
}

func (conn *Conn) GetRow(table string, row string, columns []string) (map[string]string, error) {
	params := url.Values{}
	if len(columns) > 0 {
		params.Set("columns", encodeColumns(columns))
	}
	data, err := conn.call("GET", []string{"v0", "data", table, UB64Enc(row)}, params, map[string][]byte{}, conn.ApiKey)
	if err != nil {
		return nil, err
	}
	dataParsed := map[string]string{}
	json.Unmarshal(data, &dataParsed)
	return decodeDict(dataParsed)
}

func (conn *Conn) GetTable(table string, columns []string) ([]map[string]string, error) {
	params := url.Values{}
	if len(columns) > 0 {
		params.Set("columns", encodeColumns(columns))
	}
	data, err := conn.call("GET", []string{"v0", "data", table}, params, map[string][]byte{}, conn.ApiKey)
	if err != nil {
		return nil, err
	}
	dataParsed := []map[string]string{}
	json.Unmarshal(data, &dataParsed)
	return decodeLod(dataParsed)
}

func encodeColumns(columns []string) string {
	columnsB64 := []string{}
	for _, c := range columns {
		columnsB64 = append(columnsB64, B64Enc(c))
	}
	return strings.Join(columnsB64, ",")
}

func (conn *Conn) GetSlice(table string, startRow string, stopRow string, columns []string) ([]map[string]string, error) {
	params := url.Values{}
	if len(columns) > 0 {
		params.Set("columns", encodeColumns(columns))
	}
	data, err := conn.call("GET", []string{"v0", "slice", table, UB64Enc(startRow), UB64Enc(stopRow)}, params, map[string][]byte{}, conn.ApiKey)
	if err != nil {
		return nil, err
	}
	dataParsed := []map[string]string{}
	json.Unmarshal(data, &dataParsed)
	return decodeLod(dataParsed)
}

func mapToUrlValues(data map[string]string) url.Values {
	out := url.Values{}
	for k, v := range data {
		out.Set(k, v)
	}
	return out
}

func (conn *Conn) PostSlice(table string, startRow string, stopRow string, params map[string]string) (map[string]string, error) {
	data, err := conn.call("POST", []string{"v0", "slice", table, UB64Enc(startRow), UB64Enc(stopRow)}, mapToUrlValues(encodeValues(params)), map[string][]byte{}, conn.ApiKey)
	if err != nil {
		return nil, err
	}
	dataParsed := map[string]string{}
	json.Unmarshal(data, &dataParsed)
	return decodeDict(dataParsed)
}

type Slice struct {
    StartRow, StopRow string
}

func encodeSlices(slices []Slice) string {
	out := []string{}
	for _, v := range slices {
		out = append(out, B64Enc(v.StartRow) + "," + B64Enc(v.StopRow))
	}
	return strings.Join(out, ";")
}

func (conn *Conn) PostTable(table string, params map[string]string, files map[string][]byte, slices []Slice) (map[string]string, error) {
	if len(slices) > 0 {
		params["slices"] = encodeSlices(slices)
	}
	data, err := conn.call("POST", []string{"v0", "data", table}, mapToUrlValues(encodeDict(params)), encodeFiles(files), conn.ApiKey)
	if err != nil {
		return nil, err
	}
	dataParsed := map[string]string{}
	json.Unmarshal(data, &dataParsed)
	return decodeValues(dataParsed)
}

func (conn *Conn) PostRow(table string, row string, params map[string]string) (map[string]string, error) {
	data, err := conn.call("POST", []string{"v0", "data", table, UB64Enc(row)}, mapToUrlValues(encodeValues(params)), map[string][]byte{}, conn.ApiKey)
	if err != nil {
		return nil, err
	}
	dataParsed := map[string]string{}
	json.Unmarshal(data, &dataParsed)
	return decodeDict(dataParsed)
}

func (conn *Conn) PatchRow(table string, row string, params map[string]string, files map[string][]byte) (map[string]string, error) {
	data, err := conn.call("PATCH", []string{"v0", "data", table, UB64Enc(row)}, mapToUrlValues(encodeDict(params)), encodeFiles(files), conn.ApiKey)
	if err != nil {
		return nil, err
	}
	dataParsed := map[string]string{}
	json.Unmarshal(data, &dataParsed)
	return decodeDict(dataParsed)
}

func UB64Dec(s string) string {
	decoded, err := base64.URLEncoding.DecodeString(s)
	if err != nil {
		panic(err)
	}
	return string(decoded)
}

func B64Dec(s string) string {
	decoded, err := base64.StdEncoding.DecodeString(s)
	if err != nil {
		fmt.Println(s)
		panic(err)
	}
	return string(decoded)
}

func B64DecBytes(s string) []byte {
	decoded, err := base64.StdEncoding.DecodeString(s)
	if err != nil {
		fmt.Println(s)
		panic(err)
	}
	return decoded
}

func UB64Enc(s string) string {
	return base64.URLEncoding.EncodeToString([]byte(s))
}

func B64Enc(s string) string {
	return base64.StdEncoding.EncodeToString([]byte(s))
}

func (conn *Conn) WatchJob(row string) error {
	for {
		data, err := conn.GetRow("jobs", row, []string{})
		if err != nil {
			return err
		}
		fmt.Println(data)
		if data["status"] == "completed" {
			return nil
		}
		time.Sleep(1000 * time.Millisecond)
	}
}<|endoftext|>


<commit_before>package go_geohash

import "strings"

// Default precision to use
const default_precision uint8 = 9

// Static array of 0-9, a-z
var base32_codes [32]string
var base32_map map[string]uint8

func init() {
	base32_codes = [...]string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "b", "c", "d", "e", "f", "g", "h", "j", "k", "m", "n", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"}

	base32_map = map[string]uint8{}
	for i, c := range base32_codes {
		base32_map[c] = uint8(i)
	}
}

//
// Bounded box for parsing latitude and longitude
//
type DecodedBoundBox struct {
	MinLatitude  float64
	MinLongitude float64

	MaxLatitude  float64
	MaxLongitude float64
}

//
// Decoded Latitude,Longitude + error position
//
type DecodedPosition struct {
	Latitude  float64
	Longitude float64

	LatitudeError  float64
	LongitudeError float64
}

//
// Encode a Latitude and Longitude as a string
//
// Arguments:
//  latitude  float64
//  longitude float64
//  precision uint8  (ie how long should the hash string be?)
//
func Encode(latitude float64, longitude float64, precision uint8) string {
	// Pre-Allocate the hash string
	// var output string = strings.Repeat(" ", int(precision))

	// DecodedBoundBox for the lat/lon + errors
	var bbox DecodedBoundBox = DecodedBoundBox{MaxLatitude: 90, MaxLongitude: 180, MinLatitude: -90, MinLongitude: -180}

	var min_max_avg float64 = 0
	var islon bool = true
	var num_bits uint = 0
	var hash_index int = 0

	var buffer = make([]string, precision)
	var output_length uint8 = 0
	for output_length < precision {
		if islon {
			min_max_avg = (bbox.MaxLongitude + bbox.MinLongitude) / 2
			if longitude > min_max_avg {
				hash_index = (hash_index << 1) + 1
				bbox.MinLongitude = min_max_avg
			} else {
				hash_index = (hash_index << 1) + 0
				bbox.MaxLongitude = min_max_avg
			}
		} else {
			min_max_avg = (bbox.MaxLatitude + bbox.MinLatitude) / 2
			if latitude > min_max_avg {
				hash_index = (hash_index << 1) + 1
				bbox.MinLatitude = min_max_avg
			} else {
				hash_index = (hash_index << 1) + 0
				bbox.MaxLatitude = min_max_avg
			}
		}
		islon = !islon

		num_bits++
		if 5 == num_bits {
			buffer[output_length] = base32_codes[hash_index]
			// output[output_length] = base32_codes[hash_index]

			output_length++
			num_bits = 0
			hash_index = 0
		}
	}

	var output = strings.Join(buffer, "")

	return output
}

func DecodeBoundBox(hash_string string) DecodedBoundBox {
	// Downcase the input string
	hash_string = strings.ToLower(hash_string)

	var output DecodedBoundBox = DecodedBoundBox{MaxLatitude: 90, MaxLongitude: 180, MinLatitude: -90, MinLongitude: -180}

	var islon bool = true
	for _, c := range hash_string {
		var index_of uint8 = base32_map[string(c)]

		for bits := 4; bits >= 0; bits-- {
			bit := (index_of >> uint8(bits)) & 1
			if islon {
				mid := (output.MaxLongitude + output.MinLongitude) / 2
				if bit == 1 {
					output.MinLongitude = mid
				} else {
					output.MaxLongitude = mid
				}
			} else {
				mid := (output.MaxLatitude + output.MinLatitude) / 2
				if bit == 1 {
					output.MinLatitude = mid
				} else {
					output.MaxLatitude = mid
				}
			}
			islon = !islon
		}
	}

	return output
}

func Decode(hash_string string) DecodedPosition {
	var bbox DecodedBoundBox = DecodeBoundBox(hash_string)
	var output DecodedPosition
	// Mid point of box
	output.Latitude = (bbox.MinLatitude + bbox.MaxLatitude) / 2
	output.Longitude = (bbox.MinLongitude + bbox.MaxLongitude) / 2
	// Mid Point -  Min/Max ==> Error
	output.LatitudeError = bbox.MaxLatitude - output.Latitude
	output.LongitudeError = bbox.MaxLongitude - output.Longitude

	return output
}

func Neighbor(hash_string string, direction [2]int) string {
	// Adjust the DecodedPosition for the direction of the neighbors
	var position DecodedPosition = Decode(hash_string)
	precision := uint8(len(hash_string))
	lat := position.Latitude + float64(direction[0])*position.LatitudeError*2
	lon := position.Longitude + float64(direction[1])*position.LongitudeError*2

	return Encode(lat, lon, precision)
}
<commit_msg>Added basic tests to geohashing library<commit_after>package go_geohash

import "strings"

// Default precision to use
const default_precision uint8 = 9

// Static array of 0-9, a-z
const base32_codes [32]string

// Static map of character in "base32_codes" to it's position
// This is to improve performance of the DecodeBoundBox method
const base32_map map[string]uint8

func init() {
	base32_codes = [...]string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "b", "c", "d", "e", "f", "g", "h", "j", "k", "m", "n", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"}

	base32_map = map[string]uint8{}
	for i, c := range base32_codes {
		base32_map[c] = uint8(i)
	}
}

//
// Bounded box for parsing latitude and longitude
//
type DecodedBoundBox struct {
	MinLatitude  float64
	MinLongitude float64

	MaxLatitude  float64
	MaxLongitude float64
}

//
// Decoded Latitude,Longitude + error position
//
type DecodedPosition struct {
	Latitude  float64
	Longitude float64

	LatitudeError  float64
	LongitudeError float64
}

//
// Encode a Latitude and Longitude as a string
//
// Arguments:
//  latitude  float64
//  longitude float64
//  precision uint8  (ie how long should the hash string be?)
//
func Encode(latitude float64, longitude float64, precision uint8) string {
	// Pre-Allocate the hash string
	// var output string = strings.Repeat(" ", int(precision))

	// DecodedBoundBox for the lat/lon + errors
	var bbox DecodedBoundBox = DecodedBoundBox{MaxLatitude: 90, MaxLongitude: 180, MinLatitude: -90, MinLongitude: -180}

	var min_max_avg float64 = 0
	var islon bool = true
	var num_bits uint = 0
	var hash_index int = 0

	var buffer = make([]string, precision)
	var output_length uint8 = 0
	for output_length < precision {
		if islon {
			min_max_avg = (bbox.MaxLongitude + bbox.MinLongitude) / 2
			if longitude > min_max_avg {
				hash_index = (hash_index << 1) + 1
				bbox.MinLongitude = min_max_avg
			} else {
				hash_index = (hash_index << 1) + 0
				bbox.MaxLongitude = min_max_avg
			}
		} else {
			min_max_avg = (bbox.MaxLatitude + bbox.MinLatitude) / 2
			if latitude > min_max_avg {
				hash_index = (hash_index << 1) + 1
				bbox.MinLatitude = min_max_avg
			} else {
				hash_index = (hash_index << 1) + 0
				bbox.MaxLatitude = min_max_avg
			}
		}
		islon = !islon

		num_bits++
		if 5 == num_bits {
			buffer[output_length] = base32_codes[hash_index]
			// output[output_length] = base32_codes[hash_index]

			output_length++
			num_bits = 0
			hash_index = 0
		}
	}

	var output = strings.Join(buffer, "")

	return output
}

func DecodeBoundBox(hash_string string) DecodedBoundBox {
	// Downcase the input string
	hash_string = strings.ToLower(hash_string)

	var output DecodedBoundBox = DecodedBoundBox{MaxLatitude: 90, MaxLongitude: 180, MinLatitude: -90, MinLongitude: -180}

	var islon bool = true
	for _, c := range hash_string {
		var index_of uint8 = base32_map[string(c)]

		for bits := 4; bits >= 0; bits-- {
			bit := (index_of >> uint8(bits)) & 1
			if islon {
				mid := (output.MaxLongitude + output.MinLongitude) / 2
				if bit == 1 {
					output.MinLongitude = mid
				} else {
					output.MaxLongitude = mid
				}
			} else {
				mid := (output.MaxLatitude + output.MinLatitude) / 2
				if bit == 1 {
					output.MinLatitude = mid
				} else {
					output.MaxLatitude = mid
				}
			}
			islon = !islon
		}
	}

	return output
}

func Decode(hash_string string) DecodedPosition {
	var bbox DecodedBoundBox = DecodeBoundBox(hash_string)
	var output DecodedPosition
	// Mid point of box
	output.Latitude = (bbox.MinLatitude + bbox.MaxLatitude) / 2
	output.Longitude = (bbox.MinLongitude + bbox.MaxLongitude) / 2
	// Mid Point -  Min/Max ==> Error
	output.LatitudeError = bbox.MaxLatitude - output.Latitude
	output.LongitudeError = bbox.MaxLongitude - output.Longitude

	return output
}

func Neighbor(hash_string string, direction [2]int) string {
	// Adjust the DecodedPosition for the direction of the neighbors
	var position DecodedPosition = Decode(hash_string)
	precision := uint8(len(hash_string))
	lat := position.Latitude + float64(direction[0])*position.LatitudeError*2
	lon := position.Longitude + float64(direction[1])*position.LongitudeError*2

	return Encode(lat, lon, precision)
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2018 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package upstream

import (
	"encoding/binary"
	"fmt"
	"github.com/bitmark-inc/bitmarkd/announce"
	"github.com/bitmark-inc/bitmarkd/blockdigest"
	"github.com/bitmark-inc/bitmarkd/counter"
	"github.com/bitmark-inc/bitmarkd/fault"
	"github.com/bitmark-inc/bitmarkd/messagebus"
	"github.com/bitmark-inc/bitmarkd/mode"
	"github.com/bitmark-inc/bitmarkd/util"
	"github.com/bitmark-inc/bitmarkd/zmqutil"
	"github.com/bitmark-inc/logger"
	zmq "github.com/pebbe/zmq4"
	"sync"
	"time"
)

const (
	cycleInterval = 30 * time.Second
	queueSize     = 50 // 0 => synchronous queue
)

type Upstream struct {
	sync.RWMutex
	log         *logger.L
	client      *zmqutil.Client
	registered  bool
	blockHeight uint64
	shutdown    chan<- struct{}
}

// atomically incremented counter for log names
var upstreamCounter counter.Counter

func New(privateKey []byte, publicKey []byte, timeout time.Duration) (*Upstream, error) {
	client, err := zmqutil.NewClient(zmq.REQ, privateKey, publicKey, timeout)
	if nil != err {
		return nil, err
	}

	n := upstreamCounter.Increment()

	shutdown := make(chan struct{})
	u := &Upstream{
		log:         logger.New(fmt.Sprintf("upstream@%d", n)),
		client:      client,
		registered:  false,
		blockHeight: 0,
		shutdown:    shutdown,
	}
	go upstreamRunner(u, shutdown)
	return u, nil
}

func (u *Upstream) Destroy() {
	close(u.shutdown)
}

// check the current destination
//
// does not mean actually connected, as could be in a timeout and
// reconnect state
func (u *Upstream) IsConnectedTo(serverPublicKey []byte) bool {
	return u.client.IsConnectedTo(serverPublicKey)
}

// if registered the have avalid connection
func (u *Upstream) IsOK() bool {
	return u.registered
}

// if registered the have avalid connection
func (u *Upstream) ConnectedTo() *zmqutil.Connected {
	return u.client.ConnectedTo()
}

// connect (or reconnect) to a specific server
func (u *Upstream) Connect(address *util.Connection, serverPublicKey []byte) error {
	u.log.Infof("connecting to address: %s", address)
	u.log.Infof("connecting to server: %x", serverPublicKey)
	u.Lock()
	err := u.client.Connect(address, serverPublicKey, mode.ChainName())
	if nil == err {
		err = register(u.client, u.log)
	}
	u.Unlock()
	return err
}

// fetch height from last polled value
func (u *Upstream) GetHeight() uint64 {
	return u.blockHeight
}

// fetch block digest
func (u *Upstream) GetBlockDigest(blockNumber uint64) (blockdigest.Digest, error) {
	parameter := make([]byte, 8)
	binary.BigEndian.PutUint64(parameter, blockNumber)

	// critical section - lock out the runner process
	u.Lock()
	var data [][]byte
	err := u.client.Send("H", parameter)
	if nil == err {
		data, err = u.client.Receive(0)
	}
	u.Unlock()

	if nil != err {
		return blockdigest.Digest{}, err
	}

	if 2 != len(data) {
		return blockdigest.Digest{}, fault.ErrInvalidPeerResponse
	}

	switch string(data[0]) {
	case "E":
		return blockdigest.Digest{}, fault.InvalidError(string(data[1]))
	case "H":
		d := blockdigest.Digest{}
		if blockdigest.Length == len(data[1]) {
			err := blockdigest.DigestFromBytes(&d, data[1])
			return d, err
		}
	default:
	}
	return blockdigest.Digest{}, fault.ErrInvalidPeerResponse
}

// fetch block data
func (u *Upstream) GetBlockData(blockNumber uint64) ([]byte, error) {
	parameter := make([]byte, 8)
	binary.BigEndian.PutUint64(parameter, blockNumber)

	// critical section - lock out the runner process
	u.Lock()
	var data [][]byte
	err := u.client.Send("B", parameter)
	if nil == err {
		data, err = u.client.Receive(0)
	}
	u.Unlock()

	if nil != err {
		return nil, err
	}

	if 2 != len(data) {
		return nil, fault.ErrInvalidPeerResponse
	}

	switch string(data[0]) {
	case "E":
		return nil, fault.InvalidError(string(data[1]))
	case "B":
		return data[1], nil
	default:
	}
	return nil, fault.ErrInvalidPeerResponse
}

func upstreamRunner(u *Upstream, shutdown <-chan struct{}) {
	log := u.log

	log.Info("starting…")

	queue := messagebus.Bus.Broadcast.Chan(queueSize)

loop:
	for {
		log.Info("waiting…")

		select {
		case <-shutdown:
			break loop

		case item := <-queue:
			log.Infof("from queue: %q  %x", item.Command, item.Parameters)
			if u.registered {
				u.Lock()
				err := push(u.client, u.log, &item)
				if nil != err {
					log.Errorf("push: error: %s", err)
					err := u.client.Reconnect()
					if nil != err {
						log.Errorf("push: reconnect error: %s", err)
					}
				}
				u.Unlock()
			}

		case <-time.After(cycleInterval):
			u.Lock()
			if !u.registered {
				err := register(u.client, u.log)
				if fault.ErrNotConnected == err {
					log.Infof("register: %s", err)
					u.Unlock()
					continue loop // try again late
				} else if nil != err {
					log.Errorf("register: error: %s", err)
					err := u.client.Reconnect()
					if nil != err {
						log.Errorf("register: reconnect error: %s", err)
					}
					u.Unlock()
					continue loop // try again later
				}
				u.registered = true
			}

			h, err := getHeight(u.client, u.log)
			if nil == err {
				u.blockHeight = h
			} else {
				u.registered = false
				log.Errorf("getHeight: error: %s", err)
				err := u.client.Reconnect()
				if nil != err {
					log.Errorf("highestBlock: reconnect error: %s", err)
				}
			}
			u.Unlock()
		}
	}
	log.Info("shutting down…")
	u.client.Close()
	log.Info("stopped")
}

func register(client *zmqutil.Client, log *logger.L) error {

	log.Debugf("register: client: %s", client)

	err := announce.SendRegistration(client, "R")
	if nil != err {
		return err
	}

	data, err := client.Receive(0)
	if nil != err {
		return err
	}

	if len(data) < 2 {
		return fmt.Errorf("register received: %d  expected at least: 2", len(data))
	}

	switch string(data[0]) {
	case "E":
		return fmt.Errorf("register error: %q", data[1])
	case "R":
		if len(data) < 5 {
			return fmt.Errorf("register response incorrect: %x", data)
		}
		chain := mode.ChainName()
		received := string(data[1])
		if received != chain {
			log.Criticalf("expected chain: %q but received: %q", chain, received)
			logger.Panicf("expected chain: %q but received: %q", chain, received)
		}

		timestamp := binary.BigEndian.Uint64(data[4])
		log.Infof("register replied: public key: %x:  listeners: %x  timestamp: %d", data[2], data[3], timestamp)
		announce.AddPeer(data[2], data[3], timestamp) // publicKey, broadcasts, listeners
		return nil
	default:
		return fmt.Errorf("rpc unexpected response: %q", data[0])
	}
}

func getHeight(client *zmqutil.Client, log *logger.L) (uint64, error) {

	log.Debugf("getHeight: client: %s", client)

	err := client.Send("N")
	if nil != err {
		return 0, err
	}

	data, err := client.Receive(0)
	if nil != err {
		return 0, err
	}
	if 2 != len(data) {
		return 0, fmt.Errorf("getHeight received: %d  expected: 2", len(data))
	}

	switch string(data[0]) {
	case "E":
		return 0, fmt.Errorf("rpc error response: %q", data[1])
	case "N":
		if 8 != len(data[1]) {
			return 0, fmt.Errorf("highestBlock: rpc invalid response: %q", data[1])
		}
		height := binary.BigEndian.Uint64(data[1])
		log.Infof("height: %d", height)
		return height, nil
	default:
		return 0, fmt.Errorf("rpc unexpected response: %q", data[0])
	}
}

func push(client *zmqutil.Client, log *logger.L, item *messagebus.Message) error {

	log.Debugf("push: client: %s  %q %x", client, item.Command, item.Parameters)

	err := client.Send(item.Command, item.Parameters)
	if nil != err {
		log.Errorf("push: %s send error: %s", client, err)
		return err
	}

	data, err := client.Receive(0)
	if nil != err {
		log.Errorf("push: %s receive error: %s", client, err)
		return err
	}
	if 2 != len(data) {
		return fmt.Errorf("push received: %d  expected: 2", len(data))
	}

	switch string(data[0]) {
	case "E":
		return fmt.Errorf("rpc error response: %q", data[1])
	case item.Command:
		log.Debugf("push: client: %s complete: %q", client, data[1])
		return nil
	default:
		return fmt.Errorf("rpc unexpected response: %q", data[0])
	}
}
<commit_msg>[peer] add nil pointer check<commit_after>// Copyright (c) 2014-2018 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package upstream

import (
	"encoding/binary"
	"fmt"
	"github.com/bitmark-inc/bitmarkd/announce"
	"github.com/bitmark-inc/bitmarkd/blockdigest"
	"github.com/bitmark-inc/bitmarkd/counter"
	"github.com/bitmark-inc/bitmarkd/fault"
	"github.com/bitmark-inc/bitmarkd/messagebus"
	"github.com/bitmark-inc/bitmarkd/mode"
	"github.com/bitmark-inc/bitmarkd/util"
	"github.com/bitmark-inc/bitmarkd/zmqutil"
	"github.com/bitmark-inc/logger"
	zmq "github.com/pebbe/zmq4"
	"sync"
	"time"
)

const (
	cycleInterval = 30 * time.Second
	queueSize     = 50 // 0 => synchronous queue
)

type Upstream struct {
	sync.RWMutex
	log         *logger.L
	client      *zmqutil.Client
	registered  bool
	blockHeight uint64
	shutdown    chan<- struct{}
}

// atomically incremented counter for log names
var upstreamCounter counter.Counter

func New(privateKey []byte, publicKey []byte, timeout time.Duration) (*Upstream, error) {
	client, err := zmqutil.NewClient(zmq.REQ, privateKey, publicKey, timeout)
	if nil != err {
		return nil, err
	}

	n := upstreamCounter.Increment()

	shutdown := make(chan struct{})
	u := &Upstream{
		log:         logger.New(fmt.Sprintf("upstream@%d", n)),
		client:      client,
		registered:  false,
		blockHeight: 0,
		shutdown:    shutdown,
	}
	go upstreamRunner(u, shutdown)
	return u, nil
}

func (u *Upstream) Destroy() {
	if nil != u {
		close(u.shutdown)
	}
}

// check the current destination
//
// does not mean actually connected, as could be in a timeout and
// reconnect state
func (u *Upstream) IsConnectedTo(serverPublicKey []byte) bool {
	return u.client.IsConnectedTo(serverPublicKey)
}

// if registered the have avalid connection
func (u *Upstream) IsOK() bool {
	return u.registered
}

// if registered the have avalid connection
func (u *Upstream) ConnectedTo() *zmqutil.Connected {
	return u.client.ConnectedTo()
}

// connect (or reconnect) to a specific server
func (u *Upstream) Connect(address *util.Connection, serverPublicKey []byte) error {
	u.log.Infof("connecting to address: %s", address)
	u.log.Infof("connecting to server: %x", serverPublicKey)
	u.Lock()
	err := u.client.Connect(address, serverPublicKey, mode.ChainName())
	if nil == err {
		err = register(u.client, u.log)
	}
	u.Unlock()
	return err
}

// fetch height from last polled value
func (u *Upstream) GetHeight() uint64 {
	return u.blockHeight
}

// fetch block digest
func (u *Upstream) GetBlockDigest(blockNumber uint64) (blockdigest.Digest, error) {
	parameter := make([]byte, 8)
	binary.BigEndian.PutUint64(parameter, blockNumber)

	// critical section - lock out the runner process
	u.Lock()
	var data [][]byte
	err := u.client.Send("H", parameter)
	if nil == err {
		data, err = u.client.Receive(0)
	}
	u.Unlock()

	if nil != err {
		return blockdigest.Digest{}, err
	}

	if 2 != len(data) {
		return blockdigest.Digest{}, fault.ErrInvalidPeerResponse
	}

	switch string(data[0]) {
	case "E":
		return blockdigest.Digest{}, fault.InvalidError(string(data[1]))
	case "H":
		d := blockdigest.Digest{}
		if blockdigest.Length == len(data[1]) {
			err := blockdigest.DigestFromBytes(&d, data[1])
			return d, err
		}
	default:
	}
	return blockdigest.Digest{}, fault.ErrInvalidPeerResponse
}

// fetch block data
func (u *Upstream) GetBlockData(blockNumber uint64) ([]byte, error) {
	parameter := make([]byte, 8)
	binary.BigEndian.PutUint64(parameter, blockNumber)

	// critical section - lock out the runner process
	u.Lock()
	var data [][]byte
	err := u.client.Send("B", parameter)
	if nil == err {
		data, err = u.client.Receive(0)
	}
	u.Unlock()

	if nil != err {
		return nil, err
	}

	if 2 != len(data) {
		return nil, fault.ErrInvalidPeerResponse
	}

	switch string(data[0]) {
	case "E":
		return nil, fault.InvalidError(string(data[1]))
	case "B":
		return data[1], nil
	default:
	}
	return nil, fault.ErrInvalidPeerResponse
}

func upstreamRunner(u *Upstream, shutdown <-chan struct{}) {
	log := u.log

	log.Info("starting…")

	queue := messagebus.Bus.Broadcast.Chan(queueSize)

loop:
	for {
		log.Info("waiting…")

		select {
		case <-shutdown:
			break loop

		case item := <-queue:
			log.Infof("from queue: %q  %x", item.Command, item.Parameters)
			if u.registered {
				u.Lock()
				err := push(u.client, u.log, &item)
				if nil != err {
					log.Errorf("push: error: %s", err)
					err := u.client.Reconnect()
					if nil != err {
						log.Errorf("push: reconnect error: %s", err)
					}
				}
				u.Unlock()
			}

		case <-time.After(cycleInterval):
			u.Lock()
			if !u.registered {
				err := register(u.client, u.log)
				if fault.ErrNotConnected == err {
					log.Infof("register: %s", err)
					u.Unlock()
					continue loop // try again later
				} else if nil != err {
					log.Errorf("register: error: %s", err)
					err := u.client.Reconnect()
					if nil != err {
						log.Errorf("register: reconnect error: %s", err)
					}
					u.Unlock()
					continue loop // try again later
				}
				u.registered = true
			}

			h, err := getHeight(u.client, u.log)
			if nil == err {
				u.blockHeight = h
			} else {
				u.registered = false
				log.Errorf("getHeight: error: %s", err)
				err := u.client.Reconnect()
				if nil != err {
					log.Errorf("highestBlock: reconnect error: %s", err)
				}
			}
			u.Unlock()
		}
	}
	log.Info("shutting down…")
	u.client.Close()
	log.Info("stopped")
}

func register(client *zmqutil.Client, log *logger.L) error {

	log.Debugf("register: client: %s", client)

	err := announce.SendRegistration(client, "R")
	if nil != err {
		return err
	}

	data, err := client.Receive(0)
	if nil != err {
		return err
	}

	if len(data) < 2 {
		return fmt.Errorf("register received: %d  expected at least: 2", len(data))
	}

	switch string(data[0]) {
	case "E":
		return fmt.Errorf("register error: %q", data[1])
	case "R":
		if len(data) < 5 {
			return fmt.Errorf("register response incorrect: %x", data)
		}
		chain := mode.ChainName()
		received := string(data[1])
		if received != chain {
			log.Criticalf("expected chain: %q but received: %q", chain, received)
			logger.Panicf("expected chain: %q but received: %q", chain, received)
		}

		timestamp := binary.BigEndian.Uint64(data[4])
		log.Infof("register replied: public key: %x:  listeners: %x  timestamp: %d", data[2], data[3], timestamp)
		announce.AddPeer(data[2], data[3], timestamp) // publicKey, broadcasts, listeners
		return nil
	default:
		return fmt.Errorf("rpc unexpected response: %q", data[0])
	}
}

func getHeight(client *zmqutil.Client, log *logger.L) (uint64, error) {

	log.Debugf("getHeight: client: %s", client)

	err := client.Send("N")
	if nil != err {
		return 0, err
	}

	data, err := client.Receive(0)
	if nil != err {
		return 0, err
	}
	if 2 != len(data) {
		return 0, fmt.Errorf("getHeight received: %d  expected: 2", len(data))
	}

	switch string(data[0]) {
	case "E":
		return 0, fmt.Errorf("rpc error response: %q", data[1])
	case "N":
		if 8 != len(data[1]) {
			return 0, fmt.Errorf("highestBlock: rpc invalid response: %q", data[1])
		}
		height := binary.BigEndian.Uint64(data[1])
		log.Infof("height: %d", height)
		return height, nil
	default:
		return 0, fmt.Errorf("rpc unexpected response: %q", data[0])
	}
}

func push(client *zmqutil.Client, log *logger.L, item *messagebus.Message) error {

	log.Debugf("push: client: %s  %q %x", client, item.Command, item.Parameters)

	err := client.Send(item.Command, item.Parameters)
	if nil != err {
		log.Errorf("push: %s send error: %s", client, err)
		return err
	}

	data, err := client.Receive(0)
	if nil != err {
		log.Errorf("push: %s receive error: %s", client, err)
		return err
	}
	if 2 != len(data) {
		return fmt.Errorf("push received: %d  expected: 2", len(data))
	}

	switch string(data[0]) {
	case "E":
		return fmt.Errorf("rpc error response: %q", data[1])
	case item.Command:
		log.Debugf("push: client: %s complete: %q", client, data[1])
		return nil
	default:
		return fmt.Errorf("rpc unexpected response: %q", data[0])
	}
}
<|endoftext|>


<commit_before>package main

import (
	"testing"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

func TestNewContact(t *testing.T) {
	sess, err := mgo.Dial("localhost")
	if err != nil {
		t.Errorf("%s", err)
	}
	defer sess.Close()
	sess.SetSafe(&mgo.Safe{})
	collection := sess.DB("test").C("newcontact")
	fakeContact, err := NewContact(
		collection,
		"Encom Inc.",
		"Flynn",
		"flynn@encom.com",
		"",
		"",
		"USA",
	)
	if err != nil {
		t.Errorf("%s", err)
	}
	var fetchedContact contact
	err = sess.DB("test").C("newcontact").Find(bson.M{}).One(&fetchedContact)
	if err != nil {
		t.Errorf("%s", err)
	}

	// fakeContact is a pointer, because NewContact returns a pointer to a struct of contact type.
	// That's why we check fetchedContact with *fakeContact.

	if fetchedContact != *fakeContact {
		t.Errorf("inserted contact is not the fetched contact")
	}
	err = collection.DropCollection()
	if err != nil {
		t.Errorf("%s", err)
	}
}
<commit_msg>Add tests for GetContact() function<commit_after>package main

import (
	"testing"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

func TestNewContact(t *testing.T) {
	sess, err := mgo.Dial("localhost")
	if err != nil {
		t.Errorf("%s", err)
	}
	defer sess.Close()
	sess.SetSafe(&mgo.Safe{})
	collection := sess.DB("test").C("newcontact")
	fakeContact, err := NewContact(
		collection,
		"Encom Inc.",
		"Flynn",
		"flynn@encom.com",
		"",
		"",
		"USA",
	)
	if err != nil {
		t.Errorf("%s", err)
	}
	var fetchedContact contact
	err = sess.DB("test").C("newcontact").Find(bson.M{}).One(&fetchedContact)
	if err != nil {
		t.Errorf("%s", err)
	}

	// fakeContact is a pointer, because NewContact returns a pointer to a struct of contact type.
	// That's why we check fetchedContact with *fakeContact.

	if fetchedContact != *fakeContact {
		t.Errorf("inserted contact is not the fetched contact")
	}
	err = collection.DropCollection()
	if err != nil {
		t.Errorf("%s", err)
	}
}

func TestGetContact(t *testing.T) {
	sess, err := mgo.Dial("localhost")
	if err != nil {
		t.Errorf("%s", err)
	}
	defer sess.Close()
	sess.SetSafe(&mgo.Safe{})
	collection := sess.DB("test").C("newcontact")
	fakeContact, err := NewContact(
		collection,
		"Encom Inc.",
		"Flynn",
		"flynn@encom.com",
		"",
		"",
		"USA",
	)
	if err != nil {
		t.Errorf("%s", err)
	}
	id := fakeContact.Id
	fetchedContact, err := GetContact(id)
	if err != nil {
		t.Errorf("%s", err)
	}
	if *fakeContact != *fetchedContact {
		t.Errorf("Expected %+v, but got %+v\n", *fakeContact, *fetchedContact)
	}
	err = collection.DropCollection()
	if err != nil {
		t.Errorf("%s", err)
	}
}
<|endoftext|>


<commit_before>package container551

import (
	"errors"
	"github.com/go51/auth551"
	"github.com/go51/cookie551"
	"github.com/go51/log551"
	"github.com/go51/memcache551"
	"github.com/go51/model551"
	"github.com/go51/mysql551"
	"github.com/go51/repository551"
	"github.com/go51/secure551"
	"github.com/go51/string551"
	"net/http"
	"strconv"
)

type urlFunc func(name string, parameter ...string) string

type Container struct {
	sid         string
	ssid        string
	w           http.ResponseWriter
	r           *http.Request
	logger      *log551.Log551
	cookie      *cookie551.Cookie
	db          *mysql551.Mysql
	session     *memcache551.Memcache
	model       *model551.Model
	auth        *auth551.Auth
	user        *auth551.UserModel
	options     map[string]string
	urlFunction urlFunc
}

func New() *Container {
	return &Container{}
}

func (c *Container) SetSID(sid string) {
	c.sid = sid
	c.ssid = sid[:10]
}

func (c *Container) SID() string {
	return c.sid
}
func (c *Container) SSID() string {
	return c.ssid
}

func (c *Container) SetResponseWriter(w http.ResponseWriter) {
	c.w = w
}

func (c *Container) ResponseWriter() http.ResponseWriter {
	return c.w
}

func (c *Container) SetRequest(r *http.Request) {
	c.r = r
}

func (c *Container) Request() *http.Request {
	return c.r
}

func (c *Container) SetLogger(logger *log551.Log551) {
	c.logger = logger
}

func (c *Container) Logger() *log551.Log551 {
	return c.logger
}

func (c *Container) SetCookie(cookie *cookie551.Cookie) {
	c.cookie = cookie
}

func (c *Container) Cookie() *cookie551.Cookie {
	return c.cookie
}

func (c *Container) SetDb(db *mysql551.Mysql) {
	c.db = db
}

func (c *Container) Db() *mysql551.Mysql {
	return c.db
}

func (c *Container) SetSession(session *memcache551.Memcache) {
	c.session = session
}

func (c *Container) Session() *memcache551.Memcache {
	return c.session
}

func (c *Container) SetModel(modelManager *model551.Model) {
	c.model = modelManager
}

func (c *Container) ModelManager() *model551.Model {
	return c.model
}

func (c *Container) SetAuth(auth *auth551.Auth) {
	c.auth = auth

	if c.user != nil {
		return
	}

	// Load user from session
	c.session.GetModel("reminder_user", &c.user)

	// Get user id from cookie
	id, err := c.getRemindId()
	if err != nil {
		return
	}

	// Get user model from database
	c.user = c.getUser(id)

	// Set user model to session
	c.session.Set("reminder_user", c.user)

	return

}

func (c *Container) getRemindId() (int64, error) {
	cookieId, err := c.cookie.Get(c.auth.CookieKeyName())
	if err != nil {
		return 0, err
	}

	sid := secure551.Decrypted(cookieId, c.auth.MasterKey())
	id, err := strconv.ParseInt(sid, 10, 64)
	if err != nil {
		return 0, err
	}

	return id, nil

}

func (c *Container) getUser(id int64) *auth551.UserModel {
	repo := repository551.Load()
	miUser := c.ModelManager().Get("UserModel")
	mUser := repo.Find(c.db, miUser, id)
	user, ok := mUser.(*auth551.UserModel)
	if !ok {
		return nil
	}

	return user
}

func (c *Container) Auth() *auth551.Auth {
	return c.auth
}

func (c *Container) SignIn(user *auth551.UserModel) {
	// Set remind id to cookie
	id := string551.Right("0000000000000000"+strconv.FormatInt(user.Id, 10), 16)
	secureId := secure551.Encrypted(id, c.auth.MasterKey())
	c.cookie.Set(c.auth.CookieKeyName(), secureId, 60*60*24*365)

	// Set user model to session
	c.session.Set("reminder_user", user)

}

func (c *Container) SignOut() {
	c.cookie.Delete(c.auth.CookieKeyName())
	c.session.Delete("reminder_user")
}

func (c *Container) IsSignIn() bool {
	return c.user != nil
}

func (c *Container) User() *auth551.UserModel {
	return c.user
}

func (c *Container) UpdateUser(user *auth551.UserModel) {
	c.user = user
}

func (c *Container) SetCommandOptions(options map[string]string) {
	c.options = options
}

func (c *Container) CommandOption(name string) string {
	return c.options[name]
}

func (c *Container) Segment(number int) string {
	c.logger.Debugf("%s [ URL.Path ] %s", c.ssid, c.r.URL.Path[1:])
	paths := string551.Split(c.r.URL.Path[1:], "/")
	c.logger.Debugf("%s [ URL.Path ] %#v", c.ssid, paths)
	if len(paths) < number+1 {
		return ""
	}
	return paths[number]
}

func (c *Container) SegmentInt64(number int) (int64, error) {
	c.logger.Debugf("%s [ URL.Path ] %s", c.ssid, c.r.URL.Path[1:])
	paths := string551.Split(c.r.URL.Path[1:], "/")
	c.logger.Debugf("%s [ URL.Path ] %#v", c.ssid, paths)
	if len(paths) < number+1 {
		return 0, errors.New("invalid memory address or nil pointer dereference")
	}
	segment := paths[number]

	return strconv.ParseInt(segment, 10, 64)

}

func (c *Container) SetUrlFunc(urlFunction urlFunc) {
	c.urlFunction = urlFunction
}

func (c *Container) URL(name string, parameter ...string) string {
	return c.urlFunction(name, parameter...)
}
<commit_msg>refs #35 絶対 URL を返す UrlAbs を実装する<commit_after>package container551

import (
	"errors"
	"github.com/go51/auth551"
	"github.com/go51/cookie551"
	"github.com/go51/log551"
	"github.com/go51/memcache551"
	"github.com/go51/model551"
	"github.com/go51/mysql551"
	"github.com/go51/repository551"
	"github.com/go51/secure551"
	"github.com/go51/string551"
	"net/http"
	"strconv"
)

type urlFunc func(name string, parameter ...string) string

type Container struct {
	sid         string
	ssid        string
	w           http.ResponseWriter
	r           *http.Request
	logger      *log551.Log551
	cookie      *cookie551.Cookie
	db          *mysql551.Mysql
	session     *memcache551.Memcache
	model       *model551.Model
	auth        *auth551.Auth
	user        *auth551.UserModel
	options     map[string]string
	urlFunction urlFunc
	baseUrl     string
}

func New() *Container {
	return &Container{}
}

func (c *Container) SetSID(sid string) {
	c.sid = sid
	c.ssid = sid[:10]
}

func (c *Container) SID() string {
	return c.sid
}
func (c *Container) SSID() string {
	return c.ssid
}

func (c *Container) SetResponseWriter(w http.ResponseWriter) {
	c.w = w
}

func (c *Container) ResponseWriter() http.ResponseWriter {
	return c.w
}

func (c *Container) SetRequest(r *http.Request) {
	c.r = r
}

func (c *Container) Request() *http.Request {
	return c.r
}

func (c *Container) SetLogger(logger *log551.Log551) {
	c.logger = logger
}

func (c *Container) Logger() *log551.Log551 {
	return c.logger
}

func (c *Container) SetCookie(cookie *cookie551.Cookie) {
	c.cookie = cookie
}

func (c *Container) Cookie() *cookie551.Cookie {
	return c.cookie
}

func (c *Container) SetDb(db *mysql551.Mysql) {
	c.db = db
}

func (c *Container) Db() *mysql551.Mysql {
	return c.db
}

func (c *Container) SetSession(session *memcache551.Memcache) {
	c.session = session
}

func (c *Container) Session() *memcache551.Memcache {
	return c.session
}

func (c *Container) SetModel(modelManager *model551.Model) {
	c.model = modelManager
}

func (c *Container) ModelManager() *model551.Model {
	return c.model
}

func (c *Container) SetAuth(auth *auth551.Auth) {
	c.auth = auth

	if c.user != nil {
		return
	}

	// Load user from session
	c.session.GetModel("reminder_user", &c.user)

	// Get user id from cookie
	id, err := c.getRemindId()
	if err != nil {
		return
	}

	// Get user model from database
	c.user = c.getUser(id)

	// Set user model to session
	c.session.Set("reminder_user", c.user)

	return

}

func (c *Container) getRemindId() (int64, error) {
	cookieId, err := c.cookie.Get(c.auth.CookieKeyName())
	if err != nil {
		return 0, err
	}

	sid := secure551.Decrypted(cookieId, c.auth.MasterKey())
	id, err := strconv.ParseInt(sid, 10, 64)
	if err != nil {
		return 0, err
	}

	return id, nil

}

func (c *Container) getUser(id int64) *auth551.UserModel {
	repo := repository551.Load()
	miUser := c.ModelManager().Get("UserModel")
	mUser := repo.Find(c.db, miUser, id)
	user, ok := mUser.(*auth551.UserModel)
	if !ok {
		return nil
	}

	return user
}

func (c *Container) Auth() *auth551.Auth {
	return c.auth
}

func (c *Container) SignIn(user *auth551.UserModel) {
	// Set remind id to cookie
	id := string551.Right("0000000000000000"+strconv.FormatInt(user.Id, 10), 16)
	secureId := secure551.Encrypted(id, c.auth.MasterKey())
	c.cookie.Set(c.auth.CookieKeyName(), secureId, 60*60*24*365)

	// Set user model to session
	c.session.Set("reminder_user", user)

}

func (c *Container) SignOut() {
	c.cookie.Delete(c.auth.CookieKeyName())
	c.session.Delete("reminder_user")
}

func (c *Container) IsSignIn() bool {
	return c.user != nil
}

func (c *Container) User() *auth551.UserModel {
	return c.user
}

func (c *Container) UpdateUser(user *auth551.UserModel) {
	c.user = user
}

func (c *Container) SetCommandOptions(options map[string]string) {
	c.options = options
}

func (c *Container) CommandOption(name string) string {
	return c.options[name]
}

func (c *Container) Segment(number int) string {
	c.logger.Debugf("%s [ URL.Path ] %s", c.ssid, c.r.URL.Path[1:])
	paths := string551.Split(c.r.URL.Path[1:], "/")
	c.logger.Debugf("%s [ URL.Path ] %#v", c.ssid, paths)
	if len(paths) < number+1 {
		return ""
	}
	return paths[number]
}

func (c *Container) SegmentInt64(number int) (int64, error) {
	c.logger.Debugf("%s [ URL.Path ] %s", c.ssid, c.r.URL.Path[1:])
	paths := string551.Split(c.r.URL.Path[1:], "/")
	c.logger.Debugf("%s [ URL.Path ] %#v", c.ssid, paths)
	if len(paths) < number+1 {
		return 0, errors.New("invalid memory address or nil pointer dereference")
	}
	segment := paths[number]

	return strconv.ParseInt(segment, 10, 64)

}

func (c *Container) SetUrlFunc(urlFunction urlFunc) {
	c.urlFunction = urlFunction
}

func (c *Container) SetBaseURL(url string) {
	c.baseUrl = url
}

func (c *Container) URL(name string, parameter ...string) string {
	return c.urlFunction(name, parameter...)
}

func (c *Container) URLAbs(name string, parameter ...string) string {
	return c.baseUrl + c.urlFunction(name, parameter...)
}
<|endoftext|>


<commit_before>/*
Package goushahidi provides a client for accessing data from Ushahidi API.

Access different parts of the World Bank Open Data API using the various
services:
         client := goushahidi.NewClient(nil)

         // list all incidences
         incidences, err := client.GetIncidences()

The full Ushahidi API is documented at https://wiki.ushahidi.com/display/WIKI/Ushahidi+3.x+REST+API.
*/
package goushahidi

const (
	libraryVersion = "0.1"
	defaultBaseURL = "http://demo.ushahidi.com" // Demo Ushahidi Instance
)

type UshahidiClient struct {
}

func NewClient() *UshahidiClient {
	return &UshahidiClient{}

}
<commit_msg>updated doc<commit_after>/*
Package goushahidi provides a client for accessing data from Ushahidi API.

Access different data using the API:
         client := goushahidi.NewClient(nil)

         // list all incidences
         incidences, err := client.GetIncidences()

The full Ushahidi API is documented at https://wiki.ushahidi.com/display/WIKI/Ushahidi+3.x+REST+API.
*/
package goushahidi

const (
	libraryVersion = "0.1"
	defaultBaseURL = "http://demo.ushahidi.com" // Demo Ushahidi Instance
)

type UshahidiClient struct {
}

func NewClient() *UshahidiClient {
	return &UshahidiClient{}

}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"os"
	"path"
	"testing"
	"time"
)

func TestZip(t *testing.T) {

	zippath := "test.zip"

	ioutil.WriteFile(zippath, []byte("<possibly an exefile>"), 0644)

	os.MkdirAll("files/emptydir", 0755)
	ioutil.WriteFile("hello.txt", []byte("Hello World"), 0777)
	ioutil.WriteFile("files/hello.tpl", []byte("
Hello World
"), 0644)

	testfileheader := "hello.txt"
	info, err := os.Stat(testfileheader)
	if err != nil {
		t.Error(err)
	}
	actualperms := info.Mode().Perm()

	filetime := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
	err = os.Chtimes(testfileheader, filetime, filetime)
	if err != nil {
		t.Error(err)
	}

	Zip(zippath, []string{"files", "hello.txt"})

	if !IsZip(zippath) {
		t.Error("zip test failed")
	}

	os.RemoveAll("files")
	os.Remove("hello.txt")

	list, err := UnzipList(zippath)
	if err != nil || len(list) != 4 {
		t.Error("unzip list failed")
	}

	if err := Unzip(zippath, "extract"); err != nil {
		t.Error("unzip failed")
	}

	if _, err := os.Stat("extract/files/hello.tpl"); os.IsNotExist(err) {
		t.Error("unzip didn't work")
	}
	if _, err := os.Stat("extract/files/emptydir"); os.IsNotExist(err) {
		t.Error("unzip didn't create empty dir")
	}

	info, err = os.Stat(path.Join("extract", testfileheader))
	if err != nil {
		t.Error(err)
		t.FailNow()
	}
	if info.Mode().Perm() != actualperms {
		t.Error("unzip didn't set file perms")
	}
	if !info.ModTime().Equal(filetime) {
		t.Error("unzip didn't set file modtime")
	}

	if !t.Failed() {
		os.Remove("hello.txt")
		os.Remove("test.zip")
		os.RemoveAll("files")
		os.RemoveAll("extract")
	}

}
<commit_msg>Update tests<commit_after>package gozip

import (
	"io/ioutil"
	"os"
	"path"
	"testing"
	"time"
)

func TestZip(t *testing.T) {

	zippath := "test.zip"

	ioutil.WriteFile(zippath, []byte("<possibly an exefile>"), 0644)

	os.MkdirAll("files/emptydir", 0755)
	ioutil.WriteFile("hello.txt", []byte("Hello World"), 0777)
	ioutil.WriteFile("files/hello.tpl", []byte("
Hello World
"), 0644)

	testfileheader := "hello.txt"
	info, err := os.Stat(testfileheader)
	if err != nil {
		t.Error(err)
	}
	actualperms := info.Mode().Perm()

	filetime := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
	err = os.Chtimes(testfileheader, filetime, filetime)
	if err != nil {
		t.Error(err)
	}

	Zip(zippath, []string{"files", "hello.txt"})

	if !IsZip(zippath) {
		t.Error("zip test failed")
	}

	os.RemoveAll("files")
	os.Remove("hello.txt")

	list, err := UnzipList(zippath)
	if err != nil || len(list) != 4 {
		t.Error("unzip list failed")
	}

	if err := Unzip(zippath, "extract"); err != nil {
		t.Error("unzip failed")
	}

	if _, err := os.Stat("extract/files/hello.tpl"); os.IsNotExist(err) {
		t.Error("unzip didn't work")
	}
	if _, err := os.Stat("extract/files/emptydir"); os.IsNotExist(err) {
		t.Error("unzip didn't create empty dir")
	}

	info, err = os.Stat(path.Join("extract", testfileheader))
	if err != nil {
		t.Error(err)
		t.FailNow()
	}
	if info.Mode().Perm() != actualperms {
		t.Error("unzip didn't set file perms")
	}
	if !info.ModTime().Equal(filetime) {
		t.Error("unzip didn't set file modtime")
	}

	if !t.Failed() {
		os.Remove("hello.txt")
		os.Remove("test.zip")
		os.RemoveAll("files")
		os.RemoveAll("extract")
	}

}
<|endoftext|>


<commit_before>// Copyright (c) Alex Ellis 2017. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

package proxy

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/openfaas/faas-cli/stack"
	"github.com/openfaas/faas/gateway/requests"
)

// FunctionResourceRequest defines a request to set function resources
type FunctionResourceRequest struct {
	Limits   *stack.FunctionResources
	Requests *stack.FunctionResources
}

func DeployFunction(fprocess string, gateway string, functionName string, image string,
	registryAuth string, language string, replace bool, envVars map[string]string,
	network string, constraints []string, update bool, secrets []string,
	labels map[string]string, functionResourceRequest1 FunctionResourceRequest) {

	rollingUpdateInfo := fmt.Sprintf("Function %s already exists, attempting rolling-update.", functionName)
	statusCode, deployOutput := Deploy(fprocess, gateway, functionName, image, registryAuth, language, replace, envVars, network, constraints, update, secrets, labels, functionResourceRequest1)

	if update == true && statusCode == http.StatusNotFound {
		// Re-run the function with update=false
		_, deployOutput = Deploy(fprocess, gateway, functionName, image, registryAuth, language, replace, envVars, network, constraints, false, secrets, labels, functionResourceRequest1)
	} else if statusCode == http.StatusOK {
		fmt.Println(rollingUpdateInfo)
	}
	fmt.Println()
	fmt.Println(deployOutput)
}

func Deploy(fprocess string, gateway string, functionName string, image string,
	registryAuth string, language string, replace bool, envVars map[string]string,
	network string, constraints []string, update bool, secrets []string,
	labels map[string]string, functionResourceRequest1 FunctionResourceRequest) (int, string) {

	var deployOutput string
	// Need to alter Gateway to allow nil/empty string as fprocess, to avoid this repetition.
	var fprocessTemplate string
	if len(fprocess) > 0 {
		fprocessTemplate = fprocess
	}

	if (registryAuth != "") && !strings.HasPrefix(gateway, "https") {
		fmt.Println("WARNING! Communication is not secure, please consider using HTTPS. Letsencrypt.org offers free SSL/TLS certificates.")
	}

	gateway = strings.TrimRight(gateway, "/")

	if replace {
		DeleteFunction(gateway, functionName)
	}

	req := requests.CreateFunctionRequest{
		EnvProcess:   fprocessTemplate,
		Image:        image,
		RegistryAuth: registryAuth,
		Network:      network,
		Service:      functionName,
		EnvVars:      envVars,
		Constraints:  constraints,
		Secrets:      secrets,
		Labels:       &labels,
	}

	hasLimits := false
	req.Limits = &requests.FunctionResources{}
	if functionResourceRequest1.Limits != nil && len(functionResourceRequest1.Limits.Memory) > 0 {
		hasLimits = true
		req.Limits.Memory = functionResourceRequest1.Limits.Memory
	}
	if functionResourceRequest1.Limits != nil && len(functionResourceRequest1.Limits.CPU) > 0 {
		hasLimits = true
		req.Limits.CPU = functionResourceRequest1.Limits.CPU
	}
	if !hasLimits {
		req.Limits = nil
	}

	hasRequests := false
	req.Requests = &requests.FunctionResources{}
	if functionResourceRequest1.Requests != nil && len(functionResourceRequest1.Requests.Memory) > 0 {
		hasRequests = true
		req.Requests.Memory = functionResourceRequest1.Requests.Memory
	}
	if functionResourceRequest1.Requests != nil && len(functionResourceRequest1.Requests.CPU) > 0 {
		hasRequests = true
		req.Requests.CPU = functionResourceRequest1.Requests.CPU
	}

	if !hasRequests {
		req.Requests = nil
	}

	reqBytes, _ := json.Marshal(&req)
	reader := bytes.NewReader(reqBytes)
	var request *http.Request

	timeout := 60 * time.Second
	client := MakeHTTPClient(&timeout)

	method := http.MethodPost
	// "application/json"
	if update {
		method = http.MethodPut
	}

	var err error
	request, err = http.NewRequest(method, gateway+"/system/functions", reader)
	SetAuth(request, gateway)
	if err != nil {
		deployOutput += fmt.Sprintln(err)
		return http.StatusInternalServerError, deployOutput
	}

	res, err := client.Do(request)
	if err != nil {
		deployOutput += fmt.Sprintln("Is FaaS deployed? Do you need to specify the --gateway flag?")
		deployOutput += fmt.Sprintln(err)
		return http.StatusInternalServerError, deployOutput
	}

	if res.Body != nil {
		defer res.Body.Close()
	}

	switch res.StatusCode {
	case http.StatusOK, http.StatusCreated, http.StatusAccepted:
		deployOutput += fmt.Sprintf("Deployed. %s.\n", res.Status)

		deployedURL := fmt.Sprintf("URL: %s/function/%s", gateway, functionName)
		deployOutput += fmt.Sprintln(deployedURL)
	case http.StatusUnauthorized:
		deployOutput += fmt.Sprintln("unauthorized access, run \"faas-cli login\" to setup authentication for this server")
		/*
			case http.StatusNotFound:
				if replace && !update {
					deployOutput += fmt.Sprintln("Could not delete-and-replace function because it is not found (404)")
				}
		*/
	default:
		bytesOut, err := ioutil.ReadAll(res.Body)
		if err == nil {
			deployOutput += fmt.Sprintf("Unexpected status: %d, message: %s\n", res.StatusCode, string(bytesOut))
		}
	}

	return res.StatusCode, deployOutput
}
<commit_msg>Fixes issue with double warning about insecure TLS<commit_after>// Copyright (c) Alex Ellis 2017. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

package proxy

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/openfaas/faas-cli/stack"
	"github.com/openfaas/faas/gateway/requests"
)

// FunctionResourceRequest defines a request to set function resources
type FunctionResourceRequest struct {
	Limits   *stack.FunctionResources
	Requests *stack.FunctionResources
}

// DeployFunction first tries to deploy a function and if it exists will then attempt
// a rolling update. Warnings are suppressed for the second API call (if required.)
func DeployFunction(fprocess string, gateway string, functionName string, image string,
	registryAuth string, language string, replace bool, envVars map[string]string,
	network string, constraints []string, update bool, secrets []string,
	labels map[string]string, functionResourceRequest1 FunctionResourceRequest) {

	rollingUpdateInfo := fmt.Sprintf("Function %s already exists, attempting rolling-update.", functionName)
	warnInsecureGateway := true
	statusCode, deployOutput := Deploy(fprocess, gateway, functionName, image, registryAuth, language, replace, envVars, network, constraints, update, secrets, labels, functionResourceRequest1, warnInsecureGateway)

	warnInsecureGateway = false
	if update == true && statusCode == http.StatusNotFound {
		// Re-run the function with update=false

		_, deployOutput = Deploy(fprocess, gateway, functionName, image, registryAuth, language, replace, envVars, network, constraints, false, secrets, labels, functionResourceRequest1, warnInsecureGateway)
	} else if statusCode == http.StatusOK {
		fmt.Println(rollingUpdateInfo)
	}
	fmt.Println()
	fmt.Println(deployOutput)
}

// Deploy a function to an OpenFaaS gateway over REST
func Deploy(fprocess string, gateway string, functionName string, image string,
	registryAuth string, language string, replace bool, envVars map[string]string,
	network string, constraints []string, update bool, secrets []string,
	labels map[string]string, functionResourceRequest1 FunctionResourceRequest,
	warnInsecureGateway bool) (int, string) {

	var deployOutput string
	// Need to alter Gateway to allow nil/empty string as fprocess, to avoid this repetition.
	var fprocessTemplate string
	if len(fprocess) > 0 {
		fprocessTemplate = fprocess
	}

	if warnInsecureGateway {
		if (registryAuth != "") && !strings.HasPrefix(gateway, "https") {
			fmt.Println("WARNING! Communication is not secure, please consider using HTTPS. Letsencrypt.org offers free SSL/TLS certificates.")
		}
	}

	gateway = strings.TrimRight(gateway, "/")

	if replace {
		DeleteFunction(gateway, functionName)
	}

	req := requests.CreateFunctionRequest{
		EnvProcess:   fprocessTemplate,
		Image:        image,
		RegistryAuth: registryAuth,
		Network:      network,
		Service:      functionName,
		EnvVars:      envVars,
		Constraints:  constraints,
		Secrets:      secrets,
		Labels:       &labels,
	}

	hasLimits := false
	req.Limits = &requests.FunctionResources{}
	if functionResourceRequest1.Limits != nil && len(functionResourceRequest1.Limits.Memory) > 0 {
		hasLimits = true
		req.Limits.Memory = functionResourceRequest1.Limits.Memory
	}
	if functionResourceRequest1.Limits != nil && len(functionResourceRequest1.Limits.CPU) > 0 {
		hasLimits = true
		req.Limits.CPU = functionResourceRequest1.Limits.CPU
	}
	if !hasLimits {
		req.Limits = nil
	}

	hasRequests := false
	req.Requests = &requests.FunctionResources{}
	if functionResourceRequest1.Requests != nil && len(functionResourceRequest1.Requests.Memory) > 0 {
		hasRequests = true
		req.Requests.Memory = functionResourceRequest1.Requests.Memory
	}
	if functionResourceRequest1.Requests != nil && len(functionResourceRequest1.Requests.CPU) > 0 {
		hasRequests = true
		req.Requests.CPU = functionResourceRequest1.Requests.CPU
	}

	if !hasRequests {
		req.Requests = nil
	}

	reqBytes, _ := json.Marshal(&req)
	reader := bytes.NewReader(reqBytes)
	var request *http.Request

	timeout := 60 * time.Second
	client := MakeHTTPClient(&timeout)

	method := http.MethodPost
	// "application/json"
	if update {
		method = http.MethodPut
	}

	var err error
	request, err = http.NewRequest(method, gateway+"/system/functions", reader)
	SetAuth(request, gateway)
	if err != nil {
		deployOutput += fmt.Sprintln(err)
		return http.StatusInternalServerError, deployOutput
	}

	res, err := client.Do(request)
	if err != nil {
		deployOutput += fmt.Sprintln("Is FaaS deployed? Do you need to specify the --gateway flag?")
		deployOutput += fmt.Sprintln(err)
		return http.StatusInternalServerError, deployOutput
	}

	if res.Body != nil {
		defer res.Body.Close()
	}

	switch res.StatusCode {
	case http.StatusOK, http.StatusCreated, http.StatusAccepted:
		deployOutput += fmt.Sprintf("Deployed. %s.\n", res.Status)

		deployedURL := fmt.Sprintf("URL: %s/function/%s", gateway, functionName)
		deployOutput += fmt.Sprintln(deployedURL)
	case http.StatusUnauthorized:
		deployOutput += fmt.Sprintln("unauthorized access, run \"faas-cli login\" to setup authentication for this server")
		/*
			case http.StatusNotFound:
				if replace && !update {
					deployOutput += fmt.Sprintln("Could not delete-and-replace function because it is not found (404)")
				}
		*/
	default:
		bytesOut, err := ioutil.ReadAll(res.Body)
		if err == nil {
			deployOutput += fmt.Sprintf("Unexpected status: %d, message: %s\n", res.StatusCode, string(bytesOut))
		}
	}

	return res.StatusCode, deployOutput
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"sync"
	"syscall"
	"time"

	"github.com/voxelbrain/goptions"
)

const (
	VERSION = "1.2.0"
)

var (
	DefaultBlocksize = 4 * KiloByte
	options          = struct {
		Cores     int           `goptions:"-c, --cores, description='Number of cores to use'"`
		Threads   int           `goptions:"-t, --threads, description='Number of threads to use'"`
		Blocksize *Datasize     `goptions:"-b, --block-size, description='Number of bytes to write with each call'"`
		Filesize  *Datasize     `goptions:"-f, --file-size, obligatory, description='Number of zeroes to write to each file'"`
		OutputDir string        `goptions:"-o, --output-dir, description='Output directory'"`
		Sync      bool          `goptions:"-s, --sync, description='Sync after every written block'"`
		KeepFiles bool          `goptions:"-k, --keep-files, description='Dont delete files when done'"`
		Help      goptions.Help `goptions:"-h, --help, description='Show this help'"`
	}{
		Cores:     runtime.NumCPU(),
		Threads:   runtime.NumCPU(),
		Blocksize: &DefaultBlocksize,
		OutputDir: os.TempDir(),
	}
	Timestamp = fmt.Sprintf("%d", time.Now().UnixNano())
)

func init() {
	goptions.ParseAndFail(&options)
	options.OutputDir = filepath.Clean(options.OutputDir + "/iowhip_" + Timestamp)
}

type Result struct {
	Index    int
	Duration time.Duration
}

func main() {
	err := os.MkdirAll(options.OutputDir, os.FileMode(0755))
	if err != nil {
		log.Fatalf("Could not create output directory %s: %s", options.OutputDir, err)
	}
	cleanup := make(chan os.Signal)
	signal.Notify(cleanup, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGINT)
	go func() {
		<-cleanup
		if !options.KeepFiles {
			os.RemoveAll(options.OutputDir)
		}
	}()

	runtime.GOMAXPROCS(options.Cores)
	log.Printf("Starting %d workers on %d cores writing %s bytes (%s per call)...", options.Threads, options.Cores, options.Filesize, options.Blocksize)

	results := make(chan Result, options.Threads)
	wg := &sync.WaitGroup{}
	wg.Add(options.Threads)
	for i := 0; i < options.Threads; i++ {
		go writeFile(i, results, wg)
	}
	wg.Wait()
	for i := 0; i < options.Threads; i++ {
		r := <-results
		log.Printf("Thread %d: %s, %s/s", r.Index, r.Duration, Datasize(int64(float64(*options.Filesize)*float64(time.Second)/float64(r.Duration))))
	}
	close(results)
	cleanup <- syscall.SIGINT
}

func writeFile(idx int, c chan Result, wg *sync.WaitGroup) {
	result := Result{
		Index: idx,
	}
	defer func() { c <- result }()
	defer wg.Done()
	filename := fmt.Sprintf("%s/%d", options.OutputDir, idx)
	log.Printf("Thread %d: Using %s", idx, filename)

	amount := *options.Filesize
	data := make([]byte, int(*options.Blocksize))
	start := time.Now()
	f, err := os.Create(filename)
	if err != nil {
		log.Printf("Thread %d: Could not open file %s: %s", idx, filename, err)
		return
	}
	defer f.Close()

	for amount > 0 {
		n, err := f.Write(data)
		if err != nil {
			log.Printf("Thread %d: Write to %s failed: %s", idx, filename, err)
			return
		}
		amount -= Datasize(n)
		if options.Sync {
			f.Sync()
		}
	}
	f.Sync()
	f.Close()
	result.Duration = time.Now().Sub(start)
}
<commit_msg>Measure pure writing AND total. Do prettyprint<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"sync"
	"syscall"
	"text/tabwriter"
	"time"

	"github.com/voxelbrain/goptions"
)

const (
	VERSION = "1.2.0"
)

var (
	DefaultBlocksize = 4 * KiloByte
	options          = struct {
		Cores     int           `goptions:"-c, --cores, description='Number of cores to use'"`
		Threads   int           `goptions:"-t, --threads, description='Number of threads to use'"`
		Blocksize *Datasize     `goptions:"-b, --block-size, description='Number of bytes to write with each call'"`
		Filesize  *Datasize     `goptions:"-f, --file-size, obligatory, description='Number of zeroes to write to each file'"`
		OutputDir string        `goptions:"-o, --output-dir, description='Output directory'"`
		Sync      bool          `goptions:"-s, --sync, description='Sync after every written block'"`
		KeepFiles bool          `goptions:"-k, --keep-files, description='Dont delete files when done'"`
		Help      goptions.Help `goptions:"-h, --help, description='Show this help'"`
	}{
		Cores:     runtime.NumCPU(),
		Threads:   runtime.NumCPU(),
		Blocksize: &DefaultBlocksize,
		OutputDir: os.TempDir(),
	}
	Timestamp = fmt.Sprintf("%d", time.Now().UnixNano())
)

func init() {
	goptions.ParseAndFail(&options)
	options.OutputDir = filepath.Clean(options.OutputDir + "/iowhip_" + Timestamp)
}

type Result struct {
	Index       int
	TotalTime   time.Duration
	WritingTime time.Duration
}

func main() {
	err := os.MkdirAll(options.OutputDir, os.FileMode(0755))
	if err != nil {
		log.Fatalf("Could not create output directory %s: %s", options.OutputDir, err)
	}
	cleanup := make(chan os.Signal)
	signal.Notify(cleanup, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGINT)
	go func() {
		<-cleanup
		if !options.KeepFiles {
			os.RemoveAll(options.OutputDir)
		}
	}()

	runtime.GOMAXPROCS(options.Cores)
	log.Printf("Starting %d workers on %d cores writing %s bytes (%s per call)...", options.Threads, options.Cores, options.Filesize, options.Blocksize)

	results := make(chan Result)
	wg := &sync.WaitGroup{}
	wg.Add(options.Threads)
	for i := 0; i < options.Threads; i++ {
		go writeFile(i, results, wg)
	}
	wg.Wait()

	prettyPrint(results)
	cleanup <- syscall.SIGINT
}

func prettyPrint(results <-chan Result) {
	w := tabwriter.NewWriter(os.Stdout, 3, 1, 2, ' ', tabwriter.AlignRight)
	defer w.Flush()
	fmt.Fprintf(w, "\nThread\tTotal Time\tTotal Speed\tWriting Time\tWriting Speed\t\n")
	for i := 0; i < options.Threads; i++ {
		r := <-results
		fmt.Fprintf(w, "%d\t%s\t%s/s\t%s\t%s/s\t\n", r.Index,
			r.TotalTime, Datasize(int64(float64(*options.Filesize)*float64(time.Second)/float64(r.TotalTime))),
			r.WritingTime, Datasize(int64(float64(*options.Filesize)*float64(time.Second)/float64(r.WritingTime))))
	}
}

func writeFile(idx int, c chan Result, wg *sync.WaitGroup) {
	result := Result{
		Index: idx,
	}
	defer func() { c <- result }()
	defer wg.Done()
	filename := fmt.Sprintf("%s/%d", options.OutputDir, idx)
	log.Printf("Thread %d: Using %s", idx, filename)

	amount := *options.Filesize
	data := make([]byte, int(*options.Blocksize))
	start_creating := time.Now()
	f, err := os.Create(filename)
	if err != nil {
		log.Printf("Thread %d: Could not open file %s: %s", idx, filename, err)
		return
	}
	defer f.Close()

	start_writing := time.Now()
	for amount > 0 {
		n, err := f.Write(data)
		if err != nil {
			log.Printf("Thread %d: Write to %s failed: %s", idx, filename, err)
			return
		}
		amount -= Datasize(n)
		if options.Sync {
			f.Sync()
		}
	}
	f.Sync()
	result.WritingTime = time.Since(start_writing)
	f.Close()
	result.TotalTime = time.Since(start_creating)
}
<|endoftext|>