<commit_before>package parsing

import (
	"encoding/json"
	"fmt"
	"log"
	"reflect"
	"strconv"
	"strings"
	"unicode"
    "golang.org/x/text/unicode/rangetable"
	"runtime/debug"
)

func marshError(input interface{}, stage string, err error) {
	if err != nil {
		fmt.Println(input)
		fmt.Println(stage)
		debug.PrintStack()
		log.Fatal("Remashalling error! ", err)

	}
}

func Remarshal(input interface{}) KeyValue {
	// This is just a nasty type conversions, marshals an interface and then back into our Keyvalue map type
	var back KeyValue
	out, e := json.Marshal(input)
	marshError(input, "Marshal", e)
	e = json.Unmarshal([]byte(out), &back)
	marshError(input, "Unmarshal", e)
	return back
}

// GetSliceOfKeys creates a slice of keys from an object
func GetSliceOfKeys(input KeyValue) []string {
	// Creates an array of key names given a Keyvalue map
	var r []string
	for key := range input {
		r = append(r, key)
	}
	return r
}

// CreatePath: Given an array, construct it into a jmespath expression (string with . separator)
func CreatePath(input []string) string {
	var r string
	for i := range input {
		str := input[i]
		// Escape a . in string name for parsing later
		str = strings.Replace(str, ".", "\\.", -1)
		if i == (len(input) - 1) {
			r = r + str
		} else {
			r = r + str + "."
		}
	}
	return r
}

// IndexOf: Finds index of an object in a given array
func IndexOf(inputList []string, inputKey string) int {
	for i, v := range inputList {
		if v == inputKey {
			return i
		}
	}
	return -1
}

// UnorderedKeyMatch: Returns a bool dependant on all 'keys' in a map matching.
func UnorderedKeyMatch(o map[string]interface{}, m map[string]interface{}) bool {
	istanbool := true
	oSlice := GetSliceOfKeys(o)
	mSlice := GetSliceOfKeys(m)
	for k := range oSlice {
		val := IndexOf(mSlice, oSlice[k])
		if val == -1 {
			istanbool = false
		}
	}

	for k := range mSlice {
		val := IndexOf(oSlice, mSlice[k])
		if val == -1 {
			istanbool = false
		}
	}
	return istanbool
}

// SliceIndex: Adds an 'index' value to the last string in the slice, used for the 'path' to handle arrays.
func SliceIndex(i int, path []string) []string {

	nPath := make([]string, len(path))
	copy(nPath, path)
	iter := len(nPath) - 1
	nPath[iter] = nPath[iter] + "[" + strconv.Itoa(i) + "]"
	return nPath
}

func MatchAny(compare interface{}, compareSlice []interface{}) bool {
	for i := range compareSlice {
		if reflect.DeepEqual(compare, compareSlice[i]) {
			return true
		}
	}
	return false
}

// DoMapArrayKeysMatch: Uses 'UnorderedKeyMatch' to return a bool for two interfaces if they're both maps
func DoMapArrayKeysMatch(o interface{}, m interface{}) bool {
	if reflect.TypeOf(o).Kind() == reflect.Map && reflect.TypeOf(m).Kind() == reflect.Map {
		return UnorderedKeyMatch(Remarshal(o), Remarshal(m))
	}
	return false
}

// PathSplit: Splits up jmespath format path into a slice, will ignore escaped '.' ; opposite of CreatePath
func PathSplit(input string) []string {

	str := escape(input)
	for i := range str {
		str[i] = strings.Replace(str[i], "\\.", ".", -1)
	}
	return str
}

func escape(input string) []string {
	slashRange := rangetable.New(rune('\\'))
	dotRange := rangetable.New(rune('.'))
	old := rune(0)
	f := func(c rune) bool {
		switch {
		case old == rune('\\'):
			old = rune(0)
			return false
		case old != rune(0):
			return false
		case unicode.In(c, slashRange):
			old = c
			return false
		default:
			return  unicode.In(c, dotRange)

		}
	}
	return strings.FieldsFunc(input, f)

}

// \ = U+005C
// . = U+002E<commit_msg>fixing up comments<commit_after>package parsing

import (
	"encoding/json"
	"fmt"
	"log"
	"reflect"
	"strconv"
	"strings"
	"unicode"
    "golang.org/x/text/unicode/rangetable"
	"runtime/debug"
)

func marshError(input interface{}, stage string, err error) {
	if err != nil {
		fmt.Println(input)
		fmt.Println(stage)
		debug.PrintStack()
		log.Fatal("Remashalling error! ", err)

	}
}

// Remarshal deprecated
func Remarshal(input interface{}) KeyValue {
	// This is just a nasty type conversions, marshals an interface and then back into our Keyvalue map type
	var back KeyValue
	out, e := json.Marshal(input)
	marshError(input, "Marshal", e)
	e = json.Unmarshal([]byte(out), &back)
	marshError(input, "Unmarshal", e)
	return back
}

// GetSliceOfKeys creates a slice of keys from an object
func GetSliceOfKeys(input KeyValue) []string {
	// Creates an array of key names given a Keyvalue map
	var r []string
	for key := range input {
		r = append(r, key)
	}
	return r
}

// CreatePath: Given an array, construct it into a jmespath expression (string with . separator)
func CreatePath(input []string) string {
	var r string
	for i := range input {
		str := input[i]
		// Escape a . in string name for parsing later
		str = strings.Replace(str, ".", "\\.", -1)
		if i == (len(input) - 1) {
			r = r + str
		} else {
			r = r + str + "."
		}
	}
	return r
}

// IndexOf: Finds index of an object in a given array
func IndexOf(inputList []string, inputKey string) int {
	for i, v := range inputList {
		if v == inputKey {
			return i
		}
	}
	return -1
}

// UnorderedKeyMatch: Returns a bool dependant on all 'keys' in a map matching.
func UnorderedKeyMatch(o map[string]interface{}, m map[string]interface{}) bool {
	istanbool := true
	oSlice := GetSliceOfKeys(o)
	mSlice := GetSliceOfKeys(m)
	for k := range oSlice {
		val := IndexOf(mSlice, oSlice[k])
		if val == -1 {
			istanbool = false
		}
	}

	for k := range mSlice {
		val := IndexOf(oSlice, mSlice[k])
		if val == -1 {
			istanbool = false
		}
	}
	return istanbool
}

// SliceIndex Adds an 'index' value to the last string in the slice, used for the 'path' to handle arrays.
func SliceIndex(i int, path []string) []string {

	nPath := make([]string, len(path))
	copy(nPath, path)
	iter := len(nPath) - 1
	nPath[iter] = nPath[iter] + "[" + strconv.Itoa(i) + "]"
	return nPath
}

func MatchAny(compare interface{}, compareSlice []interface{}) bool {
	for i := range compareSlice {
		if reflect.DeepEqual(compare, compareSlice[i]) {
			return true
		}
	}
	return false
}

// DoMapArrayKeysMatch Uses 'UnorderedKeyMatch' to return a bool for two interfaces if they're both maps
func DoMapArrayKeysMatch(o interface{}, m interface{}) bool {
	if reflect.TypeOf(o).Kind() == reflect.Map && reflect.TypeOf(m).Kind() == reflect.Map {
		return UnorderedKeyMatch(Remarshal(o), Remarshal(m))
	}
	return false
}

// PathSplit Splits up jmespath format path into a slice, will ignore escaped '.' ; opposite of CreatePath
func PathSplit(input string) []string {

	str := escape(input)
	for i := range str {
		str[i] = strings.Replace(str[i], "\\.", ".", -1)
	}
	return str
}

func escape(input string) []string {
	slashRange := rangetable.New(rune('\\'))
	dotRange := rangetable.New(rune('.'))
	old := rune(0)
	f := func(c rune) bool {
		switch {
		case old == rune('\\'):
			old = rune(0)
			return false
		case old != rune(0):
			return false
		case unicode.In(c, slashRange):
			old = c
			return false
		default:
			return  unicode.In(c, dotRange)

		}
	}
	return strings.FieldsFunc(input, f)

}

// \ = U+005C
// . = U+002E<|endoftext|>


<commit_before>// Copyright 2019, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This test is too large for the race detector.  This SDK uses no locks
// that the race detector would help with, anyway.
// +build !race

package metric_test

import (
	"context"
	"math/rand"
	"testing"
	"time"

	"go.opentelemetry.io/otel/api/core"
	"go.opentelemetry.io/otel/sdk/export/metric"
	"go.opentelemetry.io/otel/sdk/metric/aggregator/histogram"
)

func TestStressInt64Histogram(t *testing.T) {
	desc := metric.NewDescriptor("some_metric", metric.MeasureKind, nil, "", "", core.Int64NumberKind)
	h := histogram.New(desc, []core.Number{core.NewInt64Number(25), core.NewInt64Number(50), core.NewInt64Number(75)})

	go func() {
		rnd := rand.New(rand.NewSource(time.Now().Unix()))
		for {
			_ = h.Update(context.Background(), core.NewInt64Number(rnd.Int63()), desc)
		}
	}()

	startTime := time.Now()
	for time.Since(startTime) < time.Second {
		h.Checkpoint(context.Background(), desc)

		b, _ := h.Histogram()
		c, _ := h.Count()

		var realCount int64
		for _, c := range b.Counts {
			v := c.AsInt64()
			realCount += v
		}

		if realCount != c {
			t.Fail()
		}
	}
}
<commit_msg>Eliminate goroutine leak in histogram stress test (#547)<commit_after>// Copyright 2020, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This test is too large for the race detector.  This SDK uses no locks
// that the race detector would help with, anyway.
// +build !race

package metric_test

import (
	"context"
	"math/rand"
	"testing"
	"time"

	"go.opentelemetry.io/otel/api/core"
	"go.opentelemetry.io/otel/sdk/export/metric"
	"go.opentelemetry.io/otel/sdk/metric/aggregator/histogram"
)

func TestStressInt64Histogram(t *testing.T) {
	desc := metric.NewDescriptor("some_metric", metric.MeasureKind, nil, "", "", core.Int64NumberKind)
	h := histogram.New(desc, []core.Number{core.NewInt64Number(25), core.NewInt64Number(50), core.NewInt64Number(75)})

	ctx, cancelFunc := context.WithCancel(context.Background())
	defer cancelFunc()
	go func() {
		rnd := rand.New(rand.NewSource(time.Now().Unix()))
		for {
			select {
			case <-ctx.Done():
				return
			default:
				_ = h.Update(ctx, core.NewInt64Number(rnd.Int63()%100), desc)
			}
		}
	}()

	startTime := time.Now()
	for time.Since(startTime) < time.Second {
		h.Checkpoint(context.Background(), desc)

		b, _ := h.Histogram()
		c, _ := h.Count()

		var realCount int64
		for _, c := range b.Counts {
			v := c.AsInt64()
			realCount += v
		}

		if realCount != c {
			t.Fail()
		}
	}
}
<|endoftext|>


<commit_before>package persist

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os"
	"strings"

	"gitlab.com/NebulousLabs/Sia/build"
	"gitlab.com/NebulousLabs/Sia/crypto"

	"gitlab.com/NebulousLabs/errors"
)

// readJSON will try to read a persisted json object from a file.
func readJSON(meta Metadata, object interface{}, filename string) error {
	// Open the file.
	file, err := os.Open(filename)
	if os.IsNotExist(err) {
		return err
	}
	if err != nil {
		return build.ExtendErr("unable to open persisted json object file", err)
	}
	defer file.Close()

	// Read the metadata from the file.
	var header, version string
	dec := json.NewDecoder(file)
	if err := dec.Decode(&header); err != nil {
		return build.ExtendErr("unable to read header from persisted json object file", err)
	}
	if header != meta.Header {
		return ErrBadHeader
	}
	if err := dec.Decode(&version); err != nil {
		return build.ExtendErr("unable to read version from persisted json object file", err)
	}
	if version != meta.Version {
		return ErrBadVersion
	}

	// Read everything else.
	remainingBytes, err := ioutil.ReadAll(dec.Buffered())
	if err != nil {
		return build.ExtendErr("unable to read persisted json object data", err)
	}
	// The buffer may or may not have read the rest of the file, read the rest
	// of the file to be certain.
	remainingBytesExtra, err := ioutil.ReadAll(file)
	if err != nil {
		return build.ExtendErr("unable to read persisted json object data", err)
	}
	remainingBytes = append(remainingBytes, remainingBytesExtra...)

	// Determine whether the leading bytes contain a checksum. A proper checksum
	// will be 67 bytes (quote, 64 byte checksum, quote, newline). A manual
	// checksum will be the characters "manual\n" (9 characters). If neither
	// decode correctly, it is assumed that there is no checksum at all.
	var checksum crypto.Hash
	err = json.Unmarshal(remainingBytes[:67], &checksum)
	if err == nil && checksum == crypto.HashBytes(remainingBytes[68:]) {
		// Checksum is proper, and matches the data. Update the data portion to
		// exclude the checksum.
		remainingBytes = remainingBytes[68:]
	} else {
		// Cryptographic checksum failed, try interpreting a manual checksum.
		var manualChecksum string
		err := json.Unmarshal(remainingBytes[:8], &manualChecksum)
		if err == nil && manualChecksum == "manual" {
			// Manual checksum is proper. Update the remaining data to exclude
			// the manual checksum.
			remainingBytes = remainingBytes[9:]
		}
	}

	// Any valid checksum has been stripped off. There is also the case that no
	// checksum was written at all, which is ignored as a case - it's needed to
	// preserve compatibility with previous persist files.

	// Parse the json object.
	return json.Unmarshal(remainingBytes, &object)
}

// LoadJSON will load a persisted json object from disk.
func LoadJSON(meta Metadata, object interface{}, filename string) error {
	// Verify that the filename does not have the persist temp suffix.
	if strings.HasSuffix(filename, tempSuffix) {
		return ErrBadFilenameSuffix
	}

	// Verify that no other thread is using this filename.
	err := func() error {
		activeFilesMu.Lock()
		defer activeFilesMu.Unlock()

		_, exists := activeFiles[filename]
		if exists {
			build.Critical(ErrFileInUse, filename)
			return ErrFileInUse
		}
		activeFiles[filename] = struct{}{}
		return nil
	}()
	if err != nil {
		return err
	}
	// Release the lock at the end of the function.
	defer func() {
		activeFilesMu.Lock()
		delete(activeFiles, filename)
		activeFilesMu.Unlock()
	}()

	// Try opening the primary file.
	err = readJSON(meta, object, filename)
	if err == ErrBadHeader || err == ErrBadVersion || os.IsNotExist(err) {
		return err
	}
	if err != nil {
		// Try opening the temp file.
		err := readJSON(meta, object, filename+tempSuffix)
		if err != nil {
			return build.ExtendErr("unable to read persisted json object from disk", err)
		}
	}

	// Success.
	return nil
}

// SaveJSON will save a json object to disk in a durable, atomic way. The
// resulting file will have a checksum of the data as the third line. If
// manually editing files, the checksum line can be replaced with the 8
// characters "manual". This will cause the reader to accept the checksum even
// though the file has been changed.
func SaveJSON(meta Metadata, object interface{}, filename string) error {
	// Verify that the filename does not have the persist temp suffix.
	if strings.HasSuffix(filename, tempSuffix) {
		return ErrBadFilenameSuffix
	}

	// Verify that no other thread is using this filename.
	err := func() error {
		activeFilesMu.Lock()
		defer activeFilesMu.Unlock()

		_, exists := activeFiles[filename]
		if exists {
			build.Critical(ErrFileInUse, filename)
			return ErrFileInUse
		}
		activeFiles[filename] = struct{}{}
		return nil
	}()
	if err != nil {
		return err
	}
	// Release the lock at the end of the function.
	defer func() {
		activeFilesMu.Lock()
		delete(activeFiles, filename)
		activeFilesMu.Unlock()
	}()

	// Write the metadata to the buffer.
	buf := new(bytes.Buffer)
	enc := json.NewEncoder(buf)
	if err := enc.Encode(meta.Header); err != nil {
		return build.ExtendErr("unable to encode metadata header", err)
	}
	if err := enc.Encode(meta.Version); err != nil {
		return build.ExtendErr("unable to encode metadata version", err)
	}

	// Marshal the object into json and write the checksum + result to the
	// buffer.
	objBytes, err := json.MarshalIndent(object, "", "\t")
	if err != nil {
		return build.ExtendErr("unable to marshal the provided object", err)
	}
	checksum := crypto.HashBytes(objBytes)
	if err := enc.Encode(checksum); err != nil {
		return build.ExtendErr("unable to encode checksum", err)
	}
	buf.Write(objBytes)

	// Write out the data to the temp file, with a sync.
	data := buf.Bytes()
	err = func() (err error) {
		file, err := os.OpenFile(filename+tempSuffix, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0600)
		if err != nil {
			return build.ExtendErr("unable to open temp file", err)
		}
		defer func() {
			err = build.ComposeErrors(err, file.Close())
		}()

		// If the data is greater than the file, we truncate it first to make
		// sure we don't run out of disk space mid-write.
		fi, err := file.Stat()
		if err != nil {
			return errors.AddContext(err, "failed to get FileInfo")
		}
		if int64(len(data)) > fi.Size() {
			if err := file.Truncate(int64(len(data))); err != nil {
				return errors.AddContext(err, "failed to reserve space for file")
			}
		}

		// Write and sync.
		_, err = file.Write(data)
		if err != nil {
			return build.ExtendErr("unable to write temp file", err)
		}
		err = file.Sync()
		if err != nil {
			return build.ExtendErr("unable to sync temp file", err)
		}
		return nil
	}()
	if err != nil {
		return err
	}

	// Write out the data to the real file, with a sync.
	err = func() (err error) {
		file, err := os.OpenFile(filename, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0600)
		if err != nil {
			return build.ExtendErr("unable to open file", err)
		}
		defer func() {
			err = build.ComposeErrors(err, file.Close())
		}()

		// If the data is greater than the file, we truncate it first to make
		// sure we don't run out of disk space mid-write.
		fi, err := file.Stat()
		if err != nil {
			return errors.AddContext(err, "failed to get FileInfo")
		}
		if int64(len(data)) > fi.Size() {
			if err := file.Truncate(int64(len(data))); err != nil {
				return errors.AddContext(err, "failed to reserve space for file")
			}
		}

		// Write and sync.
		_, err = file.Write(data)
		if err != nil {
			return build.ExtendErr("unable to write file", err)
		}
		err = file.Sync()
		if err != nil {
			return build.ExtendErr("unable to sync temp file", err)
		}
		return nil
	}()
	if err != nil {
		return err
	}

	// Success
	return nil
}
<commit_msg>Revert "Merge branch '3153-contractor-persistence-corrupted-when-disk-is-full' into 'master'"<commit_after>package persist

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os"
	"strings"

	"gitlab.com/NebulousLabs/Sia/build"
	"gitlab.com/NebulousLabs/Sia/crypto"
)

// readJSON will try to read a persisted json object from a file.
func readJSON(meta Metadata, object interface{}, filename string) error {
	// Open the file.
	file, err := os.Open(filename)
	if os.IsNotExist(err) {
		return err
	}
	if err != nil {
		return build.ExtendErr("unable to open persisted json object file", err)
	}
	defer file.Close()

	// Read the metadata from the file.
	var header, version string
	dec := json.NewDecoder(file)
	if err := dec.Decode(&header); err != nil {
		return build.ExtendErr("unable to read header from persisted json object file", err)
	}
	if header != meta.Header {
		return ErrBadHeader
	}
	if err := dec.Decode(&version); err != nil {
		return build.ExtendErr("unable to read version from persisted json object file", err)
	}
	if version != meta.Version {
		return ErrBadVersion
	}

	// Read everything else.
	remainingBytes, err := ioutil.ReadAll(dec.Buffered())
	if err != nil {
		return build.ExtendErr("unable to read persisted json object data", err)
	}
	// The buffer may or may not have read the rest of the file, read the rest
	// of the file to be certain.
	remainingBytesExtra, err := ioutil.ReadAll(file)
	if err != nil {
		return build.ExtendErr("unable to read persisted json object data", err)
	}
	remainingBytes = append(remainingBytes, remainingBytesExtra...)

	// Determine whether the leading bytes contain a checksum. A proper checksum
	// will be 67 bytes (quote, 64 byte checksum, quote, newline). A manual
	// checksum will be the characters "manual\n" (9 characters). If neither
	// decode correctly, it is assumed that there is no checksum at all.
	var checksum crypto.Hash
	err = json.Unmarshal(remainingBytes[:67], &checksum)
	if err == nil && checksum == crypto.HashBytes(remainingBytes[68:]) {
		// Checksum is proper, and matches the data. Update the data portion to
		// exclude the checksum.
		remainingBytes = remainingBytes[68:]
	} else {
		// Cryptographic checksum failed, try interpreting a manual checksum.
		var manualChecksum string
		err := json.Unmarshal(remainingBytes[:8], &manualChecksum)
		if err == nil && manualChecksum == "manual" {
			// Manual checksum is proper. Update the remaining data to exclude
			// the manual checksum.
			remainingBytes = remainingBytes[9:]
		}
	}

	// Any valid checksum has been stripped off. There is also the case that no
	// checksum was written at all, which is ignored as a case - it's needed to
	// preserve compatibility with previous persist files.

	// Parse the json object.
	return json.Unmarshal(remainingBytes, &object)
}

// LoadJSON will load a persisted json object from disk.
func LoadJSON(meta Metadata, object interface{}, filename string) error {
	// Verify that the filename does not have the persist temp suffix.
	if strings.HasSuffix(filename, tempSuffix) {
		return ErrBadFilenameSuffix
	}

	// Verify that no other thread is using this filename.
	err := func() error {
		activeFilesMu.Lock()
		defer activeFilesMu.Unlock()

		_, exists := activeFiles[filename]
		if exists {
			build.Critical(ErrFileInUse, filename)
			return ErrFileInUse
		}
		activeFiles[filename] = struct{}{}
		return nil
	}()
	if err != nil {
		return err
	}
	// Release the lock at the end of the function.
	defer func() {
		activeFilesMu.Lock()
		delete(activeFiles, filename)
		activeFilesMu.Unlock()
	}()

	// Try opening the primary file.
	err = readJSON(meta, object, filename)
	if err == ErrBadHeader || err == ErrBadVersion || os.IsNotExist(err) {
		return err
	}
	if err != nil {
		// Try opening the temp file.
		err := readJSON(meta, object, filename+tempSuffix)
		if err != nil {
			return build.ExtendErr("unable to read persisted json object from disk", err)
		}
	}

	// Success.
	return nil
}

// SaveJSON will save a json object to disk in a durable, atomic way. The
// resulting file will have a checksum of the data as the third line. If
// manually editing files, the checksum line can be replaced with the 8
// characters "manual". This will cause the reader to accept the checksum even
// though the file has been changed.
func SaveJSON(meta Metadata, object interface{}, filename string) error {
	// Verify that the filename does not have the persist temp suffix.
	if strings.HasSuffix(filename, tempSuffix) {
		return ErrBadFilenameSuffix
	}

	// Verify that no other thread is using this filename.
	err := func() error {
		activeFilesMu.Lock()
		defer activeFilesMu.Unlock()

		_, exists := activeFiles[filename]
		if exists {
			build.Critical(ErrFileInUse, filename)
			return ErrFileInUse
		}
		activeFiles[filename] = struct{}{}
		return nil
	}()
	if err != nil {
		return err
	}
	// Release the lock at the end of the function.
	defer func() {
		activeFilesMu.Lock()
		delete(activeFiles, filename)
		activeFilesMu.Unlock()
	}()

	// Write the metadata to the buffer.
	buf := new(bytes.Buffer)
	enc := json.NewEncoder(buf)
	if err := enc.Encode(meta.Header); err != nil {
		return build.ExtendErr("unable to encode metadata header", err)
	}
	if err := enc.Encode(meta.Version); err != nil {
		return build.ExtendErr("unable to encode metadata version", err)
	}

	// Marshal the object into json and write the checksum + result to the
	// buffer.
	objBytes, err := json.MarshalIndent(object, "", "\t")
	if err != nil {
		return build.ExtendErr("unable to marshal the provided object", err)
	}
	checksum := crypto.HashBytes(objBytes)
	if err := enc.Encode(checksum); err != nil {
		return build.ExtendErr("unable to encode checksum", err)
	}
	buf.Write(objBytes)

	// Write out the data to the temp file, with a sync.
	data := buf.Bytes()
	err = func() (err error) {
		file, err := os.OpenFile(filename+tempSuffix, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0600)
		if err != nil {
			return build.ExtendErr("unable to open temp file", err)
		}
		defer func() {
			err = build.ComposeErrors(err, file.Close())
		}()

		// Write and sync.
		_, err = file.Write(data)
		if err != nil {
			return build.ExtendErr("unable to write temp file", err)
		}
		err = file.Sync()
		if err != nil {
			return build.ExtendErr("unable to sync temp file", err)
		}
		return nil
	}()
	if err != nil {
		return err
	}

	// Write out the data to the real file, with a sync.
	err = func() (err error) {
		file, err := os.OpenFile(filename, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0600)
		if err != nil {
			return build.ExtendErr("unable to open file", err)
		}
		defer func() {
			err = build.ComposeErrors(err, file.Close())
		}()

		// Write and sync.
		_, err = file.Write(data)
		if err != nil {
			return build.ExtendErr("unable to write file", err)
		}
		err = file.Sync()
		if err != nil {
			return build.ExtendErr("unable to sync temp file", err)
		}
		return nil
	}()
	if err != nil {
		return err
	}

	// Success
	return nil
}
<|endoftext|>


<commit_before>package pixy

import (
	"crypto/rand"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/mailgun/kafka-pixy/Godeps/_workspace/src/github.com/mailgun/log"
	"github.com/mailgun/kafka-pixy/Godeps/_workspace/src/github.com/mailgun/sarama"
	"github.com/mailgun/kafka-pixy/Godeps/_workspace/src/github.com/wvanbergen/kazoo-go"
)

type Config struct {
	// A unix domain socket address that the service should listen at.
	UnixAddr string
	// A TCP address that the service should listen at.
	TCPAddr string
	// A unique id that identifies this particular Kafka-Pixy instance in both
	// Kafka and ZooKeeper.
	ClientID string

	Kafka struct {
		// A list of seed Kafka peers in the form "<host>:<port>" that the
		// service will try to connect to to resolve the cluster topology.
		SeedPeers []string
	}
	ZooKeeper struct {
		// A list of seed ZooKeeper peers in the form "<host>:<port>" that the
		// service will try to connect to to resolve the cluster topology.
		SeedPeers []string
		// The root directory where Kafka keeps all its znodes.
		Chroot string
	}
	Producer struct {
		// The period of time that a proxy should allow to `sarama.Producer` to
		// submit buffered messages to Kafka. It should be large enough to avoid
		// event loss when shutdown is performed during Kafka leader election.
		ShutdownTimeout time.Duration
		// DeadMessageCh is a channel to dump undelivered messages into. It is
		// used in testing only.
		DeadMessageCh chan<- *sarama.ProducerMessage
	}
	Consumer struct {
		// A consume request will wait at most this long until a message from
		// the specified group/topic becomes available. This timeout is
		// necessary to account for consumer rebalancing that happens whenever
		// a new consumer joins a group or subscribes to a topic.
		LongPollingTimeout time.Duration
		// The period of time that a proxy should keep registration with a
		// consumer group or subscription for a topic in the absence of requests
		// to the aforementioned consumer group or topic.
		RegistrationTimeout time.Duration
		// If a request to a KafkaBroker fails for any reason then the proxy
		// should wait this long before retrying.
		BackOffTimeout time.Duration
		// A consumer should wait this long after it gets notification that a
		// consumer joined/left its consumer group before it should rebalance.
		RebalanceDelay time.Duration
	}
	// All buffered channels created by the service will have this size.
	ChannelBufferSize int
	// testing sections contains parameters that are used in tests only.
	testing struct {
		// If this channel is not `nil` then exclusive consumers will use it to
		// notify when they fetch the very first message.
		firstMessageFetchedCh chan *exclusiveConsumer
	}
}

func NewConfig() *Config {
	config := &Config{}
	config.ClientID = newClientID()
	config.ChannelBufferSize = 256

	config.Producer.ShutdownTimeout = 30 * time.Second

	config.Consumer.LongPollingTimeout = 3 * time.Second
	config.Consumer.RegistrationTimeout = 20 * time.Second
	config.Consumer.BackOffTimeout = 500 * time.Millisecond
	config.Consumer.RebalanceDelay = 250 * time.Millisecond

	return config
}

// saramaConfig generates a `Shopify/sarama` library config.
func (c *Config) saramaConfig() *sarama.Config {
	saramaConfig := sarama.NewConfig()
	saramaConfig.ClientID = c.ClientID
	saramaConfig.ChannelBufferSize = c.ChannelBufferSize

	saramaConfig.Producer.RequiredAcks = sarama.WaitForAll
	saramaConfig.Producer.Return.Successes = true
	saramaConfig.Producer.Return.Errors = true
	saramaConfig.Producer.Compression = sarama.CompressionSnappy
	saramaConfig.Producer.Retry.Backoff = 4 * time.Second
	saramaConfig.Producer.Retry.Max = 5
	saramaConfig.Producer.Flush.Frequency = 500 * time.Millisecond
	saramaConfig.Producer.Flush.Bytes = 1024 * 1024

	saramaConfig.Consumer.Offsets.CommitInterval = 50 * time.Millisecond
	saramaConfig.Consumer.Retry.Backoff = c.Consumer.BackOffTimeout
	saramaConfig.Consumer.Fetch.Default = 512000

	return saramaConfig
}

// saramaConfig generates a `wvanbergen/kazoo-go` library config.
func (c *Config) kazooConfig() *kazoo.Config {
	kazooConfig := kazoo.NewConfig()
	kazooConfig.Chroot = c.ZooKeeper.Chroot
	// ZooKeeper documentation says following about the session timeout: "The
	// current (ZooKeeper) implementation requires that the timeout be a
	// minimum of 2 times the tickTime (as set in the server configuration) and
	// a maximum of 20 times the tickTime". The default tickTime is 2 seconds.
	//
	// See http://zookeeper.apache.org/doc/trunk/zookeeperProgrammers.html#ch_zkSessions
	kazooConfig.Timeout = 15 * time.Second
	return kazooConfig
}

type Service struct {
	producer   *GracefulProducer
	consumer   *SmartConsumer
	admin      *Admin
	unixServer *HTTPAPIServer
	tcpServer  *HTTPAPIServer
	quitCh     chan struct{}
	wg         sync.WaitGroup
}

func SpawnService(config *Config) (*Service, error) {
	producer, err := SpawnGracefulProducer(config)
	if err != nil {
		return nil, fmt.Errorf("failed to spawn producer, err=(%s)", err)
	}
	consumer, err := SpawnSmartConsumer(config)
	if err != nil {
		return nil, fmt.Errorf("failed to spawn consumer, err=(%s)", err)
	}
	admin, err := SpawnAdmin(config)
	if err != nil {
		return nil, fmt.Errorf("failed to spawn admin, err=(%s)", err)
	}
	unixServer, err := NewHTTPAPIServer(NetworkUnix, config.UnixAddr, producer, consumer, admin)
	if err != nil {
		producer.Stop()
		return nil, fmt.Errorf("failed to start Unix socket based HTTP API, err=(%s)", err)
	}
	var tcpServer *HTTPAPIServer
	if config.TCPAddr != "" {
		tcpServer, err = NewHTTPAPIServer(NetworkTCP, config.TCPAddr, producer, consumer, admin)
		if err != nil {
			producer.Stop()
			return nil, fmt.Errorf("failed to start TCP socket based HTTP API, err=(%s)", err)
		}
	}
	s := &Service{
		producer:   producer,
		consumer:   consumer,
		admin:      admin,
		unixServer: unixServer,
		tcpServer:  tcpServer,
		quitCh:     make(chan struct{}),
	}
	spawn(&s.wg, s.supervisor)
	return s, nil
}

func (s *Service) Stop() {
	close(s.quitCh)
	s.wg.Wait()
}

// supervisor takes care of the service graceful shutdown.
func (s *Service) supervisor() {
	defer sarama.RootCID.NewChild("supervisor").LogScope()()
	var tcpServerErrorCh <-chan error

	s.unixServer.Start()
	if s.tcpServer != nil {
		s.tcpServer.Start()
		tcpServerErrorCh = s.tcpServer.ErrorCh()
	}
	// Block to wait for quit signal or an API server crash.
	select {
	case <-s.quitCh:
	case err, ok := <-s.unixServer.ErrorCh():
		if ok {
			log.Errorf("Unix socket based HTTP API crashed, err=(%s)", err)
		}
	case err, ok := <-tcpServerErrorCh:
		if ok {
			log.Errorf("TCP socket based HTTP API crashed, err=(%s)", err)
		}
	}
	// Initiate stop of all API servers.
	s.unixServer.AsyncStop()
	if s.tcpServer != nil {
		s.tcpServer.AsyncStop()
	}
	// Wait until all API servers are stopped.
	for range s.unixServer.ErrorCh() {
		// Drain the errors channel until it is closed.
	}
	if s.tcpServer != nil {
		for range s.tcpServer.ErrorCh() {
			// Drain the errors channel until it is closed.
		}
	}
	// There are no more requests in flight at this point so it is safe to stop
	// all Kafka clients.
	var wg sync.WaitGroup
	spawn(&wg, s.producer.Stop)
	spawn(&wg, s.consumer.Stop)
	spawn(&wg, s.admin.Stop)
	wg.Wait()
}

// newClientID creates a unique id that identifies this particular Kafka-Pixy
// in both Kafka and ZooKeeper.
func newClientID() string {
	hostname, err := os.Hostname()
	if err != nil {
		ip, err := getIP()
		if err != nil {
			buffer := make([]byte, 8)
			_, _ = rand.Read(buffer)
			hostname = fmt.Sprintf("%X", buffer)

		} else {
			hostname = ip.String()
		}
	}
	timestamp := time.Now().UTC().Format(time.RFC3339)
	return fmt.Sprintf("pixy_%s_%d_%s", hostname, os.Getpid(), timestamp)
}
<commit_msg>Increase the default fetch size<commit_after>package pixy

import (
	"crypto/rand"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/mailgun/kafka-pixy/Godeps/_workspace/src/github.com/mailgun/log"
	"github.com/mailgun/kafka-pixy/Godeps/_workspace/src/github.com/mailgun/sarama"
	"github.com/mailgun/kafka-pixy/Godeps/_workspace/src/github.com/wvanbergen/kazoo-go"
)

type Config struct {
	// A unix domain socket address that the service should listen at.
	UnixAddr string
	// A TCP address that the service should listen at.
	TCPAddr string
	// A unique id that identifies this particular Kafka-Pixy instance in both
	// Kafka and ZooKeeper.
	ClientID string

	Kafka struct {
		// A list of seed Kafka peers in the form "<host>:<port>" that the
		// service will try to connect to to resolve the cluster topology.
		SeedPeers []string
	}
	ZooKeeper struct {
		// A list of seed ZooKeeper peers in the form "<host>:<port>" that the
		// service will try to connect to to resolve the cluster topology.
		SeedPeers []string
		// The root directory where Kafka keeps all its znodes.
		Chroot string
	}
	Producer struct {
		// The period of time that a proxy should allow to `sarama.Producer` to
		// submit buffered messages to Kafka. It should be large enough to avoid
		// event loss when shutdown is performed during Kafka leader election.
		ShutdownTimeout time.Duration
		// DeadMessageCh is a channel to dump undelivered messages into. It is
		// used in testing only.
		DeadMessageCh chan<- *sarama.ProducerMessage
	}
	Consumer struct {
		// A consume request will wait at most this long until a message from
		// the specified group/topic becomes available. This timeout is
		// necessary to account for consumer rebalancing that happens whenever
		// a new consumer joins a group or subscribes to a topic.
		LongPollingTimeout time.Duration
		// The period of time that a proxy should keep registration with a
		// consumer group or subscription for a topic in the absence of requests
		// to the aforementioned consumer group or topic.
		RegistrationTimeout time.Duration
		// If a request to a KafkaBroker fails for any reason then the proxy
		// should wait this long before retrying.
		BackOffTimeout time.Duration
		// A consumer should wait this long after it gets notification that a
		// consumer joined/left its consumer group before it should rebalance.
		RebalanceDelay time.Duration
	}
	// All buffered channels created by the service will have this size.
	ChannelBufferSize int
	// testing sections contains parameters that are used in tests only.
	testing struct {
		// If this channel is not `nil` then exclusive consumers will use it to
		// notify when they fetch the very first message.
		firstMessageFetchedCh chan *exclusiveConsumer
	}
}

func NewConfig() *Config {
	config := &Config{}
	config.ClientID = newClientID()
	config.ChannelBufferSize = 256

	config.Producer.ShutdownTimeout = 30 * time.Second

	config.Consumer.LongPollingTimeout = 3 * time.Second
	config.Consumer.RegistrationTimeout = 20 * time.Second
	config.Consumer.BackOffTimeout = 500 * time.Millisecond
	config.Consumer.RebalanceDelay = 250 * time.Millisecond

	return config
}

// saramaConfig generates a `Shopify/sarama` library config.
func (c *Config) saramaConfig() *sarama.Config {
	saramaConfig := sarama.NewConfig()
	saramaConfig.ClientID = c.ClientID
	saramaConfig.ChannelBufferSize = c.ChannelBufferSize

	saramaConfig.Producer.RequiredAcks = sarama.WaitForAll
	saramaConfig.Producer.Return.Successes = true
	saramaConfig.Producer.Return.Errors = true
	saramaConfig.Producer.Compression = sarama.CompressionSnappy
	saramaConfig.Producer.Retry.Backoff = 4 * time.Second
	saramaConfig.Producer.Retry.Max = 5
	saramaConfig.Producer.Flush.Frequency = 500 * time.Millisecond
	saramaConfig.Producer.Flush.Bytes = 1024 * 1024

	saramaConfig.Consumer.Offsets.CommitInterval = 50 * time.Millisecond
	saramaConfig.Consumer.Retry.Backoff = c.Consumer.BackOffTimeout
	saramaConfig.Consumer.Fetch.Default = 1024 * 1024

	return saramaConfig
}

// saramaConfig generates a `wvanbergen/kazoo-go` library config.
func (c *Config) kazooConfig() *kazoo.Config {
	kazooConfig := kazoo.NewConfig()
	kazooConfig.Chroot = c.ZooKeeper.Chroot
	// ZooKeeper documentation says following about the session timeout: "The
	// current (ZooKeeper) implementation requires that the timeout be a
	// minimum of 2 times the tickTime (as set in the server configuration) and
	// a maximum of 20 times the tickTime". The default tickTime is 2 seconds.
	//
	// See http://zookeeper.apache.org/doc/trunk/zookeeperProgrammers.html#ch_zkSessions
	kazooConfig.Timeout = 15 * time.Second
	return kazooConfig
}

type Service struct {
	producer   *GracefulProducer
	consumer   *SmartConsumer
	admin      *Admin
	unixServer *HTTPAPIServer
	tcpServer  *HTTPAPIServer
	quitCh     chan struct{}
	wg         sync.WaitGroup
}

func SpawnService(config *Config) (*Service, error) {
	producer, err := SpawnGracefulProducer(config)
	if err != nil {
		return nil, fmt.Errorf("failed to spawn producer, err=(%s)", err)
	}
	consumer, err := SpawnSmartConsumer(config)
	if err != nil {
		return nil, fmt.Errorf("failed to spawn consumer, err=(%s)", err)
	}
	admin, err := SpawnAdmin(config)
	if err != nil {
		return nil, fmt.Errorf("failed to spawn admin, err=(%s)", err)
	}
	unixServer, err := NewHTTPAPIServer(NetworkUnix, config.UnixAddr, producer, consumer, admin)
	if err != nil {
		producer.Stop()
		return nil, fmt.Errorf("failed to start Unix socket based HTTP API, err=(%s)", err)
	}
	var tcpServer *HTTPAPIServer
	if config.TCPAddr != "" {
		tcpServer, err = NewHTTPAPIServer(NetworkTCP, config.TCPAddr, producer, consumer, admin)
		if err != nil {
			producer.Stop()
			return nil, fmt.Errorf("failed to start TCP socket based HTTP API, err=(%s)", err)
		}
	}
	s := &Service{
		producer:   producer,
		consumer:   consumer,
		admin:      admin,
		unixServer: unixServer,
		tcpServer:  tcpServer,
		quitCh:     make(chan struct{}),
	}
	spawn(&s.wg, s.supervisor)
	return s, nil
}

func (s *Service) Stop() {
	close(s.quitCh)
	s.wg.Wait()
}

// supervisor takes care of the service graceful shutdown.
func (s *Service) supervisor() {
	defer sarama.RootCID.NewChild("supervisor").LogScope()()
	var tcpServerErrorCh <-chan error

	s.unixServer.Start()
	if s.tcpServer != nil {
		s.tcpServer.Start()
		tcpServerErrorCh = s.tcpServer.ErrorCh()
	}
	// Block to wait for quit signal or an API server crash.
	select {
	case <-s.quitCh:
	case err, ok := <-s.unixServer.ErrorCh():
		if ok {
			log.Errorf("Unix socket based HTTP API crashed, err=(%s)", err)
		}
	case err, ok := <-tcpServerErrorCh:
		if ok {
			log.Errorf("TCP socket based HTTP API crashed, err=(%s)", err)
		}
	}
	// Initiate stop of all API servers.
	s.unixServer.AsyncStop()
	if s.tcpServer != nil {
		s.tcpServer.AsyncStop()
	}
	// Wait until all API servers are stopped.
	for range s.unixServer.ErrorCh() {
		// Drain the errors channel until it is closed.
	}
	if s.tcpServer != nil {
		for range s.tcpServer.ErrorCh() {
			// Drain the errors channel until it is closed.
		}
	}
	// There are no more requests in flight at this point so it is safe to stop
	// all Kafka clients.
	var wg sync.WaitGroup
	spawn(&wg, s.producer.Stop)
	spawn(&wg, s.consumer.Stop)
	spawn(&wg, s.admin.Stop)
	wg.Wait()
}

// newClientID creates a unique id that identifies this particular Kafka-Pixy
// in both Kafka and ZooKeeper.
func newClientID() string {
	hostname, err := os.Hostname()
	if err != nil {
		ip, err := getIP()
		if err != nil {
			buffer := make([]byte, 8)
			_, _ = rand.Read(buffer)
			hostname = fmt.Sprintf("%X", buffer)

		} else {
			hostname = ip.String()
		}
	}
	timestamp := time.Now().UTC().Format(time.RFC3339)
	return fmt.Sprintf("pixy_%s_%d_%s", hostname, os.Getpid(), timestamp)
}
<|endoftext|>


<commit_before>package commandevaluators

import (
	"errors"
	"log"
	"strings"

	"github.com/byuoitav/av-api/base"
	"github.com/byuoitav/av-api/dbo"
	"github.com/byuoitav/configuration-database-microservice/accessors"
	"github.com/byuoitav/event-router-microservice/eventinfrastructure"
)

/*

	Video Switchers are a little different by way of port identification.
	Basically our ports are combinations of input + output

	so 0:0 is the input zero set to output zero. So here we run a split on the ':' to assign the input and output separately.

*/

//ChangeVideoInputVideoswitcher the struct that implements the CommandEvaluation struct
type ChangeVideoInputVideoSwitcher struct {
}

//Evaluate fulfills the CommmandEvaluation evaluate requirement
func (c *ChangeVideoInputVideoSwitcher) Evaluate(room base.PublicRoom) ([]base.ActionStructure, error) {
	actionList := []base.ActionStructure{}

	if len(room.CurrentVideoInput) != 0 {
		devices, err := dbo.GetDevicesByBuildingAndRoomAndRole(room.Building, room.Room, "VideoOut")
		if err != nil {
			return []base.ActionStructure{}, err
		}

		for _, device := range devices {
			action, err := GetSwitcherAndCreateAction(room, device, room.CurrentVideoInput, "ChangeVideoInputVideoSwitcher")
			if err != nil {
				return []base.ActionStructure{}, err
			}
			actionList = append(actionList, action)
		}
	}

	// if there is at least one display
	if len(room.Displays) != 0 {

		// interate through all displays in the room, create an ActionStructure if it has an input
		for _, display := range room.Displays {

			// if the display has an input, create the action
			if len(display.Input) != 0 {
				device, err := dbo.GetDeviceByName(room.Building, room.Room, display.Name)
				if err != nil {
					return []base.ActionStructure{}, err
				}

				action, err := GetSwitcherAndCreateAction(room, device, display.Input, "ChangeVideoInputVideoSwitcher")
				if err != nil {
					return []base.ActionStructure{}, err
				}
				//Undecode the format into the
				actionList = append(actionList, action)
			}
		}

	}

	for _, action := range actionList {
		p := action.Parameters["output"]
		splitP := strings.Split(p, ":")

		if len(splitP) != 2 {
			return actionList, errors.New("Invalid port for a video switcher")
		}

		action.Parameters["input"] = splitP[0]
		action.Parameters["output"] = splitP[1]
	}
	return actionList, nil
}

//GetSwitcherAndCreateAction gets the videoswitcher in a room, matches the destination port to the new port
// and creates an action
func GetSwitcherAndCreateAction(room base.PublicRoom, device accessors.Device, selectedInput string, generatingEvaluator string) (base.ActionStructure, error) {

	switcher, err := dbo.GetDevicesByBuildingAndRoomAndRole(room.Building, room.Room, "VideoSwitcher")
	if err != nil {
		return base.ActionStructure{}, err
	}

	if len(switcher) != 1 {
		return base.ActionStructure{}, errors.New("too many switchers/none available")
	}

	log.Printf("Evaluating device %s for a port connecting %s to %s", switcher[0].GetFullName(), selectedInput, device.GetFullName())
	for _, port := range switcher[0].Ports {

		if port.Destination == device.Name && port.Source == selectedInput {

			m := make(map[string]string)
			m["output"] = port.Name

			eventInfo := eventinfrastructure.EventInfo{
				Type:           eventinfrastructure.CORESTATE,
				EventCause:     eventinfrastructure.USERINPUT,
				Device:         switcher[0].Name,
				EventInfoKey:   "input",
				EventInfoValue: m["output"],
			}

			tempAction := base.ActionStructure{
				Action:              "ChangeInput",
				GeneratingEvaluator: generatingEvaluator,
				Device:              switcher[0],
				Parameters:          m,
				DeviceSpecific:      false,
				Overridden:          false,
				EventLog:            []eventinfrastructure.EventInfo{eventInfo},
			}

			return tempAction, nil
		}
	}

	return base.ActionStructure{}, errors.New("no switcher found with the matching port")
}

//Validate f
func (c *ChangeVideoInputVideoSwitcher) Validate(action base.ActionStructure) error {
	log.Printf("Validating action for command %v", action.Action)

	// check if ChangeInput is a valid name of a command (ok is a bool)
	ok, _ := CheckCommands(action.Device.Commands, "ChangeInput")

	// returns and error if the ChangeInput command doesn't exist or if the command isn't ChangeInput
	if !ok || action.Action != "ChangeInput" {
		log.Printf("ERROR. %s is an invalid command for %s", action.Action, action.Device.Name)
		return errors.New(action.Action + "is not an invalid command for " + action.Device.Name)
	}

	log.Print("done.")
	return nil
}

//GetIncompatibleCommands f
func (c *ChangeVideoInputVideoSwitcher) GetIncompatibleCommands() []string {
	return nil
}
<commit_msg>changing the event stuff<commit_after>package commandevaluators

import (
	"errors"
	"log"
	"strings"

	"github.com/byuoitav/av-api/base"
	"github.com/byuoitav/av-api/dbo"
	"github.com/byuoitav/configuration-database-microservice/accessors"
	"github.com/byuoitav/event-router-microservice/eventinfrastructure"
)

/*

	Video Switchers are a little different by way of port identification.
	Basically our ports are combinations of input + output

	so 0:0 is the input zero set to output zero. So here we run a split on the ':' to assign the input and output separately.

*/

//ChangeVideoInputVideoswitcher the struct that implements the CommandEvaluation struct
type ChangeVideoInputVideoSwitcher struct {
}

//Evaluate fulfills the CommmandEvaluation evaluate requirement
func (c *ChangeVideoInputVideoSwitcher) Evaluate(room base.PublicRoom) ([]base.ActionStructure, error) {
	actionList := []base.ActionStructure{}

	if len(room.CurrentVideoInput) != 0 {
		devices, err := dbo.GetDevicesByBuildingAndRoomAndRole(room.Building, room.Room, "VideoOut")
		if err != nil {
			return []base.ActionStructure{}, err
		}

		for _, device := range devices {
			action, err := GetSwitcherAndCreateAction(room, device, room.CurrentVideoInput, "ChangeVideoInputVideoSwitcher")
			if err != nil {
				return []base.ActionStructure{}, err
			}
			actionList = append(actionList, action)
		}
	}

	// if there is at least one display
	if len(room.Displays) != 0 {

		// interate through all displays in the room, create an ActionStructure if it has an input
		for _, display := range room.Displays {

			// if the display has an input, create the action
			if len(display.Input) != 0 {
				device, err := dbo.GetDeviceByName(room.Building, room.Room, display.Name)
				if err != nil {
					return []base.ActionStructure{}, err
				}

				action, err := GetSwitcherAndCreateAction(room, device, display.Input, "ChangeVideoInputVideoSwitcher")
				if err != nil {
					return []base.ActionStructure{}, err
				}
				//Undecode the format into the
				actionList = append(actionList, action)
			}
		}

	}

	for _, action := range actionList {
		p := action.Parameters["output"]
		splitP := strings.Split(p, ":")

		if len(splitP) != 2 {
			return actionList, errors.New("Invalid port for a video switcher")
		}

		action.Parameters["input"] = splitP[0]
		action.Parameters["output"] = splitP[1]
	}
	return actionList, nil
}

//GetSwitcherAndCreateAction gets the videoswitcher in a room, matches the destination port to the new port
// and creates an action
func GetSwitcherAndCreateAction(room base.PublicRoom, device accessors.Device, selectedInput string, generatingEvaluator string) (base.ActionStructure, error) {

	switcher, err := dbo.GetDevicesByBuildingAndRoomAndRole(room.Building, room.Room, "VideoSwitcher")
	if err != nil {
		return base.ActionStructure{}, err
	}

	if len(switcher) != 1 {
		return base.ActionStructure{}, errors.New("too many switchers/none available")
	}

	log.Printf("Evaluating device %s for a port connecting %s to %s", switcher[0].GetFullName(), selectedInput, device.GetFullName())
	for _, port := range switcher[0].Ports {

		if port.Destination == device.Name && port.Source == selectedInput {

			m := make(map[string]string)
			m["output"] = port.Name

			eventInfo := eventinfrastructure.EventInfo{
				Type:           eventinfrastructure.CORESTATE,
				EventCause:     eventinfrastructure.USERINPUT,
				Device:         device.GetFullName(),
				EventInfoKey:   "input",
				EventInfoValue: selectedInput,
			}

			tempAction := base.ActionStructure{
				Action:              "ChangeInput",
				GeneratingEvaluator: generatingEvaluator,
				Device:              switcher[0],
				Parameters:          m,
				DeviceSpecific:      false,
				Overridden:          false,
				EventLog:            []eventinfrastructure.EventInfo{eventInfo},
			}

			return tempAction, nil
		}
	}

	return base.ActionStructure{}, errors.New("no switcher found with the matching port")
}

//Validate f
func (c *ChangeVideoInputVideoSwitcher) Validate(action base.ActionStructure) error {
	log.Printf("Validating action for command %v", action.Action)

	// check if ChangeInput is a valid name of a command (ok is a bool)
	ok, _ := CheckCommands(action.Device.Commands, "ChangeInput")

	// returns and error if the ChangeInput command doesn't exist or if the command isn't ChangeInput
	if !ok || action.Action != "ChangeInput" {
		log.Printf("ERROR. %s is an invalid command for %s", action.Action, action.Device.Name)
		return errors.New(action.Action + "is not an invalid command for " + action.Device.Name)
	}

	log.Print("done.")
	return nil
}

//GetIncompatibleCommands f
func (c *ChangeVideoInputVideoSwitcher) GetIncompatibleCommands() []string {
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"../nsq"
	"../util"
	"github.com/bmizerany/assert"
	// "io/ioutil"
	// "log"
	"net"
	"testing"
	"time"
)

// exercise the basic operations of the V2 protocol
func TestBasicV2(t *testing.T) {
	// log.SetOutput(ioutil.Discard)
	// defer log.SetOutput(os.Stdout)

	tcpAddr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:5170")
	tcpListener, err := net.Listen("tcp", tcpAddr.String())
	defer tcpListener.Close()

	go TopicFactory(10, ".")
	go util.TcpServer(tcpListener, tcpClientHandler)

	msg := nsq.NewMessage(util.Uuid(), []byte("test body"))
	topic := GetTopic("test_v2")
	topic.PutMessage(msg)

	consumer := nsq.NewConsumer(tcpAddr)

	err = consumer.Connect()
	assert.Equal(t, err, nil)

	err = consumer.Version(nsq.ProtocolV2Magic)
	assert.Equal(t, err, nil)

	err = consumer.WriteCommand(consumer.Subscribe("test_v2", "ch"))
	assert.Equal(t, err, nil)

	err = consumer.WriteCommand(consumer.Ready(1))
	assert.Equal(t, err, nil)

	resp, err := consumer.ReadResponse()
	assert.Equal(t, err, nil)
	frameType, msgInterface, err := consumer.UnpackResponse(resp)
	msgOut := msgInterface.(*nsq.Message)
	assert.Equal(t, frameType, nsq.FrameTypeMessage)
	assert.Equal(t, msgOut.Uuid, msg.Uuid)
	assert.Equal(t, msgOut.Body, msg.Body)
	assert.Equal(t, msgOut.Retries, uint16(1))
}

func TestMultipleConsumerV2(t *testing.T) {
	// log.SetOutput(ioutil.Discard)

	readyChan := make(chan int)
	msgChan := make(chan *nsq.Message)

	tcpAddr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:5170")
	tcpListener, err := net.Listen("tcp", tcpAddr.String())
	defer tcpListener.Close()

	go TopicFactory(10, ".")
	go util.TcpServer(tcpListener, tcpClientHandler)

	msg := nsq.NewMessage(util.Uuid(), []byte("test body"))
	topic := GetTopic("test_multiple_v2")

	for _, i := range []string{"1", "2"} {
		consumer := nsq.NewConsumer(tcpAddr)
		err = consumer.Connect()
		assert.Equal(t, err, nil)

		err = consumer.Version(nsq.ProtocolV2Magic)
		assert.Equal(t, err, nil)

		err = consumer.WriteCommand(consumer.Subscribe("test_multiple_v2", "ch"+i))
		assert.Equal(t, err, nil)

		err = consumer.WriteCommand(consumer.Ready(1))
		assert.Equal(t, err, nil)

		go func() {
			<-time.After(time.Duration(time.Second))
			readyChan <- 1
		}()

		go func(c *nsq.Consumer) {
			resp, _ := c.ReadResponse()
			_, msgInterface, _ := c.UnpackResponse(resp)
			msgChan <- msgInterface.(*nsq.Message)
		}(consumer)
	}

	<-readyChan
	<-readyChan

	topic.PutMessage(msg)

	msgOut := <-msgChan
	assert.Equal(t, msgOut.Uuid, msg.Uuid)
	assert.Equal(t, msgOut.Body, msg.Body)
	assert.Equal(t, msgOut.Retries, uint16(1))
	msgOut = <-msgChan
	assert.Equal(t, msgOut.Uuid, msg.Uuid)
	assert.Equal(t, msgOut.Body, msg.Body)
	assert.Equal(t, msgOut.Retries, uint16(1))
}
<commit_msg>dont rely on time for multiple consumer test<commit_after>package main

import (
	"../nsq"
	"../util"
	"github.com/bmizerany/assert"
	// "io/ioutil"
	// "log"
	"net"
	"testing"
)

// exercise the basic operations of the V2 protocol
func TestBasicV2(t *testing.T) {
	// log.SetOutput(ioutil.Discard)
	// defer log.SetOutput(os.Stdout)

	tcpAddr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:5170")
	tcpListener, err := net.Listen("tcp", tcpAddr.String())
	assert.Equal(t, err, nil)
	defer tcpListener.Close()

	go TopicFactory(10, ".")
	go util.TcpServer(tcpListener, tcpClientHandler)

	msg := nsq.NewMessage(util.Uuid(), []byte("test body"))
	topic := GetTopic("test_v2")
	topic.PutMessage(msg)

	consumer := nsq.NewConsumer(tcpAddr)

	err = consumer.Connect()
	assert.Equal(t, err, nil)

	err = consumer.Version(nsq.ProtocolV2Magic)
	assert.Equal(t, err, nil)

	err = consumer.WriteCommand(consumer.Subscribe("test_v2", "ch"))
	assert.Equal(t, err, nil)

	err = consumer.WriteCommand(consumer.Ready(1))
	assert.Equal(t, err, nil)

	resp, err := consumer.ReadResponse()
	assert.Equal(t, err, nil)
	frameType, msgInterface, err := consumer.UnpackResponse(resp)
	msgOut := msgInterface.(*nsq.Message)
	assert.Equal(t, frameType, nsq.FrameTypeMessage)
	assert.Equal(t, msgOut.Uuid, msg.Uuid)
	assert.Equal(t, msgOut.Body, msg.Body)
	assert.Equal(t, msgOut.Retries, uint16(1))
}

func TestMultipleConsumerV2(t *testing.T) {
	// log.SetOutput(ioutil.Discard)
	// defer log.SetOutput(os.Stdout)

	msgChan := make(chan *nsq.Message)

	tcpAddr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:5170")
	tcpListener, err := net.Listen("tcp", tcpAddr.String())
	assert.Equal(t, err, nil)
	defer tcpListener.Close()

	go TopicFactory(10, ".")
	go util.TcpServer(tcpListener, tcpClientHandler)

	msg := nsq.NewMessage(util.Uuid(), []byte("test body"))
	topic := GetTopic("test_multiple_v2")
	topic.GetChannel("ch1")
	topic.GetChannel("ch2")
	topic.PutMessage(msg)

	for _, i := range []string{"1", "2"} {
		consumer := nsq.NewConsumer(tcpAddr)
		err = consumer.Connect()
		assert.Equal(t, err, nil)

		err = consumer.Version(nsq.ProtocolV2Magic)
		assert.Equal(t, err, nil)

		err = consumer.WriteCommand(consumer.Subscribe("test_multiple_v2", "ch"+i))
		assert.Equal(t, err, nil)

		err = consumer.WriteCommand(consumer.Ready(1))
		assert.Equal(t, err, nil)

		go func(c *nsq.Consumer) {
			resp, _ := c.ReadResponse()
			_, msgInterface, _ := c.UnpackResponse(resp)
			msgChan <- msgInterface.(*nsq.Message)
		}(consumer)
	}

	msgOut := <-msgChan
	assert.Equal(t, msgOut.Uuid, msg.Uuid)
	assert.Equal(t, msgOut.Body, msg.Body)
	assert.Equal(t, msgOut.Retries, uint16(1))
	msgOut = <-msgChan
	assert.Equal(t, msgOut.Uuid, msg.Uuid)
	assert.Equal(t, msgOut.Body, msg.Body)
	assert.Equal(t, msgOut.Retries, uint16(1))
}
<|endoftext|>


<commit_before>/*
   Copyright 2017 GitHub Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package remote

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/github/orchestrator/go/config"
	"github.com/github/orchestrator/go/inst"
	orcos "github.com/github/orchestrator/go/os"
	"github.com/github/orchestrator/go/process"
	"github.com/outbrain/golib/log"
)

func TestRemoteCommandOnInstance(instanceKey *inst.InstanceKey) error {
	sudoCommand := ""
	if config.Config.RemoteSSHCommandUseSudo {
		sudoCommand = "sudo -i"
	}

	tempFile, err := ioutil.TempFile("", "orchestrator-remote-test-")
	if err != nil {
		return err
	}
	defer os.Remove(tempFile.Name())

	randomToken := process.NewToken()

	command := config.Config.RemoteSSHCommand
	command = strings.Replace(command, "{hostname}", instanceKey.Hostname, -1)
	command = fmt.Sprintf("%s '%s echo %s' > %s", command, sudoCommand, randomToken.Hash, tempFile.Name())
	if err := orcos.CommandRun(command, orcos.EmptyEnv); err != nil {
		return err
	}
	bytes, err := ioutil.ReadFile(tempFile.Name())
	if err != nil {
		return err
	}
	if content := strings.TrimSpace(string(bytes)); content != randomToken.Hash {
		return fmt.Errorf("TestRemoteCommandOnInstance: expected %s, got %s", randomToken.Hash, content)
	}
	return nil
}

// AlignViaRelaylogCorrelation will align siblings by applying relaylogs from one to the other, via remote SSH
func AlignViaRelaylogCorrelation(instance, fromInstance *inst.Instance) (*inst.Instance, error) {
	if config.Config.RemoteSSHCommand == "" {
		return instance, fmt.Errorf("RemoteSSHCommand not configured")
	}
	log.Debugf("Testing SSH on %+v", instance.Key)
	if err := TestRemoteCommandOnInstance(&instance.Key); err != nil {
		return instance, err
	}

	log.Debugf("AlignViaRelaylogCorrelation: stopping replication")
	if instance.ReplicaRunning() {
		return instance, log.Errorf("AlignViaRelaylogCorrelation: replication on %+v must not run", instance.Key)
	}
	if fromInstance.ReplicaRunning() {
		return instance, log.Errorf("AlignViaRelaylogCorrelation: replication on %+v must not run", fromInstance.Key)
	}

	log.Debugf("AlignViaRelaylogCorrelation: correlating coordinates of %+v on %+v", instance.Key, fromInstance.Key)
	_, _, nextCoordinates, found, err := inst.CorrelateRelaylogCoordinates(instance, nil, fromInstance)
	if err != nil {
		return instance, err
	}
	if !found {
		return instance, err
	}
	log.Debugf("AlignViaRelaylogCorrelation: correlated next-coordinates are %+v", *nextCoordinates)

	// We now have the correlation info needed to proceed with remote calls
	sudoCommand := ""
	if config.Config.RemoteSSHCommandUseSudo {
		sudoCommand = "sudo -i"
	}

	// Write get-relaylogs script locally
	getRelayLogContentsScriptFile, err := ioutil.TempFile("", "orchestrator-remote-get-relaylogs-content-")
	if err != nil {
		return instance, err
	}
	{
		defer os.Remove(getRelayLogContentsScriptFile.Name())
		script := GetRelayLogContentsScript
		script = strings.Replace(script, "$MAGIC_FIRST_RELAYLOG_FILE", nextCoordinates.LogFile, -1)
		script = strings.Replace(script, "$MAGIC_START_POSITION", fmt.Sprintf("%d", nextCoordinates.LogPos), -1)
		if err := ioutil.WriteFile(getRelayLogContentsScriptFile.Name(), []byte(script), 0640); err != nil {
			return instance, err
		}
	}
	log.Debugf("getRelayLogContentsScriptFile: %+v", getRelayLogContentsScriptFile.Name())

	// Get relay log contents, save locally
	localRelayLogContentsFile, err := ioutil.TempFile("", "orchestrator-remote-relaylogs-content-")
	if err != nil {
		return instance, err
	}
	defer os.Remove(localRelayLogContentsFile.Name())
	localRelayLogContentsCopyFileName := fmt.Sprintf("%s.copy", localRelayLogContentsFile.Name())
	{
		command := config.Config.RemoteSSHCommand
		command = strings.Replace(command, "{hostname}", fromInstance.Key.Hostname, -1)
		command = fmt.Sprintf("cat %s | %s '%s' > %s", getRelayLogContentsScriptFile.Name(), command, sudoCommand, localRelayLogContentsFile.Name())
		if err := orcos.CommandRun(command, orcos.EmptyEnv); err != nil {
			return instance, err
		}
	}
	log.Debugf("Have fetched relay logs from %s, output file is %s", fromInstance.Key.Hostname, localRelayLogContentsFile.Name())
	// Copy local relay log contents to target host:
	{
		command := config.Config.RemoteSSHCommand
		command = strings.Replace(command, "{hostname}", instance.Key.Hostname, -1)
		command = fmt.Sprintf("cat %s | %s '%s cat - > %s'", localRelayLogContentsFile.Name(), command, sudoCommand, localRelayLogContentsCopyFileName)
		if err := orcos.CommandRun(command, orcos.EmptyEnv); err != nil {
			return instance, err
		}
	}
	log.Debugf("Have copied contents file to %s, output file is %s", instance.Key.Hostname, localRelayLogContentsFile.Name())

	// Generate the apply-relaylogs script, locally
	applyRelayLogContentsScriptFile, err := ioutil.TempFile("", "orchestrator-remote-apply-relaylogs-content-")
	if err != nil {
		return instance, err
	}
	{
		defer os.Remove(applyRelayLogContentsScriptFile.Name())
		script := ApplyRelayLogContentsScript
		script = strings.Replace(script, "$MAGIC_MYSQL_COMMAND", "", -1)
		script = strings.Replace(script, "$MAGIC_CONTENTS_FILE", localRelayLogContentsCopyFileName, -1)

		if err := ioutil.WriteFile(applyRelayLogContentsScriptFile.Name(), []byte(script), 0640); err != nil {
			return instance, err
		}
	}
	log.Debugf("applyRelayLogContentsScriptFile: %+v", applyRelayLogContentsScriptFile.Name())

	if *config.RuntimeCLIFlags.Noop {
		return instance, fmt.Errorf("noop: Not really applying scripts onto %+v; signalling error but nothing went wrong", instance.Key)
	}

	// apply relaylog contents on target host:
	{
		command := config.Config.RemoteSSHCommand
		command = strings.Replace(command, "{hostname}", instance.Key.Hostname, -1)
		command = fmt.Sprintf("cat %s | %s '%s'", applyRelayLogContentsScriptFile.Name(), command, sudoCommand)
		if err := orcos.CommandRun(command, orcos.EmptyEnv); err != nil {
			return instance, err
		}
	}
	log.Debugf("Have successfully applied relay logs on %s", instance.Key.Hostname)

	instance, err = inst.ChangeMasterTo(&instance.Key, &fromInstance.MasterKey, &fromInstance.ExecBinlogCoordinates, false, inst.GTIDHintNeutral)
	if err != nil {
		return instance, err
	}
	inst.AuditOperation("align-via-relaylogs-remote", &instance.Key, fmt.Sprintf("aligned %+v by relaylogs from %+v", instance.Key, fromInstance.Key))
	return instance, err
}

func SyncReplicasRelayLogs(masterKey *inst.InstanceKey) (syncedReplicas, failedReplicas [](*inst.Instance), err error) {
	var replicas [](*inst.Instance)
	if replicas, err = inst.GetSortedReplicas(masterKey, true); err != nil {
		return syncedReplicas, replicas, err
	}
	if len(replicas) <= 1 {
		// Nothing to be done
		return syncedReplicas, replicas, err
	}
	applyFromReplica := replicas[0]
	applyToReplicas := replicas[1:]

	log.Debugf("Testing SSH on %+v", applyFromReplica.Key)
	if err := TestRemoteCommandOnInstance(&applyFromReplica.Key); err != nil {
		return syncedReplicas, replicas, err
	}

	defer inst.StartSlave(&applyFromReplica.Key)

	barrier := make(chan *inst.InstanceKey)
	allErrors := make(chan error, len(applyToReplicas))
	synchedReplicasChan := make(chan *inst.Instance, len(applyToReplicas))
	failedReplicasChan := make(chan *inst.Instance, len(applyToReplicas))
	for _, applyToReplica := range applyToReplicas {
		applyToReplica := applyToReplica
		go func() {
			defer func() {
				defer func() { barrier <- &applyToReplica.Key }()
				inst.StartSlave(&applyToReplica.Key)
			}()
			if _, err := AlignViaRelaylogCorrelation(applyToReplica, applyFromReplica); err == nil {
				synchedReplicasChan <- applyToReplica
			} else {
				failedReplicasChan <- applyToReplica
				allErrors <- err
			}
		}()
	}
	for range applyToReplicas {
		<-barrier
	}
	syncedReplicas = append(syncedReplicas, applyFromReplica)
	for len(synchedReplicasChan) > 0 {
		syncedReplicas = append(syncedReplicas, <-synchedReplicasChan)
	}
	for len(synchedReplicasChan) > 0 {
		failedReplicas = append(failedReplicas, <-failedReplicasChan)
	}
	countErrors := len(allErrors)
	for len(allErrors) > 0 {
		log.Errore(<-allErrors)
	}
	inst.AuditOperation("sync-replicas-relaylogs", masterKey, fmt.Sprintf("aligned %+v replicas by relaylogs from %+v, got %+v errors", len(applyToReplicas), applyFromReplica.Key, countErrors))
	return syncedReplicas, failedReplicas, err
}
<commit_msg>fixed failedReplicasChan iteration<commit_after>/*
   Copyright 2017 GitHub Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package remote

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/github/orchestrator/go/config"
	"github.com/github/orchestrator/go/inst"
	orcos "github.com/github/orchestrator/go/os"
	"github.com/github/orchestrator/go/process"
	"github.com/outbrain/golib/log"
)

func TestRemoteCommandOnInstance(instanceKey *inst.InstanceKey) error {
	sudoCommand := ""
	if config.Config.RemoteSSHCommandUseSudo {
		sudoCommand = "sudo -i"
	}

	tempFile, err := ioutil.TempFile("", "orchestrator-remote-test-")
	if err != nil {
		return err
	}
	defer os.Remove(tempFile.Name())

	randomToken := process.NewToken()

	command := config.Config.RemoteSSHCommand
	command = strings.Replace(command, "{hostname}", instanceKey.Hostname, -1)
	command = fmt.Sprintf("%s '%s echo %s' > %s", command, sudoCommand, randomToken.Hash, tempFile.Name())
	if err := orcos.CommandRun(command, orcos.EmptyEnv); err != nil {
		return err
	}
	bytes, err := ioutil.ReadFile(tempFile.Name())
	if err != nil {
		return err
	}
	if content := strings.TrimSpace(string(bytes)); content != randomToken.Hash {
		return fmt.Errorf("TestRemoteCommandOnInstance: expected %s, got %s", randomToken.Hash, content)
	}
	return nil
}

// AlignViaRelaylogCorrelation will align siblings by applying relaylogs from one to the other, via remote SSH
func AlignViaRelaylogCorrelation(instance, fromInstance *inst.Instance) (*inst.Instance, error) {
	if config.Config.RemoteSSHCommand == "" {
		return instance, fmt.Errorf("RemoteSSHCommand not configured")
	}
	log.Debugf("Testing SSH on %+v", instance.Key)
	if err := TestRemoteCommandOnInstance(&instance.Key); err != nil {
		return instance, err
	}

	log.Debugf("AlignViaRelaylogCorrelation: stopping replication")
	if instance.ReplicaRunning() {
		return instance, log.Errorf("AlignViaRelaylogCorrelation: replication on %+v must not run", instance.Key)
	}
	if fromInstance.ReplicaRunning() {
		return instance, log.Errorf("AlignViaRelaylogCorrelation: replication on %+v must not run", fromInstance.Key)
	}

	log.Debugf("AlignViaRelaylogCorrelation: correlating coordinates of %+v on %+v", instance.Key, fromInstance.Key)
	_, _, nextCoordinates, found, err := inst.CorrelateRelaylogCoordinates(instance, nil, fromInstance)
	if err != nil {
		return instance, err
	}
	if !found {
		return instance, err
	}
	log.Debugf("AlignViaRelaylogCorrelation: correlated next-coordinates are %+v", *nextCoordinates)

	// We now have the correlation info needed to proceed with remote calls
	sudoCommand := ""
	if config.Config.RemoteSSHCommandUseSudo {
		sudoCommand = "sudo -i"
	}

	// Write get-relaylogs script locally
	getRelayLogContentsScriptFile, err := ioutil.TempFile("", "orchestrator-remote-get-relaylogs-content-")
	if err != nil {
		return instance, err
	}
	{
		defer os.Remove(getRelayLogContentsScriptFile.Name())
		script := GetRelayLogContentsScript
		script = strings.Replace(script, "$MAGIC_FIRST_RELAYLOG_FILE", nextCoordinates.LogFile, -1)
		script = strings.Replace(script, "$MAGIC_START_POSITION", fmt.Sprintf("%d", nextCoordinates.LogPos), -1)
		if err := ioutil.WriteFile(getRelayLogContentsScriptFile.Name(), []byte(script), 0640); err != nil {
			return instance, err
		}
	}
	log.Debugf("getRelayLogContentsScriptFile: %+v", getRelayLogContentsScriptFile.Name())

	// Get relay log contents, save locally
	localRelayLogContentsFile, err := ioutil.TempFile("", "orchestrator-remote-relaylogs-content-")
	if err != nil {
		return instance, err
	}
	defer os.Remove(localRelayLogContentsFile.Name())
	localRelayLogContentsCopyFileName := fmt.Sprintf("%s.copy", localRelayLogContentsFile.Name())
	{
		command := config.Config.RemoteSSHCommand
		command = strings.Replace(command, "{hostname}", fromInstance.Key.Hostname, -1)
		command = fmt.Sprintf("cat %s | %s '%s' > %s", getRelayLogContentsScriptFile.Name(), command, sudoCommand, localRelayLogContentsFile.Name())
		if err := orcos.CommandRun(command, orcos.EmptyEnv); err != nil {
			return instance, err
		}
	}
	log.Debugf("Have fetched relay logs from %s, output file is %s", fromInstance.Key.Hostname, localRelayLogContentsFile.Name())
	// Copy local relay log contents to target host:
	{
		command := config.Config.RemoteSSHCommand
		command = strings.Replace(command, "{hostname}", instance.Key.Hostname, -1)
		command = fmt.Sprintf("cat %s | %s '%s cat - > %s'", localRelayLogContentsFile.Name(), command, sudoCommand, localRelayLogContentsCopyFileName)
		if err := orcos.CommandRun(command, orcos.EmptyEnv); err != nil {
			return instance, err
		}
	}
	log.Debugf("Have copied contents file to %s, output file is %s", instance.Key.Hostname, localRelayLogContentsFile.Name())

	// Generate the apply-relaylogs script, locally
	applyRelayLogContentsScriptFile, err := ioutil.TempFile("", "orchestrator-remote-apply-relaylogs-content-")
	if err != nil {
		return instance, err
	}
	{
		defer os.Remove(applyRelayLogContentsScriptFile.Name())
		script := ApplyRelayLogContentsScript
		script = strings.Replace(script, "$MAGIC_MYSQL_COMMAND", "", -1)
		script = strings.Replace(script, "$MAGIC_CONTENTS_FILE", localRelayLogContentsCopyFileName, -1)

		if err := ioutil.WriteFile(applyRelayLogContentsScriptFile.Name(), []byte(script), 0640); err != nil {
			return instance, err
		}
	}
	log.Debugf("applyRelayLogContentsScriptFile: %+v", applyRelayLogContentsScriptFile.Name())

	if *config.RuntimeCLIFlags.Noop {
		return instance, fmt.Errorf("noop: Not really applying scripts onto %+v; signalling error but nothing went wrong", instance.Key)
	}

	// apply relaylog contents on target host:
	{
		command := config.Config.RemoteSSHCommand
		command = strings.Replace(command, "{hostname}", instance.Key.Hostname, -1)
		command = fmt.Sprintf("cat %s | %s '%s'", applyRelayLogContentsScriptFile.Name(), command, sudoCommand)
		if err := orcos.CommandRun(command, orcos.EmptyEnv); err != nil {
			return instance, err
		}
	}
	log.Debugf("Have successfully applied relay logs on %s", instance.Key.Hostname)

	instance, err = inst.ChangeMasterTo(&instance.Key, &fromInstance.MasterKey, &fromInstance.ExecBinlogCoordinates, false, inst.GTIDHintNeutral)
	if err != nil {
		return instance, err
	}
	inst.AuditOperation("align-via-relaylogs-remote", &instance.Key, fmt.Sprintf("aligned %+v by relaylogs from %+v", instance.Key, fromInstance.Key))
	return instance, err
}

func SyncReplicasRelayLogs(masterKey *inst.InstanceKey) (syncedReplicas, failedReplicas [](*inst.Instance), err error) {
	var replicas [](*inst.Instance)
	if replicas, err = inst.GetSortedReplicas(masterKey, true); err != nil {
		return syncedReplicas, replicas, err
	}
	if len(replicas) <= 1 {
		// Nothing to be done
		return syncedReplicas, replicas, err
	}
	applyFromReplica := replicas[0]
	applyToReplicas := replicas[1:]

	log.Debugf("Testing SSH on %+v", applyFromReplica.Key)
	if err := TestRemoteCommandOnInstance(&applyFromReplica.Key); err != nil {
		return syncedReplicas, replicas, err
	}

	defer inst.StartSlave(&applyFromReplica.Key)

	barrier := make(chan *inst.InstanceKey)
	allErrors := make(chan error, len(applyToReplicas))
	synchedReplicasChan := make(chan *inst.Instance, len(applyToReplicas))
	failedReplicasChan := make(chan *inst.Instance, len(applyToReplicas))
	for _, applyToReplica := range applyToReplicas {
		applyToReplica := applyToReplica
		go func() {
			defer func() {
				defer func() { barrier <- &applyToReplica.Key }()
				inst.StartSlave(&applyToReplica.Key)
			}()
			if _, err := AlignViaRelaylogCorrelation(applyToReplica, applyFromReplica); err == nil {
				synchedReplicasChan <- applyToReplica
			} else {
				failedReplicasChan <- applyToReplica
				allErrors <- err
			}
		}()
	}
	for range applyToReplicas {
		<-barrier
	}
	syncedReplicas = append(syncedReplicas, applyFromReplica)
	for len(synchedReplicasChan) > 0 {
		syncedReplicas = append(syncedReplicas, <-synchedReplicasChan)
	}
	for len(failedReplicasChan) > 0 {
		failedReplicas = append(failedReplicas, <-failedReplicasChan)
	}
	countErrors := len(allErrors)
	for len(allErrors) > 0 {
		log.Errore(<-allErrors)
	}
	inst.AuditOperation("sync-replicas-relaylogs", masterKey, fmt.Sprintf("aligned %+v replicas by relaylogs from %+v, got %+v errors", len(applyToReplicas), applyFromReplica.Key, countErrors))
	return syncedReplicas, failedReplicas, err
}
<|endoftext|>


<commit_before>// A simple tool for log rotation.
/*
   Copyright (C) 2016, Library of the University of Tübingen

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as
   published by the Free Software Foundation, either version 3 of the
   License, or (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
package main

import "flag"
import "fmt"
import "os"
import "path"
import "path/filepath"
import "strconv"

// Handle command-line arguments.
func processFlags(maxRotationCount *int) {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: %s [--max-rotation-count max_rotations] log_base_names\n", path.Base(os.Args[0]))
		flag.PrintDefaults()
		os.Exit(1)
	}
	localMaxRotationCount := flag.Int("max-rotation-count", 5, "The maximum number of log file rotations.")
	flag.Parse()
	if *localMaxRotationCount < 1 {
		fmt.Fprintf(os.Stderr, "%s: max-rotation-count must be positive!\n", path.Base(os.Args[0]))
		os.Exit(1)
	}
	*maxRotationCount = *localMaxRotationCount
}

// Either return the log file names provided on the command-line, or, if there are none,
// return a list of files matching "*.log".
func getLogNames() []string {
	logNames := flag.Args()
	if len(logNames) == 0 {
		logNames, _ = filepath.Glob("*.log")
	}
	return logNames
}

func Exists(path string) bool {
	_, err := os.Stat(path)
	return err == nil
}

func processName(logFileName string, maxRotationCount int) {
	if !Exists(logFileName) {
		return
	} else {
		os.Remove(logFileName + "." + strconv.Itoa(maxRotationCount))
	}

	for countSuffix := maxRotationCount; countSuffix > 1; countSuffix-- {
		os.Rename(logFileName+"."+strconv.Itoa(countSuffix-1), logFileName+"."+strconv.Itoa(countSuffix))
	}
	os.Rename(logFileName, logFileName+".1")
}

func main() {
	var maxRotationCount int
	processFlags(&maxRotationCount)
	fmt.Printf("maxRotationCount = %d\n", maxRotationCount)
	logNames := getLogNames()
	for _, name := range logNames {
		processName(name, maxRotationCount)
	}
}
<commit_msg>Changed capitalisation of a function that need not be exported.<commit_after>// A simple tool for log rotation.
/*
   Copyright (C) 2016, Library of the University of Tübingen

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as
   published by the Free Software Foundation, either version 3 of the
   License, or (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
package main

import "flag"
import "fmt"
import "os"
import "path"
import "path/filepath"
import "strconv"

// Handle command-line arguments.
func processFlags(maxRotationCount *int) {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: %s [--max-rotation-count max_rotations] log_base_names\n", path.Base(os.Args[0]))
		flag.PrintDefaults()
		os.Exit(1)
	}
	localMaxRotationCount := flag.Int("max-rotation-count", 5, "The maximum number of log file rotations.")
	flag.Parse()
	if *localMaxRotationCount < 1 {
		fmt.Fprintf(os.Stderr, "%s: max-rotation-count must be positive!\n", path.Base(os.Args[0]))
		os.Exit(1)
	}
	*maxRotationCount = *localMaxRotationCount
}

// Either return the log file names provided on the command-line, or, if there are none,
// return a list of files matching "*.log".
func getLogNames() []string {
	logNames := flag.Args()
	if len(logNames) == 0 {
		logNames, _ = filepath.Glob("*.log")
	}
	return logNames
}

func exists(path string) bool {
	_, err := os.Stat(path)
	return err == nil
}

func processName(logFileName string, maxRotationCount int) {
	if !exists(logFileName) {
		return
	} else {
		os.Remove(logFileName + "." + strconv.Itoa(maxRotationCount))
	}

	for countSuffix := maxRotationCount; countSuffix > 1; countSuffix-- {
		os.Rename(logFileName+"."+strconv.Itoa(countSuffix-1), logFileName+"."+strconv.Itoa(countSuffix))
	}
	os.Rename(logFileName, logFileName+".1")
}

func main() {
	var maxRotationCount int
	processFlags(&maxRotationCount)
	fmt.Printf("maxRotationCount = %d\n", maxRotationCount)
	logNames := getLogNames()
	for _, name := range logNames {
		processName(name, maxRotationCount)
	}
}
<|endoftext|>


<commit_before>package s3manager_test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"sync"
	"testing"

	"github.com/awslabs/aws-sdk-go/aws"
	"github.com/awslabs/aws-sdk-go/aws/awserr"
	"github.com/awslabs/aws-sdk-go/aws/awsutil"
	"github.com/awslabs/aws-sdk-go/internal/test/unit"
	"github.com/awslabs/aws-sdk-go/service/s3"
	"github.com/awslabs/aws-sdk-go/service/s3/s3manager"
	"github.com/stretchr/testify/assert"
)

var _ = unit.Imported
var buf12MB = make([]byte, 1024*1024*12)
var buf2MB = make([]byte, 1024*1024*2)

func val(i interface{}, s string) interface{} {
	return awsutil.ValuesAtPath(i, s)[0]
}

func loggingSvc() (*s3.S3, *[]string, *[]interface{}) {
	var m sync.Mutex
	partNum := 0
	names := []string{}
	params := []interface{}{}
	svc := s3.New(nil)
	svc.Handlers.Unmarshal.Clear()
	svc.Handlers.UnmarshalMeta.Clear()
	svc.Handlers.UnmarshalError.Clear()
	svc.Handlers.Send.Clear()
	svc.Handlers.Send.PushBack(func(r *aws.Request) {
		m.Lock()
		defer m.Unlock()

		names = append(names, r.Operation.Name)
		params = append(params, r.Params)

		r.HTTPResponse = &http.Response{
			StatusCode: 200,
			Body:       ioutil.NopCloser(bytes.NewReader([]byte{})),
		}

		switch data := r.Data.(type) {
		case *s3.CreateMultipartUploadOutput:
			data.UploadID = aws.String("UPLOAD-ID")
		case *s3.UploadPartOutput:
			partNum++
			data.ETag = aws.String(fmt.Sprintf("ETAG%d", partNum))
		case *s3.CompleteMultipartUploadOutput:
			data.Location = aws.String("https://location")
		}
	})

	return svc, &names, &params
}

func buflen(i interface{}) int {
	r := i.(io.Reader)
	b, _ := ioutil.ReadAll(r)
	return len(b)
}

func TestUploadOrderMulti(t *testing.T) {
	s, ops, args := loggingSvc()
	resp, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   bytes.NewReader(buf12MB),
	}, nil)

	assert.NoError(t, err)
	assert.Equal(t, []string{"CreateMultipartUpload", "UploadPart", "UploadPart", "UploadPart", "CompleteMultipartUpload"}, *ops)
	assert.Equal(t, "https://location", resp.Location)
	assert.Equal(t, "UPLOAD-ID", resp.UploadID)

	// Validate input values

	// UploadPart
	assert.Equal(t, "UPLOAD-ID", val((*args)[1], "UploadID"))
	assert.Equal(t, "UPLOAD-ID", val((*args)[2], "UploadID"))
	assert.Equal(t, "UPLOAD-ID", val((*args)[3], "UploadID"))

	// CompleteMultipartUpload
	assert.Equal(t, "UPLOAD-ID", val((*args)[4], "UploadID"))
	assert.Equal(t, int64(1), val((*args)[4], "MultipartUpload.Parts[0].PartNumber"))
	assert.Equal(t, int64(2), val((*args)[4], "MultipartUpload.Parts[1].PartNumber"))
	assert.Equal(t, int64(3), val((*args)[4], "MultipartUpload.Parts[2].PartNumber"))
	assert.Regexp(t, `^ETAG\d+$`, val((*args)[4], "MultipartUpload.Parts[0].ETag"))
	assert.Regexp(t, `^ETAG\d+$`, val((*args)[4], "MultipartUpload.Parts[1].ETag"))
	assert.Regexp(t, `^ETAG\d+$`, val((*args)[4], "MultipartUpload.Parts[2].ETag"))
}

func TestUploadOrderMultiDifferentPartSize(t *testing.T) {
	s, ops, args := loggingSvc()
	_, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   bytes.NewReader(buf12MB),
	}, &s3manager.UploadOptions{PartSize: 1024 * 1024 * 7, Concurrency: 1})

	assert.NoError(t, err)
	assert.Equal(t, []string{"CreateMultipartUpload", "UploadPart", "UploadPart", "CompleteMultipartUpload"}, *ops)

	// Part lengths
	assert.Equal(t, 1024*1024*7, buflen(val((*args)[1], "Body")))
	assert.Equal(t, 1024*1024*5, buflen(val((*args)[2], "Body")))
}

func TestUploadOrderSingle(t *testing.T) {
	s, ops, _ := loggingSvc()
	resp, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   bytes.NewReader(buf2MB),
	}, nil)

	assert.NoError(t, err)
	assert.Equal(t, []string{"PutObject"}, *ops)
	assert.NotEqual(t, "", resp.Location)
	assert.Equal(t, "", resp.UploadID)
}

func TestUploadOrderSingleFailure(t *testing.T) {
	s, ops, _ := loggingSvc()
	s.Handlers.Send.PushBack(func(r *aws.Request) {
		r.HTTPResponse.StatusCode = 400
	})
	resp, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   bytes.NewReader(buf2MB),
	}, nil)

	assert.Error(t, err)
	assert.Equal(t, []string{"PutObject"}, *ops)
	assert.Nil(t, resp)
}

func TestUploadOrderZero(t *testing.T) {
	s, ops, args := loggingSvc()
	resp, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   bytes.NewReader(make([]byte, 0)),
	}, nil)

	assert.NoError(t, err)
	assert.Equal(t, []string{"PutObject"}, *ops)
	assert.NotEqual(t, "", resp.Location)
	assert.Equal(t, "", resp.UploadID)
	assert.Equal(t, 0, buflen(val((*args)[0], "Body")))
}

func TestUploadOrderMultiFailure(t *testing.T) {
	s, ops, _ := loggingSvc()
	s.Handlers.Send.PushBack(func(r *aws.Request) {
		switch t := r.Data.(type) {
		case *s3.UploadPartOutput:
			if *t.ETag == "ETAG2" {
				r.HTTPResponse.StatusCode = 400
			}
		}
	})
	_, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   bytes.NewReader(buf12MB),
	}, &s3manager.UploadOptions{Concurrency: 1})

	assert.Error(t, err)
	assert.Equal(t, []string{"CreateMultipartUpload", "UploadPart", "UploadPart", "AbortMultipartUpload"}, *ops)
}

func TestUploadOrderMultiFailureOnComplete(t *testing.T) {
	s, ops, _ := loggingSvc()
	s.Handlers.Send.PushBack(func(r *aws.Request) {
		switch r.Data.(type) {
		case *s3.CompleteMultipartUploadOutput:
			r.HTTPResponse.StatusCode = 400
		}
	})
	_, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   bytes.NewReader(buf12MB),
	}, nil)

	assert.Error(t, err)
	assert.Equal(t, []string{"CreateMultipartUpload", "UploadPart", "UploadPart",
		"UploadPart", "CompleteMultipartUpload", "AbortMultipartUpload"}, *ops)
}

func TestUploadOrderMultiFailureOnCreate(t *testing.T) {
	s, ops, _ := loggingSvc()
	s.Handlers.Send.PushBack(func(r *aws.Request) {
		switch r.Data.(type) {
		case *s3.CreateMultipartUploadOutput:
			r.HTTPResponse.StatusCode = 400
		}
	})
	_, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   bytes.NewReader(make([]byte, 1024*1024*12)),
	}, nil)

	assert.Error(t, err)
	assert.Equal(t, []string{"CreateMultipartUpload"}, *ops)
}

func TestUploadOrderMultiFailureLeaveParts(t *testing.T) {
	s, ops, _ := loggingSvc()
	s.Handlers.Send.PushBack(func(r *aws.Request) {
		switch data := r.Data.(type) {
		case *s3.UploadPartOutput:
			if *data.ETag == "ETAG2" {
				r.HTTPResponse.StatusCode = 400
			}
		}
	})
	_, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   bytes.NewReader(make([]byte, 1024*1024*12)),
	}, &s3manager.UploadOptions{Concurrency: 1, LeavePartsOnError: true})

	assert.Error(t, err)
	assert.Equal(t, []string{"CreateMultipartUpload", "UploadPart", "UploadPart"}, *ops)
}

var failreaderCount = 0

type failreader struct{ times int }

func (f failreader) Read(b []byte) (int, error) {
	failreaderCount++
	if failreaderCount >= f.times {
		return 0, fmt.Errorf("random failure")
	}
	return len(b), nil
}

func TestUploadOrderReadFail1(t *testing.T) {
	failreaderCount = 0
	s, ops, _ := loggingSvc()
	_, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   failreader{1},
	}, nil)

	assert.Equal(t, "ReadRequestBody", err.(awserr.Error).Code())
	assert.EqualError(t, err.(awserr.Error).OrigErr(), "random failure")
	assert.Equal(t, []string{}, *ops)
}

func TestUploadOrderReadFail2(t *testing.T) {
	failreaderCount = 0
	s, ops, _ := loggingSvc()
	_, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   failreader{2},
	}, nil)

	assert.Equal(t, "MultipartUpload", err.(awserr.Error).Code())
	assert.Equal(t, "ReadRequestBody", err.(awserr.Error).OrigErr().(awserr.Error).Code())
	assert.EqualError(t, err.(awserr.Error).OrigErr().(awserr.Error).OrigErr(), "random failure")
	assert.Equal(t, []string{"CreateMultipartUpload", "AbortMultipartUpload"}, *ops)
}
<commit_msg>service/s3/s3manager: Add tests for additional UploadInput parameters<commit_after>package s3manager_test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"sync"
	"testing"

	"github.com/awslabs/aws-sdk-go/aws"
	"github.com/awslabs/aws-sdk-go/aws/awserr"
	"github.com/awslabs/aws-sdk-go/aws/awsutil"
	"github.com/awslabs/aws-sdk-go/internal/test/unit"
	"github.com/awslabs/aws-sdk-go/service/s3"
	"github.com/awslabs/aws-sdk-go/service/s3/s3manager"
	"github.com/stretchr/testify/assert"
)

var _ = unit.Imported
var buf12MB = make([]byte, 1024*1024*12)
var buf2MB = make([]byte, 1024*1024*2)

func val(i interface{}, s string) interface{} {
	return awsutil.ValuesAtPath(i, s)[0]
}

func loggingSvc() (*s3.S3, *[]string, *[]interface{}) {
	var m sync.Mutex
	partNum := 0
	names := []string{}
	params := []interface{}{}
	svc := s3.New(nil)
	svc.Handlers.Unmarshal.Clear()
	svc.Handlers.UnmarshalMeta.Clear()
	svc.Handlers.UnmarshalError.Clear()
	svc.Handlers.Send.Clear()
	svc.Handlers.Send.PushBack(func(r *aws.Request) {
		m.Lock()
		defer m.Unlock()

		names = append(names, r.Operation.Name)
		params = append(params, r.Params)

		r.HTTPResponse = &http.Response{
			StatusCode: 200,
			Body:       ioutil.NopCloser(bytes.NewReader([]byte{})),
		}

		switch data := r.Data.(type) {
		case *s3.CreateMultipartUploadOutput:
			data.UploadID = aws.String("UPLOAD-ID")
		case *s3.UploadPartOutput:
			partNum++
			data.ETag = aws.String(fmt.Sprintf("ETAG%d", partNum))
		case *s3.CompleteMultipartUploadOutput:
			data.Location = aws.String("https://location")
		}
	})

	return svc, &names, &params
}

func buflen(i interface{}) int {
	r := i.(io.Reader)
	b, _ := ioutil.ReadAll(r)
	return len(b)
}

func TestUploadOrderMulti(t *testing.T) {
	s, ops, args := loggingSvc()
	resp, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket:               aws.String("Bucket"),
		Key:                  aws.String("Key"),
		Body:                 bytes.NewReader(buf12MB),
		ServerSideEncryption: aws.String("AES256"),
		ContentType:          aws.String("content/type"),
	}, nil)

	assert.NoError(t, err)
	assert.Equal(t, []string{"CreateMultipartUpload", "UploadPart", "UploadPart", "UploadPart", "CompleteMultipartUpload"}, *ops)
	assert.Equal(t, "https://location", resp.Location)
	assert.Equal(t, "UPLOAD-ID", resp.UploadID)

	// Validate input values

	// UploadPart
	assert.Equal(t, "UPLOAD-ID", val((*args)[1], "UploadID"))
	assert.Equal(t, "UPLOAD-ID", val((*args)[2], "UploadID"))
	assert.Equal(t, "UPLOAD-ID", val((*args)[3], "UploadID"))

	// CompleteMultipartUpload
	assert.Equal(t, "UPLOAD-ID", val((*args)[4], "UploadID"))
	assert.Equal(t, int64(1), val((*args)[4], "MultipartUpload.Parts[0].PartNumber"))
	assert.Equal(t, int64(2), val((*args)[4], "MultipartUpload.Parts[1].PartNumber"))
	assert.Equal(t, int64(3), val((*args)[4], "MultipartUpload.Parts[2].PartNumber"))
	assert.Regexp(t, `^ETAG\d+$`, val((*args)[4], "MultipartUpload.Parts[0].ETag"))
	assert.Regexp(t, `^ETAG\d+$`, val((*args)[4], "MultipartUpload.Parts[1].ETag"))
	assert.Regexp(t, `^ETAG\d+$`, val((*args)[4], "MultipartUpload.Parts[2].ETag"))

	// Custom headers
	assert.Equal(t, "AES256", val((*args)[0], "ServerSideEncryption"))
	assert.Equal(t, "content/type", val((*args)[0], "ContentType"))
}

func TestUploadOrderMultiDifferentPartSize(t *testing.T) {
	s, ops, args := loggingSvc()
	_, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   bytes.NewReader(buf12MB),
	}, &s3manager.UploadOptions{PartSize: 1024 * 1024 * 7, Concurrency: 1})

	assert.NoError(t, err)
	assert.Equal(t, []string{"CreateMultipartUpload", "UploadPart", "UploadPart", "CompleteMultipartUpload"}, *ops)

	// Part lengths
	assert.Equal(t, 1024*1024*7, buflen(val((*args)[1], "Body")))
	assert.Equal(t, 1024*1024*5, buflen(val((*args)[2], "Body")))
}

func TestUploadOrderSingle(t *testing.T) {
	s, ops, args := loggingSvc()
	resp, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket:               aws.String("Bucket"),
		Key:                  aws.String("Key"),
		Body:                 bytes.NewReader(buf2MB),
		ServerSideEncryption: aws.String("AES256"),
		ContentType:          aws.String("content/type"),
	}, nil)

	assert.NoError(t, err)
	assert.Equal(t, []string{"PutObject"}, *ops)
	assert.NotEqual(t, "", resp.Location)
	assert.Equal(t, "", resp.UploadID)
	assert.Equal(t, "AES256", val((*args)[0], "ServerSideEncryption"))
	assert.Equal(t, "content/type", val((*args)[0], "ContentType"))
}

func TestUploadOrderSingleFailure(t *testing.T) {
	s, ops, _ := loggingSvc()
	s.Handlers.Send.PushBack(func(r *aws.Request) {
		r.HTTPResponse.StatusCode = 400
	})
	resp, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   bytes.NewReader(buf2MB),
	}, nil)

	assert.Error(t, err)
	assert.Equal(t, []string{"PutObject"}, *ops)
	assert.Nil(t, resp)
}

func TestUploadOrderZero(t *testing.T) {
	s, ops, args := loggingSvc()
	resp, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   bytes.NewReader(make([]byte, 0)),
	}, nil)

	assert.NoError(t, err)
	assert.Equal(t, []string{"PutObject"}, *ops)
	assert.NotEqual(t, "", resp.Location)
	assert.Equal(t, "", resp.UploadID)
	assert.Equal(t, 0, buflen(val((*args)[0], "Body")))
}

func TestUploadOrderMultiFailure(t *testing.T) {
	s, ops, _ := loggingSvc()
	s.Handlers.Send.PushBack(func(r *aws.Request) {
		switch t := r.Data.(type) {
		case *s3.UploadPartOutput:
			if *t.ETag == "ETAG2" {
				r.HTTPResponse.StatusCode = 400
			}
		}
	})
	_, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   bytes.NewReader(buf12MB),
	}, &s3manager.UploadOptions{Concurrency: 1})

	assert.Error(t, err)
	assert.Equal(t, []string{"CreateMultipartUpload", "UploadPart", "UploadPart", "AbortMultipartUpload"}, *ops)
}

func TestUploadOrderMultiFailureOnComplete(t *testing.T) {
	s, ops, _ := loggingSvc()
	s.Handlers.Send.PushBack(func(r *aws.Request) {
		switch r.Data.(type) {
		case *s3.CompleteMultipartUploadOutput:
			r.HTTPResponse.StatusCode = 400
		}
	})
	_, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   bytes.NewReader(buf12MB),
	}, nil)

	assert.Error(t, err)
	assert.Equal(t, []string{"CreateMultipartUpload", "UploadPart", "UploadPart",
		"UploadPart", "CompleteMultipartUpload", "AbortMultipartUpload"}, *ops)
}

func TestUploadOrderMultiFailureOnCreate(t *testing.T) {
	s, ops, _ := loggingSvc()
	s.Handlers.Send.PushBack(func(r *aws.Request) {
		switch r.Data.(type) {
		case *s3.CreateMultipartUploadOutput:
			r.HTTPResponse.StatusCode = 400
		}
	})
	_, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   bytes.NewReader(make([]byte, 1024*1024*12)),
	}, nil)

	assert.Error(t, err)
	assert.Equal(t, []string{"CreateMultipartUpload"}, *ops)
}

func TestUploadOrderMultiFailureLeaveParts(t *testing.T) {
	s, ops, _ := loggingSvc()
	s.Handlers.Send.PushBack(func(r *aws.Request) {
		switch data := r.Data.(type) {
		case *s3.UploadPartOutput:
			if *data.ETag == "ETAG2" {
				r.HTTPResponse.StatusCode = 400
			}
		}
	})
	_, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   bytes.NewReader(make([]byte, 1024*1024*12)),
	}, &s3manager.UploadOptions{Concurrency: 1, LeavePartsOnError: true})

	assert.Error(t, err)
	assert.Equal(t, []string{"CreateMultipartUpload", "UploadPart", "UploadPart"}, *ops)
}

var failreaderCount = 0

type failreader struct{ times int }

func (f failreader) Read(b []byte) (int, error) {
	failreaderCount++
	if failreaderCount >= f.times {
		return 0, fmt.Errorf("random failure")
	}
	return len(b), nil
}

func TestUploadOrderReadFail1(t *testing.T) {
	failreaderCount = 0
	s, ops, _ := loggingSvc()
	_, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   failreader{1},
	}, nil)

	assert.Equal(t, "ReadRequestBody", err.(awserr.Error).Code())
	assert.EqualError(t, err.(awserr.Error).OrigErr(), "random failure")
	assert.Equal(t, []string{}, *ops)
}

func TestUploadOrderReadFail2(t *testing.T) {
	failreaderCount = 0
	s, ops, _ := loggingSvc()
	_, err := s3manager.Upload(s, &s3manager.UploadInput{
		Bucket: aws.String("Bucket"),
		Key:    aws.String("Key"),
		Body:   failreader{2},
	}, nil)

	assert.Equal(t, "MultipartUpload", err.(awserr.Error).Code())
	assert.Equal(t, "ReadRequestBody", err.(awserr.Error).OrigErr().(awserr.Error).Code())
	assert.EqualError(t, err.(awserr.Error).OrigErr().(awserr.Error).OrigErr(), "random failure")
	assert.Equal(t, []string{"CreateMultipartUpload", "AbortMultipartUpload"}, *ops)
}
<|endoftext|>