<commit_before>// weed master
package weedo

import (
	"errors"
	"io"
	"log"
	"net/http"
	"net/url"
	"strconv"
)

type Master struct {
	Url string
}

func NewMaster(addr string) *Master {
	return &Master{
		Url: addr,
	}
}

// Assign a file key
func (m *Master) Assign() (string, error) {
	return m.AssignArgs(url.Values{})
}

// Assign multi file keys
func (m *Master) AssignN(count int) (fid string, err error) {
	args := url.Values{}
	if count > 0 {
		args.Set("count", strconv.Itoa(count))
	}

	return m.AssignArgs(args)
}

type assignResp struct {
	Count     int
	Fid       string
	Url       string
	PublicUrl string
	Size      int64
	Error     string
}

// v0.4 or later only
func (m *Master) AssignArgs(args url.Values) (fid string, err error) {
	u := url.URL{
		Scheme:   "http",
		Host:     m.Url,
		Path:     "/dir/assign",
		RawQuery: args.Encode(),
	}

	resp, err := http.Get(u.String())
	if err != nil {
		return
	}
	defer resp.Body.Close()

	assign := new(assignResp)
	if err = decodeJson(resp.Body, assign); err != nil {
		log.Println(err)
		return
	}

	if assign.Error != "" {
		err = errors.New(assign.Error)
		log.Println(err)
		return
	}

	fid = assign.Fid

	return
}

type lookupResp struct {
	Locations []Location
	Error     string
}

type Location struct {
	Url       string
	PublicUrl string
}

// Lookup Volume
func (m *Master) lookup(volumeId, collection string) (*Volume, error) {
	args := url.Values{}
	args.Set("volumeId", volumeId)
	args.Set("collection", collection)

	u := url.URL{
		Scheme:   "http",
		Host:     m.Url,
		Path:     "/dir/lookup",
		RawQuery: args.Encode(),
	}
	resp, err := http.Get(u.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	lookup := new(lookupResp)
	if err = decodeJson(resp.Body, lookup); err != nil {
		log.Println(err)
		return nil, err
	}

	if lookup.Error != "" {
		return nil, errors.New(lookup.Error)
	}

	return NewVolume(lookup.Locations), nil
}

// Force Garbage Collection
func (m *Master) GC(threshold float64) error {
	args := url.Values{}
	args.Set("garbageThreshold", strconv.FormatFloat(threshold, 'f', -1, 64))
	u := url.URL{
		Scheme:   "http",
		Host:     m.Url,
		Path:     "/vol/vacuum",
		RawQuery: args.Encode(),
	}
	resp, err := http.Get(u.String())
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	// TODO: handle result
	return nil
}

// Pre-Allocate Volumes
func (m *Master) Grow(count int, collection, replication, dataCenter string) error {
	args := url.Values{}
	if count > 0 {
		args.Set("count", strconv.Itoa(count))
	}
	args.Set("collection", collection)
	args.Set("replication", replication)
	args.Set("dataCenter", dataCenter)

	return m.GrowArgs(args)
}

// v0.4 or later only
func (m *Master) GrowArgs(args url.Values) error {
	u := url.URL{
		Scheme:   "http",
		Host:     m.Url,
		Path:     "/vol/grow",
		RawQuery: args.Encode(),
	}
	resp, err := http.Get(u.String())
	resp.Body.Close()

	return err
}

// Upload File Directly
func (m *Master) Submit(filename, mimeType string, file io.Reader) (fid string, size int64, err error) {
	data, contentType, err := makeFormData(filename, mimeType, file)
	if err != nil {
		return
	}

	u := url.URL{
		Scheme: "http",
		Host:   m.Url,
		Path:   "/submit",
	}

	resp, err := upload(u.String(), contentType, data)
	if err == nil {
		fid = resp.Fid
		size = resp.Size
	}

	return
}

type systemStatus struct {
	Topology topology
	Version  string
	Error    string
}

type topology struct {
	DataCenters dataCenter
	Free        int
	Max         int
	Layouts     []layout
}

type dataCenter struct {
	Free  int
	Max   int
	Racks []rack
}

type rack struct {
	DataNodes []dataNode
	Free      int
	Max       int
}

type dataNode struct {
	Free      int
	Max       int
	PublicUrl string
	Url       string
	Volumes   int
}

type layout struct {
	Replication string
	Writables   []uint64
}

// Check System Status
func (m *Master) Status() (err error) {
	u := url.URL{
		Scheme: "http",
		Host:   m.Url,
		Path:   "/dir/status",
	}
	resp, err := http.Get(u.String())
	if err != nil {
		return
	}

	defer resp.Body.Close()

	status := new(systemStatus)
	if err = decodeJson(resp.Body, status); err != nil {
		log.Println(err)
		return
	}

	if status.Error != "" {
		err = errors.New(status.Error)
		log.Println(err)
		return
	}
	return
}
<commit_msg>submit support args<commit_after>// weed master
package weedo

import (
	"errors"
	"io"
	"log"
	"net/http"
	"net/url"
	"strconv"
)

type Master struct {
	Url string
}

func NewMaster(addr string) *Master {
	return &Master{
		Url: addr,
	}
}

// Assign a file key
func (m *Master) Assign() (string, error) {
	return m.AssignArgs(url.Values{})
}

// Assign multi file keys
func (m *Master) AssignN(count int) (fid string, err error) {
	args := url.Values{}
	if count > 0 {
		args.Set("count", strconv.Itoa(count))
	}

	return m.AssignArgs(args)
}

type assignResp struct {
	Count     int
	Fid       string
	Url       string
	PublicUrl string
	Size      int64
	Error     string
}

// v0.4 or later only
func (m *Master) AssignArgs(args url.Values) (fid string, err error) {
	u := url.URL{
		Scheme:   "http",
		Host:     m.Url,
		Path:     "/dir/assign",
		RawQuery: args.Encode(),
	}

	resp, err := http.Get(u.String())
	if err != nil {
		return
	}
	defer resp.Body.Close()

	assign := new(assignResp)
	if err = decodeJson(resp.Body, assign); err != nil {
		log.Println(err)
		return
	}

	if assign.Error != "" {
		err = errors.New(assign.Error)
		log.Println(err)
		return
	}

	fid = assign.Fid

	return
}

type lookupResp struct {
	Locations []Location
	Error     string
}

type Location struct {
	Url       string
	PublicUrl string
}

// Lookup Volume
func (m *Master) lookup(volumeId, collection string) (*Volume, error) {
	args := url.Values{}
	args.Set("volumeId", volumeId)
	args.Set("collection", collection)

	u := url.URL{
		Scheme:   "http",
		Host:     m.Url,
		Path:     "/dir/lookup",
		RawQuery: args.Encode(),
	}
	resp, err := http.Get(u.String())
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	lookup := new(lookupResp)
	if err = decodeJson(resp.Body, lookup); err != nil {
		log.Println(err)
		return nil, err
	}

	if lookup.Error != "" {
		return nil, errors.New(lookup.Error)
	}

	return NewVolume(lookup.Locations), nil
}

// Force Garbage Collection
func (m *Master) GC(threshold float64) error {
	args := url.Values{}
	args.Set("garbageThreshold", strconv.FormatFloat(threshold, 'f', -1, 64))
	u := url.URL{
		Scheme:   "http",
		Host:     m.Url,
		Path:     "/vol/vacuum",
		RawQuery: args.Encode(),
	}
	resp, err := http.Get(u.String())
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	// TODO: handle result
	return nil
}

// Pre-Allocate Volumes
func (m *Master) Grow(count int, collection, replication, dataCenter string) error {
	args := url.Values{}
	if count > 0 {
		args.Set("count", strconv.Itoa(count))
	}
	args.Set("collection", collection)
	args.Set("replication", replication)
	args.Set("dataCenter", dataCenter)

	return m.GrowArgs(args)
}

// v0.4 or later only
func (m *Master) GrowArgs(args url.Values) error {
	u := url.URL{
		Scheme:   "http",
		Host:     m.Url,
		Path:     "/vol/grow",
		RawQuery: args.Encode(),
	}
	resp, err := http.Get(u.String())
	resp.Body.Close()

	return err
}

func (m *Master) Submit(filename, mimeType string, file io.Reader) (fid string, size int64, err error) {
	return m.SubmitArgs(filename, mimeType, file, url.Values{})
}

// Upload File Directly
func (m *Master) SubmitArgs(filename, mimeType string, file io.Reader, args url.Values) (fid string, size int64, err error) {
	data, contentType, err := makeFormData(filename, mimeType, file)
	if err != nil {
		return
	}

	u := url.URL{
		Scheme: "http",
		Host:   m.Url,
		Path:   "/submit",
		RawQuery: args.Encode(),
	}

	resp, err := upload(u.String(), contentType, data)
	if err == nil {
		fid = resp.Fid
		size = resp.Size
	}

	return
}

type systemStatus struct {
	Topology topology
	Version  string
	Error    string
}

type topology struct {
	DataCenters dataCenter
	Free        int
	Max         int
	Layouts     []layout
}

type dataCenter struct {
	Free  int
	Max   int
	Racks []rack
}

type rack struct {
	DataNodes []dataNode
	Free      int
	Max       int
}

type dataNode struct {
	Free      int
	Max       int
	PublicUrl string
	Url       string
	Volumes   int
}

type layout struct {
	Replication string
	Writables   []uint64
}

// Check System Status
func (m *Master) Status() (err error) {
	u := url.URL{
		Scheme: "http",
		Host:   m.Url,
		Path:   "/dir/status",
	}
	resp, err := http.Get(u.String())
	if err != nil {
		return
	}

	defer resp.Body.Close()

	status := new(systemStatus)
	if err = decodeJson(resp.Body, status); err != nil {
		log.Println(err)
		return
	}

	if status.Error != "" {
		err = errors.New(status.Error)
		log.Println(err)
		return
	}
	return
}
<|endoftext|>


<commit_before>package gofigure

import (
	"fmt"
	"reflect"
)

func mergeMapAndStruct(theMap map[string]interface{}, spec interface{}) error {
	s := reflect.ValueOf(spec).Elem()
	if s.Kind() != reflect.Struct {
		return fmt.Errorf("Invalid spec! Needs to be a struct.")
	}
	typeOfSpec := s.Type()
	for i := 0; i < s.NumField(); i++ {
		f := s.Field(i)
		if f.CanSet() {
			fieldName := typeOfSpec.Field(i).Name
			value, ok := theMap[fieldName]
			if !ok {
				fmt.Println("Field %s not found in %s", fieldName, spec)
				continue
			}
			switch f.Kind() {
			case reflect.String:
				f.SetString(value.(string))
			case reflect.Bool:
				f.SetBool(value.(bool))
			case reflect.Float32, reflect.Float64:
				switch value.(type) {
				case float32:
					f.SetFloat(float64(value.(float32)))
				case float64:
					f.SetFloat(value.(float64))
				}
			case reflect.Int:
				switch value.(type) {
				case int:
					f.SetInt(int64(value.(int)))
				case int16:
					f.SetInt(int64(value.(int16)))
				case int32:
					f.SetInt(int64(value.(int32)))
				case int64:
					f.SetInt(value.(int64))
				default:
					panic("Non-int!")
				}
			}
		}
	}
	return nil
}
<commit_msg>handle int{,8,16,32,64} instead of just int<commit_after>package gofigure

import (
	"fmt"
	"reflect"
)

func mergeMapAndStruct(theMap map[string]interface{}, spec interface{}) error {
	s := reflect.ValueOf(spec).Elem()
	if s.Kind() != reflect.Struct {
		return fmt.Errorf("Invalid spec! Needs to be a struct.")
	}
	typeOfSpec := s.Type()
	for i := 0; i < s.NumField(); i++ {
		f := s.Field(i)
		if f.CanSet() {
			fieldName := typeOfSpec.Field(i).Name
			value, ok := theMap[fieldName]
			if !ok {
				// the value isn't in the map. Skip dat.
				continue
			}
			switch f.Kind() {
			case reflect.String:
				f.SetString(value.(string))
			case reflect.Bool:
				f.SetBool(value.(bool))
			case reflect.Float32, reflect.Float64:
				switch value.(type) {
				case float32:
					f.SetFloat(float64(value.(float32)))
				case float64:
					f.SetFloat(value.(float64))
				}
			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
				switch value.(type) {
				case int:
					f.SetInt(int64(value.(int)))
				case int16:
					f.SetInt(int64(value.(int16)))
				case int32:
					f.SetInt(int64(value.(int32)))
				case int64:
					f.SetInt(value.(int64))
				default:
					panic("This should never happen. A non-int has been detected as an int by 'reflect'")
				}
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package myaws

import (
	"fmt"
	"github.com/aws/aws-sdk-go/service/ecs"

	"github.com/aws/aws-sdk-go/aws/awsutil"
	"github.com/aws/aws-sdk-go/service/autoscaling"
	"github.com/pkg/errors"
)

// ECSNodeRenewOptions customize the behavior of the Renew command.
type ECSNodeRenewOptions struct {
	Cluster string
	AsgName string
}

// ECSNodeRenew renew ECS container instances with blue-green deployment.
// This method is an automation process to renew your ECS container instances
// if you update the AMI. creates new instances, drains the old instances,
// and discards the old instances.
func (client *Client) ECSNodeRenew(options ECSNodeRenewOptions) error {
	fmt.Fprintf(client.stdout, "start: ecs node renew\noptions: %s\n", awsutil.Prettify(options))

	if err := client.printECSStatus(options.Cluster); err != nil {
		return err
	}

	// get the current desired capacity
	desiredCapacity, err := client.getAutoScalingGroupDesiredCapacity(options.AsgName)
	if err != nil {
		return err
	}

	// list the current container instances
	oldNodes, err := client.findECSNodes(options.Cluster)
	if err != nil {
		return err
	}

	if len(oldNodes) != int(desiredCapacity) {
		return errors.Errorf("assertion failed: currentCapacity(%d) != desiredCapacity(%d)", len(oldNodes), desiredCapacity)
	}

	// Update the desired capacity and wait until new instances are InService
	// We simply double the number of instances here.
	// If you need more flexible control, please implement a strategy such as
	// rolling update.
	targetCapacity := desiredCapacity * 2

	fmt.Fprintf(client.stdout, "Update autoscaling group %s (DesiredCapacity: %d => %d)\n", options.AsgName, desiredCapacity, targetCapacity)

	err = client.AutoscalingUpdate(AutoscalingUpdateOptions{
		AsgName:         options.AsgName,
		DesiredCapacity: targetCapacity,
		Wait:            true,
	})
	if err != nil {
		return err
	}

	if err = client.printECSStatus(options.Cluster); err != nil {
		return err
	}

	// A status of instance in autoscaling group is InService doesn't mean the
	// container instance is registered. We should make sure container instances
	// are registered
	fmt.Fprintln(client.stdout, "Wait until ECS container instances are registered...")
	err = client.WaitUntilECSContainerInstancesAreRegistered(options.Cluster, targetCapacity)
	if err != nil {
		return err
	}

	if err = client.printECSStatus(options.Cluster); err != nil {
		return err
	}

	// drain old container instances and wait until no task running
	oldNodeArns := []*string{}
	for _, oldNode := range oldNodes {
		oldNodeArns = append(oldNodeArns, oldNode.ContainerInstanceArn)
	}
	fmt.Fprintf(client.stdout, "Drain old container instances and wait until no task running...\n%v\n", awsutil.Prettify(oldNodeArns))
	err = client.ECSNodeDrain(ECSNodeDrainOptions{
		Cluster:            options.Cluster,
		ContainerInstances: oldNodeArns,
		Wait:               true,
	})
	if err != nil {
		return err
	}

	if err = client.printECSStatus(options.Cluster); err != nil {
		return err
	}

	// All old container instances are drained doesn't mean all services are stable.
	// It depends on the deployment strategy of each service.
	// We should make sure all services are stable
	fmt.Fprintln(client.stdout, "Wait until all ECS services stable...")
	err = client.WaitUntilECSAllServicesStable(options.Cluster)
	if err != nil {
		return err
	}

	if err = client.printECSStatus(options.Cluster); err != nil {
		return err
	}

	// A stable state for all services does not mean that all targets are healthy.
	// We need to explicitly confirm it.
	fmt.Fprintln(client.stdout, "Wait until all targets healthy...")
	err = client.WaitUntilECSAllTargetsInService(options.Cluster)
	if err != nil {
		return err
	}

	if err = client.printECSStatus(options.Cluster); err != nil {
		return err
	}

	// Select instances to protect from scale in.
	// By setting "scale-in protection" to instances created at scale-out,
	// the intended instances (instances created before scale-in) are only terminated at scale-in process.
	protectInstanceIds, err := client.selectInstanceToProtectFromScaleIn(oldNodes, options.Cluster)
	if err != nil {
		return err
	}

	fmt.Fprintln(client.stdout, "Setting scale in protection: ", awsutil.Prettify(protectInstanceIds))
	// set "scale in protection" to instances created at scale-out.
	err = client.AutoScalingSetInstanceProtection(AutoScalingSetInstanceProtectionOptions{
		options.AsgName,
		protectInstanceIds,
		true})
	if err != nil {
		return err
	}

	// restore the desired capacity and wait until old instances are discarded
	fmt.Fprintf(client.stdout, "Update autoscaling group %s (DesiredCapacity: %d => %d)\n", options.AsgName, targetCapacity, desiredCapacity)

	err = client.AutoscalingUpdate(AutoscalingUpdateOptions{
		AsgName:         options.AsgName,
		DesiredCapacity: desiredCapacity,
		Wait:            true,
	})
	if err != nil {
		return err
	}

	// remove "scale in protection" to instances created at scale-out.
	fmt.Fprintln(client.stdout, "Removing scale in protection: ", awsutil.Prettify(protectInstanceIds))
	err = client.AutoScalingSetInstanceProtection(AutoScalingSetInstanceProtectionOptions{
		options.AsgName,
		protectInstanceIds,
		false})
	if err != nil {
		return err
	}

	if err = client.printECSStatus(options.Cluster); err != nil {
		return err
	}

	fmt.Fprintln(client.stdout, "end: ecs node renew")
	return nil
}

// selectInstanceToProtectFromScaleIn selects instance to protect from Scale in.
// instance select rule:
//   instances after scale out - instances before scale out - instances which already set `InstanceProtection==true`
func (client *Client) selectInstanceToProtectFromScaleIn(oldNodes []*ecs.ContainerInstance, cluster string) ([]*string, error) {
	// Get a list of instance IDs before auto scaling
	var oldInstanceIds []*string
	for _, oldNode := range oldNodes {
		oldInstanceIds = append(oldInstanceIds, oldNode.Ec2InstanceId)
	}

	// Get a list of instances after auto scaling
	allNodes, err := client.findECSNodes(cluster)
	if err != nil {
		return nil, err
	}

	// Get a list of instance IDs after auto scaling
	var allInstanceIds []*string
	for _, allNode := range allNodes {
		allInstanceIds = append(allInstanceIds, allNode.Ec2InstanceId)
	}

	// get newly created nodes (allInstanceIds - oldInstanceIds)
	newInstanceIds := difference(allInstanceIds, oldInstanceIds)

	// exclude ProtectedFromScaleIn == true nodes
	params := &autoscaling.DescribeAutoScalingInstancesInput{
		InstanceIds: newInstanceIds,
	}
	response, err := client.AutoScaling.DescribeAutoScalingInstances(params)
	if err != nil {
		return nil, errors.Wrap(err, "DescribeAutoScalingGroups failed:")
	}

	var targetInstanceIds []*string
	for _, instance := range response.AutoScalingInstances {
		if *instance.ProtectedFromScaleIn == false {
			targetInstanceIds = append(targetInstanceIds, instance.InstanceId)
		}
	}
	return targetInstanceIds, nil
}

// difference returns the elements in `a` that aren't in `b`.
func difference(a, b []*string) []*string {
	mb := make(map[string]struct{}, len(b))
	for _, x := range b {
		mb[*x] = struct{}{}
	}
	var diff []*string
	for _, x := range a {
		if _, ok := mb[*x]; !ok {
			diff = append(diff, x)
		}
	}
	return diff
}
<commit_msg>add comment about scale in protection.<commit_after>package myaws

import (
	"fmt"
	"github.com/aws/aws-sdk-go/service/ecs"

	"github.com/aws/aws-sdk-go/aws/awsutil"
	"github.com/aws/aws-sdk-go/service/autoscaling"
	"github.com/pkg/errors"
)

// ECSNodeRenewOptions customize the behavior of the Renew command.
type ECSNodeRenewOptions struct {
	Cluster string
	AsgName string
}

// ECSNodeRenew renew ECS container instances with blue-green deployment.
// This method is an automation process to renew your ECS container instances
// if you update the AMI. creates new instances, drains the old instances,
// and discards the old instances.
func (client *Client) ECSNodeRenew(options ECSNodeRenewOptions) error {
	fmt.Fprintf(client.stdout, "start: ecs node renew\noptions: %s\n", awsutil.Prettify(options))

	if err := client.printECSStatus(options.Cluster); err != nil {
		return err
	}

	// get the current desired capacity
	desiredCapacity, err := client.getAutoScalingGroupDesiredCapacity(options.AsgName)
	if err != nil {
		return err
	}

	// list the current container instances
	oldNodes, err := client.findECSNodes(options.Cluster)
	if err != nil {
		return err
	}

	if len(oldNodes) != int(desiredCapacity) {
		return errors.Errorf("assertion failed: currentCapacity(%d) != desiredCapacity(%d)", len(oldNodes), desiredCapacity)
	}

	// Update the desired capacity and wait until new instances are InService
	// We simply double the number of instances here.
	// If you need more flexible control, please implement a strategy such as
	// rolling update.
	targetCapacity := desiredCapacity * 2

	fmt.Fprintf(client.stdout, "Update autoscaling group %s (DesiredCapacity: %d => %d)\n", options.AsgName, desiredCapacity, targetCapacity)

	err = client.AutoscalingUpdate(AutoscalingUpdateOptions{
		AsgName:         options.AsgName,
		DesiredCapacity: targetCapacity,
		Wait:            true,
	})
	if err != nil {
		return err
	}

	if err = client.printECSStatus(options.Cluster); err != nil {
		return err
	}

	// A status of instance in autoscaling group is InService doesn't mean the
	// container instance is registered. We should make sure container instances
	// are registered
	fmt.Fprintln(client.stdout, "Wait until ECS container instances are registered...")
	err = client.WaitUntilECSContainerInstancesAreRegistered(options.Cluster, targetCapacity)
	if err != nil {
		return err
	}

	if err = client.printECSStatus(options.Cluster); err != nil {
		return err
	}

	// drain old container instances and wait until no task running
	oldNodeArns := []*string{}
	for _, oldNode := range oldNodes {
		oldNodeArns = append(oldNodeArns, oldNode.ContainerInstanceArn)
	}
	fmt.Fprintf(client.stdout, "Drain old container instances and wait until no task running...\n%v\n", awsutil.Prettify(oldNodeArns))
	err = client.ECSNodeDrain(ECSNodeDrainOptions{
		Cluster:            options.Cluster,
		ContainerInstances: oldNodeArns,
		Wait:               true,
	})
	if err != nil {
		return err
	}

	if err = client.printECSStatus(options.Cluster); err != nil {
		return err
	}

	// All old container instances are drained doesn't mean all services are stable.
	// It depends on the deployment strategy of each service.
	// We should make sure all services are stable
	fmt.Fprintln(client.stdout, "Wait until all ECS services stable...")
	err = client.WaitUntilECSAllServicesStable(options.Cluster)
	if err != nil {
		return err
	}

	if err = client.printECSStatus(options.Cluster); err != nil {
		return err
	}

	// A stable state for all services does not mean that all targets are healthy.
	// We need to explicitly confirm it.
	fmt.Fprintln(client.stdout, "Wait until all targets healthy...")
	err = client.WaitUntilECSAllTargetsInService(options.Cluster)
	if err != nil {
		return err
	}

	if err = client.printECSStatus(options.Cluster); err != nil {
		return err
	}

	// Select instances to protect from scale in.
	// By setting "scale-in protection" to instances created at scale-out,
	// the intended instances (instances created before scale-in) are only terminated at scale-in process.
	protectInstanceIds, err := client.selectInstanceToProtectFromScaleIn(oldNodes, options.Cluster)
	if err != nil {
		return err
	}

	// Enable scale in protection for specific instances before scaling in.
	// In the default termination policy of auto scaling, instances close to the next billing time will terminate when the launch configuration is the same.
	// By running this function, your auto scaling group scales out, then scales in.
	// During scale in, instances created during scale out may be subject to termination.
	// To prevent this, set scale in protection for instances created at scale out.
	// https://docs.aws.amazon.com/autoscaling/ec2/userguide/as-instance-termination.html
	fmt.Fprintln(client.stdout, "Setting scale in protection: ", awsutil.Prettify(protectInstanceIds))
	// set "scale in protection" to instances created at scale-out.
	err = client.AutoScalingSetInstanceProtection(AutoScalingSetInstanceProtectionOptions{
		options.AsgName,
		protectInstanceIds,
		true})
	if err != nil {
		return err
	}

	// restore the desired capacity and wait until old instances are discarded
	fmt.Fprintf(client.stdout, "Update autoscaling group %s (DesiredCapacity: %d => %d)\n", options.AsgName, targetCapacity, desiredCapacity)

	err = client.AutoscalingUpdate(AutoscalingUpdateOptions{
		AsgName:         options.AsgName,
		DesiredCapacity: desiredCapacity,
		Wait:            true,
	})
	if err != nil {
		return err
	}

	// remove "scale in protection" to instances created at scale-out.
	fmt.Fprintln(client.stdout, "Removing scale in protection: ", awsutil.Prettify(protectInstanceIds))
	err = client.AutoScalingSetInstanceProtection(AutoScalingSetInstanceProtectionOptions{
		options.AsgName,
		protectInstanceIds,
		false})
	if err != nil {
		return err
	}

	if err = client.printECSStatus(options.Cluster); err != nil {
		return err
	}

	fmt.Fprintln(client.stdout, "end: ecs node renew")
	return nil
}

// selectInstanceToProtectFromScaleIn selects instance to protect from Scale in.
// instance select rule:
//   instances after scale out - instances before scale out - instances which already set `InstanceProtection==true`
func (client *Client) selectInstanceToProtectFromScaleIn(oldNodes []*ecs.ContainerInstance, cluster string) ([]*string, error) {
	// Get a list of instance IDs before auto scaling
	var oldInstanceIds []*string
	for _, oldNode := range oldNodes {
		oldInstanceIds = append(oldInstanceIds, oldNode.Ec2InstanceId)
	}

	// Get a list of instances after auto scaling
	allNodes, err := client.findECSNodes(cluster)
	if err != nil {
		return nil, err
	}

	// Get a list of instance IDs after auto scaling
	var allInstanceIds []*string
	for _, allNode := range allNodes {
		allInstanceIds = append(allInstanceIds, allNode.Ec2InstanceId)
	}

	// get newly created nodes (allInstanceIds - oldInstanceIds)
	newInstanceIds := difference(allInstanceIds, oldInstanceIds)

	// exclude ProtectedFromScaleIn == true nodes
	params := &autoscaling.DescribeAutoScalingInstancesInput{
		InstanceIds: newInstanceIds,
	}
	response, err := client.AutoScaling.DescribeAutoScalingInstances(params)
	if err != nil {
		return nil, errors.Wrap(err, "DescribeAutoScalingGroups failed:")
	}

	var targetInstanceIds []*string
	for _, instance := range response.AutoScalingInstances {
		if *instance.ProtectedFromScaleIn == false {
			targetInstanceIds = append(targetInstanceIds, instance.InstanceId)
		}
	}
	return targetInstanceIds, nil
}

// difference returns the elements in `a` that aren't in `b`.
func difference(a, b []*string) []*string {
	mb := make(map[string]struct{}, len(b))
	for _, x := range b {
		mb[*x] = struct{}{}
	}
	var diff []*string
	for _, x := range a {
		if _, ok := mb[*x]; !ok {
			diff = append(diff, x)
		}
	}
	return diff
}
<|endoftext|>


<commit_before>package minnow

import (
	"crypto/hmac"
	"crypto/sha512"
	"encoding/binary"
	"errors"
	"hash"
	"io"
	"log"
	"sync"
	"time"

	cryptoRand "crypto/rand"
	mathRand "math/rand"
)

var ErrAlreadyClosed = errors.New("Writer has already been closed")
var ErrMaxLengthExceeded = errors.New("Maximum message length (1<<64 - 1) exceeded")

var InitialMessageSize = 1024

type Packet struct {
       Header  PacketHeader
       Payload []byte
}

type PacketHeader struct {
	SequenceN uint64
	Mac       [64]byte
	Size      uint64
}

type MessageWriteCloser struct {
	closed bool
	hash hash.Hash
	message []byte
	destination io.WriteCloser
	wg *sync.WaitGroup
}

func NewMessageWriteCloser(secret []byte, w io.WriteCloser) *MessageWriteCloser {
	return &MessageWriteCloser{
		closed: false,
		hash: hmac.New(sha512.New, secret),
		message: make([]byte, InitialMessageSize),
		destination: w,
		wg: new(sync.WaitGroup),
	}
}

// Writes a message fragment to the internal buffer
// Return value is always (len(m), nil)
func (mw *MessageWriteCloser) Write(m []byte) (int, error) {
	mw.message = append(mw.message, m...)
	return len(m), nil
}

// Close is when we actually get the end of the message and can write it
func (mw *MessageWriteCloser) Close() (error) {
	if !mw.closed {
		log.Printf("Writing message:\n%s", string(mw.message))
		mw.closed = true
		// Write chaff asynchronously
		mw.wg.Add(1)
		go binaryChaff(mw.destination, mw.wg.Done)

		// Write the real message
		_, err := mw.writeMessage()

		// Wait for chaff to finish writing
		mw.wg.Wait()

		// Either close and handle error or just close
		if err != nil {
			// Always close the destination
			mw.destination.Close()
			return err
		}
		return mw.destination.Close()
	}
	return ErrAlreadyClosed
}

func (mw *MessageWriteCloser) writeMessage() (n int, err error) {
	for n, _ = range mw.message {
		_, err = mw.writePacket(uint64(n), mw.message[n:n + 1])
		log.Printf("Wrote %s with err: %v", string(mw.message[n:n + 1]), err)
		if err != nil {
			return
		}
		time.Sleep(time.Duration(mathRand.Intn(10)) * time.Millisecond)
	}
	return
}

func (mw *MessageWriteCloser) writePacket(seqn uint64, packet []byte) (int, error) {
	var mbuf [64]byte

	mw.hash.Write(packet)
	defer mw.hash.Reset()

	m := mw.hash.Sum(nil)

	for i, _ := range mbuf {
		mbuf[i] = m[i]
	}

	h := PacketHeader{
		SequenceN: seqn,
		Mac:       mbuf,
		Size:      uint64(len(packet)),
	}

	err := binary.Write(mw.destination, binary.BigEndian, h)
	if err != nil {
		return 0, err
	}

	n, err := mw.destination.Write(packet)
	if err != nil {
		return n, err
	}

	return len(packet), nil
}

func binaryChaff(w io.Writer, done func()) {
	defer done()

	var err error
	ph := &PacketHeader{Size: 1}
	macbuf := make([]byte, 64, 64)
	payloadbuf := make([]byte, 1, 1)
	sz := uint64(0)
	for sz == 0 {
		//FIXME Need to base amount of chaff of message size without
		//      hinting at message size
		sz = uint64(mathRand.Int63()) / 100000000000000
	}

	log.Printf("Writing %d chaff packets", sz)

	for ; ph.SequenceN < sz; ph.SequenceN++ {
		_, err = cryptoRand.Read(macbuf)
		if err != nil {
			return
		}

		for i, _ := range macbuf {
			ph.Mac[i] = macbuf[i]
		}

		err = binary.Write(w, binary.BigEndian, ph)
		if err != nil {
			log.Printf("Failed to write chaff header: %s", err)
			return
		}

		_, err = cryptoRand.Read(payloadbuf)
		if err != nil {
			return
		}

		_, err = w.Write(payloadbuf)
		if err != nil {
			return
		}
		time.Sleep(time.Duration(mathRand.Intn(2)) * time.Millisecond)
	}

}
<commit_msg>Properly use append<commit_after>package minnow

import (
	"crypto/hmac"
	"crypto/sha512"
	"encoding/binary"
	"errors"
	"hash"
	"io"
	"log"
	"sync"
	"time"

	cryptoRand "crypto/rand"
	mathRand "math/rand"
)

var ErrAlreadyClosed = errors.New("Writer has already been closed")
var ErrMaxLengthExceeded = errors.New("Maximum message length (1<<64 - 1) exceeded")

type Packet struct {
       Header  PacketHeader
       Payload []byte
}

type PacketHeader struct {
	SequenceN uint64
	Mac       [64]byte
	Size      uint64
}

type MessageWriteCloser struct {
	closed bool
	hash hash.Hash
	message []byte
	destination io.WriteCloser
	wg *sync.WaitGroup
}

func NewMessageWriteCloser(secret []byte, w io.WriteCloser) *MessageWriteCloser {
	return &MessageWriteCloser{
		closed: false,
		hash: hmac.New(sha512.New, secret),
		message: make([]byte, 0),
		destination: w,
		wg: new(sync.WaitGroup),
	}
}

// Writes a message fragment to the internal buffer
// Return value is always (len(m), nil)
func (mw *MessageWriteCloser) Write(m []byte) (int, error) {
	mw.message = append(mw.message, m...)
	return len(m), nil
}

// Close is when we actually get the end of the message and can write it
func (mw *MessageWriteCloser) Close() (error) {
	if !mw.closed {
		log.Printf("Writing message:\n%s", string(mw.message))
		mw.closed = true
		// Write chaff asynchronously
		mw.wg.Add(1)
		go binaryChaff(mw.destination, mw.wg.Done)

		// Write the real message
		_, err := mw.writeMessage()

		// Wait for chaff to finish writing before closing connection
		mw.wg.Wait()

		// Either close and handle error or just close
		if err != nil {
			// Always close the destination
			mw.destination.Close()
			return err
		}
		return mw.destination.Close()
	}
	return ErrAlreadyClosed
}

func (mw *MessageWriteCloser) writeMessage() (n int, err error) {
	log.Printf("Writing message <<%s>> len: %d cap: %d", string(mw.message), len(mw.message), cap(mw.message))
	for n, _ = range mw.message {
		_, err = mw.writePacket(uint64(n), mw.message[n:n + 1])
		log.Printf("Wrote seq %d: %v with err: %v", n, mw.message[n:n + 1], err)
		if err != nil {
			return
		}
		time.Sleep(time.Duration(mathRand.Intn(10)) * time.Millisecond)
	}
	return
}

func (mw *MessageWriteCloser) writePacket(seqn uint64, packet []byte) (int, error) {
	var mbuf [64]byte

	mw.hash.Write(packet)
	defer mw.hash.Reset()

	m := mw.hash.Sum(nil)

	for i, _ := range mbuf {
		mbuf[i] = m[i]
	}

	h := PacketHeader{
		SequenceN: seqn,
		Mac:       mbuf,
		Size:      uint64(len(packet)),
	}

	err := binary.Write(mw.destination, binary.BigEndian, h)
	if err != nil {
		return 0, err
	}

	n, err := mw.destination.Write(packet)
	if err != nil {
		return n, err
	}

	return len(packet), nil
}

func binaryChaff(w io.Writer, done func()) {
	defer done()

	var err error
	ph := &PacketHeader{Size: 1}
	macbuf := make([]byte, 64, 64)
	payloadbuf := make([]byte, 1, 1)
	sz := uint64(0)
	for sz == 0 {
		//FIXME Need to base amount of chaff of message size without
		//      hinting at message size
		sz = uint64(mathRand.Int63()) / 100000000000000
	}

	log.Printf("Writing %d chaff packets", sz)

	for ; ph.SequenceN < sz; ph.SequenceN++ {
		_, err = cryptoRand.Read(macbuf)
		if err != nil {
			return
		}

		for i, _ := range macbuf {
			ph.Mac[i] = macbuf[i]
		}

		err = binary.Write(w, binary.BigEndian, ph)
		if err != nil {
			log.Printf("Failed to write chaff header: %s", err)
			return
		}

		_, err = cryptoRand.Read(payloadbuf)
		if err != nil {
			return
		}

		_, err = w.Write(payloadbuf)
		if err != nil {
			return
		}
		time.Sleep(time.Duration(mathRand.Intn(2)) * time.Millisecond)
	}

}
<|endoftext|>


<commit_before>package mixmux

import (
	"net/http"

	"github.com/dimfeld/httptreemux"
	"github.com/julienschmidt/httprouter"
)

// MixMuxer defines available methods for wrapped multiplexers.
type MixMuxer interface {
	Options(path string, h http.Handler)
	Get(path string, h http.Handler)
	Post(path string, h http.Handler)
	Put(path string, h http.Handler)
	Patch(path string, h http.Handler)
	Delete(path string, h http.Handler)
	Head(path string, h http.Handler)
}

// Router wraps HTTPRouter.
type Router struct {
	*httprouter.Router
}

// NewHTTPRouter returns a wrapped HTTPRouter.
func NewHTTPRouter() *Router {
	return &Router{httprouter.New()}
}

// Options takes a path and http.Handler and adds them to the mux.
func (r *Router) Options(path string, h http.Handler) {
	r.Handle("OPTIONS", path, httpRouterWrapHandler(h))
}

// Get takes a path and http.Handler and adds them to the mux.
func (r *Router) Get(path string, h http.Handler) {
	r.Handle("GET", path, httpRouterWrapHandler(h))
}

// Post takes a path and http.Handler and adds them to the mux.
func (r *Router) Post(path string, h http.Handler) {
	r.Handle("POST", path, httpRouterWrapHandler(h))
}

// Put takes a path and http.Handler and adds them to the mux.
func (r *Router) Put(path string, h http.Handler) {
	r.Handle("PUT", path, httpRouterWrapHandler(h))
}

// Patch takes a path and http.Handler and adds them to the mux.
func (r *Router) Patch(path string, h http.Handler) {
	r.Handle("PATCH", path, httpRouterWrapHandler(h))
}

// Delete takes a path and http.Handler and adds them to the mux.
func (r *Router) Delete(path string, h http.Handler) {
	r.Handle("DELETE", path, httpRouterWrapHandler(h))
}

// Head takes a path and http.Handler and adds them to the mux.
func (r *Router) Head(path string, h http.Handler) {
	r.Handle("HEAD", path, httpRouterWrapHandler(h))
}

func httpRouterWrapHandler(h http.Handler) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
		h.ServeHTTP(w, r)
	}
}

// TreeMux wraps HTTPTreeMux.
type TreeMux struct {
	*httptreemux.TreeMux
}

// NewTreeMux returns a wrapped HTTPTreeMux.
func NewTreeMux() *TreeMux {
	return &TreeMux{httptreemux.New()}
}

// Options takes a path and http.Handler and adds them to the mux.
func (tm *TreeMux) Options(path string, h http.Handler) {
	tm.Handle("OPTIONS", path, treeMuxWrapHandler(h))
}

// Get takes a path and http.Handler and adds them to the mux.
func (tm *TreeMux) Get(path string, h http.Handler) {
	tm.Handle("GET", path, treeMuxWrapHandler(h))
}

// Post takes a path and http.Handler and adds them to the mux.
func (tm *TreeMux) Post(path string, h http.Handler) {
	tm.Handle("POST", path, treeMuxWrapHandler(h))
}

// Put takes a path and http.Handler and adds them to the mux.
func (tm *TreeMux) Put(path string, h http.Handler) {
	tm.Handle("PUT", path, treeMuxWrapHandler(h))
}

// Patch takes a path and http.Handler and adds them to the mux.
func (tm *TreeMux) Patch(path string, h http.Handler) {
	tm.Handle("PATCH", path, treeMuxWrapHandler(h))
}

// Delete takes a path and http.Handler and adds them to the mux.
func (tm *TreeMux) Delete(path string, h http.Handler) {
	tm.Handle("DELETE", path, treeMuxWrapHandler(h))
}

// Head takes a path and http.Handler and adds them to the mux.
func (tm *TreeMux) Head(path string, h http.Handler) {
	tm.Handle("HEAD", path, treeMuxWrapHandler(h))
}

func treeMuxWrapHandler(h http.Handler) httptreemux.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request, _ map[string]string) {
		h.ServeHTTP(w, r)
	}
}
<commit_msg>Added package comment.<commit_after>// Package mixmux wraps HTTPRouter and HTTPTreeMux to provide more consistent
// and more idiomatic APIs.
package mixmux

import (
	"net/http"

	"github.com/dimfeld/httptreemux"
	"github.com/julienschmidt/httprouter"
)

// MixMuxer defines available methods for wrapped multiplexers.
type MixMuxer interface {
	Options(path string, h http.Handler)
	Get(path string, h http.Handler)
	Post(path string, h http.Handler)
	Put(path string, h http.Handler)
	Patch(path string, h http.Handler)
	Delete(path string, h http.Handler)
	Head(path string, h http.Handler)
}

// Router wraps HTTPRouter.
type Router struct {
	*httprouter.Router
}

// NewHTTPRouter returns a wrapped HTTPRouter.
func NewHTTPRouter() *Router {
	return &Router{httprouter.New()}
}

// Options takes a path and http.Handler and adds them to the mux.
func (r *Router) Options(path string, h http.Handler) {
	r.Handle("OPTIONS", path, httpRouterWrapHandler(h))
}

// Get takes a path and http.Handler and adds them to the mux.
func (r *Router) Get(path string, h http.Handler) {
	r.Handle("GET", path, httpRouterWrapHandler(h))
}

// Post takes a path and http.Handler and adds them to the mux.
func (r *Router) Post(path string, h http.Handler) {
	r.Handle("POST", path, httpRouterWrapHandler(h))
}

// Put takes a path and http.Handler and adds them to the mux.
func (r *Router) Put(path string, h http.Handler) {
	r.Handle("PUT", path, httpRouterWrapHandler(h))
}

// Patch takes a path and http.Handler and adds them to the mux.
func (r *Router) Patch(path string, h http.Handler) {
	r.Handle("PATCH", path, httpRouterWrapHandler(h))
}

// Delete takes a path and http.Handler and adds them to the mux.
func (r *Router) Delete(path string, h http.Handler) {
	r.Handle("DELETE", path, httpRouterWrapHandler(h))
}

// Head takes a path and http.Handler and adds them to the mux.
func (r *Router) Head(path string, h http.Handler) {
	r.Handle("HEAD", path, httpRouterWrapHandler(h))
}

func httpRouterWrapHandler(h http.Handler) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
		h.ServeHTTP(w, r)
	}
}

// TreeMux wraps HTTPTreeMux.
type TreeMux struct {
	*httptreemux.TreeMux
}

// NewTreeMux returns a wrapped HTTPTreeMux.
func NewTreeMux() *TreeMux {
	return &TreeMux{httptreemux.New()}
}

// Options takes a path and http.Handler and adds them to the mux.
func (tm *TreeMux) Options(path string, h http.Handler) {
	tm.Handle("OPTIONS", path, treeMuxWrapHandler(h))
}

// Get takes a path and http.Handler and adds them to the mux.
func (tm *TreeMux) Get(path string, h http.Handler) {
	tm.Handle("GET", path, treeMuxWrapHandler(h))
}

// Post takes a path and http.Handler and adds them to the mux.
func (tm *TreeMux) Post(path string, h http.Handler) {
	tm.Handle("POST", path, treeMuxWrapHandler(h))
}

// Put takes a path and http.Handler and adds them to the mux.
func (tm *TreeMux) Put(path string, h http.Handler) {
	tm.Handle("PUT", path, treeMuxWrapHandler(h))
}

// Patch takes a path and http.Handler and adds them to the mux.
func (tm *TreeMux) Patch(path string, h http.Handler) {
	tm.Handle("PATCH", path, treeMuxWrapHandler(h))
}

// Delete takes a path and http.Handler and adds them to the mux.
func (tm *TreeMux) Delete(path string, h http.Handler) {
	tm.Handle("DELETE", path, treeMuxWrapHandler(h))
}

// Head takes a path and http.Handler and adds them to the mux.
func (tm *TreeMux) Head(path string, h http.Handler) {
	tm.Handle("HEAD", path, treeMuxWrapHandler(h))
}

func treeMuxWrapHandler(h http.Handler) httptreemux.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request, _ map[string]string) {
		h.ServeHTTP(w, r)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"net/http"

	log "github.com/Sirupsen/logrus"
	"github.com/elazarl/goproxy"
	"io/ioutil"
)

type DBClient struct {
	cache Cache
	http  *http.Client
}

// request holds structure for request
type request struct {
	details requestDetails
}

var emptyResp = &http.Response{}

// requestDetails stores information about request, it's used for creating unique hash and also as a payload structure
type requestDetails struct {
	Path        string              `json:"path"`
	Method      string              `json:"method"`
	Destination string              `json:"destination"`
	Scheme      string              `json:"scheme"`
	Query       string              `json:"query"`
	Body        string              `json:"body"`
	RemoteAddr  string              `json:"remoteAddr"`
	Headers     map[string][]string `json:"headers"`
}

// hash returns unique hash key for request
func (r *request) hash() string {
	h := md5.New()
	io.WriteString(h, fmt.Sprintf("%s%s%s%s", r.details.Destination, r.details.Path, r.details.Method, r.details.Query))
	return fmt.Sprintf("%x", h.Sum(nil))
}

// res structure hold response body from external service, body is not decoded and is supposed
// to be bytes, however headers should provide all required information for later decoding
// by the client.
type response struct {
	Status  int                 `json:"status"`
	Body    string              `json:"body"`
	Headers map[string][]string `json:"headers"`
}

// Payload structure holds request and response structure
type Payload struct {
	Response response       `json:"response"`
	Request  requestDetails `json:"request"`
	ID       string         `json:"id"`
}

// recordRequest saves request for later playback
func (d *DBClient) captureRequest(req *http.Request) (*http.Response, error) {

	// this is mainly for testing, since when you create
	if req.Body == nil {
		req.Body = ioutil.NopCloser(bytes.NewBuffer([]byte("")))
	}

	reqBody, err := ioutil.ReadAll(req.Body)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
			"mode":  AppConfig.mode,
		}).Error("Got error when reading request body")
	}
	log.WithFields(log.Fields{
		"body": string(reqBody),
	}).Info("got request body")
	req.Body = ioutil.NopCloser(bytes.NewBuffer(reqBody))

	// forwarding request
	resp, err := d.doRequest(req)

	if err == nil {
		respBody, err := extractBody(resp)
		if err != nil {
			// copying the response body did not work
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
				}).Error("Failed to copy response body.")
			}
		}

		// saving response body with request/response meta to cache
		go d.save(req, resp, respBody, reqBody)
	}

	// return new response or error here
	return resp, err
}

func copyBody(body io.ReadCloser) (resp1, resp2 io.ReadCloser, err error) {
	var buf bytes.Buffer
	if _, err = buf.ReadFrom(body); err != nil {
		return nil, nil, err
	}
	if err = body.Close(); err != nil {
		return nil, nil, err
	}
	return ioutil.NopCloser(&buf), ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil
}

func extractBody(resp *http.Response) (extract []byte, err error) {
	save := resp.Body
	savecl := resp.ContentLength

	save, resp.Body, err = copyBody(resp.Body)

	if err != nil {
		return
	}
	defer resp.Body.Close()
	extract, err = ioutil.ReadAll(resp.Body)

	resp.Body = save
	resp.ContentLength = savecl
	if err != nil {
		return nil, err
	}
	return extract, nil
}

// doRequest performs original request and returns response that should be returned to client and error (if there is one)
func (d *DBClient) doRequest(request *http.Request) (*http.Response, error) {
	// We can't have this set. And it only contains "/pkg/net/http/" anyway
	request.RequestURI = ""

	if AppConfig.middleware != "" {
		var payload Payload

		c := NewConstructor(request, payload)
		c.ApplyMiddleware(AppConfig.middleware)

		request = c.reconstructRequest()

	}

	resp, err := d.http.Do(request)

	if err != nil {
		log.WithFields(log.Fields{
			"error":  err.Error(),
			"host":   request.Host,
			"method": request.Method,
			"path":   request.URL.Path,
		}).Error("Could not forward request.")
		return nil, err
	}

	log.WithFields(log.Fields{
		"host":   request.Host,
		"method": request.Method,
		"path":   request.URL.Path,
	}).Info("Response got successfuly!")

	resp.Header.Set("hoverfly", "Was-Here")
	return resp, nil

}

// save gets request fingerprint, extracts request body, status code and headers, then saves it to cache
func (d *DBClient) save(req *http.Request, resp *http.Response, respBody []byte, reqBody []byte) {
	// record request here
	key := getRequestFingerprint(req)

	if resp == nil {
		resp = emptyResp
	} else {
		responseObj := response{
			Status:  resp.StatusCode,
			Body:    string(respBody),
			Headers: resp.Header,
		}

		log.WithFields(log.Fields{
			"path":          req.URL.Path,
			"rawQuery":      req.URL.RawQuery,
			"requestMethod": req.Method,
			"bodyLen":       len(reqBody),
			"destination":   req.Host,
			"hashKey":       key,
		}).Info("Capturing")

		requestObj := requestDetails{
			Path:        req.URL.Path,
			Method:      req.Method,
			Destination: req.Host,
			Scheme:      req.URL.Scheme,
			Query:       req.URL.RawQuery,
			Body:        string(reqBody),
			RemoteAddr:  req.RemoteAddr,
			Headers:     req.Header,
		}

		payload := Payload{
			Response: responseObj,
			Request:  requestObj,
			ID:       key,
		}
		// converting it to json bytes
		bts, err := json.Marshal(payload)

		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to marshal json")
		} else {
			d.cache.set(key, bts)
		}
	}

}

// getAllRecordsRaw returns raw (json string) for all records
func (d *DBClient) getAllRecordsRaw() ([]string, error) {
	keys, err := d.cache.getAllKeys()

	if err == nil {

		// checking if there are any records
		if len(keys) == 0 {
			return nil, nil
		}

		jsonStrs, err := d.cache.getAllValues(keys)

		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to get all values (raw)")
			return nil, err
		} else {
			return jsonStrs, nil
		}

	} else {
		return nil, err
	}
}

// getAllRecords returns all stored
func (d *DBClient) getAllRecords() ([]Payload, error) {
	var payloads []Payload

	jsonStrs, err := d.getAllRecordsRaw()

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to get all values")
	} else {

		if jsonStrs != nil {
			for _, v := range jsonStrs {
				var pl Payload
				err = json.Unmarshal([]byte(v), &pl)
				if err != nil {
					log.WithFields(log.Fields{
						"error": err.Error(),
						"json":  v,
					}).Warning("Failed to deserialize json")
				} else {
					payloads = append(payloads, pl)
				}
			}
		}
	}

	return payloads, err

}

// deleteAllRecords deletes all recorded requests
func (d *DBClient) deleteAllRecords() error {
	keys, err := d.cache.getAllKeys()

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Warning("Failed to get keys, cannot delete all records")
		return err
	} else {
		for _, v := range keys {
			err := d.cache.delete(v)
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
					"key":   v,
				}).Warning("Failed to delete key...")
			}
		}
		return nil
	}
}

// getRequestFingerprint returns request hash
func getRequestFingerprint(req *http.Request) string {
	details := requestDetails{Path: req.URL.Path, Method: req.Method, Destination: req.Host, Query: req.URL.RawQuery}
	r := request{details: details}
	return r.hash()
}

// getResponse returns stored response from cache
func (d *DBClient) getResponse(req *http.Request) *http.Response {

	key := getRequestFingerprint(req)
	var payload Payload

	payloadBts, err := d.cache.get(key)

	if err == nil {
		// getting cache response
		err = json.Unmarshal(payloadBts, &payload)
		if err != nil {
			log.Error(err)
			// what now?
		}

		c := NewConstructor(req, payload)

		if AppConfig.middleware != "" {
			_ = c.ApplyMiddleware(AppConfig.middleware)
		}

		response := c.reconstructResponse()

		log.WithFields(log.Fields{
			"key":        key,
			"status":     payload.Response.Status,
			"bodyLength": response.ContentLength,
			"mode":       AppConfig.mode,
		}).Info("Response found, returning")

		return response

	} else {
		log.WithFields(log.Fields{
			"error":       err.Error(),
			"mode":        AppConfig.mode,
			"query":       req.URL.RawQuery,
			"path":        req.URL.RawPath,
			"destination": req.Host,
			"method":      req.Method,
		}).Warn("Failed to retrieve response from cache")
		// return error? if we return nil - proxy forwards request to original destination
		return goproxy.NewResponse(req,
			goproxy.ContentTypeText, http.StatusPreconditionFailed,
			"Coudldn't find recorded request, please record it first!")
	}

}

// modifyRequestResponse modifies outgoing request and then modifies incoming response, neither request nor response
// is saved to cache.
func (d *DBClient) modifyRequestResponse(req *http.Request, middleware string) (*http.Response, error) {

	// modifying request
	resp, err := d.doRequest(req)

	if err != nil {
		return nil, err
	}

	// preparing payload
	bodyBytes, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		log.WithFields(log.Fields{
			"error":      err.Error(),
			"middleware": middleware,
		}).Error("Failed to read response body after sending modified request")
		return nil, err
	}

	r := response{
		Status:  resp.StatusCode,
		Body:    string(bodyBytes),
		Headers: resp.Header,
	}
	payload := Payload{Response: r}

	c := NewConstructor(req, payload)
	// applying middleware to modify response
	err = c.ApplyMiddleware(middleware)

	if err != nil {
		return nil, err
	}

	newResponse := c.reconstructResponse()

	log.WithFields(log.Fields{
		"status":     newResponse.StatusCode,
		"middleware": middleware,
		"mode":       AppConfig.mode,
	}).Info("Response modified, returning")

	return newResponse, nil

}
<commit_msg>using bolt methods<commit_after>package main

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"net/http"

	log "github.com/Sirupsen/logrus"
	"github.com/elazarl/goproxy"
	"io/ioutil"
)

type DBClient struct {
	cache Cache
	http  *http.Client
}

// request holds structure for request
type request struct {
	details requestDetails
}

var emptyResp = &http.Response{}

// requestDetails stores information about request, it's used for creating unique hash and also as a payload structure
type requestDetails struct {
	Path        string              `json:"path"`
	Method      string              `json:"method"`
	Destination string              `json:"destination"`
	Scheme      string              `json:"scheme"`
	Query       string              `json:"query"`
	Body        string              `json:"body"`
	RemoteAddr  string              `json:"remoteAddr"`
	Headers     map[string][]string `json:"headers"`
}

// hash returns unique hash key for request
func (r *request) hash() string {
	h := md5.New()
	io.WriteString(h, fmt.Sprintf("%s%s%s%s", r.details.Destination, r.details.Path, r.details.Method, r.details.Query))
	return fmt.Sprintf("%x", h.Sum(nil))
}

// res structure hold response body from external service, body is not decoded and is supposed
// to be bytes, however headers should provide all required information for later decoding
// by the client.
type response struct {
	Status  int                 `json:"status"`
	Body    string              `json:"body"`
	Headers map[string][]string `json:"headers"`
}

// Payload structure holds request and response structure
type Payload struct {
	Response response       `json:"response"`
	Request  requestDetails `json:"request"`
	ID       string         `json:"id"`
}

// recordRequest saves request for later playback
func (d *DBClient) captureRequest(req *http.Request) (*http.Response, error) {

	// this is mainly for testing, since when you create
	if req.Body == nil {
		req.Body = ioutil.NopCloser(bytes.NewBuffer([]byte("")))
	}

	reqBody, err := ioutil.ReadAll(req.Body)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
			"mode":  AppConfig.mode,
		}).Error("Got error when reading request body")
	}
	log.WithFields(log.Fields{
		"body": string(reqBody),
	}).Info("got request body")
	req.Body = ioutil.NopCloser(bytes.NewBuffer(reqBody))

	// forwarding request
	resp, err := d.doRequest(req)

	if err == nil {
		respBody, err := extractBody(resp)
		if err != nil {
			// copying the response body did not work
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
				}).Error("Failed to copy response body.")
			}
		}

		// saving response body with request/response meta to cache
		go d.save(req, resp, respBody, reqBody)
	}

	// return new response or error here
	return resp, err
}

func copyBody(body io.ReadCloser) (resp1, resp2 io.ReadCloser, err error) {
	var buf bytes.Buffer
	if _, err = buf.ReadFrom(body); err != nil {
		return nil, nil, err
	}
	if err = body.Close(); err != nil {
		return nil, nil, err
	}
	return ioutil.NopCloser(&buf), ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil
}

func extractBody(resp *http.Response) (extract []byte, err error) {
	save := resp.Body
	savecl := resp.ContentLength

	save, resp.Body, err = copyBody(resp.Body)

	if err != nil {
		return
	}
	defer resp.Body.Close()
	extract, err = ioutil.ReadAll(resp.Body)

	resp.Body = save
	resp.ContentLength = savecl
	if err != nil {
		return nil, err
	}
	return extract, nil
}

// doRequest performs original request and returns response that should be returned to client and error (if there is one)
func (d *DBClient) doRequest(request *http.Request) (*http.Response, error) {
	// We can't have this set. And it only contains "/pkg/net/http/" anyway
	request.RequestURI = ""

	if AppConfig.middleware != "" {
		var payload Payload

		c := NewConstructor(request, payload)
		c.ApplyMiddleware(AppConfig.middleware)

		request = c.reconstructRequest()

	}

	resp, err := d.http.Do(request)

	if err != nil {
		log.WithFields(log.Fields{
			"error":  err.Error(),
			"host":   request.Host,
			"method": request.Method,
			"path":   request.URL.Path,
		}).Error("Could not forward request.")
		return nil, err
	}

	log.WithFields(log.Fields{
		"host":   request.Host,
		"method": request.Method,
		"path":   request.URL.Path,
	}).Info("Response got successfuly!")

	resp.Header.Set("hoverfly", "Was-Here")
	return resp, nil

}

// save gets request fingerprint, extracts request body, status code and headers, then saves it to cache
func (d *DBClient) save(req *http.Request, resp *http.Response, respBody []byte, reqBody []byte) {
	// record request here
	key := getRequestFingerprint(req)

	if resp == nil {
		resp = emptyResp
	} else {
		responseObj := response{
			Status:  resp.StatusCode,
			Body:    string(respBody),
			Headers: resp.Header,
		}

		log.WithFields(log.Fields{
			"path":          req.URL.Path,
			"rawQuery":      req.URL.RawQuery,
			"requestMethod": req.Method,
			"bodyLen":       len(reqBody),
			"destination":   req.Host,
			"hashKey":       key,
		}).Info("Capturing")

		requestObj := requestDetails{
			Path:        req.URL.Path,
			Method:      req.Method,
			Destination: req.Host,
			Scheme:      req.URL.Scheme,
			Query:       req.URL.RawQuery,
			Body:        string(reqBody),
			RemoteAddr:  req.RemoteAddr,
			Headers:     req.Header,
		}

		payload := Payload{
			Response: responseObj,
			Request:  requestObj,
			ID:       key,
		}
		// converting it to json bytes
		bts, err := json.Marshal(payload)

		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to marshal json")
		} else {
			d.cache.Set([]byte(key), bts)
		}
	}

}

// getAllRecordsRaw returns raw (json string) for all records
func (d *DBClient) getAllRecordsRaw() ([]string, error) {
	keys, err := d.cache.getAllKeys()

	if err == nil {

		// checking if there are any records
		if len(keys) == 0 {
			return nil, nil
		}

		jsonStrs, err := d.cache.getAllValues(keys)

		if err != nil {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Error("Failed to get all values (raw)")
			return nil, err
		} else {
			return jsonStrs, nil
		}

	} else {
		return nil, err
	}
}

// getAllRecords returns all stored
func (d *DBClient) getAllRecords() ([]Payload, error) {
	var payloads []Payload

	jsonStrs, err := d.getAllRecordsRaw()

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Failed to get all values")
	} else {

		if jsonStrs != nil {
			for _, v := range jsonStrs {
				var pl Payload
				err = json.Unmarshal([]byte(v), &pl)
				if err != nil {
					log.WithFields(log.Fields{
						"error": err.Error(),
						"json":  v,
					}).Warning("Failed to deserialize json")
				} else {
					payloads = append(payloads, pl)
				}
			}
		}
	}

	return payloads, err

}

// deleteAllRecords deletes all recorded requests
func (d *DBClient) deleteAllRecords() error {
	keys, err := d.cache.getAllKeys()

	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Warning("Failed to get keys, cannot delete all records")
		return err
	} else {
		for _, v := range keys {
			err := d.cache.delete(v)
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
					"key":   v,
				}).Warning("Failed to delete key...")
			}
		}
		return nil
	}
}

// getRequestFingerprint returns request hash
func getRequestFingerprint(req *http.Request) string {
	details := requestDetails{Path: req.URL.Path, Method: req.Method, Destination: req.Host, Query: req.URL.RawQuery}
	r := request{details: details}
	return r.hash()
}

// getResponse returns stored response from cache
func (d *DBClient) getResponse(req *http.Request) *http.Response {

	key := getRequestFingerprint(req)
	var payload Payload

	payloadBts, err := d.cache.Get([]byte(key))

	if err == nil {
		// getting cache response
		err = json.Unmarshal(payloadBts, &payload)
		if err != nil {
			log.Error(err)
			// what now?
		}

		c := NewConstructor(req, payload)

		if AppConfig.middleware != "" {
			_ = c.ApplyMiddleware(AppConfig.middleware)
		}

		response := c.reconstructResponse()

		log.WithFields(log.Fields{
			"key":        key,
			"status":     payload.Response.Status,
			"bodyLength": response.ContentLength,
			"mode":       AppConfig.mode,
		}).Info("Response found, returning")

		return response

	} else {
		log.WithFields(log.Fields{
			"error":       err.Error(),
			"mode":        AppConfig.mode,
			"query":       req.URL.RawQuery,
			"path":        req.URL.RawPath,
			"destination": req.Host,
			"method":      req.Method,
		}).Warn("Failed to retrieve response from cache")
		// return error? if we return nil - proxy forwards request to original destination
		return goproxy.NewResponse(req,
			goproxy.ContentTypeText, http.StatusPreconditionFailed,
			"Coudldn't find recorded request, please record it first!")
	}

}

// modifyRequestResponse modifies outgoing request and then modifies incoming response, neither request nor response
// is saved to cache.
func (d *DBClient) modifyRequestResponse(req *http.Request, middleware string) (*http.Response, error) {

	// modifying request
	resp, err := d.doRequest(req)

	if err != nil {
		return nil, err
	}

	// preparing payload
	bodyBytes, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		log.WithFields(log.Fields{
			"error":      err.Error(),
			"middleware": middleware,
		}).Error("Failed to read response body after sending modified request")
		return nil, err
	}

	r := response{
		Status:  resp.StatusCode,
		Body:    string(bodyBytes),
		Headers: resp.Header,
	}
	payload := Payload{Response: r}

	c := NewConstructor(req, payload)
	// applying middleware to modify response
	err = c.ApplyMiddleware(middleware)

	if err != nil {
		return nil, err
	}

	newResponse := c.reconstructResponse()

	log.WithFields(log.Fields{
		"status":     newResponse.StatusCode,
		"middleware": middleware,
		"mode":       AppConfig.mode,
	}).Info("Response modified, returning")

	return newResponse, nil

}
<|endoftext|>


<commit_before>// Copyright 2011 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package twittergo

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

const (
	H_LIMIT              = "X-Rate-Limit-Limit"
	H_LIMIT_REMAIN       = "X-Rate-Limit-Remaining"
	H_LIMIT_RESET        = "X-Rate-Limit-Reset"
	H_MEDIA_LIMIT        = "X-MediaRateLimit-Limit"
	H_MEDIA_LIMIT_REMAIN = "X-MediaRateLimit-Remaining"
	H_MEDIA_LIMIT_RESET  = "X-MediaRateLimit-Reset"
)

const (
	STATUS_OK           = 200
	STATUS_CREATED      = 201
	STATUS_ACCEPTED     = 202
	STATUS_NO_CONTENT   = 204
	STATUS_INVALID      = 400
	STATUS_UNAUTHORIZED = 401
	STATUS_FORBIDDEN    = 403
	STATUS_NOTFOUND     = 404
	STATUS_LIMIT        = 429
	STATUS_GATEWAY      = 502
)

// Error returned if there was an issue parsing the response body.
type ResponseError struct {
	Body string
	Code int
}

func NewResponseError(code int, body string) ResponseError {
	return ResponseError{Code: code, Body: body}
}

func (e ResponseError) Error() string {
	return fmt.Sprintf(
		"Unable to handle response (status code %d): `%v`",
		e.Code,
		e.Body)
}

type Error map[string]interface{}

func (e Error) Code() int64 {
	return int64(e["code"].(float64))
}

func (e Error) Message() string {
	return e["message"].(string)
}

func (e Error) Error() string {
	msg := "Error %v: %v"
	return fmt.Sprintf(msg, e.Code(), e.Message())
}

type Errors map[string]interface{}

func (e Errors) Error() string {
	var (
		msg string = ""
		err Error
		ok  bool
	)
	if e["errors"] == nil {
		return msg
	}
	for _, val := range e["errors"].([]interface{}) {
		if err, ok = val.(map[string]interface{}); ok {
			msg += err.Error() + ". "
		}
	}
	return msg
}

func (e Errors) String() string {
	return e.Error()
}

func (e Errors) Errors() []Error {
	var errs = e["errors"].([]interface{})
	var out = make([]Error, len(errs))
	for i, val := range errs {
		out[i] = Error(val.(map[string]interface{}))
	}
	return out
}

type RateLimitError struct {
	Limit     uint32
	Remaining uint32
	Reset     time.Time
}

func (e RateLimitError) Error() string {
	msg := "Rate limit: %v, Remaining: %v, Reset: %v"
	return fmt.Sprintf(msg, e.Limit, e.Remaining, e.Reset)
}

type APIResponse http.Response

func (r APIResponse) HasRateLimit() bool {
	return r.Header.Get(H_LIMIT) != ""
}

func (r APIResponse) RateLimit() uint32 {
	h := r.Header.Get(H_LIMIT)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) RateLimitRemaining() uint32 {
	h := r.Header.Get(H_LIMIT_REMAIN)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) RateLimitReset() time.Time {
	h := r.Header.Get(H_LIMIT_RESET)
	i, _ := strconv.ParseUint(h, 10, 32)
	t := time.Unix(int64(i), 0)
	return t
}

func (r APIResponse) HasMediaRateLimit() bool {
	return r.Header.Get(H_MEDIA_LIMIT) != ""
}

func (r APIResponse) MediaRateLimit() uint32 {
	h := r.Header.Get(H_MEDIA_LIMIT)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) MediaRateLimitRemaining() uint32 {
	h := r.Header.Get(H_MEDIA_LIMIT_REMAIN)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) MediaRateLimitReset() time.Time {
	h := r.Header.Get(H_MEDIA_LIMIT_RESET)
	i, _ := strconv.ParseUint(h, 10, 32)
	t := time.Unix(int64(i), 0)
	return t
}

func (r APIResponse) readBody() (b []byte, err error) {
	var (
		header string
		reader io.Reader
	)
	defer r.Body.Close()
	header = strings.ToLower(r.Header.Get("Content-Encoding"))
	if header == "" || strings.Index(header, "gzip") == -1 {
		reader = r.Body
	} else {
		if reader, err = gzip.NewReader(r.Body); err != nil {
			return
		}
	}
	b, err = ioutil.ReadAll(reader)
	return
}

func (r APIResponse) ReadBody() string {
	var (
		b   []byte
		err error
	)
	if b, err = r.readBody(); err != nil {
		return ""
	}
	return string(b)
}

// Parses a JSON encoded HTTP response into the supplied interface.
func (r APIResponse) Parse(out interface{}) (err error) {
	var b []byte
	switch r.StatusCode {
	case STATUS_UNAUTHORIZED:
		fallthrough
	case STATUS_NOTFOUND:
		fallthrough
	case STATUS_GATEWAY:
		fallthrough
	case STATUS_FORBIDDEN:
		fallthrough
	case STATUS_INVALID:
		e := &Errors{}
		if b, err = r.readBody(); err != nil {
			return
		}
		if err = json.Unmarshal(b, e); err != nil {
			err = NewResponseError(r.StatusCode, string(b))
		} else {
			err = *e
		}
		return
	case STATUS_LIMIT:
		err = RateLimitError{
			Limit:     r.RateLimit(),
			Remaining: r.RateLimitRemaining(),
			Reset:     r.RateLimitReset(),
		}
		// consume the request body even if we don't need it
		r.readBody()
		return
	case STATUS_NO_CONTENT:
		return
	case STATUS_CREATED:
		fallthrough
	case STATUS_ACCEPTED:
		fallthrough
	case STATUS_OK:
		if b, err = r.readBody(); err != nil {
			return
		}
		err = json.Unmarshal(b, out)
		if err == io.EOF {
			err = nil
		}
	default:
		if b, err = r.readBody(); err != nil {
			return
		}
		err = NewResponseError(r.StatusCode, string(b))
	}
	return
}

// It's a user!
type User map[string]interface{}

func (u User) Id() uint64 {
	id, _ := strconv.ParseUint(u["id_str"].(string), 10, 64)
	return id
}

func (u User) IdStr() string {
	return u["id_str"].(string)
}

func (u User) Name() string {
	return u["name"].(string)
}

func (u User) ScreenName() string {
	return u["screen_name"].(string)
}

// It's a Tweet! (Adorably referred to by the API as a "status").
type Tweet map[string]interface{}

func (t Tweet) Id() (id uint64) {
	var (
		err error
		src = t["id_str"].(string)
	)
	if id, err = strconv.ParseUint(src, 10, 64); err != nil {
		panic(fmt.Sprintf("Could not parse ID: %v", err))
	}
	return
}

func (t Tweet) IdStr() string {
	return t["id_str"].(string)
}

func (t Tweet) Text() string {
	return t["text"].(string)
}

func (t Tweet) User() User {
	return User(t["user"].(map[string]interface{}))
}

func (t Tweet) Language() string {
	return t["lang"].(string)
}

func (t Tweet) CreatedAt() (out time.Time) {
	var (
		err error
		src = t["created_at"].(string)
	)
	if out, err = time.Parse(time.RubyDate, src); err != nil {
		panic(fmt.Sprintf("Could not parse time: %v", err))
	}
	return
}

// It's a less structured list of Tweets!
type Timeline []Tweet

// It's a structured list of Tweets!
type SearchResults map[string]interface{}

func (sr SearchResults) Statuses() []Tweet {
	var a []interface{} = sr["statuses"].([]interface{})
	b := make([]Tweet, len(a))
	for i, v := range a {
		b[i] = v.(map[string]interface{})
	}
	return b
}

func (sr SearchResults) SearchMetadata() map[string]interface{} {
	a := sr["search_metadata"].(map[string]interface{})
	return a
}

func (sr SearchResults) NextQuery() (val url.Values, err error) {
	var (
		sm   map[string]interface{}
		n    interface{}
		next string
		ok   bool
	)
	sm = sr.SearchMetadata()
	if n, ok = sm["next_results"]; !ok {
		err = fmt.Errorf("Could not get next_results from search")
		return
	}
	if next, ok = n.(string); !ok {
		err = fmt.Errorf("Could not convert next_results to str: %v", n)
		return
	}
	if next[0] == '?' {
		next = next[1:]
	}
	val, err = url.ParseQuery(next)
	return
}

// A List!
type List map[string]interface{}

func (l List) User() User {
	return User(l["user"].(map[string]interface{}))
}

func (l List) Id() (id uint64) {
	var (
		err error
		src = l["id_str"].(string)
	)
	if id, err = strconv.ParseUint(src, 10, 64); err != nil {
		panic(fmt.Sprintf("Could not parse ID: %v", err))
	}
	return
}

func (l List) IdStr() string {
	return l["id_str"].(string)
}

func (l List) Mode() string {
	return l["mode"].(string)
}

func (l List) Name() string {
	return l["name"].(string)
}

func (l List) Slug() string {
	return l["slug"].(string)
}

func (l List) SubscriberCount() int64 {
	return l["subscriber_count"].(int64)
}

func (l List) MemberCount() int64 {
	return l["member_count"].(int64)
}

// It's a less structured list of Lists!
type Lists []List

// It's a cursored list of Lists!

type CursoredLists map[string]interface{}

func (cl CursoredLists) NextCursorStr() string {
	return cl["next_cursor_str"].(string)
}

func (cl CursoredLists) PreviousCursorStr() string {
	return cl["previous_cursor_str"].(string)
}

func (cl CursoredLists) Lists() Lists {
	var a []interface{} = cl["lists"].([]interface{})
	b := make([]List, len(a))
	for i, v := range a {
		b[i] = v.(map[string]interface{})
	}
	return b
}

// Nested response structure for video uploads.
type VideoUpload map[string]interface{}

func (v VideoUpload) Type() string {
	return v["video_type"].(string)
}

// Response for media upload requests.
type MediaResponse map[string]interface{}

func (r MediaResponse) MediaId() int64 {
	return r["media_id"].(int64)
}

func (r MediaResponse) Size() int64 {
	return r["size"].(int64)
}

func (r MediaResponse) ExpiresAfterSecs() int32 {
	return r["expires_after_secs"].(int32)
}

func (r MediaResponse) Video() VideoUpload {
	return VideoUpload(r["video"].(map[string]interface{}))
}
<commit_msg>Add common interface between RateLimitError and APIResponse<commit_after>// Copyright 2011 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package twittergo

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

const (
	H_LIMIT              = "X-Rate-Limit-Limit"
	H_LIMIT_REMAIN       = "X-Rate-Limit-Remaining"
	H_LIMIT_RESET        = "X-Rate-Limit-Reset"
	H_MEDIA_LIMIT        = "X-MediaRateLimit-Limit"
	H_MEDIA_LIMIT_REMAIN = "X-MediaRateLimit-Remaining"
	H_MEDIA_LIMIT_RESET  = "X-MediaRateLimit-Reset"
)

const (
	STATUS_OK           = 200
	STATUS_CREATED      = 201
	STATUS_ACCEPTED     = 202
	STATUS_NO_CONTENT   = 204
	STATUS_INVALID      = 400
	STATUS_UNAUTHORIZED = 401
	STATUS_FORBIDDEN    = 403
	STATUS_NOTFOUND     = 404
	STATUS_LIMIT        = 429
	STATUS_GATEWAY      = 502
)

// Error returned if there was an issue parsing the response body.
type ResponseError struct {
	Body string
	Code int
}

func NewResponseError(code int, body string) ResponseError {
	return ResponseError{Code: code, Body: body}
}

func (e ResponseError) Error() string {
	return fmt.Sprintf(
		"Unable to handle response (status code %d): `%v`",
		e.Code,
		e.Body)
}

type Error map[string]interface{}

func (e Error) Code() int64 {
	return int64(e["code"].(float64))
}

func (e Error) Message() string {
	return e["message"].(string)
}

func (e Error) Error() string {
	msg := "Error %v: %v"
	return fmt.Sprintf(msg, e.Code(), e.Message())
}

type Errors map[string]interface{}

func (e Errors) Error() string {
	var (
		msg string = ""
		err Error
		ok  bool
	)
	if e["errors"] == nil {
		return msg
	}
	for _, val := range e["errors"].([]interface{}) {
		if err, ok = val.(map[string]interface{}); ok {
			msg += err.Error() + ". "
		}
	}
	return msg
}

func (e Errors) String() string {
	return e.Error()
}

func (e Errors) Errors() []Error {
	var errs = e["errors"].([]interface{})
	var out = make([]Error, len(errs))
	for i, val := range errs {
		out[i] = Error(val.(map[string]interface{}))
	}
	return out
}

// RateLimitResponse is implemented by both RateLimitError and APIResponse.
type RateLimitResponse interface {
	// HasRateLimit returns false if the ratelimiting information is
	// optional and missing.
	HasRateLimit() bool
	// RateLimit returns the requests per time period capacity of the
	// limit.
	RateLimit() uint32
	// RateLimitRemaining returns how many requests are still available
	// in the current time period.
	RateLimitRemaining() uint32
	// RateLimitReset returns when the rate limit will reset.
	RateLimitReset() time.Time
}

// RateLimitError is returned from SendRequest when a rate limit is encountered.
type RateLimitError struct {
	Limit     uint32
	Remaining uint32
	Reset     time.Time
}

func (e RateLimitError) Error() string {
	msg := "Rate limit: %v, Remaining: %v, Reset: %v"
	return fmt.Sprintf(msg, e.Limit, e.Remaining, e.Reset)
}

func (e RateLimitError) HasRateLimit() bool {
	return true
}

func (e RateLimitError) RateLimit() uint32 {
	return e.Limit
}

func (e RateLimitError) RateLimitRemaining() uint32 {
	return e.Remaining
}

func (e RateLimitError) RateLimitReset() time.Time {
	return e.Reset
}

// APIResponse provides methods for retrieving information from the HTTP
// headers in a Twitter API response.
type APIResponse http.Response

func (r APIResponse) HasRateLimit() bool {
	return r.Header.Get(H_LIMIT) != ""
}

func (r APIResponse) RateLimit() uint32 {
	h := r.Header.Get(H_LIMIT)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) RateLimitRemaining() uint32 {
	h := r.Header.Get(H_LIMIT_REMAIN)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) RateLimitReset() time.Time {
	h := r.Header.Get(H_LIMIT_RESET)
	i, _ := strconv.ParseUint(h, 10, 32)
	t := time.Unix(int64(i), 0)
	return t
}

func (r APIResponse) HasMediaRateLimit() bool {
	return r.Header.Get(H_MEDIA_LIMIT) != ""
}

func (r APIResponse) MediaRateLimit() uint32 {
	h := r.Header.Get(H_MEDIA_LIMIT)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) MediaRateLimitRemaining() uint32 {
	h := r.Header.Get(H_MEDIA_LIMIT_REMAIN)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) MediaRateLimitReset() time.Time {
	h := r.Header.Get(H_MEDIA_LIMIT_RESET)
	i, _ := strconv.ParseUint(h, 10, 32)
	t := time.Unix(int64(i), 0)
	return t
}

func (r APIResponse) readBody() (b []byte, err error) {
	var (
		header string
		reader io.Reader
	)
	defer r.Body.Close()
	header = strings.ToLower(r.Header.Get("Content-Encoding"))
	if header == "" || strings.Index(header, "gzip") == -1 {
		reader = r.Body
	} else {
		if reader, err = gzip.NewReader(r.Body); err != nil {
			return
		}
	}
	b, err = ioutil.ReadAll(reader)
	return
}

// ReadBody returns the body of the response as a string.
// Only one of ReadBody and Parse may be called on a given APIResponse.
func (r APIResponse) ReadBody() string {
	var (
		b   []byte
		err error
	)
	if b, err = r.readBody(); err != nil {
		return ""
	}
	return string(b)
}

// Parse unmarshals a JSON encoded HTTP response into the supplied interface,
// with handling for the various kinds of errors the Twitter API can return.
//
// The returned error may be of the type Errors, RateLimitError,
// ResponseError, or an error returned from io.Reader.Read().
func (r APIResponse) Parse(out interface{}) (err error) {
	var b []byte
	switch r.StatusCode {
	case STATUS_UNAUTHORIZED:
		fallthrough
	case STATUS_NOTFOUND:
		fallthrough
	case STATUS_GATEWAY:
		fallthrough
	case STATUS_FORBIDDEN:
		fallthrough
	case STATUS_INVALID:
		e := &Errors{}
		if b, err = r.readBody(); err != nil {
			return
		}
		if err = json.Unmarshal(b, e); err != nil {
			err = NewResponseError(r.StatusCode, string(b))
		} else {
			err = *e
		}
		return
	case STATUS_LIMIT:
		err = RateLimitError{
			Limit:     r.RateLimit(),
			Remaining: r.RateLimitRemaining(),
			Reset:     r.RateLimitReset(),
		}
		// consume the request body even if we don't need it
		r.readBody()
		return
	case STATUS_NO_CONTENT:
		return
	case STATUS_CREATED:
		fallthrough
	case STATUS_ACCEPTED:
		fallthrough
	case STATUS_OK:
		if b, err = r.readBody(); err != nil {
			return
		}
		err = json.Unmarshal(b, out)
		if err == io.EOF {
			err = nil
		}
	default:
		if b, err = r.readBody(); err != nil {
			return
		}
		err = NewResponseError(r.StatusCode, string(b))
	}
	return
}

// It's a user!
type User map[string]interface{}

func (u User) Id() uint64 {
	id, _ := strconv.ParseUint(u["id_str"].(string), 10, 64)
	return id
}

func (u User) IdStr() string {
	return u["id_str"].(string)
}

func (u User) Name() string {
	return u["name"].(string)
}

func (u User) ScreenName() string {
	return u["screen_name"].(string)
}

// It's a Tweet! (Adorably referred to by the API as a "status").
type Tweet map[string]interface{}

func (t Tweet) Id() (id uint64) {
	var (
		err error
		src = t["id_str"].(string)
	)
	if id, err = strconv.ParseUint(src, 10, 64); err != nil {
		panic(fmt.Sprintf("Could not parse ID: %v", err))
	}
	return
}

func (t Tweet) IdStr() string {
	return t["id_str"].(string)
}

func (t Tweet) Text() string {
	return t["text"].(string)
}

func (t Tweet) User() User {
	return User(t["user"].(map[string]interface{}))
}

func (t Tweet) Language() string {
	return t["lang"].(string)
}

func (t Tweet) CreatedAt() (out time.Time) {
	var (
		err error
		src = t["created_at"].(string)
	)
	if out, err = time.Parse(time.RubyDate, src); err != nil {
		panic(fmt.Sprintf("Could not parse time: %v", err))
	}
	return
}

// It's a less structured list of Tweets!
type Timeline []Tweet

// It's a structured list of Tweets!
type SearchResults map[string]interface{}

func (sr SearchResults) Statuses() []Tweet {
	var a []interface{} = sr["statuses"].([]interface{})
	b := make([]Tweet, len(a))
	for i, v := range a {
		b[i] = v.(map[string]interface{})
	}
	return b
}

func (sr SearchResults) SearchMetadata() map[string]interface{} {
	a := sr["search_metadata"].(map[string]interface{})
	return a
}

func (sr SearchResults) NextQuery() (val url.Values, err error) {
	var (
		sm   map[string]interface{}
		n    interface{}
		next string
		ok   bool
	)
	sm = sr.SearchMetadata()
	if n, ok = sm["next_results"]; !ok {
		err = fmt.Errorf("Could not get next_results from search")
		return
	}
	if next, ok = n.(string); !ok {
		err = fmt.Errorf("Could not convert next_results to str: %v", n)
		return
	}
	if next[0] == '?' {
		next = next[1:]
	}
	val, err = url.ParseQuery(next)
	return
}

// A List!
type List map[string]interface{}

func (l List) User() User {
	return User(l["user"].(map[string]interface{}))
}

func (l List) Id() (id uint64) {
	var (
		err error
		src = l["id_str"].(string)
	)
	if id, err = strconv.ParseUint(src, 10, 64); err != nil {
		panic(fmt.Sprintf("Could not parse ID: %v", err))
	}
	return
}

func (l List) IdStr() string {
	return l["id_str"].(string)
}

func (l List) Mode() string {
	return l["mode"].(string)
}

func (l List) Name() string {
	return l["name"].(string)
}

func (l List) Slug() string {
	return l["slug"].(string)
}

func (l List) SubscriberCount() int64 {
	return l["subscriber_count"].(int64)
}

func (l List) MemberCount() int64 {
	return l["member_count"].(int64)
}

// It's a less structured list of Lists!
type Lists []List

// It's a cursored list of Lists!

type CursoredLists map[string]interface{}

func (cl CursoredLists) NextCursorStr() string {
	return cl["next_cursor_str"].(string)
}

func (cl CursoredLists) PreviousCursorStr() string {
	return cl["previous_cursor_str"].(string)
}

func (cl CursoredLists) Lists() Lists {
	var a []interface{} = cl["lists"].([]interface{})
	b := make([]List, len(a))
	for i, v := range a {
		b[i] = v.(map[string]interface{})
	}
	return b
}

// Nested response structure for video uploads.
type VideoUpload map[string]interface{}

func (v VideoUpload) Type() string {
	return v["video_type"].(string)
}

// Response for media upload requests.
type MediaResponse map[string]interface{}

func (r MediaResponse) MediaId() int64 {
	return r["media_id"].(int64)
}

func (r MediaResponse) Size() int64 {
	return r["size"].(int64)
}

func (r MediaResponse) ExpiresAfterSecs() int32 {
	return r["expires_after_secs"].(int32)
}

func (r MediaResponse) Video() VideoUpload {
	return VideoUpload(r["video"].(map[string]interface{}))
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "container/list"
)

// Job description
type Job struct {
    Name string
    id int
}

func (j *Job) Print() {
    fmt.Printf("job: name=%s, id=%d\n", j.Name, j.id)
}

// JobQueue is the dockbroker queue of jobs
type JobQueue struct {
    l *list.List
    lastID int
}

// QueueElement is a more generic interface (than specific the Job structure)
// for the JobQueue.
type QueueElement interface {
    Print()
}

// Enqueue adds a job to the queue.
func (jq *JobQueue) Enqueue(qe QueueElement) {
    jq.l.PushBack(qe)
}

// Print an overview of the current queue.
func (jq *JobQueue) Print() {
    for j := jq.l.Front(); j != nil; j = j.Next() {
        j.Value.(QueueElement).Print()
    }
}

// NewJob creates a new Job and enqueues it.
func (jq *JobQueue) NewJob(name string) {
    jq.lastID++
    job := Job{name, jq.lastID}
    jq.Enqueue(&job)
}

// the job queue for this broker
var Queue = JobQueue{list.New(), 0}

<commit_msg>more linter comments<commit_after>package main

import (
    "fmt"
    "container/list"
)

// Job description
type Job struct {
    Name string
    id int
}

// Print Job information.
func (j *Job) Print() {
    fmt.Printf("job: name=%s, id=%d\n", j.Name, j.id)
}

// JobQueue is the dockbroker queue of jobs
type JobQueue struct {
    l *list.List
    lastID int
}

// QueueElement is a more generic interface (than specific the Job structure)
// for the JobQueue.
type QueueElement interface {
    Print()
}

// Enqueue adds a job to the queue.
func (jq *JobQueue) Enqueue(qe QueueElement) {
    jq.l.PushBack(qe)
}

// Print an overview of the current queue.
func (jq *JobQueue) Print() {
    for j := jq.l.Front(); j != nil; j = j.Next() {
        j.Value.(QueueElement).Print()
    }
}

// NewJob creates a new Job and enqueues it.
func (jq *JobQueue) NewJob(name string) {
    jq.lastID++
    job := Job{name, jq.lastID}
    jq.Enqueue(&job)
}

// the job queue for this broker
var Queue = JobQueue{list.New(), 0}

<|endoftext|>