<commit_before>package multiaddr

import (
	"bytes"
	"encoding/base32"
	"encoding/base64"
	"encoding/binary"
	"fmt"
	"net"
	"strconv"
	"strings"

	mh "github.com/multiformats/go-multihash"
)

type Transcoder interface {
	StringToBytes(string) ([]byte, error)
	BytesToString([]byte) (string, error)
	ValidateBytes([]byte) error
}

func NewTranscoderFromFunctions(
	s2b func(string) ([]byte, error),
	b2s func([]byte) (string, error),
	val func([]byte) error,
) Transcoder {
	return twrp{s2b, b2s, val}
}

type twrp struct {
	strtobyte func(string) ([]byte, error)
	bytetostr func([]byte) (string, error)
	validbyte func([]byte) error
}

func (t twrp) StringToBytes(s string) ([]byte, error) {
	return t.strtobyte(s)
}
func (t twrp) BytesToString(b []byte) (string, error) {
	return t.bytetostr(b)
}

func (t twrp) ValidateBytes(b []byte) error {
	if t.validbyte == nil {
		return nil
	}
	return t.validbyte(b)
}

var TranscoderIP4 = NewTranscoderFromFunctions(ip4StB, ip4BtS, nil)
var TranscoderIP6 = NewTranscoderFromFunctions(ip6StB, ip6BtS, nil)
var TranscoderIP6Zone = NewTranscoderFromFunctions(ip6zoneStB, ip6zoneBtS, ip6zoneVal)

func ip4StB(s string) ([]byte, error) {
	i := net.ParseIP(s).To4()
	if i == nil {
		return nil, fmt.Errorf("failed to parse ip4 addr: %s", s)
	}
	return i, nil
}

func ip6zoneStB(s string) ([]byte, error) {
	if len(s) == 0 {
		return nil, fmt.Errorf("empty ip6zone")
	}
	return []byte(s), nil
}

func ip6zoneBtS(b []byte) (string, error) {
	if len(b) == 0 {
		return "", fmt.Errorf("invalid length (should be > 0)")
	}
	return string(b), nil
}

func ip6zoneVal(b []byte) error {
	if len(b) == 0 {
		return fmt.Errorf("invalid length (should be > 0)")
	}
	// Not supported as this would break multiaddrs.
	if bytes.IndexByte(b, '/') >= 0 {
		return fmt.Errorf("IPv6 zone ID contains '/': %s", string(b))
	}
	return nil
}

func ip6StB(s string) ([]byte, error) {
	i := net.ParseIP(s).To16()
	if i == nil {
		return nil, fmt.Errorf("failed to parse ip6 addr: %s", s)
	}
	return i, nil
}

func ip6BtS(b []byte) (string, error) {
	ip := net.IP(b)
	if ip4 := ip.To4(); ip4 != nil {
		// Go fails to prepend the `::ffff:` part.
		return "::ffff:" + ip4.String(), nil
	}
	return ip.String(), nil
}

func ip4BtS(b []byte) (string, error) {
	return net.IP(b).String(), nil
}

var TranscoderPort = NewTranscoderFromFunctions(portStB, portBtS, nil)

func portStB(s string) ([]byte, error) {
	i, err := strconv.Atoi(s)
	if err != nil {
		return nil, fmt.Errorf("failed to parse port addr: %s", err)
	}
	if i >= 65536 {
		return nil, fmt.Errorf("failed to parse port addr: %s", "greater than 65536")
	}
	b := make([]byte, 2)
	binary.BigEndian.PutUint16(b, uint16(i))
	return b, nil
}

func portBtS(b []byte) (string, error) {
	i := binary.BigEndian.Uint16(b)
	return strconv.Itoa(int(i)), nil
}

var TranscoderOnion = NewTranscoderFromFunctions(onionStB, onionBtS, nil)

func onionStB(s string) ([]byte, error) {
	addr := strings.Split(s, ":")
	if len(addr) != 2 {
		return nil, fmt.Errorf("failed to parse onion addr: %s does not contain a port number.", s)
	}

	// onion address without the ".onion" substring
	if len(addr[0]) != 16 {
		return nil, fmt.Errorf("failed to parse onion addr: %s not a Tor onion address.", s)
	}
	onionHostBytes, err := base32.StdEncoding.DecodeString(strings.ToUpper(addr[0]))
	if err != nil {
		return nil, fmt.Errorf("failed to decode base32 onion addr: %s %s", s, err)
	}

	// onion port number
	i, err := strconv.Atoi(addr[1])
	if err != nil {
		return nil, fmt.Errorf("failed to parse onion addr: %s", err)
	}
	if i >= 65536 {
		return nil, fmt.Errorf("failed to parse onion addr: %s", "port greater than 65536")
	}
	if i < 1 {
		return nil, fmt.Errorf("failed to parse onion addr: %s", "port less than 1")
	}

	onionPortBytes := make([]byte, 2)
	binary.BigEndian.PutUint16(onionPortBytes, uint16(i))
	bytes := []byte{}
	bytes = append(bytes, onionHostBytes...)
	bytes = append(bytes, onionPortBytes...)
	return bytes, nil
}

func onionBtS(b []byte) (string, error) {
	addr := strings.ToLower(base32.StdEncoding.EncodeToString(b[0:10]))
	port := binary.BigEndian.Uint16(b[10:12])
	return addr + ":" + strconv.Itoa(int(port)), nil
}

var TranscoderOnion3 = NewTranscoderFromFunctions(onion3StB, onion3BtS, nil)

func onion3StB(s string) ([]byte, error) {
	addr := strings.Split(s, ":")
	if len(addr) != 2 {
		return nil, fmt.Errorf("failed to parse onion addr: %s does not contain a port number.", s)
	}

	// onion address without the ".onion" substring
	if len(addr[0]) != 56 {
		return nil, fmt.Errorf("failed to parse onion addr: %s not a Tor onionv3 address. len == %d", s, len(addr[0]))
	}
	onionHostBytes, err := base32.StdEncoding.DecodeString(strings.ToUpper(addr[0]))
	if err != nil {
		return nil, fmt.Errorf("failed to decode base32 onion addr: %s %s", s, err)
	}

	// onion port number
	i, err := strconv.Atoi(addr[1])
	if err != nil {
		return nil, fmt.Errorf("failed to parse onion addr: %s", err)
	}
	if i >= 65536 {
		return nil, fmt.Errorf("failed to parse onion addr: %s", "port greater than 65536")
	}
	if i < 1 {
		return nil, fmt.Errorf("failed to parse onion addr: %s", "port less than 1")
	}

	onionPortBytes := make([]byte, 2)
	binary.BigEndian.PutUint16(onionPortBytes, uint16(i))
	bytes := []byte{}
	bytes = append(bytes, onionHostBytes[0:35]...)
	bytes = append(bytes, onionPortBytes...)
	return bytes, nil
}

func onion3BtS(b []byte) (string, error) {
	addr := strings.ToLower(base32.StdEncoding.EncodeToString(b[0:35]))
	port := binary.BigEndian.Uint16(b[35:37])
	str := addr + ":" + strconv.Itoa(int(port))
	return str, nil
}

var TranscoderGarlic64 = NewTranscoderFromFunctions(garlic64StB, garlic64BtS, garlic64Validate)

// i2p uses an alternate character set for base64 addresses. This returns an appropriate encoder.
var garlicBase64Encoding = base64.NewEncoding("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-~")

func garlic64StB(s string) ([]byte, error) {
	// i2p base64 address
	if len(s) < 516 || len(s) > 616 {
		return nil, fmt.Errorf("failed to parse garlic addr: %s not an i2p base64 address. len: %d\n", s, len(s))
	}
	garlicHostBytes, err := garlicBase64Encoding.DecodeString(s)
	if err != nil {
		return nil, fmt.Errorf("failed to decode base64 i2p addr: %s %s", s, err)
	}

	return garlicHostBytes, nil
}

func garlic64BtS(b []byte) (string, error) {
	if len(b) < 386 {
		return "", fmt.Errorf("failed to validate garlic addr: %s not an i2p base64 address. len: %d\n", b, len(b))
	}
	addr := garlicBase64Encoding.EncodeToString(b)
	return addr, nil
}

func garlic64Validate(b []byte) error {
	if len(b) < 386 {
		return fmt.Errorf("failed to validate garlic addr: %s not an i2p base64 address. len: %d\n", b, len(b))
	}
	return nil
}

var TranscoderGarlic32 = NewTranscoderFromFunctions(garlic32StB, garlic32BtS, garlic32Validate)

var garlicBase32Encoding = base32.NewEncoding("abcdefghijklmnopqrstuvwxyz234567")

func garlic32StB(s string) ([]byte, error) {
	//s = strings.Replace(s, ".b32.i2p", "", -1)
	// garlic address without the ".b32.i2p" substring

	// an i2p base32 address with a length of greater than 55 characters is
	// using an Encrypted Leaseset v2.
	if len(s) < 55 {
		if len(s) != 52 {
			// all other base32 addresses will always be exactly 52 characters
			return nil, fmt.Errorf("failed to parse garlic addr: %s not a i2p base32 address. len: %d", s, len(s))
		}
	}
	//compute the length to pad the address to, usually 56 or 64
	x := int((len(s)/8)+1) * 8
	for len(s) < x {
		s += "="
	}
	garlicHostBytes, err := garlicBase32Encoding.DecodeString(s[0:56])
	if err != nil {
		return nil, fmt.Errorf("failed to decode base32 garlic addr: %s, err: %v len: %v", s, err, len(s))
	}
	return garlicHostBytes, nil
}

func garlic32BtS(b []byte) (string, error) {
	if err := garlic32Validate(b); err != nil {
		return "", err
	}
	return strings.TrimRight(garlicBase32Encoding.EncodeToString(b), "="), nil
}

func garlic32Validate(b []byte) error {
	// an i2p base64 for an Encrypted Leaseset v2 will be at least 35 bytes
	// long
	if len(b) < 35 {
		// other than that, they will be exactly 32 bytes
		if len(b) != 32 {
			return fmt.Errorf("failed to validate garlic addr: %s not an i2p base32 address. len: %d\n", b, len(b))
		}
	}
	return nil
}

var TranscoderP2P = NewTranscoderFromFunctions(p2pStB, p2pBtS, p2pVal)

func p2pStB(s string) ([]byte, error) {
	// the address is a varint prefixed multihash string representation
	m, err := mh.FromB58String(s)
	if err != nil {
		return nil, fmt.Errorf("failed to parse p2p addr: %s %s", s, err)
	}
	return m, nil
}

func p2pVal(b []byte) error {
	_, err := mh.Cast(b)
	return err
}

func p2pBtS(b []byte) (string, error) {
	m, err := mh.Cast(b)
	if err != nil {
		return "", err
	}
	return m.B58String(), nil
}

var TranscoderUnix = NewTranscoderFromFunctions(unixStB, unixBtS, nil)

func unixStB(s string) ([]byte, error) {
	return []byte(s), nil
}

func unixBtS(b []byte) (string, error) {
	return string(b), nil
}
<commit_msg>fix the padding<commit_after>package multiaddr

import (
	"bytes"
	"encoding/base32"
	"encoding/base64"
	"encoding/binary"
	"fmt"
	"net"
	"strconv"
	"strings"

	mh "github.com/multiformats/go-multihash"
)

type Transcoder interface {
	StringToBytes(string) ([]byte, error)
	BytesToString([]byte) (string, error)
	ValidateBytes([]byte) error
}

func NewTranscoderFromFunctions(
	s2b func(string) ([]byte, error),
	b2s func([]byte) (string, error),
	val func([]byte) error,
) Transcoder {
	return twrp{s2b, b2s, val}
}

type twrp struct {
	strtobyte func(string) ([]byte, error)
	bytetostr func([]byte) (string, error)
	validbyte func([]byte) error
}

func (t twrp) StringToBytes(s string) ([]byte, error) {
	return t.strtobyte(s)
}
func (t twrp) BytesToString(b []byte) (string, error) {
	return t.bytetostr(b)
}

func (t twrp) ValidateBytes(b []byte) error {
	if t.validbyte == nil {
		return nil
	}
	return t.validbyte(b)
}

var TranscoderIP4 = NewTranscoderFromFunctions(ip4StB, ip4BtS, nil)
var TranscoderIP6 = NewTranscoderFromFunctions(ip6StB, ip6BtS, nil)
var TranscoderIP6Zone = NewTranscoderFromFunctions(ip6zoneStB, ip6zoneBtS, ip6zoneVal)

func ip4StB(s string) ([]byte, error) {
	i := net.ParseIP(s).To4()
	if i == nil {
		return nil, fmt.Errorf("failed to parse ip4 addr: %s", s)
	}
	return i, nil
}

func ip6zoneStB(s string) ([]byte, error) {
	if len(s) == 0 {
		return nil, fmt.Errorf("empty ip6zone")
	}
	return []byte(s), nil
}

func ip6zoneBtS(b []byte) (string, error) {
	if len(b) == 0 {
		return "", fmt.Errorf("invalid length (should be > 0)")
	}
	return string(b), nil
}

func ip6zoneVal(b []byte) error {
	if len(b) == 0 {
		return fmt.Errorf("invalid length (should be > 0)")
	}
	// Not supported as this would break multiaddrs.
	if bytes.IndexByte(b, '/') >= 0 {
		return fmt.Errorf("IPv6 zone ID contains '/': %s", string(b))
	}
	return nil
}

func ip6StB(s string) ([]byte, error) {
	i := net.ParseIP(s).To16()
	if i == nil {
		return nil, fmt.Errorf("failed to parse ip6 addr: %s", s)
	}
	return i, nil
}

func ip6BtS(b []byte) (string, error) {
	ip := net.IP(b)
	if ip4 := ip.To4(); ip4 != nil {
		// Go fails to prepend the `::ffff:` part.
		return "::ffff:" + ip4.String(), nil
	}
	return ip.String(), nil
}

func ip4BtS(b []byte) (string, error) {
	return net.IP(b).String(), nil
}

var TranscoderPort = NewTranscoderFromFunctions(portStB, portBtS, nil)

func portStB(s string) ([]byte, error) {
	i, err := strconv.Atoi(s)
	if err != nil {
		return nil, fmt.Errorf("failed to parse port addr: %s", err)
	}
	if i >= 65536 {
		return nil, fmt.Errorf("failed to parse port addr: %s", "greater than 65536")
	}
	b := make([]byte, 2)
	binary.BigEndian.PutUint16(b, uint16(i))
	return b, nil
}

func portBtS(b []byte) (string, error) {
	i := binary.BigEndian.Uint16(b)
	return strconv.Itoa(int(i)), nil
}

var TranscoderOnion = NewTranscoderFromFunctions(onionStB, onionBtS, nil)

func onionStB(s string) ([]byte, error) {
	addr := strings.Split(s, ":")
	if len(addr) != 2 {
		return nil, fmt.Errorf("failed to parse onion addr: %s does not contain a port number.", s)
	}

	// onion address without the ".onion" substring
	if len(addr[0]) != 16 {
		return nil, fmt.Errorf("failed to parse onion addr: %s not a Tor onion address.", s)
	}
	onionHostBytes, err := base32.StdEncoding.DecodeString(strings.ToUpper(addr[0]))
	if err != nil {
		return nil, fmt.Errorf("failed to decode base32 onion addr: %s %s", s, err)
	}

	// onion port number
	i, err := strconv.Atoi(addr[1])
	if err != nil {
		return nil, fmt.Errorf("failed to parse onion addr: %s", err)
	}
	if i >= 65536 {
		return nil, fmt.Errorf("failed to parse onion addr: %s", "port greater than 65536")
	}
	if i < 1 {
		return nil, fmt.Errorf("failed to parse onion addr: %s", "port less than 1")
	}

	onionPortBytes := make([]byte, 2)
	binary.BigEndian.PutUint16(onionPortBytes, uint16(i))
	bytes := []byte{}
	bytes = append(bytes, onionHostBytes...)
	bytes = append(bytes, onionPortBytes...)
	return bytes, nil
}

func onionBtS(b []byte) (string, error) {
	addr := strings.ToLower(base32.StdEncoding.EncodeToString(b[0:10]))
	port := binary.BigEndian.Uint16(b[10:12])
	return addr + ":" + strconv.Itoa(int(port)), nil
}

var TranscoderOnion3 = NewTranscoderFromFunctions(onion3StB, onion3BtS, nil)

func onion3StB(s string) ([]byte, error) {
	addr := strings.Split(s, ":")
	if len(addr) != 2 {
		return nil, fmt.Errorf("failed to parse onion addr: %s does not contain a port number.", s)
	}

	// onion address without the ".onion" substring
	if len(addr[0]) != 56 {
		return nil, fmt.Errorf("failed to parse onion addr: %s not a Tor onionv3 address. len == %d", s, len(addr[0]))
	}
	onionHostBytes, err := base32.StdEncoding.DecodeString(strings.ToUpper(addr[0]))
	if err != nil {
		return nil, fmt.Errorf("failed to decode base32 onion addr: %s %s", s, err)
	}

	// onion port number
	i, err := strconv.Atoi(addr[1])
	if err != nil {
		return nil, fmt.Errorf("failed to parse onion addr: %s", err)
	}
	if i >= 65536 {
		return nil, fmt.Errorf("failed to parse onion addr: %s", "port greater than 65536")
	}
	if i < 1 {
		return nil, fmt.Errorf("failed to parse onion addr: %s", "port less than 1")
	}

	onionPortBytes := make([]byte, 2)
	binary.BigEndian.PutUint16(onionPortBytes, uint16(i))
	bytes := []byte{}
	bytes = append(bytes, onionHostBytes[0:35]...)
	bytes = append(bytes, onionPortBytes...)
	return bytes, nil
}

func onion3BtS(b []byte) (string, error) {
	addr := strings.ToLower(base32.StdEncoding.EncodeToString(b[0:35]))
	port := binary.BigEndian.Uint16(b[35:37])
	str := addr + ":" + strconv.Itoa(int(port))
	return str, nil
}

var TranscoderGarlic64 = NewTranscoderFromFunctions(garlic64StB, garlic64BtS, garlic64Validate)

// i2p uses an alternate character set for base64 addresses. This returns an appropriate encoder.
var garlicBase64Encoding = base64.NewEncoding("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-~")

func garlic64StB(s string) ([]byte, error) {
	// i2p base64 address
	if len(s) < 516 || len(s) > 616 {
		return nil, fmt.Errorf("failed to parse garlic addr: %s not an i2p base64 address. len: %d\n", s, len(s))
	}
	garlicHostBytes, err := garlicBase64Encoding.DecodeString(s)
	if err != nil {
		return nil, fmt.Errorf("failed to decode base64 i2p addr: %s %s", s, err)
	}

	return garlicHostBytes, nil
}

func garlic64BtS(b []byte) (string, error) {
	if len(b) < 386 {
		return "", fmt.Errorf("failed to validate garlic addr: %s not an i2p base64 address. len: %d\n", b, len(b))
	}
	addr := garlicBase64Encoding.EncodeToString(b)
	return addr, nil
}

func garlic64Validate(b []byte) error {
	if len(b) < 386 {
		return fmt.Errorf("failed to validate garlic addr: %s not an i2p base64 address. len: %d\n", b, len(b))
	}
	return nil
}

var TranscoderGarlic32 = NewTranscoderFromFunctions(garlic32StB, garlic32BtS, garlic32Validate)

var garlicBase32Encoding = base32.NewEncoding("abcdefghijklmnopqrstuvwxyz234567")

func garlic32StB(s string) ([]byte, error) {
	//s = strings.Replace(s, ".b32.i2p", "", -1)
	// garlic address without the ".b32.i2p" substring

	// an i2p base32 address with a length of greater than 55 characters is
	// using an Encrypted Leaseset v2.
	if len(s) < 55 {
		if len(s) != 52 {
			// all other base32 addresses will always be exactly 52 characters
			return nil, fmt.Errorf("failed to parse garlic addr: %s not a i2p base32 address. len: %d", s, len(s))
		}
	}
	//compute the length to pad the address to, usually 56 or 64
	padout := func(s string) string {
		if len(s)%8 == 0 {
			return s
		}
		x := int((len(s)/8)+1) * 8
		for len(s) < x {
			s += "="
		}
		return s
	}

	garlicHostBytes, err := garlicBase32Encoding.DecodeString(padout(s))
	if err != nil {
		return nil, fmt.Errorf("failed to decode base32 garlic addr: %s, err: %v len: %v", s, err, len(s))
	}
	return garlicHostBytes, nil
}

func garlic32BtS(b []byte) (string, error) {
	if err := garlic32Validate(b); err != nil {
		return "", err
	}
	return strings.TrimRight(garlicBase32Encoding.EncodeToString(b), "="), nil
}

func garlic32Validate(b []byte) error {
	// an i2p base64 for an Encrypted Leaseset v2 will be at least 35 bytes
	// long
	if len(b) < 35 {
		// other than that, they will be exactly 32 bytes
		if len(b) != 32 {
			return fmt.Errorf("failed to validate garlic addr: %s not an i2p base32 address. len: %d\n", b, len(b))
		}
	}
	return nil
}

var TranscoderP2P = NewTranscoderFromFunctions(p2pStB, p2pBtS, p2pVal)

func p2pStB(s string) ([]byte, error) {
	// the address is a varint prefixed multihash string representation
	m, err := mh.FromB58String(s)
	if err != nil {
		return nil, fmt.Errorf("failed to parse p2p addr: %s %s", s, err)
	}
	return m, nil
}

func p2pVal(b []byte) error {
	_, err := mh.Cast(b)
	return err
}

func p2pBtS(b []byte) (string, error) {
	m, err := mh.Cast(b)
	if err != nil {
		return "", err
	}
	return m.B58String(), nil
}

var TranscoderUnix = NewTranscoderFromFunctions(unixStB, unixBtS, nil)

func unixStB(s string) ([]byte, error) {
	return []byte(s), nil
}

func unixBtS(b []byte) (string, error) {
	return string(b), nil
}
<|endoftext|>


<commit_before>package workflows

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/stelligent/mu/common"
)

var ecsImagePattern = "amzn-ami-*-amazon-ecs-optimized"
var ec2ImagePattern = "amzn-ami-hvm-*-x86_64-gp2"

// NewEnvironmentUpserter create a new workflow for upserting an environment
func NewEnvironmentUpserter(ctx *common.Context, environmentName string) Executor {

	workflow := new(environmentWorkflow)
	ecsStackParams := make(map[string]string)
	elbStackParams := make(map[string]string)
	workflow.codeRevision = ctx.Config.Repo.Revision
	workflow.repoName = ctx.Config.Repo.Slug

	return newPipelineExecutor(
		workflow.environmentFinder(&ctx.Config, environmentName),
		workflow.environmentNormalizer(),
		workflow.environmentRolesetUpserter(ctx.RolesetManager, ctx.RolesetManager, ecsStackParams),
		workflow.environmentVpcUpserter(ctx.Config.Namespace, ecsStackParams, elbStackParams, ctx.StackManager, ctx.StackManager, ctx.StackManager, ctx.StackManager),
		workflow.environmentElbUpserter(ctx.Config.Namespace, ecsStackParams, elbStackParams, ctx.StackManager, ctx.StackManager, ctx.StackManager),
		workflow.environmentUpserter(ctx.Config.Namespace, ecsStackParams, ctx.StackManager, ctx.StackManager, ctx.StackManager),
	)
}

// Find an environment in config, by name and set the reference
func (workflow *environmentWorkflow) environmentFinder(config *common.Config, environmentName string) Executor {

	return func() error {
		for _, e := range config.Environments {
			if strings.EqualFold(e.Name, environmentName) {
				workflow.environment = &e
				return nil
			}
		}
		return common.Warningf("Unable to find environment named '%s' in configuration", environmentName)
	}
}

func (workflow *environmentWorkflow) environmentVpcUpserter(namespace string,
	ecsStackParams map[string]string, elbStackParams map[string]string, imageFinder common.ImageFinder,
	stackUpserter common.StackUpserter, stackWaiter common.StackWaiter, azCounter common.AZCounter) Executor {
	return func() error {
		environment := workflow.environment
		vpcStackParams := make(map[string]string)
		var err error

		var vpcStackName string
		var vpcTemplateName string
		if environment.VpcTarget.Environment != "" {
			targetNamespace := environment.VpcTarget.Namespace
			if targetNamespace == "" {
				targetNamespace = namespace
			}
			log.Debugf("VpcTarget exists for different environment; targeting that VPC")
			vpcStackName = common.CreateStackName(targetNamespace, common.StackTypeVpc, environment.VpcTarget.Environment)
		} else if environment.VpcTarget.VpcID != "" {
			log.Debugf("VpcTarget exists, so we will upsert the VPC stack that references the VPC attributes")
			vpcStackName = common.CreateStackName(namespace, common.StackTypeTarget, environment.Name)
			vpcTemplateName = "vpc-target.yml"

			// target VPC referenced from config
			vpcStackParams["VpcId"] = environment.VpcTarget.VpcID
			vpcStackParams["ElbSubnetIds"] = strings.Join(environment.VpcTarget.ElbSubnetIds, ",")
			vpcStackParams["InstanceSubnetIds"] = strings.Join(environment.VpcTarget.InstanceSubnetIds, ",")
		} else {
			log.Debugf("No VpcTarget, so we will upsert the VPC stack that manages the VPC")
			vpcStackName = common.CreateStackName(namespace, common.StackTypeVpc, environment.Name)
			vpcTemplateName = "vpc.yml"

			if environment.Cluster.InstanceTenancy != "" {
				vpcStackParams["InstanceTenancy"] = string(environment.Cluster.InstanceTenancy)
			}
			if environment.Cluster.SSHAllow != "" {
				vpcStackParams["SshAllow"] = environment.Cluster.SSHAllow
			} else {
				vpcStackParams["SshAllow"] = "0.0.0.0/0"
			}
			if environment.Cluster.KeyName != "" {
				vpcStackParams["BastionKeyName"] = environment.Cluster.KeyName
				vpcStackParams["BastionImageId"], err = imageFinder.FindLatestImageID(ec2ImagePattern)
				if err != nil {
					return err
				}
			}

			vpcStackParams["ElbInternal"] = strconv.FormatBool(environment.Loadbalancer.Internal)
		}

		azCount, err := azCounter.CountAZs()
		if err != nil {
			return err
		}
		if azCount < 2 {
			return fmt.Errorf("Only found %v availability zones...need at least 2", azCount)
		}
		vpcStackParams["AZCount"] = strconv.Itoa(azCount)

		if vpcTemplateName != "" {
			log.Noticef("Upserting VPC environment '%s' ...", environment.Name)

			tags := createTagMap(&EnvironmentTags{
				Environment: environment.Name,
				Type:        string(common.StackTypeVpc),
				Provider:    string(environment.Provider),
				Revision:    workflow.codeRevision,
				Repo:        workflow.repoName,
			})

			err = stackUpserter.UpsertStack(vpcStackName, vpcTemplateName, environment, vpcStackParams, tags, workflow.cloudFormationRoleArn)
			if err != nil {
				return err
			}

			log.Debugf("Waiting for stack '%s' to complete", vpcStackName)
			stack := stackWaiter.AwaitFinalStatus(vpcStackName)

			if stack == nil {
				return fmt.Errorf("Unable to create stack %s", vpcStackName)
			}
			if strings.HasSuffix(stack.Status, "ROLLBACK_COMPLETE") || !strings.HasSuffix(stack.Status, "_COMPLETE") {
				return fmt.Errorf("Ended in failed status %s %s", stack.Status, stack.StatusReason)
			}
		}

		ecsStackParams["VpcId"] = fmt.Sprintf("%s-VpcId", vpcStackName)
		ecsStackParams["InstanceSubnetIds"] = fmt.Sprintf("%s-InstanceSubnetIds", vpcStackName)

		elbStackParams["VpcId"] = fmt.Sprintf("%s-VpcId", vpcStackName)
		elbStackParams["ElbSubnetIds"] = fmt.Sprintf("%s-ElbSubnetIds", vpcStackName)

		return nil
	}
}

func (workflow *environmentWorkflow) environmentRolesetUpserter(rolesetUpserter common.RolesetUpserter, rolesetGetter common.RolesetGetter, ecsStackParams map[string]string) Executor {
	return func() error {
		err := rolesetUpserter.UpsertCommonRoleset()
		if err != nil {
			return err
		}

		commonRoleset, err := rolesetGetter.GetCommonRoleset()
		if err != nil {
			return err
		}

		workflow.cloudFormationRoleArn = commonRoleset["CloudFormationRoleArn"]

		err = rolesetUpserter.UpsertEnvironmentRoleset(workflow.environment.Name)
		if err != nil {
			return err
		}

		environmentRoleset, err := rolesetGetter.GetEnvironmentRoleset(workflow.environment.Name)
		if err != nil {
			return err
		}

		ecsStackParams["EC2InstanceProfileArn"] = environmentRoleset["EC2InstanceProfileArn"]

		return nil
	}
}

func (workflow *environmentWorkflow) environmentElbUpserter(namespace string, ecsStackParams map[string]string, elbStackParams map[string]string, imageFinder common.ImageFinder, stackUpserter common.StackUpserter, stackWaiter common.StackWaiter) Executor {
	return func() error {
		environment := workflow.environment
		envStackName := common.CreateStackName(namespace, common.StackTypeLoadBalancer, environment.Name)

		log.Noticef("Upserting ELB environment '%s' ...", environment.Name)

		stackParams := elbStackParams

		if environment.Loadbalancer.Certificate != "" {
			stackParams["ElbCert"] = environment.Loadbalancer.Certificate
		}

		if environment.Loadbalancer.HostedZone != "" {
			stackParams["ElbDomainName"] = environment.Loadbalancer.HostedZone

			if environment.Loadbalancer.Name == "" {
				// default to env name
				stackParams["ElbHostName"] = environment.Name
			} else {
				stackParams["ElbHostName"] = environment.Loadbalancer.Name
			}
		}

		if environment.Discovery.Name == "" {
			stackParams["ServiceDiscoveryName"] = fmt.Sprintf("%s.%s.local", environment.Name, namespace)
		} else {
			stackParams["ServiceDiscoveryName"] = environment.Discovery.Name
		}

		stackParams["ElbInternal"] = strconv.FormatBool(environment.Loadbalancer.Internal)

		tags := createTagMap(&EnvironmentTags{
			Environment: environment.Name,
			Type:        string(common.StackTypeLoadBalancer),
			Provider:    string(environment.Provider),
			Revision:    workflow.codeRevision,
			Repo:        workflow.repoName,
		})

		err := stackUpserter.UpsertStack(envStackName, "elb.yml", environment, stackParams, tags, workflow.cloudFormationRoleArn)
		if err != nil {
			return err
		}
		log.Debugf("Waiting for stack '%s' to complete", envStackName)
		stack := stackWaiter.AwaitFinalStatus(envStackName)

		if stack == nil {
			return fmt.Errorf("Unable to create stack %s", envStackName)
		}
		if strings.HasSuffix(stack.Status, "ROLLBACK_COMPLETE") || !strings.HasSuffix(stack.Status, "_COMPLETE") {
			return fmt.Errorf("Ended in failed status %s %s", stack.Status, stack.StatusReason)
		}

		ecsStackParams["ElbSecurityGroup"] = stack.Outputs["ElbInstanceSecurityGroup"]

		return nil
	}
}

func (workflow *environmentWorkflow) environmentUpserter(namespace string, ecsStackParams map[string]string,
	imageFinder common.ImageFinder, stackUpserter common.StackUpserter,
	stackWaiter common.StackWaiter) Executor {
	return func() error {
		log.Debugf("Using provider '%s' for environment", workflow.environment.Provider)

		environment := workflow.environment
		envStackName := common.CreateStackName(namespace, common.StackTypeEnv, environment.Name)

		stackParams := ecsStackParams

		var templateName string
		var imagePattern string
		envMapping := map[common.EnvProvider]map[string]string{
			common.EnvProviderEcs: map[string]string{
				"templateName": "env-ecs.yml",
				"imagePattern": ecsImagePattern,
				"launchType":   "EC2"},
			common.EnvProviderEcsFargate: map[string]string{
				"templateName": "env-ecs.yml",
				"imagePattern": ecsImagePattern,
				"launchType":   "FARGATE"},
			common.EnvProviderEc2: map[string]string{
				"templateName": "env-ec2.yml",
				"imagePattern": ec2ImagePattern,
				"launchType":   ""}}
		templateName = envMapping[environment.Provider]["templateName"]
		imagePattern = envMapping[environment.Provider]["imagePattern"]
		stackParams["LaunchType"] = envMapping[environment.Provider]["launchType"]

		log.Noticef("Upserting environment '%s' ...", environment.Name)

		if environment.Cluster.SSHAllow != "" {
			stackParams["SshAllow"] = environment.Cluster.SSHAllow
		} else {
			stackParams["SshAllow"] = "0.0.0.0/0"
		}
		if environment.Cluster.InstanceType != "" {
			stackParams["InstanceType"] = environment.Cluster.InstanceType
		}
		if environment.Cluster.ExtraUserData != "" {
			stackParams["ExtraUserData"] = environment.Cluster.ExtraUserData
		}
		if environment.Cluster.ImageID != "" {
			stackParams["ImageId"] = environment.Cluster.ImageID
		} else {
			var err error
			stackParams["ImageId"], err = imageFinder.FindLatestImageID(imagePattern)
			if err != nil {
				return err
			}

		}
		if environment.Cluster.ImageOsType != "" {
			stackParams["ImageOsType"] = environment.Cluster.ImageOsType
		}
		if environment.Cluster.DesiredCapacity != 0 {
			stackParams["DesiredCapacity"] = strconv.Itoa(environment.Cluster.DesiredCapacity)
		}
		if environment.Cluster.MinSize != 0 {
			stackParams["MinSize"] = strconv.Itoa(environment.Cluster.MinSize)
		}
		if environment.Cluster.MaxSize != 0 {
			stackParams["MaxSize"] = strconv.Itoa(environment.Cluster.MaxSize)
		}
		if environment.Cluster.KeyName != "" {
			stackParams["KeyName"] = environment.Cluster.KeyName
		}
		if environment.Cluster.TargetCPUReservation != 0 {
			stackParams["TargetCPUReservation"] = strconv.Itoa(environment.Cluster.TargetCPUReservation)
		}
		if environment.Cluster.TargetMemoryReservation != 0 {
			stackParams["TargetMemoryReservation"] = strconv.Itoa(environment.Cluster.TargetMemoryReservation)
		}
		if environment.Cluster.HTTPProxy != "" {
			stackParams["HttpProxy"] = environment.Cluster.HTTPProxy
		}

		tags := createTagMap(&EnvironmentTags{
			Environment: environment.Name,
			Type:        string(common.StackTypeEnv),
			Provider:    string(environment.Provider),
			Revision:    workflow.codeRevision,
			Repo:        workflow.repoName,
		})

		err := stackUpserter.UpsertStack(envStackName, templateName, environment, stackParams, tags, workflow.cloudFormationRoleArn)
		if err != nil {
			return err
		}
		log.Debugf("Waiting for stack '%s' to complete", envStackName)
		stack := stackWaiter.AwaitFinalStatus(envStackName)

		if stack == nil {
			return fmt.Errorf("Unable to create stack %s", envStackName)
		}
		if strings.HasSuffix(stack.Status, "ROLLBACK_COMPLETE") || !strings.HasSuffix(stack.Status, "_COMPLETE") {
			return fmt.Errorf("Ended in failed status %s %s", stack.Status, stack.StatusReason)
		}

		return nil
	}
}
<commit_msg>replace conditionals with helper fn<commit_after>package workflows

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/stelligent/mu/common"
)

var ecsImagePattern = "amzn-ami-*-amazon-ecs-optimized"
var ec2ImagePattern = "amzn-ami-hvm-*-x86_64-gp2"

// NewEnvironmentUpserter create a new workflow for upserting an environment
func NewEnvironmentUpserter(ctx *common.Context, environmentName string) Executor {

	workflow := new(environmentWorkflow)
	ecsStackParams := make(map[string]string)
	elbStackParams := make(map[string]string)
	workflow.codeRevision = ctx.Config.Repo.Revision
	workflow.repoName = ctx.Config.Repo.Slug

	return newPipelineExecutor(
		workflow.environmentFinder(&ctx.Config, environmentName),
		workflow.environmentNormalizer(),
		workflow.environmentRolesetUpserter(ctx.RolesetManager, ctx.RolesetManager, ecsStackParams),
		workflow.environmentVpcUpserter(ctx.Config.Namespace, ecsStackParams, elbStackParams, ctx.StackManager, ctx.StackManager, ctx.StackManager, ctx.StackManager),
		workflow.environmentElbUpserter(ctx.Config.Namespace, ecsStackParams, elbStackParams, ctx.StackManager, ctx.StackManager, ctx.StackManager),
		workflow.environmentUpserter(ctx.Config.Namespace, ecsStackParams, ctx.StackManager, ctx.StackManager, ctx.StackManager),
	)
}

// Find an environment in config, by name and set the reference
func (workflow *environmentWorkflow) environmentFinder(config *common.Config, environmentName string) Executor {

	return func() error {
		for _, e := range config.Environments {
			if strings.EqualFold(e.Name, environmentName) {
				workflow.environment = &e
				return nil
			}
		}
		return common.Warningf("Unable to find environment named '%s' in configuration", environmentName)
	}
}

func (workflow *environmentWorkflow) environmentVpcUpserter(namespace string,
	ecsStackParams map[string]string, elbStackParams map[string]string, imageFinder common.ImageFinder,
	stackUpserter common.StackUpserter, stackWaiter common.StackWaiter, azCounter common.AZCounter) Executor {
	return func() error {
		environment := workflow.environment
		vpcStackParams := make(map[string]string)
		var err error

		var vpcStackName string
		var vpcTemplateName string
		if environment.VpcTarget.Environment != "" {
			targetNamespace := environment.VpcTarget.Namespace
			if targetNamespace == "" {
				targetNamespace = namespace
			}
			log.Debugf("VpcTarget exists for different environment; targeting that VPC")
			vpcStackName = common.CreateStackName(targetNamespace, common.StackTypeVpc, environment.VpcTarget.Environment)
		} else if environment.VpcTarget.VpcID != "" {
			log.Debugf("VpcTarget exists, so we will upsert the VPC stack that references the VPC attributes")
			vpcStackName = common.CreateStackName(namespace, common.StackTypeTarget, environment.Name)
			vpcTemplateName = "vpc-target.yml"

			// target VPC referenced from config
			vpcStackParams["VpcId"] = environment.VpcTarget.VpcID
			vpcStackParams["ElbSubnetIds"] = strings.Join(environment.VpcTarget.ElbSubnetIds, ",")
			vpcStackParams["InstanceSubnetIds"] = strings.Join(environment.VpcTarget.InstanceSubnetIds, ",")
		} else {
			log.Debugf("No VpcTarget, so we will upsert the VPC stack that manages the VPC")
			vpcStackName = common.CreateStackName(namespace, common.StackTypeVpc, environment.Name)
			vpcTemplateName = "vpc.yml"

			if environment.Cluster.InstanceTenancy != "" {
				vpcStackParams["InstanceTenancy"] = string(environment.Cluster.InstanceTenancy)
			}
			if environment.Cluster.SSHAllow != "" {
				vpcStackParams["SshAllow"] = environment.Cluster.SSHAllow
			} else {
				vpcStackParams["SshAllow"] = "0.0.0.0/0"
			}
			if environment.Cluster.KeyName != "" {
				vpcStackParams["BastionKeyName"] = environment.Cluster.KeyName
				vpcStackParams["BastionImageId"], err = imageFinder.FindLatestImageID(ec2ImagePattern)
				if err != nil {
					return err
				}
			}

			vpcStackParams["ElbInternal"] = strconv.FormatBool(environment.Loadbalancer.Internal)
		}

		azCount, err := azCounter.CountAZs()
		if err != nil {
			return err
		}
		if azCount < 2 {
			return fmt.Errorf("Only found %v availability zones...need at least 2", azCount)
		}
		vpcStackParams["AZCount"] = strconv.Itoa(azCount)

		if vpcTemplateName != "" {
			log.Noticef("Upserting VPC environment '%s' ...", environment.Name)

			tags := createTagMap(&EnvironmentTags{
				Environment: environment.Name,
				Type:        string(common.StackTypeVpc),
				Provider:    string(environment.Provider),
				Revision:    workflow.codeRevision,
				Repo:        workflow.repoName,
			})

			err = stackUpserter.UpsertStack(vpcStackName, vpcTemplateName, environment, vpcStackParams, tags, workflow.cloudFormationRoleArn)
			if err != nil {
				return err
			}

			log.Debugf("Waiting for stack '%s' to complete", vpcStackName)
			stack := stackWaiter.AwaitFinalStatus(vpcStackName)

			if stack == nil {
				return fmt.Errorf("Unable to create stack %s", vpcStackName)
			}
			if strings.HasSuffix(stack.Status, "ROLLBACK_COMPLETE") || !strings.HasSuffix(stack.Status, "_COMPLETE") {
				return fmt.Errorf("Ended in failed status %s %s", stack.Status, stack.StatusReason)
			}
		}

		ecsStackParams["VpcId"] = fmt.Sprintf("%s-VpcId", vpcStackName)
		ecsStackParams["InstanceSubnetIds"] = fmt.Sprintf("%s-InstanceSubnetIds", vpcStackName)

		elbStackParams["VpcId"] = fmt.Sprintf("%s-VpcId", vpcStackName)
		elbStackParams["ElbSubnetIds"] = fmt.Sprintf("%s-ElbSubnetIds", vpcStackName)

		return nil
	}
}

func (workflow *environmentWorkflow) environmentRolesetUpserter(rolesetUpserter common.RolesetUpserter, rolesetGetter common.RolesetGetter, ecsStackParams map[string]string) Executor {
	return func() error {
		err := rolesetUpserter.UpsertCommonRoleset()
		if err != nil {
			return err
		}

		commonRoleset, err := rolesetGetter.GetCommonRoleset()
		if err != nil {
			return err
		}

		workflow.cloudFormationRoleArn = commonRoleset["CloudFormationRoleArn"]

		err = rolesetUpserter.UpsertEnvironmentRoleset(workflow.environment.Name)
		if err != nil {
			return err
		}

		environmentRoleset, err := rolesetGetter.GetEnvironmentRoleset(workflow.environment.Name)
		if err != nil {
			return err
		}

		ecsStackParams["EC2InstanceProfileArn"] = environmentRoleset["EC2InstanceProfileArn"]

		return nil
	}
}

func (workflow *environmentWorkflow) environmentElbUpserter(namespace string, ecsStackParams map[string]string, elbStackParams map[string]string, imageFinder common.ImageFinder, stackUpserter common.StackUpserter, stackWaiter common.StackWaiter) Executor {
	return func() error {
		environment := workflow.environment
		envStackName := common.CreateStackName(namespace, common.StackTypeLoadBalancer, environment.Name)

		log.Noticef("Upserting ELB environment '%s' ...", environment.Name)

		stackParams := elbStackParams

		if environment.Loadbalancer.Certificate != "" {
			stackParams["ElbCert"] = environment.Loadbalancer.Certificate
		}

		if environment.Loadbalancer.HostedZone != "" {
			stackParams["ElbDomainName"] = environment.Loadbalancer.HostedZone

			if environment.Loadbalancer.Name == "" {
				// default to env name
				stackParams["ElbHostName"] = environment.Name
			} else {
				stackParams["ElbHostName"] = environment.Loadbalancer.Name
			}
		}

		if environment.Discovery.Name == "" {
			stackParams["ServiceDiscoveryName"] = fmt.Sprintf("%s.%s.local", environment.Name, namespace)
		} else {
			stackParams["ServiceDiscoveryName"] = environment.Discovery.Name
		}

		stackParams["ElbInternal"] = strconv.FormatBool(environment.Loadbalancer.Internal)

		tags := createTagMap(&EnvironmentTags{
			Environment: environment.Name,
			Type:        string(common.StackTypeLoadBalancer),
			Provider:    string(environment.Provider),
			Revision:    workflow.codeRevision,
			Repo:        workflow.repoName,
		})

		err := stackUpserter.UpsertStack(envStackName, "elb.yml", environment, stackParams, tags, workflow.cloudFormationRoleArn)
		if err != nil {
			return err
		}
		log.Debugf("Waiting for stack '%s' to complete", envStackName)
		stack := stackWaiter.AwaitFinalStatus(envStackName)

		if stack == nil {
			return fmt.Errorf("Unable to create stack %s", envStackName)
		}
		if strings.HasSuffix(stack.Status, "ROLLBACK_COMPLETE") || !strings.HasSuffix(stack.Status, "_COMPLETE") {
			return fmt.Errorf("Ended in failed status %s %s", stack.Status, stack.StatusReason)
		}

		ecsStackParams["ElbSecurityGroup"] = stack.Outputs["ElbInstanceSecurityGroup"]

		return nil
	}
}

func (workflow *environmentWorkflow) environmentUpserter(namespace string, ecsStackParams map[string]string,
	imageFinder common.ImageFinder, stackUpserter common.StackUpserter,
	stackWaiter common.StackWaiter) Executor {
	return func() error {
		log.Debugf("Using provider '%s' for environment", workflow.environment.Provider)

		environment := workflow.environment
		envStackName := common.CreateStackName(namespace, common.StackTypeEnv, environment.Name)

		stackParams := ecsStackParams

		var templateName string
		var imagePattern string
		envMapping := map[common.EnvProvider]map[string]string{
			common.EnvProviderEcs: map[string]string{
				"templateName": "env-ecs.yml",
				"imagePattern": ecsImagePattern,
				"launchType":   "EC2"},
			common.EnvProviderEcsFargate: map[string]string{
				"templateName": "env-ecs.yml",
				"imagePattern": ecsImagePattern,
				"launchType":   "FARGATE"},
			common.EnvProviderEc2: map[string]string{
				"templateName": "env-ec2.yml",
				"imagePattern": ec2ImagePattern,
				"launchType":   ""}}
		templateName = envMapping[environment.Provider]["templateName"]
		imagePattern = envMapping[environment.Provider]["imagePattern"]
		stackParams["LaunchType"] = envMapping[environment.Provider]["launchType"]

		log.Noticef("Upserting environment '%s' ...", environment.Name)

		// Default SshAllow if none defined
		stackParams["SshAllow"] = "0.0.0.0/0"
		stackParams["SshAllow"] = common.NewStringIfNotEmpty(stackParams["SshAllow"], environment.Cluster.SSHAllow)
		stackParams["InstanceType"] = common.NewStringIfNotEmpty(stackParams["InstanceType"], environment.Cluster.InstanceType)
		stackParams["ExtraUserData"] = common.NewStringIfNotEmpty(stackParams["ExtraUserData"], environment.Cluster.ExtraUserData)
		stackParams["ImageId"] = common.NewStringIfNotEmpty(stackParams["ImageId"], environment.Cluster.ImageID)

		if environment.Cluster.ImageID == "" {
			var err error
			stackParams["ImageId"], err = imageFinder.FindLatestImageID(imagePattern)
			if err != nil {
				return err
			}
		}
		stackParams["ImageOsType"] = common.NewStringIfNotEmpty(stackParams["ImageOsType"], environment.Cluster.ImageOsType)

		stackParams["DesiredCapacity"] = common.NewStringIfNotZero(stackParams["DesiredCapacity"], environment.Cluster.DesiredCapacity)
		stackParams["MinSize"] = common.NewStringIfNotZero(stackParams["MinSize"], environment.Cluster.MinSize)
		stackParams["MaxSize"] = common.NewStringIfNotZero(stackParams["MaxSize"], environment.Cluster.MaxSize)

		stackParams["KeyName"] = common.NewStringIfNotEmpty(stackParams["KeyName"], environment.Cluster.KeyName)

		stackParams["TargetCPUReservation"] = common.NewStringIfNotZero(stackParams["TargetCPUReservation"], environment.Cluster.TargetCPUReservation)
		stackParams["TargetMemoryReservation"] = common.NewStringIfNotZero(stackParams["TargetMemoryReservation"], environment.Cluster.TargetMemoryReservation)

		stackParams["HttpProxy"] = common.NewStringIfNotEmpty(stackParams["HttpProxy"], environment.Cluster.HTTPProxy)

		tags := createTagMap(&EnvironmentTags{
			Environment: environment.Name,
			Type:        string(common.StackTypeEnv),
			Provider:    string(environment.Provider),
			Revision:    workflow.codeRevision,
			Repo:        workflow.repoName,
		})

		err := stackUpserter.UpsertStack(envStackName, templateName, environment, stackParams, tags, workflow.cloudFormationRoleArn)
		if err != nil {
			return err
		}
		log.Debugf("Waiting for stack '%s' to complete", envStackName)
		stack := stackWaiter.AwaitFinalStatus(envStackName)

		if stack == nil {
			return fmt.Errorf("Unable to create stack %s", envStackName)
		}
		if strings.HasSuffix(stack.Status, "ROLLBACK_COMPLETE") || !strings.HasSuffix(stack.Status, "_COMPLETE") {
			return fmt.Errorf("Ended in failed status %s %s", stack.Status, stack.StatusReason)
		}

		return nil
	}
}
<|endoftext|>


<commit_before>package worldone

func Add(a int, b int) {
	return a + b
}
<commit_msg>think i fixed build<commit_after>package worldone

func Add(a int, b int) int {
	return a + b
}
<|endoftext|>


<commit_before>package client

import (
	"log"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/hashicorp/nomad/client/driver"
	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/testutil"

	ctestutil "github.com/hashicorp/nomad/client/testutil"
)

func testLogger() *log.Logger {
	return log.New(os.Stderr, "", log.LstdFlags)
}

type MockTaskStateUpdater struct {
	Count       int
	Name        []string
	Status      []string
	Description []string
}

func (m *MockTaskStateUpdater) Update(name, status, desc string) {
	m.Count += 1
	m.Name = append(m.Name, name)
	m.Status = append(m.Status, status)
	m.Description = append(m.Description, desc)
}

func testTaskRunner() (*MockTaskStateUpdater, *TaskRunner) {
	logger := testLogger()
	conf := DefaultConfig()
	conf.StateDir = "/tmp"
	upd := &MockTaskStateUpdater{}
	ctx := driver.NewExecContext()
	alloc := mock.Alloc()
	task := alloc.Job.TaskGroups[0].Tasks[0]
	tr := NewTaskRunner(logger, conf, upd.Update, ctx, alloc.ID, task)
	return upd, tr
}

func TestTaskRunner_SimpleRun(t *testing.T) {
	ctestutil.ExecCompatible(t)
	upd, tr := testTaskRunner()
	go tr.Run()
	defer tr.Destroy()

	select {
	case <-tr.WaitCh():
	case <-time.After(2 * time.Second):
		t.Fatalf("timeout")
	}

	if upd.Count != 2 {
		t.Fatalf("should have 2 updates: %#v", upd)
	}
	if upd.Name[0] != tr.task.Name {
		t.Fatalf("bad: %#v", upd.Name)
	}
	if upd.Status[0] != structs.AllocClientStatusRunning {
		t.Fatalf("bad: %#v", upd.Status)
	}
	if upd.Description[0] != "task started" {
		t.Fatalf("bad: %#v", upd.Description)
	}

	if upd.Name[1] != tr.task.Name {
		t.Fatalf("bad: %#v", upd.Name)
	}
	if upd.Status[1] != structs.AllocClientStatusDead {
		t.Fatalf("bad: %#v", upd.Status)
	}
	if upd.Description[1] != "task completed" {
		t.Fatalf("bad: %#v", upd.Description)
	}
}

func TestTaskRunner_Destroy(t *testing.T) {
	ctestutil.ExecCompatible(t)
	upd, tr := testTaskRunner()

	// Change command to ensure we run for a bit
	tr.task.Config["command"] = "/bin/sleep"
	tr.task.Config["args"] = "10"
	go tr.Run()

	// Begin the tear down
	go func() {
		time.Sleep(100 * time.Millisecond)
		tr.Destroy()
	}()

	select {
	case <-tr.WaitCh():
	case <-time.After(2 * time.Second):
		t.Fatalf("timeout")
	}

	if upd.Count != 2 {
		t.Fatalf("should have 2 updates: %#v", upd)
	}
	if upd.Status[0] != structs.AllocClientStatusRunning {
		t.Fatalf("bad: %#v", upd.Status)
	}
	if upd.Status[1] != structs.AllocClientStatusDead {
		t.Fatalf("bad: %#v", upd.Status)
	}
	if !strings.Contains(upd.Description[1], "task failed") {
		t.Fatalf("bad: %#v", upd.Description)
	}
}

func TestTaskRunner_Update(t *testing.T) {
	ctestutil.ExecCompatible(t)
	_, tr := testTaskRunner()

	// Change command to ensure we run for a bit
	tr.task.Config["command"] = "/bin/sleep"
	tr.task.Config["args"] = "10"
	go tr.Run()
	defer tr.Destroy()

	// Update the task definition
	newTask := new(structs.Task)
	*newTask = *tr.task
	newTask.Driver = "foobar"
	tr.Update(newTask)

	// Wait for update to take place
	testutil.WaitForResult(func() (bool, error) {
		return tr.task == newTask, nil
	}, func(err error) {
		t.Fatalf("err: %v", err)
	})
}

/*
TODO: This test is disabled til a follow-up api changes the restore state interface.
The driver/executor interface will be changed from Open to Cleanup, in which
clean-up tears down previous allocs.

func TestTaskRunner_SaveRestoreState(t *testing.T) {
	upd, tr := testTaskRunner()

	// Change command to ensure we run for a bit
	tr.task.Config["command"] = "/bin/sleep"
	tr.task.Config["args"] = "10"
	go tr.Run()
	defer tr.Destroy()

	// Snapshot state
	time.Sleep(200 * time.Millisecond)
	err := tr.SaveState()
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Create a new task runner
	tr2 := NewTaskRunner(tr.logger, tr.config, upd.Update,
		tr.ctx, tr.allocID, &structs.Task{Name: tr.task.Name})
	err = tr2.RestoreState()
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	go tr2.Run()
	defer tr2.Destroy()

	// Destroy and wait
	tr2.Destroy()

	select {
	case <-tr.WaitCh():
	case <-time.After(2 * time.Second):
		t.Fatalf("timeout")
	}
}
*/
<commit_msg>Initialize reserved ports in task_runner<commit_after>package client

import (
	"log"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/hashicorp/nomad/client/driver"
	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/testutil"

	ctestutil "github.com/hashicorp/nomad/client/testutil"
)

func testLogger() *log.Logger {
	return log.New(os.Stderr, "", log.LstdFlags)
}

type MockTaskStateUpdater struct {
	Count       int
	Name        []string
	Status      []string
	Description []string
}

func (m *MockTaskStateUpdater) Update(name, status, desc string) {
	m.Count += 1
	m.Name = append(m.Name, name)
	m.Status = append(m.Status, status)
	m.Description = append(m.Description, desc)
}

func testTaskRunner() (*MockTaskStateUpdater, *TaskRunner) {
	logger := testLogger()
	conf := DefaultConfig()
	conf.StateDir = "/tmp"
	upd := &MockTaskStateUpdater{}
	ctx := driver.NewExecContext()
	alloc := mock.Alloc()
	task := alloc.Job.TaskGroups[0].Tasks[0]
	tr := NewTaskRunner(logger, conf, upd.Update, ctx, alloc.ID, task)
	return upd, tr
}

func testTaskRunnerPostOffer(*MockTaskStateUpdater, *TaskRunner) {
	state, runner := testTaskRunner()
	// testTaskRunner gives us a job in pre-offer state. We need it in post-
	// offer state some some other things are initialized. We'll init them here.
	runner.task.Resources.Networks[0].ReservedPorts[0] = 80
	return state, runner
}

func TestTaskRunner_SimpleRun(t *testing.T) {
	ctestutil.ExecCompatible(t)
	upd, tr := testTaskRunnerPostOffer()
	go tr.Run()
	defer tr.Destroy()

	select {
	case <-tr.WaitCh():
	case <-time.After(2 * time.Second):
		t.Fatalf("timeout")
	}

	if upd.Count != 2 {
		t.Fatalf("should have 2 updates: %#v", upd)
	}
	if upd.Name[0] != tr.task.Name {
		t.Fatalf("bad: %#v", upd.Name)
	}
	if upd.Status[0] != structs.AllocClientStatusRunning {
		t.Fatalf("bad: %#v", upd.Status)
	}
	if upd.Description[0] != "task started" {
		t.Fatalf("bad: %#v", upd.Description)
	}

	if upd.Name[1] != tr.task.Name {
		t.Fatalf("bad: %#v", upd.Name)
	}
	if upd.Status[1] != structs.AllocClientStatusDead {
		t.Fatalf("bad: %#v", upd.Status)
	}
	if upd.Description[1] != "task completed" {
		t.Fatalf("bad: %#v", upd.Description)
	}
}

func TestTaskRunner_Destroy(t *testing.T) {
	ctestutil.ExecCompatible(t)
	upd, tr := testTaskRunner()

	// Change command to ensure we run for a bit
	tr.task.Config["command"] = "/bin/sleep"
	tr.task.Config["args"] = "10"
	go tr.Run()

	// Begin the tear down
	go func() {
		time.Sleep(100 * time.Millisecond)
		tr.Destroy()
	}()

	select {
	case <-tr.WaitCh():
	case <-time.After(2 * time.Second):
		t.Fatalf("timeout")
	}

	if upd.Count != 2 {
		t.Fatalf("should have 2 updates: %#v", upd)
	}
	if upd.Status[0] != structs.AllocClientStatusRunning {
		t.Fatalf("bad: %#v", upd.Status)
	}
	if upd.Status[1] != structs.AllocClientStatusDead {
		t.Fatalf("bad: %#v", upd.Status)
	}
	if !strings.Contains(upd.Description[1], "task failed") {
		t.Fatalf("bad: %#v", upd.Description)
	}
}

func TestTaskRunner_Update(t *testing.T) {
	ctestutil.ExecCompatible(t)
	_, tr := testTaskRunner()

	// Change command to ensure we run for a bit
	tr.task.Config["command"] = "/bin/sleep"
	tr.task.Config["args"] = "10"
	go tr.Run()
	defer tr.Destroy()

	// Update the task definition
	newTask := new(structs.Task)
	*newTask = *tr.task
	newTask.Driver = "foobar"
	tr.Update(newTask)

	// Wait for update to take place
	testutil.WaitForResult(func() (bool, error) {
		return tr.task == newTask, nil
	}, func(err error) {
		t.Fatalf("err: %v", err)
	})
}

/*
TODO: This test is disabled til a follow-up api changes the restore state interface.
The driver/executor interface will be changed from Open to Cleanup, in which
clean-up tears down previous allocs.

func TestTaskRunner_SaveRestoreState(t *testing.T) {
	upd, tr := testTaskRunner()

	// Change command to ensure we run for a bit
	tr.task.Config["command"] = "/bin/sleep"
	tr.task.Config["args"] = "10"
	go tr.Run()
	defer tr.Destroy()

	// Snapshot state
	time.Sleep(200 * time.Millisecond)
	err := tr.SaveState()
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	// Create a new task runner
	tr2 := NewTaskRunner(tr.logger, tr.config, upd.Update,
		tr.ctx, tr.allocID, &structs.Task{Name: tr.task.Name})
	err = tr2.RestoreState()
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	go tr2.Run()
	defer tr2.Destroy()

	// Destroy and wait
	tr2.Destroy()

	select {
	case <-tr.WaitCh():
	case <-time.After(2 * time.Second):
		t.Fatalf("timeout")
	}
}
*/
<|endoftext|>


<commit_before>/*

	This package implements decision trees.

	ID3DecisionTree:
		Builds a decision tree using the ID3 algorithm
			by picking the Attribute which maximises
			Information Gain at each node.

		Attributes must be CategoricalAttributes at
			present, so discretise beforehand (see
			filters)

	RandomTree:
		Builds a decision tree using the ID3 algorithm
			by picking the Attribute amongst those
			randomly selected that maximises Information
			Gain

		Attributes must be CategoricalAttributes at
			present, so discretise beforehand (see
			filters)

*/

package trees
<commit_msg>Updating package level details<commit_after>/*

	This package implements decision trees.

	ID3DecisionTree:
		Builds a decision tree using the ID3 algorithm
			by picking the Attribute which maximises
			Information Gain at each node.

		Attributes must be CategoricalAttributes at
			present, so discretise beforehand (see
			filters)

  CART (Classification and Regression Trees):
    Builds a binary decision tree using the CART algorithm
      using a greedy approach to find the best split at each node.

    Can be used for regression and classficiation.
      Attributes have to be FloatAttributes even for classification.
      Hence, convert to Integer Labels before hand for Classficiation.

	RandomTree:
		Builds a decision tree using the ID3 algorithm
			by picking the Attribute amongst those
			randomly selected that maximises Information
			Gain

		Attributes must be CategoricalAttributes at
			present, so discretise beforehand (see
			filters)

*/

package trees
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"github.com/thomersch/grandine/lib/csv"
	"github.com/thomersch/grandine/lib/geojson"
	"github.com/thomersch/grandine/lib/mapping"
	"github.com/thomersch/grandine/lib/spaten"
	"github.com/thomersch/grandine/lib/spatial"
)

type filelist []string

func (fl *filelist) String() string {
	return fmt.Sprintf("%s", *fl)
}

func (fl *filelist) Set(value string) error {
	for _, s := range strings.Split(value, ",") {
		*fl = append(*fl, strings.TrimSpace(s))
	}
	return nil
}

func main() {
	var (
		infiles filelist
		conds   []mapping.Condition
	)
	dest := flag.String("out", "", "")
	mapFilePath := flag.String("mapping", "", "Path to mapping file which will be used to transform data.")
	csvLatColumn := flag.Int("csv-lat", 1, "If parsing CSV, which column contains the Latitude. Zero-indexed.")
	csvLonColumn := flag.Int("csv-lon", 2, "If parsing CSV, which column contains the Longitude. Zero-indexed.")
	csvDelimiter := flag.String("csv-delim", ",", "If parsing CSV, what is the delimiter between values")
	flag.Var(&infiles, "in", "infile(s)")
	flag.Parse()

	if len(*csvDelimiter) > 1 {
		log.Fatal("CSV Delimiter: only single character delimiters are allowed")
	}

	if len(*mapFilePath) != 0 {
		mf, err := os.Open(*mapFilePath)
		if err != nil {
			log.Fatal(err)
		}
		conds, err = mapping.ParseMapping(mf)
		if err != nil {
			log.Fatal(err)
		}

		if len(conds) > 0 {
			log.Printf("input file(s) will be filtered using %v conditions", len(conds))
		}
	}

	availableCodecs := []spatial.Codec{
		&geojson.Codec{},
		&spaten.Codec{},
		&csv.Codec{
			LatCol: *csvLatColumn,
			LonCol: *csvLonColumn,
			Delim:  rune((*csvDelimiter)[0]),
		},
	}

	// Determining which codec we will be using.
	var (
		enc interface{}
		err error
	)
	if len(*dest) == 0 {
		enc = &spaten.Codec{}
	} else {
		enc, err = guessCodec(*dest, availableCodecs)
		if err != nil {
			log.Fatalf("file type of %s is not supported (please check for correct file extension)", *dest)
		}
	}
	encoder, ok := enc.(spatial.Encoder)
	if !ok {
		log.Fatalf("%v codec does not support writing", enc)
	}

	// Determine wheter we're writing to a stream or file.
	var out io.WriteCloser
	if len(*dest) == 0 {
		out = os.Stdout
	} else {
		out, err = os.Create(*dest)
		if err != nil {
			log.Fatal(err)
		}
	}
	defer out.Close()

	var fc spatial.FeatureCollection
	for _, infileName := range infiles {
		dec, err := guessCodec(infileName, availableCodecs)
		if err != nil {
			log.Fatalf("file type of %s is not supported (please check for correct file extension)", infileName)
		}
		decoder, ok := dec.(spatial.Decoder)
		if !ok {
			log.Fatalf("%T codec does not support reading", decoder)
		}

		r, err := os.Open(infileName)
		if err != nil {
			log.Fatalf("could not open %v for reading: %v", infileName, err)
		}
		defer r.Close()

		var finished func() error
		switch d := dec.(type) {
		case spatial.ChunkedDecoder:
			chunks, err := d.ChunkedDecode(r)
			if err != nil {
				log.Fatalf("could not decode %v: %v", infileName, err)
			}
			for chunks.Next() {
				chunks.Scan(&fc)
				finished, err = write(out, &fc, encoder, conds)
				if err != nil {
					log.Fatal(err)
				}
				fc.Features = []spatial.Feature{}
			}
			err = finished()
			if err != nil {
				log.Fatal(err)
			}
		case spatial.Decoder:
			err = decoder.Decode(r, &fc)
			if err != nil {
				log.Fatalf("could not decode %v: %v", infileName, err)
			}
			finished, err = write(out, &fc, encoder, conds)
			if err != nil {
				log.Fatal(err)
			}
			finished()
		}
	}
}

var featBuf []spatial.FeatureCollection // TODO: this is not optimal, needs better wrapping

func write(w io.Writer, fs *spatial.FeatureCollection, enc spatial.Encoder, conds []mapping.Condition) (flush func() error, err error) {
	if len(conds) > 0 {
		var filtered []spatial.Feature
		for _, ft := range fs.Features {
			for _, cond := range conds {
				if cond.Matches(ft.Props) {
					nft := ft
					nft.Props = cond.Map(ft.Props)
					filtered = append(filtered, nft)
				}
			}
		}
		fs.Features = filtered
	}

	if e, ok := enc.(spatial.ChunkedEncoder); ok {
		err = e.EncodeChunk(w, fs)
		if err != nil {
			return func() error { return nil }, err
		}
		return func() error { return e.Close() }, nil
	}

	featBuf = append(featBuf, *fs)
	return func() error {
		var flat spatial.FeatureCollection
		for _, ftc := range featBuf {
			flat.Features = append(flat.Features, ftc.Features...)
			flat.SRID = ftc.SRID
		}
		return enc.Encode(w, &flat)
	}, nil
}

func guessCodec(filename string, codecs []spatial.Codec) (spatial.Codec, error) {
	fn := strings.ToLower(filename)
	for _, cd := range codecs {
		for _, ext := range cd.Extensions() {
			if strings.HasSuffix(fn, "."+ext) {
				return cd, nil
			}
		}
	}
	return nil, errors.New("file type not supported")
}
<commit_msg>cmd/converter: stream handling (in and out)<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"github.com/thomersch/grandine/lib/csv"
	"github.com/thomersch/grandine/lib/geojson"
	"github.com/thomersch/grandine/lib/mapping"
	"github.com/thomersch/grandine/lib/spaten"
	"github.com/thomersch/grandine/lib/spatial"
)

type filelist []string

func (fl *filelist) String() string {
	return fmt.Sprintf("%s", *fl)
}

func (fl *filelist) Set(value string) error {
	for _, s := range strings.Split(value, ",") {
		*fl = append(*fl, strings.TrimSpace(s))
	}
	return nil
}

func main() {
	var (
		infiles filelist
		conds   []mapping.Condition
	)
	dest := flag.String("out", "", "")
	mapFilePath := flag.String("mapping", "", "Path to mapping file which will be used to transform data.")
	csvLatColumn := flag.Int("csv-lat", 1, "If parsing CSV, which column contains the Latitude. Zero-indexed.")
	csvLonColumn := flag.Int("csv-lon", 2, "If parsing CSV, which column contains the Longitude. Zero-indexed.")
	csvDelimiter := flag.String("csv-delim", ",", "If parsing CSV, what is the delimiter between values")
	inCodecName := flag.String("in-codec", "spaten", "Specify codec for in-files. Only used for read from stdin.")
	flag.Var(&infiles, "in", "infile(s)")
	flag.Parse()

	if len(*csvDelimiter) > 1 {
		log.Fatal("CSV Delimiter: only single character delimiters are allowed")
	}

	if len(*mapFilePath) != 0 {
		mf, err := os.Open(*mapFilePath)
		if err != nil {
			log.Fatal(err)
		}
		conds, err = mapping.ParseMapping(mf)
		if err != nil {
			log.Fatal(err)
		}

		if len(conds) > 0 {
			log.Printf("input file(s) will be filtered using %v conditions", len(conds))
		}
	}

	availableCodecs := []spatial.Codec{
		&geojson.Codec{},
		&spaten.Codec{},
		&csv.Codec{
			LatCol: *csvLatColumn,
			LonCol: *csvLonColumn,
			Delim:  rune((*csvDelimiter)[0]),
		},
	}

	// Determining which codec we will be using for the output.
	var (
		enc interface{}
		err error
	)
	if len(*dest) == 0 {
		enc = &spaten.Codec{}
	} else {
		enc, err = guessCodec(*dest, availableCodecs)
		if err != nil {
			log.Fatalf("file type of %s is not supported (please check for correct file extension)", *dest)
		}
	}
	encoder, ok := enc.(spatial.Encoder)
	if !ok {
		log.Fatalf("%v codec does not support writing", enc)
	}

	// Determine whether we're writing to a stream or file.
	var out io.WriteCloser
	if len(*dest) == 0 {
		out = os.Stdout
	} else {
		out, err = os.Create(*dest)
		if err != nil {
			log.Fatal(err)
		}
	}
	defer out.Close()

	var (
		fc       spatial.FeatureCollection
		finished func() error
	)

	if len(infiles) == 0 {
		log.Println("No input files specified. Reading from stdin.")
		incodec, err := guessCodec("."+*inCodecName, availableCodecs)
		if err != nil {
			log.Fatalf("could not use incodec: %v", err)
		}
		inc, ok := incodec.(spatial.ChunkedDecoder)
		if !ok {
			log.Fatal("codec cannot be used for decoding")
		}
		icd, err := inc.ChunkedDecode(os.Stdin)
		if err != nil {
			log.Fatal(err)
		}
		for icd.Next() {
			err = icd.Scan(&fc)
			if err != nil {
				log.Fatal(err)
			}
			finished, err = write(out, &fc, encoder, conds)
		}
		finished()
	}

	for _, infileName := range infiles {
		dec, err := guessCodec(infileName, availableCodecs)
		if err != nil {
			log.Fatalf("file type of %s is not supported (please check for correct file extension)", infileName)
		}
		decoder, ok := dec.(spatial.Decoder)
		if !ok {
			log.Fatalf("%T codec does not support reading", decoder)
		}

		r, err := os.Open(infileName)
		if err != nil {
			log.Fatalf("could not open %v for reading: %v", infileName, err)
		}
		defer r.Close()

		var finished func() error
		switch d := dec.(type) {
		case spatial.ChunkedDecoder:
			chunks, err := d.ChunkedDecode(r)
			if err != nil {
				log.Fatalf("could not decode %v: %v", infileName, err)
			}
			for chunks.Next() {
				chunks.Scan(&fc)
				finished, err = write(out, &fc, encoder, conds)
				if err != nil {
					log.Fatal(err)
				}
				fc.Features = []spatial.Feature{}
			}
			err = finished()
			if err != nil {
				log.Fatal(err)
			}
		case spatial.Decoder:
			err = decoder.Decode(r, &fc)
			if err != nil {
				log.Fatalf("could not decode %v: %v", infileName, err)
			}
			finished, err = write(out, &fc, encoder, conds)
			if err != nil {
				log.Fatal(err)
			}
			finished()
		}
	}
}

var featBuf []spatial.FeatureCollection // TODO: this is not optimal, needs better wrapping

func write(w io.Writer, fs *spatial.FeatureCollection, enc spatial.Encoder, conds []mapping.Condition) (flush func() error, err error) {
	if len(conds) > 0 {
		var filtered []spatial.Feature
		for _, ft := range fs.Features {
			for _, cond := range conds {
				if cond.Matches(ft.Props) {
					nft := ft
					nft.Props = cond.Map(ft.Props)
					filtered = append(filtered, nft)
				}
			}
		}
		fs.Features = filtered
	}

	if e, ok := enc.(spatial.ChunkedEncoder); ok {
		err = e.EncodeChunk(w, fs)
		if err != nil {
			return func() error { return nil }, err
		}
		return func() error { return e.Close() }, nil
	}

	featBuf = append(featBuf, *fs)
	return func() error {
		var flat spatial.FeatureCollection
		for _, ftc := range featBuf {
			flat.Features = append(flat.Features, ftc.Features...)
			flat.SRID = ftc.SRID
		}
		return enc.Encode(w, &flat)
	}, nil
}

func guessCodec(filename string, codecs []spatial.Codec) (spatial.Codec, error) {
	fn := strings.ToLower(filename)
	for _, cd := range codecs {
		for _, ext := range cd.Extensions() {
			if strings.HasSuffix(fn, "."+ext) {
				return cd, nil
			}
		}
	}
	return nil, errors.New("file type not supported")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	log "github.com/bitrise-io/cocoapods-install/logger"
	"github.com/bitrise-io/cocoapods-install/run"
	"github.com/bitrise-io/cocoapods-install/sorting"
	"github.com/bitrise-io/go-utils/fileutil"
	"github.com/bitrise-io/go-utils/pathutil"
)

var (
	gitFolderName           = ".git"
	podsFolderName          = "Pods"
	carthageFolderName      = "Carthage"
	scanFolderNameBlackList = []string{gitFolderName, podsFolderName, carthageFolderName}

	frameworkExt           = ".framework"
	scanFolderExtBlackList = []string{frameworkExt}
)

func validateRequiredInput(key, value string) {
	if value == "" {
		log.Fail("Missing required input: %s", key)
	}
}

func fileList(searchDir string) ([]string, error) {
	fileList := []string{}

	if err := filepath.Walk(searchDir, func(path string, f os.FileInfo, err error) error {
		fileList = append(fileList, path)
		return err
	}); err != nil {
		return []string{}, err
	}

	return fileList, nil
}

func caseInsensitiveEquals(s1, s2 string) bool {
	s1, s2 = strings.ToLower(s1), strings.ToLower(s2)
	return s1 == s2
}

func isPathContainsComponent(pth, component string) bool {
	pathComponents := strings.Split(pth, string(filepath.Separator))
	for _, c := range pathComponents {
		if c == component {
			return true
		}
	}
	return false
}

func isPathContainsComponentWithExtension(pth, ext string) bool {
	pathComponents := strings.Split(pth, string(filepath.Separator))
	for _, c := range pathComponents {
		e := filepath.Ext(c)
		if e == ext {
			return true
		}
	}
	return false
}

func isRelevantPodfile(pth string) bool {
	basename := filepath.Base(pth)
	if !caseInsensitiveEquals(basename, "podfile") {
		return false
	}

	for _, folderName := range scanFolderNameBlackList {
		if isPathContainsComponent(pth, folderName) {
			return false
		}
	}

	for _, folderExt := range scanFolderExtBlackList {
		if isPathContainsComponentWithExtension(pth, folderExt) {
			return false
		}
	}

	return true
}

func findMostRootPodfile(fileList []string) string {
	podfiles := []string{}

	for _, file := range fileList {
		if isRelevantPodfile(file) {
			podfiles = append(podfiles, file)
		}
	}

	if len(podfiles) == 0 {
		return ""
	}

	sort.Sort(sorting.ByComponents(podfiles))
	return podfiles[0]
}

func cocoapodsVersionFromGemfileLockContent(content string) string {
	relevantLines := []string{}
	lines := strings.Split(content, "\n")

	specsStart := false
	for _, line := range lines {
		if strings.Contains(line, "specs:") {
			specsStart = true
		}

		trimmed := strings.Trim(line, " ")
		if trimmed == "" {
			break
		}

		if specsStart {
			relevantLines = append(relevantLines, line)
		}
	}

	exp := regexp.MustCompile(`cocoapods \((.+)\)`)
	for _, line := range relevantLines {
		match := exp.FindStringSubmatch(line)
		if match != nil && len(match) == 2 {
			return match[1]
		}
	}

	return ""
}

func cocoapodsVersionFromGemfileLock(gemfileLockPth string) (string, error) {
	content, err := fileutil.ReadStringFromFile(gemfileLockPth)
	if err != nil {
		return "", err
	}
	return cocoapodsVersionFromGemfileLockContent(content), nil
}

func cocoapodsVersionFromPodfileLockContent(content string) string {
	exp := regexp.MustCompile("COCOAPODS: (.+)")
	match := exp.FindStringSubmatch(content)
	if match != nil && len(match) == 2 {
		return match[1]
	}
	return ""
}

func cocoapodsVersionFromPodfileLock(podfileLockPth string) (string, error) {
	content, err := fileutil.ReadStringFromFile(podfileLockPth)
	if err != nil {
		return "", err
	}
	return cocoapodsVersionFromPodfileLockContent(content), nil
}

func main() {
	//
	// Inputs
	if os.Getenv("is_update_cocoapods") != "false" {
		log.Warn("`is_update_cocoapods` is deprecated!")
		log.Warn("CocoaPods version is determined based on the Gemfile.lock or the Podfile.lock in the Podfile's directory.")
	}

	if os.Getenv("install_cocoapods_version") != "" {
		log.Warn("`install_cocoapods_version` is deprecated!")
		log.Warn("CocoaPods version is determined based on the Gemfile.lock or the Podfile.lock in the Podfile's directory.")
	}

	sourceRootPath := os.Getenv("source_root_path")
	podfilePath := os.Getenv("podfile_path")

	rubyCommand, err := run.NewRubyCommandModel()
	if err != nil {
		log.Fail("Failed to create ruby command, err: %s", err)
	}

	systemCocoapodsVersion := rubyCommand.GetPodVersion()

	log.Configs(sourceRootPath, podfilePath, systemCocoapodsVersion)
	validateRequiredInput("source_root_path", sourceRootPath)

	//
	// Search for Podfile
	if podfilePath == "" {
		log.Info("Searching for Podfile")

		fileList, err := fileList(sourceRootPath)
		if err != nil {
			log.Fail("Failed to list files at: %s", sourceRootPath)
		}

		podfilePath = findMostRootPodfile(fileList)
		if podfilePath == "" {
			log.Fail("No Podfile found")
		}

		log.Done("Found Podfile: %s", podfilePath)
	}

	podfileDir := filepath.Dir(podfilePath)

	//
	// Install required cocoapods version
	log.Info("Determining required cocoapods version")

	useCocoapodsFromGemfile := false
	useCocoapodsVersion := ""

	gemfileLockPth := filepath.Join(podfileDir, "Gemfile.lock")
	log.Details("Searching for Gemfile.lock with cocoapods gem")

	if exist, err := pathutil.IsPathExists(gemfileLockPth); err != nil {
		log.Fail("Failed to check Gemfile.lock at: %s, error: %s", gemfileLockPth, err)
	} else if exist {
		version, err := cocoapodsVersionFromGemfileLock(gemfileLockPth)
		if err != nil {
			log.Fail("Failed to check if Gemfile.lock contains cocopods, error: %s", err)
		}

		if version != "" {
			log.Details("Found Gemfile.lock: %s", gemfileLockPth)
			log.Done("Gemfile.lock defined cocoapods version: %s", version)

			bundleInstallCmd := []string{"bundle", "install"}
			if err := rubyCommand.Execute(podfileDir, false, bundleInstallCmd); err != nil {
				log.Fail("Command failed, error: %s", err)
			}

			useCocoapodsFromGemfile = true
		}
	} else {
		log.Details("No Gemfile.lock with cocoapods gem found at: %s", gemfileLockPth)
	}

	if !useCocoapodsFromGemfile {
		fmt.Println("")
		log.Details("Searching for Podfile.lock")
		// Check Podfile.lock for CocoaPods version
		podfileLockPth := filepath.Join(podfileDir, "Podfile.lock")
		if exist, err := pathutil.IsPathExists(podfileLockPth); err != nil {
			log.Fail("Failed to check Podfile.lock at: %s, error: %s", podfileLockPth, err)
		} else if exist {
			// Podfile.lock exist scearch for version
			log.Details("Found Podfile.lock: %s", podfileLockPth)

			version, err := cocoapodsVersionFromPodfileLock(podfileLockPth)
			if err != nil {
				log.Fail("Failed to determin CocoaPods version, error: %s", err)
			}

			log.Done("CocoaPods version: %s", version)
			if version != systemCocoapodsVersion {
				useCocoapodsVersion = version
			}
		} else {
			// Use system installed cocoapods
			log.Warn("No Podfile.lock found at: %s", podfileLockPth)
			log.Warn("Make sure it's committed into your repository!")
			log.Done("Using system installed CocoaPods version")
		}
	}

	// Install cocoapods version
	podCmd := []string{"pod"}
	if !useCocoapodsFromGemfile && useCocoapodsVersion != "" {
		installed, err := rubyCommand.IsGemInstalled("cocoapods", useCocoapodsVersion)
		if err != nil {
			log.Fail("Command failed, error: %s", err)
		}

		if !installed {
			log.Info("Installing cocoapods: %s", useCocoapodsVersion)
			if err := rubyCommand.GemInstall("cocoapods", useCocoapodsVersion); err != nil {
				log.Fail("Command failed, error: %s", err)
			}
		}

		podCmd = append(podCmd, fmt.Sprintf("_%s_", useCocoapodsVersion))
	}

	// Run pod install
	log.Info("Installing Pods")

	podVersionCmd := append(podCmd, "--version")
	if err := rubyCommand.Execute(podfileDir, useCocoapodsFromGemfile, podVersionCmd); err != nil {
		log.Fail("Command failed, error: %s", err)
	}

	podInstallNoUpdateCmd := append(podCmd, "install", "--verbose", "--no-repo-update")
	if err := rubyCommand.Execute(podfileDir, useCocoapodsFromGemfile, podInstallNoUpdateCmd); err != nil {
		log.Warn("Command failed with error: %s, retrying without --no-repo-update ...", err)

		podRepoUpdateCmd := append(podCmd, "repo", "update")
		if err := rubyCommand.Execute(podfileDir, useCocoapodsFromGemfile, podRepoUpdateCmd); err != nil {
			log.Fail("Command failed, error: %s", err)
		}

		podInstallCmd := append(podCmd, "install", "--verbose")
		if err := rubyCommand.Execute(podfileDir, useCocoapodsFromGemfile, podInstallCmd); err != nil {
			log.Fail("Command failed, error: %s", err)
		}
	}
	log.Done("Success!")
}
<commit_msg>bundle install - parallel and retry<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	log "github.com/bitrise-io/cocoapods-install/logger"
	"github.com/bitrise-io/cocoapods-install/run"
	"github.com/bitrise-io/cocoapods-install/sorting"
	"github.com/bitrise-io/go-utils/fileutil"
	"github.com/bitrise-io/go-utils/pathutil"
)

var (
	gitFolderName           = ".git"
	podsFolderName          = "Pods"
	carthageFolderName      = "Carthage"
	scanFolderNameBlackList = []string{gitFolderName, podsFolderName, carthageFolderName}

	frameworkExt           = ".framework"
	scanFolderExtBlackList = []string{frameworkExt}
)

func validateRequiredInput(key, value string) {
	if value == "" {
		log.Fail("Missing required input: %s", key)
	}
}

func fileList(searchDir string) ([]string, error) {
	fileList := []string{}

	if err := filepath.Walk(searchDir, func(path string, f os.FileInfo, err error) error {
		fileList = append(fileList, path)
		return err
	}); err != nil {
		return []string{}, err
	}

	return fileList, nil
}

func caseInsensitiveEquals(s1, s2 string) bool {
	s1, s2 = strings.ToLower(s1), strings.ToLower(s2)
	return s1 == s2
}

func isPathContainsComponent(pth, component string) bool {
	pathComponents := strings.Split(pth, string(filepath.Separator))
	for _, c := range pathComponents {
		if c == component {
			return true
		}
	}
	return false
}

func isPathContainsComponentWithExtension(pth, ext string) bool {
	pathComponents := strings.Split(pth, string(filepath.Separator))
	for _, c := range pathComponents {
		e := filepath.Ext(c)
		if e == ext {
			return true
		}
	}
	return false
}

func isRelevantPodfile(pth string) bool {
	basename := filepath.Base(pth)
	if !caseInsensitiveEquals(basename, "podfile") {
		return false
	}

	for _, folderName := range scanFolderNameBlackList {
		if isPathContainsComponent(pth, folderName) {
			return false
		}
	}

	for _, folderExt := range scanFolderExtBlackList {
		if isPathContainsComponentWithExtension(pth, folderExt) {
			return false
		}
	}

	return true
}

func findMostRootPodfile(fileList []string) string {
	podfiles := []string{}

	for _, file := range fileList {
		if isRelevantPodfile(file) {
			podfiles = append(podfiles, file)
		}
	}

	if len(podfiles) == 0 {
		return ""
	}

	sort.Sort(sorting.ByComponents(podfiles))
	return podfiles[0]
}

func cocoapodsVersionFromGemfileLockContent(content string) string {
	relevantLines := []string{}
	lines := strings.Split(content, "\n")

	specsStart := false
	for _, line := range lines {
		if strings.Contains(line, "specs:") {
			specsStart = true
		}

		trimmed := strings.Trim(line, " ")
		if trimmed == "" {
			break
		}

		if specsStart {
			relevantLines = append(relevantLines, line)
		}
	}

	exp := regexp.MustCompile(`cocoapods \((.+)\)`)
	for _, line := range relevantLines {
		match := exp.FindStringSubmatch(line)
		if match != nil && len(match) == 2 {
			return match[1]
		}
	}

	return ""
}

func cocoapodsVersionFromGemfileLock(gemfileLockPth string) (string, error) {
	content, err := fileutil.ReadStringFromFile(gemfileLockPth)
	if err != nil {
		return "", err
	}
	return cocoapodsVersionFromGemfileLockContent(content), nil
}

func cocoapodsVersionFromPodfileLockContent(content string) string {
	exp := regexp.MustCompile("COCOAPODS: (.+)")
	match := exp.FindStringSubmatch(content)
	if match != nil && len(match) == 2 {
		return match[1]
	}
	return ""
}

func cocoapodsVersionFromPodfileLock(podfileLockPth string) (string, error) {
	content, err := fileutil.ReadStringFromFile(podfileLockPth)
	if err != nil {
		return "", err
	}
	return cocoapodsVersionFromPodfileLockContent(content), nil
}

func main() {
	//
	// Inputs
	if os.Getenv("is_update_cocoapods") != "false" {
		log.Warn("`is_update_cocoapods` is deprecated!")
		log.Warn("CocoaPods version is determined based on the Gemfile.lock or the Podfile.lock in the Podfile's directory.")
	}

	if os.Getenv("install_cocoapods_version") != "" {
		log.Warn("`install_cocoapods_version` is deprecated!")
		log.Warn("CocoaPods version is determined based on the Gemfile.lock or the Podfile.lock in the Podfile's directory.")
	}

	sourceRootPath := os.Getenv("source_root_path")
	podfilePath := os.Getenv("podfile_path")

	rubyCommand, err := run.NewRubyCommandModel()
	if err != nil {
		log.Fail("Failed to create ruby command, err: %s", err)
	}

	systemCocoapodsVersion := rubyCommand.GetPodVersion()

	log.Configs(sourceRootPath, podfilePath, systemCocoapodsVersion)
	validateRequiredInput("source_root_path", sourceRootPath)

	//
	// Search for Podfile
	if podfilePath == "" {
		log.Info("Searching for Podfile")

		fileList, err := fileList(sourceRootPath)
		if err != nil {
			log.Fail("Failed to list files at: %s", sourceRootPath)
		}

		podfilePath = findMostRootPodfile(fileList)
		if podfilePath == "" {
			log.Fail("No Podfile found")
		}

		log.Done("Found Podfile: %s", podfilePath)
	}

	podfileDir := filepath.Dir(podfilePath)

	//
	// Install required cocoapods version
	log.Info("Determining required cocoapods version")

	useCocoapodsFromGemfile := false
	useCocoapodsVersion := ""

	gemfileLockPth := filepath.Join(podfileDir, "Gemfile.lock")
	log.Details("Searching for Gemfile.lock with cocoapods gem")

	if exist, err := pathutil.IsPathExists(gemfileLockPth); err != nil {
		log.Fail("Failed to check Gemfile.lock at: %s, error: %s", gemfileLockPth, err)
	} else if exist {
		version, err := cocoapodsVersionFromGemfileLock(gemfileLockPth)
		if err != nil {
			log.Fail("Failed to check if Gemfile.lock contains cocopods, error: %s", err)
		}

		if version != "" {
			log.Details("Found Gemfile.lock: %s", gemfileLockPth)
			log.Done("Gemfile.lock defined cocoapods version: %s", version)

			bundleInstallCmd := []string{"bundle", "install", "--jobs", "20", "--retry", "5"}
			if err := rubyCommand.Execute(podfileDir, false, bundleInstallCmd); err != nil {
				log.Fail("Command failed, error: %s", err)
			}

			useCocoapodsFromGemfile = true
		}
	} else {
		log.Details("No Gemfile.lock with cocoapods gem found at: %s", gemfileLockPth)
	}

	if !useCocoapodsFromGemfile {
		fmt.Println("")
		log.Details("Searching for Podfile.lock")
		// Check Podfile.lock for CocoaPods version
		podfileLockPth := filepath.Join(podfileDir, "Podfile.lock")
		if exist, err := pathutil.IsPathExists(podfileLockPth); err != nil {
			log.Fail("Failed to check Podfile.lock at: %s, error: %s", podfileLockPth, err)
		} else if exist {
			// Podfile.lock exist scearch for version
			log.Details("Found Podfile.lock: %s", podfileLockPth)

			version, err := cocoapodsVersionFromPodfileLock(podfileLockPth)
			if err != nil {
				log.Fail("Failed to determin CocoaPods version, error: %s", err)
			}

			log.Done("CocoaPods version: %s", version)
			if version != systemCocoapodsVersion {
				useCocoapodsVersion = version
			}
		} else {
			// Use system installed cocoapods
			log.Warn("No Podfile.lock found at: %s", podfileLockPth)
			log.Warn("Make sure it's committed into your repository!")
			log.Done("Using system installed CocoaPods version")
		}
	}

	// Install cocoapods version
	podCmd := []string{"pod"}
	if !useCocoapodsFromGemfile && useCocoapodsVersion != "" {
		installed, err := rubyCommand.IsGemInstalled("cocoapods", useCocoapodsVersion)
		if err != nil {
			log.Fail("Command failed, error: %s", err)
		}

		if !installed {
			log.Info("Installing cocoapods: %s", useCocoapodsVersion)
			if err := rubyCommand.GemInstall("cocoapods", useCocoapodsVersion); err != nil {
				log.Fail("Command failed, error: %s", err)
			}
		}

		podCmd = append(podCmd, fmt.Sprintf("_%s_", useCocoapodsVersion))
	}

	// Run pod install
	log.Info("Installing Pods")

	podVersionCmd := append(podCmd, "--version")
	if err := rubyCommand.Execute(podfileDir, useCocoapodsFromGemfile, podVersionCmd); err != nil {
		log.Fail("Command failed, error: %s", err)
	}

	podInstallNoUpdateCmd := append(podCmd, "install", "--verbose", "--no-repo-update")
	if err := rubyCommand.Execute(podfileDir, useCocoapodsFromGemfile, podInstallNoUpdateCmd); err != nil {
		log.Warn("Command failed with error: %s, retrying without --no-repo-update ...", err)

		podRepoUpdateCmd := append(podCmd, "repo", "update")
		if err := rubyCommand.Execute(podfileDir, useCocoapodsFromGemfile, podRepoUpdateCmd); err != nil {
			log.Fail("Command failed, error: %s", err)
		}

		podInstallCmd := append(podCmd, "install", "--verbose")
		if err := rubyCommand.Execute(podfileDir, useCocoapodsFromGemfile, podInstallCmd); err != nil {
			log.Fail("Command failed, error: %s", err)
		}
	}
	log.Done("Success!")
}
<|endoftext|>


<commit_before>// inspired by a solution in https://stackoverflow.com/questions/47134293/compare-structs-except-one-field-golang/47134781

// need to extend this to cover
// 	nested structs
// 	other data types like slice and maps

// see go-cmp-except.go for better solution

package main

import (
	"fmt"
	"reflect"
	"time"

	mapset "github.com/deckarep/golang-set"
)

type Parent struct {
	Name string
	// NameP *string
	Date1 time.Time
	Date2 time.Time
	Child child
}

type child struct {
	Address string
}

func EqualExcept(f *Parent, other *Parent, ExceptField mapset.Set) bool {
	val := reflect.ValueOf(f).Elem()
	otherFields := reflect.Indirect(reflect.ValueOf(other))

	for i := 0; i < val.NumField(); i++ {
		typeField := val.Type().Field(i)
		if ExceptField.Contains(typeField.Name) {
			continue
		}

		value := val.Field(i)
		otherValue := otherFields.FieldByName(typeField.Name)

		if value.Interface() != otherValue.Interface() {
			return false
		}
	}
	return true
}

func main() {
	n1 := "NNN"
	n2 := "NNN"
	// p1 := "PPP"
	// p2 := "PPP"

	f1 := &Parent{
		Name: n1,
		// &p1,
		Date1: time.Now(),
		Date2: time.Now(),
		Child: child{Address: "AAA"},
	}

	f2 := &Parent{
		Name: n2,
		// &p2,
		Date1: time.Now(),
		Date2: time.Now(),
		Child: child{Address: "AAA"},
	}

	fmt.Println(EqualExcept(f1, f2, mapset.NewSet("Date1", "Date2")))
	fmt.Println(EqualExcept(f1, f2, mapset.NewSet("Name")))

}<commit_msg>ignoring fields of struct with slice is working<commit_after>// inspired by a solution in https://stackoverflow.com/questions/47134293/compare-structs-except-one-field-golang/47134781

// need to extend this to cover
//  pointers
// 	nested structs
// 	other data types like slice and maps

// see go-cmp-except.go for better solution

package main

import (
	"fmt"
	"reflect"
	"time"

	mapset "github.com/deckarep/golang-set"
	"github.com/google/go-cmp/cmp"
)

type Parent struct {
	Name string
	// NameP  *string
	Date1  time.Time
	Date2  time.Time
	Family []string
	Child  child
}

type child struct {
	Address string
}

func EqualExcept(f *Parent, other *Parent, ExceptField mapset.Set) bool {
	val := reflect.ValueOf(f).Elem()
	otherFields := reflect.Indirect(reflect.ValueOf(other))

	for i := 0; i < val.NumField(); i++ {
		typeField := val.Type().Field(i)
		if ExceptField.Contains(typeField.Name) {
			continue
		}

		value := val.Field(i)
		otherValue := otherFields.FieldByName(typeField.Name)

		if diff := cmp.Diff(value.Interface(), otherValue.Interface()); diff != "" {
			return false
		}
	}
	return true
}

func main() {
	n1 := "NNN"
	n2 := "NNN"
	// p1 := "PPP"
	// p2 := "PPP"

	f1 := &Parent{
		Name: n1,
		// NameP:  &p1,
		Date1:  time.Now(),
		Date2:  time.Now(),
		Family: []string{"1", "2"},
		Child:  child{Address: "AAA"},
	}

	f2 := &Parent{
		Name: n2,
		// NameP:  &p2,
		Date1:  time.Now(),
		Date2:  time.Now(),
		Family: []string{"1", "2"},
		Child:  child{Address: "AAA"},
	}

	fmt.Println(EqualExcept(f1, f2, mapset.NewSet("Date1", "Date2")))
	fmt.Println(EqualExcept(f1, f2, mapset.NewSet("Name")))

}<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"strings"

	"github.com/GeoNet/kit/aws/s3"
	fdsnS3 "github.com/GeoNet/kit/aws/s3"
	"github.com/GeoNet/kit/aws/sqs"
)

var (
	bucketName string
	keyPrefix  string
	sqsUrl     string
	s3Client   *s3.S3
	sqsClient  sqs.SQS
)

func init() {
	flag.StringVar(&bucketName, "bucket-name", "", "S3 bucket name which holds miniSEEDs.")
	flag.StringVar(&keyPrefix, "key-prefix", "", "Key prefix to search in the S3 bucket.")
	flag.StringVar(&sqsUrl, "sqs-url", "", "SQS queue url to send notifications to. Omit this parameter to show the list of matched keys only.")
	flag.Parse()

	var err error
	s3c, err := fdsnS3.NewWithMaxRetries(100)
	if err != nil {
		log.Fatalf("error creating S3 client: %s", err)
	}
	s3Client = &s3c
	sqsClient, err = sqs.NewWithMaxRetries(100)
	if err != nil {
		log.Fatalf("error creating SQS client: %s", err)
	}
}

func main() {
	if bucketName == "" {
		flag.Usage()
		return
	}
	fmt.Println("Checking S3 bucket:", bucketName)
	fmt.Println("Search key prefix:", keyPrefix)
	if sqsUrl == "" {
		fmt.Println("No sqs-url specified. Displaying matched key only.")
	} else {
		fmt.Println("Send to SQS:", sqsUrl)
	}

	keys, err := s3Client.ListAll(bucketName, keyPrefix)
	if err != nil {
		log.Fatalf("error listing S3 objects: %s", err)
	}
	for _, k := range keys {
		if strings.HasSuffix(k, "/") {
			continue // directories have trailing /
		}
		if err := sendSQS(k); err != nil {
			log.Fatal(err)
			break
		}
	}

	fmt.Println("Total keys matched:", len(keys))
}

func sendSQS(key string) error {
	fmt.Println("Key:", key)

	if sqsUrl == "" {
		return nil
	}

	e := fdsnS3.Event{
		Records: []fdsnS3.EventRecord{
			{
				EventName: "ObjectCreated:Put",
				S3: fdsnS3.EventS3{
					Object: fdsnS3.EventObject{
						Key: key,
					},
					Bucket: fdsnS3.EventBucket{
						Name: bucketName,
					},
				},
			},
		},
	}

	b, err := json.Marshal(e)
	if err != nil {
		return err
	}

	return sqsClient.Send(sqsUrl, string(b))
}
<commit_msg>Remove redundant package rename based on code review<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"strings"

	"github.com/GeoNet/kit/aws/s3"
	"github.com/GeoNet/kit/aws/sqs"
)

var (
	bucketName string
	keyPrefix  string
	sqsUrl     string
	s3Client   *s3.S3
	sqsClient  sqs.SQS
)

func init() {
	flag.StringVar(&bucketName, "bucket-name", "", "S3 bucket name which holds miniSEEDs.")
	flag.StringVar(&keyPrefix, "key-prefix", "", "Key prefix to search in the S3 bucket.")
	flag.StringVar(&sqsUrl, "sqs-url", "", "SQS queue url to send notifications to. Omit this parameter to show the list of matched keys only.")
	flag.Parse()

	var err error
	s3c, err := s3.NewWithMaxRetries(100)
	if err != nil {
		log.Fatalf("error creating S3 client: %s", err)
	}
	s3Client = &s3c
	sqsClient, err = sqs.NewWithMaxRetries(100)
	if err != nil {
		log.Fatalf("error creating SQS client: %s", err)
	}
}

func main() {
	if bucketName == "" {
		flag.Usage()
		return
	}
	fmt.Println("Checking S3 bucket:", bucketName)
	fmt.Println("Search key prefix:", keyPrefix)
	if sqsUrl == "" {
		fmt.Println("No sqs-url specified. Displaying matched key only.")
	} else {
		fmt.Println("Send to SQS:", sqsUrl)
	}

	keys, err := s3Client.ListAll(bucketName, keyPrefix)
	if err != nil {
		log.Fatalf("error listing S3 objects: %s", err)
	}
	for _, k := range keys {
		if strings.HasSuffix(k, "/") {
			continue // directories have trailing /
		}
		if err := sendSQS(k); err != nil {
			log.Fatal(err)
			break
		}
	}

	fmt.Println("Total keys matched:", len(keys))
}

func sendSQS(key string) error {
	fmt.Println("Key:", key)

	if sqsUrl == "" {
		return nil
	}

	e := s3.Event{
		Records: []s3.EventRecord{
			{
				EventName: "ObjectCreated:Put",
				S3: s3.EventS3{
					Object: s3.EventObject{
						Key: key,
					},
					Bucket: s3.EventBucket{
						Name: bucketName,
					},
				},
			},
		},
	}

	b, err := json.Marshal(e)
	if err != nil {
		return err
	}

	return sqsClient.Send(sqsUrl, string(b))
}
<|endoftext|>


<commit_before>// +build linux

package fs2

import (
	"bufio"
	"math"
	"os"
	"strconv"

	"github.com/opencontainers/runc/libcontainer/cgroups"
	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
	"github.com/opencontainers/runc/libcontainer/configs"
	"github.com/pkg/errors"
)

// numToStr converts an int64 value to a string for writing to a
// cgroupv2 files with .min, .max, .low, or .high suffix.
// The value of -1 is converted to "max" for cgroupv1 compatibility
// (which used to write -1 to remove the limit).
func numToStr(value int64) (ret string) {
	switch {
	case value == 0:
		ret = ""
	case value == -1:
		ret = "max"
	default:
		ret = strconv.FormatInt(value, 10)
	}

	return ret
}

func isMemorySet(cgroup *configs.Cgroup) bool {
	return cgroup.Resources.MemoryReservation != 0 ||
		cgroup.Resources.Memory != 0 || cgroup.Resources.MemorySwap != 0
}

func setMemory(dirPath string, cgroup *configs.Cgroup) error {
	if !isMemorySet(cgroup) {
		return nil
	}
	swap, err := cgroups.ConvertMemorySwapToCgroupV2Value(cgroup.Resources.MemorySwap, cgroup.Resources.Memory)
	if err != nil {
		return err
	}
	swapStr := numToStr(swap)
	if swapStr == "" && swap == 0 && cgroup.Resources.MemorySwap > 0 {
		// memory and memorySwap set to the same value -- disable swap
		swapStr = "0"
	}
	// never write empty string to `memory.swap.max`, it means set to 0.
	if swapStr != "" {
		if err := fscommon.WriteFile(dirPath, "memory.swap.max", swapStr); err != nil {
			return err
		}
	}

	if val := numToStr(cgroup.Resources.Memory); val != "" {
		if err := fscommon.WriteFile(dirPath, "memory.max", val); err != nil {
			return err
		}
	}

	// cgroup.Resources.KernelMemory is ignored

	if val := numToStr(cgroup.Resources.MemoryReservation); val != "" {
		if err := fscommon.WriteFile(dirPath, "memory.low", val); err != nil {
			return err
		}
	}

	return nil
}

func statMemory(dirPath string, stats *cgroups.Stats) error {
	// Set stats from memory.stat.
	statsFile, err := fscommon.OpenFile(dirPath, "memory.stat", os.O_RDONLY)
	if err != nil {
		return err
	}
	defer statsFile.Close()

	sc := bufio.NewScanner(statsFile)
	for sc.Scan() {
		t, v, err := fscommon.ParseKeyValue(sc.Text())
		if err != nil {
			return errors.Wrapf(err, "failed to parse memory.stat (%q)", sc.Text())
		}
		stats.MemoryStats.Stats[t] = v
	}
	stats.MemoryStats.Cache = stats.MemoryStats.Stats["file"]
	// Unlike cgroup v1 which has memory.use_hierarchy binary knob,
	// cgroup v2 is always hierarchical.
	stats.MemoryStats.UseHierarchy = true

	memoryUsage, err := getMemoryDataV2(dirPath, "")
	if err != nil {
		return err
	}
	stats.MemoryStats.Usage = memoryUsage
	swapUsage, err := getMemoryDataV2(dirPath, "swap")
	if err != nil {
		return err
	}
	// As cgroup v1 reports SwapUsage values as mem+swap combined,
	// while in cgroup v2 swap values do not include memory,
	// report combined mem+swap for v1 compatibility.
	swapUsage.Usage += memoryUsage.Usage
	if swapUsage.Limit != math.MaxUint64 {
		swapUsage.Limit += memoryUsage.Limit
	}
	stats.MemoryStats.SwapUsage = swapUsage

	return nil
}

func getMemoryDataV2(path, name string) (cgroups.MemoryData, error) {
	memoryData := cgroups.MemoryData{}

	moduleName := "memory"
	if name != "" {
		moduleName = "memory." + name
	}
	usage := moduleName + ".current"
	limit := moduleName + ".max"

	value, err := fscommon.GetCgroupParamUint(path, usage)
	if err != nil {
		if name != "" && os.IsNotExist(err) {
			// Ignore EEXIST as there's no swap accounting
			// if kernel CONFIG_MEMCG_SWAP is not set or
			// swapaccount=0 kernel boot parameter is given.
			return cgroups.MemoryData{}, nil
		}
		return cgroups.MemoryData{}, errors.Wrapf(err, "failed to parse %s", usage)
	}
	memoryData.Usage = value

	value, err = fscommon.GetCgroupParamUint(path, limit)
	if err != nil {
		return cgroups.MemoryData{}, errors.Wrapf(err, "failed to parse %s", limit)
	}
	memoryData.Limit = value

	return memoryData, nil
}
<commit_msg>libct/cg/fs2/memory.Stat: add usage for root cgroup<commit_after>// +build linux

package fs2

import (
	"bufio"
	"math"
	"os"
	"strconv"
	"strings"

	"github.com/opencontainers/runc/libcontainer/cgroups"
	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
	"github.com/opencontainers/runc/libcontainer/configs"
	"github.com/pkg/errors"
	"golang.org/x/sys/unix"
)

// numToStr converts an int64 value to a string for writing to a
// cgroupv2 files with .min, .max, .low, or .high suffix.
// The value of -1 is converted to "max" for cgroupv1 compatibility
// (which used to write -1 to remove the limit).
func numToStr(value int64) (ret string) {
	switch {
	case value == 0:
		ret = ""
	case value == -1:
		ret = "max"
	default:
		ret = strconv.FormatInt(value, 10)
	}

	return ret
}

func isMemorySet(cgroup *configs.Cgroup) bool {
	return cgroup.Resources.MemoryReservation != 0 ||
		cgroup.Resources.Memory != 0 || cgroup.Resources.MemorySwap != 0
}

func setMemory(dirPath string, cgroup *configs.Cgroup) error {
	if !isMemorySet(cgroup) {
		return nil
	}
	swap, err := cgroups.ConvertMemorySwapToCgroupV2Value(cgroup.Resources.MemorySwap, cgroup.Resources.Memory)
	if err != nil {
		return err
	}
	swapStr := numToStr(swap)
	if swapStr == "" && swap == 0 && cgroup.Resources.MemorySwap > 0 {
		// memory and memorySwap set to the same value -- disable swap
		swapStr = "0"
	}
	// never write empty string to `memory.swap.max`, it means set to 0.
	if swapStr != "" {
		if err := fscommon.WriteFile(dirPath, "memory.swap.max", swapStr); err != nil {
			return err
		}
	}

	if val := numToStr(cgroup.Resources.Memory); val != "" {
		if err := fscommon.WriteFile(dirPath, "memory.max", val); err != nil {
			return err
		}
	}

	// cgroup.Resources.KernelMemory is ignored

	if val := numToStr(cgroup.Resources.MemoryReservation); val != "" {
		if err := fscommon.WriteFile(dirPath, "memory.low", val); err != nil {
			return err
		}
	}

	return nil
}

func statMemory(dirPath string, stats *cgroups.Stats) error {
	// Set stats from memory.stat.
	statsFile, err := fscommon.OpenFile(dirPath, "memory.stat", os.O_RDONLY)
	if err != nil {
		return err
	}
	defer statsFile.Close()

	sc := bufio.NewScanner(statsFile)
	for sc.Scan() {
		t, v, err := fscommon.ParseKeyValue(sc.Text())
		if err != nil {
			return errors.Wrapf(err, "failed to parse memory.stat (%q)", sc.Text())
		}
		stats.MemoryStats.Stats[t] = v
	}
	stats.MemoryStats.Cache = stats.MemoryStats.Stats["file"]
	// Unlike cgroup v1 which has memory.use_hierarchy binary knob,
	// cgroup v2 is always hierarchical.
	stats.MemoryStats.UseHierarchy = true

	memoryUsage, err := getMemoryDataV2(dirPath, "")
	if err != nil {
		if errors.Is(err, unix.ENOENT) && dirPath == UnifiedMountpoint {
			// The root cgroup does not have memory.{current,max}
			// so emulate those using data from /proc/meminfo.
			return statsFromMeminfo(stats)
		}
		return err
	}
	stats.MemoryStats.Usage = memoryUsage
	swapUsage, err := getMemoryDataV2(dirPath, "swap")
	if err != nil {
		return err
	}
	// As cgroup v1 reports SwapUsage values as mem+swap combined,
	// while in cgroup v2 swap values do not include memory,
	// report combined mem+swap for v1 compatibility.
	swapUsage.Usage += memoryUsage.Usage
	if swapUsage.Limit != math.MaxUint64 {
		swapUsage.Limit += memoryUsage.Limit
	}
	stats.MemoryStats.SwapUsage = swapUsage

	return nil
}

func getMemoryDataV2(path, name string) (cgroups.MemoryData, error) {
	memoryData := cgroups.MemoryData{}

	moduleName := "memory"
	if name != "" {
		moduleName = "memory." + name
	}
	usage := moduleName + ".current"
	limit := moduleName + ".max"

	value, err := fscommon.GetCgroupParamUint(path, usage)
	if err != nil {
		if name != "" && os.IsNotExist(err) {
			// Ignore EEXIST as there's no swap accounting
			// if kernel CONFIG_MEMCG_SWAP is not set or
			// swapaccount=0 kernel boot parameter is given.
			return cgroups.MemoryData{}, nil
		}
		return cgroups.MemoryData{}, errors.Wrapf(err, "failed to parse %s", usage)
	}
	memoryData.Usage = value

	value, err = fscommon.GetCgroupParamUint(path, limit)
	if err != nil {
		return cgroups.MemoryData{}, errors.Wrapf(err, "failed to parse %s", limit)
	}
	memoryData.Limit = value

	return memoryData, nil
}

func statsFromMeminfo(stats *cgroups.Stats) error {
	f, err := os.Open("/proc/meminfo")
	if err != nil {
		return err
	}
	defer f.Close()

	// Fields we are interested in.
	var (
		swap_free  uint64
		swap_total uint64
		main_total uint64
		main_free  uint64
	)
	mem := map[string]*uint64{
		"SwapFree":  &swap_free,
		"SwapTotal": &swap_total,
		"MemTotal":  &main_total,
		"MemFree":   &main_free,
	}

	found := 0
	sc := bufio.NewScanner(f)
	for sc.Scan() {
		parts := strings.SplitN(sc.Text(), ":", 3)
		if len(parts) != 2 {
			// Should not happen.
			continue
		}
		k := parts[0]
		p, ok := mem[k]
		if !ok {
			// Unknown field -- not interested.
			continue
		}
		vStr := strings.TrimSpace(strings.TrimSuffix(parts[1], " kB"))
		*p, err = strconv.ParseUint(vStr, 10, 64)
		if err != nil {
			return errors.Wrap(err, "parsing /proc/meminfo "+k)
		}

		found++
		if found == len(mem) {
			// Got everything we need -- skip the rest.
			break
		}
	}
	if sc.Err() != nil {
		return sc.Err()
	}

	stats.MemoryStats.SwapUsage.Usage = (swap_total - swap_free) * 1024
	stats.MemoryStats.SwapUsage.Limit = math.MaxUint64

	stats.MemoryStats.Usage.Usage = (main_total - main_free) * 1024
	stats.MemoryStats.Usage.Limit = math.MaxUint64

	return nil
}
<|endoftext|>