<commit_before>package duktape

/*
#cgo linux LDFLAGS: -lm
#cgo CFLAGS: -DDUK_OPT_DEEP_C_STACK

# include "duktape.h"
extern duk_ret_t goFunctionCall(duk_context *ctx);
extern void goFinalizeCall(duk_context *ctx);

*/
import "C"
import (
	"errors"
	"fmt"
	"regexp"
	"sync"
	"unsafe"
)

var reFuncName = regexp.MustCompile("^[a-z_][a-z0-9_]*([A-Z_][a-z0-9_]*)*$")

const (
	goFunctionPtrProp = "\xff" + "goFunctionPtrProp"
	goContextPtrProp  = "\xff" + "goContextPtrProp"
)

type Context struct {
	*context
}

// transmute replaces the value from Context with the value of pointer
func (c *Context) transmute(p unsafe.Pointer) {
	*c = *(*Context)(p)
}

// this is a pojo containing only the values of the Context
type context struct {
	sync.Mutex
	duk_context *C.duk_context
	fnIndex     *functionIndex
	timerIndex  *timerIndex
}

// New returns plain initialized duktape context object
// See: http://duktape.org/api.html#duk_create_heap_default
func New() *Context {
	return &Context{
		&context{
			duk_context: C.duk_create_heap(nil, nil, nil, nil, nil),
			fnIndex:     newFunctionIndex(),
			timerIndex:  &timerIndex{},
		},
	}
}

func contextFromPointer(ctx *C.duk_context) *Context {
	return &Context{&context{duk_context: ctx}}
}

// PushGlobalGoFunction push the given function into duktape global object
// Returns non-negative index (relative to stack bottom) of the pushed function
// also returns error if the function name is invalid
func (d *Context) PushGlobalGoFunction(name string, fn func(*Context) int) (int, error) {
	if !reFuncName.MatchString(name) {
		return -1, errors.New("Malformed function name '" + name + "'")
	}

	d.PushGlobalObject()
	idx := d.PushGoFunction(fn)
	d.PutPropString(-2, name)
	d.Pop()

	return idx, nil
}

// PushGoFunction push the given function into duktape stack, returns non-negative
// index (relative to stack bottom) of the pushed function
func (d *Context) PushGoFunction(fn func(*Context) int) int {
	funPtr := d.fnIndex.Add(fn)
	ctxPtr := unsafe.Pointer(d)

	idx := d.PushCFunction((*[0]byte)(C.goFunctionCall), C.DUK_VARARGS)
	d.PushCFunction((*[0]byte)(C.goFinalizeCall), 1)
	d.PushPointer(funPtr)
	d.PutPropString(-2, goFunctionPtrProp)
	d.PushPointer(ctxPtr)
	d.PutPropString(-2, goContextPtrProp)
	d.SetFinalizer(-2)

	d.PushPointer(funPtr)
	d.PutPropString(-2, goFunctionPtrProp)
	d.PushPointer(ctxPtr)
	d.PutPropString(-2, goContextPtrProp)

	return idx
}

//export goFunctionCall
func goFunctionCall(cCtx *C.duk_context) C.duk_ret_t {
	d := contextFromPointer(cCtx)

	funPtr, ctxPtr := d.getFunctionPtrs()
	d.transmute(ctxPtr)

	result := d.fnIndex.Get(funPtr)(d)

	return C.duk_ret_t(result)
}

//export goFinalizeCall
func goFinalizeCall(cCtx *C.duk_context) {
	d := contextFromPointer(cCtx)

	funPtr, ctxPtr := d.getFunctionPtrs()
	d.transmute(ctxPtr)

	d.fnIndex.Delete(funPtr)
}

func (d *Context) getFunctionPtrs() (funPtr, ctxPtr unsafe.Pointer) {
	d.PushCurrentFunction()
	d.GetPropString(-1, goFunctionPtrProp)
	funPtr = d.GetPointer(-1)

	d.Pop()

	d.GetPropString(-1, goContextPtrProp)
	ctxPtr = d.GetPointer(-1)

	d.Pop2()
	return
}

// Destroy destroy all the references to the functions and freed the pointers
func (d *Context) Destroy() {
	d.fnIndex.Destroy()
}

type Error struct {
	Type       string
	Message    string
	FileName   string
	LineNumber int
	Stack      string
}

func (e *Error) Error() string {
	return fmt.Sprintf("%s: %s", e.Type, e.Message)
}

type Type int

func (t Type) IsNone() bool      { return t == TypeNone }
func (t Type) IsUndefined() bool { return t == TypeUndefined }
func (t Type) IsNull() bool      { return t == TypeNull }
func (t Type) IsBool() bool      { return t == TypeBoolean }
func (t Type) IsNumber() bool    { return t == TypeNumber }
func (t Type) IsString() bool    { return t == TypeString }
func (t Type) IsObject() bool    { return t == TypeObject }
func (t Type) IsBuffer() bool    { return t == TypeBuffer }
func (t Type) IsPointer() bool   { return t == TypePointer }
func (t Type) IsLightFunc() bool { return t == TypeLightFunc }

func (t Type) String() string {
	switch t {
	case TypeNone:
		return "None"
	case TypeUndefined:
		return "Undefined"
	case TypeNull:
		return "Null"
	case TypeBoolean:
		return "Boolean"
	case TypeNumber:
		return "Number"
	case TypeString:
		return "String"
	case TypeObject:
		return "Object"
	case TypeBuffer:
		return "Buffer"
	case TypePointer:
		return "Pointer"
	case TypeLightFunc:
		return "LightFunc"
	default:
		return "Unknown"
	}
}

type functionIndex struct {
	functions map[unsafe.Pointer]func(*Context) int
	sync.Mutex
}

type timerIndex struct {
	c float64
	sync.Mutex
}

func (t *timerIndex) get() float64 {
	t.Lock()
	defer t.Unlock()
	t.c++
	return t.c
}

func newFunctionIndex() *functionIndex {
	return &functionIndex{
		functions: make(map[unsafe.Pointer]func(*Context) int, 0),
	}
}

func (i *functionIndex) Add(fn func(*Context) int) unsafe.Pointer {
	ptr := C.malloc(1)

	i.Lock()
	defer i.Unlock()
	i.functions[ptr] = fn

	return ptr
}

func (i *functionIndex) Get(ptr unsafe.Pointer) func(*Context) int {
	i.Lock()
	defer i.Unlock()

	return i.functions[ptr]
}

func (i *functionIndex) Delete(ptr unsafe.Pointer) {
	i.Lock()
	defer i.Unlock()

	delete(i.functions, ptr)
	C.free(ptr)
}

func (i *functionIndex) Destroy() {
	i.Lock()
	defer i.Unlock()

	for ptr, _ := range i.functions {
		delete(i.functions, ptr)
		C.free(ptr)
	}
}
<commit_msg>Drop DUK_OPT_DEEP_C_STACK option<commit_after>package duktape

/*
#cgo linux LDFLAGS: -lm

# include "duktape.h"
extern duk_ret_t goFunctionCall(duk_context *ctx);
extern void goFinalizeCall(duk_context *ctx);

*/
import "C"
import (
	"errors"
	"fmt"
	"regexp"
	"sync"
	"unsafe"
)

var reFuncName = regexp.MustCompile("^[a-z_][a-z0-9_]*([A-Z_][a-z0-9_]*)*$")

const (
	goFunctionPtrProp = "\xff" + "goFunctionPtrProp"
	goContextPtrProp  = "\xff" + "goContextPtrProp"
)

type Context struct {
	*context
}

// transmute replaces the value from Context with the value of pointer
func (c *Context) transmute(p unsafe.Pointer) {
	*c = *(*Context)(p)
}

// this is a pojo containing only the values of the Context
type context struct {
	sync.Mutex
	duk_context *C.duk_context
	fnIndex     *functionIndex
	timerIndex  *timerIndex
}

// New returns plain initialized duktape context object
// See: http://duktape.org/api.html#duk_create_heap_default
func New() *Context {
	return &Context{
		&context{
			duk_context: C.duk_create_heap(nil, nil, nil, nil, nil),
			fnIndex:     newFunctionIndex(),
			timerIndex:  &timerIndex{},
		},
	}
}

func contextFromPointer(ctx *C.duk_context) *Context {
	return &Context{&context{duk_context: ctx}}
}

// PushGlobalGoFunction push the given function into duktape global object
// Returns non-negative index (relative to stack bottom) of the pushed function
// also returns error if the function name is invalid
func (d *Context) PushGlobalGoFunction(name string, fn func(*Context) int) (int, error) {
	if !reFuncName.MatchString(name) {
		return -1, errors.New("Malformed function name '" + name + "'")
	}

	d.PushGlobalObject()
	idx := d.PushGoFunction(fn)
	d.PutPropString(-2, name)
	d.Pop()

	return idx, nil
}

// PushGoFunction push the given function into duktape stack, returns non-negative
// index (relative to stack bottom) of the pushed function
func (d *Context) PushGoFunction(fn func(*Context) int) int {
	funPtr := d.fnIndex.Add(fn)
	ctxPtr := unsafe.Pointer(d)

	idx := d.PushCFunction((*[0]byte)(C.goFunctionCall), C.DUK_VARARGS)
	d.PushCFunction((*[0]byte)(C.goFinalizeCall), 1)
	d.PushPointer(funPtr)
	d.PutPropString(-2, goFunctionPtrProp)
	d.PushPointer(ctxPtr)
	d.PutPropString(-2, goContextPtrProp)
	d.SetFinalizer(-2)

	d.PushPointer(funPtr)
	d.PutPropString(-2, goFunctionPtrProp)
	d.PushPointer(ctxPtr)
	d.PutPropString(-2, goContextPtrProp)

	return idx
}

//export goFunctionCall
func goFunctionCall(cCtx *C.duk_context) C.duk_ret_t {
	d := contextFromPointer(cCtx)

	funPtr, ctxPtr := d.getFunctionPtrs()
	d.transmute(ctxPtr)

	result := d.fnIndex.Get(funPtr)(d)

	return C.duk_ret_t(result)
}

//export goFinalizeCall
func goFinalizeCall(cCtx *C.duk_context) {
	d := contextFromPointer(cCtx)

	funPtr, ctxPtr := d.getFunctionPtrs()
	d.transmute(ctxPtr)

	d.fnIndex.Delete(funPtr)
}

func (d *Context) getFunctionPtrs() (funPtr, ctxPtr unsafe.Pointer) {
	d.PushCurrentFunction()
	d.GetPropString(-1, goFunctionPtrProp)
	funPtr = d.GetPointer(-1)

	d.Pop()

	d.GetPropString(-1, goContextPtrProp)
	ctxPtr = d.GetPointer(-1)

	d.Pop2()
	return
}

// Destroy destroy all the references to the functions and freed the pointers
func (d *Context) Destroy() {
	d.fnIndex.Destroy()
}

type Error struct {
	Type       string
	Message    string
	FileName   string
	LineNumber int
	Stack      string
}

func (e *Error) Error() string {
	return fmt.Sprintf("%s: %s", e.Type, e.Message)
}

type Type int

func (t Type) IsNone() bool      { return t == TypeNone }
func (t Type) IsUndefined() bool { return t == TypeUndefined }
func (t Type) IsNull() bool      { return t == TypeNull }
func (t Type) IsBool() bool      { return t == TypeBoolean }
func (t Type) IsNumber() bool    { return t == TypeNumber }
func (t Type) IsString() bool    { return t == TypeString }
func (t Type) IsObject() bool    { return t == TypeObject }
func (t Type) IsBuffer() bool    { return t == TypeBuffer }
func (t Type) IsPointer() bool   { return t == TypePointer }
func (t Type) IsLightFunc() bool { return t == TypeLightFunc }

func (t Type) String() string {
	switch t {
	case TypeNone:
		return "None"
	case TypeUndefined:
		return "Undefined"
	case TypeNull:
		return "Null"
	case TypeBoolean:
		return "Boolean"
	case TypeNumber:
		return "Number"
	case TypeString:
		return "String"
	case TypeObject:
		return "Object"
	case TypeBuffer:
		return "Buffer"
	case TypePointer:
		return "Pointer"
	case TypeLightFunc:
		return "LightFunc"
	default:
		return "Unknown"
	}
}

type functionIndex struct {
	functions map[unsafe.Pointer]func(*Context) int
	sync.Mutex
}

type timerIndex struct {
	c float64
	sync.Mutex
}

func (t *timerIndex) get() float64 {
	t.Lock()
	defer t.Unlock()
	t.c++
	return t.c
}

func newFunctionIndex() *functionIndex {
	return &functionIndex{
		functions: make(map[unsafe.Pointer]func(*Context) int, 0),
	}
}

func (i *functionIndex) Add(fn func(*Context) int) unsafe.Pointer {
	ptr := C.malloc(1)

	i.Lock()
	defer i.Unlock()
	i.functions[ptr] = fn

	return ptr
}

func (i *functionIndex) Get(ptr unsafe.Pointer) func(*Context) int {
	i.Lock()
	defer i.Unlock()

	return i.functions[ptr]
}

func (i *functionIndex) Delete(ptr unsafe.Pointer) {
	i.Lock()
	defer i.Unlock()

	delete(i.functions, ptr)
	C.free(ptr)
}

func (i *functionIndex) Destroy() {
	i.Lock()
	defer i.Unlock()

	for ptr, _ := range i.functions {
		delete(i.functions, ptr)
		C.free(ptr)
	}
}
<|endoftext|>


<commit_before>package types

import (
	"encoding/json"
	"github.com/pkg/errors"
	"golang.org/x/text/currency"
	"strconv"
	"time"
)

// UnmarshalJSON implements json.Unmarshaller for a Journey
func (j *Journey) UnmarshalJSON(b []byte) error {
	// First let's create the analogous structure
	// We define some of the value as pointers to the real values, allowing us to bypass copying in cases where we don't need to process the data
	data := &struct {
		Duration  int64 `json:"duration"`
		Transfers *uint `json:"nb_transfers"`

		Departure string `json:"departure_date_time"`
		Requested string `json:"requested_date_time"`
		Arrival   string `json:"arrival_date_time"`

		Sections *[]Section `json:"sections"`

		From PlaceCountainer `json:"from"`
		To   PlaceCountainer `json:"to"`

		Type *JourneyQualification `json:"type"`

		Fare *Fare `json:"fare"`

		Status *JourneyStatus `json:"status"`
	}{
		Transfers: &j.Transfers,
		Sections:  &j.Sections,
		Type:      &j.Type,
		Fare:      &j.Fare,
		Status:    &j.Status,
	}

	// Now unmarshall the raw data into the analogous structure
	err := json.Unmarshal(b, data)
	if err != nil {
		return errors.Wrap(err, "Error while unmarshalling journey")
	}

	// As the given duration is in second, let's multiply it by one second to have the correct value
	j.Duration = time.Duration(data.Duration) * time.Second

	// For departure, requested and arrival, we use parseDateTime
	j.Departure, err = parseDateTime(data.Departure)
	if err != nil {
		return unmarshalErr(err, "Departure", "departure_date_time", data.Departure, "parseDateTime failed")
	}
	j.Requested, err = parseDateTime(data.Requested)
	if err != nil {
		return unmarshalErr(err, "Requested", "requested_date_time", data.Requested, "parseDateTime failed")
	}
	j.Arrival, err = parseDateTime(data.Arrival)
	if err != nil {
		return unmarshalErr(err, "Arrival", "arrival_date_time", data.Arrival, "parseDateTime failed")
	}

	// For the places, we directly use the embedded type !
	j.From, err = data.From.Place()
	if err != nil {
		return unmarshalErr(err, "From", "from", data.From, " .Place() failed")
	}
	j.To, err = data.To.Place()
	if err != nil {
		return unmarshalErr(err, "To", "to", data.To, " .Place() failed")
	}

	return nil
}

// UnmarshalJSON implements json.Unmarshaller for a Fare
func (f *Fare) UnmarshalJSON(b []byte) error {
	// First let's create the analogous structure
	// We define some of the value as pointers to the real values, allowing us to bypass copying in cases where we don't need to process the data
	data := &struct {
		Found *bool `json:"found"`
		Cost  struct {
			Value    string `json:"value"`
			Currency string `json:"currency"`
		} `json:"cost"`
	}{
		Found: &f.Found,
	}

	// Now unmarshall the raw data into the analogous structure
	err := json.Unmarshal(b, data)
	if err != nil {
		return errors.Wrap(err, "Error while unmarshalling journey")
	}

	// Let's convert the cost now
	// If we have no defined fare, let's skip that part
	if data.Cost.Value == "" || data.Cost.Currency == "" {
		return nil
	}
	// First get the currency unit
	unit, err := currency.ParseISO(data.Cost.Currency)
	if err != nil {
		return errors.Wrap(err, "Error while dealing with currency unit !")
	}
	// Now let's create the correct amount
	f.Total = unit.Amount(data.Cost.Value)

	return nil
}

// UnmarshalJSON implements json.Unmarshaller for CO2Emissions
func (c *CO2Emissions) UnmarshalJSON(b []byte) error {
	// First let's create the analogous structure
	// We define some of the value as pointers to the real values, allowing us to bypass copying in cases where we don't need to process the data
	data := &struct {
		Unit  *string `json:"unit"`
		Value string  `json:"value"`
	}{
		Unit: &c.Unit,
	}

	// Now unmarshall the raw data into the analogous structure
	err := json.Unmarshal(b, data)
	if err != nil {
		return errors.Wrap(err, "Error while unmarshalling journey")
	}

	// Now parse the value
	f, err := strconv.ParseFloat(data.Value, 64)
	if err != nil {
		return errors.Wrap(err, "Error while parsing CO2 emissions value")
	}
	c.Value = f

	return nil
}
<commit_msg>types: in cases of coordinates form of addresses IDs, no need for query escaping<commit_after>package types

import (
	"encoding/json"
	"github.com/pkg/errors"
	"golang.org/x/text/currency"
	"strconv"
	"time"
)

// UnmarshalJSON implements json.Unmarshaller for a Journey
func (j *Journey) UnmarshalJSON(b []byte) error {
	// First let's create the analogous structure
	// We define some of the value as pointers to the real values, allowing us to bypass copying in cases where we don't need to process the data
	data := &struct {
		Duration  int64 `json:"duration"`
		Transfers *uint `json:"nb_transfers"`

		Departure string `json:"departure_date_time"`
		Requested string `json:"requested_date_time"`
		Arrival   string `json:"arrival_date_time"`

		Sections *[]Section `json:"sections"`

		From PlaceCountainer `json:"from"`
		To   PlaceCountainer `json:"to"`

		Type *JourneyQualification `json:"type"`

		Fare *Fare `json:"fare"`

		Status *JourneyStatus `json:"status"`
	}{
		Transfers: &j.Transfers,
		Sections:  &j.Sections,
		Type:      &j.Type,
		Fare:      &j.Fare,
		Status:    &j.Status,
	}

	// Now unmarshall the raw data into the analogous structure
	err := json.Unmarshal(b, data)
	if err != nil {
		return errors.Wrap(err, "Error while unmarshalling journey")
	}

	// As the given duration is in second, let's multiply it by one second to have the correct value
	j.Duration = time.Duration(data.Duration) * time.Second

	// For departure, requested and arrival, we use parseDateTime
	j.Departure, err = parseDateTime(data.Departure)
	if err != nil {
		return unmarshalErr(err, "Departure", "departure_date_time", data.Departure, "parseDateTime failed")
	}
	j.Requested, err = parseDateTime(data.Requested)
	if err != nil {
		return unmarshalErr(err, "Requested", "requested_date_time", data.Requested, "parseDateTime failed")
	}
	j.Arrival, err = parseDateTime(data.Arrival)
	if err != nil {
		return unmarshalErr(err, "Arrival", "arrival_date_time", data.Arrival, "parseDateTime failed")
	}

	// For the places, we directly use the embedded type !
	// Warning: it is possible for a countainer to be empty, in those cases a nil Place will be present
	j.From, err = data.From.Place()
	if err != nil {
		return unmarshalErr(err, "From", "from", data.From, " .Place() failed")
	}
	j.To, err = data.To.Place()
	if err != nil {
		return unmarshalErr(err, "To", "to", data.To, " .Place() failed")
	}

	return nil
}

// UnmarshalJSON implements json.Unmarshaller for a Fare
func (f *Fare) UnmarshalJSON(b []byte) error {
	// First let's create the analogous structure
	// We define some of the value as pointers to the real values, allowing us to bypass copying in cases where we don't need to process the data
	data := &struct {
		Found *bool `json:"found"`
		Cost  struct {
			Value    string `json:"value"`
			Currency string `json:"currency"`
		} `json:"cost"`
	}{
		Found: &f.Found,
	}

	// Now unmarshall the raw data into the analogous structure
	err := json.Unmarshal(b, data)
	if err != nil {
		return errors.Wrap(err, "Error while unmarshalling journey")
	}

	// Let's convert the cost now
	// If we have no defined fare, let's skip that part
	if data.Cost.Value == "" || data.Cost.Currency == "" {
		return nil
	}
	// First get the currency unit
	unit, err := currency.ParseISO(data.Cost.Currency)
	if err != nil {
		return errors.Wrap(err, "Error while dealing with currency unit !")
	}
	// Now let's create the correct amount
	f.Total = unit.Amount(data.Cost.Value)

	return nil
}

// UnmarshalJSON implements json.Unmarshaller for CO2Emissions
func (c *CO2Emissions) UnmarshalJSON(b []byte) error {
	// First let's create the analogous structure
	// We define some of the value as pointers to the real values, allowing us to bypass copying in cases where we don't need to process the data
	data := &struct {
		Unit  *string `json:"unit"`
		Value string  `json:"value"`
	}{
		Unit: &c.Unit,
	}

	// Now unmarshall the raw data into the analogous structure
	err := json.Unmarshal(b, data)
	if err != nil {
		return errors.Wrap(err, "Error while unmarshalling journey")
	}

	// Now parse the value
	f, err := strconv.ParseFloat(data.Value, 64)
	if err != nil {
		return errors.Wrap(err, "Error while parsing CO2 emissions value")
	}
	c.Value = f

	return nil
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"testing"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform-plugin-sdk/helper/acctest"
	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/terraform"
)

func TestAccAwsEc2ClientVpnNetworkAssociation_basic(t *testing.T) {
	var assoc1 ec2.TargetNetwork
	rStr := acctest.RandString(5)

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAwsEc2ClientVpnNetworkAssociationDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccEc2ClientVpnNetworkAssociationConfig(rStr),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsEc2ClientVpnNetworkAssociationExists("aws_ec2_client_vpn_network_association.test", &assoc1),
				),
			},
		},
	})
}

func TestAccAwsEc2ClientVpnNetworkAssociation_disappears(t *testing.T) {
	var assoc1 ec2.TargetNetwork
	rStr := acctest.RandString(5)

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAwsEc2ClientVpnNetworkAssociationDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccEc2ClientVpnNetworkAssociationConfig(rStr),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsEc2ClientVpnNetworkAssociationExists("aws_ec2_client_vpn_network_association.test", &assoc1),
					testAccCheckAwsEc2ClientVpnNetworkAssociationDisappears(&assoc1),
				),
				ExpectNonEmptyPlan: true,
			},
		},
	})
}

func testAccCheckAwsEc2ClientVpnNetworkAssociationDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).ec2conn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_ec2_client_vpn_network_association" {
			continue
		}

		resp, _ := conn.DescribeClientVpnTargetNetworks(&ec2.DescribeClientVpnTargetNetworksInput{
			ClientVpnEndpointId: aws.String(rs.Primary.Attributes["client_vpn_endpoint_id"]),
			AssociationIds:      []*string{aws.String(rs.Primary.ID)},
		})

		for _, v := range resp.ClientVpnTargetNetworks {
			if *v.AssociationId == rs.Primary.ID && !(*v.Status.Code == "Disassociated") {
				return fmt.Errorf("[DESTROY ERROR] Client VPN network association (%s) not deleted", rs.Primary.ID)
			}
		}
	}

	return nil
}

func testAccCheckAwsEc2ClientVpnNetworkAssociationDisappears(targetNetwork *ec2.TargetNetwork) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		conn := testAccProvider.Meta().(*AWSClient).ec2conn

		_, err := conn.DisassociateClientVpnTargetNetwork(&ec2.DisassociateClientVpnTargetNetworkInput{
			AssociationId:       targetNetwork.AssociationId,
			ClientVpnEndpointId: targetNetwork.ClientVpnEndpointId,
		})

		if err != nil {
			return err
		}

		stateConf := &resource.StateChangeConf{
			Pending: []string{ec2.AssociationStatusCodeDisassociating},
			Target:  []string{ec2.AssociationStatusCodeDisassociated},
			Refresh: clientVpnNetworkAssociationRefreshFunc(conn, aws.StringValue(targetNetwork.AssociationId), aws.StringValue(targetNetwork.ClientVpnEndpointId)),
			Timeout: 10 * time.Minute,
		}

		_, err = stateConf.WaitForState()

		return err
	}
}

func testAccCheckAwsEc2ClientVpnNetworkAssociationExists(name string, assoc *ec2.TargetNetwork) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[name]
		if !ok {
			return fmt.Errorf("Not found: %s", name)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).ec2conn

		resp, err := conn.DescribeClientVpnTargetNetworks(&ec2.DescribeClientVpnTargetNetworksInput{
			ClientVpnEndpointId: aws.String(rs.Primary.Attributes["client_vpn_endpoint_id"]),
			AssociationIds:      []*string{aws.String(rs.Primary.ID)},
		})

		if err != nil {
			return fmt.Errorf("Error reading Client VPN network association (%s): %s", rs.Primary.ID, err)
		}

		for _, a := range resp.ClientVpnTargetNetworks {
			if *a.AssociationId == rs.Primary.ID && !(*a.Status.Code == "Disassociated") {
				*assoc = *a
				return nil
			}
		}

		return fmt.Errorf("Client VPN network association (%s) not found", rs.Primary.ID)
	}
}

func testAccEc2ClientVpnNetworkAssociationConfigAcmCertificateBase() string {
	key := tlsRsaPrivateKeyPem(2048)
	certificate := tlsRsaX509SelfSignedCertificatePem(key, "example.com")

	return fmt.Sprintf(`
resource "aws_acm_certificate" "test" {
  certificate_body = "%[1]s"
  private_key      = "%[2]s"
}
`, tlsPemEscapeNewlines(certificate), tlsPemEscapeNewlines(key))
}

func testAccEc2ClientVpnNetworkAssociationConfig(rName string) string {
	return testAccEc2ClientVpnNetworkAssociationConfigAcmCertificateBase() + fmt.Sprintf(`
resource "aws_vpc" "test" {
  cidr_block = "10.1.0.0/16"

  tags = {
    Name = "terraform-testacc-subnet-%s"
  }
}

resource "aws_subnet" "test" {
  cidr_block              = "10.1.1.0/24"
  vpc_id                  = "${aws_vpc.test.id}"
  map_public_ip_on_launch = true

  tags = {
    Name = "tf-acc-subnet-%s"
  }
}

resource "aws_ec2_client_vpn_endpoint" "test" {
  description            = "terraform-testacc-clientvpn-%s"
  server_certificate_arn = "${aws_acm_certificate.test.arn}"
  client_cidr_block      = "10.0.0.0/16"

  authentication_options {
    type                       = "certificate-authentication"
    root_certificate_chain_arn = "${aws_acm_certificate.test.arn}"
  }

  connection_log_options {
    enabled = false
  }
}

resource "aws_ec2_client_vpn_network_association" "test" {
  client_vpn_endpoint_id = "${aws_ec2_client_vpn_endpoint.test.id}"
  subnet_id              = "${aws_subnet.test.id}"
}
`, rName, rName, rName)
}
<commit_msg>tests/resource/aws_ec2_client_vpn_network_association: Blacklist usw2-az4 which does not support this functionality (#11717)<commit_after>package aws

import (
	"fmt"
	"testing"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform-plugin-sdk/helper/acctest"
	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/terraform"
)

func TestAccAwsEc2ClientVpnNetworkAssociation_basic(t *testing.T) {
	var assoc1 ec2.TargetNetwork
	rStr := acctest.RandString(5)

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAwsEc2ClientVpnNetworkAssociationDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccEc2ClientVpnNetworkAssociationConfig(rStr),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsEc2ClientVpnNetworkAssociationExists("aws_ec2_client_vpn_network_association.test", &assoc1),
				),
			},
		},
	})
}

func TestAccAwsEc2ClientVpnNetworkAssociation_disappears(t *testing.T) {
	var assoc1 ec2.TargetNetwork
	rStr := acctest.RandString(5)

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAwsEc2ClientVpnNetworkAssociationDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccEc2ClientVpnNetworkAssociationConfig(rStr),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsEc2ClientVpnNetworkAssociationExists("aws_ec2_client_vpn_network_association.test", &assoc1),
					testAccCheckAwsEc2ClientVpnNetworkAssociationDisappears(&assoc1),
				),
				ExpectNonEmptyPlan: true,
			},
		},
	})
}

func testAccCheckAwsEc2ClientVpnNetworkAssociationDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).ec2conn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_ec2_client_vpn_network_association" {
			continue
		}

		resp, _ := conn.DescribeClientVpnTargetNetworks(&ec2.DescribeClientVpnTargetNetworksInput{
			ClientVpnEndpointId: aws.String(rs.Primary.Attributes["client_vpn_endpoint_id"]),
			AssociationIds:      []*string{aws.String(rs.Primary.ID)},
		})

		for _, v := range resp.ClientVpnTargetNetworks {
			if *v.AssociationId == rs.Primary.ID && !(*v.Status.Code == "Disassociated") {
				return fmt.Errorf("[DESTROY ERROR] Client VPN network association (%s) not deleted", rs.Primary.ID)
			}
		}
	}

	return nil
}

func testAccCheckAwsEc2ClientVpnNetworkAssociationDisappears(targetNetwork *ec2.TargetNetwork) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		conn := testAccProvider.Meta().(*AWSClient).ec2conn

		_, err := conn.DisassociateClientVpnTargetNetwork(&ec2.DisassociateClientVpnTargetNetworkInput{
			AssociationId:       targetNetwork.AssociationId,
			ClientVpnEndpointId: targetNetwork.ClientVpnEndpointId,
		})

		if err != nil {
			return err
		}

		stateConf := &resource.StateChangeConf{
			Pending: []string{ec2.AssociationStatusCodeDisassociating},
			Target:  []string{ec2.AssociationStatusCodeDisassociated},
			Refresh: clientVpnNetworkAssociationRefreshFunc(conn, aws.StringValue(targetNetwork.AssociationId), aws.StringValue(targetNetwork.ClientVpnEndpointId)),
			Timeout: 10 * time.Minute,
		}

		_, err = stateConf.WaitForState()

		return err
	}
}

func testAccCheckAwsEc2ClientVpnNetworkAssociationExists(name string, assoc *ec2.TargetNetwork) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[name]
		if !ok {
			return fmt.Errorf("Not found: %s", name)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).ec2conn

		resp, err := conn.DescribeClientVpnTargetNetworks(&ec2.DescribeClientVpnTargetNetworksInput{
			ClientVpnEndpointId: aws.String(rs.Primary.Attributes["client_vpn_endpoint_id"]),
			AssociationIds:      []*string{aws.String(rs.Primary.ID)},
		})

		if err != nil {
			return fmt.Errorf("Error reading Client VPN network association (%s): %s", rs.Primary.ID, err)
		}

		for _, a := range resp.ClientVpnTargetNetworks {
			if *a.AssociationId == rs.Primary.ID && !(*a.Status.Code == "Disassociated") {
				*assoc = *a
				return nil
			}
		}

		return fmt.Errorf("Client VPN network association (%s) not found", rs.Primary.ID)
	}
}

func testAccEc2ClientVpnNetworkAssociationConfigAcmCertificateBase() string {
	key := tlsRsaPrivateKeyPem(2048)
	certificate := tlsRsaX509SelfSignedCertificatePem(key, "example.com")

	return fmt.Sprintf(`
resource "aws_acm_certificate" "test" {
  certificate_body = "%[1]s"
  private_key      = "%[2]s"
}
`, tlsPemEscapeNewlines(certificate), tlsPemEscapeNewlines(key))
}

func testAccEc2ClientVpnNetworkAssociationConfig(rName string) string {
	return testAccEc2ClientVpnNetworkAssociationConfigAcmCertificateBase() + fmt.Sprintf(`
data "aws_availability_zones" "available" {
  # InvalidParameterValue: AZ us-west-2d is not currently supported. Please choose another az in this region
  blacklisted_zone_ids = ["usw2-az4"]
  state                = "available"
}

resource "aws_vpc" "test" {
  cidr_block = "10.1.0.0/16"

  tags = {
    Name = "terraform-testacc-subnet-%s"
  }
}

resource "aws_subnet" "test" {
  availability_zone       = data.aws_availability_zones.available.names[0]
  cidr_block              = "10.1.1.0/24"
  vpc_id                  = "${aws_vpc.test.id}"
  map_public_ip_on_launch = true

  tags = {
    Name = "tf-acc-subnet-%s"
  }
}

resource "aws_ec2_client_vpn_endpoint" "test" {
  description            = "terraform-testacc-clientvpn-%s"
  server_certificate_arn = "${aws_acm_certificate.test.arn}"
  client_cidr_block      = "10.0.0.0/16"

  authentication_options {
    type                       = "certificate-authentication"
    root_certificate_chain_arn = "${aws_acm_certificate.test.arn}"
  }

  connection_log_options {
    enabled = false
  }
}

resource "aws_ec2_client_vpn_network_association" "test" {
  client_vpn_endpoint_id = "${aws_ec2_client_vpn_endpoint.test.id}"
  subnet_id              = "${aws_subnet.test.id}"
}
`, rName, rName, rName)
}
<|endoftext|>


<commit_before>package machine

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/rancher/norman/clientbase"
	"github.com/rancher/norman/event"
	"github.com/rancher/norman/types/convert"
	"github.com/rancher/norman/types/values"
	"github.com/rancher/rancher/pkg/machine/store"
	machineconfig "github.com/rancher/rancher/pkg/machine/store/config"
	"github.com/rancher/types/apis/management.cattle.io/v3"
	"github.com/rancher/types/config"
	"github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	typedv1 "k8s.io/client-go/kubernetes/typed/core/v1"
)

const (
	defaultEngineInstallURL = "https://releases.rancher.com/install-docker/17.03.2.sh"
)

func Register(management *config.ManagementContext) {
	secretStore, err := machineconfig.NewStore(management)
	if err != nil {
		logrus.Fatal(err)
	}

	machineClient := management.Management.Machines("")

	machineLifecycle := &Lifecycle{
		secretStore:                  secretStore,
		machineClient:                machineClient,
		machineTemplateClient:        management.Management.MachineTemplates(""),
		machineTemplateGenericClient: management.Management.MachineTemplates("").ObjectClient().UnstructuredClient(),
		configMapGetter:              management.K8sClient.CoreV1(),
		logger:                       management.EventLogger,
		clusterLister:                management.Management.Clusters("").Controller().Lister(),
	}

	machineClient.AddLifecycle("machine-controller", machineLifecycle)
}

type Lifecycle struct {
	secretStore                  *store.GenericEncryptedStore
	machineTemplateGenericClient *clientbase.ObjectClient
	machineClient                v3.MachineInterface
	machineTemplateClient        v3.MachineTemplateInterface
	configMapGetter              typedv1.ConfigMapsGetter
	logger                       event.Logger
	clusterLister                v3.ClusterLister
}

func (m *Lifecycle) setupCustom(obj *v3.Machine) {
	obj.Status.NodeConfig = &v3.RKEConfigNode{
		MachineName:      obj.Spec.ClusterName + ":" + obj.Name,
		Role:             obj.Spec.Role,
		HostnameOverride: obj.Spec.RequestedHostname,
		Address:          obj.Spec.CustomConfig.Address,
		InternalAddress:  obj.Spec.CustomConfig.InternalAddress,
		User:             obj.Spec.CustomConfig.User,
		DockerSocket:     obj.Spec.CustomConfig.DockerSocket,
		SSHKey:           obj.Spec.CustomConfig.SSHKey,
	}

	obj.Status.SSHUser = obj.Status.NodeConfig.User
	if obj.Status.SSHUser == "" {
		obj.Status.SSHUser = "root"
	}
}

func isCustom(obj *v3.Machine) bool {
	return obj.Spec.CustomConfig != nil && obj.Spec.CustomConfig.Address != ""
}

func (m *Lifecycle) Create(obj *v3.Machine) (*v3.Machine, error) {
	if isCustom(obj) {
		m.setupCustom(obj)
	}

	if obj.Spec.MachineTemplateName == "" {
		return obj, nil
	}

	newObj, err := v3.MachineConditionInitialized.Once(obj, func() (runtime.Object, error) {
		template, err := m.machineTemplateClient.Get(obj.Spec.MachineTemplateName, metav1.GetOptions{})
		if err != nil {
			return obj, err
		}
		obj.Status.MachineTemplateSpec = &template.Spec
		if obj.Spec.RequestedHostname == "" {
			obj.Spec.RequestedHostname = obj.Name
		}

		if obj.Status.MachineTemplateSpec.EngineInstallURL == "" {
			obj.Status.MachineTemplateSpec.EngineInstallURL = defaultEngineInstallURL
		}

		rawTemplate, err := m.machineTemplateGenericClient.Get(obj.Spec.MachineTemplateName, metav1.GetOptions{})
		if err != nil {
			return obj, err
		}

		rawConfig, ok := values.GetValue(rawTemplate.(*unstructured.Unstructured).Object, template.Spec.Driver+"Config")
		if !ok {
			return obj, fmt.Errorf("machine config not specified")
		}

		sshUser, ok := convert.ToMapInterface(rawConfig)["sshUser"]
		if ok {
			obj.Status.SSHUser = convert.ToString(sshUser)
		}

		if obj.Status.SSHUser == "" {
			obj.Status.SSHUser = "root"
		}

		bytes, err := json.Marshal(rawConfig)
		if err != nil {
			return obj, errors.Wrap(err, "failed to marshal machine driver confg")
		}

		obj.Status.MachineDriverConfig = string(bytes)
		return obj, nil
	})

	return newObj.(*v3.Machine), err
}

func (m *Lifecycle) Remove(obj *v3.Machine) (*v3.Machine, error) {
	if obj.Status.MachineTemplateSpec == nil {
		return obj, nil
	}
	found, err := m.isNodeInAppliedSpec(obj)
	if err != nil {
		return obj, err
	}
	if found {
		return obj, fmt.Errorf("Machine [%s] still not deleted from cluster spec", obj.Name)
	}
	config, err := machineconfig.NewMachineConfig(m.secretStore, obj)
	if err != nil {
		return obj, err
	}
	if err := config.Restore(); err != nil {
		return obj, err
	}
	defer config.Remove()

	mExists, err := machineExists(config.Dir(), obj.Spec.RequestedHostname)
	if err != nil {
		return obj, err
	}

	if mExists {
		m.logger.Infof(obj, "Removing machine %s", obj.Spec.RequestedHostname)
		if err := deleteMachine(config.Dir(), obj); err != nil {
			return obj, err
		}
		m.logger.Infof(obj, "Removing machine %s done", obj.Spec.RequestedHostname)
	}

	return obj, nil
}

func (m *Lifecycle) provision(machineDir string, obj *v3.Machine) (*v3.Machine, error) {
	configRawMap := map[string]interface{}{}
	if err := json.Unmarshal([]byte(obj.Status.MachineDriverConfig), &configRawMap); err != nil {
		return obj, errors.Wrap(err, "failed to unmarshal machine config")
	}

	// Since we know this will take a long time persist so user sees status
	obj, err := m.machineClient.Update(obj)
	if err != nil {
		return obj, err
	}

	createCommandsArgs := buildCreateCommand(obj, configRawMap)
	cmd := buildCommand(machineDir, createCommandsArgs)
	m.logger.Infof(obj, "Provisioning machine %s", obj.Spec.RequestedHostname)

	stdoutReader, stderrReader, err := startReturnOutput(cmd)
	if err != nil {
		return obj, err
	}
	defer stdoutReader.Close()
	defer stderrReader.Close()
	defer cmd.Wait()

	hostExist := false
	obj, err = m.reportStatus(stdoutReader, stderrReader, obj)
	if err != nil {
		if strings.Contains(err.Error(), "Host already exists") {
			hostExist = true
		}
		if !hostExist {
			return obj, err
		}
	}

	if err := cmd.Wait(); err != nil && !hostExist {
		return obj, err
	}

	m.logger.Infof(obj, "Provisioning machine %s done", obj.Spec.RequestedHostname)
	return obj, nil
}

func (m *Lifecycle) ready(obj *v3.Machine) (*v3.Machine, error) {
	config, err := machineconfig.NewMachineConfig(m.secretStore, obj)
	if err != nil {
		return obj, err
	}
	defer config.Cleanup()

	if err := config.Restore(); err != nil {
		return obj, err
	}

	// Provision in the background so we can poll and save the config
	done := make(chan error)
	go func() {
		newObj, err := v3.MachineConditionProvisioned.Once(obj, func() (runtime.Object, error) {
			return m.provision(config.Dir(), obj)
		})
		obj = newObj.(*v3.Machine)
		done <- err
	}()

	// Poll and save config
outer:
	for {
		select {
		case err = <-done:
			break outer
		case <-time.After(5 * time.Second):
			config.Save()
		}
	}

	newObj, saveError := v3.MachineConditionConfigSaved.Once(obj, func() (runtime.Object, error) {
		return m.saveConfig(config, config.Dir(), obj)
	})
	obj = newObj.(*v3.Machine)
	if err == nil {
		return obj, saveError
	}
	return obj, err
}

func (m *Lifecycle) Updated(obj *v3.Machine) (*v3.Machine, error) {
	if obj.Status.MachineTemplateSpec == nil {
		return obj, nil
	}

	newObj, err := v3.MachineConditionConfigReady.Once(obj, func() (runtime.Object, error) {
		return m.ready(obj)
	})
	obj = newObj.(*v3.Machine)

	return obj, err
}

func (m *Lifecycle) saveConfig(config *machineconfig.MachineConfig, machineDir string, obj *v3.Machine) (*v3.Machine, error) {
	logrus.Infof("Generating and uploading machine config %s", obj.Spec.RequestedHostname)
	if err := config.Save(); err != nil {
		return obj, err
	}

	ip, err := config.IP()
	if err != nil {
		return obj, err
	}

	interalAddress, err := config.InternalIP()
	if err != nil {
		return obj, err
	}

	sshKey, err := getSSHKey(machineDir, obj)
	if err != nil {
		return obj, err
	}

	if err := config.Save(); err != nil {
		return obj, err
	}

	obj.Status.NodeConfig = &v3.RKEConfigNode{
		MachineName:      obj.Spec.ClusterName + ":" + obj.Name,
		Address:          ip,
		InternalAddress:  interalAddress,
		User:             obj.Status.SSHUser,
		Role:             obj.Spec.Role,
		HostnameOverride: obj.Spec.RequestedHostname,
		SSHKey:           sshKey,
	}

	if len(obj.Status.NodeConfig.Role) == 0 {
		obj.Status.NodeConfig.Role = []string{"worker"}
	}

	return obj, nil
}

func (m *Lifecycle) isNodeInAppliedSpec(machine *v3.Machine) (bool, error) {
	cluster, err := m.clusterLister.Get("", machine.Spec.ClusterName)
	if err != nil {
		return false, err
	}
	if cluster == nil {
		return false, nil
	}
	if cluster.DeletionTimestamp != nil {
		return false, nil
	}
	if cluster.Status.AppliedSpec.RancherKubernetesEngineConfig == nil {
		return false, nil
	}

	for _, node := range cluster.Status.AppliedSpec.RancherKubernetesEngineConfig.Nodes {
		machineName := node.MachineName
		if len(machineName) == 0 {
			continue
		}
		if machineName == fmt.Sprintf("%s:%s", machine.Namespace, machine.Name) {
			return true, nil
		}
	}
	return false, nil
}
<commit_msg>fix machine delete issue<commit_after>package machine

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/rancher/norman/clientbase"
	"github.com/rancher/norman/event"
	"github.com/rancher/norman/types/convert"
	"github.com/rancher/norman/types/values"
	"github.com/rancher/rancher/pkg/machine/store"
	machineconfig "github.com/rancher/rancher/pkg/machine/store/config"
	"github.com/rancher/types/apis/management.cattle.io/v3"
	"github.com/rancher/types/config"
	"github.com/sirupsen/logrus"
	kerror "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	typedv1 "k8s.io/client-go/kubernetes/typed/core/v1"
)

const (
	defaultEngineInstallURL = "https://releases.rancher.com/install-docker/17.03.2.sh"
)

func Register(management *config.ManagementContext) {
	secretStore, err := machineconfig.NewStore(management)
	if err != nil {
		logrus.Fatal(err)
	}

	machineClient := management.Management.Machines("")

	machineLifecycle := &Lifecycle{
		secretStore:                  secretStore,
		machineClient:                machineClient,
		machineTemplateClient:        management.Management.MachineTemplates(""),
		machineTemplateGenericClient: management.Management.MachineTemplates("").ObjectClient().UnstructuredClient(),
		configMapGetter:              management.K8sClient.CoreV1(),
		logger:                       management.EventLogger,
		clusterLister:                management.Management.Clusters("").Controller().Lister(),
	}

	machineClient.AddLifecycle("machine-controller", machineLifecycle)
}

type Lifecycle struct {
	secretStore                  *store.GenericEncryptedStore
	machineTemplateGenericClient *clientbase.ObjectClient
	machineClient                v3.MachineInterface
	machineTemplateClient        v3.MachineTemplateInterface
	configMapGetter              typedv1.ConfigMapsGetter
	logger                       event.Logger
	clusterLister                v3.ClusterLister
}

func (m *Lifecycle) setupCustom(obj *v3.Machine) {
	obj.Status.NodeConfig = &v3.RKEConfigNode{
		MachineName:      obj.Spec.ClusterName + ":" + obj.Name,
		Role:             obj.Spec.Role,
		HostnameOverride: obj.Spec.RequestedHostname,
		Address:          obj.Spec.CustomConfig.Address,
		InternalAddress:  obj.Spec.CustomConfig.InternalAddress,
		User:             obj.Spec.CustomConfig.User,
		DockerSocket:     obj.Spec.CustomConfig.DockerSocket,
		SSHKey:           obj.Spec.CustomConfig.SSHKey,
	}

	obj.Status.SSHUser = obj.Status.NodeConfig.User
	if obj.Status.SSHUser == "" {
		obj.Status.SSHUser = "root"
	}
}

func isCustom(obj *v3.Machine) bool {
	return obj.Spec.CustomConfig != nil && obj.Spec.CustomConfig.Address != ""
}

func (m *Lifecycle) Create(obj *v3.Machine) (*v3.Machine, error) {
	if isCustom(obj) {
		m.setupCustom(obj)
	}

	if obj.Spec.MachineTemplateName == "" {
		return obj, nil
	}

	newObj, err := v3.MachineConditionInitialized.Once(obj, func() (runtime.Object, error) {
		template, err := m.machineTemplateClient.Get(obj.Spec.MachineTemplateName, metav1.GetOptions{})
		if err != nil {
			return obj, err
		}
		obj.Status.MachineTemplateSpec = &template.Spec
		if obj.Spec.RequestedHostname == "" {
			obj.Spec.RequestedHostname = obj.Name
		}

		if obj.Status.MachineTemplateSpec.EngineInstallURL == "" {
			obj.Status.MachineTemplateSpec.EngineInstallURL = defaultEngineInstallURL
		}

		rawTemplate, err := m.machineTemplateGenericClient.Get(obj.Spec.MachineTemplateName, metav1.GetOptions{})
		if err != nil {
			return obj, err
		}

		rawConfig, ok := values.GetValue(rawTemplate.(*unstructured.Unstructured).Object, template.Spec.Driver+"Config")
		if !ok {
			return obj, fmt.Errorf("machine config not specified")
		}

		sshUser, ok := convert.ToMapInterface(rawConfig)["sshUser"]
		if ok {
			obj.Status.SSHUser = convert.ToString(sshUser)
		}

		if obj.Status.SSHUser == "" {
			obj.Status.SSHUser = "root"
		}

		bytes, err := json.Marshal(rawConfig)
		if err != nil {
			return obj, errors.Wrap(err, "failed to marshal machine driver confg")
		}

		obj.Status.MachineDriverConfig = string(bytes)
		return obj, nil
	})

	return newObj.(*v3.Machine), err
}

func (m *Lifecycle) Remove(obj *v3.Machine) (*v3.Machine, error) {
	if obj.Status.MachineTemplateSpec == nil {
		return obj, nil
	}
	found, err := m.isNodeInAppliedSpec(obj)
	if err != nil {
		return obj, err
	}
	if found {
		return obj, fmt.Errorf("Machine [%s] still not deleted from cluster spec", obj.Name)
	}
	config, err := machineconfig.NewMachineConfig(m.secretStore, obj)
	if err != nil {
		return obj, err
	}
	if err := config.Restore(); err != nil {
		return obj, err
	}
	defer config.Remove()

	mExists, err := machineExists(config.Dir(), obj.Spec.RequestedHostname)
	if err != nil {
		return obj, err
	}

	if mExists {
		m.logger.Infof(obj, "Removing machine %s", obj.Spec.RequestedHostname)
		if err := deleteMachine(config.Dir(), obj); err != nil {
			return obj, err
		}
		m.logger.Infof(obj, "Removing machine %s done", obj.Spec.RequestedHostname)
	}

	return obj, nil
}

func (m *Lifecycle) provision(machineDir string, obj *v3.Machine) (*v3.Machine, error) {
	configRawMap := map[string]interface{}{}
	if err := json.Unmarshal([]byte(obj.Status.MachineDriverConfig), &configRawMap); err != nil {
		return obj, errors.Wrap(err, "failed to unmarshal machine config")
	}

	// Since we know this will take a long time persist so user sees status
	obj, err := m.machineClient.Update(obj)
	if err != nil {
		return obj, err
	}

	createCommandsArgs := buildCreateCommand(obj, configRawMap)
	cmd := buildCommand(machineDir, createCommandsArgs)
	m.logger.Infof(obj, "Provisioning machine %s", obj.Spec.RequestedHostname)

	stdoutReader, stderrReader, err := startReturnOutput(cmd)
	if err != nil {
		return obj, err
	}
	defer stdoutReader.Close()
	defer stderrReader.Close()
	defer cmd.Wait()

	hostExist := false
	obj, err = m.reportStatus(stdoutReader, stderrReader, obj)
	if err != nil {
		if strings.Contains(err.Error(), "Host already exists") {
			hostExist = true
		}
		if !hostExist {
			return obj, err
		}
	}

	if err := cmd.Wait(); err != nil && !hostExist {
		return obj, err
	}

	m.logger.Infof(obj, "Provisioning machine %s done", obj.Spec.RequestedHostname)
	return obj, nil
}

func (m *Lifecycle) ready(obj *v3.Machine) (*v3.Machine, error) {
	config, err := machineconfig.NewMachineConfig(m.secretStore, obj)
	if err != nil {
		return obj, err
	}
	defer config.Cleanup()

	if err := config.Restore(); err != nil {
		return obj, err
	}

	// Provision in the background so we can poll and save the config
	done := make(chan error)
	go func() {
		newObj, err := v3.MachineConditionProvisioned.Once(obj, func() (runtime.Object, error) {
			return m.provision(config.Dir(), obj)
		})
		obj = newObj.(*v3.Machine)
		done <- err
	}()

	// Poll and save config
outer:
	for {
		select {
		case err = <-done:
			break outer
		case <-time.After(5 * time.Second):
			config.Save()
		}
	}

	newObj, saveError := v3.MachineConditionConfigSaved.Once(obj, func() (runtime.Object, error) {
		return m.saveConfig(config, config.Dir(), obj)
	})
	obj = newObj.(*v3.Machine)
	if err == nil {
		return obj, saveError
	}
	return obj, err
}

func (m *Lifecycle) Updated(obj *v3.Machine) (*v3.Machine, error) {
	if obj.Status.MachineTemplateSpec == nil {
		return obj, nil
	}

	newObj, err := v3.MachineConditionConfigReady.Once(obj, func() (runtime.Object, error) {
		return m.ready(obj)
	})
	obj = newObj.(*v3.Machine)

	return obj, err
}

func (m *Lifecycle) saveConfig(config *machineconfig.MachineConfig, machineDir string, obj *v3.Machine) (*v3.Machine, error) {
	logrus.Infof("Generating and uploading machine config %s", obj.Spec.RequestedHostname)
	if err := config.Save(); err != nil {
		return obj, err
	}

	ip, err := config.IP()
	if err != nil {
		return obj, err
	}

	interalAddress, err := config.InternalIP()
	if err != nil {
		return obj, err
	}

	sshKey, err := getSSHKey(machineDir, obj)
	if err != nil {
		return obj, err
	}

	if err := config.Save(); err != nil {
		return obj, err
	}

	obj.Status.NodeConfig = &v3.RKEConfigNode{
		MachineName:      obj.Spec.ClusterName + ":" + obj.Name,
		Address:          ip,
		InternalAddress:  interalAddress,
		User:             obj.Status.SSHUser,
		Role:             obj.Spec.Role,
		HostnameOverride: obj.Spec.RequestedHostname,
		SSHKey:           sshKey,
	}

	if len(obj.Status.NodeConfig.Role) == 0 {
		obj.Status.NodeConfig.Role = []string{"worker"}
	}

	return obj, nil
}

func (m *Lifecycle) isNodeInAppliedSpec(machine *v3.Machine) (bool, error) {
	cluster, err := m.clusterLister.Get("", machine.Spec.ClusterName)
	if err != nil {
		if kerror.IsNotFound(err) {
			return false, nil
		}
		return false, err
	}
	if cluster == nil {
		return false, nil
	}
	if cluster.DeletionTimestamp != nil {
		return false, nil
	}
	if cluster.Status.AppliedSpec.RancherKubernetesEngineConfig == nil {
		return false, nil
	}

	for _, node := range cluster.Status.AppliedSpec.RancherKubernetesEngineConfig.Nodes {
		machineName := node.MachineName
		if len(machineName) == 0 {
			continue
		}
		if machineName == fmt.Sprintf("%s:%s", machine.Namespace, machine.Name) {
			return true, nil
		}
	}
	return false, nil
}
<|endoftext|>


<commit_before><commit_msg>Switched from os.Rename to filesystem.Rename in staging.<commit_after><|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
	"github.com/pivotal-cf/cf-redis-broker/brokerconfig"
	"github.com/pivotal-cf/cf-redis-broker/redis"
	"github.com/pivotal-cf/cf-redis-broker/redis/client"
	"github.com/pivotal-cf/cf-redis-broker/redisconf"
	"github.com/pivotal-golang/lager"

	cf_lager "github.com/cloudfoundry-incubator/cf-lager"
)

var S3RegionNotFoundErr error = errors.New("S3 region not found")
var logger = cf_lager.New("backup")

func main() {
	config := parseConfig()
	if !config.RedisConfiguration.BackupConfiguration.Enabled() {
		return
	}

	bucket, err := getBucket(config.RedisConfiguration.BackupConfiguration)
	if err != nil {
		log.Fatal(err)
	}

	instanceDirs, err := ioutil.ReadDir(config.RedisConfiguration.InstanceDataDirectory)
	if err != nil {
		log.Fatal(err)
	}

	backupErrors := []error{}
	for _, instanceDir := range instanceDirs {
		if instanceDir.Name()[0] == '.' {
			continue
		}
		err = backupInstance(instanceDir, config, bucket)
		if err != nil {
			backupErrors = append(backupErrors, err)
			logger.Error("error backing up instance", err, lager.Data{
				"instance_id": instanceDir.Name(),
			})
		}
	}

	if len(backupErrors) > 0 {
		os.Exit(1)
	}
}

func parseConfig() brokerconfig.Config {
	config, err := brokerconfig.ParseConfig(configPath())
	if err != nil {
		log.Fatal(err)
	}
	return config
}

func backupInstance(instanceDir os.FileInfo, config brokerconfig.Config, bucket *s3.Bucket) error {

	pathToInstanceDirectory := filepath.Join(config.RedisConfiguration.InstanceDataDirectory, instanceDir.Name())
	if !fileExists(pathToInstanceDirectory) {
		logger.Info("instance directory not found, skipping instance backup", lager.Data{
			"Local file": pathToInstanceDirectory,
		})
		return nil
	}

	err := saveAndWaitUntilFinished(instanceDir, config)
	if err != nil {
		return err
	}

	pathToRdbFile := filepath.Join(config.RedisConfiguration.InstanceDataDirectory, instanceDir.Name(), "db", "dump.rdb")
	if !fileExists(pathToRdbFile) {
		logger.Info("dump.rb not found, skipping instance backup", lager.Data{
			"Local file": pathToRdbFile,
		})
		return nil
	}

	rdbBytes, err := ioutil.ReadFile(pathToRdbFile)
	if err != nil {
		return err
	}

	remotePath := fmt.Sprintf("%s/%s", config.RedisConfiguration.BackupConfiguration.Path, instanceDir.Name())

	logger.Info("Backing up instance", lager.Data{
		"Local file":  pathToRdbFile,
		"Remote file": remotePath,
	})

	return bucket.Put(remotePath, rdbBytes, "", "")
}

func fileExists(path string) bool {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return false
	}
	return true
}

func getBucket(backupConfig brokerconfig.BackupConfiguration) (*s3.Bucket, error) {
	client, err := configureS3Client(backupConfig)
	if err != nil {
		return nil, err
	}
	bucket := client.Bucket(backupConfig.BucketName)

	foundBucketOnS3 := false

	resp, err := client.ListBuckets()
	if err != nil {
		return nil, err
	}

	for _, bucket := range resp.Buckets {
		if bucket.Name == backupConfig.BucketName {
			foundBucketOnS3 = true
			logger.Info("Bucket exists", lager.Data{"bucket": backupConfig.BucketName})
			break
		}
	}

	if !foundBucketOnS3 {
		logger.Info("Creating bucket", lager.Data{"bucket": backupConfig.BucketName})
		err = bucket.PutBucket(s3.Private)
		if err != nil {
			return nil, err
		}
	}

	return bucket, nil
}

func saveAndWaitUntilFinished(instanceDir os.FileInfo, config brokerconfig.Config) error {

	instanceID := instanceDir.Name()

	client, err := buildRedisClient(instanceID, config)
	if err != nil {
		return err
	}

	lastSaveTime, err := client.LastRDBSaveTime()
	if err != nil {
		return err
	}

	// ensure new save time can't match last save time
	time.Sleep(time.Second)

	err = client.RunBGSave()
	if err != nil {
		return err
	}

	// wait for save to complete
	timeout := config.RedisConfiguration.BackupConfiguration.BGSaveTimeoutSeconds
	for i := 0; i < timeout; i++ {
		saveTime, err := client.LastRDBSaveTime()
		if err != nil {
			return err
		}

		if saveTime > lastSaveTime {
			return nil
		}

		time.Sleep(time.Second)
	}

	return errors.New("Timed out waiting for background save to complete")
}

func buildRedisClient(instanceID string, config brokerconfig.Config) (*client.Client, error) {

	localRepo := redis.LocalRepository{RedisConf: config.RedisConfiguration}
	instance, err := localRepo.FindByID(instanceID)
	if err != nil {
		return nil, err
	}

	instanceConf, err := redisconf.Load(localRepo.InstanceConfigPath(instanceID))
	if err != nil {
		return nil, err
	}

	return client.Connect(instance.Host, uint(instance.Port), instance.Password, instanceConf)
}

func configureS3Client(backupConfig brokerconfig.BackupConfiguration) (*s3.S3, error) {
	// Warning! Fake S3 server does not appear to honour region constraints or authentication.
	// The fact that this uses the configured region and authentication is untested.
	region, err := getRegion(backupConfig)
	if err != nil {
		return nil, err
	}

	auth := aws.Auth{
		AccessKey: backupConfig.AccessKeyId,
		SecretKey: backupConfig.SecretAccessKey,
	}

	return s3.New(auth, region), nil
}

func getRegion(backupConfig brokerconfig.BackupConfiguration) (aws.Region, error) {
	for _, region := range aws.Regions {
		if region.S3Endpoint == backupConfig.EndpointUrl {
			return region, nil
		}
	}

	if backupConfig.S3Region != "" {
		return aws.Region{
			Name:                 backupConfig.S3Region,
			S3Endpoint:           backupConfig.EndpointUrl,
			S3LocationConstraint: true,
		}, nil
	}

	return aws.Region{}, S3RegionNotFoundErr
}

func configPath() string {
	brokerConfigYamlPath := os.Getenv("BROKER_CONFIG_PATH")
	if brokerConfigYamlPath == "" {
		panic("BROKER_CONFIG_PATH not set")
	}
	return brokerConfigYamlPath
}
<commit_msg>tiny readability improvement in backup/main.go<commit_after>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
	"github.com/pivotal-cf/cf-redis-broker/brokerconfig"
	"github.com/pivotal-cf/cf-redis-broker/redis"
	"github.com/pivotal-cf/cf-redis-broker/redis/client"
	"github.com/pivotal-cf/cf-redis-broker/redisconf"
	"github.com/pivotal-golang/lager"

	cf_lager "github.com/cloudfoundry-incubator/cf-lager"
)

var S3RegionNotFoundErr error = errors.New("S3 region not found")
var logger = cf_lager.New("backup")

func main() {
	config := parseConfig()
	if !config.RedisConfiguration.BackupConfiguration.Enabled() {
		return
	}

	bucket, err := getBucket(config.RedisConfiguration.BackupConfiguration)
	if err != nil {
		log.Fatal(err)
	}

	instanceDirs, err := ioutil.ReadDir(config.RedisConfiguration.InstanceDataDirectory)
	if err != nil {
		log.Fatal(err)
	}

	backupErrors := []error{}
	for _, instanceDir := range instanceDirs {
		if strings.HasPrefix(instanceDir.Name(), ".") {
			continue
		}
		err = backupInstance(instanceDir, config, bucket)
		if err != nil {
			backupErrors = append(backupErrors, err)
			logger.Error("error backing up instance", err, lager.Data{
				"instance_id": instanceDir.Name(),
			})
		}
	}

	if len(backupErrors) > 0 {
		os.Exit(1)
	}
}

func parseConfig() brokerconfig.Config {
	config, err := brokerconfig.ParseConfig(configPath())
	if err != nil {
		log.Fatal(err)
	}
	return config
}

func backupInstance(instanceDir os.FileInfo, config brokerconfig.Config, bucket *s3.Bucket) error {

	pathToInstanceDirectory := filepath.Join(config.RedisConfiguration.InstanceDataDirectory, instanceDir.Name())
	if !fileExists(pathToInstanceDirectory) {
		logger.Info("instance directory not found, skipping instance backup", lager.Data{
			"Local file": pathToInstanceDirectory,
		})
		return nil
	}

	err := saveAndWaitUntilFinished(instanceDir, config)
	if err != nil {
		return err
	}

	pathToRdbFile := filepath.Join(config.RedisConfiguration.InstanceDataDirectory, instanceDir.Name(), "db", "dump.rdb")
	if !fileExists(pathToRdbFile) {
		logger.Info("dump.rb not found, skipping instance backup", lager.Data{
			"Local file": pathToRdbFile,
		})
		return nil
	}

	rdbBytes, err := ioutil.ReadFile(pathToRdbFile)
	if err != nil {
		return err
	}

	remotePath := fmt.Sprintf("%s/%s", config.RedisConfiguration.BackupConfiguration.Path, instanceDir.Name())

	logger.Info("Backing up instance", lager.Data{
		"Local file":  pathToRdbFile,
		"Remote file": remotePath,
	})

	return bucket.Put(remotePath, rdbBytes, "", "")
}

func fileExists(path string) bool {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return false
	}
	return true
}

func getBucket(backupConfig brokerconfig.BackupConfiguration) (*s3.Bucket, error) {
	client, err := configureS3Client(backupConfig)
	if err != nil {
		return nil, err
	}
	bucket := client.Bucket(backupConfig.BucketName)

	foundBucketOnS3 := false

	resp, err := client.ListBuckets()
	if err != nil {
		return nil, err
	}

	for _, bucket := range resp.Buckets {
		if bucket.Name == backupConfig.BucketName {
			foundBucketOnS3 = true
			logger.Info("Bucket exists", lager.Data{"bucket": backupConfig.BucketName})
			break
		}
	}

	if !foundBucketOnS3 {
		logger.Info("Creating bucket", lager.Data{"bucket": backupConfig.BucketName})
		err = bucket.PutBucket(s3.Private)
		if err != nil {
			return nil, err
		}
	}

	return bucket, nil
}

func saveAndWaitUntilFinished(instanceDir os.FileInfo, config brokerconfig.Config) error {

	instanceID := instanceDir.Name()

	client, err := buildRedisClient(instanceID, config)
	if err != nil {
		return err
	}

	lastSaveTime, err := client.LastRDBSaveTime()
	if err != nil {
		return err
	}

	// ensure new save time can't match last save time
	time.Sleep(time.Second)

	err = client.RunBGSave()
	if err != nil {
		return err
	}

	// wait for save to complete
	timeout := config.RedisConfiguration.BackupConfiguration.BGSaveTimeoutSeconds
	for i := 0; i < timeout; i++ {
		saveTime, err := client.LastRDBSaveTime()
		if err != nil {
			return err
		}

		if saveTime > lastSaveTime {
			return nil
		}

		time.Sleep(time.Second)
	}

	return errors.New("Timed out waiting for background save to complete")
}

func buildRedisClient(instanceID string, config brokerconfig.Config) (*client.Client, error) {

	localRepo := redis.LocalRepository{RedisConf: config.RedisConfiguration}
	instance, err := localRepo.FindByID(instanceID)
	if err != nil {
		return nil, err
	}

	instanceConf, err := redisconf.Load(localRepo.InstanceConfigPath(instanceID))
	if err != nil {
		return nil, err
	}

	return client.Connect(instance.Host, uint(instance.Port), instance.Password, instanceConf)
}

func configureS3Client(backupConfig brokerconfig.BackupConfiguration) (*s3.S3, error) {
	// Warning! Fake S3 server does not appear to honour region constraints or authentication.
	// The fact that this uses the configured region and authentication is untested.
	region, err := getRegion(backupConfig)
	if err != nil {
		return nil, err
	}

	auth := aws.Auth{
		AccessKey: backupConfig.AccessKeyId,
		SecretKey: backupConfig.SecretAccessKey,
	}

	return s3.New(auth, region), nil
}

func getRegion(backupConfig brokerconfig.BackupConfiguration) (aws.Region, error) {
	for _, region := range aws.Regions {
		if region.S3Endpoint == backupConfig.EndpointUrl {
			return region, nil
		}
	}

	if backupConfig.S3Region != "" {
		return aws.Region{
			Name:                 backupConfig.S3Region,
			S3Endpoint:           backupConfig.EndpointUrl,
			S3LocationConstraint: true,
		}, nil
	}

	return aws.Region{}, S3RegionNotFoundErr
}

func configPath() string {
	brokerConfigYamlPath := os.Getenv("BROKER_CONFIG_PATH")
	if brokerConfigYamlPath == "" {
		panic("BROKER_CONFIG_PATH not set")
	}
	return brokerConfigYamlPath
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package codegen

import (
	"bytes"
	"encoding/json"
	"io"
	"os"
	"path/filepath"
	"strings"
	tmpl "text/template"

	"github.com/pkg/errors"
	"github.com/uber/zanzibar/codegen/template_bundle"
)

// AssetProvider provides access to template assets
type AssetProvider interface {
	// AssetNames returns a list of named assets available
	AssetNames() []string
	// Asset returns the bytes for a provided asset name
	Asset(string) ([]byte, error)
}

type defaultAssetCollection struct{}

func (*defaultAssetCollection) AssetNames() []string {
	return templates.AssetNames()
}

func (*defaultAssetCollection) Asset(assetName string) ([]byte, error) {
	return templates.Asset(assetName)
}

var funcMap = tmpl.FuncMap{
	"lower":         strings.ToLower,
	"title":         strings.Title,
	"fullTypeName":  fullTypeName,
	"camel":         camelCase,
	"split":         strings.Split,
	"dec":           decrement,
	"basePath":      filepath.Base,
	"pascal":        pascalCase,
	"jsonMarshal":   jsonMarshal,
	"isPointerType": isPointerType,
	"unref":         unref,
}

func fullTypeName(typeName, packageName string) string {
	if typeName == "" || strings.Contains(typeName, ".") {
		return typeName
	}
	return packageName + "." + typeName
}

func decrement(num int) int {
	return num - 1
}

func jsonMarshal(jsonObj map[string]interface{}) (string, error) {
	str, err := json.Marshal(jsonObj)
	return string(str), err
}

func isPointerType(t string) bool {
	return strings.HasPrefix(t, "*")
}

func unref(t string) string {
	if strings.HasPrefix(t, "*") {
		return strings.TrimLeft(t, "*")
	}
	return t
}

// Template generates code for edge gateway clients and edgegateway endpoints.
type Template struct {
	template *tmpl.Template
}

// NewDefaultTemplate creates a bundle of templates.
func NewDefaultTemplate() (*Template, error) {
	return NewTemplate(&defaultAssetCollection{})
}

// NewTemplate returns a template helper for the provided asset collection
func NewTemplate(
	assetProvider AssetProvider,
) (*Template, error) {
	t := tmpl.New("main").Funcs(funcMap)
	for _, file := range assetProvider.AssetNames() {
		fileContent, err := assetProvider.Asset(file)
		if err != nil {
			return nil, errors.Wrapf(
				err,
				"Could not read bin data for template %s",
				file,
			)
		}
		if _, err := t.New(file).Parse(string(fileContent)); err != nil {
			return nil, errors.Wrapf(err, "Could not parse template %s", file)
		}
	}
	return &Template{
		template: t,
	}, nil
}

func (t *Template) execTemplate(
	tplName string,
	tplData interface{},
	pkgHelper *PackageHelper,
) ([]byte, error) {
	tplBuffer := bytes.NewBuffer(nil)

	_, err := io.WriteString(tplBuffer, "// Code generated by zanzibar \n"+
		"// @generated \n \n")
	if err != nil {
		return nil, errors.Wrapf(err, "failed to write to file: %s", err)
	}

	_, err = io.WriteString(tplBuffer, pkgHelper.copyrightHeader)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to write to file: %s", err)
	}
	_, err = io.WriteString(tplBuffer, "\n\n")
	if err != nil {
		return nil, errors.Wrapf(err, "failed to write to file: %s", err)
	}

	if err := t.template.ExecuteTemplate(
		tplBuffer,
		tplName,
		tplData,
	); err != nil {
		return nil, errors.Wrapf(
			err,
			"Error generating template %s",
			tplName,
		)
	}

	return tplBuffer.Bytes(), nil
}

func openFileOrCreate(file string) (*os.File, error) {
	if _, err := os.Stat(file); os.IsNotExist(err) {
		if err := os.MkdirAll(filepath.Dir(file), os.ModePerm); err != nil {
			return nil, errors.Wrapf(
				err, "could not make directory: %s", file,
			)
		}
	}
	return os.OpenFile(file, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
}
<commit_msg>make template FuncMap configurable<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package codegen

import (
	"bytes"
	"encoding/json"
	"io"
	"os"
	"path/filepath"
	"strings"
	tmpl "text/template"

	"github.com/pkg/errors"
	"github.com/uber/zanzibar/codegen/template_bundle"
)

// AssetProvider provides access to template assets
type AssetProvider interface {
	// AssetNames returns a list of named assets available
	AssetNames() []string
	// Asset returns the bytes for a provided asset name
	Asset(string) ([]byte, error)
}

type defaultAssetCollection struct{}

func (*defaultAssetCollection) AssetNames() []string {
	return templates.AssetNames()
}

func (*defaultAssetCollection) Asset(assetName string) ([]byte, error) {
	return templates.Asset(assetName)
}

var defaultFuncMap = tmpl.FuncMap{
	"lower":         strings.ToLower,
	"title":         strings.Title,
	"fullTypeName":  fullTypeName,
	"camel":         camelCase,
	"split":         strings.Split,
	"dec":           decrement,
	"basePath":      filepath.Base,
	"pascal":        pascalCase,
	"jsonMarshal":   jsonMarshal,
	"isPointerType": isPointerType,
	"unref":         unref,
}

func fullTypeName(typeName, packageName string) string {
	if typeName == "" || strings.Contains(typeName, ".") {
		return typeName
	}
	return packageName + "." + typeName
}

func decrement(num int) int {
	return num - 1
}

func jsonMarshal(jsonObj map[string]interface{}) (string, error) {
	str, err := json.Marshal(jsonObj)
	return string(str), err
}

func isPointerType(t string) bool {
	return strings.HasPrefix(t, "*")
}

func unref(t string) string {
	if strings.HasPrefix(t, "*") {
		return strings.TrimLeft(t, "*")
	}
	return t
}

// Template generates code for edge gateway clients and edgegateway endpoints.
type Template struct {
	template *tmpl.Template
}

// NewDefaultTemplate creates a bundle of templates.
func NewDefaultTemplate() (*Template, error) {
	return NewTemplate(
		&defaultAssetCollection{},
		defaultFuncMap,
	)
}

// NewTemplate returns a template helper for the provided asset collection
func NewTemplate(
	assetProvider AssetProvider,
	functionMap tmpl.FuncMap,
) (*Template, error) {
	t := tmpl.New("main").Funcs(functionMap)
	for _, file := range assetProvider.AssetNames() {
		fileContent, err := assetProvider.Asset(file)
		if err != nil {
			return nil, errors.Wrapf(
				err,
				"Could not read bin data for template %s",
				file,
			)
		}
		if _, err := t.New(file).Parse(string(fileContent)); err != nil {
			return nil, errors.Wrapf(err, "Could not parse template %s", file)
		}
	}
	return &Template{
		template: t,
	}, nil
}

func (t *Template) execTemplate(
	tplName string,
	tplData interface{},
	pkgHelper *PackageHelper,
) ([]byte, error) {
	tplBuffer := bytes.NewBuffer(nil)

	_, err := io.WriteString(tplBuffer, "// Code generated by zanzibar \n"+
		"// @generated \n \n")
	if err != nil {
		return nil, errors.Wrapf(err, "failed to write to file: %s", err)
	}

	_, err = io.WriteString(tplBuffer, pkgHelper.copyrightHeader)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to write to file: %s", err)
	}
	_, err = io.WriteString(tplBuffer, "\n\n")
	if err != nil {
		return nil, errors.Wrapf(err, "failed to write to file: %s", err)
	}

	if err := t.template.ExecuteTemplate(
		tplBuffer,
		tplName,
		tplData,
	); err != nil {
		return nil, errors.Wrapf(
			err,
			"Error generating template %s",
			tplName,
		)
	}

	return tplBuffer.Bytes(), nil
}

func openFileOrCreate(file string) (*os.File, error) {
	if _, err := os.Stat(file); os.IsNotExist(err) {
		if err := os.MkdirAll(filepath.Dir(file), os.ModePerm); err != nil {
			return nil, errors.Wrapf(
				err, "could not make directory: %s", file,
			)
		}
	}
	return os.OpenFile(file, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"database/sql"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/user"
	"strconv"
	"strings"

	"github.com/phicode/blkidx/fs"

	. "github.com/phicode/blkidx"

	_ "github.com/mattn/go-sqlite3"
)

var (
	flagDb          *string
	flagConcurrency = flag.Int("c", 1, "concurrency")
	logger          = log.New(os.Stderr, "", log.Ldate|log.Ltime|log.Lmicroseconds)
)

func init() {
	var db string
	if user, err := user.Current(); err == nil {
		db = user.HomeDir + string(os.PathSeparator) + ".blkidx.sqlite3"
	}
	flagDb = flag.String("db", db, "sqlite database file to store")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `
%s [options] command <path>

commands:

  index [path...]            add or update files to the index.

  remove [path...]           remove files from the index.

  remove-missing [path...]   remove files from the index which
                             are also missing on the filesystem.

  list                       list all files that are currently in the index.

  list-missing [path...]     list only files that are in the index
                             but not on the filesystem.

  dups [path...]             show all files in the index which
                             have the same checksums.

  rm-dups [path...]          interactive duplicate removal.

options:
`, os.Args[0])
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()
	args := flag.Args()
	if *flagDb == "" || len(args) == 0 {
		errUsage()
	}
	found, err := run(args, *flagDb)
	if !found {
		errUsage()
	}
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(2)
	}
}

func errUsage() {
	flag.Usage()
	os.Exit(1)
}

func run(args []string, dbUrl string) (found bool, err error) {
	idx, dbCloser, err := openDbIndex(dbUrl)
	if err != nil {
		return true, fmt.Errorf("failed to open the sqlite3 database: %v", err)
	}
	defer dbCloser.Close()

	var paths fs.Paths
	if len(args) == 1 {
		paths, err = fs.WorkingDirectory()
	} else {
		paths, err = fs.NewPaths(args[1:]...)
	}
	if err != nil {
		return true, err
	}

	switch args[0] {
	case "index":
		index(idx, paths)

	case "remove":
		err = remove(idx, paths, nil)

	case "remove-missing":
		removeMissing(idx, paths)

	case "list":
		err = list(idx)

	case "list-missing":
		err = listMissing(idx, paths)

	case "dups":
		err = dups(idx, paths, false)

	case "rm-dups":
		err = dups(idx, paths, true)

	default:
		return false, nil
	}
	return true, err
}

func openDbIndex(dbUrl string) (Index, io.Closer, error) {
	// TODO: doesn'nt work, see comment below
	//dbUrl := "file:" + *flagDb + "?cache=shared&mode=rwc"
	db, err := sql.Open("sqlite3", dbUrl)
	if err != nil {
		return nil, nil, err
	}
	idx, err := NewSqlIndex(db)
	if err != nil {
		db.Close()
		return nil, nil, err
	}
	// TODO: get sqlite locking/serialization right so that we do not have to
	// wrap access the database through a single thread
	return &LockedIndex{Backend: idx}, db, nil
}

func index(idx Index, paths fs.Paths) {
	var indexer = &Indexer{
		Index:       idx,
		Log:         logger,
		Concurrency: *flagConcurrency,
	}

	indexer.IndexAll(fs.WalkFiles(paths))
}

//TODO: review
func remove(idx Index, paths fs.Paths, exclude fs.Paths) error {
	names, err := idx.AllNames()
	if err != nil {
		return err
	}

	for path, _ := range paths {
		var remove Names
		for _, name := range names {
			if strings.HasPrefix(name, path) {
				if _, found := exclude[name]; !found {
					remove = append(remove, name)
				}
			}
		}
		if len(remove) > 0 {
			if err := idx.Remove(remove); err != nil {
				return err
			}
		}
	}

	c, _ := idx.Count()
	fmt.Println("files removed:", len(names)-c, "remaining:", c)
	return nil
}

func removeMissing(idx Index, paths fs.Paths) {
	var exclude fs.Paths = findAllFiles(paths)
	remove(idx, paths, exclude)
}

func list(idx Index) error {
	names, err := idx.AllNames()
	if err != nil {
		return err
	}
	names.Sort()

	for _, name := range names {
		fmt.Println(name)
	}
	fmt.Fprintln(os.Stderr)
	fmt.Fprintln(os.Stderr, "files listed:", len(names))
	return nil
}

// TODO: review
func listMissing(idx Index, paths fs.Paths) error {
	names, err := getMissing(idx, paths, findAllFiles(paths))
	if err != nil {
		return err
	}
	var ns Names
	for n, _ := range names {
		ns = append(ns, n)
	}
	ns.Sort()
	for _, name := range ns {
		fmt.Println(name)
	}
	fmt.Fprintln(os.Stderr)
	fmt.Fprintln(os.Stderr, "files missing:", len(ns))
	return nil
}

// TODO: review
func getMissing(idx Index, paths fs.Paths, present fs.Paths) (fs.Paths, error) {
	names, err := idx.AllNames()
	if err != nil {
		return nil, err
	}

	missing := make(fs.Paths)
	for path, _ := range paths {
		for _, name := range names {
			if strings.HasPrefix(name, path) {
				if _, found := present[name]; !found {
					missing[name] = struct{}{}
				}
			}
		}
	}

	return missing, nil
}

func dups(idx Index, paths fs.Paths, rm bool) error {
	equalBlobs, err := idx.FindEqualHashes()
	if err != nil {
		return fmt.Errorf("find duplicates failed: %v", err)
	}
	equalBlobs = reduceEqualBlobs(equalBlobs, findAllFiles(paths))
	if len(equalBlobs) == 0 {
		fmt.Println("no duplicates found")
		return nil
	}

	var savings int64
	separator := strings.Repeat("-", 80)
	for _, equal := range equalBlobs {
		fmt.Println(separator)
		for i, name := range equal.Names {
			fmt.Printf("%d - %s\n", (i + 1), name)
		}

		if rm {
			n, err := askRemove(idx, equal)
			if err != nil {
				return err
			}
			savings += equal.Size * int64(n)
		} else {
			savings += (equal.Size * (int64(len(equal.Names) - 1)))
		}
	}
	fmt.Fprintln(os.Stderr)
	if rm {
		fmt.Fprintln(os.Stderr, "removed", sizePretty(savings))
	} else {
		fmt.Fprintln(os.Stderr, "removing all duplicates would save", sizePretty(savings))
	}
	return nil
}

func askRemove(idx Index, equal EqualBlobs) (int, error) {
	r := bufio.NewReader(os.Stdin)

	fmt.Println(`enter space-separated file indexes to delete or enter to delete-nothing
!!! this really deleted the file !!!`)
	indexes, err := readIntFieldsLine(r, -1)
	if err != nil {
		return 0, err
	}
	if len(indexes) == 0 {
		return 0, nil
	}
	for _, index := range indexes {
		if index >= 0 && index < len(equal.Names) {
			file := equal.Names[index]
			fmt.Println("deleting", file)
			if err := os.Remove(file); err != nil {
				return 0, err // TODO: still report how much was already saved
			}
			var deleted Names = Names{file}
			if err := idx.Remove(deleted); err != nil {
				return 0, fmt.Errorf("file removed but still in index due do: %v", err) // TODO: same as above
			}
		}
	}

	return len(indexes), nil
}

func readIntFieldsLine(r *bufio.Reader, offset int) ([]int, error) {
	line, err := r.ReadString('\n')
	if err != nil {
		return nil, err
	}
	if strings.TrimSpace(line) == "" {
		return nil, nil
	}
	parts := strings.Fields(line)
	var indexes []int
	for _, p := range parts {
		index, err := strconv.Atoi(p)
		if err != nil {
			return nil, err
		}
		indexes = append(indexes, index+offset)
	}
	return indexes, nil
}

func findAllFiles(paths fs.Paths) fs.Paths {
	c := fs.WalkFiles(paths)
	return fs.AggregateLogErrors(c, logger)
}

func reduceEqualBlobs(ebs []EqualBlobs, filesInPaths fs.Paths) []EqualBlobs {
	var rv []EqualBlobs
	for _, eb := range ebs {
		if eb.ContainsAnyName(filesInPaths) {
			rv = append(rv, eb)
		}
	}
	return rv
}

func sizePretty(s int64) string {
	var o order = orders[0]
	if s < o.v {
		return fmt.Sprintf("%d bytes", s)
	}
	for _, x := range orders {
		if s < x.v {
			break
		}
		o = x
	}
	return fmt.Sprintf("%.3f %s", float64(s)/float64(o.v), o.s)
}

type order struct {
	v int64
	s string
}

var orders = []order{
	{1 << 10, "KiB"},
	{1 << 20, "MiB"},
	{1 << 30, "GiB"},
	{1 << 40, "TiB"},
	{1 << 50, "PiB"},
	{1 << 60, "EiB"},
}
<commit_msg>function naming<commit_after>package main

import (
	"bufio"
	"database/sql"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/user"
	"strconv"
	"strings"

	"github.com/phicode/blkidx/fs"

	. "github.com/phicode/blkidx"

	_ "github.com/mattn/go-sqlite3"
)

var (
	flagDb          *string
	flagConcurrency = flag.Int("c", 1, "concurrency")
	logger          = log.New(os.Stderr, "", log.Ldate|log.Ltime|log.Lmicroseconds)
)

func init() {
	var db string
	if user, err := user.Current(); err == nil {
		db = user.HomeDir + string(os.PathSeparator) + ".blkidx.sqlite3"
	}
	flagDb = flag.String("db", db, "sqlite database file to store")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `
%s [options] command <path>

commands:

  index [path...]            add or update files to the index.

  remove [path...]           remove files from the index.

  remove-missing [path...]   remove files from the index which
                             are also missing on the filesystem.

  list                       list all files that are currently in the index.

  list-missing [path...]     list only files that are in the index
                             but not on the filesystem.

  dups [path...]             show all files in the index which
                             have the same checksums.

  rm-dups [path...]          interactive duplicate removal.

options:
`, os.Args[0])
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()
	args := flag.Args()
	if *flagDb == "" || len(args) == 0 {
		errUsage()
	}
	found, err := run(args, *flagDb)
	if !found {
		errUsage()
	}
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(2)
	}
}

func errUsage() {
	flag.Usage()
	os.Exit(1)
}

func run(args []string, dbUrl string) (found bool, err error) {
	idx, dbCloser, err := openDbIndex(dbUrl)
	if err != nil {
		return true, fmt.Errorf("failed to open the sqlite3 database: %v", err)
	}
	defer dbCloser.Close()

	var paths fs.Paths
	if len(args) == 1 {
		paths, err = fs.WorkingDirectory()
	} else {
		paths, err = fs.NewPaths(args[1:]...)
	}
	if err != nil {
		return true, err
	}

	switch args[0] {
	case "index":
		index(idx, paths)

	case "remove":
		err = remove(idx, paths, nil)

	case "remove-missing":
		removeMissing(idx, paths)

	case "list":
		err = list(idx)

	case "list-missing":
		err = listMissing(idx, paths)

	case "dups":
		err = dups(idx, paths, false)

	case "rm-dups":
		err = dups(idx, paths, true)

	default:
		return false, nil
	}
	return true, err
}

func openDbIndex(dbUrl string) (Index, io.Closer, error) {
	// TODO: doesn'nt work, see comment below
	//dbUrl := "file:" + *flagDb + "?cache=shared&mode=rwc"
	db, err := sql.Open("sqlite3", dbUrl)
	if err != nil {
		return nil, nil, err
	}
	idx, err := NewSqlIndex(db)
	if err != nil {
		db.Close()
		return nil, nil, err
	}
	// TODO: get sqlite locking/serialization right so that we do not have to
	// wrap access the database through a single thread
	return &LockedIndex{Backend: idx}, db, nil
}

func index(idx Index, paths fs.Paths) {
	var indexer = &Indexer{
		Index:       idx,
		Log:         logger,
		Concurrency: *flagConcurrency,
	}

	indexer.IndexAll(fs.WalkFiles(paths))
}

//TODO: review
func remove(idx Index, paths fs.Paths, exclude fs.Paths) error {
	names, err := idx.AllNames()
	if err != nil {
		return err
	}

	for path, _ := range paths {
		var remove Names
		for _, name := range names {
			if strings.HasPrefix(name, path) {
				if _, found := exclude[name]; !found {
					remove = append(remove, name)
				}
			}
		}
		if len(remove) > 0 {
			if err := idx.Remove(remove); err != nil {
				return err
			}
		}
	}

	c, _ := idx.Count()
	fmt.Println("files removed:", len(names)-c, "remaining:", c)
	return nil
}

func removeMissing(idx Index, paths fs.Paths) {
	var exclude fs.Paths = findAllFiles(paths)
	remove(idx, paths, exclude)
}

func list(idx Index) error {
	names, err := idx.AllNames()
	if err != nil {
		return err
	}
	names.Sort()

	for _, name := range names {
		fmt.Println(name)
	}
	fmt.Fprintln(os.Stderr)
	fmt.Fprintln(os.Stderr, "files listed:", len(names))
	return nil
}

// TODO: review
func listMissing(idx Index, paths fs.Paths) error {
	names, err := getMissing(idx, paths, findAllFiles(paths))
	if err != nil {
		return err
	}
	var ns Names
	for n, _ := range names {
		ns = append(ns, n)
	}
	ns.Sort()
	for _, name := range ns {
		fmt.Println(name)
	}
	fmt.Fprintln(os.Stderr)
	fmt.Fprintln(os.Stderr, "files missing:", len(ns))
	return nil
}

// TODO: review
func getMissing(idx Index, paths fs.Paths, present fs.Paths) (fs.Paths, error) {
	names, err := idx.AllNames()
	if err != nil {
		return nil, err
	}

	missing := make(fs.Paths)
	for path, _ := range paths {
		for _, name := range names {
			if strings.HasPrefix(name, path) {
				if _, found := present[name]; !found {
					missing[name] = struct{}{}
				}
			}
		}
	}

	return missing, nil
}

func dups(idx Index, paths fs.Paths, rm bool) error {
	equalBlobs, err := idx.FindEqualHashes()
	if err != nil {
		return fmt.Errorf("find duplicates failed: %v", err)
	}
	equalBlobs = reduceEqualBlobs(equalBlobs, findAllFiles(paths))
	if len(equalBlobs) == 0 {
		fmt.Println("no duplicates found")
		return nil
	}

	var savings int64
	separator := strings.Repeat("-", 80)
	for _, equal := range equalBlobs {
		fmt.Println(separator)
		for i, name := range equal.Names {
			fmt.Printf("%d - %s\n", (i + 1), name)
		}

		if rm {
			n, err := askRemove(idx, equal)
			if err != nil {
				return err
			}
			savings += equal.Size * int64(n)
		} else {
			savings += (equal.Size * (int64(len(equal.Names) - 1)))
		}
	}
	fmt.Fprintln(os.Stderr)
	if rm {
		fmt.Fprintln(os.Stderr, "removed", formatSize(savings))
	} else {
		fmt.Fprintln(os.Stderr, "removing all duplicates would save", formatSize(savings))
	}
	return nil
}

func askRemove(idx Index, equal EqualBlobs) (int, error) {
	r := bufio.NewReader(os.Stdin)

	fmt.Println(`enter space-separated file indexes to delete or enter to delete-nothing
!!! this really deleted the file !!!`)
	indexes, err := readIntFieldsLine(r, -1)
	if err != nil {
		return 0, err
	}
	if len(indexes) == 0 {
		return 0, nil
	}
	for _, index := range indexes {
		if index >= 0 && index < len(equal.Names) {
			file := equal.Names[index]
			fmt.Println("deleting", file)
			if err := os.Remove(file); err != nil {
				return 0, err // TODO: still report how much was already saved
			}
			var deleted Names = Names{file}
			if err := idx.Remove(deleted); err != nil {
				return 0, fmt.Errorf("file removed but still in index due do: %v", err) // TODO: same as above
			}
		}
	}

	return len(indexes), nil
}

func readIntFieldsLine(r *bufio.Reader, offset int) ([]int, error) {
	line, err := r.ReadString('\n')
	if err != nil {
		return nil, err
	}
	if strings.TrimSpace(line) == "" {
		return nil, nil
	}
	parts := strings.Fields(line)
	var indexes []int
	for _, p := range parts {
		index, err := strconv.Atoi(p)
		if err != nil {
			return nil, err
		}
		indexes = append(indexes, index+offset)
	}
	return indexes, nil
}

func findAllFiles(paths fs.Paths) fs.Paths {
	c := fs.WalkFiles(paths)
	return fs.AggregateLogErrors(c, logger)
}

func reduceEqualBlobs(ebs []EqualBlobs, filesInPaths fs.Paths) []EqualBlobs {
	var rv []EqualBlobs
	for _, eb := range ebs {
		if eb.ContainsAnyName(filesInPaths) {
			rv = append(rv, eb)
		}
	}
	return rv
}

func formatSize(s int64) string {
	o := orders[0]
	if s < o.v {
		return fmt.Sprintf("%d bytes", s)
	}
	for _, x := range orders {
		if s < x.v {
			break
		}
		o = x
	}
	return fmt.Sprintf("%.3f %s", float64(s)/float64(o.v), o.s)
}

type order struct {
	v int64
	s string
}

var orders = [...]order{
	{1 << 10, "KiB"},
	{1 << 20, "MiB"},
	{1 << 30, "GiB"},
	{1 << 40, "TiB"},
	{1 << 50, "PiB"},
	{1 << 60, "EiB"},
}
<|endoftext|>


<commit_before>/*
Copyright 2013 The Camlistore Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"os"

	"camlistore.org/pkg/cmdmain"
	"camlistore.org/pkg/osutil"
)

var envMap = map[string]func() string{
	"configdir":    osutil.CamliConfigDir,
	"clientconfig": osutil.UserClientConfigPath,
	"serverconfig": osutil.UserServerConfigPath,
}

type envCmd struct{}

func init() {
	cmdmain.RegisterCommand("env", func(flags *flag.FlagSet) cmdmain.CommandRunner {
		return new(envCmd)
	})
}

func (c *envCmd) Describe() string {
	return "Return Camlistore environment information"
}

func (c *envCmd) Usage() {
	fmt.Fprintf(os.Stderr, "camtool env [key]\n")
}

func (c *envCmd) RunCommand(args []string) error {
	if len(args) == 0 {
		for k, fn := range envMap {
			fmt.Printf("%s: %s\n", k, fn())
		}
		return nil
	}
	if len(args) > 1 {
		return cmdmain.UsageError("only 0 or 1 arguments allowed")
	}
	fn := envMap[args[0]]
	if fn == nil {
		return fmt.Errorf("unknown environment key %q", args[0])
	}
	fmt.Println(fn())
	return nil
}
<commit_msg>camtool: add env camsrcroot<commit_after>/*
Copyright 2013 The Camlistore Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"

	"camlistore.org/pkg/cmdmain"
	"camlistore.org/pkg/osutil"
)

var envMap = map[string]func() string{
	"configdir":    osutil.CamliConfigDir,
	"clientconfig": osutil.UserClientConfigPath,
	"serverconfig": osutil.UserServerConfigPath,
	"camsrcroot":   srcRoot,
}

type envCmd struct{}

func init() {
	cmdmain.RegisterCommand("env", func(flags *flag.FlagSet) cmdmain.CommandRunner {
		return new(envCmd)
	})
}

func (c *envCmd) Describe() string {
	return "Return Camlistore environment information"
}

func (c *envCmd) Usage() {
	fmt.Fprintf(os.Stderr, "camtool env [key]\n")
}

func (c *envCmd) RunCommand(args []string) error {
	if len(args) == 0 {
		for k, fn := range envMap {
			fmt.Printf("%s: %s\n", k, fn())
		}
		return nil
	}
	if len(args) > 1 {
		return cmdmain.UsageError("only 0 or 1 arguments allowed")
	}
	fn := envMap[args[0]]
	if fn == nil {
		return fmt.Errorf("unknown environment key %q", args[0])
	}
	fmt.Println(fn())
	return nil
}

func srcRoot() string {
	for _, dir := range filepath.SplitList(os.Getenv("GOPATH")) {
		if d := filepath.Join(dir, "src", "camlistore.org"); osutil.DirExists(d) {
			return d
		}
	}
	return ""
}
<|endoftext|>