<commit_before>package ns1

import (
	"errors"
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/hashicorp/terraform/helper/schema"

	"github.com/mitchellh/hashstructure"
	ns1 "gopkg.in/ns1/ns1-go.v2/rest"
	"gopkg.in/ns1/ns1-go.v2/rest/model/data"
	"gopkg.in/ns1/ns1-go.v2/rest/model/dns"
	"gopkg.in/ns1/ns1-go.v2/rest/model/filter"
)

var recordTypeStringEnum *StringEnum = NewStringEnum([]string{
	"A",
	"AAAA",
	"ALIAS",
	"AFSDB",
	"CNAME",
	"DNAME",
	"HINFO",
	"MX",
	"NAPTR",
	"NS",
	"PTR",
	"RP",
	"SPF",
	"SRV",
	"TXT",
})

func recordResource() *schema.Resource {
	return &schema.Resource{
		Schema: map[string]*schema.Schema{
			// Required
			"zone": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"domain": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"type": &schema.Schema{
				Type:         schema.TypeString,
				Required:     true,
				ForceNew:     true,
				ValidateFunc: recordTypeStringEnum.ValidateFunc,
			},
			// Optional
			"ttl": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
				Computed: true,
			},
			// "meta": metaSchema,
			"link": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"use_client_subnet": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			},
			"answers": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"answer": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
						},
						"region": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
						},
						// "meta": metaSchema,
					},
				},
				Set: genericHasher,
			},
			"regions": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"name": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},
						// "meta": metaSchema,
					},
				},
				Set: genericHasher,
			},
			"filters": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"filter": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},
						"disabled": &schema.Schema{
							Type:     schema.TypeBool,
							Optional: true,
						},
						"config": &schema.Schema{
							Type:     schema.TypeMap,
							Optional: true,
						},
					},
				},
			},
			// Computed
			"id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
		Create:   RecordCreate,
		Read:     RecordRead,
		Update:   RecordUpdate,
		Delete:   RecordDelete,
		Importer: &schema.ResourceImporter{State: RecordStateFunc},
	}
}

func genericHasher(v interface{}) int {
	hash, err := hashstructure.Hash(v, nil)
	if err != nil {
		panic(fmt.Sprintf("error computing hash code for %#v: %s", v, err.Error()))
	}
	return int(hash)
}

func recordToResourceData(d *schema.ResourceData, r *dns.Record) error {
	d.SetId(r.ID)
	d.Set("domain", r.Domain)
	d.Set("zone", r.Zone)
	d.Set("type", r.Type)
	d.Set("ttl", r.TTL)
	if r.Link != "" {
		d.Set("link", r.Link)
	}
	// if r.Meta != nil {
	// 	d.State()
	// 	t := metaStructToDynamic(r.Meta)
	// 	d.Set("meta", t)
	// }
	if len(r.Filters) > 0 {
		filters := make([]map[string]interface{}, len(r.Filters))
		for i, f := range r.Filters {
			m := make(map[string]interface{})
			m["filter"] = f.Type
			if f.Disabled {
				m["disabled"] = true
			}
			if f.Config != nil {
				m["config"] = f.Config
			}
			filters[i] = m
		}
		d.Set("filters", filters)
	}
	if len(r.Answers) > 0 {
		ans := &schema.Set{
			F: genericHasher,
		}
		log.Printf("Got back from ns1 answers: %+v", r.Answers)
		for _, answer := range r.Answers {
			ans.Add(answerToMap(*answer))
		}
		log.Printf("Setting answers %+v", ans)
		err := d.Set("answers", ans)
		if err != nil {
			return fmt.Errorf("[DEBUG] Error setting answers for: %s, error: %#v", r.Domain, err)
		}
	}
	if len(r.Regions) > 0 {
		regions := make([]map[string]interface{}, 0, len(r.Regions))
		for regionName, _ := range r.Regions {
			newRegion := make(map[string]interface{})
			newRegion["name"] = regionName
			// newRegion["meta"] = metaStructToDynamic(&region.Meta)
			regions = append(regions, newRegion)
		}
		log.Printf("Setting regions %+v", regions)
		err := d.Set("regions", regions)
		if err != nil {
			return fmt.Errorf("[DEBUG] Error setting regions for: %s, error: %#v", r.Domain, err)
		}
	}
	return nil
}

func answerToMap(a dns.Answer) map[string]interface{} {
	m := make(map[string]interface{})
	m["answer"] = strings.Join(a.Rdata, " ")
	if a.RegionName != "" {
		m["region"] = a.RegionName
	}
	// if a.Meta != nil {
	// 	m["meta"] = metaStructToDynamic(a.Meta)
	// }
	return m
}

func btoi(b bool) int {
	if b {
		return 1
	}
	return 0
}

func resourceDataToRecord(r *dns.Record, d *schema.ResourceData) error {
	r.ID = d.Id()
	if answers := d.Get("answers").(*schema.Set); answers.Len() > 0 {
		al := make([]*dns.Answer, answers.Len())
		for i, answerRaw := range answers.List() {
			answer := answerRaw.(map[string]interface{})
			var a *dns.Answer
			v := answer["answer"].(string)
			switch d.Get("type") {
			case "TXT":
				a = dns.NewTXTAnswer(v)
			default:
				a = dns.NewAnswer(strings.Split(v, " "))
			}
			if v, ok := answer["region"]; ok {
				a.RegionName = v.(string)
			}

			// if v, ok := answer["meta"]; ok {
			// 	metaDynamicToStruct(a.Meta, v)
			// }
			al[i] = a
		}
		r.Answers = al
		if _, ok := d.GetOk("link"); ok {
			return errors.New("Cannot have both link and answers in a record")
		}
	}
	if v, ok := d.GetOk("ttl"); ok {
		r.TTL = v.(int)
	}
	if v, ok := d.GetOk("link"); ok {
		r.LinkTo(v.(string))
	}
	// if v, ok := d.GetOk("meta"); ok {
	// 	metaDynamicToStruct(r.Meta, v)
	// }
	useClientSubnetVal := d.Get("use_client_subnet").(bool)
	if v := strconv.FormatBool(useClientSubnetVal); v != "" {
		r.UseClientSubnet = &useClientSubnetVal
	}

	if rawFilters := d.Get("filters").([]interface{}); len(rawFilters) > 0 {
		f := make([]*filter.Filter, len(rawFilters))
		for i, filterRaw := range rawFilters {
			fi := filterRaw.(map[string]interface{})
			config := make(map[string]interface{})
			filter := filter.Filter{
				Type:   fi["filter"].(string),
				Config: config,
			}
			if disabled, ok := fi["disabled"]; ok {
				filter.Disabled = disabled.(bool)
			}
			if rawConfig, ok := fi["config"]; ok {
				for k, v := range rawConfig.(map[string]interface{}) {
					if i, err := strconv.Atoi(v.(string)); err == nil {
						filter.Config[k] = i
					} else {
						filter.Config[k] = v
					}
				}
			}
			f[i] = &filter
		}
		r.Filters = f
	}
	if regions := d.Get("regions").(*schema.Set); regions.Len() > 0 {
		for _, regionRaw := range regions.List() {
			region := regionRaw.(map[string]interface{})
			ns1R := data.Region{
				Meta: data.Meta{},
			}
			// if v, ok := region["meta"]; ok {
			// 	metaDynamicToStruct(&ns1R.Meta, v)
			// }

			r.Regions[region["name"].(string)] = ns1R
		}
	}
	return nil
}

// RecordCreate creates DNS record in ns1
func RecordCreate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*ns1.Client)
	r := dns.NewRecord(d.Get("zone").(string), d.Get("domain").(string), d.Get("type").(string))
	if err := resourceDataToRecord(r, d); err != nil {
		return err
	}
	if _, err := client.Records.Create(r); err != nil {
		return err
	}
	return recordToResourceData(d, r)
}

// RecordRead reads the DNS record from ns1
func RecordRead(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*ns1.Client)

	r, _, err := client.Records.Get(d.Get("zone").(string), d.Get("domain").(string), d.Get("type").(string))
	if err != nil {
		return err
	}

	return recordToResourceData(d, r)
}

// RecordDelete deltes the DNS record from ns1
func RecordDelete(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*ns1.Client)
	_, err := client.Records.Delete(d.Get("zone").(string), d.Get("domain").(string), d.Get("type").(string))
	d.SetId("")
	return err
}

// RecordUpdate updates the given dns record in ns1
func RecordUpdate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*ns1.Client)
	r := dns.NewRecord(d.Get("zone").(string), d.Get("domain").(string), d.Get("type").(string))
	if err := resourceDataToRecord(r, d); err != nil {
		return err
	}
	if _, err := client.Records.Update(r); err != nil {
		return err
	}
	return recordToResourceData(d, r)
}

func RecordStateFunc(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) {
	parts := strings.Split(d.Id(), "/")
	if len(parts) != 3 {
		return nil, fmt.Errorf("Invalid record specifier.  Expecting 2 slashes (\"zone/domain/type\"), got %d.", len(parts)-1)
	}

	d.Set("zone", parts[0])
	d.Set("domain", parts[1])
	d.Set("type", parts[2])

	return []*schema.ResourceData{d}, nil
}
<commit_msg>provider/ns1/record: Fix "use_client_subnet". (#11368)<commit_after>package ns1

import (
	"errors"
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/hashicorp/terraform/helper/schema"

	"github.com/mitchellh/hashstructure"
	ns1 "gopkg.in/ns1/ns1-go.v2/rest"
	"gopkg.in/ns1/ns1-go.v2/rest/model/data"
	"gopkg.in/ns1/ns1-go.v2/rest/model/dns"
	"gopkg.in/ns1/ns1-go.v2/rest/model/filter"
)

var recordTypeStringEnum *StringEnum = NewStringEnum([]string{
	"A",
	"AAAA",
	"ALIAS",
	"AFSDB",
	"CNAME",
	"DNAME",
	"HINFO",
	"MX",
	"NAPTR",
	"NS",
	"PTR",
	"RP",
	"SPF",
	"SRV",
	"TXT",
})

func recordResource() *schema.Resource {
	return &schema.Resource{
		Schema: map[string]*schema.Schema{
			// Required
			"zone": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"domain": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"type": &schema.Schema{
				Type:         schema.TypeString,
				Required:     true,
				ForceNew:     true,
				ValidateFunc: recordTypeStringEnum.ValidateFunc,
			},
			// Optional
			"ttl": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
				Computed: true,
			},
			// "meta": metaSchema,
			"link": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"use_client_subnet": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			},
			"answers": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"answer": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
						},
						"region": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
						},
						// "meta": metaSchema,
					},
				},
				Set: genericHasher,
			},
			"regions": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"name": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},
						// "meta": metaSchema,
					},
				},
				Set: genericHasher,
			},
			"filters": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"filter": &schema.Schema{
							Type:     schema.TypeString,
							Required: true,
						},
						"disabled": &schema.Schema{
							Type:     schema.TypeBool,
							Optional: true,
						},
						"config": &schema.Schema{
							Type:     schema.TypeMap,
							Optional: true,
						},
					},
				},
			},
			// Computed
			"id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
		Create:   RecordCreate,
		Read:     RecordRead,
		Update:   RecordUpdate,
		Delete:   RecordDelete,
		Importer: &schema.ResourceImporter{State: RecordStateFunc},
	}
}

func genericHasher(v interface{}) int {
	hash, err := hashstructure.Hash(v, nil)
	if err != nil {
		panic(fmt.Sprintf("error computing hash code for %#v: %s", v, err.Error()))
	}
	return int(hash)
}

func recordToResourceData(d *schema.ResourceData, r *dns.Record) error {
	d.SetId(r.ID)
	d.Set("domain", r.Domain)
	d.Set("zone", r.Zone)
	d.Set("type", r.Type)
	d.Set("ttl", r.TTL)
	if r.Link != "" {
		d.Set("link", r.Link)
	}
	// if r.Meta != nil {
	// 	d.State()
	// 	t := metaStructToDynamic(r.Meta)
	// 	d.Set("meta", t)
	// }
	if r.UseClientSubnet != nil {
		d.Set("use_client_subnet", *r.UseClientSubnet)
	}
	if len(r.Filters) > 0 {
		filters := make([]map[string]interface{}, len(r.Filters))
		for i, f := range r.Filters {
			m := make(map[string]interface{})
			m["filter"] = f.Type
			if f.Disabled {
				m["disabled"] = true
			}
			if f.Config != nil {
				m["config"] = f.Config
			}
			filters[i] = m
		}
		d.Set("filters", filters)
	}
	if len(r.Answers) > 0 {
		ans := &schema.Set{
			F: genericHasher,
		}
		log.Printf("Got back from ns1 answers: %+v", r.Answers)
		for _, answer := range r.Answers {
			ans.Add(answerToMap(*answer))
		}
		log.Printf("Setting answers %+v", ans)
		err := d.Set("answers", ans)
		if err != nil {
			return fmt.Errorf("[DEBUG] Error setting answers for: %s, error: %#v", r.Domain, err)
		}
	}
	if len(r.Regions) > 0 {
		regions := make([]map[string]interface{}, 0, len(r.Regions))
		for regionName, _ := range r.Regions {
			newRegion := make(map[string]interface{})
			newRegion["name"] = regionName
			// newRegion["meta"] = metaStructToDynamic(&region.Meta)
			regions = append(regions, newRegion)
		}
		log.Printf("Setting regions %+v", regions)
		err := d.Set("regions", regions)
		if err != nil {
			return fmt.Errorf("[DEBUG] Error setting regions for: %s, error: %#v", r.Domain, err)
		}
	}
	return nil
}

func answerToMap(a dns.Answer) map[string]interface{} {
	m := make(map[string]interface{})
	m["answer"] = strings.Join(a.Rdata, " ")
	if a.RegionName != "" {
		m["region"] = a.RegionName
	}
	// if a.Meta != nil {
	// 	m["meta"] = metaStructToDynamic(a.Meta)
	// }
	return m
}

func btoi(b bool) int {
	if b {
		return 1
	}
	return 0
}

func resourceDataToRecord(r *dns.Record, d *schema.ResourceData) error {
	r.ID = d.Id()
	if answers := d.Get("answers").(*schema.Set); answers.Len() > 0 {
		al := make([]*dns.Answer, answers.Len())
		for i, answerRaw := range answers.List() {
			answer := answerRaw.(map[string]interface{})
			var a *dns.Answer
			v := answer["answer"].(string)
			switch d.Get("type") {
			case "TXT":
				a = dns.NewTXTAnswer(v)
			default:
				a = dns.NewAnswer(strings.Split(v, " "))
			}
			if v, ok := answer["region"]; ok {
				a.RegionName = v.(string)
			}

			// if v, ok := answer["meta"]; ok {
			// 	metaDynamicToStruct(a.Meta, v)
			// }
			al[i] = a
		}
		r.Answers = al
		if _, ok := d.GetOk("link"); ok {
			return errors.New("Cannot have both link and answers in a record")
		}
	}
	if v, ok := d.GetOk("ttl"); ok {
		r.TTL = v.(int)
	}
	if v, ok := d.GetOk("link"); ok {
		r.LinkTo(v.(string))
	}
	// if v, ok := d.GetOk("meta"); ok {
	// 	metaDynamicToStruct(r.Meta, v)
	// }
	if v, ok := d.GetOk("use_client_subnet"); ok {
		copy := v.(bool)
		r.UseClientSubnet = &copy
	}

	if rawFilters := d.Get("filters").([]interface{}); len(rawFilters) > 0 {
		f := make([]*filter.Filter, len(rawFilters))
		for i, filterRaw := range rawFilters {
			fi := filterRaw.(map[string]interface{})
			config := make(map[string]interface{})
			filter := filter.Filter{
				Type:   fi["filter"].(string),
				Config: config,
			}
			if disabled, ok := fi["disabled"]; ok {
				filter.Disabled = disabled.(bool)
			}
			if rawConfig, ok := fi["config"]; ok {
				for k, v := range rawConfig.(map[string]interface{}) {
					if i, err := strconv.Atoi(v.(string)); err == nil {
						filter.Config[k] = i
					} else {
						filter.Config[k] = v
					}
				}
			}
			f[i] = &filter
		}
		r.Filters = f
	}
	if regions := d.Get("regions").(*schema.Set); regions.Len() > 0 {
		for _, regionRaw := range regions.List() {
			region := regionRaw.(map[string]interface{})
			ns1R := data.Region{
				Meta: data.Meta{},
			}
			// if v, ok := region["meta"]; ok {
			// 	metaDynamicToStruct(&ns1R.Meta, v)
			// }

			r.Regions[region["name"].(string)] = ns1R
		}
	}
	return nil
}

// RecordCreate creates DNS record in ns1
func RecordCreate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*ns1.Client)
	r := dns.NewRecord(d.Get("zone").(string), d.Get("domain").(string), d.Get("type").(string))
	if err := resourceDataToRecord(r, d); err != nil {
		return err
	}
	if _, err := client.Records.Create(r); err != nil {
		return err
	}
	return recordToResourceData(d, r)
}

// RecordRead reads the DNS record from ns1
func RecordRead(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*ns1.Client)

	r, _, err := client.Records.Get(d.Get("zone").(string), d.Get("domain").(string), d.Get("type").(string))
	if err != nil {
		return err
	}

	return recordToResourceData(d, r)
}

// RecordDelete deltes the DNS record from ns1
func RecordDelete(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*ns1.Client)
	_, err := client.Records.Delete(d.Get("zone").(string), d.Get("domain").(string), d.Get("type").(string))
	d.SetId("")
	return err
}

// RecordUpdate updates the given dns record in ns1
func RecordUpdate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*ns1.Client)
	r := dns.NewRecord(d.Get("zone").(string), d.Get("domain").(string), d.Get("type").(string))
	if err := resourceDataToRecord(r, d); err != nil {
		return err
	}
	if _, err := client.Records.Update(r); err != nil {
		return err
	}
	return recordToResourceData(d, r)
}

func RecordStateFunc(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) {
	parts := strings.Split(d.Id(), "/")
	if len(parts) != 3 {
		return nil, fmt.Errorf("Invalid record specifier.  Expecting 2 slashes (\"zone/domain/type\"), got %d.", len(parts)-1)
	}

	d.Set("zone", parts[0])
	d.Set("domain", parts[1])
	d.Set("type", parts[2])

	return []*schema.ResourceData{d}, nil
}
<|endoftext|>


<commit_before>package thesaurus

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"os"
	"sort"
	"strings"

	"github.com/gyuho/goling/utils"
)

// func main() {
// 	api, err := New()
// 	if err != nil {
// 		log.Fatal(err)
// 	}
// 	// map[verb:map[syn:[accredit account ascribe assign attribute bank calculate impute rely swear trust] ant:[debit]] noun:map[syn:[recognition credit entry deferred payment course credit citation cite acknowledgment reference mention quotation accomplishment accounting entry achievement annotation approval assets attainment commendation entry ledger entry notation note payment title] ant:[cash debit]]]
// 	rs, err := api.GetSynonyms("health")
// 	if err != nil {
// 		log.Fatal(err)
// 	}
// 	fmt.Println(rs)
// }

// API implements Merriam-Webster API client.
type API struct {
	key string
	*http.Client
}

// New returns a new API with default http.Client.
func New() (*API, error) {
	akey := os.Getenv("THESAURUS_KEY")
	if akey == "" {
		return nil, errors.New("no environment variable set THESAURUS_KEY")
	}
	api := API{
		key:    akey,
		Client: http.DefaultClient,
	}
	return &api, nil
}

// NewCustom returns a new API with customized http.Client.
func NewCustom(client *http.Client) (*API, error) {
	akey := os.Getenv("THESAURUS_KEY")
	if akey == "" {
		return nil, errors.New("no environment variable set THESAURUS_KEY")
	}
	api := API{
		key:    akey,
		Client: client,
	}
	return &api, nil
}

const endpoint = "http://words.bighugelabs.com/api/2/%s/%s/json"

// GetSynonyms returns the synonyms of an input word.
func (a *API) GetSynonyms(word string) ([]string, error) {
	nword := strings.TrimSpace(word)
	nword = strings.ToLower(nword)
	url := fmt.Sprintf(endpoint, a.key, nword)
	resp, err := a.Client.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	// body, err := ioutil.ReadAll(resp.Body)
	// fmt.Println(string(body))
	result := map[string]map[string][]string{}
	err = json.NewDecoder(resp.Body).Decode(&result)
	if err != nil {
		return nil, err
	}
	smap := make(map[string]bool)
	for _, val := range result {
		for k, v := range val {
			if k == "syn" {
				for _, elem := range v {
					smap[elem] = true
				}
			}
		}
	}
	temSplice := []string{}
	for key := range smap {
		words := utils.SplitToWords(strings.ToLower(key))
		temSplice = append(temSplice, words...)
	}
	found := make(map[string]bool)
	for _, elem := range temSplice {
		found[elem] = true
	}
	slice := []string{}
	for k := range found {
		slice = append(slice, k)
	}
	sort.Strings(slice)
	return slice, nil
}
<commit_msg>Update<commit_after>package thesaurus

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"os"
	"sort"
	"strings"

	"github.com/gyuho/goling/utils"
)

// func main() {
// 	api, err := New()
// 	if err != nil {
// 		log.Fatal(err)
// 	}
// 	// map[verb:map[syn:[accredit account ascribe assign attribute bank calculate impute rely swear trust] ant:[debit]] noun:map[syn:[recognition credit entry deferred payment course credit citation cite acknowledgment reference mention quotation accomplishment accounting entry achievement annotation approval assets attainment commendation entry ledger entry notation note payment title] ant:[cash debit]]]
// 	rs, err := api.Get("health")
// 	if err != nil {
// 		log.Fatal(err)
// 	}
// 	fmt.Println(rs)
// }

// API implements Merriam-Webster API client.
type API struct {
	key string
	*http.Client
}

// New returns a new API with default http.Client.
func New() (*API, error) {
	akey := os.Getenv("THESAURUS_KEY")
	if akey == "" {
		return nil, errors.New("no environment variable set THESAURUS_KEY")
	}
	api := API{
		key:    akey,
		Client: http.DefaultClient,
	}
	return &api, nil
}

// NewCustom returns a new API with customized http.Client.
func NewCustom(client *http.Client) (*API, error) {
	akey := os.Getenv("THESAURUS_KEY")
	if akey == "" {
		return nil, errors.New("no environment variable set THESAURUS_KEY")
	}
	api := API{
		key:    akey,
		Client: client,
	}
	return &api, nil
}

const endpoint = "http://words.bighugelabs.com/api/2/%s/%s/json"

// Get returns the synonyms of an input word.
func (a *API) Get(word string) ([]string, error) {
	nword := strings.TrimSpace(word)
	nword = strings.ToLower(nword)
	url := fmt.Sprintf(endpoint, a.key, nword)
	resp, err := a.Client.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	// body, err := ioutil.ReadAll(resp.Body)
	// fmt.Println(string(body))
	result := map[string]map[string][]string{}
	err = json.NewDecoder(resp.Body).Decode(&result)
	if err != nil {
		return nil, err
	}
	smap := make(map[string]bool)
	for _, val := range result {
		for k, v := range val {
			if k == "syn" {
				for _, elem := range v {
					smap[elem] = true
				}
			}
		}
	}
	temSplice := []string{}
	for key := range smap {
		words := utils.SplitToWords(strings.ToLower(key))
		temSplice = append(temSplice, words...)
	}
	found := make(map[string]bool)
	for _, elem := range temSplice {
		found[elem] = true
	}
	slice := []string{}
	for k := range found {
		slice = append(slice, k)
	}
	sort.Strings(slice)
	return slice, nil
}
<|endoftext|>


<commit_before>package awstasks

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/golang/glog"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
	"strings"
)

//go:generate fitask -type=DNSZone
type DNSZone struct {
	Name *string
	ID   *string
}

var _ fi.CompareWithID = &DNSZone{}

func (e *DNSZone) CompareWithID() *string {
	return e.Name
}

func (e *DNSZone) Find(c *fi.Context) (*DNSZone, error) {
	cloud := c.Cloud.(*awsup.AWSCloud)

	z, err := e.findExisting(cloud)
	if err != nil {
		return nil, err
	}

	if z == nil {
		return nil, nil
	}

	actual := &DNSZone{}
	actual.Name = e.Name
	actual.ID = z.Id

	if e.ID == nil {
		e.ID = actual.ID
	}

	return actual, nil
}

func (e *DNSZone) findExisting(cloud *awsup.AWSCloud) (*route53.HostedZone, error) {
	findID := ""
	if e.ID != nil {
		findID = *e.ID
	} else if e.Name != nil && !strings.Contains(*e.Name, ".") {
		// Looks like a hosted zone ID
		findID = *e.Name
	}
	if findID != "" {
		request := &route53.GetHostedZoneInput{
			Id: aws.String(findID),
		}

		response, err := cloud.Route53.GetHostedZone(request)
		if err != nil {
			if awsup.AWSErrorCode(err) == "NoSuchHostedZone" {
				if e.ID != nil {
					return nil, nil
				}
				// Otherwise continue ... maybe the name was not an id after all...
			} else {
				return nil, fmt.Errorf("error fetching DNS HostedZone %q: %v", *e.ID, err)
			}
		} else {
			return response.HostedZone, nil
		}
	}

	findName := fi.StringValue(e.Name)
	if findName == "" {
		return nil, nil
	}
	if !strings.HasSuffix(findName, ".") {
		findName += "."
	}
	request := &route53.ListHostedZonesByNameInput{
		DNSName: aws.String(findName),
	}

	response, err := cloud.Route53.ListHostedZonesByName(request)
	if err != nil {
		return nil, fmt.Errorf("error listing DNS HostedZones: %v", err)
	}

	var zones []*route53.HostedZone
	for _, zone := range response.HostedZones {
		if aws.StringValue(zone.Name) == findName {
			zones = append(zones, zone)
		}
	}
	if len(zones) == 0 {
		return nil, nil
	}
	if len(zones) != 1 {
		return nil, fmt.Errorf("found multiple hosted zones matched name %q", findName)
	}

	return zones[0], nil
}

func (e *DNSZone) Run(c *fi.Context) error {
	return fi.DefaultDeltaRunMethod(e, c)
}

func (s *DNSZone) CheckChanges(a, e, changes *DNSZone) error {
	if fi.StringValue(e.Name) == "" {
		return fi.RequiredField("Name")
	}
	return nil
}

func (_ *DNSZone) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *DNSZone) error {
	if a == nil {
		request := &route53.CreateHostedZoneInput{}
		request.Name = e.Name

		glog.V(2).Infof("Creating Route53 HostedZone with Name %q", e.Name)

		response, err := t.Cloud.Route53.CreateHostedZone(request)
		if err != nil {
			return fmt.Errorf("error creating DNS HostedZone: %v", err)
		}

		e.ID = response.HostedZone.Id
	}

	// We don't tag the zone - we expect it to be shared
	return nil
}

type terraformRoute53Zone struct {
	Name *string           `json:"name"`
	Tags map[string]string `json:"tags,omitempty"`
}

func (_ *DNSZone) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *DNSZone) error {
	cloud := t.Cloud.(*awsup.AWSCloud)

	// As a special case, we check for an existing zone
	// It is really painful to have TF create a new one...
	// (you have to reconfigure the DNS NS records)
	glog.Infof("Check for existing route53 zone to re-use with name %q", *e.Name)
	z, err := e.findExisting(cloud)
	if err != nil {
		return err
	}

	if z != nil {
		glog.Infof("Existing zone %q found; will configure TF to reuse", aws.StringValue(z.Name))

		e.ID = z.Id
		return nil
	}

	tf := &terraformRoute53Zone{
		Name: e.Name,
		//Tags:               cloud.BuildTags(e.Name, nil),
	}

	return t.RenderResource("aws_route53_zone", *e.Name, tf)
}

func (e *DNSZone) TerraformLink() *terraform.Literal {
	if e.ID != nil {
		glog.V(4).Infof("reusing existing route53 zone with id %q", *e.ID)
		return terraform.LiteralFromStringValue(*e.ID)
	}

	return terraform.LiteralSelfLink("aws_route53_zone", *e.Name)
}
<commit_msg>Fix DNS zone creation<commit_after>package awstasks

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/golang/glog"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
	"math/rand"
	"strconv"
	"strings"
)

//go:generate fitask -type=DNSZone
type DNSZone struct {
	Name *string
	ID   *string
}

var _ fi.CompareWithID = &DNSZone{}

func (e *DNSZone) CompareWithID() *string {
	return e.Name
}

func (e *DNSZone) Find(c *fi.Context) (*DNSZone, error) {
	cloud := c.Cloud.(*awsup.AWSCloud)

	z, err := e.findExisting(cloud)
	if err != nil {
		return nil, err
	}

	if z == nil {
		return nil, nil
	}

	actual := &DNSZone{}
	actual.Name = e.Name
	actual.ID = z.Id

	if e.ID == nil {
		e.ID = actual.ID
	}

	return actual, nil
}

func (e *DNSZone) findExisting(cloud *awsup.AWSCloud) (*route53.HostedZone, error) {
	findID := ""
	if e.ID != nil {
		findID = *e.ID
	} else if e.Name != nil && !strings.Contains(*e.Name, ".") {
		// Looks like a hosted zone ID
		findID = *e.Name
	}
	if findID != "" {
		request := &route53.GetHostedZoneInput{
			Id: aws.String(findID),
		}

		response, err := cloud.Route53.GetHostedZone(request)
		if err != nil {
			if awsup.AWSErrorCode(err) == "NoSuchHostedZone" {
				if e.ID != nil {
					return nil, nil
				}
				// Otherwise continue ... maybe the name was not an id after all...
			} else {
				return nil, fmt.Errorf("error fetching DNS HostedZone %q: %v", *e.ID, err)
			}
		} else {
			return response.HostedZone, nil
		}
	}

	findName := fi.StringValue(e.Name)
	if findName == "" {
		return nil, nil
	}
	if !strings.HasSuffix(findName, ".") {
		findName += "."
	}
	request := &route53.ListHostedZonesByNameInput{
		DNSName: aws.String(findName),
	}

	response, err := cloud.Route53.ListHostedZonesByName(request)
	if err != nil {
		return nil, fmt.Errorf("error listing DNS HostedZones: %v", err)
	}

	var zones []*route53.HostedZone
	for _, zone := range response.HostedZones {
		if aws.StringValue(zone.Name) == findName {
			zones = append(zones, zone)
		}
	}
	if len(zones) == 0 {
		return nil, nil
	}
	if len(zones) != 1 {
		return nil, fmt.Errorf("found multiple hosted zones matched name %q", findName)
	}

	return zones[0], nil
}

func (e *DNSZone) Run(c *fi.Context) error {
	return fi.DefaultDeltaRunMethod(e, c)
}

func (s *DNSZone) CheckChanges(a, e, changes *DNSZone) error {
	if fi.StringValue(e.Name) == "" {
		return fi.RequiredField("Name")
	}
	return nil
}

func (_ *DNSZone) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *DNSZone) error {
	if a == nil {
		request := &route53.CreateHostedZoneInput{}
		request.Name = e.Name
		nonce := rand.Int63()
		request.CallerReference = aws.String(strconv.FormatInt(nonce, 10))

		glog.V(2).Infof("Creating Route53 HostedZone with Name %q", e.Name)

		response, err := t.Cloud.Route53.CreateHostedZone(request)
		if err != nil {
			return fmt.Errorf("error creating DNS HostedZone: %v", err)
		}

		e.ID = response.HostedZone.Id
	}

	// We don't tag the zone - we expect it to be shared
	return nil
}

type terraformRoute53Zone struct {
	Name *string           `json:"name"`
	Tags map[string]string `json:"tags,omitempty"`
}

func (_ *DNSZone) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *DNSZone) error {
	cloud := t.Cloud.(*awsup.AWSCloud)

	// As a special case, we check for an existing zone
	// It is really painful to have TF create a new one...
	// (you have to reconfigure the DNS NS records)
	glog.Infof("Check for existing route53 zone to re-use with name %q", *e.Name)
	z, err := e.findExisting(cloud)
	if err != nil {
		return err
	}

	if z != nil {
		glog.Infof("Existing zone %q found; will configure TF to reuse", aws.StringValue(z.Name))

		e.ID = z.Id
		return nil
	}

	tf := &terraformRoute53Zone{
		Name: e.Name,
		//Tags:               cloud.BuildTags(e.Name, nil),
	}

	return t.RenderResource("aws_route53_zone", *e.Name, tf)
}

func (e *DNSZone) TerraformLink() *terraform.Literal {
	if e.ID != nil {
		glog.V(4).Infof("reusing existing route53 zone with id %q", *e.ID)
		return terraform.LiteralFromStringValue(*e.ID)
	}

	return terraform.LiteralSelfLink("aws_route53_zone", *e.Name)
}
<|endoftext|>


<commit_before>// Command merge-docker-save repacks output of docker save command called for
// single image to a tar stream with merged content of all image layers
//
// Usage:
//
// 	docker save image:tag | merge-docker-save > image-fs.tar
package main

import (
	"archive/tar"
	"compress/gzip"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/artyom/autoflags"
)

func main() {
	args := struct {
		File string `flag:"o,file to write output to instead of stdout"`
		Gzip bool   `flag:"gzip,compress output with gzip"`
	}{}
	autoflags.Parse(&args)
	if err := do(args.File, args.Gzip, os.Stdin); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func do(name string, gzip bool, input io.Reader) error {
	output, err := openOutput(name, gzip)
	if err != nil {
		return err
	}
	defer output.Close()
	if err := repack(output, input); err != nil {
		return err
	}
	return output.Close()
}

func repack(out io.Writer, input io.Reader) error {
	tr := tar.NewReader(input)
	tw := tar.NewWriter(out)
	layers := make(map[string]*os.File)
	var mlayers []*layerMeta
	defer func() {
		for _, f := range layers {
			f.Close()
		}
	}()
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			if err := fillSkips(layers, mlayers); err != nil {
				return err
			}
			for _, meta := range mlayers {
				f, ok := layers[meta.name]
				if !ok {
					return fmt.Errorf("manifest references unknown layer %q", meta.name)
				}
				if _, err := f.Seek(0, io.SeekStart); err != nil {
					return err
				}
				if err := copyStream(tw, tar.NewReader(f), meta.skip); err != nil {
					return err
				}
			}
			return tw.Close()
		}
		if err != nil {
			return err
		}
		if strings.HasSuffix(hdr.Name, "/layer.tar") {
			f, err := dumpStream(tr)
			if err != nil {
				return err
			}
			layers[hdr.Name] = f
			continue
		}
		if hdr.Name == "manifest.json" {
			if mlayers, err = decodeLayerList(tr); err != nil {
				return err
			}
		}
	}
}

func copyStream(tw *tar.Writer, tr *tar.Reader, skip map[string]struct{}) error {
tarLoop:
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			return nil
		}
		if err != nil {
			return err
		}
		if _, ok := skip[hdr.Name]; ok {
			continue
		}
		if strings.HasPrefix(path.Base(hdr.Name), tombstone) {
			continue
		}
		for prefix := range skip {
			if strings.HasPrefix(hdr.Name, prefix+"/") {
				continue tarLoop
			}
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		if _, err := io.Copy(tw, tr); err != nil {
			return err
		}
	}
}

func decodeLayerList(r io.Reader) ([]*layerMeta, error) {
	data := []struct {
		Layers []string
	}{}
	if err := json.NewDecoder(r).Decode(&data); err != nil {
		return nil, err
	}
	if l := len(data); l != 1 {
		return nil, fmt.Errorf("manifest.json describes %d objects, call docker save for a single image", l)
	}
	out := make([]*layerMeta, len(data[0].Layers))
	for i, name := range data[0].Layers {
		out[i] = &layerMeta{name: name}
	}
	return out, nil
}

func dumpStream(r io.Reader) (*os.File, error) {
	f, err := ioutil.TempFile("", "merge-docker-save-")
	if err != nil {
		return nil, err
	}
	os.Remove(f.Name())
	if _, err := io.Copy(f, r); err != nil {
		f.Close()
		return nil, err
	}
	return f, nil
}

func openOutput(name string, compress bool) (io.WriteCloser, error) {
	var wc io.WriteCloser = os.Stdout
	if name != "" {
		f, err := os.Create(name)
		if err != nil {
			return nil, err
		}
		wc = f
	}
	if !compress {
		return wc, nil
	}
	return &writerChain{gzip.NewWriter(wc), wc}, nil
}

type writerChain []io.WriteCloser

// Write implements io.Writer by writing to the first Writer in writerChain
func (w writerChain) Write(b []byte) (int, error) { return w[0].Write(b) }

// Close implements io.Closer by closing every Closer in a writerChain and
// returning the first captured non-nil error it encountered.
func (w writerChain) Close() error {
	var err error
	for _, c := range w {
		if err2 := c.Close(); err2 != nil && err == nil {
			err = err2
		}
	}
	return err
}

type layerMeta struct {
	name string
	skip map[string]struct{}
}

// fillSkips fills skip fields of mlayers elements from the tombstone items
// discovered in files referenced in layers map. skip fields filled in such
// a way that for each layer it holds a set of names that should be skipped when
// repacking tar stream since these items would be removed by the following
// layers.
func fillSkips(layers map[string]*os.File, mlayers []*layerMeta) error {
	for i := len(mlayers) - 1; i > 0; i-- {
		meta := mlayers[i]
		f, ok := layers[meta.name]
		if !ok {
			return fmt.Errorf("manifest references unknown layer %q", meta.name)
		}
		skips, err := findSkips(f)
		if err != nil {
			return err
		}
		if skips == nil {
			continue
		}
		for _, meta := range mlayers[:i] {
			if meta.skip == nil {
				meta.skip = make(map[string]struct{})
			}
			for _, s := range skips {
				meta.skip[s] = struct{}{}
			}
		}
	}
	return nil
}

// findSkips scans tar archive for tombstone items and returns list of
// corresponding file names.
func findSkips(f io.ReadSeeker) ([]string, error) {
	if _, err := f.Seek(0, io.SeekStart); err != nil {
		return nil, err
	}
	var skips []string
	tr := tar.NewReader(f)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			return skips, nil
		}
		if err != nil {
			return nil, err
		}
		if base := path.Base(hdr.Name); strings.HasPrefix(base, tombstone) && base != tombstone {
			skips = append(skips, path.Join(path.Dir(hdr.Name), strings.TrimPrefix(base, tombstone)))
		}
	}
}

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: docker save image:tag | %s > image-fs.tar\n", filepath.Base(os.Args[0]))
		flag.PrintDefaults()
	}
}

const tombstone = ".wh." // prefix docker uses to mark deleted files
<commit_msg>Skip files which are going to be overwritten later<commit_after>// Command merge-docker-save repacks output of docker save command called for
// single image to a tar stream with merged content of all image layers
//
// Usage:
//
// 	docker save image:tag | merge-docker-save > image-fs.tar
package main

import (
	"archive/tar"
	"compress/gzip"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/artyom/autoflags"
)

func main() {
	args := struct {
		File string `flag:"o,file to write output to instead of stdout"`
		Gzip bool   `flag:"gzip,compress output with gzip"`
	}{}
	autoflags.Parse(&args)
	if err := do(args.File, args.Gzip, os.Stdin); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func do(name string, gzip bool, input io.Reader) error {
	output, err := openOutput(name, gzip)
	if err != nil {
		return err
	}
	defer output.Close()
	if err := repack(output, input); err != nil {
		return err
	}
	return output.Close()
}

func repack(out io.Writer, input io.Reader) error {
	tr := tar.NewReader(input)
	tw := tar.NewWriter(out)
	layers := make(map[string]*os.File)
	var mlayers []*layerMeta
	defer func() {
		for _, f := range layers {
			f.Close()
		}
	}()
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			if err := fillSkips(layers, mlayers); err != nil {
				return err
			}
			for _, meta := range mlayers {
				f, ok := layers[meta.name]
				if !ok {
					return fmt.Errorf("manifest references unknown layer %q", meta.name)
				}
				if _, err := f.Seek(0, io.SeekStart); err != nil {
					return err
				}
				if err := copyStream(tw, tar.NewReader(f), meta.skip); err != nil {
					return err
				}
			}
			return tw.Close()
		}
		if err != nil {
			return err
		}
		if strings.HasSuffix(hdr.Name, "/layer.tar") {
			f, err := dumpStream(tr)
			if err != nil {
				return err
			}
			layers[hdr.Name] = f
			continue
		}
		if hdr.Name == "manifest.json" {
			if mlayers, err = decodeLayerList(tr); err != nil {
				return err
			}
		}
	}
}

func copyStream(tw *tar.Writer, tr *tar.Reader, skip map[string]struct{}) error {
tarLoop:
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			return nil
		}
		if err != nil {
			return err
		}
		if _, ok := skip[hdr.Name]; ok {
			continue
		}
		if strings.HasPrefix(path.Base(hdr.Name), tombstone) {
			continue
		}
		for prefix := range skip {
			if strings.HasPrefix(hdr.Name, prefix+"/") {
				continue tarLoop
			}
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		if _, err := io.Copy(tw, tr); err != nil {
			return err
		}
	}
}

func decodeLayerList(r io.Reader) ([]*layerMeta, error) {
	data := []struct {
		Layers []string
	}{}
	if err := json.NewDecoder(r).Decode(&data); err != nil {
		return nil, err
	}
	if l := len(data); l != 1 {
		return nil, fmt.Errorf("manifest.json describes %d objects, call docker save for a single image", l)
	}
	out := make([]*layerMeta, len(data[0].Layers))
	for i, name := range data[0].Layers {
		out[i] = &layerMeta{name: name}
	}
	return out, nil
}

func dumpStream(r io.Reader) (*os.File, error) {
	f, err := ioutil.TempFile("", "merge-docker-save-")
	if err != nil {
		return nil, err
	}
	os.Remove(f.Name())
	if _, err := io.Copy(f, r); err != nil {
		f.Close()
		return nil, err
	}
	return f, nil
}

func openOutput(name string, compress bool) (io.WriteCloser, error) {
	var wc io.WriteCloser = os.Stdout
	if name != "" {
		f, err := os.Create(name)
		if err != nil {
			return nil, err
		}
		wc = f
	}
	if !compress {
		return wc, nil
	}
	return &writerChain{gzip.NewWriter(wc), wc}, nil
}

type writerChain []io.WriteCloser

// Write implements io.Writer by writing to the first Writer in writerChain
func (w writerChain) Write(b []byte) (int, error) { return w[0].Write(b) }

// Close implements io.Closer by closing every Closer in a writerChain and
// returning the first captured non-nil error it encountered.
func (w writerChain) Close() error {
	var err error
	for _, c := range w {
		if err2 := c.Close(); err2 != nil && err == nil {
			err = err2
		}
	}
	return err
}

type layerMeta struct {
	name string
	skip map[string]struct{}
}

// fillSkips fills skip fields of mlayers elements from the tombstone items
// discovered in files referenced in layers map. skip fields filled in such
// a way that for each layer it holds a set of names that should be skipped when
// repacking tar stream since these items would be removed by the following
// layers.
func fillSkips(layers map[string]*os.File, mlayers []*layerMeta) error {
	for i := len(mlayers) - 1; i > 0; i-- {
		meta := mlayers[i]
		f, ok := layers[meta.name]
		if !ok {
			return fmt.Errorf("manifest references unknown layer %q", meta.name)
		}
		skips, err := findSkips(f)
		if err != nil {
			return err
		}
		if skips == nil {
			continue
		}
		for _, meta := range mlayers[:i] {
			if meta.skip == nil {
				meta.skip = make(map[string]struct{})
			}
			for _, s := range skips {
				meta.skip[s] = struct{}{}
			}
		}
	}
	return nil
}

// findSkips scans tar archive for tombstone items and returns list of
// corresponding file names.
func findSkips(f io.ReadSeeker) ([]string, error) {
	if _, err := f.Seek(0, io.SeekStart); err != nil {
		return nil, err
	}
	var skips []string
	tr := tar.NewReader(f)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			return skips, nil
		}
		if err != nil {
			return nil, err
		}
		if base := path.Base(hdr.Name); strings.HasPrefix(base, tombstone) && base != tombstone {
			skips = append(skips, path.Join(path.Dir(hdr.Name), strings.TrimPrefix(base, tombstone)))
		} else {
			// workaround for GNU tar bug: https://gist.github.com/artyom/926ec9c49a2077f2820053274f0b1b16
			skips = append(skips, hdr.Name)
		}
	}
}

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: docker save image:tag | %s > image-fs.tar\n", filepath.Base(os.Args[0]))
		flag.PrintDefaults()
	}
}

const tombstone = ".wh." // prefix docker uses to mark deleted files
<|endoftext|>


<commit_before>// Copyright (C) 2019-2022, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package timer

import (
	"container/heap"
	"errors"
	"fmt"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/message"
	"github.com/ava-labs/avalanchego/utils/math"
	"github.com/ava-labs/avalanchego/utils/timer/mockable"
	"github.com/ava-labs/avalanchego/utils/wrappers"
)

var (
	errNonPositiveHalflife = errors.New("timeout halflife must be positive")

	_ heap.Interface         = &timeoutQueue{}
	_ AdaptiveTimeoutManager = &adaptiveTimeoutManager{}
)

type adaptiveTimeout struct {
	index    int           // Index in the wait queue
	id       ids.ID        // Unique ID of this timeout
	handler  func()        // Function to execute if timed out
	duration time.Duration // How long this timeout was set for
	deadline time.Time     // When this timeout should be fired
	op       message.Op    // Type of this outstanding request
}

type timeoutQueue []*adaptiveTimeout

func (tq timeoutQueue) Len() int           { return len(tq) }
func (tq timeoutQueue) Less(i, j int) bool { return tq[i].deadline.Before(tq[j].deadline) }
func (tq timeoutQueue) Swap(i, j int) {
	tq[i], tq[j] = tq[j], tq[i]
	tq[i].index = i
	tq[j].index = j
}

// Push adds an item to this priority queue. x must have type *adaptiveTimeout
func (tq *timeoutQueue) Push(x interface{}) {
	item := x.(*adaptiveTimeout)
	item.index = len(*tq)
	*tq = append(*tq, item)
}

// Pop returns the next item in this queue
func (tq *timeoutQueue) Pop() interface{} {
	n := len(*tq)
	item := (*tq)[n-1]
	(*tq)[n-1] = nil // make sure the item is freed from memory
	*tq = (*tq)[:n-1]
	return item
}

// AdaptiveTimeoutConfig contains the parameters provided to the
// adaptive timeout manager.
type AdaptiveTimeoutConfig struct {
	InitialTimeout time.Duration `json:"initialTimeout"`
	MinimumTimeout time.Duration `json:"minimumTimeout"`
	MaximumTimeout time.Duration `json:"maximumTimeout"`
	// Timeout is [timeoutCoefficient] * average response time
	// [timeoutCoefficient] must be > 1
	TimeoutCoefficient float64 `json:"timeoutCoefficient"`
	// Larger halflife --> less volatile timeout
	// [timeoutHalfLife] must be positive
	TimeoutHalflife time.Duration `json:"timeoutHalflife"`
}

type AdaptiveTimeoutManager interface {
	// Start the timeout manager.
	// Must be called before any other method.
	// Must only be called once.
	Dispatch()
	// Stop the timeout manager.
	// Must only be called once.
	Stop()
	// Returns the current network timeout duration.
	TimeoutDuration() time.Duration
	// Registers a timeout for the item with the given [id].
	// If the timeout occurs before the item is Removed, [timeoutHandler] is called.
	Put(id ids.ID, op message.Op, timeoutHandler func())
	// Remove the timeout associated with [id].
	// Its timeout handler will not be called.
	Remove(id ids.ID)
	// ObserveLatency manually registers a response latency.
	// We use this to pretend that it a query to a benched validator
	// timed out when actually, we never even sent them a request.
	ObserveLatency(latency time.Duration)
}

type adaptiveTimeoutManager struct {
	lock sync.Mutex
	// Tells the time. Can be faked for testing.
	clock                            mockable.Clock
	networkTimeoutMetric, avgLatency prometheus.Gauge
	numTimeouts                      prometheus.Counter
	// Averages the response time from all peers
	averager math.Averager
	// Timeout is [timeoutCoefficient] * average response time
	// [timeoutCoefficient] must be > 1
	timeoutCoefficient float64
	minimumTimeout     time.Duration
	maximumTimeout     time.Duration
	currentTimeout     time.Duration // Amount of time before a timeout
	timeoutMap         map[ids.ID]*adaptiveTimeout
	timeoutQueue       timeoutQueue
	timer              *Timer // Timer that will fire to clear the timeouts
}

func NewAdaptiveTimeoutManager(
	config *AdaptiveTimeoutConfig,
	metricsNamespace string,
	metricsRegister prometheus.Registerer,
) (AdaptiveTimeoutManager, error) {
	switch {
	case config.InitialTimeout > config.MaximumTimeout:
		return nil, fmt.Errorf("initial timeout (%s) > maximum timeout (%s)", config.InitialTimeout, config.MaximumTimeout)
	case config.InitialTimeout < config.MinimumTimeout:
		return nil, fmt.Errorf("initial timeout (%s) < minimum timeout (%s)", config.InitialTimeout, config.MinimumTimeout)
	case config.TimeoutCoefficient < 1:
		return nil, fmt.Errorf("timeout coefficient must be >= 1 but got %f", config.TimeoutCoefficient)
	case config.TimeoutHalflife <= 0:
		return nil, errNonPositiveHalflife
	}

	tm := &adaptiveTimeoutManager{
		networkTimeoutMetric: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: metricsNamespace,
			Name:      "current_timeout",
			Help:      "Duration of current network timeout in nanoseconds",
		}),
		avgLatency: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: metricsNamespace,
			Name:      "average_latency",
			Help:      "Average network latency in nanoseconds",
		}),
		numTimeouts: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: metricsNamespace,
			Name:      "timeouts",
			Help:      "Number of timed out requests",
		}),
		minimumTimeout:     config.MinimumTimeout,
		maximumTimeout:     config.MaximumTimeout,
		currentTimeout:     config.InitialTimeout,
		timeoutCoefficient: config.TimeoutCoefficient,
		timeoutMap:         make(map[ids.ID]*adaptiveTimeout),
	}
	tm.timer = NewTimer(tm.timeout)
	tm.averager = math.NewAverager(float64(config.InitialTimeout), config.TimeoutHalflife, tm.clock.Time())

	errs := &wrappers.Errs{}
	errs.Add(metricsRegister.Register(tm.networkTimeoutMetric))
	errs.Add(metricsRegister.Register(tm.avgLatency))
	errs.Add(metricsRegister.Register(tm.numTimeouts))
	return tm, errs.Err
}

func (tm *adaptiveTimeoutManager) TimeoutDuration() time.Duration {
	tm.lock.Lock()
	defer tm.lock.Unlock()

	return tm.currentTimeout
}

func (tm *adaptiveTimeoutManager) Dispatch() { tm.timer.Dispatch() }

func (tm *adaptiveTimeoutManager) Stop() { tm.timer.Stop() }

func (tm *adaptiveTimeoutManager) Put(id ids.ID, op message.Op, timeoutHandler func()) {
	tm.lock.Lock()
	defer tm.lock.Unlock()

	tm.put(id, op, timeoutHandler)
}

// Assumes [tm.lock] is held
func (tm *adaptiveTimeoutManager) put(id ids.ID, op message.Op, handler func()) {
	now := tm.clock.Time()
	tm.remove(id, now)

	timeout := &adaptiveTimeout{
		id:       id,
		handler:  handler,
		duration: tm.currentTimeout,
		deadline: now.Add(tm.currentTimeout),
		op:       op,
	}
	tm.timeoutMap[id] = timeout
	heap.Push(&tm.timeoutQueue, timeout)

	tm.setNextTimeoutTime()
}

func (tm *adaptiveTimeoutManager) Remove(id ids.ID) {
	tm.lock.Lock()
	defer tm.lock.Unlock()

	tm.remove(id, tm.clock.Time())
}

// Assumes [tm.lock] is held
func (tm *adaptiveTimeoutManager) remove(id ids.ID, now time.Time) {
	timeout, exists := tm.timeoutMap[id]
	if !exists {
		return
	}

	// Observe the response time to update average network response time.
	// Don't include Get requests in calculation, since an adversary
	// can cause you to issue a Get request and then cause it to timeout,
	// increasing your timeout.
	if timeout.op != message.Get {
		timeoutRegisteredAt := timeout.deadline.Add(-1 * timeout.duration)
		latency := now.Sub(timeoutRegisteredAt)
		tm.observeLatencyAndUpdateTimeout(latency, now)
	}

	// Remove the timeout from the map
	delete(tm.timeoutMap, id)

	// Remove the timeout from the queue
	heap.Remove(&tm.timeoutQueue, timeout.index)
}

// Assumes [tm.lock] is not held.
func (tm *adaptiveTimeoutManager) timeout() {
	tm.lock.Lock()
	defer tm.lock.Unlock()

	now := tm.clock.Time()
	for {
		// getNextTimeoutHandler returns nil once there is nothing left to remove
		timeoutHandler := tm.getNextTimeoutHandler(now)
		if timeoutHandler == nil {
			break
		}
		tm.numTimeouts.Inc()

		// Don't execute a callback with a lock held
		tm.lock.Unlock()
		timeoutHandler()
		tm.lock.Lock()
	}
	tm.setNextTimeoutTime()
}

func (tm *adaptiveTimeoutManager) ObserveLatency(latency time.Duration) {
	tm.lock.Lock()
	defer tm.lock.Unlock()

	tm.observeLatencyAndUpdateTimeout(latency, tm.clock.Time())
}

// Assumes [tm.lock] is held
func (tm *adaptiveTimeoutManager) observeLatencyAndUpdateTimeout(latency time.Duration, now time.Time) {
	tm.averager.Observe(float64(latency), now)
	avgLatency := tm.averager.Read()
	tm.currentTimeout = time.Duration(tm.timeoutCoefficient * avgLatency)
	if tm.currentTimeout > tm.maximumTimeout {
		tm.currentTimeout = tm.maximumTimeout
	} else if tm.currentTimeout < tm.minimumTimeout {
		tm.currentTimeout = tm.minimumTimeout
	}
	// Update the metrics
	tm.networkTimeoutMetric.Set(float64(tm.currentTimeout))
	tm.avgLatency.Set(avgLatency)
}

// Returns the handler function associated with the next timeout.
// If there are no timeouts, or if the next timeout is after [now],
// returns nil.
// Assumes [tm.lock] is held
func (tm *adaptiveTimeoutManager) getNextTimeoutHandler(now time.Time) func() {
	if tm.timeoutQueue.Len() == 0 {
		return nil
	}

	nextTimeout := tm.timeoutQueue[0]
	if nextTimeout.deadline.After(now) {
		return nil
	}
	tm.remove(nextTimeout.id, now)
	return nextTimeout.handler
}

// Calculate the time of the next timeout and set
// the timer to fire at that time.
func (tm *adaptiveTimeoutManager) setNextTimeoutTime() {
	if tm.timeoutQueue.Len() == 0 {
		// There are no pending timeouts
		tm.timer.Cancel()
		return
	}

	now := tm.clock.Time()
	nextTimeout := tm.timeoutQueue[0]
	timeToNextTimeout := nextTimeout.deadline.Sub(now)
	tm.timer.SetTimeoutIn(timeToNextTimeout)
}
<commit_msg>Add metric for pending network timeouts (#2035)<commit_after>// Copyright (C) 2019-2022, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package timer

import (
	"container/heap"
	"errors"
	"fmt"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/message"
	"github.com/ava-labs/avalanchego/utils/math"
	"github.com/ava-labs/avalanchego/utils/timer/mockable"
	"github.com/ava-labs/avalanchego/utils/wrappers"
)

var (
	errNonPositiveHalflife = errors.New("timeout halflife must be positive")

	_ heap.Interface         = &timeoutQueue{}
	_ AdaptiveTimeoutManager = &adaptiveTimeoutManager{}
)

type adaptiveTimeout struct {
	index    int           // Index in the wait queue
	id       ids.ID        // Unique ID of this timeout
	handler  func()        // Function to execute if timed out
	duration time.Duration // How long this timeout was set for
	deadline time.Time     // When this timeout should be fired
	op       message.Op    // Type of this outstanding request
}

type timeoutQueue []*adaptiveTimeout

func (tq timeoutQueue) Len() int           { return len(tq) }
func (tq timeoutQueue) Less(i, j int) bool { return tq[i].deadline.Before(tq[j].deadline) }
func (tq timeoutQueue) Swap(i, j int) {
	tq[i], tq[j] = tq[j], tq[i]
	tq[i].index = i
	tq[j].index = j
}

// Push adds an item to this priority queue. x must have type *adaptiveTimeout
func (tq *timeoutQueue) Push(x interface{}) {
	item := x.(*adaptiveTimeout)
	item.index = len(*tq)
	*tq = append(*tq, item)
}

// Pop returns the next item in this queue
func (tq *timeoutQueue) Pop() interface{} {
	n := len(*tq)
	item := (*tq)[n-1]
	(*tq)[n-1] = nil // make sure the item is freed from memory
	*tq = (*tq)[:n-1]
	return item
}

// AdaptiveTimeoutConfig contains the parameters provided to the
// adaptive timeout manager.
type AdaptiveTimeoutConfig struct {
	InitialTimeout time.Duration `json:"initialTimeout"`
	MinimumTimeout time.Duration `json:"minimumTimeout"`
	MaximumTimeout time.Duration `json:"maximumTimeout"`
	// Timeout is [timeoutCoefficient] * average response time
	// [timeoutCoefficient] must be > 1
	TimeoutCoefficient float64 `json:"timeoutCoefficient"`
	// Larger halflife --> less volatile timeout
	// [timeoutHalfLife] must be positive
	TimeoutHalflife time.Duration `json:"timeoutHalflife"`
}

type AdaptiveTimeoutManager interface {
	// Start the timeout manager.
	// Must be called before any other method.
	// Must only be called once.
	Dispatch()
	// Stop the timeout manager.
	// Must only be called once.
	Stop()
	// Returns the current network timeout duration.
	TimeoutDuration() time.Duration
	// Registers a timeout for the item with the given [id].
	// If the timeout occurs before the item is Removed, [timeoutHandler] is called.
	Put(id ids.ID, op message.Op, timeoutHandler func())
	// Remove the timeout associated with [id].
	// Its timeout handler will not be called.
	Remove(id ids.ID)
	// ObserveLatency manually registers a response latency.
	// We use this to pretend that it a query to a benched validator
	// timed out when actually, we never even sent them a request.
	ObserveLatency(latency time.Duration)
}

type adaptiveTimeoutManager struct {
	lock sync.Mutex
	// Tells the time. Can be faked for testing.
	clock                            mockable.Clock
	networkTimeoutMetric, avgLatency prometheus.Gauge
	numTimeouts                      prometheus.Counter
	numPendingTimeouts               prometheus.Gauge
	// Averages the response time from all peers
	averager math.Averager
	// Timeout is [timeoutCoefficient] * average response time
	// [timeoutCoefficient] must be > 1
	timeoutCoefficient float64
	minimumTimeout     time.Duration
	maximumTimeout     time.Duration
	currentTimeout     time.Duration // Amount of time before a timeout
	timeoutMap         map[ids.ID]*adaptiveTimeout
	timeoutQueue       timeoutQueue
	timer              *Timer // Timer that will fire to clear the timeouts
}

func NewAdaptiveTimeoutManager(
	config *AdaptiveTimeoutConfig,
	metricsNamespace string,
	metricsRegister prometheus.Registerer,
) (AdaptiveTimeoutManager, error) {
	switch {
	case config.InitialTimeout > config.MaximumTimeout:
		return nil, fmt.Errorf("initial timeout (%s) > maximum timeout (%s)", config.InitialTimeout, config.MaximumTimeout)
	case config.InitialTimeout < config.MinimumTimeout:
		return nil, fmt.Errorf("initial timeout (%s) < minimum timeout (%s)", config.InitialTimeout, config.MinimumTimeout)
	case config.TimeoutCoefficient < 1:
		return nil, fmt.Errorf("timeout coefficient must be >= 1 but got %f", config.TimeoutCoefficient)
	case config.TimeoutHalflife <= 0:
		return nil, errNonPositiveHalflife
	}

	tm := &adaptiveTimeoutManager{
		networkTimeoutMetric: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: metricsNamespace,
			Name:      "current_timeout",
			Help:      "Duration of current network timeout in nanoseconds",
		}),
		avgLatency: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: metricsNamespace,
			Name:      "average_latency",
			Help:      "Average network latency in nanoseconds",
		}),
		numTimeouts: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: metricsNamespace,
			Name:      "timeouts",
			Help:      "Number of timed out requests",
		}),
		numPendingTimeouts: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: metricsNamespace,
			Name:      "pending_timeouts",
			Help:      "Number of pending timeouts",
		}),
		minimumTimeout:     config.MinimumTimeout,
		maximumTimeout:     config.MaximumTimeout,
		currentTimeout:     config.InitialTimeout,
		timeoutCoefficient: config.TimeoutCoefficient,
		timeoutMap:         make(map[ids.ID]*adaptiveTimeout),
	}
	tm.timer = NewTimer(tm.timeout)
	tm.averager = math.NewAverager(float64(config.InitialTimeout), config.TimeoutHalflife, tm.clock.Time())

	errs := &wrappers.Errs{}
	errs.Add(
		metricsRegister.Register(tm.networkTimeoutMetric),
		metricsRegister.Register(tm.avgLatency),
		metricsRegister.Register(tm.numTimeouts),
		metricsRegister.Register(tm.numPendingTimeouts),
	)
	return tm, errs.Err
}

func (tm *adaptiveTimeoutManager) TimeoutDuration() time.Duration {
	tm.lock.Lock()
	defer tm.lock.Unlock()

	return tm.currentTimeout
}

func (tm *adaptiveTimeoutManager) Dispatch() { tm.timer.Dispatch() }

func (tm *adaptiveTimeoutManager) Stop() { tm.timer.Stop() }

func (tm *adaptiveTimeoutManager) Put(id ids.ID, op message.Op, timeoutHandler func()) {
	tm.lock.Lock()
	defer tm.lock.Unlock()

	tm.put(id, op, timeoutHandler)
}

// Assumes [tm.lock] is held
func (tm *adaptiveTimeoutManager) put(id ids.ID, op message.Op, handler func()) {
	now := tm.clock.Time()
	tm.remove(id, now)

	timeout := &adaptiveTimeout{
		id:       id,
		handler:  handler,
		duration: tm.currentTimeout,
		deadline: now.Add(tm.currentTimeout),
		op:       op,
	}
	tm.timeoutMap[id] = timeout
	tm.numPendingTimeouts.Set(float64(len(tm.timeoutMap)))
	heap.Push(&tm.timeoutQueue, timeout)

	tm.setNextTimeoutTime()
}

func (tm *adaptiveTimeoutManager) Remove(id ids.ID) {
	tm.lock.Lock()
	defer tm.lock.Unlock()

	tm.remove(id, tm.clock.Time())
}

// Assumes [tm.lock] is held
func (tm *adaptiveTimeoutManager) remove(id ids.ID, now time.Time) {
	timeout, exists := tm.timeoutMap[id]
	if !exists {
		return
	}

	// Observe the response time to update average network response time.
	// Don't include Get requests in calculation, since an adversary
	// can cause you to issue a Get request and then cause it to timeout,
	// increasing your timeout.
	if timeout.op != message.Get {
		timeoutRegisteredAt := timeout.deadline.Add(-1 * timeout.duration)
		latency := now.Sub(timeoutRegisteredAt)
		tm.observeLatencyAndUpdateTimeout(latency, now)
	}

	// Remove the timeout from the map
	delete(tm.timeoutMap, id)
	tm.numPendingTimeouts.Set(float64(len(tm.timeoutMap)))

	// Remove the timeout from the queue
	heap.Remove(&tm.timeoutQueue, timeout.index)
}

// Assumes [tm.lock] is not held.
func (tm *adaptiveTimeoutManager) timeout() {
	tm.lock.Lock()
	defer tm.lock.Unlock()

	now := tm.clock.Time()
	for {
		// getNextTimeoutHandler returns nil once there is nothing left to remove
		timeoutHandler := tm.getNextTimeoutHandler(now)
		if timeoutHandler == nil {
			break
		}
		tm.numTimeouts.Inc()

		// Don't execute a callback with a lock held
		tm.lock.Unlock()
		timeoutHandler()
		tm.lock.Lock()
	}
	tm.setNextTimeoutTime()
}

func (tm *adaptiveTimeoutManager) ObserveLatency(latency time.Duration) {
	tm.lock.Lock()
	defer tm.lock.Unlock()

	tm.observeLatencyAndUpdateTimeout(latency, tm.clock.Time())
}

// Assumes [tm.lock] is held
func (tm *adaptiveTimeoutManager) observeLatencyAndUpdateTimeout(latency time.Duration, now time.Time) {
	tm.averager.Observe(float64(latency), now)
	avgLatency := tm.averager.Read()
	tm.currentTimeout = time.Duration(tm.timeoutCoefficient * avgLatency)
	if tm.currentTimeout > tm.maximumTimeout {
		tm.currentTimeout = tm.maximumTimeout
	} else if tm.currentTimeout < tm.minimumTimeout {
		tm.currentTimeout = tm.minimumTimeout
	}
	// Update the metrics
	tm.networkTimeoutMetric.Set(float64(tm.currentTimeout))
	tm.avgLatency.Set(avgLatency)
}

// Returns the handler function associated with the next timeout.
// If there are no timeouts, or if the next timeout is after [now],
// returns nil.
// Assumes [tm.lock] is held
func (tm *adaptiveTimeoutManager) getNextTimeoutHandler(now time.Time) func() {
	if tm.timeoutQueue.Len() == 0 {
		return nil
	}

	nextTimeout := tm.timeoutQueue[0]
	if nextTimeout.deadline.After(now) {
		return nil
	}
	tm.remove(nextTimeout.id, now)
	return nextTimeout.handler
}

// Calculate the time of the next timeout and set
// the timer to fire at that time.
func (tm *adaptiveTimeoutManager) setNextTimeoutTime() {
	if tm.timeoutQueue.Len() == 0 {
		// There are no pending timeouts
		tm.timer.Cancel()
		return
	}

	now := tm.clock.Time()
	nextTimeout := tm.timeoutQueue[0]
	timeToNextTimeout := nextTimeout.deadline.Sub(now)
	tm.timer.SetTimeoutIn(timeToNextTimeout)
}
<|endoftext|>


<commit_before>package migrator

import (
	"log"

	"github.com/awslabs/aws-sdk-go/aws"
	"github.com/awslabs/aws-sdk-go/gen/kinesis"
	"github.com/awslabs/aws-sdk-go/gen/swf"
	. "github.com/sclasen/swfsm/sugar"
	//"github.com/awslabs/aws-sdk-go/gen/dynamodb"
	"time"
)

// TypesMigrator is composed of a DomainMigrator, a WorkflowTypeMigrator and an ActivityTypeMigrator.
type TypesMigrator struct {
	DomainMigrator       *DomainMigrator
	WorkflowTypeMigrator *WorkflowTypeMigrator
	ActivityTypeMigrator *ActivityTypeMigrator
	StreamMigrator       *StreamMigrator
}

type SWFOps interface {
	DeprecateActivityType(req *swf.DeprecateActivityTypeInput) (err error)
	DeprecateDomain(req *swf.DeprecateDomainInput) (err error)
	DeprecateWorkflowType(req *swf.DeprecateWorkflowTypeInput) (err error)
	DescribeActivityType(req *swf.DescribeActivityTypeInput) (resp *swf.ActivityTypeDetail, err error)
	DescribeDomain(req *swf.DescribeDomainInput) (resp *swf.DomainDetail, err error)
	DescribeWorkflowExecution(req *swf.DescribeWorkflowExecutionInput) (resp *swf.WorkflowExecutionDetail, err error)
	DescribeWorkflowType(req *swf.DescribeWorkflowTypeInput) (resp *swf.WorkflowTypeDetail, err error)
	RegisterActivityType(req *swf.RegisterActivityTypeInput) (err error)
	RegisterDomain(req *swf.RegisterDomainInput) (err error)
	RegisterWorkflowType(req *swf.RegisterWorkflowTypeInput) (err error)
}

type KinesisOps interface {
	CreateStream(req *kinesis.CreateStreamInput) (err error)
	DescribeStream(req *kinesis.DescribeStreamInput) (resp *kinesis.DescribeStreamOutput, err error)
}

// Migrate runs Migrate on the underlying DomainMigrator, a WorkflowTypeMigrator and ActivityTypeMigrator.
func (t *TypesMigrator) Migrate() {
	if t.ActivityTypeMigrator == nil {
		t.ActivityTypeMigrator = new(ActivityTypeMigrator)
	}
	if t.DomainMigrator == nil {
		t.DomainMigrator = new(DomainMigrator)
	}
	if t.WorkflowTypeMigrator == nil {
		t.WorkflowTypeMigrator = new(WorkflowTypeMigrator)
	}
	if t.StreamMigrator == nil {
		t.StreamMigrator = new(StreamMigrator)
	}
	t.DomainMigrator.Migrate()
	t.WorkflowTypeMigrator.Migrate()
	t.ActivityTypeMigrator.Migrate()
	t.StreamMigrator.Migrate()
}

// DomainMigrator will register or deprecate the configured domains as required.
type DomainMigrator struct {
	RegisteredDomains []swf.RegisterDomainInput
	DeprecatedDomains []swf.DeprecateDomainInput
	Client            SWFOps
}

// Migrate asserts that DeprecatedDomains are deprecated or deprecates them, then asserts that RegisteredDomains are registered or registers them.
func (d *DomainMigrator) Migrate() {
	for _, dd := range d.DeprecatedDomains {
		if d.isDeprecated(dd.Name) {
			log.Printf("action=migrate at=deprecate-domain domain=%s status=previously-deprecated", LS(dd.Name))
		} else {
			d.deprecate(dd)
			log.Printf("action=migrate at=deprecate-domain domain=%s status=deprecated", LS(dd.Name))
		}
	}
	for _, r := range d.RegisteredDomains {
		if d.isRegisteredNotDeprecated(r) {
			log.Printf("action=migrate at=register-domain domain=%s status=previously-registered", LS(r.Name))
		} else {
			d.register(r)
			log.Printf("action=migrate at=register-domain domain=%s status=registered", LS(r.Name))
		}
	}
}

func (d *DomainMigrator) isRegisteredNotDeprecated(rd swf.RegisterDomainInput) bool {
	desc, err := d.describe(rd.Name)
	if err != nil {
		if ae, ok := err.(aws.APIError); ok && ae.Type == ErrorTypeUnknownResourceFault {
			return false
		}

		panic(err)

	}

	return *desc.DomainInfo.Status == swf.RegistrationStatusRegistered
}

func (d *DomainMigrator) register(rd swf.RegisterDomainInput) {
	err := d.Client.RegisterDomain(&rd)
	if err != nil {
		panic(err)
	}
}

func (d *DomainMigrator) isDeprecated(domain aws.StringValue) bool {
	desc, err := d.describe(domain)
	if err != nil {
		log.Printf("action=migrate at=is-dep domain=%s error=%s", LS(domain), err.Error())
		return false
	}

	return *desc.DomainInfo.Status == swf.RegistrationStatusDeprecated
}

func (d *DomainMigrator) deprecate(dd swf.DeprecateDomainInput) {
	err := d.Client.DeprecateDomain(&dd)
	if err != nil {
		panic(err)
	}
}

func (d *DomainMigrator) describe(domain aws.StringValue) (*swf.DomainDetail, error) {
	resp, err := d.Client.DescribeDomain(&swf.DescribeDomainInput{Name: domain})
	if err != nil {
		return nil, err
	}
	return resp, nil
}

// WorkflowTypeMigrator will register or deprecate the configured workflow types as required.
type WorkflowTypeMigrator struct {
	RegisteredWorkflowTypes []swf.RegisterWorkflowTypeInput
	DeprecatedWorkflowTypes []swf.DeprecateWorkflowTypeInput
	Client                  SWFOps
}

// Migrate asserts that DeprecatedWorkflowTypes are deprecated or deprecates them, then asserts that RegisteredWorkflowTypes are registered or registers them.
func (w *WorkflowTypeMigrator) Migrate() {
	for _, dd := range w.DeprecatedWorkflowTypes {
		if w.isDeprecated(dd.Domain, dd.WorkflowType.Name, dd.WorkflowType.Version) {
			log.Printf("action=migrate at=deprecate-workflow domain=%s workflow=%s version=%s status=previously-deprecated", LS(dd.Domain), LS(dd.WorkflowType.Name), LS(dd.WorkflowType.Version))
		} else {
			w.deprecate(dd)
			log.Printf("action=migrate at=deprecate-workflow domain=%s  workflow=%s version=%s status=deprecate", LS(dd.Domain), LS(dd.WorkflowType.Name), LS(dd.WorkflowType.Version))
		}
	}
	for _, r := range w.RegisteredWorkflowTypes {
		if w.isRegisteredNotDeprecated(r) {
			log.Printf("action=migrate at=register-workflow domain=%s workflow=%s version=%s status=previously-registered", LS(r.Domain), LS(r.Name), LS(r.Version))
		} else {
			w.register(r)
			log.Printf("action=migrate at=register-workflow domain=%s  workflow=%s version=%s status=registered", LS(r.Domain), LS(r.Name), LS(r.Version))
		}
	}
}

func (w *WorkflowTypeMigrator) isRegisteredNotDeprecated(rd swf.RegisterWorkflowTypeInput) bool {
	desc, err := w.describe(rd.Domain, rd.Name, rd.Version)
	if err != nil {
		if ae, ok := err.(aws.APIError); ok && ae.Type == ErrorTypeUnknownResourceFault {
			return false
		}

		panic(err)

	}

	return *desc.TypeInfo.Status == swf.RegistrationStatusRegistered
}

func (w *WorkflowTypeMigrator) register(rd swf.RegisterWorkflowTypeInput) {
	err := w.Client.RegisterWorkflowType(&rd)
	if err != nil {
		panic(err)
	}
}

func (w *WorkflowTypeMigrator) isDeprecated(domain aws.StringValue, name aws.StringValue, version aws.StringValue) bool {
	desc, err := w.describe(domain, name, version)
	if err != nil {
		log.Printf("action=migrate at=is-dep domain=%s workflow=%s version=%s error=%s", LS(domain), LS(name), LS(version), err.Error())
		return false
	}

	return *desc.TypeInfo.Status == swf.RegistrationStatusDeprecated
}

func (w *WorkflowTypeMigrator) deprecate(dd swf.DeprecateWorkflowTypeInput) {
	err := w.Client.DeprecateWorkflowType(&dd)
	if err != nil {
		panic(err)
	}
}

func (w *WorkflowTypeMigrator) describe(domain aws.StringValue, name aws.StringValue, version aws.StringValue) (*swf.WorkflowTypeDetail, error) {
	resp, err := w.Client.DescribeWorkflowType(&swf.DescribeWorkflowTypeInput{Domain: domain, WorkflowType: &swf.WorkflowType{Name: name, Version: version}})
	if err != nil {
		return nil, err
	}
	return resp, nil
}

// ActivityTypeMigrator will register or deprecate the configured activity types as required.
type ActivityTypeMigrator struct {
	RegisteredActivityTypes []swf.RegisterActivityTypeInput
	DeprecatedActivityTypes []swf.DeprecateActivityTypeInput
	Client                  SWFOps
}

// Migrate asserts that DeprecatedActivityTypes are deprecated or deprecates them, then asserts that RegisteredActivityTypes are registered or registers them.
func (a *ActivityTypeMigrator) Migrate() {
	for _, d := range a.DeprecatedActivityTypes {
		if a.isDeprecated(d.Domain, d.ActivityType.Name, d.ActivityType.Version) {
			log.Printf("action=migrate at=deprecate-activity domain=%s activity=%s version=%s status=previously-deprecated", LS(d.Domain), LS(d.ActivityType.Name), LS(d.ActivityType.Version))
		} else {
			a.deprecate(d)
			log.Printf("action=migrate at=depreacate-activity domain=%s activity=%s version=%s status=deprecated", LS(d.Domain), LS(d.ActivityType.Name), LS(d.ActivityType.Version))
		}
	}
	for _, r := range a.RegisteredActivityTypes {
		if a.isRegisteredNotDeprecated(r) {
			log.Printf("action=migrate at=register-activity domain=%s activity=%s version=%s status=previously-registered", LS(r.Domain), LS(r.Name), LS(r.Version))
		} else {
			a.register(r)
			log.Printf("action=migrate at=register-activity domain=%s activity=%s version=%s status=registered", LS(r.Domain), LS(r.Name), LS(r.Version))
		}
	}
}

func (a *ActivityTypeMigrator) isRegisteredNotDeprecated(rd swf.RegisterActivityTypeInput) bool {
	desc, err := a.describe(rd.Domain, rd.Name, rd.Version)
	if err != nil {
		if ae, ok := err.(aws.APIError); ok && ae.Type == ErrorTypeUnknownResourceFault {
			return false
		}

		panic(err)

	}

	return *desc.TypeInfo.Status == swf.RegistrationStatusRegistered
}

func (a *ActivityTypeMigrator) register(rd swf.RegisterActivityTypeInput) {
	err := a.Client.RegisterActivityType(&rd)
	if err != nil {
		panic(err)
	}
}

func (a *ActivityTypeMigrator) isDeprecated(domain aws.StringValue, name aws.StringValue, version aws.StringValue) bool {
	desc, err := a.describe(domain, name, version)
	if err != nil {
		log.Printf("action=migrate at=is-dep domain=%s activity=%s version=%s error=%s", LS(domain), LS(name), LS(version), err.Error())
		return false
	}

	return *desc.TypeInfo.Status == swf.RegistrationStatusDeprecated
}

func (a *ActivityTypeMigrator) deprecate(dd swf.DeprecateActivityTypeInput) {
	err := a.Client.DeprecateActivityType(&dd)
	if err != nil {
		panic(err)
	}
}

func (a *ActivityTypeMigrator) describe(domain aws.StringValue, name aws.StringValue, version aws.StringValue) (*swf.ActivityTypeDetail, error) {
	resp, err := a.Client.DescribeActivityType(&swf.DescribeActivityTypeInput{Domain: domain, ActivityType: &swf.ActivityType{Name: name, Version: version}})
	if err != nil {
		return nil, err
	}
	return resp, nil
}

// StreamMigrator will create any Kinesis Streams required.
type StreamMigrator struct {
	Streams []kinesis.CreateStreamInput
	Client  KinesisOps
}

// Migrate checks that the desired streams have been created and if they have not, creates them.s
func (s *StreamMigrator) Migrate() {
	for _, st := range s.Streams {
		if s.isCreated(st) {
			log.Printf("action=migrate at=create-stream stream=%s status=previously-created", LS(st.StreamName))
		} else {
			s.create(st)
			log.Printf("action=migrate at=create-stream stream=%s status=created", LS(st.StreamName))
		}
		s.awaitActive(st.StreamName, 30)
	}
}

func (s *StreamMigrator) isCreated(st kinesis.CreateStreamInput) bool {
	_, err := s.describe(st)
	if err != nil {
		if ae, ok := err.(aws.APIError); ok && ae.Type == ErrorTypeStreamNotFound {
			return false
		}
		panic(err)

	}

	return true
}

func (s *StreamMigrator) create(st kinesis.CreateStreamInput) {
	err := s.Client.CreateStream(&st)
	if err != nil {
		panic(err)
	}
}

func (s *StreamMigrator) describe(st kinesis.CreateStreamInput) (*kinesis.DescribeStreamOutput, error) {
	req := kinesis.DescribeStreamInput{
		StreamName: st.StreamName,
	}
	resp, err := s.Client.DescribeStream(&req)
	if err != nil {
		return nil, err
	}
	return resp, nil
}

func (s *StreamMigrator) awaitActive(stream aws.StringValue, atMostSeconds int) {

	waited := 0
	status := kinesis.StreamStatusCreating
	for status != kinesis.StreamStatusActive {
		desc, err := s.Client.DescribeStream(&kinesis.DescribeStreamInput{
			StreamName: stream,
		})
		if err != nil {
			log.Printf("component=kinesis-migrator fn=awaitActive at=describe-error error=%s", err)
			panic(err)
		}
		log.Printf("component=kinesis-migrator fn=awaitActive stream=%s at=describe status=%s", *stream, *desc.StreamDescription.StreamStatus)
		status = *desc.StreamDescription.StreamStatus
		time.Sleep(1 * time.Second)
		waited++
		if waited >= atMostSeconds {
			log.Printf("component=kinesis-migrator fn=awaitActive streal=%s  at=error error=exeeeded-max-wait", *stream)
			panic("waited too long")
		}
	}
}
<commit_msg>unwrap aws errors when migrations fail<commit_after>package migrator

import (
	"log"

	"github.com/awslabs/aws-sdk-go/aws"
	"github.com/awslabs/aws-sdk-go/gen/kinesis"
	"github.com/awslabs/aws-sdk-go/gen/swf"
	. "github.com/sclasen/swfsm/sugar"
	//"github.com/awslabs/aws-sdk-go/gen/dynamodb"
	"time"
    "fmt"
)

// TypesMigrator is composed of a DomainMigrator, a WorkflowTypeMigrator and an ActivityTypeMigrator.
type TypesMigrator struct {
	DomainMigrator       *DomainMigrator
	WorkflowTypeMigrator *WorkflowTypeMigrator
	ActivityTypeMigrator *ActivityTypeMigrator
	StreamMigrator       *StreamMigrator
}

type SWFOps interface {
	DeprecateActivityType(req *swf.DeprecateActivityTypeInput) (err error)
	DeprecateDomain(req *swf.DeprecateDomainInput) (err error)
	DeprecateWorkflowType(req *swf.DeprecateWorkflowTypeInput) (err error)
	DescribeActivityType(req *swf.DescribeActivityTypeInput) (resp *swf.ActivityTypeDetail, err error)
	DescribeDomain(req *swf.DescribeDomainInput) (resp *swf.DomainDetail, err error)
	DescribeWorkflowExecution(req *swf.DescribeWorkflowExecutionInput) (resp *swf.WorkflowExecutionDetail, err error)
	DescribeWorkflowType(req *swf.DescribeWorkflowTypeInput) (resp *swf.WorkflowTypeDetail, err error)
	RegisterActivityType(req *swf.RegisterActivityTypeInput) (err error)
	RegisterDomain(req *swf.RegisterDomainInput) (err error)
	RegisterWorkflowType(req *swf.RegisterWorkflowTypeInput) (err error)
}

type KinesisOps interface {
	CreateStream(req *kinesis.CreateStreamInput) (err error)
	DescribeStream(req *kinesis.DescribeStreamInput) (resp *kinesis.DescribeStreamOutput, err error)
}

// Migrate runs Migrate on the underlying DomainMigrator, a WorkflowTypeMigrator and ActivityTypeMigrator.
func (t *TypesMigrator) Migrate() {
	if t.ActivityTypeMigrator == nil {
		t.ActivityTypeMigrator = new(ActivityTypeMigrator)
	}
	if t.DomainMigrator == nil {
		t.DomainMigrator = new(DomainMigrator)
	}
	if t.WorkflowTypeMigrator == nil {
		t.WorkflowTypeMigrator = new(WorkflowTypeMigrator)
	}
	if t.StreamMigrator == nil {
		t.StreamMigrator = new(StreamMigrator)
	}
	t.DomainMigrator.Migrate()
	t.WorkflowTypeMigrator.Migrate()
	t.ActivityTypeMigrator.Migrate()
	t.StreamMigrator.Migrate()
}

// DomainMigrator will register or deprecate the configured domains as required.
type DomainMigrator struct {
	RegisteredDomains []swf.RegisterDomainInput
	DeprecatedDomains []swf.DeprecateDomainInput
	Client            SWFOps
}

// Migrate asserts that DeprecatedDomains are deprecated or deprecates them, then asserts that RegisteredDomains are registered or registers them.
func (d *DomainMigrator) Migrate() {
	for _, dd := range d.DeprecatedDomains {
		if d.isDeprecated(dd.Name) {
			log.Printf("action=migrate at=deprecate-domain domain=%s status=previously-deprecated", LS(dd.Name))
		} else {
			d.deprecate(dd)
			log.Printf("action=migrate at=deprecate-domain domain=%s status=deprecated", LS(dd.Name))
		}
	}
	for _, r := range d.RegisteredDomains {
		if d.isRegisteredNotDeprecated(r) {
			log.Printf("action=migrate at=register-domain domain=%s status=previously-registered", LS(r.Name))
		} else {
			d.register(r)
			log.Printf("action=migrate at=register-domain domain=%s status=registered", LS(r.Name))
		}
	}
}

func (d *DomainMigrator) isRegisteredNotDeprecated(rd swf.RegisterDomainInput) bool {
	desc, err := d.describe(rd.Name)
	if err != nil {
		if ae, ok := err.(aws.APIError); ok && ae.Type == ErrorTypeUnknownResourceFault {
			return false
		}

		panicWithError(err)

	}

	return *desc.DomainInfo.Status == swf.RegistrationStatusRegistered
}

func (d *DomainMigrator) register(rd swf.RegisterDomainInput) {
	err := d.Client.RegisterDomain(&rd)
	if err != nil {
		panicWithError(err)
	}
}

func (d *DomainMigrator) isDeprecated(domain aws.StringValue) bool {
	desc, err := d.describe(domain)
	if err != nil {
		log.Printf("action=migrate at=is-dep domain=%s error=%s", LS(domain), err.Error())
		return false
	}

	return *desc.DomainInfo.Status == swf.RegistrationStatusDeprecated
}

func (d *DomainMigrator) deprecate(dd swf.DeprecateDomainInput) {
	err := d.Client.DeprecateDomain(&dd)
	if err != nil {
		panicWithError(err)
	}
}

func (d *DomainMigrator) describe(domain aws.StringValue) (*swf.DomainDetail, error) {
	resp, err := d.Client.DescribeDomain(&swf.DescribeDomainInput{Name: domain})
	if err != nil {
		return nil, err
	}
	return resp, nil
}

// WorkflowTypeMigrator will register or deprecate the configured workflow types as required.
type WorkflowTypeMigrator struct {
	RegisteredWorkflowTypes []swf.RegisterWorkflowTypeInput
	DeprecatedWorkflowTypes []swf.DeprecateWorkflowTypeInput
	Client                  SWFOps
}

// Migrate asserts that DeprecatedWorkflowTypes are deprecated or deprecates them, then asserts that RegisteredWorkflowTypes are registered or registers them.
func (w *WorkflowTypeMigrator) Migrate() {
	for _, dd := range w.DeprecatedWorkflowTypes {
		if w.isDeprecated(dd.Domain, dd.WorkflowType.Name, dd.WorkflowType.Version) {
			log.Printf("action=migrate at=deprecate-workflow domain=%s workflow=%s version=%s status=previously-deprecated", LS(dd.Domain), LS(dd.WorkflowType.Name), LS(dd.WorkflowType.Version))
		} else {
			w.deprecate(dd)
			log.Printf("action=migrate at=deprecate-workflow domain=%s  workflow=%s version=%s status=deprecate", LS(dd.Domain), LS(dd.WorkflowType.Name), LS(dd.WorkflowType.Version))
		}
	}
	for _, r := range w.RegisteredWorkflowTypes {
		if w.isRegisteredNotDeprecated(r) {
			log.Printf("action=migrate at=register-workflow domain=%s workflow=%s version=%s status=previously-registered", LS(r.Domain), LS(r.Name), LS(r.Version))
		} else {
			w.register(r)
			log.Printf("action=migrate at=register-workflow domain=%s  workflow=%s version=%s status=registered", LS(r.Domain), LS(r.Name), LS(r.Version))
		}
	}
}

func (w *WorkflowTypeMigrator) isRegisteredNotDeprecated(rd swf.RegisterWorkflowTypeInput) bool {
	desc, err := w.describe(rd.Domain, rd.Name, rd.Version)
	if err != nil {
		if ae, ok := err.(aws.APIError); ok && ae.Type == ErrorTypeUnknownResourceFault {
			return false
		}

		panicWithError(err)

	}

	return *desc.TypeInfo.Status == swf.RegistrationStatusRegistered
}

func (w *WorkflowTypeMigrator) register(rd swf.RegisterWorkflowTypeInput) {
	err := w.Client.RegisterWorkflowType(&rd)
	if err != nil {
		panicWithError(err)
	}
}

func (w *WorkflowTypeMigrator) isDeprecated(domain aws.StringValue, name aws.StringValue, version aws.StringValue) bool {
	desc, err := w.describe(domain, name, version)
	if err != nil {
		log.Printf("action=migrate at=is-dep domain=%s workflow=%s version=%s error=%s", LS(domain), LS(name), LS(version), err.Error())
		return false
	}

	return *desc.TypeInfo.Status == swf.RegistrationStatusDeprecated
}

func (w *WorkflowTypeMigrator) deprecate(dd swf.DeprecateWorkflowTypeInput) {
	err := w.Client.DeprecateWorkflowType(&dd)
	if err != nil {
		panicWithError(err)
	}
}

func (w *WorkflowTypeMigrator) describe(domain aws.StringValue, name aws.StringValue, version aws.StringValue) (*swf.WorkflowTypeDetail, error) {
	resp, err := w.Client.DescribeWorkflowType(&swf.DescribeWorkflowTypeInput{Domain: domain, WorkflowType: &swf.WorkflowType{Name: name, Version: version}})
	if err != nil {
		return nil, err
	}
	return resp, nil
}

// ActivityTypeMigrator will register or deprecate the configured activity types as required.
type ActivityTypeMigrator struct {
	RegisteredActivityTypes []swf.RegisterActivityTypeInput
	DeprecatedActivityTypes []swf.DeprecateActivityTypeInput
	Client                  SWFOps
}

// Migrate asserts that DeprecatedActivityTypes are deprecated or deprecates them, then asserts that RegisteredActivityTypes are registered or registers them.
func (a *ActivityTypeMigrator) Migrate() {
	for _, d := range a.DeprecatedActivityTypes {
		if a.isDeprecated(d.Domain, d.ActivityType.Name, d.ActivityType.Version) {
			log.Printf("action=migrate at=deprecate-activity domain=%s activity=%s version=%s status=previously-deprecated", LS(d.Domain), LS(d.ActivityType.Name), LS(d.ActivityType.Version))
		} else {
			a.deprecate(d)
			log.Printf("action=migrate at=depreacate-activity domain=%s activity=%s version=%s status=deprecated", LS(d.Domain), LS(d.ActivityType.Name), LS(d.ActivityType.Version))
		}
	}
	for _, r := range a.RegisteredActivityTypes {
		if a.isRegisteredNotDeprecated(r) {
			log.Printf("action=migrate at=register-activity domain=%s activity=%s version=%s status=previously-registered", LS(r.Domain), LS(r.Name), LS(r.Version))
		} else {
			a.register(r)
			log.Printf("action=migrate at=register-activity domain=%s activity=%s version=%s status=registered", LS(r.Domain), LS(r.Name), LS(r.Version))
		}
	}
}

func (a *ActivityTypeMigrator) isRegisteredNotDeprecated(rd swf.RegisterActivityTypeInput) bool {
	desc, err := a.describe(rd.Domain, rd.Name, rd.Version)
	if err != nil {
		if ae, ok := err.(aws.APIError); ok && ae.Type == ErrorTypeUnknownResourceFault {
			return false
		}

		panicWithError(err)

	}

	return *desc.TypeInfo.Status == swf.RegistrationStatusRegistered
}

func (a *ActivityTypeMigrator) register(rd swf.RegisterActivityTypeInput) {
	err := a.Client.RegisterActivityType(&rd)
	if err != nil {
		panicWithError(err)
	}
}

func (a *ActivityTypeMigrator) isDeprecated(domain aws.StringValue, name aws.StringValue, version aws.StringValue) bool {
	desc, err := a.describe(domain, name, version)
	if err != nil {
		log.Printf("action=migrate at=is-dep domain=%s activity=%s version=%s error=%s", LS(domain), LS(name), LS(version), err.Error())
		return false
	}

	return *desc.TypeInfo.Status == swf.RegistrationStatusDeprecated
}

func (a *ActivityTypeMigrator) deprecate(dd swf.DeprecateActivityTypeInput) {
	err := a.Client.DeprecateActivityType(&dd)
	if err != nil {
		panicWithError(err)
	}
}

func (a *ActivityTypeMigrator) describe(domain aws.StringValue, name aws.StringValue, version aws.StringValue) (*swf.ActivityTypeDetail, error) {
	resp, err := a.Client.DescribeActivityType(&swf.DescribeActivityTypeInput{Domain: domain, ActivityType: &swf.ActivityType{Name: name, Version: version}})
	if err != nil {
		return nil, err
	}
	return resp, nil
}

// StreamMigrator will create any Kinesis Streams required.
type StreamMigrator struct {
	Streams []kinesis.CreateStreamInput
	Client  KinesisOps
}

// Migrate checks that the desired streams have been created and if they have not, creates them.s
func (s *StreamMigrator) Migrate() {
	for _, st := range s.Streams {
		if s.isCreated(st) {
			log.Printf("action=migrate at=create-stream stream=%s status=previously-created", LS(st.StreamName))
		} else {
			s.create(st)
			log.Printf("action=migrate at=create-stream stream=%s status=created", LS(st.StreamName))
		}
		s.awaitActive(st.StreamName, 30)
	}
}

func (s *StreamMigrator) isCreated(st kinesis.CreateStreamInput) bool {
	_, err := s.describe(st)
	if err != nil {
		if ae, ok := err.(aws.APIError); ok && ae.Type == ErrorTypeStreamNotFound {
			return false
		}
		panicWithError(err)

	}

	return true
}

func (s *StreamMigrator) create(st kinesis.CreateStreamInput) {
	err := s.Client.CreateStream(&st)
	if err != nil {
		panicWithError(err)
	}
}

func (s *StreamMigrator) describe(st kinesis.CreateStreamInput) (*kinesis.DescribeStreamOutput, error) {
	req := kinesis.DescribeStreamInput{
		StreamName: st.StreamName,
	}
	resp, err := s.Client.DescribeStream(&req)
	if err != nil {
		return nil, err
	}
	return resp, nil
}

func (s *StreamMigrator) awaitActive(stream aws.StringValue, atMostSeconds int) {

	waited := 0
	status := kinesis.StreamStatusCreating
	for status != kinesis.StreamStatusActive {
		desc, err := s.Client.DescribeStream(&kinesis.DescribeStreamInput{
			StreamName: stream,
		})
		if err != nil {
			log.Printf("component=kinesis-migrator fn=awaitActive at=describe-error error=%s", err)
			panicWithError(err)
		}
		log.Printf("component=kinesis-migrator fn=awaitActive stream=%s at=describe status=%s", *stream, *desc.StreamDescription.StreamStatus)
		status = *desc.StreamDescription.StreamStatus
		time.Sleep(1 * time.Second)
		waited++
		if waited >= atMostSeconds {
			log.Printf("component=kinesis-migrator fn=awaitActive streal=%s  at=error error=exeeeded-max-wait", *stream)
			panic("waited too long")
		}
	}
}

func panicWithError(err error){
    if ae, ok := err.(aws.APIError); ok {
        panic(fmt.Sprintf("aws error while migrating type=%s message=%s code=%s request-id=%s", ae.Type, ae.Message, ae.Code, ae.RequestID))
    }

    panic(err)
}
<|endoftext|>


<commit_before>package irckit

import (
	"fmt"
	"net"
	"strings"
	"sync"
	"time"

	"github.com/42wim/matterircd/bridge"
	"github.com/desertbit/timer"
	"github.com/sorcix/irc"
	"github.com/spf13/viper"
)

// NewUser creates a *User, wrapping a connection with metadata we need for our server.
func NewUser(c Conn) *User {
	return &User{
		Conn: c,
		UserInfo: &bridge.UserInfo{
			Host: "*",
		},
		channels: map[Channel]struct{}{},
		DecodeCh: make(chan *irc.Message),
	}
}

// NewUserNet creates a *User from a net.Conn connection.
func NewUserNet(c net.Conn) *User {
	return NewUser(&conn{
		Conn:    c,
		Encoder: irc.NewEncoder(c),
		Decoder: irc.NewDecoder(c),
	})
}

const defaultCloseMsg = "Closed."

type User struct {
	Conn

	sync.RWMutex
	*bridge.UserInfo

	BufferedMsg *irc.Message
	DecodeCh    chan *irc.Message

	channels map[Channel]struct{}

	v *viper.Viper

	UserBridge
}

func (u *User) ID() string {
	// return strings.ToLower(u.Nick)
	return strings.ToLower(u.User)
}

func (u *User) Prefix() *irc.Prefix {
	return &irc.Prefix{
		Name: u.Nick,
		User: u.User,
		Host: u.Host,
	}
}

func (u *User) Close() error {
	for ch := range u.channels {
		ch.Part(u, defaultCloseMsg)
	}
	return u.Conn.Close()
}

func (u *User) String() string {
	return u.Prefix().String()
}

func (u *User) NumChannels() int {
	u.RLock()
	defer u.RUnlock()
	return len(u.channels)
}

func (u *User) Channels() []Channel {
	u.RLock()
	channels := make([]Channel, 0, len(u.channels))
	for ch := range u.channels {
		channels = append(channels, ch)
	}
	u.RUnlock()
	return channels
}

func (u *User) VisibleTo() []*User {
	seen := map[*User]struct{}{}
	seen[u] = struct{}{}

	channels := u.Channels()
	num := 0
	for _, ch := range channels {
		// Don't include self
		num += ch.Len()
	}

	// Pre-allocate
	users := make([]*User, 0, num)
	if num == 0 {
		return users
	}

	// Get all unique users
	for _, ch := range channels {
		for _, other := range ch.Users() {
			if _, dupe := seen[other]; dupe {
				continue
			}
			seen[other] = struct{}{}
			// TODO: Check visibility (once it's implemented)
			users = append(users, other)
		}
	}
	return users
}

// Encode and send each msg until an error occurs, then returns.
func (u *User) Encode(msgs ...*irc.Message) (err error) {
	if u.Ghost {
		return nil
	}

	for _, msg := range msgs {
		if msg.Command == "PRIVMSG" && (msg.Prefix.Name == "slack" || msg.Prefix.Name == "mattermost") && msg.Prefix.Host == "service" && strings.Contains(msg.Trailing, "token") {
			logger.Debugf("-> %s %s %s", msg.Command, msg.Prefix.Name, "[token redacted]")

			err := u.Conn.Encode(msg)
			if err != nil {
				return err
			}

			continue
		}

		logger.Debugf("-> %s", msg)

		err := u.Conn.Encode(msg)
		if err != nil {
			return err
		}
	}

	return nil
}

// Decode will receive and return a decoded message, or an error.
// nolint:funlen,gocognit,gocyclo
func (u *User) Decode() {
	if u.Ghost {
		// block
		c := make(chan struct{})
		<-c
	}
	buffer := make(chan *irc.Message)
	stop := make(chan struct{})
	bufferTimeout := u.v.GetInt("PasteBufferTimeout")
	// we need at least 100
	if bufferTimeout < 100 {
		bufferTimeout = 100
	}
	logger.Debugf("using paste buffer timeout: %#v\n", bufferTimeout)
	t := timer.NewTimer(time.Duration(bufferTimeout) * time.Millisecond)
	t.Stop()
	go func(buffer chan *irc.Message, stop chan struct{}) {
		for {
			select {
			case msg := <-buffer:
				// are we starting a new buffer ?
				if u.BufferedMsg == nil {
					u.BufferedMsg = msg
					// start timer now
					t.Reset(time.Duration(bufferTimeout) * time.Millisecond)
				} else {
					if strings.HasPrefix(msg.Trailing, "\x01ACTION") {
						// flush buffer
						logger.Debug("flushing buffer because of /me")
						u.BufferedMsg.Trailing = strings.TrimSpace(u.BufferedMsg.Trailing)
						u.DecodeCh <- u.BufferedMsg
						u.BufferedMsg = nil
						// send CTCP message
						u.DecodeCh <- msg
						continue
					}
					// make sure we're sending to the same recipient in the buffer
					if u.BufferedMsg.Params[0] == msg.Params[0] {
						u.BufferedMsg.Trailing += "\n" + msg.Trailing
					} else {
						u.DecodeCh <- msg
					}
				}
			case <-t.C:
				if u.BufferedMsg != nil {
					// trim last newline
					u.BufferedMsg.Trailing = strings.TrimSpace(u.BufferedMsg.Trailing)
					logger.Debugf("flushing buffer: %#v\n", u.BufferedMsg)
					u.DecodeCh <- u.BufferedMsg
					// clear buffer
					u.BufferedMsg = nil
					t.Stop()
				}
			case <-stop:
				logger.Debug("closing decode()")
				return
			}
		}
	}(buffer, stop)
	for {
		msg, err := u.Conn.Decode()
		if err != nil {
			close(stop)
			if err.Error() != "EOF" {
				logger.Errorf("msg: %s err: %s", msg, err)
			}
			break
		}

		if msg == nil {
			continue
		}

		dmsg := fmt.Sprintf("<- %s", msg)
		if msg.Command == "PRIVMSG" && msg.Params != nil && (msg.Params[0] == "slack" || msg.Params[0] == "mattermost") {
			// Don't log sensitive information
			trail := strings.Split(msg.Trailing, " ")
			if (msg.Trailing != "" && trail[0] == "login") || (len(msg.Params) > 1 && msg.Params[1] == "login") {
				dmsg = fmt.Sprintf("<- PRIVMSG %s :login [redacted]", msg.Params[0])
			}
		}
		// PRIVMSG can be buffered
		if msg.Command == "PRIVMSG" {
			logger.Debugf("B: %#v\n", dmsg)
			buffer <- msg
		} else {
			logger.Debug(dmsg)
			u.DecodeCh <- msg
		}
	}
}

func (u *User) createService(nick string, what string) {
	u.CreateUserFromInfo(
		&bridge.UserInfo{
			Nick:  nick,
			User:  nick,
			Real:  what,
			Host:  "service",
			Ghost: true,
		})
}
<commit_msg>Flush buffers on reactions, replies to threads or message modifications (#387)<commit_after>package irckit

import (
	"fmt"
	"net"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/42wim/matterircd/bridge"
	"github.com/desertbit/timer"
	"github.com/sorcix/irc"
	"github.com/spf13/viper"
)

// NewUser creates a *User, wrapping a connection with metadata we need for our server.
func NewUser(c Conn) *User {
	return &User{
		Conn: c,
		UserInfo: &bridge.UserInfo{
			Host: "*",
		},
		channels: map[Channel]struct{}{},
		DecodeCh: make(chan *irc.Message),
	}
}

// NewUserNet creates a *User from a net.Conn connection.
func NewUserNet(c net.Conn) *User {
	return NewUser(&conn{
		Conn:    c,
		Encoder: irc.NewEncoder(c),
		Decoder: irc.NewDecoder(c),
	})
}

const defaultCloseMsg = "Closed."

type User struct {
	Conn

	sync.RWMutex
	*bridge.UserInfo

	BufferedMsg *irc.Message
	DecodeCh    chan *irc.Message

	channels map[Channel]struct{}

	v *viper.Viper

	UserBridge
}

func (u *User) ID() string {
	// return strings.ToLower(u.Nick)
	return strings.ToLower(u.User)
}

func (u *User) Prefix() *irc.Prefix {
	return &irc.Prefix{
		Name: u.Nick,
		User: u.User,
		Host: u.Host,
	}
}

func (u *User) Close() error {
	for ch := range u.channels {
		ch.Part(u, defaultCloseMsg)
	}
	return u.Conn.Close()
}

func (u *User) String() string {
	return u.Prefix().String()
}

func (u *User) NumChannels() int {
	u.RLock()
	defer u.RUnlock()
	return len(u.channels)
}

func (u *User) Channels() []Channel {
	u.RLock()
	channels := make([]Channel, 0, len(u.channels))
	for ch := range u.channels {
		channels = append(channels, ch)
	}
	u.RUnlock()
	return channels
}

func (u *User) VisibleTo() []*User {
	seen := map[*User]struct{}{}
	seen[u] = struct{}{}

	channels := u.Channels()
	num := 0
	for _, ch := range channels {
		// Don't include self
		num += ch.Len()
	}

	// Pre-allocate
	users := make([]*User, 0, num)
	if num == 0 {
		return users
	}

	// Get all unique users
	for _, ch := range channels {
		for _, other := range ch.Users() {
			if _, dupe := seen[other]; dupe {
				continue
			}
			seen[other] = struct{}{}
			// TODO: Check visibility (once it's implemented)
			users = append(users, other)
		}
	}
	return users
}

// Encode and send each msg until an error occurs, then returns.
func (u *User) Encode(msgs ...*irc.Message) (err error) {
	if u.Ghost {
		return nil
	}

	for _, msg := range msgs {
		if msg.Command == "PRIVMSG" && (msg.Prefix.Name == "slack" || msg.Prefix.Name == "mattermost") && msg.Prefix.Host == "service" && strings.Contains(msg.Trailing, "token") {
			logger.Debugf("-> %s %s %s", msg.Command, msg.Prefix.Name, "[token redacted]")

			err := u.Conn.Encode(msg)
			if err != nil {
				return err
			}

			continue
		}

		logger.Debugf("-> %s", msg)

		err := u.Conn.Encode(msg)
		if err != nil {
			return err
		}
	}

	return nil
}

// Decode will receive and return a decoded message, or an error.
// nolint:funlen,gocognit,gocyclo
func (u *User) Decode() {
	if u.Ghost {
		// block
		c := make(chan struct{})
		<-c
	}
	buffer := make(chan *irc.Message)
	stop := make(chan struct{})
	bufferTimeout := u.v.GetInt("PasteBufferTimeout")
	// we need at least 100
	if bufferTimeout < 100 {
		bufferTimeout = 100
	}
	logger.Debugf("using paste buffer timeout: %#v\n", bufferTimeout)
	t := timer.NewTimer(time.Duration(bufferTimeout) * time.Millisecond)
	t.Stop()
	go func(buffer chan *irc.Message, stop chan struct{}) {
		for {
			select {
			case msg := <-buffer:
				// are we starting a new buffer ?
				if u.BufferedMsg == nil {
					u.BufferedMsg = msg
					// start timer now
					t.Reset(time.Duration(bufferTimeout) * time.Millisecond)
				} else {
					re := regexp.MustCompile(`^(?:\@\@|s/)(?:[0-9a-f]{3}|[0-9a-z]{26}|!!)|/`)
					if strings.HasPrefix(msg.Trailing, "\x01ACTION") || re.MatchString(msg.Trailing) {
						// flush buffer
						logger.Debug("flushing buffer because of /me, replies to threads, and message modifications")
						u.BufferedMsg.Trailing = strings.TrimSpace(u.BufferedMsg.Trailing)
						u.DecodeCh <- u.BufferedMsg
						u.BufferedMsg = nil
						// send CTCP message
						u.DecodeCh <- msg
						continue
					}
					// make sure we're sending to the same recipient in the buffer
					if u.BufferedMsg.Params[0] == msg.Params[0] {
						u.BufferedMsg.Trailing += "\n" + msg.Trailing
					} else {
						u.DecodeCh <- msg
					}
				}
			case <-t.C:
				if u.BufferedMsg != nil {
					// trim last newline
					u.BufferedMsg.Trailing = strings.TrimSpace(u.BufferedMsg.Trailing)
					logger.Debugf("flushing buffer: %#v\n", u.BufferedMsg)
					u.DecodeCh <- u.BufferedMsg
					// clear buffer
					u.BufferedMsg = nil
					t.Stop()
				}
			case <-stop:
				logger.Debug("closing decode()")
				return
			}
		}
	}(buffer, stop)
	for {
		msg, err := u.Conn.Decode()
		if err != nil {
			close(stop)
			if err.Error() != "EOF" {
				logger.Errorf("msg: %s err: %s", msg, err)
			}
			break
		}

		if msg == nil {
			continue
		}

		dmsg := fmt.Sprintf("<- %s", msg)
		if msg.Command == "PRIVMSG" && msg.Params != nil && (msg.Params[0] == "slack" || msg.Params[0] == "mattermost") {
			// Don't log sensitive information
			trail := strings.Split(msg.Trailing, " ")
			if (msg.Trailing != "" && trail[0] == "login") || (len(msg.Params) > 1 && msg.Params[1] == "login") {
				dmsg = fmt.Sprintf("<- PRIVMSG %s :login [redacted]", msg.Params[0])
			}
		}
		// PRIVMSG can be buffered
		if msg.Command == "PRIVMSG" {
			logger.Debugf("B: %#v\n", dmsg)
			buffer <- msg
		} else {
			logger.Debug(dmsg)
			u.DecodeCh <- msg
		}
	}
}

func (u *User) createService(nick string, what string) {
	u.CreateUserFromInfo(
		&bridge.UserInfo{
			Nick:  nick,
			User:  nick,
			Real:  what,
			Host:  "service",
			Ghost: true,
		})
}
<|endoftext|>