<commit_before>package alicloud

import (
	"fmt"

	"github.com/denverdino/aliyungo/common"
	"github.com/denverdino/aliyungo/ecs"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
	"log"
	"time"
)

func resourceAliyunSecurityGroup() *schema.Resource {
	return &schema.Resource{
		Create: resourceAliyunSecurityGroupCreate,
		Read:   resourceAliyunSecurityGroupRead,
		Update: resourceAliyunSecurityGroupUpdate,
		Delete: resourceAliyunSecurityGroupDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:         schema.TypeString,
				Optional:     true,
				ValidateFunc: validateSecurityGroupName,
			},

			"description": &schema.Schema{
				Type:         schema.TypeString,
				Optional:     true,
				ValidateFunc: validateSecurityGroupDescription,
			},

			"vpc_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
		},
	}
}

func resourceAliyunSecurityGroupCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AliyunClient).ecsconn

	args, err := buildAliyunSecurityGroupArgs(d, meta)
	if err != nil {
		return err
	}

	securityGroupID, err := conn.CreateSecurityGroup(args)
	if err != nil {
		return err
	}

	d.SetId(securityGroupID)
	return resourceAliyunSecurityGroupRead(d, meta)
}

func resourceAliyunSecurityGroupRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AliyunClient).ecsconn

	args := &ecs.DescribeSecurityGroupAttributeArgs{
		SecurityGroupId: d.Id(),
		RegionId:        getRegion(d, meta),
	}
	//err := resource.Retry(3*time.Minute, func() *resource.RetryError {
	var sg *ecs.DescribeSecurityGroupAttributeResponse
	err := resource.Retry(3*time.Minute, func() *resource.RetryError {
		group, e := conn.DescribeSecurityGroupAttribute(args)
		if e != nil && !NotFoundError(e) {
			return resource.NonRetryableError(fmt.Errorf("Error DescribeSecurityGroupAttribute: %#v", e))
		}
		if group != nil {
			sg = group
			return nil
		}
		return resource.RetryableError(fmt.Errorf("Security group is creating - try again while describe security group"))
	})
	//sg, err := conn.DescribeSecurityGroupAttribute(args)
	//if err != nil {
	//	if NotFoundError(err) {
	//		d.SetId("")
	//		return nil
	//	}
	//	return fmt.Errorf("Error DescribeSecurityGroupAttribute: %#v", err)
	//}
	if err != nil {
		return err
	}
	if sg == nil {
		d.SetId("")
		return nil
	}

	d.Set("name", sg.SecurityGroupName)
	d.Set("description", sg.Description)

	return nil
}

func resourceAliyunSecurityGroupUpdate(d *schema.ResourceData, meta interface{}) error {

	conn := meta.(*AliyunClient).ecsconn

	d.Partial(true)
	attributeUpdate := false
	args := &ecs.ModifySecurityGroupAttributeArgs{
		SecurityGroupId: d.Id(),
		RegionId:        getRegion(d, meta),
	}

	if d.HasChange("name") {
		d.SetPartial("name")
		args.SecurityGroupName = d.Get("name").(string)

		attributeUpdate = true
	}

	if d.HasChange("description") {
		d.SetPartial("description")
		args.Description = d.Get("description").(string)

		attributeUpdate = true
	}
	if attributeUpdate {
		if err := conn.ModifySecurityGroupAttribute(args); err != nil {
			return err
		}
	}

	return nil
}

func resourceAliyunSecurityGroupDelete(d *schema.ResourceData, meta interface{}) error {

	conn := meta.(*AliyunClient).ecsconn

	return resource.Retry(5*time.Minute, func() *resource.RetryError {
		err := conn.DeleteSecurityGroup(getRegion(d, meta), d.Id())

		if err != nil {
			e, _ := err.(*common.Error)
			if e.ErrorResponse.Code == SgDependencyViolation {
				return resource.RetryableError(fmt.Errorf("Security group in use - trying again while it is deleted."))
			}
		}

		sg, err := conn.DescribeSecurityGroupAttribute(&ecs.DescribeSecurityGroupAttributeArgs{
			RegionId:        getRegion(d, meta),
			SecurityGroupId: d.Id(),
		})

		if err != nil {
			e, _ := err.(*common.Error)
			if e.ErrorResponse.Code == InvalidSecurityGroupIdNotFound {
				return nil
			}
			return resource.NonRetryableError(err)
		} else if sg == nil {
			return nil
		}

		return resource.RetryableError(fmt.Errorf("Security group in use - trying again while it is deleted."))
	})

}

func buildAliyunSecurityGroupArgs(d *schema.ResourceData, meta interface{}) (*ecs.CreateSecurityGroupArgs, error) {

	args := &ecs.CreateSecurityGroupArgs{
		RegionId: getRegion(d, meta),
	}

	if v := d.Get("name").(string); v != "" {
		args.SecurityGroupName = v
	}

	if v := d.Get("description").(string); v != "" {
		args.Description = v
	}

	if v := d.Get("vpc_id").(string); v != "" {
		args.VpcId = v
	}

	return args, nil
}
<commit_msg>remove useless annotation<commit_after>package alicloud

import (
	"fmt"

	"github.com/denverdino/aliyungo/common"
	"github.com/denverdino/aliyungo/ecs"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
	"log"
	"time"
)

func resourceAliyunSecurityGroup() *schema.Resource {
	return &schema.Resource{
		Create: resourceAliyunSecurityGroupCreate,
		Read:   resourceAliyunSecurityGroupRead,
		Update: resourceAliyunSecurityGroupUpdate,
		Delete: resourceAliyunSecurityGroupDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:         schema.TypeString,
				Optional:     true,
				ValidateFunc: validateSecurityGroupName,
			},

			"description": &schema.Schema{
				Type:         schema.TypeString,
				Optional:     true,
				ValidateFunc: validateSecurityGroupDescription,
			},

			"vpc_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
		},
	}
}

func resourceAliyunSecurityGroupCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AliyunClient).ecsconn

	args, err := buildAliyunSecurityGroupArgs(d, meta)
	if err != nil {
		return err
	}

	securityGroupID, err := conn.CreateSecurityGroup(args)
	if err != nil {
		return err
	}

	d.SetId(securityGroupID)
	return resourceAliyunSecurityGroupRead(d, meta)
}

func resourceAliyunSecurityGroupRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AliyunClient).ecsconn

	args := &ecs.DescribeSecurityGroupAttributeArgs{
		SecurityGroupId: d.Id(),
		RegionId:        getRegion(d, meta),
	}
	//err := resource.Retry(3*time.Minute, func() *resource.RetryError {
	var sg *ecs.DescribeSecurityGroupAttributeResponse
	err := resource.Retry(3*time.Minute, func() *resource.RetryError {
		group, e := conn.DescribeSecurityGroupAttribute(args)
		if e != nil && !NotFoundError(e) {
			return resource.NonRetryableError(fmt.Errorf("Error DescribeSecurityGroupAttribute: %#v", e))
		}
		if group != nil {
			sg = group
			return nil
		}
		return resource.RetryableError(fmt.Errorf("Security group is creating - try again while describe security group"))
	})

	if err != nil {
		return err
	}
	if sg == nil {
		d.SetId("")
		return nil
	}

	d.Set("name", sg.SecurityGroupName)
	d.Set("description", sg.Description)

	return nil
}

func resourceAliyunSecurityGroupUpdate(d *schema.ResourceData, meta interface{}) error {

	conn := meta.(*AliyunClient).ecsconn

	d.Partial(true)
	attributeUpdate := false
	args := &ecs.ModifySecurityGroupAttributeArgs{
		SecurityGroupId: d.Id(),
		RegionId:        getRegion(d, meta),
	}

	if d.HasChange("name") {
		d.SetPartial("name")
		args.SecurityGroupName = d.Get("name").(string)

		attributeUpdate = true
	}

	if d.HasChange("description") {
		d.SetPartial("description")
		args.Description = d.Get("description").(string)

		attributeUpdate = true
	}
	if attributeUpdate {
		if err := conn.ModifySecurityGroupAttribute(args); err != nil {
			return err
		}
	}

	return nil
}

func resourceAliyunSecurityGroupDelete(d *schema.ResourceData, meta interface{}) error {

	conn := meta.(*AliyunClient).ecsconn

	return resource.Retry(5*time.Minute, func() *resource.RetryError {
		err := conn.DeleteSecurityGroup(getRegion(d, meta), d.Id())

		if err != nil {
			e, _ := err.(*common.Error)
			if e.ErrorResponse.Code == SgDependencyViolation {
				return resource.RetryableError(fmt.Errorf("Security group in use - trying again while it is deleted."))
			}
		}

		sg, err := conn.DescribeSecurityGroupAttribute(&ecs.DescribeSecurityGroupAttributeArgs{
			RegionId:        getRegion(d, meta),
			SecurityGroupId: d.Id(),
		})

		if err != nil {
			e, _ := err.(*common.Error)
			if e.ErrorResponse.Code == InvalidSecurityGroupIdNotFound {
				return nil
			}
			return resource.NonRetryableError(err)
		} else if sg == nil {
			return nil
		}

		return resource.RetryableError(fmt.Errorf("Security group in use - trying again while it is deleted."))
	})

}

func buildAliyunSecurityGroupArgs(d *schema.ResourceData, meta interface{}) (*ecs.CreateSecurityGroupArgs, error) {

	args := &ecs.CreateSecurityGroupArgs{
		RegionId: getRegion(d, meta),
	}

	if v := d.Get("name").(string); v != "" {
		args.SecurityGroupName = v
	}

	if v := d.Get("description").(string); v != "" {
		args.Description = v
	}

	if v := d.Get("vpc_id").(string); v != "" {
		args.VpcId = v
	}

	return args, nil
}
<|endoftext|>


<commit_before>// ############## Page 19 ##############
package main

import "fmt"

type Gear struct {
  Chainring float64
  Cog       float64
}

func NewGear(chainring, cog float64) Gear {
  return Gear{chainring, cog}
}

func (gear Gear) Ratio() float64 {
  return gear.Chainring / gear.Cog
}

func main() {
  fmt.Println(NewGear(52, 11).Ratio()) // => 4.7272727272727275
  fmt.Println(NewGear(30, 27).Ratio()) // => 1.1111111111111112
}
<commit_msg>number of teeth is most likely a natural number<commit_after>// ############## Page 19 ##############
package main

import "fmt"

type Gear struct {
  Chainring int // number of teath
  Cog       int
}

func NewGear(chainring, cog int) Gear {
  return Gear{chainring, cog}
}

func (gear Gear) Ratio() float64 {
  return float64(gear.Chainring) / float64(gear.Cog)
}

func main() {
  fmt.Println(NewGear(52, 11).Ratio()) // => 4.7272727272727275
  fmt.Println(NewGear(30, 27).Ratio()) // => 1.1111111111111112
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"net/url"
	"path"
	"regexp"
	"strings"

	"github.com/PuerkitoBio/goquery"
	log "github.com/Sirupsen/logrus"
	"github.com/alphagov/govuk_crawler_worker/http_crawler"
	"github.com/streadway/amqp"
)

type CrawlerMessageItem struct {
	amqp.Delivery
	Response *http_crawler.CrawlerResponse

	rootURL        *url.URL
	blacklistPaths []string
}

func NewCrawlerMessageItem(delivery amqp.Delivery, rootURL *url.URL, blacklistPaths []string) *CrawlerMessageItem {
	return &CrawlerMessageItem{
		Delivery:       delivery,
		rootURL:        rootURL,
		blacklistPaths: blacklistPaths,
	}
}

func (c *CrawlerMessageItem) URL() string {
	return string(c.Body)
}

func (c *CrawlerMessageItem) RelativeFilePath() (string, error) {
	var filePath string

	urlParts, err := url.Parse(c.URL())
	if err != nil {
		return "", err
	}

	filePath = urlParts.Path

	contentType, err := c.Response.ParseContentType()
	if err != nil {
		return "", err
	}

	if contentType == http_crawler.HTML {
		r, err := regexp.Compile(`.(html|htm)$`)

		if err != nil {
			return "", err
		}

		switch {
		case strings.HasSuffix(filePath, "/"):
			filePath += "index.html"
		case !r.MatchString(filePath): // extension not .html or .htm
			filePath += ".html"
		}
	}

	filePath = path.Clean(filePath)
	filePath = strings.TrimPrefix(filePath, "/")

	return filePath, nil
}

func (c *CrawlerMessageItem) ExtractURLs() ([]*url.URL, error) {
	extractedURLs := []*url.URL{}

	document, err := goquery.NewDocumentFromReader(bytes.NewBuffer(c.Response.Body))
	if err != nil {
		return extractedURLs, err
	}

	urlElementMatches := [][]string{
		[]string{"a", "href"},
		[]string{"img", "src"},
		[]string{"link", "href"},
		[]string{"script", "src"},
	}

	var hrefs []string
	var urls []*url.URL

	for _, attr := range urlElementMatches {
		element, attr := attr[0], attr[1]

		hrefs = findHrefsByElementAttribute(document, element, attr)
		urls, err = parseURLs(hrefs)

		if err != nil {
			return extractedURLs, err
		}

		urls = convertURLsToAbsolute(c.rootURL, urls)
		urls = filterURLsByHost(c.rootURL.Host, urls)
		urls = filterBlacklistedURLs(c.blacklistPaths, urls)
		urls = removeFragmentFromURLs(urls)

		extractedURLs = append(extractedURLs, urls...)
	}

	extractedURLs = filterDuplicateURLs(extractedURLs)

	return extractedURLs, err
}

func (c *CrawlerMessageItem) IsBlacklisted() bool {
	urlParts, err := url.Parse(c.URL())
	if err != nil {
		log.Warningln("Malformed URL", c.URL())
		return false
	}
	return isBlacklistedPath(urlParts.Path, c.blacklistPaths)
}

func parseURLs(urls []string) ([]*url.URL, error) {
	var parsedURLs []*url.URL
	var err error

	for _, u := range urls {
		u, err := url.Parse(u)
		if err != nil {
			return parsedURLs, err
		}
		parsedURLs = append(parsedURLs, u)
	}

	return parsedURLs, err
}

func convertURLsToAbsolute(rootURL *url.URL, urls []*url.URL) []*url.URL {
	return mapURLs(urls, func(url *url.URL) *url.URL {
		return rootURL.ResolveReference(url)
	})
}

func removeFragmentFromURLs(urls []*url.URL) []*url.URL {
	return mapURLs(urls, func(url *url.URL) *url.URL {
		url.Fragment = ""
		return url
	})
}

func filterURLsByHost(host string, urls []*url.URL) []*url.URL {
	return filterURLs(urls, func(url *url.URL) bool {
		return url.Host == host
	})
}

func filterBlacklistedURLs(blacklistedPaths []string, urls []*url.URL) []*url.URL {
	return filterURLs(urls, func(url *url.URL) bool {
		return !isBlacklistedPath(url.Path, blacklistedPaths)
	})
}

func filterDuplicateURLs(urls []*url.URL) []*url.URL {
	urlMap := make(map[string]*url.URL)
	for _, url := range urls {
		urlMap[url.String()] = url
	}

	uniqueUrls := make([]*url.URL, 0, len(urlMap))
	for _, url := range urlMap {
		uniqueUrls = append(uniqueUrls, url)
	}

	return uniqueUrls
}

// Filter an array of *url.URL objects based on a filter function that
// returns a boolean. Only elements that return true for this filter
// function will be kept. Returns a new array.
func filterURLs(urls []*url.URL, filterFunc func(u *url.URL) bool) []*url.URL {
	var filteredURLs []*url.URL

	for _, url := range urls {
		if filterFunc(url) {
			filteredURLs = append(filteredURLs, url)
		}
	}

	return filteredURLs
}

// Map a function to each element of a *url.URL array. Returns a new
// array but will edit any url.URL objects in place should the mapFunc
// mutate state.
func mapURLs(urls []*url.URL, mapFunc func(u *url.URL) *url.URL) []*url.URL {
	for index, url := range urls {
		urls[index] = mapFunc(url)
	}

	return urls
}

func findHrefsByElementAttribute(
	document *goquery.Document,
	element string,
	attr string) []string {

	hrefs := []string{}

	document.Find(element).Each(func(_ int, element *goquery.Selection) {
		href, _ := element.Attr(attr)
		unescapedHref, _ := url.QueryUnescape(href)
		trimmedHref := strings.TrimSpace(unescapedHref)
		hrefs = append(hrefs, trimmedHref)
	})

	return hrefs
}

func isBlacklistedPath(path string, blacklistedPaths []string) bool {
	for _, blacklistedPath := range blacklistedPaths {
		if strings.HasPrefix(path, blacklistedPath) {
			return true
		}
	}

	return false
}
<commit_msg>Simplify code in crawler_message_item.go<commit_after>package main

import (
	"bytes"
	"net/url"
	"path"
	"regexp"
	"strings"

	"github.com/PuerkitoBio/goquery"
	log "github.com/Sirupsen/logrus"
	"github.com/alphagov/govuk_crawler_worker/http_crawler"
	"github.com/streadway/amqp"
)

type CrawlerMessageItem struct {
	amqp.Delivery
	Response *http_crawler.CrawlerResponse

	rootURL        *url.URL
	blacklistPaths []string
}

func NewCrawlerMessageItem(delivery amqp.Delivery, rootURL *url.URL, blacklistPaths []string) *CrawlerMessageItem {
	return &CrawlerMessageItem{
		Delivery:       delivery,
		rootURL:        rootURL,
		blacklistPaths: blacklistPaths,
	}
}

func (c *CrawlerMessageItem) URL() string {
	return string(c.Body)
}

func (c *CrawlerMessageItem) RelativeFilePath() (string, error) {
	var filePath string

	urlParts, err := url.Parse(c.URL())
	if err != nil {
		return "", err
	}

	filePath = urlParts.Path

	contentType, err := c.Response.ParseContentType()
	if err != nil {
		return "", err
	}

	if contentType == http_crawler.HTML {
		r, err := regexp.Compile(`.(html|htm)$`)

		if err != nil {
			return "", err
		}

		switch {
		case strings.HasSuffix(filePath, "/"):
			filePath += "index.html"
		case !r.MatchString(filePath): // extension not .html or .htm
			filePath += ".html"
		}
	}

	filePath = path.Clean(filePath)
	filePath = strings.TrimPrefix(filePath, "/")

	return filePath, nil
}

func (c *CrawlerMessageItem) ExtractURLs() ([]*url.URL, error) {
	extractedURLs := []*url.URL{}

	document, err := goquery.NewDocumentFromReader(bytes.NewBuffer(c.Response.Body))
	if err != nil {
		return extractedURLs, err
	}

	urlElementMatches := [][]string{
		{"a", "href"},
		{"img", "src"},
		{"link", "href"},
		{"script", "src"},
	}

	var hrefs []string
	var urls []*url.URL

	for _, attr := range urlElementMatches {
		element, attr := attr[0], attr[1]

		hrefs = findHrefsByElementAttribute(document, element, attr)
		urls, err = parseURLs(hrefs)

		if err != nil {
			return extractedURLs, err
		}

		urls = convertURLsToAbsolute(c.rootURL, urls)
		urls = filterURLsByHost(c.rootURL.Host, urls)
		urls = filterBlacklistedURLs(c.blacklistPaths, urls)
		urls = removeFragmentFromURLs(urls)

		extractedURLs = append(extractedURLs, urls...)
	}

	extractedURLs = filterDuplicateURLs(extractedURLs)

	return extractedURLs, err
}

func (c *CrawlerMessageItem) IsBlacklisted() bool {
	urlParts, err := url.Parse(c.URL())
	if err != nil {
		log.Warningln("Malformed URL", c.URL())
		return false
	}
	return isBlacklistedPath(urlParts.Path, c.blacklistPaths)
}

func parseURLs(urls []string) ([]*url.URL, error) {
	var parsedURLs []*url.URL
	var err error

	for _, u := range urls {
		u, err := url.Parse(u)
		if err != nil {
			return parsedURLs, err
		}
		parsedURLs = append(parsedURLs, u)
	}

	return parsedURLs, err
}

func convertURLsToAbsolute(rootURL *url.URL, urls []*url.URL) []*url.URL {
	return mapURLs(urls, func(url *url.URL) *url.URL {
		return rootURL.ResolveReference(url)
	})
}

func removeFragmentFromURLs(urls []*url.URL) []*url.URL {
	return mapURLs(urls, func(url *url.URL) *url.URL {
		url.Fragment = ""
		return url
	})
}

func filterURLsByHost(host string, urls []*url.URL) []*url.URL {
	return filterURLs(urls, func(url *url.URL) bool {
		return url.Host == host
	})
}

func filterBlacklistedURLs(blacklistedPaths []string, urls []*url.URL) []*url.URL {
	return filterURLs(urls, func(url *url.URL) bool {
		return !isBlacklistedPath(url.Path, blacklistedPaths)
	})
}

func filterDuplicateURLs(urls []*url.URL) []*url.URL {
	urlMap := make(map[string]*url.URL)
	for _, url := range urls {
		urlMap[url.String()] = url
	}

	uniqueUrls := make([]*url.URL, 0, len(urlMap))
	for _, url := range urlMap {
		uniqueUrls = append(uniqueUrls, url)
	}

	return uniqueUrls
}

// Filter an array of *url.URL objects based on a filter function that
// returns a boolean. Only elements that return true for this filter
// function will be kept. Returns a new array.
func filterURLs(urls []*url.URL, filterFunc func(u *url.URL) bool) []*url.URL {
	var filteredURLs []*url.URL

	for _, url := range urls {
		if filterFunc(url) {
			filteredURLs = append(filteredURLs, url)
		}
	}

	return filteredURLs
}

// Map a function to each element of a *url.URL array. Returns a new
// array but will edit any url.URL objects in place should the mapFunc
// mutate state.
func mapURLs(urls []*url.URL, mapFunc func(u *url.URL) *url.URL) []*url.URL {
	for index, url := range urls {
		urls[index] = mapFunc(url)
	}

	return urls
}

func findHrefsByElementAttribute(
	document *goquery.Document,
	element string,
	attr string) []string {

	hrefs := []string{}

	document.Find(element).Each(func(_ int, element *goquery.Selection) {
		href, _ := element.Attr(attr)
		unescapedHref, _ := url.QueryUnescape(href)
		trimmedHref := strings.TrimSpace(unescapedHref)
		hrefs = append(hrefs, trimmedHref)
	})

	return hrefs
}

func isBlacklistedPath(path string, blacklistedPaths []string) bool {
	for _, blacklistedPath := range blacklistedPaths {
		if strings.HasPrefix(path, blacklistedPath) {
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>package runkeeper

import (
	"log"
	"time"

	runkeeper "github.com/svdberg/syncmysport-runkeeper/Godeps/_workspace/src/github.com/c9s/go-runkeeper"
	dm "github.com/svdberg/syncmysport-runkeeper/datamodel"
)

const API = "API"

func ConvertToActivity(rkActivity *runkeeper.FitnessActivity) *dm.Activity {
	returnActivity := dm.CreateActivity()
	if rkActivity.Type == "Other" {
		returnActivity.Type = "Activity"
	} else {
		returnActivity.Type = rkActivity.Type
	}

	//RK time is 'Local', convert to UTC
	//TODO. This might be wrong and perhaps should be the negative zone (e.g. -1 * ..)
	negativeOffset := -1 * rkActivity.UtcOffset * 60 * 60
	sourceLocation := time.FixedZone("RKSourceLocation", negativeOffset)
	correctedTime := time.Time(rkActivity.StartTime).In(sourceLocation)
	log.Printf("RK Local date: %s, start date: %s, unix: %d, offset: %d", time.Time(rkActivity.StartTime), correctedTime, time.Time(rkActivity.StartTime).Unix(), rkActivity.UtcOffset)
	returnActivity.StartTime = int(time.Time(correctedTime).Unix())
	returnActivity.UtcOffSet = rkActivity.UtcOffset
	returnActivity.Duration = int(rkActivity.Duration)
	returnActivity.Name = rkActivity.Notes
	returnActivity.Notes = rkActivity.Notes
	returnActivity.Private = false
	returnActivity.Stationary = rkActivity.HasMap
	returnActivity.AverageHeartRate = rkActivity.AverageHeartRate
	returnActivity.Calories = rkActivity.TotalCalories
	returnActivity.Distance = rkActivity.TotalDistance
	returnActivity.GPS = convertFromPath(rkActivity.Path)
	returnActivity.HeartRate = convertFromHR(rkActivity.HeartRate)

	// log.Printf("INPUT: %s, OUTPUT: %s", rkActivity, returnActivity)
	return returnActivity
}

func ConvertToRkActivity(activity *dm.Activity) *runkeeper.FitnessActivityNew {
	rkActivity := runkeeper.CreateNewFitnessActivity(activity.Name, float64(activity.Duration))

	rkActivity.Type = activity.Type
	//runkeeper only nows the following types:
	//Running, Cycling, Mountain Biking, Walking,
	//Hiking, Downhill Skiing, Cross-Country Skiing,
	//Snowboarding, Skating, Swimming, Wheelchair, Rowing, Elliptical, Other
	//
	//check if Type is one of these, otherwise Other.
	rkKnownTypes := map[string]string{
		"Running":              "Running",
		"Cycling":              "Cycling",
		"Mountain Biking":      "Mountain Biking",
		"Walking":              "Walking",
		"Hiking":               "Hiking",
		"Downhill Skiing":      "Downhill Skiing",
		"Cross-Country Skiing": "Cross-Country Skiing",
		"Snowboarding":         "Snowboarding",
		"Skating":              "Skating",
		"Swimming":             "Swimming",
		"Wheelchair":           "Wheelchair",
		"Rowing":               "Rowing",
		"Elliptical":           "Elliptical",
		"Other":                "Other"}

	_, ok := rkKnownTypes[activity.Type]
	if !ok {
		rkActivity.Type = "Other"
	}

	//runkeeper times are in local timezones, so covert back to the local time
	rkLocalLocation := time.FixedZone("rkZone", activity.UtcOffSet*60*60)
	rkActivity.StartTime = runkeeper.Time(time.Unix(int64(activity.StartTime), 0).In(rkLocalLocation))
	log.Printf("SMS time: %s, converted to RK time: %s for offset: %d", activity.StartTime, rkActivity.StartTime, activity.UtcOffSet)
	rkActivity.Notes = activity.Name
	rkActivity.TotalDistance = activity.Distance
	rkActivity.AverageHeartRate = activity.AverageHeartRate
	rkActivity.TotalCalories = activity.Calories
	rkActivity.Source = activity.Source
	rkActivity.EntryMode = API

	rkActivity.Path = convertToPath(activity.GPS)
	rkActivity.HeartRate = convertToHR(activity.HeartRate)
	return rkActivity
}

func convertToPath(gps []dm.GPS) []runkeeper.Path {
	rkPath := make([]runkeeper.Path, len(gps))
	for i, gp := range gps {
		rkPath[i] = runkeeper.Path{gp.Altitude, gp.Longitude, "gps", gp.Latitude, gp.Timestamp}
	}
	return rkPath
}

func convertFromPath(path []runkeeper.Path) []dm.GPS {
	dmPath := make([]dm.GPS, len(path))
	for i, rp := range path {
		dmPath[i] = dm.GPS{rp.Timestamp, rp.Altitude, rp.Longitude, rp.Latitude}
	}
	return dmPath
}

func convertToHR(hr []dm.HeartRate) []runkeeper.HeartRate {
	rkHr := make([]runkeeper.HeartRate, len(hr))
	for i, h := range hr {
		rkHr[i] = runkeeper.HeartRate{h.Timestamp, h.Heartrate}
	}
	return rkHr
}

func convertFromHR(rkHr []runkeeper.HeartRate) []dm.HeartRate {
	dmHr := make([]dm.HeartRate, len(rkHr))
	for i, h := range rkHr {
		dmHr[i] = dm.HeartRate{h.TimeStamp, h.HearRateNr}
	}
	return dmHr
}
<commit_msg>Subtract on the unix ts level<commit_after>package runkeeper

import (
	"log"
	"time"

	runkeeper "github.com/svdberg/syncmysport-runkeeper/Godeps/_workspace/src/github.com/c9s/go-runkeeper"
	dm "github.com/svdberg/syncmysport-runkeeper/datamodel"
)

const API = "API"

func ConvertToActivity(rkActivity *runkeeper.FitnessActivity) *dm.Activity {
	returnActivity := dm.CreateActivity()
	if rkActivity.Type == "Other" {
		returnActivity.Type = "Activity"
	} else {
		returnActivity.Type = rkActivity.Type
	}

	//RK time is 'Local', convert to UTC
	//TODO. This might be wrong and perhaps should be the negative zone (e.g. -1 * ..)
	negativeOffset := -1 * rkActivity.UtcOffset * 60 * 60
	// sourceLocation := time.FixedZone("RKSourceLocation", negativeOffset)
	// correctedTime := time.Time(rkActivity.StartTime).In(sourceLocation)
	log.Printf("RK Local date: %s, start date: %s, unix: %d, offset: %d", time.Time(rkActivity.StartTime), correctedTime, time.Time(rkActivity.StartTime).Unix(), rkActivity.UtcOffset)
	returnActivity.StartTime = int(rkActivity.StartTime.Unix() + negativeOffset)
	returnActivity.UtcOffSet = rkActivity.UtcOffset
	returnActivity.Duration = int(rkActivity.Duration)
	returnActivity.Name = rkActivity.Notes
	returnActivity.Notes = rkActivity.Notes
	returnActivity.Private = false
	returnActivity.Stationary = rkActivity.HasMap
	returnActivity.AverageHeartRate = rkActivity.AverageHeartRate
	returnActivity.Calories = rkActivity.TotalCalories
	returnActivity.Distance = rkActivity.TotalDistance
	returnActivity.GPS = convertFromPath(rkActivity.Path)
	returnActivity.HeartRate = convertFromHR(rkActivity.HeartRate)

	// log.Printf("INPUT: %s, OUTPUT: %s", rkActivity, returnActivity)
	return returnActivity
}

func ConvertToRkActivity(activity *dm.Activity) *runkeeper.FitnessActivityNew {
	rkActivity := runkeeper.CreateNewFitnessActivity(activity.Name, float64(activity.Duration))

	rkActivity.Type = activity.Type
	//runkeeper only nows the following types:
	//Running, Cycling, Mountain Biking, Walking,
	//Hiking, Downhill Skiing, Cross-Country Skiing,
	//Snowboarding, Skating, Swimming, Wheelchair, Rowing, Elliptical, Other
	//
	//check if Type is one of these, otherwise Other.
	rkKnownTypes := map[string]string{
		"Running":              "Running",
		"Cycling":              "Cycling",
		"Mountain Biking":      "Mountain Biking",
		"Walking":              "Walking",
		"Hiking":               "Hiking",
		"Downhill Skiing":      "Downhill Skiing",
		"Cross-Country Skiing": "Cross-Country Skiing",
		"Snowboarding":         "Snowboarding",
		"Skating":              "Skating",
		"Swimming":             "Swimming",
		"Wheelchair":           "Wheelchair",
		"Rowing":               "Rowing",
		"Elliptical":           "Elliptical",
		"Other":                "Other"}

	_, ok := rkKnownTypes[activity.Type]
	if !ok {
		rkActivity.Type = "Other"
	}

	//runkeeper times are in local timezones, so covert back to the local time
	rkLocalLocation := time.FixedZone("rkZone", activity.UtcOffSet*60*60)
	rkActivity.StartTime = runkeeper.Time(time.Unix(int64(activity.StartTime), 0).In(rkLocalLocation))
	log.Printf("SMS time: %s, converted to RK time: %s for offset: %d", activity.StartTime, rkActivity.StartTime, activity.UtcOffSet)
	rkActivity.Notes = activity.Name
	rkActivity.TotalDistance = activity.Distance
	rkActivity.AverageHeartRate = activity.AverageHeartRate
	rkActivity.TotalCalories = activity.Calories
	rkActivity.Source = activity.Source
	rkActivity.EntryMode = API

	rkActivity.Path = convertToPath(activity.GPS)
	rkActivity.HeartRate = convertToHR(activity.HeartRate)
	return rkActivity
}

func convertToPath(gps []dm.GPS) []runkeeper.Path {
	rkPath := make([]runkeeper.Path, len(gps))
	for i, gp := range gps {
		rkPath[i] = runkeeper.Path{gp.Altitude, gp.Longitude, "gps", gp.Latitude, gp.Timestamp}
	}
	return rkPath
}

func convertFromPath(path []runkeeper.Path) []dm.GPS {
	dmPath := make([]dm.GPS, len(path))
	for i, rp := range path {
		dmPath[i] = dm.GPS{rp.Timestamp, rp.Altitude, rp.Longitude, rp.Latitude}
	}
	return dmPath
}

func convertToHR(hr []dm.HeartRate) []runkeeper.HeartRate {
	rkHr := make([]runkeeper.HeartRate, len(hr))
	for i, h := range hr {
		rkHr[i] = runkeeper.HeartRate{h.Timestamp, h.Heartrate}
	}
	return rkHr
}

func convertFromHR(rkHr []runkeeper.HeartRate) []dm.HeartRate {
	dmHr := make([]dm.HeartRate, len(rkHr))
	for i, h := range rkHr {
		dmHr[i] = dm.HeartRate{h.TimeStamp, h.HearRateNr}
	}
	return dmHr
}
<|endoftext|>


<commit_before>// Package taskqueue provides the functionality for receiving, handling and executing tasks.
// In this file are the routines for the taskqueue itself.
package taskqueue

import (
	"fmt"
	"github.com/jpillora/backoff"
	"github.com/mediocregopher/radix.v2/pool"
	"github.com/mediocregopher/radix.v2/redis"
	"github.com/nevsnode/gordon/config"
	"github.com/nevsnode/gordon/output"
	"github.com/nevsnode/gordon/stats"
	"sync"
	"time"
)

type failedTask struct {
	configTask config.Task
	queueTask  QueueTask
}

var (
	errorNoNewTask          = fmt.Errorf("No new task available")
	errorNoNewTasksAccepted = fmt.Errorf("No new tasks accepted")

	conf            config.Config
	shutdownChan    chan bool
	waitGroup       sync.WaitGroup
	waitGroupFailed sync.WaitGroup
	redisPool       *pool.Pool
	failedChan      chan failedTask
)

func init() {
	workerCount = make(map[string]int)
	workerBackoff = make(map[string]*backoff.Backoff)
}

// Start initialises several variables and creates necessary go-routines
func Start(c config.Config) {
	conf = c

	var err error
	redisPool, err = pool.NewCustom(conf.RedisNetwork, conf.RedisAddress, 0, redisDialFunction)
	if err != nil {
		output.NotifyError("redis pool.NewCustom():", err)
	}

	stats.InitTasks(conf.Tasks)

	failedChan = make(chan failedTask)
	shutdownChan = make(chan bool, 1)

	for _, ct := range conf.Tasks {
		createWorkerCount(ct.Type)
	}

	waitGroupFailed.Add(1)
	go failedTaskWorker()

	waitGroup.Add(1)
	go queueWorker()
}

// Stop will cause the taskqueue to stop accepting new tasks and shutdown the
// worker routines after they've finished their current tasks
func Stop() {
	if isShuttingDown() {
		return
	}

	setShutdown()
	shutdownChan <- true
}

// Wait waits, to keep the application running as long as there are workers
func Wait() {
	waitGroup.Wait()
	output.Debug("Finished task-workers")

	close(failedChan)
	waitGroupFailed.Wait()
	output.Debug("Finished failed-task-worker")
}

func queueWorker() {
	interval := backoff.Backoff{
		Min:    time.Duration(conf.IntervalMin) * time.Millisecond,
		Max:    time.Duration(conf.IntervalMax) * time.Millisecond,
		Factor: conf.IntervalFactor,
	}

	runIntervalLoop := make(chan bool)
	doneIntervalLoop := make(chan bool)

	go func() {
		for {
			<-doneIntervalLoop
			time.Sleep(interval.Duration())

			if isShuttingDown() {
				break
			}

			runIntervalLoop <- true
		}
	}()

	go func() {
		for {
			select {
			case <-shutdownChan:
				runIntervalLoop <- false
			}
		}
	}()

	doneIntervalLoop <- true

intervalLoop:
	for <-runIntervalLoop {
		for taskType, configTask := range conf.Tasks {
			if isShuttingDown() {
				break intervalLoop
			}

			output.Debug("Checking for new tasks (" + taskType + ")")

			// check if there are available workers
			if !isWorkerAvailable(taskType) {
				continue
			}

			queueKey := conf.RedisQueueKey + ":" + taskType

			llen, err := redisPool.Cmd("LLEN", queueKey).Int()
			if err != nil {
				// Errors here are likely redis-connection errors, so we'll
				// need to notify about it
				output.NotifyError("redisPool.Cmd() Error:", err)
				break
			}

			// there are no new tasks in redis
			if llen == 0 {
				continue
			}

			// iterate over all entries in redis, until no more are available,
			// or all workers are busy, for a maximum of 2 * workers
			for i := 0; i < (configTask.Workers * 2); i++ {
				if !isWorkerAvailable(taskType) {
					break
				}

				value, err := redisPool.Cmd("LPOP", queueKey).Str()
				if err != nil {
					// no more tasks found
					break
				}

				output.Debug("Fetched task for type", taskType, "with payload", value)

				task, err := NewQueueTask(value)
				if err != nil {
					output.NotifyError("NewQueueTask():", err)
					continue
				}

				// spawn worker go-routine
				claimWorker(taskType)
				go taskWorker(task, configTask)

				// we've actually are handling new tasks so reset the interval
				interval.Reset()
			}
		}

		doneIntervalLoop <- true
	}

	Stop()
	waitGroup.Done()
	output.Debug("Finished queue-worker")
}

func taskWorker(task QueueTask, ct config.Task) {
	defer returnWorker(ct.Type)

	if ct.BackoffEnabled {
		doErrorBackoff(ct.Type)
	}

	payload, _ := task.GetJSONString()
	output.Debug("Executing task type", ct.Type, "- Payload:", payload)
	txn := stats.StartedTask(ct.Type)

	err := task.Execute(ct.Script)

	if err != nil {
		txn.NoticeError(err)
	}
	txn.End()

	if err == nil {
		resetErrorBackoff(ct.Type)
	}

	if err != nil {
		task.ErrorMessage = fmt.Sprintf("%s", err)
		failedChan <- failedTask{
			configTask: ct,
			queueTask:  task,
		}

		msg := fmt.Sprintf("Failed executing task for type \"%s\"\nPayload:\n%s\n\n%s", ct.Type, payload, err)
		output.NotifyError(msg)
	}

	output.Debug("Finished task type", ct.Type, "- Payload:", payload)
}

func failedTaskWorker() {
	defer waitGroupFailed.Done()

	for ft := range failedChan {
		ct := ft.configTask
		qt := ft.queueTask

		if ct.FailedTasksTTL == 0 {
			continue
		}

		rc, err := redisPool.Get()
		if err != nil {
			output.NotifyError("redisPool.Get():", err)
			continue
		}
		defer redisPool.Put(rc)

		queueKey := conf.RedisQueueKey + ":" + ct.Type + ":failed"

		jsonString, err := qt.GetJSONString()
		if err != nil {
			output.NotifyError("failedTaskWorker(), qt.GetJSONString():", err)
			continue
		}

		// add to list
		reply := rc.Cmd("RPUSH", queueKey, jsonString)
		if reply.Err != nil {
			output.NotifyError("failedTaskWorker(), RPUSH:", reply.Err)
			continue
		}

		// set expire
		reply = rc.Cmd("EXPIRE", queueKey, ct.FailedTasksTTL)
		if reply.Err != nil {
			output.NotifyError("failedTaskWorker(), EXPIRE:", reply.Err)
			continue
		}
	}
}

func redisDialFunction(network, addr string) (*redis.Client, error) {
	return redis.DialTimeout(network, addr, time.Duration(10)*time.Second)
}

var (
	shutdownLock sync.RWMutex
	shutdown     = false
)

func isShuttingDown() bool {
	shutdownLock.RLock()
	defer shutdownLock.RUnlock()
	return shutdown
}

func setShutdown() {
	shutdownLock.Lock()
	defer shutdownLock.Unlock()
	shutdown = true
}

var (
	workerCount     map[string]int
	workerCountLock sync.Mutex
)

func createWorkerCount(taskType string) {
	workerCountLock.Lock()
	defer workerCountLock.Unlock()

	workerCount[taskType] = 0
}

func isWorkerAvailable(taskType string) bool {
	workerCountLock.Lock()
	defer workerCountLock.Unlock()

	currentCount := workerCount[taskType]
	maxCount := conf.Tasks[taskType].Workers

	return currentCount < maxCount
}

func claimWorker(taskType string) {
	waitGroup.Add(1)

	workerCountLock.Lock()
	defer workerCountLock.Unlock()

	workerCount[taskType]++
}

func returnWorker(taskType string) {
	workerCountLock.Lock()
	defer workerCountLock.Unlock()

	workerCount[taskType]--
	waitGroup.Done()
}

var (
	workerBackoff     map[string]*backoff.Backoff
	workerBackoffLock sync.Mutex
)

func doErrorBackoff(taskType string) {
	workerBackoffLock.Lock()
	defer workerBackoffLock.Unlock()

	if workerBackoff[taskType] == nil {
		ct := conf.Tasks[taskType]
		workerBackoff[taskType] = &backoff.Backoff{
			Min:    time.Duration(ct.BackoffMin) * time.Millisecond,
			Max:    time.Duration(ct.BackoffMax) * time.Millisecond,
			Factor: ct.BackoffFactor,
			Jitter: true,
		}
	}

	time.Sleep(workerBackoff[taskType].Duration())
}

func resetErrorBackoff(taskType string) {
	workerBackoffLock.Lock()
	defer workerBackoffLock.Unlock()

	if workerBackoff[taskType] == nil {
		return
	}

	workerBackoff[taskType].Reset()
}
<commit_msg>attempt to improve redis-command reliability for instance in flaky network situations<commit_after>// Package taskqueue provides the functionality for receiving, handling and executing tasks.
// In this file are the routines for the taskqueue itself.
package taskqueue

import (
	"fmt"
	"github.com/jpillora/backoff"
	"github.com/mediocregopher/radix.v2/pool"
	"github.com/mediocregopher/radix.v2/redis"
	"github.com/nevsnode/gordon/config"
	"github.com/nevsnode/gordon/output"
	"github.com/nevsnode/gordon/stats"
	"math"
	"sync"
	"time"
)

type failedTask struct {
	configTask config.Task
	queueTask  QueueTask
}

var (
	errorNoNewTask          = fmt.Errorf("No new task available")
	errorNoNewTasksAccepted = fmt.Errorf("No new tasks accepted")

	conf            config.Config
	shutdownChan    chan bool
	waitGroup       sync.WaitGroup
	waitGroupFailed sync.WaitGroup
	redisPool       *pool.Pool
	failedChan      chan failedTask
)

func init() {
	workerCount = make(map[string]int)
	workerBackoff = make(map[string]*backoff.Backoff)
}

// Start initialises several variables and creates necessary go-routines
func Start(c config.Config) {
	conf = c

	var err error
	redisPool, err = pool.NewCustom(conf.RedisNetwork, conf.RedisAddress, 0, redisDialFunction)
	if err != nil {
		output.NotifyError("redis pool.NewCustom():", err)
	}

	stats.InitTasks(conf.Tasks)

	failedChan = make(chan failedTask)
	shutdownChan = make(chan bool, 1)

	for _, ct := range conf.Tasks {
		createWorkerCount(ct.Type)
	}

	waitGroupFailed.Add(1)
	go failedTaskWorker()

	waitGroup.Add(1)
	go queueWorker()
}

// Stop will cause the taskqueue to stop accepting new tasks and shutdown the
// worker routines after they've finished their current tasks
func Stop() {
	if isShuttingDown() {
		return
	}

	setShutdown()
	shutdownChan <- true
}

// Wait waits, to keep the application running as long as there are workers
func Wait() {
	waitGroup.Wait()
	output.Debug("Finished task-workers")

	close(failedChan)
	waitGroupFailed.Wait()
	output.Debug("Finished failed-task-worker")
}

func queueWorker() {
	interval := backoff.Backoff{
		Min:    time.Duration(conf.IntervalMin) * time.Millisecond,
		Max:    time.Duration(conf.IntervalMax) * time.Millisecond,
		Factor: conf.IntervalFactor,
	}

	runIntervalLoop := make(chan bool)
	doneIntervalLoop := make(chan bool)

	go func() {
		for {
			<-doneIntervalLoop
			time.Sleep(interval.Duration())

			if isShuttingDown() {
				break
			}

			runIntervalLoop <- true
		}
	}()

	go func() {
		for {
			select {
			case <-shutdownChan:
				runIntervalLoop <- false
			}
		}
	}()

	doneIntervalLoop <- true

intervalLoop:
	for <-runIntervalLoop {
		for taskType, configTask := range conf.Tasks {
			if isShuttingDown() {
				break intervalLoop
			}

			output.Debug("Checking for new tasks (" + taskType + ")")

			// check if there are available workers
			if !isWorkerAvailable(taskType) {
				continue
			}

			queueKey := conf.RedisQueueKey + ":" + taskType

			llen, err := redisPoolCmd(3, "LLEN", queueKey).Int()
			if err != nil {
				// Errors here are likely redis-connection errors, so we'll
				// need to notify about it
				output.NotifyError("redisPoolCmd() Error:", err)
				break
			}

			// there are no new tasks in redis
			if llen == 0 {
				continue
			}

			// iterate over all entries in redis, until no more are available,
			// or all workers are busy, for a maximum of 2 * workers
			for i := 0; i < (configTask.Workers * 2); i++ {
				if !isWorkerAvailable(taskType) {
					break
				}

				value, err := redisPoolCmd(1, "LPOP", queueKey).Str()
				if err != nil {
					// most likely no more tasks found
					break
				}

				output.Debug("Fetched task for type", taskType, "with payload", value)

				task, err := NewQueueTask(value)
				if err != nil {
					output.NotifyError("NewQueueTask():", err)
					continue
				}

				// spawn worker go-routine
				claimWorker(taskType)
				go taskWorker(task, configTask)

				// we've actually are handling new tasks so reset the interval
				interval.Reset()
			}
		}

		doneIntervalLoop <- true
	}

	Stop()
	waitGroup.Done()
	output.Debug("Finished queue-worker")
}

func taskWorker(task QueueTask, ct config.Task) {
	defer returnWorker(ct.Type)

	if ct.BackoffEnabled {
		doErrorBackoff(ct.Type)
	}

	payload, _ := task.GetJSONString()
	output.Debug("Executing task type", ct.Type, "- Payload:", payload)
	txn := stats.StartedTask(ct.Type)

	err := task.Execute(ct.Script)

	if err != nil {
		txn.NoticeError(err)
	}
	txn.End()

	if err == nil {
		resetErrorBackoff(ct.Type)
	}

	if err != nil {
		task.ErrorMessage = fmt.Sprintf("%s", err)
		failedChan <- failedTask{
			configTask: ct,
			queueTask:  task,
		}

		msg := fmt.Sprintf("Failed executing task for type \"%s\"\nPayload:\n%s\n\n%s", ct.Type, payload, err)
		output.NotifyError(msg)
	}

	output.Debug("Finished task type", ct.Type, "- Payload:", payload)
}

func failedTaskWorker() {
	defer waitGroupFailed.Done()

	for ft := range failedChan {
		ct := ft.configTask
		qt := ft.queueTask

		if ct.FailedTasksTTL == 0 {
			continue
		}

		queueKey := conf.RedisQueueKey + ":" + ct.Type + ":failed"

		jsonString, err := qt.GetJSONString()
		if err != nil {
			output.NotifyError("failedTaskWorker(), qt.GetJSONString():", err)
			continue
		}

		// add to list
		reply := redisPoolCmd(3, "RPUSH", queueKey, jsonString)
		if reply.Err != nil {
			output.NotifyError("failedTaskWorker(), RPUSH:", reply.Err, "\nPayload:\n", jsonString)
			continue
		}

		// set expire
		reply = redisPoolCmd(3, "EXPIRE", queueKey, ct.FailedTasksTTL)
		if reply.Err != nil {
			output.NotifyError("failedTaskWorker(), EXPIRE:", reply.Err, "\nPayload:\n", jsonString)
			continue
		}
	}
}

func redisDialFunction(network, addr string) (*redis.Client, error) {
	return redis.DialTimeout(network, addr, time.Duration(10)*time.Second)
}

func redisPoolCmd(retries int, cmd string, args ...interface{}) (resp *redis.Resp) {
	cmdBackoff := backoff.Backoff{
		Min:    time.Duration(250) * time.Millisecond,
		Max:    time.Duration(2000) * time.Millisecond,
		Factor: math.E,
		Jitter: true,
	}

	i := 0
	for i < retries {
		resp = redisPool.Cmd(cmd, args...)
		if resp.Err == nil {
			break
		}

		output.Debug("redisPool.Cmd() Error:", resp.Err, "\nCommand:\n"+cmd, fmt.Sprint(args...))
		i++

		if i < retries {
			time.Sleep(cmdBackoff.Duration())
		}
	}

	return resp
}

var (
	shutdownLock sync.RWMutex
	shutdown     = false
)

func isShuttingDown() bool {
	shutdownLock.RLock()
	defer shutdownLock.RUnlock()
	return shutdown
}

func setShutdown() {
	shutdownLock.Lock()
	defer shutdownLock.Unlock()
	shutdown = true
}

var (
	workerCount     map[string]int
	workerCountLock sync.Mutex
)

func createWorkerCount(taskType string) {
	workerCountLock.Lock()
	defer workerCountLock.Unlock()

	workerCount[taskType] = 0
}

func isWorkerAvailable(taskType string) bool {
	workerCountLock.Lock()
	defer workerCountLock.Unlock()

	currentCount := workerCount[taskType]
	maxCount := conf.Tasks[taskType].Workers

	return currentCount < maxCount
}

func claimWorker(taskType string) {
	waitGroup.Add(1)

	workerCountLock.Lock()
	defer workerCountLock.Unlock()

	workerCount[taskType]++
}

func returnWorker(taskType string) {
	workerCountLock.Lock()
	defer workerCountLock.Unlock()

	workerCount[taskType]--
	waitGroup.Done()
}

var (
	workerBackoff     map[string]*backoff.Backoff
	workerBackoffLock sync.Mutex
)

func doErrorBackoff(taskType string) {
	workerBackoffLock.Lock()
	defer workerBackoffLock.Unlock()

	if workerBackoff[taskType] == nil {
		ct := conf.Tasks[taskType]
		workerBackoff[taskType] = &backoff.Backoff{
			Min:    time.Duration(ct.BackoffMin) * time.Millisecond,
			Max:    time.Duration(ct.BackoffMax) * time.Millisecond,
			Factor: ct.BackoffFactor,
			Jitter: true,
		}
	}

	time.Sleep(workerBackoff[taskType].Duration())
}

func resetErrorBackoff(taskType string) {
	workerBackoffLock.Lock()
	defer workerBackoffLock.Unlock()

	if workerBackoff[taskType] == nil {
		return
	}

	workerBackoff[taskType].Reset()
}
<|endoftext|>


<commit_before>package bitio

import (
	"bytes"
	"errors"
	"io"
	"math/rand"
	"testing"
	"time"
)

func TestReader(t *testing.T) {
	data := []byte{3, 255, 0xcc, 0x1a, 0xbc, 0xde, 0x80, 0x01, 0x02, 0xf8, 0x08, 0xf0}

	r := NewReader(bytes.NewBuffer(data))

	var nExp interface{}
	check := func(n interface{}, err error) {
		if n != nExp || err != nil {
			t.Errorf("Got %x, want %x, error: %v", n, nExp, err)
		}
	}

	nExp = byte(3)
	check(r.ReadByte())
	nExp = uint64(255)
	check(r.ReadBits(8))

	nExp = uint64(0xc)
	check(r.ReadBits(4))

	nExp = uint64(0xc1)
	check(r.ReadBits(8))

	nExp = uint64(0xabcde)
	check(r.ReadBits(20))

	if b, err := r.ReadBool(); !b || err != nil {
		t.Errorf("Got %v, want %v, error: %v", b, false, err)
	}
	if b, err := r.ReadBool(); b || err != nil {
		t.Errorf("Got %v, want %v, error: %v", b, true, err)
	}

	if n := r.Align(); n != 6 {
		t.Errorf("Got %v, want %v", n, 6)
	}

	s := make([]byte, 2)
	if n, err := r.Read(s); n != 2 || err != nil || !bytes.Equal(s, []byte{0x01, 0x02}) {
		t.Errorf("Got %v, want %v, error: %v", s, []byte{0x01, 0x02}, err)
	}

	nExp = uint64(0xf)
	check(r.ReadBits(4))

	if n, err := r.Read(s); n != 2 || err != nil || !bytes.Equal(s, []byte{0x80, 0x8f}) {
		t.Errorf("Got %v, want %v, error: %v", s, []byte{0x80, 0x8f}, err)
	}
}

func TestWriter(t *testing.T) {
	b := &bytes.Buffer{}

	w := NewWriter(b)

	expected := []byte{0xc1, 0x7f, 0xac, 0x89, 0x24, 0x78, 0x01, 0x02, 0xf8, 0x08, 0xf0, 0xff, 0x80}

	errs := []error{}
	errs = append(errs, w.WriteByte(0xc1))
	errs = append(errs, w.WriteBool(false))
	errs = append(errs, w.WriteBits(0x3f, 6))
	errs = append(errs, w.WriteBool(true))
	errs = append(errs, w.WriteByte(0xac))
	errs = append(errs, w.WriteBits(0x01, 1))
	errs = append(errs, w.WriteBits(0x1248f, 20))

	var nExp interface{}
	check := func(n interface{}, err error) {
		if n != nExp || err != nil {
			t.Errorf("Got %x, want %x, error: %v", n, nExp, err)
		}
	}

	nExp = byte(3)
	check(w.Align())

	nExp = int(2)
	check(w.Write([]byte{0x01, 0x02}))

	errs = append(errs, w.WriteBits(0x0f, 4))

	check(w.Write([]byte{0x80, 0x8f}))

	nExp = byte(4)
	check(w.Align())
	nExp = byte(0)
	check(w.Align())
	if err := w.WriteBits(0x01, 1); err != nil {
		t.Error("Got error:", err)
	}
	if err := w.WriteByte(0xff); err != nil {
		t.Error("Got error:", err)
	}

	errs = append(errs, w.Close())

	for _, v := range errs {
		if v != nil {
			t.Error("Got error:", v)
		}
	}

	if !bytes.Equal(b.Bytes(), expected) {
		t.Errorf("Got: %x, want: %x", b.Bytes(), expected)
	}
}

func TestReaderEOF(t *testing.T) {
	r := NewReader(bytes.NewBuffer([]byte{0x01}))

	if b, err := r.ReadByte(); b != 1 || err != nil {
		t.Errorf("Got %x, want %x, error: %v", b, 1, err)
	}
	if _, err := r.ReadByte(); err != io.EOF {
		t.Errorf("Got %v, want %v", err, io.EOF)
	}
	if _, err := r.ReadBool(); err != io.EOF {
		t.Errorf("Got %v, want %v", err, io.EOF)
	}
	if _, err := r.ReadBits(1); err != io.EOF {
		t.Errorf("Got %v, want %v", err, io.EOF)
	}
	if n, err := r.Read(make([]byte, 2)); n != 0 || err != io.EOF {
		t.Errorf("Got %v, want %v", err, io.EOF)
	}
}

func TestReaderEOF2(t *testing.T) {
	r := NewReader(bytes.NewBuffer([]byte{0x01}))
	if _, err := r.ReadBits(17); err != io.EOF {
		t.Errorf("Got %v, want %v", err, io.EOF)
	}

	// Byte spreading byte boundary (readUnalignedByte)
	r = NewReader(bytes.NewBuffer([]byte{0xc1, 0x01}))
	if b, err := r.ReadBool(); !b || err != nil {
		t.Errorf("Got %v, want %v, error: %v", b, false, err)
	}
	if b, err := r.ReadByte(); b != 0x82 || err != nil {
		t.Errorf("Got %x, want %x, error: %v", b, 0x82, err)
	}
	// readUnalignedByte resulting in EOF
	if _, err := r.ReadByte(); err != io.EOF {
		t.Errorf("Got %v, want %v", err, io.EOF)
	}

	r = NewReader(bytes.NewBuffer([]byte{0xc1, 0x01}))
	if b, err := r.ReadBool(); !b || err != nil {
		t.Errorf("Got %v, want %v, error: %v", b, false, err)
	}
	if n, err := r.Read(make([]byte, 2)); n != 1 || err != io.EOF {
		t.Errorf("Got %v, want %v", err, io.EOF)
	}
}

type nonByteReaderWriter struct {
	io.Reader
	io.Writer
}

func TestNonByteReaderWriter(t *testing.T) {
	NewReader(nonByteReaderWriter{})
	NewWriter(nonByteReaderWriter{})
}

type errWriter struct {
	limit int
}

func (e *errWriter) WriteByte(c byte) error {
	if e.limit == 0 {
		return errors.New("Can't write more!")
	}
	e.limit--
	return nil
}

func (e *errWriter) Write(p []byte) (n int, err error) {
	for i, v := range p {
		if err := e.WriteByte(v); err != nil {
			return i, err
		}
	}
	return len(p), nil
}

type errCloser struct {
	errWriter
}

func (e *errCloser) Close() error {
	return errors.New("Obliged not to close!")
}

func TestWriterError(t *testing.T) {
	w := NewWriter(&errWriter{1})
	if err := w.WriteBool(true); err != nil {
		t.Error("Got error:", err)
	}
	if n, err := w.Write([]byte{0x01, 0x02}); n != 1 || err == nil {
		t.Errorf("Got %x, want %x, error: %v", n, 2, err)
	}
	if err := w.Close(); err == nil {
		t.Error("Got no error:", err)
	}

	w = NewWriter(&errWriter{0})
	if err := w.WriteBits(0x00, 9); err == nil {
		t.Error("Got no error:", err)
	}

	w = NewWriter(&errWriter{1})
	if err := w.WriteBits(0x00, 17); err == nil {
		t.Error("Got no error:", err)
	}

	w = NewWriter(&errWriter{})
	if err := w.WriteBits(0x00, 7); err != nil {
		t.Error("Got error:", err)
	}
	if err := w.WriteBool(false); err == nil {
		t.Error("Got no error:", err)
	}

	w = NewWriter(&errWriter{})
	if err := w.WriteBool(true); err != nil {
		t.Error("Got error:", err)
	}
	if _, err := w.Align(); err == nil {
		t.Error("Got no error:", err)
	}

	w = NewWriter(&errCloser{})
	if err := w.Close(); err == nil {
		t.Error("Got no error:", err)
	}
}

func TestChain(t *testing.T) {
	b := &bytes.Buffer{}
	w := NewWriter(b)

	rand.Seed(time.Now().UnixNano())

	expected := make([]uint64, 100000)
	bits := make([]byte, len(expected))

	// Writing (generating)
	for i := range expected {
		expected[i] = uint64(rand.Int63())
		bits[i] = byte(1 + rand.Int31n(60))
		expected[i] &= uint64(1)<<bits[i] - 1
		w.WriteBits(expected[i], bits[i])
	}
	if err := w.Close(); err != nil {
		t.Error("Got error:", err)
	}

	r := NewReader(bytes.NewBuffer(b.Bytes()))

	// Reading (verifying)
	for i, v := range expected {
		if u, err := r.ReadBits(bits[i]); u != v || err != nil {
			t.Errorf("Idx: %d, Got: %x, want: %x, bits: %d, error: %v", i, u, v, bits[i], err)
		}
	}
}
<commit_msg>Utilized github.com/icza/mighty for testing.<commit_after>package bitio

import (
	"bytes"
	"errors"
	"github.com/icza/mighty"
	"io"
	"math/rand"
	"testing"
	"time"
)

func TestReader(t *testing.T) {
	data := []byte{3, 255, 0xcc, 0x1a, 0xbc, 0xde, 0x80, 0x01, 0x02, 0xf8, 0x08, 0xf0}

	r := NewReader(bytes.NewBuffer(data))

	eq := mighty.Eq(t)
	var exp interface{}
	check := func(got interface{}, err error) {
		eq(exp, got, err)
	}

	exp = byte(3)
	check(r.ReadByte())
	exp = uint64(255)
	check(r.ReadBits(8))

	exp = uint64(0xc)
	check(r.ReadBits(4))

	exp = uint64(0xc1)
	check(r.ReadBits(8))

	exp = uint64(0xabcde)
	check(r.ReadBits(20))

	exp = true
	check(r.ReadBool())
	exp = false
	check(r.ReadBool())

	eq(byte(6), r.Align())

	s := make([]byte, 2)
	exp = 2
	check(r.Read(s))
	eq(true, bytes.Equal(s, []byte{0x01, 0x02}))

	exp = uint64(0xf)
	check(r.ReadBits(4))

	exp = 2
	check(r.Read(s))
	eq(true, bytes.Equal(s, []byte{0x80, 0x8f}))
}

func TestWriter(t *testing.T) {
	b := &bytes.Buffer{}

	w := NewWriter(b)

	expected := []byte{0xc1, 0x7f, 0xac, 0x89, 0x24, 0x78, 0x01, 0x02, 0xf8, 0x08, 0xf0, 0xff, 0x80}

	eq := mighty.Eq(t)

	eq(nil, w.WriteByte(0xc1))
	eq(nil, w.WriteBool(false))
	eq(nil, w.WriteBits(0x3f, 6))
	eq(nil, w.WriteBool(true))
	eq(nil, w.WriteByte(0xac))
	eq(nil, w.WriteBits(0x01, 1))
	eq(nil, w.WriteBits(0x1248f, 20))

	var exp interface{}
	check := func(got interface{}, err error) {
		eq(exp, got, err)
	}

	exp = byte(3)
	check(w.Align())

	exp = int(2)
	check(w.Write([]byte{0x01, 0x02}))

	eq(nil, w.WriteBits(0x0f, 4))

	check(w.Write([]byte{0x80, 0x8f}))

	exp = byte(4)
	check(w.Align())
	exp = byte(0)
	check(w.Align())
	eq(nil, w.WriteBits(0x01, 1))
	eq(nil, w.WriteByte(0xff))

	eq(nil, w.Close())

	eq(true, bytes.Equal(b.Bytes(), expected))
}

func TestReaderEOF(t *testing.T) {
	eq := mighty.Eq(t)

	r := NewReader(bytes.NewBuffer([]byte{0x01}))

	b, err := r.ReadByte()
	eq(byte(1), b)
	eq(nil, err)
	_, err = r.ReadByte()
	eq(io.EOF, err)
	_, err = r.ReadBool()
	eq(io.EOF, err)
	_, err = r.ReadBits(1)
	eq(io.EOF, err)
	n, err := r.Read(make([]byte, 2))
	eq(0, n)
	eq(io.EOF, err)
}

func TestReaderEOF2(t *testing.T) {
	eq := mighty.Eq(t)

	var exp interface{}
	var err error

	check := func(got interface{}, err error) {
		eq(exp, got, err)
	}

	r := NewReader(bytes.NewBuffer([]byte{0x01}))
	_, err = r.ReadBits(17)
	eq(io.EOF, err)

	// Byte spreading byte boundary (readUnalignedByte)
	r = NewReader(bytes.NewBuffer([]byte{0xc1, 0x01}))
	exp = true
	check(r.ReadBool())
	exp = byte(0x82)
	check(r.ReadByte())
	// readUnalignedByte resulting in EOF
	_, err = r.ReadByte()
	eq(io.EOF, err)

	r = NewReader(bytes.NewBuffer([]byte{0xc1, 0x01}))
	exp = true
	check(r.ReadBool())
	got, err := r.Read(make([]byte, 2))
	eq(1, got)
	eq(io.EOF, err)
}

type nonByteReaderWriter struct {
	io.Reader
	io.Writer
}

func TestNonByteReaderWriter(t *testing.T) {
	NewReader(nonByteReaderWriter{})
	NewWriter(nonByteReaderWriter{})
}

type errWriter struct {
	limit int
}

func (e *errWriter) WriteByte(c byte) error {
	if e.limit == 0 {
		return errors.New("Can't write more!")
	}
	e.limit--
	return nil
}

func (e *errWriter) Write(p []byte) (n int, err error) {
	for i, v := range p {
		if err := e.WriteByte(v); err != nil {
			return i, err
		}
	}
	return len(p), nil
}

type errCloser struct {
	errWriter
}

func (e *errCloser) Close() error {
	return errors.New("Obliged not to close!")
}

func TestWriterError(t *testing.T) {
	eq, neq := mighty.EqNeq(t)

	w := NewWriter(&errWriter{1})
	eq(nil, w.WriteBool(true))
	got, err := w.Write([]byte{0x01, 0x02})
	eq(1, got)
	neq(nil, err)
	neq(nil, w.Close())

	w = NewWriter(&errWriter{0})
	neq(nil, w.WriteBits(0x00, 9))

	w = NewWriter(&errWriter{1})
	neq(nil, w.WriteBits(0x00, 17))

	w = NewWriter(&errWriter{})
	eq(nil, w.WriteBits(0x00, 7))
	neq(nil, w.WriteBool(false))

	w = NewWriter(&errWriter{})
	eq(nil, w.WriteBool(true))
	_, err = w.Align()
	neq(nil, err)

	w = NewWriter(&errCloser{})
	neq(nil, w.Close())
}

func TestChain(t *testing.T) {
	eq := mighty.Eq(t)

	b := &bytes.Buffer{}
	w := NewWriter(b)

	rand.Seed(time.Now().UnixNano())

	expected := make([]uint64, 100000)
	bits := make([]byte, len(expected))

	// Writing (generating)
	for i := range expected {
		expected[i] = uint64(rand.Int63())
		bits[i] = byte(1 + rand.Int31n(60))
		expected[i] &= uint64(1)<<bits[i] - 1
		w.WriteBits(expected[i], bits[i])
	}

	eq(nil, w.Close())

	r := NewReader(bytes.NewBuffer(b.Bytes()))

	// Reading (verifying)
	for i, v := range expected {
		u, err := r.ReadBits(bits[i])
		eq(v, u, err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package topics

import (
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"

	"github.com/gorilla/mux"

	"istio.io/pkg/ctrlz/fw"
	"istio.io/pkg/ctrlz/topics/assets"
	"istio.io/pkg/log"
)

type scopeTopic struct {
}

type scopeInfo struct {
	Name            string `json:"name"`
	Description     string `json:"description"`
	OutputLevel     string `json:"output_level"`
	StackTraceLevel string `json:"stack_trace_level"`
	LogCallers      bool   `json:"log_callers"`
}

var levelToString = map[log.Level]string{
	log.DebugLevel: "debug",
	log.InfoLevel:  "info",
	log.WarnLevel:  "warn",
	log.ErrorLevel: "error",
	log.NoneLevel:  "none",
}

var stringToLevel = map[string]log.Level{
	"debug": log.DebugLevel,
	"info":  log.InfoLevel,
	"warn":  log.WarnLevel,
	"error": log.ErrorLevel,
	"none":  log.NoneLevel,
}

// ScopeTopic returns a ControlZ topic that allows visualization of process logging scopes.
func ScopeTopic() fw.Topic {
	return scopeTopic{}
}

func (scopeTopic) Title() string {
	return "Logging Scopes"
}

func (scopeTopic) Prefix() string {
	return "scope"
}

func getScopeInfo(s *log.Scope) *scopeInfo {
	return &scopeInfo{
		Name:            s.Name(),
		Description:     s.Description(),
		OutputLevel:     levelToString[s.GetOutputLevel()],
		StackTraceLevel: levelToString[s.GetStackTraceLevel()],
		LogCallers:      s.GetLogCallers(),
	}
}

func (scopeTopic) Activate(context fw.TopicContext) {
	tmpl := template.Must(context.Layout().Parse(string(assets.MustAsset("templates/scopes.html"))))

	_ = context.HTMLRouter().NewRoute().HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		allScopes := log.Scopes()
		s := make([]scopeInfo, 0, len(allScopes))
		for _, scope := range allScopes {
			s = append(s, *getScopeInfo(scope))
		}
		fw.RenderHTML(w, tmpl, s)
	})

	_ = context.JSONRouter().StrictSlash(true).NewRoute().Methods("GET").Path("/").HandlerFunc(getAllScopes)
	_ = context.JSONRouter().NewRoute().Methods("GET").Path("/{scope}").HandlerFunc(getScope)
	_ = context.JSONRouter().NewRoute().Methods("PUT").Path("/{scope}").HandlerFunc(putScope)
}

func getAllScopes(w http.ResponseWriter, _ *http.Request) {
	allScopes := log.Scopes()

	scopeInfos := make([]scopeInfo, 0, len(allScopes))
	for _, s := range allScopes {
		scopeInfos = append(scopeInfos, *getScopeInfo(s))
	}

	fw.RenderJSON(w, http.StatusOK, scopeInfos)
}

func getScope(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	name := vars["scope"]

	if s := log.FindScope(name); s != nil {
		fw.RenderJSON(w, http.StatusOK, getScopeInfo(s))
		return
	}

	fw.RenderError(w, http.StatusBadRequest, fmt.Errorf("unknown scope name: %s", name))
}

func putScope(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	name := vars["scope"]

	var info scopeInfo
	if err := json.NewDecoder(req.Body).Decode(&info); err != nil {
		fw.RenderError(w, http.StatusBadRequest, fmt.Errorf("unable to decode request: %v", err))
		return
	}

	if s := log.FindScope(name); s != nil {
		level, ok := stringToLevel[info.OutputLevel]
		if ok {
			s.SetOutputLevel(level)
		}

		level, ok = stringToLevel[info.StackTraceLevel]
		if ok {
			s.SetStackTraceLevel(level)
		}

		s.SetLogCallers(info.LogCallers)
		w.WriteHeader(http.StatusAccepted)
		return
	}

	fw.RenderError(w, http.StatusBadRequest, fmt.Errorf("unknown scope name: %s", name))
}
<commit_msg>Sort scopes for controlz (#172)<commit_after>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package topics

import (
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"sort"

	"github.com/gorilla/mux"

	"istio.io/pkg/ctrlz/fw"
	"istio.io/pkg/ctrlz/topics/assets"
	"istio.io/pkg/log"
)

type scopeTopic struct {
}

type scopeInfo struct {
	Name            string `json:"name"`
	Description     string `json:"description"`
	OutputLevel     string `json:"output_level"`
	StackTraceLevel string `json:"stack_trace_level"`
	LogCallers      bool   `json:"log_callers"`
}

var levelToString = map[log.Level]string{
	log.DebugLevel: "debug",
	log.InfoLevel:  "info",
	log.WarnLevel:  "warn",
	log.ErrorLevel: "error",
	log.NoneLevel:  "none",
}

var stringToLevel = map[string]log.Level{
	"debug": log.DebugLevel,
	"info":  log.InfoLevel,
	"warn":  log.WarnLevel,
	"error": log.ErrorLevel,
	"none":  log.NoneLevel,
}

// ScopeTopic returns a ControlZ topic that allows visualization of process logging scopes.
func ScopeTopic() fw.Topic {
	return scopeTopic{}
}

func (scopeTopic) Title() string {
	return "Logging Scopes"
}

func (scopeTopic) Prefix() string {
	return "scope"
}

func getScopeInfo(s *log.Scope) *scopeInfo {
	return &scopeInfo{
		Name:            s.Name(),
		Description:     s.Description(),
		OutputLevel:     levelToString[s.GetOutputLevel()],
		StackTraceLevel: levelToString[s.GetStackTraceLevel()],
		LogCallers:      s.GetLogCallers(),
	}
}

func (scopeTopic) Activate(context fw.TopicContext) {
	tmpl := template.Must(context.Layout().Parse(string(assets.MustAsset("templates/scopes.html"))))

	_ = context.HTMLRouter().NewRoute().HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		allScopes := log.Scopes()
		s := make([]scopeInfo, 0, len(allScopes))
		for _, scope := range allScopes {
			s = append(s, *getScopeInfo(scope))
		}
		sort.Slice(s, func(i, j int) bool {
			return s[i].Name < s[j].Name
		})

		fw.RenderHTML(w, tmpl, s)
	})

	_ = context.JSONRouter().StrictSlash(true).NewRoute().Methods("GET").Path("/").HandlerFunc(getAllScopes)
	_ = context.JSONRouter().NewRoute().Methods("GET").Path("/{scope}").HandlerFunc(getScope)
	_ = context.JSONRouter().NewRoute().Methods("PUT").Path("/{scope}").HandlerFunc(putScope)
}

func getAllScopes(w http.ResponseWriter, _ *http.Request) {
	allScopes := log.Scopes()

	scopeInfos := make([]scopeInfo, 0, len(allScopes))
	for _, s := range allScopes {
		scopeInfos = append(scopeInfos, *getScopeInfo(s))
	}
	sort.Slice(scopeInfos, func(i, j int) bool {
		return scopeInfos[i].Name < scopeInfos[j].Name
	})

	fw.RenderJSON(w, http.StatusOK, scopeInfos)
}

func getScope(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	name := vars["scope"]

	if s := log.FindScope(name); s != nil {
		fw.RenderJSON(w, http.StatusOK, getScopeInfo(s))
		return
	}

	fw.RenderError(w, http.StatusBadRequest, fmt.Errorf("unknown scope name: %s", name))
}

func putScope(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	name := vars["scope"]

	var info scopeInfo
	if err := json.NewDecoder(req.Body).Decode(&info); err != nil {
		fw.RenderError(w, http.StatusBadRequest, fmt.Errorf("unable to decode request: %v", err))
		return
	}

	if s := log.FindScope(name); s != nil {
		level, ok := stringToLevel[info.OutputLevel]
		if ok {
			s.SetOutputLevel(level)
		}

		level, ok = stringToLevel[info.StackTraceLevel]
		if ok {
			s.SetStackTraceLevel(level)
		}

		s.SetLogCallers(info.LogCallers)
		w.WriteHeader(http.StatusAccepted)
		return
	}

	fw.RenderError(w, http.StatusBadRequest, fmt.Errorf("unknown scope name: %s", name))
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path"

	log "github.com/Sirupsen/logrus"

	"github.com/docker/hyperkit/go"

	"github.com/docker/infrakit/pkg/spi/instance"
	"github.com/docker/infrakit/pkg/types"
)

// NewHyperKitPlugin creates an instance plugin for hyperkit.
func NewHyperKitPlugin(vmDir, hyperkit, vpnkitSock string) instance.Plugin {
	return &hyperkitPlugin{VMDir: vmDir,
		HyperKit:   hyperkit,
		VPNKitSock: vpnkitSock,
		DiskDir:    path.Join(vmDir, "disks"),
	}
}

type hyperkitPlugin struct {
	// VMDir is the path to a directory where per VM state is kept
	VMDir string

	// Hyperkit is the path to the hyperkit executable
	HyperKit string

	// VPNKitSock is the path to the VPNKit Unix domain socket.
	VPNKitSock string

	// DiskDir is the path to persistent (across reboots) disk images
	DiskDir string
}

// Validate performs local validation on a provision request.
func (p hyperkitPlugin) Validate(req *types.Any) error {
	return nil
}

// Provision creates a new instance.
func (p hyperkitPlugin) Provision(spec instance.Spec) (*instance.ID, error) {

	var properties map[string]interface{}

	if spec.Properties != nil {
		if err := spec.Properties.Decode(&properties); err != nil {
			return nil, fmt.Errorf("Invalid instance properties: %s", err)
		}
	}

	if properties["Moby"] == nil {
		return nil, errors.New("Property 'Moby' must be set")
	}
	if properties["CPUs"] == nil {
		properties["CPUs"] = 1
	}
	if properties["Memory"] == nil {
		properties["Memory"] = 512
	}
	diskSize := 0
	if properties["Disk"] != nil {
		diskSize = int(properties["Disk"].(float64))
	}

	instanceDir, err := ioutil.TempDir(p.VMDir, "infrakit-")
	if err != nil {
		return nil, err
	}
	id := instance.ID(path.Base(instanceDir))
	log.Infof("[%s] New instance", id)

	logicalID := string(id)
	uuidStr := ""

	diskImage := ""
	if spec.LogicalID != nil {
		logicalID = string(*spec.LogicalID)
		// The LogicalID may be a IP address. If so, translate
		// it into a magic UUID which cause VPNKit to assign a
		// fixed IP address
		if ip := net.ParseIP(logicalID); len(ip) > 0 {
			uuid := make([]byte, 16)
			uuid[12] = ip.To4()[0]
			uuid[13] = ip.To4()[1]
			uuid[14] = ip.To4()[2]
			uuid[15] = ip.To4()[3]
			uuidStr = fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:])
		}
		// If a LogicalID is supplied and the Disk size is
		// non-zero, we place the disk in a special directory
		// so it persists across reboots.
		if diskSize != 0 {
			diskImage = path.Join(p.DiskDir, logicalID+".img")
			// Make sure the directory exists
			err = os.MkdirAll(p.DiskDir, 0755)
			if err != nil {
				return nil, err
			}
		}
	}
	log.Infof("[%s] LogicalID: %s", id, logicalID)
	log.Debugf("[%s] UUID: %s", id, uuidStr)

	// Start a HyperKit instance
	h, err := hyperkit.New(p.HyperKit, instanceDir, p.VPNKitSock, diskImage)
	if err != nil {
		return nil, err
	}
	h.Kernel = properties["Moby"].(string) + "-bzImage"
	h.Initrd = properties["Moby"].(string) + "-initrd.img"
	h.CPUs = int(properties["CPUs"].(float64))
	h.Memory = int(properties["Memory"].(float64))
	h.DiskSize = diskSize
	h.UUID = uuidStr
	h.UserData = spec.Init
	h.Console = hyperkit.ConsoleFile
	log.Infof("[%s] Booting: %s/%s", id, h.Kernel, h.Initrd)
	log.Infof("[%s] %d CPUs, %dMB Memory, %dMB Disk (%s)", id, h.CPUs, h.Memory, h.DiskSize, h.DiskImage)

	err = h.Start("console=ttyS0")
	if err != nil {
		return nil, err
	}
	log.Infof("[%s] Started", id)

	if err := ioutil.WriteFile(path.Join(instanceDir, "logical.id"), []byte(logicalID), 0644); err != nil {
		return nil, err
	}

	tagData, err := types.AnyValue(spec.Tags)
	if err != nil {
		return nil, err
	}

	log.Debugf("[%s] tags: %s", id, tagData)
	if err := ioutil.WriteFile(path.Join(instanceDir, "tags"), tagData.Bytes(), 0644); err != nil {
		return nil, err
	}

	return &id, nil
}

// Label labels the instance
func (p hyperkitPlugin) Label(instance instance.ID, labels map[string]string) error {
	instanceDir := path.Join(p.VMDir, string(instance))
	tagFile := path.Join(instanceDir, "tags")
	buff, err := ioutil.ReadFile(tagFile)
	if err != nil {
		return err
	}

	tags := map[string]string{}
	err = types.AnyBytes(buff).Decode(&tags)
	if err != nil {
		return err
	}

	for k, v := range labels {
		tags[k] = v
	}

	encoded, err := types.AnyValue(tags)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(tagFile, encoded.Bytes(), 0644)
}

// Destroy terminates an existing instance.
func (p hyperkitPlugin) Destroy(id instance.ID) error {
	log.Info("Destroying VM: ", id)

	instanceDir := path.Join(p.VMDir, string(id))
	_, err := os.Stat(instanceDir)
	if err != nil {
		if os.IsNotExist(err) {
			return errors.New("Instance does not exist")
		}
	}

	h, err := hyperkit.FromState(instanceDir)
	if err != nil {
		return err
	}
	err = h.Stop()
	if err != nil {
		return err
	}
	err = h.Remove(false)
	if err != nil {
		return err
	}
	return nil
}

// DescribeInstances returns descriptions of all instances matching all of the provided tags.
func (p hyperkitPlugin) DescribeInstances(tags map[string]string, properties bool) ([]instance.Description, error) {
	files, err := ioutil.ReadDir(p.VMDir)
	if err != nil {
		return nil, err
	}

	descriptions := []instance.Description{}

	for _, file := range files {
		if !file.IsDir() {
			continue
		}

		instanceDir := path.Join(p.VMDir, file.Name())

		tagData, err := ioutil.ReadFile(path.Join(instanceDir, "tags"))
		if err != nil {
			if os.IsNotExist(err) {
				continue
			}

			return nil, err
		}

		instanceTags := map[string]string{}
		if err := types.AnyBytes(tagData).Decode(&instanceTags); err != nil {
			return nil, err
		}

		allMatched := true
		for k, v := range tags {
			value, exists := instanceTags[k]
			if !exists || v != value {
				allMatched = false
				break
			}
		}

		if allMatched {
			var logicalID *instance.LogicalID
			id := instance.ID(file.Name())

			h, err := hyperkit.FromState(instanceDir)
			if err != nil {
				log.Warningln("Could not get instance data. Id: ", id)
				p.Destroy(id)
				continue
			}
			if !h.IsRunning() {
				log.Warningln("Instance is not running. Id: ", id)
				p.Destroy(id)
				continue
			}

			lidData, err := ioutil.ReadFile(path.Join(instanceDir, "logical.id"))
			if err != nil {
				log.Warningln("Could not get logical ID. Id: ", id)
				p.Destroy(id)
				continue
			}
			lid := instance.LogicalID(lidData)
			logicalID = &lid

			descriptions = append(descriptions, instance.Description{
				ID:        id,
				LogicalID: logicalID,
				Tags:      instanceTags,
			})
		}
	}

	return descriptions, nil
}
<commit_msg>infrakit: Adjust hyperkit instance plugin to new API<commit_after>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/hyperkit/go"
	"github.com/docker/infrakit/pkg/spi/instance"
	"github.com/docker/infrakit/pkg/types"
	"github.com/rneugeba/iso9660wrap"
)

// NewHyperKitPlugin creates an instance plugin for hyperkit.
func NewHyperKitPlugin(vmDir, hyperkit, vpnkitSock string) instance.Plugin {
	return &hyperkitPlugin{VMDir: vmDir,
		HyperKit:   hyperkit,
		VPNKitSock: vpnkitSock,
		DiskDir:    path.Join(vmDir, "disks"),
	}
}

type hyperkitPlugin struct {
	// VMDir is the path to a directory where per VM state is kept
	VMDir string

	// Hyperkit is the path to the hyperkit executable
	HyperKit string

	// VPNKitSock is the path to the VPNKit Unix domain socket.
	VPNKitSock string

	// DiskDir is the path to persistent (across reboots) disk images
	DiskDir string
}

// Validate performs local validation on a provision request.
func (p hyperkitPlugin) Validate(req *types.Any) error {
	return nil
}

// Provision creates a new instance.
func (p hyperkitPlugin) Provision(spec instance.Spec) (*instance.ID, error) {

	var properties map[string]interface{}

	if spec.Properties != nil {
		if err := spec.Properties.Decode(&properties); err != nil {
			return nil, fmt.Errorf("Invalid instance properties: %s", err)
		}
	}

	if properties["Moby"] == nil {
		return nil, errors.New("Property 'Moby' must be set")
	}
	if properties["CPUs"] == nil {
		properties["CPUs"] = 1
	}
	if properties["Memory"] == nil {
		properties["Memory"] = 512
	}
	diskSize := 0
	if properties["Disk"] != nil {
		diskSize = int(properties["Disk"].(float64))
	}

	instanceDir, err := ioutil.TempDir(p.VMDir, "infrakit-")
	if err != nil {
		return nil, err
	}
	id := instance.ID(path.Base(instanceDir))
	log.Infof("[%s] New instance", id)

	logicalID := string(id)
	uuidStr := ""

	diskImage := ""
	if spec.LogicalID != nil {
		logicalID = string(*spec.LogicalID)
		// The LogicalID may be a IP address. If so, translate
		// it into a magic UUID which cause VPNKit to assign a
		// fixed IP address
		if ip := net.ParseIP(logicalID); len(ip) > 0 {
			uuid := make([]byte, 16)
			uuid[12] = ip.To4()[0]
			uuid[13] = ip.To4()[1]
			uuid[14] = ip.To4()[2]
			uuid[15] = ip.To4()[3]
			uuidStr = fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:])
		}
		// If a LogicalID is supplied and the Disk size is
		// non-zero, we place the disk in a special directory
		// so it persists across reboots.
		if diskSize != 0 {
			diskImage = path.Join(p.DiskDir, logicalID+".img")
		}
	}

	isoImage := ""
	if spec.Init != "" {
		isoImage = path.Join(instanceDir, "data.iso")
		outfh, err := os.OpenFile(isoImage, os.O_CREATE|os.O_WRONLY, 0644)
		if err != nil {
			log.Fatalf("Cannot create user data ISO: %s", err)
		}
		err = iso9660wrap.WriteBuffer(outfh, []byte(spec.Init), "config")
		if err != nil {
			log.Fatalf("Cannot write user data ISO: %s", err)
		}
		outfh.Close()
	}

	log.Infof("[%s] LogicalID: %s", id, logicalID)
	log.Debugf("[%s] UUID: %s", id, uuidStr)

	// Start a HyperKit instance
	h, err := hyperkit.New(p.HyperKit, p.VPNKitSock, instanceDir)
	if err != nil {
		return nil, err
	}
	h.Kernel = properties["Moby"].(string) + "-bzImage"
	h.Initrd = properties["Moby"].(string) + "-initrd.img"
	h.UUID = uuidStr
	h.DiskImage = diskImage
	h.ISOImage = isoImage
	h.CPUs = int(properties["CPUs"].(float64))
	h.Memory = int(properties["Memory"].(float64))
	h.DiskSize = diskSize
	h.Console = hyperkit.ConsoleFile
	log.Infof("[%s] Booting: %s/%s", id, h.Kernel, h.Initrd)
	log.Infof("[%s] %d CPUs, %dMB Memory, %dMB Disk (%s)", id, h.CPUs, h.Memory, h.DiskSize, h.DiskImage)

	err = h.Start("console=ttyS0")
	if err != nil {
		return nil, err
	}
	log.Infof("[%s] Started", id)

	if err := ioutil.WriteFile(path.Join(instanceDir, "logical.id"), []byte(logicalID), 0644); err != nil {
		return nil, err
	}

	tagData, err := types.AnyValue(spec.Tags)
	if err != nil {
		return nil, err
	}

	log.Debugf("[%s] tags: %s", id, tagData)
	if err := ioutil.WriteFile(path.Join(instanceDir, "tags"), tagData.Bytes(), 0644); err != nil {
		return nil, err
	}

	return &id, nil
}

// Label labels the instance
func (p hyperkitPlugin) Label(instance instance.ID, labels map[string]string) error {
	instanceDir := path.Join(p.VMDir, string(instance))
	tagFile := path.Join(instanceDir, "tags")
	buff, err := ioutil.ReadFile(tagFile)
	if err != nil {
		return err
	}

	tags := map[string]string{}
	err = types.AnyBytes(buff).Decode(&tags)
	if err != nil {
		return err
	}

	for k, v := range labels {
		tags[k] = v
	}

	encoded, err := types.AnyValue(tags)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(tagFile, encoded.Bytes(), 0644)
}

// Destroy terminates an existing instance.
func (p hyperkitPlugin) Destroy(id instance.ID) error {
	log.Info("Destroying VM: ", id)

	instanceDir := path.Join(p.VMDir, string(id))
	_, err := os.Stat(instanceDir)
	if err != nil {
		if os.IsNotExist(err) {
			return errors.New("Instance does not exist")
		}
	}

	h, err := hyperkit.FromState(instanceDir)
	if err != nil {
		return err
	}
	err = h.Stop()
	if err != nil {
		return err
	}
	err = h.Remove(false)
	if err != nil {
		return err
	}
	return nil
}

// DescribeInstances returns descriptions of all instances matching all of the provided tags.
func (p hyperkitPlugin) DescribeInstances(tags map[string]string, properties bool) ([]instance.Description, error) {
	files, err := ioutil.ReadDir(p.VMDir)
	if err != nil {
		return nil, err
	}

	descriptions := []instance.Description{}

	for _, file := range files {
		if !file.IsDir() {
			continue
		}

		instanceDir := path.Join(p.VMDir, file.Name())

		tagData, err := ioutil.ReadFile(path.Join(instanceDir, "tags"))
		if err != nil {
			if os.IsNotExist(err) {
				continue
			}

			return nil, err
		}

		instanceTags := map[string]string{}
		if err := types.AnyBytes(tagData).Decode(&instanceTags); err != nil {
			return nil, err
		}

		allMatched := true
		for k, v := range tags {
			value, exists := instanceTags[k]
			if !exists || v != value {
				allMatched = false
				break
			}
		}

		if allMatched {
			var logicalID *instance.LogicalID
			id := instance.ID(file.Name())

			h, err := hyperkit.FromState(instanceDir)
			if err != nil {
				log.Warningln("Could not get instance data. Id: ", id)
				p.Destroy(id)
				continue
			}
			if !h.IsRunning() {
				log.Warningln("Instance is not running. Id: ", id)
				p.Destroy(id)
				continue
			}

			lidData, err := ioutil.ReadFile(path.Join(instanceDir, "logical.id"))
			if err != nil {
				log.Warningln("Could not get logical ID. Id: ", id)
				p.Destroy(id)
				continue
			}
			lid := instance.LogicalID(lidData)
			logicalID = &lid

			descriptions = append(descriptions, instance.Description{
				ID:        id,
				LogicalID: logicalID,
				Tags:      instanceTags,
			})
		}
	}

	return descriptions, nil
}
<|endoftext|>


<commit_before>// Package v1alpha1 implements all the required types and methods for parsing
// resources for v1alpha1 versioned ClusterServiceVersions.
package v1alpha1

import (
	"encoding/json"

	"github.com/coreos/go-semver/semver"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	GroupVersion = "v1alpha1" // used in registering ClusterServiceVersion scheme

	ClusterServiceVersionCRDName       = "clusterserviceversion-v1s.app.coreos.com"
	ClusterServiceVersionKind          = "ClusterServiceVersion-v1"
	ClusterServiceVersionCRDAPIVersion = "apiextensions.k8s.io/v1beta1" // API version w/ CRD support

)

// NamedInstallStrategy represents the block of an ClusterServiceVersion resource
// where the install strategy is specified.
type NamedInstallStrategy struct {
	StrategyName    string          `json:"strategy"`
	StrategySpecRaw json.RawMessage `json:"spec"`
}

// StatusDescriptor describes a field in a status block of a CRD so that ALM can consume it
type StatusDescriptor struct {
	Path         string          `json:"path"`
	DisplayName  string          `json:"displayName,omitempty"`
	Description  string          `json:"description,omitempty"`
	XDescriptors []string        `json:"x-descriptors,omitempty"`
	Value        json.RawMessage `json:"value,omitempty"`
}

// CRDDescription provides details to ALM about the CRDs
type CRDDescription struct {
	Name              string             `json:"name"`
	DisplayName       string             `json:"displayName,omitempty"`
	Description       string             `json:"description,omitempty"`
	StatusDescriptors []StatusDescriptor `json:"statusDescriptors,omitempty"`
}

// CustomResourceDefinitions declares all of the CRDs managed or required by
// an operator being ran by ClusterServiceVersion.
//
// If the CRD is present in the Owned list, it is implicitly required.
type CustomResourceDefinitions struct {
	Owned    []CRDDescription `json:"owned"`
	Required []CRDDescription `json:"required,omitempty"`
}

// ClusterServiceVersionSpec declarations tell the ALM how to install an operator
// that can manage apps for given version and AppType.
type ClusterServiceVersionSpec struct {
	InstallStrategy           NamedInstallStrategy      `json:"install"`
	Version                   semver.Version            `json:"version"`
	Maturity                  string                    `json:"maturity"`
	CustomResourceDefinitions CustomResourceDefinitions `json:"customresourcedefinitions"`
	DisplayName               string                    `json:"displayName"`
	Description               string                    `json:"description"`
	Keywords                  []string                  `json:"keywords"`
	Maintainers               []Maintainer              `json:"maintainers"`
	Provider                  AppLink                   `json:"provider"`
	Links                     []AppLink                 `json:"links"`
	Icon                      []Icon                    `json:"icon"`

	// The name of a CSV this one replaces. Should match the `metadata.Name` field of the old CSV.
	// +optional
	Replaces string `json:"replaces,omitempty"`

	// Map of string keys and values that can be used to organize and categorize
	// (scope and select) objects.
	// +optional
	Labels map[string]string `json:"labels,omitempty" protobuf:"bytes,11,rep,name=labels"`

	// Annotations is an unstructured key value map stored with a resource that may be
	// set by external tools to store and retrieve arbitrary metadata.
	// +optional
	Annotations map[string]string `json:"annotations,omitempty" protobuf:"bytes,12,rep,name=annotations"`

	// Label selector for related resources.
	// +optional
	Selector *metav1.LabelSelector `json:"selector,omitempty" protobuf:"bytes,2,opt,name=selector"`
}

type Maintainer struct {
	Name  string `json:"name"`
	Email string `json:"email"`
}

type AppLink struct {
	Name string `json:"name"`
	URL  string `json:"url"`
}

type Icon struct {
	Data      string `json:"base64data"`
	MediaType string `json:"mediatype"`
}

// ClusterServiceVersionPhase is a label for the condition of a ClusterServiceVersion at the current time.
type ClusterServiceVersionPhase string

// These are the valid phases of ClusterServiceVersion
const (
	CSVPhaseNone = ""
	// CSVPending means the csv has been accepted by the system, but the install strategy has not been attempted.
	// This is likely because there are unmet requirements.
	CSVPhasePending ClusterServiceVersionPhase = "Pending"
	// CSVRunning means that the requirements are met but the install strategy has not been run.
	CSVPhaseInstalling ClusterServiceVersionPhase = "Installing"
	// CSVSucceeded means that the resources in the CSV were created successfully.
	CSVPhaseSucceeded ClusterServiceVersionPhase = "Succeeded"
	// CSVFailed means that the install strategy could not be successfully completed.
	CSVPhaseFailed ClusterServiceVersionPhase = "Failed"
	// CSVUnknown means that for some reason the state of the csv could not be obtained.
	CSVPhaseUnknown ClusterServiceVersionPhase = "Unknown"
)

// ConditionReason is a camelcased reason for the state transition
type ConditionReason string

const (
	CSVReasonRequirementsUnknown ConditionReason = "RequirementsUnknown"
	CSVReasonRequirementsNotMet  ConditionReason = "RequirementsNotMet"
	CSVReasonRequirementsMet     ConditionReason = "AllRequirementsMet"
	CSVReasonComponentFailed     ConditionReason = "InstallComponentFailed"
	CSVReasonInstallSuccessful   ConditionReason = "InstallSucceeded"
	CSVReasonInstallCheckFailed  ConditionReason = "InstallCheckFailed"
	CSVReasonComponentUnhealthy  ConditionReason = "ComponentUnhealthy"
)

// Conditions appear in the status as a record of state transitions on the ClusterServiceVersion
type ClusterServiceVersionCondition struct {
	// Condition of the ClusterServiceVersion
	Phase ClusterServiceVersionPhase `json:"phase,omitempty"`
	// A human readable message indicating details about why the ClusterServiceVersion is in this condition.
	// +optional
	Message string `json:"message,omitempty"`
	// A brief CamelCase message indicating details about why the ClusterServiceVersion is in this state.
	// e.g. 'RequirementsNotMet'
	// +optional
	Reason ConditionReason `json:"reason,omitempty"`
	// Last time we updated the status
	// +optional
	LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
	// Last time the status transitioned from one status to another.
	// +optional
	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
}

// OwnsCRD determines whether the current CSV owns a paritcular CRD.
func (csv ClusterServiceVersion) OwnsCRD(name string) bool {
	for _, crdDescription := range csv.Spec.CustomResourceDefinitions.Owned {
		if crdDescription.Name == name {
			return true
		}
	}

	return false
}

type RequirementStatus struct {
	Group   string `json:"group"`
	Version string `json:"version"`
	Kind    string `json:"kind"`
	Name    string `json:"name"`
	Status  string `json:"status"`
	UUID    string `json:"uuid,omitempty"`
}

// ClusterServiceVersionStatus represents information about the status of a pod. Status may trail the actual
// state of a system.
type ClusterServiceVersionStatus struct {
	// Current condition of the ClusterServiceVersion
	Phase ClusterServiceVersionPhase `json:"phase,omitempty"`
	// A human readable message indicating details about why the ClusterServiceVersion is in this condition.
	// +optional
	Message string `json:"message,omitempty"`
	// A brief CamelCase message indicating details about why the ClusterServiceVersion is in this state.
	// e.g. 'RequirementsNotMet'
	// +optional
	Reason ConditionReason `json:"reason,omitempty"`
	// Last time we updated the status
	// +optional
	LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
	// Last time the status transitioned from one status to another.
	// +optional
	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
	// List of conditions, a history of state transitions
	Conditions []ClusterServiceVersionCondition `json:"conditions,omitempty"`
	// The status of each requirement for this CSV
	RequirementStatus []RequirementStatus `json:"requirementStatus,omitempty"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// ClusterServiceVersion is a Custom Resource of type `ClusterServiceVersionSpec`.
type ClusterServiceVersion struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata"`

	Spec   ClusterServiceVersionSpec   `json:"spec"`
	Status ClusterServiceVersionStatus `json:"status"`
}

// ClusterServiceVersionList represents a list of ClusterServiceVersions.
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
type ClusterServiceVersionList struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata"`

	Items []ClusterServiceVersion `json:"items"`
}

func (crd CustomResourceDefinitions) GetAllCrds() []CRDDescription {
	setOfCrds := map[string]CRDDescription{}
	for _, requiredCrd := range crd.Required {
		setOfCrds[requiredCrd.Name] = requiredCrd
	}

	for _, ownedCrd := range crd.Owned {
		setOfCrds[ownedCrd.Name] = ownedCrd
	}

	allCrds := []CRDDescription{}
	for _, value := range setOfCrds {
		allCrds = append(allCrds, value)
	}

	return allCrds
}
<commit_msg>fix(apis/clusterserviceversion): correct json tag<commit_after>// Package v1alpha1 implements all the required types and methods for parsing
// resources for v1alpha1 versioned ClusterServiceVersions.
package v1alpha1

import (
	"encoding/json"

	"github.com/coreos/go-semver/semver"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	GroupVersion = "v1alpha1" // used in registering ClusterServiceVersion scheme

	ClusterServiceVersionCRDName       = "clusterserviceversion-v1s.app.coreos.com"
	ClusterServiceVersionKind          = "ClusterServiceVersion-v1"
	ClusterServiceVersionCRDAPIVersion = "apiextensions.k8s.io/v1beta1" // API version w/ CRD support

)

// NamedInstallStrategy represents the block of an ClusterServiceVersion resource
// where the install strategy is specified.
type NamedInstallStrategy struct {
	StrategyName    string          `json:"strategy"`
	StrategySpecRaw json.RawMessage `json:"spec,omitempty"`
}

// StatusDescriptor describes a field in a status block of a CRD so that ALM can consume it
type StatusDescriptor struct {
	Path         string          `json:"path"`
	DisplayName  string          `json:"displayName,omitempty"`
	Description  string          `json:"description,omitempty"`
	XDescriptors []string        `json:"x-descriptors,omitempty"`
	Value        json.RawMessage `json:"value,omitempty"`
}

// CRDDescription provides details to ALM about the CRDs
type CRDDescription struct {
	Name              string             `json:"name"`
	DisplayName       string             `json:"displayName,omitempty"`
	Description       string             `json:"description,omitempty"`
	StatusDescriptors []StatusDescriptor `json:"statusDescriptors,omitempty"`
}

// CustomResourceDefinitions declares all of the CRDs managed or required by
// an operator being ran by ClusterServiceVersion.
//
// If the CRD is present in the Owned list, it is implicitly required.
type CustomResourceDefinitions struct {
	Owned    []CRDDescription `json:"owned"`
	Required []CRDDescription `json:"required,omitempty"`
}

// ClusterServiceVersionSpec declarations tell the ALM how to install an operator
// that can manage apps for given version and AppType.
type ClusterServiceVersionSpec struct {
	InstallStrategy           NamedInstallStrategy      `json:"install"`
	Version                   semver.Version            `json:"version"`
	Maturity                  string                    `json:"maturity"`
	CustomResourceDefinitions CustomResourceDefinitions `json:"customresourcedefinitions"`
	DisplayName               string                    `json:"displayName"`
	Description               string                    `json:"description"`
	Keywords                  []string                  `json:"keywords"`
	Maintainers               []Maintainer              `json:"maintainers"`
	Provider                  AppLink                   `json:"provider"`
	Links                     []AppLink                 `json:"links"`
	Icon                      []Icon                    `json:"icon"`

	// The name of a CSV this one replaces. Should match the `metadata.Name` field of the old CSV.
	// +optional
	Replaces string `json:"replaces,omitempty"`

	// Map of string keys and values that can be used to organize and categorize
	// (scope and select) objects.
	// +optional
	Labels map[string]string `json:"labels,omitempty" protobuf:"bytes,11,rep,name=labels"`

	// Annotations is an unstructured key value map stored with a resource that may be
	// set by external tools to store and retrieve arbitrary metadata.
	// +optional
	Annotations map[string]string `json:"annotations,omitempty" protobuf:"bytes,12,rep,name=annotations"`

	// Label selector for related resources.
	// +optional
	Selector *metav1.LabelSelector `json:"selector,omitempty" protobuf:"bytes,2,opt,name=selector"`
}

type Maintainer struct {
	Name  string `json:"name"`
	Email string `json:"email"`
}

type AppLink struct {
	Name string `json:"name"`
	URL  string `json:"url"`
}

type Icon struct {
	Data      string `json:"base64data"`
	MediaType string `json:"mediatype"`
}

// ClusterServiceVersionPhase is a label for the condition of a ClusterServiceVersion at the current time.
type ClusterServiceVersionPhase string

// These are the valid phases of ClusterServiceVersion
const (
	CSVPhaseNone = ""
	// CSVPending means the csv has been accepted by the system, but the install strategy has not been attempted.
	// This is likely because there are unmet requirements.
	CSVPhasePending ClusterServiceVersionPhase = "Pending"
	// CSVRunning means that the requirements are met but the install strategy has not been run.
	CSVPhaseInstalling ClusterServiceVersionPhase = "Installing"
	// CSVSucceeded means that the resources in the CSV were created successfully.
	CSVPhaseSucceeded ClusterServiceVersionPhase = "Succeeded"
	// CSVFailed means that the install strategy could not be successfully completed.
	CSVPhaseFailed ClusterServiceVersionPhase = "Failed"
	// CSVUnknown means that for some reason the state of the csv could not be obtained.
	CSVPhaseUnknown ClusterServiceVersionPhase = "Unknown"
)

// ConditionReason is a camelcased reason for the state transition
type ConditionReason string

const (
	CSVReasonRequirementsUnknown ConditionReason = "RequirementsUnknown"
	CSVReasonRequirementsNotMet  ConditionReason = "RequirementsNotMet"
	CSVReasonRequirementsMet     ConditionReason = "AllRequirementsMet"
	CSVReasonComponentFailed     ConditionReason = "InstallComponentFailed"
	CSVReasonInstallSuccessful   ConditionReason = "InstallSucceeded"
	CSVReasonInstallCheckFailed  ConditionReason = "InstallCheckFailed"
	CSVReasonComponentUnhealthy  ConditionReason = "ComponentUnhealthy"
)

// Conditions appear in the status as a record of state transitions on the ClusterServiceVersion
type ClusterServiceVersionCondition struct {
	// Condition of the ClusterServiceVersion
	Phase ClusterServiceVersionPhase `json:"phase,omitempty"`
	// A human readable message indicating details about why the ClusterServiceVersion is in this condition.
	// +optional
	Message string `json:"message,omitempty"`
	// A brief CamelCase message indicating details about why the ClusterServiceVersion is in this state.
	// e.g. 'RequirementsNotMet'
	// +optional
	Reason ConditionReason `json:"reason,omitempty"`
	// Last time we updated the status
	// +optional
	LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
	// Last time the status transitioned from one status to another.
	// +optional
	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
}

// OwnsCRD determines whether the current CSV owns a paritcular CRD.
func (csv ClusterServiceVersion) OwnsCRD(name string) bool {
	for _, crdDescription := range csv.Spec.CustomResourceDefinitions.Owned {
		if crdDescription.Name == name {
			return true
		}
	}

	return false
}

type RequirementStatus struct {
	Group   string `json:"group"`
	Version string `json:"version"`
	Kind    string `json:"kind"`
	Name    string `json:"name"`
	Status  string `json:"status"`
	UUID    string `json:"uuid,omitempty"`
}

// ClusterServiceVersionStatus represents information about the status of a pod. Status may trail the actual
// state of a system.
type ClusterServiceVersionStatus struct {
	// Current condition of the ClusterServiceVersion
	Phase ClusterServiceVersionPhase `json:"phase,omitempty"`
	// A human readable message indicating details about why the ClusterServiceVersion is in this condition.
	// +optional
	Message string `json:"message,omitempty"`
	// A brief CamelCase message indicating details about why the ClusterServiceVersion is in this state.
	// e.g. 'RequirementsNotMet'
	// +optional
	Reason ConditionReason `json:"reason,omitempty"`
	// Last time we updated the status
	// +optional
	LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
	// Last time the status transitioned from one status to another.
	// +optional
	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
	// List of conditions, a history of state transitions
	Conditions []ClusterServiceVersionCondition `json:"conditions,omitempty"`
	// The status of each requirement for this CSV
	RequirementStatus []RequirementStatus `json:"requirementStatus,omitempty"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// ClusterServiceVersion is a Custom Resource of type `ClusterServiceVersionSpec`.
type ClusterServiceVersion struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata"`

	Spec   ClusterServiceVersionSpec   `json:"spec"`
	Status ClusterServiceVersionStatus `json:"status"`
}

// ClusterServiceVersionList represents a list of ClusterServiceVersions.
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
type ClusterServiceVersionList struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata"`

	Items []ClusterServiceVersion `json:"items"`
}

func (crd CustomResourceDefinitions) GetAllCrds() []CRDDescription {
	setOfCrds := map[string]CRDDescription{}
	for _, requiredCrd := range crd.Required {
		setOfCrds[requiredCrd.Name] = requiredCrd
	}

	for _, ownedCrd := range crd.Owned {
		setOfCrds[ownedCrd.Name] = ownedCrd
	}

	allCrds := []CRDDescription{}
	for _, value := range setOfCrds {
		allCrds = append(allCrds, value)
	}

	return allCrds
}
<|endoftext|>