<commit_before>package appinsights

// NOTE: This file was automatically generated.

import (
	"github.com/jjjordanmsft/ApplicationInsights-Go/appinsights/contracts"
	"strconv"
)

type TelemetryContext interface {
	InstrumentationKey() string

	Application() ApplicationContext
	Device() DeviceContext
	Location() LocationContext
	Operation() OperationContext
	Session() SessionContext
	User() UserContext
	Cloud() CloudContext
	Internal() InternalContext
}

type ApplicationContext interface {
	GetVer() string
	SetVer(value string)
}

type DeviceContext interface {
	GetId() string
	SetId(value string)
	GetLocale() string
	SetLocale(value string)
	GetModel() string
	SetModel(value string)
	GetOemName() string
	SetOemName(value string)
	GetOsVersion() string
	SetOsVersion(value string)
	GetType() string
	SetType(value string)
}

type LocationContext interface {
	GetIp() string
	SetIp(value string)
}

type OperationContext interface {
	GetId() string
	SetId(value string)
	GetName() string
	SetName(value string)
	GetParentId() string
	SetParentId(value string)
	GetSyntheticSource() string
	SetSyntheticSource(value string)
	GetCorrelationVector() string
	SetCorrelationVector(value string)
}

type SessionContext interface {
	GetId() string
	SetId(value string)
	GetIsFirst() string
	SetIsFirst(value string)
}

type UserContext interface {
	GetAccountId() string
	SetAccountId(value string)
	GetId() string
	SetId(value string)
	GetAuthUserId() string
	SetAuthUserId(value string)
}

type CloudContext interface {
	GetRole() string
	SetRole(value string)
	GetRoleInstance() string
	SetRoleInstance(value string)
}

type InternalContext interface {
	GetSdkVersion() string
	SetSdkVersion(value string)
	GetAgentVersion() string
	SetAgentVersion(value string)
	GetNodeName() string
	SetNodeName(value string)
}

type telemetryContext struct {
	iKey string
	tags map[string]string
}

type applicationContext struct {
	context *telemetryContext
}

type deviceContext struct {
	context *telemetryContext
}

type locationContext struct {
	context *telemetryContext
}

type operationContext struct {
	context *telemetryContext
}

type sessionContext struct {
	context *telemetryContext
}

type userContext struct {
	context *telemetryContext
}

type cloudContext struct {
	context *telemetryContext
}

type internalContext struct {
	context *telemetryContext
}

func NewTelemetryContext() TelemetryContext {
	return &telemetryContext{
		tags: make(map[string]string),
	}
}

func (context *telemetryContext) InstrumentationKey() string {
	return context.iKey
}

func (context *telemetryContext) Application() ApplicationContext {
	return &applicationContext{context: context}
}

func (context *telemetryContext) Device() DeviceContext {
	return &deviceContext{context: context}
}

func (context *telemetryContext) Location() LocationContext {
	return &locationContext{context: context}
}

func (context *telemetryContext) Operation() OperationContext {
	return &operationContext{context: context}
}

func (context *telemetryContext) Session() SessionContext {
	return &sessionContext{context: context}
}

func (context *telemetryContext) User() UserContext {
	return &userContext{context: context}
}

func (context *telemetryContext) Cloud() CloudContext {
	return &cloudContext{context: context}
}

func (context *telemetryContext) Internal() InternalContext {
	return &internalContext{context: context}
}

func (context *telemetryContext) getStringTag(key string) string {
	if result, ok := context.tags[key]; ok {
		return result
	}

	return ""
}

func (context *telemetryContext) setStringTag(key, value string) {
	if value != "" {
		context.tags[key] = value
	} else {
		delete(context.tags, key)
	}
}

func (context *telemetryContext) getBoolTag(key string) bool {
	if result, ok := context.tags[key]; ok {
		if value, err := strconv.ParseBool(result); err == nil {
			return value
		}
	}

	return false
}

func (context *telemetryContext) setBoolTag(key string, value bool) {
	if value {
		context.tags[key] = "true"
	} else {
		delete(context.tags, key)
	}
}

func (context *applicationContext) GetVer() string {
	return context.context.getStringTag("ai.application.ver")
}

func (context *applicationContext) SetVer(value string) {
	context.context.setStringTag("ai.application.ver", value)
}

func (context *deviceContext) GetId() string {
	return context.context.getStringTag("ai.device.id")
}

func (context *deviceContext) SetId(value string) {
	context.context.setStringTag("ai.device.id", value)
}

func (context *deviceContext) GetLocale() string {
	return context.context.getStringTag("ai.device.locale")
}

func (context *deviceContext) SetLocale(value string) {
	context.context.setStringTag("ai.device.locale", value)
}

func (context *deviceContext) GetModel() string {
	return context.context.getStringTag("ai.device.model")
}

func (context *deviceContext) SetModel(value string) {
	context.context.setStringTag("ai.device.model", value)
}

func (context *deviceContext) GetOemName() string {
	return context.context.getStringTag("ai.device.oemName")
}

func (context *deviceContext) SetOemName(value string) {
	context.context.setStringTag("ai.device.oemName", value)
}

func (context *deviceContext) GetOsVersion() string {
	return context.context.getStringTag("ai.device.osVersion")
}

func (context *deviceContext) SetOsVersion(value string) {
	context.context.setStringTag("ai.device.osVersion", value)
}

func (context *deviceContext) GetType() string {
	return context.context.getStringTag("ai.device.type")
}

func (context *deviceContext) SetType(value string) {
	context.context.setStringTag("ai.device.type", value)
}

func (context *locationContext) GetIp() string {
	return context.context.getStringTag("ai.location.ip")
}

func (context *locationContext) SetIp(value string) {
	context.context.setStringTag("ai.location.ip", value)
}

func (context *operationContext) GetId() string {
	return context.context.getStringTag("ai.operation.id")
}

func (context *operationContext) SetId(value string) {
	context.context.setStringTag("ai.operation.id", value)
}

func (context *operationContext) GetName() string {
	return context.context.getStringTag("ai.operation.name")
}

func (context *operationContext) SetName(value string) {
	context.context.setStringTag("ai.operation.name", value)
}

func (context *operationContext) GetParentId() string {
	return context.context.getStringTag("ai.operation.parentId")
}

func (context *operationContext) SetParentId(value string) {
	context.context.setStringTag("ai.operation.parentId", value)
}

func (context *operationContext) GetSyntheticSource() string {
	return context.context.getStringTag("ai.operation.syntheticSource")
}

func (context *operationContext) SetSyntheticSource(value string) {
	context.context.setStringTag("ai.operation.syntheticSource", value)
}

func (context *operationContext) GetCorrelationVector() string {
	return context.context.getStringTag("ai.operation.correlationVector")
}

func (context *operationContext) SetCorrelationVector(value string) {
	context.context.setStringTag("ai.operation.correlationVector", value)
}

func (context *sessionContext) GetId() string {
	return context.context.getStringTag("ai.session.id")
}

func (context *sessionContext) SetId(value string) {
	context.context.setStringTag("ai.session.id", value)
}

func (context *sessionContext) GetIsFirst() string {
	return context.context.getStringTag("ai.session.isFirst")
}

func (context *sessionContext) SetIsFirst(value string) {
	context.context.setStringTag("ai.session.isFirst", value)
}

func (context *userContext) GetAccountId() string {
	return context.context.getStringTag("ai.user.accountId")
}

func (context *userContext) SetAccountId(value string) {
	context.context.setStringTag("ai.user.accountId", value)
}

func (context *userContext) GetId() string {
	return context.context.getStringTag("ai.user.id")
}

func (context *userContext) SetId(value string) {
	context.context.setStringTag("ai.user.id", value)
}

func (context *userContext) GetAuthUserId() string {
	return context.context.getStringTag("ai.user.authUserId")
}

func (context *userContext) SetAuthUserId(value string) {
	context.context.setStringTag("ai.user.authUserId", value)
}

func (context *cloudContext) GetRole() string {
	return context.context.getStringTag("ai.cloud.role")
}

func (context *cloudContext) SetRole(value string) {
	context.context.setStringTag("ai.cloud.role", value)
}

func (context *cloudContext) GetRoleInstance() string {
	return context.context.getStringTag("ai.cloud.roleInstance")
}

func (context *cloudContext) SetRoleInstance(value string) {
	context.context.setStringTag("ai.cloud.roleInstance", value)
}

func (context *internalContext) GetSdkVersion() string {
	return context.context.getStringTag("ai.internal.sdkVersion")
}

func (context *internalContext) SetSdkVersion(value string) {
	context.context.setStringTag("ai.internal.sdkVersion", value)
}

func (context *internalContext) GetAgentVersion() string {
	return context.context.getStringTag("ai.internal.agentVersion")
}

func (context *internalContext) SetAgentVersion(value string) {
	context.context.setStringTag("ai.internal.agentVersion", value)
}

func (context *internalContext) GetNodeName() string {
	return context.context.getStringTag("ai.internal.nodeName")
}

func (context *internalContext) SetNodeName(value string) {
	context.context.setStringTag("ai.internal.nodeName", value)
}
<commit_msg>Remove unused reference<commit_after>package appinsights

// NOTE: This file was automatically generated.

import "strconv"

type TelemetryContext interface {
	InstrumentationKey() string

	Application() ApplicationContext
	Device() DeviceContext
	Location() LocationContext
	Operation() OperationContext
	Session() SessionContext
	User() UserContext
	Cloud() CloudContext
	Internal() InternalContext
}

type ApplicationContext interface {
	GetVer() string
	SetVer(value string)
}

type DeviceContext interface {
	GetId() string
	SetId(value string)
	GetLocale() string
	SetLocale(value string)
	GetModel() string
	SetModel(value string)
	GetOemName() string
	SetOemName(value string)
	GetOsVersion() string
	SetOsVersion(value string)
	GetType() string
	SetType(value string)
}

type LocationContext interface {
	GetIp() string
	SetIp(value string)
}

type OperationContext interface {
	GetId() string
	SetId(value string)
	GetName() string
	SetName(value string)
	GetParentId() string
	SetParentId(value string)
	GetSyntheticSource() string
	SetSyntheticSource(value string)
	GetCorrelationVector() string
	SetCorrelationVector(value string)
}

type SessionContext interface {
	GetId() string
	SetId(value string)
	GetIsFirst() string
	SetIsFirst(value string)
}

type UserContext interface {
	GetAccountId() string
	SetAccountId(value string)
	GetId() string
	SetId(value string)
	GetAuthUserId() string
	SetAuthUserId(value string)
}

type CloudContext interface {
	GetRole() string
	SetRole(value string)
	GetRoleInstance() string
	SetRoleInstance(value string)
}

type InternalContext interface {
	GetSdkVersion() string
	SetSdkVersion(value string)
	GetAgentVersion() string
	SetAgentVersion(value string)
	GetNodeName() string
	SetNodeName(value string)
}

type telemetryContext struct {
	iKey string
	tags map[string]string
}

type applicationContext struct {
	context *telemetryContext
}

type deviceContext struct {
	context *telemetryContext
}

type locationContext struct {
	context *telemetryContext
}

type operationContext struct {
	context *telemetryContext
}

type sessionContext struct {
	context *telemetryContext
}

type userContext struct {
	context *telemetryContext
}

type cloudContext struct {
	context *telemetryContext
}

type internalContext struct {
	context *telemetryContext
}

func NewTelemetryContext() TelemetryContext {
	return &telemetryContext{
		tags: make(map[string]string),
	}
}

func (context *telemetryContext) InstrumentationKey() string {
	return context.iKey
}

func (context *telemetryContext) Application() ApplicationContext {
	return &applicationContext{context: context}
}

func (context *telemetryContext) Device() DeviceContext {
	return &deviceContext{context: context}
}

func (context *telemetryContext) Location() LocationContext {
	return &locationContext{context: context}
}

func (context *telemetryContext) Operation() OperationContext {
	return &operationContext{context: context}
}

func (context *telemetryContext) Session() SessionContext {
	return &sessionContext{context: context}
}

func (context *telemetryContext) User() UserContext {
	return &userContext{context: context}
}

func (context *telemetryContext) Cloud() CloudContext {
	return &cloudContext{context: context}
}

func (context *telemetryContext) Internal() InternalContext {
	return &internalContext{context: context}
}

func (context *telemetryContext) getStringTag(key string) string {
	if result, ok := context.tags[key]; ok {
		return result
	}

	return ""
}

func (context *telemetryContext) setStringTag(key, value string) {
	if value != "" {
		context.tags[key] = value
	} else {
		delete(context.tags, key)
	}
}

func (context *telemetryContext) getBoolTag(key string) bool {
	if result, ok := context.tags[key]; ok {
		if value, err := strconv.ParseBool(result); err == nil {
			return value
		}
	}

	return false
}

func (context *telemetryContext) setBoolTag(key string, value bool) {
	if value {
		context.tags[key] = "true"
	} else {
		delete(context.tags, key)
	}
}

func (context *applicationContext) GetVer() string {
	return context.context.getStringTag("ai.application.ver")
}

func (context *applicationContext) SetVer(value string) {
	context.context.setStringTag("ai.application.ver", value)
}

func (context *deviceContext) GetId() string {
	return context.context.getStringTag("ai.device.id")
}

func (context *deviceContext) SetId(value string) {
	context.context.setStringTag("ai.device.id", value)
}

func (context *deviceContext) GetLocale() string {
	return context.context.getStringTag("ai.device.locale")
}

func (context *deviceContext) SetLocale(value string) {
	context.context.setStringTag("ai.device.locale", value)
}

func (context *deviceContext) GetModel() string {
	return context.context.getStringTag("ai.device.model")
}

func (context *deviceContext) SetModel(value string) {
	context.context.setStringTag("ai.device.model", value)
}

func (context *deviceContext) GetOemName() string {
	return context.context.getStringTag("ai.device.oemName")
}

func (context *deviceContext) SetOemName(value string) {
	context.context.setStringTag("ai.device.oemName", value)
}

func (context *deviceContext) GetOsVersion() string {
	return context.context.getStringTag("ai.device.osVersion")
}

func (context *deviceContext) SetOsVersion(value string) {
	context.context.setStringTag("ai.device.osVersion", value)
}

func (context *deviceContext) GetType() string {
	return context.context.getStringTag("ai.device.type")
}

func (context *deviceContext) SetType(value string) {
	context.context.setStringTag("ai.device.type", value)
}

func (context *locationContext) GetIp() string {
	return context.context.getStringTag("ai.location.ip")
}

func (context *locationContext) SetIp(value string) {
	context.context.setStringTag("ai.location.ip", value)
}

func (context *operationContext) GetId() string {
	return context.context.getStringTag("ai.operation.id")
}

func (context *operationContext) SetId(value string) {
	context.context.setStringTag("ai.operation.id", value)
}

func (context *operationContext) GetName() string {
	return context.context.getStringTag("ai.operation.name")
}

func (context *operationContext) SetName(value string) {
	context.context.setStringTag("ai.operation.name", value)
}

func (context *operationContext) GetParentId() string {
	return context.context.getStringTag("ai.operation.parentId")
}

func (context *operationContext) SetParentId(value string) {
	context.context.setStringTag("ai.operation.parentId", value)
}

func (context *operationContext) GetSyntheticSource() string {
	return context.context.getStringTag("ai.operation.syntheticSource")
}

func (context *operationContext) SetSyntheticSource(value string) {
	context.context.setStringTag("ai.operation.syntheticSource", value)
}

func (context *operationContext) GetCorrelationVector() string {
	return context.context.getStringTag("ai.operation.correlationVector")
}

func (context *operationContext) SetCorrelationVector(value string) {
	context.context.setStringTag("ai.operation.correlationVector", value)
}

func (context *sessionContext) GetId() string {
	return context.context.getStringTag("ai.session.id")
}

func (context *sessionContext) SetId(value string) {
	context.context.setStringTag("ai.session.id", value)
}

func (context *sessionContext) GetIsFirst() string {
	return context.context.getStringTag("ai.session.isFirst")
}

func (context *sessionContext) SetIsFirst(value string) {
	context.context.setStringTag("ai.session.isFirst", value)
}

func (context *userContext) GetAccountId() string {
	return context.context.getStringTag("ai.user.accountId")
}

func (context *userContext) SetAccountId(value string) {
	context.context.setStringTag("ai.user.accountId", value)
}

func (context *userContext) GetId() string {
	return context.context.getStringTag("ai.user.id")
}

func (context *userContext) SetId(value string) {
	context.context.setStringTag("ai.user.id", value)
}

func (context *userContext) GetAuthUserId() string {
	return context.context.getStringTag("ai.user.authUserId")
}

func (context *userContext) SetAuthUserId(value string) {
	context.context.setStringTag("ai.user.authUserId", value)
}

func (context *cloudContext) GetRole() string {
	return context.context.getStringTag("ai.cloud.role")
}

func (context *cloudContext) SetRole(value string) {
	context.context.setStringTag("ai.cloud.role", value)
}

func (context *cloudContext) GetRoleInstance() string {
	return context.context.getStringTag("ai.cloud.roleInstance")
}

func (context *cloudContext) SetRoleInstance(value string) {
	context.context.setStringTag("ai.cloud.roleInstance", value)
}

func (context *internalContext) GetSdkVersion() string {
	return context.context.getStringTag("ai.internal.sdkVersion")
}

func (context *internalContext) SetSdkVersion(value string) {
	context.context.setStringTag("ai.internal.sdkVersion", value)
}

func (context *internalContext) GetAgentVersion() string {
	return context.context.getStringTag("ai.internal.agentVersion")
}

func (context *internalContext) SetAgentVersion(value string) {
	context.context.setStringTag("ai.internal.agentVersion", value)
}

func (context *internalContext) GetNodeName() string {
	return context.context.getStringTag("ai.internal.nodeName")
}

func (context *internalContext) SetNodeName(value string) {
	context.context.setStringTag("ai.internal.nodeName", value)
}
<|endoftext|>


<commit_before>// Package poly provides polygon detection methods.
package poly

import "fmt"

// Point is simple 2D point
type Point struct {
	X, Y, Z float64
}

// InsideRect detects point is inside of another rect
func (p Point) InsideRect(rect Rect) bool {
	if p.X < rect.Min.X || p.X > rect.Max.X {
		return false
	}
	if p.Y < rect.Min.Y || p.Y > rect.Max.Y {
		return false
	}
	return true
}

// Polygon is series of points that make up a polygon
type Polygon []Point

// InsideRect detects polygon is inside of another rect
func (p Polygon) InsideRect(rect Rect) bool {
	if len(p) == 0 {
		return false
	}
	for _, p := range p {
		if !p.InsideRect(rect) {
			return false
		}
	}
	return true
}

// IntersectsRect detects polygon is inside of another rect
func (p Polygon) IntersectsRect(rect Rect) bool {
	if len(p) == 0 {
		return false
	}
	rectPoly := Polygon{rect.Min, {rect.Min.X, rect.Max.Y, 0}, rect.Max, {rect.Max.X, rect.Min.Y, 0}, rect.Min}
	return p.Intersects(rectPoly, nil)
}

// String returns a string representation of the polygon.
func (p Polygon) String() string {
	s := "{"
	for i, p := range p {
		if i > 0 {
			s += ", "
		}
		s += fmt.Sprintf("{%v, %v}", p.X, p.Y)
	}
	s += "}"
	return s
}

// Rect is rectangle
type Rect struct {
	Min, Max Point
}

// Polygon returns a polygon for the rect
func (r Rect) Polygon() Polygon {
	p := Polygon(make([]Point, 5))
	p[0] = Point{X: r.Min.X, Y: r.Max.Y}
	p[1] = Point{X: r.Max.X, Y: r.Max.Y}
	p[2] = Point{X: r.Max.X, Y: r.Min.Y}
	p[3] = Point{X: r.Min.X, Y: r.Min.Y}
	p[4] = Point{X: r.Min.X, Y: r.Max.Y}
	return p
}

// Rect returns the bounding box rectangle for the polygon
func (p Polygon) Rect() Rect {
	var bbox Rect
	for i, p := range p {
		if i == 0 {
			bbox.Min = p
			bbox.Max = p
		} else {
			if p.X < bbox.Min.X {
				bbox.Min.X = p.X
			} else if p.X > bbox.Max.X {
				bbox.Max.X = p.X
			}
			if p.Y < bbox.Min.Y {
				bbox.Min.Y = p.Y
			} else if p.Y > bbox.Max.Y {
				bbox.Max.Y = p.Y
			}
		}
	}
	return bbox
}

// IntersectsRect detects if two bboxes intersect.
func (r Rect) IntersectsRect(rect Rect) bool {
	if r.Min.Y > rect.Max.Y || r.Max.Y < rect.Min.Y {
		return false
	}
	if r.Min.X > rect.Max.X || r.Max.X < rect.Min.X {
		return false
	}
	return true
}

// InsideRect detects rect is inside of another rect
func (r Rect) InsideRect(rect Rect) bool {
	if r.Min.X < rect.Min.X || r.Max.X > rect.Max.X {
		return false
	}
	if r.Min.Y < rect.Min.Y || r.Max.Y > rect.Max.Y {
		return false
	}
	return true
}
<commit_msg>documentation: Point is [lat lon z]<commit_after>// Package poly provides polygon detection methods.
package poly

import "fmt"

// Point is simple 2D point
type Point struct {
	X, Y, Z float64 // X=latitude, Y=longitude, (Z is optional, e.g. for elevation or timestamp)
}

// InsideRect detects point is inside of another rect
func (p Point) InsideRect(rect Rect) bool {
	if p.X < rect.Min.X || p.X > rect.Max.X {
		return false
	}
	if p.Y < rect.Min.Y || p.Y > rect.Max.Y {
		return false
	}
	return true
}

// Polygon is series of points that make up a polygon
type Polygon []Point

// InsideRect detects polygon is inside of another rect
func (p Polygon) InsideRect(rect Rect) bool {
	if len(p) == 0 {
		return false
	}
	for _, p := range p {
		if !p.InsideRect(rect) {
			return false
		}
	}
	return true
}

// IntersectsRect detects polygon is inside of another rect
func (p Polygon) IntersectsRect(rect Rect) bool {
	if len(p) == 0 {
		return false
	}
	rectPoly := Polygon{rect.Min, {rect.Min.X, rect.Max.Y, 0}, rect.Max, {rect.Max.X, rect.Min.Y, 0}, rect.Min}
	return p.Intersects(rectPoly, nil)
}

// String returns a string representation of the polygon.
func (p Polygon) String() string {
	s := "{"
	for i, p := range p {
		if i > 0 {
			s += ", "
		}
		s += fmt.Sprintf("{%v, %v}", p.X, p.Y)
	}
	s += "}"
	return s
}

// Rect is rectangle
type Rect struct {
	Min, Max Point
}

// Polygon returns a polygon for the rect
func (r Rect) Polygon() Polygon {
	p := Polygon(make([]Point, 5))
	p[0] = Point{X: r.Min.X, Y: r.Max.Y}
	p[1] = Point{X: r.Max.X, Y: r.Max.Y}
	p[2] = Point{X: r.Max.X, Y: r.Min.Y}
	p[3] = Point{X: r.Min.X, Y: r.Min.Y}
	p[4] = Point{X: r.Min.X, Y: r.Max.Y}
	return p
}

// Rect returns the bounding box rectangle for the polygon
func (p Polygon) Rect() Rect {
	var bbox Rect
	for i, p := range p {
		if i == 0 {
			bbox.Min = p
			bbox.Max = p
		} else {
			if p.X < bbox.Min.X {
				bbox.Min.X = p.X
			} else if p.X > bbox.Max.X {
				bbox.Max.X = p.X
			}
			if p.Y < bbox.Min.Y {
				bbox.Min.Y = p.Y
			} else if p.Y > bbox.Max.Y {
				bbox.Max.Y = p.Y
			}
		}
	}
	return bbox
}

// IntersectsRect detects if two bboxes intersect.
func (r Rect) IntersectsRect(rect Rect) bool {
	if r.Min.Y > rect.Max.Y || r.Max.Y < rect.Min.Y {
		return false
	}
	if r.Min.X > rect.Max.X || r.Max.X < rect.Min.X {
		return false
	}
	return true
}

// InsideRect detects rect is inside of another rect
func (r Rect) InsideRect(rect Rect) bool {
	if r.Min.X < rect.Min.X || r.Max.X > rect.Max.X {
		return false
	}
	if r.Min.Y < rect.Min.Y || r.Max.Y > rect.Max.Y {
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/rancher/norman/types/convert"
	libhelm "github.com/rancher/rancher/pkg/catalog/helm"
	"github.com/rancher/types/apis/management.cattle.io/v3"
	"github.com/rancher/types/image"
	"gopkg.in/yaml.v2"
)

var (
	scriptMap = map[string]string{
		"linux-save":     linuxSaveScript,
		"linux-load":     linuxLoadScript,
		"linux-mirror":   linuxMirrorScript,
		"windows-save":   windowsSaveScirpt,
		"windows-load":   windowsLoadScript,
		"windows-mirror": windowsMirrorScript,
	}
	scriptNameMap = map[string]string{
		"linux-save":     "rancher-save-images.sh",
		"linux-load":     "rancher-load-images.sh",
		"linux-mirror":   "rancher-mirror-to-rancher-org.sh",
		"windows-save":   "rancher-save-images.ps1",
		"windows-load":   "rancher-load-images.ps1",
		"windows-mirror": "rancher-mirror-to-rancher-org.ps1",
	}
	filenameMap = map[string]string{
		"linux":   "rancher-images.txt",
		"windows": "rancher-windows-images.txt",
	}
)

func main() {
	if len(os.Args) < 2 {
		log.Fatal("system charts path is required, please set it as the first parameter")
	}
	images, err := getImagesFromCharts(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}

	images = append(images, os.Args[2:]...)

	if err := run(images...); err != nil {
		log.Fatal(err)
	}
}

func run(images ...string) error {
	targetImages, err := collectionImages(v3.K8sVersionToRKESystemImages, v3.ToolsSystemImages)
	if err != nil {
		return err
	}

	for _, i := range images {
		targetImages = append(targetImages, image.Mirror(i))
	}

	targetWindowsImages, err := collectionImages(v3.K8sVersionWindowsSystemImages)
	if err != nil {
		return err
	}

	if agentImage := getWindowsAgentImage(); agentImage != "" {
		targetWindowsImages = append(targetWindowsImages, image.Mirror(agentImage))
	}

	for arch, images := range map[string][]string{
		"linux":   targetImages,
		"windows": targetWindowsImages,
	} {
		err = imagesText(arch, images)
		if err != nil {
			return err
		}

		err = mirrorScript(arch, images)
		if err != nil {
			return err
		}

		err = saveScript(arch, images)
		if err != nil {
			return err
		}

		err = loadScript(arch, images)
		if err != nil {
			return err
		}
	}

	return nil
}

func loadScript(arch string, targetImages []string) error {
	loadScriptName := getScriptFilename(arch, "load")
	log.Printf("Creating %s\n", loadScriptName)
	load, err := os.Create(loadScriptName)
	if err != nil {
		return err
	}
	defer load.Close()
	load.Chmod(0755)

	fmt.Fprintf(load, getScript(arch, "load"))
	return nil
}

func saveImages(targetImages []string) []string {
	var saveImages []string
	for _, targetImage := range targetImages {
		_, ok := image.Mirrors[targetImage]
		if !ok {
			continue
		}

		saveImages = append(saveImages, targetImage)
	}
	return saveImages
}

func saveScript(arch string, targetImages []string) error {
	filename := getScriptFilename(arch, "save")
	log.Printf("Creating %s\n", filename)
	save, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer save.Close()
	save.Chmod(0755)

	fmt.Fprintf(save, getScript(arch, "save"))

	return nil
}

func imagesText(arch string, targetImages []string) error {
	filename := filenameMap[arch]
	log.Printf("Creating %s\n", filename)
	save, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer save.Close()
	save.Chmod(0755)

	for _, image := range saveImages(targetImages) {
		log.Println("Image:", image)
		fmt.Fprintln(save, image)
	}

	return nil
}

func mirrorScript(arch string, targetImages []string) error {
	filename := getScriptFilename(arch, "mirror")
	log.Printf("Creating %s\n", filename)
	mirror, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer mirror.Close()
	mirror.Chmod(0755)

	scriptStarter := getScript(arch, "mirror")
	fmt.Fprintf(mirror, scriptStarter)

	var saveImages []string
	for _, targetImage := range targetImages {
		srcImage, ok := image.Mirrors[targetImage]
		if !ok {
			continue
		}

		saveImages = append(saveImages, targetImage)
		fmt.Fprintf(mirror, "docker pull %s\n", srcImage)
		if targetImage != srcImage {
			fmt.Fprintf(mirror, "docker tag %s %s\n", srcImage, targetImage)
			fmt.Fprintf(mirror, "docker push %s\n", targetImage)
		}
	}

	return nil
}

func collectionImages(objs ...interface{}) ([]string, error) {
	images := map[string]bool{}

	for _, obj := range objs {
		data := map[string]interface{}{}
		if err := convert.ToObj(obj, &data); err != nil {
			return nil, err
		}
		findStrings(data, images)
	}

	var result []string
	for k := range images {
		result = append(result, k)
	}
	sort.Slice(result, func(i, j int) bool {
		return result[i] < result[j]
	})

	return result, nil
}

func findStrings(obj map[string]interface{}, found map[string]bool) {
	for _, v := range obj {
		switch t := v.(type) {
		case string:
			found[t] = true
		case map[string]interface{}:
			findStrings(t, found)
		}
	}
}

func getImagesFromCharts(path string) ([]string, error) {
	var images []string
	imageMap := map[string]struct{}{}
	chartVersion, err := getChartAndVersion(path)
	if err != nil {
		return nil, err
	}
	if err := filepath.Walk(path, func(p string, info os.FileInfo, err error) error {
		return walkFunc(imageMap, chartVersion, path, p, info, err)
	}); err != nil {
		return images, err
	}
	for value := range imageMap {
		images = append(images, value)
	}
	return images, nil
}

func getChartAndVersion(path string) (map[string]string, error) {
	rtn := map[string]string{}
	helm := libhelm.Helm{
		LocalPath: path,
		IconPath:  path,
		Hash:      "",
	}
	index, err := helm.LoadIndex()
	if err != nil {
		return nil, err
	}
	for k, versions := range index.IndexFile.Entries {
		// because versions is sorted in reverse order, the first one will be the latest version
		if len(versions) > 0 {
			rtn[k] = versions[0].Dir
		}
	}

	return rtn, nil
}

func walkFunc(images map[string]struct{}, versions map[string]string, basePath, path string, info os.FileInfo, err error) error {
	relPath, err := filepath.Rel(basePath, path)
	if err != nil {
		return err
	}
	var found bool
	for _, v := range versions {
		if strings.HasPrefix(relPath, v) {
			found = true
			break
		}
	}
	if !found || info.Name() != "values.yaml" {
		return nil
	}
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	dataInterface := map[interface{}]interface{}{}
	if err := yaml.Unmarshal(data, &dataInterface); err != nil {
		return err
	}

	walkthroughMap(dataInterface, func(inputMap map[interface{}]interface{}) {
		generateImages(inputMap, images)
	})
	return nil
}

func generateImages(inputMap map[interface{}]interface{}, output map[string]struct{}) {
	r, repoOk := inputMap["repository"]
	t, tagOk := inputMap["tag"]
	if !repoOk || !tagOk {
		return
	}
	repo, repoOk := r.(string)
	tag, tagOk := t.(string)
	if !repoOk || !tagOk {
		return
	}

	output[fmt.Sprintf("%s:%s", repo, tag)] = struct{}{}

	return
}

func walkthroughMap(inputMap map[interface{}]interface{}, walkFunc func(map[interface{}]interface{})) {
	walkFunc(inputMap)
	for _, value := range inputMap {
		if v, ok := value.(map[interface{}]interface{}); ok {
			walkthroughMap(v, walkFunc)
		}
	}
}

func getWindowsAgentImage() string {
	tag, ok := os.LookupEnv("TAG")
	if !ok {
		return ""
	}
	repo, ok := os.LookupEnv("REPO")
	if !ok {
		return ""
	}
	return fmt.Sprintf("%s/rancher-agent:%s", repo, tag)
}

func getScript(arch, fileType string) string {
	return scriptMap[fmt.Sprintf("%s-%s", arch, fileType)]
}

func getScriptFilename(arch, fileType string) string {
	return scriptNameMap[fmt.Sprintf("%s-%s", arch, fileType)]
}

const (
	linuxLoadScript = `#!/bin/bash
list="rancher-images.txt"
images="rancher-images.tar.gz"

POSITIONAL=()
while [[ $# -gt 0 ]]; do
    key="$1"
    case $key in
        -r|--registry)
        reg="$2"
        shift # past argument
        shift # past value
        ;;
        -l|--image-list)
        list="$2"
        shift # past argument
        shift # past value
        ;;
        -i|--images)
        images="$2"
        shift # past argument
        shift # past value
        ;;
        -h|--help)
        help="true"
        shift
        ;;
    esac
done

usage () {
    echo "USAGE: $0 [--image-list rancher-images.txt] [--images rancher-images.tar.gz] --registry my.registry.com:5000"
    echo "  [-l|--images-list path] text file with list of images. 1 per line."
    echo "  [-l|--images path] tar.gz generated by docker save."
    echo "  [-r|--registry registry:port] target private registry:port."
    echo "  [-h|--help] Usage message"
}

if [[ -z $reg ]]; then
    usage
    exit 1
fi
if [[ $help ]]; then
    usage
    exit 0
fi

set -e -x

docker load --input ${images}

for i in $(cat ${list}); do
    docker tag ${i} ${reg}/${i}
    docker push ${reg}/${i}
done
`
	linuxSaveScript = `#!/bin/bash
list="rancher-images.txt"
images="rancher-images.tar.gz"

POSITIONAL=()
while [[ $# -gt 0 ]]; do
	key="$1"
	case $key in
		-i|--images)
		images="$2"
		shift # past argument
		shift # past value
		;;
		-l|--image-list)
		list="$2"
		shift # past argument
		shift # past value
		;;
		-h|--help)
		help="true"
		shift
		;;
	esac
done

usage () {
	echo "USAGE: $0 [--image-list rancher-images.txt] [--images rancher-images.tar.gz]"
	echo "  [-l|--images-list path] text file with list of images. 1 per line."
	echo "  [-l|--images path] tar.gz generated by docker save."
	echo "  [-h|--help] Usage message"
}

if [[ $help ]]; then
	usage
	exit 0
fi

set -e -x

for i in $(cat ${list}); do
	docker pull ${i}
done

docker save $(cat ${list} | tr '\n' ' ') | gzip -c > ${images}
`
	linuxMirrorScript = "#!/bin/sh\nset -e -x\n\n"
	windowsLoadScript = `<#
    .PARAMETER  registry
    target private registry:port.
    .PARAMETER image-list
    text file with list of images. 1 per line. default is %s
    .PARAMETER images
    tar.gz generated by docker save. default is rancher-windows-images.tar.gz
#>
param(
    [PARAMETER(Mandatory=$true,Position=0,HelpMessage="target private registry:port.")][string]$registry,
    [string]${image-list}="%s",
    [string]$images="rancher-windows-images.tar.gz"
)

$content=Get-Content -path ${image-list}

docker load --input ${images}

foreach ($item in $content) {
    docker tag $item $Registry/$item
    docker push $Registry/$item
}
`
	windowsSaveScirpt = `#Requires -Version 5.0
<#
	.PARAMETER image-list
	text file with list of images. 1 per line.
	.PARAMETER images
	tar.gz generated by docker save.
#>
param(
	[string]${image-list}="rancher-windows-images.txt",
	[string]$images="rancher-windows-images.tar.gz"
)

$content=Get-Content -path ${image-list}

foreach ($item in $content) {
	docker pull $item
}

docker save $($content) -o $images
`
	windowsMirrorScript = ``
)
<commit_msg>Push registry:2 to priv.repo/rancher/registry:2<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/rancher/norman/types/convert"
	libhelm "github.com/rancher/rancher/pkg/catalog/helm"
	"github.com/rancher/types/apis/management.cattle.io/v3"
	"github.com/rancher/types/image"
	"gopkg.in/yaml.v2"
)

var (
	scriptMap = map[string]string{
		"linux-save":     linuxSaveScript,
		"linux-load":     linuxLoadScript,
		"linux-mirror":   linuxMirrorScript,
		"windows-save":   windowsSaveScirpt,
		"windows-load":   windowsLoadScript,
		"windows-mirror": windowsMirrorScript,
	}
	scriptNameMap = map[string]string{
		"linux-save":     "rancher-save-images.sh",
		"linux-load":     "rancher-load-images.sh",
		"linux-mirror":   "rancher-mirror-to-rancher-org.sh",
		"windows-save":   "rancher-save-images.ps1",
		"windows-load":   "rancher-load-images.ps1",
		"windows-mirror": "rancher-mirror-to-rancher-org.ps1",
	}
	filenameMap = map[string]string{
		"linux":   "rancher-images.txt",
		"windows": "rancher-windows-images.txt",
	}
)

func main() {
	if len(os.Args) < 2 {
		log.Fatal("system charts path is required, please set it as the first parameter")
	}
	images, err := getImagesFromCharts(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}

	images = append(images, os.Args[2:]...)

	if err := run(images...); err != nil {
		log.Fatal(err)
	}
}

func run(images ...string) error {
	targetImages, err := collectionImages(v3.K8sVersionToRKESystemImages, v3.ToolsSystemImages)
	if err != nil {
		return err
	}

	for _, i := range images {
		targetImages = append(targetImages, image.Mirror(i))
	}

	targetWindowsImages, err := collectionImages(v3.K8sVersionWindowsSystemImages)
	if err != nil {
		return err
	}

	if agentImage := getWindowsAgentImage(); agentImage != "" {
		targetWindowsImages = append(targetWindowsImages, image.Mirror(agentImage))
	}

	for arch, images := range map[string][]string{
		"linux":   targetImages,
		"windows": targetWindowsImages,
	} {
		err = imagesText(arch, images)
		if err != nil {
			return err
		}

		err = mirrorScript(arch, images)
		if err != nil {
			return err
		}

		err = saveScript(arch, images)
		if err != nil {
			return err
		}

		err = loadScript(arch, images)
		if err != nil {
			return err
		}
	}

	return nil
}

func loadScript(arch string, targetImages []string) error {
	loadScriptName := getScriptFilename(arch, "load")
	log.Printf("Creating %s\n", loadScriptName)
	load, err := os.Create(loadScriptName)
	if err != nil {
		return err
	}
	defer load.Close()
	load.Chmod(0755)

	fmt.Fprintf(load, getScript(arch, "load"))
	return nil
}

func saveImages(targetImages []string) []string {
	var saveImages []string
	for _, targetImage := range targetImages {
		_, ok := image.Mirrors[targetImage]
		if !ok {
			continue
		}

		saveImages = append(saveImages, targetImage)
	}
	return saveImages
}

func saveScript(arch string, targetImages []string) error {
	filename := getScriptFilename(arch, "save")
	log.Printf("Creating %s\n", filename)
	save, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer save.Close()
	save.Chmod(0755)

	fmt.Fprintf(save, getScript(arch, "save"))

	return nil
}

func imagesText(arch string, targetImages []string) error {
	filename := filenameMap[arch]
	log.Printf("Creating %s\n", filename)
	save, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer save.Close()
	save.Chmod(0755)

	for _, image := range saveImages(targetImages) {
		log.Println("Image:", image)
		fmt.Fprintln(save, image)
	}

	return nil
}

func mirrorScript(arch string, targetImages []string) error {
	filename := getScriptFilename(arch, "mirror")
	log.Printf("Creating %s\n", filename)
	mirror, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer mirror.Close()
	mirror.Chmod(0755)

	scriptStarter := getScript(arch, "mirror")
	fmt.Fprintf(mirror, scriptStarter)

	var saveImages []string
	for _, targetImage := range targetImages {
		srcImage, ok := image.Mirrors[targetImage]
		if !ok {
			continue
		}

		saveImages = append(saveImages, targetImage)
		fmt.Fprintf(mirror, "docker pull %s\n", srcImage)
		if targetImage != srcImage {
			fmt.Fprintf(mirror, "docker tag %s %s\n", srcImage, targetImage)
			fmt.Fprintf(mirror, "docker push %s\n", targetImage)
		}
	}

	return nil
}

func collectionImages(objs ...interface{}) ([]string, error) {
	images := map[string]bool{}

	for _, obj := range objs {
		data := map[string]interface{}{}
		if err := convert.ToObj(obj, &data); err != nil {
			return nil, err
		}
		findStrings(data, images)
	}

	var result []string
	for k := range images {
		result = append(result, k)
	}
	sort.Slice(result, func(i, j int) bool {
		return result[i] < result[j]
	})

	return result, nil
}

func findStrings(obj map[string]interface{}, found map[string]bool) {
	for _, v := range obj {
		switch t := v.(type) {
		case string:
			found[t] = true
		case map[string]interface{}:
			findStrings(t, found)
		}
	}
}

func getImagesFromCharts(path string) ([]string, error) {
	var images []string
	imageMap := map[string]struct{}{}
	chartVersion, err := getChartAndVersion(path)
	if err != nil {
		return nil, err
	}
	if err := filepath.Walk(path, func(p string, info os.FileInfo, err error) error {
		return walkFunc(imageMap, chartVersion, path, p, info, err)
	}); err != nil {
		return images, err
	}
	for value := range imageMap {
		images = append(images, value)
	}
	return images, nil
}

func getChartAndVersion(path string) (map[string]string, error) {
	rtn := map[string]string{}
	helm := libhelm.Helm{
		LocalPath: path,
		IconPath:  path,
		Hash:      "",
	}
	index, err := helm.LoadIndex()
	if err != nil {
		return nil, err
	}
	for k, versions := range index.IndexFile.Entries {
		// because versions is sorted in reverse order, the first one will be the latest version
		if len(versions) > 0 {
			rtn[k] = versions[0].Dir
		}
	}

	return rtn, nil
}

func walkFunc(images map[string]struct{}, versions map[string]string, basePath, path string, info os.FileInfo, err error) error {
	relPath, err := filepath.Rel(basePath, path)
	if err != nil {
		return err
	}
	var found bool
	for _, v := range versions {
		if strings.HasPrefix(relPath, v) {
			found = true
			break
		}
	}
	if !found || info.Name() != "values.yaml" {
		return nil
	}
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	dataInterface := map[interface{}]interface{}{}
	if err := yaml.Unmarshal(data, &dataInterface); err != nil {
		return err
	}

	walkthroughMap(dataInterface, func(inputMap map[interface{}]interface{}) {
		generateImages(inputMap, images)
	})
	return nil
}

func generateImages(inputMap map[interface{}]interface{}, output map[string]struct{}) {
	r, repoOk := inputMap["repository"]
	t, tagOk := inputMap["tag"]
	if !repoOk || !tagOk {
		return
	}
	repo, repoOk := r.(string)
	tag, tagOk := t.(string)
	if !repoOk || !tagOk {
		return
	}

	output[fmt.Sprintf("%s:%s", repo, tag)] = struct{}{}

	return
}

func walkthroughMap(inputMap map[interface{}]interface{}, walkFunc func(map[interface{}]interface{})) {
	walkFunc(inputMap)
	for _, value := range inputMap {
		if v, ok := value.(map[interface{}]interface{}); ok {
			walkthroughMap(v, walkFunc)
		}
	}
}

func getWindowsAgentImage() string {
	tag, ok := os.LookupEnv("TAG")
	if !ok {
		return ""
	}
	repo, ok := os.LookupEnv("REPO")
	if !ok {
		return ""
	}
	return fmt.Sprintf("%s/rancher-agent:%s", repo, tag)
}

func getScript(arch, fileType string) string {
	return scriptMap[fmt.Sprintf("%s-%s", arch, fileType)]
}

func getScriptFilename(arch, fileType string) string {
	return scriptNameMap[fmt.Sprintf("%s-%s", arch, fileType)]
}

const (
	linuxLoadScript = `#!/bin/bash
list="rancher-images.txt"
images="rancher-images.tar.gz"

POSITIONAL=()
while [[ $# -gt 0 ]]; do
    key="$1"
    case $key in
        -r|--registry)
        reg="$2"
        shift # past argument
        shift # past value
        ;;
        -l|--image-list)
        list="$2"
        shift # past argument
        shift # past value
        ;;
        -i|--images)
        images="$2"
        shift # past argument
        shift # past value
        ;;
        -h|--help)
        help="true"
        shift
        ;;
    esac
done

usage () {
    echo "USAGE: $0 [--image-list rancher-images.txt] [--images rancher-images.tar.gz] --registry my.registry.com:5000"
    echo "  [-l|--images-list path] text file with list of images. 1 per line."
    echo "  [-l|--images path] tar.gz generated by docker save."
    echo "  [-r|--registry registry:port] target private registry:port."
    echo "  [-h|--help] Usage message"
}

if [[ -z $reg ]]; then
    usage
    exit 1
fi
if [[ $help ]]; then
    usage
    exit 0
fi

set -e -x

docker load --input ${images}

for i in $(cat ${list}); do
    case $i in
    */*)
        docker tag ${i} ${reg}/${i}
        docker push ${reg}/${i}
        ;;
    *)
        docker tag ${i} ${reg}/rancher/${i}
        docker push ${reg}/rancher/${i}
        ;;
    esac
done
`
	linuxSaveScript = `#!/bin/bash
list="rancher-images.txt"
images="rancher-images.tar.gz"

POSITIONAL=()
while [[ $# -gt 0 ]]; do
	key="$1"
	case $key in
		-i|--images)
		images="$2"
		shift # past argument
		shift # past value
		;;
		-l|--image-list)
		list="$2"
		shift # past argument
		shift # past value
		;;
		-h|--help)
		help="true"
		shift
		;;
	esac
done

usage () {
	echo "USAGE: $0 [--image-list rancher-images.txt] [--images rancher-images.tar.gz]"
	echo "  [-l|--images-list path] text file with list of images. 1 per line."
	echo "  [-l|--images path] tar.gz generated by docker save."
	echo "  [-h|--help] Usage message"
}

if [[ $help ]]; then
	usage
	exit 0
fi

set -e -x

for i in $(cat ${list}); do
	docker pull ${i}
done

docker save $(cat ${list} | tr '\n' ' ') | gzip -c > ${images}
`
	linuxMirrorScript = "#!/bin/sh\nset -e -x\n\n"
	windowsLoadScript = `<#
    .PARAMETER  registry
    target private registry:port.
    .PARAMETER image-list
    text file with list of images. 1 per line. default is %s
    .PARAMETER images
    tar.gz generated by docker save. default is rancher-windows-images.tar.gz
#>
param(
    [PARAMETER(Mandatory=$true,Position=0,HelpMessage="target private registry:port.")][string]$registry,
    [string]${image-list}="%s",
    [string]$images="rancher-windows-images.tar.gz"
)

$content=Get-Content -path ${image-list}

docker load --input ${images}

foreach ($item in $content) {
    docker tag $item $Registry/$item
    docker push $Registry/$item
}
`
	windowsSaveScirpt = `#Requires -Version 5.0
<#
	.PARAMETER image-list
	text file with list of images. 1 per line.
	.PARAMETER images
	tar.gz generated by docker save.
#>
param(
	[string]${image-list}="rancher-windows-images.txt",
	[string]$images="rancher-windows-images.tar.gz"
)

$content=Get-Content -path ${image-list}

foreach ($item in $content) {
	docker pull $item
}

docker save $($content) -o $images
`
	windowsMirrorScript = ``
)
<|endoftext|>


<commit_before>package dnsimple

import (
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/dnsimple/dnsimple-go/dnsimple"
	"github.com/juju/ratelimit"
	"github.com/rancher/external-dns/providers"
	"github.com/rancher/external-dns/utils"
)

type DNSimpleProvider struct {
	client    *dnsimple.Client
	accountID string
	root      string
	limiter   *ratelimit.Bucket
}

func init() {
	providers.RegisterProvider("dnsimple", &DNSimpleProvider{})
}

func (d *DNSimpleProvider) Init(rootDomainName string) error {
	var oauthToken string

	if len(os.Getenv("DNSIMPLE_EMAIL")) > 0 {
		return fmt.Errorf("DNSimple API v2 requires an account identifier and the new OAuth token. Please upgrade your configuration.")
	}

	if oauthToken = os.Getenv("DNSIMPLE_TOKEN"); len(oauthToken) == 0 {
		return fmt.Errorf("DNSIMPLE_TOKEN is not set")
	}

	d.root = utils.UnFqdn(rootDomainName)
	d.client = dnsimple.NewClient(dnsimple.NewOauthTokenCredentials(oauthToken))
	d.limiter = ratelimit.NewBucketWithRate(1.5, 5)

	whoamiResponse, err := d.client.Identity.Whoami()
	if err != nil {
		return fmt.Errorf("DNSimple Authentication failed: %v", err)
	}
	if whoamiResponse.Data.Account == nil {
		return fmt.Errorf("DNSimple User tokens are not supported, use an Account token")
	}
	d.accountID = strconv.Itoa(whoamiResponse.Data.Account.ID)

	_, err = d.client.Zones.GetZone(d.accountID, d.root)
	if err != nil {
		return fmt.Errorf("Failed to get zone for '%s': %v", d.root)
	}

	logrus.Infof("Configured %s with zone '%s'", d.GetName(), d.root)
	return nil
}

func (*DNSimpleProvider) GetName() string {
	return "DNSimple"
}

func (d *DNSimpleProvider) HealthCheck() error {
	d.limiter.Wait(1)
	_, err := d.client.Identity.Whoami()
	return err
}

func (d *DNSimpleProvider) parseName(record utils.DnsRecord) string {
	name := strings.TrimSuffix(record.Fqdn, fmt.Sprintf(".%s.", d.root))
	return name
}

func (d *DNSimpleProvider) AddRecord(record utils.DnsRecord) error {
	name := d.parseName(record)
	for _, rec := range record.Records {
		recordInput := dnsimple.ZoneRecord{
			Name:    name,
			TTL:     record.TTL,
			Type:    record.Type,
			Content: rec,
		}
		d.limiter.Wait(1)
		_, err := d.client.Zones.CreateRecord(d.accountID, d.root, recordInput)
		if err != nil {
			return fmt.Errorf("DNSimple API call has failed: %v", err)
		}
	}

	return nil
}

func (d *DNSimpleProvider) findRecords(record utils.DnsRecord) ([]dnsimple.ZoneRecord, error) {
	var zoneRecords []dnsimple.ZoneRecord

	d.limiter.Wait(1)
	recordsResponse, err := d.client.Zones.ListRecords(d.accountID, d.root, nil)
	if err != nil {
		return zoneRecords, fmt.Errorf("DNSimple API call has failed: %v", err)
	}

	name := d.parseName(record)
	for _, zoneRecord := range recordsResponse.Data {
		if zoneRecord.Name == name && zoneRecord.Type == record.Type {
			zoneRecords = append(zoneRecords, zoneRecord)
		}
	}

	return zoneRecords, nil
}

func (d *DNSimpleProvider) UpdateRecord(record utils.DnsRecord) error {
	err := d.RemoveRecord(record)
	if err != nil {
		return err
	}

	return d.AddRecord(record)
}

func (d *DNSimpleProvider) RemoveRecord(record utils.DnsRecord) error {
	zoneRecords, err := d.findRecords(record)
	if err != nil {
		return err
	}

	for _, zoneRecord := range zoneRecords {
		d.limiter.Wait(1)
		_, err := d.client.Zones.DeleteRecord(d.accountID, d.root, zoneRecord.ID)
		if err != nil {
			return fmt.Errorf("DNSimple API call has failed: %v", err)
		}
	}

	return nil
}

func (d *DNSimpleProvider) GetRecords() ([]utils.DnsRecord, error) {
	var records []utils.DnsRecord

	d.limiter.Wait(1)
	recordsResponse, err := d.client.Zones.ListRecords(d.accountID, d.root, nil)
	if err != nil {
		return records, fmt.Errorf("DNSimple API call has failed: %v", err)
	}

	recordMap := map[string]map[string][]string{}
	recordTTLs := map[string]map[string]int{}

	for _, zoneRecord := range recordsResponse.Data {
		var fqdn string
		if zoneRecord.Name == "" {
			fqdn = d.root + "."
		} else {
			fqdn = fmt.Sprintf("%s.%s.", zoneRecord.Name, d.root)
		}

		recordTTLs[fqdn] = map[string]int{}
		recordTTLs[fqdn][zoneRecord.Type] = zoneRecord.TTL
		recordSet, exists := recordMap[fqdn]
		if exists {
			recordSlice, sliceExists := recordSet[zoneRecord.Type]
			if sliceExists {
				recordSlice = append(recordSlice, zoneRecord.Content)
				recordSet[zoneRecord.Type] = recordSlice
			} else {
				recordSet[zoneRecord.Type] = []string{zoneRecord.Content}
			}
		} else {
			recordMap[fqdn] = map[string][]string{}
			recordMap[fqdn][zoneRecord.Type] = []string{zoneRecord.Content}
		}
	}

	for fqdn, recordSet := range recordMap {
		for recordType, recordSlice := range recordSet {
			ttl := recordTTLs[fqdn][recordType]
			record := utils.DnsRecord{Fqdn: fqdn, Records: recordSlice, Type: recordType, TTL: ttl}
			records = append(records, record)
		}
	}
	return records, nil
}
<commit_msg>Revert "Switch to API v2"<commit_after>package dnsimple

import (
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/dnsimple/dnsimple-go/dnsimple"
	"github.com/juju/ratelimit"
	"github.com/rancher/external-dns/providers"
	"github.com/rancher/external-dns/utils"
	api "github.com/weppos/go-dnsimple/dnsimple"
)

type DNSimpleProvider struct {
	client    *api.Client
	client2   *dnsimple.Client
	accountID string
	root      string
	limiter   *ratelimit.Bucket
}

func init() {
	providers.RegisterProvider("dnsimple", &DNSimpleProvider{})
}

func (d *DNSimpleProvider) Init(rootDomainName string) error {
	var email, apiToken, oauthToken string

	if email = os.Getenv("DNSIMPLE_EMAIL"); len(email) == 0 {
		return fmt.Errorf("DNSIMPLE_EMAIL is not set")
	}

	if apiToken = os.Getenv("DNSIMPLE_TOKEN"); len(apiToken) == 0 {
		return fmt.Errorf("DNSIMPLE_TOKEN is not set")
	}

	if oauthToken = os.Getenv("DNSIMPLE_TOKEN"); len(oauthToken) == 0 {
		return fmt.Errorf("DNSIMPLE_TOKEN is not set")
	}

	d.root = utils.UnFqdn(rootDomainName)
	d.client = api.NewClient(apiToken, email)
	d.client2 = dnsimple.NewClient(dnsimple.NewOauthTokenCredentials(oauthToken))
	d.limiter = ratelimit.NewBucketWithRate(1.5, 5)

	whoamiResponse, err := d.client2.Identity.Whoami()
	if err != nil {
		return fmt.Errorf("DNSimple Authentication failed: %v", err)
	}
	if whoamiResponse.Data.Account == nil {
		return fmt.Errorf("DNSimple User tokens are not supported, use an Account token")
	}
	d.accountID = strconv.Itoa(whoamiResponse.Data.Account.ID)

	_, err = d.client2.Zones.GetZone(d.accountID, d.root)
	if err != nil {
		return fmt.Errorf("Failed to get zone for '%s': %v", d.root)
	}

	logrus.Infof("Configured %s with zone '%s'", d.GetName(), d.root)
	return nil
}

func (*DNSimpleProvider) GetName() string {
	return "DNSimple"
}

func (d *DNSimpleProvider) HealthCheck() error {
	d.limiter.Wait(1)
	_, err := d.client2.Identity.Whoami()
	return err
}

func (d *DNSimpleProvider) parseName(record utils.DnsRecord) string {
	name := strings.TrimSuffix(record.Fqdn, fmt.Sprintf(".%s.", d.root))
	return name
}

func (d *DNSimpleProvider) AddRecord(record utils.DnsRecord) error {
	name := d.parseName(record)
	for _, rec := range record.Records {
		recordInput := dnsimple.ZoneRecord{
			Name:    name,
			TTL:     record.TTL,
			Type:    record.Type,
			Content: rec,
		}
		d.limiter.Wait(1)
		_, err := d.client2.Zones.CreateRecord(d.accountID, d.root, recordInput)
		if err != nil {
			return fmt.Errorf("DNSimple API call has failed: %v", err)
		}
	}

	return nil
}

func (d *DNSimpleProvider) findRecords(record utils.DnsRecord) ([]dnsimple.ZoneRecord, error) {
	var zoneRecords []dnsimple.ZoneRecord

	d.limiter.Wait(1)
	recordsResponse, err := d.client2.Zones.ListRecords(d.accountID, d.root, nil)
	if err != nil {
		return zoneRecords, fmt.Errorf("DNSimple API call has failed: %v", err)
	}

	name := d.parseName(record)
	for _, zoneRecord := range recordsResponse.Data {
		if zoneRecord.Name == name && zoneRecord.Type == record.Type {
			zoneRecords = append(zoneRecords, zoneRecord)
		}
	}

	return zoneRecords, nil
}

func (d *DNSimpleProvider) UpdateRecord(record utils.DnsRecord) error {
	err := d.RemoveRecord(record)
	if err != nil {
		return err
	}

	return d.AddRecord(record)
}

func (d *DNSimpleProvider) RemoveRecord(record utils.DnsRecord) error {
	zoneRecords, err := d.findRecords(record)
	if err != nil {
		return err
	}

	for _, zoneRecord := range zoneRecords {
		d.limiter.Wait(1)
		_, err := d.client2.Zones.DeleteRecord(d.accountID, d.root, zoneRecord.ID)
		if err != nil {
			return fmt.Errorf("DNSimple API call has failed: %v", err)
		}
	}

	return nil
}

func (d *DNSimpleProvider) GetRecords() ([]utils.DnsRecord, error) {
	var records []utils.DnsRecord

	d.limiter.Wait(1)
	recordsResponse, err := d.client2.Zones.ListRecords(d.accountID, d.root, nil)
	if err != nil {
		return records, fmt.Errorf("DNSimple API call has failed: %v", err)
	}

	recordMap := map[string]map[string][]string{}
	recordTTLs := map[string]map[string]int{}

	for _, zoneRecord := range recordsResponse.Data {
		var fqdn string
		if zoneRecord.Name == "" {
			fqdn = d.root + "."
		} else {
			fqdn = fmt.Sprintf("%s.%s.", zoneRecord.Name, d.root)
		}

		recordTTLs[fqdn] = map[string]int{}
		recordTTLs[fqdn][zoneRecord.Type] = zoneRecord.TTL
		recordSet, exists := recordMap[fqdn]
		if exists {
			recordSlice, sliceExists := recordSet[zoneRecord.Type]
			if sliceExists {
				recordSlice = append(recordSlice, zoneRecord.Content)
				recordSet[zoneRecord.Type] = recordSlice
			} else {
				recordSet[zoneRecord.Type] = []string{zoneRecord.Content}
			}
		} else {
			recordMap[fqdn] = map[string][]string{}
			recordMap[fqdn][zoneRecord.Type] = []string{zoneRecord.Content}
		}
	}

	for fqdn, recordSet := range recordMap {
		for recordType, recordSlice := range recordSet {
			ttl := recordTTLs[fqdn][recordType]
			record := utils.DnsRecord{Fqdn: fqdn, Records: recordSlice, Type: recordType, TTL: ttl}
			records = append(records, record)
		}
	}
	return records, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"net/http"
	"sort"
	"strings"

	"github.com/fsouza/go-dockerclient"
	"github.com/fsouza/go-dockerclient/testing"
	"github.com/globalsign/mgo/bson"
	"github.com/tsuru/config"
	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/tsuru/app"
	"gopkg.in/check.v1"
)

func (s *S) TestMigrateImages(c *check.C) {
	var requests []*http.Request
	server, err := testing.NewServer("127.0.0.1:0", nil, func(r *http.Request) {
		requests = append(requests, r)
	})
	c.Assert(err, check.IsNil)
	defer server.Stop()
	var p dockerProvisioner
	err = p.Initialize()
	c.Assert(err, check.IsNil)
	p.cluster, _ = cluster.New(nil, &cluster.MapStorage{}, "",
		cluster.Node{Address: server.URL()})
	app1 := app.App{Name: "app1"}
	app2 := app.App{Name: "app2"}
	app3 := app.App{Name: "app-app2"}
	c1, err := s.newContainer(&newContainerOpts{Image: "tsuru/app1", AppName: "app1"}, &p)
	c.Assert(err, check.IsNil)
	defer s.removeTestContainer(c1)
	c2, err := s.newContainer(&newContainerOpts{Image: "tsuru/app1", AppName: "app1"}, &p)
	c.Assert(err, check.IsNil)
	defer s.removeTestContainer(c2)
	c3, err := s.newContainer(&newContainerOpts{Image: "tsuru/app1", AppName: "app1"}, &p)
	c.Assert(err, check.IsNil)
	defer s.removeTestContainer(c3)
	c4, err := s.newContainer(&newContainerOpts{Image: "tsuru/app-app2", AppName: "app2"}, &p)
	c.Assert(err, check.IsNil)
	defer s.removeTestContainer(c4)
	c5, err := s.newContainer(&newContainerOpts{Image: "tsuru/app-app2", AppName: "app-app2"}, &p)
	c.Assert(err, check.IsNil)
	defer s.removeTestContainer(c5)
	err = s.conn.Apps().Insert(app1, app2, app3)
	c.Assert(err, check.IsNil)
	mainDockerProvisioner = &p
	err = MigrateImages()
	c.Assert(err, check.IsNil)
	contApp1, err := p.ListContainers(bson.M{"appname": app1.Name, "image": "tsuru/app-app1"})
	c.Assert(err, check.IsNil)
	c.Assert(contApp1, check.HasLen, 3)
	contApp2, err := p.ListContainers(bson.M{"appname": app2.Name, "image": "tsuru/app-app-app2"})
	c.Assert(err, check.IsNil)
	c.Assert(contApp2, check.HasLen, 0)
	contApp3, err := p.ListContainers(bson.M{"appname": app3.Name, "image": "tsuru/app-app-app2"})
	c.Assert(err, check.IsNil)
	c.Assert(contApp3, check.HasLen, 1)
}

func (s *S) TestMigrateImagesWithoutImageInStorage(c *check.C) {
	var requests []*http.Request
	server, err := testing.NewServer("127.0.0.1:0", nil, func(r *http.Request) {
		requests = append(requests, r)
	})
	c.Assert(err, check.IsNil)
	defer server.Stop()
	var p dockerProvisioner
	err = p.Initialize()
	c.Assert(err, check.IsNil)
	p.cluster, _ = cluster.New(nil, &cluster.MapStorage{}, "",
		cluster.Node{Address: server.URL()})
	app1 := app.App{Name: "app1"}
	err = s.conn.Apps().Insert(app1)
	c.Assert(err, check.IsNil)
	mainDockerProvisioner = &p
	err = MigrateImages()
	c.Assert(err, check.IsNil)
	client, err := docker.NewClient(server.URL())
	c.Assert(err, check.IsNil)
	images, err := client.ListImages(docker.ListImagesOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(images, check.HasLen, 0)
}

func (s *S) TestMigrateImagesWithRegistry(c *check.C) {
	var requests []*http.Request
	server, err := testing.NewServer("127.0.0.1:0", nil, func(r *http.Request) {
		requests = append(requests, r)
	})
	c.Assert(err, check.IsNil)
	defer server.Stop()
	config.Set("docker:registry", "localhost:3030")
	defer config.Unset("docker:registry")
	var p dockerProvisioner
	err = p.Initialize()
	c.Assert(err, check.IsNil)
	p.cluster, _ = cluster.New(nil, &cluster.MapStorage{}, "",
		cluster.Node{Address: server.URL()})
	app1 := app.App{Name: "app1"}
	app2 := app.App{Name: "app2"}
	err = s.conn.Apps().Insert(app1, app2)
	c.Assert(err, check.IsNil)
	err = newFakeImage(&p, "localhost:3030/tsuru/app1", nil)
	c.Assert(err, check.IsNil)
	err = newFakeImage(&p, "localhost:3030/tsuru/app2", nil)
	c.Assert(err, check.IsNil)
	mainDockerProvisioner = &p
	err = MigrateImages()
	c.Assert(err, check.IsNil)
	client, err := docker.NewClient(server.URL())
	c.Assert(err, check.IsNil)
	images, err := client.ListImages(docker.ListImagesOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(images, check.HasLen, 2)
	sort.Slice(images, func(i, j int) bool {
		return strings.Join(images[i].RepoTags, "") < strings.Join(images[j].RepoTags, "")
	})
	tags1 := images[0].RepoTags
	sort.Strings(tags1)
	tags2 := images[1].RepoTags
	sort.Strings(tags2)
	c.Assert(tags1, check.DeepEquals, []string{"localhost:3030/tsuru/app-app1", "localhost:3030/tsuru/app1"})
	c.Assert(tags2, check.DeepEquals, []string{"localhost:3030/tsuru/app-app2", "localhost:3030/tsuru/app2"})
}
<commit_msg>provision/docker: fix flaky test<commit_after>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"net/http"
	"sort"
	"strings"

	"github.com/fsouza/go-dockerclient"
	"github.com/fsouza/go-dockerclient/testing"
	"github.com/globalsign/mgo/bson"
	"github.com/tsuru/config"
	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/tsuru/app"
	"gopkg.in/check.v1"
)

func (s *S) TestMigrateImages(c *check.C) {
	var requests []*http.Request
	server, err := testing.NewServer("127.0.0.1:0", nil, func(r *http.Request) {
		requests = append(requests, r)
	})
	c.Assert(err, check.IsNil)
	defer server.Stop()
	var p dockerProvisioner
	err = p.Initialize()
	c.Assert(err, check.IsNil)
	p.cluster, _ = cluster.New(nil, &cluster.MapStorage{}, "",
		cluster.Node{Address: server.URL()})
	app1 := app.App{Name: "app1"}
	app2 := app.App{Name: "app2"}
	app3 := app.App{Name: "app-app2"}
	c1, err := s.newContainer(&newContainerOpts{Image: "tsuru/app1", AppName: "app1"}, &p)
	c.Assert(err, check.IsNil)
	defer s.removeTestContainer(c1)
	c2, err := s.newContainer(&newContainerOpts{Image: "tsuru/app1", AppName: "app1"}, &p)
	c.Assert(err, check.IsNil)
	defer s.removeTestContainer(c2)
	c3, err := s.newContainer(&newContainerOpts{Image: "tsuru/app1", AppName: "app1"}, &p)
	c.Assert(err, check.IsNil)
	defer s.removeTestContainer(c3)
	c4, err := s.newContainer(&newContainerOpts{Image: "tsuru/app-app2", AppName: "app2"}, &p)
	c.Assert(err, check.IsNil)
	defer s.removeTestContainer(c4)
	c5, err := s.newContainer(&newContainerOpts{Image: "tsuru/app-app2", AppName: "app-app2"}, &p)
	c.Assert(err, check.IsNil)
	defer s.removeTestContainer(c5)
	err = s.conn.Apps().Insert(app1, app2, app3)
	c.Assert(err, check.IsNil)
	mainDockerProvisioner = &p
	err = MigrateImages()
	c.Assert(err, check.IsNil)
	contApp1, err := p.ListContainers(bson.M{"appname": app1.Name, "image": "tsuru/app-app1"})
	c.Assert(err, check.IsNil)
	c.Assert(contApp1, check.HasLen, 3)
	contApp2, err := p.ListContainers(bson.M{"appname": app2.Name, "image": "tsuru/app-app-app2"})
	c.Assert(err, check.IsNil)
	c.Assert(contApp2, check.HasLen, 0)
	contApp3, err := p.ListContainers(bson.M{"appname": app3.Name, "image": "tsuru/app-app-app2"})
	c.Assert(err, check.IsNil)
	c.Assert(contApp3, check.HasLen, 1)
}

func (s *S) TestMigrateImagesWithoutImageInStorage(c *check.C) {
	var requests []*http.Request
	server, err := testing.NewServer("127.0.0.1:0", nil, func(r *http.Request) {
		requests = append(requests, r)
	})
	c.Assert(err, check.IsNil)
	defer server.Stop()
	var p dockerProvisioner
	err = p.Initialize()
	c.Assert(err, check.IsNil)
	p.cluster, _ = cluster.New(nil, &cluster.MapStorage{}, "",
		cluster.Node{Address: server.URL()})
	app1 := app.App{Name: "app1"}
	err = s.conn.Apps().Insert(app1)
	c.Assert(err, check.IsNil)
	mainDockerProvisioner = &p
	err = MigrateImages()
	c.Assert(err, check.IsNil)
	client, err := docker.NewClient(server.URL())
	c.Assert(err, check.IsNil)
	images, err := client.ListImages(docker.ListImagesOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(images, check.HasLen, 0)
}

func (s *S) TestMigrateImagesWithRegistry(c *check.C) {
	var requests []*http.Request
	server, err := testing.NewServer("127.0.0.1:0", nil, func(r *http.Request) {
		requests = append(requests, r)
	})
	c.Assert(err, check.IsNil)
	defer server.Stop()
	config.Set("docker:registry", "localhost:3030")
	defer config.Unset("docker:registry")
	var p dockerProvisioner
	err = p.Initialize()
	c.Assert(err, check.IsNil)
	p.cluster, _ = cluster.New(nil, &cluster.MapStorage{}, "",
		cluster.Node{Address: server.URL()})
	app1 := app.App{Name: "app1"}
	app2 := app.App{Name: "app2"}
	err = s.conn.Apps().Insert(app1, app2)
	c.Assert(err, check.IsNil)
	err = newFakeImage(&p, "localhost:3030/tsuru/app1", nil)
	c.Assert(err, check.IsNil)
	err = newFakeImage(&p, "localhost:3030/tsuru/app2", nil)
	c.Assert(err, check.IsNil)
	mainDockerProvisioner = &p
	err = MigrateImages()
	c.Assert(err, check.IsNil)
	client, err := docker.NewClient(server.URL())
	c.Assert(err, check.IsNil)
	images, err := client.ListImages(docker.ListImagesOptions{All: true})
	c.Assert(err, check.IsNil)
	c.Assert(images, check.HasLen, 2)
	sort.Strings(images[0].RepoTags)
	sort.Strings(images[1].RepoTags)
	sort.Slice(images, func(i, j int) bool {
		return strings.Join(images[i].RepoTags, "") < strings.Join(images[j].RepoTags, "")
	})
	c.Assert(images[0].RepoTags, check.DeepEquals, []string{"localhost:3030/tsuru/app-app1", "localhost:3030/tsuru/app1"})
	c.Assert(images[1].RepoTags, check.DeepEquals, []string{"localhost:3030/tsuru/app-app2", "localhost:3030/tsuru/app2"})
}
<|endoftext|>


<commit_before>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/tsuru/config"
	"github.com/tsuru/monsterqueue"
	"github.com/tsuru/tsuru/app"
	"github.com/tsuru/tsuru/iaas"
	"github.com/tsuru/tsuru/log"
)

const runBsTaskName = "run-bs"

type runBs struct{}

func (runBs) Name() string {
	return runBsTaskName
}

func (t runBs) Run(job monsterqueue.Job) {
	params := job.Parameters()
	dockerEndpoint := params["endpoint"].(string)
	machineID := params["machine"].(string)
	err := t.waitDocker(dockerEndpoint)
	if err != nil {
		job.Error(err)
		return
	}
	err = t.createBsContainer(dockerEndpoint)
	if err != nil {
		job.Error(err)
		t.destroyMachine(machineID)
		return
	}
	rawMetadata := params["metadata"].(monsterqueue.JobParams)
	metadata := make(map[string]string, len(rawMetadata))
	for key, value := range rawMetadata {
		metadata[key] = value.(string)
	}
	_, err = mainDockerProvisioner.getCluster().Register(dockerEndpoint, metadata)
	if err != nil {
		job.Error(err)
		t.destroyMachine(machineID)
		return
	}
}

func (runBs) waitDocker(endpoint string) error {
	client, err := docker.NewClient(endpoint)
	if err != nil {
		return err
	}
	timeout, _ := config.GetInt("docker:api-timeout")
	if timeout == 0 {
		timeout = 600
	}
	timeoutChan := time.After(time.Duration(timeout) * time.Second)
	pong := make(chan error, 1)
	go func() {
		for {
			err := client.Ping()
			if err == nil {
				pong <- nil
				return
			}
			if e, ok := err.(*docker.Error); ok && e.Status > 499 {
				pong <- err
				return
			}
		}
	}()
	select {
	case err := <-pong:
		return err
	case <-timeoutChan:
		return fmt.Errorf("Docker API at %q didn't respond after %d seconds", endpoint, timeout)
	}
}

func (runBs) createBsContainer(dockerEndpoint string) error {
	client, err := docker.NewClient(dockerEndpoint)
	if err != nil {
		return err
	}
	bsImage, _ := config.GetString("docker:bs:image")
	if bsImage == "" {
		bsImage = "tsuru/bs"
	}
	tsuruEndpoint, _ := config.GetString("host")
	if !strings.HasPrefix(tsuruEndpoint, "http://") && !strings.HasPrefix(tsuruEndpoint, "https://") {
		tsuruEndpoint = "http://" + tsuruEndpoint
	}
	interval, _ := config.GetInt("docker:bs:reporter-interval")
	tsuruEndpoint = strings.TrimRight(tsuruEndpoint, "/") + "/"
	token, err := app.AuthScheme.AppLogin(app.InternalAppName)
	if err != nil {
		return err
	}
	sentinelEnvVar := "TSURU_APPNAME"
	hostConfig := docker.HostConfig{RestartPolicy: docker.AlwaysRestart()}
	endpoint := dockerEndpoint
	socket, _ := config.GetString("docker:bs:socket")
	if socket != "" {
		hostConfig.Binds = []string{fmt.Sprintf("%s:/var/run/docker.sock:rw", socket)}
		endpoint = "unix:///var/run/docker.sock"
	}
	sysLogInternalPort, _ := config.GetInt("docker:bs:syslog-internal-port")
	if sysLogInternalPort == 0 {
		sysLogInternalPort = 514
	}
	sysLogExternalPort := getBsSysLogPort()
	hostConfig.PortBindings = map[docker.Port][]docker.PortBinding{
		docker.Port(strconv.Itoa(sysLogInternalPort) + "/udp"): {
			docker.PortBinding{
				HostIP:   "0.0.0.0",
				HostPort: strconv.Itoa(sysLogExternalPort),
			},
		},
	}
	env := []string{
		"DOCKER_ENDPOINT=" + endpoint,
		"TSURU_ENDPOINT=" + tsuruEndpoint,
		"TSURU_TOKEN=" + token.GetValue(),
		"TSURU_SENTINEL_ENV_VAR=" + sentinelEnvVar,
		"STATUS_INTERVAL=" + strconv.Itoa(interval),
		"SYSLOG_LISTEN_ADDRESS=" + "udp://0.0.0.0:" + strconv.Itoa(sysLogInternalPort),
	}
	opts := docker.CreateContainerOptions{
		Name:       "big-sibling",
		HostConfig: &hostConfig,
		Config: &docker.Config{
			Image: bsImage,
			Env:   env,
			ExposedPorts: map[docker.Port]struct{}{
				docker.Port(strconv.Itoa(sysLogExternalPort) + "/udp"): {},
			},
		},
	}
	container, err := client.CreateContainer(opts)
	if err == docker.ErrNoSuchImage {
		pullOpts := docker.PullImageOptions{Repository: bsImage}
		err = client.PullImage(pullOpts, getRegistryAuthConfig())
		if err != nil {
			return err
		}
		container, err = client.CreateContainer(opts)
	}
	if err != nil {
		return err
	}
	return client.StartContainer(container.ID, &hostConfig)
}

func getBsSysLogPort() int {
	bsPort, _ := config.GetInt("docker:bs:syslog-external-port")
	if bsPort == 0 {
		bsPort = 1514
	}
	return bsPort
}

func (runBs) destroyMachine(id string) {
	if id != "" {
		machine, err := iaas.FindMachineById(id)
		if err != nil {
			log.Errorf("failed to remove machine %q: %s", id, err)
			return
		}
		err = machine.Destroy()
		if err != nil {
			log.Errorf("failed to remove machine %q: %s", id, err)
			return
		}
	}
}
<commit_msg>provision/docker: make sure node adding task call success when done<commit_after>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/tsuru/config"
	"github.com/tsuru/monsterqueue"
	"github.com/tsuru/tsuru/app"
	"github.com/tsuru/tsuru/iaas"
	"github.com/tsuru/tsuru/log"
)

const runBsTaskName = "run-bs"

type runBs struct{}

func (runBs) Name() string {
	return runBsTaskName
}

func (t runBs) Run(job monsterqueue.Job) {
	params := job.Parameters()
	dockerEndpoint := params["endpoint"].(string)
	machineID := params["machine"].(string)
	err := t.waitDocker(dockerEndpoint)
	if err != nil {
		job.Error(err)
		return
	}
	err = t.createBsContainer(dockerEndpoint)
	if err != nil {
		job.Error(err)
		t.destroyMachine(machineID)
		return
	}
	rawMetadata := params["metadata"].(monsterqueue.JobParams)
	metadata := make(map[string]string, len(rawMetadata))
	for key, value := range rawMetadata {
		metadata[key] = value.(string)
	}
	_, err = mainDockerProvisioner.getCluster().Register(dockerEndpoint, metadata)
	if err != nil {
		job.Error(err)
		t.destroyMachine(machineID)
		return
	}
	job.Success(nil)
}

func (runBs) waitDocker(endpoint string) error {
	client, err := docker.NewClient(endpoint)
	if err != nil {
		return err
	}
	timeout, _ := config.GetInt("docker:api-timeout")
	if timeout == 0 {
		timeout = 600
	}
	timeoutChan := time.After(time.Duration(timeout) * time.Second)
	pong := make(chan error, 1)
	go func() {
		for {
			err := client.Ping()
			if err == nil {
				pong <- nil
				return
			}
			if e, ok := err.(*docker.Error); ok && e.Status > 499 {
				pong <- err
				return
			}
		}
	}()
	select {
	case err := <-pong:
		return err
	case <-timeoutChan:
		return fmt.Errorf("Docker API at %q didn't respond after %d seconds", endpoint, timeout)
	}
}

func (runBs) createBsContainer(dockerEndpoint string) error {
	client, err := docker.NewClient(dockerEndpoint)
	if err != nil {
		return err
	}
	bsImage, _ := config.GetString("docker:bs:image")
	if bsImage == "" {
		bsImage = "tsuru/bs"
	}
	tsuruEndpoint, _ := config.GetString("host")
	if !strings.HasPrefix(tsuruEndpoint, "http://") && !strings.HasPrefix(tsuruEndpoint, "https://") {
		tsuruEndpoint = "http://" + tsuruEndpoint
	}
	interval, _ := config.GetInt("docker:bs:reporter-interval")
	tsuruEndpoint = strings.TrimRight(tsuruEndpoint, "/") + "/"
	token, err := app.AuthScheme.AppLogin(app.InternalAppName)
	if err != nil {
		return err
	}
	sentinelEnvVar := "TSURU_APPNAME"
	hostConfig := docker.HostConfig{RestartPolicy: docker.AlwaysRestart()}
	endpoint := dockerEndpoint
	socket, _ := config.GetString("docker:bs:socket")
	if socket != "" {
		hostConfig.Binds = []string{fmt.Sprintf("%s:/var/run/docker.sock:rw", socket)}
		endpoint = "unix:///var/run/docker.sock"
	}
	sysLogInternalPort, _ := config.GetInt("docker:bs:syslog-internal-port")
	if sysLogInternalPort == 0 {
		sysLogInternalPort = 514
	}
	sysLogExternalPort := getBsSysLogPort()
	hostConfig.PortBindings = map[docker.Port][]docker.PortBinding{
		docker.Port(strconv.Itoa(sysLogInternalPort) + "/udp"): {
			docker.PortBinding{
				HostIP:   "0.0.0.0",
				HostPort: strconv.Itoa(sysLogExternalPort),
			},
		},
	}
	env := []string{
		"DOCKER_ENDPOINT=" + endpoint,
		"TSURU_ENDPOINT=" + tsuruEndpoint,
		"TSURU_TOKEN=" + token.GetValue(),
		"TSURU_SENTINEL_ENV_VAR=" + sentinelEnvVar,
		"STATUS_INTERVAL=" + strconv.Itoa(interval),
		"SYSLOG_LISTEN_ADDRESS=" + "udp://0.0.0.0:" + strconv.Itoa(sysLogInternalPort),
	}
	opts := docker.CreateContainerOptions{
		Name:       "big-sibling",
		HostConfig: &hostConfig,
		Config: &docker.Config{
			Image: bsImage,
			Env:   env,
			ExposedPorts: map[docker.Port]struct{}{
				docker.Port(strconv.Itoa(sysLogExternalPort) + "/udp"): {},
			},
		},
	}
	container, err := client.CreateContainer(opts)
	if err == docker.ErrNoSuchImage {
		pullOpts := docker.PullImageOptions{Repository: bsImage}
		err = client.PullImage(pullOpts, getRegistryAuthConfig())
		if err != nil {
			return err
		}
		container, err = client.CreateContainer(opts)
	}
	if err != nil {
		return err
	}
	return client.StartContainer(container.ID, &hostConfig)
}

func getBsSysLogPort() int {
	bsPort, _ := config.GetInt("docker:bs:syslog-external-port")
	if bsPort == 0 {
		bsPort = 1514
	}
	return bsPort
}

func (runBs) destroyMachine(id string) {
	if id != "" {
		machine, err := iaas.FindMachineById(id)
		if err != nil {
			log.Errorf("failed to remove machine %q: %s", id, err)
			return
		}
		err = machine.Destroy()
		if err != nil {
			log.Errorf("failed to remove machine %q: %s", id, err)
			return
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"github.com/globocom/config"
	"github.com/globocom/tsuru/db"
	ftesting "github.com/globocom/tsuru/fs/testing"
	"launchpad.net/gocheck"
	"os"
	"testing"
)

func Test(t *testing.T) { gocheck.TestingT(t) }

type S struct {
	collName      string
	imageCollName string
	conn          *db.Storage
	gitHost       string
	repoNamespace string
	deployCmd     string
	runBin        string
	runArgs       string
	port          string
	hostAddr      string
}

var _ = gocheck.Suite(&S{})

func (s *S) SetUpSuite(c *gocheck.C) {
	s.collName = "docker_unit"
	s.imageCollName = "docker_image"
	s.gitHost = "my.gandalf.com"
	s.repoNamespace = "tsuru"
	s.hostAddr = "10.0.0.4"
	config.Set("git:host", s.gitHost)
	config.Set("database:url", "127.0.0.1:27017")
	config.Set("database:name", "docker_provision_tests_s")
	config.Set("docker:repository-namespace", s.repoNamespace)
	config.Set("docker:binary", "docker")
	config.Set("docker:router", "fake")
	config.Set("docker:collection", s.collName)
	config.Set("docker:host-address", s.hostAddr)
	config.Set("docker:deploy-cmd", "/var/lib/tsuru/deploy")
	config.Set("docker:run-cmd:bin", "/usr/local/bin/circusd")
	config.Set("docker:run-cmd:args", "/etc/circus/circus.ini")
	config.Set("docker:run-cmd:port", "8888")
	config.Set("docker:ssh:add-key-cmd", "/var/lib/tsuru/add-key")
	s.deployCmd = "/var/lib/tsuru/deploy"
	s.runBin = "/usr/local/bin/circusd"
	s.runArgs = "/etc/circus/circus.ini"
	s.port = "8888"
	var err error
	s.conn, err = db.Conn()
	c.Assert(err, gocheck.IsNil)
	fsystem = &ftesting.RecordingFs{}
	f, err := fsystem.Create(os.ExpandEnv("${HOME}/.ssh/id_rsa.pub"))
	c.Assert(err, gocheck.IsNil)
	f.Write([]byte("key-content"))
	f.Close()
}

func (s *S) TearDownSuite(c *gocheck.C) {
	s.conn.Collection(s.collName).Database.DropDatabase()
	fsystem = nil
}
<commit_msg>provision/docker: set docker:ssh:user in SetUpSuite<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"github.com/globocom/config"
	"github.com/globocom/tsuru/db"
	ftesting "github.com/globocom/tsuru/fs/testing"
	"launchpad.net/gocheck"
	"os"
	"testing"
)

func Test(t *testing.T) { gocheck.TestingT(t) }

type S struct {
	collName      string
	imageCollName string
	conn          *db.Storage
	gitHost       string
	repoNamespace string
	deployCmd     string
	runBin        string
	runArgs       string
	port          string
	hostAddr      string
	sshUser       string
}

var _ = gocheck.Suite(&S{})

func (s *S) SetUpSuite(c *gocheck.C) {
	s.collName = "docker_unit"
	s.imageCollName = "docker_image"
	s.gitHost = "my.gandalf.com"
	s.repoNamespace = "tsuru"
	s.hostAddr = "10.0.0.4"
	s.sshUser = "root"
	config.Set("git:host", s.gitHost)
	config.Set("database:url", "127.0.0.1:27017")
	config.Set("database:name", "docker_provision_tests_s")
	config.Set("docker:repository-namespace", s.repoNamespace)
	config.Set("docker:binary", "docker")
	config.Set("docker:router", "fake")
	config.Set("docker:collection", s.collName)
	config.Set("docker:host-address", s.hostAddr)
	config.Set("docker:deploy-cmd", "/var/lib/tsuru/deploy")
	config.Set("docker:run-cmd:bin", "/usr/local/bin/circusd")
	config.Set("docker:run-cmd:args", "/etc/circus/circus.ini")
	config.Set("docker:run-cmd:port", "8888")
	config.Set("docker:ssh:add-key-cmd", "/var/lib/tsuru/add-key")
	config.Set("docker:ssh:user", s.sshUser)
	s.deployCmd = "/var/lib/tsuru/deploy"
	s.runBin = "/usr/local/bin/circusd"
	s.runArgs = "/etc/circus/circus.ini"
	s.port = "8888"
	var err error
	s.conn, err = db.Conn()
	c.Assert(err, gocheck.IsNil)
	fsystem = &ftesting.RecordingFs{}
	f, err := fsystem.Create(os.ExpandEnv("${HOME}/.ssh/id_rsa.pub"))
	c.Assert(err, gocheck.IsNil)
	f.Write([]byte("key-content"))
	f.Close()
}

func (s *S) TearDownSuite(c *gocheck.C) {
	s.conn.Collection(s.collName).Database.DropDatabase()
	fsystem = nil
}
<|endoftext|>


<commit_before>package application

import (
	"fmt"
	"net"

	"github.com/rosenhouse/tubes/lib/awsclient"
	"github.com/rosenhouse/tubes/lib/director"
	"github.com/rosenhouse/tubes/lib/manifests"
)

type directorManifestGenerator interface {
	Generate(config director.DirectorConfig) (manifests.Manifest, error)
}

type boshIOClient interface {
	LatestRelease(releasePath string) (director.Artifact, error)
	LatestStemcell(stemcellName string) (director.Artifact, error)
}

type credentialsGenerator interface {
	Fill(interface{}) error
}

type ManifestBuilder struct {
	DirectorManifestGenerator directorManifestGenerator
	BoshIOClient              boshIOClient
	CredentialsGenerator      credentialsGenerator
}

func (b *ManifestBuilder) getLatestSoftware() (director.Software, error) {
	config := director.Software{}
	var err error
	config.Stemcell, err = b.BoshIOClient.LatestStemcell("bosh-aws-xen-hvm-ubuntu-trusty-go_agent")
	if err != nil {
		return config, err
	}
	config.BoshAWSCPIRelease, err = b.BoshIOClient.LatestRelease("github.com/cloudfoundry-incubator/bosh-aws-cpi-release")
	if err != nil {
		return config, err
	}
	config.BoshDirectorRelease, err = b.BoshIOClient.LatestRelease("github.com/cloudfoundry/bosh")
	if err != nil {
		return config, err
	}
	return config, nil
}

func (b *ManifestBuilder) getInternalIP(subnetCIDR string) (string, error) {
	ip, _, err := net.ParseCIDR(subnetCIDR)
	if err != nil {
		return "", err
	}
	ip = ip.To4()
	ip = director.IncrementIP(ip, 6)
	return ip.To4().String(), nil
}

func (b *ManifestBuilder) getAWSNetwork(resources awsclient.BaseStackResources) director.AWSNetwork {
	return director.AWSNetwork{
		AvailabilityZone: resources.AvailabilityZone,
		BOSHSubnetID:     resources.BOSHSubnetID,
		BOSHSubnetCIDR:   resources.BOSHSubnetCIDR,
		ElasticIP:        resources.BOSHElasticIP,
		SecurityGroup:    resources.BOSHSecurityGroup,
	}
}

func (b *ManifestBuilder) Build(stackName string, resources awsclient.BaseStackResources, accessKey, secretKey string) ([]byte, error) {
	if accessKey == "" {
		return nil, fmt.Errorf("missing access key")
	}
	if secretKey == "" {
		return nil, fmt.Errorf("missing secret key")
	}

	config := director.DirectorConfig{}

	var err error
	config.Software, err = b.getLatestSoftware()
	if err != nil {
		return nil, err
	}

	err = b.CredentialsGenerator.Fill(&config.Credentials)
	if err != nil {
		return nil, err
	}

	config.InternalIP, err = b.getInternalIP(resources.BOSHSubnetCIDR)
	if err != nil {
		return nil, err
	}

	config.AWSNetwork = b.getAWSNetwork(resources)
	config.AWSSSHKey.Name = stackName
	config.AWSSSHKey.Path = "./ssh-key"
	config.AWSCredentials = director.AWSCredentials{
		Region:          resources.AWSRegion,
		AccessKeyID:     accessKey,
		SecretAccessKey: secretKey,
	}

	manifest, err := b.DirectorManifestGenerator.Generate(config)
	if err != nil {
		return nil, err // not tested
	}

	return []byte(manifest.String()), nil
}
<commit_msg>Remove incorrect old comment<commit_after>package application

import (
	"fmt"
	"net"

	"github.com/rosenhouse/tubes/lib/awsclient"
	"github.com/rosenhouse/tubes/lib/director"
	"github.com/rosenhouse/tubes/lib/manifests"
)

type directorManifestGenerator interface {
	Generate(config director.DirectorConfig) (manifests.Manifest, error)
}

type boshIOClient interface {
	LatestRelease(releasePath string) (director.Artifact, error)
	LatestStemcell(stemcellName string) (director.Artifact, error)
}

type credentialsGenerator interface {
	Fill(interface{}) error
}

type ManifestBuilder struct {
	DirectorManifestGenerator directorManifestGenerator
	BoshIOClient              boshIOClient
	CredentialsGenerator      credentialsGenerator
}

func (b *ManifestBuilder) getLatestSoftware() (director.Software, error) {
	config := director.Software{}
	var err error
	config.Stemcell, err = b.BoshIOClient.LatestStemcell("bosh-aws-xen-hvm-ubuntu-trusty-go_agent")
	if err != nil {
		return config, err
	}
	config.BoshAWSCPIRelease, err = b.BoshIOClient.LatestRelease("github.com/cloudfoundry-incubator/bosh-aws-cpi-release")
	if err != nil {
		return config, err
	}
	config.BoshDirectorRelease, err = b.BoshIOClient.LatestRelease("github.com/cloudfoundry/bosh")
	if err != nil {
		return config, err
	}
	return config, nil
}

func (b *ManifestBuilder) getInternalIP(subnetCIDR string) (string, error) {
	ip, _, err := net.ParseCIDR(subnetCIDR)
	if err != nil {
		return "", err
	}
	ip = ip.To4()
	ip = director.IncrementIP(ip, 6)
	return ip.To4().String(), nil
}

func (b *ManifestBuilder) getAWSNetwork(resources awsclient.BaseStackResources) director.AWSNetwork {
	return director.AWSNetwork{
		AvailabilityZone: resources.AvailabilityZone,
		BOSHSubnetID:     resources.BOSHSubnetID,
		BOSHSubnetCIDR:   resources.BOSHSubnetCIDR,
		ElasticIP:        resources.BOSHElasticIP,
		SecurityGroup:    resources.BOSHSecurityGroup,
	}
}

func (b *ManifestBuilder) Build(stackName string, resources awsclient.BaseStackResources, accessKey, secretKey string) ([]byte, error) {
	if accessKey == "" {
		return nil, fmt.Errorf("missing access key")
	}
	if secretKey == "" {
		return nil, fmt.Errorf("missing secret key")
	}

	config := director.DirectorConfig{}

	var err error
	config.Software, err = b.getLatestSoftware()
	if err != nil {
		return nil, err
	}

	err = b.CredentialsGenerator.Fill(&config.Credentials)
	if err != nil {
		return nil, err
	}

	config.InternalIP, err = b.getInternalIP(resources.BOSHSubnetCIDR)
	if err != nil {
		return nil, err
	}

	config.AWSNetwork = b.getAWSNetwork(resources)
	config.AWSSSHKey.Name = stackName
	config.AWSSSHKey.Path = "./ssh-key"
	config.AWSCredentials = director.AWSCredentials{
		Region:          resources.AWSRegion,
		AccessKeyID:     accessKey,
		SecretAccessKey: secretKey,
	}

	manifest, err := b.DirectorManifestGenerator.Generate(config)
	if err != nil {
		return nil, err
	}

	return []byte(manifest.String()), nil
}
<|endoftext|>


<commit_before>package cache

import (
	"hash/crc32"
	"math"
	"sync"
	"time"
)

const bucketsSize = 256

type Cache struct {
	// This are the Read/Write locks shared by key
	// So the prevents to lock all keys when only one is being written
	entriesLock [bucketsSize]*sync.RWMutex
	entries     [bucketsSize]map[string]*CacheEntry
	storage     Storage
}

type CacheEntry struct {
	// This lock is used only to prevent concurrent access to the same upstream
	// It is not meant to protect access to values, the `mutex` of storage is used for that.
	valuesLock *sync.RWMutex
	values     []*Value
}

type Value struct {
	expiration time.Time

	// This lock prevents deleting the content on disk
	// While there is a request reading from it
	refLock *sync.RWMutex

	// ref is a previous request/response stored
	// This can easily be a Clear() interface but
	// Using HttpCacheEntry prevents having to cast
	// everywhere. This is because the lock of the map
	// Is used to prevent deleting it from the map
	// But the lock is released after the entry is read from the map
	// It does not mean that some go routine is not using it
	ref *HttpCacheEntry
}

func NewCache(storage Storage) *Cache {
	return &Cache{storage: storage}
}

func (s *Cache) Setup() error {
	for i := 0; i < int(bucketsSize); i++ {
		s.entriesLock[i] = new(sync.RWMutex)
		s.entries[i] = make(map[string]*CacheEntry)
	}

	return s.storage.Setup()
}

func (s *Cache) Clear() error {
	// TODO
	return nil
	// return s.storage.Clean()
}

func (s *Cache) getEntry(key string) *CacheEntry {
	i := s.getBucketIndexForKey(key)

	s.entriesLock[i].Lock()
	entry, ok := s.entries[i][key]
	if !ok {
		s.entries[i][key] = &CacheEntry{
			values:     []*Value{},
			valuesLock: new(sync.RWMutex),
		}
		entry = s.entries[i][key]
	}
	s.entriesLock[i].Unlock()

	return entry
}

/**
 * Given a key and a matcher it fetches the searched value.
 * If it exists it calls the handler with that value.
 * If it doesn't it calls the value with nil and the return value
 * of the handler will be pushed.
 */
func (s *Cache) GetOrSet(key string, condition func(*HttpCacheEntry) bool, handler func(*HttpCacheEntry) (*HttpCacheEntry, error)) error {
	entry := s.getEntry(key)

	// While iterating throw the values is important that nobody else writes on it
	// Until the resource is found in the entries' list or is fetched
	entry.valuesLock.Lock()

	for _, value := range entry.values {
		if condition(value.ref) {
			// The searched resource if found, the list can be unlocked
			entry.valuesLock.Unlock()

			// Read lock the content so it is not expired while using it in the handler
			value.refLock.RLock()
			//noinspection GoDeferInLoop
			defer value.refLock.RUnlock()

			// Call the handler
			newValue, err := handler(value.ref)

			// The case when newValue is not nil is when a previous time called
			// was not cacheable but now it is. Should rarely happen
			if err == nil && newValue != nil {
				entry.valuesLock.Lock()
				s.unsafePush(key, entry, newValue)
				entry.valuesLock.Unlock()
			}

			return err
		}
	}

	// If the entry is not on the list wait until it is fetched from upstream
	defer entry.valuesLock.Unlock()

	newValue, err := handler(nil)
	if err != nil || newValue == nil {
		return err
	}

	s.unsafePush(key, entry, newValue)
	return nil
}

func (s *Cache) unsafePush(key string, entry *CacheEntry, newValue *HttpCacheEntry) {
	value := &Value{
		ref:        newValue,
		refLock:    new(sync.RWMutex),
		expiration: newValue.Expiration,
	}

	// This pushes the new entry on top of the slice
	// This is useful to use the most recent values first
	entry.values = append([]*Value{value}, entry.values...)

	// Launch a new go routine that will expire the content
	go s.expire(key, newValue.Expiration)
}

func (s *Cache) getBucketIndexForKey(key string) uint32 {
	return uint32(math.Mod(float64(crc32.ChecksumIEEE([]byte(key))), float64(bucketsSize)))
}

func (s *Cache) NewContent(key string) (StorageContent, error) {
	return s.storage.NewContent(key)
}

func (s *Cache) expire(key string, expiration time.Time) {
	// Sleep until is time to cleanup entry
	time.Sleep(expiration.Sub(time.Now().UTC()))

	// Get bucket and lock
	bucket := s.getBucketIndexForKey(key)
	s.entriesLock[bucket].Lock()
	entry, ok := s.entries[bucket][key]
	s.entriesLock[bucket].Unlock()

	if !ok {
		// Possible bug, how it ended here?
		return
	}

	entry.valuesLock.Lock()
	defer entry.valuesLock.Unlock()

	// This should prevent creating a new array
	// Uses the same slice
	notExpiredValues := entry.values[:0]
	for _, value := range entry.values {
		// Check which entry for the key is expired
		if value.expiration.After(time.Now().UTC()) {
			notExpiredValues = append(notExpiredValues, value)
		} else {
			// Clear the content in other go routine
			// If it is being red it can block others
			go func(value *Value) {
				// Get lock to prevent any other go routine read from it
				value.refLock.Lock()
				// Delete it if it is on disk
				value.ref.Clear()
			}(value) // Copying the pointer to the go routine is required to avoid reading an invalid value
		}
	}
	entry.values = notExpiredValues
}
<commit_msg>Move lock to prevent possible race condition<commit_after>package cache

import (
	"hash/crc32"
	"math"
	"sync"
	"time"
)

const bucketsSize = 256

type Cache struct {
	// This are the Read/Write locks shared by key
	// So the prevents to lock all keys when only one is being written
	entriesLock [bucketsSize]*sync.RWMutex
	entries     [bucketsSize]map[string]*CacheEntry
	storage     Storage
}

type CacheEntry struct {
	// This lock is used only to prevent concurrent access to the same upstream
	// It is not meant to protect access to values, the `mutex` of storage is used for that.
	valuesLock *sync.RWMutex
	values     []*Value
}

type Value struct {
	expiration time.Time

	// This lock prevents deleting the content on disk
	// While there is a request reading from it
	refLock *sync.RWMutex

	// ref is a previous request/response stored
	// This can easily be a Clear() interface but
	// Using HttpCacheEntry prevents having to cast
	// everywhere. This is because the lock of the map
	// Is used to prevent deleting it from the map
	// But the lock is released after the entry is read from the map
	// It does not mean that some go routine is not using it
	ref *HttpCacheEntry
}

func NewCache(storage Storage) *Cache {
	return &Cache{storage: storage}
}

func (s *Cache) Setup() error {
	for i := 0; i < int(bucketsSize); i++ {
		s.entriesLock[i] = new(sync.RWMutex)
		s.entries[i] = make(map[string]*CacheEntry)
	}

	return s.storage.Setup()
}

func (s *Cache) Clear() error {
	// TODO
	return nil
	// return s.storage.Clean()
}

func (s *Cache) getEntry(key string) *CacheEntry {
	i := s.getBucketIndexForKey(key)

	s.entriesLock[i].Lock()
	entry, ok := s.entries[i][key]
	if !ok {
		s.entries[i][key] = &CacheEntry{
			values:     []*Value{},
			valuesLock: new(sync.RWMutex),
		}
		entry = s.entries[i][key]
	}
	s.entriesLock[i].Unlock()

	return entry
}

/**
 * Given a key and a matcher it fetches the searched value.
 * If it exists it calls the handler with that value.
 * If it doesn't it calls the value with nil and the return value
 * of the handler will be pushed.
 */
func (s *Cache) GetOrSet(key string, condition func(*HttpCacheEntry) bool, handler func(*HttpCacheEntry) (*HttpCacheEntry, error)) error {
	entry := s.getEntry(key)

	// While iterating throw the values is important that nobody else writes on it
	// Until the resource is found in the entries' list or is fetched
	entry.valuesLock.Lock()

	for _, value := range entry.values {
		if condition(value.ref) {
			// Read lock the content so it is not expired while using it in the handler
			value.refLock.RLock()
			//noinspection GoDeferInLoop
			defer value.refLock.RUnlock()

			// The searched resource if found, the list can be unlocked
			entry.valuesLock.Unlock()

			// Call the handler
			newValue, err := handler(value.ref)

			// The case when newValue is not nil is when a previous time called
			// was not cacheable but now it is. Should rarely happen
			if err == nil && newValue != nil {
				entry.valuesLock.Lock()
				s.unsafePush(key, entry, newValue)
				entry.valuesLock.Unlock()
			}

			return err
		}
	}

	// If the entry is not on the list wait until it is fetched from upstream
	defer entry.valuesLock.Unlock()

	newValue, err := handler(nil)
	if err != nil || newValue == nil {
		return err
	}

	s.unsafePush(key, entry, newValue)
	return nil
}

func (s *Cache) unsafePush(key string, entry *CacheEntry, newValue *HttpCacheEntry) {
	value := &Value{
		ref:        newValue,
		refLock:    new(sync.RWMutex),
		expiration: newValue.Expiration,
	}

	// This pushes the new entry on top of the slice
	// This is useful to use the most recent values first
	entry.values = append([]*Value{value}, entry.values...)

	// Launch a new go routine that will expire the content
	go s.expire(key, newValue.Expiration)
}

func (s *Cache) getBucketIndexForKey(key string) uint32 {
	return uint32(math.Mod(float64(crc32.ChecksumIEEE([]byte(key))), float64(bucketsSize)))
}

func (s *Cache) NewContent(key string) (StorageContent, error) {
	return s.storage.NewContent(key)
}

func (s *Cache) expire(key string, expiration time.Time) {
	// Sleep until is time to cleanup entry
	time.Sleep(expiration.Sub(time.Now().UTC()))

	// Get bucket and lock
	bucket := s.getBucketIndexForKey(key)
	s.entriesLock[bucket].Lock()
	entry, ok := s.entries[bucket][key]
	s.entriesLock[bucket].Unlock()

	if !ok {
		// Possible bug, how it ended here?
		return
	}

	entry.valuesLock.Lock()
	defer entry.valuesLock.Unlock()

	// This should prevent creating a new array
	// Uses the same slice
	notExpiredValues := entry.values[:0]
	for _, value := range entry.values {
		// Check which entry for the key is expired
		if value.expiration.After(time.Now().UTC()) {
			notExpiredValues = append(notExpiredValues, value)
		} else {
			// Clear the content in other go routine
			// If it is being red it can block others
			go func(value *Value) {
				// Get lock to prevent any other go routine read from it
				value.refLock.Lock()
				// Delete it if it is on disk
				value.ref.Clear()
			}(value) // Copying the pointer to the go routine is required to avoid reading an invalid value
		}
	}
	entry.values = notExpiredValues
}
<|endoftext|>