<commit_before>package proto

import (
	"butler/null"
	pb "code.google.com/p/goprotobuf/proto"
)

func (ins *Instruction) Iterate(itFunc func(*Instruction)) {
	itFunc(ins)
	if ins.Children == nil {
		return
	}
	for _, child := range ins.Children {
		child.Iterate(itFunc)
	}
}

func (ins *Instruction) IterateAll(itFunc func(*Instruction)) {
	itFunc(ins)
	if ins.Arguments != nil {
		for _, child := range ins.Arguments {
			child.Iterate(itFunc)
		}
	}
	if ins.Children != nil {
		for _, child := range ins.Children {
			child.Iterate(itFunc)
		}
	}
}

func ListInstructions(instrs ...*Instruction) []*Instruction {
	return append(make([]*Instruction, 0), instrs...)
}

func FoldLeft(funcName string, base *Instruction, seq []*Instruction) (acc *Instruction) {
	for acc = base; len(seq) > 0; seq = seq[1:] {
		acc = MakeFunctionCall(funcName, ListInstructions(acc, seq[0]), nil, *base.LineNumber)
	}
	return acc
}

func MakeText(text string, lineNum int32) *Instruction {
	return &Instruction{
		Type:       NewInstruction_InstructionType(Instruction_TEXT),
		Value:      pb.String(text),
		LineNumber: pb.Int32(lineNum),
	}
}

func MakePosition(pos string, lineNum int32) *Instruction {
	return &Instruction{
		Type:       NewInstruction_InstructionType(Instruction_POSITION),
		Value:      pb.String(pos),
		LineNumber: pb.Int32(lineNum),
	}
}

func MakeComment(comment string, lineNum int32) *Instruction {
	return &Instruction{
		Type:       NewInstruction_InstructionType(Instruction_COMMENT),
		Value:      pb.String(comment),
		LineNumber: pb.Int32(lineNum),
	}
}

func MakeImport(path string, lineNum int32) *Instruction {
	return &Instruction{
		Type:       NewInstruction_InstructionType(Instruction_IMPORT),
		Value:      pb.String(path),
		LineNumber: pb.Int32(lineNum),
	}
}

func MakeLocalVar(name string, val *Instruction, block []*Instruction, lineNum int32) *Instruction {
	node := &Instruction{
		Type:       NewInstruction_InstructionType(Instruction_LOCAL_VAR),
		Value:      pb.String(name),
		Children:   block,
		LineNumber: pb.Int32(lineNum),
	}
	if val == nil {
		node.Arguments = nil
	} else {
		node.Arguments = ListInstructions(val)
	}
	return node
}

func MakeFunctionCall(name string, args []*Instruction, block []*Instruction, lineNum int32) *Instruction {
	return &Instruction{
		Type:       NewInstruction_InstructionType(Instruction_FUNCTION_CALL),
		Value:      pb.String(name),
		Arguments:  args,
		Children:   block,
		LineNumber: pb.Int32(lineNum),
	}
}

func MakeBlock(children []*Instruction, lineNum int32) *Instruction {
	return &Instruction{
		Type:       NewInstruction_InstructionType(Instruction_BLOCK),
		Children:   children,
		LineNumber: pb.Int32(lineNum),
	}
}

func (ins *Instruction) GetFunction(pkg *Package) *Function {
	funId := null.GetInt32(ins.FunctionId)
	return pkg.Functions[int(funId)]
}

func (instr *Instruction) Append(more ...*Instruction) {
	if instr.Children == nil {
		instr.Children = more
	} else {
		instr.Children = append(instr.Children, more...)
	}
}

func (instr *Instruction) ConcatList(more []*Instruction) {
	instr.Append(more...)
}

func (instr *Instruction) ConcatBlock(more *Instruction) {
	instr.Append(more.Children...)
}
<commit_msg>use new goprotobuf interface<commit_after>package proto

import (
	"butler/null"
	pb "code.google.com/p/goprotobuf/proto"
)

func (ins *Instruction) Iterate(itFunc func(*Instruction)) {
	itFunc(ins)
	if ins.Children == nil {
		return
	}
	for _, child := range ins.Children {
		child.Iterate(itFunc)
	}
}

func (ins *Instruction) IterateAll(itFunc func(*Instruction)) {
	itFunc(ins)
	if ins.Arguments != nil {
		for _, child := range ins.Arguments {
			child.Iterate(itFunc)
		}
	}
	if ins.Children != nil {
		for _, child := range ins.Children {
			child.Iterate(itFunc)
		}
	}
}

func ListInstructions(instrs ...*Instruction) []*Instruction {
	return append(make([]*Instruction, 0), instrs...)
}

func FoldLeft(funcName string, base *Instruction, seq []*Instruction) (acc *Instruction) {
	for acc = base; len(seq) > 0; seq = seq[1:] {
		acc = MakeFunctionCall(funcName, ListInstructions(acc, seq[0]), nil, *base.LineNumber)
	}
	return acc
}

func MakeText(text string, lineNum int32) *Instruction {
	return &Instruction{
		Type:       Instruction_TEXT.Enum(),
		Value:      pb.String(text),
		LineNumber: pb.Int32(lineNum),
	}
}

func MakePosition(pos string, lineNum int32) *Instruction {
	return &Instruction{
		Type:       Instruction_POSITION.Enum(),
		Value:      pb.String(pos),
		LineNumber: pb.Int32(lineNum),
	}
}

func MakeComment(comment string, lineNum int32) *Instruction {
	return &Instruction{
		Type:       Instruction_COMMENT.Enum(),
		Value:      pb.String(comment),
		LineNumber: pb.Int32(lineNum),
	}
}

func MakeImport(path string, lineNum int32) *Instruction {
	return &Instruction{
		Type:       Instruction_IMPORT.Enum(),
		Value:      pb.String(path),
		LineNumber: pb.Int32(lineNum),
	}
}

func MakeLocalVar(name string, val *Instruction, block []*Instruction, lineNum int32) *Instruction {
	node := &Instruction{
		Type:       Instruction_LOCAL_VAR.Enum(),
		Value:      pb.String(name),
		Children:   block,
		LineNumber: pb.Int32(lineNum),
	}
	if val == nil {
		node.Arguments = nil
	} else {
		node.Arguments = ListInstructions(val)
	}
	return node
}

func MakeFunctionCall(name string, args []*Instruction, block []*Instruction, lineNum int32) *Instruction {
	return &Instruction{
		Type:       Instruction_FUNCTION_CALL.Enum(),
		Value:      pb.String(name),
		Arguments:  args,
		Children:   block,
		LineNumber: pb.Int32(lineNum),
	}
}

func MakeBlock(children []*Instruction, lineNum int32) *Instruction {
	return &Instruction{
		Type:       Instruction_BLOCK.Enum(),
		Children:   children,
		LineNumber: pb.Int32(lineNum),
	}
}

func (ins *Instruction) GetFunction(pkg *Package) *Function {
	funId := null.GetInt32(ins.FunctionId)
	return pkg.Functions[int(funId)]
}

func (instr *Instruction) Append(more ...*Instruction) {
	if instr.Children == nil {
		instr.Children = more
	} else {
		instr.Children = append(instr.Children, more...)
	}
}

func (instr *Instruction) ConcatList(more []*Instruction) {
	instr.Append(more...)
}

func (instr *Instruction) ConcatBlock(more *Instruction) {
	instr.Append(more.Children...)
}
<|endoftext|>


<commit_before>package protobuf

import (
	"fmt"

	alog "github.com/apex/log"
	proto "github.com/golang/protobuf/proto"
)

// Marshal is an implementation of a MarshalFunc specifically for use
// with this handler.  Although it accepts an empty interface type, it
// will only work with an apex.log.Entry type, and will panic if any
// other type is passed in.  Not that this mechanism also enforces the
// rule that any fields set must either be strings or satisfy the
// fmt.Stringer interface.
func Marshal(x interface{}) ([]byte, error) {
	var logEntry *alog.Entry
	var timestamp []byte
	var ok bool
	var fields map[string]string
	var stringer fmt.Stringer
	var str string

	if logEntry, ok = x.(*alog.Entry); !ok {
		return nil, fmt.Errorf("Attempted to marshal a type other than apex.log.Entry")
	}
	timestamp, err := logEntry.Timestamp.MarshalText()
	if err != nil {
		return nil, err
	}

	fields = make(map[string]string, len(logEntry.Fields))
	for key, value := range logEntry.Fields {
		if str, ok = value.(string); ok {
			fields[key] = str
			continue
		}
		if stringer, ok = value.(fmt.Stringer); !ok {
			err := fmt.Errorf("Value for field %s is not a string, nor does it satisfy fmt.Stringer", key)
			return nil, err
		}
		fields[key] = stringer.String()
	}
	entry := &Entry{
		Level:     logEntry.Level.String(),
		Timestamp: timestamp,
		Message:   logEntry.Message,
		Fields:    fields,
	}
	return proto.Marshal(entry)
}

// Unmarshal is an implementation of a UnmarshalFunc specifically for
// unmarshalling an Entry back into apex.log.Entry.
func Unmarshal(data []byte, v interface{}) error {
	var entry *Entry
	var logEntry *alog.Entry
	var ok bool

	if logEntry, ok = v.(*alog.Entry); !ok {
		return fmt.Errorf("Attempted to unmarshal to a type other than apex.log.Entry")
	}

	entry = &Entry{}
	err := proto.Unmarshal(data, entry)
	if err != nil {
		return err
	}
	logEntry.Level = alog.MustParseLevel(entry.Level)
	logEntry.Timestamp.UnmarshalText(entry.Timestamp)
	logEntry.Message = entry.Message
	if logEntry.Fields == nil {
		logEntry.Fields = make(map[string]interface{}, len(entry.Fields))
	}
	for key, value := range entry.Fields {
		logEntry.Fields[key] = value
	}
	return nil
}
<commit_msg>Comment on why we enforce strings for fields<commit_after>package protobuf

import (
	"fmt"

	alog "github.com/apex/log"
	proto "github.com/golang/protobuf/proto"
)

// Marshal is an implementation of a MarshalFunc specifically for use
// with this handler.  Although it accepts an empty interface type, it
// will only work with an apex.log.Entry type, and will panic if any
// other type is passed in.  Not that this mechanism also enforces the
// rule that any fields set must either be strings or satisfy the
// fmt.Stringer interface.
func Marshal(x interface{}) ([]byte, error) {
	var logEntry *alog.Entry
	var timestamp []byte
	var ok bool
	var fields map[string]string
	var stringer fmt.Stringer
	var str string

	if logEntry, ok = x.(*alog.Entry); !ok {
		return nil, fmt.Errorf("Attempted to marshal a type other than apex.log.Entry")
	}
	timestamp, err := logEntry.Timestamp.MarshalText()
	if err != nil {
		return nil, err
	}

	fields = make(map[string]string, len(logEntry.Fields))
	for key, value := range logEntry.Fields {
		// Enforcing the string or fmt.Stringer is a simple
		// way to ensure we can always push field data over
		// the line.  If we ever really want to reconstruct
		// types on the other side of NSQ then we'd probably
		// be just as well off wrapping gob.Encode and
		// gob.Decode as Marshal/Unmarshal.
		if str, ok = value.(string); ok {
			fields[key] = str
			continue
		}
		if stringer, ok = value.(fmt.Stringer); !ok {
			err := fmt.Errorf("Value for field %s is not a string, nor does it satisfy fmt.Stringer", key)
			return nil, err
		}
		fields[key] = stringer.String()
	}
	entry := &Entry{
		Level:     logEntry.Level.String(),
		Timestamp: timestamp,
		Message:   logEntry.Message,
		Fields:    fields,
	}
	return proto.Marshal(entry)
}

// Unmarshal is an implementation of a UnmarshalFunc specifically for
// unmarshalling an Entry back into apex.log.Entry.
func Unmarshal(data []byte, v interface{}) error {
	var entry *Entry
	var logEntry *alog.Entry
	var ok bool

	if logEntry, ok = v.(*alog.Entry); !ok {
		return fmt.Errorf("Attempted to unmarshal to a type other than apex.log.Entry")
	}

	entry = &Entry{}
	err := proto.Unmarshal(data, entry)
	if err != nil {
		return err
	}
	logEntry.Level = alog.MustParseLevel(entry.Level)
	logEntry.Timestamp.UnmarshalText(entry.Timestamp)
	logEntry.Message = entry.Message
	if logEntry.Fields == nil {
		logEntry.Fields = make(map[string]interface{}, len(entry.Fields))
	}
	for key, value := range entry.Fields {
		logEntry.Fields[key] = value
	}
	return nil
}
<|endoftext|>


<commit_before>package plans

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"koding/db/models"
	"koding/kites/kloud/api/amazon"
	"koding/kites/kloud/contexthelper/session"

	"github.com/mitchellh/goamz/ec2"
	"golang.org/x/net/context"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

type KodingChecker struct {
	NetworkUsageEndpoint string
}

func (k *KodingChecker) Fetch(ctx context.Context, planName string) (Checker, error) {
	sess, ok := session.FromContext(ctx)
	if !ok {
		return nil, errors.New("Koding checker couldn't obtain session context")
	}

	if k.NetworkUsageEndpoint == "" {
		return nil, errors.New("Network usage endpoint is not set")
	}

	plan, ok := Plans[strings.ToLower(planName)]
	if !ok {
		return nil, fmt.Errorf("could not find plan. There is no plan called '%s'", planName)
	}

	plan.networkUsageEndpoint = k.NetworkUsageEndpoint
	plan.DB = sess.DB
	plan.AWSClient = sess.AWSClient
	plan.Environment = sess.Kite.Config.Environment
	plan.Log = sess.Log

	return plan, nil
}

type networkUsageResponse struct {
	CanStart     bool    `json:"canStart"`
	Reason       string  `json:"reason"`
	AllowedUsage float64 `json:"allowedUsage"`
	CurrentUsage float64 `json:"currentUsage"`
}

func (p *Plan) NetworkUsage(username string) error {
	if p.networkUsageEndpoint == "" {
		return errors.New("Network usage endpoint is not set")
	}

	networkEndpoint, err := url.Parse(p.networkUsageEndpoint)
	if err != nil {
		p.Log.Debug("Failed to parse network-usage endpoint: %v. err: %v",
			p.networkUsageEndpoint, err)
		return err
	}

	var account *models.Account
	if err := p.DB.Run("jAccounts", func(c *mgo.Collection) error {
		return c.Find(bson.M{"profile.nickname": username}).One(&account)
	}); err != nil {
		p.Log.Warning("Failed to fetch user information while checking network-usage. err: %v",
			err)
		return err
	}

	// in case of error fetching network usage, assume it's ok to start
	var usageResponse = &networkUsageResponse{}
	usageResponse.CanStart = true

	q := networkEndpoint.Query()
	q.Set("account_id", account.Id.Hex())
	networkEndpoint.RawQuery = q.Encode()

	resp, err := http.Get(networkEndpoint.String())
	if err != nil {
		p.Log.Warning("Failed to fetch network-usage because network-usage providing api host seems down. err: %v",
			err)
		return nil
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		p.Log.Debug("Network-usage response code is not 200. It's %v",
			resp.StatusCode)
		return nil
	}

	if err := json.NewDecoder(resp.Body).Decode(&usageResponse); err != nil {
		p.Log.Warning("Failed to decode network-usage response. err: %v",
			err)
		return nil
	}
	if !usageResponse.CanStart {
		p.Log.Debug("Network-usage limit is reached. Allowed usage: %v MiB, Current usage: %v MiB",
			usageResponse.AllowedUsage, usageResponse.CurrentUsage)

		err := fmt.Errorf("%s; allowed: %v, current: %v",
			usageResponse.Reason, usageResponse.AllowedUsage,
			usageResponse.CurrentUsage,
		)

		return err
	}
	return nil
}

func (p *Plan) AllowedInstances(wantInstance InstanceType) error {
	if _, ok := p.allowedInstances[wantInstance]; ok {
		return nil
	}

	return fmt.Errorf("not allowed to create instance type: %s", wantInstance)
}

func (p *Plan) AlwaysOn(username string) error {
	// get all alwaysOn machines that belongs to this user
	alwaysOnMachines := 0
	if err := p.DB.Run("jMachines", func(c *mgo.Collection) error {
		var err error
		alwaysOnMachines, err = c.Find(bson.M{
			"credential":    username,
			"meta.alwaysOn": true,
		}).Count()

		return err
	}); err != nil && err != mgo.ErrNotFound {
		// if it's something else just return an error, needs to be fixed
		return err
	}

	p.Log.Debug("checking alwaysOn limit. current alwaysOn count: %d (plan limit: %d, plan: %s)",
		alwaysOnMachines, p.AlwaysOnLimit, p)

	// the user has still not reached the limit
	if alwaysOnMachines <= p.AlwaysOnLimit {
		p.Log.Debug("allowing user '%s'. current alwaysOn count: %d (plan limit: %d, plan: %s)",
			username, alwaysOnMachines, p.AlwaysOnLimit, p)
		return nil // allow user, it didn't reach the limit
	}

	p.Log.Info("denying user '%s'. current alwaysOn count: %d (plan limit: %d, plan: %s)",
		username, alwaysOnMachines, p.AlwaysOnLimit, p)
	return fmt.Errorf("total alwaysOn limit has been reached. Current count: %d Plan limit: %d",
		alwaysOnMachines, p.AlwaysOnLimit)
}

func (p *Plan) Total(username string) error {
	instances, err := p.userInstances(username)

	// no match, allow to create instance
	if err == amazon.ErrNoInstances {
		p.Log.Debug("allowing user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
			username, len(instances), p.TotalLimit, p)
		return nil
	}

	// if it's something else don't allow it until it's solved
	if err != nil {
		return err
	}

	if len(instances) >= p.TotalLimit {
		p.Log.Debug("denying user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
			username, len(instances), p.TotalLimit, p)
		return fmt.Errorf("total machine limit has been reached. Current count: %d Plan limit: %d",
			len(instances), p.TotalLimit)
	}

	p.Log.Debug("allowing user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
		username, len(instances), p.TotalLimit, p)
	return nil
}

func (p *Plan) SnapshotTotal(machineId, username string) error {
	// lazy return
	if p.SnapshotTotalLimit == 0 {
		p.Log.Debug("denying user to for snapshots, limit is zero already")
		return fmt.Errorf("total snapshot limit has been reached. Plan limit: %d", p.SnapshotTotalLimit)
	}

	currentSnapshotCount := 0
	if err := p.DB.Run("jSnapshots", func(c *mgo.Collection) error {
		var err error
		currentSnapshotCount, err = c.Find(bson.M{
			"machineId": bson.ObjectIdHex(machineId),
		}).Count()

		return err
	}); err != nil && err != mgo.ErrNotFound {
		// if it's something else just return an error, needs to be fixed
		return err
	}

	p.Log.Debug("checking snapshot limit. current count: %d, plan limit: %d (plan: %s)",
		currentSnapshotCount, p.SnapshotTotalLimit, p)

	// the user has still not reached the limit
	if currentSnapshotCount <= p.SnapshotTotalLimit {
		p.Log.Debug("allowing user '%s'. current snapshot count: %d (plan limit: %d, plan: %s)",
			username, currentSnapshotCount, p.SnapshotTotalLimit, p)
		return nil // allow user, it didn't reach the limit
	}

	p.Log.Info("denying user '%s'. current snapshot count: %d (plan limit: %d, plan: %s)",
		username, currentSnapshotCount, p.SnapshotTotalLimit, p)
	return fmt.Errorf("total snapshot limit has been reached. Current count: %d Plan limit: %d",
		currentSnapshotCount, p.SnapshotTotalLimit)
}

func (p *Plan) Storage(wantStorage int, username string) error {
	// no need for errors because instances will be empty in case of an error
	instances, _ := p.userInstances(username)

	// we need to fetch JAccount here to get earnedRewards if exists
	var account *models.Account
	if err := p.DB.Run("jAccounts", func(c *mgo.Collection) error {
		return c.Find(bson.M{"profile.nickname": username}).One(&account)
	}); err != nil {
		p.Log.Warning("Failed to fetch user information while checking storage. err: %v",
			err)
		return err
	}

	rewardAmount := 0

	// querying the earnedReward of given account
	var reward *models.EarnedReward
	if err := p.DB.Run("jEarnedRewards", func(c *mgo.Collection) error {
		return c.Find(bson.M{
			"originId": account.Id,
			"type":     "disk",
			"unit":     "MB",
		}).One(&reward)
	}); err != nil {
		// if there is a different error rather
		// than notFound we should stop here
		if err != mgo.ErrNotFound {
			return err
		}
	} else {
		// we got the amount as MB but aws only supports GB
		// dividing with 1000 not 1024.
		rewardAmount = reward.Amount / 1000
	}

	// and adding it to p.StorageLimit
	// if there is no reward it will be 0 in this state
	p.StorageLimit += rewardAmount

	// i hate for loops too, but unfortunaly the responses are always in form
	// of slices
	currentStorage := 0
	for _, instance := range instances {
		for _, blockDevice := range instance.BlockDevices {
			volumes, err := p.AWSClient.Client.Volumes([]string{blockDevice.VolumeId}, ec2.NewFilter())
			if err != nil {
				return err
			}

			for _, volume := range volumes.Volumes {
				volumeStorage, err := strconv.Atoi(volume.Size)
				if err != nil {
					return err
				}

				currentStorage += volumeStorage
			}
		}
	}

	p.Log.Debug("Checking storage. Current: %dGB. Want: %dGB (plan limit: %dGB, plan: %s)",
		currentStorage, wantStorage, p.StorageLimit, p)

	if currentStorage+wantStorage > p.StorageLimit {
		return fmt.Errorf("total storage limit has been reached. Can have %dGB. User wants %dGB (plan: %s)",
			p.StorageLimit, currentStorage+wantStorage, p)
	}

	p.Log.Debug("Allowing user '%s'. Current: %dGB. Want: %dGB (plan limit: %dGB, plan: %s)",
		username, currentStorage, wantStorage, p.StorageLimit, p)

	// allow to create storage
	return nil
}

func (p *Plan) userInstances(username string) ([]ec2.Instance, error) {
	filter := ec2.NewFilter()
	filter.Add("tag-value", username)

	// Anything except "terminated" and "shutting-down"
	filter.Add("instance-state-name", "pending", "running", "stopping", "stopped")

	instances, err := p.AWSClient.InstancesByFilter(filter)
	if err != nil {
		return nil, err
	}

	filtered := []ec2.Instance{}

	// we don't use filters because they are timing out for us due to high
	// instances count we have. However it seems the filter `tag-value` has an
	// index internally inside AWS so somehow that one is not timing out.
	for _, instance := range instances {
		for _, tag := range instance.Tags {
			if tag.Key == "koding-user" && tag.Value == username {
				for _, tag := range instance.Tags {
					if tag.Key == "koding-env" && tag.Value == p.Environment {

						// now we have the instance that matches both the correct username
						// and environment
						filtered = append(filtered, instance)
					}
				}
			}
		}
	}

	// garbage collect it
	instances = nil
	return filtered, nil
}
<commit_msg>kloud/plans: migrate to aws-sdk-go<commit_after>package plans

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strings"

	"koding/db/models"
	"koding/kites/kloud/api/amazon"
	"koding/kites/kloud/contexthelper/session"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"golang.org/x/net/context"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

type KodingChecker struct {
	NetworkUsageEndpoint string
}

func (k *KodingChecker) Fetch(ctx context.Context, planName string) (Checker, error) {
	sess, ok := session.FromContext(ctx)
	if !ok {
		return nil, errors.New("Koding checker couldn't obtain session context")
	}

	if k.NetworkUsageEndpoint == "" {
		return nil, errors.New("Network usage endpoint is not set")
	}

	plan, ok := Plans[strings.ToLower(planName)]
	if !ok {
		return nil, fmt.Errorf("could not find plan. There is no plan called '%s'", planName)
	}

	plan.networkUsageEndpoint = k.NetworkUsageEndpoint
	plan.DB = sess.DB
	plan.AWSClient = sess.AWSClient
	plan.Environment = sess.Kite.Config.Environment
	plan.Log = sess.Log

	return plan, nil
}

type networkUsageResponse struct {
	CanStart     bool    `json:"canStart"`
	Reason       string  `json:"reason"`
	AllowedUsage float64 `json:"allowedUsage"`
	CurrentUsage float64 `json:"currentUsage"`
}

func (p *Plan) NetworkUsage(username string) error {
	if p.networkUsageEndpoint == "" {
		return errors.New("Network usage endpoint is not set")
	}

	networkEndpoint, err := url.Parse(p.networkUsageEndpoint)
	if err != nil {
		p.Log.Debug("Failed to parse network-usage endpoint: %v. err: %v",
			p.networkUsageEndpoint, err)
		return err
	}

	var account *models.Account
	if err := p.DB.Run("jAccounts", func(c *mgo.Collection) error {
		return c.Find(bson.M{"profile.nickname": username}).One(&account)
	}); err != nil {
		p.Log.Warning("Failed to fetch user information while checking network-usage. err: %v",
			err)
		return err
	}

	// in case of error fetching network usage, assume it's ok to start
	var usageResponse = &networkUsageResponse{}
	usageResponse.CanStart = true

	q := networkEndpoint.Query()
	q.Set("account_id", account.Id.Hex())
	networkEndpoint.RawQuery = q.Encode()

	resp, err := http.Get(networkEndpoint.String())
	if err != nil {
		p.Log.Warning("Failed to fetch network-usage because network-usage providing api host seems down. err: %v",
			err)
		return nil
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		p.Log.Debug("Network-usage response code is not 200. It's %v",
			resp.StatusCode)
		return nil
	}

	if err := json.NewDecoder(resp.Body).Decode(&usageResponse); err != nil {
		p.Log.Warning("Failed to decode network-usage response. err: %v",
			err)
		return nil
	}
	if !usageResponse.CanStart {
		p.Log.Debug("Network-usage limit is reached. Allowed usage: %v MiB, Current usage: %v MiB",
			usageResponse.AllowedUsage, usageResponse.CurrentUsage)

		err := fmt.Errorf("%s; allowed: %v, current: %v",
			usageResponse.Reason, usageResponse.AllowedUsage,
			usageResponse.CurrentUsage,
		)

		return err
	}
	return nil
}

func (p *Plan) AllowedInstances(wantInstance InstanceType) error {
	if _, ok := p.allowedInstances[wantInstance]; ok {
		return nil
	}

	return fmt.Errorf("not allowed to create instance type: %s", wantInstance)
}

func (p *Plan) AlwaysOn(username string) error {
	// get all alwaysOn machines that belongs to this user
	alwaysOnMachines := 0
	if err := p.DB.Run("jMachines", func(c *mgo.Collection) error {
		var err error
		alwaysOnMachines, err = c.Find(bson.M{
			"credential":    username,
			"meta.alwaysOn": true,
		}).Count()

		return err
	}); err != nil && err != mgo.ErrNotFound {
		// if it's something else just return an error, needs to be fixed
		return err
	}

	p.Log.Debug("checking alwaysOn limit. current alwaysOn count: %d (plan limit: %d, plan: %s)",
		alwaysOnMachines, p.AlwaysOnLimit, p)

	// the user has still not reached the limit
	if alwaysOnMachines <= p.AlwaysOnLimit {
		p.Log.Debug("allowing user '%s'. current alwaysOn count: %d (plan limit: %d, plan: %s)",
			username, alwaysOnMachines, p.AlwaysOnLimit, p)
		return nil // allow user, it didn't reach the limit
	}

	p.Log.Info("denying user '%s'. current alwaysOn count: %d (plan limit: %d, plan: %s)",
		username, alwaysOnMachines, p.AlwaysOnLimit, p)
	return fmt.Errorf("total alwaysOn limit has been reached. Current count: %d Plan limit: %d",
		alwaysOnMachines, p.AlwaysOnLimit)
}

func (p *Plan) Total(username string) error {
	instances, err := p.userInstances(username)

	// no match, allow to create instance
	if amazon.IsNotFound(err) {
		p.Log.Debug("allowing user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
			username, len(instances), p.TotalLimit, p)
		return nil
	}

	// if it's something else don't allow it until it's solved
	if err != nil {
		return err
	}

	if len(instances) >= p.TotalLimit {
		p.Log.Debug("denying user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
			username, len(instances), p.TotalLimit, p)
		return fmt.Errorf("total machine limit has been reached. Current count: %d Plan limit: %d",
			len(instances), p.TotalLimit)
	}

	p.Log.Debug("allowing user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
		username, len(instances), p.TotalLimit, p)
	return nil
}

func (p *Plan) SnapshotTotal(machineId, username string) error {
	// lazy return
	if p.SnapshotTotalLimit == 0 {
		p.Log.Debug("denying user to for snapshots, limit is zero already")
		return fmt.Errorf("total snapshot limit has been reached. Plan limit: %d", p.SnapshotTotalLimit)
	}

	currentSnapshotCount := 0
	if err := p.DB.Run("jSnapshots", func(c *mgo.Collection) error {
		var err error
		currentSnapshotCount, err = c.Find(bson.M{
			"machineId": bson.ObjectIdHex(machineId),
		}).Count()

		return err
	}); err != nil && err != mgo.ErrNotFound {
		// if it's something else just return an error, needs to be fixed
		return err
	}

	p.Log.Debug("checking snapshot limit. current count: %d, plan limit: %d (plan: %s)",
		currentSnapshotCount, p.SnapshotTotalLimit, p)

	// the user has still not reached the limit
	if currentSnapshotCount <= p.SnapshotTotalLimit {
		p.Log.Debug("allowing user '%s'. current snapshot count: %d (plan limit: %d, plan: %s)",
			username, currentSnapshotCount, p.SnapshotTotalLimit, p)
		return nil // allow user, it didn't reach the limit
	}

	p.Log.Info("denying user '%s'. current snapshot count: %d (plan limit: %d, plan: %s)",
		username, currentSnapshotCount, p.SnapshotTotalLimit, p)
	return fmt.Errorf("total snapshot limit has been reached. Current count: %d Plan limit: %d",
		currentSnapshotCount, p.SnapshotTotalLimit)
}

func (p *Plan) Storage(wantStorage int, username string) error {
	// no need for errors because instances will be empty in case of an error
	instances, _ := p.userInstances(username)

	// we need to fetch JAccount here to get earnedRewards if exists
	var account *models.Account
	if err := p.DB.Run("jAccounts", func(c *mgo.Collection) error {
		return c.Find(bson.M{"profile.nickname": username}).One(&account)
	}); err != nil {
		p.Log.Warning("Failed to fetch user information while checking storage. err: %v",
			err)
		return err
	}

	rewardAmount := 0

	// querying the earnedReward of given account
	var reward *models.EarnedReward
	if err := p.DB.Run("jEarnedRewards", func(c *mgo.Collection) error {
		return c.Find(bson.M{
			"originId": account.Id,
			"type":     "disk",
			"unit":     "MB",
		}).One(&reward)
	}); err != nil {
		// if there is a different error rather
		// than notFound we should stop here
		if err != mgo.ErrNotFound {
			return err
		}
	} else {
		// we got the amount as MB but aws only supports GB
		// dividing with 1000 not 1024.
		rewardAmount = reward.Amount / 1000
	}

	// and adding it to p.StorageLimit
	// if there is no reward it will be 0 in this state
	p.StorageLimit += rewardAmount

	// i hate for loops too, but unfortunaly the responses are always in form
	// of slices
	currentStorage := 0
	for _, instance := range instances {
		for _, dev := range instance.BlockDeviceMappings {
			// When instance is runnig this shouldn't be nil, however
			// defensive check it either way.
			if dev.Ebs == nil {
				return fmt.Errorf("block device %q has not EBS parameters associated",
					aws.StringValue(dev.DeviceName))
			}
			vol, err := p.AWSClient.Client.VolumeByID(aws.StringValue(dev.Ebs.VolumeId))
			if err != nil {
				return err
			}
			currentStorage += int(aws.Int64Value(vol.Size))
		}
	}

	p.Log.Debug("Checking storage. Current: %dGB. Want: %dGB (plan limit: %dGB, plan: %s)",
		currentStorage, wantStorage, p.StorageLimit, p)

	if currentStorage+wantStorage > p.StorageLimit {
		return fmt.Errorf("total storage limit has been reached. Can have %dGB. User wants %dGB (plan: %s)",
			p.StorageLimit, currentStorage+wantStorage, p)
	}

	p.Log.Debug("Allowing user '%s'. Current: %dGB. Want: %dGB (plan limit: %dGB, plan: %s)",
		username, currentStorage, wantStorage, p.StorageLimit, p)

	// allow to create storage
	return nil
}

func (p *Plan) userInstances(username string) ([]*ec2.Instance, error) {
	filters := map[string][]string{
		// Anything except "terminated" and "shutting-down"
		"instance-state-name": {"pending", "running", "stopping", "stopped"},
		"tag-value":           {username},
	}
	instances, err := p.AWSClient.InstancesByFilters(filters)
	if err != nil {
		return nil, err
	}
	var filtered []*ec2.Instance
	// we don't use filters because they are timing out for us due to high
	// instances count we have. However it seems the filter `tag-value` has an
	// index internally inside AWS so somehow that one is not timing out.
	tags := map[string]string{
		"koding-user": username,
		"koding-env":  p.Environment,
	}
	for _, instance := range instances {
		if amazon.TagsMatch(instance.Tags, tags) {
			// now we have the instance that matches both the correct username
			// and environment
			filtered = append(filtered, instance)
		}
	}
	return filtered, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tabletmanager

import (
	"flag"
	"time"

	"golang.org/x/net/context"

	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/logutil"
	"vitess.io/vitess/go/vt/mysqlctl"
	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
	"vitess.io/vitess/go/vt/topo"
	"vitess.io/vitess/go/vt/topo/topoproto"
	"vitess.io/vitess/go/vt/vterrors"
)

var (
	shardSyncRetryDelay = flag.Duration("shard_sync_retry_delay", 30*time.Second, "delay between retries of updates to keep the tablet and its shard record in sync")
)

// shardSyncLoop is a loop that tries to keep the tablet state and the
// shard record in sync.
//
// It is launched as a background goroutine in the tablet because it may need to
// initiate a tablet state change in response to an incoming watch event for the
// shard record, and it may need to continually retry updating the shard record
// if it's out of sync with the tablet state. At steady state, when the tablet
// and shard record are in sync, this goroutine goes to sleep waiting for
// something to change in either the tablet state or in the shard record.
//
// This goroutine gets woken up for shard record changes by maintaining a
// topo watch on the shard record. It gets woken up for tablet state changes by
// a notification signal from setTablet().
func (agent *ActionAgent) shardSyncLoop(ctx context.Context) {
	// Make a copy of the channel so we don't race when stopShardSync() clears it.
	agent.mutex.Lock()
	notifyChan := agent._shardSyncChan
	agent.mutex.Unlock()

	// retryChan is how we wake up after going to sleep between retries.
	// If no retry is pending, this channel will be nil, which means it's fine
	// to always select on it -- a nil channel is never ready.
	var retryChan <-chan time.Time

	// shardWatch is how we get notified when the shard record is updated.
	// We only watch the shard record while we are master.
	shardWatch := &shardWatcher{}
	defer shardWatch.stop()

	// This loop sleeps until it's notified that something may have changed.
	// Then it wakes up to check if anything needs to be synchronized.
	for {
		select {
		case <-notifyChan:
			// Something may have changed in the tablet state.
			log.Info("Change to tablet state")
		case <-retryChan:
			// It's time to retry a previous failed sync attempt.
			log.Info("Retry sync")
		case event := <-shardWatch.watchChan:
			// Something may have changed in the shard record.
			// We don't use the watch event except to know that we should
			// re-read the shard record, and to know if the watch dies.
			log.Info("Change in shard record")
			if event.Err != nil {
				// The watch failed. Stop it so we start a new one if needed.
				log.Errorf("Shard watch failed: %v", event.Err)
				shardWatch.stop()
			}
		case <-ctx.Done():
			// Our context was cancelled. Terminate the loop.
			return
		}

		// Disconnect any pending retry timer since we're already retrying for
		// another reason.
		retryChan = nil

		// Get the latest internal tablet value, representing what we think we are.
		tablet := agent.Tablet()

		switch tablet.Type {
		case topodatapb.TabletType_MASTER:
			// If we think we're master, check if we need to update the shard record.
			masterAlias, err := syncShardMaster(ctx, agent.TopoServer, tablet, agent.masterTermStartTime())
			if err != nil {
				log.Errorf("Failed to sync shard record: %v", err)
				// Start retry timer and go back to sleep.
				retryChan = time.After(*shardSyncRetryDelay)
				continue
			}
			if !topoproto.TabletAliasEqual(masterAlias, tablet.Alias) {
				// Another master has taken over while we still think we're master.
				if err := agent.abortMasterTerm(ctx, masterAlias); err != nil {
					log.Errorf("Failed to abort master term: %v", err)
					// Start retry timer and go back to sleep.
					retryChan = time.After(*shardSyncRetryDelay)
					continue
				}
				// We're not master anymore, so stop watching the shard record.
				shardWatch.stop()
				continue
			}

			// As long as we're master, watch the shard record so we'll be
			// notified if another master takes over.
			if shardWatch.active() {
				// We already have an active watch. Nothing to do.
				continue
			}
			if err := shardWatch.start(ctx, agent.TopoServer, tablet.Keyspace, tablet.Shard); err != nil {
				log.Errorf("Failed to start shard watch: %v", err)
				// Start retry timer and go back to sleep.
				retryChan = time.After(*shardSyncRetryDelay)
				continue
			}
		default:
			// If we're not master, stop watching the shard record,
			// so only masters contribute to global topo watch load.
			shardWatch.stop()
		}
	}
}

// syncShardMaster is called when we think we're master.
// It checks that the shard record agrees, and updates it if possible.
//
// If the returned error is nil, the returned masterAlias indicates the current
// master tablet according to the shard record.
//
// If the shard record indicates a new master has taken over, this returns
// success (we successfully synchronized), but the returned masterAlias will be
// different from the input tablet.Alias.
func syncShardMaster(ctx context.Context, ts *topo.Server, tablet *topodatapb.Tablet, masterTermStartTime time.Time) (masterAlias *topodatapb.TabletAlias, err error) {
	ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout)
	defer cancel()

	var shardInfo *topo.ShardInfo
	_, err = ts.UpdateShardFields(ctx, tablet.Keyspace, tablet.Shard, func(si *topo.ShardInfo) error {
		lastTerm := logutil.ProtoToTime(si.MasterTermStartTime)

		// Save the ShardInfo so we can check it afterward.
		// We can't use the return value of UpdateShardFields because it might be nil.
		shardInfo = si

		// Only attempt an update if our term is more recent.
		if !masterTermStartTime.After(lastTerm) {
			return topo.NewError(topo.NoUpdateNeeded, si.ShardName())
		}

		aliasStr := topoproto.TabletAliasString(tablet.Alias)
		log.Infof("Updating shard record: master_alias=%v, master_term_start_time=%v", aliasStr, masterTermStartTime)
		si.MasterAlias = tablet.Alias
		si.MasterTermStartTime = logutil.TimeToProto(masterTermStartTime)
		return nil
	})
	if err != nil {
		return nil, err
	}

	return shardInfo.MasterAlias, nil
}

// abortMasterTerm is called when we unexpectedly lost mastership.
//
// Under normal circumstances, we should be gracefully demoted before a new
// master appears. This function is only reached when that graceful demotion
// failed or was skipped, so we only found out we're no longer master after the
// new master started advertising itself.
//
// If active reparents are enabled, we demote our own MySQL to a replica and
// update our tablet type to REPLICA.
//
// If active reparents are disabled, we don't touch our MySQL.
// We just directly update our tablet type to REPLICA.
func (agent *ActionAgent) abortMasterTerm(ctx context.Context, masterAlias *topodatapb.TabletAlias) error {
	masterAliasStr := topoproto.TabletAliasString(masterAlias)
	log.Warningf("Another tablet (%v) has won master election. Stepping down to REPLICA.", masterAliasStr)

	if *mysqlctl.DisableActiveReparents {
		// Don't touch anything at the MySQL level. Just update tablet state.
		log.Infof("Active reparents are disabled; updating tablet state only.")
		changeTypeCtx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout)
		defer cancel()
		if err := agent.ChangeType(changeTypeCtx, topodatapb.TabletType_REPLICA); err != nil {
			return vterrors.Wrap(err, "failed to change type to REPLICA")
		}
		return nil
	}

	// Do a full demotion to convert MySQL into a replica.
	// We do not revert on partial failure here because this code path only
	// triggers after a new master has taken over, so we are past the point of
	// no return. Instead, we should leave partial results and retry the rest
	// later.
	log.Infof("Active reparents are enabled; converting MySQL to replica.")
	demoteMasterCtx, cancelDemoteMaster := context.WithTimeout(ctx, *topo.RemoteOperationTimeout)
	defer cancelDemoteMaster()
	if _, err := agent.demoteMaster(demoteMasterCtx, false /* revertPartialFailure */); err != nil {
		return vterrors.Wrap(err, "failed to demote master")
	}
	setMasterCtx, cancelSetMaster := context.WithTimeout(ctx, *topo.RemoteOperationTimeout)
	defer cancelSetMaster()
	log.Infof("Attempting to reparent self to new master %v.", masterAliasStr)
	if err := agent.SetMaster(setMasterCtx, masterAlias, 0, true); err != nil {
		return vterrors.Wrap(err, "failed to reparent self to new master")
	}
	return nil
}

func (agent *ActionAgent) startShardSync() {
	// Use a buffer size of 1 so we can remember we need to check the state
	// even if the receiver is busy. We can drop any additional send attempts
	// if the buffer is full because all we care about is that the receiver will
	// be told it needs to recheck the state.
	agent.mutex.Lock()
	agent._shardSyncChan = make(chan struct{}, 1)
	ctx, cancel := context.WithCancel(context.Background())
	agent._shardSyncCancel = cancel
	agent.mutex.Unlock()

	// Queue up a pending notification to force the loop to run once at startup.
	agent.notifyShardSync()

	// Start the sync loop in the background.
	go agent.shardSyncLoop(ctx)
}

func (agent *ActionAgent) stopShardSync() {
	agent.mutex.Lock()
	if agent._shardSyncCancel != nil {
		agent._shardSyncCancel()
		agent._shardSyncCancel = nil
		agent._shardSyncChan = nil
	}
	agent.mutex.Unlock()
}

func (agent *ActionAgent) notifyShardSync() {
	// If this is called before the shard sync is started, do nothing.
	agent.mutex.Lock()
	defer agent.mutex.Unlock()

	if agent._shardSyncChan == nil {
		return
	}

	// Try to send. If the channel buffer is full, it means a notification is
	// already pending, so we don't need to do anything.
	select {
	case agent._shardSyncChan <- struct{}{}:
	default:
	}
}

// setMasterTermStartTime remembers the time when our term as master began.
//
// If another tablet claims to be master and offers a more recent time,
// that tablet will be trusted over us.
func (agent *ActionAgent) setMasterTermStartTime(t time.Time) {
	agent.mutex.Lock()
	agent._masterTermStartTime = t
	agent._replicationDelay = 0
	agent.mutex.Unlock()

	// Notify the shard sync loop that the tablet state changed.
	agent.notifyShardSync()
}

func (agent *ActionAgent) masterTermStartTime() time.Time {
	agent.mutex.Lock()
	defer agent.mutex.Unlock()
	return agent._masterTermStartTime
}
<commit_msg>shard_sync should check for nil MasterAlias, which can happen when starting with InitTablet<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tabletmanager

import (
	"flag"
	"time"

	"golang.org/x/net/context"

	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/logutil"
	"vitess.io/vitess/go/vt/mysqlctl"
	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
	"vitess.io/vitess/go/vt/topo"
	"vitess.io/vitess/go/vt/topo/topoproto"
	"vitess.io/vitess/go/vt/vterrors"
)

var (
	shardSyncRetryDelay = flag.Duration("shard_sync_retry_delay", 30*time.Second, "delay between retries of updates to keep the tablet and its shard record in sync")
)

// shardSyncLoop is a loop that tries to keep the tablet state and the
// shard record in sync.
//
// It is launched as a background goroutine in the tablet because it may need to
// initiate a tablet state change in response to an incoming watch event for the
// shard record, and it may need to continually retry updating the shard record
// if it's out of sync with the tablet state. At steady state, when the tablet
// and shard record are in sync, this goroutine goes to sleep waiting for
// something to change in either the tablet state or in the shard record.
//
// This goroutine gets woken up for shard record changes by maintaining a
// topo watch on the shard record. It gets woken up for tablet state changes by
// a notification signal from setTablet().
func (agent *ActionAgent) shardSyncLoop(ctx context.Context) {
	// Make a copy of the channel so we don't race when stopShardSync() clears it.
	agent.mutex.Lock()
	notifyChan := agent._shardSyncChan
	agent.mutex.Unlock()

	// retryChan is how we wake up after going to sleep between retries.
	// If no retry is pending, this channel will be nil, which means it's fine
	// to always select on it -- a nil channel is never ready.
	var retryChan <-chan time.Time

	// shardWatch is how we get notified when the shard record is updated.
	// We only watch the shard record while we are master.
	shardWatch := &shardWatcher{}
	defer shardWatch.stop()

	// This loop sleeps until it's notified that something may have changed.
	// Then it wakes up to check if anything needs to be synchronized.
	for {
		select {
		case <-notifyChan:
			// Something may have changed in the tablet state.
			log.Info("Change to tablet state")
		case <-retryChan:
			// It's time to retry a previous failed sync attempt.
			log.Info("Retry sync")
		case event := <-shardWatch.watchChan:
			// Something may have changed in the shard record.
			// We don't use the watch event except to know that we should
			// re-read the shard record, and to know if the watch dies.
			log.Info("Change in shard record")
			if event.Err != nil {
				// The watch failed. Stop it so we start a new one if needed.
				log.Errorf("Shard watch failed: %v", event.Err)
				shardWatch.stop()
			}
		case <-ctx.Done():
			// Our context was cancelled. Terminate the loop.
			return
		}

		// Disconnect any pending retry timer since we're already retrying for
		// another reason.
		retryChan = nil

		// Get the latest internal tablet value, representing what we think we are.
		tablet := agent.Tablet()

		switch tablet.Type {
		case topodatapb.TabletType_MASTER:
			// If we think we're master, check if we need to update the shard record.
			masterAlias, err := syncShardMaster(ctx, agent.TopoServer, tablet, agent.masterTermStartTime())
			if err != nil {
				log.Errorf("Failed to sync shard record: %v", err)
				// Start retry timer and go back to sleep.
				retryChan = time.After(*shardSyncRetryDelay)
				continue
			}
			// If the shard was initialized via InitTablet, there is no masterAlias set
			// so we are the true master
			if masterAlias != nil && !topoproto.TabletAliasEqual(masterAlias, tablet.Alias) {
				// Another master has taken over while we still think we're master.
				if err := agent.abortMasterTerm(ctx, masterAlias); err != nil {
					log.Errorf("Failed to abort master term: %v", err)
					// Start retry timer and go back to sleep.
					retryChan = time.After(*shardSyncRetryDelay)
					continue
				}
				// We're not master anymore, so stop watching the shard record.
				shardWatch.stop()
				continue
			}

			// As long as we're master, watch the shard record so we'll be
			// notified if another master takes over.
			if shardWatch.active() {
				// We already have an active watch. Nothing to do.
				continue
			}
			if err := shardWatch.start(ctx, agent.TopoServer, tablet.Keyspace, tablet.Shard); err != nil {
				log.Errorf("Failed to start shard watch: %v", err)
				// Start retry timer and go back to sleep.
				retryChan = time.After(*shardSyncRetryDelay)
				continue
			}
		default:
			// If we're not master, stop watching the shard record,
			// so only masters contribute to global topo watch load.
			shardWatch.stop()
		}
	}
}

// syncShardMaster is called when we think we're master.
// It checks that the shard record agrees, and updates it if possible.
//
// If the returned error is nil, the returned masterAlias indicates the current
// master tablet according to the shard record.
//
// If the shard record indicates a new master has taken over, this returns
// success (we successfully synchronized), but the returned masterAlias will be
// different from the input tablet.Alias.
func syncShardMaster(ctx context.Context, ts *topo.Server, tablet *topodatapb.Tablet, masterTermStartTime time.Time) (masterAlias *topodatapb.TabletAlias, err error) {
	ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout)
	defer cancel()

	var shardInfo *topo.ShardInfo
	_, err = ts.UpdateShardFields(ctx, tablet.Keyspace, tablet.Shard, func(si *topo.ShardInfo) error {
		lastTerm := logutil.ProtoToTime(si.MasterTermStartTime)

		// Save the ShardInfo so we can check it afterward.
		// We can't use the return value of UpdateShardFields because it might be nil.
		shardInfo = si

		// Only attempt an update if our term is more recent.
		if !masterTermStartTime.After(lastTerm) {
			return topo.NewError(topo.NoUpdateNeeded, si.ShardName())
		}

		aliasStr := topoproto.TabletAliasString(tablet.Alias)
		log.Infof("Updating shard record: master_alias=%v, master_term_start_time=%v", aliasStr, masterTermStartTime)
		si.MasterAlias = tablet.Alias
		si.MasterTermStartTime = logutil.TimeToProto(masterTermStartTime)
		return nil
	})
	if err != nil {
		return nil, err
	}

	return shardInfo.MasterAlias, nil
}

// abortMasterTerm is called when we unexpectedly lost mastership.
//
// Under normal circumstances, we should be gracefully demoted before a new
// master appears. This function is only reached when that graceful demotion
// failed or was skipped, so we only found out we're no longer master after the
// new master started advertising itself.
//
// If active reparents are enabled, we demote our own MySQL to a replica and
// update our tablet type to REPLICA.
//
// If active reparents are disabled, we don't touch our MySQL.
// We just directly update our tablet type to REPLICA.
func (agent *ActionAgent) abortMasterTerm(ctx context.Context, masterAlias *topodatapb.TabletAlias) error {
	masterAliasStr := topoproto.TabletAliasString(masterAlias)
	log.Warningf("Another tablet (%v) has won master election. Stepping down to REPLICA.", masterAliasStr)

	if *mysqlctl.DisableActiveReparents {
		// Don't touch anything at the MySQL level. Just update tablet state.
		log.Infof("Active reparents are disabled; updating tablet state only.")
		changeTypeCtx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout)
		defer cancel()
		if err := agent.ChangeType(changeTypeCtx, topodatapb.TabletType_REPLICA); err != nil {
			return vterrors.Wrap(err, "failed to change type to REPLICA")
		}
		return nil
	}

	// Do a full demotion to convert MySQL into a replica.
	// We do not revert on partial failure here because this code path only
	// triggers after a new master has taken over, so we are past the point of
	// no return. Instead, we should leave partial results and retry the rest
	// later.
	log.Infof("Active reparents are enabled; converting MySQL to replica.")
	demoteMasterCtx, cancelDemoteMaster := context.WithTimeout(ctx, *topo.RemoteOperationTimeout)
	defer cancelDemoteMaster()
	if _, err := agent.demoteMaster(demoteMasterCtx, false /* revertPartialFailure */); err != nil {
		return vterrors.Wrap(err, "failed to demote master")
	}
	setMasterCtx, cancelSetMaster := context.WithTimeout(ctx, *topo.RemoteOperationTimeout)
	defer cancelSetMaster()
	log.Infof("Attempting to reparent self to new master %v.", masterAliasStr)
	if err := agent.SetMaster(setMasterCtx, masterAlias, 0, true); err != nil {
		return vterrors.Wrap(err, "failed to reparent self to new master")
	}
	return nil
}

func (agent *ActionAgent) startShardSync() {
	// Use a buffer size of 1 so we can remember we need to check the state
	// even if the receiver is busy. We can drop any additional send attempts
	// if the buffer is full because all we care about is that the receiver will
	// be told it needs to recheck the state.
	agent.mutex.Lock()
	agent._shardSyncChan = make(chan struct{}, 1)
	ctx, cancel := context.WithCancel(context.Background())
	agent._shardSyncCancel = cancel
	agent.mutex.Unlock()

	// Queue up a pending notification to force the loop to run once at startup.
	agent.notifyShardSync()

	// Start the sync loop in the background.
	go agent.shardSyncLoop(ctx)
}

func (agent *ActionAgent) stopShardSync() {
	agent.mutex.Lock()
	if agent._shardSyncCancel != nil {
		agent._shardSyncCancel()
		agent._shardSyncCancel = nil
		agent._shardSyncChan = nil
	}
	agent.mutex.Unlock()
}

func (agent *ActionAgent) notifyShardSync() {
	// If this is called before the shard sync is started, do nothing.
	agent.mutex.Lock()
	defer agent.mutex.Unlock()

	if agent._shardSyncChan == nil {
		return
	}

	// Try to send. If the channel buffer is full, it means a notification is
	// already pending, so we don't need to do anything.
	select {
	case agent._shardSyncChan <- struct{}{}:
	default:
	}
}

// setMasterTermStartTime remembers the time when our term as master began.
//
// If another tablet claims to be master and offers a more recent time,
// that tablet will be trusted over us.
func (agent *ActionAgent) setMasterTermStartTime(t time.Time) {
	agent.mutex.Lock()
	agent._masterTermStartTime = t
	agent._replicationDelay = 0
	agent.mutex.Unlock()

	// Notify the shard sync loop that the tablet state changed.
	agent.notifyShardSync()
}

func (agent *ActionAgent) masterTermStartTime() time.Time {
	agent.mutex.Lock()
	defer agent.mutex.Unlock()
	return agent._masterTermStartTime
}
<|endoftext|>


<commit_before>package commands

import (
	"bufio"
	"database/sql"
	"fmt"
	"github.com/fatih/color"
	"github.com/zuzuleinen/dave/reminder"
	"os"
	"strconv"
	"strings"
	"time"
)

func Remind(db *sql.DB) {
	fmt.Print("Task name: ")

	reader := bufio.NewReader(os.Stdin)
	taskName, _ := reader.ReadString('\n')
	taskName = strings.TrimSuffix(taskName, "\n")
	dbItems := []reminder.Reminder{{taskName}}

	year := scanYear()
	month := scanMonth()
	hour := scanHour()

	reminder.Save(db, dbItems)
	color.Green("Great I will remind you to %s.", taskName)
	fmt.Println("Time is", year, month, hour)
}

func scanYear() int {
	var y int
	var prompt string

	fmt.Print("Year[2016]:")
	fmt.Scanln(&prompt)

	if prompt == "" {
		y = time.Now().Year()
	} else {
		y, _ = strconv.Atoi(prompt)
	}

	return y
}

func scanMonth() string {
	var prompt string
	var m string

	fmt.Print("Month[10]:")
	fmt.Scanln(&prompt)

	if prompt == "" {
		m = time.Now().Month().String()
	} else {
		monthIndex, _ := strconv.Atoi(prompt)
		m = time.Month(monthIndex).String()
	}

	return m
}

func scanHour() string {
	var hour string

	fmt.Print("Time(HH:mm):")
	fmt.Scanln(&hour)

	return hour
}
<commit_msg>add default date to terminal on remind<commit_after>package commands

import (
	"bufio"
	"database/sql"
	"fmt"
	"github.com/fatih/color"
	"github.com/zuzuleinen/dave/reminder"
	"os"
	"strconv"
	"strings"
	"time"
)

func Remind(db *sql.DB) {
	fmt.Print("Task name: ")

	reader := bufio.NewReader(os.Stdin)
	taskName, _ := reader.ReadString('\n')
	taskName = strings.TrimSuffix(taskName, "\n")
	dbItems := []reminder.Reminder{{taskName}}

	year := scanYear()
	month := scanMonth()
	day := scanDay()
	hour := scanHour()

	reminder.Save(db, dbItems)
	color.Green("Great I will remind you to %s.", taskName)
	fmt.Println("Time is", year, day, month, hour)
}

func scanYear() int {
	var y int
	var prompt string

	fmt.Print(fmt.Sprintf("Year[%d]:", time.Now().Year()))
	fmt.Scanln(&prompt)

	if prompt == "" {
		y = time.Now().Year()
	} else {
		y, _ = strconv.Atoi(prompt)
	}

	return y
}

func scanMonth() string {
	var prompt string
	var m string

	fmt.Print(fmt.Sprintf("Month[%d]:", time.Now().Month()))
	fmt.Scanln(&prompt)

	if prompt == "" {
		m = time.Now().Month().String()
	} else {
		monthIndex, _ := strconv.Atoi(prompt)
		m = time.Month(monthIndex).String()
	}

	return m
}

func scanHour() string {
	var prompt string

	fmt.Print("Time(HH:mm):")
	fmt.Scanln(&prompt)

	return prompt
}

func scanDay() int {
	var d int
	var prompt string

	fmt.Print(fmt.Sprintf("Day[%d]:", time.Now().Day()))
	fmt.Scanln(&prompt)

	if prompt == "" {
		d = time.Now().Day()
	} else {
		d, _ = strconv.Atoi(prompt)
	}

	return d
}
<|endoftext|>


<commit_before>package workers

import (
	//	"encoding/json"
	"fmt"
	//	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/dpn/models"
	"github.com/APTrust/exchange/dpn/network"
	"github.com/nsqio/go-nsq"
	//	"net/url"
	"strconv"
	"strings"
	//	"time"
)

// Keep the files in S3 up to 60 days, in case we're
// having system problems and we need to attempt the
// restore multiple times. We'll have other processes
// clean out the S3 bucket when necessary.
const DAYS_TO_KEEP_IN_S3 = 5

// Requests that an object be restored from Glacier to S3. This is
// the first step toward performing fixity checks on DPN bags, and
// restoring DPN bags, all of which are stored in Glacier.
type DPNGlacierRestoreInit struct {
	// Context includes logging, config, network connections, and
	// other general resources for the worker.
	Context *context.Context
	// LocalDPNRestClient lets us talk to our local DPN server.
	LocalDPNRestClient *network.DPNRestClient
	// RequestChannel is for requesting an item be moved from Glacier
	// into S3.
	RequestChannel chan *models.DPNGlacierRestoreState
	// CleanupChannel is for housekeeping, like updating NSQ.
	CleanupChannel chan *models.DPNGlacierRestoreState
	// PostTestChannel is for testing only. In production, nothing listens
	// on this channel.
	PostTestChannel chan *models.DPNGlacierRestoreState
	// S3Url is a custom URL that the S3 client should connect to.
	// We use this only in testing, when we want the client to talk
	// to a local test server. This should not be set in demo or
	// production.
	S3Url string
}

func DPNNewGlacierRestoreInit(_context *context.Context) (*DPNGlacierRestoreInit, error) {
	restorer := &DPNGlacierRestoreInit{
		Context: _context,
	}
	// Set up buffered channels
	restorerBufferSize := _context.Config.DPN.DPNGlacierRestoreWorker.NetworkConnections * 4
	workerBufferSize := _context.Config.DPN.DPNGlacierRestoreWorker.Workers * 10
	restorer.RequestChannel = make(chan *models.DPNGlacierRestoreState, restorerBufferSize)
	restorer.CleanupChannel = make(chan *models.DPNGlacierRestoreState, workerBufferSize)
	// Set up a limited number of go routines to handle the work.
	for i := 0; i < _context.Config.DPN.DPNGlacierRestoreWorker.NetworkConnections; i++ {
		go restorer.RequestRestore()
	}
	for i := 0; i < _context.Config.DPN.DPNGlacierRestoreWorker.Workers; i++ {
		go restorer.Cleanup()
	}
	// Set up a client to talk to our local DPN server.
	var err error
	restorer.LocalDPNRestClient, err = network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.RestClient.LocalAPIRoot,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	return restorer, err
}

// This is the callback that NSQ workers use to handle messages from NSQ.
func (restorer *DPNGlacierRestoreInit) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()

	state := restorer.GetRestoreState(message)
	restorer.SaveDPNWorkItem(state)
	if state.ErrorMessage != "" {
		restorer.Context.MessageLog.Error("Error setting up state for WorkItem %s: %s",
			string(message.Body), state.ErrorMessage)
		// No use proceeding...
		restorer.CleanupChannel <- state
		return fmt.Errorf(state.ErrorMessage)
	}
	if state.DPNWorkItem.IsCompletedOrCancelled() {
		restorer.Context.MessageLog.Info("Skipping WorkItem %d because status is %s",
			state.DPNWorkItem.Id, state.DPNWorkItem.Status)
		restorer.CleanupChannel <- state
		return nil
	}

	// OK, we're good
	restorer.RequestChannel <- state
	return nil
}

func (restorer *DPNGlacierRestoreInit) RequestRestore() {
	// for state := range restorer.RequestChannel {
	// 	// Request restore from Glacier
	// }
}

func (restorer *DPNGlacierRestoreInit) Cleanup() {
	for state := range restorer.CleanupChannel {
		if state.ErrorMessage != "" {
			restorer.FinishWithError(state)
		} else {
			restorer.FinishWithSuccess(state)
		}
		// For testing only. The test code creates the PostTestChannel.
		// When running in demo & production, this channel is nil.
		if restorer.PostTestChannel != nil {
			restorer.PostTestChannel <- state
		}
	}
}

func (restorer *DPNGlacierRestoreInit) FinishWithSuccess(state *models.DPNGlacierRestoreState) {
	state.DPNWorkItem.ClearNodeAndPid()
	note := "Awaiting availability in S3 for fixity check"
	if state.IsAvailableInS3 {
		note = "Item is available in S3 for fixity check"
	}
	state.DPNWorkItem.Note = &note
	restorer.SaveDPNWorkItem(state)
	state.NSQMessage.Finish()

	// Move to download queue
}

func (restorer *DPNGlacierRestoreInit) FinishWithError(state *models.DPNGlacierRestoreState) {
	state.DPNWorkItem.ClearNodeAndPid()
	state.DPNWorkItem.Note = &state.ErrorMessage
	restorer.SaveDPNWorkItem(state)
	state.NSQMessage.Finish()
}

// GetWorkItem returns the WorkItem with the specified Id from Pharos,
// or nil.
func (restorer *DPNGlacierRestoreInit) GetRestoreState(message *nsq.Message) *models.DPNGlacierRestoreState {
	msgBody := strings.TrimSpace(string(message.Body))
	restorer.Context.MessageLog.Info("NSQ Message body: '%s'", msgBody)
	state := &models.DPNGlacierRestoreState{}

	// Get the DPN work item
	dpnWorkItemId, err := strconv.Atoi(string(msgBody))
	if err != nil || dpnWorkItemId == 0 {
		state.ErrorMessage = fmt.Sprintf("Could not get DPNWorkItem Id from NSQ message body: %v", err)
		return state
	}
	resp := restorer.Context.PharosClient.DPNWorkItemGet(dpnWorkItemId)
	if resp.Error != nil {
		state.ErrorMessage = fmt.Sprintf("Error getting DPNWorkItem %d from Pharos: %v", dpnWorkItemId, resp.Error)
		return state
	}
	dpnWorkItem := resp.DPNWorkItem()
	if dpnWorkItem == nil {
		state.ErrorMessage = fmt.Sprintf("Pharos returned nil for WorkItem %d", dpnWorkItemId)
		return state
	}
	state.DPNWorkItem = dpnWorkItem
	state.DPNWorkItem.SetNodeAndPid()
	note := "Requesting Glacier restoration for fixity"
	state.DPNWorkItem.Note = &note

	// Get the DPN Bag from the DPN REST server.
	dpnResp := restorer.LocalDPNRestClient.DPNBagGet(dpnWorkItem.Identifier)
	if dpnResp.Error != nil {
		state.ErrorMessage = fmt.Sprintf("Error getting DPN bag %s from %s: %v", dpnWorkItem.Identifier,
			restorer.Context.Config.DPN.RestClient.LocalServiceURL, resp.Error)
		return state
	}
	dpnBag := dpnResp.Bag()
	if dpnBag == nil {
		state.ErrorMessage = fmt.Sprintf("DPN REST server returned nil for bag %s", dpnWorkItem.Identifier)
		return state
	}
	state.DPNBag = dpnBag

	// Although this is duplicate info, we record it in the state object
	// so we can see it in the Pharos UI when we're checking on the state
	// of an item.
	state.GlacierBucket = restorer.Context.Config.DPN.DPNGlacierRegion
	state.GlacierKey = dpnBag.UUID

	return state
}

func (restorer *DPNGlacierRestoreInit) SaveDPNWorkItem(state *models.DPNGlacierRestoreState) {
	jsonData, err := state.ToJson()
	if err != nil {
		msg := fmt.Sprintf("Could not marshal DPNGlacierRestoreState "+
			"for DPNWorkItem %d: %v", state.DPNWorkItem.Id, err)
		restorer.Context.MessageLog.Error(msg)
		note := "[JSON serialization error]"
		state.DPNWorkItem.Note = &note
	}

	// Update the DPNWorkItem
	state.DPNWorkItem.State = &jsonData
	state.DPNWorkItem.Retry = !state.ErrorIsFatal

	resp := restorer.Context.PharosClient.DPNWorkItemSave(state.DPNWorkItem)
	if resp.Error != nil {
		msg := fmt.Sprintf("Could not save DPNWorkItem %d "+
			"for fixity on bag %s to Pharos: %v",
			state.DPNWorkItem.Id, state.DPNWorkItem.Identifier, err)
		restorer.Context.MessageLog.Error(msg)
		if state.ErrorMessage == "" {
			state.ErrorMessage = msg
		}
	}
}
<commit_msg>Working on workflow/business logic for DPN Glacier restore<commit_after>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/dpn/models"
	dpn_network "github.com/APTrust/exchange/dpn/network"
	apt_network "github.com/APTrust/exchange/network"
	"github.com/nsqio/go-nsq"
	"strconv"
	"strings"
	"time"
)

// Standard retrieval is 3-5 hours.
// Bulk is 5-12 hours, and is cheaper.
// There's no rush on DPN fixity checking, so use the cheaper option.
// https://docs.aws.amazon.com/amazonglacier/latest/dev/downloading-an-archive-two-steps.html#api-downloading-an-archive-two-steps-retrieval-options
// For retrieval pricing, see https://aws.amazon.com/glacier/pricing/
const RETRIEVAL_OPTION = "Bulk"

// Keep the files in S3 up to 60 days, in case we're
// having system problems and we need to attempt the
// restore multiple times. We'll have other processes
// clean out the S3 bucket when necessary.
const DAYS_TO_KEEP_IN_S3 = 60

// Requests that an object be restored from Glacier to S3. This is
// the first step toward performing fixity checks on DPN bags, and
// restoring DPN bags, all of which are stored in Glacier.
type DPNGlacierRestoreInit struct {
	// Context includes logging, config, network connections, and
	// other general resources for the worker.
	Context *context.Context
	// LocalDPNRestClient lets us talk to our local DPN server.
	LocalDPNRestClient *dpn_network.DPNRestClient
	// RequestChannel is for requesting an item be moved from Glacier
	// into S3.
	RequestChannel chan *models.DPNGlacierRestoreState
	// CleanupChannel is for housekeeping, like updating NSQ.
	CleanupChannel chan *models.DPNGlacierRestoreState
	// PostTestChannel is for testing only. In production, nothing listens
	// on this channel.
	PostTestChannel chan *models.DPNGlacierRestoreState
	// S3Url is a custom URL that the S3 client should connect to.
	// We use this only in testing, when we want the client to talk
	// to a local test server. This should not be set in demo or
	// production.
	S3Url string
}

func DPNNewGlacierRestoreInit(_context *context.Context) (*DPNGlacierRestoreInit, error) {
	restorer := &DPNGlacierRestoreInit{
		Context: _context,
	}
	// Set up buffered channels
	restorerBufferSize := _context.Config.DPN.DPNGlacierRestoreWorker.NetworkConnections * 4
	workerBufferSize := _context.Config.DPN.DPNGlacierRestoreWorker.Workers * 10
	restorer.RequestChannel = make(chan *models.DPNGlacierRestoreState, restorerBufferSize)
	restorer.CleanupChannel = make(chan *models.DPNGlacierRestoreState, workerBufferSize)
	// Set up a limited number of go routines to handle the work.
	for i := 0; i < _context.Config.DPN.DPNGlacierRestoreWorker.NetworkConnections; i++ {
		go restorer.RequestRestore()
	}
	for i := 0; i < _context.Config.DPN.DPNGlacierRestoreWorker.Workers; i++ {
		go restorer.Cleanup()
	}
	// Set up a client to talk to our local DPN server.
	var err error
	restorer.LocalDPNRestClient, err = dpn_network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.RestClient.LocalAPIRoot,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	return restorer, err
}

// This is the callback that NSQ workers use to handle messages from NSQ.
func (restorer *DPNGlacierRestoreInit) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()

	state := restorer.GetRestoreState(message)
	restorer.SaveDPNWorkItem(state)
	if state.ErrorMessage != "" {
		restorer.Context.MessageLog.Error("Error setting up state for WorkItem %s: %s",
			string(message.Body), state.ErrorMessage)
		// No use proceeding...
		restorer.CleanupChannel <- state
		return fmt.Errorf(state.ErrorMessage)
	}
	if state.DPNWorkItem.IsCompletedOrCancelled() {
		restorer.Context.MessageLog.Info("Skipping WorkItem %d because status is %s",
			state.DPNWorkItem.Id, state.DPNWorkItem.Status)
		restorer.CleanupChannel <- state
		return nil
	}

	// OK, we're good. Ask Glacier to move the file into S3.
	restorer.RequestChannel <- state
	return nil
}

func (restorer *DPNGlacierRestoreInit) RequestRestore() {
	for state := range restorer.RequestChannel {
		restorer.InitializeRetrieval(state)
	}
}

func (restorer *DPNGlacierRestoreInit) Cleanup() {
	for state := range restorer.CleanupChannel {
		if state.ErrorMessage != "" {
			restorer.FinishWithError(state)
		} else {
			restorer.FinishWithSuccess(state)
		}
		// For testing only. The test code creates the PostTestChannel.
		// When running in demo & production, this channel is nil.
		if restorer.PostTestChannel != nil {
			restorer.PostTestChannel <- state
		}
	}
}

func (restorer *DPNGlacierRestoreInit) FinishWithSuccess(state *models.DPNGlacierRestoreState) {
	state.DPNWorkItem.ClearNodeAndPid()
	note := "Awaiting availability in S3 for fixity check"
	if state.IsAvailableInS3 {
		note = "Item is available in S3 for fixity check"
	}
	state.DPNWorkItem.Note = &note
	restorer.Context.MessageLog.Info("Requested %s from Glacier. %s", state.GlacierKey, note)
	restorer.SaveDPNWorkItem(state)
	state.NSQMessage.Finish()

	// TODO: Push to download queue
}

func (restorer *DPNGlacierRestoreInit) FinishWithError(state *models.DPNGlacierRestoreState) {
	state.DPNWorkItem.ClearNodeAndPid()
	state.DPNWorkItem.Note = &state.ErrorMessage
	restorer.Context.MessageLog.Error(state.ErrorMessage)
	restorer.SaveDPNWorkItem(state)

	attempts := int(state.NSQMessage.Attempts)
	maxAttempts := int(restorer.Context.Config.DPN.DPNGlacierRestoreWorker.MaxAttempts)

	if state.ErrorIsFatal {
		restorer.Context.MessageLog.Error("Error for %s is fatal. Not requeueing.", state.GlacierKey)
		state.NSQMessage.Finish()
	} else if attempts > maxAttempts {
		restorer.Context.MessageLog.Error("Attempt to restore %s failed %d times. Not requeuing.",
			attempts, state.GlacierKey)
		state.NSQMessage.Finish()
	} else {
		restorer.Context.MessageLog.Info("Error for %s is transient. Requeueing.", state.GlacierKey)
		state.NSQMessage.Requeue(1 * time.Minute)
	}
}

func (restorer *DPNGlacierRestoreInit) InitializeRetrieval(state *models.DPNGlacierRestoreState) {

	// Request restore from Glacier
	restorer.Context.MessageLog.Info("Requesting Glacier retrieval of %s from %s",
		state.GlacierKey, state.GlacierBucket)

	restoreClient := apt_network.NewS3Restore(
		restorer.Context.Config.GetAWSAccessKeyId(),
		restorer.Context.Config.GetAWSSecretAccessKey(),
		restorer.Context.Config.DPN.DPNGlacierRegion,
		state.GlacierBucket,
		state.GlacierKey,
		RETRIEVAL_OPTION,
		DAYS_TO_KEEP_IN_S3)

	// Custom S3Url is for testing only.
	if restorer.S3Url != "" {
		restorer.Context.MessageLog.Warning("Setting S3 URL to %s. This should happen only in testing!",
			restorer.S3Url)
		restoreClient.TestURL = restorer.S3Url
		restoreClient.BucketName = ""
	}

	// Figure out approximately how long this item will
	// be available in S3, once we restore it.
	now := time.Now().UTC()
	estimatedDeletionFromS3 := now.AddDate(0, 0, DAYS_TO_KEEP_IN_S3)

	// This is where me make the actual request to Glacier.
	restoreClient.Restore()
	if restoreClient.ErrorMessage != "" {
		state.ErrorMessage = fmt.Sprintf("Glacier retrieval request returned an error for %s at %s: %v",
			state.GlacierBucket, state.GlacierKey, restoreClient.ErrorMessage)
	}

	// Update this info.
	state.RequestAccepted = (restoreClient.ErrorMessage == "")
	state.RequestedAt = now
	state.EstimatedDeletionFromS3 = estimatedDeletionFromS3
}

// GetWorkItem returns the WorkItem with the specified Id from Pharos,
// or nil.
func (restorer *DPNGlacierRestoreInit) GetRestoreState(message *nsq.Message) *models.DPNGlacierRestoreState {
	msgBody := strings.TrimSpace(string(message.Body))
	restorer.Context.MessageLog.Info("NSQ Message body: '%s'", msgBody)
	state := &models.DPNGlacierRestoreState{}

	// Get the DPN work item
	dpnWorkItemId, err := strconv.Atoi(string(msgBody))
	if err != nil || dpnWorkItemId == 0 {
		state.ErrorMessage = fmt.Sprintf("Could not get DPNWorkItem Id from NSQ message body: %v", err)
		return state
	}
	resp := restorer.Context.PharosClient.DPNWorkItemGet(dpnWorkItemId)
	if resp.Error != nil {
		state.ErrorMessage = fmt.Sprintf("Error getting DPNWorkItem %d from Pharos: %v", dpnWorkItemId, resp.Error)
		return state
	}
	dpnWorkItem := resp.DPNWorkItem()
	if dpnWorkItem == nil {
		state.ErrorMessage = fmt.Sprintf("Pharos returned nil for WorkItem %d", dpnWorkItemId)
		return state
	}
	state.DPNWorkItem = dpnWorkItem
	state.DPNWorkItem.SetNodeAndPid()
	note := "Requesting Glacier restoration for fixity"
	state.DPNWorkItem.Note = &note

	// Get the DPN Bag from the DPN REST server.
	dpnResp := restorer.LocalDPNRestClient.DPNBagGet(dpnWorkItem.Identifier)
	if dpnResp.Error != nil {
		state.ErrorMessage = fmt.Sprintf("Error getting DPN bag %s from %s: %v", dpnWorkItem.Identifier,
			restorer.Context.Config.DPN.RestClient.LocalServiceURL, resp.Error)
		return state
	}
	dpnBag := dpnResp.Bag()
	if dpnBag == nil {
		state.ErrorMessage = fmt.Sprintf("DPN REST server returned nil for bag %s", dpnWorkItem.Identifier)
		return state
	}
	state.DPNBag = dpnBag

	// Although this is duplicate info, we record it in the state object
	// so we can see it in the Pharos UI when we're checking on the state
	// of an item.
	state.GlacierBucket = restorer.Context.Config.DPN.DPNGlacierRegion
	state.GlacierKey = dpnBag.UUID

	return state
}

func (restorer *DPNGlacierRestoreInit) SaveDPNWorkItem(state *models.DPNGlacierRestoreState) {
	jsonData, err := state.ToJson()
	if err != nil {
		msg := fmt.Sprintf("Could not marshal DPNGlacierRestoreState "+
			"for DPNWorkItem %d: %v", state.DPNWorkItem.Id, err)
		restorer.Context.MessageLog.Error(msg)
		note := "[JSON serialization error]"
		state.DPNWorkItem.Note = &note
	}

	// Update the DPNWorkItem
	state.DPNWorkItem.State = &jsonData
	state.DPNWorkItem.Retry = !state.ErrorIsFatal

	resp := restorer.Context.PharosClient.DPNWorkItemSave(state.DPNWorkItem)
	if resp.Error != nil {
		msg := fmt.Sprintf("Could not save DPNWorkItem %d "+
			"for fixity on bag %s to Pharos: %v",
			state.DPNWorkItem.Id, state.DPNWorkItem.Identifier, err)
		restorer.Context.MessageLog.Error(msg)
		if state.ErrorMessage == "" {
			state.ErrorMessage = msg
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	"runtime"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

type Tintptr *int // assignable to *int
type Tint int     // *Tint implements Tinter, interface{}

func (t *Tint) m() {}

type Tinter interface {
	m()
}

func TestFinalizerType(t *testing.T) {
	if runtime.GOARCH != "amd64" {
		t.Skipf("Skipping on non-amd64 machine")
	}

	ch := make(chan bool, 10)
	finalize := func(x *int) {
		if *x != 97531 {
			t.Errorf("finalizer %d, want %d", *x, 97531)
		}
		ch <- true
	}

	var finalizerTests = []struct {
		convert   func(*int) interface{}
		finalizer interface{}
	}{
		{func(x *int) interface{} { return x }, func(v *int) { finalize(v) }},
		{func(x *int) interface{} { return Tintptr(x) }, func(v Tintptr) { finalize(v) }},
		{func(x *int) interface{} { return Tintptr(x) }, func(v *int) { finalize(v) }},
		{func(x *int) interface{} { return (*Tint)(x) }, func(v *Tint) { finalize((*int)(v)) }},
		{func(x *int) interface{} { return (*Tint)(x) }, func(v Tinter) { finalize((*int)(v.(*Tint))) }},
	}

	for _, tt := range finalizerTests {
		func() {
			v := new(int)
			*v = 97531
			runtime.SetFinalizer(tt.convert(v), tt.finalizer)
			v = nil
		}()
		runtime.GC()
		select {
		case <-ch:
		case <-time.After(time.Second * 4):
			t.Errorf("Finalizer of type %T didn't run", tt.finalizer)
		}
	}
}

type bigValue struct {
	fill uint64
	it   bool
	up   string
}

func TestFinalizerInterfaceBig(t *testing.T) {
	if runtime.GOARCH != "amd64" {
		t.Skipf("Skipping on non-amd64 machine")
	}
	ch := make(chan bool)
	func() {
		v := &bigValue{0xDEADBEEFDEADBEEF, true, "It matters not how strait the gate"}
		runtime.SetFinalizer(v, func(v interface{}) {
			i, ok := v.(*bigValue)
			if !ok {
				t.Errorf("Expected *bigValue from interface{} in finalizer, got %v", *i)
			}
			if i.fill != 0xDEADBEEFDEADBEEF && i.it != true && i.up != "It matters not how strait the gate" {
				t.Errorf("*bigValue from interface{} has the wrong value: %v\n", *i)
			}
			close(ch)
		})
		v = nil
	}()
	runtime.GC()
	select {
	case <-ch:
	case <-time.After(time.Second * 4):
		t.Errorf("Finalizer set by SetFinalizer(*bigValue, func(interface{})) didn't run")
	}
}

func fin(v *int) {
}

func BenchmarkFinalizer(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	var wg sync.WaitGroup
	wg.Add(procs)
	for p := 0; p < procs; p++ {
		go func() {
			var data [CallsPerSched]*int
			for i := 0; i < CallsPerSched; i++ {
				data[i] = new(int)
			}
			for atomic.AddInt32(&N, -1) >= 0 {
				runtime.Gosched()
				for i := 0; i < CallsPerSched; i++ {
					runtime.SetFinalizer(data[i], fin)
				}
				for i := 0; i < CallsPerSched; i++ {
					runtime.SetFinalizer(data[i], nil)
				}
			}
			wg.Done()
		}()
	}
	wg.Wait()
}

func BenchmarkFinalizerRun(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	var wg sync.WaitGroup
	wg.Add(procs)
	for p := 0; p < procs; p++ {
		go func() {
			for atomic.AddInt32(&N, -1) >= 0 {
				runtime.Gosched()
				for i := 0; i < CallsPerSched; i++ {
					v := new(int)
					runtime.SetFinalizer(v, fin)
				}
				runtime.GC()
			}
			wg.Done()
		}()
	}
	wg.Wait()
}
<commit_msg>runtime: fix finalizer test on amd64<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	"runtime"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

type Tintptr *int // assignable to *int
type Tint int     // *Tint implements Tinter, interface{}

func (t *Tint) m() {}

type Tinter interface {
	m()
}

func TestFinalizerType(t *testing.T) {
	if runtime.GOARCH != "amd64" {
		t.Skipf("Skipping on non-amd64 machine")
	}

	ch := make(chan bool, 10)
	finalize := func(x *int) {
		if *x != 97531 {
			t.Errorf("finalizer %d, want %d", *x, 97531)
		}
		ch <- true
	}

	var finalizerTests = []struct {
		convert   func(*int) interface{}
		finalizer interface{}
	}{
		{func(x *int) interface{} { return x }, func(v *int) { finalize(v) }},
		{func(x *int) interface{} { return Tintptr(x) }, func(v Tintptr) { finalize(v) }},
		{func(x *int) interface{} { return Tintptr(x) }, func(v *int) { finalize(v) }},
		{func(x *int) interface{} { return (*Tint)(x) }, func(v *Tint) { finalize((*int)(v)) }},
		{func(x *int) interface{} { return (*Tint)(x) }, func(v Tinter) { finalize((*int)(v.(*Tint))) }},
	}

	for _, tt := range finalizerTests {
		go func() {
			v := new(int)
			*v = 97531
			runtime.SetFinalizer(tt.convert(v), tt.finalizer)
			v = nil
		}()
		time.Sleep(1 * time.Second)
		runtime.GC()
		select {
		case <-ch:
		case <-time.After(time.Second * 4):
			t.Errorf("finalizer for type %T didn't run", tt.finalizer)
		}
	}
}

type bigValue struct {
	fill uint64
	it   bool
	up   string
}

func TestFinalizerInterfaceBig(t *testing.T) {
	if runtime.GOARCH != "amd64" {
		t.Skipf("Skipping on non-amd64 machine")
	}
	ch := make(chan bool)
	go func() {
		v := &bigValue{0xDEADBEEFDEADBEEF, true, "It matters not how strait the gate"}
		old := *v
		runtime.SetFinalizer(v, func(v interface{}) {
			i, ok := v.(*bigValue)
			if !ok {
				t.Errorf("finalizer called with type %T, want *bigValue", v)
			}
			if *i != old {
				t.Errorf("finalizer called with %+v, want %+v", *i, old)
			}
			close(ch)
		})
		v = nil
	}()
	time.Sleep(1 * time.Second)
	runtime.GC()
	select {
	case <-ch:
	case <-time.After(4 * time.Second):
		t.Errorf("finalizer for type *bigValue didn't run")
	}
}

func fin(v *int) {
}

func BenchmarkFinalizer(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	var wg sync.WaitGroup
	wg.Add(procs)
	for p := 0; p < procs; p++ {
		go func() {
			var data [CallsPerSched]*int
			for i := 0; i < CallsPerSched; i++ {
				data[i] = new(int)
			}
			for atomic.AddInt32(&N, -1) >= 0 {
				runtime.Gosched()
				for i := 0; i < CallsPerSched; i++ {
					runtime.SetFinalizer(data[i], fin)
				}
				for i := 0; i < CallsPerSched; i++ {
					runtime.SetFinalizer(data[i], nil)
				}
			}
			wg.Done()
		}()
	}
	wg.Wait()
}

func BenchmarkFinalizerRun(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	var wg sync.WaitGroup
	wg.Add(procs)
	for p := 0; p < procs; p++ {
		go func() {
			for atomic.AddInt32(&N, -1) >= 0 {
				runtime.Gosched()
				for i := 0; i < CallsPerSched; i++ {
					v := new(int)
					runtime.SetFinalizer(v, fin)
				}
				runtime.GC()
			}
			wg.Done()
		}()
	}
	wg.Wait()
}
<|endoftext|>


<commit_before>package builds

import (
	"context"
	"sync"

	"code.cloudfoundry.org/lager/lagerctx"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/engine"
	"github.com/concourse/concourse/atc/metric"
	"github.com/concourse/concourse/atc/util"
)

func NewTracker(
	buildFactory db.BuildFactory,
	engine engine.Engine,
) *Tracker {
	return &Tracker{
		buildFactory: buildFactory,
		engine:       engine,
		running:      &sync.Map{},
	}
}

type Tracker struct {
	buildFactory db.BuildFactory
	engine       engine.Engine

	running *sync.Map
}

func (bt *Tracker) Run(ctx context.Context) error {
	logger := lagerctx.FromContext(ctx)

	logger.Debug("start")
	defer logger.Debug("done")

	builds, err := bt.buildFactory.GetAllStartedBuilds()
	if err != nil {
		logger.Error("failed-to-lookup-started-builds", err)
		return err
	}

	for _, b := range builds {
		if _, exists := bt.running.LoadOrStore(b.ID(), true); !exists {
			go func(build db.Build) {
				loggerData := build.LagerData()
				defer func() {
					err := util.DumpPanic(recover(), "tracking build %d", build.ID())
					if err != nil {
						logger.Error("panic-in-tracker-build-run", err)

						build.Finish(db.BuildStatusErrored)
					}
				}()

				defer bt.running.Delete(build.ID())

				if b.Name() == db.CheckBuildName {
					metric.Metrics.CheckBuildsRunning.Inc()
				} else {
					metric.Metrics.BuildsRunning.Inc()
				}

				defer func(b db.Build) {
					if b.Name() == db.CheckBuildName {
						metric.Metrics.CheckBuildsRunning.Dec()
					} else {
						metric.Metrics.BuildsRunning.Dec()
					}
				}(b)

				bt.engine.NewBuild(build).Run(
					lagerctx.NewContext(
						context.Background(),
						logger.Session("run", loggerData),
					),
				)
			}(b)
		}
	}

	return nil
}

func (bt *Tracker) Drain(ctx context.Context) {
	bt.engine.Drain(ctx)
}
<commit_msg>fix test failure.<commit_after>package builds

import (
	"context"
	"sync"

	"code.cloudfoundry.org/lager/lagerctx"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/engine"
	"github.com/concourse/concourse/atc/metric"
	"github.com/concourse/concourse/atc/util"
)

func NewTracker(
	buildFactory db.BuildFactory,
	engine engine.Engine,
) *Tracker {
	return &Tracker{
		buildFactory: buildFactory,
		engine:       engine,
		running:      &sync.Map{},
	}
}

type Tracker struct {
	buildFactory db.BuildFactory
	engine       engine.Engine

	running *sync.Map
}

func (bt *Tracker) Run(ctx context.Context) error {
	logger := lagerctx.FromContext(ctx)

	logger.Debug("start")
	defer logger.Debug("done")

	builds, err := bt.buildFactory.GetAllStartedBuilds()
	if err != nil {
		logger.Error("failed-to-lookup-started-builds", err)
		return err
	}

	for _, b := range builds {
		if _, exists := bt.running.LoadOrStore(b.ID(), true); !exists {
			go func(build db.Build) {
				loggerData := build.LagerData()
				defer func() {
					err := util.DumpPanic(recover(), "tracking build %d", build.ID())
					if err != nil {
						logger.Error("panic-in-tracker-build-run", err)

						build.Finish(db.BuildStatusErrored)
					}
				}()

				defer bt.running.Delete(build.ID())

				if build.Name() == db.CheckBuildName {
					metric.Metrics.CheckBuildsRunning.Inc()
				} else {
					metric.Metrics.BuildsRunning.Inc()
				}

				defer func(build db.Build) {
					if build.Name() == db.CheckBuildName {
						metric.Metrics.CheckBuildsRunning.Dec()
					} else {
						metric.Metrics.BuildsRunning.Dec()
					}
				}(build)

				bt.engine.NewBuild(build).Run(
					lagerctx.NewContext(
						context.Background(),
						logger.Session("run", loggerData),
					),
				)
			}(b)
		}
	}

	return nil
}

func (bt *Tracker) Drain(ctx context.Context) {
	bt.engine.Drain(ctx)
}
<|endoftext|>


<commit_before>package tfinstall

import (
	"context"
	"fmt"
	"io/ioutil"
	"log"
	"os/exec"
	"runtime"

	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/plumbing"
)

type GitRefOption struct {
	installDir string
	repoURL    string
	ref        string
}

var _ ExecPathFinder = &GitRefOption{}

func GitRef(ref, repo, installDir string) *GitRefOption {
	return &GitRefOption{
		installDir: installDir,
		repoURL:    repo,
		ref:        ref,
	}
}

func (opt *GitRefOption) ExecPath(ctx context.Context) (string, error) {
	installDir, err := ensureInstallDir(opt.installDir)
	if err != nil {
		return "", err
	}

	ref := plumbing.ReferenceName(opt.ref)
	if opt.ref == "" {
		ref = plumbing.ReferenceName("refs/heads/master")
	}

	repoURL := opt.repoURL
	if repoURL == "" {
		repoURL = "https://github.com/hashicorp/terraform.git"
	}

	_, err = git.PlainClone(installDir, false, &git.CloneOptions{
		URL:           repoURL,
		ReferenceName: ref,

		Depth: 1,
		Tags:  git.NoTags,
	})
	if err != nil {
		return "", fmt.Errorf("unable to clone %q: %w", repoURL, err)
	}

	var binName string
	{
		// TODO: maybe there is a better way to make sure this filename is available?
		// I guess we could locate it in a different dir, or nest the git underneath
		// the root tmp dir, etc.
		binPattern := "terraform"
		if runtime.GOOS == "windows" {
			binPattern = "terraform*.exe"
		}
		binFile, err := ioutil.TempFile(installDir, binPattern)
		if err != nil {
			return "", fmt.Errorf("unable to create bin file: %w", err)
		}
		binName = binFile.Name()
		binFile.Close()
	}

	cmd := exec.CommandContext(ctx, "go", "build", "-mod", "vendor", "-o", binName)
	cmd.Dir = installDir
	out, err := cmd.CombinedOutput()
	log.Print(string(out))
	if err != nil {
		return "", fmt.Errorf("unable to build Terraform: %w\n%s", err, out)
	}

	return binName, nil
}
<commit_msg>Avoid -mod=vendor when there's no vendor folder<commit_after>package tfinstall

import (
	"context"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"

	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/plumbing"
)

type GitRefOption struct {
	installDir string
	repoURL    string
	ref        string
}

var _ ExecPathFinder = &GitRefOption{}

func GitRef(ref, repo, installDir string) *GitRefOption {
	return &GitRefOption{
		installDir: installDir,
		repoURL:    repo,
		ref:        ref,
	}
}

func (opt *GitRefOption) ExecPath(ctx context.Context) (string, error) {
	installDir, err := ensureInstallDir(opt.installDir)
	if err != nil {
		return "", err
	}

	ref := plumbing.ReferenceName(opt.ref)
	if opt.ref == "" {
		ref = plumbing.ReferenceName("refs/heads/master")
	}

	repoURL := opt.repoURL
	if repoURL == "" {
		repoURL = "https://github.com/hashicorp/terraform.git"
	}

	_, err = git.PlainClone(installDir, false, &git.CloneOptions{
		URL:           repoURL,
		ReferenceName: ref,

		Depth: 1,
		Tags:  git.NoTags,
	})
	if err != nil {
		return "", fmt.Errorf("unable to clone %q: %w", repoURL, err)
	}

	var binName string
	{
		// TODO: maybe there is a better way to make sure this filename is available?
		// I guess we could locate it in a different dir, or nest the git underneath
		// the root tmp dir, etc.
		binPattern := "terraform"
		if runtime.GOOS == "windows" {
			binPattern = "terraform*.exe"
		}
		binFile, err := ioutil.TempFile(installDir, binPattern)
		if err != nil {
			return "", fmt.Errorf("unable to create bin file: %w", err)
		}
		binName = binFile.Name()
		binFile.Close()
	}

	goArgs := []string{"build", "-o", binName}

	vendorDir := filepath.Join(installDir, "vendor")
	if fi, err := os.Stat(vendorDir); err == nil && fi.IsDir() {
		goArgs = append(goArgs, "-mod", "vendor")
	}

	cmd := exec.CommandContext(ctx, "go", goArgs...)
	cmd.Dir = installDir
	out, err := cmd.CombinedOutput()
	log.Print(string(out))
	if err != nil {
		return "", fmt.Errorf("unable to build Terraform: %w\n%s", err, out)
	}

	return binName, nil
}
<|endoftext|>


<commit_before>package util

import "strings"
import "github.com/appPlant/alpinepass/src/data"
import "encoding/json"

import "os"
import "fmt"

//CheckError throws an exception if an error exists.
func CheckError(err error) {
	if err != nil {
		panic(err)
	}
}

// Filler is used to replace spaces in strings.
const Filler string = "_"

//CleanString removes whitespace in string.
func CleanString(s string) string {
	return strings.Replace(s, " ", Filler, -1)
}

//ThrowError prints the error information and exits the application.
func ThrowError(message string) {
	os.Stderr.WriteString(message)
	fmt.Println()
	os.Exit(-1)
}

//ThrowConfigError prints the error, some information about a Config and exits the application.
func ThrowConfigError(message string, config data.Config) {
	var configJSON []byte
	var err error
	configJSON, err = json.MarshalIndent(config, "", "    ")
	CheckError(err)

	os.Stderr.WriteString(message)
	fmt.Println()
	os.Stderr.WriteString(string(configJSON))
	fmt.Println()
	os.Exit(-1)
}
<commit_msg>Commit add helper function for checking if a string is contained in an array<commit_after>package util

import "strings"
import "github.com/appPlant/alpinepass/src/data"
import "encoding/json"

import "os"
import "fmt"

//CheckError throws an exception if an error exists.
func CheckError(err error) {
	if err != nil {
		panic(err)
	}
}

// Filler is used to replace spaces in strings.
const Filler string = "_"

//CleanString removes whitespace in string.
func CleanString(s string) string {
	return strings.Replace(s, " ", Filler, -1)
}

//ThrowError prints the error information and exits the application.
func ThrowError(message string) {
	os.Stderr.WriteString(message)
	fmt.Println()
	os.Exit(-1)
}

//ThrowConfigError prints the error, some information about a Config and exits the application.
func ThrowConfigError(message string, config data.Config) {
	var configJSON []byte
	var err error
	configJSON, err = json.MarshalIndent(config, "", "    ")
	CheckError(err)

	os.Stderr.WriteString(message)
	fmt.Println()
	os.Stderr.WriteString(string(configJSON))
	fmt.Println()
	os.Exit(-1)
}

//StringInArray checks if a string is contained in an array.
func StringInArray(item string, array []string) bool {
	for i := 0; i < len(array); i++ {
		if item == array[i] {
			return true
		}
	}
	return false
}
<|endoftext|>