<commit_before>package worker

import (
	"encoding/json"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/stitchfix/flotilla-os/config"
	"github.com/stitchfix/flotilla-os/execution/engine"
	flotillaLog "github.com/stitchfix/flotilla-os/log"
	"github.com/stitchfix/flotilla-os/queue"
	"github.com/stitchfix/flotilla-os/state"
	"gopkg.in/tomb.v2"
	"strings"
	"time"
)

type cloudtrailWorker struct {
	sm           state.Manager
	qm           queue.Manager
	conf         config.Config
	log          flotillaLog.Logger
	pollInterval time.Duration
	t            tomb.Tomb
	queue        string
	engine       *string
	s3Client     *s3.S3
}

func (ctw *cloudtrailWorker) Initialize(conf config.Config, sm state.Manager, ee engine.Engine, log flotillaLog.Logger, pollInterval time.Duration, engine *string, qm queue.Manager) error {
	ctw.pollInterval = pollInterval
	ctw.conf = conf
	ctw.sm = sm
	ctw.qm = qm
	ctw.log = log
	ctw.engine = engine
	ctw.queue = conf.GetString("cloudtrail_queue")
	_ = ctw.qm.Initialize(ctw.conf, "eks")

	awsRegion := conf.GetString("eks.manifest.storage.options.region")
	sess := session.Must(session.NewSession(&aws.Config{Region: aws.String(awsRegion)}))
	ctw.s3Client = s3.New(sess, aws.NewConfig().WithRegion(awsRegion))

	return nil
}

func (ctw *cloudtrailWorker) GetTomb() *tomb.Tomb {
	return &ctw.t
}

//
// Run lists queues, consumes runs from them, and executes them using the execution engine
//
func (ctw *cloudtrailWorker) Run() error {
	for {
		select {
		case <-ctw.t.Dying():
			_ = ctw.log.Log("message", "A CloudTrail worker was terminated")
			return nil
		default:
			ctw.runOnce()
			time.Sleep(ctw.pollInterval)
		}
	}
}

func (ctw *cloudtrailWorker) runOnce() {
	qurl, err := ctw.qm.QurlFor(ctw.queue, false)
	if err != nil {
		_ = ctw.log.Log("message", "Error receiving CloudTrail queue", "error", fmt.Sprintf("%+v", err))
		return
	}
	cloudTrailS3File, err := ctw.qm.ReceiveCloudTrail(qurl)
	if err != nil {
		_ = ctw.log.Log("message", "Error receiving CloudTrail file", "error", fmt.Sprintf("%+v", err))
		return
	}

	ctw.processS3Keys(cloudTrailS3File)
	_ = cloudTrailS3File.Done()
}

func (ctw *cloudtrailWorker) processS3Keys(cloudTrailS3File state.CloudTrailS3File) {
	var ctn state.CloudTrailNotifications
	defaultRegion := ctw.conf.GetString("aws_default_region")
	for _, keyName := range cloudTrailS3File.S3ObjectKey {
		if !strings.Contains(keyName, defaultRegion) {
			continue
		}
		getObjectOutput, err := ctw.s3Client.GetObject(&s3.GetObjectInput{
			Bucket: aws.String(cloudTrailS3File.S3Bucket),
			Key:    aws.String(keyName),
		})

		if err != nil {
			_ = ctw.log.Log("message", "Error receiving CloudTrail file - no object", "error", fmt.Sprintf("%+v", err))
			return
		}

		err = json.NewDecoder(getObjectOutput.Body).Decode(&ctn)

		_ = ctw.log.Log("message", "CloudTrail processing file", "key", fmt.Sprintf("s3://%s/%s", cloudTrailS3File.S3Bucket, keyName), "len", len(ctn.Records))
		ctw.processCloudTrailNotifications(ctn)

		getObjectOutput.Body.Close()
	}
}

func (ctw *cloudtrailWorker) processCloudTrailNotifications(ctn state.CloudTrailNotifications) {
	sa := ctw.conf.GetString("eks.service_account")
	runIdRecordMap := make(map[string][]state.Record)
	for _, record := range ctn.Records {
		if strings.Contains(record.UserIdentity.Arn, sa) && strings.Contains(record.UserIdentity.Arn, "eks-") {
			runId := ctw.getRunId(record)
			runIdRecordMap[runId] = append(runIdRecordMap[runId], record)
		}
	}

	for runId, records := range runIdRecordMap {
		_ = ctw.log.Log("message", "Saving CloudTrail Events", "run_id", runId, len(records))
		run, err := ctw.sm.GetRun(runId)
		if err == nil {
			run.CloudTrailNotifications.Records = append(run.CloudTrailNotifications.Records, records...)
			_, err = ctw.sm.UpdateRun(runId, run)
			if err != nil {
				_ = ctw.log.Log("message", "Error updating run", "error", fmt.Sprintf("%+v", err))
			}
		}
	}
}
func (ctw *cloudtrailWorker) getRunId(record state.Record) string {
	splits := strings.Split(record.UserIdentity.Arn, "/")
	runId := splits[len(splits)-1]
	return runId
}
<commit_msg>fix assigning ctn<commit_after>package worker

import (
	"encoding/json"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/stitchfix/flotilla-os/config"
	"github.com/stitchfix/flotilla-os/execution/engine"
	flotillaLog "github.com/stitchfix/flotilla-os/log"
	"github.com/stitchfix/flotilla-os/queue"
	"github.com/stitchfix/flotilla-os/state"
	"gopkg.in/tomb.v2"
	"strings"
	"time"
)

type cloudtrailWorker struct {
	sm           state.Manager
	qm           queue.Manager
	conf         config.Config
	log          flotillaLog.Logger
	pollInterval time.Duration
	t            tomb.Tomb
	queue        string
	engine       *string
	s3Client     *s3.S3
}

func (ctw *cloudtrailWorker) Initialize(conf config.Config, sm state.Manager, ee engine.Engine, log flotillaLog.Logger, pollInterval time.Duration, engine *string, qm queue.Manager) error {
	ctw.pollInterval = pollInterval
	ctw.conf = conf
	ctw.sm = sm
	ctw.qm = qm
	ctw.log = log
	ctw.engine = engine
	ctw.queue = conf.GetString("cloudtrail_queue")
	_ = ctw.qm.Initialize(ctw.conf, "eks")

	awsRegion := conf.GetString("eks.manifest.storage.options.region")
	sess := session.Must(session.NewSession(&aws.Config{Region: aws.String(awsRegion)}))
	ctw.s3Client = s3.New(sess, aws.NewConfig().WithRegion(awsRegion))

	return nil
}

func (ctw *cloudtrailWorker) GetTomb() *tomb.Tomb {
	return &ctw.t
}

//
// Run lists queues, consumes runs from them, and executes them using the execution engine
//
func (ctw *cloudtrailWorker) Run() error {
	for {
		select {
		case <-ctw.t.Dying():
			_ = ctw.log.Log("message", "A CloudTrail worker was terminated")
			return nil
		default:
			ctw.runOnce()
			time.Sleep(ctw.pollInterval)
		}
	}
}

func (ctw *cloudtrailWorker) runOnce() {
	qurl, err := ctw.qm.QurlFor(ctw.queue, false)
	if err != nil {
		_ = ctw.log.Log("message", "Error receiving CloudTrail queue", "error", fmt.Sprintf("%+v", err))
		return
	}
	cloudTrailS3File, err := ctw.qm.ReceiveCloudTrail(qurl)
	if err != nil {
		_ = ctw.log.Log("message", "Error receiving CloudTrail file", "error", fmt.Sprintf("%+v", err))
		return
	}

	ctw.processS3Keys(cloudTrailS3File)
	_ = cloudTrailS3File.Done()
}

func (ctw *cloudtrailWorker) processS3Keys(cloudTrailS3File state.CloudTrailS3File) {
	var ctn state.CloudTrailNotifications
	defaultRegion := ctw.conf.GetString("aws_default_region")
	for _, keyName := range cloudTrailS3File.S3ObjectKey {
		if !strings.Contains(keyName, defaultRegion) {
			continue
		}
		getObjectOutput, err := ctw.s3Client.GetObject(&s3.GetObjectInput{
			Bucket: aws.String(cloudTrailS3File.S3Bucket),
			Key:    aws.String(keyName),
		})

		if err != nil {
			_ = ctw.log.Log("message", "Error receiving CloudTrail file - no object", "error", fmt.Sprintf("%+v", err))
			return
		}

		err = json.NewDecoder(getObjectOutput.Body).Decode(&ctn)

		_ = ctw.log.Log("message", "CloudTrail processing file", "key", fmt.Sprintf("s3://%s/%s", cloudTrailS3File.S3Bucket, keyName), "len", len(ctn.Records))
		ctw.processCloudTrailNotifications(ctn)

		getObjectOutput.Body.Close()
	}
}

func (ctw *cloudtrailWorker) processCloudTrailNotifications(ctn state.CloudTrailNotifications) {
	sa := ctw.conf.GetString("eks.service_account")
	runIdRecordMap := make(map[string][]state.Record)
	for _, record := range ctn.Records {
		if strings.Contains(record.UserIdentity.Arn, sa) && strings.Contains(record.UserIdentity.Arn, "eks-") {
			runId := ctw.getRunId(record)
			runIdRecordMap[runId] = append(runIdRecordMap[runId], record)
		}
	}

	for runId, records := range runIdRecordMap {
		_ = ctw.log.Log("message", "Saving CloudTrail Events", "run_id", runId, len(records))
		run, err := ctw.sm.GetRun(runId)
		if err == nil {
			if run.CloudTrailNotifications == nil || len((*run.CloudTrailNotifications).Records) == 0 {
				run.CloudTrailNotifications = &state.CloudTrailNotifications{Records: records}
			} else {
				run.CloudTrailNotifications = &state.CloudTrailNotifications{Records: append((*run.CloudTrailNotifications).Records, records...)}
			}

			_, err = ctw.sm.UpdateRun(runId, run)
			if err != nil {
				_ = ctw.log.Log("message", "Error updating run", "error", fmt.Sprintf("%+v", err))
			}
		}
	}
}
func (ctw *cloudtrailWorker) getRunId(record state.Record) string {
	splits := strings.Split(record.UserIdentity.Arn, "/")
	runId := splits[len(splits)-1]
	return runId
}
<|endoftext|>


<commit_before>package singular

<commit_msg>worker/singular: initial implementation<commit_after>package singular

import (
	"fmt"
	"time"

	"launchpad.net/tomb"
	"github.com/juju/loggo"

	"launchpad.net/juju-core/worker"
)

var logger = loggo.GetLogger("juju.worker")

var pingInterval = 10 * time.Second

type runner struct {
	tomb     tomb.Tomb
	isMaster bool
	worker.Runner
}

// Conn represents a connection to some resource.
type Conn interface {
	// IsMaster reports whether this connection is currently held by
	// the (singular) master of the resource.
	IsMaster() (bool, error)

	// Ping probes the resource and returns an error if the the
	// connection has failed. If the master changes, this method
	// must return an error.
	Ping() error
}

// New returns a Runner that can be used to start workers that will only
// run a single instance. The conn value is used to determine whether to
// run the workers or not.
//
// Any workers started will be started on the underlying runner, but
// will do nothing if the connection is not currently held by the master
// of the resource.
func New(underlying worker.Runner, conn Conn) (worker.Runner, error) {
	isMaster, err := conn.IsMaster()
	if err != nil {
		return nil, fmt.Errorf("cannot get master status: %v", err)
	}
	r := &runner{
		isMaster: isMaster,
		Runner:   underlying,
	}
	go func() {
		defer r.tomb.Done()
		if !isMaster {
			// We're not master, so start a pinger so that we know when
			// the connection master changes.
			r.pinger(conn)
		} else {
			// We *are* master; the started workers should
			// encounter an error as they do what they're supposed
			// to do - no need for an extra pinger to tell us.
			// We just wait to be killed.
			<-r.tomb.Dying()
		}
	}()
	return r, nil
}

func (r *runner) pinger(conn Conn) {
	timer := time.NewTimer(0)
	for {
		err := conn.Ping()
		if err != nil {
			logger.Infof("ping error: %v", err)
			return
		}
		timer.Reset(pingInterval)
		select {
		case <-timer.C:
		case <-r.tomb.Dying():
			return
		}
	}
}

func (r *runner) StartWorker(id string, startFunc func() (worker.Worker, error)) error {
	if r.isMaster {
		return r.Runner.StartWorker(id, startFunc)
	}
	return r.Runner.StartWorker(id, func() (worker.Worker, error) {
		return worker.NewSimpleWorker(func(stop <-chan struct{}) error {
			logger.Infof("not starting %q because we are not master", id)
			select {
			case <-stop:
			case <-r.tomb.Dying():
			}
			return nil
		}), nil
	})
}
<|endoftext|>


<commit_before>package node

import (
	"encoding/binary"
	"erlang/dist"
	"erlang/epmd"
	"erlang/term"
	"flag"
	"fmt"
	"log"
	"net"
	"strconv"
	"strings"
)

var nTrace bool

func init() {
	flag.BoolVar(&nTrace, "erlang.node.trace", false, "trace erlang node")
}

func nLog(f string, a ...interface{}) {
	if nTrace {
		log.Printf(f, a...)
	}
}

type regReq struct {
	replyTo  chan term.Pid
	channels procChannels
}

type registryChan struct {
	storeChan chan regReq
}

type nodeConn struct {
	conn  net.Conn
	wchan chan []term.Term
}

type Node struct {
	epmd.NodeInfo
	Cookie     string
	port       int32
	registry   *registryChan
	channels   map[term.Pid]procChannels
	registered map[term.Atom]term.Pid
	neighbors  map[term.Atom]nodeConn
}

type procChannels struct {
	in     chan term.Term
	inFrom chan term.Tuple
	ctl    chan term.Term
}

type Behaviour interface {
	ProcessLoop(n *Node, pid term.Pid, pcs procChannels, pd Process, args ...interface{})
}

type Process interface {
	Behaviour() (behaviour Behaviour, options map[string]interface{})
}

func NewNode(name string, cookie string) (node *Node) {
	nLog("Start with name '%s' and cookie '%s'", name, cookie)
	// TODO: add fqdn support
	ns := strings.Split(name, "@")
	nodeInfo := epmd.NodeInfo{
		FullName: name,
		Name:     ns[0],
		Domain:   ns[1],
		Port:     0,
		Type:     77, // or 72 if hidden
		Protocol: 0,
		HighVsn:  5,
		LowVsn:   5,
		Creation: 0,
	}

	registry := &registryChan{
		storeChan: make(chan regReq),
	}

	node = &Node{
		NodeInfo:   nodeInfo,
		Cookie:     cookie,
		registry:   registry,
		channels:   make(map[term.Pid]procChannels),
		registered: make(map[term.Atom]term.Pid),
		neighbors:  make(map[term.Atom]nodeConn),
	}
	return node
}

func (n *Node) prepareProcesses() {
	nk := new(netKernel)
	nkPid := n.Spawn(nk, n)
	n.Register(term.Atom("net_kernel"), nkPid)

	gns := new(globalNameServer)
	gnsPid := n.Spawn(gns, n)
	n.Register(term.Atom("global_name_server"), gnsPid)

	rex := new(rexRPC)
	rexPid := n.Spawn(rex, n)
	n.Register(term.Atom("rex"), rexPid)

}

func (n *Node) Spawn(pd Process, args ...interface{}) (pid term.Pid) {
	behaviour, options := pd.Behaviour()
	chanSize, ok := options["chan-size"].(int)
	if !ok {
		chanSize = 100
	}
	ctlChanSize, ok := options["chan-size"].(int)
	if !ok {
		chanSize = 100
	}
	in := make(chan term.Term, chanSize)
	inFrom := make(chan term.Tuple, chanSize)
	ctl := make(chan term.Term, ctlChanSize)
	pcs := procChannels{
		in:     in,
		inFrom: inFrom,
		ctl:    ctl,
	}
	pid = n.storeProcess(pcs)
	go behaviour.ProcessLoop(n, pid, pcs, pd, args...)
	return
}

func (n *Node) Register(name term.Atom, pid term.Pid) {
	n.registered[name] = pid
}

func (n *Node) Registered() (pids []term.Atom) {
	pids = make([]term.Atom, len(n.registered))
	i := 0
	for p, _ := range n.registered {
		pids[i] = p
		i++
	}
	return
}

func (n *Node) registrator() {
	for {
		select {
		case req := <-n.registry.storeChan:
			// FIXME: make proper allocation, now it just stub
			var id uint32 = 0
			for k, _ := range n.channels {
				if k.Id >= id {
					id = k.Id + 1
				}
			}
			var pid term.Pid
			pid.Node = term.Atom(n.FullName)
			pid.Id = id
			pid.Serial = 0 // FIXME
			pid.Creation = byte(n.Creation)

			n.channels[pid] = req.channels
			req.replyTo <- pid
		}
	}
}

func (n *Node) storeProcess(chs procChannels) (pid term.Pid) {
	myChan := make(chan term.Pid)
	n.registry.storeChan <- regReq{replyTo: myChan, channels: chs}
	pid = <-myChan
	return pid
}

func (n *Node) Connect(remote string) {

}

func (n *Node) Publish(port int) (err error) {
	nLog("Publish ENode at %d", port)
	l, err := net.Listen("tcp", net.JoinHostPort("", strconv.Itoa(port)))
	if err != nil {
		return
	}
	n.Port = uint16(port)
	aliveResp := make(chan uint16)
	go epmdC(n, aliveResp)
	creation := <-aliveResp
	switch creation {
	case 99:
		return fmt.Errorf("Duplicate name '%s'", n.Name)
	case 100:
		return fmt.Errorf("Cannot connect to EPMD")
	default:
		n.Creation = creation
	}

	go func() {
		for {
			conn, err := l.Accept()
			nLog("Accept new at ENode")
			if err != nil {
				nLog(err.Error())
			} else {
				wchan := make(chan []term.Term, 10)
				ndchan := make(chan *dist.NodeDesc)
				go n.mLoopReader(conn, wchan, ndchan)
				go n.mLoopWriter(conn, wchan, ndchan)
			}
		}
	}()
	go n.registrator()
	n.prepareProcesses()
	return nil
}

func (currNode *Node) mLoopReader(c net.Conn, wchan chan []term.Term, ndchan chan *dist.NodeDesc) {

	currNd := dist.NewNodeDesc(currNode.FullName, currNode.Cookie, false)
	ndchan <- currNd
	for {
		terms, err := currNd.ReadMessage(c)
		if err != nil {
			nLog("Enode error: %s", err.Error())
			break
		}
		currNode.handleTerms(c, wchan, terms)
	}
	c.Close()
}

func (currNode *Node) mLoopWriter(c net.Conn, wchan chan []term.Term, ndchan chan *dist.NodeDesc) {

	currNd := <-ndchan

	for {
		terms := <-wchan
		err := currNd.WriteMessage(c, terms)
		if err != nil {
			nLog("Enode error: %s", err.Error())
			break
		}
	}
	c.Close()
}

func (currNode *Node) handleTerms(c net.Conn, wchan chan []term.Term, terms []term.Term) {
	nLog("Node terms: %#v", terms)

	if len(terms) == 0 {
		return
	}
	switch t := terms[0].(type) {
	case term.Tuple:
		if len(t) > 0 {
			switch act := t.Element(1).(type) {
			case term.Int:
				switch act {
				case REG_SEND:
					if len(terms) == 2 {
						currNode.RegSend(t.Element(2), t.Element(4), terms[1])
					} else {
						nLog("*** ERROR: bad REG_SEND: %#v", terms)
					}
				default:
					nLog("Unhandled node message: %#v", t)
				}
			case term.Atom:
				switch act {
				case term.Atom("$go_set_node"):
					nLog("SET NODE %#v", t)
					currNode.neighbors[t[1].(term.Atom)] = nodeConn{conn: c, wchan: wchan}
				}
			}
		}
	}
}

func (currNode *Node) RegSend(from, to term.Term, message term.Term) {
	nLog("REG_SEND: From: %#v, To: %#v, Message: %#v", from, to, message)
	var toPid term.Pid
	switch tp := to.(type) {
	case term.Pid:
		toPid = tp
	case term.Atom:
		toPid = currNode.Whereis(tp)
	}
	currNode.SendFrom(from, toPid, message)
}

func (currNode *Node) Whereis(who term.Atom) (pid term.Pid) {
	pid, _ = currNode.registered[who]
	return
}

func (currNode *Node) SendFrom(from term.Term, to term.Pid, message term.Term) {
	nLog("SendFrom: %#v, %#v, %#v", from, to, message)
	pcs := currNode.channels[to]
	pcs.inFrom <- term.Tuple{from, message}
}

func (currNode *Node) Send(to term.Pid, message term.Term) {
	nLog("Send: %#v, %#v", to, message)
	if string(to.Node) == currNode.FullName {
		nLog("Send to local node")
		pcs := currNode.channels[to]
		pcs.in <- message
	} else {
		nLog("Send to remote node: %#v, %#v", to, currNode.neighbors[to.Node])

		msg := []term.Term{term.Tuple{SEND, term.Atom(""), to}, message}
		currNode.neighbors[to.Node].wchan <- msg
	}
}

func epmdC(n *Node, resp chan uint16) {
	conn, err := net.Dial("tcp", ":4369")
	defer conn.Close()
	if err != nil {
		resp <- 100
		return
	}

	epmdFROM := make(chan []byte)
	go epmdREADER(conn, epmdFROM)

	epmdTO := make(chan []byte)
	go epmdWRITER(conn, epmdFROM, epmdTO)

	epmdTO <- epmd.Compose_ALIVE2_REQ(&n.NodeInfo)

	for {

		select {
		case reply := <-epmdFROM:
			nLog("From EPMD: %v", reply)

			switch epmd.MessageId(reply[0]) {
			case epmd.ALIVE2_RESP:
				if creation, ok := epmd.Read_ALIVE2_RESP(reply); ok {
					resp <- creation
				} else {
					resp <- 99
				}
			}

		}

	}
}

func epmdREADER(conn net.Conn, in chan []byte) {
	for {
		buf := make([]byte, 1024)
		n, err := conn.Read(buf)
		if err != nil {
			in <- buf[0:n]
			in <- []byte{}
			return
		}
		nLog("Read from EPMD %d: %v", n, buf[:n])
		in <- buf[:n]
	}
}

func epmdWRITER(conn net.Conn, in chan []byte, out chan []byte) {
	for {
		select {
		case data := <-out:
			buf := make([]byte, 2)
			binary.BigEndian.PutUint16(buf[0:2], uint16(len(data)))
			buf = append(buf, data...)
			_, err := conn.Write(buf)
			if err != nil {
				in <- []byte{}
			}
		}
	}
}
<commit_msg>Async registration<commit_after>package node

import (
	"encoding/binary"
	"erlang/dist"
	"erlang/epmd"
	"erlang/term"
	"flag"
	"fmt"
	"log"
	"net"
	"strconv"
	"strings"
)

var nTrace bool

func init() {
	flag.BoolVar(&nTrace, "erlang.node.trace", false, "trace erlang node")
}

func nLog(f string, a ...interface{}) {
	if nTrace {
		log.Printf(f, a...)
	}
}

type regReq struct {
	replyTo  chan term.Pid
	channels procChannels
}

type regNameReq struct {
	name term.Atom
	pid term.Pid
}

type unregNameReq struct {
	name term.Atom
}

type registryChan struct {
	storeChan chan regReq
	regNameChan chan regNameReq
	unregNameChan chan unregNameReq
}

type nodeConn struct {
	conn  net.Conn
	wchan chan []term.Term
}

type Node struct {
	epmd.NodeInfo
	Cookie     string
	port       int32
	registry   *registryChan
	channels   map[term.Pid]procChannels
	registered map[term.Atom]term.Pid
	neighbors  map[term.Atom]nodeConn
}

type procChannels struct {
	in     chan term.Term
	inFrom chan term.Tuple
	ctl    chan term.Term
}

type Behaviour interface {
	ProcessLoop(n *Node, pid term.Pid, pcs procChannels, pd Process, args ...interface{})
}

type Process interface {
	Behaviour() (behaviour Behaviour, options map[string]interface{})
}

func NewNode(name string, cookie string) (node *Node) {
	nLog("Start with name '%s' and cookie '%s'", name, cookie)
	// TODO: add fqdn support
	ns := strings.Split(name, "@")
	nodeInfo := epmd.NodeInfo{
		FullName: name,
		Name:     ns[0],
		Domain:   ns[1],
		Port:     0,
		Type:     77, // or 72 if hidden
		Protocol: 0,
		HighVsn:  5,
		LowVsn:   5,
		Creation: 0,
	}

	registry := &registryChan{
		storeChan: make(chan regReq),
		regNameChan: make(chan regNameReq),
		unregNameChan: make(chan unregNameReq),
	}

	node = &Node{
		NodeInfo:   nodeInfo,
		Cookie:     cookie,
		registry:   registry,
		channels:   make(map[term.Pid]procChannels),
		registered: make(map[term.Atom]term.Pid),
		neighbors:  make(map[term.Atom]nodeConn),
	}
	return node
}

func (n *Node) prepareProcesses() {
	nk := new(netKernel)
	nkPid := n.Spawn(nk, n)
	n.Register(term.Atom("net_kernel"), nkPid)

	gns := new(globalNameServer)
	gnsPid := n.Spawn(gns, n)
	n.Register(term.Atom("global_name_server"), gnsPid)

	rex := new(rexRPC)
	rexPid := n.Spawn(rex, n)
	n.Register(term.Atom("rex"), rexPid)

}

func (n *Node) Spawn(pd Process, args ...interface{}) (pid term.Pid) {
	behaviour, options := pd.Behaviour()
	chanSize, ok := options["chan-size"].(int)
	if !ok {
		chanSize = 100
	}
	ctlChanSize, ok := options["chan-size"].(int)
	if !ok {
		chanSize = 100
	}
	in := make(chan term.Term, chanSize)
	inFrom := make(chan term.Tuple, chanSize)
	ctl := make(chan term.Term, ctlChanSize)
	pcs := procChannels{
		in:     in,
		inFrom: inFrom,
		ctl:    ctl,
	}
	pid = n.storeProcess(pcs)
	go behaviour.ProcessLoop(n, pid, pcs, pd, args...)
	return
}

func (n *Node) Register(name term.Atom, pid term.Pid) {
	r := regNameReq{name: name, pid: pid}
	n.registry.regNameChan <- r
}

func (n *Node) Unregister(name term.Atom) {
	r := unregNameReq{name: name}
	n.registry.unregNameChan <- r
}

func (n *Node) Registered() (pids []term.Atom) {
	pids = make([]term.Atom, len(n.registered))
	i := 0
	for p, _ := range n.registered {
		pids[i] = p
		i++
	}
	return
}

func (n *Node) registrator() {
	for {
		select {
		case req := <-n.registry.storeChan:
			// FIXME: make proper allocation, now it just stub
			var id uint32 = 0
			for k, _ := range n.channels {
				if k.Id >= id {
					id = k.Id + 1
				}
			}
			var pid term.Pid
			pid.Node = term.Atom(n.FullName)
			pid.Id = id
			pid.Serial = 0 // FIXME
			pid.Creation = byte(n.Creation)

			n.channels[pid] = req.channels
			req.replyTo <- pid
		case req := <-n.registry.regNameChan:
			n.registered[req.name] = req.pid
		case req := <-n.registry.unregNameChan:
			delete(n.registered, req.name)
		}
	}
}

func (n *Node) storeProcess(chs procChannels) (pid term.Pid) {
	myChan := make(chan term.Pid)
	n.registry.storeChan <- regReq{replyTo: myChan, channels: chs}
	pid = <-myChan
	return pid
}

func (n *Node) Connect(remote string) {

}

func (n *Node) Publish(port int) (err error) {
	nLog("Publish ENode at %d", port)
	l, err := net.Listen("tcp", net.JoinHostPort("", strconv.Itoa(port)))
	if err != nil {
		return
	}
	n.Port = uint16(port)
	aliveResp := make(chan uint16)
	go epmdC(n, aliveResp)
	creation := <-aliveResp
	switch creation {
	case 99:
		return fmt.Errorf("Duplicate name '%s'", n.Name)
	case 100:
		return fmt.Errorf("Cannot connect to EPMD")
	default:
		n.Creation = creation
	}

	go func() {
		for {
			conn, err := l.Accept()
			nLog("Accept new at ENode")
			if err != nil {
				nLog(err.Error())
			} else {
				wchan := make(chan []term.Term, 10)
				ndchan := make(chan *dist.NodeDesc)
				go n.mLoopReader(conn, wchan, ndchan)
				go n.mLoopWriter(conn, wchan, ndchan)
			}
		}
	}()
	go n.registrator()
	n.prepareProcesses()
	return nil
}

func (currNode *Node) mLoopReader(c net.Conn, wchan chan []term.Term, ndchan chan *dist.NodeDesc) {

	currNd := dist.NewNodeDesc(currNode.FullName, currNode.Cookie, false)
	ndchan <- currNd
	for {
		terms, err := currNd.ReadMessage(c)
		if err != nil {
			nLog("Enode error: %s", err.Error())
			break
		}
		currNode.handleTerms(c, wchan, terms)
	}
	c.Close()
}

func (currNode *Node) mLoopWriter(c net.Conn, wchan chan []term.Term, ndchan chan *dist.NodeDesc) {

	currNd := <-ndchan

	for {
		terms := <-wchan
		err := currNd.WriteMessage(c, terms)
		if err != nil {
			nLog("Enode error: %s", err.Error())
			break
		}
	}
	c.Close()
}

func (currNode *Node) handleTerms(c net.Conn, wchan chan []term.Term, terms []term.Term) {
	nLog("Node terms: %#v", terms)

	if len(terms) == 0 {
		return
	}
	switch t := terms[0].(type) {
	case term.Tuple:
		if len(t) > 0 {
			switch act := t.Element(1).(type) {
			case term.Int:
				switch act {
				case REG_SEND:
					if len(terms) == 2 {
						currNode.RegSend(t.Element(2), t.Element(4), terms[1])
					} else {
						nLog("*** ERROR: bad REG_SEND: %#v", terms)
					}
				default:
					nLog("Unhandled node message: %#v", t)
				}
			case term.Atom:
				switch act {
				case term.Atom("$go_set_node"):
					nLog("SET NODE %#v", t)
					currNode.neighbors[t[1].(term.Atom)] = nodeConn{conn: c, wchan: wchan}
				}
			}
		}
	}
}

func (currNode *Node) RegSend(from, to term.Term, message term.Term) {
	nLog("REG_SEND: From: %#v, To: %#v, Message: %#v", from, to, message)
	var toPid term.Pid
	switch tp := to.(type) {
	case term.Pid:
		toPid = tp
	case term.Atom:
		toPid = currNode.Whereis(tp)
	}
	currNode.SendFrom(from, toPid, message)
}

func (currNode *Node) Whereis(who term.Atom) (pid term.Pid) {
	pid, _ = currNode.registered[who]
	return
}

func (currNode *Node) SendFrom(from term.Term, to term.Pid, message term.Term) {
	nLog("SendFrom: %#v, %#v, %#v", from, to, message)
	pcs := currNode.channels[to]
	pcs.inFrom <- term.Tuple{from, message}
}

func (currNode *Node) Send(to term.Pid, message term.Term) {
	nLog("Send: %#v, %#v", to, message)
	if string(to.Node) == currNode.FullName {
		nLog("Send to local node")
		pcs := currNode.channels[to]
		pcs.in <- message
	} else {
		nLog("Send to remote node: %#v, %#v", to, currNode.neighbors[to.Node])

		msg := []term.Term{term.Tuple{SEND, term.Atom(""), to}, message}
		currNode.neighbors[to.Node].wchan <- msg
	}
}

func epmdC(n *Node, resp chan uint16) {
	conn, err := net.Dial("tcp", ":4369")
	defer conn.Close()
	if err != nil {
		resp <- 100
		return
	}

	epmdFROM := make(chan []byte)
	go epmdREADER(conn, epmdFROM)

	epmdTO := make(chan []byte)
	go epmdWRITER(conn, epmdFROM, epmdTO)

	epmdTO <- epmd.Compose_ALIVE2_REQ(&n.NodeInfo)

	for {

		select {
		case reply := <-epmdFROM:
			nLog("From EPMD: %v", reply)

			switch epmd.MessageId(reply[0]) {
			case epmd.ALIVE2_RESP:
				if creation, ok := epmd.Read_ALIVE2_RESP(reply); ok {
					resp <- creation
				} else {
					resp <- 99
				}
			}

		}

	}
}

func epmdREADER(conn net.Conn, in chan []byte) {
	for {
		buf := make([]byte, 1024)
		n, err := conn.Read(buf)
		if err != nil {
			in <- buf[0:n]
			in <- []byte{}
			return
		}
		nLog("Read from EPMD %d: %v", n, buf[:n])
		in <- buf[:n]
	}
}

func epmdWRITER(conn net.Conn, in chan []byte, out chan []byte) {
	for {
		select {
		case data := <-out:
			buf := make([]byte, 2)
			binary.BigEndian.PutUint16(buf[0:2], uint16(len(data)))
			buf = append(buf, data...)
			_, err := conn.Write(buf)
			if err != nil {
				in <- []byte{}
			}
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>golangTraining seeing address<commit_after>package main

import "fmt"

func zero(z int) {
	fmt.Printf("%p\n", &z)
	fmt.Println(&z)

	z = 0
}

func main() {
	x := 5
	fmt.Printf("%p\n", &x)
	fmt.Println(&x)
	zero(x)
	fmt.Println(x) // still
}<|endoftext|>


<commit_before>package progress

import (
	"errors"
	"fmt"
	"io"
	"os"
	"os/signal"
	"time"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/api/types/swarm"
	"github.com/docker/docker/client"
	"github.com/docker/docker/pkg/progress"
	"github.com/docker/docker/pkg/streamformatter"
	"github.com/docker/docker/pkg/stringid"
	"golang.org/x/net/context"
)

var (
	numberedStates = map[swarm.TaskState]int64{
		swarm.TaskStateNew:       1,
		swarm.TaskStateAllocated: 2,
		swarm.TaskStatePending:   3,
		swarm.TaskStateAssigned:  4,
		swarm.TaskStateAccepted:  5,
		swarm.TaskStatePreparing: 6,
		swarm.TaskStateReady:     7,
		swarm.TaskStateStarting:  8,
		swarm.TaskStateRunning:   9,
	}

	longestState int
)

const (
	maxProgress     = 9
	maxProgressBars = 20
)

type progressUpdater interface {
	update(service swarm.Service, tasks []swarm.Task, activeNodes map[string]swarm.Node, rollback bool) (bool, error)
}

func init() {
	for state := range numberedStates {
		if len(state) > longestState {
			longestState = len(state)
		}
	}
}

func stateToProgress(state swarm.TaskState, rollback bool) int64 {
	if !rollback {
		return numberedStates[state]
	}
	return int64(len(numberedStates)) - numberedStates[state]
}

// ServiceProgress outputs progress information for convergence of a service.
// nolint: gocyclo
func ServiceProgress(ctx context.Context, client client.APIClient, serviceID string, progressWriter io.WriteCloser) error {
	defer progressWriter.Close()

	progressOut := streamformatter.NewJSONProgressOutput(progressWriter, false)

	sigint := make(chan os.Signal, 1)
	signal.Notify(sigint, os.Interrupt)
	defer signal.Stop(sigint)

	taskFilter := filters.NewArgs()
	taskFilter.Add("service", serviceID)
	taskFilter.Add("_up-to-date", "true")

	getUpToDateTasks := func() ([]swarm.Task, error) {
		return client.TaskList(ctx, types.TaskListOptions{Filters: taskFilter})
	}

	var (
		updater     progressUpdater
		converged   bool
		convergedAt time.Time
		monitor     = 5 * time.Second
		rollback    bool
	)

	for {
		service, _, err := client.ServiceInspectWithRaw(ctx, serviceID, types.ServiceInspectOptions{})
		if err != nil {
			return err
		}

		if service.Spec.UpdateConfig != nil && service.Spec.UpdateConfig.Monitor != 0 {
			monitor = service.Spec.UpdateConfig.Monitor
		}

		if updater == nil {
			updater, err = initializeUpdater(service, progressOut)
			if err != nil {
				return err
			}
		}

		if service.UpdateStatus != nil {
			switch service.UpdateStatus.State {
			case swarm.UpdateStateUpdating:
				rollback = false
			case swarm.UpdateStateCompleted:
				if !converged {
					return nil
				}
			case swarm.UpdateStatePaused:
				return fmt.Errorf("service update paused: %s", service.UpdateStatus.Message)
			case swarm.UpdateStateRollbackStarted:
				if !rollback && service.UpdateStatus.Message != "" {
					progressOut.WriteProgress(progress.Progress{
						ID:     "rollback",
						Action: service.UpdateStatus.Message,
					})
				}
				rollback = true
			case swarm.UpdateStateRollbackPaused:
				return fmt.Errorf("service rollback paused: %s", service.UpdateStatus.Message)
			case swarm.UpdateStateRollbackCompleted:
				if !converged {
					return fmt.Errorf("service rolled back: %s", service.UpdateStatus.Message)
				}
			}
		}
		if converged && time.Since(convergedAt) >= monitor {
			return nil
		}

		tasks, err := getUpToDateTasks()
		if err != nil {
			return err
		}

		activeNodes, err := getActiveNodes(ctx, client)
		if err != nil {
			return err
		}

		converged, err = updater.update(service, tasks, activeNodes, rollback)
		if err != nil {
			return err
		}
		if converged {
			if convergedAt.IsZero() {
				convergedAt = time.Now()
			}
			wait := monitor - time.Since(convergedAt)
			if wait >= 0 {
				progressOut.WriteProgress(progress.Progress{
					// Ideally this would have no ID, but
					// the progress rendering code behaves
					// poorly on an "action" with no ID. It
					// returns the cursor to the beginning
					// of the line, so the first character
					// may be difficult to read. Then the
					// output is overwritten by the shell
					// prompt when the command finishes.
					ID:     "verify",
					Action: fmt.Sprintf("Waiting %d seconds to verify that tasks are stable...", wait/time.Second+1),
				})
			}
		} else {
			if !convergedAt.IsZero() {
				progressOut.WriteProgress(progress.Progress{
					ID:     "verify",
					Action: "Detected task failure",
				})
			}
			convergedAt = time.Time{}
		}

		select {
		case <-time.After(200 * time.Millisecond):
		case <-sigint:
			if !converged {
				progress.Message(progressOut, "", "Operation continuing in background.")
				progress.Messagef(progressOut, "", "Use `docker service ps %s` to check progress.", serviceID)
			}
			return nil
		}
	}
}

func getActiveNodes(ctx context.Context, client client.APIClient) (map[string]swarm.Node, error) {
	nodes, err := client.NodeList(ctx, types.NodeListOptions{})
	if err != nil {
		return nil, err
	}

	activeNodes := make(map[string]swarm.Node)
	for _, n := range nodes {
		if n.Status.State != swarm.NodeStateDown {
			activeNodes[n.ID] = n
		}
	}
	return activeNodes, nil
}

func initializeUpdater(service swarm.Service, progressOut progress.Output) (progressUpdater, error) {
	if service.Spec.Mode.Replicated != nil && service.Spec.Mode.Replicated.Replicas != nil {
		return &replicatedProgressUpdater{
			progressOut: progressOut,
		}, nil
	}
	if service.Spec.Mode.Global != nil {
		return &globalProgressUpdater{
			progressOut: progressOut,
		}, nil
	}
	return nil, errors.New("unrecognized service mode")
}

func writeOverallProgress(progressOut progress.Output, numerator, denominator int, rollback bool) {
	if rollback {
		progressOut.WriteProgress(progress.Progress{
			ID:     "overall progress",
			Action: fmt.Sprintf("rolling back update: %d out of %d tasks", numerator, denominator),
		})
		return
	}
	progressOut.WriteProgress(progress.Progress{
		ID:     "overall progress",
		Action: fmt.Sprintf("%d out of %d tasks", numerator, denominator),
	})
}

type replicatedProgressUpdater struct {
	progressOut progress.Output

	// used for mapping slots to a contiguous space
	// this also causes progress bars to appear in order
	slotMap map[int]int

	initialized bool
	done        bool
}

// nolint: gocyclo
func (u *replicatedProgressUpdater) update(service swarm.Service, tasks []swarm.Task, activeNodes map[string]swarm.Node, rollback bool) (bool, error) {
	if service.Spec.Mode.Replicated == nil || service.Spec.Mode.Replicated.Replicas == nil {
		return false, errors.New("no replica count")
	}
	replicas := *service.Spec.Mode.Replicated.Replicas

	if !u.initialized {
		u.slotMap = make(map[int]int)

		// Draw progress bars in order
		writeOverallProgress(u.progressOut, 0, int(replicas), rollback)

		if replicas <= maxProgressBars {
			for i := uint64(1); i <= replicas; i++ {
				progress.Update(u.progressOut, fmt.Sprintf("%d/%d", i, replicas), " ")
			}
		}
		u.initialized = true
	}

	// If there are multiple tasks with the same slot number, favor the one
	// with the *lowest* desired state. This can happen in restart
	// scenarios.
	tasksBySlot := make(map[int]swarm.Task)
	for _, task := range tasks {
		if numberedStates[task.DesiredState] == 0 {
			continue
		}
		if existingTask, ok := tasksBySlot[task.Slot]; ok {
			if numberedStates[existingTask.DesiredState] <= numberedStates[task.DesiredState] {
				continue
			}
		}
		if task.NodeID != "" {
			if _, nodeActive := activeNodes[task.NodeID]; nodeActive {
				tasksBySlot[task.Slot] = task
			}
		} else {
			tasksBySlot[task.Slot] = task
		}
	}

	// If we had reached a converged state, check if we are still converged.
	if u.done {
		for _, task := range tasksBySlot {
			if task.Status.State != swarm.TaskStateRunning {
				u.done = false
				break
			}
		}
	}

	running := uint64(0)

	for _, task := range tasksBySlot {
		mappedSlot := u.slotMap[task.Slot]
		if mappedSlot == 0 {
			mappedSlot = len(u.slotMap) + 1
			u.slotMap[task.Slot] = mappedSlot
		}

		if !u.done && replicas <= maxProgressBars && uint64(mappedSlot) <= replicas {
			u.progressOut.WriteProgress(progress.Progress{
				ID:         fmt.Sprintf("%d/%d", mappedSlot, replicas),
				Action:     fmt.Sprintf("%-[1]*s", longestState, task.Status.State),
				Current:    stateToProgress(task.Status.State, rollback),
				Total:      maxProgress,
				HideCounts: true,
			})
		}
		if task.Status.State == swarm.TaskStateRunning {
			running++
		}
	}

	if !u.done {
		writeOverallProgress(u.progressOut, int(running), int(replicas), rollback)

		if running == replicas {
			u.done = true
		}
	}

	return running == replicas, nil
}

type globalProgressUpdater struct {
	progressOut progress.Output

	initialized bool
	done        bool
}

func (u *globalProgressUpdater) update(service swarm.Service, tasks []swarm.Task, activeNodes map[string]swarm.Node, rollback bool) (bool, error) {
	// If there are multiple tasks with the same node ID, favor the one
	// with the *lowest* desired state. This can happen in restart
	// scenarios.
	tasksByNode := make(map[string]swarm.Task)
	for _, task := range tasks {
		if numberedStates[task.DesiredState] == 0 {
			continue
		}
		if existingTask, ok := tasksByNode[task.NodeID]; ok {
			if numberedStates[existingTask.DesiredState] <= numberedStates[task.DesiredState] {
				continue
			}
		}
		tasksByNode[task.NodeID] = task
	}

	// We don't have perfect knowledge of how many nodes meet the
	// constraints for this service. But the orchestrator creates tasks
	// for all eligible nodes at the same time, so we should see all those
	// nodes represented among the up-to-date tasks.
	nodeCount := len(tasksByNode)

	if !u.initialized {
		if nodeCount == 0 {
			// Two possibilities: either the orchestrator hasn't created
			// the tasks yet, or the service doesn't meet constraints for
			// any node. Either way, we wait.
			u.progressOut.WriteProgress(progress.Progress{
				ID:     "overall progress",
				Action: "waiting for new tasks",
			})
			return false, nil
		}

		writeOverallProgress(u.progressOut, 0, nodeCount, rollback)
		u.initialized = true
	}

	// If we had reached a converged state, check if we are still converged.
	if u.done {
		for _, task := range tasksByNode {
			if task.Status.State != swarm.TaskStateRunning {
				u.done = false
				break
			}
		}
	}

	running := 0

	for _, task := range tasksByNode {
		if node, nodeActive := activeNodes[task.NodeID]; nodeActive {
			if !u.done && nodeCount <= maxProgressBars {
				u.progressOut.WriteProgress(progress.Progress{
					ID:         stringid.TruncateID(node.ID),
					Action:     fmt.Sprintf("%-[1]*s", longestState, task.Status.State),
					Current:    stateToProgress(task.Status.State, rollback),
					Total:      maxProgress,
					HideCounts: true,
				})
			}
			if task.Status.State == swarm.TaskStateRunning {
				running++
			}
		}
	}

	if !u.done {
		writeOverallProgress(u.progressOut, running, nodeCount, rollback)

		if running == nodeCount {
			u.done = true
		}
	}

	return running == nodeCount, nil
}
<commit_msg>progress: Show success message when finished counting down<commit_after>package progress

import (
	"errors"
	"fmt"
	"io"
	"os"
	"os/signal"
	"time"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/api/types/swarm"
	"github.com/docker/docker/client"
	"github.com/docker/docker/pkg/progress"
	"github.com/docker/docker/pkg/streamformatter"
	"github.com/docker/docker/pkg/stringid"
	"golang.org/x/net/context"
)

var (
	numberedStates = map[swarm.TaskState]int64{
		swarm.TaskStateNew:       1,
		swarm.TaskStateAllocated: 2,
		swarm.TaskStatePending:   3,
		swarm.TaskStateAssigned:  4,
		swarm.TaskStateAccepted:  5,
		swarm.TaskStatePreparing: 6,
		swarm.TaskStateReady:     7,
		swarm.TaskStateStarting:  8,
		swarm.TaskStateRunning:   9,
	}

	longestState int
)

const (
	maxProgress     = 9
	maxProgressBars = 20
)

type progressUpdater interface {
	update(service swarm.Service, tasks []swarm.Task, activeNodes map[string]swarm.Node, rollback bool) (bool, error)
}

func init() {
	for state := range numberedStates {
		if len(state) > longestState {
			longestState = len(state)
		}
	}
}

func stateToProgress(state swarm.TaskState, rollback bool) int64 {
	if !rollback {
		return numberedStates[state]
	}
	return int64(len(numberedStates)) - numberedStates[state]
}

// ServiceProgress outputs progress information for convergence of a service.
// nolint: gocyclo
func ServiceProgress(ctx context.Context, client client.APIClient, serviceID string, progressWriter io.WriteCloser) error {
	defer progressWriter.Close()

	progressOut := streamformatter.NewJSONProgressOutput(progressWriter, false)

	sigint := make(chan os.Signal, 1)
	signal.Notify(sigint, os.Interrupt)
	defer signal.Stop(sigint)

	taskFilter := filters.NewArgs()
	taskFilter.Add("service", serviceID)
	taskFilter.Add("_up-to-date", "true")

	getUpToDateTasks := func() ([]swarm.Task, error) {
		return client.TaskList(ctx, types.TaskListOptions{Filters: taskFilter})
	}

	var (
		updater     progressUpdater
		converged   bool
		convergedAt time.Time
		monitor     = 5 * time.Second
		rollback    bool
	)

	for {
		service, _, err := client.ServiceInspectWithRaw(ctx, serviceID, types.ServiceInspectOptions{})
		if err != nil {
			return err
		}

		if service.Spec.UpdateConfig != nil && service.Spec.UpdateConfig.Monitor != 0 {
			monitor = service.Spec.UpdateConfig.Monitor
		}

		if updater == nil {
			updater, err = initializeUpdater(service, progressOut)
			if err != nil {
				return err
			}
		}

		if service.UpdateStatus != nil {
			switch service.UpdateStatus.State {
			case swarm.UpdateStateUpdating:
				rollback = false
			case swarm.UpdateStateCompleted:
				if !converged {
					return nil
				}
			case swarm.UpdateStatePaused:
				return fmt.Errorf("service update paused: %s", service.UpdateStatus.Message)
			case swarm.UpdateStateRollbackStarted:
				if !rollback && service.UpdateStatus.Message != "" {
					progressOut.WriteProgress(progress.Progress{
						ID:     "rollback",
						Action: service.UpdateStatus.Message,
					})
				}
				rollback = true
			case swarm.UpdateStateRollbackPaused:
				return fmt.Errorf("service rollback paused: %s", service.UpdateStatus.Message)
			case swarm.UpdateStateRollbackCompleted:
				if !converged {
					return fmt.Errorf("service rolled back: %s", service.UpdateStatus.Message)
				}
			}
		}
		if converged && time.Since(convergedAt) >= monitor {
			progressOut.WriteProgress(progress.Progress{
				ID:     "verify",
				Action: "Service converged",
			})

			return nil
		}

		tasks, err := getUpToDateTasks()
		if err != nil {
			return err
		}

		activeNodes, err := getActiveNodes(ctx, client)
		if err != nil {
			return err
		}

		converged, err = updater.update(service, tasks, activeNodes, rollback)
		if err != nil {
			return err
		}
		if converged {
			if convergedAt.IsZero() {
				convergedAt = time.Now()
			}
			wait := monitor - time.Since(convergedAt)
			if wait >= 0 {
				progressOut.WriteProgress(progress.Progress{
					// Ideally this would have no ID, but
					// the progress rendering code behaves
					// poorly on an "action" with no ID. It
					// returns the cursor to the beginning
					// of the line, so the first character
					// may be difficult to read. Then the
					// output is overwritten by the shell
					// prompt when the command finishes.
					ID:     "verify",
					Action: fmt.Sprintf("Waiting %d seconds to verify that tasks are stable...", wait/time.Second+1),
				})
			}
		} else {
			if !convergedAt.IsZero() {
				progressOut.WriteProgress(progress.Progress{
					ID:     "verify",
					Action: "Detected task failure",
				})
			}
			convergedAt = time.Time{}
		}

		select {
		case <-time.After(200 * time.Millisecond):
		case <-sigint:
			if !converged {
				progress.Message(progressOut, "", "Operation continuing in background.")
				progress.Messagef(progressOut, "", "Use `docker service ps %s` to check progress.", serviceID)
			}
			return nil
		}
	}
}

func getActiveNodes(ctx context.Context, client client.APIClient) (map[string]swarm.Node, error) {
	nodes, err := client.NodeList(ctx, types.NodeListOptions{})
	if err != nil {
		return nil, err
	}

	activeNodes := make(map[string]swarm.Node)
	for _, n := range nodes {
		if n.Status.State != swarm.NodeStateDown {
			activeNodes[n.ID] = n
		}
	}
	return activeNodes, nil
}

func initializeUpdater(service swarm.Service, progressOut progress.Output) (progressUpdater, error) {
	if service.Spec.Mode.Replicated != nil && service.Spec.Mode.Replicated.Replicas != nil {
		return &replicatedProgressUpdater{
			progressOut: progressOut,
		}, nil
	}
	if service.Spec.Mode.Global != nil {
		return &globalProgressUpdater{
			progressOut: progressOut,
		}, nil
	}
	return nil, errors.New("unrecognized service mode")
}

func writeOverallProgress(progressOut progress.Output, numerator, denominator int, rollback bool) {
	if rollback {
		progressOut.WriteProgress(progress.Progress{
			ID:     "overall progress",
			Action: fmt.Sprintf("rolling back update: %d out of %d tasks", numerator, denominator),
		})
		return
	}
	progressOut.WriteProgress(progress.Progress{
		ID:     "overall progress",
		Action: fmt.Sprintf("%d out of %d tasks", numerator, denominator),
	})
}

type replicatedProgressUpdater struct {
	progressOut progress.Output

	// used for mapping slots to a contiguous space
	// this also causes progress bars to appear in order
	slotMap map[int]int

	initialized bool
	done        bool
}

// nolint: gocyclo
func (u *replicatedProgressUpdater) update(service swarm.Service, tasks []swarm.Task, activeNodes map[string]swarm.Node, rollback bool) (bool, error) {
	if service.Spec.Mode.Replicated == nil || service.Spec.Mode.Replicated.Replicas == nil {
		return false, errors.New("no replica count")
	}
	replicas := *service.Spec.Mode.Replicated.Replicas

	if !u.initialized {
		u.slotMap = make(map[int]int)

		// Draw progress bars in order
		writeOverallProgress(u.progressOut, 0, int(replicas), rollback)

		if replicas <= maxProgressBars {
			for i := uint64(1); i <= replicas; i++ {
				progress.Update(u.progressOut, fmt.Sprintf("%d/%d", i, replicas), " ")
			}
		}
		u.initialized = true
	}

	// If there are multiple tasks with the same slot number, favor the one
	// with the *lowest* desired state. This can happen in restart
	// scenarios.
	tasksBySlot := make(map[int]swarm.Task)
	for _, task := range tasks {
		if numberedStates[task.DesiredState] == 0 {
			continue
		}
		if existingTask, ok := tasksBySlot[task.Slot]; ok {
			if numberedStates[existingTask.DesiredState] <= numberedStates[task.DesiredState] {
				continue
			}
		}
		if task.NodeID != "" {
			if _, nodeActive := activeNodes[task.NodeID]; nodeActive {
				tasksBySlot[task.Slot] = task
			}
		} else {
			tasksBySlot[task.Slot] = task
		}
	}

	// If we had reached a converged state, check if we are still converged.
	if u.done {
		for _, task := range tasksBySlot {
			if task.Status.State != swarm.TaskStateRunning {
				u.done = false
				break
			}
		}
	}

	running := uint64(0)

	for _, task := range tasksBySlot {
		mappedSlot := u.slotMap[task.Slot]
		if mappedSlot == 0 {
			mappedSlot = len(u.slotMap) + 1
			u.slotMap[task.Slot] = mappedSlot
		}

		if !u.done && replicas <= maxProgressBars && uint64(mappedSlot) <= replicas {
			u.progressOut.WriteProgress(progress.Progress{
				ID:         fmt.Sprintf("%d/%d", mappedSlot, replicas),
				Action:     fmt.Sprintf("%-[1]*s", longestState, task.Status.State),
				Current:    stateToProgress(task.Status.State, rollback),
				Total:      maxProgress,
				HideCounts: true,
			})
		}
		if task.Status.State == swarm.TaskStateRunning {
			running++
		}
	}

	if !u.done {
		writeOverallProgress(u.progressOut, int(running), int(replicas), rollback)

		if running == replicas {
			u.done = true
		}
	}

	return running == replicas, nil
}

type globalProgressUpdater struct {
	progressOut progress.Output

	initialized bool
	done        bool
}

func (u *globalProgressUpdater) update(service swarm.Service, tasks []swarm.Task, activeNodes map[string]swarm.Node, rollback bool) (bool, error) {
	// If there are multiple tasks with the same node ID, favor the one
	// with the *lowest* desired state. This can happen in restart
	// scenarios.
	tasksByNode := make(map[string]swarm.Task)
	for _, task := range tasks {
		if numberedStates[task.DesiredState] == 0 {
			continue
		}
		if existingTask, ok := tasksByNode[task.NodeID]; ok {
			if numberedStates[existingTask.DesiredState] <= numberedStates[task.DesiredState] {
				continue
			}
		}
		tasksByNode[task.NodeID] = task
	}

	// We don't have perfect knowledge of how many nodes meet the
	// constraints for this service. But the orchestrator creates tasks
	// for all eligible nodes at the same time, so we should see all those
	// nodes represented among the up-to-date tasks.
	nodeCount := len(tasksByNode)

	if !u.initialized {
		if nodeCount == 0 {
			// Two possibilities: either the orchestrator hasn't created
			// the tasks yet, or the service doesn't meet constraints for
			// any node. Either way, we wait.
			u.progressOut.WriteProgress(progress.Progress{
				ID:     "overall progress",
				Action: "waiting for new tasks",
			})
			return false, nil
		}

		writeOverallProgress(u.progressOut, 0, nodeCount, rollback)
		u.initialized = true
	}

	// If we had reached a converged state, check if we are still converged.
	if u.done {
		for _, task := range tasksByNode {
			if task.Status.State != swarm.TaskStateRunning {
				u.done = false
				break
			}
		}
	}

	running := 0

	for _, task := range tasksByNode {
		if node, nodeActive := activeNodes[task.NodeID]; nodeActive {
			if !u.done && nodeCount <= maxProgressBars {
				u.progressOut.WriteProgress(progress.Progress{
					ID:         stringid.TruncateID(node.ID),
					Action:     fmt.Sprintf("%-[1]*s", longestState, task.Status.State),
					Current:    stateToProgress(task.Status.State, rollback),
					Total:      maxProgress,
					HideCounts: true,
				})
			}
			if task.Status.State == swarm.TaskStateRunning {
				running++
			}
		}
	}

	if !u.done {
		writeOverallProgress(u.progressOut, running, nodeCount, rollback)

		if running == nodeCount {
			u.done = true
		}
	}

	return running == nodeCount, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 Ole Krüger.
// Licensed under the MIT license which can be found in the LICENSE file.

package dpt

import (
	"fmt"
)

// DPT_7001 represents DPT 7.001 / Value 2 Ucount.
type DPT_7001 uint16

func (d DPT_7001) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7001) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7001) Unit() string {
	return "pulses"
}

func (d DPT_7001) String() string {
	return fmt.Sprintf("%d pulses", uint16(d))
}

// DPT_7002 represents DPT 7.002 / Time Period MSec.
type DPT_7002 uint16

func (d DPT_7002) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7002) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7002) Unit() string {
	return "ms"
}

func (d DPT_7002) String() string {
	return fmt.Sprintf("%d ms", uint16(d))
}

// DPT_7003 represents DPT 7.003 / Time Period 10 MSec.
type DPT_7003 uint16

func (d DPT_7003) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7003) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7003) Unit() string {
	return "s"
}

func (d DPT_7003) String() string {
	return fmt.Sprintf("%d s", uint16(d))
}

// DPT_7004 represents DPT 7.004 / Time Period 100 MSec.
type DPT_7004 uint16

func (d DPT_7004) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7004) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7004) Unit() string {
	return "s"
}

func (d DPT_7004) String() string {
	return fmt.Sprintf("%d s", uint16(d))
}

// DPT_7005 represents DPT 7.005 / Time Period Sec.
type DPT_7005 uint16

func (d DPT_7005) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7005) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7005) Unit() string {
	return "s"
}

func (d DPT_7005) String() string {
	return fmt.Sprintf("%d s", uint16(d))
}

// DPT_7006 represents DPT 7.006 / Time Period Min.
type DPT_7006 uint16

func (d DPT_7006) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7006) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7006) Unit() string {
	return "m"
}

func (d DPT_7006) String() string {
	return fmt.Sprintf("%d m", uint16(d))
}

// DPT_7007 represents DPT 7.007 / Time Period Hrs.
type DPT_7007 uint16

func (d DPT_7007) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7007) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7007) Unit() string {
	return "h"
}

func (d DPT_7007) String() string {
	return fmt.Sprintf("%d h", uint16(d))
}

// DPT_7010 represents DPT 7.010 / Property DataType.
type DPT_7010 uint16

func (d DPT_7010) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7010) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7010) Unit() string {
	return ""
}

func (d DPT_7010) String() string {
	return fmt.Sprintf("%d", uint16(d))
}

// DPT_7011 represents DPT 7.011 / Length mm.
type DPT_7011 uint16

func (d DPT_7011) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7011) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7011) Unit() string {
	return "mm"
}

func (d DPT_7011) String() string {
	return fmt.Sprintf("%d mm", uint16(d))
}

// DPT_7012 represents DPT 7.012 / Current mA.
type DPT_7012 uint16

func (d DPT_7012) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7012) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7012) Unit() string {
	return "mA"
}

func (d DPT_7012) String() string {
	return fmt.Sprintf("%d mA", uint16(d))
}

// DPT_7013 represents DPT 7.013 / Brightness lux.
type DPT_7013 uint16

func (d DPT_7013) Pack() []byte {
 return packU16(uint16(d))
}

func (d *DPT_7013) Unpack(data []byte) error {
 return unpackU16(data, (*uint16)(d))
}

func (d DPT_7013) Unit() string {
 return "lux"
}

func (d DPT_7013) String() string {
 return fmt.Sprintf("%d lux", uint16(d))
}
<commit_msg>Apply Go formatter<commit_after>// Copyright 2017 Ole Krüger.
// Licensed under the MIT license which can be found in the LICENSE file.

package dpt

import (
	"fmt"
)

// DPT_7001 represents DPT 7.001 / Value 2 Ucount.
type DPT_7001 uint16

func (d DPT_7001) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7001) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7001) Unit() string {
	return "pulses"
}

func (d DPT_7001) String() string {
	return fmt.Sprintf("%d pulses", uint16(d))
}

// DPT_7002 represents DPT 7.002 / Time Period MSec.
type DPT_7002 uint16

func (d DPT_7002) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7002) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7002) Unit() string {
	return "ms"
}

func (d DPT_7002) String() string {
	return fmt.Sprintf("%d ms", uint16(d))
}

// DPT_7003 represents DPT 7.003 / Time Period 10 MSec.
type DPT_7003 uint16

func (d DPT_7003) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7003) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7003) Unit() string {
	return "s"
}

func (d DPT_7003) String() string {
	return fmt.Sprintf("%d s", uint16(d))
}

// DPT_7004 represents DPT 7.004 / Time Period 100 MSec.
type DPT_7004 uint16

func (d DPT_7004) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7004) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7004) Unit() string {
	return "s"
}

func (d DPT_7004) String() string {
	return fmt.Sprintf("%d s", uint16(d))
}

// DPT_7005 represents DPT 7.005 / Time Period Sec.
type DPT_7005 uint16

func (d DPT_7005) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7005) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7005) Unit() string {
	return "s"
}

func (d DPT_7005) String() string {
	return fmt.Sprintf("%d s", uint16(d))
}

// DPT_7006 represents DPT 7.006 / Time Period Min.
type DPT_7006 uint16

func (d DPT_7006) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7006) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7006) Unit() string {
	return "m"
}

func (d DPT_7006) String() string {
	return fmt.Sprintf("%d m", uint16(d))
}

// DPT_7007 represents DPT 7.007 / Time Period Hrs.
type DPT_7007 uint16

func (d DPT_7007) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7007) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7007) Unit() string {
	return "h"
}

func (d DPT_7007) String() string {
	return fmt.Sprintf("%d h", uint16(d))
}

// DPT_7010 represents DPT 7.010 / Property DataType.
type DPT_7010 uint16

func (d DPT_7010) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7010) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7010) Unit() string {
	return ""
}

func (d DPT_7010) String() string {
	return fmt.Sprintf("%d", uint16(d))
}

// DPT_7011 represents DPT 7.011 / Length mm.
type DPT_7011 uint16

func (d DPT_7011) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7011) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7011) Unit() string {
	return "mm"
}

func (d DPT_7011) String() string {
	return fmt.Sprintf("%d mm", uint16(d))
}

// DPT_7012 represents DPT 7.012 / Current mA.
type DPT_7012 uint16

func (d DPT_7012) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7012) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7012) Unit() string {
	return "mA"
}

func (d DPT_7012) String() string {
	return fmt.Sprintf("%d mA", uint16(d))
}

// DPT_7013 represents DPT 7.013 / Brightness lux.
type DPT_7013 uint16

func (d DPT_7013) Pack() []byte {
	return packU16(uint16(d))
}

func (d *DPT_7013) Unpack(data []byte) error {
	return unpackU16(data, (*uint16)(d))
}

func (d DPT_7013) Unit() string {
	return "lux"
}

func (d DPT_7013) String() string {
	return fmt.Sprintf("%d lux", uint16(d))
}
<|endoftext|>


<commit_before>// Copyright (c) 2021 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package zapcore

import (
	"bufio"
	"sync"
	"time"

	"go.uber.org/multierr"
)

const (
	// _defaultBufferSize specifies the default size used by Buffer.
	_defaultBufferSize = 256 * 1024 // 256 kB

	// _defaultFlushInterval specifies the default flush interval for
	// Buffer.
	_defaultFlushInterval = 30 * time.Second
)

// A BufferedWriteSyncer is a WriteSyncer that buffers writes in-memory before
// flushing them to a wrapped WriteSyncer after reaching some limit, or at some
// fixed interval--whichever comes first.
//
// BufferedWriteSyncer is safe for concurrent use. You don't need to use
// zapcore.Lock for WriteSyncers with BufferedWriteSyncer.
type BufferedWriteSyncer struct {
	// WS is the WriteSyncer around which BufferedWriteSyncer will buffer
	// writes.
	//
	// This field is required.
	WS WriteSyncer

	// Size specifies the maximum amount of data the writer will buffered
	// before flushing.
	//
	// Defaults to 256 kB if unspecified.
	Size int

	// FlushInterval specifies how often the writer should flush data if
	// there have been no writes.
	//
	// Defaults to 30 seconds if unspecified.
	FlushInterval time.Duration

	// Clock, if specified, provides control of the source of time for the
	// writer.
	//
	// Defaults to the system clock.
	Clock Clock

	// unexported fields for state
	mu          sync.Mutex
	initialized bool // whether initialize() has run
	stopped     bool // whether Stop() has run
	writer      *bufio.Writer
	ticker      *time.Ticker
	stop        chan struct{} // closed when flushLoop should stop
	done        chan struct{} // closed when flushLoop has stopped
}

func (s *BufferedWriteSyncer) initialize() {
	size := s.Size
	if size == 0 {
		size = _defaultBufferSize
	}

	flushInterval := s.FlushInterval
	if flushInterval == 0 {
		flushInterval = _defaultFlushInterval
	}

	if s.Clock == nil {
		s.Clock = DefaultClock
	}

	s.ticker = s.Clock.NewTicker(flushInterval)
	s.writer = bufio.NewWriterSize(s.WS, size)
	s.stop = make(chan struct{})
	s.done = make(chan struct{})
	s.initialized = true
	go s.flushLoop()
}

// Write writes log data into buffer syncer directly, multiple Write calls will be batched,
// and log data will be flushed to disk when the buffer is full or periodically.
func (s *BufferedWriteSyncer) Write(bs []byte) (int, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	if !s.initialized {
		s.initialize()
	}

	// To avoid partial writes from being flushed, we manually flush the existing buffer if:
	// * The current write doesn't fit into the buffer fully, and
	// * The buffer is not empty (since bufio will not split large writes when the buffer is empty)
	if len(bs) > s.writer.Available() && s.writer.Buffered() > 0 {
		if err := s.writer.Flush(); err != nil {
			return 0, err
		}
	}

	return s.writer.Write(bs)
}

// Sync flushes buffered log data into disk directly.
func (s *BufferedWriteSyncer) Sync() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	var err error
	if s.initialized {
		err = s.writer.Flush()
	}

	return multierr.Append(err, s.WS.Sync())
}

// flushLoop flushes the buffer at the configured interval until Stop is
// called.
func (s *BufferedWriteSyncer) flushLoop() {
	defer close(s.done)

	for {
		select {
		case <-s.ticker.C:
			// we just simply ignore error here
			// because the underlying bufio writer stores any errors
			// and we return any error from Sync() as part of the close
			_ = s.Sync()
		case <-s.stop:
			return
		}
	}
}

// Stop closes the buffer, cleans up background goroutines, and flushes
// remaining unwritten data.
func (s *BufferedWriteSyncer) Stop() (err error) {
	var stopped bool

	// Critical section.
	func() {
		s.mu.Lock()
		defer s.mu.Unlock()

		if !s.initialized {
			return
		}

		stopped = s.stopped
		if stopped {
			return
		}
		s.stopped = true

		s.ticker.Stop()
		close(s.stop) // tell flushLoop to stop
		<-s.done      // and wait until it has
	}()

	// Don't call Sync on consecutive Stops.
	if !stopped {
		err = s.Sync()
	}

	return err
}
<commit_msg>zapcore/BufferedWriteSyncer: More docs (#1139)<commit_after>// Copyright (c) 2021 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package zapcore

import (
	"bufio"
	"sync"
	"time"

	"go.uber.org/multierr"
)

const (
	// _defaultBufferSize specifies the default size used by Buffer.
	_defaultBufferSize = 256 * 1024 // 256 kB

	// _defaultFlushInterval specifies the default flush interval for
	// Buffer.
	_defaultFlushInterval = 30 * time.Second
)

// A BufferedWriteSyncer is a WriteSyncer that buffers writes in-memory before
// flushing them to a wrapped WriteSyncer after reaching some limit, or at some
// fixed interval--whichever comes first.
//
// BufferedWriteSyncer is safe for concurrent use. You don't need to use
// zapcore.Lock for WriteSyncers with BufferedWriteSyncer.
//
// To set up a BufferedWriteSyncer, construct a WriteSyncer for your log
// destination (*os.File is a valid WriteSyncer), wrap it with
// BufferedWriteSyncer, and defer a Stop() call for when you no longer need the
// object.
//
//   func main() {
//     ws := ... // your log destination
//     bws := &zapcore.BufferedWriteSyncer{WS: ws}
//     defer bws.Stop()
//
//     // ...
//     core := zapcore.NewCore(enc, bws, lvl)
//     logger := zap.New(core)
//
//     // ...
//  }
//
// By default, a BufferedWriteSyncer will buffer up to 256 kilobytes of logs,
// waiting at most 30 seconds between flushes.
// You can customize these parameters by setting the Size or FlushInterval
// fields.
// For example, the following buffers up to 512 kB of logs before flushing them
// to Stderr, with a maximum of one minute between each flush.
//
//   ws := &BufferedWriteSyncer{
//     WS:            os.Stderr,
//     Size:          512 * 1024, // 512 kB
//     FlushInterval: time.Minute,
//   }
//   defer ws.Stop()
type BufferedWriteSyncer struct {
	// WS is the WriteSyncer around which BufferedWriteSyncer will buffer
	// writes.
	//
	// This field is required.
	WS WriteSyncer

	// Size specifies the maximum amount of data the writer will buffered
	// before flushing.
	//
	// Defaults to 256 kB if unspecified.
	Size int

	// FlushInterval specifies how often the writer should flush data if
	// there have been no writes.
	//
	// Defaults to 30 seconds if unspecified.
	FlushInterval time.Duration

	// Clock, if specified, provides control of the source of time for the
	// writer.
	//
	// Defaults to the system clock.
	Clock Clock

	// unexported fields for state
	mu          sync.Mutex
	initialized bool // whether initialize() has run
	stopped     bool // whether Stop() has run
	writer      *bufio.Writer
	ticker      *time.Ticker
	stop        chan struct{} // closed when flushLoop should stop
	done        chan struct{} // closed when flushLoop has stopped
}

func (s *BufferedWriteSyncer) initialize() {
	size := s.Size
	if size == 0 {
		size = _defaultBufferSize
	}

	flushInterval := s.FlushInterval
	if flushInterval == 0 {
		flushInterval = _defaultFlushInterval
	}

	if s.Clock == nil {
		s.Clock = DefaultClock
	}

	s.ticker = s.Clock.NewTicker(flushInterval)
	s.writer = bufio.NewWriterSize(s.WS, size)
	s.stop = make(chan struct{})
	s.done = make(chan struct{})
	s.initialized = true
	go s.flushLoop()
}

// Write writes log data into buffer syncer directly, multiple Write calls will be batched,
// and log data will be flushed to disk when the buffer is full or periodically.
func (s *BufferedWriteSyncer) Write(bs []byte) (int, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	if !s.initialized {
		s.initialize()
	}

	// To avoid partial writes from being flushed, we manually flush the existing buffer if:
	// * The current write doesn't fit into the buffer fully, and
	// * The buffer is not empty (since bufio will not split large writes when the buffer is empty)
	if len(bs) > s.writer.Available() && s.writer.Buffered() > 0 {
		if err := s.writer.Flush(); err != nil {
			return 0, err
		}
	}

	return s.writer.Write(bs)
}

// Sync flushes buffered log data into disk directly.
func (s *BufferedWriteSyncer) Sync() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	var err error
	if s.initialized {
		err = s.writer.Flush()
	}

	return multierr.Append(err, s.WS.Sync())
}

// flushLoop flushes the buffer at the configured interval until Stop is
// called.
func (s *BufferedWriteSyncer) flushLoop() {
	defer close(s.done)

	for {
		select {
		case <-s.ticker.C:
			// we just simply ignore error here
			// because the underlying bufio writer stores any errors
			// and we return any error from Sync() as part of the close
			_ = s.Sync()
		case <-s.stop:
			return
		}
	}
}

// Stop closes the buffer, cleans up background goroutines, and flushes
// remaining unwritten data.
func (s *BufferedWriteSyncer) Stop() (err error) {
	var stopped bool

	// Critical section.
	func() {
		s.mu.Lock()
		defer s.mu.Unlock()

		if !s.initialized {
			return
		}

		stopped = s.stopped
		if stopped {
			return
		}
		s.stopped = true

		s.ticker.Stop()
		close(s.stop) // tell flushLoop to stop
		<-s.done      // and wait until it has
	}()

	// Don't call Sync on consecutive Stops.
	if !stopped {
		err = s.Sync()
	}

	return err
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2016 Dennis Chen
//
// This file is part of Clashr.
//
// Clashr is free software: you can redistribute it and/or modify it under the
// terms of the GNU Affero General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// Clashr is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
// FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
// more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with Clashr.  If not, see <http://www.gnu.org/licenses/>.
//

package handlers

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"

	"github.com/gorilla/mux"

	"github.com/clashr/go-servr/models"
)

func ChallengeIndex(w http.ResponseWriter, r *http.Request) {
	challenges := new(models.Challenges)
	if err := engine.Find(challenges); err != nil {
		log.Fatalln(err)
	}

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(challenges); err != nil {
		log.Fatalln(err)
	}
}

func ChallengeShow(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	challengeId := vars["challengeId"]
	challenge := new(models.Challenge)
	exists, err := engine.Id(challengeId).Get(challenge)
	if err != nil {
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(http.StatusInternalServerError)
		if err := json.NewEncoder(w).Encode(err); err != nil {
			log.Fatalln(err)
		}
		return
	}
	if !exists {
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(challenge); err != nil {
		log.Fatalln(err)
	}
}

func ChallengeCreate(w http.ResponseWriter, r *http.Request) {
	var challenge models.Challenge
	body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1048576))
	if err != nil {
		log.Fatalln(err)
	}
	if err := r.Body.Close(); err != nil {
		log.Fatalln(err)
	}
	if err := json.Unmarshal(body, &challenge); err != nil {
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(422)
		if err := json.NewEncoder(w).Encode(err); err != nil {
			log.Fatalln(err)
		}
		return
	}

	affected, err := engine.Insert(&challenge)
	if err != nil {
		log.Fatalln(err)
	}
	log.Println(affected)

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.WriteHeader(http.StatusCreated)
	if err := json.NewEncoder(w).Encode(affected); err != nil {
		log.Fatalln(err)
	}
}

func ChallengeUpdate(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	challengeId := vars["challengeId"]
	fmt.Fprintf(w, "Challenge Update:", challengeId)
}

func ChallengeDestroy(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	challengeId := vars["challengeId"]
	fmt.Fprintf(w, "Challenge Destroy:", challengeId)
}
<commit_msg>handlers: update creation to redirect to newly created resource<commit_after>//
// Copyright (c) 2016 Dennis Chen
//
// This file is part of Clashr.
//
// Clashr is free software: you can redistribute it and/or modify it under the
// terms of the GNU Affero General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// Clashr is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
// FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
// more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with Clashr.  If not, see <http://www.gnu.org/licenses/>.
//

package handlers

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"

	"github.com/gorilla/mux"

	"github.com/clashr/go-servr/models"
)

func ChallengeIndex(w http.ResponseWriter, r *http.Request) {
	challenges := new(models.Challenges)
	if err := engine.Find(challenges); err != nil {
		log.Fatalln(err)
	}

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(challenges); err != nil {
		log.Fatalln(err)
	}
}

func ChallengeShow(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	challengeId := vars["challengeId"]
	challenge := new(models.Challenge)
	exists, err := engine.Id(challengeId).Get(challenge)
	if err != nil {
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(http.StatusInternalServerError)
		if err := json.NewEncoder(w).Encode(err); err != nil {
			log.Fatalln(err)
		}
		return
	}
	if !exists {
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(http.StatusNotFound)
		return
	}

	if err := json.NewEncoder(w).Encode(challenge); err != nil {
		log.Fatalln(err)
	}
}

func ChallengeCreate(w http.ResponseWriter, r *http.Request) {
	var challenge models.Challenge
	body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1048576))
	if err != nil {
		log.Fatalln(err)
	}
	if err := r.Body.Close(); err != nil {
		log.Fatalln(err)
	}
	if err := json.Unmarshal(body, &challenge); err != nil {
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(422)
		if err := json.NewEncoder(w).Encode(err); err != nil {
			log.Fatalln(err)
		}
		return
	}

	affected, err := engine.Insert(&challenge)
	if err != nil {
		log.Fatalln(err)
	}

	target := fmt.Sprintf("/challenges/%d", affected)
	http.Redirect(w, r, target, http.StatusSeeOther)
}

func ChallengeUpdate(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	challengeId := vars["challengeId"]
	fmt.Fprintf(w, "Challenge Update:", challengeId)
}

func ChallengeDestroy(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	challengeId := vars["challengeId"]
	fmt.Fprintf(w, "Challenge Destroy:", challengeId)
}
<|endoftext|>


<commit_before>package workflows

import (
	"encoding/base64"
	"fmt"
	"os"
	"strings"

	"github.com/docker/docker/api/types"
	"github.com/pkg/errors"
	"github.com/stelligent/mu/common"
)

type serviceWorkflow struct {
	envStack                      *common.Stack
	lbStack                       *common.Stack
	artifactProvider              common.ArtifactProvider
	serviceName                   string
	serviceTag                    string
	serviceImage                  string
	registryAuth                  string
	registryAuthConfig            map[string]types.AuthConfig
	priority                      int
	codeRevision                  string
	repoName                      string
	appName                       string
	appRevisionBucket             string
	appRevisionKey                string
	databaseName                  string
	cloudFormationRoleArn         string
	microserviceTaskDefinitionArn string
	ecsEventsRoleArn              string
	kubernetesResourceManager     common.KubernetesResourceManager
}

// Find a service in config, by name and set the reference
func (workflow *serviceWorkflow) serviceLoader(ctx *common.Context, tag string, provider string) Executor {
	return func() error {
		err := workflow.serviceInput(ctx, "")()
		if err != nil {
			return err
		}

		// Tag
		if tag != "" {
			workflow.serviceTag = tag
		} else if ctx.Config.Repo.Revision != "" {
			workflow.serviceTag = ctx.Config.Repo.Revision
		} else {
			workflow.serviceTag = "latest"
		}
		workflow.appRevisionKey = fmt.Sprintf("%s/%s.zip", workflow.serviceName, workflow.serviceTag)

		workflow.databaseName = ctx.Config.Service.Database.Name
		workflow.codeRevision = ctx.Config.Repo.Revision
		workflow.repoName = ctx.Config.Repo.Slug
		workflow.priority = ctx.Config.Service.Priority

		if provider == "" {
			dockerfile := ctx.Config.Service.Dockerfile
			if dockerfile == "" {
				dockerfile = "Dockerfile"
			}

			dockerfilePath := fmt.Sprintf("%s/%s", ctx.Config.Basedir, dockerfile)
			log.Debugf("Determining repo provider by checking for existence of '%s'", dockerfilePath)

			if _, err := os.Stat(dockerfilePath); !os.IsNotExist(err) {
				log.Infof("Dockerfile found, assuming ECR pipeline")
				workflow.artifactProvider = common.ArtifactProviderEcr
			} else {
				log.Infof("No Dockerfile found, assuming CodeDeploy pipeline")
				workflow.artifactProvider = common.ArtifactProviderS3
			}
		} else {
			workflow.artifactProvider = common.ArtifactProvider(provider)
		}

		log.Debugf("Working with service:'%s' tag:'%s'", workflow.serviceName, workflow.serviceTag)
		return nil
	}
}

func (workflow *serviceWorkflow) isEcrProvider() Conditional {
	return func() bool {
		return strings.EqualFold(string(workflow.artifactProvider), string(common.ArtifactProviderEcr))
	}
}

func (workflow *serviceWorkflow) isS3Provider() Conditional {
	return func() bool {
		return strings.EqualFold(string(workflow.artifactProvider), string(common.ArtifactProviderS3))
	}
}

func (workflow *serviceWorkflow) isEcsProvider() Conditional {
	return func() bool {
		return strings.EqualFold(string(workflow.envStack.Tags["provider"]), string(common.EnvProviderEcs)) ||
			strings.EqualFold(string(workflow.envStack.Tags["provider"]), string(common.EnvProviderEcsFargate))
	}
}
func (workflow *serviceWorkflow) isEksProvider() Conditional {
	return func() bool {
		return strings.EqualFold(string(workflow.envStack.Tags["provider"]), string(common.EnvProviderEks)) ||
			strings.EqualFold(string(workflow.envStack.Tags["provider"]), string(common.EnvProviderEksFargate))
	}
}
func (workflow *serviceWorkflow) isFargateProvider() Conditional {
	return func() bool {
		return strings.EqualFold(string(workflow.envStack.Tags["provider"]), string(common.EnvProviderEcsFargate))
	}
}

func (workflow *serviceWorkflow) isEc2Provider() Conditional {
	return func() bool {
		return strings.EqualFold(string(workflow.envStack.Tags["provider"]), string(common.EnvProviderEc2))
	}
}

func (workflow *serviceWorkflow) serviceInput(ctx *common.Context, serviceName string) Executor {
	return func() error {
		// Repo Name
		if serviceName != "" {
			workflow.serviceName = serviceName
		} else if ctx.Config.Service.Name != "" {
			workflow.serviceName = ctx.Config.Service.Name
		} else if ctx.Config.Repo.Name != "" {
			workflow.serviceName = ctx.Config.Repo.Name
		} else {
			return errors.New("Service name must be provided")
		}
		return nil
	}
}

func (workflow *serviceWorkflow) serviceRepoUpserter(namespace string, service *common.Service, stackUpserter common.StackUpserter, stackWaiter common.StackWaiter) Executor {
	return func() error {
		if service.ImageRepository != "" {
			log.Noticef("Using repo '%s' for service '%s'", service.ImageRepository, workflow.serviceName)
			workflow.serviceImage = service.ImageRepository
			return nil
		}

		log.Noticef("Upsert repo for service '%s'", workflow.serviceName)

		ecrStackName := common.CreateStackName(namespace, common.StackTypeRepo, workflow.serviceName)

		stackParams := make(map[string]string)
		stackParams["RepoName"] = fmt.Sprintf("%s-%s", namespace, workflow.serviceName)

		tags := createTagMap(&ServiceTags{
			Service:  workflow.serviceName,
			Type:     string(common.StackTypeRepo),
			Provider: "",
			Revision: workflow.codeRevision,
			Repo:     workflow.repoName,
		})

		err := stackUpserter.UpsertStack(ecrStackName, common.TemplateRepo, nil, stackParams, tags, "", "")
		if err != nil {
			return err
		}

		log.Debugf("Waiting for stack '%s' to complete", ecrStackName)
		stack := stackWaiter.AwaitFinalStatus(ecrStackName)
		if stack == nil {
			return fmt.Errorf("Unable to create stack %s", ecrStackName)
		}
		if strings.HasSuffix(stack.Status, "ROLLBACK_COMPLETE") || !strings.HasSuffix(stack.Status, "_COMPLETE") {
			return fmt.Errorf("Ended in failed status %s %s", stack.Status, stack.StatusReason)
		}
		workflow.serviceImage = fmt.Sprintf("%s:%s", stack.Outputs["RepoUrl"], workflow.serviceTag)
		return nil
	}
}
func (workflow *serviceWorkflow) serviceAppUpserter(namespace string, service *common.Service, stackUpserter common.StackUpserter, stackWaiter common.StackWaiter) Executor {
	return func() error {
		log.Noticef("Upsert app for service '%s'", workflow.serviceName)

		appStackName := common.CreateStackName(namespace, common.StackTypeApp, workflow.serviceName)

		stackParams := make(map[string]string)

		tags := createTagMap(&EnvironmentTags{
			Environment: workflow.serviceName,
			Type:        string(common.StackTypeApp),
			Provider:    "",
			Revision:    workflow.codeRevision,
			Repo:        workflow.repoName,
		})

		err := stackUpserter.UpsertStack(appStackName, common.TemplateApp, nil, stackParams, tags, "", workflow.cloudFormationRoleArn)
		if err != nil {
			return err
		}

		log.Debugf("Waiting for stack '%s' to complete", appStackName)
		stack := stackWaiter.AwaitFinalStatus(appStackName)
		if stack == nil {
			return fmt.Errorf("Unable to create stack %s", appStackName)
		}
		if strings.HasSuffix(stack.Status, "ROLLBACK_COMPLETE") || !strings.HasSuffix(stack.Status, "_COMPLETE") {
			return fmt.Errorf("Ended in failed status %s %s", stack.Status, stack.StatusReason)
		}
		workflow.appName = stack.Outputs["ApplicationName"]
		return nil
	}
}

func (workflow *serviceWorkflow) serviceBucketUpserter(namespace string, service *common.Service, stackUpserter common.StackUpserter, stackWaiter common.StackWaiter) Executor {
	return func() error {

		if service.Pipeline.Build.Bucket != "" {
			workflow.appRevisionBucket = service.Pipeline.Build.Bucket
		} else {
			bucketStackName := common.CreateStackName(namespace, common.StackTypeBucket, "codedeploy")
			log.Noticef("Upserting Bucket for CodeDeploy")
			bucketParams := make(map[string]string)
			bucketParams["Namespace"] = namespace
			bucketParams["BucketPrefix"] = "codedeploy"

			tags := createTagMap(&PipelineTags{
				Type: common.StackTypeBucket,
			})

			err := stackUpserter.UpsertStack(bucketStackName, common.TemplateBucket, nil, bucketParams, tags, "", workflow.cloudFormationRoleArn)
			if err != nil {
				return err
			}

			log.Debugf("Waiting for stack '%s' to complete", bucketStackName)
			stack := stackWaiter.AwaitFinalStatus(bucketStackName)
			if stack == nil {
				return fmt.Errorf("Unable to create stack %s", bucketStackName)
			}
			if strings.HasSuffix(stack.Status, "ROLLBACK_COMPLETE") || !strings.HasSuffix(stack.Status, "_COMPLETE") {
				return fmt.Errorf("Ended in failed status %s %s", stack.Status, stack.StatusReason)
			}

			workflow.appRevisionBucket = stack.Outputs["Bucket"]

		}

		return nil
	}
}

func (workflow *serviceWorkflow) serviceRegistryAuthenticator(authenticator common.RepositoryAuthenticator) Executor {
	return func() error {
		log.Debugf("Authenticating to registry '%s'", workflow.serviceImage)
		registryAuth, err := authenticator.AuthenticateRepository(workflow.serviceImage)
		if err != nil {
			return err
		}

		data, err := base64.StdEncoding.DecodeString(registryAuth)
		if err != nil {
			return err
		}

		authParts := strings.Split(string(data), ":")

		workflow.registryAuth = base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("{\"username\":\"%s\", \"password\":\"%s\"}", authParts[0], authParts[1])))

		// ImageBuild pull auth
		var authConfigs2 map[string]types.AuthConfig = make(map[string]types.AuthConfig)
		var serviceImagePart string = strings.Split(workflow.serviceImage, ":")[0]

		authConfigs2[serviceImagePart] = types.AuthConfig{
			Username:      authParts[0],
			Password:      authParts[1],
			ServerAddress: fmt.Sprintf("https://%s", serviceImagePart),
		}

		workflow.registryAuthConfig = authConfigs2

		return nil
	}
}

func (workflow *serviceWorkflow) connectKubernetes(provider common.KubernetesResourceManagerProvider) Executor {
	return func() error {
		clusterName := workflow.envStack.Name
		kubernetesResourceManager, err := provider.GetResourceManager(clusterName)
		workflow.kubernetesResourceManager = kubernetesResourceManager
		return err
	}
}
<commit_msg>cleanup<commit_after>package workflows

import (
	"encoding/base64"
	"fmt"
	"os"
	"strings"

	"github.com/docker/docker/api/types"
	"github.com/pkg/errors"
	"github.com/stelligent/mu/common"
)

type serviceWorkflow struct {
	envStack                      *common.Stack
	lbStack                       *common.Stack
	artifactProvider              common.ArtifactProvider
	serviceName                   string
	serviceTag                    string
	serviceImage                  string
	registryAuth                  string
	registryAuthConfig            map[string]types.AuthConfig
	priority                      int
	codeRevision                  string
	repoName                      string
	appName                       string
	appRevisionBucket             string
	appRevisionKey                string
	databaseName                  string
	cloudFormationRoleArn         string
	microserviceTaskDefinitionArn string
	ecsEventsRoleArn              string
	kubernetesResourceManager     common.KubernetesResourceManager
}

// Find a service in config, by name and set the reference
func (workflow *serviceWorkflow) serviceLoader(ctx *common.Context, tag string, provider string) Executor {
	return func() error {
		err := workflow.serviceInput(ctx, "")()
		if err != nil {
			return err
		}

		// Tag
		if tag != "" {
			workflow.serviceTag = tag
		} else if ctx.Config.Repo.Revision != "" {
			workflow.serviceTag = ctx.Config.Repo.Revision
		} else {
			workflow.serviceTag = "latest"
		}
		workflow.appRevisionKey = fmt.Sprintf("%s/%s.zip", workflow.serviceName, workflow.serviceTag)

		workflow.databaseName = ctx.Config.Service.Database.Name
		workflow.codeRevision = ctx.Config.Repo.Revision
		workflow.repoName = ctx.Config.Repo.Slug
		workflow.priority = ctx.Config.Service.Priority

		if provider == "" {
			dockerfile := ctx.Config.Service.Dockerfile
			if dockerfile == "" {
				dockerfile = "Dockerfile"
			}

			dockerfilePath := fmt.Sprintf("%s/%s", ctx.Config.Basedir, dockerfile)
			log.Debugf("Determining repo provider by checking for existence of '%s'", dockerfilePath)

			if _, err := os.Stat(dockerfilePath); !os.IsNotExist(err) {
				log.Infof("Dockerfile found, assuming ECR pipeline")
				workflow.artifactProvider = common.ArtifactProviderEcr
			} else {
				log.Infof("No Dockerfile found, assuming CodeDeploy pipeline")
				workflow.artifactProvider = common.ArtifactProviderS3
			}
		} else {
			workflow.artifactProvider = common.ArtifactProvider(provider)
		}

		log.Debugf("Working with service:'%s' tag:'%s'", workflow.serviceName, workflow.serviceTag)
		return nil
	}
}

func (workflow *serviceWorkflow) isEcrProvider() Conditional {
	return func() bool {
		return strings.EqualFold(string(workflow.artifactProvider), string(common.ArtifactProviderEcr))
	}
}

func (workflow *serviceWorkflow) isS3Provider() Conditional {
	return func() bool {
		return strings.EqualFold(string(workflow.artifactProvider), string(common.ArtifactProviderS3))
	}
}

func (workflow *serviceWorkflow) isEcsProvider() Conditional {
	return func() bool {
		return strings.EqualFold(string(workflow.envStack.Tags["provider"]), string(common.EnvProviderEcs)) ||
			strings.EqualFold(string(workflow.envStack.Tags["provider"]), string(common.EnvProviderEcsFargate))
	}
}
func (workflow *serviceWorkflow) isEksProvider() Conditional {
	return func() bool {
		return strings.EqualFold(string(workflow.envStack.Tags["provider"]), string(common.EnvProviderEks)) ||
			strings.EqualFold(string(workflow.envStack.Tags["provider"]), string(common.EnvProviderEksFargate))
	}
}
func (workflow *serviceWorkflow) isFargateProvider() Conditional {
	return func() bool {
		return strings.EqualFold(string(workflow.envStack.Tags["provider"]), string(common.EnvProviderEcsFargate))
	}
}

func (workflow *serviceWorkflow) isEc2Provider() Conditional {
	return func() bool {
		return strings.EqualFold(string(workflow.envStack.Tags["provider"]), string(common.EnvProviderEc2))
	}
}

func (workflow *serviceWorkflow) serviceInput(ctx *common.Context, serviceName string) Executor {
	return func() error {
		// Repo Name
		if serviceName != "" {
			workflow.serviceName = serviceName
		} else if ctx.Config.Service.Name != "" {
			workflow.serviceName = ctx.Config.Service.Name
		} else if ctx.Config.Repo.Name != "" {
			workflow.serviceName = ctx.Config.Repo.Name
		} else {
			return errors.New("Service name must be provided")
		}
		return nil
	}
}

func (workflow *serviceWorkflow) serviceRepoUpserter(namespace string, service *common.Service, stackUpserter common.StackUpserter, stackWaiter common.StackWaiter) Executor {
	return func() error {
		if service.ImageRepository != "" {
			log.Noticef("Using repo '%s' for service '%s'", service.ImageRepository, workflow.serviceName)
			workflow.serviceImage = service.ImageRepository
			return nil
		}

		log.Noticef("Upsert repo for service '%s'", workflow.serviceName)

		ecrStackName := common.CreateStackName(namespace, common.StackTypeRepo, workflow.serviceName)

		stackParams := make(map[string]string)
		stackParams["RepoName"] = fmt.Sprintf("%s-%s", namespace, workflow.serviceName)

		tags := createTagMap(&ServiceTags{
			Service:  workflow.serviceName,
			Type:     string(common.StackTypeRepo),
			Provider: "",
			Revision: workflow.codeRevision,
			Repo:     workflow.repoName,
		})

		err := stackUpserter.UpsertStack(ecrStackName, common.TemplateRepo, nil, stackParams, tags, "", "")
		if err != nil {
			return err
		}

		log.Debugf("Waiting for stack '%s' to complete", ecrStackName)
		stack := stackWaiter.AwaitFinalStatus(ecrStackName)
		if stack == nil {
			return fmt.Errorf("Unable to create stack %s", ecrStackName)
		}
		if strings.HasSuffix(stack.Status, "ROLLBACK_COMPLETE") || !strings.HasSuffix(stack.Status, "_COMPLETE") {
			return fmt.Errorf("Ended in failed status %s %s", stack.Status, stack.StatusReason)
		}
		workflow.serviceImage = fmt.Sprintf("%s:%s", stack.Outputs["RepoUrl"], workflow.serviceTag)
		return nil
	}
}
func (workflow *serviceWorkflow) serviceAppUpserter(namespace string, service *common.Service, stackUpserter common.StackUpserter, stackWaiter common.StackWaiter) Executor {
	return func() error {
		log.Noticef("Upsert app for service '%s'", workflow.serviceName)

		appStackName := common.CreateStackName(namespace, common.StackTypeApp, workflow.serviceName)

		stackParams := make(map[string]string)

		tags := createTagMap(&EnvironmentTags{
			Environment: workflow.serviceName,
			Type:        string(common.StackTypeApp),
			Provider:    "",
			Revision:    workflow.codeRevision,
			Repo:        workflow.repoName,
		})

		err := stackUpserter.UpsertStack(appStackName, common.TemplateApp, nil, stackParams, tags, "", workflow.cloudFormationRoleArn)
		if err != nil {
			return err
		}

		log.Debugf("Waiting for stack '%s' to complete", appStackName)
		stack := stackWaiter.AwaitFinalStatus(appStackName)
		if stack == nil {
			return fmt.Errorf("Unable to create stack %s", appStackName)
		}
		if strings.HasSuffix(stack.Status, "ROLLBACK_COMPLETE") || !strings.HasSuffix(stack.Status, "_COMPLETE") {
			return fmt.Errorf("Ended in failed status %s %s", stack.Status, stack.StatusReason)
		}
		workflow.appName = stack.Outputs["ApplicationName"]
		return nil
	}
}

func (workflow *serviceWorkflow) serviceBucketUpserter(namespace string, service *common.Service, stackUpserter common.StackUpserter, stackWaiter common.StackWaiter) Executor {
	return func() error {

		if service.Pipeline.Build.Bucket != "" {
			workflow.appRevisionBucket = service.Pipeline.Build.Bucket
		} else {
			bucketStackName := common.CreateStackName(namespace, common.StackTypeBucket, "codedeploy")
			log.Noticef("Upserting Bucket for CodeDeploy")
			bucketParams := make(map[string]string)
			bucketParams["Namespace"] = namespace
			bucketParams["BucketPrefix"] = "codedeploy"

			tags := createTagMap(&PipelineTags{
				Type: common.StackTypeBucket,
			})

			err := stackUpserter.UpsertStack(bucketStackName, common.TemplateBucket, nil, bucketParams, tags, "", workflow.cloudFormationRoleArn)
			if err != nil {
				return err
			}

			log.Debugf("Waiting for stack '%s' to complete", bucketStackName)
			stack := stackWaiter.AwaitFinalStatus(bucketStackName)
			if stack == nil {
				return fmt.Errorf("Unable to create stack %s", bucketStackName)
			}
			if strings.HasSuffix(stack.Status, "ROLLBACK_COMPLETE") || !strings.HasSuffix(stack.Status, "_COMPLETE") {
				return fmt.Errorf("Ended in failed status %s %s", stack.Status, stack.StatusReason)
			}

			workflow.appRevisionBucket = stack.Outputs["Bucket"]

		}

		return nil
	}
}

func (workflow *serviceWorkflow) serviceRegistryAuthenticator(authenticator common.RepositoryAuthenticator) Executor {
	return func() error {
		log.Debugf("Authenticating to registry '%s'", workflow.serviceImage)
		registryAuth, err := authenticator.AuthenticateRepository(workflow.serviceImage)
		if err != nil {
			return err
		}

		data, err := base64.StdEncoding.DecodeString(registryAuth)
		if err != nil {
			return err
		}

		authParts := strings.Split(string(data), ":")

		workflow.registryAuth = base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("{\"username\":\"%s\", \"password\":\"%s\"}", authParts[0], authParts[1])))

		// ImageBuild pull auth
		var authConfigs2 map[string]types.AuthConfig = make(map[string]types.AuthConfig)
		var serviceImagePart = strings.Split(workflow.serviceImage, ":")[0]

		authConfigs2[serviceImagePart] = types.AuthConfig{
			Username:      authParts[0],
			Password:      authParts[1],
			ServerAddress: fmt.Sprintf("https://%s", serviceImagePart),
		}

		workflow.registryAuthConfig = authConfigs2

		return nil
	}
}

func (workflow *serviceWorkflow) connectKubernetes(provider common.KubernetesResourceManagerProvider) Executor {
	return func() error {
		clusterName := workflow.envStack.Name
		kubernetesResourceManager, err := provider.GetResourceManager(clusterName)
		workflow.kubernetesResourceManager = kubernetesResourceManager
		return err
	}
}
<|endoftext|>