<commit_before>package nats

import (
	"bufio"
	"io"
	"sync"
	"net/textproto"
)

type Connection struct {
	rw io.ReadWriteCloser

	r     *bufio.Reader
	w     *bufio.Writer
	re    error
	we    error
	rec   chan error
	wec   chan error
	rLock sync.Mutex
	wLock sync.Mutex

	sc chan bool

	// Sequencer for PINGs/receiving corresponding PONGs
	ps textproto.Pipeline

	// Channel for receiving PONGs
	pc chan bool

	// Channel for receiving messages
	oc chan readObject
}

func NewConnection(rw io.ReadWriteCloser) *Connection {
	var c = new(Connection)

	c.rw = rw

	c.r = bufio.NewReader(rw)
	c.w = bufio.NewWriter(rw)
	c.rec = make(chan error, 1)
	c.wec = make(chan error, 1)

	c.sc = make(chan bool, 1)

	c.pc = make(chan bool)
	c.oc = make(chan readObject)

	return c
}

func (c *Connection) setReadError(e error) {
	if c.re == nil {
		c.re = e
		c.rec <- e
	}
}

func (c *Connection) setWriteError(e error) {
	if c.we == nil {
		c.we = e
		c.wec <- e
	}
}

func (c *Connection) acquireReader() *bufio.Reader {
	c.rLock.Lock()
	return c.r
}

func (c *Connection) releaseReader(r *bufio.Reader) {
	c.r = r
	c.rLock.Unlock()
}

func (c *Connection) acquireWriter() *bufio.Writer {
	c.wLock.Lock()
	return c.w
}

func (c *Connection) releaseWriter(w *bufio.Writer) {
	c.w = w
	c.wLock.Unlock()
}

func (c *Connection) read(r *bufio.Reader) (readObject, error) {
	var o readObject
	var e error

	o, e = read(r)
	if e != nil {
		c.setReadError(e)
		return nil, e
	}

	return o, nil
}

func (c *Connection) write(w *bufio.Writer, o writeObject) error {
	var e error

	e = write(w, o)
	if e != nil {
		c.setWriteError(e)
		return e
	}

	e = w.Flush()
	if e != nil {
		c.setWriteError(e)
		return e
	}

	return e
}

func (c *Connection) pingAndWaitForPong(w *bufio.Writer) bool {
	var e error

	// Write PING and grab sequence number
	e = c.write(w, &writePing{})
	if e != nil {
		c.releaseWriter(w)
		return false
	}

	seq := c.ps.Next()
	c.releaseWriter(w)

	// Wait for PONG
	c.ps.StartResponse(seq)
	_, ok := <-c.pc
	c.ps.EndResponse(seq)

	return ok
}

func (c *Connection) Ping() bool {
	var w *bufio.Writer

	w = c.acquireWriter()
	return c.pingAndWaitForPong(w)
}

func (c *Connection) WriteChannel(oc chan writeObject) bool {
	var w *bufio.Writer
	var e error

	w = c.acquireWriter()
	defer c.releaseWriter(w)

	// Write until EOF
	for o := range oc {
		if e == nil {
			e = c.write(w, o)
		}
	}

	return e == nil
}

func (c *Connection) Write(o writeObject) bool {
	var w *bufio.Writer
	var e error

	w = c.acquireWriter()
	e = c.write(w, o)
	if e != nil {
		c.releaseWriter(w)
		return false
	}

	c.releaseWriter(w)
	return true
}

func (c *Connection) WriteAndPing(o writeObject) bool {
	var w *bufio.Writer
	var e error

	w = c.acquireWriter()
	e = c.write(w, o)
	if e != nil {
		c.releaseWriter(w)
		return false
	}

	return c.pingAndWaitForPong(w)
}

func (c *Connection) Stop() {
	c.sc <- true
}

func (c *Connection) Run() error {
	var r *bufio.Reader
	var rc chan readObject

	r = c.acquireReader()
	rc = make(chan readObject)

	defer c.releaseReader(r)

	go func() {
		var o readObject
		var e error

		defer close(rc)

		for {
			o, e = c.read(r)
			if e != nil {
				break
			}

			rc <- o
		}
	}()

	var stop bool
	var e error
	var ok bool

	for !stop {
		var o readObject

		select {
		case <-c.sc:
			stop = true
		case e = <-c.rec:
			stop = true
		case e = <-c.wec:
			stop = true
		case o, ok = <-rc:
			if ok {
				switch o.(type) {
				case *readPing:
					go func() {
						c.Write(&writePong{})
					}()
				case *readPong:
					c.pc <- true
				default:
					c.oc <- o
				}
			}
		}
	}

	// Close connection
	c.rw.Close()

	// Drain readObject channel to make read goroutine quit
	for _ = range rc {
	}

	// Can't receive more PONGs
	close(c.pc)

	// Can't receive more messages
	close(c.oc)

	return e
}
<commit_msg>Don't pass read/writer when releasing lock<commit_after>package nats

import (
	"bufio"
	"io"
	"sync"
	"net/textproto"
)

type Connection struct {
	rw io.ReadWriteCloser

	r     *bufio.Reader
	w     *bufio.Writer
	re    error
	we    error
	rec   chan error
	wec   chan error
	rLock sync.Mutex
	wLock sync.Mutex

	sc chan bool

	// Sequencer for PINGs/receiving corresponding PONGs
	ps textproto.Pipeline

	// Channel for receiving PONGs
	pc chan bool

	// Channel for receiving messages
	oc chan readObject
}

func NewConnection(rw io.ReadWriteCloser) *Connection {
	var c = new(Connection)

	c.rw = rw

	c.r = bufio.NewReader(rw)
	c.w = bufio.NewWriter(rw)
	c.rec = make(chan error, 1)
	c.wec = make(chan error, 1)

	c.sc = make(chan bool, 1)

	c.pc = make(chan bool)
	c.oc = make(chan readObject)

	return c
}

func (c *Connection) setReadError(e error) {
	if c.re == nil {
		c.re = e
		c.rec <- e
	}
}

func (c *Connection) setWriteError(e error) {
	if c.we == nil {
		c.we = e
		c.wec <- e
	}
}

func (c *Connection) acquireReader() *bufio.Reader {
	c.rLock.Lock()
	return c.r
}

func (c *Connection) releaseReader() {
	c.rLock.Unlock()
}

func (c *Connection) acquireWriter() *bufio.Writer {
	c.wLock.Lock()
	return c.w
}

func (c *Connection) releaseWriter() {
	c.wLock.Unlock()
}

func (c *Connection) read(r *bufio.Reader) (readObject, error) {
	var o readObject
	var e error

	o, e = read(r)
	if e != nil {
		c.setReadError(e)
		return nil, e
	}

	return o, nil
}

func (c *Connection) write(w *bufio.Writer, o writeObject) error {
	var e error

	e = write(w, o)
	if e != nil {
		c.setWriteError(e)
		return e
	}

	e = w.Flush()
	if e != nil {
		c.setWriteError(e)
		return e
	}

	return e
}

func (c *Connection) pingAndWaitForPong(w *bufio.Writer) bool {
	var e error

	// Write PING and grab sequence number
	e = c.write(w, &writePing{})
	if e != nil {
		c.releaseWriter()
		return false
	}

	seq := c.ps.Next()
	c.releaseWriter()

	// Wait for PONG
	c.ps.StartResponse(seq)
	_, ok := <-c.pc
	c.ps.EndResponse(seq)

	return ok
}

func (c *Connection) Ping() bool {
	var w *bufio.Writer

	w = c.acquireWriter()
	return c.pingAndWaitForPong(w)
}

func (c *Connection) WriteChannel(oc chan writeObject) bool {
	var w *bufio.Writer
	var e error

	w = c.acquireWriter()
	defer c.releaseWriter()

	// Write until EOF
	for o := range oc {
		if e == nil {
			e = c.write(w, o)
		}
	}

	return e == nil
}

func (c *Connection) Write(o writeObject) bool {
	var w *bufio.Writer
	var e error

	w = c.acquireWriter()
	e = c.write(w, o)
	if e != nil {
		c.releaseWriter()
		return false
	}

	c.releaseWriter()
	return true
}

func (c *Connection) WriteAndPing(o writeObject) bool {
	var w *bufio.Writer
	var e error

	w = c.acquireWriter()
	e = c.write(w, o)
	if e != nil {
		c.releaseWriter()
		return false
	}

	return c.pingAndWaitForPong(w)
}

func (c *Connection) Stop() {
	c.sc <- true
}

func (c *Connection) Run() error {
	var r *bufio.Reader
	var rc chan readObject

	r = c.acquireReader()
	rc = make(chan readObject)

	defer c.releaseReader()

	go func() {
		var o readObject
		var e error

		defer close(rc)

		for {
			o, e = c.read(r)
			if e != nil {
				break
			}

			rc <- o
		}
	}()

	var stop bool
	var e error
	var ok bool

	for !stop {
		var o readObject

		select {
		case <-c.sc:
			stop = true
		case e = <-c.rec:
			stop = true
		case e = <-c.wec:
			stop = true
		case o, ok = <-rc:
			if ok {
				switch o.(type) {
				case *readPing:
					go func() {
						c.Write(&writePong{})
					}()
				case *readPong:
					c.pc <- true
				default:
					c.oc <- o
				}
			}
		}
	}

	// Close connection
	c.rw.Close()

	// Drain readObject channel to make read goroutine quit
	for _ = range rc {
	}

	// Can't receive more PONGs
	close(c.pc)

	// Can't receive more messages
	close(c.oc)

	return e
}
<|endoftext|>


<commit_before>// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

package mysql

import (
	"crypto/tls"
	"database/sql/driver"
	"errors"
	"net"
	"strings"
	"time"
)

type mysqlConn struct {
	buf              *buffer
	netConn          net.Conn
	affectedRows     uint64
	insertId         uint64
	cfg              *config
	maxPacketAllowed int
	maxWriteSize     int
	flags            clientFlag
	sequence         uint8
	parseTime        bool
	strict           bool
}

type config struct {
	user              string
	passwd            string
	net               string
	addr              string
	dbname            string
	params            map[string]string
	loc               *time.Location
	timeout           time.Duration
	tls               *tls.Config
	allowAllFiles     bool
	allowOldPasswords bool
	clientFoundRows   bool
}

// Handles parameters set in DSN
func (mc *mysqlConn) handleParams() (err error) {
	for param, val := range mc.cfg.params {
		switch param {
		// Charset
		case "charset":
			charsets := strings.Split(val, ",")
			for i := range charsets {
				// ignore errors here - a charset may not exist
				err = mc.exec("SET NAMES " + charsets[i])
				if err == nil {
					break
				}
			}
			if err != nil {
				return
			}

		// time.Time parsing
		case "parseTime":
			var isBool bool
			mc.parseTime, isBool = readBool(val)
			if !isBool {
				return errors.New("Invalid Bool value: " + val)
			}

		// Strict mode
		case "strict":
			var isBool bool
			mc.strict, isBool = readBool(val)
			if !isBool {
				return errors.New("Invalid Bool value: " + val)
			}

		// Compression
		case "compress":
			err = errors.New("Compression not implemented yet")
			return

		// System Vars
		default:
			err = mc.exec("SET " + param + "=" + val + "")
			if err != nil {
				return
			}
		}
	}

	return
}

func (mc *mysqlConn) Begin() (driver.Tx, error) {
	if mc.netConn == nil {
		errLog.Print(errInvalidConn)
		return nil, driver.ErrBadConn
	}
	err := mc.exec("START TRANSACTION")
	if err == nil {
		return &mysqlTx{mc}, err
	}

	return nil, err
}

func (mc *mysqlConn) Close() (err error) {
	// Makes Close idempotent
	if mc.netConn != nil {
		mc.writeCommandPacket(comQuit)
		mc.netConn.Close()
		mc.netConn = nil
	}

	mc.cfg = nil
	mc.buf = nil

	return
}

func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) {
	if mc.netConn == nil {
		errLog.Print(errInvalidConn)
		return nil, driver.ErrBadConn
	}
	// Send command
	err := mc.writeCommandPacketStr(comStmtPrepare, query)
	if err != nil {
		return nil, err
	}

	stmt := &mysqlStmt{
		mc: mc,
	}

	// Read Result
	columnCount, err := stmt.readPrepareResultPacket()
	if err == nil {
		if stmt.paramCount > 0 {
			if err = mc.readUntilEOF(); err != nil {
				return nil, err
			}
		}

		if columnCount > 0 {
			err = mc.readUntilEOF()
		}
	}

	return stmt, err
}

func (mc *mysqlConn) Exec(query string, args []driver.Value) (driver.Result, error) {
	if mc.netConn == nil {
		errLog.Print(errInvalidConn)
		return nil, driver.ErrBadConn
	}
	if len(args) == 0 { // no args, fastpath
		mc.affectedRows = 0
		mc.insertId = 0

		err := mc.exec(query)
		if err == nil {
			return &mysqlResult{
				affectedRows: int64(mc.affectedRows),
				insertId:     int64(mc.insertId),
			}, err
		}
		return nil, err
	}

	// with args, must use prepared stmt
	return nil, driver.ErrSkip

}

// Internal function to execute commands
func (mc *mysqlConn) exec(query string) error {
	// Send command
	err := mc.writeCommandPacketStr(comQuery, query)
	if err != nil {
		return err
	}

	// Read Result
	resLen, err := mc.readResultSetHeaderPacket()
	if err == nil && resLen > 0 {
		if err = mc.readUntilEOF(); err != nil {
			return err
		}

		err = mc.readUntilEOF()
	}

	return err
}

func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, error) {
	if mc.netConn == nil {
		errLog.Print(errInvalidConn)
		return nil, driver.ErrBadConn
	}
	if len(args) == 0 { // no args, fastpath
		// Send command
		err := mc.writeCommandPacketStr(comQuery, query)
		if err == nil {
			// Read Result
			var resLen int
			resLen, err = mc.readResultSetHeaderPacket()
			if err == nil {
				rows := new(textRows)
				rows.mc = mc

				if resLen > 0 {
					// Columns
					rows.columns, err = mc.readColumns(resLen)
				}
				return rows, err
			}
		}
		return nil, err
	}

	// with args, must use prepared stmt
	return nil, driver.ErrSkip
}

// Gets the value of the given MySQL System Variable
// The returned byte slice is only valid until the next read
func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) {
	// Send command
	if err := mc.writeCommandPacketStr(comQuery, "SELECT @@"+name); err != nil {
		return nil, err
	}

	// Read Result
	resLen, err := mc.readResultSetHeaderPacket()
	if err == nil {
		rows := new(textRows)
		rows.mc = mc

		if resLen > 0 {
			// Columns
			if err := mc.readUntilEOF(); err != nil {
				return nil, err
			}
		}

		dest := make([]driver.Value, resLen)
		if err = rows.readRow(dest); err == nil {
			return dest[0].([]byte), mc.readUntilEOF()
		}
	}
	return nil, err
}
<commit_msg>return first err in mysqlConn.Close<commit_after>// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

package mysql

import (
	"crypto/tls"
	"database/sql/driver"
	"errors"
	"net"
	"strings"
	"time"
)

type mysqlConn struct {
	buf              *buffer
	netConn          net.Conn
	affectedRows     uint64
	insertId         uint64
	cfg              *config
	maxPacketAllowed int
	maxWriteSize     int
	flags            clientFlag
	sequence         uint8
	parseTime        bool
	strict           bool
}

type config struct {
	user              string
	passwd            string
	net               string
	addr              string
	dbname            string
	params            map[string]string
	loc               *time.Location
	timeout           time.Duration
	tls               *tls.Config
	allowAllFiles     bool
	allowOldPasswords bool
	clientFoundRows   bool
}

// Handles parameters set in DSN
func (mc *mysqlConn) handleParams() (err error) {
	for param, val := range mc.cfg.params {
		switch param {
		// Charset
		case "charset":
			charsets := strings.Split(val, ",")
			for i := range charsets {
				// ignore errors here - a charset may not exist
				err = mc.exec("SET NAMES " + charsets[i])
				if err == nil {
					break
				}
			}
			if err != nil {
				return
			}

		// time.Time parsing
		case "parseTime":
			var isBool bool
			mc.parseTime, isBool = readBool(val)
			if !isBool {
				return errors.New("Invalid Bool value: " + val)
			}

		// Strict mode
		case "strict":
			var isBool bool
			mc.strict, isBool = readBool(val)
			if !isBool {
				return errors.New("Invalid Bool value: " + val)
			}

		// Compression
		case "compress":
			err = errors.New("Compression not implemented yet")
			return

		// System Vars
		default:
			err = mc.exec("SET " + param + "=" + val + "")
			if err != nil {
				return
			}
		}
	}

	return
}

func (mc *mysqlConn) Begin() (driver.Tx, error) {
	if mc.netConn == nil {
		errLog.Print(errInvalidConn)
		return nil, driver.ErrBadConn
	}
	err := mc.exec("START TRANSACTION")
	if err == nil {
		return &mysqlTx{mc}, err
	}

	return nil, err
}

func (mc *mysqlConn) Close() (err error) {
	// Makes Close idempotent
	if mc.netConn != nil {
		err = mc.writeCommandPacket(comQuit)
		if err == nil {
			err = mc.netConn.Close()
		} else {
			mc.netConn.Close()
		}
		mc.netConn = nil
	}

	mc.cfg = nil
	mc.buf = nil

	return
}

func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) {
	if mc.netConn == nil {
		errLog.Print(errInvalidConn)
		return nil, driver.ErrBadConn
	}
	// Send command
	err := mc.writeCommandPacketStr(comStmtPrepare, query)
	if err != nil {
		return nil, err
	}

	stmt := &mysqlStmt{
		mc: mc,
	}

	// Read Result
	columnCount, err := stmt.readPrepareResultPacket()
	if err == nil {
		if stmt.paramCount > 0 {
			if err = mc.readUntilEOF(); err != nil {
				return nil, err
			}
		}

		if columnCount > 0 {
			err = mc.readUntilEOF()
		}
	}

	return stmt, err
}

func (mc *mysqlConn) Exec(query string, args []driver.Value) (driver.Result, error) {
	if mc.netConn == nil {
		errLog.Print(errInvalidConn)
		return nil, driver.ErrBadConn
	}
	if len(args) == 0 { // no args, fastpath
		mc.affectedRows = 0
		mc.insertId = 0

		err := mc.exec(query)
		if err == nil {
			return &mysqlResult{
				affectedRows: int64(mc.affectedRows),
				insertId:     int64(mc.insertId),
			}, err
		}
		return nil, err
	}

	// with args, must use prepared stmt
	return nil, driver.ErrSkip

}

// Internal function to execute commands
func (mc *mysqlConn) exec(query string) error {
	// Send command
	err := mc.writeCommandPacketStr(comQuery, query)
	if err != nil {
		return err
	}

	// Read Result
	resLen, err := mc.readResultSetHeaderPacket()
	if err == nil && resLen > 0 {
		if err = mc.readUntilEOF(); err != nil {
			return err
		}

		err = mc.readUntilEOF()
	}

	return err
}

func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, error) {
	if mc.netConn == nil {
		errLog.Print(errInvalidConn)
		return nil, driver.ErrBadConn
	}
	if len(args) == 0 { // no args, fastpath
		// Send command
		err := mc.writeCommandPacketStr(comQuery, query)
		if err == nil {
			// Read Result
			var resLen int
			resLen, err = mc.readResultSetHeaderPacket()
			if err == nil {
				rows := new(textRows)
				rows.mc = mc

				if resLen > 0 {
					// Columns
					rows.columns, err = mc.readColumns(resLen)
				}
				return rows, err
			}
		}
		return nil, err
	}

	// with args, must use prepared stmt
	return nil, driver.ErrSkip
}

// Gets the value of the given MySQL System Variable
// The returned byte slice is only valid until the next read
func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) {
	// Send command
	if err := mc.writeCommandPacketStr(comQuery, "SELECT @@"+name); err != nil {
		return nil, err
	}

	// Read Result
	resLen, err := mc.readResultSetHeaderPacket()
	if err == nil {
		rows := new(textRows)
		rows.mc = mc

		if resLen > 0 {
			// Columns
			if err := mc.readUntilEOF(); err != nil {
				return nil, err
			}
		}

		dest := make([]driver.Value, resLen)
		if err = rows.readRow(dest); err == nil {
			return dest[0].([]byte), mc.readUntilEOF()
		}
	}
	return nil, err
}
<|endoftext|>


<commit_before>package main
/*
 * Copyright 2016 ThoughtWorks, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"fmt"
	"os"
	"os/exec"
	"io"
	"bytes"
	"io/ioutil"
	"strings"
	"flag"
)

var goAgentFilename = "gocd-golang-agent"

var targetOS = map[string][]string{
	"darwin" : {"amd64"},
	"linux" : {"386", "amd64"},
//	"windows" : {"386", "amd64"},  // Windows build is broken because of undefined syscall.Statfs_t and syscall.Statfs
}

var goAgent = "github.com/gocd-contrib/gocd-golang-agent"

var targetOSmap = map[string]string{
	"darwin" : "MacOS",
	"linux" : "Linux",
	"windows" : "Windows",
	"solaris" : "Solaris",
	"netbsd" : "NetBSD",
	"freebsd" : "FreeBSD",
}

var ext_libraries []string =  []string{
	"golang.org/x/net/websocket",
	"golang.org/x/text",
	"golang.org/x/crypto/ssh",
	"github.com/satori/go.uuid",
	"github.com/xli/assert",
	"github.com/bmatcuk/doublestar"}

var testReport = "testreport.xml"

func shouldSkipDependent(exlib string) bool {
	exlibGoEnv := "GO_REVISION_SRC_" + strings.ToUpper(strings.Join(strings.Split(strings.Join(strings.Split(exlib,"."),"_"),"/"),"_"))
	if len(os.Getenv(exlibGoEnv)) > 0 {
		return true
	}
	return false
}

func getDependencies(excludeLib string) {
	fmt.Println("==================================")
	fmt.Println("Download Dependencies")
	go_args := []string{}
	args := []string{"get","-u"}
	for _, exlib := range ext_libraries {
		if strings.Contains(excludeLib,exlib) {
			fmt.Printf("Exclude from go get. Please manually run : go get %s\n", exlib)
		}else {
			if !shouldSkipDependent(exlib) {
				go_args = append(args, exlib)
				_, err := exec.Command("go", go_args...).Output()
				if err != nil {
					fmt.Println(err)
				}
				fmt.Printf("Get : %s\n", exlib)
			} else {
				fmt.Printf("Skip %s since it is part of GoCD Material\n", exlib)
			}
		}
	}
}

func getGitHash(pwd string) string {
	runCmd := exec.Command("git", "rev-parse", "HEAD")
	runCmd.Dir = pwd + "/src/" + goAgent
	out, err := runCmd.Output()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(2)
	}
	return string(out)
}

func buildBinary(pwd string){
	fmt.Println("==================================")
	fmt.Println("Building Binary")
	os.RemoveAll("output")
	os.Mkdir("output",0755)
	os.Setenv("CGO_ENABLED","0")
	for buildOS, buildArchs := range targetOS {
		for _, buildArch := range buildArchs {
			fmt.Println("---> " + targetOSmap[buildOS] + " - " + buildArch)
			os.Setenv("GOOS", buildOS)
			os.Setenv("GOARCH", buildArch)
			ldFlags := "-w -X main.Githash=" + getGitHash(pwd)
			buildVersion := os.Getenv("BUILD_VERSION")
			if len(buildVersion) > 0 {
				ldFlags = ldFlags + "-X main.Version=" + buildVersion
			}
			out, err := exec.Command("go", "build", "-a", "-tags", "netgo", "-ldflags", ldFlags, "-o", "output/" + goAgentFilename + "_" + buildOS + "_" + buildArch, goAgent).Output()
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(2)
			}
			if len(string(out)) > 0 {
				fmt.Println(out)
			}
		}
	}

}

func runTest(pwd string){
	fmt.Println("==================================")
	fmt.Println("Running Test")
	args := []string{"get","-u","github.com/jstemmer/go-junit-report"}
	reportCmd := pwd + "/bin/go-junit-report"
	reportCmd_args := []string{" > ", testReport}

	_, err := exec.Command("go", args...).Output()
	if err != nil{
		fmt.Println(err)
	}
	_, err = os.Stat(testReport)
	if err == nil {
		os.Remove(testReport)
	}
	args = []string{"test", "-test.v", goAgent + "..." }
	gotest := exec.Command("go", args...)
	goreport := exec.Command(reportCmd,reportCmd_args...)
	reader, writer := io.Pipe()
	gotest.Stdout = writer
	goreport.Stdin = reader

	var buf bytes.Buffer
	goreport.Stdout = &buf

	gotest.Start()
	goreport.Start()
	gotest.Wait()
	writer.Close()
	goreport.Wait()
	err = ioutil.WriteFile(testReport,buf.Bytes(),0644)
	if err != nil{
		panic(err)
	}
}

func main() {

	var excludeLib string

	flag.StringVar(&excludeLib, "excludelib", "github.com/gocd-contrib/fake_agent", "exclude dependencies in comma separated format, eg github.com/gocd-contrib/fake_agent,")
	flag.Parse()

	pwd, err := os.Getwd()
	if err == nil {
		os.Setenv("GOPATH",pwd)
		os.Setenv("GOBIN", pwd + "/bin")
	}

	getDependencies(excludeLib)
	runTest(pwd)
	buildBinary(pwd)

}<commit_msg>Separate test, binary building, etc.<commit_after>package main
/*
 * Copyright 2016 ThoughtWorks, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"fmt"
	"os"
	"os/exec"
	"io"
	"bytes"
	"io/ioutil"
	"strings"
	"flag"
	"runtime"
)

var goAgentFilename = "gocd-golang-agent"

var targetOS = map[string][]string{
	"darwin" : {"amd64"},
	"linux" : {"386", "amd64"},
//	"windows" : {"386", "amd64"},  // Windows build is broken because of undefined syscall.Statfs_t and syscall.Statfs
}

var goAgent = "github.com/gocd-contrib/gocd-golang-agent"

var targetOSmap = map[string]string{
	"darwin" : "MacOS",
	"linux" : "Linux",
	"windows" : "Windows",
	"solaris" : "Solaris",
	"netbsd" : "NetBSD",
	"freebsd" : "FreeBSD",
}

var ext_libraries []string =  []string{
	"golang.org/x/net/websocket",
	"golang.org/x/text",
	"golang.org/x/crypto/ssh",
	"github.com/satori/go.uuid",
	"github.com/xli/assert",
	"github.com/bmatcuk/doublestar"}

var testReport = "testreport.xml"

func shouldSkipDependent(exlib string) bool {
	exlibGoEnv := "GO_REVISION_SRC_" + strings.ToUpper(strings.Join(strings.Split(strings.Join(strings.Split(exlib,"."),"_"),"/"),"_"))
	if len(os.Getenv(exlibGoEnv)) > 0 {
		return true
	}
	return false
}

func getDependencies(excludeLib string) {
	fmt.Println("==================================")
	fmt.Println("Download Dependencies")
	go_args := []string{}
	args := []string{"get","-u"}
	for _, exlib := range ext_libraries {
		if strings.Contains(excludeLib,exlib) {
			fmt.Printf("Exclude from go get. Please manually run : go get %s\n", exlib)
		}else {
			if !shouldSkipDependent(exlib) {
				go_args = append(args, exlib)
				_, err := exec.Command("go", go_args...).Output()
				if err != nil {
					fmt.Println(err)
				}
				fmt.Printf("Get : %s\n", exlib)
			} else {
				fmt.Printf("Skip %s since it is part of GoCD Material\n", exlib)
			}
		}
	}
}

func getGitHash(pwd string) string {
	runCmd := exec.Command("git", "rev-parse", "HEAD")
	runCmd.Dir = pwd + "/src/" + goAgent
	out, err := runCmd.Output()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(2)
	}
	return string(out)
}

func buildBinary(pwd string, binAllbinary bool){
	fmt.Println("==================================")
	fmt.Println("Building Binary")
	os.RemoveAll("output")
	os.Mkdir("output",0755)
	os.Setenv("CGO_ENABLED","0")
	if binAllbinary {
		for buildOS, buildArchs := range targetOS {
			for _, buildArch := range buildArchs {
				os.Setenv("GOOS", buildOS)
				os.Setenv("GOARCH", buildArch)
				compileApp(pwd, buildOS, buildArch)
			}
		}
	}else{
		compileApp(pwd, runtime.GOOS, runtime.GOARCH)
	}

}

func compileApp(pwd string, targetOS string, targetArch string){
	fmt.Println("---> " + targetOSmap[targetOS] + " - " + targetArch)
	ldFlags := "-w -X main.Githash=" + getGitHash(pwd)
	buildVersion := os.Getenv("BUILD_VERSION")
	if len(buildVersion) > 0 {
		ldFlags = ldFlags + "-X main.Version=" + buildVersion
	}
	out, err := exec.Command("go", "build", "-a", "-tags", "netgo", "-ldflags", ldFlags, "-o", "output/" + goAgentFilename + "_" + targetOS + "_" + targetArch, goAgent).Output()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(2)
	}
	if len(string(out)) > 0 {
		fmt.Println(out)
	}

}

func runTest(pwd string){
	fmt.Println("==================================")
	fmt.Println("Running Test")
	args := []string{"get","-u","github.com/jstemmer/go-junit-report"}
	reportCmd := pwd + "/bin/go-junit-report"
	reportCmd_args := []string{" > ", testReport}

	_, err := exec.Command("go", args...).Output()
	if err != nil{
		fmt.Println(err)
	}
	_, err = os.Stat(testReport)
	if err == nil {
		os.Remove(testReport)
	}
	args = []string{"test", "-test.v", goAgent + "..." }
	gotest := exec.Command("go", args...)
	goreport := exec.Command(reportCmd,reportCmd_args...)
	reader, writer := io.Pipe()
	gotest.Stdout = writer
	goreport.Stdin = reader

	var buf bytes.Buffer
	goreport.Stdout = &buf

	gotest.Start()
	goreport.Start()
	gotest.Wait()
	writer.Close()
	goreport.Wait()
	err = ioutil.WriteFile(testReport,buf.Bytes(),0644)
	if err != nil{
		panic(err)
	}
}

func main() {

	var (
		excludeLib string
		runAllTest bool
		buildLocalBinary bool
		buildAll bool
	)

	flag.StringVar(&excludeLib, "excludelib", "", "exclude dependencies in comma separated format, eg github.com/gocd-contrib/fake_agent,github.com/gocd-contrib/fake_server")
	flag.BoolVar(&runAllTest,"runtest", true, "Run all Tests")
	flag.BoolVar(&buildLocalBinary,"buildbinary", true, "Build local GoAgent binary" )
	flag.BoolVar(&buildAll,"buildall", false, "Build GoAgent binary for all platforms" )
	flag.Parse()

	pwd, err := os.Getwd()
	if err == nil {
		os.Setenv("GOPATH",pwd)
		os.Setenv("GOBIN", pwd + "/bin")
	}

	getDependencies(excludeLib)

	if runAllTest {
		runTest(pwd)
	}

	if buildAll {
		buildBinary(pwd,true)
	}else{
		if buildLocalBinary {
			buildBinary(pwd, false)
		}
	}

}<|endoftext|>


<commit_before>package basichost

import (
	"net"
	"strconv"
	"sync"

	goprocess "github.com/jbenet/goprocess"
	goprocessctx "github.com/jbenet/goprocess/context"
	"github.com/libp2p/go-libp2p-core/network"
	inat "github.com/libp2p/go-libp2p-nat"
	ma "github.com/multiformats/go-multiaddr"
)

// A simple interface to manage NAT devices.
type NATManager interface {

	// Get the NAT device managed by the NAT manager.
	NAT() *inat.NAT

	// Receive a notification when the NAT device is ready for use.
	Ready() <-chan struct{}

	// Close all resources associated with a NAT manager.
	Close() error
}

// Create a NAT manager.
func NewNATManager(net network.Network) NATManager {
	return newNatManager(net)
}

// natManager takes care of adding + removing port mappings to the nat.
// Initialized with the host if it has a NATPortMap option enabled.
// natManager receives signals from the network, and check on nat mappings:
//  * natManager listens to the network and adds or closes port mappings
//    as the network signals Listen() or ListenClose().
//  * closing the natManager closes the nat and its mappings.
type natManager struct {
	net   network.Network
	natmu sync.RWMutex
	nat   *inat.NAT

	ready chan struct{} // closed once the nat is ready to process port mappings

	syncMu sync.Mutex

	proc goprocess.Process // natManager has a process + children. can be closed.
}

func newNatManager(net network.Network) *natManager {
	nmgr := &natManager{
		net:   net,
		ready: make(chan struct{}),
	}

	nmgr.proc = goprocess.WithTeardown(func() error {
		// on closing, unregister from network notifications.
		net.StopNotify((*nmgrNetNotifiee)(nmgr))
		return nil
	})

	// discover the nat.
	nmgr.discoverNAT()
	return nmgr
}

// Close closes the natManager, closing the underlying nat
// and unregistering from network events.
func (nmgr *natManager) Close() error {
	return nmgr.proc.Close()
}

// Ready returns a channel which will be closed when the NAT has been found
// and is ready to be used, or the search process is done.
func (nmgr *natManager) Ready() <-chan struct{} {
	return nmgr.ready
}

func (nmgr *natManager) discoverNAT() {
	nmgr.proc.Go(func(worker goprocess.Process) {
		// inat.DiscoverNAT blocks until the nat is found or a timeout
		// is reached. we unfortunately cannot specify timeouts-- the
		// library we're using just blocks.
		//
		// Note: on early shutdown, there may be a case where we're trying
		// to close before DiscoverNAT() returns. Since we cant cancel it
		// (library) we can choose to (1) drop the result and return early,
		// or (2) wait until it times out to exit. For now we choose (2),
		// to avoid leaking resources in a non-obvious way. the only case
		// this affects is when the daemon is being started up and _immediately_
		// asked to close. other services are also starting up, so ok to wait.

		natInstance, err := inat.DiscoverNAT(goprocessctx.OnClosingContext(worker))
		if err != nil {
			log.Info("DiscoverNAT error:", err)
			close(nmgr.ready)
			return
		}

		nmgr.natmu.Lock()
		nmgr.nat = natInstance
		nmgr.natmu.Unlock()
		close(nmgr.ready)

		// wire up the nat to close when nmgr closes.
		// nmgr.proc is our parent, and waiting for us.
		nmgr.proc.AddChild(nmgr.nat.Process())

		// sign natManager up for network notifications
		// we need to sign up here to avoid missing some notifs
		// before the NAT has been found.
		nmgr.net.Notify((*nmgrNetNotifiee)(nmgr))
		nmgr.sync()
	})
}

// syncs the current NAT mappings, removing any outdated mappings and adding any
// new mappings.
func (nmgr *natManager) sync() {
	nat := nmgr.NAT()
	if nat == nil {
		// Nothing to do.
		return
	}

	nmgr.proc.Go(func(_ goprocess.Process) {
		nmgr.syncMu.Lock()
		defer nmgr.syncMu.Unlock()

		ports := map[string]map[int]bool{
			"tcp": map[int]bool{},
			"udp": map[int]bool{},
		}
		for _, maddr := range nmgr.net.ListenAddresses() {
			// Strip the IP
			maIP, rest := ma.SplitFirst(maddr)
			if maIP == nil || rest == nil {
				continue
			}

			switch maIP.Protocol().Code {
			case ma.P_IP6, ma.P_IP4:
			default:
				continue
			}

			// Only bother if we're listening on a
			// unicast/unspecified IP.
			ip := net.IP(maIP.RawValue())
			if !(ip.IsGlobalUnicast() || ip.IsUnspecified()) {
				continue
			}

			// Extract the port/protocol
			proto, _ := ma.SplitFirst(rest)
			if proto == nil {
				continue
			}

			var protocol string
			switch proto.Protocol().Code {
			case ma.P_TCP:
				protocol = "tcp"
			case ma.P_UDP:
				protocol = "udp"
			default:
				continue
			}

			port, err := strconv.ParseUint(proto.Value(), 10, 16)
			if err != nil {
				// bug in multiaddr
				panic(err)
			}
			ports[protocol][int(port)] = false
		}

		var wg sync.WaitGroup
		defer wg.Wait()

		// Close old mappings
		for _, m := range nat.Mappings() {
			mappedPort := m.InternalPort()
			if _, ok := ports[m.Protocol()][mappedPort]; !ok {
				// No longer need this mapping.
				wg.Add(1)
				go func(m inat.Mapping) {
					defer wg.Done()
					m.Close()
				}(m)
			} else {
				// already mapped
				ports[m.Protocol()][mappedPort] = true
			}
		}

		// Create new mappings.
		for proto, pports := range ports {
			for port, mapped := range pports {
				if mapped {
					continue
				}
				wg.Add(1)
				go func(proto string, port int) {
					defer wg.Done()
					_, err := nat.NewMapping(proto, port)
					if err != nil {
						log.Errorf("failed to port-map %s port %d: %s", proto, port, err)
					}
				}(proto, port)
			}
		}
	})
}

// NAT returns the natManager's nat object. this may be nil, if
// (a) the search process is still ongoing, or (b) the search process
// found no nat. Clients must check whether the return value is nil.
func (nmgr *natManager) NAT() *inat.NAT {
	nmgr.natmu.Lock()
	defer nmgr.natmu.Unlock()
	return nmgr.nat
}

type nmgrNetNotifiee natManager

func (nn *nmgrNetNotifiee) natManager() *natManager {
	return (*natManager)(nn)
}

func (nn *nmgrNetNotifiee) Listen(n network.Network, addr ma.Multiaddr) {
	nn.natManager().sync()
}

func (nn *nmgrNetNotifiee) ListenClose(n network.Network, addr ma.Multiaddr) {
	nn.natManager().sync()
}

func (nn *nmgrNetNotifiee) Connected(network.Network, network.Conn)      {}
func (nn *nmgrNetNotifiee) Disconnected(network.Network, network.Conn)   {}
func (nn *nmgrNetNotifiee) OpenedStream(network.Network, network.Stream) {}
func (nn *nmgrNetNotifiee) ClosedStream(network.Network, network.Stream) {}
<commit_msg>fix: avoid a close deadlock in the natmanager<commit_after>package basichost

import (
	"net"
	"strconv"
	"sync"

	goprocess "github.com/jbenet/goprocess"
	goprocessctx "github.com/jbenet/goprocess/context"
	"github.com/libp2p/go-libp2p-core/network"
	inat "github.com/libp2p/go-libp2p-nat"
	ma "github.com/multiformats/go-multiaddr"
)

// A simple interface to manage NAT devices.
type NATManager interface {

	// Get the NAT device managed by the NAT manager.
	NAT() *inat.NAT

	// Receive a notification when the NAT device is ready for use.
	Ready() <-chan struct{}

	// Close all resources associated with a NAT manager.
	Close() error
}

// Create a NAT manager.
func NewNATManager(net network.Network) NATManager {
	return newNatManager(net)
}

// natManager takes care of adding + removing port mappings to the nat.
// Initialized with the host if it has a NATPortMap option enabled.
// natManager receives signals from the network, and check on nat mappings:
//  * natManager listens to the network and adds or closes port mappings
//    as the network signals Listen() or ListenClose().
//  * closing the natManager closes the nat and its mappings.
type natManager struct {
	net   network.Network
	natmu sync.RWMutex
	nat   *inat.NAT

	ready    chan struct{} // closed once the nat is ready to process port mappings
	syncFlag chan struct{}

	proc goprocess.Process // natManager has a process + children. can be closed.
}

func newNatManager(net network.Network) *natManager {
	nmgr := &natManager{
		net:      net,
		ready:    make(chan struct{}),
		syncFlag: make(chan struct{}, 1),
	}

	nmgr.proc = goprocess.WithParent(goprocess.Background())

	nmgr.start()
	return nmgr
}

// Close closes the natManager, closing the underlying nat
// and unregistering from network events.
func (nmgr *natManager) Close() error {
	return nmgr.proc.Close()
}

// Ready returns a channel which will be closed when the NAT has been found
// and is ready to be used, or the search process is done.
func (nmgr *natManager) Ready() <-chan struct{} {
	return nmgr.ready
}

func (nmgr *natManager) start() {
	nmgr.proc.Go(func(worker goprocess.Process) {
		// inat.DiscoverNAT blocks until the nat is found or a timeout
		// is reached. we unfortunately cannot specify timeouts-- the
		// library we're using just blocks.
		//
		// Note: on early shutdown, there may be a case where we're trying
		// to close before DiscoverNAT() returns. Since we cant cancel it
		// (library) we can choose to (1) drop the result and return early,
		// or (2) wait until it times out to exit. For now we choose (2),
		// to avoid leaking resources in a non-obvious way. the only case
		// this affects is when the daemon is being started up and _immediately_
		// asked to close. other services are also starting up, so ok to wait.

		natInstance, err := inat.DiscoverNAT(goprocessctx.OnClosingContext(worker))
		if err != nil {
			log.Info("DiscoverNAT error:", err)
			close(nmgr.ready)
			return
		}

		nmgr.natmu.Lock()
		nmgr.nat = natInstance
		nmgr.natmu.Unlock()
		close(nmgr.ready)

		// wire up the nat to close when nmgr closes.
		// nmgr.proc is our parent, and waiting for us.
		nmgr.proc.AddChild(nmgr.nat.Process())

		// sign natManager up for network notifications
		// we need to sign up here to avoid missing some notifs
		// before the NAT has been found.
		nmgr.net.Notify((*nmgrNetNotifiee)(nmgr))
		defer nmgr.net.StopNotify((*nmgrNetNotifiee)(nmgr))

		nmgr.doSync() // sync one first.
		for {
			select {
			case <-nmgr.syncFlag:
				nmgr.doSync() // sync when our listen addresses chnage.
			case <-worker.Closing():
				return
			}
		}
	})
}

func (nmgr *natManager) sync() {
	select {
	case nmgr.syncFlag <- struct{}{}:
	default:
	}
}

// doSync syncs the current NAT mappings, removing any outdated mappings and adding any
// new mappings.
func (nmgr *natManager) doSync() {
	ports := map[string]map[int]bool{
		"tcp": map[int]bool{},
		"udp": map[int]bool{},
	}
	for _, maddr := range nmgr.net.ListenAddresses() {
		// Strip the IP
		maIP, rest := ma.SplitFirst(maddr)
		if maIP == nil || rest == nil {
			continue
		}

		switch maIP.Protocol().Code {
		case ma.P_IP6, ma.P_IP4:
		default:
			continue
		}

		// Only bother if we're listening on a
		// unicast/unspecified IP.
		ip := net.IP(maIP.RawValue())
		if !(ip.IsGlobalUnicast() || ip.IsUnspecified()) {
			continue
		}

		// Extract the port/protocol
		proto, _ := ma.SplitFirst(rest)
		if proto == nil {
			continue
		}

		var protocol string
		switch proto.Protocol().Code {
		case ma.P_TCP:
			protocol = "tcp"
		case ma.P_UDP:
			protocol = "udp"
		default:
			continue
		}

		port, err := strconv.ParseUint(proto.Value(), 10, 16)
		if err != nil {
			// bug in multiaddr
			panic(err)
		}
		ports[protocol][int(port)] = false
	}

	var wg sync.WaitGroup
	defer wg.Wait()

	// Close old mappings
	for _, m := range nmgr.nat.Mappings() {
		mappedPort := m.InternalPort()
		if _, ok := ports[m.Protocol()][mappedPort]; !ok {
			// No longer need this mapping.
			wg.Add(1)
			go func(m inat.Mapping) {
				defer wg.Done()
				m.Close()
			}(m)
		} else {
			// already mapped
			ports[m.Protocol()][mappedPort] = true
		}
	}

	// Create new mappings.
	for proto, pports := range ports {
		for port, mapped := range pports {
			if mapped {
				continue
			}
			wg.Add(1)
			go func(proto string, port int) {
				defer wg.Done()
				_, err := nmgr.nat.NewMapping(proto, port)
				if err != nil {
					log.Errorf("failed to port-map %s port %d: %s", proto, port, err)
				}
			}(proto, port)
		}
	}
}

// NAT returns the natManager's nat object. this may be nil, if
// (a) the search process is still ongoing, or (b) the search process
// found no nat. Clients must check whether the return value is nil.
func (nmgr *natManager) NAT() *inat.NAT {
	nmgr.natmu.Lock()
	defer nmgr.natmu.Unlock()
	return nmgr.nat
}

type nmgrNetNotifiee natManager

func (nn *nmgrNetNotifiee) natManager() *natManager {
	return (*natManager)(nn)
}

func (nn *nmgrNetNotifiee) Listen(n network.Network, addr ma.Multiaddr) {
	nn.natManager().sync()
}

func (nn *nmgrNetNotifiee) ListenClose(n network.Network, addr ma.Multiaddr) {
	nn.natManager().sync()
}

func (nn *nmgrNetNotifiee) Connected(network.Network, network.Conn)      {}
func (nn *nmgrNetNotifiee) Disconnected(network.Network, network.Conn)   {}
func (nn *nmgrNetNotifiee) OpenedStream(network.Network, network.Stream) {}
func (nn *nmgrNetNotifiee) ClosedStream(network.Network, network.Stream) {}
<|endoftext|>


<commit_before>package auth

// HostCredentialsFromMap converts a map of key-value pairs from a credentials
// definition provided by the user (e.g. in a config file, or via a credentials
// helper) into a HostCredentials object if possible, or returns nil if
// no credentials could be extracted from the map.
//
// This function ignores map keys it is unfamiliar with, to allow for future
// expansion of the credentials map format for new credential types.
func HostCredentialsFromMap(m map[string]interface{}) HostCredentials {
	if m == nil {
		return nil
	}
	if token, ok := m["token"].(string); ok {
		return HostCredentialsToken(token)
	}
	return nil
}
<commit_msg>command/cliconfig: New CredentialsSource implementation<commit_after>package auth

import (
	"github.com/zclconf/go-cty/cty"
)

// HostCredentialsFromMap converts a map of key-value pairs from a credentials
// definition provided by the user (e.g. in a config file, or via a credentials
// helper) into a HostCredentials object if possible, or returns nil if
// no credentials could be extracted from the map.
//
// This function ignores map keys it is unfamiliar with, to allow for future
// expansion of the credentials map format for new credential types.
func HostCredentialsFromMap(m map[string]interface{}) HostCredentials {
	if m == nil {
		return nil
	}
	if token, ok := m["token"].(string); ok {
		return HostCredentialsToken(token)
	}
	return nil
}

// HostCredentialsFromObject converts a cty.Value of an object type into a
// HostCredentials object if possible, or returns nil if no credentials could
// be extracted from the map.
//
// This function ignores object attributes it is unfamiliar with, to allow for
// future expansion of the credentials object structure for new credential types.
//
// If the given value is not of an object type, this function will panic.
func HostCredentialsFromObject(obj cty.Value) HostCredentials {
	if !obj.Type().HasAttribute("token") {
		return nil
	}

	tokenV := obj.GetAttr("token")
	if tokenV.IsNull() || !tokenV.IsKnown() {
		return nil
	}
	if !cty.String.Equals(tokenV.Type()) {
		// Weird, but maybe some future Terraform version accepts an object
		// here for some reason, so we'll be resilient.
		return nil
	}

	return HostCredentialsToken(tokenV.AsString())
}
<|endoftext|>


<commit_before>package minify // import "github.com/tdewolff/minify"

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

var errDummy = errors.New("dummy error")

// from os/exec/exec_test.go
func helperCommand(t *testing.T, s ...string) *exec.Cmd {
	cs := []string{"-test.run=TestHelperProcess", "--"}
	cs = append(cs, s...)
	cmd := exec.Command(os.Args[0], cs...)
	cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}
	return cmd
}

func helperMinifyString(t *testing.T, m *M, mediatype string) string {
	s, err := m.String(mediatype, "")
	assert.Nil(t, err, "minifier must not return error for '"+mediatype+"'")
	return s
}

////////////////////////////////////////////////////////////////

var m *M

func init() {
	m = New()
	m.AddFunc("dummy/copy", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		io.Copy(w, r)
		return nil
	})
	m.AddFunc("dummy/nil", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		return nil
	})
	m.AddFunc("dummy/err", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		return errDummy
	})
	m.AddFunc("dummy/charset", func(m *M, w io.Writer, r io.Reader, params map[string]string) error {
		w.Write([]byte(params["charset"]))
		return nil
	})
	m.AddFunc("dummy/params", func(m *M, w io.Writer, r io.Reader, params map[string]string) error {
		return m.Minify(params["type"]+"/"+params["sub"], w, r)
	})
	m.AddFunc("type/sub", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		w.Write([]byte("type/sub"))
		return nil
	})
	m.AddFuncRegexp(regexp.MustCompile("^type/.+$"), func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		w.Write([]byte("type/*"))
		return nil
	})
	m.AddFuncRegexp(regexp.MustCompile("^.+/.+$"), func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		w.Write([]byte("*/*"))
		return nil
	})
}

func TestMinify(t *testing.T) {
	assert.Equal(t, ErrNotExist, m.Minify("?", nil, nil), "must return ErrNotExist when minifier doesn't exist")
	assert.Nil(t, m.Minify("dummy/nil", nil, nil), "must return nil for dummy/nil")
	assert.Equal(t, errDummy, m.Minify("dummy/err", nil, nil), "must return errDummy for dummy/err")

	b := []byte("test")
	out, err := m.Bytes("dummy/nil", b)
	assert.Nil(t, err, "must not return error for dummy/nil")
	assert.Equal(t, []byte{}, out, "must return empty byte array for dummy/nil")
	out, err = m.Bytes("?", b)
	assert.Equal(t, ErrNotExist, err, "must return ErrNotExist when minifier doesn't exist")
	assert.Equal(t, b, out, "must return input byte array when minifier doesn't exist")

	s := "test"
	out2, err := m.String("dummy/nil", s)
	assert.Nil(t, err, "must not return error for dummy/nil")
	assert.Equal(t, "", out2, "must return empty string for dummy/nil")
	out2, err = m.String("?", s)
	assert.Equal(t, ErrNotExist, err, "must return ErrNotExist when minifier doesn't exist")
	assert.Equal(t, s, out2, "must return input string when minifier doesn't exist")
}

func TestAdd(t *testing.T) {
	m := New()
	w := &bytes.Buffer{}
	r := bytes.NewBufferString("test")
	m.AddFunc("dummy/err", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		return errDummy
	})
	assert.Equal(t, errDummy, m.Minify("dummy/err", nil, nil), "must return errDummy for dummy/err")

	m.AddCmd("dummy/copy", helperCommand(t, "dummy/copy"))
	m.AddCmd("dummy/err", helperCommand(t, "dummy/err"))
	m.AddCmdRegexp(regexp.MustCompile("err$"), helperCommand(t, "werr"))
	assert.Nil(t, m.Minify("dummy/copy", w, r), "must return nil for dummy/copy command")
	assert.Equal(t, "test", w.String(), "must return input string for dummy/copy command")
	assert.Equal(t, "exit status 1", m.Minify("dummy/err", w, r).Error(), "must return proper exit status when command encounters error")
	assert.Equal(t, "exit status 2", m.Minify("werr", w, r).Error(), "must return proper exit status when command encounters error")
	assert.Equal(t, "exit status 2", m.Minify("stderr", w, r).Error(), "must return proper exit status when command encounters error")
}

func TestWildcard(t *testing.T) {
	assert.Equal(t, "type/sub", helperMinifyString(t, m, "type/sub"), "must return type/sub for type/sub")
	assert.Equal(t, "type/*", helperMinifyString(t, m, "type/*"), "must return type/* for type/*")
	assert.Equal(t, "*/*", helperMinifyString(t, m, "*/*"), "must return */* for */*")
	assert.Equal(t, "type/*", helperMinifyString(t, m, "type/sub2"), "must return type/* for type/sub2")
	assert.Equal(t, "*/*", helperMinifyString(t, m, "type2/sub"), "must return */* for type2/sub")
	assert.Equal(t, "UTF-8", helperMinifyString(t, m, "dummy/charset;charset=UTF-8"), "must return UTF-8 for dummy/charset;charset=UTF-8")
	assert.Equal(t, "UTF-8", helperMinifyString(t, m, "dummy/charset; charset = UTF-8 "), "must return UTF-8 for ' dummy/charset; charset = UTF-8 '")
	assert.Equal(t, "type/sub", helperMinifyString(t, m, "dummy/params;type=type;sub=sub"), "must return type/sub for dummy/params;type=type;sub=sub")
}

func TestHelperProcess(*testing.T) {
	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
		return
	}
	args := os.Args
	for len(args) > 0 {
		if args[0] == "--" {
			args = args[1:]
			break
		}
		args = args[1:]
	}
	if len(args) == 0 {
		fmt.Fprintf(os.Stderr, "No command\n")
		os.Exit(2)
	}
	cmd, args := args[0], args[1:]
	switch cmd {
	case "dummy/copy":
		io.Copy(os.Stdout, os.Stdin)
	case "dummy/err":
		os.Exit(1)
	default:
		os.Exit(2)
	}
	os.Exit(0)
}

////////////////////////////////////////////////////////////////

func ExampleMinify_custom() {
	m := New()
	m.AddFunc("text/plain", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		// remove all newlines and spaces
		rb := bufio.NewReader(r)
		for {
			line, err := rb.ReadString('\n')
			if err != nil && err != io.EOF {
				return err
			}
			if _, errws := io.WriteString(w, strings.Replace(line, " ", "", -1)); errws != nil {
				return errws
			}
			if err == io.EOF {
				break
			}
		}
		return nil
	})

	in := "Because my coffee was too cold, I heated it in the microwave."
	out, err := m.String("text/plain", in)
	if err != nil {
		panic(err)
	}
	fmt.Println(out)
	// Output: Becausemycoffeewastoocold,Iheateditinthemicrowave.
}

func ExampleReader() {
	b := bytes.NewReader([]byte("input"))

	m := New()
	// add minfiers

	r := m.Reader("mime/type", b)
	if _, err := io.Copy(os.Stdout, r); err != nil {
		if _, err := io.Copy(os.Stdout, b); err != nil {
			panic(err)
		}
	}
}

func ExampleWriter() {
	m := New()
	// add minfiers

	w := m.Writer("mime/type", os.Stdout)
	if _, err := w.Write([]byte("input")); err != nil {
		panic(err)
	}
	if err := w.Close(); err != nil {
		panic(err)
	}
}

type MinifierResponseWriter struct {
	http.ResponseWriter
	io.Writer
}

func (m MinifierResponseWriter) Write(b []byte) (int, error) {
	return m.Writer.Write(b)
}

func ExampleMinify_responseWriter(res http.ResponseWriter) http.ResponseWriter {
	m := New()
	// add minfiers

	pr, pw := io.Pipe()
	go func(w io.Writer) {
		if err := m.Minify("mime/type", w, pr); err != nil {
			panic(err)
		}
	}(res)
	return MinifierResponseWriter{res, pw}
}
<commit_msg>Fix example names<commit_after>package minify // import "github.com/tdewolff/minify"

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

var errDummy = errors.New("dummy error")

// from os/exec/exec_test.go
func helperCommand(t *testing.T, s ...string) *exec.Cmd {
	cs := []string{"-test.run=TestHelperProcess", "--"}
	cs = append(cs, s...)
	cmd := exec.Command(os.Args[0], cs...)
	cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}
	return cmd
}

func helperMinifyString(t *testing.T, m *M, mediatype string) string {
	s, err := m.String(mediatype, "")
	assert.Nil(t, err, "minifier must not return error for '"+mediatype+"'")
	return s
}

////////////////////////////////////////////////////////////////

var m *M

func init() {
	m = New()
	m.AddFunc("dummy/copy", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		io.Copy(w, r)
		return nil
	})
	m.AddFunc("dummy/nil", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		return nil
	})
	m.AddFunc("dummy/err", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		return errDummy
	})
	m.AddFunc("dummy/charset", func(m *M, w io.Writer, r io.Reader, params map[string]string) error {
		w.Write([]byte(params["charset"]))
		return nil
	})
	m.AddFunc("dummy/params", func(m *M, w io.Writer, r io.Reader, params map[string]string) error {
		return m.Minify(params["type"]+"/"+params["sub"], w, r)
	})
	m.AddFunc("type/sub", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		w.Write([]byte("type/sub"))
		return nil
	})
	m.AddFuncRegexp(regexp.MustCompile("^type/.+$"), func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		w.Write([]byte("type/*"))
		return nil
	})
	m.AddFuncRegexp(regexp.MustCompile("^.+/.+$"), func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		w.Write([]byte("*/*"))
		return nil
	})
}

func TestMinify(t *testing.T) {
	assert.Equal(t, ErrNotExist, m.Minify("?", nil, nil), "must return ErrNotExist when minifier doesn't exist")
	assert.Nil(t, m.Minify("dummy/nil", nil, nil), "must return nil for dummy/nil")
	assert.Equal(t, errDummy, m.Minify("dummy/err", nil, nil), "must return errDummy for dummy/err")

	b := []byte("test")
	out, err := m.Bytes("dummy/nil", b)
	assert.Nil(t, err, "must not return error for dummy/nil")
	assert.Equal(t, []byte{}, out, "must return empty byte array for dummy/nil")
	out, err = m.Bytes("?", b)
	assert.Equal(t, ErrNotExist, err, "must return ErrNotExist when minifier doesn't exist")
	assert.Equal(t, b, out, "must return input byte array when minifier doesn't exist")

	s := "test"
	out2, err := m.String("dummy/nil", s)
	assert.Nil(t, err, "must not return error for dummy/nil")
	assert.Equal(t, "", out2, "must return empty string for dummy/nil")
	out2, err = m.String("?", s)
	assert.Equal(t, ErrNotExist, err, "must return ErrNotExist when minifier doesn't exist")
	assert.Equal(t, s, out2, "must return input string when minifier doesn't exist")
}

func TestAdd(t *testing.T) {
	m := New()
	w := &bytes.Buffer{}
	r := bytes.NewBufferString("test")
	m.AddFunc("dummy/err", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		return errDummy
	})
	assert.Equal(t, errDummy, m.Minify("dummy/err", nil, nil), "must return errDummy for dummy/err")

	m.AddCmd("dummy/copy", helperCommand(t, "dummy/copy"))
	m.AddCmd("dummy/err", helperCommand(t, "dummy/err"))
	m.AddCmdRegexp(regexp.MustCompile("err$"), helperCommand(t, "werr"))
	assert.Nil(t, m.Minify("dummy/copy", w, r), "must return nil for dummy/copy command")
	assert.Equal(t, "test", w.String(), "must return input string for dummy/copy command")
	assert.Equal(t, "exit status 1", m.Minify("dummy/err", w, r).Error(), "must return proper exit status when command encounters error")
	assert.Equal(t, "exit status 2", m.Minify("werr", w, r).Error(), "must return proper exit status when command encounters error")
	assert.Equal(t, "exit status 2", m.Minify("stderr", w, r).Error(), "must return proper exit status when command encounters error")
}

func TestWildcard(t *testing.T) {
	assert.Equal(t, "type/sub", helperMinifyString(t, m, "type/sub"), "must return type/sub for type/sub")
	assert.Equal(t, "type/*", helperMinifyString(t, m, "type/*"), "must return type/* for type/*")
	assert.Equal(t, "*/*", helperMinifyString(t, m, "*/*"), "must return */* for */*")
	assert.Equal(t, "type/*", helperMinifyString(t, m, "type/sub2"), "must return type/* for type/sub2")
	assert.Equal(t, "*/*", helperMinifyString(t, m, "type2/sub"), "must return */* for type2/sub")
	assert.Equal(t, "UTF-8", helperMinifyString(t, m, "dummy/charset;charset=UTF-8"), "must return UTF-8 for dummy/charset;charset=UTF-8")
	assert.Equal(t, "UTF-8", helperMinifyString(t, m, "dummy/charset; charset = UTF-8 "), "must return UTF-8 for ' dummy/charset; charset = UTF-8 '")
	assert.Equal(t, "type/sub", helperMinifyString(t, m, "dummy/params;type=type;sub=sub"), "must return type/sub for dummy/params;type=type;sub=sub")
}

func TestHelperProcess(*testing.T) {
	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
		return
	}
	args := os.Args
	for len(args) > 0 {
		if args[0] == "--" {
			args = args[1:]
			break
		}
		args = args[1:]
	}
	if len(args) == 0 {
		fmt.Fprintf(os.Stderr, "No command\n")
		os.Exit(2)
	}
	cmd, args := args[0], args[1:]
	switch cmd {
	case "dummy/copy":
		io.Copy(os.Stdout, os.Stdin)
	case "dummy/err":
		os.Exit(1)
	default:
		os.Exit(2)
	}
	os.Exit(0)
}

////////////////////////////////////////////////////////////////

func ExampleM_Minify_custom() {
	m := New()
	m.AddFunc("text/plain", func(m *M, w io.Writer, r io.Reader, _ map[string]string) error {
		// remove all newlines and spaces
		rb := bufio.NewReader(r)
		for {
			line, err := rb.ReadString('\n')
			if err != nil && err != io.EOF {
				return err
			}
			if _, errws := io.WriteString(w, strings.Replace(line, " ", "", -1)); errws != nil {
				return errws
			}
			if err == io.EOF {
				break
			}
		}
		return nil
	})

	in := "Because my coffee was too cold, I heated it in the microwave."
	out, err := m.String("text/plain", in)
	if err != nil {
		panic(err)
	}
	fmt.Println(out)
	// Output: Becausemycoffeewastoocold,Iheateditinthemicrowave.
}

func ExampleM_Reader() {
	b := bytes.NewReader([]byte("input"))

	m := New()
	// add minfiers

	r := m.Reader("mime/type", b)
	if _, err := io.Copy(os.Stdout, r); err != nil {
		if _, err := io.Copy(os.Stdout, b); err != nil {
			panic(err)
		}
	}
}

func ExampleM_Writer() {
	m := New()
	// add minfiers

	w := m.Writer("mime/type", os.Stdout)
	if _, err := w.Write([]byte("input")); err != nil {
		panic(err)
	}
	if err := w.Close(); err != nil {
		panic(err)
	}
}

type MinifierResponseWriter struct {
	http.ResponseWriter
	io.Writer
}

func (m MinifierResponseWriter) Write(b []byte) (int, error) {
	return m.Writer.Write(b)
}

func ExampleM_Minify_responseWriter(res http.ResponseWriter) http.ResponseWriter {
	m := New()
	// add minfiers

	pr, pw := io.Pipe()
	go func(w io.Writer) {
		if err := m.Minify("mime/type", w, pr); err != nil {
			panic(err)
		}
	}(res)
	return MinifierResponseWriter{res, pw}
}
<|endoftext|>


<commit_before>package minion

import (
	"os"
	"os/exec"
	"os/signal"
	"log"
	"bytes"
	"time"
	"strings"
	"strconv"
	"path/filepath"
	"encoding/json"

	"code.google.com/p/go-uuid/uuid"
	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
	etcdclient "github.com/coreos/etcd/client"
)

// Key spaces in etcd
const EtcdRootKeySpace = "/gru"

// Global keyspace
var EtcdGlobalSpace = filepath.Join(EtcdRootKeySpace, "global")

// Global keyspace for task logs
var EtcdGlobalLogSpace = filepath.Join(EtcdGlobalSpace, "log")

// Global keyspace for classifiers
var EtcdGlobalClassifierSpace = filepath.Join(EtcdGlobalSpace, "classifier")

// Keyspace where minions store their configuration
var EtcdMinionSpace = filepath.Join(EtcdRootKeySpace, "minion")

// Etcd Minion
type EtcdMinion struct {
	// Name of this minion
	Name string

	// Minion root node in etcd
	MinionRootDir string

	// Minion queue node in etcd
	QueueDir string

	// Log directory to keep previously executed tasks
	LogDir string

	// Root node for classifiers in etcd
	ClassifierDir string

	// Minion unique identifier
	UUID uuid.UUID

	// KeysAPI client to etcd
	KAPI etcdclient.KeysAPI
}

// Etcd minion task
type EtcdTask struct {
	// Command to be executed by the minion
	Command string

	// Command arguments
	Args []string

	// Time when the command was sent for processing
	TimeReceived int64

	// Time when the command was processed
	TimeProcessed int64

	// Task unique identifier
	TaskID uuid.UUID

	// Result of task after processing
	Result string

	// If true this task can run concurrently with other tasks
	TaskIsConcurrent bool

	// Task error, if any
	Error string
}

// Unmarshals task from etcd and removes it from the queue
func UnmarshalEtcdTask(node *etcdclient.Node) (*EtcdTask, error) {
	task := new(EtcdTask)
	err := json.Unmarshal([]byte(node.Value), &task)

	if err != nil {
		log.Printf("Invalid task: key: %s\n", node.Key)
		log.Printf("Invalid task: value: %s\n", node.Value)
		log.Printf("Invalid task: error: %s\n", err)
	}

	return task, err
}

func NewEtcdTask(command string, args ...string) MinionTask {
	t := &EtcdTask{
		Command: command,
		Args: args,
		TimeReceived: time.Now().Unix(),
		TaskID: uuid.NewRandom(),
	}

	return t
}

// Gets the task unique identifier
func (t *EtcdTask) GetTaskID() uuid.UUID {
	return t.TaskID
}

// Gets the task command to be executed
func (t *EtcdTask) GetCommand() (string, error) {
	return t.Command, nil
}

// Gets the task arguments
func (t *EtcdTask) GetArgs() ([]string, error) {
	return t.Args, nil
}

// Returns the time a task has been received for processing
func (t *EtcdTask) GetTimeReceived() (int64, error) {
	return t.TimeReceived, nil
}

// Returns the time when a task has been processed
func (t *EtcdTask) GetTimeProcessed() (int64, error) {
	return t.TimeProcessed, nil
}

// Returns the result of the task
func (t *EtcdTask) GetResult() (string, error) {
	return t.Result, nil
}

// Returns the task error, if any
func (t *EtcdTask) GetError() string {
	return t.Error
}

// Returns a boolean whether or not the task can run
// concurrently with other tasks
func (t *EtcdTask) IsConcurrent() bool {
	return t.TaskIsConcurrent
}

// Sets the flag whether or not this task can run
// concurrently with other tasks
func (t *EtcdTask) SetConcurrent(c bool) error {
	t.TaskIsConcurrent = c

	return nil
}

// Processes a task
func (t *EtcdTask) Process() error {
	var buf bytes.Buffer
	taskID := t.GetTaskID()
	command, _ := t.GetCommand()
	args, _ := t.GetArgs()
	cmd := exec.Command(command, args...)
	cmd.Stdout = &buf
	cmd.Stderr = &buf

	log.Printf("Processing task %s\n", taskID)

	cmdError := cmd.Run()
	t.TimeProcessed = time.Now().Unix()
	t.Result = buf.String()

	if cmdError != nil {
		log.Printf("Failed to process task %s\n", taskID)
		t.Error = cmdError.Error()
	} else {
		log.Printf("Finished processing task %s\n", taskID)
	}

	return cmdError
}

// Creates a new etcd minion
func NewEtcdMinion(name string, cfg etcdclient.Config) Minion {
	c, err := etcdclient.New(cfg)
	if err != nil {
		log.Fatal(err)
	}

	kapi := etcdclient.NewKeysAPI(c)
	minionUUID := GenerateUUID(name)
	minionRootDir := filepath.Join(EtcdMinionSpace, minionUUID.String())
	queueDir := filepath.Join(minionRootDir, "queue")
	classifierDir := filepath.Join(minionRootDir, "classifier")
	logDir := filepath.Join(minionRootDir, "log")

	log.Printf("Created minion with uuid %s\n", minionUUID)

	m := &EtcdMinion{
		Name: name,
		MinionRootDir: minionRootDir,
		QueueDir: queueDir,
		ClassifierDir: classifierDir,
		LogDir: logDir,
		UUID: minionUUID,
		KAPI: kapi,
	}

	return m
}

// Get the minion UUID
func (m *EtcdMinion) GetUUID() uuid.UUID {
	return m.UUID
}

// Get the human-readable name of the minion
func (m *EtcdMinion) GetName() (string, error) {
	return m.Name, nil
}

// Set the human-readable name of the minion
func (m *EtcdMinion) SetName(name string) error {
	key := filepath.Join(m.MinionRootDir, "name")
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.KAPI.Set(context.Background(), key, m.Name, opts)

	return err
}

// Set the time the minion was last seen in seconds since the Epoch
func (m *EtcdMinion) SetLastseen(s int64) error {
	key := filepath.Join(m.MinionRootDir, "lastseen")
	lastseen := strconv.FormatInt(s, 10)
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.KAPI.Set(context.Background(), key, lastseen, opts)

	return err
}

// Get classifier for a minion
func (m *EtcdMinion) GetClassifier(key string) (MinionClassifier, error) {
	klassifier := new(SimpleClassifier)
	klassifierNode := filepath.Join(m.ClassifierDir, key, "info")

	// Get classifier from etcd and deserialize
	resp, err := m.KAPI.Get(context.Background(), klassifierNode, nil)

	if err != nil {
		return klassifier, err
	}

	err = json.Unmarshal([]byte(resp.Node.Value), &klassifier)

	return klassifier, err
}

// Classify a minion with a given key and value
func (m *EtcdMinion) SetClassifier(c MinionClassifier) error {
	// Classifiers in etcd expire after an hour
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
		TTL: time.Hour,
	}

	// Get classifier values
	classifierKey, err := c.GetKey()
	classifierDescription, err := c.GetDescription()
	classifierValue, err := c.GetValue(m)

	if err != nil {
		return err
	}

	// Create a simple classifier and serialize to JSON
	klassifier := NewSimpleClassifier(classifierKey, classifierValue, classifierDescription)
	data, err := json.Marshal(klassifier)

	if err != nil {
		log.Printf("Failed to serialize classifier: %s\n", classifierKey)
		return err
	}

	// Set classifier in the minion's space
	klassifierNode := filepath.Join(m.ClassifierDir, classifierKey)
	_, err = m.KAPI.Set(context.Background(), klassifierNode, string(data), opts)

	if err != nil {
		log.Printf("Failed to set classifier %s: %s\n", classifierKey, err)
	}

	// Set a classifier reference in the global classifier space as well
	// The global classifier is simply a reference to the minion's classifier key
	klassifierRef := filepath.Join(EtcdGlobalClassifierSpace, classifierKey, m.UUID.String())
	_, err = m.KAPI.Set(context.Background(), klassifierRef, klassifierNode, opts)

	if err != nil {
		log.Printf("Failed to set global classifier ref %s: %s\n", classifierKey, err)
	}

	return err
}

// Runs periodic jobs such as refreshing classifiers and updating lastseen
func (m *EtcdMinion) PeriodicRunner(ticker *time.Ticker) error {
	for {
		// Update classifiers
		for _, classifier := range ClassifierRegistry {
			m.SetClassifier(classifier)
		}

		// Update lastseen time
		now := time.Now().Unix()
		err := m.SetLastseen(now)
		if err != nil {
			log.Printf("Failed to update lastseen time: %s\n", err)
		}

		<- ticker.C
	}

	return nil
}

// Monitors etcd for new tasks for processing
func (m *EtcdMinion) TaskListener(c chan<- MinionTask) error {
	log.Printf("Starting task listener")

	watcherOpts := &etcdclient.WatcherOptions{
		Recursive: true,
	}
	watcher := m.KAPI.Watcher(m.QueueDir, watcherOpts)

	for {
		resp, err := watcher.Next(context.Background())
		if err != nil {
			log.Printf("Failed to read task: %s\n", err)
			continue
		}

		// Ignore "delete" events when removing a task from the queue
		action := strings.ToLower(resp.Action)
		if strings.EqualFold(action, "delete") {
			continue
		}

		task, err := UnmarshalEtcdTask(resp.Node)
		m.KAPI.Delete(context.Background(), resp.Node.Key, nil)

		if err != nil {
			continue
		}

		log.Printf("Received task %s\n", task.GetTaskID())

		c <- task
	}

	return nil
}

// Processes new tasks
func (m *EtcdMinion) TaskRunner(c <-chan MinionTask) error {
	for {
		task := <-c

		// TODO: Run task concurrently with others if task.IsConcurrent()
		task.Process()
		m.SaveTaskResult(task)
	}

	return nil
}

// Saves a task in the minion's log of previously executed tasks
func (m *EtcdMinion) SaveTaskResult(t MinionTask) error {
	taskID := t.GetTaskID()
	taskNode := filepath.Join(m.LogDir, taskID.String())

	// Serialize task to JSON
	data, err := json.Marshal(t)
	if err != nil {
		log.Printf("Failed to serialize task %s: %s\n", taskID, err)
		return err
	}

	// Save task result in the minion's space
	_, err = m.KAPI.Create(context.Background(), taskNode, string(data))
	if err != nil {
		log.Printf("Failed to save task %s: %s\n", taskID, err)
		return err
	}

	// Save a reference to the task result in the global keyspace as well
	// This is the place where all minions that were classified to process
	// this task would put a reference to their result, so that we can
	// get the results of all minions from a single location instead of
	// recursing through all minions
	taskRef := filepath.Join(EtcdGlobalLogSpace, taskID.String(), m.UUID.String())
	_, err = m.KAPI.Create(context.Background(), taskRef, taskNode)

	if err != nil {
		log.Printf("Failed to set global task ref for %s: %s\n", taskID, err)
	}

	return err
}

// Checks for any tasks in backlog
func (m *EtcdMinion) CheckForBacklog(c chan<- MinionTask) error {
	opts := &etcdclient.GetOptions{
		Recursive: true,
		Sort: true,
	}

	// Get backlog tasks if any
	resp, err := m.KAPI.Get(context.Background(), m.QueueDir, opts)
	if err != nil {
		return nil
	}

	backlog := resp.Node.Nodes

	if len(backlog) == 0 {
		// No backlog tasks found
		return nil
	}

	log.Printf("Found %d tasks in backlog", len(backlog))
	for _, node := range backlog {
		task, err := UnmarshalEtcdTask(node)
		m.KAPI.Delete(context.Background(), node.Key, nil)

		if err != nil {
			continue
		}

		c <- task
	}

	return nil
}

// Main entry point of the minion
func (m *EtcdMinion) Serve() error {
	// Set name of the minion
	m.SetName(m.Name)

	// Channel on which we send the quit signal
	quit := make(chan os.Signal, 1)
	signal.Notify(quit, os.Interrupt)

	// Run any periodic tasks every hour
	ticker := time.NewTicker(time.Minute * 15)
	go m.PeriodicRunner(ticker)

	// Check for backlog tasks and start task listener and runner
	tasks := make(chan MinionTask)
	go m.TaskListener(tasks)
	go m.CheckForBacklog(tasks)
	go m.TaskRunner(tasks)

	// Block until a stop signal is received
	s := <-quit
	log.Printf("Received %s signal, shutting down", s)
	close(quit)
	close(tasks)

	return nil
}

<commit_msg>Be able to run concurrent tasks<commit_after>package minion

import (
	"os"
	"os/exec"
	"os/signal"
	"log"
	"bytes"
	"time"
	"strings"
	"strconv"
	"path/filepath"
	"encoding/json"

	"code.google.com/p/go-uuid/uuid"
	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
	etcdclient "github.com/coreos/etcd/client"
)

// Key spaces in etcd
const EtcdRootKeySpace = "/gru"

// Global keyspace
var EtcdGlobalSpace = filepath.Join(EtcdRootKeySpace, "global")

// Global keyspace for task logs
var EtcdGlobalLogSpace = filepath.Join(EtcdGlobalSpace, "log")

// Global keyspace for classifiers
var EtcdGlobalClassifierSpace = filepath.Join(EtcdGlobalSpace, "classifier")

// Keyspace where minions store their configuration
var EtcdMinionSpace = filepath.Join(EtcdRootKeySpace, "minion")

// Etcd Minion
type EtcdMinion struct {
	// Name of this minion
	Name string

	// Minion root node in etcd
	MinionRootDir string

	// Minion queue node in etcd
	QueueDir string

	// Log directory to keep previously executed tasks
	LogDir string

	// Root node for classifiers in etcd
	ClassifierDir string

	// Minion unique identifier
	UUID uuid.UUID

	// KeysAPI client to etcd
	KAPI etcdclient.KeysAPI
}

// Etcd minion task
type EtcdTask struct {
	// Command to be executed by the minion
	Command string

	// Command arguments
	Args []string

	// Time when the command was sent for processing
	TimeReceived int64

	// Time when the command was processed
	TimeProcessed int64

	// Task unique identifier
	TaskID uuid.UUID

	// Result of task after processing
	Result string

	// If true this task can run concurrently with other tasks
	TaskIsConcurrent bool

	// Task error, if any
	Error string
}

// Unmarshals task from etcd and removes it from the queue
func UnmarshalEtcdTask(node *etcdclient.Node) (*EtcdTask, error) {
	task := new(EtcdTask)
	err := json.Unmarshal([]byte(node.Value), &task)

	if err != nil {
		log.Printf("Invalid task: key: %s\n", node.Key)
		log.Printf("Invalid task: value: %s\n", node.Value)
		log.Printf("Invalid task: error: %s\n", err)
	}

	return task, err
}

func NewEtcdTask(command string, args ...string) MinionTask {
	t := &EtcdTask{
		Command: command,
		Args: args,
		TimeReceived: time.Now().Unix(),
		TaskID: uuid.NewRandom(),
	}

	return t
}

// Gets the task unique identifier
func (t *EtcdTask) GetTaskID() uuid.UUID {
	return t.TaskID
}

// Gets the task command to be executed
func (t *EtcdTask) GetCommand() (string, error) {
	return t.Command, nil
}

// Gets the task arguments
func (t *EtcdTask) GetArgs() ([]string, error) {
	return t.Args, nil
}

// Returns the time a task has been received for processing
func (t *EtcdTask) GetTimeReceived() (int64, error) {
	return t.TimeReceived, nil
}

// Returns the time when a task has been processed
func (t *EtcdTask) GetTimeProcessed() (int64, error) {
	return t.TimeProcessed, nil
}

// Returns the result of the task
func (t *EtcdTask) GetResult() (string, error) {
	return t.Result, nil
}

// Returns the task error, if any
func (t *EtcdTask) GetError() string {
	return t.Error
}

// Returns a boolean whether or not the task can run
// concurrently with other tasks
func (t *EtcdTask) IsConcurrent() bool {
	return t.TaskIsConcurrent
}

// Sets the flag whether or not this task can run
// concurrently with other tasks
func (t *EtcdTask) SetConcurrent(c bool) error {
	t.TaskIsConcurrent = c

	return nil
}

// Processes a task
func (t *EtcdTask) Process() error {
	var buf bytes.Buffer
	taskID := t.GetTaskID()
	command, _ := t.GetCommand()
	args, _ := t.GetArgs()
	cmd := exec.Command(command, args...)
	cmd.Stdout = &buf
	cmd.Stderr = &buf

	log.Printf("Processing task %s\n", taskID)

	cmdError := cmd.Run()
	t.TimeProcessed = time.Now().Unix()
	t.Result = buf.String()

	if cmdError != nil {
		log.Printf("Failed to process task %s\n", taskID)
		t.Error = cmdError.Error()
	} else {
		log.Printf("Finished processing task %s\n", taskID)
	}

	return cmdError
}

// Creates a new etcd minion
func NewEtcdMinion(name string, cfg etcdclient.Config) Minion {
	c, err := etcdclient.New(cfg)
	if err != nil {
		log.Fatal(err)
	}

	kapi := etcdclient.NewKeysAPI(c)
	minionUUID := GenerateUUID(name)
	minionRootDir := filepath.Join(EtcdMinionSpace, minionUUID.String())
	queueDir := filepath.Join(minionRootDir, "queue")
	classifierDir := filepath.Join(minionRootDir, "classifier")
	logDir := filepath.Join(minionRootDir, "log")

	log.Printf("Created minion with uuid %s\n", minionUUID)

	m := &EtcdMinion{
		Name: name,
		MinionRootDir: minionRootDir,
		QueueDir: queueDir,
		ClassifierDir: classifierDir,
		LogDir: logDir,
		UUID: minionUUID,
		KAPI: kapi,
	}

	return m
}

// Get the minion UUID
func (m *EtcdMinion) GetUUID() uuid.UUID {
	return m.UUID
}

// Get the human-readable name of the minion
func (m *EtcdMinion) GetName() (string, error) {
	return m.Name, nil
}

// Set the human-readable name of the minion
func (m *EtcdMinion) SetName(name string) error {
	key := filepath.Join(m.MinionRootDir, "name")
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.KAPI.Set(context.Background(), key, m.Name, opts)

	return err
}

// Set the time the minion was last seen in seconds since the Epoch
func (m *EtcdMinion) SetLastseen(s int64) error {
	key := filepath.Join(m.MinionRootDir, "lastseen")
	lastseen := strconv.FormatInt(s, 10)
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.KAPI.Set(context.Background(), key, lastseen, opts)

	return err
}

// Get classifier for a minion
func (m *EtcdMinion) GetClassifier(key string) (MinionClassifier, error) {
	klassifier := new(SimpleClassifier)
	klassifierNode := filepath.Join(m.ClassifierDir, key, "info")

	// Get classifier from etcd and deserialize
	resp, err := m.KAPI.Get(context.Background(), klassifierNode, nil)

	if err != nil {
		return klassifier, err
	}

	err = json.Unmarshal([]byte(resp.Node.Value), &klassifier)

	return klassifier, err
}

// Classify a minion with a given key and value
func (m *EtcdMinion) SetClassifier(c MinionClassifier) error {
	// Classifiers in etcd expire after an hour
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
		TTL: time.Hour,
	}

	// Get classifier values
	classifierKey, err := c.GetKey()
	classifierDescription, err := c.GetDescription()
	classifierValue, err := c.GetValue(m)

	if err != nil {
		return err
	}

	// Create a simple classifier and serialize to JSON
	klassifier := NewSimpleClassifier(classifierKey, classifierValue, classifierDescription)
	data, err := json.Marshal(klassifier)

	if err != nil {
		log.Printf("Failed to serialize classifier: %s\n", classifierKey)
		return err
	}

	// Set classifier in the minion's space
	klassifierNode := filepath.Join(m.ClassifierDir, classifierKey)
	_, err = m.KAPI.Set(context.Background(), klassifierNode, string(data), opts)

	if err != nil {
		log.Printf("Failed to set classifier %s: %s\n", classifierKey, err)
	}

	// Set a classifier reference in the global classifier space as well
	// The global classifier is simply a reference to the minion's classifier key
	klassifierRef := filepath.Join(EtcdGlobalClassifierSpace, classifierKey, m.UUID.String())
	_, err = m.KAPI.Set(context.Background(), klassifierRef, klassifierNode, opts)

	if err != nil {
		log.Printf("Failed to set global classifier ref %s: %s\n", classifierKey, err)
	}

	return err
}

// Runs periodic jobs such as refreshing classifiers and updating lastseen
func (m *EtcdMinion) PeriodicRunner(ticker *time.Ticker) error {
	for {
		// Update classifiers
		for _, classifier := range ClassifierRegistry {
			m.SetClassifier(classifier)
		}

		// Update lastseen time
		now := time.Now().Unix()
		err := m.SetLastseen(now)
		if err != nil {
			log.Printf("Failed to update lastseen time: %s\n", err)
		}

		<- ticker.C
	}

	return nil
}

// Monitors etcd for new tasks for processing
func (m *EtcdMinion) TaskListener(c chan<- MinionTask) error {
	log.Printf("Starting task listener")

	watcherOpts := &etcdclient.WatcherOptions{
		Recursive: true,
	}
	watcher := m.KAPI.Watcher(m.QueueDir, watcherOpts)

	for {
		resp, err := watcher.Next(context.Background())
		if err != nil {
			log.Printf("Failed to read task: %s\n", err)
			continue
		}

		// Ignore "delete" events when removing a task from the queue
		action := strings.ToLower(resp.Action)
		if strings.EqualFold(action, "delete") {
			continue
		}

		// Remove task from the queue
		task, err := UnmarshalEtcdTask(resp.Node)
		m.KAPI.Delete(context.Background(), resp.Node.Key, nil)

		if err != nil {
			continue
		}

		log.Printf("Received task %s\n", task.GetTaskID())

		c <- task
	}

	return nil
}

// Processes new tasks
func (m *EtcdMinion) TaskRunner(c <-chan MinionTask) error {
	for {
		task := <-c

		runTask := func() {
			task.Process()
			m.SaveTaskResult(task)
		}

		if task.IsConcurrent() {
			go runTask()
		} else {
			runTask()
		}
	}

	return nil
}

// Saves a task in the minion's log of previously executed tasks
func (m *EtcdMinion) SaveTaskResult(t MinionTask) error {
	taskID := t.GetTaskID()
	taskNode := filepath.Join(m.LogDir, taskID.String())

	// Serialize task to JSON
	data, err := json.Marshal(t)
	if err != nil {
		log.Printf("Failed to serialize task %s: %s\n", taskID, err)
		return err
	}

	// Save task result in the minion's space
	_, err = m.KAPI.Create(context.Background(), taskNode, string(data))
	if err != nil {
		log.Printf("Failed to save task %s: %s\n", taskID, err)
		return err
	}

	// Save a reference to the task result in the global keyspace as well
	// This is the place where all minions that were classified to process
	// this task would put a reference to their result, so that we can
	// get the results of all minions from a single location instead of
	// recursing through all minions
	taskRef := filepath.Join(EtcdGlobalLogSpace, taskID.String(), m.UUID.String())
	_, err = m.KAPI.Create(context.Background(), taskRef, taskNode)

	if err != nil {
		log.Printf("Failed to set global task ref for %s: %s\n", taskID, err)
	}

	return err
}

// Checks for any tasks in backlog
func (m *EtcdMinion) CheckForBacklog(c chan<- MinionTask) error {
	opts := &etcdclient.GetOptions{
		Recursive: true,
		Sort: true,
	}

	// Get backlog tasks if any
	resp, err := m.KAPI.Get(context.Background(), m.QueueDir, opts)
	if err != nil {
		return nil
	}

	backlog := resp.Node.Nodes

	if len(backlog) == 0 {
		// No backlog tasks found
		return nil
	}

	log.Printf("Found %d tasks in backlog", len(backlog))
	for _, node := range backlog {
		task, err := UnmarshalEtcdTask(node)
		m.KAPI.Delete(context.Background(), node.Key, nil)

		if err != nil {
			continue
		}

		c <- task
	}

	return nil
}

// Main entry point of the minion
func (m *EtcdMinion) Serve() error {
	// Set name of the minion
	m.SetName(m.Name)

	// Channel on which we send the quit signal
	quit := make(chan os.Signal, 1)
	signal.Notify(quit, os.Interrupt)

	// Run any periodic tasks every hour
	ticker := time.NewTicker(time.Minute * 15)
	go m.PeriodicRunner(ticker)

	// Check for backlog tasks and start task listener and runner
	tasks := make(chan MinionTask)
	go m.TaskListener(tasks)
	go m.CheckForBacklog(tasks)
	go m.TaskRunner(tasks)

	// Block until a stop signal is received
	s := <-quit
	log.Printf("Received %s signal, shutting down", s)
	close(quit)
	close(tasks)

	return nil
}

<|endoftext|>


<commit_before>// Copyright (c) 2014-2019 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package transactionrecord_test

import (
	"bytes"
	"encoding/json"
	"reflect"
	"testing"

	"golang.org/x/crypto/ed25519"

	"github.com/bitmark-inc/bitmarkd/fault"
	"github.com/bitmark-inc/bitmarkd/merkle"
	"github.com/bitmark-inc/bitmarkd/transactionrecord"
	"github.com/bitmark-inc/bitmarkd/util"
)

// test the packing/unpacking of Share balance record
//
// ensures that pack->unpack returns the same original value
func TestPackShareBalanceOne(t *testing.T) {

	ownerOneAccount := makeAccount(ownerOne.publicKey)

	var link merkle.Digest
	err := merkleDigestFromLE("79a67be2b3d313bd490363fb0d27901c46ed53d3f7b21f60d48bc42439b06084", &link)
	if nil != err {
		t.Fatalf("hex to link error: %s", err)
	}

	r := transactionrecord.BitmarkShare{
		Link:     link,
		Quantity: 12345,
	}

	expected := []byte{
		0x08, 0x20, 0x79, 0xa6, 0x7b, 0xe2, 0xb3, 0xd3,
		0x13, 0xbd, 0x49, 0x03, 0x63, 0xfb, 0x0d, 0x27,
		0x90, 0x1c, 0x46, 0xed, 0x53, 0xd3, 0xf7, 0xb2,
		0x1f, 0x60, 0xd4, 0x8b, 0xc4, 0x24, 0x39, 0xb0,
		0x60, 0x84, 0xb9, 0x60,
	}

	expectedTxId := merkle.Digest{
		0x68, 0x95, 0x6b, 0x9a, 0x91, 0x0f, 0xaa, 0x55,
		0xf3, 0x3a, 0xcb, 0xa1, 0x17, 0x08, 0x6c, 0x2f,
		0x2d, 0x83, 0x7c, 0xba, 0x9f, 0x80, 0x79, 0x87,
		0x2a, 0x4e, 0xeb, 0x65, 0x6a, 0x42, 0xeb, 0x83,
	}

	// manually sign the record and attach signature to "expected"
	signature := ed25519.Sign(ownerOne.privateKey, expected)
	r.Signature = signature
	l := util.ToVarint64(uint64(len(signature)))
	expected = append(expected, l...)
	expected = append(expected, signature...)

	// test the packer
	packed, err := r.Pack(ownerOneAccount)
	if nil != err {
		t.Errorf("pack error: %s", err)
	}

	// if either of above fail we will have the message _without_ a signature
	if !bytes.Equal(packed, expected) {
		t.Errorf("pack record: %x  expected: %x", packed, expected)
		t.Errorf("*** GENERATED Packed:\n%s", util.FormatBytes("expected", packed))
		t.Fatal("fatal error")
	}

	t.Logf("Packed length: %d bytes", len(packed))

	// check txId
	txId := packed.MakeLink()

	if txId != expectedTxId {
		t.Errorf("pack txId: %#v  expected: %x", txId, expectedTxId)
		t.Errorf("*** GENERATED txId:\n%s", util.FormatBytes("expectedTxId", txId[:]))
		t.Fatal("fatal error")
	}

	// test the unpacker
	unpacked, n, err := packed.Unpack(true)
	if nil != err {
		t.Fatalf("unpack error: %s", err)
	}
	if len(packed) != n {
		t.Errorf("did not unpack all data: only used: %d of: %d bytes", n, len(packed))
	}

	balance, ok := unpacked.(*transactionrecord.BitmarkShare)
	if !ok {
		t.Fatalf("did not unpack to BitmarkShare")
	}

	// display a JSON version for information
	item := struct {
		TxId         merkle.Digest
		ShareBalance *transactionrecord.BitmarkShare
	}{
		txId,
		balance,
	}
	b, err := json.MarshalIndent(item, "", "  ")
	if nil != err {
		t.Fatalf("json error: %s", err)
	}

	t.Logf("Share Balance: JSON: %s", b)

	// check that structure is preserved through Pack/Unpack
	// note reg is a pointer here
	if !reflect.DeepEqual(r, *balance) {
		t.Fatalf("different, original: %v  recovered: %v", r, *balance)
	}
}

// test the packing/unpacking of Share balance record
//
// ensures that zero value fails
func TestPackShareBalanceValueNotZero(t *testing.T) {

	ownerOneAccount := makeAccount(ownerOne.publicKey)

	var link merkle.Digest
	err := merkleDigestFromLE("79a67be2b3d313bd490363fb0d27901c46ed53d3f7b21f60d48bc42439b06084", &link)
	if nil != err {
		t.Fatalf("hex to link error: %s", err)
	}

	r := transactionrecord.BitmarkShare{
		Link:     link,
		Quantity: 0,
	}

	expected := []byte{
		0x05, 0x20, 0x14, 0xeb, 0x10, 0x3a, 0x0c, 0x8f,
		0xb2, 0x2e, 0x50, 0xe7, 0x3a, 0xe9, 0xb4, 0xff,
		0x88, 0x59, 0x5b, 0x1c, 0xd5, 0xf6, 0x0c, 0x4a,
		0xfb, 0x69, 0x0d, 0x8f, 0xbd, 0x01, 0x4c, 0x3e,
		0xd0, 0x91, 0x00, 0x21, 0x13, 0x27, 0x64, 0x0e,
		0x4a, 0xab, 0x92, 0xd8, 0x7b, 0x4a, 0x6a, 0x2f,
		0x30, 0xb8, 0x81, 0xf4, 0x49, 0x29, 0xf8, 0x66,
		0x04, 0x3a, 0x84, 0x1c, 0x38, 0x14, 0xb1, 0x66,
		0xb8, 0x89, 0x44, 0xb0, 0x92,
	}

	// manually sign the record and attach signature to "expected"
	signature := ed25519.Sign(ownerTwo.privateKey, expected)
	r.Signature = signature

	// test the packer
	_, err = r.Pack(ownerOneAccount)
	if fault.ErrShareQuantityTooSmall != err {
		t.Fatalf("unexpected pack error: %s", err)
	}
}
<commit_msg>- Revert to the right signature.<commit_after>// Copyright (c) 2014-2019 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package transactionrecord_test

import (
	"bytes"
	"encoding/json"
	"reflect"
	"testing"

	"golang.org/x/crypto/ed25519"

	"github.com/bitmark-inc/bitmarkd/fault"
	"github.com/bitmark-inc/bitmarkd/merkle"
	"github.com/bitmark-inc/bitmarkd/transactionrecord"
	"github.com/bitmark-inc/bitmarkd/util"
)

// test the packing/unpacking of Share balance record
//
// ensures that pack->unpack returns the same original value
func TestPackShareBalanceOne(t *testing.T) {

	ownerOneAccount := makeAccount(ownerOne.publicKey)

	var link merkle.Digest
	err := merkleDigestFromLE("79a67be2b3d313bd490363fb0d27901c46ed53d3f7b21f60d48bc42439b06084", &link)
	if nil != err {
		t.Fatalf("hex to link error: %s", err)
	}

	r := transactionrecord.BitmarkShare{
		Link:     link,
		Quantity: 12345,
	}

	expected := []byte{
		0x08, 0x20, 0x79, 0xa6, 0x7b, 0xe2, 0xb3, 0xd3,
		0x13, 0xbd, 0x49, 0x03, 0x63, 0xfb, 0x0d, 0x27,
		0x90, 0x1c, 0x46, 0xed, 0x53, 0xd3, 0xf7, 0xb2,
		0x1f, 0x60, 0xd4, 0x8b, 0xc4, 0x24, 0x39, 0xb0,
		0x60, 0x84, 0xb9, 0x60,
	}

	expectedTxId := merkle.Digest{
		0x68, 0x95, 0x6b, 0x9a, 0x91, 0x0f, 0xaa, 0x55,
		0xf3, 0x3a, 0xcb, 0xa1, 0x17, 0x08, 0x6c, 0x2f,
		0x2d, 0x83, 0x7c, 0xba, 0x9f, 0x80, 0x79, 0x87,
		0x2a, 0x4e, 0xeb, 0x65, 0x6a, 0x42, 0xeb, 0x83,
	}

	// manually sign the record and attach signature to "expected"
	signature := ed25519.Sign(ownerOne.privateKey, expected)
	r.Signature = signature
	l := util.ToVarint64(uint64(len(signature)))
	expected = append(expected, l...)
	expected = append(expected, signature...)

	// test the packer
	packed, err := r.Pack(ownerOneAccount)
	if nil != err {
		t.Errorf("pack error: %s", err)
	}

	// if either of above fail we will have the message _without_ a signature
	if !bytes.Equal(packed, expected) {
		t.Errorf("pack record: %x  expected: %x", packed, expected)
		t.Errorf("*** GENERATED Packed:\n%s", util.FormatBytes("expected", packed))
		t.Fatal("fatal error")
	}

	t.Logf("Packed length: %d bytes", len(packed))

	// check txId
	txId := packed.MakeLink()

	if txId != expectedTxId {
		t.Errorf("pack txId: %#v  expected: %x", txId, expectedTxId)
		t.Errorf("*** GENERATED txId:\n%s", util.FormatBytes("expectedTxId", txId[:]))
		t.Fatal("fatal error")
	}

	// test the unpacker
	unpacked, n, err := packed.Unpack(true)
	if nil != err {
		t.Fatalf("unpack error: %s", err)
	}
	if len(packed) != n {
		t.Errorf("did not unpack all data: only used: %d of: %d bytes", n, len(packed))
	}

	balance, ok := unpacked.(*transactionrecord.BitmarkShare)
	if !ok {
		t.Fatalf("did not unpack to BitmarkShare")
	}

	// display a JSON version for information
	item := struct {
		TxId         merkle.Digest
		ShareBalance *transactionrecord.BitmarkShare
	}{
		txId,
		balance,
	}
	b, err := json.MarshalIndent(item, "", "  ")
	if nil != err {
		t.Fatalf("json error: %s", err)
	}

	t.Logf("Share Balance: JSON: %s", b)

	// check that structure is preserved through Pack/Unpack
	// note reg is a pointer here
	if !reflect.DeepEqual(r, *balance) {
		t.Fatalf("different, original: %v  recovered: %v", r, *balance)
	}
}

// test the packing/unpacking of Share balance record
//
// ensures that zero value fails
func TestPackShareBalanceValueNotZero(t *testing.T) {

	ownerOneAccount := makeAccount(ownerOne.publicKey)

	var link merkle.Digest
	err := merkleDigestFromLE("79a67be2b3d313bd490363fb0d27901c46ed53d3f7b21f60d48bc42439b06084", &link)
	if nil != err {
		t.Fatalf("hex to link error: %s", err)
	}

	r := transactionrecord.BitmarkShare{
		Link:     link,
		Quantity: 0,
	}

	expected := []byte{
		0x08, 0x20, 0x79, 0xa6, 0x7b, 0xe2, 0xb3, 0xd3,
		0x13, 0xbd, 0x49, 0x03, 0x63, 0xfb, 0x0d, 0x27,
		0x90, 0x1c, 0x46, 0xed, 0x53, 0xd3, 0xf7, 0xb2,
		0x1f, 0x60, 0xd4, 0x8b, 0xc4, 0x24, 0x39, 0xb0,
		0x60, 0x84, 0x00,
	}

	// manually sign the record and attach signature to "expected"
	signature := ed25519.Sign(ownerTwo.privateKey, expected)
	r.Signature = signature

	// test the packer
	_, err = r.Pack(ownerOneAccount)
	if fault.ErrShareQuantityTooSmall != err {
		t.Fatalf("unexpected pack error: %s", err)
	}
}
<|endoftext|>


<commit_before>package scipipe

import (
	"fmt"
	"os/exec"
	re "regexp"
	str "strings"
	// "time"
	"errors"
)

type ShellTask struct {
	task
	_OutOnly     bool
	InPorts      map[string]chan *FileTarget
	InPaths      map[string]string
	OutPorts     map[string]chan *FileTarget
	OutPathFuncs map[string]func() string
	Command      string
}

func NewShellTask(command string) *ShellTask {
	return &ShellTask{
		Command:      command,
		InPorts:      make(map[string]chan *FileTarget),
		InPaths:      make(map[string]string),
		OutPorts:     make(map[string]chan *FileTarget),
		OutPathFuncs: make(map[string]func() string),
	}
}

func Sh(cmd string) *ShellTask {
	// Create task
	t := NewShellTask(cmd)

	// Find in/out port names, and set up in port lists
	r, err := re.Compile("{(o|i):([^{}:]+)}")
	Check(err)
	ms := r.FindAllStringSubmatch(cmd, -1)
	for _, m := range ms {
		if len(m) < 3 {
			Check(errors.New("Too few matches"))
		}
		typ := m[1]
		name := m[2]
		if typ == "o" {
			t.OutPorts[name] = make(chan *FileTarget, BUFSIZE)
		} // else if typ == "i" {
		// Set up a channel on the inports, even though this is
		// often replaced by another tasks output port channel.
		// It might be nice to have it init'ed with a channel
		// anyways, for use cases when we want to send FileTargets
		// on the inport manually.
		// t.InPorts[name] = make(chan *FileTarget, BUFSIZE)
		// }
	}
	return t
}

func (t *ShellTask) Run() {
	fmt.Println("Entering task: ", t.Command)
	// Close output channels
	for _, ochan := range t.OutPorts {
		defer close(ochan)
	}

	// Main loop
	breakLoop := false
	for !breakLoop {
		// If there are no inports, we know we should exit the loop
		// directly after executing the command, and sending the outputs
		if len(t.InPorts) == 0 {
			breakLoop = true
		}

		// Read from inports
		inPortsOpen := t.receiveInputs()
		if !inPortsOpen {
			break
		}

		// Execute command
		t.formatAndExecute(t.Command)

		// Send
		t.sendOutputs()
	}
	fmt.Println("Exiting task:  ", t.Command)
}

func (t *ShellTask) receiveInputs() bool {
	inPortsOpen := true
	// Read input targets on in-ports and set up path mappings
	for iname, ichan := range t.InPorts {
		infile, open := <-ichan
		if !open {
			inPortsOpen = false
			continue
		}
		fmt.Println("Receiving file:", infile.GetPath())
		t.InPaths[iname] = infile.GetPath()
	}
	return inPortsOpen
}

func (t *ShellTask) sendOutputs() {
	// Send output targets on out ports
	for oname, ochan := range t.OutPorts {
		fun := t.OutPathFuncs[oname]
		baseName := fun()
		ft := NewFileTarget(baseName)
		fmt.Println("Sending file:  ", ft.GetPath())
		ochan <- ft
	}
}

func (t *ShellTask) formatAndExecute(cmd string) {
	cmd = t.ReplacePlaceholdersInCmd(cmd)
	fmt.Println("Executing cmd: ", cmd)
	_, err := exec.Command("bash", "-c", cmd).Output()
	Check(err)
}

func (t *ShellTask) ReplacePlaceholdersInCmd(cmd string) string {
	r, err := re.Compile("{(o|i):([^{}:]+)}")
	Check(err)
	ms := r.FindAllStringSubmatch(cmd, -1)
	for _, m := range ms {
		whole := m[0]
		typ := m[1]
		name := m[2]
		var newstr string
		if typ == "o" {
			if t.OutPathFuncs[name] != nil {
				newstr = t.OutPathFuncs[name]()
			} else {
				msg := fmt.Sprint("Missing outpath function for outport '", name, "' of shell task '", t.Command, "'")
				Check(errors.New(msg))
			}
		} else if typ == "i" {
			if t.InPaths[name] != "" {
				newstr = t.InPaths[name]
			} else {
				msg := fmt.Sprint("Missing inpath for inport '", name, "' of shell task '", t.Command, "'")
				Check(errors.New(msg))
			}
		}
		if newstr == "" {
			msg := fmt.Sprint("Replace failed for port ", name, " in task '", t.Command, "'")
			Check(errors.New(msg))
		}
		cmd = str.Replace(cmd, whole, newstr, -1)
	}
	return cmd
}

func (t *ShellTask) GetInPath(inPort string) string {
	var inPath string
	if t.InPaths[inPort] != "" {
		inPath = t.InPaths[inPort]
	} else {
		msg := fmt.Sprint("Missing inpath for inport '", inPort, "' of shell task '", t.Command, "'")
		Check(errors.New(msg))
	}
	return inPath
}
<commit_msg>Refactoring: Break out closeOutChans in t.Run()<commit_after>package scipipe

import (
	"fmt"
	"os/exec"
	re "regexp"
	str "strings"
	// "time"
	"errors"
)

type ShellTask struct {
	task
	_OutOnly     bool
	InPorts      map[string]chan *FileTarget
	InPaths      map[string]string
	OutPorts     map[string]chan *FileTarget
	OutPathFuncs map[string]func() string
	Command      string
}

func NewShellTask(command string) *ShellTask {
	return &ShellTask{
		Command:      command,
		InPorts:      make(map[string]chan *FileTarget),
		InPaths:      make(map[string]string),
		OutPorts:     make(map[string]chan *FileTarget),
		OutPathFuncs: make(map[string]func() string),
	}
}

func Sh(cmd string) *ShellTask {
	// Create task
	t := NewShellTask(cmd)

	// Find in/out port names, and set up in port lists
	r, err := re.Compile("{(o|i):([^{}:]+)}")
	Check(err)
	ms := r.FindAllStringSubmatch(cmd, -1)
	for _, m := range ms {
		if len(m) < 3 {
			Check(errors.New("Too few matches"))
		}
		typ := m[1]
		name := m[2]
		if typ == "o" {
			t.OutPorts[name] = make(chan *FileTarget, BUFSIZE)
		} // else if typ == "i" {
		// Set up a channel on the inports, even though this is
		// often replaced by another tasks output port channel.
		// It might be nice to have it init'ed with a channel
		// anyways, for use cases when we want to send FileTargets
		// on the inport manually.
		// t.InPorts[name] = make(chan *FileTarget, BUFSIZE)
		// }
	}
	return t
}

func (t *ShellTask) Run() {
	fmt.Println("Entering task: ", t.Command)
	defer t.closeOutChans()

	// Main loop
	continueLoop := true
	for continueLoop {
		// If there are no inports, we know we should exit the loop
		// directly after executing the command, and sending the outputs
		if len(t.InPorts) == 0 {
			continueLoop = false
		}

		// Read from inports
		inPortsOpen := t.receiveInputs()
		if !inPortsOpen {
			break
		}

		// Execute command
		t.formatAndExecute(t.Command)

		// Send
		t.sendOutputs()
	}
	fmt.Println("Exiting task:  ", t.Command)
}

func (t *ShellTask) closeOutChans() {
	// Close output channels
	for _, ochan := range t.OutPorts {
		close(ochan)
	}
}

func (t *ShellTask) receiveInputs() bool {
	inPortsOpen := true
	// Read input targets on in-ports and set up path mappings
	for iname, ichan := range t.InPorts {
		infile, open := <-ichan
		if !open {
			inPortsOpen = false
			continue
		}
		fmt.Println("Receiving file:", infile.GetPath())
		t.InPaths[iname] = infile.GetPath()
	}
	return inPortsOpen
}

func (t *ShellTask) sendOutputs() {
	// Send output targets on out ports
	for oname, ochan := range t.OutPorts {
		fun := t.OutPathFuncs[oname]
		baseName := fun()
		ft := NewFileTarget(baseName)
		fmt.Println("Sending file:  ", ft.GetPath())
		ochan <- ft
	}
}

func (t *ShellTask) formatAndExecute(cmd string) {
	cmd = t.ReplacePlaceholdersInCmd(cmd)
	fmt.Println("Executing cmd: ", cmd)
	_, err := exec.Command("bash", "-c", cmd).Output()
	Check(err)
}

func (t *ShellTask) ReplacePlaceholdersInCmd(cmd string) string {
	r, err := re.Compile("{(o|i):([^{}:]+)}")
	Check(err)
	ms := r.FindAllStringSubmatch(cmd, -1)
	for _, m := range ms {
		whole := m[0]
		typ := m[1]
		name := m[2]
		var newstr string
		if typ == "o" {
			if t.OutPathFuncs[name] != nil {
				newstr = t.OutPathFuncs[name]()
			} else {
				msg := fmt.Sprint("Missing outpath function for outport '", name, "' of shell task '", t.Command, "'")
				Check(errors.New(msg))
			}
		} else if typ == "i" {
			if t.InPaths[name] != "" {
				newstr = t.InPaths[name]
			} else {
				msg := fmt.Sprint("Missing inpath for inport '", name, "' of shell task '", t.Command, "'")
				Check(errors.New(msg))
			}
		}
		if newstr == "" {
			msg := fmt.Sprint("Replace failed for port ", name, " in task '", t.Command, "'")
			Check(errors.New(msg))
		}
		cmd = str.Replace(cmd, whole, newstr, -1)
	}
	return cmd
}

func (t *ShellTask) GetInPath(inPort string) string {
	var inPath string
	if t.InPaths[inPort] != "" {
		inPath = t.InPaths[inPort]
	} else {
		msg := fmt.Sprint("Missing inpath for inport '", inPort, "' of shell task '", t.Command, "'")
		Check(errors.New(msg))
	}
	return inPath
}
<|endoftext|>