<commit_before>package worker

import (
	"encoding/json"
	"fmt"
	"time"

	gocontext "context"

	"github.com/bitly/go-simplejson"
	"github.com/streadway/amqp"
	"github.com/travis-ci/worker/backend"
	"github.com/travis-ci/worker/context"
	"github.com/travis-ci/worker/metrics"
)

type amqpJob struct {
	conn            *amqp.Connection
	delivery        amqp.Delivery
	payload         *JobPayload
	rawPayload      *simplejson.Json
	startAttributes *backend.StartAttributes
	received        time.Time
	started         time.Time
}

func (j *amqpJob) GoString() string {
	return fmt.Sprintf("&amqpJob{conn: %#v, delivery: %#v, payload: %#v, startAttributes: %#v}",
		j.conn, j.delivery, j.payload, j.startAttributes)
}

func (j *amqpJob) Payload() *JobPayload {
	return j.payload
}

func (j *amqpJob) RawPayload() *simplejson.Json {
	return j.rawPayload
}

func (j *amqpJob) StartAttributes() *backend.StartAttributes {
	return j.startAttributes
}

func (j *amqpJob) Error(ctx gocontext.Context, errMessage string) error {
	log, err := j.LogWriter(ctx, time.Minute)
	if err != nil {
		return err
	}

	_, err = log.WriteAndClose([]byte(errMessage))
	if err != nil {
		return err
	}

	return j.Finish(ctx, FinishStateErrored)
}

func (j *amqpJob) Requeue(ctx gocontext.Context) error {
	context.LoggerFromContext(ctx).Info("requeueing job")

	metrics.Mark("worker.job.requeue")

	err := j.sendStateUpdate("job:test:reset", map[string]interface{}{
		"id":    j.Payload().Job.ID,
		"state": "reset",
	})
	if err != nil {
		return err
	}

	return j.delivery.Ack(false)
}

func (j *amqpJob) Received() error {
	j.received = time.Now()

	if j.payload.Job.QueuedAt != nil {
		metrics.TimeSince("travis.worker.job.queue_time", *j.payload.Job.QueuedAt)
	}

	return j.sendStateUpdate("job:test:receive", map[string]interface{}{
		"id":          j.Payload().Job.ID,
		"state":       "received",
		"received_at": j.received.UTC().Format(time.RFC3339),
	})
}

func (j *amqpJob) Started() error {
	j.started = time.Now()

	metrics.TimeSince("travis.worker.job.start_time", j.received)

	return j.sendStateUpdate("job:test:start", map[string]interface{}{
		"id":          j.Payload().Job.ID,
		"state":       "started",
		"received_at": j.received.UTC().Format(time.RFC3339),
		"started_at":  j.started.UTC().Format(time.RFC3339),
	})
}

func (j *amqpJob) Finish(ctx gocontext.Context, state FinishState) error {
	context.LoggerFromContext(ctx).WithField("state", state).Info("finishing job")

	finishedAt := time.Now()
	receivedAt := j.received
	if receivedAt.IsZero() {
		receivedAt = finishedAt
	}
	startedAt := j.started
	if startedAt.IsZero() {
		startedAt = finishedAt
	}

	metrics.Mark(fmt.Sprintf("travis.worker.job.finish.%s", state))
	metrics.Mark("travis.worker.job.finish")

	err := j.sendStateUpdate("job:test:finish", map[string]interface{}{
		"id":          j.Payload().Job.ID,
		"state":       state,
		"received_at": receivedAt.UTC().Format(time.RFC3339),
		"started_at":  startedAt.UTC().Format(time.RFC3339),
		"finished_at": finishedAt.UTC().Format(time.RFC3339),
	})
	if err != nil {
		return err
	}

	return j.delivery.Ack(false)
}

func (j *amqpJob) LogWriter(ctx gocontext.Context, defaultLogTimeout time.Duration) (LogWriter, error) {
	logTimeout := time.Duration(j.payload.Timeouts.LogSilence) * time.Second
	if logTimeout == 0 {
		logTimeout = defaultLogTimeout
	}

	return newAMQPLogWriter(ctx, j.conn, j.payload.Job.ID, logTimeout)
}

func (j *amqpJob) sendStateUpdate(event string, body map[string]interface{}) error {
	amqpChan, err := j.conn.Channel()
	if err != nil {
		return err
	}
	defer amqpChan.Close()

	bodyBytes, err := json.Marshal(body)
	if err != nil {
		return err
	}

	_, err = amqpChan.QueueDeclare("reporting.jobs.builds", true, false, false, false, nil)
	if err != nil {
		return err
	}

	return amqpChan.Publish("", "reporting.jobs.builds", false, false, amqp.Publishing{
		ContentType:  "application/json",
		DeliveryMode: amqp.Persistent,
		Timestamp:    time.Now().UTC(),
		Type:         event,
		Body:         bodyBytes,
	})
}
<commit_msg>amqp-job: send queued_at in state updates if known<commit_after>package worker

import (
	"encoding/json"
	"fmt"
	"time"

	gocontext "context"

	"github.com/bitly/go-simplejson"
	"github.com/streadway/amqp"
	"github.com/travis-ci/worker/backend"
	"github.com/travis-ci/worker/context"
	"github.com/travis-ci/worker/metrics"
)

type amqpJob struct {
	conn            *amqp.Connection
	delivery        amqp.Delivery
	payload         *JobPayload
	rawPayload      *simplejson.Json
	startAttributes *backend.StartAttributes
	received        time.Time
	started         time.Time
}

func (j *amqpJob) GoString() string {
	return fmt.Sprintf("&amqpJob{conn: %#v, delivery: %#v, payload: %#v, startAttributes: %#v}",
		j.conn, j.delivery, j.payload, j.startAttributes)
}

func (j *amqpJob) Payload() *JobPayload {
	return j.payload
}

func (j *amqpJob) RawPayload() *simplejson.Json {
	return j.rawPayload
}

func (j *amqpJob) StartAttributes() *backend.StartAttributes {
	return j.startAttributes
}

func (j *amqpJob) Error(ctx gocontext.Context, errMessage string) error {
	log, err := j.LogWriter(ctx, time.Minute)
	if err != nil {
		return err
	}

	_, err = log.WriteAndClose([]byte(errMessage))
	if err != nil {
		return err
	}

	return j.Finish(ctx, FinishStateErrored)
}

func (j *amqpJob) Requeue(ctx gocontext.Context) error {
	context.LoggerFromContext(ctx).Info("requeueing job")

	metrics.Mark("worker.job.requeue")

	err := j.sendStateUpdate("job:test:reset", map[string]interface{}{
		"id":    j.Payload().Job.ID,
		"state": "reset",
	})
	if err != nil {
		return err
	}

	return j.delivery.Ack(false)
}

func (j *amqpJob) Received() error {
	j.received = time.Now()

	payload := map[string]interface{}{
		"id":          j.Payload().Job.ID,
		"state":       "received",
		"received_at": j.received.UTC().Format(time.RFC3339),
	}

	if j.Payload().Job.QueuedAt != nil {
		metrics.TimeSince("travis.worker.job.queue_time", *j.payload.Job.QueuedAt)
		payload["queued_at"] = j.Payload().Job.QueuedAt.UTC().Format(time.RFC3339)
	}

	return j.sendStateUpdate("job:test:receive", payload)
}

func (j *amqpJob) Started() error {
	j.started = time.Now()

	metrics.TimeSince("travis.worker.job.start_time", j.received)

	payload := map[string]interface{}{
		"id":          j.Payload().Job.ID,
		"state":       "started",
		"received_at": j.received.UTC().Format(time.RFC3339),
		"started_at":  j.started.UTC().Format(time.RFC3339),
	}

	if j.Payload().Job.QueuedAt != nil {
		payload["queued_at"] = j.Payload().Job.QueuedAt.UTC().Format(time.RFC3339)
	}

	return j.sendStateUpdate("job:test:start", payload)
}

func (j *amqpJob) Finish(ctx gocontext.Context, state FinishState) error {
	context.LoggerFromContext(ctx).WithField("state", state).Info("finishing job")

	finishedAt := time.Now()
	receivedAt := j.received
	if receivedAt.IsZero() {
		receivedAt = finishedAt
	}
	startedAt := j.started
	if startedAt.IsZero() {
		startedAt = finishedAt
	}

	metrics.Mark(fmt.Sprintf("travis.worker.job.finish.%s", state))
	metrics.Mark("travis.worker.job.finish")

	payload := map[string]interface{}{
		"id":          j.Payload().Job.ID,
		"state":       state,
		"received_at": receivedAt.UTC().Format(time.RFC3339),
		"started_at":  startedAt.UTC().Format(time.RFC3339),
		"finished_at": finishedAt.UTC().Format(time.RFC3339),
	}

	if j.Payload().Job.QueuedAt != nil {
		payload["queued_at"] = j.Payload().Job.QueuedAt.UTC().Format(time.RFC3339)
	}

	err := j.sendStateUpdate("job:test:finish", payload)
	if err != nil {
		return err
	}

	return j.delivery.Ack(false)
}

func (j *amqpJob) LogWriter(ctx gocontext.Context, defaultLogTimeout time.Duration) (LogWriter, error) {
	logTimeout := time.Duration(j.payload.Timeouts.LogSilence) * time.Second
	if logTimeout == 0 {
		logTimeout = defaultLogTimeout
	}

	return newAMQPLogWriter(ctx, j.conn, j.payload.Job.ID, logTimeout)
}

func (j *amqpJob) sendStateUpdate(event string, body map[string]interface{}) error {
	amqpChan, err := j.conn.Channel()
	if err != nil {
		return err
	}
	defer amqpChan.Close()

	bodyBytes, err := json.Marshal(body)
	if err != nil {
		return err
	}

	_, err = amqpChan.QueueDeclare("reporting.jobs.builds", true, false, false, false, nil)
	if err != nil {
		return err
	}

	return amqpChan.Publish("", "reporting.jobs.builds", false, false, amqp.Publishing{
		ContentType:  "application/json",
		DeliveryMode: amqp.Persistent,
		Timestamp:    time.Now().UTC(),
		Type:         event,
		Body:         bodyBytes,
	})
}
<|endoftext|>


<commit_before><commit_msg>switch to reflection of definition rather than instance of configuration<commit_after><|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"syscall"

	"github.com/gorilla/websocket"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxc/config"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/gnuflag"
	"github.com/lxc/lxd/shared/i18n"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/termios"
)

type consoleCmd struct {
	showLog bool
}

func (c *consoleCmd) showByDefault() bool {
	return true
}

func (c *consoleCmd) usage() string {
	return i18n.G(
		`Usage: lxc console [<remote>:]<container> [-l]

Interact with the container's console device and log.`)
}

func (c *consoleCmd) flags() {
	gnuflag.BoolVar(&c.showLog, "show-log", false, i18n.G("Retrieve the container's console log"))
}

func (c *consoleCmd) sendTermSize(control *websocket.Conn) error {
	width, height, err := termios.GetSize(int(syscall.Stdout))
	if err != nil {
		return err
	}

	logger.Debugf("Window size is now: %dx%d", width, height)

	w, err := control.NextWriter(websocket.TextMessage)
	if err != nil {
		return err
	}

	msg := api.ContainerExecControl{}
	msg.Command = "window-resize"
	msg.Args = make(map[string]string)
	msg.Args["width"] = strconv.Itoa(width)
	msg.Args["height"] = strconv.Itoa(height)

	buf, err := json.Marshal(msg)
	if err != nil {
		return err
	}
	_, err = w.Write(buf)

	w.Close()
	return err
}

type ReadWriteCloser struct {
	io.Reader
	io.WriteCloser
}

func (c *consoleCmd) run(conf *config.Config, args []string) error {
	if len(args) < 1 {
		return errArgs
	}

	remote, name, err := conf.ParseRemote(args[0])
	if err != nil {
		return err
	}

	d, err := conf.GetContainerServer(remote)
	if err != nil {
		return err
	}

	if c.showLog {
		console := &lxd.ContainerConsoleLogArgs{}
		log, err := d.GetContainerConsoleLog(name, console)
		if err != nil {
			return err
		}

		stuff, err := ioutil.ReadAll(log)
		if err != nil {
			return err
		}

		fmt.Printf("\n"+i18n.G("Console log:")+"\n\n%s\n", string(stuff))
		return nil
	}

	cfd := int(syscall.Stdin)

	var oldttystate *termios.State
	oldttystate, err = termios.MakeRaw(cfd)
	if err != nil {
		return err
	}
	defer termios.Restore(cfd, oldttystate)

	handler := c.controlSocketHandler

	var width, height int
	width, height, err = termios.GetSize(int(syscall.Stdout))
	if err != nil {
		return err
	}

	req := api.ContainerConsolePost{
		Width:  width,
		Height: height,
	}

	consoleArgs := lxd.ContainerConsoleArgs{
		Terminal: &ReadWriteCloser{os.Stdin, os.Stdout},
		Control:  handler,
	}

	// Run the command in the container
	op, err := d.ConsoleContainer(name, req, &consoleArgs)
	if err != nil {
		return err
	}

	// Wait for the operation to complete
	err = op.Wait()
	if err != nil {
		return err
	}

	if oldttystate != nil {
		/* A bit of a special case here: we want to exit with the same code as
		 * the process inside the container, so we explicitly exit here
		 * instead of returning an error.
		 *
		 * Additionally, since os.Exit() exits without running deferred
		 * functions, we restore the terminal explicitly.
		 */
		termios.Restore(cfd, oldttystate)
	}

	return nil
}
<commit_msg>lxc: handle console detach<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strconv"

	"github.com/gorilla/websocket"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxc/config"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/gnuflag"
	"github.com/lxc/lxd/shared/i18n"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/termios"
)

type consoleCmd struct {
	showLog bool
}

func (c *consoleCmd) showByDefault() bool {
	return true
}

func (c *consoleCmd) usage() string {
	return i18n.G(
		`Usage: lxc console [<remote>:]<container> [-l]

Interact with the container's console device and log.`)
}

func (c *consoleCmd) flags() {
	gnuflag.BoolVar(&c.showLog, "show-log", false, i18n.G("Retrieve the container's console log"))
}

func (c *consoleCmd) sendTermSize(control *websocket.Conn) error {
	width, height, err := termios.GetSize(int(os.Stdout.Fd()))
	if err != nil {
		return err
	}

	logger.Debugf("Window size is now: %dx%d", width, height)

	w, err := control.NextWriter(websocket.TextMessage)
	if err != nil {
		return err
	}

	msg := api.ContainerExecControl{}
	msg.Command = "window-resize"
	msg.Args = make(map[string]string)
	msg.Args["width"] = strconv.Itoa(width)
	msg.Args["height"] = strconv.Itoa(height)

	buf, err := json.Marshal(msg)
	if err != nil {
		return err
	}
	_, err = w.Write(buf)

	w.Close()
	return err
}

type ReadWriteCloser struct {
	io.Reader
	io.WriteCloser
}

type StdinMirror struct {
	r                 io.Reader
	consoleDisconnect chan<- bool
	foundEscape       *bool
}

// The pty has been switched to raw mode so we will only ever read a single
// byte. The buffer size is therefore uninteresting to us.
func (er StdinMirror) Read(p []byte) (int, error) {
	n, err := er.r.Read(p)

	v := rune(p[0])
	if v == '\u0001' && !*er.foundEscape {
		*er.foundEscape = true
		return 0, err
	}

	if v == 'q' && *er.foundEscape {
		select {
		case er.consoleDisconnect <- true:
			return 0, err
		default:
			return 0, err
		}
	}

	*er.foundEscape = false
	return n, err
}

func (c *consoleCmd) run(conf *config.Config, args []string) error {
	if len(args) < 1 {
		return errArgs
	}

	remote, name, err := conf.ParseRemote(args[0])
	if err != nil {
		return err
	}

	d, err := conf.GetContainerServer(remote)
	if err != nil {
		return err
	}

	if c.showLog {
		console := &lxd.ContainerConsoleLogArgs{}
		log, err := d.GetContainerConsoleLog(name, console)
		if err != nil {
			return err
		}

		stuff, err := ioutil.ReadAll(log)
		if err != nil {
			return err
		}

		fmt.Printf("\n"+i18n.G("Console log:")+"\n\n%s\n", string(stuff))
		return nil
	}

	cfd := int(os.Stdin.Fd())

	var oldttystate *termios.State
	oldttystate, err = termios.MakeRaw(cfd)
	if err != nil {
		return err
	}
	defer termios.Restore(cfd, oldttystate)

	handler := c.controlSocketHandler

	var width, height int
	width, height, err = termios.GetSize(int(os.Stdin.Fd()))
	if err != nil {
		return err
	}

	req := api.ContainerConsolePost{
		Width:  width,
		Height: height,
	}

	consoleDisconnect := make(chan bool)
	sendDisconnect := make(chan bool)
	defer close(sendDisconnect)
	consoleArgs := lxd.ContainerConsoleArgs{
		Terminal: &ReadWriteCloser{StdinMirror{os.Stdin,
			sendDisconnect, new(bool)}, os.Stdout},
		Control:           handler,
		ConsoleDisconnect: consoleDisconnect,
	}

	go func() {
		<-sendDisconnect
		close(consoleDisconnect)
	}()

	// Run the command in the container
	op, err := d.ConsoleContainer(name, req, &consoleArgs)
	if err != nil {
		return err
	}

	// Wait for the operation to complete
	err = op.Wait()
	if err != nil {
		return err
	}

	if oldttystate != nil {
		/* A bit of a special case here: we want to exit with the same code as
		 * the process inside the container, so we explicitly exit here
		 * instead of returning an error.
		 *
		 * Additionally, since os.Exit() exits without running deferred
		 * functions, we restore the terminal explicitly.
		 */
		termios.Restore(cfd, oldttystate)
	}

	return nil
}
<|endoftext|>


<commit_before>// Package appsettings provides simple key/value store functionality designed
// to be used for easily storing and recalling application runtime settings
package appsettings

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"os"
	"strconv"
	"sync"
)

// DataTree is the host of key and branches of values
type DataTree interface {
	GetString(key string) (string, error)
	SetString(key string, val string)

	GetInt(key string) (int, error)
	SetInt(key string, val int)

	GetInt64(key string) (int64, error)
	SetInt64(key string, val int64)
	IncrInt64(key string) int64
	DecrInt64(key string) int64

	Delete(key string)
	DeleteTree(key string)
	GetTree(key string) DataTree

	HasTree(key string) bool
	HasLeaf(key string) bool

	GetTrees() map[string]*tree
	GetLeaves() map[string]string
}

type tree struct {
	Branches map[string]*tree
	Leaves   map[string]string

	sync.Mutex
}

// AppSettings is the root most DataTree
type AppSettings struct {
	filename string
	pretty   bool

	*tree
}

// Option sets an option of the passed AppSettings
type Option func(*AppSettings)

// OptionPrettyPrint configures AppSettings to pretty print the saved json
func OptionPrettyPrint(app *AppSettings) {
	app.pretty = true
}

// NewAppSettings gets a new AppSettings struct
func NewAppSettings(dbFilename string, options ...Option) (*AppSettings, error) {
	a := &AppSettings{
		filename: dbFilename,
		pretty:   false,

		tree: &tree{
			Branches: make(map[string]*tree),
			Leaves:   make(map[string]string),
		},
	}

	for _, option := range options {
		option(a)
	}

	if _, err := os.Stat(dbFilename); os.IsNotExist(err) {
		d1, _ := json.Marshal(a)
		err := ioutil.WriteFile(dbFilename, d1, 0644)
		if err != nil {
			return nil, err
		}
	} else {
		d1, err := ioutil.ReadFile(dbFilename)
		if err != nil {
			return nil, err
		}

		err = json.Unmarshal(d1, a)
		if err != nil {
			return nil, err
		}
	}

	return a, nil
}

// ErrUndefinedKey is returned when the key requested from get is undefined.
var ErrUndefinedKey = errors.New("undefined key")

// GetString gets the give keys leaf as a string.
//
// Returns an ErrUndefinedKey if the key is not defined
func (a *tree) GetString(key string) (string, error) {
	a.Lock()
	defer a.Unlock()

	if _, ok := a.Leaves[key]; !ok {
		return "", ErrUndefinedKey
	}

	return a.Leaves[key], nil
}

// SetString sets the give key leaf as a string.
func (a *tree) SetString(key string, val string) {
	a.Lock()
	defer a.Unlock()

	a.Leaves[key] = val
}

// GetInt gets the give keys leaf as an int.
//
// Returns an ErrUndefinedKey if the key is not defined.
//
// Will return an error if ParseInt of the value of the leaf fails.
func (a *tree) GetInt(key string) (int, error) {
	str, err := a.GetString(key)
	if err != nil {
		return 0, err
	}

	i, err := strconv.Atoi(str)
	if err != nil {
		return 0, err
	}
	return i, nil
}

// SetInt sets the give key leaf as a int.
func (a *tree) SetInt(key string, val int) {
	a.Lock()
	defer a.Unlock()

	a.Leaves[key] = strconv.Itoa(val)
}

// GetInt64 gets the give keys leaf as an int64.
//
// Returns an ErrUndefinedKey if the key is not defined.
//
// Will return an error if ParseInt(...,10, 64) of the value of the leaf fails.
func (a *tree) GetInt64(key string) (int64, error) {
	str, err := a.GetString(key)
	if err != nil {
		return 0, err
	}

	i, err := strconv.ParseInt(str, 10, 64)
	if err != nil {
		return 0, err
	}
	return i, nil
}

// SetInt64 sets the give key leaf as a int64.
func (a *tree) SetInt64(key string, val int64) {
	a.Lock()
	defer a.Unlock()

	a.Leaves[key] = strconv.FormatInt(val, 10)
}

// IncrInt64 increments an int or int64 leaf and returns the new value as int64.
//
// If the key is undefined or non-integer, this will initialize it to 0 and increment it to 1
func (a *tree) IncrInt64(key string) int64 {
	v, _ := a.GetInt64(key)
	v += 1
	a.SetInt64(key, v)

	return v
}

// DecrInt64 decrements an int or int64 leaf and returns the new value as int64.
//
// If the key is undefined or non-integer, this will initialize it to 0 and decrements it to -1
func (a *tree) DecrInt64(key string) int64 {
	v, _ := a.GetInt64(key)
	v -= 1
	a.SetInt64(key, v)

	return v
}

// Delete removes the given leaf from the branch.
func (a *tree) Delete(key string) {
	a.Lock()
	defer a.Unlock()

	delete(a.Leaves, key)
}

// Delete removes the given tree from the branch.
func (a *tree) DeleteTree(key string) {
	a.Lock()
	defer a.Unlock()

	delete(a.Branches, key)
}

func (a *tree) GetLeaves() map[string]string {
	return a.Leaves
}

func (a *tree) GetTrees() map[string]*tree {
	return a.Branches
}

// GetTree fetches a tree for app setting storage
func (a *tree) GetTree(key string) DataTree {
	a.Lock()
	defer a.Unlock()

	if _, ok := a.Branches[key]; !ok {
		a.Branches[key] = &tree{
			Branches: make(map[string]*tree),
			Leaves:   make(map[string]string),
		}
	}

	return a.Branches[key]
}

// HasTree checks if the given key is defined as a tree
func (a *tree) HasTree(key string) bool {
	a.Lock()
	defer a.Unlock()

	_, ok := a.Branches[key]
	return ok
}

// HasTree checks if the given key is defined as a leaf value
func (a *tree) HasLeaf(key string) bool {
	a.Lock()
	defer a.Unlock()

	_, ok := a.Leaves[key]
	return ok
}

// Persist causes the current state of the app settings to be persisted.
func (a *AppSettings) Persist() error {
	a.Lock()
	defer a.Unlock()

	var err error
	var d1 []byte

	if a.pretty {
		d1, err = json.MarshalIndent(a, "", "\t")
	} else {
		d1, err = json.Marshal(a)
	}
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(a.filename, d1, 0644)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Correct doc typos<commit_after>// Package appsettings provides simple key/value store functionality designed
// to be used for easily storing and recalling application runtime settings
package appsettings

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"os"
	"strconv"
	"sync"
)

// DataTree is the host of key and branches of values
type DataTree interface {
	GetString(key string) (string, error)
	SetString(key string, val string)

	GetInt(key string) (int, error)
	SetInt(key string, val int)

	GetInt64(key string) (int64, error)
	SetInt64(key string, val int64)
	IncrInt64(key string) int64
	DecrInt64(key string) int64

	Delete(key string)
	DeleteTree(key string)
	GetTree(key string) DataTree

	HasTree(key string) bool
	HasLeaf(key string) bool

	GetTrees() map[string]*tree
	GetLeaves() map[string]string
}

type tree struct {
	Branches map[string]*tree
	Leaves   map[string]string

	sync.Mutex
}

// AppSettings is the root most DataTree
type AppSettings struct {
	filename string
	pretty   bool

	*tree
}

// Option sets an option of the passed AppSettings
type Option func(*AppSettings)

// OptionPrettyPrint configures AppSettings to pretty print the saved json
func OptionPrettyPrint(app *AppSettings) {
	app.pretty = true
}

// NewAppSettings gets a new AppSettings struct
func NewAppSettings(dbFilename string, options ...Option) (*AppSettings, error) {
	a := &AppSettings{
		filename: dbFilename,
		pretty:   false,

		tree: &tree{
			Branches: make(map[string]*tree),
			Leaves:   make(map[string]string),
		},
	}

	for _, option := range options {
		option(a)
	}

	if _, err := os.Stat(dbFilename); os.IsNotExist(err) {
		d1, _ := json.Marshal(a)
		err := ioutil.WriteFile(dbFilename, d1, 0644)
		if err != nil {
			return nil, err
		}
	} else {
		d1, err := ioutil.ReadFile(dbFilename)
		if err != nil {
			return nil, err
		}

		err = json.Unmarshal(d1, a)
		if err != nil {
			return nil, err
		}
	}

	return a, nil
}

// ErrUndefinedKey is returned when the key requested from get is undefined.
var ErrUndefinedKey = errors.New("undefined key")

// GetString gets the give keys leaf as a string.
//
// Returns an ErrUndefinedKey if the key is not defined
func (a *tree) GetString(key string) (string, error) {
	a.Lock()
	defer a.Unlock()

	if _, ok := a.Leaves[key]; !ok {
		return "", ErrUndefinedKey
	}

	return a.Leaves[key], nil
}

// SetString sets the give key leaf as a string.
func (a *tree) SetString(key string, val string) {
	a.Lock()
	defer a.Unlock()

	a.Leaves[key] = val
}

// GetInt gets the give keys leaf as an int.
//
// Returns an ErrUndefinedKey if the key is not defined.
//
// Will return an error if ParseInt of the value of the leaf fails.
func (a *tree) GetInt(key string) (int, error) {
	str, err := a.GetString(key)
	if err != nil {
		return 0, err
	}

	i, err := strconv.Atoi(str)
	if err != nil {
		return 0, err
	}
	return i, nil
}

// SetInt sets the give key leaf as a int.
func (a *tree) SetInt(key string, val int) {
	a.Lock()
	defer a.Unlock()

	a.Leaves[key] = strconv.Itoa(val)
}

// GetInt64 gets the give keys leaf as an int64.
//
// Returns an ErrUndefinedKey if the key is not defined.
//
// Will return an error if ParseInt(...,10, 64) of the value of the leaf fails.
func (a *tree) GetInt64(key string) (int64, error) {
	str, err := a.GetString(key)
	if err != nil {
		return 0, err
	}

	i, err := strconv.ParseInt(str, 10, 64)
	if err != nil {
		return 0, err
	}
	return i, nil
}

// SetInt64 sets the give key leaf as a int64.
func (a *tree) SetInt64(key string, val int64) {
	a.Lock()
	defer a.Unlock()

	a.Leaves[key] = strconv.FormatInt(val, 10)
}

// IncrInt64 increments an int or int64 leaf and returns the new value as int64.
//
// If the key is undefined or non-integer, this will initialize it to 0 and increment it to 1
func (a *tree) IncrInt64(key string) int64 {
	v, _ := a.GetInt64(key)
	v += 1
	a.SetInt64(key, v)

	return v
}

// DecrInt64 decrements an int or int64 leaf and returns the new value as int64.
//
// If the key is undefined or non-integer, this will initialize it to 0 and decrements it to -1
func (a *tree) DecrInt64(key string) int64 {
	v, _ := a.GetInt64(key)
	v -= 1
	a.SetInt64(key, v)

	return v
}

// Delete removes the given leaf from the branch.
func (a *tree) Delete(key string) {
	a.Lock()
	defer a.Unlock()

	delete(a.Leaves, key)
}

// DeleteTree removes the given tree from the branch.
func (a *tree) DeleteTree(key string) {
	a.Lock()
	defer a.Unlock()

	delete(a.Branches, key)
}

func (a *tree) GetLeaves() map[string]string {
	return a.Leaves
}

func (a *tree) GetTrees() map[string]*tree {
	return a.Branches
}

// GetTree fetches a tree for app setting storage
func (a *tree) GetTree(key string) DataTree {
	a.Lock()
	defer a.Unlock()

	if _, ok := a.Branches[key]; !ok {
		a.Branches[key] = &tree{
			Branches: make(map[string]*tree),
			Leaves:   make(map[string]string),
		}
	}

	return a.Branches[key]
}

// HasTree checks if the given key is defined as a tree
func (a *tree) HasTree(key string) bool {
	a.Lock()
	defer a.Unlock()

	_, ok := a.Branches[key]
	return ok
}

// HasLeaf checks if the given key is defined as a leaf value
func (a *tree) HasLeaf(key string) bool {
	a.Lock()
	defer a.Unlock()

	_, ok := a.Leaves[key]
	return ok
}

// Persist causes the current state of the app settings to be persisted.
func (a *AppSettings) Persist() error {
	a.Lock()
	defer a.Unlock()

	var err error
	var d1 []byte

	if a.pretty {
		d1, err = json.MarshalIndent(a, "", "\t")
	} else {
		d1, err = json.Marshal(a)
	}
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(a.filename, d1, 0644)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package mails

import (
	"database/sql"
	"fmt"
	"log"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ses"
	"github.com/toomore/mailbox/campaign"
)

var svc *ses.SES

func init() {
	svc = ses.New(session.Must(session.NewSession(
		&aws.Config{
			Region: aws.String("us-east-1"),
			Credentials: credentials.NewStaticCredentials(
				os.Getenv("mailbox_ses_key"), os.Getenv("mailbox_ses_token"), ""),
		},
	),
	))
}

func baseParams() *ses.SendEmailInput {
	return &ses.SendEmailInput{
		Destination: &ses.Destination{
			ToAddresses: []*string{aws.String("")},
		},
		Message: &ses.Message{
			Body: &ses.Body{
				Html: &ses.Content{
					Data:    aws.String(""),
					Charset: aws.String("UTF-8"),
				},
			},
			Subject: &ses.Content{
				Data:    aws.String(""),
				Charset: aws.String("UTF-8"),
			},
		},
		Source: aws.String(os.Getenv("mailbox_ses_sender")),
	}
}

// GenParams is to gen email params
func GenParams(to string, message string, subject string) *ses.SendEmailInput {
	params := baseParams()
	params.Destination.ToAddresses[0] = aws.String(to)
	params.Message.Body.Html.Data = aws.String(message)
	params.Message.Subject.Data = aws.String(subject)
	return params
}

// Send is to send mail
func Send(params *ses.SendEmailInput) {
	resp, err := svc.SendEmail(params)
	if err != nil {
		log.Println(err)
	} else {
		log.Println(resp)
	}
}

// ProcessSend is to start send from rows
func ProcessSend(body []byte, rows *sql.Rows, cid string, replaceLink bool, subject string, dryRun bool) {
	var allATags []LinksData
	if replaceLink {
		allATags = FilterATags(body, cid)
	}

	var seed = campaign.GetSeed(cid)
	var count int
	for rows.Next() {
		var (
			email string
			fname string
			lname string
			msg   []byte
			no    string
		)
		rows.Scan(&no, &email, &fname, &lname)

		msg = body
		if replaceLink {
			ReplaceATag(&msg, allATags, cid, seed, no)
		}
		ReplaceFname(&msg, fname)
		ReplaceReader(&msg, cid, seed, no)
		if dryRun {
			log.Printf("%s\n", msg)
			for i, v := range allATags {
				fmt.Printf("%02d => [%s] %s\n", i, v.LinkID, v.URL)
			}
		} else {
			Send(GenParams(
				fmt.Sprintf("%s %s <%s>", fname, lname, email),
				string(msg),
				subject))
		}
		count++
	}
	log.Printf("\n  cid: %s, count: %d\n  Subject: `%s`\n", cid, count, subject)
}
<commit_msg>Add goroutine to send<commit_after>package mails

import (
	"database/sql"
	"fmt"
	"log"
	"os"
	"sync"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ses"
	"github.com/toomore/mailbox/campaign"
)

var svc *ses.SES
var ql chan struct{}

func init() {
	ql = make(chan struct{}, 10)

	svc = ses.New(session.Must(session.NewSession(
		&aws.Config{
			Region: aws.String("us-east-1"),
			Credentials: credentials.NewStaticCredentials(
				os.Getenv("mailbox_ses_key"), os.Getenv("mailbox_ses_token"), ""),
		},
	),
	))
}

func baseParams() *ses.SendEmailInput {
	return &ses.SendEmailInput{
		Destination: &ses.Destination{
			ToAddresses: []*string{aws.String("")},
		},
		Message: &ses.Message{
			Body: &ses.Body{
				Html: &ses.Content{
					Data:    aws.String(""),
					Charset: aws.String("UTF-8"),
				},
			},
			Subject: &ses.Content{
				Data:    aws.String(""),
				Charset: aws.String("UTF-8"),
			},
		},
		Source: aws.String(os.Getenv("mailbox_ses_sender")),
	}
}

// GenParams is to gen email params
func GenParams(to string, message string, subject string) *ses.SendEmailInput {
	params := baseParams()
	params.Destination.ToAddresses[0] = aws.String(to)
	params.Message.Body.Html.Data = aws.String(message)
	params.Message.Subject.Data = aws.String(subject)
	return params
}

// Send is to send mail
func Send(params *ses.SendEmailInput) {
	resp, err := svc.SendEmail(params)
	if err != nil {
		log.Println(err)
	} else {
		log.Println(resp)
	}
}

// SendWG is with WaitGroup
func SendWG(params *ses.SendEmailInput, wg *sync.WaitGroup) {
	Send(params)
	<-ql
	wg.Done()
}

// ProcessSend is to start send from rows
func ProcessSend(body []byte, rows *sql.Rows, cid string, replaceLink bool, subject string, dryRun bool) {
	var allATags []LinksData
	if replaceLink {
		allATags = FilterATags(body, cid)
	}

	var seed = campaign.GetSeed(cid)
	var count int
	var wg sync.WaitGroup
	for rows.Next() {
		var (
			email string
			fname string
			lname string
			msg   []byte
			no    string
		)
		rows.Scan(&no, &email, &fname, &lname)

		msg = body
		if replaceLink {
			ReplaceATag(&msg, allATags, cid, seed, no)
		}
		ReplaceFname(&msg, fname)
		ReplaceReader(&msg, cid, seed, no)
		if dryRun {
			log.Printf("%s\n", msg)
			for i, v := range allATags {
				fmt.Printf("%02d => [%s] %s\n", i, v.LinkID, v.URL)
			}
		} else {
			wg.Add(1)
			ql <- struct{}{}
			go SendWG(GenParams(
				fmt.Sprintf("%s %s <%s>", fname, lname, email),
				string(msg),
				subject), &wg)
		}
		count++
	}
	wg.Wait()
	log.Printf("\n  cid: %s, count: %d\n  Subject: `%s`\n", cid, count, subject)
}
<|endoftext|>


<commit_before>package admin

import (
	"flag"
	"fmt"
	"gondola/mux"
	"gondola/util"
	"io"
	"io/ioutil"
	"os"
	"reflect"
	"runtime"
	"sort"
	"strings"
)

const tabWidth = 8

var (
	commands = map[string]*command{}
)

type command struct {
	handler mux.Handler
	help    string
	flags   []*Flag
}

// Register registers a new admin command with the
// given function and options (which might be nil).
func Register(f mux.Handler, o *Options) error {
	var name string
	var help string
	var flags []*Flag
	if o != nil {
		name = o.Name
		help = o.Help
		flags = o.Flags
	}
	if name == "" {
		qname := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
		p := strings.Split(qname, ".")
		name = p[len(p)-1]
		if name == "" {
			return fmt.Errorf("could not determine name for function %v. Please, provide a name using Options.", f)
		}
	}
	cmdName := util.CamelCaseToLower(name, "-")
	if _, ok := commands[cmdName]; ok {
		return fmt.Errorf("duplicate command name %q", name)
	}
	commands[cmdName] = &command{
		handler: f,
		help:    help,
		flags:   flags,
	}
	return nil
}

// MustRegister works like Register, but panics
// if there's an error
func MustRegister(f mux.Handler, o *Options) {
	if err := Register(f, o); err != nil {
		panic(err)
	}
}

func performCommand(name string, cmd *command, args []string, m *mux.Mux) {
	// Parse command flags
	set := flag.NewFlagSet(name, flag.ContinueOnError)
	set.Usage = func() {
		commandHelp(name, -1, os.Stderr)
	}
	flags := map[string]interface{}{}
	for _, arg := range cmd.flags {
		switch arg.typ {
		case typBool:
			var b bool
			set.BoolVar(&b, arg.name, arg.def.(bool), arg.help)
			flags[arg.name] = &b
		case typInt:
			var i int
			set.IntVar(&i, arg.name, arg.def.(int), arg.help)
			flags[arg.name] = &i
		case typString:
			var s string
			set.StringVar(&s, arg.name, arg.def.(string), arg.help)
			flags[arg.name] = &s
		default:
			panic("invalid arg type")
		}
	}
	// Print error/help messages ourselves
	set.SetOutput(ioutil.Discard)
	err := set.Parse(args)
	if err != nil {
		if err == flag.ErrHelp {
			return
		}
		if strings.Contains(err.Error(), "provided but not defined") {
			flagName := strings.TrimSpace(strings.Split(err.Error(), ":")[1])
			fmt.Fprintf(os.Stderr, "command %s does not accept flag %s\n", name, flagName)
			return
		}
		panic(err)
	}
	params := map[string]string{}
	for _, arg := range cmd.flags {
		params[arg.name] = fmt.Sprintf("%v", reflect.ValueOf(flags[arg.name]).Elem().Interface())
	}
	provider := &contextProvider{
		args:   set.Args(),
		params: params,
	}
	ctx := m.NewContext(provider)
	defer m.CloseContext(ctx)
	cmd.handler(ctx)
}

func Perform(m *mux.Mux) bool {
	if !flag.Parsed() {
		flag.Parse()
	}
	args := flag.Args()
	if len(args) > 0 {
		cmd := strings.ToLower(args[0])
		for k, v := range commands {
			if cmd == k {
				performCommand(k, v, args[1:], m)
				return true
			}
		}
	}
	return false
}

// commandHelp prints the help for the given command
// to the given io.Writer
func commandHelp(name string, maxLen int, w io.Writer) {
	if maxLen < 0 {
		maxLen = len(name) + 1
	}
	ntabs := (maxLen / tabWidth) - ((len(name) + 1) / tabWidth) + 1
	tabs := strings.Repeat("\t", ntabs)
	fmt.Fprintf(w, "%s:%s%s\n", name, tabs, commands[name].help)
	if flags := commands[name].flags; len(flags) > 0 {
		indent := strings.Repeat("\t", (maxLen/tabWidth)+1)
		fmt.Fprintf(w, "%sAvailable flags for %v:\n", indent, name)
		for _, arg := range flags {
			if arg.help != "" {
				fmt.Fprintf(w, "%s-%s=%v:\t%s\n", indent, arg.name, arg.def, arg.help)
			} else {
				fmt.Fprintf(w, "%s-%s=%v\n", indent, arg.name, arg.def)
			}
		}
	}
}

// commandsHelp prints the help for all commands to the given io.Writer
func commandsHelp(w io.Writer) {
	var cmds []string
	maxLen := 0
	for k, _ := range commands {
		if l := len(k); l > maxLen {
			maxLen = l
		}
		cmds = append(cmds, k)
	}
	maxLen += 1
	sort.Strings(cmds)
	for _, v := range cmds {
		commandHelp(v, maxLen, w)
	}
}

// Implementation of the help command for Gondola apps
func help(ctx *mux.Context) {
	var cmd string
	ctx.ParseIndexValue(0, &cmd)
	if cmd != "" {
		c := strings.ToLower(cmd)
		if _, ok := commands[c]; ok {
			fmt.Fprintf(os.Stderr, "Help for administrative command %s:\n", c)
			commandHelp(c, -1, os.Stderr)
		} else {
			fmt.Fprintf(os.Stderr, "No such administrative command %q\n", cmd)
		}
	} else {
		fmt.Fprintf(os.Stderr, "Administrative commands:\n")
		commandsHelp(os.Stderr)
	}
}

func init() {
	MustRegister(help, &Options{
		Help: "Show available commands with their respective help.",
	})
}
<commit_msg>Quote default values when they're strings<commit_after>package admin

import (
	"flag"
	"fmt"
	"gondola/mux"
	"gondola/util"
	"io"
	"io/ioutil"
	"os"
	"reflect"
	"runtime"
	"sort"
	"strings"
)

const tabWidth = 8

var (
	commands = map[string]*command{}
)

type command struct {
	handler mux.Handler
	help    string
	flags   []*Flag
}

// Register registers a new admin command with the
// given function and options (which might be nil).
func Register(f mux.Handler, o *Options) error {
	var name string
	var help string
	var flags []*Flag
	if o != nil {
		name = o.Name
		help = o.Help
		flags = o.Flags
	}
	if name == "" {
		qname := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
		p := strings.Split(qname, ".")
		name = p[len(p)-1]
		if name == "" {
			return fmt.Errorf("could not determine name for function %v. Please, provide a name using Options.", f)
		}
	}
	cmdName := util.CamelCaseToLower(name, "-")
	if _, ok := commands[cmdName]; ok {
		return fmt.Errorf("duplicate command name %q", name)
	}
	commands[cmdName] = &command{
		handler: f,
		help:    help,
		flags:   flags,
	}
	return nil
}

// MustRegister works like Register, but panics
// if there's an error
func MustRegister(f mux.Handler, o *Options) {
	if err := Register(f, o); err != nil {
		panic(err)
	}
}

func performCommand(name string, cmd *command, args []string, m *mux.Mux) {
	// Parse command flags
	set := flag.NewFlagSet(name, flag.ContinueOnError)
	set.Usage = func() {
		commandHelp(name, -1, os.Stderr)
	}
	flags := map[string]interface{}{}
	for _, arg := range cmd.flags {
		switch arg.typ {
		case typBool:
			var b bool
			set.BoolVar(&b, arg.name, arg.def.(bool), arg.help)
			flags[arg.name] = &b
		case typInt:
			var i int
			set.IntVar(&i, arg.name, arg.def.(int), arg.help)
			flags[arg.name] = &i
		case typString:
			var s string
			set.StringVar(&s, arg.name, arg.def.(string), arg.help)
			flags[arg.name] = &s
		default:
			panic("invalid arg type")
		}
	}
	// Print error/help messages ourselves
	set.SetOutput(ioutil.Discard)
	err := set.Parse(args)
	if err != nil {
		if err == flag.ErrHelp {
			return
		}
		if strings.Contains(err.Error(), "provided but not defined") {
			flagName := strings.TrimSpace(strings.Split(err.Error(), ":")[1])
			fmt.Fprintf(os.Stderr, "command %s does not accept flag %s\n", name, flagName)
			return
		}
		panic(err)
	}
	params := map[string]string{}
	for _, arg := range cmd.flags {
		params[arg.name] = fmt.Sprintf("%v", reflect.ValueOf(flags[arg.name]).Elem().Interface())
	}
	provider := &contextProvider{
		args:   set.Args(),
		params: params,
	}
	ctx := m.NewContext(provider)
	defer m.CloseContext(ctx)
	cmd.handler(ctx)
}

func Perform(m *mux.Mux) bool {
	if !flag.Parsed() {
		flag.Parse()
	}
	args := flag.Args()
	if len(args) > 0 {
		cmd := strings.ToLower(args[0])
		for k, v := range commands {
			if cmd == k {
				performCommand(k, v, args[1:], m)
				return true
			}
		}
	}
	return false
}

// commandHelp prints the help for the given command
// to the given io.Writer
func commandHelp(name string, maxLen int, w io.Writer) {
	if maxLen < 0 {
		maxLen = len(name) + 1
	}
	ntabs := (maxLen / tabWidth) - ((len(name) + 1) / tabWidth) + 1
	tabs := strings.Repeat("\t", ntabs)
	fmt.Fprintf(w, "%s:%s%s\n", name, tabs, commands[name].help)
	if flags := commands[name].flags; len(flags) > 0 {
		indent := strings.Repeat("\t", (maxLen/tabWidth)+1)
		fmt.Fprintf(w, "%sAvailable flags for %v:\n", indent, name)
		for _, arg := range flags {
			if arg.help != "" {
				if arg.typ == typString {
					fmt.Fprintf(w, "%s-%s=%q:\t%s\n", indent, arg.name, arg.def, arg.help)
				} else {
					fmt.Fprintf(w, "%s-%s=%v:\t%s\n", indent, arg.name, arg.def, arg.help)
				}
			} else {
				fmt.Fprintf(w, "%s-%s=%v\n", indent, arg.name, arg.def)
			}
		}
	}
}

// commandsHelp prints the help for all commands to the given io.Writer
func commandsHelp(w io.Writer) {
	var cmds []string
	maxLen := 0
	for k, _ := range commands {
		if l := len(k); l > maxLen {
			maxLen = l
		}
		cmds = append(cmds, k)
	}
	maxLen += 1
	sort.Strings(cmds)
	for _, v := range cmds {
		commandHelp(v, maxLen, w)
	}
}

// Implementation of the help command for Gondola apps
func help(ctx *mux.Context) {
	var cmd string
	ctx.ParseIndexValue(0, &cmd)
	if cmd != "" {
		c := strings.ToLower(cmd)
		if _, ok := commands[c]; ok {
			fmt.Fprintf(os.Stderr, "Help for administrative command %s:\n", c)
			commandHelp(c, -1, os.Stderr)
		} else {
			fmt.Fprintf(os.Stderr, "No such administrative command %q\n", cmd)
		}
	} else {
		fmt.Fprintf(os.Stderr, "Administrative commands:\n")
		commandsHelp(os.Stderr)
	}
}

func init() {
	MustRegister(help, &Options{
		Help: "Show available commands with their respective help.",
	})
}
<|endoftext|>


<commit_before>// Package assess provides functions for measuring the difference between the
// values predicted by a model and the values actually observed.
package assess

import (
	"math"
)

// MSE computes the mean-square error.
//
// https://en.wikipedia.org/wiki/Mean_squared_error
func MSE(predicted, observed []float64) float64 {
	var delta, sum float64

	for i := range observed {
		delta = predicted[i] - observed[i]
		sum += delta * delta
	}

	return sum / float64(len(observed))
}

// RMSE computes the root-mean-square error.
//
// https://en.wikipedia.org/wiki/Root-mean-square_deviation
func RMSE(predicted, observed []float64) float64 {
	return math.Sqrt(MSE(predicted, observed))
}

// NRMSE computes the normalized root-mean-square error.
//
// https://en.wikipedia.org/wiki/Root-mean-square_deviation#Normalized_root-mean-square_deviation
func NRMSE(predicted, observed []float64) float64 {
	count := len(observed)
	if count == 0 {
		return 0
	}

	min, max := observed[0], observed[0]
	for i := 1; i < count; i++ {
		if observed[i] < min {
			min = observed[i]
		}
		if observed[i] > max {
			max = observed[i]
		}
	}

	return RMSE(predicted, observed) / (max - min)
}
<commit_msg>Love UTF-8!<commit_after>// Package assess provides functions for measuring the difference between the
// values predicted by a model and the values actually observed.
package assess

import (
	"math"
)

// MSE computes the mean-square error.
//
// https://en.wikipedia.org/wiki/Mean_squared_error
func MSE(predicted, observed []float64) float64 {
	var sum, Δ float64

	for i := range observed {
		Δ = predicted[i] - observed[i]
		sum += Δ * Δ
	}

	return sum / float64(len(observed))
}

// RMSE computes the root-mean-square error.
//
// https://en.wikipedia.org/wiki/Root-mean-square_deviation
func RMSE(predicted, observed []float64) float64 {
	return math.Sqrt(MSE(predicted, observed))
}

// NRMSE computes the normalized root-mean-square error.
//
// https://en.wikipedia.org/wiki/Root-mean-square_deviation#Normalized_root-mean-square_deviation
func NRMSE(predicted, observed []float64) float64 {
	count := len(observed)
	if count == 0 {
		return 0
	}

	min, max := observed[0], observed[0]
	for i := 1; i < count; i++ {
		if observed[i] < min {
			min = observed[i]
		}
		if observed[i] > max {
			max = observed[i]
		}
	}

	return RMSE(predicted, observed) / (max - min)
}
<|endoftext|>


<commit_before><commit_msg>return ErrNil if not found<commit_after><|endoftext|>


<commit_before>package matrix

import (
	"testing"
)

func TestIdentity(t *testing.T) {
	m := Identity(11)
	rows, cols := m.Dims()
	for i := 0; i < rows; i++ {
		for k := 0; k < cols; k++ {
			a := m.Get(i, k)
			if i == k {
				if a != 1 {
					t.Fatalf("(%d, %d) == %f != 1", i, k, a)
				}
			} else {
				if a != 0 {
					t.Fatalf("(%d, %d) == %f != 0", i, k, a)
				}
			}
		}
	}
}
<commit_msg>Test fixed<commit_after>package matrix

import (
	"testing"
)

func TestIdentity(t *testing.T) {
	m := DenseI(11)
	rows, cols := m.Dims()
	for i := 0; i < rows; i++ {
		for k := 0; k < cols; k++ {
			a := m.Get(i, k)
			if i == k {
				if a != 1 {
					t.Fatalf("(%d, %d) == %f != 1", i, k, a)
				}
			} else {
				if a != 0 {
					t.Fatalf("(%d, %d) == %f != 0", i, k, a)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package archivist

import (
    "archive/zip"
    "io"
    "os"
    "path/filepath"
    "strings"
)

type zipper struct {
    src    string
    dst    string
    writer *zip.Writer
}

// zipper.do initialises output file, archive and directory tree walk function
func (z *zipper) do() error {
    out, err := os.Create(z.dst)
    if err != nil {
        return err
    }

    z.writer = zip.NewWriter(out)
    if err := filepath.Walk(z.src, z.walk); err != nil {
        return err
    }

    if err := z.writer.Close(); err != nil {
        return err
    }

    return out.Close()
}

// zipper.walk gets called for each file in given directory tree
func (z *zipper) walk(path string, info os.FileInfo, err error) error {
    if err != nil { return err }

    if !info.Mode().IsRegular() || info.Size() == 0 {
        return nil
    }

    file, err := os.Open(path)
    if err != nil {
        return err
    }
    defer file.Close()

    fileName := strings.TrimPrefix(path, z.src + string(filepath.Separator))
    w, err := z.writer.Create(fileName)
    if err != nil {
        return err
    }

    // Not efficient FIXME
    _, err = io.Copy(w, file)

    return err
}

type unzipper struct {
    src    string
    dst    string
    reader *zip.ReadCloser
}

// unzipper.do initialises output file and unzips source there
func (z *unzipper) do() error {
    err := os.MkdirAll(z.dst, os.ModeDir)
    if err != nil { return err }

    z.reader, err = zip.OpenReader(z.src)
    if err != nil { return err }

    for _, f := range z.reader.File {
        r, err := f.Open()
        if err != nil { return err }

        w, err := os.Create(filepath.Join(z.dst, f.Name))
        if err != nil {
            z.reader.Close()
            r.Close()
            return err
        }

        if _, err := io.Copy(w, r); err != nil {
            z.reader.Close()
            w.Close()
            r.Close()
            return err
        }

        if err := r.Close(); err != nil {
            w.Close()
            z.reader.Close()
            return err
        }

        if err := w.Close(); err != nil {
            return err
        }
    }

    if err := z.reader.Close(); err != nil {
        return err
    }

    return nil
}

// Zip is an exported method which sanitizes io paths and starts archiving
func Zip(src, dst string) error {
    z := &zipper{
        src: filepath.Clean(filepath.FromSlash(src)),
        dst: filepath.Clean(filepath.FromSlash(dst)),
    }
    return z.do()
}

// Unzip is an exported method which sanitizes io paths and starts unzipping
func Unzip(src, dst string) error {
    z := &unzipper{
        src: filepath.Clean(filepath.FromSlash(src)),
        dst: filepath.Clean(filepath.FromSlash(dst)),
    }
    return z.do()
}
<commit_msg>Break down a large function<commit_after>package archivist

import (
    "archive/zip"
    "io"
    "os"
    "path/filepath"
    "strings"
)

type zipper struct {
    src    string
    dst    string
    writer *zip.Writer
}

// zipper.do initialises output file, archive and directory tree walk function
func (z *zipper) do() error {
    out, err := os.Create(z.dst)
    if err != nil {
        return err
    }

    z.writer = zip.NewWriter(out)
    if err := filepath.Walk(z.src, z.walk); err != nil {
        return err
    }

    if err := z.writer.Close(); err != nil {
        return err
    }

    return out.Close()
}

// zipper.walk gets called for each file in given directory tree
func (z *zipper) walk(path string, info os.FileInfo, err error) error {
    if err != nil { return err }

    if !info.Mode().IsRegular() || info.Size() == 0 {
        return nil
    }

    file, err := os.Open(path)
    if err != nil {
        return err
    }
    defer file.Close()

    fileName := strings.TrimPrefix(path, z.src + string(filepath.Separator))
    w, err := z.writer.Create(fileName)
    if err != nil {
        return err
    }

    // Not efficient FIXME
    _, err = io.Copy(w, file)

    return err
}

type unzipper struct {
    src    string
    dst    string
    reader *zip.ReadCloser
}

// unzipper.do initialises output file and unzips source there
func (z *unzipper) do() error {
    err := os.MkdirAll(z.dst, os.ModeDir)
    if err != nil { return err }

    z.reader, err = zip.OpenReader(z.src)
    if err != nil { return err }

    for _, f := range z.reader.File {
        if err := z.unzip(f); err != nil {
            z.reader.Close()
            return err
        }
    }

    if err := z.reader.Close(); err != nil {
        return err
    }

    return nil
}

func (z *unzipper) unzip(f *zip.File) error {
    r, err := f.Open()
    if err != nil { return err }

    w, err := os.Create(filepath.Join(z.dst, f.Name))
    if err != nil {
        z.reader.Close()
        r.Close()
        return err
    }

    if _, err := io.Copy(w, r); err != nil {
        z.reader.Close()
        w.Close()
        r.Close()
        return err
    }

    if err := r.Close(); err != nil {
        w.Close()
        z.reader.Close()
        return err
    }

    if err := w.Close(); err != nil {
        return err
    }

    return nil
}

// Zip is an exported method which sanitizes io paths and starts archiving
func Zip(src, dst string) error {
    z := &zipper{
        src: filepath.Clean(filepath.FromSlash(src)),
        dst: filepath.Clean(filepath.FromSlash(dst)),
    }
    return z.do()
}

// Unzip is an exported method which sanitizes io paths and starts unzipping
func Unzip(src, dst string) error {
    z := &unzipper{
        src: filepath.Clean(filepath.FromSlash(src)),
        dst: filepath.Clean(filepath.FromSlash(dst)),
    }
    return z.do()
}
<|endoftext|>


<commit_before>package main_test

import (
	. "github.com/bluemixgaragelondon/cf-blue-green-deploy"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"strings"
)

var _ = Describe("Args", func() {
	Context("With an appname only", func() {
		args := NewArgs(bgdArgs("appname"))

		It("sets the app name", func() {
			Expect(args.AppName).To(Equal("appname"))
		})

		It("does not set the smoke test file", func() {
			Expect(args.SmokeTestPath).To(BeZero())
		})

		It("does not set a manifest", func() {
			Expect(args.ManifestPath).To(BeZero())
		})
	})

	Context("With a smoke test and an appname", func() {
		args := NewArgs(bgdArgs("appname --smoke-test script/smoke-test"))

		It("sets the smoke test file", func() {
			Expect(args.SmokeTestPath).To(Equal("script/smoke-test"))
		})

		It("sets the app name", func() {
			Expect(args.AppName).To(Equal("appname"))
		})

		It("does not set a manifest", func() {
			Expect(args.ManifestPath).To(BeZero())
		})
	})

	Context("With an appname smoke test and a manifest", func() {
		args := NewArgs(bgdArgs("appname --smoke-test smokey -f manifest.yml"))

		It("sets the smoke test file", func() {
			Expect(args.SmokeTestPath).To(Equal("smokey"))
		})

		It("sets the app name", func() {
			Expect(args.AppName).To(Equal("appname"))
		})

		It("sets a manifest", func() {
			Expect(args.ManifestPath).To(Equal("manifest.yml"))
		})
	})

	Context("When a global cf flag is set with an app name", func() {
		args := NewArgs([]string{"cf", "-v", "blue-green-deploy", "app"})

		It("sets the app name", func() {
			Expect(args.AppName).To(Equal("app"))
		})
	})

	Context("When the bgd abbreviation is used", func() {
		args := NewArgs([]string{"cf", "bgd", "app"})

		It("sets the app name", func() {
			Expect(args.AppName).To(Equal("app"))
		})
	})
})

func bgdArgs(argString string) []string {
	args := strings.Split(argString, " ")
	return append([]string{"cf", "blue-green-deploy"}, args...)
}
<commit_msg>Make test slightly stronger<commit_after>package main_test

import (
	. "github.com/bluemixgaragelondon/cf-blue-green-deploy"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"strings"
)

var _ = Describe("Args", func() {
	Context("With an appname only", func() {
		args := NewArgs(bgdArgs("appname"))

		It("sets the app name", func() {
			Expect(args.AppName).To(Equal("appname"))
		})

		It("does not set the smoke test file", func() {
			Expect(args.SmokeTestPath).To(BeZero())
		})

		It("does not set a manifest", func() {
			Expect(args.ManifestPath).To(BeZero())
		})
	})

	Context("With a smoke test and an appname", func() {
		args := NewArgs(bgdArgs("appname --smoke-test script/smoke-test"))

		It("sets the smoke test file", func() {
			Expect(args.SmokeTestPath).To(Equal("script/smoke-test"))
		})

		It("sets the app name", func() {
			Expect(args.AppName).To(Equal("appname"))
		})

		It("does not set a manifest", func() {
			Expect(args.ManifestPath).To(BeZero())
		})
	})

	Context("With an appname smoke test and a manifest", func() {
		args := NewArgs(bgdArgs("appname --smoke-test smokey -f custommanifest.yml"))

		It("sets the smoke test file", func() {
			Expect(args.SmokeTestPath).To(Equal("smokey"))
		})

		It("sets the app name", func() {
			Expect(args.AppName).To(Equal("appname"))
		})

		It("sets a manifest", func() {
			Expect(args.ManifestPath).To(Equal("custommanifest.yml"))
		})
	})

	Context("When a global cf flag is set with an app name", func() {
		args := NewArgs([]string{"cf", "-v", "blue-green-deploy", "app"})

		It("sets the app name", func() {
			Expect(args.AppName).To(Equal("app"))
		})
	})

	Context("When the bgd abbreviation is used", func() {
		args := NewArgs([]string{"cf", "bgd", "app"})

		It("sets the app name", func() {
			Expect(args.AppName).To(Equal("app"))
		})
	})
})

func bgdArgs(argString string) []string {
	args := strings.Split(argString, " ")
	return append([]string{"cf", "blue-green-deploy"}, args...)
}
<|endoftext|>


<commit_before>package terraform

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"io"
	"sort"
	"strings"
	"sync"
)

// The format byte is prefixed into the diff file format so that we have
// the ability in the future to change the file format if we want for any
// reason.
const diffFormatByte byte = 1

// Diff tracks the differences between resources to apply.
type Diff struct {
	Resources map[string]*ResourceDiff
	once      sync.Once
}

// ReadDiff reads a diff structure out of a reader in the format that
// was written by WriteDiff.
func ReadDiff(src io.Reader) (*Diff, error) {
	var result *Diff

	var formatByte [1]byte
	n, err := src.Read(formatByte[:])
	if err != nil {
		return nil, err
	}
	if n != len(formatByte) {
		return nil, errors.New("failed to read diff version byte")
	}

	if formatByte[0] != diffFormatByte {
		return nil, fmt.Errorf("unknown diff file version: %d", formatByte[0])
	}

	dec := gob.NewDecoder(src)
	if err := dec.Decode(&result); err != nil {
		return nil, err
	}

	return result, nil
}

// WriteDiff writes a diff somewhere in a binary format.
func WriteDiff(d *Diff, dst io.Writer) error {
	n, err := dst.Write([]byte{diffFormatByte})
	if err != nil {
		return err
	}
	if n != 1 {
		return errors.New("failed to write diff version byte")
	}

	return gob.NewEncoder(dst).Encode(d)
}

func (d *Diff) init() {
	d.once.Do(func() {
		if d.Resources == nil {
			d.Resources = make(map[string]*ResourceDiff)
		}
	})
}

// Empty returns true if the diff has no changes.
func (d *Diff) Empty() bool {
	if len(d.Resources) == 0 {
		return true
	}

	for _, rd := range d.Resources {
		if len(rd.Attributes) > 0 {
			return false
		}
	}

	return true
}

// String outputs the diff in a long but command-line friendly output
// format that users can read to quickly inspect a diff.
func (d *Diff) String() string {
	var buf bytes.Buffer

	names := make([]string, 0, len(d.Resources))
	for name, _ := range d.Resources {
		names = append(names, name)
	}
	sort.Strings(names)

	for _, name := range names {
		rdiff := d.Resources[name]

		crud := "UPDATE"
		if rdiff.RequiresNew() {
			crud = "CREATE"
		}

		buf.WriteString(fmt.Sprintf(
			"%s: %s\n",
			crud,
			name))

		keyLen := 0
		keys := make([]string, 0, len(rdiff.Attributes))
		for key, _ := range rdiff.Attributes {
			keys = append(keys, key)
			if len(key) > keyLen {
				keyLen = len(key)
			}
		}
		sort.Strings(keys)

		for _, attrK := range keys {
			attrDiff := rdiff.Attributes[attrK]

			v := attrDiff.New
			if attrDiff.NewComputed {
				v = "<computed>"
			}

			newResource := ""
			if attrDiff.RequiresNew {
				newResource = " (forces new resource)"
			}

			buf.WriteString(fmt.Sprintf(
				"  %s:%s %#v => %#v%s\n",
				attrK,
				strings.Repeat(" ", keyLen-len(attrK)),
				attrDiff.Old,
				v,
				newResource))
		}
	}

	return buf.String()
}

// ResourceDiff is the diff of a resource from some state to another.
type ResourceDiff struct {
	Attributes map[string]*ResourceAttrDiff
}

// ResourceAttrDiff is the diff of a single attribute of a resource.
type ResourceAttrDiff struct {
	Old         string // Old Value
	New         string // New Value
	NewComputed bool   // True if new value is computed (unknown currently)
	RequiresNew bool   // True if change requires new resource
	Type        DiffAttrType
}

// DiffAttrType is an enum type that says whether a resource attribute
// diff is an input attribute (comes from the configuration) or an
// output attribute (comes as a result of applying the configuration). An
// example input would be "ami" for AWS and an example output would be
// "private_ip".
type DiffAttrType byte

const (
	DiffAttrUnknown DiffAttrType = iota
	DiffAttrInput
	DiffAttrOutput
)

// Empty returns true if this diff encapsulates no changes.
// TODO(mitchellh): test
func (d *ResourceDiff) Empty() bool {
	if d == nil {
		return true
	}

	return len(d.Attributes) == 0
}

// RequiresNew returns true if the diff requires the creation of a new
// resource (implying the destruction of the old).
func (d *ResourceDiff) RequiresNew() bool {
	if d == nil {
		return false
	}

	for _, rd := range d.Attributes {
		if rd.RequiresNew {
			return true
		}
	}

	return false
}
<commit_msg>terraform: diff has Destroy flag<commit_after>package terraform

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"io"
	"sort"
	"strings"
	"sync"
)

// The format byte is prefixed into the diff file format so that we have
// the ability in the future to change the file format if we want for any
// reason.
const diffFormatByte byte = 1

// Diff tracks the differences between resources to apply.
type Diff struct {
	Resources map[string]*ResourceDiff
	once      sync.Once
}

// ReadDiff reads a diff structure out of a reader in the format that
// was written by WriteDiff.
func ReadDiff(src io.Reader) (*Diff, error) {
	var result *Diff

	var formatByte [1]byte
	n, err := src.Read(formatByte[:])
	if err != nil {
		return nil, err
	}
	if n != len(formatByte) {
		return nil, errors.New("failed to read diff version byte")
	}

	if formatByte[0] != diffFormatByte {
		return nil, fmt.Errorf("unknown diff file version: %d", formatByte[0])
	}

	dec := gob.NewDecoder(src)
	if err := dec.Decode(&result); err != nil {
		return nil, err
	}

	return result, nil
}

// WriteDiff writes a diff somewhere in a binary format.
func WriteDiff(d *Diff, dst io.Writer) error {
	n, err := dst.Write([]byte{diffFormatByte})
	if err != nil {
		return err
	}
	if n != 1 {
		return errors.New("failed to write diff version byte")
	}

	return gob.NewEncoder(dst).Encode(d)
}

func (d *Diff) init() {
	d.once.Do(func() {
		if d.Resources == nil {
			d.Resources = make(map[string]*ResourceDiff)
		}
	})
}

// Empty returns true if the diff has no changes.
func (d *Diff) Empty() bool {
	if len(d.Resources) == 0 {
		return true
	}

	for _, rd := range d.Resources {
		if len(rd.Attributes) > 0 {
			return false
		}
	}

	return true
}

// String outputs the diff in a long but command-line friendly output
// format that users can read to quickly inspect a diff.
func (d *Diff) String() string {
	var buf bytes.Buffer

	names := make([]string, 0, len(d.Resources))
	for name, _ := range d.Resources {
		names = append(names, name)
	}
	sort.Strings(names)

	for _, name := range names {
		rdiff := d.Resources[name]

		crud := "UPDATE"
		if rdiff.RequiresNew() {
			crud = "CREATE"
		}

		buf.WriteString(fmt.Sprintf(
			"%s: %s\n",
			crud,
			name))

		keyLen := 0
		keys := make([]string, 0, len(rdiff.Attributes))
		for key, _ := range rdiff.Attributes {
			keys = append(keys, key)
			if len(key) > keyLen {
				keyLen = len(key)
			}
		}
		sort.Strings(keys)

		for _, attrK := range keys {
			attrDiff := rdiff.Attributes[attrK]

			v := attrDiff.New
			if attrDiff.NewComputed {
				v = "<computed>"
			}

			newResource := ""
			if attrDiff.RequiresNew {
				newResource = " (forces new resource)"
			}

			buf.WriteString(fmt.Sprintf(
				"  %s:%s %#v => %#v%s\n",
				attrK,
				strings.Repeat(" ", keyLen-len(attrK)),
				attrDiff.Old,
				v,
				newResource))
		}
	}

	return buf.String()
}

// ResourceDiff is the diff of a resource from some state to another.
type ResourceDiff struct {
	Attributes map[string]*ResourceAttrDiff
	Destroy    bool
}

// ResourceAttrDiff is the diff of a single attribute of a resource.
type ResourceAttrDiff struct {
	Old         string // Old Value
	New         string // New Value
	NewComputed bool   // True if new value is computed (unknown currently)
	RequiresNew bool   // True if change requires new resource
	Type        DiffAttrType
}

// DiffAttrType is an enum type that says whether a resource attribute
// diff is an input attribute (comes from the configuration) or an
// output attribute (comes as a result of applying the configuration). An
// example input would be "ami" for AWS and an example output would be
// "private_ip".
type DiffAttrType byte

const (
	DiffAttrUnknown DiffAttrType = iota
	DiffAttrInput
	DiffAttrOutput
)

// Empty returns true if this diff encapsulates no changes.
// TODO(mitchellh): test
func (d *ResourceDiff) Empty() bool {
	if d == nil {
		return true
	}

	return len(d.Attributes) == 0
}

// RequiresNew returns true if the diff requires the creation of a new
// resource (implying the destruction of the old).
func (d *ResourceDiff) RequiresNew() bool {
	if d == nil {
		return false
	}

	for _, rd := range d.Attributes {
		if rd.RequiresNew {
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>package action

import (
	"errors"
	"fmt"
	"strings"

	"golang.org/x/net/websocket"
)

var (
	errNotImplemented = errors.New("Not Implemented!")
)

// Exec execute the js code on WebBrowser
func Exec(ws *websocket.Conn, js string) error {
	return websocket.Message.Send(ws, js)
}

//Enable the target form field or button.
func Enable(ws *websocket.Conn, target string) error {
	return errNotImplemented
}

//Disable  the target form field or button.
func Disable(ws *websocket.Conn, target string) error {
	return errNotImplemented
}

//Replace target with new content
func Replace(ws *websocket.Conn, target, content string) error {
	c := strings.Replace(content, "\n", "\\n", -1)
	c = strings.Replace(c, "\"", "\\\"", -1)
	js := fmt.Sprintf("$( \"#%s\" ).replaceWith(\"%s\");", target, c)
	return websocket.Message.Send(ws, js)
}

//HTML replace target with new content
func HTML(ws *websocket.Conn, target, content string) error {
	c := strings.Replace(content, "\n", "\\n", -1)
	c = strings.Replace(c, "\"", "\\\"", -1)
	js := fmt.Sprintf("$( \"#%s\" ).html(\"%s\");", target, c)
	return websocket.Message.Send(ws, js)
}

//Remove target from the DOM
func Remove(ws *websocket.Conn, target string) error {
	return errNotImplemented
}

//InsertTop Insert content at the top of target
func InsertTop(ws *websocket.Conn, target, content string) error {
	return errNotImplemented
}

//InsertBottom Insert content at the bottom of target
func InsertBottom(ws *websocket.Conn, target, content string) error {
	return errNotImplemented
}

//InsertBefore Insert content at the before of target
func InsertBefore(ws *websocket.Conn, target, content string) error {
	return errNotImplemented
}

//InsertAfter Insert content at the after of target
func InsertAfter(ws *websocket.Conn, target, content string) error {
	return errNotImplemented
}

//Redirect to url
func Redirect(ws *websocket.Conn, url string) error {
	return errNotImplemented
}

//Set update a form element (textbox, dropdown, checkbox, etc) to set text value of TargetID.
func Set(ws *websocket.Conn, target, value string) error {
	js := fmt.Sprintf("$('#%s').val('%s');", target, value)
	err := websocket.Message.Send(ws, js)
	if err != nil {
		return err
	}
	return nil
}

//Get content of form element
func Get(ws *websocket.Conn, target string) (string, error) {
	ret := ""
	js := fmt.Sprintf("ws.send($('#%s').val());", target)
	err := websocket.Message.Send(ws, js)
	if err != nil {
		return "", err
	}

	err = websocket.Message.Receive(ws, &ret)
	if err != nil {
		return "", err
	}

	return ret, nil
}

//Wire bind an action to an event on target
func Wire(ws *websocket.Conn, target, event, act string) error {
	return errNotImplemented
}

//SendEvent send an event to server
func SendEvent(ws *websocket.Conn, event string) error {
	js := fmt.Sprintf("sendEvent('%s');", event)
	err := websocket.Message.Send(ws, js)
	if err != nil {
		return err
	}

	return nil
}

//Alert show alert message in browser
func Alert(ws *websocket.Conn, message string) error {
	js := fmt.Sprintf("alert('%s');", message)
	err := websocket.Message.Send(ws, js)
	if err != nil {
		return err
	}

	return nil
}

//BlockUI block page interaction
func BlockUI(ws *websocket.Conn) {
	Exec(ws, "$.blockUI();")
	/*	Exec(ws, "$.blockUI();ws.send('blocked');")

		ret := ""
		err := websocket.Message.Receive(ws, &ret)
		if err != nil {
			log.Println("ERROR blockUI", err)
		}*/
}

//UnblockUI block page interaction
func UnblockUI(ws *websocket.Conn) {
	Exec(ws, "$.unblockUI();")
}
<commit_msg>code commented removed<commit_after>package action

import (
	"errors"
	"fmt"
	"strings"

	"golang.org/x/net/websocket"
)

var (
	errNotImplemented = errors.New("Not Implemented!")
)

// Exec execute the js code on WebBrowser
func Exec(ws *websocket.Conn, js string) error {
	return websocket.Message.Send(ws, js)
}

//Enable the target form field or button.
func Enable(ws *websocket.Conn, target string) error {
	return errNotImplemented
}

//Disable  the target form field or button.
func Disable(ws *websocket.Conn, target string) error {
	return errNotImplemented
}

//Replace target with new content
func Replace(ws *websocket.Conn, target, content string) error {
	c := strings.Replace(content, "\n", "\\n", -1)
	c = strings.Replace(c, "\"", "\\\"", -1)
	js := fmt.Sprintf("$( \"#%s\" ).replaceWith(\"%s\");", target, c)
	return websocket.Message.Send(ws, js)
}

//HTML replace target with new content
func HTML(ws *websocket.Conn, target, content string) error {
	c := strings.Replace(content, "\n", "\\n", -1)
	c = strings.Replace(c, "\"", "\\\"", -1)
	js := fmt.Sprintf("$( \"#%s\" ).html(\"%s\");", target, c)
	return websocket.Message.Send(ws, js)
}

//Remove target from the DOM
func Remove(ws *websocket.Conn, target string) error {
	return errNotImplemented
}

//InsertTop Insert content at the top of target
func InsertTop(ws *websocket.Conn, target, content string) error {
	return errNotImplemented
}

//InsertBottom Insert content at the bottom of target
func InsertBottom(ws *websocket.Conn, target, content string) error {
	return errNotImplemented
}

//InsertBefore Insert content at the before of target
func InsertBefore(ws *websocket.Conn, target, content string) error {
	return errNotImplemented
}

//InsertAfter Insert content at the after of target
func InsertAfter(ws *websocket.Conn, target, content string) error {
	return errNotImplemented
}

//Redirect to url
func Redirect(ws *websocket.Conn, url string) error {
	return errNotImplemented
}

//Set update a form element (textbox, dropdown, checkbox, etc) to set text value of TargetID.
func Set(ws *websocket.Conn, target, value string) error {
	js := fmt.Sprintf("$('#%s').val('%s');", target, value)
	err := websocket.Message.Send(ws, js)
	if err != nil {
		return err
	}
	return nil
}

//Get content of form element
func Get(ws *websocket.Conn, target string) (string, error) {
	ret := ""
	js := fmt.Sprintf("ws.send($('#%s').val());", target)
	err := websocket.Message.Send(ws, js)
	if err != nil {
		return "", err
	}

	err = websocket.Message.Receive(ws, &ret)
	if err != nil {
		return "", err
	}

	return ret, nil
}

//Wire bind an action to an event on target
func Wire(ws *websocket.Conn, target, event, act string) error {
	return errNotImplemented
}

//SendEvent send an event to server
func SendEvent(ws *websocket.Conn, event string) error {
	js := fmt.Sprintf("sendEvent('%s');", event)
	err := websocket.Message.Send(ws, js)
	if err != nil {
		return err
	}

	return nil
}

//Alert show alert message in browser
func Alert(ws *websocket.Conn, message string) error {
	js := fmt.Sprintf("alert('%s');", message)
	err := websocket.Message.Send(ws, js)
	if err != nil {
		return err
	}

	return nil
}

//BlockUI block page interaction
func BlockUI(ws *websocket.Conn) {
	Exec(ws, "$.blockUI();")
}

//UnblockUI block page interaction
func UnblockUI(ws *websocket.Conn) {
	Exec(ws, "$.unblockUI();")
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
)

func cmdCheckout() int {

	// git-lob checkout [options] [<pathspec>...]

	// no custom options
	optDryRun := GlobalOptions.DryRun

	// All extra arguments must be <pathspec>
	var pathspecs []string
	for _, arg := range GlobalOptions.Args[1:] {
		p := filepath.Clean(arg)
		pathspecs = append(pathspecs, p)
	}

	err := Checkout(pathspecs, optDryRun)

	if err != nil {
		fmt.Fprintf(os.Stderr, "git-lob: checkout error - %v", err.Error())
		return 7
	}

	return 0
}

// Populate local placeholders with real content, if available. Do entire working copy unless limited to pathspecs
func Checkout(pathspecs []string, dryRun bool) error {
	// We're going to scan for missing git-lob content not just by checking the working copy, but
	// getting the expected content from git first. This is in case the working copy has had files
	// deleted for example. We still check the content of the working copy if the file IS there
	// in order to not overwrite modified files.

	// firstly convert any pathspecs to the root of the repo, in case this is being executed in a sub-folder
	reporoot, _ := GetRepoRoot()
	if reporoot == "" {
		return errors.New("Not in git repository?")
	}
	curdir, err := os.Getwd()
	if err != nil {
		return err
	}
	var rootedpathspecs []string
	for _, p := range pathspecs {
		var abs string
		if filepath.IsAbs(p) {
			abs = p
		} else {
			abs = filepath.Join(curdir, p)
		}
		reltoroot, err := filepath.Rel(reporoot, abs)
		if err != nil {
			return errors.New(fmt.Sprintf("Unable to make %v relative to repo root %v", p, reporoot))
		}
		rootedpathspecs = append(rootedpathspecs, reltoroot)
	}

	// Get what git thinks we should have
	filelobs, err := GetGitAllFilesAndLOBsToCheckoutAtCommit("HEAD", rootedpathspecs, nil)
	if err != nil {
		return err
	}
	var filesOK int
	var filesNotOK int
	for _, filelob := range filelobs {
		// Check each file, and if it's missing or contains the placeholder text, replace it with content
		// Otherwise, assume it's been locally modified and leave it alone (user can override this with git reset/checkout if they want)
		absfile := filepath.Join(reporoot, filelob.Filename)
		stat, err := os.Stat(absfile)
		replaceContent := false
		if err == nil {
			// File existed, check content (smoke test on size)
			if stat.Size() == int64(SHALineLen) {
				// File existed and is right size for placeholder, so check contents
				placeholderContent := getLOBPlaceholderContent(filelob.SHA)
				filebytes, err := ioutil.ReadFile(absfile)
				if err == nil && string(filebytes) == placeholderContent {
					// File content is placeholder, so replace
					replaceContent = true
				}
			}
		} else {
			// File did not exist
			replaceContent = true
		}

		if replaceContent {
			if dryRun {
				if GlobalOptions.Verbose {
					fmt.Println("Checkout:", filelob.Filename)
				}

			} else {
				err = os.MkdirAll(filepath.Dir(absfile), 0755)
				if err != nil {
					// This is not fatal but log it
					LogErrorf("ERROR: can't create parent directory of %v: %v\n", absfile, err.Error())
					continue
				}
				f, err := os.OpenFile(absfile, os.O_CREATE|os.O_TRUNC, 0644)
				if err != nil {
					// This is not fatal but log it
					LogErrorf("ERROR: can't open %v for writing: %v", absfile, err.Error())
					continue
				}
				_, err = RetrieveLOB(filelob.SHA, f)
				f.Close()
				if err != nil {
					// This is not fatal but log it
					LogErrorf("ERROR: can't retrieve content for %v: %v", filelob.Filename, err.Error())
					continue
				}

			}
			filesOK++
		}

		if !GlobalOptions.Quiet {
			if dryRun {
				fmt.Println(filesOK, "files need updating")
				fmt.Println("Run this command again without --dry-run to update these files.")
			} else {
				fmt.Println(filesOK, "files were updated")
				if filesNotOK > 0 {
					fmt.Println("WARNING:", filesNotOK, "failed to be updated, check errors above")
				}
			}
		}
	}

	if filesNotOK > 0 {
		return errors.New(fmt.Sprintf("%d files failed", filesNotOK))
	}

	return nil

}

func cmdCheckoutHelp() {
	fmt.Println(`Usage: git-lob checkout [options] [<pathspec>...]

  Populate files in the working copy with binary content where they
  currently just have placeholder content, because the real content wasn't
  available.

  NOTE: You probably won't need to run this command yourself.

        Running 'git lob pull' will both fetch (download) AND checkout, so
        most of the time you should use 'git lob pull' instead.

        Also 'git checkout' will populate the binary content correctly if
        you have it locally so you don't have to run this command after
        switching branches, unless you need to download extra content, in
        which case 'git lob pull' is once again a better bet.

  Because git-lob stores binary content separately from your git repository,
  it's possible that when you perform a 'git checkout' or 'git clone', you did
  not have the binary content available locally to populate binary files in
  your working copy. In this situation, git-lob creates placeholders in the
  working copy, whose content looks something like this:

  git-lob: <sha>

  Where <sha> is the identifier of the content of the binary file. Once you
  have downloaded the content (e.g. via 'git lob fetch'), you can then use
  'git lob checkout' to fill in these blanks.

  Specify <pathspec> to limit the checking to particular files or directories.

  Options:
    --quiet, -q   Print less output
    --verbose, -v Print more output
    --dry-run     Don't actually change any files, just report

`)
}
<commit_msg>Correctly record files that failed to checkout<commit_after>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
)

func cmdCheckout() int {

	// git-lob checkout [options] [<pathspec>...]

	// no custom options
	optDryRun := GlobalOptions.DryRun

	// All extra arguments must be <pathspec>
	var pathspecs []string
	for _, arg := range GlobalOptions.Args[1:] {
		p := filepath.Clean(arg)
		pathspecs = append(pathspecs, p)
	}

	err := Checkout(pathspecs, optDryRun)

	if err != nil {
		fmt.Fprintf(os.Stderr, "git-lob: checkout error - %v", err.Error())
		return 7
	}

	return 0
}

// Populate local placeholders with real content, if available. Do entire working copy unless limited to pathspecs
func Checkout(pathspecs []string, dryRun bool) error {
	// We're going to scan for missing git-lob content not just by checking the working copy, but
	// getting the expected content from git first. This is in case the working copy has had files
	// deleted for example. We still check the content of the working copy if the file IS there
	// in order to not overwrite modified files.

	// firstly convert any pathspecs to the root of the repo, in case this is being executed in a sub-folder
	reporoot, _ := GetRepoRoot()
	if reporoot == "" {
		return errors.New("Not in git repository?")
	}
	curdir, err := os.Getwd()
	if err != nil {
		return err
	}
	var rootedpathspecs []string
	for _, p := range pathspecs {
		var abs string
		if filepath.IsAbs(p) {
			abs = p
		} else {
			abs = filepath.Join(curdir, p)
		}
		reltoroot, err := filepath.Rel(reporoot, abs)
		if err != nil {
			return errors.New(fmt.Sprintf("Unable to make %v relative to repo root %v", p, reporoot))
		}
		rootedpathspecs = append(rootedpathspecs, reltoroot)
	}

	// Get what git thinks we should have
	filelobs, err := GetGitAllFilesAndLOBsToCheckoutAtCommit("HEAD", rootedpathspecs, nil)
	if err != nil {
		return err
	}
	var filesOK int
	var filesNotOK int
	for _, filelob := range filelobs {
		// Check each file, and if it's missing or contains the placeholder text, replace it with content
		// Otherwise, assume it's been locally modified and leave it alone (user can override this with git reset/checkout if they want)
		absfile := filepath.Join(reporoot, filelob.Filename)
		stat, err := os.Stat(absfile)
		replaceContent := false
		if err == nil {
			// File existed, check content (smoke test on size)
			if stat.Size() == int64(SHALineLen) {
				// File existed and is right size for placeholder, so check contents
				placeholderContent := getLOBPlaceholderContent(filelob.SHA)
				filebytes, err := ioutil.ReadFile(absfile)
				if err == nil && string(filebytes) == placeholderContent {
					// File content is placeholder, so replace
					replaceContent = true
				}
			}
		} else {
			// File did not exist
			replaceContent = true
		}

		if replaceContent {
			if dryRun {
				if GlobalOptions.Verbose {
					fmt.Println("Checkout:", filelob.Filename)
				}

			} else {
				err = os.MkdirAll(filepath.Dir(absfile), 0755)
				if err != nil {
					// This is not fatal but log it
					LogErrorf("ERROR: can't create parent directory of %v: %v\n", absfile, err.Error())
					filesNotOK++
					continue
				}
				f, err := os.OpenFile(absfile, os.O_CREATE|os.O_TRUNC, 0644)
				if err != nil {
					// This is not fatal but log it
					LogErrorf("ERROR: can't open %v for writing: %v", absfile, err.Error())
					filesNotOK++
					continue
				}
				_, err = RetrieveLOB(filelob.SHA, f)
				f.Close()
				if err != nil {
					// This is not fatal but log it
					LogErrorf("ERROR: can't retrieve content for %v: %v", filelob.Filename, err.Error())
					filesNotOK++
					continue
				}

			}
			filesOK++
		}

		if !GlobalOptions.Quiet {
			if dryRun {
				fmt.Println(filesOK, "files need updating")
				fmt.Println("Run this command again without --dry-run to update these files.")
			} else {
				fmt.Println(filesOK, "files were updated")
				if filesNotOK > 0 {
					fmt.Println("WARNING:", filesNotOK, "failed to be updated, check errors above")
				}
			}
		}
	}

	if filesNotOK > 0 {
		return errors.New(fmt.Sprintf("%d files failed", filesNotOK))
	}

	return nil

}

func cmdCheckoutHelp() {
	fmt.Println(`Usage: git-lob checkout [options] [<pathspec>...]

  Populate files in the working copy with binary content where they
  currently just have placeholder content, because the real content wasn't
  available.

  NOTE: You probably won't need to run this command yourself.

        Running 'git lob pull' will both fetch (download) AND checkout, so
        most of the time you should use 'git lob pull' instead.

        Also 'git checkout' will populate the binary content correctly if
        you have it locally so you don't have to run this command after
        switching branches, unless you need to download extra content, in
        which case 'git lob pull' is once again a better bet.

  Because git-lob stores binary content separately from your git repository,
  it's possible that when you perform a 'git checkout' or 'git clone', you did
  not have the binary content available locally to populate binary files in
  your working copy. In this situation, git-lob creates placeholders in the
  working copy, whose content looks something like this:

  git-lob: <sha>

  Where <sha> is the identifier of the content of the binary file. Once you
  have downloaded the content (e.g. via 'git lob fetch'), you can then use
  'git lob checkout' to fill in these blanks.

  Specify <pathspec> to limit the checking to particular files or directories.

  Options:
    --quiet, -q   Print less output
    --verbose, -v Print more output
    --dry-run     Don't actually change any files, just report

`)
}
<|endoftext|>