<commit_before>package worker

import (
	"encoding/json"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/go-redis/redis"
	"github.com/pkg/errors"
	"github.com/stitchfix/flotilla-os/clients/metrics"
	"github.com/stitchfix/flotilla-os/config"
	"github.com/stitchfix/flotilla-os/execution/engine"
	flotillaLog "github.com/stitchfix/flotilla-os/log"
	"github.com/stitchfix/flotilla-os/queue"
	"github.com/stitchfix/flotilla-os/state"
	"gopkg.in/tomb.v2"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strings"
	"time"
)

type statusWorker struct {
	sm                       state.Manager
	ee                       engine.Engine
	conf                     config.Config
	log                      flotillaLog.Logger
	pollInterval             time.Duration
	t                        tomb.Tomb
	engine                   *string
	redisClient              *redis.Client
	workerId                 string
	exceptionExtractorClient *http.Client
	exceptionExtractorUrl    string
	emrEngine                engine.Engine
}

func (sw *statusWorker) Initialize(conf config.Config, sm state.Manager, eksEngine engine.Engine, emrEngine engine.Engine, log flotillaLog.Logger, pollInterval time.Duration, qm queue.Manager) error {
	sw.pollInterval = pollInterval
	sw.conf = conf
	sw.sm = sm
	sw.ee = eksEngine
	sw.log = log
	sw.workerId = fmt.Sprintf("workerid:%d", rand.Int())
	sw.engine = &state.EKSEngine
	sw.emrEngine = emrEngine
	if sw.conf.IsSet("eks.exception_extractor_url") {
		sw.exceptionExtractorClient = &http.Client{
			Timeout: time.Second * 5,
		}
		sw.exceptionExtractorUrl = sw.conf.GetString("eks.exception_extractor_url")
	}
	sw.setupRedisClient(conf)
	_ = sw.log.Log("message", "initialized a status worker")
	return nil
}

func (sw *statusWorker) setupRedisClient(conf config.Config) {
	if *sw.engine == state.EKSEngine {
		sw.redisClient = redis.NewClient(&redis.Options{Addr: conf.GetString("redis_address"), DB: conf.GetInt("redis_db")})
	}
}

func (sw *statusWorker) GetTomb() *tomb.Tomb {
	return &sw.t
}

//
// Run updates status of tasks
//
func (sw *statusWorker) Run() error {
	for {
		select {
		case <-sw.t.Dying():
			sw.log.Log("message", "A status worker was terminated")
			return nil
		default:
			if *sw.engine == state.EKSEngine {
				sw.runOnceEKS()
				sw.runOnceEMR()
				time.Sleep(sw.pollInterval)
			}
		}
	}
}

func (sw *statusWorker) runOnceEMR() {
	rl, err := sw.sm.ListRuns(1000, 0, "started_at", "asc", map[string][]string{
		"queued_at_since": {
			time.Now().AddDate(0, 0, -30).Format(time.RFC3339),
		},
		"task_type": {state.DefaultTaskType},
		"status":    {state.StatusNeedsRetry, state.StatusRunning, state.StatusQueued, state.StatusPending},
	}, nil, []string{state.EKSSparkEngine})

	if err != nil {
		_ = sw.log.Log("message", "unable to receive runs", "error", fmt.Sprintf("%+v", err))
		return
	}
	runs := rl.Runs
	sw.processEMRRuns(runs)
}

func (sw *statusWorker) processEMRRuns(runs []state.Run) {
	for _, run := range runs {
		if run.QueuedAt != nil && run.ActiveDeadlineSeconds != nil {
			runningDuration := time.Now().Sub(*run.QueuedAt)
			if int64(runningDuration.Seconds()) > *run.ActiveDeadlineSeconds {
				err := sw.emrEngine.Terminate(run)
				if err == nil {
					exitCode := int64(1)
					finishedAt := time.Now()
					_, err = sw.sm.UpdateRun(run.RunID, state.Run{
						Status:     state.StatusStopped,
						ExitReason: aws.String(fmt.Sprintf("JobRun exceeded specified timeout of %v seconds", *run.ActiveDeadlineSeconds)),
						ExitCode:   &exitCode,
						FinishedAt: &finishedAt,
					})
				}
			}
		}
	}
}

func (sw *statusWorker) runOnceEKS() {
	rl, err := sw.sm.ListRuns(1000, 0, "started_at", "asc", map[string][]string{
		"queued_at_since": {
			time.Now().AddDate(0, 0, -30).Format(time.RFC3339),
		},
		"task_type": {state.DefaultTaskType},
		"status":    {state.StatusNeedsRetry, state.StatusRunning, state.StatusQueued, state.StatusPending},
	}, nil, []string{state.EKSEngine})

	if err != nil {
		_ = sw.log.Log("message", "unable to receive runs", "error", fmt.Sprintf("%+v", err))
		return
	}
	runs := rl.Runs
	sw.processEKSRuns(runs)
}

func (sw *statusWorker) processEKSRuns(runs []state.Run) {
	var lockedRuns []state.Run
	for _, run := range runs {
		duration := time.Duration(45) * time.Second
		lock := sw.acquireLock(run, "status", duration)
		if lock {
			lockedRuns = append(lockedRuns, run)
		}
	}
	_ = metrics.Increment(metrics.StatusWorkerLockedRuns, []string{sw.workerId}, float64(len(lockedRuns)))
	for _, run := range lockedRuns {
		start := time.Now()
		go sw.processEKSRun(run)
		_ = metrics.Timing(metrics.StatusWorkerProcessEKSRun, time.Since(start), []string{sw.workerId}, 1)
	}
}
func (sw *statusWorker) acquireLock(run state.Run, purpose string, expiration time.Duration) bool {
	start := time.Now()
	key := fmt.Sprintf("%s-%s", run.RunID, purpose)
	ttl, err := sw.redisClient.TTL(key).Result()
	if err == nil && ttl.Nanoseconds() < 0 {
		_, err = sw.redisClient.Del(key).Result()
	}
	set, err := sw.redisClient.SetNX(key, sw.workerId, expiration).Result()
	if err != nil {
		_ = sw.log.Log("message", "unable to set lock", "error", fmt.Sprintf("%+v", err))
		return true
	}
	_ = metrics.Timing(metrics.StatusWorkerAcquireLock, time.Since(start), []string{sw.workerId}, 1)
	return set
}

func (sw *statusWorker) processEKSRun(run state.Run) {
	reloadRun, err := sw.sm.GetRun(run.RunID)
	if err == nil && reloadRun.Status == state.StatusStopped {
		// Run was updated by another worker process.
		return
	}
	start := time.Now()
	updatedRunWithMetrics, _ := sw.ee.FetchPodMetrics(run)
	_ = metrics.Timing(metrics.StatusWorkerFetchPodMetrics, time.Since(start), []string{sw.workerId}, 1)

	start = time.Now()
	updatedRun, err := sw.ee.FetchUpdateStatus(updatedRunWithMetrics)
	if err != nil {
		_ = sw.log.Log("message", "fetch update status", "run", run.RunID, "error", fmt.Sprintf("%+v", err))
	}
	_ = metrics.Timing(metrics.StatusWorkerFetchUpdateStatus, time.Since(start), []string{sw.workerId}, 1)

	if err == nil {
		subRuns, err := sw.sm.ListRuns(1000, 0, "status", "desc", nil, map[string]string{"PARENT_FLOTILLA_RUN_ID": run.RunID}, state.Engines)
		if err == nil && subRuns.Total > 0 {
			var spawnedRuns state.SpawnedRuns
			for _, subRun := range subRuns.Runs {
				spawnedRuns = append(spawnedRuns, state.SpawnedRun{RunID: subRun.RunID})
			}
			updatedRun.SpawnedRuns = &spawnedRuns
		}
	}
	if err != nil {
		message := fmt.Sprintf("%+v", err)
		minutesInQueue := time.Now().Sub(*run.QueuedAt).Minutes()
		if strings.Contains(message, "not found") && minutesInQueue > float64(30) {
			stoppedAt := time.Now()
			reason := "Job either timed out or not found on the EKS cluster."
			updatedRun.Status = state.StatusStopped
			updatedRun.FinishedAt = &stoppedAt
			updatedRun.ExitReason = &reason
			_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
		}

	} else {
		if run.Status != updatedRun.Status && (updatedRun.PodName == run.PodName) {
			sw.logStatusUpdate(updatedRun)
			if updatedRun.ExitCode != nil {
				go sw.cleanupRun(run.RunID)
			}
			_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
			if err != nil {
				_ = sw.log.Log("message", "unable to save eks runs", "error", fmt.Sprintf("%+v", err))
			}

			if updatedRun.Status == state.StatusStopped {
				//TODO - move to a separate worker.
				//_ = sw.eksEngine.Terminate(run)
			}
		} else {
			if updatedRun.MaxMemoryUsed != run.MaxMemoryUsed ||
				updatedRun.MaxCpuUsed != run.MaxCpuUsed ||
				updatedRun.Cpu != run.Cpu ||
				updatedRun.PodName != run.PodName ||
				updatedRun.Memory != run.Memory ||
				updatedRun.PodEvents != run.PodEvents ||
				updatedRun.SpawnedRuns != run.SpawnedRuns {
				_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
			}
		}
	}
}

func (sw *statusWorker) cleanupRun(runID string) {
	//Logs maybe delayed before being persisted to S3.
	time.Sleep(120 * time.Second)
	run, err := sw.sm.GetRun(runID)
	if err == nil {
		//Delete run from Kubernetes
		_ = sw.ee.Terminate(run)
	}
}

func (sw *statusWorker) extractExceptions(runID string) {
	//Logs maybe delayed before being persisted to S3.
	time.Sleep(60 * time.Second)
	run, err := sw.sm.GetRun(runID)
	if err == nil {
		jobUrl := fmt.Sprintf("%s/extract/%s", sw.exceptionExtractorUrl, run.RunID)
		res, err := sw.exceptionExtractorClient.Get(jobUrl)
		if err == nil && res != nil && res.Body != nil {
			body, err := ioutil.ReadAll(res.Body)
			if body != nil {
				defer res.Body.Close()
				runExceptions := state.RunExceptions{}
				err = json.Unmarshal(body, &runExceptions)
				if err == nil {
					run.RunExceptions = &runExceptions
				}
			}
			_, _ = sw.sm.UpdateRun(run.RunID, run)
		}
	}
}

func (sw *statusWorker) processEKSRunMetrics(run state.Run) {
	updatedRun, err := sw.ee.FetchPodMetrics(run)
	if err == nil {
		if updatedRun.MaxMemoryUsed != run.MaxMemoryUsed ||
			updatedRun.MaxCpuUsed != run.MaxCpuUsed {
			_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
		}
	}
}

func (sw *statusWorker) logStatusUpdate(update state.Run) {
	var err error
	var startedAt, finishedAt time.Time
	var env state.EnvList
	var command string

	if update.StartedAt != nil {
		startedAt = *update.StartedAt
	}

	if update.FinishedAt != nil {
		finishedAt = *update.FinishedAt
	}

	if update.Env != nil {
		env = *update.Env
	}

	if update.Command != nil {
		command = *update.Command
	}

	if update.ExitCode != nil {
		err = sw.log.Event("eventClassName", "FlotillaTaskStatus",
			"run_id", update.RunID,
			"definition_id", update.DefinitionID,
			"alias", update.Alias,
			"image", update.Image,
			"cluster_name", update.ClusterName,
			"command", command,
			"exit_code", *update.ExitCode,
			"status", update.Status,
			"started_at", startedAt,
			"finished_at", finishedAt,
			"instance_id", update.InstanceID,
			"instance_dns_name", update.InstanceDNSName,
			"group_name", update.GroupName,
			"user", update.User,
			"task_type", update.TaskType,
			"env", env,
			"executable_id", update.ExecutableID,
			"executable_type", update.ExecutableType)
	} else {
		err = sw.log.Event("eventClassName", "FlotillaTaskStatus",
			"run_id", update.RunID,
			"definition_id", update.DefinitionID,
			"alias", update.Alias,
			"image", update.Image,
			"cluster_name", update.ClusterName,
			"command", command,
			"status", update.Status,
			"started_at", startedAt,
			"finished_at", finishedAt,
			"instance_id", update.InstanceID,
			"instance_dns_name", update.InstanceDNSName,
			"group_name", update.GroupName,
			"user", update.User,
			"task_type", update.TaskType,
			"env", env,
			"executable_id", update.ExecutableID,
			"executable_type", update.ExecutableType)
	}

	if err != nil {
		sw.log.Log("message", "Failed to emit status event", "run_id", update.RunID, "error", err.Error())
	}
}

func (sw *statusWorker) findRun(taskArn string) (state.Run, error) {
	var engines []string
	if sw.engine != nil {
		engines = []string{*sw.engine}
	} else {
		engines = nil
	}

	runs, err := sw.sm.ListRuns(1, 0, "started_at", "asc", map[string][]string{
		"task_arn": {taskArn},
	}, nil, engines)
	if err != nil {
		return state.Run{}, errors.Wrapf(err, "problem finding run by task arn [%s]", taskArn)
	}
	if runs.Total > 0 && len(runs.Runs) > 0 {
		return runs.Runs[0], nil
	}
	return state.Run{}, errors.Errorf("no run found for [%s]", taskArn)
}
<commit_msg>use StartedAt for timing out EMR/EKS jobs<commit_after>package worker

import (
	"encoding/json"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/go-redis/redis"
	"github.com/pkg/errors"
	"github.com/stitchfix/flotilla-os/clients/metrics"
	"github.com/stitchfix/flotilla-os/config"
	"github.com/stitchfix/flotilla-os/execution/engine"
	flotillaLog "github.com/stitchfix/flotilla-os/log"
	"github.com/stitchfix/flotilla-os/queue"
	"github.com/stitchfix/flotilla-os/state"
	"gopkg.in/tomb.v2"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strings"
	"time"
)

type statusWorker struct {
	sm                       state.Manager
	ee                       engine.Engine
	conf                     config.Config
	log                      flotillaLog.Logger
	pollInterval             time.Duration
	t                        tomb.Tomb
	engine                   *string
	redisClient              *redis.Client
	workerId                 string
	exceptionExtractorClient *http.Client
	exceptionExtractorUrl    string
	emrEngine                engine.Engine
}

func (sw *statusWorker) Initialize(conf config.Config, sm state.Manager, eksEngine engine.Engine, emrEngine engine.Engine, log flotillaLog.Logger, pollInterval time.Duration, qm queue.Manager) error {
	sw.pollInterval = pollInterval
	sw.conf = conf
	sw.sm = sm
	sw.ee = eksEngine
	sw.log = log
	sw.workerId = fmt.Sprintf("workerid:%d", rand.Int())
	sw.engine = &state.EKSEngine
	sw.emrEngine = emrEngine
	if sw.conf.IsSet("eks.exception_extractor_url") {
		sw.exceptionExtractorClient = &http.Client{
			Timeout: time.Second * 5,
		}
		sw.exceptionExtractorUrl = sw.conf.GetString("eks.exception_extractor_url")
	}
	sw.setupRedisClient(conf)
	_ = sw.log.Log("message", "initialized a status worker")
	return nil
}

func (sw *statusWorker) setupRedisClient(conf config.Config) {
	if *sw.engine == state.EKSEngine {
		sw.redisClient = redis.NewClient(&redis.Options{Addr: conf.GetString("redis_address"), DB: conf.GetInt("redis_db")})
	}
}

func (sw *statusWorker) GetTomb() *tomb.Tomb {
	return &sw.t
}

//
// Run updates status of tasks
//
func (sw *statusWorker) Run() error {
	for {
		select {
		case <-sw.t.Dying():
			sw.log.Log("message", "A status worker was terminated")
			return nil
		default:
			if *sw.engine == state.EKSEngine {
				sw.runOnceEKS()
				sw.runOnceEMR()
				time.Sleep(sw.pollInterval)
			}
		}
	}
}

func (sw *statusWorker) runOnceEMR() {
	rl, err := sw.sm.ListRuns(1000, 0, "started_at", "asc", map[string][]string{
		"queued_at_since": {
			time.Now().AddDate(0, 0, -30).Format(time.RFC3339),
		},
		"task_type": {state.DefaultTaskType},
		"status":    {state.StatusNeedsRetry, state.StatusRunning, state.StatusQueued, state.StatusPending},
	}, nil, []string{state.EKSSparkEngine})

	if err != nil {
		_ = sw.log.Log("message", "unable to receive runs", "error", fmt.Sprintf("%+v", err))
		return
	}
	runs := rl.Runs
	sw.processEMRRuns(runs)
}

func (sw *statusWorker) processEMRRuns(runs []state.Run) {
	for _, run := range runs {
		if run.QueuedAt != nil && run.ActiveDeadlineSeconds != nil {
			runningDuration := time.Now().Sub(*run.StartedAt)
			if int64(runningDuration.Seconds()) > *run.ActiveDeadlineSeconds {
				err := sw.emrEngine.Terminate(run)
				if err == nil {
					exitCode := int64(1)
					finishedAt := time.Now()
					_, err = sw.sm.UpdateRun(run.RunID, state.Run{
						Status:     state.StatusStopped,
						ExitReason: aws.String(fmt.Sprintf("JobRun exceeded specified timeout of %v seconds", *run.ActiveDeadlineSeconds)),
						ExitCode:   &exitCode,
						FinishedAt: &finishedAt,
					})
				}
			}
		}
	}
}

func (sw *statusWorker) runOnceEKS() {
	rl, err := sw.sm.ListRuns(1000, 0, "started_at", "asc", map[string][]string{
		"queued_at_since": {
			time.Now().AddDate(0, 0, -30).Format(time.RFC3339),
		},
		"task_type": {state.DefaultTaskType},
		"status":    {state.StatusNeedsRetry, state.StatusRunning, state.StatusQueued, state.StatusPending},
	}, nil, []string{state.EKSEngine})

	if err != nil {
		_ = sw.log.Log("message", "unable to receive runs", "error", fmt.Sprintf("%+v", err))
		return
	}
	runs := rl.Runs
	sw.processEKSRuns(runs)
}

func (sw *statusWorker) processEKSRuns(runs []state.Run) {
	var lockedRuns []state.Run
	for _, run := range runs {
		duration := time.Duration(45) * time.Second
		lock := sw.acquireLock(run, "status", duration)
		if lock {
			lockedRuns = append(lockedRuns, run)
		}
	}
	_ = metrics.Increment(metrics.StatusWorkerLockedRuns, []string{sw.workerId}, float64(len(lockedRuns)))
	for _, run := range lockedRuns {
		start := time.Now()
		go sw.processEKSRun(run)
		_ = metrics.Timing(metrics.StatusWorkerProcessEKSRun, time.Since(start), []string{sw.workerId}, 1)
	}
}
func (sw *statusWorker) acquireLock(run state.Run, purpose string, expiration time.Duration) bool {
	start := time.Now()
	key := fmt.Sprintf("%s-%s", run.RunID, purpose)
	ttl, err := sw.redisClient.TTL(key).Result()
	if err == nil && ttl.Nanoseconds() < 0 {
		_, err = sw.redisClient.Del(key).Result()
	}
	set, err := sw.redisClient.SetNX(key, sw.workerId, expiration).Result()
	if err != nil {
		_ = sw.log.Log("message", "unable to set lock", "error", fmt.Sprintf("%+v", err))
		return true
	}
	_ = metrics.Timing(metrics.StatusWorkerAcquireLock, time.Since(start), []string{sw.workerId}, 1)
	return set
}

func (sw *statusWorker) processEKSRun(run state.Run) {
	reloadRun, err := sw.sm.GetRun(run.RunID)
	if err == nil && reloadRun.Status == state.StatusStopped {
		// Run was updated by another worker process.
		return
	}
	start := time.Now()
	updatedRunWithMetrics, _ := sw.ee.FetchPodMetrics(run)
	_ = metrics.Timing(metrics.StatusWorkerFetchPodMetrics, time.Since(start), []string{sw.workerId}, 1)

	start = time.Now()
	updatedRun, err := sw.ee.FetchUpdateStatus(updatedRunWithMetrics)
	if err != nil {
		_ = sw.log.Log("message", "fetch update status", "run", run.RunID, "error", fmt.Sprintf("%+v", err))
	}
	_ = metrics.Timing(metrics.StatusWorkerFetchUpdateStatus, time.Since(start), []string{sw.workerId}, 1)

	if err == nil {
		subRuns, err := sw.sm.ListRuns(1000, 0, "status", "desc", nil, map[string]string{"PARENT_FLOTILLA_RUN_ID": run.RunID}, state.Engines)
		if err == nil && subRuns.Total > 0 {
			var spawnedRuns state.SpawnedRuns
			for _, subRun := range subRuns.Runs {
				spawnedRuns = append(spawnedRuns, state.SpawnedRun{RunID: subRun.RunID})
			}
			updatedRun.SpawnedRuns = &spawnedRuns
		}
	}
	if err != nil {
		message := fmt.Sprintf("%+v", err)
		minutesInQueue := time.Now().Sub(*run.QueuedAt).Minutes()
		if strings.Contains(message, "not found") && minutesInQueue > float64(30) {
			stoppedAt := time.Now()
			reason := "Job either timed out or not found on the EKS cluster."
			updatedRun.Status = state.StatusStopped
			updatedRun.FinishedAt = &stoppedAt
			updatedRun.ExitReason = &reason
			_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
		}

	} else {
		if run.Status != updatedRun.Status && (updatedRun.PodName == run.PodName) {
			sw.logStatusUpdate(updatedRun)
			if updatedRun.ExitCode != nil {
				go sw.cleanupRun(run.RunID)
			}
			_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
			if err != nil {
				_ = sw.log.Log("message", "unable to save eks runs", "error", fmt.Sprintf("%+v", err))
			}

			if updatedRun.Status == state.StatusStopped {
				//TODO - move to a separate worker.
				//_ = sw.eksEngine.Terminate(run)
			}
		} else {
			if updatedRun.MaxMemoryUsed != run.MaxMemoryUsed ||
				updatedRun.MaxCpuUsed != run.MaxCpuUsed ||
				updatedRun.Cpu != run.Cpu ||
				updatedRun.PodName != run.PodName ||
				updatedRun.Memory != run.Memory ||
				updatedRun.PodEvents != run.PodEvents ||
				updatedRun.SpawnedRuns != run.SpawnedRuns {
				_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
			}
		}
	}
}

func (sw *statusWorker) cleanupRun(runID string) {
	//Logs maybe delayed before being persisted to S3.
	time.Sleep(120 * time.Second)
	run, err := sw.sm.GetRun(runID)
	if err == nil {
		//Delete run from Kubernetes
		_ = sw.ee.Terminate(run)
	}
}

func (sw *statusWorker) extractExceptions(runID string) {
	//Logs maybe delayed before being persisted to S3.
	time.Sleep(60 * time.Second)
	run, err := sw.sm.GetRun(runID)
	if err == nil {
		jobUrl := fmt.Sprintf("%s/extract/%s", sw.exceptionExtractorUrl, run.RunID)
		res, err := sw.exceptionExtractorClient.Get(jobUrl)
		if err == nil && res != nil && res.Body != nil {
			body, err := ioutil.ReadAll(res.Body)
			if body != nil {
				defer res.Body.Close()
				runExceptions := state.RunExceptions{}
				err = json.Unmarshal(body, &runExceptions)
				if err == nil {
					run.RunExceptions = &runExceptions
				}
			}
			_, _ = sw.sm.UpdateRun(run.RunID, run)
		}
	}
}

func (sw *statusWorker) processEKSRunMetrics(run state.Run) {
	updatedRun, err := sw.ee.FetchPodMetrics(run)
	if err == nil {
		if updatedRun.MaxMemoryUsed != run.MaxMemoryUsed ||
			updatedRun.MaxCpuUsed != run.MaxCpuUsed {
			_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
		}
	}
}

func (sw *statusWorker) logStatusUpdate(update state.Run) {
	var err error
	var startedAt, finishedAt time.Time
	var env state.EnvList
	var command string

	if update.StartedAt != nil {
		startedAt = *update.StartedAt
	}

	if update.FinishedAt != nil {
		finishedAt = *update.FinishedAt
	}

	if update.Env != nil {
		env = *update.Env
	}

	if update.Command != nil {
		command = *update.Command
	}

	if update.ExitCode != nil {
		err = sw.log.Event("eventClassName", "FlotillaTaskStatus",
			"run_id", update.RunID,
			"definition_id", update.DefinitionID,
			"alias", update.Alias,
			"image", update.Image,
			"cluster_name", update.ClusterName,
			"command", command,
			"exit_code", *update.ExitCode,
			"status", update.Status,
			"started_at", startedAt,
			"finished_at", finishedAt,
			"instance_id", update.InstanceID,
			"instance_dns_name", update.InstanceDNSName,
			"group_name", update.GroupName,
			"user", update.User,
			"task_type", update.TaskType,
			"env", env,
			"executable_id", update.ExecutableID,
			"executable_type", update.ExecutableType)
	} else {
		err = sw.log.Event("eventClassName", "FlotillaTaskStatus",
			"run_id", update.RunID,
			"definition_id", update.DefinitionID,
			"alias", update.Alias,
			"image", update.Image,
			"cluster_name", update.ClusterName,
			"command", command,
			"status", update.Status,
			"started_at", startedAt,
			"finished_at", finishedAt,
			"instance_id", update.InstanceID,
			"instance_dns_name", update.InstanceDNSName,
			"group_name", update.GroupName,
			"user", update.User,
			"task_type", update.TaskType,
			"env", env,
			"executable_id", update.ExecutableID,
			"executable_type", update.ExecutableType)
	}

	if err != nil {
		sw.log.Log("message", "Failed to emit status event", "run_id", update.RunID, "error", err.Error())
	}
}

func (sw *statusWorker) findRun(taskArn string) (state.Run, error) {
	var engines []string
	if sw.engine != nil {
		engines = []string{*sw.engine}
	} else {
		engines = nil
	}

	runs, err := sw.sm.ListRuns(1, 0, "started_at", "asc", map[string][]string{
		"task_arn": {taskArn},
	}, nil, engines)
	if err != nil {
		return state.Run{}, errors.Wrapf(err, "problem finding run by task arn [%s]", taskArn)
	}
	if runs.Total > 0 && len(runs.Runs) > 0 {
		return runs.Runs[0], nil
	}
	return state.Run{}, errors.Errorf("no run found for [%s]", taskArn)
}
<|endoftext|>


<commit_before>package helpers

import (
	"html/template"
	"io"
)

const (
	HeaderName = "header"
	FooterName = "footer"
)

// The Page type (for injecting data)
type Page struct {
	Name string
	Data interface{}
}

// Loading a template safely (just by name)
func LoadTemplate(name string) (*template.Template, error) {
	temp, err := template.ParseFiles("templates/" + name + ".html")

	return temp, err
}

// Loading a template unsafely (just by name)
func LoadTemplateUnsafe(name string) *template.Template {
	temp, err := LoadTemplate(name)

	if err != nil {
		panic(err)
	}

	return temp
}

// Loading the header and footer templates
func loadHeader() *template.Template { return LoadTemplateUnsafe(HeaderName) }
func loadFooter() *template.Template { return LoadTemplateUnsafe(FooterName) }

// Sending a file with the header and
// footer templates included
func SendPage(w io.Writer, name string, data interface{}) error {
	page, err := LoadTemplate(name)

	if err != nil {
		return err
	}

	header := loadHeader()
	footer := loadFooter()

	v := Page{Name: name,
		Data: data}

	header.Execute(w, v)
	page.Execute(w, v)
	footer.Execute(w, v)

	return nil
}
<commit_msg>Added functions to safely and unsafely render a page into a string.<commit_after>package helpers

import (
	"html/template"
	"io"
	"log"
)

const (
	HeaderName = "header"
	FooterName = "footer"
)

// The Page type (for injecting data)
type Page struct {
	Name string
	Data interface{}
}

// Loading a template safely (just by name)
func LoadTemplate(name string) (*template.Template, error) {
	temp, err := template.ParseFiles("templates/" + name + ".html")

	return temp, err
}

// Loading a template unsafely (just by name)
func LoadTemplateUnsafe(name string) *template.Template {
	temp, err := LoadTemplate(name)

	if err != nil {
		log.Fatal(err)
	}

	return temp
}

// Loading the header and footer templates
func loadHeader() *template.Template { return LoadTemplateUnsafe(HeaderName) }
func loadFooter() *template.Template { return LoadTemplateUnsafe(FooterName) }

// Sending a file with the header and
// footer templates included
func SendPage(w io.Writer, name string, data interface{}) error {
	page, err := LoadTemplate(name)

	if err != nil {
		return err
	}

	header := loadHeader()
	footer := loadFooter()

	v := Page{Name: name,
		Data: data}

	header.Execute(w, v)
	page.Execute(w, v)
	footer.Execute(w, v)

	return nil
}

// Loading a template to a string
func RenderPage(name string, data interface{}) (string, error) {
	sw := NewStringWriter()

	err := SendPage(sw, name, data)

	if err != nil {
		return "", err
	}

	return sw.String(), nil
}

// Unsafely rendering the page
func RenderPageUnsafe(name string, data interface{}) string {
	str, err := RenderPage(name, data)
	if err != nil {
		log.Fatal(err)
	}
	return str
}
<|endoftext|>


<commit_before>package executor

import (
	"bytes"
	"reflect"
	"testing"
)

func TestE_RunCommands(t *testing.T) {
	var tests = []struct {
		commands     []string
		passCommands []string
		failCommands []string
		showOutput   bool
		args         []string

		expectedResult *Result
		expectedOutput string
	}{
		// Test showOutput variations
		{ // Hide output when disabled
			[]string{"echo foo"},
			[]string{},
			[]string{},
			false,
			[]string{},
			&Result{true, true, true},
			"",
		},
		{ // Show output when enabled
			[]string{"echo foo"},
			[]string{},
			[]string{},
			true,
			[]string{},
			&Result{true, true, true},
			"foo\n",
		},
		{ // Multiple commands
			[]string{"echo foo", "echo bar"},
			[]string{},
			[]string{},
			true,
			[]string{},
			&Result{true, true, true},
			"foo\nbar\n",
		},
		{ // Show error when output disabled
			[]string{"echo foo 1>&2"},
			[]string{},
			[]string{},
			false,
			[]string{},
			&Result{true, true, true},
			"foo\n",
		},
		{ // Show error when output enabled
			[]string{"echo foo 1>&2"},
			[]string{},
			[]string{},
			true,
			[]string{},
			&Result{true, true, true},
			"foo\n",
		},
		{ // Show output and error when output enabled
			[]string{"echo foo; echo bar 1>&2"},
			[]string{},
			[]string{},
			true,
			[]string{},
			&Result{true, true, true},
			"foo\nbar\n",
		},
		{ // Output comes before error
			[]string{"echo foo; echo bar 1>&2; echo baz"},
			[]string{},
			[]string{},
			true,
			[]string{},
			&Result{true, true, true},
			"foo\nbaz\nbar\n",
		},

		// Test passing
		{ // Output comes before error
			[]string{"exit 1"},
			[]string{},
			[]string{},
			false,
			[]string{},
			&Result{false, true, true},
			"",
		},
		{ // Show output as error on failures
			[]string{"echo foo; exit 1"},
			[]string{},
			[]string{},
			false,
			[]string{},
			&Result{false, true, true},
			"foo\n",
		},
		{ // Show error as error on failures
			[]string{"echo foo; exit 1"},
			[]string{},
			[]string{},
			false,
			[]string{},
			&Result{false, true, true},
			"foo\n",
		},
		{ // Show output and error as error on failures
			[]string{"echo foo; echo bar 1>&2; exit 1"},
			[]string{},
			[]string{},
			false,
			[]string{},
			&Result{false, true, true},
			"foo\nbar\n",
		},

		// Passing/failing commands
		{ // Run pass commands on success
			[]string{"echo foo; exit 0"},
			[]string{"echo yes"},
			[]string{"echo no"},
			true,
			[]string{},
			&Result{true, true, true},
			"foo\nyes\n",
		},
		{ // Run fail commands on failure
			[]string{"echo foo; exit 1"},
			[]string{"echo yes"},
			[]string{"echo no"},
			true,
			[]string{},
			&Result{false, true, true},
			"foo\nno\n",
		},
		{ // Multiple pass commands
			[]string{"echo foo; exit 0"},
			[]string{"echo yes", "echo again"},
			[]string{"echo no"},
			true,
			[]string{},
			&Result{true, true, true},
			"foo\nyes\nagain\n",
		},
		{ // Multiple fail commands
			[]string{"echo foo; exit 1"},
			[]string{"echo yes"},
			[]string{"echo no", "echo again"},
			true,
			[]string{},
			&Result{false, true, true},
			"foo\nno\nagain\n",
		},
		{ // Pass command failure doesn't influence overall result
			[]string{"echo foo; exit 0"},
			[]string{"echo yes; exit 1"},
			[]string{"echo no"},
			true,
			[]string{},
			&Result{true, true, true},
			"foo\nyes\n",
		},
		{ // Fail command failure doesn't.. uh.. magically make the overall result success?
			[]string{"echo foo; exit 1"},
			[]string{"echo yes"},
			[]string{"echo no; exit 1"},
			true,
			[]string{},
			&Result{false, true, true},
			"foo\nno\n",
		},
		{ // Pass command doesn't output when output disabled
			[]string{"echo foo; exit 0"},
			[]string{"echo yes"},
			[]string{"echo no"},
			false,
			[]string{},
			&Result{true, true, true},
			"",
		},
		{ // Pass command errors output when output disabled
			[]string{"echo foo; exit 0"},
			[]string{"echo yes 1>&2"},
			[]string{"echo no"},
			false,
			[]string{},
			&Result{true, true, true},
			"yes\n",
		},
		{ // Fail command doesn't output when output disabled
			[]string{"echo foo; exit 1"},
			[]string{"echo yes"},
			[]string{"echo no"},
			false,
			[]string{},
			&Result{false, true, true},
			"foo\n",
		},
		{ // Fail command shows error when output disabled
			[]string{"echo foo; exit 1"},
			[]string{"echo yes"},
			[]string{"echo no 1>&2"},
			false,
			[]string{},
			&Result{false, true, true},
			"foo\nno\n",
		},
	}

	for i, test := range tests {
		var outBuf bytes.Buffer

		e := New(test.commands,
			test.passCommands,
			test.failCommands,
			"",
			"",
			test.showOutput)
		e.out = &outBuf
		e.err = &outBuf

		r := e.RunCommands("", test.args)
		outStr := outBuf.String()

		if !reflect.DeepEqual(r, test.expectedResult) {
			t.Errorf("%d. unexpected result:\nexpected=%#v,\nactual=%#v\n", i, test.expectedResult, r)
		}
		if outStr != test.expectedOutput {
			t.Errorf("%d. unexpected output/error:\nexpected=%#v,\nactual=%#v\n", i, test.expectedOutput, outStr)
		}
	}
}
<commit_msg>Fix tests<commit_after>package executor

import (
	"bytes"
	"testing"
	"time"
)

func TestE_RunCommands(t *testing.T) {
	var tests = []struct {
		commands     []string
		passCommands []string
		failCommands []string
		showOutput   bool
		args         []string

		expectedResult *Result
		expectedOutput string
	}{
		// Test showOutput variations
		{ // Hide output when disabled
			[]string{"echo foo"},
			[]string{},
			[]string{},
			false,
			[]string{},
			&Result{true, true, true, 0, time.Time{}},
			"",
		},
		{ // Show output when enabled
			[]string{"echo foo"},
			[]string{},
			[]string{},
			true,
			[]string{},
			&Result{true, true, true, 0, time.Time{}},
			"foo\n",
		},
		{ // Multiple commands
			[]string{"echo foo", "echo bar"},
			[]string{},
			[]string{},
			true,
			[]string{},
			&Result{true, true, true, 0, time.Time{}},
			"foo\nbar\n",
		},
		{ // Show error when output disabled
			[]string{"echo foo 1>&2"},
			[]string{},
			[]string{},
			false,
			[]string{},
			&Result{true, true, true, 0, time.Time{}},
			"foo\n",
		},
		{ // Show error when output enabled
			[]string{"echo foo 1>&2"},
			[]string{},
			[]string{},
			true,
			[]string{},
			&Result{true, true, true, 0, time.Time{}},
			"foo\n",
		},
		{ // Show output and error when output enabled
			[]string{"echo foo; echo bar 1>&2"},
			[]string{},
			[]string{},
			true,
			[]string{},
			&Result{true, true, true, 0, time.Time{}},
			"foo\nbar\n",
		},
		{ // Output comes before error
			[]string{"echo foo; echo bar 1>&2; echo baz"},
			[]string{},
			[]string{},
			true,
			[]string{},
			&Result{true, true, true, 0, time.Time{}},
			"foo\nbaz\nbar\n",
		},

		// Test passing
		{ // Output comes before error
			[]string{"exit 1"},
			[]string{},
			[]string{},
			false,
			[]string{},
			&Result{false, true, true, 0, time.Time{}},
			"",
		},
		{ // Show output as error on failures
			[]string{"echo foo; exit 1"},
			[]string{},
			[]string{},
			false,
			[]string{},
			&Result{false, true, true, 0, time.Time{}},
			"foo\n",
		},
		{ // Show error as error on failures
			[]string{"echo foo; exit 1"},
			[]string{},
			[]string{},
			false,
			[]string{},
			&Result{false, true, true, 0, time.Time{}},
			"foo\n",
		},
		{ // Show output and error as error on failures
			[]string{"echo foo; echo bar 1>&2; exit 1"},
			[]string{},
			[]string{},
			false,
			[]string{},
			&Result{false, true, true, 0, time.Time{}},
			"foo\nbar\n",
		},

		// Passing/failing commands
		{ // Run pass commands on success
			[]string{"echo foo; exit 0"},
			[]string{"echo yes"},
			[]string{"echo no"},
			true,
			[]string{},
			&Result{true, true, true, 0, time.Time{}},
			"foo\nyes\n",
		},
		{ // Run fail commands on failure
			[]string{"echo foo; exit 1"},
			[]string{"echo yes"},
			[]string{"echo no"},
			true,
			[]string{},
			&Result{false, true, true, 0, time.Time{}},
			"foo\nno\n",
		},
		{ // Multiple pass commands
			[]string{"echo foo; exit 0"},
			[]string{"echo yes", "echo again"},
			[]string{"echo no"},
			true,
			[]string{},
			&Result{true, true, true, 0, time.Time{}},
			"foo\nyes\nagain\n",
		},
		{ // Multiple fail commands
			[]string{"echo foo; exit 1"},
			[]string{"echo yes"},
			[]string{"echo no", "echo again"},
			true,
			[]string{},
			&Result{false, true, true, 0, time.Time{}},
			"foo\nno\nagain\n",
		},
		{ // Pass command failure doesn't influence overall result
			[]string{"echo foo; exit 0"},
			[]string{"echo yes; exit 1"},
			[]string{"echo no"},
			true,
			[]string{},
			&Result{true, true, true, 0, time.Time{}},
			"foo\nyes\n",
		},
		{ // Fail command failure doesn't.. uh.. magically make the overall result success?
			[]string{"echo foo; exit 1"},
			[]string{"echo yes"},
			[]string{"echo no; exit 1"},
			true,
			[]string{},
			&Result{false, true, true, 0, time.Time{}},
			"foo\nno\n",
		},
		{ // Pass command doesn't output when output disabled
			[]string{"echo foo; exit 0"},
			[]string{"echo yes"},
			[]string{"echo no"},
			false,
			[]string{},
			&Result{true, true, true, 0, time.Time{}},
			"",
		},
		{ // Pass command errors output when output disabled
			[]string{"echo foo; exit 0"},
			[]string{"echo yes 1>&2"},
			[]string{"echo no"},
			false,
			[]string{},
			&Result{true, true, true, 0, time.Time{}},
			"yes\n",
		},
		{ // Fail command doesn't output when output disabled
			[]string{"echo foo; exit 1"},
			[]string{"echo yes"},
			[]string{"echo no"},
			false,
			[]string{},
			&Result{false, true, true, 0, time.Time{}},
			"foo\n",
		},
		{ // Fail command shows error when output disabled
			[]string{"echo foo; exit 1"},
			[]string{"echo yes"},
			[]string{"echo no 1>&2"},
			false,
			[]string{},
			&Result{false, true, true, 0, time.Time{}},
			"foo\nno\n",
		},
	}

	for i, test := range tests {
		var outBuf bytes.Buffer

		e := New(test.commands,
			test.passCommands,
			test.failCommands,
			"",
			"",
			test.showOutput)
		e.out = &outBuf
		e.err = &outBuf

		r := e.RunCommands("", test.args)
		outStr := outBuf.String()

		if !resultsEqual(r, test.expectedResult) {
			t.Errorf("%d. unexpected result:\nexpected=%#v,\nactual=%#v\n", i, test.expectedResult, r)
		}
		if outStr != test.expectedOutput {
			t.Errorf("%d. unexpected output/error:\nexpected=%#v,\nactual=%#v\n", i, test.expectedOutput, outStr)
		}
	}
}

func resultsEqual(actual *Result, expected *Result) bool {
	if actual.First != expected.First {
		return false
	}
	if actual.Passing != expected.Passing {
		return false
	}
	if actual.WasPassing != expected.WasPassing {
		return false
	}
	// Ignore times and durations
	return true
}<|endoftext|>


<commit_before>package searcher

import (
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/etsy/hound/config"
	"github.com/etsy/hound/index"
	"github.com/etsy/hound/vcs"
)

type Searcher struct {
	idx  *index.Index
	lck  sync.RWMutex
	Repo *config.Repo

	// The channel is used to request updates from the API and
	// to signal that it is ok for searchers to begin polling.
	// It has a buffer size of 1 to allow at most one pending
	// update at a time.
	updateCh chan time.Time
}

type limiter chan bool

/**
 * Holds a set of IndexRefs that were found in the dbpath at startup,
 * these indexes can be 'claimed' and re-used by newly created searchers.
 */
type foundRefs struct {
	refs    []*index.IndexRef
	claimed map[*index.IndexRef]bool
}

func makeLimiter(n int) limiter {
	return limiter(make(chan bool, n))
}

func (l limiter) Acquire() {
	l <- true
}

func (l limiter) Release() {
	<-l
}

/**
 * Find an Index ref for the repo url and rev, returns nil if no such
 * ref exists.
 */
func (r *foundRefs) find(url, rev string) *index.IndexRef {
	for _, ref := range r.refs {
		if ref.Url == url && ref.Rev == rev {
			return ref
		}
	}
	return nil
}

/**
 * Claim a ref for reuse. This ensures they ref will not be garbage
 * collected at the end of startup.
 */
func (r *foundRefs) claim(ref *index.IndexRef) {
	r.claimed[ref] = true
}

/**
 * Delete the directorires associated with all IndexRefs that were
 * found in the dbpath but were not claimed during startup.
 */
func (r *foundRefs) removeUnclaimed() error {
	for _, ref := range r.refs {
		if r.claimed[ref] {
			continue
		}

		if err := ref.Remove(); err != nil {
			return err
		}
	}
	return nil
}

// Perform atomic swap of index in the searcher so that the new
// index is made "live".
func (s *Searcher) swapIndexes(idx *index.Index) error {
	s.lck.Lock()
	defer s.lck.Unlock()

	oldIdx := s.idx
	s.idx = idx

	return oldIdx.Destroy()
}

// Perform a basic search on the current index using the supplied pattern
// and the options.
//
// TODO(knorton): pat should really just be a part of SearchOptions
func (s *Searcher) Search(pat string, opt *index.SearchOptions) (*index.SearchResponse, error) {
	s.lck.RLock()
	defer s.lck.RUnlock()
	return s.idx.Search(pat, opt)
}

// Get the excluded files as a JSON string. This is only used for returning
// the data directly to clients (thus JSON).
func (s *Searcher) GetExcludedFiles() string {
	path := filepath.Join(s.idx.GetDir(), "excluded_files.json")
	dat, err := ioutil.ReadFile(path)
	if err != nil {
		log.Printf("Couldn't read excluded_files.json %v\n", err)
	}
	return string(dat)
}

// Triggers an immediate poll of the repository.
func (s *Searcher) Update() bool {
	if !s.Repo.PushUpdatesEnabled() {
		return false
	}

	// schedule an update if one is not already scheduled
	select {
	case s.updateCh <- time.Now():
	default:
		// don't wait to enqueue another update
	}

	return true
}

// Wait for either the delay period to expire or an update request to
// arrive. Note that an empty delay will result in an infinite timeout.
func (s *Searcher) waitForUpdate(delay time.Duration) {
	var tch <-chan time.Time
	if delay.Nanoseconds() > 0 {
		tch = time.After(delay)
	}

	// wait for either the timeout or the update channel signal
	select {
	case <-s.updateCh:
	case <-tch:
	}
}

// Signal the searcher that it is ok to begin polling the repository.
func (s *Searcher) begin() {
	s.updateCh <- time.Now()
}

// Generate a new index directory in the dbpath. The names are based
// on pseudo-randomness with a time-based seed.
func nextIndexDir(dbpath string) string {
	r := uint64(rand.Uint32())<<32 | uint64(rand.Uint32())
	return filepath.Join(dbpath, fmt.Sprintf("idx-%08x", r))
}

// Read the refs associated with each of the index dirs
// in the given dbpath.
func findExistingRefs(dbpath string) (*foundRefs, error) {
	dirs, err := filepath.Glob(filepath.Join(dbpath, "idx-*"))
	if err != nil {
		return nil, err
	}

	var refs []*index.IndexRef
	for _, dir := range dirs {
		r, _ := index.Read(dir)
		refs = append(refs, r)
	}

	return &foundRefs{
		refs:    refs,
		claimed: map[*index.IndexRef]bool{},
	}, nil
}

// Open an index at the given path. If the idxDir is already present, it will
// simply open and use that index. If, however, the idxDir does not exist a new
// one will be built.
func buildAndOpenIndex(
	opt *index.IndexOptions,
	dbpath,
	vcsDir,
	idxDir,
	url,
	rev string) (*index.Index, error) {
	if _, err := os.Stat(idxDir); err != nil {
		r, err := index.Build(opt, idxDir, vcsDir, url, rev)
		if err != nil {
			return nil, err
		}

		return r.Open()
	}

	return index.Open(idxDir)
}

// Simply prints out statistics about the heap. When hound rebuilds a new
// index it will expand the heap with a decent amount of garbage. This is
// helpful to ensure the heap growth looks sane.
func reportOnMemory() {
	var ms runtime.MemStats

	// Print out interesting heap info.
	runtime.ReadMemStats(&ms)
	fmt.Printf("HeapInUse = %0.2f\n", float64(ms.HeapInuse)/1e6)
	fmt.Printf("HeapIdle  = %0.2f\n", float64(ms.HeapIdle)/1e6)
}

// Utility function for producing a hex encoded sha1 hash for a string.
func hashFor(name string) string {
	h := sha1.New()
	h.Write([]byte(name))
	return hex.EncodeToString(h.Sum(nil))
}

// Create a normalized name for the vcs directory of this repo.
func vcsDirFor(repo *config.Repo) string {
	return fmt.Sprintf("vcs-%s", hashFor(repo.Url))
}

func init() {
	rand.Seed(time.Now().UnixNano())
}

// Make a searcher for each repo in the Config. This function kind of has a notion
// of partial errors. First, if the error returned is non-nil then a fatal error has
// occurred and no other return values are valid. If an error occurs that is specific
// to a particular searcher, that searcher will not be present in the searcher map and
// will have an error entry in the error map.
//
// NOTE: The keys in the searcher map will be normalized to lower case, but not such
// transformation will be done on the error map to make it easier to match those errors
// back to the original repo name.
func MakeAll(cfg *config.Config) (map[string]*Searcher, map[string]error, error) {
	errs := map[string]error{}
	searchers := map[string]*Searcher{}

	refs, err := findExistingRefs(cfg.DbPath)
	if err != nil {
		return nil, nil, err
	}

	lim := makeLimiter(cfg.MaxConcurrentIndexers)

	for name, repo := range cfg.Repos {
		s, err := newSearcher(cfg.DbPath, name, repo, refs, lim)
		if err != nil {
			log.Print(err)
			errs[name] = err
			continue
		}

		searchers[strings.ToLower(name)] = s
	}

	if err := refs.removeUnclaimed(); err != nil {
		return nil, nil, err
	}

	// after all the repos are in good shape, we start their polling
	for _, s := range searchers {
		s.begin()
	}

	return searchers, errs, nil
}

// Creates a new Searcher that is available for searches as soon as this returns.
// This will pull or clone the target repo and start watching the repo for changes.
func New(dbpath, name string, repo *config.Repo) (*Searcher, error) {
	s, err := newSearcher(dbpath, name, repo, &foundRefs{}, makeLimiter(1))
	if err != nil {
		return nil, err
	}

	s.begin()

	return s, nil
}

// Update the vcs and reindex the given repo.
func updateAndReindex(
	s *Searcher,
	dbpath,
	vcsDir,
	name,
	rev string,
	wd *vcs.WorkDir,
	opt *index.IndexOptions,
	lim limiter) (string, bool) {

	// acquire a token from the rate limiter
	lim.Acquire()
	defer lim.Release()

	repo := s.Repo
	newRev, err := wd.PullOrClone(vcsDir, repo.Url)

	if err != nil {
		log.Printf("vcs pull error (%s - %s): %s", name, repo.Url, err)
		return rev, false
	}

	if newRev == rev {
		return rev, false
	}

	log.Printf("Rebuilding %s for %s", name, newRev)
	idx, err := buildAndOpenIndex(
		opt,
		dbpath,
		vcsDir,
		nextIndexDir(dbpath),
		repo.Url,
		newRev)
	if err != nil {
		log.Printf("failed index build (%s): %s", name, err)
		return rev, false
	}

	if err := s.swapIndexes(idx); err != nil {
		log.Printf("failed index swap (%s): %s", name, err)
		if err := idx.Destroy(); err != nil {
			log.Printf("failed to destroy index (%s): %s\n", name, err)
		}
		return rev, false
	}

	return newRev, true
}

// Creates a new Searcher that is capable of re-claiming an existing index directory
// from a set of existing manifests.
func newSearcher(
	dbpath, name string,
	repo *config.Repo,
	refs *foundRefs,
	lim limiter) (*Searcher, error) {

	vcsDir := filepath.Join(dbpath, vcsDirFor(repo))

	log.Printf("Searcher started for %s", name)

	wd, err := vcs.New(repo.Vcs, repo.VcsConfig())
	if err != nil {
		return nil, err
	}

	opt := &index.IndexOptions{
		ExcludeDotFiles: repo.ExcludeDotFiles,
		SpecialFiles:    wd.SpecialFiles(),
	}

	rev, err := wd.PullOrClone(vcsDir, repo.Url)
	if err != nil {
		return nil, err
	}

	var idxDir string
	ref := refs.find(repo.Url, rev)
	if ref == nil {
		idxDir = nextIndexDir(dbpath)
	} else {
		idxDir = ref.Dir()
		refs.claim(ref)
	}

	idx, err := buildAndOpenIndex(
		opt,
		dbpath,
		vcsDir,
		idxDir,
		repo.Url,
		rev)
	if err != nil {
		return nil, err
	}

	s := &Searcher{
		idx:      idx,
		updateCh: make(chan time.Time, 1),
		Repo:     repo,
	}

	go func() {

		// each searcher's poller is held until begin is called.
		<-s.updateCh

		// if all forms of updating are turned off, we're done here.
		if !repo.PollUpdatesEnabled() && !repo.PushUpdatesEnabled() {
			return
		}

		var delay time.Duration
		if repo.PollUpdatesEnabled() {
			delay = time.Duration(repo.MsBetweenPolls) * time.Millisecond
		}

		for {

			// Wait for a signal to proceed
			s.waitForUpdate(delay)

			// attempt to update and reindex this searcher
			newRev, ok := updateAndReindex(s, dbpath, vcsDir, name, rev, wd, opt, lim)
			if !ok {
				continue
			}

			rev = newRev

			// This is just a good time to GC since we know there will be a
			// whole set of dead posting lists on the heap. Ensuring these
			// go away quickly helps to prevent the heap from expanding
			// uncessarily.
			runtime.GC()

			reportOnMemory()
		}
	}()

	return s, nil
}
<commit_msg>Log the vcs dir when rebuilding.  (fingerscrossed)<commit_after>package searcher

import (
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/etsy/hound/config"
	"github.com/etsy/hound/index"
	"github.com/etsy/hound/vcs"
)

type Searcher struct {
	idx  *index.Index
	lck  sync.RWMutex
	Repo *config.Repo

	// The channel is used to request updates from the API and
	// to signal that it is ok for searchers to begin polling.
	// It has a buffer size of 1 to allow at most one pending
	// update at a time.
	updateCh chan time.Time
}

type limiter chan bool

/**
 * Holds a set of IndexRefs that were found in the dbpath at startup,
 * these indexes can be 'claimed' and re-used by newly created searchers.
 */
type foundRefs struct {
	refs    []*index.IndexRef
	claimed map[*index.IndexRef]bool
}

func makeLimiter(n int) limiter {
	return limiter(make(chan bool, n))
}

func (l limiter) Acquire() {
	l <- true
}

func (l limiter) Release() {
	<-l
}

/**
 * Find an Index ref for the repo url and rev, returns nil if no such
 * ref exists.
 */
func (r *foundRefs) find(url, rev string) *index.IndexRef {
	for _, ref := range r.refs {
		if ref.Url == url && ref.Rev == rev {
			return ref
		}
	}
	return nil
}

/**
 * Claim a ref for reuse. This ensures they ref will not be garbage
 * collected at the end of startup.
 */
func (r *foundRefs) claim(ref *index.IndexRef) {
	r.claimed[ref] = true
}

/**
 * Delete the directorires associated with all IndexRefs that were
 * found in the dbpath but were not claimed during startup.
 */
func (r *foundRefs) removeUnclaimed() error {
	for _, ref := range r.refs {
		if r.claimed[ref] {
			continue
		}

		if err := ref.Remove(); err != nil {
			return err
		}
	}
	return nil
}

// Perform atomic swap of index in the searcher so that the new
// index is made "live".
func (s *Searcher) swapIndexes(idx *index.Index) error {
	s.lck.Lock()
	defer s.lck.Unlock()

	oldIdx := s.idx
	s.idx = idx

	return oldIdx.Destroy()
}

// Perform a basic search on the current index using the supplied pattern
// and the options.
//
// TODO(knorton): pat should really just be a part of SearchOptions
func (s *Searcher) Search(pat string, opt *index.SearchOptions) (*index.SearchResponse, error) {
	s.lck.RLock()
	defer s.lck.RUnlock()
	return s.idx.Search(pat, opt)
}

// Get the excluded files as a JSON string. This is only used for returning
// the data directly to clients (thus JSON).
func (s *Searcher) GetExcludedFiles() string {
	path := filepath.Join(s.idx.GetDir(), "excluded_files.json")
	dat, err := ioutil.ReadFile(path)
	if err != nil {
		log.Printf("Couldn't read excluded_files.json %v\n", err)
	}
	return string(dat)
}

// Triggers an immediate poll of the repository.
func (s *Searcher) Update() bool {
	if !s.Repo.PushUpdatesEnabled() {
		return false
	}

	// schedule an update if one is not already scheduled
	select {
	case s.updateCh <- time.Now():
	default:
		// don't wait to enqueue another update
	}

	return true
}

// Wait for either the delay period to expire or an update request to
// arrive. Note that an empty delay will result in an infinite timeout.
func (s *Searcher) waitForUpdate(delay time.Duration) {
	var tch <-chan time.Time
	if delay.Nanoseconds() > 0 {
		tch = time.After(delay)
	}

	// wait for either the timeout or the update channel signal
	select {
	case <-s.updateCh:
	case <-tch:
	}
}

// Signal the searcher that it is ok to begin polling the repository.
func (s *Searcher) begin() {
	s.updateCh <- time.Now()
}

// Generate a new index directory in the dbpath. The names are based
// on pseudo-randomness with a time-based seed.
func nextIndexDir(dbpath string) string {
	r := uint64(rand.Uint32())<<32 | uint64(rand.Uint32())
	return filepath.Join(dbpath, fmt.Sprintf("idx-%08x", r))
}

// Read the refs associated with each of the index dirs
// in the given dbpath.
func findExistingRefs(dbpath string) (*foundRefs, error) {
	dirs, err := filepath.Glob(filepath.Join(dbpath, "idx-*"))
	if err != nil {
		return nil, err
	}

	var refs []*index.IndexRef
	for _, dir := range dirs {
		r, _ := index.Read(dir)
		refs = append(refs, r)
	}

	return &foundRefs{
		refs:    refs,
		claimed: map[*index.IndexRef]bool{},
	}, nil
}

// Open an index at the given path. If the idxDir is already present, it will
// simply open and use that index. If, however, the idxDir does not exist a new
// one will be built.
func buildAndOpenIndex(
	opt *index.IndexOptions,
	dbpath,
	vcsDir,
	idxDir,
	url,
	rev string) (*index.Index, error) {
	if _, err := os.Stat(idxDir); err != nil {
		r, err := index.Build(opt, idxDir, vcsDir, url, rev)
		if err != nil {
			return nil, err
		}

		return r.Open()
	}

	return index.Open(idxDir)
}

// Simply prints out statistics about the heap. When hound rebuilds a new
// index it will expand the heap with a decent amount of garbage. This is
// helpful to ensure the heap growth looks sane.
func reportOnMemory() {
	var ms runtime.MemStats

	// Print out interesting heap info.
	runtime.ReadMemStats(&ms)
	fmt.Printf("HeapInUse = %0.2f\n", float64(ms.HeapInuse)/1e6)
	fmt.Printf("HeapIdle  = %0.2f\n", float64(ms.HeapIdle)/1e6)
}

// Utility function for producing a hex encoded sha1 hash for a string.
func hashFor(name string) string {
	h := sha1.New()
	h.Write([]byte(name))
	return hex.EncodeToString(h.Sum(nil))
}

// Create a normalized name for the vcs directory of this repo.
func vcsDirFor(repo *config.Repo) string {
	return fmt.Sprintf("vcs-%s", hashFor(repo.Url))
}

func init() {
	rand.Seed(time.Now().UnixNano())
}

// Make a searcher for each repo in the Config. This function kind of has a notion
// of partial errors. First, if the error returned is non-nil then a fatal error has
// occurred and no other return values are valid. If an error occurs that is specific
// to a particular searcher, that searcher will not be present in the searcher map and
// will have an error entry in the error map.
//
// NOTE: The keys in the searcher map will be normalized to lower case, but not such
// transformation will be done on the error map to make it easier to match those errors
// back to the original repo name.
func MakeAll(cfg *config.Config) (map[string]*Searcher, map[string]error, error) {
	errs := map[string]error{}
	searchers := map[string]*Searcher{}

	refs, err := findExistingRefs(cfg.DbPath)
	if err != nil {
		return nil, nil, err
	}

	lim := makeLimiter(cfg.MaxConcurrentIndexers)

	for name, repo := range cfg.Repos {
		s, err := newSearcher(cfg.DbPath, name, repo, refs, lim)
		if err != nil {
			log.Print(err)
			errs[name] = err
			continue
		}

		searchers[strings.ToLower(name)] = s
	}

	if err := refs.removeUnclaimed(); err != nil {
		return nil, nil, err
	}

	// after all the repos are in good shape, we start their polling
	for _, s := range searchers {
		s.begin()
	}

	return searchers, errs, nil
}

// Creates a new Searcher that is available for searches as soon as this returns.
// This will pull or clone the target repo and start watching the repo for changes.
func New(dbpath, name string, repo *config.Repo) (*Searcher, error) {
	s, err := newSearcher(dbpath, name, repo, &foundRefs{}, makeLimiter(1))
	if err != nil {
		return nil, err
	}

	s.begin()

	return s, nil
}

// Update the vcs and reindex the given repo.
func updateAndReindex(
	s *Searcher,
	dbpath,
	vcsDir,
	name,
	rev string,
	wd *vcs.WorkDir,
	opt *index.IndexOptions,
	lim limiter) (string, bool) {

	// acquire a token from the rate limiter
	lim.Acquire()
	defer lim.Release()

	repo := s.Repo
	newRev, err := wd.PullOrClone(vcsDir, repo.Url)

	if err != nil {
		log.Printf("vcs pull error (%s - %s): %s", name, repo.Url, err)
		return rev, false
	}

	if newRev == rev {
		return rev, false
	}

	log.Printf("Rebuilding %s for %s [%s]", name, newRev, vcsDir)
	idx, err := buildAndOpenIndex(
		opt,
		dbpath,
		vcsDir,
		nextIndexDir(dbpath),
		repo.Url,
		newRev)
	if err != nil {
		log.Printf("failed index build (%s): %s", name, err)
		return rev, false
	}

	if err := s.swapIndexes(idx); err != nil {
		log.Printf("failed index swap (%s): %s", name, err)
		if err := idx.Destroy(); err != nil {
			log.Printf("failed to destroy index (%s): %s\n", name, err)
		}
		return rev, false
	}

	return newRev, true
}

// Creates a new Searcher that is capable of re-claiming an existing index directory
// from a set of existing manifests.
func newSearcher(
	dbpath, name string,
	repo *config.Repo,
	refs *foundRefs,
	lim limiter) (*Searcher, error) {

	vcsDir := filepath.Join(dbpath, vcsDirFor(repo))

	log.Printf("Searcher started for %s", name)

	wd, err := vcs.New(repo.Vcs, repo.VcsConfig())
	if err != nil {
		return nil, err
	}

	opt := &index.IndexOptions{
		ExcludeDotFiles: repo.ExcludeDotFiles,
		SpecialFiles:    wd.SpecialFiles(),
	}

	rev, err := wd.PullOrClone(vcsDir, repo.Url)
	if err != nil {
		return nil, err
	}

	var idxDir string
	ref := refs.find(repo.Url, rev)
	if ref == nil {
		idxDir = nextIndexDir(dbpath)
	} else {
		idxDir = ref.Dir()
		refs.claim(ref)
	}

	idx, err := buildAndOpenIndex(
		opt,
		dbpath,
		vcsDir,
		idxDir,
		repo.Url,
		rev)
	if err != nil {
		return nil, err
	}

	s := &Searcher{
		idx:      idx,
		updateCh: make(chan time.Time, 1),
		Repo:     repo,
	}

	go func() {

		// each searcher's poller is held until begin is called.
		<-s.updateCh

		// if all forms of updating are turned off, we're done here.
		if !repo.PollUpdatesEnabled() && !repo.PushUpdatesEnabled() {
			return
		}

		var delay time.Duration
		if repo.PollUpdatesEnabled() {
			delay = time.Duration(repo.MsBetweenPolls) * time.Millisecond
		}

		for {

			// Wait for a signal to proceed
			s.waitForUpdate(delay)

			// attempt to update and reindex this searcher
			newRev, ok := updateAndReindex(s, dbpath, vcsDir, name, rev, wd, opt, lim)
			if !ok {
				continue
			}

			rev = newRev

			// This is just a good time to GC since we know there will be a
			// whole set of dead posting lists on the heap. Ensuring these
			// go away quickly helps to prevent the heap from expanding
			// uncessarily.
			runtime.GC()

			reportOnMemory()
		}
	}()

	return s, nil
}
<|endoftext|>


<commit_before>// +build linux

package modules

import (
	"archive/tar"
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"strconv"
)

func init() {
	Modules["geekbench"] = &Geekbench{}
}

type Geekbench struct {
	SingleCore int
	MultiCore  int
	URL        string
	ID         int
}

func (stat *Geekbench) Run() error {
	res, err := http.Get("http://cdn.primatelabs.com/Geekbench-4.1.0-Linux.tar.gz")
	if err != nil {
		return err
	}

	if err := stat.extract(res.Body); err != nil {
		return err
	}

	gb := exec.Command("build.pulse/dist/Geekbench-4.1.0-Linux/geekbench4")

	out, err := gb.Output()
	if err != nil {
		fmt.Print(err.(*exec.ExitError).Stderr)
		return err
	}

	if err := os.RemoveAll("build.pulse"); err != nil {
		return err
	}

	r, err := regexp.Compile("https://browser.geekbench.com/v4/cpu/([0-9]*)")
	if err != nil {
		return err
	}

	match := r.FindStringSubmatch(string(out))
	if len(match) < 2 {
		return errors.New("geekbench did not return result url")
	}

	stat.URL = match[0]
	id, _ := strconv.Atoi(match[1])
	stat.ID = id

	stat.SingleCore, stat.MultiCore, err = stat.scrape(stat.URL)
	if err != nil {
		return err
	}

	return nil
}

func (stat *Geekbench) Print() {
	fmt.Printf("Single-Core Score  : %d\n", stat.SingleCore)
	fmt.Printf("Multi-Core Score   : %d\n", stat.MultiCore)
	fmt.Printf("Result URL         : %s\n", stat.URL)
}

func (stat *Geekbench) scrape(url string) (int, int, error) {
	res, err := http.Get(url)
	if err != nil {
		return 0, 0, err
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return 0, 0, err
	}

	singleReg, err := regexp.Compile("Single-Core Score	\n([0-9]*)	")
	if err != nil {
		return 0, 0, err
	}

	single := singleReg.FindStringSubmatch(string(body))
	if len(single) < 2 {
		return 0, 0, errors.New("failed to scrape score")
	}

	multiReg, err := regexp.Compile("Multi-Core Score	\n([0-9]*)	")
	if err != nil {
		return 0, 0, err
	}

	multi := multiReg.FindStringSubmatch(string(body))
	if len(multi) < 2 {
		return 0, 0, errors.New("failed to scrape score")
	}

	singleScore, _ := strconv.Atoi(single[1])
	multiScore, _ := strconv.Atoi(multi[1])

	return singleScore, multiScore, nil
}

func (stat *Geekbench) extract(r io.Reader) error {

	gzf, err := gzip.NewReader(r)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	tarReader := tar.NewReader(gzf)

	for {
		header, err := tarReader.Next()
		if err == io.EOF {
			return nil
		}
		if err != nil {
			return err
		}

		switch header.Typeflag {
		case tar.TypeDir:
			if err := os.MkdirAll(header.Name, os.ModePerm); err != nil {
				return err
			}
		case tar.TypeReg:
			f, err := os.Create(header.Name)
			if err != nil {
				return err
			}
			defer f.Close()

			if _, err := io.Copy(f, tarReader); err != nil {
				return err
			}

			os.Chmod(header.Name, os.ModePerm)
		default:
			return fmt.Errorf("%s %c %s %s", "unexpected file type", header.Typeflag, "in file", header.Name)
		}
	}

}
<commit_msg>geekbench: fix error log<commit_after>// +build linux

package modules

import (
	"archive/tar"
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"strconv"
)

func init() {
	Modules["geekbench"] = &Geekbench{}
}

type Geekbench struct {
	SingleCore int
	MultiCore  int
	URL        string
	ID         int
}

func (stat *Geekbench) Run() error {
	res, err := http.Get("http://cdn.primatelabs.com/Geekbench-4.1.0-Linux.tar.gz")
	if err != nil {
		return err
	}

	if err := stat.extract(res.Body); err != nil {
		return err
	}

	gb := exec.Command("build.pulse/dist/Geekbench-4.1.0-Linux/geekbench4")

	out, err := gb.Output()
	if err != nil {
		fmt.Printf("%s", err.(*exec.ExitError).Stderr)
		return err
	}

	if err := os.RemoveAll("build.pulse"); err != nil {
		return err
	}

	r, err := regexp.Compile("https://browser.geekbench.com/v4/cpu/([0-9]*)")
	if err != nil {
		return err
	}

	match := r.FindStringSubmatch(string(out))
	if len(match) < 2 {
		return errors.New("geekbench did not return result url")
	}

	stat.URL = match[0]
	id, _ := strconv.Atoi(match[1])
	stat.ID = id

	stat.SingleCore, stat.MultiCore, err = stat.scrape(stat.URL)
	if err != nil {
		return err
	}

	return nil
}

func (stat *Geekbench) Print() {
	fmt.Printf("Single-Core Score  : %d\n", stat.SingleCore)
	fmt.Printf("Multi-Core Score   : %d\n", stat.MultiCore)
	fmt.Printf("Result URL         : %s\n", stat.URL)
}

func (stat *Geekbench) scrape(url string) (int, int, error) {
	res, err := http.Get(url)
	if err != nil {
		return 0, 0, err
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return 0, 0, err
	}

	singleReg, err := regexp.Compile("Single-Core Score	\n([0-9]*)	")
	if err != nil {
		return 0, 0, err
	}

	single := singleReg.FindStringSubmatch(string(body))
	if len(single) < 2 {
		return 0, 0, errors.New("failed to scrape score")
	}

	multiReg, err := regexp.Compile("Multi-Core Score	\n([0-9]*)	")
	if err != nil {
		return 0, 0, err
	}

	multi := multiReg.FindStringSubmatch(string(body))
	if len(multi) < 2 {
		return 0, 0, errors.New("failed to scrape score")
	}

	singleScore, _ := strconv.Atoi(single[1])
	multiScore, _ := strconv.Atoi(multi[1])

	return singleScore, multiScore, nil
}

func (stat *Geekbench) extract(r io.Reader) error {

	gzf, err := gzip.NewReader(r)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	tarReader := tar.NewReader(gzf)

	for {
		header, err := tarReader.Next()
		if err == io.EOF {
			return nil
		}
		if err != nil {
			return err
		}

		switch header.Typeflag {
		case tar.TypeDir:
			if err := os.MkdirAll(header.Name, os.ModePerm); err != nil {
				return err
			}
		case tar.TypeReg:
			f, err := os.Create(header.Name)
			if err != nil {
				return err
			}
			defer f.Close()

			if _, err := io.Copy(f, tarReader); err != nil {
				return err
			}

			os.Chmod(header.Name, os.ModePerm)
		default:
			return fmt.Errorf("%s %c %s %s", "unexpected file type", header.Typeflag, "in file", header.Name)
		}
	}

}
<|endoftext|>


<commit_before>package secure

import (
	"testing"
	"time"
	"crypto/aes"
	"crypto/cipher"
)

var (
	original = []byte("Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird!")
)

func BenchmarkEncryptAesCbc(b *testing.B) {
	key := randomBytes(16)
	iv := make([]byte, 16)
	counter := 0
	aesCipher, err := aes.NewCipher(key)
	if err != nil {
		b.Fatal(err)
	}
	enc := cipher.NewCBCEncrypter(aesCipher, iv)
	decrypted := PKCS5Padding(original, aes.BlockSize)
	dataLen := len(decrypted)
	encrypted := make([]byte, len(decrypted))
	t := time.Now()
	b.SetBytes(int64(len(decrypted)))
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		enc.CryptBlocks(encrypted, decrypted)
		counter += dataLen
	}
	b.StopTimer()
	ts := time.Since(t)
	b.Logf("encryption speed: %.2f Mbit/s", float64(counter) * 8 / ts.Seconds() / 1024 / 1024)
}

func BenchmarkDescryptAesCbc(b *testing.B) {
	key := randomBytes(16)
	iv := randomBytes(16)
	encrypted, err := EncryptIV(original, key, iv)
	if err != nil {
		b.Fatal(err)
	}
	counter := 0
	dataLen := len(encrypted)

	aesCipher, err := aes.NewCipher(key)
	if err != nil {
		b.Fatal(err)
	}
	dec := cipher.NewCBCDecrypter(aesCipher, iv)
	t := time.Now()
	decrypted := make([]byte, len(encrypted))
	b.SetBytes(int64(len(encrypted)))
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		dec.CryptBlocks(decrypted, encrypted)
		counter += dataLen
	}
	b.StopTimer()
	ts := time.Since(t)
	b.Logf("decryption speed: %.2f Mbit/s", float64(counter) * 8 / ts.Seconds() / 1024 / 1024)
}

func BenchmarkEncryptAESGCM(b *testing.B) {
	key := make([]byte, 32)
	a, _ := aes.NewCipher(key)
	c, _ := cipher.NewGCM(a)
	benchmarkAEAD(b, c)
}

const benchSize = 1024 * 1024

func benchmarkAEAD(b *testing.B, c cipher.AEAD) {
	b.SetBytes(benchSize)
	input := make([]byte, benchSize)
	output := make([]byte, benchSize)
	nonce := make([]byte, c.NonceSize())
	counter := 0
	t := time.Now()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		c.Seal(output[:0], nonce, input, nil)
		counter += len(output)
	}
	ts := time.Since(t)
	b.Logf("aes-gcm speed: %.2f Mbit/s", float64(counter) * 8 / ts.Seconds() / 1024 / 1024)
}
<commit_msg>update aes-gcm benchmarks<commit_after>package secure

import (
	"testing"
	"time"
	"crypto/aes"
	"crypto/cipher"
)

var (
	original = []byte("Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird! Hello, World from MeshBird!")
)

func BenchmarkEncryptAesCbc(b *testing.B) {
	key := randomBytes(16)
	iv := make([]byte, 16)
	counter := 0
	aesCipher, err := aes.NewCipher(key)
	if err != nil {
		b.Fatal(err)
	}
	enc := cipher.NewCBCEncrypter(aesCipher, iv)
	decrypted := PKCS5Padding(original, aes.BlockSize)
	dataLen := len(decrypted)
	encrypted := make([]byte, len(decrypted))
	t := time.Now()
	b.SetBytes(int64(len(decrypted)))
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		enc.CryptBlocks(encrypted, decrypted)
		counter += dataLen
	}
	b.StopTimer()
	ts := time.Since(t)
	b.Logf("encryption speed: %.2f Mbit/s", float64(counter) * 8 / ts.Seconds() / 1024 / 1024)
}

func BenchmarkDescryptAesCbc(b *testing.B) {
	key := randomBytes(16)
	iv := randomBytes(16)
	encrypted, err := EncryptIV(original, key, iv)
	if err != nil {
		b.Fatal(err)
	}
	counter := 0
	dataLen := len(encrypted)

	aesCipher, err := aes.NewCipher(key)
	if err != nil {
		b.Fatal(err)
	}
	dec := cipher.NewCBCDecrypter(aesCipher, iv)
	t := time.Now()
	decrypted := make([]byte, len(encrypted))
	b.SetBytes(int64(len(encrypted)))
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		dec.CryptBlocks(decrypted, encrypted)
		counter += dataLen
	}
	b.StopTimer()
	ts := time.Since(t)
	b.Logf("decryption speed: %.2f Mbit/s", float64(counter) * 8 / ts.Seconds() / 1024 / 1024)
}

func BenchmarkEncryptAESGCM(b *testing.B) {
	key := make([]byte, 32)
	a, _ := aes.NewCipher(key)
	c, _ := cipher.NewGCM(a)
	benchmarkAEAD(b, c)
}

const benchSize = 1500

func benchmarkAEAD(b *testing.B, c cipher.AEAD) {
	b.SetBytes(benchSize)
	input := make([]byte, benchSize)
	output := make([]byte, benchSize)
	nonce := make([]byte, c.NonceSize())
	counter := 0
	t := time.Now()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		c.Seal(output[:0], nonce, input, nil)
		counter += len(output)
	}
	ts := time.Since(t)
	b.Logf("aes-gcm speed: %.2f Mbit/s", float64(counter) * 8 / ts.Seconds() / 1024 / 1024)
}
<|endoftext|>


<commit_before>package referenceblas

import (
	"github.com/gonum/blas"
)

var badOrder string = "referenceblas: illegal order"
var mLT0 string = "referenceblas: m < 0"
var nLT0 string = "referenceblas: m < 0"

func verifyLevel2inputs(o blas.Order, tA blas.Transpose, m, n, incX, incY, lda int) {
	if o != blas.RowMajor && o != blas.ColMajor {
		panic("blas: illegal order")
	}
	if tA != blas.NoTrans && tA != blas.Trans && tA != blas.ConjTrans {
		panic("blas: illegal transpose")
	}
	if m < 0 {
		panic(mLT0)
	}
	if n < 0 {
		panic(nLT0)
	}
	if o == blas.ColMajor {
		if lda > m && lda > 1 {
			panic("blas: lda must be less than max(1,m) for row major")
		}
	} else {
		if lda > n && lda > 0 {
			panic("blas: lda must be less than max(1,n) for col major")
		}
	}
	if incX == 0 {
		panic(zeroInc)
	}
	if incY == 0 {
		panic(zeroInc)
	}
}

func getLevel2Indexes(o blas.Order, tA blas.Transpose, m, n, incX, incY int) (lenx, leny, kx, ky int) {
	// Set up the lengths of the vectors and start up points
	// TODO: Figure out how this works with order
	lenx = m
	leny = n
	if tA == blas.NoTrans {
		lenx = n
		leny = m
	}
	if incX > 0 {
		kx = 0
	} else {
		kx = -(lenx - 1) * incX
	}
	if incY > 0 {
		ky = 0
	} else {
		ky = -(leny - 1) * incY
	}
	return lenx, leny, kx, ky
}

// Dgemv computes y = alpha*a*x + beta*y if tA = blas.NoTrans
// or alpha*A^T*x + beta*y if tA = blas.Trans or blas.ConjTrans
func (b Blas) Dgemv(o blas.Order, tA blas.Transpose, m, n int, alpha float64, a []float64, lda int, x []float64, incX int, beta float64, y []float64, incY int) {
	verifyLevel2inputs(o, tA, m, n, incX, incY, lda)

	// Quick return if possible
	if m == 0 || n == 0 || (alpha == 0 && beta == 1) {
		return
	}

	_, lenY, kx, ky := getLevel2Indexes(o, tA, m, n, incX, incY)

	// First form y := beta * y
	b.Dscal(lenY, beta, y, incY)

	if alpha == 0 {
		return
	}

	// Form y := alpha * A * x + y
	switch {
	case tA == blas.NoTrans && o == blas.RowMajor:
		jx := kx
		for j := 0; j < n; j++ {
			temp := alpha * x[jx]
			iy := ky
			for i := 0; i < m; i++ {
				y[iy] += temp * a[lda*i+j]
				iy += incY
			}
			jx += incX
		}
	case (tA == blas.Trans || tA == blas.ConjTrans) && o == blas.RowMajor:
		jy := ky
		for j := 0; j < n; j++ {
			var temp float64
			ix := kx
			for i := 0; i < m; i++ {
				temp += a[lda*i+j] * x[ix]
				ix += incX
			}
			y[jy] += alpha * temp
			jy += incY
		}
	default:
		// TODO: Add in other switch cases
		panic("Not yet implemented")
	}
}

/*
// Level 2 routines.
        Dgemv(o Order, tA Transpose, m, n int, alpha float64, a []float64, lda int, x []float64, incX int, beta float64, y []float64, incY int)
        Dgbmv(o Order, tA Transpose, m, n, kL, kU int, alpha float64, a []float64, lda int, x []float64, incX int, beta float64, y []float64, incY int)
        Dtrmv(o Order, ul Uplo, tA Transpose, d Diag, n int, a []float64, lda int, x []float64, incX int)
        Dtbmv(o Order, ul Uplo, tA Transpose, d Diag, n, k int, a []float64, lda int, x []float64, incX int)
        Dtpmv(o Order, ul Uplo, tA Transpose, d Diag, n int, ap []float64, x []float64, incX int)
        Dtrsv(o Order, ul Uplo, tA Transpose, d Diag, n int, a []float64, lda int, x []float64, incX int)
        Dtbsv(o Order, ul Uplo, tA Transpose, d Diag, n, k int, a []float64, lda int, x []float64, incX int)
        Dtpsv(o Order, ul Uplo, tA Transpose, d Diag, n int, ap []float64, x []float64, incX int)
        Dsymv(o Order, ul Uplo, n int, alpha float64, a []float64, lda int, x []float64, incX int, beta float64, y []float64, incY int)
        Dsbmv(o Order, ul Uplo, n, k int, alpha float64, a []float64, lda int, x []float64, incX int, beta float64, y []float64, incY int)
        Dspmv(o Order, ul Uplo, n int, alpha float64, ap []float64, x []float64, incX int, beta float64, y []float64, incY int)
        Dger(o Order, m, n int, alpha float64, x []float64, incX int, y []float64, incY int, a []float64, lda int)
        Dsyr(o Order, ul Uplo, n int, alpha float64, x []float64, incX int, a []float64, lda int)
        Dspr(o Order, ul Uplo, n int, alpha float64, x []float64, incX int, ap []float64)
        Dsyr2(o Order, ul Uplo, n int, alpha float64, x []float64, incX int, y []float64, incY int, a []float64, lda int)
        Dspr2(o Order, ul Uplo, n int, alpha float64, x []float64, incX int, y []float64, incY int, a []float64)
*/
<commit_msg>Added Dtrmv and Dtrsv<commit_after>package referenceblas

import (
	"github.com/gonum/blas"
)

const (
	badOrder     string = "referenceblas: illegal order"
	mLT0         string = "referenceblas: m < 0"
	nLT0         string = "referenceblas: m < 0"
	badUplo      string = "referenceblas: illegal triangularization"
	badTranspose string = "referenceblas: illegal transpose"
	badDiag      string = "referenceblas: illegal diag"
)

func getLevel2Indexes(o blas.Order, tA blas.Transpose, m, n, incX, incY int) (lenx, leny, kx, ky int) {
	// Set up the lengths of the vectors and start up points
	// TODO: Figure out how this works with order
	lenx = m
	leny = n
	if tA == blas.NoTrans {
		lenx = n
		leny = m
	}
	if incX > 0 {
		kx = 0
	} else {
		kx = -(lenx - 1) * incX
	}
	if incY > 0 {
		ky = 0
	} else {
		ky = -(leny - 1) * incY
	}
	return lenx, leny, kx, ky
}

// Dgemv computes y = alpha*a*x + beta*y if tA = blas.NoTrans
// or alpha*A^T*x + beta*y if tA = blas.Trans or blas.ConjTrans
func (b Blas) Dgemv(o blas.Order, tA blas.Transpose, m, n int, alpha float64, a []float64, lda int, x []float64, incX int, beta float64, y []float64, incY int) {
	if o != blas.RowMajor && o != blas.ColMajor {
		panic(badOrder)
	}
	if tA != blas.NoTrans && tA != blas.Trans && tA != blas.ConjTrans {
		panic(badTranspose)
	}
	if m < 0 {
		panic(mLT0)
	}
	if n < 0 {
		panic(nLT0)
	}
	if o == blas.ColMajor {
		if lda > m && lda > 1 {
			panic("blas: lda must be less than max(1,m) for row major")
		}
	} else {
		if lda > n && lda > 0 {
			panic("blas: lda must be less than max(1,n) for col major")
		}
	}
	if incX == 0 {
		panic(zeroInc)
	}
	if incY == 0 {
		panic(zeroInc)
	}

	// Quick return if possible
	if m == 0 || n == 0 || (alpha == 0 && beta == 1) {
		return
	}

	_, lenY, kx, ky := getLevel2Indexes(o, tA, m, n, incX, incY)

	// First form y := beta * y
	b.Dscal(lenY, beta, y, incY)

	if alpha == 0 {
		return
	}

	// Form y := alpha * A * x + y
	switch {
	case tA == blas.NoTrans && o == blas.RowMajor:
		jx := kx
		for j := 0; j < n; j++ {
			temp := alpha * x[jx]
			iy := ky
			for i := 0; i < m; i++ {
				y[iy] += temp * a[lda*i+j]
				iy += incY
			}
			jx += incX
		}
	case (tA == blas.Trans || tA == blas.ConjTrans) && o == blas.RowMajor:
		jy := ky
		for j := 0; j < n; j++ {
			var temp float64
			ix := kx
			for i := 0; i < m; i++ {
				temp += a[lda*i+j] * x[ix]
				ix += incX
			}
			y[jy] += alpha * temp
			jy += incY
		}
	default:
		// TODO: Add in other switch cases
		panic("Not yet implemented")
	}
}

// DTRMV  performs one of the matrix-vector operations
// 		x := A*x,   or   x := A**T*x,
// where x is an n element vector and  A is an n by n unit, or non-unit,
// upper or lower triangular matrix.
func Dtrmv(o blas.Order, ul blas.Uplo, tA blas.Transpose, d blas.Diag, n int, a []float64, lda int, x []float64, incX int) {
	// Verify inputs
	if o != blas.RowMajor && o != blas.ColMajor {
		panic(badOrder)
	}
	if ul != blas.Lower && ul != blas.Upper {
		panic(badUplo)
	}
	if tA != blas.NoTrans && tA != blas.Trans && tA != blas.ConjTrans {
		panic(badTranspose)
	}
	if d != blas.NonUnit && d != blas.Unit {
		panic(badDiag)
	}
	if n < 0 {
		panic(nLT0)
	}
	if lda > n && lda > 1 {
		panic("blas: lda must be less than max(1,n)")
	}
	if incX == 0 {
		panic(zeroInc)
	}
	if n == 0 {
		return
	}
	var kx int
	if incX <= 0 {
		kx = -(n - 1) * incX
	}
	switch {
	case o == blas.RowMajor && tA == blas.NoTrans && ul == blas.Upper:
		jx := kx
		for j := 0; j < n; j++ {
			if x[jx] != 0 {
				temp := x[jx]
				ix := kx
				for i := 0; i < j-1; i++ {
					x[ix] += temp * a[lda*i+j]
					ix += incX
				}
				if d == blas.NonUnit {
					x[jx] *= a[lda*j+j]
				}
			}
			jx += incX
		}
	case o == blas.RowMajor && tA == blas.NoTrans && ul == blas.Lower:
		kx += (n - 1) * incX
		jx := kx
		for j := n - 1; j >= 0; j-- {
			if x[jx] != 0 {
				tmp := x[jx]
				ix := kx
				for i := n - 1; i >= j; i-- {
					x[ix] += tmp * a[lda*i+j]
				}
				if d == blas.NonUnit {
					x[jx] *= a[lda*j+j]
				}
			}
		}
	case o == blas.RowMajor && (tA == blas.Trans || tA == blas.ConjTrans) && ul == blas.Upper:
		jx := kx + (n-1)*incX
		for j := n - 1; j >= 0; j-- {
			tmp := x[jx]
			ix := jx
			if d == blas.NonUnit {
				tmp *= a[lda*j+j]
			}
			for i := j - 2; j >= 0; j-- {
				ix -= incX
				tmp += a[lda*i+j] * x[ix]
			}
			x[jx] = tmp
			jx -= incX
		}
	case o == blas.RowMajor && (tA == blas.Trans || tA == blas.ConjTrans) && ul == blas.Lower:
		jx := kx
		for j := 0; j < n; j++ {
			tmp := x[jx]
			ix := jx
			if d == blas.NonUnit {
				tmp *= a[lda*j+j]
				for i := j; i < n; i++ {
					ix += incX
					tmp += a[lda*i+j] * x[ix]
				}
				x[jx] = tmp
				jx += incX
			}
		}
	default:
		panic("not yet implemented")
	}
}

// Dtrsv  solves one of the systems of equations
//    A*x = b,   or   A**T*x = b,
// where b and x are n element vectors and A is an n by n unit, or
// non-unit, upper or lower triangular matrix.
//
// No test for singularity or near-singularity is included in this
// routine. Such tests must be performed before calling this routine.
func Dtrsv(o blas.Order, ul blas.Uplo, tA blas.Transpose, d blas.Diag, n int, a []float64, lda int, x []float64, incX int) {
	// Test the input parameters
	// Verify inputs
	if o != blas.RowMajor && o != blas.ColMajor {
		panic(badOrder)
	}
	if ul != blas.Lower && ul != blas.Upper {
		panic(badUplo)
	}
	if tA != blas.NoTrans && tA != blas.Trans && tA != blas.ConjTrans {
		panic(badTranspose)
	}
	if d != blas.NonUnit && d != blas.Unit {
		panic(badDiag)
	}
	if n < 0 {
		panic(nLT0)
	}
	if lda > n && lda > 1 {
		panic("blas: lda must be less than max(1,n)")
	}
	if incX == 0 {
		panic(zeroInc)
	}
	// Quick return if possible
	if n == 0 {
		return
	}

	var kx int
	if incX < 0 {
		kx = -(n - 1) * incX
	}

	switch {
	case o == blas.RowMajor && tA == blas.NoTrans && ul == blas.Upper:
		jx := kx + (n-1)*incX
		for j := n; j >= 0; j-- {
			if x[jx] != 0 {
				if d == blas.NonUnit {
					x[jx] /= a[lda*j+j]
				}
				tmp := x[jx]
				ix := jx
				for i := j - 2; i >= 0; i-- {
					ix -= incX
					x[ix] -= tmp * a[lda*i+j]
				}
			}
			jx -= incX
		}
	case o == blas.RowMajor && tA == blas.NoTrans && ul == blas.Lower:
		jx := kx
		for j := 0; j < n; j++ {
			if x[jx] != 0 {
				if d == blas.NonUnit {
					x[jx] /= a[lda*j+j]
				}
				tmp := x[jx]
				ix := jx
				for i := j; i < n; j++ {
					ix += incX
					x[ix] -= tmp * a[lda*i+j]
				}
			}
			jx += incX
		}
	case o == blas.RowMajor && (tA == blas.Trans || tA == blas.ConjTrans) && ul == blas.Upper:
		jx := kx
		for j := 0; j < n; j++ {
			tmp := x[jx]
			ix := kx
			for i := 0; i < j-1; i++ {
				tmp -= a[lda*i+j] * x[ix]
				ix += incX
			}
			if d == blas.NonUnit {
				tmp /= a[lda*j+j]
			}
			x[jx] = tmp
			jx += incX
		}
	case o == blas.RowMajor && (tA == blas.Trans || tA == blas.ConjTrans) && ul == blas.Lower:
		kx += (n - 1) * incX
		jx := kx
		for j := n - 1; j >= 0; j-- {
			tmp := x[jx]
			ix := kx
			for i := n - 1; i >= j; i-- {
				tmp -= a[lda*i+j] * x[ix]
				ix -= incX
			}
			if d == blas.NonUnit {
				tmp /= a[lda*j+j]
				x[jx] = tmp
				jx -= incX
			}
		}
	}
}

/*
// Level 2 routines.
        Dgbmv(o Order, tA Transpose, m, n, kL, kU int, alpha float64, a []float64, lda int, x []float64, incX int, beta float64, y []float64, incY int)
        Dtbmv(o Order, ul Uplo, tA Transpose, d Diag, n, k int, a []float64, lda int, x []float64, incX int)
        Dtpmv(o Order, ul Uplo, tA Transpose, d Diag, n int, ap []float64, x []float64, incX int)
        Dtbsv(o Order, ul Uplo, tA Transpose, d Diag, n, k int, a []float64, lda int, x []float64, incX int)
        Dtpsv(o Order, ul Uplo, tA Transpose, d Diag, n int, ap []float64, x []float64, incX int)
        Dsbmv(o Order, ul Uplo, n, k int, alpha float64, a []float64, lda int, x []float64, incX int, beta float64, y []float64, incY int)
        Dspmv(o Order, ul Uplo, n int, alpha float64, ap []float64, x []float64, incX int, beta float64, y []float64, incY int)
		Dspr(o Order, ul Uplo, n int, alpha float64, x []float64, incX int, ap []float64)
		Dspr2(o Order, ul Uplo, n int, alpha float64, x []float64, incX int, y []float64, incY int, a []float64)

        Dsymv(o Order, ul Uplo, n int, alpha float64, a []float64, lda int, x []float64, incX int, beta float64, y []float64, incY int)
        Dger(o Order, m, n int, alpha float64, x []float64, incX int, y []float64, incY int, a []float64, lda int)
        Dsyr(o Order, ul Uplo, n int, alpha float64, x []float64, incX int, a []float64, lda int)
        Dsyr2(o Order, ul Uplo, n int, alpha float64, x []float64, incX int, y []float64, incY int, a []float64, lda int)
*/
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"

	"github.com/jingweno/jqplay/Godeps/_workspace/src/gopkg.in/unrolled/render.v1"
	"github.com/jingweno/jqplay/jq"
)

const (
	JSONPayloadLimit   = JSONPayloadLimitMB * OneMB
	JSONPayloadLimitMB = 5
	OneMB              = 1024000
)

type JQHandlerContext struct {
	*Config
	JQ string
}

func (c *JQHandlerContext) Asset(path string) string {
	return fmt.Sprintf("%s/%s", c.Config.AssetHost, path)
}

func (c *JQHandlerContext) ShouldInitJQ() bool {
	return c.JQ != ""
}

type JQHandler struct {
	r *render.Render
	c *Config
}

func (h *JQHandler) handleIndex(rw http.ResponseWriter, r *http.Request) {
	h.r.HTML(rw, 200, "index", &JQHandlerContext{Config: h.c})
}

func (h *JQHandler) handleJqPost(rw http.ResponseWriter, r *http.Request) {
	if r.ContentLength == -1 {
		log.Printf("Error: Content length is unknown")
	}

	if r.ContentLength > JSONPayloadLimit {
		msg := fmt.Sprintf("JSON payload size is %.1fMB, larger than limit %dMB.", float64(r.ContentLength)/OneMB, JSONPayloadLimitMB)
		log.Printf("Error: %s", msg)
		h.r.JSON(rw, 403, map[string]string{
			"message": msg,
		})

		return
	}

	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		h.r.JSON(rw, 422, map[string]string{"message": err.Error()})
		return
	}
	defer r.Body.Close()

	var jq *jq.JQ
	err = json.Unmarshal(b, &jq)
	if err != nil {
		h.r.JSON(rw, 422, map[string]string{"message": err.Error()})
		return
	}

	re, err := jq.Eval()
	if err != nil {
		h.r.JSON(rw, 422, map[string]string{"message": err.Error()})
		return
	}

	h.r.JSON(rw, 200, map[string]string{"result": re})
}

func (h *JQHandler) handleJqGet(rw http.ResponseWriter, r *http.Request) {
	q := r.URL.Query()
	jq := &jq.JQ{
		J: q.Get("j"),
		Q: q.Get("q"),
	}

	var jqData string
	if err := jq.Validate(); err == nil {
		d, err := json.Marshal(jq)
		if err == nil {
			jqData = string(d)
		}
	}

	h.r.HTML(rw, 200, "index", &JQHandlerContext{Config: h.c, JQ: jqData})
}

func (h *JQHandler) handleJq(rw http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		h.handleJqPost(rw, r)
		return
	}

	if r.Method == "GET" {
		h.handleJqGet(rw, r)
		return
	}

	h.r.JSON(rw, 500, nil)
}
<commit_msg>Log `jq` when there's an error<commit_after>package server

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"

	"github.com/jingweno/jqplay/Godeps/_workspace/src/gopkg.in/unrolled/render.v1"
	"github.com/jingweno/jqplay/jq"
)

const (
	JSONPayloadLimit   = JSONPayloadLimitMB * OneMB
	JSONPayloadLimitMB = 5
	OneMB              = 1024000
)

type JQHandlerContext struct {
	*Config
	JQ string
}

func (c *JQHandlerContext) Asset(path string) string {
	return fmt.Sprintf("%s/%s", c.Config.AssetHost, path)
}

func (c *JQHandlerContext) ShouldInitJQ() bool {
	return c.JQ != ""
}

type JQHandler struct {
	r *render.Render
	c *Config
}

func (h *JQHandler) handleIndex(rw http.ResponseWriter, r *http.Request) {
	h.r.HTML(rw, 200, "index", &JQHandlerContext{Config: h.c})
}

func (h *JQHandler) handleJqPost(rw http.ResponseWriter, r *http.Request) {
	if r.ContentLength == -1 {
		log.Printf("Error: Content length is unknown")
	}

	if r.ContentLength > JSONPayloadLimit {
		msg := fmt.Sprintf("JSON payload size is %.1fMB, larger than limit %dMB.", float64(r.ContentLength)/OneMB, JSONPayloadLimitMB)
		log.Printf("Error: %s", msg)
		h.r.JSON(rw, 403, map[string]string{
			"message": msg,
		})

		return
	}

	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		h.r.JSON(rw, 422, map[string]string{"message": err.Error()})
		return
	}
	defer r.Body.Close()

	var jq *jq.JQ
	err = json.Unmarshal(b, &jq)
	if err != nil {
		h.r.JSON(rw, 422, map[string]string{"message": err.Error()})
		return
	}

	re, err := jq.Eval()
	if err != nil {
		log.Printf("%s\n", jq)
		h.r.JSON(rw, 422, map[string]string{"message": err.Error()})
		return
	}

	h.r.JSON(rw, 200, map[string]string{"result": re})
}

func (h *JQHandler) handleJqGet(rw http.ResponseWriter, r *http.Request) {
	q := r.URL.Query()
	jq := &jq.JQ{
		J: q.Get("j"),
		Q: q.Get("q"),
	}

	var jqData string
	if err := jq.Validate(); err == nil {
		d, err := json.Marshal(jq)
		if err == nil {
			jqData = string(d)
		}
	}

	h.r.HTML(rw, 200, "index", &JQHandlerContext{Config: h.c, JQ: jqData})
}

func (h *JQHandler) handleJq(rw http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		h.handleJqPost(rw, r)
		return
	}

	if r.Method == "GET" {
		h.handleJqGet(rw, r)
		return
	}

	h.r.JSON(rw, 500, nil)
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"time"

	"github.com/Sirupsen/logrus"
	log "github.com/Sirupsen/logrus"
	"github.com/jingweno/jqplay/config"
	"github.com/jingweno/jqplay/jq"
	"golang.org/x/net/context"
	"gopkg.in/gin-gonic/gin.v1"
)

const (
	JSONPayloadLimit   = JSONPayloadLimitMB * OneMB
	JSONPayloadLimitMB = 10
	OneMB              = 1024000
	JQExecTimeout      = 15 * time.Second
)

type JQHandlerContext struct {
	*config.Config
	JQ string
}

func (c *JQHandlerContext) Asset(path string) string {
	return fmt.Sprintf("%s/%s", c.AssetHost, path)
}

func (c *JQHandlerContext) ShouldInitJQ() bool {
	return c.JQ != ""
}

type JQHandler struct {
	DB     *DB
	Config *config.Config
}

func (h *JQHandler) handleIndex(c *gin.Context) {
	c.HTML(200, "index.tmpl", &JQHandlerContext{Config: h.Config})
}

func (h *JQHandler) checkReqSize(c *gin.Context) (float64, error) {
	if c.Request.ContentLength > JSONPayloadLimit {
		size := float64(c.Request.ContentLength) / OneMB
		return size, fmt.Errorf("JSON payload size is %.1fMB, larger than limit %dMB.", size, JSONPayloadLimitMB)
	}

	return 0, nil
}

func (h *JQHandler) handleJqPost(c *gin.Context) {
	if size, err := h.checkReqSize(c); err != nil {
		h.logger(c).WithField("size", size).WithError(err)
		c.String(http.StatusExpectationFailed, err.Error())
		return
	}

	c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, JSONPayloadLimit)

	var j *jq.JQ
	if err := c.BindJSON(&j); err != nil {
		err = fmt.Errorf("error parsing JSON: %s", err)
		h.logger(c).WithError(err).Infof("error parsing JSON")
		c.String(http.StatusUnprocessableEntity, err.Error())
		return
	}

	ctx, cancel := context.WithTimeout(c.Request.Context(), JQExecTimeout)
	defer cancel()

	// Evaling into ResponseWriter sets the status code to 200
	// appending error message in the end if there's any
	var debug bytes.Buffer
	w := io.MultiWriter(c.Writer, &debug)
	if err := j.Eval(ctx, w); err != nil {
		fmt.Fprint(c.Writer, err.Error())

		if shouldLogJQError(err) {
			h.logger(c).WithError(err).WithFields(log.Fields{
				"j": j.J,
				"q": j.Q,
				"o": j.Opts(),
				"r": debug.String(),
			}).Info("jq error")
		}
	}
}

func (h *JQHandler) handleJqGet(c *gin.Context) {
	jq := &jq.JQ{
		J: c.Query("j"),
		Q: c.Query("q"),
	}

	var jqData string
	if err := jq.Validate(); err == nil {
		d, err := json.Marshal(jq)
		if err == nil {
			jqData = string(d)
		}
	}

	c.HTML(http.StatusOK, "index.tmpl", &JQHandlerContext{Config: h.Config, JQ: jqData})
}

func (h *JQHandler) handleJqSharePost(c *gin.Context) {
	if size, err := h.checkReqSize(c); err != nil {
		h.logger(c).WithField("size", size).WithError(err)
		c.String(http.StatusExpectationFailed, err.Error())
		return
	}

	c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, JSONPayloadLimit)

	var jq *jq.JQ
	if err := c.BindJSON(&jq); err != nil {
		err = fmt.Errorf("error parsing JSON: %s", err)
		h.logger(c).WithError(err)
		c.String(http.StatusUnprocessableEntity, err.Error())
		return
	}

	if err := jq.Validate(); err != nil {
		c.String(http.StatusUnprocessableEntity, err.Error())
		return
	}

	id, err := h.DB.UpsertSnippet(FromJQ(jq))
	if err != nil {
		h.logger(c).WithError(err)
		c.String(http.StatusUnprocessableEntity, "error sharing snippet")
		return
	}

	c.String(http.StatusCreated, id)
}

func (h *JQHandler) handleJqShareGet(c *gin.Context) {
	id := c.Param("id")

	s, err := h.DB.GetSnippet(id)
	if err != nil {
		h.logger(c).WithError(err).WithField("id", id).Info("error getting snippet")
		c.Redirect(http.StatusFound, "/")
		return
	}

	var jqData string
	d, err := json.Marshal(ToJQ(s))
	if err == nil {
		jqData = string(d)
	}

	c.HTML(200, "index.tmpl", &JQHandlerContext{
		Config: h.Config,
		JQ:     jqData,
	})
}

func (h *JQHandler) logger(c *gin.Context) *logrus.Entry {
	l, _ := c.Get("logger")
	return l.(*logrus.Entry)
}

func shouldLogJQError(err error) bool {
	return err == jq.ExecTimeoutError
}
<commit_msg>Better error logging<commit_after>package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"time"

	"github.com/Sirupsen/logrus"
	log "github.com/Sirupsen/logrus"
	"github.com/jingweno/jqplay/config"
	"github.com/jingweno/jqplay/jq"
	"golang.org/x/net/context"
	"gopkg.in/gin-gonic/gin.v1"
)

const (
	JSONPayloadLimit   = JSONPayloadLimitMB * OneMB
	JSONPayloadLimitMB = 10
	OneMB              = 1024000
	JQExecTimeout      = 15 * time.Second
)

type JQHandlerContext struct {
	*config.Config
	JQ string
}

func (c *JQHandlerContext) Asset(path string) string {
	return fmt.Sprintf("%s/%s", c.AssetHost, path)
}

func (c *JQHandlerContext) ShouldInitJQ() bool {
	return c.JQ != ""
}

type JQHandler struct {
	DB     *DB
	Config *config.Config
}

func (h *JQHandler) handleIndex(c *gin.Context) {
	c.HTML(200, "index.tmpl", &JQHandlerContext{Config: h.Config})
}

func (h *JQHandler) checkReqSize(c *gin.Context) (float64, error) {
	if c.Request.ContentLength > JSONPayloadLimit {
		size := float64(c.Request.ContentLength) / OneMB
		return size, fmt.Errorf("JSON payload size is %.1fMB, larger than limit %dMB.", size, JSONPayloadLimitMB)
	}

	return 0, nil
}

func (h *JQHandler) handleJqPost(c *gin.Context) {
	if size, err := h.checkReqSize(c); err != nil {
		h.logger(c).WithField("size", size).WithError(err).Info("req too large")
		c.String(http.StatusExpectationFailed, err.Error())
		return
	}

	c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, JSONPayloadLimit)

	var j *jq.JQ
	if err := c.BindJSON(&j); err != nil {
		err = fmt.Errorf("error parsing JSON: %s", err)
		h.logger(c).WithError(err).Info("error parsing JSON")
		c.String(http.StatusUnprocessableEntity, err.Error())
		return
	}

	ctx, cancel := context.WithTimeout(c.Request.Context(), JQExecTimeout)
	defer cancel()

	// Evaling into ResponseWriter sets the status code to 200
	// appending error message in the end if there's any
	var debug bytes.Buffer
	w := io.MultiWriter(c.Writer, &debug)
	if err := j.Eval(ctx, w); err != nil {
		fmt.Fprint(c.Writer, err.Error())

		if shouldLogJQError(err) {
			h.logger(c).WithError(err).WithFields(log.Fields{
				"j": j.J,
				"q": j.Q,
				"o": j.Opts(),
				"r": debug.String(),
			}).Info("jq error")
		}
	}
}

func (h *JQHandler) handleJqGet(c *gin.Context) {
	jq := &jq.JQ{
		J: c.Query("j"),
		Q: c.Query("q"),
	}

	var jqData string
	if err := jq.Validate(); err == nil {
		d, err := json.Marshal(jq)
		if err == nil {
			jqData = string(d)
		}
	}

	c.HTML(http.StatusOK, "index.tmpl", &JQHandlerContext{Config: h.Config, JQ: jqData})
}

func (h *JQHandler) handleJqSharePost(c *gin.Context) {
	if size, err := h.checkReqSize(c); err != nil {
		h.logger(c).WithField("size", size).WithError(err).Info("req too large")
		c.String(http.StatusExpectationFailed, err.Error())
		return
	}

	c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, JSONPayloadLimit)

	var jq *jq.JQ
	if err := c.BindJSON(&jq); err != nil {
		err = fmt.Errorf("error parsing JSON: %s", err)
		h.logger(c).WithError(err).Info("error parsing JSON")
		c.String(http.StatusUnprocessableEntity, err.Error())
		return
	}

	if err := jq.Validate(); err != nil {
		c.String(http.StatusUnprocessableEntity, err.Error())
		return
	}

	id, err := h.DB.UpsertSnippet(FromJQ(jq))
	if err != nil {
		h.logger(c).WithError(err).Info("error upserting snippet")
		c.String(http.StatusUnprocessableEntity, "error sharing snippet")
		return
	}

	c.String(http.StatusCreated, id)
}

func (h *JQHandler) handleJqShareGet(c *gin.Context) {
	id := c.Param("id")

	s, err := h.DB.GetSnippet(id)
	if err != nil {
		h.logger(c).WithError(err).WithField("id", id).Info("error getting snippet")
		c.Redirect(http.StatusFound, "/")
		return
	}

	var jqData string
	d, err := json.Marshal(ToJQ(s))
	if err == nil {
		jqData = string(d)
	}

	c.HTML(200, "index.tmpl", &JQHandlerContext{
		Config: h.Config,
		JQ:     jqData,
	})
}

func (h *JQHandler) logger(c *gin.Context) *logrus.Entry {
	l, _ := c.Get("logger")
	return l.(*logrus.Entry)
}

func shouldLogJQError(err error) bool {
	return err == jq.ExecTimeoutError
}
<|endoftext|>