<commit_before>package main

import (
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"encoding/base64"
	"github.com/realglobe-Inc/edo/util"
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"
)

const (
	headerTaId       = "X-Edo-Ta-Id"
	headerTaToken    = "X-Edo-Ta-Token"
	headerTaTokenSig = "X-Edo-Ta-Token-Sign"
	headerHashFunc   = "X-Edo-Hash-Function"

	headerAccProxErr = "X-Edo-Access-Proxy-Error"

	headerTaAuthErr = "X-Edo-Ta-Auth-Error"

	cookieTaSess = "X-Edo-Ta-Session"
)

// Web プロキシ。
func proxyApi(sys *system, w http.ResponseWriter, r *http.Request) error {

	if !strings.HasPrefix(r.RequestURI, "http://") && !strings.HasPrefix(r.RequestURI, "https://") {
		return erro.Wrap(util.NewHttpStatusError(http.StatusBadRequest, "no scheme in request uri", nil))
	}

	taId := r.Header.Get(headerTaId)
	if taId == "" {
		taId = sys.taId
	}

	sess, _, err := sys.session(uriBase(r.URL), taId, nil)
	if err != nil {
		return erro.Wrap(err)
	}

	if sess != nil {
		// セッション確立済み。
		log.Debug("authenticated session is exist")
		return forward(sys, w, r, taId, sess)
	} else {
		// セッション未確立。
		log.Debug("session is not exist")
		return startSession(sys, w, r, taId)
	}
}

// 転送する。
func forward(sys *system, w http.ResponseWriter, r *http.Request, taId string, sess *session) error {
	r.AddCookie(&http.Cookie{Name: cookieTaSess, Value: sess.id})
	r.RequestURI = ""

	////////////////////////////////////////////////////////////
	util.LogRequest(r, true)
	////////////////////////////////////////////////////////////
	resp, err := sess.cli.Do(r)
	if err != nil {
		err = erro.Wrap(err)
		switch erro.Unwrap(err).(type) {
		case *net.OpError:
			return erro.Wrap(util.NewHttpStatusError(http.StatusNotFound, "cannot connect "+uriBase(r.URL), err))
		default:
			return err
		}
	}
	defer resp.Body.Close()
	////////////////////////////////////////////////////////////
	util.LogResponse(resp, true)
	////////////////////////////////////////////////////////////

	log.Debug("forwarded")

	if resp.StatusCode == http.StatusUnauthorized && resp.Header.Get(headerTaAuthErr) != "" {
		// edo-auth で 401 Unauthorized なら、タイミングの問題なので startSession からやり直す。
		// 古いセッションは上書きされるので消す必要無し。
		return startSession(sys, w, r, taId)
	}

	return copyResponse(resp, w)
}

// セッション開始。
func startSession(sys *system, w http.ResponseWriter, r *http.Request, taId string) error {

	cli := &http.Client{}

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return erro.Wrap(err)
	}

	r.RequestURI = ""
	r.Body = ioutil.NopCloser(bytes.NewReader(body))

	////////////////////////////////////////////////////////////
	util.LogRequest(r, true)
	////////////////////////////////////////////////////////////
	resp, err := cli.Do(r)
	if err != nil {
		err = erro.Wrap(err)
		switch erro.Unwrap(err).(type) {
		case *net.OpError:
			return erro.Wrap(util.NewHttpStatusError(http.StatusNotFound, "cannot connect "+uriBase(r.URL), err))
		default:
			return err
		}
	}
	defer resp.Body.Close()
	////////////////////////////////////////////////////////////
	util.LogResponse(resp, true)
	////////////////////////////////////////////////////////////

	log.Debug("sent raw request")

	if resp.Header.Get(headerTaAuthErr) == "" || resp.StatusCode != http.StatusUnauthorized {
		// 相手側が TA 認証を必要としていなかったのかもしれない。
		return copyResponse(resp, w)
	}

	// 相手側 TA も認証始めた。
	log.Debug("authentication started")

	sess, sessToken := parseSession(resp)
	if sess == nil {
		return erro.Wrap(util.NewHttpStatusError(http.StatusForbidden, "no cookie "+cookieTaSess, nil))
	} else if sessToken == "" {
		return erro.Wrap(util.NewHttpStatusError(http.StatusForbidden, "no header field "+headerTaToken, nil))
	}

	expiDate := getExpirationDate(sess)

	// 認証用データが揃ってた。
	log.Debug("authentication data was found")

	priKey, _, err := sys.privateKey(taId, nil)
	if err != nil {
		return erro.Wrap(err)
	} else if priKey == nil {
		return erro.Wrap(util.NewHttpStatusError(http.StatusForbidden, "no private key of "+taId, nil))
	}

	// 秘密鍵を用意できた。
	log.Debug("private key of " + taId + " is exist")

	hashName := r.Header.Get(headerHashFunc)
	if hashName == "" {
		hashName = sys.hashName
	}

	tokenSign, err := sign(priKey, hashName, sessToken)
	if err != nil {
		return erro.Wrap(err)
	}

	// 署名できた。
	log.Debug("signed")

	r.AddCookie(&http.Cookie{Name: cookieTaSess, Value: sess.Value})
	r.Header.Set(headerTaId, taId)
	r.Header.Set(headerTaTokenSig, tokenSign)
	r.Header.Set(headerHashFunc, hashName)
	r.RequestURI = ""
	r.Body = ioutil.NopCloser(bytes.NewReader(body))

	////////////////////////////////////////////////////////////
	util.LogRequest(r, true)
	////////////////////////////////////////////////////////////
	resp, err = cli.Do(r)
	if err != nil {
		err = erro.Wrap(err)
		switch erro.Unwrap(err).(type) {
		case *net.OpError:
			return erro.Wrap(util.NewHttpStatusError(http.StatusNotFound, "cannot connect "+uriBase(r.URL), err))
		default:
			return err
		}
	}
	defer resp.Body.Close()
	////////////////////////////////////////////////////////////
	util.LogResponse(resp, true)
	////////////////////////////////////////////////////////////

	// 認証された。
	log.Debug("authentication finished")

	if resp.Header.Get(headerTaAuthErr) == "" {
		// セッションを保存。
		if _, err := sys.addSession(&session{id: sess.Value, uri: uriBase(r.URL), taId: taId, cli: cli}, expiDate); err != nil {
			err = erro.Wrap(err)
			log.Err(erro.Unwrap(err))
			log.Debug(err)
		} else {
			log.Debug("session was saved")
		}
	}

	return copyResponse(resp, w)
}

// クエリパラメータ等を除いた URL を得る。
func uriBase(url *url.URL) string {
	return url.Scheme + "://" + url.Host + url.Path
}

// プロキシ先がおかしいかどうか。
func isDestinationError(err error) bool {
	for {
		switch e := erro.Unwrap(err).(type) {
		case *net.OpError:
			return true
		case *url.Error:
			if e.Err != nil {
				err = e.Err
			} else {
				return false
			}
		case *erro.Tracer:
			err = e.Cause()
		case *util.HttpStatusError:
			if e.Cause() != nil {
				err = e.Cause()
			} else {
				return false
			}
		default:
			return false
		}
	}
}

// 相手側 TA の認証開始レスポンスから必要情報を抜き出す。
func parseSession(resp *http.Response) (sess *http.Cookie, sessToken string) {
	for _, cookie := range resp.Cookies() {
		if cookie.Name == cookieTaSess {
			sess = cookie
			break
		}
	}

	return sess, resp.Header.Get(headerTaToken)
}

// 相手側 TA からのレスポンスをリクエスト元へのレスポンスに写す。
func copyResponse(resp *http.Response, w http.ResponseWriter) error {
	// ヘッダフィールドのコピー。
	for key, values := range resp.Header {
		for _, value := range values {
			w.Header().Add(key, value)
		}
	}

	// ステータスのコピー。
	w.WriteHeader(resp.StatusCode)

	// ボディのコピー。
	if _, err := io.Copy(w, resp.Body); err != nil {
		return erro.Wrap(err)
	}

	return nil
}

// 相手側 TA からのお題に署名する。
func sign(priKey *rsa.PrivateKey, hashName, token string) (string, error) {
	hash, err := util.ParseHashFunction(hashName)
	if err != nil {
		return "", erro.Wrap(err)
	}

	h := hash.New()
	h.Write([]byte(token))
	buff, err := rsa.SignPKCS1v15(rand.Reader, priKey, hash, h.Sum(nil))
	if err != nil {
		return "", erro.Wrap(err)
	}

	return base64.StdEncoding.EncodeToString(buff), nil
}

// 相手側 TA が提示したセッションの有効期限を読み取る。
func getExpirationDate(sess *http.Cookie) (expiDate time.Time) {
	if sess.MaxAge != 0 {
		return time.Now().Add(time.Duration(sess.MaxAge))
	} else {
		return sess.Expires
	}
}
<commit_msg>通信先の問題かどうか判定している部分を整理<commit_after>package main

import (
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"encoding/base64"
	"github.com/realglobe-Inc/edo/util"
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"
)

const (
	headerTaId       = "X-Edo-Ta-Id"
	headerTaToken    = "X-Edo-Ta-Token"
	headerTaTokenSig = "X-Edo-Ta-Token-Sign"
	headerHashFunc   = "X-Edo-Hash-Function"

	headerAccProxErr = "X-Edo-Access-Proxy-Error"

	headerTaAuthErr = "X-Edo-Ta-Auth-Error"

	cookieTaSess = "X-Edo-Ta-Session"
)

// Web プロキシ。
func proxyApi(sys *system, w http.ResponseWriter, r *http.Request) error {

	if !strings.HasPrefix(r.RequestURI, "http://") && !strings.HasPrefix(r.RequestURI, "https://") {
		return erro.Wrap(util.NewHttpStatusError(http.StatusBadRequest, "no scheme in request uri", nil))
	}

	taId := r.Header.Get(headerTaId)
	if taId == "" {
		taId = sys.taId
	}

	sess, _, err := sys.session(uriBase(r.URL), taId, nil)
	if err != nil {
		return erro.Wrap(err)
	}

	if sess != nil {
		// セッション確立済み。
		log.Debug("authenticated session is exist")
		return forward(sys, w, r, taId, sess)
	} else {
		// セッション未確立。
		log.Debug("session is not exist")
		return startSession(sys, w, r, taId)
	}
}

// 転送する。
func forward(sys *system, w http.ResponseWriter, r *http.Request, taId string, sess *session) error {
	r.AddCookie(&http.Cookie{Name: cookieTaSess, Value: sess.id})
	r.RequestURI = ""

	////////////////////////////////////////////////////////////
	util.LogRequest(r, true)
	////////////////////////////////////////////////////////////
	resp, err := sess.cli.Do(r)
	if err != nil {
		err = erro.Wrap(err)
		if isDestinationError(err) {
			return erro.Wrap(util.NewHttpStatusError(http.StatusNotFound, "cannot connect "+uriBase(r.URL), err))
		} else {
			return err
		}
	}
	defer resp.Body.Close()
	////////////////////////////////////////////////////////////
	util.LogResponse(resp, true)
	////////////////////////////////////////////////////////////

	log.Debug("forwarded")

	if resp.StatusCode == http.StatusUnauthorized && resp.Header.Get(headerTaAuthErr) != "" {
		// edo-auth で 401 Unauthorized なら、タイミングの問題なので startSession からやり直す。
		// 古いセッションは上書きされるので消す必要無し。
		return startSession(sys, w, r, taId)
	}

	return copyResponse(resp, w)
}

// セッション開始。
func startSession(sys *system, w http.ResponseWriter, r *http.Request, taId string) error {

	cli := &http.Client{}

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return erro.Wrap(err)
	}

	r.RequestURI = ""
	r.Body = ioutil.NopCloser(bytes.NewReader(body))

	////////////////////////////////////////////////////////////
	util.LogRequest(r, true)
	////////////////////////////////////////////////////////////
	resp, err := cli.Do(r)
	if err != nil {
		err = erro.Wrap(err)
		if isDestinationError(err) {
			return erro.Wrap(util.NewHttpStatusError(http.StatusNotFound, "cannot connect "+uriBase(r.URL), err))
		} else {
			return err
		}
	}
	defer resp.Body.Close()
	////////////////////////////////////////////////////////////
	util.LogResponse(resp, true)
	////////////////////////////////////////////////////////////

	log.Debug("sent raw request")

	if resp.Header.Get(headerTaAuthErr) == "" || resp.StatusCode != http.StatusUnauthorized {
		// 相手側が TA 認証を必要としていなかったのかもしれない。
		return copyResponse(resp, w)
	}

	// 相手側 TA も認証始めた。
	log.Debug("authentication started")

	sess, sessToken := parseSession(resp)
	if sess == nil {
		return erro.Wrap(util.NewHttpStatusError(http.StatusForbidden, "no cookie "+cookieTaSess, nil))
	} else if sessToken == "" {
		return erro.Wrap(util.NewHttpStatusError(http.StatusForbidden, "no header field "+headerTaToken, nil))
	}

	expiDate := getExpirationDate(sess)

	// 認証用データが揃ってた。
	log.Debug("authentication data was found")

	priKey, _, err := sys.privateKey(taId, nil)
	if err != nil {
		return erro.Wrap(err)
	} else if priKey == nil {
		return erro.Wrap(util.NewHttpStatusError(http.StatusForbidden, "no private key of "+taId, nil))
	}

	// 秘密鍵を用意できた。
	log.Debug("private key of " + taId + " is exist")

	hashName := r.Header.Get(headerHashFunc)
	if hashName == "" {
		hashName = sys.hashName
	}

	tokenSign, err := sign(priKey, hashName, sessToken)
	if err != nil {
		return erro.Wrap(err)
	}

	// 署名できた。
	log.Debug("signed")

	r.AddCookie(&http.Cookie{Name: cookieTaSess, Value: sess.Value})
	r.Header.Set(headerTaId, taId)
	r.Header.Set(headerTaTokenSig, tokenSign)
	r.Header.Set(headerHashFunc, hashName)
	r.RequestURI = ""
	r.Body = ioutil.NopCloser(bytes.NewReader(body))

	////////////////////////////////////////////////////////////
	util.LogRequest(r, true)
	////////////////////////////////////////////////////////////
	resp, err = cli.Do(r)
	if err != nil {
		err = erro.Wrap(err)
		if isDestinationError(err) {
			return erro.Wrap(util.NewHttpStatusError(http.StatusNotFound, "cannot connect "+uriBase(r.URL), err))
		} else {
			return err
		}
	}
	defer resp.Body.Close()
	////////////////////////////////////////////////////////////
	util.LogResponse(resp, true)
	////////////////////////////////////////////////////////////

	// 認証された。
	log.Debug("authentication finished")

	if resp.Header.Get(headerTaAuthErr) == "" {
		// セッションを保存。
		if _, err := sys.addSession(&session{id: sess.Value, uri: uriBase(r.URL), taId: taId, cli: cli}, expiDate); err != nil {
			err = erro.Wrap(err)
			log.Err(erro.Unwrap(err))
			log.Debug(err)
		} else {
			log.Debug("session was saved")
		}
	}

	return copyResponse(resp, w)
}

// クエリパラメータ等を除いた URL を得る。
func uriBase(url *url.URL) string {
	return url.Scheme + "://" + url.Host + url.Path
}

// プロキシ先がおかしいかどうか。
func isDestinationError(err error) bool {
	for {
		switch e := erro.Unwrap(err).(type) {
		case *net.OpError:
			return true
		case *url.Error:
			if e.Err != nil {
				err = e.Err
			} else {
				return false
			}
		case *erro.Tracer:
			err = e.Cause()
		case *util.HttpStatusError:
			if e.Cause() != nil {
				err = e.Cause()
			} else {
				return false
			}
		default:
			return false
		}
	}
}

// 相手側 TA の認証開始レスポンスから必要情報を抜き出す。
func parseSession(resp *http.Response) (sess *http.Cookie, sessToken string) {
	for _, cookie := range resp.Cookies() {
		if cookie.Name == cookieTaSess {
			sess = cookie
			break
		}
	}

	return sess, resp.Header.Get(headerTaToken)
}

// 相手側 TA からのレスポンスをリクエスト元へのレスポンスに写す。
func copyResponse(resp *http.Response, w http.ResponseWriter) error {
	// ヘッダフィールドのコピー。
	for key, values := range resp.Header {
		for _, value := range values {
			w.Header().Add(key, value)
		}
	}

	// ステータスのコピー。
	w.WriteHeader(resp.StatusCode)

	// ボディのコピー。
	if _, err := io.Copy(w, resp.Body); err != nil {
		return erro.Wrap(err)
	}

	return nil
}

// 相手側 TA からのお題に署名する。
func sign(priKey *rsa.PrivateKey, hashName, token string) (string, error) {
	hash, err := util.ParseHashFunction(hashName)
	if err != nil {
		return "", erro.Wrap(err)
	}

	h := hash.New()
	h.Write([]byte(token))
	buff, err := rsa.SignPKCS1v15(rand.Reader, priKey, hash, h.Sum(nil))
	if err != nil {
		return "", erro.Wrap(err)
	}

	return base64.StdEncoding.EncodeToString(buff), nil
}

// 相手側 TA が提示したセッションの有効期限を読み取る。
func getExpirationDate(sess *http.Cookie) (expiDate time.Time) {
	if sess.MaxAge != 0 {
		return time.Now().Add(time.Duration(sess.MaxAge))
	} else {
		return sess.Expires
	}
}
<|endoftext|>


<commit_before>package worker

import (
	"fmt"
	"github.com/go-redis/redis"
	"github.com/stitchfix/flotilla-os/queue"
	"math/rand"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/stitchfix/flotilla-os/config"
	"github.com/stitchfix/flotilla-os/execution/engine"
	flotillaLog "github.com/stitchfix/flotilla-os/log"
	"github.com/stitchfix/flotilla-os/state"
	"gopkg.in/tomb.v2"
)

type statusWorker struct {
	sm           state.Manager
	ee           engine.Engine
	conf         config.Config
	log          flotillaLog.Logger
	pollInterval time.Duration
	t            tomb.Tomb
	engine       *string
	redisClient  *redis.Client
	workerId     string
}

func (sw *statusWorker) Initialize(conf config.Config, sm state.Manager, ee engine.Engine, log flotillaLog.Logger, pollInterval time.Duration, engine *string, qm queue.Manager) error {
	sw.pollInterval = pollInterval
	sw.conf = conf
	sw.sm = sm
	sw.ee = ee
	sw.log = log
	sw.engine = engine
	sw.workerId = fmt.Sprintf("%d", rand.Int())
	sw.setupRedisClient(conf)
	_ = sw.log.Log("message", "initialized a status worker", "engine", *engine)
	return nil
}

func (sw *statusWorker) setupRedisClient(conf config.Config) {
	if *sw.engine == state.EKSEngine {
		sw.redisClient = redis.NewClient(&redis.Options{Addr: conf.GetString("redis_address"), DB: conf.GetInt("redis_db")})
	}
}

func (sw *statusWorker) GetTomb() *tomb.Tomb {
	return &sw.t
}

//
// Run updates status of tasks
//
func (sw *statusWorker) Run() error {
	for {
		select {
		case <-sw.t.Dying():
			sw.log.Log("message", "A status worker was terminated")
			return nil
		default:
			if *sw.engine == state.ECSEngine {
				sw.runOnceECS()
				time.Sleep(sw.pollInterval)
			}

			if *sw.engine == state.EKSEngine {
				sw.runOnceEKS()
				time.Sleep(sw.pollInterval)
			}
		}
	}
}

func (sw *statusWorker) runOnceEKS() {
	rl, err := sw.sm.ListRuns(1000, 0, "queued_at", "asc", map[string][]string{
		"queued_at_since": {
			time.Now().AddDate(0, 0, -3).Format(time.RFC3339),
		},
		"status": {state.StatusNeedsRetry, state.StatusRunning, state.StatusQueued, state.StatusPending},
	}, nil, []string{state.EKSEngine})

	if err != nil {
		_ = sw.log.Log("message", "unable to receive runs", "error", fmt.Sprintf("%+v", err))
		return
	}
	runs := rl.Runs
	sw.processEKSRuns(runs)
}

func (sw *statusWorker) processEKSRuns(runs []state.Run) {
	for _, run := range runs {
		reloadRun, err := sw.sm.GetRun(run.RunID)
		if err == nil && reloadRun.Status != state.StatusStopped {
			if sw.acquireLock(run, "status", 10*time.Second) == true {
				sw.processEKSRun(run)
				sw.processEKSRunMetrics(run)
			}
		}
	}
}
func (sw *statusWorker) acquireLock(run state.Run, purpose string, expiration time.Duration) bool {
	set, err := sw.redisClient.SetNX(fmt.Sprintf("%s-%s", run.RunID, purpose), sw.workerId, expiration).Result()
	if err != nil {
		// Turn off in dev mode; too noisy.
		if sw.conf.GetString("flotilla_mode") != "dev" {
			_ = sw.log.Log("message", "unable to set lock", "error", fmt.Sprintf("%+v", err))
		}
		return false
	}
	return set
}

func (sw *statusWorker) processEKSRun(run state.Run) {
	reloadRun, err := sw.sm.GetRun(run.RunID)
	if err == nil && reloadRun.Status == state.StatusStopped {
		// Run was updated by another worker process.
		return
	}
	updatedRun, err := sw.ee.FetchUpdateStatus(run)
	if err != nil {
		message := fmt.Sprintf("%+v", err)
		_ = sw.log.Log("message", "unable to receive eks runs", "error", message)

		minutesInQueue := time.Now().Sub(*run.QueuedAt).Minutes()
		if strings.Contains(message, "not found") && minutesInQueue > float64(30) {
			stoppedAt := time.Now()
			reason := "Job either timed out or not found on the EKS cluster."
			updatedRun.Status = state.StatusStopped
			updatedRun.FinishedAt = &stoppedAt
			updatedRun.ExitReason = &reason
			_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
		}

	} else {
		if run.Status != updatedRun.Status {
			_ = sw.log.Log("message", "updating eks run", "run", updatedRun.RunID, "status", updatedRun.Status)
			_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
			if err != nil {
				_ = sw.log.Log("message", "unable to save eks runs", "error", fmt.Sprintf("%+v", err))
			}

			if updatedRun.Status == state.StatusStopped {
				//TODO - move to a separate worker.
				//_ = sw.ee.Terminate(run)
			}
		} else {
			if updatedRun.MaxMemoryUsed != run.MaxMemoryUsed ||
				updatedRun.MaxCpuUsed != run.MaxCpuUsed ||
				updatedRun.Cpu != run.Cpu ||
				updatedRun.Memory != run.Memory ||
				updatedRun.PodEvents != run.PodEvents {
				_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
			}
		}
	}
}

func (sw *statusWorker) processEKSRunMetrics(run state.Run) {
	updatedRun, err := sw.ee.FetchPodMetrics(run)
	if err == nil {
		if updatedRun.MaxMemoryUsed != run.MaxMemoryUsed ||
			updatedRun.MaxCpuUsed != run.MaxCpuUsed {
			_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
		}
	}
}

func (sw *statusWorker) runOnceECS() {
	runReceipt, err := sw.ee.PollStatus()
	if err != nil {
		sw.log.Log("message", "unable to receive status message", "error", fmt.Sprintf("%+v", err))
		return
	}

	// Ensure update is in the env required, otherwise, ack without taking action
	update := runReceipt.Run
	if update != nil {
		//
		// Relies on the reserved env var, FLOTILLA_SERVER_MODE to ensure update
		// belongs to -this- mode of Flotilla
		//
		var serverMode string
		if update.Env != nil {
			for _, kv := range *update.Env {
				if kv.Name == "FLOTILLA_SERVER_MODE" {
					serverMode = kv.Value
				}
			}
		}

		shouldProcess := len(serverMode) > 0 && serverMode == sw.conf.GetString("flotilla_mode")
		if shouldProcess {
			run, err := sw.findRun(update.TaskArn)
			if err != nil {
				sw.log.Log("message", "unable to find run to apply update to", "error", fmt.Sprintf("%+v", err))
				return
			}

			_, err = sw.sm.UpdateRun(run.RunID, *update)
			if err != nil {
				sw.log.Log("message", "error applying status update", "run", run.RunID, "error", fmt.Sprintf("%+v", err))
				return
			}

			// emit status update event
			sw.logStatusUpdate(*update)
		}

		if sw.conf.GetString("flotilla_mode") != "dev" {
			_ = sw.log.Log("message", "Acking status update", "arn", update.TaskArn)
		}
		if err = runReceipt.Done(); err != nil {
			sw.log.Log("message", "Acking status update failed", "arn", update.TaskArn, "error", fmt.Sprintf("%+v", err))
		}
	}
}

func (sw *statusWorker) logStatusUpdate(update state.Run) {
	var err error
	var startedAt, finishedAt time.Time
	var env state.EnvList

	if update.StartedAt != nil {
		startedAt = *update.StartedAt
	}

	if update.FinishedAt != nil {
		finishedAt = *update.FinishedAt
	}

	if update.Env != nil {
		env = *update.Env
	}

	if update.ExitCode != nil {
		err = sw.log.Event("eventClassName", "FlotillaTaskStatus",
			"run_id", update.RunID,
			"task_arn", update.TaskArn,
			"definition_id", update.DefinitionID,
			"alias", update.Alias,
			"image", update.Image,
			"cluster_name", update.ClusterName,
			"exit_code", *update.ExitCode,
			"status", update.Status,
			"started_at", startedAt,
			"finished_at", finishedAt,
			"instance_id", update.InstanceID,
			"instance_dns_name", update.InstanceDNSName,
			"group_name", update.GroupName,
			"user", update.User,
			"task_type", update.TaskType,
			"env", env,
			"executable_id", update.ExecutableID,
			"executable_type", update.ExecutableType)
	} else {
		err = sw.log.Event("eventClassName", "FlotillaTaskStatus",
			"run_id", update.RunID,
			"task_arn", update.TaskArn,
			"definition_id", update.DefinitionID,
			"alias", update.Alias,
			"image", update.Image,
			"cluster_name", update.ClusterName,
			"status", update.Status,
			"started_at", startedAt,
			"finished_at", finishedAt,
			"instance_id", update.InstanceID,
			"instance_dns_name", update.InstanceDNSName,
			"group_name", update.GroupName,
			"user", update.User,
			"task_type", update.TaskType,
			"env", env,
			"executable_id", update.ExecutableID,
			"executable_type", update.ExecutableType)
	}

	if err != nil {
		sw.log.Log("message", "Failed to emit status event", "run_id", update.RunID, "error", err.Error())
	}
}

func (sw *statusWorker) findRun(taskArn string) (state.Run, error) {
	var engines []string
	if sw.engine != nil {
		engines = []string{*sw.engine}
	} else {
		engines = nil
	}

	runs, err := sw.sm.ListRuns(1, 0, "started_at", "asc", map[string][]string{
		"task_arn": {taskArn},
	}, nil, engines)
	if err != nil {
		return state.Run{}, errors.Wrapf(err, "problem finding run by task arn [%s]", taskArn)
	}
	if runs.Total > 0 && len(runs.Runs) > 0 {
		return runs.Runs[0], nil
	}
	return state.Run{}, errors.Errorf("no run found for [%s]", taskArn)
}
<commit_msg>make list runs based on started_at time<commit_after>package worker

import (
	"fmt"
	"github.com/go-redis/redis"
	"github.com/stitchfix/flotilla-os/queue"
	"math/rand"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/stitchfix/flotilla-os/config"
	"github.com/stitchfix/flotilla-os/execution/engine"
	flotillaLog "github.com/stitchfix/flotilla-os/log"
	"github.com/stitchfix/flotilla-os/state"
	"gopkg.in/tomb.v2"
)

type statusWorker struct {
	sm           state.Manager
	ee           engine.Engine
	conf         config.Config
	log          flotillaLog.Logger
	pollInterval time.Duration
	t            tomb.Tomb
	engine       *string
	redisClient  *redis.Client
	workerId     string
}

func (sw *statusWorker) Initialize(conf config.Config, sm state.Manager, ee engine.Engine, log flotillaLog.Logger, pollInterval time.Duration, engine *string, qm queue.Manager) error {
	sw.pollInterval = pollInterval
	sw.conf = conf
	sw.sm = sm
	sw.ee = ee
	sw.log = log
	sw.engine = engine
	sw.workerId = fmt.Sprintf("%d", rand.Int())
	sw.setupRedisClient(conf)
	_ = sw.log.Log("message", "initialized a status worker", "engine", *engine)
	return nil
}

func (sw *statusWorker) setupRedisClient(conf config.Config) {
	if *sw.engine == state.EKSEngine {
		sw.redisClient = redis.NewClient(&redis.Options{Addr: conf.GetString("redis_address"), DB: conf.GetInt("redis_db")})
	}
}

func (sw *statusWorker) GetTomb() *tomb.Tomb {
	return &sw.t
}

//
// Run updates status of tasks
//
func (sw *statusWorker) Run() error {
	for {
		select {
		case <-sw.t.Dying():
			sw.log.Log("message", "A status worker was terminated")
			return nil
		default:
			if *sw.engine == state.ECSEngine {
				sw.runOnceECS()
				time.Sleep(sw.pollInterval)
			}

			if *sw.engine == state.EKSEngine {
				sw.runOnceEKS()
				time.Sleep(sw.pollInterval)
			}
		}
	}
}

func (sw *statusWorker) runOnceEKS() {
	rl, err := sw.sm.ListRuns(1000, 0, "started_at", "asc", map[string][]string{
		"queued_at_since": {
			time.Now().AddDate(0, 0, -3).Format(time.RFC3339),
		},
		"status": {state.StatusNeedsRetry, state.StatusRunning, state.StatusQueued, state.StatusPending},
	}, nil, []string{state.EKSEngine})

	if err != nil {
		_ = sw.log.Log("message", "unable to receive runs", "error", fmt.Sprintf("%+v", err))
		return
	}
	runs := rl.Runs
	sw.processEKSRuns(runs)
}

func (sw *statusWorker) processEKSRuns(runs []state.Run) {
	for _, run := range runs {
		reloadRun, err := sw.sm.GetRun(run.RunID)
		if err == nil && reloadRun.Status != state.StatusStopped {
			if sw.acquireLock(run, "status", 10*time.Second) == true {
				sw.processEKSRun(run)
				sw.processEKSRunMetrics(run)
			}
		}
	}
}
func (sw *statusWorker) acquireLock(run state.Run, purpose string, expiration time.Duration) bool {
	set, err := sw.redisClient.SetNX(fmt.Sprintf("%s-%s", run.RunID, purpose), sw.workerId, expiration).Result()
	if err != nil {
		// Turn off in dev mode; too noisy.
		if sw.conf.GetString("flotilla_mode") != "dev" {
			_ = sw.log.Log("message", "unable to set lock", "error", fmt.Sprintf("%+v", err))
		}
		return false
	}
	return set
}

func (sw *statusWorker) processEKSRun(run state.Run) {
	reloadRun, err := sw.sm.GetRun(run.RunID)
	if err == nil && reloadRun.Status == state.StatusStopped {
		// Run was updated by another worker process.
		return
	}
	updatedRun, err := sw.ee.FetchUpdateStatus(run)
	if err != nil {
		message := fmt.Sprintf("%+v", err)
		_ = sw.log.Log("message", "unable to receive eks runs", "error", message)

		minutesInQueue := time.Now().Sub(*run.QueuedAt).Minutes()
		if strings.Contains(message, "not found") && minutesInQueue > float64(30) {
			stoppedAt := time.Now()
			reason := "Job either timed out or not found on the EKS cluster."
			updatedRun.Status = state.StatusStopped
			updatedRun.FinishedAt = &stoppedAt
			updatedRun.ExitReason = &reason
			_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
		}

	} else {
		if run.Status != updatedRun.Status {
			_ = sw.log.Log("message", "updating eks run", "run", updatedRun.RunID, "status", updatedRun.Status)
			_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
			if err != nil {
				_ = sw.log.Log("message", "unable to save eks runs", "error", fmt.Sprintf("%+v", err))
			}

			if updatedRun.Status == state.StatusStopped {
				//TODO - move to a separate worker.
				//_ = sw.ee.Terminate(run)
			}
		} else {
			if updatedRun.MaxMemoryUsed != run.MaxMemoryUsed ||
				updatedRun.MaxCpuUsed != run.MaxCpuUsed ||
				updatedRun.Cpu != run.Cpu ||
				updatedRun.Memory != run.Memory ||
				updatedRun.PodEvents != run.PodEvents {
				_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
			}
		}
	}
}

func (sw *statusWorker) processEKSRunMetrics(run state.Run) {
	updatedRun, err := sw.ee.FetchPodMetrics(run)
	if err == nil {
		if updatedRun.MaxMemoryUsed != run.MaxMemoryUsed ||
			updatedRun.MaxCpuUsed != run.MaxCpuUsed {
			_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
		}
	}
}

func (sw *statusWorker) runOnceECS() {
	runReceipt, err := sw.ee.PollStatus()
	if err != nil {
		sw.log.Log("message", "unable to receive status message", "error", fmt.Sprintf("%+v", err))
		return
	}

	// Ensure update is in the env required, otherwise, ack without taking action
	update := runReceipt.Run
	if update != nil {
		//
		// Relies on the reserved env var, FLOTILLA_SERVER_MODE to ensure update
		// belongs to -this- mode of Flotilla
		//
		var serverMode string
		if update.Env != nil {
			for _, kv := range *update.Env {
				if kv.Name == "FLOTILLA_SERVER_MODE" {
					serverMode = kv.Value
				}
			}
		}

		shouldProcess := len(serverMode) > 0 && serverMode == sw.conf.GetString("flotilla_mode")
		if shouldProcess {
			run, err := sw.findRun(update.TaskArn)
			if err != nil {
				sw.log.Log("message", "unable to find run to apply update to", "error", fmt.Sprintf("%+v", err))
				return
			}

			_, err = sw.sm.UpdateRun(run.RunID, *update)
			if err != nil {
				sw.log.Log("message", "error applying status update", "run", run.RunID, "error", fmt.Sprintf("%+v", err))
				return
			}

			// emit status update event
			sw.logStatusUpdate(*update)
		}

		if sw.conf.GetString("flotilla_mode") != "dev" {
			_ = sw.log.Log("message", "Acking status update", "arn", update.TaskArn)
		}
		if err = runReceipt.Done(); err != nil {
			sw.log.Log("message", "Acking status update failed", "arn", update.TaskArn, "error", fmt.Sprintf("%+v", err))
		}
	}
}

func (sw *statusWorker) logStatusUpdate(update state.Run) {
	var err error
	var startedAt, finishedAt time.Time
	var env state.EnvList

	if update.StartedAt != nil {
		startedAt = *update.StartedAt
	}

	if update.FinishedAt != nil {
		finishedAt = *update.FinishedAt
	}

	if update.Env != nil {
		env = *update.Env
	}

	if update.ExitCode != nil {
		err = sw.log.Event("eventClassName", "FlotillaTaskStatus",
			"run_id", update.RunID,
			"task_arn", update.TaskArn,
			"definition_id", update.DefinitionID,
			"alias", update.Alias,
			"image", update.Image,
			"cluster_name", update.ClusterName,
			"exit_code", *update.ExitCode,
			"status", update.Status,
			"started_at", startedAt,
			"finished_at", finishedAt,
			"instance_id", update.InstanceID,
			"instance_dns_name", update.InstanceDNSName,
			"group_name", update.GroupName,
			"user", update.User,
			"task_type", update.TaskType,
			"env", env,
			"executable_id", update.ExecutableID,
			"executable_type", update.ExecutableType)
	} else {
		err = sw.log.Event("eventClassName", "FlotillaTaskStatus",
			"run_id", update.RunID,
			"task_arn", update.TaskArn,
			"definition_id", update.DefinitionID,
			"alias", update.Alias,
			"image", update.Image,
			"cluster_name", update.ClusterName,
			"status", update.Status,
			"started_at", startedAt,
			"finished_at", finishedAt,
			"instance_id", update.InstanceID,
			"instance_dns_name", update.InstanceDNSName,
			"group_name", update.GroupName,
			"user", update.User,
			"task_type", update.TaskType,
			"env", env,
			"executable_id", update.ExecutableID,
			"executable_type", update.ExecutableType)
	}

	if err != nil {
		sw.log.Log("message", "Failed to emit status event", "run_id", update.RunID, "error", err.Error())
	}
}

func (sw *statusWorker) findRun(taskArn string) (state.Run, error) {
	var engines []string
	if sw.engine != nil {
		engines = []string{*sw.engine}
	} else {
		engines = nil
	}

	runs, err := sw.sm.ListRuns(1, 0, "started_at", "asc", map[string][]string{
		"task_arn": {taskArn},
	}, nil, engines)
	if err != nil {
		return state.Run{}, errors.Wrapf(err, "problem finding run by task arn [%s]", taskArn)
	}
	if runs.Total > 0 && len(runs.Runs) > 0 {
		return runs.Runs[0], nil
	}
	return state.Run{}, errors.Errorf("no run found for [%s]", taskArn)
}
<|endoftext|>


<commit_before>package main

import (
	log "github.com/Sirupsen/logrus"

	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

type UsersResource struct {
	Data []User `json:"data"`
}

type UserResource struct {
	Data User `json:"data"`
}

// Structure representing error
type errorResponse struct {
	Msg string `json:"msg"`
}

func responseDetailsFromMongoError(error interface{}) (content errorResponse, code int) {
	content = errorResponse{Msg: fmt.Sprint(error)}
	code = 400
	if content.Msg == "not found" {
		code = 404
	}
	return content, code
}

// write json response to http response
func writeJsonResponse(w http.ResponseWriter, content *[]byte, code int) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(code)
	w.Write(*content)
}

// addUserHandler used to add new user
func (h *HTTPClientHandler) addUserHandler(w http.ResponseWriter, r *http.Request) {
	// adding new user to database
	var userRequest User

	defer r.Body.Close()
	body, err := ioutil.ReadAll(r.Body)

	if err != nil {
		// failed to read response body
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Could not read response body!")
		http.Error(w, "Failed to read request body.", 400)
		return
	}

	err = json.Unmarshal(body, &userRequest)

	if err != nil {
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(422) // can't process this entity
		return
	}

	log.WithFields(log.Fields{
		"firstName":     userRequest.FirstName,
		"lastName":      userRequest.LastName,
		"userID":        userRequest.UserID,
		"profilePicUrl": userRequest.ProfilePicUrl,
		"gender":        userRequest.Gender,
		"body":          string(body),
	}).Info("Got user info")

	// adding user
	err = h.db.addUser(userRequest)

	if err == nil {
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(201) // user inserted
		return
	} else {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Warn("Failed to insert..")

		content, code := responseDetailsFromMongoError(err)

		// Marshal provided interface into JSON structure
		uj, _ := json.Marshal(content)

		// Write content-type, statuscode, payload
		writeJsonResponse(w, &uj, code)

	}

}

// getAllUsersHandler used to get all users
func (h *HTTPClientHandler) getAllUsersHandler(w http.ResponseWriter, r *http.Request) {

	userid, _ := r.URL.Query()["q"]
	// looking for specific user
	if len(userid) > 0 {
		log.WithFields(log.Fields{
			"userid": userid[0],
		}).Info("Looking for user..")

		user, err := h.db.getUser(userid[0])

		if err == nil {
			// Marshal provided interface into JSON structure
			response := UserResource{Data: user}
			uj, _ := json.Marshal(response)

			// Write content-type, statuscode, payload
			w.Header().Set("Content-Type", "application/json")
			w.WriteHeader(200)
			fmt.Fprintf(w, "%s", uj)
			return
		} else {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Warn("Failed to insert..")

			content, code := responseDetailsFromMongoError(err)

			// Marshal provided interface into JSON structure
			uj, _ := json.Marshal(content)

			// Write content-type, statuscode, payload
			writeJsonResponse(w, &uj, code)
			return

		}
	}

	log.Warn(len(userid))
	// displaying all users
	results, err := h.db.getUsers()
	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Got error when tried to get all users")
	}

	log.WithFields(log.Fields{
		"count": len(results),
	}).Info("number of users")

	// Marshal provided interface into JSON structure
	response := UsersResource{Data: results}
	uj, _ := json.Marshal(response)

	// Write content-type, statuscode, payload
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(200)
	fmt.Fprintf(w, "%s", uj)
}

func (h *HTTPClientHandler) updateUserHandler(w http.ResponseWriter, r *http.Request) {

}

// addPlaceHandler add new hosting place, provide json
func (h *HTTPClientHandler) addPlaceHandler(w http.ResponseWriter, r *http.Request) {
	// adding new hosting place to database
	var hostingPlaceRequest HostingPlace

	defer r.Body.Close()
	body, err := ioutil.ReadAll(r.Body)

	if err != nil {
		// failed to read response body
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Could not read response body!")
		http.Error(w, "Failed to read request body.", 400)
		return
	}

	err = json.Unmarshal(body, &hostingPlaceRequest)

	err = h.db.addHostingPlace(hostingPlaceRequest)

	if err == nil {
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(201) // place inserted
		return
	} else {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Warn("Failed to insert hosting place..")

		content, code := responseDetailsFromMongoError(err)

		// Marshal provided interface into JSON structure
		uj, _ := json.Marshal(content)

		// Write content-type, statuscode, payload
		writeJsonResponse(w, &uj, code)

	}

}
<commit_msg>logging, mongo is not the best database<commit_after>package main

import (
	log "github.com/Sirupsen/logrus"

	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

type UsersResource struct {
	Data []User `json:"data"`
}

type UserResource struct {
	Data User `json:"data"`
}

// Structure representing error
type errorResponse struct {
	Msg string `json:"msg"`
}

func responseDetailsFromMongoError(error interface{}) (content errorResponse, code int) {
	content = errorResponse{Msg: fmt.Sprint(error)}
	code = 400
	if content.Msg == "not found" {
		code = 404
	}
	return content, code
}

// write json response to http response
func writeJsonResponse(w http.ResponseWriter, content *[]byte, code int) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(code)
	w.Write(*content)
}

// addUserHandler used to add new user
func (h *HTTPClientHandler) addUserHandler(w http.ResponseWriter, r *http.Request) {
	// adding new user to database
	var userRequest User

	defer r.Body.Close()
	body, err := ioutil.ReadAll(r.Body)

	if err != nil {
		// failed to read response body
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Could not read response body!")
		http.Error(w, "Failed to read request body.", 400)
		return
	}

	err = json.Unmarshal(body, &userRequest)

	if err != nil {
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(422) // can't process this entity
		return
	}

	log.WithFields(log.Fields{
		"firstName":     userRequest.FirstName,
		"lastName":      userRequest.LastName,
		"userID":        userRequest.UserID,
		"profilePicUrl": userRequest.ProfilePicUrl,
		"gender":        userRequest.Gender,
		"body":          string(body),
	}).Info("Got user info")

	// adding user
	err = h.db.addUser(userRequest)

	if err == nil {
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(201) // user inserted
		return
	} else {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Warn("Failed to insert..")

		content, code := responseDetailsFromMongoError(err)

		// Marshal provided interface into JSON structure
		uj, _ := json.Marshal(content)

		// Write content-type, statuscode, payload
		writeJsonResponse(w, &uj, code)

	}

}

// getAllUsersHandler used to get all users
func (h *HTTPClientHandler) getAllUsersHandler(w http.ResponseWriter, r *http.Request) {

	userid, _ := r.URL.Query()["q"]
	// looking for specific user
	if len(userid) > 0 {
		log.WithFields(log.Fields{
			"userid": userid[0],
		}).Info("Looking for user..")

		user, err := h.db.getUser(userid[0])

		if err == nil {
			// Marshal provided interface into JSON structure
			response := UserResource{Data: user}
			uj, _ := json.Marshal(response)

			// Write content-type, statuscode, payload
			w.Header().Set("Content-Type", "application/json")
			w.WriteHeader(200)
			fmt.Fprintf(w, "%s", uj)
			return
		} else {
			log.WithFields(log.Fields{
				"error": err.Error(),
			}).Warn("Failed to insert..")

			content, code := responseDetailsFromMongoError(err)

			// Marshal provided interface into JSON structure
			uj, _ := json.Marshal(content)

			// Write content-type, statuscode, payload
			writeJsonResponse(w, &uj, code)
			return

		}
	}

	log.Warn(len(userid))
	// displaying all users
	results, err := h.db.getUsers()
	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Got error when tried to get all users")
	}

	log.WithFields(log.Fields{
		"count": len(results),
	}).Info("number of users")

	// Marshal provided interface into JSON structure
	response := UsersResource{Data: results}
	uj, _ := json.Marshal(response)

	// Write content-type, statuscode, payload
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(200)
	fmt.Fprintf(w, "%s", uj)
}

// addPlaceHandler add new hosting place, provide json
func (h *HTTPClientHandler) addPlaceHandler(w http.ResponseWriter, r *http.Request) {
	// adding new hosting place to database
	var hostingPlaceRequest HostingPlace
	log.Info("adding place........")
	defer r.Body.Close()
	body, err := ioutil.ReadAll(r.Body)

	if err != nil {
		// failed to read response body
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Could not read response body!")
		http.Error(w, "Failed to read request body.", 400)
		return
	}

	err = json.Unmarshal(body, &hostingPlaceRequest)

	log.WithFields(log.Fields{
		"body":   string(body),
		"host":   hostingPlaceRequest.Host,
		"active": hostingPlaceRequest.Active,
		"lat":    hostingPlaceRequest.Lat,
		"long":   hostingPlaceRequest.Long,
	}).Info("Got place info")

	err = h.db.addHostingPlace(hostingPlaceRequest)

	if err == nil {
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(201) // place inserted
		return
	} else {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Warn("Failed to insert hosting place..")

		content, code := responseDetailsFromMongoError(err)

		// Marshal provided interface into JSON structure
		uj, _ := json.Marshal(content)

		// Write content-type, statuscode, payload
		writeJsonResponse(w, &uj, code)

	}

}

func (h *HTTPClientHandler) getPlaceHandler(w http.ResponseWriter, r *http.Request) {

}
<|endoftext|>


<commit_before>package worker

import (
	"fmt"
	"github.com/go-redis/redis"
	"math/rand"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/stitchfix/flotilla-os/config"
	"github.com/stitchfix/flotilla-os/execution/engine"
	flotillaLog "github.com/stitchfix/flotilla-os/log"
	"github.com/stitchfix/flotilla-os/state"
	"gopkg.in/tomb.v2"
)

type statusWorker struct {
	sm           state.Manager
	ee           engine.Engine
	conf         config.Config
	log          flotillaLog.Logger
	pollInterval time.Duration
	t            tomb.Tomb
	engine       *string
	redisClient  *redis.Client
	workerId     string
}

func (sw *statusWorker) Initialize(conf config.Config, sm state.Manager, ee engine.Engine, log flotillaLog.Logger, pollInterval time.Duration, engine *string) error {
	sw.pollInterval = pollInterval
	sw.conf = conf
	sw.sm = sm
	sw.ee = ee
	sw.log = log
	sw.engine = engine
	sw.workerId = fmt.Sprintf("%d", rand.Int())
	sw.setupRedisClient(conf)
	_ = sw.log.Log("message", "initialized a status worker", "engine", *engine)
	return nil
}

func (sw *statusWorker) setupRedisClient(conf config.Config) {
	if *sw.engine == state.EKSEngine {
		sw.redisClient = redis.NewClient(&redis.Options{Addr: conf.GetString("redis_address"), DB: conf.GetInt("redis_db")})
	}
}

func (sw *statusWorker) GetTomb() *tomb.Tomb {
	return &sw.t
}

//
// Run updates status of tasks
//
func (sw *statusWorker) Run() error {
	for {
		select {
		case <-sw.t.Dying():
			sw.log.Log("message", "A status worker was terminated")
			return nil
		default:
			if *sw.engine == state.ECSEngine {
				sw.runOnceECS()
				time.Sleep(sw.pollInterval)
			}

			if *sw.engine == state.EKSEngine {
				sw.runOnceEKS()
				time.Sleep(sw.pollInterval)
			}
		}
	}
}

func (sw *statusWorker) runOnceEKS() {
	rl, err := sw.sm.ListRuns(1000, 0, "status", "asc", map[string][]string{
		"queued_at_since": {
			time.Now().AddDate(0, 0, -30).Format(time.RFC3339),
		},
		"status": {state.StatusNeedsRetry, state.StatusRunning, state.StatusQueued, state.StatusPending},
	}, nil, []string{state.EKSEngine})

	if err != nil {
		_ = sw.log.Log("message", "unable to receive runs", "error", fmt.Sprintf("%+v", err))
		return
	}
	sw.processRuns(rl.Runs)
}

func (sw *statusWorker) processRuns(runs []state.Run) {
	for _, run := range runs {
		if sw.acquireLock(run, "status", 15*time.Second) == true {
			sw.processRun(run)

		}

		if sw.acquireLock(run, "metrics", 5*time.Second) == true {
			sw.processRunMetrics(run)
		}
	}
}
func (sw *statusWorker) acquireLock(run state.Run, purpose string, expiration time.Duration) bool {
	set, err := sw.redisClient.SetNX(fmt.Sprintf("%s-%s", run.RunID, purpose), sw.workerId, expiration).Result()
	if err != nil {
		_ = sw.log.Log("message", "unable to set lock", "error", fmt.Sprintf("%+v", err))
		return false
	}
	return set
}

func (sw *statusWorker) processRun(run state.Run) {
	reloadRun, err := sw.sm.GetRun(run.RunID)
	if err == nil && reloadRun.Status == state.StatusStopped {
		// Run was updated by another worker process.
		return
	}
	updatedRun, err := sw.ee.FetchUpdateStatus(run)
	if err != nil {
		message := fmt.Sprintf("%+v", err)
		_ = sw.log.Log("message", "unable to receive eks runs", "error", message)

		minutesInQueue := time.Now().Sub(*run.QueuedAt).Minutes()
		if strings.Contains(message, "not found") && minutesInQueue > float64(30) {
			stoppedAt := time.Now()
			reason := "Job either timed out or not found on the EKS cluster."
			updatedRun.Status = state.StatusStopped
			updatedRun.FinishedAt = &stoppedAt
			updatedRun.ExitReason = &reason
			_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
		}

	} else {
		if run.Status != updatedRun.Status {
			_ = sw.log.Log("message", "updating eks run", "run", updatedRun.RunID, "status", updatedRun.Status)
			_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
			if err != nil {
				_ = sw.log.Log("message", "unable to save eks runs", "error", fmt.Sprintf("%+v", err))
			}

			if updatedRun.Status == state.StatusStopped {
				//TODO - move to a separate worker.
				//_ = sw.ee.Terminate(run)
			}
		} else {
			if updatedRun.MaxMemoryUsed != run.MaxMemoryUsed ||
				updatedRun.MaxCpuUsed != run.MaxCpuUsed ||
				updatedRun.Cpu != run.Cpu ||
				updatedRun.Memory != run.Memory ||
				updatedRun.PodEvents != run.PodEvents {
				_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
			}
		}
	}
}

func (sw *statusWorker) processRunMetrics(run state.Run) {
	updatedRun, err := sw.ee.FetchPodMetrics(run)
	if err == nil {
		if updatedRun.MaxMemoryUsed != run.MaxMemoryUsed ||
			updatedRun.MaxCpuUsed != run.MaxCpuUsed {
			_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
		}
	}
}

func (sw *statusWorker) runOnceECS() {
	runReceipt, err := sw.ee.PollStatus()
	if err != nil {
		sw.log.Log("message", "unable to receive status message", "error", fmt.Sprintf("%+v", err))
		return
	}

	// Ensure update is in the env required, otherwise, ack without taking action
	update := runReceipt.Run
	if update != nil {
		//
		// Relies on the reserved env var, FLOTILLA_SERVER_MODE to ensure update
		// belongs to -this- mode of Flotilla
		//
		var serverMode string
		if update.Env != nil {
			for _, kv := range *update.Env {
				if kv.Name == "FLOTILLA_SERVER_MODE" {
					serverMode = kv.Value
				}
			}
		}

		shouldProcess := len(serverMode) > 0 && serverMode == sw.conf.GetString("flotilla_mode")
		if shouldProcess {
			run, err := sw.findRun(update.TaskArn)
			if err != nil {
				sw.log.Log("message", "unable to find run to apply update to", "error", fmt.Sprintf("%+v", err))
				return
			}

			_, err = sw.sm.UpdateRun(run.RunID, *update)
			if err != nil {
				sw.log.Log("message", "error applying status update", "run", run.RunID, "error", fmt.Sprintf("%+v", err))
				return
			}

			// emit status update event
			sw.logStatusUpdate(*update)
		}

		sw.log.Log("message", "Acking status update", "arn", update.TaskArn)
		if err = runReceipt.Done(); err != nil {
			sw.log.Log("message", "Acking status update failed", "arn", update.TaskArn, "error", fmt.Sprintf("%+v", err))
		}
	}
}

func (sw *statusWorker) logStatusUpdate(update state.Run) {
	var err error
	var startedAt, finishedAt time.Time
	var env state.EnvList

	if update.StartedAt != nil {
		startedAt = *update.StartedAt
	}

	if update.FinishedAt != nil {
		finishedAt = *update.FinishedAt
	}

	if update.Env != nil {
		env = *update.Env
	}

	if update.ExitCode != nil {
		err = sw.log.Event("eventClassName", "FlotillaTaskStatus",
			"run_id", update.RunID,
			"task_arn", update.TaskArn,
			"definition_id", update.DefinitionID,
			"alias", update.Alias,
			"image", update.Image,
			"cluster_name", update.ClusterName,
			"exit_code", *update.ExitCode,
			"status", update.Status,
			"started_at", startedAt,
			"finished_at", finishedAt,
			"instance_id", update.InstanceID,
			"instance_dns_name", update.InstanceDNSName,
			"group_name", update.GroupName,
			"user", update.User,
			"task_type", update.TaskType,
			"env", env)
	} else {
		err = sw.log.Event("eventClassName", "FlotillaTaskStatus",
			"run_id", update.RunID,
			"task_arn", update.TaskArn,
			"definition_id", update.DefinitionID,
			"alias", update.Alias,
			"image", update.Image,
			"cluster_name", update.ClusterName,
			"status", update.Status,
			"started_at", startedAt,
			"finished_at", finishedAt,
			"instance_id", update.InstanceID,
			"instance_dns_name", update.InstanceDNSName,
			"group_name", update.GroupName,
			"user", update.User,
			"task_type", update.TaskType,
			"env", env)
	}

	if err != nil {
		sw.log.Log("message", "Failed to emit status event", "run_id", update.RunID, "error", err.Error())
	}
}

func (sw *statusWorker) findRun(taskArn string) (state.Run, error) {
	var engines []string
	if sw.engine != nil {
		engines = []string{*sw.engine}
	} else {
		engines = nil
	}

	runs, err := sw.sm.ListRuns(1, 0, "started_at", "asc", map[string][]string{
		"task_arn": {taskArn},
	}, nil, engines)
	if err != nil {
		return state.Run{}, errors.Wrapf(err, "problem finding run by task arn [%s]", taskArn)
	}
	if runs.Total > 0 && len(runs.Runs) > 0 {
		return runs.Runs[0], nil
	}
	return state.Run{}, errors.Errorf("no run found for [%s]", taskArn)
}
<commit_msg>shuffle order in which each worker processes it's run objects (#280)<commit_after>package worker

import (
	"fmt"
	"github.com/go-redis/redis"
	"math/rand"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/stitchfix/flotilla-os/config"
	"github.com/stitchfix/flotilla-os/execution/engine"
	flotillaLog "github.com/stitchfix/flotilla-os/log"
	"github.com/stitchfix/flotilla-os/state"
	"gopkg.in/tomb.v2"
)

type statusWorker struct {
	sm           state.Manager
	ee           engine.Engine
	conf         config.Config
	log          flotillaLog.Logger
	pollInterval time.Duration
	t            tomb.Tomb
	engine       *string
	redisClient  *redis.Client
	workerId     string
}

func (sw *statusWorker) Initialize(conf config.Config, sm state.Manager, ee engine.Engine, log flotillaLog.Logger, pollInterval time.Duration, engine *string) error {
	sw.pollInterval = pollInterval
	sw.conf = conf
	sw.sm = sm
	sw.ee = ee
	sw.log = log
	sw.engine = engine
	sw.workerId = fmt.Sprintf("%d", rand.Int())
	sw.setupRedisClient(conf)
	_ = sw.log.Log("message", "initialized a status worker", "engine", *engine)
	return nil
}

func (sw *statusWorker) setupRedisClient(conf config.Config) {
	if *sw.engine == state.EKSEngine {
		sw.redisClient = redis.NewClient(&redis.Options{Addr: conf.GetString("redis_address"), DB: conf.GetInt("redis_db")})
	}
}

func (sw *statusWorker) GetTomb() *tomb.Tomb {
	return &sw.t
}

//
// Run updates status of tasks
//
func (sw *statusWorker) Run() error {
	for {
		select {
		case <-sw.t.Dying():
			sw.log.Log("message", "A status worker was terminated")
			return nil
		default:
			if *sw.engine == state.ECSEngine {
				sw.runOnceECS()
				time.Sleep(sw.pollInterval)
			}

			if *sw.engine == state.EKSEngine {
				sw.runOnceEKS()
				time.Sleep(sw.pollInterval)
			}
		}
	}
}

func (sw *statusWorker) runOnceEKS() {
	rl, err := sw.sm.ListRuns(1000, 0, "status", "asc", map[string][]string{
		"queued_at_since": {
			time.Now().AddDate(0, 0, -30).Format(time.RFC3339),
		},
		"status": {state.StatusNeedsRetry, state.StatusRunning, state.StatusQueued, state.StatusPending},
	}, nil, []string{state.EKSEngine})

	if err != nil {
		_ = sw.log.Log("message", "unable to receive runs", "error", fmt.Sprintf("%+v", err))
		return
	}
	runs := rl.Runs

	rand.Seed(time.Now().UnixNano())
	rand.Shuffle(len(runs), func(i, j int) { runs[i], runs[j] = runs[j], runs[i] })
	sw.processRuns(runs)
}

func (sw *statusWorker) processRuns(runs []state.Run) {
	for _, run := range runs {
		if sw.acquireLock(run, "status", 10*time.Second) == true {
			sw.processRun(run)
			sw.processRunMetrics(run)
		}
	}
}
func (sw *statusWorker) acquireLock(run state.Run, purpose string, expiration time.Duration) bool {
	set, err := sw.redisClient.SetNX(fmt.Sprintf("%s-%s", run.RunID, purpose), sw.workerId, expiration).Result()
	if err != nil {
		_ = sw.log.Log("message", "unable to set lock", "error", fmt.Sprintf("%+v", err))
		return false
	}
	return set
}

func (sw *statusWorker) processRun(run state.Run) {
	reloadRun, err := sw.sm.GetRun(run.RunID)
	if err == nil && reloadRun.Status == state.StatusStopped {
		// Run was updated by another worker process.
		return
	}
	updatedRun, err := sw.ee.FetchUpdateStatus(run)
	if err != nil {
		message := fmt.Sprintf("%+v", err)
		_ = sw.log.Log("message", "unable to receive eks runs", "error", message)

		minutesInQueue := time.Now().Sub(*run.QueuedAt).Minutes()
		if strings.Contains(message, "not found") && minutesInQueue > float64(30) {
			stoppedAt := time.Now()
			reason := "Job either timed out or not found on the EKS cluster."
			updatedRun.Status = state.StatusStopped
			updatedRun.FinishedAt = &stoppedAt
			updatedRun.ExitReason = &reason
			_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
		}

	} else {
		if run.Status != updatedRun.Status {
			_ = sw.log.Log("message", "updating eks run", "run", updatedRun.RunID, "status", updatedRun.Status)
			_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
			if err != nil {
				_ = sw.log.Log("message", "unable to save eks runs", "error", fmt.Sprintf("%+v", err))
			}

			if updatedRun.Status == state.StatusStopped {
				//TODO - move to a separate worker.
				//_ = sw.ee.Terminate(run)
			}
		} else {
			if updatedRun.MaxMemoryUsed != run.MaxMemoryUsed ||
				updatedRun.MaxCpuUsed != run.MaxCpuUsed ||
				updatedRun.Cpu != run.Cpu ||
				updatedRun.Memory != run.Memory ||
				updatedRun.PodEvents != run.PodEvents {
				_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
			}
		}
	}
}

func (sw *statusWorker) processRunMetrics(run state.Run) {
	updatedRun, err := sw.ee.FetchPodMetrics(run)
	if err == nil {
		if updatedRun.MaxMemoryUsed != run.MaxMemoryUsed ||
			updatedRun.MaxCpuUsed != run.MaxCpuUsed {
			_, err = sw.sm.UpdateRun(updatedRun.RunID, updatedRun)
		}
	}
}

func (sw *statusWorker) runOnceECS() {
	runReceipt, err := sw.ee.PollStatus()
	if err != nil {
		sw.log.Log("message", "unable to receive status message", "error", fmt.Sprintf("%+v", err))
		return
	}

	// Ensure update is in the env required, otherwise, ack without taking action
	update := runReceipt.Run
	if update != nil {
		//
		// Relies on the reserved env var, FLOTILLA_SERVER_MODE to ensure update
		// belongs to -this- mode of Flotilla
		//
		var serverMode string
		if update.Env != nil {
			for _, kv := range *update.Env {
				if kv.Name == "FLOTILLA_SERVER_MODE" {
					serverMode = kv.Value
				}
			}
		}

		shouldProcess := len(serverMode) > 0 && serverMode == sw.conf.GetString("flotilla_mode")
		if shouldProcess {
			run, err := sw.findRun(update.TaskArn)
			if err != nil {
				sw.log.Log("message", "unable to find run to apply update to", "error", fmt.Sprintf("%+v", err))
				return
			}

			_, err = sw.sm.UpdateRun(run.RunID, *update)
			if err != nil {
				sw.log.Log("message", "error applying status update", "run", run.RunID, "error", fmt.Sprintf("%+v", err))
				return
			}

			// emit status update event
			sw.logStatusUpdate(*update)
		}

		sw.log.Log("message", "Acking status update", "arn", update.TaskArn)
		if err = runReceipt.Done(); err != nil {
			sw.log.Log("message", "Acking status update failed", "arn", update.TaskArn, "error", fmt.Sprintf("%+v", err))
		}
	}
}

func (sw *statusWorker) logStatusUpdate(update state.Run) {
	var err error
	var startedAt, finishedAt time.Time
	var env state.EnvList

	if update.StartedAt != nil {
		startedAt = *update.StartedAt
	}

	if update.FinishedAt != nil {
		finishedAt = *update.FinishedAt
	}

	if update.Env != nil {
		env = *update.Env
	}

	if update.ExitCode != nil {
		err = sw.log.Event("eventClassName", "FlotillaTaskStatus",
			"run_id", update.RunID,
			"task_arn", update.TaskArn,
			"definition_id", update.DefinitionID,
			"alias", update.Alias,
			"image", update.Image,
			"cluster_name", update.ClusterName,
			"exit_code", *update.ExitCode,
			"status", update.Status,
			"started_at", startedAt,
			"finished_at", finishedAt,
			"instance_id", update.InstanceID,
			"instance_dns_name", update.InstanceDNSName,
			"group_name", update.GroupName,
			"user", update.User,
			"task_type", update.TaskType,
			"env", env)
	} else {
		err = sw.log.Event("eventClassName", "FlotillaTaskStatus",
			"run_id", update.RunID,
			"task_arn", update.TaskArn,
			"definition_id", update.DefinitionID,
			"alias", update.Alias,
			"image", update.Image,
			"cluster_name", update.ClusterName,
			"status", update.Status,
			"started_at", startedAt,
			"finished_at", finishedAt,
			"instance_id", update.InstanceID,
			"instance_dns_name", update.InstanceDNSName,
			"group_name", update.GroupName,
			"user", update.User,
			"task_type", update.TaskType,
			"env", env)
	}

	if err != nil {
		sw.log.Log("message", "Failed to emit status event", "run_id", update.RunID, "error", err.Error())
	}
}

func (sw *statusWorker) findRun(taskArn string) (state.Run, error) {
	var engines []string
	if sw.engine != nil {
		engines = []string{*sw.engine}
	} else {
		engines = nil
	}

	runs, err := sw.sm.ListRuns(1, 0, "started_at", "asc", map[string][]string{
		"task_arn": {taskArn},
	}, nil, engines)
	if err != nil {
		return state.Run{}, errors.Wrapf(err, "problem finding run by task arn [%s]", taskArn)
	}
	if runs.Total > 0 && len(runs.Runs) > 0 {
		return runs.Runs[0], nil
	}
	return state.Run{}, errors.Errorf("no run found for [%s]", taskArn)
}
<|endoftext|>


<commit_before>package pigae

import (
	"fmt"
	"net/http"
	"strconv"
	"time"

	. "github.com/Deleplace/programming-idioms/pig"

	"appengine"
)

func implSave(w http.ResponseWriter, r *http.Request) error {
	idiomIDStr := r.FormValue("idiom_id")
	existingIDStr := r.FormValue("impl_id")
	username := r.FormValue("user_nickname")

	if !toggles["anonymousWrite"] {
		if username == "" {
			return PiError{"Username is mandatory. No anonymous edit.", http.StatusBadRequest}
		}
	}

	setNicknameCookie(w, username)

	if existingIDStr == "" {
		return newImplSave(w, r, username, idiomIDStr)
	}
	return existingImplSave(w, r, username, idiomIDStr, existingIDStr)
}

func newImplSave(w http.ResponseWriter, r *http.Request, username string, idiomIDStr string) error {
	if err := togglesMissing(w, r, "implAddition"); err != nil {
		return err
	}
	if err := parametersMissing(w, r, "impl_language"); err != nil {
		return err
	}

	c := appengine.NewContext(r)
	language := normLang(r.FormValue("impl_language"))
	imports := r.FormValue("impl_imports")
	code := r.FormValue("impl_code")
	comment := r.FormValue("impl_comment")
	attributionURL := r.FormValue("impl_attribution_url")
	demoURL := r.FormValue("impl_demo_url")
	docURL := r.FormValue("impl_doc_url")
	editSummary := fmt.Sprintf("New %v implementation by user [%v]", language, username)
	c.Infof("[%v] is creating new %v impl for idiom %v", username, language, idiomIDStr)

	if !StringSliceContains(allLanguages(), language) {
		return PiError{fmt.Sprintf("Sorry, [%v] is currently not a supported language. Supported languages are %v.", r.FormValue("impl_language"), allNiceLangs), http.StatusBadRequest}
	}

	idiomID := String2Int(idiomIDStr)
	if idiomID == -1 {
		return PiError{idiomIDStr + " is not a valid idiom id.", http.StatusBadRequest}
	}

	key, idiom, err := dao.getIdiom(c, idiomID)
	if err != nil {
		return PiError{"Could not find idiom " + idiomIDStr, http.StatusNotFound}
	}

	if err := validateURLFormatOrEmpty(attributionURL); err != nil {
		return PiError{"Can't accept URL [" + attributionURL + "]", http.StatusBadRequest}
	}

	if err := validateURLFormatOrEmpty(demoURL); err != nil {
		return PiError{"Can't accept URL [" + demoURL + "]", http.StatusBadRequest}
	}

	implID, err := dao.nextImplID(c)
	if err != nil {
		return err
	}
	now := time.Now()
	newImpl := Impl{
		Id:                     implID,
		OrigId:                 implID,
		Author:                 username,
		CreationDate:           now,
		LastEditor:             username,
		LanguageName:           language,
		ImportsBlock:           imports,
		CodeBlock:              code,
		AuthorComment:          comment,
		OriginalAttributionURL: attributionURL,
		DemoURL:                demoURL,
		DocumentationURL:       docURL,
		Version:                1,
		VersionDate:            now,
	}
	idiom.Implementations = append(idiom.Implementations, newImpl)
	idiom.EditSummary = editSummary
	idiom.LastEditedImplID = implID

	err = dao.saveExistingIdiom(c, key, idiom)
	if err != nil {
		return err
	}

	http.Redirect(w, r, NiceImplURL(idiom, implID, language), http.StatusFound)
	return nil
}

func existingImplSave(w http.ResponseWriter, r *http.Request, username string, idiomIDStr string, existingImplIDStr string) error {
	if err := togglesMissing(w, r, "implEditing"); err != nil {
		return err
	}
	if err := parametersMissing(w, r, "impl_version"); err != nil {
		return err
	}

	c := appengine.NewContext(r)
	imports := r.FormValue("impl_imports")
	code := r.FormValue("impl_code")
	comment := r.FormValue("impl_comment")
	attributionURL := r.FormValue("impl_attribution_url")
	demoURL := r.FormValue("impl_demo_url")
	docURL := r.FormValue("impl_doc_url")
	c.Infof("[%v] is updating impl %v of idiom %v", username, existingImplIDStr, idiomIDStr)

	idiomID := String2Int(idiomIDStr)
	if idiomID == -1 {
		return PiError{idiomIDStr + " is not a valid idiom id.", http.StatusBadRequest}
	}

	key, idiom, err := dao.getIdiom(c, idiomID)
	if err != nil {
		return PiError{"Could not find implementation " + existingImplIDStr + " for idiom " + idiomIDStr, http.StatusNotFound}
	}

	implID := String2Int(existingImplIDStr)
	if implID == -1 {
		return PiError{existingImplIDStr + " is not a valid implementation id.", http.StatusBadRequest}
	}

	_, impl, _ := idiom.FindImplInIdiom(implID)

	if r.FormValue("impl_version") != strconv.Itoa(impl.Version) {
		return PiError{fmt.Sprintf("Implementation has been concurrently modified (editing version %v, current version is %v)", r.FormValue("impl_version"), impl.Version), http.StatusConflict}
	}

	if err := validateURLFormatOrEmpty(attributionURL); err != nil {
		return PiError{"Can't accept URL [" + attributionURL + "]", http.StatusBadRequest}
	}

	if err := validateURLFormatOrEmpty(demoURL); err != nil {
		return PiError{"Can't accept URL [" + demoURL + "]", http.StatusBadRequest}
	}

	idiom.EditSummary = "[" + impl.LanguageName + "]" + r.FormValue("edit_summary")
	idiom.LastEditedImplID = implID
	impl.ImportsBlock = imports
	impl.CodeBlock = code
	impl.AuthorComment = comment
	impl.LastEditor = username
	impl.OriginalAttributionURL = attributionURL
	impl.DemoURL = demoURL
	impl.DocumentationURL = docURL
	impl.Version = impl.Version + 1
	impl.VersionDate = time.Now()

	err = dao.saveExistingIdiom(c, key, idiom)
	if err != nil {
		return err
	}

	http.Redirect(w, r, NiceImplURL(idiom, implID, impl.LanguageName), http.StatusFound)
	return nil
}
<commit_msg>Fixed missing space in EditSummary.<commit_after>package pigae

import (
	"fmt"
	"net/http"
	"strconv"
	"time"

	. "github.com/Deleplace/programming-idioms/pig"

	"appengine"
)

func implSave(w http.ResponseWriter, r *http.Request) error {
	idiomIDStr := r.FormValue("idiom_id")
	existingIDStr := r.FormValue("impl_id")
	username := r.FormValue("user_nickname")

	if !toggles["anonymousWrite"] {
		if username == "" {
			return PiError{"Username is mandatory. No anonymous edit.", http.StatusBadRequest}
		}
	}

	setNicknameCookie(w, username)

	if existingIDStr == "" {
		return newImplSave(w, r, username, idiomIDStr)
	}
	return existingImplSave(w, r, username, idiomIDStr, existingIDStr)
}

func newImplSave(w http.ResponseWriter, r *http.Request, username string, idiomIDStr string) error {
	if err := togglesMissing(w, r, "implAddition"); err != nil {
		return err
	}
	if err := parametersMissing(w, r, "impl_language"); err != nil {
		return err
	}

	c := appengine.NewContext(r)
	language := normLang(r.FormValue("impl_language"))
	imports := r.FormValue("impl_imports")
	code := r.FormValue("impl_code")
	comment := r.FormValue("impl_comment")
	attributionURL := r.FormValue("impl_attribution_url")
	demoURL := r.FormValue("impl_demo_url")
	docURL := r.FormValue("impl_doc_url")
	editSummary := fmt.Sprintf("New %v implementation by user [%v]", language, username)
	c.Infof("[%v] is creating new %v impl for idiom %v", username, language, idiomIDStr)

	if !StringSliceContains(allLanguages(), language) {
		return PiError{fmt.Sprintf("Sorry, [%v] is currently not a supported language. Supported languages are %v.", r.FormValue("impl_language"), allNiceLangs), http.StatusBadRequest}
	}

	idiomID := String2Int(idiomIDStr)
	if idiomID == -1 {
		return PiError{idiomIDStr + " is not a valid idiom id.", http.StatusBadRequest}
	}

	key, idiom, err := dao.getIdiom(c, idiomID)
	if err != nil {
		return PiError{"Could not find idiom " + idiomIDStr, http.StatusNotFound}
	}

	if err := validateURLFormatOrEmpty(attributionURL); err != nil {
		return PiError{"Can't accept URL [" + attributionURL + "]", http.StatusBadRequest}
	}

	if err := validateURLFormatOrEmpty(demoURL); err != nil {
		return PiError{"Can't accept URL [" + demoURL + "]", http.StatusBadRequest}
	}

	implID, err := dao.nextImplID(c)
	if err != nil {
		return err
	}
	now := time.Now()
	newImpl := Impl{
		Id:                     implID,
		OrigId:                 implID,
		Author:                 username,
		CreationDate:           now,
		LastEditor:             username,
		LanguageName:           language,
		ImportsBlock:           imports,
		CodeBlock:              code,
		AuthorComment:          comment,
		OriginalAttributionURL: attributionURL,
		DemoURL:                demoURL,
		DocumentationURL:       docURL,
		Version:                1,
		VersionDate:            now,
	}
	idiom.Implementations = append(idiom.Implementations, newImpl)
	idiom.EditSummary = editSummary
	idiom.LastEditedImplID = implID

	err = dao.saveExistingIdiom(c, key, idiom)
	if err != nil {
		return err
	}

	http.Redirect(w, r, NiceImplURL(idiom, implID, language), http.StatusFound)
	return nil
}

func existingImplSave(w http.ResponseWriter, r *http.Request, username string, idiomIDStr string, existingImplIDStr string) error {
	if err := togglesMissing(w, r, "implEditing"); err != nil {
		return err
	}
	if err := parametersMissing(w, r, "impl_version"); err != nil {
		return err
	}

	c := appengine.NewContext(r)
	imports := r.FormValue("impl_imports")
	code := r.FormValue("impl_code")
	comment := r.FormValue("impl_comment")
	attributionURL := r.FormValue("impl_attribution_url")
	demoURL := r.FormValue("impl_demo_url")
	docURL := r.FormValue("impl_doc_url")
	c.Infof("[%v] is updating impl %v of idiom %v", username, existingImplIDStr, idiomIDStr)

	idiomID := String2Int(idiomIDStr)
	if idiomID == -1 {
		return PiError{idiomIDStr + " is not a valid idiom id.", http.StatusBadRequest}
	}

	key, idiom, err := dao.getIdiom(c, idiomID)
	if err != nil {
		return PiError{"Could not find implementation " + existingImplIDStr + " for idiom " + idiomIDStr, http.StatusNotFound}
	}

	implID := String2Int(existingImplIDStr)
	if implID == -1 {
		return PiError{existingImplIDStr + " is not a valid implementation id.", http.StatusBadRequest}
	}

	_, impl, _ := idiom.FindImplInIdiom(implID)

	if r.FormValue("impl_version") != strconv.Itoa(impl.Version) {
		return PiError{fmt.Sprintf("Implementation has been concurrently modified (editing version %v, current version is %v)", r.FormValue("impl_version"), impl.Version), http.StatusConflict}
	}

	if err := validateURLFormatOrEmpty(attributionURL); err != nil {
		return PiError{"Can't accept URL [" + attributionURL + "]", http.StatusBadRequest}
	}

	if err := validateURLFormatOrEmpty(demoURL); err != nil {
		return PiError{"Can't accept URL [" + demoURL + "]", http.StatusBadRequest}
	}

	idiom.EditSummary = "[" + impl.LanguageName + "] " + r.FormValue("edit_summary")
	idiom.LastEditedImplID = implID
	impl.ImportsBlock = imports
	impl.CodeBlock = code
	impl.AuthorComment = comment
	impl.LastEditor = username
	impl.OriginalAttributionURL = attributionURL
	impl.DemoURL = demoURL
	impl.DocumentationURL = docURL
	impl.Version = impl.Version + 1
	impl.VersionDate = time.Now()

	err = dao.saveExistingIdiom(c, key, idiom)
	if err != nil {
		return err
	}

	http.Redirect(w, r, NiceImplURL(idiom, implID, impl.LanguageName), http.StatusFound)
	return nil
}
<|endoftext|>


<commit_before>package workers

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"github.com/Supernomad/quantum/common"
	"github.com/Supernomad/quantum/etcd"
)

var (
	outgoing *Outgoing
	incoming *Incoming
)

var (
	resolveIncomingResult, verifyResult, unsealResult,
	resolveOutgoingResult, sealResult, signResult *common.Payload
)

var resolveIncomingMapping, resolveOutgoingMapping, testMapping *common.Mapping
var intRemoteIp, intLocalIp uint32

func init() {
	key := make([]byte, 32)
	rand.Read(key)
	block, _ := aes.NewCipher(key)
	aesgcm, _ := cipher.NewGCM(block)

	remoteIp := "10.8.0.1"
	localIp := "10.8.0.2"

	intRemoteIp = etcd.IP4toInt(remoteIp)
	intLocalIp = etcd.IP4toInt(localIp)

	outgoing = NewOutgoing(nil, "10.8.0.2", nil, nil, nil)
	incoming = NewIncoming(nil, "10.8.0.1", nil, nil, nil)

	testMapping = &common.Mapping{PublicKey: make([]byte, 32), SecretKey: key, Cipher: aesgcm}

	outgoing.Mappings = make(map[uint32]*common.Mapping)
	outgoing.Mappings[intRemoteIp] = testMapping
	outgoing.Mappings[intLocalIp] = testMapping

	incoming.Mappings = make(map[uint32]*common.Mapping)
	incoming.Mappings[intRemoteIp] = testMapping
	incoming.Mappings[intLocalIp] = testMapping

	for i := 5; i < 10000; i++ {
		outgoing.Mappings[intRemoteIp+uint32(i)] = &common.Mapping{PublicKey: make([]byte, 32)}
		incoming.Mappings[intRemoteIp+uint32(i)] = &common.Mapping{PublicKey: make([]byte, 32)}
	}
}
<commit_msg>Fixed failing tests<commit_after>package workers

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"github.com/Supernomad/quantum/common"
)

var (
	outgoing *Outgoing
	incoming *Incoming
)

var (
	resolveIncomingResult, verifyResult, unsealResult,
	resolveOutgoingResult, sealResult, signResult *common.Payload
)

var resolveIncomingMapping, resolveOutgoingMapping, testMapping *common.Mapping
var intRemoteIp, intLocalIp uint32

func init() {
	key := make([]byte, 32)
	rand.Read(key)
	block, _ := aes.NewCipher(key)
	aesgcm, _ := cipher.NewGCM(block)

	remoteIp := "10.8.0.1"
	localIp := "10.8.0.2"

	intRemoteIp = common.IPtoInt(remoteIp)
	intLocalIp = common.IPtoInt(localIp)

	outgoing = NewOutgoing(nil, "10.8.0.2", nil, nil, nil)
	incoming = NewIncoming(nil, "10.8.0.1", nil, nil, nil)

	testMapping = &common.Mapping{PublicKey: make([]byte, 32), SecretKey: key, Cipher: aesgcm}

	outgoing.Mappings = make(map[uint32]*common.Mapping)
	outgoing.Mappings[intRemoteIp] = testMapping
	outgoing.Mappings[intLocalIp] = testMapping

	incoming.Mappings = make(map[uint32]*common.Mapping)
	incoming.Mappings[intRemoteIp] = testMapping
	incoming.Mappings[intLocalIp] = testMapping

	for i := 5; i < 10000; i++ {
		outgoing.Mappings[intRemoteIp+uint32(i)] = &common.Mapping{PublicKey: make([]byte, 32)}
		incoming.Mappings[intRemoteIp+uint32(i)] = &common.Mapping{PublicKey: make([]byte, 32)}
	}
}
<|endoftext|>


<commit_before>package apidVerifyApiKey

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
)

type sucResponseDetail struct {
	Key             string `json:"key"`
	ExpiresAt       int64  `json:"expiresAt"`
	IssuedAt        int64  `json:"issuedAt"`
	Status          string `json:"status"`
	RedirectionURIs string `json:"redirectionURIs"`
	DeveloperAppId  string `json:"developerId"`
	DeveloperAppNam string `json:"developerAppName"`
}

type errResultDetail struct {
	ErrorCode string `json:"errorCode"`
	Reason    string `json:"reason"`
}

type kmsResponseSuccess struct {
	RspInfo sucResponseDetail `json:"result"`
	Type    string            `json:"type"`
}

type kmsResponseFail struct {
	ErrInfo errResultDetail `json:"result"`
	Type    string          `json:"type"`
}

// handle client API
func handleRequest(w http.ResponseWriter, r *http.Request) {

	db := getDB()
	if db == nil {
		w.WriteHeader(http.StatusServiceUnavailable)
		w.Write([]byte("initializing"))
		return
	}

	err := r.ParseForm()
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Unable to parse form"))
		return
	}

	f := r.Form
	elems := []string{"action", "key", "uriPath", "scopeuuid"}
	for _, elem := range elems {
		if f.Get(elem) == "" {
			w.WriteHeader(http.StatusBadRequest)
			w.Write([]byte(fmt.Sprintf("Missing element: %s", elem)))
			return
		}
	}

	b, err := verifyAPIKey(f)
	if err != nil {
		log.Errorf("error: %s", err)
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(err.Error()))
		return
	}

	log.Debugf("handleVerifyAPIKey result %s", b)
	w.Write(b)
}

// returns []byte to be written to client
func verifyAPIKey(f url.Values) ([]byte, error) {

	key := f.Get("key")
	scopeuuid := f.Get("scopeuuid")
	path := f.Get("uriPath")
	action := f.Get("action")

	if key == "" || scopeuuid == "" || path == "" || action != "verify" {
		log.Error("Input params Invalid/Incomplete")
		reason := "Input Params Incomplete or Invalid"
		errorCode := "INCORRECT_USER_INPUT"
		return errorResponse(reason, errorCode)
	}

	db := getDB()

	// DANGER: This relies on an external TABLE - DATA_SCOPE is maintained by apidApigeeSync
	var env, tenantId string
	error := db.QueryRow("SELECT env, scope FROM DATA_SCOPE WHERE id = ?;", scopeuuid).Scan(&env, &tenantId)

	switch {
	case error == sql.ErrNoRows:
		reason := "ENV Validation Failed"
		errorCode := "ENV_VALIDATION_FAILED"
		return errorResponse(reason, errorCode)
	case error != nil:
		reason := error.Error()
		errorCode := "SEARCH_INTERNAL_ERROR"
		return errorResponse(reason, errorCode)
	}

	log.Debug("Found tenant_id='", tenantId, "' with env='", env, "' for scopeuuid='", scopeuuid, "'")

	sSql := `
		SELECT
			ap.api_resources,
			ap.environments,
			c.issued_at,
			c.status,
			a.callback_url,
			ad.name,
			ad.id
		FROM
			APP_CREDENTIAL AS c
			INNER JOIN APP AS a ON c.app_id = a.id
			INNER JOIN ALL_DEVELOPERS AS ad
				ON (ad.id = a.company_id OR ad.id = a.developer_id)
			INNER JOIN APP_CREDENTIAL_APIPRODUCT_MAPPER as mp
				ON mp.appcred_id = c.id
			INNER JOIN API_PRODUCT as ap ON ap.id = mp.apiprdt_id
		WHERE (UPPER(ad.status) = 'ACTIVE'
			AND mp.apiprdt_id = ap.id
			AND mp.app_id = a.id
			AND mp.appcred_id = c.id
			AND UPPER(mp.status) = 'APPROVED'
			AND UPPER(a.status) = 'APPROVED'
			AND c.id = $1
			AND c.tenant_id = $2)
		UNION
		SELECT
			ap.api_resources,
			ap.environments,
			c.issued_at,
			c.status,
			a.callback_url,
			ad.name,
			ad.id
		FROM
			APP_CREDENTIAL AS c
			INNER JOIN APP AS a ON c.app_id = a.id
			INNER JOIN ALL_DEVELOPERS AS ad
				ON (ad.id = a.company_id OR ad.id = a.developer_id)
			INNER JOIN APP_CREDENTIAL_APIPRODUCT_MAPPER as mp
				ON mp.appcred_id = c.id
			INNER JOIN API_PRODUCT as ap ON ap.id = mp.apiprdt_id
		WHERE (UPPER(ad.status) = 'ACTIVE'
			AND mp.apiprdt_id = ap.id
			AND mp.app_id = a.id
			AND mp.appcred_id = c.id
			AND UPPER(mp.status) = 'APPROVED'
			AND UPPER(a.status) = 'APPROVED'
			AND c.id = $1
			AND c.tenant_id = $2)
	;`

	var status, redirectionURIs, developerAppName, developerId, resName, resEnv string
	var issuedAt int64
	err := db.QueryRow(sSql, key, tenantId).Scan(&resName, &resEnv, &issuedAt, &status,
		&redirectionURIs, &developerAppName, &developerId)
	switch {
	case err == sql.ErrNoRows:
		reason := "API Key verify failed for (" + key + ", " + scopeuuid + ", " + path + ")"
		errorCode := "REQ_ENTRY_NOT_FOUND"
		return errorResponse(reason, errorCode)

	case err != nil:
		reason := err.Error()
		errorCode := "SEARCH_INTERNAL_ERROR"
		return errorResponse(reason, errorCode)
	}

	/*
	 * Perform all validations related to the Query made with the data
	 * we just retrieved
	 */
	result := validatePath(resName, path)
	if result == false {
		reason := "Path Validation Failed (" + resName + " vs " + path + ")"
		errorCode := "PATH_VALIDATION_FAILED"
		return errorResponse(reason, errorCode)

	}

	/* Verify if the ENV matches */
	result = validateEnv(resEnv, env)
	if result == false {
		reason := "ENV Validation Failed (" + resEnv + " vs " + env + ")"
		errorCode := "ENV_VALIDATION_FAILED"
		return errorResponse(reason, errorCode)
	}

	var expiresAt int64 = -1
	resp := kmsResponseSuccess{
		Type: "APIKeyContext",
		RspInfo: sucResponseDetail{
			Key:             key,
			ExpiresAt:       expiresAt,
			IssuedAt:        issuedAt,
			Status:          status,
			RedirectionURIs: redirectionURIs,
			DeveloperAppId:  developerId,
			DeveloperAppNam: developerAppName},
	}
	return json.Marshal(resp)
}

func errorResponse(reason, errorCode string) ([]byte, error) {

	log.Error(reason)
	resp := kmsResponseFail{
		Type: "ErrorResult",
		ErrInfo: errResultDetail{
			Reason:    reason,
			ErrorCode: errorCode},
	}
	return json.Marshal(resp)
}
<commit_msg>Use union all - slightly more efficient<commit_after>package apidVerifyApiKey

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
)

type sucResponseDetail struct {
	Key             string `json:"key"`
	ExpiresAt       int64  `json:"expiresAt"`
	IssuedAt        int64  `json:"issuedAt"`
	Status          string `json:"status"`
	RedirectionURIs string `json:"redirectionURIs"`
	DeveloperAppId  string `json:"developerId"`
	DeveloperAppNam string `json:"developerAppName"`
}

type errResultDetail struct {
	ErrorCode string `json:"errorCode"`
	Reason    string `json:"reason"`
}

type kmsResponseSuccess struct {
	RspInfo sucResponseDetail `json:"result"`
	Type    string            `json:"type"`
}

type kmsResponseFail struct {
	ErrInfo errResultDetail `json:"result"`
	Type    string          `json:"type"`
}

// handle client API
func handleRequest(w http.ResponseWriter, r *http.Request) {

	db := getDB()
	if db == nil {
		w.WriteHeader(http.StatusServiceUnavailable)
		w.Write([]byte("initializing"))
		return
	}

	err := r.ParseForm()
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Unable to parse form"))
		return
	}

	f := r.Form
	elems := []string{"action", "key", "uriPath", "scopeuuid"}
	for _, elem := range elems {
		if f.Get(elem) == "" {
			w.WriteHeader(http.StatusBadRequest)
			w.Write([]byte(fmt.Sprintf("Missing element: %s", elem)))
			return
		}
	}

	b, err := verifyAPIKey(f)
	if err != nil {
		log.Errorf("error: %s", err)
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte(err.Error()))
		return
	}

	log.Debugf("handleVerifyAPIKey result %s", b)
	w.Write(b)
}

// returns []byte to be written to client
func verifyAPIKey(f url.Values) ([]byte, error) {

	key := f.Get("key")
	scopeuuid := f.Get("scopeuuid")
	path := f.Get("uriPath")
	action := f.Get("action")

	if key == "" || scopeuuid == "" || path == "" || action != "verify" {
		log.Error("Input params Invalid/Incomplete")
		reason := "Input Params Incomplete or Invalid"
		errorCode := "INCORRECT_USER_INPUT"
		return errorResponse(reason, errorCode)
	}

	db := getDB()

	// DANGER: This relies on an external TABLE - DATA_SCOPE is maintained by apidApigeeSync
	var env, tenantId string
	error := db.QueryRow("SELECT env, scope FROM DATA_SCOPE WHERE id = ?;", scopeuuid).Scan(&env, &tenantId)

	switch {
	case error == sql.ErrNoRows:
		reason := "ENV Validation Failed"
		errorCode := "ENV_VALIDATION_FAILED"
		return errorResponse(reason, errorCode)
	case error != nil:
		reason := error.Error()
		errorCode := "SEARCH_INTERNAL_ERROR"
		return errorResponse(reason, errorCode)
	}

	log.Debug("Found tenant_id='", tenantId, "' with env='", env, "' for scopeuuid='", scopeuuid, "'")

	sSql := `
		SELECT
			ap.api_resources,
			ap.environments,
			c.issued_at,
			c.status,
			a.callback_url,
			ad.name,
			ad.id
		FROM
			APP_CREDENTIAL AS c
			INNER JOIN APP AS a ON c.app_id = a.id
			INNER JOIN ALL_DEVELOPERS AS ad
				ON (ad.id = a.company_id OR ad.id = a.developer_id)
			INNER JOIN APP_CREDENTIAL_APIPRODUCT_MAPPER as mp
				ON mp.appcred_id = c.id
			INNER JOIN API_PRODUCT as ap ON ap.id = mp.apiprdt_id
		WHERE (UPPER(ad.status) = 'ACTIVE'
			AND mp.apiprdt_id = ap.id
			AND mp.app_id = a.id
			AND mp.appcred_id = c.id
			AND UPPER(mp.status) = 'APPROVED'
			AND UPPER(a.status) = 'APPROVED'
			AND c.id = $1
			AND c.tenant_id = $2)
		UNION ALL
		SELECT
			ap.api_resources,
			ap.environments,
			c.issued_at,
			c.status,
			a.callback_url,
			ad.name,
			ad.id
		FROM
			APP_CREDENTIAL AS c
			INNER JOIN APP AS a ON c.app_id = a.id
			INNER JOIN ALL_DEVELOPERS AS ad
				ON (ad.id = a.company_id OR ad.id = a.developer_id)
			INNER JOIN APP_CREDENTIAL_APIPRODUCT_MAPPER as mp
				ON mp.appcred_id = c.id
			INNER JOIN API_PRODUCT as ap ON ap.id = mp.apiprdt_id
		WHERE (UPPER(ad.status) = 'ACTIVE'
			AND mp.apiprdt_id = ap.id
			AND mp.app_id = a.id
			AND mp.appcred_id = c.id
			AND UPPER(mp.status) = 'APPROVED'
			AND UPPER(a.status) = 'APPROVED'
			AND c.id = $1
			AND c.tenant_id = $2)
	;`

	var status, redirectionURIs, developerAppName, developerId, resName, resEnv string
	var issuedAt int64
	err := db.QueryRow(sSql, key, tenantId).Scan(&resName, &resEnv, &issuedAt, &status,
		&redirectionURIs, &developerAppName, &developerId)
	switch {
	case err == sql.ErrNoRows:
		reason := "API Key verify failed for (" + key + ", " + scopeuuid + ", " + path + ")"
		errorCode := "REQ_ENTRY_NOT_FOUND"
		return errorResponse(reason, errorCode)

	case err != nil:
		reason := err.Error()
		errorCode := "SEARCH_INTERNAL_ERROR"
		return errorResponse(reason, errorCode)
	}

	/*
	 * Perform all validations related to the Query made with the data
	 * we just retrieved
	 */
	result := validatePath(resName, path)
	if result == false {
		reason := "Path Validation Failed (" + resName + " vs " + path + ")"
		errorCode := "PATH_VALIDATION_FAILED"
		return errorResponse(reason, errorCode)

	}

	/* Verify if the ENV matches */
	result = validateEnv(resEnv, env)
	if result == false {
		reason := "ENV Validation Failed (" + resEnv + " vs " + env + ")"
		errorCode := "ENV_VALIDATION_FAILED"
		return errorResponse(reason, errorCode)
	}

	var expiresAt int64 = -1
	resp := kmsResponseSuccess{
		Type: "APIKeyContext",
		RspInfo: sucResponseDetail{
			Key:             key,
			ExpiresAt:       expiresAt,
			IssuedAt:        issuedAt,
			Status:          status,
			RedirectionURIs: redirectionURIs,
			DeveloperAppId:  developerId,
			DeveloperAppNam: developerAppName},
	}
	return json.Marshal(resp)
}

func errorResponse(reason, errorCode string) ([]byte, error) {

	log.Error(reason)
	resp := kmsResponseFail{
		Type: "ErrorResult",
		ErrInfo: errResultDetail{
			Reason:    reason,
			ErrorCode: errorCode},
	}
	return json.Marshal(resp)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/kataras/iris"
)

func main() {
	iris.Get("/hi", func(ctx *iris.Context) {
		ctx.SetCookieKV("name", "iris")
		ctx.Write("Hello world %s", ctx.GetCookie("name"))
	})
	fmt.Println("Start Service....")
	iris.Listen(":8080")
}
<commit_msg>- update code<commit_after>package main

import (
	"fmt"

	"github.com/kataras/iris"
)

func main() {
	iris.Get("/hi", func(ctx *iris.Context) {
		ctx.SetCookieKV("name", "iris")

		ctx.SetHeader("Data", "1234567")

		ctx.Write("Hello world %s", ctx.GetCookie("name"))
	})
	fmt.Println("Start Service....")
	iris.Listen(":8080")
}
<|endoftext|>


<commit_before>package dnsimple

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/hashicorp/cleanhttp"
)

// Client provides a client to the DNSimple API
type Client struct {
	// Access Token
	Token string

	// User Email
	Email string

	// Domain Token
	DomainToken string

	// URL to the DO API to use
	URL string

	// HttpClient is the client to use. A client with
	// default values will be used if not provided.
	Http *http.Client
}

// DNSimpleError is the error format that they return
// to us if there is a problem
type DNSimpleError struct {
	Errors map[string][]string `json:"errors"`
}

func (d *DNSimpleError) Join() string {
	var errs []string

	for k, v := range d.Errors {
		errs = append(errs, fmt.Sprintf("%s errors: %s", k, strings.Join(v, ", ")))
	}

	return strings.Join(errs, ", ")
}

// NewClient returns a new dnsimple client,
// requires an authorization token. You can generate
// an OAuth token by visiting the Apps & API section
// of the DNSimple control panel for your account.
func NewClient(email string, token string) (*Client, error) {
	client := Client{
		Token: token,
		Email: email,
		URL:   "https://api.dnsimple.com/v1",
		Http:  cleanhttp.DefaultClient(),
	}
	return &client, nil
}

func NewClientWithDomainToken(domainToken string) (*Client, error) {
	client := Client{
		DomainToken: domainToken,
		URL:         "https://api.dnsimple.com/v1",
		Http:        cleanhttp.DefaultClient(),
	}
	return &client, nil
}

// Creates a new request with the params
func (c *Client) NewRequest(body map[string]interface{}, method string, endpoint string) (*http.Request, error) {
	u, err := url.Parse(c.URL + endpoint)

	if err != nil {
		return nil, fmt.Errorf("Error parsing base URL: %s", err)
	}

	rBody, err := encodeBody(body)
	if err != nil {
		return nil, fmt.Errorf("Error encoding request body: %s", err)
	}

	// Build the request
	req, err := http.NewRequest(method, u.String(), rBody)
	if err != nil {
		return nil, fmt.Errorf("Error creating request: %s", err)
	}

	// Add the authorization header
	if c.DomainToken != "" {
		req.Header.Add("X-DNSimple-Domain-Token", c.DomainToken)
	} else {
		req.Header.Add("X-DNSimple-Token", fmt.Sprintf("%s:%s", c.Email, c.Token))
	}
	req.Header.Add("Accept", "application/json")

	// If it's a not a get, add a content-type
	if method != "GET" {
		req.Header.Add("Content-Type", "application/json")
	}

	return req, nil

}

// parseErr is used to take an error json resp
// and return a single string for use in error messages
func parseErr(resp *http.Response) error {
	dnsError := DNSimpleError{}

	err := decodeBody(resp, &dnsError)

	// if there was an error decoding the body, just return that
	if err != nil {
		return fmt.Errorf("Error parsing error body for non-200 request: %s", err)
	}

	return fmt.Errorf("API Error: %s", dnsError.Join())
}

// decodeBody is used to JSON decode a body
func decodeBody(resp *http.Response, out interface{}) error {
	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		return err
	}

	if err = json.Unmarshal(body, &out); err != nil {
		return err
	}

	return nil
}

func encodeBody(obj interface{}) (io.Reader, error) {
	buf := bytes.NewBuffer(nil)
	enc := json.NewEncoder(buf)
	if err := enc.Encode(obj); err != nil {
		return nil, err
	}
	return buf, nil
}

// checkResp wraps http.Client.Do() and verifies that the
// request was successful. A non-200 request returns an error
// formatted to included any validation problems or otherwise
func checkResp(resp *http.Response, err error) (*http.Response, error) {
	// If the err is already there, there was an error higher
	// up the chain, so just return that
	if err != nil {
		return resp, err
	}

	switch i := resp.StatusCode; {
	case i == 200:
		return resp, nil
	case i == 201:
		return resp, nil
	case i == 202:
		return resp, nil
	case i == 204:
		return resp, nil
	case i == 422:
		return nil, fmt.Errorf("API Error: %s", resp.Status)
	case i == 400:
		return nil, parseErr(resp)
	default:
		return nil, fmt.Errorf("API Error: %s", resp.Status)
	}
}
<commit_msg>Update the location of cleanhttp<commit_after>package dnsimple

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/hashicorp/go-cleanhttp"
)

// Client provides a client to the DNSimple API
type Client struct {
	// Access Token
	Token string

	// User Email
	Email string

	// Domain Token
	DomainToken string

	// URL to the DO API to use
	URL string

	// HttpClient is the client to use. A client with
	// default values will be used if not provided.
	Http *http.Client
}

// DNSimpleError is the error format that they return
// to us if there is a problem
type DNSimpleError struct {
	Errors map[string][]string `json:"errors"`
}

func (d *DNSimpleError) Join() string {
	var errs []string

	for k, v := range d.Errors {
		errs = append(errs, fmt.Sprintf("%s errors: %s", k, strings.Join(v, ", ")))
	}

	return strings.Join(errs, ", ")
}

// NewClient returns a new dnsimple client,
// requires an authorization token. You can generate
// an OAuth token by visiting the Apps & API section
// of the DNSimple control panel for your account.
func NewClient(email string, token string) (*Client, error) {
	client := Client{
		Token: token,
		Email: email,
		URL:   "https://api.dnsimple.com/v1",
		Http:  cleanhttp.DefaultClient(),
	}
	return &client, nil
}

func NewClientWithDomainToken(domainToken string) (*Client, error) {
	client := Client{
		DomainToken: domainToken,
		URL:         "https://api.dnsimple.com/v1",
		Http:        cleanhttp.DefaultClient(),
	}
	return &client, nil
}

// Creates a new request with the params
func (c *Client) NewRequest(body map[string]interface{}, method string, endpoint string) (*http.Request, error) {
	u, err := url.Parse(c.URL + endpoint)

	if err != nil {
		return nil, fmt.Errorf("Error parsing base URL: %s", err)
	}

	rBody, err := encodeBody(body)
	if err != nil {
		return nil, fmt.Errorf("Error encoding request body: %s", err)
	}

	// Build the request
	req, err := http.NewRequest(method, u.String(), rBody)
	if err != nil {
		return nil, fmt.Errorf("Error creating request: %s", err)
	}

	// Add the authorization header
	if c.DomainToken != "" {
		req.Header.Add("X-DNSimple-Domain-Token", c.DomainToken)
	} else {
		req.Header.Add("X-DNSimple-Token", fmt.Sprintf("%s:%s", c.Email, c.Token))
	}
	req.Header.Add("Accept", "application/json")

	// If it's a not a get, add a content-type
	if method != "GET" {
		req.Header.Add("Content-Type", "application/json")
	}

	return req, nil

}

// parseErr is used to take an error json resp
// and return a single string for use in error messages
func parseErr(resp *http.Response) error {
	dnsError := DNSimpleError{}

	err := decodeBody(resp, &dnsError)

	// if there was an error decoding the body, just return that
	if err != nil {
		return fmt.Errorf("Error parsing error body for non-200 request: %s", err)
	}

	return fmt.Errorf("API Error: %s", dnsError.Join())
}

// decodeBody is used to JSON decode a body
func decodeBody(resp *http.Response, out interface{}) error {
	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		return err
	}

	if err = json.Unmarshal(body, &out); err != nil {
		return err
	}

	return nil
}

func encodeBody(obj interface{}) (io.Reader, error) {
	buf := bytes.NewBuffer(nil)
	enc := json.NewEncoder(buf)
	if err := enc.Encode(obj); err != nil {
		return nil, err
	}
	return buf, nil
}

// checkResp wraps http.Client.Do() and verifies that the
// request was successful. A non-200 request returns an error
// formatted to included any validation problems or otherwise
func checkResp(resp *http.Response, err error) (*http.Response, error) {
	// If the err is already there, there was an error higher
	// up the chain, so just return that
	if err != nil {
		return resp, err
	}

	switch i := resp.StatusCode; {
	case i == 200:
		return resp, nil
	case i == 201:
		return resp, nil
	case i == 202:
		return resp, nil
	case i == 204:
		return resp, nil
	case i == 422:
		return nil, fmt.Errorf("API Error: %s", resp.Status)
	case i == 400:
		return nil, parseErr(resp)
	default:
		return nil, fmt.Errorf("API Error: %s", resp.Status)
	}
}
<|endoftext|>


<commit_before>package bitclient

import (
	"fmt"
)

func (bc *BitClient) GetSonarSettings(projectKey string, repositorySlug string) (SonarSettings, error) {

	params := SonarSettings{}

	_, err := bc.DoGet(
		fmt.Sprintf("/rest/sonar4stash/1.0/projects/%s/repos/%s/settings", projectKey, repositorySlug),
		nil,
		&params,
	)

	return params, err
}

func (bc *BitClient) SetSonarSettings(projectKey string, repositorySlug string, settings SonarSettings) error {

	_, err := bc.DoPost(
		fmt.Sprintf("/rest/sonar4stash/1.0/projects/%s/repos/%s/settings", projectKey, repositorySlug),
		settings,
		nil,
	)

	return err
}
<commit_msg>Fixed sonar endpoints<commit_after>package bitclient

import (
	"fmt"
)

func (bc *BitClient) GetSonarSettings(projectKey string, repositorySlug string) (SonarSettings, error) {
	rError := new(ErrorResponse)
	settings := SonarSettings{}

	url := fmt.Sprintf("/rest/sonar4stash/1.0/projects/%s/repos/%s/settings", projectKey, repositorySlug)
	resp, _ := bc.sling.New().Get(url).Receive(&settings, rError)

	resp, err := bc.checkReponse(resp, rError)

	return settings, err
}

func (bc *BitClient) SetSonarSettings(projectKey string, repositorySlug string, settings SonarSettings) error {
	rError := new(ErrorResponse)

	url := fmt.Sprintf("/rest/sonar4stash/1.0/projects/%s/repos/%s/settings", projectKey, repositorySlug)
	resp, _ := bc.sling.New().Post(url).BodyJSON(settings).Receive(nil, rError)

	resp, err := bc.checkReponse(resp, rError)

	return err
}
<|endoftext|>


<commit_before><commit_msg>adding support for discarding on captured data<commit_after><|endoftext|>