<commit_before>package agent

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"

	"github.com/fnproject/fn/api/agent/drivers"
	"github.com/fnproject/fn/api/common"
	"github.com/fnproject/fn/api/id"
	"github.com/fnproject/fn/api/models"
	"github.com/go-openapi/strfmt"
	"github.com/opentracing/opentracing-go"
	"github.com/sirupsen/logrus"
)

type Call interface {
	// Model will return the underlying models.Call configuration for this call.
	// TODO we could respond to async correctly from agent but layering, this
	// is only because the front end has different responses based on call type.
	// try to discourage use elsewhere until this gets pushed down more...
	Model() *models.Call

	// Start will be called before this call is executed, it may be used to
	// guarantee mutual exclusion, check docker permissions, update timestamps,
	// etc.
	// TODO Start and End can likely be unexported as they are only used in the agent,
	// and on a type which is constructed in a specific agent. meh.
	Start(ctx context.Context) error

	// End will be called immediately after attempting a call execution,
	// regardless of whether the execution failed or not. An error will be passed
	// to End, which if nil indicates a successful execution. Any error returned
	// from End will be returned as the error from Submit.
	End(ctx context.Context, err error) error
}

// TODO build w/o closures... lazy
type CallOpt func(a *agent, c *call) error

type Param struct {
	Key   string
	Value string
}
type Params []Param

func fixupRequestURL(req *http.Request) string {
	if req.URL.Scheme == "" {
		if req.TLS == nil {
			req.URL.Scheme = "http"
		} else {
			req.URL.Scheme = "https"
		}
	}
	if req.URL.Host == "" {
		req.URL.Host = req.Host
	}
	return req.URL.String()
}

func FromRequest(appName, path string, req *http.Request) CallOpt {
	return func(a *agent, c *call) error {
		app, err := a.da.GetApp(req.Context(), appName)
		if err != nil {
			return err
		}

		route, err := a.da.GetRoute(req.Context(), appName, path)
		if err != nil {
			return err
		}

		if route.Format == "" {
			route.Format = models.FormatDefault
		}

		id := id.New().String()

		// TODO this relies on ordering of opts, but tests make sure it works, probably re-plumb/destroy headers
		// TODO async should probably supply an http.ResponseWriter that records the logs, to attach response headers to
		if rw, ok := c.w.(http.ResponseWriter); ok {
			rw.Header().Add("FN_CALL_ID", id)
			for k, vs := range route.Headers {
				for _, v := range vs {
					// pre-write in these headers to response
					rw.Header().Add(k, v)
				}
			}
		}

		// add our per call headers in here
		req.Header.Set("FN_METHOD", req.Method)
		req.Header.Set("FN_REQUEST_URL", reqURL(req))
		req.Header.Set("FN_CALL_ID", id)

		// this ensures that there is an image, path, timeouts, memory, etc are valid.
		// NOTE: this means assign any changes above into route's fields
		err = route.Validate()
		if err != nil {
			return err
		}

		c.Call = &models.Call{
			ID:      id,
			AppName: appName,
			Path:    route.Path,
			Image:   route.Image,
			// Delay: 0,
			Type:   route.Type,
			Format: route.Format,
			// Payload: TODO,
			Priority:    new(int32), // TODO this is crucial, apparently
			Timeout:     route.Timeout,
			IdleTimeout: route.IdleTimeout,
			Memory:      route.Memory,
			Config:      buildConfig(app, route),
			Headers:     req.Header,
			CreatedAt:   strfmt.DateTime(time.Now()),
			URL:         reqURL(req),
			Method:      req.Method,
		}

		c.req = req
		return nil
	}
}

func buildConfig(app *models.App, route *models.Route) models.Config {
	conf := make(models.Config, 8+len(app.Config)+len(route.Config))
	for k, v := range app.Config {
		conf[k] = v
	}
	for k, v := range route.Config {
		conf[k] = v
	}

	conf["FN_FORMAT"] = route.Format
	conf["FN_APP_NAME"] = app.Name
	conf["FN_PATH"] = route.Path
	// TODO: might be a good idea to pass in: "FN_BASE_PATH" = fmt.Sprintf("/r/%s", appName) || "/" if using DNS entries per app
	conf["FN_MEMORY"] = fmt.Sprintf("%d", route.Memory)
	conf["FN_TYPE"] = route.Type
	return conf
}

func reqURL(req *http.Request) string {
	if req.URL.Scheme == "" {
		if req.TLS == nil {
			req.URL.Scheme = "http"
		} else {
			req.URL.Scheme = "https"
		}
	}
	if req.URL.Host == "" {
		req.URL.Host = req.Host
	}
	return req.URL.String()
}

// TODO this currently relies on FromRequest having happened before to create the model
// here, to be a fully qualified model. We probably should double check but having a way
// to bypass will likely be what's used anyway unless forced.
func FromModel(mCall *models.Call) CallOpt {
	return func(a *agent, c *call) error {
		c.Call = mCall

		req, err := http.NewRequest(c.Method, c.URL, strings.NewReader(c.Payload))
		if err != nil {
			return err
		}
		req.Header = c.Headers

		c.req = req
		// TODO anything else really?
		return nil
	}
}

// TODO this should be required
func WithWriter(w io.Writer) CallOpt {
	return func(a *agent, c *call) error {
		c.w = w
		return nil
	}
}

// GetCall builds a Call that can be used to submit jobs to the agent.
//
// TODO where to put this? async and sync both call this
func (a *agent) GetCall(opts ...CallOpt) (Call, error) {
	var c call

	for _, o := range opts {
		err := o(a, &c)
		if err != nil {
			return nil, err
		}
	}

	// TODO typed errors to test
	if c.req == nil || c.Call == nil {
		return nil, errors.New("no model or request provided for call")
	}

	c.da = a.da
	c.ct = a

	ctx, _ := common.LoggerWithFields(c.req.Context(),
		logrus.Fields{"id": c.ID, "app": c.AppName, "route": c.Path})
	c.req = c.req.WithContext(ctx)

	// setup stderr logger separate (don't inherit ctx vars)
	logger := logrus.WithFields(logrus.Fields{"user_log": true, "app_name": c.AppName, "path": c.Path, "image": c.Image, "call_id": c.ID})
	c.stderr = setupLogger(logger)
	if c.w == nil {
		// send STDOUT to logs if no writer given (async...)
		// TODO we could/should probably make this explicit to GetCall, ala 'WithLogger', but it's dupe code (who cares?)
		c.w = c.stderr
	}

	now := time.Now()
	slotDeadline := now.Add(time.Duration(c.Call.Timeout) * time.Second / 2)
	execDeadline := now.Add(time.Duration(c.Call.Timeout) * time.Second)

	c.slotDeadline = slotDeadline
	c.execDeadline = execDeadline

	execDeadlineStr := strfmt.DateTime(execDeadline).String()

	// these 2 headers buckets are the same but for posterity!
	if c.Headers == nil {
		c.Headers = make(http.Header)
		c.req.Header = c.Headers
	}
	c.Headers.Set("FN_DEADLINE", execDeadlineStr)
	c.req.Header.Set("FN_DEADLINE", execDeadlineStr)

	return &c, nil
}

type call struct {
	*models.Call

	da           DataAccess
	w            io.Writer
	req          *http.Request
	stderr       io.ReadWriteCloser
	ct           callTrigger
	slots        *slotQueue
	slotDeadline time.Time
	execDeadline time.Time
}

func (c *call) Model() *models.Call { return c.Call }

func (c *call) Start(ctx context.Context) error {
	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_call_start")
	defer span.Finish()

	// Check context timeouts, errors
	if ctx.Err() != nil {
		return ctx.Err()
	}

	c.StartedAt = strfmt.DateTime(time.Now())
	c.Status = "running"

	if rw, ok := c.w.(http.ResponseWriter); ok { // TODO need to figure out better way to wire response headers in
		rw.Header().Set("XXX-FXLB-WAIT", time.Time(c.StartedAt).Sub(time.Time(c.CreatedAt)).String())
	}

	if c.Type == models.TypeAsync {
		// XXX (reed): make sure MQ reservation is lengthy. to skirt MQ semantics,
		// we could add a new message to MQ w/ delay of call.Timeout and delete the
		// old one (in that order), after marking the call as running in the db
		// (see below)

		// XXX (reed): should we store the updated started_at + status? we could
		// use this so that if we pick up a call from mq and find its status is
		// running to avoid running the call twice and potentially mark it as
		// errored (built in long running task detector, so to speak...)

		err := c.da.Start(ctx, c.Model())
		if err != nil {
			return err // let another thread try this
		}
	}

	err := c.ct.fireBeforeCall(ctx, c.Model())
	if err != nil {
		return fmt.Errorf("BeforeCall: %v", err)
	}

	return nil
}

func (c *call) End(ctx context.Context, errIn error) error {
	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_call_end")
	defer span.Finish()

	c.CompletedAt = strfmt.DateTime(time.Now())

	switch errIn {
	case nil:
		c.Status = "success"
	case context.DeadlineExceeded:
		c.Status = "timeout"
	default:
		c.Status = "error"
		c.Error = errIn.Error()
	}

	// ensure stats histogram is reasonably bounded
	c.Call.Stats = drivers.Decimate(240, c.Call.Stats)

	if err := c.da.Finish(ctx, c.Model(), c.stderr, c.Type == models.TypeAsync); err != nil {
		common.Logger(ctx).WithError(err).Error("error finalizing call on datastore/mq")
		// note: Not returning err here since the job could have already finished successfully.
	}

	// NOTE call this after InsertLog or the buffer will get reset
	c.stderr.Close()

	if err := c.ct.fireAfterCall(ctx, c.Model()); err != nil {
		return fmt.Errorf("AfterCall: %v", err)
	}

	return errIn // original error, important for use in sync call returns
}
<commit_msg>fn: removed dead code (#672)<commit_after>package agent

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"

	"github.com/fnproject/fn/api/agent/drivers"
	"github.com/fnproject/fn/api/common"
	"github.com/fnproject/fn/api/id"
	"github.com/fnproject/fn/api/models"
	"github.com/go-openapi/strfmt"
	"github.com/opentracing/opentracing-go"
	"github.com/sirupsen/logrus"
)

type Call interface {
	// Model will return the underlying models.Call configuration for this call.
	// TODO we could respond to async correctly from agent but layering, this
	// is only because the front end has different responses based on call type.
	// try to discourage use elsewhere until this gets pushed down more...
	Model() *models.Call

	// Start will be called before this call is executed, it may be used to
	// guarantee mutual exclusion, check docker permissions, update timestamps,
	// etc.
	// TODO Start and End can likely be unexported as they are only used in the agent,
	// and on a type which is constructed in a specific agent. meh.
	Start(ctx context.Context) error

	// End will be called immediately after attempting a call execution,
	// regardless of whether the execution failed or not. An error will be passed
	// to End, which if nil indicates a successful execution. Any error returned
	// from End will be returned as the error from Submit.
	End(ctx context.Context, err error) error
}

// TODO build w/o closures... lazy
type CallOpt func(a *agent, c *call) error

type Param struct {
	Key   string
	Value string
}
type Params []Param

func FromRequest(appName, path string, req *http.Request) CallOpt {
	return func(a *agent, c *call) error {
		app, err := a.da.GetApp(req.Context(), appName)
		if err != nil {
			return err
		}

		route, err := a.da.GetRoute(req.Context(), appName, path)
		if err != nil {
			return err
		}

		if route.Format == "" {
			route.Format = models.FormatDefault
		}

		id := id.New().String()

		// TODO this relies on ordering of opts, but tests make sure it works, probably re-plumb/destroy headers
		// TODO async should probably supply an http.ResponseWriter that records the logs, to attach response headers to
		if rw, ok := c.w.(http.ResponseWriter); ok {
			rw.Header().Add("FN_CALL_ID", id)
			for k, vs := range route.Headers {
				for _, v := range vs {
					// pre-write in these headers to response
					rw.Header().Add(k, v)
				}
			}
		}

		// add our per call headers in here
		req.Header.Set("FN_METHOD", req.Method)
		req.Header.Set("FN_REQUEST_URL", reqURL(req))
		req.Header.Set("FN_CALL_ID", id)

		// this ensures that there is an image, path, timeouts, memory, etc are valid.
		// NOTE: this means assign any changes above into route's fields
		err = route.Validate()
		if err != nil {
			return err
		}

		c.Call = &models.Call{
			ID:      id,
			AppName: appName,
			Path:    route.Path,
			Image:   route.Image,
			// Delay: 0,
			Type:   route.Type,
			Format: route.Format,
			// Payload: TODO,
			Priority:    new(int32), // TODO this is crucial, apparently
			Timeout:     route.Timeout,
			IdleTimeout: route.IdleTimeout,
			Memory:      route.Memory,
			Config:      buildConfig(app, route),
			Headers:     req.Header,
			CreatedAt:   strfmt.DateTime(time.Now()),
			URL:         reqURL(req),
			Method:      req.Method,
		}

		c.req = req
		return nil
	}
}

func buildConfig(app *models.App, route *models.Route) models.Config {
	conf := make(models.Config, 8+len(app.Config)+len(route.Config))
	for k, v := range app.Config {
		conf[k] = v
	}
	for k, v := range route.Config {
		conf[k] = v
	}

	conf["FN_FORMAT"] = route.Format
	conf["FN_APP_NAME"] = app.Name
	conf["FN_PATH"] = route.Path
	// TODO: might be a good idea to pass in: "FN_BASE_PATH" = fmt.Sprintf("/r/%s", appName) || "/" if using DNS entries per app
	conf["FN_MEMORY"] = fmt.Sprintf("%d", route.Memory)
	conf["FN_TYPE"] = route.Type
	return conf
}

func reqURL(req *http.Request) string {
	if req.URL.Scheme == "" {
		if req.TLS == nil {
			req.URL.Scheme = "http"
		} else {
			req.URL.Scheme = "https"
		}
	}
	if req.URL.Host == "" {
		req.URL.Host = req.Host
	}
	return req.URL.String()
}

// TODO this currently relies on FromRequest having happened before to create the model
// here, to be a fully qualified model. We probably should double check but having a way
// to bypass will likely be what's used anyway unless forced.
func FromModel(mCall *models.Call) CallOpt {
	return func(a *agent, c *call) error {
		c.Call = mCall

		req, err := http.NewRequest(c.Method, c.URL, strings.NewReader(c.Payload))
		if err != nil {
			return err
		}
		req.Header = c.Headers

		c.req = req
		// TODO anything else really?
		return nil
	}
}

// TODO this should be required
func WithWriter(w io.Writer) CallOpt {
	return func(a *agent, c *call) error {
		c.w = w
		return nil
	}
}

// GetCall builds a Call that can be used to submit jobs to the agent.
//
// TODO where to put this? async and sync both call this
func (a *agent) GetCall(opts ...CallOpt) (Call, error) {
	var c call

	for _, o := range opts {
		err := o(a, &c)
		if err != nil {
			return nil, err
		}
	}

	// TODO typed errors to test
	if c.req == nil || c.Call == nil {
		return nil, errors.New("no model or request provided for call")
	}

	c.da = a.da
	c.ct = a

	ctx, _ := common.LoggerWithFields(c.req.Context(),
		logrus.Fields{"id": c.ID, "app": c.AppName, "route": c.Path})
	c.req = c.req.WithContext(ctx)

	// setup stderr logger separate (don't inherit ctx vars)
	logger := logrus.WithFields(logrus.Fields{"user_log": true, "app_name": c.AppName, "path": c.Path, "image": c.Image, "call_id": c.ID})
	c.stderr = setupLogger(logger)
	if c.w == nil {
		// send STDOUT to logs if no writer given (async...)
		// TODO we could/should probably make this explicit to GetCall, ala 'WithLogger', but it's dupe code (who cares?)
		c.w = c.stderr
	}

	now := time.Now()
	slotDeadline := now.Add(time.Duration(c.Call.Timeout) * time.Second / 2)
	execDeadline := now.Add(time.Duration(c.Call.Timeout) * time.Second)

	c.slotDeadline = slotDeadline
	c.execDeadline = execDeadline

	execDeadlineStr := strfmt.DateTime(execDeadline).String()

	// these 2 headers buckets are the same but for posterity!
	if c.Headers == nil {
		c.Headers = make(http.Header)
		c.req.Header = c.Headers
	}
	c.Headers.Set("FN_DEADLINE", execDeadlineStr)
	c.req.Header.Set("FN_DEADLINE", execDeadlineStr)

	return &c, nil
}

type call struct {
	*models.Call

	da           DataAccess
	w            io.Writer
	req          *http.Request
	stderr       io.ReadWriteCloser
	ct           callTrigger
	slots        *slotQueue
	slotDeadline time.Time
	execDeadline time.Time
}

func (c *call) Model() *models.Call { return c.Call }

func (c *call) Start(ctx context.Context) error {
	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_call_start")
	defer span.Finish()

	// Check context timeouts, errors
	if ctx.Err() != nil {
		return ctx.Err()
	}

	c.StartedAt = strfmt.DateTime(time.Now())
	c.Status = "running"

	if rw, ok := c.w.(http.ResponseWriter); ok { // TODO need to figure out better way to wire response headers in
		rw.Header().Set("XXX-FXLB-WAIT", time.Time(c.StartedAt).Sub(time.Time(c.CreatedAt)).String())
	}

	if c.Type == models.TypeAsync {
		// XXX (reed): make sure MQ reservation is lengthy. to skirt MQ semantics,
		// we could add a new message to MQ w/ delay of call.Timeout and delete the
		// old one (in that order), after marking the call as running in the db
		// (see below)

		// XXX (reed): should we store the updated started_at + status? we could
		// use this so that if we pick up a call from mq and find its status is
		// running to avoid running the call twice and potentially mark it as
		// errored (built in long running task detector, so to speak...)

		err := c.da.Start(ctx, c.Model())
		if err != nil {
			return err // let another thread try this
		}
	}

	err := c.ct.fireBeforeCall(ctx, c.Model())
	if err != nil {
		return fmt.Errorf("BeforeCall: %v", err)
	}

	return nil
}

func (c *call) End(ctx context.Context, errIn error) error {
	span, ctx := opentracing.StartSpanFromContext(ctx, "agent_call_end")
	defer span.Finish()

	c.CompletedAt = strfmt.DateTime(time.Now())

	switch errIn {
	case nil:
		c.Status = "success"
	case context.DeadlineExceeded:
		c.Status = "timeout"
	default:
		c.Status = "error"
		c.Error = errIn.Error()
	}

	// ensure stats histogram is reasonably bounded
	c.Call.Stats = drivers.Decimate(240, c.Call.Stats)

	if err := c.da.Finish(ctx, c.Model(), c.stderr, c.Type == models.TypeAsync); err != nil {
		common.Logger(ctx).WithError(err).Error("error finalizing call on datastore/mq")
		// note: Not returning err here since the job could have already finished successfully.
	}

	// NOTE call this after InsertLog or the buffer will get reset
	c.stderr.Close()

	if err := c.ct.fireAfterCall(ctx, c.Model()); err != nil {
		return fmt.Errorf("AfterCall: %v", err)
	}

	return errIn // original error, important for use in sync call returns
}
<|endoftext|>


<commit_before><commit_msg>Adding tests for Gid.<commit_after><|endoftext|>


<commit_before>package evedata

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/jmcvetta/napping"
)

var stations map[int64]int64

func goEMDRCrestBridge(c *AppContext) {

	type marketRegions struct {
		RegionID   int64  `db:"regionID"`
		RegionName string `db:"regionName"`
	}

	type marketTypes struct {
		TypeID   int64  `db:"typeID"`
		TypeName string `db:"typeName"`
	}

	regions := []marketRegion{}
	err := c.Db.Select(&regions, `
		SELECT 	regionID, regionName
		FROM 	mapRegions
		WHERE 	regionID < 11000000
			AND regionID NOT IN(10000017, 10000019, 10000004);
	`)
	if err != nil {
		log.Fatal("EMDRCrestBridge:", err)
	}
	log.Printf("EMDRCrestBridge: Loaded %d Regions", len(regions))

	types := []marketTypes{}
	err = c.Db.Select(&types, `
		SELECT 	typeID, typeName
		FROM 	invTypes
		WHERE 	marketGroupID IS NOT NULL
			AND typeID < 250000;
	`)
	if err != nil {
		log.Fatal("EMDRCrestBridge:", err)
	}
	log.Printf("EMDRCrestBridge: Loaded %d items", len(types))

	stations = make(map[int64]int64)
	rows, err := c.Db.Query(`
		SELECT stationID, solarSystemID
		FROM staStations;
	`)
	for rows.Next() {

		var stationID int64
		var systemID int64

		if err := rows.Scan(&stationID, &systemID); err != nil {
			log.Fatal("EMDRCrestBridge: ", err)
		}
		stations[stationID] = systemID
	}
	rows.Close()

	if err != nil {
		log.Fatal("EMDRCrestBridge: ", err)
	}
	log.Printf("EMDRCrestBridge: Loaded %d stations", len(stations))

	// Throttle Crest Requests
	rate := time.Second / 8
	throttle := time.Tick(rate)

	// semaphore to prevent runaways
	sem := make(chan bool, c.Conf.EMDRCrestBridge.MaxGoRoutines)

	// CREST Session
	crest := napping.Session{}

	for {
		// loop through all regions
		for _, r := range regions {
			// and each item per region
			for _, t := range types {
				<-throttle // impliment throttle

				sem <- true
				go func() {
					defer func() { <-sem }()
					// Process Market History
					h := marketHistory{}
					url := fmt.Sprintf("https://public-crest.eveonline.com/market/%d/types/%d/history/", r.RegionID, t.TypeID)
					response, err := crest.Get(url, nil, &h, nil)
					if err != nil {
						log.Printf("EMDRCrestBridge: %s", err)
						return
					}
					if response.Status() == 200 {
						if len(h.Items) > 0 {
							go postHistory(sem, h, c, t.TypeID, r.RegionID)
						}
					}
				}()

				sem <- true
				go func() {
					defer func() { <-sem }()
					// Process Market Buy Orders
					b := marketOrders{}
					url := fmt.Sprintf("https://public-crest.eveonline.com/market/%d/orders/buy/?type=https://public-crest.eveonline.com/types/%d/", r.RegionID, t.TypeID)
					response, err := crest.Get(url, nil, &b, nil)
					if err != nil {
						log.Printf("EMDRCrestBridge: %s", err)
						return
					}
					if response.Status() == 200 {
						if len(b.Items) > 0 {
							go postOrders(sem, b, c, 1, t.TypeID, r.RegionID)
						}
					}
				}()

				sem <- true
				go func() {
					defer func() { <-sem }()
					// Process Market Sell Orders
					s := marketOrders{}
					url := fmt.Sprintf("https://public-crest.eveonline.com/market/%d/orders/sell/?type=https://public-crest.eveonline.com/types/%d/", r.RegionID, t.TypeID)
					response, err := crest.Get(url, nil, &s, nil)
					if err != nil {
						log.Printf("EMDRCrestBridge: %s", err)
						return
					}
					if response.Status() == 200 {
						if len(s.Items) > 0 {
							go postOrders(sem, s, c, 0, t.TypeID, r.RegionID)
						}
					}
				}()
			}
		}
	}
}

func postHistory(sem chan bool, h marketHistory, c *AppContext, typeID int64, regionID int64) {
	sem <- true
	defer func() { <-sem }()

	if c.Conf.EMDRCrestBridge.Import {
		historyUpdate, err := c.Db.Prepare(`
			INSERT IGNORE INTO market_history
				(date, low, high, mean, quantity, orders, itemID, regionID)
				VALUES(?,?,?,?,?,?,?,?);
		`)
		defer historyUpdate.Close()
		if err != nil {
			log.Printf("EMDRCrestBridge: %s", err)
		} else {
			tx, _ := c.Db.Begin()
			for _, e := range h.Items {
				tx.Stmt(historyUpdate).Exec(e.Date, e.LowPrice, e.HighPrice, e.AvgPrice, typeID, regionID)
			}
			tx.Commit()
		}
	}

	if c.Conf.EMDRCrestBridge.Upload {
		u := newUUDIFHeader()
		u.ResultType = "history"
		u.Columns = []string{"date", "orders", "quantity", "low", "high", "average"}

		u.Rowsets = make([]rowsetsUUDIF, 1)

		u.Rowsets[0].RegionID = regionID
		u.Rowsets[0].TypeID = typeID
		u.Rowsets[0].GeneratedAt = time.Now()

		u.Rowsets[0].Rows = make([][]interface{}, len(h.Items))

		for i, e := range h.Items {
			u.Rowsets[0].Rows[i] = make([]interface{}, 6)
			u.Rowsets[0].Rows[i][0] = e.Date + "+00:00"
			u.Rowsets[0].Rows[i][1] = e.OrderCount
			u.Rowsets[0].Rows[i][2] = e.Volume
			u.Rowsets[0].Rows[i][3] = e.LowPrice
			u.Rowsets[0].Rows[i][4] = e.HighPrice
			u.Rowsets[0].Rows[i][5] = e.AvgPrice
		}

		enc, err := json.Marshal(u)
		if err != nil {
			log.Println("EMDRCrestBridge:", err)
		} else {
			postUUDIF(c.Conf.EMDRCrestBridge.URL, enc)
		}
	}
}

func postOrders(sem chan bool, o marketOrders, c *AppContext, buy int, typeID int64, regionID int64) {
	sem <- true
	defer func() { <-sem }()

	if c.Conf.EMDRCrestBridge.Import {
		orderUpdate, err := c.Db.Prepare(`
					INSERT INTO market
						(orderID, price, remainingVolume, typeID, enteredVolume, minVolume, bid, issued, duration, stationID, regionID, systemID, reported)
						VALUES(?,?,?,?,?,?,?,?,?,?,?,?,NOW())
						ON DUPLICATE KEY UPDATE price=VALUES(price),
												remainingVolume=VALUES(remainingVolume),
												issued=VALUES(issued),
												duration=VALUES(duration),
												reported=VALUES(reported),
												done=0;
				`)
		defer orderUpdate.Close()

		orderMark, err := c.Db.Prepare(`
					UPDATE market SET done = 1 WHERE regionID = ? AND typeID =?
				`)
		defer orderMark.Close()

		if err != nil {
			log.Fatalf("EMDRCrestBridge: %s", err)
		} else {

			// Mark orders complete
			tx, err := c.Db.Begin()
			tx.Stmt(orderMark).Exec(regionID, typeID)

			// Add or update orders
			for _, e := range o.Items {
				tx.Stmt(orderUpdate).Exec(e.ID, e.Price, e.Volume, typeID, e.VolumeEntered, e.MinVolume, buy, e.Issued, e.Duration, e.Location.ID, regionID, stations[e.Location.ID])
			}

			err = tx.Commit()

			if err != nil {
				log.Println("EMDRCrestBridge:", err)
			}
		}
	}

	if c.Conf.EMDRCrestBridge.Upload {
		u := newUUDIFHeader()
		u.ResultType = "orders"
		u.Columns = []string{"price", "volRemaining", "range", "orderID", "volEntered", "minVolume", "bid", "issueDate", "duration", "stationID", "solarSystemID"}

		u.Rowsets = make([]rowsetsUUDIF, 1)

		u.Rowsets[0].RegionID = regionID
		u.Rowsets[0].TypeID = typeID
		u.Rowsets[0].GeneratedAt = time.Now()

		u.Rowsets[0].Rows = make([][]interface{}, len(o.Items))

		for i, e := range o.Items {

			var r int
			switch {
			case e.Range == "station":
				r = -1
			case e.Range == "solarsystem":
				r = 0
			case e.Range == "region":
				r = 32767
			default:
				r, _ = strconv.Atoi(e.Range)
			}

			u.Rowsets[0].Rows[i] = make([]interface{}, 11)
			u.Rowsets[0].Rows[i][0] = e.Price
			u.Rowsets[0].Rows[i][1] = e.Volume
			u.Rowsets[0].Rows[i][2] = r
			u.Rowsets[0].Rows[i][3] = e.ID
			u.Rowsets[0].Rows[i][4] = e.VolumeEntered
			u.Rowsets[0].Rows[i][5] = e.MinVolume
			u.Rowsets[0].Rows[i][6] = e.Buy
			u.Rowsets[0].Rows[i][7] = e.Issued + "+00:00"
			u.Rowsets[0].Rows[i][8] = e.Duration
			u.Rowsets[0].Rows[i][9] = e.Location.ID
			u.Rowsets[0].Rows[i][10] = stations[e.Location.ID]
		}

		enc, err := json.Marshal(u)
		if err != nil {
			log.Println("EMDRCrestBridge:", err)
		} else {
			postUUDIF(c.Conf.EMDRCrestBridge.URL, enc)
		}
	}
}

func postUUDIF(url string, j []byte) {
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(j))
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Println("EMDRCrestBridge:", err)
	} else {
		defer resp.Body.Close()

		if resp.Status != "200 OK" {
			body, _ := ioutil.ReadAll(resp.Body)
			log.Println("EMDRCrestBridge:", string(body))
			log.Println("EMDRCrestBridge:", string(resp.Status))
		}
	}
}

func newUUDIFHeader() marketUUDIF {
	n := marketUUDIF{}

	n.Version = "0.1"

	n.Generator.Name = "EveData.Org"
	n.Generator.Version = "0.025a"

	n.UploadKeys = make([]uploadKeysUUDIF, 1)
	n.UploadKeys[0] = uploadKeysUUDIF{"EveData.Org", "TheCheeseIsBree"}

	n.CurrentTime = time.Now()

	return n
}

type rowsetsUUDIF struct {
	GeneratedAt time.Time       `json:"generatedAt"`
	RegionID    int64           `json:"regionID"`
	TypeID      int64           `json:"typeID"`
	Rows        [][]interface{} `json:"rows"`
}

type uploadKeysUUDIF struct {
	Name string `json:"name"`
	Key  string `json:"key"`
}

type marketUUDIF struct {
	ResultType string            `json:"resultType"`
	Version    string            `json:"version"`
	UploadKeys []uploadKeysUUDIF `json:"uploadKeys"`
	Generator  struct {
		Name    string `json:"name"`
		Version string `json:"version"`
	} `json:"generator"`
	Columns     []string       `json:"columns"`
	CurrentTime time.Time      `json:"currentTime"`
	Rowsets     []rowsetsUUDIF `json:"rowsets"`
}

type marketHistory struct {
	TotalCount_Str string
	Items          []struct {
		OrderCount int64
		LowPrice   float64
		HighPrice  float64
		AvgPrice   float64
		Volume     int64
		Date       string
	}
	PageCount  int64
	TotalCount int64
}

type marketOrders struct {
	Items []struct {
		Buy           bool
		Issued        string
		Price         float64
		VolumeEntered int64
		MinVolume     int64
		Volume        int64
		Range         string
		Duration      int64
		ID            int64
		Location      struct {
			ID   int64
			Name string
		}
		Type struct {
			ID   int64
			Name string
		}
	}
	PageCount  int64
	TotalCount int64
}
<commit_msg>Allow posting of empty items so sites are aware of dead items.<commit_after>package evedata

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/jmcvetta/napping"
)

var stations map[int64]int64

func goEMDRCrestBridge(c *AppContext) {

	type marketRegions struct {
		RegionID   int64  `db:"regionID"`
		RegionName string `db:"regionName"`
	}

	type marketTypes struct {
		TypeID   int64  `db:"typeID"`
		TypeName string `db:"typeName"`
	}

	regions := []marketRegion{}
	err := c.Db.Select(&regions, `
		SELECT 	regionID, regionName
		FROM 	mapRegions
		WHERE 	regionID < 11000000
			AND regionID NOT IN(10000017, 10000019, 10000004);
	`)
	if err != nil {
		log.Fatal("EMDRCrestBridge:", err)
	}
	log.Printf("EMDRCrestBridge: Loaded %d Regions", len(regions))

	types := []marketTypes{}
	err = c.Db.Select(&types, `
		SELECT 	typeID, typeName
		FROM 	invTypes
		WHERE 	marketGroupID IS NOT NULL
			AND typeID < 250000;
	`)
	if err != nil {
		log.Fatal("EMDRCrestBridge:", err)
	}
	log.Printf("EMDRCrestBridge: Loaded %d items", len(types))

	stations = make(map[int64]int64)
	rows, err := c.Db.Query(`
		SELECT stationID, solarSystemID
		FROM staStations;
	`)
	for rows.Next() {

		var stationID int64
		var systemID int64

		if err := rows.Scan(&stationID, &systemID); err != nil {
			log.Fatal("EMDRCrestBridge: ", err)
		}
		stations[stationID] = systemID
	}
	rows.Close()

	if err != nil {
		log.Fatal("EMDRCrestBridge: ", err)
	}
	log.Printf("EMDRCrestBridge: Loaded %d stations", len(stations))

	// Throttle Crest Requests
	rate := time.Second / 8
	throttle := time.Tick(rate)

	// semaphore to prevent runaways
	sem := make(chan bool, c.Conf.EMDRCrestBridge.MaxGoRoutines)

	// CREST Session
	crest := napping.Session{}

	for {
		// loop through all regions
		for _, r := range regions {
			// and each item per region
			for _, t := range types {
				<-throttle // impliment throttle

				sem <- true
				go func() {
					defer func() { <-sem }()
					// Process Market History
					h := marketHistory{}
					url := fmt.Sprintf("https://public-crest.eveonline.com/market/%d/types/%d/history/", r.RegionID, t.TypeID)
					response, err := crest.Get(url, nil, &h, nil)
					if err != nil {
						log.Printf("EMDRCrestBridge: %s", err)
						return
					}
					if response.Status() == 200 {
						go postHistory(sem, h, c, t.TypeID, r.RegionID)

					}
				}()

				sem <- true
				go func() {
					defer func() { <-sem }()
					// Process Market Buy Orders
					b := marketOrders{}
					url := fmt.Sprintf("https://public-crest.eveonline.com/market/%d/orders/buy/?type=https://public-crest.eveonline.com/types/%d/", r.RegionID, t.TypeID)
					response, err := crest.Get(url, nil, &b, nil)
					if err != nil {
						log.Printf("EMDRCrestBridge: %s", err)
						return
					}
					if response.Status() == 200 {

						go postOrders(sem, b, c, 1, t.TypeID, r.RegionID)

					}
				}()

				sem <- true
				go func() {
					defer func() { <-sem }()
					// Process Market Sell Orders
					s := marketOrders{}
					url := fmt.Sprintf("https://public-crest.eveonline.com/market/%d/orders/sell/?type=https://public-crest.eveonline.com/types/%d/", r.RegionID, t.TypeID)
					response, err := crest.Get(url, nil, &s, nil)
					if err != nil {
						log.Printf("EMDRCrestBridge: %s", err)
						return
					}
					if response.Status() == 200 {

						go postOrders(sem, s, c, 0, t.TypeID, r.RegionID)

					}
				}()
			}
		}
	}
}

func postHistory(sem chan bool, h marketHistory, c *AppContext, typeID int64, regionID int64) {
	sem <- true
	defer func() { <-sem }()

	if c.Conf.EMDRCrestBridge.Import {
		historyUpdate, err := c.Db.Prepare(`
			INSERT IGNORE INTO market_history
				(date, low, high, mean, quantity, orders, itemID, regionID)
				VALUES(?,?,?,?,?,?,?,?);
		`)
		defer historyUpdate.Close()
		if err != nil {
			log.Printf("EMDRCrestBridge: %s", err)
		} else {
			tx, _ := c.Db.Begin()
			for _, e := range h.Items {
				tx.Stmt(historyUpdate).Exec(e.Date, e.LowPrice, e.HighPrice, e.AvgPrice, typeID, regionID)
			}
			tx.Commit()
		}
	}

	if c.Conf.EMDRCrestBridge.Upload {
		u := newUUDIFHeader()
		u.ResultType = "history"
		u.Columns = []string{"date", "orders", "quantity", "low", "high", "average"}

		u.Rowsets = make([]rowsetsUUDIF, 1)

		u.Rowsets[0].RegionID = regionID
		u.Rowsets[0].TypeID = typeID
		u.Rowsets[0].GeneratedAt = time.Now()

		u.Rowsets[0].Rows = make([][]interface{}, len(h.Items))

		for i, e := range h.Items {
			u.Rowsets[0].Rows[i] = make([]interface{}, 6)
			u.Rowsets[0].Rows[i][0] = e.Date + "+00:00"
			u.Rowsets[0].Rows[i][1] = e.OrderCount
			u.Rowsets[0].Rows[i][2] = e.Volume
			u.Rowsets[0].Rows[i][3] = e.LowPrice
			u.Rowsets[0].Rows[i][4] = e.HighPrice
			u.Rowsets[0].Rows[i][5] = e.AvgPrice
		}

		enc, err := json.Marshal(u)
		if err != nil {
			log.Println("EMDRCrestBridge:", err)
		} else {
			postUUDIF(c.Conf.EMDRCrestBridge.URL, enc)
		}
	}
}

func postOrders(sem chan bool, o marketOrders, c *AppContext, buy int, typeID int64, regionID int64) {
	sem <- true
	defer func() { <-sem }()

	if c.Conf.EMDRCrestBridge.Import {
		orderUpdate, err := c.Db.Prepare(`
					INSERT INTO market
						(orderID, price, remainingVolume, typeID, enteredVolume, minVolume, bid, issued, duration, stationID, regionID, systemID, reported)
						VALUES(?,?,?,?,?,?,?,?,?,?,?,?,NOW())
						ON DUPLICATE KEY UPDATE price=VALUES(price),
												remainingVolume=VALUES(remainingVolume),
												issued=VALUES(issued),
												duration=VALUES(duration),
												reported=VALUES(reported),
												done=0;
				`)
		defer orderUpdate.Close()

		orderMark, err := c.Db.Prepare(`
					UPDATE market SET done = 1 WHERE regionID = ? AND typeID =?
				`)
		defer orderMark.Close()

		if err != nil {
			log.Fatalf("EMDRCrestBridge: %s", err)
		} else {

			// Mark orders complete
			tx, err := c.Db.Begin()
			tx.Stmt(orderMark).Exec(regionID, typeID)

			// Add or update orders
			for _, e := range o.Items {
				tx.Stmt(orderUpdate).Exec(e.ID, e.Price, e.Volume, typeID, e.VolumeEntered, e.MinVolume, buy, e.Issued, e.Duration, e.Location.ID, regionID, stations[e.Location.ID])
			}

			err = tx.Commit()

			if err != nil {
				log.Println("EMDRCrestBridge:", err)
			}
		}
	}

	if c.Conf.EMDRCrestBridge.Upload {
		u := newUUDIFHeader()
		u.ResultType = "orders"
		u.Columns = []string{"price", "volRemaining", "range", "orderID", "volEntered", "minVolume", "bid", "issueDate", "duration", "stationID", "solarSystemID"}

		u.Rowsets = make([]rowsetsUUDIF, 1)

		u.Rowsets[0].RegionID = regionID
		u.Rowsets[0].TypeID = typeID
		u.Rowsets[0].GeneratedAt = time.Now()

		u.Rowsets[0].Rows = make([][]interface{}, len(o.Items))

		for i, e := range o.Items {

			var r int
			switch {
			case e.Range == "station":
				r = -1
			case e.Range == "solarsystem":
				r = 0
			case e.Range == "region":
				r = 32767
			default:
				r, _ = strconv.Atoi(e.Range)
			}

			u.Rowsets[0].Rows[i] = make([]interface{}, 11)
			u.Rowsets[0].Rows[i][0] = e.Price
			u.Rowsets[0].Rows[i][1] = e.Volume
			u.Rowsets[0].Rows[i][2] = r
			u.Rowsets[0].Rows[i][3] = e.ID
			u.Rowsets[0].Rows[i][4] = e.VolumeEntered
			u.Rowsets[0].Rows[i][5] = e.MinVolume
			u.Rowsets[0].Rows[i][6] = e.Buy
			u.Rowsets[0].Rows[i][7] = e.Issued + "+00:00"
			u.Rowsets[0].Rows[i][8] = e.Duration
			u.Rowsets[0].Rows[i][9] = e.Location.ID
			u.Rowsets[0].Rows[i][10] = stations[e.Location.ID]
		}

		enc, err := json.Marshal(u)
		if err != nil {
			log.Println("EMDRCrestBridge:", err)
		} else {
			postUUDIF(c.Conf.EMDRCrestBridge.URL, enc)
		}
	}
}

func postUUDIF(url string, j []byte) {
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(j))
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Println("EMDRCrestBridge:", err)
	} else {
		defer resp.Body.Close()

		if resp.Status != "200 OK" {
			body, _ := ioutil.ReadAll(resp.Body)
			log.Println("EMDRCrestBridge:", string(body))
			log.Println("EMDRCrestBridge:", string(resp.Status))
		}
	}
}

func newUUDIFHeader() marketUUDIF {
	n := marketUUDIF{}

	n.Version = "0.1"

	n.Generator.Name = "EveData.Org"
	n.Generator.Version = "0.025a"

	n.UploadKeys = make([]uploadKeysUUDIF, 1)
	n.UploadKeys[0] = uploadKeysUUDIF{"EveData.Org", "TheCheeseIsBree"}

	n.CurrentTime = time.Now()

	return n
}

type rowsetsUUDIF struct {
	GeneratedAt time.Time       `json:"generatedAt"`
	RegionID    int64           `json:"regionID"`
	TypeID      int64           `json:"typeID"`
	Rows        [][]interface{} `json:"rows"`
}

type uploadKeysUUDIF struct {
	Name string `json:"name"`
	Key  string `json:"key"`
}

type marketUUDIF struct {
	ResultType string            `json:"resultType"`
	Version    string            `json:"version"`
	UploadKeys []uploadKeysUUDIF `json:"uploadKeys"`
	Generator  struct {
		Name    string `json:"name"`
		Version string `json:"version"`
	} `json:"generator"`
	Columns     []string       `json:"columns"`
	CurrentTime time.Time      `json:"currentTime"`
	Rowsets     []rowsetsUUDIF `json:"rowsets"`
}

type marketHistory struct {
	TotalCount_Str string
	Items          []struct {
		OrderCount int64
		LowPrice   float64
		HighPrice  float64
		AvgPrice   float64
		Volume     int64
		Date       string
	}
	PageCount  int64
	TotalCount int64
}

type marketOrders struct {
	Items []struct {
		Buy           bool
		Issued        string
		Price         float64
		VolumeEntered int64
		MinVolume     int64
		Volume        int64
		Range         string
		Duration      int64
		ID            int64
		Location      struct {
			ID   int64
			Name string
		}
		Type struct {
			ID   int64
			Name string
		}
	}
	PageCount  int64
	TotalCount int64
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"os/exec"
	"regexp"
	"strings"
	"time"

	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	pb "github.com/brotherlogic/gobuildslave/proto"
	pbgs "github.com/brotherlogic/goserver/proto"
	"github.com/brotherlogic/goserver/utils"
)

// RunJob - runs the job
func (s *Server) RunJob(ctx context.Context, req *pb.RunRequest) (*pb.RunResponse, error) {
	if !req.GetJob().GetBreakout() &&
		(s.Registry.Identifier == "clust6" ||
			s.Registry.Identifier == "clust3" ||
			s.Registry.Identifier == "clust7" ||
			s.Registry.Identifier == "clust8" ||
			s.Registry.Identifier == "clust4") {
		return &pb.RunResponse{}, status.Errorf(codes.FailedPrecondition, "we only run the basic set of jobs")
	}

	if req.GetBits() > 0 && s.Bits != int(req.GetBits()) {
		return &pb.RunResponse{}, status.Errorf(codes.FailedPrecondition, "Cannot run %v bits on this server", req.GetBits())
	}

	if !s.doesBuild && !req.Job.Breakout {
		return &pb.RunResponse{}, status.Errorf(codes.FailedPrecondition, "Refusing to build")
	}
	s.nMut.Lock()
	defer s.nMut.Unlock()
	if _, ok := s.njobs[req.GetJob().GetName()]; ok {
		return &pb.RunResponse{}, fmt.Errorf("Already running this job!")
	}

	if len(s.njobs) > s.maxJobs && !req.GetJob().GetBreakout() {
		return nil, status.Errorf(codes.FailedPrecondition, "We're running %v jobs, can't run no more", len(s.njobs))
	}

	s.CtxLog(ctx, "Running %v")

	s.njobs[req.GetJob().GetName()] = &pb.JobAssignment{Job: req.GetJob(), LastTransitionTime: time.Now().Unix(), Bits: int32(s.Bits)}
	go s.nmonitor(s.njobs[req.GetJob().GetName()])

	return &pb.RunResponse{}, nil
}

// KillJob - kills the job
func (s *Server) KillJob(ctx context.Context, req *pb.KillRequest) (*pb.KillResponse, error) {
	s.nMut.Lock()
	defer s.nMut.Unlock()

	if _, ok := s.njobs[req.GetJob().GetName()]; !ok {
		return nil, fmt.Errorf("Job was not running")
	}

	s.njobs[req.GetJob().GetName()].State = pb.State_KILLING
	return &pb.KillResponse{}, nil
}

//UpdateJob - updates the job
func (s *Server) UpdateJob(ctx context.Context, req *pb.UpdateRequest) (*pb.UpdateResponse, error) {
	s.nMut.Lock()
	defer s.nMut.Unlock()
	if _, ok := s.njobs[req.GetJob().GetName()]; !ok {
		return nil, fmt.Errorf("Job was not running")
	}

	s.njobs[req.GetJob().GetName()].State = pb.State_UPDATE_STARTING
	return &pb.UpdateResponse{}, nil
}

// ListJobs - lists the jobs
func (s *Server) ListJobs(ctx context.Context, req *pb.ListRequest) (*pb.ListResponse, error) {
	s.nMut.Lock()
	defer s.nMut.Unlock()
	resp := &pb.ListResponse{}
	for _, job := range s.njobs {
		resp.Jobs = append(resp.Jobs, job)
	}
	return resp, nil
}

func extractBitRate(output string) (string, string) {
	matcher := regexp.MustCompile("Rate=(.*?) ")
	matches := matcher.FindStringSubmatch(output)

	matcher2 := regexp.MustCompile("Access Point. ([A-F0-9:]*)")
	matches2 := matcher2.FindStringSubmatch(output)
	if len(matches) > 0 && len(matches2) > 0 {
		return strings.TrimRight(matches[1], " "), strings.TrimRight(matches2[1], " ")
	}
	return "", ""
}

// SlaveConfig gets the config for this slave
func (s *Server) SlaveConfig(ctx context.Context, req *pb.ConfigRequest) (*pb.ConfigResponse, error) {
	disks := s.disker.getDisks()
	requirements := make([]*pb.Requirement, 0)
	for _, disk := range disks {
		requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_DISK, Properties: disk})
	}
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_SERVER, Properties: s.Registry.Identifier})
	if s.Registry.Identifier == "argon" {
		requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_EXTERNAL, Properties: "external_ready"})
	}

	data, err := exec.Command("/usr/bin/lsusb").Output()
	if err != nil {
		return nil, fmt.Errorf("error listing usb components: %v", err)
	}
	s.CtxLog(ctx, fmt.Sprintf("USBRES: %v", string(data)))
	if strings.Contains(string(data), "TSP100II") {
		requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_RECEIPT_PRINTER})
	}

	out, _ := exec.Command("/sbin/iwconfig").Output()
	br, ap := extractBitRate(string(out))
	s.accessPoint = ap
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_NETWORK, Properties: br})
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_ACCESS_POINT, Properties: ap})

	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_BITS, Properties: fmt.Sprintf("%v", s.Bits)})

	out, _ = exec.Command("cat", "/sys/firmware/devicetree/base/model").Output()
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_HOST_TYPE, Properties: string(out)})

	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_ZONE, Properties: s.Registry.Zone})

	return &pb.ConfigResponse{Config: &pb.SlaveConfig{Requirements: requirements}}, nil
}

func (s *Server) FullShutdown(ctx context.Context, req *pb.ShutdownRequest) (*pb.ShutdownResponse, error) {
	defer func() {
		time.Sleep(time.Minute)

		exec.Command("sudo", "shutdown", "-h", "now")
	}()

	jobs, err := s.ListJobs(ctx, &pb.ListRequest{})
	if err != nil {
		return nil, err
	}

	s.CtxLog(ctx, fmt.Sprintf("Shutting down %v jobs", len(jobs.GetJobs())))

	for _, job := range jobs.GetJobs() {
		if job.GetPort() != 0 {
			conn, err := utils.LFDial(fmt.Sprintf("%v:%v", job.GetHost(), job.GetPort()))
			if err != nil {
				return nil, err
			}
			s.CtxLog(ctx, fmt.Sprintf("Calling shutdown on %v", job))
			gsclient := pbgs.NewGoserverServiceClient(conn)
			_, err = gsclient.Shutdown(ctx, &pbgs.ShutdownRequest{})
			if err != nil {
				return nil, err
			}
		} else {
			s.CtxLog(ctx, fmt.Sprintf("Not shutting down %v", job))
		}
	}

	return &pb.ShutdownResponse{}, nil
}
<commit_msg>Actually shutdown<commit_after>package main

import (
	"context"
	"fmt"
	"os/exec"
	"regexp"
	"strings"
	"time"

	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	pb "github.com/brotherlogic/gobuildslave/proto"
	pbgs "github.com/brotherlogic/goserver/proto"
	"github.com/brotherlogic/goserver/utils"
)

// RunJob - runs the job
func (s *Server) RunJob(ctx context.Context, req *pb.RunRequest) (*pb.RunResponse, error) {
	if !req.GetJob().GetBreakout() &&
		(s.Registry.Identifier == "clust6" ||
			s.Registry.Identifier == "clust3" ||
			s.Registry.Identifier == "clust7" ||
			s.Registry.Identifier == "clust8" ||
			s.Registry.Identifier == "clust4") {
		return &pb.RunResponse{}, status.Errorf(codes.FailedPrecondition, "we only run the basic set of jobs")
	}

	if req.GetBits() > 0 && s.Bits != int(req.GetBits()) {
		return &pb.RunResponse{}, status.Errorf(codes.FailedPrecondition, "Cannot run %v bits on this server", req.GetBits())
	}

	if !s.doesBuild && !req.Job.Breakout {
		return &pb.RunResponse{}, status.Errorf(codes.FailedPrecondition, "Refusing to build")
	}
	s.nMut.Lock()
	defer s.nMut.Unlock()
	if _, ok := s.njobs[req.GetJob().GetName()]; ok {
		return &pb.RunResponse{}, fmt.Errorf("Already running this job!")
	}

	if len(s.njobs) > s.maxJobs && !req.GetJob().GetBreakout() {
		return nil, status.Errorf(codes.FailedPrecondition, "We're running %v jobs, can't run no more", len(s.njobs))
	}

	s.CtxLog(ctx, "Running %v")

	s.njobs[req.GetJob().GetName()] = &pb.JobAssignment{Job: req.GetJob(), LastTransitionTime: time.Now().Unix(), Bits: int32(s.Bits)}
	go s.nmonitor(s.njobs[req.GetJob().GetName()])

	return &pb.RunResponse{}, nil
}

// KillJob - kills the job
func (s *Server) KillJob(ctx context.Context, req *pb.KillRequest) (*pb.KillResponse, error) {
	s.nMut.Lock()
	defer s.nMut.Unlock()

	if _, ok := s.njobs[req.GetJob().GetName()]; !ok {
		return nil, fmt.Errorf("Job was not running")
	}

	s.njobs[req.GetJob().GetName()].State = pb.State_KILLING
	return &pb.KillResponse{}, nil
}

//UpdateJob - updates the job
func (s *Server) UpdateJob(ctx context.Context, req *pb.UpdateRequest) (*pb.UpdateResponse, error) {
	s.nMut.Lock()
	defer s.nMut.Unlock()
	if _, ok := s.njobs[req.GetJob().GetName()]; !ok {
		return nil, fmt.Errorf("Job was not running")
	}

	s.njobs[req.GetJob().GetName()].State = pb.State_UPDATE_STARTING
	return &pb.UpdateResponse{}, nil
}

// ListJobs - lists the jobs
func (s *Server) ListJobs(ctx context.Context, req *pb.ListRequest) (*pb.ListResponse, error) {
	s.nMut.Lock()
	defer s.nMut.Unlock()
	resp := &pb.ListResponse{}
	for _, job := range s.njobs {
		resp.Jobs = append(resp.Jobs, job)
	}
	return resp, nil
}

func extractBitRate(output string) (string, string) {
	matcher := regexp.MustCompile("Rate=(.*?) ")
	matches := matcher.FindStringSubmatch(output)

	matcher2 := regexp.MustCompile("Access Point. ([A-F0-9:]*)")
	matches2 := matcher2.FindStringSubmatch(output)
	if len(matches) > 0 && len(matches2) > 0 {
		return strings.TrimRight(matches[1], " "), strings.TrimRight(matches2[1], " ")
	}
	return "", ""
}

// SlaveConfig gets the config for this slave
func (s *Server) SlaveConfig(ctx context.Context, req *pb.ConfigRequest) (*pb.ConfigResponse, error) {
	disks := s.disker.getDisks()
	requirements := make([]*pb.Requirement, 0)
	for _, disk := range disks {
		requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_DISK, Properties: disk})
	}
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_SERVER, Properties: s.Registry.Identifier})
	if s.Registry.Identifier == "argon" {
		requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_EXTERNAL, Properties: "external_ready"})
	}

	data, err := exec.Command("/usr/bin/lsusb").Output()
	if err != nil {
		return nil, fmt.Errorf("error listing usb components: %v", err)
	}
	s.CtxLog(ctx, fmt.Sprintf("USBRES: %v", string(data)))
	if strings.Contains(string(data), "TSP100II") {
		requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_RECEIPT_PRINTER})
	}

	out, _ := exec.Command("/sbin/iwconfig").Output()
	br, ap := extractBitRate(string(out))
	s.accessPoint = ap
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_NETWORK, Properties: br})
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_ACCESS_POINT, Properties: ap})

	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_BITS, Properties: fmt.Sprintf("%v", s.Bits)})

	out, _ = exec.Command("cat", "/sys/firmware/devicetree/base/model").Output()
	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_HOST_TYPE, Properties: string(out)})

	requirements = append(requirements, &pb.Requirement{Category: pb.RequirementCategory_ZONE, Properties: s.Registry.Zone})

	return &pb.ConfigResponse{Config: &pb.SlaveConfig{Requirements: requirements}}, nil
}

func (s *Server) FullShutdown(ctx context.Context, req *pb.ShutdownRequest) (*pb.ShutdownResponse, error) {
	defer func() {
		time.Sleep(time.Minute)

		exec.Command("sudo", "shutdown", "-h", "now").Run()
	}()

	jobs, err := s.ListJobs(ctx, &pb.ListRequest{})
	if err != nil {
		return nil, err
	}

	s.CtxLog(ctx, fmt.Sprintf("Shutting down %v jobs", len(jobs.GetJobs())))

	for _, job := range jobs.GetJobs() {
		if job.GetPort() != 0 {
			conn, err := utils.LFDial(fmt.Sprintf("%v:%v", job.GetHost(), job.GetPort()))
			if err != nil {
				return nil, err
			}
			s.CtxLog(ctx, fmt.Sprintf("Calling shutdown on %v", job))
			gsclient := pbgs.NewGoserverServiceClient(conn)
			_, err = gsclient.Shutdown(ctx, &pbgs.ShutdownRequest{})
			if err != nil {
				s.CtxLog(ctx, fmt.Sprintf("Failed shutdown: %v", err))
			}
		} else {
			s.CtxLog(ctx, fmt.Sprintf("Not shutting down %v", job))
		}
	}

	return &pb.ShutdownResponse{}, nil
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"net/http"
	"strconv"

	eram "github.com/Onefootball/entity-rest-api/manager"
	"github.com/ant0ine/go-json-rest/rest"
)

const Offset string = "0"
const Limit string = "10"
const OrderDir string = "ASC"

type EntityRestAPI struct {
	em *eram.EntityDbManager
}

func NewEntityRestAPI(em *eram.EntityDbManager) *EntityRestAPI {
	return &EntityRestAPI{
		em,
	}
}

func (api *EntityRestAPI) GetAllEntities(w rest.ResponseWriter, r *rest.Request) {
	entity := r.PathParam("entity")
	qs := r.Request.URL.Query()

	limit, offset, orderBy, orderDir := qs.Get("_perPage"), qs.Get("_page"), qs.Get("_sortField"), qs.Get("_sortDir")

	qs.Del("_perPage")
	qs.Del("_page")
	qs.Del("_sortField")
	qs.Del("_sortDir")

	filterParams := make(map[string]string)

	// remaining GET parameters are used to filter the result
	for filterName, _ := range qs {
		filterParams[filterName] = qs.Get(filterName)
	}

	if offset == "" {
		offset = Offset
	}

	if limit == "" {
		limit = Limit
	}

	if orderBy == "" {
		orderBy = api.em.GetIdColumn(entity)
	}

	if orderDir == "" {
		orderDir = OrderDir
	}

	allResults, count, dbErr := api.em.GetEntities(entity, filterParams, limit, offset, orderBy, orderDir)

	if dbErr != nil {
		rest.Error(w, dbErr.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Access-Control-Expose-Headers", "X-Total-Count")
	w.Header().Set("X-Total-Count", fmt.Sprintf("%d", count))

	w.WriteJson(allResults)
}

func (api *EntityRestAPI) GetEntity(w rest.ResponseWriter, r *rest.Request) {
	id := r.PathParam("id")
	entity := r.PathParam("entity")
	result, err := api.em.GetEntity(entity, id)
	if err != nil {
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	} else if len(result) <= 0 {
		rest.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	w.WriteJson(result)
}

func (api *EntityRestAPI) PostEntity(w rest.ResponseWriter, r *rest.Request) {
	entity := r.PathParam("entity")
	postData := map[string]interface{}{}
	if err := r.DecodeJsonPayload(&postData); err != nil {
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	newId, err := api.em.PostEntity(entity, postData)
	if err != nil {
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	insertedEntity, err := api.em.GetEntity(entity, strconv.FormatInt(newId, 10))
	if err != nil {
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Location", fmt.Sprintf("%s/%d", entity, insertedEntity[api.em.GetIdColumn(entity)]))

	w.WriteHeader(http.StatusCreated)
	w.WriteJson(insertedEntity)
}

func (api *EntityRestAPI) PutEntity(w rest.ResponseWriter, r *rest.Request) {
	id := r.PathParam("id")
	entity := r.PathParam("entity")
	updated := map[string]interface{}{}
	if err := r.DecodeJsonPayload(&updated); err != nil {
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	rowsAffected, updatedEntity, err := api.em.UpdateEntity(entity, id, updated)
	if err != nil {
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	} else if len(updatedEntity) <= 0 {
		rest.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	if rowsAffected == 0 {
		w.WriteHeader(http.StatusNoContent)
	} else {
		w.WriteHeader(http.StatusOK)
	}

	w.WriteJson(updatedEntity)
}

func (api *EntityRestAPI) DeleteEntity(w rest.ResponseWriter, r *rest.Request) {
	id := r.PathParam("id")
	entity := r.PathParam("entity")
	rowsAffected, err := api.em.DeleteEntity(entity, id)
	if err != nil {
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	if rowsAffected == 0 {
		w.WriteHeader(http.StatusNotFound)
	} else {
		w.WriteHeader(http.StatusOK)
	}
}
<commit_msg>Exposing X-Status-Code and X-Entity-ID<commit_after>package api

import (
	"fmt"
	"net/http"
	"strconv"

	eram "github.com/Onefootball/entity-rest-api/manager"
	"github.com/ant0ine/go-json-rest/rest"
)

const (
	Offset           = "0"
	Limit            = "10"
	OrderDir         = "ASC"
	StatusCodeHeader = "X-Status-Code"
	EntityIDHeader   = "X-Entity-ID"
	LocationHeader   = "Location"
)

type EntityRestAPI struct {
	em *eram.EntityDbManager
}

func NewEntityRestAPI(em *eram.EntityDbManager) *EntityRestAPI {
	return &EntityRestAPI{
		em,
	}
}

func (api *EntityRestAPI) GetAllEntities(w rest.ResponseWriter, r *rest.Request) {
	entity := r.PathParam("entity")
	qs := r.Request.URL.Query()

	limit, offset, orderBy, orderDir := qs.Get("_perPage"), qs.Get("_page"), qs.Get("_sortField"), qs.Get("_sortDir")

	qs.Del("_perPage")
	qs.Del("_page")
	qs.Del("_sortField")
	qs.Del("_sortDir")

	filterParams := make(map[string]string)

	// remaining GET parameters are used to filter the result
	for filterName, _ := range qs {
		filterParams[filterName] = qs.Get(filterName)
	}

	if offset == "" {
		offset = Offset
	}

	if limit == "" {
		limit = Limit
	}

	if orderBy == "" {
		orderBy = api.em.GetIdColumn(entity)
	}

	if orderDir == "" {
		orderDir = OrderDir
	}

	allResults, count, dbErr := api.em.GetEntities(entity, filterParams, limit, offset, orderBy, orderDir)

	if dbErr != nil {
		rest.Error(w, dbErr.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Access-Control-Expose-Headers", "X-Total-Count")
	w.Header().Set("X-Total-Count", fmt.Sprintf("%d", count))

	w.WriteJson(allResults)
}

func (api *EntityRestAPI) GetEntity(w rest.ResponseWriter, r *rest.Request) {
	id := r.PathParam("id")
	entity := r.PathParam("entity")
	result, err := api.em.GetEntity(entity, id)
	if err != nil {
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	} else if len(result) <= 0 {
		rest.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	w.WriteJson(result)
}

func (api *EntityRestAPI) PostEntity(w rest.ResponseWriter, r *rest.Request) {
	w.Header().Add("Access-Control-Expose-Headers", StatusCodeHeader)
	w.Header().Add("Access-Control-Expose-Headers", EntityIDHeader)

	entity := r.PathParam("entity")
	postData := map[string]interface{}{}
	if err := r.DecodeJsonPayload(&postData); err != nil {
		w.Header().Set(StatusCodeHeader, fmt.Sprintf("%d", http.StatusInternalServerError))
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	newId, err := api.em.PostEntity(entity, postData)
	if err != nil {
		w.Header().Set(StatusCodeHeader, fmt.Sprintf("%d", http.StatusInternalServerError))
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	insertedEntity, err := api.em.GetEntity(entity, strconv.FormatInt(newId, 10))
	if err != nil {
		w.Header().Set(StatusCodeHeader, fmt.Sprintf("%d", http.StatusInternalServerError))
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set(LocationHeader, fmt.Sprintf("%s/%s", entity, insertedEntity[api.em.GetIdColumn(entity)]))
	w.Header().Set(StatusCodeHeader, fmt.Sprintf("%d", http.StatusCreated))
	w.Header().Set(EntityIDHeader, fmt.Sprintf("%s", insertedEntity[api.em.GetIdColumn(entity)]))

	w.WriteHeader(http.StatusCreated)
	w.WriteJson(insertedEntity)
}

func (api *EntityRestAPI) PutEntity(w rest.ResponseWriter, r *rest.Request) {
	id := r.PathParam("id")
	entity := r.PathParam("entity")
	updated := map[string]interface{}{}
	if err := r.DecodeJsonPayload(&updated); err != nil {
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	rowsAffected, updatedEntity, err := api.em.UpdateEntity(entity, id, updated)
	if err != nil {
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	} else if len(updatedEntity) <= 0 {
		rest.Error(w, "Not Found", http.StatusNotFound)
		return
	}

	if rowsAffected == 0 {
		w.WriteHeader(http.StatusNoContent)
	} else {
		w.WriteHeader(http.StatusOK)
	}

	w.WriteJson(updatedEntity)
}

func (api *EntityRestAPI) DeleteEntity(w rest.ResponseWriter, r *rest.Request) {
	id := r.PathParam("id")
	entity := r.PathParam("entity")
	rowsAffected, err := api.em.DeleteEntity(entity, id)
	if err != nil {
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	if rowsAffected == 0 {
		w.WriteHeader(http.StatusNotFound)
	} else {
		w.WriteHeader(http.StatusOK)
	}
}
<|endoftext|>


<commit_before>package repo

import (
	"context"
	"fmt"
	"net/url"
	"strings"

	"github.com/bleenco/abstruse/pkg/gitscm"
	"github.com/bleenco/abstruse/server/core"
	"github.com/drone/go-scm/scm"
	"github.com/jinzhu/gorm"
)

// New returns a new RepositoryStore.
func New(db *gorm.DB) core.RepositoryStore {
	return repositoryStore{db}
}

type repositoryStore struct {
	db *gorm.DB
}

func (s repositoryStore) Find(id, userID uint) (core.Repository, error) {
	var repo core.Repository

	db := s.db.Preload("Provider")
	db = db.Joins("LEFT JOIN permissions ON permissions.repository_id = repositories.id").
		Joins("LEFT JOIN teams ON teams.id = permissions.team_id").
		Joins("LEFT JOIN team_users ON team_users.team_id = teams.id")
	db = db.Where("repositories.id = ? AND repositories.user_id = ?", id, userID).
		Or("repositories.id = ? AND team_users.user_id = ? AND permissions.read = ?", id, userID, true)

	err := db.First(&repo).Error
	if err != nil {
		return repo, err
	}
	repo.Perms = s.GetPermissions(repo.ID, userID)

	return repo, err
}

func (s repositoryStore) FindUID(uid string) (core.Repository, error) {
	var repo core.Repository
	err := s.db.Where("uid = ?", uid).Preload("Provider").First(&repo).Error
	return repo, err
}

func (s repositoryStore) FindClone(clone string) (core.Repository, error) {
	var repo core.Repository
	err := s.db.Where("clone = ?", clone).Preload("Provider").First(&repo).Error
	return repo, err
}

func (s repositoryStore) FindToken(token string) (*core.Repository, error) {
	var repo core.Repository
	err := s.db.Where("token = ?", token).First(&repo).Error
	return &repo, err
}

func (s repositoryStore) List(filters core.RepositoryFilter) ([]core.Repository, int, error) {
	var repos []core.Repository
	var count int
	var err error
	keyword := fmt.Sprintf("%%%s%%", filters.Keyword)

	db := s.db.Preload("Provider")

	db = db.
		Joins("LEFT JOIN permissions ON permissions.repository_id = repositories.id").
		Joins("LEFT JOIN teams ON teams.id = permissions.team_id").
		Joins("LEFT JOIN team_users ON team_users.team_id = teams.id")

	if filters.UserID != 0 {
		db = db.Where("repositories.user_id = ? AND repositories.full_name LIKE ?", filters.UserID, keyword).
			Or("team_users.user_id = ? AND permissions.read = ? AND repositories.full_name LIKE ?", filters.UserID, true, keyword)
	}

	if filters.Limit != 0 {
		db = db.Limit(filters.Limit)
	}

	if filters.Offset != 0 {
		db = db.Offset(filters.Offset)
	}

	err = db.Order("active desc, name asc").Group("repositories.id").Find(&repos).Limit(-1).Offset(-1).Count(&count).Error
	if err != nil || count == 0 {
		return repos, count, err
	}

	for i, repo := range repos {
		repos[i].Perms = s.GetPermissions(repo.ID, filters.UserID)
	}

	return repos, count, err
}

func (s repositoryStore) Create(repo core.Repository) error {
	return s.db.Create(&repo).Error
}

func (s repositoryStore) Update(repo core.Repository) error {
	return s.db.Model(&repo).Updates(&repo).Error
}

func (s repositoryStore) CreateOrUpdate(repo core.Repository) error {
	if s.db.Where("uid = ? AND clone = ?", repo.UID, repo.Clone).First(&repo).RecordNotFound() {
		return s.db.Create(&repo).Error
	}

	return s.db.Model(&repo).Where("uid = ? AND clone = ?", repo.UID, repo.Clone).Updates(&repo).Error
}

func (s repositoryStore) Delete(repo core.Repository) error {
	return s.db.Delete(&repo).Error
}

func (s repositoryStore) SetActive(id uint, active bool) error {
	var repo core.Repository
	if s.db.Where("id = ?", id).First(&repo).RecordNotFound() {
		return fmt.Errorf("repository not found")
	}

	return s.db.Model(&repo).Update("active", active).Error
}

func (s repositoryStore) GetPermissions(id, userID uint) core.Perms {
	perms := core.Perms{Read: false, Write: false, Exec: false}

	var user core.User
	err := s.db.Model(&user).Where("id = ?", userID).First(&user).Error
	if err != nil {
		return perms
	}
	if user.Role == "admin" {
		return core.Perms{Read: true, Write: true, Exec: true}
	}

	var repo core.Repository
	if err := s.db.Where("id = ?", id).First(&repo).Error; err == nil {
		if repo.UserID == userID {
			perms.Read = true
			perms.Write = true
			perms.Exec = true
			return perms
		}
	}

	db := s.db

	db = db.
		Joins("LEFT JOIN team_users ON team_users.team_id = permissions.team_id").
		Where("team_users.user_id = ? AND permissions.repository_id = ?", userID, id)

	var permissions []*core.Permission
	err = db.Find(&permissions).Error
	if err != nil {
		return perms
	}

	r, w, x := false, false, false

	for _, p := range permissions {
		if p.Read {
			r = true
		}
		if p.Write {
			w = true
		}
		if p.Exec {
			x = true
		}
	}

	return core.Perms{Read: r, Write: w, Exec: x}
}

func (s repositoryStore) ListHooks(id, userID uint) ([]*scm.Hook, error) {
	repo, err := s.Find(id, userID)
	if err != nil {
		return nil, err
	}

	gitscm, err := gitscm.New(context.Background(), repo.Provider.Name, repo.Provider.URL, repo.Provider.AccessToken)
	if err != nil {
		return nil, err
	}

	hooks, err := gitscm.ListHooks(repo.FullName)
	if err != nil {
		return nil, err
	}

	return filterHooks(hooks, repo.Provider), nil
}

func (s repositoryStore) CreateHook(id, userID uint, data gitscm.HookForm) error {
	if err := s.DeleteHooks(id, userID); err != nil {
		return err
	}

	repo, err := s.Find(id, userID)
	if err != nil {
		return err
	}

	gitscm, err := gitscm.New(context.Background(), repo.Provider.Name, repo.Provider.URL, repo.Provider.AccessToken)
	if err != nil {
		return err
	}

	if !data.Branch && !data.PullRequest && !data.Push && !data.Tag {
		return nil
	}

	target := fmt.Sprintf("%s/webhooks", repo.Provider.Host)
	_, err = gitscm.CreateHook(repo.FullName, target, repo.Provider.Name, repo.Provider.Secret, data)
	return err
}

func (s repositoryStore) DeleteHooks(id, userID uint) error {
	repo, err := s.Find(id, userID)
	if err != nil {
		return err
	}

	gitscm, err := gitscm.New(context.Background(), repo.Provider.Name, repo.Provider.URL, repo.Provider.AccessToken)
	if err != nil {
		return err
	}

	hooks, err := gitscm.ListHooks(repo.FullName)
	if err != nil {
		return err
	}

	webhooks := filterHooks(hooks, repo.Provider)

	for _, webhook := range webhooks {
		if err := gitscm.DeleteHook(repo.FullName, webhook.ID); err != nil {
			return err
		}
	}

	return nil
}

func filterHooks(hooks []*scm.Hook, provider core.Provider) []*scm.Hook {
	var webhooks []*scm.Hook

	for _, hook := range hooks {
		url, _ := url.Parse(hook.Target)
		if strings.HasPrefix(hook.Target, provider.Host) && strings.HasSuffix(url.Path, "/webhooks") {
			webhooks = append(webhooks, hook)
		}
	}

	return webhooks
}
<commit_msg>fix(webhooks): fixed params on create Github Hook<commit_after>package repo

import (
	"context"
	"fmt"
	"net/url"
	"strings"

	"github.com/bleenco/abstruse/pkg/gitscm"
	"github.com/bleenco/abstruse/server/core"
	"github.com/drone/go-scm/scm"
	"github.com/jinzhu/gorm"
)

// New returns a new RepositoryStore.
func New(db *gorm.DB) core.RepositoryStore {
	return repositoryStore{db}
}

type repositoryStore struct {
	db *gorm.DB
}

func (s repositoryStore) Find(id, userID uint) (core.Repository, error) {
	var repo core.Repository

	db := s.db.Preload("Provider")
	db = db.Joins("LEFT JOIN permissions ON permissions.repository_id = repositories.id").
		Joins("LEFT JOIN teams ON teams.id = permissions.team_id").
		Joins("LEFT JOIN team_users ON team_users.team_id = teams.id")
	db = db.Where("repositories.id = ? AND repositories.user_id = ?", id, userID).
		Or("repositories.id = ? AND team_users.user_id = ? AND permissions.read = ?", id, userID, true)

	err := db.First(&repo).Error
	if err != nil {
		return repo, err
	}
	repo.Perms = s.GetPermissions(repo.ID, userID)

	return repo, err
}

func (s repositoryStore) FindUID(uid string) (core.Repository, error) {
	var repo core.Repository
	err := s.db.Where("uid = ?", uid).Preload("Provider").First(&repo).Error
	return repo, err
}

func (s repositoryStore) FindClone(clone string) (core.Repository, error) {
	var repo core.Repository
	err := s.db.Where("clone = ?", clone).Preload("Provider").First(&repo).Error
	return repo, err
}

func (s repositoryStore) FindToken(token string) (*core.Repository, error) {
	var repo core.Repository
	err := s.db.Where("token = ?", token).First(&repo).Error
	return &repo, err
}

func (s repositoryStore) List(filters core.RepositoryFilter) ([]core.Repository, int, error) {
	var repos []core.Repository
	var count int
	var err error
	keyword := fmt.Sprintf("%%%s%%", filters.Keyword)

	db := s.db.Preload("Provider")

	db = db.
		Joins("LEFT JOIN permissions ON permissions.repository_id = repositories.id").
		Joins("LEFT JOIN teams ON teams.id = permissions.team_id").
		Joins("LEFT JOIN team_users ON team_users.team_id = teams.id")

	if filters.UserID != 0 {
		db = db.Where("repositories.user_id = ? AND repositories.full_name LIKE ?", filters.UserID, keyword).
			Or("team_users.user_id = ? AND permissions.read = ? AND repositories.full_name LIKE ?", filters.UserID, true, keyword)
	}

	if filters.Limit != 0 {
		db = db.Limit(filters.Limit)
	}

	if filters.Offset != 0 {
		db = db.Offset(filters.Offset)
	}

	err = db.Order("active desc, name asc").Group("repositories.id").Find(&repos).Limit(-1).Offset(-1).Count(&count).Error
	if err != nil || count == 0 {
		return repos, count, err
	}

	for i, repo := range repos {
		repos[i].Perms = s.GetPermissions(repo.ID, filters.UserID)
	}

	return repos, count, err
}

func (s repositoryStore) Create(repo core.Repository) error {
	return s.db.Create(&repo).Error
}

func (s repositoryStore) Update(repo core.Repository) error {
	return s.db.Model(&repo).Updates(&repo).Error
}

func (s repositoryStore) CreateOrUpdate(repo core.Repository) error {
	if s.db.Where("uid = ? AND clone = ?", repo.UID, repo.Clone).First(&repo).RecordNotFound() {
		return s.db.Create(&repo).Error
	}

	return s.db.Model(&repo).Where("uid = ? AND clone = ?", repo.UID, repo.Clone).Updates(&repo).Error
}

func (s repositoryStore) Delete(repo core.Repository) error {
	return s.db.Delete(&repo).Error
}

func (s repositoryStore) SetActive(id uint, active bool) error {
	var repo core.Repository
	if s.db.Where("id = ?", id).First(&repo).RecordNotFound() {
		return fmt.Errorf("repository not found")
	}

	return s.db.Model(&repo).Update("active", active).Error
}

func (s repositoryStore) GetPermissions(id, userID uint) core.Perms {
	perms := core.Perms{Read: false, Write: false, Exec: false}

	var user core.User
	err := s.db.Model(&user).Where("id = ?", userID).First(&user).Error
	if err != nil {
		return perms
	}
	if user.Role == "admin" {
		return core.Perms{Read: true, Write: true, Exec: true}
	}

	var repo core.Repository
	if err := s.db.Where("id = ?", id).First(&repo).Error; err == nil {
		if repo.UserID == userID {
			perms.Read = true
			perms.Write = true
			perms.Exec = true
			return perms
		}
	}

	db := s.db

	db = db.
		Joins("LEFT JOIN team_users ON team_users.team_id = permissions.team_id").
		Where("team_users.user_id = ? AND permissions.repository_id = ?", userID, id)

	var permissions []*core.Permission
	err = db.Find(&permissions).Error
	if err != nil {
		return perms
	}

	r, w, x := false, false, false

	for _, p := range permissions {
		if p.Read {
			r = true
		}
		if p.Write {
			w = true
		}
		if p.Exec {
			x = true
		}
	}

	return core.Perms{Read: r, Write: w, Exec: x}
}

func (s repositoryStore) ListHooks(id, userID uint) ([]*scm.Hook, error) {
	repo, err := s.Find(id, userID)
	if err != nil {
		return nil, err
	}

	gitscm, err := gitscm.New(context.Background(), repo.Provider.Name, repo.Provider.URL, repo.Provider.AccessToken)
	if err != nil {
		return nil, err
	}

	hooks, err := gitscm.ListHooks(repo.FullName)
	if err != nil {
		return nil, err
	}

	return filterHooks(hooks, repo.Provider), nil
}

func (s repositoryStore) CreateHook(id, userID uint, data gitscm.HookForm) error {
	if err := s.DeleteHooks(id, userID); err != nil {
		return err
	}

	repo, err := s.Find(id, userID)
	if err != nil {
		return err
	}

	gitscm, err := gitscm.New(context.Background(), repo.Provider.Name, repo.Provider.URL, repo.Provider.AccessToken)
	if err != nil {
		return err
	}

	if !data.Branch && !data.PullRequest && !data.Push && !data.Tag {
		return nil
	}

	target := fmt.Sprintf("%s/webhooks", repo.Provider.Host)
	_, err = gitscm.CreateHook(repo.FullName, target, repo.Provider.Secret, repo.Provider.Name, data)
	return err
}

func (s repositoryStore) DeleteHooks(id, userID uint) error {
	repo, err := s.Find(id, userID)
	if err != nil {
		return err
	}

	gitscm, err := gitscm.New(context.Background(), repo.Provider.Name, repo.Provider.URL, repo.Provider.AccessToken)
	if err != nil {
		return err
	}

	hooks, err := gitscm.ListHooks(repo.FullName)
	if err != nil {
		return err
	}

	webhooks := filterHooks(hooks, repo.Provider)

	for _, webhook := range webhooks {
		if err := gitscm.DeleteHook(repo.FullName, webhook.ID); err != nil {
			return err
		}
	}

	return nil
}

func filterHooks(hooks []*scm.Hook, provider core.Provider) []*scm.Hook {
	var webhooks []*scm.Hook

	for _, hook := range hooks {
		url, _ := url.Parse(hook.Target)
		if strings.HasPrefix(hook.Target, provider.Host) && strings.HasSuffix(url.Path, "/webhooks") {
			webhooks = append(webhooks, hook)
		}
	}

	return webhooks
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// build the protos
//go:generate $GOPATH/src/github.com/apigee/istio-mixer-adapter/bin/codegen.sh -f apigee/config/config.proto
//go:generate $GOPATH/src/github.com/apigee/istio-mixer-adapter/bin/codegen.sh -t template/analytics/template.proto

package apigee

import (
	"context"
	"fmt"
	"net/url"

	"github.com/apigee/istio-mixer-adapter/apigee/analytics"
	"github.com/apigee/istio-mixer-adapter/apigee/auth"
	"github.com/apigee/istio-mixer-adapter/apigee/config"
	"github.com/apigee/istio-mixer-adapter/apigee/product"
	"github.com/apigee/istio-mixer-adapter/apigee/quota"
	analyticsT "github.com/apigee/istio-mixer-adapter/template/analytics"
	"istio.io/istio/mixer/pkg/adapter"
	"istio.io/istio/mixer/pkg/status"
	"istio.io/istio/mixer/template/apikey"
	authT "istio.io/istio/mixer/template/authorization"
	quotaT "istio.io/istio/mixer/template/quota"
)

type (
	builder struct {
		adapterConfig *config.Params
	}

	handler struct {
		env          adapter.Env
		apigeeBase   url.URL
		customerBase url.URL
		orgName      string
		envName      string
		key          string
		secret       string
	}
)

// make handler implement Context...

func (h *handler) Log() adapter.Logger {
	return h.env.Logger()
}
func (h *handler) ApigeeBase() url.URL {
	return h.apigeeBase
}
func (h *handler) CustomerBase() url.URL {
	return h.customerBase
}
func (h *handler) Organization() string {
	return h.orgName
}
func (h *handler) Environment() string {
	return h.envName
}
func (h *handler) Key() string {
	return h.key
}
func (h *handler) Secret() string {
	return h.secret
}

// Ensure required interfaces are implemented.
var (
	// Builder
	_ adapter.HandlerBuilder    = &builder{}
	_ quotaT.HandlerBuilder     = &builder{}
	_ analyticsT.HandlerBuilder = &builder{}
	_ apikey.HandlerBuilder     = &builder{}
	_ authT.HandlerBuilder      = &builder{}

	// Handler
	_ adapter.Handler    = &handler{}
	_ quotaT.Handler     = &handler{}
	_ analyticsT.Handler = &handler{}
	_ apikey.Handler     = &handler{}
	_ authT.Handler      = &handler{}
)

////////////////// GetInfo //////////////////////////

// GetInfo returns the adapter.Info associated with this implementation.
func GetInfo() adapter.Info {
	return adapter.Info{
		Name:        "apigee",
		Impl:        "istio.io/istio/mixer/adapter/apigee",
		Description: "Apigee adapter",
		SupportedTemplates: []string{
			analyticsT.TemplateName,
			apikey.TemplateName,
			authT.TemplateName,
			quotaT.TemplateName,
		},
		DefaultConfig: &config.Params{},
		NewBuilder:    func() adapter.HandlerBuilder { return &builder{} },
	}
}

////////////////// adapter.Builder //////////////////////////

// Implements adapter.HandlerBuilder
func (b *builder) SetAdapterConfig(cfg adapter.Config) {
	b.adapterConfig = cfg.(*config.Params)
}

// Implements adapter.HandlerBuilder
func (b *builder) Build(context context.Context, env adapter.Env) (adapter.Handler, error) {

	apigeeBase, err := url.Parse(b.adapterConfig.ApigeeBase)
	if err != nil {
		return nil, err
	}

	customerBase, err := url.Parse(b.adapterConfig.CustomerBase)
	if err != nil {
		return nil, err
	}

	h := &handler{
		env:          env,
		apigeeBase:   *apigeeBase,
		customerBase: *customerBase,
		orgName:      b.adapterConfig.OrgName,
		envName:      b.adapterConfig.EnvName,
		key:          b.adapterConfig.Key,
		secret:       b.adapterConfig.Secret,
	}

	product.Start(h.CustomerBase(), h.Log(), env)
	auth.Start(env)

	return h, nil
}

// Implements adapter.HandlerBuilder
func (b *builder) Validate() (errs *adapter.ConfigErrors) {

	if b.adapterConfig.ApigeeBase == "" {
		errs = errs.Append("apigee_base", fmt.Errorf("required"))
	} else if _, err := url.Parse(b.adapterConfig.ApigeeBase); err != nil {
		errs = errs.Append("apigee_base", fmt.Errorf("must be a valid url: %v", err))
	}

	if b.adapterConfig.CustomerBase == "" {
		errs = errs.Append("customer_base", fmt.Errorf("required"))
	} else if _, err := url.Parse(b.adapterConfig.CustomerBase); err != nil {
		errs = errs.Append("customer_base", fmt.Errorf("must be a valid url: %v", err))
	}

	if b.adapterConfig.OrgName == "" {
		errs = errs.Append("org_name", fmt.Errorf("required"))
	}

	if b.adapterConfig.EnvName == "" {
		errs = errs.Append("env_name", fmt.Errorf("required"))
	}

	if b.adapterConfig.Key == "" {
		errs = errs.Append("key", fmt.Errorf("required"))
	}

	if b.adapterConfig.Secret == "" {
		errs = errs.Append("secret", fmt.Errorf("required"))
	}

	return errs
}

func (*builder) SetAnalyticsTypes(map[string]*analyticsT.Type) {}
func (*builder) SetApiKeyTypes(map[string]*apikey.Type)        {}
func (*builder) SetAuthorizationTypes(map[string]*authT.Type)  {}
func (*builder) SetQuotaTypes(map[string]*quotaT.Type)         {}

////////////////// adapter.Handler //////////////////////////

// Implements adapter.Handler
func (h *handler) Close() error {
	product.Stop()
	auth.Stop()
	return nil
}

// important: This assumes that the Auth is the same for all records!
func (h *handler) HandleAnalytics(ctx context.Context, instances []*analyticsT.Instance) error {

	var authContext *auth.Context
	var records []analytics.Record

	for _, inst := range instances {
		h.Log().Infof("HandleAnalytics: %v\n", inst)

		record := analytics.Record{
			ClientReceivedStartTimestamp: analytics.TimeToUnix(inst.ClientReceivedStartTimestamp),
			ClientReceivedEndTimestamp:   analytics.TimeToUnix(inst.ClientReceivedStartTimestamp),
			ClientSentStartTimestamp:     analytics.TimeToUnix(inst.ClientSentStartTimestamp),
			ClientSentEndTimestamp:       analytics.TimeToUnix(inst.ClientSentEndTimestamp),
			TargetReceivedStartTimestamp: analytics.TimeToUnix(inst.TargetReceivedStartTimestamp),
			TargetReceivedEndTimestamp:   analytics.TimeToUnix(inst.TargetReceivedEndTimestamp),
			TargetSentStartTimestamp:     analytics.TimeToUnix(inst.TargetSentStartTimestamp),
			TargetSentEndTimestamp:       analytics.TimeToUnix(inst.TargetSentEndTimestamp),
			APIProxy:                     inst.ApiProxy,
			RequestURI:                   inst.RequestUri,
			RequestPath:                  inst.RequestPath,
			RequestVerb:                  inst.RequestVerb,
			ClientIP:                     inst.ClientIp.String(),
			UserAgent:                    inst.Useragent,
			ResponseStatusCode:           int(inst.ResponseStatusCode),
		}

		if authContext == nil {
			ac, err := auth.Authenticate(h, inst.ApiKey, convertClaims(inst.ApiClaims))
			if err != nil {
				return err
			}
			authContext = &ac
		}

		records = append(records, record)
	}

	return analytics.SendRecords(authContext, records)
}

func (h *handler) HandleApiKey(ctx context.Context, inst *apikey.Instance) (adapter.CheckResult, error) {
	h.Log().Infof("HandleApiKey: %v\n", inst)

	if inst.ApiKey == "" || inst.Api == "" || inst.ApiOperation == "" {
		h.Log().Infof("missing properties")
		return adapter.CheckResult{
			Status: status.WithPermissionDenied("missing authentication"),
		}, nil
	}

	authContext, err := auth.Authenticate(h, inst.ApiKey, nil)
	if err != nil {
		h.Log().Errorf("authenticate err: %v", err)
		return adapter.CheckResult{
			Status: status.WithPermissionDenied(err.Error()),
		}, err
	}

	// todo: need to do better response for fail
	if authContext.ClientID == "" {
		h.Log().Infof("authenticate failed")
		return adapter.CheckResult{
			Status: status.WithPermissionDenied("authentication failed"),
		}, nil
	}

	return authorize(authContext, inst.Api, inst.ApiOperation)
}

func (h *handler) HandleAuthorization(ctx context.Context, inst *authT.Instance) (adapter.CheckResult, error) {
	h.Log().Infof("HandleAuthorization: %v\n", inst)

	if inst.Subject == nil || inst.Subject.Properties == nil || inst.Action.Service == "" || inst.Action.Path == "" {
		h.Log().Infof("missing properties")
		return adapter.CheckResult{
			Status: status.WithPermissionDenied("missing authentication"),
		}, nil
	}

	claims, ok := inst.Subject.Properties["claims"].(map[string]string)
	if !ok {
		return adapter.CheckResult{}, fmt.Errorf("wrong claims type: %v\n", inst.Subject.Properties["claims"])
	}

	authContext, err := auth.Authenticate(h, "", convertClaims(claims))
	if err != nil {
		h.Log().Errorf("authenticate err: %v", err)
		return adapter.CheckResult{
			Status: status.WithPermissionDenied(err.Error()),
		}, err
	}

	if authContext.ClientID == "" {
		h.Log().Infof("authenticate failed")
		return adapter.CheckResult{
			Status: status.WithPermissionDenied("not authenticated"),
		}, nil
	}

	return authorize(authContext, inst.Action.Service, inst.Action.Path)
}

// authorize: check service, path, scopes
func authorize(authContext auth.Context, service, path string) (adapter.CheckResult, error) {

	products := product.Resolve(authContext, service, path)
	if len(products) > 0 {
		return adapter.CheckResult{
			Status: status.OK,
		}, nil
	}

	return adapter.CheckResult{
		Status: status.WithPermissionDenied("not authorized"),
	}, nil
}

// Istio doesn't understand our Quotas, so it cannot be allowed to cache
func (h *handler) HandleQuota(ctx context.Context, inst *quotaT.Instance, args adapter.QuotaArgs) (adapter.QuotaResult, error) {
	h.Log().Infof("HandleQuota: %v args: %v\n", inst, args)

	// skip < 0 to eliminate Istio prefetch returns
	if args.QuotaAmount <= 0 {
		return adapter.QuotaResult{}, nil
	}

	path := inst.Dimensions["path"].(string)
	if path == "" {
		return adapter.QuotaResult{}, fmt.Errorf("path attribute required")
	}
	apiKey := inst.Dimensions["api_key"].(string)
	api := inst.Dimensions["api"].(string)

	h.Log().Infof("api: %v, key: %v, path: %v", api, apiKey, path)

	// not sure about actual format
	claims, ok := inst.Dimensions["api_claims"].(map[string]string)
	if !ok {
		return adapter.QuotaResult{}, fmt.Errorf("wrong claims type: %v\n", inst.Dimensions["api_claims"])
	}

	authContext, err := auth.Authenticate(h, apiKey, convertClaims(claims))
	if err != nil {
		return adapter.QuotaResult{}, err
	}

	h.Log().Infof("auth: %v", authContext)

	// get relevant products
	prods := product.Resolve(authContext, api, path)

	if len(prods) == 0 { // no quotas, allow
		return adapter.QuotaResult{
			Amount:        args.QuotaAmount,
			ValidDuration: 0,
		}, nil
	}

	// todo: support args.DeduplicationID
	// todo: converting our quotas to Istio is weird, anything better?
	// todo: set QuotaAmount to 1 to eliminate Istio prefetch (also renders BestEffort meaningless)
	args.QuotaAmount = 1
	var exceeded int64
	var anyErr error
	for _, p := range prods {
		if p.QuotaLimit != "" {
			result, err := quota.Apply(authContext, p, args)
			if err != nil {
				anyErr = err
			} else if result.Exceeded > 0 {
				exceeded = result.Exceeded
			}
		}
	}
	if anyErr != nil {
		return adapter.QuotaResult{}, anyErr
	}
	if exceeded > 0 {
		return adapter.QuotaResult{
			Status:        status.OK,
			ValidDuration: 0,
			Amount:        0,
		}, nil
	}

	return adapter.QuotaResult{
		Status:        status.OK,
		ValidDuration: 0,
		Amount:        args.QuotaAmount,
	}, nil
}

func convertClaims(claims map[string]string) map[string]interface{} {
	var claimsOut map[string]interface{}
	for k, v := range claims {
		claimsOut[k] = v
	}
	return claimsOut
}
<commit_msg>add some logging<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// build the protos
//go:generate $GOPATH/src/github.com/apigee/istio-mixer-adapter/bin/codegen.sh -f apigee/config/config.proto
//go:generate $GOPATH/src/github.com/apigee/istio-mixer-adapter/bin/codegen.sh -t template/analytics/template.proto

package apigee

import (
	"context"
	"fmt"
	"net/url"

	"github.com/apigee/istio-mixer-adapter/apigee/analytics"
	"github.com/apigee/istio-mixer-adapter/apigee/auth"
	"github.com/apigee/istio-mixer-adapter/apigee/config"
	"github.com/apigee/istio-mixer-adapter/apigee/product"
	"github.com/apigee/istio-mixer-adapter/apigee/quota"
	analyticsT "github.com/apigee/istio-mixer-adapter/template/analytics"
	"istio.io/istio/mixer/pkg/adapter"
	"istio.io/istio/mixer/pkg/status"
	"istio.io/istio/mixer/template/apikey"
	authT "istio.io/istio/mixer/template/authorization"
	quotaT "istio.io/istio/mixer/template/quota"
)

type (
	builder struct {
		adapterConfig *config.Params
	}

	handler struct {
		env          adapter.Env
		apigeeBase   url.URL
		customerBase url.URL
		orgName      string
		envName      string
		key          string
		secret       string
	}
)

// make handler implement Context...

func (h *handler) Log() adapter.Logger {
	return h.env.Logger()
}
func (h *handler) ApigeeBase() url.URL {
	return h.apigeeBase
}
func (h *handler) CustomerBase() url.URL {
	return h.customerBase
}
func (h *handler) Organization() string {
	return h.orgName
}
func (h *handler) Environment() string {
	return h.envName
}
func (h *handler) Key() string {
	return h.key
}
func (h *handler) Secret() string {
	return h.secret
}

// Ensure required interfaces are implemented.
var (
	// Builder
	_ adapter.HandlerBuilder    = &builder{}
	_ quotaT.HandlerBuilder     = &builder{}
	_ analyticsT.HandlerBuilder = &builder{}
	_ apikey.HandlerBuilder     = &builder{}
	_ authT.HandlerBuilder      = &builder{}

	// Handler
	_ adapter.Handler    = &handler{}
	_ quotaT.Handler     = &handler{}
	_ analyticsT.Handler = &handler{}
	_ apikey.Handler     = &handler{}
	_ authT.Handler      = &handler{}
)

////////////////// GetInfo //////////////////////////

// GetInfo returns the adapter.Info associated with this implementation.
func GetInfo() adapter.Info {
	return adapter.Info{
		Name:        "apigee",
		Impl:        "istio.io/istio/mixer/adapter/apigee",
		Description: "Apigee adapter",
		SupportedTemplates: []string{
			analyticsT.TemplateName,
			apikey.TemplateName,
			authT.TemplateName,
			quotaT.TemplateName,
		},
		DefaultConfig: &config.Params{},
		NewBuilder:    func() adapter.HandlerBuilder { return &builder{} },
	}
}

////////////////// adapter.Builder //////////////////////////

// Implements adapter.HandlerBuilder
func (b *builder) SetAdapterConfig(cfg adapter.Config) {
	b.adapterConfig = cfg.(*config.Params)
}

// Implements adapter.HandlerBuilder
func (b *builder) Build(context context.Context, env adapter.Env) (adapter.Handler, error) {

	apigeeBase, err := url.Parse(b.adapterConfig.ApigeeBase)
	if err != nil {
		return nil, err
	}

	customerBase, err := url.Parse(b.adapterConfig.CustomerBase)
	if err != nil {
		return nil, err
	}

	h := &handler{
		env:          env,
		apigeeBase:   *apigeeBase,
		customerBase: *customerBase,
		orgName:      b.adapterConfig.OrgName,
		envName:      b.adapterConfig.EnvName,
		key:          b.adapterConfig.Key,
		secret:       b.adapterConfig.Secret,
	}

	product.Start(h.CustomerBase(), h.Log(), env)
	h.Log().Infof("product manager started")
	auth.Start(env)
	h.Log().Infof("auth manager started")

	return h, nil
}

// Implements adapter.HandlerBuilder
func (b *builder) Validate() (errs *adapter.ConfigErrors) {

	if b.adapterConfig.ApigeeBase == "" {
		errs = errs.Append("apigee_base", fmt.Errorf("required"))
	} else if _, err := url.Parse(b.adapterConfig.ApigeeBase); err != nil {
		errs = errs.Append("apigee_base", fmt.Errorf("must be a valid url: %v", err))
	}

	if b.adapterConfig.CustomerBase == "" {
		errs = errs.Append("customer_base", fmt.Errorf("required"))
	} else if _, err := url.Parse(b.adapterConfig.CustomerBase); err != nil {
		errs = errs.Append("customer_base", fmt.Errorf("must be a valid url: %v", err))
	}

	if b.adapterConfig.OrgName == "" {
		errs = errs.Append("org_name", fmt.Errorf("required"))
	}

	if b.adapterConfig.EnvName == "" {
		errs = errs.Append("env_name", fmt.Errorf("required"))
	}

	if b.adapterConfig.Key == "" {
		errs = errs.Append("key", fmt.Errorf("required"))
	}

	if b.adapterConfig.Secret == "" {
		errs = errs.Append("secret", fmt.Errorf("required"))
	}

	return errs
}

func (*builder) SetAnalyticsTypes(map[string]*analyticsT.Type) {}
func (*builder) SetApiKeyTypes(map[string]*apikey.Type)        {}
func (*builder) SetAuthorizationTypes(map[string]*authT.Type)  {}
func (*builder) SetQuotaTypes(map[string]*quotaT.Type)         {}

////////////////// adapter.Handler //////////////////////////

// Implements adapter.Handler
func (h *handler) Close() error {
	product.Stop()
	h.Log().Infof("product manager stopped")
	auth.Stop()
	h.Log().Infof("auth manager stopped")
	return nil
}

// important: This assumes that the Auth is the same for all records!
func (h *handler) HandleAnalytics(ctx context.Context, instances []*analyticsT.Instance) error {

	var authContext *auth.Context
	var records []analytics.Record

	for _, inst := range instances {
		h.Log().Infof("HandleAnalytics: %v\n", inst)

		record := analytics.Record{
			ClientReceivedStartTimestamp: analytics.TimeToUnix(inst.ClientReceivedStartTimestamp),
			ClientReceivedEndTimestamp:   analytics.TimeToUnix(inst.ClientReceivedStartTimestamp),
			ClientSentStartTimestamp:     analytics.TimeToUnix(inst.ClientSentStartTimestamp),
			ClientSentEndTimestamp:       analytics.TimeToUnix(inst.ClientSentEndTimestamp),
			TargetReceivedStartTimestamp: analytics.TimeToUnix(inst.TargetReceivedStartTimestamp),
			TargetReceivedEndTimestamp:   analytics.TimeToUnix(inst.TargetReceivedEndTimestamp),
			TargetSentStartTimestamp:     analytics.TimeToUnix(inst.TargetSentStartTimestamp),
			TargetSentEndTimestamp:       analytics.TimeToUnix(inst.TargetSentEndTimestamp),
			APIProxy:                     inst.ApiProxy,
			RequestURI:                   inst.RequestUri,
			RequestPath:                  inst.RequestPath,
			RequestVerb:                  inst.RequestVerb,
			ClientIP:                     inst.ClientIp.String(),
			UserAgent:                    inst.Useragent,
			ResponseStatusCode:           int(inst.ResponseStatusCode),
		}

		if authContext == nil {
			ac, err := auth.Authenticate(h, inst.ApiKey, convertClaims(inst.ApiClaims))
			if err != nil {
				return err
			}
			authContext = &ac
		}

		records = append(records, record)
	}

	return analytics.SendRecords(authContext, records)
}

func (h *handler) HandleApiKey(ctx context.Context, inst *apikey.Instance) (adapter.CheckResult, error) {
	h.Log().Infof("HandleApiKey: %v\n", inst)

	if inst.ApiKey == "" || inst.Api == "" || inst.ApiOperation == "" {
		h.Log().Infof("missing properties")
		return adapter.CheckResult{
			Status: status.WithPermissionDenied("missing authentication"),
		}, nil
	}

	authContext, err := auth.Authenticate(h, inst.ApiKey, nil)
	if err != nil {
		h.Log().Errorf("authenticate err: %v", err)
		return adapter.CheckResult{
			Status: status.WithPermissionDenied(err.Error()),
		}, err
	}

	// todo: need to do better response for fail
	if authContext.ClientID == "" {
		h.Log().Infof("authenticate failed")
		return adapter.CheckResult{
			Status: status.WithPermissionDenied("authentication failed"),
		}, nil
	}

	return authorize(authContext, inst.Api, inst.ApiOperation)
}

func (h *handler) HandleAuthorization(ctx context.Context, inst *authT.Instance) (adapter.CheckResult, error) {
	h.Log().Infof("HandleAuthorization: %v\n", inst)

	if inst.Subject == nil || inst.Subject.Properties == nil || inst.Action.Service == "" || inst.Action.Path == "" {
		h.Log().Infof("missing properties")
		return adapter.CheckResult{
			Status: status.WithPermissionDenied("missing authentication"),
		}, nil
	}

	claims, ok := inst.Subject.Properties["claims"].(map[string]string)
	if !ok {
		return adapter.CheckResult{}, fmt.Errorf("wrong claims type: %v\n", inst.Subject.Properties["claims"])
	}

	authContext, err := auth.Authenticate(h, "", convertClaims(claims))
	if err != nil {
		h.Log().Errorf("authenticate err: %v", err)
		return adapter.CheckResult{
			Status: status.WithPermissionDenied(err.Error()),
		}, err
	}

	if authContext.ClientID == "" {
		h.Log().Infof("authenticate failed")
		return adapter.CheckResult{
			Status: status.WithPermissionDenied("not authenticated"),
		}, nil
	}

	return authorize(authContext, inst.Action.Service, inst.Action.Path)
}

// authorize: check service, path, scopes
func authorize(authContext auth.Context, service, path string) (adapter.CheckResult, error) {

	products := product.Resolve(authContext, service, path)
	if len(products) > 0 {
		return adapter.CheckResult{
			Status: status.OK,
		}, nil
	}

	return adapter.CheckResult{
		Status: status.WithPermissionDenied("not authorized"),
	}, nil
}

// Istio doesn't understand our Quotas, so it cannot be allowed to cache
func (h *handler) HandleQuota(ctx context.Context, inst *quotaT.Instance, args adapter.QuotaArgs) (adapter.QuotaResult, error) {
	h.Log().Infof("HandleQuota: %v args: %v\n", inst, args)

	// skip < 0 to eliminate Istio prefetch returns
	if args.QuotaAmount <= 0 {
		return adapter.QuotaResult{}, nil
	}

	path := inst.Dimensions["path"].(string)
	if path == "" {
		return adapter.QuotaResult{}, fmt.Errorf("path attribute required")
	}
	apiKey := inst.Dimensions["api_key"].(string)
	api := inst.Dimensions["api"].(string)

	h.Log().Infof("api: %v, key: %v, path: %v", api, apiKey, path)

	// not sure about actual format
	claims, ok := inst.Dimensions["api_claims"].(map[string]string)
	if !ok {
		return adapter.QuotaResult{}, fmt.Errorf("wrong claims type: %v\n", inst.Dimensions["api_claims"])
	}

	authContext, err := auth.Authenticate(h, apiKey, convertClaims(claims))
	if err != nil {
		return adapter.QuotaResult{}, err
	}

	h.Log().Infof("auth: %v", authContext)

	// get relevant products
	prods := product.Resolve(authContext, api, path)

	if len(prods) == 0 { // no quotas, allow
		return adapter.QuotaResult{
			Amount:        args.QuotaAmount,
			ValidDuration: 0,
		}, nil
	}

	// todo: support args.DeduplicationID
	// todo: converting our quotas to Istio is weird, anything better?
	// todo: set QuotaAmount to 1 to eliminate Istio prefetch (also renders BestEffort meaningless)
	args.QuotaAmount = 1
	var exceeded int64
	var anyErr error
	for _, p := range prods {
		if p.QuotaLimit != "" {
			result, err := quota.Apply(authContext, p, args)
			if err != nil {
				anyErr = err
			} else if result.Exceeded > 0 {
				exceeded = result.Exceeded
			}
		}
	}
	if anyErr != nil {
		return adapter.QuotaResult{}, anyErr
	}
	if exceeded > 0 {
		return adapter.QuotaResult{
			Status:        status.OK,
			ValidDuration: 0,
			Amount:        0,
		}, nil
	}

	return adapter.QuotaResult{
		Status:        status.OK,
		ValidDuration: 0,
		Amount:        args.QuotaAmount,
	}, nil
}

func convertClaims(claims map[string]string) map[string]interface{} {
	var claimsOut map[string]interface{}
	for k, v := range claims {
		claimsOut[k] = v
	}
	return claimsOut
}
<|endoftext|>