<commit_before>package kit

// InstrumentingTemplate
var InstrumentingTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/metrics"
	"golang.org/x/net/context"
)

// DefaultMiddlewares provides bare bones for default middlewares with
// requestLatency, requestCount and requestLogging
func DefaultMiddlewares(method string, requestCount metrics.Counter, requestLatency metrics.TimeHistogram, logger log.Logger) endpoint.Middleware {
	return endpoint.Chain(
		RequestLatencyMiddleware(method, requestLatency),
		RequestCountMiddleware(method, requestCount),
		RequestLoggingMiddleware(method, logger),
	)
}

// RequestCountMiddleware prepares a request counter endpoint.Middleware for
// package wide usage
func RequestCountMiddleware(method string, requestCount metrics.Counter) endpoint.Middleware {
	return func(next endpoint.Endpoint) endpoint.Endpoint {
		return func(ctx context.Context, request interface{}) (response interface{}, err error) {
			defer func() {
				methodField := metrics.Field{Key: "method", Value: method}
				errorField := metrics.Field{Key: "error", Value: fmt.Sprintf("%v", err)}
				requestCount.With(methodField).With(errorField).Add(1)
			}()

			response, err = next(ctx, request)
			return
		}
	}
}

// RequestLatencyMiddleware prepares a request latency calculator
// endpoint.Middleware for package wide usage
func RequestLatencyMiddleware(method string, requestLatency metrics.TimeHistogram) endpoint.Middleware {
	return func(next endpoint.Endpoint) endpoint.Endpoint {
		return func(ctx context.Context, request interface{}) (response interface{}, err error) {
			defer func(begin time.Time) {
				methodField := metrics.Field{Key: "method", Value: method}
				errorField := metrics.Field{Key: "error", Value: fmt.Sprintf("%v", err)}
				requestLatency.With(methodField).With(errorField).Observe(time.Since(begin))
			}(time.Now())

			response, err = next(ctx, request)
			return
		}
	}
}

// RequestLoggingMiddleware prepares a request logger endpoint.Middleware for
// package wide usage
func RequestLoggingMiddleware(method string, logger log.Logger) endpoint.Middleware {
	return func(next endpoint.Endpoint) endpoint.Endpoint {
		return func(ctx context.Context, request interface{}) (response interface{}, err error) {
			defer func(begin time.Time) {
				input, _ := json.Marshal(request)
				output, _ := json.Marshal(response)
				_ = logger.Log(
					"method", method,
					"input", string(input),
					"output", string(output),
					"err", err,
					"took", time.Since(begin),
				)
			}(time.Now())
			response, err = next(ctx, request)
			return
		}
	}
}
`

// InterfaceTemplate
var InterfaceTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

type {{$title}}Service interface {
{{range $funcKey, $funcValue := $schema.Functions}}
{{$funcKey}}(ctx context.Context, req *{{Argumentize $funcValue.Properties.incoming}}) (res *{{Argumentize $funcValue.Properties.outgoing}}, err error){{end}}
}
`

// TransportHTTPServerTemplate
var TransportHTTPServerTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

import (
	"encoding/json"
	"net/http"

	"golang.org/x/net/context"

	"github.com/go-kit/kit/endpoint"
	httptransport "github.com/go-kit/kit/transport/http"
)

// Handler functions

{{range $funcKey, $funcValue := $schema.Functions}}
func New{{$funcKey}}Handler(ctx context.Context, svc {{$title}}Service, middleware endpoint.Middleware, options ...httptransport.ServerOption) *httptransport.Server {
	return httptransport.NewServer(
		ctx,
		middleware(make{{$funcKey}}Endpoint(svc)),
		decode{{$funcKey}}Request,
		encodeResponse,
		options...,
	)
}
{{end}}

// Endpoint functions

{{range $funcKey, $funcValue := $schema.Functions}}
func make{{$funcKey}}Endpoint(svc {{$title}}Service) endpoint.Endpoint {
	return func(ctx context.Context, request interface{}) (interface{}, error) {
		req := request.(*{{Argumentize $funcValue.Properties.incoming}})
		return svc.{{$funcKey}}(ctx, req)
	}
}
{{end}}
`

// TransportHTTPClientTemplate
var TransportHTTPClientTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

import (
	jujuratelimit "github.com/juju/ratelimit"
	"github.com/sony/gobreaker"
	"golang.org/x/net/context"

	"github.com/go-kit/kit/circuitbreaker"
	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/loadbalancer"
	"github.com/go-kit/kit/loadbalancer/static"
	"github.com/go-kit/kit/log"
	kitratelimit "github.com/go-kit/kit/ratelimit"
	httptransport "github.com/go-kit/kit/transport/http"
)

// {{$title}}Client holds remote endpoint functions
// Satisfies {{$title}}Service interface
type {{$title}}Client struct {
{{range $funcKey, $funcValue := $schema.Functions}}// {{$funcKey}}Endpoint provides remote call to {{ToLower $funcKey}} endpoint
	{{$funcKey}}Endpoint endpoint.Endpoint

{{end}}}

// New{{$title}}Client creates a new client for {{$title}}Service
func  New{{$title}}Client(proxies []string, ctx context.Context, maxAttempt int, maxTime time.Duration, qps int, logger log.Logger) *{{$title}}Client {
return &{{$title}}Client{
{{range $funcKey, $funcValue := $schema.Functions}}
{{$funcKey}}Endpoint : new{{$funcKey}}ClientEndpoint(proxies, ctx, maxAttempt, maxTime, qps, logger),{{end}}
}
}

{{range $funcKey, $funcValue := $schema.Functions}}
{{AsComment $funcValue.Description}}func ({{Pointerize $title}} *{{$title}}Client) {{$funcKey}}(ctx context.Context, req *{{Argumentize $funcValue.Properties.incoming}}) (*{{Argumentize $funcValue.Properties.outgoing}}, error) {
	res, err := {{Pointerize $title}}.{{$funcKey}}Endpoint(ctx, req)
	if err != nil {
		return nil, err
	}

	return res.(*{{Argumentize $funcValue.Properties.outgoing}}), nil
}
{{end}}

// Client Endpoint functions
{{range $funcKey, $funcValue := $schema.Functions}}
func new{{$funcKey}}ClientEndpoint(proxies []string, ctx context.Context, maxAttempt int, maxTime time.Duration, qps int, logger log.Logger) endpoint.Endpoint {
	factory := createFactory(ctx, qps, make{{$funcKey}}Proxy)
	return defaultClientEndpointCreator(proxies, maxAttempt, maxTime, logger, factory)
}
{{end}}

{{range $funcKey, $funcValue := $schema.Functions}}
func make{{$funcKey}}Proxy(ctx context.Context, instance string) endpoint.Endpoint {
	return httptransport.NewClient(
		"POST",
		createProxyURL(instance, "{{ToLower $funcKey}}"),
		encodeRequest,
		decode{{$funcKey}}Response,
	).Endpoint()
}
{{end}}

// Proxy functions

func createProxyURL(instance, endpoint string) *url.URL {
	if !strings.HasPrefix(instance, "http") {
		instance = "http://" + instance
	}
	u, err := url.Parse(instance)
	if err != nil {
		panic(err)
	}
	if u.Path == "" {
		u.Path = endpoint
	}

	return u
}

type proxyFunc func(context.Context, string) endpoint.Endpoint

func createFactory(ctx context.Context, qps int, pf proxyFunc) loadbalancer.Factory {
	return func(instance string) (endpoint.Endpoint, io.Closer, error) {
		var e endpoint.Endpoint
		e = pf(ctx, instance)
		e = circuitbreaker.Gobreaker(gobreaker.NewCircuitBreaker(gobreaker.Settings{}))(e)
		e = kitratelimit.NewTokenBucketLimiter(jujuratelimit.NewBucketWithRate(float64(qps), int64(qps)))(e)
		return e, nil, nil
	}
}

func defaultClientEndpointCreator(
	proxies []string,
	maxAttempts int,
	maxTime time.Duration,
	logger log.Logger,
	factory loadbalancer.Factory,
) endpoint.Endpoint {

	publisher := static.NewPublisher(
		proxies,
		factory,
		logger,
	)

	lb := loadbalancer.NewRoundRobin(publisher)
	return loadbalancer.Retry(maxAttempts, maxTime, lb)
}

`

// TransportHTTPSemioticsTemplate
var TransportHTTPSemioticsTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

import (
	"encoding/json"
	"net/http"

	"golang.org/x/net/context"

	"github.com/go-kit/kit/endpoint"
	httptransport "github.com/go-kit/kit/transport/http"
)

// Decode Request functions

{{range $funcKey, $funcValue := $schema.Functions}}
func decode{{$funcKey}}Request(r *http.Request) (interface{}, error) {
	var req {{Argumentize $funcValue.Properties.incoming}}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		return nil, err
	}
	return &req, nil
}
{{end}}

// Decode Response functions

{{range $funcKey, $funcValue := $schema.Functions}}
func decode{{$funcKey}}Response(r *http.Response) (interface{}, error) {
	var res {{Argumentize $funcValue.Properties.incoming}}
	if err := json.NewDecoder(r.Body).Decode(&res); err != nil {
		return nil, err
	}
	return &res, nil
}
{{end}}

// Encode request function

func encodeRequest(r *http.Request, request interface{}) error {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(request); err != nil {
		return err
	}
	r.Body = ioutil.NopCloser(&buf)
	return nil
}

// Encode response function

func encodeResponse(rw http.ResponseWriter, response interface{}) error {
	return json.NewEncoder(rw).Encode(response)
}
`
<commit_msg>Kit: added documentations of endpoints functions<commit_after>package kit

// InstrumentingTemplate
var InstrumentingTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/metrics"
	"golang.org/x/net/context"
)

// DefaultMiddlewares provides bare bones for default middlewares with
// requestLatency, requestCount and requestLogging
func DefaultMiddlewares(method string, requestCount metrics.Counter, requestLatency metrics.TimeHistogram, logger log.Logger) endpoint.Middleware {
	return endpoint.Chain(
		RequestLatencyMiddleware(method, requestLatency),
		RequestCountMiddleware(method, requestCount),
		RequestLoggingMiddleware(method, logger),
	)
}

// RequestCountMiddleware prepares a request counter endpoint.Middleware for
// package wide usage
func RequestCountMiddleware(method string, requestCount metrics.Counter) endpoint.Middleware {
	return func(next endpoint.Endpoint) endpoint.Endpoint {
		return func(ctx context.Context, request interface{}) (response interface{}, err error) {
			defer func() {
				methodField := metrics.Field{Key: "method", Value: method}
				errorField := metrics.Field{Key: "error", Value: fmt.Sprintf("%v", err)}
				requestCount.With(methodField).With(errorField).Add(1)
			}()

			response, err = next(ctx, request)
			return
		}
	}
}

// RequestLatencyMiddleware prepares a request latency calculator
// endpoint.Middleware for package wide usage
func RequestLatencyMiddleware(method string, requestLatency metrics.TimeHistogram) endpoint.Middleware {
	return func(next endpoint.Endpoint) endpoint.Endpoint {
		return func(ctx context.Context, request interface{}) (response interface{}, err error) {
			defer func(begin time.Time) {
				methodField := metrics.Field{Key: "method", Value: method}
				errorField := metrics.Field{Key: "error", Value: fmt.Sprintf("%v", err)}
				requestLatency.With(methodField).With(errorField).Observe(time.Since(begin))
			}(time.Now())

			response, err = next(ctx, request)
			return
		}
	}
}

// RequestLoggingMiddleware prepares a request logger endpoint.Middleware for
// package wide usage
func RequestLoggingMiddleware(method string, logger log.Logger) endpoint.Middleware {
	return func(next endpoint.Endpoint) endpoint.Endpoint {
		return func(ctx context.Context, request interface{}) (response interface{}, err error) {
			defer func(begin time.Time) {
				input, _ := json.Marshal(request)
				output, _ := json.Marshal(response)
				_ = logger.Log(
					"method", method,
					"input", string(input),
					"output", string(output),
					"err", err,
					"took", time.Since(begin),
				)
			}(time.Now())
			response, err = next(ctx, request)
			return
		}
	}
}
`

// InterfaceTemplate
var InterfaceTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

{{AsComment $schema.Description}} type {{$title}}Service interface { {{range $funcKey, $funcValue := $schema.Functions}}
{{AsComment $funcValue.Description}} {{$funcKey}}(ctx context.Context, req *{{Argumentize $funcValue.Properties.incoming}}) (res *{{Argumentize $funcValue.Properties.outgoing}}, err error)
{{end}}
}
`

// TransportHTTPServerTemplate
var TransportHTTPServerTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

import (
	"encoding/json"
	"net/http"

	"golang.org/x/net/context"

	"github.com/go-kit/kit/endpoint"
	httptransport "github.com/go-kit/kit/transport/http"
)

// Handler functions

{{range $funcKey, $funcValue := $schema.Functions}}
func New{{$funcKey}}Handler(ctx context.Context, svc {{$title}}Service, middleware endpoint.Middleware, options ...httptransport.ServerOption) *httptransport.Server {
	return httptransport.NewServer(
		ctx,
		middleware(make{{$funcKey}}Endpoint(svc)),
		decode{{$funcKey}}Request,
		encodeResponse,
		options...,
	)
}
{{end}}

// Endpoint functions

{{range $funcKey, $funcValue := $schema.Functions}}
func make{{$funcKey}}Endpoint(svc {{$title}}Service) endpoint.Endpoint {
	return func(ctx context.Context, request interface{}) (interface{}, error) {
		req := request.(*{{Argumentize $funcValue.Properties.incoming}})
		return svc.{{$funcKey}}(ctx, req)
	}
}
{{end}}
`

// TransportHTTPClientTemplate
var TransportHTTPClientTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

import (
	jujuratelimit "github.com/juju/ratelimit"
	"github.com/sony/gobreaker"
	"golang.org/x/net/context"

	"github.com/go-kit/kit/circuitbreaker"
	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/loadbalancer"
	"github.com/go-kit/kit/loadbalancer/static"
	"github.com/go-kit/kit/log"
	kitratelimit "github.com/go-kit/kit/ratelimit"
	httptransport "github.com/go-kit/kit/transport/http"
)

// {{$title}}Client holds remote endpoint functions
// Satisfies {{$title}}Service interface
type {{$title}}Client struct {
	{{range $funcKey, $funcValue := $schema.Functions}}// {{$funcKey}}LoadBalancer provides remote call to {{ToLower $funcKey}} endpoints
		{{$funcKey}}LoadBalancer loadbalancer.LoadBalancer

	{{end}}
}

// New{{$title}}Client creates a new client for {{$title}}Service
func  New{{$title}}Client(proxies []string, logger log.Logger, clientOpts []httptransport.ClientOption, middlewares []endpoint.Middleware) *{{$title}}Client {
	return &{{$title}}Client{ {{range $funcKey, $funcValue := $schema.Functions}}
		{{$funcKey}}LoadBalancer : createClientLoadBalancer(semiotics["{{ToLower $funcKey}}"], proxies, logger, clientOpts, middlewares),{{end}}
	}
}

{{range $funcKey, $funcValue := $schema.Functions}}
{{AsComment $funcValue.Description}}func ({{Pointerize $title}} *{{$title}}Client) {{$funcKey}}(ctx context.Context, req *{{Argumentize $funcValue.Properties.incoming}}) (*{{Argumentize $funcValue.Properties.outgoing}}, error) {
	res, err := {{Pointerize $title}}.{{$funcKey}}Endpoint(ctx, req)
	if err != nil {
		return nil, err
	}

	return res.(*{{Argumentize $funcValue.Properties.outgoing}}), nil
}
{{end}}

// Client Endpoint functions
{{range $funcKey, $funcValue := $schema.Functions}}
func new{{$funcKey}}ClientEndpoint(proxies []string, ctx context.Context, maxAttempt int, maxTime time.Duration, qps int, logger log.Logger) endpoint.Endpoint {
	factory := createFactory(ctx, qps, make{{$funcKey}}Proxy)
	return defaultClientEndpointCreator(proxies, maxAttempt, maxTime, logger, factory)
}
{{end}}

{{range $funcKey, $funcValue := $schema.Functions}}
func make{{$funcKey}}Proxy(ctx context.Context, instance string) endpoint.Endpoint {
	return httptransport.NewClient(
		"POST",
		createProxyURL(instance, "{{ToLower $funcKey}}"),
		encodeRequest,
		decode{{$funcKey}}Response,
	).Endpoint()
}
{{end}}

// Proxy functions

func createProxyURL(instance, endpoint string) *url.URL {
	if !strings.HasPrefix(instance, "http") {
		instance = "http://" + instance
	}
	u, err := url.Parse(instance)
	if err != nil {
		panic(err)
	}
	if u.Path == "" {
		u.Path = endpoint
	}

	return u
}

type proxyFunc func(context.Context, string) endpoint.Endpoint

func createFactory(ctx context.Context, qps int, pf proxyFunc) loadbalancer.Factory {
	return func(instance string) (endpoint.Endpoint, io.Closer, error) {
		var e endpoint.Endpoint
		e = pf(ctx, instance)
		e = circuitbreaker.Gobreaker(gobreaker.NewCircuitBreaker(gobreaker.Settings{}))(e)
		e = kitratelimit.NewTokenBucketLimiter(jujuratelimit.NewBucketWithRate(float64(qps), int64(qps)))(e)
		return e, nil, nil
	}
}

func defaultClientEndpointCreator(
	proxies []string,
	maxAttempts int,
	maxTime time.Duration,
	logger log.Logger,
	factory loadbalancer.Factory,
) endpoint.Endpoint {

	publisher := static.NewPublisher(
		proxies,
		factory,
		logger,
	)

	lb := loadbalancer.NewRoundRobin(publisher)
	return loadbalancer.Retry(maxAttempts, maxTime, lb)
}

`

// TransportHTTPSemioticsTemplate
var TransportHTTPSemioticsTemplate = `
{{$schema := .Schema}}
{{$title := ToUpperFirst .Schema.Title}}

package {{ToLower $title}}

import (
	"encoding/json"
	"net/http"

	"golang.org/x/net/context"

	"github.com/go-kit/kit/endpoint"
	httptransport "github.com/go-kit/kit/transport/http"
)

// Decode Request functions

{{range $funcKey, $funcValue := $schema.Functions}}
func decode{{$funcKey}}Request(r *http.Request) (interface{}, error) {
	var req {{Argumentize $funcValue.Properties.incoming}}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		return nil, err
	}
	return &req, nil
}
{{end}}

// Decode Response functions

{{range $funcKey, $funcValue := $schema.Functions}}
func decode{{$funcKey}}Response(r *http.Response) (interface{}, error) {
	var res {{Argumentize $funcValue.Properties.incoming}}
	if err := json.NewDecoder(r.Body).Decode(&res); err != nil {
		return nil, err
	}
	return &res, nil
}
{{end}}

// Encode request function

func encodeRequest(r *http.Request, request interface{}) error {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(request); err != nil {
		return err
	}
	r.Body = ioutil.NopCloser(&buf)
	return nil
}

// Encode response function

func encodeResponse(rw http.ResponseWriter, response interface{}) error {
	return json.NewEncoder(rw).Encode(response)
}
`
<|endoftext|>


<commit_before>package kernel

import (
	"log"
	"os"
	"path"
	"fmt"
	"strings"
	"net/http"
	"encoding/json"

	"bytengine/kernel/modules"

	"github.com/gorilla/mux"
	"github.com/gorilla/schema"
)

// Global vars
//var Eng *CommandEngine
var EngReqChan chan modules.Request

// web vars
var decoder = schema.NewDecoder()
var ROOT_DIR string

func renderView(view string) []byte {
	var b []byte

	// load file
	view_f := path.Join(ROOT_DIR,"templates",view)
	file, err := os.Open(view_f)
	if err != nil {
		return b
	}
	defer file.Close()

	// get file size
	stat, err := file.Stat()
	if err != nil {
		return b
	}

	// read file
	b = make([]byte, stat.Size())
	_, err = file.Read(b)
	if err != nil {
		return b
	}

	return b
}

func welcomeHandler(w http.ResponseWriter, r *http.Request) {
	b := renderView("index.html")
	w.Header().Set("Content-Type","text/html")
	w.Write(b)
}

func documentationHandler(w http.ResponseWriter, r *http.Request) {
	b := renderView("documentation.html")
	w.Header().Set("Content-Type","text/html")
	w.Write(b)
}

func commandsHandler(w http.ResponseWriter, r *http.Request) {
	b := renderView("commands.html")
	w.Header().Set("Content-Type","text/html")
	w.Write(b)
}

func staticFileHandler(w http.ResponseWriter, r *http.Request) {
	_prefix := "/static/"
	_path := path.Clean(r.URL.Path)
	if !strings.HasPrefix(_path, _prefix) {
		http.Error(w, "", http.StatusNotFound)
		return
	}
	_path = path.Join(ROOT_DIR, _path)
	http.ServeFile(w, r, _path)
}

func pingHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type","text/plain")
	fmt.Fprintf(w, "pong!")
}

func readUploadData(outpath string, r *http.Request) (int, error) {
	total := 0 // total bytes
	maxbytes := 102400 // hard limit of 100kb just for test

	// create read buffer
	var bsize int64 = 16 * 1024 // 16 kb
	buffer := make([]byte, bsize)

	// get stream
	mr, err := r.MultipartReader()
	if err != nil {
		return total, err
	}
	in_f, err := mr.NextPart()
	if err != nil {
		return total, err
	}
	defer in_f.Close()

	// create output file
	out_f, err := os.OpenFile(outpath, os.O_WRONLY|os.O_CREATE, 0777)
	if err != nil {
		return total, err
	}
	defer out_f.Close()

	// start reading/writing

	for {
		// read
		n, err := in_f.Read(buffer)
		if n == 0 { break }
		if err != nil {
			return total, err
		}
		// update total bytes
		total += n
		if total > maxbytes {
			return total, fmt.Errorf("exceeded maximum file size of %d bytes",maxbytes)
		}
		// write
		n, err = out_f.Write(buffer[:n])
		if err != nil {
			return total, err
		}
	}

	return total, nil
}

func writeDownloadData(filep, mime string, size int64, w http.ResponseWriter) {
	// create read buffer
	var bsize int64 = 16 * 1024 // 16 kb
	buffer := make([]byte, bsize)

	err_msg := "Error reading content. Contact admin."

	// open attachment
	out_f, err := os.Open(filep)
	if err != nil {
		fmt.Println(err)
		http.Error(w, err_msg, http.StatusNotFound)
		return
	}
	defer out_f.Close()

	// set response headers
	w.Header().Set("Content-Type",mime)
	w.Header().Set("Content-Length",fmt.Sprintf("%d",int64(size)))

	// start reading/writing
	for {
		// read
		n, err := out_f.Read(buffer)
		if n == 0 { break }
		if err != nil {
			fmt.Println(err)
			http.Error(w, err_msg, http.StatusNotFound)
			return
		}
		// write
		n, err = w.Write(buffer[:n])
		if err != nil {
			fmt.Println(err)
			http.Error(w, err_msg, http.StatusNotFound)
			return
		}
	}
	// finish
	return
}

func uploadAttachmentHandler(w http.ResponseWriter, r *http.Request) {
	// get ticket from url
	vars := mux.Vars(r)
	ticket := vars["ticket"]
	req := modules.NewUploadTCheckRequest(ticket)
    success, failure := req.GetChannels()
    EngReqChan <- req
    for {
        select {
        case msg := <- success:
        	// deserialize ticket info
        	b := msg[1]
        	var tmp interface{}
        	err := json.Unmarshal(b,&tmp)
        	if err != nil {
        		b2 := modules.ErrorResponse("Upload failed: ticket not verified", modules.EngineError)
        		w.Header().Set("Content-Type",string(msg[0]))
                w.WriteHeader(http.StatusBadRequest)
        		w.Write(b2[1])
        		return
        	}
        	// upload data to temp file
        	ticket_info := tmp.(map[string]interface{})["data"].([]interface{})
        	db := ticket_info[0].(string) // database to save to
        	bpath := ticket_info[1].(string) // bfs content path to attach to
        	tpath := ticket_info[2].(string) // uploaded file path

        	total, err := readUploadData(tpath, r)
        	if err != nil {
        		b2 := modules.ErrorResponse("Upload failed: error read/write data", modules.EngineError)
        		w.Header().Set("Content-Type",string(msg[0]))
                w.WriteHeader(http.StatusBadRequest)
        		w.Write(b2[1])
        		return
        	}
        	// send upload complete request
        	req2 := modules.NewUploadCompleteRequest(db, tpath, bpath, total)
        	success2, failure2 := req2.GetChannels()
    		EngReqChan <- req2
    		for {
    			select {
    				case msg2 := <- success2:
    					w.Header().Set("Content-Type",string(msg2[0]))
            			w.Write(msg2[1])
            			return
    				case msg2 := <- failure2:
    					w.Header().Set("Content-Type",string(msg2[0]))
                        w.WriteHeader(http.StatusBadRequest) // error 400
			            w.Write(msg2[1])
			            return
    			}
    		}
            return
        case msg := <- failure:
        	w.Header().Set("Content-Type",string(msg[0]))
            w.WriteHeader(http.StatusBadRequest) // error 400
            w.Write(msg[1])
            return
        }
    }
}

func runCommandHandler(w http.ResponseWriter, r *http.Request) {
	req, b := modules.ParsePostRequest(r)
    if len(b) > 0 {
        w.Header().Set("Content-Type",string(b[0]))
    	w.Write(b[1])
        return
    }

    success, failure := req.GetChannels()
    EngReqChan <- req
    for {
        select {
        case msg := <- success:
        	switch req.(type) {
        	case modules.DownloadRequest:
        		var j map[string]interface{}
        		err := json.Unmarshal(msg[1], &j)
        		if err != nil {
                    w.Header().Set("Content-Type",string(msg[0]))
        			w.Write([]byte(""))
		            return
        		}
        		filep := j["data"].(map[string]interface{})["filepointer"].(string)
        		size := j["data"].(map[string]interface{})["size"].(float64)
        		mime := j["data"].(map[string]interface{})["mime"].(string)

        		writeDownloadData(filep, mime, int64(size), w)

				return
        	default:
        		w.Header().Set("Content-Type",string(msg[0]))
	            w.Write(msg[1])
	            return
        	}
        case msg := <- failure:
            w.Header().Set("Content-Type",string(msg[0]))
        	w.Write(msg[1])
            return
        }
    }
}

// GET requests
func getRequestHandler(w http.ResponseWriter, r *http.Request) {
	req, b := modules.ParseGetRequest(r)
    if len(b) > 0 {
        w.WriteHeader(http.StatusBadRequest) // error 400
        w.Write(b[1])
        return
    }

    success, failure := req.GetChannels()
    EngReqChan <- req
    for {
        select {
        case msg := <- success:
        	w.Header().Set("Content-Type",string(msg[0]))
            w.Write(msg[1])
            return
        case msg := <- failure:
            w.WriteHeader(http.StatusBadRequest) // error 400
            w.Write(msg[1])
            return
        }
    }
}

func contentDileveryHandler(w http.ResponseWriter, r *http.Request){
	// get values from url
	vars := mux.Vars(r)
	db := vars["database"]
	typ := vars["type"]
	path := "/" + vars["path"]

	req := modules.NewDirectAccessRequest(path, typ, db)
	success, failure := req.GetChannels()
    EngReqChan <- req
    for {
        select {
        case msg := <- success:
        	if typ == "fa" {
        		var j map[string]interface{}
        		err := json.Unmarshal(msg[1], &j)
        		if err != nil {
        			w.WriteHeader(http.StatusNotFound) // error 400
		            w.Write([]byte(""))
		            return
        		}
        		filep := j["data"].(map[string]interface{})["filepointer"].(string)
        		size := j["data"].(map[string]interface{})["size"].(float64)
        		mime := j["data"].(map[string]interface{})["mime"].(string)

        		writeDownloadData(filep, mime, int64(size), w)

				return
        	}
        	w.Header().Set("Content-Type",string(msg[0]))
            w.Write(msg[1])
            return
        case msg := <- failure:
        	w.WriteHeader(http.StatusBadRequest) // error 400
            w.Write(msg[1])
            return
        }
    }
}

// Build url router
func buildRoutes() (*mux.Router, error) {
	// create router
	r := mux.NewRouter()

	// API urls
	r.HandleFunc("/bfs/prq/{type}", runCommandHandler).Methods("POST")
	r.HandleFunc("/bfs/upload/{ticket}", uploadAttachmentHandler).Methods("POST")
	r.HandleFunc("/cds/{type:fd|fa}/{database}/{path:[\\w\\.0-9/_\\-]+}", contentDileveryHandler).Methods("GET")

	/*
		Info Url
		=================
		/info/version --> bytengine server version
	*/
	r.HandleFunc("/bfs/grq/{type}/{query:[\\w\\.0-9_@]*}", getRequestHandler).Methods("GET")

	// Web UI
	r.HandleFunc("/", welcomeHandler).Methods("GET")
	r.HandleFunc("/ping", pingHandler).Methods("GET")
	r.HandleFunc("/docs", documentationHandler).Methods("GET")
	r.HandleFunc("/commands", commandsHandler).Methods("GET")

	// Static files
	r.HandleFunc("/static/{path:[\\w\\.0-9/_\\-]+}",staticFileHandler).Methods("GET")

	return r, nil
}

func Run(configfile string) {
	// start command engine
	EngReqChan = make(chan modules.Request)
	engine := modules.NewCommandEngine(configfile)
    go engine.Run(EngReqChan)

	// set root directory
	ROOT_DIR = path.Join(path.Dir(configfile),"..","core","web")

	// create router
	r, err := buildRoutes()
	if err != nil {
		log.Panic(err)
	}

	// create app handler
	http.Handle("/", r)
	log.Fatal(http.ListenAndServe(engine.SysInit.WebUrl(), nil))
}<commit_msg>Updated web handlers and addded Http "HEAD" to methods<commit_after>package kernel

import (
	"log"
	"os"
	"path"
	"fmt"
	"strings"
	"net/http"
	"encoding/json"

	"bytengine/kernel/modules"

	"github.com/gorilla/mux"
	"github.com/gorilla/schema"
)

// Global vars
//var Eng *CommandEngine
var EngReqChan chan modules.Request

// web vars
var decoder = schema.NewDecoder()
var ROOT_DIR string

func renderView(view string) []byte {
	var b []byte

	// load file
	view_f := path.Join(ROOT_DIR,"templates",view)
	file, err := os.Open(view_f)
	if err != nil {
		return b
	}
	defer file.Close()

	// get file size
	stat, err := file.Stat()
	if err != nil {
		return b
	}

	// read file
	b = make([]byte, stat.Size())
	_, err = file.Read(b)
	if err != nil {
		return b
	}

	return b
}

func welcomeHandler(w http.ResponseWriter, r *http.Request) {
	b := renderView("index.html")
	w.Header().Set("Content-Type","text/html")
	w.Write(b)
}

func documentationHandler(w http.ResponseWriter, r *http.Request) {
	b := renderView("documentation.html")
	w.Header().Set("Content-Type","text/html")
	w.Write(b)
}

func commandsHandler(w http.ResponseWriter, r *http.Request) {
	b := renderView("commands.html")
	w.Header().Set("Content-Type","text/html")
	w.Write(b)
}

func staticFileHandler(w http.ResponseWriter, r *http.Request) {
	_prefix := "/static/"
	_path := path.Clean(r.URL.Path)
	if !strings.HasPrefix(_path, _prefix) {
		http.Error(w, "", http.StatusNotFound)
		return
	}
	_path = path.Join(ROOT_DIR, _path)
	http.ServeFile(w, r, _path)
}

func pingHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type","text/plain")
	fmt.Fprintf(w, "pong!")
}

func readUploadData(outpath string, r *http.Request) (int, error) {
	total := 0 // total bytes
	maxbytes := 102400 // hard limit of 100kb just for test

	// create read buffer
	var bsize int64 = 16 * 1024 // 16 kb
	buffer := make([]byte, bsize)

	// get stream
	mr, err := r.MultipartReader()
	if err != nil {
		return total, err
	}
	in_f, err := mr.NextPart()
	if err != nil {
		return total, err
	}
	defer in_f.Close()

	// create output file
	out_f, err := os.OpenFile(outpath, os.O_WRONLY|os.O_CREATE, 0777)
	if err != nil {
		return total, err
	}
	defer out_f.Close()

	// start reading/writing

	for {
		// read
		n, err := in_f.Read(buffer)
		if n == 0 { break }
		if err != nil {
			return total, err
		}
		// update total bytes
		total += n
		if total > maxbytes {
			return total, fmt.Errorf("exceeded maximum file size of %d bytes",maxbytes)
		}
		// write
		n, err = out_f.Write(buffer[:n])
		if err != nil {
			return total, err
		}
	}

	return total, nil
}

func writeDownloadData(filep, mime string, size int64, w http.ResponseWriter) {
	// create read buffer
	var bsize int64 = 16 * 1024 // 16 kb
	buffer := make([]byte, bsize)

	err_msg := "Error reading content. Contact admin."

	// open attachment
	out_f, err := os.Open(filep)
	if err != nil {
		fmt.Println(err)
		http.Error(w, err_msg, http.StatusNotFound)
		return
	}
	defer out_f.Close()

	// set response headers
	w.Header().Set("Content-Type",mime)
	w.Header().Set("Content-Length",fmt.Sprintf("%d",int64(size)))

	// start reading/writing
	for {
		// read
		n, err := out_f.Read(buffer)
		if n == 0 { break }
		if err != nil {
			fmt.Println(err)
			http.Error(w, err_msg, http.StatusNotFound)
			return
		}
		// write
		n, err = w.Write(buffer[:n])
		if err != nil {
			fmt.Println(err)
			http.Error(w, err_msg, http.StatusNotFound)
			return
		}
	}
	// finish
	return
}

func uploadAttachmentHandler(w http.ResponseWriter, r *http.Request) {
	// get ticket from url
	vars := mux.Vars(r)
	ticket := vars["ticket"]
	req := modules.NewUploadTCheckRequest(ticket)
    success, failure := req.GetChannels()
    EngReqChan <- req
    for {
        select {
        case msg := <- success:
        	// deserialize ticket info
        	b := msg[1]
        	var tmp interface{}
        	err := json.Unmarshal(b,&tmp)
        	if err != nil {
        		b2 := modules.ErrorResponse("Upload failed: ticket not verified", modules.EngineError)
        		w.Header().Set("Content-Type",string(msg[0]))
                w.WriteHeader(http.StatusBadRequest)
        		w.Write(b2[1])
        		return
        	}
        	// upload data to temp file
        	ticket_info := tmp.(map[string]interface{})["data"].([]interface{})
        	db := ticket_info[0].(string) // database to save to
        	bpath := ticket_info[1].(string) // bfs content path to attach to
        	tpath := ticket_info[2].(string) // uploaded file path

        	total, err := readUploadData(tpath, r)
        	if err != nil {
        		b2 := modules.ErrorResponse("Upload failed: error read/write data", modules.EngineError)
        		w.Header().Set("Content-Type",string(msg[0]))
                w.WriteHeader(http.StatusBadRequest)
        		w.Write(b2[1])
        		return
        	}
        	// send upload complete request
        	req2 := modules.NewUploadCompleteRequest(db, tpath, bpath, total)
        	success2, failure2 := req2.GetChannels()
    		EngReqChan <- req2
    		for {
    			select {
    				case msg2 := <- success2:
    					w.Header().Set("Content-Type",string(msg2[0]))
            			w.Write(msg2[1])
            			return
    				case msg2 := <- failure2:
    					w.Header().Set("Content-Type",string(msg2[0]))
                        w.WriteHeader(http.StatusBadRequest) // error 400
			            w.Write(msg2[1])
			            return
    			}
    		}
            return
        case msg := <- failure:
        	w.Header().Set("Content-Type",string(msg[0]))
            w.WriteHeader(http.StatusBadRequest) // error 400
            w.Write(msg[1])
            return
        }
    }
}

func runCommandHandler(w http.ResponseWriter, r *http.Request) {
	req, b := modules.ParsePostRequest(r)
    if len(b) > 0 {
        w.Header().Set("Content-Type",string(b[0]))
    	w.Write(b[1])
        return
    }

    success, failure := req.GetChannels()
    EngReqChan <- req
    for {
        select {
        case msg := <- success:
        	switch req.(type) {
        	case modules.DownloadRequest:
        		var j map[string]interface{}
        		err := json.Unmarshal(msg[1], &j)
        		if err != nil {
                    w.Header().Set("Content-Type",string(msg[0]))
        			w.Write([]byte(""))
		            return
        		}
        		filep := j["data"].(map[string]interface{})["filepointer"].(string)
        		size := j["data"].(map[string]interface{})["size"].(float64)
        		mime := j["data"].(map[string]interface{})["mime"].(string)

        		writeDownloadData(filep, mime, int64(size), w)

				return
        	default:
        		w.Header().Set("Content-Type",string(msg[0]))
	            w.Write(msg[1])
	            return
        	}
        case msg := <- failure:
            w.Header().Set("Content-Type",string(msg[0]))
        	w.Write(msg[1])
            return
        }
    }
}

// GET requests
func getRequestHandler(w http.ResponseWriter, r *http.Request) {
	req, b := modules.ParseGetRequest(r)
    if len(b) > 0 {
        w.WriteHeader(http.StatusBadRequest) // error 400
        w.Write(b[1])
        return
    }

    success, failure := req.GetChannels()
    EngReqChan <- req
    for {
        select {
        case msg := <- success:
        	w.Header().Set("Content-Type",string(msg[0]))
            w.Write(msg[1])
            return
        case msg := <- failure:
            w.WriteHeader(http.StatusBadRequest) // error 400
            w.Write(msg[1])
            return
        }
    }
}

func contentDileveryHandler(w http.ResponseWriter, r *http.Request){
	// get values from url
	vars := mux.Vars(r)
	db := vars["database"]
	typ := vars["type"]
	path := "/" + vars["path"]

	req := modules.NewDirectAccessRequest(path, typ, db)
	success, failure := req.GetChannels()
    EngReqChan <- req
    for {
        select {
        case msg := <- success:
        	if typ == "fa" {
        		var j map[string]interface{}
        		err := json.Unmarshal(msg[1], &j)
        		if err != nil {
        			w.WriteHeader(http.StatusNotFound) // error 400
		            w.Write([]byte(""))
		            return
        		}
        		filep := j["data"].(map[string]interface{})["filepointer"].(string)
        		size := j["data"].(map[string]interface{})["size"].(float64)
        		mime := j["data"].(map[string]interface{})["mime"].(string)

        		writeDownloadData(filep, mime, int64(size), w)

				return
        	}
        	w.Header().Set("Content-Type",string(msg[0]))
            w.Write(msg[1])
            return
        case msg := <- failure:
        	w.WriteHeader(http.StatusBadRequest) // error 400
            w.Write(msg[1])
            return
        }
    }
}

// Build url router
func buildRoutes() (*mux.Router, error) {
	// create router
	r := mux.NewRouter()

	// API urls
	r.HandleFunc("/bfs/prq/{type}", runCommandHandler).Methods("POST","HEAD")
	r.HandleFunc("/bfs/upload/{ticket}", uploadAttachmentHandler).Methods("POST","HEAD")
	r.HandleFunc("/cds/{type:fd|fa}/{database}/{path:[\\w\\.0-9/_\\-]+}", contentDileveryHandler).Methods("GET","HEAD")

	/*
		Info Url
		=================
		/info/version --> bytengine server version
	*/
	r.HandleFunc("/bfs/grq/{type}/{query:[\\w\\.0-9_@]*}", getRequestHandler).Methods("GET","HEAD")

	// Web UI
	r.HandleFunc("/", welcomeHandler).Methods("GET","HEAD")
	r.HandleFunc("/ping", pingHandler).Methods("GET","HEAD")
	r.HandleFunc("/docs", documentationHandler).Methods("GET","HEAD")
	r.HandleFunc("/commands", commandsHandler).Methods("GET","HEAD")

	// Static files
	r.HandleFunc("/static/{path:[\\w\\.0-9/_\\-]+}",staticFileHandler).Methods("GET","HEAD")

	return r, nil
}

func Run(configfile string) {
	// start command engine
	EngReqChan = make(chan modules.Request)
	engine := modules.NewCommandEngine(configfile)
    go engine.Run(EngReqChan)

	// set root directory
	ROOT_DIR = path.Join(path.Dir(configfile),"..","core","web")

	// create router
	r, err := buildRoutes()
	if err != nil {
		log.Panic(err)
	}

	// create app handler
	http.Handle("/", r)
	log.Fatal(http.ListenAndServe(engine.SysInit.WebUrl(), nil))
}<|endoftext|>


<commit_before>package auth

import (
	"net/http/cookiejar"
	"net/url"

	"github.com/MyHomeworkSpace/api-server/blackbaud"
)

func DaltonLogin(username string, password string) (map[string]interface{}, string, error) {
	schoolSlug := "dalton"

	// set up ajax token and stuff
	ajaxToken, err := blackbaud.GetAjaxToken(schoolSlug)
	if err != nil {
		return nil, "internal_server_error", err
	}

	jar, err := cookiejar.New(nil)
	if err != nil {
		return nil, "internal_server_error", err
	}

	// sign in to blackbaud
	response, err := blackbaud.Request(schoolSlug, "POST", "SignIn", url.Values{}, map[string]interface{}{
		"From":            "",
		"InterfaceSource": "WebApp",
		"Password":        password,
		"Username":        username,
		"remember":        "false",
	}, jar, ajaxToken)

	if err != nil {
		return nil, "dalton_creds_incorrect", nil
	}

	result, worked := (response.(map[string]interface{}))["AuthenticationResult"].(float64)

	if worked && result == 5 {
		return nil, "bb_signin_rate_limit", nil
	}

	if !worked || result == 2 {
		return nil, "dalton_creds_incorrect", nil
	}

	// get user id
	response, err = blackbaud.Request(schoolSlug, "GET", "webapp/context", url.Values{}, map[string]interface{}{}, jar, ajaxToken)
	if err != nil {
		return nil, "internal_server_error", err
	}

	// bbUserId := int(((response.(map[string]interface{}))["UserInfo"].(map[string]interface{}))["UserId"].(float64))
	firstName := ((response.(map[string]interface{}))["UserInfo"].(map[string]interface{}))["FirstName"].(string)
	lastName := ((response.(map[string]interface{}))["UserInfo"].(map[string]interface{}))["LastName"].(string)
	fullName := firstName + " " + lastName

	data := map[string]interface{}{}

	data["fullname"] = fullName
	roles := []string{}

	personas := response.(map[string]interface{})["Personas"].([]interface{})
	for _, persona := range personas {
		roles = append(roles, persona.(map[string]interface{})["UrlFriendlyDescription"].(string))
	}

	data["roles"] = roles

	return data, "", nil
}
<commit_msg>fix server crash when a login attempt is made with an incorrect username<commit_after>package auth

import (
	"net/http/cookiejar"
	"net/url"

	"github.com/MyHomeworkSpace/api-server/blackbaud"
)

func DaltonLogin(username string, password string) (map[string]interface{}, string, error) {
	schoolSlug := "dalton"

	// set up ajax token and stuff
	ajaxToken, err := blackbaud.GetAjaxToken(schoolSlug)
	if err != nil {
		return nil, "internal_server_error", err
	}

	jar, err := cookiejar.New(nil)
	if err != nil {
		return nil, "internal_server_error", err
	}

	// sign in to blackbaud
	response, err := blackbaud.Request(schoolSlug, "POST", "SignIn", url.Values{}, map[string]interface{}{
		"From":            "",
		"InterfaceSource": "WebApp",
		"Password":        password,
		"Username":        username,
		"remember":        "false",
	}, jar, ajaxToken)

	if err != nil {
		return nil, "dalton_creds_incorrect", nil
	}

	result, worked := (response.(map[string]interface{}))["AuthenticationResult"].(float64)

	if worked && result == 5 {
		return nil, "bb_signin_rate_limit", nil
	}

	if !worked || result == 1 || result == 2 {
		return nil, "dalton_creds_incorrect", nil
	}

	// get user id
	response, err = blackbaud.Request(schoolSlug, "GET", "webapp/context", url.Values{}, map[string]interface{}{}, jar, ajaxToken)
	if err != nil {
		return nil, "internal_server_error", err
	}

	// bbUserId := int(((response.(map[string]interface{}))["UserInfo"].(map[string]interface{}))["UserId"].(float64))
	firstName := ((response.(map[string]interface{}))["UserInfo"].(map[string]interface{}))["FirstName"].(string)
	lastName := ((response.(map[string]interface{}))["UserInfo"].(map[string]interface{}))["LastName"].(string)
	fullName := firstName + " " + lastName

	data := map[string]interface{}{}

	data["fullname"] = fullName
	roles := []string{}

	personas := response.(map[string]interface{})["Personas"].([]interface{})
	for _, persona := range personas {
		roles = append(roles, persona.(map[string]interface{})["UrlFriendlyDescription"].(string))
	}

	data["roles"] = roles

	return data, "", nil
}
<|endoftext|>


<commit_before>// Package authcookie implements creation and verification of signed
// authentication cookies.
//
// Cookie is a Base64 encoded (using URLEncoding, from RFC 4648) string, which
// consists of concatenation of expiration time, login, and signature:
//
// 	expiration time || login || signature
//
// where expiration time is the number of seconds since Unix epoch UTC
// indicating when this cookie must expire (8 bytes, big-endian, uint64), login
// is a byte string of arbitrary length (at least 1 byte, not null-terminated),
// and signature is 32 bytes of HMAC-SHA256(expiration_time || login, k), where
// k = HMAC-SHA256(expiration_time || login, secret key).
//
// Example:
//
//	secret := []byte("my secret key")
//
//	// Generate cookie valid for 24 hours for user "bender"
//	cookie := authcookie.NewSinceNow("bender", 60*60*24, secret)
//
//	// cookie is now:
//	// AAAAAE2ozc9iZW5kZXIJ8B7Av9N--nwafka4slEdceRxDJqBBIjRQspA7mjrgQ==
//	// send it to user's browser..
//
//	// To authenticate a user later, receive cookie and:
//	login := authcookie.Login(cookie, secret)
//	if login != "" {
//		// access for login granted
//	} else {
//		// access denied
//	}
//
// Note that login and expiration time are not encrypted, they are only signed
// and Base64 encoded.
package authcookie

import (
	"crypto/hmac"
	"crypto/subtle"
	"encoding/base64"
	"encoding/binary"
	"os"
	"time"
)

func getSignature(b []byte, secret []byte) []byte {
	keym := hmac.NewSHA256(secret)
	keym.Write(b)
	m := hmac.NewSHA256(keym.Sum())
	m.Write(b)
	return m.Sum()
}

// New returns a signed authentication cookie for the given login,
// expiration time in seconds since Unix epoch UTC, and secret key.
func New(login string, expires int64, secret []byte) string {
	llen := len(login)
	b := make([]byte, llen+8+32)
	// Put expiration time
	binary.BigEndian.PutUint64(b, uint64(expires))
	// Put login
	copy(b[8:], []byte(login))
	// Calculate and put signature
	sig := getSignature([]byte(b[:8+llen]), secret)
	copy(b[8+llen:], sig)
	// Base64-encode
	cookie := make([]byte, base64.URLEncoding.EncodedLen(len(b)))
	base64.URLEncoding.Encode(cookie, b)
	return string(cookie)
}

// NewSinceNow returns a signed authetication cookie for the given login,
// expiration time in seconds since current time, and secret key.
func NewSinceNow(login string, sec int64, secret []byte) string {
	return New(login, sec+time.Seconds(), secret)
}

// Parse verifies the given cookie with the secret key and returns login and
// expiration time extracted from the cookie. If the cookie fails verification
// or is not well-formed, the function returns an error.
//
// Callers must:
//
// 1. Check for the returned error and deny access if it's present.
//
// 2. Check the returned expiration time and deny access if it's in the past.
//
func Parse(cookie string, secret []byte) (login string, expires int64, err os.Error) {
	blen := base64.URLEncoding.DecodedLen(len(cookie))
	// Avoid allocation if cookie is too short
	if blen <= 8+32 {
		err = os.NewError("malformed cookie: too short")
		return
	}
	b := make([]byte, blen)
	blen, err = base64.URLEncoding.Decode(b, []byte(cookie))
	if err != nil {
		return
	}
	// Decoded length may be bifferent from max length, which
	// we allocated, so check it, and set new length for b
	if blen <= 8+32 {
		err = os.NewError("malformed cookie: too short")
		return
	}
	b = b[:blen]

	sig := b[blen-32:]
	data := b[:blen-32]

	realSig := getSignature(data, secret)
	if subtle.ConstantTimeCompare(realSig, sig) != 1 {
		err = os.NewError("wrong cookie signature")
		return
	}
	expires = int64(binary.BigEndian.Uint64(data[:8]))
	login = string(data[8:])
	return
}

// Login returns a valid login extracted from the given cookie and verified
// using the given secret key.  If verification fails or the cookie expired,
// the function returns an empty string.
func Login(cookie string, secret []byte) string {
	l, exp, err := Parse(cookie, secret)
	if err != nil {
		return ""
	}
	if exp < time.Seconds() {
		return ""
	}
	return l
}
<commit_msg>Simplify Login function<commit_after>// Package authcookie implements creation and verification of signed
// authentication cookies.
//
// Cookie is a Base64 encoded (using URLEncoding, from RFC 4648) string, which
// consists of concatenation of expiration time, login, and signature:
//
// 	expiration time || login || signature
//
// where expiration time is the number of seconds since Unix epoch UTC
// indicating when this cookie must expire (8 bytes, big-endian, uint64), login
// is a byte string of arbitrary length (at least 1 byte, not null-terminated),
// and signature is 32 bytes of HMAC-SHA256(expiration_time || login, k), where
// k = HMAC-SHA256(expiration_time || login, secret key).
//
// Example:
//
//	secret := []byte("my secret key")
//
//	// Generate cookie valid for 24 hours for user "bender"
//	cookie := authcookie.NewSinceNow("bender", 60*60*24, secret)
//
//	// cookie is now:
//	// AAAAAE2ozc9iZW5kZXIJ8B7Av9N--nwafka4slEdceRxDJqBBIjRQspA7mjrgQ==
//	// send it to user's browser..
//
//	// To authenticate a user later, receive cookie and:
//	login := authcookie.Login(cookie, secret)
//	if login != "" {
//		// access for login granted
//	} else {
//		// access denied
//	}
//
// Note that login and expiration time are not encrypted, they are only signed
// and Base64 encoded.
package authcookie

import (
	"crypto/hmac"
	"crypto/subtle"
	"encoding/base64"
	"encoding/binary"
	"os"
	"time"
)

func getSignature(b []byte, secret []byte) []byte {
	keym := hmac.NewSHA256(secret)
	keym.Write(b)
	m := hmac.NewSHA256(keym.Sum())
	m.Write(b)
	return m.Sum()
}

// New returns a signed authentication cookie for the given login,
// expiration time in seconds since Unix epoch UTC, and secret key.
func New(login string, expires int64, secret []byte) string {
	llen := len(login)
	b := make([]byte, llen+8+32)
	// Put expiration time
	binary.BigEndian.PutUint64(b, uint64(expires))
	// Put login
	copy(b[8:], []byte(login))
	// Calculate and put signature
	sig := getSignature([]byte(b[:8+llen]), secret)
	copy(b[8+llen:], sig)
	// Base64-encode
	cookie := make([]byte, base64.URLEncoding.EncodedLen(len(b)))
	base64.URLEncoding.Encode(cookie, b)
	return string(cookie)
}

// NewSinceNow returns a signed authetication cookie for the given login,
// expiration time in seconds since current time, and secret key.
func NewSinceNow(login string, sec int64, secret []byte) string {
	return New(login, sec+time.Seconds(), secret)
}

// Parse verifies the given cookie with the secret key and returns login and
// expiration time extracted from the cookie. If the cookie fails verification
// or is not well-formed, the function returns an error.
//
// Callers must:
//
// 1. Check for the returned error and deny access if it's present.
//
// 2. Check the returned expiration time and deny access if it's in the past.
//
func Parse(cookie string, secret []byte) (login string, expires int64, err os.Error) {
	blen := base64.URLEncoding.DecodedLen(len(cookie))
	// Avoid allocation if cookie is too short
	if blen <= 8+32 {
		err = os.NewError("malformed cookie: too short")
		return
	}
	b := make([]byte, blen)
	blen, err = base64.URLEncoding.Decode(b, []byte(cookie))
	if err != nil {
		return
	}
	// Decoded length may be bifferent from max length, which
	// we allocated, so check it, and set new length for b
	if blen <= 8+32 {
		err = os.NewError("malformed cookie: too short")
		return
	}
	b = b[:blen]

	sig := b[blen-32:]
	data := b[:blen-32]

	realSig := getSignature(data, secret)
	if subtle.ConstantTimeCompare(realSig, sig) != 1 {
		err = os.NewError("wrong cookie signature")
		return
	}
	expires = int64(binary.BigEndian.Uint64(data[:8]))
	login = string(data[8:])
	return
}

// Login returns a valid login extracted from the given cookie and verified
// using the given secret key.  If verification fails or the cookie expired,
// the function returns an empty string.
func Login(cookie string, secret []byte) string {
	l, exp, err := Parse(cookie, secret)
	if err != nil || exp < time.Seconds() {
		return ""
	}
	return l
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"

	mp "github.com/mackerelio/go-mackerel-plugin"
	"github.com/mackerelio/mackerel-agent/logging"
)

var logger = logging.GetLogger("metrics.plugin.jvm")

type JVMPlugin struct {
	Target    string
	Lvmid     string
	JstatPath string
	JavaName  string
	Tempfile  string
}

// # jps
// 26547 NettyServer
// 6438 Jps
func FetchLvmidByAppname(appname, target, jpsPath string) (string, error) {
	out, err := exec.Command(jpsPath, target).Output()
	if err != nil {
		logger.Errorf("Failed to run exec jps. %s", err)
		return "", err
	}
	for _, line := range strings.Split(string(out), "\n") {
		words := strings.Split(line, " ")
		if len(words) != 2 {
			continue
		}
		lvmid, name := words[0], words[1]
		if name == appname {
			return lvmid, nil
		}
	}
	return "", errors.New(fmt.Sprintf("Cannot get lvmid from %s", appname))
}

func fetchJstatMetrics(lvmid, option, jstatPath string) (map[string]float64, error) {
	out, err := exec.Command(jstatPath, option, lvmid).Output()
	if err != nil {
		logger.Errorf("Failed to run exec jstat. %s", err)
		return nil, err
	}

	lines := strings.Split(string(out), "\n")
	keys := strings.Fields(lines[0])
	values := strings.Fields(lines[1])

	stat := make(map[string]float64)
	for i, key := range keys {
		value, err := strconv.ParseFloat(values[i], 64)
		if err != nil {
			logger.Warningf("Failed to parse value. %s", err)
		}
		stat[key] = value
	}

	return stat, nil
}

func mergeStat(dst, src map[string]float64) {
	for k, v := range src {
		dst[k] = v
	}
}

// # jstat -gc <vmid>
//  S0C    S1C    S0U    S1U      EC       EU        OC         OU       PC     PU    YGC     YGCT    FGC    FGCT     GCT
// 3584.0 3584.0 2528.0  0.0   692224.0 19062.4  1398272.0   485450.1  72704.0 72611.3   3152   30.229   0      0.000   30.229

// # jstat -gccapacity  <vmid>
//  NGCMN    NGCMX     NGC     S0C   S1C       EC      OGCMN      OGCMX       OGC         OC      PGCMN    PGCMX     PGC       PC     YGC    FGC
// 699392.0 699392.0 699392.0 4096.0 4096.0 691200.0  1398272.0  1398272.0  1398272.0  1398272.0  21504.0 524288.0  72704.0  72704.0   4212     0

// # jstat -gcnew  <vmid>
//  S0C    S1C    S0U    S1U   TT MTT  DSS      EC       EU     YGC     YGCT
// 3072.0 3072.0    0.0 2848.0  1  15 3072.0 693248.0 626782.2   3463   33.658

func (m JVMPlugin) FetchMetrics() (map[string]float64, error) {
	gcStat, err := fetchJstatMetrics(m.Lvmid, "-gc", m.JstatPath)
	if err != nil {
		return nil, err
	}
	gcCapacityStat, err := fetchJstatMetrics(m.Lvmid, "-gccapacity", m.JstatPath)
	if err != nil {
		return nil, err
	}
	gcNewStat, err := fetchJstatMetrics(m.Lvmid, "-gcnew", m.JstatPath)
	if err != nil {
		return nil, err
	}
	gcOldStat, err := fetchJstatMetrics(m.Lvmid, "-gcold", m.JstatPath)
	if err != nil {
		return nil, err
	}

	stat := make(map[string]float64)
	mergeStat(stat, gcStat)
	mergeStat(stat, gcCapacityStat)
	mergeStat(stat, gcNewStat)
	mergeStat(stat, gcOldStat)

	return stat, nil
}

func (m JVMPlugin) GraphDefinition() map[string](mp.Graphs) {
	rawJavaName := m.JavaName
	lowerJavaName := strings.ToLower(m.JavaName)
	return map[string](mp.Graphs){
		fmt.Sprintf("jvm.%s.gc_events", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s GC events", rawJavaName),
			Unit:  "integer",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "YGC", Label: "Young GC event", Diff: true},
				mp.Metrics{Name: "FGC", Label: "Full GC event", Diff: true},
			},
		},
		fmt.Sprintf("jvm.%s.gc_time", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s GC time (msec)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "YGCT", Label: "Young GC time", Diff: true},
				mp.Metrics{Name: "FGCT", Label: "Full GC time", Diff: true},
			},
		},
		fmt.Sprintf("jvm.%s.new_space", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s New Space memory (KB)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "NGCMX", Label: "New max", Diff: false},
				mp.Metrics{Name: "NGC", Label: "New current", Diff: false},
				mp.Metrics{Name: "EU", Label: "Eden used", Diff: false},
				mp.Metrics{Name: "S0U", Label: "Survivor0 used", Diff: false},
				mp.Metrics{Name: "S1U", Label: "Survivor1 used", Diff: false},
			},
		},
		fmt.Sprintf("jvm.%s.old_space", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s Old Space memory (KB)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "OGCMX", Label: "Old max", Diff: false},
				mp.Metrics{Name: "OGC", Label: "Old current", Diff: false},
				mp.Metrics{Name: "OU", Label: "Old used", Diff: false},
			},
		},
		fmt.Sprintf("jvm.%s.perm_space", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s Permanent Space (KB)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "PGCMX", Label: "Perm max", Diff: false},
				mp.Metrics{Name: "PGC", Label: "Perm current", Diff: false},
				mp.Metrics{Name: "PU", Label: "Perm used", Diff: false},
			},
		},
	}
}

func main() {
	optHost := flag.String("host", "localhost", "Hostname")
	optPort := flag.String("port", "1099", "Port")
	optJstatPath := flag.String("jstatpath", "/usr/bin/jstat", "jstat path")
	optJpsPath := flag.String("jpspath", "/usr/bin/jps", "jps path")
	optJavaName := flag.String("javaname", "", "Java app name")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	if *optJavaName == "" {
		logger.Errorf("javaname is required")
		flag.PrintDefaults()
		os.Exit(1)
	}

	var jvm JVMPlugin
	jvm.Target = fmt.Sprintf("%s:%s", *optHost, *optPort)
	lvmid, err := FetchLvmidByAppname(*optJavaName, jvm.Target, *optJpsPath)
	if err != nil {
		logger.Errorf("Failed to fetch lvmid. %s", err)
		os.Exit(1)
	}
	jvm.Lvmid = lvmid
	jvm.JstatPath = *optJstatPath
	jvm.JavaName = *optJavaName

	helper := mp.NewMackerelPlugin(jvm)
	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.Tempfile = fmt.Sprintf("/tmp/mackerel-plugin-jvm-%s", *optHost)
	}

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<commit_msg>Add GC time percentage graph<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"

	mp "github.com/mackerelio/go-mackerel-plugin"
	"github.com/mackerelio/mackerel-agent/logging"
)

var logger = logging.GetLogger("metrics.plugin.jvm")

type JVMPlugin struct {
	Target    string
	Lvmid     string
	JstatPath string
	JavaName  string
	Tempfile  string
}

// # jps
// 26547 NettyServer
// 6438 Jps
func FetchLvmidByAppname(appname, target, jpsPath string) (string, error) {
	out, err := exec.Command(jpsPath, target).Output()
	if err != nil {
		logger.Errorf("Failed to run exec jps. %s", err)
		return "", err
	}
	for _, line := range strings.Split(string(out), "\n") {
		words := strings.Split(line, " ")
		if len(words) != 2 {
			continue
		}
		lvmid, name := words[0], words[1]
		if name == appname {
			return lvmid, nil
		}
	}
	return "", errors.New(fmt.Sprintf("Cannot get lvmid from %s", appname))
}

func fetchJstatMetrics(lvmid, option, jstatPath string) (map[string]float64, error) {
	out, err := exec.Command(jstatPath, option, lvmid).Output()
	if err != nil {
		logger.Errorf("Failed to run exec jstat. %s", err)
		return nil, err
	}

	lines := strings.Split(string(out), "\n")
	keys := strings.Fields(lines[0])
	values := strings.Fields(lines[1])

	stat := make(map[string]float64)
	for i, key := range keys {
		value, err := strconv.ParseFloat(values[i], 64)
		if err != nil {
			logger.Warningf("Failed to parse value. %s", err)
		}
		stat[key] = value
	}

	return stat, nil
}

func mergeStat(dst, src map[string]float64) {
	for k, v := range src {
		dst[k] = v
	}
}

// # jstat -gc <vmid>
//  S0C    S1C    S0U    S1U      EC       EU        OC         OU       PC     PU    YGC     YGCT    FGC    FGCT     GCT
// 3584.0 3584.0 2528.0  0.0   692224.0 19062.4  1398272.0   485450.1  72704.0 72611.3   3152   30.229   0      0.000   30.229

// # jstat -gccapacity  <vmid>
//  NGCMN    NGCMX     NGC     S0C   S1C       EC      OGCMN      OGCMX       OGC         OC      PGCMN    PGCMX     PGC       PC     YGC    FGC
// 699392.0 699392.0 699392.0 4096.0 4096.0 691200.0  1398272.0  1398272.0  1398272.0  1398272.0  21504.0 524288.0  72704.0  72704.0   4212     0

// # jstat -gcnew  <vmid>
//  S0C    S1C    S0U    S1U   TT MTT  DSS      EC       EU     YGC     YGCT
// 3072.0 3072.0    0.0 2848.0  1  15 3072.0 693248.0 626782.2   3463   33.658

func (m JVMPlugin) FetchMetrics() (map[string]float64, error) {
	gcStat, err := fetchJstatMetrics(m.Lvmid, "-gc", m.JstatPath)
	if err != nil {
		return nil, err
	}
	gcCapacityStat, err := fetchJstatMetrics(m.Lvmid, "-gccapacity", m.JstatPath)
	if err != nil {
		return nil, err
	}
	gcNewStat, err := fetchJstatMetrics(m.Lvmid, "-gcnew", m.JstatPath)
	if err != nil {
		return nil, err
	}
	gcOldStat, err := fetchJstatMetrics(m.Lvmid, "-gcold", m.JstatPath)
	if err != nil {
		return nil, err
	}

	stat := make(map[string]float64)
	mergeStat(stat, gcStat)
	mergeStat(stat, gcCapacityStat)
	mergeStat(stat, gcNewStat)
	mergeStat(stat, gcOldStat)

	return stat, nil
}

func (m JVMPlugin) GraphDefinition() map[string](mp.Graphs) {
	rawJavaName := m.JavaName
	lowerJavaName := strings.ToLower(m.JavaName)
	return map[string](mp.Graphs){
		fmt.Sprintf("jvm.%s.gc_events", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s GC events", rawJavaName),
			Unit:  "integer",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "YGC", Label: "Young GC event", Diff: true},
				mp.Metrics{Name: "FGC", Label: "Full GC event", Diff: true},
			},
		},
		fmt.Sprintf("jvm.%s.gc_time", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s GC time (msec)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "YGCT", Label: "Young GC time", Diff: true},
				mp.Metrics{Name: "FGCT", Label: "Full GC time", Diff: true},
			},
		},
		fmt.Sprintf("jvm.%s.gc_time_percentage", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s GC time percentage", rawJavaName),
			Unit:  "percentage",
			Metrics: [](mp.Metrics){
				// gc_time_percentage is the percentage of gc time to 60 sec.
				mp.Metrics{Name: "YGCT", Label: "Young GC time", Diff: true, Scale: (0.001 / 60)},
				mp.Metrics{Name: "FGCT", Label: "Full GC time", Diff: true, Scale: (0.001 / 60)},
			},
		},
		fmt.Sprintf("jvm.%s.new_space", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s New Space memory (KB)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "NGCMX", Label: "New max", Diff: false},
				mp.Metrics{Name: "NGC", Label: "New current", Diff: false},
				mp.Metrics{Name: "EU", Label: "Eden used", Diff: false},
				mp.Metrics{Name: "S0U", Label: "Survivor0 used", Diff: false},
				mp.Metrics{Name: "S1U", Label: "Survivor1 used", Diff: false},
			},
		},
		fmt.Sprintf("jvm.%s.old_space", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s Old Space memory (KB)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "OGCMX", Label: "Old max", Diff: false},
				mp.Metrics{Name: "OGC", Label: "Old current", Diff: false},
				mp.Metrics{Name: "OU", Label: "Old used", Diff: false},
			},
		},
		fmt.Sprintf("jvm.%s.perm_space", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s Permanent Space (KB)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "PGCMX", Label: "Perm max", Diff: false},
				mp.Metrics{Name: "PGC", Label: "Perm current", Diff: false},
				mp.Metrics{Name: "PU", Label: "Perm used", Diff: false},
			},
		},
	}
}

func main() {
	optHost := flag.String("host", "localhost", "Hostname")
	optPort := flag.String("port", "1099", "Port")
	optJstatPath := flag.String("jstatpath", "/usr/bin/jstat", "jstat path")
	optJpsPath := flag.String("jpspath", "/usr/bin/jps", "jps path")
	optJavaName := flag.String("javaname", "", "Java app name")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	if *optJavaName == "" {
		logger.Errorf("javaname is required")
		flag.PrintDefaults()
		os.Exit(1)
	}

	var jvm JVMPlugin
	jvm.Target = fmt.Sprintf("%s:%s", *optHost, *optPort)
	lvmid, err := FetchLvmidByAppname(*optJavaName, jvm.Target, *optJpsPath)
	if err != nil {
		logger.Errorf("Failed to fetch lvmid. %s", err)
		os.Exit(1)
	}
	jvm.Lvmid = lvmid
	jvm.JstatPath = *optJstatPath
	jvm.JavaName = *optJavaName

	helper := mp.NewMackerelPlugin(jvm)
	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.Tempfile = fmt.Sprintf("/tmp/mackerel-plugin-jvm-%s", *optHost)
	}

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package ghcache implements an HTTP cache optimized for caching responses
// from the GitHub API (https://api.github.com).
//
// Specifically, it enforces a cache policy that revalidates every cache hit
// with a conditional request to upstream regardless of cache entry freshness
// because conditional requests for unchanged resources don't cost any API
// tokens!!! See: https://developer.github.com/v3/#conditional-requests
//
// It also provides request coalescing and prometheus instrumentation.
package ghcache

import (
	"context"
	"crypto/sha256"
	"fmt"
	"net/http"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/gomodule/redigo/redis"
	"github.com/gregjones/httpcache"
	"github.com/gregjones/httpcache/diskcache"
	rediscache "github.com/gregjones/httpcache/redis"
	"github.com/peterbourgon/diskv"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/sirupsen/logrus"
	"golang.org/x/sync/semaphore"
	"k8s.io/test-infra/ghproxy/ghmetrics"
)

type CacheResponseMode string

// Cache response modes describe how ghcache fulfilled a request.
const (
	CacheModeHeader = "X-Cache-Mode"

	ModeError   CacheResponseMode = "ERROR"    // internal error handling request
	ModeNoStore CacheResponseMode = "NO-STORE" // response not cacheable
	ModeMiss    CacheResponseMode = "MISS"     // not in cache, request proxied and response cached.
	ModeChanged CacheResponseMode = "CHANGED"  // cache value invalid: resource changed, cache updated
	// The modes below are the happy cases in which the request is fulfilled for
	// free (no API tokens used).
	ModeCoalesced   CacheResponseMode = "COALESCED"   // coalesced request, this is a copied response
	ModeRevalidated CacheResponseMode = "REVALIDATED" // cached value revalidated and returned
)

func CacheModeIsFree(mode CacheResponseMode) bool {
	switch mode {
	case ModeCoalesced:
		return true
	case ModeRevalidated:
		return true
	case ModeError:
		// In this case we did not successfully communicate with the GH API, so no
		// token is used, but we also don't return a response, so ModeError won't
		// ever be returned as a value of CacheModeHeader.
		return true
	}
	return false
}

// cacheCounter provides the 'ghcache_responses' counter vec that is indexed
// by the cache response mode.
var cacheCounter = prometheus.NewCounterVec(
	prometheus.CounterOpts{
		Name: "ghcache_responses",
		Help: "How many cache responses of each cache response mode there are.",
	},
	[]string{"mode"},
)

// outboundConcurrencyGauge provides the 'concurrent_outbound_requests' gauge that
// is global to the proxy.
var outboundConcurrencyGauge = prometheus.NewGauge(prometheus.GaugeOpts{
	Name: "concurrent_outbound_requests",
	Help: "How many concurrent requests are in flight to GitHub servers.",
})

// pendingOutboundConnectionsGauge provides the 'pending_outbound_requests' gauge that
// is global to the proxy.
var pendingOutboundConnectionsGauge = prometheus.NewGauge(prometheus.GaugeOpts{
	Name: "pending_outbound_requests",
	Help: "How many pending requests are waiting to be sent to GitHub servers.",
})

func init() {
	prometheus.MustRegister(cacheCounter)
	prometheus.MustRegister(outboundConcurrencyGauge)
	prometheus.MustRegister(pendingOutboundConnectionsGauge)
}

func cacheResponseMode(headers http.Header) CacheResponseMode {
	if strings.Contains(headers.Get("Cache-Control"), "no-store") {
		return ModeNoStore
	}
	if strings.Contains(headers.Get("Status"), "304 Not Modified") {
		return ModeRevalidated
	}
	if headers.Get("X-Conditional-Request") != "" {
		return ModeChanged
	}
	return ModeMiss
}

func newThrottlingTransport(maxConcurrency int, delegate http.RoundTripper) http.RoundTripper {
	return &throttlingTransport{sem: semaphore.NewWeighted(int64(maxConcurrency)), delegate: delegate}
}

// throttlingTransport throttles outbound concurrency from the proxy
type throttlingTransport struct {
	sem      *semaphore.Weighted
	delegate http.RoundTripper
}

func (c *throttlingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	pendingOutboundConnectionsGauge.Inc()
	if err := c.sem.Acquire(context.Background(), 1); err != nil {
		logrus.WithField("cache-key", req.URL.String()).WithError(err).Error("Internal error acquiring semaphore.")
		return nil, err
	}
	defer c.sem.Release(1)
	pendingOutboundConnectionsGauge.Dec()
	outboundConcurrencyGauge.Inc()
	defer outboundConcurrencyGauge.Dec()
	return c.delegate.RoundTrip(req)
}

// upstreamTransport changes response headers from upstream before they
// reach the cache layer in order to force the caching policy we require.
//
// By default github responds to PR requests with:
//    Cache-Control: private, max-age=60, s-maxage=60
// Which means the httpcache would not consider anything stale for 60 seconds.
// However, we want to always revalidate cache entries using ETags and last
// modified times so this RoundTripper overrides response headers to:
//    Cache-Control: no-cache
// This instructs the cache to store the response, but always consider it stale.
type upstreamTransport struct {
	delegate http.RoundTripper
}

func (u upstreamTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	etag := req.Header.Get("if-none-match")

	// get authorization header to convert to sha256
	authHeader := req.Header.Get("Authorization")
	if authHeader == "" {
		logrus.Warn("Couldn't retrieve 'Authorization' header, adding to unknown bucket")
		authHeader = "unknown"
	}
	hasher := sha256.New()
	hasher.Write([]byte(authHeader))
	authHeaderHash := fmt.Sprintf("%x", hasher.Sum(nil)) // use %x to make this a utf-8 string for use as a label

	reqStartTime := time.Now()
	// Don't modify request, just pass to delegate.
	resp, err := u.delegate.RoundTrip(req)
	if err != nil {
		logrus.WithField("cache-key", req.URL.String()).WithError(err).Error("Error from upstream (GitHub).")
		return nil, err
	}
	responseTime := time.Now()
	roundTripTime := responseTime.Sub(reqStartTime)

	if resp.StatusCode >= 400 {
		// Don't store errors. They can't be revalidated to save API tokens.
		resp.Header.Set("Cache-Control", "no-store")
	} else {
		resp.Header.Set("Cache-Control", "no-cache")
	}
	if etag != "" {
		resp.Header.Set("X-Conditional-Request", etag)
	}

	apiVersion := "v3"
	if strings.HasPrefix(req.URL.Path, "graphql") || strings.HasPrefix(req.URL.Path, "/graphql") {
		apiVersion = "v4"
	}

	ghmetrics.CollectGitHubTokenMetrics(authHeaderHash, apiVersion, resp.Header, reqStartTime, responseTime)
	ghmetrics.CollectGitHubRequestMetrics(authHeaderHash, req.URL.Path, strconv.Itoa(resp.StatusCode), roundTripTime.Seconds())

	return resp, nil
}

// NewDiskCache creates a GitHub cache RoundTripper that is backed by a disk
// cache.
func NewDiskCache(delegate http.RoundTripper, cacheDir string, cacheSizeGB, maxConcurrency int) http.RoundTripper {
	return NewFromCache(delegate, diskcache.NewWithDiskv(
		diskv.New(diskv.Options{
			BasePath:     path.Join(cacheDir, "data"),
			TempDir:      path.Join(cacheDir, "temp"),
			CacheSizeMax: uint64(cacheSizeGB) * uint64(1000000000), // convert G to B
		})),
		maxConcurrency,
	)
}

// NewMemCache creates a GitHub cache RoundTripper that is backed by a memory
// cache.
func NewMemCache(delegate http.RoundTripper, maxConcurrency int) http.RoundTripper {
	return NewFromCache(delegate, httpcache.NewMemoryCache(), maxConcurrency)
}

// NewFromCache creates a GitHub cache RoundTripper that is backed by the
// specified httpcache.Cache implementation.
func NewFromCache(delegate http.RoundTripper, cache httpcache.Cache, maxConcurrency int) http.RoundTripper {
	cacheTransport := httpcache.NewTransport(cache)
	cacheTransport.Transport = newThrottlingTransport(maxConcurrency, upstreamTransport{delegate: delegate})
	return &requestCoalescer{
		keys:     make(map[string]*responseWaiter),
		delegate: cacheTransport,
	}
}

// NewRedisCache creates a GitHub cache RoundTripper that is backed by a Redis
// cache.
func NewRedisCache(delegate http.RoundTripper, redisAddress string, maxConcurrency int) http.RoundTripper {
	conn, err := redis.Dial("tcp", redisAddress)
	if err != nil {
		logrus.WithError(err).Fatal("Error connecting to Redis")
	}
	return NewFromCache(delegate, rediscache.NewWithClient(conn), maxConcurrency)
}
<commit_msg>Ensure ghproxy not to use cache for github v4 api<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package ghcache implements an HTTP cache optimized for caching responses
// from the GitHub API (https://api.github.com).
//
// Specifically, it enforces a cache policy that revalidates every cache hit
// with a conditional request to upstream regardless of cache entry freshness
// because conditional requests for unchanged resources don't cost any API
// tokens!!! See: https://developer.github.com/v3/#conditional-requests
//
// It also provides request coalescing and prometheus instrumentation.
package ghcache

import (
	"context"
	"crypto/sha256"
	"fmt"
	"net/http"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/gomodule/redigo/redis"
	"github.com/gregjones/httpcache"
	"github.com/gregjones/httpcache/diskcache"
	rediscache "github.com/gregjones/httpcache/redis"
	"github.com/peterbourgon/diskv"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/sirupsen/logrus"
	"golang.org/x/sync/semaphore"
	"k8s.io/test-infra/ghproxy/ghmetrics"
)

type CacheResponseMode string

// Cache response modes describe how ghcache fulfilled a request.
const (
	CacheModeHeader = "X-Cache-Mode"

	ModeError   CacheResponseMode = "ERROR"    // internal error handling request
	ModeNoStore CacheResponseMode = "NO-STORE" // response not cacheable
	ModeMiss    CacheResponseMode = "MISS"     // not in cache, request proxied and response cached.
	ModeChanged CacheResponseMode = "CHANGED"  // cache value invalid: resource changed, cache updated
	// The modes below are the happy cases in which the request is fulfilled for
	// free (no API tokens used).
	ModeCoalesced   CacheResponseMode = "COALESCED"   // coalesced request, this is a copied response
	ModeRevalidated CacheResponseMode = "REVALIDATED" // cached value revalidated and returned
)

func CacheModeIsFree(mode CacheResponseMode) bool {
	switch mode {
	case ModeCoalesced:
		return true
	case ModeRevalidated:
		return true
	case ModeError:
		// In this case we did not successfully communicate with the GH API, so no
		// token is used, but we also don't return a response, so ModeError won't
		// ever be returned as a value of CacheModeHeader.
		return true
	}
	return false
}

// cacheCounter provides the 'ghcache_responses' counter vec that is indexed
// by the cache response mode.
var cacheCounter = prometheus.NewCounterVec(
	prometheus.CounterOpts{
		Name: "ghcache_responses",
		Help: "How many cache responses of each cache response mode there are.",
	},
	[]string{"mode"},
)

// outboundConcurrencyGauge provides the 'concurrent_outbound_requests' gauge that
// is global to the proxy.
var outboundConcurrencyGauge = prometheus.NewGauge(prometheus.GaugeOpts{
	Name: "concurrent_outbound_requests",
	Help: "How many concurrent requests are in flight to GitHub servers.",
})

// pendingOutboundConnectionsGauge provides the 'pending_outbound_requests' gauge that
// is global to the proxy.
var pendingOutboundConnectionsGauge = prometheus.NewGauge(prometheus.GaugeOpts{
	Name: "pending_outbound_requests",
	Help: "How many pending requests are waiting to be sent to GitHub servers.",
})

func init() {
	prometheus.MustRegister(cacheCounter)
	prometheus.MustRegister(outboundConcurrencyGauge)
	prometheus.MustRegister(pendingOutboundConnectionsGauge)
}

func cacheResponseMode(headers http.Header) CacheResponseMode {
	if strings.Contains(headers.Get("Cache-Control"), "no-store") {
		return ModeNoStore
	}
	if strings.Contains(headers.Get("Status"), "304 Not Modified") {
		return ModeRevalidated
	}
	if headers.Get("X-Conditional-Request") != "" {
		return ModeChanged
	}
	return ModeMiss
}

func newThrottlingTransport(maxConcurrency int, delegate http.RoundTripper) http.RoundTripper {
	return &throttlingTransport{sem: semaphore.NewWeighted(int64(maxConcurrency)), delegate: delegate}
}

// throttlingTransport throttles outbound concurrency from the proxy
type throttlingTransport struct {
	sem      *semaphore.Weighted
	delegate http.RoundTripper
}

func (c *throttlingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	pendingOutboundConnectionsGauge.Inc()
	if err := c.sem.Acquire(context.Background(), 1); err != nil {
		logrus.WithField("cache-key", req.URL.String()).WithError(err).Error("Internal error acquiring semaphore.")
		return nil, err
	}
	defer c.sem.Release(1)
	pendingOutboundConnectionsGauge.Dec()
	outboundConcurrencyGauge.Inc()
	defer outboundConcurrencyGauge.Dec()
	return c.delegate.RoundTrip(req)
}

// upstreamTransport changes response headers from upstream before they
// reach the cache layer in order to force the caching policy we require.
//
// By default github responds to PR requests with:
//    Cache-Control: private, max-age=60, s-maxage=60
// Which means the httpcache would not consider anything stale for 60 seconds.
// However, we want to always revalidate cache entries using ETags and last
// modified times so this RoundTripper overrides response headers to:
//    Cache-Control: no-cache
// This instructs the cache to store the response, but always consider it stale.
type upstreamTransport struct {
	delegate http.RoundTripper
}

func (u upstreamTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	etag := req.Header.Get("if-none-match")

	// get authorization header to convert to sha256
	authHeader := req.Header.Get("Authorization")
	if authHeader == "" {
		logrus.Warn("Couldn't retrieve 'Authorization' header, adding to unknown bucket")
		authHeader = "unknown"
	}
	hasher := sha256.New()
	hasher.Write([]byte(authHeader))
	authHeaderHash := fmt.Sprintf("%x", hasher.Sum(nil)) // use %x to make this a utf-8 string for use as a label

	reqStartTime := time.Now()
	// Don't modify request, just pass to delegate.
	resp, err := u.delegate.RoundTrip(req)
	if err != nil {
		logrus.WithField("cache-key", req.URL.String()).WithError(err).Error("Error from upstream (GitHub).")
		return nil, err
	}
	responseTime := time.Now()
	roundTripTime := responseTime.Sub(reqStartTime)

	if resp.StatusCode >= 400 {
		// Don't store errors. They can't be revalidated to save API tokens.
		resp.Header.Set("Cache-Control", "no-store")
	} else {
		resp.Header.Set("Cache-Control", "no-cache")
	}
	if etag != "" {
		resp.Header.Set("X-Conditional-Request", etag)
	}

	apiVersion := "v3"
	if strings.HasPrefix(req.URL.Path, "graphql") || strings.HasPrefix(req.URL.Path, "/graphql") {
		resp.Header.Set("Cache-Control", "no-store")
		apiVersion = "v4"
	}

	ghmetrics.CollectGitHubTokenMetrics(authHeaderHash, apiVersion, resp.Header, reqStartTime, responseTime)
	ghmetrics.CollectGitHubRequestMetrics(authHeaderHash, req.URL.Path, strconv.Itoa(resp.StatusCode), roundTripTime.Seconds())

	return resp, nil
}

// NewDiskCache creates a GitHub cache RoundTripper that is backed by a disk
// cache.
func NewDiskCache(delegate http.RoundTripper, cacheDir string, cacheSizeGB, maxConcurrency int) http.RoundTripper {
	return NewFromCache(delegate, diskcache.NewWithDiskv(
		diskv.New(diskv.Options{
			BasePath:     path.Join(cacheDir, "data"),
			TempDir:      path.Join(cacheDir, "temp"),
			CacheSizeMax: uint64(cacheSizeGB) * uint64(1000000000), // convert G to B
		})),
		maxConcurrency,
	)
}

// NewMemCache creates a GitHub cache RoundTripper that is backed by a memory
// cache.
func NewMemCache(delegate http.RoundTripper, maxConcurrency int) http.RoundTripper {
	return NewFromCache(delegate, httpcache.NewMemoryCache(), maxConcurrency)
}

// NewFromCache creates a GitHub cache RoundTripper that is backed by the
// specified httpcache.Cache implementation.
func NewFromCache(delegate http.RoundTripper, cache httpcache.Cache, maxConcurrency int) http.RoundTripper {
	cacheTransport := httpcache.NewTransport(cache)
	cacheTransport.Transport = newThrottlingTransport(maxConcurrency, upstreamTransport{delegate: delegate})
	return &requestCoalescer{
		keys:     make(map[string]*responseWaiter),
		delegate: cacheTransport,
	}
}

// NewRedisCache creates a GitHub cache RoundTripper that is backed by a Redis
// cache.
func NewRedisCache(delegate http.RoundTripper, redisAddress string, maxConcurrency int) http.RoundTripper {
	conn, err := redis.Dial("tcp", redisAddress)
	if err != nil {
		logrus.WithError(err).Fatal("Error connecting to Redis")
	}
	return NewFromCache(delegate, rediscache.NewWithClient(conn), maxConcurrency)
}
<|endoftext|>


<commit_before>package monitor

import (
	"fmt"
	"io"
	"regexp"
	"time"

	"github.com/cloudfoundry-community/go-cfclient"
)

type Report struct {
	SuccessCount int64
	FailureCount int64
	WarningCount int64
	ErrorCount   int64
	Errors       map[*Task]map[string]int // number of errors by error message by task
	Warnings     map[*Task]map[string]int // number of warnings by error message by task
	Elapsed      time.Duration
}

func (r *Report) String() string {
	total := r.SuccessCount + r.FailureCount + r.WarningCount
	s := "\nReport:\n"
	s += "==============\n"
	s += fmt.Sprintf("Total task executions: %d\n", total)
	s += fmt.Sprintf("Total successes:       %d\n", r.SuccessCount)
	s += fmt.Sprintf("Total failures:        %d\n", r.FailureCount)
	s += fmt.Sprintf("Total warnings:        %d\n", r.WarningCount)
	s += fmt.Sprintf("Total errors:          %d\n", r.ErrorCount)
	s += fmt.Sprintf("Elapsed time:          %s\n", r.Elapsed.String())
	s += fmt.Sprintf("Average rate:          %.2f tasks/sec\n", float64(total)/r.Elapsed.Seconds())

	if len(r.Errors) > 0 {
		s += fmt.Sprintf("\nErrors:\n")
		for task, errCounts := range r.Errors {
			s += fmt.Sprintf("\n    %s:\n", task.name)
			for err, count := range errCounts {
				s += fmt.Sprintf("        %s (%d failures)\n", err, count)
			}
		}
	}
	if len(r.Warnings) > 0 {
		s += fmt.Sprintf("\nWarnings:\n")
		for task, warningCounts := range r.Warnings {
			s += fmt.Sprintf("\n    %s:\n", task.name)
			for warning, count := range warningCounts {
				s += fmt.Sprintf("        %s (%d warnings)\n", warning, count)
			}
		}
	}
	return s
}

type TaskFunc func(cfg *cfclient.Config) error

type Task struct {
	name string
	fn   TaskFunc
}

type result struct {
	task    *Task
	err     error
	started time.Time
	ended   time.Time
}

// Monitor continuously runs each Task given to Add in parallel
// and gathers basic statistics on

type Monitor struct {
	tasks             []*Task
	taskRatePerSecond int64
	reporter          chan *Report
	queue             chan *Task
	results           chan *result
	halt              chan bool
	clientCfg         *cfclient.Config
	logger            io.Writer
	numWorkers        int
	warningMatchers   []*regexp.Regexp
}

func (m *Monitor) Add(name string, fn TaskFunc) {
	m.tasks = append(m.tasks, &Task{
		name: name,
		fn:   fn,
	})
}

func (m *Monitor) statsCollector() {
	report := &Report{
		Errors:   map[*Task]map[string]int{},
		Warnings: map[*Task]map[string]int{},
	}
	started := time.Now()
	for {
		select {
		case result := <-m.results:

			err := result.err
			if err != nil {
				msg := err.Error()

				ignored := false
				for _, warningMatcher := range m.warningMatchers {
					if warningMatcher.MatchString(msg) {
						ignored = true
						break
					}
				}
				if !ignored {
					if report.Errors[result.task] == nil {
						report.Errors[result.task] = map[string]int{}
					}
					report.Errors[result.task][msg]++
					report.ErrorCount++
					report.FailureCount++
				} else {
					if report.Warnings[result.task] == nil {
						report.Warnings[result.task] = map[string]int{}
					}
					report.Warnings[result.task][msg]++
					report.WarningCount++
				}
				fmt.Fprintf(
					m.logger,
					"%s - %s %s: %s\n",
					result.started.Format("2006-01-02 15:04:05.00 MST"),
					result.ended.Format("2006-01-02 15:04:05.00 MST"),
					result.task.name,
					msg,
				)
			} else {
				report.SuccessCount++
			}
		case <-m.halt:
			report.Elapsed = time.Since(started)
			m.reporter <- report
			return
		}
	}
}

// Run launches N worker routines to continually execute tasks and blocks until Stop is called
func (m *Monitor) Run() *Report {
	if m.halt != nil {
		panic("Run() called twice")
	}
	m.queue = make(chan *Task, m.numWorkers)
	m.halt = make(chan bool)
	for i := 0; i < m.numWorkers; i++ {
		go m.worker()
	}
	go m.statsCollector()
	go m.producer()
	return <-m.reporter
}

// worker executes a task from the queue and sends the result to the statsCollector
func (m *Monitor) worker() {
	for {
		select {
		case task := <-m.queue:
			res := &result{
				task:    task,
				started: time.Now(),
			}
			res.err = func() error {
				var cfg = *m.clientCfg
				return task.fn(&cfg)
			}()
			res.ended = time.Now()
			m.results <- res
		case <-m.halt:
			return
		}
	}
}

// producer continually fills the queue with tasks
func (m *Monitor) producer() {
	for {
		for _, task := range m.tasks {
			select {
			case <-m.halt:
				return
			case <-time.After(m.taskRateToDuration()):
				m.queue <- task
			}
		}
	}
}

func (m *Monitor) Stop() {
	select {
	case <-m.halt:
		return
	default:
		close(m.halt)
	}
}

func (m *Monitor) taskRateToDuration() time.Duration {
	return time.Second / time.Duration(m.taskRatePerSecond)
}

func NewMonitor(
	clientCfg *cfclient.Config,
	logger io.Writer,
	numWorkers int,
	warningMatchers []*regexp.Regexp,
	taskRatePerSecond int64,
) *Monitor {
	m := &Monitor{
		reporter:          make(chan *Report),
		results:           make(chan *result, numWorkers),
		clientCfg:         clientCfg,
		numWorkers:        numWorkers,
		warningMatchers:   warningMatchers,
		logger:            logger,
		taskRatePerSecond: taskRatePerSecond,
	}
	return m
}
<commit_msg>Rm dup value ErrorCount from api availability test<commit_after>package monitor

import (
	"fmt"
	"io"
	"regexp"
	"time"

	"github.com/cloudfoundry-community/go-cfclient"
)

type Report struct {
	SuccessCount int64
	FailureCount int64
	WarningCount int64
	Errors       map[*Task]map[string]int // number of errors by error message by task
	Warnings     map[*Task]map[string]int // number of warnings by error message by task
	Elapsed      time.Duration
}

func (r *Report) String() string {
	total := r.SuccessCount + r.FailureCount + r.WarningCount
	s := "\nReport:\n"
	s += "==============\n"
	s += fmt.Sprintf("Total task executions: %d\n", total)
	s += fmt.Sprintf("Total successes:       %d\n", r.SuccessCount)
	s += fmt.Sprintf("Total failures:        %d\n", r.FailureCount)
	s += fmt.Sprintf("Total warnings:        %d\n", r.WarningCount)
	s += fmt.Sprintf("Elapsed time:          %s\n", r.Elapsed.String())
	s += fmt.Sprintf("Average rate:          %.2f tasks/sec\n", float64(total)/r.Elapsed.Seconds())

	if len(r.Errors) > 0 {
		s += fmt.Sprintf("\nErrors:\n")
		for task, errCounts := range r.Errors {
			s += fmt.Sprintf("\n    %s:\n", task.name)
			for err, count := range errCounts {
				s += fmt.Sprintf("        %s (%d failures)\n", err, count)
			}
		}
	}
	if len(r.Warnings) > 0 {
		s += fmt.Sprintf("\nWarnings:\n")
		for task, warningCounts := range r.Warnings {
			s += fmt.Sprintf("\n    %s:\n", task.name)
			for warning, count := range warningCounts {
				s += fmt.Sprintf("        %s (%d warnings)\n", warning, count)
			}
		}
	}
	return s
}

type TaskFunc func(cfg *cfclient.Config) error

type Task struct {
	name string
	fn   TaskFunc
}

type result struct {
	task    *Task
	err     error
	started time.Time
	ended   time.Time
}

// Monitor continuously runs each Task given to Add in parallel
// and gathers basic statistics on

type Monitor struct {
	tasks             []*Task
	taskRatePerSecond int64
	reporter          chan *Report
	queue             chan *Task
	results           chan *result
	halt              chan bool
	clientCfg         *cfclient.Config
	logger            io.Writer
	numWorkers        int
	warningMatchers   []*regexp.Regexp
}

func (m *Monitor) Add(name string, fn TaskFunc) {
	m.tasks = append(m.tasks, &Task{
		name: name,
		fn:   fn,
	})
}

func (m *Monitor) statsCollector() {
	report := &Report{
		Errors:   map[*Task]map[string]int{},
		Warnings: map[*Task]map[string]int{},
	}
	started := time.Now()
	for {
		select {
		case result := <-m.results:

			err := result.err
			if err != nil {
				msg := err.Error()

				ignored := false
				for _, warningMatcher := range m.warningMatchers {
					if warningMatcher.MatchString(msg) {
						ignored = true
						break
					}
				}
				if !ignored {
					if report.Errors[result.task] == nil {
						report.Errors[result.task] = map[string]int{}
					}
					report.Errors[result.task][msg]++
					report.FailureCount++
				} else {
					if report.Warnings[result.task] == nil {
						report.Warnings[result.task] = map[string]int{}
					}
					report.Warnings[result.task][msg]++
					report.WarningCount++
				}
				fmt.Fprintf(
					m.logger,
					"%s - %s %s: %s\n",
					result.started.Format("2006-01-02 15:04:05.00 MST"),
					result.ended.Format("2006-01-02 15:04:05.00 MST"),
					result.task.name,
					msg,
				)
			} else {
				report.SuccessCount++
			}
		case <-m.halt:
			report.Elapsed = time.Since(started)
			m.reporter <- report
			return
		}
	}
}

// Run launches N worker routines to continually execute tasks and blocks until Stop is called
func (m *Monitor) Run() *Report {
	if m.halt != nil {
		panic("Run() called twice")
	}
	m.queue = make(chan *Task, m.numWorkers)
	m.halt = make(chan bool)
	for i := 0; i < m.numWorkers; i++ {
		go m.worker()
	}
	go m.statsCollector()
	go m.producer()
	return <-m.reporter
}

// worker executes a task from the queue and sends the result to the statsCollector
func (m *Monitor) worker() {
	for {
		select {
		case task := <-m.queue:
			res := &result{
				task:    task,
				started: time.Now(),
			}
			res.err = func() error {
				var cfg = *m.clientCfg
				return task.fn(&cfg)
			}()
			res.ended = time.Now()
			m.results <- res
		case <-m.halt:
			return
		}
	}
}

// producer continually fills the queue with tasks
func (m *Monitor) producer() {
	for {
		for _, task := range m.tasks {
			select {
			case <-m.halt:
				return
			case <-time.After(m.taskRateToDuration()):
				m.queue <- task
			}
		}
	}
}

func (m *Monitor) Stop() {
	select {
	case <-m.halt:
		return
	default:
		close(m.halt)
	}
}

func (m *Monitor) taskRateToDuration() time.Duration {
	return time.Second / time.Duration(m.taskRatePerSecond)
}

func NewMonitor(
	clientCfg *cfclient.Config,
	logger io.Writer,
	numWorkers int,
	warningMatchers []*regexp.Regexp,
	taskRatePerSecond int64,
) *Monitor {
	m := &Monitor{
		reporter:          make(chan *Report),
		results:           make(chan *result, numWorkers),
		clientCfg:         clientCfg,
		numWorkers:        numWorkers,
		warningMatchers:   warningMatchers,
		logger:            logger,
		taskRatePerSecond: taskRatePerSecond,
	}
	return m
}
<|endoftext|>


<commit_before>package paddlecloud

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"mime/multipart"
	"os"
	"path/filepath"

	pfsmod "github.com/PaddlePaddle/cloud/go/filemanager/pfsmodules"
	log "github.com/golang/glog"
)

func remoteChunkMeta(path string,
	chunkSize int64) ([]pfsmod.ChunkMeta, error) {
	cmd := pfsmod.ChunkMetaCmd{
		Method:    pfsmod.ChunkMetaCmdName,
		FilePath:  path,
		ChunkSize: chunkSize,
	}

	t := fmt.Sprintf("%s/api/v1/chunks", config.ActiveConfig.Endpoint)
	ret, err := GetCall(t, cmd.ToURLParam())
	if err != nil {
		return nil, err
	}

	type chunkMetaResponse struct {
		Err     string             `json:"err"`
		Results []pfsmod.ChunkMeta `json:"results"`
	}

	resp := chunkMetaResponse{}
	if err := json.Unmarshal(ret, &resp); err != nil {
		return nil, err
	}

	if len(resp.Err) == 0 {
		return resp.Results, nil
	}

	return resp.Results, errors.New(resp.Err)
}

func getChunkData(target string, chunk pfsmod.Chunk, dst string) error {
	log.V(1).Info("target url: " + target)

	resp, err := GetChunk(target, chunk.ToURLParam())
	if err != nil {
		return err
	}
	defer pfsmod.Close(resp.Body)

	if resp.Status != HTTPOK {
		return errors.New("http server returned non-200 status: " + resp.Status)
	}

	partReader := multipart.NewReader(resp.Body, pfsmod.DefaultMultiPartBoundary)
	for {
		part, error := partReader.NextPart()
		if error == io.EOF {
			break
		}

		if part.FormName() == "chunk" {
			recvCmd, err := pfsmod.ParseChunk(part.FileName())
			if err != nil {
				return errors.New(err.Error())
			}

			recvCmd.Path = dst

			if err := recvCmd.SaveChunkData(part); err != nil {
				return err
			}
		}
	}

	return nil
}

func downloadChunks(src string,
	dst string, diffMeta []pfsmod.ChunkMeta) error {
	if len(diffMeta) == 0 {
		log.V(1).Infof("srcfile:%s and dstfile:%s are already same\n", src, dst)
		fmt.Printf("download ok\n")
		return nil
	}

	t := fmt.Sprintf("%s/api/v1/storage/chunks", config.ActiveConfig.Endpoint)
	for _, meta := range diffMeta {
		chunk := pfsmod.Chunk{
			Path:   src,
			Offset: meta.Offset,
			Size:   meta.Len,
		}

		err := getChunkData(t, chunk, dst)
		if err != nil {
			return err
		}
	}

	return nil
}

func downloadFile(src string, srcFileSize int64, dst string) error {
	srcMeta, err := remoteChunkMeta(src, defaultChunkSize)
	if err != nil {
		return err
	}
	log.V(4).Infof("srcMeta:%#v\n\n", srcMeta)

	dstMeta, err := pfsmod.GetChunkMeta(dst, defaultChunkSize)
	if err != nil {
		if os.IsNotExist(err) {
			if err := pfsmod.CreateSizedFile(dst, srcFileSize); err != nil {
				return err
			}
		} else {
			return err
		}
	}
	log.V(4).Infof("dstMeta:%#v\n", dstMeta)

	diffMeta, err := pfsmod.GetDiffChunkMeta(srcMeta, dstMeta)
	if err != nil {
		return err
	}

	err = downloadChunks(src, dst, diffMeta)
	return err
}

func checkBeforeDownLoad(src []pfsmod.LsResult, dst string) (bool, error) {
	var bDir bool
	fi, err := os.Stat(dst)
	if err == nil {
		bDir = fi.IsDir()
		if !fi.IsDir() && len(src) > 1 {
			return bDir, errors.New(pfsmod.StatusDestShouldBeDirectory)
		}
	} else if os.IsNotExist(err) {
		return false, nil
	}

	return bDir, err
}

// Download function downloads src to dst.
func download(src, dst string) error {
	log.V(1).Infof("download %s to %s\n", src, dst)
	lsRet, err := RemoteLs(pfsmod.NewLsCmd(true, src))
	if err != nil {
		return err
	}

	bDir, err := checkBeforeDownLoad(lsRet, dst)
	if err != nil {
		return err
	}

	for _, attr := range lsRet {
		if attr.IsDir {
			return errors.New(pfsmod.StatusOnlySupportFiles)
		}

		realSrc := attr.Path
		realDst := dst

		if bDir {
			_, file := filepath.Split(attr.Path)
			realDst = dst + "/" + file
		}

		fmt.Printf("download src_path:%s dst_path:%s\n", realSrc, realDst)
		if err := downloadFile(realSrc, attr.Size, realDst); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>rm not need code<commit_after>package paddlecloud

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"mime/multipart"
	"os"
	"path/filepath"

	pfsmod "github.com/PaddlePaddle/cloud/go/filemanager/pfsmodules"
	log "github.com/golang/glog"
)

func remoteChunkMeta(path string,
	chunkSize int64) ([]pfsmod.ChunkMeta, error) {
	cmd := pfsmod.ChunkMetaCmd{
		Method:    pfsmod.ChunkMetaCmdName,
		FilePath:  path,
		ChunkSize: chunkSize,
	}

	t := fmt.Sprintf("%s/api/v1/chunks", config.ActiveConfig.Endpoint)
	ret, err := GetCall(t, cmd.ToURLParam())
	if err != nil {
		return nil, err
	}

	type chunkMetaResponse struct {
		Err     string             `json:"err"`
		Results []pfsmod.ChunkMeta `json:"results"`
	}

	resp := chunkMetaResponse{}
	if err := json.Unmarshal(ret, &resp); err != nil {
		return nil, err
	}

	if len(resp.Err) == 0 {
		return resp.Results, nil
	}

	return resp.Results, errors.New(resp.Err)
}

func getChunkData(target string, chunk pfsmod.Chunk, dst string) error {
	log.V(1).Info("target url: " + target)

	resp, err := GetChunk(target, chunk.ToURLParam())
	if err != nil {
		return err
	}
	defer pfsmod.Close(resp.Body)

	if resp.Status != HTTPOK {
		return errors.New("http server returned non-200 status: " + resp.Status)
	}

	partReader := multipart.NewReader(resp.Body, pfsmod.DefaultMultiPartBoundary)
	for {
		part, error := partReader.NextPart()
		if error == io.EOF {
			break
		}

		if part.FormName() == "chunk" {
			recvCmd, err := pfsmod.ParseChunk(part.FileName())
			if err != nil {
				return errors.New(err.Error())
			}

			recvCmd.Path = dst

			if err := recvCmd.SaveChunkData(part); err != nil {
				return err
			}
		}
	}

	return nil
}

func downloadChunks(src string,
	dst string, diffMeta []pfsmod.ChunkMeta) error {
	if len(diffMeta) == 0 {
		log.V(1).Infof("srcfile:%s and dstfile:%s are already same\n", src, dst)
		fmt.Printf("download ok\n")
		return nil
	}

	t := fmt.Sprintf("%s/api/v1/storage/chunks", config.ActiveConfig.Endpoint)
	for _, meta := range diffMeta {
		chunk := pfsmod.Chunk{
			Path:   src,
			Offset: meta.Offset,
			Size:   meta.Len,
		}

		err := getChunkData(t, chunk, dst)
		if err != nil {
			return err
		}
	}

	return nil
}

func downloadFile(src string, srcFileSize int64, dst string) error {
	srcMeta, err := remoteChunkMeta(src, defaultChunkSize)
	if err != nil {
		return err
	}
	log.V(4).Infof("srcMeta:%#v\n\n", srcMeta)

	dstMeta, err := pfsmod.GetChunkMeta(dst, defaultChunkSize)
	if err != nil {
		if os.IsNotExist(err) {
			if err := pfsmod.CreateSizedFile(dst, srcFileSize); err != nil {
				return err
			}
		} else {
			return err
		}
	}
	log.V(4).Infof("dstMeta:%#v\n", dstMeta)

	diffMeta, err := pfsmod.GetDiffChunkMeta(srcMeta, dstMeta)
	if err != nil {
		return err
	}

	err = downloadChunks(src, dst, diffMeta)
	return err
}

func checkBeforeDownLoad(src []pfsmod.LsResult, dst string) (bool, error) {
	var bDir bool
	fi, err := os.Stat(dst)
	if err == nil {
		bDir = fi.IsDir()
		if !fi.IsDir() && len(src) > 1 {
			return bDir, errors.New(pfsmod.StatusDestShouldBeDirectory)
		}
	} else if os.IsNotExist(err) {
		return false, nil
	}

	return bDir, err
}

func download(src, dst string) error {
	log.V(1).Infof("download %s to %s\n", src, dst)
	lsRet, err := RemoteLs(pfsmod.NewLsCmd(true, src))
	if err != nil {
		return err
	}

	bDir, err := checkBeforeDownLoad(lsRet, dst)
	if err != nil {
		return err
	}

	for _, attr := range lsRet {
		if attr.IsDir {
			return errors.New(pfsmod.StatusOnlySupportFiles)
		}

		realSrc := attr.Path
		realDst := dst

		if bDir {
			_, file := filepath.Split(attr.Path)
			realDst = dst + "/" + file
		}

		fmt.Printf("download src_path:%s dst_path:%s\n", realSrc, realDst)
		if err := downloadFile(realSrc, attr.Size, realDst); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>