<commit_before>package v1batch

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"path"
	"sync"

	"github.com/nerdalize/nerd/nerd/client"
	v1payload "github.com/nerdalize/nerd/nerd/client/batch/v1/payload"
)

const (
	//AuthHeader is the name of the HTTP Authorization header.
	AuthHeader = "Authorization"

	projectsPrefix = "projects"

	tasksEndpoint      = "tasks"
	tokensEndpoint     = "tokens"
	datasetEndpoint    = "datasets"
	workloadsEndpoint  = "workload"
	placementsEndpoint = "placements"
)

//Client is a client for the Nerdalize API.
type Client struct {
	ClientConfig
	cred string
	m    sync.Mutex
}

//ClientConfig provides config details to create a Nerd client.
type ClientConfig struct {
	Doer        Doer
	JWTProvider JWTProvider
	Base        *url.URL
	Logger      client.Logger
}

//ClientInterface is an interface so client calls can be mocked.
type ClientInterface interface {
	ClientDatasetInterface
	ClientPingInterface
	ClientWorkloadInterface
	ClientTaskInterface
	ClientRunInterface
	ClientTokenInterface
	ClientUploadInterface
	ClientPlacementInterface
}

// Force compile errors when Client doesn't implement ClientInterface.
var _ ClientInterface = &Client{}

// Doer executes http requests.  It is implemented by *http.Client.
type Doer interface {
	Do(req *http.Request) (*http.Response, error)
}

//NewClient creates a new Nerd client from a config object. The http.DefaultClient
//will be used as default Doer.
func NewClient(conf ClientConfig) *Client {
	if conf.Doer == nil {
		conf.Doer = http.DefaultClient
	}
	if conf.Base.Path != "" && conf.Base.Path[len(conf.Base.Path)-1] != '/' {
		conf.Base.Path = conf.Base.Path + "/"
	}
	cl := &Client{
		ClientConfig: conf,
		cred:         "",
		m:            sync.Mutex{},
	}
	return cl
}

//getJWT atomically gets the JWT from the JWT provider.
func (c *Client) getJWT() (string, error) {
	c.m.Lock()
	defer c.m.Unlock()
	if c.JWTProvider == nil {
		return "", fmt.Errorf("No JWT provider found")
	}
	if c.cred == "" || c.JWTProvider.IsExpired() {
		cred, err := c.JWTProvider.Retrieve()
		if err != nil {
			return "", err
		}
		c.cred = cred
	}
	return c.cred, nil
}

//doRequest requests the server and decodes the output into the `output` field.
//
//doRequest will set the Authentication header with the JWT provided by the JWTProvider.
//When a status code >= 400 is returned by the server the returned error will be of type HTTPError.
func (c *Client) doRequest(method, urlPath string, input, output interface{}) (err error) {
	cred, err := c.getJWT()
	if err != nil {
		return err
	}

	path, err := url.Parse(urlPath)
	if err != nil {
		return client.NewError("invalid url path provided", err)
	}

	resolved := c.Base.ResolveReference(path)

	var req *http.Request
	if input != nil && method != http.MethodGet {
		buf := bytes.NewBuffer(nil)
		enc := json.NewEncoder(buf)
		err = enc.Encode(input)
		if err != nil {
			return client.NewError("failed to encode the request body", err)
		}
		req, err = http.NewRequest(method, resolved.String(), buf)
		if err != nil {
			return client.NewError("failed to create HTTP request", err)
		}
	} else {
		req, err = http.NewRequest(method, resolved.String(), nil)
		if err != nil {
			return client.NewError("failed to create HTTP request", err)
		}
	}

	req.Header.Set(AuthHeader, "Bearer "+cred)
	client.LogRequest(req, c.Logger)
	resp, err := c.Doer.Do(req)
	if err != nil {
		return client.NewError("failed to create HTTP request", err)
	}
	client.LogResponse(resp, c.Logger)

	dec := json.NewDecoder(resp.Body)
	defer resp.Body.Close()
	if resp.StatusCode > 399 {
		errv := &v1payload.Error{}
		err = dec.Decode(errv)
		if err != nil {
			return client.NewError(fmt.Sprintf("failed to decode unexpected HTTP response (%s)", resp.Status), err)
		}

		return &HTTPError{
			StatusCode: resp.StatusCode,
			Err:        errv,
		}
	}

	if output != nil {
		err = dec.Decode(output)
		if err != nil {
			return client.NewError(fmt.Sprintf("failed to decode successfull HTTP response (%s)", resp.Status), err)
		}
	}

	return nil
}

//createPath is a convienent wrapper for creating a resource path prefixed by the project namespace.
func createPath(projectID string, elem ...string) string {
	return path.Join(projectsPrefix, projectID, path.Join(elem...))
}
<commit_msg>Fix typo<commit_after>package v1batch

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"path"
	"sync"

	"github.com/nerdalize/nerd/nerd/client"
	v1payload "github.com/nerdalize/nerd/nerd/client/batch/v1/payload"
)

const (
	//AuthHeader is the name of the HTTP Authorization header.
	AuthHeader = "Authorization"

	projectsPrefix = "projects"

	tasksEndpoint      = "tasks"
	tokensEndpoint     = "tokens"
	datasetEndpoint    = "datasets"
	workloadsEndpoint  = "workloads"
	placementsEndpoint = "placements"
)

//Client is a client for the Nerdalize API.
type Client struct {
	ClientConfig
	cred string
	m    sync.Mutex
}

//ClientConfig provides config details to create a Nerd client.
type ClientConfig struct {
	Doer        Doer
	JWTProvider JWTProvider
	Base        *url.URL
	Logger      client.Logger
}

//ClientInterface is an interface so client calls can be mocked.
type ClientInterface interface {
	ClientDatasetInterface
	ClientPingInterface
	ClientWorkloadInterface
	ClientTaskInterface
	ClientRunInterface
	ClientTokenInterface
	ClientUploadInterface
	ClientPlacementInterface
}

// Force compile errors when Client doesn't implement ClientInterface.
var _ ClientInterface = &Client{}

// Doer executes http requests.  It is implemented by *http.Client.
type Doer interface {
	Do(req *http.Request) (*http.Response, error)
}

//NewClient creates a new Nerd client from a config object. The http.DefaultClient
//will be used as default Doer.
func NewClient(conf ClientConfig) *Client {
	if conf.Doer == nil {
		conf.Doer = http.DefaultClient
	}
	if conf.Base.Path != "" && conf.Base.Path[len(conf.Base.Path)-1] != '/' {
		conf.Base.Path = conf.Base.Path + "/"
	}
	cl := &Client{
		ClientConfig: conf,
		cred:         "",
		m:            sync.Mutex{},
	}
	return cl
}

//getJWT atomically gets the JWT from the JWT provider.
func (c *Client) getJWT() (string, error) {
	c.m.Lock()
	defer c.m.Unlock()
	if c.JWTProvider == nil {
		return "", fmt.Errorf("No JWT provider found")
	}
	if c.cred == "" || c.JWTProvider.IsExpired() {
		cred, err := c.JWTProvider.Retrieve()
		if err != nil {
			return "", err
		}
		c.cred = cred
	}
	return c.cred, nil
}

//doRequest requests the server and decodes the output into the `output` field.
//
//doRequest will set the Authentication header with the JWT provided by the JWTProvider.
//When a status code >= 400 is returned by the server the returned error will be of type HTTPError.
func (c *Client) doRequest(method, urlPath string, input, output interface{}) (err error) {
	cred, err := c.getJWT()
	if err != nil {
		return err
	}

	path, err := url.Parse(urlPath)
	if err != nil {
		return client.NewError("invalid url path provided", err)
	}

	resolved := c.Base.ResolveReference(path)

	var req *http.Request
	if input != nil && method != http.MethodGet {
		buf := bytes.NewBuffer(nil)
		enc := json.NewEncoder(buf)
		err = enc.Encode(input)
		if err != nil {
			return client.NewError("failed to encode the request body", err)
		}
		req, err = http.NewRequest(method, resolved.String(), buf)
		if err != nil {
			return client.NewError("failed to create HTTP request", err)
		}
	} else {
		req, err = http.NewRequest(method, resolved.String(), nil)
		if err != nil {
			return client.NewError("failed to create HTTP request", err)
		}
	}

	req.Header.Set(AuthHeader, "Bearer "+cred)
	client.LogRequest(req, c.Logger)
	resp, err := c.Doer.Do(req)
	if err != nil {
		return client.NewError("failed to create HTTP request", err)
	}
	client.LogResponse(resp, c.Logger)

	dec := json.NewDecoder(resp.Body)
	defer resp.Body.Close()
	if resp.StatusCode > 399 {
		errv := &v1payload.Error{}
		err = dec.Decode(errv)
		if err != nil {
			return client.NewError(fmt.Sprintf("failed to decode unexpected HTTP response (%s)", resp.Status), err)
		}

		return &HTTPError{
			StatusCode: resp.StatusCode,
			Err:        errv,
		}
	}

	if output != nil {
		err = dec.Decode(output)
		if err != nil {
			return client.NewError(fmt.Sprintf("failed to decode successfull HTTP response (%s)", resp.Status), err)
		}
	}

	return nil
}

//createPath is a convienent wrapper for creating a resource path prefixed by the project namespace.
func createPath(projectID string, elem ...string) string {
	return path.Join(projectsPrefix, projectID, path.Join(elem...))
}
<|endoftext|>


<commit_before>package wavreader

import "testing"

func TestAtoI(t *testing.T) {
	b := []byte{1, 0, 0, 128}
	tatoi(t, b[2:4], 1<<15)
	tatoi(t, b[1:4], 1<<23)
	tatoi(t, b, 1<<31+1)
}

func tatoi(t *testing.T, buf []byte, exp int) {
	i := atoi(buf)
	if i != exp {
		t.Errorf("atoi failed for %02x: expected %d (%02x); got %d (%02x)", buf, exp, exp, i, i)
	}
}

func TestAtoSI(t *testing.T) {
	b := []byte{0, 0, 0, 128}
	tatosi(t, b[2:4], -1*1<<15)
	tatosi(t, b[1:4], -1*1<<23)
}

func tatosi(t *testing.T, buf []byte, exp int) {
	i := atosi(buf)
	if i != exp {
		t.Errorf("atosi failed for %02x: expected %d (%02x); got %d (%02x)", buf, exp, exp, i, i)
	}
}

func TestItoa8bit(t *testing.T) {
	titoa8(t, 0, 0)
	titoa8(t, 6, 6)
	titoa8(t, 200, 200)
	titoa8(t, -128, 0)
	titoa8(t, 3000, 255)
}

func titoa8(t *testing.T, v int, exp byte) {
	b := []byte{3}
	itoa(b, v)
	if b[0] != exp {
		t.Errorf("itoa(8) failed, expected %d (%02x), got %d (%02x)", exp, exp, b[0], b[0])
	}
}
<commit_msg>Make the tests work on 32-bit platforms<commit_after>package wavreader

import "testing"

func TestAtoI(t *testing.T) {
	b := []byte{1, 0, 0, 64}
	tatoi(t, b[2:4], 1<<14)
	tatoi(t, b[1:4], 1<<22)
	tatoi(t, b, 1<<30+1)
}

func tatoi(t *testing.T, buf []byte, exp int) {
	i := atoi(buf)
	if i != exp {
		t.Errorf("atoi failed for %02x: expected %d (%02x); got %d (%02x)", buf, exp, exp, i, i)
	}
}

func TestAtoSI(t *testing.T) {
	b := []byte{0, 0, 0, 128}
	tatosi(t, b[2:4], -1*1<<15)
	tatosi(t, b[1:4], -1*1<<23)
}

func tatosi(t *testing.T, buf []byte, exp int) {
	i := atosi(buf)
	if i != exp {
		t.Errorf("atosi failed for %02x: expected %d (%02x); got %d (%02x)", buf, exp, exp, i, i)
	}
}

func TestItoa8bit(t *testing.T) {
	titoa8(t, 0, 0)
	titoa8(t, 6, 6)
	titoa8(t, 200, 200)
	titoa8(t, -128, 0)
	titoa8(t, 3000, 255)
}

func titoa8(t *testing.T, v int, exp byte) {
	b := []byte{3}
	itoa(b, v)
	if b[0] != exp {
		t.Errorf("itoa(8) failed, expected %d (%02x), got %d (%02x)", exp, exp, b[0], b[0])
	}
}
<|endoftext|>


<commit_before>package rest

import (
	"code.google.com/p/goprotobuf/proto"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/golang/glog"
	"github.com/gorilla/mux"
	"github.com/qorio/api"
	"github.com/qorio/omni/auth"
	omni_http "github.com/qorio/omni/http"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
)

var (
	ERROR_MISSING_INPUT        = errors.New("error-missing-input")
	ERROR_UNKNOWN_CONTENT_TYPE = errors.New("error-no-content-type")
	ERROR_UNKNOWN_METHOD       = errors.New("error-unknown-method")
)

var (
	json_marshaler = func(contentType string, resp http.ResponseWriter, typed proto.Message) error {
		if buff, err := json.Marshal(typed); err == nil {
			omni_http.SetCORSHeaders(resp)
			resp.Header().Add("Content-Type", contentType)
			resp.Write(buff)
			return nil
		} else {
			return err
		}
	}

	json_unmarshaler = func(body io.ReadCloser, typed proto.Message) error {
		dec := json.NewDecoder(body)
		return dec.Decode(typed)
	}

	proto_marshaler = func(contentType string, resp http.ResponseWriter, typed proto.Message) error {
		if buff, err := proto.Marshal(typed); err == nil {
			omni_http.SetCORSHeaders(resp)
			resp.Header().Add("Content-Type", contentType)
			resp.Write(buff)
			return nil
		} else {
			return err
		}
	}

	proto_unmarshaler = func(body io.ReadCloser, typed proto.Message) error {
		buff, err := ioutil.ReadAll(body)
		if err != nil {
			return err
		}
		return proto.Unmarshal(buff, typed)
	}

	marshalers = map[string]func(string, http.ResponseWriter, proto.Message) error{
		"":                     json_marshaler,
		"application/json":     json_marshaler,
		"application/protobuf": proto_marshaler,
	}

	unmarshalers = map[string]func(io.ReadCloser, proto.Message) error{
		"":                     json_unmarshaler,
		"application/json":     json_unmarshaler,
		"application/protobuf": proto_unmarshaler,
	}
)

type Handler func(http.ResponseWriter, *http.Request)

type ServiceMethodImpl struct {
	Api                  api.MethodSpec // note this is by copy -- so that behavior is deterministic after initialization
	Handler              Handler
	AuthenticatedHandler auth.HttpHandler
	ServiceId            string
}

func SetHandler(m api.MethodSpec, h Handler) *ServiceMethodImpl {
	if m.AuthScope != "" {
		panic(errors.New(fmt.Sprintf("Method %s has oauth scopes but binding to unauthed handler.", m)))
	}
	return &ServiceMethodImpl{
		Api:     m,
		Handler: h,
	}
}

func SetAuthenticatedHandler(serviceId string, m api.MethodSpec, h auth.HttpHandler) *ServiceMethodImpl {
	if m.AuthScope == "" {
		panic(errors.New(fmt.Sprintf("Method %s has no oauth scopes but binding to authenticated handler.", m)))
	}
	return &ServiceMethodImpl{
		Api:                  m,
		AuthenticatedHandler: h,
		ServiceId:            serviceId,
	}
}

type EngineEvent struct {
	Service       string
	ServiceMethod api.ServiceMethod
	Body          interface{}
}

type Engine interface {
	Bind(...*ServiceMethodImpl)
	ServeHTTP(http.ResponseWriter, *http.Request)
	NewAuthToken() *auth.Token
	SignedString(*auth.Token) (string, error)
	GetUrlParameter(*http.Request, string) string
	Unmarshal(*http.Request, proto.Message) error
	Marshal(*http.Request, proto.Message, http.ResponseWriter) error
	MarshalJSON(*http.Request, interface{}, http.ResponseWriter) error
	HandleError(http.ResponseWriter, *http.Request, string, int) error
	EventChannel() chan<- *EngineEvent
}

type engine struct {
	spec       *api.ServiceMethods
	router     *mux.Router
	auth       auth.Service
	event_chan chan *EngineEvent
	done_chan  chan bool
	webhooks   WebHooksService
}

func NewEngine(spec *api.ServiceMethods, auth auth.Service, webhooks WebHooksService) *engine {
	e := &engine{
		spec:       spec,
		router:     mux.NewRouter(),
		auth:       auth,
		event_chan: make(chan *EngineEvent),
		done_chan:  make(chan bool),
		webhooks:   webhooks,
	}
	return e
}

func (this *engine) Router() *mux.Router {
	return this.router
}

func (this *engine) NewAuthToken() *auth.Token {
	return this.auth.NewToken()
}

func (this *engine) SignedString(token *auth.Token) (string, error) {
	return this.auth.SignedString(token)
}

func (this *engine) ServeHTTP(resp http.ResponseWriter, request *http.Request) {
	// Also start listening on the event channel for any webhook calls
	go func() {
		for {
			select {

			case message := <-this.event_chan:
				this.do_callback(message)

			case done := <-this.done_chan:
				if done {
					glog.Infoln("REST engine event channel stopped.")
					return
				}
			}
		}
	}()

	this.router.ServeHTTP(resp, request)
	glog.Infoln("Stopping event channel")
	this.done_chan <- true
}

func (this *engine) GetUrlParameter(req *http.Request, key string) string {
	vars := mux.Vars(req)
	if val, has := vars[key]; has {
		return val
	} else if err := req.ParseForm(); err == nil {
		if _, has := req.Form[key]; has {
			return req.Form[key][0]
		}
	}
	return ""
}

func (this *engine) Bind(endpoints ...*ServiceMethodImpl) {
	for i, ep := range endpoints {
		switch {
		case ep.Handler != nil:
			this.router.HandleFunc(ep.Api.UrlRoute, ep.Handler).Methods(ep.Api.HttpMethod)

		case ep.AuthenticatedHandler != nil:
			this.router.HandleFunc(ep.Api.UrlRoute,
				this.auth.RequiresAuth(ep.Api.AuthScope, func(token *auth.Token) []string {
					return strings.Split(token.GetString(ep.ServiceId+"/@scopes"), ",")
				}, ep.AuthenticatedHandler)).Methods(ep.Api.HttpMethod)

		case ep.Handler == nil && ep.AuthenticatedHandler == nil:
			panic(errors.New(fmt.Sprintf("No implementation for REST endpoint[%d]: %s", i, ep)))
		}

		// check the content type
		for _, ct := range ep.Api.ContentTypes {
			if _, has := marshalers[ct]; !has {
				panic(errors.New(fmt.Sprintf("Bad content type: %s", ct)))
			}
			if _, has := unmarshalers[ct]; !has {
				panic(errors.New(fmt.Sprintf("Bad content type: %s", ct)))
			}
		}
	}
}

func (this *engine) Unmarshal(req *http.Request, typed proto.Message) (err error) {
	contentType := req.Header.Get("Content-Type")
	if unmarshaler, has := unmarshalers[contentType]; has {
		return unmarshaler(req.Body, typed)
	} else {
		return ERROR_UNKNOWN_CONTENT_TYPE
	}
}

func (this *engine) MarshalJSON(req *http.Request, any interface{}, resp http.ResponseWriter) (err error) {
	if buff, err := json.Marshal(any); err == nil {
		omni_http.SetCORSHeaders(resp)
		resp.Header().Add("Content-Type", "application/json")
		resp.Write(buff)
		return nil
	} else {
		return err
	}
}

func (this *engine) Marshal(req *http.Request, typed proto.Message, resp http.ResponseWriter) (err error) {
	contentType := req.Header.Get("Content-Type") // Usually the case when a client POSTs
	if contentType == "" {
		contentType = req.Header.Get("Accept") // Usually the case with GET
	}
	if marshaler, has := marshalers[contentType]; has {
		return marshaler(contentType, resp, typed)
	} else {
		return ERROR_UNKNOWN_CONTENT_TYPE
	}
}

func (this *engine) HandleError(resp http.ResponseWriter, req *http.Request, message string, code int) (err error) {
	resp.WriteHeader(code)
	resp.Write([]byte(fmt.Sprintf("{\"error\":\"%s\"}", message)))
	return
}

func (this *engine) EventChannel() chan<- *EngineEvent {
	return this.event_chan
}

func (this *engine) do_callback(message *EngineEvent) error {
	if this.webhooks == nil {
		return nil
	}
	//methods := *this.spec
	if m, has := (*this.spec)[message.ServiceMethod]; has {
		if m.CallbackEvent != api.EventKey("") {
			return this.webhooks.Send(message.Service, string(m.CallbackEvent), message.Body, m.CallbackBodyTemplate)
		}
	}
	return ERROR_UNKNOWN_METHOD
}
<commit_msg>Removed unnecessary comment and sending to channel during http serve<commit_after>package rest

import (
	"code.google.com/p/goprotobuf/proto"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/golang/glog"
	"github.com/gorilla/mux"
	"github.com/qorio/api"
	"github.com/qorio/omni/auth"
	omni_http "github.com/qorio/omni/http"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
)

var (
	ERROR_MISSING_INPUT        = errors.New("error-missing-input")
	ERROR_UNKNOWN_CONTENT_TYPE = errors.New("error-no-content-type")
	ERROR_UNKNOWN_METHOD       = errors.New("error-unknown-method")
)

var (
	json_marshaler = func(contentType string, resp http.ResponseWriter, typed proto.Message) error {
		if buff, err := json.Marshal(typed); err == nil {
			omni_http.SetCORSHeaders(resp)
			resp.Header().Add("Content-Type", contentType)
			resp.Write(buff)
			return nil
		} else {
			return err
		}
	}

	json_unmarshaler = func(body io.ReadCloser, typed proto.Message) error {
		dec := json.NewDecoder(body)
		return dec.Decode(typed)
	}

	proto_marshaler = func(contentType string, resp http.ResponseWriter, typed proto.Message) error {
		if buff, err := proto.Marshal(typed); err == nil {
			omni_http.SetCORSHeaders(resp)
			resp.Header().Add("Content-Type", contentType)
			resp.Write(buff)
			return nil
		} else {
			return err
		}
	}

	proto_unmarshaler = func(body io.ReadCloser, typed proto.Message) error {
		buff, err := ioutil.ReadAll(body)
		if err != nil {
			return err
		}
		return proto.Unmarshal(buff, typed)
	}

	marshalers = map[string]func(string, http.ResponseWriter, proto.Message) error{
		"":                     json_marshaler,
		"application/json":     json_marshaler,
		"application/protobuf": proto_marshaler,
	}

	unmarshalers = map[string]func(io.ReadCloser, proto.Message) error{
		"":                     json_unmarshaler,
		"application/json":     json_unmarshaler,
		"application/protobuf": proto_unmarshaler,
	}
)

type Handler func(http.ResponseWriter, *http.Request)

type ServiceMethodImpl struct {
	Api                  api.MethodSpec // note this is by copy -- so that behavior is deterministic after initialization
	Handler              Handler
	AuthenticatedHandler auth.HttpHandler
	ServiceId            string
}

func SetHandler(m api.MethodSpec, h Handler) *ServiceMethodImpl {
	if m.AuthScope != "" {
		panic(errors.New(fmt.Sprintf("Method %s has oauth scopes but binding to unauthed handler.", m)))
	}
	return &ServiceMethodImpl{
		Api:     m,
		Handler: h,
	}
}

func SetAuthenticatedHandler(serviceId string, m api.MethodSpec, h auth.HttpHandler) *ServiceMethodImpl {
	if m.AuthScope == "" {
		panic(errors.New(fmt.Sprintf("Method %s has no oauth scopes but binding to authenticated handler.", m)))
	}
	return &ServiceMethodImpl{
		Api:                  m,
		AuthenticatedHandler: h,
		ServiceId:            serviceId,
	}
}

type EngineEvent struct {
	Service       string
	ServiceMethod api.ServiceMethod
	Body          interface{}
}

type Engine interface {
	Bind(...*ServiceMethodImpl)
	ServeHTTP(http.ResponseWriter, *http.Request)
	NewAuthToken() *auth.Token
	SignedString(*auth.Token) (string, error)
	GetUrlParameter(*http.Request, string) string
	Unmarshal(*http.Request, proto.Message) error
	Marshal(*http.Request, proto.Message, http.ResponseWriter) error
	MarshalJSON(*http.Request, interface{}, http.ResponseWriter) error
	HandleError(http.ResponseWriter, *http.Request, string, int) error
	EventChannel() chan<- *EngineEvent
}

type engine struct {
	spec       *api.ServiceMethods
	router     *mux.Router
	auth       auth.Service
	event_chan chan *EngineEvent
	done_chan  chan bool
	webhooks   WebHooksService
}

func NewEngine(spec *api.ServiceMethods, auth auth.Service, webhooks WebHooksService) *engine {
	e := &engine{
		spec:       spec,
		router:     mux.NewRouter(),
		auth:       auth,
		event_chan: make(chan *EngineEvent),
		done_chan:  make(chan bool),
		webhooks:   webhooks,
	}
	return e
}

func (this *engine) Router() *mux.Router {
	return this.router
}

func (this *engine) NewAuthToken() *auth.Token {
	return this.auth.NewToken()
}

func (this *engine) SignedString(token *auth.Token) (string, error) {
	return this.auth.SignedString(token)
}

func (this *engine) ServeHTTP(resp http.ResponseWriter, request *http.Request) {
	// Also start listening on the event channel for any webhook calls
	go func() {
		for {
			select {

			case message := <-this.event_chan:
				this.do_callback(message)

			case done := <-this.done_chan:
				if done {
					glog.Infoln("REST engine event channel stopped.")
					return
				}
			}
		}
	}()

	this.router.ServeHTTP(resp, request)
}

func (this *engine) GetUrlParameter(req *http.Request, key string) string {
	vars := mux.Vars(req)
	if val, has := vars[key]; has {
		return val
	} else if err := req.ParseForm(); err == nil {
		if _, has := req.Form[key]; has {
			return req.Form[key][0]
		}
	}
	return ""
}

func (this *engine) Bind(endpoints ...*ServiceMethodImpl) {
	for i, ep := range endpoints {
		switch {
		case ep.Handler != nil:
			this.router.HandleFunc(ep.Api.UrlRoute, ep.Handler).Methods(ep.Api.HttpMethod)

		case ep.AuthenticatedHandler != nil:
			this.router.HandleFunc(ep.Api.UrlRoute,
				this.auth.RequiresAuth(ep.Api.AuthScope, func(token *auth.Token) []string {
					return strings.Split(token.GetString(ep.ServiceId+"/@scopes"), ",")
				}, ep.AuthenticatedHandler)).Methods(ep.Api.HttpMethod)

		case ep.Handler == nil && ep.AuthenticatedHandler == nil:
			panic(errors.New(fmt.Sprintf("No implementation for REST endpoint[%d]: %s", i, ep)))
		}

		// check the content type
		for _, ct := range ep.Api.ContentTypes {
			if _, has := marshalers[ct]; !has {
				panic(errors.New(fmt.Sprintf("Bad content type: %s", ct)))
			}
			if _, has := unmarshalers[ct]; !has {
				panic(errors.New(fmt.Sprintf("Bad content type: %s", ct)))
			}
		}
	}
}

func (this *engine) Unmarshal(req *http.Request, typed proto.Message) (err error) {
	contentType := req.Header.Get("Content-Type")
	if unmarshaler, has := unmarshalers[contentType]; has {
		return unmarshaler(req.Body, typed)
	} else {
		return ERROR_UNKNOWN_CONTENT_TYPE
	}
}

func (this *engine) MarshalJSON(req *http.Request, any interface{}, resp http.ResponseWriter) (err error) {
	if buff, err := json.Marshal(any); err == nil {
		omni_http.SetCORSHeaders(resp)
		resp.Header().Add("Content-Type", "application/json")
		resp.Write(buff)
		return nil
	} else {
		return err
	}
}

func (this *engine) Marshal(req *http.Request, typed proto.Message, resp http.ResponseWriter) (err error) {
	contentType := req.Header.Get("Content-Type") // Usually the case when a client POSTs
	if contentType == "" {
		contentType = req.Header.Get("Accept") // Usually the case with GET
	}
	if marshaler, has := marshalers[contentType]; has {
		return marshaler(contentType, resp, typed)
	} else {
		return ERROR_UNKNOWN_CONTENT_TYPE
	}
}

func (this *engine) HandleError(resp http.ResponseWriter, req *http.Request, message string, code int) (err error) {
	resp.WriteHeader(code)
	resp.Write([]byte(fmt.Sprintf("{\"error\":\"%s\"}", message)))
	return
}

func (this *engine) EventChannel() chan<- *EngineEvent {
	return this.event_chan
}

func (this *engine) do_callback(message *EngineEvent) error {
	if this.webhooks == nil {
		return nil
	}
	//methods := *this.spec
	if m, has := (*this.spec)[message.ServiceMethod]; has {
		if m.CallbackEvent != api.EventKey("") {
			return this.webhooks.Send(message.Service, string(m.CallbackEvent), message.Body, m.CallbackBodyTemplate)
		}
	}
	return ERROR_UNKNOWN_METHOD
}
<|endoftext|>


<commit_before>package compiler

import (
	"code.google.com/p/go.exp/fsnotify"
	"bytes"
	"log"
)

type Compiler struct {
	files map[CmplrFile]int // Filename to cache index
	cache [][]byte
}

func New(fs []string) *Compiler {
	c := &Compiler {
		files: make(map[CmplrFile]int),
		cache: make([][]byte, len(fs)),
	}

	for i, f := range fs {
		c.files[CmplrFile(f)] = i
	}

	return c
}

func (c *Compiler) Compile() (b []byte, err error) {
	for f, i := range c.files {
		b, err = f.Compile()
		if err != nil {
			return
		}
		c.cache[i] = b
	}

	b = bytes.Join(c.cache, []byte{})
	return
}

func (c *Compiler) Watch() chan []byte {
	w, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	ifUpdate := make(chan bool)
	go c.monitor(w, ifUpdate)

	for f := range c.files {
		err = w.WatchFlags(string(f), fsnotify.FSN_MODIFY)
		if err != nil {
			log.Fatal(err)
		}
	}

	ofUpdate := make(chan []byte)
	go func() {
		for {
			<-ifUpdate
			ofUpdate <- bytes.Join(c.cache, []byte{})
		}
	}()
	return ofUpdate
}

func (c *Compiler) monitor(w *fsnotify.Watcher, u chan bool) {
	for {
		select {
			case ev := <- w.Event:
				f := CmplrFile(ev.Name)
				log.Println(f, "- Recompiling")
				if out, err := f.Compile(); err == nil {
					c.cache[c.files[f]] = out
					u <- true
				} else {
					log.Println(err)
				}
			case err := <- w.Error:
				log.Println("error:", err)
		}
	}
}
<commit_msg>Only update output file if compiled source changes<commit_after>package compiler

import (
	"code.google.com/p/go.exp/fsnotify"
	"bytes"
	"log"
)

type Compiler struct {
	files map[CmplrFile]int // Filename to cache index
	cache [][]byte
}

func New(fs []string) *Compiler {
	c := &Compiler {
		files: make(map[CmplrFile]int),
		cache: make([][]byte, len(fs)),
	}

	for i, f := range fs {
		c.files[CmplrFile(f)] = i
	}

	return c
}

func (c *Compiler) Compile() (b []byte, err error) {
	for f, i := range c.files {
		b, err = f.Compile()
		if err != nil {
			return
		}
		c.cache[i] = b
	}

	b = bytes.Join(c.cache, []byte{})
	return
}

func (c *Compiler) Watch() chan []byte {
	w, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	ifUpdate := make(chan bool)
	go c.monitor(w, ifUpdate)

	for f := range c.files {
		err = w.WatchFlags(string(f), fsnotify.FSN_MODIFY)
		if err != nil {
			log.Fatal(err)
		}
	}

	ofUpdate := make(chan []byte)
	go func() {
		for {
			<-ifUpdate
			ofUpdate <- bytes.Join(c.cache, []byte{})
		}
	}()
	return ofUpdate
}

func (c *Compiler) monitor(w *fsnotify.Watcher, u chan bool) {
	for {
		select {
			case ev := <- w.Event:
				f := CmplrFile(ev.Name)
				if out, err := f.Compile(); err == nil {
					if !bytes.Equal(c.cache[c.files[f]], out) {
						log.Println(f, "- Recompiling")
						c.cache[c.files[f]] = out
						u <- true
					}
				} else {
					log.Println(err)
				}
			case err := <- w.Error:
				log.Println("error:", err)
		}
	}
}
<|endoftext|>


<commit_before>// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

// +build integration

package esutil_test

import (
	"context"
	"fmt"
	"os"
	"strconv"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"github.com/elastic/go-elasticsearch/v8"
	"github.com/elastic/go-elasticsearch/v8/estransport"
	"github.com/elastic/go-elasticsearch/v8/esutil"
)

func TestBulkIndexerIntegration(t *testing.T) {
	body := `{"body":"Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat."}`

	testCases := []struct {
		name                       string
		CompressRequestBodyEnabled bool
	}{
		{
			name:                       "Without body compression",
			CompressRequestBodyEnabled: false,
		},
		{
			name:                       "With body compression",
			CompressRequestBodyEnabled: true,
		},
	}

	for _, tt := range testCases {
		t.Run(tt.name, func(t *testing.T) {
			t.Run("Default", func(t *testing.T) {
				var countSuccessful uint64
				indexName := "test-bulk-integration"

				es, _ := elasticsearch.NewClient(elasticsearch.Config{
					CompressRequestBody: tt.CompressRequestBodyEnabled,
					Logger:              &estransport.ColorLogger{Output: os.Stdout},
				})

				es.Indices.Delete([]string{indexName}, es.Indices.Delete.WithIgnoreUnavailable(true))
				es.Indices.Create(
					indexName,
					es.Indices.Create.WithBody(strings.NewReader(`{"settings": {"number_of_shards": 1, "number_of_replicas": 0, "refresh_interval":"5s"}}`)),
					es.Indices.Create.WithWaitForActiveShards("1"))

				bi, _ := esutil.NewBulkIndexer(esutil.BulkIndexerConfig{
					Index:  indexName,
					Client: es,
					// FlushBytes: 3e+6,
				})

				numItems := 100000
				start := time.Now().UTC()

				for i := 1; i <= numItems; i++ {
					err := bi.Add(context.Background(), esutil.BulkIndexerItem{
						Action:     "index",
						DocumentID: strconv.Itoa(i),
						Body:       strings.NewReader(body),
						OnSuccess: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem) {
							atomic.AddUint64(&countSuccessful, 1)
						},
					})
					if err != nil {
						t.Fatalf("Unexpected error: %s", err)
					}
				}

				if err := bi.Close(context.Background()); err != nil {
					t.Errorf("Unexpected error: %s", err)
				}

				stats := bi.Stats()

				if stats.NumAdded != uint64(numItems) {
					t.Errorf("Unexpected NumAdded: want=%d, got=%d", numItems, stats.NumAdded)
				}

				if stats.NumIndexed != uint64(numItems) {
					t.Errorf("Unexpected NumIndexed: want=%d, got=%d", numItems, stats.NumIndexed)
				}

				if stats.NumFailed != 0 {
					t.Errorf("Unexpected NumFailed: want=0, got=%d", stats.NumFailed)
				}

				if countSuccessful != uint64(numItems) {
					t.Errorf("Unexpected countSuccessful: want=%d, got=%d", numItems, countSuccessful)
				}

				fmt.Printf("  Added %d documents to indexer. Succeeded: %d. Failed: %d. Requests: %d. Duration: %s (%.0f docs/sec)\n",
					stats.NumAdded,
					stats.NumFlushed,
					stats.NumFailed,
					stats.NumRequests,
					time.Since(start).Truncate(time.Millisecond),
					1000.0/float64(time.Since(start)/time.Millisecond)*float64(stats.NumFlushed))
			})

			t.Run("Multiple indices", func(t *testing.T) {
				es, _ := elasticsearch.NewClient(elasticsearch.Config{
					CompressRequestBody: tt.CompressRequestBodyEnabled,
					Logger:              &estransport.ColorLogger{Output: os.Stdout},
				})

				bi, _ := esutil.NewBulkIndexer(esutil.BulkIndexerConfig{
					Index:  "test-index-a",
					Client: es,
				})

				// Default index
				for i := 1; i <= 10; i++ {
					err := bi.Add(context.Background(), esutil.BulkIndexerItem{
						Action:     "index",
						DocumentID: strconv.Itoa(i),
						Body:       strings.NewReader(body),
					})
					if err != nil {
						t.Fatalf("Unexpected error: %s", err)
					}
				}

				// Index 1
				for i := 1; i <= 10; i++ {
					err := bi.Add(context.Background(), esutil.BulkIndexerItem{
						Action: "index",
						Index:  "test-index-b",
						Body:   strings.NewReader(body),
					})
					if err != nil {
						t.Fatalf("Unexpected error: %s", err)
					}
				}

				// Index 2
				for i := 1; i <= 10; i++ {
					err := bi.Add(context.Background(), esutil.BulkIndexerItem{
						Action: "index",
						Index:  "test-index-c",
						Body:   strings.NewReader(body),
					})
					if err != nil {
						t.Fatalf("Unexpected error: %s", err)
					}
				}

				if err := bi.Close(context.Background()); err != nil {
					t.Errorf("Unexpected error: %s", err)
				}
				stats := bi.Stats()

				expectedIndexed := 10 + 10 + 10
				if stats.NumIndexed != uint64(expectedIndexed) {
					t.Errorf("Unexpected NumIndexed: want=%d, got=%d", expectedIndexed, stats.NumIndexed)
				}

				res, err := es.Indices.Exists([]string{"test-index-a", "test-index-b", "test-index-c"})
				if err != nil {
					t.Fatalf("Unexpected error: %s", err)
				}
				if res.StatusCode != 200 {
					t.Errorf("Expected indices to exist, but got a [%s] response", res.Status())
				}
			})
		})
	}
}<commit_msg>CI: Util: Add integration tests to bulkindexer for external versioning (#360)<commit_after>// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

//go:build integration
// +build integration

package esutil_test

import (
	"context"
	"fmt"
	"os"
	"strconv"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"github.com/elastic/go-elasticsearch/v8"
	"github.com/elastic/go-elasticsearch/v8/estransport"
	"github.com/elastic/go-elasticsearch/v8/esutil"
)

func TestBulkIndexerIntegration(t *testing.T) {
	body := `{"body":"Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat."}`

	testCases := []struct {
		name                       string
		CompressRequestBodyEnabled bool
	}{
		{
			name:                       "Without body compression",
			CompressRequestBodyEnabled: false,
		},
		{
			name:                       "With body compression",
			CompressRequestBodyEnabled: true,
		},
	}

	for _, tt := range testCases {
		t.Run(tt.name, func(t *testing.T) {
			t.Run("Default", func(t *testing.T) {
				var countSuccessful uint64
				indexName := "test-bulk-integration"

				es, _ := elasticsearch.NewClient(elasticsearch.Config{
					CompressRequestBody: tt.CompressRequestBodyEnabled,
					Logger:              &estransport.ColorLogger{Output: os.Stdout},
				})

				es.Indices.Delete([]string{indexName}, es.Indices.Delete.WithIgnoreUnavailable(true))
				es.Indices.Create(
					indexName,
					es.Indices.Create.WithBody(strings.NewReader(`{"settings": {"number_of_shards": 1, "number_of_replicas": 0, "refresh_interval":"5s"}}`)),
					es.Indices.Create.WithWaitForActiveShards("1"))

				bi, _ := esutil.NewBulkIndexer(esutil.BulkIndexerConfig{
					Index:  indexName,
					Client: es,
					// FlushBytes: 3e+6,
				})

				numItems := 100000
				start := time.Now().UTC()

				for i := 1; i <= numItems; i++ {
					err := bi.Add(context.Background(), esutil.BulkIndexerItem{
						Action:     "index",
						DocumentID: strconv.Itoa(i),
						Body:       strings.NewReader(body),
						OnSuccess: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem) {
							atomic.AddUint64(&countSuccessful, 1)
						},
					})
					if err != nil {
						t.Fatalf("Unexpected error: %s", err)
					}
				}

				if err := bi.Close(context.Background()); err != nil {
					t.Errorf("Unexpected error: %s", err)
				}

				stats := bi.Stats()

				if stats.NumAdded != uint64(numItems) {
					t.Errorf("Unexpected NumAdded: want=%d, got=%d", numItems, stats.NumAdded)
				}

				if stats.NumIndexed != uint64(numItems) {
					t.Errorf("Unexpected NumIndexed: want=%d, got=%d", numItems, stats.NumIndexed)
				}

				if stats.NumFailed != 0 {
					t.Errorf("Unexpected NumFailed: want=0, got=%d", stats.NumFailed)
				}

				if countSuccessful != uint64(numItems) {
					t.Errorf("Unexpected countSuccessful: want=%d, got=%d", numItems, countSuccessful)
				}

				fmt.Printf("  Added %d documents to indexer. Succeeded: %d. Failed: %d. Requests: %d. Duration: %s (%.0f docs/sec)\n",
					stats.NumAdded,
					stats.NumFlushed,
					stats.NumFailed,
					stats.NumRequests,
					time.Since(start).Truncate(time.Millisecond),
					1000.0/float64(time.Since(start)/time.Millisecond)*float64(stats.NumFlushed))
			})

			t.Run("Multiple indices", func(t *testing.T) {
				es, _ := elasticsearch.NewClient(elasticsearch.Config{
					CompressRequestBody: tt.CompressRequestBodyEnabled,
					Logger:              &estransport.ColorLogger{Output: os.Stdout},
				})

				bi, _ := esutil.NewBulkIndexer(esutil.BulkIndexerConfig{
					Index:  "test-index-a",
					Client: es,
				})

				// Default index
				for i := 1; i <= 10; i++ {
					err := bi.Add(context.Background(), esutil.BulkIndexerItem{
						Action:     "index",
						DocumentID: strconv.Itoa(i),
						Body:       strings.NewReader(body),
					})
					if err != nil {
						t.Fatalf("Unexpected error: %s", err)
					}
				}

				// Index 1
				for i := 1; i <= 10; i++ {
					err := bi.Add(context.Background(), esutil.BulkIndexerItem{
						Action: "index",
						Index:  "test-index-b",
						Body:   strings.NewReader(body),
					})
					if err != nil {
						t.Fatalf("Unexpected error: %s", err)
					}
				}

				// Index 2
				for i := 1; i <= 10; i++ {
					err := bi.Add(context.Background(), esutil.BulkIndexerItem{
						Action: "index",
						Index:  "test-index-c",
						Body:   strings.NewReader(body),
					})
					if err != nil {
						t.Fatalf("Unexpected error: %s", err)
					}
				}

				if err := bi.Close(context.Background()); err != nil {
					t.Errorf("Unexpected error: %s", err)
				}
				stats := bi.Stats()

				expectedIndexed := 10 + 10 + 10
				if stats.NumIndexed != uint64(expectedIndexed) {
					t.Errorf("Unexpected NumIndexed: want=%d, got=%d", expectedIndexed, stats.NumIndexed)
				}

				res, err := es.Indices.Exists([]string{"test-index-a", "test-index-b", "test-index-c"})
				if err != nil {
					t.Fatalf("Unexpected error: %s", err)
				}
				if res.StatusCode != 200 {
					t.Errorf("Expected indices to exist, but got a [%s] response", res.Status())
				}
			})

			t.Run("External version", func(t *testing.T) {
				var index string = "test-index-a"

				es, _ := elasticsearch.NewClient(elasticsearch.Config{
					CompressRequestBody: tt.CompressRequestBodyEnabled,
					Logger:              &estransport.ColorLogger{Output: os.Stdout},
				})

				es.Indices.Delete([]string{index}, es.Indices.Delete.WithIgnoreUnavailable(true))
				es.Indices.Create(index, es.Indices.Create.WithWaitForActiveShards("1"))

				bulkIndex := func(bulkIndexer esutil.BulkIndexer, baseVersion int) {
					var countTotal int = 500
					var countSuccessful uint64
					for i := 0; i < countTotal; i++ {
						version := int64(baseVersion + i)
						item := esutil.BulkIndexerItem{
							Action:      "index",
							Index:       index,
							DocumentID:  strconv.Itoa(i),
							Body:        strings.NewReader(body),
							Version:     &version,
							VersionType: "external",
							OnSuccess: func(ctx context.Context, item esutil.BulkIndexerItem, item2 esutil.BulkIndexerResponseItem) {
								if version != item2.Version &&
									version != *item.Version &&
									item2.Result != "created" {
									t.Fatalf("version mismatch, expected: %d, got: %d && %d", version, item.Version, item2.Version)
								}
								atomic.AddUint64(&countSuccessful, 1)
							},
						}
						err := bulkIndexer.Add(context.Background(), item)
						if err != nil {
							t.Fatal(err)
						}
					}
					if err := bulkIndexer.Close(context.Background()); err != nil {
						t.Fatal(err)
					}

					if int(countSuccessful) != countTotal {
						t.Fatalf("Unexpected countSuccessful, expected %d, got: %d", countTotal, countSuccessful)
					}
				}

				bi, _ := esutil.NewBulkIndexer(esutil.BulkIndexerConfig{
					Index:  index,
					Client: es,
				})
				bulkIndex(bi, 500)

				bi, _ = esutil.NewBulkIndexer(esutil.BulkIndexerConfig{
					Index:  index,
					Client: es,
				})
				bulkIndex(bi, 900)
			})
		})
	}
}
<|endoftext|>


<commit_before>package provision

import (
	"github.com/docker/machine/drivers"
)

const (
	// TODO: eventually the RPM install process will be integrated
	// into the get.docker.com install script; for now
	// we install via vendored RPMs
	dockerFedoraRPMPath = "https://docker-mcn.s3.amazonaws.com/public/fedora/rpms/docker-engine-1.6.1-0.0.20150511.171646.git1b47f9f.fc21.x86_64.rpm"
)

func init() {
	Register("Fedora", &RegisteredProvisioner{
		New: NewFedoraProvisioner,
	})
}

func NewFedoraProvisioner(d drivers.Driver) Provisioner {
	g := GenericProvisioner{
		DockerOptionsDir:  "/etc/docker",
		DaemonOptionsFile: "/etc/systemd/system/docker.service",
		OsReleaseId:       "fedora",
		Packages:          []string{},
		Driver:            d,
	}
	p := &FedoraProvisioner{
		RedHatProvisioner{
			g,
			dockerFedoraRPMPath,
		},
	}
	return p
}

type FedoraProvisioner struct {
	RedHatProvisioner
}
<commit_msg>fedora: provisioner based off of redhat<commit_after>package provision

import (
	"github.com/docker/machine/drivers"
)

const (
	// TODO: eventually the RPM install process will be integrated
	// into the get.docker.com install script; for now
	// we install via vendored RPMs
	dockerFedoraRPMPath = "https://docker-mcn.s3.amazonaws.com/public/fedora/rpms/docker-engine-1.6.1-0.0.20150511.171646.git1b47f9f.fc21.x86_64.rpm"
)

func init() {
	Register("Fedora", &RegisteredProvisioner{
		New: NewFedoraProvisioner,
	})
}

func NewFedoraProvisioner(d drivers.Driver) Provisioner {
	g := GenericProvisioner{
		DockerOptionsDir:  "/etc/docker",
		DaemonOptionsFile: "/etc/systemd/system/docker.service",
		OsReleaseId:       "fedora",
		Packages: []string{
			"curl",
			"yum-utils",
		},
		Driver: d,
	}
	p := &FedoraProvisioner{
		RedHatProvisioner{
			g,
			dockerFedoraRPMPath,
		},
	}
	return p
}

type FedoraProvisioner struct {
	RedHatProvisioner
}
<|endoftext|>


<commit_before>package provision

import (
	"bytes"
	"errors"
	"fmt"
	"text/template"

	"github.com/docker/machine/libmachine/auth"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/engine"
	"github.com/docker/machine/libmachine/log"
	"github.com/docker/machine/libmachine/mcnutils"
	"github.com/docker/machine/libmachine/provision/pkgaction"
	"github.com/docker/machine/libmachine/provision/serviceaction"
	"github.com/docker/machine/libmachine/swarm"
)

var (
	ErrUnknownYumOsRelease = errors.New("unknown OS for Yum repository")

	packageListTemplate = `[docker]
name=Docker Stable Repository
baseurl=https://yum.dockerproject.org/repo/main/{{.OsRelease}}/{{.OsReleaseVersion}}
priority=1
enabled=1
gpgkey=https://yum.dockerproject.org/gpg
`
	engineConfigTemplate = `[Service]
ExecStart=/usr/bin/docker -d -H tcp://0.0.0.0:{{.DockerPort}} -H unix:///var/run/docker.sock --storage-driver {{.EngineOptions.StorageDriver}} --tlsverify --tlscacert {{.AuthOptions.CaCertRemotePath}} --tlscert {{.AuthOptions.ServerCertRemotePath}} --tlskey {{.AuthOptions.ServerKeyRemotePath}} {{ range .EngineOptions.Labels }}--label {{.}} {{ end }}{{ range .EngineOptions.InsecureRegistry }}--insecure-registry {{.}} {{ end }}{{ range .EngineOptions.RegistryMirror }}--registry-mirror {{.}} {{ end }}{{ range .EngineOptions.ArbitraryFlags }}--{{.}} {{ end }}
MountFlags=slave
LimitNOFILE=1048576
LimitNPROC=1048576
LimitCORE=infinity
Environment={{range .EngineOptions.Env}}{{ printf "%q" . }} {{end}}
`
)

type PackageListInfo struct {
	OsRelease        string
	OsReleaseVersion string
}

func init() {
	Register("RedHat", &RegisteredProvisioner{
		New: func(d drivers.Driver) Provisioner {
			return NewRedHatProvisioner("rhel", d)
		},
	})
}

func NewRedHatProvisioner(osReleaseID string, d drivers.Driver) *RedHatProvisioner {
	systemdProvisioner := NewSystemdProvisioner(osReleaseID, d)
	systemdProvisioner.SSHCommander = RedHatSSHCommander{Driver: d}
	return &RedHatProvisioner{
		systemdProvisioner,
	}
}

type RedHatProvisioner struct {
	SystemdProvisioner
}

func (provisioner *RedHatProvisioner) String() string {
	return "redhat"
}

func (provisioner *RedHatProvisioner) SetHostname(hostname string) error {
	// we have to have SetHostname here as well to use the RedHat provisioner
	// SSHCommand to add the tty allocation
	if _, err := provisioner.SSHCommand(fmt.Sprintf(
		"sudo hostname %s && echo %q | sudo tee /etc/hostname",
		hostname,
		hostname,
	)); err != nil {
		return err
	}

	// ubuntu/debian use 127.0.1.1 for non "localhost" loopback hostnames: https://www.debian.org/doc/manuals/debian-reference/ch05.en.html#_the_hostname_resolution
	if _, err := provisioner.SSHCommand(fmt.Sprintf(
		"if grep -xq 127.0.1.1.* /etc/hosts; then sudo sed -i 's/^127.0.1.1.*/127.0.1.1 %s/g' /etc/hosts; else echo '127.0.1.1 %s' | sudo tee -a /etc/hosts; fi",
		hostname,
		hostname,
	)); err != nil {
		return err
	}

	return nil
}

func (provisioner *RedHatProvisioner) Package(name string, action pkgaction.PackageAction) error {
	var packageAction string

	switch action {
	case pkgaction.Install:
		packageAction = "install"
	case pkgaction.Remove:
		packageAction = "remove"
	case pkgaction.Upgrade:
		packageAction = "upgrade"
	}

	command := fmt.Sprintf("sudo -E yum %s -y %s", packageAction, name)

	if _, err := provisioner.SSHCommand(command); err != nil {
		return err
	}

	return nil
}

func installDocker(provisioner *RedHatProvisioner) error {
	if err := provisioner.installOfficialDocker(); err != nil {
		return err
	}

	if err := provisioner.Service("docker", serviceaction.Restart); err != nil {
		return err
	}

	if err := provisioner.Service("docker", serviceaction.Enable); err != nil {
		return err
	}

	return nil
}

func (provisioner *RedHatProvisioner) installOfficialDocker() error {
	log.Debug("installing docker")

	if err := provisioner.ConfigurePackageList(); err != nil {
		return err
	}

	if _, err := provisioner.SSHCommand("sudo yum install -y docker-engine"); err != nil {
		return err
	}

	return nil
}

func (provisioner *RedHatProvisioner) dockerDaemonResponding() bool {
	log.Debug("checking docker daemon")

	if out, err := provisioner.SSHCommand("sudo docker version"); err != nil {
		log.Warnf("Error getting SSH command to check if the daemon is up: %s", err)
		log.Debugf("'sudo docker version' output:\n%s", out)
		return false
	}

	// The daemon is up if the command worked.  Carry on.
	return true
}

func (provisioner *RedHatProvisioner) Provision(swarmOptions swarm.Options, authOptions auth.Options, engineOptions engine.Options) error {
	provisioner.SwarmOptions = swarmOptions
	provisioner.AuthOptions = authOptions
	provisioner.EngineOptions = engineOptions
	swarmOptions.Env = engineOptions.Env

	// set default storage driver for redhat
	if provisioner.EngineOptions.StorageDriver == "" {
		provisioner.EngineOptions.StorageDriver = "devicemapper"
	}

	if err := provisioner.SetHostname(provisioner.Driver.GetMachineName()); err != nil {
		return err
	}

	for _, pkg := range provisioner.Packages {
		log.Debugf("installing base package: name=%s", pkg)
		if err := provisioner.Package(pkg, pkgaction.Install); err != nil {
			return err
		}
	}

	// update OS -- this is needed for libdevicemapper and the docker install
	if _, err := provisioner.SSHCommand("sudo yum -y update"); err != nil {
		return err
	}

	// install docker
	if err := installDocker(provisioner); err != nil {
		return err
	}

	if err := mcnutils.WaitFor(provisioner.dockerDaemonResponding); err != nil {
		return err
	}

	if err := makeDockerOptionsDir(provisioner); err != nil {
		return err
	}

	provisioner.AuthOptions = setRemoteAuthOptions(provisioner)

	if err := ConfigureAuth(provisioner); err != nil {
		return err
	}

	if err := configureSwarm(provisioner, swarmOptions, provisioner.AuthOptions); err != nil {
		return err
	}

	return nil
}

func (provisioner *RedHatProvisioner) GenerateDockerOptions(dockerPort int) (*DockerOptions, error) {
	var (
		engineCfg  bytes.Buffer
		configPath = provisioner.DaemonOptionsFile
	)

	driverNameLabel := fmt.Sprintf("provider=%s", provisioner.Driver.DriverName())
	provisioner.EngineOptions.Labels = append(provisioner.EngineOptions.Labels, driverNameLabel)

	// systemd / redhat will not load options if they are on newlines
	// instead, it just continues with a different set of options; yeah...
	t, err := template.New("engineConfig").Parse(engineConfigTemplate)
	if err != nil {
		return nil, err
	}

	engineConfigContext := EngineConfigContext{
		DockerPort:       dockerPort,
		AuthOptions:      provisioner.AuthOptions,
		EngineOptions:    provisioner.EngineOptions,
		DockerOptionsDir: provisioner.DockerOptionsDir,
	}

	t.Execute(&engineCfg, engineConfigContext)

	daemonOptsDir := configPath
	return &DockerOptions{
		EngineOptions:     engineCfg.String(),
		EngineOptionsPath: daemonOptsDir,
	}, nil
}

func generateYumRepoList(provisioner Provisioner) (*bytes.Buffer, error) {
	packageListInfo := &PackageListInfo{}

	releaseInfo, err := provisioner.GetOsReleaseInfo()
	if err != nil {
		return nil, err
	}

	switch releaseInfo.ID {
	case "rhel", "centos":
		// rhel and centos both use the "centos" repo
		packageListInfo.OsRelease = "centos"
		packageListInfo.OsReleaseVersion = "7"
	case "fedora":
		packageListInfo.OsRelease = "fedora"
		packageListInfo.OsReleaseVersion = "22"
	default:
		return nil, ErrUnknownYumOsRelease
	}

	t, err := template.New("packageList").Parse(packageListTemplate)
	if err != nil {
		return nil, err
	}

	var buf bytes.Buffer

	if err := t.Execute(&buf, packageListInfo); err != nil {
		return nil, err
	}

	return &buf, nil
}

func (provisioner *RedHatProvisioner) ConfigurePackageList() error {
	buf, err := generateYumRepoList(provisioner)
	if err != nil {
		return err
	}

	// we cannot use %q here as it combines the newlines in the formatting
	// on transport causing yum to not use the repo
	packageCmd := fmt.Sprintf("echo \"%s\" | sudo tee /etc/yum.repos.d/docker.repo", buf.String())
	if _, err := provisioner.SSHCommand(packageCmd); err != nil {
		return err
	}

	return nil
}
<commit_msg>Changed to experimental repo and replaced "-d" option with "daemon"<commit_after>package provision

import (
	"bytes"
	"errors"
	"fmt"
	"text/template"

	"github.com/docker/machine/libmachine/auth"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/engine"
	"github.com/docker/machine/libmachine/log"
	"github.com/docker/machine/libmachine/mcnutils"
	"github.com/docker/machine/libmachine/provision/pkgaction"
	"github.com/docker/machine/libmachine/provision/serviceaction"
	"github.com/docker/machine/libmachine/swarm"
)

var (
	ErrUnknownYumOsRelease = errors.New("unknown OS for Yum repository")

	packageListTemplate = `[docker]
name=Docker Stable Repository
baseurl=https://yum.dockerproject.org/repo/experimental/{{.OsRelease}}/{{.OsReleaseVersion}}
priority=1
enabled=1
gpgkey=https://yum.dockerproject.org/gpg
`
	engineConfigTemplate = `[Service]
ExecStart=/usr/bin/docker daemon -H tcp://0.0.0.0:{{.DockerPort}} -H unix:///var/run/docker.sock --storage-driver {{.EngineOptions.StorageDriver}} --tlsverify --tlscacert {{.AuthOptions.CaCertRemotePath}} --tlscert {{.AuthOptions.ServerCertRemotePath}} --tlskey {{.AuthOptions.ServerKeyRemotePath}} {{ range .EngineOptions.Labels }}--label {{.}} {{ end }}{{ range .EngineOptions.InsecureRegistry }}--insecure-registry {{.}} {{ end }}{{ range .EngineOptions.RegistryMirror }}--registry-mirror {{.}} {{ end }}{{ range .EngineOptions.ArbitraryFlags }}--{{.}} {{ end }}
MountFlags=slave
LimitNOFILE=1048576
LimitNPROC=1048576
LimitCORE=infinity
Environment={{range .EngineOptions.Env}}{{ printf "%q" . }} {{end}}
`
)

type PackageListInfo struct {
	OsRelease        string
	OsReleaseVersion string
}

func init() {
	Register("RedHat", &RegisteredProvisioner{
		New: func(d drivers.Driver) Provisioner {
			return NewRedHatProvisioner("rhel", d)
		},
	})
}

func NewRedHatProvisioner(osReleaseID string, d drivers.Driver) *RedHatProvisioner {
	systemdProvisioner := NewSystemdProvisioner(osReleaseID, d)
	systemdProvisioner.SSHCommander = RedHatSSHCommander{Driver: d}
	return &RedHatProvisioner{
		systemdProvisioner,
	}
}

type RedHatProvisioner struct {
	SystemdProvisioner
}

func (provisioner *RedHatProvisioner) String() string {
	return "redhat"
}

func (provisioner *RedHatProvisioner) SetHostname(hostname string) error {
	// we have to have SetHostname here as well to use the RedHat provisioner
	// SSHCommand to add the tty allocation
	if _, err := provisioner.SSHCommand(fmt.Sprintf(
		"sudo hostname %s && echo %q | sudo tee /etc/hostname",
		hostname,
		hostname,
	)); err != nil {
		return err
	}

	// ubuntu/debian use 127.0.1.1 for non "localhost" loopback hostnames: https://www.debian.org/doc/manuals/debian-reference/ch05.en.html#_the_hostname_resolution
	if _, err := provisioner.SSHCommand(fmt.Sprintf(
		"if grep -xq 127.0.1.1.* /etc/hosts; then sudo sed -i 's/^127.0.1.1.*/127.0.1.1 %s/g' /etc/hosts; else echo '127.0.1.1 %s' | sudo tee -a /etc/hosts; fi",
		hostname,
		hostname,
	)); err != nil {
		return err
	}

	return nil
}

func (provisioner *RedHatProvisioner) Package(name string, action pkgaction.PackageAction) error {
	var packageAction string

	switch action {
	case pkgaction.Install:
		packageAction = "install"
	case pkgaction.Remove:
		packageAction = "remove"
	case pkgaction.Upgrade:
		packageAction = "upgrade"
	}

	command := fmt.Sprintf("sudo -E yum %s -y %s", packageAction, name)

	if _, err := provisioner.SSHCommand(command); err != nil {
		return err
	}

	return nil
}

func installDocker(provisioner *RedHatProvisioner) error {
	if err := provisioner.installOfficialDocker(); err != nil {
		return err
	}

	if err := provisioner.Service("docker", serviceaction.Restart); err != nil {
		return err
	}

	if err := provisioner.Service("docker", serviceaction.Enable); err != nil {
		return err
	}

	return nil
}

func (provisioner *RedHatProvisioner) installOfficialDocker() error {
	log.Debug("installing docker")

	if err := provisioner.ConfigurePackageList(); err != nil {
		return err
	}

	if _, err := provisioner.SSHCommand("sudo yum install -y docker-engine"); err != nil {
		return err
	}

	return nil
}

func (provisioner *RedHatProvisioner) dockerDaemonResponding() bool {
	log.Debug("checking docker daemon")

	if out, err := provisioner.SSHCommand("sudo docker version"); err != nil {
		log.Warnf("Error getting SSH command to check if the daemon is up: %s", err)
		log.Debugf("'sudo docker version' output:\n%s", out)
		return false
	}

	// The daemon is up if the command worked.  Carry on.
	return true
}

func (provisioner *RedHatProvisioner) Provision(swarmOptions swarm.Options, authOptions auth.Options, engineOptions engine.Options) error {
	provisioner.SwarmOptions = swarmOptions
	provisioner.AuthOptions = authOptions
	provisioner.EngineOptions = engineOptions
	swarmOptions.Env = engineOptions.Env

	// set default storage driver for redhat
	if provisioner.EngineOptions.StorageDriver == "" {
		provisioner.EngineOptions.StorageDriver = "devicemapper"
	}

	if err := provisioner.SetHostname(provisioner.Driver.GetMachineName()); err != nil {
		return err
	}

	for _, pkg := range provisioner.Packages {
		log.Debugf("installing base package: name=%s", pkg)
		if err := provisioner.Package(pkg, pkgaction.Install); err != nil {
			return err
		}
	}

	// update OS -- this is needed for libdevicemapper and the docker install
	if _, err := provisioner.SSHCommand("sudo yum -y update"); err != nil {
		return err
	}

	// install docker
	if err := installDocker(provisioner); err != nil {
		return err
	}

	if err := mcnutils.WaitFor(provisioner.dockerDaemonResponding); err != nil {
		return err
	}

	if err := makeDockerOptionsDir(provisioner); err != nil {
		return err
	}

	provisioner.AuthOptions = setRemoteAuthOptions(provisioner)

	if err := ConfigureAuth(provisioner); err != nil {
		return err
	}

	if err := configureSwarm(provisioner, swarmOptions, provisioner.AuthOptions); err != nil {
		return err
	}

	return nil
}

func (provisioner *RedHatProvisioner) GenerateDockerOptions(dockerPort int) (*DockerOptions, error) {
	var (
		engineCfg  bytes.Buffer
		configPath = provisioner.DaemonOptionsFile
	)

	driverNameLabel := fmt.Sprintf("provider=%s", provisioner.Driver.DriverName())
	provisioner.EngineOptions.Labels = append(provisioner.EngineOptions.Labels, driverNameLabel)

	// systemd / redhat will not load options if they are on newlines
	// instead, it just continues with a different set of options; yeah...
	t, err := template.New("engineConfig").Parse(engineConfigTemplate)
	if err != nil {
		return nil, err
	}

	engineConfigContext := EngineConfigContext{
		DockerPort:       dockerPort,
		AuthOptions:      provisioner.AuthOptions,
		EngineOptions:    provisioner.EngineOptions,
		DockerOptionsDir: provisioner.DockerOptionsDir,
	}

	t.Execute(&engineCfg, engineConfigContext)

	daemonOptsDir := configPath
	return &DockerOptions{
		EngineOptions:     engineCfg.String(),
		EngineOptionsPath: daemonOptsDir,
	}, nil
}

func generateYumRepoList(provisioner Provisioner) (*bytes.Buffer, error) {
	packageListInfo := &PackageListInfo{}

	releaseInfo, err := provisioner.GetOsReleaseInfo()
	if err != nil {
		return nil, err
	}

	switch releaseInfo.ID {
	case "rhel", "centos":
		// rhel and centos both use the "centos" repo
		packageListInfo.OsRelease = "centos"
		packageListInfo.OsReleaseVersion = "7"
	case "fedora":
		packageListInfo.OsRelease = "fedora"
		packageListInfo.OsReleaseVersion = "22"
	default:
		return nil, ErrUnknownYumOsRelease
	}

	t, err := template.New("packageList").Parse(packageListTemplate)
	if err != nil {
		return nil, err
	}

	var buf bytes.Buffer

	if err := t.Execute(&buf, packageListInfo); err != nil {
		return nil, err
	}

	return &buf, nil
}

func (provisioner *RedHatProvisioner) ConfigurePackageList() error {
	buf, err := generateYumRepoList(provisioner)
	if err != nil {
		return err
	}

	// we cannot use %q here as it combines the newlines in the formatting
	// on transport causing yum to not use the repo
	packageCmd := fmt.Sprintf("echo \"%s\" | sudo tee /etc/yum.repos.d/docker.repo", buf.String())
	if _, err := provisioner.SSHCommand(packageCmd); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package components

import (
	"fmt"
	"github.com/thethingsnetwork/core"
	"github.com/thethingsnetwork/core/lorawan/semtech"
	"github.com/thethingsnetwork/core/utils/log"
	"time"
)

const (
	EXPIRY_DELAY   = time.Hour * 8
	UP_POOL_SIZE   = 1
	DOWN_POOL_SIZE = 1
)

// Router represents a concrete router of TTN architecture. Use the New() method to create a new
// one and then connect it to its adapters.
type Router struct {
	brokers       []core.BrokerAddress // Brokers known by the router
	Logger        log.Logger           // Specify a logger for the router. NOTE Having this exported isn't thread-safe.
	addressKeeper addressKeeper        // Local storage that maps end-device addresses to broker addresses
	up            chan upMsg           // Internal communication channel which sends data to the up adapter
	down          chan downMsg         // Internal communication channel which sends data to the down adapter
}

// upMsg materializes messages that flow along the up channel
type upMsg struct {
	packet  semtech.Packet      // The packet to transfer
	gateway core.GatewayAddress // The recipient gateway to reach
}

// downMsg materializes messages that flow along the down channel
type downMsg struct {
	payload semtech.Payload      // The payload to transfer
	brokers []core.BrokerAddress // The recipient broker to reach. If nil or empty, assume that all broker should be reached
}

// NewRouter constructs a Router and setup its internal structure
func NewRouter(brokers ...core.BrokerAddress) (*Router, error) {
	localDB, err := NewLocalDB(EXPIRY_DELAY)

	if err != nil {
		return nil, err
	}

	if len(brokers) == 0 {
		return nil, core.ErrBadOptions
	}

	return &Router{
		brokers:       brokers,
		addressKeeper: localDB,
		up:            make(chan upMsg),
		down:          make(chan downMsg),
		Logger:        log.VoidLogger{},
	}, nil
}

// HandleUplink implements the core.Router interface
func (r *Router) HandleUplink(packet semtech.Packet, gateway core.GatewayAddress) {
	r.ensure()

	switch packet.Identifier {
	case semtech.PULL_DATA:
		r.log("receives PULL_DATA, sending ack")
		r.up <- upMsg{
			packet: semtech.Packet{
				Version:    semtech.VERSION,
				Identifier: semtech.PULL_ACK,
				Token:      packet.Token,
			},
			gateway: gateway,
		}
	case semtech.PUSH_DATA:
		// 1. Send an ack
		r.log("receives PUSH_DATA, sending ack")
		r.up <- upMsg{
			packet: semtech.Packet{
				Version:    semtech.VERSION,
				Identifier: semtech.PUSH_ACK,
				Token:      packet.Token,
			},
			gateway: gateway,
		}

		// 2. Determine payloads related to different end-devices present in the packet
		// NOTE So far, Stats are ignored.
		if packet.Payload == nil || len(packet.Payload.RXPK) == 0 {
			r.log("Ignores inconsistent PUSH_DATA packet")
			return
		}

		payloads := make(map[semtech.DeviceAddress]*semtech.Payload)
		for _, rxpk := range packet.Payload.RXPK {
			devAddr := rxpk.DevAddr()
			if devAddr == nil {
				r.log("Unable to determine end-device address for rxpk: %+v", rxpk)
				continue
			}

			if _, ok := payloads[*devAddr]; !ok {
				payloads[*devAddr] = &semtech.Payload{
					RXPK: make([]semtech.RXPK, 0),
				}
			}

			payload := payloads[*devAddr]
			(*payload).RXPK = append(payloads[*devAddr].RXPK, rxpk)
		}

		// 3. Broadcast or Forward payloads depending wether or not the brokers are known
		for devAddr, payload := range payloads {
			brokers, err := r.addressKeeper.lookup(devAddr)
			if err != nil {
				r.log("Forward payload to known brokers %+v", payload)
				r.down <- downMsg{
					payload: *payload,
					brokers: brokers,
				}
				continue
			}

			r.log("Broadcast payload to all brokers %+v", payload)
			r.down <- downMsg{payload: *payload}
		}
	default:
		r.log("Unexpected packet receive from uplink %+v", packet)
	}
}

// HandleDownlink implements the core.Router interface
func (r *Router) HandleDownlink(payload semtech.Payload, broker core.BrokerAddress) {
	// TODO MileStone 4
}

// RegisterDevice implements the core.Router interface
func (r *Router) RegisterDevice(devAddr semtech.DeviceAddress, broAddrs ...core.BrokerAddress) {
	r.ensure()
	r.addressKeeper.store(devAddr, broAddrs...) // TODO handle the error
}

// RegisterDevice implements the core.Router interface
func (r *Router) HandleError(err interface{}) {
	r.ensure()

	switch err.(type) {
	default:
		fmt.Println(err) // Wow, much handling, very reliable
	}
}

// Connect implements the core.Router interface
func (r *Router) Connect(upAdapter core.GatewayRouterAdapter, downAdapter core.RouterBrokerAdapter) {
	r.ensure()

	for i := 0; i < UP_POOL_SIZE; i += 1 {
		go r.connectUpAdapter(upAdapter)
	}

	for i := 0; i < DOWN_POOL_SIZE; i += 1 {
		go r.connectDownAdapter(downAdapter)
	}
}

// Consume messages sent to r.up channel
func (r *Router) connectUpAdapter(upAdapter core.GatewayRouterAdapter) {
	for msg := range r.up {
		upAdapter.Ack(r, msg.packet, msg.gateway)
	}
}

// Consume messages sent to r.down channel
func (r *Router) connectDownAdapter(downAdapter core.RouterBrokerAdapter) {
	for msg := range r.down {
		if len(msg.brokers) == 0 {
			downAdapter.Broadcast(r, msg.payload)
			continue
		}
		downAdapter.Forward(r, msg.payload, msg.brokers...)
	}
}

// ensure checks whether or not the current Router has been created via New(). It panics if not.
func (r *Router) ensure() {
	if r == nil || r.addressKeeper == nil {
		panic("Call method on non-initialized Router")
	}
}

// log is a shortcut to access the router logger
func (r *Router) log(format string, a ...interface{}) {
	if r.Logger == nil {
		return
	}
	r.Logger.Log(format, a...)
}
<commit_msg>[router] Add log line in router.go<commit_after>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package components

import (
	"fmt"
	"github.com/thethingsnetwork/core"
	"github.com/thethingsnetwork/core/lorawan/semtech"
	"github.com/thethingsnetwork/core/utils/log"
	"time"
)

const (
	EXPIRY_DELAY   = time.Hour * 8
	UP_POOL_SIZE   = 1
	DOWN_POOL_SIZE = 1
)

// Router represents a concrete router of TTN architecture. Use the New() method to create a new
// one and then connect it to its adapters.
type Router struct {
	brokers       []core.BrokerAddress // Brokers known by the router
	Logger        log.Logger           // Specify a logger for the router. NOTE Having this exported isn't thread-safe.
	addressKeeper addressKeeper        // Local storage that maps end-device addresses to broker addresses
	up            chan upMsg           // Internal communication channel which sends data to the up adapter
	down          chan downMsg         // Internal communication channel which sends data to the down adapter
}

// upMsg materializes messages that flow along the up channel
type upMsg struct {
	packet  semtech.Packet      // The packet to transfer
	gateway core.GatewayAddress // The recipient gateway to reach
}

// downMsg materializes messages that flow along the down channel
type downMsg struct {
	payload semtech.Payload      // The payload to transfer
	brokers []core.BrokerAddress // The recipient broker to reach. If nil or empty, assume that all broker should be reached
}

// NewRouter constructs a Router and setup its internal structure
func NewRouter(brokers ...core.BrokerAddress) (*Router, error) {
	localDB, err := NewLocalDB(EXPIRY_DELAY)

	if err != nil {
		return nil, err
	}

	if len(brokers) == 0 {
		return nil, core.ErrBadOptions
	}

	return &Router{
		brokers:       brokers,
		addressKeeper: localDB,
		up:            make(chan upMsg),
		down:          make(chan downMsg),
		Logger:        log.VoidLogger{},
	}, nil
}

// HandleUplink implements the core.Router interface
func (r *Router) HandleUplink(packet semtech.Packet, gateway core.GatewayAddress) {
	r.ensure()

	switch packet.Identifier {
	case semtech.PULL_DATA:
		r.log("receives PULL_DATA, sending ack")
		r.up <- upMsg{
			packet: semtech.Packet{
				Version:    semtech.VERSION,
				Identifier: semtech.PULL_ACK,
				Token:      packet.Token,
			},
			gateway: gateway,
		}
	case semtech.PUSH_DATA:
		// 1. Send an ack
		r.log("receives PUSH_DATA, sending ack")
		r.up <- upMsg{
			packet: semtech.Packet{
				Version:    semtech.VERSION,
				Identifier: semtech.PUSH_ACK,
				Token:      packet.Token,
			},
			gateway: gateway,
		}

		// 2. Determine payloads related to different end-devices present in the packet
		// NOTE So far, Stats are ignored.
		if packet.Payload == nil || len(packet.Payload.RXPK) == 0 {
			r.log("Ignores inconsistent PUSH_DATA packet")
			return
		}

		payloads := make(map[semtech.DeviceAddress]*semtech.Payload)
		for _, rxpk := range packet.Payload.RXPK {
			devAddr := rxpk.DevAddr()
			if devAddr == nil {
				r.log("Unable to determine end-device address for rxpk: %+v", rxpk)
				continue
			}

			if _, ok := payloads[*devAddr]; !ok {
				payloads[*devAddr] = &semtech.Payload{
					RXPK: make([]semtech.RXPK, 0),
				}
			}

			payload := payloads[*devAddr]
			(*payload).RXPK = append(payloads[*devAddr].RXPK, rxpk)
		}

		// 3. Broadcast or Forward payloads depending wether or not the brokers are known
		for devAddr, payload := range payloads {
			brokers, err := r.addressKeeper.lookup(devAddr)
			if err == nil {
				r.log("Forward payload to known brokers %+v", payload)
				r.down <- downMsg{
					payload: *payload,
					brokers: brokers,
				}
				continue
			}

			r.log("Broadcast payload to all brokers %+v", payload)
			r.down <- downMsg{payload: *payload}
		}
	default:
		r.log("Unexpected packet receive from uplink %+v", packet)
	}
}

// HandleDownlink implements the core.Router interface
func (r *Router) HandleDownlink(payload semtech.Payload, broker core.BrokerAddress) {
	// TODO MileStone 4
}

// RegisterDevice implements the core.Router interface
func (r *Router) RegisterDevice(devAddr semtech.DeviceAddress, broAddrs ...core.BrokerAddress) {
	r.ensure()
	r.log("Register device %+x to brokers: %v", devAddr, broAddrs)
	r.addressKeeper.store(devAddr, broAddrs...) // TODO handle the error
}

// RegisterDevice implements the core.Router interface
func (r *Router) HandleError(err interface{}) {
	r.ensure()

	switch err.(type) {
	default:
		fmt.Println(err) // Wow, much handling, very reliable
	}
}

// Connect implements the core.Router interface
func (r *Router) Connect(upAdapter core.GatewayRouterAdapter, downAdapter core.RouterBrokerAdapter) {
	r.ensure()

	for i := 0; i < UP_POOL_SIZE; i += 1 {
		go r.connectUpAdapter(upAdapter)
	}

	for i := 0; i < DOWN_POOL_SIZE; i += 1 {
		go r.connectDownAdapter(downAdapter)
	}
}

// Consume messages sent to r.up channel
func (r *Router) connectUpAdapter(upAdapter core.GatewayRouterAdapter) {
	for msg := range r.up {
		upAdapter.Ack(r, msg.packet, msg.gateway)
	}
}

// Consume messages sent to r.down channel
func (r *Router) connectDownAdapter(downAdapter core.RouterBrokerAdapter) {
	for msg := range r.down {
		if len(msg.brokers) == 0 {
			downAdapter.Broadcast(r, msg.payload)
			continue
		}
		downAdapter.Forward(r, msg.payload, msg.brokers...)
	}
}

// ensure checks whether or not the current Router has been created via New(). It panics if not.
func (r *Router) ensure() {
	if r == nil || r.addressKeeper == nil {
		panic("Call method on non-initialized Router")
	}
}

// log is a shortcut to access the router logger
func (r *Router) log(format string, a ...interface{}) {
	if r.Logger == nil {
		return
	}
	r.Logger.Log(format, a...)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"time"
)

// 配列やスライスはconstで定義できない。
// const numbers = [9]int{1, 2, 3, 4, 5, 6, 7, 8, 9}

func spinner(message string, delay time.Duration) {
	for {
		for _, r := range `-\|/` {
			fmt.Printf("\r%c%v", r, message)
			time.Sleep(delay)
		}
	}
}

func main() {
	var delay time.Duration
	var acc time.Duration

	go spinner("コンピュータ思考中...", 140*time.Millisecond)

	// 乱数のシードをナノ秒にしておかないと、デフォルトは秒なので短時間の処理では乱数が分布しない。

	// http://qiita.com/cubicdaiya/items/819886c57e9d17e4b019
	// ↑ここのコメント欄から

	// ちなみに↓ここでも同じことをして、Pythonのshuffleを模倣している。
	// http://marcelom.github.io/2013/06/07/goshuffle.html
	rand.Seed(time.Now().UnixNano())

	for _, x := range rand.Perm(9)[:5] {
		delay = time.Duration(100*x) * time.Millisecond
		acc += delay
		time.Sleep(delay)

		// fmt.Println(x, 100*x*time.Millisecond) // これはコンパイルエラー

		// ?? どういうこと?
		// invalid operation: 100 * x * time.Millisecond (mismatched types int and time.Duration)
		// time.Durationはint64の別名型なのでダメらしい
		// 掛け算とわり算でやり方が違う点がまたわかりにくい

		// https://golang.org/pkg/time/#Duration
		// fmt.Print(int64(second/time.Millisecond)) // prints 1000
		// fmt.Print(time.Duration(seconds)*time.Second) // prints 10s

		// [追記 11:45]

		// > 掛け算とわり算でやり方が違う

		// これはそういうことじゃなくて、割り算の例は、Duration -> int64に、
		// 掛け算の例はint64 -> Durationに変換している例だと思う。

		// time.SleepはDurationを引数に取るので注意
	}

	// ANSIエスケープコードを使ってコンソール制御
	// http://www.mm2d.net/main/prog/c/console-02.html

	// fmt.Printf("\033[H\033[2J") // これだと画面を消してしまう
	fmt.Printf("\033[2K\033[G") // 行を消し、その後カーソルを行頭に移動
	// [ok] だけ色を緑に変える
	fmt.Printf("\033[32m[ok]\033[39m %v かかりました\n", acc)
}
<commit_msg>garden<commit_after>package main

import (
	"fmt"
	"math/rand"
	"time"
)

// 配列やスライスはconstで定義できない。
// const numbers = [9]int{1, 2, 3, 4, 5, 6, 7, 8, 9}

func spinner(message string, delay time.Duration) {
	for {
		for _, r := range `-\|/` {
			fmt.Printf("\r%c%v", r, message)
			time.Sleep(delay)
		}
	}
}

func main() {
	var delay time.Duration
	var acc time.Duration

	go spinner("コンピュータ思考中...", 140*time.Millisecond)

	// 乱数のシードをナノ秒にしておかないと、デフォルトは秒なので短時間の処理では乱数が分布しない。

	// http://qiita.com/cubicdaiya/items/819886c57e9d17e4b019
	// ↑ここのコメント欄から

	// ちなみに↓ここでも同じことをして、Pythonのshuffleを模倣している。
	// http://marcelom.github.io/2013/06/07/goshuffle.html
	rand.Seed(time.Now().UnixNano())

	for _, x := range rand.Perm(9)[:5] {
		delay = time.Duration(100*x) * time.Millisecond
		acc += delay
		time.Sleep(delay)

		// fmt.Println(x, 100*x*time.Millisecond) // これはコンパイルエラー

		// ?? どういうこと?
		// invalid operation: 100 * x * time.Millisecond (mismatched types int and time.Duration)
		// time.Durationはint64の別名型なのでダメらしい
		// 掛け算とわり算でやり方が違う点がまたわかりにくい

		// https://golang.org/pkg/time/#Duration
		// fmt.Print(int64(second/time.Millisecond)) // prints 1000
		// fmt.Print(time.Duration(seconds)*time.Second) // prints 10s

		// [追記 11:45]

		// > 掛け算とわり算でやり方が違う

		// これはそういうことじゃなくて、割り算の例は、Duration -> int64に、
		// 掛け算の例はint64 -> Durationに変換している例だと思う。

		// time.SleepはDurationを引数に取るので注意
	}

	// ANSIエスケープコードを使ってコンソール制御
	// http://www.mm2d.net/main/prog/c/console-02.html

	// fmt.Printf("\033[H\033[2J") // これだと画面を消してしまう
	fmt.Printf("\033[2K\033[G") // 行を消し、その後カーソルを行頭に移動
	// [ok] だけ色を緑に変える
	fmt.Printf("\033[32m[ok]\033[39m %v かかりました\n", acc)
	// fmt.Printf("\r%v かかりました\n", acc)
}
<|endoftext|>


<commit_before>package tgff

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"testing"

	"github.com/goesd/support/assert"
)

const (
	fixturePath = "fixtures"
)

func TestParseSuccess(t *testing.T) {
	file := openFixture("simple")
	defer file.Close()

	r, err := Parse(file)

	assert.Success(err, t)

	assert.Equal(r.Period, uint32(1180), t)
	assert.Equal(len(r.Graphs), 5, t)
	assert.Equal(len(r.Tables), 3, t)

	graphs := []struct {
		period    uint32
		tasks     int
		arcs      int
		deadlines int
	}{
		{590, 12, 19, 1},
		{1180, 20, 25, 6},
		{1180, 24, 28, 8},
		{590, 8, 7, 3},
		{1180, 20, 24, 6},
	}

	for i, graph := range graphs {
		assert.Equal(r.Graphs[i].Name, "TASK_GRAPH", t)
		assert.Equal(r.Graphs[i].Number, uint32(i), t)
		assert.Equal(r.Graphs[i].Period, graph.period, t)

		assert.Equal(len(r.Graphs[i].Tasks), graph.tasks, t)
		assert.Equal(len(r.Graphs[i].Arcs), graph.arcs, t)
		assert.Equal(len(r.Graphs[i].Deadlines), graph.deadlines, t)
	}

	tables := []struct {
		price float64
	}{
		{70.1121},
		{71.4235},
		{80.491},
	}

	for i, table := range tables {
		assert.Equal(r.Tables[i].Name, "COMMUN", t)
		assert.Equal(r.Tables[i].Number, uint32(i), t)
		assert.Equal(r.Tables[i].Attributes["price"], table.price, t)

		assert.Equal(len(r.Tables[i].Columns), 2, t)
		assert.Equal(r.Tables[i].Columns[0].Name, "type", t)
		assert.Equal(r.Tables[i].Columns[1].Name, "exec_time", t)
	}

	assert.DeepEqual(r.Tables[2].Columns[1].Data, fixtureSimpleTable2Column1, t)
}

func TestParseFailure(t *testing.T) {
	reader := strings.NewReader("  @ garbage")

	_, err := Parse(reader)

	assert.Failure(err, t)
}

func BenchmarkParseSimple(b *testing.B) {
	data := readFixture("simple")

	for i := 0; i < b.N; i++ {
		Parse(bytes.NewReader(data))
	}
}

func BenchmarkParseComplex(b *testing.B) {
	data := readFixture("complex")

	for i := 0; i < b.N; i++ {
		Parse(bytes.NewReader(data))
	}
}

func ExampleParseFile() {
	result, _ := ParseFile("fixtures/simple.tgff")

	fmt.Println("Task graphs:", len(result.Graphs))
	fmt.Println("Data tables:", len(result.Tables))

	// Output:
	// Task graphs: 5
	// Data tables: 3
}

func readFixture(name string) []byte {
	path := path.Join(fixturePath, fmt.Sprintf("%s.tgff", name))
	data, _ := ioutil.ReadFile(path)

	return data
}

func openFixture(name string) *os.File {
	path := path.Join(fixturePath, fmt.Sprintf("%s.tgff", name))
	file, _ := os.Open(path)

	return file
}

var fixtureSimpleTable2Column1 = []float64{
	48.5893,
	33.4384,
	34.2468,
	51.2027,
	51.3571,
	30.3827,
	43.3982,
	60.9097,
	36.0322,
	34.7446,
	45.3479,
	31.7221,
	49.6842,
	52.0635,
	44.7690,
	37.7183,
	54.7523,
	58.4432,
	33.1266,
	48.2143,
	31.2946,
	45.9168,
	36.4521,
	61.6448,
	49.4966,
	37.1130,
	40.1642,
	38.9454,
	41.6213,
	42.1084,
	42.4186,
	42.5145,
	34.4180,
	33.4178,
	32.4243,
	63.7925,
	50.3810,
	51.9030,
	46.4714,
	35.0566,
	41.8399,
	30.1513,
	31.7449,
	57.3263,
	61.2321,
	44.9932,
	32.0830,
	37.9489,
	62.4774,
	39.2500,
}
<commit_msg>Fixed a typo<commit_after>package tgff

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"testing"

	"github.com/goesd/support/assert"
)

const (
	fixturePath = "fixtures"
)

func TestParseSuccess(t *testing.T) {
	file := openFixture("simple")
	defer file.Close()

	r, err := Parse(file)

	assert.Success(err, t)

	assert.Equal(r.Period, uint32(1180), t)
	assert.Equal(len(r.Graphs), 5, t)
	assert.Equal(len(r.Tables), 3, t)

	graphs := []struct {
		period    uint32
		tasks     int
		arcs      int
		deadlines int
	}{
		{590, 12, 19, 1},
		{1180, 20, 25, 6},
		{1180, 24, 28, 8},
		{590, 8, 7, 3},
		{1180, 20, 24, 6},
	}

	for i, graph := range graphs {
		assert.Equal(r.Graphs[i].Name, "TASK_GRAPH", t)
		assert.Equal(r.Graphs[i].Number, uint32(i), t)
		assert.Equal(r.Graphs[i].Period, graph.period, t)

		assert.Equal(len(r.Graphs[i].Tasks), graph.tasks, t)
		assert.Equal(len(r.Graphs[i].Arcs), graph.arcs, t)
		assert.Equal(len(r.Graphs[i].Deadlines), graph.deadlines, t)
	}

	tables := []struct {
		price float64
	}{
		{70.1121},
		{71.4235},
		{80.491},
	}

	for i, table := range tables {
		assert.Equal(r.Tables[i].Name, "COMMUN", t)
		assert.Equal(r.Tables[i].Number, uint32(i), t)
		assert.Equal(r.Tables[i].Attributes["price"], table.price, t)

		assert.Equal(len(r.Tables[i].Columns), 2, t)
		assert.Equal(r.Tables[i].Columns[0].Name, "type", t)
		assert.Equal(r.Tables[i].Columns[1].Name, "exec_time", t)
	}

	assert.DeepEqual(r.Tables[1].Columns[1].Data, fixtureSimpleTable1Column1, t)
}

func TestParseFailure(t *testing.T) {
	reader := strings.NewReader("  @ garbage")

	_, err := Parse(reader)

	assert.Failure(err, t)
}

func BenchmarkParseSimple(b *testing.B) {
	data := readFixture("simple")

	for i := 0; i < b.N; i++ {
		Parse(bytes.NewReader(data))
	}
}

func BenchmarkParseComplex(b *testing.B) {
	data := readFixture("complex")

	for i := 0; i < b.N; i++ {
		Parse(bytes.NewReader(data))
	}
}

func ExampleParseFile() {
	result, _ := ParseFile("fixtures/simple.tgff")

	fmt.Println("Task graphs:", len(result.Graphs))
	fmt.Println("Data tables:", len(result.Tables))

	// Output:
	// Task graphs: 5
	// Data tables: 3
}

func readFixture(name string) []byte {
	path := path.Join(fixturePath, fmt.Sprintf("%s.tgff", name))
	data, _ := ioutil.ReadFile(path)

	return data
}

func openFixture(name string) *os.File {
	path := path.Join(fixturePath, fmt.Sprintf("%s.tgff", name))
	file, _ := os.Open(path)

	return file
}

var fixtureSimpleTable1Column1 = []float64{
	48.5893,
	33.4384,
	34.2468,
	51.2027,
	51.3571,
	30.3827,
	43.3982,
	60.9097,
	36.0322,
	34.7446,
	45.3479,
	31.7221,
	49.6842,
	52.0635,
	44.7690,
	37.7183,
	54.7523,
	58.4432,
	33.1266,
	48.2143,
	31.2946,
	45.9168,
	36.4521,
	61.6448,
	49.4966,
	37.1130,
	40.1642,
	38.9454,
	41.6213,
	42.1084,
	42.4186,
	42.5145,
	34.4180,
	33.4178,
	32.4243,
	63.7925,
	50.3810,
	51.9030,
	46.4714,
	35.0566,
	41.8399,
	30.1513,
	31.7449,
	57.3263,
	61.2321,
	44.9932,
	32.0830,
	37.9489,
	62.4774,
	39.2500,
}
<|endoftext|>


<commit_before>package download

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/subutai-io/agent/log"
	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
)

type ListItem struct {
	ID           string            `json:"id"`
	Size         string            `json:"size,omitempty"`
	Name         string            `json:"name,omitempty"`
	Owner        []string          `json:"owner,omitempty"`
	Parent       string            `json:"parent,omitempty"`
	Version      string            `json:"version,omitempty"`
	Filename     string            `json:"filename,omitempty"`
	Prefsize     string            `json:"prefsize,omitempty"`
	Signature    map[string]string `json:"signature,omitempty"`
	Description  string            `json:"description,omitempty"`
	Architecture string            `json:"architecture,omitempty"`
}

func Handler(repo string, w http.ResponseWriter, r *http.Request) {
	hash := r.URL.Query().Get("hash")
	name := r.URL.Query().Get("name")
	if len(r.URL.Query().Get("id")) > 0 {
		hash = r.URL.Query().Get("id")
		if tmp := strings.Split(hash, "."); len(tmp) > 1 {
			hash = tmp[1]
		}
	}
	if len(hash) == 0 && len(name) == 0 {
		io.WriteString(w, "Please specify hash or name")
		return
	} else if len(name) != 0 {
		hash = db.LastHash(name, repo)
	}

	if len(db.Read(hash)) > 0 && !db.Public(hash) && !db.CheckShare(hash, db.CheckToken(r.URL.Query().Get("token"))) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("Not found"))
		return
	}

	// if len(db.Read(hash)) == 0 && repo == "template" && !torrent.IsDownloaded(hash) {
	// 	torrent.AddTorrent(hash)
	// 	w.WriteHeader(http.StatusAccepted)
	// 	w.Write([]byte(torrent.Info(hash)))
	// 	return
	// }

	f, err := os.Open(config.Storage.Path + hash)
	defer f.Close()

	if log.Check(log.WarnLevel, "Opening file "+config.Storage.Path+hash, err) || len(hash) == 0 {
		if len(config.CDN.Node) > 0 {
			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
			resp, err := client.Get(config.CDN.Node + r.URL.RequestURI())
			if !log.Check(log.WarnLevel, "Getting file from CDN", err) {
				w.Header().Set("Content-Length", resp.Header.Get("Content-Length"))
				w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
				w.Header().Set("Last-Modified", resp.Header.Get("Last-Modified"))
				w.Header().Set("Content-Disposition", resp.Header.Get("Content-Disposition"))

				io.Copy(w, resp.Body)
				resp.Body.Close()
				return
			}
		}

		w.WriteHeader(http.StatusNotFound)
		io.WriteString(w, "File not found")
		return
	}
	fi, _ := f.Stat()

	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && fi.ModTime().Unix() <= t.Unix() {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	w.Header().Set("Content-Length", fmt.Sprint(fi.Size()))
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))

	if name = db.Read(hash); len(name) == 0 && len(config.CDN.Node) > 0 {
		httpclient := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
		resp, err := httpclient.Get(config.CDN.Node + "/kurjun/rest/template/info?id=" + hash)
		if !log.Check(log.WarnLevel, "Getting info from CDN", err) {
			var info ListItem
			rsp, err := ioutil.ReadAll(resp.Body)
			if log.Check(log.WarnLevel, "Reading from CDN response", err) {
				w.WriteHeader(http.StatusNotFound)
				io.WriteString(w, "File not found")
				return
			}
			if !log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &info)) {
				w.Header().Set("Content-Disposition", "attachment; filename=\""+info.Filename+"\"")
			}
			resp.Body.Close()
		}
	} else {
		w.Header().Set("Content-Disposition", "attachment; filename=\""+db.Read(hash)+"\"")
	}

	io.Copy(w, f)
}

func Info(repo string, r *http.Request) []byte {
	var items []ListItem
	var info map[string]string
	p := []int{0, 1000}

	id := r.URL.Query().Get("id")
	name := r.URL.Query().Get("name")
	page := r.URL.Query().Get("page")
	owner := r.URL.Query().Get("owner")
	token := r.URL.Query().Get("token")
	version := r.URL.Query().Get("version")
	verified := r.URL.Query().Get("verified")

	list := db.Search(name)
	if len(id) > 0 {
		list = append(list[:0], id)
	} else if verified == "true" {
		items := append(items, getVerified(list, name, repo))
		output, err := json.Marshal(items)
		if err != nil || string(output) == "null" {
			return nil
		}
		return output
	}

	pstr := strings.Split(page, ",")
	p[0], _ = strconv.Atoi(pstr[0])
	if len(pstr) == 2 {
		p[1], _ = strconv.Atoi(pstr[1])
	}

	for _, k := range list {
		if (!db.Public(k) && !db.CheckShare(k, db.CheckToken(token))) ||
			(len(owner) > 0 && db.CheckRepo(owner, repo, k) == 0) ||
			db.CheckRepo("", repo, k) == 0 {
			continue
		}

		if len(items) < p[0] {
			continue
		}

		if name == "management" && repo == "template" {
			info = db.LatestTmpl(name, version)
			if len(info["name"]) == 0 {
				continue
			}
		} else {
			info = db.Info(k)
		}

		item := formatItem(info, repo, name)

		if strings.HasPrefix(info["name"], name+"-subutai-template") || name == info["name"] {
			if (len(version) == 0 || strings.Contains(info["version"], version)) && k == db.LastHash(info["name"], repo) {
				items = []ListItem{item}
				break
			}
			continue
		}

		if len(items) == p[0]+p[1] {
			break
		}
		items = append(items, item)
	}
	output, err := json.Marshal(items)
	if err != nil || string(output) == "null" {
		return nil
	}
	return output
}

// ProxyList retrieves list of artifacts from main CDN nodes if no data found in local database
// It creates simple JSON list of artifacts to provide it to Subutai Social.
func ProxyList(t string) []byte {
	if len(config.CDN.Node) == 0 {
		return nil
	}
	list := make([]ListItem, 0)

	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
	resp, err := client.Get(config.CDN.Node + "/kurjun/rest/" + t + "/list")
	defer resp.Body.Close()
	if log.Check(log.WarnLevel, "Getting list from CDN", err) {
		return nil
	}

	rsp, err := ioutil.ReadAll(resp.Body)
	if log.Check(log.WarnLevel, "Reading from CDN response", err) {
		return nil
	}

	if log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &list)) {
		return nil
	}

	output, err := json.Marshal(list)
	if log.Check(log.WarnLevel, "Marshaling list", err) {
		return nil
	}
	return output
}

// ProxyInfo retrieves information from main CDN nodes if no data found in local database
// It creates simple info JSON to provide it to Subutai Social.
func ProxyInfo(uri string) []byte {
	if len(config.CDN.Node) == 0 {
		return nil
	}
	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
	resp, err := client.Get(config.CDN.Node + uri)
	defer resp.Body.Close()
	if log.Check(log.WarnLevel, "Getting list of templates from CDN", err) {
		return nil
	}

	rsp, err := ioutil.ReadAll(resp.Body)
	if log.Check(log.WarnLevel, "Reading from CDN response", err) {
		return nil
	}
	return rsp
}

func in(str string, list []string) bool {
	for _, s := range list {
		if s == str {
			return true
		}
	}
	return false
}

func getVerified(list []string, name, repo string) ListItem {
	for _, k := range list {
		if info := db.Info(k); db.CheckRepo("", repo, k) > 0 {
			if info["name"] == name || (strings.HasPrefix(info["name"], name+"-subutai-template") && repo == "template") {
				for _, owner := range db.FileOwner(info["id"]) {
					if in(owner, []string{"subutai", "jenkins", "docker"}) {
						return formatItem(info, repo, name)
					}
				}
			}
		}
	}
	return ListItem{}
}

func formatItem(info map[string]string, repo, name string) ListItem {
	if len(info["prefsize"]) == 0 && repo == "template" {
		info["prefsize"] = "tiny"
	}
	item := ListItem{
		ID:           info["id"],
		Name:         strings.Split(info["name"], "-subutai-template")[0],
		Size:         info["size"],
		Owner:        db.FileOwner(info["id"]),
		Version:      info["version"],
		Filename:     info["name"],
		Parent:       info["parent"],
		Prefsize:     info["prefsize"],
		Architecture: strings.ToUpper(info["arch"]),
		Signature:    db.FileSignatures(info["id"], name),
		Description:  info["Description"],
	}

	if repo == "apt" {
		item.Architecture = info["Architecture"]
	}

	return item
}
<commit_msg>Changes based on static code analyzer<commit_after>package download

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/subutai-io/agent/log"
	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
)

// ListItem describes Gorjun entity. It can be APT package, Subutai template or Raw file.
type ListItem struct {
	ID           string            `json:"id"`
	Size         string            `json:"size,omitempty"`
	Name         string            `json:"name,omitempty"`
	Owner        []string          `json:"owner,omitempty"`
	Parent       string            `json:"parent,omitempty"`
	Version      string            `json:"version,omitempty"`
	Filename     string            `json:"filename,omitempty"`
	Prefsize     string            `json:"prefsize,omitempty"`
	Signature    map[string]string `json:"signature,omitempty"`
	Description  string            `json:"description,omitempty"`
	Architecture string            `json:"architecture,omitempty"`
}

// Handler provides download functionality for all artifacts.
func Handler(repo string, w http.ResponseWriter, r *http.Request) {
	hash := r.URL.Query().Get("hash")
	name := r.URL.Query().Get("name")
	if len(r.URL.Query().Get("id")) > 0 {
		hash = r.URL.Query().Get("id")
		if tmp := strings.Split(hash, "."); len(tmp) > 1 {
			hash = tmp[1]
		}
	}
	if len(hash) == 0 && len(name) == 0 {
		io.WriteString(w, "Please specify hash or name")
		return
	} else if len(name) != 0 {
		hash = db.LastHash(name, repo)
	}

	if len(db.Read(hash)) > 0 && !db.Public(hash) && !db.CheckShare(hash, db.CheckToken(r.URL.Query().Get("token"))) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("Not found"))
		return
	}

	// if len(db.Read(hash)) == 0 && repo == "template" && !torrent.IsDownloaded(hash) {
	// 	torrent.AddTorrent(hash)
	// 	w.WriteHeader(http.StatusAccepted)
	// 	w.Write([]byte(torrent.Info(hash)))
	// 	return
	// }

	f, err := os.Open(config.Storage.Path + hash)
	defer f.Close()

	if log.Check(log.WarnLevel, "Opening file "+config.Storage.Path+hash, err) || len(hash) == 0 {
		if len(config.CDN.Node) > 0 {
			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
			resp, err := client.Get(config.CDN.Node + r.URL.RequestURI())
			if !log.Check(log.WarnLevel, "Getting file from CDN", err) {
				w.Header().Set("Content-Length", resp.Header.Get("Content-Length"))
				w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
				w.Header().Set("Last-Modified", resp.Header.Get("Last-Modified"))
				w.Header().Set("Content-Disposition", resp.Header.Get("Content-Disposition"))

				io.Copy(w, resp.Body)
				resp.Body.Close()
				return
			}
		}

		w.WriteHeader(http.StatusNotFound)
		io.WriteString(w, "File not found")
		return
	}
	fi, _ := f.Stat()

	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && fi.ModTime().Unix() <= t.Unix() {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	w.Header().Set("Content-Length", fmt.Sprint(fi.Size()))
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))

	if name = db.Read(hash); len(name) == 0 && len(config.CDN.Node) > 0 {
		httpclient := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
		resp, err := httpclient.Get(config.CDN.Node + "/kurjun/rest/template/info?id=" + hash)
		if !log.Check(log.WarnLevel, "Getting info from CDN", err) {
			var info ListItem
			rsp, err := ioutil.ReadAll(resp.Body)
			if log.Check(log.WarnLevel, "Reading from CDN response", err) {
				w.WriteHeader(http.StatusNotFound)
				io.WriteString(w, "File not found")
				return
			}
			if !log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &info)) {
				w.Header().Set("Content-Disposition", "attachment; filename=\""+info.Filename+"\"")
			}
			resp.Body.Close()
		}
	} else {
		w.Header().Set("Content-Disposition", "attachment; filename=\""+db.Read(hash)+"\"")
	}

	io.Copy(w, f)
}

// Info returns JSON formatted list of elements. It allows to apply some filters to Search.
func Info(repo string, r *http.Request) []byte {
	var items []ListItem
	var info map[string]string
	p := []int{0, 1000}

	id := r.URL.Query().Get("id")
	name := r.URL.Query().Get("name")
	page := r.URL.Query().Get("page")
	owner := r.URL.Query().Get("owner")
	token := r.URL.Query().Get("token")
	version := r.URL.Query().Get("version")
	verified := r.URL.Query().Get("verified")

	list := db.Search(name)
	if len(id) > 0 {
		list = append(list[:0], id)
	} else if verified == "true" {
		items := append(items, getVerified(list, name, repo))
		output, err := json.Marshal(items)
		if err != nil || string(output) == "null" {
			return nil
		}
		return output
	}

	pstr := strings.Split(page, ",")
	p[0], _ = strconv.Atoi(pstr[0])
	if len(pstr) == 2 {
		p[1], _ = strconv.Atoi(pstr[1])
	}

	for _, k := range list {
		if (!db.Public(k) && !db.CheckShare(k, db.CheckToken(token))) ||
			(len(owner) > 0 && db.CheckRepo(owner, repo, k) == 0) ||
			db.CheckRepo("", repo, k) == 0 {
			continue
		}

		if len(items) < p[0] {
			continue
		}

		if name == "management" && repo == "template" {
			info = db.LatestTmpl(name, version)
			if len(info["name"]) == 0 {
				continue
			}
		} else {
			info = db.Info(k)
		}

		item := formatItem(info, repo, name)

		if strings.HasPrefix(info["name"], name+"-subutai-template") || name == info["name"] {
			if (len(version) == 0 || strings.Contains(info["version"], version)) && k == db.LastHash(info["name"], repo) {
				items = []ListItem{item}
				break
			}
			continue
		}

		if len(items) == p[0]+p[1] {
			break
		}
		items = append(items, item)
	}
	output, err := json.Marshal(items)
	if err != nil || string(output) == "null" {
		return nil
	}
	return output
}

// ProxyList retrieves list of artifacts from main CDN nodes if no data found in local database
// It creates simple JSON list of artifacts to provide it to Subutai Social.
func ProxyList(t string) []byte {
	if len(config.CDN.Node) == 0 {
		return nil
	}
	list := make([]ListItem, 0)

	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
	resp, err := client.Get(config.CDN.Node + "/kurjun/rest/" + t + "/list")
	if log.Check(log.WarnLevel, "Getting list from CDN", err) {
		return nil
	}
	defer resp.Body.Close()

	rsp, err := ioutil.ReadAll(resp.Body)
	if log.Check(log.WarnLevel, "Reading from CDN response", err) {
		return nil
	}

	if log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &list)) {
		return nil
	}

	output, err := json.Marshal(list)
	if log.Check(log.WarnLevel, "Marshaling list", err) {
		return nil
	}
	return output
}

// ProxyInfo retrieves information from main CDN nodes if no data found in local database
// It creates simple info JSON to provide it to Subutai Social.
func ProxyInfo(uri string) []byte {
	if len(config.CDN.Node) == 0 {
		return nil
	}
	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
	resp, err := client.Get(config.CDN.Node + uri)
	if log.Check(log.WarnLevel, "Getting list of templates from CDN", err) {
		return nil
	}
	defer resp.Body.Close()

	rsp, err := ioutil.ReadAll(resp.Body)
	if log.Check(log.WarnLevel, "Reading from CDN response", err) {
		return nil
	}
	return rsp
}

func in(str string, list []string) bool {
	for _, s := range list {
		if s == str {
			return true
		}
	}
	return false
}

func getVerified(list []string, name, repo string) ListItem {
	for _, k := range list {
		if info := db.Info(k); db.CheckRepo("", repo, k) > 0 {
			if info["name"] == name || (strings.HasPrefix(info["name"], name+"-subutai-template") && repo == "template") {
				for _, owner := range db.FileOwner(info["id"]) {
					if in(owner, []string{"subutai", "jenkins", "docker"}) {
						return formatItem(info, repo, name)
					}
				}
			}
		}
	}
	return ListItem{}
}

func formatItem(info map[string]string, repo, name string) ListItem {
	if len(info["prefsize"]) == 0 && repo == "template" {
		info["prefsize"] = "tiny"
	}
	item := ListItem{
		ID:           info["id"],
		Name:         strings.Split(info["name"], "-subutai-template")[0],
		Size:         info["size"],
		Owner:        db.FileOwner(info["id"]),
		Version:      info["version"],
		Filename:     info["name"],
		Parent:       info["parent"],
		Prefsize:     info["prefsize"],
		Architecture: strings.ToUpper(info["arch"]),
		Signature:    db.FileSignatures(info["id"], name),
		Description:  info["Description"],
	}

	if repo == "apt" {
		item.Architecture = info["Architecture"]
	}

	return item
}
<|endoftext|>