<commit_before>package consul

import (
	"fmt"
	"time"

	"github.com/armon/go-metrics"
	"github.com/hashicorp/consul/consul/structs"
)

// KVS endpoint is used to manipulate the Key-Value store
type KVS struct {
	srv *Server
}

// Apply is used to apply a KVS request to the data store. This should
// only be used for operations that modify the data
func (k *KVS) Apply(args *structs.KVSRequest, reply *bool) error {
	if done, err := k.srv.forward("KVS.Apply", args, args, reply); done {
		return err
	}
	defer metrics.MeasureSince([]string{"consul", "kvs", "apply"}, time.Now())

	// Verify the args
	if args.DirEnt.Key == "" && args.Op != structs.KVSDeleteTree {
		return fmt.Errorf("Must provide key")
	}

	// Apply the ACL policy if any
	acl, err := k.srv.resolveToken(args.Token)
	if err != nil {
		return err
	} else if acl != nil {
		switch args.Op {
		case structs.KVSDeleteTree:
			if !acl.KeyWritePrefix(args.DirEnt.Key) {
				return permissionDeniedErr
			}
		default:
			if !acl.KeyWrite(args.DirEnt.Key) {
				return permissionDeniedErr
			}
		}
	}

	// If this is a lock, we must check for a lock-delay. Since lock-delay
	// is based on wall-time, each peer expire the lock-delay at a slightly
	// different time. This means the enforcement of lock-delay cannot be done
	// after the raft log is committed as it would lead to inconsistent FSMs.
	// Instead, the lock-delay must be enforced before commit. This means that
	// only the wall-time of the leader node is used, preventing any inconsistencies.
	if args.Op == structs.KVSLock {
		state := k.srv.fsm.State()
		expires := state.KVSLockDelay(args.DirEnt.Key)
		if expires.After(time.Now()) {
			k.srv.logger.Printf("[WARN] consul.kvs: Rejecting lock of %s due to lock-delay until %v",
				args.DirEnt.Key, expires)
			*reply = false
			return nil
		}
	}

	// Apply the update
	resp, err := k.srv.raftApply(structs.KVSRequestType, args)
	if err != nil {
		k.srv.logger.Printf("[ERR] consul.kvs: Apply failed: %v", err)
		return err
	}
	if respErr, ok := resp.(error); ok {
		return respErr
	}

	// Check if the return type is a bool
	if respBool, ok := resp.(bool); ok {
		*reply = respBool
	}
	return nil
}

// Get is used to lookup a single key
func (k *KVS) Get(args *structs.KeyRequest, reply *structs.IndexedDirEntries) error {
	if done, err := k.srv.forward("KVS.Get", args, args, reply); done {
		return err
	}

	acl, err := k.srv.resolveToken(args.Token)
	if err != nil {
		return err
	}

	// Get the local state
	state := k.srv.fsm.State()
	opts := blockingRPCOptions{
		queryOpts: &args.QueryOptions,
		queryMeta: &reply.QueryMeta,
		tables:    nil,
		kvWatch:   true,
		kvPrefix:  args.Key,
		run: func() error {
			index, ent, err := state.KVSGet(args.Key)
			if err != nil {
				return err
			}
			if acl != nil && !acl.KeyRead(args.Key) {
				ent = nil
			}
			if ent == nil {
				// Must provide non-zero index to prevent blocking
				// Index 1 is impossible anyways (due to Raft internals)
				if index == 0 {
					reply.Index = 1
				} else {
					reply.Index = index
				}
				reply.Entries = nil
			} else {
				reply.Index = ent.ModifyIndex
				reply.Entries = structs.DirEntries{ent}
			}
			return nil
		},
	}
	return k.srv.blockingRPCOpt(&opts)
}

// List is used to list all keys with a given prefix
func (k *KVS) List(args *structs.KeyRequest, reply *structs.IndexedDirEntries) error {
	if done, err := k.srv.forward("KVS.List", args, args, reply); done {
		return err
	}

	acl, err := k.srv.resolveToken(args.Token)
	if err != nil {
		return err
	}

	// Get the local state
	state := k.srv.fsm.State()
	opts := blockingRPCOptions{
		queryOpts: &args.QueryOptions,
		queryMeta: &reply.QueryMeta,
		tables:    nil,
		kvWatch:   true,
		kvPrefix:  args.Key,
		run: func() error {
			tombIndex, index, ent, err := state.KVSList(args.Key)
			if err != nil {
				return err
			}
			if acl != nil {
				ent = FilterDirEnt(acl, ent)
			}

			if len(ent) == 0 {
				// Must provide non-zero index to prevent blocking
				// Index 1 is impossible anyways (due to Raft internals)
				if index == 0 {
					reply.Index = 1
				} else {
					reply.Index = index
				}
			} else {
				// Determine the maximum affected index
				var maxIndex uint64
				for _, e := range ent {
					if e.ModifyIndex > maxIndex {
						maxIndex = e.ModifyIndex
					}
				}
				if tombIndex > maxIndex {
					maxIndex = tombIndex
				}
				reply.Index = maxIndex
				reply.Entries = ent
			}
			return nil
		},
	}
	return k.srv.blockingRPCOpt(&opts)
}

// ListKeys is used to list all keys with a given prefix to a seperator
func (k *KVS) ListKeys(args *structs.KeyListRequest, reply *structs.IndexedKeyList) error {
	if done, err := k.srv.forward("KVS.ListKeys", args, args, reply); done {
		return err
	}

	acl, err := k.srv.resolveToken(args.Token)
	if err != nil {
		return err
	}

	// Get the local state
	state := k.srv.fsm.State()
	opts := blockingRPCOptions{
		queryOpts: &args.QueryOptions,
		queryMeta: &reply.QueryMeta,
		tables:    nil,
		kvWatch:   true,
		kvPrefix:  args.Prefix,
		run: func() error {
			index, keys, err := state.KVSListKeys(args.Prefix, args.Seperator)
			reply.Index = index
			if acl != nil {
				keys = FilterKeys(acl, keys)
			}
			reply.Keys = keys
			return err

		},
	}
	return k.srv.blockingRPCOpt(&opts)
}
<commit_msg>consul: Remove dead code<commit_after>package consul

import (
	"fmt"
	"time"

	"github.com/armon/go-metrics"
	"github.com/hashicorp/consul/consul/structs"
)

// KVS endpoint is used to manipulate the Key-Value store
type KVS struct {
	srv *Server
}

// Apply is used to apply a KVS request to the data store. This should
// only be used for operations that modify the data
func (k *KVS) Apply(args *structs.KVSRequest, reply *bool) error {
	if done, err := k.srv.forward("KVS.Apply", args, args, reply); done {
		return err
	}
	defer metrics.MeasureSince([]string{"consul", "kvs", "apply"}, time.Now())

	// Verify the args
	if args.DirEnt.Key == "" && args.Op != structs.KVSDeleteTree {
		return fmt.Errorf("Must provide key")
	}

	// Apply the ACL policy if any
	acl, err := k.srv.resolveToken(args.Token)
	if err != nil {
		return err
	} else if acl != nil {
		switch args.Op {
		case structs.KVSDeleteTree:
			if !acl.KeyWritePrefix(args.DirEnt.Key) {
				return permissionDeniedErr
			}
		default:
			if !acl.KeyWrite(args.DirEnt.Key) {
				return permissionDeniedErr
			}
		}
	}

	// If this is a lock, we must check for a lock-delay. Since lock-delay
	// is based on wall-time, each peer expire the lock-delay at a slightly
	// different time. This means the enforcement of lock-delay cannot be done
	// after the raft log is committed as it would lead to inconsistent FSMs.
	// Instead, the lock-delay must be enforced before commit. This means that
	// only the wall-time of the leader node is used, preventing any inconsistencies.
	if args.Op == structs.KVSLock {
		state := k.srv.fsm.State()
		expires := state.KVSLockDelay(args.DirEnt.Key)
		if expires.After(time.Now()) {
			k.srv.logger.Printf("[WARN] consul.kvs: Rejecting lock of %s due to lock-delay until %v",
				args.DirEnt.Key, expires)
			*reply = false
			return nil
		}
	}

	// Apply the update
	resp, err := k.srv.raftApply(structs.KVSRequestType, args)
	if err != nil {
		k.srv.logger.Printf("[ERR] consul.kvs: Apply failed: %v", err)
		return err
	}
	if respErr, ok := resp.(error); ok {
		return respErr
	}

	// Check if the return type is a bool
	if respBool, ok := resp.(bool); ok {
		*reply = respBool
	}
	return nil
}

// Get is used to lookup a single key
func (k *KVS) Get(args *structs.KeyRequest, reply *structs.IndexedDirEntries) error {
	if done, err := k.srv.forward("KVS.Get", args, args, reply); done {
		return err
	}

	acl, err := k.srv.resolveToken(args.Token)
	if err != nil {
		return err
	}

	// Get the local state
	state := k.srv.fsm.State()
	opts := blockingRPCOptions{
		queryOpts: &args.QueryOptions,
		queryMeta: &reply.QueryMeta,
		kvWatch:   true,
		kvPrefix:  args.Key,
		run: func() error {
			index, ent, err := state.KVSGet(args.Key)
			if err != nil {
				return err
			}
			if acl != nil && !acl.KeyRead(args.Key) {
				ent = nil
			}
			if ent == nil {
				// Must provide non-zero index to prevent blocking
				// Index 1 is impossible anyways (due to Raft internals)
				if index == 0 {
					reply.Index = 1
				} else {
					reply.Index = index
				}
				reply.Entries = nil
			} else {
				reply.Index = ent.ModifyIndex
				reply.Entries = structs.DirEntries{ent}
			}
			return nil
		},
	}
	return k.srv.blockingRPCOpt(&opts)
}

// List is used to list all keys with a given prefix
func (k *KVS) List(args *structs.KeyRequest, reply *structs.IndexedDirEntries) error {
	if done, err := k.srv.forward("KVS.List", args, args, reply); done {
		return err
	}

	acl, err := k.srv.resolveToken(args.Token)
	if err != nil {
		return err
	}

	// Get the local state
	state := k.srv.fsm.State()
	opts := blockingRPCOptions{
		queryOpts: &args.QueryOptions,
		queryMeta: &reply.QueryMeta,
		kvWatch:   true,
		kvPrefix:  args.Key,
		run: func() error {
			tombIndex, index, ent, err := state.KVSList(args.Key)
			if err != nil {
				return err
			}
			if acl != nil {
				ent = FilterDirEnt(acl, ent)
			}

			if len(ent) == 0 {
				// Must provide non-zero index to prevent blocking
				// Index 1 is impossible anyways (due to Raft internals)
				if index == 0 {
					reply.Index = 1
				} else {
					reply.Index = index
				}
			} else {
				// Determine the maximum affected index
				var maxIndex uint64
				for _, e := range ent {
					if e.ModifyIndex > maxIndex {
						maxIndex = e.ModifyIndex
					}
				}
				if tombIndex > maxIndex {
					maxIndex = tombIndex
				}
				reply.Index = maxIndex
				reply.Entries = ent
			}
			return nil
		},
	}
	return k.srv.blockingRPCOpt(&opts)
}

// ListKeys is used to list all keys with a given prefix to a seperator
func (k *KVS) ListKeys(args *structs.KeyListRequest, reply *structs.IndexedKeyList) error {
	if done, err := k.srv.forward("KVS.ListKeys", args, args, reply); done {
		return err
	}

	acl, err := k.srv.resolveToken(args.Token)
	if err != nil {
		return err
	}

	// Get the local state
	state := k.srv.fsm.State()
	opts := blockingRPCOptions{
		queryOpts: &args.QueryOptions,
		queryMeta: &reply.QueryMeta,
		kvWatch:   true,
		kvPrefix:  args.Prefix,
		run: func() error {
			index, keys, err := state.KVSListKeys(args.Prefix, args.Seperator)
			reply.Index = index
			if acl != nil {
				keys = FilterKeys(acl, keys)
			}
			reply.Keys = keys
			return err

		},
	}
	return k.srv.blockingRPCOpt(&opts)
}
<|endoftext|>


<commit_before>package backends

import (
	"encoding/json"
	"fmt"
	"github.com/docker/libswarm/beam"
	"github.com/dotcloud/docker/api"
	"github.com/dotcloud/docker/pkg/version"
	"github.com/gorilla/mux"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
	"time"
)

func DockerServer() beam.Sender {
	backend := beam.NewServer()
	backend.OnSpawn(beam.Handler(func(ctx *beam.Message) error {
		instance := beam.Task(func(in beam.Receiver, out beam.Sender) {
			url := "tcp://localhost:4243"
			if len(ctx.Args) > 0 {
				url = ctx.Args[0]
			}
			err := listenAndServe(url, out)
			if err != nil {
				fmt.Printf("listenAndServe: %v", err)
			}
		})
		_, err := ctx.Ret.Send(&beam.Message{Verb: beam.Ack, Ret: instance})
		return err
	}))
	return backend
}

type HttpApiFunc func(out beam.Sender, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error

func listenAndServe(url string, out beam.Sender) error {
	fmt.Println("Starting Docker server...")
	r, err := createRouter(out)
	if err != nil {
		return err
	}
	arr := strings.Split(url, "://")
	proto := arr[0]
	addr := arr[1]
	l, err := net.Listen(proto, addr)
	if err != nil {
		return err
	}
	httpSrv := http.Server{Addr: addr, Handler: r}
	return httpSrv.Serve(l)
}

func ping(out beam.Sender, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	_, err := w.Write([]byte{'O', 'K'})
	return err
}

func getContainersJSON(out beam.Sender, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := r.ParseForm(); err != nil {
		return err
	}

	o := beam.Obj(out)
	names, err := o.Ls()
	if err != nil {
		return err
	}

	var responses []interface{}

	for _, name := range names {
		_, containerOut, err := o.Attach(name)
		if err != nil {
			return err
		}
		container := beam.Obj(containerOut)
		responseJson, err := container.Get()
		if err != nil {
			return err
		}
		var response struct {
			ID      string
			Created string
			Name    string
			Config  struct {
				Cmd   []string
				Image string
			}
			State struct {
				Running    bool
				StartedAt  string
				FinishedAt string
				ExitCode   int
			}
		}
		if err = json.Unmarshal([]byte(responseJson), &response); err != nil {
			return err
		}
		created, err := time.Parse(time.RFC3339, response.Created)
		if err != nil {
			return err
		}
		var state string
		if response.State.Running {
			state = "Up"
		} else {
			state = fmt.Sprintf("Exited (%d)", response.State.ExitCode)
		}
		responses = append(responses, map[string]interface{}{
			"Id":      response.ID,
			"Command": strings.Join(response.Config.Cmd, " "),
			"Created": created.Unix(),
			"Image":   response.Config.Image,
			"Names":   []string{response.Name},
			"Ports":   []string{},
			"Status":  state,
		})
	}

	return writeJSON(w, http.StatusOK, responses)
}

func postContainersCreate(out beam.Sender, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := r.ParseForm(); err != nil {
		return nil
	}

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return err
	}

	container, err := beam.Obj(out).Spawn(string(body))
	if err != nil {
		return err
	}

	responseJson, err := container.Get()
	if err != nil {
		return err
	}

	var response struct{ Id string }
	if err = json.Unmarshal([]byte(responseJson), &response); err != nil {
		return err
	}
	return writeJSON(w, http.StatusCreated, response)
}

func postContainersStart(out beam.Sender, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if vars == nil {
		return fmt.Errorf("Missing parameter")
	}

	// TODO: r.Body

	name := vars["name"]
	_, containerOut, err := beam.Obj(out).Attach(name)
	container := beam.Obj(containerOut)
	if err != nil {
		return err
	}
	if err := container.Start(); err != nil {
		return err
	}

	w.WriteHeader(http.StatusNoContent)
	return nil
}

func postContainersStop(out beam.Sender, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if vars == nil {
		return fmt.Errorf("Missing parameter")
	}

	name := vars["name"]
	_, containerOut, err := beam.Obj(out).Attach(name)
	container := beam.Obj(containerOut)
	if err != nil {
		return err
	}
	if err := container.Stop(); err != nil {
		return err
	}

	w.WriteHeader(http.StatusNoContent)
	return nil
}

func createRouter(out beam.Sender) (*mux.Router, error) {
	r := mux.NewRouter()
	m := map[string]map[string]HttpApiFunc{
		"GET": {
			"/_ping":           ping,
			"/containers/json": getContainersJSON,
		},
		"POST": {
			"/containers/create":          postContainersCreate,
			"/containers/{name:.*}/start": postContainersStart,
			"/containers/{name:.*}/stop":  postContainersStop,
		},
		"DELETE":  {},
		"OPTIONS": {},
	}

	for method, routes := range m {
		for route, fct := range routes {
			localRoute := route
			localFct := fct
			localMethod := method

			f := makeHttpHandler(out, localMethod, localRoute, localFct, version.Version("0.11.0"))

			// add the new route
			if localRoute == "" {
				r.Methods(localMethod).HandlerFunc(f)
			} else {
				r.Path("/v{version:[0-9.]+}" + localRoute).Methods(localMethod).HandlerFunc(f)
				r.Path(localRoute).Methods(localMethod).HandlerFunc(f)
			}
		}
	}

	return r, nil
}

func makeHttpHandler(out beam.Sender, localMethod string, localRoute string, handlerFunc HttpApiFunc, dockerVersion version.Version) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		// log the request
		fmt.Printf("Calling %s %s\n", localMethod, localRoute)

		version := version.Version(mux.Vars(r)["version"])
		if version == "" {
			version = api.APIVERSION
		}

		if err := handlerFunc(out, version, w, r, mux.Vars(r)); err != nil {
			fmt.Printf("Error: %s", err)
		}
	}
}

func writeJSON(w http.ResponseWriter, code int, v interface{}) error {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(code)
	enc := json.NewEncoder(w)
	return enc.Encode(v)
}
<commit_msg>Parse the URL with the net/url Parse() function.<commit_after>package backends

import (
	"encoding/json"
	"fmt"
	"github.com/docker/libswarm/beam"
	"github.com/dotcloud/docker/api"
	"github.com/dotcloud/docker/pkg/version"
	"github.com/gorilla/mux"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"
)

func DockerServer() beam.Sender {
	backend := beam.NewServer()
	backend.OnSpawn(beam.Handler(func(ctx *beam.Message) error {
		instance := beam.Task(func(in beam.Receiver, out beam.Sender) {
			url := "tcp://localhost:4243"
			if len(ctx.Args) > 0 {
				url = ctx.Args[0]
			}
			err := listenAndServe(url, out)
			if err != nil {
				fmt.Printf("listenAndServe: %v", err)
			}
		})
		_, err := ctx.Ret.Send(&beam.Message{Verb: beam.Ack, Ret: instance})
		return err
	}))
	return backend
}

type HttpApiFunc func(out beam.Sender, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error

func listenAndServe(urlStr string, out beam.Sender) error {
	fmt.Println("Starting Docker server...")
	r, err := createRouter(out)
	if err != nil {
		return err
	}

	parsedUrl, err := url.Parse(urlStr)
	if err != nil {
		return err
	}

	l, err := net.Listen(parsedUrl.Scheme, parsedUrl.Host)
	if err != nil {
		return err
	}
	httpSrv := http.Server{Addr: parsedUrl.Host, Handler: r}
	return httpSrv.Serve(l)
}

func ping(out beam.Sender, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	_, err := w.Write([]byte{'O', 'K'})
	return err
}

func getContainersJSON(out beam.Sender, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := r.ParseForm(); err != nil {
		return err
	}

	o := beam.Obj(out)
	names, err := o.Ls()
	if err != nil {
		return err
	}

	var responses []interface{}

	for _, name := range names {
		_, containerOut, err := o.Attach(name)
		if err != nil {
			return err
		}
		container := beam.Obj(containerOut)
		responseJson, err := container.Get()
		if err != nil {
			return err
		}
		var response struct {
			ID      string
			Created string
			Name    string
			Config  struct {
				Cmd   []string
				Image string
			}
			State struct {
				Running    bool
				StartedAt  string
				FinishedAt string
				ExitCode   int
			}
		}
		if err = json.Unmarshal([]byte(responseJson), &response); err != nil {
			return err
		}
		created, err := time.Parse(time.RFC3339, response.Created)
		if err != nil {
			return err
		}
		var state string
		if response.State.Running {
			state = "Up"
		} else {
			state = fmt.Sprintf("Exited (%d)", response.State.ExitCode)
		}
		responses = append(responses, map[string]interface{}{
			"Id":      response.ID,
			"Command": strings.Join(response.Config.Cmd, " "),
			"Created": created.Unix(),
			"Image":   response.Config.Image,
			"Names":   []string{response.Name},
			"Ports":   []string{},
			"Status":  state,
		})
	}

	return writeJSON(w, http.StatusOK, responses)
}

func postContainersCreate(out beam.Sender, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := r.ParseForm(); err != nil {
		return nil
	}

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return err
	}

	container, err := beam.Obj(out).Spawn(string(body))
	if err != nil {
		return err
	}

	responseJson, err := container.Get()
	if err != nil {
		return err
	}

	var response struct{ Id string }
	if err = json.Unmarshal([]byte(responseJson), &response); err != nil {
		return err
	}
	return writeJSON(w, http.StatusCreated, response)
}

func postContainersStart(out beam.Sender, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if vars == nil {
		return fmt.Errorf("Missing parameter")
	}

	// TODO: r.Body

	name := vars["name"]
	_, containerOut, err := beam.Obj(out).Attach(name)
	container := beam.Obj(containerOut)
	if err != nil {
		return err
	}
	if err := container.Start(); err != nil {
		return err
	}

	w.WriteHeader(http.StatusNoContent)
	return nil
}

func postContainersStop(out beam.Sender, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if vars == nil {
		return fmt.Errorf("Missing parameter")
	}

	name := vars["name"]
	_, containerOut, err := beam.Obj(out).Attach(name)
	container := beam.Obj(containerOut)
	if err != nil {
		return err
	}
	if err := container.Stop(); err != nil {
		return err
	}

	w.WriteHeader(http.StatusNoContent)
	return nil
}

func createRouter(out beam.Sender) (*mux.Router, error) {
	r := mux.NewRouter()
	m := map[string]map[string]HttpApiFunc{
		"GET": {
			"/_ping":           ping,
			"/containers/json": getContainersJSON,
		},
		"POST": {
			"/containers/create":          postContainersCreate,
			"/containers/{name:.*}/start": postContainersStart,
			"/containers/{name:.*}/stop":  postContainersStop,
		},
		"DELETE":  {},
		"OPTIONS": {},
	}

	for method, routes := range m {
		for route, fct := range routes {
			localRoute := route
			localFct := fct
			localMethod := method

			f := makeHttpHandler(out, localMethod, localRoute, localFct, version.Version("0.11.0"))

			// add the new route
			if localRoute == "" {
				r.Methods(localMethod).HandlerFunc(f)
			} else {
				r.Path("/v{version:[0-9.]+}" + localRoute).Methods(localMethod).HandlerFunc(f)
				r.Path(localRoute).Methods(localMethod).HandlerFunc(f)
			}
		}
	}

	return r, nil
}

func makeHttpHandler(out beam.Sender, localMethod string, localRoute string, handlerFunc HttpApiFunc, dockerVersion version.Version) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		// log the request
		fmt.Printf("Calling %s %s\n", localMethod, localRoute)

		version := version.Version(mux.Vars(r)["version"])
		if version == "" {
			version = api.APIVERSION
		}

		if err := handlerFunc(out, version, w, r, mux.Vars(r)); err != nil {
			fmt.Printf("Error: %s", err)
		}
	}
}

func writeJSON(w http.ResponseWriter, code int, v interface{}) error {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(code)
	enc := json.NewEncoder(w)
	return enc.Encode(v)
}
<|endoftext|>


<commit_before>package httputils

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"runtime"
	"strconv"
	"time"

	// Below is a port of the exponential backoff implementation from
	// google-http-java-client.
	"github.com/cenkalti/backoff"
	"github.com/fiorix/go-web/autogzip"
	"github.com/skia-dev/glog"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/timer"
	"go.skia.org/infra/go/util"
)

const (
	DIAL_TIMEOUT    = time.Minute
	REQUEST_TIMEOUT = 5 * time.Minute

	// Exponential backoff defaults.
	INITIAL_INTERVAL     = 500 * time.Millisecond
	RANDOMIZATION_FACTOR = 0.5
	BACKOFF_MULTIPLIER   = 1.5
	MAX_INTERVAL         = 60 * time.Second
	MAX_ELAPSED_TIME     = 5 * time.Minute

	MAX_BYTES_IN_RESPONSE_BODY = 10 * 1024 //10 KB
)

// DialTimeout is a dialer that sets a timeout.
func DialTimeout(network, addr string) (net.Conn, error) {
	return net.DialTimeout(network, addr, DIAL_TIMEOUT)
}

// NewTimeoutClient creates a new http.Client with both a dial timeout and a
// request timeout.
func NewTimeoutClient() *http.Client {
	return &http.Client{
		Transport: &http.Transport{
			Dial: DialTimeout,
		},
		Timeout: REQUEST_TIMEOUT,
	}
}

type BackOffConfig struct {
	initialInterval     time.Duration
	maxInterval         time.Duration
	maxElapsedTime      time.Duration
	randomizationFactor float64
	backOffMultiplier   float64
}

// NewBackOffTransport creates a BackOffTransport with default values. Look at
// NewConfiguredBackOffTransport for an example of how the values impact behavior.
func NewBackOffTransport() http.RoundTripper {
	config := &BackOffConfig{
		initialInterval:     INITIAL_INTERVAL,
		maxInterval:         MAX_INTERVAL,
		maxElapsedTime:      MAX_ELAPSED_TIME,
		randomizationFactor: RANDOMIZATION_FACTOR,
		backOffMultiplier:   BACKOFF_MULTIPLIER,
	}
	return NewConfiguredBackOffTransport(config)
}

type BackOffTransport struct {
	http.Transport
	backOffConfig *BackOffConfig
}

type ResponsePagination struct {
	Offset int `json:"offset"`
	Size   int `json:"size"`
	Total  int `json:"total"`
}

// NewBackOffTransport creates a BackOffTransport with the specified config.
//
// Example: The default retry_interval is .5 seconds, default randomization_factor
// is 0.5, default multiplier is 1.5 and the default max_interval is 1 minute. For
// 10 tries the sequence will be (values in seconds) and assuming we go over the
// max_elapsed_time on the 10th try:
//
//  request#     retry_interval     randomized_interval
//  1             0.5                [0.25,   0.75]
//  2             0.75               [0.375,  1.125]
//  3             1.125              [0.562,  1.687]
//  4             1.687              [0.8435, 2.53]
//  5             2.53               [1.265,  3.795]
//  6             3.795              [1.897,  5.692]
//  7             5.692              [2.846,  8.538]
//  8             8.538              [4.269, 12.807]
//  9            12.807              [6.403, 19.210]
//  10           19.210              backoff.Stop
func NewConfiguredBackOffTransport(config *BackOffConfig) http.RoundTripper {
	return &BackOffTransport{
		Transport:     http.Transport{Dial: DialTimeout},
		backOffConfig: config,
	}
}

// RoundTrip implements the RoundTripper interface.
func (t *BackOffTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	// Initialize the exponential backoff client.
	backOffClient := &backoff.ExponentialBackOff{
		InitialInterval:     t.backOffConfig.initialInterval,
		RandomizationFactor: t.backOffConfig.randomizationFactor,
		Multiplier:          t.backOffConfig.backOffMultiplier,
		MaxInterval:         t.backOffConfig.maxInterval,
		MaxElapsedTime:      t.backOffConfig.maxElapsedTime,
		Clock:               backoff.SystemClock,
	}
	// Make a copy of the request's Body so that we can reuse it if the request
	// needs to be backed off and retried.
	bodyBuf := bytes.Buffer{}
	if req.Body != nil {
		if _, err := bodyBuf.ReadFrom(req.Body); err != nil {
			return nil, fmt.Errorf("Failed to read request body: %v", err)
		}
	}

	var resp *http.Response
	var err error
	roundTripOp := func() error {
		if req.Body != nil {
			req.Body = ioutil.NopCloser(bytes.NewBufferString(bodyBuf.String()))
		}
		resp, err = t.Transport.RoundTrip(req)
		if err != nil {
			return fmt.Errorf("Error while making the round trip: %s", err)
		}
		if resp != nil {
			if resp.StatusCode >= 500 && resp.StatusCode <= 599 {
				// We can't close the resp.Body on success, so we must do it in each of the failure cases.
				return fmt.Errorf("Got server error statuscode %d while making the HTTP %s request to %s\nResponse: %s", resp.StatusCode, req.Method, req.URL, readAndClose(resp.Body))
			} else if resp.StatusCode < 200 || resp.StatusCode > 299 {
				// We can't close the resp.Body on success, so we must do it in each of the failure cases.
				// Stop backing off if there are non server errors.
				backOffClient.MaxElapsedTime = backoff.Stop
				return fmt.Errorf("Got non server error statuscode %d while making the HTTP %s request to %s\nResponse: %s", resp.StatusCode, req.Method, req.URL, readAndClose(resp.Body))
			}
		}
		return nil
	}
	notifyFunc := func(err error, wait time.Duration) {
		glog.Warningf("Got error: %s. Retrying HTTP request after sleeping for %s", err, wait)
	}

	if err := backoff.RetryNotify(roundTripOp, backOffClient, notifyFunc); err != nil {
		return nil, fmt.Errorf("HTTP request failed inspite of exponential backoff: %s", err)
	}
	return resp, nil
}

// readAndClose reads the content of a ReadCloser (e.g. http Response), and returns it as a string.
// If the response was nil or there was a problem, it will return empty string.  The reader,
//if non-null, will be closed by this function.
func readAndClose(r io.ReadCloser) string {
	if r != nil {
		defer util.Close(r)
		if b, err := ioutil.ReadAll(io.LimitReader(r, MAX_BYTES_IN_RESPONSE_BODY)); err != nil {
			glog.Warningf("There was a potential problem reading the response body: %s", err)
		} else {
			return fmt.Sprintf("%q", string(b))
		}
	}
	return ""
}

// TODO(stephana): Remove 'r' from the argument list since it's not used. It would
// be also useful if we could specify a return status explicitly.

// ReportError formats an HTTP error response and also logs the detailed error message.
// The message parameter is returned in the HTTP response. If it is not provided then
// "Unknown error" will be returned instead.
func ReportError(w http.ResponseWriter, r *http.Request, err error, message string) {
	glog.Errorln(message, err)
	if err != io.ErrClosedPipe {
		httpErrMsg := message
		if message == "" {
			httpErrMsg = "Unknown error"
		}
		http.Error(w, httpErrMsg, 500)
	}
}

// responseProxy implements http.ResponseWriter and records the status codes.
type responseProxy struct {
	http.ResponseWriter
	wroteHeader bool
}

func (rp *responseProxy) WriteHeader(code int) {
	if !rp.wroteHeader {
		glog.Infof("Response Code: %d", code)
		metrics2.GetCounter("http.response", map[string]string{"statuscode": strconv.Itoa(code)}).Inc(1)
		rp.ResponseWriter.WriteHeader(code)
		rp.wroteHeader = true
	}
}

// recordResponse returns a wrapped http.Handler that records the status codes of the
// responses.
//
// Note that if a handler doesn't explicitly set a response code and goes with
// the default of 200 then this will never record anything.
func recordResponse(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		h.ServeHTTP(&responseProxy{ResponseWriter: w}, r)
	})
}

// LoggingGzipRequestResponse records parts of the request and the response to the logs.
func LoggingGzipRequestResponse(h http.Handler) http.Handler {
	// Closure to capture the request.
	f := func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if err := recover(); err != nil {
				const size = 64 << 10
				buf := make([]byte, size)
				buf = buf[:runtime.Stack(buf, false)]
				glog.Errorf("panic serving %v: %v\n%s", r.URL.Path, err, buf)
			}
		}()
		defer timer.New(fmt.Sprintf("Request: %s %s %#v Content Length: %d Latency:", r.URL.Path, r.Method, r.URL, r.ContentLength)).Stop()
		h.ServeHTTP(w, r)
	}

	return autogzip.Handle(recordResponse(http.HandlerFunc(f)))
}

// MakeResourceHandler is an HTTP handler function designed for serving files.
func MakeResourceHandler(resourcesDir string) func(http.ResponseWriter, *http.Request) {
	fileServer := http.FileServer(http.Dir(resourcesDir))
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Add("Cache-Control", "max-age=300")
		fileServer.ServeHTTP(w, r)
	}
}

// CorsHandler is an HTTP handler function which adds the necessary header for CORS.
func CorsHandler(h func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Add("Access-Control-Allow-Origin", "*")
		h(w, r)
	}
}

// PaginationParams is helper function to extract pagination parameters from a
// URL query string. It assumes that 'offset' and 'size' are the query parameters
// used for pagination. It parses the values and returns an error if they are
// not integers. If the params are not set the defaults are proviced.
// Further it ensures that size is never above max size.
func PaginationParams(query url.Values, defaultOffset, defaultSize, maxSize int) (int, int, error) {
	size, err := getPositiveInt(query, "size", defaultSize)
	if err != nil {
		return 0, 0, err
	}

	offset, err := getPositiveInt(query, "offset", defaultOffset)
	if err != nil {
		return 0, 0, err
	}

	return offset, util.MinInt(size, maxSize), nil
}

// getPositiveInt parses the param in query and ensures it is >= 0 using
// default value when necessary.
func getPositiveInt(query url.Values, param string, defaultVal int) (int, error) {
	var val int
	var err error
	if valStr := query.Get(param); valStr == "" {
		return defaultVal, nil
	} else {
		val, err = strconv.Atoi(valStr)
		if err != nil {
			return 0, fmt.Errorf("Not a valid integer value.")
		}
	}
	if val < 0 {
		return defaultVal, nil
	}
	return val, nil
}
<commit_msg>Add LoggingRequestResponse to httputils.<commit_after>package httputils

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"runtime"
	"strconv"
	"time"

	// Below is a port of the exponential backoff implementation from
	// google-http-java-client.
	"github.com/cenkalti/backoff"
	"github.com/fiorix/go-web/autogzip"
	"github.com/skia-dev/glog"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/timer"
	"go.skia.org/infra/go/util"
)

const (
	DIAL_TIMEOUT    = time.Minute
	REQUEST_TIMEOUT = 5 * time.Minute

	// Exponential backoff defaults.
	INITIAL_INTERVAL     = 500 * time.Millisecond
	RANDOMIZATION_FACTOR = 0.5
	BACKOFF_MULTIPLIER   = 1.5
	MAX_INTERVAL         = 60 * time.Second
	MAX_ELAPSED_TIME     = 5 * time.Minute

	MAX_BYTES_IN_RESPONSE_BODY = 10 * 1024 //10 KB
)

// DialTimeout is a dialer that sets a timeout.
func DialTimeout(network, addr string) (net.Conn, error) {
	return net.DialTimeout(network, addr, DIAL_TIMEOUT)
}

// NewTimeoutClient creates a new http.Client with both a dial timeout and a
// request timeout.
func NewTimeoutClient() *http.Client {
	return &http.Client{
		Transport: &http.Transport{
			Dial: DialTimeout,
		},
		Timeout: REQUEST_TIMEOUT,
	}
}

type BackOffConfig struct {
	initialInterval     time.Duration
	maxInterval         time.Duration
	maxElapsedTime      time.Duration
	randomizationFactor float64
	backOffMultiplier   float64
}

// NewBackOffTransport creates a BackOffTransport with default values. Look at
// NewConfiguredBackOffTransport for an example of how the values impact behavior.
func NewBackOffTransport() http.RoundTripper {
	config := &BackOffConfig{
		initialInterval:     INITIAL_INTERVAL,
		maxInterval:         MAX_INTERVAL,
		maxElapsedTime:      MAX_ELAPSED_TIME,
		randomizationFactor: RANDOMIZATION_FACTOR,
		backOffMultiplier:   BACKOFF_MULTIPLIER,
	}
	return NewConfiguredBackOffTransport(config)
}

type BackOffTransport struct {
	http.Transport
	backOffConfig *BackOffConfig
}

type ResponsePagination struct {
	Offset int `json:"offset"`
	Size   int `json:"size"`
	Total  int `json:"total"`
}

// NewBackOffTransport creates a BackOffTransport with the specified config.
//
// Example: The default retry_interval is .5 seconds, default randomization_factor
// is 0.5, default multiplier is 1.5 and the default max_interval is 1 minute. For
// 10 tries the sequence will be (values in seconds) and assuming we go over the
// max_elapsed_time on the 10th try:
//
//  request#     retry_interval     randomized_interval
//  1             0.5                [0.25,   0.75]
//  2             0.75               [0.375,  1.125]
//  3             1.125              [0.562,  1.687]
//  4             1.687              [0.8435, 2.53]
//  5             2.53               [1.265,  3.795]
//  6             3.795              [1.897,  5.692]
//  7             5.692              [2.846,  8.538]
//  8             8.538              [4.269, 12.807]
//  9            12.807              [6.403, 19.210]
//  10           19.210              backoff.Stop
func NewConfiguredBackOffTransport(config *BackOffConfig) http.RoundTripper {
	return &BackOffTransport{
		Transport:     http.Transport{Dial: DialTimeout},
		backOffConfig: config,
	}
}

// RoundTrip implements the RoundTripper interface.
func (t *BackOffTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	// Initialize the exponential backoff client.
	backOffClient := &backoff.ExponentialBackOff{
		InitialInterval:     t.backOffConfig.initialInterval,
		RandomizationFactor: t.backOffConfig.randomizationFactor,
		Multiplier:          t.backOffConfig.backOffMultiplier,
		MaxInterval:         t.backOffConfig.maxInterval,
		MaxElapsedTime:      t.backOffConfig.maxElapsedTime,
		Clock:               backoff.SystemClock,
	}
	// Make a copy of the request's Body so that we can reuse it if the request
	// needs to be backed off and retried.
	bodyBuf := bytes.Buffer{}
	if req.Body != nil {
		if _, err := bodyBuf.ReadFrom(req.Body); err != nil {
			return nil, fmt.Errorf("Failed to read request body: %v", err)
		}
	}

	var resp *http.Response
	var err error
	roundTripOp := func() error {
		if req.Body != nil {
			req.Body = ioutil.NopCloser(bytes.NewBufferString(bodyBuf.String()))
		}
		resp, err = t.Transport.RoundTrip(req)
		if err != nil {
			return fmt.Errorf("Error while making the round trip: %s", err)
		}
		if resp != nil {
			if resp.StatusCode >= 500 && resp.StatusCode <= 599 {
				// We can't close the resp.Body on success, so we must do it in each of the failure cases.
				return fmt.Errorf("Got server error statuscode %d while making the HTTP %s request to %s\nResponse: %s", resp.StatusCode, req.Method, req.URL, readAndClose(resp.Body))
			} else if resp.StatusCode < 200 || resp.StatusCode > 299 {
				// We can't close the resp.Body on success, so we must do it in each of the failure cases.
				// Stop backing off if there are non server errors.
				backOffClient.MaxElapsedTime = backoff.Stop
				return fmt.Errorf("Got non server error statuscode %d while making the HTTP %s request to %s\nResponse: %s", resp.StatusCode, req.Method, req.URL, readAndClose(resp.Body))
			}
		}
		return nil
	}
	notifyFunc := func(err error, wait time.Duration) {
		glog.Warningf("Got error: %s. Retrying HTTP request after sleeping for %s", err, wait)
	}

	if err := backoff.RetryNotify(roundTripOp, backOffClient, notifyFunc); err != nil {
		return nil, fmt.Errorf("HTTP request failed inspite of exponential backoff: %s", err)
	}
	return resp, nil
}

// readAndClose reads the content of a ReadCloser (e.g. http Response), and returns it as a string.
// If the response was nil or there was a problem, it will return empty string.  The reader,
//if non-null, will be closed by this function.
func readAndClose(r io.ReadCloser) string {
	if r != nil {
		defer util.Close(r)
		if b, err := ioutil.ReadAll(io.LimitReader(r, MAX_BYTES_IN_RESPONSE_BODY)); err != nil {
			glog.Warningf("There was a potential problem reading the response body: %s", err)
		} else {
			return fmt.Sprintf("%q", string(b))
		}
	}
	return ""
}

// TODO(stephana): Remove 'r' from the argument list since it's not used. It would
// be also useful if we could specify a return status explicitly.

// ReportError formats an HTTP error response and also logs the detailed error message.
// The message parameter is returned in the HTTP response. If it is not provided then
// "Unknown error" will be returned instead.
func ReportError(w http.ResponseWriter, r *http.Request, err error, message string) {
	glog.Errorln(message, err)
	if err != io.ErrClosedPipe {
		httpErrMsg := message
		if message == "" {
			httpErrMsg = "Unknown error"
		}
		http.Error(w, httpErrMsg, 500)
	}
}

// responseProxy implements http.ResponseWriter and records the status codes.
type responseProxy struct {
	http.ResponseWriter
	wroteHeader bool
}

func (rp *responseProxy) WriteHeader(code int) {
	if !rp.wroteHeader {
		glog.Infof("Response Code: %d", code)
		metrics2.GetCounter("http.response", map[string]string{"statuscode": strconv.Itoa(code)}).Inc(1)
		rp.ResponseWriter.WriteHeader(code)
		rp.wroteHeader = true
	}
}

// recordResponse returns a wrapped http.Handler that records the status codes of the
// responses.
//
// Note that if a handler doesn't explicitly set a response code and goes with
// the default of 200 then this will never record anything.
func recordResponse(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		h.ServeHTTP(&responseProxy{ResponseWriter: w}, r)
	})
}

// LoggingGzipRequestResponse records parts of the request and the response to
// the logs and gzips responses when appropriate.
func LoggingGzipRequestResponse(h http.Handler) http.Handler {
	return autogzip.Handle(LoggingRequestResponse(h))
}

// LoggingRequestResponse records parts of the request and the response to the logs.
func LoggingRequestResponse(h http.Handler) http.Handler {
	// Closure to capture the request.
	f := func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if err := recover(); err != nil {
				const size = 64 << 10
				buf := make([]byte, size)
				buf = buf[:runtime.Stack(buf, false)]
				glog.Errorf("panic serving %v: %v\n%s", r.URL.Path, err, buf)
			}
		}()
		defer timer.New(fmt.Sprintf("Request: %s %s %#v Content Length: %d Latency:", r.URL.Path, r.Method, r.URL, r.ContentLength)).Stop()
		h.ServeHTTP(w, r)
	}

	return recordResponse(http.HandlerFunc(f))
}

// MakeResourceHandler is an HTTP handler function designed for serving files.
func MakeResourceHandler(resourcesDir string) func(http.ResponseWriter, *http.Request) {
	fileServer := http.FileServer(http.Dir(resourcesDir))
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Add("Cache-Control", "max-age=300")
		fileServer.ServeHTTP(w, r)
	}
}

// CorsHandler is an HTTP handler function which adds the necessary header for CORS.
func CorsHandler(h func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Add("Access-Control-Allow-Origin", "*")
		h(w, r)
	}
}

// PaginationParams is helper function to extract pagination parameters from a
// URL query string. It assumes that 'offset' and 'size' are the query parameters
// used for pagination. It parses the values and returns an error if they are
// not integers. If the params are not set the defaults are proviced.
// Further it ensures that size is never above max size.
func PaginationParams(query url.Values, defaultOffset, defaultSize, maxSize int) (int, int, error) {
	size, err := getPositiveInt(query, "size", defaultSize)
	if err != nil {
		return 0, 0, err
	}

	offset, err := getPositiveInt(query, "offset", defaultOffset)
	if err != nil {
		return 0, 0, err
	}

	return offset, util.MinInt(size, maxSize), nil
}

// getPositiveInt parses the param in query and ensures it is >= 0 using
// default value when necessary.
func getPositiveInt(query url.Values, param string, defaultVal int) (int, error) {
	var val int
	var err error
	if valStr := query.Get(param); valStr == "" {
		return defaultVal, nil
	} else {
		val, err = strconv.Atoi(valStr)
		if err != nil {
			return 0, fmt.Errorf("Not a valid integer value.")
		}
	}
	if val < 0 {
		return defaultVal, nil
	}
	return val, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package saltpack

// MessageType is an int used to describe what "type" of message it is.
type MessageType int

// packetSeqno is a special int type used to describe which packet in the
// sequence we're dealing with.  The header is always at seqno=0. Other packets
// follow. Note that there is a distinction between packetSeqno and encryptionBlockNumber.
// In general, the former is one more than the latter.
type packetSeqno uint64

// MessageTypeEncryption is a packet type to describe an
// encryption message.
const MessageTypeEncryption MessageType = 0

// MessageTypeAttachedSignature is a packet type to describe an
// attached signature.
const MessageTypeAttachedSignature MessageType = 1

// MessageTypeDetachedSignature is a packet type to describe a
// detached signature.
const MessageTypeDetachedSignature MessageType = 2

// SaltpackCurrentVersion is currently the only supported packet
// version, 1.0.
var SaltpackCurrentVersion = Version{Major: 1, Minor: 0}

// encryptionBlockSize is by default 1MB and can't currently be tweaked.
const encryptionBlockSize int = 1048576

const EncryptionArmorString = "ENCRYPTED MESSAGE"
const SignedArmorString = "SIGNED MESSAGE"
const DetachedSignatureArmorString = "DETACHED SIGNATURE"

// SaltpackFormatName is the publicly advertised name of the format,
// used in the header of the message and also in Nonce creation.
const SaltpackFormatName = "saltpack"

// NoncePrefixEncryption is the prefix used to create the nonce when
// using the nonce for encryption.
const NoncePrefixEncryption = "encryption nonce prefix"

// signatureBlockSize is by default 1MB and can't currently be tweaked.
const signatureBlockSize int = 1048576

// signatureAttachedString is part of the data that is signed in
// each payload packet.
const signatureAttachedString = "saltpack attached signature\x00"

// signatureDetachedString is part of the data that is signed in
// a detached signature.
const signatureDetachedString = "saltpack detached signature\x00"

// We truncate HMAC512 to the same link that NaCl's crypto_auth function does.
const cryptoAuthBytes = 32

const cryptoAuthKeyBytes = 32

type readState int

const (
	stateBody readState = iota
	stateEndOfStream
)

func (m MessageType) String() string {
	switch m {
	case MessageTypeEncryption:
		return "an encrypted message"
	case MessageTypeDetachedSignature:
		return "a detached signature"
	case MessageTypeAttachedSignature:
		return "an attached signature"
	default:
		return "an unknown message type"
	}
}
<commit_msg>delete NoncePrefixEncryption<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package saltpack

// MessageType is an int used to describe what "type" of message it is.
type MessageType int

// packetSeqno is a special int type used to describe which packet in the
// sequence we're dealing with.  The header is always at seqno=0. Other packets
// follow. Note that there is a distinction between packetSeqno and encryptionBlockNumber.
// In general, the former is one more than the latter.
type packetSeqno uint64

// MessageTypeEncryption is a packet type to describe an
// encryption message.
const MessageTypeEncryption MessageType = 0

// MessageTypeAttachedSignature is a packet type to describe an
// attached signature.
const MessageTypeAttachedSignature MessageType = 1

// MessageTypeDetachedSignature is a packet type to describe a
// detached signature.
const MessageTypeDetachedSignature MessageType = 2

// SaltpackCurrentVersion is currently the only supported packet
// version, 1.0.
var SaltpackCurrentVersion = Version{Major: 1, Minor: 0}

// encryptionBlockSize is by default 1MB and can't currently be tweaked.
const encryptionBlockSize int = 1048576

const EncryptionArmorString = "ENCRYPTED MESSAGE"
const SignedArmorString = "SIGNED MESSAGE"
const DetachedSignatureArmorString = "DETACHED SIGNATURE"

// SaltpackFormatName is the publicly advertised name of the format,
// used in the header of the message and also in Nonce creation.
const SaltpackFormatName = "saltpack"

// signatureBlockSize is by default 1MB and can't currently be tweaked.
const signatureBlockSize int = 1048576

// signatureAttachedString is part of the data that is signed in
// each payload packet.
const signatureAttachedString = "saltpack attached signature\x00"

// signatureDetachedString is part of the data that is signed in
// a detached signature.
const signatureDetachedString = "saltpack detached signature\x00"

// We truncate HMAC512 to the same link that NaCl's crypto_auth function does.
const cryptoAuthBytes = 32

const cryptoAuthKeyBytes = 32

type readState int

const (
	stateBody readState = iota
	stateEndOfStream
)

func (m MessageType) String() string {
	switch m {
	case MessageTypeEncryption:
		return "an encrypted message"
	case MessageTypeDetachedSignature:
		return "a detached signature"
	case MessageTypeAttachedSignature:
		return "an attached signature"
	default:
		return "an unknown message type"
	}
}
<|endoftext|>


<commit_before>package simulate

import (
	"fmt"
	"log"
	"xpfunds"
	"xpfunds/median"
)

func MedianPerformance(funds []*xpfunds.Fund, maxDuration, minMonths, maxNumFunds int, s Strategy) float64 {
	var perfs []float64

	for time := maxDuration - 1; time >= 1; time-- {
		var active []*xpfunds.Fund
		withMinMonths := 0
		for _, f := range funds {
			if f.Duration() >= time+1 {
				active = append(active, f)
			}
			if f.Duration() >= time+minMonths {
				withMinMonths++
			}
		}
		if len(active) < maxNumFunds+1 || withMinMonths == 0 {
			continue
		}
		perfs = append(perfs, performance(active, s, time))
	}
	return median.Median(perfs)
}

func performance(funds []*xpfunds.Fund, s Strategy, time int) float64 {
	chosenFunds := s.Choose(funds, time)
	total := 0.0
	for _, f := range chosenFunds {
		total += f.Return(0, time)
	}
	return total / float64(len(chosenFunds))
}

type Strategy interface {
	Name() string

	Choose(funds []*xpfunds.Fund, end int) []*xpfunds.Fund
}

type Weighted struct {
	monthsToRead        int
	ignoreWithoutMonths int
	weight              []float64
}

func NewWeighted(maxMonths int, weight []float64) *Weighted {
	return &Weighted{
		0,      // int(math.Round((weight[len(weight)-2] + 1) / 2 * float64(maxMonths))),
		6,      // int(math.Round((weight[len(weight)-1] + 1) / 2 * float64(maxMonths))),
		weight, // weight[:len(weight)-2],
	}
}

func (w *Weighted) Name() string {
	return fmt.Sprintf("Weighted(%v,%v,%v)", w.monthsToRead, w.ignoreWithoutMonths, w.weight)
}

func (w *Weighted) Choose(funds []*xpfunds.Fund, end int) []*xpfunds.Fund {
	featureCount := funds[0].FeatureCount()
	numFunds := len(w.weight) / featureCount
	chosen := make(map[*xpfunds.Fund]bool)
	for i := 0; i < numFunds; i++ {
		var bestFund *xpfunds.Fund
		bestValue := -999999.99
		for _, f := range funds {
			if chosen[f] || f.Duration()-end < w.monthsToRead+w.ignoreWithoutMonths {
				continue
			}
			start := end + w.monthsToRead
			if w.monthsToRead == 0 {
				start = f.Duration()
			}
			value := f.Weighted(w.weight[0:featureCount], end, start)
			if value > bestValue {
				bestValue = value
				bestFund = f
			}
		}
		if bestFund == nil {
			break
		}
		chosen[bestFund] = true
	}
	if len(chosen) == 0 {
		for _, f := range funds {
			log.Print(f.Duration() - end)
		}
		log.Fatal("len(funds)=", len(funds), " w=", w.Name())
	}
	ret := make([]*xpfunds.Fund, len(chosen))
	i := 0
	for f := range chosen {
		ret[i] = f
		i++
	}
	return ret
}

func (w *Weighted) FeatureCount() int {
	return 0
}
<commit_msg>individual<commit_after>package simulate

import (
	"fmt"
	"log"
	"xpfunds"
	"xpfunds/median"
)

func MedianPerformance(funds []*xpfunds.Fund, maxDuration, minMonths, maxNumFunds int, s Strategy) float64 {
	var perfs []float64

	for time := maxDuration - 1; time >= 1; time-- {
		var active []*xpfunds.Fund
		withMinMonths := 0
		for _, f := range funds {
			if f.Duration() >= time+1 {
				active = append(active, f)
			}
			if f.Duration() >= time+minMonths {
				withMinMonths++
			}
		}
		if len(active) < maxNumFunds+1 || withMinMonths == 0 {
			continue
		}
		perfs = append(perfs, performance(active, s, time))
	}
	return median.Median(perfs)
}

func performance(funds []*xpfunds.Fund, s Strategy, time int) float64 {
	chosenFunds := s.Choose(funds, time)
	total := 0.0
	for _, f := range chosenFunds {
		total += f.Return(0, time)
	}
	return total / float64(len(chosenFunds))
}

type Strategy interface {
	Name() string

	Choose(funds []*xpfunds.Fund, end int) []*xpfunds.Fund
}

type Weighted struct {
	monthsToRead        int
	ignoreWithoutMonths int
	weight              []float64
}

func NewWeighted(maxMonths int, weight []float64) *Weighted {
	return &Weighted{
		0,      // int(math.Round((weight[len(weight)-2] + 1) / 2 * float64(maxMonths))),
		6,      // int(math.Round((weight[len(weight)-1] + 1) / 2 * float64(maxMonths))),
		weight, // weight[:len(weight)-2],
	}
}

func (w *Weighted) Name() string {
	return fmt.Sprintf("Weighted(%v,%v,%v)", w.monthsToRead, w.ignoreWithoutMonths, w.weight)
}

func (w *Weighted) Choose(funds []*xpfunds.Fund, end int) []*xpfunds.Fund {
	featureCount := funds[0].FeatureCount()
	numFunds := len(w.weight) / featureCount
	chosen := make(map[*xpfunds.Fund]bool)
	for i := 0; i < numFunds; i++ {
		var bestFund *xpfunds.Fund
		bestValue := -999999.99
		for _, f := range funds {
			if chosen[f] || f.Duration()-end < w.monthsToRead+w.ignoreWithoutMonths {
				continue
			}
			start := end + w.monthsToRead
			if w.monthsToRead == 0 {
				start = f.Duration()
			}
			value := f.Weighted(w.weight[i*featureCount:(i+1)*featureCount], end, start)
			if value > bestValue {
				bestValue = value
				bestFund = f
			}
		}
		if bestFund == nil {
			break
		}
		chosen[bestFund] = true
	}
	if len(chosen) == 0 {
		for _, f := range funds {
			log.Print(f.Duration() - end)
		}
		log.Fatal("len(funds)=", len(funds), " w=", w.Name())
	}
	ret := make([]*xpfunds.Fund, len(chosen))
	i := 0
	for f := range chosen {
		ret[i] = f
		i++
	}
	return ret
}

func (w *Weighted) FeatureCount() int {
	return 0
}
<|endoftext|>


<commit_before>package profile

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"sync"
	"time"

	"github.com/NebulousLabs/Sia/persist"
)

// There's a global lock on cpu and memory profiling, because I'm not sure what
// happens if multiple threads call each at the same time. This lock might be
// unnecessary.
var (
	cpuActive bool
	cpuLock   sync.Mutex
	memActive bool
	memLock   sync.Mutex
)

// startCPUProfile starts cpu profiling. An error will be returned if a cpu
// profiler is already running.
func StartCPUProfile(profileDir, identifier string) error {
	// Lock the cpu profile lock so that only one profiler is running at a
	// time.
	cpuLock.Lock()
	if cpuActive {
		cpuLock.Unlock()
		return errors.New("cannot start cpu profilier, a profiler is already running")
	}
	cpuActive = true
	cpuLock.Unlock()

	// Start profiling into the profile dir, using the identifer. The timestamp
	// of the start time of the profiling will be included in the filename.
	cpuProfileFile, err := os.Create(filepath.Join(profileDir, "cpu-profile-"+identifier+"-"+time.Now().Format(time.RFC3339Nano)+".prof"))
	if err != nil {
		return err
	}
	pprof.StartCPUProfile(cpuProfileFile)
	return nil
}

// stopCPUProfile stops cpu profiling.
func StopCPUProfile() {
	cpuLock.Lock()
	if cpuActive {
		pprof.StopCPUProfile()
		cpuActive = false
	}
	cpuLock.Unlock()
}

// saveMemProfile saves the current memory structure of the program. An error
// will be returned if memory profiling is already in progress. Unlike for cpu
// profiling, there is no 'stopMemProfile' call - everything happens at once.
func SaveMemProfile(profileDir, identifier string) error {
	memLock.Lock()
	if memActive {
		memLock.Unlock()
		return errors.New("cannot start memory profiler, a memory profiler is already running")
	}
	memActive = true
	memLock.Unlock()

	// Save the memory profile.
	memFile, err := os.Create(filepath.Join(profileDir, "mem-profile-"+identifier+"-"+time.Now().Format(time.RFC3339Nano)+".prof"))
	if err != nil {
		return err
	}
	pprof.WriteHeapProfile(memFile)

	memLock.Lock()
	memActive = false
	memLock.Unlock()
	return nil
}

// StartContinuousProfiling will continuously print statistics about the cpu
// usage, memory usage, and runtime stats of the program.
func StartContinuousProfile(profileDir string) {
	// Create the folder for all of the profiling results.
	err := os.MkdirAll(profileDir, 0700)
	if err != nil {
		fmt.Println(err)
		return
	}

	// Continuously log statistics about the running Sia application.
	go func() {
		// Create the logger.
		log, err := persist.NewFileLogger(filepath.Join(profileDir, "continuousProfiling.log"))
		if err != nil {
			fmt.Println("Profile logging failed:", err)
			return
		}

		// Collect statistics in an infinite loop.
		sleepTime := time.Second * 20
		for {
			// Sleep for an exponential amount of time each iteration, this
			// keeps the size of the log small while still providing lots of
			// information.
			StopCPUProfile()
			SaveMemProfile(profileDir, "continuousProfilingMem")
			StartCPUProfile(profileDir, "continuousProfilingCPU")
			time.Sleep(sleepTime)
			sleepTime = time.Duration(1.5 * float64(sleepTime))

			var m runtime.MemStats
			runtime.ReadMemStats(&m)
			log.Printf("\n\tGoroutines: %v\n\tAlloc: %v\n\tTotalAlloc: %v\n\tHeapAlloc: %v\n\tHeapSys: %v\n", runtime.NumGoroutine(), m.Alloc, m.TotalAlloc, m.HeapAlloc, m.HeapSys)
		}
	}()
}
<commit_msg>profile.go: update function descriptions<commit_after>package profile

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"sync"
	"time"

	"github.com/NebulousLabs/Sia/persist"
)

// There's a global lock on cpu and memory profiling, because I'm not sure what
// happens if multiple threads call each at the same time. This lock might be
// unnecessary.
var (
	cpuActive bool
	cpuLock   sync.Mutex
	memActive bool
	memLock   sync.Mutex
)

// StartCPUProfile starts cpu profiling. An error will be returned if a cpu
// profiler is already running.
func StartCPUProfile(profileDir, identifier string) error {
	// Lock the cpu profile lock so that only one profiler is running at a
	// time.
	cpuLock.Lock()
	if cpuActive {
		cpuLock.Unlock()
		return errors.New("cannot start cpu profilier, a profiler is already running")
	}
	cpuActive = true
	cpuLock.Unlock()

	// Start profiling into the profile dir, using the identifer. The timestamp
	// of the start time of the profiling will be included in the filename.
	cpuProfileFile, err := os.Create(filepath.Join(profileDir, "cpu-profile-"+identifier+"-"+time.Now().Format(time.RFC3339Nano)+".prof"))
	if err != nil {
		return err
	}
	pprof.StartCPUProfile(cpuProfileFile)
	return nil
}

// StopCPUProfile stops cpu profiling.
func StopCPUProfile() {
	cpuLock.Lock()
	if cpuActive {
		pprof.StopCPUProfile()
		cpuActive = false
	}
	cpuLock.Unlock()
}

// SaveMemProfile saves the current memory structure of the program. An error
// will be returned if memory profiling is already in progress. Unlike for cpu
// profiling, there is no 'stopMemProfile' call - everything happens at once.
func SaveMemProfile(profileDir, identifier string) error {
	memLock.Lock()
	if memActive {
		memLock.Unlock()
		return errors.New("cannot start memory profiler, a memory profiler is already running")
	}
	memActive = true
	memLock.Unlock()

	// Save the memory profile.
	memFile, err := os.Create(filepath.Join(profileDir, "mem-profile-"+identifier+"-"+time.Now().Format(time.RFC3339Nano)+".prof"))
	if err != nil {
		return err
	}
	pprof.WriteHeapProfile(memFile)

	memLock.Lock()
	memActive = false
	memLock.Unlock()
	return nil
}

// StartContinuousProfiling will continuously print statistics about the cpu
// usage, memory usage, and runtime stats of the program.
func StartContinuousProfile(profileDir string) {
	// Create the folder for all of the profiling results.
	err := os.MkdirAll(profileDir, 0700)
	if err != nil {
		fmt.Println(err)
		return
	}

	// Continuously log statistics about the running Sia application.
	go func() {
		// Create the logger.
		log, err := persist.NewFileLogger(filepath.Join(profileDir, "continuousProfiling.log"))
		if err != nil {
			fmt.Println("Profile logging failed:", err)
			return
		}

		// Collect statistics in an infinite loop.
		sleepTime := time.Second * 20
		for {
			// Sleep for an exponential amount of time each iteration, this
			// keeps the size of the log small while still providing lots of
			// information.
			StopCPUProfile()
			SaveMemProfile(profileDir, "continuousProfilingMem")
			StartCPUProfile(profileDir, "continuousProfilingCPU")
			time.Sleep(sleepTime)
			sleepTime = time.Duration(1.5 * float64(sleepTime))

			var m runtime.MemStats
			runtime.ReadMemStats(&m)
			log.Printf("\n\tGoroutines: %v\n\tAlloc: %v\n\tTotalAlloc: %v\n\tHeapAlloc: %v\n\tHeapSys: %v\n", runtime.NumGoroutine(), m.Alloc, m.TotalAlloc, m.HeapAlloc, m.HeapSys)
		}
	}()
}
<|endoftext|>


<commit_before>package eloqua

import (
	"fmt"
)

// EmailGroupService provides access to all the endpoints related
// to email group data within eloqua
//
// Eloqua API docs: https://docs.oracle.com/cloud/latest/marketingcs_gs/OMCAB/#Developers/RESTAPI/1.0 Endpoints/Email groups/post-assets-emailGroup.htm
type EmailGroupService struct {
	client *Client
}

// EmailGroup represents an Eloqua email group object.
type EmailGroup struct {
	Type          string   `json:"type,omitempty"`
	ID            int      `json:"id,omitempty,string"`
	CreatedAt     int      `json:"createdAt,omitempty,string"`
	CreatedBy     int      `json:"createdBy,omitempty,string"`
	RequestDepth  string   `json:"depth,omitempty"`
	Name          string   `json:"name,omitempty"`
	Permissions   []string `json:"permissions,omitempty"`
	Description   string   `json:"description,omitempty"`
	UpdatedAt     int      `json:"updatedAt,omitempty,string"`
	UpdatedBy     int      `json:"updatedBy,omitempty,string"`
	EmailHeaderID int      `json:"emailHeaderId,omitempty,string"`
	EmailFooterID int      `json:"emailFooterId,omitempty,string"`
	EmailIDs      []int    `json:"emailIds,omitempty,string"`

	IsVisibleInOutlookPlugin          bool   `json:"isVisibleInOutlookPlugin,omitempty,string"`
	IsVisibleInPublicSubscriptionList bool   `json:"isVisibleInPublicSubscriptionList,omitempty,string"`
	SubscriptionListDataLookupID      string `json:"subscriptionListDataLookupId,omitempty"`
	SubscriptionListID                int    `json:"subscriptionListId,omitempty,string"`
	SubscriptionLandingPageId         int    `json:"subscriptionLandingPageId,omitempty,string"`
	UnSubscriptionListDataLookupId    string `json:"unSubscriptionListDataLookupId,omitempty"`
	UnSubscriptionListId              int    `json:"unSubscriptionListId,omitempty,string"`
	UnsubscriptionLandingPageId       int    `json:"unsubscriptionLandingPageId,omitempty,string"`
}

// Create a new email group in eloqua
// During testing subscriptionLandingPageId & subscriptionLandingPageId seemed to be required but
// as this is not as per the documentation it is not required in this method.
// If you get ObjectValidationError's it may be due to this.
func (e *EmailGroupService) Create(name string, emailGroup *EmailGroup) (*EmailGroup, *Response, error) {
	if emailGroup == nil {
		emailGroup = &EmailGroup{}
	}

	emailGroup.Name = name
	endpoint := "/assets/email/group"
	resp, err := e.client.postRequestDecode(endpoint, emailGroup)
	return emailGroup, resp, err
}

// Get a email group object via its ID
func (e *EmailGroupService) Get(id int) (*EmailGroup, *Response, error) {
	endpoint := fmt.Sprintf("/assets/email/group/%d?depth=complete", id)
	emailGroup := &EmailGroup{}
	resp, err := e.client.getRequestDecode(endpoint, emailGroup)
	return emailGroup, resp, err
}

// List many eloqua email groups
func (e *EmailGroupService) List(opts *ListOptions) ([]EmailGroup, *Response, error) {
	endpoint := "/assets/email/groups"
	emailGroups := new([]EmailGroup)
	resp, err := e.client.getRequestListDecode(endpoint, emailGroups, opts)
	return *emailGroups, resp, err
}

// Update an existing email group in eloqua
// During testing subscriptionLandingPageId & subscriptionLandingPageId seemed to be required but
// as this is not as per the documentation it is not required in this method.
// If you get ObjectValidationError's it may be due to this.
func (e *EmailGroupService) Update(id int, name string, emailGroup *EmailGroup) (*EmailGroup, *Response, error) {
	if emailGroup == nil {
		emailGroup = &EmailGroup{}
	}

	emailGroup.ID = id
	emailGroup.Name = name

	endpoint := fmt.Sprintf("/assets/email/group/%d", emailGroup.ID)
	resp, err := e.client.putRequestDecode(endpoint, emailGroup)
	return emailGroup, resp, err
}

// Delete an existing email group from eloqua
func (e *EmailGroupService) Delete(id int) (*Response, error) {
	emailGroup := &EmailGroup{ID: id}
	endpoint := fmt.Sprintf("/assets/email/group/%d", emailGroup.ID)
	resp, err := e.client.deleteRequest(endpoint, emailGroup)
	return resp, err
}
<commit_msg>Fixed linting issues<commit_after>package eloqua

import (
	"fmt"
)

// EmailGroupService provides access to all the endpoints related
// to email group data within eloqua
//
// Eloqua API docs: https://docs.oracle.com/cloud/latest/marketingcs_gs/OMCAB/#Developers/RESTAPI/1.0 Endpoints/Email groups/post-assets-emailGroup.htm
type EmailGroupService struct {
	client *Client
}

// EmailGroup represents an Eloqua email group object.
type EmailGroup struct {
	Type          string   `json:"type,omitempty"`
	ID            int      `json:"id,omitempty,string"`
	CreatedAt     int      `json:"createdAt,omitempty,string"`
	CreatedBy     int      `json:"createdBy,omitempty,string"`
	RequestDepth  string   `json:"depth,omitempty"`
	Name          string   `json:"name,omitempty"`
	Permissions   []string `json:"permissions,omitempty"`
	Description   string   `json:"description,omitempty"`
	UpdatedAt     int      `json:"updatedAt,omitempty,string"`
	UpdatedBy     int      `json:"updatedBy,omitempty,string"`
	EmailHeaderID int      `json:"emailHeaderId,omitempty,string"`
	EmailFooterID int      `json:"emailFooterId,omitempty,string"`
	EmailIDs      []int    `json:"emailIds,omitempty,string"`

	IsVisibleInOutlookPlugin          bool   `json:"isVisibleInOutlookPlugin,omitempty,string"`
	IsVisibleInPublicSubscriptionList bool   `json:"isVisibleInPublicSubscriptionList,omitempty,string"`
	SubscriptionListDataLookupID      string `json:"subscriptionListDataLookupId,omitempty"`
	SubscriptionListID                int    `json:"subscriptionListId,omitempty,string"`
	SubscriptionLandingPageID         int    `json:"subscriptionLandingPageId,omitempty,string"`
	UnSubscriptionListDataLookupID    string `json:"unSubscriptionListDataLookupId,omitempty"`
	UnSubscriptionListID              int    `json:"unSubscriptionListId,omitempty,string"`
	UnsubscriptionLandingPageID       int    `json:"unsubscriptionLandingPageId,omitempty,string"`
}

// Create a new email group in eloqua
// During testing subscriptionLandingPageId & subscriptionLandingPageId seemed to be required but
// as this is not as per the documentation it is not required in this method.
// If you get ObjectValidationError's it may be due to this.
func (e *EmailGroupService) Create(name string, emailGroup *EmailGroup) (*EmailGroup, *Response, error) {
	if emailGroup == nil {
		emailGroup = &EmailGroup{}
	}

	emailGroup.Name = name
	endpoint := "/assets/email/group"
	resp, err := e.client.postRequestDecode(endpoint, emailGroup)
	return emailGroup, resp, err
}

// Get a email group object via its ID
func (e *EmailGroupService) Get(id int) (*EmailGroup, *Response, error) {
	endpoint := fmt.Sprintf("/assets/email/group/%d?depth=complete", id)
	emailGroup := &EmailGroup{}
	resp, err := e.client.getRequestDecode(endpoint, emailGroup)
	return emailGroup, resp, err
}

// List many eloqua email groups
func (e *EmailGroupService) List(opts *ListOptions) ([]EmailGroup, *Response, error) {
	endpoint := "/assets/email/groups"
	emailGroups := new([]EmailGroup)
	resp, err := e.client.getRequestListDecode(endpoint, emailGroups, opts)
	return *emailGroups, resp, err
}

// Update an existing email group in eloqua
// During testing subscriptionLandingPageId & subscriptionLandingPageId seemed to be required but
// as this is not as per the documentation it is not required in this method.
// If you get ObjectValidationError's it may be due to this.
func (e *EmailGroupService) Update(id int, name string, emailGroup *EmailGroup) (*EmailGroup, *Response, error) {
	if emailGroup == nil {
		emailGroup = &EmailGroup{}
	}

	emailGroup.ID = id
	emailGroup.Name = name

	endpoint := fmt.Sprintf("/assets/email/group/%d", emailGroup.ID)
	resp, err := e.client.putRequestDecode(endpoint, emailGroup)
	return emailGroup, resp, err
}

// Delete an existing email group from eloqua
func (e *EmailGroupService) Delete(id int) (*Response, error) {
	emailGroup := &EmailGroup{ID: id}
	endpoint := fmt.Sprintf("/assets/email/group/%d", emailGroup.ID)
	resp, err := e.client.deleteRequest(endpoint, emailGroup)
	return resp, err
}
<|endoftext|>


<commit_before>// Package grpc is a grpc proxy built for the go-micro/server
package grpc

import (
	"context"
	"io"
	"strings"

	"github.com/micro/go-micro/v3/client"
	grpcc "github.com/micro/go-micro/v3/client/grpc"
	"github.com/micro/go-micro/v3/codec"
	"github.com/micro/go-micro/v3/codec/bytes"
	"github.com/micro/go-micro/v3/errors"
	"github.com/micro/go-micro/v3/logger"
	"github.com/micro/go-micro/v3/proxy"
	"github.com/micro/go-micro/v3/server"
	"google.golang.org/grpc"
)

// Proxy will transparently proxy requests to an endpoint.
// If no endpoint is specified it will call a service using the client.
type Proxy struct {
	// embed options
	options proxy.Options

	// The client to use for outbound requests in the local network
	Client client.Client

	// Endpoint to route all calls to
	Endpoint string
}

// read client request and write to server
func readLoop(r server.Request, s client.Stream) error {
	// request to backend server
	req := s.Request()

	for {
		// get data from client
		//  no need to decode it
		body, err := r.Read()
		if err == io.EOF {
			return nil
		}

		if err != nil {
			return err
		}

		// get the header from client
		hdr := r.Header()
		msg := &codec.Message{
			Type:   codec.Request,
			Header: hdr,
			Body:   body,
		}

		// write the raw request
		err = req.Codec().Write(msg, nil)
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}
	}
}

// ProcessMessage acts as a message exchange and forwards messages to ongoing topics
// TODO: should we look at p.Endpoint and only send to the local endpoint? probably
func (p *Proxy) ProcessMessage(ctx context.Context, msg server.Message) error {
	// TODO: check that we're not broadcast storming by sending to the same topic
	// that we're actually subscribed to

	if logger.V(logger.TraceLevel, logger.DefaultLogger) {
		logger.Tracef("Proxy received message for %s", msg.Topic())
	}

	// directly publish to the local client
	return p.Client.Publish(ctx, msg)
}

// ServeRequest honours the server.Router interface
func (p *Proxy) ServeRequest(ctx context.Context, req server.Request, rsp server.Response) error {
	// service name to call
	service := req.Service()
	// endpoint to call
	endpoint := req.Endpoint()

	if len(service) == 0 {
		return errors.BadRequest("go.micro.proxy", "service name is blank")
	}

	if logger.V(logger.TraceLevel, logger.DefaultLogger) {
		logger.Tracef("Proxy received request for %s %s", service, endpoint)
	}

	var opts []client.CallOption

	// call a specific backend
	if len(p.Endpoint) > 0 {
		// address:port
		if parts := strings.Split(p.Endpoint, ":"); len(parts) > 1 {
			opts = append(opts, client.WithAddress(p.Endpoint))
			// use as service name
		} else {
			service = p.Endpoint
		}
	}

	// serve the normal way
	return p.serveRequest(ctx, p.Client, service, endpoint, req, rsp, opts...)
}

func (p *Proxy) serveRequest(ctx context.Context, link client.Client, service, endpoint string, req server.Request, rsp server.Response, opts ...client.CallOption) error {
	// read initial request
	body, err := req.Read()
	if err != nil {
		return err
	}

	// create new request with raw bytes body
	creq := link.NewRequest(service, endpoint, &bytes.Frame{Data: body}, client.WithContentType(req.ContentType()))

	// not a stream so make a client.Call request
	if !req.Stream() {
		crsp := new(bytes.Frame)

		// make a call to the backend
		if err := link.Call(ctx, creq, crsp, opts...); err != nil {
			return err
		}

		// write the response
		if err := rsp.Write(crsp.Data); err != nil {
			return err
		}

		return nil
	}

	// new context with cancel
	ctx, cancel := context.WithCancel(ctx)

	// create new stream
	stream, err := link.Stream(ctx, creq, opts...)
	if err != nil {
		return err
	}
	defer stream.Close()

	// with a grpc stream we have to refire the initial request
	// client request to start the server side

	// get the header from client
	msg := &codec.Message{
		Type:   codec.Request,
		Header: req.Header(),
		Body:   body,
	}

	// write the raw request
	err = stream.Request().Codec().Write(msg, nil)
	if err == io.EOF {
		return nil
	} else if err != nil {
		return err
	}

	// create client request read loop if streaming
	go func() {
		err := readLoop(req, stream)
		if err != nil && err != io.EOF {
			// cancel the context
			cancel()
		}
	}()

	// get raw response
	resp := stream.Response()

	// create server response write loop
	for {
		// read backend response body
		body, err := resp.Read()
		if err != nil {
			// when we're done if its a grpc stream we have to set the trailer
			if cc, ok := stream.(grpc.ClientStream); ok {
				if ss, ok := resp.Codec().(grpc.ServerStream); ok {
					ss.SetTrailer(cc.Trailer())
				}
			}
		}

		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}

		// read backend response header
		hdr := resp.Header()

		// write raw response header to client
		rsp.WriteHeader(hdr)

		// write raw response body to client
		err = rsp.Write(body)
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}
	}
}

func (p *Proxy) String() string {
	return "grpc"
}

// NewProxy returns a new proxy which will route based on mucp headers
func NewProxy(opts ...proxy.Option) proxy.Proxy {
	var options proxy.Options

	for _, o := range opts {
		o(&options)
	}

	// create a new grpc proxy
	p := new(Proxy)
	p.options = options

	// set the client
	p.Client = options.Client
	// set the endpoint
	p.Endpoint = options.Endpoint

	// set the default client
	if p.Client == nil {
		p.Client = grpcc.NewClient()
	}

	return p
}
<commit_msg>proxy/grpc: fix client streaming bug (EOF not sent to the server) (#2011)<commit_after>// Package grpc is a grpc proxy built for the go-micro/server
package grpc

import (
	"context"
	"io"
	"strings"

	"github.com/micro/go-micro/v3/client"
	grpcc "github.com/micro/go-micro/v3/client/grpc"
	"github.com/micro/go-micro/v3/codec"
	"github.com/micro/go-micro/v3/codec/bytes"
	"github.com/micro/go-micro/v3/errors"
	"github.com/micro/go-micro/v3/logger"
	"github.com/micro/go-micro/v3/proxy"
	"github.com/micro/go-micro/v3/server"
	"google.golang.org/grpc"
)

// Proxy will transparently proxy requests to an endpoint.
// If no endpoint is specified it will call a service using the client.
type Proxy struct {
	// embed options
	options proxy.Options

	// The client to use for outbound requests in the local network
	Client client.Client

	// Endpoint to route all calls to
	Endpoint string
}

// read client request and write to server
func readLoop(r server.Request, s client.Stream) error {
	// request to backend server
	req := s.Request()

	for {
		// get data from client
		//  no need to decode it
		body, err := r.Read()
		if err == io.EOF {
			return s.Close()
		} else if err != nil {
			return err
		}

		// get the header from client
		hdr := r.Header()
		msg := &codec.Message{
			Type:   codec.Request,
			Header: hdr,
			Body:   body,
		}

		// send the message to the stream
		if err := req.Codec().Write(msg, nil); err != nil {
			return err
		}
	}
}

// ProcessMessage acts as a message exchange and forwards messages to ongoing topics
// TODO: should we look at p.Endpoint and only send to the local endpoint? probably
func (p *Proxy) ProcessMessage(ctx context.Context, msg server.Message) error {
	// TODO: check that we're not broadcast storming by sending to the same topic
	// that we're actually subscribed to

	if logger.V(logger.TraceLevel, logger.DefaultLogger) {
		logger.Tracef("Proxy received message for %s", msg.Topic())
	}

	// directly publish to the local client
	return p.Client.Publish(ctx, msg)
}

// ServeRequest honours the server.Router interface
func (p *Proxy) ServeRequest(ctx context.Context, req server.Request, rsp server.Response) error {
	// service name to call
	service := req.Service()
	// endpoint to call
	endpoint := req.Endpoint()

	if len(service) == 0 {
		return errors.BadRequest("go.micro.proxy", "service name is blank")
	}

	if logger.V(logger.TraceLevel, logger.DefaultLogger) {
		logger.Tracef("Proxy received request for %s %s", service, endpoint)
	}

	var opts []client.CallOption

	// call a specific backend
	if len(p.Endpoint) > 0 {
		// address:port
		if parts := strings.Split(p.Endpoint, ":"); len(parts) > 1 {
			opts = append(opts, client.WithAddress(p.Endpoint))
			// use as service name
		} else {
			service = p.Endpoint
		}
	}

	// serve the normal way
	return p.serveRequest(ctx, p.Client, service, endpoint, req, rsp, opts...)
}

func (p *Proxy) serveRequest(ctx context.Context, link client.Client, service, endpoint string, req server.Request, rsp server.Response, opts ...client.CallOption) error {
	// read initial request
	body, err := req.Read()
	if err != nil {
		return err
	}

	// create new request with raw bytes body
	creq := link.NewRequest(service, endpoint, &bytes.Frame{Data: body}, client.WithContentType(req.ContentType()))

	// not a stream so make a client.Call request
	if !req.Stream() {
		crsp := new(bytes.Frame)

		// make a call to the backend
		if err := link.Call(ctx, creq, crsp, opts...); err != nil {
			return err
		}

		// write the response
		if err := rsp.Write(crsp.Data); err != nil {
			return err
		}

		return nil
	}

	// new context with cancel
	ctx, cancel := context.WithCancel(ctx)

	// create new stream
	stream, err := link.Stream(ctx, creq, opts...)
	if err != nil {
		return err
	}
	defer stream.Close()

	// with a grpc stream we have to refire the initial request
	// client request to start the server side

	// get the header from client
	msg := &codec.Message{
		Type:   codec.Request,
		Header: req.Header(),
		Body:   body,
	}

	// write the raw request
	err = stream.Request().Codec().Write(msg, nil)
	if err == io.EOF {
		return nil
	} else if err != nil {
		return err
	}

	// create client request read loop if streaming
	go func() {
		if err := readLoop(req, stream); err != nil {
			// cancel the context
			cancel()
		}
	}()

	// get raw response
	resp := stream.Response()

	// create server response write loop
	for {
		// read backend response body
		body, err := resp.Read()
		if err != nil {
			// when we're done if its a grpc stream we have to set the trailer
			if cc, ok := stream.(grpc.ClientStream); ok {
				if ss, ok := resp.Codec().(grpc.ServerStream); ok {
					ss.SetTrailer(cc.Trailer())
				}
			}
		}

		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}

		// read backend response header
		hdr := resp.Header()

		// write raw response header to client
		rsp.WriteHeader(hdr)

		// write raw response body to client
		err = rsp.Write(body)
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}
	}
}

func (p *Proxy) String() string {
	return "grpc"
}

// NewProxy returns a new proxy which will route based on mucp headers
func NewProxy(opts ...proxy.Option) proxy.Proxy {
	var options proxy.Options

	for _, o := range opts {
		o(&options)
	}

	// create a new grpc proxy
	p := new(Proxy)
	p.options = options

	// set the client
	p.Client = options.Client
	// set the endpoint
	p.Endpoint = options.Endpoint

	// set the default client
	if p.Client == nil {
		p.Client = grpcc.NewClient()
	}

	return p
}
<|endoftext|>