<commit_before>package template

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"text/template"

	"github.com/BurntSushi/toml"
	"github.com/kelseyhightower/confd/backends"
	"github.com/kelseyhightower/confd/log"
	"github.com/kelseyhightower/memkv"
)

type Config struct {
	ConfDir       string
	ConfigDir     string
	KeepStageFile bool
	Noop          bool
	Prefix        string
	StoreClient   backends.StoreClient
	TemplateDir   string
}

// TemplateResourceConfig holds the parsed template resource.
type TemplateResourceConfig struct {
	TemplateResource TemplateResource `toml:"template"`
}

// TemplateResource is the representation of a parsed template resource.
type TemplateResource struct {
	CheckCmd      string `toml:"check_cmd"`
	Dest          string
	FileMode      os.FileMode
	Gid           int
	Keys          []string
	Mode          string
	Prefix        string
	ReloadCmd     string `toml:"reload_cmd"`
	Src           string
	StageFile     *os.File
	Uid           int
	funcMap       map[string]interface{}
	lastIndex     uint64
	keepStageFile bool
	noop          bool
	prefix        string
	store         memkv.Store
	storeClient   backends.StoreClient
}

var ErrEmptySrc = errors.New("empty src template")

// NewTemplateResource creates a TemplateResource.
func NewTemplateResource(path string, config Config) (*TemplateResource, error) {
	if config.StoreClient == nil {
		return nil, errors.New("A valid StoreClient is required.")
	}
	var tc *TemplateResourceConfig
	log.Debug("Loading template resource from " + path)
	_, err := toml.DecodeFile(path, &tc)
	if err != nil {
		return nil, fmt.Errorf("Cannot process template resource %s - %s", path, err.Error())
	}
	tr := tc.TemplateResource
	tr.keepStageFile = config.KeepStageFile
	tr.noop = config.Noop
	tr.storeClient = config.StoreClient
	tr.funcMap = newFuncMap()
	tr.store = memkv.New()
	addFuncs(tr.funcMap, tr.store.FuncMap)
	tr.prefix = filepath.Join("/", config.Prefix, tr.Prefix)
	if tr.Src == "" {
		return nil, ErrEmptySrc
	}
	tr.Src = filepath.Join(config.TemplateDir, tr.Src)
	return &tr, nil
}

// setVars sets the Vars for template resource.
func (t *TemplateResource) setVars() error {
	var err error
	log.Debug("Retrieving keys from store")
	log.Debug("Key prefix set to " + t.prefix)
	result, err := t.storeClient.GetValues(appendPrefix(t.prefix, t.Keys))
	if err != nil {
		return err
	}
	t.store.Purge()
	for k, v := range result {
		t.store.Set(filepath.Join("/", strings.TrimPrefix(k, t.prefix)), v)
	}
	return nil
}

// createStageFile stages the src configuration file by processing the src
// template and setting the desired owner, group, and mode. It also sets the
// StageFile for the template resource.
// It returns an error if any.
func (t *TemplateResource) createStageFile() error {
	log.Debug("Using source template " + t.Src)

	if !isFileExist(t.Src) {
		return errors.New("Missing template: " + t.Src)
	}

	log.Debug("Compiling source template " + t.Src)
	tmpl, err := template.New(path.Base(t.Src)).Funcs(t.funcMap).ParseFiles(t.Src)
	if err != nil {
		return fmt.Errorf("Unable to process template %s, %s", t.Src, err)
	}

	// create TempFile in Dest directory to avoid cross-filesystem issues
	temp, err := ioutil.TempFile(filepath.Dir(t.Dest), "."+filepath.Base(t.Dest))
	if err != nil {
		return err
	}

	if err = tmpl.Execute(temp, nil); err != nil {
		temp.Close()
		os.Remove(temp.Name())
		return err
	}
	defer temp.Close()

	// Set the owner, group, and mode on the stage file now to make it easier to
	// compare against the destination configuration file later.
	os.Chmod(temp.Name(), t.FileMode)
	os.Chown(temp.Name(), t.Uid, t.Gid)
	t.StageFile = temp
	return nil
}

// sync compares the staged and dest config files and attempts to sync them
// if they differ. sync will run a config check command if set before
// overwriting the target config file. Finally, sync will run a reload command
// if set to have the application or service pick up the changes.
// It returns an error if any.
func (t *TemplateResource) sync() error {
	staged := t.StageFile.Name()
	if t.keepStageFile {
		log.Info("Keeping staged file: " + staged)
	} else {
		defer os.Remove(staged)
	}

	log.Debug("Comparing candidate config to " + t.Dest)
	ok, err := sameConfig(staged, t.Dest)
	if err != nil {
		log.Error(err.Error())
	}
	if t.noop {
		log.Warning("Noop mode enabled. " + t.Dest + " will not be modified")
		return nil
	}
	if !ok {
		log.Info("Target config " + t.Dest + " out of sync")
		if t.CheckCmd != "" {
			if err := t.check(); err != nil {
				return errors.New("Config check failed: " + err.Error())
			}
		}
		log.Debug("Overwriting target config " + t.Dest)
		err := os.Rename(staged, t.Dest)
		if err != nil {
			if strings.Contains(err.Error(), "device or resource busy") {
				log.Debug("Rename failed - target is likely a mount. Trying to write instead")
				// try to open the file and write to it
				var contents []byte
				var rerr error
				contents, rerr = ioutil.ReadFile(staged)
				if rerr != nil {
					return rerr
				}
				err := ioutil.WriteFile(t.Dest, contents, t.FileMode)
				// make sure owner and group match the temp file, in case the file was created with WriteFile
				os.Chown(t.Dest, t.Uid, t.Gid)
				if err != nil {
					return err
				}
			} else {
				return err
			}
		}
		if t.ReloadCmd != "" {
			if err := t.reload(); err != nil {
				return err
			}
		}
		log.Info("Target config " + t.Dest + " has been updated")
	} else {
		log.Debug("Target config " + t.Dest + " in sync")
	}
	return nil
}

// check executes the check command to validate the staged config file. The
// command is modified so that any references to src template are substituted
// with a string representing the full path of the staged file. This allows the
// check to be run on the staged file before overwriting the destination config
// file.
// It returns nil if the check command returns 0 and there are no other errors.
func (t *TemplateResource) check() error {
	var cmdBuffer bytes.Buffer
	data := make(map[string]string)
	data["src"] = t.StageFile.Name()
	tmpl, err := template.New("checkcmd").Parse(t.CheckCmd)
	if err != nil {
		return err
	}
	if err := tmpl.Execute(&cmdBuffer, data); err != nil {
		return err
	}
	log.Debug("Running " + cmdBuffer.String())
	c := exec.Command("/bin/sh", "-c", cmdBuffer.String())
	output, err := c.CombinedOutput()
	if err != nil {
		log.Error(fmt.Sprintf("%q", string(output)))
		return err
	}
	log.Debug(fmt.Sprintf("%q", string(output)))
	return nil
}

// reload executes the reload command.
// It returns nil if the reload command returns 0.
func (t *TemplateResource) reload() error {
	log.Debug("Running " + t.ReloadCmd)
	c := exec.Command("/bin/sh", "-c", t.ReloadCmd)
	output, err := c.CombinedOutput()
	if err != nil {
		log.Error(fmt.Sprintf("%q", string(output)))
		return err
	}
	log.Debug(fmt.Sprintf("%q", string(output)))
	return nil
}

// process is a convenience function that wraps calls to the three main tasks
// required to keep local configuration files in sync. First we gather vars
// from the store, then we stage a candidate configuration file, and finally sync
// things up.
// It returns an error if any.
func (t *TemplateResource) process() error {
	if err := t.setFileMode(); err != nil {
		return err
	}
	if err := t.setVars(); err != nil {
		return err
	}
	if err := t.createStageFile(); err != nil {
		return err
	}
	if err := t.sync(); err != nil {
		return err
	}
	return nil
}

// setFileMode sets the FileMode.
func (t *TemplateResource) setFileMode() error {
	if t.Mode == "" {
		if !isFileExist(t.Dest) {
			t.FileMode = 0644
		} else {
			fi, err := os.Stat(t.Dest)
			if err != nil {
				return err
			}
			t.FileMode = fi.Mode()
		}
	} else {
		mode, err := strconv.ParseUint(t.Mode, 0, 32)
		if err != nil {
			return err
		}
		t.FileMode = os.FileMode(mode)
	}
	return nil
}
<commit_msg>Added atomicity to the reload cmd<commit_after>package template

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"text/template"

	"github.com/BurntSushi/toml"
	"github.com/kelseyhightower/confd/backends"
	"github.com/kelseyhightower/confd/log"
	"github.com/kelseyhightower/memkv"
	"time"
)

type Config struct {
	ConfDir       string
	ConfigDir     string
	KeepStageFile bool
	Noop          bool
	Prefix        string
	StoreClient   backends.StoreClient
	TemplateDir   string
}

// TemplateResourceConfig holds the parsed template resource.
type TemplateResourceConfig struct {
	TemplateResource TemplateResource `toml:"template"`
}

// TemplateResource is the representation of a parsed template resource.
type TemplateResource struct {
	CheckCmd      string `toml:"check_cmd"`
	Dest          string
	FileMode      os.FileMode
	Gid           int
	Keys          []string
	Mode          string
	Prefix        string
	ReloadCmd     string `toml:"reload_cmd"`
	Src           string
	StageFile     *os.File
	Uid           int
	funcMap       map[string]interface{}
	lastIndex     uint64
	keepStageFile bool
	noop          bool
	prefix        string
	store         memkv.Store
	storeClient   backends.StoreClient
}

var ErrEmptySrc = errors.New("empty src template")
const RELOAD_CMD_MARKER_PATH = "/var/lib/confd/"

// NewTemplateResource creates a TemplateResource.
func NewTemplateResource(path string, config Config) (*TemplateResource, error) {
	if config.StoreClient == nil {
		return nil, errors.New("A valid StoreClient is required.")
	}
	var tc *TemplateResourceConfig
	log.Debug("Loading template resource from " + path)
	_, err := toml.DecodeFile(path, &tc)
	if err != nil {
		return nil, fmt.Errorf("Cannot process template resource %s - %s", path, err.Error())
	}
	tr := tc.TemplateResource
	tr.keepStageFile = config.KeepStageFile
	tr.noop = config.Noop
	tr.storeClient = config.StoreClient
	tr.funcMap = newFuncMap()
	tr.store = memkv.New()
	addFuncs(tr.funcMap, tr.store.FuncMap)
	tr.prefix = filepath.Join("/", config.Prefix, tr.Prefix)
	if tr.Src == "" {
		return nil, ErrEmptySrc
	}
	tr.Src = filepath.Join(config.TemplateDir, tr.Src)
	return &tr, nil
}

// setVars sets the Vars for template resource.
func (t *TemplateResource) setVars() error {
	var err error
	log.Debug("Retrieving keys from store")
	log.Debug("Key prefix set to " + t.prefix)
	result, err := t.storeClient.GetValues(appendPrefix(t.prefix, t.Keys))
	if err != nil {
		return err
	}
	t.store.Purge()
	for k, v := range result {
		t.store.Set(filepath.Join("/", strings.TrimPrefix(k, t.prefix)), v)
	}
	return nil
}

// createStageFile stages the src configuration file by processing the src
// template and setting the desired owner, group, and mode. It also sets the
// StageFile for the template resource.
// It returns an error if any.
func (t *TemplateResource) createStageFile() error {
	log.Debug("Using source template " + t.Src)

	if !isFileExist(t.Src) {
		return errors.New("Missing template: " + t.Src)
	}

	log.Debug("Compiling source template " + t.Src)
	tmpl, err := template.New(path.Base(t.Src)).Funcs(t.funcMap).ParseFiles(t.Src)
	if err != nil {
		return fmt.Errorf("Unable to process template %s, %s", t.Src, err)
	}

	// create TempFile in Dest directory to avoid cross-filesystem issues
	temp, err := ioutil.TempFile(filepath.Dir(t.Dest), "."+filepath.Base(t.Dest))
	if err != nil {
		return err
	}

	if err = tmpl.Execute(temp, nil); err != nil {
		temp.Close()
		os.Remove(temp.Name())
		return err
	}
	defer temp.Close()

	// Set the owner, group, and mode on the stage file now to make it easier to
	// compare against the destination configuration file later.
	os.Chmod(temp.Name(), t.FileMode)
	os.Chown(temp.Name(), t.Uid, t.Gid)
	t.StageFile = temp
	return nil
}

// sync compares the staged and dest config files and attempts to sync them
// if they differ. sync will run a config check command if set before
// overwriting the target config file. Finally, sync will run a reload command
// if set to have the application or service pick up the changes.
// It returns an error if any.
func (t *TemplateResource) sync() error {
	staged := t.StageFile.Name()
	if t.keepStageFile {
		log.Info("Keeping staged file: " + staged)
	} else {
		defer os.Remove(staged)
	}

	log.Debug("Comparing candidate config to " + t.Dest)
	ok, err := sameConfig(staged, t.Dest)
	if err != nil {
		log.Error(err.Error())
	}
	if t.noop {
		log.Warning("Noop mode enabled. " + t.Dest + " will not be modified")
		return nil
	}
	if !ok {
		log.Info("Target config " + t.Dest + " out of sync")
		if t.CheckCmd != "" {
			if err := t.check(); err != nil {
				return errors.New("Config check failed: " + err.Error())
			}
		}
		log.Debug("Overwriting target config " + t.Dest)
		err := os.Rename(staged, t.Dest)
		if err != nil {
			if strings.Contains(err.Error(), "device or resource busy") {
				log.Debug("Rename failed - target is likely a mount. Trying to write instead")
				// try to open the file and write to it
				var contents []byte
				var rerr error
				contents, rerr = ioutil.ReadFile(staged)
				if rerr != nil {
					return rerr
				}
				err := ioutil.WriteFile(t.Dest, contents, t.FileMode)
				// make sure owner and group match the temp file, in case the file was created with WriteFile
				os.Chown(t.Dest, t.Uid, t.Gid)
				if err != nil {
					return err
				}
			} else {
				return err
			}
		}
		if t.ReloadCmd != "" {
			err := t.reloadAndCreateMarker()
			if err != nil {
				log.Error(err.Error())
			}
		}
		log.Info("Target config " + t.Dest + " has been updated")
	} else {
		if t.ReloadCmd != "" {
			reloadedOk, err := sameConfig(t.Dest, t.reloadCmdMarkerFilePath())
			if err != nil {
				log.Error(err.Error())
			}

			if !reloadedOk {
				err := t.createReloadCmdMarkerFile()
				if err != nil {
					log.Error(err.Error())
				}
			} else {
				log.Debug("Reload command already ran for the dest file")
			}
		}
		log.Debug("Target config " + t.Dest + " in sync")
	}
	return nil
}

func (t *TemplateResource) reloadAndCreateMarker() error {
	t.reloadWithRetry()
	log.Debug("Reload command executed successfully")
	err := t.createReloadCmdMarkerFile()
	if err != nil {
		return err
	}
	log.Debug("Created Reload Marker file successfully")
	return nil
}

func (t *TemplateResource) createReloadCmdMarkerFile() error {
	contents, err := ioutil.ReadFile(t.Dest)
	if err != nil {
		return err
	}
	os.MkdirAll(RELOAD_CMD_MARKER_PATH, os.FileMode(0755))
	err = ioutil.WriteFile(t.reloadCmdMarkerFilePath(), contents, t.FileMode)
	// make sure owner and group match the temp file, in case the file was created with WriteFile
	os.Chown(t.Dest, t.Uid, t.Gid)
	if err != nil {
		return err
	}
	return nil
}

// check executes the check command to validate the staged config file. The
// command is modified so that any references to src template are substituted
// with a string representing the full path of the staged file. This allows the
// check to be run on the staged file before overwriting the destination config
// file.
// It returns nil if the check command returns 0 and there are no other errors.
func (t *TemplateResource) check() error {
	var cmdBuffer bytes.Buffer
	data := make(map[string]string)
	data["src"] = t.StageFile.Name()
	tmpl, err := template.New("checkcmd").Parse(t.CheckCmd)
	if err != nil {
		return err
	}
	if err := tmpl.Execute(&cmdBuffer, data); err != nil {
		return err
	}
	log.Debug("Running " + cmdBuffer.String())
	c := exec.Command("/bin/sh", "-c", cmdBuffer.String())
	output, err := c.CombinedOutput()
	if err != nil {
		log.Error(fmt.Sprintf("%q", string(output)))
		return err
	}
	log.Debug(fmt.Sprintf("%q", string(output)))
	return nil
}

// reload executes the reload command.
// It returns nil if the reload command returns 0.
func (t *TemplateResource) reload() error {
	log.Debug("Running " + t.ReloadCmd)
	c := exec.Command("/bin/sh", "-c", t.ReloadCmd)
	output, err := c.CombinedOutput()
	if err != nil {
		log.Error(fmt.Sprintf("%q", string(output)))
		return err
	}
	log.Debug(fmt.Sprintf("%q", string(output)))
	return nil
}

// Keep trying to execute reload command till
// it exits with one
func (t *TemplateResource) reloadWithRetry() {
	for {
		if err := t.reload(); err == nil {
			return;
		}
		time.Sleep(time.Second * 20)
	}
}

func (t *TemplateResource) reloadCmdMarkerName() string {
	trimedString := strings.TrimPrefix(t.Prefix + t.Src + t.Dest, "/")
	replacedString := strings.Replace(trimedString, ".", "_", -1)
	return strings.Replace(replacedString, "/", "_", -1)
}

func (t *TemplateResource) reloadCmdMarkerFilePath() string {
	return RELOAD_CMD_MARKER_PATH + t.reloadCmdMarkerName()
}

// process is a convenience function that wraps calls to the three main tasks
// required to keep local configuration files in sync. First we gather vars
// from the store, then we stage a candidate configuration file, and finally sync
// things up.
// It returns an error if any.
func (t *TemplateResource) process() error {
	if err := t.setFileMode(); err != nil {
		return err
	}
	if err := t.setVars(); err != nil {
		return err
	}
	if err := t.createStageFile(); err != nil {
		return err
	}
	if err := t.sync(); err != nil {
		return err
	}
	return nil
}

// setFileMode sets the FileMode.
func (t *TemplateResource) setFileMode() error {
	if t.Mode == "" {
		if !isFileExist(t.Dest) {
			t.FileMode = 0644
		} else {
			fi, err := os.Stat(t.Dest)
			if err != nil {
				return err
			}
			t.FileMode = fi.Mode()
		}
	} else {
		mode, err := strconv.ParseUint(t.Mode, 0, 32)
		if err != nil {
			return err
		}
		t.FileMode = os.FileMode(mode)
	}
	return nil
}
<|endoftext|>


<commit_before>// Package alice provides a convenient way to chain http handlers, together with contexts.
// Modified to no longer only chain handlers, but also pass the contexts like
// suggested by google : https://blog.golang.org/context
package alice

import (
	"net/http"

	"code.google.com/p/go.net/context"
)

// A constructor for a piece of middleware, also for the final method called by .Then
type Constructor func(context.Context, CtxHandler) CtxHandler
type CtxHandlerFunc func(context.Context, http.ResponseWriter, *http.Request)
type CtxHandler interface {
	ServeHTTP(context.Context, http.ResponseWriter, *http.Request)
}

// Chain acts as a list of http.Handler constructors.
// Chain is effectively immutable:
// once created, it will always hold
// the same set of constructors in the same order.
type Chain struct {
	constructors []Constructor
}

// New creates a new chain,
// memorizing the given list of middleware constructors.
// New serves no other function,
// constructors are only called upon a call to Then().
func New(constructors ...Constructor) Chain {
	c := Chain{}
	c.constructors = append(c.constructors, constructors...)

	return c
}

// Then chains the middleware and returns the final http.Handler.
//     New(m1, m2, m3).Then(h)
// is equivalent to:
//     m1(m2(m3(h)))
// When the request comes in, it will be passed to m1, then m2, then m3
// and finally, the given handler
// (assuming every middleware calls the following one).
//
// A chain can be safely reused by calling Then() several times.
//     stdStack := alice.New(ratelimitHandler, csrfHandler)
//     indexPipe = stdStack.Then(indexHandler)
//     authPipe = stdStack.Then(authHandler)
// Note that constructors are called on every call to Then()
// and thus several instances of the same middleware will be created
// when a chain is reused in this way.
// For proper middleware, this should cause no problems.
//
// nil is not allowed for Then()
func (c Chain) Then(h CtxHandler) (wrappedFinal http.Handler) {
	var final CtxHandler

	ctx := context.TODO()

	if h != nil {
		final = h
	} else {
		panic("nil is not allowed")
	}

	for i := len(c.constructors) - 1; i >= 0; i-- {
		final = c.constructors[i](ctx, final)
	}
	wrappedFinal = http.HandlerFunc(CtxHandlerToHandlerFunc(ctx, final))
	return
}

// Same as Then, but with CtxHandler instead of wrapped-http-Handler
func (c Chain) ThenContext(h CtxHandler) (final CtxHandler) {

	ctx := context.TODO()

	if h != nil {
		final = h
	} else {
		panic("nil is not allowed")
	}

	for i := len(c.constructors) - 1; i >= 0; i-- {
		final = c.constructors[i](ctx, final)
	}
	return
}

// Same as ThenFunc, but with CtxHandler instead of wrapped-http-Handler
func (c Chain) ThenFuncContext(fn CtxHandlerFunc) (final CtxHandler) {

	if fn == nil {
		return c.Then(nil)
	}

	return c.ThenContext(CtxHandlerFunc(fn))
}

func CtxHandlerToHandlerFunc(ctx context.Context, fn CtxHandler) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) { fn.ServeHTTP(ctx, w, r) }
}

// ThenFunc works identically to Then, but takes
// a HandlerFunc instead of a Handler.
//
// The following two statements are equivalent:
//     c.Then(http.HandlerFunc(fn))
//     c.ThenFunc(fn)
//
// ThenFunc provides all the guarantees of Then.
func (c Chain) ThenFunc(fn CtxHandlerFunc) http.Handler {
	if fn == nil {
		return c.Then(nil)
	}

	return c.Then(CtxHandlerFunc(fn))
}

// Append extends a chain, adding the specified constructors
// as the last ones in the request flow.
//
// Append returns a new chain, leaving the original one untouched.
//
//     stdChain := alice.New(m1, m2)
//     extChain := stdChain.Append(m3, m4)
//     // requests in stdChain go m1 -> m2
//     // requests in extChain go m1 -> m2 -> m3 -> m4
func (c Chain) Append(constructors ...Constructor) Chain {
	newCons := make([]Constructor, len(c.constructors))
	copy(newCons, c.constructors)
	newCons = append(newCons, constructors...)

	newChain := New(newCons...)
	return newChain
}

// ServeHTTP calls f(ctx,w, r).
func (f CtxHandlerFunc) ServeHTTP(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	f(ctx, w, r)
}
<commit_msg>fixed ThenFuncContext<commit_after>// Package alice provides a convenient way to chain http handlers, together with contexts.
// Modified to no longer only chain handlers, but also pass the contexts like
// suggested by google : https://blog.golang.org/context
package alice

import (
	"net/http"

	"code.google.com/p/go.net/context"
)

// A constructor for a piece of middleware, also for the final method called by .Then
type Constructor func(context.Context, CtxHandler) CtxHandler
type CtxHandlerFunc func(context.Context, http.ResponseWriter, *http.Request)
type CtxHandler interface {
	ServeHTTP(context.Context, http.ResponseWriter, *http.Request)
}

// Chain acts as a list of http.Handler constructors.
// Chain is effectively immutable:
// once created, it will always hold
// the same set of constructors in the same order.
type Chain struct {
	constructors []Constructor
}

// New creates a new chain,
// memorizing the given list of middleware constructors.
// New serves no other function,
// constructors are only called upon a call to Then().
func New(constructors ...Constructor) Chain {
	c := Chain{}
	c.constructors = append(c.constructors, constructors...)

	return c
}

// Then chains the middleware and returns the final http.Handler.
//     New(m1, m2, m3).Then(h)
// is equivalent to:
//     m1(m2(m3(h)))
// When the request comes in, it will be passed to m1, then m2, then m3
// and finally, the given handler
// (assuming every middleware calls the following one).
//
// A chain can be safely reused by calling Then() several times.
//     stdStack := alice.New(ratelimitHandler, csrfHandler)
//     indexPipe = stdStack.Then(indexHandler)
//     authPipe = stdStack.Then(authHandler)
// Note that constructors are called on every call to Then()
// and thus several instances of the same middleware will be created
// when a chain is reused in this way.
// For proper middleware, this should cause no problems.
//
// nil is not allowed for Then()
func (c Chain) Then(h CtxHandler) (wrappedFinal http.Handler) {
	var final CtxHandler

	ctx := context.TODO()

	if h != nil {
		final = h
	} else {
		panic("nil is not allowed")
	}

	for i := len(c.constructors) - 1; i >= 0; i-- {
		final = c.constructors[i](ctx, final)
	}
	wrappedFinal = http.HandlerFunc(CtxHandlerToHandlerFunc(ctx, final))
	return
}

// Same as Then, but with CtxHandler instead of wrapped-http-Handler
func (c Chain) ThenContext(h CtxHandler) (final CtxHandler) {

	ctx := context.TODO()

	if h != nil {
		final = h
	} else {
		panic("nil is not allowed")
	}

	for i := len(c.constructors) - 1; i >= 0; i-- {
		final = c.constructors[i](ctx, final)
	}
	return
}

// Same as ThenFunc, but with CtxHandler instead of wrapped-http-Handler
func (c Chain) ThenFuncContext(fn CtxHandlerFunc) (final CtxHandler) {

	if fn == nil {
		return c.ThenContext(nil)
	}

	return c.ThenContext(CtxHandlerFunc(fn))
}

func CtxHandlerToHandlerFunc(ctx context.Context, fn CtxHandler) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) { fn.ServeHTTP(ctx, w, r) }
}

// ThenFunc works identically to Then, but takes
// a HandlerFunc instead of a Handler.
//
// The following two statements are equivalent:
//     c.Then(http.HandlerFunc(fn))
//     c.ThenFunc(fn)
//
// ThenFunc provides all the guarantees of Then.
func (c Chain) ThenFunc(fn CtxHandlerFunc) http.Handler {
	if fn == nil {
		return c.Then(nil)
	}

	return c.Then(CtxHandlerFunc(fn))
}

// Append extends a chain, adding the specified constructors
// as the last ones in the request flow.
//
// Append returns a new chain, leaving the original one untouched.
//
//     stdChain := alice.New(m1, m2)
//     extChain := stdChain.Append(m3, m4)
//     // requests in stdChain go m1 -> m2
//     // requests in extChain go m1 -> m2 -> m3 -> m4
func (c Chain) Append(constructors ...Constructor) Chain {
	newCons := make([]Constructor, len(c.constructors))
	copy(newCons, c.constructors)
	newCons = append(newCons, constructors...)

	newChain := New(newCons...)
	return newChain
}

// ServeHTTP calls f(ctx,w, r).
func (f CtxHandlerFunc) ServeHTTP(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	f(ctx, w, r)
}
<|endoftext|>


<commit_before>package sacloud

// SSHKey 公開鍵
type SSHKey struct {
	*Resource       // ID
	propName        // 名称
	propDescription // 説明
	propCreatedAt   // 作成日時

	PublicKey   string `json:",omitempty"` // 公開鍵
	Fingerprint string `json:",omitempty"` // フィンガープリント
}

// SSHKeyGenerated 公開鍵生成戻り値(秘密鍵のダウンロード用)
type SSHKeyGenerated struct {
	SSHKey
	PrivateKey string `json:",omitempty"` // 秘密鍵
}
<commit_msg>Add getter to SSHKey<commit_after>package sacloud

// SSHKey 公開鍵
type SSHKey struct {
	*Resource       // ID
	propName        // 名称
	propDescription // 説明
	propCreatedAt   // 作成日時

	PublicKey   string `json:",omitempty"` // 公開鍵
	Fingerprint string `json:",omitempty"` // フィンガープリント
}

// SSHKeyGenerated 公開鍵生成戻り値(秘密鍵のダウンロード用)
type SSHKeyGenerated struct {
	SSHKey
	PrivateKey string `json:",omitempty"` // 秘密鍵
}

func (k *SSHKey) GetPublicKey() string {
	return k.PublicKey
}

func (k *SSHKey) SetPublicKey(pKey string) {
	k.PublicKey = pKey
}

func (k *SSHKey) GetFingerpinrt() string {
	return k.Fingerprint
}

func (k *SSHKeyGenerated) GetPrivateKey() string {
	return k.PrivateKey
}
<|endoftext|>


<commit_before>// Package chartjs simplifies making chartjs.org plots in go.
package chartjs

import (
	"bytes"
	"encoding/json"
	"fmt"
	"math"

	"github.com/brentp/go-chartjs/annotation"
	"github.com/brentp/go-chartjs/types"
)

var True = types.True
var False = types.False

var chartTypes = [...]string{
	"line",
	"bar",
	"bubble",
}

type chartType int

func (c chartType) MarshalJSON() ([]byte, error) {
	return []byte(`"` + chartTypes[c] + `"`), nil
}

const (
	// Line is a "line" plot
	Line chartType = iota
	// Bar is a "bar" plot
	Bar
	// Bubble is a "bubble" plot
	Bubble
)

type interpMode int

const (
	_ interpMode = iota
	InterpMonotone
	InterpDefault
)

var interpModes = [...]string{
	"",
	"monotone",
	"default",
}

func (m interpMode) MarshalJSON() ([]byte, error) {
	return []byte(`"` + interpModes[m] + `"`), nil
}

// XFloatFormat determines how many decimal places are sent in the JSON for X values.
var XFloatFormat = "%.2f"

// YFloatFormat determines how many decimal places are sent in the JSON for Y values.
var YFloatFormat = "%.2f"

// Values dictates the interface of data to be plotted.
type Values interface {
	// X-axis values. If only these are specified then it must be a Bar plot.
	Xs() []float64
	// Optional Y values.
	Ys() []float64
	// Rs are used to size points for chartType `Bubble`
	Rs() []float64
}

func marshalValuesJSON(v Values) ([]byte, error) {
	xs, ys, rs := v.Xs(), v.Ys(), v.Rs()
	if len(xs) == 0 {
		if len(rs) != 0 {
			return nil, fmt.Errorf("chart: bad format of Values data")
		}
		xs = ys[:len(ys)]
		ys = nil
	}
	buf := bytes.NewBuffer(make([]byte, 0, 8*len(xs)))
	buf.WriteRune('[')
	if len(rs) > 0 {
		if len(xs) != len(ys) || len(xs) != len(rs) {
			return nil, fmt.Errorf("chart: bad format of Values. All axes must be of the same length")
		}
		var err error
		for i, x := range xs {
			if i > 0 {
				buf.WriteRune(',')
			}
			y, r := ys[i], rs[i]
			if math.IsNaN(y) {
				_, err = buf.WriteString(fmt.Sprintf(("{\"x\":" + XFloatFormat + ",\"y\": null,\"r\":" + YFloatFormat + "}"), x, r))
			} else {
				_, err = buf.WriteString(fmt.Sprintf(("{\"x\":" + XFloatFormat + ",\"y\":" + YFloatFormat + ",\"r\":" + YFloatFormat + "}"), x, y, r))
			}
			if err != nil {
				return nil, err
			}
		}
	} else if len(ys) > 0 {
		if len(xs) != len(ys) {
			return nil, fmt.Errorf("chart: bad format of Values. X and Y must be of the same length")
		}
		var err error
		for i, x := range xs {
			if i > 0 {
				buf.WriteRune(',')
			}
			y := ys[i]
			if math.IsNaN(y) {
				_, err = buf.WriteString(fmt.Sprintf(("{\"x\":" + XFloatFormat + ",\"y\": null }"), x))
			} else {
				_, err = buf.WriteString(fmt.Sprintf(("{\"x\":" + XFloatFormat + ",\"y\":" + YFloatFormat + "}"), x, y))
			}
			if err != nil {
				return nil, err
			}
		}

	} else {
		for i, x := range xs {
			if i > 0 {
				buf.WriteRune(',')
			}
			_, err := buf.WriteString(fmt.Sprintf(XFloatFormat, x))
			if err != nil {
				return nil, err
			}
		}
	}

	buf.WriteRune(']')
	return buf.Bytes(), nil
}

// shape indicates the type of marker used for plotting.
type shape int

var shapes = []string{
	"",
	"circle",
	"triangle",
	"rect",
	"rectRot",
	"cross",
	"crossRot",
	"star",
	"line",
	"dash",
}

const (
	empty = iota
	Circle
	Triangle
	Rect
	RectRot
	Cross
	CrossRot
	Star
	LinePoint
	Dash
)

func (s shape) MarshalJSON() ([]byte, error) {
	return []byte(`"` + shapes[s] + `"`), nil
}

// Dataset wraps the "dataset" JSON
type Dataset struct {
	Data            Values      `json:"-"`
	Type            chartType   `json:"type,omitempty"`
	BackgroundColor *types.RGBA `json:"backgroundColor,omitempty"`
	// BorderColor is the color of the line.
	BorderColor *types.RGBA `json:"borderColor,omitempty"`
	// BorderWidth is the width of the line.
	BorderWidth int `json:"borderWidth,omitempty"`

	// Label indicates the name of the dataset to be shown in the legend.
	Label string     `json:"label,omitempty"`
	Fill  types.Bool `json:"fill,omitempty"`

	// SteppedLine of true means dont interpolate and ignore line tension.
	SteppedLine            types.Bool  `json:"steppedLine,omitempty"`
	LineTension            float64     `json:"lineTension,omitempty"`
	CubicInterpolationMode interpMode  `json:"cubicInterpolationMode,omitempty"`
	PointBackgroundColor   *types.RGBA `json:"pointBackgroundColor,omitempty"`
	PointBorderColor       *types.RGBA `json:"pointBorderColor,omitempty"`
	PointBorderWidth       int         `json:"pointBorderWidth,omitempty"`
	PointRadius            float64     `json:"pointRadius"`
	PointHoverBorderColor  *types.RGBA `json:"pointHoverBorderColor,omitempty"`
	PointStyle             shape       `json:"pointStyle,omitempty"`

	ShowLine types.Bool `json:"showLine,omitempty"`
	SpanGaps types.Bool `json:"spanGaps,omitempty"`

	// Axis ID that matches the ID on the Axis where this dataset is to be drawn.
	XAxisID string `json:"xAxisID,omitempty"`
	YAxisID string `json:"yAxisID,omitempty"`
}

// MarshalJSON implements json.Marshaler interface.
func (d Dataset) MarshalJSON() ([]byte, error) {
	o, err := marshalValuesJSON(d.Data)
	// avoid recursion by creating an alias.
	type alias Dataset
	buf, err := json.Marshal(alias(d))
	if err != nil {
		return nil, err
	}
	// replace '}' with ',' to continue struct
	if len(buf) > 0 {
		buf[len(buf)-1] = ','
	}
	buf = append(buf, []byte(`"data":`)...)
	buf = append(buf, o...)
	buf = append(buf, '}')
	return buf, nil
}

// Data wraps the "data" JSON
type Data struct {
	Datasets []Dataset `json:"datasets"`
	Labels   []string  `json:"labels"`
}

type axisType int

var axisTypes = []string{
	"category",
	"linear",
	"logarithmic",
	"time",
	"radialLinear",
}

const (
	// Category is a categorical axis (this is the default),
	// used for bar plots.
	Category axisType = iota
	// Linear axis should be use for scatter plots.
	Linear
	// Log axis
	Log
	// Time axis
	Time
	// Radial axis
	Radial
)

func (t axisType) MarshalJSON() ([]byte, error) {
	return []byte("\"" + axisTypes[t] + "\""), nil
}

type axisPosition int

const (
	// Bottom puts the axis on the bottom (used for Y-axis)
	Bottom axisPosition = iota + 1
	// Top puts the axis on the bottom (used for Y-axis)
	Top
	// Left puts the axis on the bottom (used for X-axis)
	Left
	// Right puts the axis on the bottom (used for X-axis)
	Right
)

var axisPositions = []string{
	"",
	"bottom",
	"top",
	"left",
	"right",
}

func (p axisPosition) MarshalJSON() ([]byte, error) {
	return []byte(`"` + axisPositions[p] + `"`), nil
}

// Axis corresponds to 'scale' in chart.js lingo.
type Axis struct {
	Type      axisType     `json:"type"`
	Position  axisPosition `json:"position,omitempty"`
	Label     string       `json:"label,omitempty"`
	ID        string       `json:"id,omitempty"`
	GridLines types.Bool   `json:"gridLine,omitempty"`
	Stacked   types.Bool   `json:"stacked,omitempty"`

	// Bool differentiates between false and empty by use of pointer.
	Display    types.Bool  `json:"display,omitempty"`
	ScaleLabel *ScaleLabel `json:"scaleLabel,omitempty"`
	Tick       *Tick       `json:"ticks,omitempty"`
}

// Tick lets us set the range of the data.
type Tick struct {
	Min         float64    `json:"min,omitempty"`
	Max         float64    `json:"max,omitempty"`
	BeginAtZero types.Bool `json:"beginAtZero,omitempty"`
	// TODO: add additional options from: tick options.
}

// ScaleLabel corresponds to scale title.
// Display: True must be specified for this to be shown.
type ScaleLabel struct {
	Display     types.Bool  `json:"display,omitempty"`
	LabelString string      `json:"labelString,omitempty"`
	FontColor   *types.RGBA `json:"fontColor,omitempty"`
	FontFamily  string      `json:"fontFamily,omitempty"`
	FontSize    int         `json:"fontSize,omitempty"`
	FontStyle   string      `json:"fontStyle,omitempty"`
}

// Axes holds the X and Y axies. Its simpler to use Chart.AddXAxis, Chart.AddYAxis.
type Axes struct {
	XAxes []Axis `json:"xAxes,omitempty"`
	YAxes []Axis `json:"yAxes,omitempty"`
}

// AddX adds a X-Axis.
func (a *Axes) AddX(x Axis) {
	a.XAxes = append(a.XAxes, x)
}

// AddY adds a Y-Axis.
func (a *Axes) AddY(y Axis) {
	a.YAxes = append(a.YAxes, y)
}

// Option wraps the chartjs "option"
type Option struct {
	Responsive          types.Bool `json:"responsive,omitempty"`
	MaintainAspectRatio types.Bool `json:"maintainAspectRatio,omitempty"`
}

type Annotation struct {
	Annotations []annotation.Annotation `json:"annotations,omitempty"`
}

// Options wraps the chartjs "options"
type Options struct {
	Option
	Scales     Axes       `json:"scales,omitempty"`
	Annotation Annotation `json:"annotation,omitempty"`
	Legend     *Legend    `json:"legend,omitempty"`
}

type Legend struct {
	Display types.Bool `json:"display,omitempty"`
}

// Chart is the top-level type from chartjs.
type Chart struct {
	Type    chartType `json:"type"`
	Label   string    `json:"label,omitempty"`
	Data    Data      `json:"data,omitempty"`
	Options Options   `json:"options,omitempty"`
}

// AddDataset adds a dataset to the chart.
func (c *Chart) AddDataset(d Dataset) {
	c.Data.Datasets = append(c.Data.Datasets, d)
}

// AddXAxis adds an x-axis to the chart and returns the ID of the added axis.
func (c *Chart) AddXAxis(x Axis) (string, error) {
	if x.ID == "" {
		x.ID = fmt.Sprintf("xaxis%d", len(c.Options.Scales.XAxes))
	}
	if x.Position == Left || x.Position == Right {
		return "", fmt.Errorf("chart: added x-axis to left or right")
	}
	c.Options.Scales.XAxes = append(c.Options.Scales.XAxes, x)
	return x.ID, nil
}

// AddYAxis adds an y-axis to the chart and return the ID of the added axis.
func (c *Chart) AddYAxis(y Axis) (string, error) {
	if y.ID == "" {
		y.ID = fmt.Sprintf("yaxis%d", len(c.Options.Scales.YAxes))
	}
	if y.Position == Top || y.Position == Bottom {
		return "", fmt.Errorf("chart: added y-axis to top or bottom")
	}
	c.Options.Scales.YAxes = append(c.Options.Scales.YAxes, y)
	return y.ID, nil
}
<commit_msg>allow Widths to be float<commit_after>// Package chartjs simplifies making chartjs.org plots in go.
package chartjs

import (
	"bytes"
	"encoding/json"
	"fmt"
	"math"

	"github.com/brentp/go-chartjs/annotation"
	"github.com/brentp/go-chartjs/types"
)

var True = types.True
var False = types.False

var chartTypes = [...]string{
	"line",
	"bar",
	"bubble",
}

type chartType int

func (c chartType) MarshalJSON() ([]byte, error) {
	return []byte(`"` + chartTypes[c] + `"`), nil
}

const (
	// Line is a "line" plot
	Line chartType = iota
	// Bar is a "bar" plot
	Bar
	// Bubble is a "bubble" plot
	Bubble
)

type interpMode int

const (
	_ interpMode = iota
	InterpMonotone
	InterpDefault
)

var interpModes = [...]string{
	"",
	"monotone",
	"default",
}

func (m interpMode) MarshalJSON() ([]byte, error) {
	return []byte(`"` + interpModes[m] + `"`), nil
}

// XFloatFormat determines how many decimal places are sent in the JSON for X values.
var XFloatFormat = "%.2f"

// YFloatFormat determines how many decimal places are sent in the JSON for Y values.
var YFloatFormat = "%.2f"

// Values dictates the interface of data to be plotted.
type Values interface {
	// X-axis values. If only these are specified then it must be a Bar plot.
	Xs() []float64
	// Optional Y values.
	Ys() []float64
	// Rs are used to size points for chartType `Bubble`
	Rs() []float64
}

func marshalValuesJSON(v Values) ([]byte, error) {
	xs, ys, rs := v.Xs(), v.Ys(), v.Rs()
	if len(xs) == 0 {
		if len(rs) != 0 {
			return nil, fmt.Errorf("chart: bad format of Values data")
		}
		xs = ys[:len(ys)]
		ys = nil
	}
	buf := bytes.NewBuffer(make([]byte, 0, 8*len(xs)))
	buf.WriteRune('[')
	if len(rs) > 0 {
		if len(xs) != len(ys) || len(xs) != len(rs) {
			return nil, fmt.Errorf("chart: bad format of Values. All axes must be of the same length")
		}
		var err error
		for i, x := range xs {
			if i > 0 {
				buf.WriteRune(',')
			}
			y, r := ys[i], rs[i]
			if math.IsNaN(y) {
				_, err = buf.WriteString(fmt.Sprintf(("{\"x\":" + XFloatFormat + ",\"y\": null,\"r\":" + YFloatFormat + "}"), x, r))
			} else {
				_, err = buf.WriteString(fmt.Sprintf(("{\"x\":" + XFloatFormat + ",\"y\":" + YFloatFormat + ",\"r\":" + YFloatFormat + "}"), x, y, r))
			}
			if err != nil {
				return nil, err
			}
		}
	} else if len(ys) > 0 {
		if len(xs) != len(ys) {
			return nil, fmt.Errorf("chart: bad format of Values. X and Y must be of the same length")
		}
		var err error
		for i, x := range xs {
			if i > 0 {
				buf.WriteRune(',')
			}
			y := ys[i]
			if math.IsNaN(y) {
				_, err = buf.WriteString(fmt.Sprintf(("{\"x\":" + XFloatFormat + ",\"y\": null }"), x))
			} else {
				_, err = buf.WriteString(fmt.Sprintf(("{\"x\":" + XFloatFormat + ",\"y\":" + YFloatFormat + "}"), x, y))
			}
			if err != nil {
				return nil, err
			}
		}

	} else {
		for i, x := range xs {
			if i > 0 {
				buf.WriteRune(',')
			}
			_, err := buf.WriteString(fmt.Sprintf(XFloatFormat, x))
			if err != nil {
				return nil, err
			}
		}
	}

	buf.WriteRune(']')
	return buf.Bytes(), nil
}

// shape indicates the type of marker used for plotting.
type shape int

var shapes = []string{
	"",
	"circle",
	"triangle",
	"rect",
	"rectRot",
	"cross",
	"crossRot",
	"star",
	"line",
	"dash",
}

const (
	empty = iota
	Circle
	Triangle
	Rect
	RectRot
	Cross
	CrossRot
	Star
	LinePoint
	Dash
)

func (s shape) MarshalJSON() ([]byte, error) {
	return []byte(`"` + shapes[s] + `"`), nil
}

// Dataset wraps the "dataset" JSON
type Dataset struct {
	Data            Values      `json:"-"`
	Type            chartType   `json:"type,omitempty"`
	BackgroundColor *types.RGBA `json:"backgroundColor,omitempty"`
	// BorderColor is the color of the line.
	BorderColor *types.RGBA `json:"borderColor,omitempty"`
	// BorderWidth is the width of the line.
	BorderWidth float64 `json:"borderWidth,omitempty"`

	// Label indicates the name of the dataset to be shown in the legend.
	Label string     `json:"label,omitempty"`
	Fill  types.Bool `json:"fill,omitempty"`

	// SteppedLine of true means dont interpolate and ignore line tension.
	SteppedLine            types.Bool  `json:"steppedLine,omitempty"`
	LineTension            float64     `json:"lineTension,omitempty"`
	CubicInterpolationMode interpMode  `json:"cubicInterpolationMode,omitempty"`
	PointBackgroundColor   *types.RGBA `json:"pointBackgroundColor,omitempty"`
	PointBorderColor       *types.RGBA `json:"pointBorderColor,omitempty"`
	PointBorderWidth       float64     `json:"pointBorderWidth,omitempty"`
	PointRadius            float64     `json:"pointRadius"`
	PointHoverBorderColor  *types.RGBA `json:"pointHoverBorderColor,omitempty"`
	PointStyle             shape       `json:"pointStyle,omitempty"`

	ShowLine types.Bool `json:"showLine,omitempty"`
	SpanGaps types.Bool `json:"spanGaps,omitempty"`

	// Axis ID that matches the ID on the Axis where this dataset is to be drawn.
	XAxisID string `json:"xAxisID,omitempty"`
	YAxisID string `json:"yAxisID,omitempty"`
}

// MarshalJSON implements json.Marshaler interface.
func (d Dataset) MarshalJSON() ([]byte, error) {
	o, err := marshalValuesJSON(d.Data)
	// avoid recursion by creating an alias.
	type alias Dataset
	buf, err := json.Marshal(alias(d))
	if err != nil {
		return nil, err
	}
	// replace '}' with ',' to continue struct
	if len(buf) > 0 {
		buf[len(buf)-1] = ','
	}
	buf = append(buf, []byte(`"data":`)...)
	buf = append(buf, o...)
	buf = append(buf, '}')
	return buf, nil
}

// Data wraps the "data" JSON
type Data struct {
	Datasets []Dataset `json:"datasets"`
	Labels   []string  `json:"labels"`
}

type axisType int

var axisTypes = []string{
	"category",
	"linear",
	"logarithmic",
	"time",
	"radialLinear",
}

const (
	// Category is a categorical axis (this is the default),
	// used for bar plots.
	Category axisType = iota
	// Linear axis should be use for scatter plots.
	Linear
	// Log axis
	Log
	// Time axis
	Time
	// Radial axis
	Radial
)

func (t axisType) MarshalJSON() ([]byte, error) {
	return []byte("\"" + axisTypes[t] + "\""), nil
}

type axisPosition int

const (
	// Bottom puts the axis on the bottom (used for Y-axis)
	Bottom axisPosition = iota + 1
	// Top puts the axis on the bottom (used for Y-axis)
	Top
	// Left puts the axis on the bottom (used for X-axis)
	Left
	// Right puts the axis on the bottom (used for X-axis)
	Right
)

var axisPositions = []string{
	"",
	"bottom",
	"top",
	"left",
	"right",
}

func (p axisPosition) MarshalJSON() ([]byte, error) {
	return []byte(`"` + axisPositions[p] + `"`), nil
}

// Axis corresponds to 'scale' in chart.js lingo.
type Axis struct {
	Type      axisType     `json:"type"`
	Position  axisPosition `json:"position,omitempty"`
	Label     string       `json:"label,omitempty"`
	ID        string       `json:"id,omitempty"`
	GridLines types.Bool   `json:"gridLine,omitempty"`
	Stacked   types.Bool   `json:"stacked,omitempty"`

	// Bool differentiates between false and empty by use of pointer.
	Display    types.Bool  `json:"display,omitempty"`
	ScaleLabel *ScaleLabel `json:"scaleLabel,omitempty"`
	Tick       *Tick       `json:"ticks,omitempty"`
}

// Tick lets us set the range of the data.
type Tick struct {
	Min         float64    `json:"min,omitempty"`
	Max         float64    `json:"max,omitempty"`
	BeginAtZero types.Bool `json:"beginAtZero,omitempty"`
	// TODO: add additional options from: tick options.
}

// ScaleLabel corresponds to scale title.
// Display: True must be specified for this to be shown.
type ScaleLabel struct {
	Display     types.Bool  `json:"display,omitempty"`
	LabelString string      `json:"labelString,omitempty"`
	FontColor   *types.RGBA `json:"fontColor,omitempty"`
	FontFamily  string      `json:"fontFamily,omitempty"`
	FontSize    int         `json:"fontSize,omitempty"`
	FontStyle   string      `json:"fontStyle,omitempty"`
}

// Axes holds the X and Y axies. Its simpler to use Chart.AddXAxis, Chart.AddYAxis.
type Axes struct {
	XAxes []Axis `json:"xAxes,omitempty"`
	YAxes []Axis `json:"yAxes,omitempty"`
}

// AddX adds a X-Axis.
func (a *Axes) AddX(x Axis) {
	a.XAxes = append(a.XAxes, x)
}

// AddY adds a Y-Axis.
func (a *Axes) AddY(y Axis) {
	a.YAxes = append(a.YAxes, y)
}

// Option wraps the chartjs "option"
type Option struct {
	Responsive          types.Bool `json:"responsive,omitempty"`
	MaintainAspectRatio types.Bool `json:"maintainAspectRatio,omitempty"`
}

type Annotation struct {
	Annotations []annotation.Annotation `json:"annotations,omitempty"`
}

// Options wraps the chartjs "options"
type Options struct {
	Option
	Scales     Axes       `json:"scales,omitempty"`
	Annotation Annotation `json:"annotation,omitempty"`
	Legend     *Legend    `json:"legend,omitempty"`
}

type Legend struct {
	Display types.Bool `json:"display,omitempty"`
}

// Chart is the top-level type from chartjs.
type Chart struct {
	Type    chartType `json:"type"`
	Label   string    `json:"label,omitempty"`
	Data    Data      `json:"data,omitempty"`
	Options Options   `json:"options,omitempty"`
}

// AddDataset adds a dataset to the chart.
func (c *Chart) AddDataset(d Dataset) {
	c.Data.Datasets = append(c.Data.Datasets, d)
}

// AddXAxis adds an x-axis to the chart and returns the ID of the added axis.
func (c *Chart) AddXAxis(x Axis) (string, error) {
	if x.ID == "" {
		x.ID = fmt.Sprintf("xaxis%d", len(c.Options.Scales.XAxes))
	}
	if x.Position == Left || x.Position == Right {
		return "", fmt.Errorf("chart: added x-axis to left or right")
	}
	c.Options.Scales.XAxes = append(c.Options.Scales.XAxes, x)
	return x.ID, nil
}

// AddYAxis adds an y-axis to the chart and return the ID of the added axis.
func (c *Chart) AddYAxis(y Axis) (string, error) {
	if y.ID == "" {
		y.ID = fmt.Sprintf("yaxis%d", len(c.Options.Scales.YAxes))
	}
	if y.Position == Top || y.Position == Bottom {
		return "", fmt.Errorf("chart: added y-axis to top or bottom")
	}
	c.Options.Scales.YAxes = append(c.Options.Scales.YAxes, y)
	return y.ID, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bigquery

import (
	"errors"
	"fmt"
	"reflect"

	"golang.org/x/net/context"
	bq "google.golang.org/api/bigquery/v2"
)

// An Uploader does streaming inserts into a BigQuery table.
// It is safe for concurrent use.
type Uploader struct {
	t *Table

	// SkipInvalidRows causes rows containing invalid data to be silently
	// ignored. The default value is false, which causes the entire request to
	// fail if there is an attempt to insert an invalid row.
	SkipInvalidRows bool

	// IgnoreUnknownValues causes values not matching the schema to be ignored.
	// The default value is false, which causes records containing such values
	// to be treated as invalid records.
	IgnoreUnknownValues bool

	// A TableTemplateSuffix allows Uploaders to create tables automatically.
	//
	// Experimental: this option is experimental and may be modified or removed in future versions,
	// regardless of any other documented package stability guarantees.
	//
	// When you specify a suffix, the table you upload data to
	// will be used as a template for creating a new table, with the same schema,
	// called <table> + <suffix>.
	//
	// More information is available at
	// https://cloud.google.com/bigquery/streaming-data-into-bigquery#template-tables
	TableTemplateSuffix string
}

// Uploader returns an Uploader that can be used to append rows to t.
// The returned Uploader may optionally be further configured before its Put method is called.
func (t *Table) Uploader() *Uploader {
	return &Uploader{t: t}
}

// Put uploads one or more rows to the BigQuery service.
//
// If src is ValueSaver, then its Save method is called to produce a row for uploading.
//
// If src is a struct or pointer to a struct, then a schema is inferred from it
// and used to create a StructSaver. The InsertID of the StructSaver will be
// empty.
//
// If src is a slice of ValueSavers, structs, or struct pointers, then each
// element of the slice is treated as above, and multiple rows are uploaded.
//
// Put returns a PutMultiError if one or more rows failed to be uploaded.
// The PutMultiError contains a RowInsertionError for each failed row.
//
// Put will retry on temporary errors (see
// https://cloud.google.com/bigquery/troubleshooting-errors). This can result
// in duplicate rows if you do not use insert IDs. Also, if the error persists,
// the call will run indefinitely. Pass a context with a timeout to prevent
// hanging calls.
func (u *Uploader) Put(ctx context.Context, src interface{}) error {
	savers, err := valueSavers(src)
	if err != nil {
		return err
	}
	return u.putMulti(ctx, savers)
}

func valueSavers(src interface{}) ([]ValueSaver, error) {
	saver, ok, err := toValueSaver(src)
	if err != nil {
		return nil, err
	}
	if ok {
		return []ValueSaver{saver}, nil
	}
	srcVal := reflect.ValueOf(src)
	if srcVal.Kind() != reflect.Slice {
		return nil, fmt.Errorf("%T is not a ValueSaver, struct, struct pointer, or slice", src)

	}
	var savers []ValueSaver
	for i := 0; i < srcVal.Len(); i++ {
		s := srcVal.Index(i).Interface()
		saver, ok, err := toValueSaver(s)
		if err != nil {
			return nil, err
		}
		if !ok {
			return nil, fmt.Errorf("src[%d] has type %T, which is not a ValueSaver, struct or struct pointer", i, s)
		}
		savers = append(savers, saver)
	}
	return savers, nil
}

// Make a ValueSaver from x, which must implement ValueSaver already
// or be a struct or pointer to struct.
func toValueSaver(x interface{}) (ValueSaver, bool, error) {
	if _, ok := x.(StructSaver); ok {
		return nil, false, errors.New("bigquery: use &StructSaver, not StructSaver")
	}
	var insertID string
	// Handle StructSavers specially so we can infer the schema if necessary.
	if ss, ok := x.(*StructSaver); ok && ss.Schema == nil {
		x = ss.Struct
		insertID = ss.InsertID
		// Fall through so we can infer the schema.
	}
	if saver, ok := x.(ValueSaver); ok {
		return saver, ok, nil
	}
	v := reflect.ValueOf(x)
	// Support Put with []interface{}
	if v.Kind() == reflect.Interface {
		v = v.Elem()
	}
	if v.Kind() == reflect.Ptr {
		v = v.Elem()
	}
	if v.Kind() != reflect.Struct {
		return nil, false, nil
	}
	schema, err := inferSchemaReflectCached(v.Type())
	if err != nil {
		return nil, false, err
	}
	return &StructSaver{
		Struct:   x,
		InsertID: insertID,
		Schema:   schema,
	}, true, nil
}

func (u *Uploader) putMulti(ctx context.Context, src []ValueSaver) error {
	req, err := u.newInsertRequest(src)
	if err != nil {
		return err
	}
	if req == nil {
		return nil
	}
	call := u.t.c.bqs.Tabledata.InsertAll(u.t.ProjectID, u.t.DatasetID, u.t.TableID, req)
	call = call.Context(ctx)
	setClientHeader(call.Header())
	var res *bq.TableDataInsertAllResponse
	err = runWithRetry(ctx, func() (err error) {
		res, err = call.Do()
		return err
	})
	if err != nil {
		return err
	}
	return handleInsertErrors(res.InsertErrors, req.Rows)
}

func (u *Uploader) newInsertRequest(savers []ValueSaver) (*bq.TableDataInsertAllRequest, error) {
	if savers == nil { // If there are no rows, do nothing.
		return nil, nil
	}
	req := &bq.TableDataInsertAllRequest{
		TemplateSuffix:      u.TableTemplateSuffix,
		IgnoreUnknownValues: u.IgnoreUnknownValues,
		SkipInvalidRows:     u.SkipInvalidRows,
	}
	for _, saver := range savers {
		row, insertID, err := saver.Save()
		if err != nil {
			return nil, err
		}
		if insertID == "" {
			insertID = randomIDFn()
		}
		m := make(map[string]bq.JsonValue)
		for k, v := range row {
			m[k] = bq.JsonValue(v)
		}
		req.Rows = append(req.Rows, &bq.TableDataInsertAllRequestRows{
			InsertId: insertID,
			Json:     m,
		})
	}
	return req, nil
}

func handleInsertErrors(ierrs []*bq.TableDataInsertAllResponseInsertErrors, rows []*bq.TableDataInsertAllRequestRows) error {
	if len(ierrs) == 0 {
		return nil
	}
	var errs PutMultiError
	for _, e := range ierrs {
		if int(e.Index) > len(rows) {
			return fmt.Errorf("internal error: unexpected row index: %v", e.Index)
		}
		rie := RowInsertionError{
			InsertID: rows[e.Index].InsertId,
			RowIndex: int(e.Index),
		}
		for _, errp := range e.Errors {
			rie.Errors = append(rie.Errors, bqToError(errp))
		}
		errs = append(errs, rie)
	}
	return errs
}
<commit_msg>bigquery: document how to stream into date-partitioned tables.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bigquery

import (
	"errors"
	"fmt"
	"reflect"

	"golang.org/x/net/context"
	bq "google.golang.org/api/bigquery/v2"
)

// An Uploader does streaming inserts into a BigQuery table.
// It is safe for concurrent use.
type Uploader struct {
	t *Table

	// SkipInvalidRows causes rows containing invalid data to be silently
	// ignored. The default value is false, which causes the entire request to
	// fail if there is an attempt to insert an invalid row.
	SkipInvalidRows bool

	// IgnoreUnknownValues causes values not matching the schema to be ignored.
	// The default value is false, which causes records containing such values
	// to be treated as invalid records.
	IgnoreUnknownValues bool

	// A TableTemplateSuffix allows Uploaders to create tables automatically.
	//
	// Experimental: this option is experimental and may be modified or removed in future versions,
	// regardless of any other documented package stability guarantees.
	//
	// When you specify a suffix, the table you upload data to
	// will be used as a template for creating a new table, with the same schema,
	// called <table> + <suffix>.
	//
	// More information is available at
	// https://cloud.google.com/bigquery/streaming-data-into-bigquery#template-tables
	TableTemplateSuffix string
}

// Uploader returns an Uploader that can be used to append rows to t.
// The returned Uploader may optionally be further configured before its Put method is called.
//
// To stream rows into a date-partitioned table at a particular date, add the
// $yyyymmdd suffix to the table name when constructing the Table.
func (t *Table) Uploader() *Uploader {
	return &Uploader{t: t}
}

// Put uploads one or more rows to the BigQuery service.
//
// If src is ValueSaver, then its Save method is called to produce a row for uploading.
//
// If src is a struct or pointer to a struct, then a schema is inferred from it
// and used to create a StructSaver. The InsertID of the StructSaver will be
// empty.
//
// If src is a slice of ValueSavers, structs, or struct pointers, then each
// element of the slice is treated as above, and multiple rows are uploaded.
//
// Put returns a PutMultiError if one or more rows failed to be uploaded.
// The PutMultiError contains a RowInsertionError for each failed row.
//
// Put will retry on temporary errors (see
// https://cloud.google.com/bigquery/troubleshooting-errors). This can result
// in duplicate rows if you do not use insert IDs. Also, if the error persists,
// the call will run indefinitely. Pass a context with a timeout to prevent
// hanging calls.
func (u *Uploader) Put(ctx context.Context, src interface{}) error {
	savers, err := valueSavers(src)
	if err != nil {
		return err
	}
	return u.putMulti(ctx, savers)
}

func valueSavers(src interface{}) ([]ValueSaver, error) {
	saver, ok, err := toValueSaver(src)
	if err != nil {
		return nil, err
	}
	if ok {
		return []ValueSaver{saver}, nil
	}
	srcVal := reflect.ValueOf(src)
	if srcVal.Kind() != reflect.Slice {
		return nil, fmt.Errorf("%T is not a ValueSaver, struct, struct pointer, or slice", src)

	}
	var savers []ValueSaver
	for i := 0; i < srcVal.Len(); i++ {
		s := srcVal.Index(i).Interface()
		saver, ok, err := toValueSaver(s)
		if err != nil {
			return nil, err
		}
		if !ok {
			return nil, fmt.Errorf("src[%d] has type %T, which is not a ValueSaver, struct or struct pointer", i, s)
		}
		savers = append(savers, saver)
	}
	return savers, nil
}

// Make a ValueSaver from x, which must implement ValueSaver already
// or be a struct or pointer to struct.
func toValueSaver(x interface{}) (ValueSaver, bool, error) {
	if _, ok := x.(StructSaver); ok {
		return nil, false, errors.New("bigquery: use &StructSaver, not StructSaver")
	}
	var insertID string
	// Handle StructSavers specially so we can infer the schema if necessary.
	if ss, ok := x.(*StructSaver); ok && ss.Schema == nil {
		x = ss.Struct
		insertID = ss.InsertID
		// Fall through so we can infer the schema.
	}
	if saver, ok := x.(ValueSaver); ok {
		return saver, ok, nil
	}
	v := reflect.ValueOf(x)
	// Support Put with []interface{}
	if v.Kind() == reflect.Interface {
		v = v.Elem()
	}
	if v.Kind() == reflect.Ptr {
		v = v.Elem()
	}
	if v.Kind() != reflect.Struct {
		return nil, false, nil
	}
	schema, err := inferSchemaReflectCached(v.Type())
	if err != nil {
		return nil, false, err
	}
	return &StructSaver{
		Struct:   x,
		InsertID: insertID,
		Schema:   schema,
	}, true, nil
}

func (u *Uploader) putMulti(ctx context.Context, src []ValueSaver) error {
	req, err := u.newInsertRequest(src)
	if err != nil {
		return err
	}
	if req == nil {
		return nil
	}
	call := u.t.c.bqs.Tabledata.InsertAll(u.t.ProjectID, u.t.DatasetID, u.t.TableID, req)
	call = call.Context(ctx)
	setClientHeader(call.Header())
	var res *bq.TableDataInsertAllResponse
	err = runWithRetry(ctx, func() (err error) {
		res, err = call.Do()
		return err
	})
	if err != nil {
		return err
	}
	return handleInsertErrors(res.InsertErrors, req.Rows)
}

func (u *Uploader) newInsertRequest(savers []ValueSaver) (*bq.TableDataInsertAllRequest, error) {
	if savers == nil { // If there are no rows, do nothing.
		return nil, nil
	}
	req := &bq.TableDataInsertAllRequest{
		TemplateSuffix:      u.TableTemplateSuffix,
		IgnoreUnknownValues: u.IgnoreUnknownValues,
		SkipInvalidRows:     u.SkipInvalidRows,
	}
	for _, saver := range savers {
		row, insertID, err := saver.Save()
		if err != nil {
			return nil, err
		}
		if insertID == "" {
			insertID = randomIDFn()
		}
		m := make(map[string]bq.JsonValue)
		for k, v := range row {
			m[k] = bq.JsonValue(v)
		}
		req.Rows = append(req.Rows, &bq.TableDataInsertAllRequestRows{
			InsertId: insertID,
			Json:     m,
		})
	}
	return req, nil
}

func handleInsertErrors(ierrs []*bq.TableDataInsertAllResponseInsertErrors, rows []*bq.TableDataInsertAllRequestRows) error {
	if len(ierrs) == 0 {
		return nil
	}
	var errs PutMultiError
	for _, e := range ierrs {
		if int(e.Index) > len(rows) {
			return fmt.Errorf("internal error: unexpected row index: %v", e.Index)
		}
		rie := RowInsertionError{
			InsertID: rows[e.Index].InsertId,
			RowIndex: int(e.Index),
		}
		for _, errp := range e.Errors {
			rie.Errors = append(rie.Errors, bqToError(errp))
		}
		errs = append(errs, rie)
	}
	return errs
}
<|endoftext|>


<commit_before>package algoliaconnector

import (
	"encoding/json"
	"errors"
	"fmt"
	"socialapi/models"
	"socialapi/request"
	"strconv"
	"time"

	"github.com/algolia/algoliasearch-client-go/algoliasearch"
	"github.com/koding/logging"
	"github.com/streadway/amqp"
)

var (
	ErrAlgoliaObjectIdNotFoundMsg = "ObjectID does not exist"
	ErrAlgoliaIndexNotExistMsg    = "Index messages.test does not exist"
)

type IndexSet map[string]*algoliasearch.Index

type Controller struct {
	log             logging.Logger
	client          *algoliasearch.Client
	indexes         *IndexSet
	kodingChannelId string
}

// IsAlgoliaError checks if the given algolia error string and given messages
// are same according their data structure
func IsAlgoliaError(err error, message string) bool {
	if err == nil {
		return false
	}

	v := &algoliaErrorRes{}

	if err := json.Unmarshal([]byte(err.Error()), v); err != nil {
		return false
	}

	if v.Message == message {
		return true
	}

	return false
}

type algoliaErrorRes struct {
	Message string `json:"message"`
	Status  int    `json:"status"`
}

func (i *IndexSet) Get(name string) (*algoliasearch.Index, error) {
	index, ok := (*i)[name]
	if !ok {
		return nil, fmt.Errorf("Unknown index: '%s'", name)
	}
	return index, nil
}

func (c *Controller) DefaultErrHandler(delivery amqp.Delivery, err error) bool {
	c.log.Error(err.Error())
	return false
}

func New(log logging.Logger, client *algoliasearch.Client, indexSuffix string) *Controller {
	// TODO later on listen channel_participant_added event and remove this koding channel fetch
	c := models.NewChannel()
	q := request.NewQuery()
	q.GroupName = "koding"
	q.Name = "public"
	q.Type = models.Channel_TYPE_GROUP

	channel, err := c.ByName(q)
	if err != nil {
		log.Error("Could not fetch koding channel: %s:", err)
	}
	var channelId string
	if channel.Id != 0 {
		channelId = strconv.FormatInt(channel.Id, 10)
	}

	return &Controller{
		log:    log,
		client: client,
		indexes: &IndexSet{
			"topics":   client.InitIndex("topics" + indexSuffix),
			"accounts": client.InitIndex("accounts" + indexSuffix),
			"messages": client.InitIndex("messages" + indexSuffix),
		},
		kodingChannelId: channelId,
	}
}

func (f *Controller) TopicSaved(data *models.Channel) error {
	if data.TypeConstant != models.Channel_TYPE_TOPIC {
		return nil
	}
	return f.insert("topics", map[string]interface{}{
		"objectID": strconv.FormatInt(data.Id, 10),
		"name":     data.Name,
		"purpose":  data.Purpose,
	})
}

func (f *Controller) AccountSaved(data *models.Account) error {
	return f.insert("accounts", map[string]interface{}{
		"objectID": data.OldId,
		"nick":     data.Nick,
		"_tags":    []string{f.kodingChannelId},
	})
}

func (f *Controller) MessageListSaved(listing *models.ChannelMessageList) error {
	message := models.NewChannelMessage()

	if err := message.ById(listing.MessageId); err != nil {
		return err
	}

	// no need to index join/leave messages
	if message.TypeConstant != models.ChannelMessage_TYPE_POST &&
		message.TypeConstant != models.ChannelMessage_TYPE_REPLY {
		return nil
	}

	objectId := strconv.FormatInt(message.Id, 10)
	channelId := strconv.FormatInt(listing.ChannelId, 10)

	record, err := f.get("messages", objectId)
	if err != nil &&
		!IsAlgoliaError(err, ErrAlgoliaObjectIdNotFoundMsg) &&
		!IsAlgoliaError(err, ErrAlgoliaIndexNotExistMsg) {
		return err
	}

	if record == nil {
		return f.insert("messages", map[string]interface{}{
			"objectID": objectId,
			"body":     message.Body,
			"_tags":    []string{channelId},
		})
	}

	return f.partialUpdate("messages", map[string]interface{}{
		"objectID": objectId,
		"_tags":    appendTag(record, channelId),
	})
}

func (f *Controller) MessageListDeleted(listing *models.ChannelMessageList) error {
	index, err := f.indexes.Get("messages")
	if err != nil {
		return err
	}

	objectId := strconv.FormatInt(listing.MessageId, 10)

	record, err := f.get("messages", objectId)
	if err != nil &&
		!IsAlgoliaError(err, ErrAlgoliaObjectIdNotFoundMsg) &&
		!IsAlgoliaError(err, ErrAlgoliaIndexNotExistMsg) {
		return err
	}

	if tags, ok := record["_tags"]; ok {
		if t, ok := tags.([]interface{}); ok && len(t) == 1 {
			if _, err = index.DeleteObject(objectId); err != nil {
				return err
			}
			return nil
		}
	}

	return f.partialUpdate("messages", map[string]interface{}{
		"objectID": objectId,
		"_tags":    removeMessageTag(record, strconv.FormatInt(listing.ChannelId, 10)),
	})
}

func (f *Controller) MessageUpdated(message *models.ChannelMessage) error {
	return f.partialUpdate("messages", map[string]interface{}{
		"objectID": strconv.FormatInt(message.Id, 10),
		"body":     message.Body,
	})
}

const accountIndexName = "accounts"

// ParticipantDeleted operates with the participant deleted events, removes
// deleted tag from algolia document
func (f *Controller) ParticipantUpdated(p *models.ChannelParticipant) error {
	// if status of the participant is left, then just notify the current user
	if p.StatusConstant == models.ChannelParticipant_STATUS_LEFT {
		return f.handleParticipantOperation(p, removeTag)
	}

	if p.StatusConstant == models.ChannelParticipant_STATUS_ACTIVE {
		return f.handleParticipantOperation(p, appendTag)
	}

	f.log.Debug("ignoring the error status: %s", p.StatusConstant)

	return nil
}

// ParticipantCreated operates with the participant createad event, adds new
// tag to the algolia document
func (f *Controller) ParticipantCreated(p *models.ChannelParticipant) error {
	err := f.handleParticipantOperation(p, appendTag)
	if err != nil {
		f.log.Error("err while handling participant created event: %s", err.Error())
	}

	return err
}

func (f *Controller) handleParticipantOperation(p *models.ChannelParticipant, tagOperator func(record map[string]interface{}, channelId string) []interface{}) error {
	if p.ChannelId == 0 {
		return nil
	}

	if p.AccountId == 0 {
		return nil
	}

	a := models.NewAccount()
	if err := a.ById(p.AccountId); err != nil {
		f.log.Error("err while fetching account: %s", err.Error())
		return nil
	}

	if a.Id == 0 {
		f.log.Critical("account found but id is 0 %+v", a)
		return nil
	}

	record, err := f.get(accountIndexName, a.OldId)
	if err != nil &&
		!IsAlgoliaError(err, ErrAlgoliaObjectIdNotFoundMsg) &&
		!IsAlgoliaError(err, ErrAlgoliaIndexNotExistMsg) {
		return err
	}

	if record == nil {
		// first create the account
		if err := f.AccountSaved(a); err != nil {
			return err
		}

		// make sure account is there, before start processing it
		err := makeSureAccount(f, a.OldId, func(record map[string]interface{}, err error) bool {
			if err != nil {
				return false
			}

			if record == nil {
				return false
			}

			return true
		})
		if err != nil {
			return err
		}
	}

	record, err = f.get(accountIndexName, a.OldId)
	if err != nil {
		return err
	}

	channelId := strconv.FormatInt(p.ChannelId, 10)

	return f.partialUpdate(accountIndexName, map[string]interface{}{
		"objectID": a.OldId,
		"_tags":    tagOperator(record, channelId),
	})
}

var errDeadline = errors.New("deadline reached")

// makeSureAccount checks if the given id's get request returns the desired
// err, it will re-try every 100ms until deadline of 2 minutes reached. Algolia
// doesnt index the records right away, so try to go to a desired state
func makeSureAccount(handler *Controller, id string, f func(map[string]interface{}, error) bool) error {
	deadLine := time.After(time.Minute * 2)
	tick := time.Tick(time.Millisecond * 100)
	for {
		select {
		case <-tick:
			record, err := handler.get("accounts", id)
			if f(record, err) {
				return nil
			}
		case <-deadLine:
			return errDeadline
		}
	}
}
<commit_msg>Socialapi: fix docs<commit_after>package algoliaconnector

import (
	"encoding/json"
	"errors"
	"fmt"
	"socialapi/models"
	"socialapi/request"
	"strconv"
	"time"

	"github.com/algolia/algoliasearch-client-go/algoliasearch"
	"github.com/koding/logging"
	"github.com/streadway/amqp"
)

var (
	ErrAlgoliaObjectIdNotFoundMsg = "ObjectID does not exist"
	ErrAlgoliaIndexNotExistMsg    = "Index messages.test does not exist"
)

type IndexSet map[string]*algoliasearch.Index

type Controller struct {
	log             logging.Logger
	client          *algoliasearch.Client
	indexes         *IndexSet
	kodingChannelId string
}

// IsAlgoliaError checks if the given algolia error string and given messages
// are same according their data structure
func IsAlgoliaError(err error, message string) bool {
	if err == nil {
		return false
	}

	v := &algoliaErrorRes{}

	if err := json.Unmarshal([]byte(err.Error()), v); err != nil {
		return false
	}

	if v.Message == message {
		return true
	}

	return false
}

type algoliaErrorRes struct {
	Message string `json:"message"`
	Status  int    `json:"status"`
}

func (i *IndexSet) Get(name string) (*algoliasearch.Index, error) {
	index, ok := (*i)[name]
	if !ok {
		return nil, fmt.Errorf("Unknown index: '%s'", name)
	}
	return index, nil
}

func (c *Controller) DefaultErrHandler(delivery amqp.Delivery, err error) bool {
	c.log.Error(err.Error())
	return false
}

func New(log logging.Logger, client *algoliasearch.Client, indexSuffix string) *Controller {
	// TODO later on listen channel_participant_added event and remove this koding channel fetch
	c := models.NewChannel()
	q := request.NewQuery()
	q.GroupName = "koding"
	q.Name = "public"
	q.Type = models.Channel_TYPE_GROUP

	channel, err := c.ByName(q)
	if err != nil {
		log.Error("Could not fetch koding channel: %s:", err)
	}
	var channelId string
	if channel.Id != 0 {
		channelId = strconv.FormatInt(channel.Id, 10)
	}

	return &Controller{
		log:    log,
		client: client,
		indexes: &IndexSet{
			"topics":   client.InitIndex("topics" + indexSuffix),
			"accounts": client.InitIndex("accounts" + indexSuffix),
			"messages": client.InitIndex("messages" + indexSuffix),
		},
		kodingChannelId: channelId,
	}
}

func (f *Controller) TopicSaved(data *models.Channel) error {
	if data.TypeConstant != models.Channel_TYPE_TOPIC {
		return nil
	}
	return f.insert("topics", map[string]interface{}{
		"objectID": strconv.FormatInt(data.Id, 10),
		"name":     data.Name,
		"purpose":  data.Purpose,
	})
}

func (f *Controller) AccountSaved(data *models.Account) error {
	return f.insert("accounts", map[string]interface{}{
		"objectID": data.OldId,
		"nick":     data.Nick,
		"_tags":    []string{f.kodingChannelId},
	})
}

func (f *Controller) MessageListSaved(listing *models.ChannelMessageList) error {
	message := models.NewChannelMessage()

	if err := message.ById(listing.MessageId); err != nil {
		return err
	}

	// no need to index join/leave messages
	if message.TypeConstant != models.ChannelMessage_TYPE_POST &&
		message.TypeConstant != models.ChannelMessage_TYPE_REPLY {
		return nil
	}

	objectId := strconv.FormatInt(message.Id, 10)
	channelId := strconv.FormatInt(listing.ChannelId, 10)

	record, err := f.get("messages", objectId)
	if err != nil &&
		!IsAlgoliaError(err, ErrAlgoliaObjectIdNotFoundMsg) &&
		!IsAlgoliaError(err, ErrAlgoliaIndexNotExistMsg) {
		return err
	}

	if record == nil {
		return f.insert("messages", map[string]interface{}{
			"objectID": objectId,
			"body":     message.Body,
			"_tags":    []string{channelId},
		})
	}

	return f.partialUpdate("messages", map[string]interface{}{
		"objectID": objectId,
		"_tags":    appendTag(record, channelId),
	})
}

func (f *Controller) MessageListDeleted(listing *models.ChannelMessageList) error {
	index, err := f.indexes.Get("messages")
	if err != nil {
		return err
	}

	objectId := strconv.FormatInt(listing.MessageId, 10)

	record, err := f.get("messages", objectId)
	if err != nil &&
		!IsAlgoliaError(err, ErrAlgoliaObjectIdNotFoundMsg) &&
		!IsAlgoliaError(err, ErrAlgoliaIndexNotExistMsg) {
		return err
	}

	if tags, ok := record["_tags"]; ok {
		if t, ok := tags.([]interface{}); ok && len(t) == 1 {
			if _, err = index.DeleteObject(objectId); err != nil {
				return err
			}
			return nil
		}
	}

	return f.partialUpdate("messages", map[string]interface{}{
		"objectID": objectId,
		"_tags":    removeMessageTag(record, strconv.FormatInt(listing.ChannelId, 10)),
	})
}

func (f *Controller) MessageUpdated(message *models.ChannelMessage) error {
	return f.partialUpdate("messages", map[string]interface{}{
		"objectID": strconv.FormatInt(message.Id, 10),
		"body":     message.Body,
	})
}

const accountIndexName = "accounts"

// ParticipantUpdated operates with the participant deleted/created events,
// removes from algolia if status is left, adds to algolia if the state is
// active
func (f *Controller) ParticipantUpdated(p *models.ChannelParticipant) error {
	// if status of the participant is left, then just notify the current user
	if p.StatusConstant == models.ChannelParticipant_STATUS_LEFT {
		return f.handleParticipantOperation(p, removeTag)
	}

	if p.StatusConstant == models.ChannelParticipant_STATUS_ACTIVE {
		return f.handleParticipantOperation(p, appendTag)
	}

	f.log.Debug("ignoring the error status: %s", p.StatusConstant)

	return nil
}

// ParticipantCreated operates with the participant createad event, adds new
// tag to the algolia document
func (f *Controller) ParticipantCreated(p *models.ChannelParticipant) error {
	err := f.handleParticipantOperation(p, appendTag)
	if err != nil {
		f.log.Error("err while handling participant created event: %s", err.Error())
	}

	return err
}

func (f *Controller) handleParticipantOperation(p *models.ChannelParticipant, tagOperator func(record map[string]interface{}, channelId string) []interface{}) error {
	if p.ChannelId == 0 {
		return nil
	}

	if p.AccountId == 0 {
		return nil
	}

	a := models.NewAccount()
	if err := a.ById(p.AccountId); err != nil {
		f.log.Error("err while fetching account: %s", err.Error())
		return nil
	}

	if a.Id == 0 {
		f.log.Critical("account found but id is 0 %+v", a)
		return nil
	}

	record, err := f.get(accountIndexName, a.OldId)
	if err != nil &&
		!IsAlgoliaError(err, ErrAlgoliaObjectIdNotFoundMsg) &&
		!IsAlgoliaError(err, ErrAlgoliaIndexNotExistMsg) {
		return err
	}

	if record == nil {
		// first create the account
		if err := f.AccountSaved(a); err != nil {
			return err
		}

		// make sure account is there, before start processing it
		err := makeSureAccount(f, a.OldId, func(record map[string]interface{}, err error) bool {
			if err != nil {
				return false
			}

			if record == nil {
				return false
			}

			return true
		})
		if err != nil {
			return err
		}
	}

	record, err = f.get(accountIndexName, a.OldId)
	if err != nil {
		return err
	}

	channelId := strconv.FormatInt(p.ChannelId, 10)

	return f.partialUpdate(accountIndexName, map[string]interface{}{
		"objectID": a.OldId,
		"_tags":    tagOperator(record, channelId),
	})
}

var errDeadline = errors.New("deadline reached")

// makeSureAccount checks if the given id's get request returns the desired
// err, it will re-try every 100ms until deadline of 2 minutes reached. Algolia
// doesnt index the records right away, so try to go to a desired state
func makeSureAccount(handler *Controller, id string, f func(map[string]interface{}, error) bool) error {
	deadLine := time.After(time.Minute * 2)
	tick := time.Tick(time.Millisecond * 100)
	for {
		select {
		case <-tick:
			record, err := handler.get("accounts", id)
			if f(record, err) {
				return nil
			}
		case <-deadLine:
			return errDeadline
		}
	}
}
<|endoftext|>


<commit_before>package loraserver

import (
	"fmt"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/brocaar/loraserver/models"
	"github.com/brocaar/lorawan"
)

type dataDownProperties struct {
	rx1Channel int
	rx1DR      int
	rxDelay    time.Duration
}

func getDataDownProperties(rxInfo models.RXInfo, ns models.NodeSession) (dataDownProperties, error) {
	var err error
	var prop dataDownProperties

	// get TX DR
	uplinkDR, err := Band.GetDataRate(rxInfo.DataRate)
	if err != nil {
		return prop, err
	}

	// get TX channel
	uplinkChannel, err := Band.GetChannel(rxInfo.Frequency, uplinkDR)
	if err != nil {
		return prop, err
	}

	// get RX1 channel
	prop.rx1Channel = Band.GetRX1Channel(uplinkChannel)

	// get RX1 DR
	prop.rx1DR, err = Band.GetRX1DataRateForOffset(uplinkDR, int(ns.RX1DROffset))
	if err != nil {
		return prop, err
	}

	// get rx delay
	prop.rxDelay = Band.ReceiveDelay1
	if ns.RXDelay > 0 {
		prop.rxDelay = time.Duration(ns.RXDelay) * time.Second
	}

	return prop, nil
}

// getNextValidTXPayloadForDRFromQueue returns the next valid TXPayload from the
// queue for the given data-rate. When it exceeds the max size, the payload will
// be discarded and a notification will be sent to the application.
func getNextValidTXPayloadForDRFromQueue(ctx Context, ns models.NodeSession, dataRate int) (*models.TXPayload, error) {
	for {
		txPayload, err := getTXPayloadFromQueue(ctx.RedisPool, ns.DevEUI)
		if err != nil {
			if err == errEmptyQueue {
				return nil, nil
			}
			return nil, err
		}

		if len(txPayload.Data) <= Band.MaxPayloadSize[dataRate].N {
			return &txPayload, nil
		}

		// the payload exceeded the max payload size for the current data-rate
		// we'll remove the payload from the queue and notify the application
		if _, err = clearInProcessTXPayload(ctx.RedisPool, ns.DevEUI); err != nil {
			return nil, err
		}

		// log a warning
		log.WithFields(log.Fields{
			"dev_eui":             ns.DevEUI,
			"data_rate":           dataRate,
			"frmpayload_size":     len(txPayload.Data),
			"max_frmpayload_size": Band.MaxPayloadSize[dataRate].N,
			"reference":           txPayload.Reference,
		}).Warning("downlink payload max size exceeded")

		// notify the application
		err = ctx.Application.SendNotification(ns.AppEUI, ns.DevEUI, models.ErrorNotificationType, models.ErrorPayload{
			Reference: txPayload.Reference,
			DevEUI:    ns.DevEUI,
			Message:   fmt.Sprintf("downlink payload max size exceeded (dr: %d, allowed: %d, got: %d)", dataRate, Band.MaxPayloadSize[dataRate].N, len(txPayload.Data)),
		})
		if err != nil {
			return nil, err
		}
	}
}

func handleDataDownReply(ctx Context, rxPacket models.RXPacket, ns models.NodeSession) error {
	macPL, ok := rxPacket.PHYPayload.MACPayload.(*lorawan.MACPayload)
	if !ok {
		return fmt.Errorf("expected *lorawan.MACPayload, got: %T", rxPacket.PHYPayload.MACPayload)
	}

	// get data down properies
	properties, err := getDataDownProperties(rxPacket.RXInfo, ns)
	if err != nil {
		return fmt.Errorf("get data down properties error: %s", err)
	}

	var frmMACCommands bool
	var macPayloads []models.MACPayload
	allMACPayloads, err := readMACPayloadTXQueue(ctx.RedisPool, ns.DevAddr)
	if err != nil {
		return fmt.Errorf("read mac-payload tx queue error: %s", err)
	}

	if len(allMACPayloads) > 0 {
		if allMACPayloads[0].FRMPayload {
			// the first mac-commands must be sent as FRMPayload, filter the rest
			// of the MACPayload items with the same property, respecting the
			// max FRMPayload size for the data-rate.
			frmMACCommands = true
			macPayloads = filterMACPayloads(allMACPayloads, true, Band.MaxPayloadSize[properties.rx1DR].N)
		} else {
			// the first mac-command must be sent as FOpts, filter the rest of
			// the MACPayload items with the same property, respecting the
			// max FOpts size of 15.
			macPayloads = filterMACPayloads(allMACPayloads, false, 15)
		}
	}

	// if the MACCommands (if any) are not sent as FRMPayload, check if there
	// is a tx-payload in the queue and validate if the FOpts + FRMPayload
	// does not exceed the max payload size.
	var txPayload *models.TXPayload
	if !frmMACCommands {
		// check if there are payloads pending in the queue
		txPayload, err = getNextValidTXPayloadForDRFromQueue(ctx, ns, properties.rx1DR)
		if err != nil {
			return fmt.Errorf("get next valid tx-payload error: %s", err)
		}

		var macByteCount int
		for _, mac := range macPayloads {
			macByteCount += len(mac.MACCommand)
		}

		if txPayload != nil && len(txPayload.Data)+macByteCount > Band.MaxPayloadSize[properties.rx1DR].N {
			log.WithFields(log.Fields{
				"data_rate": properties.rx1DR,
				"dev_eui":   ns.DevEUI,
				"reference": txPayload.Reference,
			}).Info("scheduling tx-payload for next downlink, mac-commands + payload exceeds max size")
			txPayload = nil
		}
	}

	// convert the MACPayload items into MACCommand items
	var macCommmands []lorawan.MACCommand
	for _, pl := range macPayloads {
		var mac lorawan.MACCommand
		if err := mac.UnmarshalBinary(false, pl.MACCommand); err != nil {
			// in case the mac commands can't be unmarshaled, the payload
			// is ignored and an error sent to the network-controller
			errStr := fmt.Sprintf("unmarshal mac command error: %s", err)
			log.WithFields(log.Fields{
				"dev_eui":   ns.DevEUI,
				"reference": pl.Reference,
			}).Warning(errStr)
			err = ctx.Controller.SendErrorPayload(ns.AppEUI, ns.DevEUI, models.ErrorPayload{
				Reference: pl.Reference,
				DevEUI:    ns.DevEUI,
				Message:   errStr,
			})
			if err != nil {
				return fmt.Errorf("send error payload to network-controller error: %s", err)
			}
			continue
		}
		macCommmands = append(macCommmands, mac)
	}

	// uplink was unconfirmed and no downlink data in queue and no mac commands to send
	if txPayload == nil && rxPacket.PHYPayload.MHDR.MType == lorawan.UnconfirmedDataUp && len(macCommmands) == 0 {
		return nil
	}

	// get the queue size (the size includes the current payload)
	queueSize, err := getTXPayloadQueueSize(ctx.RedisPool, ns.DevEUI)
	if err != nil {
		return err
	}
	if txPayload != nil {
		queueSize-- // substract the current tx-payload from the queue-size
	}

	phy := lorawan.PHYPayload{
		MHDR: lorawan.MHDR{
			MType: lorawan.UnconfirmedDataDown,
			Major: lorawan.LoRaWANR1,
		},
	}
	macPL = &lorawan.MACPayload{
		FHDR: lorawan.FHDR{
			DevAddr: ns.DevAddr,
			FCtrl: lorawan.FCtrl{
				ACK:      rxPacket.PHYPayload.MHDR.MType == lorawan.ConfirmedDataUp, // set ACK when uplink packet was of type ConfirmedDataUp
				FPending: queueSize > 0 || len(allMACPayloads) != len(macPayloads),  // items in the queue or not all mac commands being sent
			},
			FCnt: ns.FCntDown,
		},
	}
	phy.MACPayload = macPL

	if len(macCommmands) > 0 {
		if frmMACCommands {
			var fPort uint8 // 0
			var frmPayload []lorawan.Payload
			for _, pl := range macCommmands {
				frmPayload = append(frmPayload, &pl)
			}
			macPL.FPort = &fPort
			macPL.FRMPayload = frmPayload
		} else {
			macPL.FHDR.FOpts = macCommmands
		}
	}

	// add the payload to FRMPayload field
	// note that txPayload is by definition nil when there are mac commands
	// to send in the FRMPayload field.
	if txPayload != nil {
		if txPayload.Confirmed {
			phy.MHDR.MType = lorawan.ConfirmedDataDown
		}

		macPL.FPort = &txPayload.FPort
		macPL.FRMPayload = []lorawan.Payload{
			&lorawan.DataPayload{Bytes: txPayload.Data},
		}
	}

	// if there is no payload set, encrypt will just do nothing
	if err := phy.EncryptFRMPayload(ns.AppSKey); err != nil {
		return fmt.Errorf("encrypt FRMPayload error: %s", err)
	}

	if err := phy.SetMIC(ns.NwkSKey); err != nil {
		return fmt.Errorf("set MIC error: %s", err)
	}

	txPacket := models.TXPacket{
		TXInfo: models.TXInfo{
			MAC:       rxPacket.RXInfo.MAC,
			Timestamp: rxPacket.RXInfo.Timestamp + uint32(properties.rxDelay/time.Microsecond),
			Frequency: Band.DownlinkChannels[properties.rx1Channel].Frequency,
			Power:     Band.DefaultTXPower,
			DataRate:  Band.DataRates[properties.rx1DR],
			CodeRate:  rxPacket.RXInfo.CodeRate,
		},
		PHYPayload: phy,
	}

	// window 1
	if err := ctx.Gateway.SendTXPacket(txPacket); err != nil {
		return fmt.Errorf("send tx packet (rx window 1) to gateway error: %s", err)
	}

	// increment the FCntDown when MType != ConfirmedDataDown and clear
	// in-process queue. In case of ConfirmedDataDown we increment on ACK.
	if phy.MHDR.MType != lorawan.ConfirmedDataDown {
		ns.FCntDown++
		if err = saveNodeSession(ctx.RedisPool, ns); err != nil {
			return err
		}

		if _, err = clearInProcessTXPayload(ctx.RedisPool, ns.DevEUI); err != nil {
			return err
		}
	}

	// remove the mac commands from the queue
	for _, pl := range macPayloads {
		if err = deleteMACPayloadFromTXQueue(ctx.RedisPool, ns.DevAddr, pl); err != nil {
			return fmt.Errorf("delete mac-payload from tx queue error: %s", err)
		}
	}

	return nil
}
<commit_msg>Fix mac-command range, encryption and clear in-process queue.<commit_after>package loraserver

import (
	"fmt"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/brocaar/loraserver/models"
	"github.com/brocaar/lorawan"
)

type dataDownProperties struct {
	rx1Channel int
	rx1DR      int
	rxDelay    time.Duration
}

func getDataDownProperties(rxInfo models.RXInfo, ns models.NodeSession) (dataDownProperties, error) {
	var err error
	var prop dataDownProperties

	// get TX DR
	uplinkDR, err := Band.GetDataRate(rxInfo.DataRate)
	if err != nil {
		return prop, err
	}

	// get TX channel
	uplinkChannel, err := Band.GetChannel(rxInfo.Frequency, uplinkDR)
	if err != nil {
		return prop, err
	}

	// get RX1 channel
	prop.rx1Channel = Band.GetRX1Channel(uplinkChannel)

	// get RX1 DR
	prop.rx1DR, err = Band.GetRX1DataRateForOffset(uplinkDR, int(ns.RX1DROffset))
	if err != nil {
		return prop, err
	}

	// get rx delay
	prop.rxDelay = Band.ReceiveDelay1
	if ns.RXDelay > 0 {
		prop.rxDelay = time.Duration(ns.RXDelay) * time.Second
	}

	return prop, nil
}

// getNextValidTXPayloadForDRFromQueue returns the next valid TXPayload from the
// queue for the given data-rate. When it exceeds the max size, the payload will
// be discarded and a notification will be sent to the application.
func getNextValidTXPayloadForDRFromQueue(ctx Context, ns models.NodeSession, dataRate int) (*models.TXPayload, error) {
	for {
		txPayload, err := getTXPayloadFromQueue(ctx.RedisPool, ns.DevEUI)
		if err != nil {
			if err == errEmptyQueue {
				return nil, nil
			}
			return nil, err
		}

		if len(txPayload.Data) <= Band.MaxPayloadSize[dataRate].N {
			return &txPayload, nil
		}

		// the payload exceeded the max payload size for the current data-rate
		// we'll remove the payload from the queue and notify the application
		if _, err = clearInProcessTXPayload(ctx.RedisPool, ns.DevEUI); err != nil {
			return nil, err
		}

		// log a warning
		log.WithFields(log.Fields{
			"dev_eui":             ns.DevEUI,
			"data_rate":           dataRate,
			"frmpayload_size":     len(txPayload.Data),
			"max_frmpayload_size": Band.MaxPayloadSize[dataRate].N,
			"reference":           txPayload.Reference,
		}).Warning("downlink payload max size exceeded")

		// notify the application
		err = ctx.Application.SendNotification(ns.AppEUI, ns.DevEUI, models.ErrorNotificationType, models.ErrorPayload{
			Reference: txPayload.Reference,
			DevEUI:    ns.DevEUI,
			Message:   fmt.Sprintf("downlink payload max size exceeded (dr: %d, allowed: %d, got: %d)", dataRate, Band.MaxPayloadSize[dataRate].N, len(txPayload.Data)),
		})
		if err != nil {
			return nil, err
		}
	}
}

func handleDataDownReply(ctx Context, rxPacket models.RXPacket, ns models.NodeSession) error {
	macPL, ok := rxPacket.PHYPayload.MACPayload.(*lorawan.MACPayload)
	if !ok {
		return fmt.Errorf("expected *lorawan.MACPayload, got: %T", rxPacket.PHYPayload.MACPayload)
	}

	// get data down properies
	properties, err := getDataDownProperties(rxPacket.RXInfo, ns)
	if err != nil {
		return fmt.Errorf("get data down properties error: %s", err)
	}

	var frmMACCommands bool
	var macPayloads []models.MACPayload
	allMACPayloads, err := readMACPayloadTXQueue(ctx.RedisPool, ns.DevAddr)
	if err != nil {
		return fmt.Errorf("read mac-payload tx queue error: %s", err)
	}

	if len(allMACPayloads) > 0 {
		if allMACPayloads[0].FRMPayload {
			// the first mac-commands must be sent as FRMPayload, filter the rest
			// of the MACPayload items with the same property, respecting the
			// max FRMPayload size for the data-rate.
			frmMACCommands = true
			macPayloads = filterMACPayloads(allMACPayloads, true, Band.MaxPayloadSize[properties.rx1DR].N)
		} else {
			// the first mac-command must be sent as FOpts, filter the rest of
			// the MACPayload items with the same property, respecting the
			// max FOpts size of 15.
			macPayloads = filterMACPayloads(allMACPayloads, false, 15)
		}
	}

	// if the MACCommands (if any) are not sent as FRMPayload, check if there
	// is a tx-payload in the queue and validate if the FOpts + FRMPayload
	// does not exceed the max payload size.
	var txPayload *models.TXPayload
	if !frmMACCommands {
		// check if there are payloads pending in the queue
		txPayload, err = getNextValidTXPayloadForDRFromQueue(ctx, ns, properties.rx1DR)
		if err != nil {
			return fmt.Errorf("get next valid tx-payload error: %s", err)
		}

		var macByteCount int
		for _, mac := range macPayloads {
			macByteCount += len(mac.MACCommand)
		}

		if txPayload != nil && len(txPayload.Data)+macByteCount > Band.MaxPayloadSize[properties.rx1DR].N {
			log.WithFields(log.Fields{
				"data_rate": properties.rx1DR,
				"dev_eui":   ns.DevEUI,
				"reference": txPayload.Reference,
			}).Info("scheduling tx-payload for next downlink, mac-commands + payload exceeds max size")
			txPayload = nil
		}
	}

	// convert the MACPayload items into MACCommand items
	var macCommmands []lorawan.MACCommand
	for _, pl := range macPayloads {
		var mac lorawan.MACCommand
		if err := mac.UnmarshalBinary(false, pl.MACCommand); err != nil {
			// in case the mac commands can't be unmarshaled, the payload
			// is ignored and an error sent to the network-controller
			errStr := fmt.Sprintf("unmarshal mac command error: %s", err)
			log.WithFields(log.Fields{
				"dev_eui":   ns.DevEUI,
				"reference": pl.Reference,
			}).Warning(errStr)
			err = ctx.Controller.SendErrorPayload(ns.AppEUI, ns.DevEUI, models.ErrorPayload{
				Reference: pl.Reference,
				DevEUI:    ns.DevEUI,
				Message:   errStr,
			})
			if err != nil {
				return fmt.Errorf("send error payload to network-controller error: %s", err)
			}
			continue
		}
		macCommmands = append(macCommmands, mac)
	}

	// uplink was unconfirmed and no downlink data in queue and no mac commands to send
	if txPayload == nil && rxPacket.PHYPayload.MHDR.MType == lorawan.UnconfirmedDataUp && len(macCommmands) == 0 {
		return nil
	}

	// get the queue size (the size includes the current payload)
	queueSize, err := getTXPayloadQueueSize(ctx.RedisPool, ns.DevEUI)
	if err != nil {
		return err
	}
	if txPayload != nil {
		queueSize-- // substract the current tx-payload from the queue-size
	}

	phy := lorawan.PHYPayload{
		MHDR: lorawan.MHDR{
			MType: lorawan.UnconfirmedDataDown,
			Major: lorawan.LoRaWANR1,
		},
	}
	macPL = &lorawan.MACPayload{
		FHDR: lorawan.FHDR{
			DevAddr: ns.DevAddr,
			FCtrl: lorawan.FCtrl{
				ACK:      rxPacket.PHYPayload.MHDR.MType == lorawan.ConfirmedDataUp, // set ACK when uplink packet was of type ConfirmedDataUp
				FPending: queueSize > 0 || len(allMACPayloads) != len(macPayloads),  // items in the queue or not all mac commands being sent
			},
			FCnt: ns.FCntDown,
		},
	}
	phy.MACPayload = macPL

	if len(macCommmands) > 0 {
		if frmMACCommands {
			var fPort uint8 // 0
			var frmPayload []lorawan.Payload
			for i, _ := range macCommmands {
				frmPayload = append(frmPayload, &macCommmands[i])
			}
			macPL.FPort = &fPort
			macPL.FRMPayload = frmPayload
		} else {
			macPL.FHDR.FOpts = macCommmands
		}
	}

	// add the payload to FRMPayload field
	// note that txPayload is by definition nil when there are mac commands
	// to send in the FRMPayload field.
	if txPayload != nil {
		if txPayload.Confirmed {
			phy.MHDR.MType = lorawan.ConfirmedDataDown
		}

		macPL.FPort = &txPayload.FPort
		macPL.FRMPayload = []lorawan.Payload{
			&lorawan.DataPayload{Bytes: txPayload.Data},
		}
	}

	// if there is no payload set, encrypt will just do nothing
	if len(macCommmands) > 0 && frmMACCommands {
		if err := phy.EncryptFRMPayload(ns.NwkSKey); err != nil {
			return fmt.Errorf("encrypt FRMPayload error: %s", err)
		}
	} else {
		if err := phy.EncryptFRMPayload(ns.AppSKey); err != nil {
			return fmt.Errorf("encrypt FRMPayload error: %s", err)
		}
	}

	if err := phy.SetMIC(ns.NwkSKey); err != nil {
		return fmt.Errorf("set MIC error: %s", err)
	}

	txPacket := models.TXPacket{
		TXInfo: models.TXInfo{
			MAC:       rxPacket.RXInfo.MAC,
			Timestamp: rxPacket.RXInfo.Timestamp + uint32(properties.rxDelay/time.Microsecond),
			Frequency: Band.DownlinkChannels[properties.rx1Channel].Frequency,
			Power:     Band.DefaultTXPower,
			DataRate:  Band.DataRates[properties.rx1DR],
			CodeRate:  rxPacket.RXInfo.CodeRate,
		},
		PHYPayload: phy,
	}

	// window 1
	if err := ctx.Gateway.SendTXPacket(txPacket); err != nil {
		return fmt.Errorf("send tx packet (rx window 1) to gateway error: %s", err)
	}

	// increment the FCntDown when MType != ConfirmedDataDown and clear
	// in-process queue. In case of ConfirmedDataDown we increment on ACK.
	if phy.MHDR.MType != lorawan.ConfirmedDataDown {
		ns.FCntDown++
		if err = saveNodeSession(ctx.RedisPool, ns); err != nil {
			return err
		}

		if txPayload != nil {
			if _, err = clearInProcessTXPayload(ctx.RedisPool, ns.DevEUI); err != nil {
				return err
			}
		}
	}

	// remove the mac commands from the queue
	for _, pl := range macPayloads {
		if err = deleteMACPayloadFromTXQueue(ctx.RedisPool, ns.DevAddr, pl); err != nil {
			return fmt.Errorf("delete mac-payload from tx queue error: %s", err)
		}
	}

	return nil
}
<|endoftext|>