<commit_before>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/globocom/config"
	"github.com/globocom/tsuru/api/auth"
	"github.com/globocom/tsuru/api/bind"
	"github.com/globocom/tsuru/api/service"
	"github.com/globocom/tsuru/db"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/repository"
	"io"
	"labix.org/v2/mgo/bson"
	"launchpad.net/goyaml"
	"os"
	"os/exec"
	"path"
	"sort"
	"strings"
	"time"
)

const confSep = "========"

type App struct {
	Env       map[string]bind.EnvVar
	Framework string
	Logs      []applog
	Name      string
	State     string
	Units     []Unit
	Teams     []string
}

func (a *App) MarshalJSON() ([]byte, error) {
	result := make(map[string]interface{})
	result["Name"] = a.Name
	result["State"] = a.State
	result["Framework"] = a.Framework
	result["Teams"] = a.Teams
	result["Units"] = a.Units
	result["Repository"] = repository.GetUrl(a.Name)
	return json.Marshal(&result)
}

type applog struct {
	Date    time.Time
	Message string
}

type conf struct {
	PreRestart []string `yaml:"pre-restart"`
	PosRestart []string `yaml:"pos-restart"`
}

func (a *App) Get() error {
	return db.Session.Apps().Find(bson.M{"name": a.Name}).One(a)
}

// createApp creates a new app.
//
// Creating a new app is a process composed of two steps:
//
//       1. Saves the app in the database
//       2. Deploys juju charm
func createApp(a *App) error {
	a.State = "pending"
	err := db.Session.Apps().Insert(a)
	if err != nil {
		return err
	}
	return deploy(a)
}

// Deploys an app.
func deploy(a *App) error {
	a.log(fmt.Sprintf("creating app %s", a.Name))
	cmd := exec.Command("juju", "deploy", "--repository=/home/charms", "local:"+a.Framework, a.Name)
	log.Printf("deploying %s with name %s", a.Framework, a.Name)
	out, err := cmd.CombinedOutput()
	outStr := string(out)
	a.log(outStr)
	log.Printf("executing %s", outStr)
	if err != nil {
		a.log(fmt.Sprintf("juju finished with exit status: %s", err))
		db.Session.Apps().Remove(bson.M{"name": a.Name})
		return errors.New(outStr)
	}
	return nil
}

func (a *App) unbind() error {
	var instances []service.ServiceInstance
	err := db.Session.ServiceInstances().Find(bson.M{"apps": bson.M{"$in": []string{a.Name}}}).All(&instances)
	if err != nil {
		return err
	}
	var msg string
	var addMsg = func(instanceName string, reason error) {
		if msg == "" {
			msg = "Failed to unbind the following instances:\n"
		}
		msg += fmt.Sprintf("- %s (%s)", instanceName, reason.Error())
	}
	for _, instance := range instances {
		err = instance.Unbind(a)
		if err != nil {
			addMsg(instance.Name, err)
		}
	}
	if msg != "" {
		return errors.New(msg)
	}
	return nil
}

func (a *App) destroy() error {
	out, err := a.unit().destroy()
	msg := string(out)
	log.Print(msg)
	if err != nil {
		return errors.New(msg)
	}
	err = a.unbind()
	if err != nil {
		return err
	}
	return db.Session.Apps().Remove(bson.M{"name": a.Name})
}

func (a *App) AddUnit(u *Unit) {
	for i, unt := range a.Units {
		if unt.Machine == u.Machine {
			a.Units[i] = *u
			return
		}
	}
	a.Units = append(a.Units, *u)
}

func (a *App) find(team *auth.Team) (int, bool) {
	pos := sort.Search(len(a.Teams), func(i int) bool {
		return a.Teams[i] >= team.Name
	})
	return pos, pos < len(a.Teams) && a.Teams[pos] == team.Name
}

func (a *App) grant(team *auth.Team) error {
	pos, found := a.find(team)
	if found {
		return errors.New("This team already has access to this app")
	}
	a.Teams = append(a.Teams, "")
	tmp := a.Teams[pos]
	for i := pos; i < len(a.Teams)-1; i++ {
		a.Teams[i+1], tmp = tmp, a.Teams[i]
	}
	a.Teams[pos] = team.Name
	return nil
}

func (a *App) revoke(team *auth.Team) error {
	index, found := a.find(team)
	if !found {
		return errors.New("This team does not have access to this app")
	}
	copy(a.Teams[index:], a.Teams[index+1:])
	a.Teams = a.Teams[:len(a.Teams)-1]
	return nil
}

func (a *App) teams() []auth.Team {
	var teams []auth.Team
	db.Session.Teams().Find(bson.M{"_id": bson.M{"$in": a.Teams}}).All(&teams)
	return teams
}

func (a *App) setTeams(teams []auth.Team) {
	a.Teams = make([]string, len(teams))
	for i, team := range teams {
		a.Teams[i] = team.Name
	}
	sort.Strings(a.Teams)
}

func (a *App) setEnv(env bind.EnvVar) {
	if a.Env == nil {
		a.Env = make(map[string]bind.EnvVar)
	}
	a.Env[env.Name] = env
	a.log(fmt.Sprintf("setting env %s with value %s", env.Name, env.Value))
}

func (a *App) getEnv(name string) (bind.EnvVar, error) {
	var (
		env bind.EnvVar
		err error
		ok  bool
	)
	if env, ok = a.Env[name]; !ok {
		err = errors.New("Environment variable not declared for this app.")
	}
	return env, err
}

func (a *App) InstanceEnv(name string) map[string]bind.EnvVar {
	envs := make(map[string]bind.EnvVar)
	for k, env := range a.Env {
		if env.InstanceName == name {
			envs[k] = bind.EnvVar(env)
		}
	}
	return envs
}

func deployHookAbsPath(p string) (string, error) {
	repoPath, err := config.GetString("git:unit-repo")
	if err != nil {
		return "", nil
	}
	cmdArgs := strings.Fields(p)
	abs := path.Join(repoPath, cmdArgs[0])
	_, err = os.Stat(abs)
	if os.IsNotExist(err) {
		return p, nil
	}
	cmdArgs[0] = abs
	return strings.Join(cmdArgs, " "), nil
}

// Returns app.conf located at app's git repository
func (a *App) conf() (conf, error) {
	var c conf
	uRepo, err := repository.GetPath()
	if err != nil {
		a.log(fmt.Sprintf("Got error while getting repository path: %s", err.Error()))
		return c, err
	}
	cPath := path.Join(uRepo, "app.conf")
	cmd := fmt.Sprintf(`echo "%s";cat %s`, confSep, cPath)
	o, err := a.unit().Command(nil, nil, cmd)
	if err != nil {
		a.log(fmt.Sprintf("Got error while executing command: %s... Skipping hooks execution", err.Error()))
		return c, nil
	}
	data := strings.Split(string(o), confSep)[1]
	err = goyaml.Unmarshal([]byte(data), &c)
	if err != nil {
		a.log(fmt.Sprintf("Got error while parsing yaml: %s", err.Error()))
		return c, err
	}
	return c, nil
}

func (a *App) runHook(cmds []string, kind string) ([]byte, error) {
	var (
		buf bytes.Buffer
		err error
	)
	a.log(fmt.Sprintf("Executing %s hook...", kind))
	for _, cmd := range cmds {
		p, err := deployHookAbsPath(cmd)
		if err != nil {
			a.log(fmt.Sprintf("Error obtaining absolute path to hook: %s.", err))
			continue
		}
		err = a.run(p, &buf)
		if err != nil {
			return nil, err
		}
		a.log(fmt.Sprintf("Output of %s (%s hook): %s", cmd, kind, buf.Bytes()))
	}
	return buf.Bytes(), err
}

/*
* preRestart is responsible for running user's pre-restart script.
* The path to this script can be found at the app.conf file, at the root of user's app repository.
 */
func (a *App) preRestart(c conf) ([]byte, error) {
	if !a.hasRestartHooks(c) {
		a.log("app.conf file does not exists or is in the right place. Skipping pre-restart hook...")
		return []byte(nil), nil
	}
	if len(c.PreRestart) == 0 {
		a.log("pre-restart hook section in app conf does not exists... Skipping pre-restart hook...")
		return []byte(nil), nil
	}
	return a.runHook(c.PreRestart, "pre-restart")
}

/*
* posRestart is responsible for running user's pos-restart script.
* The path to this script can be found at the app.conf file, at the root of user's app repository.
 */
func (a *App) posRestart(c conf) ([]byte, error) {
	if !a.hasRestartHooks(c) {
		a.log("app.conf file does not exists or is in the right place. Skipping pos-restart hook...")
		return []byte(nil), nil
	}
	if len(c.PosRestart) == 0 {
		a.log("pos-restart hook section in app conf does not exists... Skipping pos-restart hook...")
		return []byte(nil), nil
	}
	return a.runHook(c.PosRestart, "pos-restart")
}

func (a *App) hasRestartHooks(c conf) bool {
	return len(c.PreRestart) > 0 || len(c.PosRestart) > 0
}

func (a *App) run(cmd string, w io.Writer) error {
	a.log(fmt.Sprintf("running '%s'", cmd))
	cmd = fmt.Sprintf("[ -f /home/application/apprc ] && source /home/application/apprc; [ -d /home/application/current ] && cd /home/application/current; %s", cmd)
	out, err := a.unit().Command(w, w, cmd)
	a.log(string(out))
	return err
}

// restart runs the restart hook for the app
// and returns your output.
func restart(a *App, w io.Writer) ([]byte, error) {
	u := a.unit()
	a.log("executting hook to restarting")
	if w != nil {
		_, err := w.Write([]byte("\n ---> Restarting your app\n"))
		if err != nil {
			return nil, err
		}
	}
	out, err := u.executeHook(w, w, "restart")
	if err != nil {
		return out, err
	}
	a.log(string(out))
	return out, nil
}

//installDeps runs the dependencies hook for the app
//and returns your output.
func installDeps(a *App, stdout, stderr io.Writer) ([]byte, error) {
	u := a.unit()
	a.log("executting hook dependencies")
	out, err := u.executeHook(stdout, stderr, "dependencies")
	a.log(string(out))
	if err != nil {
		return out, err
	}
	return out, nil
}

func (a *App) unit() *Unit {
	if len(a.Units) > 0 {
		unit := a.Units[0]
		unit.app = a
		return &unit
	}
	return &Unit{app: a}
}

func (a *App) GetUnits() []bind.Unit {
	var units []bind.Unit
	for _, u := range a.Units {
		u.app = a
		units = append(units, &u)
	}
	return units
}

func (a *App) GetName() string {
	return a.Name
}

func (a *App) SetEnvs(envs []bind.EnvVar, publicOnly bool) error {
	e := make([]bind.EnvVar, len(envs))
	for i, env := range envs {
		e[i] = bind.EnvVar(env)
	}
	return setEnvsToApp(a, e, publicOnly)
}

func (a *App) UnsetEnvs(envs []string, publicOnly bool) error {
	return unsetEnvFromApp(a, envs, publicOnly)
}

func (a *App) log(message string) error {
	log.Printf(message)
	l := applog{Date: time.Now(), Message: message}
	a.Logs = append(a.Logs, l)
	return db.Session.Apps().Update(bson.M{"name": a.Name}, a)
}
<commit_msg>api/app: logging the entire command<commit_after>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/globocom/config"
	"github.com/globocom/tsuru/api/auth"
	"github.com/globocom/tsuru/api/bind"
	"github.com/globocom/tsuru/api/service"
	"github.com/globocom/tsuru/db"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/repository"
	"io"
	"labix.org/v2/mgo/bson"
	"launchpad.net/goyaml"
	"os"
	"os/exec"
	"path"
	"sort"
	"strings"
	"time"
)

const confSep = "========"

type App struct {
	Env       map[string]bind.EnvVar
	Framework string
	Logs      []applog
	Name      string
	State     string
	Units     []Unit
	Teams     []string
}

func (a *App) MarshalJSON() ([]byte, error) {
	result := make(map[string]interface{})
	result["Name"] = a.Name
	result["State"] = a.State
	result["Framework"] = a.Framework
	result["Teams"] = a.Teams
	result["Units"] = a.Units
	result["Repository"] = repository.GetUrl(a.Name)
	return json.Marshal(&result)
}

type applog struct {
	Date    time.Time
	Message string
}

type conf struct {
	PreRestart []string `yaml:"pre-restart"`
	PosRestart []string `yaml:"pos-restart"`
}

func (a *App) Get() error {
	return db.Session.Apps().Find(bson.M{"name": a.Name}).One(a)
}

// createApp creates a new app.
//
// Creating a new app is a process composed of two steps:
//
//       1. Saves the app in the database
//       2. Deploys juju charm
func createApp(a *App) error {
	a.State = "pending"
	err := db.Session.Apps().Insert(a)
	if err != nil {
		return err
	}
	return deploy(a)
}

// Deploys an app.
func deploy(a *App) error {
	a.log(fmt.Sprintf("creating app %s", a.Name))
	cmd := exec.Command("juju", "deploy", "--repository=/home/charms", "local:"+a.Framework, a.Name)
	log.Printf("deploying %s with name %s", a.Framework, a.Name)
	out, err := cmd.CombinedOutput()
	outStr := string(out)
	a.log(outStr)
	log.Printf("executing %s", outStr)
	if err != nil {
		a.log(fmt.Sprintf("juju finished with exit status: %s", err))
		db.Session.Apps().Remove(bson.M{"name": a.Name})
		return errors.New(outStr)
	}
	return nil
}

func (a *App) unbind() error {
	var instances []service.ServiceInstance
	err := db.Session.ServiceInstances().Find(bson.M{"apps": bson.M{"$in": []string{a.Name}}}).All(&instances)
	if err != nil {
		return err
	}
	var msg string
	var addMsg = func(instanceName string, reason error) {
		if msg == "" {
			msg = "Failed to unbind the following instances:\n"
		}
		msg += fmt.Sprintf("- %s (%s)", instanceName, reason.Error())
	}
	for _, instance := range instances {
		err = instance.Unbind(a)
		if err != nil {
			addMsg(instance.Name, err)
		}
	}
	if msg != "" {
		return errors.New(msg)
	}
	return nil
}

func (a *App) destroy() error {
	out, err := a.unit().destroy()
	msg := string(out)
	log.Print(msg)
	if err != nil {
		return errors.New(msg)
	}
	err = a.unbind()
	if err != nil {
		return err
	}
	return db.Session.Apps().Remove(bson.M{"name": a.Name})
}

func (a *App) AddUnit(u *Unit) {
	for i, unt := range a.Units {
		if unt.Machine == u.Machine {
			a.Units[i] = *u
			return
		}
	}
	a.Units = append(a.Units, *u)
}

func (a *App) find(team *auth.Team) (int, bool) {
	pos := sort.Search(len(a.Teams), func(i int) bool {
		return a.Teams[i] >= team.Name
	})
	return pos, pos < len(a.Teams) && a.Teams[pos] == team.Name
}

func (a *App) grant(team *auth.Team) error {
	pos, found := a.find(team)
	if found {
		return errors.New("This team already has access to this app")
	}
	a.Teams = append(a.Teams, "")
	tmp := a.Teams[pos]
	for i := pos; i < len(a.Teams)-1; i++ {
		a.Teams[i+1], tmp = tmp, a.Teams[i]
	}
	a.Teams[pos] = team.Name
	return nil
}

func (a *App) revoke(team *auth.Team) error {
	index, found := a.find(team)
	if !found {
		return errors.New("This team does not have access to this app")
	}
	copy(a.Teams[index:], a.Teams[index+1:])
	a.Teams = a.Teams[:len(a.Teams)-1]
	return nil
}

func (a *App) teams() []auth.Team {
	var teams []auth.Team
	db.Session.Teams().Find(bson.M{"_id": bson.M{"$in": a.Teams}}).All(&teams)
	return teams
}

func (a *App) setTeams(teams []auth.Team) {
	a.Teams = make([]string, len(teams))
	for i, team := range teams {
		a.Teams[i] = team.Name
	}
	sort.Strings(a.Teams)
}

func (a *App) setEnv(env bind.EnvVar) {
	if a.Env == nil {
		a.Env = make(map[string]bind.EnvVar)
	}
	a.Env[env.Name] = env
	a.log(fmt.Sprintf("setting env %s with value %s", env.Name, env.Value))
}

func (a *App) getEnv(name string) (bind.EnvVar, error) {
	var (
		env bind.EnvVar
		err error
		ok  bool
	)
	if env, ok = a.Env[name]; !ok {
		err = errors.New("Environment variable not declared for this app.")
	}
	return env, err
}

func (a *App) InstanceEnv(name string) map[string]bind.EnvVar {
	envs := make(map[string]bind.EnvVar)
	for k, env := range a.Env {
		if env.InstanceName == name {
			envs[k] = bind.EnvVar(env)
		}
	}
	return envs
}

func deployHookAbsPath(p string) (string, error) {
	repoPath, err := config.GetString("git:unit-repo")
	if err != nil {
		return "", nil
	}
	cmdArgs := strings.Fields(p)
	abs := path.Join(repoPath, cmdArgs[0])
	_, err = os.Stat(abs)
	if os.IsNotExist(err) {
		return p, nil
	}
	cmdArgs[0] = abs
	return strings.Join(cmdArgs, " "), nil
}

// Returns app.conf located at app's git repository
func (a *App) conf() (conf, error) {
	var c conf
	uRepo, err := repository.GetPath()
	if err != nil {
		a.log(fmt.Sprintf("Got error while getting repository path: %s", err.Error()))
		return c, err
	}
	cPath := path.Join(uRepo, "app.conf")
	cmd := fmt.Sprintf(`echo "%s";cat %s`, confSep, cPath)
	o, err := a.unit().Command(nil, nil, cmd)
	if err != nil {
		a.log(fmt.Sprintf("Got error while executing command: %s... Skipping hooks execution", err.Error()))
		return c, nil
	}
	data := strings.Split(string(o), confSep)[1]
	err = goyaml.Unmarshal([]byte(data), &c)
	if err != nil {
		a.log(fmt.Sprintf("Got error while parsing yaml: %s", err.Error()))
		return c, err
	}
	return c, nil
}

func (a *App) runHook(cmds []string, kind string) ([]byte, error) {
	var (
		buf bytes.Buffer
		err error
	)
	a.log(fmt.Sprintf("Executing %s hook...", kind))
	for _, cmd := range cmds {
		p, err := deployHookAbsPath(cmd)
		if err != nil {
			a.log(fmt.Sprintf("Error obtaining absolute path to hook: %s.", err))
			continue
		}
		err = a.run(p, &buf)
		if err != nil {
			return nil, err
		}
		a.log(fmt.Sprintf("Output of %s (%s hook): %s", p, kind, buf.Bytes()))
	}
	return buf.Bytes(), err
}

/*
* preRestart is responsible for running user's pre-restart script.
* The path to this script can be found at the app.conf file, at the root of user's app repository.
 */
func (a *App) preRestart(c conf) ([]byte, error) {
	if !a.hasRestartHooks(c) {
		a.log("app.conf file does not exists or is in the right place. Skipping pre-restart hook...")
		return []byte(nil), nil
	}
	if len(c.PreRestart) == 0 {
		a.log("pre-restart hook section in app conf does not exists... Skipping pre-restart hook...")
		return []byte(nil), nil
	}
	return a.runHook(c.PreRestart, "pre-restart")
}

/*
* posRestart is responsible for running user's pos-restart script.
* The path to this script can be found at the app.conf file, at the root of user's app repository.
 */
func (a *App) posRestart(c conf) ([]byte, error) {
	if !a.hasRestartHooks(c) {
		a.log("app.conf file does not exists or is in the right place. Skipping pos-restart hook...")
		return []byte(nil), nil
	}
	if len(c.PosRestart) == 0 {
		a.log("pos-restart hook section in app conf does not exists... Skipping pos-restart hook...")
		return []byte(nil), nil
	}
	return a.runHook(c.PosRestart, "pos-restart")
}

func (a *App) hasRestartHooks(c conf) bool {
	return len(c.PreRestart) > 0 || len(c.PosRestart) > 0
}

func (a *App) run(cmd string, w io.Writer) error {
	a.log(fmt.Sprintf("running '%s'", cmd))
	cmd = fmt.Sprintf("[ -f /home/application/apprc ] && source /home/application/apprc; [ -d /home/application/current ] && cd /home/application/current; %s", cmd)
	out, err := a.unit().Command(w, w, cmd)
	a.log(string(out))
	return err
}

// restart runs the restart hook for the app
// and returns your output.
func restart(a *App, w io.Writer) ([]byte, error) {
	u := a.unit()
	a.log("executting hook to restarting")
	if w != nil {
		_, err := w.Write([]byte("\n ---> Restarting your app\n"))
		if err != nil {
			return nil, err
		}
	}
	out, err := u.executeHook(w, w, "restart")
	if err != nil {
		return out, err
	}
	a.log(string(out))
	return out, nil
}

//installDeps runs the dependencies hook for the app
//and returns your output.
func installDeps(a *App, stdout, stderr io.Writer) ([]byte, error) {
	u := a.unit()
	a.log("executting hook dependencies")
	out, err := u.executeHook(stdout, stderr, "dependencies")
	a.log(string(out))
	if err != nil {
		return out, err
	}
	return out, nil
}

func (a *App) unit() *Unit {
	if len(a.Units) > 0 {
		unit := a.Units[0]
		unit.app = a
		return &unit
	}
	return &Unit{app: a}
}

func (a *App) GetUnits() []bind.Unit {
	var units []bind.Unit
	for _, u := range a.Units {
		u.app = a
		units = append(units, &u)
	}
	return units
}

func (a *App) GetName() string {
	return a.Name
}

func (a *App) SetEnvs(envs []bind.EnvVar, publicOnly bool) error {
	e := make([]bind.EnvVar, len(envs))
	for i, env := range envs {
		e[i] = bind.EnvVar(env)
	}
	return setEnvsToApp(a, e, publicOnly)
}

func (a *App) UnsetEnvs(envs []string, publicOnly bool) error {
	return unsetEnvFromApp(a, envs, publicOnly)
}

func (a *App) log(message string) error {
	log.Printf(message)
	l := applog{Date: time.Now(), Message: message}
	a.Logs = append(a.Logs, l)
	return db.Session.Apps().Update(bson.M{"name": a.Name}, a)
}
<|endoftext|>


<commit_before>package virtualServer

import (
	"os"
	"strconv"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"k8s.io/client-go/1.4/kubernetes/fake"
	"k8s.io/client-go/1.4/pkg/api"
	"k8s.io/client-go/1.4/pkg/api/unversioned"
	"k8s.io/client-go/1.4/pkg/api/v1"
)

func TestConfigFilename(t *testing.T) {
	assert := assert.New(t)

	pid := os.Getpid()
	expectedFilename := "/tmp/f5-k8s-controller.config." + strconv.Itoa(pid) + ".json"

	assert.Equal(expectedFilename, OutputFilename)
}

func newNode(id, rv string, unsched bool,
	addresses []v1.NodeAddress) *v1.Node {
	return &v1.Node{
		TypeMeta: unversioned.TypeMeta{
			Kind:       "Node",
			APIVersion: "v1",
		},
		ObjectMeta: v1.ObjectMeta{
			Name:            id,
			ResourceVersion: rv,
		},
		Spec: v1.NodeSpec{
			Unschedulable: unsched,
		},
		Status: v1.NodeStatus{
			Addresses: addresses,
		},
	}
}

func TestGetAddresses(t *testing.T) {
	// Existing Node data
	expectedNodes := []*v1.Node{
		newNode("node0", "0", true, []v1.NodeAddress{
			{"ExternalIP", "127.0.0.0"}}),
		newNode("node1", "1", false, []v1.NodeAddress{
			{"ExternalIP", "127.0.0.1"}}),
		newNode("node2", "2", false, []v1.NodeAddress{
			{"ExternalIP", "127.0.0.2"}}),
		newNode("node3", "3", false, []v1.NodeAddress{
			{"ExternalIP", "127.0.0.3"}}),
		newNode("node4", "4", false, []v1.NodeAddress{
			{"InternalIP", "127.0.0.4"}}),
		newNode("node5", "5", false, []v1.NodeAddress{
			{"Hostname", "127.0.0.5"}}),
	}

	expectedReturn := []string{
		"127.0.0.1",
		"127.0.0.2",
		"127.0.0.3",
	}

	fake := fake.NewSimpleClientset()

	for _, expectedNode := range expectedNodes {
		node, err := fake.Core().Nodes().Create(expectedNode)
		require.Nil(t, err, "Should not fail creating node")
		require.EqualValues(t, expectedNode, node, "Nodes should be equal")
	}

	addresses, err := getNodeAddresses(fake)
	require.Nil(t, err, "Should not fail getting addresses")
	assert.EqualValues(t, expectedReturn, addresses,
		"Should receive the correct addresses")

	for _, node := range expectedNodes {
		err := fake.Core().Nodes().Delete(node.ObjectMeta.Name,
			&api.DeleteOptions{})
		require.Nil(t, err, "Should not fail deleting node")
	}

	expectedReturn = []string{}
	addresses, err = getNodeAddresses(fake)
	require.Nil(t, err, "Should not fail getting empty addresses")
	assert.EqualValues(t, expectedReturn, addresses, "Should get no addresses")
}
<commit_msg>VEL-373 Add remainder node processing unit test<commit_after>package virtualServer

import (
	"encoding/json"
	"os"
	"strconv"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"k8s.io/client-go/1.4/kubernetes/fake"
	"k8s.io/client-go/1.4/pkg/api"
	"k8s.io/client-go/1.4/pkg/api/unversioned"
	"k8s.io/client-go/1.4/pkg/api/v1"
)

func TestConfigFilename(t *testing.T) {
	assert := assert.New(t)

	pid := os.Getpid()
	expectedFilename := "/tmp/f5-k8s-controller.config." + strconv.Itoa(pid) + ".json"

	assert.Equal(expectedFilename, OutputFilename)
}

func newNode(id, rv string, unsched bool,
	addresses []v1.NodeAddress) *v1.Node {
	return &v1.Node{
		TypeMeta: unversioned.TypeMeta{
			Kind:       "Node",
			APIVersion: "v1",
		},
		ObjectMeta: v1.ObjectMeta{
			Name:            id,
			ResourceVersion: rv,
		},
		Spec: v1.NodeSpec{
			Unschedulable: unsched,
		},
		Status: v1.NodeStatus{
			Addresses: addresses,
		},
	}
}

func TestGetAddresses(t *testing.T) {
	// Existing Node data
	expectedNodes := []*v1.Node{
		newNode("node0", "0", true, []v1.NodeAddress{
			{"ExternalIP", "127.0.0.0"}}),
		newNode("node1", "1", false, []v1.NodeAddress{
			{"ExternalIP", "127.0.0.1"}}),
		newNode("node2", "2", false, []v1.NodeAddress{
			{"ExternalIP", "127.0.0.2"}}),
		newNode("node3", "3", false, []v1.NodeAddress{
			{"ExternalIP", "127.0.0.3"}}),
		newNode("node4", "4", false, []v1.NodeAddress{
			{"InternalIP", "127.0.0.4"}}),
		newNode("node5", "5", false, []v1.NodeAddress{
			{"Hostname", "127.0.0.5"}}),
	}

	expectedReturn := []string{
		"127.0.0.1",
		"127.0.0.2",
		"127.0.0.3",
	}

	fake := fake.NewSimpleClientset()
	assert.NotNil(t, fake, "Mock client cannot be nil")

	for _, expectedNode := range expectedNodes {
		node, err := fake.Core().Nodes().Create(expectedNode)
		require.Nil(t, err, "Should not fail creating node")
		require.EqualValues(t, expectedNode, node, "Nodes should be equal")
	}

	addresses, err := getNodeAddresses(fake)
	require.Nil(t, err, "Should not fail getting addresses")
	assert.EqualValues(t, expectedReturn, addresses,
		"Should receive the correct addresses")

	for _, node := range expectedNodes {
		err := fake.Core().Nodes().Delete(node.ObjectMeta.Name,
			&api.DeleteOptions{})
		require.Nil(t, err, "Should not fail deleting node")
	}

	expectedReturn = []string{}
	addresses, err = getNodeAddresses(fake)
	require.Nil(t, err, "Should not fail getting empty addresses")
	assert.EqualValues(t, expectedReturn, addresses, "Should get no addresses")
}

func validateFile(t *testing.T) {
	configFile, err := os.Open(OutputFilename)
	if nil != err {
		assert.Nil(t, err)
		return
	}

	services := outputConfigs{}

	parser := json.NewDecoder(configFile)
	err = parser.Decode(&services)
	if nil != err {
		assert.Nil(t, err)
		return
	}

	require.EqualValues(t, outputConfigs{[]VirtualServerConfig{}}, services,
		"Should be empty config output")
}

func TestProcessNodeUpdate(t *testing.T) {
	defer os.Remove(OutputFilename)

	originalSet := []v1.Node{
		*newNode("node0", "0", true, []v1.NodeAddress{
			{"ExternalIP", "127.0.0.0"}}),
		*newNode("node1", "1", false, []v1.NodeAddress{
			{"ExternalIP", "127.0.0.1"}}),
		*newNode("node2", "2", false, []v1.NodeAddress{
			{"ExternalIP", "127.0.0.2"}}),
		*newNode("node3", "3", false, []v1.NodeAddress{
			{"ExternalIP", "127.0.0.3"}}),
		*newNode("node4", "4", false, []v1.NodeAddress{
			{"InternalIP", "127.0.0.4"}}),
		*newNode("node5", "5", false, []v1.NodeAddress{
			{"Hostname", "127.0.0.5"}}),
	}

	expectedOgSet := []string{
		"127.0.0.1",
		"127.0.0.2",
		"127.0.0.3",
	}

	fake := fake.NewSimpleClientset(&v1.NodeList{Items: originalSet})
	assert.NotNil(t, fake, "Mock client should not be nil")

	ProcessNodeUpdate(fake)
	validateFile(t)
	require.EqualValues(t, expectedOgSet, oldNodes,
		"Should have cached correct node set")

	cachedNodes := getNodesFromCache()
	require.EqualValues(t, oldNodes, cachedNodes,
		"Cached nodes should be oldNodes")
	require.EqualValues(t, expectedOgSet, cachedNodes,
		"Cached nodes should be expected set")

	// add some nodes
	_, err := fake.Core().Nodes().Create(newNode("nodeAdd", "nodeAdd", false,
		[]v1.NodeAddress{{"ExternalIP", "127.0.0.6"}}))
	require.Nil(t, err, "Create should not return err")

	_, err = fake.Core().Nodes().Create(newNode("nodeExclude", "nodeExclude",
		true, []v1.NodeAddress{{"InternalIP", "127.0.0.7"}}))

	ProcessNodeUpdate(fake)
	validateFile(t)
	expectedAddSet := append(expectedOgSet, "127.0.0.6")

	require.EqualValues(t, expectedAddSet, oldNodes)

	cachedNodes = getNodesFromCache()
	require.EqualValues(t, oldNodes, cachedNodes,
		"Cached nodes should be oldNodes")
	require.EqualValues(t, expectedAddSet, cachedNodes,
		"Cached nodes should be expected set")

	// make no changes and re-run process
	ProcessNodeUpdate(fake)
	validateFile(t)
	expectedAddSet = append(expectedOgSet, "127.0.0.6")

	require.EqualValues(t, expectedAddSet, oldNodes)

	cachedNodes = getNodesFromCache()
	require.EqualValues(t, oldNodes, cachedNodes,
		"Cached nodes should be oldNodes")
	require.EqualValues(t, expectedAddSet, cachedNodes,
		"Cached nodes should be expected set")

	// remove nodes
	err = fake.Core().Nodes().Delete("node1", &api.DeleteOptions{})
	require.Nil(t, err)
	fake.Core().Nodes().Delete("node2", &api.DeleteOptions{})
	require.Nil(t, err)
	fake.Core().Nodes().Delete("node3", &api.DeleteOptions{})
	require.Nil(t, err)

	expectedDelSet := []string{"127.0.0.6"}

	ProcessNodeUpdate(fake)
	validateFile(t)

	require.EqualValues(t, expectedDelSet, oldNodes)

	cachedNodes = getNodesFromCache()
	require.EqualValues(t, oldNodes, cachedNodes,
		"Cached nodes should be oldNodes")
	require.EqualValues(t, expectedDelSet, cachedNodes,
		"Cached nodes should be expected set")
}
<|endoftext|>


<commit_before>// Package web Cozy Stack API.
//
// Cozy is a personal platform as a service with a focus on data.
//
// Terms Of Service:
//
// there are no TOS at this moment, use at your own risk we take no responsibility
//
//     Schemes: https
//     Host: localhost
//     BasePath: /
//     Version: 0.0.1
//     License: AGPL-3.0 https://opensource.org/licenses/agpl-3.0
//     Contact: Bruno Michel <bruno@cozycloud.cc> https://cozy.io/
//
//     Consumes:
//     - application/json
//
//     Produces:
//     - application/json
//
// swagger:meta
package web

import (
	"github.com/cozy/cozy-stack/web/apps"
	"github.com/cozy/cozy-stack/web/data"
	"github.com/cozy/cozy-stack/web/files"
	"github.com/cozy/cozy-stack/web/middlewares"
	"github.com/cozy/cozy-stack/web/status"
	"github.com/cozy/cozy-stack/web/version"
	"github.com/gin-gonic/gin"
)

// SetupRoutes sets the routing for HTTP endpoints to the Go methods
func SetupRoutes(router *gin.Engine) {
	router.Use(middlewares.ParseHost())

	appsRouter = gin.New()
	appsRouter.Use(middlewares.NeedInstance(), apps.Serve)
	router.Use(middlewares.ServeApp(gin.wrapH(appsRouter)))

	router.Use(middlewares.ErrorHandler())
	apps.Routes(router.Group("/apps", middlewares.NeedInstance()))
	data.Routes(router.Group("/data", middlewares.NeedInstance()))
	files.Routes(router.Group("/files", middlewares.NeedInstance()))
	status.Routes(router.Group("/status"))
	version.Routes(router.Group("/version"))
}
<commit_msg>Allow only GET and HEAD on app sub-domains<commit_after>// Package web Cozy Stack API.
//
// Cozy is a personal platform as a service with a focus on data.
//
// Terms Of Service:
//
// there are no TOS at this moment, use at your own risk we take no responsibility
//
//     Schemes: https
//     Host: localhost
//     BasePath: /
//     Version: 0.0.1
//     License: AGPL-3.0 https://opensource.org/licenses/agpl-3.0
//     Contact: Bruno Michel <bruno@cozycloud.cc> https://cozy.io/
//
//     Consumes:
//     - application/json
//
//     Produces:
//     - application/json
//
// swagger:meta
package web

import (
	"github.com/cozy/cozy-stack/web/apps"
	"github.com/cozy/cozy-stack/web/data"
	"github.com/cozy/cozy-stack/web/files"
	"github.com/cozy/cozy-stack/web/middlewares"
	"github.com/cozy/cozy-stack/web/status"
	"github.com/cozy/cozy-stack/web/version"
	"github.com/gin-gonic/gin"
)

// SetupRoutes sets the routing for HTTP endpoints to the Go methods
func SetupRoutes(router *gin.Engine) {
	router.Use(middlewares.ParseHost())

	appsRouter := gin.New()
	appsRouter.HEAD("/*anypath", middlewares.NeedInstance(), apps.Serve)
	appsRouter.GET("/*anypath", middlewares.NeedInstance(), apps.Serve)
	router.Use(middlewares.ServeApp(gin.WrapH(appsRouter)))

	router.Use(middlewares.ErrorHandler())
	apps.Routes(router.Group("/apps", middlewares.NeedInstance()))
	data.Routes(router.Group("/data", middlewares.NeedInstance()))
	files.Routes(router.Group("/files", middlewares.NeedInstance()))
	status.Routes(router.Group("/status"))
	version.Routes(router.Group("/version"))
}
<|endoftext|>


<commit_before>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"errors"
	"io"

	"github.com/tsuru/config"
	"github.com/tsuru/go-gandalfclient"
	"github.com/tsuru/tsuru/action"
	"github.com/tsuru/tsuru/app/bind"
	"github.com/tsuru/tsuru/auth"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/log"
	"github.com/tsuru/tsuru/provision"
	"github.com/tsuru/tsuru/quota"
	"github.com/tsuru/tsuru/repository"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var (
	ErrAppAlreadyExists = errors.New("there is already an app with this name")
	ErrAppNotFound      = errors.New("App not found.")
)

// reserveUserApp reserves the app for the user, only if the user has a quota
// of apps. If the user does not have a quota, meaning that it's unlimited,
// reserveUserApp.Forward just return nil.
var reserveUserApp = action.Action{
	Name: "reserve-user-app",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		var app *App
		switch ctx.Params[0].(type) {
		case *App:
			app = ctx.Params[0].(*App)
		default:
			return nil, errors.New("First parameter must be *App.")
		}
		var user auth.User
		switch ctx.Params[1].(type) {
		case auth.User:
			user = ctx.Params[1].(auth.User)
		case *auth.User:
			user = *ctx.Params[1].(*auth.User)
		default:
			return nil, errors.New("Third parameter must be auth.User or *auth.User.")
		}
		usr, err := auth.GetUserByEmail(user.Email)
		if err != nil {
			return nil, err
		}
		if err := auth.ReserveApp(usr); err != nil {
			return nil, err
		}
		return map[string]string{"app": app.Name, "user": user.Email}, nil
	},
	Backward: func(ctx action.BWContext) {
		m := ctx.FWResult.(map[string]string)
		if user, err := auth.GetUserByEmail(m["user"]); err == nil {
			auth.ReleaseApp(user)
		}
	},
	MinParams: 2,
}

// insertApp is an action that inserts an app in the database in Forward and
// removes it in the Backward.
//
// The first argument in the context must be an App or a pointer to an App.
var insertApp = action.Action{
	Name: "insert-app",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		var app *App
		switch ctx.Params[0].(type) {
		case *App:
			app = ctx.Params[0].(*App)
		default:
			return nil, errors.New("First parameter must be *App.")
		}
		conn, err := db.Conn()
		if err != nil {
			return nil, err
		}
		defer conn.Close()
		app.Quota = quota.Unlimited
		if limit, err := config.GetInt("quota:units-per-app"); err == nil {
			app.Quota.Limit = limit
		}
		err = conn.Apps().Insert(app)
		if mgo.IsDup(err) {
			return nil, ErrAppAlreadyExists
		}
		return app, err
	},
	Backward: func(ctx action.BWContext) {
		app := ctx.FWResult.(*App)
		conn, err := db.Conn()
		if err != nil {
			log.Errorf("Could not connect to the database: %s", err)
			return
		}
		defer conn.Close()
		conn.Apps().Remove(bson.M{"name": app.Name})
	},
	MinParams: 1,
}

// exportEnvironmentsAction exports tsuru's default environment variables in a
// new app. It requires a pointer to an App instance as the first parameter.
var exportEnvironmentsAction = action.Action{
	Name: "export-environments",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		app := ctx.Params[0].(*App)
		app, err := GetByName(app.Name)
		if err != nil {
			return nil, err
		}
		t, err := AuthScheme.AppLogin(app.Name)
		if err != nil {
			return nil, err
		}
		host, _ := config.GetString("host")
		appdir, _ := config.GetString("git:unit-repo")
		envVars := []bind.EnvVar{
			{Name: "TSURU_APPNAME", Value: app.Name},
			{Name: "TSURU_APPDIR", Value: appdir},
			{Name: "TSURU_HOST", Value: host},
			{Name: "TSURU_APP_TOKEN", Value: t.GetValue()},
		}
		err = app.setEnvsToApp(envVars, false, false, nil)
		if err != nil {
			return nil, err
		}
		return ctx.Previous, nil
	},
	Backward: func(ctx action.BWContext) {
		app := ctx.Params[0].(*App)
		AuthScheme.Logout(app.Env["TSURU_APP_TOKEN"].Value)
		app, err := GetByName(app.Name)
		if err == nil {
			vars := []string{"TSURU_HOST", "TSURU_APPNAME", "TSURU_APP_TOKEN"}
			app.UnsetEnvs(vars, false, nil)
		}
	},
	MinParams: 1,
}

// createRepository creates a repository for the app in Gandalf.
var createRepository = action.Action{
	Name: "create-repository",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		var app *App
		switch ctx.Params[0].(type) {
		case *App:
			app = ctx.Params[0].(*App)
		default:
			return nil, errors.New("First parameter must be *App.")
		}
		gURL := repository.ServerURL()
		var users []string
		for _, t := range app.GetTeams() {
			users = append(users, t.Users...)
		}
		c := gandalf.Client{Endpoint: gURL}
		_, err := c.NewRepository(app.Name, users, false)
		return app, err
	},
	Backward: func(ctx action.BWContext) {
		app := ctx.FWResult.(*App)
		gURL := repository.ServerURL()
		c := gandalf.Client{Endpoint: gURL}
		c.RemoveRepository(app.Name)
	},
	MinParams: 1,
}

// provisionApp provisions the app in the provisioner. It takes two arguments:
// the app, and the number of units to create (an unsigned integer).
var provisionApp = action.Action{
	Name: "provision-app",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		var app *App
		switch ctx.Params[0].(type) {
		case *App:
			app = ctx.Params[0].(*App)
		default:
			return nil, errors.New("First parameter must be *App.")
		}
		err := Provisioner.Provision(app)
		if err != nil {
			return nil, err
		}
		return app, nil
	},
	Backward: func(ctx action.BWContext) {
		app := ctx.FWResult.(*App)
		Provisioner.Destroy(app)
	},
	MinParams: 1,
}

var setAppIp = action.Action{
	Name: "set-app-ip",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		var app *App
		switch ctx.Params[0].(type) {
		case *App:
			app = ctx.Params[0].(*App)
		default:
			return nil, errors.New("First parameter must be *App.")
		}
		conn, err := db.Conn()
		if err != nil {
			return nil, err
		}
		defer conn.Close()
		app.Ip, err = Provisioner.Addr(app)
		if err != nil {
			return nil, err
		}
		err = conn.Apps().Update(bson.M{"name": app.Name}, bson.M{"$set": bson.M{"ip": app.Ip}})
		if err != nil {
			return nil, err
		}
		return app, nil
	},
	Backward: func(ctx action.BWContext) {
		app := ctx.FWResult.(*App)
		app.Ip = ""
		conn, err := db.Conn()
		if err != nil {
			log.Errorf("Error trying to get connection to rollback setAppIp action: %s", err)
		}
		defer conn.Close()
		err = conn.Apps().Update(bson.M{"name": app.Name}, bson.M{"$unset": bson.M{"ip": ""}})
		if err != nil {
			log.Errorf("Error trying to update app to rollback setAppIp action: %s", err)
		}
	},
	MinParams: 1,
}

var reserveUnitsToAdd = action.Action{
	Name: "reserve-units-to-add",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		var app *App
		switch ctx.Params[0].(type) {
		case *App:
			app = ctx.Params[0].(*App)
		default:
			return nil, errors.New("First parameter must be *App.")
		}
		var n int
		switch ctx.Params[1].(type) {
		case int:
			n = ctx.Params[1].(int)
		case uint:
			n = int(ctx.Params[1].(uint))
		default:
			return nil, errors.New("Second parameter must be int or uint.")
		}
		conn, err := db.Conn()
		if err != nil {
			return nil, err
		}
		defer conn.Close()
		app, err = GetByName(app.Name)
		if err != nil {
			return nil, ErrAppNotFound
		}
		err = reserveUnits(app, n)
		if err != nil {
			return nil, err
		}
		return n, nil
	},
	Backward: func(ctx action.BWContext) {
		var app *App
		switch ctx.Params[0].(type) {
		case *App:
			app = ctx.Params[0].(*App)
		}
		qty := ctx.FWResult.(int)
		err := releaseUnits(app, qty)
		if err != nil {
			log.Errorf("Failed to rollback reserveUnitsToAdd: %s", err)
		}
	},
	MinParams: 2,
}

var provisionAddUnits = action.Action{
	Name: "provision-add-units",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		var app *App
		switch ctx.Params[0].(type) {
		case *App:
			app = ctx.Params[0].(*App)
		default:
			return nil, errors.New("First parameter must be *App.")
		}
		var w io.Writer
		if len(ctx.Params) >= 3 {
			w, _ = ctx.Params[2].(io.Writer)
		}
		n := ctx.Previous.(int)
		units, err := Provisioner.AddUnits(app, uint(n), w)
		if err != nil {
			return nil, err
		}
		return units, nil
	},
	Backward: func(ctx action.BWContext) {
		units := ctx.FWResult.([]provision.Unit)
		for _, unit := range units {
			Provisioner.RemoveUnit(unit)
		}
	},
	MinParams: 1,
}

var BindService = action.Action{
	Name: "bind-service",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		var app *App
		switch ctx.Params[0].(type) {
		case *App:
			app = ctx.Params[0].(*App)
		case DeployOptions:
			opts := ctx.Params[0].(DeployOptions)
			app = opts.App
		default:
			return nil, errors.New("First parameter must be *App or DeployOptions.")
		}
		units, _ := ctx.Previous.([]provision.Unit)
		app, err := GetByName(app.Name)
		if err != nil {
			return nil, ErrAppNotFound
		}
		if len(units) == 0 {
			units = app.Units()
		}
		for _, unit := range units {
			err := app.BindUnit(&unit)
			if err != nil {
				return nil, err
			}
		}
		return nil, nil
	},
	MinParams: 1,
}

// ProvisionerDeploy is an actions that call the Provisioner.Deploy.
var ProvisionerDeploy = action.Action{
	Name: "provisioner-deploy",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		opts, ok := ctx.Params[0].(DeployOptions)
		if !ok {
			return nil, errors.New("First parameter must be DeployOptions")
		}
		writer, ok := ctx.Params[1].(io.Writer)
		if !ok {
			return nil, errors.New("Second parameter must be an io.Writer")
		}
		if opts.File != nil {
			if deployer, ok := Provisioner.(provision.UploadDeployer); ok {
				return nil, deployer.UploadDeploy(opts.App, opts.File, writer)
			}
		}
		if opts.ArchiveURL != "" {
			if deployer, ok := Provisioner.(provision.ArchiveDeployer); ok {
				return nil, deployer.ArchiveDeploy(opts.App, opts.ArchiveURL, writer)
			}
		}
		err := Provisioner.(provision.GitDeployer).GitDeploy(opts.App, opts.Version, writer)
		return nil, err
	},
	Backward: func(ctx action.BWContext) {
	},
	MinParams: 2,
}

// Increment is an actions that increments the deploy number.
var IncrementDeploy = action.Action{
	Name: "increment-deploy",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		opts, ok := ctx.Params[0].(DeployOptions)
		if !ok {
			return nil, errors.New("First parameter must be DeployOptions")
		}
		err := incrementDeploy(opts.App)
		return nil, err
	},
	Backward: func(ctx action.BWContext) {
	},
	MinParams: 1,
}
<commit_msg>app: fix typo in doc<commit_after>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"errors"
	"io"

	"github.com/tsuru/config"
	"github.com/tsuru/go-gandalfclient"
	"github.com/tsuru/tsuru/action"
	"github.com/tsuru/tsuru/app/bind"
	"github.com/tsuru/tsuru/auth"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/log"
	"github.com/tsuru/tsuru/provision"
	"github.com/tsuru/tsuru/quota"
	"github.com/tsuru/tsuru/repository"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var (
	ErrAppAlreadyExists = errors.New("there is already an app with this name")
	ErrAppNotFound      = errors.New("App not found.")
)

// reserveUserApp reserves the app for the user, only if the user has a quota
// of apps. If the user does not have a quota, meaning that it's unlimited,
// reserveUserApp.Forward just return nil.
var reserveUserApp = action.Action{
	Name: "reserve-user-app",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		var app *App
		switch ctx.Params[0].(type) {
		case *App:
			app = ctx.Params[0].(*App)
		default:
			return nil, errors.New("First parameter must be *App.")
		}
		var user auth.User
		switch ctx.Params[1].(type) {
		case auth.User:
			user = ctx.Params[1].(auth.User)
		case *auth.User:
			user = *ctx.Params[1].(*auth.User)
		default:
			return nil, errors.New("Third parameter must be auth.User or *auth.User.")
		}
		usr, err := auth.GetUserByEmail(user.Email)
		if err != nil {
			return nil, err
		}
		if err := auth.ReserveApp(usr); err != nil {
			return nil, err
		}
		return map[string]string{"app": app.Name, "user": user.Email}, nil
	},
	Backward: func(ctx action.BWContext) {
		m := ctx.FWResult.(map[string]string)
		if user, err := auth.GetUserByEmail(m["user"]); err == nil {
			auth.ReleaseApp(user)
		}
	},
	MinParams: 2,
}

// insertApp is an action that inserts an app in the database in Forward and
// removes it in the Backward.
//
// The first argument in the context must be an App or a pointer to an App.
var insertApp = action.Action{
	Name: "insert-app",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		var app *App
		switch ctx.Params[0].(type) {
		case *App:
			app = ctx.Params[0].(*App)
		default:
			return nil, errors.New("First parameter must be *App.")
		}
		conn, err := db.Conn()
		if err != nil {
			return nil, err
		}
		defer conn.Close()
		app.Quota = quota.Unlimited
		if limit, err := config.GetInt("quota:units-per-app"); err == nil {
			app.Quota.Limit = limit
		}
		err = conn.Apps().Insert(app)
		if mgo.IsDup(err) {
			return nil, ErrAppAlreadyExists
		}
		return app, err
	},
	Backward: func(ctx action.BWContext) {
		app := ctx.FWResult.(*App)
		conn, err := db.Conn()
		if err != nil {
			log.Errorf("Could not connect to the database: %s", err)
			return
		}
		defer conn.Close()
		conn.Apps().Remove(bson.M{"name": app.Name})
	},
	MinParams: 1,
}

// exportEnvironmentsAction exports tsuru's default environment variables in a
// new app. It requires a pointer to an App instance as the first parameter.
var exportEnvironmentsAction = action.Action{
	Name: "export-environments",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		app := ctx.Params[0].(*App)
		app, err := GetByName(app.Name)
		if err != nil {
			return nil, err
		}
		t, err := AuthScheme.AppLogin(app.Name)
		if err != nil {
			return nil, err
		}
		host, _ := config.GetString("host")
		appdir, _ := config.GetString("git:unit-repo")
		envVars := []bind.EnvVar{
			{Name: "TSURU_APPNAME", Value: app.Name},
			{Name: "TSURU_APPDIR", Value: appdir},
			{Name: "TSURU_HOST", Value: host},
			{Name: "TSURU_APP_TOKEN", Value: t.GetValue()},
		}
		err = app.setEnvsToApp(envVars, false, false, nil)
		if err != nil {
			return nil, err
		}
		return ctx.Previous, nil
	},
	Backward: func(ctx action.BWContext) {
		app := ctx.Params[0].(*App)
		AuthScheme.Logout(app.Env["TSURU_APP_TOKEN"].Value)
		app, err := GetByName(app.Name)
		if err == nil {
			vars := []string{"TSURU_HOST", "TSURU_APPNAME", "TSURU_APP_TOKEN"}
			app.UnsetEnvs(vars, false, nil)
		}
	},
	MinParams: 1,
}

// createRepository creates a repository for the app in Gandalf.
var createRepository = action.Action{
	Name: "create-repository",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		var app *App
		switch ctx.Params[0].(type) {
		case *App:
			app = ctx.Params[0].(*App)
		default:
			return nil, errors.New("First parameter must be *App.")
		}
		gURL := repository.ServerURL()
		var users []string
		for _, t := range app.GetTeams() {
			users = append(users, t.Users...)
		}
		c := gandalf.Client{Endpoint: gURL}
		_, err := c.NewRepository(app.Name, users, false)
		return app, err
	},
	Backward: func(ctx action.BWContext) {
		app := ctx.FWResult.(*App)
		gURL := repository.ServerURL()
		c := gandalf.Client{Endpoint: gURL}
		c.RemoveRepository(app.Name)
	},
	MinParams: 1,
}

// provisionApp provisions the app in the provisioner. It takes two arguments:
// the app, and the number of units to create (an unsigned integer).
var provisionApp = action.Action{
	Name: "provision-app",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		var app *App
		switch ctx.Params[0].(type) {
		case *App:
			app = ctx.Params[0].(*App)
		default:
			return nil, errors.New("First parameter must be *App.")
		}
		err := Provisioner.Provision(app)
		if err != nil {
			return nil, err
		}
		return app, nil
	},
	Backward: func(ctx action.BWContext) {
		app := ctx.FWResult.(*App)
		Provisioner.Destroy(app)
	},
	MinParams: 1,
}

var setAppIp = action.Action{
	Name: "set-app-ip",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		var app *App
		switch ctx.Params[0].(type) {
		case *App:
			app = ctx.Params[0].(*App)
		default:
			return nil, errors.New("First parameter must be *App.")
		}
		conn, err := db.Conn()
		if err != nil {
			return nil, err
		}
		defer conn.Close()
		app.Ip, err = Provisioner.Addr(app)
		if err != nil {
			return nil, err
		}
		err = conn.Apps().Update(bson.M{"name": app.Name}, bson.M{"$set": bson.M{"ip": app.Ip}})
		if err != nil {
			return nil, err
		}
		return app, nil
	},
	Backward: func(ctx action.BWContext) {
		app := ctx.FWResult.(*App)
		app.Ip = ""
		conn, err := db.Conn()
		if err != nil {
			log.Errorf("Error trying to get connection to rollback setAppIp action: %s", err)
		}
		defer conn.Close()
		err = conn.Apps().Update(bson.M{"name": app.Name}, bson.M{"$unset": bson.M{"ip": ""}})
		if err != nil {
			log.Errorf("Error trying to update app to rollback setAppIp action: %s", err)
		}
	},
	MinParams: 1,
}

var reserveUnitsToAdd = action.Action{
	Name: "reserve-units-to-add",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		var app *App
		switch ctx.Params[0].(type) {
		case *App:
			app = ctx.Params[0].(*App)
		default:
			return nil, errors.New("First parameter must be *App.")
		}
		var n int
		switch ctx.Params[1].(type) {
		case int:
			n = ctx.Params[1].(int)
		case uint:
			n = int(ctx.Params[1].(uint))
		default:
			return nil, errors.New("Second parameter must be int or uint.")
		}
		conn, err := db.Conn()
		if err != nil {
			return nil, err
		}
		defer conn.Close()
		app, err = GetByName(app.Name)
		if err != nil {
			return nil, ErrAppNotFound
		}
		err = reserveUnits(app, n)
		if err != nil {
			return nil, err
		}
		return n, nil
	},
	Backward: func(ctx action.BWContext) {
		var app *App
		switch ctx.Params[0].(type) {
		case *App:
			app = ctx.Params[0].(*App)
		}
		qty := ctx.FWResult.(int)
		err := releaseUnits(app, qty)
		if err != nil {
			log.Errorf("Failed to rollback reserveUnitsToAdd: %s", err)
		}
	},
	MinParams: 2,
}

var provisionAddUnits = action.Action{
	Name: "provision-add-units",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		var app *App
		switch ctx.Params[0].(type) {
		case *App:
			app = ctx.Params[0].(*App)
		default:
			return nil, errors.New("First parameter must be *App.")
		}
		var w io.Writer
		if len(ctx.Params) >= 3 {
			w, _ = ctx.Params[2].(io.Writer)
		}
		n := ctx.Previous.(int)
		units, err := Provisioner.AddUnits(app, uint(n), w)
		if err != nil {
			return nil, err
		}
		return units, nil
	},
	Backward: func(ctx action.BWContext) {
		units := ctx.FWResult.([]provision.Unit)
		for _, unit := range units {
			Provisioner.RemoveUnit(unit)
		}
	},
	MinParams: 1,
}

var BindService = action.Action{
	Name: "bind-service",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		var app *App
		switch ctx.Params[0].(type) {
		case *App:
			app = ctx.Params[0].(*App)
		case DeployOptions:
			opts := ctx.Params[0].(DeployOptions)
			app = opts.App
		default:
			return nil, errors.New("First parameter must be *App or DeployOptions.")
		}
		units, _ := ctx.Previous.([]provision.Unit)
		app, err := GetByName(app.Name)
		if err != nil {
			return nil, ErrAppNotFound
		}
		if len(units) == 0 {
			units = app.Units()
		}
		for _, unit := range units {
			err := app.BindUnit(&unit)
			if err != nil {
				return nil, err
			}
		}
		return nil, nil
	},
	MinParams: 1,
}

// ProvisionerDeploy is an action that calls the Provisioner.Deploy.
var ProvisionerDeploy = action.Action{
	Name: "provisioner-deploy",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		opts, ok := ctx.Params[0].(DeployOptions)
		if !ok {
			return nil, errors.New("First parameter must be DeployOptions")
		}
		writer, ok := ctx.Params[1].(io.Writer)
		if !ok {
			return nil, errors.New("Second parameter must be an io.Writer")
		}
		if opts.File != nil {
			if deployer, ok := Provisioner.(provision.UploadDeployer); ok {
				return nil, deployer.UploadDeploy(opts.App, opts.File, writer)
			}
		}
		if opts.ArchiveURL != "" {
			if deployer, ok := Provisioner.(provision.ArchiveDeployer); ok {
				return nil, deployer.ArchiveDeploy(opts.App, opts.ArchiveURL, writer)
			}
		}
		err := Provisioner.(provision.GitDeployer).GitDeploy(opts.App, opts.Version, writer)
		return nil, err
	},
	Backward: func(ctx action.BWContext) {
	},
	MinParams: 2,
}

// Increment is an actions that increments the deploy number.
var IncrementDeploy = action.Action{
	Name: "increment-deploy",
	Forward: func(ctx action.FWContext) (action.Result, error) {
		opts, ok := ctx.Params[0].(DeployOptions)
		if !ok {
			return nil, errors.New("First parameter must be DeployOptions")
		}
		err := incrementDeploy(opts.App)
		return nil, err
	},
	Backward: func(ctx action.BWContext) {
	},
	MinParams: 1,
}
<|endoftext|>


<commit_before>// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package prog

import (
	"bytes"
	"fmt"
	"math/rand"
	"sort"
	"testing"
)

func TestNotEscaping(t *testing.T) {
	r := newRand(nil, rand.NewSource(0))
	s := &state{
		files: map[string]bool{"./file0": true},
	}
	bound := 1000000
	if testing.Short() {
		bound = 1000
	}
	for i := 0; i < bound; i++ {
		fn := r.filenameImpl(s)
		if escapingFilename(fn) {
			t.Errorf("sandbox escaping file name %q", fn)
		}
	}
}

func TestDeterminism(t *testing.T) {
	target, rs, iters := initTest(t)
	iters /= 10 // takes too long
	ct := target.DefaultChoiceTable()
	var corpus []*Prog
	for i := 0; i < iters; i++ {
		seed := rs.Int63()
		rs1 := rand.NewSource(seed)
		p1 := generateProg(t, target, rs1, ct, corpus)
		rs2 := rand.NewSource(seed)
		p2 := generateProg(t, target, rs2, ct, corpus)
		ps1 := string(p1.Serialize())
		ps2 := string(p2.Serialize())
		r1 := rs1.Int63()
		r2 := rs2.Int63()
		if r1 != r2 || ps1 != ps2 {
			t.Errorf("seed=%v\nprog 1 (%v):\n%v\nprog 2 (%v):\n%v", seed, r1, ps1, r2, ps2)
		}
		corpus = append(corpus, p1)
	}
}

func generateProg(t *testing.T, target *Target, rs rand.Source, ct *ChoiceTable, corpus []*Prog) *Prog {
	p := target.Generate(rs, 5, ct)
	p.Mutate(rs, 10, ct, corpus)
	for i, c := range p.Calls {
		comps := make(CompMap)
		for v := range extractValues(c) {
			comps.AddComp(v, v+1)
			comps.AddComp(v, v+10)
		}
		p.MutateWithHints(i, comps, func(p1 *Prog) {
			p = p1.Clone()
		})
	}
	for _, crash := range []bool{false, true} {
		p, _ = Minimize(p, -1, crash, func(*Prog, int) bool {
			return rs.Int63()%10 == 0
		})
	}
	data := p.Serialize()
	var err error
	p, err = target.Deserialize(data, NonStrict)
	if err != nil {
		t.Fatal(err)
	}
	return p
}

// Checks that a generated program contains only enabled syscalls.
func TestEnabledCalls(t *testing.T) {
	target, rs, iters := initTest(t)
	enabledCalls := map[string]bool{"open": true, "read": true, "dup3": true, "write": true, "close": true}
	enabled := make(map[*Syscall]bool)
	for c := range enabledCalls {
		enabled[target.SyscallMap[c]] = true
	}
	ct := target.BuildChoiceTable(nil, enabled)
	const tries = 10
	for i := 0; i < tries; i++ {
		p := target.Generate(rs, 50, ct)
		for it := 0; it < iters/tries; it++ {
			p.Mutate(rs, 50, ct, nil)
		}
		for _, c := range p.Calls {
			if _, ok := enabledCalls[c.Meta.Name]; !ok {
				t.Fatalf("program contains a syscall that is not enabled: %v\n", c.Meta.Name)
			}
		}
	}
}

func TestSizeGenerateConstArg(t *testing.T) {
	target, rs, iters := initRandomTargetTest(t, "test", "64")
	r := newRand(target, rs)
	ForeachType(target.Syscalls, func(typ Type, ctx TypeCtx) {
		if _, ok := typ.(*IntType); !ok {
			return
		}
		bits := typ.TypeBitSize()
		limit := uint64(1<<bits - 1)
		for i := 0; i < iters; i++ {
			newArg, _ := typ.generate(r, nil, ctx.Dir)
			newVal := newArg.(*ConstArg).Val
			if newVal > limit {
				t.Fatalf("invalid generated value: %d. (arg bitsize: %d; max value: %d)", newVal, bits, limit)
			}
		}
	})
}

func TestFlags(t *testing.T) {
	// This test does not test anything, it just prints resulting
	// distribution of values for different scenarios.
	tests := []struct {
		vv      []uint64
		bitmask bool
		old     uint64
	}{
		{[]uint64{0, 1, 2, 3}, false, 0},
		{[]uint64{0, 1, 2, 3}, false, 2},
		{[]uint64{1, 2, 3, 4}, false, 0},
		{[]uint64{1, 2, 3, 4}, false, 2},
		{[]uint64{1, 2, 4, 8}, true, 0},
		{[]uint64{1, 2, 4, 8}, true, 2},
		{[]uint64{7}, false, 0},
		{[]uint64{7}, false, 7},
		{[]uint64{1, 2}, true, 0},
		{[]uint64{1, 2}, true, 2},
	}
	target, rs, _ := initRandomTargetTest(t, "test", "64")
	r := newRand(target, rs)
	for _, test := range tests {
		results := make(map[uint64]uint64)
		const throws = 1e4
		for i := 0; i < throws; i++ {
			var v uint64
			for {
				v = r.flags(test.vv, test.bitmask, test.old)
				if test.old == 0 || test.old != v {
					break
				}
			}
			if v > 100 {
				v = 999 // to not print all possible random values we generated
			}
			results[v]++
		}
		var sorted [][2]uint64
		for v, c := range results {
			sorted = append(sorted, [2]uint64{v, c})
		}
		sort.Slice(sorted, func(i, j int) bool {
			return sorted[i][0] < sorted[j][0]
		})
		buf := new(bytes.Buffer)
		for _, p := range sorted {
			fmt.Fprintf(buf, "%v\t%v\n", p[0], p[1])
		}
		t.Logf("test: vv=%+v bitmask=%v old=%v\nvalue\ttimes (out of %v)\n%v",
			test.vv, test.bitmask, test.old, throws, buf.String())
	}
}

func TestTruncateToBitSize(t *testing.T) {
	tests := []struct{ v, bits, res uint64 }{
		{0, 1, 0},
		{1, 1, 1},
		{0x123, 4, 0x3},
		{0xabc, 4, 0xc},
		{0x123, 8, 0x23},
		{0xabc, 8, 0xbc},
		{0x12345678abcdabcd, 64, 0x12345678abcdabcd},
		{0xf2345678abcdabcd, 64, 0xf2345678abcdabcd},
	}
	for i, test := range tests {
		t.Run(fmt.Sprint(i), func(t *testing.T) {
			res := truncateToBitSize(test.v, test.bits)
			if res != test.res {
				t.Fatalf("truncateToBitSize(0x%x, %v)=0x%x, want 0x%x", test.v, test.bits, res, test.res)
			}
		})
	}
}
<commit_msg>prog: extend TestEnabledCalls test<commit_after>// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package prog

import (
	"bytes"
	"fmt"
	"math/rand"
	"sort"
	"testing"
)

func TestNotEscaping(t *testing.T) {
	r := newRand(nil, rand.NewSource(0))
	s := &state{
		files: map[string]bool{"./file0": true},
	}
	bound := 1000000
	if testing.Short() {
		bound = 1000
	}
	for i := 0; i < bound; i++ {
		fn := r.filenameImpl(s)
		if escapingFilename(fn) {
			t.Errorf("sandbox escaping file name %q", fn)
		}
	}
}

func TestDeterminism(t *testing.T) {
	target, rs, iters := initTest(t)
	iters /= 10 // takes too long
	ct := target.DefaultChoiceTable()
	var corpus []*Prog
	for i := 0; i < iters; i++ {
		seed := rs.Int63()
		rs1 := rand.NewSource(seed)
		p1 := generateProg(t, target, rs1, ct, corpus)
		rs2 := rand.NewSource(seed)
		p2 := generateProg(t, target, rs2, ct, corpus)
		ps1 := string(p1.Serialize())
		ps2 := string(p2.Serialize())
		r1 := rs1.Int63()
		r2 := rs2.Int63()
		if r1 != r2 || ps1 != ps2 {
			t.Errorf("seed=%v\nprog 1 (%v):\n%v\nprog 2 (%v):\n%v", seed, r1, ps1, r2, ps2)
		}
		corpus = append(corpus, p1)
	}
}

func generateProg(t *testing.T, target *Target, rs rand.Source, ct *ChoiceTable, corpus []*Prog) *Prog {
	p := target.Generate(rs, 5, ct)
	p.Mutate(rs, 10, ct, corpus)
	for i, c := range p.Calls {
		comps := make(CompMap)
		for v := range extractValues(c) {
			comps.AddComp(v, v+1)
			comps.AddComp(v, v+10)
		}
		p.MutateWithHints(i, comps, func(p1 *Prog) {
			p = p1.Clone()
		})
	}
	for _, crash := range []bool{false, true} {
		p, _ = Minimize(p, -1, crash, func(*Prog, int) bool {
			return rs.Int63()%10 == 0
		})
	}
	data := p.Serialize()
	var err error
	p, err = target.Deserialize(data, NonStrict)
	if err != nil {
		t.Fatal(err)
	}
	return p
}

// Checks that a generated program contains only enabled syscalls.
func TestEnabledCalls(t *testing.T) {
	target, rs, iters := initTest(t)
	enabledCalls := make(map[string]bool)
	enabled := make(map[*Syscall]bool)
	rnd := rand.New(rs)
	for i := 0; i < len(target.Syscalls)/10; i++ {
		c := target.Syscalls[rnd.Intn(len(target.Syscalls))]
		enabled[c] = true
		enabledCalls[c.Name] = true
	}
	c := target.SyscallMap["clock_gettime"]
	enabled[c] = true
	enabledCalls[c.Name] = true

	ct := target.BuildChoiceTable(nil, enabled)
	const tries = 10
	for i := 0; i < tries; i++ {
		p := target.Generate(rs, 50, ct)
		for it := 0; it < iters/tries; it++ {
			p.Mutate(rs, 50, ct, nil)
		}
		for _, c := range p.Calls {
			if _, ok := enabledCalls[c.Meta.Name]; !ok {
				t.Fatalf("program contains a syscall that is not enabled: %v\n", c.Meta.Name)
			}
		}
	}
}

func TestSizeGenerateConstArg(t *testing.T) {
	target, rs, iters := initRandomTargetTest(t, "test", "64")
	r := newRand(target, rs)
	ForeachType(target.Syscalls, func(typ Type, ctx TypeCtx) {
		if _, ok := typ.(*IntType); !ok {
			return
		}
		bits := typ.TypeBitSize()
		limit := uint64(1<<bits - 1)
		for i := 0; i < iters; i++ {
			newArg, _ := typ.generate(r, nil, ctx.Dir)
			newVal := newArg.(*ConstArg).Val
			if newVal > limit {
				t.Fatalf("invalid generated value: %d. (arg bitsize: %d; max value: %d)", newVal, bits, limit)
			}
		}
	})
}

func TestFlags(t *testing.T) {
	// This test does not test anything, it just prints resulting
	// distribution of values for different scenarios.
	tests := []struct {
		vv      []uint64
		bitmask bool
		old     uint64
	}{
		{[]uint64{0, 1, 2, 3}, false, 0},
		{[]uint64{0, 1, 2, 3}, false, 2},
		{[]uint64{1, 2, 3, 4}, false, 0},
		{[]uint64{1, 2, 3, 4}, false, 2},
		{[]uint64{1, 2, 4, 8}, true, 0},
		{[]uint64{1, 2, 4, 8}, true, 2},
		{[]uint64{7}, false, 0},
		{[]uint64{7}, false, 7},
		{[]uint64{1, 2}, true, 0},
		{[]uint64{1, 2}, true, 2},
	}
	target, rs, _ := initRandomTargetTest(t, "test", "64")
	r := newRand(target, rs)
	for _, test := range tests {
		results := make(map[uint64]uint64)
		const throws = 1e4
		for i := 0; i < throws; i++ {
			var v uint64
			for {
				v = r.flags(test.vv, test.bitmask, test.old)
				if test.old == 0 || test.old != v {
					break
				}
			}
			if v > 100 {
				v = 999 // to not print all possible random values we generated
			}
			results[v]++
		}
		var sorted [][2]uint64
		for v, c := range results {
			sorted = append(sorted, [2]uint64{v, c})
		}
		sort.Slice(sorted, func(i, j int) bool {
			return sorted[i][0] < sorted[j][0]
		})
		buf := new(bytes.Buffer)
		for _, p := range sorted {
			fmt.Fprintf(buf, "%v\t%v\n", p[0], p[1])
		}
		t.Logf("test: vv=%+v bitmask=%v old=%v\nvalue\ttimes (out of %v)\n%v",
			test.vv, test.bitmask, test.old, throws, buf.String())
	}
}

func TestTruncateToBitSize(t *testing.T) {
	tests := []struct{ v, bits, res uint64 }{
		{0, 1, 0},
		{1, 1, 1},
		{0x123, 4, 0x3},
		{0xabc, 4, 0xc},
		{0x123, 8, 0x23},
		{0xabc, 8, 0xbc},
		{0x12345678abcdabcd, 64, 0x12345678abcdabcd},
		{0xf2345678abcdabcd, 64, 0xf2345678abcdabcd},
	}
	for i, test := range tests {
		t.Run(fmt.Sprint(i), func(t *testing.T) {
			res := truncateToBitSize(test.v, test.bits)
			if res != test.res {
				t.Fatalf("truncateToBitSize(0x%x, %v)=0x%x, want 0x%x", test.v, test.bits, res, test.res)
			}
		})
	}
}
<|endoftext|>


<commit_before>// Package youtube provides loading audio from video files for given youtube channels
package youtube

import (
	"context"
	"crypto/sha1"
	"encoding/xml"
	"fmt"
	"os"
	"path"
	"strings"
	"time"

	log "github.com/go-pkgz/lgr"
	"github.com/google/uuid"
	"github.com/pkg/errors"

	rssfeed "github.com/umputun/feed-master/app/feed"
	ytfeed "github.com/umputun/feed-master/app/youtube/feed"
)

//go:generate moq -out mocks/downloader.go -pkg mocks -skip-ensure -fmt goimports . DownloaderService
//go:generate moq -out mocks/channel.go -pkg mocks -skip-ensure -fmt goimports . ChannelService
//go:generate moq -out mocks/store.go -pkg mocks -skip-ensure -fmt goimports . StoreService

// Service loads audio from youtube channels
type Service struct {
	Feeds          []FeedInfo
	Downloader     DownloaderService
	ChannelService ChannelService
	Store          StoreService
	CheckDuration  time.Duration
	RSSFileStore   RSSFileStore
	KeepPerChannel int
	RootURL        string
}

// FeedInfo contains channel or feed ID, readable name and other per-feed info
type FeedInfo struct {
	Name     string      `yaml:"name"`
	ID       string      `yaml:"id"`
	Type     ytfeed.Type `yaml:"type"`
	Keep     int         `yaml:"keep"`
	Language string      `yaml:"lang"`
}

// DownloaderService is an interface for downloading audio from youtube
type DownloaderService interface {
	Get(ctx context.Context, id string, fname string) (file string, err error)
}

// ChannelService is an interface for getting channel entries, i.e. the list of videos
type ChannelService interface {
	Get(ctx context.Context, chanID string, feedType ytfeed.Type) ([]ytfeed.Entry, error)
}

// StoreService is an interface for storing and loading metadata about downloaded audio
type StoreService interface {
	Save(entry ytfeed.Entry) (bool, error)
	Load(channelID string, max int) ([]ytfeed.Entry, error)
	Exist(entry ytfeed.Entry) (bool, error)
	RemoveOld(channelID string, keep int) ([]string, error)
	SetProcessed(entry ytfeed.Entry) error
	CheckProcessed(entry ytfeed.Entry) (found bool, ts time.Time, err error)
}

// Do is a blocking function that downloads audio from youtube channels and updates metadata
func (s *Service) Do(ctx context.Context) error {
	log.Printf("[INFO] starting youtube service")

	for _, f := range s.Feeds {
		log.Printf("[INFO] youtube feed %+v", f)
	}

	tick := time.NewTicker(s.CheckDuration)
	defer tick.Stop()

	if err := s.procChannels(ctx); err != nil {
		return errors.Wrap(err, "failed to process channels")
	}

	for {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-tick.C:
			if err := s.procChannels(ctx); err != nil {
				return errors.Wrap(err, "failed to process channels")
			}
		}
	}
}

// RSSFeed generates RSS feed for given channel
func (s *Service) RSSFeed(fi FeedInfo) (string, error) {
	entries, err := s.Store.Load(fi.ID, s.keep(fi))
	if err != nil {
		return "", errors.Wrap(err, "failed to get channel entries")
	}

	if len(entries) == 0 {
		return "", nil
	}

	items := []rssfeed.Item{}
	for _, entry := range entries {

		fileURL := s.RootURL + "/" + path.Base(entry.File)

		var fileSize int
		if fileInfo, fiErr := os.Stat(entry.File); fiErr != nil {
			log.Printf("[WARN] failed to get file size for %s (%s %s): %v",
				entry.File, entry.VideoID, entry.Title, fiErr)
		} else {
			fileSize = int(fileInfo.Size())
		}

		items = append(items, rssfeed.Item{
			Title:       entry.Title,
			Description: entry.Media.Description,
			Link:        entry.Link.Href,
			PubDate:     entry.Published.Format(time.RFC822Z),
			GUID:        entry.ChannelID + "::" + entry.VideoID,
			Author:      entry.Author.Name,
			Enclosure: rssfeed.Enclosure{
				URL:    fileURL,
				Type:   "audio/mpeg",
				Length: fileSize,
			},
			DT: time.Now(),
		})
	}

	rss := rssfeed.Rss2{
		Version:       "2.0",
		ItemList:      items,
		Title:         fi.Name,
		Description:   "generated by feed-master",
		Link:          entries[0].Author.URI,
		PubDate:       items[0].PubDate,
		LastBuildDate: time.Now().Format(time.RFC822Z),
		Language:      fi.Language,
	}

	if fi.Type == ytfeed.FTPlaylist {
		rss.Link = "https://www.youtube.com/playlist?list=" + fi.ID
	}

	b, err := xml.MarshalIndent(&rss, "", "  ")
	if err != nil {
		return "", errors.Wrap(err, "failed to marshal rss")
	}

	return string(b), nil
}

func (s *Service) procChannels(ctx context.Context) error {

	var (
		allEntries   int
		allProcessed int
		allAdded     int
		allRemoved   int
	)

	for _, feedInfo := range s.Feeds {
		entries, err := s.ChannelService.Get(ctx, feedInfo.ID, feedInfo.Type)
		if err != nil {
			log.Printf("[WARN] failed to get channel entries for %s: %s", feedInfo.ID, err)
			continue
		}
		log.Printf("[INFO] got %d entries for %s, limit to %d", len(entries), feedInfo.Name, s.keep(feedInfo))
		changed, processed := false, 0
		for i, entry := range entries {
			allEntries++
			if processed >= s.keep(feedInfo) {
				break
			}

			// check if entry already exists in store
			exists, exErr := s.Store.Exist(entry)
			if err != nil {
				return errors.Wrapf(exErr, "failed to check if entry %s exists", entry.VideoID)
			}
			if exists {
				processed++
				continue
			}

			// check if we already processed this entry.
			// this is needed to avoid infinite get/remove loop when the original feed is updated in place
			found, procTS, procErr := s.Store.CheckProcessed(entry)
			if procErr != nil {
				log.Printf("[WARN] can't get processed status for %s, %+v", entry.VideoID, feedInfo)
			}
			if procErr == nil && found {
				processed++
				log.Printf("[INFO] skipping already processed entry %s at %s, %+v",
					entry.VideoID, procTS.Format(time.RFC3339), feedInfo)
				continue
			}

			log.Printf("[INFO] new entry [%d] %s, %s, %s", i+1, entry.VideoID, entry.Title, feedInfo.Name)
			file, downErr := s.Downloader.Get(ctx, entry.VideoID, s.makeFileName(entry))
			if downErr != nil {
				log.Printf("[WARN] failed to download %s: %s", entry.VideoID, downErr)
				continue
			}
			processed++
			log.Printf("[INFO] downloaded %s (%s) to %s, channel: %+v", entry.VideoID, entry.Title, file, feedInfo)
			entry.File = file
			if !strings.Contains(entry.Title, feedInfo.Name) { // if title doesn't contains channel name add it
				entry.Title = feedInfo.Name + ": " + entry.Title
			}
			ok, saveErr := s.Store.Save(entry)
			if saveErr != nil {
				return errors.Wrapf(saveErr, "failed to save entry %+v", entry)
			}
			if !ok {
				log.Printf("[WARN] attempt to save dup entry %+v", entry)
			}
			changed = true
			if procErr = s.Store.SetProcessed(entry); procErr != nil {
				log.Printf("[WARN] failed to set processed status for %s: %v", entry.VideoID, procErr)
			}
			allAdded++
			log.Printf("[INFO] saved %s (%s) to %s, channel: %+v", entry.VideoID, entry.Title, file, feedInfo)
		}
		allProcessed += processed

		if changed { // save rss feed to fs if there are new entries
			removed, err := s.removeOld(feedInfo)
			if err != nil {
				log.Printf("[WARN] failed to remove old entries for %s: %v", feedInfo.Name, err)
			} else {
				allRemoved += removed
			}
			rss, rssErr := s.RSSFeed(feedInfo)
			if rssErr != nil {
				log.Printf("[WARN] failed to generate rss for %s: %s", feedInfo.Name, rssErr)
			} else {
				if err := s.RSSFileStore.Save(feedInfo.ID, rss); err != nil {
					log.Printf("[WARN] failed to save rss for %s: %s", feedInfo.Name, err)
				}
			}
		}
	}

	log.Printf("[INFO] all channels processed - channels: %d, entries: %d, processed: %d, updated: %d, removed: %d",
		len(s.Feeds), allEntries, allProcessed, allAdded, allRemoved)

	return nil
}

// removeOld deletes old entries from store and corresponding files
func (s *Service) removeOld(fi FeedInfo) (int, error) {
	removed := 0
	keep := s.keep(fi)
	files, err := s.Store.RemoveOld(fi.ID, keep+1)
	if err != nil {
		return 0, errors.Wrapf(err, "failed to remove old meta data for %s", fi.ID)
	}
	for _, f := range files {
		if e := os.Remove(f); e != nil {
			log.Printf("[WARN] failed to remove file %s: %v", f, e)
			continue
		}
		removed++
		log.Printf("[INFO] removed %s for %s (%s)", f, fi.ID, fi.Name)
	}
	return removed, nil
}

func (s *Service) keep(fi FeedInfo) int {
	keep := s.KeepPerChannel
	if fi.Keep > 0 {
		keep = fi.Keep
	}
	return keep
}

func (s *Service) makeFileName(entry ytfeed.Entry) string {
	h := sha1.New()
	if _, err := h.Write([]byte(entry.ChannelID + "::" + entry.VideoID)); err != nil {
		return uuid.New().String()
	}
	return fmt.Sprintf("%x", h.Sum(nil))
}
<commit_msg>enclose stats for yt processing<commit_after>// Package youtube provides loading audio from video files for given youtube channels
package youtube

import (
	"context"
	"crypto/sha1"
	"encoding/xml"
	"fmt"
	"os"
	"path"
	"strings"
	"time"

	log "github.com/go-pkgz/lgr"
	"github.com/google/uuid"
	"github.com/pkg/errors"

	rssfeed "github.com/umputun/feed-master/app/feed"
	ytfeed "github.com/umputun/feed-master/app/youtube/feed"
)

//go:generate moq -out mocks/downloader.go -pkg mocks -skip-ensure -fmt goimports . DownloaderService
//go:generate moq -out mocks/channel.go -pkg mocks -skip-ensure -fmt goimports . ChannelService
//go:generate moq -out mocks/store.go -pkg mocks -skip-ensure -fmt goimports . StoreService

// Service loads audio from youtube channels
type Service struct {
	Feeds          []FeedInfo
	Downloader     DownloaderService
	ChannelService ChannelService
	Store          StoreService
	CheckDuration  time.Duration
	RSSFileStore   RSSFileStore
	KeepPerChannel int
	RootURL        string
}

// FeedInfo contains channel or feed ID, readable name and other per-feed info
type FeedInfo struct {
	Name     string      `yaml:"name"`
	ID       string      `yaml:"id"`
	Type     ytfeed.Type `yaml:"type"`
	Keep     int         `yaml:"keep"`
	Language string      `yaml:"lang"`
}

// DownloaderService is an interface for downloading audio from youtube
type DownloaderService interface {
	Get(ctx context.Context, id string, fname string) (file string, err error)
}

// ChannelService is an interface for getting channel entries, i.e. the list of videos
type ChannelService interface {
	Get(ctx context.Context, chanID string, feedType ytfeed.Type) ([]ytfeed.Entry, error)
}

// StoreService is an interface for storing and loading metadata about downloaded audio
type StoreService interface {
	Save(entry ytfeed.Entry) (bool, error)
	Load(channelID string, max int) ([]ytfeed.Entry, error)
	Exist(entry ytfeed.Entry) (bool, error)
	RemoveOld(channelID string, keep int) ([]string, error)
	SetProcessed(entry ytfeed.Entry) error
	CheckProcessed(entry ytfeed.Entry) (found bool, ts time.Time, err error)
}

// Do is a blocking function that downloads audio from youtube channels and updates metadata
func (s *Service) Do(ctx context.Context) error {
	log.Printf("[INFO] starting youtube service")

	for _, f := range s.Feeds {
		log.Printf("[INFO] youtube feed %+v", f)
	}

	tick := time.NewTicker(s.CheckDuration)
	defer tick.Stop()

	if err := s.procChannels(ctx); err != nil {
		return errors.Wrap(err, "failed to process channels")
	}

	for {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-tick.C:
			if err := s.procChannels(ctx); err != nil {
				return errors.Wrap(err, "failed to process channels")
			}
		}
	}
}

// RSSFeed generates RSS feed for given channel
func (s *Service) RSSFeed(fi FeedInfo) (string, error) {
	entries, err := s.Store.Load(fi.ID, s.keep(fi))
	if err != nil {
		return "", errors.Wrap(err, "failed to get channel entries")
	}

	if len(entries) == 0 {
		return "", nil
	}

	items := []rssfeed.Item{}
	for _, entry := range entries {

		fileURL := s.RootURL + "/" + path.Base(entry.File)

		var fileSize int
		if fileInfo, fiErr := os.Stat(entry.File); fiErr != nil {
			log.Printf("[WARN] failed to get file size for %s (%s %s): %v",
				entry.File, entry.VideoID, entry.Title, fiErr)
		} else {
			fileSize = int(fileInfo.Size())
		}

		items = append(items, rssfeed.Item{
			Title:       entry.Title,
			Description: entry.Media.Description,
			Link:        entry.Link.Href,
			PubDate:     entry.Published.Format(time.RFC822Z),
			GUID:        entry.ChannelID + "::" + entry.VideoID,
			Author:      entry.Author.Name,
			Enclosure: rssfeed.Enclosure{
				URL:    fileURL,
				Type:   "audio/mpeg",
				Length: fileSize,
			},
			DT: time.Now(),
		})
	}

	rss := rssfeed.Rss2{
		Version:       "2.0",
		ItemList:      items,
		Title:         fi.Name,
		Description:   "generated by feed-master",
		Link:          entries[0].Author.URI,
		PubDate:       items[0].PubDate,
		LastBuildDate: time.Now().Format(time.RFC822Z),
		Language:      fi.Language,
	}

	if fi.Type == ytfeed.FTPlaylist {
		rss.Link = "https://www.youtube.com/playlist?list=" + fi.ID
	}

	b, err := xml.MarshalIndent(&rss, "", "  ")
	if err != nil {
		return "", errors.Wrap(err, "failed to marshal rss")
	}

	return string(b), nil
}

func (s *Service) procChannels(ctx context.Context) error {

	var allStats stats

	for _, feedInfo := range s.Feeds {
		entries, err := s.ChannelService.Get(ctx, feedInfo.ID, feedInfo.Type)
		if err != nil {
			log.Printf("[WARN] failed to get channel entries for %s: %s", feedInfo.ID, err)
			continue
		}
		log.Printf("[INFO] got %d entries for %s, limit to %d", len(entries), feedInfo.Name, s.keep(feedInfo))
		changed, processed := false, 0
		for i, entry := range entries {
			allStats.entries++
			if processed >= s.keep(feedInfo) {
				break
			}

			// check if entry already exists in store
			exists, exErr := s.Store.Exist(entry)
			if err != nil {
				return errors.Wrapf(exErr, "failed to check if entry %s exists", entry.VideoID)
			}
			if exists {
				allStats.skipped++
				processed++
				continue
			}

			// check if we already processed this entry.
			// this is needed to avoid infinite get/remove loop when the original feed is updated in place
			found, procTS, procErr := s.Store.CheckProcessed(entry)
			if procErr != nil {
				log.Printf("[WARN] can't get processed status for %s, %+v", entry.VideoID, feedInfo)
			}
			if procErr == nil && found {
				allStats.skipped++
				processed++
				log.Printf("[INFO] skipping already processed entry %s at %s, %+v",
					entry.VideoID, procTS.Format(time.RFC3339), feedInfo)
				continue
			}

			log.Printf("[INFO] new entry [%d] %s, %s, %s", i+1, entry.VideoID, entry.Title, feedInfo.Name)
			file, downErr := s.Downloader.Get(ctx, entry.VideoID, s.makeFileName(entry))
			if downErr != nil {
				allStats.ignored++
				log.Printf("[WARN] failed to download %s: %s", entry.VideoID, downErr)
				continue
			}
			processed++
			log.Printf("[INFO] downloaded %s (%s) to %s, channel: %+v", entry.VideoID, entry.Title, file, feedInfo)
			entry.File = file
			if !strings.Contains(entry.Title, feedInfo.Name) { // if title doesn't contains channel name add it
				entry.Title = feedInfo.Name + ": " + entry.Title
			}
			ok, saveErr := s.Store.Save(entry)
			if saveErr != nil {
				return errors.Wrapf(saveErr, "failed to save entry %+v", entry)
			}
			if !ok {
				log.Printf("[WARN] attempt to save dup entry %+v", entry)
			}
			changed = true
			if procErr = s.Store.SetProcessed(entry); procErr != nil {
				log.Printf("[WARN] failed to set processed status for %s: %v", entry.VideoID, procErr)
			}
			allStats.added++
			log.Printf("[INFO] saved %s (%s) to %s, channel: %+v", entry.VideoID, entry.Title, file, feedInfo)
		}
		allStats.processed += processed

		if changed { // save rss feed to fs if there are new entries
			removed, err := s.removeOld(feedInfo)
			if err != nil {
				log.Printf("[WARN] failed to remove old entries for %s: %v", feedInfo.Name, err)
			} else {
				allStats.removed += removed
			}
			rss, rssErr := s.RSSFeed(feedInfo)
			if rssErr != nil {
				log.Printf("[WARN] failed to generate rss for %s: %s", feedInfo.Name, rssErr)
			} else {
				if err := s.RSSFileStore.Save(feedInfo.ID, rss); err != nil {
					log.Printf("[WARN] failed to save rss for %s: %s", feedInfo.Name, err)
				}
			}
		}
	}

	log.Printf("[INFO] all channels processed - channels: %d, %s", len(s.Feeds), allStats.String())

	return nil
}

// removeOld deletes old entries from store and corresponding files
func (s *Service) removeOld(fi FeedInfo) (int, error) {
	removed := 0
	keep := s.keep(fi)
	files, err := s.Store.RemoveOld(fi.ID, keep+1)
	if err != nil {
		return 0, errors.Wrapf(err, "failed to remove old meta data for %s", fi.ID)
	}
	for _, f := range files {
		if e := os.Remove(f); e != nil {
			log.Printf("[WARN] failed to remove file %s: %v", f, e)
			continue
		}
		removed++
		log.Printf("[INFO] removed %s for %s (%s)", f, fi.ID, fi.Name)
	}
	return removed, nil
}

func (s *Service) keep(fi FeedInfo) int {
	keep := s.KeepPerChannel
	if fi.Keep > 0 {
		keep = fi.Keep
	}
	return keep
}

func (s *Service) makeFileName(entry ytfeed.Entry) string {
	h := sha1.New()
	if _, err := h.Write([]byte(entry.ChannelID + "::" + entry.VideoID)); err != nil {
		return uuid.New().String()
	}
	return fmt.Sprintf("%x", h.Sum(nil))
}

type stats struct {
	entries   int
	processed int
	added     int
	removed   int
	ignored   int
	skipped   int
}

func (st stats) String() string {
	return fmt.Sprintf("entries: %d, processed: %d, updated: %d, removed: %d, ignored: %d, skipped: %d",
		st.entries, st.processed, st.added, st.removed, st.ignored, st.skipped)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/ecc1/cc1101"
)

func main() {
	log.SetFlags(log.Ltime | log.Lmicroseconds | log.LUTC)
	if len(os.Args) != 2 {
		log.Fatalf("Usage: %s frequency\n", os.Args[0])
	}
	frequency := getFrequency(os.Args[1])
	log.Printf("setting frequency to %d\n", frequency)
	r, err := cc1101.Open()
	if err != nil {
		log.Fatal(err)
	}
	err = r.Init(frequency)
	if err != nil {
		log.Fatal(err)
	}
	for packet := range r.Incoming() {
		log.Printf("% X (RSSI = %d)\n", packet.Data, packet.Rssi)
	}
}

func getFrequency(s string) uint32 {
	MHz := 0.0
	n, err := fmt.Sscanf(s, "%f", &MHz)
	if err == nil && n == 1 && 860.0 <= MHz && MHz <= 920.0 {
		return uint32(MHz * 1000000.0)
	}
	Hz := uint32(0)
	n, err = fmt.Sscanf(s, "%d", &Hz)
	if err == nil && n == 1 && 860000000 <= Hz && Hz <= 920000000 {
		return Hz
	}
	if err != nil {
		log.Fatalf("Argument (%s): %v\n", s, err)
	}
	log.Fatalf("Argument (%s) should be the pump frequency in MHz or Hz\n", s)
	panic("unreachable")
}
<commit_msg>Improve frequency parsing in receive test program<commit_after>package main

import (
	"log"
	"os"
	"strconv"

	"github.com/ecc1/cc1101"
)

func main() {
	log.SetFlags(log.Ltime | log.Lmicroseconds | log.LUTC)
	if len(os.Args) != 2 {
		log.Fatalf("Usage: %s frequency\n", os.Args[0])
	}
	frequency := getFrequency(os.Args[1])
	log.Printf("setting frequency to %d\n", frequency)
	r, err := cc1101.Open()
	if err != nil {
		log.Fatal(err)
	}
	err = r.Init(frequency)
	if err != nil {
		log.Fatal(err)
	}
	for packet := range r.Incoming() {
		log.Printf("% X (RSSI = %d)\n", packet.Data, packet.Rssi)
	}
}

func getFrequency(s string) uint32 {
	f, err := strconv.ParseFloat(s, 64)
	if err != nil {
		log.Fatal(err)
	}
	if 860.0 <= f && f <= 920.0 {
		return uint32(f * 1000000.0)
	}
	if 860000000.0 <= f && f <= 920000000.0 {
		return uint32(f)
	}
	log.Fatalf("%s: invalid pump frequency\n", s)
	panic("unreachable")
}
<|endoftext|>


<commit_before>// Split text into words for stemming
package langword

import (
	"unicode"
	"unicode/utf8"
	"code.google.com/p/go.text/unicode/norm"
)

// Scan words in latin alphabets, suitable for bufio.Scanner. Returns lowercase words containing only
// letters that are unicode normalized (NFC).
func ScanLatinWords(data []byte, atEOF bool) (advance int, token []byte, err error) {
        // Skip leading spaces.
        start := 0
        for width := 0; start < len(data); start += width {
                var r rune
                r, width = utf8.DecodeRune(data[start:])
                if unicode.IsLetter(r) {
                        break
                }
        }
        if atEOF && len(data) == 0 {
                return 0, nil, nil
        }
        // Scan until space, marking end of word.
        for width, i := 0, start; i < len(data); i += width {
                var r rune
                r, width = utf8.DecodeRune(data[i:])

			if !unicode.IsLetter(r) {
				return i + width, mutateWord(data[start:i]), nil
			}
       }
        // If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
        if atEOF && len(data) > start {
			return len(data), mutateWord(data[start:]), nil
        }
        // Request more data.
        return 0, nil, nil
}

func mutateWord(bs []byte) []byte {
	return norm.NFC.Bytes(toLowerBytes(bs))
}

func toLowerBytes(src []byte) []byte {
	dst := make([]byte, 0, len(src))
	for width, i, di := 0,0,0; i<len(src); i+= width {
		var r rune
		r, width = utf8.DecodeRune(src[i:])
		r = unicode.ToLower(r)
		dw:=utf8.RuneLen(r)
		dst = append(dst, make([]byte, dw)...)
		utf8.EncodeRune(dst[di:], r)
		di += dw
	}
	return dst
}
<commit_msg>Return empty words for punctuation<commit_after>// Split text into words for stemming
package langword

import (
	"code.google.com/p/go.text/unicode/norm"
	"unicode"
	"unicode/utf8"
)

// Scan words in latin alphabets, suitable for bufio.Scanner. Returns lowercase words containing only
// letters that are unicode normalized (NFC).
func ScanLatinWords(data []byte, atEOF bool) (advance int, token []byte, err error) {
	// Skip leading spaces.
	start := 0
	for width := 0; start < len(data); start += width {
		var r rune
		r, width = utf8.DecodeRune(data[start:])
		if unicode.IsLetter(r) {
			break
		}
		if unicode.IsPunct(r) {
			return start + width, nil, nil
		}
	}
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	// Scan until space, marking end of word.
	for width, i := 0, start; i < len(data); i += width {
		var r rune
		r, width = utf8.DecodeRune(data[i:])

		if !unicode.IsLetter(r) {
			return i + width, mutateWord(data[start:i]), nil
		}
	}
	// If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
	if atEOF && len(data) > start {
		return len(data), mutateWord(data[start:]), nil
	}
	// Request more data.
	return 0, nil, nil
}

func mutateWord(bs []byte) []byte {
	return norm.NFC.Bytes(toLowerBytes(bs))
}

func toLowerBytes(src []byte) []byte {
	dst := make([]byte, 0, len(src))
	for width, i, di := 0, 0, 0; i < len(src); i += width {
		var r rune
		r, width = utf8.DecodeRune(src[i:])
		r = unicode.ToLower(r)
		dw := utf8.RuneLen(r)
		dst = append(dst, make([]byte, dw)...)
		utf8.EncodeRune(dst[di:], r)
		di += dw
	}
	return dst
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"

	"github.com/blevesearch/bleve"
)

// Indexer is an online asynchronous indexer.
type Indexer struct {
	store    *Store
	index    bleve.Index
	queue    *IndexQueue
	geocoder *Geocoder
	reset    chan bool
	work     chan bool
	stop     chan chan bool
}

// NewIndexer creates a new Indexer assuming it is the soler writer for
// supplied store and index.
func NewIndexer(store *Store, index bleve.Index, geocoder *Geocoder,
	queue *IndexQueue) *Indexer {

	idx := &Indexer{
		store:    store,
		index:    index,
		geocoder: geocoder,
		queue:    queue,
		reset:    make(chan bool, 1),
		work:     make(chan bool, 1),
		stop:     make(chan chan bool),
	}
	go idx.dispatch()
	return idx
}

// Close signals and waits for the indexing goroutine to terminate.
func (idx *Indexer) Close() {
	done := make(chan bool)
	idx.stop <- done
	<-done
}

// Sync makes the indexer to compare the index and store again and synchronize
// them if necessary. The synchronization is performed asynchronously.
func (idx *Indexer) Sync() {
	select {
	case idx.reset <- true:
	default:
	}
}

func (idx *Indexer) dispatch() {
	for {
		select {
		case <-idx.reset:
			log.Printf("collecting index updates")
			err := idx.resetQueue()
			if err != nil {
				log.Printf("error: indexer reset failed: %s", err)
				continue
			}
			log.Printf("collection done")
			idx.signalWork()
		case <-idx.work:
			log.Printf("indexing documents, %d updates remaining", idx.queue.Size())
			err := idx.indexSome()
			if err != nil {
				log.Println("error: indexation failed: %s", err)
			}
			log.Printf("indexation done")
		case done := <-idx.stop:
			close(done)
			return
		}
	}
}

func listIndexIds(index bleve.Index) ([]string, error) {
	idx, _, err := index.Advanced()
	if err != nil {
		return nil, err
	}
	reader, err := idx.Reader()
	if err != nil {
		return nil, err
	}
	defer reader.Close()
	idReader, err := reader.DocIDReader("", "")
	if err != nil {
		return nil, err
	}
	defer idReader.Close()
	ids := []string{}
	for {
		id, err := idReader.Next()
		if err != nil {
			return nil, err
		}
		if id == "" {
			break
		}
		ids = append(ids, id)
	}
	return ids, nil
}

func (idx *Indexer) resetQueue() error {
	ops := []Queued{}

	// For now we can live with loading both set of ids and diffing them
	stored, err := idx.store.List()
	if err != nil {
		return err
	}
	indexed, err := listIndexIds(idx.index)
	if err != nil {
		return err
	}
	added, removed := diffIds(stored, indexed)

	for _, id := range added {
		ops = append(ops, Queued{Id: id, Op: AddOp})
	}
	for _, id := range removed {
		ops = append(ops, Queued{Id: id, Op: RemoveOp})
	}
	log.Printf("queuing %d additions, %d removals", len(added), len(removed))

	// Update queue
	err = idx.queue.DeleteMany(idx.queue.Size())
	if err != nil {
		return err
	}
	return idx.queue.QueueMany(ops)
}

func (idx *Indexer) signalWork() {
	select {
	case idx.work <- true:
	default:
	}
}

func (idx *Indexer) indexSome() error {
	count := 10
	queued, err := idx.queue.FetchMany(count)
	if err != nil {
		return err
	}
	if len(queued) >= count {
		idx.signalWork()
	}
	for _, q := range queued {
		if q.Op == AddOp {
			log.Printf("indexing %s", q.Id)
			offer, err := getStoreOffer(idx.store, q.Id)
			if err != nil {
				return err
			}
			if offer != nil {
				_, _, err = geocodeOffer(idx.geocoder, offer, true)
				if err != nil {
					return err
				}
				err = idx.index.Index(offer.Id, offer)
				if err != nil {
					return err
				}
			}
		} else if q.Op == RemoveOp {
			log.Printf("deleting %s", q.Id)
			err = idx.index.Delete(q.Id)
			if err != nil {
				return err
			}
		} else {
			return fmt.Errorf("unknown operation: %v", q.Op)
		}
		err = idx.queue.DeleteMany(1)
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>webindexer: remove entries before adding new ones<commit_after>package main

import (
	"fmt"
	"log"

	"github.com/blevesearch/bleve"
)

// Indexer is an online asynchronous indexer.
type Indexer struct {
	store    *Store
	index    bleve.Index
	queue    *IndexQueue
	geocoder *Geocoder
	reset    chan bool
	work     chan bool
	stop     chan chan bool
}

// NewIndexer creates a new Indexer assuming it is the soler writer for
// supplied store and index.
func NewIndexer(store *Store, index bleve.Index, geocoder *Geocoder,
	queue *IndexQueue) *Indexer {

	idx := &Indexer{
		store:    store,
		index:    index,
		geocoder: geocoder,
		queue:    queue,
		reset:    make(chan bool, 1),
		work:     make(chan bool, 1),
		stop:     make(chan chan bool),
	}
	go idx.dispatch()
	return idx
}

// Close signals and waits for the indexing goroutine to terminate.
func (idx *Indexer) Close() {
	done := make(chan bool)
	idx.stop <- done
	<-done
}

// Sync makes the indexer to compare the index and store again and synchronize
// them if necessary. The synchronization is performed asynchronously.
func (idx *Indexer) Sync() {
	select {
	case idx.reset <- true:
	default:
	}
}

func (idx *Indexer) dispatch() {
	for {
		select {
		case <-idx.reset:
			log.Printf("collecting index updates")
			err := idx.resetQueue()
			if err != nil {
				log.Printf("error: indexer reset failed: %s", err)
				continue
			}
			log.Printf("collection done")
			idx.signalWork()
		case <-idx.work:
			log.Printf("indexing documents, %d updates remaining", idx.queue.Size())
			err := idx.indexSome()
			if err != nil {
				log.Println("error: indexation failed: %s", err)
			}
			log.Printf("indexation done")
		case done := <-idx.stop:
			close(done)
			return
		}
	}
}

func listIndexIds(index bleve.Index) ([]string, error) {
	idx, _, err := index.Advanced()
	if err != nil {
		return nil, err
	}
	reader, err := idx.Reader()
	if err != nil {
		return nil, err
	}
	defer reader.Close()
	idReader, err := reader.DocIDReader("", "")
	if err != nil {
		return nil, err
	}
	defer idReader.Close()
	ids := []string{}
	for {
		id, err := idReader.Next()
		if err != nil {
			return nil, err
		}
		if id == "" {
			break
		}
		ids = append(ids, id)
	}
	return ids, nil
}

func (idx *Indexer) resetQueue() error {
	ops := []Queued{}

	// For now we can live with loading both set of ids and diffing them
	stored, err := idx.store.List()
	if err != nil {
		return err
	}
	indexed, err := listIndexIds(idx.index)
	if err != nil {
		return err
	}
	added, removed := diffIds(stored, indexed)

	for _, id := range removed {
		ops = append(ops, Queued{Id: id, Op: RemoveOp})
	}
	for _, id := range added {
		ops = append(ops, Queued{Id: id, Op: AddOp})
	}
	log.Printf("queuing %d additions, %d removals", len(added), len(removed))

	// Update queue
	err = idx.queue.DeleteMany(idx.queue.Size())
	if err != nil {
		return err
	}
	return idx.queue.QueueMany(ops)
}

func (idx *Indexer) signalWork() {
	select {
	case idx.work <- true:
	default:
	}
}

func (idx *Indexer) indexSome() error {
	count := 10
	queued, err := idx.queue.FetchMany(count)
	if err != nil {
		return err
	}
	if len(queued) >= count {
		idx.signalWork()
	}
	for _, q := range queued {
		if q.Op == AddOp {
			log.Printf("indexing %s", q.Id)
			offer, err := getStoreOffer(idx.store, q.Id)
			if err != nil {
				return err
			}
			if offer != nil {
				_, _, err = geocodeOffer(idx.geocoder, offer, true)
				if err != nil {
					return err
				}
				err = idx.index.Index(offer.Id, offer)
				if err != nil {
					return err
				}
			}
		} else if q.Op == RemoveOp {
			log.Printf("deleting %s", q.Id)
			err = idx.index.Delete(q.Id)
			if err != nil {
				return err
			}
		} else {
			return fmt.Errorf("unknown operation: %v", q.Op)
		}
		err = idx.queue.DeleteMany(1)
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package pusher

type binders struct {
}

<commit_msg>Remove unused file.<commit_after><|endoftext|>