<commit_before>package eval

// Builtin functions.

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"os/user"
	"strconv"
)

var builtinFns []*builtinFn
var BuiltinFnNames []string

func init() {
	// Needed to work around init loop.
	builtinFns = []*builtinFn{
		&builtinFn{":", nop},
		&builtinFn{"true", nop},

		&builtinFn{"print", print},
		&builtinFn{"println", println},

		&builtinFn{"into-lines", intoLines},
		&builtinFn{"from-lines", fromLines},

		&builtinFn{"rat", ratFn},

		&builtinFn{"put", put},
		&builtinFn{"unpack", unpack},

		&builtinFn{"from-json", fromJSON},

		&builtinFn{"typeof", typeof},

		&builtinFn{"failure", failure},
		&builtinFn{"return", returnFn},
		&builtinFn{"break", breakFn},
		&builtinFn{"continue", continueFn},

		&builtinFn{"each", each},

		&builtinFn{"cd", cd},
		&builtinFn{"visited-dirs", visistedDirs},
		&builtinFn{"jump-dir", jumpDir},

		&builtinFn{"source", source},

		&builtinFn{"+", plus},
		&builtinFn{"-", minus},
		&builtinFn{"*", times},
		&builtinFn{"/", divide},

		&builtinFn{"=", eq},
	}
	for _, b := range builtinFns {
		BuiltinFnNames = append(BuiltinFnNames, b.Name)
	}
}

var (
	argsError  = newFailure("args error")
	inputError = newFailure("input error")
)

func nop(ec *evalCtx, args []Value) exitus {
	return ok
}

func put(ec *evalCtx, args []Value) exitus {
	out := ec.ports[1].ch
	for _, a := range args {
		out <- a
	}
	return ok
}

func typeof(ec *evalCtx, args []Value) exitus {
	out := ec.ports[1].ch
	for _, a := range args {
		out <- str(a.Type().String())
	}
	return ok
}

func failure(ec *evalCtx, args []Value) exitus {
	if len(args) != 1 {
		return argsError
	}
	out := ec.ports[1].ch
	out <- newFailure(toString(args[0]))
	return ok
}

func returnFn(ec *evalCtx, args []Value) exitus {
	return newFlowExitus(Return)
}

func breakFn(ec *evalCtx, args []Value) exitus {
	return newFlowExitus(Break)
}

func continueFn(ec *evalCtx, args []Value) exitus {
	return newFlowExitus(Continue)
}

func print(ec *evalCtx, args []Value) exitus {
	out := ec.ports[1].f
	for _, a := range args {
		fmt.Fprint(out, toString(a))
	}
	return ok
}

func println(ec *evalCtx, args []Value) exitus {
	args = append(args, str("\n"))
	return print(ec, args)
}

func intoLines(ec *evalCtx, args []Value) exitus {
	if len(args) > 0 {
		return argsError
	}
	in := ec.ports[0].ch
	out := ec.ports[1].f

	for v := range in {
		fmt.Fprintln(out, toString(v))
	}
	return ok
}

func fromLines(ec *evalCtx, args []Value) exitus {
	if len(args) > 0 {
		return argsError
	}
	in := ec.ports[0].f
	out := ec.ports[1].ch

	bufferedIn := bufio.NewReader(in)
	for {
		line, err := bufferedIn.ReadString('\n')
		if err == io.EOF {
			return ok
		} else if err != nil {
			return newFailure(err.Error())
		}
		out <- str(line[:len(line)-1])
	}
}

func ratFn(ec *evalCtx, args []Value) exitus {
	if len(args) != 1 {
		return argsError
	}
	out := ec.ports[1].ch
	r, err := toRat(args[0])
	if err != nil {
		return newFailure(err.Error())
	}
	out <- r
	return ok
}

// unpack takes any number of tables and output their list elements.
func unpack(ec *evalCtx, args []Value) exitus {
	if len(args) != 0 {
		return argsError
	}
	in := ec.ports[0].ch
	out := ec.ports[1].ch

	for v := range in {
		if t, ok := v.(*table); !ok {
			return inputError
		} else {
			for _, e := range t.List {
				out <- e
			}
		}
	}

	return ok
}

// fromJSON parses a stream of JSON data into Value's.
func fromJSON(ec *evalCtx, args []Value) exitus {
	if len(args) > 0 {
		return argsError
	}
	in := ec.ports[0].f
	out := ec.ports[1].ch

	dec := json.NewDecoder(in)
	var v interface{}
	for {
		err := dec.Decode(&v)
		if err != nil {
			if err == io.EOF {
				return ok
			}
			return newFailure(err.Error())
		}
		out <- fromJSONInterface(v)
	}
}

// each takes a single closure and applies it to all input values.
func each(ec *evalCtx, args []Value) exitus {
	if len(args) != 1 {
		return argsError
	}
	if f, ok := args[0].(*closure); !ok {
		return argsError
	} else {
		in := ec.ports[0].ch
	in:
		for v := range in {
			su := f.Exec(ec.copy("closure of each"), []Value{v})
			// F.Exec will put exactly one stateUpdate on the channel
			e := (<-su).Exitus
			switch e.Sort {
			case Ok, Continue:
				// nop
			case Break:
				break in
			default:
				return e
			}
		}
	}
	return ok
}

func cd(ec *evalCtx, args []Value) exitus {
	var dir string
	if len(args) == 0 {
		user, err := user.Current()
		if err == nil {
			dir = user.HomeDir
		}
	} else if len(args) == 1 {
		dir = toString(args[0])
	} else {
		return argsError
	}
	err := os.Chdir(dir)
	if err != nil {
		return newFailure(err.Error())
	}
	if ec.store != nil {
		pwd, err := os.Getwd()
		// BUG(xiaq): Possible error of os.Getwd after cd-ing is ignored.
		if err == nil {
			ec.store.AddDir(pwd)
		}
	}
	return ok
}

var storeNotConnected = newFailure("store not connected")

func visistedDirs(ec *evalCtx, args []Value) exitus {
	if ec.store == nil {
		return storeNotConnected
	}
	dirs, err := ec.store.ListDirs()
	if err != nil {
		return newFailure("store error: " + err.Error())
	}
	out := ec.ports[1].ch
	for _, dir := range dirs {
		table := newTable()
		table.Dict["path"] = str(dir.Path)
		table.Dict["score"] = str(fmt.Sprint(dir.Score))
		out <- table
	}
	return ok
}

var noMatchingDir = newFailure("no matching directory")

func jumpDir(ec *evalCtx, args []Value) exitus {
	if len(args) != 1 {
		return argsError
	}
	if ec.store == nil {
		return storeNotConnected
	}
	dirs, err := ec.store.FindDirs(toString(args[0]))
	if err != nil {
		return newFailure("store error: " + err.Error())
	}
	if len(dirs) == 0 {
		return noMatchingDir
	}
	dir := dirs[0].Path
	err = os.Chdir(dir)
	// TODO(xiaq): Remove directories that no longer exist
	if err != nil {
		return newFailure(err.Error())
	}
	ec.store.AddDir(dir)
	return ok
}

func source(ec *evalCtx, args []Value) exitus {
	if len(args) != 1 {
		return argsError
	}
	if fname, ok := args[0].(str); !ok {
		return argsError
	} else {
		ec.Source(string(fname))
	}
	return ok
}

func toFloats(args []Value) (nums []float64, err error) {
	for _, a := range args {
		a, ok := a.(str)
		if !ok {
			return nil, fmt.Errorf("must be string")
		}
		f, err := strconv.ParseFloat(string(a), 64)
		if err != nil {
			return nil, err
		}
		nums = append(nums, f)
	}
	return
}

func plus(ec *evalCtx, args []Value) exitus {
	out := ec.ports[1].ch
	nums, err := toFloats(args)
	if err != nil {
		return newFailure(err.Error())
	}
	sum := 0.0
	for _, f := range nums {
		sum += f
	}
	out <- str(fmt.Sprintf("%g", sum))
	return ok
}

func minus(ec *evalCtx, args []Value) exitus {
	out := ec.ports[1].ch
	if len(args) == 0 {
		return argsError
	}
	nums, err := toFloats(args)
	if err != nil {
		return newFailure(err.Error())
	}
	sum := nums[0]
	for _, f := range nums[1:] {
		sum -= f
	}
	out <- str(fmt.Sprintf("%g", sum))
	return ok
}

func times(ec *evalCtx, args []Value) exitus {
	out := ec.ports[1].ch
	nums, err := toFloats(args)
	if err != nil {
		return newFailure(err.Error())
	}
	prod := 1.0
	for _, f := range nums {
		prod *= f
	}
	out <- str(fmt.Sprintf("%g", prod))
	return ok
}

func divide(ec *evalCtx, args []Value) exitus {
	out := ec.ports[1].ch
	if len(args) == 0 {
		return argsError
	}
	nums, err := toFloats(args)
	if err != nil {
		return newFailure(err.Error())
	}
	prod := nums[0]
	for _, f := range nums[1:] {
		prod /= f
	}
	out <- str(fmt.Sprintf("%g", prod))
	return ok
}

func eq(ec *evalCtx, args []Value) exitus {
	out := ec.ports[1].ch
	if len(args) == 0 {
		return argsError
	}
	for i := 0; i+1 < len(args); i++ {
		if !valueEq(args[i], args[i+1]) {
			out <- boolean(false)
			return ok
		}
	}
	out <- boolean(true)
	return ok
}
<commit_msg>eval: remove BuiltinFnNames<commit_after>package eval

// Builtin functions.

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"os/user"
	"strconv"
)

var builtinFns []*builtinFn

func init() {
	// Needed to work around init loop.
	builtinFns = []*builtinFn{
		&builtinFn{":", nop},
		&builtinFn{"true", nop},

		&builtinFn{"print", print},
		&builtinFn{"println", println},

		&builtinFn{"into-lines", intoLines},
		&builtinFn{"from-lines", fromLines},

		&builtinFn{"rat", ratFn},

		&builtinFn{"put", put},
		&builtinFn{"unpack", unpack},

		&builtinFn{"from-json", fromJSON},

		&builtinFn{"typeof", typeof},

		&builtinFn{"failure", failure},
		&builtinFn{"return", returnFn},
		&builtinFn{"break", breakFn},
		&builtinFn{"continue", continueFn},

		&builtinFn{"each", each},

		&builtinFn{"cd", cd},
		&builtinFn{"visited-dirs", visistedDirs},
		&builtinFn{"jump-dir", jumpDir},

		&builtinFn{"source", source},

		&builtinFn{"+", plus},
		&builtinFn{"-", minus},
		&builtinFn{"*", times},
		&builtinFn{"/", divide},

		&builtinFn{"=", eq},
	}
}

var (
	argsError  = newFailure("args error")
	inputError = newFailure("input error")
)

func nop(ec *evalCtx, args []Value) exitus {
	return ok
}

func put(ec *evalCtx, args []Value) exitus {
	out := ec.ports[1].ch
	for _, a := range args {
		out <- a
	}
	return ok
}

func typeof(ec *evalCtx, args []Value) exitus {
	out := ec.ports[1].ch
	for _, a := range args {
		out <- str(a.Type().String())
	}
	return ok
}

func failure(ec *evalCtx, args []Value) exitus {
	if len(args) != 1 {
		return argsError
	}
	out := ec.ports[1].ch
	out <- newFailure(toString(args[0]))
	return ok
}

func returnFn(ec *evalCtx, args []Value) exitus {
	return newFlowExitus(Return)
}

func breakFn(ec *evalCtx, args []Value) exitus {
	return newFlowExitus(Break)
}

func continueFn(ec *evalCtx, args []Value) exitus {
	return newFlowExitus(Continue)
}

func print(ec *evalCtx, args []Value) exitus {
	out := ec.ports[1].f
	for _, a := range args {
		fmt.Fprint(out, toString(a))
	}
	return ok
}

func println(ec *evalCtx, args []Value) exitus {
	args = append(args, str("\n"))
	return print(ec, args)
}

func intoLines(ec *evalCtx, args []Value) exitus {
	if len(args) > 0 {
		return argsError
	}
	in := ec.ports[0].ch
	out := ec.ports[1].f

	for v := range in {
		fmt.Fprintln(out, toString(v))
	}
	return ok
}

func fromLines(ec *evalCtx, args []Value) exitus {
	if len(args) > 0 {
		return argsError
	}
	in := ec.ports[0].f
	out := ec.ports[1].ch

	bufferedIn := bufio.NewReader(in)
	for {
		line, err := bufferedIn.ReadString('\n')
		if err == io.EOF {
			return ok
		} else if err != nil {
			return newFailure(err.Error())
		}
		out <- str(line[:len(line)-1])
	}
}

func ratFn(ec *evalCtx, args []Value) exitus {
	if len(args) != 1 {
		return argsError
	}
	out := ec.ports[1].ch
	r, err := toRat(args[0])
	if err != nil {
		return newFailure(err.Error())
	}
	out <- r
	return ok
}

// unpack takes any number of tables and output their list elements.
func unpack(ec *evalCtx, args []Value) exitus {
	if len(args) != 0 {
		return argsError
	}
	in := ec.ports[0].ch
	out := ec.ports[1].ch

	for v := range in {
		if t, ok := v.(*table); !ok {
			return inputError
		} else {
			for _, e := range t.List {
				out <- e
			}
		}
	}

	return ok
}

// fromJSON parses a stream of JSON data into Value's.
func fromJSON(ec *evalCtx, args []Value) exitus {
	if len(args) > 0 {
		return argsError
	}
	in := ec.ports[0].f
	out := ec.ports[1].ch

	dec := json.NewDecoder(in)
	var v interface{}
	for {
		err := dec.Decode(&v)
		if err != nil {
			if err == io.EOF {
				return ok
			}
			return newFailure(err.Error())
		}
		out <- fromJSONInterface(v)
	}
}

// each takes a single closure and applies it to all input values.
func each(ec *evalCtx, args []Value) exitus {
	if len(args) != 1 {
		return argsError
	}
	if f, ok := args[0].(*closure); !ok {
		return argsError
	} else {
		in := ec.ports[0].ch
	in:
		for v := range in {
			su := f.Exec(ec.copy("closure of each"), []Value{v})
			// F.Exec will put exactly one stateUpdate on the channel
			e := (<-su).Exitus
			switch e.Sort {
			case Ok, Continue:
				// nop
			case Break:
				break in
			default:
				return e
			}
		}
	}
	return ok
}

func cd(ec *evalCtx, args []Value) exitus {
	var dir string
	if len(args) == 0 {
		user, err := user.Current()
		if err == nil {
			dir = user.HomeDir
		}
	} else if len(args) == 1 {
		dir = toString(args[0])
	} else {
		return argsError
	}
	err := os.Chdir(dir)
	if err != nil {
		return newFailure(err.Error())
	}
	if ec.store != nil {
		pwd, err := os.Getwd()
		// BUG(xiaq): Possible error of os.Getwd after cd-ing is ignored.
		if err == nil {
			ec.store.AddDir(pwd)
		}
	}
	return ok
}

var storeNotConnected = newFailure("store not connected")

func visistedDirs(ec *evalCtx, args []Value) exitus {
	if ec.store == nil {
		return storeNotConnected
	}
	dirs, err := ec.store.ListDirs()
	if err != nil {
		return newFailure("store error: " + err.Error())
	}
	out := ec.ports[1].ch
	for _, dir := range dirs {
		table := newTable()
		table.Dict["path"] = str(dir.Path)
		table.Dict["score"] = str(fmt.Sprint(dir.Score))
		out <- table
	}
	return ok
}

var noMatchingDir = newFailure("no matching directory")

func jumpDir(ec *evalCtx, args []Value) exitus {
	if len(args) != 1 {
		return argsError
	}
	if ec.store == nil {
		return storeNotConnected
	}
	dirs, err := ec.store.FindDirs(toString(args[0]))
	if err != nil {
		return newFailure("store error: " + err.Error())
	}
	if len(dirs) == 0 {
		return noMatchingDir
	}
	dir := dirs[0].Path
	err = os.Chdir(dir)
	// TODO(xiaq): Remove directories that no longer exist
	if err != nil {
		return newFailure(err.Error())
	}
	ec.store.AddDir(dir)
	return ok
}

func source(ec *evalCtx, args []Value) exitus {
	if len(args) != 1 {
		return argsError
	}
	if fname, ok := args[0].(str); !ok {
		return argsError
	} else {
		ec.Source(string(fname))
	}
	return ok
}

func toFloats(args []Value) (nums []float64, err error) {
	for _, a := range args {
		a, ok := a.(str)
		if !ok {
			return nil, fmt.Errorf("must be string")
		}
		f, err := strconv.ParseFloat(string(a), 64)
		if err != nil {
			return nil, err
		}
		nums = append(nums, f)
	}
	return
}

func plus(ec *evalCtx, args []Value) exitus {
	out := ec.ports[1].ch
	nums, err := toFloats(args)
	if err != nil {
		return newFailure(err.Error())
	}
	sum := 0.0
	for _, f := range nums {
		sum += f
	}
	out <- str(fmt.Sprintf("%g", sum))
	return ok
}

func minus(ec *evalCtx, args []Value) exitus {
	out := ec.ports[1].ch
	if len(args) == 0 {
		return argsError
	}
	nums, err := toFloats(args)
	if err != nil {
		return newFailure(err.Error())
	}
	sum := nums[0]
	for _, f := range nums[1:] {
		sum -= f
	}
	out <- str(fmt.Sprintf("%g", sum))
	return ok
}

func times(ec *evalCtx, args []Value) exitus {
	out := ec.ports[1].ch
	nums, err := toFloats(args)
	if err != nil {
		return newFailure(err.Error())
	}
	prod := 1.0
	for _, f := range nums {
		prod *= f
	}
	out <- str(fmt.Sprintf("%g", prod))
	return ok
}

func divide(ec *evalCtx, args []Value) exitus {
	out := ec.ports[1].ch
	if len(args) == 0 {
		return argsError
	}
	nums, err := toFloats(args)
	if err != nil {
		return newFailure(err.Error())
	}
	prod := nums[0]
	for _, f := range nums[1:] {
		prod /= f
	}
	out <- str(fmt.Sprintf("%g", prod))
	return ok
}

func eq(ec *evalCtx, args []Value) exitus {
	out := ec.ports[1].ch
	if len(args) == 0 {
		return argsError
	}
	for i := 0; i+1 < len(args); i++ {
		if !valueEq(args[i], args[i+1]) {
			out <- boolean(false)
			return ok
		}
	}
	out <- boolean(true)
	return ok
}
<|endoftext|>


<commit_before>package konnectors

import (
	"archive/tar"
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"runtime"
	"time"

	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/realtime"
	"github.com/cozy/cozy-stack/pkg/stack"
	"github.com/cozy/cozy-stack/pkg/workers/mails"
	"github.com/sirupsen/logrus"

	"github.com/spf13/afero"
)

func init() {
	jobs.AddWorker("konnector", &jobs.WorkerConfig{
		Concurrency:  runtime.NumCPU() * 2,
		MaxExecCount: 2,
		MaxExecTime:  200 * time.Second,
		Timeout:      200 * time.Second,
		WorkerFunc:   Worker,
		WorkerCommit: commit,
	})
}

// Options contains the options to execute a konnector.
type Options struct {
	Konnector    string `json:"konnector"`
	Account      string `json:"account"`
	FolderToSave string `json:"folder_to_save"`
}

// result stores the result of a konnector execution.
type result struct {
	DocID       string    `json:"_id,omitempty"`
	DocRev      string    `json:"_rev,omitempty"`
	CreatedAt   time.Time `json:"last_execution"`
	LastSuccess time.Time `json:"last_success"`
	Account     string    `json:"account"`
	State       string    `json:"state"`
	Error       string    `json:"error"`
}

func (r *result) ID() string         { return r.DocID }
func (r *result) Rev() string        { return r.DocRev }
func (r *result) DocType() string    { return consts.KonnectorResults }
func (r *result) Clone() couchdb.Doc { c := *r; return &c }
func (r *result) SetID(id string)    { r.DocID = id }
func (r *result) SetRev(rev string)  { r.DocRev = rev }

const konnectorMsgTypeError string = "error"

// const konnectorMsgTypeDebug string = "debug"
// const konnectorMsgTypeWarning string = "warning"
// const konnectorMsgTypeProgress string = "progress"

type konnectorMsg struct {
	Type    string `json:"type"`
	Message string `json:"message"`
}

// Worker is the worker that runs a konnector by executing an external process.
func Worker(ctx context.Context, m *jobs.Message) error {
	opts := &Options{}
	if err := m.Unmarshal(&opts); err != nil {
		return err
	}

	slug := opts.Konnector
	fields := struct {
		Account      string `json:"account"`
		FolderToSave string `json:"folder_to_save"`
	}{
		Account:      opts.Account,
		FolderToSave: opts.FolderToSave,
	}
	domain := ctx.Value(jobs.ContextDomainKey).(string)
	worker := ctx.Value(jobs.ContextWorkerKey).(string)
	jobID := fmt.Sprintf("%s/%s/%s", worker, slug, domain)

	inst, err := instance.Get(domain)
	if err != nil {
		return err
	}

	man, err := apps.GetKonnectorBySlug(inst, slug)
	if err != nil {
		return err
	}
	if man.State() != apps.Ready {
		return errors.New("Konnector is not ready")
	}

	token := inst.BuildKonnectorToken(man)

	osFS := afero.NewOsFs()
	workDir, err := afero.TempDir(osFS, "", "konnector-"+slug)
	if err != nil {
		return err
	}
	defer osFS.RemoveAll(workDir)
	workFS := afero.NewBasePathFs(osFS, workDir)

	fileServer := inst.KonnectorsFileServer()
	tarFile, err := fileServer.Open(slug, man.Version(), apps.KonnectorArchiveName)
	if err != nil {
		return err
	}

	tr := tar.NewReader(tarFile)
	for {
		var hdr *tar.Header
		hdr, err = tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		dirname := path.Dir(hdr.Name)
		if dirname != "." {
			if err = workFS.MkdirAll(dirname, 0755); err != nil {
				return nil
			}
		}
		var f afero.File
		f, err = workFS.OpenFile(hdr.Name, os.O_CREATE|os.O_WRONLY, 0644)
		if err != nil {
			return err
		}
		_, err = io.Copy(f, tr)
		errc := f.Close()
		if err != nil {
			return err
		}
		if errc != nil {
			return errc
		}
	}

	fieldsJSON, err := json.Marshal(fields)
	if err != nil {
		return err
	}

	konnCmd := config.GetConfig().Konnectors.Cmd
	cmd := exec.CommandContext(ctx, konnCmd, workDir) // #nosec
	cmd.Env = []string{
		"COZY_URL=" + inst.PageURL("/", nil),
		"COZY_CREDENTIALS=" + token,
		"COZY_FIELDS=" + string(fieldsJSON),
		"COZY_TYPE=" + man.Type,
		"COZY_JOB_ID=" + jobID,
	}

	cmdErr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}
	cmdOut, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	scanErr := bufio.NewScanner(cmdErr)
	scanOut := bufio.NewScanner(cmdOut)
	scanOut.Buffer(nil, 256*1024)

	var msgChan = make(chan konnectorMsg)
	var messages []konnectorMsg

	log := logger.WithDomain(domain)

	go doScanOut(jobID, scanOut, domain, msgChan, log)
	go doScanErr(jobID, scanErr, log)
	go func() {
		hub := realtime.GetHub()
		for msg := range msgChan {
			messages = append(messages, msg)
			hub.Publish(&realtime.Event{
				Verb: realtime.EventCreate,
				Doc: couchdb.JSONDoc{Type: consts.JobEvents, M: map[string]interface{}{
					"type":    msg.Type,
					"message": msg.Message,
				}},
				Domain: domain,
			})
		}
	}()

	if err = cmd.Start(); err != nil {
		return wrapErr(ctx, err)
	}

	err = cmd.Wait()
	if err != nil {
		err = wrapErr(ctx, err)
	}

	close(msgChan)
	for _, msg := range messages {
		if msg.Type == konnectorMsgTypeError {
			// konnector err is more explicit
			return errors.New(msg.Message)
		}
	}

	return err
}

func doScanOut(jobID string, scanner *bufio.Scanner, domain string,
	msgs chan konnectorMsg, log *logrus.Entry) {
	for scanner.Scan() {
		linebb := scanner.Bytes()
		from := bytes.IndexByte(linebb, '{')
		to := bytes.LastIndexByte(linebb, '}')
		var msg konnectorMsg
		log.Infof("[konnector] %s: Stdout: %s", jobID, string(linebb))
		if from > -1 && from < to && to > -1 {
			err := json.Unmarshal(linebb[from:to+1], &msg)
			if err == nil {
				msgs <- msg
				continue
			}
		}
		log.Warnf("[konnector] %s: Could not parse as JSON", jobID)
		log.Debugf("[konnector] %s: %s", jobID, string(linebb))
	}
	if err := scanner.Err(); err != nil {
		log.Errorf("[konnector] %s: Error while reading stdout: %s", jobID, err)
	}
}

func doScanErr(jobID string, scanner *bufio.Scanner, log *logrus.Entry) {
	for scanner.Scan() {
		log.Errorf("[konnector] %s: Stderr: %s", jobID, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		log.Errorf("[konnector] %s: Error while reading stderr: %s", jobID, err)
	}
}

func commit(ctx context.Context, m *jobs.Message, errjob error) error {
	opts := &Options{}
	if err := m.Unmarshal(&opts); err != nil {
		return err
	}

	slug := opts.Konnector
	domain := ctx.Value(jobs.ContextDomainKey).(string)

	log := logger.WithDomain(domain)

	inst, err := instance.Get(domain)
	if err != nil {
		return err
	}

	lastResult := &result{}
	err = couchdb.GetDoc(inst, consts.KonnectorResults, slug, lastResult)
	if err != nil {
		if !couchdb.IsNotFoundError(err) {
			return err
		}
		lastResult = nil
	}

	var state, errstr string
	var lastSuccess time.Time
	if errjob != nil {
		if lastResult != nil {
			lastSuccess = lastResult.LastSuccess
		}
		errstr = errjob.Error()
		state = jobs.Errored
	} else {
		lastSuccess = time.Now()
		state = jobs.Done
	}
	result := &result{
		DocID:       slug,
		Account:     opts.Account,
		CreatedAt:   time.Now(),
		LastSuccess: lastSuccess,
		State:       state,
		Error:       errstr,
	}
	if lastResult == nil {
		err = couchdb.CreateNamedDocWithDB(inst, result)
	} else {
		result.SetRev(lastResult.Rev())
		err = couchdb.UpdateDoc(inst, result)
	}
	if err != nil {
		return err
	}

	// if it is the first try we do not take into account an error, we bail.
	if lastResult == nil {
		return nil
	}
	// if the job has not errored, or the last one was already errored, we bail.
	if state != jobs.Errored || lastResult.State == jobs.Errored {
		return nil
	}

	konnectorURL := inst.SubDomain(consts.CollectSlug)
	konnectorURL.Fragment = "/category/all/" + slug
	mail := mails.Options{
		Mode:         mails.ModeNoReply,
		Subject:      inst.Translate("Error Konnector execution", domain),
		TemplateName: "konnector_error_" + inst.Locale,
		TemplateValues: map[string]string{
			"KonnectorName": slug,
			"KonnectorPage": konnectorURL.String(),
		},
	}
	msg, err := jobs.NewMessage(jobs.JSONEncoding, &mail)
	if err != nil {
		return err
	}
	log.Info("Konnector has failed definitively, should send mail.", mail)
	_, err = stack.GetBroker().PushJob(&jobs.JobRequest{
		Domain:     domain,
		WorkerType: "sendmail",
		Message:    msg,
	})
	return err
	return nil
}

func wrapErr(ctx context.Context, err error) error {
	if ctx.Err() == context.DeadlineExceeded {
		return context.DeadlineExceeded
	}
	return err
}
<commit_msg>Fix unreachable code<commit_after>package konnectors

import (
	"archive/tar"
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"runtime"
	"time"

	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/realtime"
	"github.com/cozy/cozy-stack/pkg/stack"
	"github.com/cozy/cozy-stack/pkg/workers/mails"
	"github.com/sirupsen/logrus"

	"github.com/spf13/afero"
)

func init() {
	jobs.AddWorker("konnector", &jobs.WorkerConfig{
		Concurrency:  runtime.NumCPU() * 2,
		MaxExecCount: 2,
		MaxExecTime:  200 * time.Second,
		Timeout:      200 * time.Second,
		WorkerFunc:   Worker,
		WorkerCommit: commit,
	})
}

// Options contains the options to execute a konnector.
type Options struct {
	Konnector    string `json:"konnector"`
	Account      string `json:"account"`
	FolderToSave string `json:"folder_to_save"`
}

// result stores the result of a konnector execution.
type result struct {
	DocID       string    `json:"_id,omitempty"`
	DocRev      string    `json:"_rev,omitempty"`
	CreatedAt   time.Time `json:"last_execution"`
	LastSuccess time.Time `json:"last_success"`
	Account     string    `json:"account"`
	State       string    `json:"state"`
	Error       string    `json:"error"`
}

func (r *result) ID() string         { return r.DocID }
func (r *result) Rev() string        { return r.DocRev }
func (r *result) DocType() string    { return consts.KonnectorResults }
func (r *result) Clone() couchdb.Doc { c := *r; return &c }
func (r *result) SetID(id string)    { r.DocID = id }
func (r *result) SetRev(rev string)  { r.DocRev = rev }

const konnectorMsgTypeError string = "error"

// const konnectorMsgTypeDebug string = "debug"
// const konnectorMsgTypeWarning string = "warning"
// const konnectorMsgTypeProgress string = "progress"

type konnectorMsg struct {
	Type    string `json:"type"`
	Message string `json:"message"`
}

// Worker is the worker that runs a konnector by executing an external process.
func Worker(ctx context.Context, m *jobs.Message) error {
	opts := &Options{}
	if err := m.Unmarshal(&opts); err != nil {
		return err
	}

	slug := opts.Konnector
	fields := struct {
		Account      string `json:"account"`
		FolderToSave string `json:"folder_to_save"`
	}{
		Account:      opts.Account,
		FolderToSave: opts.FolderToSave,
	}
	domain := ctx.Value(jobs.ContextDomainKey).(string)
	worker := ctx.Value(jobs.ContextWorkerKey).(string)
	jobID := fmt.Sprintf("%s/%s/%s", worker, slug, domain)

	inst, err := instance.Get(domain)
	if err != nil {
		return err
	}

	man, err := apps.GetKonnectorBySlug(inst, slug)
	if err != nil {
		return err
	}
	if man.State() != apps.Ready {
		return errors.New("Konnector is not ready")
	}

	token := inst.BuildKonnectorToken(man)

	osFS := afero.NewOsFs()
	workDir, err := afero.TempDir(osFS, "", "konnector-"+slug)
	if err != nil {
		return err
	}
	defer osFS.RemoveAll(workDir)
	workFS := afero.NewBasePathFs(osFS, workDir)

	fileServer := inst.KonnectorsFileServer()
	tarFile, err := fileServer.Open(slug, man.Version(), apps.KonnectorArchiveName)
	if err != nil {
		return err
	}

	tr := tar.NewReader(tarFile)
	for {
		var hdr *tar.Header
		hdr, err = tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		dirname := path.Dir(hdr.Name)
		if dirname != "." {
			if err = workFS.MkdirAll(dirname, 0755); err != nil {
				return nil
			}
		}
		var f afero.File
		f, err = workFS.OpenFile(hdr.Name, os.O_CREATE|os.O_WRONLY, 0644)
		if err != nil {
			return err
		}
		_, err = io.Copy(f, tr)
		errc := f.Close()
		if err != nil {
			return err
		}
		if errc != nil {
			return errc
		}
	}

	fieldsJSON, err := json.Marshal(fields)
	if err != nil {
		return err
	}

	konnCmd := config.GetConfig().Konnectors.Cmd
	cmd := exec.CommandContext(ctx, konnCmd, workDir) // #nosec
	cmd.Env = []string{
		"COZY_URL=" + inst.PageURL("/", nil),
		"COZY_CREDENTIALS=" + token,
		"COZY_FIELDS=" + string(fieldsJSON),
		"COZY_TYPE=" + man.Type,
		"COZY_JOB_ID=" + jobID,
	}

	cmdErr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}
	cmdOut, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	scanErr := bufio.NewScanner(cmdErr)
	scanOut := bufio.NewScanner(cmdOut)
	scanOut.Buffer(nil, 256*1024)

	var msgChan = make(chan konnectorMsg)
	var messages []konnectorMsg

	log := logger.WithDomain(domain)

	go doScanOut(jobID, scanOut, domain, msgChan, log)
	go doScanErr(jobID, scanErr, log)
	go func() {
		hub := realtime.GetHub()
		for msg := range msgChan {
			messages = append(messages, msg)
			hub.Publish(&realtime.Event{
				Verb: realtime.EventCreate,
				Doc: couchdb.JSONDoc{Type: consts.JobEvents, M: map[string]interface{}{
					"type":    msg.Type,
					"message": msg.Message,
				}},
				Domain: domain,
			})
		}
	}()

	if err = cmd.Start(); err != nil {
		return wrapErr(ctx, err)
	}

	err = cmd.Wait()
	if err != nil {
		err = wrapErr(ctx, err)
	}

	close(msgChan)
	for _, msg := range messages {
		if msg.Type == konnectorMsgTypeError {
			// konnector err is more explicit
			return errors.New(msg.Message)
		}
	}

	return err
}

func doScanOut(jobID string, scanner *bufio.Scanner, domain string,
	msgs chan konnectorMsg, log *logrus.Entry) {
	for scanner.Scan() {
		linebb := scanner.Bytes()
		from := bytes.IndexByte(linebb, '{')
		to := bytes.LastIndexByte(linebb, '}')
		var msg konnectorMsg
		log.Infof("[konnector] %s: Stdout: %s", jobID, string(linebb))
		if from > -1 && from < to && to > -1 {
			err := json.Unmarshal(linebb[from:to+1], &msg)
			if err == nil {
				msgs <- msg
				continue
			}
		}
		log.Warnf("[konnector] %s: Could not parse as JSON", jobID)
		log.Debugf("[konnector] %s: %s", jobID, string(linebb))
	}
	if err := scanner.Err(); err != nil {
		log.Errorf("[konnector] %s: Error while reading stdout: %s", jobID, err)
	}
}

func doScanErr(jobID string, scanner *bufio.Scanner, log *logrus.Entry) {
	for scanner.Scan() {
		log.Errorf("[konnector] %s: Stderr: %s", jobID, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		log.Errorf("[konnector] %s: Error while reading stderr: %s", jobID, err)
	}
}

func commit(ctx context.Context, m *jobs.Message, errjob error) error {
	opts := &Options{}
	if err := m.Unmarshal(&opts); err != nil {
		return err
	}

	slug := opts.Konnector
	domain := ctx.Value(jobs.ContextDomainKey).(string)

	log := logger.WithDomain(domain)

	inst, err := instance.Get(domain)
	if err != nil {
		return err
	}

	lastResult := &result{}
	err = couchdb.GetDoc(inst, consts.KonnectorResults, slug, lastResult)
	if err != nil {
		if !couchdb.IsNotFoundError(err) {
			return err
		}
		lastResult = nil
	}

	var state, errstr string
	var lastSuccess time.Time
	if errjob != nil {
		if lastResult != nil {
			lastSuccess = lastResult.LastSuccess
		}
		errstr = errjob.Error()
		state = jobs.Errored
	} else {
		lastSuccess = time.Now()
		state = jobs.Done
	}
	result := &result{
		DocID:       slug,
		Account:     opts.Account,
		CreatedAt:   time.Now(),
		LastSuccess: lastSuccess,
		State:       state,
		Error:       errstr,
	}
	if lastResult == nil {
		err = couchdb.CreateNamedDocWithDB(inst, result)
	} else {
		result.SetRev(lastResult.Rev())
		err = couchdb.UpdateDoc(inst, result)
	}
	if err != nil {
		return err
	}

	// if it is the first try we do not take into account an error, we bail.
	if lastResult == nil {
		return nil
	}
	// if the job has not errored, or the last one was already errored, we bail.
	if state != jobs.Errored || lastResult.State == jobs.Errored {
		return nil
	}

	konnectorURL := inst.SubDomain(consts.CollectSlug)
	konnectorURL.Fragment = "/category/all/" + slug
	mail := mails.Options{
		Mode:         mails.ModeNoReply,
		Subject:      inst.Translate("Error Konnector execution", domain),
		TemplateName: "konnector_error_" + inst.Locale,
		TemplateValues: map[string]string{
			"KonnectorName": slug,
			"KonnectorPage": konnectorURL.String(),
		},
	}
	msg, err := jobs.NewMessage(jobs.JSONEncoding, &mail)
	if err != nil {
		return err
	}
	log.Info("Konnector has failed definitively, should send mail.", mail)
	_, err = stack.GetBroker().PushJob(&jobs.JobRequest{
		Domain:     domain,
		WorkerType: "sendmail",
		Message:    msg,
	})
	return err
}

func wrapErr(ctx context.Context, err error) error {
	if ctx.Err() == context.DeadlineExceeded {
		return context.DeadlineExceeded
	}
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"fmt"
	log "github.com/sirupsen/logrus"
	"os"
	utils "signalbroker-server/examples/grpc/go/timeSync/util"
	"strconv"
	"strings"
)
import "google.golang.org/grpc"
import "signalbroker-server/examples/grpc/go/timeSync/proto_files"

// json file for connection specifics.
type Configuration struct{
	Brokerip string
	Brokerport string
}

var conf Configuration

// define signal data

type signalid struct{
	Identifier string
}

type framee struct{
	Frameid string `json:frameid`
	Sigids []signalid `json:sigids`
}

type spaces struct{
	Name  string `json:name`
	Frames []framee `json:framee`
}

type settings struct{
	Namespaces []spaces `json:namespaces`
}

type VehiclesList struct{
	Vehicles []settings `json:vehicles`
}

const(
	ddindex = 0
	hhindex = 2
	mmindex = 4
	ssindex = 6
)
func display_subscribedvalues(signals []*base.Signal){

	timestrings := []string{"00",":","00",":","00",":","00"}

	// collect the output string
	for _,asignal := range signals{
		zerofiller := ""
		if asignal.GetInteger() < 10 {
			zerofiller = "0"
		}
		switch asignal.Id.Name{
		case "Day":
			timestrings[ddindex] = zerofiller + strconv.FormatInt(asignal.GetInteger(),10)
		case "Hr":
			timestrings[hhindex] = zerofiller + strconv.FormatInt(asignal.GetInteger(),10)
		case "Mins":
			timestrings[mmindex] = zerofiller + strconv.FormatInt(asignal.GetInteger(),10)
		case "Sec":
			timestrings[ssindex] = zerofiller + strconv.FormatInt(asignal.GetInteger(),10)
		}
	}

	print(strings.Join(timestrings,""))
}

func subcribe_to_signal_set(clientconnection base.NetworkServiceClient,signals *base.SubscriberConfig,ch chan int) {

	response, err := clientconnection.SubscribeToSignals(context.Background(),signals);

	if (err != nil){
		log.Debug(" error in subscrition to signals ", err);
	} else {
		for {
			msg,err := response.Recv(); // wait for a subscription msg
			if (err != nil){
				log.Debug(" error ", err);
				break;
			}

			display_subscribedvalues(msg.GetSignal())
		}
	}

	log.Info(" Done subcribing ...")
	ch <- 1 // don't block any more.
}

func initConfiguration()(bool){
	file,err := os.Open("configuration.json")
	defer file.Close()

	if  err != nil {
		log.Error("could not open configuration.json ", err)
		return false
	} else{
		decoder := json.NewDecoder(file)
		conf = Configuration{}
		err2 := decoder.Decode(&conf)
		if err2 != nil{
			log.Error("could not parse configuration.json ", err2)
			return false
		}

	}

	inializePlotter()
	return true
}

// print current configuration to the console
func printSignalTree(clientconnection *grpc.ClientConn) {
	systemServiceClient := base.NewSystemServiceClient(clientconnection);
	configuration,err := systemServiceClient.GetConfiguration(context.Background(),&base.Empty{})

	infos := configuration.GetNetworkInfo();
	for _,element := range infos{
		printSignals(element.Namespace.Name,clientconnection);
	}

	if err != nil{
		log.Debug("could not retrieve configuration " , err);
	}

}

// print signal tree(s) to console , using fmt for this.
func printSpaces(number int){
	for k := 1; k < number; k++ {
		fmt.Print(" ");
	}
}

func printTreeBranch(){
	fmt.Print("|");
}

func getFirstNameSpace(frames []*base.FrameInfo) string{
	element := frames[0];
	return element.SignalInfo.Id.Name;
}

func printSignals(zenamespace string,clientconnection *grpc.ClientConn){
	systemServiceClient := base.NewSystemServiceClient(clientconnection)
	signallist, err := systemServiceClient.ListSignals(context.Background(),&base.NameSpace{Name : zenamespace})

	frames := signallist.GetFrame();

	rootstring := "|[" + zenamespace + "]---|";
	rootstringlength := len(rootstring);
	fmt.Println(rootstring);

	for _,element := range frames{

		printTreeBranch();
		printSpaces(rootstringlength -1);

		framestring := "|---[" + element.SignalInfo.Id.Name + "]---|";
		framestringlength := len(framestring);

		fmt.Println(framestring);
		childs := element.ChildInfo;

		for _,childelement := range childs{
			outstr := childelement.Id.Name;
			printTreeBranch();
			printSpaces(rootstringlength -1);
			printTreeBranch();
			printSpaces(framestringlength - 1);
			fmt.Println("|---{", outstr, "}");
		}
	}

	if err != nil {
		log.Debug(" could not list signals ", err);
	}
}

// hard coded predefined settings used for examples.
func subsignalDB() (*settings){
	data := &settings{
		Namespaces: []spaces{
			{Name: "BodyCANhs",
				Frames: []framee{
					{Frameid: "CEMBodyFr29",
						Sigids: []signalid{
							{Identifier: "Day"},
							{Identifier: "Hr"},
							{Identifier: "Mins"},
							{Identifier: "Sec"},
							{Identifier: "TiAndDateIndcn_UB"},
							{Identifier: "TiAndDateVld"},
						}},
				},
			},
		},
	}

   return data
}

// set signal name and namespace to grpc generated data strcuture
func getSignaId(signalName string,namespaceName string) *base.SignalId{
	return &base.SignalId{
		Name: signalName,
		Namespace:&base.NameSpace{
			Name:namespaceName},
	}
}

// set signals and namespaces to grpc subscriber configuration, see files under proto_files
func getSignals(data *settings)*base.SubscriberConfig{
	var signalids []*base.SignalId;
	var namespacename string

	for cindex := 0; cindex < len(data.Namespaces); cindex++{
		namespacename = data.Namespaces[cindex].Name;
		for _,frameelement := range data.Namespaces[cindex].Frames{
			for _,sigelement := range frameelement.Sigids{
				log.Info("subscribing to signal: " , sigelement);
				signalids = append(signalids,getSignaId(sigelement.Identifier,namespacename));
			}
		}
	}

	// add selected signals to subscriber configuration
	signals := &base.SubscriberConfig{
		ClientId: &base.ClientId{
			Id: "app_identifier",
		},
		Signals: &base.SignalIds{
			SignalId:signalids,
		},
		OnChange: false,
	}

	return signals
}

func main(){
	fmt.Println(" we are trying go with the volvo signal broker")

	// http server for output
	go utils.ServePrinter()

	initConfiguration()
	conn, err := grpc.Dial(conf.Brokerip + ":"+ string(conf.Brokerport), grpc.WithInsecure())
	if err != nil {
		log.Debug("did not connect: %v", err)
	}
	defer conn.Close()

	// get system and basic signal information from broker
	printSignalTree(conn)
	c := base.NewNetworkServiceClient(conn)

	// prevents main thread from finishing
	var ch chan int = make(chan int)

	signals := getSignals(subsignalDB())
	// start subscription thread
	go subcribe_to_signal_set(c,signals,ch);
	log.Info(" Waiting for subscription to end ...")
	fmt.Println(<-ch);
}
<commit_msg>changed a comment<commit_after>package main

import (
	"context"
	"encoding/json"
	"fmt"
	log "github.com/sirupsen/logrus"
	"os"
	utils "signalbroker-server/examples/grpc/go/timeSync/util"
	"strconv"
	"strings"
)
import "google.golang.org/grpc"
import "signalbroker-server/examples/grpc/go/timeSync/proto_files"

// json file for connection specifics.
type Configuration struct{
	Brokerip string
	Brokerport string
}

var conf Configuration

// define signal data

type signalid struct{
	Identifier string
}

type framee struct{
	Frameid string `json:frameid`
	Sigids []signalid `json:sigids`
}

type spaces struct{
	Name  string `json:name`
	Frames []framee `json:framee`
}

type settings struct{
	Namespaces []spaces `json:namespaces`
}

type VehiclesList struct{
	Vehicles []settings `json:vehicles`
}

const(
	ddindex = 0
	hhindex = 2
	mmindex = 4
	ssindex = 6
)
func display_subscribedvalues(signals []*base.Signal){

	timestrings := []string{"00",":","00",":","00",":","00"}

	// collect the output string
	for _,asignal := range signals{
		zerofiller := ""
		if asignal.GetInteger() < 10 {
			zerofiller = "0"
		}
		switch asignal.Id.Name{
		case "Day":
			timestrings[ddindex] = zerofiller + strconv.FormatInt(asignal.GetInteger(),10)
		case "Hr":
			timestrings[hhindex] = zerofiller + strconv.FormatInt(asignal.GetInteger(),10)
		case "Mins":
			timestrings[mmindex] = zerofiller + strconv.FormatInt(asignal.GetInteger(),10)
		case "Sec":
			timestrings[ssindex] = zerofiller + strconv.FormatInt(asignal.GetInteger(),10)
		}
	}

	print(strings.Join(timestrings,""))
}

func subcribe_to_signal_set(clientconnection base.NetworkServiceClient,signals *base.SubscriberConfig,ch chan int) {

	response, err := clientconnection.SubscribeToSignals(context.Background(),signals);

	if (err != nil){
		log.Debug(" error in subscrition to signals ", err);
	} else {
		for {
			msg,err := response.Recv(); // wait for a subscription msg
			if (err != nil){
				log.Debug(" error ", err);
				break;
			}

			display_subscribedvalues(msg.GetSignal())
		}
	}

	log.Info(" Done subcribing ...")
	ch <- 1 // don't block any more.
}

func initConfiguration()(bool){
	file,err := os.Open("configuration.json")
	defer file.Close()

	if  err != nil {
		log.Error("could not open configuration.json ", err)
		return false
	} else{
		decoder := json.NewDecoder(file)
		conf = Configuration{}
		err2 := decoder.Decode(&conf)
		if err2 != nil{
			log.Error("could not parse configuration.json ", err2)
			return false
		}

	}

	inializePlotter()
	return true
}

// print current configuration to the console
func printSignalTree(clientconnection *grpc.ClientConn) {
	systemServiceClient := base.NewSystemServiceClient(clientconnection);
	configuration,err := systemServiceClient.GetConfiguration(context.Background(),&base.Empty{})

	infos := configuration.GetNetworkInfo();
	for _,element := range infos{
		printSignals(element.Namespace.Name,clientconnection);
	}

	if err != nil{
		log.Debug("could not retrieve configuration " , err);
	}

}

// print signal tree(s) to console , using fmt for this.
func printSpaces(number int){
	for k := 1; k < number; k++ {
		fmt.Print(" ");
	}
}

func printTreeBranch(){
	fmt.Print("|");
}

func getFirstNameSpace(frames []*base.FrameInfo) string{
	element := frames[0];
	return element.SignalInfo.Id.Name;
}

func printSignals(zenamespace string,clientconnection *grpc.ClientConn){
	systemServiceClient := base.NewSystemServiceClient(clientconnection)
	signallist, err := systemServiceClient.ListSignals(context.Background(),&base.NameSpace{Name : zenamespace})

	frames := signallist.GetFrame();

	rootstring := "|[" + zenamespace + "]---|";
	rootstringlength := len(rootstring);
	fmt.Println(rootstring);

	for _,element := range frames{

		printTreeBranch();
		printSpaces(rootstringlength -1);

		framestring := "|---[" + element.SignalInfo.Id.Name + "]---|";
		framestringlength := len(framestring);

		fmt.Println(framestring);
		childs := element.ChildInfo;

		for _,childelement := range childs{
			outstr := childelement.Id.Name;
			printTreeBranch();
			printSpaces(rootstringlength -1);
			printTreeBranch();
			printSpaces(framestringlength - 1);
			fmt.Println("|---{", outstr, "}");
		}
	}

	if err != nil {
		log.Debug(" could not list signals ", err);
	}
}

// hard coded predefined sigbal settings used for this example.
func subsignalDB() (*settings){
	data := &settings{
		Namespaces: []spaces{
			{Name: "BodyCANhs",
				Frames: []framee{
					{Frameid: "CEMBodyFr29",
						Sigids: []signalid{
							{Identifier: "Day"},
							{Identifier: "Hr"},
							{Identifier: "Mins"},
							{Identifier: "Sec"},
							{Identifier: "TiAndDateIndcn_UB"},
							{Identifier: "TiAndDateVld"},
						}},
				},
			},
		},
	}

   return data
}

// set signal name and namespace to grpc generated data strcuture
func getSignaId(signalName string,namespaceName string) *base.SignalId{
	return &base.SignalId{
		Name: signalName,
		Namespace:&base.NameSpace{
			Name:namespaceName},
	}
}

// set signals and namespaces to grpc subscriber configuration, see files under proto_files
func getSignals(data *settings)*base.SubscriberConfig{
	var signalids []*base.SignalId;
	var namespacename string

	for cindex := 0; cindex < len(data.Namespaces); cindex++{
		namespacename = data.Namespaces[cindex].Name;
		for _,frameelement := range data.Namespaces[cindex].Frames{
			for _,sigelement := range frameelement.Sigids{
				log.Info("subscribing to signal: " , sigelement);
				signalids = append(signalids,getSignaId(sigelement.Identifier,namespacename));
			}
		}
	}

	// add selected signals to subscriber configuration
	signals := &base.SubscriberConfig{
		ClientId: &base.ClientId{
			Id: "app_identifier",
		},
		Signals: &base.SignalIds{
			SignalId:signalids,
		},
		OnChange: false,
	}

	return signals
}

func main(){
	fmt.Println(" we are trying go with the volvo signal broker")

	// http server for output
	go utils.ServePrinter()

	initConfiguration()
	conn, err := grpc.Dial(conf.Brokerip + ":"+ string(conf.Brokerport), grpc.WithInsecure())
	if err != nil {
		log.Debug("did not connect: %v", err)
	}
	defer conn.Close()

	// get system and basic signal information from broker
	printSignalTree(conn)
	c := base.NewNetworkServiceClient(conn)

	// prevents main thread from finishing
	var ch chan int = make(chan int)

	signals := getSignals(subsignalDB())
	// start subscription thread
	go subcribe_to_signal_set(c,signals,ch);
	log.Info(" Waiting for subscription to end ...")
	fmt.Println(<-ch);
}
<|endoftext|>


<commit_before>package bongo

import (
	"github.com/cenkalti/backoff"
	"github.com/koding/broker"
	"github.com/koding/logging"

	"github.com/jinzhu/gorm"
)

var B *Bongo

type Bongo struct {
	Broker *broker.Broker
	DB     *gorm.DB
	log    logging.Logger
	Cache  Cache
}

func New(b *broker.Broker, db *gorm.DB, l logging.Logger) *Bongo {
	return &Bongo{
		Broker: b,
		DB:     db,
		log:    l,
	}
}

func (b *Bongo) Connect() error {

	bo := backoff.NewExponentialBackOff()
	ticker := backoff.NewTicker(bo)
	defer ticker.Stop()

	var err error
	for _ = range ticker.C {
		if err = b.Broker.Connect(); err != nil {
			b.log.Error("err while connecting: %s  will retry...", err.Error())
			continue
		}

		break
	}

	if err != nil {
		return err
	}

	B = b

	b.log.Info("Bongo connected %t", true)
	// todo add gorm Connect()
	return nil
}

func (b *Bongo) Close() error {
	if err := b.Broker.Close(); err != nil {
		return err
	}
	b.log.Info("Bongo dis-connected %t", true)

	// todo add gorm Close()
	return nil
}
<commit_msg>bongo: get redis connection from bongo<commit_after>package bongo

import (
	"github.com/cenkalti/backoff"
	"github.com/koding/broker"
	"github.com/koding/logging"
	"github.com/koding/redis"

	"github.com/jinzhu/gorm"
)

var B *Bongo

type Bongo struct {
	Broker *broker.Broker
	DB     *gorm.DB
	log    logging.Logger
	Cache  Cache
}

func New(b *broker.Broker, db *gorm.DB, l logging.Logger) *Bongo {
	return &Bongo{
		Broker: b,
		DB:     db,
		log:    l,
	}
}

func (b *Bongo) Connect() error {

	bo := backoff.NewExponentialBackOff()
	ticker := backoff.NewTicker(bo)
	defer ticker.Stop()

	var err error
	for _ = range ticker.C {
		if err = b.Broker.Connect(); err != nil {
			b.log.Error("err while connecting: %s  will retry...", err.Error())
			continue
		}

		break
	}

	if err != nil {
		return err
	}

	B = b

	b.log.Info("Bongo connected %t", true)
	// todo add gorm Connect()
	return nil
}

func (b *Bongo) Close() error {
	if err := b.Broker.Close(); err != nil {
		return err
	}
	b.log.Info("Bongo dis-connected %t", true)

	r, ok := b.Cache.(*redis.RedisSession)
	if ok {
		r.Close()
	}

	// todo add gorm Close()
	return nil
}

func (b *Bongo) GetRedisConn() *redis.RedisSession {
	r, _ := b.Cache.(*redis.RedisSession)

	return r
}
<|endoftext|>


<commit_before>package machine

import (
	"context"
	"errors"
	"sync"
	"time"

	"github.com/koding/logging"
)

// DynamicAddrFunc is an adapter that allows to dynamically provide addresses
// from a given network. Error should be of ErrAddrNotFound type when provided
// network has no addresses.
type DynamicAddrFunc func(string) (Addr, error)

// ClientBuilder is an interface used to dynamically build remote machine clients.
type ClientBuilder interface {
	// Ping uses dynamic address provider to ping the machine. If error is nil,
	// this method should return address which was used to ping the machine.
	Ping(dynAddr DynamicAddrFunc) (Status, Addr, error)

	// Build builds new client which will connect to machine using provided
	// address.
	Build(ctx context.Context, addr Addr) Client
}

// DynamicClientOpts are the options used to configure dynamic client.
type DynamicClientOpts struct {
	// AddrFunc is a factory for dynamic machine addresses.
	AddrFunc DynamicAddrFunc

	// Builder is a factory used to build clients.
	Builder ClientBuilder

	// DynAddrInterval indicates how often dynamic client should pull address
	// function looking for new addresses.
	DynAddrInterval time.Duration

	// PingInterval indicates how often dynamic client should ping external
	// machine.
	PingInterval time.Duration

	// Log is used for logging. If nil, default logger will be created.
	Log logging.Logger
}

// Valid checks if provided options are correct.
func (opts *DynamicClientOpts) Valid() error {
	if opts.AddrFunc == nil {
		return errors.New("nil dynamic address function")
	}
	if opts.Builder == nil {
		return errors.New("nil client builder")
	}
	if opts.DynAddrInterval == 0 {
		return errors.New("dynamic address check interval is not set")
	}
	if opts.PingInterval == 0 {
		return errors.New("ping interval is not set")
	}

	return nil
}

// DynamicClient is a client that may change it's endpoint address depending
// on client builder ping function status. It is safe to use this structure
// concurrently.
type DynamicClient struct {
	opts DynamicClientOpts
	log  logging.Logger

	once sync.Once
	stop chan struct{} // channel used to close dynamic client.

	mu     sync.RWMutex
	c      Client             // current client.
	stat   Status             // current connection status.
	ctx    context.Context    // context used in current client.
	cancel context.CancelFunc // function that can close current context.
}

// NewDynamicClient starts and returns a new DynamicClient instance. The caller
// should call Close when finished, in order to shut it down.
func NewDynamicClient(opts DynamicClientOpts) (*DynamicClient, error) {
	if err := opts.Valid(); err != nil {
		return nil, err
	}

	stop := make(chan struct{}, 1) // make Close function unblocked.

	dc := &DynamicClient{
		opts: opts,
		stop: stop,
	}

	if opts.Log != nil {
		dc.log = opts.Log.New("monitor")
	} else {
		dc.log = DefaultLogger.New("monitor")
	}

	dc.disconnected() // set disconnected client.
	go dc.cron()

	return dc, nil
}

// Status gets current client status. It may return zero value when client is
// disconnected.
func (dc *DynamicClient) Status() Status {
	dc.mu.RLock()
	stat := dc.stat
	dc.mu.RUnlock()

	return stat
}

// Client returns current client.
func (dc *DynamicClient) Client() Client {
	dc.mu.RLock()
	c := dc.c
	dc.mu.RUnlock()

	return c
}

// Context returns current client's context. If client change, returned context
// will be canceled. If there is no clients available in dynamic client. Already
// canceled context is returned.
func (dc *DynamicClient) Context() context.Context {
	dc.mu.RLock()
	defer dc.mu.RUnlock()

	if dc.ctx == nil {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		return ctx
	}

	return dc.ctx
}

// Addr uses dynamic address function binded to client to obtain addresses.
func (dc *DynamicClient) Addr(network string) (Addr, error) {
	return dc.opts.AddrFunc(network)
}

// Close stops the dynamic client. After this function is called, client is
// in disconnected state and each contexts returned by it are closed.
func (dc *DynamicClient) Close() {
	dc.once.Do(func() {
		close(dc.stop)
	})
}

func (dc *DynamicClient) cron() {
	var (
		dynAddrTick = time.NewTicker(dc.opts.DynAddrInterval)
		pingTick    = time.NewTicker(dc.opts.PingInterval)
	)

	curr := Addr{}
	dc.tryUpdate(&curr)
	for {
		select {
		case <-dynAddrTick.C:
			// Look address cache for new addresses. This does not require
			// pinging remote machines because it only checks current address
			// book state. Thus, it may be run more frequently than ping.
			a, err := dc.opts.AddrFunc(curr.Network)
			if err != nil || (a.Network == curr.Network && a.Value == curr.Value) {
				break
			}
			dc.tryUpdate(&curr)
		case <-pingTick.C:
			// Ping remote machine directly in order to check its status.
			dc.tryUpdate(&curr)
		case <-dc.stop:
			// Client was closed.
			dc.mu.Lock()
			dc.disconnected()
			dc.mu.Unlock()

			// Stop tickers.
			dynAddrTick.Stop()
			pingTick.Stop()
			return
		}
	}
}

// tryUpdate uses client builder to ping the machine and updates dynamic client
// if machine address changes.
func (dc *DynamicClient) tryUpdate(addr *Addr) {
	stat, a, err := dc.opts.Builder.Ping(dc.opts.AddrFunc)
	if err != nil {
		dc.log.Warning("Machine ping error: %s", err)
		return
	}

	if a.Network == addr.Network && a.Value == addr.Value {
		// Client address did not change.
		return
	}

	// Create new client.
	dc.log.Info("Reinitializing client with %s address: %s", a.Network, a.Value)
	ctx, cancel := context.WithCancel(context.Background())
	c := dc.opts.Builder.Build(ctx, a)

	// Update current address.
	*addr = a

	dc.mu.Lock()
	if dc.cancel != nil {
		dc.cancel()
	}
	dc.c, dc.stat, dc.ctx, dc.cancel = c, stat, ctx, cancel
	dc.mu.Unlock()
}

// disconnected sets disconnected client.
func (dc *DynamicClient) disconnected() {
	if dc.cancel != nil {
		dc.cancel()
	}

	dc.c = DisconnectedClient{}
	dc.stat = Status{}
	dc.ctx = nil
	dc.cancel = nil
}
<commit_msg>klient: limit ping error logs to not print them periodically<commit_after>package machine

import (
	"context"
	"errors"
	"sync"
	"time"

	"github.com/koding/logging"
)

// DynamicAddrFunc is an adapter that allows to dynamically provide addresses
// from a given network. Error should be of ErrAddrNotFound type when provided
// network has no addresses.
type DynamicAddrFunc func(string) (Addr, error)

// ClientBuilder is an interface used to dynamically build remote machine clients.
type ClientBuilder interface {
	// Ping uses dynamic address provider to ping the machine. If error is nil,
	// this method should return address which was used to ping the machine.
	Ping(dynAddr DynamicAddrFunc) (Status, Addr, error)

	// Build builds new client which will connect to machine using provided
	// address.
	Build(ctx context.Context, addr Addr) Client
}

// DynamicClientOpts are the options used to configure dynamic client.
type DynamicClientOpts struct {
	// AddrFunc is a factory for dynamic machine addresses.
	AddrFunc DynamicAddrFunc

	// Builder is a factory used to build clients.
	Builder ClientBuilder

	// DynAddrInterval indicates how often dynamic client should pull address
	// function looking for new addresses.
	DynAddrInterval time.Duration

	// PingInterval indicates how often dynamic client should ping external
	// machine.
	PingInterval time.Duration

	// Log is used for logging. If nil, default logger will be created.
	Log logging.Logger
}

// Valid checks if provided options are correct.
func (opts *DynamicClientOpts) Valid() error {
	if opts.AddrFunc == nil {
		return errors.New("nil dynamic address function")
	}
	if opts.Builder == nil {
		return errors.New("nil client builder")
	}
	if opts.DynAddrInterval == 0 {
		return errors.New("dynamic address check interval is not set")
	}
	if opts.PingInterval == 0 {
		return errors.New("ping interval is not set")
	}

	return nil
}

// DynamicClient is a client that may change it's endpoint address depending
// on client builder ping function status. It is safe to use this structure
// concurrently.
type DynamicClient struct {
	opts DynamicClientOpts
	log  logging.Logger

	once sync.Once
	stop chan struct{} // channel used to close dynamic client.

	mu     sync.RWMutex
	c      Client             // current client.
	stat   Status             // current connection status.
	ctx    context.Context    // context used in current client.
	cancel context.CancelFunc // function that can close current context.
}

// NewDynamicClient starts and returns a new DynamicClient instance. The caller
// should call Close when finished, in order to shut it down.
func NewDynamicClient(opts DynamicClientOpts) (*DynamicClient, error) {
	if err := opts.Valid(); err != nil {
		return nil, err
	}

	stop := make(chan struct{}, 1) // make Close function unblocked.

	dc := &DynamicClient{
		opts: opts,
		stop: stop,
	}

	if opts.Log != nil {
		dc.log = opts.Log.New("monitor")
	} else {
		dc.log = DefaultLogger.New("monitor")
	}

	dc.disconnected() // set disconnected client.
	go dc.cron()

	return dc, nil
}

// Status gets current client status. It may return zero value when client is
// disconnected.
func (dc *DynamicClient) Status() Status {
	dc.mu.RLock()
	stat := dc.stat
	dc.mu.RUnlock()

	return stat
}

// Client returns current client.
func (dc *DynamicClient) Client() Client {
	dc.mu.RLock()
	c := dc.c
	dc.mu.RUnlock()

	return c
}

// Context returns current client's context. If client change, returned context
// will be canceled. If there is no clients available in dynamic client. Already
// canceled context is returned.
func (dc *DynamicClient) Context() context.Context {
	dc.mu.RLock()
	defer dc.mu.RUnlock()

	if dc.ctx == nil {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		return ctx
	}

	return dc.ctx
}

// Addr uses dynamic address function binded to client to obtain addresses.
func (dc *DynamicClient) Addr(network string) (Addr, error) {
	return dc.opts.AddrFunc(network)
}

// Close stops the dynamic client. After this function is called, client is
// in disconnected state and each contexts returned by it are closed.
func (dc *DynamicClient) Close() {
	dc.once.Do(func() {
		close(dc.stop)
	})
}

func (dc *DynamicClient) cron() {
	var (
		dynAddrTick = time.NewTicker(dc.opts.DynAddrInterval)
		pingTick    = time.NewTicker(dc.opts.PingInterval)
	)

	curr, islog := Addr{}, true
	dc.tryUpdate(&curr, &islog)
	for {
		select {
		case <-dynAddrTick.C:
			// Look address cache for new addresses. This does not require
			// pinging remote machines because it only checks current address
			// book state. Thus, it may be run more frequently than ping.
			a, err := dc.opts.AddrFunc(curr.Network)
			if err != nil || (a.Network == curr.Network && a.Value == curr.Value) {
				break
			}
			dc.tryUpdate(&curr, &islog)
		case <-pingTick.C:
			// Ping remote machine directly in order to check its status.
			dc.tryUpdate(&curr, &islog)
		case <-dc.stop:
			// Client was closed.
			dc.mu.Lock()
			dc.disconnected()
			dc.mu.Unlock()

			// Stop tickers.
			dynAddrTick.Stop()
			pingTick.Stop()
			return
		}
	}
}

// tryUpdate uses client builder to ping the machine and updates dynamic client
// if machine address changes.
func (dc *DynamicClient) tryUpdate(addr *Addr, islog *bool) {
	stat, a, err := dc.opts.Builder.Ping(dc.opts.AddrFunc)
	if err != nil {
		if *islog {
			// Log only once in order to not spam log files.
			dc.log.Warning("Machine ping error: %s", err)
			*islog = false
		}
		return
	}

	if a.Network == addr.Network && a.Value == addr.Value {
		// Client address did not change.
		return
	}

	// Create new client.
	*islog = true
	dc.log.Info("Reinitializing client with %s address: %s", a.Network, a.Value)
	ctx, cancel := context.WithCancel(context.Background())
	c := dc.opts.Builder.Build(ctx, a)

	// Update current address.
	*addr = a

	dc.mu.Lock()
	if dc.cancel != nil {
		dc.cancel()
	}
	dc.c, dc.stat, dc.ctx, dc.cancel = c, stat, ctx, cancel
	dc.mu.Unlock()
}

// disconnected sets disconnected client.
func (dc *DynamicClient) disconnected() {
	if dc.cancel != nil {
		dc.cancel()
	}

	dc.c = DisconnectedClient{}
	dc.stat = Status{}
	dc.ctx = nil
	dc.cancel = nil
}
<|endoftext|>


<commit_before>package models

import (
	"socialapi/workers/common/runner"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestChannelCreate(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while creating channel", t, func() {
		Convey("channel name should not be empty", func(){
			c := NewChannel()
			c.Name = ""
			So(c.Create().Error(), ShouldContainSubstring, "Validation failed")
		})

		Convey("channel groupName should not be empty", func(){
			c := NewChannel()
			c.GroupName = ""
			So(c.Create().Error(), ShouldContainSubstring, "Validation failed")
		})

		Convey("channel typeConstant should not be empty", func(){
			c := NewChannel()
			c.TypeConstant = ""
			So(c.Create().Error(), ShouldContainSubstring, "Validation failed")
		})

		Convey("channel name should not contain whitespace", func(){
			c := NewChannel()
			c.Name = "name channel"
			So(c.Create().Error(), ShouldContainSubstring, "has empty space")
		})
	})

}

func TestChannelTableName(t *testing.T) {
	Convey("while testing TableName()", t, func() {
		So(NewChannel().TableName(), ShouldEqual, ChannelTableName)
	})
}

func TestChannelCanOpen(t *testing.T) {
	Convey("while testing channel permissions", t, func() {
		Convey("can not open uninitialized channel", func() {
			c := NewChannel()
			canOpen, err := c.CanOpen(1231)
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, ErrChannelIdIsNotSet)
			So(canOpen, ShouldBeFalse)
		})

		Convey("uninitialized account can not open channel", func() {
			c := NewChannel()
			c.Id = 123
			canOpen, err := c.CanOpen(0)
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, ErrAccountIdIsNotSet)
			So(canOpen, ShouldBeFalse)
		})

		Convey("participants can open group channel", func() {
			// init account
			account, err := createAccount()
			So(err, ShouldBeNil)
			So(account, ShouldNotBeNil)
			So(account.Id, ShouldNotEqual, 0)

			// init channel
			c := NewChannel()
			c.CreatorId = account.Id
			c.TypeConstant = Channel_TYPE_GROUP

			So(c.Create(), ShouldBeNil)

			cp, err := c.AddParticipant(account.Id)
			So(err, ShouldBeNil)
			So(cp, ShouldNotBeNil)

			canOpen, err := c.CanOpen(account.Id)
			So(err, ShouldBeNil)
			So(canOpen, ShouldBeTrue)
		})

		Convey("everyone can open group channel", func() {
			// init account
			account, err := createAccount()
			So(err, ShouldBeNil)
			So(account, ShouldNotBeNil)
			So(account.Id, ShouldNotEqual, 0)

			// init channel
			c := NewChannel()
			c.CreatorId = account.Id
			c.TypeConstant = Channel_TYPE_GROUP

			So(c.Create(), ShouldBeNil)

			// 1 is just a random id
			canOpen, err := c.CanOpen(1)
			So(err, ShouldBeNil)
			So(canOpen, ShouldBeTrue)
		})

		Convey("everyone can open topic channel", func() {
			// init account
			account, err := createAccount()
			So(err, ShouldBeNil)
			So(account, ShouldNotBeNil)
			So(account.Id, ShouldNotEqual, 0)

			// init channel
			c := NewChannel()
			c.CreatorId = account.Id
			c.TypeConstant = Channel_TYPE_TOPIC

			So(c.Create(), ShouldBeNil)

			// 1 is just a random id
			canOpen, err := c.CanOpen(1)
			So(err, ShouldBeNil)
			So(canOpen, ShouldBeTrue)
		})

		Convey("participants can open pinned activity channel", func(){
			// init account
			account, err := createAccount()
			So(err, ShouldBeNil)
			So(account, ShouldNotBeNil)
			So(account.Id, ShouldNotEqual, 0)

			// init channel
			c := NewChannel()
			c.CreatorId = account.Id
			c.TypeConstant = Channel_TYPE_PINNED_ACTIVITY

			So(c.Create(), ShouldBeNil)

			cp, err := c.AddParticipant(account.Id)
			So(err, ShouldBeNil)
			So(cp, ShouldNotBeNil)

			canOpen, err := c.CanOpen(account.Id)
			So(err, ShouldBeNil)
			So(canOpen, ShouldBeFalse)
		})

		Convey("participants can open private message channel", func(){
			// init account
			account, err := createAccount()
			So(err, ShouldBeNil)
			So(account, ShouldNotBeNil)
			So(account.Id, ShouldNotEqual, 0)

			// init channel
			c := NewChannel()
			c.CreatorId = account.Id
			c.TypeConstant = Channel_TYPE_PRIVATE_MESSAGE

			So(c.Create(), ShouldBeNil)

			// add participant to the channel
			cp, err := c.AddParticipant(account.Id)
			So(err, ShouldBeNil)
			So(cp, ShouldNotBeNil)

			canOpen, err := c.CanOpen(account.Id)
			So(err, ShouldBeNil)
			So(canOpen, ShouldBeTrue)
		})

		Convey("non-participants can open private message channel", func(){
			// init account
			account, err := createAccount()
			So(err, ShouldBeNil)
			So(account, ShouldNotBeNil)
			So(account.Id, ShouldNotEqual, 0)

			// init channel
			c := NewChannel()
			c.CreatorId = account.Id
			c.TypeConstant = Channel_TYPE_PRIVATE_MESSAGE

			So(c.Create(), ShouldBeNil)

			canOpen, err := c.CanOpen(account.Id)
			So(err, ShouldBeNil)
			So(canOpen, ShouldBeFalse)
		})

	})

}
<commit_msg>social/tests: created function which creates account & inits channel, required changes has been applied.<commit_after>package models

import (
	"socialapi/workers/common/runner"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

// createNewChannelWithTest creates a new account
// And inits a channel
func createNewChannelWithTest() *Channel {
	c := NewChannel()

	// init account
	account, err := createAccount()
	So(err, ShouldBeNil)
	So(account, ShouldNotBeNil)
	So(account.Id, ShouldNotEqual, 0)

	// init channel
	c := NewChannel()
	// set Creator id
	c.CreatorId = account.Id
	return c
}

func TestChannelCreate(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while creating channel", t, func() {
		Convey("channel name should not be empty", func(){
			c := NewChannel()
			c.Name = ""
			So(c.Create().Error(), ShouldContainSubstring, "Validation failed")
		})

		Convey("channel groupName should not be empty", func(){
			c := NewChannel()
			c.GroupName = ""
			So(c.Create().Error(), ShouldContainSubstring, "Validation failed")
		})

		Convey("channel typeConstant should not be empty", func(){
			c := NewChannel()
			c.TypeConstant = ""
			So(c.Create().Error(), ShouldContainSubstring, "Validation failed")
		})

		Convey("channel name should not contain whitespace", func(){
			c := NewChannel()
			c.Name = "name channel"
			So(c.Create().Error(), ShouldContainSubstring, "has empty space")
		})
	})

}

func TestChannelTableName(t *testing.T) {
	Convey("while testing TableName()", t, func() {
		So(NewChannel().TableName(), ShouldEqual, ChannelTableName)
	})
}

func TestChannelCanOpen(t *testing.T) {
	Convey("while testing channel permissions", t, func() {
		Convey("can not open uninitialized channel", func() {
			c := NewChannel()
			canOpen, err := c.CanOpen(1231)
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, ErrChannelIdIsNotSet)
			So(canOpen, ShouldBeFalse)
		})

		Convey("uninitialized account can not open channel", func() {
			c := NewChannel()
			c.Id = 123
			canOpen, err := c.CanOpen(0)
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, ErrAccountIdIsNotSet)
			So(canOpen, ShouldBeFalse)
		})

		Convey("participants can open group channel", func() {
			c :=createNewChannelWithTest()
			c.TypeConstant = Channel_TYPE_GROUP

			So(c.Create(), ShouldBeNil)

			cp, err := c.AddParticipant(account.Id)
			So(err, ShouldBeNil)
			So(cp, ShouldNotBeNil)

			canOpen, err := c.CanOpen(account.Id)
			So(err, ShouldBeNil)
			So(canOpen, ShouldBeTrue)
		})

		Convey("everyone can open group channel", func() {
			c :=createNewChannelWithTest()
			// set required constant to open chanel
			c.TypeConstant = Channel_TYPE_GROUP

			So(c.Create(), ShouldBeNil)

			// 1 is just a random id
			canOpen, err := c.CanOpen(1)
			So(err, ShouldBeNil)
			So(canOpen, ShouldBeTrue)
		})

		Convey("everyone can open topic channel", func() {
			c :=createNewChannelWithTest()
			c.TypeConstant = Channel_TYPE_TOPIC

			So(c.Create(), ShouldBeNil)

			// 1 is just a random id
			canOpen, err := c.CanOpen(1)
			So(err, ShouldBeNil)
			So(canOpen, ShouldBeTrue)
		})

		Convey("participants can open pinned activity channel", func(){
			c :=createNewChannelWithTest()
			c.TypeConstant = Channel_TYPE_PINNED_ACTIVITY

			So(c.Create(), ShouldBeNil)

			cp, err := c.AddParticipant(account.Id)
			So(err, ShouldBeNil)
			So(cp, ShouldNotBeNil)

			canOpen, err := c.CanOpen(account.Id)
			So(err, ShouldBeNil)
			So(canOpen, ShouldBeFalse)
		})

		Convey("participants can open private message channel", func(){
			c :=createNewChannelWithTest()
			c.TypeConstant = Channel_TYPE_PRIVATE_MESSAGE

			So(c.Create(), ShouldBeNil)

			// add participant to the channel
			cp, err := c.AddParticipant(account.Id)
			So(err, ShouldBeNil)
			So(cp, ShouldNotBeNil)

			canOpen, err := c.CanOpen(account.Id)
			So(err, ShouldBeNil)
			So(canOpen, ShouldBeTrue)
		})

		Convey("non-participants can open private message channel", func(){
			c :=createNewChannelWithTest()
			c.TypeConstant = Channel_TYPE_PRIVATE_MESSAGE

			So(c.Create(), ShouldBeNil)

			canOpen, err := c.CanOpen(account.Id)
			So(err, ShouldBeNil)
			So(canOpen, ShouldBeFalse)
		})

	})

}
<|endoftext|>


<commit_before><commit_msg>Move kubeClient setup to BeforeEach<commit_after><|endoftext|>


<commit_before>// Copyright 2013-2014 Aerospike, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package atomic

import (
	"sync"
)

// AtomicInt implements an int value with atomic semantics
type AtomicInt struct {
	val   int
	mutex sync.RWMutex
}

// NewAtomicInt generates a new AtomicInt instance.
func NewAtomicInt(value int) *AtomicInt {
	return &AtomicInt{
		val: value,
	}
}

// AddAndGet atomically adds the given value to the current value.
func (ai *AtomicInt) AddAndGet(delta int) int {
	ai.mutex.Lock()
	ai.val += delta
	res := ai.val
	ai.mutex.Unlock()
	return res
}

// CompareAndSet atomically sets the value to the given updated value if the current value == expected value.
// Returns true if the expectation was met
func (ai *AtomicInt) CompareAndSet(expect int, update int) bool {
	res := false
	ai.mutex.Lock()
	if ai.val == expect {
		ai.val = update
		res = true
	}
	ai.mutex.Unlock()
	return res
}

// DecrementAndGet atomically decrements current value by one and returns the result.
func (ai *AtomicInt) DecrementAndGet() int {
	ai.mutex.Lock()
	ai.val--
	res := ai.val
	ai.mutex.Unlock()
	return res
}

// Get atomically retrieves the current value.
func (ai *AtomicInt) Get() int {
	ai.mutex.RLock()
	res := ai.val
	ai.mutex.RUnlock()
	return res
}

// GetAndAdd atomically adds the given delta to the current value and returns the result.
func (ai *AtomicInt) GetAndAdd(delta int) int {
	ai.mutex.Lock()
	old := ai.val
	ai.val += delta
	ai.mutex.Unlock()
	return old
}

// GetAndDecrement atomically decrements the current value by one and returns the result.
func (ai *AtomicInt) GetAndDecrement() int {
	ai.mutex.Lock()
	old := ai.val
	ai.val--
	ai.mutex.Unlock()
	return old
}

// GetAndIncrement atomically increments current value by one and returns the result.
func (ai *AtomicInt) GetAndIncrement() int {
	ai.mutex.Lock()
	old := ai.val
	ai.val++
	ai.mutex.Unlock()
	return old
}

// GetAndSet atomically sets current value to the given value and returns the old value.
func (ai *AtomicInt) GetAndSet(newValue int) int {
	ai.mutex.Lock()
	old := ai.val
	ai.val = newValue
	ai.mutex.Unlock()
	return old
}

// IncrementAndGet atomically increments current value by one and returns the result.
func (ai *AtomicInt) IncrementAndGet() int {
	ai.mutex.Lock()
	ai.val++
	res := ai.val
	ai.mutex.Unlock()
	return res
}

// Set atomically sets current value to the given value.
func (ai *AtomicInt) Set(newValue int) {
	ai.mutex.Lock()
	ai.val = newValue
	ai.mutex.Unlock()
}
<commit_msg>use atomic aps instead of locks<commit_after>// Copyright 2013-2014 Aerospike, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package atomic

import "sync/atomic"

// AtomicInt implements an int value with atomic semantics
type AtomicInt struct {
	val int64
}

// NewAtomicInt generates a newVal AtomicInt instance.
func NewAtomicInt(value int) *AtomicInt {
	return &AtomicInt{
		val: int64(value),
	}
}

// AddAndGet atomically adds the given value to the current value.
func (ai *AtomicInt) AddAndGet(delta int) int {
	return int(atomic.AddInt64(&ai.val, int64(delta)))
}

// CompareAndSet atomically sets the value to the given updated value if the current value == expected value.
// Returns true if the expectation was met
func (ai *AtomicInt) CompareAndSet(expect int, update int) bool {
	return atomic.CompareAndSwapInt64(&ai.val, int64(expect), int64(update))
}

// DecrementAndGet atomically decrements current value by one and returns the result.
func (ai *AtomicInt) DecrementAndGet() int {
	return int(atomic.AddInt64(&ai.val, -1))
}

// Get atomically retrieves the current value.
func (ai *AtomicInt) Get() int {
	return int(atomic.LoadInt64(&ai.val))
}

// GetAndAdd atomically adds the given delta to the current value and returns the result.
func (ai *AtomicInt) GetAndAdd(delta int) int {
	newVal := atomic.AddInt64(&ai.val, int64(delta))
	return int(newVal - int64(delta))
}

// GetAndDecrement atomically decrements the current value by one and returns the result.
func (ai *AtomicInt) GetAndDecrement() int {
	newVal := atomic.AddInt64(&ai.val, -1)
	return int(newVal + 1)
}

// GetAndIncrement atomically increments current value by one and returns the result.
func (ai *AtomicInt) GetAndIncrement() int {
	newVal := atomic.AddInt64(&ai.val, 1)
	return int(newVal - 1)
}

// GetAndSet atomically sets current value to the given value and returns the old value.
func (ai *AtomicInt) GetAndSet(newValue int) int {
	return int(atomic.SwapInt64(&ai.val, int64(newValue)))
}

// IncrementAndGet atomically increments current value by one and returns the result.
func (ai *AtomicInt) IncrementAndGet() int {
	return int(atomic.AddInt64(&ai.val, 1))
}

// Set atomically sets current value to the given value.
func (ai *AtomicInt) Set(newValue int) {
	atomic.StoreInt64(&ai.val, int64(newValue))
}
<|endoftext|>


<commit_before>package tsm1_test

import (
	"fmt"
	"math/rand"
	"reflect"
	"runtime"
	"sort"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/influxdata/influxdb/v2/tsdb/tsm1"
)

func TestCacheCheckConcurrentReadsAreSafe(t *testing.T) {
	values := make(tsm1.Values, 1000)
	timestamps := make([]int64, len(values))
	series := make([][]byte, 100)
	for i := range timestamps {
		timestamps[i] = int64(rand.Int63n(int64(len(values))))
	}

	for i := range values {
		values[i] = tsm1.NewValue(timestamps[i*len(timestamps)/len(values)], float64(i))
	}

	for i := range series {
		series[i] = []byte(fmt.Sprintf("series%d", i))
	}

	wg := sync.WaitGroup{}
	c := tsm1.NewCache(1000000)

	ch := make(chan struct{})
	for _, s := range series {
		for _, v := range values {
			c.Write(s, tsm1.Values{v})
		}
		wg.Add(3)
		go func(s []byte) {
			defer wg.Done()
			<-ch
			c.Values(s)
		}(s)
		go func(s []byte) {
			defer wg.Done()
			<-ch
			c.Values(s)
		}(s)
		go func(s []byte) {
			defer wg.Done()
			<-ch
			c.Values(s)
		}(s)
	}
	close(ch)
	wg.Wait()
}

func TestCacheRace(t *testing.T) {
	values := make(tsm1.Values, 1000)
	timestamps := make([]int64, len(values))
	series := make([][]byte, 100)
	for i := range timestamps {
		timestamps[i] = int64(rand.Int63n(int64(len(values))))
	}

	for i := range values {
		values[i] = tsm1.NewValue(timestamps[i*len(timestamps)/len(values)], float64(i))
	}

	for i := range series {
		series[i] = []byte(fmt.Sprintf("series%d", i))
	}

	wg := sync.WaitGroup{}
	c := tsm1.NewCache(1000000)

	ch := make(chan struct{})
	for _, s := range series {
		for _, v := range values {
			c.Write(s, tsm1.Values{v})
		}
		wg.Add(1)
		go func(s []byte) {
			defer wg.Done()
			<-ch
			c.Values(s)
		}(s)
	}

	errC := make(chan error)
	wg.Add(1)
	go func() {
		defer wg.Done()
		<-ch
		s, err := c.Snapshot()
		if err == tsm1.ErrSnapshotInProgress {
			return
		}

		if err != nil {
			errC <- fmt.Errorf("failed to snapshot cache: %v", err)
			return
		}

		s.Deduplicate()
		c.ClearSnapshot(true)
	}()

	close(ch)

	go func() {
		wg.Wait()
		close(errC)
	}()

	for err := range errC {
		if err != nil {
			t.Error(err)
		}
	}
}

func TestCacheRace2Compacters(t *testing.T) {
	values := make(tsm1.Values, 1000)
	timestamps := make([]int64, len(values))
	series := make([][]byte, 100)
	for i := range timestamps {
		timestamps[i] = int64(rand.Int63n(int64(len(values))))
	}

	for i := range values {
		values[i] = tsm1.NewValue(timestamps[i*len(timestamps)/len(values)], float64(i))
	}

	for i := range series {
		series[i] = []byte(fmt.Sprintf("series%d", i))
	}

	wg := sync.WaitGroup{}
	c := tsm1.NewCache(1000000)

	ch := make(chan struct{})
	for _, s := range series {
		for _, v := range values {
			c.Write(s, tsm1.Values{v})
		}
		wg.Add(1)
		go func(s []byte) {
			defer wg.Done()
			<-ch
			c.Values(s)
		}(s)
	}
	fileCounter := 0
	mapFiles := map[int]bool{}
	mu := sync.Mutex{}
	errC := make(chan error)
	for i := 0; i < 2; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			<-ch
			s, err := c.Snapshot()
			if err == tsm1.ErrSnapshotInProgress {
				return
			}

			if err != nil {
				errC <- fmt.Errorf("failed to snapshot cache: %v", err)
				return
			}

			mu.Lock()
			mapFiles[fileCounter] = true
			fileCounter++
			myFiles := map[int]bool{}
			for k, e := range mapFiles {
				myFiles[k] = e
			}
			mu.Unlock()
			s.Deduplicate()
			c.ClearSnapshot(true)
			mu.Lock()
			defer mu.Unlock()
			for k := range myFiles {
				if _, ok := mapFiles[k]; !ok {
					errC <- fmt.Errorf("something else deleted one of my files")
					return
				} else {
					delete(mapFiles, k)
				}
			}
		}()
	}
	close(ch)

	go func() {
		wg.Wait()
		close(errC)
	}()

	for err := range errC {
		if err != nil {
			t.Error(err)
		}
	}
}

func TestConcurrentReadAfterWrite(t *testing.T) {
	t.Parallel()

	var starttime int64 = 1594785691
	series := [][]byte{[]byte("key1"), []byte("key2")}

	concurrency := runtime.GOMAXPROCS(0) * 2
	batch := 1024

	errCh := make(chan error, concurrency)
	closing := make(chan struct{})
	var wg sync.WaitGroup

	c := tsm1.NewCache(1024 * 1024 * 16)
	for i := 0; i < concurrency; i++ {
		wg.Add(1)
		// read after read concurrently
		go func() {
			defer wg.Done()
			for {

				select {
				case <-closing:
					errCh <- nil
					return
				default:
				}

				ts := atomic.AddInt64(&starttime, int64(batch))
				writes := make(tsm1.Values, 0, batch)
				for j := 0; j < batch; j++ {
					writes = append(writes,
						tsm1.NewValue(ts+int64(j), ts+int64(j)))
				}
				for _, key := range series {
					if err := c.Write(key, writes); err != nil {
						errCh <- err
						return
					}
				}
				for _, key := range series {
					// check the read result
					reads := c.Values(key)

					if len(reads) < len(writes) {
						errCh <- fmt.Errorf("read count: %v less than write count: %v", len(reads), len(writes))
						return
					}

					sort.Slice(reads, func(i, j int) bool {
						return reads[i].UnixNano() < reads[j].UnixNano()
					})

					k := 0
					for j := range writes {
						write := writes[j].Value()

						found := false
						for k < len(reads) {
							read := reads[k].Value()
							if reflect.DeepEqual(read, write) {
								found = true
								break
							}
							k++
						}

						if !found {
							errCh <- fmt.Errorf("write value: %v not found in reads", write)
							return
						}
					}
				}
			}
		}()
	}

	// sleep for a little while and check
	time.Sleep(time.Second * 20)
	close(closing)
	wg.Wait()

	for i := 0; i < concurrency; i++ {
		err := <-errCh
		if err != nil {
			t.Fatal(err)
			return
		}
	}
}
<commit_msg>test: set bigger max size of cache in TestConcurrentReadAfterWrite<commit_after>package tsm1_test

import (
	"fmt"
	"math/rand"
	"reflect"
	"runtime"
	"sort"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/influxdata/influxdb/v2/tsdb/tsm1"
)

func TestCacheCheckConcurrentReadsAreSafe(t *testing.T) {
	values := make(tsm1.Values, 1000)
	timestamps := make([]int64, len(values))
	series := make([][]byte, 100)
	for i := range timestamps {
		timestamps[i] = int64(rand.Int63n(int64(len(values))))
	}

	for i := range values {
		values[i] = tsm1.NewValue(timestamps[i*len(timestamps)/len(values)], float64(i))
	}

	for i := range series {
		series[i] = []byte(fmt.Sprintf("series%d", i))
	}

	wg := sync.WaitGroup{}
	c := tsm1.NewCache(1000000)

	ch := make(chan struct{})
	for _, s := range series {
		for _, v := range values {
			c.Write(s, tsm1.Values{v})
		}
		wg.Add(3)
		go func(s []byte) {
			defer wg.Done()
			<-ch
			c.Values(s)
		}(s)
		go func(s []byte) {
			defer wg.Done()
			<-ch
			c.Values(s)
		}(s)
		go func(s []byte) {
			defer wg.Done()
			<-ch
			c.Values(s)
		}(s)
	}
	close(ch)
	wg.Wait()
}

func TestCacheRace(t *testing.T) {
	values := make(tsm1.Values, 1000)
	timestamps := make([]int64, len(values))
	series := make([][]byte, 100)
	for i := range timestamps {
		timestamps[i] = int64(rand.Int63n(int64(len(values))))
	}

	for i := range values {
		values[i] = tsm1.NewValue(timestamps[i*len(timestamps)/len(values)], float64(i))
	}

	for i := range series {
		series[i] = []byte(fmt.Sprintf("series%d", i))
	}

	wg := sync.WaitGroup{}
	c := tsm1.NewCache(1000000)

	ch := make(chan struct{})
	for _, s := range series {
		for _, v := range values {
			c.Write(s, tsm1.Values{v})
		}
		wg.Add(1)
		go func(s []byte) {
			defer wg.Done()
			<-ch
			c.Values(s)
		}(s)
	}

	errC := make(chan error)
	wg.Add(1)
	go func() {
		defer wg.Done()
		<-ch
		s, err := c.Snapshot()
		if err == tsm1.ErrSnapshotInProgress {
			return
		}

		if err != nil {
			errC <- fmt.Errorf("failed to snapshot cache: %v", err)
			return
		}

		s.Deduplicate()
		c.ClearSnapshot(true)
	}()

	close(ch)

	go func() {
		wg.Wait()
		close(errC)
	}()

	for err := range errC {
		if err != nil {
			t.Error(err)
		}
	}
}

func TestCacheRace2Compacters(t *testing.T) {
	values := make(tsm1.Values, 1000)
	timestamps := make([]int64, len(values))
	series := make([][]byte, 100)
	for i := range timestamps {
		timestamps[i] = int64(rand.Int63n(int64(len(values))))
	}

	for i := range values {
		values[i] = tsm1.NewValue(timestamps[i*len(timestamps)/len(values)], float64(i))
	}

	for i := range series {
		series[i] = []byte(fmt.Sprintf("series%d", i))
	}

	wg := sync.WaitGroup{}
	c := tsm1.NewCache(1000000)

	ch := make(chan struct{})
	for _, s := range series {
		for _, v := range values {
			c.Write(s, tsm1.Values{v})
		}
		wg.Add(1)
		go func(s []byte) {
			defer wg.Done()
			<-ch
			c.Values(s)
		}(s)
	}
	fileCounter := 0
	mapFiles := map[int]bool{}
	mu := sync.Mutex{}
	errC := make(chan error)
	for i := 0; i < 2; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			<-ch
			s, err := c.Snapshot()
			if err == tsm1.ErrSnapshotInProgress {
				return
			}

			if err != nil {
				errC <- fmt.Errorf("failed to snapshot cache: %v", err)
				return
			}

			mu.Lock()
			mapFiles[fileCounter] = true
			fileCounter++
			myFiles := map[int]bool{}
			for k, e := range mapFiles {
				myFiles[k] = e
			}
			mu.Unlock()
			s.Deduplicate()
			c.ClearSnapshot(true)
			mu.Lock()
			defer mu.Unlock()
			for k := range myFiles {
				if _, ok := mapFiles[k]; !ok {
					errC <- fmt.Errorf("something else deleted one of my files")
					return
				} else {
					delete(mapFiles, k)
				}
			}
		}()
	}
	close(ch)

	go func() {
		wg.Wait()
		close(errC)
	}()

	for err := range errC {
		if err != nil {
			t.Error(err)
		}
	}
}

func TestConcurrentReadAfterWrite(t *testing.T) {
	t.Parallel()

	var starttime int64 = 1594785691
	series := [][]byte{[]byte("key1"), []byte("key2")}

	concurrency := runtime.GOMAXPROCS(0) * 2
	batch := 1024

	errCh := make(chan error, concurrency)
	closing := make(chan struct{})
	var wg sync.WaitGroup

	c := tsm1.NewCache(1024 * 1024 * 128)
	for i := 0; i < concurrency; i++ {
		wg.Add(1)
		// read after read concurrently
		go func() {
			defer wg.Done()
			for {

				select {
				case <-closing:
					errCh <- nil
					return
				default:
				}

				ts := atomic.AddInt64(&starttime, int64(batch))
				writes := make(tsm1.Values, 0, batch)
				for j := 0; j < batch; j++ {
					writes = append(writes,
						tsm1.NewValue(ts+int64(j), ts+int64(j)))
				}
				for _, key := range series {
					if err := c.Write(key, writes); err != nil {
						errCh <- err
						return
					}
				}
				for _, key := range series {
					// check the read result
					reads := c.Values(key)

					if len(reads) < len(writes) {
						errCh <- fmt.Errorf("read count: %v less than write count: %v", len(reads), len(writes))
						return
					}

					sort.Slice(reads, func(i, j int) bool {
						return reads[i].UnixNano() < reads[j].UnixNano()
					})

					k := 0
					for j := range writes {
						write := writes[j].Value()

						found := false
						for k < len(reads) {
							read := reads[k].Value()
							if reflect.DeepEqual(read, write) {
								found = true
								break
							}
							k++
						}

						if !found {
							errCh <- fmt.Errorf("write value: %v not found in reads", write)
							return
						}
					}
				}
			}
		}()
	}

	// sleep for a little while and check
	time.Sleep(time.Second * 20)
	close(closing)
	wg.Wait()

	for i := 0; i < concurrency; i++ {
		err := <-errCh
		if err != nil {
			t.Fatal(err)
			return
		}
	}
}
<|endoftext|>


<commit_before>package runtimectl

import (
	"errors"
	"github.com/graniticio/granitic/ctl"
	"github.com/graniticio/granitic/instance"
	"github.com/graniticio/granitic/ioc"
	"github.com/graniticio/granitic/logging"
	"github.com/graniticio/granitic/ws"
	"strconv"
	"strings"
)

const (
	compCommandName = "components"
	compSummary     = "Show a list of the names of components managed by the IoC container."
	compUsage       = "components [-fw true]"
	compHelp        = "Lists the name of all of the user-defined components currently present in the IoC Container."
	compHelpTwo     = "If the '-fw true' argument is supplied, the list will show built-in Granitic framework components instead of user-defined components."
	fwArg           = "fw"
)

type ComponentsCommand struct {
	FrameworkLogger logging.Logger
	container       *ioc.ComponentContainer
}

func (c *ComponentsCommand) Container(container *ioc.ComponentContainer) {
	c.container = container
}

func (c *ComponentsCommand) ExecuteCommand(qualifiers []string, args map[string]string) (*ctl.CommandOutcome, []*ws.CategorisedError) {

	var frameworkOnly bool
	var err error

	if frameworkOnly, err = showBuiltin(args); err != nil {
		return nil, []*ws.CategorisedError{ctl.NewCommandClientError(err.Error())}
	}

	names := make([][]string, 0)

	for _, c := range c.container.AllComponents() {

		fw := strings.HasPrefix(c.Name, instance.FrameworkPrefix)

		if (fw && frameworkOnly) || (!fw && !frameworkOnly) {
			names = append(names, []string{c.Name})
		}
	}

	co := new(ctl.CommandOutcome)
	co.OutputBody = names
	co.RenderHint = ctl.Columns

	return co, nil
}

func showBuiltin(args map[string]string) (bool, error) {

	if args == nil || len(args) == 0 {
		return false, nil
	}

	for k, v := range args {

		if k != fwArg {
			return false, errors.New("fw is the only argument supported by the components command.")
		}

		if choice, err := strconv.ParseBool(v); err == nil {
			return choice, nil
		} else {
			return false, errors.New("Value of fw argument cannot be interpreted as a bool")
		}

	}

	return false, nil

}

func (c *ComponentsCommand) Name() string {
	return compCommandName
}

func (c *ComponentsCommand) Summmary() string {
	return compSummary
}

func (c *ComponentsCommand) Usage() string {
	return compUsage
}

func (c *ComponentsCommand) Help() []string {
	return []string{compHelp, compHelpTwo}
}
<commit_msg>grnc-ctl tool<commit_after>package runtimectl

import (
	"errors"
	"fmt"
	"github.com/graniticio/granitic/ctl"
	"github.com/graniticio/granitic/instance"
	"github.com/graniticio/granitic/ioc"
	"github.com/graniticio/granitic/logging"
	"github.com/graniticio/granitic/ws"
	"strconv"
	"strings"
)

const (
	compCommandName = "components"
	compSummary     = "Show a list of the names of components managed by the IoC container."
	compUsage       = "components [-fw true] [-lc start|stop|suspend]"
	compHelp        = "Lists the name of all of the user-defined components currently present in the IoC Container."
	compHelpTwo     = "If the '-fw true' argument is supplied, the list will show built-in Granitic framework components instead of user-defined components."
	compHelpThree   = "If the '-lc true' argument is supplied with one of the values start/stop/suspend then only those components that implement the corresponding " +
		"lifecycle interface (ioc.Startable, ioc.Stoppable, ioc.Suspendable) will be displayed"
	fwArg = "fw"
	lcArg = "lc"
)

type lifecycleFilter int

const (
	all = iota
	stop
	start
	suspend
)

func fromFilterArg(arg string) (lifecycleFilter, error) {

	s := strings.ToLower(arg)

	switch s {
	case "", "all":
		return all, nil
	case "stop":
		return stop, nil
	case "start":
		return start, nil
	case "suspend":
		return suspend, nil
	}

	m := fmt.Sprintf("%s is not a recognised lifecycle filter (all, stop, start, suspend)", arg)

	return all, errors.New(m)

}

type ComponentsCommand struct {
	FrameworkLogger logging.Logger
	container       *ioc.ComponentContainer
}

func (c *ComponentsCommand) Container(container *ioc.ComponentContainer) {
	c.container = container
}

func (c *ComponentsCommand) ExecuteCommand(qualifiers []string, args map[string]string) (*ctl.CommandOutcome, []*ws.CategorisedError) {

	var frameworkOnly bool
	var lcFilter lifecycleFilter
	var err error

	if frameworkOnly, err = showBuiltin(args); err != nil {
		return nil, []*ws.CategorisedError{ctl.NewCommandClientError(err.Error())}
	}

	if lcFilter, err = findLifecycleFilter(args); err != nil {
		return nil, []*ws.CategorisedError{ctl.NewCommandClientError(err.Error())}
	}

	names := make([][]string, 0)

	for _, c := range c.container.AllComponents() {

		fw := strings.HasPrefix(c.Name, instance.FrameworkPrefix)

		if ((fw && frameworkOnly) || (!fw && !frameworkOnly)) && matchesFilter(lcFilter, c.Instance) {
			names = append(names, []string{c.Name})
		}
	}

	co := new(ctl.CommandOutcome)
	co.OutputBody = names
	co.RenderHint = ctl.Columns

	return co, nil
}

func matchesFilter(f lifecycleFilter, i interface{}) bool {

	switch f {
	case all:
		return true

	case start:
		_, found := i.(ioc.Startable)
		return found

	case stop:
		_, found := i.(ioc.Stoppable)
		return found

	case suspend:
		_, found := i.(ioc.Suspendable)
		return found

	}

	return true
}

func findLifecycleFilter(args map[string]string) (lifecycleFilter, error) {

	if args == nil || len(args) == 0 || args[lcArg] == "" {
		return all, nil
	}

	v := args[lcArg]

	return fromFilterArg(v)

}

func showBuiltin(args map[string]string) (bool, error) {

	if args == nil || len(args) == 0 || args[fwArg] == "" {
		return false, nil
	}

	v := args[fwArg]

	if choice, err := strconv.ParseBool(v); err == nil {
		return choice, nil
	} else {
		return false, errors.New("Value of fw argument cannot be interpreted as a bool")
	}

	return false, nil

}

func (c *ComponentsCommand) Name() string {
	return compCommandName
}

func (c *ComponentsCommand) Summmary() string {
	return compSummary
}

func (c *ComponentsCommand) Usage() string {
	return compUsage
}

func (c *ComponentsCommand) Help() []string {
	return []string{compHelp, compHelpTwo, compHelpThree}
}
<|endoftext|>


<commit_before>package ttl_hash_set

import (
	"time"

	"github.com/fzzy/radix/redis"
)

type TTLHashSet struct {
	client *redis.Client
	prefix string
}

func NewTTLHashSet(prefix string, address string) (*TTLHashSet, error) {
	client, err := redis.Dial("tcp", address)
	if err != nil {
		return nil, err
	}

	return &TTLHashSet{
		client: client,
		prefix: prefix,
	}, nil
}

func (t *TTLHashSet) Add(key string) (bool, error) {
	localKey := prefixKey(t.prefix, key)

	// Use pipelining to set the key and set expiry in one go.
	t.client.Append("SET", localKey, 1)
	t.client.Append("EXPIRE", localKey, (24 * time.Hour).Seconds())

	return t.client.GetReply().Bool()
}

func (t *TTLHashSet) Close() error {
	return t.client.Close()
}

func (t *TTLHashSet) Exists(key string) (bool, error) {
	localKey := prefixKey(t.prefix, key)
	return t.client.Cmd("EXISTS", localKey).Bool()
}

func (t *TTLHashSet) TTL(key string) (int, error) {
	localKey := prefixKey(t.prefix, key)
	return t.client.Cmd("TTL", localKey).Int()
}

func prefixKey(prefix string, key string) string {
	return prefix + ":" + key
}
<commit_msg>Lock operations that happen with a single TTLHashSet object<commit_after>package ttl_hash_set

import (
	"sync"
	"time"

	"github.com/fzzy/radix/redis"
)

type TTLHashSet struct {
	client *redis.Client
	mutex  sync.Mutex
	prefix string
}

func NewTTLHashSet(prefix string, address string) (*TTLHashSet, error) {
	client, err := redis.Dial("tcp", address)
	if err != nil {
		return nil, err
	}

	return &TTLHashSet{
		client: client,
		prefix: prefix,
	}, nil
}

func (t *TTLHashSet) Add(key string) (bool, error) {
	localKey := prefixKey(t.prefix, key)

	// Use pipelining to set the key and set expiry in one go.
	t.mutex.Lock()
	t.client.Append("SET", localKey, 1)
	t.client.Append("EXPIRE", localKey, (24 * time.Hour).Seconds())
	add, err := t.client.GetReply().Bool()
	t.mutex.Unlock()

	return add, err
}

func (t *TTLHashSet) Close() error {
	t.mutex.Lock()
	err := t.client.Close()
	t.mutex.Unlock()

	return err
}

func (t *TTLHashSet) Exists(key string) (bool, error) {
	localKey := prefixKey(t.prefix, key)

	t.mutex.Lock()
	exists, err := t.client.Cmd("EXISTS", localKey).Bool()
	t.mutex.Unlock()

	return exists, err
}

func (t *TTLHashSet) TTL(key string) (int, error) {
	localKey := prefixKey(t.prefix, key)

	t.mutex.Lock()
	ttl, err := t.client.Cmd("TTL", localKey).Int()
	t.mutex.Unlock()

	return ttl, err
}

func prefixKey(prefix string, key string) string {
	return prefix + ":" + key
}
<|endoftext|>