<commit_before>package cmd

import (
	"fmt"

	"github.com/freneticmonkey/migrate/migrate/git"
	"github.com/freneticmonkey/migrate/migrate/id"
	"github.com/freneticmonkey/migrate/migrate/migration"
	"github.com/freneticmonkey/migrate/migrate/mysql"
	"github.com/freneticmonkey/migrate/migrate/table"
	"github.com/freneticmonkey/migrate/migrate/util"
	"github.com/freneticmonkey/migrate/migrate/yaml"
	"github.com/urfave/cli"
)

// GetCreateCommand Create a new migration for the target project at the version indicated by hash
func GetCreateCommand() (setup cli.Command) {
	setup = cli.Command{
		Name:  "create",
		Usage: "This subcommand is used to create a migration and register it with the management database.",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "project",
				Value: "",
				Usage: "The target project",
			},
			cli.StringFlag{
				Name:  "version",
				Value: "",
				Usage: "The target git version",
			},
			cli.BoolFlag{
				Name:  "rollback",
				Usage: "Allows for a rollback to be created",
			},
		},
		Action: func(ctx *cli.Context) error {
			var problems int
			var ts string
			var info string
			rollback := false

			// Setup the management database and configuration settings
			configureManagement(ctx)

			// Override the project settings with the command line flags
			if ctx.IsSet("project") {
				conf.Project.Name = ctx.String("project")
			}

			if ctx.IsSet("version") {
				conf.Project.Version = ctx.String("version")
			}

			// if the version hasn't been defined
			if len(conf.Project.Name) == 0 {
				return cli.NewExitError("Creation failed.  Unable to generate a migration as no project was defined", 1)
			} else if len(conf.Project.Version) == 0 {
				return cli.NewExitError("Creation failed.  Unable to generate a migration as no version was defined to migrate to", 1)
			} else {
				git.Clone(conf.Project)
			}

			// Read the YAML files cloned from the repo
			err := yaml.ReadTables(conf.Options.WorkingPath)
			if util.ErrorCheck(err) {
				return cli.NewExitError("Creation failed. Unable to read YAML Tables", 1)
			}
			problems, err = id.ValidateSchema(yaml.Schema, "YAML Schema")
			if util.ErrorCheck(err) {
				return cli.NewExitError("Creation failed. YAML Validation Errors Detected", problems)
			}

			// Read the MySQL tables from the target database
			err = mysql.ReadTables(conf.Project)
			if util.ErrorCheck(err) {
				return cli.NewExitError("Creation failed. Unable to read MySQL Tables", 1)
			}
			problems, err = id.ValidateSchema(mysql.Schema, "Target Database Schema")
			if util.ErrorCheck(err) {
				return cli.NewExitError("Creation failed. Target Database Validation Errors Detected", problems)
			}

			forwardDiff := table.DiffTables(yaml.Schema, mysql.Schema)
			forwardOps := mysql.GenerateAlters(forwardDiff)

			backwardDiff := table.DiffTables(mysql.Schema, yaml.Schema)
			backwardOps := mysql.GenerateAlters(backwardDiff)

			ts, err = git.GetVersionTime(conf.Project.Name, conf.Project.Version)
			if util.ErrorCheck(err) {
				return cli.NewExitError("Create failed. Unable to obtain Version Timestamp from Git checkout", 1)
			}
			info, err = git.GetVersionDetails(conf.Project.Name, conf.Project.Version)
			if util.ErrorCheck(err) {
				return cli.NewExitError("Create failed. Unable to obtain Version Details from Git checkout", 1)
			}

			m, err := migration.New(migration.Param{
				Project:     conf.Project.Name,
				Version:     conf.Project.Version,
				Timestamp:   ts,
				Description: info,
				Forwards:    forwardOps,
				Backwards:   backwardOps,
				Rollback:    rollback,
			})
			if util.ErrorCheck(err) {
				return cli.NewExitError("Create failed. Unable to create new Migration in the management database", 1)
			}

			success := fmt.Sprintf("Created Migration successfully with ID: [%d]", m.MID)

			return cli.NewExitError(success, 0)
		},
	}
	return setup
}
<commit_msg>Improved create rollback command line flag help text<commit_after>package cmd

import (
	"fmt"

	"github.com/freneticmonkey/migrate/migrate/git"
	"github.com/freneticmonkey/migrate/migrate/id"
	"github.com/freneticmonkey/migrate/migrate/migration"
	"github.com/freneticmonkey/migrate/migrate/mysql"
	"github.com/freneticmonkey/migrate/migrate/table"
	"github.com/freneticmonkey/migrate/migrate/util"
	"github.com/freneticmonkey/migrate/migrate/yaml"
	"github.com/urfave/cli"
)

// GetCreateCommand Create a new migration for the target project at the version indicated by hash
func GetCreateCommand() (setup cli.Command) {
	setup = cli.Command{
		Name:  "create",
		Usage: "This subcommand is used to create a migration and register it with the management database.",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "project",
				Value: "",
				Usage: "The target project",
			},
			cli.StringFlag{
				Name:  "version",
				Value: "",
				Usage: "The target git version",
			},
			cli.BoolFlag{
				Name:  "rollback",
				Usage: "Force a rollback (backward) migration to be created",
			},
		},
		Action: func(ctx *cli.Context) error {
			var problems int
			var ts string
			var info string
			rollback := false

			// Setup the management database and configuration settings
			configureManagement(ctx)

			// Override the project settings with the command line flags
			if ctx.IsSet("project") {
				conf.Project.Name = ctx.String("project")
			}

			if ctx.IsSet("version") {
				conf.Project.Version = ctx.String("version")
			}

			// if the version hasn't been defined
			if len(conf.Project.Name) == 0 {
				return cli.NewExitError("Creation failed.  Unable to generate a migration as no project was defined", 1)
			} else if len(conf.Project.Version) == 0 {
				return cli.NewExitError("Creation failed.  Unable to generate a migration as no version was defined to migrate to", 1)
			} else {
				git.Clone(conf.Project)
			}

			// Read the YAML files cloned from the repo
			err := yaml.ReadTables(conf.Options.WorkingPath)
			if util.ErrorCheck(err) {
				return cli.NewExitError("Creation failed. Unable to read YAML Tables", 1)
			}
			problems, err = id.ValidateSchema(yaml.Schema, "YAML Schema")
			if util.ErrorCheck(err) {
				return cli.NewExitError("Creation failed. YAML Validation Errors Detected", problems)
			}

			// Read the MySQL tables from the target database
			err = mysql.ReadTables(conf.Project)
			if util.ErrorCheck(err) {
				return cli.NewExitError("Creation failed. Unable to read MySQL Tables", 1)
			}
			problems, err = id.ValidateSchema(mysql.Schema, "Target Database Schema")
			if util.ErrorCheck(err) {
				return cli.NewExitError("Creation failed. Target Database Validation Errors Detected", problems)
			}

			forwardDiff := table.DiffTables(yaml.Schema, mysql.Schema)
			forwardOps := mysql.GenerateAlters(forwardDiff)

			backwardDiff := table.DiffTables(mysql.Schema, yaml.Schema)
			backwardOps := mysql.GenerateAlters(backwardDiff)

			ts, err = git.GetVersionTime(conf.Project.Name, conf.Project.Version)
			if util.ErrorCheck(err) {
				return cli.NewExitError("Create failed. Unable to obtain Version Timestamp from Git checkout", 1)
			}
			info, err = git.GetVersionDetails(conf.Project.Name, conf.Project.Version)
			if util.ErrorCheck(err) {
				return cli.NewExitError("Create failed. Unable to obtain Version Details from Git checkout", 1)
			}

			m, err := migration.New(migration.Param{
				Project:     conf.Project.Name,
				Version:     conf.Project.Version,
				Timestamp:   ts,
				Description: info,
				Forwards:    forwardOps,
				Backwards:   backwardOps,
				Rollback:    rollback,
			})
			if util.ErrorCheck(err) {
				return cli.NewExitError("Create failed. Unable to create new Migration in the management database", 1)
			}

			success := fmt.Sprintf("Created Migration successfully with ID: [%d]", m.MID)

			return cli.NewExitError(success, 0)
		},
	}
	return setup
}
<|endoftext|>


<commit_before>// Package eval implements an evaluator of gigue.
package eval

import (
	"errors"
	"fmt"
	"github.com/suzuken/gigue/lexer"
	"github.com/suzuken/gigue/parser"
	"github.com/suzuken/gigue/types"
	"io"
	"os"
	"path/filepath"
	"strings"
)

// evalDefine evaluate (define ...) style expression.
func evalDefine(exps []types.Expression, env *Env) (types.Expression, error) {
	if len(exps) < 2 {
		return nil, errors.New("define clause must have symbol and body")
	}
	switch tt := exps[1].(type) {
	// put symbol and variables
	// (define x 1) style definition
	case types.Symbol:
		value, err := Eval(exps[2], env)
		if err != nil {
			return nil, err
		}
		env.Put(tt, value)
		return nil, nil
	// (define (hoge args) (..)) style definition
	// above style is syntax sugar for lambda.
	case []types.Expression:
		if len(tt) < 2 {
			return nil, errors.New("define statament must have more than 2 words")
		}
		caddr, ok := tt[0].(types.Symbol)
		if !ok {
			return nil, errors.New("(define x) of x should be symbol")
		}
		// create lambda and put it into environment
		env.Put(caddr, Lambda{tt[1:], exps[2], env})
		return nil, nil
	default:
		return nil, nil
	}
}

// evalIf evaluates if-clause.
// predicate consequent alternative
// like, (if (ok? yeah) (go) (not go))
func evalIf(predicate, consequent, alternative types.Expression, env *Env) (types.Expression, error) {
	bb, err := evalPredicate(predicate, env)
	if err != nil {
		return nil, err
	}
	if bb {
		return Eval(consequent, env)
	}
	return Eval(alternative, env)
}

// evalPredicate is helper for evaluate expression should be boolean.
func evalPredicate(exp types.Expression, env *Env) (types.Boolean, error) {
	b, err := Eval(exp, env)
	if err != nil {
		return false, err
	}
	bb, ok := b.(types.Boolean)
	if !ok {
		return false, fmt.Errorf("the expression should return types.Boolean, exps: %v", exp)
	}
	return bb, nil
}

func evalCond(exps []types.Expression, env *Env) (types.Expression, error) {
	for _, operand := range exps[1:] {
		tt, ok := operand.([]types.Expression)
		if !ok {
			return nil, errors.New("cond clause must have expression")
		}
		if tt[0] == types.Symbol("else") {
			return Eval(tt[1], env)
		}
		if bb, err := evalPredicate(tt[0], env); err != nil {
			return nil, err
		} else if bb {
			return Eval(tt[1], env)
		}
	}
	// unreachable
	return nil, nil
}

func evalBegin(env *Env, exps ...types.Expression) (types.Expression, error) {
	var lastExp types.Expression
	for _, beginExp := range exps {
		l, err := Eval(beginExp, env)
		if err != nil {
			return nil, err
		}
		lastExp = l
	}
	return lastExp, nil
}

// evalLoad evaluates (load "file.scm") style definition.
// loading file and evaluate it.
func evalLoad(path string, env *Env) (types.Expression, error) {
	current, err := env.Get("#current-load-path")
	if err != nil {
		return nil, err
	}
	// if path is set, search from current directory
	if p := fmt.Sprintf("%s", current); p != "" {
		// if path start with /, deal as absolute path
		// if not, deal as relative path
		if !strings.HasPrefix(path, "/") {
			path = filepath.Join(filepath.Dir(p), path)
		}
	}
	abs, err := filepath.Abs(path)
	if err != nil {
		return nil, err
	}
	// evaluate file in given environment.
	return EvalFile(abs, env)
}

func evalApplication(env *Env, operator types.Expression, operands ...types.Expression) (types.Expression, error) {
	// extend environment
	exps := make([]types.Expression, 0, len(operands)-1)
	for _, operand := range operands {
		exp, err := Eval(operand, env)
		if err != nil {
			return nil, err
		}
		exps = append(exps, exp)
	}
	// maybe, it is primitive procedure or compound procedure.
	fn, err := Eval(operator, env)
	if err != nil {
		return nil, err
	}
	return Apply(fn, exps)
}

// Eval is body of evaluator
func Eval(exp types.Expression, env *Env) (types.Expression, error) {
	switch t := exp.(type) {
	case types.Boolean, types.Number, *types.Pair, string:
		return t, nil
	case types.Symbol:
		// it's variable or expression. get value from environment
		e, err := env.Get(t)
		if err != nil {
			return nil, err
		}
		return e, nil
	case []types.Expression:
		if len(t) == 0 {
			return &types.Pair{}, nil
		}
		switch t[0] {
		case types.Symbol("define"):
			return evalDefine(t, env)
		case types.Symbol("if"):
			if len(t) < 4 {
				return nil, errors.New("syntax error: if clause must be (if predicate consequent alternative) style")
			}
			return evalIf(t[1], t[2], t[3], env)
		case types.Symbol("cond"):
			if len(t) < 2 {
				return nil, errors.New("syntax error: cond clause must be (cond predicate consequent alternative) style")
			}
			return evalCond(t, env)
		case types.Symbol("lambda"):
			if len(t) < 3 {
				return nil, errors.New("lambda must have more than 3 words")
			}
			return Lambda{t[1], t[2], env}, nil
		case types.Symbol("begin"):
			return evalBegin(env, t[1:]...)
		case types.Symbol("load"):
			path, ok := t[1].(string)
			if !ok {
				return nil, errors.New("syntax error: args of load should be string")
			}
			return evalLoad(path, env)
		default:
			return evalApplication(env, t[0], t[1:]...)
		}
	default:
		// not found any known operands. failed.
		return nil, fmt.Errorf("unknown expression type -- %v", exp)
	}
	return nil, nil
}

// EvalFile evaluate given file
func EvalFile(filename string, env *Env) (types.Expression, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	env.Put("#current-load-path", filename)
	defer f.Close()
	return EvalReader(f, env)
}

// EvalReader evaluate scheme program from io.Reader
func EvalReader(r io.Reader, env *Env) (types.Expression, error) {
	l := lexer.New(r)
	p := parser.New(l)
	if _, err := env.Get("#current-load-path"); err != nil {
		env.Put("#current-load-path", "")
	}
	var exps types.Expression
	for {
		tokens, err := p.Parse()
		if err != nil {
			return nil, err
		}
		// TODO should handle unknown token.
		if tokens == types.Symbol("") {
			break
		}
		exps, err = Eval(tokens, env)
		if err != nil {
			return nil, err
		}
	}
	return exps, nil
}

// Apply receives procedure and arguments. if procedure is compounded, evaluate on extended environment.
func Apply(procedure types.Expression, args []types.Expression) (types.Expression, error) {
	switch p := procedure.(type) {
	case Lambda:
		// extend environment base on lambda arguments
		// should bind argument to this environment.
		// for example, (define (sum x y) (+ x y)) and given (sum 1 2),
		// then creates frames which have x = 1 and y = 2.
		env := &Env{m: make(Frame), parent: p.Env}
		env.Setup()
		switch lambdaArgs := p.Args.(type) {
		case []types.Expression:
			if len(lambdaArgs) != len(args) {
				return nil, errors.New("given args is not match with lambda args")
			}
			for i, arg := range lambdaArgs {
				env.Put(arg.(types.Symbol), args[i])
			}
		default:
			env.Put(lambdaArgs.(types.Symbol), lambdaArgs)
		}
		return Eval(p.Body, env)
	case func(...types.Expression) (types.Expression, error):
		// primitive procedure
		return p(args...)
	default:
		return nil, fmt.Errorf("unknown procedure type -- %v", procedure)
	}
}
<commit_msg>go vet: remove unreachable return<commit_after>// Package eval implements an evaluator of gigue.
package eval

import (
	"errors"
	"fmt"
	"github.com/suzuken/gigue/lexer"
	"github.com/suzuken/gigue/parser"
	"github.com/suzuken/gigue/types"
	"io"
	"os"
	"path/filepath"
	"strings"
)

// evalDefine evaluate (define ...) style expression.
func evalDefine(exps []types.Expression, env *Env) (types.Expression, error) {
	if len(exps) < 2 {
		return nil, errors.New("define clause must have symbol and body")
	}
	switch tt := exps[1].(type) {
	// put symbol and variables
	// (define x 1) style definition
	case types.Symbol:
		value, err := Eval(exps[2], env)
		if err != nil {
			return nil, err
		}
		env.Put(tt, value)
		return nil, nil
	// (define (hoge args) (..)) style definition
	// above style is syntax sugar for lambda.
	case []types.Expression:
		if len(tt) < 2 {
			return nil, errors.New("define statament must have more than 2 words")
		}
		caddr, ok := tt[0].(types.Symbol)
		if !ok {
			return nil, errors.New("(define x) of x should be symbol")
		}
		// create lambda and put it into environment
		env.Put(caddr, Lambda{tt[1:], exps[2], env})
		return nil, nil
	default:
		return nil, nil
	}
}

// evalIf evaluates if-clause.
// predicate consequent alternative
// like, (if (ok? yeah) (go) (not go))
func evalIf(predicate, consequent, alternative types.Expression, env *Env) (types.Expression, error) {
	bb, err := evalPredicate(predicate, env)
	if err != nil {
		return nil, err
	}
	if bb {
		return Eval(consequent, env)
	}
	return Eval(alternative, env)
}

// evalPredicate is helper for evaluate expression should be boolean.
func evalPredicate(exp types.Expression, env *Env) (types.Boolean, error) {
	b, err := Eval(exp, env)
	if err != nil {
		return false, err
	}
	bb, ok := b.(types.Boolean)
	if !ok {
		return false, fmt.Errorf("the expression should return types.Boolean, exps: %v", exp)
	}
	return bb, nil
}

func evalCond(exps []types.Expression, env *Env) (types.Expression, error) {
	for _, operand := range exps[1:] {
		tt, ok := operand.([]types.Expression)
		if !ok {
			return nil, errors.New("cond clause must have expression")
		}
		if tt[0] == types.Symbol("else") {
			return Eval(tt[1], env)
		}
		if bb, err := evalPredicate(tt[0], env); err != nil {
			return nil, err
		} else if bb {
			return Eval(tt[1], env)
		}
	}
	// unreachable
	return nil, nil
}

func evalBegin(env *Env, exps ...types.Expression) (types.Expression, error) {
	var lastExp types.Expression
	for _, beginExp := range exps {
		l, err := Eval(beginExp, env)
		if err != nil {
			return nil, err
		}
		lastExp = l
	}
	return lastExp, nil
}

// evalLoad evaluates (load "file.scm") style definition.
// loading file and evaluate it.
func evalLoad(path string, env *Env) (types.Expression, error) {
	current, err := env.Get("#current-load-path")
	if err != nil {
		return nil, err
	}
	// if path is set, search from current directory
	if p := fmt.Sprintf("%s", current); p != "" {
		// if path start with /, deal as absolute path
		// if not, deal as relative path
		if !strings.HasPrefix(path, "/") {
			path = filepath.Join(filepath.Dir(p), path)
		}
	}
	abs, err := filepath.Abs(path)
	if err != nil {
		return nil, err
	}
	// evaluate file in given environment.
	return EvalFile(abs, env)
}

func evalApplication(env *Env, operator types.Expression, operands ...types.Expression) (types.Expression, error) {
	// extend environment
	exps := make([]types.Expression, 0, len(operands)-1)
	for _, operand := range operands {
		exp, err := Eval(operand, env)
		if err != nil {
			return nil, err
		}
		exps = append(exps, exp)
	}
	// maybe, it is primitive procedure or compound procedure.
	fn, err := Eval(operator, env)
	if err != nil {
		return nil, err
	}
	return Apply(fn, exps)
}

// Eval is body of evaluator
func Eval(exp types.Expression, env *Env) (types.Expression, error) {
	switch t := exp.(type) {
	case types.Boolean, types.Number, *types.Pair, string:
		return t, nil
	case types.Symbol:
		// it's variable or expression. get value from environment
		e, err := env.Get(t)
		if err != nil {
			return nil, err
		}
		return e, nil
	case []types.Expression:
		if len(t) == 0 {
			return &types.Pair{}, nil
		}
		switch t[0] {
		case types.Symbol("define"):
			return evalDefine(t, env)
		case types.Symbol("if"):
			if len(t) < 4 {
				return nil, errors.New("syntax error: if clause must be (if predicate consequent alternative) style")
			}
			return evalIf(t[1], t[2], t[3], env)
		case types.Symbol("cond"):
			if len(t) < 2 {
				return nil, errors.New("syntax error: cond clause must be (cond predicate consequent alternative) style")
			}
			return evalCond(t, env)
		case types.Symbol("lambda"):
			if len(t) < 3 {
				return nil, errors.New("lambda must have more than 3 words")
			}
			return Lambda{t[1], t[2], env}, nil
		case types.Symbol("begin"):
			return evalBegin(env, t[1:]...)
		case types.Symbol("load"):
			path, ok := t[1].(string)
			if !ok {
				return nil, errors.New("syntax error: args of load should be string")
			}
			return evalLoad(path, env)
		default:
			return evalApplication(env, t[0], t[1:]...)
		}
	default:
		// not found any known operands. failed.
		return nil, fmt.Errorf("unknown expression type -- %v", exp)
	}
}

// EvalFile evaluate given file
func EvalFile(filename string, env *Env) (types.Expression, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	env.Put("#current-load-path", filename)
	defer f.Close()
	return EvalReader(f, env)
}

// EvalReader evaluate scheme program from io.Reader
func EvalReader(r io.Reader, env *Env) (types.Expression, error) {
	l := lexer.New(r)
	p := parser.New(l)
	if _, err := env.Get("#current-load-path"); err != nil {
		env.Put("#current-load-path", "")
	}
	var exps types.Expression
	for {
		tokens, err := p.Parse()
		if err != nil {
			return nil, err
		}
		// TODO should handle unknown token.
		if tokens == types.Symbol("") {
			break
		}
		exps, err = Eval(tokens, env)
		if err != nil {
			return nil, err
		}
	}
	return exps, nil
}

// Apply receives procedure and arguments. if procedure is compounded, evaluate on extended environment.
func Apply(procedure types.Expression, args []types.Expression) (types.Expression, error) {
	switch p := procedure.(type) {
	case Lambda:
		// extend environment base on lambda arguments
		// should bind argument to this environment.
		// for example, (define (sum x y) (+ x y)) and given (sum 1 2),
		// then creates frames which have x = 1 and y = 2.
		env := &Env{m: make(Frame), parent: p.Env}
		env.Setup()
		switch lambdaArgs := p.Args.(type) {
		case []types.Expression:
			if len(lambdaArgs) != len(args) {
				return nil, errors.New("given args is not match with lambda args")
			}
			for i, arg := range lambdaArgs {
				env.Put(arg.(types.Symbol), args[i])
			}
		default:
			env.Put(lambdaArgs.(types.Symbol), lambdaArgs)
		}
		return Eval(p.Body, env)
	case func(...types.Expression) (types.Expression, error):
		// primitive procedure
		return p(args...)
	default:
		return nil, fmt.Errorf("unknown procedure type -- %v", procedure)
	}
}
<|endoftext|>


<commit_before>package eval

// This file implements facilities for "scanning" arguments and options.

import (
	"reflect"

	"github.com/elves/elvish/parse"
	"github.com/elves/elvish/util"
)

// ScanArgs scans arguments into pointers to supported argument types. If the
// arguments cannot be scanned, an error is thrown.
func ScanArgs(src []Value, dst ...interface{}) {
	if len(src) != len(dst) {
		throwf("arity mistmatch: want %d arguments, got %d", len(dst), len(src))
	}
	for i, value := range src {
		scanArg(value, dst[i])
	}
}

// ScanArgsVariadic is like ScanArgs, but the last element of args should be a
// pointer to a slice, and the rest of arguments will be scanned into it.
func ScanArgsVariadic(src []Value, dst ...interface{}) {
	if len(src) < len(dst)-1 {
		throwf("arity mistmatch: want at least %d arguments, got %d", len(dst)-1, len(src))
	}
	ScanArgs(src[:len(dst)-1], dst[:len(dst)-1]...)

	// Scan the rest of arguments into a slice.
	rest := src[len(dst)-1:]
	restDst := reflect.ValueOf(dst[len(dst)-1])
	if restDst.Kind() != reflect.Ptr || restDst.Elem().Kind() != reflect.Slice {
		throwf("internal bug: %T to ScanArgsVariadic, need pointer to slice", dst[len(dst)-1])
	}
	scanned := reflect.MakeSlice(restDst.Elem().Type(), len(rest), len(rest))
	for i, value := range rest {
		scanArg(value, scanned.Index(i).Addr().Interface())
	}
	reflect.Indirect(restDst).Set(scanned)
}

// ScanArgsOptionalInput is like ScanArgs, but the argument can contain an
// optional iterable value at the end containing inputs to the function. The
// return value is a function that iterates the iterable value if it exists, or
// the input otherwise.
func ScanArgsOptionalInput(ec *EvalCtx, src []Value, dst ...interface{}) func(func(Value)) {
	switch len(src) {
	case len(dst):
		ScanArgs(src, dst...)
		return ec.IterateInputs
	case len(dst) + 1:
		ScanArgs(src[:len(dst)], dst...)
		value := src[len(dst)]
		iterable, ok := value.(Iterable)
		if !ok {
			throwf("need iterable argument, got %s", value.Kind())
		}
		return func(f func(Value)) {
			iterable.Iterate(func(v Value) bool {
				f(v)
				return true
			})
		}
	default:
		throwf("arity mistmatch: want %d or %d arguments, got %d", len(dst), len(dst)+1, len(src))
		return nil
	}
}

// OptToScan is a data structure for an option that is intended to be used in
// ScanOpts.
type OptToScan struct {
	Name    string
	Ptr     interface{}
	Default Value
}

// ScanOpts scans options from a map.
func ScanOpts(m map[string]Value, opts ...OptToScan) {
	scanned := make(map[string]bool)
	for _, opt := range opts {
		a := opt.Ptr
		value, ok := m[opt.Name]
		if !ok {
			value = opt.Default
		}
		scanArg(value, a)
		scanned[opt.Name] = true
	}
	for key := range m {
		if !scanned[key] {
			throwf("unknown option %s", parse.Quote(key))
		}
	}
}

// ScanOptsToStruct scan options from a map like ScanOpts except the destination
// is a struct whose fields correspond to the options to be parsed. A field
// named FieldName corresponds to the option named field-name, unless the field
// has a explicit "name" tag.
func ScanOptsToStruct(m map[string]Value, structPtr interface{}) {
	ptrValue := reflect.ValueOf(structPtr)
	if ptrValue.Kind() != reflect.Ptr || ptrValue.Elem().Kind() != reflect.Struct {
		throwf("internal bug: need struct ptr for ScanOptsToStruct, got %T", structPtr)
	}
	struc := ptrValue.Elem()

	// fieldIdxForOpt maps option name to the index of field in struc.
	fieldIdxForOpt := make(map[string]int)
	for i := 0; i < struc.Type().NumField(); i++ {
		// ignore unexported fields
		if !struc.Field(i).CanSet() {
			continue
		}

		f := struc.Type().Field(i)
		optName := f.Tag.Get("name")
		if optName == "" {
			optName = util.CamelToDashed(f.Name)
		}
		fieldIdxForOpt[optName] = i
	}

	for k, v := range m {
		fieldIdx, ok := fieldIdxForOpt[k]
		if !ok {
			throwf("unknown option %s", parse.Quote(k))
		}
		scanArg(v, struc.Field(fieldIdx).Addr().Interface())
	}
}

func scanArg(src Value, dstPtr interface{}) {
	ptr := reflect.ValueOf(dstPtr)
	if ptr.Kind() != reflect.Ptr {
		throwf("internal bug: %T to ScanArgs, need pointer", dstPtr)
	}
	dst := reflect.Indirect(ptr)
	switch dst.Kind() {
	case reflect.Int:
		i, err := toInt(src)
		maybeThrow(err)
		dst.Set(reflect.ValueOf(i))
	case reflect.Float64:
		f, err := toFloat(src)
		maybeThrow(err)
		dst.Set(reflect.ValueOf(f))
	default:
		if reflect.TypeOf(src).ConvertibleTo(dst.Type()) {
			dst.Set(reflect.ValueOf(src).Convert(dst.Type()))
		} else {
			throwf("need %T argument, got %s", dst.Interface(), src.Kind())
		}
	}
}
<commit_msg>More renamings.<commit_after>package eval

// This file implements facilities for "scanning" arguments and options.

import (
	"reflect"

	"github.com/elves/elvish/parse"
	"github.com/elves/elvish/util"
)

// ScanArgs scans arguments into pointers to supported argument types. If the
// arguments cannot be scanned, an error is thrown.
func ScanArgs(src []Value, dstPtrs ...interface{}) {
	if len(src) != len(dstPtrs) {
		throwf("arity mistmatch: want %d arguments, got %d", len(dstPtrs), len(src))
	}
	for i, value := range src {
		scanArg(value, dstPtrs[i])
	}
}

// ScanArgsVariadic is like ScanArgs, but the last element of args should be a
// pointer to a slice, and the rest of arguments will be scanned into it.
func ScanArgsVariadic(src []Value, dstPtrs ...interface{}) {
	if len(src) < len(dstPtrs)-1 {
		throwf("arity mistmatch: want at least %d arguments, got %d", len(dstPtrs)-1, len(src))
	}
	ScanArgs(src[:len(dstPtrs)-1], dstPtrs[:len(dstPtrs)-1]...)

	// Scan the rest of arguments into a slice.
	rest := src[len(dstPtrs)-1:]
	restDst := reflect.ValueOf(dstPtrs[len(dstPtrs)-1])
	if restDst.Kind() != reflect.Ptr || restDst.Elem().Kind() != reflect.Slice {
		throwf("internal bug: %T to ScanArgsVariadic, need pointer to slice", dstPtrs[len(dstPtrs)-1])
	}
	scanned := reflect.MakeSlice(restDst.Elem().Type(), len(rest), len(rest))
	for i, value := range rest {
		scanArg(value, scanned.Index(i).Addr().Interface())
	}
	reflect.Indirect(restDst).Set(scanned)
}

// ScanArgsOptionalInput is like ScanArgs, but the argument can contain an
// optional iterable value at the end containing inputs to the function. The
// return value is a function that iterates the iterable value if it exists, or
// the input otherwise.
func ScanArgsOptionalInput(ec *EvalCtx, src []Value, dstArgs ...interface{}) func(func(Value)) {
	switch len(src) {
	case len(dstArgs):
		ScanArgs(src, dstArgs...)
		return ec.IterateInputs
	case len(dstArgs) + 1:
		ScanArgs(src[:len(dstArgs)], dstArgs...)
		value := src[len(dstArgs)]
		iterable, ok := value.(Iterable)
		if !ok {
			throwf("need iterable argument, got %s", value.Kind())
		}
		return func(f func(Value)) {
			iterable.Iterate(func(v Value) bool {
				f(v)
				return true
			})
		}
	default:
		throwf("arity mistmatch: want %d or %d arguments, got %d", len(dstArgs), len(dstArgs)+1, len(src))
		return nil
	}
}

// OptToScan is a data structure for an option that is intended to be used in
// ScanOpts.
type OptToScan struct {
	Name    string
	Ptr     interface{}
	Default Value
}

// ScanOpts scans options from a map.
func ScanOpts(m map[string]Value, opts ...OptToScan) {
	scanned := make(map[string]bool)
	for _, opt := range opts {
		a := opt.Ptr
		value, ok := m[opt.Name]
		if !ok {
			value = opt.Default
		}
		scanArg(value, a)
		scanned[opt.Name] = true
	}
	for key := range m {
		if !scanned[key] {
			throwf("unknown option %s", parse.Quote(key))
		}
	}
}

// ScanOptsToStruct scan options from a map like ScanOpts except the destination
// is a struct whose fields correspond to the options to be parsed. A field
// named FieldName corresponds to the option named field-name, unless the field
// has a explicit "name" tag.
func ScanOptsToStruct(m map[string]Value, structPtr interface{}) {
	ptrValue := reflect.ValueOf(structPtr)
	if ptrValue.Kind() != reflect.Ptr || ptrValue.Elem().Kind() != reflect.Struct {
		throwf("internal bug: need struct ptr for ScanOptsToStruct, got %T", structPtr)
	}
	struc := ptrValue.Elem()

	// fieldIdxForOpt maps option name to the index of field in struc.
	fieldIdxForOpt := make(map[string]int)
	for i := 0; i < struc.Type().NumField(); i++ {
		// ignore unexported fields
		if !struc.Field(i).CanSet() {
			continue
		}

		f := struc.Type().Field(i)
		optName := f.Tag.Get("name")
		if optName == "" {
			optName = util.CamelToDashed(f.Name)
		}
		fieldIdxForOpt[optName] = i
	}

	for k, v := range m {
		fieldIdx, ok := fieldIdxForOpt[k]
		if !ok {
			throwf("unknown option %s", parse.Quote(k))
		}
		scanArg(v, struc.Field(fieldIdx).Addr().Interface())
	}
}

func scanArg(src Value, dstPtr interface{}) {
	ptr := reflect.ValueOf(dstPtr)
	if ptr.Kind() != reflect.Ptr {
		throwf("internal bug: %T to ScanArgs, need pointer", dstPtr)
	}
	dst := reflect.Indirect(ptr)
	switch dst.Kind() {
	case reflect.Int:
		i, err := toInt(src)
		maybeThrow(err)
		dst.Set(reflect.ValueOf(i))
	case reflect.Float64:
		f, err := toFloat(src)
		maybeThrow(err)
		dst.Set(reflect.ValueOf(f))
	default:
		if reflect.TypeOf(src).ConvertibleTo(dst.Type()) {
			dst.Set(reflect.ValueOf(src).Convert(dst.Type()))
		} else {
			throwf("need %T argument, got %s", dst.Interface(), src.Kind())
		}
	}
}
<|endoftext|>


<commit_before>package EventBus

import (
	"fmt"
	"reflect"
	"sync"
)

//BusSubscriber defines subscription-related bus behavior
type BusSubscriber interface {
	Subscribe(topic string, fn interface{}) error
	SubscribeAsync(topic string, fn interface{}, transactional bool) error
	SubscribeOnce(topic string, fn interface{}) error
	SubscribeOnceAsync(topic string, fn interface{}) error
	Unsubscribe(topic string, handler interface{}) error
}

//BusPublisher defines publishing-related bus behavior
type BusPublisher interface {
	Publish(topic string, args ...interface{})
}

//BusController defines bus control behavior (checking handler's presence, synchronization)
type BusController interface {
	HasCallback(topic string) bool
	WaitAsync()
}

//Bus englobes global (subscribe, publish, control) bus behavior
type Bus interface {
	BusController
	BusSubscriber
	BusPublisher
}

// EventBus - box for handlers and callbacks.
type EventBus struct {
	handlers map[string][]*eventHandler
	lock     sync.Mutex // a lock for the map
	wg       sync.WaitGroup
}

type eventHandler struct {
	callBack      reflect.Value
	flagOnce      bool
	async         bool
	transactional bool
	called        bool
	sync.Mutex    // lock for an event handler - useful for running async callbacks serially
}

// New returns new EventBus with empty handlers.
func New() Bus {
	b := &EventBus{
		make(map[string][]*eventHandler),
		sync.Mutex{},
		sync.WaitGroup{},
	}
	return Bus(b)
}

// doSubscribe handles the subscription logic and is utilized by the public Subscribe functions
func (bus *EventBus) doSubscribe(topic string, fn interface{}, handler *eventHandler) error {
	bus.lock.Lock()
	defer bus.lock.Unlock()
	if !(reflect.TypeOf(fn).Kind() == reflect.Func) {
		return fmt.Errorf("%s is not of type reflect.Func", reflect.TypeOf(fn).Kind())
	}
	bus.handlers[topic] = append(bus.handlers[topic], handler)
	return nil
}

// Subscribe subscribes to a topic.
// Returns error if `fn` is not a function.
func (bus *EventBus) Subscribe(topic string, fn interface{}) error {
	return bus.doSubscribe(topic, fn, &eventHandler{
		reflect.ValueOf(fn), false, false, false, false, sync.Mutex{},
	})
}

// SubscribeAsync subscribes to a topic with an asynchronous callback
// Transactional determines whether subsequent callbacks for a topic are
// run serially (true) or concurrently (false)
// Returns error if `fn` is not a function.
func (bus *EventBus) SubscribeAsync(topic string, fn interface{}, transactional bool) error {
	return bus.doSubscribe(topic, fn, &eventHandler{
		reflect.ValueOf(fn), false, true, transactional, false, sync.Mutex{},
	})
}

// SubscribeOnce subscribes to a topic once. Handler will be removed after executing.
// Returns error if `fn` is not a function.
func (bus *EventBus) SubscribeOnce(topic string, fn interface{}) error {
	return bus.doSubscribe(topic, fn, &eventHandler{
		reflect.ValueOf(fn), true, false, false, false, sync.Mutex{},
	})
}

// SubscribeOnceAsync subscribes to a topic once with an asynchronous callback
// Handler will be removed after executing.
// Returns error if `fn` is not a function.
func (bus *EventBus) SubscribeOnceAsync(topic string, fn interface{}) error {
	return bus.doSubscribe(topic, fn, &eventHandler{
		reflect.ValueOf(fn), true, true, false, false, sync.Mutex{},
	})
}

// HasCallback returns true if exists any callback subscribed to the topic.
func (bus *EventBus) HasCallback(topic string) bool {
	bus.lock.Lock()
	defer bus.lock.Unlock()
	_, ok := bus.handlers[topic]
	if ok {
		return len(bus.handlers[topic]) > 0
	}
	return false
}

// Unsubscribe removes callback defined for a topic.
// Returns error if there are no callbacks subscribed to the topic.
func (bus *EventBus) Unsubscribe(topic string, handler interface{}) error {
	bus.lock.Lock()
	defer bus.lock.Unlock()
	if _, ok := bus.handlers[topic]; ok && len(bus.handlers[topic]) > 0 {
		bus.removeHandler(topic, reflect.ValueOf(handler))
		return nil
	}
	return fmt.Errorf("topic %s doesn't exist", topic)
}

// Publish executes callback defined for a topic. Any additional argument will be transferred to the callback.
func (bus *EventBus) Publish(topic string, args ...interface{}) {
	bus.lock.Lock() // will unlock if handler is not found or always after setUpPublish
	defer bus.lock.Unlock()
	if handlers, ok := bus.handlers[topic]; ok {
		for _, handler := range handlers {
			if !handler.async {
				bus.doPublish(handler, topic, args...)
			} else {
				bus.wg.Add(1)
				go bus.doPublishAsync(handler, topic, args...)
			}
		}
	}
}

func (bus *EventBus) doPublish(handler *eventHandler, topic string, args ...interface{}) {
	passedArguments := bus.setUpPublish(topic, args...)
	if handler.flagOnce {
		bus.removeHandler(topic, handler.callBack)
		if handler.called {
			return
		}
	}
	handler.called = true
	handler.callBack.Call(passedArguments)
}

func (bus *EventBus) doPublishAsync(handler *eventHandler, topic string, args ...interface{}) {
	defer bus.wg.Done()
	if handler.transactional {
		handler.Lock()
		defer handler.Unlock()
	}
	bus.doPublish(handler, topic, args...)
}

func (bus *EventBus) findHandlerIdx(topic string, callback reflect.Value) int {
	if _, ok := bus.handlers[topic]; ok {
		for idx, handler := range bus.handlers[topic] {
			if handler.callBack == callback {
				return idx
			}
		}
	}
	return -1
}

func (bus *EventBus) removeHandler(topic string, callback reflect.Value) {
	i := bus.findHandlerIdx(topic, callback)
	if i >= 0 {
		bus.handlers[topic] = append(bus.handlers[topic][:i], bus.handlers[topic][i+1:]...)
	}
}

func (bus *EventBus) setUpPublish(topic string, args ...interface{}) []reflect.Value {

	passedArguments := make([]reflect.Value, 0)
	for _, arg := range args {
		passedArguments = append(passedArguments, reflect.ValueOf(arg))
	}
	return passedArguments
}

// WaitAsync waits for all async callbacks to complete
func (bus *EventBus) WaitAsync() {
	bus.wg.Wait()
}
<commit_msg>fix(async transactional): fixed race conditions in async transactional subscribe<commit_after>package EventBus

import (
	"fmt"
	"reflect"
	"sync"
)

//BusSubscriber defines subscription-related bus behavior
type BusSubscriber interface {
	Subscribe(topic string, fn interface{}) error
	SubscribeAsync(topic string, fn interface{}, transactional bool) error
	SubscribeOnce(topic string, fn interface{}) error
	SubscribeOnceAsync(topic string, fn interface{}) error
	Unsubscribe(topic string, handler interface{}) error
}

//BusPublisher defines publishing-related bus behavior
type BusPublisher interface {
	Publish(topic string, args ...interface{})
}

//BusController defines bus control behavior (checking handler's presence, synchronization)
type BusController interface {
	HasCallback(topic string) bool
	WaitAsync()
}

//Bus englobes global (subscribe, publish, control) bus behavior
type Bus interface {
	BusController
	BusSubscriber
	BusPublisher
}

// EventBus - box for handlers and callbacks.
type EventBus struct {
	handlers map[string][]*eventHandler
	lock     sync.Mutex // a lock for the map
	wg       sync.WaitGroup
}

type eventHandler struct {
	callBack      reflect.Value
	flagOnce      bool
	async         bool
	transactional bool
	called        bool
	sync.Mutex    // lock for an event handler - useful for running async callbacks serially
}

// New returns new EventBus with empty handlers.
func New() Bus {
	b := &EventBus{
		make(map[string][]*eventHandler),
		sync.Mutex{},
		sync.WaitGroup{},
	}
	return Bus(b)
}

// doSubscribe handles the subscription logic and is utilized by the public Subscribe functions
func (bus *EventBus) doSubscribe(topic string, fn interface{}, handler *eventHandler) error {
	bus.lock.Lock()
	defer bus.lock.Unlock()
	if !(reflect.TypeOf(fn).Kind() == reflect.Func) {
		return fmt.Errorf("%s is not of type reflect.Func", reflect.TypeOf(fn).Kind())
	}
	bus.handlers[topic] = append(bus.handlers[topic], handler)
	return nil
}

// Subscribe subscribes to a topic.
// Returns error if `fn` is not a function.
func (bus *EventBus) Subscribe(topic string, fn interface{}) error {
	return bus.doSubscribe(topic, fn, &eventHandler{
		reflect.ValueOf(fn), false, false, false, false, sync.Mutex{},
	})
}

// SubscribeAsync subscribes to a topic with an asynchronous callback
// Transactional determines whether subsequent callbacks for a topic are
// run serially (true) or concurrently (false)
// Returns error if `fn` is not a function.
func (bus *EventBus) SubscribeAsync(topic string, fn interface{}, transactional bool) error {
	return bus.doSubscribe(topic, fn, &eventHandler{
		reflect.ValueOf(fn), false, true, transactional, false, sync.Mutex{},
	})
}

// SubscribeOnce subscribes to a topic once. Handler will be removed after executing.
// Returns error if `fn` is not a function.
func (bus *EventBus) SubscribeOnce(topic string, fn interface{}) error {
	return bus.doSubscribe(topic, fn, &eventHandler{
		reflect.ValueOf(fn), true, false, false, false, sync.Mutex{},
	})
}

// SubscribeOnceAsync subscribes to a topic once with an asynchronous callback
// Handler will be removed after executing.
// Returns error if `fn` is not a function.
func (bus *EventBus) SubscribeOnceAsync(topic string, fn interface{}) error {
	return bus.doSubscribe(topic, fn, &eventHandler{
		reflect.ValueOf(fn), true, true, false, false, sync.Mutex{},
	})
}

// HasCallback returns true if exists any callback subscribed to the topic.
func (bus *EventBus) HasCallback(topic string) bool {
	bus.lock.Lock()
	defer bus.lock.Unlock()
	_, ok := bus.handlers[topic]
	if ok {
		return len(bus.handlers[topic]) > 0
	}
	return false
}

// Unsubscribe removes callback defined for a topic.
// Returns error if there are no callbacks subscribed to the topic.
func (bus *EventBus) Unsubscribe(topic string, handler interface{}) error {
	bus.lock.Lock()
	defer bus.lock.Unlock()
	if _, ok := bus.handlers[topic]; ok && len(bus.handlers[topic]) > 0 {
		bus.removeHandler(topic, reflect.ValueOf(handler))
		return nil
	}
	return fmt.Errorf("topic %s doesn't exist", topic)
}

// Publish executes callback defined for a topic. Any additional argument will be transferred to the callback.
func (bus *EventBus) Publish(topic string, args ...interface{}) {
	bus.lock.Lock() // will unlock if handler is not found or always after setUpPublish
	defer bus.lock.Unlock()
	if handlers, ok := bus.handlers[topic]; ok {
		for _, handler := range handlers {
			if !handler.async {
				bus.doPublish(handler, topic, args...)
			} else {
				bus.wg.Add(1)
				if handler.transactional {
					handler.Lock()
				}
				go bus.doPublishAsync(handler, topic, args...)
			}
		}
	}
}

func (bus *EventBus) doPublish(handler *eventHandler, topic string, args ...interface{}) {
	passedArguments := bus.setUpPublish(topic, args...)
	if handler.flagOnce {
		bus.removeHandler(topic, handler.callBack)
		if handler.called {
			return
		}
	}
	handler.called = true
	handler.callBack.Call(passedArguments)
}

func (bus *EventBus) doPublishAsync(handler *eventHandler, topic string, args ...interface{}) {
	defer bus.wg.Done()
	if handler.transactional {
		defer handler.Unlock()
	}
	bus.doPublish(handler, topic, args...)
}

func (bus *EventBus) findHandlerIdx(topic string, callback reflect.Value) int {
	if _, ok := bus.handlers[topic]; ok {
		for idx, handler := range bus.handlers[topic] {
			if handler.callBack == callback {
				return idx
			}
		}
	}
	return -1
}

func (bus *EventBus) removeHandler(topic string, callback reflect.Value) {
	i := bus.findHandlerIdx(topic, callback)
	if i >= 0 {
		bus.handlers[topic] = append(bus.handlers[topic][:i], bus.handlers[topic][i+1:]...)
	}
}

func (bus *EventBus) setUpPublish(topic string, args ...interface{}) []reflect.Value {

	passedArguments := make([]reflect.Value, 0)
	for _, arg := range args {
		passedArguments = append(passedArguments, reflect.ValueOf(arg))
	}
	return passedArguments
}

// WaitAsync waits for all async callbacks to complete
func (bus *EventBus) WaitAsync() {
	bus.wg.Wait()
}
<|endoftext|>


<commit_before>package goheroku

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"github.com/Shopify/sarama"
	cluster "github.com/bsm/sarama-cluster"
	"github.com/joeshaw/envdecode"
	"log"
	"net/url"
	"strings"
)

type appConfig struct {
	URL           string `env:"KAFKA_URL,required"`
	TrustedCert   string `env:"KAFKA_TRUSTED_CERT,required"`
	ClientCertKey string `env:"KAFKA_CLIENT_CERT_KEY,required"`
	ClientCert    string `env:"KAFKA_CLIENT_CERT,required"`
	Prefix        string `env:"KAFKA_PREFIX"`
	TLSConfig     *tls.Config
	BrokerAddrs   []string
}

// This function requires that you specify the topic that the returned consumer
// will consume from and the group that it will belong to.
func NewConsumer(topic string, consumerGroup string) (*cluster.Consumer, error) {
	ac, _ := setupConnection()
	consumer, err := ac.createKafkaConsumer(topic, consumerGroup, ac.BrokerAddrs, ac.TLSConfig)
	if err != nil {
		return nil, err
	}
	return consumer, nil
}

// For move information about the difference between Async and Sync producers,
// see the Sarama docs
func NewAsyncProducer() (sarama.AsyncProducer, error) {
	ac, _ := setupConnection()
	producer, err := ac.createKafkaAsyncProducer(ac.BrokerAddrs, ac.TLSConfig)
	if err != nil {
		return nil, err
	}
	return producer, nil
}

// For move information about the difference between Async and Sync producers,
// see the Sarama docs
func NewSyncProducer() (sarama.SyncProducer, error) {
	ac, _ := setupConnection()
	producer, err := ac.createKafkaSyncProducer(ac.BrokerAddrs, ac.TLSConfig)
	if err != nil {
		return nil, err
	}
	return producer, nil
}

// To specify the topic or consumer group, the Kafka prefix needs
// to appended to it. This function makes it possible without having
// access to the app config.
func AppendPrefixTo(str string) string {
	ac := appConfig{}
	err := envdecode.Decode(&ac)
	if err != nil {
		log.Fatal(err)
	}
	if ac.Prefix != "" {
		str = strings.Join([]string{ac.Prefix, str}, "")
	}
	return str
}

func setupConnection() (*appConfig, error) {
	ac := appConfig{}
	err := envdecode.Decode(&ac)
	if err != nil {
		log.Fatal(err)
		return nil, err
	}
	err = ac.createTLSConfig()
	if err != nil {
		return nil, err
	}
	err = ac.brokerAddresses()
	if err != nil {
		return nil, err
	}
	return &ac, nil
}

// Create the TLS context, using the key and certificates provided.
func (ac *appConfig) createTLSConfig() error {
	roots := x509.NewCertPool()
	ok := roots.AppendCertsFromPEM([]byte(ac.TrustedCert))
	if !ok {
		return errors.New("Unable to parse Root Cert. Please check your Heroku environment.")
	}
	// Setup certs for Sarama
	cert, err := tls.X509KeyPair([]byte(ac.ClientCert), []byte(ac.ClientCertKey))
	if err != nil {
		return err
	}
	ac.TLSConfig = &tls.Config{
		Certificates:       []tls.Certificate{cert},
		InsecureSkipVerify: true,
		RootCAs:            roots,
	}
	return nil
}

// Extract the host:port pairs from the Kafka URL(s)
func (ac *appConfig) brokerAddresses() error {
	urls := strings.Split(ac.URL, ",")
	addrs := make([]string, len(urls))
	for i, v := range urls {
		u, err := url.Parse(v)
		if err != nil {
			return err
		}
		addrs[i] = u.Host
		ac.BrokerAddrs = addrs
	}
	return nil
}

// Consumer group will default to Sarama if there is no value passed in
func (ac *appConfig) createKafkaConsumer(topic string, consumerGroup string, brokers []string, tc *tls.Config) (*cluster.Consumer, error) {
	config := cluster.NewConfig()
	config.Net.TLS.Config = tc
	config.Net.TLS.Enable = true
	config.Group.PartitionStrategy = cluster.StrategyRoundRobin
	config.ClientID = consumerGroup
	config.Consumer.Return.Errors = true
	group := consumerGroup
	group = AppendPrefixTo(group)
	topic = AppendPrefixTo(topic)
	consumer, err := cluster.NewConsumer(brokers, group, []string{topic}, config)
	if err != nil {
		return nil, err
	}
	return consumer, nil
}

func (ac *appConfig) createKafkaAsyncProducer(brokers []string, tc *tls.Config) (sarama.AsyncProducer, error) {
	config := sarama.NewConfig()
	config.Net.TLS.Config = tc
	config.Net.TLS.Enable = true
	config.Producer.Return.Errors = true
	config.Producer.RequiredAcks = sarama.WaitForAll // Default is WaitForLocal
	producer, err := sarama.NewAsyncProducer(brokers, config)
	if err != nil {
		return nil, err
	}
	return producer, nil
}

func (ac *appConfig) createKafkaSyncProducer(brokers []string, tc *tls.Config) (sarama.SyncProducer, error) {
	config := sarama.NewConfig()
	config.Net.TLS.Config = tc
	config.Net.TLS.Enable = true
	config.Producer.Return.Errors = true
	config.Producer.RequiredAcks = sarama.WaitForAll // Default is WaitForLocal
	producer, err := sarama.NewSyncProducer(brokers, config)
	if err != nil {
		return nil, err
	}
	return producer, nil
}
<commit_msg>Removes AppConfig and dependency for handling env<commit_after>package goheroku

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"github.com/Shopify/sarama"
	cluster "github.com/bsm/sarama-cluster"
	"net/url"
	"os"
	"strings"
)

// To specify the topic or consumer group, the Kafka prefix needs
// to appended to it. This function makes it possible without having
// access to the app config.
func AppendPrefixTo(str string) string {
	prefix := os.Getenv("KAFKA_PREFIX")
	if prefix != "" {
		str = strings.Join([]string{prefix, str}, "")
	}
	return str
}

// Create the TLS context, using the key and certificates provided.
func createTLSConfig() (*tls.Config, error) {
	trustedCert, ok := os.LookupEnv("KAFKA_TRUSTED_CERT")
	if !ok {
		return nil, errors.New("Kafka Trusted Certificate not found!")
	}

	clientCertKey, ok := os.LookupEnv("KAFKA_CLIENT_CERT_KEY")
	if !ok {
		return nil, errors.New("Kafka Client Certificate Key not found!")
	}

	clientCert, ok := os.LookupEnv("KAFKA_CLIENT_CERT")
	if !ok {
		return nil, errors.New("Kafka Client Certificate not found!")
	}

	roots := x509.NewCertPool()
	ok = roots.AppendCertsFromPEM([]byte(trustedCert))
	if !ok {
		return nil, errors.New("Unable to parse Root Cert. Please check your Heroku environment.")
	}
	// Setup certs for Sarama
	cert, err := tls.X509KeyPair([]byte(clientCert), []byte(clientCertKey))
	if err != nil {
		return nil, err
	}
	return &tls.Config{
		Certificates:       []tls.Certificate{cert},
		InsecureSkipVerify: true,
		RootCAs:            roots,
	}, nil
}

// Extract the host:port pairs from the Kafka URL(s)
func brokerAddresses() ([]string, error) {
	URL := os.Getenv("KAFKA_URL")
	urls := strings.Split(URL, ",")
	addrs := make([]string, len(urls))
	for i, v := range urls {
		u, err := url.Parse(v)
		if err != nil {
			return nil, err
		}
		addrs[i] = u.Host
	}
	return addrs, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/user"
	"path"
	"time"

	"code.google.com/p/go.crypto/ssh"
	"code.google.com/p/goauth2/oauth"
	compute "code.google.com/p/google-api-go-client/compute/v1"
)

var (
	project  = flag.String("project", "proppy-containers", "project")
	zone     = flag.String("zone", "us-central1-a", "zone")
	name     = flag.String("name", "gorogoro-vm", "vm name")
	disk     = flag.String("disk", "gorogoro-disk", "disk name")
	image    = flag.String("image", "https://www.googleapis.com/compute/v1/projects/google-containers/global/images/container-vm-v20140522", "vm image")
	api      = flag.String("api", "https://www.googleapis.com/compute/v1", "api url")
	machine  = flag.String("machine", "/zones/us-central1-a/machineTypes/f1-micro", "machine type")
	cred     = flag.String("cred", ".config/gcloud/credentials", "path to gcloud credentials")
	identity = flag.String("identity", ".ssh/google_compute_engine", "path to gcloud ssh key")
)

var credentials Credentials
var key Key

// Credentials store gcloud credentials.
type Credentials struct {
	Data []struct {
		Credential struct {
			ClientId     string `json:"Client_Id"`
			ClientSecret string `json:"Client_Secret"`
			RefreshToken string `json:"Refresh_Token"`
		}
		Key struct {
			Scope string
		}
		ProjectId string `json:"projectId"`
	}
}

// Path returns gcloud credentials path.
func (c *Credentials) path() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", fmt.Errorf("enable to get current user")
	}
	return path.Join(usr.HomeDir, *cred), nil
}

// Read reads the credentials from disk.
func (c *Credentials) Read() error {
	path, err := c.path()
	if err != nil {
		return fmt.Errorf("failed to get credentials path: %v", err)
	}
	f, err := os.Open(path)
	if err != nil {
		return fmt.Errorf("unable to load credentials from %q: %v", path, err)
	}
	defer f.Close()

	if err := json.NewDecoder(f).Decode(c); err != nil {
		return fmt.Errorf("enable to decode credentials: %v", err)
	}
	if len(c.Data) == 0 {
		return fmt.Errorf("no credentials in: %q", path)
	}
	return nil
}

// Transport return a oauth2.Transport for the gcloud credentials.
func (c *Credentials) Transport() (*oauth.Transport, error) {
	t := &oauth.Transport{
		Config: &oauth.Config{
			ClientId:     c.Data[0].Credential.ClientId,
			ClientSecret: c.Data[0].Credential.ClientSecret,
			Scope:        c.Data[0].Key.Scope,
			RedirectURL:  "oob",
			AuthURL:      "https://accounts.google.com/o/oauth2/auth",
			TokenURL:     "https://accounts.google.com/o/oauth2/token",
			AccessType:   "offline",
		},
		Token:     &oauth.Token{RefreshToken: c.Data[0].Credential.RefreshToken},
		Transport: http.DefaultTransport,
	}
	err := t.Refresh()
	if err != nil {
		return nil, fmt.Errorf("failed to refresh token: %v", err)
	}
	return t, nil
}

// Compute is a Google Compute Engine services associated to a given project and zone.
type Compute struct {
	*compute.Service
	Project string
	Zone    string
}

// NewCompute returns a new Compute service.
func NewCompute(client *http.Client) (*Compute, error) {
	service, err := compute.New(client)
	if err != nil {
		return nil, fmt.Errorf("failed to create compute service: %v", err)
	}
	return &Compute{
		service,
		*project,
		*zone,
	}, nil
}

// Disk gets or creates a new root disk.
func (c *Compute) Disk(name string) (string, error) {
	disk, err := c.Disks.Get(c.Project, c.Zone, name).Do()
	if err == nil {
		log.Printf("found existing root disk: %q", disk.SelfLink)
		return disk.SelfLink, nil
	}
	log.Printf("not found, creating new root disk: %q", name)
	op, err := c.Disks.Insert(c.Project, c.Zone, &compute.Disk{
		Name: name,
	}).SourceImage(*image).Do()
	if err != nil {
		return "", fmt.Errorf("disk insert api call failed: %v", err)
	}
	if err := c.wait(op); err != nil {
		return "", fmt.Errorf("disk insert operation failed: %v", err)
	}
	log.Printf("root disk created: %q", op.TargetLink)
	return op.TargetLink, nil
}

// Instance gets or creates a new instance.
func (c *Compute) Instance(name, disk string) (string, error) {
	instance, err := c.Instances.Get(c.Project, c.Zone, name).Do()
	if err == nil {
		log.Printf("found existing instance: %q", instance.SelfLink)
		return instance.SelfLink, nil
	}
	log.Printf("not found, creating new instance: %q", name)
	prefix := *api + "/projects/" + c.Project
	op, err := c.Instances.Insert(c.Project, c.Zone, &compute.Instance{
		Name:        name,
		Description: "gorogoro vm",
		MachineType: prefix + *machine,
		Disks: []*compute.AttachedDisk{
			{
				Boot:   true,
				Type:   "PERSISTENT",
				Mode:   "READ_WRITE",
				Source: disk,
			},
		},
		NetworkInterfaces: []*compute.NetworkInterface{
			{
				AccessConfigs: []*compute.AccessConfig{
					&compute.AccessConfig{Type: "ONE_TO_ONE_NAT"},
				},
				Network: prefix + "/global/networks/default",
			},
		},
	}).Do()
	if err != nil {
		return "", fmt.Errorf("instance insert api call failed: %v", err)
	}
	if err := c.wait(op); err != nil {
		return "", fmt.Errorf("instance insert operation failed: %v", err)
	}
	log.Printf("instance created: %q", op.TargetLink)
	return op.TargetLink, nil
}

func (c *Compute) WaitConnection(name string, port int) (string, error) {
	for {
		instance, err := c.Instances.Get(c.Project, c.Zone, name).Do()
		if err != nil {
			return "", fmt.Errorf("could not find instance %q: %v", name, err)
		}

		if ip := instance.NetworkInterfaces[0].AccessConfigs[0].NatIP; ip != "" {
			addr := fmt.Sprintf("%s:%d", ip, port)
			conn, err := net.Dial("tcp", addr)
			if err != nil {
				log.Println("connection to %q failed: %v", addr, err)
				continue
			}
			defer conn.Close()
			log.Printf("connection to %q succeeded", addr)
			return ip, nil
		}
		time.Sleep(1 * time.Second)
	}
}

func (c *Compute) wait(op *compute.Operation) error {
	for {
		op, err := c.ZoneOperations.Get(c.Project, c.Zone, op.Name).Do()
		log.Printf("operation %q status: %s, %v", op.Name, op.Status, err)
		if op.Status == "DONE" {
			if op.Error != nil {
				return fmt.Errorf("operation error: %v", *op.Error.Errors[0])
			}
			break
		}
		time.Sleep(1 * time.Second)
	}
	return nil
}

type Key struct {
	ssh.Signer
}

func (k *Key) Read() error {
	path, err := k.path()
	if err != nil {
		return fmt.Errorf("failed to get ssh key path: %v", err)
	}
	f, err := os.Open(path)
	if err != nil {
		return fmt.Errorf("unable to load ssh key from %q: %v", path, err)
	}
	defer f.Close()

	bs, err := ioutil.ReadAll(f)
	if err != nil {
		return fmt.Errorf("failed to read ssh key: %v", err)
	}
	signer, err := ssh.ParsePrivateKey(bs)
	if err != nil {
		return fmt.Errorf("failed to parse ssh key: %v", err)
	}
	k.Signer = signer
	return nil
}

func (k *Key) path() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", fmt.Errorf("enable to get current user")
	}
	return path.Join(usr.HomeDir, *identity), nil
}

func (c *Compute) Tunnel(ip string, port int) (net.Conn, error) {
	usr, err := user.Current()
	if err != nil {
		return nil, fmt.Errorf("failed to get current user: %v", err)
	}
	if err := key.Read(); err != nil {
		return nil, fmt.Errorf("failed to read ssh private key: %v", err)
	}
	raddr := ip + ":22"
	conn, err := ssh.Dial("tcp", ip+":22", &ssh.ClientConfig{
		User: usr.Name,
		Auth: []ssh.AuthMethod{
			ssh.PublicKeys(key),
		},
	})
	if err != nil {
		return nil, fmt.Errorf("failed to dial ssh conn to %q: %v", raddr, err)
	}
	laddr := fmt.Sprintf("127.0.0.1:%d", port)
	return conn.Dial("tcp", laddr)
}

func main() {
	if err := credentials.Read(); err != nil {
		log.Fatalf("failed to read credentials: %v", err)
	}
	transport, err := credentials.Transport()
	if err != nil {
		log.Fatalf("failed to create transport: %v", err)
	}
	compute, err := NewCompute(transport.Client())
	if err != nil {
		log.Fatalf("failed to create compute: %v", err)
	}
	diskUrl, err := compute.Disk(*disk)
	if err != nil {
		log.Fatalf("error creating root disk")
	}
	log.Println("disk url:", diskUrl)
	instanceUrl, err := compute.Instance(*name, diskUrl)
	if err != nil {
		log.Fatalf("error creating instance", err)
	}
	log.Println("instance url:", instanceUrl)
	ip, err := compute.WaitConnection(*name, 22)
	if err != nil {
		log.Fatalf("failed to connect to instance ssh port: %v", err)
	}
	log.Println("instance ip:", ip)
	conn, err := compute.Tunnel(ip, 4243)
	if err != nil {
		log.Fatalf("failed to create ssh tunnel to docker: %v", err)
	}
	fmt.Println(conn, err)
}
<commit_msg>fix username<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/user"
	"path"
	"time"

	"code.google.com/p/go.crypto/ssh"
	"code.google.com/p/goauth2/oauth"
	compute "code.google.com/p/google-api-go-client/compute/v1"
)

var (
	project  = flag.String("project", "proppy-containers", "project")
	zone     = flag.String("zone", "us-central1-a", "zone")
	name     = flag.String("name", "gorogoro-vm", "vm name")
	disk     = flag.String("disk", "gorogoro-disk", "disk name")
	image    = flag.String("image", "https://www.googleapis.com/compute/v1/projects/google-containers/global/images/container-vm-v20140522", "vm image")
	api      = flag.String("api", "https://www.googleapis.com/compute/v1", "api url")
	machine  = flag.String("machine", "/zones/us-central1-a/machineTypes/f1-micro", "machine type")
	cred     = flag.String("cred", ".config/gcloud/credentials", "path to gcloud credentials")
	identity = flag.String("identity", ".ssh/google_compute_engine", "path to gcloud ssh key")
)

var credentials Credentials
var key Key

// Credentials store gcloud credentials.
type Credentials struct {
	Data []struct {
		Credential struct {
			ClientId     string `json:"Client_Id"`
			ClientSecret string `json:"Client_Secret"`
			RefreshToken string `json:"Refresh_Token"`
		}
		Key struct {
			Scope string
		}
		ProjectId string `json:"projectId"`
	}
}

// Path returns gcloud credentials path.
func (c *Credentials) path() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", fmt.Errorf("enable to get current user")
	}
	return path.Join(usr.HomeDir, *cred), nil
}

// Read reads the credentials from disk.
func (c *Credentials) Read() error {
	path, err := c.path()
	if err != nil {
		return fmt.Errorf("failed to get credentials path: %v", err)
	}
	f, err := os.Open(path)
	if err != nil {
		return fmt.Errorf("unable to load credentials from %q: %v", path, err)
	}
	defer f.Close()

	if err := json.NewDecoder(f).Decode(c); err != nil {
		return fmt.Errorf("enable to decode credentials: %v", err)
	}
	if len(c.Data) == 0 {
		return fmt.Errorf("no credentials in: %q", path)
	}
	return nil
}

// Transport return a oauth2.Transport for the gcloud credentials.
func (c *Credentials) Transport() (*oauth.Transport, error) {
	t := &oauth.Transport{
		Config: &oauth.Config{
			ClientId:     c.Data[0].Credential.ClientId,
			ClientSecret: c.Data[0].Credential.ClientSecret,
			Scope:        c.Data[0].Key.Scope,
			RedirectURL:  "oob",
			AuthURL:      "https://accounts.google.com/o/oauth2/auth",
			TokenURL:     "https://accounts.google.com/o/oauth2/token",
			AccessType:   "offline",
		},
		Token:     &oauth.Token{RefreshToken: c.Data[0].Credential.RefreshToken},
		Transport: http.DefaultTransport,
	}
	err := t.Refresh()
	if err != nil {
		return nil, fmt.Errorf("failed to refresh token: %v", err)
	}
	return t, nil
}

// Compute is a Google Compute Engine services associated to a given project and zone.
type Compute struct {
	*compute.Service
	Project string
	Zone    string
}

// NewCompute returns a new Compute service.
func NewCompute(client *http.Client) (*Compute, error) {
	service, err := compute.New(client)
	if err != nil {
		return nil, fmt.Errorf("failed to create compute service: %v", err)
	}
	return &Compute{
		service,
		*project,
		*zone,
	}, nil
}

// Disk gets or creates a new root disk.
func (c *Compute) Disk(name string) (string, error) {
	disk, err := c.Disks.Get(c.Project, c.Zone, name).Do()
	if err == nil {
		log.Printf("found existing root disk: %q", disk.SelfLink)
		return disk.SelfLink, nil
	}
	log.Printf("not found, creating new root disk: %q", name)
	op, err := c.Disks.Insert(c.Project, c.Zone, &compute.Disk{
		Name: name,
	}).SourceImage(*image).Do()
	if err != nil {
		return "", fmt.Errorf("disk insert api call failed: %v", err)
	}
	if err := c.wait(op); err != nil {
		return "", fmt.Errorf("disk insert operation failed: %v", err)
	}
	log.Printf("root disk created: %q", op.TargetLink)
	return op.TargetLink, nil
}

// Instance gets or creates a new instance.
func (c *Compute) Instance(name, disk string) (string, error) {
	instance, err := c.Instances.Get(c.Project, c.Zone, name).Do()
	if err == nil {
		log.Printf("found existing instance: %q", instance.SelfLink)
		return instance.SelfLink, nil
	}
	log.Printf("not found, creating new instance: %q", name)
	prefix := *api + "/projects/" + c.Project
	op, err := c.Instances.Insert(c.Project, c.Zone, &compute.Instance{
		Name:        name,
		Description: "gorogoro vm",
		MachineType: prefix + *machine,
		Disks: []*compute.AttachedDisk{
			{
				Boot:   true,
				Type:   "PERSISTENT",
				Mode:   "READ_WRITE",
				Source: disk,
			},
		},
		NetworkInterfaces: []*compute.NetworkInterface{
			{
				AccessConfigs: []*compute.AccessConfig{
					&compute.AccessConfig{Type: "ONE_TO_ONE_NAT"},
				},
				Network: prefix + "/global/networks/default",
			},
		},
	}).Do()
	if err != nil {
		return "", fmt.Errorf("instance insert api call failed: %v", err)
	}
	if err := c.wait(op); err != nil {
		return "", fmt.Errorf("instance insert operation failed: %v", err)
	}
	log.Printf("instance created: %q", op.TargetLink)
	return op.TargetLink, nil
}

func (c *Compute) WaitConnection(name string, port int) (string, error) {
	for {
		instance, err := c.Instances.Get(c.Project, c.Zone, name).Do()
		if err != nil {
			return "", fmt.Errorf("could not find instance %q: %v", name, err)
		}

		if ip := instance.NetworkInterfaces[0].AccessConfigs[0].NatIP; ip != "" {
			addr := fmt.Sprintf("%s:%d", ip, port)
			conn, err := net.Dial("tcp", addr)
			if err != nil {
				log.Println("connection to %q failed: %v", addr, err)
				continue
			}
			defer conn.Close()
			log.Printf("connection to %q succeeded", addr)
			return ip, nil
		}
		time.Sleep(1 * time.Second)
	}
}

func (c *Compute) wait(op *compute.Operation) error {
	for {
		op, err := c.ZoneOperations.Get(c.Project, c.Zone, op.Name).Do()
		log.Printf("operation %q status: %s, %v", op.Name, op.Status, err)
		if op.Status == "DONE" {
			if op.Error != nil {
				return fmt.Errorf("operation error: %v", *op.Error.Errors[0])
			}
			break
		}
		time.Sleep(1 * time.Second)
	}
	return nil
}

type Key struct {
	ssh.Signer
}

func (k *Key) Read() error {
	path, err := k.path()
	if err != nil {
		return fmt.Errorf("failed to get ssh key path: %v", err)
	}
	f, err := os.Open(path)
	if err != nil {
		return fmt.Errorf("unable to load ssh key from %q: %v", path, err)
	}
	defer f.Close()

	bs, err := ioutil.ReadAll(f)
	if err != nil {
		return fmt.Errorf("failed to read ssh key: %v", err)
	}
	signer, err := ssh.ParsePrivateKey(bs)
	if err != nil {
		return fmt.Errorf("failed to parse ssh key: %v", err)
	}
	k.Signer = signer
	return nil
}

func (k *Key) path() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", fmt.Errorf("enable to get current user")
	}
	return path.Join(usr.HomeDir, *identity), nil
}

func (c *Compute) Tunnel(ip string, port int) (net.Conn, error) {
	usr, err := user.Current()
	if err != nil {
		return nil, fmt.Errorf("failed to get current user: %v", err)
	}
	if err := key.Read(); err != nil {
		return nil, fmt.Errorf("failed to read ssh private key: %v", err)
	}
	raddr := ip + ":22"
	conn, err := ssh.Dial("tcp", ip+":22", &ssh.ClientConfig{
		User: usr.Username,
		Auth: []ssh.AuthMethod{
			ssh.PublicKeys(key),
		},
	})
	if err != nil {
		return nil, fmt.Errorf("failed to dial ssh conn to %q: %v", raddr, err)
	}
	laddr := fmt.Sprintf("127.0.0.1:%d", port)
	return conn.Dial("tcp", laddr)
}

func main() {
	if err := credentials.Read(); err != nil {
		log.Fatalf("failed to read credentials: %v", err)
	}
	transport, err := credentials.Transport()
	if err != nil {
		log.Fatalf("failed to create transport: %v", err)
	}
	compute, err := NewCompute(transport.Client())
	if err != nil {
		log.Fatalf("failed to create compute: %v", err)
	}
	diskUrl, err := compute.Disk(*disk)
	if err != nil {
		log.Fatalf("error creating root disk")
	}
	log.Println("disk url:", diskUrl)
	instanceUrl, err := compute.Instance(*name, diskUrl)
	if err != nil {
		log.Fatalf("error creating instance", err)
	}
	log.Println("instance url:", instanceUrl)
	ip, err := compute.WaitConnection(*name, 22)
	if err != nil {
		log.Fatalf("failed to connect to instance ssh port: %v", err)
	}
	log.Println("instance ip:", ip)
	conn, err := compute.Tunnel(ip, 4243)
	if err != nil {
		log.Fatalf("failed to create ssh tunnel to docker: %v", err)
	}
	fmt.Println(conn, err)
}
<|endoftext|>


<commit_before>package gtf

import (
	"bytes"
	"testing"
)

func AssertEqual(t *testing.T, buffer *bytes.Buffer, testString string) {
	if buffer.String() != testString {
		t.Errorf("Expected %s, got %s", testString, buffer.String())
	}
	buffer.Reset()
}

func ParseTest(buffer *bytes.Buffer, body string) {
	tpl := New("test").Funcs(GtfFuncMap)
	tpl.Parse(body)
	tpl.Execute(buffer, "")
}

func TestGtfFuncMap(t *testing.T) {
	var buffer bytes.Buffer

	ParseTest(&buffer, "{{ \"The Go Programming Language\" | stringReplace \" \" }}")
	AssertEqual(t, &buffer, "TheGoProgrammingLanguage")

	ParseTest(&buffer, "{{ \"The Go Programming Language\" | stringDefault \"default value\" }}")
	AssertEqual(t, &buffer, "The Go Programming Language")

	ParseTest(&buffer, "{{ \"\" | stringDefault \"default value\" }}")
	AssertEqual(t, &buffer, "default value")

	ParseTest(&buffer, "{{ \"The Go Programming Language\" | stringLength }}")
	AssertEqual(t, &buffer, "27")

	ParseTest(&buffer, "{{ \"The Go Programming Language\" | stringLower }}")
	AssertEqual(t, &buffer, "the go programming language")

	ParseTest(&buffer, "{{ \"The Go Programming Language\" | stringUpper }}")
	AssertEqual(t, &buffer, "THE GO PROGRAMMING LANGUAGE")

	ParseTest(&buffer, "{{ \"안녕하세요. 반갑습니다.\" | stringTruncatechars 12 }}")
	AssertEqual(t, &buffer, "안녕하세요. 반갑...")

	ParseTest(&buffer, "{{ \"The Go Programming Language\" | stringTruncatechars 12 }}")
	AssertEqual(t, &buffer, "The Go Pr...")

	ParseTest(&buffer, "{{ \"안녕하세요. The Go Programming Language\" | stringTruncatechars 30 }}")
	AssertEqual(t, &buffer, "안녕하세요. The Go Programming L...")

	ParseTest(&buffer, "{{ \"The\" | stringTruncatechars 30 }}")
	AssertEqual(t, &buffer, "The")

	ParseTest(&buffer, "{{ \"The Go Programming Language\" | stringTruncatechars 3 }}")
	AssertEqual(t, &buffer, "The")

	ParseTest(&buffer, "{{ \"The Go\" | stringTruncatechars 6 }}")
	AssertEqual(t, &buffer, "The Go")

	ParseTest(&buffer, "{{ \"The Go\" | stringTruncatechars 30 }}")
	AssertEqual(t, &buffer, "The Go")

	ParseTest(&buffer, "{{ \"The Go\" | stringTruncatechars 0 }}")
	AssertEqual(t, &buffer, "")

	ParseTest(&buffer, "{{ \"The Go\" | stringTruncatechars -1 }}")
	AssertEqual(t, &buffer, "The Go")

	ParseTest(&buffer, "{{ \"http://www.example.org/foo?a=b&c=d\" | stringUrlencode }}")
	AssertEqual(t, &buffer, "http%3A%2F%2Fwww.example.org%2Ffoo%3Fa%3Db%26c%3Dd")

	ParseTest(&buffer, "{{ \"The Go Programming Language\" | stringWordcount }}")
	AssertEqual(t, &buffer, "4")

	ParseTest(&buffer, "{{ \"      The      Go       Programming      Language        \" | stringWordcount }}")
	AssertEqual(t, &buffer, "4")

	ParseTest(&buffer, "{{ 21 | intDivisibleby 3 }}")
	AssertEqual(t, &buffer, "true")

	ParseTest(&buffer, "{{ 21 | intDivisibleby 4 }}")
	AssertEqual(t, &buffer, "false")

	ParseTest(&buffer, "{{ \"Go\" | stringLengthIs 2 }}")
	AssertEqual(t, &buffer, "true")

	ParseTest(&buffer, "{{ \"안녕하세요.\" | stringLengthIs 6 }}")
	AssertEqual(t, &buffer, "true")

	ParseTest(&buffer, "{{ \"안녕하세요. Go!\" | stringLengthIs 10 }}")
	AssertEqual(t, &buffer, "true")

	ParseTest(&buffer, "{{ \"       The Go Programming Language     \" | stringTrim }}")
	AssertEqual(t, &buffer, "The Go Programming Language")

	ParseTest(&buffer, "{{ \"the go programming language\" | stringCapfirst }}")
	AssertEqual(t, &buffer, "The go programming language")

	ParseTest(&buffer, "You have 0 message{{ 0 | intPluralize \"s\" }}")
	AssertEqual(t, &buffer, "You have 0 messages")

	ParseTest(&buffer, "You have 1 message{{ 1 | intPluralize \"s\" }}")
	AssertEqual(t, &buffer, "You have 1 message")

	ParseTest(&buffer, "0 cand{{ 0 | intPluralize \"y,ies\" }}")
	AssertEqual(t, &buffer, "0 candies")

	ParseTest(&buffer, "1 cand{{ 1 | intPluralize \"y,ies\" }}")
	AssertEqual(t, &buffer, "1 candy")

	ParseTest(&buffer, "2 cand{{ 2 | intPluralize \"y,ies\" }}")
	AssertEqual(t, &buffer, "2 candies")

	ParseTest(&buffer, "{{ 2 | intPluralize \"y,ies,s\" }}")
	AssertEqual(t, &buffer, "")

	ParseTest(&buffer, "{{ true | boolYesno \"yes~\" \"no~\" }}")
	AssertEqual(t, &buffer, "yes~")

	ParseTest(&buffer, "{{ false | boolYesno \"yes~\" \"no~\" }}")
	AssertEqual(t, &buffer, "no~")

	ParseTest(&buffer, "{{ \"Go\" | stringRjust 10 }}")
	AssertEqual(t, &buffer, "        Go")

	ParseTest(&buffer, "{{ \"안녕하세요\" | stringRjust 10 }}")
	AssertEqual(t, &buffer, "     안녕하세요")

	ParseTest(&buffer, "{{ \"Go\" | stringLjust 10 }}")
	AssertEqual(t, &buffer, "Go        ")

	ParseTest(&buffer, "{{ \"안녕하세요\" | stringLjust 10 }}")
	AssertEqual(t, &buffer, "안녕하세요     ")

	ParseTest(&buffer, "{{ \"Go\" | stringCenter 10 }}")
	AssertEqual(t, &buffer, "    Go    ")

	ParseTest(&buffer, "{{ \"안녕하세요\" | stringCenter 10 }}")
	AssertEqual(t, &buffer, "  안녕하세요   ")

	ParseTest(&buffer, "{{ 123456789 | filesizeformat }}")
	AssertEqual(t, &buffer, "117.7 MB")

	ParseTest(&buffer, "{{ 234 | filesizeformat }}")
	AssertEqual(t, &buffer, "234 bytes")

	ParseTest(&buffer, "{{ 12345 | filesizeformat }}")
	AssertEqual(t, &buffer, "12.1 KB")

	ParseTest(&buffer, "{{ 554832114 | filesizeformat }}")
	AssertEqual(t, &buffer, "529.1 MB")

	ParseTest(&buffer, "{{ 1048576 | filesizeformat }}")
	AssertEqual(t, &buffer, "1 MB")

	ParseTest(&buffer, "{{ 14868735121 | filesizeformat }}")
	AssertEqual(t, &buffer, "13.8 GB")

	ParseTest(&buffer, "{{ 14868735121365 | filesizeformat }}")
	AssertEqual(t, &buffer, "13.5 TB")

	ParseTest(&buffer, "{{ 1486873512136523 | filesizeformat }}")
	AssertEqual(t, &buffer, "1.3 PB")

	ParseTest(&buffer, "{{ 12345.35335 | filesizeformat }}")
	AssertEqual(t, &buffer, "12.1 KB")

	ParseTest(&buffer, "{{ 4294967293 | filesizeformat }}")
	AssertEqual(t, &buffer, "4 GB")
}
<commit_msg>Improve test coverage.<commit_after>package gtf

import (
	"bytes"
	"testing"
)

func AssertEqual(t *testing.T, buffer *bytes.Buffer, testString string) {
	if buffer.String() != testString {
		t.Errorf("Expected %s, got %s", testString, buffer.String())
	}
	buffer.Reset()
}

func ParseTest(buffer *bytes.Buffer, body string, data interface{}) {
	tpl := New("test").Funcs(GtfFuncMap)
	tpl.Parse(body)
	tpl.Execute(buffer, data)
}

func TestGtfFuncMap(t *testing.T) {
	var buffer bytes.Buffer

	ParseTest(&buffer, "{{ \"The Go Programming Language\" | stringReplace \" \" }}", "")
	AssertEqual(t, &buffer, "TheGoProgrammingLanguage")

	ParseTest(&buffer, "{{ \"The Go Programming Language\" | stringDefault \"default value\" }}", "")
	AssertEqual(t, &buffer, "The Go Programming Language")

	ParseTest(&buffer, "{{ \"\" | stringDefault \"default value\" }}", "")
	AssertEqual(t, &buffer, "default value")

	ParseTest(&buffer, "{{ \"The Go Programming Language\" | stringLength }}", "")
	AssertEqual(t, &buffer, "27")

	ParseTest(&buffer, "{{ \"The Go Programming Language\" | stringLower }}", "")
	AssertEqual(t, &buffer, "the go programming language")

	ParseTest(&buffer, "{{ \"The Go Programming Language\" | stringUpper }}", "")
	AssertEqual(t, &buffer, "THE GO PROGRAMMING LANGUAGE")

	ParseTest(&buffer, "{{ \"안녕하세요. 반갑습니다.\" | stringTruncatechars 12 }}", "")
	AssertEqual(t, &buffer, "안녕하세요. 반갑...")

	ParseTest(&buffer, "{{ \"The Go Programming Language\" | stringTruncatechars 12 }}", "")
	AssertEqual(t, &buffer, "The Go Pr...")

	ParseTest(&buffer, "{{ \"안녕하세요. The Go Programming Language\" | stringTruncatechars 30 }}", "")
	AssertEqual(t, &buffer, "안녕하세요. The Go Programming L...")

	ParseTest(&buffer, "{{ \"The\" | stringTruncatechars 30 }}", "")
	AssertEqual(t, &buffer, "The")

	ParseTest(&buffer, "{{ \"The Go Programming Language\" | stringTruncatechars 3 }}", "")
	AssertEqual(t, &buffer, "The")

	ParseTest(&buffer, "{{ \"The Go\" | stringTruncatechars 6 }}", "")
	AssertEqual(t, &buffer, "The Go")

	ParseTest(&buffer, "{{ \"The Go\" | stringTruncatechars 30 }}", "")
	AssertEqual(t, &buffer, "The Go")

	ParseTest(&buffer, "{{ \"The Go\" | stringTruncatechars 0 }}", "")
	AssertEqual(t, &buffer, "")

	ParseTest(&buffer, "{{ \"The Go\" | stringTruncatechars -1 }}", "")
	AssertEqual(t, &buffer, "The Go")

	ParseTest(&buffer, "{{ \"http://www.example.org/foo?a=b&c=d\" | stringUrlencode }}", "")
	AssertEqual(t, &buffer, "http%3A%2F%2Fwww.example.org%2Ffoo%3Fa%3Db%26c%3Dd")

	ParseTest(&buffer, "{{ \"The Go Programming Language\" | stringWordcount }}", "")
	AssertEqual(t, &buffer, "4")

	ParseTest(&buffer, "{{ \"      The      Go       Programming      Language        \" | stringWordcount }}", "")
	AssertEqual(t, &buffer, "4")

	ParseTest(&buffer, "{{ 21 | intDivisibleby 3 }}", "")
	AssertEqual(t, &buffer, "true")

	ParseTest(&buffer, "{{ 21 | intDivisibleby 4 }}", "")
	AssertEqual(t, &buffer, "false")

	ParseTest(&buffer, "{{ \"Go\" | stringLengthIs 2 }}", "")
	AssertEqual(t, &buffer, "true")

	ParseTest(&buffer, "{{ \"안녕하세요.\" | stringLengthIs 6 }}", "")
	AssertEqual(t, &buffer, "true")

	ParseTest(&buffer, "{{ \"안녕하세요. Go!\" | stringLengthIs 10 }}", "")
	AssertEqual(t, &buffer, "true")

	ParseTest(&buffer, "{{ \"       The Go Programming Language     \" | stringTrim }}", "")
	AssertEqual(t, &buffer, "The Go Programming Language")

	ParseTest(&buffer, "{{ \"the go programming language\" | stringCapfirst }}", "")
	AssertEqual(t, &buffer, "The go programming language")

	ParseTest(&buffer, "You have 0 message{{ 0 | intPluralize \"s\" }}", "")
	AssertEqual(t, &buffer, "You have 0 messages")

	ParseTest(&buffer, "You have 1 message{{ 1 | intPluralize \"s\" }}", "")
	AssertEqual(t, &buffer, "You have 1 message")

	ParseTest(&buffer, "0 cand{{ 0 | intPluralize \"y,ies\" }}", "")
	AssertEqual(t, &buffer, "0 candies")

	ParseTest(&buffer, "1 cand{{ 1 | intPluralize \"y,ies\" }}", "")
	AssertEqual(t, &buffer, "1 candy")

	ParseTest(&buffer, "2 cand{{ 2 | intPluralize \"y,ies\" }}", "")
	AssertEqual(t, &buffer, "2 candies")

	ParseTest(&buffer, "{{ 2 | intPluralize \"y,ies,s\" }}", "")
	AssertEqual(t, &buffer, "")

	ParseTest(&buffer, "{{ true | boolYesno \"yes~\" \"no~\" }}", "")
	AssertEqual(t, &buffer, "yes~")

	ParseTest(&buffer, "{{ false | boolYesno \"yes~\" \"no~\" }}", "")
	AssertEqual(t, &buffer, "no~")

	ParseTest(&buffer, "{{ \"Go\" | stringRjust 10 }}", "")
	AssertEqual(t, &buffer, "        Go")

	ParseTest(&buffer, "{{ \"안녕하세요\" | stringRjust 10 }}", "")
	AssertEqual(t, &buffer, "     안녕하세요")

	ParseTest(&buffer, "{{ \"Go\" | stringLjust 10 }}", "")
	AssertEqual(t, &buffer, "Go        ")

	ParseTest(&buffer, "{{ \"안녕하세요\" | stringLjust 10 }}", "")
	AssertEqual(t, &buffer, "안녕하세요     ")

	ParseTest(&buffer, "{{ \"Go\" | stringCenter 10 }}", "")
	AssertEqual(t, &buffer, "    Go    ")

	ParseTest(&buffer, "{{ \"안녕하세요\" | stringCenter 10 }}", "")
	AssertEqual(t, &buffer, "  안녕하세요   ")

	ParseTest(&buffer, "{{ 123456789 | filesizeformat }}", "")
	AssertEqual(t, &buffer, "117.7 MB")

	ParseTest(&buffer, "{{ 234 | filesizeformat }}", "")
	AssertEqual(t, &buffer, "234 bytes")

	ParseTest(&buffer, "{{ 12345 | filesizeformat }}", "")
	AssertEqual(t, &buffer, "12.1 KB")

	ParseTest(&buffer, "{{ 554832114 | filesizeformat }}", "")
	AssertEqual(t, &buffer, "529.1 MB")

	ParseTest(&buffer, "{{ 1048576 | filesizeformat }}", "")
	AssertEqual(t, &buffer, "1 MB")

	ParseTest(&buffer, "{{ 14868735121 | filesizeformat }}", "")
	AssertEqual(t, &buffer, "13.8 GB")

	ParseTest(&buffer, "{{ 14868735121365 | filesizeformat }}", "")
	AssertEqual(t, &buffer, "13.5 TB")

	ParseTest(&buffer, "{{ 1486873512136523 | filesizeformat }}", "")
	AssertEqual(t, &buffer, "1.3 PB")

	ParseTest(&buffer, "{{ 12345.35335 | filesizeformat }}", "")
	AssertEqual(t, &buffer, "12.1 KB")

	ParseTest(&buffer, "{{ 4294967293 | filesizeformat }}", "")
	AssertEqual(t, &buffer, "4 GB")

	ParseTest(&buffer, "{{ \"Go\" | filesizeformat }}", "")
	AssertEqual(t, &buffer, "")

	ParseTest(&buffer, "{{ . | filesizeformat }}", uint(500))
	AssertEqual(t, &buffer, "500 bytes")
}
<|endoftext|>


<commit_before>/*
 Parses JSON configuration files and exports the config struct for server-side
 use and the clientConfig struct, for JSON stringification and passing to the
 client
*/

package server

import (
	"io/ioutil"
	"os"
)

// config contains currently loaded configuration
var config struct {
	// Configuration that can not be hot-reloaded without restarting the server
	Hard struct {
		HTTP struct {
			Addr, Media, Upload, Socket, Origin         string
			ServeStatic, TrustProxies, Gzip, Websockets bool
		}
		Redis struct {
			Addr string
			Db   int
		}
		Rethinkdb struct {
			Addr, Db string
		}
		Dirs struct {
			Src, Thumb, Mid, Tmp string
		}
		Debug bool
	}
	Boards struct {
		Enabled []string
		Boards  map[string]struct {
			MaxThreads, MaxBump int
			Title               string
		}
		Default, Staff string
		Psuedo, Links  [][2]string
		Prune          bool
	}
	Lang struct {
		Enabled []string
		Def     string
	}
	Staff struct {
		Classes map[string]struct {
			Alias   string
			Members map[string]string
			Rights  map[string]bool
		}
		Keyword     string
		SessionTime int
	}
	Images struct {
		Max struct {
			Size, Width, Height, Pixels int
		}
		Thumb struct {
			Quality            int
			ThumbDims, MidDims [2]int
			HighQuality, PNG   bool
			PNGQuality         string
		}
		Formats struct {
			Webm, WebmAudio, MP3, SVG, PDF bool
		}
		DuplicateThreshold int
		Spoilers           []int
		Hats               bool
	}
	Posts struct {
		Salt, ExcludeRegex                       string
		ThreadCreationCooldown, MaxSubjectLength int
		ReadOnly, SageEnabled, ForcedAnon        bool
	}
	Recaptcha struct {
		Public, Private string
	}
	Banners, FAQ, Eightball                                        []string
	Schedule                                                       [][3]string
	Radio, Pyu, IllyaDance                                         bool
	FeedbackEmail, DefaultCSS, Frontpage, InfoBanner, InjectJSPath string
}

// clientConfig exports public settings all clients can access
var clientConfig struct {
	Hard struct {
		HTTP struct {
			Media      string `json:"media"`
			Upload     string `json:"upload"`
			Socket     string `json:"socket"`
			Websockets bool   `json:"websockets"`
		} `json:"HTTP"`
		Debug bool `json:"debug"`
	} `json:"hard"`
	Boards struct {
		Enabled []string `json:"enabled"`
		Boards  map[string]struct {
			Title string `json:"title"`
		} `json:"boards"`
		Default string      `json:"default"`
		Psuedo  [][2]string `json:"psuedo"`
		Links   [][2]string `json:"links"`
	} `json:"boards"`
	Lang struct {
		Enabled []string `json:"enabled"`
		Default string   `json:"default"`
	} `json:"lang"`
	Staff struct {
		Classes map[string]struct {
			Alias  string          `json:"alias"`
			Rights map[string]bool `json:"rights"`
		} `json:"classes"`
		Keyword string `json:"keyword"`
	} `json:"staff"`
	Images struct {
		thumb struct {
			thumbDims [2]int `json:"thumbDims"`
			MidDims   [2]int `json:"midDims"`
		}
		Spoilers []int `json:"spoilers"`
		Hats     bool  `json:"hats"`
	} `json:"images"`
	Banners       []string    `json:"banners"`
	FAQ           []string    `json:"FAQ"`
	Eightball     []string    `json:"eightball"`
	Schedule      [][3]string `json:"schedule"`
	Radio         bool        `json:"radio"`
	IllyaDance    bool        `json:"illiyaDance"`
	FeedbackEmail string      `json:"feedbackEmail"`
	DefaultCSS    string      `json:"defaultCSS"`
	InfoBanner    string      `json:"infoBanner"`
}

// configHash is the truncated MD5 hash of the JSON configuration file
var configHash string

// loadConfig reads and parses JSON config files
func loadConfig() {
	const path = "./config/config.json"
	file, err := ioutil.ReadFile(path)

	// If config file does not exist, read and copy defaults file
	if err != nil {
		if os.IsNotExist(err) {
			file, err = ioutil.ReadFile("./config/defaults.json")
			throw(err)
			throw(ioutil.WriteFile(path, file, 0600))
		} else {
			panic(err)
		}
	}

	unmarshalJSON(file, &config)
	unmarshalJSON(file, &clientConfig)
	configHash = hashBuffer(file)
}
<commit_msg>server/config.go Properly export thumbanil dimentions<commit_after>/*
 Parses JSON configuration files and exports the config struct for server-side
 use and the clientConfig struct, for JSON stringification and passing to the
 client
*/

package server

import (
	"io/ioutil"
	"os"
)

// config contains currently loaded configuration
var config struct {
	// Configuration that can not be hot-reloaded without restarting the server
	Hard struct {
		HTTP struct {
			Addr, Media, Upload, Socket, Origin         string
			ServeStatic, TrustProxies, Gzip, Websockets bool
		}
		Redis struct {
			Addr string
			Db   int
		}
		Rethinkdb struct {
			Addr, Db string
		}
		Dirs struct {
			Src, Thumb, Mid, Tmp string
		}
		Debug bool
	}
	Boards struct {
		Enabled []string
		Boards  map[string]struct {
			MaxThreads, MaxBump int
			Title               string
		}
		Default, Staff string
		Psuedo, Links  [][2]string
		Prune          bool
	}
	Lang struct {
		Enabled []string
		Def     string
	}
	Staff struct {
		Classes map[string]struct {
			Alias   string
			Members map[string]string
			Rights  map[string]bool
		}
		Keyword     string
		SessionTime int
	}
	Images struct {
		Max struct {
			Size, Width, Height, Pixels int
		}
		Thumb struct {
			Quality            int
			ThumbDims, MidDims [2]int
			HighQuality, PNG   bool
			PNGQuality         string
		}
		Formats struct {
			Webm, WebmAudio, MP3, SVG, PDF bool
		}
		DuplicateThreshold int
		Spoilers           []int
		Hats               bool
	}
	Posts struct {
		Salt, ExcludeRegex                       string
		ThreadCreationCooldown, MaxSubjectLength int
		ReadOnly, SageEnabled, ForcedAnon        bool
	}
	Recaptcha struct {
		Public, Private string
	}
	Banners, FAQ, Eightball                                        []string
	Schedule                                                       [][3]string
	Radio, Pyu, IllyaDance                                         bool
	FeedbackEmail, DefaultCSS, Frontpage, InfoBanner, InjectJSPath string
}

// clientConfig exports public settings all clients can access
var clientConfig struct {
	Hard struct {
		HTTP struct {
			Media      string `json:"media"`
			Upload     string `json:"upload"`
			Socket     string `json:"socket"`
			Websockets bool   `json:"websockets"`
		} `json:"HTTP"`
		Debug bool `json:"debug"`
	} `json:"hard"`
	Boards struct {
		Enabled []string `json:"enabled"`
		Boards  map[string]struct {
			Title string `json:"title"`
		} `json:"boards"`
		Default string      `json:"default"`
		Psuedo  [][2]string `json:"psuedo"`
		Links   [][2]string `json:"links"`
	} `json:"boards"`
	Lang struct {
		Enabled []string `json:"enabled"`
		Default string   `json:"default"`
	} `json:"lang"`
	Staff struct {
		Classes map[string]struct {
			Alias  string          `json:"alias"`
			Rights map[string]bool `json:"rights"`
		} `json:"classes"`
		Keyword string `json:"keyword"`
	} `json:"staff"`
	Images struct {
		thumb struct {
			ThumbDims [2]int `json:"thumbDims"`
			MidDims   [2]int `json:"midDims"`
		}
		Spoilers []int `json:"spoilers"`
		Hats     bool  `json:"hats"`
	} `json:"images"`
	Banners       []string    `json:"banners"`
	FAQ           []string    `json:"FAQ"`
	Eightball     []string    `json:"eightball"`
	Schedule      [][3]string `json:"schedule"`
	Radio         bool        `json:"radio"`
	IllyaDance    bool        `json:"illiyaDance"`
	FeedbackEmail string      `json:"feedbackEmail"`
	DefaultCSS    string      `json:"defaultCSS"`
	InfoBanner    string      `json:"infoBanner"`
}

// configHash is the truncated MD5 hash of the JSON configuration file
var configHash string

// loadConfig reads and parses JSON config files
func loadConfig() {
	const path = "./config/config.json"
	file, err := ioutil.ReadFile(path)

	// If config file does not exist, read and copy defaults file
	if err != nil {
		if os.IsNotExist(err) {
			file, err = ioutil.ReadFile("./config/defaults.json")
			throw(err)
			throw(ioutil.WriteFile(path, file, 0600))
		} else {
			panic(err)
		}
	}

	unmarshalJSON(file, &config)
	unmarshalJSON(file, &clientConfig)
	configHash = hashBuffer(file)
}
<|endoftext|>


<commit_before>// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package user

import (
	"fmt"
	"strings"

	"code.gitea.io/gitea/models/db"
	"code.gitea.io/gitea/modules/setting"
	"code.gitea.io/gitea/modules/structs"
	"code.gitea.io/gitea/modules/util"

	"xorm.io/builder"
	"xorm.io/xorm"
)

// SearchUserOptions contains the options for searching
type SearchUserOptions struct {
	db.ListOptions
	Keyword       string
	Type          UserType
	UID           int64
	OrderBy       db.SearchOrderBy
	Visible       []structs.VisibleType
	Actor         *User // The user doing the search
	SearchByEmail bool  // Search by email as well as username/full name

	IsActive           util.OptionalBool
	IsAdmin            util.OptionalBool
	IsRestricted       util.OptionalBool
	IsTwoFactorEnabled util.OptionalBool
	IsProhibitLogin    util.OptionalBool
}

func (opts *SearchUserOptions) toSearchQueryBase() *xorm.Session {
	var cond builder.Cond = builder.Eq{"type": opts.Type}
	if len(opts.Keyword) > 0 {
		lowerKeyword := strings.ToLower(opts.Keyword)
		keywordCond := builder.Or(
			builder.Like{"lower_name", lowerKeyword},
			builder.Like{"LOWER(full_name)", lowerKeyword},
		)
		if opts.SearchByEmail {
			keywordCond = keywordCond.Or(builder.Like{"LOWER(email)", lowerKeyword})
		}

		cond = cond.And(keywordCond)
	}

	// If visibility filtered
	if len(opts.Visible) > 0 {
		cond = cond.And(builder.In("visibility", opts.Visible))
	}

	if opts.Actor != nil {
		var exprCond builder.Cond = builder.Expr("org_user.org_id = `user`.id")

		// If Admin - they see all users!
		if !opts.Actor.IsAdmin {
			// Force visibility for privacy
			var accessCond builder.Cond
			if !opts.Actor.IsRestricted {
				accessCond = builder.Or(
					builder.In("id", builder.Select("org_id").From("org_user").LeftJoin("`user`", exprCond).Where(builder.And(builder.Eq{"uid": opts.Actor.ID}, builder.Eq{"visibility": structs.VisibleTypePrivate}))),
					builder.In("visibility", structs.VisibleTypePublic, structs.VisibleTypeLimited))
			} else {
				// restricted users only see orgs they are a member of
				accessCond = builder.In("id", builder.Select("org_id").From("org_user").LeftJoin("`user`", exprCond).Where(builder.And(builder.Eq{"uid": opts.Actor.ID})))
			}
			// Don't forget about self
			accessCond = accessCond.Or(builder.Eq{"id": opts.Actor.ID})
			cond = cond.And(accessCond)
		}

	} else {
		// Force visibility for privacy
		// Not logged in - only public users
		cond = cond.And(builder.In("visibility", structs.VisibleTypePublic))
	}

	if opts.UID > 0 {
		cond = cond.And(builder.Eq{"id": opts.UID})
	}

	if !opts.IsActive.IsNone() {
		cond = cond.And(builder.Eq{"is_active": opts.IsActive.IsTrue()})
	}

	if !opts.IsAdmin.IsNone() {
		cond = cond.And(builder.Eq{"is_admin": opts.IsAdmin.IsTrue()})
	}

	if !opts.IsRestricted.IsNone() {
		cond = cond.And(builder.Eq{"is_restricted": opts.IsRestricted.IsTrue()})
	}

	if !opts.IsProhibitLogin.IsNone() {
		cond = cond.And(builder.Eq{"prohibit_login": opts.IsProhibitLogin.IsTrue()})
	}

	e := db.GetEngine(db.DefaultContext)
	if opts.IsTwoFactorEnabled.IsNone() {
		return e.Where(cond)
	}

	// 2fa filter uses LEFT JOIN to check whether a user has a 2fa record
	// TODO: bad performance here, maybe there will be a column "is_2fa_enabled" in the future
	if opts.IsTwoFactorEnabled.IsTrue() {
		cond = cond.And(builder.Expr("two_factor.uid IS NOT NULL"))
	} else {
		cond = cond.And(builder.Expr("two_factor.uid IS NULL"))
	}

	return e.Join("LEFT OUTER", "two_factor", "two_factor.uid = `user`.id").
		Where(cond)
}

// SearchUsers takes options i.e. keyword and part of user name to search,
// it returns results in given range and number of total results.
func SearchUsers(opts *SearchUserOptions) (users []*User, _ int64, _ error) {
	sessCount := opts.toSearchQueryBase()
	defer sessCount.Close()
	count, err := sessCount.Count(new(User))
	if err != nil {
		return nil, 0, fmt.Errorf("Count: %v", err)
	}

	if len(opts.OrderBy) == 0 {
		opts.OrderBy = db.SearchOrderByAlphabetically
	}

	sessQuery := opts.toSearchQueryBase().OrderBy(opts.OrderBy.String())
	defer sessQuery.Close()
	if opts.Page != 0 {
		sessQuery = db.SetSessionPagination(sessQuery, opts)
	}

	// the sql may contain JOIN, so we must only select User related columns
	sessQuery = sessQuery.Select("`user`.*")
	users = make([]*User, 0, opts.PageSize)
	return users, count, sessQuery.Find(&users)
}

// IterateUser iterate users
func IterateUser(f func(user *User) error) error {
	var start int
	batchSize := setting.Database.IterateBufferSize
	for {
		users := make([]*User, 0, batchSize)
		if err := db.GetEngine(db.DefaultContext).Limit(batchSize, start).Find(&users); err != nil {
			return err
		}
		if len(users) == 0 {
			return nil
		}
		start += len(users)

		for _, user := range users {
			if err := f(user); err != nil {
				return err
			}
		}
	}
}
<commit_msg>Improve the comment for 2FA filter in admin panel (#18017)<commit_after>// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package user

import (
	"fmt"
	"strings"

	"code.gitea.io/gitea/models/db"
	"code.gitea.io/gitea/modules/setting"
	"code.gitea.io/gitea/modules/structs"
	"code.gitea.io/gitea/modules/util"

	"xorm.io/builder"
	"xorm.io/xorm"
)

// SearchUserOptions contains the options for searching
type SearchUserOptions struct {
	db.ListOptions
	Keyword       string
	Type          UserType
	UID           int64
	OrderBy       db.SearchOrderBy
	Visible       []structs.VisibleType
	Actor         *User // The user doing the search
	SearchByEmail bool  // Search by email as well as username/full name

	IsActive           util.OptionalBool
	IsAdmin            util.OptionalBool
	IsRestricted       util.OptionalBool
	IsTwoFactorEnabled util.OptionalBool
	IsProhibitLogin    util.OptionalBool
}

func (opts *SearchUserOptions) toSearchQueryBase() *xorm.Session {
	var cond builder.Cond = builder.Eq{"type": opts.Type}
	if len(opts.Keyword) > 0 {
		lowerKeyword := strings.ToLower(opts.Keyword)
		keywordCond := builder.Or(
			builder.Like{"lower_name", lowerKeyword},
			builder.Like{"LOWER(full_name)", lowerKeyword},
		)
		if opts.SearchByEmail {
			keywordCond = keywordCond.Or(builder.Like{"LOWER(email)", lowerKeyword})
		}

		cond = cond.And(keywordCond)
	}

	// If visibility filtered
	if len(opts.Visible) > 0 {
		cond = cond.And(builder.In("visibility", opts.Visible))
	}

	if opts.Actor != nil {
		var exprCond builder.Cond = builder.Expr("org_user.org_id = `user`.id")

		// If Admin - they see all users!
		if !opts.Actor.IsAdmin {
			// Force visibility for privacy
			var accessCond builder.Cond
			if !opts.Actor.IsRestricted {
				accessCond = builder.Or(
					builder.In("id", builder.Select("org_id").From("org_user").LeftJoin("`user`", exprCond).Where(builder.And(builder.Eq{"uid": opts.Actor.ID}, builder.Eq{"visibility": structs.VisibleTypePrivate}))),
					builder.In("visibility", structs.VisibleTypePublic, structs.VisibleTypeLimited))
			} else {
				// restricted users only see orgs they are a member of
				accessCond = builder.In("id", builder.Select("org_id").From("org_user").LeftJoin("`user`", exprCond).Where(builder.And(builder.Eq{"uid": opts.Actor.ID})))
			}
			// Don't forget about self
			accessCond = accessCond.Or(builder.Eq{"id": opts.Actor.ID})
			cond = cond.And(accessCond)
		}

	} else {
		// Force visibility for privacy
		// Not logged in - only public users
		cond = cond.And(builder.In("visibility", structs.VisibleTypePublic))
	}

	if opts.UID > 0 {
		cond = cond.And(builder.Eq{"id": opts.UID})
	}

	if !opts.IsActive.IsNone() {
		cond = cond.And(builder.Eq{"is_active": opts.IsActive.IsTrue()})
	}

	if !opts.IsAdmin.IsNone() {
		cond = cond.And(builder.Eq{"is_admin": opts.IsAdmin.IsTrue()})
	}

	if !opts.IsRestricted.IsNone() {
		cond = cond.And(builder.Eq{"is_restricted": opts.IsRestricted.IsTrue()})
	}

	if !opts.IsProhibitLogin.IsNone() {
		cond = cond.And(builder.Eq{"prohibit_login": opts.IsProhibitLogin.IsTrue()})
	}

	e := db.GetEngine(db.DefaultContext)
	if opts.IsTwoFactorEnabled.IsNone() {
		return e.Where(cond)
	}

	// 2fa filter uses LEFT JOIN to check whether a user has a 2fa record
	// While using LEFT JOIN, sometimes the performance might not be good, but it won't be a problem now, such SQL is seldom executed.
	// There are some possible methods to refactor this SQL in future when we really need to optimize the performance (but not now):
	// (1) add a column in user table (2) add a setting value in user_setting table (3) use search engines (bleve/elasticsearch)
	if opts.IsTwoFactorEnabled.IsTrue() {
		cond = cond.And(builder.Expr("two_factor.uid IS NOT NULL"))
	} else {
		cond = cond.And(builder.Expr("two_factor.uid IS NULL"))
	}

	return e.Join("LEFT OUTER", "two_factor", "two_factor.uid = `user`.id").
		Where(cond)
}

// SearchUsers takes options i.e. keyword and part of user name to search,
// it returns results in given range and number of total results.
func SearchUsers(opts *SearchUserOptions) (users []*User, _ int64, _ error) {
	sessCount := opts.toSearchQueryBase()
	defer sessCount.Close()
	count, err := sessCount.Count(new(User))
	if err != nil {
		return nil, 0, fmt.Errorf("Count: %v", err)
	}

	if len(opts.OrderBy) == 0 {
		opts.OrderBy = db.SearchOrderByAlphabetically
	}

	sessQuery := opts.toSearchQueryBase().OrderBy(opts.OrderBy.String())
	defer sessQuery.Close()
	if opts.Page != 0 {
		sessQuery = db.SetSessionPagination(sessQuery, opts)
	}

	// the sql may contain JOIN, so we must only select User related columns
	sessQuery = sessQuery.Select("`user`.*")
	users = make([]*User, 0, opts.PageSize)
	return users, count, sessQuery.Find(&users)
}

// IterateUser iterate users
func IterateUser(f func(user *User) error) error {
	var start int
	batchSize := setting.Database.IterateBufferSize
	for {
		users := make([]*User, 0, batchSize)
		if err := db.GetEngine(db.DefaultContext).Limit(batchSize, start).Find(&users); err != nil {
			return err
		}
		if len(users) == 0 {
			return nil
		}
		start += len(users)

		for _, user := range users {
			if err := f(user); err != nil {
				return err
			}
		}
	}
}
<|endoftext|>