<commit_before>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file contains the built-in primitive functions.

package golisp

import (
    "errors"
    "fmt"
)

func RegisterSpecialFormPrimitives() {
    MakePrimitiveFunction("cond", -1, CondImpl)
    MakePrimitiveFunction("case", -1, CaseImpl)
    MakePrimitiveFunction("if", -1, IfImpl)
    MakePrimitiveFunction("lambda", -1, LambdaImpl)
    MakePrimitiveFunction("define", -1, DefineImpl)
    MakePrimitiveFunction("defun", -1, DefunImpl)
    MakePrimitiveFunction("defmacro", -1, DefmacroImpl)
    MakePrimitiveFunction("let", -1, LetImpl)
    MakePrimitiveFunction("begin", -1, BeginImpl)
    MakePrimitiveFunction("do", -1, DoImpl)
    MakePrimitiveFunction("apply", 2, ApplyImpl)
    MakePrimitiveFunction("eval", 1, EvalImpl)
    MakePrimitiveFunction("->", -1, ChainImpl)
    MakePrimitiveFunction("=>", -1, TapImpl)
}

func CondImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    var condition *Data
    for c := args; NotNilP(c); c = Cdr(c) {
        clause := Car(c)
        if !PairP(clause) {
            err = errors.New("Cond expect a sequence of clauses that are lists")
            return
        }
        condition, err = Eval(Car(clause), env)
        if err != nil {
            return
        }
        if BooleanValue(condition) || StringValue(Car(clause)) == "else" {
            for e := Cdr(clause); NotNilP(e); e = Cdr(e) {
                result, err = Eval(Car(e), env)
                if err != nil {
                    return
                }
            }
            return
        }
    }
    return
}

func evalList(l *Data, env *SymbolTableFrame) (result *Data, err error) {
    for sexpr := l; NotNilP(sexpr); sexpr = Cdr(sexpr) {
        result, err = Eval(Car(sexpr), env)
        if err != nil {
            return
        }
    }
    return
}

func CaseImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    var keyValue *Data
    var targetValue *Data

    keyValue, err = Eval(Car(args), env)
    if err != nil {
        return
    }

    for clauseCell := Cdr(args); NotNilP(clauseCell); clauseCell = Cdr(clauseCell) {
        clause := Car(clauseCell)
        if PairP(clause) {
            if IsEqual(Car(clause), SymbolWithName("else")) {
                return evalList(Cdr(clause), env)
            } else {
                targetValue, err = Eval(Car(clause), env)
                if IsEqual(targetValue, keyValue) {
                    return evalList(Cdr(clause), env)
                }
            }
        } else {
            err = errors.New("Case requires non-atomic clauses")
            return
        }
    }

    return
}

func IfImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) < 2 || Length(args) > 3 {
        err = errors.New(fmt.Sprintf("IF requires 2 or 3 arguments. Received %d.", Length(args)))
        return
    }

    c, err := Eval(Car(args), env)
    if err != nil {
        return
    }
    condition := BooleanValue(c)
    thenClause := Second(args)
    elseClause := Third(args)

    if condition {
        return Eval(thenClause, env)
    } else {
        return Eval(elseClause, env)
    }
}

func LambdaImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    params := Car(args)
    body := Cdr(args)
    return FunctionWithNameParamsBodyAndParent("anonymous", params, body, env), nil
}

func DefineImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    var value *Data
    thing := Car(args)
    if SymbolP(thing) {
        value, err = Eval(Cadr(args), env)
        if err != nil {
            return
        }
    } else if PairP(thing) {
        name := Car(thing)
        params := Cdr(thing)
        thing = name
        if !SymbolP(name) {
            err = errors.New("Function name has to be a symbol")
            return
        }
        body := Cdr(args)
        value = FunctionWithNameParamsBodyAndParent(StringValue(name), params, body, env)
    } else {
        err = errors.New("Invalid definition")
        return
    }
    env.BindLocallyTo(thing, value)
    return value, nil
}

func DefunImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    var value *Data
    thing := Car(args)
    if PairP(thing) {
        name := Car(thing)
        params := Cdr(thing)
        thing = name
        if !SymbolP(name) {
            err = errors.New("Function name has to be a symbol")
            return
        }
        body := Cdr(args)
        value = FunctionWithNameParamsBodyAndParent(StringValue(name), params, body, env)
    } else {
        err = errors.New("Invalid function definition")
        return
    }
    env.BindLocallyTo(thing, value)
    return value, nil
}

func DefmacroImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    var value *Data
    thing := Car(args)
    if PairP(thing) {
        name := Car(thing)
        params := Cdr(thing)
        thing = name
        if !SymbolP(name) {
            err = errors.New("Macro name has to be a symbol")
            return
        }
        body := Cadr(args)
        value = MacroWithNameParamsBodyAndParent(StringValue(name), params, body, env)
    } else {
        err = errors.New("Invalid macro definition")
        return
    }
    env.BindLocallyTo(thing, value)
    return value, nil
}

func bindLetLocals(bindingForms *Data, env *SymbolTableFrame) (err error) {
    var name *Data
    var value *Data

    for cell := bindingForms; NotNilP(cell); cell = Cdr(cell) {
        bindingPair := Car(cell)
        if !PairP(bindingPair) {
            err = errors.New("Let requires a list of bindings (with are pairs) as it's first argument")
            return
        }
        name = Car(bindingPair)
        if !SymbolP(name) {
            err = errors.New("First part of a let binding pair must be a symbol")
        }
        value, err = Eval(Cadr(bindingPair), env)
        if err != nil {
            return
        }
        env.BindLocallyTo(name, value)
    }
    return
}

func LetImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) < 1 {
        err = errors.New("Let requires at least a list of bindings")
        return
    }

    if !PairP(Car(args)) {
        err = errors.New("Let requires a list of bindings as it's first argument")
        return
    }

    localFrame := NewSymbolTableFrameBelow(env)
    bindLetLocals(Car(args), localFrame)

    for cell := Cdr(args); NotNilP(cell); cell = Cdr(cell) {
        sexpr := Car(cell)
        result, err = Eval(sexpr, localFrame)
        if err != nil {
            return
        }
    }

    return
}

func BeginImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    for cell := args; NotNilP(cell); cell = Cdr(cell) {
        sexpr := Car(cell)
        result, err = Eval(sexpr, env)
        if err != nil {
            return
        }
    }
    return
}

func rebindDoLocals(bindingForms *Data, env *SymbolTableFrame) (err error) {
    var name *Data
    var value *Data

    for cell := bindingForms; NotNilP(cell); cell = Cdr(cell) {
        bindingTuple := Car(cell)
        name = First(bindingTuple)
        if NotNilP(Third(bindingTuple)) {
            value, err = Eval(Third(bindingTuple), env)
            if err != nil {
                return
            }
            env.BindLocallyTo(name, value)
        }
    }
    return
}

func DoImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) < 2 {
        err = errors.New("Do requires at least a list of bindings and a test clause")
        return
    }

    bindings := Car(args)
    if !PairP(bindings) {
        err = errors.New("Do requires a list of bindings as it's first argument")
        return
    }

    testClause := Cadr(args)
    if !PairP(testClause) {
        err = errors.New("Do requires a list as it's second argument")
        return
    }

    localFrame := NewSymbolTableFrameBelow(env)
    bindLetLocals(bindings, localFrame)

    body := Cddr(args)

    var shouldExit *Data

    for true {
        shouldExit, err = Eval(Car(testClause), localFrame)
        if err != nil {
            return
        }

        if BooleanValue(shouldExit) {
            for cell := Cdr(testClause); NotNilP(cell); cell = Cdr(cell) {
                sexpr := Car(cell)
                result, err = Eval(sexpr, localFrame)
                if err != nil {
                    return
                }
            }
            return
        }

        for cell := body; NotNilP(cell); cell = Cdr(cell) {
            sexpr := Car(cell)
            result, err = Eval(sexpr, localFrame)
            if err != nil {
                return
            }
        }

        rebindDoLocals(bindings, localFrame)
    }
    return
}

func ApplyImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    f, err := Eval(Car(args), env)
    if err != nil {
        return
    }

    vals, err := Eval(Cadr(args), env)
    if err != nil {
        return
    }

    return Apply(f, vals, env)
}

func EvalImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    val, err := Eval(Car(args), env)
    if err != nil {
        return
    }

    return Eval(val, env)
}

func ChainImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) == 0 {
        err = errors.New("-> requires at least an initial value.")
        return
    }

    var value *Data

    value, err = Eval(Car(args), env)
    if err != nil {
        return
    }

    for cell := Cdr(args); NotNilP(cell); cell = Cdr(cell) {
        sexpr := Car(cell)
        var newExpr *Data
        if ListP(sexpr) {
            newExpr = Cons(Car(sexpr), Cons(value, Cdr(sexpr)))
        } else {
            newExpr = Cons(sexpr, Cons(value, nil))
        }
        value, err = Eval(newExpr, env)
        if err != nil {
            return
        }
    }
    result = value
    return
}

func TapImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) == 0 {
        err = errors.New("tap requires at least an initial value.")
        return
    }

    var value *Data

    value, err = Eval(Car(args), env)
    if err != nil {
        return
    }
    result = value

    for cell := Cdr(args); NotNilP(cell); cell = Cdr(cell) {
        sexpr := Car(cell)
        var newExpr *Data
        if ListP(sexpr) {
            newExpr = Cons(Car(sexpr), Cons(value, Cdr(sexpr)))
        } else {
            newExpr = Cons(sexpr, Cons(value, nil))
        }
        _, err = Eval(newExpr, env)
        if err != nil {
            return
        }
    }
    return
}
<commit_msg>Removed defun<commit_after>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file contains the built-in primitive functions.

package golisp

import (
    "errors"
    "fmt"
)

func RegisterSpecialFormPrimitives() {
    MakePrimitiveFunction("cond", -1, CondImpl)
    MakePrimitiveFunction("case", -1, CaseImpl)
    MakePrimitiveFunction("if", -1, IfImpl)
    MakePrimitiveFunction("lambda", -1, LambdaImpl)
    MakePrimitiveFunction("define", -1, DefineImpl)
    MakePrimitiveFunction("defmacro", -1, DefmacroImpl)
    MakePrimitiveFunction("let", -1, LetImpl)
    MakePrimitiveFunction("begin", -1, BeginImpl)
    MakePrimitiveFunction("do", -1, DoImpl)
    MakePrimitiveFunction("apply", 2, ApplyImpl)
    MakePrimitiveFunction("eval", 1, EvalImpl)
    MakePrimitiveFunction("->", -1, ChainImpl)
    MakePrimitiveFunction("=>", -1, TapImpl)
}

func CondImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    var condition *Data
    for c := args; NotNilP(c); c = Cdr(c) {
        clause := Car(c)
        if !PairP(clause) {
            err = errors.New("Cond expect a sequence of clauses that are lists")
            return
        }
        condition, err = Eval(Car(clause), env)
        if err != nil {
            return
        }
        if BooleanValue(condition) || StringValue(Car(clause)) == "else" {
            for e := Cdr(clause); NotNilP(e); e = Cdr(e) {
                result, err = Eval(Car(e), env)
                if err != nil {
                    return
                }
            }
            return
        }
    }
    return
}

func evalList(l *Data, env *SymbolTableFrame) (result *Data, err error) {
    for sexpr := l; NotNilP(sexpr); sexpr = Cdr(sexpr) {
        result, err = Eval(Car(sexpr), env)
        if err != nil {
            return
        }
    }
    return
}

func CaseImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    var keyValue *Data
    var targetValue *Data

    keyValue, err = Eval(Car(args), env)
    if err != nil {
        return
    }

    for clauseCell := Cdr(args); NotNilP(clauseCell); clauseCell = Cdr(clauseCell) {
        clause := Car(clauseCell)
        if PairP(clause) {
            if IsEqual(Car(clause), SymbolWithName("else")) {
                return evalList(Cdr(clause), env)
            } else {
                targetValue, err = Eval(Car(clause), env)
                if IsEqual(targetValue, keyValue) {
                    return evalList(Cdr(clause), env)
                }
            }
        } else {
            err = errors.New("Case requires non-atomic clauses")
            return
        }
    }

    return
}

func IfImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) < 2 || Length(args) > 3 {
        err = errors.New(fmt.Sprintf("IF requires 2 or 3 arguments. Received %d.", Length(args)))
        return
    }

    c, err := Eval(Car(args), env)
    if err != nil {
        return
    }
    condition := BooleanValue(c)
    thenClause := Second(args)
    elseClause := Third(args)

    if condition {
        return Eval(thenClause, env)
    } else {
        return Eval(elseClause, env)
    }
}

func LambdaImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    params := Car(args)
    body := Cdr(args)
    return FunctionWithNameParamsBodyAndParent("anonymous", params, body, env), nil
}

func DefineImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    var value *Data
    thing := Car(args)
    if SymbolP(thing) {
        value, err = Eval(Cadr(args), env)
        if err != nil {
            return
        }
    } else if PairP(thing) {
        name := Car(thing)
        params := Cdr(thing)
        thing = name
        if !SymbolP(name) {
            err = errors.New("Function name has to be a symbol")
            return
        }
        body := Cdr(args)
        value = FunctionWithNameParamsBodyAndParent(StringValue(name), params, body, env)
    } else {
        err = errors.New("Invalid definition")
        return
    }
    env.BindLocallyTo(thing, value)
    return value, nil
}

func DefmacroImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    var value *Data
    thing := Car(args)
    if PairP(thing) {
        name := Car(thing)
        params := Cdr(thing)
        thing = name
        if !SymbolP(name) {
            err = errors.New("Macro name has to be a symbol")
            return
        }
        body := Cadr(args)
        value = MacroWithNameParamsBodyAndParent(StringValue(name), params, body, env)
    } else {
        err = errors.New("Invalid macro definition")
        return
    }
    env.BindLocallyTo(thing, value)
    return value, nil
}

func bindLetLocals(bindingForms *Data, env *SymbolTableFrame) (err error) {
    var name *Data
    var value *Data

    for cell := bindingForms; NotNilP(cell); cell = Cdr(cell) {
        bindingPair := Car(cell)
        if !PairP(bindingPair) {
            err = errors.New("Let requires a list of bindings (with are pairs) as it's first argument")
            return
        }
        name = Car(bindingPair)
        if !SymbolP(name) {
            err = errors.New("First part of a let binding pair must be a symbol")
        }
        value, err = Eval(Cadr(bindingPair), env)
        if err != nil {
            return
        }
        env.BindLocallyTo(name, value)
    }
    return
}

func LetImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) < 1 {
        err = errors.New("Let requires at least a list of bindings")
        return
    }

    if !PairP(Car(args)) {
        err = errors.New("Let requires a list of bindings as it's first argument")
        return
    }

    localFrame := NewSymbolTableFrameBelow(env)
    bindLetLocals(Car(args), localFrame)

    for cell := Cdr(args); NotNilP(cell); cell = Cdr(cell) {
        sexpr := Car(cell)
        result, err = Eval(sexpr, localFrame)
        if err != nil {
            return
        }
    }

    return
}

func BeginImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    for cell := args; NotNilP(cell); cell = Cdr(cell) {
        sexpr := Car(cell)
        result, err = Eval(sexpr, env)
        if err != nil {
            return
        }
    }
    return
}

func rebindDoLocals(bindingForms *Data, env *SymbolTableFrame) (err error) {
    var name *Data
    var value *Data

    for cell := bindingForms; NotNilP(cell); cell = Cdr(cell) {
        bindingTuple := Car(cell)
        name = First(bindingTuple)
        if NotNilP(Third(bindingTuple)) {
            value, err = Eval(Third(bindingTuple), env)
            if err != nil {
                return
            }
            env.BindLocallyTo(name, value)
        }
    }
    return
}

func DoImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) < 2 {
        err = errors.New("Do requires at least a list of bindings and a test clause")
        return
    }

    bindings := Car(args)
    if !PairP(bindings) {
        err = errors.New("Do requires a list of bindings as it's first argument")
        return
    }

    testClause := Cadr(args)
    if !PairP(testClause) {
        err = errors.New("Do requires a list as it's second argument")
        return
    }

    localFrame := NewSymbolTableFrameBelow(env)
    bindLetLocals(bindings, localFrame)

    body := Cddr(args)

    var shouldExit *Data

    for true {
        shouldExit, err = Eval(Car(testClause), localFrame)
        if err != nil {
            return
        }

        if BooleanValue(shouldExit) {
            for cell := Cdr(testClause); NotNilP(cell); cell = Cdr(cell) {
                sexpr := Car(cell)
                result, err = Eval(sexpr, localFrame)
                if err != nil {
                    return
                }
            }
            return
        }

        for cell := body; NotNilP(cell); cell = Cdr(cell) {
            sexpr := Car(cell)
            result, err = Eval(sexpr, localFrame)
            if err != nil {
                return
            }
        }

        rebindDoLocals(bindings, localFrame)
    }
    return
}

func ApplyImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    f, err := Eval(Car(args), env)
    if err != nil {
        return
    }

    vals, err := Eval(Cadr(args), env)
    if err != nil {
        return
    }

    return Apply(f, vals, env)
}

func EvalImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    val, err := Eval(Car(args), env)
    if err != nil {
        return
    }

    return Eval(val, env)
}

func ChainImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) == 0 {
        err = errors.New("-> requires at least an initial value.")
        return
    }

    var value *Data

    value, err = Eval(Car(args), env)
    if err != nil {
        return
    }

    for cell := Cdr(args); NotNilP(cell); cell = Cdr(cell) {
        sexpr := Car(cell)
        var newExpr *Data
        if ListP(sexpr) {
            newExpr = Cons(Car(sexpr), Cons(value, Cdr(sexpr)))
        } else {
            newExpr = Cons(sexpr, Cons(value, nil))
        }
        value, err = Eval(newExpr, env)
        if err != nil {
            return
        }
    }
    result = value
    return
}

func TapImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
    if Length(args) == 0 {
        err = errors.New("tap requires at least an initial value.")
        return
    }

    var value *Data

    value, err = Eval(Car(args), env)
    if err != nil {
        return
    }
    result = value

    for cell := Cdr(args); NotNilP(cell); cell = Cdr(cell) {
        sexpr := Car(cell)
        var newExpr *Data
        if ListP(sexpr) {
            newExpr = Cons(Car(sexpr), Cons(value, Cdr(sexpr)))
        } else {
            newExpr = Cons(sexpr, Cons(value, nil))
        }
        _, err = Eval(newExpr, env)
        if err != nil {
            return
        }
    }
    return
}
<|endoftext|>


<commit_before>package nsmvpp

import (
	govppapi "git.fd.io/govpp.git/api"
	"github.com/sirupsen/logrus"
)

type operation interface {
	apply(apiCh govppapi.Channel) error
	rollback() operation
}

func rollback(tx []operation, pos int, apiCh govppapi.Channel) error {
	logrus.Infof("Rolling back operations...")
	var err error
	for i := pos - 1; pos >= 0; pos-- {
		err = tx[i].rollback().apply(apiCh)
		if err != nil {
			logrus.Errorf("error while rolling back, (I will continue rollback operations): %v", err)
		}
	}
	logrus.Info("Done. I did my best to roll things back")
	return err
}

func perform(tx []operation, apiCh govppapi.Channel) (int, error) {
	logrus.Infof("Programming dataplane...")
	for i := range tx {
		err := tx[i].apply(apiCh)
		if err != nil {
			logrus.Errorf("error performing operation %v", err)
			return i, err
		}
	}
	logrus.Infof("Transaction completed!")
	return len(tx), nil
}
<commit_msg>Fix for-loop index (#407)<commit_after>package nsmvpp

import (
	govppapi "git.fd.io/govpp.git/api"
	"github.com/sirupsen/logrus"
)

type operation interface {
	apply(apiCh govppapi.Channel) error
	rollback() operation
}

func rollback(tx []operation, pos int, apiCh govppapi.Channel) error {
	logrus.Infof("Rolling back operations...")
	var err error
	for i := pos - 1; i >= 0; i-- {
		err = tx[i].rollback().apply(apiCh)
		if err != nil {
			logrus.Errorf("error while rolling back, (I will continue rollback operations): %v", err)
		}
	}
	logrus.Info("Done. I did my best to roll things back")
	return err
}

func perform(tx []operation, apiCh govppapi.Channel) (int, error) {
	logrus.Infof("Programming dataplane...")
	for i := range tx {
		err := tx[i].apply(apiCh)
		if err != nil {
			logrus.Errorf("error performing operation %v", err)
			return i, err
		}
	}
	logrus.Infof("Transaction completed!")
	return len(tx), nil
}
<|endoftext|>


<commit_before>package benchlist

import (
	"container/list"
	"math/rand"
	"sync"
	"time"

	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/snow"
	"github.com/ava-labs/avalanchego/snow/validators"
	"github.com/ava-labs/avalanchego/utils/constants"
	"github.com/ava-labs/avalanchego/utils/timer"

	safemath "github.com/ava-labs/avalanchego/utils/math"
)

// QueryBenchlist ...
type QueryBenchlist interface {
	// RegisterQuery registers a sent query and returns whether the query is subject to benchlist
	RegisterQuery(validatorID ids.ShortID, requestID uint32, msgType constants.MsgType) bool
	// RegisterResponse registers the response to a query message
	RegisterResponse(validatorID ids.ShortID, requstID uint32)
	// QueryFailed registers that a query did not receive a response within our synchrony bound
	QueryFailed(validatorID ids.ShortID, requestID uint32)
}

// If a peer consistently does not respond to queries, it will
// increase latencies on the network whenever that peer is polled.
// If we cannot terminate the poll early, then the poll will wait
// the full timeout before finalizing the poll and making progress.
// This can increase network latencies to an undesirable level.

// Therefore, a benchlist is used as a heurstic to immediately fail
// queries to nodes that are consistently not responding.

type queryBenchlist struct {
	vdrs validators.Set
	// Validator ID --> Request ID --> non-empty iff
	// there is an outstanding request to this validator
	// with the corresponding requestID
	pendingQueries map[[20]byte]map[uint32]pendingQuery
	// Map of consecutive query failures
	consecutiveFailures map[[20]byte]int

	// Maintain benchlist
	benchlistTimes map[[20]byte]time.Time
	benchlistOrder *list.List
	benchlistSet   ids.ShortSet

	threshold  int
	duration   time.Duration
	maxPortion float64

	clock timer.Clock

	metrics *metrics
	ctx     *snow.Context

	lock sync.Mutex
}

type pendingQuery struct {
	registered time.Time
	msgType    constants.MsgType
}

// NewQueryBenchlist ...
func NewQueryBenchlist(validators validators.Set, ctx *snow.Context, threshold int, duration time.Duration, maxPortion float64, summaryEnabled bool, namespace string) QueryBenchlist {
	metrics := &metrics{}
	metrics.Initialize(ctx, namespace, summaryEnabled)

	return &queryBenchlist{
		pendingQueries:      make(map[[20]byte]map[uint32]pendingQuery),
		consecutiveFailures: make(map[[20]byte]int),
		benchlistTimes:      make(map[[20]byte]time.Time),
		benchlistOrder:      list.New(),
		benchlistSet:        ids.ShortSet{},
		vdrs:                validators,
		threshold:           threshold,
		duration:            duration,
		maxPortion:          maxPortion,
		ctx:                 ctx,
		metrics:             metrics,
	}
}

// RegisterQuery attempts to register a query from [validatorID] and returns true
// if that request should be made (not subject to benchlisting)
func (b *queryBenchlist) RegisterQuery(validatorID ids.ShortID, requestID uint32, msgType constants.MsgType) bool {
	b.lock.Lock()
	defer b.lock.Unlock()

	key := validatorID.Key()
	if benched := b.benched(validatorID); benched {
		return false
	}

	validatorRequests, ok := b.pendingQueries[key]
	if !ok {
		validatorRequests = make(map[uint32]pendingQuery)
		b.pendingQueries[key] = validatorRequests
	}

	validatorRequests[requestID] = pendingQuery{
		registered: b.clock.Time(),
		msgType:    msgType,
	}
	return true
}

// RegisterResponse removes the query from pending
func (b *queryBenchlist) RegisterResponse(validatorID ids.ShortID, requestID uint32) {
	b.lock.Lock()
	defer b.lock.Unlock()

	if ok := b.removeQuery(validatorID, requestID); !ok {
		return
	}

	// Reset consecutive failures on success
	delete(b.consecutiveFailures, validatorID.Key())
}

// QueryFailed notes a failure and benchlists [validatorID] if necessary
func (b *queryBenchlist) QueryFailed(validatorID ids.ShortID, requestID uint32) {
	b.lock.Lock()
	defer b.lock.Unlock()

	if ok := b.removeQuery(validatorID, requestID); !ok {
		return
	}

	key := validatorID.Key()
	// Add a failure and benches [validatorID] if it has
	// passed the threshold
	b.consecutiveFailures[key]++
	if b.consecutiveFailures[key] >= b.threshold {
		b.bench(validatorID)
	}
}

func (b *queryBenchlist) bench(validatorID ids.ShortID) {
	if b.benchlistSet.Contains(validatorID) {
		return
	}

	key := validatorID.Key()

	// Goal:
	// Random end time in the range:
	// [max(lastEndTime,(currentTime + (duration/2)): currentTime + duration]
	// This maintains the invariant that validators in benchlistOrder are
	// ordered by the time that they should be unbenched
	currTime := b.clock.Time()
	minEndTime := currTime.Add(b.duration / 2)
	if elem := b.benchlistOrder.Back(); elem != nil {
		lastValidator := elem.Value.(ids.ShortID)
		lastEndTime := b.benchlistTimes[lastValidator.Key()]
		if lastEndTime.After(minEndTime) {
			minEndTime = lastEndTime
		}
	}
	maxEndTime := currTime.Add(b.duration)
	// Since maxEndTime is at least [duration] in the future and every element
	// added to benchlist was added in the past with an end time at most [duration]
	// in the future, this should never produce a negative duration.
	diff := maxEndTime.Sub(minEndTime)
	randomizedEndTime := minEndTime.Add(time.Duration(rand.Float64() * float64(diff))) // #nosec G404

	// Add to benchlist times with randomized delay
	b.benchlistTimes[key] = randomizedEndTime
	b.benchlistOrder.PushBack(validatorID)
	b.benchlistSet.Add(validatorID)
	delete(b.consecutiveFailures, key)
	b.ctx.Log.Debug(
		"benching validator %s after %d consecutive failed queries for %s",
		validatorID,
		b.threshold,
		randomizedEndTime.Sub(currTime),
	)

	// Note: there could be a memory leak if a large number of
	// validators were added, sampled, benched, and never sampled
	// again. Due to the minimum staking amount and durations this
	// is not a realistic concern.
	b.cleanup()
}

// benched checks if [validatorID] is currently benched
// and calls cleanup if its benching period has elapsed
func (b *queryBenchlist) benched(validatorID ids.ShortID) bool {
	key := validatorID.Key()

	end, ok := b.benchlistTimes[key]
	if !ok {
		return false
	}

	if b.clock.Time().Before(end) {
		return true
	}

	// If a benched item has expired, cleanup the benchlist
	b.cleanup()
	return false
}

// cleanup ensures that we have not benched too much stake
// and removes anything from the benchlist whose time has expired
func (b *queryBenchlist) cleanup() {
	currentWeight, err := b.vdrs.SubsetWeight(b.benchlistSet)
	if err != nil {
		// Add log for this, should never happen
		b.ctx.Log.Error("failed to calculate subset weight due to: %w... Resetting benchlist", err)
		b.reset()
		return
	}

	benchLen := b.benchlistSet.Len()
	updatedWeight := currentWeight
	totalWeight := b.vdrs.Weight()
	maxBenchlistWeight := uint64(float64(totalWeight) * b.maxPortion)

	// Iterate over elements of the benchlist in order of expiration
	for e := b.benchlistOrder.Front(); e != nil; e = e.Next() {
		validatorID := e.Value.(ids.ShortID)
		key := validatorID.Key()
		end := b.benchlistTimes[key]
		// Remove elements with the next expiration until the next item has not
		// expired and the bench has less than the maximum weight
		// Note: this creates an edge case where benchlisting a validator
		// with a sufficient stake may clear the benchlist
		if b.clock.Time().Before(end) && currentWeight < maxBenchlistWeight {
			break
		}

		removeWeight, ok := b.vdrs.GetWeight(validatorID)
		if ok {
			newWeight, err := safemath.Sub64(currentWeight, removeWeight)
			if err != nil {
				b.ctx.Log.Error("failed to calculate new subset weight due to: %w... Resetting benchlist", err)
				b.reset()
				return
			}
			updatedWeight = newWeight
		}

		b.benchlistOrder.Remove(e)
		delete(b.benchlistTimes, key)
		b.benchlistSet.Remove(validatorID)
	}

	updatedBenchLen := b.benchlistSet.Len()
	b.ctx.Log.Debug("benchlist weight: (%d/%d) -> (%d/%d). Benched Validators: %d -> %d",
		currentWeight,
		totalWeight,
		updatedWeight,
		totalWeight,
		benchLen,
		updatedBenchLen,
	)
	b.metrics.weightBenched.Set(float64(updatedWeight))
	b.metrics.numBenched.Set(float64(updatedBenchLen))
}

func (b *queryBenchlist) reset() {
	b.pendingQueries = make(map[[20]byte]map[uint32]pendingQuery)
	b.consecutiveFailures = make(map[[20]byte]int)
	b.benchlistTimes = make(map[[20]byte]time.Time)
	b.benchlistOrder.Init()
	b.benchlistSet.Clear()
	b.metrics.weightBenched.Set(0)
	b.metrics.numBenched.Set(0)
}

// removeQuery returns true if the query was present
func (b *queryBenchlist) removeQuery(validatorID ids.ShortID, requestID uint32) bool {
	key := validatorID.Key()

	validatorRequests, ok := b.pendingQueries[key]
	if !ok {
		return false
	}

	query, ok := validatorRequests[requestID]
	if !ok {
		return false
	}

	delete(validatorRequests, requestID)
	if len(validatorRequests) == 0 {
		delete(b.pendingQueries, key)
	}
	b.metrics.observe(validatorID, query.msgType, b.clock.Time().Sub(query.registered))
	return true
}
<commit_msg>Fix updated weight in benchlist<commit_after>package benchlist

import (
	"container/list"
	"math/rand"
	"sync"
	"time"

	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/snow"
	"github.com/ava-labs/avalanchego/snow/validators"
	"github.com/ava-labs/avalanchego/utils/constants"
	"github.com/ava-labs/avalanchego/utils/timer"

	safemath "github.com/ava-labs/avalanchego/utils/math"
)

// QueryBenchlist ...
type QueryBenchlist interface {
	// RegisterQuery registers a sent query and returns whether the query is subject to benchlist
	RegisterQuery(validatorID ids.ShortID, requestID uint32, msgType constants.MsgType) bool
	// RegisterResponse registers the response to a query message
	RegisterResponse(validatorID ids.ShortID, requstID uint32)
	// QueryFailed registers that a query did not receive a response within our synchrony bound
	QueryFailed(validatorID ids.ShortID, requestID uint32)
}

// If a peer consistently does not respond to queries, it will
// increase latencies on the network whenever that peer is polled.
// If we cannot terminate the poll early, then the poll will wait
// the full timeout before finalizing the poll and making progress.
// This can increase network latencies to an undesirable level.

// Therefore, a benchlist is used as a heurstic to immediately fail
// queries to nodes that are consistently not responding.

type queryBenchlist struct {
	vdrs validators.Set
	// Validator ID --> Request ID --> non-empty iff
	// there is an outstanding request to this validator
	// with the corresponding requestID
	pendingQueries map[[20]byte]map[uint32]pendingQuery
	// Map of consecutive query failures
	consecutiveFailures map[[20]byte]int

	// Maintain benchlist
	benchlistTimes map[[20]byte]time.Time
	benchlistOrder *list.List
	benchlistSet   ids.ShortSet

	threshold  int
	duration   time.Duration
	maxPortion float64

	clock timer.Clock

	metrics *metrics
	ctx     *snow.Context

	lock sync.Mutex
}

type pendingQuery struct {
	registered time.Time
	msgType    constants.MsgType
}

// NewQueryBenchlist ...
func NewQueryBenchlist(validators validators.Set, ctx *snow.Context, threshold int, duration time.Duration, maxPortion float64, summaryEnabled bool, namespace string) QueryBenchlist {
	metrics := &metrics{}
	metrics.Initialize(ctx, namespace, summaryEnabled)

	return &queryBenchlist{
		pendingQueries:      make(map[[20]byte]map[uint32]pendingQuery),
		consecutiveFailures: make(map[[20]byte]int),
		benchlistTimes:      make(map[[20]byte]time.Time),
		benchlistOrder:      list.New(),
		benchlistSet:        ids.ShortSet{},
		vdrs:                validators,
		threshold:           threshold,
		duration:            duration,
		maxPortion:          maxPortion,
		ctx:                 ctx,
		metrics:             metrics,
	}
}

// RegisterQuery attempts to register a query from [validatorID] and returns true
// if that request should be made (not subject to benchlisting)
func (b *queryBenchlist) RegisterQuery(validatorID ids.ShortID, requestID uint32, msgType constants.MsgType) bool {
	b.lock.Lock()
	defer b.lock.Unlock()

	key := validatorID.Key()
	if benched := b.benched(validatorID); benched {
		return false
	}

	validatorRequests, ok := b.pendingQueries[key]
	if !ok {
		validatorRequests = make(map[uint32]pendingQuery)
		b.pendingQueries[key] = validatorRequests
	}

	validatorRequests[requestID] = pendingQuery{
		registered: b.clock.Time(),
		msgType:    msgType,
	}
	return true
}

// RegisterResponse removes the query from pending
func (b *queryBenchlist) RegisterResponse(validatorID ids.ShortID, requestID uint32) {
	b.lock.Lock()
	defer b.lock.Unlock()

	if ok := b.removeQuery(validatorID, requestID); !ok {
		return
	}

	// Reset consecutive failures on success
	delete(b.consecutiveFailures, validatorID.Key())
}

// QueryFailed notes a failure and benchlists [validatorID] if necessary
func (b *queryBenchlist) QueryFailed(validatorID ids.ShortID, requestID uint32) {
	b.lock.Lock()
	defer b.lock.Unlock()

	if ok := b.removeQuery(validatorID, requestID); !ok {
		return
	}

	key := validatorID.Key()
	// Add a failure and benches [validatorID] if it has
	// passed the threshold
	b.consecutiveFailures[key]++
	if b.consecutiveFailures[key] >= b.threshold {
		b.bench(validatorID)
	}
}

func (b *queryBenchlist) bench(validatorID ids.ShortID) {
	if b.benchlistSet.Contains(validatorID) {
		return
	}

	key := validatorID.Key()

	// Goal:
	// Random end time in the range:
	// [max(lastEndTime,(currentTime + (duration/2)): currentTime + duration]
	// This maintains the invariant that validators in benchlistOrder are
	// ordered by the time that they should be unbenched
	currTime := b.clock.Time()
	minEndTime := currTime.Add(b.duration / 2)
	if elem := b.benchlistOrder.Back(); elem != nil {
		lastValidator := elem.Value.(ids.ShortID)
		lastEndTime := b.benchlistTimes[lastValidator.Key()]
		if lastEndTime.After(minEndTime) {
			minEndTime = lastEndTime
		}
	}
	maxEndTime := currTime.Add(b.duration)
	// Since maxEndTime is at least [duration] in the future and every element
	// added to benchlist was added in the past with an end time at most [duration]
	// in the future, this should never produce a negative duration.
	diff := maxEndTime.Sub(minEndTime)
	randomizedEndTime := minEndTime.Add(time.Duration(rand.Float64() * float64(diff))) // #nosec G404

	// Add to benchlist times with randomized delay
	b.benchlistTimes[key] = randomizedEndTime
	b.benchlistOrder.PushBack(validatorID)
	b.benchlistSet.Add(validatorID)
	delete(b.consecutiveFailures, key)
	b.ctx.Log.Debug(
		"benching validator %s after %d consecutive failed queries for %s",
		validatorID,
		b.threshold,
		randomizedEndTime.Sub(currTime),
	)

	// Note: there could be a memory leak if a large number of
	// validators were added, sampled, benched, and never sampled
	// again. Due to the minimum staking amount and durations this
	// is not a realistic concern.
	b.cleanup()
}

// benched checks if [validatorID] is currently benched
// and calls cleanup if its benching period has elapsed
func (b *queryBenchlist) benched(validatorID ids.ShortID) bool {
	key := validatorID.Key()

	end, ok := b.benchlistTimes[key]
	if !ok {
		return false
	}

	if b.clock.Time().Before(end) {
		return true
	}

	// If a benched item has expired, cleanup the benchlist
	b.cleanup()
	return false
}

// cleanup ensures that we have not benched too much stake
// and removes anything from the benchlist whose time has expired
func (b *queryBenchlist) cleanup() {
	currentWeight, err := b.vdrs.SubsetWeight(b.benchlistSet)
	if err != nil {
		// Add log for this, should never happen
		b.ctx.Log.Error("failed to calculate subset weight due to: %w... Resetting benchlist", err)
		b.reset()
		return
	}

	benchLen := b.benchlistSet.Len()
	updatedWeight := currentWeight
	totalWeight := b.vdrs.Weight()
	maxBenchlistWeight := uint64(float64(totalWeight) * b.maxPortion)

	// Iterate over elements of the benchlist in order of expiration
	for e := b.benchlistOrder.Front(); e != nil; e = e.Next() {
		validatorID := e.Value.(ids.ShortID)
		key := validatorID.Key()
		end := b.benchlistTimes[key]
		// Remove elements with the next expiration until the next item has not
		// expired and the bench has less than the maximum weight
		// Note: this creates an edge case where benchlisting a validator
		// with a sufficient stake may clear the benchlist
		if b.clock.Time().Before(end) && updatedWeight < maxBenchlistWeight {
			break
		}

		removeWeight, ok := b.vdrs.GetWeight(validatorID)
		if ok {
			newWeight, err := safemath.Sub64(updatedWeight, removeWeight)
			if err != nil {
				b.ctx.Log.Error("failed to calculate new subset weight due to: %w... Resetting benchlist", err)
				b.reset()
				return
			}
			updatedWeight = newWeight
		}

		b.benchlistOrder.Remove(e)
		delete(b.benchlistTimes, key)
		b.benchlistSet.Remove(validatorID)
	}

	updatedBenchLen := b.benchlistSet.Len()
	b.ctx.Log.Debug("benchlist weight: (%d/%d) -> (%d/%d). Benched Validators: %d -> %d",
		currentWeight,
		totalWeight,
		updatedWeight,
		totalWeight,
		benchLen,
		updatedBenchLen,
	)
	b.metrics.weightBenched.Set(float64(updatedWeight))
	b.metrics.numBenched.Set(float64(updatedBenchLen))
}

func (b *queryBenchlist) reset() {
	b.pendingQueries = make(map[[20]byte]map[uint32]pendingQuery)
	b.consecutiveFailures = make(map[[20]byte]int)
	b.benchlistTimes = make(map[[20]byte]time.Time)
	b.benchlistOrder.Init()
	b.benchlistSet.Clear()
	b.metrics.weightBenched.Set(0)
	b.metrics.numBenched.Set(0)
}

// removeQuery returns true if the query was present
func (b *queryBenchlist) removeQuery(validatorID ids.ShortID, requestID uint32) bool {
	key := validatorID.Key()

	validatorRequests, ok := b.pendingQueries[key]
	if !ok {
		return false
	}

	query, ok := validatorRequests[requestID]
	if !ok {
		return false
	}

	delete(validatorRequests, requestID)
	if len(validatorRequests) == 0 {
		delete(b.pendingQueries, key)
	}
	b.metrics.observe(validatorID, query.msgType, b.clock.Time().Sub(query.registered))
	return true
}
<|endoftext|>


<commit_before>package gce

import (
	"flag"
	"fmt"
	"net"

	"v.io/core/veyron2"
	"v.io/core/veyron2/context"
	"v.io/core/veyron2/ipc"

	"v.io/core/veyron/lib/appcycle"
	"v.io/core/veyron/lib/flags"
	"v.io/core/veyron/lib/netstate"
	"v.io/core/veyron/profiles/internal/gce"
	_ "v.io/core/veyron/runtimes/google/ipc/protocols/tcp"
	_ "v.io/core/veyron/runtimes/google/ipc/protocols/ws"
	_ "v.io/core/veyron/runtimes/google/ipc/protocols/wsh"
	grt "v.io/core/veyron/runtimes/google/rt"
)

var (
	commonFlags *flags.Flags
)

func init() {
	commonFlags = flags.CreateAndRegister(flag.CommandLine, flags.Listen)
	veyron2.RegisterProfileInit(Init)
}

func Init(ctx *context.T) (veyron2.RuntimeX, *context.T, veyron2.Shutdown, error) {
	if !gce.RunningOnGCE() {
		return nil, nil, nil, fmt.Errorf("GCE profile used on a non-GCE system")
	}

	runtime := &grt.RuntimeX{}
	ctx, shutdown, err := runtime.Init(ctx, nil)
	if err != nil {
		return nil, nil, shutdown, err
	}
	veyron2.GetLogger(ctx).VI(1).Infof("Initializing GCE profile.")

	lf := commonFlags.ListenFlags()
	listenSpec := ipc.ListenSpec{
		Addrs: ipc.ListenAddrs(lf.Addrs),
		Proxy: lf.ListenProxy,
	}

	if ip, err := gce.ExternalIPAddress(); err != nil {
		return nil, nil, shutdown, err
	} else {
		listenSpec.AddressChooser = func(network string, addrs []ipc.Address) ([]ipc.Address, error) {
			return []ipc.Address{&netstate.AddrIfc{&net.IPAddr{IP: ip}, "gce-nat", nil}}, nil
		}
	}
	ctx = runtime.SetListenSpec(ctx, listenSpec)

	ac := appcycle.New()
	ctx = runtime.SetAppCycle(ctx, ac)

	profileShutdown := func() {
		shutdown()
		ac.Shutdown()
	}

	return runtime, ctx, profileShutdown, nil
}
<commit_msg>veyron/profiles/gce: TBR: Add missing build tag.<commit_after>// +build linux

// Package gce provides a profile for Google Compute Engine and should be
// used by binaries that only ever expect to be run on GCE.
package gce

import (
	"flag"
	"fmt"
	"net"

	"v.io/core/veyron2"
	"v.io/core/veyron2/context"
	"v.io/core/veyron2/ipc"

	"v.io/core/veyron/lib/appcycle"
	"v.io/core/veyron/lib/flags"
	"v.io/core/veyron/lib/netstate"
	"v.io/core/veyron/profiles/internal/gce"
	_ "v.io/core/veyron/runtimes/google/ipc/protocols/tcp"
	_ "v.io/core/veyron/runtimes/google/ipc/protocols/ws"
	_ "v.io/core/veyron/runtimes/google/ipc/protocols/wsh"
	grt "v.io/core/veyron/runtimes/google/rt"
)

var (
	commonFlags *flags.Flags
)

func init() {
	commonFlags = flags.CreateAndRegister(flag.CommandLine, flags.Listen)
	veyron2.RegisterProfileInit(Init)
}

func Init(ctx *context.T) (veyron2.RuntimeX, *context.T, veyron2.Shutdown, error) {
	if !gce.RunningOnGCE() {
		return nil, nil, nil, fmt.Errorf("GCE profile used on a non-GCE system")
	}

	runtime := &grt.RuntimeX{}
	ctx, shutdown, err := runtime.Init(ctx, nil)
	if err != nil {
		return nil, nil, shutdown, err
	}
	veyron2.GetLogger(ctx).VI(1).Infof("Initializing GCE profile.")

	lf := commonFlags.ListenFlags()
	listenSpec := ipc.ListenSpec{
		Addrs: ipc.ListenAddrs(lf.Addrs),
		Proxy: lf.ListenProxy,
	}

	if ip, err := gce.ExternalIPAddress(); err != nil {
		return nil, nil, shutdown, err
	} else {
		listenSpec.AddressChooser = func(network string, addrs []ipc.Address) ([]ipc.Address, error) {
			return []ipc.Address{&netstate.AddrIfc{&net.IPAddr{IP: ip}, "gce-nat", nil}}, nil
		}
	}
	ctx = runtime.SetListenSpec(ctx, listenSpec)

	ac := appcycle.New()
	ctx = runtime.SetAppCycle(ctx, ac)

	profileShutdown := func() {
		shutdown()
		ac.Shutdown()
	}

	return runtime, ctx, profileShutdown, nil
}
<|endoftext|>


<commit_before>package cassandra

import (
	"flag"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/gocql/gocql"
	"github.com/raintank/met"
	"github.com/raintank/metrictank/idx"
	"github.com/raintank/metrictank/idx/memory"
	"github.com/raintank/worldping-api/pkg/log"
	"github.com/rakyll/globalconf"
	"gopkg.in/raintank/schema.v1"
)

const keyspace_schema = `CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}  AND durable_writes = true`
const table_schema = `CREATE TABLE IF NOT EXISTS %s.metric_def_idx (
    id text PRIMARY KEY,
    def blob,
) WITH compaction = {'class': 'SizeTieredCompactionStrategy'}
    AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'}`

var (
	idxCasOk             met.Count
	idxCasFail           met.Count
	idxCasAddDuration    met.Timer
	idxCasDeleteDuration met.Timer

	Enabled        bool
	keyspace       string
	hosts          string
	consistency    string
	timeout        time.Duration
	numConns       int
	writeQueueSize int
	protoVer       int
	maxStale       time.Duration
	pruneInterval  time.Duration
)

func ConfigSetup() {
	casIdx := flag.NewFlagSet("cassandra-idx", flag.ExitOnError)
	casIdx.BoolVar(&Enabled, "enabled", false, "")
	casIdx.StringVar(&keyspace, "keyspace", "metric", "Cassandra keyspace to store metricDefinitions in.")
	casIdx.StringVar(&hosts, "hosts", "localhost:9042", "comma separated list of cassandra addresses in host:port form")
	casIdx.StringVar(&consistency, "consistency", "one", "write consistency (any|one|two|three|quorum|all|local_quorum|each_quorum|local_one")
	casIdx.DurationVar(&timeout, "timeout", time.Second, "cassandra request timeout")
	casIdx.IntVar(&numConns, "num-conns", 10, "number of concurrent connections to cassandra")
	casIdx.IntVar(&writeQueueSize, "write-queue-size", 100000, "Max number of metricDefs allowed to be unwritten to cassandra")
	casIdx.IntVar(&protoVer, "protocol-version", 4, "cql protocol version to use")
	casIdx.DurationVar(&maxStale, "max-stale", 0, "clear series from the index if they have not been seen for this much time.")
	casIdx.DurationVar(&pruneInterval, "prune-interval", time.Hour*3, "Interval at which the index should be checked for stale series.")
	globalconf.Register("cassandra-idx", casIdx)
}

type writeReq struct {
	def      *schema.MetricDefinition
	recvTime time.Time
}

// Implements the the "MetricIndex" interface
type CasIdx struct {
	memory.MemoryIdx
	cluster    *gocql.ClusterConfig
	session    *gocql.Session
	writeQueue chan writeReq
	shutdown   chan struct{}
	wg         sync.WaitGroup
}

func New() *CasIdx {
	cluster := gocql.NewCluster(strings.Split(hosts, ",")...)
	cluster.Consistency = gocql.ParseConsistency(consistency)
	cluster.Timeout = timeout
	cluster.NumConns = numConns
	cluster.ProtoVersion = protoVer

	return &CasIdx{
		MemoryIdx:  *memory.New(),
		cluster:    cluster,
		writeQueue: make(chan writeReq, writeQueueSize),
		shutdown:   make(chan struct{}),
	}
}

func (c *CasIdx) Init(stats met.Backend) error {
	log.Info("initializing cassandra-idx. Hosts=%s", hosts)
	if err := c.MemoryIdx.Init(stats); err != nil {
		return err
	}

	var err error
	tmpSession, err := c.cluster.CreateSession()
	if err != nil {
		log.Error(3, "cassandra-idx failed to create cassandra session. %s", err)
		return err
	}

	// ensure the keyspace and table exist.
	err = tmpSession.Query(fmt.Sprintf(keyspace_schema, keyspace)).Exec()
	if err != nil {
		log.Error(3, "cassandra-idx failed to initialize cassandra keyspace. %s", err)
		return err
	}
	err = tmpSession.Query(fmt.Sprintf(table_schema, keyspace)).Exec()
	if err != nil {
		log.Error(3, "cassandra-idx failed to initialize cassandra table. %s", err)
		return err
	}
	tmpSession.Close()
	c.cluster.Keyspace = keyspace
	session, err := c.cluster.CreateSession()
	if err != nil {
		log.Error(3, "cassandra-idx failed to create cassandra session. %s", err)
		return err
	}

	c.session = session

	idxCasOk = stats.NewCount("idx.cassandra.ok")
	idxCasFail = stats.NewCount("idx.cassandra.fail")
	idxCasAddDuration = stats.NewTimer("idx.cassandra.add_duration", 0)
	idxCasDeleteDuration = stats.NewTimer("idx.cassandra.delete_duration", 0)

	for i := 0; i < numConns; i++ {
		c.wg.Add(1)
		go c.processWriteQueue()
	}
	//Rebuild the in-memory index.

	c.rebuildIndex()
	if maxStale > 0 {
		if pruneInterval == 0 {
			return fmt.Errorf("pruneInterval must be greater then 0")
		}
		go c.prune()
	}
	return nil
}

func (c *CasIdx) Stop() {
	log.Info("cassandra-idx stopping")
	c.MemoryIdx.Stop()
	close(c.writeQueue)
	c.wg.Wait()
	c.session.Close()
}

func (c *CasIdx) Add(data *schema.MetricData) {
	existing, err := c.MemoryIdx.Get(data.Id)
	inMemory := true
	if err != nil {
		if err == idx.DefNotFound {
			inMemory = false
		} else {
			log.Error(3, "cassandra-idx Failed to query Memory Index for %s. %s", data.Id, err)
			return
		}
	}
	if inMemory {
		log.Debug("cassandra-idx def already seen before. Just updating memory Index")
		existing.LastUpdate = data.Time
		c.MemoryIdx.AddDef(&existing)
		return
	}
	def := schema.MetricDefinitionFromMetricData(data)
	c.MemoryIdx.AddDef(def)
	c.writeQueue <- writeReq{recvTime: time.Now(), def: def}
}

func (c *CasIdx) rebuildIndex() {
	log.Info("cassandra-idx Rebuilding Memory Index from metricDefinitions in Cassandra")
	pre := time.Now()
	defs := make([]schema.MetricDefinition, 0)
	iter := c.session.Query("SELECT def from metric_def_idx").Iter()

	var data []byte
	mdef := schema.MetricDefinition{}
	for iter.Scan(&data) {
		_, err := mdef.UnmarshalMsg(data)
		if err != nil {
			log.Error(3, "cassandra-idx Bad definition in index. %s - %s", data, err)
			continue
		}
		defs = append(defs, mdef)
	}
	c.MemoryIdx.Load(defs)
	log.Info("Rebuilding Memory Index Complete. Took %s", time.Since(pre).String())
}

func (c *CasIdx) processWriteQueue() {
	log.Info("cassandra-idx writeQueue handler started.")
	data := make([]byte, 0)
	var success bool
	var attempts int
	var err error
	var req writeReq
	for req = range c.writeQueue {
		data = data[:0]
		data, err = req.def.MarshalMsg(data)
		if err != nil {
			log.Error(3, "Failed to marshal metricDef. %s", err)
			continue
		}
		success = false
		attempts = 0
		for !success {
			if err := c.session.Query(`INSERT INTO metric_def_idx (id, def) VALUES (?, ?)`, req.def.Id, data).Exec(); err != nil {
				idxCasFail.Inc(1)
				if (attempts % 20) == 0 {
					log.Warn("cassandra-idx Failed to write def to cassandra. it will be retried. %s", err)
				}
				sleepTime := 100 * attempts
				if sleepTime > 2000 {
					sleepTime = 2000
				}
				time.Sleep(time.Duration(sleepTime) * time.Millisecond)
				attempts++
			} else {
				success = true
				idxCasAddDuration.Value(time.Since(req.recvTime))
				idxCasOk.Inc(1)
				log.Debug("cassandra-idx metricDef saved to cassandra. %s", req.def.Id)
			}
		}
	}
	log.Info("cassandra-idx writeQueue handler ended.")
	c.wg.Done()
}

func (c *CasIdx) Delete(orgId int, pattern string) error {
	ids, err := c.MemoryIdx.DeleteWithReport(orgId, pattern)
	if err != nil {
		return err
	}
	for _, id := range ids {
		err := c.session.Query("DELETE FROM metric_def_idx where id=?", id).Exec()
		if err != nil {
			log.Error(3, "cassandra-idx Failed to delete metricDef %s from cassandra. %s", id, err)
			return err
		}
	}
	return nil
}

func (c *CasIdx) Prune(orgId int, oldest time.Time) ([]schema.MetricDefinition, error) {
	pruned, err := c.MemoryIdx.Prune(orgId, oldest)
	// if an error was encountered then pruned is probably a partial list of metricDefs
	// deleted, so lets still try and delete these from Cassandra.
	for _, def := range pruned {
		err := c.session.Query("DELETE FROM metric_def_idx where id=?", def.Id).Exec()
		if err != nil {
			log.Error(3, "cassandra-idx Failed to delete metricDef %s from cassandra. %s", def.Id, err)
			continue
		}
	}
	return pruned, err
}

func (c *CasIdx) prune() {
	ticker := time.NewTicker(pruneInterval)
	for range ticker.C {
		staleTs := time.Now().Add(maxStale * -1)
		_, err := c.Prune(-1, staleTs)
		if err != nil {
			log.Error(3, "cassandra-idx: prune error. %s", err)
		}
	}
}
<commit_msg>better handling of errors deleting from cassandra.<commit_after>package cassandra

import (
	"flag"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/gocql/gocql"
	"github.com/raintank/met"
	"github.com/raintank/metrictank/idx"
	"github.com/raintank/metrictank/idx/memory"
	"github.com/raintank/worldping-api/pkg/log"
	"github.com/rakyll/globalconf"
	"gopkg.in/raintank/schema.v1"
)

const keyspace_schema = `CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}  AND durable_writes = true`
const table_schema = `CREATE TABLE IF NOT EXISTS %s.metric_def_idx (
    id text PRIMARY KEY,
    def blob,
) WITH compaction = {'class': 'SizeTieredCompactionStrategy'}
    AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'}`

var (
	idxCasOk             met.Count
	idxCasFail           met.Count
	idxCasAddDuration    met.Timer
	idxCasDeleteDuration met.Timer

	Enabled        bool
	keyspace       string
	hosts          string
	consistency    string
	timeout        time.Duration
	numConns       int
	writeQueueSize int
	protoVer       int
	maxStale       time.Duration
	pruneInterval  time.Duration
)

func ConfigSetup() {
	casIdx := flag.NewFlagSet("cassandra-idx", flag.ExitOnError)
	casIdx.BoolVar(&Enabled, "enabled", false, "")
	casIdx.StringVar(&keyspace, "keyspace", "metric", "Cassandra keyspace to store metricDefinitions in.")
	casIdx.StringVar(&hosts, "hosts", "localhost:9042", "comma separated list of cassandra addresses in host:port form")
	casIdx.StringVar(&consistency, "consistency", "one", "write consistency (any|one|two|three|quorum|all|local_quorum|each_quorum|local_one")
	casIdx.DurationVar(&timeout, "timeout", time.Second, "cassandra request timeout")
	casIdx.IntVar(&numConns, "num-conns", 10, "number of concurrent connections to cassandra")
	casIdx.IntVar(&writeQueueSize, "write-queue-size", 100000, "Max number of metricDefs allowed to be unwritten to cassandra")
	casIdx.IntVar(&protoVer, "protocol-version", 4, "cql protocol version to use")
	casIdx.DurationVar(&maxStale, "max-stale", 0, "clear series from the index if they have not been seen for this much time.")
	casIdx.DurationVar(&pruneInterval, "prune-interval", time.Hour*3, "Interval at which the index should be checked for stale series.")
	globalconf.Register("cassandra-idx", casIdx)
}

type writeReq struct {
	def      *schema.MetricDefinition
	recvTime time.Time
}

// Implements the the "MetricIndex" interface
type CasIdx struct {
	memory.MemoryIdx
	cluster    *gocql.ClusterConfig
	session    *gocql.Session
	writeQueue chan writeReq
	shutdown   chan struct{}
	wg         sync.WaitGroup
}

func New() *CasIdx {
	cluster := gocql.NewCluster(strings.Split(hosts, ",")...)
	cluster.Consistency = gocql.ParseConsistency(consistency)
	cluster.Timeout = timeout
	cluster.NumConns = numConns
	cluster.ProtoVersion = protoVer

	return &CasIdx{
		MemoryIdx:  *memory.New(),
		cluster:    cluster,
		writeQueue: make(chan writeReq, writeQueueSize),
		shutdown:   make(chan struct{}),
	}
}

func (c *CasIdx) Init(stats met.Backend) error {
	log.Info("initializing cassandra-idx. Hosts=%s", hosts)
	if err := c.MemoryIdx.Init(stats); err != nil {
		return err
	}

	var err error
	tmpSession, err := c.cluster.CreateSession()
	if err != nil {
		log.Error(3, "cassandra-idx failed to create cassandra session. %s", err)
		return err
	}

	// ensure the keyspace and table exist.
	err = tmpSession.Query(fmt.Sprintf(keyspace_schema, keyspace)).Exec()
	if err != nil {
		log.Error(3, "cassandra-idx failed to initialize cassandra keyspace. %s", err)
		return err
	}
	err = tmpSession.Query(fmt.Sprintf(table_schema, keyspace)).Exec()
	if err != nil {
		log.Error(3, "cassandra-idx failed to initialize cassandra table. %s", err)
		return err
	}
	tmpSession.Close()
	c.cluster.Keyspace = keyspace
	session, err := c.cluster.CreateSession()
	if err != nil {
		log.Error(3, "cassandra-idx failed to create cassandra session. %s", err)
		return err
	}

	c.session = session

	idxCasOk = stats.NewCount("idx.cassandra.ok")
	idxCasFail = stats.NewCount("idx.cassandra.fail")
	idxCasAddDuration = stats.NewTimer("idx.cassandra.add_duration", 0)
	idxCasDeleteDuration = stats.NewTimer("idx.cassandra.delete_duration", 0)

	for i := 0; i < numConns; i++ {
		c.wg.Add(1)
		go c.processWriteQueue()
	}
	//Rebuild the in-memory index.

	c.rebuildIndex()
	if maxStale > 0 {
		if pruneInterval == 0 {
			return fmt.Errorf("pruneInterval must be greater then 0")
		}
		go c.prune()
	}
	return nil
}

func (c *CasIdx) Stop() {
	log.Info("cassandra-idx stopping")
	c.MemoryIdx.Stop()
	close(c.writeQueue)
	c.wg.Wait()
	c.session.Close()
}

func (c *CasIdx) Add(data *schema.MetricData) {
	existing, err := c.MemoryIdx.Get(data.Id)
	inMemory := true
	if err != nil {
		if err == idx.DefNotFound {
			inMemory = false
		} else {
			log.Error(3, "cassandra-idx Failed to query Memory Index for %s. %s", data.Id, err)
			return
		}
	}
	if inMemory {
		log.Debug("cassandra-idx def already seen before. Just updating memory Index")
		existing.LastUpdate = data.Time
		c.MemoryIdx.AddDef(&existing)
		return
	}
	def := schema.MetricDefinitionFromMetricData(data)
	c.MemoryIdx.AddDef(def)
	c.writeQueue <- writeReq{recvTime: time.Now(), def: def}
}

func (c *CasIdx) rebuildIndex() {
	log.Info("cassandra-idx Rebuilding Memory Index from metricDefinitions in Cassandra")
	pre := time.Now()
	defs := make([]schema.MetricDefinition, 0)
	iter := c.session.Query("SELECT def from metric_def_idx").Iter()

	var data []byte
	mdef := schema.MetricDefinition{}
	for iter.Scan(&data) {
		_, err := mdef.UnmarshalMsg(data)
		if err != nil {
			log.Error(3, "cassandra-idx Bad definition in index. %s - %s", data, err)
			continue
		}
		defs = append(defs, mdef)
	}
	c.MemoryIdx.Load(defs)
	log.Info("Rebuilding Memory Index Complete. Took %s", time.Since(pre).String())
}

func (c *CasIdx) processWriteQueue() {
	log.Info("cassandra-idx writeQueue handler started.")
	data := make([]byte, 0)
	var success bool
	var attempts int
	var err error
	var req writeReq
	for req = range c.writeQueue {
		data = data[:0]
		data, err = req.def.MarshalMsg(data)
		if err != nil {
			log.Error(3, "Failed to marshal metricDef. %s", err)
			continue
		}
		success = false
		attempts = 0
		for !success {
			if err := c.session.Query(`INSERT INTO metric_def_idx (id, def) VALUES (?, ?)`, req.def.Id, data).Exec(); err != nil {
				idxCasFail.Inc(1)
				if (attempts % 20) == 0 {
					log.Warn("cassandra-idx Failed to write def to cassandra. it will be retried. %s", err)
				}
				sleepTime := 100 * attempts
				if sleepTime > 2000 {
					sleepTime = 2000
				}
				time.Sleep(time.Duration(sleepTime) * time.Millisecond)
				attempts++
			} else {
				success = true
				idxCasAddDuration.Value(time.Since(req.recvTime))
				idxCasOk.Inc(1)
				log.Debug("cassandra-idx metricDef saved to cassandra. %s", req.def.Id)
			}
		}
	}
	log.Info("cassandra-idx writeQueue handler ended.")
	c.wg.Done()
}

func (c *CasIdx) Delete(orgId int, pattern string) error {
	ids, err := c.MemoryIdx.DeleteWithReport(orgId, pattern)
	if err != nil {
		return err
	}
	for _, id := range ids {
		err := c.session.Query("DELETE FROM metric_def_idx where id=?", id).Exec()
		if err != nil {
			log.Error(3, "cassandra-idx Failed to delete metricDef %s from cassandra. %s", id, err)
			return err
		}
	}
	return nil
}

func (c *CasIdx) Prune(orgId int, oldest time.Time) ([]schema.MetricDefinition, error) {
	pruned, err := c.MemoryIdx.Prune(orgId, oldest)
	// if an error was encountered then pruned is probably a partial list of metricDefs
	// deleted, so lets still try and delete these from Cassandra.
	for _, def := range pruned {
		attempts := 0
		deleted := false
		for !deleted && attempts < 5 {
			attempts++
			cErr := c.session.Query("DELETE FROM metric_def_idx where id=?", def.Id).Exec()
			if cErr != nil {
				log.Error(3, "cassandra-idx Failed to delete metricDef %s from cassandra. %s", def.Id, err)
				time.Sleep(time.Second)
			} else {
				deleted = true
			}
		}
	}
	return pruned, err
}

func (c *CasIdx) prune() {
	ticker := time.NewTicker(pruneInterval)
	for range ticker.C {
		staleTs := time.Now().Add(maxStale * -1)
		_, err := c.Prune(-1, staleTs)
		if err != nil {
			log.Error(3, "cassandra-idx: prune error. %s", err)
		}
	}
}
<|endoftext|>


<commit_before>//  Copyright 2013 Walter Schulze
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package funcs

import (
	"strings"
	"time"
)

//Now returns a new now function.
func Now() Int {
	return &now{}
}

type now struct{}

func (this *now) Eval() (int64, error) {
	return time.Now().UnixNano(), nil
}

func (this *now) Hash() uint64 {
	return Hash("now")
}

func (this *now) Compare(that Comparable) int {
	if this.Hash() != that.Hash() {
		if this.Hash() < that.Hash() {
			return -1
		}
		return 1
	}
	if _, ok := that.(*now); ok {
		return 0
	}
	return strings.Compare(this.String(), that.String())
}

func (this *now) String() string {
	return "now"
}

func (this *now) HasVariable() bool { return true }

func init() {
	Register("now", Now)
}
<commit_msg>now() rather than now<commit_after>//  Copyright 2013 Walter Schulze
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package funcs

import (
	"strings"
	"time"
)

//Now returns a new now function.
func Now() Int {
	return &now{}
}

type now struct{}

func (this *now) Eval() (int64, error) {
	return time.Now().UnixNano(), nil
}

func (this *now) Hash() uint64 {
	return Hash("now")
}

func (this *now) Compare(that Comparable) int {
	if this.Hash() != that.Hash() {
		if this.Hash() < that.Hash() {
			return -1
		}
		return 1
	}
	if _, ok := that.(*now); ok {
		return 0
	}
	return strings.Compare(this.String(), that.String())
}

func (this *now) String() string {
	return "now()"
}

func (this *now) HasVariable() bool { return true }

func init() {
	Register("now", Now)
}
<|endoftext|>


<commit_before>package framework

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
)

type Plugin struct {
	Id          string              `json:"id"`
	Name        string              `json:"name"`
	Version     string              `json:"version"`
	Description string              `json:"description"`
	Permissions map[string][]string `json:"permissions"`
	Author      Author              `json:"author"`
	Homepage    string              `json:"homepage"`
	HideSidebar bool                `json:"hideSidebar"`
	Tiles       []Tile              `json:"tiles"`
}

type Tile struct {
	Name        string `json:"name"`
	Description string `json:"description"`
	View        string `json:"view"`
	Size        string `json:"size"`
}

type Author struct {
	Name     string `json:"name"`
	Email    string `json:"email"`
	Homepage string `json:"homepage"`
}

func LoadPlugins(pluginPath string) View {
	files, err := ioutil.ReadDir(pluginPath)
	if err != nil {
		return Error500
	}

	output := []Plugin{}
	for _, v := range files {
		if _, err := os.Stat(filepath.Join(pluginPath, v.Name(), "package.json")); err == nil && v.IsDir() {
			packageJSON, err := os.Open(filepath.Join(pluginPath, v.Name(), "package.json"))
			if err != nil {
				fmt.Println(err)
				continue
			}

			var plugin Plugin
			decoder := json.NewDecoder(packageJSON)
			err = decoder.Decode(&plugin)
			if err != nil {
				fmt.Println(err)
				continue
			}

			if plugin.Id != v.Name() {
				continue
			}

			output = append(output, plugin)

			packageJSON.Close()
		}
	}
	bytes, err := json.Marshal(output)
	if err != nil {
		return Error500
	}
	return &RawView{bytes, "application/json"}
}
<commit_msg>Added Viewers to Plugin Package.json<commit_after>package framework

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
)

type Plugin struct {
	Id          string              `json:"id"`
	Name        string              `json:"name"`
	Version     string              `json:"version"`
	Description string              `json:"description"`
	Permissions map[string][]string `json:"permissions"`
	Author      Author              `json:"author"`
	Homepage    string              `json:"homepage"`
	HideSidebar bool                `json:"hideSidebar"`
	Tiles       map[string]Tile     `json:"tiles"`
	Viewers     map[string]Viewer   `json:"viewers"`
}

type Viewer struct {
	Type []string `json:"type"`
	View string   `json:"view"`
}

type Tile struct {
	Name        string `json:"name"`
	Description string `json:"description"`
	View        string `json:"view"`
	Size        string `json:"size"`
}

type Author struct {
	Name     string `json:"name"`
	Email    string `json:"email"`
	Homepage string `json:"homepage"`
}

func LoadPlugins(pluginPath string) View {
	files, err := ioutil.ReadDir(pluginPath)
	if err != nil {
		return Error500
	}

	output := []Plugin{}
	for _, v := range files {
		if _, err := os.Stat(filepath.Join(pluginPath, v.Name(), "package.json")); err == nil && v.IsDir() {
			packageJSON, err := os.Open(filepath.Join(pluginPath, v.Name(), "package.json"))
			if err != nil {
				fmt.Println(err)
				continue
			}

			var plugin Plugin
			decoder := json.NewDecoder(packageJSON)
			err = decoder.Decode(&plugin)
			if err != nil {
				fmt.Println(err)
				continue
			}

			if plugin.Id != v.Name() {
				continue
			}

			output = append(output, plugin)

			packageJSON.Close()
		}
	}
	bytes, err := json.Marshal(output)
	if err != nil {
		return Error500
	}
	return &RawView{bytes, "application/json"}
}
<|endoftext|>


<commit_before>package importer

import (
	"fmt"
	"log"
	"strings"

	pb "gopkg.in/cheggaaa/pb.v1"
	// "io"
	// "io/ioutil"
	// "os"
	"sync"

	"time"

	"github.com/pkg/errors"
	"github.com/wadahiro/gitss/server/config"
	"github.com/wadahiro/gitss/server/indexer"
	"github.com/wadahiro/gitss/server/repo"
	// "github.com/wadahiro/gitss/server/util"
	"bytes"

	"golang.org/x/net/html/charset"
	"golang.org/x/text/transform"
)

type GitImporter struct {
	config  *config.Config
	indexer indexer.Indexer
	reader  *repo.GitRepoReader
	debug   bool
}

func NewGitImporter(config *config.Config, indexer indexer.Indexer) *GitImporter {
	r := repo.NewGitRepoReader(config)
	return &GitImporter{config: config, indexer: indexer, reader: r, debug: config.Debug}
}

func (g *GitImporter) Run(organization string, project string, url string) {
	log.Printf("Clone from %s %s %s\n", organization, project, url)

	repo, err := g.reader.CloneGitRepo(organization, project, url)
	if err != nil {
		log.Printf("Not found the repository: %s:%s/%s %+v\n", organization, project, url, err)
		return
	}

	repo.FetchAll()

	log.Printf("Fetched all. %s %s %s \n", organization, project, url)

	// branches and tags in the git repository (include/exclude filters are applied)
	branchMap, tagMap, err := repo.GetLatestCommitIdsMap()
	if err != nil {
		log.Printf("Failed to get latest commitIds of branch and tag. %+v\n", err)
		return
	}

	// branches in the config file
	indexed := g.config.GetIndexed(organization, project, repo.Repository)

	log.Printf("Start indexing for %s:%s/%s branches: %v -> %v, tags: %v -> %v\n", organization, project, repo.Repository, indexed.Branches, branchMap, indexed.Tags, tagMap)

	// progress bar
	bar := pb.StartNew(0)
	bar.ShowPercent = true

	start := time.Now()

	err = g.runIndexing(bar, repo, url, indexed, branchMap, tagMap)
	if err != nil {
		log.Printf("Failed to index. %+v", err)
		return
	}

	// Remove index for removed branches
	removeBranches := []string{}
	for ref, _ := range indexed.Branches {
		found := false
		for branch := range branchMap {
			if ref == branch {
				found = true
				break
			}
		}
		if !found {
			removeBranches = append(removeBranches, ref)
		}
	}

	bar.Total = bar.Total + int64(len(removeBranches))

	// Remove index for removed tags
	removeTags := []string{}
	for ref, _ := range indexed.Tags {
		found := false
		for branch := range branchMap {
			if ref == branch {
				found = true
				break
			}
		}
		if !found {
			removeTags = append(removeTags, ref)
		}
	}

	bar.Total = bar.Total + int64(len(removeTags))

	if len(removeBranches) > 0 || len(removeTags) > 0 {
		log.Printf("Start index deleting for %s:%s/%s (%v) (%v)\n", organization, project, repo.Repository, removeBranches, removeTags)
		g.indexer.DeleteIndexByRefs(organization, project, repo.Repository, removeBranches, removeTags)

		bar.Add(len(removeBranches) + len(removeTags))

		// Save config after deleting index completed
		g.config.DeleteIndexed(organization, project, repo.Repository, removeBranches, removeTags)
	}

	end := time.Now()
	time := (end.Sub(start)).Seconds()

	bar.FinishPrint(fmt.Sprintf("Indexing Complete! [%f seconds] for %s:%s/%s\n", time, organization, project, repo.Repository))
}

func (g *GitImporter) runIndexing(bar *pb.ProgressBar, repo *repo.GitRepo, url string, indexed config.Indexed, branchMap config.BrancheIndexedMap, tagMap config.TagIndexedMap) error {
	// collect create file entries
	createBranches := make(map[string]string)
	updateBranches := make(map[string][2]string)
	for branch, latestCommitID := range branchMap {
		found := false
		for indexedBranch, prevCommitID := range indexed.Branches {
			if branch == indexedBranch {
				found = true
				if latestCommitID == prevCommitID {
					log.Printf("Already up-to-date. %s", getLoggingTag(repo, branch, latestCommitID))
				} else {
					updateBranches[branch] = [2]string{prevCommitID, latestCommitID}
				}
				break
			}
		}
		if !found {
			createBranches[branch] = latestCommitID
		}
	}

	createTags := make(map[string]string)
	updateTags := make(map[string][2]string)
	for tag, latestCommitID := range tagMap {
		found := false
		for indexedTag, prevCommitID := range indexed.Tags {
			if tag == indexedTag {
				found = true
				if latestCommitID == prevCommitID {
					log.Printf("Already up-to-date. %s", getLoggingTag(repo, tag, latestCommitID))
				} else {
					updateTags[tag] = [2]string{prevCommitID, latestCommitID}
				}
				break
			}
		}
		if !found {
			createTags[tag] = latestCommitID
		}
	}

	queue := make(chan indexer.FileIndexOperation, 100)

	// process
	g.UpsertIndex(queue, bar, repo, createBranches, createTags, updateBranches, updateTags)

	callBatch := func(operations []indexer.FileIndexOperation) {
		err := g.indexer.BatchFileIndex(operations)
		if err != nil {
			errors.Errorf("Batch indexed error: %+v", err)
		} else {
			// fmt.Printf("Batch indexed %d files.\n", len(operations))
		}
		bar.Add(len(operations))
	}

	// batch
	operations := []indexer.FileIndexOperation{}
	var opsSize int64 = 0
	var batchLimitSize int64 = 1024 * 1024 // 1MB

	// fmt.Println("start queue reading")

	for op := range queue {
		operations = append(operations, op)
		opsSize += op.FileIndex.Size

		// show progress
		// if len(operations)%80 == 0 {
		// 	fmt.Printf("\n")
		// }
		// fmt.Printf(".")

		if opsSize >= batchLimitSize {
			// fmt.Printf("\n")

			callBatch(operations)

			// reset
			operations = nil
			opsSize = 0
		}
	}

	// remains
	if len(operations) > 0 {
		// fmt.Printf("\n")
		callBatch(operations)
	}

	// Save config after index completed
	err := g.config.UpdateIndexed(config.Indexed{Organization: repo.Organization, Project: repo.Project, Repository: repo.Repository, Branches: branchMap, Tags: tagMap})

	if err != nil {
		return errors.Wrapf(err, "Faild to update indexed.")
	}

	return nil
}

func (g *GitImporter) UpsertIndex(queue chan indexer.FileIndexOperation, bar *pb.ProgressBar, r *repo.GitRepo, branchMap map[string]string, tagMap map[string]string, updateBranchMap map[string][2]string, updateTagMap map[string][2]string) error {
	addFiles, err := r.GetFileEntriesMap(branchMap, tagMap)
	if err != nil {
		return errors.Wrapf(err, "Failed to get file entries. branches: %v tags: %v", branchMap, tagMap)
	}

	updateAddFiles, delFiles, err := r.GetDiffFileEntriesMap(updateBranchMap, updateTagMap)
	if err != nil {
		return errors.Wrapf(err, "Failed to get diff. branches: %v tags: %v", branchMap, tagMap)
	}

	var wg sync.WaitGroup

	wg.Add(1)
	go func() {
		defer wg.Done()
		g.handleAddFiles(queue, bar, r, addFiles)
		g.handleAddFiles(queue, bar, r, updateAddFiles)
		g.handleDelFiles(queue, bar, r, delFiles)
	}()

	go func() {
		wg.Wait()
		close(queue)
	}()

	return nil
}

func (g *GitImporter) handleAddFiles(queue chan indexer.FileIndexOperation, bar *pb.ProgressBar, r *repo.GitRepo, addFiles map[string]repo.GitFile) {
	if len(addFiles) == 0 {
		return
	}

	var wg sync.WaitGroup

	for blob, file := range addFiles {
		// check size
		if file.Size > g.config.SizeLimit {
			continue
		}

		wg.Add(1)
		go func(blob string, file repo.GitFile) {
			defer wg.Done()

			for path, loc := range file.Locations {
				// check contentType and retrive the file content
				// !! this will be heavy process !!
				contentType, content, err := g.parseContent(r, blob)
				if err != nil {
					log.Printf("Failed to parse file. [%s] - %s %+v\n", blob, path, err)
					continue
					// return errors.Wrapf(err, "Failed to parse file. [%s] - %s\n", blob, path)
				}

				// @TODO Extract text from binary in the future?
				if !strings.HasPrefix(contentType, "text/") && contentType != "application/octet-stream" {
					continue
				}

				text, encoding, err := readText(content)
				if err != nil {
					text = string(content)
					encoding = "utf8"
				}

				fileIndex := indexer.FileIndex{
					Metadata: indexer.Metadata{
						Blob:         blob,
						Organization: r.Organization,
						Project:      r.Project,
						Repository:   r.Repository,
						Branches:     loc.Branches,
						Tags:         loc.Tags,
						Path:         path,
						Ext:          indexer.GetExt(path),
						Encoding:     encoding,
						Size:         file.Size,
					},
					Content: text,
				}

				bar.Total = bar.Total + 1

				queue <- indexer.FileIndexOperation{Method: indexer.ADD, FileIndex: fileIndex}
			}
		}(blob, file)
	}
	wg.Wait()
}

// How to detect encoding
// http://qiita.com/nobuhito/items/ff782f64e32f7ed95e43
func readText(body []byte) (string, string, error) {
	var f []byte
	encodings := []string{"shift_jis", "utf8"}
	var enc string
	for i := range encodings {
		enc = encodings[i]
		if enc != "" {
			ee, _ := charset.Lookup(enc)
			if ee == nil {
				continue
			}
			var buf bytes.Buffer
			ic := transform.NewWriter(&buf, ee.NewDecoder())
			_, err := ic.Write(body)
			if err != nil {
				continue
			}
			err = ic.Close()
			if err != nil {
				continue
			}
			f = buf.Bytes()
			break
		}
	}
	return string(f), enc, nil
}

func (g *GitImporter) handleDelFiles(queue chan indexer.FileIndexOperation, bar *pb.ProgressBar, r *repo.GitRepo, delFiles map[string]repo.GitFile) {
	for blob, file := range delFiles {
		for path, loc := range file.Locations {
			fileIndex := indexer.FileIndex{
				Metadata: indexer.Metadata{
					Blob:         blob,
					Organization: r.Organization,
					Project:      r.Project,
					Repository:   r.Repository,
					Branches:     loc.Branches,
					Tags:         loc.Tags,
					Path:         path,
				},
			}

			bar.Total = bar.Total + 1

			// Delete index
			queue <- indexer.FileIndexOperation{Method: indexer.DELETE, FileIndex: fileIndex}
		}
	}
}

func (g *GitImporter) parseContent(repo *repo.GitRepo, blob string) (string, []byte, error) {
	contentType, content, err := repo.DetectBlobContentType(blob)
	if err != nil {
		return "", nil, errors.Wrapf(err, "Failed to read contentType. %s", blob)
	}
	return contentType, content, nil
}

func getLoggingTag(repo *repo.GitRepo, ref string, commitId string) string {
	tag := fmt.Sprintf("%s:%s/%s (%s) [%s]", repo.Organization, repo.Project, repo.Repository, ref, commitId)
	return tag
}
<commit_msg>Add worker system for scanning git files<commit_after>package importer

import (
	"fmt"
	"log"
	"strings"

	pb "gopkg.in/cheggaaa/pb.v1"
	// "io"
	// "io/ioutil"
	// "os"
	"sync"

	"time"

	"github.com/pkg/errors"
	"github.com/wadahiro/gitss/server/config"
	"github.com/wadahiro/gitss/server/indexer"
	"github.com/wadahiro/gitss/server/repo"
	// "github.com/wadahiro/gitss/server/util"
	"bytes"

	"golang.org/x/net/html/charset"
	"golang.org/x/text/transform"
)

type GitImporter struct {
	config  *config.Config
	indexer indexer.Indexer
	reader  *repo.GitRepoReader
	debug   bool
}

func NewGitImporter(config *config.Config, indexer indexer.Indexer) *GitImporter {
	r := repo.NewGitRepoReader(config)
	return &GitImporter{config: config, indexer: indexer, reader: r, debug: config.Debug}
}

func (g *GitImporter) Run(organization string, project string, url string) {
	log.Printf("Clone from %s %s %s\n", organization, project, url)

	repo, err := g.reader.CloneGitRepo(organization, project, url)
	if err != nil {
		log.Printf("Not found the repository: %s:%s/%s %+v\n", organization, project, url, err)
		return
	}

	repo.FetchAll()

	log.Printf("Fetched all. %s %s %s \n", organization, project, url)

	// branches and tags in the git repository (include/exclude filters are applied)
	branchMap, tagMap, err := repo.GetLatestCommitIdsMap()
	if err != nil {
		log.Printf("Failed to get latest commitIds of branch and tag. %+v\n", err)
		return
	}

	// branches in the config file
	indexed := g.config.GetIndexed(organization, project, repo.Repository)

	log.Printf("Start indexing for %s:%s/%s branches: %v -> %v, tags: %v -> %v\n", organization, project, repo.Repository, indexed.Branches, branchMap, indexed.Tags, tagMap)

	// progress bar
	bar := pb.StartNew(0)
	bar.ShowPercent = true

	start := time.Now()

	err = g.runIndexing(bar, repo, url, indexed, branchMap, tagMap)
	if err != nil {
		log.Printf("Failed to index. %+v", err)
		return
	}

	// Remove index for removed branches
	removeBranches := []string{}
	for ref, _ := range indexed.Branches {
		found := false
		for branch := range branchMap {
			if ref == branch {
				found = true
				break
			}
		}
		if !found {
			removeBranches = append(removeBranches, ref)
		}
	}

	bar.Total = bar.Total + int64(len(removeBranches))

	// Remove index for removed tags
	removeTags := []string{}
	for ref, _ := range indexed.Tags {
		found := false
		for branch := range branchMap {
			if ref == branch {
				found = true
				break
			}
		}
		if !found {
			removeTags = append(removeTags, ref)
		}
	}

	bar.Total = bar.Total + int64(len(removeTags))

	if len(removeBranches) > 0 || len(removeTags) > 0 {
		log.Printf("Start index deleting for %s:%s/%s (%v) (%v)\n", organization, project, repo.Repository, removeBranches, removeTags)
		g.indexer.DeleteIndexByRefs(organization, project, repo.Repository, removeBranches, removeTags)

		bar.Add(len(removeBranches) + len(removeTags))

		// Save config after deleting index completed
		g.config.DeleteIndexed(organization, project, repo.Repository, removeBranches, removeTags)
	}

	end := time.Now()
	time := (end.Sub(start)).Seconds()

	bar.FinishPrint(fmt.Sprintf("Indexing Complete! [%f seconds] for %s:%s/%s\n", time, organization, project, repo.Repository))
}

func (g *GitImporter) runIndexing(bar *pb.ProgressBar, repo *repo.GitRepo, url string, indexed config.Indexed, branchMap config.BrancheIndexedMap, tagMap config.TagIndexedMap) error {
	// collect create file entries
	createBranches := make(map[string]string)
	updateBranches := make(map[string][2]string)
	for branch, latestCommitID := range branchMap {
		found := false
		for indexedBranch, prevCommitID := range indexed.Branches {
			if branch == indexedBranch {
				found = true
				if latestCommitID == prevCommitID {
					log.Printf("Already up-to-date. %s", getLoggingTag(repo, branch, latestCommitID))
				} else {
					updateBranches[branch] = [2]string{prevCommitID, latestCommitID}
				}
				break
			}
		}
		if !found {
			createBranches[branch] = latestCommitID
		}
	}

	createTags := make(map[string]string)
	updateTags := make(map[string][2]string)
	for tag, latestCommitID := range tagMap {
		found := false
		for indexedTag, prevCommitID := range indexed.Tags {
			if tag == indexedTag {
				found = true
				if latestCommitID == prevCommitID {
					log.Printf("Already up-to-date. %s", getLoggingTag(repo, tag, latestCommitID))
				} else {
					updateTags[tag] = [2]string{prevCommitID, latestCommitID}
				}
				break
			}
		}
		if !found {
			createTags[tag] = latestCommitID
		}
	}

	queue := make(chan indexer.FileIndexOperation, 100)

	// process
	g.UpsertIndex(queue, bar, repo, createBranches, createTags, updateBranches, updateTags)

	callBatch := func(operations []indexer.FileIndexOperation) {
		err := g.indexer.BatchFileIndex(operations)
		if err != nil {
			errors.Errorf("Batch indexed error: %+v", err)
		} else {
			// fmt.Printf("Batch indexed %d files.\n", len(operations))
		}
		bar.Add(len(operations))
	}

	// batch
	operations := []indexer.FileIndexOperation{}
	var opsSize int64 = 0
	var batchLimitSize int64 = 1024 * 512 // 512KB

	// fmt.Println("start queue reading")

	for op := range queue {
		operations = append(operations, op)
		opsSize += op.FileIndex.Size

		// show progress
		// if len(operations)%80 == 0 {
		// 	fmt.Printf("\n")
		// }
		// fmt.Printf(".")

		if opsSize >= batchLimitSize {
			// fmt.Printf("\n")

			callBatch(operations)

			// reset
			operations = nil
			opsSize = 0
		}
	}

	// remains
	if len(operations) > 0 {
		// fmt.Printf("\n")
		callBatch(operations)
	}

	// Save config after index completed
	err := g.config.UpdateIndexed(config.Indexed{Organization: repo.Organization, Project: repo.Project, Repository: repo.Repository, Branches: branchMap, Tags: tagMap})

	if err != nil {
		return errors.Wrapf(err, "Faild to update indexed.")
	}

	return nil
}

func (g *GitImporter) UpsertIndex(queue chan indexer.FileIndexOperation, bar *pb.ProgressBar, r *repo.GitRepo, branchMap map[string]string, tagMap map[string]string, updateBranchMap map[string][2]string, updateTagMap map[string][2]string) error {
	addFiles, err := r.GetFileEntriesMap(branchMap, tagMap)
	if err != nil {
		return errors.Wrapf(err, "Failed to get file entries. branches: %v tags: %v", branchMap, tagMap)
	}

	updateAddFiles, delFiles, err := r.GetDiffFileEntriesMap(updateBranchMap, updateTagMap)
	if err != nil {
		return errors.Wrapf(err, "Failed to get diff. branches: %v tags: %v", branchMap, tagMap)
	}

	var wg sync.WaitGroup

	wg.Add(1)
	go func() {
		defer wg.Done()
		g.handleAddFiles(queue, bar, r, addFiles)
		g.handleAddFiles(queue, bar, r, updateAddFiles)
		g.handleDelFiles(queue, bar, r, delFiles)
	}()

	go func() {
		wg.Wait()
		close(queue)
	}()

	return nil
}

func (g *GitImporter) handleAddFiles(queue chan indexer.FileIndexOperation, bar *pb.ProgressBar, r *repo.GitRepo, addFiles map[string]repo.GitFile) {
	if len(addFiles) == 0 {
		return
	}

	var wg sync.WaitGroup
	scanQueue := make(chan ScannedFile, 5)

	for i := 0; i < 3; i++ {
		wg.Add(1)
		go scanFiles(&wg, scanQueue, queue, g, r, bar)
	}

	for blob, file := range addFiles {
		// check size
		if file.Size > g.config.SizeLimit {
			continue
		}

		scanQueue <- ScannedFile{Blob: blob, GitFile: file}
	}

	close(scanQueue)

	wg.Wait()
}

type ScannedFile struct {
	Blob    string
	GitFile repo.GitFile
}

func scanFiles(wg *sync.WaitGroup, scanQueue chan ScannedFile, queue chan indexer.FileIndexOperation, g *GitImporter, r *repo.GitRepo, bar *pb.ProgressBar) {
	defer wg.Done()

	for {
		scannedFile, ok := <-scanQueue
		if !ok {
			return
		}

		blob := scannedFile.Blob
		file := scannedFile.GitFile

		for path, loc := range file.Locations {
			// check contentType and retrive the file content
			// !! this will be heavy process !!
			contentType, content, err := g.parseContent(r, blob)
			if err != nil {
				log.Printf("Failed to parse file. [%s] - %s %+v\n", blob, path, err)
				continue
				// return errors.Wrapf(err, "Failed to parse file. [%s] - %s\n", blob, path)
			}

			// @TODO Extract text from binary in the future?
			if !strings.HasPrefix(contentType, "text/") && contentType != "application/octet-stream" {
				continue
			}

			text, encoding, err := readText(content)
			if err != nil {
				text = string(content)
				encoding = "utf8"
			}

			fileIndex := indexer.FileIndex{
				Metadata: indexer.Metadata{
					Blob:         blob,
					Organization: r.Organization,
					Project:      r.Project,
					Repository:   r.Repository,
					Branches:     loc.Branches,
					Tags:         loc.Tags,
					Path:         path,
					Ext:          indexer.GetExt(path),
					Encoding:     encoding,
					Size:         file.Size,
				},
				Content: text,
			}

			bar.Total = bar.Total + 1

			queue <- indexer.FileIndexOperation{Method: indexer.ADD, FileIndex: fileIndex}
		}
	}
}

// How to detect encoding
// http://qiita.com/nobuhito/items/ff782f64e32f7ed95e43
func readText(body []byte) (string, string, error) {
	var f []byte
	encodings := []string{"shift_jis", "utf8"}
	var enc string
	for i := range encodings {
		enc = encodings[i]
		if enc != "" {
			ee, _ := charset.Lookup(enc)
			if ee == nil {
				continue
			}
			var buf bytes.Buffer
			ic := transform.NewWriter(&buf, ee.NewDecoder())
			_, err := ic.Write(body)
			if err != nil {
				continue
			}
			err = ic.Close()
			if err != nil {
				continue
			}
			f = buf.Bytes()
			break
		}
	}
	return string(f), enc, nil
}

func (g *GitImporter) handleDelFiles(queue chan indexer.FileIndexOperation, bar *pb.ProgressBar, r *repo.GitRepo, delFiles map[string]repo.GitFile) {
	for blob, file := range delFiles {
		for path, loc := range file.Locations {
			fileIndex := indexer.FileIndex{
				Metadata: indexer.Metadata{
					Blob:         blob,
					Organization: r.Organization,
					Project:      r.Project,
					Repository:   r.Repository,
					Branches:     loc.Branches,
					Tags:         loc.Tags,
					Path:         path,
				},
			}

			bar.Total = bar.Total + 1

			// Delete index
			queue <- indexer.FileIndexOperation{Method: indexer.DELETE, FileIndex: fileIndex}
		}
	}
}

func (g *GitImporter) parseContent(repo *repo.GitRepo, blob string) (string, []byte, error) {
	contentType, content, err := repo.DetectBlobContentType(blob)
	if err != nil {
		return "", nil, errors.Wrapf(err, "Failed to read contentType. %s", blob)
	}
	return contentType, content, nil
}

func getLoggingTag(repo *repo.GitRepo, ref string, commitId string) string {
	tag := fmt.Sprintf("%s:%s/%s (%s) [%s]", repo.Organization, repo.Project, repo.Repository, ref, commitId)
	return tag
}
<|endoftext|>


<commit_before>package imageutil

import (
	"image"

	"golang.org/x/image/draw"
)

func OverlayCenterYLeftAlign(imgBg, imgOver image.Image) image.Image {
	output := image.NewRGBA(imgBg.Bounds())
	draw.Draw(output, imgBg.Bounds(), imgBg, image.ZP, draw.Src)

	h1 := imgBg.Bounds().Dy()
	h2 := imgOver.Bounds().Dy()
	offset := image.Pt(0, (h1-h2)/2)

	draw.Draw(output, imgOver.Bounds().Add(offset), imgOver, image.Point{}, draw.Src)
	return output
}

func MergeXSameY(images []image.Image, larger bool) image.Image {
	if len(images) == 0 {
		return nil
	} else if len(images) == 1 {
		return images[0]
	}
	images = ResizeSameY(images, larger)
	_, _, minY, _, sumX, _ := SliceXY(images, -1)
	output := image.NewRGBA(image.Rect(0, 0, sumX, minY))
	sumXPrev := 0
	for i, img := range images {
		if i == 0 {
			draw.Draw(output, img.Bounds(), img, image.Point{}, draw.Src)
		} else {
			imgOffset := image.Pt(sumXPrev, 0)
			draw.Draw(output, img.Bounds().Add(imgOffset), img, image.Point{}, draw.Src)
		}
		sumXPrev += img.Bounds().Dx()
	}
	return output
}

func mergeXSameYTwo(img1, img2 image.Image, larger bool) image.Image {
	img1, img2 = ResizeSameYTwo(img1, img2, larger)
	output := image.NewRGBA(
		image.Rect(0, 0,
			img1.Bounds().Dx()+img2.Bounds().Dx(),
			img1.Bounds().Dy()))
	draw.Draw(output, img1.Bounds(), img1, image.Point{}, draw.Src)
	img2Offset := image.Pt(img1.Bounds().Dx(), 0)
	draw.Draw(output, img2.Bounds().Add(img2Offset), img2, image.Point{}, draw.Src)
	return output
}

func MergeXSameYRead(locations []string, larger bool) (image.Image, error) {
	images, err := ReadImages(locations)
	if err != nil {
		return nil, err
	}
	return MergeXSameY(images, true), nil
	/*
		img1, _, err := ReadImageAny(location1)
		if err != nil {
			return img1, err
		}
		img2, _, err := ReadImageAny(location2)
		if err != nil {
			return img2, err
		}
		return MergeXSameY(img1, img2, true), nil
	*/
}

func MergeYSameX(images []image.Image, larger bool) image.Image {
	if len(images) == 0 {
		return nil
	} else if len(images) == 1 {
		return images[0]
	}
	images = ResizeSameX(images, larger)
	minX, _, _, _, _, sumY := SliceXY(images, -1)
	output := image.NewRGBA(image.Rect(0, 0, minX, sumY))
	sumYPrev := 0
	for i, img := range images {
		if i == 0 {
			draw.Draw(output, img.Bounds(), img, image.Point{}, draw.Src)
		} else {
			imgOffset := image.Pt(0, sumYPrev)
			draw.Draw(output, img.Bounds().Add(imgOffset), img, image.Point{}, draw.Src)
		}
		sumYPrev += img.Bounds().Dy()
	}
	return output
}

func mergeYSameXTwo(img1, img2 image.Image, larger bool) image.Image {
	img1, img2 = ResizeSameXTwo(img1, img2, larger)
	output := image.NewRGBA(
		image.Rect(0, 0,
			img1.Bounds().Dx(),
			img1.Bounds().Dy()+img2.Bounds().Dy()))
	draw.Draw(output, img1.Bounds(), img1, image.Point{}, draw.Src)
	img2Offset := image.Pt(0, img1.Bounds().Dy())
	draw.Draw(output, img2.Bounds().Add(img2Offset), img2, image.Point{}, draw.Src)
	return output
}

func MergeYSameXRead(locations []string, larger bool) (image.Image, error) {
	images, err := ReadImages(locations)
	if err != nil {
		return nil, err
	}
	return MergeYSameX(images, true), nil
	/*
		img1, _, err := ReadImageAny(location1)
		if err != nil {
			return img1, err
		}
		img2, _, err := ReadImageAny(location2)
		if err != nil {
			return img2, err
		}
		return MergeYSameX(img1, img2, true), nil
	*/
}

func MatrixMergeRead(matrix [][]string, largerX, largerY bool) (image.Image, error) {
	matrixImages := [][]image.Image{}
	for _, row := range matrix {
		if len(row) == 0 {
			continue
		}
		images, err := ReadImages(row)
		if err != nil {
			return nil, err
		}
		matrixImages = append(matrixImages, images)
	}

	return MatrixMerge(matrixImages, largerX, largerY), nil
}

func MatrixMerge(matrix [][]image.Image, largerX, largerY bool) image.Image {
	if len(matrix) == 0 {
		return nil
	}
	rowImages := []image.Image{}
	for _, rowParts := range matrix {
		if len(rowParts) > 0 {
			rowImages = append(rowImages, MergeXSameY(rowParts, largerY))
		}
	}
	if len(rowImages) == 0 {
		return nil
	}
	return MergeYSameX(rowImages, largerX)
}

func merge4Read(location1, location2, location3, location4 string, larger bool) (image.Image, error) {
	img12, err := MergeXSameYRead([]string{location1, location2}, larger)
	if err != nil {
		return img12, err
	}

	img34, err := MergeXSameYRead([]string{location3, location4}, larger)
	if err != nil {
		return img34, err
	}

	return MergeYSameX([]image.Image{img12, img34}, larger), nil
}
<commit_msg>clean: imageutil: remove old code<commit_after>package imageutil

import (
	"image"

	"golang.org/x/image/draw"
)

func OverlayCenterYLeftAlign(imgBg, imgOver image.Image) image.Image {
	output := image.NewRGBA(imgBg.Bounds())
	draw.Draw(output, imgBg.Bounds(), imgBg, image.ZP, draw.Src)

	h1 := imgBg.Bounds().Dy()
	h2 := imgOver.Bounds().Dy()
	offset := image.Pt(0, (h1-h2)/2)

	draw.Draw(output, imgOver.Bounds().Add(offset), imgOver, image.Point{}, draw.Src)
	return output
}

func MergeXSameY(images []image.Image, larger bool) image.Image {
	if len(images) == 0 {
		return nil
	} else if len(images) == 1 {
		return images[0]
	}
	images = ResizeSameY(images, larger)
	_, _, minY, _, sumX, _ := SliceXY(images, -1)
	output := image.NewRGBA(image.Rect(0, 0, sumX, minY))
	sumXPrev := 0
	for i, img := range images {
		if i == 0 {
			draw.Draw(output, img.Bounds(), img, image.Point{}, draw.Src)
		} else {
			imgOffset := image.Pt(sumXPrev, 0)
			draw.Draw(output, img.Bounds().Add(imgOffset), img, image.Point{}, draw.Src)
		}
		sumXPrev += img.Bounds().Dx()
	}
	return output
}

func mergeXSameYTwo(img1, img2 image.Image, larger bool) image.Image {
	img1, img2 = ResizeSameYTwo(img1, img2, larger)
	output := image.NewRGBA(
		image.Rect(0, 0,
			img1.Bounds().Dx()+img2.Bounds().Dx(),
			img1.Bounds().Dy()))
	draw.Draw(output, img1.Bounds(), img1, image.Point{}, draw.Src)
	img2Offset := image.Pt(img1.Bounds().Dx(), 0)
	draw.Draw(output, img2.Bounds().Add(img2Offset), img2, image.Point{}, draw.Src)
	return output
}

func MergeXSameYRead(locations []string, larger bool) (image.Image, error) {
	images, err := ReadImages(locations)
	if err != nil {
		return nil, err
	}
	return MergeXSameY(images, true), nil
}

func MergeYSameX(images []image.Image, larger bool) image.Image {
	if len(images) == 0 {
		return nil
	} else if len(images) == 1 {
		return images[0]
	}
	images = ResizeSameX(images, larger)
	minX, _, _, _, _, sumY := SliceXY(images, -1)
	output := image.NewRGBA(image.Rect(0, 0, minX, sumY))
	sumYPrev := 0
	for i, img := range images {
		if i == 0 {
			draw.Draw(output, img.Bounds(), img, image.Point{}, draw.Src)
		} else {
			imgOffset := image.Pt(0, sumYPrev)
			draw.Draw(output, img.Bounds().Add(imgOffset), img, image.Point{}, draw.Src)
		}
		sumYPrev += img.Bounds().Dy()
	}
	return output
}

func mergeYSameXTwo(img1, img2 image.Image, larger bool) image.Image {
	img1, img2 = ResizeSameXTwo(img1, img2, larger)
	output := image.NewRGBA(
		image.Rect(0, 0,
			img1.Bounds().Dx(),
			img1.Bounds().Dy()+img2.Bounds().Dy()))
	draw.Draw(output, img1.Bounds(), img1, image.Point{}, draw.Src)
	img2Offset := image.Pt(0, img1.Bounds().Dy())
	draw.Draw(output, img2.Bounds().Add(img2Offset), img2, image.Point{}, draw.Src)
	return output
}

func MergeYSameXRead(locations []string, larger bool) (image.Image, error) {
	images, err := ReadImages(locations)
	if err != nil {
		return nil, err
	}
	return MergeYSameX(images, true), nil
}

func MatrixMergeRead(matrix [][]string, largerX, largerY bool) (image.Image, error) {
	matrixImages := [][]image.Image{}
	for _, row := range matrix {
		if len(row) == 0 {
			continue
		}
		images, err := ReadImages(row)
		if err != nil {
			return nil, err
		}
		matrixImages = append(matrixImages, images)
	}

	return MatrixMerge(matrixImages, largerX, largerY), nil
}

func MatrixMerge(matrix [][]image.Image, largerX, largerY bool) image.Image {
	if len(matrix) == 0 {
		return nil
	}
	rowImages := []image.Image{}
	for _, rowParts := range matrix {
		if len(rowParts) > 0 {
			rowImages = append(rowImages, MergeXSameY(rowParts, largerY))
		}
	}
	if len(rowImages) == 0 {
		return nil
	}
	return MergeYSameX(rowImages, largerX)
}

func merge4Read(location1, location2, location3, location4 string, larger bool) (image.Image, error) {
	img12, err := MergeXSameYRead([]string{location1, location2}, larger)
	if err != nil {
		return img12, err
	}

	img34, err := MergeXSameYRead([]string{location3, location4}, larger)
	if err != nil {
		return img34, err
	}

	return MergeYSameX([]image.Image{img12, img34}, larger), nil
}
<|endoftext|>