<commit_before>// Copyright 2013 Bobby Powers. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package boosd

import (
	"bytes"
	"fmt"
	"github.com/bpowers/boosd/runtime"
	"go/ast"
	"go/parser"
	"go/token"
	"log"
	"strconv"
	"text/template"
	"unicode"
)

const fileTmpl = `
{{define "modelTmpl"}}
var (
	mdl{{$.CamelName}}Name = "{{$.Name}}"
	mdl{{$.CamelName}}Vars = map[string]runtime.Var{
{{range $.Vars}}
		"{{.Name}}": runtime.Var{"{{.Name}}", runtime.{{.Type}}},{{end}}
	}
)

type sim{{$.CamelName}} struct {
	runtime.BaseSim
}

type mdl{{$.CamelName}} struct {
	runtime.BaseModel
}

func sim{{$.CamelName}}Step(s *runtime.BaseSim, dt float64) {
{{range $.Equations}}
	{{.}}{{end}}
{{range $.Stocks}}
	{{.}}{{end}}
}

func (m *mdl{{$.CamelName}}) NewSim(name string) runtime.Sim {
	ts := runtime.Timespec{
		Start:    {{$.Time.Start}},
		End:      {{$.Time.End}},
		DT:       {{$.Time.DT}},
		SaveStep: {{$.Time.SaveStep}},
	}
	tables := map[string]runtime.Table{}
	consts := runtime.Data{}

	s := new(sim{{$.CamelName}})
	s.InstanceName = name
	s.Init(m, ts, tables, consts)
	s.Step = sim{{$.CamelName}}Step
{{range $.Initials}}
	{{.}}{{end}}

	s.Curr["time"] = ts.Start

	runtime.RegisterSim(mdl{{$.CamelName}}Name, s)

	return s
}
{{end}}

package main

import (
	"github.com/bpowers/boosd/runtime"
)

{{range $.Models}}{{template "modelTmpl" .}}{{end}}

func init() {
	m := &mdlMain{
		runtime.BaseModel{
			MName: mdlMainName,
			Vars:  mdlMainVars,
		},
	}

	runtime.RegisterModel(m)
}

func main() {
	runtime.Main()
}
`

type genModel struct {
	Name      string
	CamelName string // camelcased
	Vars      map[string]runtime.Var
	Time      runtime.Timespec
	Equations []string
	Stocks    []string
	Initials  []string
	Abstract  bool
}

type generator struct {
	Models map[string]*genModel
	curr *genModel
}

func (g *generator) declList(list []Decl) {
}

// constEval returns the float64 value represented by Expr, or an
// error if it can't be evaluated at compile time.
func constEval(e Expr) (v float64, err error) {
	val, ok := e.(*UnitExpr)
	if !ok {
		err = fmt.Errorf("timespec val %T not UnitExpr", e)
		return
	}
	basic, ok := val.X.(*BasicLit)
	if !ok {
		err = fmt.Errorf("timespec val %T not BasicLit", val.X)
		return
	}
	return strconv.ParseFloat(basic.Value, 64)
}

func isConst(e Expr) bool {
	if _, err := constEval(e); err == nil {
		return true
	}
	return false
}

func kvConvert(e Expr) (k string, v Expr, err error) {
	kv, ok := e.(*KeyValueExpr)
	if !ok {
		err = fmt.Errorf("e %T not KVeyValueExpr: %v", e, e)
		return
	}
	ident, ok := kv.Key.(*Ident)
	if !ok {
		err = fmt.Errorf("e key %T not Ident: %v", kv.Key, kv.Key)
		return
	}
	return ident.Name, kv.Value, nil
}

func (g *generator) timespec(elts []Expr) {
	for _, e := range elts {
		k, val, err := kvConvert(e)
		if err != nil {
			panic(fmt.Sprintf("kvConvert(%v): %s", e, err))
		}
		v, err := constEval(val)
		if err != nil {
			panic(fmt.Sprintf("timespec constEval(%v): %s",
				val, err))
		}
		switch k {
		case "start":
			g.curr.Time.Start = v
		case "end":
			g.curr.Time.End = v
		case "dt":
			g.curr.Time.DT = v
		case "save_step":
			g.curr.Time.SaveStep = v
		default:
			panic(fmt.Sprintf("timespec unknown key %s", k))
		}
	}
}

var identAux = &Ident{Name: "aux"}

func varFromDecl(d *VarDecl) (v runtime.Var, err error) {
	if d.Type == nil {
		d.Type = identAux
	}
	return runtime.Var{d.Name.Name, runtime.TypeForName(d.Type.Name)}, nil
}

func (g *generator) initial(name string, expr Expr) (err error) {
	val, err := constEval(expr)
	if err == nil {
		init := fmt.Sprintf(`s.Curr["%s"] = %f`, name, val)
		g.curr.Initials = append(g.curr.Initials, init)
	} else {
		unitExpr, ok := expr.(*UnitExpr)
		if !ok {
			return fmt.Errorf("initial(%s): not unitexpr", name)
		}
		switch e := unitExpr.X.(type) {
		case *RefExpr:
			if _, ok := g.curr.Vars[e.Ident.Name]; ok {
				init := fmt.Sprintf(`s.Curr["%s"] = s.Curr["%s"]`,
					name, e.Name)
				g.curr.Initials = append(g.curr.Initials, init)
				err = nil
			} else {
				err = fmt.Errorf("initial(%s): non-const ident %v",
					name, e.Name)
			}
		default:
			err = fmt.Errorf("initial(%s): non-const %v (%T)",
				name, unitExpr.X, unitExpr.X)
		}
	}
	return
}

func (g *generator) stock(name string, expr Expr) error {
	cl, ok := expr.(*CompositeLit)
	if !ok {
		panic(fmt.Sprintf("stock is %T, not CompositeLit", expr))
	}
	var bi, in, out string
	for _, e := range cl.Elts {
		k, val, err := kvConvert(e)
		if err != nil {
			panic(fmt.Sprintf("kvConvert(%s): %s", name, err))
		}
		switch k {
		case "initial":
			if err := g.initial(name, val); err != nil {
				return fmt.Errorf("initial(%s, %s): %s",
					name, val, err)
			}
		case "biflow":
			bi = fmt.Sprintf("+%s", val)
		case "inflow":
			in = fmt.Sprintf("+%s", val)
		case "outflow":
			out = fmt.Sprintf("-(%s)", val)
		default:
			panic(fmt.Sprintf("stock(%s): unknown k %s",
				name, k))
		}
	}
	eqn := fmt.Sprintf(`s.Next["%s"] = s.Curr["%s"] + (%s %s %s)*dt`, name, name, bi, in, out)
	g.curr.Stocks = append(g.curr.Stocks, eqn)
	return nil
}

func (g *generator) expr(name string, expr Expr) {
	if g.curr.Vars[name].Type == runtime.TyAux {
		if isConst(expr) {
			g.initial(name, expr)
			eqn := fmt.Sprintf(`s.Next["%s"] = s.Curr["%s"]`, name, name)
			g.curr.Stocks = append(g.curr.Stocks, eqn)
		} else {
			eqn := fmt.Sprintf(`s.Curr["%s"] = %s`, name, expr)
			g.curr.Equations = append(g.curr.Equations, eqn)
		}
	} else {
		eqn := fmt.Sprintf(`s.Curr["%s"] = %s`, name, expr)
		g.curr.Equations = append(g.curr.Equations, eqn)
	}
}

func (g *generator) assign(s *AssignStmt) error {
	if s.Lhs.Name.Name == "timespec" {
		c, ok := s.Rhs.(*CompositeLit)
		if !ok {
			return fmt.Errorf("timespec is %T, not CompositeLit",
				s.Rhs)
		}
		g.timespec(c.Elts)
		return nil
	}
	v, ok := g.curr.Vars[s.Lhs.Name.Name]
	if !ok {
		return fmt.Errorf("assign: unknown v '%s'?", s.Lhs.Name.Name)
	}
	if v.Type == runtime.TyStock {
		if err := g.stock(v.Name, s.Rhs); err != nil {
			return err
		}
	} else {
		g.expr(v.Name, s.Rhs)
	}
	return nil
}

func (g *generator) stmt(s Stmt) error {
	switch ss := s.(type) {
	case *AssignStmt:
		if err := g.assign(ss); err != nil {
			return err
		}
	case *DeclStmt:
	default:
		log.Printf("stmt(%T): unimplemented - %T", s, s)
	}
	return nil
}

func (g *generator) vars(stmts ...Stmt) (err error) {
	addVar := func(vd *VarDecl) error {
		v, err := varFromDecl(vd)
		if err != nil {
			return fmt.Errorf("varFromDecl(%v): %s", vd, err)
		}
		g.curr.Vars[v.Name] = v
		return nil
	}
	for _, s := range stmts {
		switch ss := s.(type) {
		case *AssignStmt:
			err = addVar(ss.Lhs)
		case *DeclStmt:
			g.curr.Abstract = true
			err = addVar(ss.Decl)
		default:
			err = fmt.Errorf("varFromDecl(%v): unknown ty %T",
				s, ss)
		}
	}
	return
}

func (g *generator) model(m *ModelDecl) error {
	name := m.Name.Name
	camelName := fmt.Sprintf("%c%s", unicode.ToUpper(rune(name[0])), name[1:])
	g.curr = &genModel{
		Name: name,
		CamelName: camelName,
		Vars: map[string]runtime.Var{},
		Equations: []string{},
		Stocks: []string{},
		Initials: []string{},
	}
	g.vars(m.Body.List...)
	for _, s := range m.Body.List {
		if err := g.stmt(s); err != nil {
			return err
		}
	}
	g.Models[m.Name.Name] = g.curr
	g.curr = nil

	return nil
}

func (g *generator) file(f *File) ([]byte, error) {
	for _, d := range f.Decls {
		md, ok := d.(*ModelDecl)
		if !ok {
			log.Printf("top level decl that isn't a model: %v (%T)", d, d)
			continue
		}
		if err := g.model(md); err != nil {
			return nil, fmt.Errorf("g.model: %s", err)
		}
	}

	var buf bytes.Buffer
	tmpl := template.New("model.go")
	if _, err := tmpl.Parse(fileTmpl); err != nil {
		panic(fmt.Sprintf("Parse(modelTmpl): %s", err))
	}
	if err := tmpl.Execute(&buf, g); err != nil {
		panic(fmt.Sprintf("Execute(%v): %s", g, err))
	}

	return buf.Bytes(), nil
}

func GenGo(f *File) (*ast.File, error) {
	g := &generator{
		Models: map[string]*genModel{},
	}

	code, err := g.file(f)
	if err != nil {
		return nil, fmt.Errorf("g.file: %s", err)
	}
	log.Printf("c: %s", code)

	fset := token.NewFileSet()
	goFile, err := parser.ParseFile(fset, "model.go", code, 0)
	if err != nil {
		return nil, err
	}

	return goFile, nil
}
<commit_msg>boosd: don't debug print all the code<commit_after>// Copyright 2013 Bobby Powers. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package boosd

import (
	"bytes"
	"fmt"
	"github.com/bpowers/boosd/runtime"
	"go/ast"
	"go/parser"
	"go/token"
	"log"
	"strconv"
	"text/template"
	"unicode"
)

const fileTmpl = `
{{define "modelTmpl"}}
var (
	mdl{{$.CamelName}}Name = "{{$.Name}}"
	mdl{{$.CamelName}}Vars = map[string]runtime.Var{
{{range $.Vars}}
		"{{.Name}}": runtime.Var{"{{.Name}}", runtime.{{.Type}}},{{end}}
	}
)

type sim{{$.CamelName}} struct {
	runtime.BaseSim
}

type mdl{{$.CamelName}} struct {
	runtime.BaseModel
}

func sim{{$.CamelName}}Step(s *runtime.BaseSim, dt float64) {
{{range $.Equations}}
	{{.}}{{end}}
{{range $.Stocks}}
	{{.}}{{end}}
}

func (m *mdl{{$.CamelName}}) NewSim(name string) runtime.Sim {
	ts := runtime.Timespec{
		Start:    {{$.Time.Start}},
		End:      {{$.Time.End}},
		DT:       {{$.Time.DT}},
		SaveStep: {{$.Time.SaveStep}},
	}
	tables := map[string]runtime.Table{}
	consts := runtime.Data{}

	s := new(sim{{$.CamelName}})
	s.InstanceName = name
	s.Init(m, ts, tables, consts)
	s.Step = sim{{$.CamelName}}Step
{{range $.Initials}}
	{{.}}{{end}}

	s.Curr["time"] = ts.Start

	runtime.RegisterSim(mdl{{$.CamelName}}Name, s)

	return s
}
{{end}}

package main

import (
	"github.com/bpowers/boosd/runtime"
)

{{range $.Models}}{{template "modelTmpl" .}}{{end}}

func init() {
	m := &mdlMain{
		runtime.BaseModel{
			MName: mdlMainName,
			Vars:  mdlMainVars,
		},
	}

	runtime.RegisterModel(m)
}

func main() {
	runtime.Main()
}
`

type genModel struct {
	Name      string
	CamelName string // camelcased
	Vars      map[string]runtime.Var
	Time      runtime.Timespec
	Equations []string
	Stocks    []string
	Initials  []string
	Abstract  bool
}

type generator struct {
	Models map[string]*genModel
	curr *genModel
}

func (g *generator) declList(list []Decl) {
}

// constEval returns the float64 value represented by Expr, or an
// error if it can't be evaluated at compile time.
func constEval(e Expr) (v float64, err error) {
	val, ok := e.(*UnitExpr)
	if !ok {
		err = fmt.Errorf("timespec val %T not UnitExpr", e)
		return
	}
	basic, ok := val.X.(*BasicLit)
	if !ok {
		err = fmt.Errorf("timespec val %T not BasicLit", val.X)
		return
	}
	return strconv.ParseFloat(basic.Value, 64)
}

func isConst(e Expr) bool {
	if _, err := constEval(e); err == nil {
		return true
	}
	return false
}

func kvConvert(e Expr) (k string, v Expr, err error) {
	kv, ok := e.(*KeyValueExpr)
	if !ok {
		err = fmt.Errorf("e %T not KVeyValueExpr: %v", e, e)
		return
	}
	ident, ok := kv.Key.(*Ident)
	if !ok {
		err = fmt.Errorf("e key %T not Ident: %v", kv.Key, kv.Key)
		return
	}
	return ident.Name, kv.Value, nil
}

func (g *generator) timespec(elts []Expr) {
	for _, e := range elts {
		k, val, err := kvConvert(e)
		if err != nil {
			panic(fmt.Sprintf("kvConvert(%v): %s", e, err))
		}
		v, err := constEval(val)
		if err != nil {
			panic(fmt.Sprintf("timespec constEval(%v): %s",
				val, err))
		}
		switch k {
		case "start":
			g.curr.Time.Start = v
		case "end":
			g.curr.Time.End = v
		case "dt":
			g.curr.Time.DT = v
		case "save_step":
			g.curr.Time.SaveStep = v
		default:
			panic(fmt.Sprintf("timespec unknown key %s", k))
		}
	}
}

var identAux = &Ident{Name: "aux"}

func varFromDecl(d *VarDecl) (v runtime.Var, err error) {
	if d.Type == nil {
		d.Type = identAux
	}
	return runtime.Var{d.Name.Name, runtime.TypeForName(d.Type.Name)}, nil
}

func (g *generator) initial(name string, expr Expr) (err error) {
	val, err := constEval(expr)
	if err == nil {
		init := fmt.Sprintf(`s.Curr["%s"] = %f`, name, val)
		g.curr.Initials = append(g.curr.Initials, init)
	} else {
		unitExpr, ok := expr.(*UnitExpr)
		if !ok {
			return fmt.Errorf("initial(%s): not unitexpr", name)
		}
		switch e := unitExpr.X.(type) {
		case *RefExpr:
			if _, ok := g.curr.Vars[e.Ident.Name]; ok {
				init := fmt.Sprintf(`s.Curr["%s"] = s.Curr["%s"]`,
					name, e.Name)
				g.curr.Initials = append(g.curr.Initials, init)
				err = nil
			} else {
				err = fmt.Errorf("initial(%s): non-const ident %v",
					name, e.Name)
			}
		default:
			err = fmt.Errorf("initial(%s): non-const %v (%T)",
				name, unitExpr.X, unitExpr.X)
		}
	}
	return
}

func (g *generator) stock(name string, expr Expr) error {
	cl, ok := expr.(*CompositeLit)
	if !ok {
		panic(fmt.Sprintf("stock is %T, not CompositeLit", expr))
	}
	var bi, in, out string
	for _, e := range cl.Elts {
		k, val, err := kvConvert(e)
		if err != nil {
			panic(fmt.Sprintf("kvConvert(%s): %s", name, err))
		}
		switch k {
		case "initial":
			if err := g.initial(name, val); err != nil {
				return fmt.Errorf("initial(%s, %s): %s",
					name, val, err)
			}
		case "biflow":
			bi = fmt.Sprintf("+%s", val)
		case "inflow":
			in = fmt.Sprintf("+%s", val)
		case "outflow":
			out = fmt.Sprintf("-(%s)", val)
		default:
			panic(fmt.Sprintf("stock(%s): unknown k %s",
				name, k))
		}
	}
	eqn := fmt.Sprintf(`s.Next["%s"] = s.Curr["%s"] + (%s %s %s)*dt`, name, name, bi, in, out)
	g.curr.Stocks = append(g.curr.Stocks, eqn)
	return nil
}

func (g *generator) expr(name string, expr Expr) {
	if g.curr.Vars[name].Type == runtime.TyAux {
		if isConst(expr) {
			g.initial(name, expr)
			eqn := fmt.Sprintf(`s.Next["%s"] = s.Curr["%s"]`, name, name)
			g.curr.Stocks = append(g.curr.Stocks, eqn)
		} else {
			eqn := fmt.Sprintf(`s.Curr["%s"] = %s`, name, expr)
			g.curr.Equations = append(g.curr.Equations, eqn)
		}
	} else {
		eqn := fmt.Sprintf(`s.Curr["%s"] = %s`, name, expr)
		g.curr.Equations = append(g.curr.Equations, eqn)
	}
}

func (g *generator) assign(s *AssignStmt) error {
	if s.Lhs.Name.Name == "timespec" {
		c, ok := s.Rhs.(*CompositeLit)
		if !ok {
			return fmt.Errorf("timespec is %T, not CompositeLit",
				s.Rhs)
		}
		g.timespec(c.Elts)
		return nil
	}
	v, ok := g.curr.Vars[s.Lhs.Name.Name]
	if !ok {
		return fmt.Errorf("assign: unknown v '%s'?", s.Lhs.Name.Name)
	}
	if v.Type == runtime.TyStock {
		if err := g.stock(v.Name, s.Rhs); err != nil {
			return err
		}
	} else {
		g.expr(v.Name, s.Rhs)
	}
	return nil
}

func (g *generator) stmt(s Stmt) error {
	switch ss := s.(type) {
	case *AssignStmt:
		if err := g.assign(ss); err != nil {
			return err
		}
	case *DeclStmt:
	default:
		log.Printf("stmt(%T): unimplemented - %T", s, s)
	}
	return nil
}

func (g *generator) vars(stmts ...Stmt) (err error) {
	addVar := func(vd *VarDecl) error {
		v, err := varFromDecl(vd)
		if err != nil {
			return fmt.Errorf("varFromDecl(%v): %s", vd, err)
		}
		g.curr.Vars[v.Name] = v
		return nil
	}
	for _, s := range stmts {
		switch ss := s.(type) {
		case *AssignStmt:
			err = addVar(ss.Lhs)
		case *DeclStmt:
			g.curr.Abstract = true
			err = addVar(ss.Decl)
		default:
			err = fmt.Errorf("varFromDecl(%v): unknown ty %T",
				s, ss)
		}
	}
	return
}

func (g *generator) model(m *ModelDecl) error {
	name := m.Name.Name
	camelName := fmt.Sprintf("%c%s", unicode.ToUpper(rune(name[0])), name[1:])
	g.curr = &genModel{
		Name: name,
		CamelName: camelName,
		Vars: map[string]runtime.Var{},
		Equations: []string{},
		Stocks: []string{},
		Initials: []string{},
	}
	g.vars(m.Body.List...)
	for _, s := range m.Body.List {
		if err := g.stmt(s); err != nil {
			return err
		}
	}
	g.Models[m.Name.Name] = g.curr
	g.curr = nil

	return nil
}

func (g *generator) file(f *File) ([]byte, error) {
	for _, d := range f.Decls {
		md, ok := d.(*ModelDecl)
		if !ok {
			log.Printf("top level decl that isn't a model: %v (%T)", d, d)
			continue
		}
		if err := g.model(md); err != nil {
			return nil, fmt.Errorf("g.model: %s", err)
		}
	}

	var buf bytes.Buffer
	tmpl := template.New("model.go")
	if _, err := tmpl.Parse(fileTmpl); err != nil {
		panic(fmt.Sprintf("Parse(modelTmpl): %s", err))
	}
	if err := tmpl.Execute(&buf, g); err != nil {
		panic(fmt.Sprintf("Execute(%v): %s", g, err))
	}

	return buf.Bytes(), nil
}

func GenGo(f *File) (*ast.File, error) {
	g := &generator{
		Models: map[string]*genModel{},
	}

	code, err := g.file(f)
	if err != nil {
		return nil, fmt.Errorf("g.file: %s", err)
	}
	//log.Printf("c: %s", code)

	fset := token.NewFileSet()
	goFile, err := parser.ParseFile(fset, "model.go", code, 0)
	if err != nil {
		return nil, err
	}

	return goFile, nil
}
<|endoftext|>


<commit_before>package interpreter

import (
	"fmt"

	"CodeCity/server/interpreter/ast"
	"CodeCity/server/interpreter/object"
)

// Interpreter implements a JavaScript interpreter.
type Interpreter struct {
	state state
	value object.Value
}

// NewInterpreter takes a JavaScript program, in the form of an
// JSON-encoded ESTree, and creates a new Interpreter that will
// execute that program.
func NewInterpreter(astJSON string) *Interpreter {
	var this = new(Interpreter)

	tree, err := ast.NewFromJSON(astJSON)
	if err != nil {
		panic(err)
	}
	this.state = newState(nil, *tree)
	this.state.(*stateBlockStatement).interpreter = this
	return this
}

// Step performs the next step in the evaluation of program.  Returns
// true if a step was executed; false if the program has terminated.
func (this *Interpreter) Step() bool {
	if this.state == nil {
		return false
	}
	fmt.Printf("Next step is a %T\n", this.state)
	this.state = this.state.step()
	return true
}

// Run runs the program to completion.
func (this *Interpreter) Run() {
	for this.Step() {
	}
}

// Value returns the final value computed by the last statement
// expression of the program.
func (this *Interpreter) Value() object.Value {
	return this.value
}

// acceptValue receives the final value computed by the program.  It
// is normally called from the step method of final state to be
// evaluated before the program terminates.
func (this *Interpreter) acceptValue(v object.Value) {
	this.value = v
}

/********************************************************************/

// state is the interface implemented by each of the types
// representing different possible next states for the interpreter
// (roughly: one state per ast.Node implementation); each value of
// this type represents a possible state of the computation.
type state interface {
	// step performs the next step in the evaluation of the program, and
	// returns the new state execution state.
	step() state

	// acceptValue receives the value resulting from the evaluation of
	//a child expression.
	/// It is normally called by the
	// subexpression's step method, typically as follows:
	//
	//        // ... compute value to be returned ...
	//        this.parent.acceptValue(value)
	//        return this.parent
	//    }
	acceptValue(object.Value)
}

// newState creates a state object corresponding to the given AST
// node.  The parent parameter represents the state the interpreter
// should return to after evaluating the tree rooted at node.
func newState(parent state, node ast.Node) state {
	switch n := node.(type) {
	case ast.Program:
		s := stateBlockStatement{}
		s.parent = parent
		s.initFromProgram(n)
		return &s
	case ast.BlockStatement:
		s := stateBlockStatement{}
		s.parent = parent
		s.init(n)
		return &s
	case *ast.ExpressionStatement:
		s := stateExpressionStatement{}
		s.parent = parent
		s.init(*n)
		return &s
	case *ast.BinaryExpression:
		s := stateBinaryExpression{}
		s.parent = parent
		s.init(*n)
		return &s
	case *ast.Literal:
		s := stateLiteral{}
		s.parent = parent
		s.init(*n)
		return &s
	default:
		panic(fmt.Errorf("State for AST node type %T not implemented\n", n))
	}
}

/********************************************************************/

// stateCommon is a struct, intended to be embedded in most or all
// state<NodeType> types, which provides fields common to most/all
// states.
type stateCommon struct {
	// state is the state to return to once evaluation of this state is
	// finished.
	parent state
}

/********************************************************************/

type stateBlockStatement struct {
	stateCommon
	body        ast.Statements
	value       object.Value
	n           int
	interpreter *Interpreter // Used by Program nodes only
}

func (this *stateBlockStatement) initFromProgram(node ast.Program) {
	this.body = node.Body
}

func (this *stateBlockStatement) init(node ast.BlockStatement) {
	this.body = node.Body
}

func (this *stateBlockStatement) step() state {
	if this.n < len(this.body) {
		s := newState(this, (this.body)[this.n])
		this.n++
		return s
	}
	if this.interpreter != nil {
		this.interpreter.acceptValue(this.value)
	}
	return this.parent
}

func (this *stateBlockStatement) acceptValue(v object.Value) {
	this.value = v
}

/********************************************************************/

type stateExpressionStatement struct {
	stateCommon
	expr ast.Expression
}

func (this *stateExpressionStatement) init(node ast.ExpressionStatement) {
	this.expr = node.Expression
}

func (this *stateExpressionStatement) step() state {
	return newState(this.parent, ast.Node(this.expr.E))
}

func (this *stateExpressionStatement) acceptValue(v object.Value) {
	panic("should not be reached")
}

/********************************************************************/

type stateBinaryExpression struct {
	stateCommon
	op                  string
	lNode, rNode        ast.Expression
	haveLeft, haveRight bool
	left, right         object.Value
}

func (this *stateBinaryExpression) init(node ast.BinaryExpression) {
	this.op = node.Operator
	this.lNode = node.Left
	this.rNode = node.Right
	this.haveLeft = false
	this.haveRight = false
}

func (this *stateBinaryExpression) step() state {
	if !this.haveLeft {
		return newState(this, ast.Node(this.lNode.E))
	} else if !this.haveRight {
		return newState(this, ast.Node(this.rNode.E))
	}

	// FIXME: implement other operators, types

	var v object.Value
	switch this.op {
	case "+":
		v = object.Number(this.left.(object.Number) +
			this.right.(object.Number))
	case "-":
		v = object.Number(this.left.(object.Number) -
			this.right.(object.Number))
	case "*":
		v = object.Number(this.left.(object.Number) *
			this.right.(object.Number))
	case "/":
		v = object.Number(this.left.(object.Number) /
			this.right.(object.Number))
	}

	this.parent.acceptValue(v)
	return this.parent

}

func (this *stateBinaryExpression) acceptValue(v object.Value) {
	if !this.haveLeft {
		this.left = v
		this.haveLeft = true
	} else if !this.haveRight {
		this.right = v
		this.haveRight = true
	} else {
		panic(fmt.Errorf("too may values"))
	}
}

/********************************************************************/

type stateEmptyStatement struct {
	stateCommon
}

func (this *stateEmptyStatement) init(node ast.Literal) {
}

func (this *stateEmptyStatement) step() state {
	return this.parent
}

func (this *stateEmptyStatement) acceptValue(v object.Value) {
	panic(fmt.Errorf("EmptyStatement can't have subexpression"))
}

/********************************************************************/

type stateFunctionDeclaration struct {
	stateCommon
}

func (this *stateFunctionDeclaration) init(node ast.Literal) {
}

func (this *stateFunctionDeclaration) step() state {
	return this.parent
}

func (this *stateFunctionDeclaration) acceptValue(v object.Value) {
	panic(fmt.Errorf("FunctionDeclaration can't have subexpression"))
}

/********************************************************************/

type stateLiteral struct {
	stateCommon
	value object.Value
}

func (this *stateLiteral) init(node ast.Literal) {
	this.value = object.PrimitiveFromRaw(node.Raw)
}

func (this *stateLiteral) step() state {
	this.parent.acceptValue(this.value)
	return this.parent
}

func (this *stateLiteral) acceptValue(v object.Value) {
	panic(fmt.Errorf("Literal can't have subexpression"))
}
<commit_msg>Alphabetisation<commit_after>package interpreter

import (
	"fmt"

	"CodeCity/server/interpreter/ast"
	"CodeCity/server/interpreter/object"
)

// Interpreter implements a JavaScript interpreter.
type Interpreter struct {
	state state
	value object.Value
}

// NewInterpreter takes a JavaScript program, in the form of an
// JSON-encoded ESTree, and creates a new Interpreter that will
// execute that program.
func NewInterpreter(astJSON string) *Interpreter {
	var this = new(Interpreter)

	tree, err := ast.NewFromJSON(astJSON)
	if err != nil {
		panic(err)
	}
	this.state = newState(nil, *tree)
	this.state.(*stateBlockStatement).interpreter = this
	return this
}

// Step performs the next step in the evaluation of program.  Returns
// true if a step was executed; false if the program has terminated.
func (this *Interpreter) Step() bool {
	if this.state == nil {
		return false
	}
	fmt.Printf("Next step is a %T\n", this.state)
	this.state = this.state.step()
	return true
}

// Run runs the program to completion.
func (this *Interpreter) Run() {
	for this.Step() {
	}
}

// Value returns the final value computed by the last statement
// expression of the program.
func (this *Interpreter) Value() object.Value {
	return this.value
}

// acceptValue receives the final value computed by the program.  It
// is normally called from the step method of final state to be
// evaluated before the program terminates.
func (this *Interpreter) acceptValue(v object.Value) {
	this.value = v
}

/********************************************************************/

// state is the interface implemented by each of the types
// representing different possible next states for the interpreter
// (roughly: one state per ast.Node implementation); each value of
// this type represents a possible state of the computation.
type state interface {
	// step performs the next step in the evaluation of the program, and
	// returns the new state execution state.
	step() state

	// acceptValue receives the value resulting from the evaluation of
	//a child expression.
	/// It is normally called by the
	// subexpression's step method, typically as follows:
	//
	//        // ... compute value to be returned ...
	//        this.parent.acceptValue(value)
	//        return this.parent
	//    }
	acceptValue(object.Value)
}

// newState creates a state object corresponding to the given AST
// node.  The parent parameter represents the state the interpreter
// should return to after evaluating the tree rooted at node.
func newState(parent state, node ast.Node) state {
	switch n := node.(type) {
	case ast.Program:
		s := stateBlockStatement{}
		s.parent = parent
		s.initFromProgram(n)
		return &s
	case ast.BlockStatement:
		s := stateBlockStatement{}
		s.parent = parent
		s.init(n)
		return &s
	case *ast.ExpressionStatement:
		s := stateExpressionStatement{}
		s.parent = parent
		s.init(*n)
		return &s
	case *ast.BinaryExpression:
		s := stateBinaryExpression{}
		s.parent = parent
		s.init(*n)
		return &s
	case *ast.Literal:
		s := stateLiteral{}
		s.parent = parent
		s.init(*n)
		return &s
	default:
		panic(fmt.Errorf("State for AST node type %T not implemented\n", n))
	}
}

/********************************************************************/

// stateCommon is a struct, intended to be embedded in most or all
// state<NodeType> types, which provides fields common to most/all
// states.
type stateCommon struct {
	// state is the state to return to once evaluation of this state is
	// finished.
	parent state
}

/********************************************************************/

type stateBinaryExpression struct {
	stateCommon
	op                  string
	lNode, rNode        ast.Expression
	haveLeft, haveRight bool
	left, right         object.Value
}

func (this *stateBinaryExpression) init(node ast.BinaryExpression) {
	this.op = node.Operator
	this.lNode = node.Left
	this.rNode = node.Right
	this.haveLeft = false
	this.haveRight = false
}

func (this *stateBinaryExpression) step() state {
	if !this.haveLeft {
		return newState(this, ast.Node(this.lNode.E))
	} else if !this.haveRight {
		return newState(this, ast.Node(this.rNode.E))
	}

	// FIXME: implement other operators, types

	var v object.Value
	switch this.op {
	case "+":
		v = object.Number(this.left.(object.Number) +
			this.right.(object.Number))
	case "-":
		v = object.Number(this.left.(object.Number) -
			this.right.(object.Number))
	case "*":
		v = object.Number(this.left.(object.Number) *
			this.right.(object.Number))
	case "/":
		v = object.Number(this.left.(object.Number) /
			this.right.(object.Number))
	}

	this.parent.acceptValue(v)
	return this.parent

}

func (this *stateBinaryExpression) acceptValue(v object.Value) {
	if !this.haveLeft {
		this.left = v
		this.haveLeft = true
	} else if !this.haveRight {
		this.right = v
		this.haveRight = true
	} else {
		panic(fmt.Errorf("too may values"))
	}
}

/********************************************************************/

type stateBlockStatement struct {
	stateCommon
	body        ast.Statements
	value       object.Value
	n           int
	interpreter *Interpreter // Used by Program nodes only
}

func (this *stateBlockStatement) initFromProgram(node ast.Program) {
	this.body = node.Body
}

func (this *stateBlockStatement) init(node ast.BlockStatement) {
	this.body = node.Body
}

func (this *stateBlockStatement) step() state {
	if this.n < len(this.body) {
		s := newState(this, (this.body)[this.n])
		this.n++
		return s
	}
	if this.interpreter != nil {
		this.interpreter.acceptValue(this.value)
	}
	return this.parent
}

func (this *stateBlockStatement) acceptValue(v object.Value) {
	this.value = v
}

/********************************************************************/

type stateEmptyStatement struct {
	stateCommon
}

func (this *stateEmptyStatement) init(node ast.Literal) {
}

func (this *stateEmptyStatement) step() state {
	return this.parent
}

func (this *stateEmptyStatement) acceptValue(v object.Value) {
	panic(fmt.Errorf("EmptyStatement can't have subexpression"))
}

/********************************************************************/

type stateExpressionStatement struct {
	stateCommon
	expr ast.Expression
}

func (this *stateExpressionStatement) init(node ast.ExpressionStatement) {
	this.expr = node.Expression
}

func (this *stateExpressionStatement) step() state {
	return newState(this.parent, ast.Node(this.expr.E))
}

func (this *stateExpressionStatement) acceptValue(v object.Value) {
	panic("should not be reached")
}

/********************************************************************/

type stateFunctionDeclaration struct {
	stateCommon
}

func (this *stateFunctionDeclaration) init(node ast.Literal) {
}

func (this *stateFunctionDeclaration) step() state {
	return this.parent
}

func (this *stateFunctionDeclaration) acceptValue(v object.Value) {
	panic(fmt.Errorf("FunctionDeclaration can't have subexpression"))
}

/********************************************************************/

type stateLiteral struct {
	stateCommon
	value object.Value
}

func (this *stateLiteral) init(node ast.Literal) {
	this.value = object.PrimitiveFromRaw(node.Raw)
}

func (this *stateLiteral) step() state {
	this.parent.acceptValue(this.value)
	return this.parent
}

func (this *stateLiteral) acceptValue(v object.Value) {
	panic(fmt.Errorf("Literal can't have subexpression"))
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"os"

	"github.com/tarm/serial"
)

type Config struct {
	Help         bool
	SerialDevice string
	SerialSpeed  int
}

var config Config

func init() {
	flag.BoolVar(&config.Help, "h", false, "display help")
	flag.StringVar(&config.SerialDevice, "d", "", "serial device name")
	flag.IntVar(&config.SerialSpeed, "b", 9600, "serial baudrate")

	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: serialmon [options]")
		flag.PrintDefaults()
	}
}

func exitOnError(err error) {
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func openComms(config Config) io.ReadWriteCloser {

	if len(config.SerialDevice) > 0 && config.SerialSpeed != 0 {
		serialcfg := serial.Config{Name: config.SerialDevice, Baud: config.SerialSpeed}
		port, err := serial.OpenPort(&serialcfg)
		exitOnError(err)
		return port
	}

	fmt.Fprintln(os.Stderr, "comms port incorrectly specified")
	flag.Usage()
	os.Exit(1)

	return nil
}

func main() {
	flag.Parse()

	if len(flag.Args()) != 0 || config.Help {
		flag.Usage()
		os.Exit(1)
	}

	comms := openComms(config)
	defer comms.Close()
}
<commit_msg>added copy funcs<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"os"

	"github.com/tarm/serial"
)

type Config struct {
	Help         bool
	SerialDevice string
	SerialSpeed  int
}

var config Config

func init() {
	flag.BoolVar(&config.Help, "h", false, "display help")
	flag.StringVar(&config.SerialDevice, "d", "", "serial device name")
	flag.IntVar(&config.SerialSpeed, "b", 9600, "serial baudrate")

	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: serialmon [options]")
		flag.PrintDefaults()
	}
}

func exitOnError(err error) {
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func openComms(config Config) io.ReadWriteCloser {

	if len(config.SerialDevice) > 0 && config.SerialSpeed != 0 {
		serialcfg := serial.Config{Name: config.SerialDevice, Baud: config.SerialSpeed}
		port, err := serial.OpenPort(&serialcfg)
		exitOnError(err)
		return port
	}

	fmt.Fprintln(os.Stderr, "comms port incorrectly specified")
	flag.Usage()
	os.Exit(1)

	return nil
}

func main() {
	flag.Parse()

	if len(flag.Args()) != 0 || config.Help {
		flag.Usage()
		os.Exit(1)
	}

	comms := openComms(config)
	defer comms.Close()

	go func() {
		io.Copy(os.Stdout, comms)
	}()

	io.Copy(comms, os.Stdin)
}
<|endoftext|>


<commit_before>// Package implements access to Upwork API
//
// Licensed under the Upwork's API Terms of Use;
// you may not use this file except in compliance with the Terms.
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author::    Maksym Novozhylov (mnovozhilov@upwork.com)
// Copyright:: Copyright 2018(c) Upwork.com
// License::   See LICENSE.txt and TOS - https://developers.upwork.com/api-tos.html
package api

import (
    "log"
    "strings"
    "fmt"
    "bytes"
    "context"
    "net/http"
    "net/url"
    "io/ioutil"

    "golang.org/x/oauth2"
)

// Define end points
const (
    // oauth2 and api flow
    BaseHost = "https://www.upwork.com/"
    DefaultEpoint = "api"
    AuthorizationEP = BaseHost + "ab/account-security/oauth2/authorize"
    AccessTokenEP = BaseHost + DefaultEpoint + "/v3/oauth2/token"
    DataFormat = "json"
    OverloadParam = "http_method"
    UPWORK_LIBRARY_USER_AGENT = "Github Upwork API Golang Library"

    // response types
    ByteResponse = "[]byte"
    ErrorResponse = "error"
)

// Api client
type ApiClient struct {
    // oauth2
    oconf *oauth2.Config
    token *oauth2.Token
    oclient *http.Client
    config *Config

    // refresh token notify function
    rnfunc TokenNotifyFunc

    // client
    ep string
    respType string
    sendPostAsJson bool
    hasCustomHttpClient bool
}

// TokenNotifyFunc is a function that accepts an oauth2 Token upon refresh, and
// returns an error if it should not be used.
type TokenNotifyFunc func(*oauth2.Token) error

// NotifyingTokenSource is an oauth2.TokenSource that calls a function when a
// new token is obtained.
type NotifyingTokenSource struct {
    f   TokenNotifyFunc
    src oauth2.TokenSource
}

// Setup client using specific config
func Setup(config *Config) (client ApiClient) {
    var c ApiClient

    c.config = config

    c.oconf = &oauth2.Config{
        ClientID:     config.ClientId,
        ClientSecret: config.ClientSecret,
	RedirectURL: config.RedirectUri,
	Endpoint: oauth2.Endpoint{
	    TokenURL: AccessTokenEP,
	    AuthURL:  AuthorizationEP,
        },
    }

    c.hasCustomHttpClient = config.HasCustomHttpClient

    // Force setup of client_id as a parameter
    oauth2.RegisterBrokenAuthHeaderProvider(BaseHost)

    c.token = new(oauth2.Token)
    c.token.TokenType = "Bearer"
    c.token.AccessToken = config.AccessToken
    c.token.RefreshToken = config.RefreshToken
    c.token.Expiry = config.ExpiresAt

    c.SetApiResponseType(ByteResponse)
    c.SetPostAsJson(false) // send by default using PostForm

    return c
}

// NewNotifyingTokenSource creates a NotifyingTokenSource from an underlying src
// and calls f when a new token is obtained.
func NewNotifyingTokenSource(src oauth2.TokenSource, f TokenNotifyFunc) *NotifyingTokenSource {
    return &NotifyingTokenSource{f: f, src: src}
}

// Token fetches a new token from the underlying source.
func (s *NotifyingTokenSource) Token() (*oauth2.Token, error) {
    t, err := s.src.Token()
    if err != nil {
        return nil, err
    }
    if s.f == nil {
        return t, nil
    }
    return t, s.f(t)
}

// Set refresh token notify function
func (c *ApiClient) SetRefreshTokenNotifyFunc(f TokenNotifyFunc) () {
    c.rnfunc = f
}

// Set request type for non-Get requests
func (c *ApiClient) SetPostAsJson(t bool) {
    c.sendPostAsJson = t;
}

// Set API response type
func (c *ApiClient) SetApiResponseType(t string) {
    c.respType = t;
}

// Set entry point, e.g requested from a router
func (c *ApiClient) SetEntryPoint(ep string) {
    c.ep = ep
}

// Receive an authorization URL
func (c *ApiClient) GetAuthorizationUrl(stateString string) (authzUrl string) {
    url := c.oconf.AuthCodeURL(stateString)
    if url == "" {
        log.Fatal("Can not get authorization URL using OAuth2 library")
    }

    return url
}

// Get access token using a specific authorization code
func (c *ApiClient) GetToken(ctx context.Context, authzCode string) (*oauth2.Token) {
    ctx = c.config.SetOwnHttpClient(ctx)

    accessToken, err := c.oconf.Exchange(ctx, strings.Trim(authzCode, "\n"))
    if err != nil {
        log.Fatal(err)
    }

    c.token = accessToken
    c.setupOauth2Client(ctx)

    return accessToken
}

// Check if client contains already a access/refresh token pair
func (c *ApiClient) HasAccessToken(ctx context.Context) (bool) {
    has := (c.token != nil && (c.token.AccessToken != "" && c.token.RefreshToken != ""))
    if has {
        c.setupOauth2Client(ctx)
    }
    return has
}

// GET method for client
func (c *ApiClient) Get(uri string, params map[string]string) (r *http.Response, re interface{}) {
    // parameters must be encoded according to RFC 3986
    // hmmm, it seems to be the easiest trick?
    qstr := ""
    if params != nil {
        for k, v := range params {
            qstr += fmt.Sprintf("%s=%s&", k, v)
        }
        qstr = qstr[0:len(qstr)-1]
    }
    u := &url.URL{Path: qstr}

    // https://github.com/mrjones/oauth/issues/34
    encQuery := strings.Replace(u.String(), ";", "%3B", -1)
    encQuery = strings.Replace(encQuery, "./", "?", 1) // see URL.String method to understand when "./" is returned

    // non-empty string may miss "?"
    if encQuery[:1] != "?" {
        encQuery = "?" + encQuery
    }

    response, err := c.oclient.Get(formatUri(uri, c.ep) + encQuery)

    return c.getTypedResponse(response, err)
}

// POST method for client
func (c *ApiClient) Post(uri string, params map[string]string) (r *http.Response, re interface{}) {
    return c.sendPostRequest(uri, params)
}

// PUT method for client
func (c *ApiClient) Put(uri string, params map[string]string) (r *http.Response, re interface{}) {
    return c.sendPostRequest(uri, addOverloadParam(params, "put"))
}

// DELETE method for client
func (c *ApiClient) Delete(uri string, params map[string]string) (r *http.Response, re interface{}) {
    return c.sendPostRequest(uri, addOverloadParam(params, "delete"))
}

// setup/save authorized oauth2 client, based on received or provided access/refresh token pair
func (c *ApiClient) setupOauth2Client(ctx context.Context) {
    if (c.hasCustomHttpClient == false) {
	ctx = c.config.SetOwnHttpClient(ctx)
    }

    if c.rnfunc != nil {
        // setup notifier for token-refresh workflow - https://github.com/golang/oauth2/issues/84
        realSource := c.oconf.TokenSource(ctx, c.token)
        notifyingSrc := NewNotifyingTokenSource(realSource, c.rnfunc)
        notifyingWithInitialSrc := oauth2.ReuseTokenSource(c.token, notifyingSrc)
        // setup authorized oauth2 client
        c.oclient = oauth2.NewClient(ctx, notifyingWithInitialSrc)
    } else {
        c.oclient = c.oconf.Client(ctx, c.token)
    }
}

// run post/put/delete requests
func (c *ApiClient) sendPostRequest(uri string, params map[string]string) (*http.Response, interface{}) {
    if c.sendPostAsJson == true {
	// old style for backward compatibility with the old library
        var jsonStr = []byte("{}")
        if params != nil {
            str := ""
            for k, v := range params {
                str += fmt.Sprintf("\"%s\": \"%s\",", k, v)
            }
            jsonStr = []byte(fmt.Sprintf("{%s}", str[0:len(str)-1]))
        }

	response, err := c.oclient.Post(formatUri(uri, c.ep), "application/json", bytes.NewBuffer(jsonStr))

        return c.getTypedResponse(response, err)
    } else {
	// prefered
        urlValues := url.Values{}
        if params != nil {
	    for k, v := range params {
                urlValues.Add(k, v)
            }
        }

	response, err := c.oclient.PostForm(formatUri(uri, c.ep), urlValues)

        return c.getTypedResponse(response, err)
    }
}

// return proper response type
func (c *ApiClient) getTypedResponse(resp *http.Response, re error) (*http.Response, interface{}) {
    if c.respType == ByteResponse {
	r, b := formatResponse(resp, re)
	return r, b.([]byte)
    } else {
        return resp, re
    }
}

// Check and format (preparate a byte body) http response routine
func formatResponse(resp *http.Response, err error) (*http.Response, interface{}) {
    if err != nil {
        log.Fatal("Can not execute the request, " + err.Error())
    }

    defer resp.Body.Close()
    if resp.StatusCode != 200 {
        // do not exit, it can be a normal response
        // it's up to client/requester's side decide what to do
    }
    // read json http response
    jsonDataFromHttp, _ := ioutil.ReadAll(resp.Body)

    return resp, jsonDataFromHttp
}

// Create a path to a specific resource
func formatUri(uri string, ep string) (string) {
    format := ""
    if ep == DefaultEpoint {
        format += "." + DataFormat
    }
    return BaseHost + ep + uri + format
}

// add overload parameter to the map of parameters
func addOverloadParam(params map[string]string, op string) map[string]string {
    if params == nil {
        params = make(map[string]string)
    }
    params[OverloadParam] = op
    return params
}
<commit_msg>fix 'slice bounds out of range' in newer versions of golang<commit_after>// Package implements access to Upwork API
//
// Licensed under the Upwork's API Terms of Use;
// you may not use this file except in compliance with the Terms.
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author::    Maksym Novozhylov (mnovozhilov@upwork.com)
// Copyright:: Copyright 2018(c) Upwork.com
// License::   See LICENSE.txt and TOS - https://developers.upwork.com/api-tos.html
package api

import (
    "log"
    "strings"
    "fmt"
    "bytes"
    "context"
    "net/http"
    "net/url"
    "io/ioutil"

    "golang.org/x/oauth2"
)

// Define end points
const (
    // oauth2 and api flow
    BaseHost = "https://www.upwork.com/"
    DefaultEpoint = "api"
    AuthorizationEP = BaseHost + "ab/account-security/oauth2/authorize"
    AccessTokenEP = BaseHost + DefaultEpoint + "/v3/oauth2/token"
    DataFormat = "json"
    OverloadParam = "http_method"
    UPWORK_LIBRARY_USER_AGENT = "Github Upwork API Golang Library"

    // response types
    ByteResponse = "[]byte"
    ErrorResponse = "error"
)

// Api client
type ApiClient struct {
    // oauth2
    oconf *oauth2.Config
    token *oauth2.Token
    oclient *http.Client
    config *Config

    // refresh token notify function
    rnfunc TokenNotifyFunc

    // client
    ep string
    respType string
    sendPostAsJson bool
    hasCustomHttpClient bool
}

// TokenNotifyFunc is a function that accepts an oauth2 Token upon refresh, and
// returns an error if it should not be used.
type TokenNotifyFunc func(*oauth2.Token) error

// NotifyingTokenSource is an oauth2.TokenSource that calls a function when a
// new token is obtained.
type NotifyingTokenSource struct {
    f   TokenNotifyFunc
    src oauth2.TokenSource
}

// Setup client using specific config
func Setup(config *Config) (client ApiClient) {
    var c ApiClient

    c.config = config

    c.oconf = &oauth2.Config{
        ClientID:     config.ClientId,
        ClientSecret: config.ClientSecret,
	RedirectURL: config.RedirectUri,
	Endpoint: oauth2.Endpoint{
	    TokenURL: AccessTokenEP,
	    AuthURL:  AuthorizationEP,
        },
    }

    c.hasCustomHttpClient = config.HasCustomHttpClient

    // Force setup of client_id as a parameter
    oauth2.RegisterBrokenAuthHeaderProvider(BaseHost)

    c.token = new(oauth2.Token)
    c.token.TokenType = "Bearer"
    c.token.AccessToken = config.AccessToken
    c.token.RefreshToken = config.RefreshToken
    c.token.Expiry = config.ExpiresAt

    c.SetApiResponseType(ByteResponse)
    c.SetPostAsJson(false) // send by default using PostForm

    return c
}

// NewNotifyingTokenSource creates a NotifyingTokenSource from an underlying src
// and calls f when a new token is obtained.
func NewNotifyingTokenSource(src oauth2.TokenSource, f TokenNotifyFunc) *NotifyingTokenSource {
    return &NotifyingTokenSource{f: f, src: src}
}

// Token fetches a new token from the underlying source.
func (s *NotifyingTokenSource) Token() (*oauth2.Token, error) {
    t, err := s.src.Token()
    if err != nil {
        return nil, err
    }
    if s.f == nil {
        return t, nil
    }
    return t, s.f(t)
}

// Set refresh token notify function
func (c *ApiClient) SetRefreshTokenNotifyFunc(f TokenNotifyFunc) () {
    c.rnfunc = f
}

// Set request type for non-Get requests
func (c *ApiClient) SetPostAsJson(t bool) {
    c.sendPostAsJson = t;
}

// Set API response type
func (c *ApiClient) SetApiResponseType(t string) {
    c.respType = t;
}

// Set entry point, e.g requested from a router
func (c *ApiClient) SetEntryPoint(ep string) {
    c.ep = ep
}

// Receive an authorization URL
func (c *ApiClient) GetAuthorizationUrl(stateString string) (authzUrl string) {
    url := c.oconf.AuthCodeURL(stateString)
    if url == "" {
        log.Fatal("Can not get authorization URL using OAuth2 library")
    }

    return url
}

// Get access token using a specific authorization code
func (c *ApiClient) GetToken(ctx context.Context, authzCode string) (*oauth2.Token) {
    ctx = c.config.SetOwnHttpClient(ctx)

    accessToken, err := c.oconf.Exchange(ctx, strings.Trim(authzCode, "\n"))
    if err != nil {
        log.Fatal(err)
    }

    c.token = accessToken
    c.setupOauth2Client(ctx)

    return accessToken
}

// Check if client contains already a access/refresh token pair
func (c *ApiClient) HasAccessToken(ctx context.Context) (bool) {
    has := (c.token != nil && (c.token.AccessToken != "" && c.token.RefreshToken != ""))
    if has {
        c.setupOauth2Client(ctx)
    }
    return has
}

// GET method for client
func (c *ApiClient) Get(uri string, params map[string]string) (r *http.Response, re interface{}) {
    // parameters must be encoded according to RFC 3986
    // hmmm, it seems to be the easiest trick?
    qstr := ""
    if params != nil {
        for k, v := range params {
            qstr += fmt.Sprintf("%s=%s&", k, v)
        }
        qstr = qstr[0:len(qstr)-1]
    }
    u := &url.URL{Path: qstr}

    // https://github.com/mrjones/oauth/issues/34
    encQuery := strings.Replace(u.String(), ";", "%3B", -1)
    encQuery = strings.Replace(encQuery, "./", "?", 1) // see URL.String method to understand when "./" is returned

    // non-empty string may miss "?"
    if encQuery !="" && encQuery[:1] != "?" {
        encQuery = "?" + encQuery
    }

    response, err := c.oclient.Get(formatUri(uri, c.ep) + encQuery)

    return c.getTypedResponse(response, err)
}

// POST method for client
func (c *ApiClient) Post(uri string, params map[string]string) (r *http.Response, re interface{}) {
    return c.sendPostRequest(uri, params)
}

// PUT method for client
func (c *ApiClient) Put(uri string, params map[string]string) (r *http.Response, re interface{}) {
    return c.sendPostRequest(uri, addOverloadParam(params, "put"))
}

// DELETE method for client
func (c *ApiClient) Delete(uri string, params map[string]string) (r *http.Response, re interface{}) {
    return c.sendPostRequest(uri, addOverloadParam(params, "delete"))
}

// setup/save authorized oauth2 client, based on received or provided access/refresh token pair
func (c *ApiClient) setupOauth2Client(ctx context.Context) {
    if (c.hasCustomHttpClient == false) {
	ctx = c.config.SetOwnHttpClient(ctx)
    }

    if c.rnfunc != nil {
        // setup notifier for token-refresh workflow - https://github.com/golang/oauth2/issues/84
        realSource := c.oconf.TokenSource(ctx, c.token)
        notifyingSrc := NewNotifyingTokenSource(realSource, c.rnfunc)
        notifyingWithInitialSrc := oauth2.ReuseTokenSource(c.token, notifyingSrc)
        // setup authorized oauth2 client
        c.oclient = oauth2.NewClient(ctx, notifyingWithInitialSrc)
    } else {
        c.oclient = c.oconf.Client(ctx, c.token)
    }
}

// run post/put/delete requests
func (c *ApiClient) sendPostRequest(uri string, params map[string]string) (*http.Response, interface{}) {
    if c.sendPostAsJson == true {
	// old style for backward compatibility with the old library
        var jsonStr = []byte("{}")
        if params != nil {
            str := ""
            for k, v := range params {
                str += fmt.Sprintf("\"%s\": \"%s\",", k, v)
            }
            jsonStr = []byte(fmt.Sprintf("{%s}", str[0:len(str)-1]))
        }

	response, err := c.oclient.Post(formatUri(uri, c.ep), "application/json", bytes.NewBuffer(jsonStr))

        return c.getTypedResponse(response, err)
    } else {
	// prefered
        urlValues := url.Values{}
        if params != nil {
	    for k, v := range params {
                urlValues.Add(k, v)
            }
        }

	response, err := c.oclient.PostForm(formatUri(uri, c.ep), urlValues)

        return c.getTypedResponse(response, err)
    }
}

// return proper response type
func (c *ApiClient) getTypedResponse(resp *http.Response, re error) (*http.Response, interface{}) {
    if c.respType == ByteResponse {
	r, b := formatResponse(resp, re)
	return r, b.([]byte)
    } else {
        return resp, re
    }
}

// Check and format (preparate a byte body) http response routine
func formatResponse(resp *http.Response, err error) (*http.Response, interface{}) {
    if err != nil {
        log.Fatal("Can not execute the request, " + err.Error())
    }

    defer resp.Body.Close()
    if resp.StatusCode != 200 {
        // do not exit, it can be a normal response
        // it's up to client/requester's side decide what to do
    }
    // read json http response
    jsonDataFromHttp, _ := ioutil.ReadAll(resp.Body)

    return resp, jsonDataFromHttp
}

// Create a path to a specific resource
func formatUri(uri string, ep string) (string) {
    format := ""
    if ep == DefaultEpoint {
        format += "." + DataFormat
    }
    return BaseHost + ep + uri + format
}

// add overload parameter to the map of parameters
func addOverloadParam(params map[string]string, op string) map[string]string {
    if params == nil {
        params = make(map[string]string)
    }
    params[OverloadParam] = op
    return params
}
<|endoftext|>


<commit_before>// Copyright 2018 ReactiveOps
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rbacdefinition

import (
	"reflect"

	logrus "github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func (bc *RBACDefinitionController) reconcileClusterRoleBindings(
	requestedClusterRoleBindings *[]rbacv1.ClusterRoleBinding,
	existingManagedClusterRoleBindings *[]rbacv1.ClusterRoleBinding,
	ownerReferences *[]metav1.OwnerReference) {

	matchingClusterRoleBindings := []rbacv1.ClusterRoleBinding{}

	for _, requestedCRB := range *requestedClusterRoleBindings {
		alreadyExists := false
		for _, existingCRB := range *existingManagedClusterRoleBindings {
			if crbMatches(&existingCRB, &requestedCRB) {
				alreadyExists = true
				matchingClusterRoleBindings = append(matchingClusterRoleBindings, existingCRB)
				break
			}
		}

		if !alreadyExists {
			logrus.Infof("Creating Cluster Role Binding: %v", requestedCRB.Name)
			_, err := bc.kubernetesClientSet.RbacV1().ClusterRoleBindings().Create(&requestedCRB)
			if err != nil {
				logrus.Errorf("Error creating Cluster Role Binding: %v", err)
			}
		} else {
			logrus.Debugf("Cluster Role Binding already exists %v", requestedCRB.Name)
		}
	}

	for _, existingCRB := range *existingManagedClusterRoleBindings {
		if reflect.DeepEqual(existingCRB.OwnerReferences, *ownerReferences) {
			matchingRequest := false
			for _, requestedCRB := range matchingClusterRoleBindings {
				if crbMatches(&existingCRB, &requestedCRB) {
					matchingRequest = true
					break
				}
			}

			if !matchingRequest {
				logrus.Infof("Deleting Cluster Role Binding: %v", existingCRB.Name)
				err := bc.kubernetesClientSet.RbacV1().ClusterRoleBindings().Delete(existingCRB.Name, &metav1.DeleteOptions{})
				if err != nil {
					logrus.Errorf("Error deleting Cluster Role Binding: %v", err)
				}
			} else {
				logrus.Debugf("Matches requested Cluster Role Binding: %v", existingCRB.Name)
			}
		}
	}
}

func (bc *RBACDefinitionController) reconcileRoleBindings(
	requestedRoleBindings *[]rbacv1.RoleBinding,
	existingManagedRoleBindings *[]rbacv1.RoleBinding,
	ownerReferences *[]metav1.OwnerReference) {

	matchingRoleBindings := []rbacv1.RoleBinding{}

	for _, requestedRB := range *requestedRoleBindings {
		alreadyExists := false
		for _, existingRB := range *existingManagedRoleBindings {
			if rbMatches(&existingRB, &requestedRB) {
				alreadyExists = true
				matchingRoleBindings = append(matchingRoleBindings, existingRB)
				break
			}
		}

		if !alreadyExists {
			logrus.Infof("Creating Role Binding: %v", requestedRB.Name)
			_, err := bc.kubernetesClientSet.RbacV1().RoleBindings(requestedRB.ObjectMeta.Namespace).Create(&requestedRB)
			if err != nil {
				logrus.Errorf("Error creating Role Binding: %v", err)
			}
		} else {
			logrus.Debugf("Role Binding already exists %v", requestedRB.Name)
		}
	}

	for _, existingRB := range *existingManagedRoleBindings {
		if reflect.DeepEqual(existingRB.OwnerReferences, *ownerReferences) {
			matchingRequest := false
			for _, requestedRB := range matchingRoleBindings {
				if rbMatches(&existingRB, &requestedRB) {
					matchingRequest = true
					break
				}
			}

			if !matchingRequest {
				logrus.Infof("Deleting Role Binding %v", existingRB.Name)
				err := bc.kubernetesClientSet.RbacV1().RoleBindings(existingRB.Namespace).Delete(existingRB.Name, &metav1.DeleteOptions{})
				if err != nil {
					logrus.Infof("Error deleting Role Binding: %v", err)
				}
			} else {
				logrus.Debugf("Matches requested Role Binding %v", existingRB.Name)
			}
		}
	}
}

func (bc *RBACDefinitionController) reconcileServiceAccounts(
	requestedServiceAccounts *[]v1.ServiceAccount,
	existingManagedServiceAccounts *[]v1.ServiceAccount,
	ownerReferences *[]metav1.OwnerReference) {

	matchingServiceAccounts := []v1.ServiceAccount{}

	for _, requestedSA := range *requestedServiceAccounts {
		alreadyExists := false
		for _, existingSA := range *existingManagedServiceAccounts {
			if saMatches(&existingSA, &requestedSA) {
				alreadyExists = true
				matchingServiceAccounts = append(matchingServiceAccounts, existingSA)
				break
			}
		}

		if !alreadyExists {
			logrus.Infof("Creating Service Account: %v", requestedSA.Name)
			_, err := bc.kubernetesClientSet.CoreV1().ServiceAccounts(requestedSA.ObjectMeta.Namespace).Create(&requestedSA)
			if err != nil {
				logrus.Errorf("Error creating Service Account: %v", err)
			}
		} else {
			logrus.Debugf("Service Account already exists %v", requestedSA.Name)
		}
	}

	for _, existingSA := range *existingManagedServiceAccounts {
		if reflect.DeepEqual(existingSA.OwnerReferences, *ownerReferences) {
			matchingRequest := false
			for _, matchingSA := range matchingServiceAccounts {
				if saMatches(&existingSA, &matchingSA) {
					matchingRequest = true
					break
				}
			}

			if !matchingRequest {
				logrus.Infof("Deleting Service Account %v", existingSA.Name)
				err := bc.kubernetesClientSet.CoreV1().ServiceAccounts(existingSA.Namespace).Delete(existingSA.Name, &metav1.DeleteOptions{})
				if err != nil {
					logrus.Infof("Error deleting Service Account: %v", err)
				}
			} else {
				logrus.Debugf("Matches requested Service Account %v", existingSA.Name)
			}
		}
	}
}
<commit_msg>some fixes for #20<commit_after>// Copyright 2018 ReactiveOps
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rbacdefinition

import (
	"reflect"

	logrus "github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func (bc *RBACDefinitionController) reconcileClusterRoleBindings(
	requestedClusterRoleBindings *[]rbacv1.ClusterRoleBinding,
	existingManagedClusterRoleBindings *[]rbacv1.ClusterRoleBinding,
	ownerReferences *[]metav1.OwnerReference) {

	matchingClusterRoleBindings := []rbacv1.ClusterRoleBinding{}
	clusterRoleBindingsToCreate := []rbacv1.ClusterRoleBinding{}

	for _, requestedCRB := range *requestedClusterRoleBindings {
		alreadyExists := false
		for _, existingCRB := range *existingManagedClusterRoleBindings {
			if crbMatches(&existingCRB, &requestedCRB) {
				alreadyExists = true
				matchingClusterRoleBindings = append(matchingClusterRoleBindings, existingCRB)
				break
			}
		}

		if !alreadyExists {
			clusterRoleBindingsToCreate = append(clusterRoleBindingsToCreate, requestedCRB)
		} else {
			logrus.Debugf("Cluster Role Binding already exists %v", requestedCRB.Name)
		}
	}

	for _, existingCRB := range *existingManagedClusterRoleBindings {
		if reflect.DeepEqual(existingCRB.OwnerReferences, *ownerReferences) {
			matchingRequest := false
			for _, requestedCRB := range matchingClusterRoleBindings {
				if crbMatches(&existingCRB, &requestedCRB) {
					matchingRequest = true
					break
				}
			}

			if !matchingRequest {
				logrus.Infof("Deleting Cluster Role Binding: %v", existingCRB.Name)
				err := bc.kubernetesClientSet.RbacV1().ClusterRoleBindings().Delete(existingCRB.Name, &metav1.DeleteOptions{})
				if err != nil {
					logrus.Errorf("Error deleting Cluster Role Binding: %v", err)
				}
			} else {
				logrus.Debugf("Matches requested Cluster Role Binding: %v", existingCRB.Name)
			}
		}
	}

	for _, clusterRoleBindingToCreate := range clusterRoleBindingsToCreate {
		logrus.Infof("Creating Cluster Role Binding: %v", clusterRoleBindingToCreate.Name)
		_, err := bc.kubernetesClientSet.RbacV1().ClusterRoleBindings().Create(&clusterRoleBindingToCreate)
		if err != nil {
			logrus.Errorf("Error creating Cluster Role Binding: %v", err)
		}
	}
}

func (bc *RBACDefinitionController) reconcileRoleBindings(
	requestedRoleBindings *[]rbacv1.RoleBinding,
	existingManagedRoleBindings *[]rbacv1.RoleBinding,
	ownerReferences *[]metav1.OwnerReference) {

	matchingRoleBindings := []rbacv1.RoleBinding{}
	roleBindingsToCreate := []rbacv1.RoleBinding{}

	for _, requestedRB := range *requestedRoleBindings {
		alreadyExists := false
		for _, existingRB := range *existingManagedRoleBindings {
			if rbMatches(&existingRB, &requestedRB) {
				alreadyExists = true
				matchingRoleBindings = append(matchingRoleBindings, existingRB)
				break
			}
		}

		if !alreadyExists {
			roleBindingsToCreate = append(roleBindingsToCreate, requestedRB)
		} else {
			logrus.Debugf("Role Binding already exists %v", requestedRB.Name)
		}
	}

	for _, existingRB := range *existingManagedRoleBindings {
		if reflect.DeepEqual(existingRB.OwnerReferences, *ownerReferences) {
			matchingRequest := false
			for _, requestedRB := range matchingRoleBindings {
				if rbMatches(&existingRB, &requestedRB) {
					matchingRequest = true
					break
				}
			}

			if !matchingRequest {
				logrus.Infof("Deleting Role Binding %v", existingRB.Name)
				err := bc.kubernetesClientSet.RbacV1().RoleBindings(existingRB.Namespace).Delete(existingRB.Name, &metav1.DeleteOptions{})
				if err != nil {
					logrus.Infof("Error deleting Role Binding: %v", err)
				}
			} else {
				logrus.Debugf("Matches requested Role Binding %v", existingRB.Name)
			}
		}
	}

	for _, roleBindingToCreate := range roleBindingsToCreate {
		logrus.Infof("Creating Role Binding: %v", roleBindingToCreate.Name)
		_, err := bc.kubernetesClientSet.RbacV1().RoleBindings(roleBindingToCreate.ObjectMeta.Namespace).Create(&roleBindingToCreate)
		if err != nil {
			logrus.Errorf("Error creating Role Binding: %v", err)
		}
	}
}

func (bc *RBACDefinitionController) reconcileServiceAccounts(
	requestedServiceAccounts *[]v1.ServiceAccount,
	existingManagedServiceAccounts *[]v1.ServiceAccount,
	ownerReferences *[]metav1.OwnerReference) {

	matchingServiceAccounts := []v1.ServiceAccount{}
	serviceAccountsToCreate := []v1.ServiceAccount{}

	for _, requestedSA := range *requestedServiceAccounts {
		alreadyExists := false
		for _, existingSA := range *existingManagedServiceAccounts {
			if saMatches(&existingSA, &requestedSA) {
				alreadyExists = true
				matchingServiceAccounts = append(matchingServiceAccounts, existingSA)
				break
			}
		}

		if !alreadyExists {
			serviceAccountsToCreate = append(serviceAccountsToCreate, requestedSA)
		} else {
			logrus.Debugf("Service Account already exists %v", requestedSA.Name)
		}
	}

	for _, existingSA := range *existingManagedServiceAccounts {
		if reflect.DeepEqual(existingSA.OwnerReferences, *ownerReferences) {
			matchingRequest := false
			for _, matchingSA := range matchingServiceAccounts {
				if saMatches(&existingSA, &matchingSA) {
					matchingRequest = true
					break
				}
			}

			if !matchingRequest {
				logrus.Infof("Deleting Service Account %v", existingSA.Name)
				err := bc.kubernetesClientSet.CoreV1().ServiceAccounts(existingSA.Namespace).Delete(existingSA.Name, &metav1.DeleteOptions{})
				if err != nil {
					logrus.Infof("Error deleting Service Account: %v", err)
				}
			} else {
				logrus.Debugf("Matches requested Service Account %v", existingSA.Name)
			}
		}
	}

	for _, serviceAccountToCreate := range serviceAccountsToCreate {
		logrus.Infof("Creating Service Account: %v", serviceAccountToCreate.Name)
		_, err := bc.kubernetesClientSet.CoreV1().ServiceAccounts(serviceAccountToCreate.ObjectMeta.Namespace).Create(&serviceAccountToCreate)
		if err != nil {
			logrus.Errorf("Error creating Service Account: %v", err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"math"
	"math/rand"
	"net/http"
	"os"
	"sync/atomic"
	"testing"
	"time"

	"github.com/bmatsuo/lmdb-go/lmdb"
	"github.com/boltdb/bolt"
	"github.com/dgraph-io/badger"
	"github.com/dgraph-io/badger-bench/store"
	"github.com/dgraph-io/badger/table"
	"github.com/dgraph-io/badger/y"
)

var (
	ctx           = context.Background()
	numKeys       = flag.Float64("keys_mil", 10.0, "How many million keys to write.")
	flagDir       = flag.String("dir", "bench-tmp", "Where data is temporarily stored.")
	flagValueSize = flag.Int("valsz", 128, "Size of each value.")
)

const Mi int = 1000000
const Mf float64 = 1000000

func getBadger() (*badger.KV, error) {
	opt := badger.DefaultOptions
	opt.MapTablesTo = table.LoadToRAM
	opt.ValueGCRunInterval = 10 * time.Hour
	opt.Dir = *flagDir + "/badger"
	opt.ValueDir = opt.Dir
	opt.DoNotCompact = true
	opt.ValueGCThreshold = 0.0
	return badger.NewKV(&opt)
}

func getRocks() *store.Store {
	rdb, err := store.NewReadOnlyStore(*flagDir + "/rocks")
	y.Check(err)
	return rdb
}

func getLmdb() *lmdb.Env {
	lmdbEnv, err := lmdb.NewEnv()
	y.Check(err)
	err = lmdbEnv.SetMaxReaders(math.MaxInt64)
	y.Check(err)
	err = lmdbEnv.SetMaxDBs(1)
	y.Check(err)
	err = lmdbEnv.SetMapSize(1 << 38) // ~273Gb
	y.Check(err)

	err = lmdbEnv.Open(*flagDir+"/lmdb", lmdb.Readonly, 0777)
	y.Check(err)
	return lmdbEnv
}

func getBoltDB() *bolt.DB {
	opts := bolt.DefaultOptions
	opts.ReadOnly = true
	boltdb, err := bolt.Open(*flagDir+"/bolt/bolt.db", 0777, opts)
	y.Check(err)
	return boltdb
}

func newKey() []byte {
	k := rand.Int() % int(*numKeys*Mf)
	key := fmt.Sprintf("vsz=%05d-k=%010d", *flagValueSize, k) // 22 bytes.
	return []byte(key)
}

func print(count int) {
	if count%100000 == 0 {
		fmt.Printf(".")
	} else if count%Mi == 0 {
		fmt.Printf("-")
	}
}

type hitCounter struct {
	found    uint64
	notFound uint64
	errored  uint64
}

func (h *hitCounter) Reset() {
	h.found, h.notFound, h.errored = 0, 0, 0
}

func (h *hitCounter) Update(c *hitCounter) {
	atomic.AddUint64(&h.found, c.found)
	atomic.AddUint64(&h.notFound, c.notFound)
	atomic.AddUint64(&h.errored, c.errored)
}

func (h *hitCounter) Print(storeName string, b *testing.B) {
	b.Logf("%s: %d keys had valid values.", storeName, h.found)
	b.Logf("%s: %d keys had no values", storeName, h.notFound)
	b.Logf("%s: %d keys had errors", storeName, h.errored)
	b.Logf("%s: %d total keys looked at", storeName, h.found+h.notFound+h.errored)
	b.Logf("%s: hit rate : %.2f", storeName, float64(h.found)/float64(h.found+h.notFound+h.errored))
}

// A generic read benchmark that runs the doBench func for a specific key value store,
// aggregates the hit counts and prints them out.
func runRandomReadBenchmark(b *testing.B, storeName string, doBench func(*hitCounter, *testing.PB)) {
	counter := &hitCounter{}
	b.Run("read-random"+storeName, func(b *testing.B) {
		counter.Reset()
		b.RunParallel(func(pb *testing.PB) {
			c := &hitCounter{}
			doBench(c, pb)
			counter.Update(c)
		})
	})
	counter.Print(storeName, b)
}

func BenchmarkReadRandomBadger(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	defer bdb.Close()

	runRandomReadBenchmark(b, "badger", func(c *hitCounter, pb *testing.PB) {
		var item badger.KVItem
		for pb.Next() {
			key := newKey()
			err := bdb.Get(key, &item)
			if err != nil {
				c.errored++
			} else if v := item.Value(); v != nil {
				y.AssertTruef(len(v) == *flagValueSize, "Assertion failed. value size is %d, expected %d", len(v), *flagValueSize)
				c.found++
			} else {
				c.notFound++
			}
		}
	})
}

func BenchmarkReadRandomRocks(b *testing.B) {
	rdb := getRocks()
	defer rdb.Close()

	runRandomReadBenchmark(b, "rocksdb", func(c *hitCounter, pb *testing.PB) {
		for pb.Next() {
			key := newKey()
			// FIXME lookup API and increment accordingly
			if _, err := rdb.Get(key); err == nil {
				c.found++
			}
		}
	})
}

func BenchmarkReadRandomLmdb(b *testing.B) {
	lmdbEnv := getLmdb()
	defer lmdbEnv.Close()

	var lmdbDBI lmdb.DBI
	// Acquire handle
	err := lmdbEnv.View(func(txn *lmdb.Txn) error {
		var err error
		lmdbDBI, err = txn.OpenDBI("bench", 0)
		return err
	})
	y.Check(err)
	defer lmdbEnv.CloseDBI(lmdbDBI)

	runRandomReadBenchmark(b, "lmdb", func(c *hitCounter, pb *testing.PB) {
		err := lmdbEnv.View(func(txn *lmdb.Txn) error {
			txn.RawRead = true
			for pb.Next() {
				key := newKey()
				v, err := txn.Get(lmdbDBI, key)
				if lmdb.IsNotFound(err) {
					c.notFound++
					continue
				} else if err != nil {
					c.errored++
					continue
				}
				y.AssertTruef(len(v) == *flagValueSize, "Assertion failed. value size is %d, expected %d", len(v), *flagValueSize)
				c.found++
			}
			return nil
		})
		if err != nil {
			y.Check(err)
		}
	})
}

func BenchmarkReadRandomBolt(b *testing.B) {
	boltdb := getBoltDB()
	defer boltdb.Close()

	runRandomReadBenchmark(b, "bolt", func(c *hitCounter, pb *testing.PB) {
		err := boltdb.View(func(txn *bolt.Tx) error {
			boltBkt := txn.Bucket([]byte("bench"))
			y.AssertTrue(boltBkt != nil)
			for pb.Next() {
				key := newKey()
				v := boltBkt.Get(key)
				if v == nil {
					c.notFound++
					continue
				}
				y.AssertTruef(len(v) == *flagValueSize, "Assertion failed. value size is %d, expected %d", len(v), *flagValueSize)
				c.found++
			}
			return nil
		})
		y.Check(err)
	})
}

func safecopy(dst []byte, src []byte) []byte {
	if cap(dst) < len(src) {
		dst = make([]byte, len(src))
	}
	dst = dst[0:len(src)]
	copy(dst, src)
	return dst
}

func BenchmarkIterateRocks(b *testing.B) {
	rdb := getRocks()
	defer rdb.Close()
	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("rocksdb-iterate", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			itr := rdb.NewIterator()
			var count int
			for itr.SeekToFirst(); itr.Valid(); itr.Next() {
				{
					// do some processing.
					k = safecopy(k, itr.Key().Data())
					v = safecopy(v, itr.Value().Data())
				}
				count++
				print(count)
				if count >= 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateLmdb(b *testing.B) {
	lmdbEnv := getLmdb()
	defer lmdbEnv.Close()

	var lmdbDBI lmdb.DBI
	// Acquire handle
	err := lmdbEnv.View(func(txn *lmdb.Txn) error {
		var err error
		lmdbDBI, err = txn.OpenDBI("bench", 0)
		return err
	})
	y.Check(err)
	defer lmdbEnv.CloseDBI(lmdbDBI)

	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("lmdb-iterate", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			err = lmdbEnv.View(func(txn *lmdb.Txn) error {
				txn.RawRead = true
				cur, err := txn.OpenCursor(lmdbDBI)
				if err != nil {
					return err
				}
				defer cur.Close()

				for {
					k1, v1, err := cur.Get(nil, nil, lmdb.Next)
					if lmdb.IsNotFound(err) {
						return nil
					}
					if err != nil {
						return err
					}

					//fmt.Printf("%s %s\n", k, v)

					y.AssertTruef(len(v1) == *flagValueSize, "Assertion failed. value size is %d, expected %d", len(v1), *flagValueSize)

					// do some processing.
					k = safecopy(k, k1)
					v = safecopy(v, v1)

					count++
					print(count)
					if count >= 2*Mi {
						break
					}
				}
				return nil
			})
			y.Check(err)
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateBadgerOnlyKeys(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	defer bdb.Close()
	k := make([]byte, 1024)
	b.ResetTimer()

	b.Run("badger-iterate-onlykeys", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			// 100 = size, 0 = num workers, false = fwd direction.
			opt := badger.IteratorOptions{}
			opt.PrefetchSize = 10000
			itr := bdb.NewIterator(opt)
			for itr.Rewind(); itr.Valid(); itr.Next() {
				item := itr.Item()
				{
					// do some processing.
					k = safecopy(k, item.Key())
				}
				count++
				print(count)
				if count >= 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateBadgerWithValues(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	defer bdb.Close()
	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("badger-iterate-withvals", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			opt := badger.IteratorOptions{}
			opt.PrefetchSize = 10000
			opt.FetchValues = true
			itr := bdb.NewIterator(opt)
			for itr.Rewind(); itr.Valid(); itr.Next() {
				item := itr.Item()
				vsz := len(item.Value())
				y.AssertTruef(vsz == *flagValueSize, "Assertion failed. value size is %d, expected %d", vsz, *flagValueSize)
				{
					// do some processing.
					k = safecopy(k, item.Key())
					v = safecopy(v, item.Value())
				}
				count++
				print(count)
				if count >= 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func TestMain(m *testing.M) {
	flag.Parse()
	// call flag.Parse() here if TestMain uses flags
	go http.ListenAndServe(":8080", nil)
	os.Exit(m.Run())
}
<commit_msg>Add iterate benchmark for BoltDB<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"math"
	"math/rand"
	"net/http"
	"os"
	"sync/atomic"
	"testing"
	"time"

	"github.com/bmatsuo/lmdb-go/lmdb"
	"github.com/boltdb/bolt"
	"github.com/dgraph-io/badger"
	"github.com/dgraph-io/badger-bench/store"
	"github.com/dgraph-io/badger/table"
	"github.com/dgraph-io/badger/y"
)

var (
	ctx           = context.Background()
	numKeys       = flag.Float64("keys_mil", 10.0, "How many million keys to write.")
	flagDir       = flag.String("dir", "bench-tmp", "Where data is temporarily stored.")
	flagValueSize = flag.Int("valsz", 128, "Size of each value.")
)

const Mi int = 1000000
const Mf float64 = 1000000

func getBadger() (*badger.KV, error) {
	opt := badger.DefaultOptions
	opt.MapTablesTo = table.LoadToRAM
	opt.ValueGCRunInterval = 10 * time.Hour
	opt.Dir = *flagDir + "/badger"
	opt.ValueDir = opt.Dir
	opt.DoNotCompact = true
	opt.ValueGCThreshold = 0.0
	return badger.NewKV(&opt)
}

func getRocks() *store.Store {
	rdb, err := store.NewReadOnlyStore(*flagDir + "/rocks")
	y.Check(err)
	return rdb
}

func getLmdb() *lmdb.Env {
	lmdbEnv, err := lmdb.NewEnv()
	y.Check(err)
	err = lmdbEnv.SetMaxReaders(math.MaxInt64)
	y.Check(err)
	err = lmdbEnv.SetMaxDBs(1)
	y.Check(err)
	err = lmdbEnv.SetMapSize(1 << 38) // ~273Gb
	y.Check(err)

	err = lmdbEnv.Open(*flagDir+"/lmdb", lmdb.Readonly, 0777)
	y.Check(err)
	return lmdbEnv
}

func getBoltDB() *bolt.DB {
	opts := bolt.DefaultOptions
	opts.ReadOnly = true
	boltdb, err := bolt.Open(*flagDir+"/bolt/bolt.db", 0777, opts)
	y.Check(err)
	return boltdb
}

func newKey() []byte {
	k := rand.Int() % int(*numKeys*Mf)
	key := fmt.Sprintf("vsz=%05d-k=%010d", *flagValueSize, k) // 22 bytes.
	return []byte(key)
}

func print(count int) {
	if count%100000 == 0 {
		fmt.Printf(".")
	} else if count%Mi == 0 {
		fmt.Printf("-")
	}
}

type hitCounter struct {
	found    uint64
	notFound uint64
	errored  uint64
}

func (h *hitCounter) Reset() {
	h.found, h.notFound, h.errored = 0, 0, 0
}

func (h *hitCounter) Update(c *hitCounter) {
	atomic.AddUint64(&h.found, c.found)
	atomic.AddUint64(&h.notFound, c.notFound)
	atomic.AddUint64(&h.errored, c.errored)
}

func (h *hitCounter) Print(storeName string, b *testing.B) {
	b.Logf("%s: %d keys had valid values.", storeName, h.found)
	b.Logf("%s: %d keys had no values", storeName, h.notFound)
	b.Logf("%s: %d keys had errors", storeName, h.errored)
	b.Logf("%s: %d total keys looked at", storeName, h.found+h.notFound+h.errored)
	b.Logf("%s: hit rate : %.2f", storeName, float64(h.found)/float64(h.found+h.notFound+h.errored))
}

// A generic read benchmark that runs the doBench func for a specific key value store,
// aggregates the hit counts and prints them out.
func runRandomReadBenchmark(b *testing.B, storeName string, doBench func(*hitCounter, *testing.PB)) {
	counter := &hitCounter{}
	b.Run("read-random"+storeName, func(b *testing.B) {
		counter.Reset()
		b.RunParallel(func(pb *testing.PB) {
			c := &hitCounter{}
			doBench(c, pb)
			counter.Update(c)
		})
	})
	counter.Print(storeName, b)
}

func BenchmarkReadRandomBadger(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	defer bdb.Close()

	runRandomReadBenchmark(b, "badger", func(c *hitCounter, pb *testing.PB) {
		var item badger.KVItem
		for pb.Next() {
			key := newKey()
			err := bdb.Get(key, &item)
			if err != nil {
				c.errored++
			} else if v := item.Value(); v != nil {
				y.AssertTruef(len(v) == *flagValueSize, "Assertion failed. value size is %d, expected %d", len(v), *flagValueSize)
				c.found++
			} else {
				c.notFound++
			}
		}
	})
}

func BenchmarkReadRandomRocks(b *testing.B) {
	rdb := getRocks()
	defer rdb.Close()

	runRandomReadBenchmark(b, "rocksdb", func(c *hitCounter, pb *testing.PB) {
		for pb.Next() {
			key := newKey()
			// FIXME lookup API and increment accordingly
			if _, err := rdb.Get(key); err == nil {
				c.found++
			}
		}
	})
}

func BenchmarkReadRandomLmdb(b *testing.B) {
	lmdbEnv := getLmdb()
	defer lmdbEnv.Close()

	var lmdbDBI lmdb.DBI
	// Acquire handle
	err := lmdbEnv.View(func(txn *lmdb.Txn) error {
		var err error
		lmdbDBI, err = txn.OpenDBI("bench", 0)
		return err
	})
	y.Check(err)
	defer lmdbEnv.CloseDBI(lmdbDBI)

	runRandomReadBenchmark(b, "lmdb", func(c *hitCounter, pb *testing.PB) {
		err := lmdbEnv.View(func(txn *lmdb.Txn) error {
			txn.RawRead = true
			for pb.Next() {
				key := newKey()
				v, err := txn.Get(lmdbDBI, key)
				if lmdb.IsNotFound(err) {
					c.notFound++
					continue
				} else if err != nil {
					c.errored++
					continue
				}
				y.AssertTruef(len(v) == *flagValueSize, "Assertion failed. value size is %d, expected %d", len(v), *flagValueSize)
				c.found++
			}
			return nil
		})
		if err != nil {
			y.Check(err)
		}
	})
}

func BenchmarkReadRandomBolt(b *testing.B) {
	boltdb := getBoltDB()
	defer boltdb.Close()

	runRandomReadBenchmark(b, "bolt", func(c *hitCounter, pb *testing.PB) {
		err := boltdb.View(func(txn *bolt.Tx) error {
			boltBkt := txn.Bucket([]byte("bench"))
			y.AssertTrue(boltBkt != nil)
			for pb.Next() {
				key := newKey()
				v := boltBkt.Get(key)
				if v == nil {
					c.notFound++
					continue
				}
				y.AssertTruef(len(v) == *flagValueSize, "Assertion failed. value size is %d, expected %d", len(v), *flagValueSize)
				c.found++
			}
			return nil
		})
		y.Check(err)
	})
}

func safecopy(dst []byte, src []byte) []byte {
	if cap(dst) < len(src) {
		dst = make([]byte, len(src))
	}
	dst = dst[0:len(src)]
	copy(dst, src)
	return dst
}

func BenchmarkIterateRocks(b *testing.B) {
	rdb := getRocks()
	defer rdb.Close()
	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("rocksdb-iterate", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			itr := rdb.NewIterator()
			var count int
			for itr.SeekToFirst(); itr.Valid(); itr.Next() {
				{
					// do some processing.
					k = safecopy(k, itr.Key().Data())
					v = safecopy(v, itr.Value().Data())
				}
				count++
				print(count)
				if count >= 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateLmdb(b *testing.B) {
	lmdbEnv := getLmdb()
	defer lmdbEnv.Close()

	var lmdbDBI lmdb.DBI
	// Acquire handle
	err := lmdbEnv.View(func(txn *lmdb.Txn) error {
		var err error
		lmdbDBI, err = txn.OpenDBI("bench", 0)
		return err
	})
	y.Check(err)
	defer lmdbEnv.CloseDBI(lmdbDBI)

	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("lmdb-iterate", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			err = lmdbEnv.View(func(txn *lmdb.Txn) error {
				txn.RawRead = true
				cur, err := txn.OpenCursor(lmdbDBI)
				if err != nil {
					return err
				}
				defer cur.Close()

				for {
					k1, v1, err := cur.Get(nil, nil, lmdb.Next)
					if lmdb.IsNotFound(err) {
						return nil
					}
					if err != nil {
						return err
					}

					//fmt.Printf("%s %s\n", k, v)

					y.AssertTruef(len(v1) == *flagValueSize, "Assertion failed. value size is %d, expected %d", len(v1), *flagValueSize)

					// do some processing.
					k = safecopy(k, k1)
					v = safecopy(v, v1)

					count++
					print(count)
					if count >= 2*Mi {
						break
					}
				}
				return nil
			})
			y.Check(err)
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateBolt(b *testing.B) {
	boltdb := getBoltDB()
	defer boltdb.Close()

	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("boltdb-iterate", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			err := boltdb.View(func(txn *bolt.Tx) error {
				boltBkt := txn.Bucket([]byte("bench"))
				y.AssertTrue(boltBkt != nil)
				cur := boltBkt.Cursor()
				for k1, v1 := cur.First(); k1 != nil; k1, v1 = cur.Next() {
					y.AssertTruef(len(v1) == *flagValueSize, "Assertion failed. value size is %d, expected %d", len(v1), *flagValueSize)

					// do some processing.
					k = safecopy(k, k1)
					v = safecopy(v, v1)

					count++
					print(count)
					if count >= 2*Mi {
						break
					}
				}
				return nil
			})
			y.Check(err)
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateBadgerOnlyKeys(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	defer bdb.Close()
	k := make([]byte, 1024)
	b.ResetTimer()

	b.Run("badger-iterate-onlykeys", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			// 100 = size, 0 = num workers, false = fwd direction.
			opt := badger.IteratorOptions{}
			opt.PrefetchSize = 10000
			itr := bdb.NewIterator(opt)
			for itr.Rewind(); itr.Valid(); itr.Next() {
				item := itr.Item()
				{
					// do some processing.
					k = safecopy(k, item.Key())
				}
				count++
				print(count)
				if count >= 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateBadgerWithValues(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	defer bdb.Close()
	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("badger-iterate-withvals", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			opt := badger.IteratorOptions{}
			opt.PrefetchSize = 10000
			opt.FetchValues = true
			itr := bdb.NewIterator(opt)
			for itr.Rewind(); itr.Valid(); itr.Next() {
				item := itr.Item()
				vsz := len(item.Value())
				y.AssertTruef(vsz == *flagValueSize, "Assertion failed. value size is %d, expected %d", vsz, *flagValueSize)
				{
					// do some processing.
					k = safecopy(k, item.Key())
					v = safecopy(v, item.Value())
				}
				count++
				print(count)
				if count >= 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func TestMain(m *testing.M) {
	flag.Parse()
	// call flag.Parse() here if TestMain uses flags
	go http.ListenAndServe(":8080", nil)
	os.Exit(m.Run())
}
<|endoftext|>


<commit_before>package flatcar

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"time"

	"github.com/pkg/errors"
	v1 "k8s.io/api/core/v1"

	"github.com/sapcc/kubernikus/pkg/util/version"
)

const (
	stable channel = "stable"
	beta   channel = "beta"
	alpha  channel = "alpha"
)

var (
	now = time.Now

	flatcarVersionBaseURL       = "https://%s.release.flatcar-linux.net/amd64-usr/current/version.txt"
	flatcarVersionRE            = regexp.MustCompile(`FLATCAR_VERSION=(.+)`)
	flatcarVersionIdentifierRE  = regexp.MustCompile(`(\d+\.\d+\.\d+)`)
	flatcarVersionFetchInterval = 1 * time.Hour
)

type channel string

// Version is a helper that fetches and caches flatcar versions
type Version struct {
	Client    *http.Client
	versions  map[channel]*version.Version
	fetchedAt map[channel]time.Time
}

// Stable returns version of flatcar stable channel
func (d *Version) Stable() (*version.Version, error) {
	return d.latest(stable)
}

// Beta returns version of flatcar beta channel
func (d *Version) Beta() (*version.Version, error) {
	return d.latest(beta)
}

// Alpha returns version of flatcar alpha channel
func (d *Version) Alpha() (*version.Version, error) {
	return d.latest(alpha)
}

// IsNodeUptodate checkes whether a Kubernetes Node is a flatcar that needs updating
func (d *Version) IsNodeUptodate(node *v1.Node) (bool, error) {
	var availableVersion, nodeVersion *version.Version
	var err error

	availableVersion, err = d.Stable()
	if err != nil {
		return false, errors.Wrap(err, "flatcar version couldn't be retrieved.")
	}

	nodeVersion, err = ExractVersion(node)
	if err != nil {
		return false, err
	}

	return nodeVersion.AtLeast(availableVersion), nil
}

// ExractVersion returns a semantic version of the node
func ExractVersion(node *v1.Node) (*version.Version, error) {
	match := flatcarVersionIdentifierRE.FindSubmatch([]byte(node.Status.NodeInfo.OSImage))
	if len(match) < 2 {
		return nil, fmt.Errorf("Couldn't match flatcar version from NodeInfo.OSImage: %s", node.Status.NodeInfo.OSImage)
	}

	nodeVersion, err := version.ParseSemantic(string(match[1]))
	if err != nil {
		return nil, errors.Wrapf(err, "Node version can't be parsed from %s", match[1])
	}

	return nodeVersion, nil
}

func (d *Version) latest(c channel) (*version.Version, error) {
	var err error

	if d.Client == nil {
		d.Client = &http.Client{
			Timeout: time.Second * 10,
		}
	}

	if d.versions == nil {
		d.versions = make(map[channel]*version.Version)
	}

	if d.fetchedAt == nil {
		d.fetchedAt = make(map[channel]time.Time)
	}

	if d.versions[c] == nil || now().After(d.fetchedAt[c].Add(flatcarVersionFetchInterval)) {
		d.versions[c], err = d.fetch(c)
		if err != nil {
			return nil, errors.Wrapf(err, "Couldn't fetch latest %s version", c)
		}
	}

	return d.versions[c], nil
}

func (d *Version) fetch(c channel) (*version.Version, error) {
	r, err := d.Client.Get(fmt.Sprintf(flatcarVersionBaseURL, c))
	if err != nil {
		return nil, fmt.Errorf("Couldn't fetch flatcar version: %s", err)
	}

	defer r.Body.Close()

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, fmt.Errorf("Couldn't read flatcar version: %s", err)
	}

	v := flatcarVersionRE.FindSubmatch(body)
	if len(v) < 2 {
		return nil, fmt.Errorf("Couldn't parse flatcar version: %s", err)
	}

	d.fetchedAt[c] = now()

	return version.ParseSemantic(string(v[1]))
}
<commit_msg>use mutex to avoid concurent map writes<commit_after>package flatcar

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"sync"
	"time"

	"github.com/pkg/errors"
	v1 "k8s.io/api/core/v1"

	"github.com/sapcc/kubernikus/pkg/util/version"
)

const (
	stable channel = "stable"
	beta   channel = "beta"
	alpha  channel = "alpha"
)

var (
	now = time.Now

	flatcarVersionBaseURL       = "https://%s.release.flatcar-linux.net/amd64-usr/current/version.txt"
	flatcarVersionRE            = regexp.MustCompile(`FLATCAR_VERSION=(.+)`)
	flatcarVersionIdentifierRE  = regexp.MustCompile(`(\d+\.\d+\.\d+)`)
	flatcarVersionFetchInterval = 1 * time.Hour
)

type channel string

// Version is a helper that fetches and caches flatcar versions
type Version struct {
	Client    *http.Client
	versions  map[channel]*version.Version
	fetchedAt map[channel]time.Time

	mu sync.Mutex
}

// Stable returns version of flatcar stable channel
func (d *Version) Stable() (*version.Version, error) {
	return d.latest(stable)
}

// Beta returns version of flatcar beta channel
func (d *Version) Beta() (*version.Version, error) {
	return d.latest(beta)
}

// Alpha returns version of flatcar alpha channel
func (d *Version) Alpha() (*version.Version, error) {
	return d.latest(alpha)
}

// IsNodeUptodate checkes whether a Kubernetes Node is a flatcar that needs updating
func (d *Version) IsNodeUptodate(node *v1.Node) (bool, error) {
	var availableVersion, nodeVersion *version.Version
	var err error

	availableVersion, err = d.Stable()
	if err != nil {
		return false, errors.Wrap(err, "flatcar version couldn't be retrieved.")
	}

	nodeVersion, err = ExractVersion(node)
	if err != nil {
		return false, err
	}

	return nodeVersion.AtLeast(availableVersion), nil
}

// ExractVersion returns a semantic version of the node
func ExractVersion(node *v1.Node) (*version.Version, error) {
	match := flatcarVersionIdentifierRE.FindSubmatch([]byte(node.Status.NodeInfo.OSImage))
	if len(match) < 2 {
		return nil, fmt.Errorf("Couldn't match flatcar version from NodeInfo.OSImage: %s", node.Status.NodeInfo.OSImage)
	}

	nodeVersion, err := version.ParseSemantic(string(match[1]))
	if err != nil {
		return nil, errors.Wrapf(err, "Node version can't be parsed from %s", match[1])
	}

	return nodeVersion, nil
}

func (d *Version) latest(c channel) (*version.Version, error) {
	var err error
	d.mu.Lock()
	defer d.mu.Unlock()

	if d.Client == nil {
		d.Client = &http.Client{
			Timeout: time.Second * 10,
		}
	}

	if d.versions == nil {
		d.versions = make(map[channel]*version.Version)
	}

	if d.fetchedAt == nil {
		d.fetchedAt = make(map[channel]time.Time)
	}

	if d.versions[c] == nil || now().After(d.fetchedAt[c].Add(flatcarVersionFetchInterval)) {
		d.versions[c], err = d.fetch(c)
		if err != nil {
			return nil, errors.Wrapf(err, "Couldn't fetch latest %s version", c)
		}
	}

	return d.versions[c], nil
}

func (d *Version) fetch(c channel) (*version.Version, error) {
	r, err := d.Client.Get(fmt.Sprintf(flatcarVersionBaseURL, c))
	if err != nil {
		return nil, fmt.Errorf("Couldn't fetch flatcar version: %s", err)
	}

	defer r.Body.Close()

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, fmt.Errorf("Couldn't read flatcar version: %s", err)
	}

	v := flatcarVersionRE.FindSubmatch(body)
	if len(v) < 2 {
		return nil, fmt.Errorf("Couldn't parse flatcar version: %s", err)
	}

	d.fetchedAt[c] = now()

	return version.ParseSemantic(string(v[1]))
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"github.com/mitchellh/hashstructure"
	"github.com/patrickmn/go-cache"
	"sync"
	"time"
)

type AppCache struct {
	Cache *cache.Cache
}

func (a *AppCache) Get(key interface{}) interface{} {
	hash, err := hashstructure.Hash(key, nil)
	if err != nil {
		return nil
	}
	value, found := a.Cache.Get(fmt.Sprintf("%d", hash))
	if found == false {
		return nil
	}
	return value
}

func (a *AppCache) Set(key map[string]string, value interface{}) {
	hash, err := hashstructure.Hash(key, nil)
	if err != nil {
		return
	}
	a.Cache.Set(fmt.Sprint(hash), value, cache.DefaultExpiration)
}

func (a *AppCache) SetKey(key string, value interface{}) {
	a.Cache.Set(key, value, cache.DefaultExpiration)
}

func (a *AppCache) Del(key map[string]string) {
	hash, _ := hashstructure.Hash(key, nil)
	a.Cache.Delete(fmt.Sprint(hash))
}

func (a *AppCache) OnEvict(fn func(string, interface{})) {
	a.Cache.OnEvicted(fn)
}

func NewAppCache(arg ...time.Duration) AppCache {
	var retention time.Duration = 5
	var cleanup time.Duration = 10
	if len(arg) > 0 {
		retention = arg[0]
		if len(arg) > 1 {
			cleanup = arg[1]
		}
	}
	c := AppCache{}
	c.Cache = cache.New(retention*time.Minute, cleanup*time.Minute)
	return c
}

func NewQuickCache(arg ...time.Duration) AppCache {
	var retention time.Duration = 5
	var cleanup time.Duration = 10
	if len(arg) > 0 {
		retention = arg[0]
		if len(arg) > 1 {
			cleanup = arg[1]
		}
	}
	c := AppCache{}
	c.Cache = cache.New(retention * time.Second, cleanup * time.Second)
	return c
}

// ============================================================================

type KeyValueStore struct {
	cache map[string]interface{}
	sync.Mutex
}

func NewKeyValueStore() KeyValueStore {
	return KeyValueStore{ cache: make(map[string]interface{}) }
}

func (this *KeyValueStore) Get(key string) interface{} {
	this.Lock()
	val := this.cache[key]
	this.Unlock()
	return val
}

func (this *KeyValueStore) Set(key string, value interface{}) {
	this.Lock()
	this.cache[key] = value
	this.Unlock()
}

func (this *KeyValueStore) Clear() {
	this.Lock()
	this.cache = make(map[string]interface{})
	this.Unlock()
}
<commit_msg>improve (lock): leverage RWLock<commit_after>package common

import (
	"fmt"
	"github.com/mitchellh/hashstructure"
	"github.com/patrickmn/go-cache"
	"sync"
	"time"
)

type AppCache struct {
	Cache *cache.Cache
}

func (a *AppCache) Get(key interface{}) interface{} {
	hash, err := hashstructure.Hash(key, nil)
	if err != nil {
		return nil
	}
	value, found := a.Cache.Get(fmt.Sprintf("%d", hash))
	if found == false {
		return nil
	}
	return value
}

func (a *AppCache) Set(key map[string]string, value interface{}) {
	hash, err := hashstructure.Hash(key, nil)
	if err != nil {
		return
	}
	a.Cache.Set(fmt.Sprint(hash), value, cache.DefaultExpiration)
}

func (a *AppCache) SetKey(key string, value interface{}) {
	a.Cache.Set(key, value, cache.DefaultExpiration)
}

func (a *AppCache) Del(key map[string]string) {
	hash, _ := hashstructure.Hash(key, nil)
	a.Cache.Delete(fmt.Sprint(hash))
}

func (a *AppCache) OnEvict(fn func(string, interface{})) {
	a.Cache.OnEvicted(fn)
}

func NewAppCache(arg ...time.Duration) AppCache {
	var retention time.Duration = 5
	var cleanup time.Duration = 10
	if len(arg) > 0 {
		retention = arg[0]
		if len(arg) > 1 {
			cleanup = arg[1]
		}
	}
	c := AppCache{}
	c.Cache = cache.New(retention*time.Minute, cleanup*time.Minute)
	return c
}

func NewQuickCache(arg ...time.Duration) AppCache {
	var retention time.Duration = 5
	var cleanup time.Duration = 10
	if len(arg) > 0 {
		retention = arg[0]
		if len(arg) > 1 {
			cleanup = arg[1]
		}
	}
	c := AppCache{}
	c.Cache = cache.New(retention * time.Second, cleanup * time.Second)
	return c
}

// ============================================================================

type KeyValueStore struct {
	cache map[string]interface{}
	sync.RWMutex
}

func NewKeyValueStore() KeyValueStore {
	return KeyValueStore{ cache: make(map[string]interface{}) }
}

func (this *KeyValueStore) Get(key string) interface{} {
	this.RLock()
	val := this.cache[key]
	this.RUnlock()
	return val
}

func (this *KeyValueStore) Set(key string, value interface{}) {
	this.Lock()
	this.cache[key] = value
	this.Unlock()
}

func (this *KeyValueStore) Clear() {
	this.Lock()
	this.cache = make(map[string]interface{})
	this.Unlock()
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/af83/edwig/siri"
)

type Server struct {
	UUIDConsumer
	ClockConsumer

	bind        string
	startedTime time.Time
}

func NewServer(bind string) *Server {
	server := Server{bind: bind}
	server.startedTime = server.Clock().Now()
	return &server
}

func (server *Server) ListenAndServe() error {
	http.HandleFunc("/siri", server.checkStatusHandler)
	fmt.Printf("Starting server on %s\n", server.bind)
	return http.ListenAndServe(server.bind, nil)
}

func (server *Server) checkStatusHandler(w http.ResponseWriter, r *http.Request) {
	// Try to read and parse request body
	requestContent, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, "Invalid request: can't read content", 500)
		return
	}
	xmlRequest, err := siri.NewXMLCheckStatusRequestFromContent(requestContent)
	if err != nil {
		http.Error(w, "Invalid request: can't parse content", 500)
		return
	}

	fmt.Printf("CheckStatus %s\n", xmlRequest.MessageIdentifier())

	// Set Content-Type header and create a SIRICheckStatusResponse
	w.Header().Set("Content-Type", "text/xml")

	response := new(siri.SIRICheckStatusResponse)
	response.Address = strings.Join([]string{r.URL.Host, r.URL.Path}, "")
	response.ProducerRef = "Edwig"
	response.RequestMessageRef = xmlRequest.MessageIdentifier()
	response.ResponseMessageIdentifier = fmt.Sprintf("Edwig:ResponseMessage::%s:LOC", server.NewUUID())
	response.Status = true // Temp
	response.ResponseTimestamp = server.Clock().Now()
	response.ServiceStartedTime = server.startedTime

	fmt.Fprintf(w, siri.WrapSoap(response.BuildXML()))
}
<commit_msg>use SOAPEnvelope in server checkstatus handler<commit_after>package api

import (
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/af83/edwig/siri"
)

type Server struct {
	UUIDConsumer
	ClockConsumer

	bind        string
	startedTime time.Time
}

func NewServer(bind string) *Server {
	server := Server{bind: bind}
	server.startedTime = server.Clock().Now()
	return &server
}

func (server *Server) ListenAndServe() error {
	http.HandleFunc("/siri", server.checkStatusHandler)
	fmt.Printf("Starting server on %s\n", server.bind)
	return http.ListenAndServe(server.bind, nil)
}

func (server *Server) checkStatusHandler(w http.ResponseWriter, r *http.Request) {
	// Create XMLCheckStatusResponse
	envelope, err := siri.NewSOAPEnvelope(r.Body)
	if err != nil {
		http.Error(w, "Invalid request: can't read content", 400)
		return
	}
	if envelope.BodyType() != "CheckStatus" {
		fmt.Println(envelope.BodyType())
		http.Error(w, "Invalid request: not a checkstatus", 400)
		return
	}
	xmlRequest := siri.NewXMLCheckStatusRequest(envelope.Body())

	fmt.Printf("CheckStatus %s\n", xmlRequest.MessageIdentifier())

	// Set Content-Type header and create a SIRICheckStatusResponse
	w.Header().Set("Content-Type", "text/xml")

	response := new(siri.SIRICheckStatusResponse)
	response.Address = strings.Join([]string{r.URL.Host, r.URL.Path}, "")
	response.ProducerRef = "Edwig"
	response.RequestMessageRef = xmlRequest.MessageIdentifier()
	response.ResponseMessageIdentifier = fmt.Sprintf("Edwig:ResponseMessage::%s:LOC", server.NewUUID())
	response.Status = true // Temp
	response.ResponseTimestamp = server.Clock().Now()
	response.ServiceStartedTime = server.startedTime

	fmt.Fprintf(w, siri.WrapSoap(response.BuildXML()))
}
<|endoftext|>