<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package vm

import (
	"fmt"
	"regexp"
	"time"

	"github.com/google/mtail/metrics"
	"github.com/google/mtail/metrics/datum"
)

// compiler is data for the code generator.
type codegen struct {
	name string // Name of the program.

	errors ErrorList // Compile errors.
	obj    object    // The object to return

	decos []*decoNode // Decorator stack to unwind
}

// CodeGen is the function that compiles the program to bytecode and data.
func CodeGen(name string, ast astNode) (*object, error) {
	c := &codegen{name: name}
	Walk(c, ast)
	if len(c.errors) > 0 {
		return nil, c.errors
	}
	return &c.obj, nil
}

func (c *codegen) errorf(pos *position, format string, args ...interface{}) {
	e := "Internal compiler error, aborting compilation: " + fmt.Sprintf(format, args...)
	c.errors.Add(pos, e)
}

func (c *codegen) emit(i instr) {
	c.obj.prog = append(c.obj.prog, i)
}

func (c *codegen) VisitBefore(node astNode) Visitor {
	switch n := node.(type) {

	case *declNode:
		var name string
		if n.exportedName != "" {
			name = n.exportedName
		} else {
			name = n.name
		}
		m := metrics.NewMetric(name, c.name, n.kind, metrics.Int, n.keys...)
		// Scalar counters can be initialized to zero.  Dimensioned counters we
		// don't know the values of the labels yet.  Gauges and Timers we can't
		// assume start at zero.
		if len(n.keys) == 0 && n.kind == metrics.Counter {
			d, err := m.GetDatum()
			if err != nil {
				c.errorf(n.Pos(), "%s", err)
				return nil
			}
			// Initialize to zero at the zero time.
			datum.SetInt(d, 0, time.Unix(0, 0))
		}
		m.Hidden = n.hidden
		(*n.sym).Binding = m
		n.sym.Addr = len(c.obj.m)
		c.obj.m = append(c.obj.m, m)
		return nil

	case *condNode:
		if n.cond != nil {
			Walk(c, n.cond)
		}
		// Save PC of previous jump instruction emitted by the n.cond
		// compilation.  (See regexNode and relNode cases, which will emit a
		// jump as the last instr.)  This jump will skip over the truthNode.
		pc := len(c.obj.prog) - 1
		// Set matched flag false for children.
		c.emit(instr{setmatched, false})
		Walk(c, n.truthNode)
		// Re-set matched flag to true for rest of current block.
		c.emit(instr{setmatched, true})
		// Rewrite n.cond's jump target to jump to instruction after block.
		c.obj.prog[pc].opnd = len(c.obj.prog)
		// Now also emit the else clause, and a jump.
		if n.elseNode != nil {
			c.emit(instr{op: jmp})
			// Rewrite jump again to avoid this else-skipper just emitted.
			c.obj.prog[pc].opnd = len(c.obj.prog)
			// Now get the PC of the else-skipper just emitted.
			pc = len(c.obj.prog) - 1
			Walk(c, n.elseNode)
			// Rewrite else-skipper to the next PC.
			c.obj.prog[pc].opnd = len(c.obj.prog)
		}
		return nil

	case *regexNode:
		re, err := regexp.Compile(n.pattern)
		if err != nil {
			c.errorf(n.Pos(), "%s", err)
			return nil
		}
		c.obj.re = append(c.obj.re, re)
		// Store the location of this regular expression in the regexNode
		n.addr = len(c.obj.re) - 1
		c.emit(instr{match, n.addr})
		c.emit(instr{op: jnm})

	case *stringConstNode:
		c.obj.str = append(c.obj.str, n.text)
		c.emit(instr{str, len(c.obj.str) - 1})

	case *intConstNode:
		c.emit(instr{push, n.i})

	case *floatConstNode:
		c.emit(instr{push, n.f})

	case *idNode:
		if n.sym == nil || n.sym.Binding == nil {
			c.errorf(n.Pos(), "No metric bound to identifier %q", n.name)
			return nil
		}
		c.emit(instr{mload, n.sym.Addr})
		m := n.sym.Binding.(*metrics.Metric)
		c.emit(instr{dload, len(m.Keys)})

	case *caprefNode:
		if n.sym == nil || n.sym.Binding == nil {
			c.errorf(n.Pos(), "No regular expression bound to capref %q", n.name)
			return nil
		}
		rn := n.sym.Binding.(*regexNode)
		// rn.addr contains the index of the regular expression object,
		// which correlates to storage on the re slice
		c.emit(instr{push, rn.addr})
		// n.sym.addr is the capture group offset
		c.emit(instr{capref, n.sym.Addr})

	case *defNode:
		// Do nothing, defs are inlined.
		return nil

	case *decoNode:
		// Put the current block on the stack
		c.decos = append(c.decos, n)
		if n.def == nil {
			c.errorf(n.Pos(), "No definition found for decorator %q", n.name)
			return nil
		}
		// then iterate over the decorator's nodes
		Walk(c, n.def.block)
		c.decos = c.decos[:len(c.decos)-1]
		return nil

	case *nextNode:
		// Visit the 'next' block on the decorated block stack
		deco := c.decos[len(c.decos)-1]
		Walk(c, deco.block)
		return nil

	case *otherwiseNode:
		c.emit(instr{op: otherwise})
		c.emit(instr{op: jnm})

	case *delNode:
		Walk(c, n.n)
		// overwdrite the dload instruction
		pc := len(c.obj.prog) - 1
		c.obj.prog[pc].op = del
	}

	return c
}

var typedOperators = map[int]map[Type]opcode{
	PLUS: {Int: iadd,
		Float: fadd},
	MINUS: {Int: isub,
		Float: fsub},
	MUL: {Int: imul,
		Float: fmul},
	DIV: {Int: idiv,
		Float: fdiv},
	MOD: {Int: imod,
		Float: fmod},
	POW: {Int: ipow,
		Float: fpow},
	ASSIGN: {Int: iset,
		Float: fset},
}

func (c *codegen) VisitAfter(node astNode) {
	switch n := node.(type) {
	case *builtinNode:
		if n.args != nil {
			c.emit(instr{builtin[n.name], len(n.args.(*exprlistNode).children)})
		} else {
			c.emit(instr{op: builtin[n.name]})
		}
	case *unaryExprNode:
		switch n.op {
		case INC:
			c.emit(instr{op: inc})
		case NOT:
			c.emit(instr{op: not})
		}
	case *binaryExprNode:
		switch n.op {
		case LT:
			c.emit(instr{cmp, -1})
			c.emit(instr{op: jnm})
		case GT:
			c.emit(instr{cmp, 1})
			c.emit(instr{op: jnm})
		case LE:
			c.emit(instr{cmp, 1})
			c.emit(instr{op: jm})
		case GE:
			c.emit(instr{cmp, -1})
			c.emit(instr{op: jm})
		case EQ:
			c.emit(instr{cmp, 0})
			c.emit(instr{op: jnm})
		case NE:
			c.emit(instr{cmp, 0})
			c.emit(instr{op: jm})
		case PLUS, MINUS, MUL, DIV, MOD, POW, ASSIGN:
			switch n.Type() {
			case Int, Float:
				c.emit(instr{op: typedOperators[n.op][n.Type()]})
			default:
				c.errorf(n.Pos(), "Invalid type for binary expression: %q", n.Type())
			}
		case AND:
			c.emit(instr{op: and})
		case OR:
			c.emit(instr{op: or})
		case XOR:
			c.emit(instr{op: xor})
		case SHL:
			c.emit(instr{op: shl})
		case SHR:
			c.emit(instr{op: shr})
		}
	}
}
<commit_msg>Use the type of the declNode to create the type of the metric datum.<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package vm

import (
	"fmt"
	"regexp"
	"time"

	"github.com/google/mtail/metrics"
	"github.com/google/mtail/metrics/datum"
)

// compiler is data for the code generator.
type codegen struct {
	name string // Name of the program.

	errors ErrorList // Compile errors.
	obj    object    // The object to return

	decos []*decoNode // Decorator stack to unwind
}

// CodeGen is the function that compiles the program to bytecode and data.
func CodeGen(name string, ast astNode) (*object, error) {
	c := &codegen{name: name}
	Walk(c, ast)
	if len(c.errors) > 0 {
		return nil, c.errors
	}
	return &c.obj, nil
}

func (c *codegen) errorf(pos *position, format string, args ...interface{}) {
	e := "Internal compiler error, aborting compilation: " + fmt.Sprintf(format, args...)
	c.errors.Add(pos, e)
}

func (c *codegen) emit(i instr) {
	c.obj.prog = append(c.obj.prog, i)
}

var kindMap = map[Type]datum.Type{
	Int:   metrics.Int,
	Float: metrics.Float,
}

func (c *codegen) VisitBefore(node astNode) Visitor {
	switch n := node.(type) {

	case *declNode:
		var name string
		if n.exportedName != "" {
			name = n.exportedName
		} else {
			name = n.name
		}
		kind, ok := kindMap[n.Type()]
		if !ok {
			c.errorf(n.Pos(), "Unable to determine type for metric %q", name)
			return nil
		}
		m := metrics.NewMetric(name, c.name, n.kind, kind, n.keys...)
		// Scalar counters can be initialized to zero.  Dimensioned counters we
		// don't know the values of the labels yet.  Gauges and Timers we can't
		// assume start at zero.
		if len(n.keys) == 0 && n.kind == metrics.Counter {
			d, err := m.GetDatum()
			if err != nil {
				c.errorf(n.Pos(), "%s", err)
				return nil
			}
			// Initialize to zero at the zero time.
			datum.SetInt(d, 0, time.Unix(0, 0))
		}
		m.Hidden = n.hidden
		(*n.sym).Binding = m
		n.sym.Addr = len(c.obj.m)
		c.obj.m = append(c.obj.m, m)
		return nil

	case *condNode:
		if n.cond != nil {
			Walk(c, n.cond)
		}
		// Save PC of previous jump instruction emitted by the n.cond
		// compilation.  (See regexNode and relNode cases, which will emit a
		// jump as the last instr.)  This jump will skip over the truthNode.
		pc := len(c.obj.prog) - 1
		// Set matched flag false for children.
		c.emit(instr{setmatched, false})
		Walk(c, n.truthNode)
		// Re-set matched flag to true for rest of current block.
		c.emit(instr{setmatched, true})
		// Rewrite n.cond's jump target to jump to instruction after block.
		c.obj.prog[pc].opnd = len(c.obj.prog)
		// Now also emit the else clause, and a jump.
		if n.elseNode != nil {
			c.emit(instr{op: jmp})
			// Rewrite jump again to avoid this else-skipper just emitted.
			c.obj.prog[pc].opnd = len(c.obj.prog)
			// Now get the PC of the else-skipper just emitted.
			pc = len(c.obj.prog) - 1
			Walk(c, n.elseNode)
			// Rewrite else-skipper to the next PC.
			c.obj.prog[pc].opnd = len(c.obj.prog)
		}
		return nil

	case *regexNode:
		re, err := regexp.Compile(n.pattern)
		if err != nil {
			c.errorf(n.Pos(), "%s", err)
			return nil
		}
		c.obj.re = append(c.obj.re, re)
		// Store the location of this regular expression in the regexNode
		n.addr = len(c.obj.re) - 1
		c.emit(instr{match, n.addr})
		c.emit(instr{op: jnm})

	case *stringConstNode:
		c.obj.str = append(c.obj.str, n.text)
		c.emit(instr{str, len(c.obj.str) - 1})

	case *intConstNode:
		c.emit(instr{push, n.i})

	case *floatConstNode:
		c.emit(instr{push, n.f})

	case *idNode:
		if n.sym == nil || n.sym.Binding == nil {
			c.errorf(n.Pos(), "No metric bound to identifier %q", n.name)
			return nil
		}
		c.emit(instr{mload, n.sym.Addr})
		m := n.sym.Binding.(*metrics.Metric)
		c.emit(instr{dload, len(m.Keys)})

	case *caprefNode:
		if n.sym == nil || n.sym.Binding == nil {
			c.errorf(n.Pos(), "No regular expression bound to capref %q", n.name)
			return nil
		}
		rn := n.sym.Binding.(*regexNode)
		// rn.addr contains the index of the regular expression object,
		// which correlates to storage on the re slice
		c.emit(instr{push, rn.addr})
		// n.sym.addr is the capture group offset
		c.emit(instr{capref, n.sym.Addr})

	case *defNode:
		// Do nothing, defs are inlined.
		return nil

	case *decoNode:
		// Put the current block on the stack
		c.decos = append(c.decos, n)
		if n.def == nil {
			c.errorf(n.Pos(), "No definition found for decorator %q", n.name)
			return nil
		}
		// then iterate over the decorator's nodes
		Walk(c, n.def.block)
		c.decos = c.decos[:len(c.decos)-1]
		return nil

	case *nextNode:
		// Visit the 'next' block on the decorated block stack
		deco := c.decos[len(c.decos)-1]
		Walk(c, deco.block)
		return nil

	case *otherwiseNode:
		c.emit(instr{op: otherwise})
		c.emit(instr{op: jnm})

	case *delNode:
		Walk(c, n.n)
		// overwdrite the dload instruction
		pc := len(c.obj.prog) - 1
		c.obj.prog[pc].op = del
	}

	return c
}

var typedOperators = map[int]map[Type]opcode{
	PLUS: {Int: iadd,
		Float: fadd},
	MINUS: {Int: isub,
		Float: fsub},
	MUL: {Int: imul,
		Float: fmul},
	DIV: {Int: idiv,
		Float: fdiv},
	MOD: {Int: imod,
		Float: fmod},
	POW: {Int: ipow,
		Float: fpow},
	ASSIGN: {Int: iset,
		Float: fset},
}

func (c *codegen) VisitAfter(node astNode) {
	switch n := node.(type) {
	case *builtinNode:
		if n.args != nil {
			c.emit(instr{builtin[n.name], len(n.args.(*exprlistNode).children)})
		} else {
			c.emit(instr{op: builtin[n.name]})
		}
	case *unaryExprNode:
		switch n.op {
		case INC:
			c.emit(instr{op: inc})
		case NOT:
			c.emit(instr{op: not})
		}
	case *binaryExprNode:
		switch n.op {
		case LT:
			c.emit(instr{cmp, -1})
			c.emit(instr{op: jnm})
		case GT:
			c.emit(instr{cmp, 1})
			c.emit(instr{op: jnm})
		case LE:
			c.emit(instr{cmp, 1})
			c.emit(instr{op: jm})
		case GE:
			c.emit(instr{cmp, -1})
			c.emit(instr{op: jm})
		case EQ:
			c.emit(instr{cmp, 0})
			c.emit(instr{op: jnm})
		case NE:
			c.emit(instr{cmp, 0})
			c.emit(instr{op: jm})
		case PLUS, MINUS, MUL, DIV, MOD, POW, ASSIGN:
			switch n.Type() {
			case Int, Float:
				c.emit(instr{op: typedOperators[n.op][n.Type()]})
			default:
				c.errorf(n.Pos(), "Invalid type for binary expression: %q", n.Type())
			}
		case AND:
			c.emit(instr{op: and})
		case OR:
			c.emit(instr{op: or})
		case XOR:
			c.emit(instr{op: xor})
		case SHL:
			c.emit(instr{op: shl})
		case SHR:
			c.emit(instr{op: shr})
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/giantswarm/conair/btrfs"
	"github.com/giantswarm/conair/layer"
	"github.com/giantswarm/conair/nspawn"
	"github.com/giantswarm/conair/parser"
)

var cmdBuild = &Command{
	Name:        "build",
	Description: "Build an image",
	Summary:     "Build an image",
	Run:         runBuild,
}

func readFile(filename string) (*parser.Conairfile, error) {
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		return nil, err
	}
	return parser.Parse(filename)
}

func runBuild(args []string) (exit int) {
	if len(args) < 1 {
		fmt.Fprintln(os.Stderr, "Image name missing.")
		return 1
	}

	newImagePath := args[0]

	fs, _ := btrfs.Init(home)

	// remove existing layer
	if fs.Exists(newImagePath) {
		if err := fs.Remove(newImagePath); err != nil {
			fmt.Printf("%v", err)
			fmt.Fprintln(os.Stderr, fmt.Sprintf("Couldn't remove existing image. %v", err))
			return 1
		}
	}

	// read build file
	f, err := readFile("./Conairfile")
	if err != nil {
		f, err = readFile("./Dockerfile")
		if err != nil {
			fmt.Fprintln(os.Stderr, "Couldn't read Conairfile or Dockerfile.", err)
			return 1
		}
	}

	parentPath := f.From

	for i, snap := range f.Snapshots {
		paths := strings.Split(snap, ":")
		if len(paths) < 2 {
			fmt.Fprintln(os.Stderr, "SNAPSHOT definition is unreadable. Please use <snapshot name>:<container path> notation.")
			return 1
		}

		// check if snapshot exists - otherwise create a new subvolume
		snapshotPath := fmt.Sprintf(".cnr-snapshot-%s", paths[0])
		if !fs.Exists(snapshotPath) {
			if err := fs.Subvolume(snapshotPath); err != nil {
				fmt.Fprintln(os.Stderr, fmt.Sprintf("Couldn't create snapshot '%s'.", snapshotPath))
				return 1
			}
		}

		f.Snapshots[i] = fmt.Sprintf("%s/.cnr-snapshot-%s", home, snap)
	}

	for _, cmd := range f.Commands {
		l, err := layer.Create(fs, cmd.Verb, cmd.Payload, parentPath)
		if err != nil {
			fmt.Fprintln(os.Stderr, fmt.Sprintf("Couldn't create layer: %v.", err))
			return 1
		}
		fmt.Println(l.Hash, cmd.Verb, cmd.Payload)

		if l.Exists == true {
			parentPath = l.Path
			continue
		}

		c := nspawn.Init(l.Hash, fmt.Sprintf("%s/%s", home, l.Path))
		c.SetBinds(append(f.Binds, f.Snapshots...))

		if err := c.Build(cmd.Verb, cmd.Payload); err != nil {
			fmt.Fprintln(os.Stderr, fmt.Sprintf("Buildstep failed: %v.", err))
			if err = l.Remove(); err != nil {
				fmt.Fprintln(os.Stderr, "Couldn't remove temporary build container.", err)
			}
			return 1
		}

		parentPath = l.Path
	}
	if err = fs.Snapshot(parentPath, newImagePath, true); err != nil {
		fmt.Fprintln(os.Stderr, "Couldn't create filesystem for new image.", err)
		return 1
	}

	return 0
}
<commit_msg>make btrfs snapshot read/write<commit_after>package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/giantswarm/conair/btrfs"
	"github.com/giantswarm/conair/layer"
	"github.com/giantswarm/conair/nspawn"
	"github.com/giantswarm/conair/parser"
)

var cmdBuild = &Command{
	Name:        "build",
	Description: "Build an image",
	Summary:     "Build an image",
	Run:         runBuild,
}

func readFile(filename string) (*parser.Conairfile, error) {
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		return nil, err
	}
	return parser.Parse(filename)
}

func runBuild(args []string) (exit int) {
	if len(args) < 1 {
		fmt.Fprintln(os.Stderr, "Image name missing.")
		return 1
	}

	newImagePath := args[0]

	fs, _ := btrfs.Init(home)

	// remove existing layer
	if fs.Exists(newImagePath) {
		if err := fs.Remove(newImagePath); err != nil {
			fmt.Printf("%v", err)
			fmt.Fprintln(os.Stderr, fmt.Sprintf("Couldn't remove existing image. %v", err))
			return 1
		}
	}

	// read build file
	f, err := readFile("./Conairfile")
	if err != nil {
		f, err = readFile("./Dockerfile")
		if err != nil {
			fmt.Fprintln(os.Stderr, "Couldn't read Conairfile or Dockerfile.", err)
			return 1
		}
	}

	parentPath := f.From

	for i, snap := range f.Snapshots {
		paths := strings.Split(snap, ":")
		if len(paths) < 2 {
			fmt.Fprintln(os.Stderr, "SNAPSHOT definition is unreadable. Please use <snapshot name>:<container path> notation.")
			return 1
		}

		// check if snapshot exists - otherwise create a new subvolume
		snapshotPath := fmt.Sprintf(".cnr-snapshot-%s", paths[0])
		if !fs.Exists(snapshotPath) {
			if err := fs.Subvolume(snapshotPath); err != nil {
				fmt.Fprintln(os.Stderr, fmt.Sprintf("Couldn't create snapshot '%s'.", snapshotPath))
				return 1
			}
		}

		f.Snapshots[i] = fmt.Sprintf("%s/.cnr-snapshot-%s", home, snap)
	}

	for _, cmd := range f.Commands {
		l, err := layer.Create(fs, cmd.Verb, cmd.Payload, parentPath)
		if err != nil {
			fmt.Fprintln(os.Stderr, fmt.Sprintf("Couldn't create layer: %v.", err))
			return 1
		}
		fmt.Println(l.Hash, cmd.Verb, cmd.Payload)

		if l.Exists == true {
			parentPath = l.Path
			continue
		}

		c := nspawn.Init(l.Hash, fmt.Sprintf("%s/%s", home, l.Path))
		c.SetBinds(append(f.Binds, f.Snapshots...))

		if err := c.Build(cmd.Verb, cmd.Payload); err != nil {
			fmt.Fprintln(os.Stderr, fmt.Sprintf("Buildstep failed: %v.", err))
			if err = l.Remove(); err != nil {
				fmt.Fprintln(os.Stderr, "Couldn't remove temporary build container.", err)
			}
			return 1
		}

		parentPath = l.Path
	}
	if err = fs.Snapshot(parentPath, newImagePath, false); err != nil {
		fmt.Fprintln(os.Stderr, "Couldn't create filesystem for new image.", err)
		return 1
	}

	return 0
}
<|endoftext|>


<commit_before>// +build ignore

/*
Copyright (c) 2019 the Octant contributors. All Rights Reserved.
SPDX-License-Identifier: Apache-2.0
*/

package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"
)

var (
	VERSION    = "v0.13.1"
	GOPATH     = os.Getenv("GOPATH")
	GIT_COMMIT = gitCommit()
	BUILD_TIME = time.Now().UTC().Format(time.RFC3339)
	LD_FLAGS   = fmt.Sprintf("-X \"main.buildTime=%s\" -X main.gitCommit=%s", BUILD_TIME, GIT_COMMIT)
	GO_FLAGS   = fmt.Sprintf("-ldflags=%s", LD_FLAGS)
)

func main() {
	flag.Parse()
	for _, cmd := range flag.Args() {
		switch cmd {
		case "build-electron-dev":
			buildElectronDev()
		case "ci":
			test()
			vet()
			webDeps()
			webTest()
			webBuild()
			build()
		case "ci-quick":
			webDeps()
			webBuild()
			build()
		case "web-deps":
			webDeps()
		case "web-test":
			webDeps()
			webTest()
		case "web-build":
			webDeps()
			webBuild()
		case "web":
			webDeps()
			webTest()
			webBuild()
		case "clean":
			clean()
		case "generate":
			generate()
		case "vet":
			vet()
		case "test":
			test()
		case "build":
			build()
		case "run-dev":
			runDev()
		case "go-install":
			goInstall()
		case "serve":
			serve()
		case "install-test-plugin":
			installTestPlugin()
		case "version":
			version()
		case "release":
			release()
		case "docker":
			docker()
		default:
			log.Fatalf("Unknown command %q", cmd)
		}
	}
}

func runCmd(command string, env map[string]string, args ...string) {
	cmd := newCmd(command, env, args...)
	cmd.Stderr = os.Stderr
	log.Printf("Running: %s\n", cmd.String())
	if err := cmd.Run(); err != nil {
		log.Fatal(err)
	}
}

func runCmdIn(dir, command string, env map[string]string, args ...string) {
	cmd := newCmd(command, env, args...)
	cmd.Dir = dir
	log.Printf("Running in %s: %s\n", dir, cmd.String())
	if err := cmd.Run(); err != nil {
		log.Fatal(err)
	}
}

func newCmd(command string, env map[string]string, args ...string) *exec.Cmd {
	realCommand, err := exec.LookPath(command)
	if err != nil {
		log.Fatalf("unable to find command '%s'", command)
	}

	cmd := exec.Command(realCommand, args...)
	cmd.Stderr = os.Stderr

	cmd.Env = os.Environ()
	for k, v := range env {
		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
	}
	return cmd
}

func buildElectronDev() {
	listCmd := newCmd("go", nil,
		"list", "-m", "-f", "{{.Dir}}", "github.com/asticode/go-astilectron-bundler")

	abPath, err := listCmd.Output()
	if err != nil {
		log.Fatalf("unable to find astilectron-bundler: %w", err)
	}
	abPath = bytes.TrimSpace(abPath)

	runCmdIn(
		filepath.Join("cmd", "octant-electron"),
		"go",
		nil,
		"run",
		filepath.Join(string(abPath), "astilectron-bundler"),
	)
}

func goInstall() {
	pkgs := []string{
		"github.com/GeertJohan/go.rice",
		"github.com/GeertJohan/go.rice/rice",
		"github.com/golang/mock/gomock",
		"github.com/golang/mock/mockgen",
		"github.com/golang/protobuf/protoc-gen-go",
	}
	for _, pkg := range pkgs {
		runCmd("go", map[string]string{"GO111MODULE": "on"}, "install", pkg)
	}
}

func clean() {
	if err := os.Remove("pkg/icon/rice-box.go"); err != nil {
		log.Fatalf("clean: %s", err)
	}
}

func generate() {
	removeFakes()
	runCmd("go", nil, "generate", "-v", "./pkg/...", "./internal/...")
}

func build() {
	newPath := filepath.Join(".", "build")
	os.MkdirAll(newPath, 0755)

	artifact := "octant"
	if runtime.GOOS == "windows" {
		artifact = "octant.exe"
	}
	runCmd("go", nil, "build", "-o", "build/"+artifact, GO_FLAGS, "-v", "./cmd/octant")
}

func runDev() {
	env := make(map[string]string)
	for _, e := range os.Environ() {
		parts := strings.SplitN(e, "=", 2)
		env[parts[0]] = parts[1]
	}
	runCmd("build/octant", env)
}

func test() {
	runCmd("go", nil, "test", "-v", "./internal/...", "./pkg/...")
}

func vet() {
	runCmd("go", nil, "vet", "./internal/...", "./pkg/...")
}

func webDeps() {
	cmd := newCmd("npm", nil, "ci")
	cmd.Stdout = os.Stdout
	cmd.Dir = "./web"
	if err := cmd.Run(); err != nil {
		log.Fatalf("web-deps: %s", err)
	}
}

func webTest() {
	cmd := newCmd("npm", nil, "run", "test:headless")
	cmd.Stdout = os.Stdout
	cmd.Dir = "./web"
	if err := cmd.Run(); err != nil {
		log.Fatalf("web-test: %s", err)
	}
}

func webBuild() {
	cmd := newCmd("npm", nil, "run", "build")
	cmd.Stdout = os.Stdout
	cmd.Dir = "./web"
	if err := cmd.Run(); err != nil {
		log.Fatalf("web-build: %s", err)
	}
	runCmd("go", nil, "generate", "./web")
}

func serve() {
	var wg sync.WaitGroup

	uiVars := map[string]string{"API_BASE": "http://localhost:7777"}
	uiCmd := newCmd("npm", uiVars, "run", "start")
	uiCmd.Stdout = os.Stdout
	uiCmd.Dir = "./web"
	if err := uiCmd.Start(); err != nil {
		log.Fatalf("uiCmd: start: %s", err)
	}

	wg.Add(1)
	go func() {
		defer wg.Done()
		if err := uiCmd.Wait(); err != nil {
			log.Fatalf("serve: npm run: %s", err)
		}
	}()

	serverVars := map[string]string{
		"OCTANT_DISABLE_OPEN_BROWSER": "true",
		"OCTANT_LISTENER_ADDR":        "localhost:7777",
		"OCTANT_PROXY_FRONTEND":       "http://localhost:4200",
	}
	serverCmd := newCmd("go", serverVars, "run", "./cmd/octant/main.go")
	serverCmd.Stdout = os.Stdout
	if err := serverCmd.Start(); err != nil {
		log.Fatalf("serveCmd: start: %s", err)
	}

	wg.Add(1)
	go func() {
		defer wg.Done()
		if err := serverCmd.Wait(); err != nil {
			log.Fatalf("serve: go run: %s", err)
		}
	}()

	wg.Wait()
}

func installTestPlugin() {
	dir := pluginDir()
	log.Printf("Plugin path: %s", dir)
	os.MkdirAll(dir, 0755)
	filename := "octant-sample-plugin"
	if runtime.GOOS == "windows" {
		filename = "octant-sample-plugin.exe"
	}
	pluginFile := filepath.Join(dir, filename)
	runCmd("go", nil, "build", "-o", pluginFile, "github.com/vmware-tanzu/octant/cmd/octant-sample-plugin")
}

func version() {
	fmt.Println(VERSION)
}

func release() {
	runCmd("git", nil, "tag", "-a", VERSION, "-m", fmt.Sprintf("\"Release %s\"", VERSION))
	runCmd("git", nil, "push", "--follow-tags")
}

func docker() {
	dockerVars := map[string]string{
		"CGO_ENABLED": "0",
		"GOOS":        "linux",
		"GOARCH":      "amd64",
	}
	runCmd("go", dockerVars, "build", "-o", "octant", GO_FLAGS, "-v", "./cmd/octant/main.go")
}

func removeFakes() {
	checkDirs := []string{"pkg", "internal"}
	fakePaths := []string{}

	for _, dir := range checkDirs {
		err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
			if !info.IsDir() {
				return nil
			}
			if info.Name() == "fake" {
				fakePaths = append(fakePaths, filepath.Join(path, info.Name()))
			}
			return nil
		})
		if err != nil {
			log.Fatalf("generate (%s): %s", dir, err)
		}
	}

	log.Print("Removing fakes from pkg/ and internal/")
	for _, p := range fakePaths {
		os.RemoveAll(p)
	}
}

func gitCommit() string {
	cmd := newCmd("git", nil, "rev-parse", "--short", "HEAD")
	out, err := cmd.Output()
	if err != nil {
		log.Printf("gitCommit: %s", err)
		return ""
	}
	return fmt.Sprintf("%s", out)
}

func pluginDir() string {
	xdgConfigHome := os.Getenv("XDG_CONFIG_HOME")
	if xdgConfigHome != "" {
		return filepath.Join(xdgConfigHome, "octant", "plugins")
	} else if runtime.GOOS == "windows" {
		return filepath.Join(os.Getenv("LOCALAPPDATA"), "octant", "plugins")
	} else {
		return filepath.Join(os.Getenv("HOME"), ".config", "octant", "plugins")
	}
}
<commit_msg>Add std in/out/err to all cmds run by build.go<commit_after>// +build ignore

/*
Copyright (c) 2019 the Octant contributors. All Rights Reserved.
SPDX-License-Identifier: Apache-2.0
*/

package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"
)

var (
	VERSION    = "v0.13.1"
	GOPATH     = os.Getenv("GOPATH")
	GIT_COMMIT = gitCommit()
	BUILD_TIME = time.Now().UTC().Format(time.RFC3339)
	LD_FLAGS   = fmt.Sprintf("-X \"main.buildTime=%s\" -X main.gitCommit=%s", BUILD_TIME, GIT_COMMIT)
	GO_FLAGS   = fmt.Sprintf("-ldflags=%s", LD_FLAGS)
)

func main() {
	flag.Parse()
	for _, cmd := range flag.Args() {
		switch cmd {
		case "build-electron-dev":
			buildElectronDev()
		case "ci":
			test()
			vet()
			webDeps()
			webTest()
			webBuild()
			build()
		case "ci-quick":
			webDeps()
			webBuild()
			build()
		case "web-deps":
			webDeps()
		case "web-test":
			webDeps()
			webTest()
		case "web-build":
			webDeps()
			webBuild()
		case "web":
			webDeps()
			webTest()
			webBuild()
		case "clean":
			clean()
		case "generate":
			generate()
		case "vet":
			vet()
		case "test":
			test()
		case "build":
			build()
		case "run-dev":
			runDev()
		case "go-install":
			goInstall()
		case "serve":
			serve()
		case "install-test-plugin":
			installTestPlugin()
		case "version":
			version()
		case "release":
			release()
		case "docker":
			docker()
		default:
			log.Fatalf("Unknown command %q", cmd)
		}
	}
}

func runCmd(command string, env map[string]string, args ...string) {
	cmd := newCmd(command, env, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	log.Printf("Running: %s\n", cmd.String())
	if err := cmd.Run(); err != nil {
		log.Fatal(err)
	}
}

func runCmdIn(dir, command string, env map[string]string, args ...string) {
	cmd := newCmd(command, env, args...)
	cmd.Dir = dir
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	log.Printf("Running in %s: %s\n", dir, cmd.String())
	if err := cmd.Run(); err != nil {
		log.Fatal(err)
	}
}

func newCmd(command string, env map[string]string, args ...string) *exec.Cmd {
	realCommand, err := exec.LookPath(command)
	if err != nil {
		log.Fatalf("unable to find command '%s'", command)
	}

	cmd := exec.Command(realCommand, args...)
	cmd.Stderr = os.Stderr

	cmd.Env = os.Environ()
	for k, v := range env {
		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
	}
	return cmd
}

func buildElectronDev() {
	listCmd := newCmd("go", nil,
		"list", "-m", "-f", "{{.Dir}}", "github.com/asticode/go-astilectron-bundler")

	abPath, err := listCmd.Output()
	if err != nil {
		log.Fatalf("unable to find astilectron-bundler: %w", err)
	}
	abPath = bytes.TrimSpace(abPath)

	runCmdIn(
		filepath.Join("cmd", "octant-electron"),
		"go",
		nil,
		"run",
		filepath.Join(string(abPath), "astilectron-bundler"),
	)
}

func goInstall() {
	pkgs := []string{
		"github.com/GeertJohan/go.rice",
		"github.com/GeertJohan/go.rice/rice",
		"github.com/golang/mock/gomock",
		"github.com/golang/mock/mockgen",
		"github.com/golang/protobuf/protoc-gen-go",
	}
	for _, pkg := range pkgs {
		runCmd("go", map[string]string{"GO111MODULE": "on"}, "install", pkg)
	}
}

func clean() {
	if err := os.Remove("pkg/icon/rice-box.go"); err != nil {
		log.Fatalf("clean: %s", err)
	}
}

func generate() {
	removeFakes()
	runCmd("go", nil, "generate", "-v", "./pkg/...", "./internal/...")
}

func build() {
	newPath := filepath.Join(".", "build")
	os.MkdirAll(newPath, 0755)

	artifact := "octant"
	if runtime.GOOS == "windows" {
		artifact = "octant.exe"
	}
	runCmd("go", nil, "build", "-o", "build/"+artifact, GO_FLAGS, "-v", "./cmd/octant")
}

func runDev() {
	env := make(map[string]string)
	for _, e := range os.Environ() {
		parts := strings.SplitN(e, "=", 2)
		env[parts[0]] = parts[1]
	}
	runCmd("build/octant", env)
}

func test() {
	runCmd("go", nil, "test", "-v", "./internal/...", "./pkg/...")
}

func vet() {
	runCmd("go", nil, "vet", "./internal/...", "./pkg/...")
}

func webDeps() {
	cmd := newCmd("npm", nil, "ci")
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	cmd.Dir = "./web"
	if err := cmd.Run(); err != nil {
		log.Fatalf("web-deps: %s", err)
	}
}

func webTest() {
	cmd := newCmd("npm", nil, "run", "test:headless")
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	cmd.Dir = "./web"
	if err := cmd.Run(); err != nil {
		log.Fatalf("web-test: %s", err)
	}
}

func webBuild() {
	cmd := newCmd("npm", nil, "run", "build")
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	cmd.Dir = "./web"
	if err := cmd.Run(); err != nil {
		log.Fatalf("web-build: %s", err)
	}
	runCmd("go", nil, "generate", "./web")
}

func serve() {
	var wg sync.WaitGroup

	uiVars := map[string]string{"API_BASE": "http://localhost:7777"}
	uiCmd := newCmd("npm", uiVars, "run", "start")
	uiCmd.Stdout = os.Stdout
	uiCmd.Stderr = os.Stderr
	uiCmd.Stdin = os.Stdin
	uiCmd.Dir = "./web"
	if err := uiCmd.Start(); err != nil {
		log.Fatalf("uiCmd: start: %s", err)
	}

	wg.Add(1)
	go func() {
		defer wg.Done()
		if err := uiCmd.Wait(); err != nil {
			log.Fatalf("serve: npm run: %s", err)
		}
	}()

	serverVars := map[string]string{
		"OCTANT_DISABLE_OPEN_BROWSER": "true",
		"OCTANT_LISTENER_ADDR":        "localhost:7777",
		"OCTANT_PROXY_FRONTEND":       "http://localhost:4200",
	}
	serverCmd := newCmd("go", serverVars, "run", "./cmd/octant/main.go")
	serverCmd.Stdout = os.Stdout
	if err := serverCmd.Start(); err != nil {
		log.Fatalf("serveCmd: start: %s", err)
	}

	wg.Add(1)
	go func() {
		defer wg.Done()
		if err := serverCmd.Wait(); err != nil {
			log.Fatalf("serve: go run: %s", err)
		}
	}()

	wg.Wait()
}

func installTestPlugin() {
	dir := pluginDir()
	log.Printf("Plugin path: %s", dir)
	os.MkdirAll(dir, 0755)
	filename := "octant-sample-plugin"
	if runtime.GOOS == "windows" {
		filename = "octant-sample-plugin.exe"
	}
	pluginFile := filepath.Join(dir, filename)
	runCmd("go", nil, "build", "-o", pluginFile, "github.com/vmware-tanzu/octant/cmd/octant-sample-plugin")
}

func version() {
	fmt.Println(VERSION)
}

func release() {
	runCmd("git", nil, "tag", "-a", VERSION, "-m", fmt.Sprintf("\"Release %s\"", VERSION))
	runCmd("git", nil, "push", "--follow-tags")
}

func docker() {
	dockerVars := map[string]string{
		"CGO_ENABLED": "0",
		"GOOS":        "linux",
		"GOARCH":      "amd64",
	}
	runCmd("go", dockerVars, "build", "-o", "octant", GO_FLAGS, "-v", "./cmd/octant/main.go")
}

func removeFakes() {
	checkDirs := []string{"pkg", "internal"}
	fakePaths := []string{}

	for _, dir := range checkDirs {
		err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
			if !info.IsDir() {
				return nil
			}
			if info.Name() == "fake" {
				fakePaths = append(fakePaths, filepath.Join(path, info.Name()))
			}
			return nil
		})
		if err != nil {
			log.Fatalf("generate (%s): %s", dir, err)
		}
	}

	log.Print("Removing fakes from pkg/ and internal/")
	for _, p := range fakePaths {
		os.RemoveAll(p)
	}
}

func gitCommit() string {
	cmd := newCmd("git", nil, "rev-parse", "--short", "HEAD")
	out, err := cmd.Output()
	if err != nil {
		log.Printf("gitCommit: %s", err)
		return ""
	}
	return fmt.Sprintf("%s", out)
}

func pluginDir() string {
	xdgConfigHome := os.Getenv("XDG_CONFIG_HOME")
	if xdgConfigHome != "" {
		return filepath.Join(xdgConfigHome, "octant", "plugins")
	} else if runtime.GOOS == "windows" {
		return filepath.Join(os.Getenv("LOCALAPPDATA"), "octant", "plugins")
	} else {
		return filepath.Join(os.Getenv("HOME"), ".config", "octant", "plugins")
	}
}
<|endoftext|>


<commit_before>//+build ignore

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strings"

	"github.com/urfave/cli/v2"
)

var packages = []string{"cli", "altsrc"}

func main() {
	app := cli.NewApp()

	app.Name = "builder"
	app.Usage = "Generates a new urfave/cli build!"

	app.Commands = cli.Commands{
		{
			Name:   "vet",
			Action: VetActionFunc,
		},
		{
			Name:   "test",
			Action: TestActionFunc,
		},
		{
			Name:   "gfmrun",
			Action: GfmrunActionFunc,
		},
		{
			Name:   "toc",
			Action: TocActionFunc,
		},
	}

	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}

func runCmd(arg string, args ...string) error {
	cmd := exec.Command(arg, args...)

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}

func VetActionFunc(_ *cli.Context) error {
	return runCmd("go", "vet")
}

func TestActionFunc(c *cli.Context) error {
	for _, pkg := range packages {
		var packageName string

		if pkg == "cli" {
			packageName = "github.com/urfave/cli/v2"
		} else {
			packageName = fmt.Sprintf("github.com/urfave/cli/v2/%s", pkg)
		}

		coverProfile := fmt.Sprintf("--coverprofile=%s.coverprofile", pkg)

		err := runCmd("go", "test", "-v", coverProfile, packageName)
		if err != nil {
			return err
		}
	}

	return testCleanup()
}

func testCleanup() error {
	var out bytes.Buffer

	for _, pkg := range packages {
		file, err := os.Open(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}

		b, err := ioutil.ReadAll(file)
		if err != nil {
			return err
		}

		out.Write(b)
		err = file.Close()
		if err != nil {
			return err
		}

		err = os.Remove(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}
	}

	outFile, err := os.Create("coverage.txt")
	if err != nil {
		return err
	}

	_, err = out.WriteTo(outFile)
	if err != nil {
		return err
	}

	err = outFile.Close()
	if err != nil {
		return err
	}

	return nil
}

func GfmrunActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	file, err := os.Open(filename)
	if err != nil {
		return err
	}

	var counter int
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		if strings.Contains(scanner.Text(), "package main") {
			counter++
		}
	}

	err = file.Close()
	if err != nil {
		return err
	}

	err = scanner.Err()
	if err != nil {
		return err
	}

	return runCmd("gfmrun", "-c", fmt.Sprint(counter), "-s", filename)
}

func TocActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	err := runCmd("node_modules/.bin/markdown-toc", "-i", filename)
	if err != nil {
		return err
	}

	err = runCmd("git", "diff", "--exit-code")
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Close file'<commit_after>//+build ignore

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strings"

	"github.com/urfave/cli/v2"
)

var packages = []string{"cli", "altsrc"}

func main() {
	app := cli.NewApp()

	app.Name = "builder"
	app.Usage = "Generates a new urfave/cli build!"

	app.Commands = cli.Commands{
		{
			Name:   "vet",
			Action: VetActionFunc,
		},
		{
			Name:   "test",
			Action: TestActionFunc,
		},
		{
			Name:   "gfmrun",
			Action: GfmrunActionFunc,
		},
		{
			Name:   "toc",
			Action: TocActionFunc,
		},
	}

	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}

func runCmd(arg string, args ...string) error {
	cmd := exec.Command(arg, args...)

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}

func VetActionFunc(_ *cli.Context) error {
	return runCmd("go", "vet")
}

func TestActionFunc(c *cli.Context) error {
	for _, pkg := range packages {
		var packageName string

		if pkg == "cli" {
			packageName = "github.com/urfave/cli/v2"
		} else {
			packageName = fmt.Sprintf("github.com/urfave/cli/v2/%s", pkg)
		}

		coverProfile := fmt.Sprintf("--coverprofile=%s.coverprofile", pkg)

		err := runCmd("go", "test", "-v", coverProfile, packageName)
		if err != nil {
			return err
		}
	}

	return testCleanup()
}

func testCleanup() error {
	var out bytes.Buffer

	for _, pkg := range packages {
		file, err := os.Open(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}

		b, err := ioutil.ReadAll(file)
		if err != nil {
			return err
		}

		out.Write(b)
		err = file.Close()
		if err != nil {
			return err
		}

		err = os.Remove(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}
	}

	outFile, err := os.Create("coverage.txt")
	if err != nil {
		return err
	}

	_, err = out.WriteTo(outFile)
	if err != nil {
		return err
	}

	err = outFile.Close()
	if err != nil {
		return err
	}

	return nil
}

func GfmrunActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()

	var counter int
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		if strings.Contains(scanner.Text(), "package main") {
			counter++
		}
	}

	err = file.Close()
	if err != nil {
		return err
	}

	err = scanner.Err()
	if err != nil {
		return err
	}

	return runCmd("gfmrun", "-c", fmt.Sprint(counter), "-s", filename)
}

func TocActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	err := runCmd("node_modules/.bin/markdown-toc", "-i", filename)
	if err != nil {
		return err
	}

	err = runCmd("git", "diff", "--exit-code")
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package people

import (
	fthealth "github.com/Financial-Times/go-fthealth/v1_1"
	"github.com/Financial-Times/go-logger"
	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	"github.com/jmcvetta/neoism"
	"errors"
	"fmt"
	"strings"
)

const (
	messageServiceNotHealthy = "Service is NOT healthy"
	messageServiceHealthy    = "Service is healthy"
)

// Driver interface
type Driver interface {
	Read(id string, transactionID string) (person Person, found bool, err error)
	Healthchecks() []fthealth.Check
}

// CypherDriver struct
type CypherDriver struct {
	conn neoutils.NeoConnection
	env  string
}

//NewCypherDriver instantiate driver
func NewCypherDriver(conn neoutils.NeoConnection, env string) Driver {
	return &CypherDriver{conn, env}
}

// CheckConnectivity tests neo4j by running a simple cypher query
func (pcw *CypherDriver) CheckConnectivity() (string, error) {
	err := neoutils.Check(pcw.conn)
	if err != nil {
		return messageServiceNotHealthy, err
	}
	return messageServiceHealthy, nil
}

func (pcw *CypherDriver) Healthchecks() []fthealth.Check {
	checks := []fthealth.Check{fthealth.Check{
		Name:             "Neo4j Connectivity",
		BusinessImpact:   "Unable to retrieve People from Neo4j",
		PanicGuide:       "https://dewey.ft.com/public-people-api.html",
		Severity:         2,
		TechnicalSummary: "Cannot connect to Neo4j. If this check fails, check that the Neo4J cluster is responding.",
		Checker:          pcw.CheckConnectivity,
	},
	}
	return checks
}

type neoChangeEvent struct {
	StartedAt string
	EndedAt   string
}

type neoReadStruct struct {
	P struct {
		ID              string
		Types           []string
		DirectType      string
		PrefLabel       string
		Labels          []string
		Salutation      string
		BirthYear       int
		EmailAddress    string
		TwitterHandle   string
		FacebookProfile string
		LinkedinProfile string
		Description     string
		DescriptionXML  string
		ImageURL        string
	}
	M []struct {
		M struct {
			ID           string
			Types        []string
			DirectType   string
			PrefLabel    string
			Title        string
			ChangeEvents []neoChangeEvent
		}
		O struct {
			ID         string
			Types      []string
			DirectType string
			PrefLabel  string
			Labels     []string
		}
		R []struct {
			ID           string
			Types        []string
			DirectType   string
			PrefLabel    string
			ChangeEvents []neoChangeEvent
		}
	}
}

func (pcw CypherDriver) Read(uuid string, transactionID string) (Person, bool, error) {
	person := Person{}
	results := []struct {
		Rs []neoReadStruct
	}{}

	query := &neoism.CypherQuery{
		Statement: `
                        MATCH (identifier:UPPIdentifier{value:{uuid}})
                        MATCH (identifier)-[:IDENTIFIES]->(pp:Person)-[:EQUIVALENT_TO]->(canonical:Person)
						MATCH (canonical)<-[:EQUIVALENT_TO]-(p:Person)
                        OPTIONAL MATCH (p)<-[:HAS_MEMBER]-(m:Membership)
                        OPTIONAL MATCH (m)-[:HAS_ORGANISATION]->(o:Organisation)
                        OPTIONAL MATCH (m)-[rr:HAS_ROLE]->(r:MembershipRole)
                        WITH    canonical,
                                { id:o.uuid, types:labels(o), prefLabel:o.prefLabel} as o,
                                { id:m.uuid, types:labels(m), prefLabel:m.prefLabel, title:m.title, changeEvents:[{startedAt:m.inceptionDate}, {endedAt:m.terminationDate}] } as m ORDER BY m.terminationDateEpoch ASC, m.inceptionDateEpoch ASC, r, rr
						WITH 	m, o,canonical,
                        		{ id:r.uuid, types:labels(r), prefLabel:r.prefLabel, changeEvents:[{startedAt:rr.inceptionDate},{endedAt:rr.terminationDate}] } as r ORDER BY rr.terminationDateEpoch ASC, rr.inceptionDateEpoch ASC
                        WITH canonical, m, o, collect(r) as r ORDER BY o.uuid DESC
                        WITH canonical, collect({m:m, o:o, r:r}) as m
                        WITH m, { ID:canonical.prefUUID, types:labels(canonical), prefLabel:canonical.prefLabel, labels:canonical.aliases,
								birthYear:canonical.birthYear, salutation:canonical.salutation, emailAddress:canonical.emailAddress,
								twitterHandle:canonical.twitterHandle, facebookProfile:canonical.facebookProfile, linkedinProfile:canonical.linkedinProfile,
								imageUrl:canonical.imageUrl, Description:canonical.description, descriptionXML:canonical.descriptionXML} as p
                        RETURN collect ({p:p, m:m}) as rs
                        `,
		Parameters: neoism.Props{"uuid": uuid},
		Result:     &results,
	}

	err := pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Error("Error Querying Neo4J for a Person")
		return Person{}, true, err
	}

	if len(results) == 0 || (len(results[0].Rs) == 0 || results[0].Rs[0].P.ID == "") {
		p, f, e := pcw.ReadOldConcordanceModel(uuid, transactionID)
		return p, f, e
	} else if len(results) != 1 {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Errorf("Multiple people found with the same uuid: %s", uuid)
		return Person{}, true, err
	}

	person = neoReadStructToPerson(results[0].Rs[0], pcw.env)
	return person, true, nil
}

func (pcw CypherDriver) ReadOldConcordanceModel(uuid string, transactionID string) (person Person, found bool, err error) {
	person = Person{}
	results := []struct {
		Rs []neoReadStruct
	}{}

	query := &neoism.CypherQuery{
		Statement: `
                        MATCH (identifier:UPPIdentifier{value:{uuid}})
                        MATCH (identifier)-[:IDENTIFIES]->(p:Person)
                        OPTIONAL MATCH (p)<-[:HAS_MEMBER]-(m:Membership)
                        OPTIONAL MATCH (m)-[:HAS_ORGANISATION]->(o:Organisation)
                        OPTIONAL MATCH (m)-[rr:HAS_ROLE]->(r:MembershipRole)
                        WITH    p,
                                { id:o.uuid, types:labels(o), prefLabel:o.prefLabel} as o,
								{ id:m.uuid, types:labels(m), prefLabel:m.prefLabel, title:m.title, changeEvents:[{startedAt:m.inceptionDate}, {endedAt:m.terminationDate}] } as m ORDER BY m.terminationDateEpoch ASC, m.inceptionDateEpoch ASC, r, rr
						WITH   	p, m, o,
								{ id:r.uuid, types:labels(r), prefLabel:r.prefLabel, changeEvents:[{startedAt:rr.inceptionDate},{endedAt:rr.terminationDate}] } as r ORDER BY rr.terminationDateEpoch ASC, rr.inceptionDateEpoch ASC
                        WITH p, m, o, collect(r) as r ORDER BY o.uuid DESC
                        WITH p, collect({m:m, o:o, r:r}) as m
                        WITH m, { id:p.uuid, types:labels(p), prefLabel:p.prefLabel, labels:p.aliases,
												     birthYear:p.birthYear, salutation:p.salutation, emailAddress:p.emailAddress,
														 twitterHandle:p.twitterHandle, facebookProfile:p.facebookProfile, linkedinProfile:p.linkedinProfile,
														 imageUrl:p.imageUrl, Description:p.description, descriptionXML:p.descriptionXML} as p
                        RETURN collect ({p:p, m:m}) as rs
                        `,
		Parameters: neoism.Props{"uuid": uuid},
		Result:     &results,
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Error("Query execution failed")
		return Person{}, false, err
	} else if len(results) == 0 || len(results[0].Rs) == 0 {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Error("Person not found")
		return Person{}, false, nil
	} else if len(results) != 1 && len(results[0].Rs) != 1 {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Errorf("Multiple people found with the same uuid:%s !", uuid)
		return Person{}, true, err
	}

	person = neoReadStructToPerson(results[0].Rs[0], pcw.env)
	return person, true, nil
}

func neoReadStructToPerson(neo neoReadStruct, env string) Person {
	public := Person{}
	public.Thing = Thing{}
	public.ID = mapper.IDURL(neo.P.ID)
	public.APIURL = mapper.APIURL(neo.P.ID, neo.P.Types, env)
	public.Types = mapper.TypeURIs(neo.P.Types)
	public.DirectType = filterToMostSpecificType(neo.P.Types)
	public.PrefLabel = neo.P.PrefLabel
	if len(neo.P.Labels) > 0 {
		public.Labels = neo.P.Labels
	}
	public.BirthYear = neo.P.BirthYear
	public.Salutation = neo.P.Salutation
	public.Description = neo.P.Description
	public.DescriptionXML = neo.P.DescriptionXML
	public.EmailAddress = neo.P.EmailAddress
	public.TwitterHandle = neo.P.TwitterHandle
	public.FacebookProfile = neo.P.FacebookProfile
	public.ImageURL = neo.P.ImageURL

	if len(neo.M) > 0 {
		memberships := []Membership{}
		for _, neoMem := range neo.M {
			if neoMem.M.ID != "" && neoMem.O.ID != "" && len(neoMem.R) > 0 {
				membership := Membership{}
				membership.Title = neoMem.M.PrefLabel
				membership.Types = mapper.TypeURIs(neoMem.M.Types)
				membership.DirectType = filterToMostSpecificType(neoMem.M.Types)
				membership.Organisation = Organisation{}
				membership.Organisation.Thing = Thing{}
				membership.Organisation.ID = mapper.IDURL(neoMem.O.ID)
				membership.Organisation.APIURL = mapper.APIURL(neoMem.O.ID, neoMem.O.Types, env)
				membership.Organisation.Types = mapper.TypeURIs(neoMem.O.Types)
				membership.Organisation.DirectType = filterToMostSpecificType(neoMem.O.Types)
				membership.Organisation.PrefLabel = neoMem.O.PrefLabel
				if len(neoMem.O.Labels) > 0 {
					membership.Organisation.Labels = neoMem.O.Labels
				}
				if a, b := changeEvent(neoMem.M.ChangeEvents); a == true {
					membership.ChangeEvents = b
				}

				roles := []Role{}
				for _, neoRole := range neoMem.R {
					if neoRole.ID != "" {
						role := Role{}
						role.Thing = Thing{}
						role.ID = mapper.IDURL(neoRole.ID)
						role.APIURL = mapper.APIURL(neoRole.ID, neoRole.Types, env)
						role.Types = mapper.TypeURIs(neoRole.Types)
						role.DirectType = filterToMostSpecificType(neoRole.Types)
						role.PrefLabel = neoRole.PrefLabel
						if a, b := changeEvent(neoRole.ChangeEvents); a == true {
							role.ChangeEvents = b
						}
						roles = append(roles, role)
					}
				}
				if len(roles) > 0 {
					membership.Roles = roles
					memberships = append(memberships, membership)
				}
			}
			public.Memberships = memberships
		}
	}

	return public
}

func changeEvent(neoChgEvts []neoChangeEvent) (bool, []ChangeEvent) {
	var results []ChangeEvent

	if neoChgEvts[0].StartedAt == "" && neoChgEvts[1].EndedAt == "" {
		results = make([]ChangeEvent, 0, 0)
		return false, results
	}
	for _, neoChgEvt := range neoChgEvts {
		if neoChgEvt.StartedAt != "" {
			results = append(results, ChangeEvent{StartedAt: neoChgEvt.StartedAt})
		}
		if neoChgEvt.EndedAt != "" {
			results = append(results, ChangeEvent{EndedAt: neoChgEvt.EndedAt})
		}
	}
	return true, results
}

func filterToMostSpecificType(unfilteredTypes []string) string {
	mostSpecificType, err := mapper.MostSpecificType(unfilteredTypes)
	if err != nil {
		return ""
	}
	fullURI := mapper.TypeURIs([]string{mostSpecificType})
	return fullURI[0]
}

func handleEmptyError(e error, defaultMessage string) error {

	if e.Error() != "" {
		return e
	}

	neoError, ok := e.(neoism.NeoError)

	if !ok {
		return errors.New(defaultMessage)
	}

	if neoError.Exception != "" {
		neoError.Message = neoError.Exception
		return neoError
	}

	if neoError.Cause != nil {
		cause := fmt.Sprint(neoError.Cause)

		if cause != "" {
			neoError.Message = "Cause: " + cause
			return neoError
		}
	}

	if len(neoError.Stacktrace) > 0 {
		neoError.Message = strings.Join(neoError.Stacktrace, ", ")
		return neoError
	}

	neoError.Message = defaultMessage

	return neoError
}<commit_msg>Revert "Reverse membership order"<commit_after>package people

import (
	fthealth "github.com/Financial-Times/go-fthealth/v1_1"
	"github.com/Financial-Times/go-logger"
	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	"github.com/jmcvetta/neoism"
	"errors"
	"fmt"
	"strings"
)

const (
	messageServiceNotHealthy = "Service is NOT healthy"
	messageServiceHealthy    = "Service is healthy"
)

// Driver interface
type Driver interface {
	Read(id string, transactionID string) (person Person, found bool, err error)
	Healthchecks() []fthealth.Check
}

// CypherDriver struct
type CypherDriver struct {
	conn neoutils.NeoConnection
	env  string
}

//NewCypherDriver instantiate driver
func NewCypherDriver(conn neoutils.NeoConnection, env string) Driver {
	return &CypherDriver{conn, env}
}

// CheckConnectivity tests neo4j by running a simple cypher query
func (pcw *CypherDriver) CheckConnectivity() (string, error) {
	err := neoutils.Check(pcw.conn)
	if err != nil {
		return messageServiceNotHealthy, err
	}
	return messageServiceHealthy, nil
}

func (pcw *CypherDriver) Healthchecks() []fthealth.Check {
	checks := []fthealth.Check{fthealth.Check{
		Name:             "Neo4j Connectivity",
		BusinessImpact:   "Unable to retrieve People from Neo4j",
		PanicGuide:       "https://dewey.ft.com/public-people-api.html",
		Severity:         2,
		TechnicalSummary: "Cannot connect to Neo4j. If this check fails, check that the Neo4J cluster is responding.",
		Checker:          pcw.CheckConnectivity,
	},
	}
	return checks
}

type neoChangeEvent struct {
	StartedAt string
	EndedAt   string
}

type neoReadStruct struct {
	P struct {
		ID              string
		Types           []string
		DirectType      string
		PrefLabel       string
		Labels          []string
		Salutation      string
		BirthYear       int
		EmailAddress    string
		TwitterHandle   string
		FacebookProfile string
		LinkedinProfile string
		Description     string
		DescriptionXML  string
		ImageURL        string
	}
	M []struct {
		M struct {
			ID           string
			Types        []string
			DirectType   string
			PrefLabel    string
			Title        string
			ChangeEvents []neoChangeEvent
		}
		O struct {
			ID         string
			Types      []string
			DirectType string
			PrefLabel  string
			Labels     []string
		}
		R []struct {
			ID           string
			Types        []string
			DirectType   string
			PrefLabel    string
			ChangeEvents []neoChangeEvent
		}
	}
}

func (pcw CypherDriver) Read(uuid string, transactionID string) (Person, bool, error) {
	person := Person{}
	results := []struct {
		Rs []neoReadStruct
	}{}

	query := &neoism.CypherQuery{
		Statement: `
                        MATCH (identifier:UPPIdentifier{value:{uuid}})
                        MATCH (identifier)-[:IDENTIFIES]->(pp:Person)-[:EQUIVALENT_TO]->(canonical:Person)
						MATCH (canonical)<-[:EQUIVALENT_TO]-(p:Person)
                        OPTIONAL MATCH (p)<-[:HAS_MEMBER]-(m:Membership)
                        OPTIONAL MATCH (m)-[:HAS_ORGANISATION]->(o:Organisation)
                        OPTIONAL MATCH (m)-[rr:HAS_ROLE]->(r:MembershipRole)
                        WITH    canonical,
                                { id:o.uuid, types:labels(o), prefLabel:o.prefLabel} as o,
                                { id:m.uuid, types:labels(m), prefLabel:m.prefLabel, title:m.title, changeEvents:[{startedAt:m.inceptionDate}, {endedAt:m.terminationDate}] } as m ORDER BY m.terminationDateEpoch DESC, m.inceptionDateEpoch DESC, r, rr
						WITH 	m, o,canonical,
                        		{ id:r.uuid, types:labels(r), prefLabel:r.prefLabel, changeEvents:[{startedAt:rr.inceptionDate},{endedAt:rr.terminationDate}] } as r ORDER BY rr.terminationDateEpoch DESC, rr.inceptionDateEpoch DESC
                        WITH canonical, m, o, collect(r) as r ORDER BY o.uuid DESC
                        WITH canonical, collect({m:m, o:o, r:r}) as m
                        WITH m, { ID:canonical.prefUUID, types:labels(canonical), prefLabel:canonical.prefLabel, labels:canonical.aliases,
								birthYear:canonical.birthYear, salutation:canonical.salutation, emailAddress:canonical.emailAddress,
								twitterHandle:canonical.twitterHandle, facebookProfile:canonical.facebookProfile, linkedinProfile:canonical.linkedinProfile,
								imageUrl:canonical.imageUrl, Description:canonical.description, descriptionXML:canonical.descriptionXML} as p
                        RETURN collect ({p:p, m:m}) as rs
                        `,
		Parameters: neoism.Props{"uuid": uuid},
		Result:     &results,
	}

	err := pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Error("Error Querying Neo4J for a Person")
		return Person{}, true, err
	}

	if len(results) == 0 || (len(results[0].Rs) == 0 || results[0].Rs[0].P.ID == "") {
		p, f, e := pcw.ReadOldConcordanceModel(uuid, transactionID)
		return p, f, e
	} else if len(results) != 1 {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Errorf("Multiple people found with the same uuid: %s", uuid)
		return Person{}, true, err
	}

	person = neoReadStructToPerson(results[0].Rs[0], pcw.env)
	return person, true, nil
}

func (pcw CypherDriver) ReadOldConcordanceModel(uuid string, transactionID string) (person Person, found bool, err error) {
	person = Person{}
	results := []struct {
		Rs []neoReadStruct
	}{}

	query := &neoism.CypherQuery{
		Statement: `
                        MATCH (identifier:UPPIdentifier{value:{uuid}})
                        MATCH (identifier)-[:IDENTIFIES]->(p:Person)
                        OPTIONAL MATCH (p)<-[:HAS_MEMBER]-(m:Membership)
                        OPTIONAL MATCH (m)-[:HAS_ORGANISATION]->(o:Organisation)
                        OPTIONAL MATCH (m)-[rr:HAS_ROLE]->(r:MembershipRole)
                        WITH    p,
                                { id:o.uuid, types:labels(o), prefLabel:o.prefLabel} as o,
								{ id:m.uuid, types:labels(m), prefLabel:m.prefLabel, title:m.title, changeEvents:[{startedAt:m.inceptionDate}, {endedAt:m.terminationDate}] } as m ORDER BY m.terminationDateEpoch DESC, m.inceptionDateEpoch DESC, r, rr
						WITH   	p, m, o,
								{ id:r.uuid, types:labels(r), prefLabel:r.prefLabel, changeEvents:[{startedAt:rr.inceptionDate},{endedAt:rr.terminationDate}] } as r ORDER BY rr.terminationDateEpoch DESC, rr.inceptionDateEpoch DESC
                        WITH p, m, o, collect(r) as r ORDER BY o.uuid DESC
                        WITH p, collect({m:m, o:o, r:r}) as m
                        WITH m, { id:p.uuid, types:labels(p), prefLabel:p.prefLabel, labels:p.aliases,
												     birthYear:p.birthYear, salutation:p.salutation, emailAddress:p.emailAddress,
														 twitterHandle:p.twitterHandle, facebookProfile:p.facebookProfile, linkedinProfile:p.linkedinProfile,
														 imageUrl:p.imageUrl, Description:p.description, descriptionXML:p.descriptionXML} as p
                        RETURN collect ({p:p, m:m}) as rs
                        `,
		Parameters: neoism.Props{"uuid": uuid},
		Result:     &results,
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Error("Query execution failed")
		return Person{}, false, err
	} else if len(results) == 0 || len(results[0].Rs) == 0 {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Error("Person not found")
		return Person{}, false, nil
	} else if len(results) != 1 && len(results[0].Rs) != 1 {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Errorf("Multiple people found with the same uuid:%s !", uuid)
		return Person{}, true, err
	}

	person = neoReadStructToPerson(results[0].Rs[0], pcw.env)
	return person, true, nil
}

func neoReadStructToPerson(neo neoReadStruct, env string) Person {
	public := Person{}
	public.Thing = Thing{}
	public.ID = mapper.IDURL(neo.P.ID)
	public.APIURL = mapper.APIURL(neo.P.ID, neo.P.Types, env)
	public.Types = mapper.TypeURIs(neo.P.Types)
	public.DirectType = filterToMostSpecificType(neo.P.Types)
	public.PrefLabel = neo.P.PrefLabel
	if len(neo.P.Labels) > 0 {
		public.Labels = neo.P.Labels
	}
	public.BirthYear = neo.P.BirthYear
	public.Salutation = neo.P.Salutation
	public.Description = neo.P.Description
	public.DescriptionXML = neo.P.DescriptionXML
	public.EmailAddress = neo.P.EmailAddress
	public.TwitterHandle = neo.P.TwitterHandle
	public.FacebookProfile = neo.P.FacebookProfile
	public.ImageURL = neo.P.ImageURL

	if len(neo.M) > 0 {
		memberships := []Membership{}
		for _, neoMem := range neo.M {
			if neoMem.M.ID != "" && neoMem.O.ID != "" && len(neoMem.R) > 0 {
				membership := Membership{}
				membership.Title = neoMem.M.PrefLabel
				membership.Types = mapper.TypeURIs(neoMem.M.Types)
				membership.DirectType = filterToMostSpecificType(neoMem.M.Types)
				membership.Organisation = Organisation{}
				membership.Organisation.Thing = Thing{}
				membership.Organisation.ID = mapper.IDURL(neoMem.O.ID)
				membership.Organisation.APIURL = mapper.APIURL(neoMem.O.ID, neoMem.O.Types, env)
				membership.Organisation.Types = mapper.TypeURIs(neoMem.O.Types)
				membership.Organisation.DirectType = filterToMostSpecificType(neoMem.O.Types)
				membership.Organisation.PrefLabel = neoMem.O.PrefLabel
				if len(neoMem.O.Labels) > 0 {
					membership.Organisation.Labels = neoMem.O.Labels
				}
				if a, b := changeEvent(neoMem.M.ChangeEvents); a == true {
					membership.ChangeEvents = b
				}

				roles := []Role{}
				for _, neoRole := range neoMem.R {
					if neoRole.ID != "" {
						role := Role{}
						role.Thing = Thing{}
						role.ID = mapper.IDURL(neoRole.ID)
						role.APIURL = mapper.APIURL(neoRole.ID, neoRole.Types, env)
						role.Types = mapper.TypeURIs(neoRole.Types)
						role.DirectType = filterToMostSpecificType(neoRole.Types)
						role.PrefLabel = neoRole.PrefLabel
						if a, b := changeEvent(neoRole.ChangeEvents); a == true {
							role.ChangeEvents = b
						}
						roles = append(roles, role)
					}
				}
				if len(roles) > 0 {
					membership.Roles = roles
					memberships = append(memberships, membership)
				}
			}
			public.Memberships = memberships
		}
	}

	return public
}

func changeEvent(neoChgEvts []neoChangeEvent) (bool, []ChangeEvent) {
	var results []ChangeEvent

	if neoChgEvts[0].StartedAt == "" && neoChgEvts[1].EndedAt == "" {
		results = make([]ChangeEvent, 0, 0)
		return false, results
	}
	for _, neoChgEvt := range neoChgEvts {
		if neoChgEvt.StartedAt != "" {
			results = append(results, ChangeEvent{StartedAt: neoChgEvt.StartedAt})
		}
		if neoChgEvt.EndedAt != "" {
			results = append(results, ChangeEvent{EndedAt: neoChgEvt.EndedAt})
		}
	}
	return true, results
}

func filterToMostSpecificType(unfilteredTypes []string) string {
	mostSpecificType, err := mapper.MostSpecificType(unfilteredTypes)
	if err != nil {
		return ""
	}
	fullURI := mapper.TypeURIs([]string{mostSpecificType})
	return fullURI[0]
}

func handleEmptyError(e error, defaultMessage string) error {

	if e.Error() != "" {
		return e
	}

	neoError, ok := e.(neoism.NeoError)

	if !ok {
		return errors.New(defaultMessage)
	}

	if neoError.Exception != "" {
		neoError.Message = neoError.Exception
		return neoError
	}

	if neoError.Cause != nil {
		cause := fmt.Sprint(neoError.Cause)

		if cause != "" {
			neoError.Message = "Cause: " + cause
			return neoError
		}
	}

	if len(neoError.Stacktrace) > 0 {
		neoError.Message = strings.Join(neoError.Stacktrace, ", ")
		return neoError
	}

	neoError.Message = defaultMessage

	return neoError
}<|endoftext|>


<commit_before>package bitread_test

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"math"
	"testing"

	"github.com/markus-wa/gobitread"
)

func TestReadBit(t *testing.T) {
	b := make([]byte, 0xff)
	for n := byte(0); n < byte(len(b)); n++ {
		b[n] = n
	}

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	res := make([]byte, 8)
	var exp string
	for i := 0; i < len(b); i++ {
		for i := 0; i < 8; i++ {
			// Least significant bit first
			if br.ReadBit() {
				res[7-i] = '1'
			} else {
				res[7-i] = '0'
			}
		}

		exp = fmt.Sprintf("%b", b[i])
		// Pad cut off bits
		for len(exp) < 8 {
			exp = "0" + exp
		}
		if string(res) != exp {
			t.Fatalf("Expected %s got %s", exp, res)
		}
	}
}

func TestReadBytes(t *testing.T) {
	b := make([]byte, 1<<8)
	for n := 0; n < len(b); n++ {
		b[n] = byte(n)
	}

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	r := br.ReadBytes(len(b))

	for i := 0; i < len(b); i++ {
		if b[i] != r[i] {
			t.Fatalf("Expected %b got %b", b[i], r[i])
		}
	}
}

func TestReadBytesBuffered(t *testing.T) {
	b := make([]byte, 1<<8)
	for n := 0; n < len(b); n++ {
		b[n] = byte(n)
	}

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), len(b)<<1) // Make buffer large enough to hold all bytes

	r := br.ReadBytes(len(b))

	for i := 0; i < len(b); i++ {
		if b[i] != r[i] {
			t.Fatalf("Expected %b got %b", b[i], r[i])
		}
	}
}

func TestReadCString(t *testing.T) {
	s := "test"
	b := []byte(s)

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	r := br.ReadCString(len(s))
	if r != s {
		t.Fatalf("Expected %q got %q", s, r)
	}
}

func TestBitsToByte(t *testing.T) {
	b := make([]byte, 0x0f)
	for n := byte(0); n < byte(len(b)); n++ {
		b[n] = n
	}

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	for i := byte(0); i < byte(len(b)); i++ {
		r := br.ReadBitsToByte(4)
		if r != b[i] {
			t.Fatalf("Expected %b got %b", b[i], r)
		}
		br.ReadBitsToByte(4)
	}
}

func TestReadBits(t *testing.T) {
	nums := []uint64{0xa3dc, 0x48c1}
	b := make([]byte, len(nums)<<3)
	for i := 0; i < len(nums); i++ {
		binary.LittleEndian.PutUint64(b[i<<3:], nums[i])
	}

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	for i := 0; i < len(nums); i++ {
		r := binary.LittleEndian.Uint64(br.ReadBits(64))
		if r != nums[i] {
			t.Fatalf("Expected %d got %d", nums[0], r)
		}
	}
}

func TestReadInt(t *testing.T) {
	nums := []uint32{0, math.MaxUint32, 0x61cb83f0}
	b := make([]byte, len(nums)<<2)
	for i := 0; i < len(nums); i++ {
		binary.LittleEndian.PutUint32(b[i<<2:], nums[i])
	}

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	for i := 0; i < len(nums); i++ {
		r := br.ReadInt(32)
		if r != uint(nums[i]) {
			t.Fatalf("Expected %q got %q", nums[i], r)
		}
	}
}

func TestReadSignedInt(t *testing.T) {
	nums := []int32{math.MaxInt32, math.MinInt32, 0, 0x4ac71bf}
	b := make([]byte, len(nums)<<2)
	for i := 0; i < len(nums); i++ {
		binary.LittleEndian.PutUint32(b[i<<2:], uint32(nums[i]))
	}

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	for i := 0; i < len(nums); i++ {
		r := br.ReadSignedInt(32)
		if r != int(nums[i]) {
			t.Fatalf("Expected %q got %q", nums[i], r)
		}
	}
}

func TestPositions(t *testing.T) {
	b := []byte{0xac}

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	// Positions
	if br.LazyPosition() != 0 {
		t.Fatalf("LazyPosition is %d expected 0", br.LazyPosition())
	}

	if br.ActualPosition() != 0 {
		t.Fatalf("ActualPostition is %d expected 0", br.ActualPosition())
	}

	br.ReadBit()

	if br.LazyPosition() != 0 {
		t.Fatalf("LazyPosition is %d expected 0", br.LazyPosition())
	}

	if br.ActualPosition() != 1 {
		t.Fatalf("ActualPostition is %d expected 1", br.ActualPosition())
	}
}

// Tests Open, Close & Chunks
func TestLifecycle(t *testing.T) {
	b := []byte("a")

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	br.BeginChunk(2)

	br.ReadBit()
	if br.ChunkFinished() {
		t.Error("ChunkFinished() should have returned false but returned true")
	}

	br.ReadBit()
	if !br.ChunkFinished() {
		t.Error("ChunkFinished() should have returned true but returned false")
	}

	br.EndChunk()
	br.Close()

	// Test reopen
	br.Open(bytes.NewReader(b), 32)
	r := br.ReadBytes(len(b))
	for i := 0; i < len(b); i++ {
		if r[i] != b[i] {
			t.Fatal("Expected", r[i], "got", b[i])
		}
	}
}

// Tests seeking at EndChunk()
func TestChunkSeek(t *testing.T) {
	b := make([]byte, 0xff)
	for n := byte(0); n < byte(len(b)); n++ {
		b[n] = n
	}

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	br.BeginChunk(2)
	br.ReadBit()
	br.EndChunk()

	if br.ActualPosition() != 2 {
		t.Errorf("ActualPostition is %d expected 2", br.ActualPosition())
	}

	// Beyond buffer
	br.BeginChunk(32 << 3)
	br.EndChunk()

	// Check if lazyPos got updated when refilling the buffer
	if br.LazyPosition() != 32<<3 {
		t.Errorf("LazyPosition is %d expected %d", br.LazyPosition(), 16<<3)
	}

	if br.ActualPosition() != 2+(32<<3) {
		t.Errorf("ActualPostition is %d expected %d", br.ActualPosition(), 2+(16<<3))
	}

	// Get to an even position (in front of 18th byte)
	br.ReadBits(6)
	// Read 34th byte
	r := br.ReadSingleByte()
	if r != 33 {
		t.Fatalf("Expected 34th byte (33) got %d", r)
	}
}

func TestChunkExceeded(t *testing.T) {
	b := []byte("a")

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	br.BeginChunk(0)
	br.ReadBit()
	func() {
		defer func() {
			err := recover()
			if err == nil {
				t.Error("EndChunk() should have paniced after reading beyond chunk boundary")
			}
		}()
		br.EndChunk()
	}()
}

// TODO: The following type of tests
// Reading beyond a chunk boundary
// Not reading all the data in a chunk (seeking)
// With and without io.Seeker as underlying
// With data in sled
// Reopen after close

// TODO: Add Skip function (alias for private advance()) or make advance public? - BeginChunk followed by endchunk already does this tho - with more overhead
<commit_msg>Tests: pretty import<commit_after>package bitread_test

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"math"
	"testing"

	bitread "github.com/markus-wa/gobitread"
)

func TestReadBit(t *testing.T) {
	b := make([]byte, 0xff)
	for n := byte(0); n < byte(len(b)); n++ {
		b[n] = n
	}

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	res := make([]byte, 8)
	var exp string
	for i := 0; i < len(b); i++ {
		for i := 0; i < 8; i++ {
			// Least significant bit first
			if br.ReadBit() {
				res[7-i] = '1'
			} else {
				res[7-i] = '0'
			}
		}

		exp = fmt.Sprintf("%b", b[i])
		// Pad cut off bits
		for len(exp) < 8 {
			exp = "0" + exp
		}
		if string(res) != exp {
			t.Fatalf("Expected %s got %s", exp, res)
		}
	}
}

func TestReadBytes(t *testing.T) {
	b := make([]byte, 1<<8)
	for n := 0; n < len(b); n++ {
		b[n] = byte(n)
	}

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	r := br.ReadBytes(len(b))

	for i := 0; i < len(b); i++ {
		if b[i] != r[i] {
			t.Fatalf("Expected %b got %b", b[i], r[i])
		}
	}
}

func TestReadBytesBuffered(t *testing.T) {
	b := make([]byte, 1<<8)
	for n := 0; n < len(b); n++ {
		b[n] = byte(n)
	}

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), len(b)<<1) // Make buffer large enough to hold all bytes

	r := br.ReadBytes(len(b))

	for i := 0; i < len(b); i++ {
		if b[i] != r[i] {
			t.Fatalf("Expected %b got %b", b[i], r[i])
		}
	}
}

func TestReadCString(t *testing.T) {
	s := "test"
	b := []byte(s)

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	r := br.ReadCString(len(s))
	if r != s {
		t.Fatalf("Expected %q got %q", s, r)
	}
}

func TestBitsToByte(t *testing.T) {
	b := make([]byte, 0x0f)
	for n := byte(0); n < byte(len(b)); n++ {
		b[n] = n
	}

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	for i := byte(0); i < byte(len(b)); i++ {
		r := br.ReadBitsToByte(4)
		if r != b[i] {
			t.Fatalf("Expected %b got %b", b[i], r)
		}
		br.ReadBitsToByte(4)
	}
}

func TestReadBits(t *testing.T) {
	nums := []uint64{0xa3dc, 0x48c1}
	b := make([]byte, len(nums)<<3)
	for i := 0; i < len(nums); i++ {
		binary.LittleEndian.PutUint64(b[i<<3:], nums[i])
	}

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	for i := 0; i < len(nums); i++ {
		r := binary.LittleEndian.Uint64(br.ReadBits(64))
		if r != nums[i] {
			t.Fatalf("Expected %d got %d", nums[0], r)
		}
	}
}

func TestReadInt(t *testing.T) {
	nums := []uint32{0, math.MaxUint32, 0x61cb83f0}
	b := make([]byte, len(nums)<<2)
	for i := 0; i < len(nums); i++ {
		binary.LittleEndian.PutUint32(b[i<<2:], nums[i])
	}

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	for i := 0; i < len(nums); i++ {
		r := br.ReadInt(32)
		if r != uint(nums[i]) {
			t.Fatalf("Expected %q got %q", nums[i], r)
		}
	}
}

func TestReadSignedInt(t *testing.T) {
	nums := []int32{math.MaxInt32, math.MinInt32, 0, 0x4ac71bf}
	b := make([]byte, len(nums)<<2)
	for i := 0; i < len(nums); i++ {
		binary.LittleEndian.PutUint32(b[i<<2:], uint32(nums[i]))
	}

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	for i := 0; i < len(nums); i++ {
		r := br.ReadSignedInt(32)
		if r != int(nums[i]) {
			t.Fatalf("Expected %q got %q", nums[i], r)
		}
	}
}

func TestPositions(t *testing.T) {
	b := []byte{0xac}

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	// Positions
	if br.LazyPosition() != 0 {
		t.Fatalf("LazyPosition is %d expected 0", br.LazyPosition())
	}

	if br.ActualPosition() != 0 {
		t.Fatalf("ActualPostition is %d expected 0", br.ActualPosition())
	}

	br.ReadBit()

	if br.LazyPosition() != 0 {
		t.Fatalf("LazyPosition is %d expected 0", br.LazyPosition())
	}

	if br.ActualPosition() != 1 {
		t.Fatalf("ActualPostition is %d expected 1", br.ActualPosition())
	}
}

// Tests Open, Close & Chunks
func TestLifecycle(t *testing.T) {
	b := []byte("a")

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	br.BeginChunk(2)

	br.ReadBit()
	if br.ChunkFinished() {
		t.Error("ChunkFinished() should have returned false but returned true")
	}

	br.ReadBit()
	if !br.ChunkFinished() {
		t.Error("ChunkFinished() should have returned true but returned false")
	}

	br.EndChunk()
	br.Close()

	// Test reopen
	br.Open(bytes.NewReader(b), 32)
	r := br.ReadBytes(len(b))
	for i := 0; i < len(b); i++ {
		if r[i] != b[i] {
			t.Fatal("Expected", r[i], "got", b[i])
		}
	}
}

// Tests seeking at EndChunk()
func TestChunkSeek(t *testing.T) {
	b := make([]byte, 0xff)
	for n := byte(0); n < byte(len(b)); n++ {
		b[n] = n
	}

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	br.BeginChunk(2)
	br.ReadBit()
	br.EndChunk()

	if br.ActualPosition() != 2 {
		t.Errorf("ActualPostition is %d expected 2", br.ActualPosition())
	}

	// Beyond buffer
	br.BeginChunk(32 << 3)
	br.EndChunk()

	// Check if lazyPos got updated when refilling the buffer
	if br.LazyPosition() != 32<<3 {
		t.Errorf("LazyPosition is %d expected %d", br.LazyPosition(), 16<<3)
	}

	if br.ActualPosition() != 2+(32<<3) {
		t.Errorf("ActualPostition is %d expected %d", br.ActualPosition(), 2+(16<<3))
	}

	// Get to an even position (in front of 18th byte)
	br.ReadBits(6)
	// Read 34th byte
	r := br.ReadSingleByte()
	if r != 33 {
		t.Fatalf("Expected 34th byte (33) got %d", r)
	}
}

func TestChunkExceeded(t *testing.T) {
	b := []byte("a")

	br := new(bitread.BitReader)
	br.Open(bytes.NewReader(b), 32)

	br.BeginChunk(0)
	br.ReadBit()
	func() {
		defer func() {
			err := recover()
			if err == nil {
				t.Error("EndChunk() should have paniced after reading beyond chunk boundary")
			}
		}()
		br.EndChunk()
	}()
}

// TODO: The following type of tests
// Reading beyond a chunk boundary
// Not reading all the data in a chunk (seeking)
// With and without io.Seeker as underlying
// With data in sled
// Reopen after close

// TODO: Add Skip function (alias for private advance()) or make advance public? - BeginChunk followed by endchunk already does this tho - with more overhead
<|endoftext|>


<commit_before>package bloom

import (
	"math"
)

// EstimateParameters estimates requirements for the number of bits and hash functions,
// to use in the Bloom Filter.
// Computed from the number of items to add and the false positive rate
func EstimateParameters(numItems uint64, fpRate float64) (numBits uint64, numHash uint64) {
	l := uint64(math.Ceil(-1 * float64(numItems) * math.Log(fpRate) / math.Pow(math.Log(2), 2)))
	h := uint64(math.Ceil(math.Log(2) * float64(l) / float64(numItems)))
	return l, h
}

func GetLocations(k0 uint64, k1 uint64, numHash uint64, data []byte) []uint64 {
	// @todo
	return []uint64{}
}

// TestLocations returns true if all locations are set in the BitSet,
// false otherwise.
func TestLocations(b *BitSet, locations []uint64) bool {
	for i := 0; i < len(locations); i++ {
		if !b.Test(locations[i]) {
			return false
		}
	}
	return true
}

func SetLocations(b *BitSet, locations []uint64) *BitSet {
	for _, loc := range locations {
		b.Set(loc)
	}
	return b
}
<commit_msg>Adding siphash for bloom filter<commit_after>package bloom

import (
	"encoding/binary"
	"math"

	"github.com/dchest/siphash"
)

// EstimateParameters estimates requirements for the number of bits and hash functions,
// to use in the Bloom Filter.
// Computed from the number of items to add and the false positive rate
func EstimateParameters(numItems uint64, fpRate float64) (numBits uint64, numHash uint64) {
	l := uint64(math.Ceil(-1 * float64(numItems) * math.Log(fpRate) / math.Pow(math.Log(2), 2)))
	h := uint64(math.Ceil(math.Log(2) * float64(l) / float64(numItems)))
	return l, h
}

// GetLocations will generate a list of numHash bit locations
// using a cryptographic hash function keyed on a 16-byte key
// and the specified data
func GetLocations(key []byte, numHash uint64, data []byte) []uint64 {
	var loc uint64
	buf := make([]byte, 8)
	result := make([]uint64, 0)

	h := siphash.New(key)
	h.Write(data)
	for i := uint64(0); i < numHash; i++ {
		binary.PutUvarint(buf, i)
		h.Write(buf)
		loc = h.Sum64()
		result = append(result, loc)
	}
	return result
}

// TestLocations returns true if all locations are set in the BitSet,
// false otherwise.
func TestLocations(b *BitSet, locations []uint64) bool {
	for i := 0; i < len(locations); i++ {
		if !b.Test(locations[i]) {
			return false
		}
	}
	return true
}

// SetLocations will set all specified bits to 1 in the bitset
func SetLocations(b *BitSet, locations []uint64) *BitSet {
	for _, loc := range locations {
		b.Set(loc)
	}
	return b
}
<|endoftext|>


<commit_before>package cache

import (
	"errors"
	"sync"
	"sync/atomic"
	"time"
)

var (
	// KeyError is returned by Get if a key is not found in the cache.
	KeyError     = errors.New("Key not found.")
	ExpiredError = errors.New("Key expired.")
	// MaxSizeError is returned by Set on implementations of Interface that have max size limits.
	MaxSizeError = errors.New("Cache full.")
)

// Interface is the common cache interface for all implementations.
type Interface interface {
	// Interface implements Upstream and returns KeyError if a key is not found in cache.
	Upstream
	// Set assigns a value to a key and sets the expiration time
	Set(key, value interface{}, exp *time.Time) error
	// Evict drops the provided keys from cache (if they exist) and returns the new size of the cache.
	// Calling Evict without arguments returns the current cache size.
	Evict(keys ...interface{}) (size int)
	Metrics() Metrics
}

// Never is a helper that returns a nil expiration time.
func Never() *time.Time {
	return nil
}

func Exp(ttl time.Duration) *time.Time {
	exp := time.Now().Add(ttl)
	return &exp
}

// Cache implements Interface.
// Removal of expired items is the responsibility of the caller.
type Cache struct {
	values  map[interface{}]interface{}
	exp     map[interface{}]*time.Time
	maxsize int
	mu      sync.RWMutex
	metrics Metrics
}

// New returns a new Cache.
// size determines the maximum number of items the cache can hold.
// If set to zero or less the cache will not have a size limit.
func New(size int) *Cache {
	return &Cache{
		values:  make(map[interface{}]interface{}),
		exp:     make(map[interface{}]*time.Time),
		maxsize: size,
	}
}

func (c *Cache) init() {
	if c.values == nil {
		c.values = make(map[interface{}]interface{})
	}
	if c.exp == nil {
		c.exp = make(map[interface{}]*time.Time)
	}
}

// Set assigns a value to a key and sets the expiration time
// If the size limit is reached it returns MaxSizeError.
func (c *Cache) Set(k, v interface{}, exp *time.Time) error {
	c.mu.Lock()
	defer c.mu.Unlock()
	if _, ok := c.values[k]; !ok && c.maxsize > 0 && len(c.values) >= c.maxsize {
		return MaxSizeError
	}
	c.values[k] = v
	if exp != nil {
		c.exp[k] = exp
	}
	return nil
}

// Get returns a value assigned to a key and it's expiration time.
// If a key does not exist in cache KeyError is returned.
// If a key is expired ExpiredError is returned
func (c *Cache) Get(k interface{}) (v interface{}, exp *time.Time, err error) {
	c.mu.RLock()
	defer c.mu.RUnlock()
	var ok bool
	if v, ok = c.values[k]; ok {
		if exp = c.exp[k]; exp == nil || exp.After(time.Now()) {
			atomic.AddInt64(&c.metrics.Hit, 1)
			return
		}
		go func() {
			c.mu.Lock()
			defer c.mu.Unlock()
			delete(c.values, k)
			delete(c.exp, k)
		}()
		return nil, nil, ExpiredError
	}
	atomic.AddInt64(&c.metrics.Miss, 1)
	return nil, nil, KeyError
}

// Size returns size of all keys in cache both expired and fresh
func (c *Cache) Size() int {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return len(c.values)
}

// Trim removes all expired keys and returns a slice of removed keys
func (c *Cache) Trim(now time.Time) []interface{} {
	c.mu.Lock()
	defer c.mu.Unlock()
	expired := make([]interface{}, 0, len(c.exp))
	for k, exp := range c.exp {
		if exp != nil && exp.Before(now) {
			expired = append(expired, k)
		}
	}
	for _, k := range expired {
		delete(c.exp, k)
		delete(c.values, k)
	}
	c.metrics.Expired += int64(len(expired))
	return expired
}

func (c *Cache) evict(keys []interface{}) (n int) {
	for _, k := range keys {
		if _, ok := c.values[k]; ok {
			delete(c.values, k)
			delete(c.exp, k)
			n++
		}
	}
	return
}

// Evict removes items from the cache. It returns the new cache size.
func (c *Cache) Evict(keys ...interface{}) (size int) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.metrics.Evict += int64(c.evict(keys))
	return len(c.values)
}

type Metrics struct {
	Hit, Miss, Evict, Expired, Items int64
}

func (c *Cache) Metrics() (m Metrics) {
	c.mu.RLock()
	defer c.mu.RUnlock()
	m.Hit = atomic.LoadInt64(&c.metrics.Hit)
	m.Miss = atomic.LoadInt64(&c.metrics.Miss)
	m.Evict = c.metrics.Evict
	m.Expired = c.metrics.Expired
	m.Items = int64(len(c.values))
	return
}

type EvictionPolicy int

const (
	PolicyNone EvictionPolicy = iota
	PolicyFIFO
	PolicyLRU
	PolicyLFU
)

func NewCache(size int, policy EvictionPolicy) Interface {
	if size <= 0 {
		return New(0)
	}
	switch policy {
	case PolicyFIFO:
		return NewFIFO(size)
	case PolicyLRU:
		return NewLRU(size)
	case PolicyLFU:
		return NewLFU(size)
	default:
		return New(size)
	}
}
<commit_msg>Switch EvictionPolicy to string<commit_after>package cache

import (
	"errors"
	"sync"
	"sync/atomic"
	"time"
)

var (
	// KeyError is returned by Get if a key is not found in the cache.
	KeyError     = errors.New("Key not found.")
	ExpiredError = errors.New("Key expired.")
	// MaxSizeError is returned by Set on implementations of Interface that have max size limits.
	MaxSizeError = errors.New("Cache full.")
)

// Interface is the common cache interface for all implementations.
type Interface interface {
	// Interface implements Upstream and returns KeyError if a key is not found in cache.
	Upstream
	// Set assigns a value to a key and sets the expiration time
	Set(key, value interface{}, exp *time.Time) error
	// Evict drops the provided keys from cache (if they exist) and returns the new size of the cache.
	// Calling Evict without arguments returns the current cache size.
	Evict(keys ...interface{}) (size int)
	Metrics() Metrics
}

// Never is a helper that returns a nil expiration time.
func Never() *time.Time {
	return nil
}

func Exp(ttl time.Duration) *time.Time {
	exp := time.Now().Add(ttl)
	return &exp
}

// Cache implements Interface.
// Removal of expired items is the responsibility of the caller.
type Cache struct {
	values  map[interface{}]interface{}
	exp     map[interface{}]*time.Time
	maxsize int
	mu      sync.RWMutex
	metrics Metrics
}

// New returns a new Cache.
// size determines the maximum number of items the cache can hold.
// If set to zero or less the cache will not have a size limit.
func New(size int) *Cache {
	return &Cache{
		values:  make(map[interface{}]interface{}),
		exp:     make(map[interface{}]*time.Time),
		maxsize: size,
	}
}

func (c *Cache) init() {
	if c.values == nil {
		c.values = make(map[interface{}]interface{})
	}
	if c.exp == nil {
		c.exp = make(map[interface{}]*time.Time)
	}
}

// Set assigns a value to a key and sets the expiration time
// If the size limit is reached it returns MaxSizeError.
func (c *Cache) Set(k, v interface{}, exp *time.Time) error {
	c.mu.Lock()
	defer c.mu.Unlock()
	if _, ok := c.values[k]; !ok && c.maxsize > 0 && len(c.values) >= c.maxsize {
		return MaxSizeError
	}
	c.values[k] = v
	if exp != nil {
		c.exp[k] = exp
	}
	return nil
}

// Get returns a value assigned to a key and it's expiration time.
// If a key does not exist in cache KeyError is returned.
// If a key is expired ExpiredError is returned
func (c *Cache) Get(k interface{}) (v interface{}, exp *time.Time, err error) {
	c.mu.RLock()
	defer c.mu.RUnlock()
	var ok bool
	if v, ok = c.values[k]; ok {
		if exp = c.exp[k]; exp == nil || exp.After(time.Now()) {
			atomic.AddInt64(&c.metrics.Hit, 1)
			return
		}
		go func() {
			c.mu.Lock()
			defer c.mu.Unlock()
			delete(c.values, k)
			delete(c.exp, k)
		}()
		return nil, nil, ExpiredError
	}
	atomic.AddInt64(&c.metrics.Miss, 1)
	return nil, nil, KeyError
}

// Size returns size of all keys in cache both expired and fresh
func (c *Cache) Size() int {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return len(c.values)
}

// Trim removes all expired keys and returns a slice of removed keys
func (c *Cache) Trim(now time.Time) []interface{} {
	c.mu.Lock()
	defer c.mu.Unlock()
	expired := make([]interface{}, 0, len(c.exp))
	for k, exp := range c.exp {
		if exp != nil && exp.Before(now) {
			expired = append(expired, k)
		}
	}
	for _, k := range expired {
		delete(c.exp, k)
		delete(c.values, k)
	}
	c.metrics.Expired += int64(len(expired))
	return expired
}

func (c *Cache) evict(keys []interface{}) (n int) {
	for _, k := range keys {
		if _, ok := c.values[k]; ok {
			delete(c.values, k)
			delete(c.exp, k)
			n++
		}
	}
	return
}

// Evict removes items from the cache. It returns the new cache size.
func (c *Cache) Evict(keys ...interface{}) (size int) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.metrics.Evict += int64(c.evict(keys))
	return len(c.values)
}

type Metrics struct {
	Hit, Miss, Evict, Expired, Items int64
}

func (c *Cache) Metrics() (m Metrics) {
	c.mu.RLock()
	defer c.mu.RUnlock()
	m.Hit = atomic.LoadInt64(&c.metrics.Hit)
	m.Miss = atomic.LoadInt64(&c.metrics.Miss)
	m.Evict = c.metrics.Evict
	m.Expired = c.metrics.Expired
	m.Items = int64(len(c.values))
	return
}

type EvictionPolicy string

const (
	PolicyNone EvictionPolicy = ""
	PolicyFIFO EvictionPolicy = "FIFO"
	PolicyLRU  EvictionPolicy = "LRU"
	PolicyLFU  EvictionPolicy = "LFU"
)

func NewCache(size int, policy EvictionPolicy) Interface {
	if size <= 0 {
		return New(0)
	}
	switch policy {
	case PolicyFIFO:
		return NewFIFO(size)
	case PolicyLRU:
		return NewLRU(size)
	case PolicyLFU:
		return NewLFU(size)
	default:
		return New(size)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
)

type Cache struct {
	Cache map[string]Example `json:"Cache"`
}

var CacheFilename = "cache.bin"

func NewCache() *Cache {
	return &Cache{make(map[string]Example)}
}

func (c *Cache) Add(example Example) {
	c.Cache[example.Url] = example
}

func (c *Cache) Save(filename string) error {
	fmt.Println("Saving cache...")
	file, err := os.Create(filename)
	defer file.Close()
	if err != nil {
		return err
	}
	json, err := json.Marshal(*c)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(filename, json, 0644)
	if err != nil {
		return err
	}
	return nil
}

func LoadCache(filename string) (*Cache, error) {
	fmt.Println("Loading cache...")
	cache := NewCache()
	bytes, err := ioutil.ReadFile(filename)
	if err != nil {
		return cache, err
	}

	if err := json.Unmarshal(bytes, cache); err != nil {
		return cache, err
	}
	return cache, nil
}
<commit_msg>cacheの読み込み/保存はstderrに吐く<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
)

type Cache struct {
	Cache map[string]Example `json:"Cache"`
}

var CacheFilename = "cache.bin"

func NewCache() *Cache {
	return &Cache{make(map[string]Example)}
}

func (c *Cache) Add(example Example) {
	c.Cache[example.Url] = example
}

func (c *Cache) Save(filename string) error {
	fmt.Fprintln(os.Stderr, "Saving cache...")
	file, err := os.Create(filename)
	defer file.Close()
	if err != nil {
		return err
	}
	json, err := json.Marshal(*c)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(filename, json, 0644)
	if err != nil {
		return err
	}
	return nil
}

func LoadCache(filename string) (*Cache, error) {
	fmt.Fprintln(os.Stderr, "Loading cache...")
	cache := NewCache()
	bytes, err := ioutil.ReadFile(filename)
	if err != nil {
		return cache, err
	}

	if err := json.Unmarshal(bytes, cache); err != nil {
		return cache, err
	}
	return cache, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Tekton Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1beta1

import (
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	// TaskRunResultType default task run result value
	TaskRunResultType ResultType = "TaskRunResult"
	// PipelineResourceResultType default pipeline result value
	PipelineResourceResultType ResultType = "PipelineResourceResult"
	// InternalTektonResultType default internal tekton result value
	InternalTektonResultType ResultType = "InternalTektonResult"
	// UnknownResultType default unknown result type value
	UnknownResultType ResultType = ""
)

// +genclient
// +genclient:noStatus
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// Task represents a collection of sequential steps that are run as part of a
// Pipeline using a set of inputs and producing a set of outputs. Tasks execute
// when TaskRuns are created that provide the input parameters and resources and
// output resources the Task requires.
//
// +k8s:openapi-gen=true
type Task struct {
	metav1.TypeMeta `json:",inline"`
	// +optional
	metav1.ObjectMeta `json:"metadata"`

	// Spec holds the desired state of the Task from the client
	// +optional
	Spec TaskSpec `json:"spec"`
}

func (t *Task) TaskSpec() TaskSpec {
	return t.Spec
}

func (t *Task) TaskMetadata() metav1.ObjectMeta {
	return t.ObjectMeta
}

func (t *Task) Copy() TaskInterface {
	return t.DeepCopy()
}

// TaskSpec defines the desired state of Task.
type TaskSpec struct {
	// Resources is a list input and output resource to run the task
	// Resources are represented in TaskRuns as bindings to instances of
	// PipelineResources.
	// +optional
	Resources *TaskResources `json:"resources,omitempty"`

	// Params is a list of input parameters required to run the task. Params
	// must be supplied as inputs in TaskRuns unless they declare a default
	// value.
	// +optional
	Params []ParamSpec `json:"params,omitempty"`

	// Description is a user-facing description of the task that may be
	// used to populate a UI.
	// +optional
	Description string `json:"description,omitempty"`

	// Steps are the steps of the build; each step is run sequentially with the
	// source mounted into /workspace.
	Steps []Step `json:"steps,omitempty"`

	// Volumes is a collection of volumes that are available to mount into the
	// steps of the build.
	Volumes []corev1.Volume `json:"volumes,omitempty"`

	// StepTemplate can be used as the basis for all step containers within the
	// Task, so that the steps inherit settings on the base container.
	StepTemplate *corev1.Container `json:"stepTemplate,omitempty"`

	// Sidecars are run alongside the Task's step containers. They begin before
	// the steps start and end after the steps complete.
	Sidecars []Sidecar `json:"sidecars,omitempty"`

	// Workspaces are the volumes that this Task requires.
	Workspaces []WorkspaceDeclaration `json:"workspaces,omitempty"`

	// Results are values that this Task can output
	Results []TaskResult `json:"results,omitempty"`
}

// TaskResult used to describe the results of a task
type TaskResult struct {
	// Name the given name
	Name string `json:"name"`

	// Description is a human-readable description of the result
	// +optional
	Description string `json:"description"`
}

// Step embeds the Container type, which allows it to include fields not
// provided by Container.
type Step struct {
	corev1.Container `json:",inline"`

	// Script is the contents of an executable file to execute.
	//
	// If Script is not empty, the Step cannot have an Command or Args.
	Script string `json:"script,omitempty"`
	// Timeout is the time after which the step times out. Defaults to never.
	// Refer to Go's ParseDuration documentation for expected format: https://golang.org/pkg/time/#ParseDuration
	Timeout *metav1.Duration `json:"timeout,omitempty"`
}

// Sidecar has nearly the same data structure as Step, consisting of a Container and an optional Script, but does not have the ability to timeout.
type Sidecar struct {
	corev1.Container `json:",inline"`

	// Script is the contents of an executable file to execute.
	//
	// If Script is not empty, the Step cannot have an Command or Args.
	Script string `json:"script,omitempty"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// TaskList contains a list of Task
type TaskList struct {
	metav1.TypeMeta `json:",inline"`
	// +optional
	metav1.ListMeta `json:"metadata,omitempty"`
	Items           []Task `json:"items"`
}

// TaskRef can be used to refer to a specific instance of a task.
// Copied from CrossVersionObjectReference: https://github.com/kubernetes/kubernetes/blob/169df7434155cbbc22f1532cba8e0a9588e29ad8/pkg/apis/autoscaling/types.go#L64
type TaskRef struct {
	// Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names
	Name string `json:"name,omitempty"`
	// TaskKind indicates the kind of the task, namespaced or cluster scoped.
	Kind TaskKind `json:"kind,omitempty"`
	// API version of the referent
	// +optional
	APIVersion string `json:"apiVersion,omitempty"`
}

// Check that Pipeline may be validated and defaulted.
// TaskKind defines the type of Task used by the pipeline.
type TaskKind string

const (
	// NamespacedTaskKind indicates that the task type has a namespaced scope.
	NamespacedTaskKind TaskKind = "Task"
	// ClusterTaskKind indicates that task type has a cluster scope.
	ClusterTaskKind TaskKind = "ClusterTask"
)
<commit_msg>Fix invalid comment on Task Script 👼<commit_after>/*
Copyright 2019 The Tekton Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1beta1

import (
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	// TaskRunResultType default task run result value
	TaskRunResultType ResultType = "TaskRunResult"
	// PipelineResourceResultType default pipeline result value
	PipelineResourceResultType ResultType = "PipelineResourceResult"
	// InternalTektonResultType default internal tekton result value
	InternalTektonResultType ResultType = "InternalTektonResult"
	// UnknownResultType default unknown result type value
	UnknownResultType ResultType = ""
)

// +genclient
// +genclient:noStatus
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// Task represents a collection of sequential steps that are run as part of a
// Pipeline using a set of inputs and producing a set of outputs. Tasks execute
// when TaskRuns are created that provide the input parameters and resources and
// output resources the Task requires.
//
// +k8s:openapi-gen=true
type Task struct {
	metav1.TypeMeta `json:",inline"`
	// +optional
	metav1.ObjectMeta `json:"metadata"`

	// Spec holds the desired state of the Task from the client
	// +optional
	Spec TaskSpec `json:"spec"`
}

func (t *Task) TaskSpec() TaskSpec {
	return t.Spec
}

func (t *Task) TaskMetadata() metav1.ObjectMeta {
	return t.ObjectMeta
}

func (t *Task) Copy() TaskInterface {
	return t.DeepCopy()
}

// TaskSpec defines the desired state of Task.
type TaskSpec struct {
	// Resources is a list input and output resource to run the task
	// Resources are represented in TaskRuns as bindings to instances of
	// PipelineResources.
	// +optional
	Resources *TaskResources `json:"resources,omitempty"`

	// Params is a list of input parameters required to run the task. Params
	// must be supplied as inputs in TaskRuns unless they declare a default
	// value.
	// +optional
	Params []ParamSpec `json:"params,omitempty"`

	// Description is a user-facing description of the task that may be
	// used to populate a UI.
	// +optional
	Description string `json:"description,omitempty"`

	// Steps are the steps of the build; each step is run sequentially with the
	// source mounted into /workspace.
	Steps []Step `json:"steps,omitempty"`

	// Volumes is a collection of volumes that are available to mount into the
	// steps of the build.
	Volumes []corev1.Volume `json:"volumes,omitempty"`

	// StepTemplate can be used as the basis for all step containers within the
	// Task, so that the steps inherit settings on the base container.
	StepTemplate *corev1.Container `json:"stepTemplate,omitempty"`

	// Sidecars are run alongside the Task's step containers. They begin before
	// the steps start and end after the steps complete.
	Sidecars []Sidecar `json:"sidecars,omitempty"`

	// Workspaces are the volumes that this Task requires.
	Workspaces []WorkspaceDeclaration `json:"workspaces,omitempty"`

	// Results are values that this Task can output
	Results []TaskResult `json:"results,omitempty"`
}

// TaskResult used to describe the results of a task
type TaskResult struct {
	// Name the given name
	Name string `json:"name"`

	// Description is a human-readable description of the result
	// +optional
	Description string `json:"description"`
}

// Step embeds the Container type, which allows it to include fields not
// provided by Container.
type Step struct {
	corev1.Container `json:",inline"`

	// Script is the contents of an executable file to execute.
	//
	// If Script is not empty, the Step cannot have an Command and the Args will be passed to the Script.
	Script string `json:"script,omitempty"`
	// Timeout is the time after which the step times out. Defaults to never.
	// Refer to Go's ParseDuration documentation for expected format: https://golang.org/pkg/time/#ParseDuration
	Timeout *metav1.Duration `json:"timeout,omitempty"`
}

// Sidecar has nearly the same data structure as Step, consisting of a Container and an optional Script, but does not have the ability to timeout.
type Sidecar struct {
	corev1.Container `json:",inline"`

	// Script is the contents of an executable file to execute.
	//
	// If Script is not empty, the Step cannot have an Command or Args.
	Script string `json:"script,omitempty"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// TaskList contains a list of Task
type TaskList struct {
	metav1.TypeMeta `json:",inline"`
	// +optional
	metav1.ListMeta `json:"metadata,omitempty"`
	Items           []Task `json:"items"`
}

// TaskRef can be used to refer to a specific instance of a task.
// Copied from CrossVersionObjectReference: https://github.com/kubernetes/kubernetes/blob/169df7434155cbbc22f1532cba8e0a9588e29ad8/pkg/apis/autoscaling/types.go#L64
type TaskRef struct {
	// Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names
	Name string `json:"name,omitempty"`
	// TaskKind indicates the kind of the task, namespaced or cluster scoped.
	Kind TaskKind `json:"kind,omitempty"`
	// API version of the referent
	// +optional
	APIVersion string `json:"apiVersion,omitempty"`
}

// Check that Pipeline may be validated and defaulted.
// TaskKind defines the type of Task used by the pipeline.
type TaskKind string

const (
	// NamespacedTaskKind indicates that the task type has a namespaced scope.
	NamespacedTaskKind TaskKind = "Task"
	// ClusterTaskKind indicates that task type has a cluster scope.
	ClusterTaskKind TaskKind = "ClusterTask"
)
<|endoftext|>