<commit_before>package main

import (
	"fmt"
	"github.com/nsf/termbox-go"
)

func tbPrint(x, y int, fg, bg termbox.Attribute, msg string) {
	for _, c := range msg {
		termbox.SetCell(x, y, c, fg, bg)
		x++
	}
}

const horizontalLine = '─'

func tbHLine(y int, fg, bg termbox.Attribute) {
	for x := 0; x < currentWidth; x++ {
		termbox.SetCell(x, y, horizontalLine, fg, bg)
	}
}

var current string
var prog program

func mouse_button_num(k termbox.Key) int {
	switch k {
	case termbox.MouseLeft:
		return 0
	case termbox.MouseMiddle:
		return 1
	case termbox.MouseRight:
		return 2
	}
	return 0
}

const bgCol = termbox.ColorDefault
const fgCol = termbox.ColorDefault
const selCol = termbox.ColorWhite
const curCol = termbox.ColorCyan

//const bgCol = termbox.ColorBlack
//const fgCol = termbox.ColorWhite

var currentHeight, currentWidth int
var hexWidth, hexHeight, hexCols int
var basicWidth, basicHeight int
var mainHeight int

func resetSize(w, h int) {
	currentWidth, currentHeight = w, h
	mainHeight = h - 1

	hexCols = w / 3
	hexWidth = hexCols*3 + 1
	hexHeight = h / 2

	basicWidth = w
	basicHeight = mainHeight - hexHeight - 1

	termbox.Clear(fgCol, bgCol)
	tbPrint(0, mainHeight, termbox.ColorWhite, termbox.ColorBlue, "Instructions go here....  Press Esc to quit")
	tbHLine(hexHeight, fgCol, bgCol)
}

var hexStart, hexEnd int

func redrawHex() {
	i := hexStart
	for h := 0; h < hexHeight && i < len(prog.bytes); h++ {
		for w := 0; w < hexCols; w++ {
			if i < len(prog.bytes) {
				bti := prog.bytes[i]
				v := fmt.Sprintf("%02x", bti.v)
				switch {
				case bti.chkErr:
					tbPrint(w*3+1, h, termbox.ColorRed, bgCol, v)
				case bti.unclear:
					tbPrint(w*3+1, h, termbox.ColorYellow, bgCol, v)
				default:
					tbPrint(w*3+1, h, fgCol, bgCol, v)
				}
				i++
			} else {
				tbPrint(w*3+1, h, fgCol, bgCol, "  ")
			}

		}
	}
	hexEnd = i - 1
}

var hexSelStart, hexSelEnd int = 0, 20
var hexCursor = 10
var basicSel int = -1

func redrawSelection(visible bool) {
	cells := termbox.CellBuffer()
	bg := bgCol
	if hexEnd >= hexSelStart && hexStart <= hexSelEnd {
		if visible {
			bg = selCol
		}
		// Calc the hex number start and end offsets.
		sh := max(hexSelStart, hexStart) - hexStart
		eh := min(hexSelEnd, hexEnd) - hexStart

		// Calc the start row and column.
		sr := sh / hexCols
		sc := (sh % hexCols)

		// Calc the end row and column.
		er := eh / hexCols
		ec := sc + (eh - sh + 1) - (er-sr)*hexCols

		// Calc the start and end cell indexes.
		si := sc*3 + sr*currentWidth
		ei := min(ec*3, currentWidth-1) + er*currentWidth

		for i := si; i <= ei; i++ {
			cells[i].Bg = bg
		}
	}

	if hexEnd >= hexCursor && hexStart <= hexCursor {
		if visible {
			bg = termbox.ColorCyan
		}

		// Now do the equivalent for the hex cursor.
		ch := hexCursor - hexStart
		cr := ch / hexCols
		cc := (ch % hexCols)
		ci := cc*3 + cr*currentWidth
		cells[ci+1].Bg = bg
		cells[ci+2].Bg = bg
	}

	if basicSel >= basicStart && basicSel < basicStart+basicHeight {
		if visible {
			bg = selCol
		}
		si := ((basicSel - basicStart) + hexHeight + 1) * currentWidth
		ei := si + currentWidth
		for i := si; i < ei; i++ {
			cells[i].Bg = bg
		}
	}
}

var basicStart int

func redrawBasic() {
	i := basicStart
	for h := 0; h < basicHeight && i < len(prog.lines); h++ {
		tbPrint(1, h+hexHeight+1, fgCol, bgCol, prog.lines[i].v)
		i++
	}
}

func redrawAll() {
	termbox.SetOutputMode(termbox.Output256)
	w, h := termbox.Size()
	if currentWidth != w || currentHeight != h {
		resetSize(w, h)
	}

	redrawHex()
	redrawBasic()
	redrawSelection(true)

	termbox.Flush()

	// Size has sometimes changed already (during the Flush?).
	w, h = termbox.Size()
	if currentWidth != w || currentHeight != h {
		redrawAll()
	}
}

func moveHexCursor(newHexCur int) {
	if newHexCur >= 0 && newHexCur < len(prog.bytes) {
		redrawSelection(false)
		hexCursor = newHexCur
		if hexCursor > hexStart+(hexHeight-2)*hexCols {
			hexStart = max(0,
				min((len(prog.bytes)/hexCols-hexHeight+1)*hexCols,
					((hexCursor-(hexHeight-2)*hexCols)/hexCols)*hexCols))
			hexEnd = min(len(prog.bytes)-1, hexStart+(hexHeight-1)*hexCols)
			redrawHex()
		} else if hexCursor < hexStart+hexCols {
			hexStart = max(0, ((hexCursor/hexCols)-1)*hexCols)
			hexEnd = min(len(prog.bytes)-1, hexStart+hexHeight*hexCols)
			redrawHex()
		}

		for hexCursor > hexSelEnd {
			basicSel++
			if basicSel < len(prog.lines) {
				hexSelStart = prog.lines[basicSel].firstByte
				hexSelEnd = prog.lines[basicSel].lastByte
			} else {
				hexSelStart = prog.lines[len(prog.lines)-1].lastByte + 1
				hexSelEnd = len(prog.bytes) - 1
			}
		}

		for hexCursor < hexSelStart {
			basicSel--
			if basicSel >= 0 {
				hexSelStart = prog.lines[basicSel].firstByte
				hexSelEnd = prog.lines[basicSel].lastByte
			} else {
				hexSelStart = 0
				hexSelEnd = prog.lines[0].firstByte - 1
			}
		}

		redrawSelection(true)
		termbox.Flush()
	}
}

func displayUI(p program) {
	err := termbox.Init()
	if err != nil {
		fmt.Printf("%s**** %s ****%s", CLR_R, err, CLR_0)
	}
	defer termbox.Close()
	termbox.SetInputMode(termbox.InputEsc | termbox.InputMouse)

	prog = p
	hexSelStart = 0
	hexSelEnd = prog.lines[0].firstByte - 1

	redrawAll()

mainloop:
	for {
		switch ev := termbox.PollEvent(); ev.Type {
		case termbox.EventKey:
			tbPrint(0, currentHeight-1, fgCol, bgCol,
				fmt.Sprintf("EventKey: k: %d, c: %c, mod: %d", ev.Key, ev.Ch, ev.Mod))
			switch ev.Key {
			case termbox.KeyEsc:
				break mainloop
			case termbox.KeyArrowLeft, termbox.KeyCtrlB:
				moveHexCursor(hexCursor - 1)
			case termbox.KeyArrowRight, termbox.KeyCtrlF:
				moveHexCursor(hexCursor + 1)
			case termbox.KeyArrowUp:
				moveHexCursor(hexCursor - hexCols)
			case termbox.KeyArrowDown:
				moveHexCursor(hexCursor + hexCols)
			}
		case termbox.EventMouse:
			tbPrint(0, currentHeight-1, fgCol, bgCol,
				fmt.Sprintf("EventMouse: x: %d, y: %d, b: %d", ev.MouseX, ev.MouseY, mouse_button_num(ev.Key)))
		case termbox.EventNone:
			tbPrint(0, currentHeight-1, fgCol, bgCol, "EventNone")
		case termbox.EventResize:
			redrawAll()
		case termbox.EventError:
			panic(ev.Err)
		}
	}
}
<commit_msg>Exit cleanly if no TTY found.<commit_after>package main

import (
	"fmt"
	"github.com/nsf/termbox-go"
)

func tbPrint(x, y int, fg, bg termbox.Attribute, msg string) {
	for _, c := range msg {
		termbox.SetCell(x, y, c, fg, bg)
		x++
	}
}

const horizontalLine = '─'

func tbHLine(y int, fg, bg termbox.Attribute) {
	for x := 0; x < currentWidth; x++ {
		termbox.SetCell(x, y, horizontalLine, fg, bg)
	}
}

var current string
var prog program

func mouse_button_num(k termbox.Key) int {
	switch k {
	case termbox.MouseLeft:
		return 0
	case termbox.MouseMiddle:
		return 1
	case termbox.MouseRight:
		return 2
	}
	return 0
}

const bgCol = termbox.ColorDefault
const fgCol = termbox.ColorDefault
const selCol = termbox.ColorWhite
const curCol = termbox.ColorCyan

//const bgCol = termbox.ColorBlack
//const fgCol = termbox.ColorWhite

var currentHeight, currentWidth int
var hexWidth, hexHeight, hexCols int
var basicWidth, basicHeight int
var mainHeight int

func resetSize(w, h int) {
	currentWidth, currentHeight = w, h
	mainHeight = h - 1

	hexCols = w / 3
	hexWidth = hexCols*3 + 1
	hexHeight = h / 2

	basicWidth = w
	basicHeight = mainHeight - hexHeight - 1

	termbox.Clear(fgCol, bgCol)
	tbPrint(0, mainHeight, termbox.ColorWhite, termbox.ColorBlue, "Instructions go here....  Press Esc to quit")
	tbHLine(hexHeight, fgCol, bgCol)
}

var hexStart, hexEnd int

func redrawHex() {
	i := hexStart
	for h := 0; h < hexHeight && i < len(prog.bytes); h++ {
		for w := 0; w < hexCols; w++ {
			if i < len(prog.bytes) {
				bti := prog.bytes[i]
				v := fmt.Sprintf("%02x", bti.v)
				switch {
				case bti.chkErr:
					tbPrint(w*3+1, h, termbox.ColorRed, bgCol, v)
				case bti.unclear:
					tbPrint(w*3+1, h, termbox.ColorYellow, bgCol, v)
				default:
					tbPrint(w*3+1, h, fgCol, bgCol, v)
				}
				i++
			} else {
				tbPrint(w*3+1, h, fgCol, bgCol, "  ")
			}

		}
	}
	hexEnd = i - 1
}

var hexSelStart, hexSelEnd int = 0, 20
var hexCursor = 10
var basicSel int = -1

func redrawSelection(visible bool) {
	cells := termbox.CellBuffer()
	bg := bgCol
	if hexEnd >= hexSelStart && hexStart <= hexSelEnd {
		if visible {
			bg = selCol
		}
		// Calc the hex number start and end offsets.
		sh := max(hexSelStart, hexStart) - hexStart
		eh := min(hexSelEnd, hexEnd) - hexStart

		// Calc the start row and column.
		sr := sh / hexCols
		sc := (sh % hexCols)

		// Calc the end row and column.
		er := eh / hexCols
		ec := sc + (eh - sh + 1) - (er-sr)*hexCols

		// Calc the start and end cell indexes.
		si := sc*3 + sr*currentWidth
		ei := min(ec*3, currentWidth-1) + er*currentWidth

		for i := si; i <= ei; i++ {
			cells[i].Bg = bg
		}
	}

	if hexEnd >= hexCursor && hexStart <= hexCursor {
		if visible {
			bg = termbox.ColorCyan
		}

		// Now do the equivalent for the hex cursor.
		ch := hexCursor - hexStart
		cr := ch / hexCols
		cc := (ch % hexCols)
		ci := cc*3 + cr*currentWidth
		cells[ci+1].Bg = bg
		cells[ci+2].Bg = bg
	}

	if basicSel >= basicStart && basicSel < basicStart+basicHeight {
		if visible {
			bg = selCol
		}
		si := ((basicSel - basicStart) + hexHeight + 1) * currentWidth
		ei := si + currentWidth
		for i := si; i < ei; i++ {
			cells[i].Bg = bg
		}
	}
}

var basicStart int

func redrawBasic() {
	i := basicStart
	for h := 0; h < basicHeight && i < len(prog.lines); h++ {
		tbPrint(1, h+hexHeight+1, fgCol, bgCol, prog.lines[i].v)
		i++
	}
}

func redrawAll() {
	termbox.SetOutputMode(termbox.Output256)
	w, h := termbox.Size()
	if currentWidth != w || currentHeight != h {
		resetSize(w, h)
	}

	redrawHex()
	redrawBasic()
	redrawSelection(true)

	termbox.Flush()

	// Size has sometimes changed already (during the Flush?).
	w, h = termbox.Size()
	if currentWidth != w || currentHeight != h {
		redrawAll()
	}
}

func moveHexCursor(newHexCur int) {
	if newHexCur >= 0 && newHexCur < len(prog.bytes) {
		redrawSelection(false)
		hexCursor = newHexCur
		if hexCursor > hexStart+(hexHeight-2)*hexCols {
			hexStart = max(0,
				min((len(prog.bytes)/hexCols-hexHeight+1)*hexCols,
					((hexCursor-(hexHeight-2)*hexCols)/hexCols)*hexCols))
			hexEnd = min(len(prog.bytes)-1, hexStart+(hexHeight-1)*hexCols)
			redrawHex()
		} else if hexCursor < hexStart+hexCols {
			hexStart = max(0, ((hexCursor/hexCols)-1)*hexCols)
			hexEnd = min(len(prog.bytes)-1, hexStart+hexHeight*hexCols)
			redrawHex()
		}

		for hexCursor > hexSelEnd {
			basicSel++
			if basicSel < len(prog.lines) {
				hexSelStart = prog.lines[basicSel].firstByte
				hexSelEnd = prog.lines[basicSel].lastByte
			} else {
				hexSelStart = prog.lines[len(prog.lines)-1].lastByte + 1
				hexSelEnd = len(prog.bytes) - 1
			}
		}

		for hexCursor < hexSelStart {
			basicSel--
			if basicSel >= 0 {
				hexSelStart = prog.lines[basicSel].firstByte
				hexSelEnd = prog.lines[basicSel].lastByte
			} else {
				hexSelStart = 0
				hexSelEnd = prog.lines[0].firstByte - 1
			}
		}

		redrawSelection(true)
		termbox.Flush()
	}
}

func displayUI(p program) {
	err := termbox.Init()
	if err != nil {
		fmt.Printf("%s**** %s ****%s", CLR_R, err, CLR_0)
		return
	}
	defer termbox.Close()
	termbox.SetInputMode(termbox.InputEsc | termbox.InputMouse)

	prog = p
	hexSelStart = 0
	hexSelEnd = prog.lines[0].firstByte - 1

	redrawAll()

mainloop:
	for {
		switch ev := termbox.PollEvent(); ev.Type {
		case termbox.EventKey:
			tbPrint(0, currentHeight-1, fgCol, bgCol,
				fmt.Sprintf("EventKey: k: %d, c: %c, mod: %d", ev.Key, ev.Ch, ev.Mod))
			switch ev.Key {
			case termbox.KeyEsc:
				break mainloop
			case termbox.KeyArrowLeft, termbox.KeyCtrlB:
				moveHexCursor(hexCursor - 1)
			case termbox.KeyArrowRight, termbox.KeyCtrlF:
				moveHexCursor(hexCursor + 1)
			case termbox.KeyArrowUp:
				moveHexCursor(hexCursor - hexCols)
			case termbox.KeyArrowDown:
				moveHexCursor(hexCursor + hexCols)
			}
		case termbox.EventMouse:
			tbPrint(0, currentHeight-1, fgCol, bgCol,
				fmt.Sprintf("EventMouse: x: %d, y: %d, b: %d", ev.MouseX, ev.MouseY, mouse_button_num(ev.Key)))
		case termbox.EventNone:
			tbPrint(0, currentHeight-1, fgCol, bgCol, "EventNone")
		case termbox.EventResize:
			redrawAll()
		case termbox.EventError:
			panic(ev.Err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"text/template"
	"time"

	"github.com/eknkc/amber"
	"github.com/russross/blackfriday"
	"github.com/yosssi/gcss"
	"gopkg.in/yaml.v2"
)

const (
	ZSDIR  = ".zs"
	PUBDIR = ".pub"
)

type Vars map[string]string

// renameExt renames extension (if any) from oldext to newext
// If oldext is an empty string - extension is extracted automatically.
// If path has no extension - new extension is appended
func renameExt(path, oldext, newext string) string {
	if oldext == "" {
		oldext = filepath.Ext(path)
	}
	if oldext == "" || strings.HasSuffix(path, oldext) {
		return strings.TrimSuffix(path, oldext) + newext
	} else {
		return path
	}
}

// globals returns list of global OS environment variables that start
// with ZS_ prefix as Vars, so the values can be used inside templates
func globals() Vars {
	vars := Vars{}
	for _, e := range os.Environ() {
		pair := strings.Split(e, "=")
		if strings.HasPrefix(pair[0], "ZS_") {
			vars[strings.ToLower(pair[0][3:])] = pair[1]
		}
	}
	return vars
}

// run executes a command or a script. Vars define the command environment,
// each zs var is converted into OS environemnt variable with ZS_ prefix
// prepended.  Additional variable $ZS contains path to the zs binary. Command
// stderr is printed to zs stderr, command output is returned as a string.
func run(vars Vars, cmd string, args ...string) (string, error) {
	// First check if partial exists (.amber or .html)
	if b, err := ioutil.ReadFile(filepath.Join(ZSDIR, cmd+".amber")); err == nil {
		return string(b), nil
	}
	if b, err := ioutil.ReadFile(filepath.Join(ZSDIR, cmd+".html")); err == nil {
		return string(b), nil
	}

	var errbuf, outbuf bytes.Buffer
	c := exec.Command(cmd, args...)
	env := []string{"ZS=" + os.Args[0], "ZS_OUTDIR=" + PUBDIR}
	env = append(env, os.Environ()...)
	for k, v := range vars {
		env = append(env, "ZS_"+strings.ToUpper(k)+"="+v)
	}
	c.Env = env
	c.Stdout = &outbuf
	c.Stderr = &errbuf

	err := c.Run()

	if errbuf.Len() > 0 {
		log.Println("ERROR:", errbuf.String())
	}
	if err != nil {
		return "", err
	}
	return string(outbuf.Bytes()), nil
}

// getVars returns list of variables defined in a text file and actual file
// content following the variables declaration. Header is separated from
// content by an empty line. Header can be either YAML or JSON.
// If no empty newline is found - file is treated as content-only.
func getVars(path string, globals Vars) (Vars, string, error) {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, "", err
	}
	s := string(b)

	// Pick some default values for content-dependent variables
	v := Vars{}
	title := strings.Replace(strings.Replace(path, "_", " ", -1), "-", " ", -1)
	v["title"] = strings.ToTitle(title)
	v["description"] = ""

	// Copy globals (will override title and description for markdown layouts
	for name, value := range globals {
		v[name] = value
	}

	// Add default values extracted from file name/path
	if _, err := os.Stat(filepath.Join(ZSDIR, "layout.amber")); err == nil {
		v["layout"] = "layout.amber"
	} else {
		v["layout"] = "layout.html"
	}
	v["file"] = path
	v["url"] = path[:len(path)-len(filepath.Ext(path))] + ".html"
	v["output"] = filepath.Join(PUBDIR, v["url"])

	delim := "\n---\n"
	if sep := strings.Index(s, delim); sep == -1 {
		return v, s, nil
	} else {
		header := s[:sep]
		body := s[sep+len(delim):]

		vars := Vars{}
		if err := yaml.Unmarshal([]byte(header), &vars); err != nil {
			fmt.Println("ERROR: failed to parse header", err)
			return nil, "", err
		} else {
			for key, value := range vars {
				v[key] = value
			}
		}
		if strings.HasPrefix(v["url"], "./") {
			v["url"] = v["url"][2:]
		}
		return v, body, nil
	}
}

// Render expanding zs plugins and variables
func render(s string, vars Vars) (string, error) {
	delim_open := "{{"
	delim_close := "}}"

	out := &bytes.Buffer{}
	for {
		if from := strings.Index(s, delim_open); from == -1 {
			out.WriteString(s)
			return out.String(), nil
		} else {
			if to := strings.Index(s, delim_close); to == -1 {
				return "", fmt.Errorf("Close delim not found")
			} else {
				out.WriteString(s[:from])
				cmd := s[from+len(delim_open) : to]
				s = s[to+len(delim_close):]
				m := strings.Fields(cmd)
				if len(m) == 1 {
					if v, ok := vars[m[0]]; ok {
						out.WriteString(v)
						continue
					}
				}
				if res, err := run(vars, m[0], m[1:]...); err == nil {
					out.WriteString(res)
				} else {
					fmt.Println(err)
				}
			}
		}
	}
	return s, nil
}

// Renders markdown with the given layout into html expanding all the macros
func buildMarkdown(path string, w io.Writer, vars Vars) error {
	v, body, err := getVars(path, vars)
	if err != nil {
		return err
	}
	content, err := render(body, v)
	if err != nil {
		return err
	}
	v["content"] = string(blackfriday.MarkdownCommon([]byte(content)))
	if w == nil {
		out, err := os.Create(filepath.Join(PUBDIR, renameExt(path, "", ".html")))
		if err != nil {
			return err
		}
		defer out.Close()
		w = out
	}
	if strings.HasSuffix(v["layout"], ".amber") {
		return buildAmber(filepath.Join(ZSDIR, v["layout"]), w, v)
	} else {
		return buildHTML(filepath.Join(ZSDIR, v["layout"]), w, v)
	}
}

// Renders text file expanding all variable macros inside it
func buildHTML(path string, w io.Writer, vars Vars) error {
	v, body, err := getVars(path, vars)
	if err != nil {
		return err
	}
	if body, err = render(body, v); err != nil {
		return err
	}
	tmpl, err := template.New("").Delims("<%", "%>").Parse(body)
	if err != nil {
		return err
	}
	if w == nil {
		f, err := os.Create(filepath.Join(PUBDIR, path))
		if err != nil {
			return err
		}
		defer f.Close()
		w = f
	}
	return tmpl.Execute(w, vars)
}

// Renders .amber file into .html
func buildAmber(path string, w io.Writer, vars Vars) error {
	v, body, err := getVars(path, vars)
	if err != nil {
		return err
	}
	a := amber.New()
	if err := a.Parse(body); err != nil {
		fmt.Println(body)
		return err
	}

	t, err := a.Compile()
	if err != nil {
		return err
	}

	htmlBuf := &bytes.Buffer{}
	if err := t.Execute(htmlBuf, v); err != nil {
		return err
	}

	if body, err = render(string(htmlBuf.Bytes()), v); err != nil {
		return err
	}

	if w == nil {
		f, err := os.Create(filepath.Join(PUBDIR, renameExt(path, ".amber", ".html")))
		if err != nil {
			return err
		}
		defer f.Close()
		w = f
	}
	_, err = io.WriteString(w, body)
	return err
}

// Compiles .gcss into .css
func buildGCSS(path string, w io.Writer) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	if w == nil {
		s := strings.TrimSuffix(path, ".gcss") + ".css"
		css, err := os.Create(filepath.Join(PUBDIR, s))
		if err != nil {
			return err
		}
		defer css.Close()
		w = css
	}
	_, err = gcss.Compile(w, f)
	return err
}

// Copies file as is from path to writer
func buildRaw(path string, w io.Writer) error {
	in, err := os.Open(path)
	if err != nil {
		return err
	}
	defer in.Close()
	if w == nil {
		if out, err := os.Create(filepath.Join(PUBDIR, path)); err != nil {
			return err
		} else {
			defer out.Close()
			w = out
		}
	}
	_, err = io.Copy(w, in)
	return err
}

func build(path string, w io.Writer, vars Vars) error {
	ext := filepath.Ext(path)
	if ext == ".md" || ext == ".mkd" {
		return buildMarkdown(path, w, vars)
	} else if ext == ".html" || ext == ".xml" {
		return buildHTML(path, w, vars)
	} else if ext == ".amber" {
		return buildAmber(path, w, vars)
	} else if ext == ".gcss" {
		return buildGCSS(path, w)
	} else {
		return buildRaw(path, w)
	}
}

func buildAll(watch bool) {
	lastModified := time.Unix(0, 0)
	modified := false

	vars := globals()
	for {
		os.Mkdir(PUBDIR, 0755)
		filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
			// ignore hidden files and directories
			if filepath.Base(path)[0] == '.' || strings.HasPrefix(path, ".") {
				return nil
			}
			// inform user about fs walk errors, but continue iteration
			if err != nil {
				fmt.Println("error:", err)
				return nil
			}

			if info.IsDir() {
				os.Mkdir(filepath.Join(PUBDIR, path), 0755)
				return nil
			} else if info.ModTime().After(lastModified) {
				if !modified {
					// First file in this build cycle is about to be modified
					run(vars, "prehook")
					modified = true
				}
				log.Println("build:", path)
				return build(path, nil, vars)
			}
			return nil
		})
		if modified {
			// At least one file in this build cycle has been modified
			run(vars, "posthook")
			modified = false
		}
		if !watch {
			break
		}
		lastModified = time.Now()
		time.Sleep(1 * time.Second)
	}
}

func init() {
	// prepend .zs to $PATH, so plugins will be found before OS commands
	p := os.Getenv("PATH")
	p = ZSDIR + ":" + p
	os.Setenv("PATH", p)
}

func main() {
	if len(os.Args) == 1 {
		fmt.Println(os.Args[0], "<command> [args]")
		return
	}
	cmd := os.Args[1]
	args := os.Args[2:]
	switch cmd {
	case "build":
		if len(args) == 0 {
			buildAll(false)
		} else if len(args) == 1 {
			if err := build(args[0], os.Stdout, globals()); err != nil {
				fmt.Println("ERROR: " + err.Error())
			}
		} else {
			fmt.Println("ERROR: too many arguments")
		}
	case "watch":
		buildAll(true)
	case "var":
		if len(args) == 0 {
			fmt.Println("var: filename expected")
		} else {
			s := ""
			if vars, _, err := getVars(args[0], globals()); err != nil {
				fmt.Println("var: " + err.Error())
			} else {
				if len(args) > 1 {
					for _, a := range args[1:] {
						s = s + vars[a] + "\n"
					}
				} else {
					for k, v := range vars {
						s = s + k + ":" + v + "\n"
					}
				}
			}
			fmt.Println(strings.TrimSpace(s))
		}
	default:
		if s, err := run(globals(), cmd, args...); err != nil {
			fmt.Println(err)
		} else {
			fmt.Println(s)
		}
	}
}
<commit_msg>more obvious rules of variable overriding<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"text/template"
	"time"

	"github.com/eknkc/amber"
	"github.com/russross/blackfriday"
	"github.com/yosssi/gcss"
	"gopkg.in/yaml.v2"
)

const (
	ZSDIR  = ".zs"
	PUBDIR = ".pub"
)

type Vars map[string]string

// renameExt renames extension (if any) from oldext to newext
// If oldext is an empty string - extension is extracted automatically.
// If path has no extension - new extension is appended
func renameExt(path, oldext, newext string) string {
	if oldext == "" {
		oldext = filepath.Ext(path)
	}
	if oldext == "" || strings.HasSuffix(path, oldext) {
		return strings.TrimSuffix(path, oldext) + newext
	} else {
		return path
	}
}

// globals returns list of global OS environment variables that start
// with ZS_ prefix as Vars, so the values can be used inside templates
func globals() Vars {
	vars := Vars{}
	for _, e := range os.Environ() {
		pair := strings.Split(e, "=")
		if strings.HasPrefix(pair[0], "ZS_") {
			vars[strings.ToLower(pair[0][3:])] = pair[1]
		}
	}
	return vars
}

// run executes a command or a script. Vars define the command environment,
// each zs var is converted into OS environemnt variable with ZS_ prefix
// prepended.  Additional variable $ZS contains path to the zs binary. Command
// stderr is printed to zs stderr, command output is returned as a string.
func run(vars Vars, cmd string, args ...string) (string, error) {
	// First check if partial exists (.amber or .html)
	if b, err := ioutil.ReadFile(filepath.Join(ZSDIR, cmd+".amber")); err == nil {
		return string(b), nil
	}
	if b, err := ioutil.ReadFile(filepath.Join(ZSDIR, cmd+".html")); err == nil {
		return string(b), nil
	}

	var errbuf, outbuf bytes.Buffer
	c := exec.Command(cmd, args...)
	env := []string{"ZS=" + os.Args[0], "ZS_OUTDIR=" + PUBDIR}
	env = append(env, os.Environ()...)
	for k, v := range vars {
		env = append(env, "ZS_"+strings.ToUpper(k)+"="+v)
	}
	c.Env = env
	c.Stdout = &outbuf
	c.Stderr = &errbuf

	err := c.Run()

	if errbuf.Len() > 0 {
		log.Println("ERROR:", errbuf.String())
	}
	if err != nil {
		return "", err
	}
	return string(outbuf.Bytes()), nil
}

// getVars returns list of variables defined in a text file and actual file
// content following the variables declaration. Header is separated from
// content by an empty line. Header can be either YAML or JSON.
// If no empty newline is found - file is treated as content-only.
func getVars(path string, globals Vars) (Vars, string, error) {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, "", err
	}
	s := string(b)

	// Pick some default values for content-dependent variables
	v := Vars{}
	title := strings.Replace(strings.Replace(path, "_", " ", -1), "-", " ", -1)
	v["title"] = strings.ToTitle(title)
	v["description"] = ""
	v["file"] = path
	v["url"] = path[:len(path)-len(filepath.Ext(path))] + ".html"
	v["output"] = filepath.Join(PUBDIR, v["url"])

	// Override default values with globals
	for name, value := range globals {
		v[name] = value
	}

	// Add layout if none is specified
	if _, ok := v["layout"]; !ok {
		if _, err := os.Stat(filepath.Join(ZSDIR, "layout.amber")); err == nil {
			v["layout"] = "layout.amber"
		} else {
			v["layout"] = "layout.html"
		}
	}

	delim := "\n---\n"
	if sep := strings.Index(s, delim); sep == -1 {
		return v, s, nil
	} else {
		header := s[:sep]
		body := s[sep+len(delim):]

		vars := Vars{}
		if err := yaml.Unmarshal([]byte(header), &vars); err != nil {
			fmt.Println("ERROR: failed to parse header", err)
			return nil, "", err
		} else {
			// Override default values + globals with the ones defines in the file
			for key, value := range vars {
				v[key] = value
			}
		}
		if strings.HasPrefix(v["url"], "./") {
			v["url"] = v["url"][2:]
		}
		return v, body, nil
	}
}

// Render expanding zs plugins and variables
func render(s string, vars Vars) (string, error) {
	delim_open := "{{"
	delim_close := "}}"

	out := &bytes.Buffer{}
	for {
		if from := strings.Index(s, delim_open); from == -1 {
			out.WriteString(s)
			return out.String(), nil
		} else {
			if to := strings.Index(s, delim_close); to == -1 {
				return "", fmt.Errorf("Close delim not found")
			} else {
				out.WriteString(s[:from])
				cmd := s[from+len(delim_open) : to]
				s = s[to+len(delim_close):]
				m := strings.Fields(cmd)
				if len(m) == 1 {
					if v, ok := vars[m[0]]; ok {
						out.WriteString(v)
						continue
					}
				}
				if res, err := run(vars, m[0], m[1:]...); err == nil {
					out.WriteString(res)
				} else {
					fmt.Println(err)
				}
			}
		}
	}
	return s, nil
}

// Renders markdown with the given layout into html expanding all the macros
func buildMarkdown(path string, w io.Writer, vars Vars) error {
	v, body, err := getVars(path, vars)
	if err != nil {
		return err
	}
	content, err := render(body, v)
	if err != nil {
		return err
	}
	v["content"] = string(blackfriday.MarkdownCommon([]byte(content)))
	if w == nil {
		out, err := os.Create(filepath.Join(PUBDIR, renameExt(path, "", ".html")))
		if err != nil {
			return err
		}
		defer out.Close()
		w = out
	}
	if strings.HasSuffix(v["layout"], ".amber") {
		return buildAmber(filepath.Join(ZSDIR, v["layout"]), w, v)
	} else {
		return buildHTML(filepath.Join(ZSDIR, v["layout"]), w, v)
	}
}

// Renders text file expanding all variable macros inside it
func buildHTML(path string, w io.Writer, vars Vars) error {
	v, body, err := getVars(path, vars)
	if err != nil {
		return err
	}
	if body, err = render(body, v); err != nil {
		return err
	}
	tmpl, err := template.New("").Delims("<%", "%>").Parse(body)
	if err != nil {
		return err
	}
	if w == nil {
		f, err := os.Create(filepath.Join(PUBDIR, path))
		if err != nil {
			return err
		}
		defer f.Close()
		w = f
	}
	return tmpl.Execute(w, vars)
}

// Renders .amber file into .html
func buildAmber(path string, w io.Writer, vars Vars) error {
	v, body, err := getVars(path, vars)
	if err != nil {
		return err
	}
	a := amber.New()
	if err := a.Parse(body); err != nil {
		fmt.Println(body)
		return err
	}

	t, err := a.Compile()
	if err != nil {
		return err
	}

	htmlBuf := &bytes.Buffer{}
	if err := t.Execute(htmlBuf, v); err != nil {
		return err
	}

	if body, err = render(string(htmlBuf.Bytes()), v); err != nil {
		return err
	}

	if w == nil {
		f, err := os.Create(filepath.Join(PUBDIR, renameExt(path, ".amber", ".html")))
		if err != nil {
			return err
		}
		defer f.Close()
		w = f
	}
	_, err = io.WriteString(w, body)
	return err
}

// Compiles .gcss into .css
func buildGCSS(path string, w io.Writer) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	if w == nil {
		s := strings.TrimSuffix(path, ".gcss") + ".css"
		css, err := os.Create(filepath.Join(PUBDIR, s))
		if err != nil {
			return err
		}
		defer css.Close()
		w = css
	}
	_, err = gcss.Compile(w, f)
	return err
}

// Copies file as is from path to writer
func buildRaw(path string, w io.Writer) error {
	in, err := os.Open(path)
	if err != nil {
		return err
	}
	defer in.Close()
	if w == nil {
		if out, err := os.Create(filepath.Join(PUBDIR, path)); err != nil {
			return err
		} else {
			defer out.Close()
			w = out
		}
	}
	_, err = io.Copy(w, in)
	return err
}

func build(path string, w io.Writer, vars Vars) error {
	ext := filepath.Ext(path)
	if ext == ".md" || ext == ".mkd" {
		return buildMarkdown(path, w, vars)
	} else if ext == ".html" || ext == ".xml" {
		return buildHTML(path, w, vars)
	} else if ext == ".amber" {
		return buildAmber(path, w, vars)
	} else if ext == ".gcss" {
		return buildGCSS(path, w)
	} else {
		return buildRaw(path, w)
	}
}

func buildAll(watch bool) {
	lastModified := time.Unix(0, 0)
	modified := false

	vars := globals()
	for {
		os.Mkdir(PUBDIR, 0755)
		filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
			// ignore hidden files and directories
			if filepath.Base(path)[0] == '.' || strings.HasPrefix(path, ".") {
				return nil
			}
			// inform user about fs walk errors, but continue iteration
			if err != nil {
				fmt.Println("error:", err)
				return nil
			}

			if info.IsDir() {
				os.Mkdir(filepath.Join(PUBDIR, path), 0755)
				return nil
			} else if info.ModTime().After(lastModified) {
				if !modified {
					// First file in this build cycle is about to be modified
					run(vars, "prehook")
					modified = true
				}
				log.Println("build:", path)
				return build(path, nil, vars)
			}
			return nil
		})
		if modified {
			// At least one file in this build cycle has been modified
			run(vars, "posthook")
			modified = false
		}
		if !watch {
			break
		}
		lastModified = time.Now()
		time.Sleep(1 * time.Second)
	}
}

func init() {
	// prepend .zs to $PATH, so plugins will be found before OS commands
	p := os.Getenv("PATH")
	p = ZSDIR + ":" + p
	os.Setenv("PATH", p)
}

func main() {
	if len(os.Args) == 1 {
		fmt.Println(os.Args[0], "<command> [args]")
		return
	}
	cmd := os.Args[1]
	args := os.Args[2:]
	switch cmd {
	case "build":
		if len(args) == 0 {
			buildAll(false)
		} else if len(args) == 1 {
			if err := build(args[0], os.Stdout, globals()); err != nil {
				fmt.Println("ERROR: " + err.Error())
			}
		} else {
			fmt.Println("ERROR: too many arguments")
		}
	case "watch":
		buildAll(true)
	case "var":
		if len(args) == 0 {
			fmt.Println("var: filename expected")
		} else {
			s := ""
			if vars, _, err := getVars(args[0], Vars{}); err != nil {
				fmt.Println("var: " + err.Error())
			} else {
				if len(args) > 1 {
					for _, a := range args[1:] {
						s = s + vars[a] + "\n"
					}
				} else {
					for k, v := range vars {
						s = s + k + ":" + v + "\n"
					}
				}
			}
			fmt.Println(strings.TrimSpace(s))
		}
	default:
		if s, err := run(globals(), cmd, args...); err != nil {
			fmt.Println(err)
		} else {
			fmt.Println(s)
		}
	}
}
<|endoftext|>


<commit_before>package tasklistprovider

import (
	"time"

	swarmingv1 "go.chromium.org/luci/common/api/swarming/swarming/v1"
	"go.skia.org/infra/go/swarming"
)

// Provider queries the Swarming API bots that have failed in the given time frame.
type Provider struct {
	client swarming.ApiClient
}

func New(client swarming.ApiClient) *Provider {
	return &Provider{
		client: client,
	}
}

// Get returns all the tasks in the Skia pool that have failed that were started in the last time duration.
//
// Since searches on Swarming can only be done on task "start" time, the value
// of 'since' should be larger than the longest task that takes to complete.
func (p *Provider) Get(since time.Duration) ([]*swarmingv1.SwarmingRpcsTaskRequestMetadata, error) {
	return p.client.ListTasks(time.Now().Add(-1*since), time.Now(), []string{"pool:Skia"}, "completed_failure")
}
<commit_msg>[predict] Fix API call.<commit_after>package tasklistprovider

import (
	"time"

	swarmingv1 "go.chromium.org/luci/common/api/swarming/swarming/v1"
	"go.skia.org/infra/go/swarming"
)

// Provider queries the Swarming API bots that have failed in the given time frame.
type Provider struct {
	client swarming.ApiClient
}

func New(client swarming.ApiClient) *Provider {
	return &Provider{
		client: client,
	}
}

// Get returns all the tasks in the Skia pool that have failed that were started in the last time duration.
//
// Since searches on Swarming can only be done on task "start" time, the value
// of 'since' should be larger than the longest task that takes to complete.
func (p *Provider) Get(since time.Duration) ([]*swarmingv1.SwarmingRpcsTaskRequestMetadata, error) {
	return p.client.ListTasks(time.Now().Add(-1*since), time.Now(), []string{"pool:Skia"}, "COMPLETED_FAILURE")
}
<|endoftext|>


<commit_before>package throttler

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"runtime"
)

const (
	linux           = "linux"
	darwin          = "darwin"
	freebsd         = "freebsd"
	windows         = "windows"
	checkOSXVersion = "sw_vers -productVersion"
	ipfw            = "ipfw"
	pfctl           = "pfctl"
)

// Config specifies options for configuring packet filter rules.
type Config struct {
	Device           string
	Stop             bool
	Latency          int
	TargetBandwidth  int
	DefaultBandwidth int
	PacketLoss       float64
	TargetIps        []string
	TargetIps6       []string
	TargetPorts      []string
	TargetProtos     []string
	DryRun           bool
}

type throttler interface {
	setup(*Config) error
	teardown(*Config) error
	exists() bool
	check() string
}

type commander interface {
	execute(string) error
	executeGetLines(string) ([]string, error)
	commandExists(string) bool
}

type dryRunCommander struct{}

type shellCommander struct{}

var dry bool

func setup(t throttler, cfg *Config) {
	if t.exists() {
		fmt.Println("It looks like the packet rules are already setup")
		os.Exit(1)
	}

	if err := t.setup(cfg); err != nil {
		fmt.Println("I couldn't setup the packet rules: %s", err.Error())
		os.Exit(1)
	}

	fmt.Println("Packet rules setup...")
	fmt.Printf("Run `%s` to double check\n", t.check())
	fmt.Printf("Run `%s --stop` to reset\n", os.Args[0])
}

func teardown(t throttler, cfg *Config) {
	if !t.exists() {
		fmt.Println("It looks like the packet rules aren't setup")
		os.Exit(1)
	}

	if err := t.teardown(cfg); err != nil {
		fmt.Println("Failed to stop packet controls")
		os.Exit(1)
	}

	fmt.Println("Packet rules stopped...")
	fmt.Printf("Run `%s` to double check\n", t.check())
	fmt.Printf("Run `%s` to start\n", os.Args[0])
}

// Run executes the packet filter operation, either setting it up or tearing
// it down.
func Run(cfg *Config) {
	dry = cfg.DryRun
	var t throttler
	var c commander

	if cfg.DryRun {
		c = &dryRunCommander{}
	} else {
		c = &shellCommander{}
	}

	switch runtime.GOOS {
	case freebsd:
		if cfg.Device == "" {
			fmt.Println("Device not specified, unable to default to eth0 on FreeBSD.")
			os.Exit(1)
		}

		t = &ipfwThrottler{c}
	case darwin:
		// Avoid OS version pinning and choose based on what's available
		if c.commandExists(pfctl) {
			t = &pfctlThrottler{c}
		} else if c.commandExists(ipfw) {
			t = &ipfwThrottler{c}
		} else {
			fmt.Println("Could not determine an appropriate firewall tool for OSX (tried pfctl, ipfw), exiting")
			os.Exit(1)
		}

		if cfg.Device == "" {
			cfg.Device = "eth0"
		}

	case linux:
		if cfg.Device == "" {
			cfg.Device = "eth0"
		}

		t = &tcThrottler{c}
	default:
		fmt.Println("I don't support your OS: %s\n", runtime.GOOS)
		os.Exit(1)
	}

	if !cfg.Stop {
		setup(t, cfg)
	} else {
		teardown(t, cfg)
	}
}

func (c *dryRunCommander) execute(cmd string) error {
	fmt.Println(cmd)
	return nil
}

func (c *dryRunCommander) executeGetLines(cmd string) ([]string, error) {
	fmt.Println(cmd)
	return []string{}, nil
}

func (c *dryRunCommander) commandExists(cmd string) bool {
	return true
}

func (c *shellCommander) execute(cmd string) error {
	fmt.Println(cmd)
	return exec.Command("/bin/sh", "-c", cmd).Run()
}

func (c *shellCommander) executeGetLines(cmd string) ([]string, error) {
	lines := []string{}
	child := exec.Command("/bin/sh", "-c", cmd)

	out, err := child.StdoutPipe()
	if err != nil {
		return []string{}, err
	}

	err = child.Start()
	if err != nil {
		return []string{}, err
	}

	scanner := bufio.NewScanner(out)

	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		return []string{}, errors.New(fmt.Sprint("Error reading standard input:", err))
	}

	err = child.Wait()
	if err != nil {
		return []string{}, err
	}

	return lines, nil
}

func (c *shellCommander) commandExists(cmd string) bool {
	_, err := exec.LookPath(cmd)
	return err == nil
}
<commit_msg>Fix error messaging<commit_after>package throttler

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"runtime"
)

const (
	linux           = "linux"
	darwin          = "darwin"
	freebsd         = "freebsd"
	windows         = "windows"
	checkOSXVersion = "sw_vers -productVersion"
	ipfw            = "ipfw"
	pfctl           = "pfctl"
)

// Config specifies options for configuring packet filter rules.
type Config struct {
	Device           string
	Stop             bool
	Latency          int
	TargetBandwidth  int
	DefaultBandwidth int
	PacketLoss       float64
	TargetIps        []string
	TargetIps6       []string
	TargetPorts      []string
	TargetProtos     []string
	DryRun           bool
}

type throttler interface {
	setup(*Config) error
	teardown(*Config) error
	exists() bool
	check() string
}

type commander interface {
	execute(string) error
	executeGetLines(string) ([]string, error)
	commandExists(string) bool
}

type dryRunCommander struct{}

type shellCommander struct{}

var dry bool

func setup(t throttler, cfg *Config) {
	if t.exists() {
		fmt.Println("It looks like the packet rules are already setup")
		os.Exit(1)
	}

	if err := t.setup(cfg); err != nil {
		fmt.Println("I couldn't setup the packet rules:", err.Error())
		os.Exit(1)
	}

	fmt.Println("Packet rules setup...")
	fmt.Printf("Run `%s` to double check\n", t.check())
	fmt.Printf("Run `%s --stop` to reset\n", os.Args[0])
}

func teardown(t throttler, cfg *Config) {
	if !t.exists() {
		fmt.Println("It looks like the packet rules aren't setup")
		os.Exit(1)
	}

	if err := t.teardown(cfg); err != nil {
		fmt.Println("Failed to stop packet controls")
		os.Exit(1)
	}

	fmt.Println("Packet rules stopped...")
	fmt.Printf("Run `%s` to double check\n", t.check())
	fmt.Printf("Run `%s` to start\n", os.Args[0])
}

// Run executes the packet filter operation, either setting it up or tearing
// it down.
func Run(cfg *Config) {
	dry = cfg.DryRun
	var t throttler
	var c commander

	if cfg.DryRun {
		c = &dryRunCommander{}
	} else {
		c = &shellCommander{}
	}

	switch runtime.GOOS {
	case freebsd:
		if cfg.Device == "" {
			fmt.Println("Device not specified, unable to default to eth0 on FreeBSD.")
			os.Exit(1)
		}

		t = &ipfwThrottler{c}
	case darwin:
		// Avoid OS version pinning and choose based on what's available
		if c.commandExists(pfctl) {
			t = &pfctlThrottler{c}
		} else if c.commandExists(ipfw) {
			t = &ipfwThrottler{c}
		} else {
			fmt.Println("Could not determine an appropriate firewall tool for OSX (tried pfctl, ipfw), exiting")
			os.Exit(1)
		}

		if cfg.Device == "" {
			cfg.Device = "eth0"
		}

	case linux:
		if cfg.Device == "" {
			cfg.Device = "eth0"
		}

		t = &tcThrottler{c}
	default:
		fmt.Println("I don't support your OS: %s\n", runtime.GOOS)
		os.Exit(1)
	}

	if !cfg.Stop {
		setup(t, cfg)
	} else {
		teardown(t, cfg)
	}
}

func (c *dryRunCommander) execute(cmd string) error {
	fmt.Println(cmd)
	return nil
}

func (c *dryRunCommander) executeGetLines(cmd string) ([]string, error) {
	fmt.Println(cmd)
	return []string{}, nil
}

func (c *dryRunCommander) commandExists(cmd string) bool {
	return true
}

func (c *shellCommander) execute(cmd string) error {
	fmt.Println(cmd)
	return exec.Command("/bin/sh", "-c", cmd).Run()
}

func (c *shellCommander) executeGetLines(cmd string) ([]string, error) {
	lines := []string{}
	child := exec.Command("/bin/sh", "-c", cmd)

	out, err := child.StdoutPipe()
	if err != nil {
		return []string{}, err
	}

	err = child.Start()
	if err != nil {
		return []string{}, err
	}

	scanner := bufio.NewScanner(out)

	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		return []string{}, errors.New(fmt.Sprint("Error reading standard input:", err))
	}

	err = child.Wait()
	if err != nil {
		return []string{}, err
	}

	return lines, nil
}

func (c *shellCommander) commandExists(cmd string) bool {
	_, err := exec.LookPath(cmd)
	return err == nil
}
<|endoftext|>


<commit_before>package container

import (
	//"github.com/zenoss/glog"
	"github.com/zenoss/serviced/commons/subprocess"

	//"net"
	"time"
)

// ControllerOptions are options to be run when starting a new proxy server
type ControllerOptions struct {
	TentantID        string   // The top level service id
	ServiceID        string   // The uuid of the service to launch
	Command          []string // The command to launch
	MuxPort          int      // the TCP port for the remote mux
	Mux              bool     // True if a remote mux is used
	TLS              bool     // True if TLS should be used on the mux
	KeyPEMFile       string   // path to the KeyPEMfile
	CertPEMFile      string   // path to the CertPEMfile
	ServicedEndpoint string
	Autorestart      bool
	Logstash         bool
	MetricAddress        string  // MetricPort is the TCP port to host the metric service, :22350
	RemoteMetricEndpoint string  // The url to forward metric queries
}

// Controller is a object to manage the operations withing a container. For example,
// it creates the managed service instance, logstash forwarding, port forwarding, etc.
type Controller struct {
	options         ControllerOptions
	service         *subprocess.Instance
	metricForwarder *MetricForwarder
	logforwarder    *subprocess.Instance
}

type Closer interface {
	Close() error
}

func (c *Controller) Close() {
	for _, s := range []Closer{c.service, c.metricForwarder, c.logforwarder} {
		if s != nil {
			s.Close()
		}
	}
	c.service = nil
	c.metricForwarder = nil
	c.logforwarder = nil
	return
}

// NewController
func NewController(options ControllerOptions) (*Controller, error) {
	c := &Controller{}

	if options.Logstash {
		// make sure we pick up any logfile that was modified within the
		// last three years
		// TODO: Either expose the 3 years a configurable or get rid of it
		logforwarder, err := subprocess.New(time.Millisecond, time.Second,
			"/usr/local/serviced/resources/logstash/logstash-forwarder",
			"-old-files-hours=26280",
			"-config", "/etc/logstash-forwarder.conf")
		if err != nil {
			return nil, err
		}
		c.logforwarder = logforwarder
	}

	//build metric redirect url -- assumes 8444 is port mapped
	metric_redirect := "http://localhost:8444/api/metrics/store"
	metric_redirect += "?controlplane_tenant_id=" + options.TentantID
	metric_redirect += "&controlplane_service_id=" + options.ServiceID

	//build and serve the container metric forwarder
	forwarder, err := NewMetricForwarder(options.MetricAddress, metric_redirect)
	if err != nil {
		return c, err
	}
	c.metricForwarder = forwarder

	return c, nil
}

/*
func (c *Controller) Run() {

	serviceId := proxyCmd.Arg(0)
	command := proxyCmd.Arg(1)

	procexit := make(chan int)
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	// continually execute subprocess
	go func(cmdString string) {
		defer func() { procexit <- 0 }()
		for {
			glog.V(0).Info("About to execute: ", cmdString)
			cmd := exec.Command("bash", "-c", cmdString)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			cmd.Stdin = os.Stdin
			serviceExit := make(chan error)
			go func() {
				serviceExit <- cmd.Run()
			}()
			select {
			case sig := <-sigc:
				glog.V(1).Infof("Caught signal %d", sig)
				cmd.Process.Signal(sig)
				cmd.Wait()
				exitCode := 0
				if cmd.ProcessState != nil {
					exitCode, _ = cmd.ProcessState.Sys().(int)
				}
				procexit <- exitCode
			case cmderr := <-serviceExit:
				if cmderr != nil {
					client, err := serviced.NewLBClient(proxyOptions.servicedEndpoint)
					message := fmt.Sprintf("Service returned a non-zero exit code: %v. Command: \"%v\" Message: %v", serviceId, command, err)
					if err == nil {
						defer client.Close()
						glog.Errorf(message)

						// send the log message to the master
						client.SendLogMessage(serviced.ServiceLogInfo{serviceId, message}, nil)
					} else {
						glog.Errorf("Failed to create a client to endpoint %s: %s", proxyOptions.servicedEndpoint, err)
					}

					glog.Infof("%s", err)
					glog.Flush()
					if exiterr, ok := cmderr.(*exec.ExitError); ok && !proxyOptions.autorestart {
						if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
							procexit <- status.ExitStatus()
						}
					}
				}
			}

			if !proxyOptions.autorestart {
				break
			}
			glog.V(0).Info("service exited, sleeping...")
			time.Sleep(time.Minute)
		}
	}(command)

	//monitor application endpoints to mux ports
	go func() {
		for {
			func() {
				client, err := serviced.NewLBClient(proxyOptions.servicedEndpoint)
				if err != nil {
					glog.Errorf("Could not create a client to endpoint %s: %s", proxyOptions.servicedEndpoint, err)
					return
				}
				defer client.Close()

				var endpoints map[string][]*dao.ApplicationEndpoint
				err = client.GetServiceEndpoints(serviceId, &endpoints)
				if err != nil {
					glog.Errorf("Error getting application endpoints for service %s: %s", serviceId, err)
					return
				}

				for key, endpointList := range endpoints {
					if len(endpointList) <= 0 {
						if proxy, ok := proxies[key]; ok {
							emptyAddressList := make([]string, 0)
							proxy.SetNewAddresses(emptyAddressList)
						}
						continue
					}

					addresses := make([]string, len(endpointList))
					for i, endpoint := range endpointList {
						addresses[i] = fmt.Sprintf("%s:%d", endpoint.HostIp, endpoint.HostPort)
					}
					sort.Strings(addresses)

					var proxy *serviced.Proxy
					var ok bool
					if proxy, ok = proxies[key]; !ok {
						glog.Infof("Attempting port map for: %s -> %+v", key, *endpointList[0])

						// setup a new proxy
						listener, err := net.Listen("tcp4", fmt.Sprintf(":%d", endpointList[0].ContainerPort))
						if err != nil {
							glog.Errorf("Could not bind to port: %s", err)
							continue
						}
						proxy, err = serviced.NewProxy(
							fmt.Sprintf("%v", endpointList[0]),
							uint16(proxyOptions.muxport),
							proxyOptions.tls,
							listener)
						if err != nil {
							glog.Errorf("Could not build proxy %s", err)
							continue
						}

						glog.Infof("Success binding port: %s -> %+v", key, proxy)
						proxies[key] = proxy
					}
					proxy.SetNewAddresses(addresses)
				}
			}()

			time.Sleep(time.Second * 10)
		}
	}()

	//setup container metric forwarder

	exitcode := <-procexit // Wait for proc goroutine to exit

	glog.Flush()
	os.Exit(exitcode)
	return nil
}

var proxies map[string]*serviced.Proxy

func init() {
	proxies = make(map[string]*serviced.Proxy)
}
*/
<commit_msg>redo config<commit_after>package container

import (
	//"github.com/zenoss/glog"
	"github.com/zenoss/serviced/commons/subprocess"

	//"net"
	"time"
)

// ControllerOptions are options to be run when starting a new proxy server
type ControllerOptions struct {
	ServicedEndpoint string
	Service          struct {
		ID          string   // The uuid of the service to launch
		TenantID    string   // The tentant ID of the service
		Autorestart bool     // Controller will restart the service if it exits
		Command     []string // The command to launch
	}
	Mux struct { // TCPMUX configuration: RFC 1078
		Enabled     bool   // True if muxing is used
		Port        int    // the TCP port to use
		TLS         bool   // True if TLS is used
		KeyPEMFile  string // Path to the key file when TLS is used
		CertPEMFile string // Path to the cert file when TLS is used
	}
	Logforwarder struct { // Logforwarder configuration
		Enabled    bool   // True if enabled
		Path       string // Path to the logforwarder program
		ConfigFile string //
	}
	Metric struct {
		Address       string // TCP port to host the metric service, :22350
		RemoteEndoint string // The url to forward metric queries
	}
}

// Controller is a object to manage the operations withing a container. For example,
// it creates the managed service instance, logstash forwarding, port forwarding, etc.
type Controller struct {
	options         ControllerOptions
	service         *subprocess.Instance
	metricForwarder *MetricForwarder
	logforwarder    *subprocess.Instance
}

type Closer interface {
	Close() error
}

func (c *Controller) Close() error {
	for _, s := range []Closer{c.service, c.metricForwarder, c.logforwarder} {
		if s != nil {
			s.Close()
		}
	}
	c.service = nil
	c.metricForwarder = nil
	c.logforwarder = nil
	return nil
}

// NewController
func NewController(options ControllerOptions) (*Controller, error) {
	c := &Controller{}

	if options.Logforwarder.Enabled {
		// make sure we pick up any logfile that was modified within the
		// last three years
		// TODO: Either expose the 3 years a configurable or get rid of it
		logforwarder, err := subprocess.New(time.Millisecond, time.Second,
			options.Logforwarder.Path,
			"-old-files-hours=26280",
			"-config", options.Logforwarder.ConfigFile)
		if err != nil {
			return nil, err
		}
		c.logforwarder = logforwarder
	}

	//build metric redirect url -- assumes 8444 is port mapped
	metric_redirect := options.Metric.RemoteEndoint
	metric_redirect += "?controlplane_tenant_id=" + options.Service.TenantID
	metric_redirect += "&controlplane_service_id=" + options.Service.ID

	//build and serve the container metric forwarder
	forwarder, err := NewMetricForwarder(options.MetricAddress, metric_redirect)
	if err != nil {
		return c, err
	}
	c.metricForwarder = forwarder

	return c, nil
}

/*
func (c *Controller) Run() {

	serviceId := proxyCmd.Arg(0)
	command := proxyCmd.Arg(1)

	procexit := make(chan int)
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	// continually execute subprocess
	go func(cmdString string) {
		defer func() { procexit <- 0 }()
		for {
			glog.V(0).Info("About to execute: ", cmdString)
			cmd := exec.Command("bash", "-c", cmdString)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			cmd.Stdin = os.Stdin
			serviceExit := make(chan error)
			go func() {
				serviceExit <- cmd.Run()
			}()
			select {
			case sig := <-sigc:
				glog.V(1).Infof("Caught signal %d", sig)
				cmd.Process.Signal(sig)
				cmd.Wait()
				exitCode := 0
				if cmd.ProcessState != nil {
					exitCode, _ = cmd.ProcessState.Sys().(int)
				}
				procexit <- exitCode
			case cmderr := <-serviceExit:
				if cmderr != nil {
					client, err := serviced.NewLBClient(proxyOptions.servicedEndpoint)
					message := fmt.Sprintf("Service returned a non-zero exit code: %v. Command: \"%v\" Message: %v", serviceId, command, err)
					if err == nil {
						defer client.Close()
						glog.Errorf(message)

						// send the log message to the master
						client.SendLogMessage(serviced.ServiceLogInfo{serviceId, message}, nil)
					} else {
						glog.Errorf("Failed to create a client to endpoint %s: %s", proxyOptions.servicedEndpoint, err)
					}

					glog.Infof("%s", err)
					glog.Flush()
					if exiterr, ok := cmderr.(*exec.ExitError); ok && !proxyOptions.autorestart {
						if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
							procexit <- status.ExitStatus()
						}
					}
				}
			}

			if !proxyOptions.autorestart {
				break
			}
			glog.V(0).Info("service exited, sleeping...")
			time.Sleep(time.Minute)
		}
	}(command)

	//monitor application endpoints to mux ports
	go func() {
		for {
			func() {
				client, err := serviced.NewLBClient(proxyOptions.servicedEndpoint)
				if err != nil {
					glog.Errorf("Could not create a client to endpoint %s: %s", proxyOptions.servicedEndpoint, err)
					return
				}
				defer client.Close()

				var endpoints map[string][]*dao.ApplicationEndpoint
				err = client.GetServiceEndpoints(serviceId, &endpoints)
				if err != nil {
					glog.Errorf("Error getting application endpoints for service %s: %s", serviceId, err)
					return
				}

				for key, endpointList := range endpoints {
					if len(endpointList) <= 0 {
						if proxy, ok := proxies[key]; ok {
							emptyAddressList := make([]string, 0)
							proxy.SetNewAddresses(emptyAddressList)
						}
						continue
					}

					addresses := make([]string, len(endpointList))
					for i, endpoint := range endpointList {
						addresses[i] = fmt.Sprintf("%s:%d", endpoint.HostIp, endpoint.HostPort)
					}
					sort.Strings(addresses)

					var proxy *serviced.Proxy
					var ok bool
					if proxy, ok = proxies[key]; !ok {
						glog.Infof("Attempting port map for: %s -> %+v", key, *endpointList[0])

						// setup a new proxy
						listener, err := net.Listen("tcp4", fmt.Sprintf(":%d", endpointList[0].ContainerPort))
						if err != nil {
							glog.Errorf("Could not bind to port: %s", err)
							continue
						}
						proxy, err = serviced.NewProxy(
							fmt.Sprintf("%v", endpointList[0]),
							uint16(proxyOptions.muxport),
							proxyOptions.tls,
							listener)
						if err != nil {
							glog.Errorf("Could not build proxy %s", err)
							continue
						}

						glog.Infof("Success binding port: %s -> %+v", key, proxy)
						proxies[key] = proxy
					}
					proxy.SetNewAddresses(addresses)
				}
			}()

			time.Sleep(time.Second * 10)
		}
	}()

	//setup container metric forwarder

	exitcode := <-procexit // Wait for proc goroutine to exit

	glog.Flush()
	os.Exit(exitcode)
	return nil
}

var proxies map[string]*serviced.Proxy

func init() {
	proxies = make(map[string]*serviced.Proxy)
}
*/
<|endoftext|>


<commit_before>package kubernetes

import (
	"fmt"
	"time"

	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/pkg/apis/extensions/v1beta1"

	"github.com/rusenask/keel/types"
	"github.com/rusenask/keel/util/version"

	"testing"
)

var currentVersion = "0.0.2"
var newVersion = "0.0.3"

type fakeImplementer struct {
	namespaces *v1.NamespaceList
	deployment *v1beta1.Deployment
}

func (i *fakeImplementer) Namespaces() (*v1.NamespaceList, error) {
	return i.namespaces, nil
}

func (i *fakeImplementer) Deployment(namespace, name string) (*v1beta1.Deployment, error) {
	return i.deployment, nil
}

func (i *fakeImplementer) Deployments(namespace string) (*v1beta1.DeploymentList, error) {
	return nil, nil
}

func (i *fakeImplementer) Update(deployment *v1beta1.Deployment) error {
	return nil
}

func TestGetNamespaces(t *testing.T) {
	fi := &fakeImplementer{
		namespaces: &v1.NamespaceList{
			Items: []v1.Namespace{
				v1.Namespace{
					meta_v1.TypeMeta{},
					meta_v1.ObjectMeta{Name: "xxxx"},
					v1.NamespaceSpec{},
					v1.NamespaceStatus{},
				},
			},
		},
	}

	provider, err := NewProvider(fi)
	if err != nil {
		t.Fatalf("failed to get provider: %s", err)
	}

	namespaces, err := provider.namespaces()
	if err != nil {
		t.Errorf("failed to get namespaces: %s", err)
	}

	if namespaces.Items[0].Name != "xxxx" {
		t.Errorf("expected xxxx but got %s", namespaces.Items[0].Name)
	}
}

func TestGetImageName(t *testing.T) {
	name := versionreg.ReplaceAllString("gcr.io/v2-namespace/hello-world:1.1", "")
	fmt.Println(name)
}

func TestGetDeployments(t *testing.T) {
	provider, err := NewProvider(&fakeImplementer{})
	if err != nil {
		t.Fatalf("failed to get provider: %s", err)
	}

	deps, err := provider.deployments()
	if err != nil {
		t.Errorf("failed to get deployments: %s", err)
	}
	// fmt.Println(len(deps.Items))
	fmt.Println(deps)
}

func TestGetImpacted(t *testing.T) {
	provider, err := NewProvider(&fakeImplementer{})
	if err != nil {
		t.Fatalf("failed to get provider: %s", err)
	}

	repo := &types.Repository{
		Name: "karolisr/webhook-demo",
		Tag:  "0.0.3",
	}

	deps, err := provider.impactedDeployments(repo)
	if err != nil {
		t.Errorf("failed to get deployments: %s", err)
	}
	found := false
	for _, c := range deps[0].Spec.Template.Spec.Containers {
		ver, err := version.GetVersionFromImageName(c.Image)
		if err != nil {
			continue
		}
		if ver.String() == repo.Tag {
			found = true
		}
	}
	// fmt.Println(len(deps.Items))
	fmt.Println(len(deps))
	fmt.Println(found)

}
func TestProcessEvent(t *testing.T) {
	provider, err := NewProvider(&fakeImplementer{})
	if err != nil {
		t.Fatalf("failed to get provider: %s", err)
	}

	repo := types.Repository{
		Name: "karolisr/webhook-demo",
		Tag:  newVersion,
	}

	event := &types.Event{Repository: repo}
	updated, err := provider.processEvent(event)
	if err != nil {
		t.Errorf("got error while processing event: %s", err)
	}

	//
	time.Sleep(100 * time.Millisecond)
	for _, upd := range updated {
		current, err := provider.getDeployment(upd.Namespace, upd.Name)
		if err != nil {
			t.Fatalf("failed to get deployment %s, error: %s", upd.Name, err)
		}
		currentVer, err := version.GetVersionFromImageName(current.Spec.Template.Spec.Containers[0].Image)
		if err != nil {
			t.Fatalf("failed to get version from %s, error: %s", current.Spec.Template.Spec.Containers[0].Image, err)
		}

		if currentVer.String() != newVersion {
			t.Errorf("deployment version wasn't updated, got: %s while expected: %s", currentVer.String(), newVersion)
		}
	}

}
<commit_msg>testing<commit_after>package kubernetes

import (
	"fmt"
	"time"

	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/pkg/apis/extensions/v1beta1"

	"github.com/rusenask/keel/types"
	"github.com/rusenask/keel/util/version"

	"testing"
)

var currentVersion = "0.0.2"
var newVersion = "0.0.3"

type fakeImplementer struct {
	namespaces     *v1.NamespaceList
	deployment     *v1beta1.Deployment
	deploymentList *v1beta1.DeploymentList
}

func (i *fakeImplementer) Namespaces() (*v1.NamespaceList, error) {
	return i.namespaces, nil
}

func (i *fakeImplementer) Deployment(namespace, name string) (*v1beta1.Deployment, error) {
	return i.deployment, nil
}

func (i *fakeImplementer) Deployments(namespace string) (*v1beta1.DeploymentList, error) {
	return i.deploymentList, nil
}

func (i *fakeImplementer) Update(deployment *v1beta1.Deployment) error {
	return nil
}

func TestGetNamespaces(t *testing.T) {
	fi := &fakeImplementer{
		namespaces: &v1.NamespaceList{
			Items: []v1.Namespace{
				v1.Namespace{
					meta_v1.TypeMeta{},
					meta_v1.ObjectMeta{Name: "xxxx"},
					v1.NamespaceSpec{},
					v1.NamespaceStatus{},
				},
			},
		},
	}

	provider, err := NewProvider(fi)
	if err != nil {
		t.Fatalf("failed to get provider: %s", err)
	}

	namespaces, err := provider.namespaces()
	if err != nil {
		t.Errorf("failed to get namespaces: %s", err)
	}

	if namespaces.Items[0].Name != "xxxx" {
		t.Errorf("expected xxxx but got %s", namespaces.Items[0].Name)
	}
}

func TestGetImageName(t *testing.T) {
	name := versionreg.ReplaceAllString("gcr.io/v2-namespace/hello-world:1.1", "")
	if name != "gcr.io/v2-namespace/hello-world" {
		t.Errorf("expected 'gcr.io/v2-namespace/hello-world' but got '%s'", name)
	}
}

func TestGetDeployments(t *testing.T) {
	fp := &fakeImplementer{}
	fp.namespaces = &v1.NamespaceList{
		Items: []v1.Namespace{
			v1.Namespace{
				meta_v1.TypeMeta{},
				meta_v1.ObjectMeta{Name: "xxxx"},
				v1.NamespaceSpec{},
				v1.NamespaceStatus{},
			},
		},
	}
	fp.deploymentList = &v1beta1.DeploymentList{
		Items: []v1beta1.Deployment{
			v1beta1.Deployment{
				meta_v1.TypeMeta{},
				meta_v1.ObjectMeta{
					Name:      "dep-1",
					Namespace: "xxxx",
					Labels:    map[string]string{types.KeelPolicyLabel: "all"},
				},
				v1beta1.DeploymentSpec{
					Template: v1.PodTemplateSpec{
						Spec: v1.PodSpec{
							Containers: []v1.Container{
								v1.Container{
									Image: "gcr.io/v2-namespace/hello-world:1.1",
								},
							},
						},
					},
				},
				v1beta1.DeploymentStatus{},
			},
		},
	}

	provider, err := NewProvider(fp)
	if err != nil {
		t.Fatalf("failed to get provider: %s", err)
	}

	deps, err := provider.deployments()
	if err != nil {
		t.Errorf("failed to get deployments: %s", err)
	}
	if len(deps) != 1 {
		t.Errorf("expected to find 1 deployment, got: %d", len(deps))
	}

	if deps[0].Items[0].GetName() != "dep-1" {
		t.Errorf("expected name %s, got %s", "dep-1", deps[0].Items[0].GetName())
	}
}

func TestGetImpacted(t *testing.T) {
	provider, err := NewProvider(&fakeImplementer{})
	if err != nil {
		t.Fatalf("failed to get provider: %s", err)
	}

	repo := &types.Repository{
		Name: "karolisr/webhook-demo",
		Tag:  "0.0.3",
	}

	deps, err := provider.impactedDeployments(repo)
	if err != nil {
		t.Errorf("failed to get deployments: %s", err)
	}
	found := false
	for _, c := range deps[0].Spec.Template.Spec.Containers {
		ver, err := version.GetVersionFromImageName(c.Image)
		if err != nil {
			continue
		}
		if ver.String() == repo.Tag {
			found = true
		}
	}
	// fmt.Println(len(deps.Items))
	fmt.Println(len(deps))
	fmt.Println(found)

}
func TestProcessEvent(t *testing.T) {
	provider, err := NewProvider(&fakeImplementer{})
	if err != nil {
		t.Fatalf("failed to get provider: %s", err)
	}

	repo := types.Repository{
		Name: "karolisr/webhook-demo",
		Tag:  newVersion,
	}

	event := &types.Event{Repository: repo}
	updated, err := provider.processEvent(event)
	if err != nil {
		t.Errorf("got error while processing event: %s", err)
	}

	//
	time.Sleep(100 * time.Millisecond)
	for _, upd := range updated {
		current, err := provider.getDeployment(upd.Namespace, upd.Name)
		if err != nil {
			t.Fatalf("failed to get deployment %s, error: %s", upd.Name, err)
		}
		currentVer, err := version.GetVersionFromImageName(current.Spec.Template.Spec.Containers[0].Image)
		if err != nil {
			t.Fatalf("failed to get version from %s, error: %s", current.Spec.Template.Spec.Containers[0].Image, err)
		}

		if currentVer.String() != newVersion {
			t.Errorf("deployment version wasn't updated, got: %s while expected: %s", currentVer.String(), newVersion)
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/ae6rt/decap/build-container/locks"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/spf13/cobra"
)

var buildInfo string
var buildStartTime int64
var buildDuration int64
var bucketName string
var buildID string
var contentType string
var fileName string
var awsRegion string
var Log *log.Logger = log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile)

var BCToolCmd = &cobra.Command{
	Use:   "bctool",
	Short: "bctool is a multifunction build container tool.",
	Long:  `A multifunction build container tool that unlocks builds, uploads files to S3, and puts items to DynamoDb`,
}

var versionCmd = &cobra.Command{
	Use:   "version",
	Short: "Print the version number of bctool",
	Long:  `All software has versions. This is bctool's`,
	Run: func(cmd *cobra.Command, args []string) {
		Log.Println(buildInfo)
		os.Exit(0)
	},
}

var unlockBuildCmd = &cobra.Command{
	Use:   "unlock",
	Short: "Unlock a build",
	Long:  `Unlock a build`,
	Run: func(cmd *cobra.Command, args []string) {
		locks.Unlock()
	},
}

var putS3Cmd = &cobra.Command{
	Use:   "s3put",
	Short: "put a file to an S3 bucket",
	Long:  `put a file to an S3 bucket`,
	Run: func(cmd *cobra.Command, args []string) {
		config := aws.NewConfig().WithCredentials(credentials.NewEnvCredentials()).WithRegion(awsRegion).WithMaxRetries(3)
		svc := s3.New(config)
		data, err := ioutil.ReadFile(fileName)
		if err != nil {
			Log.Fatal(err)
		}
		params := &s3.PutObjectInput{
			Bucket:        aws.String(bucketName),
			Key:           aws.String(buildID),
			Body:          bytes.NewReader(data),
			ContentType:   aws.String(contentType),
			ContentLength: aws.Int64(int64(len(data))),
		}
		if _, err := svc.PutObject(params); err != nil {
			Log.Fatal(err.Error())
		} else {
			Log.Println("S3 PUT successful")
		}
	},
}

/*
{
        "buildID": {
            "S": "$BUILD_ID"
        },
        "buildTime": {
            "N": "$START"
        },
        "projectKey": {
            "S": "$PROJECT_KEY"
        },
        "branch": {
            "S": "$BRANCH_TO_BUILD"
        },
        "isBuilding": {
            "N": "1"
        }
}
*/
var buildStartCmd = &cobra.Command{
	Use:   "build-start",
	Short: "mark a build as started in DynamoDb",
	Long:  "mark a build as started in DynamoDb",
	Run: func(cmd *cobra.Command, args []string) {

		projectKey := os.Getenv("PROJECT_KEY")
		buildID := os.Getenv("BUILD_ID")
		branch := os.Getenv("BRANCH_TO_BUILD")

		config := aws.NewConfig().WithCredentials(credentials.NewEnvCredentials()).WithRegion(awsRegion).WithMaxRetries(3)
		svc := dynamodb.New(config)

		params := &dynamodb.PutItemInput{
			TableName: aws.String("decap-build-metadata"),
			Item: map[string]*dynamodb.AttributeValue{
				"projectKey": {
					S: aws.String(":pkey"),
				},
				"buildID": {
					S: aws.String(":buildID"),
				},
				"buildTime": {
					N: aws.String(":buildTime"),
				},
				"branch": {
					S: aws.String(":branch"),
				},
				"isBuildling": {
					N: aws.String("1"),
				},
			},
			ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{
				":pkey": {
					S: aws.String(projectKey),
				},
				":buildID": {
					S: aws.String(buildID),
				},
				":buildTime": {
					N: aws.String(fmt.Sprintf("%d", buildStartTime)),
				},
				":branch": {
					S: aws.String(branch),
				},
			},
		}

		_, err := svc.PutItem(params)

		if err != nil {
			Log.Fatal(err.Error())
		} else {
			Log.Println("DynamoDb PUT successful")
		}
	},
}

func init() {
	putS3Cmd.Flags().StringVarP(&bucketName, "bucket-name", "b", "", "S3 Bucket Name")
	putS3Cmd.Flags().StringVarP(&buildID, "build-id", "i", "", "Build ID")
	putS3Cmd.Flags().StringVarP(&contentType, "content-type", "t", "", "Content Type")
	putS3Cmd.Flags().StringVarP(&fileName, "filename", "f", "", "File Name")
	putS3Cmd.Flags().StringVarP(&awsRegion, "aws-region", "r", "", "AWS Region")

	buildStartCmd.Flags().Int64VarP(&buildStartTime, "build-start-time", "s", 0, "Build start Unix-time")

	BCToolCmd.AddCommand(versionCmd)
	BCToolCmd.AddCommand(unlockBuildCmd)
	BCToolCmd.AddCommand(putS3Cmd)
}

func main() {
	BCToolCmd.Execute()
}
<commit_msg>finish build-finish command<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/ae6rt/decap/build-container/locks"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/spf13/cobra"
)

var buildInfo string

var buildStartTime int64
var buildDuration int64
var buildResult int64

var bucketName string
var buildID string
var contentType string
var fileName string
var awsRegion string
var Log *log.Logger = log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile)

var BCToolCmd = &cobra.Command{
	Use:   "bctool",
	Short: "bctool is a multifunction build container tool.",
	Long:  `A multifunction build container tool that unlocks builds, uploads files to S3, and puts items to DynamoDb`,
}

var versionCmd = &cobra.Command{
	Use:   "version",
	Short: "Print the version number of bctool",
	Long:  `All software has versions. This is bctool's`,
	Run: func(cmd *cobra.Command, args []string) {
		Log.Println(buildInfo)
		os.Exit(0)
	},
}

var unlockBuildCmd = &cobra.Command{
	Use:   "unlock",
	Short: "Unlock a build",
	Long:  `Unlock a build`,
	Run: func(cmd *cobra.Command, args []string) {
		locks.Unlock()
	},
}

var putS3Cmd = &cobra.Command{
	Use:   "s3put",
	Short: "put a file to an S3 bucket",
	Long:  `put a file to an S3 bucket`,
	Run: func(cmd *cobra.Command, args []string) {
		config := aws.NewConfig().WithCredentials(credentials.NewEnvCredentials()).WithRegion(awsRegion).WithMaxRetries(3)
		svc := s3.New(config)
		data, err := ioutil.ReadFile(fileName)
		if err != nil {
			Log.Fatal(err)
		}
		params := &s3.PutObjectInput{
			Bucket:        aws.String(bucketName),
			Key:           aws.String(buildID),
			Body:          bytes.NewReader(data),
			ContentType:   aws.String(contentType),
			ContentLength: aws.Int64(int64(len(data))),
		}
		if _, err := svc.PutObject(params); err != nil {
			Log.Fatal(err.Error())
		} else {
			Log.Println("S3 PUT successful")
		}
	},
}

var buildStartCmd = &cobra.Command{
	Use:   "build-start",
	Short: "mark a build as started in DynamoDb",
	Long:  "mark a build as started in DynamoDb",
	Run: func(cmd *cobra.Command, args []string) {

		projectKey := os.Getenv("PROJECT_KEY")
		buildID := os.Getenv("BUILD_ID")
		branch := os.Getenv("BRANCH_TO_BUILD")

		config := aws.NewConfig().WithCredentials(credentials.NewEnvCredentials()).WithRegion(awsRegion).WithMaxRetries(3)
		svc := dynamodb.New(config)

		params := &dynamodb.PutItemInput{
			TableName: aws.String("decap-build-metadata"),
			Item: map[string]*dynamodb.AttributeValue{
				"buildID": {
					S: aws.String(buildID),
				},
				"projectKey": {
					S: aws.String(projectKey),
				},
				"buildTime": {
					N: aws.String(fmt.Sprintf("%d", buildStartTime)),
				},
				"branch": {
					S: aws.String(branch),
				},
				"isBuilding": {
					N: aws.String("1"),
				},
			},
		}

		if _, err := svc.PutItem(params); err != nil {
			Log.Fatal(err.Error())
		} else {
			Log.Println("DynamoDb PUT successful")
		}
	},
}

var buildFinishCmd = &cobra.Command{
	Use:   "build-finish",
	Short: "mark a build as finished in DynamoDb",
	Long:  "mark a build as finished in DynamoDb",
	Run: func(cmd *cobra.Command, args []string) {

		buildID := os.Getenv("BUILD_ID")

		config := aws.NewConfig().WithCredentials(credentials.NewEnvCredentials()).WithRegion(awsRegion).WithMaxRetries(3)
		svc := dynamodb.New(config)

		params := &dynamodb.UpdateItemInput{
			TableName: aws.String("decap-build-metadata"),
			Key: map[string]*dynamodb.AttributeValue{
				"buildID": {
					S: aws.String(buildID),
				},
			},
			UpdateExpression: aws.String("SET buildElapsedTime = :buildDuration, buildResult = :buildResult, isBuilding = :isBuilding"),
			ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{
				":buildDuration": {
					N: aws.String(fmt.Sprintf("%d", buildDuration)),
				},
				":buildResult": {
					N: aws.String(fmt.Sprintf("%d", buildResult)),
				},
				":isBuilding": {
					N: aws.String("0"),
				},
			},
		}

		if _, err := svc.UpdateItem(params); err != nil {
			Log.Fatal(err.Error())
		} else {
			Log.Println("DynamoDb PUT successful")
		}
	},
}

func init() {
	putS3Cmd.Flags().StringVarP(&bucketName, "bucket-name", "b", "", "S3 Bucket Name")
	putS3Cmd.Flags().StringVarP(&buildID, "build-id", "i", "", "Build ID")
	putS3Cmd.Flags().StringVarP(&contentType, "content-type", "t", "", "Content Type")
	putS3Cmd.Flags().StringVarP(&fileName, "filename", "f", "", "File Name")
	putS3Cmd.Flags().StringVarP(&awsRegion, "aws-region", "r", "us-west-1", "AWS Region")

	buildStartCmd.Flags().StringVarP(&awsRegion, "aws-region", "r", "us-west-1", "AWS Region")
	buildStartCmd.Flags().Int64VarP(&buildStartTime, "start-time", "s", 0, "Unix time in seconds since the epoch when the build started")

	buildFinishCmd.Flags().StringVarP(&awsRegion, "aws-region", "r", "us-west-1", "AWS Region")
	buildFinishCmd.Flags().Int64VarP(&buildResult, "build-result", "s", 0, "Unix exit code of the executed build")
	buildFinishCmd.Flags().Int64VarP(&buildDuration, "build-duration", "d", 0, "Duration of the build in seconds")

	BCToolCmd.AddCommand(versionCmd)
	BCToolCmd.AddCommand(unlockBuildCmd)
	BCToolCmd.AddCommand(putS3Cmd)
	BCToolCmd.AddCommand(buildStartCmd)
	BCToolCmd.AddCommand(buildFinishCmd)
}

func main() {
	BCToolCmd.Execute()
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
)

func forEachWarehouseAndDistrict(fn func(wID, dID int) error) error {
	for wID := 0; wID < *warehouses; wID++ {
		for dID := 1; dID < 10; dID++ {
			if err := fn(wID, dID); err != nil {
				return err
			}
		}
	}
	return nil
}

func checkConsistency(db *sql.DB) error {
	// Reused checks

	maxNewOIDs := make([]int, *warehouses*10)
	countNewOrders := make([]int, *warehouses*10)

	sliceIdx := func(wID, dID int) int {
		return wID*10 + dID - 1
	}

	// 3.3.2.1
	// W_YTD = sum(D_YTD) for all warehouses and districts
	var wYTD, sumDYTD float64
	for i := 0; i < *warehouses; i++ {
		if err := db.QueryRow("SELECT w_ytd FROM warehouse WHERE w_id=$1", i).Scan(&wYTD); err != nil {
			return err
		}
		if err := db.QueryRow("SELECT SUM(d_ytd) FROM district WHERE d_w_id=$1", i).Scan(&sumDYTD); err != nil {
			return err
		}
		if wYTD != sumDYTD {
			fmt.Printf("check failed: w_ytd=%f != sum(d_ytd)=%f for warehouse %d\n", wYTD, sumDYTD, i)
		}
	}

	// 3.3.2.2
	// D_NEXT_O_ID - 1 = max(O_ID) = max(NO_O_ID) for all districts
	var dNextOID, maxOID, maxNewOID int
	if err := forEachWarehouseAndDistrict(func(wID, dID int) error {
		if err := db.QueryRow(`SELECT d_next_o_id FROM district WHERE d_w_id=$1 AND d_id=$2`,
			wID, dID,
		).Scan(&dNextOID); err != nil {
			return err
		}
		if err := db.QueryRow(`SELECT max(o_id) FROM "order" WHERE o_w_id=$1 AND o_d_id=$2`,
			wID, dID,
		).Scan(&maxOID); err != nil {
			return err
		}
		if err := db.QueryRow(`SELECT max(no_o_id) FROM new_order WHERE no_w_id=$1 AND no_d_id=$2`,
			wID, dID,
		).Scan(&maxNewOID); err != nil {
			return err
		}
		maxNewOIDs[sliceIdx(wID, dID)] = maxNewOID

		if dNextOID != maxOID+1 || dNextOID != maxNewOID+1 {
			fmt.Printf("check failed: d_next_o_id=%d != max(o_id)+1=%d != max(no_o_id)+1=%d for warehouse %d district %d\n",
				dNextOID, maxOID, maxNewOID, wID, dID)
		}
		return nil
	}); err != nil {
		return err
	}

	// 3.3.2.3
	// max(NO_O_ID) - min(NO_O_ID) + 1 = # of rows in new_order for each warehouse/district
	var minNewOID, countNewOrder int
	if err := forEachWarehouseAndDistrict(func(wID, dID int) error {
		if err := db.QueryRow(`SELECT min(no_o_id), count(*) FROM new_order WHERE no_w_id=$1 AND no_d_id=$2`,
			wID, dID,
		).Scan(&minNewOID, &countNewOrder); err != nil {
			return err
		}
		countNewOrders[sliceIdx(wID, dID)] = countNewOrder
		maxNewOID := maxNewOIDs[sliceIdx(wID, dID)]
		if maxNewOID-minNewOID+1 != countNewOrder {
			fmt.Printf("check failed: max(no_o_id)=%d + min(no_o_id)=%d + 1 != count(*)=%d for w_id=%d d_id=%d\n",
				maxNewOID, minNewOID, countNewOrder, wID, dID)
		}
		return nil
	}); err != nil {
		return err
	}

	// 3.3.2.4
	// sum(O_OL_CNT) = # of rows in the ORDER-LINE table for each warehouse/district
	var sumOOLCnt int
	if err := forEachWarehouseAndDistrict(func(wID, dID int) error {
		if err := db.QueryRow(`SELECT sum(o_ol_cnt) FROM "order" WHERE o_w_id=$1 AND o_d_id=$2`,
			wID, dID,
		).Scan(sumOOLCnt); err != nil {
			return err
		}
		countNewOrder := countNewOrders[sliceIdx(wID, dID)]
		if countNewOrder != sumOOLCnt {
			fmt.Printf("check failed: count(new_orders)=%d != sum(o_ol_cnt)=%d for w_id=%d d_id=%d\n", countNewOrder, sumOOLCnt, wID, dID)
		}
		return nil
	}); err != nil {
		return err
	}

	// 3.3.2.8
	// W_YTD = sum(H_AMOUNT) for each warehouse
	var sumHAmount float64
	for i := 0; i < *warehouses; i++ {
		if err := db.QueryRow("SELECT w_ytd FROM warehouse WHERE w_id=$1", i).Scan(&wYTD); err != nil {
			return err
		}
		if err := db.QueryRow("SELECT SUM(h_amount) FROM history WHERE h_w_id=$1", i).Scan(&sumHAmount); err != nil {
			return err
		}
		if wYTD != sumHAmount {
			fmt.Printf("check failed: w_ytd=%f != sum(h_amount)=%f for warehouse %d\n", wYTD, sumHAmount, i)
		}
	}

	return nil
}
<commit_msg>tpcc: fix "sum(O_OL_CNT) = # of rows in the ORDER-LINE table" check<commit_after>package main

import (
	"database/sql"
	"fmt"
)

func forEachWarehouseAndDistrict(fn func(wID, dID int) error) error {
	for wID := 0; wID < *warehouses; wID++ {
		for dID := 1; dID < 10; dID++ {
			if err := fn(wID, dID); err != nil {
				return err
			}
		}
	}
	return nil
}

func checkConsistency(db *sql.DB) error {
	// Reused checks

	maxNewOIDs := make([]int, *warehouses*10)

	sliceIdx := func(wID, dID int) int {
		return wID*10 + dID - 1
	}

	// 3.3.2.1
	// W_YTD = sum(D_YTD) for all warehouses and districts
	var wYTD, sumDYTD float64
	for i := 0; i < *warehouses; i++ {
		if err := db.QueryRow("SELECT w_ytd FROM warehouse WHERE w_id=$1", i).Scan(&wYTD); err != nil {
			return err
		}
		if err := db.QueryRow("SELECT SUM(d_ytd) FROM district WHERE d_w_id=$1", i).Scan(&sumDYTD); err != nil {
			return err
		}
		if wYTD != sumDYTD {
			fmt.Printf("check failed: w_ytd=%f != sum(d_ytd)=%f for warehouse %d\n", wYTD, sumDYTD, i)
		}
	}

	// 3.3.2.2
	// D_NEXT_O_ID - 1 = max(O_ID) = max(NO_O_ID) for all districts
	var dNextOID, maxOID, maxNewOID int
	if err := forEachWarehouseAndDistrict(func(wID, dID int) error {
		if err := db.QueryRow(`SELECT d_next_o_id FROM district WHERE d_w_id=$1 AND d_id=$2`,
			wID, dID,
		).Scan(&dNextOID); err != nil {
			return err
		}
		if err := db.QueryRow(`SELECT max(o_id) FROM "order" WHERE o_w_id=$1 AND o_d_id=$2`,
			wID, dID,
		).Scan(&maxOID); err != nil {
			return err
		}
		if err := db.QueryRow(`SELECT max(no_o_id) FROM new_order WHERE no_w_id=$1 AND no_d_id=$2`,
			wID, dID,
		).Scan(&maxNewOID); err != nil {
			return err
		}
		maxNewOIDs[sliceIdx(wID, dID)] = maxNewOID

		if dNextOID != maxOID+1 || dNextOID != maxNewOID+1 {
			fmt.Printf("check failed: d_next_o_id=%d != max(o_id)+1=%d != max(no_o_id)+1=%d for warehouse %d district %d\n",
				dNextOID, maxOID, maxNewOID, wID, dID)
		}
		return nil
	}); err != nil {
		return err
	}

	// 3.3.2.3
	// max(NO_O_ID) - min(NO_O_ID) + 1 = # of rows in new_order for each warehouse/district
	var minNewOID, countNewOrder int
	if err := forEachWarehouseAndDistrict(func(wID, dID int) error {
		if err := db.QueryRow(`SELECT min(no_o_id), count(*) FROM new_order WHERE no_w_id=$1 AND no_d_id=$2`,
			wID, dID,
		).Scan(&minNewOID, &countNewOrder); err != nil {
			return err
		}
		maxNewOID := maxNewOIDs[sliceIdx(wID, dID)]
		if maxNewOID-minNewOID+1 != countNewOrder {
			fmt.Printf("check failed: max(no_o_id)=%d + min(no_o_id)=%d + 1 != count(*)=%d for w_id=%d d_id=%d\n",
				maxNewOID, minNewOID, countNewOrder, wID, dID)
		}
		return nil
	}); err != nil {
		return err
	}

	// 3.3.2.4
	// sum(O_OL_CNT) = # of rows in the ORDER-LINE table for each warehouse/district
	var sumOOLCnt, countNewOrderLine int
	if err := forEachWarehouseAndDistrict(func(wID, dID int) error {
		if err := db.QueryRow(`SELECT sum(o_ol_cnt) FROM "order" WHERE o_w_id=$1 AND o_d_id=$2`,
			wID, dID,
		).Scan(&sumOOLCnt); err != nil {
			return err
		}
		if err := db.QueryRow(`SELECT count(*) FROM order_line WHERE ol_w_id=$1 AND ol_d_id=$2`,
			wID, dID,
		).Scan(&countNewOrderLine); err != nil {
			return err
		}
		if countNewOrderLine != sumOOLCnt {
			fmt.Printf("check failed: count(order_line)=%d != sum(o_ol_cnt)=%d for w_id=%d d_id=%d\n",
				countNewOrderLine, sumOOLCnt, wID, dID)
		}
		return nil
	}); err != nil {
		return err
	}

	// 3.3.2.8
	// W_YTD = sum(H_AMOUNT) for each warehouse
	var sumHAmount float64
	for i := 0; i < *warehouses; i++ {
		if err := db.QueryRow("SELECT w_ytd FROM warehouse WHERE w_id=$1", i).Scan(&wYTD); err != nil {
			return err
		}
		if err := db.QueryRow("SELECT SUM(h_amount) FROM history WHERE h_w_id=$1", i).Scan(&sumHAmount); err != nil {
			return err
		}
		if wYTD != sumHAmount {
			fmt.Printf("check failed: w_ytd=%f != sum(h_amount)=%f for warehouse %d\n", wYTD, sumHAmount, i)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package trees

import (
	"math"
	"math/rand"
	"time"

	"github.com/sjwhitworth/golearn/base"
)

type IsolationForest struct {
	nTrees   int
	maxDepth int
	subSpace int
	trees    []regressorNode
}

func selectFeature(data [][]float64) int64 {
	rand.Seed(time.Now().UnixNano())
	return int64(rand.Intn(len(data[0])))
}

func minMax(feature int64, data [][]float64) (float64, float64) {

	var min, max float64

	min = math.Inf(1)
	max = math.Inf(-1)
	for _, instance := range data {
		if instance[feature] > max {
			max = instance[feature]
		}
		if instance[feature] < min {
			min = instance[feature]
		}
	}

	return min, max
}

func selectValue(min, max float64) float64 {
	rand.Seed(time.Now().UnixNano())
	val := min + (rand.Float64() * (max - min))
	if val == min {
		val += 0.000001
	} else if val == max {
		val -= 0.000001
	}
	return val
}

func splitData(val float64, feature int64, data [][]float64) ([][]float64, [][]float64) {
	var leftData, rightData [][]float64
	for _, instance := range data {
		if instance[feature] <= val {
			leftData = append(leftData, instance)
		} else {
			rightData = append(rightData, instance)
		}
	}
	return leftData, rightData
}

func checkData(data [][]float64) bool {
	for _, instance := range data {
		for i, val := range instance {
			if val != data[0][i] {
				return true
			}
		}
	}
	return false
}

func buildTree(data [][]float64, upperNode regressorNode, depth int, maxDepth int) regressorNode {
	depth++

	upperNode.isNodeNeeded = true
	if (depth > maxDepth) || (len(data) <= 1) || !checkData(data) {
		upperNode.isNodeNeeded = false
		return upperNode
	}

	var featureToSplit int64
	var splitVal float64
	var min, max float64
	min, max = 0.0, 0.0

	for min == max {
		featureToSplit = selectFeature(data)
		min, max = minMax(featureToSplit, data)
		splitVal = selectValue(min, max)
	}

	leftData, rightData := splitData(splitVal, featureToSplit, data)

	upperNode.Feature = featureToSplit
	upperNode.Threshold = splitVal
	upperNode.LeftPred = float64(len(leftData))
	upperNode.RightPred = float64(len(rightData))

	var leftNode, rightNode regressorNode
	leftNode = buildTree(leftData, leftNode, depth, maxDepth)
	rightNode = buildTree(rightData, rightNode, depth, maxDepth)

	if leftNode.isNodeNeeded {
		upperNode.Left = &leftNode
	}
	if rightNode.isNodeNeeded {
		upperNode.Right = &rightNode
	}

	return upperNode
}

func getRandomData(data [][]float64, subSpace int) [][]float64 {
	var randomData [][]float64
	rand.Seed(time.Now().UnixNano())
	for i := 0; i < subSpace; i++ {
		randomData = append(randomData, data[rand.Intn(len(data))])
	}
	return randomData
}

func NewIsolationForest(nTrees int, maxDepth int, subSpace int) IsolationForest {
	var iForest IsolationForest
	iForest.nTrees = nTrees
	iForest.maxDepth = maxDepth
	iForest.subSpace = subSpace
	return iForest
}

func (iForest *IsolationForest) Fit(X base.FixedDataGrid) {
	data := preprocessData(X)
	nTrees := iForest.nTrees
	subSpace := iForest.subSpace
	maxDepth := iForest.maxDepth

	var forest []regressorNode
	for i := 0; i < nTrees; i++ {
		subData := getRandomData(data, subSpace)
		var tree regressorNode

		tree = buildTree(subData, tree, 0, maxDepth)
		forest = append(forest, tree)
	}
	iForest.trees = forest
}

func pathLength(tree regressorNode, instance []float64, path float64) float64 {
	path++

	if instance[tree.Feature] <= tree.Threshold {
		if tree.Left == nil {
			if tree.LeftPred <= 1 {
				return path
			} else {
				return path + cFactor(int(tree.LeftPred))
			}
		}
		path = pathLength(*tree.Left, instance, path)
	} else {
		if tree.Right == nil {
			if tree.RightPred <= 1 {
				return path
			} else {
				return path + cFactor(int(tree.RightPred))
			}
		}
		path = pathLength(*tree.Right, instance, path)
	}
	return path
}

func evaluateInstance(instance []float64, forest []regressorNode) []float64 {
	var paths []float64
	for _, tree := range forest {
		paths = append(paths, pathLength(tree, instance, 0))
	}
	return paths
}

func cFactor(n int) float64 {
	return 2.0*(math.Log(float64(n-1))+0.5772156649) - (float64(2.0*(n-1)) / float64(n))
}

func anomalyScore(instance []float64, forest []regressorNode, n int) float64 {
	paths := evaluateInstance(instance, forest)
	E := 0.0
	for _, path := range paths {
		E += path
	}
	E /= float64(len(paths))
	c := cFactor(n)
	return math.Pow(2, (-1 * E / c))
}
func (iForest *IsolationForest) Predict(X base.FixedDataGrid) []float64 {
	data := preprocessData(X)

	var preds []float64
	for _, instance := range data {
		score := anomalyScore(instance, iForest.trees, iForest.subSpace)
		preds = append(preds, score)
	}
	return preds
}

func preprocessData(X base.FixedDataGrid) [][]float64 {
	data := convertInstancesToProblemVec(X)
	class, err := regressorConvertInstancesToLabelVec(X)
	if err != nil {
		panic(err)
	}
	for i, point := range class {
		data[i] = append(data[i], point)
	}
	return data
}
<commit_msg>Adding Comments<commit_after>package trees

import (
	"math"
	"math/rand"
	"time"

	"github.com/sjwhitworth/golearn/base"
)

type IsolationForest struct {
	nTrees   int
	maxDepth int
	subSpace int
	trees    []regressorNode
}

// Select A random feature for splitting from the data.
func selectFeature(data [][]float64) int64 {
	rand.Seed(time.Now().UnixNano())
	return int64(rand.Intn(len(data[0])))
}

// Find the minimum and maximum values of a feature. Used so that we can choose a random threshold.
func minMax(feature int64, data [][]float64) (float64, float64) {

	var min, max float64

	min = math.Inf(1)
	max = math.Inf(-1)
	for _, instance := range data {
		if instance[feature] > max {
			max = instance[feature]
		}
		if instance[feature] < min {
			min = instance[feature]
		}
	}

	return min, max
}

// Select a random threshold between the minimum and maximum of the feature.
func selectValue(min, max float64) float64 {
	rand.Seed(time.Now().UnixNano())
	val := min + (rand.Float64() * (max - min))
	if val == min {
		val += 0.000001
	} else if val == max {
		val -= 0.000001
	}
	return val
}

// Split the data based on the threshold.
func splitData(val float64, feature int64, data [][]float64) ([][]float64, [][]float64) {
	var leftData, rightData [][]float64
	for _, instance := range data {
		if instance[feature] <= val {
			leftData = append(leftData, instance)
		} else {
			rightData = append(rightData, instance)
		}
	}
	return leftData, rightData
}

// Make sure that the data can still be split (all datapoints are not duplicate)
func checkData(data [][]float64) bool {
	for _, instance := range data {
		for i, val := range instance {
			if val != data[0][i] {
				return true
			}
		}
	}
	return false
}

// Recusrively build a tree by randomly choosing a feature to split until nodes are pure or max depth is reached.
func buildTree(data [][]float64, upperNode regressorNode, depth int, maxDepth int) regressorNode {
	depth++

	upperNode.isNodeNeeded = true
	if (depth > maxDepth) || (len(data) <= 1) || !checkData(data) {
		upperNode.isNodeNeeded = false
		return upperNode
	}

	var featureToSplit int64
	var splitVal float64
	var min, max float64
	min, max = 0.0, 0.0

	for min == max {
		featureToSplit = selectFeature(data)
		min, max = minMax(featureToSplit, data)
		splitVal = selectValue(min, max)
	}

	leftData, rightData := splitData(splitVal, featureToSplit, data)

	upperNode.Feature = featureToSplit
	upperNode.Threshold = splitVal
	upperNode.LeftPred = float64(len(leftData))
	upperNode.RightPred = float64(len(rightData))

	var leftNode, rightNode regressorNode
	leftNode = buildTree(leftData, leftNode, depth, maxDepth)
	rightNode = buildTree(rightData, rightNode, depth, maxDepth)

	if leftNode.isNodeNeeded {
		upperNode.Left = &leftNode
	}
	if rightNode.isNodeNeeded {
		upperNode.Right = &rightNode
	}

	return upperNode
}

// Get a random subset of the data. Helps making each tree in forest different.
func getRandomData(data [][]float64, subSpace int) [][]float64 {
	var randomData [][]float64
	rand.Seed(time.Now().UnixNano())
	for i := 0; i < subSpace; i++ {
		randomData = append(randomData, data[rand.Intn(len(data))])
	}
	return randomData
}

// Function to create a new isolation forest. nTrees is number of trees in Forest. Maxdepth is maximum depth of each tree. Subspace is number of data points to use per tree.
func NewIsolationForest(nTrees int, maxDepth int, subSpace int) IsolationForest {
	var iForest IsolationForest
	iForest.nTrees = nTrees
	iForest.maxDepth = maxDepth
	iForest.subSpace = subSpace
	return iForest
}

// Fit the data based on hyperparameters and data.
func (iForest *IsolationForest) Fit(X base.FixedDataGrid) {
	data := preprocessData(X)
	nTrees := iForest.nTrees
	subSpace := iForest.subSpace
	maxDepth := iForest.maxDepth

	var forest []regressorNode
	for i := 0; i < nTrees; i++ {
		subData := getRandomData(data, subSpace)
		var tree regressorNode

		tree = buildTree(subData, tree, 0, maxDepth)
		forest = append(forest, tree)
	}
	iForest.trees = forest
}

// Calculate the path length to reach a leaf node for a datapoint. Outliers have smaller path lengths than standard data points.
func pathLength(tree regressorNode, instance []float64, path float64) float64 {
	path++

	if instance[tree.Feature] <= tree.Threshold {
		if tree.Left == nil {
			if tree.LeftPred <= 1 {
				return path
			} else {
				return path + cFactor(int(tree.LeftPred))
			}
		}
		path = pathLength(*tree.Left, instance, path)
	} else {
		if tree.Right == nil {
			if tree.RightPred <= 1 {
				return path
			} else {
				return path + cFactor(int(tree.RightPred))
			}
		}
		path = pathLength(*tree.Right, instance, path)
	}
	return path
}

// Find the path length of a a datapoints from all trees in forest.
func evaluateInstance(instance []float64, forest []regressorNode) []float64 {
	var paths []float64
	for _, tree := range forest {
		paths = append(paths, pathLength(tree, instance, 0))
	}
	return paths
}

// Helper function to calculate anomaly score.
func cFactor(n int) float64 {
	return 2.0*(math.Log(float64(n-1))+0.5772156649) - (float64(2.0*(n-1)) / float64(n))
}

// Anamoly Score - How anomalous is a data point. closer to 1 - higher chance of it being outlier. closer to 0 - low chance of it being outlier.
func anomalyScore(instance []float64, forest []regressorNode, n int) float64 {
	paths := evaluateInstance(instance, forest)
	E := 0.0
	for _, path := range paths {
		E += path
	}
	E /= float64(len(paths))
	c := cFactor(n)
	return math.Pow(2, (-1 * E / c))
}

// Return anamoly score for all datapoints.
func (iForest *IsolationForest) Predict(X base.FixedDataGrid) []float64 {
	data := preprocessData(X)

	var preds []float64
	for _, instance := range data {
		score := anomalyScore(instance, iForest.trees, iForest.subSpace)
		preds = append(preds, score)
	}
	return preds
}

// Extract data in the form of floats. Used in Fit and predict. Note that class labels are treated as normal data because Isolation Forest is unsupervised.
func preprocessData(X base.FixedDataGrid) [][]float64 {
	data := convertInstancesToProblemVec(X)
	class, err := regressorConvertInstancesToLabelVec(X)
	if err != nil {
		panic(err)
	}
	for i, point := range class {
		data[i] = append(data[i], point)
	}
	return data
}
<|endoftext|>


<commit_before>// Copyright 2017 Microsoft. All rights reserved.
// MIT License

package netlink

import (
	"fmt"
	"testing"
)

const ifname string = "nltest"

// Tests basic netlink messaging via echo.
func TestEcho(t *testing.T) {
	fmt.Println("Test: Echo")

	err := Echo("this is a test")

	if err != nil {
		t.Errorf("Echo failed: %+v", err)
	}
}

// Tests creating a new network interface.
func TestAddLink(t *testing.T) {
	fmt.Println("Test: AddLink")

	err := AddLink(ifname, "bridge")

	if err != nil {
		t.Errorf("AddLink failed: %+v", err)
	}
}

// Tests setting the operational state of a network interface.
func TestSetLinkState(t *testing.T) {
	fmt.Println("Test: SetLinkState")

	err := SetLinkState(ifname, true)

	if err != nil {
		t.Errorf("SetLinkState up failed: %+v", err)
	}

	err = SetLinkState(ifname, false)

	if err != nil {
		t.Errorf("SetLinkState down failed: %+v", err)
	}
}

// Tests deleting a network interface.
func TestDeleteLink(t *testing.T) {
	fmt.Println("Test: DeleteLink")

	err := DeleteLink(ifname)

	if err != nil {
		t.Errorf("DeleteLink failed: %+v", err)
	}
}
<commit_msg>Improved link tests<commit_after>// Copyright 2017 Microsoft. All rights reserved.
// MIT License

package netlink

import (
	"net"
	"testing"
)

const (
	ifName    = "nltest"
	ifName2   = "nltest2"
	dummyName = "dummy0"
)

// AddDummyInterface creates a dummy test interface used during actual tests.
func addDummyInterface(name string) (*net.Interface, error) {
	err := AddLink(&DummyLink{
		LinkInfo: LinkInfo{
			Type: LINK_TYPE_DUMMY,
			Name: name,
		},
	})

	if err != nil {
		return nil, err
	}

	dummy, err := net.InterfaceByName(name)
	if err != nil {
		return nil, err
	}

	return dummy, err
}

// TestEcho tests basic netlink messaging via echo.
func TestEcho(t *testing.T) {
	err := Echo("this is a test")

	if err != nil {
		t.Errorf("Echo failed: %+v", err)
	}
}

// TestAddDeleteBridge tests adding and deleting an ethernet bridge.
func TestAddDeleteBridge(t *testing.T) {
	link := BridgeLink{
		LinkInfo: LinkInfo{
			Type: LINK_TYPE_BRIDGE,
			Name: ifName,
		},
	}

	err := AddLink(&link)
	if err != nil {
		t.Errorf("AddLink failed: %+v", err)
	}

	err = DeleteLink(ifName)
	if err != nil {
		t.Errorf("DeleteLink failed: %+v", err)
	}

	_, err = net.InterfaceByName(ifName)
	if err == nil {
		t.Errorf("Interface not deleted")
	}
}

// TestAddDeleteVEth tests adding and deleting a virtual ethernet pair.
func TestAddDeleteVEth(t *testing.T) {
	link := VEthLink{
		LinkInfo: LinkInfo{
			Type: LINK_TYPE_VETH,
			Name: ifName,
		},
		PeerName: ifName2,
	}

	err := AddLink(&link)
	if err != nil {
		t.Errorf("AddLink failed: %+v", err)
	}

	err = DeleteLink(ifName)
	if err != nil {
		t.Errorf("DeleteLink failed: %+v", err)
	}

	_, err = net.InterfaceByName(ifName)
	if err == nil {
		t.Errorf("Interface not deleted")
	}
}

// TestAddDeleteIPVlan tests adding and deleting an IPVLAN interface.
func TestAddDeleteIPVlan(t *testing.T) {
	dummy, err := addDummyInterface(dummyName)
	if err != nil {
		t.Errorf("addDummyInterface failed: %v", err)
	}

	link := IPVlanLink{
		LinkInfo: LinkInfo{
			Type:        LINK_TYPE_IPVLAN,
			Name:        ifName,
			ParentIndex: dummy.Index,
		},
		Mode: IPVLAN_MODE_L2,
	}

	err = AddLink(&link)
	if err != nil {
		t.Errorf("AddLink failed: %+v", err)
	}

	err = DeleteLink(ifName)
	if err != nil {
		t.Errorf("DeleteLink failed: %+v", err)
	}

	_, err = net.InterfaceByName(ifName)
	if err == nil {
		t.Errorf("Interface not deleted")
	}

	err = DeleteLink(dummyName)
	if err != nil {
		t.Errorf("DeleteLink failed: %v", err)
	}
}

// TestSetLinkState tests setting the operational state of a network interface.
func TestSetLinkState(t *testing.T) {
	_, err := addDummyInterface(ifName)
	if err != nil {
		t.Errorf("addDummyInterface failed: %v", err)
	}

	err = SetLinkState(ifName, true)
	if err != nil {
		t.Errorf("SetLinkState up failed: %+v", err)
	}

	dummy, err := net.InterfaceByName(ifName)
	if err != nil || (dummy.Flags&net.FlagUp) == 0 {
		t.Errorf("Interface not up")
	}

	err = SetLinkState(ifName, false)
	if err != nil {
		t.Errorf("SetLinkState down failed: %+v", err)
	}

	dummy, err = net.InterfaceByName(ifName)
	if err != nil || (dummy.Flags&net.FlagUp) != 0 {
		t.Errorf("Interface not down")
	}

	err = DeleteLink(ifName)
	if err != nil {
		t.Errorf("DeleteLink failed: %+v", err)
	}
}
<|endoftext|>