<commit_before>//go:generate struct-markdown

package common

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/hashicorp/packer/template/interpolate"
)

// An iso (CD) containing custom files can be made available for your build.
//
// By default, no extra CD will be attached. All files listed in this setting
// get placed into the root directory of the CD and the CD is attached as the
// second CD device.
//
// This config exists to work around modern operating systems that have no
// way to mount floppy disks, which was our previous go-to for adding files at
// boot time.
type CDConfig struct {
	// A list of files to place onto a CD that is attached when the VM is
	// booted. This can include either files or directories; any directories
	// will be copied onto the CD recursively, preserving directory structure
	// hierarchy. Symlinks will have the link's target copied into the directory
	// tree on the CD where the symlink was. File globbing is allowed.
	//
	// Usage example (JSON):
	//
	// ```json
	// "cd_files": ["./somedirectory/meta-data", "./somedirectory/user-data"],
	// "cd_label": "cidata",
	// ```
	//
	// Usage example (HCL):
	//
	// ```hcl
	// cd_files = ["./somedirectory/meta-data", "./somedirectory/user-data"]
	// cd_label = "cidata"
	// ```
	//
	// The above will create a CD with two files, user-data and meta-data in the
	// CD root. This specific example is how you would create a CD that can be
	// used for an Ubuntu 20.04 autoinstall.
	//
	// Since globbing is also supported,
	//
	// ```hcl
	// cd_files = ["./somedirectory/*"]
	// cd_label = "cidata"
	// ```
	//
	// Would also be an acceptable way to define the above cd. The difference
	// between providing the directory with or without the glob is whether the
	// directory itself or its contents will be at the CD root.
	//
	// Use of this option assums that you have a command line tool isntalled
	// that can handle the iso creation. If you are running Packer from an OSX host,
	// the required tool is is hdiutil which comes preinstalled.
	// On linux hosts, you need to have mkisofs.
	// On Windows, you must have oscdimg.exe. oscdimg.exe is part of the
	// Windows ADK tooks, downloadable from
	// https://docs.microsoft.com/en-us/windows-hardware/get-started/adk-install#winADK
	CDFiles []string `mapstructure:"cd_files"`
	CDLabel string   `mapstructure:"cd_label"`
}

func (c *CDConfig) Prepare(ctx *interpolate.Context) []error {
	var errs []error
	var err error

	if c.CDFiles == nil {
		c.CDFiles = make([]string, 0)
	}

	// Create new file list based on globbing.
	var files []string
	for _, path := range c.CDFiles {
		if strings.ContainsAny(path, "*?[") {
			var globbedFiles []string
			globbedFiles, err = filepath.Glob(path)
			if len(globbedFiles) > 0 {
				files = append(files, globbedFiles...)
			}
		} else {
			_, err = os.Stat(path)
			files = append(files, path)
		}
		if err != nil {
			errs = append(errs, fmt.Errorf("Bad CD disk file '%s': %s", path, err))
		}
		c.CDFiles = files
	}

	return errs
}
<commit_msg>Update common/extra_iso_config.go<commit_after>//go:generate struct-markdown

package common

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/hashicorp/packer/template/interpolate"
)

// An iso (CD) containing custom files can be made available for your build.
//
// By default, no extra CD will be attached. All files listed in this setting
// get placed into the root directory of the CD and the CD is attached as the
// second CD device.
//
// This config exists to work around modern operating systems that have no
// way to mount floppy disks, which was our previous go-to for adding files at
// boot time.
type CDConfig struct {
	// A list of files to place onto a CD that is attached when the VM is
	// booted. This can include either files or directories; any directories
	// will be copied onto the CD recursively, preserving directory structure
	// hierarchy. Symlinks will have the link's target copied into the directory
	// tree on the CD where the symlink was. File globbing is allowed.
	//
	// Usage example (JSON):
	//
	// ```json
	// "cd_files": ["./somedirectory/meta-data", "./somedirectory/user-data"],
	// "cd_label": "cidata",
	// ```
	//
	// Usage example (HCL):
	//
	// ```hcl
	// cd_files = ["./somedirectory/meta-data", "./somedirectory/user-data"]
	// cd_label = "cidata"
	// ```
	//
	// The above will create a CD with two files, user-data and meta-data in the
	// CD root. This specific example is how you would create a CD that can be
	// used for an Ubuntu 20.04 autoinstall.
	//
	// Since globbing is also supported,
	//
	// ```hcl
	// cd_files = ["./somedirectory/*"]
	// cd_label = "cidata"
	// ```
	//
	// Would also be an acceptable way to define the above cd. The difference
	// between providing the directory with or without the glob is whether the
	// directory itself or its contents will be at the CD root.
	//
	// Use of this option assumes that you have a command line tool installed
	// that can handle the iso creation. If you are running Packer from an OSX host,
	// the required tool is hdiutil which comes preinstalled.
	// On linux hosts, you need to have mkisofs.
	// On Windows, you must have oscdimg.exe. oscdimg.exe is part of the
	// Windows ADK tooks, downloadable from
	// https://docs.microsoft.com/en-us/windows-hardware/get-started/adk-install#winADK
	CDFiles []string `mapstructure:"cd_files"`
	CDLabel string   `mapstructure:"cd_label"`
}

func (c *CDConfig) Prepare(ctx *interpolate.Context) []error {
	var errs []error
	var err error

	if c.CDFiles == nil {
		c.CDFiles = make([]string, 0)
	}

	// Create new file list based on globbing.
	var files []string
	for _, path := range c.CDFiles {
		if strings.ContainsAny(path, "*?[") {
			var globbedFiles []string
			globbedFiles, err = filepath.Glob(path)
			if len(globbedFiles) > 0 {
				files = append(files, globbedFiles...)
			}
		} else {
			_, err = os.Stat(path)
			files = append(files, path)
		}
		if err != nil {
			errs = append(errs, fmt.Errorf("Bad CD disk file '%s': %s", path, err))
		}
		c.CDFiles = files
	}

	return errs
}
<|endoftext|>


<commit_before>package clui

import (
	xs "github.com/huandu/xstrings"
	term "github.com/nsf/termbox-go"
)

/*
CheckBox control. It can be two-state one(on and off) - it is default mode - or tree-state.
State values are 0=off, 1=on, 2=third state
*/
type CheckBox struct {
	ControlBase
	state       int
	allow3state bool
}

/*
NewCheckBox creates a new CheckBox control.
view - is a View that manages the control.
parent - is container that keeps the control. The same View can be a view and a parent at the same time.
width - is minimal width of the control.
title - button title.
scale - the way of scaling the control when the parent is resized. Use DoNotScale constant if the
control should keep its original size.
CheckBox state can be changed using mouse or pressing space on keyboard while the control is active
*/
func NewCheckBox(view View, parent Control, width int, title string, scale int) *CheckBox {
	c := new(CheckBox)
	c.view = view
	c.parent = parent

	if width == AutoSize {
		width = xs.Len(title) + 4
	}

	c.SetSize(width, 1) // TODO: only one line checkboxes are supported at that moment
	c.SetConstraints(width, 1)
	c.state = 0
	c.SetTitle(title)
	c.SetTabStop(true)
	c.allow3state = false

	if parent != nil {
		parent.AddChild(c, scale)
	}

	return c
}

// Repaint draws the control on its View surface
func (c *CheckBox) Repaint() {
	x, y := c.Pos()
	w, h := c.Size()
	canvas := c.view.Canvas()
	tm := c.view.Screen().Theme()

	fg, bg := RealColor(tm, c.fg, ColorControlText), RealColor(tm, c.bg, ColorControlBack)
	if !c.Enabled() {
		fg, bg = RealColor(tm, c.fg, ColorControlDisabledText), RealColor(tm, c.bg, ColorControlDisabledBack)
	} else if c.Active() {
		fg, bg = RealColor(tm, c.fg, ColorControlActiveText), RealColor(tm, c.bg, ColorControlActiveBack)
	}

	parts := []rune(tm.SysObject(ObjCheckBox))

	cOpen, cClose, cEmpty, cCheck, cUnknown := parts[0], parts[1], parts[2], parts[3], parts[4]
	cState := []rune{cEmpty, cCheck, cUnknown}

	canvas.FillRect(x, y, w, h, term.Cell{Ch: ' ', Bg: bg})
	if w < 3 {
		return
	}

	canvas.PutSymbol(x, y, term.Cell{Ch: cOpen, Fg: fg, Bg: bg})
	canvas.PutSymbol(x+2, y, term.Cell{Ch: cClose, Fg: fg, Bg: bg})
	canvas.PutSymbol(x+1, y, term.Cell{Ch: cState[c.state], Fg: fg, Bg: bg})

	if w < 5 {
		return
	}

	shift, text := AlignText(c.title, w-4, c.align)
	canvas.PutText(x+4+shift, y, text, fg, bg)
}

//ProcessEvent processes all events come from the control parent. If a control
//   processes an event it should return true. If the method returns false it means
//   that the control do not want or cannot process the event and the caller sends
//   the event to the control parent
func (c *CheckBox) ProcessEvent(event Event) bool {
	if (!c.Active() && event.Type == EventKey) || !c.Enabled() {
		return false
	}

	if (event.Type == EventKey && event.Key == term.KeySpace) || event.Type == EventMouse {
		if c.state == 0 {
			c.state = 1
		} else if c.state == 2 {
			c.state = 0
		} else {
			if c.allow3state {
				c.state = 2
			} else {
				c.state = 0
			}
		}
		return true
	}

	return false
}

// SetState changes the current state of CheckBox
// Value must be 0 or 1 if Allow3State is off,
// and 0, 1, or 2 if Allow3State is on
func (c *CheckBox) SetState(val int) {
	if val < 0 {
		val = 0
	}
	if val > 1 && !c.allow3state {
		val = 1
	}
	if val > 2 {
		val = 2
	}

	c.state = val
}

// State returns current state of CheckBox
func (c *CheckBox) State() int {
	return c.state
}

// SetAllow3State sets if ComboBox should use 3 states. If the current
// state is unknown and one disables Allow3State option then the current
// value resets to off
func (c *CheckBox) SetAllow3State(enable bool) {
	if !enable && c.state == 2 {
		c.state = 0
	}
	c.allow3state = enable
}

// Allow3State returns true if ComboBox uses 3 states
func (c *CheckBox) Allow3State() bool {
	return c.allow3state
}

// SetSize changes control size. Constant DoNotChange can be
// used as placeholder to indicate that the control attrubute
// should be unchanged.
// Method does nothing if new size is less than minimal size
// CheckBox height cannot be changed - it equals 1 always
func (c *CheckBox) SetSize(width, height int) {
	if width != DoNotChange && (width > 1000 || width < c.minW) {
		return
	}
	if height != DoNotChange && (height > 200 || height < c.minH) {
		return
	}

	if width != DoNotChange {
		c.width = width
	}

	c.height = 1
}
<commit_msg>closes #47 - add event for checkbox state change<commit_after>package clui

import (
	xs "github.com/huandu/xstrings"
	term "github.com/nsf/termbox-go"
)

/*
CheckBox control. It can be two-state one(on and off) - it is default mode - or tree-state.
State values are 0=off, 1=on, 2=third state
*/
type CheckBox struct {
	ControlBase
	state       int
	allow3state bool

	onChange func(int)
}

/*
NewCheckBox creates a new CheckBox control.
view - is a View that manages the control.
parent - is container that keeps the control. The same View can be a view and a parent at the same time.
width - is minimal width of the control.
title - button title.
scale - the way of scaling the control when the parent is resized. Use DoNotScale constant if the
control should keep its original size.
CheckBox state can be changed using mouse or pressing space on keyboard while the control is active
*/
func NewCheckBox(view View, parent Control, width int, title string, scale int) *CheckBox {
	c := new(CheckBox)
	c.view = view
	c.parent = parent

	if width == AutoSize {
		width = xs.Len(title) + 4
	}

	c.SetSize(width, 1) // TODO: only one line checkboxes are supported at that moment
	c.SetConstraints(width, 1)
	c.state = 0
	c.SetTitle(title)
	c.SetTabStop(true)
	c.allow3state = false
	c.onChange = nil

	if parent != nil {
		parent.AddChild(c, scale)
	}

	return c
}

// Repaint draws the control on its View surface
func (c *CheckBox) Repaint() {
	x, y := c.Pos()
	w, h := c.Size()
	canvas := c.view.Canvas()
	tm := c.view.Screen().Theme()

	fg, bg := RealColor(tm, c.fg, ColorControlText), RealColor(tm, c.bg, ColorControlBack)
	if !c.Enabled() {
		fg, bg = RealColor(tm, c.fg, ColorControlDisabledText), RealColor(tm, c.bg, ColorControlDisabledBack)
	} else if c.Active() {
		fg, bg = RealColor(tm, c.fg, ColorControlActiveText), RealColor(tm, c.bg, ColorControlActiveBack)
	}

	parts := []rune(tm.SysObject(ObjCheckBox))

	cOpen, cClose, cEmpty, cCheck, cUnknown := parts[0], parts[1], parts[2], parts[3], parts[4]
	cState := []rune{cEmpty, cCheck, cUnknown}

	canvas.FillRect(x, y, w, h, term.Cell{Ch: ' ', Bg: bg})
	if w < 3 {
		return
	}

	canvas.PutSymbol(x, y, term.Cell{Ch: cOpen, Fg: fg, Bg: bg})
	canvas.PutSymbol(x+2, y, term.Cell{Ch: cClose, Fg: fg, Bg: bg})
	canvas.PutSymbol(x+1, y, term.Cell{Ch: cState[c.state], Fg: fg, Bg: bg})

	if w < 5 {
		return
	}

	shift, text := AlignText(c.title, w-4, c.align)
	canvas.PutText(x+4+shift, y, text, fg, bg)
}

//ProcessEvent processes all events come from the control parent. If a control
//   processes an event it should return true. If the method returns false it means
//   that the control do not want or cannot process the event and the caller sends
//   the event to the control parent
func (c *CheckBox) ProcessEvent(event Event) bool {
	if (!c.Active() && event.Type == EventKey) || !c.Enabled() {
		return false
	}

	if (event.Type == EventKey && event.Key == term.KeySpace) || event.Type == EventMouse {
		if c.state == 0 {
			c.SetState(1)
		} else if c.state == 2 {
			c.SetState(0)
		} else {
			if c.allow3state {
				c.SetState(2)
			} else {
				c.SetState(0)
			}
		}
		return true
	}

	return false
}

// SetState changes the current state of CheckBox
// Value must be 0 or 1 if Allow3State is off,
// and 0, 1, or 2 if Allow3State is on
func (c *CheckBox) SetState(val int) {
	if val == c.state {
		return
	}

	if val < 0 {
		val = 0
	}
	if val > 1 && !c.allow3state {
		val = 1
	}
	if val > 2 {
		val = 2
	}

	c.state = val

	if c.onChange != nil {
		go c.onChange(val)
	}
}

// State returns current state of CheckBox
func (c *CheckBox) State() int {
	return c.state
}

// SetAllow3State sets if ComboBox should use 3 states. If the current
// state is unknown and one disables Allow3State option then the current
// value resets to off
func (c *CheckBox) SetAllow3State(enable bool) {
	if !enable && c.state == 2 {
		c.state = 0
	}
	c.allow3state = enable
}

// Allow3State returns true if ComboBox uses 3 states
func (c *CheckBox) Allow3State() bool {
	return c.allow3state
}

// SetSize changes control size. Constant DoNotChange can be
// used as placeholder to indicate that the control attrubute
// should be unchanged.
// Method does nothing if new size is less than minimal size
// CheckBox height cannot be changed - it equals 1 always
func (c *CheckBox) SetSize(width, height int) {
	if width != DoNotChange && (width > 1000 || width < c.minW) {
		return
	}
	if height != DoNotChange && (height > 200 || height < c.minH) {
		return
	}

	if width != DoNotChange {
		c.width = width
	}

	c.height = 1
}

// OnChange sets the callback that is called whenever the state
// of the CheckBox is changed. Argument of callback is the current
// CheckBox state: 0 - off, 1 - on, 2 - third state
func (c *CheckBox) OnChange(fn func(int)) {
	c.onChange = fn
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
)

func cmdCheckout() int {

	// git-lob checkout [options] [<pathspec>...]

	// no custom options
	optDryRun := GlobalOptions.DryRun

	// All extra arguments must be <pathspec>
	var pathspecs []string
	for _, arg := range GlobalOptions.Args[1:] {
		p := filepath.Clean(arg)
		pathspecs = append(pathspecs, p)
	}

	err := Checkout(pathspecs, optDryRun)

	if err != nil {
		fmt.Fprintf(os.Stderr, "git-lob: checkout error - %v", err.Error())
		return 7
	}

	return 0
}

// Populate local placeholders with real content, if available. Do entire working copy unless limited to pathspecs
func Checkout(pathspecs []string, dryRun bool) error {
	// We're going to scan for missing git-lob content not just by checking the working copy, but
	// getting the expected content from git first. This is in case the working copy has had files
	// deleted for example. We still check the content of the working copy if the file IS there
	// in order to not overwrite modified files.

	// firstly convert any pathspecs to the root of the repo, in case this is being executed in a sub-folder
	reporoot, _ := GetRepoRoot()
	if reporoot == "" {
		return errors.New("Not in git repository?")
	}
	curdir, err := os.Getwd()
	if err != nil {
		return err
	}
	var rootedpathspecs []string
	for _, p := range pathspecs {
		var abs string
		if filepath.IsAbs(p) {
			abs = p
		} else {
			abs = filepath.Join(curdir, p)
		}
		reltoroot, err := filepath.Rel(reporoot, abs)
		if err != nil {
			return errors.New(fmt.Sprintf("Unable to make %v relative to repo root %v", p, reporoot))
		}
		rootedpathspecs = append(rootedpathspecs, reltoroot)
	}

	// Get what git thinks we should have
	filelobs, err := GetGitAllFilesAndLOBsToCheckoutAtCommit("HEAD", rootedpathspecs, nil)
	if err != nil {
		return err
	}
	var filesOK int
	var filesNotOK int
	for _, filelob := range filelobs {
		// Check each file, and if it's missing or contains the placeholder text, replace it with content
		// Otherwise, assume it's been locally modified and leave it alone (user can override this with git reset/checkout if they want)
		absfile := filepath.Join(reporoot, filelob.Filename)
		stat, err := os.Stat(absfile)
		replaceContent := false
		if err == nil {
			// File existed, check content (smoke test on size)
			if stat.Size() == int64(SHALineLen) {
				// File existed and is right size for placeholder, so check contents
				placeholderContent := getLOBPlaceholderContent(filelob.SHA)
				filebytes, err := ioutil.ReadFile(absfile)
				if err == nil && string(filebytes) == placeholderContent {
					// File content is placeholder, so replace
					replaceContent = true
				}
			}
		} else {
			// File did not exist
			replaceContent = true
		}

		if replaceContent {
			if dryRun {
				if GlobalOptions.Verbose {
					fmt.Println("Checkout:", filelob.Filename)
				}

			} else {
				err = os.MkdirAll(filepath.Dir(absfile), 0755)
				if err != nil {
					// This is not fatal but log it
					LogErrorf("ERROR: can't create parent directory of %v: %v\n", absfile, err.Error())
					continue
				}
				f, err := os.OpenFile(absfile, os.O_CREATE|os.O_TRUNC, 0644)
				if err != nil {
					// This is not fatal but log it
					LogErrorf("ERROR: can't open %v for writing: %v", absfile, err.Error())
					continue
				}
				_, err = RetrieveLOB(filelob.SHA, f)
				if err != nil {
					// This is not fatal but log it
					LogErrorf("ERROR: can't retrieve content for %v: %v", filelob.Filename, err.Error())
					continue
				}

			}
			filesOK++
		}

		if !GlobalOptions.Quiet {
			if dryRun {
				fmt.Println(filesOK, "files need updating")
				fmt.Println("Run this command again without --dry-run to update these files.")
			} else {
				fmt.Println(filesOK, "files were updated")
				if filesNotOK > 0 {
					fmt.Println("WARNING:", filesNotOK, "failed to be updated, check errors above")
				}
			}
		}
	}

	if filesNotOK > 0 {
		return errors.New(fmt.Sprintf("%d files failed", filesNotOK))
	}

	return nil

}

func cmdCheckoutHelp() {
	fmt.Println(`Usage: git-lob checkout [options] [<pathspec>...]

  Populate files in the working copy with binary content where they
  currently just have placeholder content, because the real content wasn't
  available.

  NOTE: You probably won't need to run this command yourself.

        Running 'git lob pull' will both fetch (download) AND checkout, so
        most of the time you should use 'git lob pull' instead.

        Also 'git checkout' will populate the binary content correctly if
        you have it locally so you don't have to run this command after
        switching branches, unless you need to download extra content, in
        which case 'git lob pull' is once again a better bet.

  Because git-lob stores binary content separately from your git repository,
  it's possible that when you perform a 'git checkout' or 'git clone', you did
  not have the binary content available locally to populate binary files in
  your working copy. In this situation, git-lob creates placeholders in the
  working copy, whose content looks something like this:

  git-lob: <sha>

  Where <sha> is the identifier of the content of the binary file. Once you
  have downloaded the content (e.g. via 'git lob fetch'), you can then use
  'git lob checkout' to fill in these blanks.

  Specify <pathspec> to limit the checking to particular files or directories.

  Options:
    --quiet, -q   Print less output
    --verbose, -v Print more output
    --dry-run     Don't actually change any files, just report

`)
}
<commit_msg>Make sure we close the file on checkout<commit_after>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
)

func cmdCheckout() int {

	// git-lob checkout [options] [<pathspec>...]

	// no custom options
	optDryRun := GlobalOptions.DryRun

	// All extra arguments must be <pathspec>
	var pathspecs []string
	for _, arg := range GlobalOptions.Args[1:] {
		p := filepath.Clean(arg)
		pathspecs = append(pathspecs, p)
	}

	err := Checkout(pathspecs, optDryRun)

	if err != nil {
		fmt.Fprintf(os.Stderr, "git-lob: checkout error - %v", err.Error())
		return 7
	}

	return 0
}

// Populate local placeholders with real content, if available. Do entire working copy unless limited to pathspecs
func Checkout(pathspecs []string, dryRun bool) error {
	// We're going to scan for missing git-lob content not just by checking the working copy, but
	// getting the expected content from git first. This is in case the working copy has had files
	// deleted for example. We still check the content of the working copy if the file IS there
	// in order to not overwrite modified files.

	// firstly convert any pathspecs to the root of the repo, in case this is being executed in a sub-folder
	reporoot, _ := GetRepoRoot()
	if reporoot == "" {
		return errors.New("Not in git repository?")
	}
	curdir, err := os.Getwd()
	if err != nil {
		return err
	}
	var rootedpathspecs []string
	for _, p := range pathspecs {
		var abs string
		if filepath.IsAbs(p) {
			abs = p
		} else {
			abs = filepath.Join(curdir, p)
		}
		reltoroot, err := filepath.Rel(reporoot, abs)
		if err != nil {
			return errors.New(fmt.Sprintf("Unable to make %v relative to repo root %v", p, reporoot))
		}
		rootedpathspecs = append(rootedpathspecs, reltoroot)
	}

	// Get what git thinks we should have
	filelobs, err := GetGitAllFilesAndLOBsToCheckoutAtCommit("HEAD", rootedpathspecs, nil)
	if err != nil {
		return err
	}
	var filesOK int
	var filesNotOK int
	for _, filelob := range filelobs {
		// Check each file, and if it's missing or contains the placeholder text, replace it with content
		// Otherwise, assume it's been locally modified and leave it alone (user can override this with git reset/checkout if they want)
		absfile := filepath.Join(reporoot, filelob.Filename)
		stat, err := os.Stat(absfile)
		replaceContent := false
		if err == nil {
			// File existed, check content (smoke test on size)
			if stat.Size() == int64(SHALineLen) {
				// File existed and is right size for placeholder, so check contents
				placeholderContent := getLOBPlaceholderContent(filelob.SHA)
				filebytes, err := ioutil.ReadFile(absfile)
				if err == nil && string(filebytes) == placeholderContent {
					// File content is placeholder, so replace
					replaceContent = true
				}
			}
		} else {
			// File did not exist
			replaceContent = true
		}

		if replaceContent {
			if dryRun {
				if GlobalOptions.Verbose {
					fmt.Println("Checkout:", filelob.Filename)
				}

			} else {
				err = os.MkdirAll(filepath.Dir(absfile), 0755)
				if err != nil {
					// This is not fatal but log it
					LogErrorf("ERROR: can't create parent directory of %v: %v\n", absfile, err.Error())
					continue
				}
				f, err := os.OpenFile(absfile, os.O_CREATE|os.O_TRUNC, 0644)
				if err != nil {
					// This is not fatal but log it
					LogErrorf("ERROR: can't open %v for writing: %v", absfile, err.Error())
					continue
				}
				_, err = RetrieveLOB(filelob.SHA, f)
				f.Close()
				if err != nil {
					// This is not fatal but log it
					LogErrorf("ERROR: can't retrieve content for %v: %v", filelob.Filename, err.Error())
					continue
				}

			}
			filesOK++
		}

		if !GlobalOptions.Quiet {
			if dryRun {
				fmt.Println(filesOK, "files need updating")
				fmt.Println("Run this command again without --dry-run to update these files.")
			} else {
				fmt.Println(filesOK, "files were updated")
				if filesNotOK > 0 {
					fmt.Println("WARNING:", filesNotOK, "failed to be updated, check errors above")
				}
			}
		}
	}

	if filesNotOK > 0 {
		return errors.New(fmt.Sprintf("%d files failed", filesNotOK))
	}

	return nil

}

func cmdCheckoutHelp() {
	fmt.Println(`Usage: git-lob checkout [options] [<pathspec>...]

  Populate files in the working copy with binary content where they
  currently just have placeholder content, because the real content wasn't
  available.

  NOTE: You probably won't need to run this command yourself.

        Running 'git lob pull' will both fetch (download) AND checkout, so
        most of the time you should use 'git lob pull' instead.

        Also 'git checkout' will populate the binary content correctly if
        you have it locally so you don't have to run this command after
        switching branches, unless you need to download extra content, in
        which case 'git lob pull' is once again a better bet.

  Because git-lob stores binary content separately from your git repository,
  it's possible that when you perform a 'git checkout' or 'git clone', you did
  not have the binary content available locally to populate binary files in
  your working copy. In this situation, git-lob creates placeholders in the
  working copy, whose content looks something like this:

  git-lob: <sha>

  Where <sha> is the identifier of the content of the binary file. Once you
  have downloaded the content (e.g. via 'git lob fetch'), you can then use
  'git lob checkout' to fill in these blanks.

  Specify <pathspec> to limit the checking to particular files or directories.

  Options:
    --quiet, -q   Print less output
    --verbose, -v Print more output
    --dry-run     Don't actually change any files, just report

`)
}
<|endoftext|>


<commit_before>package ei

import (
	"net/http"
	"bytes"
	"github.com/nightrune/wrench/logging"
  "encoding/base64"
  "encoding/json"
  "errors"
  "net/http/httputil"
  "io/ioutil"
)

const EI_URL = "https://build.electricimp.com/v4/"
const MODELS_ENDPOINT = "models"
const MODELS_REVISIONS_ENDPOINT = "revisions"
const DEVICES_ENDPOINT = "devices"
const DEVICES_LOG_ENDPOINT = "logs"
const MODELS_DEVICE_RESTART_ENDPOINT = "restart"

type DeviceListResponse struct {
  Success bool `json:"success"`
  Devices []Device `json:"devices"`
}

type Device struct {
  Id string `json:"id,omitempty"`
  Name string `json:"name,omitempty"`
  ModelId string `json:"model_id,omitempty"`
  PowerState string `json:"powerstate,omitempty"`
  Rssi int `json:"rssi,omitempty"`
  AgentId string `json:"agent_id,omitempty"`
  AgentStatus string `json:"agent_status,omitempty"`
}

type Model struct {
  Id string `json:"id,omitempty"`
  Name string `json:"name"`
  Devices []string `json:"devices,omitempty"`
}

type ModelList struct {
  Models []Model `json:"models"`
}

type ModelResponse struct {
  Model Model `json:"model"`
  Success bool `json:"success"`
}

type BuildError struct {
	Code string `json:"code"`
	ShortMessage string `json:"message_short"`
}

type CodeRevisionResponse struct {
  Success bool `json:"success"`
  Revisions CodeRevisionLong `json:"revision"`
  Error BuildError `json:"error"`
}

type CodeRevisionsResponse struct {
  Success bool `json:"success"`
  Revisions []CodeRevisionShort `json:"revisions"`
}

type CodeRevisionShort struct {
  Version int `json:"version"`
  CreatedAt string `json:"created_at"`
  ReleaseNotes string `json:"release_notes"`
}

type CodeRevisionLong struct {
  Version int `json:"version,omitempty"`
  CreatedAt string `json:"created_at,omitempty"`
  DeviceCode string `json:"device_code,omitempty"`
  AgentCode string `json:"agent_code,omitempty"`
  ReleaseNotes string `json:"release_notes,omitempty"`
}

type BuildClient struct {
  creds string
  http_client *http.Client
}

type DeviceLogEntry struct {
  Timestamp string `json:"timestamp"`
  Type string `json:"type"`
  Message string `json:"message"`
}

type DeviceLogResponse struct {
  Logs []DeviceLogEntry `json:"logs"`
  PollUrl string `json:"poll_url"`
  Success bool `json:"success"`
}

func NewBuildClient(api_key string) *BuildClient {
  client := new(BuildClient)
  client.http_client = &http.Client{}
  cred_data := []byte(api_key)
  client.creds = base64.StdEncoding.EncodeToString(cred_data)
  return client
}

func Concat(a string, b string) string {
	var buffer bytes.Buffer
	buffer.WriteString(a)
	buffer.WriteString(b)
    return buffer.String()
}

func (m BuildClient) SetAuthHeader(request *http.Request) {
  request.Header.Set("Authorization", "Basic " + m.creds)
}

func (m *BuildClient) _complete_request(method string,
	url string, data []byte)  ([]byte, error) {
  var req *http.Request
  if data != nil {
  	req, _ = http.NewRequest(method, url, bytes.NewBuffer(data))
  }  else {
  	req, _ = http.NewRequest(method, url, nil)
  }

  m.SetAuthHeader(req)
  req.Header.Set("Content-Type", "application/json")
  resp, err := m.http_client.Do(req)
  if err == nil {
    dump, err := httputil.DumpResponse(resp, true)
    logging.Debug(string(dump))
    full_response, err := ioutil.ReadAll(resp.Body)
    if err != nil {
    	return full_response, err
    }
    return full_response, nil
  } else {
  	return nil, err
  }
}

func (m *BuildClient) ListModels() (*ModelList, error) {
  list := new(ModelList)
  full_resp, err := m._complete_request("GET", Concat(EI_URL, "models"), nil)
  if err != nil {
  	logging.Debug("An error happened during model get, %s", err.Error())
  	return list, err
  }

  if err := json.Unmarshal(full_resp, list); err != nil {
    logging.Warn("Failed to unmarshal data from models.. %s", err.Error());
    return list, err
  }

  return list, nil
}

func (m *BuildClient) CreateModel(new_model *Model) (*Model, error) {
  var url bytes.Buffer
  resp := new(ModelResponse)
  url.WriteString(EI_URL)
  url.WriteString(MODELS_ENDPOINT)

  req_string, err := json.Marshal(new_model)
  logging.Debug("Request String for upload: %s", req_string)
  full_resp, err := m._complete_request("POST", url.String(), req_string)
  if err != nil {
    logging.Debug("An error happened during model creation, %s", err.Error())
    return &resp.Model, err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from model response.. %s", err.Error());
    return &resp.Model, err
  }

  return &resp.Model, nil
}

func (m *BuildClient) UpdateModel(model_id string, new_model *Model) (*Model, error) {
  var url bytes.Buffer
  resp := new(ModelResponse)
  url.WriteString(EI_URL)
  url.WriteString(MODELS_ENDPOINT)
  url.WriteString("/")
  url.WriteString(model_id)

  req_string, err := json.Marshal(new_model)
  logging.Debug("Request String for upload: %s", req_string)
  full_resp, err := m._complete_request("PUT", url.String(), req_string)
  if err != nil {
    logging.Debug("An error happened during model creation, %s", err.Error())
    return &resp.Model, err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from model response.. %s", err.Error());
    return &resp.Model, err
  }

  return &resp.Model, nil
}

func (m *BuildClient) DeleteModel(model_id string) (error) {
  var url bytes.Buffer
  resp := new(ModelResponse)
  url.WriteString(EI_URL)
  url.WriteString(MODELS_ENDPOINT)
  url.WriteString("/")
  url.WriteString(model_id)

  full_resp, err := m._complete_request("DELETE", url.String(), nil)
  if err != nil {
    logging.Debug("An error happened during model deletion, %s", err.Error())
    return err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from model response.. %s", err.Error());
    return err
  }

  if resp.Success == false {
    return errors.New("Error When retriveing Code Revisions")
  }

  return nil
}

func (m *BuildClient) RestartModelDevices(model_id string) (error) {
  var url bytes.Buffer
  resp := new(ModelResponse)
  url.WriteString(EI_URL)
  url.WriteString(MODELS_ENDPOINT)
  url.WriteString("/")
  url.WriteString(model_id)
  url.WriteString("/")
  url.WriteString(MODELS_DEVICE_RESTART_ENDPOINT)

  full_resp, err := m._complete_request("POST", url.String(), nil)
  if err != nil {
    logging.Debug("An error happened during model restart, %s", err.Error())
    return err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from model response.. %s", err.Error());
    return err
  }

  if resp.Success == false {
    return errors.New("Error When retriveing Code Revisions")
  }

  return nil
}

func (m *BuildClient) GetCodeRevisionList(model_id string) (
	[]CodeRevisionShort, error) {
  var url bytes.Buffer
  resp := new(CodeRevisionsResponse)
  url.WriteString(EI_URL)
  url.WriteString(MODELS_ENDPOINT)
  url.WriteString("/")
  url.WriteString(model_id)
  url.WriteString("/")
  url.WriteString(MODELS_REVISIONS_ENDPOINT)
  full_resp, err := m._complete_request("GET", url.String(), nil)
  if err != nil {
  	logging.Debug("Failed to get code revisions: %s", err.Error())
  	return resp.Revisions, err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from code revision.. %s", err.Error());
    return resp.Revisions, err
  }

  if resp.Success == false {
  	return resp.Revisions, errors.New("Error When retriveing Code Revisions")
  }
  return resp.Revisions, nil
}

func (m *BuildClient) GetCodeRevision(model_id string, build_num string) (CodeRevisionLong, error) {
  var url bytes.Buffer
  resp := new(CodeRevisionResponse)
  url.WriteString(EI_URL)
  url.WriteString(MODELS_ENDPOINT)
  url.WriteString("/")
  url.WriteString(model_id)
  url.WriteString("/")
  url.WriteString(MODELS_REVISIONS_ENDPOINT)
  url.WriteString("/")
  url.WriteString(build_num)
  full_resp, err := m._complete_request("GET", url.String(), nil)
  if err != nil {
  	logging.Debug("Failed to get code revisions: %s", err.Error())
  	return resp.Revisions, err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from code revision.. %s", err.Error());
    return resp.Revisions, err
  }

  if resp.Success == false {
  	return resp.Revisions, errors.New("Error When retriveing Code Revisions")
  }
  return resp.Revisions, nil
}

func (m *BuildClient) UpdateCodeRevision(model_id string,
	request *CodeRevisionLong) (CodeRevisionLong, error) {
  var url bytes.Buffer
  resp := new(CodeRevisionResponse)
  url.WriteString(EI_URL)
  url.WriteString(MODELS_ENDPOINT)
  url.WriteString("/")
  url.WriteString(model_id)
  url.WriteString("/")
  url.WriteString(MODELS_REVISIONS_ENDPOINT)

  req_string, err := json.Marshal(request)
  logging.Debug("Request String for upload: %s", req_string)
  full_resp, err := m._complete_request("POST", url.String(), req_string)
  if err != nil {
  	logging.Debug("Failed to update code revisions: %s", err.Error())
  	return resp.Revisions, err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from code revision update.. %s", err.Error());
    return resp.Revisions, err
  }

  if resp.Success == false {
  	return resp.Revisions, errors.New("Error When retriveing Code Revisions")
  }
  return resp.Revisions, nil
}

func (m *BuildClient) GetDeviceList() ([]Device, error) {
  var url bytes.Buffer
  resp := new(DeviceListResponse)
  url.WriteString(EI_URL)
  url.WriteString(DEVICES_ENDPOINT)

  full_resp, err := m._complete_request("GET", url.String(), nil)
  if err != nil {
  	logging.Debug("Failed to get device list: %s", err.Error())
  	return resp.Devices, err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from code revision update.. %s", err.Error());
    return resp.Devices, err
  }

  if resp.Success == false {
  	return resp.Devices, errors.New("Error When retriveing Code Revisions")
  }
  return resp.Devices, nil
}

func (m *BuildClient) GetDeviceLogs(device_id string) ([]DeviceLogEntry, error) {
  var url bytes.Buffer
  resp := new(DeviceLogResponse)
  url.WriteString(EI_URL)
  url.WriteString(DEVICES_ENDPOINT)
  url.WriteString("/")
  url.WriteString(device_id)
  url.WriteString("/")
  url.WriteString(DEVICES_LOG_ENDPOINT)
  full_resp, err := m._complete_request("GET", url.String(), nil)
  if err != nil {
  	logging.Debug("Failed to get device logs: %s", err.Error())
  	return resp.Logs, err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from device logs.. %s", err.Error());
    return resp.Logs, err
  }

  if resp.Success == false {
  	return resp.Logs, errors.New("Error When retriveing device logs")
  }
  return resp.Logs, nil
}

type DeviceResponse struct {
  Success bool `json:"success"`
  Device Device `json:"device"`
}

func (m *BuildClient) GetDevice(device_id string) (Device, error) {
  var url bytes.Buffer
  resp := new(DeviceResponse)
  url.WriteString(EI_URL)
  url.WriteString(DEVICES_ENDPOINT)
  url.WriteString("/")
  url.WriteString(device_id)

  full_resp, err := m._complete_request("GET", url.String(), nil)
  if err != nil {
    logging.Debug("Failed to get device list: %s", err.Error())
    return resp.Device, err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from code revision update.. %s", err.Error());
    return resp.Device, err
  }

  if resp.Success == false {
    return resp.Device, errors.New("Error When retriveing Code Revisions")
  }
  return resp.Device, nil
}

const DEVICES_RESTART_ENDPOINT = "restart"
func (m *BuildClient) RestartDevice(device_id string) (error) {
  var url bytes.Buffer
  resp := new(DeviceResponse)
  url.WriteString(EI_URL)
  url.WriteString(DEVICES_ENDPOINT)
  url.WriteString("/")
  url.WriteString(device_id)
  url.WriteString("/")
  url.WriteString(DEVICES_RESTART_ENDPOINT)

  full_resp, err := m._complete_request("POST", url.String(), nil)
  if err != nil {
    logging.Debug("Failed to get device list: %s", err.Error())
    return err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from code revision update.. %s", err.Error());
    return err
  }

  if resp.Success == false {
    return errors.New("Error When retriveing Code Revisions")
  }
  return nil
}

func (m *BuildClient) UpdateDevice(new_device *Device, device_id string) (Device, error) {
  var url bytes.Buffer
  resp := new(DeviceResponse)
  url.WriteString(EI_URL)
  url.WriteString(DEVICES_ENDPOINT)
  url.WriteString("/")
  url.WriteString(device_id)

  req_bytes, err := json.Marshal(new_device)
  full_resp, err := m._complete_request("PUT", url.String(), req_bytes)
  if err != nil {
    logging.Debug("Failed to update device: %s", err.Error())
    return resp.Device, err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from device update.. %s", err.Error());
    return resp.Device, err
  }

  if resp.Success == false {
    return resp.Device, errors.New("Error when updating device")
  }
  return resp.Device, nil
}<commit_msg>Added support for device deletion<commit_after>package ei

import (
	"net/http"
	"bytes"
	"github.com/nightrune/wrench/logging"
  "encoding/base64"
  "encoding/json"
  "errors"
  "net/http/httputil"
  "io/ioutil"
)

const EI_URL = "https://build.electricimp.com/v4/"
const MODELS_ENDPOINT = "models"
const MODELS_REVISIONS_ENDPOINT = "revisions"
const DEVICES_ENDPOINT = "devices"
const DEVICES_LOG_ENDPOINT = "logs"
const MODELS_DEVICE_RESTART_ENDPOINT = "restart"

type DeviceListResponse struct {
  Success bool `json:"success"`
  Devices []Device `json:"devices"`
}

type Device struct {
  Id string `json:"id,omitempty"`
  Name string `json:"name,omitempty"`
  ModelId string `json:"model_id,omitempty"`
  PowerState string `json:"powerstate,omitempty"`
  Rssi int `json:"rssi,omitempty"`
  AgentId string `json:"agent_id,omitempty"`
  AgentStatus string `json:"agent_status,omitempty"`
}

type Model struct {
  Id string `json:"id,omitempty"`
  Name string `json:"name"`
  Devices []string `json:"devices,omitempty"`
}

type ModelList struct {
  Models []Model `json:"models"`
}

type ModelResponse struct {
  Model Model `json:"model"`
  Success bool `json:"success"`
}

type BuildError struct {
	Code string `json:"code"`
	ShortMessage string `json:"message_short"`
}

type CodeRevisionResponse struct {
  Success bool `json:"success"`
  Revisions CodeRevisionLong `json:"revision"`
  Error BuildError `json:"error"`
}

type CodeRevisionsResponse struct {
  Success bool `json:"success"`
  Revisions []CodeRevisionShort `json:"revisions"`
}

type CodeRevisionShort struct {
  Version int `json:"version"`
  CreatedAt string `json:"created_at"`
  ReleaseNotes string `json:"release_notes"`
}

type CodeRevisionLong struct {
  Version int `json:"version,omitempty"`
  CreatedAt string `json:"created_at,omitempty"`
  DeviceCode string `json:"device_code,omitempty"`
  AgentCode string `json:"agent_code,omitempty"`
  ReleaseNotes string `json:"release_notes,omitempty"`
}

type BuildClient struct {
  creds string
  http_client *http.Client
}

type DeviceLogEntry struct {
  Timestamp string `json:"timestamp"`
  Type string `json:"type"`
  Message string `json:"message"`
}

type DeviceLogResponse struct {
  Logs []DeviceLogEntry `json:"logs"`
  PollUrl string `json:"poll_url"`
  Success bool `json:"success"`
}

func NewBuildClient(api_key string) *BuildClient {
  client := new(BuildClient)
  client.http_client = &http.Client{}
  cred_data := []byte(api_key)
  client.creds = base64.StdEncoding.EncodeToString(cred_data)
  return client
}

func Concat(a string, b string) string {
	var buffer bytes.Buffer
	buffer.WriteString(a)
	buffer.WriteString(b)
    return buffer.String()
}

func (m BuildClient) SetAuthHeader(request *http.Request) {
  request.Header.Set("Authorization", "Basic " + m.creds)
}

func (m *BuildClient) _complete_request(method string,
	url string, data []byte)  ([]byte, error) {
  var req *http.Request
  if data != nil {
  	req, _ = http.NewRequest(method, url, bytes.NewBuffer(data))
  }  else {
  	req, _ = http.NewRequest(method, url, nil)
  }

  m.SetAuthHeader(req)
  req.Header.Set("Content-Type", "application/json")
  resp, err := m.http_client.Do(req)
  if err == nil {
    dump, err := httputil.DumpResponse(resp, true)
    logging.Debug(string(dump))
    full_response, err := ioutil.ReadAll(resp.Body)
    if err != nil {
    	return full_response, err
    }
    return full_response, nil
  } else {
  	return nil, err
  }
}

func (m *BuildClient) ListModels() (*ModelList, error) {
  list := new(ModelList)
  full_resp, err := m._complete_request("GET", Concat(EI_URL, "models"), nil)
  if err != nil {
  	logging.Debug("An error happened during model get, %s", err.Error())
  	return list, err
  }

  if err := json.Unmarshal(full_resp, list); err != nil {
    logging.Warn("Failed to unmarshal data from models.. %s", err.Error());
    return list, err
  }

  return list, nil
}

func (m *BuildClient) CreateModel(new_model *Model) (*Model, error) {
  var url bytes.Buffer
  resp := new(ModelResponse)
  url.WriteString(EI_URL)
  url.WriteString(MODELS_ENDPOINT)

  req_string, err := json.Marshal(new_model)
  logging.Debug("Request String for upload: %s", req_string)
  full_resp, err := m._complete_request("POST", url.String(), req_string)
  if err != nil {
    logging.Debug("An error happened during model creation, %s", err.Error())
    return &resp.Model, err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from model response.. %s", err.Error());
    return &resp.Model, err
  }

  return &resp.Model, nil
}

func (m *BuildClient) UpdateModel(model_id string, new_model *Model) (*Model, error) {
  var url bytes.Buffer
  resp := new(ModelResponse)
  url.WriteString(EI_URL)
  url.WriteString(MODELS_ENDPOINT)
  url.WriteString("/")
  url.WriteString(model_id)

  req_string, err := json.Marshal(new_model)
  logging.Debug("Request String for upload: %s", req_string)
  full_resp, err := m._complete_request("PUT", url.String(), req_string)
  if err != nil {
    logging.Debug("An error happened during model creation, %s", err.Error())
    return &resp.Model, err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from model response.. %s", err.Error());
    return &resp.Model, err
  }

  return &resp.Model, nil
}

func (m *BuildClient) DeleteModel(model_id string) (error) {
  var url bytes.Buffer
  resp := new(ModelResponse)
  url.WriteString(EI_URL)
  url.WriteString(MODELS_ENDPOINT)
  url.WriteString("/")
  url.WriteString(model_id)

  full_resp, err := m._complete_request("DELETE", url.String(), nil)
  if err != nil {
    logging.Debug("An error happened during model deletion, %s", err.Error())
    return err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from model response.. %s", err.Error());
    return err
  }

  if resp.Success == false {
    return errors.New("Error When retriveing Code Revisions")
  }

  return nil
}

func (m *BuildClient) RestartModelDevices(model_id string) (error) {
  var url bytes.Buffer
  resp := new(ModelResponse)
  url.WriteString(EI_URL)
  url.WriteString(MODELS_ENDPOINT)
  url.WriteString("/")
  url.WriteString(model_id)
  url.WriteString("/")
  url.WriteString(MODELS_DEVICE_RESTART_ENDPOINT)

  full_resp, err := m._complete_request("POST", url.String(), nil)
  if err != nil {
    logging.Debug("An error happened during model restart, %s", err.Error())
    return err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from model response.. %s", err.Error());
    return err
  }

  if resp.Success == false {
    return errors.New("Error When retriveing Code Revisions")
  }

  return nil
}

func (m *BuildClient) GetCodeRevisionList(model_id string) (
	[]CodeRevisionShort, error) {
  var url bytes.Buffer
  resp := new(CodeRevisionsResponse)
  url.WriteString(EI_URL)
  url.WriteString(MODELS_ENDPOINT)
  url.WriteString("/")
  url.WriteString(model_id)
  url.WriteString("/")
  url.WriteString(MODELS_REVISIONS_ENDPOINT)
  full_resp, err := m._complete_request("GET", url.String(), nil)
  if err != nil {
  	logging.Debug("Failed to get code revisions: %s", err.Error())
  	return resp.Revisions, err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from code revision.. %s", err.Error());
    return resp.Revisions, err
  }

  if resp.Success == false {
  	return resp.Revisions, errors.New("Error When retriveing Code Revisions")
  }
  return resp.Revisions, nil
}

func (m *BuildClient) GetCodeRevision(model_id string, build_num string) (CodeRevisionLong, error) {
  var url bytes.Buffer
  resp := new(CodeRevisionResponse)
  url.WriteString(EI_URL)
  url.WriteString(MODELS_ENDPOINT)
  url.WriteString("/")
  url.WriteString(model_id)
  url.WriteString("/")
  url.WriteString(MODELS_REVISIONS_ENDPOINT)
  url.WriteString("/")
  url.WriteString(build_num)
  full_resp, err := m._complete_request("GET", url.String(), nil)
  if err != nil {
  	logging.Debug("Failed to get code revisions: %s", err.Error())
  	return resp.Revisions, err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from code revision.. %s", err.Error());
    return resp.Revisions, err
  }

  if resp.Success == false {
  	return resp.Revisions, errors.New("Error When retriveing Code Revisions")
  }
  return resp.Revisions, nil
}

func (m *BuildClient) UpdateCodeRevision(model_id string,
	request *CodeRevisionLong) (CodeRevisionLong, error) {
  var url bytes.Buffer
  resp := new(CodeRevisionResponse)
  url.WriteString(EI_URL)
  url.WriteString(MODELS_ENDPOINT)
  url.WriteString("/")
  url.WriteString(model_id)
  url.WriteString("/")
  url.WriteString(MODELS_REVISIONS_ENDPOINT)

  req_string, err := json.Marshal(request)
  logging.Debug("Request String for upload: %s", req_string)
  full_resp, err := m._complete_request("POST", url.String(), req_string)
  if err != nil {
  	logging.Debug("Failed to update code revisions: %s", err.Error())
  	return resp.Revisions, err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from code revision update.. %s", err.Error());
    return resp.Revisions, err
  }

  if resp.Success == false {
  	return resp.Revisions, errors.New("Error When retriveing Code Revisions")
  }
  return resp.Revisions, nil
}

func (m *BuildClient) GetDeviceList() ([]Device, error) {
  var url bytes.Buffer
  resp := new(DeviceListResponse)
  url.WriteString(EI_URL)
  url.WriteString(DEVICES_ENDPOINT)

  full_resp, err := m._complete_request("GET", url.String(), nil)
  if err != nil {
  	logging.Debug("Failed to get device list: %s", err.Error())
  	return resp.Devices, err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from code revision update.. %s", err.Error());
    return resp.Devices, err
  }

  if resp.Success == false {
  	return resp.Devices, errors.New("Error When retriveing Code Revisions")
  }
  return resp.Devices, nil
}

func (m *BuildClient) GetDeviceLogs(device_id string) ([]DeviceLogEntry, error) {
  var url bytes.Buffer
  resp := new(DeviceLogResponse)
  url.WriteString(EI_URL)
  url.WriteString(DEVICES_ENDPOINT)
  url.WriteString("/")
  url.WriteString(device_id)
  url.WriteString("/")
  url.WriteString(DEVICES_LOG_ENDPOINT)
  full_resp, err := m._complete_request("GET", url.String(), nil)
  if err != nil {
  	logging.Debug("Failed to get device logs: %s", err.Error())
  	return resp.Logs, err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from device logs.. %s", err.Error());
    return resp.Logs, err
  }

  if resp.Success == false {
  	return resp.Logs, errors.New("Error When retriveing device logs")
  }
  return resp.Logs, nil
}

type DeviceResponse struct {
  Success bool `json:"success"`
  Device Device `json:"device"`
}

func (m *BuildClient) GetDevice(device_id string) (Device, error) {
  var url bytes.Buffer
  resp := new(DeviceResponse)
  url.WriteString(EI_URL)
  url.WriteString(DEVICES_ENDPOINT)
  url.WriteString("/")
  url.WriteString(device_id)

  full_resp, err := m._complete_request("GET", url.String(), nil)
  if err != nil {
    logging.Debug("Failed to get device list: %s", err.Error())
    return resp.Device, err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from code revision update.. %s", err.Error());
    return resp.Device, err
  }

  if resp.Success == false {
    return resp.Device, errors.New("Error When retriveing Code Revisions")
  }
  return resp.Device, nil
}

const DEVICES_RESTART_ENDPOINT = "restart"
func (m *BuildClient) RestartDevice(device_id string) (error) {
  var url bytes.Buffer
  resp := new(DeviceResponse)
  url.WriteString(EI_URL)
  url.WriteString(DEVICES_ENDPOINT)
  url.WriteString("/")
  url.WriteString(device_id)
  url.WriteString("/")
  url.WriteString(DEVICES_RESTART_ENDPOINT)

  full_resp, err := m._complete_request("POST", url.String(), nil)
  if err != nil {
    logging.Debug("Failed to get device list: %s", err.Error())
    return err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from code revision update.. %s", err.Error());
    return err
  }

  if resp.Success == false {
    return errors.New("Error When retriveing Code Revisions")
  }
  return nil
}

func (m *BuildClient) UpdateDevice(new_device *Device, device_id string) (Device, error) {
  var url bytes.Buffer
  resp := new(DeviceResponse)
  url.WriteString(EI_URL)
  url.WriteString(DEVICES_ENDPOINT)
  url.WriteString("/")
  url.WriteString(device_id)

  req_bytes, err := json.Marshal(new_device)
  full_resp, err := m._complete_request("PUT", url.String(), req_bytes)
  if err != nil {
    logging.Debug("Failed to update device: %s", err.Error())
    return resp.Device, err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from device update.. %s", err.Error());
    return resp.Device, err
  }

  if resp.Success == false {
    return resp.Device, errors.New("Error when updating device")
  }
  return resp.Device, nil
}

func (m *BuildClient) DeleteDevice(device_id string) (error) {
  var url bytes.Buffer
  resp := new(DeviceResponse)
  url.WriteString(EI_URL)
  url.WriteString(DEVICES_ENDPOINT)
  url.WriteString("/")
  url.WriteString(device_id)

  full_resp, err := m._complete_request("DELETE", url.String(), nil)
  if err != nil {
    logging.Debug("Failed to delete device: %s", err.Error())
    return err
  }

  if err := json.Unmarshal(full_resp, resp); err != nil {
    logging.Warn("Failed to unmarshal data from device deletion.. %s", err.Error());
    return err
  }

  if resp.Success == false {
    return errors.New("Error when updating device")
  }
  return nil
}<|endoftext|>


<commit_before>
package main

import (
    "fmt"
    "../GoLang/fibonacci"
    "../GoLang/exponenciacao"
    "../GoLang/fatorial"
    "../GoLang/hanoi"
    "../GoLang/bubblesort"
    "../GoLang/selectionsort"
    "../GoLang/insertionsort"
    "../GoLang/mergesort"
)

func main() {
    fmt.Println("Fibonacci : ", fibonacci.Fibonacci(9))
    fmt.Println("Exponenciacao : ", exponenciacao.Exponenciacao(5, 5))
    fmt.Println("Fatorial : ", fatorial.Fatorial(6))
    fmt.Println("Hanoi : ")
    hanoi.Hanoi(0, 2, 1, 3)
    slice := []int{5, 2, 1, 6, 9, 8, 7, 3, 4}
    fmt.Println("Slice : ", slice)
    fmt.Println("BubbleSort : ", bubblesort.BubbleSort(slice))
    fmt.Println("SelectionSort : ", selectionsort.SelectionSort(slice))
    fmt.Println("InsertionSort : ", insertionsort.InsertionSort(slice))
    fmt.Println("MergeSort : ", mergesort.MergeSort(slice))
}
<commit_msg>Adicionado exemplo do shellsort no main<commit_after>
package main

import (
    "fmt"
    "../GoLang/fibonacci"
    "../GoLang/exponenciacao"
    "../GoLang/fatorial"
    "../GoLang/hanoi"
    "../GoLang/bubblesort"
    "../GoLang/selectionsort"
    "../GoLang/insertionsort"
    "../GoLang/mergesort"
    "../GoLang/shellsort"
)

func main() {
    fmt.Println("Fibonacci : ", fibonacci.Fibonacci(9))
    fmt.Println("Exponenciacao : ", exponenciacao.Exponenciacao(5, 5))
    fmt.Println("Fatorial : ", fatorial.Fatorial(6))
    fmt.Println("Hanoi : ")
    hanoi.Hanoi(0, 2, 1, 3)
    slice := []int{5, 2, 1, 6, 9, 8, 7, 3, 4}
    fmt.Println("Slice : ", slice)
    fmt.Println("BubbleSort : ", bubblesort.BubbleSort(slice))
    fmt.Println("SelectionSort : ", selectionsort.SelectionSort(slice))
    fmt.Println("InsertionSort : ", insertionsort.InsertionSort(slice))
    fmt.Println("MergeSort : ", mergesort.MergeSort(slice))
    fmt.Println("ShellSort : ", shellsort.ShellSort(slice))
}
<|endoftext|>


<commit_before>package aqua

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/carbocation/interpose"
	"github.com/gorilla/mux"
	"github.com/tolexo/aero/activity"
	"github.com/tolexo/aero/auth"
	"github.com/tolexo/aero/cache"
	"github.com/tolexo/aero/conf"
	monit "github.com/tolexo/aero/monit"
	"github.com/tolexo/aero/panik"
)

type endPoint struct {
	caller     MethodInvoker
	info       Fixture
	httpMethod string

	isStdHttpHandler bool
	needsJarInput    bool

	muxUrl    string
	muxVars   []string
	modules   []func(http.Handler) http.Handler
	stash     cache.Cacher
	serviceId string
}

func NewEndPoint(inv MethodInvoker, f Fixture, matchUrl string, httpMethod string, mods map[string]func(http.Handler) http.Handler,
	caches map[string]cache.Cacher, serviceId string) endPoint {

	out := endPoint{
		caller:           inv,
		info:             f,
		isStdHttpHandler: false,
		needsJarInput:    false,
		muxUrl:           matchUrl,
		muxVars:          extractRouteVars(matchUrl),
		httpMethod:       httpMethod,
		modules:          make([]func(http.Handler) http.Handler, 0),
		stash:            nil,
		serviceId:        serviceId,
	}

	if f.Stub == "" {
		out.isStdHttpHandler = out.signatureMatchesDefaultHttpHandler()
		out.needsJarInput = out.needsVariableJar()

		out.validateMuxVarsMatchFuncInputs()
		out.validateFuncInputsAreOfRightType()
		out.validateFuncOutputsAreCorrect()
	}

	// Tag modules used by this endpoint
	if mods != nil && f.Modules != "" {
		names := strings.Split(f.Modules, ",")
		out.modules = make([]func(http.Handler) http.Handler, 0)
		for _, name := range names {
			name = strings.TrimSpace(name)
			fn, found := mods[name]
			if !found {
				panic(fmt.Sprintf("Module:%s not found", name))
			}
			out.modules = append(out.modules, fn)
		}
	}

	// Tag the cache
	if c, ok := caches[f.Cache]; ok {
		out.stash = c
	} else if f.Cache != "" {
		panic("Cache not found: " + f.Cache + " for " + matchUrl)
	}

	return out
}

func (me *endPoint) signatureMatchesDefaultHttpHandler() bool {
	return me.caller.outCount == 0 &&
		me.caller.inpCount == 2 &&
		me.caller.inpParams[0] == "i:net/http.ResponseWriter" &&
		me.caller.inpParams[1] == "*st:net/http.Request"
}

func (me *endPoint) needsVariableJar() bool {
	// needs jar input as the last parameter
	for i := 0; i < len(me.caller.inpParams)-1; i++ {
		if me.caller.inpParams[i] == "st:github.com/tolexo/aqua.Jar" {
			panic("Jar parameter should be the last one: " + me.caller.name)
		}
	}
	return me.caller.inpCount > 0 && me.caller.inpParams[me.caller.inpCount-1] == "st:github.com/tolexo/aqua.Jar"
}

func (me *endPoint) validateMuxVarsMatchFuncInputs() {
	// for non-standard http handlers, the mux vars count should match
	// the count of inputs to the user's method
	if !me.isStdHttpHandler {
		inputs := me.caller.inpCount
		if me.needsJarInput {
			inputs += -1
		}
		if len(me.muxVars) != inputs {
			panic(fmt.Sprintf("%s has %d inputs, but the func (%s) has %d",
				me.muxUrl, len(me.muxVars), me.caller.name, inputs))
		}
	}
}

func (me *endPoint) validateFuncInputsAreOfRightType() {
	if !me.isStdHttpHandler {
		for _, s := range me.caller.inpParams {
			switch s {
			case "st:github.com/tolexo/aqua.Jar":
			case "int":
			case "string":
			default:
				panic("Func input params should be 'int' or 'string'. Observed: " + s + " in: " + me.caller.name)
			}
		}
	}
}

func (me *endPoint) validateFuncOutputsAreCorrect() {

	var accepts = make(map[string]bool)
	accepts["string"] = true
	accepts["map"] = true
	accepts["st:github.com/tolexo/aqua.Sac"] = true
	accepts["*st:github.com/tolexo/aqua.Sac"] = true

	if !me.isStdHttpHandler {
		switch me.caller.outCount {
		case 1:
			_, found := accepts[me.caller.outParams[0]]
			if !found && !strings.HasPrefix(me.caller.outParams[0], "st:") {
				fmt.Println(me.caller.outParams[0])
				panic("Incorrect return type found in: " + me.caller.name)
			}
		case 2:
			if me.caller.outParams[0] != "int" {
				panic("When a func returns two params, the first must be an int (http status code) : " + me.caller.name)
			}
			_, found := accepts[me.caller.outParams[1]]
			if !found && !strings.HasPrefix(me.caller.outParams[1], "st:") {
				panic("Incorrect return type for second return param found in: " + me.caller.name)
			}
		default:
			panic("Incorrect number of returns for Func: " + me.caller.name)
		}
	}
}

// func middleman(next http.Handler) http.Handler {
// 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// 		fmt.Println("In the middle >>>>")
// 		next.ServeHTTP(w, r)
// 		fmt.Println("And leaving middle <<<<")
// 	})
// }

func (me *endPoint) setupMuxHandlers(mux *mux.Router) {

	fn := handleIncoming(me)

	m := interpose.New()
	for i, _ := range me.modules {
		m.Use(me.modules[i])
		//fmt.Println("using module:", me.modules[i], reflect.TypeOf(me.modules[i]))
	}
	m.UseHandler(http.HandlerFunc(fn))

	if me.info.Version == "*" {
		mux.Handle(me.muxUrl, m).Methods(me.httpMethod)
	} else {
		urlWithVersion := cleanUrl(me.info.Prefix, "v"+me.info.Version, me.muxUrl)
		urlWithoutVersion := cleanUrl(me.info.Prefix, me.muxUrl)

		// versioned url
		mux.Handle(urlWithVersion, m).Methods(me.httpMethod)

		// content type (style1)
		header1 := fmt.Sprintf("application/%s-v%s+json", me.info.Vendor, me.info.Version)
		mux.Handle(urlWithoutVersion, m).Methods(me.httpMethod).Headers("Accept", header1)

		// content type (style2)
		header2 := fmt.Sprintf("application/%s+json;version=%s", me.info.Vendor, me.info.Version)
		mux.Handle(urlWithoutVersion, m).Methods(me.httpMethod).Headers("Accept", header2)
	}
}

//Copy request body
func copyReqBody(reqBody io.ReadCloser) (originalBody io.ReadCloser, copyBody interface{}) {
	bodyByte, _ := ioutil.ReadAll(reqBody)
	json.Unmarshal(bodyByte, &copyBody)
	originalBody = ioutil.NopCloser(bytes.NewBuffer(bodyByte))
	return
}

func handleIncoming(e *endPoint) func(http.ResponseWriter, *http.Request) {

	// return stub
	if e.info.Stub != "" {
		return func(w http.ResponseWriter, r *http.Request) {
			d, err := getContent(e.info.Stub)
			if err == nil {
				fmt.Fprintf(w, "%s", d)
			} else {
				w.WriteHeader(400)
				fmt.Fprintf(w, "{ message: \"%s\"}", "Stub path not found")
			}
		}
	}

	return func(w http.ResponseWriter, r *http.Request) {

		// cacheHit := false

		// TODO: create less local variables
		// TODO: move vars to closure level

		var out []reflect.Value
		//TODO: capture this using instrumentation handler

		var body interface{}
		logActivity := conf.Bool("log_activity", false)
		if logActivity == true {
			r.Body, body = copyReqBody(r.Body)
		}

		defer func(reqStartTime time.Time) {
			var (
				response     interface{}
				responseCode int64 = 200
			)
			respTime := time.Since(reqStartTime).Seconds() * 1000
			if out != nil && len(out) == 2 && e.caller.outParams[0] == "int" {
				responseCode = out[0].Int()
			}
			/*
				go func() {
					if e.serviceId != "" {
						monitorParams := monit.MonitorParams{
							ServiceId:    e.serviceId,
							RespTime:     respTime,
							ResponseCode: responseCode,
							CacheHit:     cacheHit,
						}
						monit.MonitorMe(monitorParams)
					}
				}()
			*/

			//User Activity logger start
			if logActivity == true {
				if out != nil && len(out) > 1 {
					response = out[1].Interface()
				}
				activity.LogActivity(e.serviceId, body, response,
					int(responseCode), respTime)
			}
			//User Activity logger end

			if reqR := recover(); reqR != nil {
				monit.PanicLogger(reqR, r.RequestURI+" "+e.serviceId, r.RequestURI, time.Now())
			}
		}(time.Now())

		//check authentication
		if e.info.Auth != "" {
			ok, errMsg := auth.AuthenticateRequest(r, e.info.Auth)
			if !ok { //print authentication error
				w.WriteHeader(401)
				w.Header().Set("Content-Type", "application/json")
				w.Header().Set("Content-Length", strconv.Itoa(len(errMsg)))
				fmt.Fprintf(w, "%s", errMsg)
				return
			}
		}

		var useCache bool = false
		var ttl time.Duration = 0 * time.Second
		var val []byte
		var err error

		if e.info.Ttl != "" {
			ttl, err = time.ParseDuration(e.info.Ttl)
			panik.On(err)
		}
		useCache = r.Method == "GET" && ttl > 0 && e.stash != nil

		muxVals := mux.Vars(r)
		params := make([]string, len(e.muxVars))
		for i, v := range e.muxVars {
			params[i] = muxVals[v]
		}

		if e.isStdHttpHandler {
			//TODO: caching of standard handler
			e.caller.Do([]reflect.Value{reflect.ValueOf(w), reflect.ValueOf(r)})
		} else {
			ref := convertToType(params, e.caller.inpParams)
			if e.needsJarInput {
				ref = append(ref, reflect.ValueOf(NewJar(r)))
			}

			if useCache {
				val, err = e.stash.Get(r.RequestURI)
				if err == nil {
					// cacheHit = true
					// fmt.Print(".")
					out = decomposeCachedValues(val, e.caller.outParams)
				} else {
					out = e.caller.Do(ref)
					if len(out) == 2 && e.caller.outParams[0] == "int" {
						code := out[0].Int()
						if code < 200 || code > 299 {
							useCache = false
						}
					}
					if useCache {
						bytes := prepareForCaching(out, e.caller.outParams)
						e.stash.Set(r.RequestURI, bytes, ttl)
						// fmt.Print(":", len(bytes), r.RequestURI)
					}

				}
			} else {
				out = e.caller.Do(ref)
				// fmt.Print("!")
			}
			writeOutput(w, e.caller.outParams, out, e.info.Pretty)
		}
	}
}

func prepareForCaching(r []reflect.Value, outputParams []string) []byte {

	var err error
	buf := new(bytes.Buffer)
	encd := json.NewEncoder(buf)

	for i, _ := range r {
		switch outputParams[i] {
		case "int":
			err = encd.Encode(r[i].Int())
			panik.On(err)
		case "map":
			err = encd.Encode(r[i].Interface().(map[string]interface{}))
			panik.On(err)
		case "string":
			err = encd.Encode(r[i].String())
			panik.On(err)
		case "*st:github.com/tolexo/aqua.Sac":
			err = encd.Encode(r[i].Elem().Interface().(Sac).Data)
		default:
			panic("Unknown type of output to be sent to endpoint cache: " + outputParams[i])
		}
	}

	return buf.Bytes()
}

func decomposeCachedValues(data []byte, outputParams []string) []reflect.Value {

	var err error
	buf := bytes.NewBuffer(data)
	decd := json.NewDecoder(buf)
	out := make([]reflect.Value, len(outputParams))

	for i, o := range outputParams {
		switch o {
		case "int":
			var j int
			err = decd.Decode(&j)
			panik.On(err)
			out[i] = reflect.ValueOf(j)
		case "map":
			var m map[string]interface{}
			err = decd.Decode(&m)
			panik.On(err)
			out[i] = reflect.ValueOf(m)
		case "string":
			var s string
			err = decd.Decode(&s)
			panik.On(err)
			out[i] = reflect.ValueOf(s)
		case "*st:github.com/tolexo/aqua.Sac":
			var m map[string]interface{}
			err = decd.Decode(&m)
			panik.On(err)
			s := NewSac()
			s.Data = m
			out[i] = reflect.ValueOf(s)
		default:
			panic("Unknown type of output to be decoded from endpoint cache:" + o)
		}
	}

	return out

}
<commit_msg>PRA-410: url added<commit_after>package aqua

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/carbocation/interpose"
	"github.com/gorilla/mux"
	"github.com/tolexo/aero/activity"
	"github.com/tolexo/aero/auth"
	"github.com/tolexo/aero/cache"
	"github.com/tolexo/aero/conf"
	monit "github.com/tolexo/aero/monit"
	"github.com/tolexo/aero/panik"
)

type endPoint struct {
	caller     MethodInvoker
	info       Fixture
	httpMethod string

	isStdHttpHandler bool
	needsJarInput    bool

	muxUrl    string
	muxVars   []string
	modules   []func(http.Handler) http.Handler
	stash     cache.Cacher
	serviceId string
}

func NewEndPoint(inv MethodInvoker, f Fixture, matchUrl string, httpMethod string, mods map[string]func(http.Handler) http.Handler,
	caches map[string]cache.Cacher, serviceId string) endPoint {

	out := endPoint{
		caller:           inv,
		info:             f,
		isStdHttpHandler: false,
		needsJarInput:    false,
		muxUrl:           matchUrl,
		muxVars:          extractRouteVars(matchUrl),
		httpMethod:       httpMethod,
		modules:          make([]func(http.Handler) http.Handler, 0),
		stash:            nil,
		serviceId:        serviceId,
	}

	if f.Stub == "" {
		out.isStdHttpHandler = out.signatureMatchesDefaultHttpHandler()
		out.needsJarInput = out.needsVariableJar()

		out.validateMuxVarsMatchFuncInputs()
		out.validateFuncInputsAreOfRightType()
		out.validateFuncOutputsAreCorrect()
	}

	// Tag modules used by this endpoint
	if mods != nil && f.Modules != "" {
		names := strings.Split(f.Modules, ",")
		out.modules = make([]func(http.Handler) http.Handler, 0)
		for _, name := range names {
			name = strings.TrimSpace(name)
			fn, found := mods[name]
			if !found {
				panic(fmt.Sprintf("Module:%s not found", name))
			}
			out.modules = append(out.modules, fn)
		}
	}

	// Tag the cache
	if c, ok := caches[f.Cache]; ok {
		out.stash = c
	} else if f.Cache != "" {
		panic("Cache not found: " + f.Cache + " for " + matchUrl)
	}

	return out
}

func (me *endPoint) signatureMatchesDefaultHttpHandler() bool {
	return me.caller.outCount == 0 &&
		me.caller.inpCount == 2 &&
		me.caller.inpParams[0] == "i:net/http.ResponseWriter" &&
		me.caller.inpParams[1] == "*st:net/http.Request"
}

func (me *endPoint) needsVariableJar() bool {
	// needs jar input as the last parameter
	for i := 0; i < len(me.caller.inpParams)-1; i++ {
		if me.caller.inpParams[i] == "st:github.com/tolexo/aqua.Jar" {
			panic("Jar parameter should be the last one: " + me.caller.name)
		}
	}
	return me.caller.inpCount > 0 && me.caller.inpParams[me.caller.inpCount-1] == "st:github.com/tolexo/aqua.Jar"
}

func (me *endPoint) validateMuxVarsMatchFuncInputs() {
	// for non-standard http handlers, the mux vars count should match
	// the count of inputs to the user's method
	if !me.isStdHttpHandler {
		inputs := me.caller.inpCount
		if me.needsJarInput {
			inputs += -1
		}
		if len(me.muxVars) != inputs {
			panic(fmt.Sprintf("%s has %d inputs, but the func (%s) has %d",
				me.muxUrl, len(me.muxVars), me.caller.name, inputs))
		}
	}
}

func (me *endPoint) validateFuncInputsAreOfRightType() {
	if !me.isStdHttpHandler {
		for _, s := range me.caller.inpParams {
			switch s {
			case "st:github.com/tolexo/aqua.Jar":
			case "int":
			case "string":
			default:
				panic("Func input params should be 'int' or 'string'. Observed: " + s + " in: " + me.caller.name)
			}
		}
	}
}

func (me *endPoint) validateFuncOutputsAreCorrect() {

	var accepts = make(map[string]bool)
	accepts["string"] = true
	accepts["map"] = true
	accepts["st:github.com/tolexo/aqua.Sac"] = true
	accepts["*st:github.com/tolexo/aqua.Sac"] = true

	if !me.isStdHttpHandler {
		switch me.caller.outCount {
		case 1:
			_, found := accepts[me.caller.outParams[0]]
			if !found && !strings.HasPrefix(me.caller.outParams[0], "st:") {
				fmt.Println(me.caller.outParams[0])
				panic("Incorrect return type found in: " + me.caller.name)
			}
		case 2:
			if me.caller.outParams[0] != "int" {
				panic("When a func returns two params, the first must be an int (http status code) : " + me.caller.name)
			}
			_, found := accepts[me.caller.outParams[1]]
			if !found && !strings.HasPrefix(me.caller.outParams[1], "st:") {
				panic("Incorrect return type for second return param found in: " + me.caller.name)
			}
		default:
			panic("Incorrect number of returns for Func: " + me.caller.name)
		}
	}
}

// func middleman(next http.Handler) http.Handler {
// 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// 		fmt.Println("In the middle >>>>")
// 		next.ServeHTTP(w, r)
// 		fmt.Println("And leaving middle <<<<")
// 	})
// }

func (me *endPoint) setupMuxHandlers(mux *mux.Router) {

	fn := handleIncoming(me)

	m := interpose.New()
	for i, _ := range me.modules {
		m.Use(me.modules[i])
		//fmt.Println("using module:", me.modules[i], reflect.TypeOf(me.modules[i]))
	}
	m.UseHandler(http.HandlerFunc(fn))

	if me.info.Version == "*" {
		mux.Handle(me.muxUrl, m).Methods(me.httpMethod)
	} else {
		urlWithVersion := cleanUrl(me.info.Prefix, "v"+me.info.Version, me.muxUrl)
		urlWithoutVersion := cleanUrl(me.info.Prefix, me.muxUrl)

		// versioned url
		mux.Handle(urlWithVersion, m).Methods(me.httpMethod)

		// content type (style1)
		header1 := fmt.Sprintf("application/%s-v%s+json", me.info.Vendor, me.info.Version)
		mux.Handle(urlWithoutVersion, m).Methods(me.httpMethod).Headers("Accept", header1)

		// content type (style2)
		header2 := fmt.Sprintf("application/%s+json;version=%s", me.info.Vendor, me.info.Version)
		mux.Handle(urlWithoutVersion, m).Methods(me.httpMethod).Headers("Accept", header2)
	}
}

//Copy request body
func copyReqBody(reqBody io.ReadCloser) (originalBody io.ReadCloser, copyBody interface{}) {
	bodyByte, _ := ioutil.ReadAll(reqBody)
	json.Unmarshal(bodyByte, &copyBody)
	originalBody = ioutil.NopCloser(bytes.NewBuffer(bodyByte))
	return
}

func handleIncoming(e *endPoint) func(http.ResponseWriter, *http.Request) {

	// return stub
	if e.info.Stub != "" {
		return func(w http.ResponseWriter, r *http.Request) {
			d, err := getContent(e.info.Stub)
			if err == nil {
				fmt.Fprintf(w, "%s", d)
			} else {
				w.WriteHeader(400)
				fmt.Fprintf(w, "{ message: \"%s\"}", "Stub path not found")
			}
		}
	}

	return func(w http.ResponseWriter, r *http.Request) {

		// cacheHit := false

		// TODO: create less local variables
		// TODO: move vars to closure level

		var out []reflect.Value
		//TODO: capture this using instrumentation handler

		var body interface{}
		logActivity := conf.Bool("log_activity", false)
		if logActivity == true {
			r.Body, body = copyReqBody(r.Body)
		}

		defer func(reqStartTime time.Time) {
			var (
				response     interface{}
				responseCode int64 = 200
			)
			respTime := time.Since(reqStartTime).Seconds() * 1000
			if out != nil && len(out) == 2 && e.caller.outParams[0] == "int" {
				responseCode = out[0].Int()
			}
			/*
				go func() {
					if e.serviceId != "" {
						monitorParams := monit.MonitorParams{
							ServiceId:    e.serviceId,
							RespTime:     respTime,
							ResponseCode: responseCode,
							CacheHit:     cacheHit,
						}
						monit.MonitorMe(monitorParams)
					}
				}()
			*/

			//User Activity logger start
			if logActivity == true {
				if out != nil && len(out) > 1 {
					response = out[1].Interface()
				}
				activity.LogActivity(r.RequestURI+" "+e.serviceId, body, response,
					int(responseCode), respTime)
			}
			//User Activity logger end

			if reqR := recover(); reqR != nil {
				monit.PanicLogger(reqR, e.serviceId, r.RequestURI, time.Now())
			}
		}(time.Now())

		//check authentication
		if e.info.Auth != "" {
			ok, errMsg := auth.AuthenticateRequest(r, e.info.Auth)
			if !ok { //print authentication error
				w.WriteHeader(401)
				w.Header().Set("Content-Type", "application/json")
				w.Header().Set("Content-Length", strconv.Itoa(len(errMsg)))
				fmt.Fprintf(w, "%s", errMsg)
				return
			}
		}

		var useCache bool = false
		var ttl time.Duration = 0 * time.Second
		var val []byte
		var err error

		if e.info.Ttl != "" {
			ttl, err = time.ParseDuration(e.info.Ttl)
			panik.On(err)
		}
		useCache = r.Method == "GET" && ttl > 0 && e.stash != nil

		muxVals := mux.Vars(r)
		params := make([]string, len(e.muxVars))
		for i, v := range e.muxVars {
			params[i] = muxVals[v]
		}

		if e.isStdHttpHandler {
			//TODO: caching of standard handler
			e.caller.Do([]reflect.Value{reflect.ValueOf(w), reflect.ValueOf(r)})
		} else {
			ref := convertToType(params, e.caller.inpParams)
			if e.needsJarInput {
				ref = append(ref, reflect.ValueOf(NewJar(r)))
			}

			if useCache {
				val, err = e.stash.Get(r.RequestURI)
				if err == nil {
					// cacheHit = true
					// fmt.Print(".")
					out = decomposeCachedValues(val, e.caller.outParams)
				} else {
					out = e.caller.Do(ref)
					if len(out) == 2 && e.caller.outParams[0] == "int" {
						code := out[0].Int()
						if code < 200 || code > 299 {
							useCache = false
						}
					}
					if useCache {
						bytes := prepareForCaching(out, e.caller.outParams)
						e.stash.Set(r.RequestURI, bytes, ttl)
						// fmt.Print(":", len(bytes), r.RequestURI)
					}

				}
			} else {
				out = e.caller.Do(ref)
				// fmt.Print("!")
			}
			writeOutput(w, e.caller.outParams, out, e.info.Pretty)
		}
	}
}

func prepareForCaching(r []reflect.Value, outputParams []string) []byte {

	var err error
	buf := new(bytes.Buffer)
	encd := json.NewEncoder(buf)

	for i, _ := range r {
		switch outputParams[i] {
		case "int":
			err = encd.Encode(r[i].Int())
			panik.On(err)
		case "map":
			err = encd.Encode(r[i].Interface().(map[string]interface{}))
			panik.On(err)
		case "string":
			err = encd.Encode(r[i].String())
			panik.On(err)
		case "*st:github.com/tolexo/aqua.Sac":
			err = encd.Encode(r[i].Elem().Interface().(Sac).Data)
		default:
			panic("Unknown type of output to be sent to endpoint cache: " + outputParams[i])
		}
	}

	return buf.Bytes()
}

func decomposeCachedValues(data []byte, outputParams []string) []reflect.Value {

	var err error
	buf := bytes.NewBuffer(data)
	decd := json.NewDecoder(buf)
	out := make([]reflect.Value, len(outputParams))

	for i, o := range outputParams {
		switch o {
		case "int":
			var j int
			err = decd.Decode(&j)
			panik.On(err)
			out[i] = reflect.ValueOf(j)
		case "map":
			var m map[string]interface{}
			err = decd.Decode(&m)
			panik.On(err)
			out[i] = reflect.ValueOf(m)
		case "string":
			var s string
			err = decd.Decode(&s)
			panik.On(err)
			out[i] = reflect.ValueOf(s)
		case "*st:github.com/tolexo/aqua.Sac":
			var m map[string]interface{}
			err = decd.Decode(&m)
			panik.On(err)
			s := NewSac()
			s.Data = m
			out[i] = reflect.ValueOf(s)
		default:
			panic("Unknown type of output to be decoded from endpoint cache:" + o)
		}
	}

	return out

}
<|endoftext|>


<commit_before>package handlers

import (
	"encoding/json"
	"fmt"
	"net/http"
	"path"

	"github.com/golang/glog"
	"github.com/zefer/mpd-web/mpd"
)

type FileListEntry struct {
	Path string `json:"path"`
	Type string `json:"type"`
	Base string `json:"base"`
}

func FileListHandler(client *mpd.Client) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		data, err := client.C.ListInfo(r.FormValue("uri"))
		if err != nil {
			glog.Errorln(err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		out := make([]*FileListEntry, len(data))
		for i, item := range data {
			for _, t := range []string{"file", "directory", "playlist"} {
				if p, ok := item[t]; ok {
					out[i] = &FileListEntry{
						Path: p,
						Type: t,
						Base: path.Base(p),
					}
					break
				}
			}
		}
		b, err := json.Marshal(out)
		w.Header().Add("Content-Type", "application/json")
		fmt.Fprint(w, string(b))
	})
}

func LibraryUpdateHandler(client *mpd.Client) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "GET" {
			w.WriteHeader(http.StatusMethodNotAllowed)
			return
		} else if r.Method == "POST" {
			w.WriteHeader(http.StatusMethodNotAllowed)
			return
		} else {
			// Parse the JSON body.
			decoder := json.NewDecoder(r.Body)
			var params map[string]interface{}
			err := decoder.Decode(&params)
			if err != nil {
				glog.Errorln(err)
				w.WriteHeader(http.StatusInternalServerError)
				return
			}
			uri := params["uri"].(string)
			if uri == "" {
				w.WriteHeader(http.StatusBadRequest)
				return
			}
			_, err = client.C.Update(uri)
			if err != nil {
				glog.Errorln(err)
				w.WriteHeader(http.StatusInternalServerError)
				return
			}
			w.WriteHeader(http.StatusOK)
			return
		}
	})
}
<commit_msg>This handler should only allow GETs<commit_after>package handlers

import (
	"encoding/json"
	"fmt"
	"net/http"
	"path"

	"github.com/golang/glog"
	"github.com/zefer/mpd-web/mpd"
)

type FileListEntry struct {
	Path string `json:"path"`
	Type string `json:"type"`
	Base string `json:"base"`
}

func FileListHandler(client *mpd.Client) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "GET" {
			w.WriteHeader(http.StatusMethodNotAllowed)
		}
		data, err := client.C.ListInfo(r.FormValue("uri"))
		if err != nil {
			glog.Errorln(err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		out := make([]*FileListEntry, len(data))
		for i, item := range data {
			for _, t := range []string{"file", "directory", "playlist"} {
				if p, ok := item[t]; ok {
					out[i] = &FileListEntry{
						Path: p,
						Type: t,
						Base: path.Base(p),
					}
					break
				}
			}
		}
		b, err := json.Marshal(out)
		w.Header().Add("Content-Type", "application/json")
		fmt.Fprint(w, string(b))
	})
}

func LibraryUpdateHandler(client *mpd.Client) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "GET" {
			w.WriteHeader(http.StatusMethodNotAllowed)
			return
		} else if r.Method == "POST" {
			w.WriteHeader(http.StatusMethodNotAllowed)
			return
		} else {
			// Parse the JSON body.
			decoder := json.NewDecoder(r.Body)
			var params map[string]interface{}
			err := decoder.Decode(&params)
			if err != nil {
				glog.Errorln(err)
				w.WriteHeader(http.StatusInternalServerError)
				return
			}
			uri := params["uri"].(string)
			if uri == "" {
				w.WriteHeader(http.StatusBadRequest)
				return
			}
			_, err = client.C.Update(uri)
			if err != nil {
				glog.Errorln(err)
				w.WriteHeader(http.StatusInternalServerError)
				return
			}
			w.WriteHeader(http.StatusOK)
			return
		}
	})
}
<|endoftext|>


<commit_before>package sisyphus

import (
	"math/rand"
	"os"
	"path/filepath"
	"time"

	log "github.com/sirupsen/logrus"

	"github.com/boltdb/bolt"
	"github.com/gonum/stat"
	"github.com/retailnext/hllpp"
)

// classificationPrior returns the prior probabilities for good and junk
// classes.
func classificationPrior(db *bolt.DB) (g float64, err error) {

	gTotal, jTotal, err := classificationStatistics(db)
	if err != nil {
		return g, err
	}

	return gTotal / (gTotal + jTotal), err
}

// classificationLikelihoodWordcounts gets wordcounts from database to be used
// in Likelihood calculation
func classificationLikelihoodWordcounts(db *bolt.DB, word string) (gN, jN float64, err error) {

	err = db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("Wordlists"))

		good := b.Bucket([]byte("Good"))
		gWordRaw := good.Get([]byte(word))
		if len(gWordRaw) > 0 {
			var gWordHLL *hllpp.HLLPP
			gWordHLL, err = hllpp.Unmarshal(gWordRaw)
			if err != nil {
				return err
			}
			gN = float64(gWordHLL.Count())
		}
		junk := b.Bucket([]byte("Junk"))
		jWordRaw := junk.Get([]byte(word))
		if len(jWordRaw) > 0 {
			var jWordHLL *hllpp.HLLPP
			jWordHLL, err = hllpp.Unmarshal(jWordRaw)
			if err != nil {
				return err
			}
			jN = float64(jWordHLL.Count())
		}

		return nil
	})

	return gN, jN, err
}

// classificationStatistics gets global statistics from database to
// be used in Likelihood calculation
func classificationStatistics(db *bolt.DB) (gTotal, jTotal float64, err error) {

	err = db.View(func(tx *bolt.Tx) error {
		p := tx.Bucket([]byte("Statistics"))
		gRaw := p.Get([]byte("ProcessedGood"))
		if len(gRaw) > 0 {
			var gHLL *hllpp.HLLPP
			gHLL, err = hllpp.Unmarshal(gRaw)
			if err != nil {
				return err
			}
			gTotal = float64(gHLL.Count())
		}
		jRaw := p.Get([]byte("ProcessedJunk"))
		if len(jRaw) > 0 {
			var jHLL *hllpp.HLLPP
			jHLL, err = hllpp.Unmarshal(jRaw)
			if err != nil {
				return err
			}
			jTotal = float64(jHLL.Count())
		}

		if gTotal == 0 && jTotal == 0 {
			log.Warning("no mails have yet been learned")
			return nil
		}
		if gTotal == 0 {
			log.Warning("no good mails have yet been learned")
			return nil
		}
		if jTotal == 0 {
			log.Warning("no junk mails have yet been learned")
			return nil
		}

		return nil
	})

	return gTotal, jTotal, err
}

// classificationLikelihood returns P(W|C_j) -- the probability of seeing a
// particular word W in a document of this class.
func classificationLikelihood(db *bolt.DB, word string) (g, j float64, err error) {

	gN, jN, err := classificationLikelihoodWordcounts(db, word)
	if err != nil {
		return g, j, err
	}

	gTotal, jTotal, err := classificationStatistics(db)
	if err != nil {
		return g, j, err
	}

	g = gN / gTotal
	j = jN / jTotal

	return g, j, err
}

// classificationWord produces the conditional probability of a word belonging
// to good or junk using the classic Bayes' rule.
func classificationWord(db *bolt.DB, word string) (g float64, err error) {

	priorG, err := classificationPrior(db)
	if err != nil {
		return g, err
	}

	likelihoodG, likelihoodJ, err := classificationLikelihood(db, word)
	if err != nil {
		return g, err
	}

	g = (likelihoodG * priorG) / (likelihoodG*priorG + likelihoodJ*(1-priorG))

	return g, nil
}

// Classify analyses a new mail (a mail that arrived in the "new" directory),
// decides whether it is junk and -- if so -- moves it to the Junk folder. If
// it is not junk, the mail is untouched so it can be handled by the mail
// client.
func (m *Mail) Classify(db *bolt.DB, dir Maildir) (err error) {

	m.New = true

	err = m.Load(dir)
	if err != nil {
		return err
	}

	list, err := m.cleanWordlist()
	if err != nil {
		return err
	}

	junk, prob, err := Junk(db, list)
	if err != nil {
		return err
	}

	m.Junk = junk

	log.WithFields(log.Fields{
		"mail":        m.Key,
		"junk":        m.Junk,
		"probability": prob,
		"dir":         string(dir),
	}).Info("Classified")

	// Move mail around if junk.
	if junk {
		if !m.DryRun {
			err = os.Rename(filepath.Join(string(dir), "new", m.Key), filepath.Join(string(dir), ".Junk", "cur", m.Key))
			if err != nil {
				return err
			}
		}

		var dryRun string
		if m.DryRun {
			dryRun = "-- dry run (nothing happened to this mail!)"
		}

		log.WithFields(log.Fields{
			"mail": m.Key,
		}).Info("Moved to Junk folder" + dryRun)
	}

	err = m.Unload(dir)

	return err
}

// Junk returns true if the wordlist is classified as a junk mail using Bayes'
// rule. If required, it also returns the calculated probability of being junk,
// but this is typically not needed.
func Junk(db *bolt.DB, wordlist []string) (junk bool, prob float64, err error) {
	var probabilities []float64

	// If the wordlist is too long, let us only select a random sample
	// for analysis. This prevents cheating by adding lots of good text
	// to a Junk mail
	if len(wordlist) > 50 {
		wordlistTemp := make(map[string]interface{})

		rand.Seed(time.Now().UnixNano())

		for len(wordlistTemp) < 50 {
			wordlistTemp[wordlist[rand.Intn(len(wordlist)-1)]] = nil

		}

		var wordlistTempSlice []string
		for key, _ := range wordlistTemp {
			wordlistTempSlice = append(wordlistTempSlice, key)
		}
		wordlist = wordlistTempSlice
	}

	// initial value should be no junk
	prob = 1.0

	for _, val := range wordlist {
		var p float64
		p, err = classificationWord(db, val)
		if err != nil {
			return false, 0.0, err
		}
		probabilities = append(probabilities, p)
	}

	if len(probabilities) > 0 {
		prob = stat.HarmonicMean(probabilities, nil)
	}
	if prob < 0.5 {
		return true, (1 - prob), err
	}

	return false, (1 - prob), err
}
<commit_msg>please gometalinter<commit_after>package sisyphus

import (
	"math/rand"
	"os"
	"path/filepath"
	"time"

	log "github.com/sirupsen/logrus"

	"github.com/boltdb/bolt"
	"github.com/gonum/stat"
	"github.com/retailnext/hllpp"
)

// classificationPrior returns the prior probabilities for good and junk
// classes.
func classificationPrior(db *bolt.DB) (g float64, err error) {

	gTotal, jTotal, err := classificationStatistics(db)
	if err != nil {
		return g, err
	}

	return gTotal / (gTotal + jTotal), err
}

// classificationLikelihoodWordcounts gets wordcounts from database to be used
// in Likelihood calculation
func classificationLikelihoodWordcounts(db *bolt.DB, word string) (gN, jN float64, err error) {

	err = db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("Wordlists"))

		good := b.Bucket([]byte("Good"))
		gWordRaw := good.Get([]byte(word))
		if len(gWordRaw) > 0 {
			var gWordHLL *hllpp.HLLPP
			gWordHLL, err = hllpp.Unmarshal(gWordRaw)
			if err != nil {
				return err
			}
			gN = float64(gWordHLL.Count())
		}
		junk := b.Bucket([]byte("Junk"))
		jWordRaw := junk.Get([]byte(word))
		if len(jWordRaw) > 0 {
			var jWordHLL *hllpp.HLLPP
			jWordHLL, err = hllpp.Unmarshal(jWordRaw)
			if err != nil {
				return err
			}
			jN = float64(jWordHLL.Count())
		}

		return nil
	})

	return gN, jN, err
}

// classificationStatistics gets global statistics from database to
// be used in Likelihood calculation
func classificationStatistics(db *bolt.DB) (gTotal, jTotal float64, err error) {

	err = db.View(func(tx *bolt.Tx) error {
		p := tx.Bucket([]byte("Statistics"))
		gRaw := p.Get([]byte("ProcessedGood"))
		if len(gRaw) > 0 {
			var gHLL *hllpp.HLLPP
			gHLL, err = hllpp.Unmarshal(gRaw)
			if err != nil {
				return err
			}
			gTotal = float64(gHLL.Count())
		}
		jRaw := p.Get([]byte("ProcessedJunk"))
		if len(jRaw) > 0 {
			var jHLL *hllpp.HLLPP
			jHLL, err = hllpp.Unmarshal(jRaw)
			if err != nil {
				return err
			}
			jTotal = float64(jHLL.Count())
		}

		if gTotal == 0 && jTotal == 0 {
			log.Warning("no mails have yet been learned")
			return nil
		}
		if gTotal == 0 {
			log.Warning("no good mails have yet been learned")
			return nil
		}
		if jTotal == 0 {
			log.Warning("no junk mails have yet been learned")
			return nil
		}

		return nil
	})

	return gTotal, jTotal, err
}

// classificationLikelihood returns P(W|C_j) -- the probability of seeing a
// particular word W in a document of this class.
func classificationLikelihood(db *bolt.DB, word string) (g, j float64, err error) {

	gN, jN, err := classificationLikelihoodWordcounts(db, word)
	if err != nil {
		return g, j, err
	}

	gTotal, jTotal, err := classificationStatistics(db)
	if err != nil {
		return g, j, err
	}

	g = gN / gTotal
	j = jN / jTotal

	return g, j, err
}

// classificationWord produces the conditional probability of a word belonging
// to good or junk using the classic Bayes' rule.
func classificationWord(db *bolt.DB, word string) (g float64, err error) {

	priorG, err := classificationPrior(db)
	if err != nil {
		return g, err
	}

	likelihoodG, likelihoodJ, err := classificationLikelihood(db, word)
	if err != nil {
		return g, err
	}

	g = (likelihoodG * priorG) / (likelihoodG*priorG + likelihoodJ*(1-priorG))

	return g, nil
}

// Classify analyses a new mail (a mail that arrived in the "new" directory),
// decides whether it is junk and -- if so -- moves it to the Junk folder. If
// it is not junk, the mail is untouched so it can be handled by the mail
// client.
func (m *Mail) Classify(db *bolt.DB, dir Maildir) (err error) {

	m.New = true

	err = m.Load(dir)
	if err != nil {
		return err
	}

	list, err := m.cleanWordlist()
	if err != nil {
		return err
	}

	junk, prob, err := Junk(db, list)
	if err != nil {
		return err
	}

	m.Junk = junk

	log.WithFields(log.Fields{
		"mail":        m.Key,
		"junk":        m.Junk,
		"probability": prob,
		"dir":         string(dir),
	}).Info("Classified")

	// Move mail around if junk.
	if junk {
		if !m.DryRun {
			err = os.Rename(filepath.Join(string(dir), "new", m.Key), filepath.Join(string(dir), ".Junk", "cur", m.Key))
			if err != nil {
				return err
			}
		}

		var dryRun string
		if m.DryRun {
			dryRun = "-- dry run (nothing happened to this mail!)"
		}

		log.WithFields(log.Fields{
			"mail": m.Key,
		}).Info("Moved to Junk folder" + dryRun)
	}

	err = m.Unload(dir)

	return err
}

// Junk returns true if the wordlist is classified as a junk mail using Bayes'
// rule. If required, it also returns the calculated probability of being junk,
// but this is typically not needed.
func Junk(db *bolt.DB, wordlist []string) (junk bool, prob float64, err error) {
	var probabilities []float64

	// If the wordlist is too long, let us only select a random sample
	// for analysis. This prevents cheating by adding lots of good text
	// to a Junk mail
	if len(wordlist) > 50 {
		wordlistTemp := make(map[string]interface{})

		rand.Seed(time.Now().UnixNano())

		for len(wordlistTemp) < 50 {
			wordlistTemp[wordlist[rand.Intn(len(wordlist)-1)]] = nil

		}

		var wordlistTempSlice []string
		for key := range wordlistTemp {
			wordlistTempSlice = append(wordlistTempSlice, key)
		}
		wordlist = wordlistTempSlice
	}

	// initial value should be no junk
	prob = 1.0

	for _, val := range wordlist {
		var p float64
		p, err = classificationWord(db, val)
		if err != nil {
			return false, 0.0, err
		}
		probabilities = append(probabilities, p)
	}

	if len(probabilities) > 0 {
		prob = stat.HarmonicMean(probabilities, nil)
	}
	if prob < 0.5 {
		return true, (1 - prob), err
	}

	return false, (1 - prob), err
}
<|endoftext|>