<commit_before>package main

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestGetStateFile(t *testing.T) {
	sPath := getStateFile("/var/lib", "C:/Windows/hoge")
	assert.Equal(t, sPath, "/var/lib/C/Windows/hoge", "drive letter should be cared")

	sPath = getStateFile("/var/lib", "/linux/hoge")
	assert.Equal(t, sPath, "/var/lib/linux/hoge", "drive letter should be cared")
}

func TestWriteBytesToSkip(t *testing.T) {
	f := ".tmp/fuga/piyo"
	err := writeBytesToSkip(f, 15)
	assert.Equal(t, err, nil, "err should be nil")

	skipBytes, err := getBytesToSkip(f)
	assert.Equal(t, err, nil, "err should be nil")
	assert.Equal(t, skipBytes, int64(15))
}

func TestSearchReader(t *testing.T) {
	dir, err := ioutil.TempDir("", "check-log-test")
	if err != nil {
		t.Fatalf("TempDir failed: %s", err)
	}
	defer os.RemoveAll(dir)

	opts := &logOpts{
		StateDir: dir,
		LogFile:  filepath.Join(dir, "dummy"),
		Pattern:  `FATAL`,
	}
	opts.prepare()

	content := `FATAL 11
OK
FATAL 22
Fatal
`
	r := strings.NewReader(content)
	warnNum, critNum, readBytes, errLines, err := opts.searchReader(r)

	assert.Equal(t, int64(2), warnNum, "warnNum should be 2")
	assert.Equal(t, int64(2), critNum, "critNum should be 2")
	assert.Equal(t, "FATAL 11\nFATAL 22\n", errLines, "invalid errLines")
	assert.Equal(t, int64(len(content)), readBytes, "readBytes should be 26")
}

func TestRun(t *testing.T) {
	dir, err := ioutil.TempDir("", "check-log-test")
	if err != nil {
		t.Errorf("something went wrong")
	}
	defer os.RemoveAll(dir)

	logf := filepath.Join(dir, "dummy")
	fh, _ := os.Create(logf)
	defer fh.Close()

	ptn := `FATAL`
	opts, _ := parseArgs([]string{"-s", dir, "-f", logf, "-p", ptn})
	opts.prepare()

	stateFile := getStateFile(opts.StateDir, logf)

	bytes, _ := getBytesToSkip(stateFile)
	assert.Equal(t, int64(0), bytes, "something went wrong")

	testEmpty := func() {
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(0), bytes, "something went wrong")
	}
	testEmpty()

	l1 := "FATAL\nFATAL\n"
	test2Line := func() {
		fh.WriteString(l1)
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(2), w, "something went wrong")
		assert.Equal(t, int64(2), c, "something went wrong")
		assert.Equal(t, l1, errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(len(l1)), bytes, "something went wrong")
	}
	test2Line()

	testReadAgain := func() {
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(len(l1)), bytes, "something went wrong")
	}
	testReadAgain()

	l2 := "SUCCESS\n"
	testRecover := func() {
		fh.WriteString(l2)
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(len(l1)+len(l2)), bytes, "something went wrong")
	}
	testRecover()

	testSuccessAgain := func() {
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(len(l1)+len(l2)), bytes, "something went wrong")
	}
	testSuccessAgain()

	testErrorAgain := func() {
		fh.WriteString(l1)
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(2), w, "something went wrong")
		assert.Equal(t, int64(2), c, "something went wrong")
		assert.Equal(t, l1, errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(len(l1)*2+len(l2)), bytes, "something went wrong")
	}
	testErrorAgain()

	testRecoverAgain := func() {
		fh.WriteString(l2)
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(len(l1)*2+len(l2)*2), bytes, "something went wrong")
	}
	testRecoverAgain()

	testRotate := func() {
		fh.Close()
		os.Remove(logf)
		fh, _ = os.Create(logf)

		fh.WriteString(l2)
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(len(l2)), bytes, "something went wrong")
	}
	testRotate()
}

func TestSearchReaderWithLevel(t *testing.T) {
	dir, err := ioutil.TempDir("", "check-log-test")
	if err != nil {
		t.Errorf("something went wrong")
	}
	defer os.RemoveAll(dir)

	logf := filepath.Join(dir, "dummy")
	ptn := `FATAL level:([0-9]+)`
	opts, _ := parseArgs([]string{"-s", dir, "-f", logf, "-i", "-p", ptn, "--critical-level=17", "--warning-level=11"})
	if !reflect.DeepEqual(&logOpts{
		StateDir:        dir,
		LogFile:         filepath.Join(dir, "dummy"),
		CaseInsensitive: true,
		Pattern:         `FATAL level:([0-9]+)`,
		WarnLevel:       11,
		CritLevel:       17,
	}, opts) {
		t.Errorf("something went wrong")
	}
	opts.prepare()

	content := `FATAL level:11
OK
FATAL level:22
Fatal level:17
`
	r := strings.NewReader(content)
	warnNum, critNum, readBytes, errLines, err := opts.searchReader(r)

	assert.Equal(t, int64(2), warnNum, "warnNum should be 2")
	assert.Equal(t, int64(1), critNum, "critNum should be 1")
	assert.Equal(t, "FATAL level:22\nFatal level:17\n", errLines, "invalid errLines")
	assert.Equal(t, int64(len(content)), readBytes, "readBytes should be 26")
}
<commit_msg>add failing test for the log file is not ended with newline character.<commit_after>package main

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestGetStateFile(t *testing.T) {
	sPath := getStateFile("/var/lib", "C:/Windows/hoge")
	assert.Equal(t, sPath, "/var/lib/C/Windows/hoge", "drive letter should be cared")

	sPath = getStateFile("/var/lib", "/linux/hoge")
	assert.Equal(t, sPath, "/var/lib/linux/hoge", "drive letter should be cared")
}

func TestWriteBytesToSkip(t *testing.T) {
	f := ".tmp/fuga/piyo"
	err := writeBytesToSkip(f, 15)
	assert.Equal(t, err, nil, "err should be nil")

	skipBytes, err := getBytesToSkip(f)
	assert.Equal(t, err, nil, "err should be nil")
	assert.Equal(t, skipBytes, int64(15))
}

func TestSearchReader(t *testing.T) {
	dir, err := ioutil.TempDir("", "check-log-test")
	if err != nil {
		t.Fatalf("TempDir failed: %s", err)
	}
	defer os.RemoveAll(dir)

	opts := &logOpts{
		StateDir: dir,
		LogFile:  filepath.Join(dir, "dummy"),
		Pattern:  `FATAL`,
	}
	opts.prepare()

	content := `FATAL 11
OK
FATAL 22
Fatal
`
	r := strings.NewReader(content)
	warnNum, critNum, readBytes, errLines, err := opts.searchReader(r)

	assert.Equal(t, int64(2), warnNum, "warnNum should be 2")
	assert.Equal(t, int64(2), critNum, "critNum should be 2")
	assert.Equal(t, "FATAL 11\nFATAL 22\n", errLines, "invalid errLines")
	assert.Equal(t, int64(len(content)), readBytes, "readBytes should be 26")
}

func TestRun(t *testing.T) {
	dir, err := ioutil.TempDir("", "check-log-test")
	if err != nil {
		t.Errorf("something went wrong")
	}
	defer os.RemoveAll(dir)

	logf := filepath.Join(dir, "dummy")
	fh, _ := os.Create(logf)
	defer fh.Close()

	ptn := `FATAL`
	opts, _ := parseArgs([]string{"-s", dir, "-f", logf, "-p", ptn})
	opts.prepare()

	stateFile := getStateFile(opts.StateDir, logf)

	bytes, _ := getBytesToSkip(stateFile)
	assert.Equal(t, int64(0), bytes, "something went wrong")

	testEmpty := func() {
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(0), bytes, "something went wrong")
	}
	testEmpty()

	l1 := "FATAL\nFATAL\n"
	test2Line := func() {
		fh.WriteString(l1)
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(2), w, "something went wrong")
		assert.Equal(t, int64(2), c, "something went wrong")
		assert.Equal(t, l1, errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(len(l1)), bytes, "something went wrong")
	}
	test2Line()

	testReadAgain := func() {
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(len(l1)), bytes, "something went wrong")
	}
	testReadAgain()

	l2 := "SUCCESS\n"
	testRecover := func() {
		fh.WriteString(l2)
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(len(l1)+len(l2)), bytes, "something went wrong")
	}
	testRecover()

	testSuccessAgain := func() {
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(len(l1)+len(l2)), bytes, "something went wrong")
	}
	testSuccessAgain()

	testErrorAgain := func() {
		fh.WriteString(l1)
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(2), w, "something went wrong")
		assert.Equal(t, int64(2), c, "something went wrong")
		assert.Equal(t, l1, errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(len(l1)*2+len(l2)), bytes, "something went wrong")
	}
	testErrorAgain()

	testRecoverAgain := func() {
		fh.WriteString(l2)
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(len(l1)*2+len(l2)*2), bytes, "something went wrong")
	}
	testRecoverAgain()

	testRotate := func() {
		fh.Close()
		os.Remove(logf)
		fh, _ = os.Create(logf)

		fh.WriteString(l2)
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(len(l2)), bytes, "something went wrong")
	}
	testRotate()
}

func TestRunWithMiddleOfLine(t *testing.T) {
	dir, err := ioutil.TempDir("", "check-log-test")
	if err != nil {
		t.Errorf("something went wrong")
	}
	defer os.RemoveAll(dir)

	logf := filepath.Join(dir, "dummy")
	fh, _ := os.Create(logf)
	defer fh.Close()

	ptn := `FATAL`
	opts, _ := parseArgs([]string{"-s", dir, "-f", logf, "-p", ptn})
	opts.prepare()

	stateFile := getStateFile(opts.StateDir, logf)

	bytes, _ := getBytesToSkip(stateFile)
	assert.Equal(t, int64(0), bytes, "something went wrong")

	testMiddleOfLine := func() {
		fh.WriteString("FATA")
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(0), bytes, "something went wrong")
	}
	testMiddleOfLine()

	testFail := func() {
		fh.WriteString("L\nSUCC")
		w, c, errLines, err := opts.searchLog(logf)
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(1), w, "something went wrong")
		assert.Equal(t, int64(1), c, "something went wrong")
		assert.Equal(t, "FATAL\n", errLines, "something went wrong")

		bytes, _ = getBytesToSkip(stateFile)
		assert.Equal(t, int64(len("FATAL\n")), bytes, "something went wrong")
	}
	testFail()
}

func TestSearchReaderWithLevel(t *testing.T) {
	dir, err := ioutil.TempDir("", "check-log-test")
	if err != nil {
		t.Errorf("something went wrong")
	}
	defer os.RemoveAll(dir)

	logf := filepath.Join(dir, "dummy")
	ptn := `FATAL level:([0-9]+)`
	opts, _ := parseArgs([]string{"-s", dir, "-f", logf, "-i", "-p", ptn, "--critical-level=17", "--warning-level=11"})
	if !reflect.DeepEqual(&logOpts{
		StateDir:        dir,
		LogFile:         filepath.Join(dir, "dummy"),
		CaseInsensitive: true,
		Pattern:         `FATAL level:([0-9]+)`,
		WarnLevel:       11,
		CritLevel:       17,
	}, opts) {
		t.Errorf("something went wrong")
	}
	opts.prepare()

	content := `FATAL level:11
OK
FATAL level:22
Fatal level:17
`
	r := strings.NewReader(content)
	warnNum, critNum, readBytes, errLines, err := opts.searchReader(r)

	assert.Equal(t, int64(2), warnNum, "warnNum should be 2")
	assert.Equal(t, int64(1), critNum, "critNum should be 1")
	assert.Equal(t, "FATAL level:22\nFatal level:17\n", errLines, "invalid errLines")
	assert.Equal(t, int64(len(content)), readBytes, "readBytes should be 26")
}
<|endoftext|>


<commit_before>package prompt

import (
	"github.com/antham/strumt"
	"io"

	"github.com/antham/chyle/prompt/internal/builder"
)

// Prompts held prompts
type Prompts struct {
	prompts strumt.Prompts
}

// New creates a new prompt chain
func New(reader io.Reader, writer io.Writer) Prompts {
	return Prompts{strumt.NewPromptsFromReaderAndWriter(reader, writer)}
}

func (p *Prompts) populatePrompts(prompts []strumt.Prompter) {
	for _, item := range prompts {
		switch prompt := item.(type) {
		case strumt.LinePrompter:
			p.prompts.AddLinePrompter(prompt)
		case strumt.MultilinePrompter:
			p.prompts.AddMultilinePrompter(prompt)
		}
	}
}

// Run starts a prompt session
func (p *Prompts) Run() builder.Store {
	store := &builder.Store{}
	prompts := mergePrompters(
		newMainMenu(),
		newMandatoryOption(store),
		newMatchers(store),
		newExtractors(store),
		newDecorators(store),
		newSenders(store),
	)

	p.populatePrompts(prompts)

	p.prompts.SetFirst("referenceFrom")
	p.prompts.Run()

	return *store
}
<commit_msg>ref(prompt/prompt) : remove useless statement<commit_after>package prompt

import (
	"github.com/antham/strumt"
	"io"

	"github.com/antham/chyle/prompt/internal/builder"
)

// Prompts held prompts
type Prompts struct {
	prompts strumt.Prompts
}

// New creates a new prompt chain
func New(reader io.Reader, writer io.Writer) Prompts {
	return Prompts{strumt.NewPromptsFromReaderAndWriter(reader, writer)}
}

func (p *Prompts) populatePrompts(prompts []strumt.Prompter) {
	for _, item := range prompts {
		switch prompt := item.(type) {
		case strumt.LinePrompter:
			p.prompts.AddLinePrompter(prompt)
		}
	}
}

// Run starts a prompt session
func (p *Prompts) Run() builder.Store {
	store := &builder.Store{}
	prompts := mergePrompters(
		newMainMenu(),
		newMandatoryOption(store),
		newMatchers(store),
		newExtractors(store),
		newDecorators(store),
		newSenders(store),
	)

	p.populatePrompts(prompts)

	p.prompts.SetFirst("referenceFrom")
	p.prompts.Run()

	return *store
}
<|endoftext|>


<commit_before>package prompt

import "unicode"
import "os"
import "time"
import "bytes"
import "fmt"

func Format2Prompt(format string) string {
	if format == "" {
		format = "[$P]$_$$$S"
	}
	var buffer bytes.Buffer
	lastchar := '\000'
	for _, ch := range format {
		if lastchar == '$' {
			c := unicode.ToLower(ch)
			if c == 'a' {
				buffer.WriteRune('&')
			} else if c == 'b' {
				buffer.WriteRune('|')
			} else if c == 'c' {
				buffer.WriteRune('(')
			} else if c == 'd' {
				year, month, day := time.Now().Date()
				buffer.WriteString(
					fmt.Sprintf("%04d-%02d-%02d",
						year, month+1, day))
			} else if c == 'e' {
				buffer.WriteRune('\x1B')
			} else if c == 'f' {
				buffer.WriteRune(')')
			} else if c == 'g' {
				buffer.WriteRune('>')
			} else if c == 'h' {
				buffer.WriteRune('\b')
			} else if c == 'l' {
				buffer.WriteRune('<')
			} else if c == 'n' {
				wd, err := os.Getwd()
				if err == nil {
					buffer.WriteString(wd[:2])
				}
			} else if c == 'p' {
				wd, err := os.Getwd()
				if err == nil {
					buffer.WriteString(wd)
				}
			} else if c == 'q' {
				buffer.WriteRune('=')
			} else if c == 's' {
				buffer.WriteRune(' ')
			} else if c == 't' {
				now := time.Now()
				hour, min, sec := now.Clock()
				nnn := now.Nanosecond() / 10000000
				buffer.WriteString(
					fmt.Sprintf("%02d:%02d:%02d.%02d",
						hour, min, sec, nnn))
			} else if c == 'v' {
				// Windows Version
			} else if c == '_' {
				buffer.WriteRune('\n')
			} else if c == '$' {
				buffer.WriteRune('$')
				ch = '\000'
			} else {
				buffer.WriteRune('$')
				buffer.WriteRune(ch)
			}
		} else if ch != '$' {
			buffer.WriteRune(ch)
		}
		lastchar = ch
	}
	return buffer.String()
}
<commit_msg>プロンプトで %HOME% or %USERPROFILE% を ~ に置換するようにした。<commit_after>package prompt

import "bytes"
import "fmt"
import "os"
import "strings"
import "time"
import "unicode"

func getHome() string {
	home := os.Getenv("HOME")
	if home == "" {
		home = os.Getenv("USERPROFILE")
	}
	return home
}

func Format2Prompt(format string) string {
	if format == "" {
		format = "[$P]$_$$$S"
	}
	var buffer bytes.Buffer
	lastchar := '\000'
	for _, ch := range format {
		if lastchar == '$' {
			c := unicode.ToLower(ch)
			if c == 'a' {
				buffer.WriteRune('&')
			} else if c == 'b' {
				buffer.WriteRune('|')
			} else if c == 'c' {
				buffer.WriteRune('(')
			} else if c == 'd' {
				year, month, day := time.Now().Date()
				buffer.WriteString(
					fmt.Sprintf("%04d-%02d-%02d",
						year, month+1, day))
			} else if c == 'e' {
				buffer.WriteRune('\x1B')
			} else if c == 'f' {
				buffer.WriteRune(')')
			} else if c == 'g' {
				buffer.WriteRune('>')
			} else if c == 'h' {
				buffer.WriteRune('\b')
			} else if c == 'l' {
				buffer.WriteRune('<')
			} else if c == 'n' {
				wd, err := os.Getwd()
				if err == nil {
					buffer.WriteString(wd[:2])
				}
			} else if c == 'p' {
				wd, err := os.Getwd()
				if err == nil {
					home := getHome()
					homeLen := len(home)
					if len(wd) >= homeLen && strings.EqualFold(home, wd[0:homeLen]) {
						wd = "~" + wd[homeLen:]
					}
					buffer.WriteString(strings.Replace(wd, "\\", "/", -1))
				}
			} else if c == 'q' {
				buffer.WriteRune('=')
			} else if c == 's' {
				buffer.WriteRune(' ')
			} else if c == 't' {
				now := time.Now()
				hour, min, sec := now.Clock()
				nnn := now.Nanosecond() / 10000000
				buffer.WriteString(
					fmt.Sprintf("%02d:%02d:%02d.%02d",
						hour, min, sec, nnn))
			} else if c == 'v' {
				// Windows Version
			} else if c == '_' {
				buffer.WriteRune('\n')
			} else if c == '$' {
				buffer.WriteRune('$')
				ch = '\000'
			} else {
				buffer.WriteRune('$')
				buffer.WriteRune(ch)
			}
		} else if ch != '$' {
			buffer.WriteRune(ch)
		}
		lastchar = ch
	}
	return buffer.String()
}
<|endoftext|>


<commit_before>package routers

import (
	"github.com/go-macaron/binding"
	"github.com/go-macaron/cache"
	"github.com/go-macaron/captcha"
	"github.com/go-macaron/gzip"
	"github.com/go-macaron/session"
	"github.com/go-macaron/toolbox"
	"github.com/ouqiang/gocron/modules/app"
	"github.com/ouqiang/gocron/modules/logger"
	"github.com/ouqiang/gocron/modules/utils"
	"github.com/ouqiang/gocron/routers/host"
	"github.com/ouqiang/gocron/routers/install"
	"github.com/ouqiang/gocron/routers/loginlog"
	"github.com/ouqiang/gocron/routers/manage"
	"github.com/ouqiang/gocron/routers/task"
	"github.com/ouqiang/gocron/routers/tasklog"
	"github.com/ouqiang/gocron/routers/user"
	"gopkg.in/macaron.v1"
	"html/template"
	"strconv"
	"strings"
	"time"
)

// 静态文件目录
const StaticDir = "public"

// 路由注册
func Register(m *macaron.Macaron) {
	// 所有GET方法，自动注册HEAD方法
	m.SetAutoHead(true)
	// 首页
	m.Get("/", Home)
	// 系统安装
	m.Group("/install", func() {
		m.Get("", install.Create)
		m.Post("/store", binding.Bind(install.InstallForm{}), install.Store)
	})

	// 用户
	m.Group("/user", func() {
		m.Get("", user.Index)
		m.Get("/create", user.Create)
		m.Get("/edit/:id", user.Edit)
		m.Post("/store", binding.Bind(user.UserForm{}), user.Store)
		m.Post("/remove/:id", user.Remove)
		m.Get("/login", user.Login)
		m.Post("/login", user.ValidateLogin)
		m.Get("/logout", user.Logout)
		m.Post("/enable/:id", user.Enable)
		m.Post("/disable/:id", user.Disable)
		m.Get("/editMyPassword", user.EditMyPassword)
		m.Post("/editMyPassword", user.UpdateMyPassword)
		m.Get("/editPassword/:id", user.EditPassword)
		m.Post("/editPassword/:id", user.UpdatePassword)
	})

	// 定时任务
	m.Group("/task", func() {
		m.Get("/create", task.Create)
		m.Post("/store", binding.Bind(task.TaskForm{}), task.Store)
		m.Get("/edit/:id", task.Edit)
		m.Get("", task.Index)
		m.Get("/log", tasklog.Index)
		m.Post("/log/clear", tasklog.Clear)
		m.Post("/remove/:id", task.Remove)
		m.Post("/enable/:id", task.Enable)
		m.Post("/disable/:id", task.Disable)
		m.Get("/run/:id", task.Run)
	})

	// 主机
	m.Group("/host", func() {
		m.Get("/create", host.Create)
		m.Get("/edit/:id", host.Edit)
		m.Post("/store", binding.Bind(host.HostForm{}), host.Store)
		m.Get("", host.Index)
		m.Get("/ping/:id", host.Ping)
		m.Post("/remove/:id", host.Remove)
	})

	// 管理
	m.Group("/manage", func() {
		m.Group("/slack", func() {
			m.Get("/", manage.Slack)
			m.Get("/edit", manage.EditSlack)
			m.Post("/url", manage.UpdateSlackUrl)
			m.Post("/channel", manage.CreateSlackChannel)
			m.Post("/channel/remove/:id", manage.RemoveSlackChannel)
		})
		m.Group("/mail", func() {
			m.Get("/", manage.Mail)
			m.Get("/edit", manage.EditMail)
			m.Post("/server", binding.Bind(manage.MailServerForm{}), manage.UpdateMailServer)
			m.Post("/server/clear", manage.ClearMailServer)
			m.Post("/user", manage.CreateMailUser)
			m.Post("/user/remove/:id", manage.RemoveMailUser)
		})
		m.Get("/login-log", loginlog.Index)
	})

	// API
	m.Group("/api/v1", func() {
		m.Post("/tasklog/remove/:id", tasklog.Remove)
		m.Post("/task/enable/:id", task.Enable)
		m.Post("/task/disable/:id", task.Disable)
	}, apiAuth)

	// 404错误
	m.NotFound(func(ctx *macaron.Context) {
		if isGetRequest(ctx) && !isAjaxRequest(ctx) {
			ctx.Data["Title"] = "404 - NOT FOUND"
			ctx.HTML(404, "error/404")
		} else {
			json := utils.JsonResponse{}
			ctx.Resp.Write([]byte(json.Failure(utils.NotFound, "您访问的地址不存在")))
		}
	})
	// 50x错误
	m.InternalServerError(func(ctx *macaron.Context) {
		logger.Debug("500错误")
		if isGetRequest(ctx) && !isAjaxRequest(ctx) {
			ctx.Data["Title"] = "500 - INTERNAL SERVER ERROR"
			ctx.HTML(500, "error/500")
		} else {
			json := utils.JsonResponse{}
			ctx.Resp.Write([]byte(json.Failure(utils.ServerError, "网站暂时无法访问,请稍后再试")))
		}
	})
}

// 中间件注册
func RegisterMiddleware(m *macaron.Macaron) {
	m.Use(macaron.Logger())
	m.Use(macaron.Recovery())
	if macaron.Env != macaron.DEV {
		m.Use(gzip.Gziper())
	}
	m.Use(macaron.Static(StaticDir))
	m.Use(macaron.Renderer(macaron.RenderOptions{
		Directory:  "templates",
		Extensions: []string{".html"},
		// 模板语法分隔符，默认为 ["{{", "}}"]
		Delims: macaron.Delims{"{{{", "}}}"},
		// 追加的 Content-Type 头信息，默认为 "UTF-8"
		Charset: "UTF-8",
		// 渲染具有缩进格式的 JSON，默认为不缩进
		IndentJSON: true,
		// 渲染具有缩进格式的 XML，默认为不缩进
		IndentXML: true,
		Funcs: []template.FuncMap{map[string]interface{}{
			"HostFormat": func(index int) bool {
				return (index+1)%3 == 0
			},
			"unescape": func(str string) template.HTML {
				return template.HTML(str)
			},
		}},
	}))
	m.Use(cache.Cacher())
	m.Use(captcha.Captchaer())
	m.Use(session.Sessioner(session.Options{
		Provider:       "file",
		ProviderConfig: app.DataDir + "/sessions",
	}))
	m.Use(toolbox.Toolboxer(m))
	checkAppInstall(m)
	m.Use(func(ctx *macaron.Context, sess session.Store) {
		if app.Installed {
			ipAuth(ctx)
			userAuth(ctx, sess)
			urlAuth(ctx, sess)
			setShareData(ctx, sess)
		}
	})
}

// region 自定义中间件

/** 系统未安装，重定向到安装页面 **/
func checkAppInstall(m *macaron.Macaron) {
	m.Use(func(ctx *macaron.Context) {
		installUrl := "/install"
		if strings.HasPrefix(ctx.Req.URL.Path, installUrl) {
			return
		}
		if !app.Installed {
			ctx.Redirect(installUrl)
		}
	})
}

// IP验证, 通过反向代理访问gocron，需设置Header X-Real-IP才能获取到客户端真实IP
func ipAuth(ctx *macaron.Context) {
	allowIpsStr := app.Setting.AllowIps
	if allowIpsStr == "" {
		return
	}
	clientIp := ctx.RemoteAddr()
	allowIps := strings.Split(allowIpsStr, ",")
	if !utils.InStringSlice(allowIps, clientIp) {
		logger.Warnf("非法IP访问-%s", clientIp)
		ctx.Status(403)
	}
}

// 用户认证
func userAuth(ctx *macaron.Context, sess session.Store) {
	if user.IsLogin(sess) {
		return
	}
	uri := ctx.Req.URL.Path
	found := false
	excludePaths := []string{"/install", "/user/login", "/api"}
	for _, path := range excludePaths {
		if strings.HasPrefix(uri, path) {
			found = true
			break
		}
	}
	if !found {
		ctx.Redirect("/user/login")
	}
}

// URL权限验证
func urlAuth(ctx *macaron.Context, sess session.Store)  {
	if user.IsAdmin(sess) {
		return
	}
	// 普通用户允许访问的URL地址
	allowPaths := []string{
		"",
		"/task",
		"/task/log",
		"/host",
		"/user/login",
		"/user/logout",
		"/user/editMyPassword",
	}
	uri := strings.TrimSpace(ctx.Req.URL.Path)
	uri = strings.TrimRight(uri, "/")
	for _, path := range allowPaths {
		if path == uri {
			return
		}
	}

	ctx.Status(403)

}

/** 设置共享数据 **/
func setShareData(ctx *macaron.Context, sess session.Store) {
	ctx.Data["URI"] = ctx.Req.URL.Path
	urlPath := strings.TrimPrefix(ctx.Req.URL.Path, "/")
	paths := strings.Split(urlPath, "/")
	ctx.Data["Controller"] = ""
	ctx.Data["Action"] = ""
	if len(paths) > 0 {
		ctx.Data["Controller"] = paths[0]
	}
	if len(paths) > 1 {
		ctx.Data["Action"] = paths[1]
	}
	ctx.Data["LoginUsername"] = user.Username(sess)
	ctx.Data["LoginUid"] = user.Uid(sess)
	ctx.Data["IsAdmin"] = user.IsAdmin(sess)
	ctx.Data["AppName"] = app.Setting.AppName
}

/** API接口签名验证 **/
func apiAuth(ctx *macaron.Context) {
	if !app.Setting.ApiSignEnable {
		return
	}
	apiKey := strings.TrimSpace(app.Setting.ApiKey)
	apiSecret := strings.TrimSpace(app.Setting.ApiSecret)
	json := utils.JsonResponse{}
	if apiKey == "" || apiSecret == "" {
		msg := json.CommonFailure("使用API前, 请先配置密钥")
		ctx.Write([]byte(msg))
		return
	}
	currentTimestamp := time.Now().Unix()
	time := ctx.QueryInt64("time")
	if time <= 0 {
		msg := json.CommonFailure("参数time不能为空")
		ctx.Write([]byte(msg))
		return
	}
	if time < (currentTimestamp - 1800) {
		msg := json.CommonFailure("time无效")
		ctx.Write([]byte(msg))
		return
	}
	sign := ctx.QueryTrim("sign")
	if sign == "" {
		msg := json.CommonFailure("参数sign不能为空")
		ctx.Write([]byte(msg))
		return
	}
	raw := apiKey + strconv.FormatInt(time, 10) + strings.TrimSpace(ctx.Req.URL.Path) + apiSecret
	realSign := utils.Md5(raw)
	if sign != realSign {
		msg := json.CommonFailure("签名验证失败")
		ctx.Write([]byte(msg))
		return
	}
}

// endregion

func isAjaxRequest(ctx *macaron.Context) bool {
	req := ctx.Req.Header.Get("X-Requested-With")
	if req == "XMLHttpRequest" {
		return true
	}

	return false
}

func isGetRequest(ctx *macaron.Context) bool {
	return ctx.Req.Method == "GET"
}
<commit_msg>bugfix: API调用报403错误<commit_after>package routers

import (
	"github.com/go-macaron/binding"
	"github.com/go-macaron/cache"
	"github.com/go-macaron/captcha"
	"github.com/go-macaron/gzip"
	"github.com/go-macaron/session"
	"github.com/go-macaron/toolbox"
	"github.com/ouqiang/gocron/modules/app"
	"github.com/ouqiang/gocron/modules/logger"
	"github.com/ouqiang/gocron/modules/utils"
	"github.com/ouqiang/gocron/routers/host"
	"github.com/ouqiang/gocron/routers/install"
	"github.com/ouqiang/gocron/routers/loginlog"
	"github.com/ouqiang/gocron/routers/manage"
	"github.com/ouqiang/gocron/routers/task"
	"github.com/ouqiang/gocron/routers/tasklog"
	"github.com/ouqiang/gocron/routers/user"
	"gopkg.in/macaron.v1"
	"html/template"
	"strconv"
	"strings"
	"time"
)

// 静态文件目录
const StaticDir = "public"

// 路由注册
func Register(m *macaron.Macaron) {
	// 所有GET方法，自动注册HEAD方法
	m.SetAutoHead(true)
	// 首页
	m.Get("/", Home)
	// 系统安装
	m.Group("/install", func() {
		m.Get("", install.Create)
		m.Post("/store", binding.Bind(install.InstallForm{}), install.Store)
	})

	// 用户
	m.Group("/user", func() {
		m.Get("", user.Index)
		m.Get("/create", user.Create)
		m.Get("/edit/:id", user.Edit)
		m.Post("/store", binding.Bind(user.UserForm{}), user.Store)
		m.Post("/remove/:id", user.Remove)
		m.Get("/login", user.Login)
		m.Post("/login", user.ValidateLogin)
		m.Get("/logout", user.Logout)
		m.Post("/enable/:id", user.Enable)
		m.Post("/disable/:id", user.Disable)
		m.Get("/editMyPassword", user.EditMyPassword)
		m.Post("/editMyPassword", user.UpdateMyPassword)
		m.Get("/editPassword/:id", user.EditPassword)
		m.Post("/editPassword/:id", user.UpdatePassword)
	})

	// 定时任务
	m.Group("/task", func() {
		m.Get("/create", task.Create)
		m.Post("/store", binding.Bind(task.TaskForm{}), task.Store)
		m.Get("/edit/:id", task.Edit)
		m.Get("", task.Index)
		m.Get("/log", tasklog.Index)
		m.Post("/log/clear", tasklog.Clear)
		m.Post("/remove/:id", task.Remove)
		m.Post("/enable/:id", task.Enable)
		m.Post("/disable/:id", task.Disable)
		m.Get("/run/:id", task.Run)
	})

	// 主机
	m.Group("/host", func() {
		m.Get("/create", host.Create)
		m.Get("/edit/:id", host.Edit)
		m.Post("/store", binding.Bind(host.HostForm{}), host.Store)
		m.Get("", host.Index)
		m.Get("/ping/:id", host.Ping)
		m.Post("/remove/:id", host.Remove)
	})

	// 管理
	m.Group("/manage", func() {
		m.Group("/slack", func() {
			m.Get("/", manage.Slack)
			m.Get("/edit", manage.EditSlack)
			m.Post("/url", manage.UpdateSlackUrl)
			m.Post("/channel", manage.CreateSlackChannel)
			m.Post("/channel/remove/:id", manage.RemoveSlackChannel)
		})
		m.Group("/mail", func() {
			m.Get("/", manage.Mail)
			m.Get("/edit", manage.EditMail)
			m.Post("/server", binding.Bind(manage.MailServerForm{}), manage.UpdateMailServer)
			m.Post("/server/clear", manage.ClearMailServer)
			m.Post("/user", manage.CreateMailUser)
			m.Post("/user/remove/:id", manage.RemoveMailUser)
		})
		m.Get("/login-log", loginlog.Index)
	})

	// API
	m.Group("/api/v1", func() {
		m.Post("/tasklog/remove/:id", tasklog.Remove)
		m.Post("/task/enable/:id", task.Enable)
		m.Post("/task/disable/:id", task.Disable)
	}, apiAuth)

	// 404错误
	m.NotFound(func(ctx *macaron.Context) {
		if isGetRequest(ctx) && !isAjaxRequest(ctx) {
			ctx.Data["Title"] = "404 - NOT FOUND"
			ctx.HTML(404, "error/404")
		} else {
			json := utils.JsonResponse{}
			ctx.Resp.Write([]byte(json.Failure(utils.NotFound, "您访问的地址不存在")))
		}
	})
	// 50x错误
	m.InternalServerError(func(ctx *macaron.Context) {
		logger.Debug("500错误")
		if isGetRequest(ctx) && !isAjaxRequest(ctx) {
			ctx.Data["Title"] = "500 - INTERNAL SERVER ERROR"
			ctx.HTML(500, "error/500")
		} else {
			json := utils.JsonResponse{}
			ctx.Resp.Write([]byte(json.Failure(utils.ServerError, "网站暂时无法访问,请稍后再试")))
		}
	})
}

// 中间件注册
func RegisterMiddleware(m *macaron.Macaron) {
	m.Use(macaron.Logger())
	m.Use(macaron.Recovery())
	if macaron.Env != macaron.DEV {
		m.Use(gzip.Gziper())
	}
	m.Use(macaron.Static(StaticDir))
	m.Use(macaron.Renderer(macaron.RenderOptions{
		Directory:  "templates",
		Extensions: []string{".html"},
		// 模板语法分隔符，默认为 ["{{", "}}"]
		Delims: macaron.Delims{"{{{", "}}}"},
		// 追加的 Content-Type 头信息，默认为 "UTF-8"
		Charset: "UTF-8",
		// 渲染具有缩进格式的 JSON，默认为不缩进
		IndentJSON: true,
		// 渲染具有缩进格式的 XML，默认为不缩进
		IndentXML: true,
		Funcs: []template.FuncMap{map[string]interface{}{
			"HostFormat": func(index int) bool {
				return (index+1)%3 == 0
			},
			"unescape": func(str string) template.HTML {
				return template.HTML(str)
			},
		}},
	}))
	m.Use(cache.Cacher())
	m.Use(captcha.Captchaer())
	m.Use(session.Sessioner(session.Options{
		Provider:       "file",
		ProviderConfig: app.DataDir + "/sessions",
	}))
	m.Use(toolbox.Toolboxer(m))
	checkAppInstall(m)
	m.Use(func(ctx *macaron.Context, sess session.Store) {
		if app.Installed {
			ipAuth(ctx)
			userAuth(ctx, sess)
			urlAuth(ctx, sess)
			setShareData(ctx, sess)
		}
	})
}

// region 自定义中间件

/** 系统未安装，重定向到安装页面 **/
func checkAppInstall(m *macaron.Macaron) {
	m.Use(func(ctx *macaron.Context) {
		installUrl := "/install"
		if strings.HasPrefix(ctx.Req.URL.Path, installUrl) {
			return
		}
		if !app.Installed {
			ctx.Redirect(installUrl)
		}
	})
}

// IP验证, 通过反向代理访问gocron，需设置Header X-Real-IP才能获取到客户端真实IP
func ipAuth(ctx *macaron.Context) {
	allowIpsStr := app.Setting.AllowIps
	if allowIpsStr == "" {
		return
	}
	clientIp := ctx.RemoteAddr()
	allowIps := strings.Split(allowIpsStr, ",")
	if !utils.InStringSlice(allowIps, clientIp) {
		logger.Warnf("非法IP访问-%s", clientIp)
		ctx.Status(403)
	}
}

// 用户认证
func userAuth(ctx *macaron.Context, sess session.Store) {
	if user.IsLogin(sess) {
		return
	}
	uri := ctx.Req.URL.Path
	found := false
	excludePaths := []string{"/install", "/user/login", "/api"}
	for _, path := range excludePaths {
		if strings.HasPrefix(uri, path) {
			found = true
			break
		}
	}
	if !found {
		ctx.Redirect("/user/login")
	}
}

// URL权限验证
func urlAuth(ctx *macaron.Context, sess session.Store)  {
	if user.IsAdmin(sess) {
		return
	}
	// 普通用户允许访问的URL地址
	allowPaths := []string{
		"",
		"/task",
		"/task/log",
		"/host",
		"/user/login",
		"/user/logout",
		"/user/editMyPassword",
		"/api",
	}
	uri := strings.TrimSpace(ctx.Req.URL.Path)
	uri = strings.TrimRight(uri, "/")
	for _, path := range allowPaths {
		if path == uri {
			return
		}
	}

	ctx.Status(403)

}

/** 设置共享数据 **/
func setShareData(ctx *macaron.Context, sess session.Store) {
	ctx.Data["URI"] = ctx.Req.URL.Path
	urlPath := strings.TrimPrefix(ctx.Req.URL.Path, "/")
	paths := strings.Split(urlPath, "/")
	ctx.Data["Controller"] = ""
	ctx.Data["Action"] = ""
	if len(paths) > 0 {
		ctx.Data["Controller"] = paths[0]
	}
	if len(paths) > 1 {
		ctx.Data["Action"] = paths[1]
	}
	ctx.Data["LoginUsername"] = user.Username(sess)
	ctx.Data["LoginUid"] = user.Uid(sess)
	ctx.Data["IsAdmin"] = user.IsAdmin(sess)
	ctx.Data["AppName"] = app.Setting.AppName
}

/** API接口签名验证 **/
func apiAuth(ctx *macaron.Context) {
	if !app.Setting.ApiSignEnable {
		return
	}
	apiKey := strings.TrimSpace(app.Setting.ApiKey)
	apiSecret := strings.TrimSpace(app.Setting.ApiSecret)
	json := utils.JsonResponse{}
	if apiKey == "" || apiSecret == "" {
		msg := json.CommonFailure("使用API前, 请先配置密钥")
		ctx.Write([]byte(msg))
		return
	}
	currentTimestamp := time.Now().Unix()
	time := ctx.QueryInt64("time")
	if time <= 0 {
		msg := json.CommonFailure("参数time不能为空")
		ctx.Write([]byte(msg))
		return
	}
	if time < (currentTimestamp - 1800) {
		msg := json.CommonFailure("time无效")
		ctx.Write([]byte(msg))
		return
	}
	sign := ctx.QueryTrim("sign")
	if sign == "" {
		msg := json.CommonFailure("参数sign不能为空")
		ctx.Write([]byte(msg))
		return
	}
	raw := apiKey + strconv.FormatInt(time, 10) + strings.TrimSpace(ctx.Req.URL.Path) + apiSecret
	realSign := utils.Md5(raw)
	if sign != realSign {
		msg := json.CommonFailure("签名验证失败")
		ctx.Write([]byte(msg))
		return
	}
}

// endregion

func isAjaxRequest(ctx *macaron.Context) bool {
	req := ctx.Req.Header.Get("X-Requested-With")
	if req == "XMLHttpRequest" {
		return true
	}

	return false
}

func isGetRequest(ctx *macaron.Context) bool {
	return ctx.Req.Method == "GET"
}
<|endoftext|>


<commit_before>package repo

import (
	"log"
	"os/exec"
	"strconv"
	"strings"
)

type GitRepository struct{}

func parseBranchListLine(line string) Alias {
	line = strings.Trim(line, "* ")
	splitLine := strings.Split(line, " ")
	masterName := splitLine[0]
	for _, lineComponent := range splitLine[1:] {
		if len(lineComponent) == 40 {
			revisionHash := lineComponent
			return Alias{masterName, Revision(revisionHash)}
		}
	}
	return Alias{Branch: masterName}
}

func (gitRepository GitRepository) ListBranches() []Alias {
	out, err := exec.Command("git", "branch", "-av", "--list", "--abbrev=40", "--no-color").Output()
	if err != nil {
		log.Fatal(err)
	}
	lines := strings.Split(strings.Trim(string(out), " \n"), "\n")
	aliases := make([]Alias, len(lines))
	index := 0
	for _, line := range lines {
		if line != "" {
			aliases[index] = parseBranchListLine(line)
			index += 1
		}
	}
	return aliases
}

func (gitRepository GitRepository) ReadRevisionContents(revision Revision) *RevisionContents {
	out, err := exec.Command("git", "ls-tree", "-r", string(revision)).Output()
	if err != nil {
		log.Fatal(err)
	}
	listOutput := strings.Trim(string(out), "\n ")
	lines := strings.Split(listOutput, "\n")
	paths := make([]string, len(lines))
	for index, line := range lines {
		line = strings.Replace(lines[index], "\t", " ", -1)
		lineParts := strings.Split(line, " ")
		paths[index] = lineParts[len(lineParts)-1]
	}
	return &RevisionContents{revision, paths}
}

func (gitRepository GitRepository) getSubject(revision Revision) string {
	out, err := exec.Command("git", "show", string(revision), "--format=\"format:%s\"", "-s").Output()
	if err != nil {
		log.Fatal(err)
	}
	return string(out)
}

func (gitRepository GitRepository) getAuthorName(revision Revision) string {
	out, err := exec.Command("git", "show", string(revision), "--format=\"format:%an\"", "-s").Output()
	if err != nil {
		log.Fatal(err)
	}
	return string(out)
}

func (gitRepository GitRepository) getAuthorEmail(revision Revision) string {
	out, err := exec.Command("git", "show", string(revision), "--format=\"format:%ae\"", "-s").Output()
	if err != nil {
		log.Fatal(err)
	}
	return string(out)
}

func (gitRepository GitRepository) getTimestamp(revision Revision) int64 {
	out, err := exec.Command("git", "show", string(revision), "--format=\"format:%ae\"", "-s").Output()
	if err != nil {
		log.Fatal(err)
	}
	timestamp, err := strconv.ParseInt(string(out), 10, 64)
	if err != nil {
		log.Fatal(err)
	}
	return timestamp
}

func (gitRepository GitRepository) ReadRevisionMetadata(revision Revision) RevisionMetadata {
	return RevisionMetadata{
		Revision:    revision,
		Timestamp:   gitRepository.getTimestamp(revision),
		Subject:     gitRepository.getSubject(revision),
		AuthorName:  gitRepository.getAuthorName(revision),
		AuthorEmail: gitRepository.getAuthorEmail(revision),
	}
}

func (gitRepository GitRepository) ReadFileAtRevision(revision Revision, path string) []Line {
	out, err := exec.Command("git", "blame", "-s", "--abbrev=40", string(revision), path).Output()
	if err != nil {
		log.Fatal(err)
	}
	blameOutput := strings.Trim(string(out), "\n ")
	lines := strings.Split(blameOutput, "\n")
	result := make([]Line, len(lines))
	for _, line := range lines {
		revision := Revision(line[0:40])
		lineNumberIndex := strings.Index(line, ")")
		lineNumber, err := strconv.Atoi(strings.Trim(line[41:lineNumberIndex], " "))
		if err != nil {
			log.Fatal(err)
		}
		contents := line[lineNumberIndex+2:]
		result = append(result, Line{revision, lineNumber, contents})
	}
	return result
}
<commit_msg>Split out the code for calling git into a helper method<commit_after>package repo

import (
	"log"
	"os/exec"
	"strconv"
	"strings"
)

type GitRepository struct{}

func runGitCommandOrDie(cmd *exec.Cmd) string {
	out, err := cmd.Output()
	if err != nil {
		log.Print(cmd.Args)
		log.Print(out)
		log.Fatal(err)
	}
	return strings.Trim(string(out), " \n")
}

func parseBranchListLine(line string) Alias {
	line = strings.Trim(line, "* ")
	splitLine := strings.Split(line, " ")
	masterName := splitLine[0]
	for _, lineComponent := range splitLine[1:] {
		if len(lineComponent) == 40 {
			revisionHash := lineComponent
			return Alias{masterName, Revision(revisionHash)}
		}
	}
	return Alias{Branch: masterName}
}

func (gitRepository GitRepository) ListBranches() []Alias {
	out := runGitCommandOrDie(
		exec.Command("git", "branch", "-av", "--list", "--abbrev=40", "--no-color"))
	lines := strings.Split(out, "\n")
	aliases := make([]Alias, len(lines))
	index := 0
	for _, line := range lines {
		if line != "" {
			aliases[index] = parseBranchListLine(line)
			index += 1
		}
	}
	return aliases
}

func (gitRepository GitRepository) ReadRevisionContents(revision Revision) *RevisionContents {
	out := runGitCommandOrDie(exec.Command("git", "ls-tree", "-r", string(revision)))
	lines := strings.Split(out, "\n")
	paths := make([]string, len(lines))
	for index, line := range lines {
		line = strings.Replace(lines[index], "\t", " ", -1)
		lineParts := strings.Split(line, " ")
		paths[index] = lineParts[len(lineParts)-1]
	}
	return &RevisionContents{revision, paths}
}

func (gitRepository GitRepository) getSubject(revision Revision) string {
	return runGitCommandOrDie(exec.Command(
		"git", "show", string(revision), "--format=\"format:%s\"", "-s"))
}

func (gitRepository GitRepository) getAuthorName(revision Revision) string {
	return runGitCommandOrDie(exec.Command(
		"git", "show", string(revision), "--format=\"format:%an\"", "-s"))
}

func (gitRepository GitRepository) getAuthorEmail(revision Revision) string {
	return runGitCommandOrDie(exec.Command(
		"git", "show", string(revision), "--format=\"format:%ae\"", "-s"))
}

func (gitRepository GitRepository) getTimestamp(revision Revision) int64 {
	out := runGitCommandOrDie(exec.Command(
		"git", "show", string(revision), "--format=\"format:%t\"", "-s"))
	timestamp, err := strconv.ParseInt(out, 10, 64)
	if err != nil {
		log.Fatal(err)
	}
	return timestamp
}

func (gitRepository GitRepository) ReadRevisionMetadata(revision Revision) RevisionMetadata {
	return RevisionMetadata{
		Revision:    revision,
		Timestamp:   gitRepository.getTimestamp(revision),
		Subject:     gitRepository.getSubject(revision),
		AuthorName:  gitRepository.getAuthorName(revision),
		AuthorEmail: gitRepository.getAuthorEmail(revision),
	}
}

func (gitRepository GitRepository) ReadFileAtRevision(revision Revision, path string) []Line {
	out := runGitCommandOrDie(
		exec.Command("git", "blame", "-s", "--abbrev=40", string(revision), path))
	lines := strings.Split(out, "\n")
	result := make([]Line, len(lines))
	for _, line := range lines {
		revision := Revision(line[0:40])
		lineNumberIndex := strings.Index(line, ")")
		lineNumber, err := strconv.Atoi(strings.Trim(line[41:lineNumberIndex], " "))
		if err != nil {
			log.Fatal(err)
		}
		contents := line[lineNumberIndex+2:]
		result = append(result, Line{revision, lineNumber, contents})
	}
	return result
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net"
	"os"
	"sync"
	"time"

	"github.com/cyberdelia/go-metrics-graphite"
	"github.com/rcrowley/go-metrics"
)

type Stats struct {
	registry metrics.Registry
	lock     sync.Mutex
}

func (t *Stats) Gauge(name string, value int) {
	metrics.GetOrRegisterGauge(name, t.registry).Update(int64(value))
}

func (t *Stats) Inc(stat string) {
	metrics.GetOrRegisterCounter(stat, t.registry).Inc(1)
}

func (t *Stats) GetCount(stat string) int64 {
	c := t.registry.Get(stat)
	if c == nil {
		return 0
	}
	return c.(metrics.Counter).Count()
}

func (t *Stats) Timing(stat string, d time.Duration) {
	metrics.GetOrRegisterTimer(stat, t.registry).Update(d)
}

func NewStats(sendToConsole bool, sendToGraphite, graphitePrefix string) *Stats {
	s := new(Stats)
	s.registry = metrics.NewRegistry()

	if sendToGraphite != "" {
		log.Println("Stats reporting to graphite: ", sendToGraphite)
		addr, _ := net.ResolveTCPAddr("tcp", sendToGraphite)

		go s.graphiteSender(time.Second*15, addr, graphitePrefix)
	}

	if sendToConsole {
		log.Println("Stats reporting enabled...")
		go metrics.Log(s.registry, time.Minute, log.New(os.Stderr, "metrics: ", log.Lmicroseconds))
	}
	return s
}

func (s *Stats) graphiteSender(freq time.Duration, addr *net.TCPAddr, prefix string) {
	cfg := graphite.GraphiteConfig{
		Addr:          addr,
		Registry:      s.registry,
		FlushInterval: 0,
		DurationUnit:  time.Millisecond,
		Prefix:        prefix,
		Percentiles:   []float64{0.5, 0.75, 0.95, 0.99, 0.999},
	}

	for _ = range time.Tick(freq) {
		graphite.GraphiteOnce(cfg)
		s.registry.UnregisterAll() // Hack to get ostrich-like flush to 0.
	}
}
<commit_msg>use my metrics stuff<commit_after>package main

import (
	"log"
	"net"
	"os"
	"sync"
	"time"

	"github.com/dt/go-metrics-graphite"
	"github.com/rcrowley/go-metrics"
)

type Stats struct {
	registry metrics.Registry
	lock     sync.Mutex
}

func (t *Stats) Gauge(name string, value int) {
	metrics.GetOrRegisterGauge(name, t.registry).Update(int64(value))
}

func (t *Stats) Inc(stat string) {
	metrics.GetOrRegisterCounter(stat+"-total", t.registry).Inc(1)
	metrics.GetOrRegisterMeter(stat, t.registry).Mark(1)
}

func (t *Stats) GetCount(stat string) int64 {
	c := t.registry.Get(stat)
	if c == nil {
		return 0
	}
	return c.(metrics.Counter).Count()
}

func (t *Stats) Timing(stat string, d time.Duration) {
	metrics.GetOrRegisterTimer(stat, t.registry).Update(d)
}

func NewStats(sendToConsole bool, sendToGraphite, graphitePrefix string) *Stats {
	s := new(Stats)
	s.registry = metrics.NewRegistry()

	if sendToGraphite != "" {
		log.Println("Stats reporting to graphite: ", sendToGraphite)
		addr, _ := net.ResolveTCPAddr("tcp", sendToGraphite)

		cfg := graphite.GraphiteConfig{
			Addr:          addr,
			Registry:      s.registry,
			FlushInterval: 15 * time.Second,
			DurationUnit:  time.Millisecond,
			Prefix:        graphitePrefix,
			Percentiles:   []float64{0.5, 0.75, 0.9, 0.95, 0.99, 0.999},
		}

		go graphite.GraphiteWithConfig(cfg)
	}

	if sendToConsole {
		log.Println("Stats reporting enabled...")
		go metrics.Log(s.registry, time.Minute, log.New(os.Stderr, "metrics: ", log.Lmicroseconds))
	}
	return s
}
<|endoftext|>


<commit_before>package pole

import (
	"testing"
	"os"
	"github.com/yaricom/goNEAT/neat"
	"fmt"
	"github.com/yaricom/goNEAT/neat/genetics"
	"github.com/yaricom/goNEAT/experiments"
	"math/rand"
)

// Run double pole-balancing experiment with Markov environment setup
func TestCartDoublePoleGenerationEvaluator_GenerationEvaluateMarkov(t *testing.T) {
	// to make sure we have predictable results
	rand.Seed(423)

	out_dir_path, context_path, genome_path := "../../out/pole2_markov_test", "../../data/pole2_markov.neat", "../../data/pole2_markov_startgenes"

	// Load context configuration
	configFile, err := os.Open(context_path)
	if err != nil {
		t.Error("Failed to load context", err)
		return
	}
	context := neat.LoadContext(configFile)
	neat.LogLevel = neat.LogLevelInfo

	// Load Genome
	fmt.Println("Loading start genome for POLE2 Markov experiment")
	genomeFile, err := os.Open(genome_path)
	if err != nil {
		t.Error("Failed to open genome file")
		return
	}
	start_genome, err := genetics.ReadGenome(genomeFile, 1)
	if err != nil {
		t.Error("Failed to read start genome")
		return
	}

	// Check if output dir exists
	if _, err := os.Stat(out_dir_path); err == nil {
		// clear it
		os.RemoveAll(out_dir_path)
	}
	// create output dir
	err = os.MkdirAll(out_dir_path, os.ModePerm)
	if err != nil {
		t.Errorf("Failed to create output directory, reason: %s", err)
		return
	}

	// The 10 runs POLE2 Markov experiment
	context.NumRuns = 10
	experiment := experiments.Experiment{
		Id:0,
		Trials:make(experiments.Trials, context.NumRuns),
	}
	err = experiment.Execute(context, start_genome, CartDoublePoleGenerationEvaluator{
		OutputPath:out_dir_path,
		Markov:true,
		ActionType:experiments.ContinuousAction,
	})
	if err != nil {
		t.Error("Failed to perform POLE2 Markov experiment:", err)
		return
	}

	// Find winner statistics
	avg_nodes, avg_genes, avg_evals := experiment.AvgWinnerNGE()

	// check results
	if avg_nodes < 8 {
		t.Error("avg_nodes < 8", avg_nodes)
	} else if avg_nodes > 40 {
		t.Error("avg_nodes > 40", avg_nodes)
	}

	if avg_genes < 7 {
		t.Error("avg_genes < 7", avg_genes)
	} else if avg_genes > 50 {
		t.Error("avg_genes > 50", avg_genes)
	}

	max_evals := float64(context.PopSize * context.NumGenerations)
	if avg_evals > max_evals {
		t.Error("avg_evals > max_evals", avg_evals, max_evals)
	}

	t.Logf("Average nodes: %.1f, genes: %.1f, evals: %.1f\n", avg_nodes, avg_genes, avg_evals)
	mean_complexity, mean_diversity, mean_age := 0.0, 0.0, 0.0
	for _, t := range experiment.Trials {
		mean_complexity += t.Complexity().Mean()
		mean_diversity += t.Diversity().Mean()
		mean_age += t.Age().Mean()
	}
	count := float64(len(experiment.Trials))
	mean_complexity /= count
	mean_diversity /= count
	mean_age /= count
	t.Logf("Mean: complexity=%.1f, diversity=%.1f, age=%.1f\n", mean_complexity, mean_diversity, mean_age)

	solved_trials := 0
	for _, tr := range experiment.Trials {
		if tr.Solved() {
			solved_trials++
		}
	}

	t.Logf("Trials solved/run: %d/%d", solved_trials, len(experiment.Trials))
}

// Run double pole-balancing experiment with Non-Markov environment setup
func TestCartDoublePoleGenerationEvaluator_GenerationEvaluateNonMarkov(t *testing.T) {
	// to make sure we have predictable results
	rand.Seed(423)

	out_dir_path, context_path, genome_path := "../../out/pole2_non-markov_test", "../../data/pole2_non-markov.neat", "../../data/pole2_non-markov_startgenes"

	// Load context configuration
	configFile, err := os.Open(context_path)
	if err != nil {
		t.Error("Failed to load context", err)
		return
	}
	context := neat.LoadContext(configFile)
	neat.LogLevel = neat.LogLevelInfo

	// Load Genome
	fmt.Println("Loading start genome for POLE2 Non-Markov experiment")
	genomeFile, err := os.Open(genome_path)
	if err != nil {
		t.Error("Failed to open genome file")
		return
	}
	start_genome, err := genetics.ReadGenome(genomeFile, 1)
	if err != nil {
		t.Error("Failed to read start genome")
		return
	}

	// Check if output dir exists
	if _, err := os.Stat(out_dir_path); err == nil {
		// clear it
		os.RemoveAll(out_dir_path)
	}
	// create output dir
	err = os.MkdirAll(out_dir_path, os.ModePerm)
	if err != nil {
		t.Errorf("Failed to create output directory, reason: %s", err)
		return
	}

	// The 10 runs POLE2 Non-Markov experiment
	context.NumRuns = 10
	experiment := experiments.Experiment{
		Id:0,
		Trials:make(experiments.Trials, context.NumRuns),
	}
	err = experiment.Execute(context, start_genome, CartDoublePoleGenerationEvaluator{
		OutputPath:out_dir_path,
		Markov:false,
		ActionType:experiments.ContinuousAction,
	})
	if err != nil {
		t.Error("Failed to perform POLE2 Non-Markov experiment:", err)
		return
	}

	// Find winner statistics
	avg_nodes, avg_genes, avg_evals := experiment.AvgWinnerNGE()

	// check results
	if avg_nodes < 5 {
		t.Error("avg_nodes < 5", avg_nodes)
	} else if avg_nodes > 40 {
		t.Error("avg_nodes > 40", avg_nodes)
	}

	if avg_genes < 5 {
		t.Error("avg_genes < 5", avg_genes)
	} else if avg_genes > 50 {
		t.Error("avg_genes > 50", avg_genes)
	}

	max_evals := float64(context.PopSize * context.NumGenerations)
	if avg_evals > max_evals {
		t.Error("avg_evals > max_evals", avg_evals, max_evals)
	}

	t.Logf("Average nodes: %.1f, genes: %.1f, evals: %.1f\n", avg_nodes, avg_genes, avg_evals)
	mean_complexity, mean_diversity, mean_age := 0.0, 0.0, 0.0
	for _, t := range experiment.Trials {
		mean_complexity += t.Complexity().Mean()
		mean_diversity += t.Diversity().Mean()
		mean_age += t.Age().Mean()
	}
	count := float64(len(experiment.Trials))
	mean_complexity /= count
	mean_diversity /= count
	mean_age /= count
	t.Logf("Mean: complexity=%.1f, diversity=%.1f, age=%.1f\n", mean_complexity, mean_diversity, mean_age)

	solved_trials := 0
	for _, tr := range experiment.Trials {
		if tr.Solved() {
			solved_trials++
		}
	}
	t.Logf("Trials solved/run: %d/%d\n", solved_trials, len(experiment.Trials))

	best_g_score := 0.0
	for _, tr := range experiment.Trials {
		if tr.Solved() {
			best_org_score := tr.Best().Fitness
			if best_org_score > best_g_score {
				best_g_score = best_org_score
			}
		}
	}
	t.Logf("Best Generalization Score: %.0f\n", best_g_score)
}
<commit_msg>Reduced number of trials for double pole-balancing test to fit into Travis CI timeout<commit_after>package pole

import (
	"testing"
	"os"
	"github.com/yaricom/goNEAT/neat"
	"fmt"
	"github.com/yaricom/goNEAT/neat/genetics"
	"github.com/yaricom/goNEAT/experiments"
	"math/rand"
)

// Run double pole-balancing experiment with Markov environment setup
func TestCartDoublePoleGenerationEvaluator_GenerationEvaluateMarkov(t *testing.T) {
	// to make sure we have predictable results
	rand.Seed(423)

	out_dir_path, context_path, genome_path := "../../out/pole2_markov_test", "../../data/pole2_markov.neat", "../../data/pole2_markov_startgenes"

	// Load context configuration
	configFile, err := os.Open(context_path)
	if err != nil {
		t.Error("Failed to load context", err)
		return
	}
	context := neat.LoadContext(configFile)
	neat.LogLevel = neat.LogLevelInfo

	// Load Genome
	fmt.Println("Loading start genome for POLE2 Markov experiment")
	genomeFile, err := os.Open(genome_path)
	if err != nil {
		t.Error("Failed to open genome file")
		return
	}
	start_genome, err := genetics.ReadGenome(genomeFile, 1)
	if err != nil {
		t.Error("Failed to read start genome")
		return
	}

	// Check if output dir exists
	if _, err := os.Stat(out_dir_path); err == nil {
		// clear it
		os.RemoveAll(out_dir_path)
	}
	// create output dir
	err = os.MkdirAll(out_dir_path, os.ModePerm)
	if err != nil {
		t.Errorf("Failed to create output directory, reason: %s", err)
		return
	}

	// The 10 runs POLE2 Markov experiment
	context.NumRuns = 5
	experiment := experiments.Experiment{
		Id:0,
		Trials:make(experiments.Trials, context.NumRuns),
	}
	err = experiment.Execute(context, start_genome, CartDoublePoleGenerationEvaluator{
		OutputPath:out_dir_path,
		Markov:true,
		ActionType:experiments.ContinuousAction,
	})
	if err != nil {
		t.Error("Failed to perform POLE2 Markov experiment:", err)
		return
	}

	// Find winner statistics
	avg_nodes, avg_genes, avg_evals := experiment.AvgWinnerNGE()

	// check results
	if avg_nodes < 8 {
		t.Error("avg_nodes < 8", avg_nodes)
	} else if avg_nodes > 40 {
		t.Error("avg_nodes > 40", avg_nodes)
	}

	if avg_genes < 7 {
		t.Error("avg_genes < 7", avg_genes)
	} else if avg_genes > 50 {
		t.Error("avg_genes > 50", avg_genes)
	}

	max_evals := float64(context.PopSize * context.NumGenerations)
	if avg_evals > max_evals {
		t.Error("avg_evals > max_evals", avg_evals, max_evals)
	}

	t.Logf("Average nodes: %.1f, genes: %.1f, evals: %.1f\n", avg_nodes, avg_genes, avg_evals)
	mean_complexity, mean_diversity, mean_age := 0.0, 0.0, 0.0
	for _, t := range experiment.Trials {
		mean_complexity += t.Complexity().Mean()
		mean_diversity += t.Diversity().Mean()
		mean_age += t.Age().Mean()
	}
	count := float64(len(experiment.Trials))
	mean_complexity /= count
	mean_diversity /= count
	mean_age /= count
	t.Logf("Mean: complexity=%.1f, diversity=%.1f, age=%.1f\n", mean_complexity, mean_diversity, mean_age)

	solved_trials := 0
	for _, tr := range experiment.Trials {
		if tr.Solved() {
			solved_trials++
		}
	}

	t.Logf("Trials solved/run: %d/%d", solved_trials, len(experiment.Trials))
}

// Run double pole-balancing experiment with Non-Markov environment setup
func TestCartDoublePoleGenerationEvaluator_GenerationEvaluateNonMarkov(t *testing.T) {
	// to make sure we have predictable results
	rand.Seed(423)

	out_dir_path, context_path, genome_path := "../../out/pole2_non-markov_test", "../../data/pole2_non-markov.neat", "../../data/pole2_non-markov_startgenes"

	// Load context configuration
	configFile, err := os.Open(context_path)
	if err != nil {
		t.Error("Failed to load context", err)
		return
	}
	context := neat.LoadContext(configFile)
	neat.LogLevel = neat.LogLevelInfo

	// Load Genome
	fmt.Println("Loading start genome for POLE2 Non-Markov experiment")
	genomeFile, err := os.Open(genome_path)
	if err != nil {
		t.Error("Failed to open genome file")
		return
	}
	start_genome, err := genetics.ReadGenome(genomeFile, 1)
	if err != nil {
		t.Error("Failed to read start genome")
		return
	}

	// Check if output dir exists
	if _, err := os.Stat(out_dir_path); err == nil {
		// clear it
		os.RemoveAll(out_dir_path)
	}
	// create output dir
	err = os.MkdirAll(out_dir_path, os.ModePerm)
	if err != nil {
		t.Errorf("Failed to create output directory, reason: %s", err)
		return
	}

	// The 10 runs POLE2 Non-Markov experiment
	context.NumRuns = 5
	experiment := experiments.Experiment{
		Id:0,
		Trials:make(experiments.Trials, context.NumRuns),
	}
	err = experiment.Execute(context, start_genome, CartDoublePoleGenerationEvaluator{
		OutputPath:out_dir_path,
		Markov:false,
		ActionType:experiments.ContinuousAction,
	})
	if err != nil {
		t.Error("Failed to perform POLE2 Non-Markov experiment:", err)
		return
	}

	// Find winner statistics
	avg_nodes, avg_genes, avg_evals := experiment.AvgWinnerNGE()

	// check results
	if avg_nodes < 5 {
		t.Error("avg_nodes < 5", avg_nodes)
	} else if avg_nodes > 40 {
		t.Error("avg_nodes > 40", avg_nodes)
	}

	if avg_genes < 5 {
		t.Error("avg_genes < 5", avg_genes)
	} else if avg_genes > 50 {
		t.Error("avg_genes > 50", avg_genes)
	}

	max_evals := float64(context.PopSize * context.NumGenerations)
	if avg_evals > max_evals {
		t.Error("avg_evals > max_evals", avg_evals, max_evals)
	}

	t.Logf("Average nodes: %.1f, genes: %.1f, evals: %.1f\n", avg_nodes, avg_genes, avg_evals)
	mean_complexity, mean_diversity, mean_age := 0.0, 0.0, 0.0
	for _, t := range experiment.Trials {
		mean_complexity += t.Complexity().Mean()
		mean_diversity += t.Diversity().Mean()
		mean_age += t.Age().Mean()
	}
	count := float64(len(experiment.Trials))
	mean_complexity /= count
	mean_diversity /= count
	mean_age /= count
	t.Logf("Mean: complexity=%.1f, diversity=%.1f, age=%.1f\n", mean_complexity, mean_diversity, mean_age)

	solved_trials := 0
	for _, tr := range experiment.Trials {
		if tr.Solved() {
			solved_trials++
		}
	}
	t.Logf("Trials solved/run: %d/%d\n", solved_trials, len(experiment.Trials))

	best_g_score := 0.0
	for _, tr := range experiment.Trials {
		if tr.Solved() {
			best_org_score := tr.Best().Fitness
			if best_org_score > best_g_score {
				best_g_score = best_org_score
			}
		}
	}
	t.Logf("Best Generalization Score: %.0f\n", best_g_score)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Add k8s style header for api-reference html docs; And fix the alignment.

package main

import (
	"flag"
	"io/ioutil"
	"os"
	"strings"

	"golang.org/x/net/html"
)

var (
	templateHTML = flag.String("templateHTML", "http://kubernetes.io/v1.0/index.html", "URL of the html file that has the k8s header")
)

//strip the <!DOCTYPE html> and  tag
func stripHTML(filename string) error {
	targetHTML, err := os.OpenFile(filename, os.O_RDWR, 0666)
	if err != nil {
		return err
	}
	defer targetHTML.Close()
	target, err := html.Parse(targetHTML)
	if err != nil {
		return err
	}
	//node for
	headNode := target.FirstChild.NextSibling.FirstChild

	err = targetHTML.Truncate(0)
	if err != nil {
		return err
	}
	_, err = targetHTML.Seek(0, 0)
	if err != nil {
		return err
	}
	for node := headNode; node != nil; node = node.NextSibling {
		err = html.Render(targetHTML, node)
		if err != nil {
			return err
		}
	}
	return nil
}

func nitFix(filename string) error {
	file, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}
	//TODO: This really should be fixed when generating the original html files
	newContents := strings.Replace(string(file), "
Paths
", "
Operations
", 1)
	//Safari cannot parse *zoom and reloads the page forever
	newContents = strings.Replace(newContents, "*zoom: 1", "zoom: 1", -1)

	if err = ioutil.WriteFile(filename, []byte(newContents), 0666); err != nil {
		return err
	}
	return nil
}

//move the definitions.html and operations.html to _includes
func moveHTML(filePath string, fileName string, outputDir string) error {
	return os.Rename(filePath, outputDir+"/../_includes/"+fileName)
}

func processHTML(filePath string, fileName string, outputDir string) error {
	if err := stripHTML(filePath); err != nil {
		return err
	}
	if err := nitFix(filePath); err != nil {
		return err
	}
	if err := moveHTML(filePath, fileName, outputDir); err != nil {
		return err
	}
	return nil
}
<commit_msg>replace the HEAD links with v1.0 ones in definitions.html<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Add k8s style header for api-reference html docs; And fix the alignment.

package main

import (
	"flag"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"strings"

	"golang.org/x/net/html"
)

var (
	templateHTML = flag.String("templateHTML", "http://kubernetes.io/v1.0/index.html", "URL of the html file that has the k8s header")
)

//strip the <!DOCTYPE html> and  tag
func stripHTML(filename string) error {
	targetHTML, err := os.OpenFile(filename, os.O_RDWR, 0666)
	if err != nil {
		return err
	}
	defer targetHTML.Close()
	target, err := html.Parse(targetHTML)
	if err != nil {
		return err
	}
	//node for
	headNode := target.FirstChild.NextSibling.FirstChild

	err = targetHTML.Truncate(0)
	if err != nil {
		return err
	}
	_, err = targetHTML.Seek(0, 0)
	if err != nil {
		return err
	}
	for node := headNode; node != nil; node = node.NextSibling {
		err = html.Render(targetHTML, node)
		if err != nil {
			return err
		}
	}
	return nil
}

// replace http://releases.k8s.io/HEAD/docs/xxx.md with
// http://kubernetes.io/v1.0/docs/xxx.html
func fixLinks(fileName, outputDir string) error {
	mdRE := regexp.MustCompile(`http://releases.k8s.io/HEAD/docs(.*?\.)md`)
	repl := []byte("http://kubernetes.io/" + path.Base(outputDir) + "/docs" + "${1}" + "html")
	file, err := ioutil.ReadFile(fileName)
	if err != nil {
		return err
	}
	file = mdRE.ReplaceAll(file, repl)

	if err = ioutil.WriteFile(fileName, file, 0666); err != nil {
		return err
	}
	return nil
}

func nitFix(filename string) error {
	file, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}
	//TODO: This really should be fixed when generating the original html files
	newContents := strings.Replace(string(file), "
Paths
", "
Operations
", 1)
	//Safari cannot parse *zoom and reloads the page forever
	newContents = strings.Replace(newContents, "*zoom: 1", "zoom: 1", -1)

	if err = ioutil.WriteFile(filename, []byte(newContents), 0666); err != nil {
		return err
	}
	return nil
}

//move the definitions.html and operations.html to _includes
func moveHTML(filePath string, fileName string, outputDir string) error {
	return os.Rename(filePath, outputDir+"/../_includes/"+fileName)
}

func processHTML(filePath string, fileName string, outputDir string) error {
	if err := stripHTML(filePath); err != nil {
		return err
	}
	if err := nitFix(filePath); err != nil {
		return err
	}
	if err := fixLinks(filePath, outputDir); err != nil {
		return err
	}
	if err := moveHTML(filePath, fileName, outputDir); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha1"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"time"

	"github.com/dickeyxxx/golock"
	"github.com/franela/goreq"
)

var updateTopic = &Topic{
	Name:        "update",
	Description: "update heroku-cli",
}

var updateCmd = &Command{
	Topic:       "update",
	Hidden:      true,
	Description: "updates heroku-cli",
	Args:        []Arg{{Name: "channel", Optional: true}},
	Run: func(ctx *Context) {
		channel := ctx.Args.(map[string]string)["channel"]
		if channel == "" {
			channel = Channel
		}
		Update(channel)
	},
}

var binPath = filepath.Join(AppDir, "heroku-cli")
var updateLockPath = filepath.Join(AppDir, "updating.lock")
var autoupdateFile = filepath.Join(AppDir, "autoupdate")

func init() {
	if runtime.GOOS == "windows" {
		binPath = binPath + ".exe"
	}
}

// Update updates the CLI and plugins
func Update(channel string) {
	golock.Lock(updateLockPath)
	defer golock.Unlock(updateLockPath)
	done := make(chan bool)
	go func() {
		updateCLI(channel)
		updatePlugins()
		touchAutoupdateFile()
		done <- true
	}()
	select {
	case <-time.After(time.Second * 120):
		Errln("Timed out while updating")
	case <-done:
	}
}

func updatePlugins() {
	Err("updating plugins... ")
	b, _ := node.UpdatePackages()
	Errln("done")
	if len(b) > 0 {
		Err("rebuilding plugins cache... ")
		ClearPluginCache()
		WritePluginCache(GetPlugins())
		Errln("done")
	}
}

func updateCLI(channel string) {
	manifest := getUpdateManifest(channel)
	if manifest.Version == Version && manifest.Channel == Channel {
		return
	}
	if !updatable() {
		Errf("Out of date: You are running %s but %s is out\n", Version, manifest.Version)
		return
	}
	Errf("updating v4 CLI to %s (%s)... ", manifest.Version, manifest.Channel)
	build := manifest.Builds[runtime.GOOS][runtime.GOARCH]
	// on windows we can't remove an existing file or remove the running binary
	// so we download the file to binName.new
	// move the running binary to binName.old (deleting any existing file first)
	// rename the downloaded file to binName
	if err := downloadBin(binPath+".new", build.URL); err != nil {
		panic(err)
	}
	if fileSha1(binPath+".new") != build.Sha1 {
		panic("SHA mismatch")
	}
	os.Remove(binPath + ".old")
	os.Rename(binPath, binPath+".old")
	if err := os.Rename(binPath+".new", binPath); err != nil {
		panic(err)
	}
	os.Remove(binPath + ".old")
	Errln("done")
}

// IsUpdateNeeded checks if an update is available
func IsUpdateNeeded(t string) bool {
	f, err := os.Stat(autoupdateFile)
	if err != nil {
		return true
	}
	if t == "soft" {
		return time.Since(f.ModTime()) > 4*time.Hour
	}
	return time.Since(f.ModTime()) > 168*time.Hour
}

func touchAutoupdateFile() {
	out, err := os.OpenFile(autoupdateFile, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		panic(err)
	}
	out.WriteString(time.Now().String())
}

type manifest struct {
	Channel, Version string
	Builds           map[string]map[string]struct {
		URL, Sha1 string
	}
}

func getUpdateManifest(channel string) manifest {
	res, err := goreq.Request{
		Uri: "https://d1gvo455cekpjp.cloudfront.net/" + channel + "/manifest.json",
	}.Do()
	ExitIfError(err)
	var m manifest
	res.Body.FromJsonTo(&m)
	return m
}

func updatable() bool {
	path, err := filepath.Abs(os.Args[0])
	if err != nil {
		Errln(err)
	}
	return path == binPath
}

func downloadBin(path, url string) error {
	out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
	if err != nil {
		return err
	}
	res, err := goreq.Request{
		Uri: url + ".gz",
	}.Do()
	if err != nil {
		return err
	}
	defer res.Body.Close()
	_, err = io.Copy(out, res.Body)
	if err != nil {
		return err
	}
	return out.Close()
}

func fileSha1(path string) string {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("%x", sha1.Sum(data))
}

// TriggerBackgroundUpdate will trigger an update to the client in the background
func TriggerBackgroundUpdate() {
	if err := exec.Command(binPath, "update").Start(); err != nil {
		panic(err)
	}
}

// WarnIfUpdating prints to stderr if the CLI is updating
func WarnIfUpdating() {
	if exists, _ := fileExists(updateLockPath); exists {
		// Disable since v3 calls CLI twice
		//Errln("WARNING: CLI is updating")
	}
}
<commit_msg>do not panic if it cannot background update<commit_after>package main

import (
	"crypto/sha1"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"time"

	"github.com/dickeyxxx/golock"
	"github.com/franela/goreq"
)

var updateTopic = &Topic{
	Name:        "update",
	Description: "update heroku-cli",
}

var updateCmd = &Command{
	Topic:       "update",
	Hidden:      true,
	Description: "updates heroku-cli",
	Args:        []Arg{{Name: "channel", Optional: true}},
	Run: func(ctx *Context) {
		channel := ctx.Args.(map[string]string)["channel"]
		if channel == "" {
			channel = Channel
		}
		Update(channel)
	},
}

var binPath = filepath.Join(AppDir, "heroku-cli")
var updateLockPath = filepath.Join(AppDir, "updating.lock")
var autoupdateFile = filepath.Join(AppDir, "autoupdate")

func init() {
	if runtime.GOOS == "windows" {
		binPath = binPath + ".exe"
	}
}

// Update updates the CLI and plugins
func Update(channel string) {
	golock.Lock(updateLockPath)
	defer golock.Unlock(updateLockPath)
	done := make(chan bool)
	go func() {
		updateCLI(channel)
		updatePlugins()
		touchAutoupdateFile()
		done <- true
	}()
	select {
	case <-time.After(time.Second * 120):
		Errln("Timed out while updating")
	case <-done:
	}
}

func updatePlugins() {
	Err("updating plugins... ")
	b, _ := node.UpdatePackages()
	Errln("done")
	if len(b) > 0 {
		Err("rebuilding plugins cache... ")
		ClearPluginCache()
		WritePluginCache(GetPlugins())
		Errln("done")
	}
}

func updateCLI(channel string) {
	manifest := getUpdateManifest(channel)
	if manifest.Version == Version && manifest.Channel == Channel {
		return
	}
	if !updatable() {
		Errf("Out of date: You are running %s but %s is out\n", Version, manifest.Version)
		return
	}
	Errf("updating v4 CLI to %s (%s)... ", manifest.Version, manifest.Channel)
	build := manifest.Builds[runtime.GOOS][runtime.GOARCH]
	// on windows we can't remove an existing file or remove the running binary
	// so we download the file to binName.new
	// move the running binary to binName.old (deleting any existing file first)
	// rename the downloaded file to binName
	if err := downloadBin(binPath+".new", build.URL); err != nil {
		panic(err)
	}
	if fileSha1(binPath+".new") != build.Sha1 {
		panic("SHA mismatch")
	}
	os.Remove(binPath + ".old")
	os.Rename(binPath, binPath+".old")
	if err := os.Rename(binPath+".new", binPath); err != nil {
		panic(err)
	}
	os.Remove(binPath + ".old")
	Errln("done")
}

// IsUpdateNeeded checks if an update is available
func IsUpdateNeeded(t string) bool {
	f, err := os.Stat(autoupdateFile)
	if err != nil {
		return true
	}
	if t == "soft" {
		return time.Since(f.ModTime()) > 4*time.Hour
	}
	return time.Since(f.ModTime()) > 168*time.Hour
}

func touchAutoupdateFile() {
	out, err := os.OpenFile(autoupdateFile, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		panic(err)
	}
	out.WriteString(time.Now().String())
}

type manifest struct {
	Channel, Version string
	Builds           map[string]map[string]struct {
		URL, Sha1 string
	}
}

func getUpdateManifest(channel string) manifest {
	res, err := goreq.Request{
		Uri: "https://d1gvo455cekpjp.cloudfront.net/" + channel + "/manifest.json",
	}.Do()
	ExitIfError(err)
	var m manifest
	res.Body.FromJsonTo(&m)
	return m
}

func updatable() bool {
	path, err := filepath.Abs(os.Args[0])
	if err != nil {
		Errln(err)
	}
	return path == binPath
}

func downloadBin(path, url string) error {
	out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
	if err != nil {
		return err
	}
	res, err := goreq.Request{
		Uri: url + ".gz",
	}.Do()
	if err != nil {
		return err
	}
	defer res.Body.Close()
	_, err = io.Copy(out, res.Body)
	if err != nil {
		return err
	}
	return out.Close()
}

func fileSha1(path string) string {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("%x", sha1.Sum(data))
}

// TriggerBackgroundUpdate will trigger an update to the client in the background
func TriggerBackgroundUpdate() {
	exec.Command(binPath, "update").Start()
}

// WarnIfUpdating prints to stderr if the CLI is updating
func WarnIfUpdating() {
	if exists, _ := fileExists(updateLockPath); exists {
		// Disable since v3 calls CLI twice
		//Errln("WARNING: CLI is updating")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha1"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/dickeyxxx/golock"
	"github.com/franela/goreq"
	"github.com/heroku/heroku-cli/gode"
	"github.com/kardianos/osext"
	"github.com/ulikunitz/xz"
)

var updateTopic = &Topic{
	Name:        "update",
	Description: "update heroku-cli",
}

var updateCmd = &Command{
	Topic:            "update",
	Hidden:           true,
	Description:      "updates heroku-cli",
	DisableAnalytics: true,
	Args:             []Arg{{Name: "channel", Optional: true}},
	Flags:            []Flag{{Name: "background", Hidden: true}},
	Run: func(ctx *Context) {
		channel := ctx.Args.(map[string]string)["channel"]
		if channel == "" {
			channel = Channel
		}
		t := "foreground"
		if ctx.Flags["background"] == true {
			t = "background"
		}
		Update(channel, t)
	},
}

var binPath string
var updateLockPath = filepath.Join(AppDir(), "updating.lock")
var autoupdateFile = filepath.Join(AppDir(), "autoupdate")
var tmpPath = filepath.Join(AppDir(), "tmp")

func init() {
	binPath, _ = osext.Executable()
}

// Update updates the CLI and plugins
func Update(channel string, t string) {
	if !IsUpdateNeeded(t) {
		return
	}
	touchAutoupdateFile()
	updateCLI(channel)
	SetupNode()
	SetupBuiltinPlugins()
	updatePlugins()
	truncateErrorLog()
	cleanTmpDir()
}

func updatePlugins() {
	action("heroku-cli: Updating plugins", "", func() {
		plugins := PluginNamesNotSymlinked()
		if len(plugins) == 0 {
			return
		}
		packages, err := gode.OutdatedPackages(plugins...)
		PrintError(err)
		if len(packages) > 0 {
			for name, version := range packages {
				lockPlugin(name)
				PrintError(gode.InstallPackages(name + "@" + version))
				plugin, err := ParsePlugin(name)
				PrintError(err)
				AddPluginsToCache(plugin)
				unlockPlugin(name)
			}
			Errf(" done. Updated %d %s.\n", len(packages), plural("package", len(packages)))
		} else {
			Errln(" no plugins to update.")
		}
	})
}

func updateCLI(channel string) {
	if channel == "?" {
		// do not update dev version
		return
	}
	manifest, err := getUpdateManifest(channel)
	if err != nil {
		Warn("Error updating CLI")
		PrintError(err)
		return
	}
	if manifest.Version == Version && manifest.Channel == Channel {
		return
	}
	locked, err := golock.IsLocked(updateLockPath)
	LogIfError(err)
	if locked {
		Warn("Update in progress")
		return
	}
	LogIfError(golock.Lock(updateLockPath))
	unlock := func() {
		golock.Unlock(updateLockPath)
	}
	defer unlock()
	msg := fmt.Sprintf("heroku-cli: Updating to %s", manifest.Version)
	if manifest.Channel != "master" {
		msg = fmt.Sprintf("%s (%s)", msg, manifest.Channel)
	}
	action(msg, "done", func() {
		build := manifest.Builds[runtime.GOOS][runtime.GOARCH]
		// on windows we can't remove an existing file or remove the running binary
		// so we download the file to binName.new
		// move the running binary to binName.old (deleting any existing file first)
		// rename the downloaded file to binName
		tmpBinPathNew := binPath + ".new"
		tmpBinPathOld := binPath + ".old"
		if err := downloadBin(tmpBinPathNew, build.URL); err != nil {
			panic(err)
		}
		if fileSha1(tmpBinPathNew) != build.Sha1 {
			panic("SHA mismatch")
		}
		os.Remove(tmpBinPathOld)
		os.Rename(binPath, tmpBinPathOld)
		if err := os.Rename(tmpBinPathNew, binPath); err != nil {
			panic(err)
		}
		os.Remove(tmpBinPathOld)
		unlock()
		clearAutoupdateFile() // force full update
	})
	reexec() // reexec to finish updating with new code
}

// IsUpdateNeeded checks if an update is available
func IsUpdateNeeded(t string) bool {
	f, err := os.Stat(autoupdateFile)
	if err != nil {
		return true
	}
	if t == "background" {
		return time.Since(f.ModTime()) > 4*time.Hour
	} else if t == "block" {
		return time.Since(f.ModTime()) > 2160*time.Hour // 90 days
	}
	return true
}

func touchAutoupdateFile() {
	out, err := os.OpenFile(autoupdateFile, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		panic(err)
	}
	out.WriteString(time.Now().String())
	out.Close()
}

// forces a full update on the next run
func clearAutoupdateFile() {
	PrintError(os.Remove(autoupdateFile))
}

type manifest struct {
	Channel, Version string
	Builds           map[string]map[string]struct {
		URL, Sha1 string
	}
}

func getUpdateManifest(channel string) (*manifest, error) {
	res, err := goreq.Request{
		Uri:       "https://cli-assets.heroku.com/" + channel + "/manifest.json",
		ShowDebug: debugging,
	}.Do()
	if err != nil {
		return nil, err
	}
	var m manifest
	res.Body.FromJsonTo(&m)
	return &m, nil
}

func downloadBin(path, url string) error {
	out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
	if err != nil {
		return err
	}
	defer out.Close()
	res, err := goreq.Request{
		Uri:       url + ".xz",
		ShowDebug: debugging,
	}.Do()
	if err != nil {
		return err
	}
	if res.StatusCode != 200 {
		b, _ := res.Body.ToString()
		return errors.New(b)
	}
	defer res.Body.Close()
	uncompressed, err := xz.NewReader(res.Body)
	if err != nil {
		return err
	}
	_, err = io.Copy(out, uncompressed)
	return err
}

func fileSha1(path string) string {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("%x", sha1.Sum(data))
}

// TriggerBackgroundUpdate will trigger an update to the client in the background
func TriggerBackgroundUpdate() {
	wg.Add(1)
	go func() {
		wg.Done()
		if IsUpdateNeeded("background") {
			exec.Command(binPath, "update", "--background").Start()
		}
	}()
}

// restarts the CLI with the same arguments
func reexec() {
	Debugln("reexecing new CLI...")
	cmd := exec.Command(binPath, os.Args[1:]...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	Exit(getExitCode(cmd.Run()))
}

func truncateErrorLog() {
	Debugln("truncating error log...")
	body, err := ioutil.ReadFile(ErrLogPath)
	if err != nil {
		PrintError(err)
		return
	}
	lines := strings.Split(string(body), "\n")
	lines = lines[maxint(len(lines)-1000, 0) : len(lines)-1]
	err = ioutil.WriteFile(ErrLogPath, []byte(strings.Join(lines, "\n")+"\n"), 0644)
	PrintError(err)
}

func maxint(a, b int) int {
	if a > b {
		return a
	}
	return b
}

func cleanTmpDir() {
	Debugln("cleaning up tmp dirs...")
	dirs, err := ioutil.ReadDir(tmpPath)
	if err != nil {
		PrintError(err)
		return
	}
	for _, dir := range dirs {
		if time.Since(dir.ModTime()) > 24*time.Hour {
			path := filepath.Join(tmpPath, dir.Name())
			Debugln("deleting " + path)
			PrintError(os.RemoveAll(path))
		}
	}
}
<commit_msg>hide unneeded truncation error<commit_after>package main

import (
	"crypto/sha1"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/dickeyxxx/golock"
	"github.com/franela/goreq"
	"github.com/heroku/heroku-cli/gode"
	"github.com/kardianos/osext"
	"github.com/ulikunitz/xz"
)

var updateTopic = &Topic{
	Name:        "update",
	Description: "update heroku-cli",
}

var updateCmd = &Command{
	Topic:            "update",
	Hidden:           true,
	Description:      "updates heroku-cli",
	DisableAnalytics: true,
	Args:             []Arg{{Name: "channel", Optional: true}},
	Flags:            []Flag{{Name: "background", Hidden: true}},
	Run: func(ctx *Context) {
		channel := ctx.Args.(map[string]string)["channel"]
		if channel == "" {
			channel = Channel
		}
		t := "foreground"
		if ctx.Flags["background"] == true {
			t = "background"
		}
		Update(channel, t)
	},
}

var binPath string
var updateLockPath = filepath.Join(AppDir(), "updating.lock")
var autoupdateFile = filepath.Join(AppDir(), "autoupdate")
var tmpPath = filepath.Join(AppDir(), "tmp")

func init() {
	binPath, _ = osext.Executable()
}

// Update updates the CLI and plugins
func Update(channel string, t string) {
	if !IsUpdateNeeded(t) {
		return
	}
	touchAutoupdateFile()
	updateCLI(channel)
	SetupNode()
	SetupBuiltinPlugins()
	updatePlugins()
	truncateErrorLog()
	cleanTmpDir()
}

func updatePlugins() {
	action("heroku-cli: Updating plugins", "", func() {
		plugins := PluginNamesNotSymlinked()
		if len(plugins) == 0 {
			return
		}
		packages, err := gode.OutdatedPackages(plugins...)
		PrintError(err)
		if len(packages) > 0 {
			for name, version := range packages {
				lockPlugin(name)
				PrintError(gode.InstallPackages(name + "@" + version))
				plugin, err := ParsePlugin(name)
				PrintError(err)
				AddPluginsToCache(plugin)
				unlockPlugin(name)
			}
			Errf(" done. Updated %d %s.\n", len(packages), plural("package", len(packages)))
		} else {
			Errln(" no plugins to update.")
		}
	})
}

func updateCLI(channel string) {
	if channel == "?" {
		// do not update dev version
		return
	}
	manifest, err := getUpdateManifest(channel)
	if err != nil {
		Warn("Error updating CLI")
		PrintError(err)
		return
	}
	if manifest.Version == Version && manifest.Channel == Channel {
		return
	}
	locked, err := golock.IsLocked(updateLockPath)
	LogIfError(err)
	if locked {
		Warn("Update in progress")
		return
	}
	LogIfError(golock.Lock(updateLockPath))
	unlock := func() {
		golock.Unlock(updateLockPath)
	}
	defer unlock()
	msg := fmt.Sprintf("heroku-cli: Updating to %s", manifest.Version)
	if manifest.Channel != "master" {
		msg = fmt.Sprintf("%s (%s)", msg, manifest.Channel)
	}
	action(msg, "done", func() {
		build := manifest.Builds[runtime.GOOS][runtime.GOARCH]
		// on windows we can't remove an existing file or remove the running binary
		// so we download the file to binName.new
		// move the running binary to binName.old (deleting any existing file first)
		// rename the downloaded file to binName
		tmpBinPathNew := binPath + ".new"
		tmpBinPathOld := binPath + ".old"
		if err := downloadBin(tmpBinPathNew, build.URL); err != nil {
			panic(err)
		}
		if fileSha1(tmpBinPathNew) != build.Sha1 {
			panic("SHA mismatch")
		}
		os.Remove(tmpBinPathOld)
		os.Rename(binPath, tmpBinPathOld)
		if err := os.Rename(tmpBinPathNew, binPath); err != nil {
			panic(err)
		}
		os.Remove(tmpBinPathOld)
		unlock()
		clearAutoupdateFile() // force full update
	})
	reexec() // reexec to finish updating with new code
}

// IsUpdateNeeded checks if an update is available
func IsUpdateNeeded(t string) bool {
	f, err := os.Stat(autoupdateFile)
	if err != nil {
		return true
	}
	if t == "background" {
		return time.Since(f.ModTime()) > 4*time.Hour
	} else if t == "block" {
		return time.Since(f.ModTime()) > 2160*time.Hour // 90 days
	}
	return true
}

func touchAutoupdateFile() {
	out, err := os.OpenFile(autoupdateFile, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		panic(err)
	}
	out.WriteString(time.Now().String())
	out.Close()
}

// forces a full update on the next run
func clearAutoupdateFile() {
	PrintError(os.Remove(autoupdateFile))
}

type manifest struct {
	Channel, Version string
	Builds           map[string]map[string]struct {
		URL, Sha1 string
	}
}

func getUpdateManifest(channel string) (*manifest, error) {
	res, err := goreq.Request{
		Uri:       "https://cli-assets.heroku.com/" + channel + "/manifest.json",
		ShowDebug: debugging,
	}.Do()
	if err != nil {
		return nil, err
	}
	var m manifest
	res.Body.FromJsonTo(&m)
	return &m, nil
}

func downloadBin(path, url string) error {
	out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
	if err != nil {
		return err
	}
	defer out.Close()
	res, err := goreq.Request{
		Uri:       url + ".xz",
		ShowDebug: debugging,
	}.Do()
	if err != nil {
		return err
	}
	if res.StatusCode != 200 {
		b, _ := res.Body.ToString()
		return errors.New(b)
	}
	defer res.Body.Close()
	uncompressed, err := xz.NewReader(res.Body)
	if err != nil {
		return err
	}
	_, err = io.Copy(out, uncompressed)
	return err
}

func fileSha1(path string) string {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("%x", sha1.Sum(data))
}

// TriggerBackgroundUpdate will trigger an update to the client in the background
func TriggerBackgroundUpdate() {
	wg.Add(1)
	go func() {
		wg.Done()
		if IsUpdateNeeded("background") {
			exec.Command(binPath, "update", "--background").Start()
		}
	}()
}

// restarts the CLI with the same arguments
func reexec() {
	Debugln("reexecing new CLI...")
	cmd := exec.Command(binPath, os.Args[1:]...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	Exit(getExitCode(cmd.Run()))
}

func truncateErrorLog() {
	Debugln("truncating error log...")
	body, err := ioutil.ReadFile(ErrLogPath)
	if err != nil {
		if !os.IsNotExist(err) {
			PrintError(err)
		}
		return
	}
	lines := strings.Split(string(body), "\n")
	lines = lines[maxint(len(lines)-1000, 0) : len(lines)-1]
	err = ioutil.WriteFile(ErrLogPath, []byte(strings.Join(lines, "\n")+"\n"), 0644)
	PrintError(err)
}

func maxint(a, b int) int {
	if a > b {
		return a
	}
	return b
}

func cleanTmpDir() {
	Debugln("cleaning up tmp dirs...")
	dirs, err := ioutil.ReadDir(tmpPath)
	if err != nil {
		PrintError(err)
		return
	}
	for _, dir := range dirs {
		if time.Since(dir.ModTime()) > 24*time.Hour {
			path := filepath.Join(tmpPath, dir.Name())
			Debugln("deleting " + path)
			PrintError(os.RemoveAll(path))
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"github.com/robmerrell/gosecret/vendor/github.com/kr/s3"
	"github.com/robmerrell/gosecret/vendor/github.com/kr/s3/s3util"
	"io"
	"net/http"
	"os"
)

// flags and args
var uploadBucketNameFlag string
var uploadAccessKeyFlag string
var uploadSecretKeyFlag string
var uploadFilenameArg string

var uploadDoc = `
Usage: upload [options] file

Upload a file to an s3 bucket
`

func uploadAction() error {
	// make sure that we have all of the required data
	if uploadFilenameArg == "" {
		return errors.New("Please provide a valid filename to upload")
	}
	if uploadBucketNameFlag == "" {
		return errors.New("Please provide an S3 bucket name with --bucket or $GOSECRET_BUCKET")
	}
	if uploadAccessKeyFlag == "" {
		return errors.New("Please provide an AWS access key with --access-key or $GOSECRET_ACCESS_KEY")
	}
	if uploadSecretKeyFlag == "" {
		return errors.New("Please provide an AWS secrety key with --secret-key or $GOSECRET_SECRET_KEY")
	}

	// create the config needed for the uploader
	config := &s3util.Config{
		Keys: &s3.Keys{
			AccessKey: uploadAccessKeyFlag,
			SecretKey: uploadSecretKeyFlag,
		},
		Service: s3.DefaultService,
	}

	return upload(uploadBucketNameFlag, uploadFilenameArg, config)
}

// uploadFlagInit initializes the flagset for the upload command
func uploadFlagInit(fs *flag.FlagSet) {
	defaultBucket := os.Getenv("GOSECRET_BUCKET")
	fs.StringVar(&uploadBucketNameFlag, "bucket", defaultBucket, "S3 bucket to upload into. Defaults to value in $GOSECRET_BUCKET")

	defaultAccessKey := os.Getenv("GOSECRET_ACCESS_KEY")
	fs.StringVar(&uploadAccessKeyFlag, "access-key", defaultAccessKey, "S3 Access Key. Defaults to value in $GOSECRET_ACCESS_KEY")

	defaultSecretKey := os.Getenv("GOSECRET_SECRET_KEY")
	fs.StringVar(&uploadSecretKeyFlag, "secret-key", defaultSecretKey, "S3 Secret Key. Defaults to value in $GOSECRET_SECRET_KEY")
}

// uploadFlagPostParse sets the uploadable filename from the arguments provided by the flagset
func uploadFlagPostParse(fs *flag.FlagSet) {
	// make sure the input file is reachable
	if filename := fs.Arg(0); filename != "" {
		if fi, err := os.Stat(filename); err == nil && !fi.IsDir() {
			uploadFilenameArg = filename
		}
	}
}

// upload uploads a file to an s3 bucket.
func upload(bucket, file string, config *s3util.Config) error {
	// open the local file to upload
	localFile, err := os.Open(file)
	if err != nil {
		return err
	}
	defer localFile.Close()

	headers := http.Header{}
	headers.Add("x-amz-acl", "private")
	s3File, err := s3util.Create(generateS3Url(bucket, file), headers, config)
	if err != nil {
		return err
	}
	defer s3File.Close()

	// copy the file
	_, err = io.Copy(s3File, localFile)
	return err
	// if _, err = io.Copy(s3File, localFile); err != nil {
	// 	return err
	// }

	// close the files
	// if err = localFile.Close(); err != nil {
	// 	return err
	// }
	// return s3File.Close()
}
<commit_msg>Removed unecessary code<commit_after>package main

import (
	"errors"
	"flag"
	"github.com/robmerrell/gosecret/vendor/github.com/kr/s3"
	"github.com/robmerrell/gosecret/vendor/github.com/kr/s3/s3util"
	"io"
	"net/http"
	"os"
)

// flags and args
var uploadBucketNameFlag string
var uploadAccessKeyFlag string
var uploadSecretKeyFlag string
var uploadFilenameArg string

var uploadDoc = `
Usage: upload [options] file

Upload a file to an s3 bucket
`

func uploadAction() error {
	// make sure that we have all of the required data
	if uploadFilenameArg == "" {
		return errors.New("Please provide a valid filename to upload")
	}
	if uploadBucketNameFlag == "" {
		return errors.New("Please provide an S3 bucket name with --bucket or $GOSECRET_BUCKET")
	}
	if uploadAccessKeyFlag == "" {
		return errors.New("Please provide an AWS access key with --access-key or $GOSECRET_ACCESS_KEY")
	}
	if uploadSecretKeyFlag == "" {
		return errors.New("Please provide an AWS secrety key with --secret-key or $GOSECRET_SECRET_KEY")
	}

	// create the config needed for the uploader
	config := &s3util.Config{
		Keys: &s3.Keys{
			AccessKey: uploadAccessKeyFlag,
			SecretKey: uploadSecretKeyFlag,
		},
		Service: s3.DefaultService,
	}

	return upload(uploadBucketNameFlag, uploadFilenameArg, config)
}

// uploadFlagInit initializes the flagset for the upload command
func uploadFlagInit(fs *flag.FlagSet) {
	defaultBucket := os.Getenv("GOSECRET_BUCKET")
	fs.StringVar(&uploadBucketNameFlag, "bucket", defaultBucket, "S3 bucket to upload into. Defaults to value in $GOSECRET_BUCKET")

	defaultAccessKey := os.Getenv("GOSECRET_ACCESS_KEY")
	fs.StringVar(&uploadAccessKeyFlag, "access-key", defaultAccessKey, "S3 Access Key. Defaults to value in $GOSECRET_ACCESS_KEY")

	defaultSecretKey := os.Getenv("GOSECRET_SECRET_KEY")
	fs.StringVar(&uploadSecretKeyFlag, "secret-key", defaultSecretKey, "S3 Secret Key. Defaults to value in $GOSECRET_SECRET_KEY")
}

// uploadFlagPostParse sets the uploadable filename from the arguments provided by the flagset
func uploadFlagPostParse(fs *flag.FlagSet) {
	// make sure the input file is reachable
	if filename := fs.Arg(0); filename != "" {
		if fi, err := os.Stat(filename); err == nil && !fi.IsDir() {
			uploadFilenameArg = filename
		}
	}
}

// upload uploads a file to an s3 bucket.
func upload(bucket, file string, config *s3util.Config) error {
	// open the local file to upload
	localFile, err := os.Open(file)
	if err != nil {
		return err
	}
	defer localFile.Close()

	headers := http.Header{}
	headers.Add("x-amz-acl", "private")
	s3File, err := s3util.Create(generateS3Url(bucket, file), headers, config)
	if err != nil {
		return err
	}
	defer s3File.Close()

	// copy the file
	_, err = io.Copy(s3File, localFile)
	return err
}
<|endoftext|>