<commit_before>// Copyright 2018 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package vmtest

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"testing"

	"github.com/u-root/u-root/pkg/cp"
	"github.com/u-root/u-root/pkg/golang"
	"github.com/u-root/u-root/pkg/qemu"
	"github.com/u-root/u-root/pkg/uio"
	"github.com/u-root/u-root/pkg/ulog"
	"github.com/u-root/u-root/pkg/ulog/ulogtest"
	"github.com/u-root/u-root/pkg/uroot"
	"github.com/u-root/u-root/pkg/uroot/initramfs"
)

// Options are integration test options.
type Options struct {
	// BuildOpts are u-root initramfs options.
	//
	// They are used if the test needs to generate an initramfs.
	// Fields that are not set are populated by QEMU and QEMUTest as
	// possible.
	BuildOpts uroot.Opts

	// QEMUOpts are QEMU VM options for the test.
	//
	// Fields that are not set are populated by QEMU and QEMUTest as
	// possible.
	QEMUOpts qemu.Options

	// Name is the test's name.
	//
	// If name is left empty, the calling function's function name will be
	// used as determined by runtime.Caller.
	Name string

	// Uinit is the uinit that should be added to a generated initramfs.
	//
	// If none is specified, the generic uinit will be used, which searches for
	// and runs the script generated from TestCmds.
	Uinit string

	// TestCmds are commands to execute after init.
	//
	// QEMUTest generates an Elvish script with these commands. The script is
	// shared with the VM, and is run from the generic uinit.
	TestCmds []string

	// TmpDir is the temporary directory exposed to the QEMU VM.
	TmpDir string

	// Logger logs build statements.
	Logger ulog.Logger

	// Extra environment variables to set when building (used by u-bmc)
	ExtraBuildEnv []string

	// Use virtual vfat rather than 9pfs
	UseVVFAT bool
}

func last(s string) string {
	l := strings.Split(s, ".")
	return l[len(l)-1]
}

func callerName(depth int) string {
	// Use the test name as the serial log's file name.
	pc, _, _, ok := runtime.Caller(depth)
	if !ok {
		panic("runtime caller failed")
	}
	f := runtime.FuncForPC(pc)
	return last(f.Name())
}

// TestLineWriter is an io.Writer that logs full lines of serial to tb.
func TestLineWriter(tb testing.TB, prefix string) io.WriteCloser {
	return uio.FullLineWriter(&testLineWriter{tb: tb, prefix: prefix})
}

type jsonStripper struct {
	uio.LineWriter
}

func (j jsonStripper) OneLine(p []byte) {
	// Poor man's JSON detector.
	if len(p) == 0 || p[0] == '{' {
		return
	}
	j.LineWriter.OneLine(p)
}

func JSONLessTestLineWriter(tb testing.TB, prefix string) io.WriteCloser {
	return uio.FullLineWriter(jsonStripper{&testLineWriter{tb: tb, prefix: prefix}})
}

// testLineWriter is an io.Writer that logs full lines of serial to tb.
type testLineWriter struct {
	tb     testing.TB
	prefix string
}

func (tsw *testLineWriter) OneLine(p []byte) {
	tsw.tb.Logf("%s: %s", tsw.prefix, strings.ReplaceAll(string(p), "\033", "~"))
}

// TestArch returns the architecture under test. Pass this as GOARCH when
// building Go programs to be run in the QEMU environment.
func TestArch() string {
	if env := os.Getenv("UROOT_TESTARCH"); env != "" {
		return env
	}
	return "amd64"
}

// SkipWithoutQEMU skips the test when the QEMU environment variables are not
// set. This is already called by QEMUTest(), so use if some expensive
// operations are performed before calling QEMUTest().
func SkipWithoutQEMU(t *testing.T) {
	if _, ok := os.LookupEnv("UROOT_QEMU"); !ok {
		t.Skip("QEMU test is skipped unless UROOT_QEMU is set")
	}
	if _, ok := os.LookupEnv("UROOT_KERNEL"); !ok {
		t.Skip("QEMU test is skipped unless UROOT_KERNEL is set")
	}
}

func QEMUTest(t *testing.T, o *Options) (*qemu.VM, func()) {
	SkipWithoutQEMU(t)

	if len(o.Name) == 0 {
		o.Name = callerName(2)
	}
	if o.Logger == nil {
		o.Logger = &ulogtest.Logger{TB: t}
	}
	if o.QEMUOpts.SerialOutput == nil {
		o.QEMUOpts.SerialOutput = TestLineWriter(t, "serial")
	}

	// Create or reuse a temporary directory. This is exposed to the VM.
	if o.TmpDir == "" {
		tmpDir, err := ioutil.TempDir("", "uroot-integration")
		if err != nil {
			t.Fatalf("Failed to create temp dir: %v", err)
		}
		o.TmpDir = tmpDir
	}

	qOpts, err := QEMU(o)
	if err != nil {
		t.Fatalf("Failed to create QEMU VM %s: %v", o.Name, err)
	}

	vm, err := qOpts.Start()
	if err != nil {
		t.Fatalf("Failed to start QEMU VM %s: %v", o.Name, err)
	}

	return vm, func() {
		vm.Close()
		t.Logf("QEMU command line to reproduce %s:\n%s", o.Name, vm.CmdlineQuoted())
		if t.Failed() {
			t.Log("Keeping temp dir: ", o.TmpDir)
		} else if len(o.TmpDir) == 0 {
			if err := os.RemoveAll(o.TmpDir); err != nil {
				t.Logf("failed to remove temporary directory %s: %v", o.TmpDir, err)
			}
		}
	}
}

// QEMU builds the u-root environment and prepares QEMU options given the test
// options and environment variables.
//
// QEMU will augment o.BuildOpts and o.QEMUOpts with configuration that the
// caller either requested (through the Options.Uinit field, for example) or
// that the caller did not set.
//
// QEMU returns the QEMU launch options or an error.
func QEMU(o *Options) (*qemu.Options, error) {
	if len(o.Name) == 0 {
		o.Name = callerName(2)
	}

	// Generate Elvish shell script of test commands in o.TmpDir.
	if len(o.TestCmds) > 0 {
		testFile := filepath.Join(o.TmpDir, "test.elv")

		if err := ioutil.WriteFile(
			testFile, []byte(strings.Join(o.TestCmds, "\n")), 0777); err != nil {
			return nil, err
		}
	}

	// Set the initramfs.
	if len(o.QEMUOpts.Initramfs) == 0 {
		o.QEMUOpts.Initramfs = filepath.Join(o.TmpDir, "initramfs.cpio")
		if err := ChooseTestInitramfs(o.BuildOpts, o.Uinit, o.QEMUOpts.Initramfs); err != nil {
			return nil, err
		}
	}

	if len(o.QEMUOpts.Kernel) == 0 {
		// Copy kernel to o.TmpDir for tests involving kexec.
		kernel := filepath.Join(o.TmpDir, "kernel")
		if err := cp.Copy(os.Getenv("UROOT_KERNEL"), kernel); err != nil {
			return nil, err
		}
		o.QEMUOpts.Kernel = kernel
	}

	switch TestArch() {
	case "amd64":
		o.QEMUOpts.KernelArgs += " console=ttyS0 earlyprintk=ttyS0"
	case "arm":
		o.QEMUOpts.KernelArgs += " console=ttyAMA0"
	}

	var dir qemu.Device
	if o.UseVVFAT {
		dir = qemu.ReadOnlyDirectory{Dir: o.TmpDir}
	} else {
		dir = qemu.P9Directory{Dir: o.TmpDir, Arch: TestArch()}
	}
	o.QEMUOpts.Devices = append(o.QEMUOpts.Devices, qemu.VirtioRandom{}, dir)

	return &o.QEMUOpts, nil
}

// ChooseTestInitramfs chooses which initramfs will be used for a given test and
// places it at the location given by outputFile.
// Default to the override initramfs if one is specified in the UROOT_INITRAMFS
// environment variable. Else, build an initramfs with the given parameters.
// If no uinit was provided, the generic one is used.
func ChooseTestInitramfs(o uroot.Opts, uinit, outputFile string) error {
	override := os.Getenv("UROOT_INITRAMFS")
	if len(override) > 0 {
		log.Printf("Overriding with initramfs %q", override)
		return cp.Copy(override, outputFile)
	}

	if len(uinit) == 0 {
		log.Printf("Defaulting to generic initramfs")
		uinit = "github.com/u-root/u-root/integration/testcmd/generic/uinit"
	}

	_, err := CreateTestInitramfs(o, uinit, outputFile)
	return err
}

// CreateTestInitramfs creates an initramfs with the given build options and
// uinit, and writes it to the given output file. If no output file is provided,
// one will be created.
// The output file name is returned. It is the caller's responsibility to remove
// the initramfs file after use.
func CreateTestInitramfs(o uroot.Opts, uinit, outputFile string) (string, error) {
	// TODO make uroot.Opts Env a pointer to allow user-supplied Env.
	env := golang.Default()
	env.CgoEnabled = false
	env.GOARCH = TestArch()
	o.Env = env

	logger := log.New(os.Stderr, "", 0)

	// If build opts don't specify any commands, include all commands. Else,
	// always add init and elvish.
	var cmds []string
	if len(o.Commands) == 0 {
		cmds = []string{
			"github.com/u-root/u-root/cmds/core/*",
			"github.com/u-root/u-root/cmds/exp/*",
		}
	}

	if len(uinit) != 0 {
		cmds = append(cmds, uinit)
	}

	// Add our commands to the build opts.
	o.AddBusyBoxCommands(cmds...)

	// Fill in the default build options if not specified.
	if o.BaseArchive == nil {
		o.BaseArchive = uroot.DefaultRamfs.Reader()
	}
	if len(o.InitCmd) == 0 {
		o.InitCmd = "init"
	}
	if len(o.DefaultShell) == 0 {
		o.DefaultShell = "elvish"
	}
	if len(o.TempDir) == 0 {
		tempDir, err := ioutil.TempDir("", "initramfs-tempdir")
		if err != nil {
			return "", fmt.Errorf("Failed to create temp dir: %v", err)
		}
		defer os.RemoveAll(tempDir)
		o.TempDir = tempDir
	}

	// Create an output file if one was not provided.
	if len(outputFile) == 0 {
		f, err := ioutil.TempFile("", "initramfs.cpio")
		if err != nil {
			return "", fmt.Errorf("failed to create output file: %v", err)
		}
		outputFile = f.Name()
	}
	w, err := initramfs.CPIO.OpenWriter(logger, outputFile, "", "")
	if err != nil {
		return "", fmt.Errorf("Failed to create initramfs writer: %v", err)
	}
	o.OutputFile = w

	return outputFile, uroot.CreateInitramfs(logger, o)
}
<commit_msg>Add back DontSetEnv<commit_after>// Copyright 2018 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package vmtest

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"testing"

	"github.com/u-root/u-root/pkg/cp"
	"github.com/u-root/u-root/pkg/golang"
	"github.com/u-root/u-root/pkg/qemu"
	"github.com/u-root/u-root/pkg/uio"
	"github.com/u-root/u-root/pkg/ulog"
	"github.com/u-root/u-root/pkg/ulog/ulogtest"
	"github.com/u-root/u-root/pkg/uroot"
	"github.com/u-root/u-root/pkg/uroot/initramfs"
)

// Options are integration test options.
type Options struct {
	// BuildOpts are u-root initramfs options.
	//
	// They are used if the test needs to generate an initramfs.
	// Fields that are not set are populated by QEMU and QEMUTest as
	// possible.
	BuildOpts uroot.Opts

	// QEMUOpts are QEMU VM options for the test.
	//
	// Fields that are not set are populated by QEMU and QEMUTest as
	// possible.
	QEMUOpts qemu.Options

	// DontSetEnv doesn't set the BuildOpts.Env and uses the user-supplied one.
	//
	// TODO: make uroot.Opts.Env a pointer?
	DontSetEnv bool

	// Name is the test's name.
	//
	// If name is left empty, the calling function's function name will be
	// used as determined by runtime.Caller.
	Name string

	// Uinit is the uinit that should be added to a generated initramfs.
	//
	// If none is specified, the generic uinit will be used, which searches for
	// and runs the script generated from TestCmds.
	Uinit string

	// TestCmds are commands to execute after init.
	//
	// QEMUTest generates an Elvish script with these commands. The script is
	// shared with the VM, and is run from the generic uinit.
	TestCmds []string

	// TmpDir is the temporary directory exposed to the QEMU VM.
	TmpDir string

	// Logger logs build statements.
	Logger ulog.Logger

	// Extra environment variables to set when building (used by u-bmc)
	ExtraBuildEnv []string

	// Use virtual vfat rather than 9pfs
	UseVVFAT bool
}

func last(s string) string {
	l := strings.Split(s, ".")
	return l[len(l)-1]
}

func callerName(depth int) string {
	// Use the test name as the serial log's file name.
	pc, _, _, ok := runtime.Caller(depth)
	if !ok {
		panic("runtime caller failed")
	}
	f := runtime.FuncForPC(pc)
	return last(f.Name())
}

// TestLineWriter is an io.Writer that logs full lines of serial to tb.
func TestLineWriter(tb testing.TB, prefix string) io.WriteCloser {
	return uio.FullLineWriter(&testLineWriter{tb: tb, prefix: prefix})
}

type jsonStripper struct {
	uio.LineWriter
}

func (j jsonStripper) OneLine(p []byte) {
	// Poor man's JSON detector.
	if len(p) == 0 || p[0] == '{' {
		return
	}
	j.LineWriter.OneLine(p)
}

func JSONLessTestLineWriter(tb testing.TB, prefix string) io.WriteCloser {
	return uio.FullLineWriter(jsonStripper{&testLineWriter{tb: tb, prefix: prefix}})
}

// testLineWriter is an io.Writer that logs full lines of serial to tb.
type testLineWriter struct {
	tb     testing.TB
	prefix string
}

func (tsw *testLineWriter) OneLine(p []byte) {
	tsw.tb.Logf("%s: %s", tsw.prefix, strings.ReplaceAll(string(p), "\033", "~"))
}

// TestArch returns the architecture under test. Pass this as GOARCH when
// building Go programs to be run in the QEMU environment.
func TestArch() string {
	if env := os.Getenv("UROOT_TESTARCH"); env != "" {
		return env
	}
	return "amd64"
}

// SkipWithoutQEMU skips the test when the QEMU environment variables are not
// set. This is already called by QEMUTest(), so use if some expensive
// operations are performed before calling QEMUTest().
func SkipWithoutQEMU(t *testing.T) {
	if _, ok := os.LookupEnv("UROOT_QEMU"); !ok {
		t.Skip("QEMU test is skipped unless UROOT_QEMU is set")
	}
	if _, ok := os.LookupEnv("UROOT_KERNEL"); !ok {
		t.Skip("QEMU test is skipped unless UROOT_KERNEL is set")
	}
}

func QEMUTest(t *testing.T, o *Options) (*qemu.VM, func()) {
	SkipWithoutQEMU(t)

	if len(o.Name) == 0 {
		o.Name = callerName(2)
	}
	if o.Logger == nil {
		o.Logger = &ulogtest.Logger{TB: t}
	}
	if o.QEMUOpts.SerialOutput == nil {
		o.QEMUOpts.SerialOutput = TestLineWriter(t, "serial")
	}

	// Create or reuse a temporary directory. This is exposed to the VM.
	if o.TmpDir == "" {
		tmpDir, err := ioutil.TempDir("", "uroot-integration")
		if err != nil {
			t.Fatalf("Failed to create temp dir: %v", err)
		}
		o.TmpDir = tmpDir
	}

	qOpts, err := QEMU(o)
	if err != nil {
		t.Fatalf("Failed to create QEMU VM %s: %v", o.Name, err)
	}

	vm, err := qOpts.Start()
	if err != nil {
		t.Fatalf("Failed to start QEMU VM %s: %v", o.Name, err)
	}

	return vm, func() {
		vm.Close()
		t.Logf("QEMU command line to reproduce %s:\n%s", o.Name, vm.CmdlineQuoted())
		if t.Failed() {
			t.Log("Keeping temp dir: ", o.TmpDir)
		} else if len(o.TmpDir) == 0 {
			if err := os.RemoveAll(o.TmpDir); err != nil {
				t.Logf("failed to remove temporary directory %s: %v", o.TmpDir, err)
			}
		}
	}
}

// QEMU builds the u-root environment and prepares QEMU options given the test
// options and environment variables.
//
// QEMU will augment o.BuildOpts and o.QEMUOpts with configuration that the
// caller either requested (through the Options.Uinit field, for example) or
// that the caller did not set.
//
// QEMU returns the QEMU launch options or an error.
func QEMU(o *Options) (*qemu.Options, error) {
	if len(o.Name) == 0 {
		o.Name = callerName(2)
	}

	// Generate Elvish shell script of test commands in o.TmpDir.
	if len(o.TestCmds) > 0 {
		testFile := filepath.Join(o.TmpDir, "test.elv")

		if err := ioutil.WriteFile(
			testFile, []byte(strings.Join(o.TestCmds, "\n")), 0777); err != nil {
			return nil, err
		}
	}

	// Set the initramfs.
	if len(o.QEMUOpts.Initramfs) == 0 {
		o.QEMUOpts.Initramfs = filepath.Join(o.TmpDir, "initramfs.cpio")
		if err := ChooseTestInitramfs(o.DontSetEnv, o.BuildOpts, o.Uinit, o.QEMUOpts.Initramfs); err != nil {
			return nil, err
		}
	}

	if len(o.QEMUOpts.Kernel) == 0 {
		// Copy kernel to o.TmpDir for tests involving kexec.
		kernel := filepath.Join(o.TmpDir, "kernel")
		if err := cp.Copy(os.Getenv("UROOT_KERNEL"), kernel); err != nil {
			return nil, err
		}
		o.QEMUOpts.Kernel = kernel
	}

	switch TestArch() {
	case "amd64":
		o.QEMUOpts.KernelArgs += " console=ttyS0 earlyprintk=ttyS0"
	case "arm":
		o.QEMUOpts.KernelArgs += " console=ttyAMA0"
	}

	var dir qemu.Device
	if o.UseVVFAT {
		dir = qemu.ReadOnlyDirectory{Dir: o.TmpDir}
	} else {
		dir = qemu.P9Directory{Dir: o.TmpDir, Arch: TestArch()}
	}
	o.QEMUOpts.Devices = append(o.QEMUOpts.Devices, qemu.VirtioRandom{}, dir)

	return &o.QEMUOpts, nil
}

// ChooseTestInitramfs chooses which initramfs will be used for a given test and
// places it at the location given by outputFile.
// Default to the override initramfs if one is specified in the UROOT_INITRAMFS
// environment variable. Else, build an initramfs with the given parameters.
// If no uinit was provided, the generic one is used.
func ChooseTestInitramfs(dontSetEnv bool, o uroot.Opts, uinit, outputFile string) error {
	override := os.Getenv("UROOT_INITRAMFS")
	if len(override) > 0 {
		log.Printf("Overriding with initramfs %q", override)
		return cp.Copy(override, outputFile)
	}

	if len(uinit) == 0 {
		log.Printf("Defaulting to generic initramfs")
		uinit = "github.com/u-root/u-root/integration/testcmd/generic/uinit"
	}

	_, err := CreateTestInitramfs(dontSetEnv, o, uinit, outputFile)
	return err
}

// CreateTestInitramfs creates an initramfs with the given build options and
// uinit, and writes it to the given output file. If no output file is provided,
// one will be created.
// The output file name is returned. It is the caller's responsibility to remove
// the initramfs file after use.
func CreateTestInitramfs(dontSetEnv bool, o uroot.Opts, uinit, outputFile string) (string, error) {
	if !dontSetEnv {
		env := golang.Default()
		env.CgoEnabled = false
		env.GOARCH = TestArch()
		o.Env = env
	}

	logger := log.New(os.Stderr, "", 0)

	// If build opts don't specify any commands, include all commands. Else,
	// always add init and elvish.
	var cmds []string
	if len(o.Commands) == 0 {
		cmds = []string{
			"github.com/u-root/u-root/cmds/core/*",
			"github.com/u-root/u-root/cmds/exp/*",
		}
	}

	if len(uinit) != 0 {
		cmds = append(cmds, uinit)
	}

	// Add our commands to the build opts.
	o.AddBusyBoxCommands(cmds...)

	// Fill in the default build options if not specified.
	if o.BaseArchive == nil {
		o.BaseArchive = uroot.DefaultRamfs.Reader()
	}
	if len(o.InitCmd) == 0 {
		o.InitCmd = "init"
	}
	if len(o.DefaultShell) == 0 {
		o.DefaultShell = "elvish"
	}
	if len(o.TempDir) == 0 {
		tempDir, err := ioutil.TempDir("", "initramfs-tempdir")
		if err != nil {
			return "", fmt.Errorf("Failed to create temp dir: %v", err)
		}
		defer os.RemoveAll(tempDir)
		o.TempDir = tempDir
	}

	// Create an output file if one was not provided.
	if len(outputFile) == 0 {
		f, err := ioutil.TempFile("", "initramfs.cpio")
		if err != nil {
			return "", fmt.Errorf("failed to create output file: %v", err)
		}
		outputFile = f.Name()
	}
	w, err := initramfs.CPIO.OpenWriter(logger, outputFile, "", "")
	if err != nil {
		return "", fmt.Errorf("Failed to create initramfs writer: %v", err)
	}
	o.OutputFile = w

	return outputFile, uroot.CreateInitramfs(logger, o)
}
<|endoftext|>


<commit_before>package hive

import (
	"bufio"
	"errors"
	// "fmt"
	"github.com/eaciit/errorlib"
	// "github.com/eaciit/toolkit"
	"io"
	// "log"
	"os/exec"
	// "reflect"
	"strings"
)

const (
	BEE_CLI_STR  = "0: jdbc:hive2:"
	CLOSE_SCRIPT = "!quit"
)

type DuplexTerm struct {
	Writer     *bufio.Writer
	Reader     *bufio.Reader
	Cmd        *exec.Cmd
	CmdStr     string
	Stdin      io.WriteCloser
	Stdout     io.ReadCloser
	FnReceive  FnHiveReceive
	Exec       bool
	OutputType string
	DateFormat string
}

/*func (d *DuplexTerm) Open() (e error) {
	if d.Stdin, e = d.Cmd.StdinPipe(); e != nil {
		return
	}

	if d.Stdout, e = d.Cmd.StdoutPipe(); e != nil {
		return
	}

	d.Writer = bufio.NewWriter(d.Stdin)
	d.Reader = bufio.NewReader(d.Stdout)

	e = d.Cmd.Start()
	return
}*/

var hr HiveResult

func (d *DuplexTerm) Open() (e error) {
	if d.CmdStr != "" {
		arg := append([]string{"-c"}, d.CmdStr)
		d.Cmd = exec.Command("sh", arg...)

		if d.Stdin, e = d.Cmd.StdinPipe(); e != nil {
			return
		}

		if d.Stdout, e = d.Cmd.StdoutPipe(); e != nil {
			return
		}

		d.Writer = bufio.NewWriter(d.Stdin)
		d.Reader = bufio.NewReader(d.Stdout)

		/*if d.Exec {
			go func() {
				_, e = d.Wait()
			}()
		}*/
		e = d.Cmd.Start()
	} else {
		errorlib.Error("", "", "Open", "The Connection Config not Set")
	}

	return
}

func (d *DuplexTerm) Close() {
	result, e := d.SendInput(CLOSE_SCRIPT)

	_ = result
	_ = e

	d.Cmd.Wait()
	d.Stdin.Close()
	d.Stdout.Close()
}

func (d *DuplexTerm) SendInput(input string) (result []string, e error) {
	if d.Exec {
		go func() {
			_, e = d.Wait()
		}()
	}

	iwrite, e := d.Writer.WriteString(input + "\n")
	if iwrite == 0 {
		e = errors.New("Writing only 0 byte")
	} else {
		e = d.Writer.Flush()
	}

	if e != nil {
		return
	}

	if !d.Exec {
		result, e = d.Wait()
	}

	return
}

/*func (d *DuplexTerm) SetFn(f interface{}) {
	fn := reflect.ValueOf(f)
	fnType := fn.Type()
	if fnType.Kind() != reflect.Func || fnType.NumIn() != 1 || fnType.NumOut() != 1 {
		panic("Expected a unary function returning a single value")
	}

	d.Fn = f
}*/

func (d *DuplexTerm) Wait() (result []string, e error) {
	isHeader := false
	for {
		peekBefore, _ := d.Reader.Peek(14)
		peekBeforeStr := string(peekBefore)

		bread, e := d.Reader.ReadString('\n')
		bread = strings.TrimRight(bread, "\n")

		peek, _ := d.Reader.Peek(14)
		peekStr := string(peek)

		delimiter := "\t"

		if d.OutputType == CSV {
			delimiter = ","
		}

		if isHeader {
			hr.constructHeader(bread, delimiter)
			isHeader = false
		} else if !strings.Contains(bread, BEE_CLI_STR) {
			if d.Exec {
				/*fn := reflect.ValueOf(d.Fn)
				// tp := fn.Type().In(0)
				// tmp := reflect.New(tp).Elem()

				Parse(hr.Header, bread, &hr.ResultObj, d.OutputType, d.DateFormat)
				log.Printf("tmp: %v\n", &hr.ResultObj)

				res := fn.Call([]reflect.Value{reflect.ValueOf(hr.ResultObj)})
				log.Printf("res: %v\n", res)*/

				/*fn := reflect.ValueOf(d.Fn)
				tp := fn.Type().In(0)
				tmp := reflect.New(tp)

				xTmp := toolkit.M{}

				Parse(hr.Header, bread, &xTmp, d.OutputType, d.DateFormat)
				log.Printf("tmp: %v\n", xTmp)*/
				// log.Printf("tmp: %v\n", tmp)

				/*res := fn.Call([]reflect.Value{reflect.ValueOf(hr.ResultObj)})
				log.Printf("test: %v\n", res)
				d.FnReceive(res)*/
				hr.Result = append(hr.Result, bread)
				Parse(hr.Header, hr.Result, &hr.ResultObj, d.OutputType, d.DateFormat)
				d.FnReceive(hr)
			} else {
				result = append(result, bread)
			}
		}

		if BEE_CLI_STR == peekBeforeStr {
			isHeader = true
		}

		/*if d.Exec {
			if (e != nil && e.Error() == "EOF") || (strings.Contains(peekStr, CLOSE_SCRIPT)) {
				break
			}
		} else {*/
		if (e != nil && e.Error() == "EOF") || (BEE_CLI_STR == peekStr) {
			break
		}
		// }

	}

	return
}
<commit_msg>change some function<commit_after>package hive

import (
	"bufio"
	"errors"
	// "fmt"
	"github.com/eaciit/errorlib"
	// "github.com/eaciit/toolkit"
	"io"
	// "log"
	"os/exec"
	// "reflect"
	"strings"
)

const (
	BEE_CLI_STR  = "0: jdbc:hive2:"
	CLOSE_SCRIPT = "!quit"
)

type DuplexTerm struct {
	Writer     *bufio.Writer
	Reader     *bufio.Reader
	Cmd        *exec.Cmd
	CmdStr     string
	Stdin      io.WriteCloser
	Stdout     io.ReadCloser
	FnReceive  FnHiveReceive
	Exec       bool
	OutputType string
	DateFormat string
}

/*func (d *DuplexTerm) Open() (e error) {
	if d.Stdin, e = d.Cmd.StdinPipe(); e != nil {
		return
	}

	if d.Stdout, e = d.Cmd.StdoutPipe(); e != nil {
		return
	}

	d.Writer = bufio.NewWriter(d.Stdin)
	d.Reader = bufio.NewReader(d.Stdout)

	e = d.Cmd.Start()
	return
}*/

var hr HiveResult

func (d *DuplexTerm) Open() (e error) {
	if d.CmdStr != "" {
		arg := append([]string{"-c"}, d.CmdStr)
		d.Cmd = exec.Command("sh", arg...)

		if d.Stdin, e = d.Cmd.StdinPipe(); e != nil {
			return
		}

		if d.Stdout, e = d.Cmd.StdoutPipe(); e != nil {
			return
		}

		d.Writer = bufio.NewWriter(d.Stdin)
		d.Reader = bufio.NewReader(d.Stdout)

		/*if d.Exec {
			go func() {
				_, e = d.Wait()
			}()
		}*/
		e = d.Cmd.Start()
	} else {
		errorlib.Error("", "", "Open", "The Connection Config not Set")
	}

	return
}

func (d *DuplexTerm) Close() {
	result, e := d.SendInput(CLOSE_SCRIPT)

	_ = result
	_ = e

	d.Cmd.Wait()
	d.Stdin.Close()
	d.Stdout.Close()
}

func (d *DuplexTerm) SendInput(input string) (result []string, e error) {
	if d.Exec {
		go func() {
			_, e = d.Wait()
		}()
	}

	iwrite, e := d.Writer.WriteString(input + "\n")
	if iwrite == 0 {
		e = errors.New("Writing only 0 byte")
	} else {
		e = d.Writer.Flush()
	}

	if e != nil {
		return
	}

	if !d.Exec {
		result, e = d.Wait()
	}

	return
}

/*func (d *DuplexTerm) SetFn(f interface{}) {
	fn := reflect.ValueOf(f)
	fnType := fn.Type()
	if fnType.Kind() != reflect.Func || fnType.NumIn() != 1 || fnType.NumOut() != 1 {
		panic("Expected a unary function returning a single value")
	}

	d.Fn = f
}*/

func (d *DuplexTerm) Wait() (result []string, e error) {
	isHeader := false
	for {
		peekBefore, _ := d.Reader.Peek(14)
		peekBeforeStr := string(peekBefore)

		bread, e := d.Reader.ReadString('\n')
		bread = strings.TrimRight(bread, "\n")

		peek, _ := d.Reader.Peek(14)
		peekStr := string(peek)

		delimiter := "\t"

		if d.OutputType == CSV {
			delimiter = ","
		}

		if isHeader {
			hr.constructHeader(bread, delimiter)
			isHeader = false
		} else if !strings.Contains(bread, BEE_CLI_STR) {
			if d.Exec {
				/*fn := reflect.ValueOf(d.Fn)
				// tp := fn.Type().In(0)
				// tmp := reflect.New(tp).Elem()

				Parse(hr.Header, bread, &hr.ResultObj, d.OutputType, d.DateFormat)
				log.Printf("tmp: %v\n", &hr.ResultObj)

				res := fn.Call([]reflect.Value{reflect.ValueOf(hr.ResultObj)})
				log.Printf("res: %v\n", res)*/

				/*fn := reflect.ValueOf(d.Fn)
				tp := fn.Type().In(0)
				tmp := reflect.New(tp)

				xTmp := toolkit.M{}

				Parse(hr.Header, bread, &xTmp, d.OutputType, d.DateFormat)
				log.Printf("tmp: %v\n", xTmp)*/
				// log.Printf("tmp: %v\n", tmp)

				/*res := fn.Call([]reflect.Value{reflect.ValueOf(hr.ResultObj)})
				log.Printf("test: %v\n", res)
				d.FnReceive(res)*/
				hr.Result = append(hr.Result, bread)
				Parse(hr.Header, hr.Result, &hr.ResultObj, d.OutputType, d.DateFormat)
				d.FnReceive(hr)
			} else {
				result = append(result, bread)
			}
		}

		if BEE_CLI_STR == peekBeforeStr {
			isHeader = true
		}

		if d.Exec {
			if (e != nil && e.Error() == "EOF") || (strings.Contains(peekStr, CLOSE_SCRIPT)) {
				break
			}
		} else if (e != nil && e.Error() == "EOF") || (BEE_CLI_STR == peekStr) {
			break
		}
		// }

	}

	return
}
<|endoftext|>


<commit_before>// Package waveform is capable of generating waveform images from audio streams.  MIT Licensed.
package waveform

import (
	"image"
	"image/color"
	"image/draw"
	"io"
	"math"

	"azul3d.org/audio.v1"

	// Import WAV and FLAC decoders
	_ "azul3d.org/audio/wav.v1"
	_ "github.com/azul3d/audio-flac"
)

const (
	// yDefault is the default height of the generated waveform image
	yDefault = 128

	// rmsScaleDefault is the default scaling factor used when scaling RMS value and waveform height
	// by the output image's height
	rmsScaleDefault = 3.00
)

var (
	// ErrFormat is returned when the input audio format is not a registered format
	// with the audio package.
	ErrFormat = struct{ error }{audio.ErrFormat}

	// ErrInvalidData is returned when the input audio format is recognized, but
	// the stream is invalid or corrupt in some way.
	ErrInvalidData = struct{ error }{audio.ErrInvalidData}

	// ErrUnexpectedEOS is returned when end-of-stream is encountered in the middle
	// of a fixed-size block or data structure.
	ErrUnexpectedEOS = struct{ error }{audio.ErrUnexpectedEOS}
)

// Options are used to customize properties about a waveform image.
type Options struct {
	// BackgroundColor and ForegroundColor specify the background and foreground
	// color of a waveform image, respectively.
	BackgroundColor color.Color
	ForegroundColor color.Color

	// Resolution sets the number of times audio is read and drawn
	// as a waveform, per second of audio.
	Resolution int

	// ScaleX and ScaleY are scaling factors used to scale a waveform image on its
	// X or Y axis, respectively.
	ScaleX int
	ScaleY int

	// Sharpness is used to apply a curve to a waveform image, scaled on its X-axis.
	// A higher value results in steeper curves, and a lower value results in more
	// "blocky" curves.
	Sharpness int

	// ScaleRMS specifies if the waveform image should be scaled down on its Y-axis
	// when certain RMS thresholds are reached.  This can be used to show a more
	// accurate waveform when a song reaches very high RMS thresholds.
	ScaleRMS bool
}

// DefaultOptions is a set of sane defaults, which are applied when no options are
// passed to New.
var DefaultOptions = &Options{
	// Black waveform on white background
	BackgroundColor: color.White,
	ForegroundColor: color.Black,

	// Read audio and draw waveform once per second of audio
	Resolution: 1,

	// No scaling
	ScaleX: 1,
	ScaleY: 1,

	// Normal sharpness
	Sharpness: 1,

	// Do not scale high RMS values
	ScaleRMS: false,
}

// New creates a new image.Image from a io.Reader.  An Options struct may be passed to
// enable further customization; else, DefaultOptions is used.
//
// New reads the input io.Reader, processes its input into a waveform, and returns the
// resulting image.Image.  On failure, New will return any errors which occur.
func New(r io.Reader, options *Options) (image.Image, error) {
	// If options are nil, set sane defaults
	if options == nil {
		options = DefaultOptions
	}

	// If resolution is 0 or less, set it to 1 to avoid divide-by-zero panic
	if options.Resolution <= 0 {
		options.Resolution = 1
	}

	// If either scale is 0 or less, set to 1 to avoid empty image
	if options.ScaleX <= 0 {
		options.ScaleX = 1
	}
	if options.ScaleY <= 0 {
		options.ScaleY = 1
	}

	// If sharpness is negative, set to 0
	if options.Sharpness < 0 {
		options.Sharpness = 0
	}

	// If color options are nil, set sane defaults to prevent panic
	if options.BackgroundColor == nil {
		options.BackgroundColor = DefaultOptions.BackgroundColor
	}
	if options.ForegroundColor == nil {
		options.ForegroundColor = DefaultOptions.ForegroundColor
	}

	// Open audio decoder on input stream
	decoder, _, err := audio.NewDecoder(r)
	if err != nil {
		// Unknown format
		if err == audio.ErrFormat {
			return nil, ErrFormat
		}

		// Invalid data
		if err == audio.ErrInvalidData {
			return nil, ErrInvalidData
		}

		// Unexpected end-of-stream
		if err == audio.ErrUnexpectedEOS {
			return nil, ErrUnexpectedEOS
		}

		// All other errors
		return nil, err
	}

	// rms is a slice of computed RMS values from each second of audio samples
	rms := make([]float64, 0)

	// Track the maximum RMS value computed, optionally used for scaling later
	var maxRMS float64

	// samples is a slice of float64 audio samples, used to store decoded values
	config := decoder.Config()
	samples := make(audio.F64Samples, (config.SampleRate*config.Channels)/options.Resolution)
	for {
		// Decode at specified resolution from options
		if _, err := decoder.Read(samples); err != nil {
			// On end of stream, stop reading values
			if err == audio.EOS {
				break
			}

			// On all other errors, return
			return nil, err
		}

		// Calculate RMS from float64 audio samples
		rmsSample := rmsF64Samples(samples)

		// Track the highest RMS value
		if rmsSample > maxRMS {
			maxRMS = rmsSample
		}

		// Store computed value
		rms = append(rms, rmsSample)
	}

	// Set image resolution
	imgX := len(rms) * options.ScaleX
	imgY := yDefault * options.ScaleY

	// Create output image, fill image with specified background color
	img := image.NewRGBA(image.Rect(0, 0, imgX, imgY))
	draw.Draw(img, img.Bounds(), image.NewUniform(options.BackgroundColor), image.ZP, draw.Src)

	// Calculate halfway point of Y-axis for image
	imgHalfY := img.Bounds().Max.Y / 2

	// Calculate a peak value used for smoothing scaled X-axis images
	peak := int(math.Ceil(float64(options.ScaleX)) / 2)

	// Calculate RMS scaling factor, based upon maximum RMS value found
	// If option ScaleRMS is true, when maximum value is above certain thresholds
	// the scaling factor is reduced to show an accurate waveform with less clipping
	rmsScale := rmsScaleDefault
	if options.ScaleRMS {
		if maxRMS > 0.35 {
			rmsScale -= 0.5
		}
		if maxRMS > 0.40 {
			rmsScale -= 0.25
		}
	}

	// Begin iterating all gathered RMS values
	x := 0
	for _, r := range rms {
		// Scale RMS value to an integer, using the height of the image and a constant
		// scaling factor
		scaleRMS := int(math.Floor(r * float64(img.Bounds().Max.Y) * rmsScale))

		// Calculate the halfway point for the scaled RMS value
		halfScaleRMS := scaleRMS / 2

		// Iterate image coordinates on the Y-axis, generating a symmetrical waveform
		// image above and below the center of the image
		for y := imgHalfY - halfScaleRMS; y < scaleRMS+(imgHalfY-halfScaleRMS); y++ {
			// If X-axis is being scaled, draw RMS value over several X coordinates
			for i := 0; i < options.ScaleX; i++ {
				// When scaled, adjust RMS value to be lower on either side of the peak,
				// so that the image appears more smooth and less "blocky"
				var adjust int
				if i < peak {
					// Adjust downward
					adjust = (i - peak) * options.Sharpness
				} else if i == peak {
					// No adjustment at peak
					adjust = 0
				} else {
					// Adjust downward
					adjust = (peak - i) * options.Sharpness
				}

				// On top half of the image, invert adjustment to create symmetry between
				// top and bottom halves
				if y < imgHalfY {
					adjust = -1 * adjust
				}

				// Draw using specified color at specified X and Y coordinate
				img.Set(x+i, y+adjust, options.ForegroundColor)
			}
		}

		// Increase X by scaling factor, to continue drawing at next loop
		x += options.ScaleX
	}

	// Return generated image
	return img, nil
}

// rmsF64Samples calculates the root mean square of a slice of float64 audio samples,
// enabling the measurement of magnitude over the entire set of samples.
// Derived from: http://en.wikipedia.org/wiki/Root_mean_square
func rmsF64Samples(samples audio.F64Samples) float64 {
	// Square and sum all input samples
	var sumSquare float64
	for i := range samples {
		sumSquare += math.Pow(float64(samples.At(i)), 2)
	}

	// Multiply squared sum by (1/n) coefficient, return square root
	return math.Sqrt(float64((float64(1) / float64(samples.Len()))) * sumSquare)
}
<commit_msg>Add more RMS scaling for higher max thresholds<commit_after>// Package waveform is capable of generating waveform images from audio streams.  MIT Licensed.
package waveform

import (
	"image"
	"image/color"
	"image/draw"
	"io"
	"math"

	"azul3d.org/audio.v1"

	// Import WAV and FLAC decoders
	_ "azul3d.org/audio/wav.v1"
	_ "github.com/azul3d/audio-flac"
)

const (
	// yDefault is the default height of the generated waveform image
	yDefault = 128

	// rmsScaleDefault is the default scaling factor used when scaling RMS value and waveform height
	// by the output image's height
	rmsScaleDefault = 3.00
)

var (
	// ErrFormat is returned when the input audio format is not a registered format
	// with the audio package.
	ErrFormat = struct{ error }{audio.ErrFormat}

	// ErrInvalidData is returned when the input audio format is recognized, but
	// the stream is invalid or corrupt in some way.
	ErrInvalidData = struct{ error }{audio.ErrInvalidData}

	// ErrUnexpectedEOS is returned when end-of-stream is encountered in the middle
	// of a fixed-size block or data structure.
	ErrUnexpectedEOS = struct{ error }{audio.ErrUnexpectedEOS}
)

// Options are used to customize properties about a waveform image.
type Options struct {
	// BackgroundColor and ForegroundColor specify the background and foreground
	// color of a waveform image, respectively.
	BackgroundColor color.Color
	ForegroundColor color.Color

	// Resolution sets the number of times audio is read and drawn
	// as a waveform, per second of audio.
	Resolution int

	// ScaleX and ScaleY are scaling factors used to scale a waveform image on its
	// X or Y axis, respectively.
	ScaleX int
	ScaleY int

	// Sharpness is used to apply a curve to a waveform image, scaled on its X-axis.
	// A higher value results in steeper curves, and a lower value results in more
	// "blocky" curves.
	Sharpness int

	// ScaleRMS specifies if the waveform image should be scaled down on its Y-axis
	// when certain RMS thresholds are reached.  This can be used to show a more
	// accurate waveform when a song reaches very high RMS thresholds.
	ScaleRMS bool
}

// DefaultOptions is a set of sane defaults, which are applied when no options are
// passed to New.
var DefaultOptions = &Options{
	// Black waveform on white background
	BackgroundColor: color.White,
	ForegroundColor: color.Black,

	// Read audio and draw waveform once per second of audio
	Resolution: 1,

	// No scaling
	ScaleX: 1,
	ScaleY: 1,

	// Normal sharpness
	Sharpness: 1,

	// Do not scale high RMS values
	ScaleRMS: false,
}

// New creates a new image.Image from a io.Reader.  An Options struct may be passed to
// enable further customization; else, DefaultOptions is used.
//
// New reads the input io.Reader, processes its input into a waveform, and returns the
// resulting image.Image.  On failure, New will return any errors which occur.
func New(r io.Reader, options *Options) (image.Image, error) {
	// If options are nil, set sane defaults
	if options == nil {
		options = DefaultOptions
	}

	// If resolution is 0 or less, set it to 1 to avoid divide-by-zero panic
	if options.Resolution <= 0 {
		options.Resolution = 1
	}

	// If either scale is 0 or less, set to 1 to avoid empty image
	if options.ScaleX <= 0 {
		options.ScaleX = 1
	}
	if options.ScaleY <= 0 {
		options.ScaleY = 1
	}

	// If sharpness is negative, set to 0
	if options.Sharpness < 0 {
		options.Sharpness = 0
	}

	// If color options are nil, set sane defaults to prevent panic
	if options.BackgroundColor == nil {
		options.BackgroundColor = DefaultOptions.BackgroundColor
	}
	if options.ForegroundColor == nil {
		options.ForegroundColor = DefaultOptions.ForegroundColor
	}

	// Open audio decoder on input stream
	decoder, _, err := audio.NewDecoder(r)
	if err != nil {
		// Unknown format
		if err == audio.ErrFormat {
			return nil, ErrFormat
		}

		// Invalid data
		if err == audio.ErrInvalidData {
			return nil, ErrInvalidData
		}

		// Unexpected end-of-stream
		if err == audio.ErrUnexpectedEOS {
			return nil, ErrUnexpectedEOS
		}

		// All other errors
		return nil, err
	}

	// rms is a slice of computed RMS values from each second of audio samples
	rms := make([]float64, 0)

	// Track the maximum RMS value computed, optionally used for scaling later
	var maxRMS float64

	// samples is a slice of float64 audio samples, used to store decoded values
	config := decoder.Config()
	samples := make(audio.F64Samples, (config.SampleRate*config.Channels)/options.Resolution)
	for {
		// Decode at specified resolution from options
		if _, err := decoder.Read(samples); err != nil {
			// On end of stream, stop reading values
			if err == audio.EOS {
				break
			}

			// On all other errors, return
			return nil, err
		}

		// Calculate RMS from float64 audio samples
		rmsSample := rmsF64Samples(samples)

		// Track the highest RMS value
		if rmsSample > maxRMS {
			maxRMS = rmsSample
		}

		// Store computed value
		rms = append(rms, rmsSample)
	}

	// Set image resolution
	imgX := len(rms) * options.ScaleX
	imgY := yDefault * options.ScaleY

	// Create output image, fill image with specified background color
	img := image.NewRGBA(image.Rect(0, 0, imgX, imgY))
	draw.Draw(img, img.Bounds(), image.NewUniform(options.BackgroundColor), image.ZP, draw.Src)

	// Calculate halfway point of Y-axis for image
	imgHalfY := img.Bounds().Max.Y / 2

	// Calculate a peak value used for smoothing scaled X-axis images
	peak := int(math.Ceil(float64(options.ScaleX)) / 2)

	// Calculate RMS scaling factor, based upon maximum RMS value found
	// If option ScaleRMS is true, when maximum value is above certain thresholds
	// the scaling factor is reduced to show an accurate waveform with less clipping
	rmsScale := rmsScaleDefault
	if options.ScaleRMS {
		if maxRMS > 0.35 {
			rmsScale -= 0.5
		}
		if maxRMS > 0.40 {
			rmsScale -= 0.25
		}
		if maxRMS > 0.45 {
			rmsScale -= 0.25
		}
		if maxRMS > 0.50 {
			rmsScale -= 0.25
		}
	}

	// Begin iterating all gathered RMS values
	x := 0
	for _, r := range rms {
		// Scale RMS value to an integer, using the height of the image and a constant
		// scaling factor
		scaleRMS := int(math.Floor(r * float64(img.Bounds().Max.Y) * rmsScale))

		// Calculate the halfway point for the scaled RMS value
		halfScaleRMS := scaleRMS / 2

		// Iterate image coordinates on the Y-axis, generating a symmetrical waveform
		// image above and below the center of the image
		for y := imgHalfY - halfScaleRMS; y < scaleRMS+(imgHalfY-halfScaleRMS); y++ {
			// If X-axis is being scaled, draw RMS value over several X coordinates
			for i := 0; i < options.ScaleX; i++ {
				// When scaled, adjust RMS value to be lower on either side of the peak,
				// so that the image appears more smooth and less "blocky"
				var adjust int
				if i < peak {
					// Adjust downward
					adjust = (i - peak) * options.Sharpness
				} else if i == peak {
					// No adjustment at peak
					adjust = 0
				} else {
					// Adjust downward
					adjust = (peak - i) * options.Sharpness
				}

				// On top half of the image, invert adjustment to create symmetry between
				// top and bottom halves
				if y < imgHalfY {
					adjust = -1 * adjust
				}

				// Draw using specified color at specified X and Y coordinate
				img.Set(x+i, y+adjust, options.ForegroundColor)
			}
		}

		// Increase X by scaling factor, to continue drawing at next loop
		x += options.ScaleX
	}

	// Return generated image
	return img, nil
}

// rmsF64Samples calculates the root mean square of a slice of float64 audio samples,
// enabling the measurement of magnitude over the entire set of samples.
// Derived from: http://en.wikipedia.org/wiki/Root_mean_square
func rmsF64Samples(samples audio.F64Samples) float64 {
	// Square and sum all input samples
	var sumSquare float64
	for i := range samples {
		sumSquare += math.Pow(float64(samples.At(i)), 2)
	}

	// Multiply squared sum by (1/n) coefficient, return square root
	return math.Sqrt(float64((float64(1) / float64(samples.Len()))) * sumSquare)
}
<|endoftext|>


<commit_before>package solver

import (
	"context"
	"sync"

	"github.com/moby/buildkit/solver-next/internal/pipe"
	"github.com/moby/buildkit/util/cond"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const debugScheduler = false // TODO: replace with logs in build trace

func NewScheduler(ef EdgeFactory) *Scheduler {
	s := &Scheduler{
		waitq:    map[*edge]struct{}{},
		incoming: map[*edge][]*edgePipe{},
		outgoing: map[*edge][]*edgePipe{},

		stopped: make(chan struct{}),
		closed:  make(chan struct{}),

		ef: ef,
	}
	s.cond = cond.NewStatefulCond(&s.mu)

	go s.loop()

	return s
}

type Scheduler struct {
	cond *cond.StatefulCond
	mu   sync.Mutex
	muQ  sync.Mutex

	ef EdgeFactory

	waitq       map[*edge]struct{}
	stopped     chan struct{}
	stoppedOnce sync.Once
	closed      chan struct{}

	incoming map[*edge][]*edgePipe
	outgoing map[*edge][]*edgePipe
}

func (s *Scheduler) Stop() {
	s.stoppedOnce.Do(func() {
		close(s.stopped)
	})
	<-s.closed
}

func (s *Scheduler) loop() {
	defer func() {
		close(s.closed)
	}()

	go func() {
		<-s.stopped
		s.mu.Lock()
		s.cond.Signal()
		s.mu.Unlock()
	}()

	s.mu.Lock()
	for {
		select {
		case <-s.stopped:
			s.mu.Unlock()
			return
		default:
		}
		s.muQ.Lock()
		q := s.waitq
		s.waitq = map[*edge]struct{}{}
		s.muQ.Unlock()
		if len(q) == 0 {
			s.cond.Wait()
			continue
		}

		for e := range q {
			s.dispatch(e)
		}
	}
}

// dispatch schedules an edge to be processed
func (s *Scheduler) dispatch(e *edge) {
	inc := make([]pipe.Sender, len(s.incoming[e]))
	for i, p := range s.incoming[e] {
		inc[i] = p.Sender
	}
	out := make([]pipe.Receiver, len(s.outgoing[e]))
	for i, p := range s.outgoing[e] {
		out[i] = p.Receiver
	}

	e.hasActiveOutgoing = false
	updates := []pipe.Receiver{}
	for _, p := range out {
		if ok := p.Receive(); ok {
			updates = append(updates, p)
		}
		if !p.Status().Completed {
			e.hasActiveOutgoing = true
		}
	}

	// unpark the edge
	debugSchedulerPreUnpark(e, inc, updates, out)
	e.unpark(inc, updates, out, &pipeFactory{s: s, e: e})
	debugSchedulerPostUnpark(e, inc)

	// set up new requests that didn't complete/were added by this run
	openIncoming := make([]*edgePipe, 0, len(inc))
	for _, r := range s.incoming[e] {
		if !r.Sender.Status().Completed {
			openIncoming = append(openIncoming, r)
		}
	}
	if len(openIncoming) > 0 {
		s.incoming[e] = openIncoming
	} else {
		delete(s.incoming, e)
	}

	openOutgoing := make([]*edgePipe, 0, len(out))
	for _, r := range s.outgoing[e] {
		if !r.Receiver.Status().Completed {
			openOutgoing = append(openOutgoing, r)
		}
	}
	if len(openOutgoing) > 0 {
		s.outgoing[e] = openOutgoing
	} else {
		delete(s.outgoing, e)
	}

	// if keys changed there might be possiblity for merge with other edge
	if e.keysDidChange {
		if k := e.currentIndexKey(); k != nil {
			// skip this if not at least 1 key per dep
			origEdge := e.index.LoadOrStore(k, e)
			if origEdge != nil {
				logrus.Debugf("merging edge %s to %s\n", e.edge.Vertex.Name(), origEdge.edge.Vertex.Name())
				if s.mergeTo(origEdge, e) {
					s.ef.SetEdge(e.edge, origEdge)
				}
			}
		}
		e.keysDidChange = false
	}

	// validation to avoid deadlocks/resource leaks:
	// TODO: if these start showing up in error reports they can be changed
	// to error the edge instead. They can only appear from algorithm bugs in
	// unpark(), not for any external input.
	if len(openIncoming) > 0 && len(openOutgoing) == 0 {
		panic("invalid dispatch: return leaving incoming open")
	}
	if len(openIncoming) == 0 && len(openOutgoing) > 0 {
		panic("invalid dispatch: return leaving outgoing open")
	}
}

// signal notifies that an edge needs to be processed again
func (s *Scheduler) signal(e *edge) {
	s.muQ.Lock()
	if _, ok := s.waitq[e]; !ok {
		s.waitq[e] = struct{}{}
		s.cond.Signal()
	}
	s.muQ.Unlock()
}

// build evaluates edge into a result
func (s *Scheduler) build(ctx context.Context, edge Edge) (CachedResult, error) {
	s.mu.Lock()
	e := s.ef.GetEdge(edge)
	if e == nil {
		s.mu.Unlock()
		return nil, errors.Errorf("invalid request %v for build", edge)
	}

	wait := make(chan struct{})

	var p *pipe.Pipe
	p = s.newPipe(e, nil, pipe.Request{Payload: &edgeRequest{desiredState: edgeStatusComplete}})
	p.OnSendCompletion = func() {
		p.Receiver.Receive()
		if p.Receiver.Status().Completed {
			close(wait)
		}
	}
	s.mu.Unlock()

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	go func() {
		<-ctx.Done()
		p.Receiver.Cancel()
	}()

	<-wait

	if err := p.Receiver.Status().Err; err != nil {
		return nil, err
	}
	return p.Receiver.Status().Value.(*edgeState).result.Clone(), nil
}

// newPipe creates a new request pipe between two edges
func (s *Scheduler) newPipe(target, from *edge, req pipe.Request) *pipe.Pipe {
	p := &edgePipe{
		Pipe:   pipe.New(req),
		Target: target,
		From:   from,
	}

	s.signal(target)
	if from != nil {
		p.OnSendCompletion = func() {
			p.mu.Lock()
			defer p.mu.Unlock()
			s.signal(p.From)
		}
		s.outgoing[from] = append(s.outgoing[from], p)
	}
	s.incoming[target] = append(s.incoming[target], p)
	p.OnReceiveCompletion = func() {
		p.mu.Lock()
		defer p.mu.Unlock()
		s.signal(p.Target)
	}
	return p.Pipe
}

// newRequestWithFunc creates a new request pipe that invokes a async function
func (s *Scheduler) newRequestWithFunc(e *edge, f func(context.Context) (interface{}, error)) pipe.Receiver {
	pp, start := pipe.NewWithFunction(f)
	p := &edgePipe{
		Pipe: pp,
		From: e,
	}
	p.OnSendCompletion = func() {
		p.mu.Lock()
		defer p.mu.Unlock()
		s.signal(p.From)
	}
	s.outgoing[e] = append(s.outgoing[e], p)
	go start()
	return p.Receiver
}

// mergeTo merges the state from one edge to another. source edge is discarded.
func (s *Scheduler) mergeTo(target, src *edge) bool {
	if !target.edge.Vertex.Options().IgnoreCache && src.edge.Vertex.Options().IgnoreCache {
		return false
	}
	for _, inc := range s.incoming[src] {
		inc.mu.Lock()
		inc.Target = target
		s.incoming[target] = append(s.incoming[target], inc)
		inc.mu.Unlock()
	}

	for _, out := range s.outgoing[src] {
		out.mu.Lock()
		out.From = target
		s.outgoing[target] = append(s.outgoing[target], out)
		out.mu.Unlock()
		out.Receiver.Cancel()
	}

	delete(s.incoming, src)
	delete(s.outgoing, src)
	s.signal(target)

	for i, d := range src.deps {
		for _, k := range d.keys {
			target.secondaryExporters = append(target.secondaryExporters, expDep{i, CacheKeyWithSelector{CacheKey: k, Selector: src.cacheMap.Deps[i].Selector}})
		}
		if d.slowCacheKey != nil {
			target.secondaryExporters = append(target.secondaryExporters, expDep{i, CacheKeyWithSelector{CacheKey: *d.slowCacheKey}})
		}
		if d.result != nil {
			target.secondaryExporters = append(target.secondaryExporters, expDep{i, CacheKeyWithSelector{CacheKey: d.result.CacheKey(), Selector: src.cacheMap.Deps[i].Selector}})
		}
	}

	// TODO(tonistiigi): merge cache providers

	return true
}

// EdgeFactory allows access to the edges from a shared graph
type EdgeFactory interface {
	GetEdge(Edge) *edge
	SetEdge(Edge, *edge)
}

type pipeFactory struct {
	e *edge
	s *Scheduler
}

func (pf *pipeFactory) NewInputRequest(ee Edge, req *edgeRequest) pipe.Receiver {
	target := pf.s.ef.GetEdge(ee)
	if target == nil {
		panic("failed to get edge") // TODO: return errored pipe
	}
	p := pf.s.newPipe(target, pf.e, pipe.Request{Payload: req})
	if debugScheduler {
		logrus.Debugf("> newPipe %s %p desiredState=%s", ee.Vertex.Name(), p, req.desiredState)
	}
	return p.Receiver
}

func (pf *pipeFactory) NewFuncRequest(f func(context.Context) (interface{}, error)) pipe.Receiver {
	p := pf.s.newRequestWithFunc(pf.e, f)
	if debugScheduler {
		logrus.Debugf("> newFunc %p", p)
	}
	return p
}

func debugSchedulerPreUnpark(e *edge, inc []pipe.Sender, updates, allPipes []pipe.Receiver) {
	if !debugScheduler {
		return
	}
	logrus.Debugf(">> unpark %s req=%d upt=%d out=%d state=%s %s", e.edge.Vertex.Name(), len(inc), len(updates), len(allPipes), e.state, e.edge.Vertex.Digest())

	for i, dep := range e.deps {
		des := edgeStatusInitial
		if dep.req != nil {
			des = dep.req.Request().(*edgeRequest).desiredState
		}
		logrus.Debugf(":: dep%d %s state=%s des=%s keys=%s hasslowcache=%v", i, e.edge.Vertex.Inputs()[i].Vertex.Name(), dep.state, des, len(dep.keys), e.slowCacheFunc(dep) != nil)
	}

	for i, in := range inc {
		req := in.Request()
		logrus.Debugf("> incoming-%d: %p dstate=%s canceled=%v", i, in, req.Payload.(*edgeRequest).desiredState, req.Canceled)
	}

	for i, up := range updates {
		if up == e.cacheMapReq {
			logrus.Debugf("> update-%d: %p cacheMapReq complete=%v", i, up, up.Status().Completed)
		} else if up == e.execReq {
			logrus.Debugf("> update-%d: %p execReq complete=%v", i, up, up.Status().Completed)
		} else {
			st, ok := up.Status().Value.(*edgeState)
			if ok {
				index := -1
				if dep, ok := e.depRequests[up]; ok {
					index = int(dep.index)
				}
				logrus.Debugf("> update-%d: %p input-%d keys=%d state=%s", i, up, index, len(st.keys), st.state)
			} else {
				logrus.Debugf("> update-%d: unknown", i)
			}
		}
	}
}

func debugSchedulerPostUnpark(e *edge, inc []pipe.Sender) {
	if !debugScheduler {
		return
	}
	for i, in := range inc {
		logrus.Debugf("< incoming-%d: %p completed=%v", i, in, in.Status().Completed)
	}
	logrus.Debugf("<< unpark %s\n", e.edge.Vertex.Name())
}
<commit_msg>solver: make event dispatching more predictable<commit_after>package solver

import (
	"context"
	"sync"

	"github.com/moby/buildkit/solver-next/internal/pipe"
	"github.com/moby/buildkit/util/cond"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const debugScheduler = false // TODO: replace with logs in build trace

func NewScheduler(ef EdgeFactory) *Scheduler {
	s := &Scheduler{
		waitq:    map[*edge]struct{}{},
		incoming: map[*edge][]*edgePipe{},
		outgoing: map[*edge][]*edgePipe{},

		stopped: make(chan struct{}),
		closed:  make(chan struct{}),

		ef: ef,
	}
	s.cond = cond.NewStatefulCond(&s.mu)

	go s.loop()

	return s
}

type dispatcher struct {
	next *dispatcher
	e    *edge
}

type Scheduler struct {
	cond *cond.StatefulCond
	mu   sync.Mutex
	muQ  sync.Mutex

	ef EdgeFactory

	waitq       map[*edge]struct{}
	next        *dispatcher
	last        *dispatcher
	stopped     chan struct{}
	stoppedOnce sync.Once
	closed      chan struct{}

	incoming map[*edge][]*edgePipe
	outgoing map[*edge][]*edgePipe
}

func (s *Scheduler) Stop() {
	s.stoppedOnce.Do(func() {
		close(s.stopped)
	})
	<-s.closed
}

func (s *Scheduler) loop() {
	defer func() {
		close(s.closed)
	}()

	go func() {
		<-s.stopped
		s.mu.Lock()
		s.cond.Signal()
		s.mu.Unlock()
	}()

	s.mu.Lock()
	for {
		select {
		case <-s.stopped:
			s.mu.Unlock()
			return
		default:
		}
		s.muQ.Lock()
		l := s.next
		if l != nil {
			if l == s.last {
				s.last = nil
			}
			s.next = l.next
			delete(s.waitq, l.e)
		}
		s.muQ.Unlock()
		if l == nil {
			s.cond.Wait()
			continue
		}
		s.dispatch(l.e)
	}
}

// dispatch schedules an edge to be processed
func (s *Scheduler) dispatch(e *edge) {
	inc := make([]pipe.Sender, len(s.incoming[e]))
	for i, p := range s.incoming[e] {
		inc[i] = p.Sender
	}
	out := make([]pipe.Receiver, len(s.outgoing[e]))
	for i, p := range s.outgoing[e] {
		out[i] = p.Receiver
	}

	e.hasActiveOutgoing = false
	updates := []pipe.Receiver{}
	for _, p := range out {
		if ok := p.Receive(); ok {
			updates = append(updates, p)
		}
		if !p.Status().Completed {
			e.hasActiveOutgoing = true
		}
	}

	// unpark the edge
	debugSchedulerPreUnpark(e, inc, updates, out)
	e.unpark(inc, updates, out, &pipeFactory{s: s, e: e})
	debugSchedulerPostUnpark(e, inc)

	// set up new requests that didn't complete/were added by this run
	openIncoming := make([]*edgePipe, 0, len(inc))
	for _, r := range s.incoming[e] {
		if !r.Sender.Status().Completed {
			openIncoming = append(openIncoming, r)
		}
	}
	if len(openIncoming) > 0 {
		s.incoming[e] = openIncoming
	} else {
		delete(s.incoming, e)
	}

	openOutgoing := make([]*edgePipe, 0, len(out))
	for _, r := range s.outgoing[e] {
		if !r.Receiver.Status().Completed {
			openOutgoing = append(openOutgoing, r)
		}
	}
	if len(openOutgoing) > 0 {
		s.outgoing[e] = openOutgoing
	} else {
		delete(s.outgoing, e)
	}

	// if keys changed there might be possiblity for merge with other edge
	if e.keysDidChange {
		if k := e.currentIndexKey(); k != nil {
			// skip this if not at least 1 key per dep
			origEdge := e.index.LoadOrStore(k, e)
			if origEdge != nil {
				logrus.Debugf("merging edge %s to %s\n", e.edge.Vertex.Name(), origEdge.edge.Vertex.Name())
				if s.mergeTo(origEdge, e) {
					s.ef.SetEdge(e.edge, origEdge)
				}
			}
		}
		e.keysDidChange = false
	}

	// validation to avoid deadlocks/resource leaks:
	// TODO: if these start showing up in error reports they can be changed
	// to error the edge instead. They can only appear from algorithm bugs in
	// unpark(), not for any external input.
	if len(openIncoming) > 0 && len(openOutgoing) == 0 {
		panic("invalid dispatch: return leaving incoming open")
	}
	if len(openIncoming) == 0 && len(openOutgoing) > 0 {
		panic("invalid dispatch: return leaving outgoing open")
	}
}

// signal notifies that an edge needs to be processed again
func (s *Scheduler) signal(e *edge) {
	s.muQ.Lock()
	if _, ok := s.waitq[e]; !ok {
		d := &dispatcher{e: e}
		if s.last == nil {
			s.next = d
		} else {
			s.last.next = d
		}
		s.last = d
		s.waitq[e] = struct{}{}
		s.cond.Signal()
	}
	s.muQ.Unlock()
}

// build evaluates edge into a result
func (s *Scheduler) build(ctx context.Context, edge Edge) (CachedResult, error) {
	s.mu.Lock()
	e := s.ef.GetEdge(edge)
	if e == nil {
		s.mu.Unlock()
		return nil, errors.Errorf("invalid request %v for build", edge)
	}

	wait := make(chan struct{})

	var p *pipe.Pipe
	p = s.newPipe(e, nil, pipe.Request{Payload: &edgeRequest{desiredState: edgeStatusComplete}})
	p.OnSendCompletion = func() {
		p.Receiver.Receive()
		if p.Receiver.Status().Completed {
			close(wait)
		}
	}
	s.mu.Unlock()

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	go func() {
		<-ctx.Done()
		p.Receiver.Cancel()
	}()

	<-wait

	if err := p.Receiver.Status().Err; err != nil {
		return nil, err
	}
	return p.Receiver.Status().Value.(*edgeState).result.Clone(), nil
}

// newPipe creates a new request pipe between two edges
func (s *Scheduler) newPipe(target, from *edge, req pipe.Request) *pipe.Pipe {
	p := &edgePipe{
		Pipe:   pipe.New(req),
		Target: target,
		From:   from,
	}

	s.signal(target)
	if from != nil {
		p.OnSendCompletion = func() {
			p.mu.Lock()
			defer p.mu.Unlock()
			s.signal(p.From)
		}
		s.outgoing[from] = append(s.outgoing[from], p)
	}
	s.incoming[target] = append(s.incoming[target], p)
	p.OnReceiveCompletion = func() {
		p.mu.Lock()
		defer p.mu.Unlock()
		s.signal(p.Target)
	}
	return p.Pipe
}

// newRequestWithFunc creates a new request pipe that invokes a async function
func (s *Scheduler) newRequestWithFunc(e *edge, f func(context.Context) (interface{}, error)) pipe.Receiver {
	pp, start := pipe.NewWithFunction(f)
	p := &edgePipe{
		Pipe: pp,
		From: e,
	}
	p.OnSendCompletion = func() {
		p.mu.Lock()
		defer p.mu.Unlock()
		s.signal(p.From)
	}
	s.outgoing[e] = append(s.outgoing[e], p)
	go start()
	return p.Receiver
}

// mergeTo merges the state from one edge to another. source edge is discarded.
func (s *Scheduler) mergeTo(target, src *edge) bool {
	if !target.edge.Vertex.Options().IgnoreCache && src.edge.Vertex.Options().IgnoreCache {
		return false
	}
	for _, inc := range s.incoming[src] {
		inc.mu.Lock()
		inc.Target = target
		s.incoming[target] = append(s.incoming[target], inc)
		inc.mu.Unlock()
	}

	for _, out := range s.outgoing[src] {
		out.mu.Lock()
		out.From = target
		s.outgoing[target] = append(s.outgoing[target], out)
		out.mu.Unlock()
		out.Receiver.Cancel()
	}

	delete(s.incoming, src)
	delete(s.outgoing, src)
	s.signal(target)

	for i, d := range src.deps {
		for _, k := range d.keys {
			target.secondaryExporters = append(target.secondaryExporters, expDep{i, CacheKeyWithSelector{CacheKey: k, Selector: src.cacheMap.Deps[i].Selector}})
		}
		if d.slowCacheKey != nil {
			target.secondaryExporters = append(target.secondaryExporters, expDep{i, CacheKeyWithSelector{CacheKey: *d.slowCacheKey}})
		}
		if d.result != nil {
			target.secondaryExporters = append(target.secondaryExporters, expDep{i, CacheKeyWithSelector{CacheKey: d.result.CacheKey(), Selector: src.cacheMap.Deps[i].Selector}})
		}
	}

	// TODO(tonistiigi): merge cache providers

	return true
}

// EdgeFactory allows access to the edges from a shared graph
type EdgeFactory interface {
	GetEdge(Edge) *edge
	SetEdge(Edge, *edge)
}

type pipeFactory struct {
	e *edge
	s *Scheduler
}

func (pf *pipeFactory) NewInputRequest(ee Edge, req *edgeRequest) pipe.Receiver {
	target := pf.s.ef.GetEdge(ee)
	if target == nil {
		panic("failed to get edge") // TODO: return errored pipe
	}
	p := pf.s.newPipe(target, pf.e, pipe.Request{Payload: req})
	if debugScheduler {
		logrus.Debugf("> newPipe %s %p desiredState=%s", ee.Vertex.Name(), p, req.desiredState)
	}
	return p.Receiver
}

func (pf *pipeFactory) NewFuncRequest(f func(context.Context) (interface{}, error)) pipe.Receiver {
	p := pf.s.newRequestWithFunc(pf.e, f)
	if debugScheduler {
		logrus.Debugf("> newFunc %p", p)
	}
	return p
}

func debugSchedulerPreUnpark(e *edge, inc []pipe.Sender, updates, allPipes []pipe.Receiver) {
	if !debugScheduler {
		return
	}
	logrus.Debugf(">> unpark %s req=%d upt=%d out=%d state=%s %s", e.edge.Vertex.Name(), len(inc), len(updates), len(allPipes), e.state, e.edge.Vertex.Digest())

	for i, dep := range e.deps {
		des := edgeStatusInitial
		if dep.req != nil {
			des = dep.req.Request().(*edgeRequest).desiredState
		}
		logrus.Debugf(":: dep%d %s state=%s des=%s keys=%s hasslowcache=%v", i, e.edge.Vertex.Inputs()[i].Vertex.Name(), dep.state, des, len(dep.keys), e.slowCacheFunc(dep) != nil)
	}

	for i, in := range inc {
		req := in.Request()
		logrus.Debugf("> incoming-%d: %p dstate=%s canceled=%v", i, in, req.Payload.(*edgeRequest).desiredState, req.Canceled)
	}

	for i, up := range updates {
		if up == e.cacheMapReq {
			logrus.Debugf("> update-%d: %p cacheMapReq complete=%v", i, up, up.Status().Completed)
		} else if up == e.execReq {
			logrus.Debugf("> update-%d: %p execReq complete=%v", i, up, up.Status().Completed)
		} else {
			st, ok := up.Status().Value.(*edgeState)
			if ok {
				index := -1
				if dep, ok := e.depRequests[up]; ok {
					index = int(dep.index)
				}
				logrus.Debugf("> update-%d: %p input-%d keys=%d state=%s", i, up, index, len(st.keys), st.state)
			} else {
				logrus.Debugf("> update-%d: unknown", i)
			}
		}
	}
}

func debugSchedulerPostUnpark(e *edge, inc []pipe.Sender) {
	if !debugScheduler {
		return
	}
	for i, in := range inc {
		logrus.Debugf("< incoming-%d: %p completed=%v", i, in, in.Status().Completed)
	}
	logrus.Debugf("<< unpark %s\n", e.edge.Vertex.Name())
}
<|endoftext|>


<commit_before>package orm

type Page struct {
	PageNo      int
	PageSize    int
	ResultCount int
	List        interface{}
}

func NewPage(pageNo, pageSize int) *Page {
	p := Page{}
	if pageNo == 0 {
		p.PageNo = 1
	} else {
		p.PageNo = pageNo
	}

	if pageSize == 0 {
		p.PageSize = 15
	} else {
		p.PageSize = pageSize
	}

	return &p
}

func (p *Page) StartRow() int {
	return p.PageSize*(p.PageNo-1) + 1
}

func (p *Page) PageCount() int {
	pageCount := 0
	if p.ResultCount%p.PageSize == 0 {
		pageCount = p.ResultCount / p.PageSize
	} else {
		pageCount = p.ResultCount/p.PageSize + 1
	}
	return pageCount
}
<commit_msg>delete List<commit_after>package orm

type Page struct {
	PageNo      int
	PageSize    int
	ResultCount int
}

func NewPage(pageNo, pageSize int) Page {
	p := Page{}
	if pageNo == 0 {
		p.PageNo = 1
	} else {
		p.PageNo = pageNo
	}

	if pageSize == 0 {
		p.PageSize = 15
	} else {
		p.PageSize = pageSize
	}

	return p
}

func (p *Page) StartRow() int {
	return p.PageSize*(p.PageNo-1) + 1
}

func (p *Page) PageCount() int {
	pageCount := 0
	if p.ResultCount%p.PageSize == 0 {
		pageCount = p.ResultCount / p.PageSize
	} else {
		pageCount = p.ResultCount/p.PageSize + 1
	}
	return pageCount
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"time"
	"bytes"
	"io/ioutil"
	"text/template"
	"path/filepath"
	"strings"
	blackfriday "github.com/russross/blackfriday"
	"sort"
)

type Page struct {
	Site *Site
	Config
	Content    string
	Path       string
	ModTime    time.Time
	RenderTime time.Time
}

type PageSlice []*Page

func NewPage(site *Site, path string) *Page {
	text, err := ioutil.ReadFile(path)
	errhandle(err)

	stat, err := os.Stat(path)
	errhandle(err)

	relpath, err := filepath.Rel(site.Path, path)
	errhandle(err)

	head, content := SplitHead(text)

	return &Page{
		Site:    site,
		Config:  *ParseConfig(head),
		Content: content,
		Path:    relpath,
		ModTime: stat.ModTime(),
	}
}

// Destination() returns relative path to a file in future published repository
func (page *Page) Destination() string {
	path := page.Path

	if strings.HasSuffix(path, ".html") {
		return path
	}

	if strings.HasSuffix(path, ".md") {
		path = strings.Replace(path, ".md", ".html", 1)
	}

	if filepath.Base(path) != "index.html" {
		path = strings.Replace(path, ".html", "/index.html", 1)
	}

	return path
}

func (page *Page) Url() string {
	return strings.Replace(page.Destination(), "/index.html", "/", 1)
}

func (page *Page) Rendered() []byte {
	ctmpl, err := template.New("ad-hoc").Parse(page.Content)
	errhandle(err)

	var buffer bytes.Buffer
	err = ctmpl.Execute(&buffer, page)
	errhandle(err)

	return blackfriday.MarkdownCommon(buffer.Bytes())
}

func SplitHead(text []byte) (string, string) {
	parts := bytes.SplitN(text, []byte("----\n"), 2)
	return string(parts[0]), string(parts[1])
}

// PageSlice manipulation

func (pages PageSlice) Get(i int) *Page   { return pages[i] }
func (pages PageSlice) First(i int) *Page { return pages.Get(0) }
func (pages PageSlice) Last(i int) *Page  { return pages.Get(len(pages) - 1) }

func (pages PageSlice) Len() int {
	return len(pages)
}
func (pages PageSlice) Less(i, j int) bool {
	return pages.Get(i).ModTime.Unix() < pages.Get(j).ModTime.Unix()
}
func (pages PageSlice) Swap(i, j int) {
	pages[i], pages[j] = pages[j], pages[i]
}

func (pages PageSlice) Sort() {
	sort.Sort(pages)
}

func (pages PageSlice) Children(root string) *PageSlice {
	children := make(PageSlice, 0)

	for _, page := range pages {
		if strings.HasPrefix(page.Url(), root) && page.Url() != root {
			children = append(children, page)
		}
	}

	return &children
}
<commit_msg>fix first/last<commit_after>package main

import (
	"os"
	"time"
	"bytes"
	"io/ioutil"
	"text/template"
	"path/filepath"
	"strings"
	blackfriday "github.com/russross/blackfriday"
	"sort"
)

type Page struct {
	Site *Site
	Config
	Content    string
	Path       string
	ModTime    time.Time
	RenderTime time.Time
}

type PageSlice []*Page

func NewPage(site *Site, path string) *Page {
	text, err := ioutil.ReadFile(path)
	errhandle(err)

	stat, err := os.Stat(path)
	errhandle(err)

	relpath, err := filepath.Rel(site.Path, path)
	errhandle(err)

	head, content := SplitHead(text)

	return &Page{
		Site:    site,
		Config:  *ParseConfig(head),
		Content: content,
		Path:    relpath,
		ModTime: stat.ModTime(),
	}
}

// Destination() returns relative path to a file in future published repository
func (page *Page) Destination() string {
	path := page.Path

	if strings.HasSuffix(path, ".html") {
		return path
	}

	if strings.HasSuffix(path, ".md") {
		path = strings.Replace(path, ".md", ".html", 1)
	}

	if filepath.Base(path) != "index.html" {
		path = strings.Replace(path, ".html", "/index.html", 1)
	}

	return path
}

func (page *Page) Url() string {
	return strings.Replace(page.Destination(), "/index.html", "/", 1)
}

func (page *Page) Rendered() []byte {
	ctmpl, err := template.New("ad-hoc").Parse(page.Content)
	errhandle(err)

	var buffer bytes.Buffer
	err = ctmpl.Execute(&buffer, page)
	errhandle(err)

	return blackfriday.MarkdownCommon(buffer.Bytes())
}

func SplitHead(text []byte) (string, string) {
	parts := bytes.SplitN(text, []byte("----\n"), 2)
	return string(parts[0]), string(parts[1])
}

// PageSlice manipulation

func (pages PageSlice) Get(i int) *Page { return pages[i] }
func (pages PageSlice) First() *Page    { return pages.Get(0) }
func (pages PageSlice) Last() *Page     { return pages.Get(len(pages) - 1) }

func (pages PageSlice) Len() int {
	return len(pages)
}
func (pages PageSlice) Less(i, j int) bool {
	return pages.Get(i).ModTime.Unix() < pages.Get(j).ModTime.Unix()
}
func (pages PageSlice) Swap(i, j int) {
	pages[i], pages[j] = pages[j], pages[i]
}

func (pages PageSlice) Sort() {
	sort.Sort(pages)
}

func (pages PageSlice) Children(root string) *PageSlice {
	children := make(PageSlice, 0)

	for _, page := range pages {
		if strings.HasPrefix(page.Url(), root) && page.Url() != root {
			children = append(children, page)
		}
	}

	return &children
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/chrisseto/pty"
	"github.com/nsf/termbox-go"
	"io"
	"os"
	"os/exec"
)

type Cell struct {
	termbox.Cell
	x, y int
}

type Pane struct {
	*exec.Cmd

	cx, cy        int
	sx, sy        int
	width, height uint16

	Prog string
	Args []string

	Pty          *os.File
	output       io.Reader
	cells        [][]termbox.Cell
	ShouldRedraw chan struct{}
}

func CreatePane(width, height uint16, prog string, args ...string) *Pane {
	return &Pane{
		Cmd: exec.Command(prog, args...),
		cx:  0, cy: 0,
		sx: 0, sy: 0,
		Prog: prog, Args: args,
		width: width, height: height,
		Pty: nil, ShouldRedraw: nil,
	}
}

func (p *Pane) Start() error {
	pterm, err := pty.Start(p.Cmd)
	if err != nil {
		panic(err)
	}
	if err = pty.Setsize(pterm, p.height, p.width); err != nil {
		panic(err)
	}
	p.Pty = pterm
	p.ShouldRedraw = make(chan struct{})
	p.cells = make([][]termbox.Cell, 1, p.height)
	p.cells[0] = make([]termbox.Cell, p.width)
	go p.outputPipe()
	return nil
}

func (p *Pane) Close() error {
	return p.Process.Kill()
}

func (p *Pane) Cells() [][]termbox.Cell {
	if len(p.cells) > int(p.height) {
		return p.cells[len(p.cells)-int(p.height):]
	}
	return p.cells
}

func (p *Pane) Width() uint16 {
	return p.width
}

func (p *Pane) Height() uint16 {
	return p.height
}

func (p *Pane) Redraw() {
	for y, line := range p.Cells() {
		for x, cell := range line {
			termbox.SetCell(x, y, cell.Ch, cell.Fg, cell.Bg)
		}
	}
}

func (p *Pane) outputPipe() {
	buf := make([]byte, 32*1024)

	for {
		nr, err := p.Pty.Read(buf)
		if nr > 0 {
			row := &p.cells[p.sy]

			for _, char := range buf[:nr] {
				switch char {
				case 0xA:
					p.sy++
					p.cells = append(p.cells, make([]termbox.Cell, p.width))
					row = &p.cells[p.sy]
				case 0xD:
					p.sx = 0
				case 0x8:
					p.sx--
					(*row)[p.sx] = termbox.Cell{' ', 0x0, 0x0}
				default:
					p.sx++
					(*row)[p.sx] = termbox.Cell{rune(char), 0x0, 0x0}
				}
			}

			p.ShouldRedraw <- struct{}{}
		}
		if err != nil {
			if err == io.EOF {
				break
			}
			panic(err)
		}

	}
	close(p.ShouldRedraw)
	p.ShouldRedraw = nil
}
<commit_msg>Dont close the redraw channel after command exits<commit_after>package main

import (
	"github.com/chrisseto/pty"
	"github.com/nsf/termbox-go"
	"io"
	"os"
	"os/exec"
)

type Cell struct {
	termbox.Cell
	x, y int
}

type Pane struct {
	*exec.Cmd

	cx, cy        int
	sx, sy        int
	width, height uint16

	Prog string
	Args []string

	Pty          *os.File
	output       io.Reader
	cells        [][]termbox.Cell
	ShouldRedraw chan struct{}
}

func CreatePane(width, height uint16, prog string, args ...string) *Pane {
	return &Pane{
		Cmd: exec.Command(prog, args...),
		cx:  0, cy: 0,
		sx: 0, sy: 0,
		Prog: prog, Args: args,
		width: width, height: height,
		Pty: nil, ShouldRedraw: nil,
	}
}

func (p *Pane) Start() error {
	pterm, err := pty.Start(p.Cmd)
	if err != nil {
		panic(err)
	}
	if err = pty.Setsize(pterm, p.height, p.width); err != nil {
		panic(err)
	}
	p.Pty = pterm
	p.ShouldRedraw = make(chan struct{})
	p.cells = make([][]termbox.Cell, 1, p.height)
	p.cells[0] = make([]termbox.Cell, p.width)
	go p.outputPipe()
	return nil
}

func (p *Pane) Close() error {
	return p.Process.Kill()
}

func (p *Pane) Cells() [][]termbox.Cell {
	if len(p.cells) > int(p.height) {
		return p.cells[len(p.cells)-int(p.height):]
	}
	return p.cells
}

func (p *Pane) Width() uint16 {
	return p.width
}

func (p *Pane) Height() uint16 {
	return p.height
}

func (p *Pane) Redraw() {
	for y, line := range p.Cells() {
		for x, cell := range line {
			termbox.SetCell(x, y, cell.Ch, cell.Fg, cell.Bg)
		}
	}
}

func (p *Pane) outputPipe() {
	buf := make([]byte, 32*1024)

	for {
		nr, err := p.Pty.Read(buf)
		if nr > 0 {
			row := &p.cells[p.sy]

			for _, char := range buf[:nr] {
				switch char {
				case 0xA:
					p.sy++
					p.cells = append(p.cells, make([]termbox.Cell, p.width))
					row = &p.cells[p.sy]
				case 0xD:
					p.sx = 0
				case 0x8:
					p.sx--
					(*row)[p.sx] = termbox.Cell{' ', 0x0, 0x0}
				default:
					p.sx++
					(*row)[p.sx] = termbox.Cell{rune(char), 0x0, 0x0}
				}
			}

			p.ShouldRedraw <- struct{}{}
		}
		if err != nil {
			if err == io.EOF {
				break
			}
			panic(err)
		}

	}
}
<|endoftext|>


<commit_before>package file

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/vektra/cypress"
	"github.com/vektra/neko"
	"github.com/vektra/tail"
)

func TestCLI(t *testing.T) {
	n := neko.Start(t)

	tmpdir, err := ioutil.TempDir("", "cli")
	require.NoError(t, err)

	defer os.RemoveAll(tmpdir)
	defer tail.Cleanup()

	n.It("generates a stream from lines in a file", func() {
		var buf bytes.Buffer

		file := filepath.Join(tmpdir, "blah.log")

		f, err := os.Create(file)
		require.NoError(t, err)

		defer f.Close()
		defer os.Remove(file)

		fmt.Fprintf(f, "from the file\n")

		f.Close()

		cli := &CLI{Once: true, output: &buf}

		err = cli.Execute([]string{file})
		require.NoError(t, err)

		dec, err := cypress.NewStreamDecoder(&buf)
		require.NoError(t, err)

		m, err := dec.Generate()
		require.NoError(t, err)

		src, ok := m.GetTag("source")
		require.True(t, ok)

		assert.Equal(t, "blah.log", src)

		msg, ok := m.Get("message")
		require.True(t, ok)

		assert.Equal(t, "from the file", msg)
	})

	n.It("uses the offsetdb if requested", func() {
		var buf bytes.Buffer

		file := filepath.Join(tmpdir, "blah.log")

		f, err := os.Create(file)
		require.NoError(t, err)

		defer f.Close()
		defer os.Remove(file)

		fmt.Fprintf(f, "from the file\n")

		dbpath := filepath.Join(tmpdir, "db")

		cli := &CLI{Once: true, DB: dbpath, output: &buf}

		err = cli.Execute([]string{file})
		require.NoError(t, err)

		dec, err := cypress.NewStreamDecoder(&buf)
		require.NoError(t, err)

		m, err := dec.Generate()
		require.NoError(t, err)

		src, ok := m.GetTag("source")
		require.True(t, ok)

		assert.Equal(t, "blah.log", src)

		msg, ok := m.Get("message")
		require.True(t, ok)

		assert.Equal(t, "from the file", msg)

		// add another line

		fmt.Fprint(f, "this is another line\n")

		cli = &CLI{Once: true, DB: dbpath, output: &buf}

		db, err := NewOffsetDB(dbpath)
		require.NoError(t, err)

		entry, err := db.Get(file)
		require.NoError(t, err)

		require.NoError(t, entry.CheckValid())
		require.True(t, entry.Offset > 0)

		err = cli.Execute([]string{file})
		require.NoError(t, err)

		m, err = dec.Generate()
		require.NoError(t, err)

		src, ok = m.GetTag("source")
		require.True(t, ok)

		assert.Equal(t, "blah.log", src)

		msg, ok = m.Get("message")
		require.True(t, ok)

		assert.Equal(t, "this is another line", msg)
	})

	n.Meow()
}
<commit_msg>More debugging to try and isolate the CI failure<commit_after>package file

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"syscall"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/vektra/cypress"
	"github.com/vektra/neko"
	"github.com/vektra/tail"
)

func TestCLI(t *testing.T) {
	n := neko.Start(t)

	tmpdir, err := ioutil.TempDir("", "cli")
	require.NoError(t, err)

	defer os.RemoveAll(tmpdir)
	defer tail.Cleanup()

	n.It("generates a stream from lines in a file", func() {
		var buf bytes.Buffer

		file := filepath.Join(tmpdir, "blah.log")

		f, err := os.Create(file)
		require.NoError(t, err)

		defer f.Close()
		defer os.Remove(file)

		fmt.Fprintf(f, "from the file\n")

		f.Close()

		cli := &CLI{Once: true, output: &buf}

		err = cli.Execute([]string{file})
		require.NoError(t, err)

		dec, err := cypress.NewStreamDecoder(&buf)
		require.NoError(t, err)

		m, err := dec.Generate()
		require.NoError(t, err)

		src, ok := m.GetTag("source")
		require.True(t, ok)

		assert.Equal(t, "blah.log", src)

		msg, ok := m.Get("message")
		require.True(t, ok)

		assert.Equal(t, "from the file", msg)
	})

	n.It("uses the offsetdb if requested", func() {
		var buf bytes.Buffer

		file := filepath.Join(tmpdir, "blah.log")

		f, err := os.Create(file)
		require.NoError(t, err)

		defer f.Close()
		defer os.Remove(file)

		stat, err := os.Stat(file)
		require.NoError(t, err)

		fmt.Printf("before: %#v\n", stat.Sys().(*syscall.Stat_t))

		fmt.Fprintf(f, "from the file\n")

		dbpath := filepath.Join(tmpdir, "db")

		cli := &CLI{Once: true, DB: dbpath, output: &buf}

		err = cli.Execute([]string{file})
		require.NoError(t, err)

		dec, err := cypress.NewStreamDecoder(&buf)
		require.NoError(t, err)

		m, err := dec.Generate()
		require.NoError(t, err)

		src, ok := m.GetTag("source")
		require.True(t, ok)

		assert.Equal(t, "blah.log", src)

		msg, ok := m.Get("message")
		require.True(t, ok)

		assert.Equal(t, "from the file", msg)

		// add another line

		fmt.Fprint(f, "this is another line\n")

		cli = &CLI{Once: true, DB: dbpath, output: &buf}

		db, err := NewOffsetDB(dbpath)
		require.NoError(t, err)

		entry, err := db.Get(file)
		require.NoError(t, err)

		stat, err = os.Stat(file)
		require.NoError(t, err)

		fmt.Printf("after: %#v\n", stat.Sys().(*syscall.Stat_t))

		require.NoError(t, entry.CheckValid())
		require.True(t, entry.Offset > 0)

		err = cli.Execute([]string{file})
		require.NoError(t, err)

		m, err = dec.Generate()
		require.NoError(t, err)

		src, ok = m.GetTag("source")
		require.True(t, ok)

		assert.Equal(t, "blah.log", src)

		msg, ok = m.Get("message")
		require.True(t, ok)

		assert.Equal(t, "this is another line", msg)
	})

	n.Meow()
}
<|endoftext|>


<commit_before>package main

import (
  "bytes"
  "compress/gzip"
  "compress/flate"
  "io/ioutil"
  // "crypto/aes"
  "crypto/rand"
  "encoding/json"
  "fmt"
  "code.google.com/p/go.crypto/scrypt"
)

func marshal(data *map[string]interface{}) []byte {
  result, err := json.Marshal(data)

  if err != nil {
    panic(fmt.Sprintf("Failed to marshall JSON data: %v", err))
  }

  return result
}

func unmarshal(data []byte) map[string]interface{} {
  result := make(map[string]interface{})
  err := json.Unmarshal(data, &result)

  if err != nil {
    panic(fmt.Sprintf("Failed to unmarshall JSON data: %v", err))
  }

  return result
}

// gzip some data
func compress(data []byte) []byte {
  var result bytes.Buffer
  writer, err := gzip.NewWriterLevel(&result, flate.BestCompression)

  if err != nil {
    panic(fmt.Sprintf("Failed to compress data: %v", err))
  }

  // compress our data
  writer.Write(data)
  writer.Close()

  return result.Bytes()
}

// gunzip some data
func decompress(data []byte) []byte {
  b := bytes.NewBuffer(data)
  reader, err := gzip.NewReader(b)

  if err != nil {
    panic(fmt.Sprintf("Failed to decompress data: %v", err))
  }

  // compress our data
  result, err := ioutil.ReadAll(reader)
  reader.Close()

  if err != nil {
    panic(fmt.Sprintf("Failed to read decompressed data: %v", err))
  }

  return result
}

// encrypt a data blob using our scheme
func encrypt(blob []byte, password string) []byte {
  return compress(blob)
}

// decrypt a data blob using our scheme
func decrypt(blob []byte, password string) []byte {
  return decompress(blob)
}

// given a password string, return the hashed variant
func hashPassword(password string) []byte {
  saltSize := 32

  // FIXME: handle salting better
  // get a random salt
  salt := make([]byte, saltSize)
  rand.Read(salt)

  // get the result and return it
  hash, _ := scrypt.Key([]byte(password), salt, 32768, 16, 2, 32)
  return hash
}

func main() {
  // some test database data
  data := make(map[string]interface{})
  data["hello"] = "world"
  data["foo"] = 1
  data["bar"] = 2.25
  data["baz"] = true
  data["pants"] = map[string]interface{} {
    "something_else": false,
    "bad": nil,
  }

  fmt.Printf("original: %#v\n\n", data)

  encryptedData := encrypt(marshal(&data), "password")

  fmt.Printf("encrypted: %#v\n\n", encryptedData)

  decryptedData := decrypt(encryptedData, "password")

  fmt.Printf("decrypted: %#v\n\n", decryptedData)

  decryptedJSON := unmarshal(decryptedData)

  fmt.Printf("new: %#v\n\n", decryptedJSON)

  fmt.Printf("hashedPassword: %#v\n\n", hashPassword("password"))
}
<commit_msg>Refactor<commit_after>package main

import (
  "bytes"
  "compress/gzip"
  "compress/flate"
  "io/ioutil"
  // "crypto/aes"
  "crypto/rand"
  "encoding/json"
  "fmt"
  "code.google.com/p/go.crypto/scrypt"
)

func dumpJSON(data *map[string]interface{}) []byte {
  result, err := json.Marshal(data)

  if err != nil {
    panic(fmt.Sprintf("Failed to marshall JSON data: %v", err))
  }

  return result
}

func loadJSON(data []byte) map[string]interface{} {
  result := make(map[string]interface{})
  err := json.Unmarshal(data, &result)

  if err != nil {
    panic(fmt.Sprintf("Failed to unmarshall JSON data: %v", err))
  }

  return result
}

// gzip some data
func compress(data []byte) []byte {
  var result bytes.Buffer
  writer, err := gzip.NewWriterLevel(&result, flate.BestCompression)

  if err != nil {
    panic(fmt.Sprintf("Failed to compress data: %v", err))
  }

  // compress our data
  writer.Write(data)
  writer.Close()

  return result.Bytes()
}

// gunzip some data
func decompress(data []byte) []byte {
  b := bytes.NewBuffer(data)
  reader, err := gzip.NewReader(b)

  if err != nil {
    panic(fmt.Sprintf("Failed to decompress data: %v", err))
  }

  // compress our data
  result, err := ioutil.ReadAll(reader)
  reader.Close()

  if err != nil {
    panic(fmt.Sprintf("Failed to read decompressed data: %v", err))
  }

  return result
}

// encrypt a data blob using our scheme
func encrypt(blob []byte, password string) []byte {
  return compress(blob)
}

// decrypt a data blob using our scheme
func decrypt(blob []byte, password string) []byte {
  return decompress(blob)
}

// given a password string, return the hashed variant
func hashPassword(password string) []byte {
  saltSize := 32

  // FIXME: handle salting better
  // get a random salt
  salt := make([]byte, saltSize)
  rand.Read(salt)

  // get the result and return it
  hash, _ := scrypt.Key([]byte(password), salt, 32768, 16, 2, 32)
  return hash
}

func main() {
  // some test database data
  data := make(map[string]interface{})
  data["hello"] = "world"
  data["foo"] = 1
  data["bar"] = 2.25
  data["baz"] = true
  data["pants"] = map[string]interface{} {
    "something_else": false,
    "bad": nil,
  }

  fmt.Printf("json: %#v\n\n", data)

  encryptedData := encrypt(dumpJSON(&data), "password")

  fmt.Printf("encrypted: %#v\n\n", encryptedData)

  decryptedData := decrypt(encryptedData, "password")

  fmt.Printf("decrypted: %#v\n\n", decryptedData)

  decryptedJSON := loadJSON(decryptedData)

  fmt.Printf("read json: %#v\n\n", decryptedJSON)

  fmt.Printf("hashedPassword: %#v\n\n", hashPassword("password"))
}
<|endoftext|>


<commit_before>package app

import (
	"bytes"
	"io/ioutil"
	"os"
	"path"
	"testing"
	"text/template"

	"github.com/stretchr/testify/assert"
)

func TestMain(t *testing.T) {
	tests := []struct {
		Args        []string
		ExpectError bool
	}{
		// No arguments
		{
			[]string{"foo"},
			true,
		},
	}
	for _, test := range tests {
		err := Main(test.Args, false)
		if test.ExpectError {
			assert.NoError(t, err)
		} else {
			assert.Error(t, err)
		}
	}
}

func TestGetFilehandles(t *testing.T) {
	tests := []struct {
		InputFilename  string
		OutputFilename string
		Pretty         bool
		ExpectedError  string
		ExpectedOutput map[string]string
	}{
		// Bad path for input
		{
			"foo", "bar",
			true,
			"open {{ .Dir }}/{{ .InputFilename }}: no such file or directory",
			map[string]string{},
		},
		// Bad path for output
		{
			"input_hello_world.json", "",
			true,
			"open {{ .Dir }}: is a directory",
			map[string]string{},
		},
		// Good paths
		{
			"input_hello_world.json", "output.json",
			true,
			"",
			map[string]string{
				"input_hello_world.json": `{ "hello": "world" }`,
				"output.json": `{
    "output": "was written here"
}` + "\n",
			},
		},
		// Again, with prettiness turned off
		{
			"input_hello_world.json", "output.json",
			false,
			"",
			map[string]string{
				"input_hello_world.json": `{ "hello": "world" }`,

				"output.json": `{"output":"was written here"}` + "\n",
			},
		},
	}
	for _, test := range tests {
		dir := setupTestDir(t)
		defer removeTestDir(t, dir)

		experr, err := tmplExecute(test.ExpectedError, dir, struct {
			Dir, InputFilename, OutputFilename string
		}{dir, test.InputFilename, test.OutputFilename})
		if err != nil {
			t.Fatal(err)
		}
		test.ExpectedError = experr

		input, output, err := getFilehandles(
			path.Join(dir, test.InputFilename),
			path.Join(dir, test.OutputFilename),
			test.Pretty,
		)
		if test.ExpectedError == "" {
			if assert.NoError(t, err) {
				assert.NoError(t, output.Write(map[string]interface{}{
					"output": "was written here",
				}))
			}
		} else {
			if assert.Error(t, err) {
				assert.Equal(t, test.ExpectedError, err.Error())
			}
			assert.Equal(t, nil, input)
			assert.Equal(t, nil, output)
		}

		for filename, content := range test.ExpectedOutput {
			assertFileHasContent(t, path.Join(dir, filename), content)
		}
	}
}

func tmplExecute(pattern, dir string, data interface{}) (string, error) {
	tmpl, err := template.New(dir).Parse(pattern)
	if err != nil {
		return "", err
	}
	buf := new(bytes.Buffer)
	err = tmpl.Execute(buf, data)
	return buf.String(), err

}

func assertFileHasContent(t *testing.T, filename, expected_content string) {
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		t.Fatal(err)
	}
	assert.Equal(t, expected_content, string(content))
}

func setupTestDir(t *testing.T) string {
	dir, err := ioutil.TempDir("", "djconvert-tests-")
	if err != nil {
		t.Fatal(err)
	}
	// Set up files
	files := map[string]string{
		"input_hello_world.json": `{ "hello": "world" }`,
	}
	for filename, content := range files {
		fullpath := path.Join(dir, filename)
		err = ioutil.WriteFile(fullpath, []byte(content), 0600)
		if err != nil {
			t.Fatal(err)
		}
	}
	return dir
}

func removeTestDir(t *testing.T, dir string) {
	err := os.RemoveAll(dir)
	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>#63 Finish up coverage for Main()<commit_after>package app

import (
	"bytes"
	"io/ioutil"
	"os"
	"path"
	"testing"
	"text/template"

	"github.com/stretchr/testify/assert"
)

func TestMain(t *testing.T) {
	tests := []struct {
		Args           []string
		ExpectError    bool
		ExpectedOutput map[string]string
	}{
		// No arguments
		{
			[]string{},
			true,
			map[string]string{},
		},
		// Bad arguments
		{
			[]string{"foo"},
			true,
			map[string]string{},
		},
		// Insufficient arguments
		{
			[]string{"up"},
			true,
			map[string]string{},
		},
		// Bad file locations
		{
			[]string{"up", "{{ .Dir }}", "."},
			true,
			map[string]string{},
		},
		// Up not pretty
		{
			[]string{"up",
				"{{ .Dir }}/input_hello_world.json",
				"{{ .Dir }}/doc.json",
			},
			false,
			map[string]string{"doc.json": `{"events":{"aa582b4df04ba01af5205e702d4d16ed0b2c0705":{"parent":"","handler":"SET","args":{"path":[],"value":{"hello":"world"}}}},"timestamps":["aa582b4df04ba01af5205e702d4d16ed0b2c0705"]}` + "\n"},
		},
		// Up pretty
		{
			[]string{"up",
				"{{ .Dir }}/input_hello_world.json",
				"{{ .Dir }}/doc.json",
				"--pretty",
			},
			false,
			map[string]string{"doc.json": `{
    "events": {
        "aa582b4df04ba01af5205e702d4d16ed0b2c0705": {
            "parent": "",
            "handler": "SET",
            "args": {
                "path": [],
                "value": {
                    "hello": "world"
                }
            }
        }
    },
    "timestamps": [
        "aa582b4df04ba01af5205e702d4d16ed0b2c0705"
    ]
}` + "\n"},
		},
		// Down not pretty
		{
			[]string{"down",
				"{{ .Dir }}/doc_hello_world.json",
				"{{ .Dir }}/static.json",
				"aa58",
			},
			false,
			map[string]string{"static.json": `{"hello":"world"}` + "\n"},
		},
		// Down pretty
		{
			[]string{"down",
				"{{ .Dir }}/doc_hello_world.json",
				"{{ .Dir }}/static.json",
				"aa58",
				"--pretty",
			},
			false,
			map[string]string{"static.json": `{
    "hello": "world"
}` + "\n"},
		},
	}
	for _, test := range tests {
		dir := setupTestDir(t)
		defer removeTestDir(t, dir)

		for i, arg := range test.Args {
			arg_transformed, err := tmplExecute(arg, dir,
				struct{ Dir string }{dir})
			if err != nil {
				t.Fatal(err)
			}
			test.Args[i] = arg_transformed
		}

		err := Main(test.Args, false)
		if test.ExpectError {
			assert.Error(t, err)
		} else {
			assert.NoError(t, err)
		}

		for filename, content := range test.ExpectedOutput {
			assertFileHasContent(t, path.Join(dir, filename), content)
		}
	}
}

func TestGetFilehandles(t *testing.T) {
	tests := []struct {
		InputFilename  string
		OutputFilename string
		Pretty         bool
		ExpectedError  string
		ExpectedOutput map[string]string
	}{
		// Bad path for input
		{
			"foo", "bar",
			true,
			"open {{ .Dir }}/{{ .InputFilename }}: no such file or directory",
			map[string]string{},
		},
		// Bad path for output
		{
			"input_hello_world.json", "",
			true,
			"open {{ .Dir }}: is a directory",
			map[string]string{},
		},
		// Good paths
		{
			"input_hello_world.json", "output.json",
			true,
			"",
			map[string]string{
				"input_hello_world.json": `{ "hello": "world" }`,
				"output.json": `{
    "output": "was written here"
}` + "\n",
			},
		},
		// Again, with prettiness turned off
		{
			"input_hello_world.json", "output.json",
			false,
			"",
			map[string]string{
				"input_hello_world.json": `{ "hello": "world" }`,

				"output.json": `{"output":"was written here"}` + "\n",
			},
		},
	}
	for _, test := range tests {
		dir := setupTestDir(t)
		defer removeTestDir(t, dir)

		experr, err := tmplExecute(test.ExpectedError, dir, struct {
			Dir, InputFilename, OutputFilename string
		}{dir, test.InputFilename, test.OutputFilename})
		if err != nil {
			t.Fatal(err)
		}
		test.ExpectedError = experr

		input, output, err := getFilehandles(
			path.Join(dir, test.InputFilename),
			path.Join(dir, test.OutputFilename),
			test.Pretty,
		)
		if test.ExpectedError == "" {
			if assert.NoError(t, err) {
				assert.NoError(t, output.Write(map[string]interface{}{
					"output": "was written here",
				}))
			}
		} else {
			if assert.Error(t, err) {
				assert.Equal(t, test.ExpectedError, err.Error())
			}
			assert.Equal(t, nil, input)
			assert.Equal(t, nil, output)
		}

		for filename, content := range test.ExpectedOutput {
			assertFileHasContent(t, path.Join(dir, filename), content)
		}
	}
}

func tmplExecute(pattern, dir string, data interface{}) (string, error) {
	tmpl, err := template.New(dir).Parse(pattern)
	if err != nil {
		return "", err
	}
	buf := new(bytes.Buffer)
	err = tmpl.Execute(buf, data)
	return buf.String(), err

}

func assertFileHasContent(t *testing.T, filename, expected_content string) {
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		t.Fatal(err)
	}
	assert.Equal(t, expected_content, string(content))
}

func setupTestDir(t *testing.T) string {
	dir, err := ioutil.TempDir("", "djconvert-tests-")
	if err != nil {
		t.Fatal(err)
	}
	// Set up files
	files := map[string]string{
		"input_hello_world.json": `{ "hello": "world" }`,
		"doc_hello_world.json":   `{"events":{"aa582b4df04ba01af5205e702d4d16ed0b2c0705":{"parent":"","handler":"SET","args":{"path":[],"value":{"hello":"world"}}}},"timestamps":["aa582b4df04ba01af5205e702d4d16ed0b2c0705"]}` + "\n",
	}
	for filename, content := range files {
		fullpath := path.Join(dir, filename)
		err = ioutil.WriteFile(fullpath, []byte(content), 0600)
		if err != nil {
			t.Fatal(err)
		}
	}
	return dir
}

func removeTestDir(t *testing.T, dir string) {
	err := os.RemoveAll(dir)
	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package cmd

import (
	"fmt"
	"os"
	"strings"

	cliHandler "github.com/TheThingsNetwork/ttn/utils/cli/handler"
	"github.com/apex/log"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var cfgFile string

var ctx log.Interface

// RootCmd is the entrypoint for handlerctl
var RootCmd = &cobra.Command{
	Use:   "ttnctl",
	Short: "Control The Things Network from the command line",
	Long:  `ttnctl controls The Things Network from the command line.`,
	PersistentPreRun: func(cmd *cobra.Command, args []string) {
		var logLevel = log.InfoLevel
		if viper.GetBool("debug") {
			logLevel = log.DebugLevel
		}
		ctx = &log.Logger{
			Level:   logLevel,
			Handler: cliHandler.New(os.Stdout),
		}
	},
}

// Execute runs on start
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

// init initializes the configuration and command line flags
func init() {
	cobra.OnInitialize(initConfig)

	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ttnctl.yaml)")

	RootCmd.PersistentFlags().String("ttn-router", "staging.thethingsnetwork.org:1700", "The net address of the TTN Router")
	viper.BindPFlag("ttn-router", RootCmd.PersistentFlags().Lookup("ttn-router"))

	RootCmd.PersistentFlags().String("ttn-handler", "staging.thethingsnetwork.org:1782", "The net address of the TTN Handler")
	viper.BindPFlag("ttn-handler", RootCmd.PersistentFlags().Lookup("ttn-handler"))

	RootCmd.PersistentFlags().String("mqtt-broker", "staging.thethingsnetwork.org:1883", "The address of the MQTT broker")
	viper.BindPFlag("mqtt-broker", RootCmd.PersistentFlags().Lookup("mqtt-broker"))

	RootCmd.PersistentFlags().String("app-eui", "0102030405060708", "The app EUI to use")
	viper.BindPFlag("app-eui", RootCmd.PersistentFlags().Lookup("app-eui"))

	RootCmd.PersistentFlags().String("ttn-account-server", "https://account.thethingsnetwork.org", "The address of the OAuth 2.0 server")
	viper.BindPFlag("ttn-account-server", RootCmd.PersistentFlags().Lookup("ttn-account-server"))
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	if cfgFile != "" {
		viper.SetConfigFile(cfgFile)
	}

	viper.SetConfigName(".ttnctl")
	viper.AddConfigPath("$HOME")
	viper.SetEnvPrefix("ttnctl") // set environment prefix
	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))
	viper.AutomaticEnv()

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err == nil {
		fmt.Println("Using config file:", viper.ConfigFileUsed())
	}
}
<commit_msg>[ttnctl] Only print config file info when DEBUG=true<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package cmd

import (
	"fmt"
	"os"
	"strings"

	cliHandler "github.com/TheThingsNetwork/ttn/utils/cli/handler"
	"github.com/apex/log"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var cfgFile string

var ctx log.Interface

// RootCmd is the entrypoint for handlerctl
var RootCmd = &cobra.Command{
	Use:   "ttnctl",
	Short: "Control The Things Network from the command line",
	Long:  `ttnctl controls The Things Network from the command line.`,
	PersistentPreRun: func(cmd *cobra.Command, args []string) {
		var logLevel = log.InfoLevel
		if viper.GetBool("debug") {
			logLevel = log.DebugLevel
		}
		ctx = &log.Logger{
			Level:   logLevel,
			Handler: cliHandler.New(os.Stdout),
		}
	},
}

// Execute runs on start
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

// init initializes the configuration and command line flags
func init() {
	cobra.OnInitialize(initConfig)

	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ttnctl.yaml)")

	RootCmd.PersistentFlags().String("ttn-router", "staging.thethingsnetwork.org:1700", "The net address of the TTN Router")
	viper.BindPFlag("ttn-router", RootCmd.PersistentFlags().Lookup("ttn-router"))

	RootCmd.PersistentFlags().String("ttn-handler", "staging.thethingsnetwork.org:1782", "The net address of the TTN Handler")
	viper.BindPFlag("ttn-handler", RootCmd.PersistentFlags().Lookup("ttn-handler"))

	RootCmd.PersistentFlags().String("mqtt-broker", "staging.thethingsnetwork.org:1883", "The address of the MQTT broker")
	viper.BindPFlag("mqtt-broker", RootCmd.PersistentFlags().Lookup("mqtt-broker"))

	RootCmd.PersistentFlags().String("app-eui", "0102030405060708", "The app EUI to use")
	viper.BindPFlag("app-eui", RootCmd.PersistentFlags().Lookup("app-eui"))

	RootCmd.PersistentFlags().String("ttn-account-server", "https://account.thethingsnetwork.org", "The address of the OAuth 2.0 server")
	viper.BindPFlag("ttn-account-server", RootCmd.PersistentFlags().Lookup("ttn-account-server"))
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	if cfgFile != "" {
		viper.SetConfigFile(cfgFile)
	}

	viper.SetConfigName(".ttnctl")
	viper.AddConfigPath("$HOME")
	viper.SetEnvPrefix("ttnctl") // set environment prefix
	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))
	viper.AutomaticEnv()

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err == nil && viper.GetBool("debug") {
		fmt.Println("Using config file:", viper.ConfigFileUsed())
	}
}
<|endoftext|>