<commit_before>// ipgrep scans one or more input files for valid IPv4 or IPv6 addresses and
// prints the result.
package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"unicode"

	"sync"

	"github.com/fatih/color"
	"github.com/mattn/go-colorable"
)

const (
	prog       = "ipgrep"
	banlistURL = `https://www.binarydefense.com/banlist.txt`
)

const usage = `
usage: %[1]v [file ...]

%[1]v scans one or more input files for valid IPv4 or IPv6 addresses and prints
the result. It accepts text files in any format (newline-delimited, JSON, YAML,
etc.) so long as the files contain IPs separated either by whitespace or by any
punctuation character other than '.' or ':'.

For example, these are all valid input:

	10.10.10.2 https://webserver.com
	{"ip": "172.16.2.84"}
	log -> time=13:10, event=foo, addr=192.168.0.2, desc="a foo went bar"
	IP address 8.8.8.8 is for Google DNS.

ipgrep would extract 10.10.10.2, 172.16.2.84, 192.168.0.2, and 8.8.8.8 from the
above. However, given this input —

	There’s no place like 127.0.0.1.

— ipgrep extracts nothing: The final '.' renders the address invalid, and this
utility doesn’t try quite that hard.
`

// scanResult stores the results of processing a single input file.
type scanResult struct {
	File string   // path to the input file.
	IPs  []net.IP // list of IPs parsed from the file.
	Err  error    // set if an I/O error occurs or the file is empty.
}

// Error satisfies the error interface.
func (r scanResult) Error() string {
	if r.Err == nil {
		return ""
	}
	return fmt.Sprintf("%v: %v", r.File, r.Err)
}

func main() {
	if len(os.Args) < 2 {
		help()
	}
	switch os.Args[1] {
	case "-h", "-help", "--help":
		help()
	}

	// If any of the input files cannot be read, quit with an error.
	var files []*os.File
	for _, fn := range os.Args[1:] {
		fp, err := os.Open(fn)
		if err != nil {
			die(err)
		}
		files = append(files, fp)
	}

	var (
		results = make(chan *scanResult, len(files))
		wg      sync.WaitGroup
	)
	// Create a goroutine to scan and parse each input file, collecting the
	// results in a channel.
	for _, fp := range files {
		wg.Add(1)
		go func(fp *os.File) {
			defer wg.Done()
			defer fp.Close()
			results <- scan(fp)
		}(fp)
	}
	wg.Wait()
	close(results)

	var failed []*scanResult
	for r := range results {
		// Show successfully extracted IPs first; display errors later.
		if r.Err != nil {
			failed = append(failed, r)
			continue
		}
		fmt.Printf("# results for %v:\n", r.File)
		for _, ip := range r.IPs {
			fmt.Println(ip)
		}
		fmt.Println()
	}
	if len(failed) > 0 {
		fmt.Println("# errors:")
		for _, r := range failed {
			printError(r)
		}
	}
}

// split is used to divide file content into “words” that might be valid IP
// addresses.
func split(r rune) bool {
	if unicode.IsSpace(r) || unicode.IsPunct(r) && r != '.' && r != ':' {
		return true
	}
	return false
}

// scan reads a file, splits its content in “words,” and tests each word to see
// if it is a valid IPv4 or IPv6 address. If reading the file causes an I/O
// error, or if the file is empty, *scanResult will have a non-nil Err field.
func scan(fp *os.File) *scanResult {
	var (
		res = &scanResult{File: fp.Name()}
		b   []byte
	)
	if b, res.Err = ioutil.ReadAll(fp); res.Err != nil {
		return res
	}
	if len(b) == 0 {
		res.Err = fmt.Errorf("empty file")
		return res
	}
	for _, word := range bytes.FieldsFunc(b, split) {
		if ip := net.ParseIP(string(word)); ip != nil {
			res.IPs = append(res.IPs, ip)
		}
	}
	return res
}

func help() {
	fmt.Fprintf(os.Stderr, usage, prog)
	os.Exit(0)
}

func printError(errMsg interface{}) {
	var (
		stderr = colorable.NewColorableStderr()
		red    = color.New(color.FgRed).SprintfFunc()
	)
	fmt.Fprintf(stderr, red("\n%v: error: %v\n", prog, errMsg))
}

func die(errMsg interface{}) {
	printError(errMsg)
	os.Exit(1)
}
<commit_msg>Edit comments and remove unused var<commit_after>// ipgrep scans one or more input files for valid IPv4 or IPv6 addresses and
// prints the result.
package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"unicode"

	"sync"

	"github.com/fatih/color"
	"github.com/mattn/go-colorable"
)

const prog = "ipgrep"

const usage = `
usage: %[1]v [file ...]

%[1]v scans one or more input files for valid IPv4 or IPv6 addresses and prints
the result. It accepts text files in any format (newline-delimited, JSON, YAML,
etc.) so long as the files contain IPs separated either by whitespace or by any
punctuation character other than '.' or ':'.

For example, these are all valid input:

	10.10.10.2 https://webserver.com
	{"ip": "172.16.2.84"}
	log -> time=13:10, event=foo, addr=192.168.0.2, desc="a foo went bar"
	IP address 8.8.8.8 is for Google DNS.

ipgrep would extract 10.10.10.2, 172.16.2.84, 192.168.0.2, and 8.8.8.8 from the
above. However, given this input —

	There’s no place like 127.0.0.1.

— ipgrep extracts nothing: The final '.' renders the address invalid, and this
utility doesn’t try quite that hard.
`

// scanResult stores the results of processing a single input file.
type scanResult struct {
	File string   // path to the input file.
	IPs  []net.IP // list of IPs parsed from the file.
	Err  error    // set if an I/O error occurs or the file is empty.
}

// Error satisfies the error interface.
func (r scanResult) Error() string {
	if r.Err == nil {
		return ""
	}
	return fmt.Sprintf("%v: %v", r.File, r.Err)
}

func main() {
	if len(os.Args) < 2 {
		help()
	}
	switch os.Args[1] {
	case "-h", "-help", "--help":
		help()
	}

	// If any of the input files cannot be read, quit with an error.
	var files []*os.File
	for _, fn := range os.Args[1:] {
		fp, err := os.Open(fn)
		if err != nil {
			die(err)
		}
		files = append(files, fp)
	}

	var (
		results = make(chan *scanResult, len(files))
		wg      sync.WaitGroup
	)
	// Create a goroutine to scan and parse each input file, collecting the
	// results in a channel.
	for _, fp := range files {
		wg.Add(1)
		// TODO(princebot): Best practice would be to cap the number of
		// goroutines this can launch, but this is just a simple little
		// tool, and anyone calling it with hundreds of input files
		// should prooooobably already be looking to fork/rewrite this
		// if he wants it to be performant.
		go func(fp *os.File) {
			results <- scan(fp)
			fp.Close()
			wg.Done()
		}(fp)
	}
	wg.Wait()
	close(results)

	var failed []*scanResult
	for r := range results {
		// Show successfully extracted IPs first; display errors later.
		if r.Err != nil {
			failed = append(failed, r)
			continue
		}
		fmt.Printf("# results for %v:\n", r.File)
		for _, ip := range r.IPs {
			fmt.Println(ip)
		}
		fmt.Println()
	}
	if len(failed) > 0 {
		fmt.Println("# errors:")
		for _, r := range failed {
			printError(r)
		}
	}
}

// split is used to divide file content into “words” that might be valid IP
// addresses.
func split(r rune) bool {
	if unicode.IsSpace(r) || unicode.IsPunct(r) && r != '.' && r != ':' {
		return true
	}
	return false
}

// scan reads a file, splits its content in “words,” and tests each word to see
// if it is a valid IPv4 or IPv6 address. If reading the file causes an I/O
// error, or if the file is empty, *scanResult will have a non-nil Err field.
func scan(fp *os.File) *scanResult {
	var (
		res = &scanResult{File: fp.Name()}
		b   []byte
	)
	if b, res.Err = ioutil.ReadAll(fp); res.Err != nil {
		return res
	}
	if len(b) == 0 {
		res.Err = fmt.Errorf("empty file")
		return res
	}
	for _, word := range bytes.FieldsFunc(b, split) {
		if ip := net.ParseIP(string(word)); ip != nil {
			res.IPs = append(res.IPs, ip)
		}
	}
	return res
}

func help() {
	fmt.Fprintf(os.Stderr, usage, prog)
	os.Exit(0)
}

func printError(errMsg interface{}) {
	var (
		stderr = colorable.NewColorableStderr()
		red    = color.New(color.FgRed).SprintfFunc()
	)
	fmt.Fprintf(stderr, red("\n%v: error: %v\n", prog, errMsg))
}

func die(errMsg interface{}) {
	printError(errMsg)
	os.Exit(1)
}
<|endoftext|>


<commit_before>// +build linux

package libcontainer

import (
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"

	"github.com/docker/libcontainer/configs"
	"github.com/docker/libcontainer/namespaces"
	"github.com/docker/libcontainer/network"
	"github.com/golang/glog"
)

type linuxContainer struct {
	id            string
	root          string
	config        *configs.Config
	state         *configs.State
	cgroupManager CgroupManager
	initArgs      []string
}

func (c *linuxContainer) ID() string {
	return c.id
}

func (c *linuxContainer) Config() *configs.Config {
	return c.config
}

func (c *linuxContainer) RunState() (configs.RunState, error) {
	return configs.Destroyed, nil // FIXME return a real state
}

func (c *linuxContainer) Processes() ([]int, error) {
	glog.Info("fetch container processes")
	pids, err := c.cgroupManager.GetPids()
	if err != nil {
		return nil, newGenericError(err, SystemError)
	}
	return pids, nil
}

func (c *linuxContainer) Stats() (*ContainerStats, error) {
	glog.Info("fetch container stats")
	var (
		err   error
		stats = &ContainerStats{}
	)

	if stats.CgroupStats, err = c.cgroupManager.GetStats(); err != nil {
		return stats, newGenericError(err, SystemError)
	}
	if stats.NetworkStats, err = network.GetStats(&c.state.NetworkState); err != nil {
		return stats, newGenericError(err, SystemError)
	}
	return stats, nil
}

func (c *linuxContainer) StartProcess(pconfig *ProcessConfig) (int, error) {
	state, err := c.RunState()
	if err != nil {
		return -1, err
	}

	if state != configs.Destroyed {
		glog.Info("start new container process")
		panic("not implemented")
	}

	if err := c.startInitProcess(pconfig); err != nil {
		return -1, err
	}

	return c.state.InitPid, nil
}

func (c *linuxContainer) updateStateFile() error {
	data, err := json.MarshalIndent(c.state, "", "\t")
	if err != nil {
		return newGenericError(err, SystemError)
	}

	fnew := filepath.Join(c.root, fmt.Sprintf("%s.new", stateFilename))
	f, err := os.Create(fnew)
	if err != nil {
		return newGenericError(err, SystemError)
	}

	_, err = f.Write(data)
	if err != nil {
		f.Close()
		return newGenericError(err, SystemError)
	}
	f.Close()

	fname := filepath.Join(c.root, stateFilename)
	if err := os.Rename(fnew, fname); err != nil {
		return newGenericError(err, SystemError)
	}

	return nil
}

func (c *linuxContainer) startInitProcess(config *ProcessConfig) error {
	cmd := exec.Command(c.initArgs[0], append(c.initArgs[1:], config.Args...)...)
	cmd.Stdin = config.Stdin
	cmd.Stdout = config.Stdout
	cmd.Stderr = config.Stderr

	cmd.Env = config.Env
	cmd.Dir = c.config.RootFs

	if cmd.SysProcAttr == nil {
		cmd.SysProcAttr = &syscall.SysProcAttr{}
	}

	cmd.SysProcAttr.Cloneflags = uintptr(namespaces.GetNamespaceFlags(c.config.Namespaces))
	cmd.SysProcAttr.Pdeathsig = syscall.SIGKILL

	err := namespaces.Exec(config.Args, config.Env, cmd, c.config, c.state)
	if err != nil {
		return err
	}

	err = c.updateStateFile()
	if err != nil {
		// FIXME c.Kill()
		return err
	}

	return nil
}

func (c *linuxContainer) Destroy() error {
	state, err := c.RunState()
	if err != nil {
		return err
	}

	if state != configs.Destroyed {
		return newGenericError(nil, ContainerNotStopped)
	}

	os.RemoveAll(c.root)
	return nil
}

func (c *linuxContainer) Pause() error {
	glog.Info("pause container")
	panic("not implemented")
}

func (c *linuxContainer) Resume() error {
	glog.Info("resume container")
	panic("not implemented")
}

func (c *linuxContainer) Signal(pid, signal int) error {
	glog.Infof("sending signal %d to pid %d", signal, pid)
	panic("not implemented")
}

func (c *linuxContainer) Wait() (int, error) {
	glog.Info("wait container")
	panic("not implemented")
}

func (c *linuxContainer) WaitProcess(pid int) (int, error) {
	glog.Infof("wait process %d", pid)
	panic("not implemented")
}
<commit_msg>libcontainer: optimize updateStateFile()<commit_after>// +build linux

package libcontainer

import (
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"

	"github.com/docker/libcontainer/configs"
	"github.com/docker/libcontainer/namespaces"
	"github.com/docker/libcontainer/network"
	"github.com/golang/glog"
)

type linuxContainer struct {
	id            string
	root          string
	config        *configs.Config
	state         *configs.State
	cgroupManager CgroupManager
	initArgs      []string
}

func (c *linuxContainer) ID() string {
	return c.id
}

func (c *linuxContainer) Config() *configs.Config {
	return c.config
}

func (c *linuxContainer) RunState() (configs.RunState, error) {
	return configs.Destroyed, nil // FIXME return a real state
}

func (c *linuxContainer) Processes() ([]int, error) {
	glog.Info("fetch container processes")
	pids, err := c.cgroupManager.GetPids()
	if err != nil {
		return nil, newGenericError(err, SystemError)
	}
	return pids, nil
}

func (c *linuxContainer) Stats() (*ContainerStats, error) {
	glog.Info("fetch container stats")
	var (
		err   error
		stats = &ContainerStats{}
	)

	if stats.CgroupStats, err = c.cgroupManager.GetStats(); err != nil {
		return stats, newGenericError(err, SystemError)
	}
	if stats.NetworkStats, err = network.GetStats(&c.state.NetworkState); err != nil {
		return stats, newGenericError(err, SystemError)
	}
	return stats, nil
}

func (c *linuxContainer) StartProcess(pconfig *ProcessConfig) (int, error) {
	state, err := c.RunState()
	if err != nil {
		return -1, err
	}

	if state != configs.Destroyed {
		glog.Info("start new container process")
		panic("not implemented")
	}

	if err := c.startInitProcess(pconfig); err != nil {
		return -1, err
	}

	return c.state.InitPid, nil
}

func (c *linuxContainer) updateStateFile() error {
	fnew := filepath.Join(c.root, fmt.Sprintf("%s.new", stateFilename))
	f, err := os.Create(fnew)
	if err != nil {
		return newGenericError(err, SystemError)
	}

	err = json.NewEncoder(f).Encode(c.state)
	if err != nil {
		f.Close()
		os.Remove(fnew)
		return newGenericError(err, SystemError)
	}
	f.Close()

	fname := filepath.Join(c.root, stateFilename)
	if err := os.Rename(fnew, fname); err != nil {
		return newGenericError(err, SystemError)
	}

	return nil
}

func (c *linuxContainer) startInitProcess(config *ProcessConfig) error {
	cmd := exec.Command(c.initArgs[0], append(c.initArgs[1:], config.Args...)...)
	cmd.Stdin = config.Stdin
	cmd.Stdout = config.Stdout
	cmd.Stderr = config.Stderr

	cmd.Env = config.Env
	cmd.Dir = c.config.RootFs

	if cmd.SysProcAttr == nil {
		cmd.SysProcAttr = &syscall.SysProcAttr{}
	}

	cmd.SysProcAttr.Cloneflags = uintptr(namespaces.GetNamespaceFlags(c.config.Namespaces))
	cmd.SysProcAttr.Pdeathsig = syscall.SIGKILL

	err := namespaces.Exec(config.Args, config.Env, cmd, c.config, c.state)
	if err != nil {
		return err
	}

	err = c.updateStateFile()
	if err != nil {
		// FIXME c.Kill()
		return err
	}

	return nil
}

func (c *linuxContainer) Destroy() error {
	state, err := c.RunState()
	if err != nil {
		return err
	}

	if state != configs.Destroyed {
		return newGenericError(nil, ContainerNotStopped)
	}

	os.RemoveAll(c.root)
	return nil
}

func (c *linuxContainer) Pause() error {
	glog.Info("pause container")
	panic("not implemented")
}

func (c *linuxContainer) Resume() error {
	glog.Info("resume container")
	panic("not implemented")
}

func (c *linuxContainer) Signal(pid, signal int) error {
	glog.Infof("sending signal %d to pid %d", signal, pid)
	panic("not implemented")
}

func (c *linuxContainer) Wait() (int, error) {
	glog.Info("wait container")
	panic("not implemented")
}

func (c *linuxContainer) WaitProcess(pid int) (int, error) {
	glog.Infof("wait process %d", pid)
	panic("not implemented")
}
<|endoftext|>


<commit_before>package class

import (
	"encoding/binary"
	"fmt"
	"io"
)

var byteOrder = binary.BigEndian

var initFuncs = []func(*ClassFile, io.Reader) error{
	(*ClassFile).readMagic,
	(*ClassFile).readVersion,
	(*ClassFile).readConstPool,
	(*ClassFile).readAccessFlags,
	(*ClassFile).readThisClass,
	(*ClassFile).readSuperClass,
	(*ClassFile).readInterfaces,
	(*ClassFile).readFields,
	(*ClassFile).readMethods,
	(*ClassFile).readAttributes,
}

func Parse(r io.Reader) (*ClassFile, error) {
	c := &ClassFile{}

	var err error

	for _, f := range initFuncs {
		err = f(c, r)
		if err != nil {
			return nil, err
		}
	}

	return c, nil
}

func (c *ClassFile) readMagic(r io.Reader) error {
	return binary.Read(r, byteOrder, &c.Magic)
}

func (c *ClassFile) readVersion(r io.Reader) error {
	err := binary.Read(r, byteOrder, &c.MinorVersion)
	if err != nil {
		return err
	}

	return binary.Read(r, byteOrder, &c.MajorVersion)
}

func (c *ClassFile) readConstPool(r io.Reader) error {
	err := binary.Read(r, byteOrder, &c.ConstPoolSize)
	if err != nil {
		return err
	}

	c.ConstPool = make([]*ConstInfo, 0, c.ConstPoolSize)

	for i := uint16(1); i < c.ConstPoolSize; i++ {
		info, err := c.readConstInfo(r)
		if err != nil {
			return err
		}

		c.ConstPool = append(c.ConstPool, info)
	}

	return nil
}

func (c *ClassFile) readConstInfo(r io.Reader) (*ConstInfo, error) {
	info := &ConstInfo{}

	err := binary.Read(r, byteOrder, &info.Tag)
	if err != nil {
		return nil, err
	}

	bytesToRead := uint16(0)
	switch info.Tag {
	case CONSTANT_Class, CONSTANT_String, CONSTANT_MethodType:
		bytesToRead = 2

	case CONSTANT_MethodHandle:
		bytesToRead = 3

	case CONSTANT_FieldRef, CONSTANT_MethodRef,
		CONSTANT_InterfaceMethodRef, CONSTANT_Integer,
		CONSTANT_Float, CONSTANT_NameAndType, CONSTANT_InvokeDynamic:
		bytesToRead = 4

	case CONSTANT_Long, CONSTANT_Double:
		bytesToRead = 8

	case CONSTANT_UTF8:
		err = binary.Read(r, byteOrder, &bytesToRead)
		if err != nil {
			return nil, err
		}

	default:
		fmt.Println(info.Tag)
		panic("unknown tag in class file!")
	}

	info.Info = make([]uint8, bytesToRead)

	err = binary.Read(r, byteOrder, &info.Info)
	if err != nil {
		return nil, err
	}

	return info, nil
}

func (c *ClassFile) readAccessFlags(r io.Reader) error {
	return binary.Read(r, byteOrder, &c.AccessFlags)
}

func (c *ClassFile) readThisClass(r io.Reader) error {
	return binary.Read(r, byteOrder, &c.ThisClass)
}

func (c *ClassFile) readSuperClass(r io.Reader) error {
	return binary.Read(r, byteOrder, &c.SuperClass)
}

func (c *ClassFile) readInterfaces(r io.Reader) error {
	err := binary.Read(r, byteOrder, &c.InterfacesCount)
	if err != nil {
		return err
	}

	c.Interfaces = make([]ConstPoolIndex, 0, c.InterfacesCount)

	var idx ConstPoolIndex
	for i := uint16(0); i < c.InterfacesCount; i++ {
		err = binary.Read(r, byteOrder, &idx)
		if err != nil {
			return err
		}

		c.Interfaces = append(c.Interfaces, idx)
	}

	return nil
}

func (c *ClassFile) readFields(r io.Reader) error {
	err := binary.Read(r, byteOrder, &c.FieldsCount)
	if err != nil {
		return err
	}

	c.Fields = make([]*FieldInfo, 0, c.FieldsCount)

	for i := uint16(0); i < c.FieldsCount; i++ {
		var access FieldAccessFlag
		err := binary.Read(r, byteOrder, &access)
		if err != nil {
			return err
		}

		fieldOrMethod, err := c.readFieldOrMethod(r)
		if err != nil {
			return err
		}

		field := &FieldInfo{access, *fieldOrMethod}
		c.Fields = append(c.Fields, field)
	}

	return nil
}

func (c *ClassFile) readMethods(r io.Reader) error {
	err := binary.Read(r, byteOrder, &c.MethodsCount)
	if err != nil {
		return err
	}

	c.Methods = make([]*MethodInfo, 0, c.MethodsCount)

	for i := uint16(0); i < c.MethodsCount; i++ {
		var access MethodAccessFlag
		err := binary.Read(r, byteOrder, &access)
		if err != nil {
			return err
		}

		fieldOrMethod, err := c.readFieldOrMethod(r)
		if err != nil {
			return err
		}

		method := &MethodInfo{access, *fieldOrMethod}
		c.Methods = append(c.Methods, method)
	}

	return nil
}

func (c *ClassFile) readFieldOrMethod(r io.Reader) (*fieldOrMethodInfo, error) {
	fom := &fieldOrMethodInfo{}

	errs := []error{
		binary.Read(r, byteOrder, &fom.NameIndex),
		binary.Read(r, byteOrder, &fom.DescriptorIndex),
		binary.Read(r, byteOrder, &fom.AttributesCount),
	}

	for _, err := range errs {
		if err != nil {
			return nil, err
		}
	}

	fom.Attributes = make([]*AttributeInfo, 0, fom.AttributesCount)

	for i := uint16(0); i < fom.AttributesCount; i++ {
		attr, err := c.readAttribute(r)
		if err != nil {
			return nil, err
		}

		fom.Attributes = append(fom.Attributes, attr)
	}

	return fom, nil
}

func (c *ClassFile) readAttributes(r io.Reader) error {
	err := binary.Read(r, byteOrder, &c.AttributesCount)
	if err != nil {
		return err
	}

	c.Attributes = make([]*AttributeInfo, 0, c.AttributesCount)

	var attr *AttributeInfo
	for i := uint16(0); i < c.AttributesCount; i++ {
		attr, err = c.readAttribute(r)
		if err != nil {
			return err
		}

		c.Attributes = append(c.Attributes, attr)
	}

	return nil
}

func (c *ClassFile) readAttribute(r io.Reader) (*AttributeInfo, error) {
	attr := &AttributeInfo{}

	errs := []error{
		binary.Read(r, byteOrder, &attr.NameIndex),
		binary.Read(r, byteOrder, &attr.Length),
	}

	for _, err := range errs {
		if err != nil {
			return nil, err
		}
	}

	attr.Info = make([]uint8, attr.Length)

	err := binary.Read(r, byteOrder, &attr.Info)
	if err != nil {
		return nil, err
	}

	return attr, nil
}
<commit_msg>WORST BUG EVER fixed. Sometimes (when long or doubles were used) the const pool parser would trip up.<commit_after>package class

import (
	"encoding/binary"
	"fmt"
	"io"
)

var byteOrder = binary.BigEndian

var initFuncs = []func(*ClassFile, io.Reader) error{
	(*ClassFile).readMagic,
	(*ClassFile).readVersion,
	(*ClassFile).readConstPool,
	(*ClassFile).readAccessFlags,
	(*ClassFile).readThisClass,
	(*ClassFile).readSuperClass,
	(*ClassFile).readInterfaces,
	(*ClassFile).readFields,
	(*ClassFile).readMethods,
	(*ClassFile).readAttributes,
}

func Parse(r io.Reader) (*ClassFile, error) {
	c := &ClassFile{}

	var err error

	for _, f := range initFuncs {
		err = f(c, r)
		if err != nil {
			return nil, err
		}
	}

	return c, nil
}

func (c *ClassFile) readMagic(r io.Reader) error {
	return binary.Read(r, byteOrder, &c.Magic)
}

func (c *ClassFile) readVersion(r io.Reader) error {
	err := binary.Read(r, byteOrder, &c.MinorVersion)
	if err != nil {
		return err
	}

	return binary.Read(r, byteOrder, &c.MajorVersion)
}

func (c *ClassFile) readConstPool(r io.Reader) error {
	err := binary.Read(r, byteOrder, &c.ConstPoolSize)
	if err != nil {
		return err
	}

	c.ConstPool = make([]*ConstInfo, 0, c.ConstPoolSize)

	for i := uint16(1); i < c.ConstPoolSize; i++ {
		info, err := c.readConstInfo(r)
		if err != nil {
			return err
		}

		// This is one of the WORST! bugs ever!
		// They even admit it in the JVM spec:
		//
		// "In retrospect, making 8-byte constants
		// take two constant pool entries was a poor choice."
		//
		// The problem is, that both longs & doubles
		// take up TWO!! slots in the const pool (an it
		// looks like they take 2 in the local variable
		// pool too). That's why we need to advance an
		// extra slot, iff we encounter one of them
		if info.Tag == CONSTANT_Long || info.Tag == CONSTANT_Double {
			i += 1
		}

		c.ConstPool = append(c.ConstPool, info)
	}

	return nil
}

func (c *ClassFile) readConstInfo(r io.Reader) (*ConstInfo, error) {
	info := &ConstInfo{}

	err := binary.Read(r, byteOrder, &info.Tag)
	if err != nil {
		return nil, err
	}

	bytesToRead := uint16(0)
	switch info.Tag {
	case CONSTANT_Class, CONSTANT_String, CONSTANT_MethodType:
		bytesToRead = 2

	case CONSTANT_MethodHandle:
		bytesToRead = 3

	case CONSTANT_FieldRef, CONSTANT_MethodRef,
		CONSTANT_InterfaceMethodRef, CONSTANT_Integer,
		CONSTANT_Float, CONSTANT_NameAndType, CONSTANT_InvokeDynamic:
		bytesToRead = 4

	case CONSTANT_Long, CONSTANT_Double:
		bytesToRead = 8

	case CONSTANT_UTF8:
		err = binary.Read(r, byteOrder, &bytesToRead)
		if err != nil {
			return nil, err
		}

	default:
		panic("unknown tag in class file!")
	}

	info.Info = make([]uint8, bytesToRead)

	err = binary.Read(r, byteOrder, &info.Info)
	if err != nil {
		return nil, err
	}

	return info, nil
}

func (c *ClassFile) readAccessFlags(r io.Reader) error {
	return binary.Read(r, byteOrder, &c.AccessFlags)
}

func (c *ClassFile) readThisClass(r io.Reader) error {
	return binary.Read(r, byteOrder, &c.ThisClass)
}

func (c *ClassFile) readSuperClass(r io.Reader) error {
	return binary.Read(r, byteOrder, &c.SuperClass)
}

func (c *ClassFile) readInterfaces(r io.Reader) error {
	err := binary.Read(r, byteOrder, &c.InterfacesCount)
	if err != nil {
		return err
	}

	c.Interfaces = make([]ConstPoolIndex, 0, c.InterfacesCount)

	var idx ConstPoolIndex
	for i := uint16(0); i < c.InterfacesCount; i++ {
		err = binary.Read(r, byteOrder, &idx)
		if err != nil {
			return err
		}

		c.Interfaces = append(c.Interfaces, idx)
	}

	return nil
}

func (c *ClassFile) readFields(r io.Reader) error {
	err := binary.Read(r, byteOrder, &c.FieldsCount)
	if err != nil {
		return err
	}

	c.Fields = make([]*FieldInfo, 0, c.FieldsCount)

	for i := uint16(0); i < c.FieldsCount; i++ {
		var access FieldAccessFlag
		err := binary.Read(r, byteOrder, &access)
		if err != nil {
			return err
		}

		fieldOrMethod, err := c.readFieldOrMethod(r)
		if err != nil {
			return err
		}

		field := &FieldInfo{access, *fieldOrMethod}
		c.Fields = append(c.Fields, field)
	}

	return nil
}

func (c *ClassFile) readMethods(r io.Reader) error {
	err := binary.Read(r, byteOrder, &c.MethodsCount)
	if err != nil {
		return err
	}

	c.Methods = make([]*MethodInfo, 0, c.MethodsCount)

	for i := uint16(0); i < c.MethodsCount; i++ {
		var access MethodAccessFlag
		err := binary.Read(r, byteOrder, &access)
		if err != nil {
			return err
		}

		fieldOrMethod, err := c.readFieldOrMethod(r)
		if err != nil {
			return err
		}

		method := &MethodInfo{access, *fieldOrMethod}
		c.Methods = append(c.Methods, method)
	}

	return nil
}

func (c *ClassFile) readFieldOrMethod(r io.Reader) (*fieldOrMethodInfo, error) {
	fom := &fieldOrMethodInfo{}

	errs := []error{
		binary.Read(r, byteOrder, &fom.NameIndex),
		binary.Read(r, byteOrder, &fom.DescriptorIndex),
		binary.Read(r, byteOrder, &fom.AttributesCount),
	}

	for _, err := range errs {
		if err != nil {
			return nil, err
		}
	}

	fom.Attributes = make([]*AttributeInfo, 0, fom.AttributesCount)

	for i := uint16(0); i < fom.AttributesCount; i++ {
		attr, err := c.readAttribute(r)
		if err != nil {
			return nil, err
		}

		fom.Attributes = append(fom.Attributes, attr)
	}

	return fom, nil
}

func (c *ClassFile) readAttributes(r io.Reader) error {
	err := binary.Read(r, byteOrder, &c.AttributesCount)
	if err != nil {
		return err
	}

	c.Attributes = make([]*AttributeInfo, 0, c.AttributesCount)

	var attr *AttributeInfo
	for i := uint16(0); i < c.AttributesCount; i++ {
		attr, err = c.readAttribute(r)
		if err != nil {
			return err
		}

		c.Attributes = append(c.Attributes, attr)
	}

	return nil
}

func (c *ClassFile) readAttribute(r io.Reader) (*AttributeInfo, error) {
	attr := &AttributeInfo{}

	errs := []error{
		binary.Read(r, byteOrder, &attr.NameIndex),
		binary.Read(r, byteOrder, &attr.Length),
	}

	for _, err := range errs {
		if err != nil {
			return nil, err
		}
	}

	attr.Info = make([]uint8, attr.Length)

	err := binary.Read(r, byteOrder, &attr.Info)
	if err != nil {
		return nil, err
	}

	return attr, nil
}
<|endoftext|>


<commit_before>package jobhub

import (
	"fmt"
	"os"
	"os/exec"
	"syscall"
	"time"

	"github.com/sirupsen/logrus"
)

var nextPipelineID = 1

type Pipeline struct {
	Name string
	Log  logrus.FieldLogger

	id              int
	nextJobID       int
	jobContainer    []Job
	jobByID         map[int]Job
	jobDependency   map[int][]int
	startingJob     map[int]bool
	recursionLevels map[int]int
}

type Job struct {
	Name string
	Path string

	pipelineID int
	id         int
}

type PipelineStatus struct {
	PipelineName string
	Status       ExecutionStatus
	JobStatus    []JobStatus
}

type JobStatus struct {
	Job        Job
	JobID      int
	LastStatus ExecutionStatus
	Statuses   []ExecutionStatus
}

type ExecutionStatus struct {
	ExecutionStart time.Time
	Code           StatusCode
	Runtime        time.Duration
}

type StatusCode int

const (
	Created   StatusCode = 0
	Scheduled StatusCode = 1
	Failed    StatusCode = 2
	Succeeded StatusCode = 3
)

var statusDescription = map[StatusCode]string{
	Created:   "Created",
	Scheduled: "Scheduled",
	Failed:    "Failed",
	Succeeded: "Succeeded",
}

func (j Job) String() string {
	return fmt.Sprintf("Pipeline ID: %d | ID: %d | Name: %s", j.pipelineID, j.id, j.Name)
}

func (sCode StatusCode) String() string {
	return statusDescription[sCode]
}

func (jStatus JobStatus) LastExecutionStatus() *ExecutionStatus {
	if len(jStatus.Statuses) > 0 {
		return &jStatus.Statuses[len(jStatus.Statuses)-1]
	}
	return nil
}

func NewPipeline() *Pipeline {
	return &Pipeline{
		Log:             logrus.StandardLogger(),
		jobByID:         make(map[int]Job),
		jobDependency:   make(map[int][]int),
		startingJob:     make(map[int]bool),
		recursionLevels: make(map[int]int),
	}
}

func nextIDPipeline() int {
	tempID := nextPipelineID
	nextPipelineID++
	return tempID
}

func (p *Pipeline) nextIDJob() int {
	p.nextJobID++
	return p.nextJobID
}

func (p *Pipeline) AddJob(job Job) Job {
	for _, j := range p.jobContainer {
		if j.id == job.id {
			p.Log.Panicf("Pipeline [%d][%s] | Job [%d][%s] has already been added",
				p.id, p.Name, job.id, job.Name)
		}
	}
	if p.id == 0 {
		p.id = nextIDPipeline()
	}
	job.pipelineID = p.id
	job.id = p.nextIDJob()
	p.jobContainer = append(p.jobContainer, job)
	p.jobByID[job.id] = job
	p.startingJob[job.id] = true
	return job
}

func (p *Pipeline) AddJobDependency(job Job, deps ...Job) {
	var tempContainer []Job
	tempContainer = append(deps, job)
	if p.id == 0 {
		p.Log.Panicf("Pipeline [%d][%s] | Pipeline not initialized",
			p.id, p.Name)
	}
	if p.jobContainer == nil {
		p.Log.Panicf("Pipeline [%d][%s] | Job Container is empty",
			p.id, p.Name)
	}
	for _, givenJob := range tempContainer {
		if givenJob.pipelineID != p.id {
			p.Log.Panicf("Pipeline [%d][%s] | Job [%d][%s] does not belong to this pipeline",
				p.id, p.Name, givenJob.id, givenJob.Name)
		}
	}
	for _, d := range deps {
		p.jobDependency[job.id] = append(p.jobDependency[job.id], d.id)
		delete(p.startingJob, d.id)
	}
}

func (p *Pipeline) resolveDependencyRecursion(jobID, level int) {
	if l := p.recursionLevels[jobID]; l >= level {
		return
	}
	p.recursionLevels[jobID] = level
	level++
	for _, depID := range p.jobDependency[jobID] {
		p.resolveDependencyRecursion(depID, level)
	}
}

func (p *Pipeline) resolveDependency() []int {
	//TODO(amwolff) switch to sort.SortSlice
	var queue []int
	tempRL := make(map[int]int)
	for startID, _ := range p.startingJob {
		p.resolveDependencyRecursion(startID, 1)
	}
	for jID, l := range p.recursionLevels {
		tempRL[jID] = l
	}
	for len(tempRL) > 0 {
		var jobID int
		max := 0
		for jID, l := range tempRL {
			if l > max {
				max = l
				jobID = jID
			}
		}
		queue = append(queue, jobID)
		delete(tempRL, jobID)
	}
	return queue
}

func (p *Pipeline) runJob(job Job) (ExecutionStatus, error) {
	ret := ExecutionStatus{ExecutionStart: time.Now().UTC()}
	_, err := os.Stat(job.Path)
	if err != nil {
		ret.Code = Failed
		return ret, err
	}
	process := exec.Command(job.Path)
	start := time.Now()
	err = process.Run()
	ret.Runtime = time.Since(start)
	if err != nil {
		exitError, ok := err.(*exec.ExitError)
		if !ok {
			p.Log.Errorf("Cannot cast to exitError: %s", err)
		}
		p.Log.Error(exitError.Sys().(syscall.WaitStatus))
		ret.Code = Failed
		return ret, err
	}
	ret.Code = Succeeded
	return ret, err
}

func (p *Pipeline) Run() PipelineStatus {
	queue := p.resolveDependency()
	if queue == nil {
		p.Log.Panicf("Pipeline [%d][%s] | No jobs in queue", p.id, p.Name)
	}
	pipelineStatus := PipelineStatus{JobStatus: make([]JobStatus, len(queue))}
	pipelineStatus.PipelineName = p.Name
	for i, jID := range queue {
		pipelineStatus.JobStatus[i].Job = p.jobByID[jID]
		pipelineStatus.JobStatus[i].JobID = jID
		pipelineStatus.JobStatus[i].LastStatus.Code = Scheduled
	}
	pipelineStatus.Status.ExecutionStart = time.Now().UTC()
	for i, jID := range queue {
		executionStatus, err := p.runJob(p.jobByID[jID])
		pipelineStatus.JobStatus[i].Statuses = append(pipelineStatus.JobStatus[i].Statuses, executionStatus)
		pipelineStatus.JobStatus[i].LastStatus = *pipelineStatus.JobStatus[i].LastExecutionStatus()
		pipelineStatus.Status.Runtime += executionStatus.Runtime
		if err != nil {
			p.Log.Errorf("Pipeline [%d][%s] | Job [%d][%s][Runtime: %v][StatusCode: %d] | %s",
				p.id, p.Name, jID, p.jobByID[jID].Name, executionStatus.Runtime, executionStatus.Code, err)
			pipelineStatus.Status.Code = Failed
			return pipelineStatus
		}
	}
	pipelineStatus.Status.Code = Succeeded
	return pipelineStatus
}
<commit_msg>Passing arguments list to executed binaries. (#12)<commit_after>package jobhub

import (
	"fmt"
	"os"
	"os/exec"
	"syscall"
	"time"

	"github.com/sirupsen/logrus"
)

var nextPipelineID = 1

type Pipeline struct {
	Name string
	Log  logrus.FieldLogger

	id              int
	nextJobID       int
	jobContainer    []Job
	jobByID         map[int]Job
	jobDependency   map[int][]int
	startingJob     map[int]bool
	recursionLevels map[int]int
}

type Job struct {
	Name string
	Path string
	Args []string

	pipelineID int
	id         int
}

type PipelineStatus struct {
	PipelineName string
	Status       ExecutionStatus
	JobStatus    []JobStatus
}

type JobStatus struct {
	Job        Job
	JobID      int
	LastStatus ExecutionStatus
	Statuses   []ExecutionStatus
}

type ExecutionStatus struct {
	ExecutionStart time.Time
	Code           StatusCode
	Runtime        time.Duration
}

type StatusCode int

const (
	Created   StatusCode = 0
	Scheduled StatusCode = 1
	Failed    StatusCode = 2
	Succeeded StatusCode = 3
)

var statusDescription = map[StatusCode]string{
	Created:   "Created",
	Scheduled: "Scheduled",
	Failed:    "Failed",
	Succeeded: "Succeeded",
}

func (j Job) String() string {
	return fmt.Sprintf("Pipeline ID: %d | ID: %d | Name: %s", j.pipelineID, j.id, j.Name)
}

func (sCode StatusCode) String() string {
	return statusDescription[sCode]
}

func (jStatus JobStatus) LastExecutionStatus() *ExecutionStatus {
	if len(jStatus.Statuses) > 0 {
		return &jStatus.Statuses[len(jStatus.Statuses)-1]
	}
	return nil
}

func NewPipeline() *Pipeline {
	return &Pipeline{
		Log:             logrus.StandardLogger(),
		jobByID:         make(map[int]Job),
		jobDependency:   make(map[int][]int),
		startingJob:     make(map[int]bool),
		recursionLevels: make(map[int]int),
	}
}

func nextIDPipeline() int {
	tempID := nextPipelineID
	nextPipelineID++
	return tempID
}

func (p *Pipeline) nextIDJob() int {
	p.nextJobID++
	return p.nextJobID
}

func (p *Pipeline) AddJob(job Job) Job {
	for _, j := range p.jobContainer {
		if j.id == job.id {
			p.Log.Panicf("Pipeline [%d][%s] | Job [%d][%s] has already been added",
				p.id, p.Name, job.id, job.Name)
		}
	}
	if p.id == 0 {
		p.id = nextIDPipeline()
	}
	job.pipelineID = p.id
	job.id = p.nextIDJob()
	p.jobContainer = append(p.jobContainer, job)
	p.jobByID[job.id] = job
	p.startingJob[job.id] = true
	return job
}

func (p *Pipeline) AddJobDependency(job Job, deps ...Job) {
	var tempContainer []Job
	tempContainer = append(deps, job)
	if p.id == 0 {
		p.Log.Panicf("Pipeline [%d][%s] | Pipeline not initialized",
			p.id, p.Name)
	}
	if p.jobContainer == nil {
		p.Log.Panicf("Pipeline [%d][%s] | Job Container is empty",
			p.id, p.Name)
	}
	for _, givenJob := range tempContainer {
		if givenJob.pipelineID != p.id {
			p.Log.Panicf("Pipeline [%d][%s] | Job [%d][%s] does not belong to this pipeline",
				p.id, p.Name, givenJob.id, givenJob.Name)
		}
	}
	for _, d := range deps {
		p.jobDependency[job.id] = append(p.jobDependency[job.id], d.id)
		delete(p.startingJob, d.id)
	}
}

func (p *Pipeline) resolveDependencyRecursion(jobID, level int) {
	if l := p.recursionLevels[jobID]; l >= level {
		return
	}
	p.recursionLevels[jobID] = level
	level++
	for _, depID := range p.jobDependency[jobID] {
		p.resolveDependencyRecursion(depID, level)
	}
}

func (p *Pipeline) resolveDependency() []int {
	//TODO(amwolff) switch to sort.SortSlice
	var queue []int
	tempRL := make(map[int]int)
	for startID, _ := range p.startingJob {
		p.resolveDependencyRecursion(startID, 1)
	}
	for jID, l := range p.recursionLevels {
		tempRL[jID] = l
	}
	for len(tempRL) > 0 {
		var jobID int
		max := 0
		for jID, l := range tempRL {
			if l > max {
				max = l
				jobID = jID
			}
		}
		queue = append(queue, jobID)
		delete(tempRL, jobID)
	}
	return queue
}

func (p *Pipeline) runJob(job Job) (ExecutionStatus, error) {
	ret := ExecutionStatus{ExecutionStart: time.Now().UTC()}
	_, err := os.Stat(job.Path)
	if err != nil {
		ret.Code = Failed
		return ret, err
	}
	process := exec.Command(job.Path, job.Args...)
	start := time.Now()
	err = process.Run()
	ret.Runtime = time.Since(start)
	if err != nil {
		exitError, ok := err.(*exec.ExitError)
		if !ok {
			p.Log.Errorf("Cannot cast to exitError (%s)", err)
		}
		p.Log.Error(exitError.Sys().(syscall.WaitStatus))
		ret.Code = Failed
		return ret, err
	}
	ret.Code = Succeeded
	return ret, err
}

func (p *Pipeline) Run() PipelineStatus {
	queue := p.resolveDependency()
	if queue == nil {
		p.Log.Panicf("Pipeline [%d][%s] | No jobs in queue", p.id, p.Name)
	}
	pipelineStatus := PipelineStatus{JobStatus: make([]JobStatus, len(queue))}
	pipelineStatus.PipelineName = p.Name
	for i, jID := range queue {
		pipelineStatus.JobStatus[i].Job = p.jobByID[jID]
		pipelineStatus.JobStatus[i].JobID = jID
		pipelineStatus.JobStatus[i].LastStatus.Code = Scheduled
	}
	pipelineStatus.Status.ExecutionStart = time.Now().UTC()
	for i, jID := range queue {
		executionStatus, err := p.runJob(p.jobByID[jID])
		pipelineStatus.JobStatus[i].Statuses = append(pipelineStatus.JobStatus[i].Statuses, executionStatus)
		pipelineStatus.JobStatus[i].LastStatus = *pipelineStatus.JobStatus[i].LastExecutionStatus()
		pipelineStatus.Status.Runtime += executionStatus.Runtime
		if err != nil {
			p.Log.Errorf("Pipeline [%d][%s] | Job [%d][%s][Runtime: %v][StatusCode: %d] | %s",
				p.id, p.Name, jID, p.jobByID[jID].Name, executionStatus.Runtime, executionStatus.Code, err)
			pipelineStatus.Status.Code = Failed
			return pipelineStatus
		}
	}
	pipelineStatus.Status.Code = Succeeded
	return pipelineStatus
}
<|endoftext|>


<commit_before>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2016 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package js

import (
	log "github.com/Sirupsen/logrus"
	"github.com/loadimpact/k6/lib"
	"github.com/loadimpact/k6/stats"
	"github.com/robertkrimen/otto"
	"strconv"
	"sync/atomic"
	"time"
)

type JSAPI struct {
	vu *VU
}

func (a JSAPI) Sleep(secs float64) {
	d := time.Duration(secs * float64(time.Second))
	t := time.NewTimer(d)
	select {
	case <-t.C:
	case <-a.vu.ctx.Done():
	}
	t.Stop()
}

func (a JSAPI) Log(level int, msg string, args []otto.Value) {
	fields := make(log.Fields, len(args))
	for i, arg := range args {
		if arg.IsObject() {
			obj := arg.Object()
			for _, key := range obj.Keys() {
				v, err := obj.Get(key)
				if err != nil {
					throw(a.vu.vm, err)
				}
				fields[key] = v.String()
			}
			continue
		}
		fields["arg"+strconv.Itoa(i)] = arg.String()
	}

	entry := log.WithFields(fields)
	switch level {
	case 0:
		entry.Debug(msg)
	case 1:
		entry.Info(msg)
	case 2:
		entry.Warn(msg)
	case 3:
		entry.Error(msg)
	}
}

func (a JSAPI) DoGroup(call otto.FunctionCall) otto.Value {
	name := call.Argument(0).String()
	group, err := a.vu.group.Group(name)
	if err != nil {
		throw(call.Otto, err)
	}
	a.vu.group = group
	defer func() { a.vu.group = group.Parent }()

	fn := call.Argument(1)
	if !fn.IsFunction() {
		panic(call.Otto.MakeSyntaxError("fn must be a function"))
	}

	val, err := fn.Call(call.This)
	if err != nil {
		throw(call.Otto, err)
	}

	if val.IsUndefined() {
		return otto.TrueValue()
	}
	return val
}

func (a JSAPI) DoCheck(call otto.FunctionCall) otto.Value {
	if len(call.ArgumentList) < 2 {
		return otto.UndefinedValue()
	}

	t := time.Now()
	samples := make([]stats.Sample, len(call.ArgumentList)-1)

	success := true
	arg0 := call.Argument(0)
	for i, v := range call.ArgumentList[1:] {
		obj := v.Object()
		if obj == nil {
			panic(call.Otto.MakeTypeError("checks must be objects"))
		}
		for _, name := range obj.Keys() {
			val, err := obj.Get(name)
			if err != nil {
				throw(call.Otto, err)
			}

			result, err := Check(val, arg0)
			if err != nil {
				throw(call.Otto, err)
			}

			check, err := a.vu.group.Check(name)
			if err != nil {
				throw(call.Otto, err)
			}

			sampleValue := 1.0
			if result {
				atomic.AddInt64(&(check.Passes), 1)
			} else {
				atomic.AddInt64(&(check.Fails), 1)
				success = false
				sampleValue = 0.0
			}

			samples[i] = stats.Sample{
				Time:   t,
				Metric: lib.MetricChecks,
				Tags: map[string]string{
					"group_id": check.Group.ID,
					"check_id": check.ID,
					"path":     check.Path, // Included for human readability.
				},
				Value: sampleValue,
			}
		}
	}

	a.vu.Samples = append(a.vu.Samples, samples...)

	if !success {
		a.vu.Taint = true
		return otto.FalseValue()
	}
	return otto.TrueValue()
}

func (a JSAPI) Taint() {
	a.vu.Taint = true
}

func (a JSAPI) ElapsedMs() float64 {
	return float64(time.Since(a.vu.started)) / float64(time.Millisecond)
}
<commit_msg>[fix] Only the last try in a check got emitted<commit_after>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2016 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package js

import (
	log "github.com/Sirupsen/logrus"
	"github.com/loadimpact/k6/lib"
	"github.com/loadimpact/k6/stats"
	"github.com/robertkrimen/otto"
	"strconv"
	"sync/atomic"
	"time"
)

type JSAPI struct {
	vu *VU
}

func (a JSAPI) Sleep(secs float64) {
	d := time.Duration(secs * float64(time.Second))
	t := time.NewTimer(d)
	select {
	case <-t.C:
	case <-a.vu.ctx.Done():
	}
	t.Stop()
}

func (a JSAPI) Log(level int, msg string, args []otto.Value) {
	fields := make(log.Fields, len(args))
	for i, arg := range args {
		if arg.IsObject() {
			obj := arg.Object()
			for _, key := range obj.Keys() {
				v, err := obj.Get(key)
				if err != nil {
					throw(a.vu.vm, err)
				}
				fields[key] = v.String()
			}
			continue
		}
		fields["arg"+strconv.Itoa(i)] = arg.String()
	}

	entry := log.WithFields(fields)
	switch level {
	case 0:
		entry.Debug(msg)
	case 1:
		entry.Info(msg)
	case 2:
		entry.Warn(msg)
	case 3:
		entry.Error(msg)
	}
}

func (a JSAPI) DoGroup(call otto.FunctionCall) otto.Value {
	name := call.Argument(0).String()
	group, err := a.vu.group.Group(name)
	if err != nil {
		throw(call.Otto, err)
	}
	a.vu.group = group
	defer func() { a.vu.group = group.Parent }()

	fn := call.Argument(1)
	if !fn.IsFunction() {
		panic(call.Otto.MakeSyntaxError("fn must be a function"))
	}

	val, err := fn.Call(call.This)
	if err != nil {
		throw(call.Otto, err)
	}

	if val.IsUndefined() {
		return otto.TrueValue()
	}
	return val
}

func (a JSAPI) DoCheck(call otto.FunctionCall) otto.Value {
	if len(call.ArgumentList) < 2 {
		return otto.UndefinedValue()
	}

	t := time.Now()

	success := true
	arg0 := call.Argument(0)
	for _, v := range call.ArgumentList[1:] {
		samples := make([]stats.Sample, len(call.ArgumentList)-1)
		obj := v.Object()
		if obj == nil {
			panic(call.Otto.MakeTypeError("checks must be objects"))
		}
		for i, name := range obj.Keys() {
			val, err := obj.Get(name)
			if err != nil {
				throw(call.Otto, err)
			}

			result, err := Check(val, arg0)
			if err != nil {
				throw(call.Otto, err)
			}

			check, err := a.vu.group.Check(name)
			if err != nil {
				throw(call.Otto, err)
			}

			sampleValue := 1.0
			if result {
				atomic.AddInt64(&(check.Passes), 1)
			} else {
				atomic.AddInt64(&(check.Fails), 1)
				success = false
				sampleValue = 0.0
			}

			samples[i] = stats.Sample{
				Time:   t,
				Metric: lib.MetricChecks,
				Tags: map[string]string{
					"group_id": check.Group.ID,
					"check_id": check.ID,
					"path":     check.Path, // Included for human readability.
				},
				Value: sampleValue,
			}
		}
		a.vu.Samples = append(a.vu.Samples, samples...)
	}

	if !success {
		a.vu.Taint = true
		return otto.FalseValue()
	}
	return otto.TrueValue()
}

func (a JSAPI) Taint() {
	a.vu.Taint = true
}

func (a JSAPI) ElapsedMs() float64 {
	return float64(time.Since(a.vu.started)) / float64(time.Millisecond)
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package ldb

import (
	"bytes"
	"encoding/binary"

	"github.com/conformal/btcdb"
	"github.com/conformal/btcwire"
	"github.com/conformal/goleveldb/leveldb"
)

type txUpdateObj struct {
	txSha     *btcwire.ShaHash
	blkHeight int64
	txoff     int
	txlen     int
	ntxout    int
	spentData []byte
	delete    bool
}

type spentTx struct {
	blkHeight int64
	txoff     int
	txlen     int
	numTxO    int
	delete    bool
}
type spentTxUpdate struct {
	txl    []*spentTx
	delete bool
}

// InsertTx inserts a tx hash and its associated data into the database.
func (db *LevelDb) InsertTx(txsha *btcwire.ShaHash, height int64, txoff int, txlen int, spentbuf []byte) (err error) {
	db.dbLock.Lock()
	defer db.dbLock.Unlock()

	return db.insertTx(txsha, height, txoff, txlen, spentbuf)
}

// insertTx inserts a tx hash and its associated data into the database.
// Must be called with db lock held.
func (db *LevelDb) insertTx(txSha *btcwire.ShaHash, height int64, txoff int, txlen int, spentbuf []byte) (err error) {
	var txU txUpdateObj

	txU.txSha = txSha
	txU.blkHeight = height
	txU.txoff = txoff
	txU.txlen = txlen
	txU.spentData = spentbuf

	db.txUpdateMap[*txSha] = &txU

	return nil
}

// formatTx generates the value buffer for the Tx db.
func (db *LevelDb) formatTx(txu *txUpdateObj) []byte {
	blkHeight := uint64(txu.blkHeight)
	txOff := uint32(txu.txoff)
	txLen := uint32(txu.txlen)
	spentbuf := txu.spentData

	txW := make([]byte, 16+len(spentbuf))
	binary.LittleEndian.PutUint64(txW[0:8], blkHeight)
	binary.LittleEndian.PutUint32(txW[8:12], txOff)
	binary.LittleEndian.PutUint32(txW[12:16], txLen)
	copy(txW[16:], spentbuf)

	return txW[:]
}

func (db *LevelDb) getTxData(txsha *btcwire.ShaHash) (int64, int, int, []byte, error) {
	key := shaTxToKey(txsha)
	buf, err := db.lDb.Get(key, db.ro)
	if err != nil {
		return 0, 0, 0, nil, err
	}

	blkHeight := binary.LittleEndian.Uint64(buf[0:8])
	txOff := binary.LittleEndian.Uint32(buf[8:12])
	txLen := binary.LittleEndian.Uint32(buf[12:16])

	spentBuf := make([]byte, len(buf)-16)
	copy(spentBuf, buf[16:])

	return int64(blkHeight), int(txOff), int(txLen), spentBuf, nil
}

func (db *LevelDb) getTxFullySpent(txsha *btcwire.ShaHash) ([]*spentTx, error) {

	var badTxList, spentTxList []*spentTx

	key := shaSpentTxToKey(txsha)
	buf, err := db.lDb.Get(key, db.ro)
	if err == leveldb.ErrNotFound {
		return badTxList, btcdb.TxShaMissing
	} else if err != nil {
		return badTxList, err
	}
	txListLen := len(buf) / 20

	spentTxList = make([]*spentTx, txListLen, txListLen)
	for i := range spentTxList {
		offset := i * 20

		blkHeight := binary.LittleEndian.Uint64(buf[offset:offset+8])
		txOff := binary.LittleEndian.Uint32(buf[offset+8:offset+12])
		txLen := binary.LittleEndian.Uint32(buf[offset+12:offset+16])
		numTxO := binary.LittleEndian.Uint32(buf[offset+16:offset+20])

		sTx := spentTx{
			blkHeight: int64(blkHeight),
			txoff:     int(txOff),
			txlen:     int(txLen),
			numTxO:    int(numTxO),
		}

		spentTxList[i] = &sTx
	}

	return spentTxList, nil
}

func (db *LevelDb) formatTxFullySpent(sTxList []*spentTx) []byte {
	txW := make([]byte, 20*len(sTxList))

	for i, sTx := range sTxList {
		blkHeight := uint64(sTx.blkHeight)
		txOff := uint32(sTx.txoff)
		txLen := uint32(sTx.txlen)
		numTxO := uint32(sTx.numTxO)
		offset := i * 20

		binary.LittleEndian.PutUint64(txW[offset:offset+8], blkHeight)
		binary.LittleEndian.PutUint32(txW[offset+8:offset+12], txOff)
		binary.LittleEndian.PutUint32(txW[offset+12:offset+16], txLen)
		binary.LittleEndian.PutUint32(txW[offset+16:offset+20], numTxO)
	}

	return txW
}

// ExistsTxSha returns if the given tx sha exists in the database
func (db *LevelDb) ExistsTxSha(txsha *btcwire.ShaHash) (exists bool) {
	db.dbLock.Lock()
	defer db.dbLock.Unlock()

	return db.existsTxSha(txsha)
}

// existsTxSha returns if the given tx sha exists in the database.o
// Must be called with the db lock held.
func (db *LevelDb) existsTxSha(txSha *btcwire.ShaHash) (exists bool) {
	_, _, _, _, err := db.getTxData(txSha)
	if err == nil {
		return true
	}

	// BUG(drahn) If there was an error beside non-existant deal with it.

	return false
}

// FetchTxByShaList returns the most recent tx of the name fully spent or not
func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply {
	db.dbLock.Lock()
	defer db.dbLock.Unlock()

	// until the fully spent separation of tx is complete this is identical
	// to FetchUnSpentTxByShaList
	replies := make([]*btcdb.TxListReply, len(txShaList))
	for i, txsha := range txShaList {
		tx, blockSha, height, txspent, err := db.fetchTxDataBySha(txsha)
		btxspent := []bool{}
		if err == nil {
			btxspent = make([]bool, len(tx.TxOut), len(tx.TxOut))
			for idx := range tx.TxOut {
				byteidx := idx / 8
				byteoff := uint(idx % 8)
				btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0
			}
		}
		if err == btcdb.TxShaMissing {
			// if the unspent pool did not have the tx,
			// look in the fully spent pool (only last instance

			sTxList, fSerr := db.getTxFullySpent(txsha)
			if fSerr == nil && len(sTxList) != 0 {
				idx := len(sTxList) - 1
				stx := sTxList[idx]

				tx, blockSha, _, _, err = db.fetchTxDataByLoc(
					stx.blkHeight, stx.txoff, stx.txlen, []byte{})
				if err == nil {
					btxspent = make([]bool, len(tx.TxOut))
					for i := range btxspent {
						btxspent[i] = true
					}
				}
			}
		}
		txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blockSha, Height: height, TxSpent: btxspent, Err: err}
		replies[i] = &txlre
	}
	return replies
}

// FetchUnSpentTxByShaList given a array of ShaHash, look up the transactions
// and return them in a TxListReply array.
func (db *LevelDb) FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply {
	db.dbLock.Lock()
	defer db.dbLock.Unlock()

	replies := make([]*btcdb.TxListReply, len(txShaList))
	for i, txsha := range txShaList {
		tx, blockSha, height, txspent, err := db.fetchTxDataBySha(txsha)
		btxspent := []bool{}
		if err == nil {
			btxspent = make([]bool, len(tx.TxOut), len(tx.TxOut))
			for idx := range tx.TxOut {
				byteidx := idx / 8
				byteoff := uint(idx % 8)
				btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0
			}
		}
		txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blockSha, Height: height, TxSpent: btxspent, Err: err}
		replies[i] = &txlre
	}
	return replies
}

// fetchTxDataBySha returns several pieces of data regarding the given sha.
func (db *LevelDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rblksha *btcwire.ShaHash, rheight int64, rtxspent []byte, err error) {
	var blkHeight int64
	var txspent []byte
	var txOff, txLen int

	blkHeight, txOff, txLen, txspent, err = db.getTxData(txsha)
	if err != nil {
		if err == leveldb.ErrNotFound {
			err = btcdb.TxShaMissing
		}
		return
	}
	return db.fetchTxDataByLoc(blkHeight, txOff, txLen, txspent)
}

// fetchTxDataByLoc returns several pieces of data regarding the given tx
// located by the block/offset/size location
func (db *LevelDb) fetchTxDataByLoc(blkHeight int64, txOff int, txLen int, txspent []byte) (rtx *btcwire.MsgTx, rblksha *btcwire.ShaHash, rheight int64, rtxspent []byte, err error) {
	var blksha *btcwire.ShaHash
	var blkbuf []byte

	blksha, blkbuf, err = db.getBlkByHeight(blkHeight)
	if err != nil {
		if err == leveldb.ErrNotFound {
			err = btcdb.TxShaMissing
		}
		return
	}

	//log.Trace("transaction %v is at block %v %v txoff %v, txlen %v\n",
	//	txsha, blksha, blkHeight, txOff, txLen)

	if len(blkbuf) < txOff+txLen {
		err = btcdb.TxShaMissing
		return
	}
	rbuf := bytes.NewReader(blkbuf[txOff : txOff+txLen])

	var tx btcwire.MsgTx
	err = tx.Deserialize(rbuf)
	if err != nil {
		log.Warnf("unable to decode tx block %v %v txoff %v txlen %v",
			blkHeight, blksha, txOff, txLen)
		return
	}

	return &tx, blksha, blkHeight, txspent, nil
}

// FetchTxBySha returns some data for the given Tx Sha.
func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, error) {
	db.dbLock.Lock()
	defer db.dbLock.Unlock()

	replylen := 0
	replycnt := 0

	tx, blksha, height, txspent, txerr := db.fetchTxDataBySha(txsha)
	if txerr == nil {
		replylen++
	} else {
		if txerr != btcdb.TxShaMissing {
			return []*btcdb.TxListReply{}, txerr
		}
	}

	sTxList, fSerr := db.getTxFullySpent(txsha)

	if fSerr != nil {
		if fSerr != btcdb.TxShaMissing {
			return []*btcdb.TxListReply{}, fSerr
		}
	} else {
		replylen += len(sTxList)
	}

	replies := make([]*btcdb.TxListReply, replylen)

	if fSerr == nil {
		for _, stx := range sTxList {
			tx, blksha, _, _, err := db.fetchTxDataByLoc(
				stx.blkHeight, stx.txoff, stx.txlen, []byte{})
			if err != nil {
				if err != leveldb.ErrNotFound {
					return []*btcdb.TxListReply{}, err
				}
				continue
			}
			btxspent := make([]bool, len(tx.TxOut), len(tx.TxOut))
			for i := range btxspent {
				btxspent[i] = true
			}
			txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blksha, Height: stx.blkHeight, TxSpent: btxspent, Err: nil}
			replies[replycnt] = &txlre
			replycnt++
		}
	}
	if txerr == nil {
		btxspent := make([]bool, len(tx.TxOut), len(tx.TxOut))
		for idx := range tx.TxOut {
			byteidx := idx / 8
			byteoff := uint(idx % 8)
			btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0
		}
		txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blksha, Height: height, TxSpent: btxspent, Err: nil}
		replies[replycnt] = &txlre
		replycnt++
	}
	return replies, nil
}
<commit_msg>go fmt<commit_after>// Copyright (c) 2013-2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package ldb

import (
	"bytes"
	"encoding/binary"

	"github.com/conformal/btcdb"
	"github.com/conformal/btcwire"
	"github.com/conformal/goleveldb/leveldb"
)

type txUpdateObj struct {
	txSha     *btcwire.ShaHash
	blkHeight int64
	txoff     int
	txlen     int
	ntxout    int
	spentData []byte
	delete    bool
}

type spentTx struct {
	blkHeight int64
	txoff     int
	txlen     int
	numTxO    int
	delete    bool
}
type spentTxUpdate struct {
	txl    []*spentTx
	delete bool
}

// InsertTx inserts a tx hash and its associated data into the database.
func (db *LevelDb) InsertTx(txsha *btcwire.ShaHash, height int64, txoff int, txlen int, spentbuf []byte) (err error) {
	db.dbLock.Lock()
	defer db.dbLock.Unlock()

	return db.insertTx(txsha, height, txoff, txlen, spentbuf)
}

// insertTx inserts a tx hash and its associated data into the database.
// Must be called with db lock held.
func (db *LevelDb) insertTx(txSha *btcwire.ShaHash, height int64, txoff int, txlen int, spentbuf []byte) (err error) {
	var txU txUpdateObj

	txU.txSha = txSha
	txU.blkHeight = height
	txU.txoff = txoff
	txU.txlen = txlen
	txU.spentData = spentbuf

	db.txUpdateMap[*txSha] = &txU

	return nil
}

// formatTx generates the value buffer for the Tx db.
func (db *LevelDb) formatTx(txu *txUpdateObj) []byte {
	blkHeight := uint64(txu.blkHeight)
	txOff := uint32(txu.txoff)
	txLen := uint32(txu.txlen)
	spentbuf := txu.spentData

	txW := make([]byte, 16+len(spentbuf))
	binary.LittleEndian.PutUint64(txW[0:8], blkHeight)
	binary.LittleEndian.PutUint32(txW[8:12], txOff)
	binary.LittleEndian.PutUint32(txW[12:16], txLen)
	copy(txW[16:], spentbuf)

	return txW[:]
}

func (db *LevelDb) getTxData(txsha *btcwire.ShaHash) (int64, int, int, []byte, error) {
	key := shaTxToKey(txsha)
	buf, err := db.lDb.Get(key, db.ro)
	if err != nil {
		return 0, 0, 0, nil, err
	}

	blkHeight := binary.LittleEndian.Uint64(buf[0:8])
	txOff := binary.LittleEndian.Uint32(buf[8:12])
	txLen := binary.LittleEndian.Uint32(buf[12:16])

	spentBuf := make([]byte, len(buf)-16)
	copy(spentBuf, buf[16:])

	return int64(blkHeight), int(txOff), int(txLen), spentBuf, nil
}

func (db *LevelDb) getTxFullySpent(txsha *btcwire.ShaHash) ([]*spentTx, error) {

	var badTxList, spentTxList []*spentTx

	key := shaSpentTxToKey(txsha)
	buf, err := db.lDb.Get(key, db.ro)
	if err == leveldb.ErrNotFound {
		return badTxList, btcdb.TxShaMissing
	} else if err != nil {
		return badTxList, err
	}
	txListLen := len(buf) / 20

	spentTxList = make([]*spentTx, txListLen, txListLen)
	for i := range spentTxList {
		offset := i * 20

		blkHeight := binary.LittleEndian.Uint64(buf[offset : offset+8])
		txOff := binary.LittleEndian.Uint32(buf[offset+8 : offset+12])
		txLen := binary.LittleEndian.Uint32(buf[offset+12 : offset+16])
		numTxO := binary.LittleEndian.Uint32(buf[offset+16 : offset+20])

		sTx := spentTx{
			blkHeight: int64(blkHeight),
			txoff:     int(txOff),
			txlen:     int(txLen),
			numTxO:    int(numTxO),
		}

		spentTxList[i] = &sTx
	}

	return spentTxList, nil
}

func (db *LevelDb) formatTxFullySpent(sTxList []*spentTx) []byte {
	txW := make([]byte, 20*len(sTxList))

	for i, sTx := range sTxList {
		blkHeight := uint64(sTx.blkHeight)
		txOff := uint32(sTx.txoff)
		txLen := uint32(sTx.txlen)
		numTxO := uint32(sTx.numTxO)
		offset := i * 20

		binary.LittleEndian.PutUint64(txW[offset:offset+8], blkHeight)
		binary.LittleEndian.PutUint32(txW[offset+8:offset+12], txOff)
		binary.LittleEndian.PutUint32(txW[offset+12:offset+16], txLen)
		binary.LittleEndian.PutUint32(txW[offset+16:offset+20], numTxO)
	}

	return txW
}

// ExistsTxSha returns if the given tx sha exists in the database
func (db *LevelDb) ExistsTxSha(txsha *btcwire.ShaHash) (exists bool) {
	db.dbLock.Lock()
	defer db.dbLock.Unlock()

	return db.existsTxSha(txsha)
}

// existsTxSha returns if the given tx sha exists in the database.o
// Must be called with the db lock held.
func (db *LevelDb) existsTxSha(txSha *btcwire.ShaHash) (exists bool) {
	_, _, _, _, err := db.getTxData(txSha)
	if err == nil {
		return true
	}

	// BUG(drahn) If there was an error beside non-existant deal with it.

	return false
}

// FetchTxByShaList returns the most recent tx of the name fully spent or not
func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply {
	db.dbLock.Lock()
	defer db.dbLock.Unlock()

	// until the fully spent separation of tx is complete this is identical
	// to FetchUnSpentTxByShaList
	replies := make([]*btcdb.TxListReply, len(txShaList))
	for i, txsha := range txShaList {
		tx, blockSha, height, txspent, err := db.fetchTxDataBySha(txsha)
		btxspent := []bool{}
		if err == nil {
			btxspent = make([]bool, len(tx.TxOut), len(tx.TxOut))
			for idx := range tx.TxOut {
				byteidx := idx / 8
				byteoff := uint(idx % 8)
				btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0
			}
		}
		if err == btcdb.TxShaMissing {
			// if the unspent pool did not have the tx,
			// look in the fully spent pool (only last instance

			sTxList, fSerr := db.getTxFullySpent(txsha)
			if fSerr == nil && len(sTxList) != 0 {
				idx := len(sTxList) - 1
				stx := sTxList[idx]

				tx, blockSha, _, _, err = db.fetchTxDataByLoc(
					stx.blkHeight, stx.txoff, stx.txlen, []byte{})
				if err == nil {
					btxspent = make([]bool, len(tx.TxOut))
					for i := range btxspent {
						btxspent[i] = true
					}
				}
			}
		}
		txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blockSha, Height: height, TxSpent: btxspent, Err: err}
		replies[i] = &txlre
	}
	return replies
}

// FetchUnSpentTxByShaList given a array of ShaHash, look up the transactions
// and return them in a TxListReply array.
func (db *LevelDb) FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply {
	db.dbLock.Lock()
	defer db.dbLock.Unlock()

	replies := make([]*btcdb.TxListReply, len(txShaList))
	for i, txsha := range txShaList {
		tx, blockSha, height, txspent, err := db.fetchTxDataBySha(txsha)
		btxspent := []bool{}
		if err == nil {
			btxspent = make([]bool, len(tx.TxOut), len(tx.TxOut))
			for idx := range tx.TxOut {
				byteidx := idx / 8
				byteoff := uint(idx % 8)
				btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0
			}
		}
		txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blockSha, Height: height, TxSpent: btxspent, Err: err}
		replies[i] = &txlre
	}
	return replies
}

// fetchTxDataBySha returns several pieces of data regarding the given sha.
func (db *LevelDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rblksha *btcwire.ShaHash, rheight int64, rtxspent []byte, err error) {
	var blkHeight int64
	var txspent []byte
	var txOff, txLen int

	blkHeight, txOff, txLen, txspent, err = db.getTxData(txsha)
	if err != nil {
		if err == leveldb.ErrNotFound {
			err = btcdb.TxShaMissing
		}
		return
	}
	return db.fetchTxDataByLoc(blkHeight, txOff, txLen, txspent)
}

// fetchTxDataByLoc returns several pieces of data regarding the given tx
// located by the block/offset/size location
func (db *LevelDb) fetchTxDataByLoc(blkHeight int64, txOff int, txLen int, txspent []byte) (rtx *btcwire.MsgTx, rblksha *btcwire.ShaHash, rheight int64, rtxspent []byte, err error) {
	var blksha *btcwire.ShaHash
	var blkbuf []byte

	blksha, blkbuf, err = db.getBlkByHeight(blkHeight)
	if err != nil {
		if err == leveldb.ErrNotFound {
			err = btcdb.TxShaMissing
		}
		return
	}

	//log.Trace("transaction %v is at block %v %v txoff %v, txlen %v\n",
	//	txsha, blksha, blkHeight, txOff, txLen)

	if len(blkbuf) < txOff+txLen {
		err = btcdb.TxShaMissing
		return
	}
	rbuf := bytes.NewReader(blkbuf[txOff : txOff+txLen])

	var tx btcwire.MsgTx
	err = tx.Deserialize(rbuf)
	if err != nil {
		log.Warnf("unable to decode tx block %v %v txoff %v txlen %v",
			blkHeight, blksha, txOff, txLen)
		return
	}

	return &tx, blksha, blkHeight, txspent, nil
}

// FetchTxBySha returns some data for the given Tx Sha.
func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, error) {
	db.dbLock.Lock()
	defer db.dbLock.Unlock()

	replylen := 0
	replycnt := 0

	tx, blksha, height, txspent, txerr := db.fetchTxDataBySha(txsha)
	if txerr == nil {
		replylen++
	} else {
		if txerr != btcdb.TxShaMissing {
			return []*btcdb.TxListReply{}, txerr
		}
	}

	sTxList, fSerr := db.getTxFullySpent(txsha)

	if fSerr != nil {
		if fSerr != btcdb.TxShaMissing {
			return []*btcdb.TxListReply{}, fSerr
		}
	} else {
		replylen += len(sTxList)
	}

	replies := make([]*btcdb.TxListReply, replylen)

	if fSerr == nil {
		for _, stx := range sTxList {
			tx, blksha, _, _, err := db.fetchTxDataByLoc(
				stx.blkHeight, stx.txoff, stx.txlen, []byte{})
			if err != nil {
				if err != leveldb.ErrNotFound {
					return []*btcdb.TxListReply{}, err
				}
				continue
			}
			btxspent := make([]bool, len(tx.TxOut), len(tx.TxOut))
			for i := range btxspent {
				btxspent[i] = true
			}
			txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blksha, Height: stx.blkHeight, TxSpent: btxspent, Err: nil}
			replies[replycnt] = &txlre
			replycnt++
		}
	}
	if txerr == nil {
		btxspent := make([]bool, len(tx.TxOut), len(tx.TxOut))
		for idx := range tx.TxOut {
			byteidx := idx / 8
			byteoff := uint(idx % 8)
			btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0
		}
		txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blksha, Height: height, TxSpent: btxspent, Err: nil}
		replies[replycnt] = &txlre
		replycnt++
	}
	return replies, nil
}
<|endoftext|>


<commit_before>// Package imstor enables you to create copies (or thumbnails) of your images and stores
// them along with the original image on your filesystem. The image and its
// copies are are stored in a file structure based on the (zero-prefixed, decimal)
// CRC 64 checksum of the original image. The last 2 characters of the checksum
// are used as the lvl 1 directory name.
//
// Example folder name and contents, given the checksum 08446744073709551615 and
// sizes named "small" and "large":
//
//  /configured/root/path/15/08446744073709551615/original.jpeg
//  /configured/root/path/15/08446744073709551615/small.jpeg
//  /configured/root/path/15/08446744073709551615/large.jpeg
package imstor

import (
	"errors"
	"fmt"
	"hash/crc64"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/vincent-petithory/dataurl"
)

var crcTable = crc64.MakeTable(crc64.ISO)

const (
	originalImageName = "original"
)

type storage struct {
	conf    *Config
	resizer Resizer
}

// Storage is the engine that can be used to store images and retrieve their paths
type Storage interface {
	Store(mediaType string, data []byte) error
	StoreDataURL(string) error
	Checksum([]byte) string
	ChecksumDataURL(string) (string, error)
	PathFor(checksum string) (string, error)
	PathForSize(checksum, size string) (string, error)
	HasSizesForChecksum(checksum string, sizes []string) (bool, error)
}

// New creates a storage engine using the default Resizer
func New(conf *Config) Storage {
	return storage{
		conf:    conf,
		resizer: DefaultResizer,
	}
}

// NewWithCustomResizer creates a storage engine using a custom resizer
func NewWithCustomResizer(conf *Config, resizer Resizer) Storage {
	return storage{
		conf:    conf,
		resizer: resizer,
	}
}

func getStructuredFolderPath(checksum string) string {
	lvl1Dir := checksum[len(checksum)-2:]
	return path.Join(lvl1Dir, checksum)
}

func (s storage) ChecksumDataURL(str string) (string, error) {
	dataURL, err := dataurl.DecodeString(str)
	if err != nil {
		return "", err
	}
	return s.Checksum(dataURL.Data), nil
}

func (s storage) Checksum(data []byte) string {
	crc := crc64.Checksum(data, crcTable)
	return fmt.Sprintf("%020d", crc)
}

func (s storage) PathFor(sum string) (string, error) {
	return s.PathForSize(sum, originalImageName)
}

func (s storage) PathForSize(sum, size string) (string, error) {
	dir := getStructuredFolderPath(sum)
	absDirPath := filepath.Join(s.conf.RootPath, filepath.FromSlash(dir))
	files, err := ioutil.ReadDir(absDirPath)
	if err != nil {
		return "", err
	}
	for _, file := range files {
		if !file.IsDir() && hasNameWithoutExtension(file.Name(), size) {
			return filepath.Join(dir, file.Name()), nil
		}
	}
	return "", errors.New("File not found!")
}

func (s storage) HasSizesForChecksum(sum string, sizes []string) (bool, error) {
	dir := getStructuredFolderPath(sum)
	absDirPath := filepath.Join(s.conf.RootPath, filepath.FromSlash(dir))
	files, err := ioutil.ReadDir(absDirPath)
	if os.IsNotExist(err) {
		return false, nil
	} else if err != nil {
		return false, err
	}
LoopSizes:
	for _, size := range sizes {
		for _, file := range files {
			if !file.IsDir() && hasNameWithoutExtension(file.Name(), size) {
				continue LoopSizes
			}
		}
		return false, nil
	}
	return true, nil
}

func hasNameWithoutExtension(fileName, name string) bool {
	extension := path.Ext(fileName)
	nameWithoutExtension := strings.TrimSuffix(fileName, extension)
	return nameWithoutExtension == name
}
<commit_msg>Add a way to decode the images<commit_after>// Package imstor enables you to create copies (or thumbnails) of your images and stores
// them along with the original image on your filesystem. The image and its
// copies are are stored in a file structure based on the (zero-prefixed, decimal)
// CRC 64 checksum of the original image. The last 2 characters of the checksum
// are used as the lvl 1 directory name.
//
// Example folder name and contents, given the checksum 08446744073709551615 and
// sizes named "small" and "large":
//
//  /configured/root/path/15/08446744073709551615/original.jpeg
//  /configured/root/path/15/08446744073709551615/small.jpeg
//  /configured/root/path/15/08446744073709551615/large.jpeg
package imstor

import (
	"errors"
	"fmt"
	"hash/crc64"
	"image"
	_ "image/jpeg"
	_ "image/png"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/vincent-petithory/dataurl"
)

var crcTable = crc64.MakeTable(crc64.ISO)

const (
	originalImageName = "original"
)

type storage struct {
	conf    *Config
	resizer Resizer
}

// Storage is the engine that can be used to store images and retrieve their paths
type Storage interface {
	Store(mediaType string, data []byte) error
	StoreDataURL(string) error
	Checksum([]byte) string
	ChecksumDataURL(string) (string, error)
	PathFor(checksum string) (string, error)
	PathForSize(checksum, size string) (string, error)
	HasSizesForChecksum(checksum string, sizes []string) (bool, error)
	GetSize(checksum, size string) (image.Image, error)
}

// New creates a storage engine using the default Resizer
func New(conf *Config) Storage {
	return storage{
		conf:    conf,
		resizer: DefaultResizer,
	}
}

// NewWithCustomResizer creates a storage engine using a custom resizer
func NewWithCustomResizer(conf *Config, resizer Resizer) Storage {
	return storage{
		conf:    conf,
		resizer: resizer,
	}
}

func getStructuredFolderPath(checksum string) string {
	lvl1Dir := checksum[len(checksum)-2:]
	return path.Join(lvl1Dir, checksum)
}

func (s storage) ChecksumDataURL(str string) (string, error) {
	dataURL, err := dataurl.DecodeString(str)
	if err != nil {
		return "", err
	}
	return s.Checksum(dataURL.Data), nil
}

func (s storage) Checksum(data []byte) string {
	crc := crc64.Checksum(data, crcTable)
	return fmt.Sprintf("%020d", crc)
}

func (s storage) PathFor(sum string) (string, error) {
	return s.PathForSize(sum, originalImageName)
}

func (s storage) PathForSize(sum, size string) (string, error) {
	dir := getStructuredFolderPath(sum)
	absDirPath := filepath.Join(s.conf.RootPath, filepath.FromSlash(dir))
	files, err := ioutil.ReadDir(absDirPath)
	if err != nil {
		return "", err
	}
	for _, file := range files {
		if !file.IsDir() && hasNameWithoutExtension(file.Name(), size) {
			return filepath.Join(dir, file.Name()), nil
		}
	}
	return "", errors.New("File not found!")
}

func (s storage) HasSizesForChecksum(sum string, sizes []string) (bool, error) {
	dir := getStructuredFolderPath(sum)
	absDirPath := filepath.Join(s.conf.RootPath, filepath.FromSlash(dir))
	files, err := ioutil.ReadDir(absDirPath)
	if os.IsNotExist(err) {
		return false, nil
	} else if err != nil {
		return false, err
	}
LoopSizes:
	for _, size := range sizes {
		for _, file := range files {
			if !file.IsDir() && hasNameWithoutExtension(file.Name(), size) {
				continue LoopSizes
			}
		}
		return false, nil
	}
	return true, nil
}

func hasNameWithoutExtension(fileName, name string) bool {
	extension := path.Ext(fileName)
	nameWithoutExtension := strings.TrimSuffix(fileName, extension)
	return nameWithoutExtension == name
}

func (s storage) GetSize(sum, size string) (image.Image, error) {
	relPath, err := s.PathForSize(sum, size)
	if err != nil {
		return nil, err
	}
	absPath := filepath.Join(s.conf.RootPath, relPath)

	file, err := os.Open(absPath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	// TODO: currently relies on `_ "image/jpeg"` to be imported here. The Format type needs some rework
	// before it can be used to properly decode written files. (Say, having only a PNG2JPEG Format would not
	// work, because the Format doesn't know how to decode the jpeg file)
	image, _, err := image.Decode(file)
	return image, err
}
<|endoftext|>


<commit_before>/*
 * jQuery File Upload Plugin GAE Go Example 1.1
 * https://github.com/blueimp/jQuery-File-Upload
 *
 * Copyright 2011, Sebastian Tschan
 * https://blueimp.net
 *
 * Licensed under the MIT license:
 * http://creativecommons.org/licenses/MIT/
 */

package app

import (
	"appengine"
	"appengine/blobstore"
	"appengine/memcache"
	"appengine/taskqueue"
	"bytes"
	"encoding/base64"
	"fmt"
	"http"
	"image"
	"image/png"
	"io"
	"json"
	"log"
	"mime/multipart"
	"os"
	"regexp"
	"resize"
	"strings"
	"url"
)

import _ "image/gif"
import _ "image/jpeg"

const (
	WEBSITE              = "http://blueimp.github.com/jQuery-File-Upload/"
	MIN_FILE_SIZE        = 1       // bytes
	MAX_FILE_SIZE        = 5000000 // bytes
	IMAGE_TYPES          = "image/(gif|p?jpeg|(x-)?png)"
	ACCEPT_FILE_TYPES    = IMAGE_TYPES
	EXPIRATION_TIME      = 300 // seconds
	THUMBNAIL_MAX_WIDTH  = 80
	THUMBNAIL_MAX_HEIGHT = THUMBNAIL_MAX_WIDTH
)

var (
	imageTypes      = regexp.MustCompile(IMAGE_TYPES)
	acceptFileTypes = regexp.MustCompile(ACCEPT_FILE_TYPES)
)

type FileInfo struct {
	Key          appengine.BlobKey `json:"-"`
	Index        int               `json:"-"`
	Url          string            `json:"url,omitempty"`
	ThumbnailUrl string            `json:"thumbnail_url,omitempty"`
	Name         string            `json:"name"`
	Type         string            `json:"type"`
	Size         int64             `json:"size"`
	Error        string            `json:"error,omitempty"`
	DeleteUrl    string            `json:"delete_url,omitempty"`
	DeleteType   string            `json:"delete_type,omitempty"`
}

func (fi *FileInfo) Validate() (valid bool) {
	if fi.Size < MIN_FILE_SIZE {
		fi.Error = "minFileSize"
	} else if fi.Size > MAX_FILE_SIZE {
		fi.Error = "maxFileSize"
	} else if !acceptFileTypes.MatchString(fi.Type) {
		fi.Error = "acceptFileTypes"
	} else {
		return true
	}
	return false
}

func (fi *FileInfo) CreateUrls(r *http.Request, c appengine.Context) {
	u := &url.URL{
		Scheme: r.URL.Scheme,
		Host:   appengine.DefaultVersionHostname(c),
	}
	u.Path = "/" + url.QueryEscape(string(fi.Key)) + "/" +
		url.QueryEscape(string(fi.Name))
	fi.Url = u.String()
	fi.DeleteUrl = fi.Url
	fi.DeleteType = "DELETE"
	if fi.ThumbnailUrl != "" && -1 == strings.Index(
		r.Header.Get("Accept"),
		"application/json",
	) {
		u.Path = "/thumbnails/" + url.QueryEscape(string(fi.Key))
		fi.ThumbnailUrl = u.String()
	}
}

func (fi *FileInfo) CreateThumbnail(r io.Reader, c appengine.Context) (data []byte, err os.Error) {
	defer func() {
		if rec := recover(); rec != nil {
			log.Println(rec)
			// 1x1 pixel transparent GIf, bas64 encoded:
			s := "R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=="
			data, _ = base64.StdEncoding.DecodeString(s)
			fi.ThumbnailUrl = "data:image/gif;base64," + s
		}
		memcache.Add(c, &memcache.Item{
			Key:        string(fi.Key),
			Value:      data,
			Expiration: EXPIRATION_TIME,
		})
	}()
	img, _, err := image.Decode(r)
	check(err)
	if b := img.Bounds(); b.Dx() > THUMBNAIL_MAX_WIDTH ||
		b.Dy() > THUMBNAIL_MAX_HEIGHT {
		w, h := THUMBNAIL_MAX_WIDTH, THUMBNAIL_MAX_HEIGHT
		if b.Dx() > b.Dy() {
			h = b.Dy() * h / b.Dx()
		} else {
			w = b.Dx() * w / b.Dy()
		}
		img = resize.Resize(img, img.Bounds(), w, h)
	}
	buffer := bytes.NewBuffer(make([]byte, 0, 32768))
	err = png.Encode(buffer, img)
	check(err)
	data = buffer.Bytes()
	fi.ThumbnailUrl = "data:image/png;base64," +
		base64.StdEncoding.EncodeToString(data)
	return
}

func check(err os.Error) {
	if err != nil {
		panic(err)
	}
}

func queueDeletion(c appengine.Context, fi *FileInfo) {
	if key := string(fi.Key); key != "" {
		task := &taskqueue.Task{
			Path:   "/" + url.QueryEscape(key) + "/-",
			Method: "DELETE", Delay: EXPIRATION_TIME * 1000000,
		}
		taskqueue.Add(c, task, "")
	}
}

func handleUpload(r *http.Request, fh *multipart.FileHeader, i int, c chan *FileInfo) {
	fi := FileInfo{Index: i, Name: fh.Filename}
	context := appengine.NewContext(r)
	defer func() {
		queueDeletion(context, &fi)
		if rec := recover(); rec != nil {
			log.Println(rec)
			fi.Error = rec.(os.Error).String()
		}
		c <- &fi
	}()
	if h, b := fh.Header["Content-Type"]; b && len(h) > 0 {
		fi.Type = h[0]
	}
	f, err := fh.Open()
	defer f.Close()
	check(err)
	fi.Size, err = f.Seek(0, 2)
	check(err)
	if fi.Validate() {
		_, err = f.Seek(0, 0)
		check(err)
		var w *blobstore.Writer
		w, err = blobstore.Create(context, fi.Type)
		defer func() {
			w.Close()
			fi.Key, err = w.Key()
			check(err)
			if imageTypes.MatchString(fi.Type) {
				_, err = f.Seek(0, 0)
				check(err)
				fi.CreateThumbnail(f, context)
			}
			fi.CreateUrls(r, context)
		}()
		check(err)
		_, err = io.Copy(w, f)
	}
}

func handleUploads(r *http.Request) (fileInfos []*FileInfo) {
	if r.MultipartForm != nil && r.MultipartForm.File != nil {
		for _, mfs := range r.MultipartForm.File {
			mfsLen := len(mfs)
			c := make(chan *FileInfo, mfsLen)
			for i, fh := range mfs {
				go handleUpload(r, fh, i, c)
			}
			fileInfos = make([]*FileInfo, mfsLen)
			for i := 0; i < mfsLen; i++ {
				fi := <-c
				fileInfos[fi.Index] = fi
			}
		}
	}
	return
}

func get(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "/" {
		http.Redirect(w, r, WEBSITE, http.StatusFound)
		return
	}
	parts := strings.Split(r.URL.Path, "/")
	if len(parts) == 3 {
		if key := parts[1]; key != "" {
			blobKey := appengine.BlobKey(key)
			bi, err := blobstore.Stat(appengine.NewContext(r), blobKey)
			if err == nil {
				w.Header().Add(
					"Cache-Control",
					fmt.Sprintf("public,max-age=%d", EXPIRATION_TIME),
				)
				if imageTypes.MatchString(bi.ContentType) {
					w.Header().Add("X-Content-Type-Options", "nosniff")
				} else {
					w.Header().Add("Content-Type", "application/octet-stream")
					w.Header().Add(
						"Content-Disposition:",
						fmt.Sprintf("attachment; filename=%s;", bi.Filename),
					)
				}
				blobstore.Send(w, appengine.BlobKey(key))
				return
			}
		}
	}
	http.Error(w, "404 Not Found", http.StatusNotFound)
}

func post(w http.ResponseWriter, r *http.Request) {
	b, err := json.Marshal(handleUploads(r))
	check(err)
	if redirect := r.FormValue("redirect"); redirect != "" {
		http.Redirect(w, r, fmt.Sprintf(
			redirect,
			url.QueryEscape(string(b)),
		), http.StatusFound)
		return
	}
	jsonType := "application/json"
	if strings.Index(r.Header.Get("Accept"), jsonType) != -1 {
		w.Header().Set("Content-Type", jsonType)
	}
	fmt.Fprintln(w, string(b))
}

func delete(w http.ResponseWriter, r *http.Request) {
	parts := strings.Split(r.URL.Path, "/")
	if len(parts) != 3 {
		return
	}
	if key := parts[1]; key != "" {
		c := appengine.NewContext(r)
		err := blobstore.Delete(c, appengine.BlobKey(key))
		check(err)
		memcache.Delete(c, key)
	}
}

func serveThumbnail(w http.ResponseWriter, r *http.Request) {
	parts := strings.Split(r.URL.Path, "/")
	if len(parts) == 3 {
		if key := parts[2]; key != "" {
			var data []byte
			c := appengine.NewContext(r)
			item, err := memcache.Get(c, key)
			if err == nil {
				data = item.Value
			} else {
				blobKey := appengine.BlobKey(key)
				if _, err = blobstore.Stat(c, blobKey); err == nil {
					fi := FileInfo{Key: blobKey}
					data, _ = fi.CreateThumbnail(
						blobstore.NewReader(c, blobKey),
						c,
					)
				}
			}
			if err == nil && len(data) > 3 {
				w.Header().Add(
					"Cache-Control",
					fmt.Sprintf("public,max-age=%d", EXPIRATION_TIME),
				)
				contentType := "image/png"
				if string(data[:3]) == "GIF" {
					contentType = "image/gif"
				} else if string(data[1:4]) != "PNG" {
					contentType = "image/jpeg"
				}
				w.Header().Set("Content-Type", contentType)
				fmt.Fprintln(w, string(data))
				return
			}
		}
	}
	http.Error(w, "404 Not Found", http.StatusNotFound)
}

func handle(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Access-Control-Allow-Origin", "*")
	w.Header().Add(
		"Access-Control-Allow-Methods",
		"OPTIONS, HEAD, GET, POST, PUT, DELETE",
	)
	switch r.Method {
	case "OPTIONS":
	case "HEAD":
	case "GET":
		get(w, r)
	case "POST":
		if r.FormValue("_method") == "DELETE" {
			delete(w, r)
		} else {
			post(w, r)
		}
	case "DELETE":
		delete(w, r)
	default:
		http.Error(w, "501 Not Implemented", http.StatusNotImplemented)
	}
}

func init() {
	http.HandleFunc("/", handle)
	http.HandleFunc("/thumbnails/", serveThumbnail)
}
<commit_msg>Process the request body as a stream.<commit_after>/*
 * jQuery File Upload Plugin GAE Go Example 1.2
 * https://github.com/blueimp/jQuery-File-Upload
 *
 * Copyright 2011, Sebastian Tschan
 * https://blueimp.net
 *
 * Licensed under the MIT license:
 * http://creativecommons.org/licenses/MIT/
 */

package app

import (
	"appengine"
	"appengine/blobstore"
	"appengine/memcache"
	"appengine/taskqueue"
	"bytes"
	"encoding/base64"
	"fmt"
	"http"
	"image"
	"image/png"
	"io"
	"json"
	"log"
	"mime/multipart"
	"os"
	"regexp"
	"resize"
	"strings"
	"url"
)

import _ "image/gif"
import _ "image/jpeg"

const (
	WEBSITE              = "http://blueimp.github.com/jQuery-File-Upload/"
	MIN_FILE_SIZE        = 1       // bytes
	MAX_FILE_SIZE        = 5000000 // bytes
	IMAGE_TYPES          = "image/(gif|p?jpeg|(x-)?png)"
	ACCEPT_FILE_TYPES    = IMAGE_TYPES
	EXPIRATION_TIME      = 300 // seconds
	THUMBNAIL_MAX_WIDTH  = 80
	THUMBNAIL_MAX_HEIGHT = THUMBNAIL_MAX_WIDTH
)

var (
	imageTypes      = regexp.MustCompile(IMAGE_TYPES)
	acceptFileTypes = regexp.MustCompile(ACCEPT_FILE_TYPES)
)

type FileInfo struct {
	Key          appengine.BlobKey `json:"-"`
	Url          string            `json:"url,omitempty"`
	ThumbnailUrl string            `json:"thumbnail_url,omitempty"`
	Name         string            `json:"name"`
	Type         string            `json:"type"`
	Size         int64             `json:"size"`
	Error        string            `json:"error,omitempty"`
	DeleteUrl    string            `json:"delete_url,omitempty"`
	DeleteType   string            `json:"delete_type,omitempty"`
}

func (fi *FileInfo) ValidateType() (valid bool) {
	if acceptFileTypes.MatchString(fi.Type) {
		return true
	}
	fi.Error = "acceptFileTypes"
	return false
}

func (fi *FileInfo) ValidateSize() (valid bool) {
	if fi.Size < MIN_FILE_SIZE {
		fi.Error = "minFileSize"
	} else if fi.Size > MAX_FILE_SIZE {
		fi.Error = "maxFileSize"
	} else {
		return true
	}
	return false
}

func (fi *FileInfo) CreateUrls(r *http.Request, c appengine.Context) {
	u := &url.URL{
		Scheme: r.URL.Scheme,
		Host:   appengine.DefaultVersionHostname(c),
	}
	u.Path = "/" + url.QueryEscape(string(fi.Key)) + "/" +
		url.QueryEscape(string(fi.Name))
	fi.Url = u.String()
	fi.DeleteUrl = fi.Url
	fi.DeleteType = "DELETE"
	if fi.ThumbnailUrl != "" && -1 == strings.Index(
		r.Header.Get("Accept"),
		"application/json",
	) {
		u.Path = "/thumbnails/" + url.QueryEscape(string(fi.Key))
		fi.ThumbnailUrl = u.String()
	}
}

func (fi *FileInfo) CreateThumbnail(r io.Reader, c appengine.Context) (data []byte, err os.Error) {
	defer func() {
		if rec := recover(); rec != nil {
			log.Println(rec)
			// 1x1 pixel transparent GIf, bas64 encoded:
			s := "R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=="
			data, _ = base64.StdEncoding.DecodeString(s)
			fi.ThumbnailUrl = "data:image/gif;base64," + s
		}
		memcache.Add(c, &memcache.Item{
			Key:        string(fi.Key),
			Value:      data,
			Expiration: EXPIRATION_TIME,
		})
	}()
	img, _, err := image.Decode(r)
	check(err)
	if bounds := img.Bounds(); bounds.Dx() > THUMBNAIL_MAX_WIDTH ||
		bounds.Dy() > THUMBNAIL_MAX_HEIGHT {
		w, h := THUMBNAIL_MAX_WIDTH, THUMBNAIL_MAX_HEIGHT
		if bounds.Dx() > bounds.Dy() {
			h = bounds.Dy() * h / bounds.Dx()
		} else {
			w = bounds.Dx() * w / bounds.Dy()
		}
		img = resize.Resize(img, img.Bounds(), w, h)
	}
	var b bytes.Buffer
	err = png.Encode(&b, img)
	check(err)
	data = b.Bytes()
	fi.ThumbnailUrl = "data:image/png;base64," +
		base64.StdEncoding.EncodeToString(data)
	return
}

func check(err os.Error) {
	if err != nil {
		panic(err)
	}
}

func delayedDelete(c appengine.Context, fi *FileInfo) {
	if key := string(fi.Key); key != "" {
		task := &taskqueue.Task{
			Path:   "/" + url.QueryEscape(key) + "/-",
			Method: "DELETE", Delay: EXPIRATION_TIME * 1000000,
		}
		taskqueue.Add(c, task, "")
	}
}

func handleUpload(r *http.Request, p *multipart.Part) (fi *FileInfo) {
	fi = &FileInfo{
		Name: p.FileName(),
		Type: p.Header.Get("Content-Type"),
	}
	if !fi.ValidateType() {
		return
	}
	defer func() {
		if rec := recover(); rec != nil {
			log.Println(rec)
			fi.Error = rec.(os.Error).String()
		}
	}()
	var b bytes.Buffer
	lr := &io.LimitedReader{p, MAX_FILE_SIZE + 1}
	context := appengine.NewContext(r)
	w, err := blobstore.Create(context, fi.Type)
	defer func() {
		w.Close()
		fi.Size = MAX_FILE_SIZE + 1 - lr.N
		fi.Key, err = w.Key()
		check(err)
		if !fi.ValidateSize() {
			err := blobstore.Delete(context, fi.Key)
			check(err)
			return
		}
		delayedDelete(context, fi)
		if b.Len() > 0 {
			fi.CreateThumbnail(&b, context)
		}
		fi.CreateUrls(r, context)
	}()
	check(err)
	var wr io.Writer = w
	if imageTypes.MatchString(fi.Type) {
		wr = io.MultiWriter(&b, w)
	}
	_, err = io.Copy(wr, lr)
	return
}

func getFormValue(p *multipart.Part) string {
	var b bytes.Buffer
	io.Copyn(&b, p, int64(1<<20)) // Copy max: 1 MiB
	return b.String()
}

func handleUploads(r *http.Request) (fileInfos []*FileInfo) {
	fileInfos = make([]*FileInfo, 0)
	mr, err := r.MultipartReader()
	check(err)
	r.Form, err = url.ParseQuery(r.URL.RawQuery)
	check(err)
	part, err := mr.NextPart()
	for err == nil {
		if name := part.FormName(); name != "" {
			if part.FileName() != "" {
				fileInfos = append(fileInfos, handleUpload(r, part))
			} else {
				r.Form[name] = append(r.Form[name], getFormValue(part))
			}
		}
		part, err = mr.NextPart()
	}
	return
}

func get(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "/" {
		http.Redirect(w, r, WEBSITE, http.StatusFound)
		return
	}
	parts := strings.Split(r.URL.Path, "/")
	if len(parts) == 3 {
		if key := parts[1]; key != "" {
			blobKey := appengine.BlobKey(key)
			bi, err := blobstore.Stat(appengine.NewContext(r), blobKey)
			if err == nil {
				w.Header().Add(
					"Cache-Control",
					fmt.Sprintf("public,max-age=%d", EXPIRATION_TIME),
				)
				if imageTypes.MatchString(bi.ContentType) {
					w.Header().Add("X-Content-Type-Options", "nosniff")
				} else {
					w.Header().Add("Content-Type", "application/octet-stream")
					w.Header().Add(
						"Content-Disposition:",
						fmt.Sprintf("attachment; filename=%s;", parts[2]),
					)
				}
				blobstore.Send(w, appengine.BlobKey(key))
				return
			}
		}
	}
	http.Error(w, "404 Not Found", http.StatusNotFound)
}

func post(w http.ResponseWriter, r *http.Request) {
	b, err := json.Marshal(handleUploads(r))
	check(err)
	if redirect := r.FormValue("redirect"); redirect != "" {
		http.Redirect(w, r, fmt.Sprintf(
			redirect,
			url.QueryEscape(string(b)),
		), http.StatusFound)
		return
	}
	jsonType := "application/json"
	if strings.Index(r.Header.Get("Accept"), jsonType) != -1 {
		w.Header().Set("Content-Type", jsonType)
	}
	fmt.Fprintln(w, string(b))
}

func delete(w http.ResponseWriter, r *http.Request) {
	parts := strings.Split(r.URL.Path, "/")
	if len(parts) != 3 {
		return
	}
	if key := parts[1]; key != "" {
		c := appengine.NewContext(r)
		err := blobstore.Delete(c, appengine.BlobKey(key))
		check(err)
		memcache.Delete(c, key)
	}
}

func serveThumbnail(w http.ResponseWriter, r *http.Request) {
	parts := strings.Split(r.URL.Path, "/")
	if len(parts) == 3 {
		if key := parts[2]; key != "" {
			var data []byte
			c := appengine.NewContext(r)
			item, err := memcache.Get(c, key)
			if err == nil {
				data = item.Value
			} else {
				blobKey := appengine.BlobKey(key)
				if _, err = blobstore.Stat(c, blobKey); err == nil {
					fi := FileInfo{Key: blobKey}
					data, _ = fi.CreateThumbnail(
						blobstore.NewReader(c, blobKey),
						c,
					)
				}
			}
			if err == nil && len(data) > 3 {
				w.Header().Add(
					"Cache-Control",
					fmt.Sprintf("public,max-age=%d", EXPIRATION_TIME),
				)
				contentType := "image/png"
				if string(data[:3]) == "GIF" {
					contentType = "image/gif"
				} else if string(data[1:4]) != "PNG" {
					contentType = "image/jpeg"
				}
				w.Header().Set("Content-Type", contentType)
				fmt.Fprintln(w, string(data))
				return
			}
		}
	}
	http.Error(w, "404 Not Found", http.StatusNotFound)
}

func handle(w http.ResponseWriter, r *http.Request) {
	params, err := url.ParseQuery(r.URL.RawQuery)
	check(err)
	w.Header().Add("Access-Control-Allow-Origin", "*")
	w.Header().Add(
		"Access-Control-Allow-Methods",
		"OPTIONS, HEAD, GET, POST, PUT, DELETE",
	)
	switch r.Method {
	case "OPTIONS":
	case "HEAD":
	case "GET":
		get(w, r)
	case "POST":
		if len(params["_method"]) > 0 && params["_method"][0] == "DELETE" {
			delete(w, r)
		} else {
			post(w, r)
		}
	case "DELETE":
		delete(w, r)
	default:
		http.Error(w, "501 Not Implemented", http.StatusNotImplemented)
	}
}

func init() {
	http.HandleFunc("/", handle)
	http.HandleFunc("/thumbnails/", serveThumbnail)
}
<|endoftext|>