<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gen

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"hash"
	"hash/fnv"
	"io"
	"log"
	"os"
	"reflect"
	"strings"
	"unicode"
	"unicode/utf8"
)

// This file contains utilities for generating code.

// TODO: other write methods like:
// - slices, maps, types, etc.

// CodeWriter is a utility for writing structured code. It computes the content
// hash and size of written content. It ensures there are newlines between
// written code blocks.
type CodeWriter struct {
	buf  bytes.Buffer
	Size int
	Hash hash.Hash32 // content hash
	gob  *gob.Encoder
	// For comments we skip the usual one-line separator if they are followed by
	// a code block.
	skipSep bool
}

func (w *CodeWriter) Write(p []byte) (n int, err error) {
	return w.buf.Write(p)
}

// NewCodeWriter returns a new CodeWriter.
func NewCodeWriter() *CodeWriter {
	h := fnv.New32()
	return &CodeWriter{Hash: h, gob: gob.NewEncoder(h)}
}

// WriteGoFile appends the buffer with the total size of all created structures
// and writes it as a Go file to the the given file with the given package name.
func (w *CodeWriter) WriteGoFile(filename, pkg string) {
	f, err := os.Create(filename)
	if err != nil {
		log.Fatalf("Could not create file %s: %v", filename, err)
	}
	defer f.Close()
	if _, err = w.WriteGo(f, pkg); err != nil {
		log.Fatalf("Error writing file %s: %v", filename, err)
	}
}

// WriteGo appends the buffer with the total size of all created structures and
// writes it as a Go file to the the given writer with the given package name.
func (w *CodeWriter) WriteGo(out io.Writer, pkg string) (n int, err error) {
	sz := w.Size
	w.WriteComment("Total table size %d bytes (%dKiB); checksum: %X\n", sz, sz/1024, w.Hash.Sum32())
	defer w.buf.Reset()
	return WriteGo(out, pkg, w.buf.Bytes())
}

func (w *CodeWriter) printf(f string, x ...interface{}) {
	fmt.Fprintf(w, f, x...)
}

func (w *CodeWriter) insertSep() {
	if w.skipSep {
		w.skipSep = false
		return
	}
	// Use at least two newlines to ensure a blank space between the previous
	// block. WriteGoFile will remove extraneous newlines.
	w.printf("\n\n")
}

// WriteComment writes a comment block. All line starts are prefixed with "//".
// Initial empty lines are gobbled. The indentation for the first line is
// stripped from consecutive lines.
func (w *CodeWriter) WriteComment(comment string, args ...interface{}) {
	s := fmt.Sprintf(comment, args...)
	s = strings.Trim(s, "\n")

	// Use at least two newlines to ensure a blank space between the previous
	// block. WriteGoFile will remove extraneous newlines.
	w.printf("\n\n// ")
	w.skipSep = true

	// strip first indent level.
	sep := "\n"
	for ; len(s) > 0 && (s[0] == '\t' || s[0] == ' '); s = s[1:] {
		sep += s[:1]
	}

	strings.NewReplacer(sep, "\n// ", "\n", "\n// ").WriteString(w, s)

	w.printf("\n")
}

func (w *CodeWriter) writeSizeInfo(size int) {
	w.printf("// Size: %d bytes\n", size)
}

// WriteConst writes a constant of the given name and value.
func (w *CodeWriter) WriteConst(name string, x interface{}) {
	w.insertSep()
	v := reflect.ValueOf(x)

	switch v.Type().Kind() {
	case reflect.String:
		// See golang.org/issue/13145.
		const arbitraryCutoff = 16
		if v.Len() > arbitraryCutoff {
			w.printf("var %s %s = ", name, typeName(x))
		} else {
			w.printf("const %s %s = ", name, typeName(x))
		}
		w.WriteString(v.String())
		w.printf("\n")
	default:
		w.printf("const %s = %#v\n", name, x)
	}
}

// WriteVar writes a variable of the given name and value.
func (w *CodeWriter) WriteVar(name string, x interface{}) {
	w.insertSep()
	v := reflect.ValueOf(x)
	oldSize := w.Size
	sz := int(v.Type().Size())
	w.Size += sz

	switch v.Type().Kind() {
	case reflect.String:
		w.printf("var %s %s = ", name, typeName(x))
		w.WriteString(v.String())
	case reflect.Struct:
		w.gob.Encode(x)
		fallthrough
	case reflect.Slice, reflect.Array:
		w.printf("var %s = ", name)
		w.writeValue(v)
		w.writeSizeInfo(w.Size - oldSize)
	default:
		w.printf("var %s %s = ", name, typeName(x))
		w.gob.Encode(x)
		w.writeValue(v)
		w.writeSizeInfo(w.Size - oldSize)
	}
	w.printf("\n")
}

func (w *CodeWriter) writeValue(v reflect.Value) {
	x := v.Interface()
	switch v.Kind() {
	case reflect.String:
		w.WriteString(v.String())
	case reflect.Array:
		// Don't double count: callers of WriteArray count on the size being
		// added, so we need to discount it here.
		w.Size -= int(v.Type().Size())
		w.writeSlice(x, true)
	case reflect.Slice:
		w.writeSlice(x, false)
	case reflect.Struct:
		w.printf("%s{\n", typeName(v.Interface()))
		t := v.Type()
		for i := 0; i < v.NumField(); i++ {
			w.printf("%s: ", t.Field(i).Name)
			w.writeValue(v.Field(i))
			w.printf(",\n")
		}
		w.printf("}")
	default:
		w.printf("%#v", x)
	}
}

// WriteString writes a string literal.
func (w *CodeWriter) WriteString(s string) {
	io.WriteString(w.Hash, s) // content hash
	w.Size += len(s)

	const maxInline = 40
	if len(s) <= maxInline {
		w.printf("%q", s)
		return
	}

	// We will render the string as a multi-line string.
	const maxWidth = 80 - 4 - len(`"`) - len(`" +`)

	// When starting on its own line, go fmt indents line 2+ an extra level.
	n, max := maxWidth, maxWidth-4

	// Print "" +\n, if a string does not start on its own line.
	b := w.buf.Bytes()
	if p := len(bytes.TrimRight(b, " \t")); p > 0 && b[p-1] != '\n' {
		w.printf("\"\" + // Size: %d bytes\n", len(s))
		n, max = maxWidth, maxWidth
	}

	w.printf(`"`)

	for sz, p := 0, 0; p < len(s); {
		var r rune
		r, sz = utf8.DecodeRuneInString(s[p:])
		out := s[p : p+sz]
		chars := 1
		if !unicode.IsPrint(r) || r == utf8.RuneError {
			switch sz {
			case 1:
				out = fmt.Sprintf("\\x%02x", s[p])
			case 2, 3:
				out = fmt.Sprintf("\\u%04x", r)
			case 4:
				out = fmt.Sprintf("\\U%08x", r)
			}
			chars = len(out)
		}
		if n -= chars; n < 0 {
			w.printf("\" +\n\"")
			n = max - len(out)
		}
		w.printf("%s", out)
		p += sz
	}
	w.printf(`"`)
}

// WriteSlice writes a slice value.
func (w *CodeWriter) WriteSlice(x interface{}) {
	w.writeSlice(x, false)
}

// WriteArray writes an array value.
func (w *CodeWriter) WriteArray(x interface{}) {
	w.writeSlice(x, true)
}

func (w *CodeWriter) writeSlice(x interface{}, isArray bool) {
	v := reflect.ValueOf(x)
	w.gob.Encode(v.Len())
	w.Size += v.Len() * int(v.Type().Elem().Size())
	name := typeName(x)
	if isArray {
		name = fmt.Sprintf("[%d]%s", v.Len(), name[strings.Index(name, "]")+1:])
	}
	if isArray {
		w.printf("%s{\n", name)
	} else {
		w.printf("%s{ // %d elements\n", name, v.Len())
	}

	switch kind := v.Type().Elem().Kind(); kind {
	case reflect.String:
		for _, s := range x.([]string) {
			w.WriteString(s)
			w.printf(",\n")
		}
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		// nLine and nBlock are the number of elements per line and block.
		nLine, nBlock, format := 8, 64, "%d,"
		switch kind {
		case reflect.Uint8:
			format = "%#02x,"
		case reflect.Uint16:
			format = "%#04x,"
		case reflect.Uint32:
			nLine, nBlock, format = 4, 32, "%#08x,"
		case reflect.Uint, reflect.Uint64:
			nLine, nBlock, format = 4, 32, "%#016x,"
		case reflect.Int8:
			nLine = 16
		}
		n := nLine
		for i := 0; i < v.Len(); i++ {
			if i%nBlock == 0 && v.Len() > nBlock {
				w.printf("// Entry %X - %X\n", i, i+nBlock-1)
			}
			x := v.Index(i).Interface()
			w.gob.Encode(x)
			w.printf(format, x)
			if n--; n == 0 {
				n = nLine
				w.printf("\n")
			}
		}
		w.printf("\n")
	case reflect.Struct:
		zero := reflect.Zero(v.Type().Elem()).Interface()
		for i := 0; i < v.Len(); i++ {
			x := v.Index(i).Interface()
			w.gob.EncodeValue(v)
			if !reflect.DeepEqual(zero, x) {
				line := fmt.Sprintf("%#v,\n", x)
				line = line[strings.IndexByte(line, '{'):]
				w.printf("%d: ", i)
				w.printf(line)
			}
		}
	case reflect.Array:
		for i := 0; i < v.Len(); i++ {
			w.printf("%d: %#v,\n", i, v.Index(i).Interface())
		}
	default:
		panic("gen: slice elem type not supported")
	}
	w.printf("}")
}

// WriteType writes a definition of the type of the given value and returns the
// type name.
func (w *CodeWriter) WriteType(x interface{}) string {
	t := reflect.TypeOf(x)
	w.printf("type %s struct {\n", t.Name())
	for i := 0; i < t.NumField(); i++ {
		w.printf("\t%s %s\n", t.Field(i).Name, t.Field(i).Type)
	}
	w.printf("}\n")
	return t.Name()
}

// typeName returns the name of the go type of x.
func typeName(x interface{}) string {
	t := reflect.ValueOf(x).Type()
	return strings.Replace(fmt.Sprint(t), "main.", "", 1)
}
<commit_msg>internal/gen: bug fix: escape '"'<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gen

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"hash"
	"hash/fnv"
	"io"
	"log"
	"os"
	"reflect"
	"strings"
	"unicode"
	"unicode/utf8"
)

// This file contains utilities for generating code.

// TODO: other write methods like:
// - slices, maps, types, etc.

// CodeWriter is a utility for writing structured code. It computes the content
// hash and size of written content. It ensures there are newlines between
// written code blocks.
type CodeWriter struct {
	buf  bytes.Buffer
	Size int
	Hash hash.Hash32 // content hash
	gob  *gob.Encoder
	// For comments we skip the usual one-line separator if they are followed by
	// a code block.
	skipSep bool
}

func (w *CodeWriter) Write(p []byte) (n int, err error) {
	return w.buf.Write(p)
}

// NewCodeWriter returns a new CodeWriter.
func NewCodeWriter() *CodeWriter {
	h := fnv.New32()
	return &CodeWriter{Hash: h, gob: gob.NewEncoder(h)}
}

// WriteGoFile appends the buffer with the total size of all created structures
// and writes it as a Go file to the the given file with the given package name.
func (w *CodeWriter) WriteGoFile(filename, pkg string) {
	f, err := os.Create(filename)
	if err != nil {
		log.Fatalf("Could not create file %s: %v", filename, err)
	}
	defer f.Close()
	if _, err = w.WriteGo(f, pkg); err != nil {
		log.Fatalf("Error writing file %s: %v", filename, err)
	}
}

// WriteGo appends the buffer with the total size of all created structures and
// writes it as a Go file to the the given writer with the given package name.
func (w *CodeWriter) WriteGo(out io.Writer, pkg string) (n int, err error) {
	sz := w.Size
	w.WriteComment("Total table size %d bytes (%dKiB); checksum: %X\n", sz, sz/1024, w.Hash.Sum32())
	defer w.buf.Reset()
	return WriteGo(out, pkg, w.buf.Bytes())
}

func (w *CodeWriter) printf(f string, x ...interface{}) {
	fmt.Fprintf(w, f, x...)
}

func (w *CodeWriter) insertSep() {
	if w.skipSep {
		w.skipSep = false
		return
	}
	// Use at least two newlines to ensure a blank space between the previous
	// block. WriteGoFile will remove extraneous newlines.
	w.printf("\n\n")
}

// WriteComment writes a comment block. All line starts are prefixed with "//".
// Initial empty lines are gobbled. The indentation for the first line is
// stripped from consecutive lines.
func (w *CodeWriter) WriteComment(comment string, args ...interface{}) {
	s := fmt.Sprintf(comment, args...)
	s = strings.Trim(s, "\n")

	// Use at least two newlines to ensure a blank space between the previous
	// block. WriteGoFile will remove extraneous newlines.
	w.printf("\n\n// ")
	w.skipSep = true

	// strip first indent level.
	sep := "\n"
	for ; len(s) > 0 && (s[0] == '\t' || s[0] == ' '); s = s[1:] {
		sep += s[:1]
	}

	strings.NewReplacer(sep, "\n// ", "\n", "\n// ").WriteString(w, s)

	w.printf("\n")
}

func (w *CodeWriter) writeSizeInfo(size int) {
	w.printf("// Size: %d bytes\n", size)
}

// WriteConst writes a constant of the given name and value.
func (w *CodeWriter) WriteConst(name string, x interface{}) {
	w.insertSep()
	v := reflect.ValueOf(x)

	switch v.Type().Kind() {
	case reflect.String:
		// See golang.org/issue/13145.
		const arbitraryCutoff = 16
		if v.Len() > arbitraryCutoff {
			w.printf("var %s %s = ", name, typeName(x))
		} else {
			w.printf("const %s %s = ", name, typeName(x))
		}
		w.WriteString(v.String())
		w.printf("\n")
	default:
		w.printf("const %s = %#v\n", name, x)
	}
}

// WriteVar writes a variable of the given name and value.
func (w *CodeWriter) WriteVar(name string, x interface{}) {
	w.insertSep()
	v := reflect.ValueOf(x)
	oldSize := w.Size
	sz := int(v.Type().Size())
	w.Size += sz

	switch v.Type().Kind() {
	case reflect.String:
		w.printf("var %s %s = ", name, typeName(x))
		w.WriteString(v.String())
	case reflect.Struct:
		w.gob.Encode(x)
		fallthrough
	case reflect.Slice, reflect.Array:
		w.printf("var %s = ", name)
		w.writeValue(v)
		w.writeSizeInfo(w.Size - oldSize)
	default:
		w.printf("var %s %s = ", name, typeName(x))
		w.gob.Encode(x)
		w.writeValue(v)
		w.writeSizeInfo(w.Size - oldSize)
	}
	w.printf("\n")
}

func (w *CodeWriter) writeValue(v reflect.Value) {
	x := v.Interface()
	switch v.Kind() {
	case reflect.String:
		w.WriteString(v.String())
	case reflect.Array:
		// Don't double count: callers of WriteArray count on the size being
		// added, so we need to discount it here.
		w.Size -= int(v.Type().Size())
		w.writeSlice(x, true)
	case reflect.Slice:
		w.writeSlice(x, false)
	case reflect.Struct:
		w.printf("%s{\n", typeName(v.Interface()))
		t := v.Type()
		for i := 0; i < v.NumField(); i++ {
			w.printf("%s: ", t.Field(i).Name)
			w.writeValue(v.Field(i))
			w.printf(",\n")
		}
		w.printf("}")
	default:
		w.printf("%#v", x)
	}
}

// WriteString writes a string literal.
func (w *CodeWriter) WriteString(s string) {
	io.WriteString(w.Hash, s) // content hash
	w.Size += len(s)

	const maxInline = 40
	if len(s) <= maxInline {
		w.printf("%q", s)
		return
	}

	// We will render the string as a multi-line string.
	const maxWidth = 80 - 4 - len(`"`) - len(`" +`)

	// When starting on its own line, go fmt indents line 2+ an extra level.
	n, max := maxWidth, maxWidth-4

	// Print "" +\n, if a string does not start on its own line.
	b := w.buf.Bytes()
	if p := len(bytes.TrimRight(b, " \t")); p > 0 && b[p-1] != '\n' {
		w.printf("\"\" + // Size: %d bytes\n", len(s))
		n, max = maxWidth, maxWidth
	}

	w.printf(`"`)

	for sz, p := 0, 0; p < len(s); {
		var r rune
		r, sz = utf8.DecodeRuneInString(s[p:])
		out := s[p : p+sz]
		chars := 1
		if !unicode.IsPrint(r) || r == utf8.RuneError || r == '"' {
			switch sz {
			case 1:
				out = fmt.Sprintf("\\x%02x", s[p])
			case 2, 3:
				out = fmt.Sprintf("\\u%04x", r)
			case 4:
				out = fmt.Sprintf("\\U%08x", r)
			}
			chars = len(out)
		}
		if n -= chars; n < 0 {
			w.printf("\" +\n\"")
			n = max - len(out)
		}
		w.printf("%s", out)
		p += sz
	}
	w.printf(`"`)
}

// WriteSlice writes a slice value.
func (w *CodeWriter) WriteSlice(x interface{}) {
	w.writeSlice(x, false)
}

// WriteArray writes an array value.
func (w *CodeWriter) WriteArray(x interface{}) {
	w.writeSlice(x, true)
}

func (w *CodeWriter) writeSlice(x interface{}, isArray bool) {
	v := reflect.ValueOf(x)
	w.gob.Encode(v.Len())
	w.Size += v.Len() * int(v.Type().Elem().Size())
	name := typeName(x)
	if isArray {
		name = fmt.Sprintf("[%d]%s", v.Len(), name[strings.Index(name, "]")+1:])
	}
	if isArray {
		w.printf("%s{\n", name)
	} else {
		w.printf("%s{ // %d elements\n", name, v.Len())
	}

	switch kind := v.Type().Elem().Kind(); kind {
	case reflect.String:
		for _, s := range x.([]string) {
			w.WriteString(s)
			w.printf(",\n")
		}
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		// nLine and nBlock are the number of elements per line and block.
		nLine, nBlock, format := 8, 64, "%d,"
		switch kind {
		case reflect.Uint8:
			format = "%#02x,"
		case reflect.Uint16:
			format = "%#04x,"
		case reflect.Uint32:
			nLine, nBlock, format = 4, 32, "%#08x,"
		case reflect.Uint, reflect.Uint64:
			nLine, nBlock, format = 4, 32, "%#016x,"
		case reflect.Int8:
			nLine = 16
		}
		n := nLine
		for i := 0; i < v.Len(); i++ {
			if i%nBlock == 0 && v.Len() > nBlock {
				w.printf("// Entry %X - %X\n", i, i+nBlock-1)
			}
			x := v.Index(i).Interface()
			w.gob.Encode(x)
			w.printf(format, x)
			if n--; n == 0 {
				n = nLine
				w.printf("\n")
			}
		}
		w.printf("\n")
	case reflect.Struct:
		zero := reflect.Zero(v.Type().Elem()).Interface()
		for i := 0; i < v.Len(); i++ {
			x := v.Index(i).Interface()
			w.gob.EncodeValue(v)
			if !reflect.DeepEqual(zero, x) {
				line := fmt.Sprintf("%#v,\n", x)
				line = line[strings.IndexByte(line, '{'):]
				w.printf("%d: ", i)
				w.printf(line)
			}
		}
	case reflect.Array:
		for i := 0; i < v.Len(); i++ {
			w.printf("%d: %#v,\n", i, v.Index(i).Interface())
		}
	default:
		panic("gen: slice elem type not supported")
	}
	w.printf("}")
}

// WriteType writes a definition of the type of the given value and returns the
// type name.
func (w *CodeWriter) WriteType(x interface{}) string {
	t := reflect.TypeOf(x)
	w.printf("type %s struct {\n", t.Name())
	for i := 0; i < t.NumField(); i++ {
		w.printf("\t%s %s\n", t.Field(i).Name, t.Field(i).Type)
	}
	w.printf("}\n")
	return t.Name()
}

// typeName returns the name of the go type of x.
func typeName(x interface{}) string {
	t := reflect.ValueOf(x).Type()
	return strings.Replace(fmt.Sprint(t), "main.", "", 1)
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"github.com/NeowayLabs/wabbit"
)

const (
	QueueMaxLen = 2 << 8
)

type Queue struct {
	name string
	data chan wabbit.Delivery
}

func NewQueue(name string) *Queue {
	return &Queue{
		name: name,
		data: make(chan wabbit.Delivery, QueueMaxLen),
	}
}

func (q *Queue) Consumers() int {
	return 0
}

func (q *Queue) Name() string {
	return q.name
}

func (q *Queue) Messages() int {
	return 0
}

func (q *Queue) dispatch(route string, delivery *Delivery) error {
	if q.data != nil {
		q.data <- delivery
		return nil
	}

	return fmt.Errorf("No data in queue: %s", route)
}
<commit_msg>Fix codecov<commit_after>package server

import "github.com/NeowayLabs/wabbit"

const (
	QueueMaxLen = 2 << 8
)

type Queue struct {
	name string
	data chan wabbit.Delivery
}

func NewQueue(name string) *Queue {
	return &Queue{
		name: name,
		data: make(chan wabbit.Delivery, QueueMaxLen),
	}
}

func (q *Queue) Consumers() int {
	return 0
}

func (q *Queue) Name() string {
	return q.name
}

func (q *Queue) Messages() int {
	return 0
}
<|endoftext|>


<commit_before>package teff

import (
	"bytes"
	"fmt"
	"h12.me/teff/core"
	"reflect"
	"strconv"
)

func Marshal(v interface{}) ([]byte, error) {
	if v == nil {
		return []byte("nil"), nil
	}
	list, err := marshalList(reflectValue(reflect.ValueOf(v)))
	if err != nil {
		return nil, err
	}
	return list.Marshal("", "\t")
}

func Unmarshal(data []byte, v interface{}) error {
	if string(data) == "nil" {
		return nil
	}
	list, err := core.Parse(bytes.NewBuffer(data))
	if err != nil {
		return err
	}
	return unmarshalList(list, reflect.ValueOf(v))
}

func marshalList(v reflect.Value) (core.List, error) {
	switch v.Type().Kind() {
	case reflect.Int, reflect.String:
		node, err := marshalNode(v)
		if err != nil {
			return nil, err
		}
		return core.List{node}, nil
	case reflect.Slice:
		list := make(core.List, v.Len())
		for i := 0; i < v.Len(); i++ {
			node, err := marshalNode(v.Index(i))
			if err != nil {
				return nil, err
			}
			list[i] = node
		}
		return list, nil
	}
	return nil, fmt.Errorf("marshal unsupported")
}

func marshalNode(v reflect.Value) (core.Node, error) {
	switch v.Type().Kind() {
	case reflect.Int:
		return core.Node{Value: fmt.Sprint(v.Interface())}, nil
	case reflect.String:
		return core.Node{Value: strconv.Quote(v.Interface().(string))}, nil
	case reflect.Ptr:
		return marshalNode(v.Elem())
	}
	return core.Node{}, fmt.Errorf("marshal unsupported")

}

func unmarshalList(list core.List, v reflect.Value) error {
	switch v.Type().Kind() {
	case reflect.Int, reflect.String:
		return unmarshalNode(list[0], v)
	case reflect.Slice:
		for i, node := range list {
			v.Set(reflect.Append(v, reflect.New(v.Type().Elem()).Elem()))
			elem := v.Index(i)
			if err := unmarshalNode(node, elem); err != nil {
				return err
			}
		}
		return nil
	case reflect.Ptr:
		return unmarshalList(list, allocValue(v))
	}
	return fmt.Errorf("unmarshal unsupported")
}

func unmarshalNode(node core.Node, v reflect.Value) error {
	switch v.Type().Kind() {
	case reflect.Int:
		i, err := strconv.Atoi(node.Value)
		if err != nil {
			return err
		}
		v.SetInt(int64(i))
		return nil
	case reflect.String:
		s, _ := strconv.Unquote(node.Value)
		v.SetString(s)
		return nil
	case reflect.Ptr:
		return unmarshalNode(node, allocValue(v))
	}
	return fmt.Errorf("unmarshal unsupported")
}

func reflectValue(v reflect.Value) reflect.Value {
	for v.Type().Kind() == reflect.Ptr && !v.IsNil() {
		v = reflect.Indirect(v)
	}
	return v
}

func allocValue(v reflect.Value) reflect.Value {
	for v.Type().Kind() == reflect.Ptr {
		if v.IsNil() {
			v.Set(reflect.New(v.Type().Elem()))
		}
		v = reflect.Indirect(v)
	}
	return v
}
<commit_msg>add MarshalIndent.<commit_after>package teff

import (
	"bytes"
	"fmt"
	"h12.me/teff/core"
	"reflect"
	"strconv"
)

func Marshal(v interface{}) ([]byte, error) {
	return MarshalIndent(v, "", "\t")
}

func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) {
	if v == nil {
		return []byte("nil"), nil
	}
	list, err := marshalList(reflectValue(reflect.ValueOf(v)))
	if err != nil {
		return nil, err
	}
	return list.Marshal(prefix, indent)
}

func Unmarshal(data []byte, v interface{}) error {
	if string(data) == "nil" {
		return nil
	}
	list, err := core.Parse(bytes.NewBuffer(data))
	if err != nil {
		return err
	}
	return unmarshalList(list, reflect.ValueOf(v))
}

func marshalList(v reflect.Value) (core.List, error) {
	switch v.Type().Kind() {
	case reflect.Int, reflect.String:
		node, err := marshalNode(v)
		if err != nil {
			return nil, err
		}
		return core.List{node}, nil
	case reflect.Slice:
		list := make(core.List, v.Len())
		for i := 0; i < v.Len(); i++ {
			node, err := marshalNode(v.Index(i))
			if err != nil {
				return nil, err
			}
			list[i] = node
		}
		return list, nil
	}
	return nil, fmt.Errorf("marshal unsupported")
}

func marshalNode(v reflect.Value) (core.Node, error) {
	switch v.Type().Kind() {
	case reflect.Int:
		return core.Node{Value: fmt.Sprint(v.Interface())}, nil
	case reflect.String:
		return core.Node{Value: strconv.Quote(v.Interface().(string))}, nil
	case reflect.Ptr:
		return marshalNode(v.Elem())
	}
	return core.Node{}, fmt.Errorf("marshal unsupported")

}

func unmarshalList(list core.List, v reflect.Value) error {
	switch v.Type().Kind() {
	case reflect.Int, reflect.String:
		return unmarshalNode(list[0], v)
	case reflect.Slice:
		for i, node := range list {
			v.Set(reflect.Append(v, reflect.New(v.Type().Elem()).Elem()))
			elem := v.Index(i)
			if err := unmarshalNode(node, elem); err != nil {
				return err
			}
		}
		return nil
	case reflect.Ptr:
		return unmarshalList(list, allocValue(v))
	}
	return fmt.Errorf("unmarshal unsupported")
}

func unmarshalNode(node core.Node, v reflect.Value) error {
	switch v.Type().Kind() {
	case reflect.Int:
		i, err := strconv.Atoi(node.Value)
		if err != nil {
			return err
		}
		v.SetInt(int64(i))
		return nil
	case reflect.String:
		s, _ := strconv.Unquote(node.Value)
		v.SetString(s)
		return nil
	case reflect.Ptr:
		return unmarshalNode(node, allocValue(v))
	}
	return fmt.Errorf("unmarshal unsupported")
}

func reflectValue(v reflect.Value) reflect.Value {
	for v.Type().Kind() == reflect.Ptr && !v.IsNil() {
		v = reflect.Indirect(v)
	}
	return v
}

func allocValue(v reflect.Value) reflect.Value {
	for v.Type().Kind() == reflect.Ptr {
		if v.IsNil() {
			v.Set(reflect.New(v.Type().Elem()))
		}
		v = reflect.Indirect(v)
	}
	return v
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"strconv"
	"time"

	"github.com/boltdb/bolt"
)

var (
	KEY_TOKEN       = []byte("token")
	KEY_FORWARD_URL = []byte("url")
	KEY_CAPACITY    = []byte("capacity")
	KEY_TOTAL_COUNT = []byte("total")
	KEY_REQUESTS    = []byte("requests")
)

/// Basket interface ///

type boltBasket struct {
}

/// BasketsDatabase interface ///

type boltDatabase struct {
	db *bolt.DB
}

func (bdb *boltDatabase) Create(name string, config BasketConfig) (BasketAuth, error) {
	auth := BasketAuth{}
	token, err := GenerateToken()
	if err != nil {
		return auth, fmt.Errorf("Failed to generate token: %s", err)
	}

	err = bdb.db.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucket([]byte(name))
		if err != nil {
			return fmt.Errorf("Failed to create bucket: %s - %s", name, err)
		}

		// initialize basket bucket (assume no issues arised)
		b.Put(KEY_TOKEN, []byte(token))
		b.Put(KEY_FORWARD_URL, []byte(config.ForwardUrl))
		b.Put(KEY_CAPACITY, []byte(strconv.Itoa(config.Capacity)))
		b.Put(KEY_TOTAL_COUNT, []byte("0"))
		b.CreateBucketIfNotExists(KEY_REQUESTS)

		return err
	})

	if err != nil {
		return auth, err
	}

	auth.Token = token

	return auth, nil
}

func (bdb *boltDatabase) Release() {
	log.Printf("Releasing bolt database resources")
	err := bdb.db.Close()
	if err != nil {
		log.Fatal(err)
	}
}

// NewBoltDatabase creates an instance of Baskets Database backed with Bolt DB
func NewBoltDatabase(file string) *boltDatabase {
	log.Printf("Bolt database location: %s", file)
	db, err := bolt.Open(file, 0600, &bolt.Options{Timeout: 10 * time.Second})
	if err != nil {
		log.Fatal(err)
		return nil
	}

	return &boltDatabase{db}
}
<commit_msg>bolt: implemented database interface, partially implemented basket interface<commit_after>package main

import (
	"fmt"
	"log"
	"strconv"
	"time"

	"github.com/boltdb/bolt"
)

var (
	KEY_TOKEN       = []byte("token")
	KEY_FORWARD_URL = []byte("url")
	KEY_CAPACITY    = []byte("capacity")
	KEY_TOTAL_COUNT = []byte("total")
	KEY_COUNT       = []byte("count")
	KEY_REQUESTS    = []byte("requests")
)

/// Basket interface ///

type boltBasket struct {
	db   *bolt.DB
	name string
}

func (basket *boltBasket) Config() BasketConfig {
	config := BasketConfig{}

	basket.db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(basket.name))
		if b != nil {
			url := b.Get(KEY_FORWARD_URL)
			cap := b.Get(KEY_CAPACITY)

			config.ForwardUrl = string(url[:])
			config.Capacity, _ = strconv.Atoi(string(cap[:]))
		} else {
			log.Printf("Failed to locate bucket: %s", basket.name)
		}

		return nil
	})

	return config
}

/// BasketsDatabase interface ///

type boltDatabase struct {
	db *bolt.DB
}

func (bdb *boltDatabase) Create(name string, config BasketConfig) (BasketAuth, error) {
	auth := BasketAuth{}
	token, err := GenerateToken()
	if err != nil {
		return auth, fmt.Errorf("Failed to generate token: %s", err)
	}

	err = bdb.db.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucket([]byte(name))
		if err != nil {
			return fmt.Errorf("Failed to create bucket: %s - %s", name, err)
		}

		// initialize basket bucket (assume no issues arised)
		b.Put(KEY_TOKEN, []byte(token))
		b.Put(KEY_FORWARD_URL, []byte(config.ForwardUrl))
		b.Put(KEY_CAPACITY, []byte(strconv.Itoa(config.Capacity)))
		b.Put(KEY_TOTAL_COUNT, []byte("0"))
		b.Put(KEY_COUNT, []byte("0"))
		b.CreateBucket(KEY_REQUESTS)

		return err
	})

	if err != nil {
		return auth, err
	}

	auth.Token = token

	return auth, nil
}

func (bdb *boltDatabase) Get(name string) *boltBasket {
	err := bdb.db.View(func(tx *bolt.Tx) error {
		if tx.Bucket([]byte(name)) != nil {
			return nil
		} else {
			return fmt.Errorf("No basket found: %s", name)
		}
	})

	if err != nil {
		log.Print(err)
		return nil
	} else {
		return &boltBasket{bdb.db, name}
	}
}

func (bdb *boltDatabase) Delete(name string) {
	err := bdb.db.Update(func(tx *bolt.Tx) error {
		return tx.DeleteBucket([]byte(name))
	})

	if err != nil {
		log.Print(err)
	}
}

func (bdb *boltDatabase) Size() int {
	// TODO : introduce bucket with statistics (e.g. "/stats", or ".stats"), see https://github.com/boltdb/bolt/issues/276
	size := 0

	bdb.db.View(func(tx *bolt.Tx) error {
		cur := tx.Cursor()
		for key, _ := cur.First(); key != nil; key, _ = cur.Next() {
			size++
		}
		return nil
	})

	return size
}

func (bdb *boltDatabase) GetNames(max int, skip int) BasketNamesPage {
	last := skip + max
	page := BasketNamesPage{make([]string, 0, max), 0, false}

	bdb.db.View(func(tx *bolt.Tx) error {
		cur := tx.Cursor()
		for key, _ := cur.First(); key != nil; key, _ = cur.Next() {
			if page.Count >= skip && page.Count < last {
				page.Names[len(page.Names)] = string(key[:])
			} else if page.Count >= last {
				page.HasMore = true
			}
			page.Count++
		}
		return nil
	})

	return page
}

func (bdb *boltDatabase) Release() {
	log.Printf("Releasing bolt database resources")
	err := bdb.db.Close()
	if err != nil {
		log.Print(err)
	}
}

// NewBoltDatabase creates an instance of Baskets Database backed with Bolt DB
func NewBoltDatabase(file string) *boltDatabase {
	log.Printf("Bolt database location: %s", file)
	db, err := bolt.Open(file, 0600, &bolt.Options{Timeout: 10 * time.Second})
	if err != nil {
		log.Fatal(err)
		return nil
	}

	return &boltDatabase{db}
}
<|endoftext|>


<commit_before>// Copyright 2013-2017 Aerospike, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aerospike

// BatchPolicy encapsulates parameters for policy attributes used in write operations.
// This object is passed into methods where database writes can occur.
type BatchPolicy struct {
	BasePolicy

	// Maximum number of concurrent batch request goroutines to server nodes at any point in time.
	// If there are 16 node/namespace combinations requested and ConcurrentNodes is 8,
	// then batch requests will be made for 8 node/namespace combinations in concurrent goroutines.
	// When a request completes, a new request will be issued until all 16 goroutines are complete.
	//
	// Values:
	// 1: Issue batch requests sequentially.  This mode has a performance advantage for small
	// to medium sized batch sizes because requests can be issued in the main transaction goroutine.
	// This is the default.
	// 0: Issue all batch requests in concurrent goroutines.  This mode has a performance
	// advantage for extremely large batch sizes because each node can process the request
	// immediately.  The downside is extra goroutines will need to be created (or taken from
	// a goroutine pool).
	// > 0: Issue up to ConcurrentNodes batch requests in concurrent goroutines.  When a request
	// completes, a new request will be issued until all goroutines are complete.  This mode
	// prevents too many concurrent goroutines being created for large cluster implementations.
	// The downside is extra goroutines will still need to be created (or taken from a goroutine pool).
	ConcurrentNodes int // = 1

	// Use old batch direct protocol where batch reads are handled by direct low-level batch server
	// database routines.  The batch direct protocol can be faster when there is a single namespace,
	// but there is one important drawback.  The batch direct protocol will not proxy to a different
	// server node when the mapped node has migrated a record to another node (resulting in not
	// found record).
	//
	// This can happen after a node has been added/removed from the cluster and there is a lag
	// between records being migrated and client partition map update (once per second).
	//
	// The new batch index protocol will perform this record proxy when necessary.
	UseBatchDirect bool // = false

	// Allow batch to be processed immediately in the server's receiving thread when the server
	// deems it to be appropriate.  If false, the batch will always be processed in separate
	// transaction goroutines.  This field is only relevant for the new batch index protocol.
	//
	// For batch exists or batch reads of smaller sized records (<= 1K per record), inline
	// processing will be significantly faster on "in memory" namespaces.  The server disables
	// inline processing on disk based namespaces regardless of this policy field.
	//
	// Inline processing can introduce the possibility of unfairness because the server
	// can process the entire batch before moving onto the next command.
	AllowInline bool //= true

	// AllowPartialResults determines if the results for some nodes should be returned in case
	// some nodes encounter an error. The result for the unreceived records will be nil.
	//
	// It is only supported for BatchGet and BatchGetHeader methods. BatchGetComplex always returns
	// partial results by design.
	AllowPartialResults bool //= false

	// Send set name field to server for every key in the batch for batch index protocol.
	// This is only necessary when authentication is enabled and security roles are defined
	// on a per set basis.
	SendSetName bool //= false
}

// NewBatchPolicy initializes a new BatchPolicy instance with default parameters.
func NewBatchPolicy() *BatchPolicy {
	return &BatchPolicy{
		ConcurrentNodes:     1,
		UseBatchDirect:      false,
		AllowInline:         true,
		AllowPartialResults: false,
		SendSetName:         false,
	}
}
<commit_msg>Use default BasePolicy values in BatchPolicy to avoid confusion<commit_after>// Copyright 2013-2017 Aerospike, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aerospike

// BatchPolicy encapsulates parameters for policy attributes used in write operations.
// This object is passed into methods where database writes can occur.
type BatchPolicy struct {
	BasePolicy

	// Maximum number of concurrent batch request goroutines to server nodes at any point in time.
	// If there are 16 node/namespace combinations requested and ConcurrentNodes is 8,
	// then batch requests will be made for 8 node/namespace combinations in concurrent goroutines.
	// When a request completes, a new request will be issued until all 16 goroutines are complete.
	//
	// Values:
	// 1: Issue batch requests sequentially.  This mode has a performance advantage for small
	// to medium sized batch sizes because requests can be issued in the main transaction goroutine.
	// This is the default.
	// 0: Issue all batch requests in concurrent goroutines.  This mode has a performance
	// advantage for extremely large batch sizes because each node can process the request
	// immediately.  The downside is extra goroutines will need to be created (or taken from
	// a goroutine pool).
	// > 0: Issue up to ConcurrentNodes batch requests in concurrent goroutines.  When a request
	// completes, a new request will be issued until all goroutines are complete.  This mode
	// prevents too many concurrent goroutines being created for large cluster implementations.
	// The downside is extra goroutines will still need to be created (or taken from a goroutine pool).
	ConcurrentNodes int // = 1

	// Use old batch direct protocol where batch reads are handled by direct low-level batch server
	// database routines.  The batch direct protocol can be faster when there is a single namespace,
	// but there is one important drawback.  The batch direct protocol will not proxy to a different
	// server node when the mapped node has migrated a record to another node (resulting in not
	// found record).
	//
	// This can happen after a node has been added/removed from the cluster and there is a lag
	// between records being migrated and client partition map update (once per second).
	//
	// The new batch index protocol will perform this record proxy when necessary.
	UseBatchDirect bool // = false

	// Allow batch to be processed immediately in the server's receiving thread when the server
	// deems it to be appropriate.  If false, the batch will always be processed in separate
	// transaction goroutines.  This field is only relevant for the new batch index protocol.
	//
	// For batch exists or batch reads of smaller sized records (<= 1K per record), inline
	// processing will be significantly faster on "in memory" namespaces.  The server disables
	// inline processing on disk based namespaces regardless of this policy field.
	//
	// Inline processing can introduce the possibility of unfairness because the server
	// can process the entire batch before moving onto the next command.
	AllowInline bool //= true

	// AllowPartialResults determines if the results for some nodes should be returned in case
	// some nodes encounter an error. The result for the unreceived records will be nil.
	//
	// It is only supported for BatchGet and BatchGetHeader methods. BatchGetComplex always returns
	// partial results by design.
	AllowPartialResults bool //= false

	// Send set name field to server for every key in the batch for batch index protocol.
	// This is only necessary when authentication is enabled and security roles are defined
	// on a per set basis.
	SendSetName bool //= false
}

// NewBatchPolicy initializes a new BatchPolicy instance with default parameters.
func NewBatchPolicy() *BatchPolicy {
	return &BatchPolicy{
		BasePolicy:          *NewPolicy(),
		ConcurrentNodes:     1,
		UseBatchDirect:      false,
		AllowInline:         true,
		AllowPartialResults: false,
		SendSetName:         false,
	}
}
<|endoftext|>


<commit_before>package memrecorder

import (
	"testing"
	"time"

	"github.com/docwhat/docker-gc/types"
)

func TestMatchesInterface(t *testing.T) {
	var _ types.Recorder = NewMemRecorder()
}

func TestSawImageTagAt(t *testing.T) {
	r := NewMemRecorder()

	tag := "flibbit"

	now := time.Now()
	then := time.Date(2009, 11, 17, 20, 34, 58, 651387237, time.UTC)

	r.SawImageTagAt(tag, then)
	r.Sweep(func(_ string, w time.Time) bool {
		if w != then {
			t.Error("The timestamp wasn't stored for the tag")
		}
		return false
	})

	r.SawImageTagAt(tag, now)
	r.Sweep(func(_ string, w time.Time) bool {
		if w != now {
			t.Errorf("The timestamp should be now, not %v", w)
		}
		return false
	})

	r.SawImageTagAt(tag, then)
	r.Sweep(func(_ string, w time.Time) bool {
		if w != now {
			t.Errorf("The timestamp should remain now, not %v", w)
		}
		return false
	})
}
<commit_msg>Added more memrecorder tests<commit_after>package memrecorder

import (
	"testing"
	"time"

	"github.com/docwhat/docker-gc/types"
)

func TestMatchesInterface(t *testing.T) {
	var _ types.Recorder = NewMemRecorder()
}

func TestSawImageTagAt(t *testing.T) {
	r := NewMemRecorder()

	tag := "flibbit"

	now := time.Now()
	then := time.Date(2009, 11, 17, 20, 34, 58, 651387237, time.UTC)

	r.SawImageTagAt(tag, then)
	r.Sweep(func(_ string, w time.Time) bool {
		if w != then {
			t.Error("The timestamp wasn't stored for the tag")
		}
		return false
	})

	r.SawImageTagAt(tag, now)
	r.Sweep(func(_ string, w time.Time) bool {
		if w != now {
			t.Errorf("The timestamp should be now, not %v", w)
		}
		return false
	})

	r.SawImageTagAt(tag, then)
	r.Sweep(func(_ string, w time.Time) bool {
		if w != now {
			t.Errorf("The timestamp should remain now, not %v", w)
		}
		return false
	})
}

func TestSawImageTag(t *testing.T) {
	r := NewMemRecorder()

	tag := "frobnitz"

	r.SawImageTag(tag)
	r.Sweep(func(found string, _ time.Time) bool {
		if found != tag {
			t.Error("I expected to find the tag")
		}
		return false
	})
}

func TestForget(t *testing.T) {
	r := NewMemRecorder()

	expectedTag := "frobnitz"
	unexpectedTag := "foobar"

	r.SawImageTag(expectedTag)
	r.SawImageTag(unexpectedTag)
	r.Forget(unexpectedTag)
	r.Sweep(func(found string, _ time.Time) bool {
		if found != expectedTag {
			t.Errorf("I expected to find the tag %v; found %v", expectedTag, found)
		}
		return false
	})
}

func TestSweep(t *testing.T) {
	r := NewMemRecorder()

	tag := "frobnitz"

	r.SawImageTag(tag)

	r.Sweep(func(_ string, _ time.Time) bool {
		return false
	})

	r.Sweep(func(found string, _ time.Time) bool {
		if found != tag {
			t.Error("I expected to find the tag")
		}
		return true
	})

	r.Sweep(func(_ string, _ time.Time) bool {
		t.Error("I shouldn't find any tags")
		return false
	})
}
<|endoftext|>


<commit_before>package vmware

import (
	"errors"
	"fmt"
	"github.com/mitchellh/mapstructure"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/builder/common"
	"github.com/mitchellh/packer/packer"
	"log"
	"math/rand"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"
)

const BuilderId = "mitchellh.vmware"

type Builder struct {
	config config
	driver Driver
	runner multistep.Runner
}

type config struct {
	DiskName        string            `mapstructure:"vmdk_name"`
	DiskSize        uint              `mapstructure:"disk_size"`
	GuestOSType     string            `mapstructure:"guest_os_type"`
	ISOMD5          string            `mapstructure:"iso_md5"`
	ISOUrl          string            `mapstructure:"iso_url"`
	VMName          string            `mapstructure:"vm_name"`
	OutputDir       string            `mapstructure:"output_directory"`
	HTTPDir         string            `mapstructure:"http_directory"`
	HTTPPortMin     uint              `mapstructure:"http_port_min"`
	HTTPPortMax     uint              `mapstructure:"http_port_max"`
	BootCommand     []string          `mapstructure:"boot_command"`
	BootWait        time.Duration     ``
	ShutdownCommand string            `mapstructure:"shutdown_command"`
	ShutdownTimeout time.Duration     ``
	SSHUser         string            `mapstructure:"ssh_username"`
	SSHPassword     string            `mapstructure:"ssh_password"`
	SSHPort         uint              `mapstructure:"ssh_port"`
	SSHWaitTimeout  time.Duration     ``
	VMXData         map[string]string `mapstructure:"vmx_data"`
	VNCPortMin      uint              `mapstructure:"vnc_port_min"`
	VNCPortMax      uint              `mapstructure:"vnc_port_max"`

	PackerDebug bool `mapstructure:"packer_debug"`

	RawBootWait        string `mapstructure:"boot_wait"`
	RawShutdownTimeout string `mapstructure:"shutdown_timeout"`
	RawSSHWaitTimeout  string `mapstructure:"ssh_wait_timeout"`
}

func (b *Builder) Prepare(raws ...interface{}) error {
	for _, raw := range raws {
		err := mapstructure.Decode(raw, &b.config)
		if err != nil {
			return err
		}
	}

	if b.config.DiskName == "" {
		b.config.DiskName = "disk"
	}

	if b.config.DiskSize == 0 {
		b.config.DiskSize = 40000
	}

	if b.config.GuestOSType == "" {
		b.config.GuestOSType = "other"
	}

	if b.config.VMName == "" {
		b.config.VMName = "packer"
	}

	if b.config.HTTPPortMin == 0 {
		b.config.HTTPPortMin = 8000
	}

	if b.config.HTTPPortMax == 0 {
		b.config.HTTPPortMax = 9000
	}

	if b.config.VNCPortMin == 0 {
		b.config.VNCPortMin = 5900
	}

	if b.config.VNCPortMax == 0 {
		b.config.VNCPortMax = 6000
	}

	if b.config.OutputDir == "" {
		b.config.OutputDir = "vmware"
	}

	if b.config.SSHPort == 0 {
		b.config.SSHPort = 22
	}

	// Accumulate any errors
	var err error
	errs := make([]error, 0)

	if b.config.HTTPPortMin > b.config.HTTPPortMax {
		errs = append(errs, errors.New("http_port_min must be less than http_port_max"))
	}

	if b.config.ISOMD5 == "" {
		errs = append(errs, errors.New("Due to large file sizes, an iso_md5 is required"))
	} else {
		b.config.ISOMD5 = strings.ToLower(b.config.ISOMD5)
	}

	if b.config.ISOUrl == "" {
		errs = append(errs, errors.New("An iso_url must be specified."))
	} else {
		url, err := url.Parse(b.config.ISOUrl)
		if err != nil {
			errs = append(errs, fmt.Errorf("iso_url is not a valid URL: %s", err))
		} else {
			if url.Scheme == "" {
				url.Scheme = "file"
			}

			if url.Scheme == "file" {
				if _, err := os.Stat(b.config.ISOUrl); err != nil {
					errs = append(errs, fmt.Errorf("iso_url points to bad file: %s", err))
				}
			} else {
				supportedSchemes := []string{"file", "http", "https"}
				scheme := strings.ToLower(url.Scheme)

				found := false
				for _, supported := range supportedSchemes {
					if scheme == supported {
						found = true
						break
					}
				}

				if !found {
					errs = append(errs, fmt.Errorf("Unsupported URL scheme in iso_url: %s", scheme))
				}
			}
		}

		if len(errs) == 0 {
			// Put the URL back together since we may have modified it
			b.config.ISOUrl = url.String()
		}
	}

	if _, err := os.Stat(b.config.OutputDir); err == nil {
		errs = append(errs, errors.New("Output directory already exists. It must not exist."))
	}

	if b.config.SSHUser == "" {
		errs = append(errs, errors.New("An ssh_username must be specified."))
	}

	if b.config.RawBootWait != "" {
		b.config.BootWait, err = time.ParseDuration(b.config.RawBootWait)
		if err != nil {
			errs = append(errs, fmt.Errorf("Failed parsing boot_wait: %s", err))
		}
	}

	if b.config.RawShutdownTimeout == "" {
		b.config.RawShutdownTimeout = "5m"
	}

	b.config.ShutdownTimeout, err = time.ParseDuration(b.config.RawShutdownTimeout)
	if err != nil {
		errs = append(errs, fmt.Errorf("Failed parsing shutdown_timeout: %s", err))
	}

	if b.config.RawSSHWaitTimeout == "" {
		b.config.RawSSHWaitTimeout = "20m"
	}

	b.config.SSHWaitTimeout, err = time.ParseDuration(b.config.RawSSHWaitTimeout)
	if err != nil {
		errs = append(errs, fmt.Errorf("Failed parsing ssh_wait_timeout: %s", err))
	}

	if b.config.VNCPortMin > b.config.VNCPortMax {
		errs = append(errs, fmt.Errorf("vnc_port_min must be less than vnc_port_max"))
	}

	b.driver, err = b.newDriver()
	if err != nil {
		errs = append(errs, fmt.Errorf("Failed creating VMware driver: %s", err))
	}

	if len(errs) > 0 {
		return &packer.MultiError{errs}
	}

	return nil
}

func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) {
	// Seed the random number generator
	rand.Seed(time.Now().UTC().UnixNano())

	steps := []multistep.Step{
		&stepDownloadISO{},
		&stepPrepareOutputDir{},
		&stepCreateDisk{},
		&stepCreateVMX{},
		&stepHTTPServer{},
		&stepConfigureVNC{},
		&stepRun{},
		&stepTypeBootCommand{},
		&stepWaitForSSH{},
		&stepProvision{},
		&stepShutdown{},
	}

	// Setup the state bag
	state := make(map[string]interface{})
	state["cache"] = cache
	state["config"] = &b.config
	state["driver"] = b.driver
	state["hook"] = hook
	state["ui"] = ui

	// Run!
	if b.config.PackerDebug {
		b.runner = &multistep.DebugRunner{
			Steps:   steps,
			PauseFn: common.MultistepDebugFn(ui),
		}
	} else {
		b.runner = &multistep.BasicRunner{Steps: steps}
	}

	b.runner.Run(state)

	// If there was an error, return that
	if rawErr, ok := state["error"]; ok {
		return nil, rawErr.(error)
	}

	// If we were interrupted or cancelled, then just exit.
	if _, ok := state[multistep.StateCancelled]; ok {
		return nil, errors.New("Build was cancelled.")
	}

	if _, ok := state[multistep.StateHalted]; ok {
		return nil, errors.New("Build was halted.")
	}

	// Compile the artifact list
	files := make([]string, 0, 10)
	visit := func(path string, info os.FileInfo, err error) error {
		files = append(files, path)
		return err
	}

	if err := filepath.Walk(b.config.OutputDir, visit); err != nil {
		return nil, err
	}

	return &Artifact{b.config.OutputDir, files}, nil
}

func (b *Builder) Cancel() {
	if b.runner != nil {
		log.Println("Cancelling the step runner...")
		b.runner.Cancel()
	}
}

func (b *Builder) newDriver() (Driver, error) {
	fusionAppPath := "/Applications/VMware Fusion.app"
	driver := &Fusion5Driver{fusionAppPath}
	if err := driver.Verify(); err != nil {
		return nil, err
	}

	return driver, nil
}
<commit_msg>builder/vmware: properly test file URLs<commit_after>package vmware

import (
	"errors"
	"fmt"
	"github.com/mitchellh/mapstructure"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/builder/common"
	"github.com/mitchellh/packer/packer"
	"log"
	"math/rand"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"
)

const BuilderId = "mitchellh.vmware"

type Builder struct {
	config config
	driver Driver
	runner multistep.Runner
}

type config struct {
	DiskName        string            `mapstructure:"vmdk_name"`
	DiskSize        uint              `mapstructure:"disk_size"`
	GuestOSType     string            `mapstructure:"guest_os_type"`
	ISOMD5          string            `mapstructure:"iso_md5"`
	ISOUrl          string            `mapstructure:"iso_url"`
	VMName          string            `mapstructure:"vm_name"`
	OutputDir       string            `mapstructure:"output_directory"`
	HTTPDir         string            `mapstructure:"http_directory"`
	HTTPPortMin     uint              `mapstructure:"http_port_min"`
	HTTPPortMax     uint              `mapstructure:"http_port_max"`
	BootCommand     []string          `mapstructure:"boot_command"`
	BootWait        time.Duration     ``
	ShutdownCommand string            `mapstructure:"shutdown_command"`
	ShutdownTimeout time.Duration     ``
	SSHUser         string            `mapstructure:"ssh_username"`
	SSHPassword     string            `mapstructure:"ssh_password"`
	SSHPort         uint              `mapstructure:"ssh_port"`
	SSHWaitTimeout  time.Duration     ``
	VMXData         map[string]string `mapstructure:"vmx_data"`
	VNCPortMin      uint              `mapstructure:"vnc_port_min"`
	VNCPortMax      uint              `mapstructure:"vnc_port_max"`

	PackerDebug bool `mapstructure:"packer_debug"`

	RawBootWait        string `mapstructure:"boot_wait"`
	RawShutdownTimeout string `mapstructure:"shutdown_timeout"`
	RawSSHWaitTimeout  string `mapstructure:"ssh_wait_timeout"`
}

func (b *Builder) Prepare(raws ...interface{}) error {
	for _, raw := range raws {
		err := mapstructure.Decode(raw, &b.config)
		if err != nil {
			return err
		}
	}

	if b.config.DiskName == "" {
		b.config.DiskName = "disk"
	}

	if b.config.DiskSize == 0 {
		b.config.DiskSize = 40000
	}

	if b.config.GuestOSType == "" {
		b.config.GuestOSType = "other"
	}

	if b.config.VMName == "" {
		b.config.VMName = "packer"
	}

	if b.config.HTTPPortMin == 0 {
		b.config.HTTPPortMin = 8000
	}

	if b.config.HTTPPortMax == 0 {
		b.config.HTTPPortMax = 9000
	}

	if b.config.VNCPortMin == 0 {
		b.config.VNCPortMin = 5900
	}

	if b.config.VNCPortMax == 0 {
		b.config.VNCPortMax = 6000
	}

	if b.config.OutputDir == "" {
		b.config.OutputDir = "vmware"
	}

	if b.config.SSHPort == 0 {
		b.config.SSHPort = 22
	}

	// Accumulate any errors
	var err error
	errs := make([]error, 0)

	if b.config.HTTPPortMin > b.config.HTTPPortMax {
		errs = append(errs, errors.New("http_port_min must be less than http_port_max"))
	}

	if b.config.ISOMD5 == "" {
		errs = append(errs, errors.New("Due to large file sizes, an iso_md5 is required"))
	} else {
		b.config.ISOMD5 = strings.ToLower(b.config.ISOMD5)
	}

	if b.config.ISOUrl == "" {
		errs = append(errs, errors.New("An iso_url must be specified."))
	} else {
		url, err := url.Parse(b.config.ISOUrl)
		if err != nil {
			errs = append(errs, fmt.Errorf("iso_url is not a valid URL: %s", err))
		} else {
			if url.Scheme == "" {
				url.Scheme = "file"
			}

			if url.Scheme == "file" {
				if _, err := os.Stat(url.Path); err != nil {
					errs = append(errs, fmt.Errorf("iso_url points to bad file: %s", err))
				}
			} else {
				supportedSchemes := []string{"file", "http", "https"}
				scheme := strings.ToLower(url.Scheme)

				found := false
				for _, supported := range supportedSchemes {
					if scheme == supported {
						found = true
						break
					}
				}

				if !found {
					errs = append(errs, fmt.Errorf("Unsupported URL scheme in iso_url: %s", scheme))
				}
			}
		}

		if len(errs) == 0 {
			// Put the URL back together since we may have modified it
			b.config.ISOUrl = url.String()
		}
	}

	if _, err := os.Stat(b.config.OutputDir); err == nil {
		errs = append(errs, errors.New("Output directory already exists. It must not exist."))
	}

	if b.config.SSHUser == "" {
		errs = append(errs, errors.New("An ssh_username must be specified."))
	}

	if b.config.RawBootWait != "" {
		b.config.BootWait, err = time.ParseDuration(b.config.RawBootWait)
		if err != nil {
			errs = append(errs, fmt.Errorf("Failed parsing boot_wait: %s", err))
		}
	}

	if b.config.RawShutdownTimeout == "" {
		b.config.RawShutdownTimeout = "5m"
	}

	b.config.ShutdownTimeout, err = time.ParseDuration(b.config.RawShutdownTimeout)
	if err != nil {
		errs = append(errs, fmt.Errorf("Failed parsing shutdown_timeout: %s", err))
	}

	if b.config.RawSSHWaitTimeout == "" {
		b.config.RawSSHWaitTimeout = "20m"
	}

	b.config.SSHWaitTimeout, err = time.ParseDuration(b.config.RawSSHWaitTimeout)
	if err != nil {
		errs = append(errs, fmt.Errorf("Failed parsing ssh_wait_timeout: %s", err))
	}

	if b.config.VNCPortMin > b.config.VNCPortMax {
		errs = append(errs, fmt.Errorf("vnc_port_min must be less than vnc_port_max"))
	}

	b.driver, err = b.newDriver()
	if err != nil {
		errs = append(errs, fmt.Errorf("Failed creating VMware driver: %s", err))
	}

	if len(errs) > 0 {
		return &packer.MultiError{errs}
	}

	return nil
}

func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) {
	// Seed the random number generator
	rand.Seed(time.Now().UTC().UnixNano())

	steps := []multistep.Step{
		&stepDownloadISO{},
		&stepPrepareOutputDir{},
		&stepCreateDisk{},
		&stepCreateVMX{},
		&stepHTTPServer{},
		&stepConfigureVNC{},
		&stepRun{},
		&stepTypeBootCommand{},
		&stepWaitForSSH{},
		&stepProvision{},
		&stepShutdown{},
	}

	// Setup the state bag
	state := make(map[string]interface{})
	state["cache"] = cache
	state["config"] = &b.config
	state["driver"] = b.driver
	state["hook"] = hook
	state["ui"] = ui

	// Run!
	if b.config.PackerDebug {
		b.runner = &multistep.DebugRunner{
			Steps:   steps,
			PauseFn: common.MultistepDebugFn(ui),
		}
	} else {
		b.runner = &multistep.BasicRunner{Steps: steps}
	}

	b.runner.Run(state)

	// If there was an error, return that
	if rawErr, ok := state["error"]; ok {
		return nil, rawErr.(error)
	}

	// If we were interrupted or cancelled, then just exit.
	if _, ok := state[multistep.StateCancelled]; ok {
		return nil, errors.New("Build was cancelled.")
	}

	if _, ok := state[multistep.StateHalted]; ok {
		return nil, errors.New("Build was halted.")
	}

	// Compile the artifact list
	files := make([]string, 0, 10)
	visit := func(path string, info os.FileInfo, err error) error {
		files = append(files, path)
		return err
	}

	if err := filepath.Walk(b.config.OutputDir, visit); err != nil {
		return nil, err
	}

	return &Artifact{b.config.OutputDir, files}, nil
}

func (b *Builder) Cancel() {
	if b.runner != nil {
		log.Println("Cancelling the step runner...")
		b.runner.Cancel()
	}
}

func (b *Builder) newDriver() (Driver, error) {
	fusionAppPath := "/Applications/VMware Fusion.app"
	driver := &Fusion5Driver{fusionAppPath}
	if err := driver.Verify(); err != nil {
		return nil, err
	}

	return driver, nil
}
<|endoftext|>


<commit_before>/*
Tiny web server to handle slack outgoing webhook and push the data to resque
*/
package main

import (
	"fmt"
	"net/http"
	"net/url"
	"encoding/json"
	"strings"
	"github.com/fzzy/radix"
)

func main() {
	http.HandleFunc("/api", apiHandler)
	http.ListenAndServe(":8765", nil)
}

func apiHandler(w http.ResponseWriter, r *http.Request) {
	type Response struct {
		Text string `json:"text"`
	}

	body := make([]byte, r.ContentLength)

	if r.Method == "POST" {
		r.Body.Read(body)
		data, err := url.ParseQuery(string(body))
		if err != nil {
			fmt.Println("error:", err)
		}
		res := Response{
			Text: "@" + data.Get("user_name") + ": " + strings.TrimPrefix(data.Get("text"), "slackops: "),
		}
		jres, err := json.Marshal(res)
		if err != nil {
			fmt.Println("error:", err)
		}
		w.Header().Set("Content-type", "application/json")
		fmt.Fprintf(w, string(jres))
	}
}
<commit_msg>push job to redis<commit_after>/*
Tiny web server to handle slack outgoing webhook and push the data to resque
*/
package main

import (
	"fmt"
	"net/http"
	"net/url"
	"encoding/json"
	"strings"
	"github.com/fzzy/radix/redis"
)

// some default global variable
var (
	PORT string = ":8765"
	API_PATH string = "/api"
	REDIS string = "localhost:6379"
	CLASS string = "SlackOPS"
	QUEUE string = "slackops"
)

func main() {
	// connect to redis and add queue
	rcon, err := redis.Dial("tcp", REDIS)
	if err != nil {
		fmt.Println("error:", err)
	}
	rcon.Cmd("SADD", "resque:queues", QUEUE)
	rcon.Close()

	// start web app
	http.HandleFunc(API_PATH, apiHandler)
	http.ListenAndServe(PORT, nil)
}

func apiHandler(w http.ResponseWriter, r *http.Request) {
	type Response struct {
		Text string `json:"text"`
	}

	type Job struct {
		Class string `json:"class"`
		Vars map[string]string `json:"vars"`
	}

	body := make([]byte, r.ContentLength)

	// accept only POST
	if r.Method == "POST" {
		// make connection to redis
		rcon, err := redis.Dial("tcp", REDIS)
		if err != nil {
			fmt.Println("error:", err)
		}

		// read and parse request body content
		r.Body.Read(body)
		data, err := url.ParseQuery(string(body))
		if err != nil {
			fmt.Println("error:", err)
		}

		// create job
		job := Job{
			Class: CLASS,
			Vars: map[string]string {
			"request": strings.TrimPrefix(data.Get("text"), data.Get("trigger_word")),
			"user": data.Get("user_name"),
			"channel": data.Get("channel_name"),
			"timestamp": data.Get("timestamp"),
			},
		}
		jjob, err := json.Marshal(job)
		if err != nil {
			fmt.Println("error:", err)
		}
		// push job to resque
		queue := "resque:queue:" + QUEUE
		rcon.Cmd("RPUSH", queue, string(jjob))
		rcon.Close()

		// response to slack
		res := Response{
			Text: "@" + data.Get("user_name") + ": " + strings.TrimPrefix(data.Get("text"), "slackops: "),
		}
		jres, err := json.Marshal(res)
		if err != nil {
			fmt.Println("error:", err)
		}
		w.Header().Set("Content-type", "application/json")
		fmt.Fprintf(w, string(jres))
	}
}
<|endoftext|>


<commit_before>package crawler

import (
	"encoding/json"
	"fmt"
	"github.com/dokterbob/ipfs-search/indexer"
	"github.com/dokterbob/ipfs-search/queue"
	"gopkg.in/ipfs/go-ipfs-api.v1"
	"log"
	"os"
	"os/exec"
	"strings"
)

type CrawlerArgs struct {
	Hash       string
	Name       string
	Size       uint64
	ParentHash string
	ParentName string
}

type Crawler struct {
	sh *shell.Shell
	id *indexer.Indexer
	fq *queue.TaskQueue
	hq *queue.TaskQueue
}

func NewCrawler(sh *shell.Shell, id *indexer.Indexer, fq *queue.TaskQueue, hq *queue.TaskQueue) *Crawler {
	return &Crawler{
		sh: sh,
		id: id,
		fq: fq,
		hq: hq,
	}
}

func hashUrl(hash string) string {
	return fmt.Sprintf("/ipfs/%s", hash)
}

// Helper function for creating reference structure
/*
	'<hash>': {
		'references': {
			[{
				'parent_hash'
				'hash'
				'name'
			}, ]
		}
	}

	if (document_exists) {
		if (references_exists) {
			add_parent_hash to references
		} else {
			add references to document
		}
	} else {
		create document with references as only information
	}
*/
func construct_references(name string, parent_hash string, parent_name string) []map[string]interface{} {
	references := []map[string]interface{}{}

	if name != "" {
		references = []map[string]interface{}{
			{
				"name":        name,
				"parent_hash": parent_hash,
				"parent_name": parent_name,
			},
		}
	}

	return references
}

// Given a particular hash (file or directory), start crawling
func (c Crawler) CrawlHash(hash string, name string, parent_hash string, parent_name string) error {
	indexed, err := c.id.IsIndexed(hash)
	if err != nil {
		return err
	}

	if indexed {
		log.Printf("Already indexed '%s', skipping", hash)
		return nil
	}

	log.Printf("Crawling hash '%s' (%s)", hash, name)

	url := hashUrl(hash)

	list, err := c.sh.FileList(url)
	if _, ok := err.(*shell.Error); ok && strings.Contains(err.Error(), "proto") {
		// We're not recovering from protocol errors, so panic
		panic(err)

	}
	if err != nil {
		return err
	}

	switch list.Type {
	case "File":
		// Add to file crawl queue
		// Note: we're expecting no references here, see comment below
		args := CrawlerArgs{
			Hash: hash,
			Name: name,
			Size: list.Size,
		}

		err = c.fq.AddTask(args)
		if err != nil {
			// failed to send the task
			return err
		}
	case "Directory":
		// Index name and size for directory and directory items
		properties := map[string]interface{}{
			"links":      list.Links,
			"size":       list.Size,
			"references": construct_references(name, parent_hash, parent_name),
		}

		c.id.IndexItem("directory", hash, properties)

		for _, link := range list.Links {
			args := CrawlerArgs{
				Hash:       link.Hash,
				Name:       link.Name,
				Size:       link.Size,
				ParentHash: hash,
				ParentName: name,
			}

			switch link.Type {
			case "File":
				// Add file to crawl queue
				err = c.fq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}

			case "Directory":
				// Add directory to crawl queue
				c.hq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}
			default:
				log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
			}
		}
	default:
		log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
	}

	log.Printf("Finished hash %s", hash)

	return nil
}

func getMetadata(path string, metadata *map[string]interface{}) error {
	cmd := exec.Command("tika", "-j", path)

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	// Standard error to system standard error
	cmd.Stderr = os.Stderr

	if err := cmd.Start(); err != nil {
		return err
	}

	if err := json.NewDecoder(stdout).Decode(&metadata); err != nil {
		return err
	}

	return cmd.Wait()
}

// Crawl a single object, known to be a file
func (c Crawler) CrawlFile(hash string, name string, parent_hash string, parent_name string, size uint64) error {
	log.Printf("Crawling file %s\n", hash)

	metadata := make(map[string]interface{})

	if size > 0 {
		var path string
		if name != "" && parent_hash != "" {
			path = fmt.Sprintf("/ipfs/%s/%s", parent_hash, name)
		} else {
			path = fmt.Sprintf("/ipfs/%s", hash)
		}

		if err := getMetadata(path, &metadata); err != nil {
			return err
		}
	}

	metadata["size"] = size
	metadata["references"] = construct_references(name, parent_hash, parent_name)

	c.id.IndexItem("file", hash, metadata)

	log.Printf("Finished file %s", hash)

	return nil
}
<commit_msg>Only index files once.<commit_after>package crawler

import (
	"encoding/json"
	"fmt"
	"github.com/dokterbob/ipfs-search/indexer"
	"github.com/dokterbob/ipfs-search/queue"
	"gopkg.in/ipfs/go-ipfs-api.v1"
	"log"
	"os"
	"os/exec"
	"strings"
)

type CrawlerArgs struct {
	Hash       string
	Name       string
	Size       uint64
	ParentHash string
	ParentName string
}

type Crawler struct {
	sh *shell.Shell
	id *indexer.Indexer
	fq *queue.TaskQueue
	hq *queue.TaskQueue
}

func NewCrawler(sh *shell.Shell, id *indexer.Indexer, fq *queue.TaskQueue, hq *queue.TaskQueue) *Crawler {
	return &Crawler{
		sh: sh,
		id: id,
		fq: fq,
		hq: hq,
	}
}

func hashUrl(hash string) string {
	return fmt.Sprintf("/ipfs/%s", hash)
}

// Helper function for creating reference structure
/*
	'<hash>': {
		'references': {
			[{
				'parent_hash'
				'hash'
				'name'
			}, ]
		}
	}

	if (document_exists) {
		if (references_exists) {
			add_parent_hash to references
		} else {
			add references to document
		}
	} else {
		create document with references as only information
	}
*/
func construct_references(name string, parent_hash string, parent_name string) []map[string]interface{} {
	references := []map[string]interface{}{}

	if name != "" {
		references = []map[string]interface{}{
			{
				"name":        name,
				"parent_hash": parent_hash,
				"parent_name": parent_name,
			},
		}
	}

	return references
}

// Given a particular hash (file or directory), start crawling
func (c Crawler) CrawlHash(hash string, name string, parent_hash string, parent_name string) error {
	indexed, err := c.id.IsIndexed(hash)
	if err != nil {
		return err
	}

	if indexed {
		log.Printf("Already indexed '%s', skipping", hash)
		return nil
	}

	log.Printf("Crawling hash '%s' (%s)", hash, name)

	url := hashUrl(hash)

	list, err := c.sh.FileList(url)
	if _, ok := err.(*shell.Error); ok && strings.Contains(err.Error(), "proto") {
		// We're not recovering from protocol errors, so panic
		panic(err)

	}
	if err != nil {
		return err
	}

	switch list.Type {
	case "File":
		// Add to file crawl queue
		// Note: we're expecting no references here, see comment below
		args := CrawlerArgs{
			Hash: hash,
			Name: name,
			Size: list.Size,
		}

		err = c.fq.AddTask(args)
		if err != nil {
			// failed to send the task
			return err
		}
	case "Directory":
		// Index name and size for directory and directory items
		properties := map[string]interface{}{
			"links":      list.Links,
			"size":       list.Size,
			"references": construct_references(name, parent_hash, parent_name),
		}

		c.id.IndexItem("directory", hash, properties)

		for _, link := range list.Links {
			args := CrawlerArgs{
				Hash:       link.Hash,
				Name:       link.Name,
				Size:       link.Size,
				ParentHash: hash,
				ParentName: name,
			}

			switch link.Type {
			case "File":
				// Add file to crawl queue
				err = c.fq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}

			case "Directory":
				// Add directory to crawl queue
				c.hq.AddTask(args)
				if err != nil {
					// failed to send the task
					return err
				}
			default:
				log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
			}
		}
	default:
		log.Printf("Type '%s' skipped for '%s'", list.Type, hash)
	}

	log.Printf("Finished hash %s", hash)

	return nil
}

func getMetadata(path string, metadata *map[string]interface{}) error {
	cmd := exec.Command("tika", "-j", path)

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	// Standard error to system standard error
	cmd.Stderr = os.Stderr

	if err := cmd.Start(); err != nil {
		return err
	}

	if err := json.NewDecoder(stdout).Decode(&metadata); err != nil {
		return err
	}

	return cmd.Wait()
}

// Crawl a single object, known to be a file
func (c Crawler) CrawlFile(hash string, name string, parent_hash string, parent_name string, size uint64) error {
	indexed, err := c.id.IsIndexed(hash)
	if err != nil {
		return err
	}

	if indexed {
		log.Printf("Already indexed '%s', skipping", hash)
		return nil
	}

	log.Printf("Crawling file %s\n", hash)

	metadata := make(map[string]interface{})

	if size > 0 {
		var path string
		if name != "" && parent_hash != "" {
			path = fmt.Sprintf("/ipfs/%s/%s", parent_hash, name)
		} else {
			path = fmt.Sprintf("/ipfs/%s", hash)
		}

		if err := getMetadata(path, &metadata); err != nil {
			return err
		}
	}

	metadata["size"] = size
	metadata["references"] = construct_references(name, parent_hash, parent_name)

	c.id.IndexItem("file", hash, metadata)

	log.Printf("Finished file %s", hash)

	return nil
}
<|endoftext|>


<commit_before>package xml // import "github.com/tdewolff/parse/xml"

// TODO: optimize

import (
	"io"
	"strconv"

	"github.com/tdewolff/buffer"
	"github.com/tdewolff/parse"
)

////////////////////////////////////////////////////////////////

// TokenType determines the type of token, eg. a number or a semicolon.
type TokenType uint32

// TokenType values.
const (
	ErrorToken TokenType = iota // extra token when errors occur
	CommentToken
	CDATAToken
	StartTagToken
	StartTagCloseToken
	StartTagCloseVoidToken
	StartTagClosePIToken
	EndTagToken
	AttributeToken
	TextToken
)

// String returns the string representation of a TokenType.
func (tt TokenType) String() string {
	switch tt {
	case ErrorToken:
		return "Error"
	case CommentToken:
		return "Comment"
	case CDATAToken:
		return "CDATA"
	case StartTagToken:
		return "StartTag"
	case StartTagCloseToken:
		return "StartTagClose"
	case StartTagCloseVoidToken:
		return "StartTagCloseVoid"
	case StartTagClosePIToken:
		return "StartTagClosePI"
	case EndTagToken:
		return "EndTag"
	case AttributeToken:
		return "Attribute"
	case TextToken:
		return "Text"
	}
	return "Invalid(" + strconv.Itoa(int(tt)) + ")"
}

////////////////////////////////////////////////////////////////

// Tokenizer is the state for the tokenizer.
type Tokenizer struct {
	r *buffer.Shifter

	inTag   bool
	attrVal []byte
}

// NewTokenizer returns a new Tokenizer for a given io.Reader.
func NewTokenizer(r io.Reader) *Tokenizer {
	return &Tokenizer{
		r: buffer.NewShifter(r),
	}
}

// Err returns the error encountered during tokenization, this is often io.EOF but also other errors can be returned.
func (z Tokenizer) Err() error {
	return z.r.Err()
}

// IsEOF returns true when it has encountered EOF and thus loaded the last buffer in memory.
func (z Tokenizer) IsEOF() bool {
	return z.r.IsEOF()
}

// Next returns the next Token. It returns ErrorToken when an error was encountered. Using Err() one can retrieve the error message.
func (z *Tokenizer) Next() (TokenType, []byte) {
	var c byte
	if z.inTag {
		z.attrVal = nil
		c = z.r.Peek(0)
		if c != '>' && (c != '/' && c != '?' || z.r.Peek(1) != '>') {
			return AttributeToken, z.shiftAttribute()
		}
		z.inTag = false
		if c == '/' {
			z.r.Move(2)
			return StartTagCloseVoidToken, z.r.Shift()
		} else if c == '?' {
			z.r.Move(2)
			return StartTagClosePIToken, z.r.Shift()
		} else {
			z.r.Move(1)
			return StartTagCloseToken, z.r.Shift()
		}
	}

	for {
		c = z.r.Peek(0)
		if c == 0 {
			if z.r.Pos() > 0 {
				return TextToken, z.r.Shift()
			}
			return ErrorToken, []byte{}
		} else if c == '<' {
			if z.r.Pos() > 0 {
				return TextToken, z.r.Shift()
			}
			c = z.r.Peek(1)
			if c == '/' {
				z.r.Move(2)
				z.r.Skip()
				return EndTagToken, z.shiftEndTag()
			} else if c == '!' {
				z.r.Move(2)
				if z.at('-', '-') {
					z.r.Move(2)
					return CommentToken, z.shiftCommentText()
				} else if z.at('[', 'C', 'D', 'A', 'T', 'A', '[') {
					z.r.Move(7)
					return CDATAToken, z.shiftCDATAText()
				}
				z.r.Move(-2)
			}
			z.r.Move(1)
			z.r.Skip()
			z.inTag = true
			return StartTagToken, z.shiftStartTag()
		}
		z.r.Move(1)
	}
	return ErrorToken, []byte{}
}

func (z *Tokenizer) AttrVal() []byte {
	return z.attrVal
}

////////////////////////////////////////////////////////////////

/*
The following functions follow the specifications at http://www.w3.org/html/wg/drafts/html/master/syntax.html
*/

func (z *Tokenizer) shiftCDATAText() []byte {
	z.r.Skip()
	for {
		if z.r.Peek(0) == 0 {
			return z.r.Shift()
		} else if z.at(']', ']', '>') {
			cdata := z.r.Shift()
			z.r.Move(3)
			z.r.Skip()
			return cdata
		}
		z.r.Move(1)
	}
}

func (z *Tokenizer) shiftCommentText() []byte {
	z.r.Skip()
	for {
		if z.r.Peek(0) == 0 {
			return z.r.Shift()
		} else if z.at('-', '-', '>') {
			comment := z.r.Shift()
			z.r.Move(3)
			z.r.Skip()
			return comment
		}
		z.r.Move(1)
	}
}

func (z *Tokenizer) shiftStartTag() []byte {
	for {
		if c := z.r.Peek(0); c == ' ' || c == '>' || c == '/' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == 0 {
			break
		}
		z.r.Move(1)
	}
	nameEnd := z.r.Pos()
	z.moveWhitespace() // before attribute name state
	return parse.ToLower(z.r.Shift()[:nameEnd])
}

func (z *Tokenizer) shiftAttribute() []byte {
	for { // attribute name state
		if c := z.r.Peek(0); c == ' ' || c == '=' || c == '>' || c == '/' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == 0 {
			break
		}
		z.r.Move(1)
	}
	nameEnd := z.r.Pos()
	z.moveWhitespace() // after attribute name state
	if z.r.Peek(0) == '=' {
		z.r.Move(1)
		z.moveWhitespace() // before attribute value state
		attrPos := z.r.Pos()
		delim := z.r.Peek(0)
		if delim == '"' || delim == '\'' { // attribute value single- and double-quoted state
			z.r.Move(1)
			for {
				if z.r.Peek(0) == delim {
					break
				}
				z.r.Move(1)
			}
			z.r.Move(1)
		} else { // attribute value unquoted state
			for {
				if c := z.r.Peek(0); c == ' ' || c == '>' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == 0 {
					break
				}
				z.r.Move(1)
			}
		}
		attrEnd := z.r.Pos()
		z.moveWhitespace() // before attribute name state or after attribute quoted value state
		z.attrVal = z.r.Bytes()[attrPos:attrEnd]
	} else {
		z.attrVal = nil
	}
	return parse.ToLower(z.r.Shift()[:nameEnd])
}

func (z *Tokenizer) shiftEndTag() []byte {
	for {
		c := z.r.Peek(0)
		if c == 0 {
			return z.r.Shift()
		} else if c == '>' {
			name := parse.ToLower(z.r.Shift())
			z.r.Move(1)
			z.r.Skip()
			return name
		}
		z.r.Move(1)
	}
}

func (z *Tokenizer) moveWhitespace() {
	for {
		c := z.r.Peek(0)
		if c != ' ' && c != '\t' && c != '\n' && c != '\r' || c == 0 {
			break
		}
		z.r.Move(1)
	}
}

func (z *Tokenizer) at(b ...byte) bool {
	for i, c := range b {
		if z.r.Peek(i) != c {
			return false
		}
	}
	return true
}
<commit_msg>XML doctype added<commit_after>package xml // import "github.com/tdewolff/parse/xml"

// TODO: optimize

import (
	"io"
	"strconv"

	"github.com/tdewolff/buffer"
	"github.com/tdewolff/parse"
)

////////////////////////////////////////////////////////////////

// TokenType determines the type of token, eg. a number or a semicolon.
type TokenType uint32

// TokenType values.
const (
	ErrorToken TokenType = iota // extra token when errors occur
	CommentToken
	DOCTYPEToken
	CDATAToken
	StartTagToken
	StartTagCloseToken
	StartTagCloseVoidToken
	StartTagClosePIToken
	EndTagToken
	AttributeToken
	TextToken
)

// String returns the string representation of a TokenType.
func (tt TokenType) String() string {
	switch tt {
	case ErrorToken:
		return "Error"
	case CommentToken:
		return "Comment"
	case DOCTYPEToken:
		return "DOCTYPE"
	case CDATAToken:
		return "CDATA"
	case StartTagToken:
		return "StartTag"
	case StartTagCloseToken:
		return "StartTagClose"
	case StartTagCloseVoidToken:
		return "StartTagCloseVoid"
	case StartTagClosePIToken:
		return "StartTagClosePI"
	case EndTagToken:
		return "EndTag"
	case AttributeToken:
		return "Attribute"
	case TextToken:
		return "Text"
	}
	return "Invalid(" + strconv.Itoa(int(tt)) + ")"
}

////////////////////////////////////////////////////////////////

// Tokenizer is the state for the tokenizer.
type Tokenizer struct {
	r *buffer.Shifter

	inTag   bool
	attrVal []byte
}

// NewTokenizer returns a new Tokenizer for a given io.Reader.
func NewTokenizer(r io.Reader) *Tokenizer {
	return &Tokenizer{
		r: buffer.NewShifter(r),
	}
}

// Err returns the error encountered during tokenization, this is often io.EOF but also other errors can be returned.
func (z Tokenizer) Err() error {
	return z.r.Err()
}

// IsEOF returns true when it has encountered EOF and thus loaded the last buffer in memory.
func (z Tokenizer) IsEOF() bool {
	return z.r.IsEOF()
}

// Next returns the next Token. It returns ErrorToken when an error was encountered. Using Err() one can retrieve the error message.
func (z *Tokenizer) Next() (TokenType, []byte) {
	var c byte
	if z.inTag {
		z.attrVal = nil
		c = z.r.Peek(0)
		if c != '>' && (c != '/' && c != '?' || z.r.Peek(1) != '>') {
			return AttributeToken, z.shiftAttribute()
		}
		z.inTag = false
		if c == '/' {
			z.r.Move(2)
			return StartTagCloseVoidToken, z.r.Shift()
		} else if c == '?' {
			z.r.Move(2)
			return StartTagClosePIToken, z.r.Shift()
		} else {
			z.r.Move(1)
			return StartTagCloseToken, z.r.Shift()
		}
	}

	for {
		c = z.r.Peek(0)
		if c == 0 {
			if z.r.Pos() > 0 {
				return TextToken, z.r.Shift()
			}
			return ErrorToken, []byte{}
		} else if c == '<' {
			if z.r.Pos() > 0 {
				return TextToken, z.r.Shift()
			}
			c = z.r.Peek(1)
			if c == '/' {
				z.r.Move(2)
				z.r.Skip()
				return EndTagToken, z.shiftEndTag()
			} else if c == '!' {
				z.r.Move(2)
				if z.at('-', '-') {
					z.r.Move(2)
					return CommentToken, z.shiftCommentText()
				} else if z.at('[', 'C', 'D', 'A', 'T', 'A', '[') {
					z.r.Move(7)
					return CDATAToken, z.shiftCDATAText()
				} else if z.at('D', 'O', 'C', 'T', 'Y', 'P', 'E') {
					z.r.Move(7)
					return DOCTYPEToken, z.shiftDOCTYPEText()
				}
				z.r.Move(-2)
			}
			z.r.Move(1)
			z.r.Skip()
			z.inTag = true
			return StartTagToken, z.shiftStartTag()
		}
		z.r.Move(1)
	}
	return ErrorToken, []byte{}
}

func (z *Tokenizer) AttrVal() []byte {
	return z.attrVal
}

////////////////////////////////////////////////////////////////

/*
The following functions follow the specifications at http://www.w3.org/html/wg/drafts/html/master/syntax.html
*/

func (z *Tokenizer) shiftDOCTYPEText() []byte {
	z.r.Skip()
	inString := false
	inBrackets := false
	for {
		c := z.r.Peek(0)
		if c == 0 {
			return z.r.Shift()
		} else if c == '>' && !inString && !inBrackets {
			doctype := z.r.Shift()
			z.r.Move(1)
			z.r.Skip()
			return doctype
		} else if c == '"' {
			inString = !inString
		} else if c == '[' {
			inBrackets = true
		} else if c == ']' {
			inBrackets = false
		}
		z.r.Move(1)
	}
}

func (z *Tokenizer) shiftCDATAText() []byte {
	z.r.Skip()
	for {
		if z.r.Peek(0) == 0 {
			return z.r.Shift()
		} else if z.at(']', ']', '>') {
			cdata := z.r.Shift()
			z.r.Move(3)
			z.r.Skip()
			return cdata
		}
		z.r.Move(1)
	}
}

func (z *Tokenizer) shiftCommentText() []byte {
	z.r.Skip()
	for {
		if z.r.Peek(0) == 0 {
			return z.r.Shift()
		} else if z.at('-', '-', '>') {
			comment := z.r.Shift()
			z.r.Move(3)
			z.r.Skip()
			return comment
		}
		z.r.Move(1)
	}
}

func (z *Tokenizer) shiftStartTag() []byte {
	for {
		if c := z.r.Peek(0); c == ' ' || c == '>' || c == '/' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == 0 {
			break
		}
		z.r.Move(1)
	}
	nameEnd := z.r.Pos()
	z.moveWhitespace() // before attribute name state
	return parse.ToLower(z.r.Shift()[:nameEnd])
}

func (z *Tokenizer) shiftAttribute() []byte {
	for { // attribute name state
		if c := z.r.Peek(0); c == ' ' || c == '=' || c == '>' || c == '/' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == 0 {
			break
		}
		z.r.Move(1)
	}
	nameEnd := z.r.Pos()
	z.moveWhitespace() // after attribute name state
	if z.r.Peek(0) == '=' {
		z.r.Move(1)
		z.moveWhitespace() // before attribute value state
		attrPos := z.r.Pos()
		delim := z.r.Peek(0)
		if delim == '"' || delim == '\'' { // attribute value single- and double-quoted state
			z.r.Move(1)
			for {
				if z.r.Peek(0) == delim {
					break
				}
				z.r.Move(1)
			}
			z.r.Move(1)
		} else { // attribute value unquoted state
			for {
				if c := z.r.Peek(0); c == ' ' || c == '>' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == 0 {
					break
				}
				z.r.Move(1)
			}
		}
		attrEnd := z.r.Pos()
		z.moveWhitespace() // before attribute name state or after attribute quoted value state
		z.attrVal = z.r.Bytes()[attrPos:attrEnd]
	} else {
		z.attrVal = nil
	}
	return parse.ToLower(z.r.Shift()[:nameEnd])
}

func (z *Tokenizer) shiftEndTag() []byte {
	for {
		c := z.r.Peek(0)
		if c == 0 {
			return z.r.Shift()
		} else if c == '>' {
			name := parse.ToLower(z.r.Shift())
			z.r.Move(1)
			z.r.Skip()
			return name
		}
		z.r.Move(1)
	}
}

func (z *Tokenizer) moveWhitespace() {
	for {
		c := z.r.Peek(0)
		if c != ' ' && c != '\t' && c != '\n' && c != '\r' || c == 0 {
			break
		}
		z.r.Move(1)
	}
}

func (z *Tokenizer) at(b ...byte) bool {
	for i, c := range b {
		if z.r.Peek(i) != c {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package kontrol

import (
	"errors"
	"strings"

	"github.com/coreos/etcd/store"
	"github.com/coreos/go-etcd/etcd"
)

type Storage interface {
	Get(key string) (*Node, error)
}

type Etcd struct {
	client *etcd.Client
}

func NewEtcd() (*Etcd, error) {
	machines := []string{"http://127.0.0.1:4001"}
	client := etcd.NewClient(machines)

	ok := client.SetCluster(machines)
	if !ok {
		return nil, errors.New("cannot connect to etcd cluster: " + strings.Join(machines, ","))
	}

	return &Etcd{
		client: client,
	}, nil
}

func (e *Etcd) Get(key string) (*Node, error) {
	resp, err := e.client.Get(key, false, true)
	if err != nil {
		return nil, err
	}

	nodeExtern := convertNodeToNodeExtern(resp.Node)

	return NewNode(nodeExtern), nil
}

func convertNodeToNodeExtern(node *etcd.Node) *store.NodeExtern {
	s := &store.NodeExtern{
		Key:           node.Key,
		Value:         &node.Value,
		Dir:           node.Dir,
		Expiration:    node.Expiration,
		TTL:           node.TTL,
		ModifiedIndex: node.ModifiedIndex,
		CreatedIndex:  node.CreatedIndex,
		Nodes:         make(store.NodeExterns, len(node.Nodes)),
	}

	for i, n := range node.Nodes {
		s.Nodes[i] = convertNodeToNodeExtern(n)
	}

	return s
}
<commit_msg>kontrol: add some etcd primitives<commit_after>package kontrol

import (
	"errors"
	"strings"

	"github.com/coreos/etcd/store"
	"github.com/coreos/go-etcd/etcd"
)

type Storage interface {
	Get(key string) (*Node, error)
	Set(key, value string) error
	Delete(key string) error
}

type Etcd struct {
	client *etcd.Client
}

func NewEtcd() (*Etcd, error) {
	machines := []string{"http://127.0.0.1:4001"}
	client := etcd.NewClient(machines)

	ok := client.SetCluster(machines)
	if !ok {
		return nil, errors.New("cannot connect to etcd cluster: " + strings.Join(machines, ","))
	}

	return &Etcd{
		client: client,
	}, nil
}

func (e *Etcd) Delete(key string) error {
	_, err := e.client.Delete(key, true)
	return err
}

func (e *Etcd) Set(key, value string) error {
	_, err := e.client.Set(key, value, uint64(HeartbeatDelay))
	return err
}

func (e *Etcd) Update(key, value string) error {
	_, err := e.client.Update(key, value, uint64(HeartbeatDelay))
	return err
}

func (e *Etcd) Get(key string) (*Node, error) {
	resp, err := e.client.Get(key, false, true)
	if err != nil {
		return nil, err
	}

	nodeExtern := convertNodeToNodeExtern(resp.Node)

	return NewNode(nodeExtern), nil
}

func convertNodeToNodeExtern(node *etcd.Node) *store.NodeExtern {
	s := &store.NodeExtern{
		Key:           node.Key,
		Value:         &node.Value,
		Dir:           node.Dir,
		Expiration:    node.Expiration,
		TTL:           node.TTL,
		ModifiedIndex: node.ModifiedIndex,
		CreatedIndex:  node.CreatedIndex,
		Nodes:         make(store.NodeExterns, len(node.Nodes)),
	}

	for i, n := range node.Nodes {
		s.Nodes[i] = convertNodeToNodeExtern(n)
	}

	return s
}
<|endoftext|>


<commit_before>// Package xsrf is a container for the gorilla csrf package
package xsrf

import (
	"net/http"

	"github.com/stowelly/core/view"
	"github.com/gorilla/csrf"
)

// Info holds the config.
type Info struct {
	AuthKey string
	Secure  bool
}

// Token sets token in the template to the CSRF token.
func Token(w http.ResponseWriter, r *http.Request, v *view.Info) {
	v.Vars["token"] = csrf.Token(r)
}
<commit_msg>csrf token stuff<commit_after>// Package xsrf is a container for the gorilla csrf package
package xsrf

import (
	"net/http"

	"github.com/gorilla/csrf"
	"github.com/stowelly/core/view"
)

// Info holds the config.
type Info struct {
	AuthKey string
	Secure  bool
}

// Token sets token in the template to the CSRF token.
func Token(w http.ResponseWriter, r *http.Request, v *view.Info) {
	w.Header().Set("X-CSRF-Token", csrf.Token(r))
}
<|endoftext|>


<commit_before>/*
 * This is the main core of the yaag package
 */
package yaag

import (
	"encoding/json"
	"github.com/betacraft/yaag/yaag/models"
	"html/template"
	"io"
	"log"
	"os"
	"path/filepath"
)

var count int
var config *Config

// Initial empty spec
var spec *models.Spec = &models.Spec{}

func IsOn() bool {
	return config.On
}

func Init(conf *Config) {
	config = conf
	// load the config file
	filePath, err := filepath.Abs(conf.DocPath + ".json")
	dataFile, err := os.Open(filePath)
	defer dataFile.Close()
	if err == nil {
		json.NewDecoder(io.Reader(dataFile)).Decode(spec)
		generateHtml()
	}
}

func add(x, y int) int {
	return x + y
}

func mult(x, y int) int {
	return (x + 1) * y
}

func GenerateHtml(apiCall *models.ApiCall) {
	shouldAddPathSpec := true
	for k, apiSpec := range spec.ApiSpecs {
		if apiSpec.Path == apiCall.CurrentPath && apiSpec.HttpVerb == apiCall.MethodType {
			shouldAddPathSpec = false
			apiCall.Id = count
			count += 1
			deleteCommonHeaders(apiCall)
			avoid := false
			for _, currentApiCall := range spec.ApiSpecs[k].Calls {
				if apiCall.RequestBody == currentApiCall.RequestBody &&
					apiCall.ResponseCode == currentApiCall.ResponseCode &&
					apiCall.ResponseBody == currentApiCall.ResponseBody {
					avoid = true
				}
			}
			if !avoid {
				spec.ApiSpecs[k].Calls = append(apiSpec.Calls, *apiCall)
			}
		}
	}

	if shouldAddPathSpec {
		apiSpec := models.ApiSpec{
			HttpVerb: apiCall.MethodType,
			Path:     apiCall.CurrentPath,
		}
		apiCall.Id = count
		count += 1
		deleteCommonHeaders(apiCall)
		apiSpec.Calls = append(apiSpec.Calls, *apiCall)
		spec.ApiSpecs = append(spec.ApiSpecs, apiSpec)
	}
	filePath, err := filepath.Abs(config.DocPath)
	dataFile, err := os.Create(filePath + ".json")
	if err != nil {
		log.Println(err)
		return
	}
	defer dataFile.Close()
	data, err := json.Marshal(spec)
	if err != nil {
		log.Println(err)
		return
	}
	_, err = dataFile.Write(data)
	if err != nil {
		log.Println(err)
		return
	}
	generateHtml()
}

func generateHtml() {
	funcs := template.FuncMap{"add": add, "mult": mult}
	t := template.New("API Documentation").Funcs(funcs)
	htmlString := Template
	t, err := t.Parse(htmlString)
	if err != nil {
		log.Println(err)
		return
	}
	filePath, err := filepath.Abs(config.DocPath)
	if err != nil {
		panic("Error while creating file path : " + err.Error())
		return
	}
	homeHtmlFile, err := os.Create(filePath)
	defer homeHtmlFile.Close()
	if err != nil {
		panic("Error while creating documentation file : " + err.Error())
		return
	}
	homeWriter := io.Writer(homeHtmlFile)
	t.Execute(homeWriter, map[string]interface{}{"array": spec.ApiSpecs,
		"baseUrls": config.BaseUrls, "Title": config.DocTitle})
}

func deleteCommonHeaders(call *models.ApiCall) {
	delete(call.RequestHeader, "Accept")
	delete(call.RequestHeader, "Accept-Encoding")
	delete(call.RequestHeader, "Accept-Language")
	delete(call.RequestHeader, "Cache-Control")
	delete(call.RequestHeader, "Connection")
	delete(call.RequestHeader, "Cookie")
	delete(call.RequestHeader, "Origin")
	delete(call.RequestHeader, "User-Agent")
}
<commit_msg>issue fix #11, remove redundant return after panic<commit_after>/*
 * This is the main core of the yaag package
 */
package yaag

import (
	"encoding/json"
	"html/template"
	"io"
	"log"
	"os"
	"path/filepath"

	"github.com/betacraft/yaag/yaag/models"
)

var count int
var config *Config

// Initial empty spec
var spec *models.Spec = &models.Spec{}

func IsOn() bool {
	return config.On
}

func Init(conf *Config) {
	config = conf
	// load the config file
	filePath, err := filepath.Abs(conf.DocPath + ".json")
	dataFile, err := os.Open(filePath)
	defer dataFile.Close()
	if err == nil {
		json.NewDecoder(io.Reader(dataFile)).Decode(spec)
		generateHtml()
	}
}

func add(x, y int) int {
	return x + y
}

func mult(x, y int) int {
	return (x + 1) * y
}

func GenerateHtml(apiCall *models.ApiCall) {
	shouldAddPathSpec := true
	for k, apiSpec := range spec.ApiSpecs {
		if apiSpec.Path == apiCall.CurrentPath && apiSpec.HttpVerb == apiCall.MethodType {
			shouldAddPathSpec = false
			apiCall.Id = count
			count += 1
			deleteCommonHeaders(apiCall)
			avoid := false
			for _, currentApiCall := range spec.ApiSpecs[k].Calls {
				if apiCall.RequestBody == currentApiCall.RequestBody &&
					apiCall.ResponseCode == currentApiCall.ResponseCode &&
					apiCall.ResponseBody == currentApiCall.ResponseBody {
					avoid = true
				}
			}
			if !avoid {
				spec.ApiSpecs[k].Calls = append(apiSpec.Calls, *apiCall)
			}
		}
	}

	if shouldAddPathSpec {
		apiSpec := models.ApiSpec{
			HttpVerb: apiCall.MethodType,
			Path:     apiCall.CurrentPath,
		}
		apiCall.Id = count
		count += 1
		deleteCommonHeaders(apiCall)
		apiSpec.Calls = append(apiSpec.Calls, *apiCall)
		spec.ApiSpecs = append(spec.ApiSpecs, apiSpec)
	}
	filePath, err := filepath.Abs(config.DocPath)
	dataFile, err := os.Create(filePath + ".json")
	if err != nil {
		log.Println(err)
		return
	}
	defer dataFile.Close()
	data, err := json.Marshal(spec)
	if err != nil {
		log.Println(err)
		return
	}
	_, err = dataFile.Write(data)
	if err != nil {
		log.Println(err)
		return
	}
	generateHtml()
}

func generateHtml() {
	funcs := template.FuncMap{"add": add, "mult": mult}
	t := template.New("API Documentation").Funcs(funcs)
	htmlString := Template
	t, err := t.Parse(htmlString)
	if err != nil {
		log.Println(err)
		return
	}
	filePath, err := filepath.Abs(config.DocPath)
	if err != nil {
		panic("Error while creating file path : " + err.Error())
	}
	homeHtmlFile, err := os.Create(filePath)
	defer homeHtmlFile.Close()
	if err != nil {
		panic("Error while creating documentation file : " + err.Error())
	}
	homeWriter := io.Writer(homeHtmlFile)
	t.Execute(homeWriter, map[string]interface{}{"array": spec.ApiSpecs,
		"baseUrls": config.BaseUrls, "Title": config.DocTitle})
}

func deleteCommonHeaders(call *models.ApiCall) {
	delete(call.RequestHeader, "Accept")
	delete(call.RequestHeader, "Accept-Encoding")
	delete(call.RequestHeader, "Accept-Language")
	delete(call.RequestHeader, "Cache-Control")
	delete(call.RequestHeader, "Connection")
	delete(call.RequestHeader, "Cookie")
	delete(call.RequestHeader, "Origin")
	delete(call.RequestHeader, "User-Agent")
}
<|endoftext|>