<commit_before>package export

import (
	"encoding/csv"
	"fmt"
	"io"
	"math"
	"reflect"
	"text/tabwriter"
	"time"
)

// Interplay Extractor-Accessor-Formatter-Dumper
// Extractor constructs a list of Accessors
// Dumper writes rows to output.
// The output format depends basically on the used Dumper, i.e. only the
// Dumper knows how to format the values. So no formatter is needed.
// But it would be nice if Accessors could help here.

var m = math.Floor

func isTime(x reflect.Type) bool {
	return x.PkgPath() == "time" && x.Kind() == reflect.Struct && x.Name() == "Time"
}

// -------------------------------------------------------------------------
// Field Types

// FieldType represents the basisc type of a field.
type FieldType uint

const (
	NA FieldType = iota
	Bool
	Int
	Float
	String
	Time
)

// String representation of ft.
func (ft FieldType) String() string {
	return []string{"NA", "Bool", "Int", "Float", "String", "Time"}[ft]
}

// -------------------------------------------------------------------------
// Formating Options

// Format describes how different fields types will be formated
type Format struct {
	True, False string // String values of boolean true and false.
	IntFmt      string // Package fmt style verb for int64 printing.
	FloatFmt    string // Package fmt style verb for float64 printing.
	StringFmt   string // Package fmt style verb for string printing.
	TimeFmt     string // A package time layout string.

	// TimeLoc is the location in which times are presented.
	// If a nil TimeLoc is used the times are presented in their
	// original location.
	TimeLoc *time.Location

	NA string // Representation of a missing value.
}

// DefaultFormat are the default formating options.
var DefaultFormat = Format{
	True:      "true",
	False:     "false",
	IntFmt:    "%d",
	FloatFmt:  "%.5g",
	StringFmt: "%s",
	TimeFmt:   "2006-01-02T15:04:05.999",
	TimeLoc:   time.Local,
	NA:        "",
}

// RFormat is a useful format for dumping stuff you want to read into R.
var RFormat = Format{
	True:      "TRUE",
	False:     "FALSE",
	IntFmt:    "%d",
	FloatFmt:  "%.6g",
	StringFmt: "%s",
	TimeFmt:   "2006-01-02 15:04:05",
	TimeLoc:   time.Local,
	NA:        "NA",
}

// -------------------------------------------------------------------------
// Field

// Field represents a column in a data frame.
type Field struct {
	Name  string                  // The name of the field
	Type  FieldType               // The type of the field
	Value func(i int) interface{} // The value, maybe nil
}

//	HasValue(i int) bool // Check if a value is present

// Print the i'th entry of f according to the given format.
func (f Field) Print(format Format, i int) string {
	val := f.Value(i)
	if val == nil {
		return format.NA
	}
	switch f.Type {
	case Bool:
		if val.(bool) {
			return format.True
		}
		return format.False
	case Int:
		return fmt.Sprintf(format.IntFmt, val.(int64))
	case Float:
		return fmt.Sprintf(format.FloatFmt, val.(float64))
	case String:
		return fmt.Sprintf(format.StringFmt, val.(string))
	case Time:
		t := val.(time.Time)
		if format.TimeLoc != nil {
			t = t.In(format.TimeLoc)
		}
		return t.Format(format.TimeFmt)
	}

	return fmt.Sprintf("Ooops: %v", val)
}

// -------------------------------------------------------------------------
// Dumper

type Dumper interface {
	Dump(e Extractor, format Format) error
}

// CSVDumper dumps values in CSV format
type CSVDumper struct {
	Writer     *csv.Writer // The csv.Writer to output the data.
	OmitHeader bool        // OmitHeader suppresses the header line in the generated CSV.
}

// Dump dumps the fields from e to d.
func (d CSVDumper) Dump(e Extractor, format Format) error {
	row := make([]string, len(e.Fields))
	if !d.OmitHeader {
		for i, field := range e.Fields {
			row[i] = field.Name
		}
		d.Writer.Write(row)
	}
	for r := 0; r < e.N; r++ {
		for col, field := range e.Fields {
			row[col] = field.Print(format, r)
		}
		err := d.Writer.Write(row)
		if err != nil {
			return err
		}
	}
	d.Writer.Flush()
	return d.Writer.Error()
}

type TabDumper struct {
	Writer io.Writer
	Flags  uint
}

// Dump dumps the fields from e to d.
func (d TabDumper) Dump(e Extractor, format Format) error {
	w := new(tabwriter.Writer)
	w.Init(d.Writer, 1, 8, 1, ' ', d.Flags)
	for i, field := range e.Fields {
		if i == 0 {
			fmt.Fprintf(w, "%s", field.Name)
		} else {
			fmt.Fprintf(w, "\t%s", field.Name)
		}
	}
	fmt.Fprintln(w)
	for r := 0; r < e.N; r++ {
		for i, field := range e.Fields {
			if i == 0 {
				fmt.Fprintf(w, "%s", field.Print(format, r))
			} else {
				fmt.Fprintf(w, "\t%s", field.Print(format, r))
			}
		}
		fmt.Fprintln(w)
	}
	w.Flush()
	return nil
}

// RVecDumper dumps as a R vectors.
type RVecDumper struct {
	Writer io.Writer

	// If Name is nonempty a data frame named Name consisting of all
	// fields is constructed too.
	Name string
}

// Dump dumps the fields from e to d.
func (d RVecDumper) Dump(e Extractor, format Format) error {
	all := ""
	for f, field := range e.Fields {
		if _, err := fmt.Fprintf(d.Writer, "%s <- c(", field.Name); err != nil {
			return err
		}
		for r := 0; r < e.N; r++ {
			s := field.Print(format, r)
			if r < e.N-1 {
				if r%10 == 9 {
					s += ",\n"
				} else {
					s += ", "
				}
			}
			if _, err := fmt.Fprintf(d.Writer, "%s", s); err != nil {
				return err
			}
		}
		if _, err := fmt.Fprintf(d.Writer, ")\n"); err != nil {
			return err
		}
		if f > 0 {
			all += ", "
		}
		all += field.Name
	}

	if d.Name != "" {
		if _, err := fmt.Fprintf(d.Writer, "%s <- data.frame(%s)\n", d.Name, all); err != nil {
			return err
		}
	}
	return nil
}

// -------------------------------------------------------------------------
// Extractor

type Extractor struct {
	// N is the number of elements/rows/measurements.
	N int

	// Field contains all the fields, i.e. the columns to extract.
	Fields []Field
}

// NewExtractor returns an extractor for the given fieldnames of data.
// The order of fieldnames determines the columns....
func NewExtractor(data interface{}, fieldnames ...string) (Extractor, error) {
	t := reflect.TypeOf(data)
	switch t.Kind() {
	case reflect.Slice:
		return newSOMExtractor(data, fieldnames...)
	case reflect.Struct:
		panic("COS data frame not implemented")
	}
	return Extractor{}, fmt.Errorf("Cannot build Extrator for %s", t.String())
}

func canHandle(t reflect.Type) bool {
	switch t.Kind() {
	case reflect.Bool:
		return true
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return true
	case reflect.String:
		return true
	case reflect.Float32, reflect.Float64:
		return true
	case reflect.Struct:
		return isTime(t)
	}
	return false
}

var (
	errorInterfaceType = reflect.TypeOf((*error)(nil)).Elem()
)

func newSOMExtractor(data interface{}, fieldnames ...string) (Extractor, error) {
	t := reflect.TypeOf(data).Elem()
	v := reflect.ValueOf(data)
	ex := Extractor{}
	ex.N = v.Len()

	for _, name := range fieldnames {
		field := Field{
			Type: NA,
			Name: name,
		}

		// Fields.
		for n := 0; n < t.NumField(); n++ {
			f := t.Field(n)
			if f.Name != name {
				continue
			}
			if !canHandle(f.Type) {
				return ex, fmt.Errorf("export: cannot usefield %q", name)
			}

			switch f.Type.Kind() {
			case reflect.Bool:
				field.Type = Bool
				field.Value = func(i int) interface{} {
					return v.Index(i).Field(n).Bool()
				}
			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
				field.Type = Int
				field.Value = func(i int) interface{} {
					return v.Index(i).Field(n).Int()
				}
			case reflect.String:
				field.Type = String
				field.Value = func(i int) interface{} {
					return v.Index(i).Field(n).String()
				}
			case reflect.Float32, reflect.Float64:
				field.Type = Float
				field.Value = func(i int) interface{} {
					return v.Index(i).Field(n).Float()
				}
			case reflect.Struct: // Checked above for beeing time.Time
				field.Type = Time
				field.Value = func(i int) interface{} {
					return v.Index(i).Field(n).Interface()
				}
			}
			break
		}
		if field.Type != NA {
			ex.Fields = append(ex.Fields, field)
			continue
		}

		// The same for methods.
		// TODO: As n is not used in the field value closures a simple
		// MethodByName would be much nicer here.
		for n := 0; n < t.NumMethod(); n++ {
			m := t.Method(n)
			if m.Name != name {
				continue
			}
			// Look for methods with signatures like
			//   func(elemtype) [int,string,float,time]
			// or
			//   func(elemtype) ([int,string,float,time], error)
			mt := m.Type
			numOut := mt.NumOut()
			if mt.NumIn() != 1 || (numOut != 1 && numOut != 2) ||
				!canHandle(mt.Out(0)) {
				return ex, fmt.Errorf("export: cannot use method %q", name)
			}
			mayFail := false
			if numOut == 2 {
				if mt.Out(1).Kind() == reflect.Interface &&
					mt.Out(1).Implements(errorInterfaceType) {
					mayFail = true
				} else {
					return ex, fmt.Errorf("export: cannot use method %q", name)
				}
			}

			// TODO: Move mayFail code out of function closure.
			switch mt.Out(0).Kind() {
			case reflect.Bool:
				field.Type = Bool
				field.Value = func(i int) interface{} {
					z := m.Func.Call([]reflect.Value{v.Index(i)})
					if mayFail && z[1].Interface() != nil {
						return nil
					}
					return z[0].Bool()
				}
			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
				field.Type = Int
				field.Value = func(i int) interface{} {
					z := m.Func.Call([]reflect.Value{v.Index(i)})
					if mayFail && z[1].Interface() != nil {
						return nil
					}
					return z[0].Int()
				}
			case reflect.String:
				field.Type = String
				field.Value = func(i int) interface{} {
					z := m.Func.Call([]reflect.Value{v.Index(i)})
					if mayFail && z[1].Interface() != nil {
						return nil
					}
					return z[0].String()
				}
			case reflect.Float32, reflect.Float64:
				field.Type = Float
				field.Value = func(i int) interface{} {
					z := m.Func.Call([]reflect.Value{v.Index(i)})
					if mayFail && z[1].Interface() != nil {
						return nil
					}
					return z[0].Float()
				}
			case reflect.Struct: // checked above for beeing time.Time
				field.Type = Time
				field.Value = func(i int) interface{} {
					z := m.Func.Call([]reflect.Value{v.Index(i)})
					if mayFail && z[1].Interface() != nil {
						return nil
					}
					return z[0].Interface()
				}
			default:
				panic("Oooops")
			}

			break
		}
		if field.Type != NA {
			ex.Fields = append(ex.Fields, field)
			continue
		}

		return ex, fmt.Errorf("export: no such field or method %q", name)

		// TODO: Maybe pointer methods too?
		// v.Addr().MethodByName()
	}

	return ex, nil
}
<commit_msg>Refactored binding to data to own function.<commit_after>package export

import (
	"encoding/csv"
	"fmt"
	"io"
	"math"
	"reflect"
	"text/tabwriter"
	"time"
)

// New idea: Register a type first. This sets an internal description
// of the fields, their types, etc. This step may fail, e.g. if unknown
// fields are used.
// Then bind an Extractor to any such type. This process constructs
// the actual closures to get the values and doesn't fail. ANd uses
// the actual number of elements.

var m = math.Floor

func isTime(x reflect.Type) bool {
	return x.PkgPath() == "time" && x.Kind() == reflect.Struct && x.Name() == "Time"
}

// -------------------------------------------------------------------------
// Field Types

// FieldType represents the basisc type of a field.
type FieldType uint

const (
	NA FieldType = iota
	Bool
	Int
	Float
	String
	Time
)

// String representation of ft.
func (ft FieldType) String() string {
	return []string{"NA", "Bool", "Int", "Float", "String", "Time"}[ft]
}

// -------------------------------------------------------------------------
// Formating Options

// Format describes how different fields types will be formated
type Format struct {
	True, False string // String values of boolean true and false.
	IntFmt      string // Package fmt style verb for int64 printing.
	FloatFmt    string // Package fmt style verb for float64 printing.
	StringFmt   string // Package fmt style verb for string printing.
	TimeFmt     string // A package time layout string.

	// TimeLoc is the location in which times are presented.
	// If a nil TimeLoc is used the times are presented in their
	// original location.
	TimeLoc *time.Location

	NA string // Representation of a missing value.
}

// DefaultFormat are the default formating options.
var DefaultFormat = Format{
	True:      "true",
	False:     "false",
	IntFmt:    "%d",
	FloatFmt:  "%.5g",
	StringFmt: "%s",
	TimeFmt:   "2006-01-02T15:04:05.999",
	TimeLoc:   time.Local,
	NA:        "",
}

// RFormat is a useful format for dumping stuff you want to read into R.
var RFormat = Format{
	True:      "TRUE",
	False:     "FALSE",
	IntFmt:    "%d",
	FloatFmt:  "%.6g",
	StringFmt: "%s",
	TimeFmt:   "2006-01-02 15:04:05",
	TimeLoc:   time.Local,
	NA:        "NA",
}

// -------------------------------------------------------------------------
// Field

// Field represents a column in a data frame.
type Field struct {
	Name  string                  // The name of the field
	Type  FieldType               // The type of the field
	Value func(i int) interface{} // The value, maybe nil

	MayFail bool

	fieldNo int           // >= 0 ==> a field; <0 ==> a method
	mfunc   reflect.Value // the function of the method if fieldNo < 0
}

// Print the i'th entry of f according to the given format.
func (f Field) Print(format Format, i int) string {
	val := f.Value(i)
	if val == nil {
		return format.NA
	}
	switch f.Type {
	case Bool:
		if val.(bool) {
			return format.True
		}
		return format.False
	case Int:
		return fmt.Sprintf(format.IntFmt, val.(int64))
	case Float:
		return fmt.Sprintf(format.FloatFmt, val.(float64))
	case String:
		return fmt.Sprintf(format.StringFmt, val.(string))
	case Time:
		t := val.(time.Time)
		if format.TimeLoc != nil {
			t = t.In(format.TimeLoc)
		}
		return t.Format(format.TimeFmt)
	}

	return fmt.Sprintf("Ooops: %v", val)
}

// -------------------------------------------------------------------------
// Dumper

type Dumper interface {
	Dump(e Extractor, format Format) error
}

// CSVDumper dumps values in CSV format
type CSVDumper struct {
	Writer     *csv.Writer // The csv.Writer to output the data.
	OmitHeader bool        // OmitHeader suppresses the header line in the generated CSV.
}

// Dump dumps the fields from e to d.
func (d CSVDumper) Dump(e *Extractor, format Format) error {
	row := make([]string, len(e.Fields))
	if !d.OmitHeader {
		for i, field := range e.Fields {
			row[i] = field.Name
		}
		d.Writer.Write(row)
	}
	for r := 0; r < e.N; r++ {
		for col, field := range e.Fields {
			row[col] = field.Print(format, r)
		}
		err := d.Writer.Write(row)
		if err != nil {
			return err
		}
	}
	d.Writer.Flush()
	return d.Writer.Error()
}

type TabDumper struct {
	Writer io.Writer
	Flags  uint
}

// Dump dumps the fields from e to d.
func (d TabDumper) Dump(e *Extractor, format Format) error {
	w := new(tabwriter.Writer)
	w.Init(d.Writer, 1, 8, 1, ' ', d.Flags)
	for i, field := range e.Fields {
		if i == 0 {
			fmt.Fprintf(w, "%s", field.Name)
		} else {
			fmt.Fprintf(w, "\t%s", field.Name)
		}
	}
	fmt.Fprintln(w)
	for r := 0; r < e.N; r++ {
		for i, field := range e.Fields {
			if i == 0 {
				fmt.Fprintf(w, "%s", field.Print(format, r))
			} else {
				fmt.Fprintf(w, "\t%s", field.Print(format, r))
			}
		}
		fmt.Fprintln(w)
	}
	w.Flush()
	return nil
}

// RVecDumper dumps as a R vectors.
type RVecDumper struct {
	Writer io.Writer

	// If Name is nonempty a data frame named Name consisting of all
	// fields is constructed too.
	Name string
}

// Dump dumps the fields from e to d.
func (d RVecDumper) Dump(e *Extractor, format Format) error {
	all := ""
	for f, field := range e.Fields {
		if _, err := fmt.Fprintf(d.Writer, "%s <- c(", field.Name); err != nil {
			return err
		}
		for r := 0; r < e.N; r++ {
			s := field.Print(format, r)
			if r < e.N-1 {
				if r%10 == 9 {
					s += ",\n"
				} else {
					s += ", "
				}
			}
			if _, err := fmt.Fprintf(d.Writer, "%s", s); err != nil {
				return err
			}
		}
		if _, err := fmt.Fprintf(d.Writer, ")\n"); err != nil {
			return err
		}
		if f > 0 {
			all += ", "
		}
		all += field.Name
	}

	if d.Name != "" {
		if _, err := fmt.Fprintf(d.Writer, "%s <- data.frame(%s)\n", d.Name, all); err != nil {
			return err
		}
	}
	return nil
}

// -------------------------------------------------------------------------
// Extractor

type Extractor struct {
	N int // N is the numer of elements in the currently bound data.

	// Field contains all the fields, i.e. the columns to extract.
	Fields []Field

	som   bool          // true for slice-of-measurement, false for collection-of-slices
	value reflect.Value // the current binding
	typ   reflect.Type
}

// NewExtractor returns an extractor for the given fieldnames of data.
// The order of fieldnames determines the columns....
func NewExtractor(data interface{}, fieldnames ...string) (*Extractor, error) {
	t := reflect.TypeOf(data)
	switch t.Kind() {
	case reflect.Slice:
		ex, err := newSOMExtractor(data, fieldnames...)
		if err != nil {
			return ex, err
		}
		ex.bindSOM(data)
		return ex, nil
	case reflect.Struct:
		panic("COS data frame not implemented")
	}
	return &Extractor{}, fmt.Errorf("Cannot build Extrator for %s", t.String())
}

// Bind (re)binds e to data which must be of the same type as data used
// during constructing e.
func (e *Extractor) Bind(data interface{}) {
	t := reflect.TypeOf(data).Elem()
	if t != e.typ {
		panic(fmt.Sprintf("Cannot bind extractor for %s to data of type %s", e.typ, t))
	}
	if e.som {
		e.bindSOM(data)
	} else {
		panic("COS data frame not implemented")
	}
}

func (e *Extractor) bindSOM(data interface{}) {
	v := reflect.ValueOf(data)
	e.value = v

	for fn, field := range e.Fields {
		n := field.fieldNo
		f := field.mfunc
		if n >= 0 {
			// Plain field access
			switch field.Type {
			case Bool:
				e.Fields[fn].Value = func(i int) interface{} {
					return v.Index(i).Field(n).Bool()
				}
			case Int:
				e.Fields[fn].Value = func(i int) interface{} {
					return v.Index(i).Field(n).Int()
				}
			case Float:
				e.Fields[fn].Value = func(i int) interface{} {
					return v.Index(i).Field(n).Float()
				}
			case String:
				e.Fields[fn].Value = func(i int) interface{} {
					return v.Index(i).Field(n).String()
				}
			case Time:
				e.Fields[fn].Value = func(i int) interface{} {
					return v.Index(i).Field(n).Interface()
				}
			}
		} else if field.MayFail {
			// Method access with possible failure
			switch field.Type {
			case Bool:
				e.Fields[fn].Value = func(i int) interface{} {
					z := f.Call([]reflect.Value{v.Index(i)})
					if z[1].Interface() != nil {
						return nil
					}
					return z[0].Bool()
				}
			case Int:
				e.Fields[fn].Value = func(i int) interface{} {
					z := f.Call([]reflect.Value{v.Index(i)})
					if z[1].Interface() != nil {
						return nil
					}
					return z[0].Int()
				}
			case Float:
				e.Fields[fn].Value = func(i int) interface{} {
					z := f.Call([]reflect.Value{v.Index(i)})
					if z[1].Interface() != nil {
						return nil
					}
					return z[0].Float()
				}
			case String:
				e.Fields[fn].Value = func(i int) interface{} {
					z := f.Call([]reflect.Value{v.Index(i)})
					if z[1].Interface() != nil {
						return nil
					}
					return z[0].String()
				}
			case Time:
				e.Fields[fn].Value = func(i int) interface{} {
					z := f.Call([]reflect.Value{v.Index(i)})
					if z[1].Interface() != nil {
						return nil
					}
					return z[0].Interface()
				}
			}
		} else {
			// Method access without failure
			switch field.Type {
			case Bool:
				e.Fields[fn].Value = func(i int) interface{} {
					return f.Call([]reflect.Value{v.Index(i)})[0].Bool()
				}
			case Int:
				e.Fields[fn].Value = func(i int) interface{} {
					return f.Call([]reflect.Value{v.Index(i)})[0].Int()
				}
			case Float:
				e.Fields[fn].Value = func(i int) interface{} {
					return f.Call([]reflect.Value{v.Index(i)})[0].Float()
				}
			case String:
				e.Fields[fn].Value = func(i int) interface{} {
					return f.Call([]reflect.Value{v.Index(i)})[0].String()
				}
			case Time:
				e.Fields[fn].Value = func(i int) interface{} {
					return f.Call([]reflect.Value{v.Index(i)})[0].Interface()
				}
			}
		}
	}
}

func superType(t reflect.Type) FieldType {
	switch t.Kind() {
	case reflect.Bool:
		return Bool
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return Int
	case reflect.String:
		return String
	case reflect.Float32, reflect.Float64:
		return Float
	case reflect.Struct:
		if isTime(t) {
			return Time
		}
	}
	return NA
}

var (
	errorInterfaceType = reflect.TypeOf((*error)(nil)).Elem()
)

func newSOMExtractor(data interface{}, fieldnames ...string) (*Extractor, error) {
	t := reflect.TypeOf(data).Elem()
	v := reflect.ValueOf(data)
	ex := Extractor{}
	ex.N = v.Len()
	ex.value = v
	ex.typ = t

	for _, name := range fieldnames {
		field := Field{
			Type: NA,
			Name: name,
		}
		// Fields.
		for n := 0; n < t.NumField(); n++ {
			f := t.Field(n)
			if f.Name != name {
				continue
			}
			st := superType(f.Type)
			if st == NA {
				return &ex, fmt.Errorf("export: cannot use field %q", name)
			}
			field.MayFail = false
			field.fieldNo = n
			field.Type = st
			break
		}
		if field.Type != NA {
			ex.Fields = append(ex.Fields, field)
			continue
		}

		// The same for methods.
		// TODO: As n is not used in the field value closures a simple
		// MethodByName would be much nicer here.
		for n := 0; n < t.NumMethod(); n++ {
			m := t.Method(n)
			if m.Name != name {
				continue
			}
			// Look for methods with signatures like
			//   func(elemtype) [int,string,float,time]
			// or
			//   func(elemtype) ([int,string,float,time], error)
			mt := m.Type
			numOut := mt.NumOut()
			st := superType(mt.Out(0))
			if mt.NumIn() != 1 || (numOut != 1 && numOut != 2) || st == NA {
				return &ex, fmt.Errorf("export: cannot use method %q", name)
			}
			mayFail := false
			if numOut == 2 {
				if mt.Out(1).Kind() == reflect.Interface &&
					mt.Out(1).Implements(errorInterfaceType) {
					mayFail = true
				} else {
					return &ex, fmt.Errorf("export: cannot use method %q", name)
				}
			}
			field.MayFail = mayFail
			field.fieldNo = -1
			field.mfunc = m.Func
			field.Type = st
			break
		}
		if field.Type != NA {
			ex.Fields = append(ex.Fields, field)
			continue
		}

		return &ex, fmt.Errorf("export: no such field or method %q", name)

		// TODO: Maybe pointer methods too?
		// v.Addr().MethodByName()
	}

	return &ex, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

var cmdExport = &Command{
	Run:   runExport,
	Usage: "export [dir]",
	Short: "Export metadata to a local directory",
	Long: `
Export metadata to a local directory

Examples:

  force export

  force export org/schema
`,
}

func runExport(cmd *Command, args []string) {
	// Get path from args if available
	var err error
	var root string
	if len(args) == 1 {
		root, err = filepath.Abs(args[0])
	}
	if err != nil {
		fmt.Printf("Error obtaining file path\n")
		ErrorAndExit(err.Error())
	}
	force, _ := ActiveForce()
	emailTemplatesResult, _, err := force.Query(fmt.Sprintf("%s", "SELECT EmailTemplate.DeveloperName, EmailTemplate.Folder.DeveloperName from EmailTemplate"), false)
	emailTemplates := make([]string, 1, 100)
	emailTemplates[0] = "*"
	for _, et := range emailTemplatesResult.Records {
		fmt.Println(et)
		if et["Folder"] != nil {
			folderName := et["Folder"].(map[string]interface{})
			emailTemplates = append(emailTemplates, folderName["DeveloperName"].(string)+"/"+et["DeveloperName"].(string))
		}
	}
	sobjects, err := force.ListSobjects()
	if err != nil {
		ErrorAndExit(err.Error())
	}
	stdObjects := make([]string, 1, len(sobjects)+1)
	stdObjects[0] = "*"
	for _, sobject := range sobjects {
		name := sobject["name"].(string)
		if !sobject["custom"].(bool) && !strings.HasSuffix(name, "__Tag") && !strings.HasSuffix(name, "__History") && !strings.HasSuffix(name, "__Share") {
			stdObjects = append(stdObjects, name)
		}
	}
	query := ForceMetadataQuery{
		{Name: "AccountSettings", Members: []string{"*"}},
		{Name: "ActivitiesSettings", Members: []string{"*"}},
		{Name: "AddressSettings", Members: []string{"*"}},
		{Name: "AnalyticSnapshot", Members: []string{"*"}},
		{Name: "ApexClass", Members: []string{"*"}},
		{Name: "ApexComponent", Members: []string{"*"}},
		{Name: "ApexPage", Members: []string{"*"}},
		{Name: "ApexTrigger", Members: []string{"*"}},
		{Name: "ApprovalProcess", Members: []string{"*"}},
		{Name: "AssignmentRules", Members: []string{"*"}},
		{Name: "AuthProvider", Members: []string{"*"}},
		{Name: "AutoResponseRules", Members: []string{"*"}},
		{Name: "BusinessHoursSettings", Members: []string{"*"}},
		{Name: "BusinessProcess", Members: []string{"*"}},
		{Name: "CallCenter", Members: []string{"*"}},
		{Name: "CaseSettings", Members: []string{"*"}},
		{Name: "ChatterAnswersSettings", Members: []string{"*"}},
		{Name: "CompanySettings", Members: []string{"*"}},
		{Name: "Community", Members: []string{"*"}},
		{Name: "CompactLayout", Members: []string{"*"}},
		{Name: "ConnectedApp", Members: []string{"*"}},
		{Name: "ContractSettings", Members: []string{"*"}},
		{Name: "CustomApplication", Members: []string{"*"}},
		{Name: "CustomApplicationComponent", Members: []string{"*"}},
		{Name: "CustomApplication", Members: []string{"*"}},
		{Name: "CustomField", Members: []string{"*"}},
		{Name: "CustomLabels", Members: []string{"*"}},
		{Name: "CustomObject", Members: stdObjects},
		{Name: "CustomObjectTranslation", Members: []string{"*"}},
		{Name: "CustomPageWebLink", Members: []string{"*"}},
		{Name: "CustomPermission", Members: []string{"*"}},
		{Name: "CustomSite", Members: []string{"*"}},
		{Name: "CustomTab", Members: []string{"*"}},
		{Name: "Dashboard", Members: []string{"*"}},
		{Name: "DataCategoryGroup", Members: []string{"*"}},
		{Name: "Document", Members: []string{"*"}},
		{Name: "EmailTemplate", Members: emailTemplates},
		{Name: "EntitlementProcess", Members: []string{"*"}},
		{Name: "EntitlementSettings", Members: []string{"*"}},
		{Name: "EntitlementTemplate", Members: []string{"*"}},
		{Name: "ExternalDataSource", Members: []string{"*"}},
		{Name: "FieldSet", Members: []string{"*"}},
		{Name: "Flow", Members: []string{"*"}},
		{Name: "Folder", Members: []string{"*"}},
		{Name: "ForecastingSettings", Members: []string{"*"}},
		{Name: "Group", Members: []string{"*"}},
		{Name: "HomePageComponent", Members: []string{"*"}},
		{Name: "HomePageLayout", Members: []string{"*"}},
		{Name: "IdeasSettings", Members: []string{"*"}},
		{Name: "KnowledgeSettings", Members: []string{"*"}},
		{Name: "Layout", Members: []string{"*"}},
		{Name: "Letterhead", Members: []string{"*"}},
		{Name: "ListView", Members: []string{"*"}},
		{Name: "LiveAgentSettings", Members: []string{"*"}},
		{Name: "LiveChatAgentConfig", Members: []string{"*"}},
		{Name: "LiveChatButton", Members: []string{"*"}},
		{Name: "LiveChatDeployment", Members: []string{"*"}},
		{Name: "MilestoneType", Members: []string{"*"}},
		{Name: "MobileSettings", Members: []string{"*"}},
		{Name: "NamedFilter", Members: []string{"*"}},
		{Name: "Network", Members: []string{"*"}},
		{Name: "OpportunitySettings", Members: []string{"*"}},
		{Name: "PermissionSet", Members: []string{"*"}},
		{Name: "Portal", Members: []string{"*"}},
		{Name: "PostTemplate", Members: []string{"*"}},
		{Name: "ProductSettings", Members: []string{"*"}},
		{Name: "Profile", Members: []string{"*"}},
		{Name: "Queue", Members: []string{"*"}},
		{Name: "QuickAction", Members: []string{"*"}},
		{Name: "QuoteSettings", Members: []string{"*"}},
		{Name: "RecordType", Members: []string{"*"}},
		{Name: "RemoteSiteSetting", Members: []string{"*"}},
		{Name: "Report", Members: []string{"*"}},
		{Name: "ReportType", Members: []string{"*"}},
		{Name: "Role", Members: []string{"*"}},
		{Name: "SamlSsoConfig", Members: []string{"*"}},
		{Name: "Scontrol", Members: []string{"*"}},
		{Name: "SecuritySettings", Members: []string{"*"}},
		{Name: "SharingReason", Members: []string{"*"}},
		{Name: "SharingRules", Members: []string{"*"}},
		{Name: "Skill", Members: []string{"*"}},
		{Name: "StaticResource", Members: []string{"*"}},
		{Name: "Territory", Members: []string{"*"}},
		{Name: "Translations", Members: []string{"*"}},
		{Name: "ValidationRule", Members: []string{"*"}},
		{Name: "Workflow", Members: []string{"*"}},
	}
	root, err = GetSourceDir()
	if err != nil {
		fmt.Printf("Error obtaining root directory\n")
		ErrorAndExit(err.Error())
	}
	files, err := force.Metadata.Retrieve(query)
	if err != nil {
		fmt.Printf("Encountered and error with retrieve...\n")
		ErrorAndExit(err.Error())
	}
	for name, data := range files {
		file := filepath.Join(root, name)
		dir := filepath.Dir(file)
		if err := os.MkdirAll(dir, 0755); err != nil {
			ErrorAndExit(err.Error())
		}
		if err := ioutil.WriteFile(filepath.Join(root, name), data, 0644); err != nil {
			ErrorAndExit(err.Error())
		}
	}
	fmt.Printf("Exported to %s\n", root)
}
<commit_msg>Export also Report, Document and Dashboard and Folders metadata<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

var cmdExport = &Command{
	Run:   runExport,
	Usage: "export [dir]",
	Short: "Export metadata to a local directory",
	Long: `
Export metadata to a local directory

Examples:

  force export

  force export org/schema
`,
}

func getMetadataType(metadataType string, folders map[string]string) (member []string) {
	force, _ := ActiveForce()
	var queryString string
	if metadataType == "Report" {
		queryString = "SELECT Id, OwnerId ,DeveloperName from Report"
	} else {
		queryString = "SELECT Id, DeveloperName, Folder.DeveloperName from " + metadataType
	}
	queryResult, _, _ := force.Query(fmt.Sprintf("%s", queryString), false)
	metadataItems := make([]string, 1, 1000)
	metadataItems[0] = "*"
	for _, folderName := range folders {
		metadataItems = append(metadataItems, folderName)
	}

	for _, metadataItem := range queryResult.Records {
		folderName := ""
		if metadataType == "Report" {
			folderName, _ = folders[metadataItem["OwnerId"].(string)]
		} else {
			folderData, _ := metadataItem["Folder"].(map[string]interface{})
			if folderData != nil {
				folderName = folderData["DeveloperName"].(string)
			}
		}
		if folderName != "" {
			metadataItems = append(metadataItems, folderName+"/"+metadataItem["DeveloperName"].(string))
		}
	}
	return metadataItems
}

func runExport(cmd *Command, args []string) {
	// Get path from args if available
	var err error
	var root string
	if len(args) == 1 {
		root, err = filepath.Abs(args[0])
	}
	if err != nil {
		fmt.Printf("Error obtaining file path\n")
		ErrorAndExit(err.Error())
	}
	force, _ := ActiveForce()
	sobjects, err := force.ListSobjects()
	if err != nil {
		ErrorAndExit(err.Error())
	}
	stdObjects := make([]string, 1, len(sobjects)+1)
	stdObjects[0] = "*"
	for _, sobject := range sobjects {
		name := sobject["name"].(string)
		if !sobject["custom"].(bool) && !strings.HasSuffix(name, "__Tag") && !strings.HasSuffix(name, "__History") && !strings.HasSuffix(name, "__Share") {
			stdObjects = append(stdObjects, name)
		}
	}
	query := ForceMetadataQuery{
		{Name: "AccountSettings", Members: []string{"*"}},
		{Name: "ActivitiesSettings", Members: []string{"*"}},
		{Name: "AddressSettings", Members: []string{"*"}},
		{Name: "AnalyticSnapshot", Members: []string{"*"}},
		{Name: "ApexClass", Members: []string{"*"}},
		{Name: "ApexComponent", Members: []string{"*"}},
		{Name: "ApexPage", Members: []string{"*"}},
		{Name: "ApexTrigger", Members: []string{"*"}},
		{Name: "ApprovalProcess", Members: []string{"*"}},
		{Name: "AssignmentRules", Members: []string{"*"}},
		{Name: "AuthProvider", Members: []string{"*"}},
		{Name: "AutoResponseRules", Members: []string{"*"}},
		{Name: "BusinessHoursSettings", Members: []string{"*"}},
		{Name: "BusinessProcess", Members: []string{"*"}},
		{Name: "CallCenter", Members: []string{"*"}},
		{Name: "CaseSettings", Members: []string{"*"}},
		{Name: "ChatterAnswersSettings", Members: []string{"*"}},
		{Name: "CompanySettings", Members: []string{"*"}},
		{Name: "Community", Members: []string{"*"}},
		{Name: "CompactLayout", Members: []string{"*"}},
		{Name: "ConnectedApp", Members: []string{"*"}},
		{Name: "ContractSettings", Members: []string{"*"}},
		{Name: "CustomApplication", Members: []string{"*"}},
		{Name: "CustomApplicationComponent", Members: []string{"*"}},
		{Name: "CustomApplication", Members: []string{"*"}},
		{Name: "CustomField", Members: []string{"*"}},
		{Name: "CustomLabels", Members: []string{"*"}},
		{Name: "CustomObject", Members: stdObjects},
		{Name: "CustomObjectTranslation", Members: []string{"*"}},
		{Name: "CustomPageWebLink", Members: []string{"*"}},
		{Name: "CustomPermission", Members: []string{"*"}},
		{Name: "CustomSite", Members: []string{"*"}},
		{Name: "CustomTab", Members: []string{"*"}},
		{Name: "DataCategoryGroup", Members: []string{"*"}},
		{Name: "EntitlementProcess", Members: []string{"*"}},
		{Name: "EntitlementSettings", Members: []string{"*"}},
		{Name: "EntitlementTemplate", Members: []string{"*"}},
		{Name: "ExternalDataSource", Members: []string{"*"}},
		{Name: "FieldSet", Members: []string{"*"}},
		{Name: "Flow", Members: []string{"*"}},
		{Name: "Folder", Members: []string{"*"}},
		{Name: "ForecastingSettings", Members: []string{"*"}},
		{Name: "Group", Members: []string{"*"}},
		{Name: "HomePageComponent", Members: []string{"*"}},
		{Name: "HomePageLayout", Members: []string{"*"}},
		{Name: "IdeasSettings", Members: []string{"*"}},
		{Name: "KnowledgeSettings", Members: []string{"*"}},
		{Name: "Layout", Members: []string{"*"}},
		{Name: "Letterhead", Members: []string{"*"}},
		{Name: "ListView", Members: []string{"*"}},
		{Name: "LiveAgentSettings", Members: []string{"*"}},
		{Name: "LiveChatAgentConfig", Members: []string{"*"}},
		{Name: "LiveChatButton", Members: []string{"*"}},
		{Name: "LiveChatDeployment", Members: []string{"*"}},
		{Name: "MilestoneType", Members: []string{"*"}},
		{Name: "MobileSettings", Members: []string{"*"}},
		{Name: "NamedFilter", Members: []string{"*"}},
		{Name: "Network", Members: []string{"*"}},
		{Name: "OpportunitySettings", Members: []string{"*"}},
		{Name: "PermissionSet", Members: []string{"*"}},
		{Name: "Portal", Members: []string{"*"}},
		{Name: "PostTemplate", Members: []string{"*"}},
		{Name: "ProductSettings", Members: []string{"*"}},
		{Name: "Profile", Members: []string{"*"}},
		{Name: "Queue", Members: []string{"*"}},
		{Name: "QuickAction", Members: []string{"*"}},
		{Name: "QuoteSettings", Members: []string{"*"}},
		{Name: "RecordType", Members: []string{"*"}},
		{Name: "RemoteSiteSetting", Members: []string{"*"}},
		{Name: "ReportType", Members: []string{"*"}},
		{Name: "Role", Members: []string{"*"}},
		{Name: "SamlSsoConfig", Members: []string{"*"}},
		{Name: "Scontrol", Members: []string{"*"}},
		{Name: "SecuritySettings", Members: []string{"*"}},
		{Name: "SharingReason", Members: []string{"*"}},
		{Name: "SharingRules", Members: []string{"*"}},
		{Name: "Skill", Members: []string{"*"}},
		{Name: "StaticResource", Members: []string{"*"}},
		{Name: "Territory", Members: []string{"*"}},
		{Name: "Translations", Members: []string{"*"}},
		{Name: "ValidationRule", Members: []string{"*"}},
		{Name: "Workflow", Members: []string{"*"}},
	}

	folderResult, _, err := force.Query(fmt.Sprintf("%s", "SELECT Id, Type, DeveloperName from Folder"), false)
	folders := make(map[string]map[string]string)
	for _, folder := range folderResult.Records {
		if folder["DeveloperName"] != nil {
			folderType := folder["Type"].(string)
			m, ok := folders[folderType]
			if !ok {
				m = make(map[string]string)
				folders[folderType] = m
			}
			m[folder["Id"].(string)] = folder["DeveloperName"].(string)
		}
	}
	for foldersType, foldersName := range folders {
		if foldersType == "Email" {
			foldersType = "EmailTemplate"
		}
		members := getMetadataType(foldersType, foldersName)
		query = append(query, ForceMetadataQueryElement{Name: foldersType, Members: members})
	}

	root, err = GetSourceDir()
	if err != nil {
		fmt.Printf("Error obtaining root directory\n")
		ErrorAndExit(err.Error())
	}
	files, err := force.Metadata.Retrieve(query)
	if err != nil {
		fmt.Printf("Encountered and error with retrieve...\n")
		ErrorAndExit(err.Error())
	}
	for name, data := range files {
		file := filepath.Join(root, name)
		dir := filepath.Dir(file)
		if err := os.MkdirAll(dir, 0755); err != nil {
			ErrorAndExit(err.Error())
		}
		if err := ioutil.WriteFile(filepath.Join(root, name), data, 0644); err != nil {
			ErrorAndExit(err.Error())
		}
	}
	fmt.Printf("Exported to %s\n", root)
}
<|endoftext|>


<commit_before>package itchio

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"reflect"
	"strings"
)

type Client struct {
	Key        string
	HTTPClient *http.Client
	BaseURL    string
}

type Response struct {
	Errors []string
}

type User struct {
	ID       int64
	Username string
	CoverUrl string `json:"cover_url"`
}

type Game struct {
	ID  int64
	Url string
}

type Upload struct {
	ID       int64
	Filename string
	Size     int64

	OSX     bool `json:"p_osx"`
	Linux   bool `json:"p_linux"`
	Windows bool `json:"p_windows"`
	Android bool `json:"p_android"`
}

func ClientWithKey(key string) *Client {
	return &Client{
		Key:        key,
		HTTPClient: &http.Client{},
		BaseURL:    "https://itch.io/api/1",
	}
}

type MyGamesResponse struct {
	Response

	Games []Game
}

func (c *Client) MyGames() (r MyGamesResponse, err error) {
	path := c.MakePath("my-games")

	resp, err := c.Get(path)
	if err != nil {
		return
	}

	err = ParseAPIResponse(&r, resp)
	return
}

type GameUploadsResponse struct {
	Response

	Uploads []Upload `json:"uploads"`
}

func (c *Client) GameUploads(gameID int64) (r GameUploadsResponse, err error) {
	path := c.MakePath("game/%d/uploads", gameID)

	resp, err := c.Get(path)
	if err != nil {
		return
	}

	err = ParseAPIResponse(&r, resp)
	return
}

type UploadDownloadResponse struct {
	Response

	Url string
}

func (c *Client) UploadDownload(uploadID int64) (r UploadDownloadResponse, err error) {
	path := c.MakePath("upload/%d/download", uploadID)

	resp, err := c.Get(path)
	if err != nil {
		return
	}

	err = ParseAPIResponse(&r, resp)
	return r, err
}

type NewBuildResponse struct {
	Response

	Build struct {
		ID          int64 `json:"id"`
		UploadID    int64 `json:"upload_id"`
		ParentBuild struct {
			ID int64 `json:"id"`
		} `json:"parent_build"`
	}
}

func (c *Client) CreateBuild(target string, channel string) (r NewBuildResponse, err error) {
	path := c.MakePath("wharf/builds")

	form := url.Values{}
	form.Add("target", target)
	form.Add("channel", channel)

	resp, err := c.PostForm(path, form)
	if err != nil {
		return
	}

	err = ParseAPIResponse(&r, resp)
	return
}

type BuildFileType string

const (
	BuildFileType_PATCH     BuildFileType = "patch"
	BuildFileType_ARCHIVE                 = "archive"
	BuildFileType_SIGNATURE               = "signature"
)

type BuildFile struct {
	ID   int64
	Type BuildFileType
	Size int64
}

type ListBuildFilesResponse struct {
	Response

	Files []BuildFile
}

func (c *Client) ListBuildFiles(buildID int64) (r ListBuildFilesResponse, err error) {
	path := c.MakePath("wharf/builds/%d/files", buildID)

	resp, err := c.Get(path)
	if err != nil {
		return
	}

	err = ParseAPIResponse(&r, resp)
	return
}

type NewBuildFileResponse struct {
	Response

	File struct {
		ID           int64
		UploadURL    string            `json:"upload_url"`
		UploadParams map[string]string `json:"upload_params"`
	}
}

func (c *Client) CreateBuildFile(buildID int64, fileType BuildFileType) (r NewBuildFileResponse, err error) {
	path := c.MakePath("wharf/builds/%d/files", buildID)

	form := url.Values{}
	form.Add("type", string(fileType))

	resp, err := c.PostForm(path, form)
	if err != nil {
		return
	}

	err = ParseAPIResponse(&r, resp)
	return
}

type FinalizeBuildFileResponse struct {
	Response
}

func (c *Client) FinalizeBuildFile(buildID int64, fileID int64, size int64) (r FinalizeBuildFileResponse, err error) {
	path := c.MakePath("wharf/builds/%d/files/%d", buildID, fileID)

	form := url.Values{}
	form.Add("size", fmt.Sprintf("%d", size))

	resp, err := c.PostForm(path, form)
	if err != nil {
		return
	}

	err = ParseAPIResponse(&r, resp)
	return
}

type DownloadBuildFileResponse struct {
	Response

	URL string
}

func (c *Client) DownloadBuildFile(buildID int64, fileID int64) (reader io.ReadCloser, err error) {
	path := c.MakePath("wharf/builds/%d/files/%d/download", buildID, fileID)

	resp, err := c.Get(path)
	if err != nil {
		return
	}

	var r DownloadBuildFileResponse
	err = ParseAPIResponse(&r, resp)
	if err != nil {
		return
	}

	req, err := http.NewRequest("GET", r.URL, nil)
	if err != nil {
		return
	}

	// not an API request, going directly with http's DefaultClient
	dlResp, err := http.DefaultClient.Do(req)
	if err != nil {
		return
	}

	reader = dlResp.Body
	return
}

type BuildEventType string

const (
	BuildEvent_JOB_STARTED   BuildEventType = "job_started"
	BuildEvent_JOB_FAILED                   = "job_failed"
	BuildEvent_JOB_COMPLETED                = "job_completed"
)

type BuildEventData map[string]interface{}

type NewBuildEventResponse struct {
	Response
}

func (c *Client) CreateBuildEvent(buildID int64, eventType BuildEventType, message string, data BuildEventData) (r NewBuildEventResponse, err error) {
	path := c.MakePath("wharf/builds/%d/events", buildID)

	form := url.Values{}
	form.Add("type", string(eventType))
	form.Add("message", message)

	jsonData, err := json.Marshal(data)
	if err != nil {
		return
	}
	form.Add("data", string(jsonData))

	resp, err := c.PostForm(path, form)
	if err != nil {
		return
	}

	err = ParseAPIResponse(&r, resp)
	return
}

type BuildEvent struct {
	Type    BuildEventType
	Message string
	Data    BuildEventData
}

type ListBuildEventsResponse struct {
	Response

	Events []BuildEvent
}

func (c *Client) ListBuildEvents(buildID int64) (r ListBuildEventsResponse, err error) {
	path := c.MakePath("wharf/builds/%d/events", buildID)

	resp, err := c.Get(path)
	if err != nil {
		return
	}

	err = ParseAPIResponse(&r, resp)
	return
}

// Helpers

func (c *Client) Get(url string) (*http.Response, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	return c.Do(req)
}

func (c *Client) PostForm(url string, data url.Values) (*http.Response, error) {
	req, err := http.NewRequest("POST", url, strings.NewReader(data.Encode()))
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	return c.Do(req)
}

func (c *Client) Do(req *http.Request) (*http.Response, error) {
	if strings.HasPrefix(c.Key, "jwt:") {
		req.Header.Add("Authorization", strings.Split(c.Key, ":")[1])
	}
	return http.DefaultClient.Do(req)
}

func (c *Client) MakePath(format string, a ...interface{}) string {
	base := strings.Trim(c.BaseURL, "/")
	subPath := strings.Trim(fmt.Sprintf(format, a...), "/")

	var key string
	if strings.HasPrefix(c.Key, "jwt:") {
		key = "jwt"
	} else {
		key = c.Key
	}
	return fmt.Sprintf("%s/%s/%s", base, key, subPath)
}

func ParseAPIResponse(dst interface{}, res *http.Response) error {
	bodyReader := res.Body
	defer bodyReader.Close()

	if res.StatusCode/100 != 2 {
		return fmt.Errorf("Server returned %s for %s", res.Status, res.Request.URL.String())
	}

	err := json.NewDecoder(bodyReader).Decode(dst)
	if err != nil {
		return err
	}

	errs := reflect.Indirect(reflect.ValueOf(dst)).FieldByName("Errors")
	if errs.Len() > 0 {
		// TODO: handle other errors too
		return errors.New(errs.Index(0).String())
	}

	return nil
}
<commit_msg>Check for download server response status<commit_after>package itchio

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"reflect"
	"strings"
)

type Client struct {
	Key        string
	HTTPClient *http.Client
	BaseURL    string
}

type Response struct {
	Errors []string
}

type User struct {
	ID       int64
	Username string
	CoverUrl string `json:"cover_url"`
}

type Game struct {
	ID  int64
	Url string
}

type Upload struct {
	ID       int64
	Filename string
	Size     int64

	OSX     bool `json:"p_osx"`
	Linux   bool `json:"p_linux"`
	Windows bool `json:"p_windows"`
	Android bool `json:"p_android"`
}

func ClientWithKey(key string) *Client {
	return &Client{
		Key:        key,
		HTTPClient: &http.Client{},
		BaseURL:    "https://itch.io/api/1",
	}
}

type MyGamesResponse struct {
	Response

	Games []Game
}

func (c *Client) MyGames() (r MyGamesResponse, err error) {
	path := c.MakePath("my-games")

	resp, err := c.Get(path)
	if err != nil {
		return
	}

	err = ParseAPIResponse(&r, resp)
	return
}

type GameUploadsResponse struct {
	Response

	Uploads []Upload `json:"uploads"`
}

func (c *Client) GameUploads(gameID int64) (r GameUploadsResponse, err error) {
	path := c.MakePath("game/%d/uploads", gameID)

	resp, err := c.Get(path)
	if err != nil {
		return
	}

	err = ParseAPIResponse(&r, resp)
	return
}

type UploadDownloadResponse struct {
	Response

	Url string
}

func (c *Client) UploadDownload(uploadID int64) (r UploadDownloadResponse, err error) {
	path := c.MakePath("upload/%d/download", uploadID)

	resp, err := c.Get(path)
	if err != nil {
		return
	}

	err = ParseAPIResponse(&r, resp)
	return r, err
}

type NewBuildResponse struct {
	Response

	Build struct {
		ID          int64 `json:"id"`
		UploadID    int64 `json:"upload_id"`
		ParentBuild struct {
			ID int64 `json:"id"`
		} `json:"parent_build"`
	}
}

func (c *Client) CreateBuild(target string, channel string) (r NewBuildResponse, err error) {
	path := c.MakePath("wharf/builds")

	form := url.Values{}
	form.Add("target", target)
	form.Add("channel", channel)

	resp, err := c.PostForm(path, form)
	if err != nil {
		return
	}

	err = ParseAPIResponse(&r, resp)
	return
}

type BuildFileType string

const (
	BuildFileType_PATCH     BuildFileType = "patch"
	BuildFileType_ARCHIVE                 = "archive"
	BuildFileType_SIGNATURE               = "signature"
)

type BuildFile struct {
	ID   int64
	Type BuildFileType
	Size int64
}

type ListBuildFilesResponse struct {
	Response

	Files []BuildFile
}

func (c *Client) ListBuildFiles(buildID int64) (r ListBuildFilesResponse, err error) {
	path := c.MakePath("wharf/builds/%d/files", buildID)

	resp, err := c.Get(path)
	if err != nil {
		return
	}

	err = ParseAPIResponse(&r, resp)
	return
}

type NewBuildFileResponse struct {
	Response

	File struct {
		ID           int64
		UploadURL    string            `json:"upload_url"`
		UploadParams map[string]string `json:"upload_params"`
	}
}

func (c *Client) CreateBuildFile(buildID int64, fileType BuildFileType) (r NewBuildFileResponse, err error) {
	path := c.MakePath("wharf/builds/%d/files", buildID)

	form := url.Values{}
	form.Add("type", string(fileType))

	resp, err := c.PostForm(path, form)
	if err != nil {
		return
	}

	err = ParseAPIResponse(&r, resp)
	return
}

type FinalizeBuildFileResponse struct {
	Response
}

func (c *Client) FinalizeBuildFile(buildID int64, fileID int64, size int64) (r FinalizeBuildFileResponse, err error) {
	path := c.MakePath("wharf/builds/%d/files/%d", buildID, fileID)

	form := url.Values{}
	form.Add("size", fmt.Sprintf("%d", size))

	resp, err := c.PostForm(path, form)
	if err != nil {
		return
	}

	err = ParseAPIResponse(&r, resp)
	return
}

type DownloadBuildFileResponse struct {
	Response

	URL string
}

func (c *Client) DownloadBuildFile(buildID int64, fileID int64) (reader io.ReadCloser, err error) {
	path := c.MakePath("wharf/builds/%d/files/%d/download", buildID, fileID)

	resp, err := c.Get(path)
	if err != nil {
		return
	}

	var r DownloadBuildFileResponse
	err = ParseAPIResponse(&r, resp)
	if err != nil {
		return
	}

	req, err := http.NewRequest("GET", r.URL, nil)
	if err != nil {
		return
	}

	// not an API request, going directly with http's DefaultClient
	dlResp, err := http.DefaultClient.Do(req)
	if err != nil {
		return
	}

	if dlResp.StatusCode != 200 {
		err = fmt.Errorf("Can't download: %s", dlResp.Status)
		return
	}

	reader = dlResp.Body
	return
}

type BuildEventType string

const (
	BuildEvent_JOB_STARTED   BuildEventType = "job_started"
	BuildEvent_JOB_FAILED                   = "job_failed"
	BuildEvent_JOB_COMPLETED                = "job_completed"
)

type BuildEventData map[string]interface{}

type NewBuildEventResponse struct {
	Response
}

func (c *Client) CreateBuildEvent(buildID int64, eventType BuildEventType, message string, data BuildEventData) (r NewBuildEventResponse, err error) {
	path := c.MakePath("wharf/builds/%d/events", buildID)

	form := url.Values{}
	form.Add("type", string(eventType))
	form.Add("message", message)

	jsonData, err := json.Marshal(data)
	if err != nil {
		return
	}
	form.Add("data", string(jsonData))

	resp, err := c.PostForm(path, form)
	if err != nil {
		return
	}

	err = ParseAPIResponse(&r, resp)
	return
}

type BuildEvent struct {
	Type    BuildEventType
	Message string
	Data    BuildEventData
}

type ListBuildEventsResponse struct {
	Response

	Events []BuildEvent
}

func (c *Client) ListBuildEvents(buildID int64) (r ListBuildEventsResponse, err error) {
	path := c.MakePath("wharf/builds/%d/events", buildID)

	resp, err := c.Get(path)
	if err != nil {
		return
	}

	err = ParseAPIResponse(&r, resp)
	return
}

// Helpers

func (c *Client) Get(url string) (*http.Response, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	return c.Do(req)
}

func (c *Client) PostForm(url string, data url.Values) (*http.Response, error) {
	req, err := http.NewRequest("POST", url, strings.NewReader(data.Encode()))
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	return c.Do(req)
}

func (c *Client) Do(req *http.Request) (*http.Response, error) {
	if strings.HasPrefix(c.Key, "jwt:") {
		req.Header.Add("Authorization", strings.Split(c.Key, ":")[1])
	}
	return http.DefaultClient.Do(req)
}

func (c *Client) MakePath(format string, a ...interface{}) string {
	base := strings.Trim(c.BaseURL, "/")
	subPath := strings.Trim(fmt.Sprintf(format, a...), "/")

	var key string
	if strings.HasPrefix(c.Key, "jwt:") {
		key = "jwt"
	} else {
		key = c.Key
	}
	return fmt.Sprintf("%s/%s/%s", base, key, subPath)
}

func ParseAPIResponse(dst interface{}, res *http.Response) error {
	bodyReader := res.Body
	defer bodyReader.Close()

	if res.StatusCode/100 != 2 {
		return fmt.Errorf("Server returned %s for %s", res.Status, res.Request.URL.String())
	}

	err := json.NewDecoder(bodyReader).Decode(dst)
	if err != nil {
		return err
	}

	errs := reflect.Indirect(reflect.ValueOf(dst)).FieldByName("Errors")
	if errs.Len() > 0 {
		// TODO: handle other errors too
		return errors.New(errs.Index(0).String())
	}

	return nil
}
<|endoftext|>


<commit_before>package model

import (
	"database/sql"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strconv"

	"github.com/af83/edwig/logger"
)

/* CSV Structure

stop_area,Id,ParentId,ModelName,Name,ObjectIDs,LineIds,Attributes,References,NextCollectAt,CollectedAt,CollectedUntil,CollectedAlways,CollectChildren
line,Id,ModelName,Name,ObjectIDs,Attributes,References,CollectGeneralMessages
vehicle_journey,Id,ModelName,Name,ObjectIDs,LineId,Attributes,References
stop_visit,Id,ModelName,ObjectIDs,StopAreaId,VehicleJourneyId,ArrivalStatus,DepartureStatus,Schedules,Attributes,References,Collected,VehicleAtStop,PassageOrder

Comments are '#'
Separators are ',' leading spaces are trimed

Escape quotes with another quote ex: "[""1234"",""5678""]"
*/

func LoadFromCSV(filePath string, referentialSlug string) error {
	prepareDatabase()

	file, err := os.Open(filePath)
	defer file.Close()
	if err != nil {
		return fmt.Errorf("Error while opening file: %v", err)
	}

	reader := csv.NewReader(file)
	reader.Comment = '#'
	reader.FieldsPerRecord = -1
	reader.LazyQuotes = true
	reader.TrimLeadingSpace = true

	importedStopAreas := 0
	importedLines := 0
	importedVehicleJourneys := 0
	importedStopVisits := 0
	importedOperators := 0

	var i int
	for {
		i++
		record, err := reader.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			logger.Log.Debugf("Error while reading %v", err)
			continue
		}

		switch record[0] {
		case "stop_area":
			err := handleStopArea(record, referentialSlug)
			if err != nil {
				logger.Log.Debugf("Error on line %d: %v", i, err)
			} else {
				importedStopAreas++
			}
		case "line":
			err := handleLine(record, referentialSlug)
			if err != nil {
				logger.Log.Debugf("Error on line %d: %v", i, err)
			} else {
				importedLines++
			}
		case "vehicle_journey":
			err := handleVehicleJourney(record, referentialSlug)
			if err != nil {
				logger.Log.Debugf("Error on line %d: %v", i, err)
			} else {
				importedVehicleJourneys++
			}
		case "stop_visit":
			err := handleStopVisit(record, referentialSlug)
			if err != nil {
				logger.Log.Debugf("Error on line %d: %v", i, err)
			} else {
				importedStopVisits++
			}
		case "operator":
			err := handleOperator(record, referentialSlug)
			if err != nil {
				logger.Log.Debugf("Error on line %d: %v", i, err)
			} else {
				importedOperators++
			}
		default:
			logger.Log.Debugf("Unknown record type: %v", record[0])
			continue
		}
	}

	logger.Log.Debugf("Import successful")
	logger.Log.Debugf("  %v StopAreas", importedStopAreas)
	logger.Log.Debugf("  %v Lines", importedLines)
	logger.Log.Debugf("  %v VehicleJourneys", importedVehicleJourneys)
	logger.Log.Debugf("  %v StopVisits", importedStopVisits)
	logger.Log.Debugf("  %v Operators", importedOperators)

	return nil
}

func prepareDatabase() {
	Database.AddTableWithName(DatabaseStopArea{}, "stop_areas")
	Database.AddTableWithName(DatabaseLine{}, "lines")
	Database.AddTableWithName(DatabaseVehicleJourney{}, "vehicle_journeys")
	Database.AddTableWithName(DatabaseStopVisit{}, "stop_visits")
	Database.AddTableWithName(DatabaseOperator{}, "operators")
}

func handleStopArea(record []string, referentialSlug string) error {
	if len(record) != 11 {
		return fmt.Errorf("Wrong number of entries, expected 11 got %v", len(record))
	}

	var err error
	parseErrors := make(map[string]string)
	var parent sql.NullString
	if record[2] != "" {
		parent = sql.NullString{
			String: record[2],
			Valid:  true,
		}
	}

	var collectedAlways bool
	if record[9] != "" {
		collectedAlways, err = strconv.ParseBool(record[9])
		if err != nil {
			parseErrors["CollectedAlways"] = err.Error()
		}
	}

	var collectChildren bool
	if record[10] != "" {
		collectChildren, err = strconv.ParseBool(record[10])
		if err != nil {
			parseErrors["CollectChildren"] = err.Error()
		}
	}

	if len(parseErrors) != 0 {
		json, _ := json.Marshal(parseErrors)
		return fmt.Errorf(string(json))
	}

	stopArea := DatabaseStopArea{
		Id:              record[1],
		ReferentialSlug: referentialSlug,
		ParentId:        parent,
		ModelName:       record[3],
		Name:            record[4],
		ObjectIDs:       record[5],
		LineIds:         record[6],
		Attributes:      record[7],
		References:      record[8],
		CollectedAlways: collectedAlways,
		CollectChildren: collectChildren,
	}

	err = Database.Insert(&stopArea)
	if err != nil {
		return err
	}

	return nil
}

func handleOperator(record []string, referentialSlug string) error {
	if len(record) != 5 {
		return fmt.Errorf("Wrong number of entries, expected 5 got %v", len(record))
	}

	operator := DatabaseOperator{
		Id:              record[1],
		ReferentialSlug: referentialSlug,
		ModelName:       record[2],
		Name:            record[3],
		ObjectIDs:       record[4],
	}

	err := Database.Insert(&operator)
	if err != nil {
		return err
	}

	return nil
}

func handleLine(record []string, referentialSlug string) error {
	if len(record) != 8 {
		return fmt.Errorf("Wrong number of entries, expected 8 got %v", len(record))
	}

	var err error
	parseErrors := make(map[string]string)

	var collectGeneralMessages bool
	if record[7] != "" {
		collectGeneralMessages, err = strconv.ParseBool(record[7])
		if err != nil {
			parseErrors["CollectGeneralMessages"] = err.Error()
		}
	}

	line := DatabaseLine{
		Id:                     record[1],
		ReferentialSlug:        referentialSlug,
		ModelName:              record[2],
		Name:                   record[3],
		ObjectIDs:              record[4],
		Attributes:             record[5],
		References:             record[6],
		CollectGeneralMessages: collectGeneralMessages,
	}

	err = Database.Insert(&line)
	if err != nil {
		return err
	}

	return nil
}

func handleVehicleJourney(record []string, referentialSlug string) error {
	if len(record) != 8 {
		return fmt.Errorf("Wrong number of entries, expected 8 got %v", len(record))
	}

	vehicleJourney := DatabaseVehicleJourney{
		Id:              record[1],
		ReferentialSlug: referentialSlug,
		ModelName:       record[2],
		Name:            record[3],
		ObjectIDs:       record[4],
		LineId:          record[5],
		Attributes:      record[6],
		References:      record[7],
	}

	err := Database.Insert(&vehicleJourney)
	if err != nil {
		return err
	}

	return nil
}

func handleStopVisit(record []string, referentialSlug string) error {
	if len(record) != 14 {
		return fmt.Errorf("Wrong number of entries, expected 14 got %v", len(record))
	}

	var err error
	parseErrors := make(map[string]string)

	var collected bool
	if record[11] != "" {
		collected, err = strconv.ParseBool(record[11])
		if err != nil {
			parseErrors["Collected"] = err.Error()
		}
	}

	var vehicleAtStop bool
	if record[12] != "" {
		vehicleAtStop, err = strconv.ParseBool(record[12])
		if err != nil {
			parseErrors["VehicleAtStop"] = err.Error()
		}
	}

	var passageOrder int
	if record[13] != "" {
		passageOrder, err = strconv.Atoi(record[13])
		if err != nil {
			parseErrors["PassageOrder"] = err.Error()
		}
	}

	if len(parseErrors) != 0 {
		json, _ := json.Marshal(parseErrors)
		return fmt.Errorf(string(json))
	}

	stopVisit := DatabaseStopVisit{
		Id:               record[1],
		ReferentialSlug:  referentialSlug,
		ModelName:        record[2],
		ObjectIDs:        record[3],
		StopAreaId:       record[4],
		VehicleJourneyId: record[5],
		ArrivalStatus:    record[6],
		DepartureStatus:  record[7],
		Schedules:        record[8],
		Attributes:       record[9],
		References:       record[10],
		Collected:        collected,
		VehicleAtStop:    vehicleAtStop,
		PassageOrder:     passageOrder,
	}

	err = Database.Insert(&stopVisit)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Refs #5239 We now display error / success message on their standard output<commit_after>package model

import (
	"database/sql"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strconv"

	"github.com/af83/edwig/logger"
)

/* CSV Structure

stop_area,Id,ParentId,ModelName,Name,ObjectIDs,LineIds,Attributes,References,NextCollectAt,CollectedAt,CollectedUntil,CollectedAlways,CollectChildren
line,Id,ModelName,Name,ObjectIDs,Attributes,References,CollectGeneralMessages
vehicle_journey,Id,ModelName,Name,ObjectIDs,LineId,Attributes,References
stop_visit,Id,ModelName,ObjectIDs,StopAreaId,VehicleJourneyId,ArrivalStatus,DepartureStatus,Schedules,Attributes,References,Collected,VehicleAtStop,PassageOrder

Comments are '#'
Separators are ',' leading spaces are trimed

Escape quotes with another quote ex: "[""1234"",""5678""]"
*/

func LoadFromCSV(filePath string, referentialSlug string) error {
	prepareDatabase()

	file, err := os.Open(filePath)
	defer file.Close()
	if err != nil {
		return fmt.Errorf("Error while opening file: %v", err)
	}

	reader := csv.NewReader(file)
	reader.Comment = '#'
	reader.FieldsPerRecord = -1
	reader.LazyQuotes = true
	reader.TrimLeadingSpace = true

	importedStopAreas := 0
	importedLines := 0
	importedVehicleJourneys := 0
	importedStopVisits := 0
	importedOperators := 0

	var i int
	for {
		i++
		record, err := reader.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			logger.Log.Debugf("Error while reading %v", err)
			fmt.Errorf("Error while reading %v", err)
			continue
		}

		switch record[0] {
		case "stop_area":
			err := handleStopArea(record, referentialSlug)
			if err != nil {
				logger.Log.Debugf("Error on line %d: %v", i, err)
				fmt.Errorf("Error on line %d: %v", i, err)
			} else {
				importedStopAreas++
			}
		case "line":
			err := handleLine(record, referentialSlug)
			if err != nil {
				logger.Log.Debugf("Error on line %d: %v", i, err)
				fmt.Errorf("Error on line %d: %v", i, err)
			} else {
				importedLines++
			}
		case "vehicle_journey":
			err := handleVehicleJourney(record, referentialSlug)
			if err != nil {
				logger.Log.Debugf("Error on line %d: %v", i, err)
				fmt.Errorf("Error on line %d: %v", i, err)
			} else {
				importedVehicleJourneys++
			}
		case "stop_visit":
			err := handleStopVisit(record, referentialSlug)
			if err != nil {
				logger.Log.Debugf("Error on line %d: %v", i, err)
				fmt.Errorf("Error on line %d: %v", i, err)
			} else {
				importedStopVisits++
			}
		case "operator":
			err := handleOperator(record, referentialSlug)
			if err != nil {
				logger.Log.Debugf("Error on line %d: %v", i, err)
				fmt.Errorf("Error on line %d: %v", i, err)
			} else {
				importedOperators++
			}
		default:
			logger.Log.Debugf("Unknown record type: %v", record[0])
			fmt.Errorf("Unknown record type: %v", record[0])
			continue
		}
	}

	logger.Log.Debugf("Import successful")
	logger.Log.Debugf("  %v StopAreas", importedStopAreas)
	logger.Log.Debugf("  %v Lines", importedLines)
	logger.Log.Debugf("  %v VehicleJourneys", importedVehicleJourneys)
	logger.Log.Debugf("  %v StopVisits", importedStopVisits)
	logger.Log.Debugf("  %v Operators", importedOperators)

	fmt.Println("Import successful")
	fmt.Printf("  %v StopAreas", importedStopAreas)
	fmt.Printf("  %v Lines", importedLines)
	fmt.Printf("  %v VehicleJourneys", importedVehicleJourneys)
	fmt.Printf("  %v StopVisits", importedStopVisits)
	fmt.Printf("  %v Operators", importedOperators)

	return nil
}

func prepareDatabase() {
	Database.AddTableWithName(DatabaseStopArea{}, "stop_areas")
	Database.AddTableWithName(DatabaseLine{}, "lines")
	Database.AddTableWithName(DatabaseVehicleJourney{}, "vehicle_journeys")
	Database.AddTableWithName(DatabaseStopVisit{}, "stop_visits")
	Database.AddTableWithName(DatabaseOperator{}, "operators")
}

func handleStopArea(record []string, referentialSlug string) error {
	if len(record) != 11 {
		return fmt.Errorf("Wrong number of entries, expected 11 got %v", len(record))
	}

	var err error
	parseErrors := make(map[string]string)
	var parent sql.NullString
	if record[2] != "" {
		parent = sql.NullString{
			String: record[2],
			Valid:  true,
		}
	}

	var collectedAlways bool
	if record[9] != "" {
		collectedAlways, err = strconv.ParseBool(record[9])
		if err != nil {
			parseErrors["CollectedAlways"] = err.Error()
		}
	}

	var collectChildren bool
	if record[10] != "" {
		collectChildren, err = strconv.ParseBool(record[10])
		if err != nil {
			parseErrors["CollectChildren"] = err.Error()
		}
	}

	if len(parseErrors) != 0 {
		json, _ := json.Marshal(parseErrors)
		return fmt.Errorf(string(json))
	}

	stopArea := DatabaseStopArea{
		Id:              record[1],
		ReferentialSlug: referentialSlug,
		ParentId:        parent,
		ModelName:       record[3],
		Name:            record[4],
		ObjectIDs:       record[5],
		LineIds:         record[6],
		Attributes:      record[7],
		References:      record[8],
		CollectedAlways: collectedAlways,
		CollectChildren: collectChildren,
	}

	err = Database.Insert(&stopArea)
	if err != nil {
		return err
	}

	return nil
}

func handleOperator(record []string, referentialSlug string) error {
	if len(record) != 5 {
		return fmt.Errorf("Wrong number of entries, expected 5 got %v", len(record))
	}

	operator := DatabaseOperator{
		Id:              record[1],
		ReferentialSlug: referentialSlug,
		ModelName:       record[2],
		Name:            record[3],
		ObjectIDs:       record[4],
	}

	err := Database.Insert(&operator)
	if err != nil {
		return err
	}

	return nil
}

func handleLine(record []string, referentialSlug string) error {
	if len(record) != 8 {
		return fmt.Errorf("Wrong number of entries, expected 8 got %v", len(record))
	}

	var err error
	parseErrors := make(map[string]string)

	var collectGeneralMessages bool
	if record[7] != "" {
		collectGeneralMessages, err = strconv.ParseBool(record[7])
		if err != nil {
			parseErrors["CollectGeneralMessages"] = err.Error()
		}
	}

	line := DatabaseLine{
		Id:                     record[1],
		ReferentialSlug:        referentialSlug,
		ModelName:              record[2],
		Name:                   record[3],
		ObjectIDs:              record[4],
		Attributes:             record[5],
		References:             record[6],
		CollectGeneralMessages: collectGeneralMessages,
	}

	err = Database.Insert(&line)
	if err != nil {
		return err
	}

	return nil
}

func handleVehicleJourney(record []string, referentialSlug string) error {
	if len(record) != 8 {
		return fmt.Errorf("Wrong number of entries, expected 8 got %v", len(record))
	}

	vehicleJourney := DatabaseVehicleJourney{
		Id:              record[1],
		ReferentialSlug: referentialSlug,
		ModelName:       record[2],
		Name:            record[3],
		ObjectIDs:       record[4],
		LineId:          record[5],
		Attributes:      record[6],
		References:      record[7],
	}

	err := Database.Insert(&vehicleJourney)
	if err != nil {
		return err
	}

	return nil
}

func handleStopVisit(record []string, referentialSlug string) error {
	if len(record) != 14 {
		return fmt.Errorf("Wrong number of entries, expected 14 got %v", len(record))
	}

	var err error
	parseErrors := make(map[string]string)

	var collected bool
	if record[11] != "" {
		collected, err = strconv.ParseBool(record[11])
		if err != nil {
			parseErrors["Collected"] = err.Error()
		}
	}

	var vehicleAtStop bool
	if record[12] != "" {
		vehicleAtStop, err = strconv.ParseBool(record[12])
		if err != nil {
			parseErrors["VehicleAtStop"] = err.Error()
		}
	}

	var passageOrder int
	if record[13] != "" {
		passageOrder, err = strconv.Atoi(record[13])
		if err != nil {
			parseErrors["PassageOrder"] = err.Error()
		}
	}

	if len(parseErrors) != 0 {
		json, _ := json.Marshal(parseErrors)
		return fmt.Errorf(string(json))
	}

	stopVisit := DatabaseStopVisit{
		Id:               record[1],
		ReferentialSlug:  referentialSlug,
		ModelName:        record[2],
		ObjectIDs:        record[3],
		StopAreaId:       record[4],
		VehicleJourneyId: record[5],
		ArrivalStatus:    record[6],
		DepartureStatus:  record[7],
		Schedules:        record[8],
		Attributes:       record[9],
		References:       record[10],
		Collected:        collected,
		VehicleAtStop:    vehicleAtStop,
		PassageOrder:     passageOrder,
	}

	err = Database.Insert(&stopVisit)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"github.com/nutrun/lentil"
	"sort"
	"strconv"
	"strings"
)

type JobQueue struct {
	q     *lentil.Beanstalkd
	tubes map[int]*Tubes
}

type Tube struct {
	majorPriority int
	minorPriority int
	jobs          int
	name          string
}

type Group struct {
	tubes []*Tube
}

func (this *Group) AddTube(tube *Tube) {
	this.tubes = append(this.tubes, tube)
}

func (this *Group) Jobs() int {
	jobs := 0
	for _, job := range this.tubes {
		jobs += job.jobs
	}
	return jobs
}

type Tubes struct {
	tubes map[int]*Group
}

func (this *Tubes) AddTube(tube *Tube) {
	if _, found := this.tubes[tube.minorPriority]; !found {
		this.tubes[tube.minorPriority] = &Group{make([]*Tube, 0)}
	}
	this.tubes[tube.minorPriority].AddTube(tube)
}

func (this *Tubes) SortMapKeys(in map[int]*Group) []int {
	keys := make([]int, 0)
	for k, _ := range in {
		keys = append(keys, k)
	}
	sort.Ints(keys)
	return keys
}

func (this *Tubes) Groups() chan *Group {
	in := make(chan *Group)
	keys := this.SortMapKeys(this.tubes)
	go func() {
		for _, key := range keys {
			in <- this.tubes[key]
		}
		close(in)
	}()
	return in
}

func NewJobQueue(q *lentil.Beanstalkd) *JobQueue {
	this := new(JobQueue)
	this.q = q
	return this
}

func (this *JobQueue) SortMapKeys(in map[int]*Tubes) []int {
	keys := make([]int, 0)
	for k, _ := range in {
		keys = append(keys, k)
	}
	sort.Ints(keys)
	return keys
}

func (this *JobQueue) Tubes() chan *Tubes {
	in := make(chan *Tubes)
	keys := this.SortMapKeys(this.tubes)
	go func() {
		for _, key := range keys {
			in <- this.tubes[key]
		}
		close(in)
	}()
	return in
}

func (this *JobQueue) ReserveFromGroup(group *Group) (*lentil.Job, error) {
	for _, tube := range group.tubes {
		_, e := this.q.Watch(tube.name)
		if e != nil {
			return nil, e
		}
	}
	job, res_err := this.q.ReserveWithTimeout(0)
	for _, ignored := range group.tubes {
		_, e := this.q.Ignore(ignored.name)
		if e != nil {
			return nil, e
		}
	}
	if res_err != nil {
		if group.Jobs() > 0 {
			return nil, res_err
		}
		return nil, nil
	}
	return job, nil
}

func (this *JobQueue) Next() (*lentil.Job, error) {
	this.refreshTubes()
	for tube := range this.Tubes() {
		for group := range tube.Groups() {
			job, err := this.ReserveFromGroup(group)
			if err != nil {
				return nil, err
			}
			if job != nil {
				return job, nil
			}
		}
		return nil, errors.New("TIMED_OUT")
	}
	return nil, errors.New("TIMED_OUT")
}

func (this *JobQueue) Delete(id uint64) error {
	return this.q.Delete(id)
}

func (this *JobQueue) priority(tube string) (int, int, error) {
	index := strings.LastIndex(tube, "_")
	priority, err := strconv.Atoi(tube[index+1:])
	if err != nil {
		return 0, 0, err
	}
	return int(priority >> 16), int(priority & 0x0000FFFF), nil
}

func (this *JobQueue) Stats(f func(tubes []*Tube)) {
	this.refreshTubes()
	for tube := range this.Tubes() {
		for group := range tube.Groups() {
			f(group.tubes)
		}
	}
}

func (this *JobQueue) refreshTubes() error {
	this.tubes = make(map[int]*Tubes)
	names, e := this.q.ListTubes()
	if e != nil {
		return e
	}
	for _, tube := range names {
		if tube == "default" {
			continue
		}
		major, minor, e := this.priority(tube)
		if e != nil {
			continue
		}
		tubestats, e := this.q.StatsTube(tube)
		if e != nil {
			return e
		}
		ready, _ := strconv.Atoi(tubestats["current-jobs-ready"])
		reserved, _ := strconv.Atoi(tubestats["current-jobs-reserved"])
		delayed, _ := strconv.Atoi(tubestats["current-jobs-delayed"])
		if ready+reserved+delayed > 0 {
			if _, found := this.tubes[major]; !found {
				this.tubes[major] = &Tubes{make(map[int]*Group)}
			}
			this.tubes[major].AddTube(&Tube{major, minor, ready + reserved, tube})
		}
	}
	return nil
}
<commit_msg>got rid of go routines for ranges<commit_after>package main

import (
	"errors"
	"github.com/nutrun/lentil"
	"sort"
	"strconv"
	"strings"
)

type JobQueue struct {
	q     *lentil.Beanstalkd
	tubes map[int]*Tubes
}

type Tube struct {
	majorPriority int
	minorPriority int
	jobs          int
	name          string
}

type Group struct {
	tubes []*Tube
}

func (this *Group) AddTube(tube *Tube) {
	this.tubes = append(this.tubes, tube)
}

func (this *Group) Jobs() int {
	jobs := 0
	for _, job := range this.tubes {
		jobs += job.jobs
	}
	return jobs
}

type Tubes struct {
	tubes map[int]*Group
}

func (this *Tubes) AddTube(tube *Tube) {
	if _, found := this.tubes[tube.minorPriority]; !found {
		this.tubes[tube.minorPriority] = &Group{make([]*Tube, 0)}
	}
	this.tubes[tube.minorPriority].AddTube(tube)
}

func (this *Tubes) SortMapKeys(in map[int]*Group) []int {
	keys := make([]int, 0)
	for k, _ := range in {
		keys = append(keys, k)
	}
	sort.Ints(keys)
	return keys
}

func (this *Tubes) Groups() []*Group {
	out := make([]*Group, 0)
	keys := this.SortMapKeys(this.tubes)
	for _, key := range keys {
		out = append(out, this.tubes[key])
	}
	return out
}

func NewJobQueue(q *lentil.Beanstalkd) *JobQueue {
	this := new(JobQueue)
	this.q = q
	return this
}

func (this *JobQueue) SortMapKeys(in map[int]*Tubes) []int {
	keys := make([]int, 0)
	for k, _ := range in {
		keys = append(keys, k)
	}
	sort.Ints(keys)
	return keys
}

func (this *JobQueue) Tubes() []*Tubes {
	out := make([]*Tubes, 0)
	keys := this.SortMapKeys(this.tubes)
	for _, key := range keys {
		out = append(out, this.tubes[key])
	}
	return out
}

func (this *JobQueue) ReserveFromGroup(group *Group) (*lentil.Job, error) {
	for _, tube := range group.tubes {
		_, e := this.q.Watch(tube.name)
		if e != nil {
			return nil, e
		}
	}
	job, res_err := this.q.ReserveWithTimeout(0)
	for _, ignored := range group.tubes {
		_, e := this.q.Ignore(ignored.name)
		if e != nil {
			return nil, e
		}
	}
	if res_err != nil {
		if group.Jobs() > 0 {
			return nil, res_err
		}
		return nil, nil
	}
	return job, nil
}

func (this *JobQueue) Next() (*lentil.Job, error) {
	this.refreshTubes()
	for _, tube := range this.Tubes() {
		for _, group := range tube.Groups() {
			job, err := this.ReserveFromGroup(group)
			if err != nil {
				return nil, err
			}
			if job != nil {
				return job, nil
			}
		}
		return nil, errors.New("TIMED_OUT")
	}
	return nil, errors.New("TIMED_OUT")
}

func (this *JobQueue) Delete(id uint64) error {
	return this.q.Delete(id)
}

func (this *JobQueue) priority(tube string) (int, int, error) {
	index := strings.LastIndex(tube, "_")
	priority, err := strconv.Atoi(tube[index+1:])
	if err != nil {
		return 0, 0, err
	}
	return int(priority >> 16), int(priority & 0x0000FFFF), nil
}

func (this *JobQueue) Stats(f func(tubes []*Tube)) {
	this.refreshTubes()
	for _, tube := range this.Tubes() {
		for _, group := range tube.Groups() {
			f(group.tubes)
		}
	}
}

func (this *JobQueue) refreshTubes() error {
	this.tubes = make(map[int]*Tubes)
	names, e := this.q.ListTubes()
	if e != nil {
		return e
	}
	for _, tube := range names {
		if tube == "default" {
			continue
		}
		major, minor, e := this.priority(tube)
		if e != nil {
			continue
		}
		tubestats, e := this.q.StatsTube(tube)
		if e != nil {
			return e
		}
		ready, _ := strconv.Atoi(tubestats["current-jobs-ready"])
		reserved, _ := strconv.Atoi(tubestats["current-jobs-reserved"])
		delayed, _ := strconv.Atoi(tubestats["current-jobs-delayed"])
		if ready+reserved+delayed > 0 {
			if _, found := this.tubes[major]; !found {
				this.tubes[major] = &Tubes{make(map[int]*Group)}
			}
			this.tubes[major].AddTube(&Tube{major, minor, ready + reserved, tube})
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"fmt"
	roman "github.com/StefanSchroeder/Golang-Roman"
	"io/ioutil"
	"log"
	"math/rand"
	"strconv"
	"strings"
	"time"
)

// Cracklib is a random-pop interface to the cracklib wordlist
type Cracklib []string

var words Cracklib

func init() {
	cracklib_location := os.Getenv("DF_CRACKLIB_LOCATION")
	default_cracklib_location := "/usr/share/cracklib/cracklib-small"
	if cracklib_location == "" {
		log.Print("Cracklib location set to default: " + default_cracklib_location)

		cracklib_location = default_cracklib_location
	}
	out, err := ioutil.ReadFile(cracklib_location)
	if err != nil {
		log.Print("Cracklib failed to load. Set location by env variable DF_CRACKLIB_LOCATION")
		return
	}
	words = removeApostrophes(strings.Split(string(out), "\n"))
	log.Printf("Loaded %d cracklib strings", len(words))

	// Also reset the random...
	rand.Seed(time.Now().UnixNano())
}

// Random returns a random string from cracklib
func (c *Cracklib) Random() string {
	return words[rand.Intn(len(words))]
}

// Title returns a random string from cracklib in Title Case
func (c *Cracklib) Title() string {
	return strings.Title(c.Random())
}

// Upper returns a random string from cracklib in UPPERCASE
func (c *Cracklib) Upper() string {
	return strings.ToUpper(c.Random())
}

// FakeName returns a fake player name
func FakeName() string {
	var prefix string
	// Add a last name prefix 80% of the time
	if percentTrue(80) {
		p := []string{"von ", "Mc", "of ", "the ", "De"}
		prefix = p[rand.Intn(len(p))]
	}

	return fmt.Sprintf(
		"%s %s%s",
		words.Title(),
		prefix,
		words.Title(),
	)
}

// FakeNick returns a fake player nickname
func FakeNick() string {
	var tag, prefix, suffix, number, char string

	// Add a clan tag
	if percentTrue(25) {
		content := words.Upper()

		if percentTrue(50) {
			// Half of the time, pick three random uppercase characters.
			b := make([]rune, 3)
			letters := []rune("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
			for i := range b {
				b[i] = letters[rand.Intn(len(letters))]
			}
			content = string(b)
		}

		tag = fmt.Sprintf("[%s]", content)
	}

	// Add a prefix
	if percentTrue(20) {
		p := []string{"sexy", "super", "muscle", "ultra", "mega"}
		prefix = p[rand.Intn(len(p))]
	}

	// Add a suffix
	if percentTrue(33) {
		s := []string{"gamer", "dude", "girl", "man", "lady", ""}
		suffix = s[rand.Intn(len(s))]
	}

	// Add a number between 250 and 1000
	if percentTrue(10) {
		number = strconv.Itoa(rand.Intn(750) + 250)
	}

	// Add a random character at the end
	if percentTrue(10) {
		c := []string{"-", "_", "^", "$", "#", "`", ">"}
		char = c[rand.Intn(len(c))]
	}

	return fmt.Sprintf(
		"%s%s%s%s%s%s",
		tag,
		prefix,
		words.Title(),
		suffix,
		number,
		char,
	)
}

// FakeTournamentTitle returns a fake tournament title and a numeral
func FakeTournamentTitle() (string, string) {
	f := []string{"Faking", "Testing", "Durnkern", "Meta", "Jalo", "Poser"}
	prefix := f[rand.Intn(len(f))]

	// Add a roman numeral between 20 and 400
	x := roman.Roman(rand.Intn(380) + 20)

	// Fake a subtitle
	var subtitle string
	switch rand.Intn(5) {
	case 0:
		subtitle = fmt.Sprintf("The %s of %s", words.Title(), words.Title())
	case 1:
		subtitle = fmt.Sprintf("%s and %s", words.Title(), words.Title())
	case 2:
		subtitle = fmt.Sprintf("%s and %s for %s", words.Title(), words.Title(), words.Title())
	case 3:
		subtitle = fmt.Sprintf("Barry Trotter and the %s of %s", words.Title(), words.Title())
	case 4:
		subtitle = words.Title()
	}

	title := fmt.Sprintf("%sFall %s: %s", prefix, x, subtitle)
	return title, strings.ToLower(x)
}

// FakeAvatar returns a fake avatar URL
func FakeAvatar() string {
	dir := "static/avatars/fake/"
	avatars, err := ioutil.ReadDir(dir)
	if err != nil {
		log.Fatal("No fake avatars available")
	}

	avatar := avatars[rand.Intn(len(avatars))]

	// Add a slash so that it makes sense as a URL
	return "/" + dir + avatar.Name()
}

func percentTrue(n int) bool {
	return rand.Intn(100) <= n
}

func removeApostrophes(c Cracklib) Cracklib {
	var vsf Cracklib
	for _, v := range c {
		if !strings.Contains(v, "'") {
			vsf = append(vsf, v)
		}
	}
	return vsf
}
<commit_msg>Fix linter errors in faking.go<commit_after>package main

import (
	"fmt"
	roman "github.com/StefanSchroeder/Golang-Roman"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"strconv"
	"strings"
	"time"
)

// Cracklib is a random-pop interface to the cracklib wordlist
type Cracklib []string

var words Cracklib

func init() {
	cracklibLocation := os.Getenv("DF_CRACKLIB_LOCATION")
	defaultCracklibLocation := "/usr/share/cracklib/cracklib-small"
	if cracklibLocation == "" {
		log.Print("Cracklib location set to default: " + defaultCracklibLocation)

		cracklibLocation = defaultCracklibLocation
	}
	out, err := ioutil.ReadFile(cracklibLocation)
	if err != nil {
		log.Print("Cracklib failed to load. Set location by env variable DF_CRACKLIB_LOCATION")
		return
	}
	words = removeApostrophes(strings.Split(string(out), "\n"))
	log.Printf("Loaded %d cracklib strings", len(words))

	// Also reset the random...
	rand.Seed(time.Now().UnixNano())
}

// Random returns a random string from cracklib
func (c *Cracklib) Random() string {
	return words[rand.Intn(len(words))]
}

// Title returns a random string from cracklib in Title Case
func (c *Cracklib) Title() string {
	return strings.Title(c.Random())
}

// Upper returns a random string from cracklib in UPPERCASE
func (c *Cracklib) Upper() string {
	return strings.ToUpper(c.Random())
}

// FakeName returns a fake player name
func FakeName() string {
	var prefix string
	// Add a last name prefix 80% of the time
	if percentTrue(80) {
		p := []string{"von ", "Mc", "of ", "the ", "De"}
		prefix = p[rand.Intn(len(p))]
	}

	return fmt.Sprintf(
		"%s %s%s",
		words.Title(),
		prefix,
		words.Title(),
	)
}

// FakeNick returns a fake player nickname
func FakeNick() string {
	var tag, prefix, suffix, number, char string

	// Add a clan tag
	if percentTrue(25) {
		content := words.Upper()

		if percentTrue(50) {
			// Half of the time, pick three random uppercase characters.
			b := make([]rune, 3)
			letters := []rune("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
			for i := range b {
				b[i] = letters[rand.Intn(len(letters))]
			}
			content = string(b)
		}

		tag = fmt.Sprintf("[%s]", content)
	}

	// Add a prefix
	if percentTrue(20) {
		p := []string{"sexy", "super", "muscle", "ultra", "mega"}
		prefix = p[rand.Intn(len(p))]
	}

	// Add a suffix
	if percentTrue(33) {
		s := []string{"gamer", "dude", "girl", "man", "lady", ""}
		suffix = s[rand.Intn(len(s))]
	}

	// Add a number between 250 and 1000
	if percentTrue(10) {
		number = strconv.Itoa(rand.Intn(750) + 250)
	}

	// Add a random character at the end
	if percentTrue(10) {
		c := []string{"-", "_", "^", "$", "#", "`", ">"}
		char = c[rand.Intn(len(c))]
	}

	return fmt.Sprintf(
		"%s%s%s%s%s%s",
		tag,
		prefix,
		words.Title(),
		suffix,
		number,
		char,
	)
}

// FakeTournamentTitle returns a fake tournament title and a numeral
func FakeTournamentTitle() (string, string) {
	f := []string{"Faking", "Testing", "Durnkern", "Meta", "Jalo", "Poser"}
	prefix := f[rand.Intn(len(f))]

	// Add a roman numeral between 20 and 400
	x := roman.Roman(rand.Intn(380) + 20)

	// Fake a subtitle
	var subtitle string
	switch rand.Intn(5) {
	case 0:
		subtitle = fmt.Sprintf("The %s of %s", words.Title(), words.Title())
	case 1:
		subtitle = fmt.Sprintf("%s and %s", words.Title(), words.Title())
	case 2:
		subtitle = fmt.Sprintf("%s and %s for %s", words.Title(), words.Title(), words.Title())
	case 3:
		subtitle = fmt.Sprintf("Barry Trotter and the %s of %s", words.Title(), words.Title())
	case 4:
		subtitle = words.Title()
	}

	title := fmt.Sprintf("%sFall %s: %s", prefix, x, subtitle)
	return title, strings.ToLower(x)
}

// FakeAvatar returns a fake avatar URL
func FakeAvatar() string {
	dir := "static/avatars/fake/"
	avatars, err := ioutil.ReadDir(dir)
	if err != nil {
		log.Fatal("No fake avatars available")
	}

	avatar := avatars[rand.Intn(len(avatars))]

	// Add a slash so that it makes sense as a URL
	return "/" + dir + avatar.Name()
}

func percentTrue(n int) bool {
	return rand.Intn(100) <= n
}

func removeApostrophes(c Cracklib) Cracklib {
	var vsf Cracklib
	for _, v := range c {
		if !strings.Contains(v, "'") {
			vsf = append(vsf, v)
		}
	}
	return vsf
}
<|endoftext|>


<commit_before>package gateway

import (
	"errors"
	"sync"
	"sync/atomic"

	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
)

type rpcID [8]byte

func (id rpcID) String() string {
	for i := range id {
		if id[i] == 0 {
			id[i] = ' '
		}
	}
	return string(id[:])
}

// handlerName truncates a string to 8 bytes. If len(name) < 8, the remaining
// bytes are 0. A handlerName is specified at the beginning of each network
// call, indicating which function should handle the connection.
func handlerName(name string) (id rpcID) {
	copy(id[:], name)
	return
}

// RPC calls an RPC on the given address. RPC cannot be called on an address
// that the Gateway is not connected to.
func (g *Gateway) RPC(addr modules.NetAddress, name string, fn modules.RPCFunc) error {
	g.log.Printf("INFO: calling RPC \"%v\" on %v", name, addr)
	peer, ok := g.peers[addr]
	if !ok {
		return errors.New("can't call RPC on unconnected peer " + string(addr))
	}

	conn, err := peer.open()
	if err != nil {
		return err
	}
	defer conn.Close()

	// write header
	if err := encoding.WriteObject(conn, handlerName(name)); err != nil {
		return err
	}
	// call fn
	err = fn(conn)
	if err != nil {
		// TODO: log error here?
		// give peer a strike
		atomic.AddUint32(&peer.strikes, 1)
	}
	return err
}

// RegisterRPC registers a function as an RPC handler for a given identifier.
// To call an RPC, use gateway.RPC, supplying the same identifier given to
// RegisterRPC. Identifiers should always use PascalCase.
func (g *Gateway) RegisterRPC(name string, fn modules.RPCFunc) {
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	g.handlerMap[handlerName(name)] = fn
}

// listenPeer listens for new streams on a peer connection and serves them via
// threadedHandleConn.
func (g *Gateway) listenPeer(p *peer) {
	for {
		conn, err := p.accept()
		if err != nil {
			g.log.Println("WARN: lost connection to peer", p.addr)
			break
		}

		// it is the handler's responsibility to close the connection
		go g.threadedHandleConn(conn)
	}
	g.Disconnect(p.addr)
}

// threadedHandleConn reads header data from a connection, then routes it to the
// appropriate handler for further processing.
func (g *Gateway) threadedHandleConn(conn modules.PeerConn) {
	defer conn.Close()
	var id rpcID
	if err := encoding.ReadObject(conn, &id, 8); err != nil {
		g.log.Printf("WARN: could not read RPC identifier from incoming conn %v: %v", conn.CallbackAddr(), err)
		return
	}
	// call registered handler for this ID
	lockid := g.mu.RLock()
	fn, ok := g.handlerMap[id]
	g.mu.RUnlock(lockid)
	if !ok {
		// TODO: write this error to conn?
		g.log.Printf("WARN: incoming conn %v requested unknown RPC \"%v\"", conn.CallbackAddr(), id)
		return
	}

	g.log.Printf("INFO: handling RPC \"%v\" from %v", id, conn.CallbackAddr())
	if err := fn(conn); err != nil {
		g.log.Printf("WARN: incoming RPC \"%v\" failed: %v", id, err)
	}
}

// Broadcast calls an RPC on all of the peers in the Gateway's peer list. The
// calls are run in parallel. Broadcasts are restricted to "one-way" RPCs,
// which simply write an object and disconnect. This is why Broadcast takes an
// interface{} instead of an RPCFunc.
func (g *Gateway) Broadcast(name string, obj interface{}) {
	peers := g.Peers()
	g.log.Printf("INFO: broadcasting RPC \"%v\" to %v peers", name, len(peers))

	// only encode obj once, instead of using WriteObject
	enc := encoding.Marshal(obj)
	fn := func(conn modules.PeerConn) error {
		return encoding.WritePrefix(conn, enc)
	}

	var wg sync.WaitGroup
	wg.Add(len(peers))
	for _, addr := range peers {
		go func(addr modules.NetAddress) {
			err := g.RPC(addr, name, fn)
			if err != nil {
				g.log.Printf("WARN: broadcast: calling RPC \"%v\" on peer %v returned error: %v", name, addr, err)
			}
			wg.Done()
		}(addr)
	}
	wg.Wait()
}
<commit_msg>fix RPC race condition<commit_after>package gateway

import (
	"errors"
	"sync"
	"sync/atomic"

	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
)

type rpcID [8]byte

func (id rpcID) String() string {
	for i := range id {
		if id[i] == 0 {
			id[i] = ' '
		}
	}
	return string(id[:])
}

// handlerName truncates a string to 8 bytes. If len(name) < 8, the remaining
// bytes are 0. A handlerName is specified at the beginning of each network
// call, indicating which function should handle the connection.
func handlerName(name string) (id rpcID) {
	copy(id[:], name)
	return
}

// RPC calls an RPC on the given address. RPC cannot be called on an address
// that the Gateway is not connected to.
func (g *Gateway) RPC(addr modules.NetAddress, name string, fn modules.RPCFunc) error {
	g.log.Printf("INFO: calling RPC \"%v\" on %v", name, addr)
	id := g.mu.RLock()
	peer, ok := g.peers[addr]
	g.mu.RUnlock(id)
	if !ok {
		return errors.New("can't call RPC on unconnected peer " + string(addr))
	}

	conn, err := peer.open()
	if err != nil {
		return err
	}
	defer conn.Close()

	// write header
	if err := encoding.WriteObject(conn, handlerName(name)); err != nil {
		return err
	}
	// call fn
	err = fn(conn)
	if err != nil {
		// TODO: log error here?
		// give peer a strike
		atomic.AddUint32(&peer.strikes, 1)
	}
	return err
}

// RegisterRPC registers a function as an RPC handler for a given identifier.
// To call an RPC, use gateway.RPC, supplying the same identifier given to
// RegisterRPC. Identifiers should always use PascalCase.
func (g *Gateway) RegisterRPC(name string, fn modules.RPCFunc) {
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	g.handlerMap[handlerName(name)] = fn
}

// listenPeer listens for new streams on a peer connection and serves them via
// threadedHandleConn.
func (g *Gateway) listenPeer(p *peer) {
	for {
		conn, err := p.accept()
		if err != nil {
			g.log.Println("WARN: lost connection to peer", p.addr)
			break
		}

		// it is the handler's responsibility to close the connection
		go g.threadedHandleConn(conn)
	}
	g.Disconnect(p.addr)
}

// threadedHandleConn reads header data from a connection, then routes it to the
// appropriate handler for further processing.
func (g *Gateway) threadedHandleConn(conn modules.PeerConn) {
	defer conn.Close()
	var id rpcID
	if err := encoding.ReadObject(conn, &id, 8); err != nil {
		g.log.Printf("WARN: could not read RPC identifier from incoming conn %v: %v", conn.CallbackAddr(), err)
		return
	}
	// call registered handler for this ID
	lockid := g.mu.RLock()
	fn, ok := g.handlerMap[id]
	g.mu.RUnlock(lockid)
	if !ok {
		// TODO: write this error to conn?
		g.log.Printf("WARN: incoming conn %v requested unknown RPC \"%v\"", conn.CallbackAddr(), id)
		return
	}

	g.log.Printf("INFO: handling RPC \"%v\" from %v", id, conn.CallbackAddr())
	if err := fn(conn); err != nil {
		g.log.Printf("WARN: incoming RPC \"%v\" failed: %v", id, err)
	}
}

// Broadcast calls an RPC on all of the peers in the Gateway's peer list. The
// calls are run in parallel. Broadcasts are restricted to "one-way" RPCs,
// which simply write an object and disconnect. This is why Broadcast takes an
// interface{} instead of an RPCFunc.
func (g *Gateway) Broadcast(name string, obj interface{}) {
	peers := g.Peers()
	g.log.Printf("INFO: broadcasting RPC \"%v\" to %v peers", name, len(peers))

	// only encode obj once, instead of using WriteObject
	enc := encoding.Marshal(obj)
	fn := func(conn modules.PeerConn) error {
		return encoding.WritePrefix(conn, enc)
	}

	var wg sync.WaitGroup
	wg.Add(len(peers))
	for _, addr := range peers {
		go func(addr modules.NetAddress) {
			err := g.RPC(addr, name, fn)
			if err != nil {
				g.log.Printf("WARN: broadcast: calling RPC \"%v\" on peer %v returned error: %v", name, addr, err)
			}
			wg.Done()
		}(addr)
	}
	wg.Wait()
}
<|endoftext|>


<commit_before>// Copyright 2014 The zhgo Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package db

import (
    "reflect"
)

// Model struct
type Model struct {
    // Module name, as DB name.
    Module string

    // table instance
    Table Table
}

// Server list
var Servers = make(map[string]*Server)

// Insert
func (m *Model) Insert() *Query {
    q := NewQuery(Servers[m.Module])
    q.InsertInto(m.Table.Name)
    return q
}

// Update
func (m *Model) Update() *Query {
    q := NewQuery(Servers[m.Module])
    q.Update(m.Table.Name)
    return q
}

// Delete
func (m *Model) Delete() *Query {
    q := NewQuery(Servers[m.Module])
    q.DeleteFrom(m.Table.Name)
    return q
}

// Select
func (m *Model) Select(f ...string) *Query {
    if len(f) == 0 {
        f = m.Table.Fields
    }
    q := NewQuery(Servers[m.Module])
    q.Select(f...)
    q.From(m.Table.Name)
    return q
}

// New Model
func NewModel(module string, table Table) *Model {
    return &Model{Module: module, Table: table}
}

// Table struct
type Table struct {
    // Table name
    Name string

    // Table primary
    Primary string

    // All fields, except primary
    Fields []string

    // Entity
    EntityType reflect.Type
}

// New Table
func NewTable(tableName string, entity interface{}) Table {
    p, f := tableFields(entity)
    t := Table{Name: tableName, Primary: p, Fields: f, EntityType: reflect.ValueOf(entity).Elem().Type()}
    return t
}
<commit_msg>Update repo.<commit_after>// Copyright 2014 The zhgo Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package db

import (
    "reflect"
)

// Model struct
type Model struct {
    // Module name, as DB name.
    Module string

    // table instance
    Table Table
}

// Table struct
type Table struct {
    // Table name
    Name string

    // Table primary
    Primary string

    // All fields, except primary
    Fields []string

    // Entity
    EntityType reflect.Type
}

// Server list
var Servers = make(map[string]*Server)

// Insert
func (m *Model) Insert() *Query {
    q := NewQuery(Servers[m.Module])
    q.InsertInto(m.Table.Name)
    return q
}

// Update
func (m *Model) Update() *Query {
    q := NewQuery(Servers[m.Module])
    q.Update(m.Table.Name)
    return q
}

// Delete
func (m *Model) Delete() *Query {
    q := NewQuery(Servers[m.Module])
    q.DeleteFrom(m.Table.Name)
    return q
}

// Select
func (m *Model) Select(f ...string) *Query {
    if len(f) == 0 {
        f = m.Table.Fields
    }
    q := NewQuery(Servers[m.Module])
    q.Select(f...)
    q.From(m.Table.Name)
    return q
}

// New Model
func NewModel(module string, table Table) *Model {
    return &Model{Module: module, Table: table}
}

// New Table
func NewTable(tableName string, entity interface{}) Table {
    p, f := tableFields(entity)
    t := Table{Name: tableName, Primary: p, Fields: f, EntityType: reflect.ValueOf(entity).Elem().Type()}
    return t
}
<|endoftext|>