<commit_before>package dnsr

var root = `
;       This file holds the information on root name servers needed to
;       initialize cache of Internet domain name servers
;       (e.g. reference this file in the "cache  .  <file>"
;       configuration file of BIND domain name servers).
;
;       This file is made available by InterNIC
;       under anonymous FTP as
;           file                /domain/named.cache
;           on server           FTP.INTERNIC.NET
;       -OR-                    RS.INTERNIC.NET
;
;       last update:     November 22, 2021
;       related version of root zone:     2021112201
;
; FORMERLY NS.INTERNIC.NET
;
.                        3600000      NS    A.ROOT-SERVERS.NET.
A.ROOT-SERVERS.NET.      3600000      A     198.41.0.4
A.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:ba3e::2:30
;
; FORMERLY NS1.ISI.EDU
;
.                        3600000      NS    B.ROOT-SERVERS.NET.
B.ROOT-SERVERS.NET.      3600000      A     199.9.14.201
B.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:200::b
;
; FORMERLY C.PSI.NET
;
.                        3600000      NS    C.ROOT-SERVERS.NET.
C.ROOT-SERVERS.NET.      3600000      A     192.33.4.12
C.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2::c
;
; FORMERLY TERP.UMD.EDU
;
.                        3600000      NS    D.ROOT-SERVERS.NET.
D.ROOT-SERVERS.NET.      3600000      A     199.7.91.13
D.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2d::d
;
; FORMERLY NS.NASA.GOV
;
.                        3600000      NS    E.ROOT-SERVERS.NET.
E.ROOT-SERVERS.NET.      3600000      A     192.203.230.10
E.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:a8::e
;
; FORMERLY NS.ISC.ORG
;
.                        3600000      NS    F.ROOT-SERVERS.NET.
F.ROOT-SERVERS.NET.      3600000      A     192.5.5.241
F.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2f::f
;
; FORMERLY NS.NIC.DDN.MIL
;
.                        3600000      NS    G.ROOT-SERVERS.NET.
G.ROOT-SERVERS.NET.      3600000      A     192.112.36.4
G.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:12::d0d
;
; FORMERLY AOS.ARL.ARMY.MIL
;
.                        3600000      NS    H.ROOT-SERVERS.NET.
H.ROOT-SERVERS.NET.      3600000      A     198.97.190.53
H.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:1::53
;
; FORMERLY NIC.NORDU.NET
;
.                        3600000      NS    I.ROOT-SERVERS.NET.
I.ROOT-SERVERS.NET.      3600000      A     192.36.148.17
I.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fe::53
;
; OPERATED BY VERISIGN, INC.
;
.                        3600000      NS    J.ROOT-SERVERS.NET.
J.ROOT-SERVERS.NET.      3600000      A     192.58.128.30
J.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:c27::2:30
;
; OPERATED BY RIPE NCC
;
.                        3600000      NS    K.ROOT-SERVERS.NET.
K.ROOT-SERVERS.NET.      3600000      A     193.0.14.129
K.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fd::1
;
; OPERATED BY ICANN
;
.                        3600000      NS    L.ROOT-SERVERS.NET.
L.ROOT-SERVERS.NET.      3600000      A     199.7.83.42
L.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:9f::42
;
; OPERATED BY WIDE
;
.                        3600000      NS    M.ROOT-SERVERS.NET.
M.ROOT-SERVERS.NET.      3600000      A     202.12.27.33
M.ROOT-SERVERS.NET.      3600000      AAAA  2001:dc3::35
; End of file`
<commit_msg>Automatic update for Wed Dec  8 08:19:47 UTC 2021<commit_after>package dnsr

var root = `
;       This file holds the information on root name servers needed to
;       initialize cache of Internet domain name servers
;       (e.g. reference this file in the "cache  .  <file>"
;       configuration file of BIND domain name servers).
;
;       This file is made available by InterNIC
;       under anonymous FTP as
;           file                /domain/named.cache
;           on server           FTP.INTERNIC.NET
;       -OR-                    RS.INTERNIC.NET
;
;       last update:     December 07, 2021
;       related version of root zone:     2021120701
;
; FORMERLY NS.INTERNIC.NET
;
.                        3600000      NS    A.ROOT-SERVERS.NET.
A.ROOT-SERVERS.NET.      3600000      A     198.41.0.4
A.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:ba3e::2:30
;
; FORMERLY NS1.ISI.EDU
;
.                        3600000      NS    B.ROOT-SERVERS.NET.
B.ROOT-SERVERS.NET.      3600000      A     199.9.14.201
B.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:200::b
;
; FORMERLY C.PSI.NET
;
.                        3600000      NS    C.ROOT-SERVERS.NET.
C.ROOT-SERVERS.NET.      3600000      A     192.33.4.12
C.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2::c
;
; FORMERLY TERP.UMD.EDU
;
.                        3600000      NS    D.ROOT-SERVERS.NET.
D.ROOT-SERVERS.NET.      3600000      A     199.7.91.13
D.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2d::d
;
; FORMERLY NS.NASA.GOV
;
.                        3600000      NS    E.ROOT-SERVERS.NET.
E.ROOT-SERVERS.NET.      3600000      A     192.203.230.10
E.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:a8::e
;
; FORMERLY NS.ISC.ORG
;
.                        3600000      NS    F.ROOT-SERVERS.NET.
F.ROOT-SERVERS.NET.      3600000      A     192.5.5.241
F.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2f::f
;
; FORMERLY NS.NIC.DDN.MIL
;
.                        3600000      NS    G.ROOT-SERVERS.NET.
G.ROOT-SERVERS.NET.      3600000      A     192.112.36.4
G.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:12::d0d
;
; FORMERLY AOS.ARL.ARMY.MIL
;
.                        3600000      NS    H.ROOT-SERVERS.NET.
H.ROOT-SERVERS.NET.      3600000      A     198.97.190.53
H.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:1::53
;
; FORMERLY NIC.NORDU.NET
;
.                        3600000      NS    I.ROOT-SERVERS.NET.
I.ROOT-SERVERS.NET.      3600000      A     192.36.148.17
I.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fe::53
;
; OPERATED BY VERISIGN, INC.
;
.                        3600000      NS    J.ROOT-SERVERS.NET.
J.ROOT-SERVERS.NET.      3600000      A     192.58.128.30
J.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:c27::2:30
;
; OPERATED BY RIPE NCC
;
.                        3600000      NS    K.ROOT-SERVERS.NET.
K.ROOT-SERVERS.NET.      3600000      A     193.0.14.129
K.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fd::1
;
; OPERATED BY ICANN
;
.                        3600000      NS    L.ROOT-SERVERS.NET.
L.ROOT-SERVERS.NET.      3600000      A     199.7.83.42
L.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:9f::42
;
; OPERATED BY WIDE
;
.                        3600000      NS    M.ROOT-SERVERS.NET.
M.ROOT-SERVERS.NET.      3600000      A     202.12.27.33
M.ROOT-SERVERS.NET.      3600000      AAAA  2001:dc3::35
; End of file`
<|endoftext|>


<commit_before>package core

import (
	"database/sql"
	"errors"
	"reflect"
	"sync"
)

type Rows struct {
	*sql.Rows
	Mapper IMapper
}

func (rs *Rows) ToMapString() ([]map[string]string, error) {
	cols, err := rs.Columns()
	if err != nil {
		return nil, err
	}

	var results = make([]map[string]string, 0, 10)
	for rs.Next() {
		var record = make(map[string]string, len(cols))
		err = rs.ScanMap(&record)
		if err != nil {
			return nil, err
		}
		results = append(results, record)
	}
	return results, nil
}

// scan data to a struct's pointer according field index
func (rs *Rows) ScanStructByIndex(dest ...interface{}) error {
	if len(dest) == 0 {
		return errors.New("at least one struct")
	}

	vvvs := make([]reflect.Value, len(dest))
	for i, s := range dest {
		vv := reflect.ValueOf(s)
		if vv.Kind() != reflect.Ptr || vv.Elem().Kind() != reflect.Struct {
			return errors.New("dest should be a struct's pointer")
		}

		vvvs[i] = vv.Elem()
	}

	cols, err := rs.Columns()
	if err != nil {
		return err
	}
	newDest := make([]interface{}, len(cols))

	var i = 0
	for _, vvv := range vvvs {
		for j := 0; j < vvv.NumField(); j++ {
			newDest[i] = vvv.Field(j).Addr().Interface()
			i = i + 1
		}
	}

	return rs.Rows.Scan(newDest...)
}

var (
	fieldCache      = make(map[reflect.Type]map[string]int)
	fieldCacheMutex sync.RWMutex
)

func fieldByName(v reflect.Value, name string) reflect.Value {
	t := v.Type()
	fieldCacheMutex.RLock()
	cache, ok := fieldCache[t]
	fieldCacheMutex.RUnlock()
	if !ok {
		cache = make(map[string]int)
		for i := 0; i < v.NumField(); i++ {
			cache[t.Field(i).Name] = i
		}
		fieldCacheMutex.Lock()
		fieldCache[t] = cache
		fieldCacheMutex.Unlock()
	}

	if i, ok := cache[name]; ok {
		return v.Field(i)
	}

	return reflect.Zero(t)
}

// scan data to a struct's pointer according field name
func (rs *Rows) ScanStructByName(dest interface{}) error {
	vv := reflect.ValueOf(dest)
	if vv.Kind() != reflect.Ptr || vv.Elem().Kind() != reflect.Struct {
		return errors.New("dest should be a struct's pointer")
	}

	cols, err := rs.Columns()
	if err != nil {
		return err
	}

	newDest := make([]interface{}, len(cols))
	var v EmptyScanner
	for j, name := range cols {
		f := fieldByName(vv.Elem(), rs.Mapper.Table2Obj(name))
		if f.IsValid() {
			newDest[j] = f.Addr().Interface()
		} else {
			newDest[j] = &v
		}
	}

	return rs.Rows.Scan(newDest...)
}

type cacheStruct struct {
	value reflect.Value
	idx   int
}

var (
	reflectCache      = make(map[reflect.Type]*cacheStruct)
	reflectCacheMutex sync.RWMutex
)

func ReflectNew(typ reflect.Type) reflect.Value {
	reflectCacheMutex.RLock()
	cs, ok := reflectCache[typ]
	reflectCacheMutex.RUnlock()

	const newSize = 200

	if !ok || cs.idx+1 > newSize-1 {
		cs = &cacheStruct{reflect.MakeSlice(reflect.SliceOf(typ), newSize, newSize), 0}
		reflectCacheMutex.Lock()
		reflectCache[typ] = cs
		reflectCacheMutex.Unlock()
	} else {
		reflectCacheMutex.Lock()
		cs.idx = cs.idx + 1
		reflectCacheMutex.Unlock()
	}
	return cs.value.Index(cs.idx).Addr()
}

// scan data to a slice's pointer, slice's length should equal to columns' number
func (rs *Rows) ScanSlice(dest interface{}) error {
	vv := reflect.ValueOf(dest)
	if vv.Kind() != reflect.Ptr || vv.Elem().Kind() != reflect.Slice {
		return errors.New("dest should be a slice's pointer")
	}

	vvv := vv.Elem()
	cols, err := rs.Columns()
	if err != nil {
		return err
	}

	newDest := make([]interface{}, len(cols))

	for j := 0; j < len(cols); j++ {
		if j >= vvv.Len() {
			newDest[j] = reflect.New(vvv.Type().Elem()).Interface()
		} else {
			newDest[j] = vvv.Index(j).Addr().Interface()
		}
	}

	err = rs.Rows.Scan(newDest...)
	if err != nil {
		return err
	}

	srcLen := vvv.Len()
	for i := srcLen; i < len(cols); i++ {
		vvv = reflect.Append(vvv, reflect.ValueOf(newDest[i]).Elem())
	}
	return nil
}

// scan data to a map's pointer
func (rs *Rows) ScanMap(dest interface{}) error {
	vv := reflect.ValueOf(dest)
	if vv.Kind() != reflect.Ptr || vv.Elem().Kind() != reflect.Map {
		return errors.New("dest should be a map's pointer")
	}

	cols, err := rs.Columns()
	if err != nil {
		return err
	}

	newDest := make([]interface{}, len(cols))
	vvv := vv.Elem()

	for i, _ := range cols {
		newDest[i] = ReflectNew(vvv.Type().Elem()).Interface()
		//v := reflect.New(vvv.Type().Elem())
		//newDest[i] = v.Interface()
	}

	err = rs.Rows.Scan(newDest...)
	if err != nil {
		return err
	}

	for i, name := range cols {
		vname := reflect.ValueOf(name)
		vvv.SetMapIndex(vname, reflect.ValueOf(newDest[i]).Elem())
	}

	return nil
}

/*func (rs *Rows) ScanMap(dest interface{}) error {
	vv := reflect.ValueOf(dest)
	if vv.Kind() != reflect.Ptr || vv.Elem().Kind() != reflect.Map {
		return errors.New("dest should be a map's pointer")
	}

	cols, err := rs.Columns()
	if err != nil {
		return err
	}

	newDest := make([]interface{}, len(cols))
	err = rs.ScanSlice(newDest)
	if err != nil {
		return err
	}

	vvv := vv.Elem()

	for i, name := range cols {
		vname := reflect.ValueOf(name)
		vvv.SetMapIndex(vname, reflect.ValueOf(newDest[i]).Elem())
	}

	return nil
}*/
type Row struct {
	rows *Rows
	// One of these two will be non-nil:
	err error // deferred error for easy chaining
}

func (row *Row) Columns() ([]string, error) {
	if row.err != nil {
		return nil, row.err
	}
	return row.rows.Columns()
}

func (row *Row) Scan(dest ...interface{}) error {
	if row.err != nil {
		return row.err
	}
	defer row.rows.Close()

	for _, dp := range dest {
		if _, ok := dp.(*sql.RawBytes); ok {
			return errors.New("sql: RawBytes isn't allowed on Row.Scan")
		}
	}

	if !row.rows.Next() {
		if err := row.rows.Err(); err != nil {
			return err
		}
		return sql.ErrNoRows
	}
	err := row.rows.Scan(dest...)
	if err != nil {
		return err
	}
	// Make sure the query can be processed to completion with no errors.
	return row.rows.Close()
}

func (row *Row) ScanStructByName(dest interface{}) error {
	if row.err != nil {
		return row.err
	}
	defer row.rows.Close()

	if !row.rows.Next() {
		if err := row.rows.Err(); err != nil {
			return err
		}
		return sql.ErrNoRows
	}
	err := row.rows.ScanStructByName(dest)
	if err != nil {
		return err
	}
	// Make sure the query can be processed to completion with no errors.
	return row.rows.Close()
}

func (row *Row) ScanStructByIndex(dest interface{}) error {
	if row.err != nil {
		return row.err
	}
	defer row.rows.Close()

	if !row.rows.Next() {
		if err := row.rows.Err(); err != nil {
			return err
		}
		return sql.ErrNoRows
	}
	err := row.rows.ScanStructByIndex(dest)
	if err != nil {
		return err
	}
	// Make sure the query can be processed to completion with no errors.
	return row.rows.Close()
}

// scan data to a slice's pointer, slice's length should equal to columns' number
func (row *Row) ScanSlice(dest interface{}) error {
	if row.err != nil {
		return row.err
	}
	defer row.rows.Close()

	if !row.rows.Next() {
		if err := row.rows.Err(); err != nil {
			return err
		}
		return sql.ErrNoRows
	}
	err := row.rows.ScanSlice(dest)
	if err != nil {
		return err
	}

	// Make sure the query can be processed to completion with no errors.
	return row.rows.Close()
}

// scan data to a map's pointer
func (row *Row) ScanMap(dest interface{}) error {
	if row.err != nil {
		return row.err
	}
	defer row.rows.Close()

	if !row.rows.Next() {
		if err := row.rows.Err(); err != nil {
			return err
		}
		return sql.ErrNoRows
	}
	err := row.rows.ScanMap(dest)
	if err != nil {
		return err
	}

	// Make sure the query can be processed to completion with no errors.
	return row.rows.Close()
}

func (row *Row) ToMapString() (map[string]string, error) {
	cols, err := row.Columns()
	if err != nil {
		return nil, err
	}

	var record = make(map[string]string, len(cols))
	err = row.ScanMap(&record)
	if err != nil {
		return nil, err
	}

	return record, nil
}
<commit_msg>add ErrorRow function<commit_after>package core

import (
	"database/sql"
	"errors"
	"reflect"
	"sync"
)

type Rows struct {
	*sql.Rows
	Mapper IMapper
}

func (rs *Rows) ToMapString() ([]map[string]string, error) {
	cols, err := rs.Columns()
	if err != nil {
		return nil, err
	}

	var results = make([]map[string]string, 0, 10)
	for rs.Next() {
		var record = make(map[string]string, len(cols))
		err = rs.ScanMap(&record)
		if err != nil {
			return nil, err
		}
		results = append(results, record)
	}
	return results, nil
}

// scan data to a struct's pointer according field index
func (rs *Rows) ScanStructByIndex(dest ...interface{}) error {
	if len(dest) == 0 {
		return errors.New("at least one struct")
	}

	vvvs := make([]reflect.Value, len(dest))
	for i, s := range dest {
		vv := reflect.ValueOf(s)
		if vv.Kind() != reflect.Ptr || vv.Elem().Kind() != reflect.Struct {
			return errors.New("dest should be a struct's pointer")
		}

		vvvs[i] = vv.Elem()
	}

	cols, err := rs.Columns()
	if err != nil {
		return err
	}
	newDest := make([]interface{}, len(cols))

	var i = 0
	for _, vvv := range vvvs {
		for j := 0; j < vvv.NumField(); j++ {
			newDest[i] = vvv.Field(j).Addr().Interface()
			i = i + 1
		}
	}

	return rs.Rows.Scan(newDest...)
}

var (
	fieldCache      = make(map[reflect.Type]map[string]int)
	fieldCacheMutex sync.RWMutex
)

func fieldByName(v reflect.Value, name string) reflect.Value {
	t := v.Type()
	fieldCacheMutex.RLock()
	cache, ok := fieldCache[t]
	fieldCacheMutex.RUnlock()
	if !ok {
		cache = make(map[string]int)
		for i := 0; i < v.NumField(); i++ {
			cache[t.Field(i).Name] = i
		}
		fieldCacheMutex.Lock()
		fieldCache[t] = cache
		fieldCacheMutex.Unlock()
	}

	if i, ok := cache[name]; ok {
		return v.Field(i)
	}

	return reflect.Zero(t)
}

// scan data to a struct's pointer according field name
func (rs *Rows) ScanStructByName(dest interface{}) error {
	vv := reflect.ValueOf(dest)
	if vv.Kind() != reflect.Ptr || vv.Elem().Kind() != reflect.Struct {
		return errors.New("dest should be a struct's pointer")
	}

	cols, err := rs.Columns()
	if err != nil {
		return err
	}

	newDest := make([]interface{}, len(cols))
	var v EmptyScanner
	for j, name := range cols {
		f := fieldByName(vv.Elem(), rs.Mapper.Table2Obj(name))
		if f.IsValid() {
			newDest[j] = f.Addr().Interface()
		} else {
			newDest[j] = &v
		}
	}

	return rs.Rows.Scan(newDest...)
}

type cacheStruct struct {
	value reflect.Value
	idx   int
}

var (
	reflectCache      = make(map[reflect.Type]*cacheStruct)
	reflectCacheMutex sync.RWMutex
)

func ReflectNew(typ reflect.Type) reflect.Value {
	reflectCacheMutex.RLock()
	cs, ok := reflectCache[typ]
	reflectCacheMutex.RUnlock()

	const newSize = 200

	if !ok || cs.idx+1 > newSize-1 {
		cs = &cacheStruct{reflect.MakeSlice(reflect.SliceOf(typ), newSize, newSize), 0}
		reflectCacheMutex.Lock()
		reflectCache[typ] = cs
		reflectCacheMutex.Unlock()
	} else {
		reflectCacheMutex.Lock()
		cs.idx = cs.idx + 1
		reflectCacheMutex.Unlock()
	}
	return cs.value.Index(cs.idx).Addr()
}

// scan data to a slice's pointer, slice's length should equal to columns' number
func (rs *Rows) ScanSlice(dest interface{}) error {
	vv := reflect.ValueOf(dest)
	if vv.Kind() != reflect.Ptr || vv.Elem().Kind() != reflect.Slice {
		return errors.New("dest should be a slice's pointer")
	}

	vvv := vv.Elem()
	cols, err := rs.Columns()
	if err != nil {
		return err
	}

	newDest := make([]interface{}, len(cols))

	for j := 0; j < len(cols); j++ {
		if j >= vvv.Len() {
			newDest[j] = reflect.New(vvv.Type().Elem()).Interface()
		} else {
			newDest[j] = vvv.Index(j).Addr().Interface()
		}
	}

	err = rs.Rows.Scan(newDest...)
	if err != nil {
		return err
	}

	srcLen := vvv.Len()
	for i := srcLen; i < len(cols); i++ {
		vvv = reflect.Append(vvv, reflect.ValueOf(newDest[i]).Elem())
	}
	return nil
}

// scan data to a map's pointer
func (rs *Rows) ScanMap(dest interface{}) error {
	vv := reflect.ValueOf(dest)
	if vv.Kind() != reflect.Ptr || vv.Elem().Kind() != reflect.Map {
		return errors.New("dest should be a map's pointer")
	}

	cols, err := rs.Columns()
	if err != nil {
		return err
	}

	newDest := make([]interface{}, len(cols))
	vvv := vv.Elem()

	for i, _ := range cols {
		newDest[i] = ReflectNew(vvv.Type().Elem()).Interface()
		//v := reflect.New(vvv.Type().Elem())
		//newDest[i] = v.Interface()
	}

	err = rs.Rows.Scan(newDest...)
	if err != nil {
		return err
	}

	for i, name := range cols {
		vname := reflect.ValueOf(name)
		vvv.SetMapIndex(vname, reflect.ValueOf(newDest[i]).Elem())
	}

	return nil
}

/*func (rs *Rows) ScanMap(dest interface{}) error {
	vv := reflect.ValueOf(dest)
	if vv.Kind() != reflect.Ptr || vv.Elem().Kind() != reflect.Map {
		return errors.New("dest should be a map's pointer")
	}

	cols, err := rs.Columns()
	if err != nil {
		return err
	}

	newDest := make([]interface{}, len(cols))
	err = rs.ScanSlice(newDest)
	if err != nil {
		return err
	}

	vvv := vv.Elem()

	for i, name := range cols {
		vname := reflect.ValueOf(name)
		vvv.SetMapIndex(vname, reflect.ValueOf(newDest[i]).Elem())
	}

	return nil
}*/
type Row struct {
	rows *Rows
	// One of these two will be non-nil:
	err error // deferred error for easy chaining
}

// ErrorRow return an error row
func ErrorRow(err error) *Row {
	return &Row{
		err: err,
	}
}

func (row *Row) Columns() ([]string, error) {
	if row.err != nil {
		return nil, row.err
	}
	return row.rows.Columns()
}

func (row *Row) Scan(dest ...interface{}) error {
	if row.err != nil {
		return row.err
	}
	defer row.rows.Close()

	for _, dp := range dest {
		if _, ok := dp.(*sql.RawBytes); ok {
			return errors.New("sql: RawBytes isn't allowed on Row.Scan")
		}
	}

	if !row.rows.Next() {
		if err := row.rows.Err(); err != nil {
			return err
		}
		return sql.ErrNoRows
	}
	err := row.rows.Scan(dest...)
	if err != nil {
		return err
	}
	// Make sure the query can be processed to completion with no errors.
	return row.rows.Close()
}

func (row *Row) ScanStructByName(dest interface{}) error {
	if row.err != nil {
		return row.err
	}
	defer row.rows.Close()

	if !row.rows.Next() {
		if err := row.rows.Err(); err != nil {
			return err
		}
		return sql.ErrNoRows
	}
	err := row.rows.ScanStructByName(dest)
	if err != nil {
		return err
	}
	// Make sure the query can be processed to completion with no errors.
	return row.rows.Close()
}

func (row *Row) ScanStructByIndex(dest interface{}) error {
	if row.err != nil {
		return row.err
	}
	defer row.rows.Close()

	if !row.rows.Next() {
		if err := row.rows.Err(); err != nil {
			return err
		}
		return sql.ErrNoRows
	}
	err := row.rows.ScanStructByIndex(dest)
	if err != nil {
		return err
	}
	// Make sure the query can be processed to completion with no errors.
	return row.rows.Close()
}

// scan data to a slice's pointer, slice's length should equal to columns' number
func (row *Row) ScanSlice(dest interface{}) error {
	if row.err != nil {
		return row.err
	}
	defer row.rows.Close()

	if !row.rows.Next() {
		if err := row.rows.Err(); err != nil {
			return err
		}
		return sql.ErrNoRows
	}
	err := row.rows.ScanSlice(dest)
	if err != nil {
		return err
	}

	// Make sure the query can be processed to completion with no errors.
	return row.rows.Close()
}

// scan data to a map's pointer
func (row *Row) ScanMap(dest interface{}) error {
	if row.err != nil {
		return row.err
	}
	defer row.rows.Close()

	if !row.rows.Next() {
		if err := row.rows.Err(); err != nil {
			return err
		}
		return sql.ErrNoRows
	}
	err := row.rows.ScanMap(dest)
	if err != nil {
		return err
	}

	// Make sure the query can be processed to completion with no errors.
	return row.rows.Close()
}

func (row *Row) ToMapString() (map[string]string, error) {
	cols, err := row.Columns()
	if err != nil {
		return nil, err
	}

	var record = make(map[string]string, len(cols))
	err = row.ScanMap(&record)
	if err != nil {
		return nil, err
	}

	return record, nil
}
<|endoftext|>


<commit_before>package roulette

import (
	"reflect"
	"sort"
	"strconv"
	"strings"
	"text/template"
)

// Ruleset ...
type Ruleset interface {
	Execute(vals interface{})
}

type ruleConfig struct {
	delimLeft    string
	delimRight   string
	defaultfuncs template.FuncMap
	userfuncs    template.FuncMap
	allfuncs     template.FuncMap

	expectTypes    []string
	expectTypesErr error
	noResultFunc   bool

	template    *template.Template
	templateErr error
}

// Rule is a single rule expression. A rule expression is a valid go text/template
type Rule struct {
	Name     string `xml:"name,attr"`
	Priority int    `xml:"priority,attr"`
	Expr     string `xml:",innerxml"`
	config   ruleConfig
}

func (r Rule) isValid(filterTypesArr []string) error {

	if r.config.expectTypes == nil || filterTypesArr == nil {
		return r.config.expectTypesErr
	}

	if len(filterTypesArr) == 1 && filterTypesArr[0] == "map[string]interface {}" {
		return nil
	}

	// less
	if len(filterTypesArr) < len(r.config.expectTypes) {
		return r.config.expectTypesErr
	}

	// equal to
	if len(filterTypesArr) == len(r.config.expectTypes) {
		for i := range r.config.expectTypes {
			if filterTypesArr[i] != r.config.expectTypes[i] {
				return r.config.expectTypesErr
			}
		}
	}

	// greater than
	// all expected types should be present in the template data.
	for _, expectedType := range r.config.expectTypes {
		j := sort.SearchStrings(filterTypesArr, expectedType)
		found := j < len(filterTypesArr) && filterTypesArr[j] == expectedType
		if !found {
			return r.config.expectTypesErr
		}
	}

	return nil
}

type textTemplateRulesetConfig struct {
	result         Result
	filterTypesArr []string
	workflowMatch  bool
}

// TextTemplateRuleset is a collection of rules for a valid go type
type TextTemplateRuleset struct {
	Name            string `xml:"name,attr"`
	FilterTypes     string `xml:"filterTypes,attr"`
	FilterStrict    bool   `xml:"filterStrict,attr"`
	DataKey         string `xml:"dataKey,attr"`
	ResultKey       string `xml:"resultKey,attr"`
	Rules           []Rule `xml:"rule"`
	PrioritiesCount string `xml:"prioritiesCount,attr"`
	Workflow        string `xml:"workflow,attr"`
	config          textTemplateRulesetConfig
	bytesBuf        *bytesPool
	mapBuf          *mapPool
	limit           int
}

// sort rules by priority
func (t TextTemplateRuleset) Len() int {
	return len(t.Rules)
}
func (t TextTemplateRuleset) Swap(i, j int) {
	t.Rules[i], t.Rules[j] = t.Rules[j], t.Rules[i]
}
func (t TextTemplateRuleset) Less(i, j int) bool {
	return t.Rules[i].Priority < t.Rules[j].Priority
}

func (t TextTemplateRuleset) isValidForTypes(filterTypesArr ...string) bool {

	if len(filterTypesArr) == 0 {
		return false
	}

	if len(t.config.filterTypesArr) != len(filterTypesArr) && t.FilterStrict {
		return false
	}

	// if not filterStrict, look for atleast one match
	// if filterStrict look for atleast one mismatch
	for i, v := range t.config.filterTypesArr {
		j := sort.SearchStrings(t.config.filterTypesArr, v)
		found := j < len(t.config.filterTypesArr) && t.config.filterTypesArr[i] == v
		if !found {
			if t.FilterStrict {
				return false
			}
		} else {

			// filtering is not strict and one atleast one match found.
			if !t.FilterStrict {
				return true
			}
		}
	}

	return false
}

func (t TextTemplateRuleset) getTypes(vals interface{}) []string {
	var types []string
	switch vals.(type) {
	case []interface{}:
		var typeName string
		size := len(vals.([]interface{}))
		types = make([]string, size)
		for i, v := range vals.([]interface{}) {
			if reflect.ValueOf(v).Kind() == reflect.Ptr || reflect.ValueOf(v).Kind() == reflect.Interface {
				typeName = reflect.TypeOf(v).Elem().String()
			} else {
				typeName = reflect.TypeOf(v).String()
			}

			types[i] = typeName
		}

		break

	default:
		types = make([]string, 1)
		typeName := reflect.TypeOf(vals).Elem().String()
		types[0] = typeName
	}

	return types
}

func (t TextTemplateRuleset) getTemplateData(tmplData map[string]interface{}, vals interface{}) {

	//fmt.Println("getTemplateData", reflect.TypeOf(vals))
	// flatten multiple types in template map so that they can be referred by
	// dataKey

	valsData := t.mapBuf.get()
	defer t.mapBuf.put(valsData)
	// index array of same types
	typeArrayIndex := make(map[string]int)
	var pkgPaths []string

	switch vals.(type) {
	case []interface{}:
		nestedMap := t.mapBuf.get()
		defer t.mapBuf.put(nestedMap)

		for i, val := range vals.([]interface{}) {

			switch val.(type) {
			case []string, []int, []int32, []int64, []bool, []float32, []float64, []interface{}:
				typ := reflect.TypeOf(val).String()
				typeName := strings.Trim(typ, "[]")
				typeName = strings.Trim(typ, "{}")
				valsData[typeName+"slice"+strconv.Itoa(i)] = val

				break

			case map[string]int, map[string]string, map[string]bool:
				typ := reflect.TypeOf(val).String()
				typeName := strings.Trim(typ, "[")
				typeName = strings.Trim(typ, "]")
				valsData[typeName+strconv.Itoa(i)] = val
				break
			case map[string]interface{}:
				valsData = val.(map[string]interface{})
				break

			case bool, int, int32, int64, float32, float64:
				typeName := reflect.TypeOf(val).String()
				valsData[typeName+strconv.Itoa(i)] = val
				break

			default:
				var typeName string
				if reflect.ValueOf(val).Kind() == reflect.Ptr || reflect.ValueOf(val).Kind() == reflect.Interface {
					typeName = reflect.TypeOf(val).Elem().String()
				} else {
					typeName = reflect.TypeOf(val).String()
				}

				var pkgTypeName string
				if reflect.TypeOf(val).PkgPath() != "" {
					pkgTypeName = reflect.TypeOf(val).PkgPath()
				} else {
					pkgPaths = strings.Split(typeName, ".")
					pkgTypeName = pkgPaths[len(pkgPaths)-1]
				}

				indexPkgTypeName := pkgTypeName

				_, ok := typeArrayIndex[pkgTypeName]
				if !ok {
					typeArrayIndex[pkgTypeName] = 0
					nestedMap[pkgTypeName] = val

				} else {
					typeArrayIndex[pkgTypeName]++
				}

				indexPkgTypeName = pkgTypeName + strconv.Itoa(typeArrayIndex[pkgTypeName])
				nestedMap[indexPkgTypeName] = val

				packagePath := ""
				for _, p := range pkgPaths[:len(pkgPaths)-1] {
					packagePath = packagePath + p
				}
				//	fmt.Println("packagePath", packagePath)
				valsData[packagePath] = nestedMap
			}

		}

		break
	default:
		typeName := reflect.TypeOf(vals).Elem().String()
		pkgPaths = strings.Split(typeName, ".")
		//fmt.Println("default", pkgPaths)
		valsData[pkgPaths[0]] = map[string]interface{}{
			pkgPaths[1]: vals,
		}
	}

	valsData[t.ResultKey] = t.config.result
	tmplData[t.DataKey] = valsData

	//fmt.Println("map", tmplData)

}

// Execute ...
func (t TextTemplateRuleset) Execute(vals interface{}) {

	if !t.config.workflowMatch {
		//log.Warnf("ruleset %s is not valid for the current parser %s %s", t.Name, t.Workflow)
		return
	}

	types := t.getTypes(vals)
	sort.Strings(types)
	if !t.isValidForTypes(types...) {
		//	log.Warnf("invalid types %s skipping ruleset %s", types, t.Name)
		return
	}

	defer func() {
		types = types[:0]
	}()
	//	fmt.Println("types:", types)
	tmplData := t.mapBuf.get()
	defer t.mapBuf.put(tmplData)
	t.getTemplateData(tmplData, vals)

	successCount := 0

	for _, rule := range t.Rules {

		if rule.config.noResultFunc {
			//log.Warnf("rule expression contains result func but no type Result interface was set %s", rule.Name)
			continue
		}

		// validate if one of the types exist in the expression.
		err := rule.isValid(types)
		if err != nil {
			//	log.Warnf("invalid rule %s, error: %v", rule.Name, err)
			continue
		}

		if rule.config.templateErr != nil {
			//	log.Warnf("invalid rule %s, error: %v", rule.Name, rule.config.templateErr)
			continue
		}

		buf := t.bytesBuf.get()
		defer t.bytesBuf.put(buf)

		err = rule.config.template.Execute(buf, tmplData)
		if err != nil {
			//	log.Warnf("invalid rule %s, error: %v", rule.Name, err)
			continue
		}

		//log.Infof("matched rule %s", rule.Name)

		var result bool

		res := strings.TrimSpace(buf.String())

		result, err = strconv.ParseBool(res)
		if err != nil {
			//log.Warnf("parse result error", err)
			continue
		}

		// n high priority rules successful, break
		if result {
			successCount++
			if successCount == t.limit {
				break
			}
		}

	}
}
<commit_msg>let's not sort types.<commit_after>package roulette

import (
	"reflect"
	"sort"
	"strconv"
	"strings"
	"text/template"
)

// Ruleset ...
type Ruleset interface {
	Execute(vals interface{})
}

type ruleConfig struct {
	delimLeft    string
	delimRight   string
	defaultfuncs template.FuncMap
	userfuncs    template.FuncMap
	allfuncs     template.FuncMap

	expectTypes    []string
	expectTypesErr error
	noResultFunc   bool

	template    *template.Template
	templateErr error
}

// Rule is a single rule expression. A rule expression is a valid go text/template
type Rule struct {
	Name     string `xml:"name,attr"`
	Priority int    `xml:"priority,attr"`
	Expr     string `xml:",innerxml"`
	config   ruleConfig
}

func (r Rule) isValid(filterTypesArr []string) error {

	if r.config.expectTypes == nil || filterTypesArr == nil {
		return r.config.expectTypesErr
	}

	if len(filterTypesArr) == 1 && filterTypesArr[0] == "map[string]interface {}" {
		return nil
	}

	// less
	if len(filterTypesArr) < len(r.config.expectTypes) {
		return r.config.expectTypesErr
	}

	// equal to
	if len(filterTypesArr) == len(r.config.expectTypes) {
		for i := range r.config.expectTypes {
			if filterTypesArr[i] != r.config.expectTypes[i] {
				return r.config.expectTypesErr
			}
		}
	}

	// greater than
	// all expected types should be present in the template data.
	for _, expectedType := range r.config.expectTypes {
		j := sort.SearchStrings(filterTypesArr, expectedType)
		found := j < len(filterTypesArr) && filterTypesArr[j] == expectedType
		if !found {
			return r.config.expectTypesErr
		}
	}

	return nil
}

type textTemplateRulesetConfig struct {
	result         Result
	filterTypesArr []string
	workflowMatch  bool
}

// TextTemplateRuleset is a collection of rules for a valid go type
type TextTemplateRuleset struct {
	Name            string `xml:"name,attr"`
	FilterTypes     string `xml:"filterTypes,attr"`
	FilterStrict    bool   `xml:"filterStrict,attr"`
	DataKey         string `xml:"dataKey,attr"`
	ResultKey       string `xml:"resultKey,attr"`
	Rules           []Rule `xml:"rule"`
	PrioritiesCount string `xml:"prioritiesCount,attr"`
	Workflow        string `xml:"workflow,attr"`
	config          textTemplateRulesetConfig
	bytesBuf        *bytesPool
	mapBuf          *mapPool
	limit           int
}

// sort rules by priority
func (t TextTemplateRuleset) Len() int {
	return len(t.Rules)
}
func (t TextTemplateRuleset) Swap(i, j int) {
	t.Rules[i], t.Rules[j] = t.Rules[j], t.Rules[i]
}
func (t TextTemplateRuleset) Less(i, j int) bool {
	return t.Rules[i].Priority < t.Rules[j].Priority
}

func (t TextTemplateRuleset) isValidForTypes(filterTypesArr ...string) bool {

	if len(filterTypesArr) == 0 {
		return false
	}

	if len(t.config.filterTypesArr) != len(filterTypesArr) && t.FilterStrict {
		return false
	}

	// if not filterStrict, look for atleast one match
	// if filterStrict look for atleast one mismatch
	for i, v := range t.config.filterTypesArr {
		j := sort.SearchStrings(t.config.filterTypesArr, v)
		found := j < len(t.config.filterTypesArr) && t.config.filterTypesArr[i] == v
		if !found {
			if t.FilterStrict {
				return false
			}
		} else {

			// filtering is not strict and one atleast one match found.
			if !t.FilterStrict {
				return true
			}
		}
	}

	return false
}

func (t TextTemplateRuleset) getTypes(vals interface{}) []string {
	var types []string
	switch vals.(type) {
	case []interface{}:
		var typeName string
		size := len(vals.([]interface{}))
		types = make([]string, size)
		for i, v := range vals.([]interface{}) {
			if reflect.ValueOf(v).Kind() == reflect.Ptr || reflect.ValueOf(v).Kind() == reflect.Interface {
				typeName = reflect.TypeOf(v).Elem().String()
			} else {
				typeName = reflect.TypeOf(v).String()
			}

			types[i] = typeName
		}

		break

	default:
		types = make([]string, 1)
		typeName := reflect.TypeOf(vals).Elem().String()
		types[0] = typeName
	}

	return types
}

func (t TextTemplateRuleset) getTemplateData(tmplData map[string]interface{}, vals interface{}) {

	//fmt.Println("getTemplateData", reflect.TypeOf(vals))
	// flatten multiple types in template map so that they can be referred by
	// dataKey

	valsData := t.mapBuf.get()
	defer t.mapBuf.put(valsData)
	// index array of same types
	typeArrayIndex := make(map[string]int)
	var pkgPaths []string

	switch vals.(type) {
	case []interface{}:
		nestedMap := t.mapBuf.get()
		defer t.mapBuf.put(nestedMap)

		for i, val := range vals.([]interface{}) {

			switch val.(type) {
			case []string, []int, []int32, []int64, []bool, []float32, []float64, []interface{}:
				typ := reflect.TypeOf(val).String()
				typeName := strings.Trim(typ, "[]")
				typeName = strings.Trim(typ, "{}")
				valsData[typeName+"slice"+strconv.Itoa(i)] = val

				break

			case map[string]int, map[string]string, map[string]bool:
				typ := reflect.TypeOf(val).String()
				typeName := strings.Trim(typ, "[")
				typeName = strings.Trim(typ, "]")
				valsData[typeName+strconv.Itoa(i)] = val
				break
			case map[string]interface{}:
				valsData = val.(map[string]interface{})
				break

			case bool, int, int32, int64, float32, float64:
				typeName := reflect.TypeOf(val).String()
				valsData[typeName+strconv.Itoa(i)] = val
				break

			default:
				var typeName string
				if reflect.ValueOf(val).Kind() == reflect.Ptr || reflect.ValueOf(val).Kind() == reflect.Interface {
					typeName = reflect.TypeOf(val).Elem().String()
				} else {
					typeName = reflect.TypeOf(val).String()
				}

				var pkgTypeName string
				if reflect.TypeOf(val).PkgPath() != "" {
					pkgTypeName = reflect.TypeOf(val).PkgPath()
				} else {
					pkgPaths = strings.Split(typeName, ".")
					pkgTypeName = pkgPaths[len(pkgPaths)-1]
				}

				indexPkgTypeName := pkgTypeName

				_, ok := typeArrayIndex[pkgTypeName]
				if !ok {
					typeArrayIndex[pkgTypeName] = 0
					nestedMap[pkgTypeName] = val

				} else {
					typeArrayIndex[pkgTypeName]++
				}

				indexPkgTypeName = pkgTypeName + strconv.Itoa(typeArrayIndex[pkgTypeName])
				nestedMap[indexPkgTypeName] = val

				packagePath := ""
				for _, p := range pkgPaths[:len(pkgPaths)-1] {
					packagePath = packagePath + p
				}
				//	fmt.Println("packagePath", packagePath)
				valsData[packagePath] = nestedMap
			}

		}

		break
	default:
		typeName := reflect.TypeOf(vals).Elem().String()
		pkgPaths = strings.Split(typeName, ".")
		//fmt.Println("default", pkgPaths)
		valsData[pkgPaths[0]] = map[string]interface{}{
			pkgPaths[1]: vals,
		}
	}

	valsData[t.ResultKey] = t.config.result
	tmplData[t.DataKey] = valsData

	//fmt.Println("map", tmplData)

}

// Execute ...
func (t TextTemplateRuleset) Execute(vals interface{}) {

	if !t.config.workflowMatch {
		//log.Warnf("ruleset %s is not valid for the current parser %s %s", t.Name, t.Workflow)
		return
	}

	types := t.getTypes(vals)

	if !t.isValidForTypes(types...) {
		//	log.Warnf("invalid types %s skipping ruleset %s", types, t.Name)
		return
	}

	defer func() {
		types = types[:0]
	}()
	//	fmt.Println("types:", types)
	tmplData := t.mapBuf.get()
	defer t.mapBuf.put(tmplData)
	t.getTemplateData(tmplData, vals)

	successCount := 0

	for _, rule := range t.Rules {

		if rule.config.noResultFunc {
			//log.Warnf("rule expression contains result func but no type Result interface was set %s", rule.Name)
			continue
		}

		// validate if one of the types exist in the expression.
		err := rule.isValid(types)
		if err != nil {
			//	log.Warnf("invalid rule %s, error: %v", rule.Name, err)
			continue
		}

		if rule.config.templateErr != nil {
			//	log.Warnf("invalid rule %s, error: %v", rule.Name, rule.config.templateErr)
			continue
		}

		buf := t.bytesBuf.get()
		defer t.bytesBuf.put(buf)

		err = rule.config.template.Execute(buf, tmplData)
		if err != nil {
			//	log.Warnf("invalid rule %s, error: %v", rule.Name, err)
			continue
		}

		//log.Infof("matched rule %s", rule.Name)

		var result bool

		res := strings.TrimSpace(buf.String())

		result, err = strconv.ParseBool(res)
		if err != nil {
			//log.Warnf("parse result error", err)
			continue
		}

		// n high priority rules successful, break
		if result {
			successCount++
			if successCount == t.limit {
				break
			}
		}

	}
}
<|endoftext|>


<commit_before>package runc

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"
	"time"

	specs "github.com/opencontainers/runtime-spec/specs-go"
)

// Format is the type of log formatting options avaliable
type Format string

const (
	none Format = ""
	JSON Format = "json"
	Text Format = "text"
	// DefaultCommand is the default command for Runc
	DefaultCommand string = "runc"
)

// Runc is the client to the runc cli
type Runc struct {
	//If command is empty, DefaultCommand is used
	Command      string
	Root         string
	Debug        bool
	Log          string
	LogFormat    Format
	PdeathSignal syscall.Signal
}

// List returns all containers created inside the provided runc root directory
func (r *Runc) List(context context.Context) ([]*Container, error) {
	data, err := r.command(context, "list", "--format=json").Output()
	if err != nil {
		return nil, err
	}
	var out []*Container
	if err := json.Unmarshal(data, &out); err != nil {
		return nil, err
	}
	return out, nil
}

// State returns the state for the container provided by id
func (r *Runc) State(context context.Context, id string) (*Container, error) {
	data, err := r.command(context, "state", id).CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("%s: %s", err, data)
	}
	var c Container
	if err := json.Unmarshal(data, &c); err != nil {
		return nil, err
	}
	return &c, nil
}

type CreateOpts struct {
	IO
	// PidFile is a path to where a pid file should be created
	PidFile       string
	ConsoleSocket *ConsoleSocket
	Detach        bool
	NoPivot       bool
	NoNewKeyring  bool
}

func (o *CreateOpts) args() (out []string, err error) {
	if o.PidFile != "" {
		abs, err := filepath.Abs(o.PidFile)
		if err != nil {
			return nil, err
		}
		out = append(out, "--pid-file", abs)
	}
	if o.ConsoleSocket != nil {
		out = append(out, "--console-socket", o.ConsoleSocket.Path())
	}
	if o.NoPivot {
		out = append(out, "--no-pivot")
	}
	if o.NoNewKeyring {
		out = append(out, "--no-new-keyring")
	}
	if o.Detach {
		out = append(out, "--detach")
	}
	return out, nil
}

// Create creates a new container and returns its pid if it was created successfully
func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOpts) error {
	args := []string{"create", "--bundle", bundle}
	if opts != nil {
		oargs, err := opts.args()
		if err != nil {
			return err
		}
		args = append(args, oargs...)
	}
	cmd := r.command(context, append(args, id)...)
	if opts != nil && opts.IO != nil {
		opts.Set(cmd)
	}
	if cmd.Stdout == nil && cmd.Stderr == nil {
		data, err := cmd.CombinedOutput()
		if err != nil {
			return fmt.Errorf("%s: %s", err, data)
		}
		return nil
	}
	if err := cmd.Start(); err != nil {
		return err
	}
	if opts != nil && opts.IO != nil {
		if c, ok := opts.IO.(StartCloser); ok {
			if err := c.CloseAfterStart(); err != nil {
				return err
			}
		}
	}
	return cmd.Wait()
}

// Start will start an already created container
func (r *Runc) Start(context context.Context, id string) error {
	return runOrError(r.command(context, "start", id))
}

type ExecOpts struct {
	IO
	PidFile       string
	Uid           int
	Gid           int
	Cwd           string
	Tty           bool
	ConsoleSocket *ConsoleSocket
	Detach        bool
}

func (o *ExecOpts) args() (out []string, err error) {
	out = append(out, "--user", fmt.Sprintf("%d:%d", o.Uid, o.Gid))
	if o.Tty {
		out = append(out, "--tty")
	}
	if o.ConsoleSocket != nil {
		out = append(out, "--console-socket", o.ConsoleSocket.Path())
	}
	if o.Cwd != "" {
		out = append(out, "--cwd", o.Cwd)
	}
	if o.Detach {
		out = append(out, "--detach")
	}
	if o.PidFile != "" {
		abs, err := filepath.Abs(o.PidFile)
		if err != nil {
			return nil, err
		}
		out = append(out, "--pid-file", abs)
	}
	return out, nil
}

// Exec executres and additional process inside the container based on a full
// OCI Process specification
func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts *ExecOpts) error {
	f, err := ioutil.TempFile("", "-process")
	if err != nil {
		return err
	}
	defer os.Remove(f.Name())
	err = json.NewEncoder(f).Encode(spec)
	f.Close()
	if err != nil {
		return err
	}
	args := []string{"exec", "--process", f.Name()}
	if opts != nil {
		oargs, err := opts.args()
		if err != nil {
			return err
		}
		args = append(args, oargs...)
	}
	cmd := r.command(context, append(args, id)...)
	if opts != nil {
		opts.Set(cmd)
	}
	return runOrError(cmd)
}

// Run runs the create, start, delete lifecycle of the container
// and returns its exit status after it has exited
func (r *Runc) Run(context context.Context, id, bundle string, opts *CreateOpts) (int, error) {
	args := []string{"run", "--bundle", bundle}
	if opts != nil {
		oargs, err := opts.args()
		if err != nil {
			return -1, err
		}
		args = append(args, oargs...)
	}
	cmd := r.command(context, append(args, id)...)
	if opts != nil {
		opts.Set(cmd)
	}
	if err := cmd.Start(); err != nil {
		return -1, err
	}
	status, err := cmd.Process.Wait()
	if err != nil {
		return -1, err
	}
	return status.Sys().(syscall.WaitStatus).ExitStatus(), nil
}

// Delete deletes the container
func (r *Runc) Delete(context context.Context, id string) error {
	return runOrError(r.command(context, "delete", id))
}

// KillOpts specifies options for killing a container and its processes
type KillOpts struct {
	All bool
}

func (o *KillOpts) args() (out []string) {
	if o.All {
		out = append(out, "--all")
	}
	return out
}

// Kill sends the specified signal to the container
func (r *Runc) Kill(context context.Context, id string, sig int, opts *KillOpts) error {
	args := []string{
		"kill",
	}
	if opts != nil {
		args = append(args, opts.args()...)
	}
	return runOrError(r.command(context, append(args, id, strconv.Itoa(sig))...))
}

// Stats return the stats for a container like cpu, memory, and io
func (r *Runc) Stats(context context.Context, id string) (*Stats, error) {
	cmd := r.command(context, "events", "--stats", id)
	rd, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	defer func() {
		rd.Close()
		cmd.Wait()
	}()
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	var e Event
	if err := json.NewDecoder(rd).Decode(&e); err != nil {
		return nil, err
	}
	return e.Stats, nil
}

// Events returns an event stream from runc for a container with stats and OOM notifications
func (r *Runc) Events(context context.Context, id string, interval time.Duration) (chan *Event, error) {
	cmd := r.command(context, "events", fmt.Sprintf("--interval=%ds", int(interval.Seconds())), id)
	rd, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	if err := cmd.Start(); err != nil {
		rd.Close()
		return nil, err
	}
	var (
		dec = json.NewDecoder(rd)
		c   = make(chan *Event, 128)
	)
	go func() {
		defer func() {
			close(c)
			rd.Close()
			cmd.Wait()
		}()
		for {
			var e Event
			if err := dec.Decode(&e); err != nil {
				if err == io.EOF {
					return
				}
				e = Event{
					Type: "error",
					Err:  err,
				}
			}
			c <- &e
		}
	}()
	return c, nil
}

// Pause the container with the provided id
func (r *Runc) Pause(context context.Context, id string) error {
	return runOrError(r.command(context, "pause", id))
}

// Resume the container with the provided id
func (r *Runc) Resume(context context.Context, id string) error {
	return runOrError(r.command(context, "resume", id))
}

// Ps lists all the processes inside the container returning their pids
func (r *Runc) Ps(context context.Context, id string) ([]int, error) {
	data, err := r.command(context, "ps", "--format", "json", id).CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("%s: %s", err, data)
	}
	var pids []int
	if err := json.Unmarshal(data, &pids); err != nil {
		return nil, err
	}
	return pids, nil
}

func (r *Runc) args() (out []string) {
	if r.Root != "" {
		out = append(out, "--root", r.Root)
	}
	if r.Debug {
		out = append(out, "--debug")
	}
	if r.Log != "" {
		out = append(out, "--log", r.Log)
	}
	if r.LogFormat != none {
		out = append(out, "--log-format", string(r.LogFormat))
	}
	return out
}

func (r *Runc) command(context context.Context, args ...string) *exec.Cmd {
	command := r.Command
	if command == "" {
		command = DefaultCommand
	}
	cmd := exec.CommandContext(context, command, append(r.args(), args...)...)
	if r.PdeathSignal != 0 {
		cmd.SysProcAttr = &syscall.SysProcAttr{
			Pdeathsig: r.PdeathSignal,
		}
	}
	return cmd
}
<commit_msg>Add nil check for exec<commit_after>package runc

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"
	"time"

	specs "github.com/opencontainers/runtime-spec/specs-go"
)

// Format is the type of log formatting options avaliable
type Format string

const (
	none Format = ""
	JSON Format = "json"
	Text Format = "text"
	// DefaultCommand is the default command for Runc
	DefaultCommand string = "runc"
)

// Runc is the client to the runc cli
type Runc struct {
	//If command is empty, DefaultCommand is used
	Command      string
	Root         string
	Debug        bool
	Log          string
	LogFormat    Format
	PdeathSignal syscall.Signal
}

// List returns all containers created inside the provided runc root directory
func (r *Runc) List(context context.Context) ([]*Container, error) {
	data, err := r.command(context, "list", "--format=json").Output()
	if err != nil {
		return nil, err
	}
	var out []*Container
	if err := json.Unmarshal(data, &out); err != nil {
		return nil, err
	}
	return out, nil
}

// State returns the state for the container provided by id
func (r *Runc) State(context context.Context, id string) (*Container, error) {
	data, err := r.command(context, "state", id).CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("%s: %s", err, data)
	}
	var c Container
	if err := json.Unmarshal(data, &c); err != nil {
		return nil, err
	}
	return &c, nil
}

type CreateOpts struct {
	IO
	// PidFile is a path to where a pid file should be created
	PidFile       string
	ConsoleSocket *ConsoleSocket
	Detach        bool
	NoPivot       bool
	NoNewKeyring  bool
}

func (o *CreateOpts) args() (out []string, err error) {
	if o.PidFile != "" {
		abs, err := filepath.Abs(o.PidFile)
		if err != nil {
			return nil, err
		}
		out = append(out, "--pid-file", abs)
	}
	if o.ConsoleSocket != nil {
		out = append(out, "--console-socket", o.ConsoleSocket.Path())
	}
	if o.NoPivot {
		out = append(out, "--no-pivot")
	}
	if o.NoNewKeyring {
		out = append(out, "--no-new-keyring")
	}
	if o.Detach {
		out = append(out, "--detach")
	}
	return out, nil
}

// Create creates a new container and returns its pid if it was created successfully
func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOpts) error {
	args := []string{"create", "--bundle", bundle}
	if opts != nil {
		oargs, err := opts.args()
		if err != nil {
			return err
		}
		args = append(args, oargs...)
	}
	cmd := r.command(context, append(args, id)...)
	if opts != nil && opts.IO != nil {
		opts.Set(cmd)
	}
	if cmd.Stdout == nil && cmd.Stderr == nil {
		data, err := cmd.CombinedOutput()
		if err != nil {
			return fmt.Errorf("%s: %s", err, data)
		}
		return nil
	}
	if err := cmd.Start(); err != nil {
		return err
	}
	if opts != nil && opts.IO != nil {
		if c, ok := opts.IO.(StartCloser); ok {
			if err := c.CloseAfterStart(); err != nil {
				return err
			}
		}
	}
	return cmd.Wait()
}

// Start will start an already created container
func (r *Runc) Start(context context.Context, id string) error {
	return runOrError(r.command(context, "start", id))
}

type ExecOpts struct {
	IO
	PidFile       string
	Uid           int
	Gid           int
	Cwd           string
	Tty           bool
	ConsoleSocket *ConsoleSocket
	Detach        bool
}

func (o *ExecOpts) args() (out []string, err error) {
	out = append(out, "--user", fmt.Sprintf("%d:%d", o.Uid, o.Gid))
	if o.Tty {
		out = append(out, "--tty")
	}
	if o.ConsoleSocket != nil {
		out = append(out, "--console-socket", o.ConsoleSocket.Path())
	}
	if o.Cwd != "" {
		out = append(out, "--cwd", o.Cwd)
	}
	if o.Detach {
		out = append(out, "--detach")
	}
	if o.PidFile != "" {
		abs, err := filepath.Abs(o.PidFile)
		if err != nil {
			return nil, err
		}
		out = append(out, "--pid-file", abs)
	}
	return out, nil
}

// Exec executres and additional process inside the container based on a full
// OCI Process specification
func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts *ExecOpts) error {
	f, err := ioutil.TempFile("", "-process")
	if err != nil {
		return err
	}
	defer os.Remove(f.Name())
	err = json.NewEncoder(f).Encode(spec)
	f.Close()
	if err != nil {
		return err
	}
	args := []string{"exec", "--process", f.Name()}
	if opts != nil {
		oargs, err := opts.args()
		if err != nil {
			return err
		}
		args = append(args, oargs...)
	}
	cmd := r.command(context, append(args, id)...)
	if opts != nil && opts.IO != nil {
		opts.Set(cmd)
	}
	return runOrError(cmd)
}

// Run runs the create, start, delete lifecycle of the container
// and returns its exit status after it has exited
func (r *Runc) Run(context context.Context, id, bundle string, opts *CreateOpts) (int, error) {
	args := []string{"run", "--bundle", bundle}
	if opts != nil {
		oargs, err := opts.args()
		if err != nil {
			return -1, err
		}
		args = append(args, oargs...)
	}
	cmd := r.command(context, append(args, id)...)
	if opts != nil {
		opts.Set(cmd)
	}
	if err := cmd.Start(); err != nil {
		return -1, err
	}
	status, err := cmd.Process.Wait()
	if err != nil {
		return -1, err
	}
	return status.Sys().(syscall.WaitStatus).ExitStatus(), nil
}

// Delete deletes the container
func (r *Runc) Delete(context context.Context, id string) error {
	return runOrError(r.command(context, "delete", id))
}

// KillOpts specifies options for killing a container and its processes
type KillOpts struct {
	All bool
}

func (o *KillOpts) args() (out []string) {
	if o.All {
		out = append(out, "--all")
	}
	return out
}

// Kill sends the specified signal to the container
func (r *Runc) Kill(context context.Context, id string, sig int, opts *KillOpts) error {
	args := []string{
		"kill",
	}
	if opts != nil {
		args = append(args, opts.args()...)
	}
	return runOrError(r.command(context, append(args, id, strconv.Itoa(sig))...))
}

// Stats return the stats for a container like cpu, memory, and io
func (r *Runc) Stats(context context.Context, id string) (*Stats, error) {
	cmd := r.command(context, "events", "--stats", id)
	rd, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	defer func() {
		rd.Close()
		cmd.Wait()
	}()
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	var e Event
	if err := json.NewDecoder(rd).Decode(&e); err != nil {
		return nil, err
	}
	return e.Stats, nil
}

// Events returns an event stream from runc for a container with stats and OOM notifications
func (r *Runc) Events(context context.Context, id string, interval time.Duration) (chan *Event, error) {
	cmd := r.command(context, "events", fmt.Sprintf("--interval=%ds", int(interval.Seconds())), id)
	rd, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	if err := cmd.Start(); err != nil {
		rd.Close()
		return nil, err
	}
	var (
		dec = json.NewDecoder(rd)
		c   = make(chan *Event, 128)
	)
	go func() {
		defer func() {
			close(c)
			rd.Close()
			cmd.Wait()
		}()
		for {
			var e Event
			if err := dec.Decode(&e); err != nil {
				if err == io.EOF {
					return
				}
				e = Event{
					Type: "error",
					Err:  err,
				}
			}
			c <- &e
		}
	}()
	return c, nil
}

// Pause the container with the provided id
func (r *Runc) Pause(context context.Context, id string) error {
	return runOrError(r.command(context, "pause", id))
}

// Resume the container with the provided id
func (r *Runc) Resume(context context.Context, id string) error {
	return runOrError(r.command(context, "resume", id))
}

// Ps lists all the processes inside the container returning their pids
func (r *Runc) Ps(context context.Context, id string) ([]int, error) {
	data, err := r.command(context, "ps", "--format", "json", id).CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("%s: %s", err, data)
	}
	var pids []int
	if err := json.Unmarshal(data, &pids); err != nil {
		return nil, err
	}
	return pids, nil
}

func (r *Runc) args() (out []string) {
	if r.Root != "" {
		out = append(out, "--root", r.Root)
	}
	if r.Debug {
		out = append(out, "--debug")
	}
	if r.Log != "" {
		out = append(out, "--log", r.Log)
	}
	if r.LogFormat != none {
		out = append(out, "--log-format", string(r.LogFormat))
	}
	return out
}

func (r *Runc) command(context context.Context, args ...string) *exec.Cmd {
	command := r.Command
	if command == "" {
		command = DefaultCommand
	}
	cmd := exec.CommandContext(context, command, append(r.args(), args...)...)
	if r.PdeathSignal != 0 {
		cmd.SysProcAttr = &syscall.SysProcAttr{
			Pdeathsig: r.PdeathSignal,
		}
	}
	return cmd
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"os"
	"fmt"
	"log"
	"time"
	"strconv"
	"os/exec"
	"net/http"
	"github.com/unrolled/render"
	"github.com/zenazn/goji/web"
)

func (run *Run) updateStatus(status string) {
	go func() {
		run.Status = status
		run.plan.run_update <- 0
	}()
	<- run.plan.run_update
}

func (run *Run) finished() {
	go func() {
		run.Duration = time.Now().Sub(run.Start)
		run.plan.run_update <- 0
	}()
	<- run.plan.run_update
}
/*
 * Execute the steps for this plan.
 */
func (run *Run) Execute() {

	run.updateStatus("executing")

	for index, step := range run.plan.Steps {
		log.Printf("running step %s\n", step.Name)
		stepPath := fmt.Sprintf("%s/step%d", run.path, index)

		/*
		 * We want to capture the stdout and stderr from each step, so set that up here.
		 */
		stdout, err := os.Create(stepPath + ".out")
		if err != nil {
			run.updateStatus("failure")
			log.Printf("cannot create stdout log for run! out of disk space or inodes?\n")
			break
		}

		stderr, err := os.Create(stepPath + ".err")
		if err != nil {
			run.updateStatus("failure")
			log.Printf("cannot create stderr log for run! out of disk space or inodes?\n")
			break
		}

		/*
		 * Take the payload from this step, turn it into an executable script, and run it.
		 */
		exe, err := os.Create(stepPath)
		if err != nil {
			run.updateStatus("failure")
			log.Printf("cannot create file for payload! out of disk space or inodes?\n")
			break
		}

		exe.WriteString(step.Payload)
		exe.Chmod(0755)
		exe.Close()

		cmd := exec.Command(stepPath)
		cmd.Stdout = stdout
		cmd.Stderr = stderr

		/*
		 * Grab the current environment, and add an env var pointing at the trigger that
		 * caused this run. The body from an HTTP trigger is already dumped into this file,
		 * If the run was triggered by crontab-style execution, the trigger file will just
		 * contain the bare string "scheduled" (without quotes).
		 */
		env := os.Environ()
		env = append(env, fmt.Sprintf("CHANNON_TRIGGER=%s/trigger", run.path))
		cmd.Env = env

		err = cmd.Run()
		stdout.Close()
		stderr.Close()
		if err != nil {
			log.Printf("err was not nil! shit")
			log.Printf(err.Error())
			run.updateStatus("failure")
			break
		}
	}

	run.finished()

	if run.Status != "failure" {
		run.updateStatus("success")
	}

	go func() {
		for _, notification := range run.plan.Notifications {
			notification.Execute(run)
		}
	}()
}

/*
 * This handler will trigger a run from the current plan.
 */
func addRunHandler(pm *PlanManager) func(web.C, http.ResponseWriter, *http.Request) {
	return func(c web.C, w http.ResponseWriter, r *http.Request) {
		planName := c.URLParams["planName"]
		plan := pm.plans[planName]

		path, _ := os.Getwd()
		path = fmt.Sprintf("%s/plans/%s/runs/%d", path, planName, len(plan.Runs))

		// Save the trigger's body before we do anything else
		os.MkdirAll(path, 0755)
		trigger, _ := os.Create(path + "/trigger")
		io.Copy(trigger, r.Body)
		trigger.Close()

		newRunID := uint(len(plan.Runs))
		newRun := Run{Id: newRunID, Status: "pending", Trigger: "post", Start: time.Now(), plan: plan, path: path}

		go func() {
			plan.Runs = append(plan.Runs, &newRun)
			go newRun.Execute()
			plan.run_update <- 0
		}()
		<- plan.run_update

		ren := render.New(render.Options{})
		ren.JSON(w, http.StatusOK, map[string]string{"runID": fmt.Sprintf("%d", newRunID)})
	}
}

/*
 * Get the list of runs for a plan
 */
func listRunsHandler(pm *PlanManager) func(web.C, http.ResponseWriter, *http.Request) {
	return func(c web.C, w http.ResponseWriter, r *http.Request) {
		planName := c.URLParams["planName"]
		plan := pm.plans[planName]

		go func() {
			log.Printf("length of runs list: %d", len(plan.Runs))

			ren := render.New(render.Options{})
			ren.JSON(w, http.StatusOK, plan.Runs)
			plan.run_update <- 0
		}()
		<- plan.run_update
	}
}

/*
 * Get the info for a specific run
 */
func getRunHandler(pm *PlanManager) func(web.C, http.ResponseWriter, *http.Request) {
	return func(c web.C, w http.ResponseWriter, r *http.Request) {
		planName := c.URLParams["planName"]
		runID, err := strconv.ParseInt(c.URLParams["runID"], 10, 32)
		if err != nil {
			http.Error(w, err.Error(), 500)
		}

		plan := pm.plans[planName]
		run := plan.Runs[runID]

		ren := render.New(render.Options{})
		ren.JSON(w, http.StatusOK, run)
	}
}

/*
 * Delete a specific run
 */
func deleteRunHandler(pm *PlanManager) func(web.C, http.ResponseWriter, *http.Request) {
	return func(c web.C, w http.ResponseWriter, r *http.Request) {
		planName := c.URLParams["planName"]
		runID, err := strconv.ParseInt(c.URLParams["runID"], 10, 32)
		if err != nil {
			http.Error(w, err.Error(), 500)
		}

		plan := pm.plans[planName]

		go func() {
			plan.Runs[runID] = &Run{}
			plan.run_update <- 0
		}()
		<- plan.run_update
		ren := render.New(render.Options{})
		ren.JSON(w, http.StatusOK, map[string]string{"deleted": fmt.Sprintf("%d", runID)})
	}
}
<commit_msg>Execute all notifications for a run simultaneously<commit_after>package main

import (
	"io"
	"os"
	"fmt"
	"log"
	"time"
	"strconv"
	"os/exec"
	"net/http"
	"github.com/unrolled/render"
	"github.com/zenazn/goji/web"
)

func (run *Run) updateStatus(status string) {
	go func() {
		run.Status = status
		run.plan.run_update <- 0
	}()
	<- run.plan.run_update
}

func (run *Run) finished() {
	go func() {
		run.Duration = time.Now().Sub(run.Start)
		run.plan.run_update <- 0
	}()
	<- run.plan.run_update
}
/*
 * Execute the steps for this plan.
 */
func (run *Run) Execute() {

	run.updateStatus("executing")

	for index, step := range run.plan.Steps {
		log.Printf("running step %s\n", step.Name)
		stepPath := fmt.Sprintf("%s/step%d", run.path, index)

		/*
		 * We want to capture the stdout and stderr from each step, so set that up here.
		 */
		stdout, err := os.Create(stepPath + ".out")
		if err != nil {
			run.updateStatus("failure")
			log.Printf("cannot create stdout log for run! out of disk space or inodes?\n")
			break
		}

		stderr, err := os.Create(stepPath + ".err")
		if err != nil {
			run.updateStatus("failure")
			log.Printf("cannot create stderr log for run! out of disk space or inodes?\n")
			break
		}

		/*
		 * Take the payload from this step, turn it into an executable script, and run it.
		 */
		exe, err := os.Create(stepPath)
		if err != nil {
			run.updateStatus("failure")
			log.Printf("cannot create file for payload! out of disk space or inodes?\n")
			break
		}

		exe.WriteString(step.Payload)
		exe.Chmod(0755)
		exe.Close()

		cmd := exec.Command(stepPath)
		cmd.Stdout = stdout
		cmd.Stderr = stderr

		/*
		 * Grab the current environment, and add an env var pointing at the trigger that
		 * caused this run. The body from an HTTP trigger is already dumped into this file,
		 * If the run was triggered by crontab-style execution, the trigger file will just
		 * contain the bare string "scheduled" (without quotes).
		 */
		env := os.Environ()
		env = append(env, fmt.Sprintf("CHANNON_TRIGGER=%s/trigger", run.path))
		cmd.Env = env

		err = cmd.Run()
		stdout.Close()
		stderr.Close()
		if err != nil {
			log.Printf("err was not nil! shit")
			log.Printf(err.Error())
			run.updateStatus("failure")
			break
		}
	}

	run.finished()

	if run.Status != "failure" {
		run.updateStatus("success")
	}

	for _, n := range run.plan.Notifications {
		n := n // the shadow knows. http://golang.org/doc/faq#closures_and_goroutines
		go n.Execute(run)
	}
}

/*
 * This handler will trigger a run from the current plan.
 */
func addRunHandler(pm *PlanManager) func(web.C, http.ResponseWriter, *http.Request) {
	return func(c web.C, w http.ResponseWriter, r *http.Request) {
		planName := c.URLParams["planName"]
		plan := pm.plans[planName]

		path, _ := os.Getwd()
		path = fmt.Sprintf("%s/plans/%s/runs/%d", path, planName, len(plan.Runs))

		// Save the trigger's body before we do anything else
		os.MkdirAll(path, 0755)
		trigger, _ := os.Create(path + "/trigger")
		io.Copy(trigger, r.Body)
		trigger.Close()

		newRunID := uint(len(plan.Runs))
		newRun := Run{Id: newRunID, Status: "pending", Trigger: "post", Start: time.Now(), plan: plan, path: path}

		go func() {
			plan.Runs = append(plan.Runs, &newRun)
			go newRun.Execute()
			plan.run_update <- 0
		}()
		<- plan.run_update

		ren := render.New(render.Options{})
		ren.JSON(w, http.StatusOK, map[string]string{"runID": fmt.Sprintf("%d", newRunID)})
	}
}

/*
 * Get the list of runs for a plan
 */
func listRunsHandler(pm *PlanManager) func(web.C, http.ResponseWriter, *http.Request) {
	return func(c web.C, w http.ResponseWriter, r *http.Request) {
		planName := c.URLParams["planName"]
		plan := pm.plans[planName]

		go func() {
			log.Printf("length of runs list: %d", len(plan.Runs))

			ren := render.New(render.Options{})
			ren.JSON(w, http.StatusOK, plan.Runs)
			plan.run_update <- 0
		}()
		<- plan.run_update
	}
}

/*
 * Get the info for a specific run
 */
func getRunHandler(pm *PlanManager) func(web.C, http.ResponseWriter, *http.Request) {
	return func(c web.C, w http.ResponseWriter, r *http.Request) {
		planName := c.URLParams["planName"]
		runID, err := strconv.ParseInt(c.URLParams["runID"], 10, 32)
		if err != nil {
			http.Error(w, err.Error(), 500)
		}

		plan := pm.plans[planName]
		run := plan.Runs[runID]

		ren := render.New(render.Options{})
		ren.JSON(w, http.StatusOK, run)
	}
}

/*
 * Delete a specific run
 */
func deleteRunHandler(pm *PlanManager) func(web.C, http.ResponseWriter, *http.Request) {
	return func(c web.C, w http.ResponseWriter, r *http.Request) {
		planName := c.URLParams["planName"]
		runID, err := strconv.ParseInt(c.URLParams["runID"], 10, 32)
		if err != nil {
			http.Error(w, err.Error(), 500)
		}

		plan := pm.plans[planName]

		go func() {
			plan.Runs[runID] = &Run{}
			plan.run_update <- 0
		}()
		<- plan.run_update
		ren := render.New(render.Options{})
		ren.JSON(w, http.StatusOK, map[string]string{"deleted": fmt.Sprintf("%d", runID)})
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Sam Whited. All rights reserved.
// Use of this source code is governed by the MIT
// license which can be found in the LICENSE file.

// Package sass provides a cgo wrapper for libsass which can be used to compile
// SASS and SCSS to CSS.
package sass

// #cgo LDFLAGS: -lsass
/*
#include <sass_interface.h>
#include <stdlib.h>
void set_source(char* source_string, struct sass_context* ctx) {
	ctx->source_string = source_string;
}
void set_file_path(char* input_path, struct sass_file_context* ctx) {
	ctx->input_path = input_path;
}
void set_folder_paths(char* search_path, char* output_path, struct sass_folder_context* ctx) {
	ctx->search_path = search_path;
	ctx->output_path = output_path;
}
void set_options(struct sass_options options, struct sass_context* ctx) {
	ctx->options = options;
}
void set_file_options(struct sass_options options, struct sass_file_context* ctx) {
	ctx->options = options;
}
void set_folder_options(struct sass_options options, struct sass_folder_context* ctx) {
	ctx->options = options;
}
struct sass_options create_options(int output_style, int source_comments, char* image_path, char* include_paths) {
	struct sass_options options;
	options.output_style = output_style;
	options.source_comments = source_comments;
	options.image_path = image_path;
	options.include_paths = include_paths;

	return options;
}
char* get_output(struct sass_context* ctx) {
	return ctx->output_string;
}
char* get_file_output(struct sass_file_context* ctx) {
	return ctx->output_string;
}
*/
import "C"
import "unsafe"

// Output CSS styles for compiling SASS.
const (
	STYLE_NESTED = iota
	STYLE_EXPANDED
	STYLE_COMPACT
	STYLE_COMPRESSED
)

// Output style for comments.
const (
	SOURCE_COMMENTS_NONE = iota
	SOURCE_COMMENTS_DEFAULT
	SOURCE_COMMENTS_MAP
)

// A set of options to pass to the SASS compiler.
type options struct {
	output_style    int
	source_comments int
	include_paths   string
	image_path      string
}

// Returns a new options struct with the defaults initialized.
func NewOptions() options {
	return options{
		output_style:    STYLE_NESTED,
		source_comments: SOURCE_COMMENTS_NONE,
		include_paths:   "",
		image_path:      "images",
	}
}

// Compile the given sass string.
func Compile(source string, opts options) (string, error) {
	var (
		ctx *C.struct_sass_context
		ret *C.char
	)

	ctx = C.sass_new_context()
	defer C.sass_free_context(ctx)
	defer C.free(unsafe.Pointer(ret))

	ctx.setOptions(opts)
	ctx.setSource(source)
	_, err := C.sass_compile(ctx)
	ret = C.get_output(ctx)
	out := C.GoString(ret)

	return out, err
}

// Compile the given file
func CompileFile(path string, opts options) (string, error) {
	var (
		ctx *C.struct_sass_file_context
		ret *C.char
	)

	ctx = C.sass_new_file_context()
	defer C.sass_free_file_context(ctx)
	defer C.free(unsafe.Pointer(ret))

	ctx.setOptions(opts)
	ctx.setPath(path)
	_, err := C.sass_compile_file(ctx)
	ret = C.get_file_output(ctx)
	out := C.GoString(ret)

	return out, err
}

// Compile the given directory
func CompileDir(
	searchPath string,
	outPath string,
	opts options) error {

	ctx := C.sass_new_folder_context()
	defer C.sass_free_folder_context(ctx)

	ctx.setOptions(opts)
	ctx.setPaths(searchPath, outPath)
	_, err := C.sass_compile_folder(ctx)

	return err
}

// Sets the source for the given context.
func (ctx *_Ctype_struct_sass_context) setSource(source string) error {
	csource := C.CString(source)
	_, err := C.set_source(csource, ctx)
	return err
}

// Sets the path for the given file context.
func (ctx *_Ctype_struct_sass_file_context) setPath(path string) error {
	cpath := C.CString(path)
	_, err := C.set_file_path(cpath, ctx)
	return err
}

// Sets the search path and output path for the given folder context.
func (ctx *_Ctype_struct_sass_folder_context) setPaths(
	searchPath string, outPath string) error {
	cspath := C.CString(searchPath)
	copath := C.CString(outPath)
	_, err := C.set_folder_paths(cspath, copath, ctx)
	return err
}

// Sets the options for the given context
func (ctx *_Ctype_struct_sass_context) setOptions(opts options) error {
	var (
		coptions C.struct_sass_options
		cim      = C.CString(opts.image_path)
		cin      = C.CString(opts.include_paths)
		cos      = C.int(opts.output_style)
		csc      = C.int(opts.source_comments)
	)

	coptions, err := C.create_options(cos, csc, cim, cin)
	if err != nil {
		return err
	}
	_, err = C.set_options(coptions, ctx)

	return err
}

// Sets the options for the given file context
func (ctx *_Ctype_struct_sass_file_context) setOptions(opts options) error {
	coptions, err := createCOptions(opts)
	if err != nil {
		return err
	}
	_, err = C.set_file_options(coptions, ctx)

	return err
}

// Sets the options for the given folder context
func (ctx *_Ctype_struct_sass_folder_context) setOptions(opts options) error {
	coptions, err := createCOptions(opts)
	if err != nil {
		return err
	}
	_, err = C.set_folder_options(coptions, ctx)

	return err
}

func createCOptions(opts options) (C.struct_sass_options, error) {
	var (
		coptions C.struct_sass_options
		cim      = C.CString(opts.image_path)
		cin      = C.CString(opts.include_paths)
		cos      = C.int(opts.output_style)
		csc      = C.int(opts.source_comments)
	)

	coptions, err := C.create_options(cos, csc, cim, cin)
	return coptions, err
}
<commit_msg>Add some more comments<commit_after>// Copyright 2014 Sam Whited. All rights reserved.
// Use of this source code is governed by the MIT
// license which can be found in the LICENSE file.

// Package sass provides a cgo wrapper for libsass which can be used to compile
// SASS and SCSS to CSS.
package sass

// #cgo LDFLAGS: -lsass
/*
#include <sass_interface.h>
#include <stdlib.h>
void set_source(char* source_string, struct sass_context* ctx) {
	ctx->source_string = source_string;
}
void set_file_path(char* input_path, struct sass_file_context* ctx) {
	ctx->input_path = input_path;
}
void set_folder_paths(char* search_path, char* output_path, struct sass_folder_context* ctx) {
	ctx->search_path = search_path;
	ctx->output_path = output_path;
}
void set_options(struct sass_options options, struct sass_context* ctx) {
	ctx->options = options;
}
void set_file_options(struct sass_options options, struct sass_file_context* ctx) {
	ctx->options = options;
}
void set_folder_options(struct sass_options options, struct sass_folder_context* ctx) {
	ctx->options = options;
}
struct sass_options create_options(int output_style, int source_comments, char* image_path, char* include_paths) {
	struct sass_options options;
	options.output_style = output_style;
	options.source_comments = source_comments;
	options.image_path = image_path;
	options.include_paths = include_paths;

	return options;
}
char* get_output(struct sass_context* ctx) {
	return ctx->output_string;
}
char* get_file_output(struct sass_file_context* ctx) {
	return ctx->output_string;
}
*/
import "C"
import "unsafe"

// Output CSS styles for compiling SASS.
const (
	STYLE_NESTED = iota
	STYLE_EXPANDED
	STYLE_COMPACT
	STYLE_COMPRESSED
)

// Output style for comments.
const (
	SOURCE_COMMENTS_NONE = iota
	SOURCE_COMMENTS_DEFAULT
	SOURCE_COMMENTS_MAP
)

// A set of options to pass to the SASS compiler.
type options struct {
	output_style    int
	source_comments int
	include_paths   string
	image_path      string
}

// Returns a new options struct with the defaults initialized.
func NewOptions() options {
	return options{
		output_style:    STYLE_NESTED,
		source_comments: SOURCE_COMMENTS_NONE,
		include_paths:   "",
		image_path:      "images",
	}
}

// Compile the given sass string.
func Compile(source string, opts options) (string, error) {
	var (
		ctx *C.struct_sass_context
		ret *C.char
	)

	ctx = C.sass_new_context()
	defer C.sass_free_context(ctx)
	defer C.free(unsafe.Pointer(ret))

	ctx.setOptions(opts)
	ctx.setSource(source)
	_, err := C.sass_compile(ctx)
	ret = C.get_output(ctx)
	out := C.GoString(ret)

	return out, err
}

// Compile the given file.
func CompileFile(path string, opts options) (string, error) {
	var (
		ctx *C.struct_sass_file_context
		ret *C.char
	)

	ctx = C.sass_new_file_context()
	defer C.sass_free_file_context(ctx)
	defer C.free(unsafe.Pointer(ret))

	ctx.setOptions(opts)
	ctx.setPath(path)
	_, err := C.sass_compile_file(ctx)
	ret = C.get_file_output(ctx)
	out := C.GoString(ret)

	return out, err
}

// Compile the given directory.
func CompileDir(
	searchPath string,
	outPath string,
	opts options) error {

	ctx := C.sass_new_folder_context()
	defer C.sass_free_folder_context(ctx)

	ctx.setOptions(opts)
	ctx.setPaths(searchPath, outPath)
	_, err := C.sass_compile_folder(ctx)

	return err
}

// Sets the source for the given libsass context.
func (ctx *_Ctype_struct_sass_context) setSource(source string) error {
	csource := C.CString(source)
	_, err := C.set_source(csource, ctx)
	return err
}

// Sets the path for the given libsass file context.
func (ctx *_Ctype_struct_sass_file_context) setPath(path string) error {
	cpath := C.CString(path)
	_, err := C.set_file_path(cpath, ctx)
	return err
}

// Sets the search path and output path for the given libsass folder context.
func (ctx *_Ctype_struct_sass_folder_context) setPaths(
	searchPath string, outPath string) error {
	cspath := C.CString(searchPath)
	copath := C.CString(outPath)
	_, err := C.set_folder_paths(cspath, copath, ctx)
	return err
}

// Sets the libsass options for the given context.
func (ctx *_Ctype_struct_sass_context) setOptions(opts options) error {
	var (
		coptions C.struct_sass_options
		cim      = C.CString(opts.image_path)
		cin      = C.CString(opts.include_paths)
		cos      = C.int(opts.output_style)
		csc      = C.int(opts.source_comments)
	)

	coptions, err := C.create_options(cos, csc, cim, cin)
	if err != nil {
		return err
	}
	_, err = C.set_options(coptions, ctx)

	return err
}

// Sets the libsass options for the given file context.
func (ctx *_Ctype_struct_sass_file_context) setOptions(opts options) error {
	coptions, err := createCOptions(opts)
	if err != nil {
		return err
	}
	_, err = C.set_file_options(coptions, ctx)

	return err
}

// Sets the libsass options for the given folder context.
func (ctx *_Ctype_struct_sass_folder_context) setOptions(opts options) error {
	coptions, err := createCOptions(opts)
	if err != nil {
		return err
	}
	_, err = C.set_folder_options(coptions, ctx)

	return err
}

// Create a C options struct for libsass from some Go options.
func createCOptions(opts options) (C.struct_sass_options, error) {
	var (
		coptions C.struct_sass_options
		cim      = C.CString(opts.image_path)
		cin      = C.CString(opts.include_paths)
		cos      = C.int(opts.output_style)
		csc      = C.int(opts.source_comments)
	)

	coptions, err := C.create_options(cos, csc, cim, cin)
	return coptions, err
}
<|endoftext|>


<commit_before>package main

import "fmt"
import md5 "crypto/md5"
import hex "encoding/hex"
import "encoding/json"
import "flag"
import "log"
import "os"
import "path"
import "sync"
import "time"
import storage	"cloud.google.com/go/storage"
import terminal  "golang.org/x/crypto/ssh/terminal"
import "strings"
import "os/exec"
import "strconv"
import "bytes"

// SRSLY?
func min(x, y int64) int64 {
    if x < y {
        return x
    }
    return y
}

type Chunk struct {
	Path     string
	FileSize     int64
	FileModTime  time.Time
	FilePerm     os.FileMode
	Offset   int64
	Md5sum   string
	data     []byte
}

func walkDirectory(root string) <-chan Chunk {
	out := make(chan Chunk)
	var queue []string
	queue = append(queue, root)

	go func() {
		for len(queue) > 0 {
			d := queue[0]
			queue = queue[1:]

			f,err := os.Open(d)
			if err != nil {
				log.Print("Unable to open: %s\n", d)
				continue
			}

			for {
				infos,err := f.Readdir(100)
				if len(infos) == 0 {
					break;
				}
				if err != nil {
					log.Fatal("Readdir failed")
				}

				for _,stat := range infos {
					full_path := path.Join(d, stat.Name())
					if stat.IsDir() {
						queue = append(queue, full_path)
					} else {
						var o int64
						o = 0
						for o < stat.Size() {
							size := min(1 << 20, stat.Size() - o)
							c := Chunk{Path: full_path, FileSize: stat.Size(), FileModTime: stat.ModTime(), FilePerm: stat.Mode(), Offset: o, Md5sum: "empty", data: make([]byte, size)}
							out <- c
							o += (1 << 20)
						}
					}
				}
			}

		}
		close(out)
	}()

	return out
}

func hashFiles(chunks <-chan Chunk) <-chan Chunk {
	out := make(chan Chunk)

	go func() {
		for c := range chunks {
			f,err := os.Open(c.Path)
			if err != nil {
				log.Printf("Couldn't open %s. Skipping it.", c.Path)
				continue
			}
			n,err := f.ReadAt(c.data, c.Offset)
			if err != nil {
				log.Fatal("Non EOF error on ", f.Name())
			} else if  n != len(c.data) {
				log.Fatal("Short read: %d v %d (on %s)\n", n, len(c.data), c.Path)
			}
			csum := md5.Sum(c.data[:])
			c.Md5sum = hex.EncodeToString(csum[:])
			out <- c
		}
		close(out)
	}()
	return out
}

func writeJSON(chunks <-chan Chunk, writer *storage.Writer) {
	enc := json.NewEncoder(writer)
	for c := range chunks {
		err := enc.Encode(c)
		if (err != nil) {
			log.Fatal("Failed to encode")
		}

	}
	writer.Close()
}

func uploadChunks(chunks <-chan Chunk, bucket string) <-chan Chunk{
	out := make(chan Chunk)
	go func() {
		var wg sync.WaitGroup
		wg.Add(50)
		for i := 0; i < 50; i++ {
			go func() {
				for c := range chunks {
					CreateChunk(bucket, c.Md5sum, c.data)
					out <- c
				}
				wg.Done()
			}()
		}
		wg.Wait()
		close(out)
	}()
	return out
}

func mergeTwo(a <-chan Chunk, b <-chan Chunk) <-chan Chunk {
	var wg sync.WaitGroup
	out := make(chan Chunk)

	output := func(c <-chan Chunk) {
		for n := range c {
			out <- n
		}
		wg.Done()
	}

	wg.Add(2)
	go output(a)
	go output(b)

	// Start a goroutine to close out once all the output goroutines are
	// done.  This must start after the wg.Add call.
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}

func filterChunks(chunks <-chan Chunk, existing map[string]bool) (<-chan Chunk, <-chan Chunk) {
	out_existing := make(chan Chunk)
	out_new := make(chan Chunk)
	go func() {
		for c := range chunks {
			if (existing[c.Md5sum] == true) {
				out_existing <- c
			} else {
				out_new <- c
			}
		}
		close(out_new)
		close(out_existing)
	}()
	return out_new, out_existing
}

func Restore(bucket string, chunks []Chunk) {
	// TODO: verify all chunks are present
	for _,c := range chunks {
		// add in the data
		c.data = readObject(bucket, c.Md5sum)
		c.Path = c.Path[1:]
		err := os.MkdirAll(path.Dir(c.Path), 0777)
		if err != nil {
			log.Fatal(err)
		}
		f, err := os.OpenFile(c.Path, os.O_RDWR | os.O_CREATE, c.FilePerm)
		if err != nil {
			log.Fatal("Error opening: ", err)
		}
		n, err := f.WriteAt(c.data, c.Offset)
		if err != nil || n != len(c.data) {
			log.Fatal("error writing to ", c.Path, " ", err)
		}
	}
}

func RestoreAll(bucket string, metadata string) {
	chunks := make(chan Chunk)
	var wg sync.WaitGroup

	wg.Add(50)
	for i := 0; i < 50; i++ {
		go func() {
			for c := range chunks {
				// add in the data
				c.data = readObject(bucket, c.Md5sum)

				err := os.MkdirAll(path.Dir(c.Path), 0777)
				if err != nil {
					log.Fatal(err)
				}
				f, err := os.OpenFile(c.Path, os.O_RDWR | os.O_CREATE, c.FilePerm)
				if err != nil {
					log.Fatal("Error opening: ", err)
				}
				n, err := f.WriteAt(c.data, c.Offset)
				if err != nil || n != len(c.data) {
					log.Fatal("error writing to ", c.Path, " ", err)
				}
				f.Close()
			}
			wg.Done()

		}()
	}

	r := GetReader(bucket, metadata)
	dec := json.NewDecoder(r)
	for dec.More() {
		var c Chunk
		err := dec.Decode(&c)
		if err != nil {
			log.Fatal(err)
		}

		// make relative
		c.Path = c.Path[1:]
		chunks <- c
	}
	close(chunks)

	wg.Wait()
}

type FsState struct {
	pwd *FsEntry
	term *terminal.Terminal
}

type Command struct {
	name string
	min_args int
	max_args int
	usage string
	cmd func(pwd *FsState, args []string)
}

func VerifyCommand(cmd *Command, args []string) bool {
	num_args := len(args) - 1;  // -1 for command name in args[0]
	if num_args > cmd.max_args || num_args < cmd.min_args {
		return false
	}
	return true
}

func DiskUsage(dir string) int64 {
	path, err := exec.LookPath("du")
	if err != nil {
		log.Fatal("Install du")
	}
	cmd := exec.Command(path, "-sb", dir)
	var out bytes.Buffer
	cmd.Stdout = &out
	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
	s := out.String()
	parts := strings.Split(s, "\t")
	ret,_ := strconv.ParseInt(parts[0], 10, 64)
	return ret
}

func ProgressBar(total_bytes int64, in <-chan Chunk) <-chan Chunk{
	out := make(chan Chunk)
	go func() {
		var done_bytes int64
		done_bytes = 0
		for chunk := range in {
			done_bytes += int64(len(chunk.data))
			chunk.data = nil
			fmt.Printf("\r Finished %d of %d", done_bytes, total_bytes)
		}
		close(out)
	}()
	return out
}

func main() {
	root := flag.String("directory", "", "Directory to scan")
	bucket := flag.String("bucket", "", "Bucket for chunks")
	mode := flag.String("mode", "", "backup|restore")

	flag.Parse()

	initStorageClient()

	if *mode == "backup" {
		bytes := DiskUsage(*root)
		existing := make(map[string]bool)
		for s := range ListBucket(*bucket) {
			existing[s] = true  // really dumb set
		}

		chunks := walkDirectory(*root)  // get all chunks in source file system
		n, e := filterChunks(hashFiles(chunks), existing)  // hash them to find new and existing ones
		u := uploadChunks(n, *bucket) // upload the new ones, spit out chunks after uploaded
		j := mergeTwo(e, u)  // write everything to the JSON file (if a chunk gets here it's in GCS)

		// generate a name for the backup: metadata/hostname/YY/MM/DD/HH/MM
		t := time.Now()
		host,_ := os.Hostname();
		prefix := t.Format("06/01/02/03/04")
		metadata_filename := "/metadata/" + host + "/" + prefix + "/backup.json"
		fmt.Println("Writing metadata to ", metadata_filename)
		w := GetWriter(*bucket, metadata_filename, "application/json")
		writeJSON(ProgressBar(bytes, j), w)
	} else if (*mode == "restore") {
//		restore(*bucket, *restore_json)
	} else if (*mode == "interactive") {
		// Set up the terminal
		if !terminal.IsTerminal(0) {
			log.Fatal("stdin not a terminal")
		}
		oldState, err := terminal.MakeRaw(0)
		if err != nil {
			log.Fatal(err)
		}
		defer terminal.Restore(0, oldState)
		n := terminal.NewTerminal(os.Stdin, ">")

		// Insert the metadata directories:
		root := MakeDirEntry("", nil)
		for s := range ListMetadata(*bucket) {
			d := InsertPath(strings.TrimSuffix(strings.TrimPrefix(s, "/metadata"), "backup.json"), root)
			d.lazy_file_maker = func() { InsertFromJSON(d, "dumpy", s) }
		}

		var fs_state *FsState
		fs_state = &FsState{root, n}

		// Set up commands:
		cmds := make(map[string]Command)
		cmds["ls"] = Command{"ls", 0, 0, "ls ; List current directory", func(state *FsState, args []string) {
			c := ListDir(state.pwd)
			for f := range c {
				n.Write([]byte(FormatFilename(f) + "\r\n"))
			}
		}}
		cmds["cd"] = Command{"cd", 1, 1, "cd dir ; Change directory", func(state *FsState, args []string) {
			new := ChangeDir(state.pwd, args[1])
			if new == nil {
				n.Write([]byte("Error changing to" + args[1] + "\r\n"))
			} else {
				state.pwd = new
			}
		}}
		cmds["restore"] = Command{"restore", 1, 1, "restore target ; Restore a file or directory", func(state *FsState, args []string) {
			f := GetFSEntry(state.pwd, args[1])
			if f == nil {
				state.term.Write([]byte("Failed to open " + args[1] + "\r\n"))
				return
			}
			if f.file {
				Restore(*bucket, f.chunks)
			} else {
				restore_func := func(f *FsEntry, path []*FsEntry) {
					if f.file {
						state.term.Write([]byte("Restoring: " + f.name + "..."))
						Restore(*bucket, f.chunks)
						state.term.Write([]byte("done.\r\n"));
					}
				}
				Walk(state.pwd, 1023, restore_func)
			}
		}}

		// Wait for commands:
		for {
			line,_ := n.ReadLine()
			if line == "exit" {
				break
			}
			parts := strings.Split(line, " ")

			cmd,ok := cmds[parts[0]]
			if !ok {
				fs_state.term.Write([]byte("Unknown command " + parts[0] + "\r\n"))
			} else {
				if VerifyCommand(&cmd, parts) {
					cmd.cmd(fs_state, parts)
				} else {
					fs_state.term.Write([]byte(cmd.usage + "\r\n"))
				}
			}
		}
	}
}
<commit_msg>Don't swallow metadata in progress bar<commit_after>package main

import "fmt"
import md5 "crypto/md5"
import hex "encoding/hex"
import "encoding/json"
import "flag"
import "log"
import "os"
import "path"
import "sync"
import "time"
import storage	"cloud.google.com/go/storage"
import terminal  "golang.org/x/crypto/ssh/terminal"
import "strings"
import "os/exec"
import "strconv"
import "bytes"

// SRSLY?
func min(x, y int64) int64 {
    if x < y {
        return x
    }
    return y
}

type Chunk struct {
	Path     string
	FileSize     int64
	FileModTime  time.Time
	FilePerm     os.FileMode
	Offset   int64
	Md5sum   string
	data     []byte
}

func walkDirectory(root string) <-chan Chunk {
	out := make(chan Chunk)
	var queue []string
	queue = append(queue, root)

	go func() {
		for len(queue) > 0 {
			d := queue[0]
			queue = queue[1:]

			f,err := os.Open(d)
			if err != nil {
				log.Print("Unable to open: %s\n", d)
				continue
			}

			for {
				infos,err := f.Readdir(100)
				if len(infos) == 0 {
					break;
				}
				if err != nil {
					log.Fatal("Readdir failed")
				}

				for _,stat := range infos {
					full_path := path.Join(d, stat.Name())
					if stat.IsDir() {
						queue = append(queue, full_path)
					} else {
						var o int64
						o = 0
						for o < stat.Size() {
							size := min(1 << 20, stat.Size() - o)
							c := Chunk{Path: full_path, FileSize: stat.Size(), FileModTime: stat.ModTime(), FilePerm: stat.Mode(), Offset: o, Md5sum: "empty", data: make([]byte, size)}
							out <- c
							o += (1 << 20)
						}
					}
				}
			}

		}
		close(out)
	}()

	return out
}

func hashFiles(chunks <-chan Chunk) <-chan Chunk {
	out := make(chan Chunk)

	go func() {
		for c := range chunks {
			f,err := os.Open(c.Path)
			if err != nil {
				log.Printf("Couldn't open %s. Skipping it.", c.Path)
				continue
			}
			n,err := f.ReadAt(c.data, c.Offset)
			if err != nil {
				log.Fatal("Non EOF error on ", f.Name())
			} else if  n != len(c.data) {
				log.Fatal("Short read: %d v %d (on %s)\n", n, len(c.data), c.Path)
			}
			csum := md5.Sum(c.data[:])
			c.Md5sum = hex.EncodeToString(csum[:])
			out <- c
		}
		close(out)
	}()
	return out
}

func writeJSON(chunks <-chan Chunk, writer *storage.Writer) {
	enc := json.NewEncoder(writer)
	for c := range chunks {
		err := enc.Encode(c)
		if (err != nil) {
			log.Fatal("Failed to encode")
		}

	}
	writer.Close()
}

func uploadChunks(chunks <-chan Chunk, bucket string) <-chan Chunk{
	out := make(chan Chunk)
	go func() {
		var wg sync.WaitGroup
		wg.Add(50)
		for i := 0; i < 50; i++ {
			go func() {
				for c := range chunks {
					CreateChunk(bucket, c.Md5sum, c.data)
					out <- c
				}
				wg.Done()
			}()
		}
		wg.Wait()
		close(out)
	}()
	return out
}

func mergeTwo(a <-chan Chunk, b <-chan Chunk) <-chan Chunk {
	var wg sync.WaitGroup
	out := make(chan Chunk)

	output := func(c <-chan Chunk) {
		for n := range c {
			out <- n
		}
		wg.Done()
	}

	wg.Add(2)
	go output(a)
	go output(b)

	// Start a goroutine to close out once all the output goroutines are
	// done.  This must start after the wg.Add call.
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}

func filterChunks(chunks <-chan Chunk, existing map[string]bool) (<-chan Chunk, <-chan Chunk) {
	out_existing := make(chan Chunk)
	out_new := make(chan Chunk)
	go func() {
		for c := range chunks {
			if (existing[c.Md5sum] == true) {
				out_existing <- c
			} else {
				out_new <- c
			}
		}
		close(out_new)
		close(out_existing)
	}()
	return out_new, out_existing
}

func Restore(bucket string, chunks []Chunk) {
	// TODO: verify all chunks are present
	for _,c := range chunks {
		// add in the data
		c.data = readObject(bucket, c.Md5sum)
		c.Path = c.Path[1:]
		err := os.MkdirAll(path.Dir(c.Path), 0777)
		if err != nil {
			log.Fatal(err)
		}
		f, err := os.OpenFile(c.Path, os.O_RDWR | os.O_CREATE, c.FilePerm)
		if err != nil {
			log.Fatal("Error opening: ", err)
		}
		n, err := f.WriteAt(c.data, c.Offset)
		if err != nil || n != len(c.data) {
			log.Fatal("error writing to ", c.Path, " ", err)
		}
	}
}

func RestoreAll(bucket string, metadata string) {
	chunks := make(chan Chunk)
	var wg sync.WaitGroup

	wg.Add(50)
	for i := 0; i < 50; i++ {
		go func() {
			for c := range chunks {
				// add in the data
				c.data = readObject(bucket, c.Md5sum)

				err := os.MkdirAll(path.Dir(c.Path), 0777)
				if err != nil {
					log.Fatal(err)
				}
				f, err := os.OpenFile(c.Path, os.O_RDWR | os.O_CREATE, c.FilePerm)
				if err != nil {
					log.Fatal("Error opening: ", err)
				}
				n, err := f.WriteAt(c.data, c.Offset)
				if err != nil || n != len(c.data) {
					log.Fatal("error writing to ", c.Path, " ", err)
				}
				f.Close()
			}
			wg.Done()

		}()
	}

	r := GetReader(bucket, metadata)
	dec := json.NewDecoder(r)
	for dec.More() {
		var c Chunk
		err := dec.Decode(&c)
		if err != nil {
			log.Fatal(err)
		}

		// make relative
		c.Path = c.Path[1:]
		chunks <- c
	}
	close(chunks)

	wg.Wait()
}

type FsState struct {
	pwd *FsEntry
	term *terminal.Terminal
}

type Command struct {
	name string
	min_args int
	max_args int
	usage string
	cmd func(pwd *FsState, args []string)
}

func VerifyCommand(cmd *Command, args []string) bool {
	num_args := len(args) - 1;  // -1 for command name in args[0]
	if num_args > cmd.max_args || num_args < cmd.min_args {
		return false
	}
	return true
}

func DiskUsage(dir string) int64 {
	path, err := exec.LookPath("du")
	if err != nil {
		log.Fatal("Install du")
	}
	cmd := exec.Command(path, "-sb", dir)
	var out bytes.Buffer
	cmd.Stdout = &out
	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
	s := out.String()
	parts := strings.Split(s, "\t")
	ret,_ := strconv.ParseInt(parts[0], 10, 64)
	return ret
}

func ProgressBar(total_bytes int64, in <-chan Chunk) <-chan Chunk{
	out := make(chan Chunk)
	go func() {
		var done_bytes int64
		done_bytes = 0
		for chunk := range in {
			done_bytes += int64(len(chunk.data))
			chunk.data = nil
			out <- chunk
			fmt.Printf("\r Finished %d of %d", done_bytes, total_bytes)
		}
		close(out)
	}()
	return out
}

func main() {
	root := flag.String("directory", "", "Directory to scan")
	bucket := flag.String("bucket", "", "Bucket for chunks")
	mode := flag.String("mode", "", "backup|restore")

	flag.Parse()

	initStorageClient()

	if *mode == "backup" {
		bytes := DiskUsage(*root)
		existing := make(map[string]bool)
		for s := range ListBucket(*bucket) {
			existing[s] = true  // really dumb set
		}

		chunks := walkDirectory(*root)  // get all chunks in source file system
		n, e := filterChunks(hashFiles(chunks), existing)  // hash them to find new and existing ones
		u := uploadChunks(n, *bucket) // upload the new ones, spit out chunks after uploaded
		j := mergeTwo(e, u)  // write everything to the JSON file (if a chunk gets here it's in GCS)

		// generate a name for the backup: metadata/hostname/YY/MM/DD/HH/MM
		t := time.Now()
		host,_ := os.Hostname();
		prefix := t.Format("06/01/02/03/04")
		metadata_filename := "/metadata/" + host + "/" + prefix + "/backup.json"
		fmt.Println("Writing metadata to ", metadata_filename)
		w := GetWriter(*bucket, metadata_filename, "application/json")
		writeJSON(ProgressBar(bytes, j), w)
	} else if (*mode == "restore") {
//		restore(*bucket, *restore_json)
	} else if (*mode == "interactive") {
		// Set up the terminal
		if !terminal.IsTerminal(0) {
			log.Fatal("stdin not a terminal")
		}
		oldState, err := terminal.MakeRaw(0)
		if err != nil {
			log.Fatal(err)
		}
		defer terminal.Restore(0, oldState)
		n := terminal.NewTerminal(os.Stdin, ">")

		// Insert the metadata directories:
		root := MakeDirEntry("", nil)
		for s := range ListMetadata(*bucket) {
			d := InsertPath(strings.TrimSuffix(strings.TrimPrefix(s, "/metadata"), "backup.json"), root)
			d.lazy_file_maker = func() { InsertFromJSON(d, "dumpy", s) }
		}

		var fs_state *FsState
		fs_state = &FsState{root, n}

		// Set up commands:
		cmds := make(map[string]Command)
		cmds["ls"] = Command{"ls", 0, 0, "ls ; List current directory", func(state *FsState, args []string) {
			c := ListDir(state.pwd)
			for f := range c {
				n.Write([]byte(FormatFilename(f) + "\r\n"))
			}
		}}
		cmds["cd"] = Command{"cd", 1, 1, "cd dir ; Change directory", func(state *FsState, args []string) {
			new := ChangeDir(state.pwd, args[1])
			if new == nil {
				n.Write([]byte("Error changing to" + args[1] + "\r\n"))
			} else {
				state.pwd = new
			}
		}}
		cmds["restore"] = Command{"restore", 1, 1, "restore target ; Restore a file or directory", func(state *FsState, args []string) {
			f := GetFSEntry(state.pwd, args[1])
			if f == nil {
				state.term.Write([]byte("Failed to open " + args[1] + "\r\n"))
				return
			}
			if f.file {
				Restore(*bucket, f.chunks)
			} else {
				restore_func := func(f *FsEntry, path []*FsEntry) {
					if f.file {
						state.term.Write([]byte("Restoring: " + f.name + "..."))
						Restore(*bucket, f.chunks)
						state.term.Write([]byte("done.\r\n"));
					}
				}
				Walk(state.pwd, 1023, restore_func)
			}
		}}

		// Wait for commands:
		for {
			line,_ := n.ReadLine()
			if line == "exit" {
				break
			}
			parts := strings.Split(line, " ")

			cmd,ok := cmds[parts[0]]
			if !ok {
				fs_state.term.Write([]byte("Unknown command " + parts[0] + "\r\n"))
			} else {
				if VerifyCommand(&cmd, parts) {
					cmd.cmd(fs_state, parts)
				} else {
					fs_state.term.Write([]byte(cmd.usage + "\r\n"))
				}
			}
		}
	}
}
<|endoftext|>