<commit_before>package dsunit

import (
	"bytes"
	"encoding/json"
	"github.com/pkg/errors"
	"github.com/viant/assertly"
	"github.com/viant/dsunit/sv"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/storage"
	"github.com/viant/toolbox/url"
	"io"
	"io/ioutil"
	"sort"
	"strings"
)

const (
	AutoincrementDirective  = "@autoincrement@"
	FromQueryDirective      = "@fromQuery@"
	FromQueryAliasDirective = "@fromQueryAlias@"
)

//Records represent data records
type Dataset struct {
	Table   string  `required:"true"`
	Records Records `required:"true"`
}

//NewDataset creates a new dataset for supplied table and records.
func NewDataset(table string, records ...map[string]interface{}) *Dataset {
	return &Dataset{
		Table:   table,
		Records: records,
	}
}

//Records represents table records
type Records []map[string]interface{}

//Records returns non empty records //directive a filtered out
func (r *Records) Expand(context toolbox.Context, includeDirectives bool) (result []interface{}, err error) {
	result = make([]interface{}, 0)

	var evaluator = assertly.NewDefaultMacroEvaluator()

	for _, candidate := range *r {
		record := Record(candidate)
		recordValues := make(map[string]interface{})
		var keys = record.Columns()
		if includeDirectives {
			keys = toolbox.MapKeysToStringSlice(record)
		}
		for _, k := range keys {
			v := record[k]
			recordValues[k] = v
			if text, ok := v.(string); ok {
				if recordValues[k], err = evaluator.Expand(context, text); err != nil {
					return nil, err
				}
			}
		}
		if len(recordValues) > 0 {
			result = append(result, recordValues)
		}
	}
	return result, nil
}

//ShouldDeleteAll checks if dataset contains empty record (indicator to delete all)
func (r *Records) ShouldDeleteAll() bool {
	var result = len(*r) == 0
	directiveScan(*r, func(record Record) {
		if record == nil || len(record) == 0 {
			result = true
		}
	})
	return result
}

//UniqueKeys returns value for unique key directive, it test keys in the following order: @Autoincrement@, @IndexBy@
func (r *Records) UniqueKeys() []string {
	var result []string
	directiveScan(*r, func(record Record) {
		for k, v := range record {
			if k == AutoincrementDirective || k == assertly.IndexByDirective {
				if keys, ok := v.([]string); ok {
					result = keys
				} else {
					result = strings.Split(toolbox.AsString(v), ",")
				}
			}
		}
	})
	return result
}

//FromQuery returns value for @FromQuery@ directive
func (r *Records) FromQuery() (string, string) {
	var fromQuery string
	var alias string
	directiveScan(*r, func(record Record) {
		for k, v := range record {
			if k == FromQueryDirective {
				fromQuery = toolbox.AsString(v)
			}
			if k == FromQueryAliasDirective {
				alias = toolbox.AsString(v)
			}
		}
	})
	return fromQuery, alias
}

//PrimaryKey returns primary key directive if matched in the following order: @Autoincrement@, @IndexBy@
func (r *Records) Autoincrement() bool {
	var result = false
	directiveScan(*r, func(record Record) {
		for k := range record {
			if k == AutoincrementDirective {
				result = true
			}
		}
	})
	return result
}

//Columns returns unique column names for this dataset
func (r *Records) Columns() []string {
	var result = make([]string, 0)
	var unique = make(map[string]bool)
	for _, record := range *r {
		var actualRecord = Record(record)
		for _, column := range actualRecord.Columns() {
			if _, has := unique[column]; has {
				continue
			}
			unique[column] = true
			result = append(result, column)
		}
	}
	sort.Strings(result)
	return result
}

//DatastoreDatasets represents a collection of datastore datasets
type DatastoreDatasets struct {
	Datastore string                              `required:"true" description:"register datastore"`
	Datasets  []*Dataset                          `description:"collection of dataset per table"`
	Data      map[string][]map[string]interface{} `description:"map, where each pair represent table name and records (backwad compatiblity)"`
}

//DatasetResource represents a dataset resource
type DatasetResource struct {
	*url.Resource      ` description:"data file location, csv, json, ndjson formats are supported"`
	*DatastoreDatasets `required:"true" description:"datastore datasets"`
	Prefix             string ` description:"location data file prefix"`  //apply prefix
	Postfix            string ` description:"location data file postgix"` //apply suffix
	loaded             bool   //flag to indicate load is called
}

func (r *DatasetResource) loadDataset() (err error) {
	if r.Resource.URL == "" {
		return errors.New("resource was empty")
	}

	r.Resource.Init()
	var storageService storage.Service
	storageService, err = storage.NewServiceForURL(r.URL, r.Credentials)
	if err != nil {
		return err
	}
	var candidates []storage.Object
	candidates, err = storageService.List(r.Resource.URL)
	if err != nil {
		return err
	}
	for _, candidate := range candidates {
		if candidate.FileInfo().IsDir() {
			continue
		}
		err = r.load(storageService, candidate)
		if err != nil {
			return err
		}
	}
	return err
}

//Loads dataset from specified resource or data map
func (r *DatasetResource) Load() (err error) {
	if r == nil {
		return errors.New("dataset resource was empty")
	}
	if r.loaded {
		return nil
	}
	r.loaded = true
	if len(r.Datasets) == 0 {
		r.Datasets = make([]*Dataset, 0)
	}
	if r.Resource != nil && r.Resource.URL != "" {
		if err = r.loadDataset(); err != nil {
			return err
		}
	}
	if len(r.Data) > 0 {
		for k, v := range r.Data {
			r.Datasets = append(r.Datasets, NewDataset(k, v...))
		}
	}
	return nil
}

func (r *DatasetResource) Init() error {
	return nil
}

func (r *DatasetResource) load(service storage.Service, object storage.Object) (err error) {
	if len(r.Datasets) == 0 {
		r.Datasets = make([]*Dataset, 0)
	}
	datafile := NewDatafileInfo(object.FileInfo().Name(), r.Prefix, r.Postfix)
	if datafile == nil {
		return nil
	}
	var loader func(datafile *DatafileInfo, data []byte) error
	switch datafile.Ext {
	case "json":
		loader = r.loadJSON

	case "csv":
		loader = r.loadCSV
	case "tsv":
		loader = r.loadTSV
	}
	if loader != nil {
		var reader io.ReadCloser
		if reader, err = service.Download(object); err == nil {
			defer reader.Close()
			var content []byte
			if content, err = ioutil.ReadAll(reader); err == nil {
				err = loader(datafile, content)
			}
		}
	}
	return nil
}

func (r *DatasetResource) loadJSON(datafile *DatafileInfo, data []byte) error {
	var dataSet = &Dataset{
		Table:   datafile.Name,
		Records: make([]map[string]interface{}, 0),
	}
	if toolbox.IsNewLineDelimitedJSON(string(data)) {
		if records, err := toolbox.NewLineDelimitedJSON(string(data)); err == nil {
			for _, record := range records {
				if recordMap, ok := record.(map[string]interface{}); ok {
					dataSet.Records = append(dataSet.Records, recordMap)
				}
			}
			r.Datasets = append(r.Datasets, dataSet)
			return nil
		}

	}
	err := json.NewDecoder(bytes.NewReader(data)).Decode(&dataSet.Records)
	if err != nil {
		return err
	}
	r.Datasets = append(r.Datasets, dataSet)
	return nil
}

func (r *DatasetResource) loadCSV(datafile *DatafileInfo, data []byte) error {
	return r.loadSeparatedData(",", datafile, data)
}

func (r *DatasetResource) loadTSV(datafile *DatafileInfo, data []byte) error {
	return r.loadSeparatedData("\t", datafile, data)
}

func (r *DatasetResource) loadSeparatedData(delimiter string, datafile *DatafileInfo, data []byte) error {
	records, err := sv.NewSeparatedValueParser(delimiter).Parse(data)
	if err != nil {
		return err
	}
	var dataSet = &Dataset{
		Table:   datafile.Name,
		Records: records,
	}

	r.Datasets = append(r.Datasets, dataSet)
	return nil
}

func NewDatasetResource(datastore string, URL, prefix, postfix string, datasets ...*Dataset) *DatasetResource {
	var result = &DatasetResource{
		Resource: url.NewResource(URL),
		DatastoreDatasets: &DatastoreDatasets{
			Datastore: datastore,
			Datasets:  datasets,
		},
		Prefix:  prefix,
		Postfix: postfix,
	}
	return result
}
<commit_msg>added error check in load<commit_after>package dsunit

import (
	"bytes"
	"encoding/json"
	"github.com/pkg/errors"
	"github.com/viant/assertly"
	"github.com/viant/dsunit/sv"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/storage"
	"github.com/viant/toolbox/url"
	"io"
	"io/ioutil"
	"sort"
	"strings"
)

const (
	AutoincrementDirective  = "@autoincrement@"
	FromQueryDirective      = "@fromQuery@"
	FromQueryAliasDirective = "@fromQueryAlias@"
)

//Records represent data records
type Dataset struct {
	Table   string  `required:"true"`
	Records Records `required:"true"`
}

//NewDataset creates a new dataset for supplied table and records.
func NewDataset(table string, records ...map[string]interface{}) *Dataset {
	return &Dataset{
		Table:   table,
		Records: records,
	}
}

//Records represents table records
type Records []map[string]interface{}

//Records returns non empty records //directive a filtered out
func (r *Records) Expand(context toolbox.Context, includeDirectives bool) (result []interface{}, err error) {
	result = make([]interface{}, 0)

	var evaluator = assertly.NewDefaultMacroEvaluator()

	for _, candidate := range *r {
		record := Record(candidate)
		recordValues := make(map[string]interface{})
		var keys = record.Columns()
		if includeDirectives {
			keys = toolbox.MapKeysToStringSlice(record)
		}
		for _, k := range keys {
			v := record[k]
			recordValues[k] = v
			if text, ok := v.(string); ok {
				if recordValues[k], err = evaluator.Expand(context, text); err != nil {
					return nil, err
				}
			}
		}
		if len(recordValues) > 0 {
			result = append(result, recordValues)
		}
	}
	return result, nil
}

//ShouldDeleteAll checks if dataset contains empty record (indicator to delete all)
func (r *Records) ShouldDeleteAll() bool {
	var result = len(*r) == 0
	directiveScan(*r, func(record Record) {
		if record == nil || len(record) == 0 {
			result = true
		}
	})
	return result
}

//UniqueKeys returns value for unique key directive, it test keys in the following order: @Autoincrement@, @IndexBy@
func (r *Records) UniqueKeys() []string {
	var result []string
	directiveScan(*r, func(record Record) {
		for k, v := range record {
			if k == AutoincrementDirective || k == assertly.IndexByDirective {
				if keys, ok := v.([]string); ok {
					result = keys
				} else {
					result = strings.Split(toolbox.AsString(v), ",")
				}
			}
		}
	})
	return result
}

//FromQuery returns value for @FromQuery@ directive
func (r *Records) FromQuery() (string, string) {
	var fromQuery string
	var alias string
	directiveScan(*r, func(record Record) {
		for k, v := range record {
			if k == FromQueryDirective {
				fromQuery = toolbox.AsString(v)
			}
			if k == FromQueryAliasDirective {
				alias = toolbox.AsString(v)
			}
		}
	})
	return fromQuery, alias
}

//PrimaryKey returns primary key directive if matched in the following order: @Autoincrement@, @IndexBy@
func (r *Records) Autoincrement() bool {
	var result = false
	directiveScan(*r, func(record Record) {
		for k := range record {
			if k == AutoincrementDirective {
				result = true
			}
		}
	})
	return result
}

//Columns returns unique column names for this dataset
func (r *Records) Columns() []string {
	var result = make([]string, 0)
	var unique = make(map[string]bool)
	for _, record := range *r {
		var actualRecord = Record(record)
		for _, column := range actualRecord.Columns() {
			if _, has := unique[column]; has {
				continue
			}
			unique[column] = true
			result = append(result, column)
		}
	}
	sort.Strings(result)
	return result
}

//DatastoreDatasets represents a collection of datastore datasets
type DatastoreDatasets struct {
	Datastore string                              `required:"true" description:"register datastore"`
	Datasets  []*Dataset                          `description:"collection of dataset per table"`
	Data      map[string][]map[string]interface{} `description:"map, where each pair represent table name and records (backwad compatiblity)"`
}

//DatasetResource represents a dataset resource
type DatasetResource struct {
	*url.Resource      ` description:"data file location, csv, json, ndjson formats are supported"`
	*DatastoreDatasets `required:"true" description:"datastore datasets"`
	Prefix             string ` description:"location data file prefix"`  //apply prefix
	Postfix            string ` description:"location data file postgix"` //apply suffix
	loaded             bool   //flag to indicate load is called
}

func (r *DatasetResource) loadDataset() (err error) {
	if r.Resource.URL == "" {
		return errors.New("resource was empty")
	}

	r.Resource.Init()
	var storageService storage.Service
	storageService, err = storage.NewServiceForURL(r.URL, r.Credentials)
	if err != nil {
		return err
	}
	var candidates []storage.Object
	candidates, err = storageService.List(r.Resource.URL)
	if err != nil {
		return err
	}
	for _, candidate := range candidates {
		if candidate.FileInfo().IsDir() {
			continue
		}
		err = r.load(storageService, candidate)
		if err != nil {
			return err
		}
	}
	return err
}

//Loads dataset from specified resource or data map
func (r *DatasetResource) Load() (err error) {
	if r == nil {
		return errors.New("dataset resource was empty")
	}
	if r.loaded {
		return nil
	}
	r.loaded = true
	if len(r.Datasets) == 0 {
		r.Datasets = make([]*Dataset, 0)
	}
	if r.Resource != nil && r.Resource.URL != "" {
		if err = r.loadDataset(); err != nil {
			return err
		}
	}
	if len(r.Data) > 0 {
		for k, v := range r.Data {
			r.Datasets = append(r.Datasets, NewDataset(k, v...))
		}
	}
	return nil
}

func (r *DatasetResource) Init() error {
	return nil
}

func (r *DatasetResource) load(service storage.Service, object storage.Object) (err error) {
	if len(r.Datasets) == 0 {
		r.Datasets = make([]*Dataset, 0)
	}
	datafile := NewDatafileInfo(object.FileInfo().Name(), r.Prefix, r.Postfix)
	if datafile == nil {
		return nil
	}
	var loader func(datafile *DatafileInfo, data []byte) error
	switch datafile.Ext {
	case "json":
		loader = r.loadJSON

	case "csv":
		loader = r.loadCSV
	case "tsv":
		loader = r.loadTSV
	}
	if loader != nil {
		var reader io.ReadCloser
		if reader, err = service.Download(object); err == nil {
			defer reader.Close()
			var content []byte
			if content, err = ioutil.ReadAll(reader); err == nil {
				if err = loader(datafile, content);err != nil {
					return err
				}
			}
		}
	}
	return err
}

func (r *DatasetResource) loadJSON(datafile *DatafileInfo, data []byte) error {
	var dataSet = &Dataset{
		Table:   datafile.Name,
		Records: make([]map[string]interface{}, 0),
	}
	if toolbox.IsNewLineDelimitedJSON(string(data)) {
		if records, err := toolbox.NewLineDelimitedJSON(string(data)); err == nil {
			for _, record := range records {
				if recordMap, ok := record.(map[string]interface{}); ok {
					dataSet.Records = append(dataSet.Records, recordMap)
				}
			}
			r.Datasets = append(r.Datasets, dataSet)
			return nil
		}

	}
	err := json.NewDecoder(bytes.NewReader(data)).Decode(&dataSet.Records)
	if err != nil {
		return err
	}
	r.Datasets = append(r.Datasets, dataSet)
	return nil
}

func (r *DatasetResource) loadCSV(datafile *DatafileInfo, data []byte) error {
	return r.loadSeparatedData(",", datafile, data)
}

func (r *DatasetResource) loadTSV(datafile *DatafileInfo, data []byte) error {
	return r.loadSeparatedData("\t", datafile, data)
}

func (r *DatasetResource) loadSeparatedData(delimiter string, datafile *DatafileInfo, data []byte) error {
	records, err := sv.NewSeparatedValueParser(delimiter).Parse(data)
	if err != nil {
		return err
	}
	var dataSet = &Dataset{
		Table:   datafile.Name,
		Records: records,
	}

	r.Datasets = append(r.Datasets, dataSet)
	return nil
}

func NewDatasetResource(datastore string, URL, prefix, postfix string, datasets ...*Dataset) *DatasetResource {
	var result = &DatasetResource{
		Resource: url.NewResource(URL),
		DatastoreDatasets: &DatastoreDatasets{
			Datastore: datastore,
			Datasets:  datasets,
		},
		Prefix:  prefix,
		Postfix: postfix,
	}
	return result
}
<|endoftext|>


<commit_before>// Copyright (c) 2012 VMware, Inc.

package gosigar

/*
#include <stdlib.h>
#include <sys/sysctl.h>
#include <sys/mount.h>
#include <mach/mach_init.h>
#include <mach/mach_host.h>
#include <mach/host_info.h>
#include <libproc.h>
#include <mach/processor_info.h>
#include <mach/vm_map.h>
*/
import "C"

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"os/user"
	"runtime"
	"strconv"
	"syscall"
	"time"
	"unsafe"
)

func (self *LoadAverage) Get() error {
	avg := []C.double{0, 0, 0}

	C.getloadavg(&avg[0], C.int(len(avg)))

	self.One = float64(avg[0])
	self.Five = float64(avg[1])
	self.Fifteen = float64(avg[2])

	return nil
}

func (self *Uptime) Get() error {
	tv := syscall.Timeval32{}

	if err := sysctlbyname("kern.boottime", &tv); err != nil {
		return err
	}

	self.Length = time.Since(time.Unix(int64(tv.Sec), int64(tv.Usec)*1000)).Seconds()

	return nil
}

func (self *Mem) Get() error {
	var vmstat C.vm_statistics_data_t

	if err := sysctlbyname("hw.memsize", &self.Total); err != nil {
		return err
	}

	if err := vm_info(&vmstat); err != nil {
		return err
	}

	kern := uint64(vmstat.inactive_count) << 12
	self.Free = uint64(vmstat.free_count) << 12

	self.Used = self.Total - self.Free
	self.ActualFree = self.Free + kern
	self.ActualUsed = self.Used - kern

	return nil
}

type xsw_usage struct {
	Total, Avail, Used uint64
}

func (self *Swap) Get() error {
	sw_usage := xsw_usage{}

	if err := sysctlbyname("vm.swapusage", &sw_usage); err != nil {
		return err
	}

	self.Total = sw_usage.Total
	self.Used = sw_usage.Used
	self.Free = sw_usage.Avail

	return nil
}

func (self *Cpu) Get() error {
	var count C.mach_msg_type_number_t = C.HOST_CPU_LOAD_INFO_COUNT
	var cpuload C.host_cpu_load_info_data_t

	status := C.host_statistics(C.host_t(C.mach_host_self()),
		C.HOST_CPU_LOAD_INFO,
		C.host_info_t(unsafe.Pointer(&cpuload)),
		&count)

	if status != C.KERN_SUCCESS {
		return fmt.Errorf("host_statistics error=%d", status)
	}

	self.User = uint64(cpuload.cpu_ticks[C.CPU_STATE_USER])
	self.Sys = uint64(cpuload.cpu_ticks[C.CPU_STATE_SYSTEM])
	self.Idle = uint64(cpuload.cpu_ticks[C.CPU_STATE_IDLE])
	self.Nice = uint64(cpuload.cpu_ticks[C.CPU_STATE_NICE])

	return nil
}

func (self *CpuList) Get() error {
	var count C.mach_msg_type_number_t
	var cpuload *C.processor_cpu_load_info_data_t
	var ncpu C.natural_t

	status := C.host_processor_info(C.host_t(C.mach_host_self()),
		C.PROCESSOR_CPU_LOAD_INFO,
		&ncpu,
		(*C.processor_info_array_t)(unsafe.Pointer(&cpuload)),
		&count)

	if status != C.KERN_SUCCESS {
		return fmt.Errorf("host_processor_info error=%d", status)
	}

	// jump through some cgo casting hoops and ensure we properly free
	// the memory that cpuload points to
	target := C.vm_map_t(C.mach_task_self_)
	address := C.vm_address_t(uintptr(unsafe.Pointer(cpuload)))
	defer C.vm_deallocate(target, address, C.vm_size_t(ncpu))

	// the body of struct processor_cpu_load_info
	// aka processor_cpu_load_info_data_t
	var cpu_ticks [C.CPU_STATE_MAX]uint32

	// copy the cpuload array to a []byte buffer
	// where we can binary.Read the data
	size := int(ncpu) * binary.Size(cpu_ticks)
	buf := C.GoBytes(unsafe.Pointer(cpuload), C.int(size))

	bbuf := bytes.NewBuffer(buf)

	self.List = make([]Cpu, 0, ncpu)

	for i := 0; i < int(ncpu); i++ {
		cpu := Cpu{}

		err := binary.Read(bbuf, binary.LittleEndian, &cpu_ticks)
		if err != nil {
			return err
		}

		cpu.User = uint64(cpu_ticks[C.CPU_STATE_USER])
		cpu.Sys = uint64(cpu_ticks[C.CPU_STATE_SYSTEM])
		cpu.Idle = uint64(cpu_ticks[C.CPU_STATE_IDLE])
		cpu.Nice = uint64(cpu_ticks[C.CPU_STATE_NICE])

		self.List = append(self.List, cpu)
	}

	return nil
}

func (self *FDUsage) Get() error {
	return ErrNotImplemented{runtime.GOOS}
}

func (self *FileSystemList) Get() error {
	num, err := syscall.Getfsstat(nil, C.MNT_NOWAIT)
	if err != nil {
		return err
	}

	buf := make([]syscall.Statfs_t, num)

	_, err = syscall.Getfsstat(buf, C.MNT_NOWAIT)
	if err != nil {
		return err
	}

	fslist := make([]FileSystem, 0, num)

	for i := 0; i < num; i++ {
		fs := FileSystem{}

		fs.DirName = bytePtrToString(&buf[i].Mntonname[0])
		fs.DevName = bytePtrToString(&buf[i].Mntfromname[0])
		fs.SysTypeName = bytePtrToString(&buf[i].Fstypename[0])

		fslist = append(fslist, fs)
	}

	self.List = fslist

	return err
}

func (self *ProcList) Get() error {
	n := C.proc_listpids(C.PROC_ALL_PIDS, 0, nil, 0)
	if n <= 0 {
		return syscall.EINVAL
	}
	buf := make([]byte, n)
	n = C.proc_listpids(C.PROC_ALL_PIDS, 0, unsafe.Pointer(&buf[0]), n)
	if n <= 0 {
		return syscall.ENOMEM
	}

	var pid int32
	num := int(n) / binary.Size(pid)
	list := make([]int, 0, num)
	bbuf := bytes.NewBuffer(buf)

	for i := 0; i < num; i++ {
		if err := binary.Read(bbuf, binary.LittleEndian, &pid); err != nil {
			return err
		}
		if pid == 0 {
			continue
		}

		list = append(list, int(pid))
	}

	self.List = list

	return nil
}

func (self *ProcState) Get(pid int) error {
	info := C.struct_proc_taskallinfo{}

	if err := task_info(pid, &info); err != nil {
		return err
	}

	self.Name = C.GoString(&info.pbsd.pbi_comm[0])

	switch info.pbsd.pbi_status {
	case C.SIDL:
		self.State = RunStateIdle
	case C.SRUN:
		self.State = RunStateRun
	case C.SSLEEP:
		self.State = RunStateSleep
	case C.SSTOP:
		self.State = RunStateStop
	case C.SZOMB:
		self.State = RunStateZombie
	default:
		self.State = RunStateUnknown
	}

	self.Ppid = int(info.pbsd.pbi_ppid)

	self.Pgid = int(info.pbsd.pbi_pgid)

	self.Tty = int(info.pbsd.e_tdev)

	self.Priority = int(info.ptinfo.pti_priority)

	self.Nice = int(info.pbsd.pbi_nice)

	// Get process username. Fallback to UID if username is not available.
	uid := strconv.Itoa(int(info.pbsd.pbi_uid))
	user, err := user.LookupId(uid)
	if err == nil && user.Username != "" {
		self.Username = user.Username
	} else {
		self.Username = uid
	}

	return nil
}

func (self *ProcMem) Get(pid int) error {
	info := C.struct_proc_taskallinfo{}

	if err := task_info(pid, &info); err != nil {
		return err
	}

	self.Size = uint64(info.ptinfo.pti_virtual_size)
	self.Resident = uint64(info.ptinfo.pti_resident_size)
	self.PageFaults = uint64(info.ptinfo.pti_faults)

	return nil
}

func (self *ProcTime) Get(pid int) error {
	info := C.struct_proc_taskallinfo{}

	if err := task_info(pid, &info); err != nil {
		return err
	}

	self.User =
		uint64(info.ptinfo.pti_total_user) / uint64(time.Millisecond)

	self.Sys =
		uint64(info.ptinfo.pti_total_system) / uint64(time.Millisecond)

	self.Total = self.User + self.Sys

	self.StartTime = (uint64(info.pbsd.pbi_start_tvsec) * 1000) +
		(uint64(info.pbsd.pbi_start_tvusec) / 1000)

	return nil
}

func (self *ProcArgs) Get(pid int) error {
	var args []string

	argv := func(arg string) {
		args = append(args, arg)
	}

	err := kern_procargs(pid, nil, argv, nil)

	self.List = args

	return err
}

func (self *ProcExe) Get(pid int) error {
	exe := func(arg string) {
		self.Name = arg
	}

	return kern_procargs(pid, exe, nil, nil)
}

func (self *ProcFDUsage) Get(pid int) error {
	return ErrNotImplemented{runtime.GOOS}
}

// wrapper around sysctl KERN_PROCARGS2
// callbacks params are optional,
// up to the caller as to which pieces of data they want
func kern_procargs(pid int,
	exe func(string),
	argv func(string),
	env func(string, string)) error {

	mib := []C.int{C.CTL_KERN, C.KERN_PROCARGS2, C.int(pid)}
	argmax := uintptr(C.ARG_MAX)
	buf := make([]byte, argmax)
	err := sysctl(mib, &buf[0], &argmax, nil, 0)
	if err != nil {
		return nil
	}

	bbuf := bytes.NewBuffer(buf)
	bbuf.Truncate(int(argmax))

	var argc int32
	binary.Read(bbuf, binary.LittleEndian, &argc)

	path, err := bbuf.ReadBytes(0)
	if exe != nil {
		exe(string(chop(path)))
	}
	if err != nil {
		return fmt.Errorf("Error reading the argv[0]: %v", err)
	}

	// skip trailing \0's
	for {
		c, err := bbuf.ReadByte()
		if err != nil {
			return fmt.Errorf("Error skipping nils: %v", err)
		}
		if c != 0 {
			bbuf.UnreadByte()
			break // start of argv[0]
		}
	}

	for i := 0; i < int(argc); i++ {
		arg, err := bbuf.ReadBytes(0)
		if err == io.EOF {
			break
		}
		if err != nil {
			return fmt.Errorf("Error reading args: %v", err)
		}
		if argv != nil {
			argv(string(chop(arg)))
		}
	}

	if env == nil {
		return nil
	}

	delim := []byte{61} // "="

	for {
		line, err := bbuf.ReadBytes(0)
		if err == io.EOF || line[0] == 0 {
			break
		}
		if err != nil {
			return fmt.Errorf("Error reading args: %v", err)
		}
		pair := bytes.SplitN(chop(line), delim, 2)
		env(string(pair[0]), string(pair[1]))
	}

	return nil
}

// XXX copied from zsyscall_darwin_amd64.go
func sysctl(mib []C.int, old *byte, oldlen *uintptr,
	new *byte, newlen uintptr) (err error) {
	var p0 unsafe.Pointer
	p0 = unsafe.Pointer(&mib[0])
	_, _, e1 := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(p0),
		uintptr(len(mib)),
		uintptr(unsafe.Pointer(old)), uintptr(unsafe.Pointer(oldlen)),
		uintptr(unsafe.Pointer(new)), uintptr(newlen))
	if e1 != 0 {
		err = e1
	}
	return
}

func vm_info(vmstat *C.vm_statistics_data_t) error {
	var count C.mach_msg_type_number_t = C.HOST_VM_INFO_COUNT

	status := C.host_statistics(
		C.host_t(C.mach_host_self()),
		C.HOST_VM_INFO,
		C.host_info_t(unsafe.Pointer(vmstat)),
		&count)

	if status != C.KERN_SUCCESS {
		return fmt.Errorf("host_statistics=%d", status)
	}

	return nil
}

// generic Sysctl buffer unmarshalling
func sysctlbyname(name string, data interface{}) (err error) {
	val, err := syscall.Sysctl(name)
	if err != nil {
		return err
	}

	buf := []byte(val)

	switch v := data.(type) {
	case *uint64:
		*v = *(*uint64)(unsafe.Pointer(&buf[0]))
		return
	}

	bbuf := bytes.NewBuffer([]byte(val))
	return binary.Read(bbuf, binary.LittleEndian, data)
}

func task_info(pid int, info *C.struct_proc_taskallinfo) error {
	size := C.int(unsafe.Sizeof(*info))
	ptr := unsafe.Pointer(info)

	n := C.proc_pidinfo(C.int(pid), C.PROC_PIDTASKALLINFO, 0, ptr, size)
	if n != size {
		return fmt.Errorf("Could not read process info for pid %d", pid)
	}

	return nil
}
<commit_msg>Check errors before using the result (#48)<commit_after>// Copyright (c) 2012 VMware, Inc.

package gosigar

/*
#include <stdlib.h>
#include <sys/sysctl.h>
#include <sys/mount.h>
#include <mach/mach_init.h>
#include <mach/mach_host.h>
#include <mach/host_info.h>
#include <libproc.h>
#include <mach/processor_info.h>
#include <mach/vm_map.h>
*/
import "C"

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"os/user"
	"runtime"
	"strconv"
	"syscall"
	"time"
	"unsafe"
)

func (self *LoadAverage) Get() error {
	avg := []C.double{0, 0, 0}

	C.getloadavg(&avg[0], C.int(len(avg)))

	self.One = float64(avg[0])
	self.Five = float64(avg[1])
	self.Fifteen = float64(avg[2])

	return nil
}

func (self *Uptime) Get() error {
	tv := syscall.Timeval32{}

	if err := sysctlbyname("kern.boottime", &tv); err != nil {
		return err
	}

	self.Length = time.Since(time.Unix(int64(tv.Sec), int64(tv.Usec)*1000)).Seconds()

	return nil
}

func (self *Mem) Get() error {
	var vmstat C.vm_statistics_data_t

	if err := sysctlbyname("hw.memsize", &self.Total); err != nil {
		return err
	}

	if err := vm_info(&vmstat); err != nil {
		return err
	}

	kern := uint64(vmstat.inactive_count) << 12
	self.Free = uint64(vmstat.free_count) << 12

	self.Used = self.Total - self.Free
	self.ActualFree = self.Free + kern
	self.ActualUsed = self.Used - kern

	return nil
}

type xsw_usage struct {
	Total, Avail, Used uint64
}

func (self *Swap) Get() error {
	sw_usage := xsw_usage{}

	if err := sysctlbyname("vm.swapusage", &sw_usage); err != nil {
		return err
	}

	self.Total = sw_usage.Total
	self.Used = sw_usage.Used
	self.Free = sw_usage.Avail

	return nil
}

func (self *Cpu) Get() error {
	var count C.mach_msg_type_number_t = C.HOST_CPU_LOAD_INFO_COUNT
	var cpuload C.host_cpu_load_info_data_t

	status := C.host_statistics(C.host_t(C.mach_host_self()),
		C.HOST_CPU_LOAD_INFO,
		C.host_info_t(unsafe.Pointer(&cpuload)),
		&count)

	if status != C.KERN_SUCCESS {
		return fmt.Errorf("host_statistics error=%d", status)
	}

	self.User = uint64(cpuload.cpu_ticks[C.CPU_STATE_USER])
	self.Sys = uint64(cpuload.cpu_ticks[C.CPU_STATE_SYSTEM])
	self.Idle = uint64(cpuload.cpu_ticks[C.CPU_STATE_IDLE])
	self.Nice = uint64(cpuload.cpu_ticks[C.CPU_STATE_NICE])

	return nil
}

func (self *CpuList) Get() error {
	var count C.mach_msg_type_number_t
	var cpuload *C.processor_cpu_load_info_data_t
	var ncpu C.natural_t

	status := C.host_processor_info(C.host_t(C.mach_host_self()),
		C.PROCESSOR_CPU_LOAD_INFO,
		&ncpu,
		(*C.processor_info_array_t)(unsafe.Pointer(&cpuload)),
		&count)

	if status != C.KERN_SUCCESS {
		return fmt.Errorf("host_processor_info error=%d", status)
	}

	// jump through some cgo casting hoops and ensure we properly free
	// the memory that cpuload points to
	target := C.vm_map_t(C.mach_task_self_)
	address := C.vm_address_t(uintptr(unsafe.Pointer(cpuload)))
	defer C.vm_deallocate(target, address, C.vm_size_t(ncpu))

	// the body of struct processor_cpu_load_info
	// aka processor_cpu_load_info_data_t
	var cpu_ticks [C.CPU_STATE_MAX]uint32

	// copy the cpuload array to a []byte buffer
	// where we can binary.Read the data
	size := int(ncpu) * binary.Size(cpu_ticks)
	buf := C.GoBytes(unsafe.Pointer(cpuload), C.int(size))

	bbuf := bytes.NewBuffer(buf)

	self.List = make([]Cpu, 0, ncpu)

	for i := 0; i < int(ncpu); i++ {
		cpu := Cpu{}

		err := binary.Read(bbuf, binary.LittleEndian, &cpu_ticks)
		if err != nil {
			return err
		}

		cpu.User = uint64(cpu_ticks[C.CPU_STATE_USER])
		cpu.Sys = uint64(cpu_ticks[C.CPU_STATE_SYSTEM])
		cpu.Idle = uint64(cpu_ticks[C.CPU_STATE_IDLE])
		cpu.Nice = uint64(cpu_ticks[C.CPU_STATE_NICE])

		self.List = append(self.List, cpu)
	}

	return nil
}

func (self *FDUsage) Get() error {
	return ErrNotImplemented{runtime.GOOS}
}

func (self *FileSystemList) Get() error {
	num, err := syscall.Getfsstat(nil, C.MNT_NOWAIT)
	if err != nil {
		return err
	}

	buf := make([]syscall.Statfs_t, num)

	_, err = syscall.Getfsstat(buf, C.MNT_NOWAIT)
	if err != nil {
		return err
	}

	fslist := make([]FileSystem, 0, num)

	for i := 0; i < num; i++ {
		fs := FileSystem{}

		fs.DirName = bytePtrToString(&buf[i].Mntonname[0])
		fs.DevName = bytePtrToString(&buf[i].Mntfromname[0])
		fs.SysTypeName = bytePtrToString(&buf[i].Fstypename[0])

		fslist = append(fslist, fs)
	}

	self.List = fslist

	return err
}

func (self *ProcList) Get() error {
	n := C.proc_listpids(C.PROC_ALL_PIDS, 0, nil, 0)
	if n <= 0 {
		return syscall.EINVAL
	}
	buf := make([]byte, n)
	n = C.proc_listpids(C.PROC_ALL_PIDS, 0, unsafe.Pointer(&buf[0]), n)
	if n <= 0 {
		return syscall.ENOMEM
	}

	var pid int32
	num := int(n) / binary.Size(pid)
	list := make([]int, 0, num)
	bbuf := bytes.NewBuffer(buf)

	for i := 0; i < num; i++ {
		if err := binary.Read(bbuf, binary.LittleEndian, &pid); err != nil {
			return err
		}
		if pid == 0 {
			continue
		}

		list = append(list, int(pid))
	}

	self.List = list

	return nil
}

func (self *ProcState) Get(pid int) error {
	info := C.struct_proc_taskallinfo{}

	if err := task_info(pid, &info); err != nil {
		return err
	}

	self.Name = C.GoString(&info.pbsd.pbi_comm[0])

	switch info.pbsd.pbi_status {
	case C.SIDL:
		self.State = RunStateIdle
	case C.SRUN:
		self.State = RunStateRun
	case C.SSLEEP:
		self.State = RunStateSleep
	case C.SSTOP:
		self.State = RunStateStop
	case C.SZOMB:
		self.State = RunStateZombie
	default:
		self.State = RunStateUnknown
	}

	self.Ppid = int(info.pbsd.pbi_ppid)

	self.Pgid = int(info.pbsd.pbi_pgid)

	self.Tty = int(info.pbsd.e_tdev)

	self.Priority = int(info.ptinfo.pti_priority)

	self.Nice = int(info.pbsd.pbi_nice)

	// Get process username. Fallback to UID if username is not available.
	uid := strconv.Itoa(int(info.pbsd.pbi_uid))
	user, err := user.LookupId(uid)
	if err == nil && user.Username != "" {
		self.Username = user.Username
	} else {
		self.Username = uid
	}

	return nil
}

func (self *ProcMem) Get(pid int) error {
	info := C.struct_proc_taskallinfo{}

	if err := task_info(pid, &info); err != nil {
		return err
	}

	self.Size = uint64(info.ptinfo.pti_virtual_size)
	self.Resident = uint64(info.ptinfo.pti_resident_size)
	self.PageFaults = uint64(info.ptinfo.pti_faults)

	return nil
}

func (self *ProcTime) Get(pid int) error {
	info := C.struct_proc_taskallinfo{}

	if err := task_info(pid, &info); err != nil {
		return err
	}

	self.User =
		uint64(info.ptinfo.pti_total_user) / uint64(time.Millisecond)

	self.Sys =
		uint64(info.ptinfo.pti_total_system) / uint64(time.Millisecond)

	self.Total = self.User + self.Sys

	self.StartTime = (uint64(info.pbsd.pbi_start_tvsec) * 1000) +
		(uint64(info.pbsd.pbi_start_tvusec) / 1000)

	return nil
}

func (self *ProcArgs) Get(pid int) error {
	var args []string

	argv := func(arg string) {
		args = append(args, arg)
	}

	err := kern_procargs(pid, nil, argv, nil)

	self.List = args

	return err
}

func (self *ProcExe) Get(pid int) error {
	exe := func(arg string) {
		self.Name = arg
	}

	return kern_procargs(pid, exe, nil, nil)
}

func (self *ProcFDUsage) Get(pid int) error {
	return ErrNotImplemented{runtime.GOOS}
}

// wrapper around sysctl KERN_PROCARGS2
// callbacks params are optional,
// up to the caller as to which pieces of data they want
func kern_procargs(pid int,
	exe func(string),
	argv func(string),
	env func(string, string)) error {

	mib := []C.int{C.CTL_KERN, C.KERN_PROCARGS2, C.int(pid)}
	argmax := uintptr(C.ARG_MAX)
	buf := make([]byte, argmax)
	err := sysctl(mib, &buf[0], &argmax, nil, 0)
	if err != nil {
		return nil
	}

	bbuf := bytes.NewBuffer(buf)
	bbuf.Truncate(int(argmax))

	var argc int32
	binary.Read(bbuf, binary.LittleEndian, &argc)

	path, err := bbuf.ReadBytes(0)
	if err != nil {
		return fmt.Errorf("Error reading the argv[0]: %v", err)
	}
	if exe != nil {
		exe(string(chop(path)))
	}

	// skip trailing \0's
	for {
		c, err := bbuf.ReadByte()
		if err != nil {
			return fmt.Errorf("Error skipping nils: %v", err)
		}
		if c != 0 {
			bbuf.UnreadByte()
			break // start of argv[0]
		}
	}

	for i := 0; i < int(argc); i++ {
		arg, err := bbuf.ReadBytes(0)
		if err == io.EOF {
			break
		}
		if err != nil {
			return fmt.Errorf("Error reading args: %v", err)
		}
		if argv != nil {
			argv(string(chop(arg)))
		}
	}

	if env == nil {
		return nil
	}

	delim := []byte{61} // "="

	for {
		line, err := bbuf.ReadBytes(0)
		if err == io.EOF || line[0] == 0 {
			break
		}
		if err != nil {
			return fmt.Errorf("Error reading args: %v", err)
		}
		pair := bytes.SplitN(chop(line), delim, 2)
		env(string(pair[0]), string(pair[1]))
	}

	return nil
}

// XXX copied from zsyscall_darwin_amd64.go
func sysctl(mib []C.int, old *byte, oldlen *uintptr,
	new *byte, newlen uintptr) (err error) {
	var p0 unsafe.Pointer
	p0 = unsafe.Pointer(&mib[0])
	_, _, e1 := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(p0),
		uintptr(len(mib)),
		uintptr(unsafe.Pointer(old)), uintptr(unsafe.Pointer(oldlen)),
		uintptr(unsafe.Pointer(new)), uintptr(newlen))
	if e1 != 0 {
		err = e1
	}
	return
}

func vm_info(vmstat *C.vm_statistics_data_t) error {
	var count C.mach_msg_type_number_t = C.HOST_VM_INFO_COUNT

	status := C.host_statistics(
		C.host_t(C.mach_host_self()),
		C.HOST_VM_INFO,
		C.host_info_t(unsafe.Pointer(vmstat)),
		&count)

	if status != C.KERN_SUCCESS {
		return fmt.Errorf("host_statistics=%d", status)
	}

	return nil
}

// generic Sysctl buffer unmarshalling
func sysctlbyname(name string, data interface{}) (err error) {
	val, err := syscall.Sysctl(name)
	if err != nil {
		return err
	}

	buf := []byte(val)

	switch v := data.(type) {
	case *uint64:
		*v = *(*uint64)(unsafe.Pointer(&buf[0]))
		return
	}

	bbuf := bytes.NewBuffer([]byte(val))
	return binary.Read(bbuf, binary.LittleEndian, data)
}

func task_info(pid int, info *C.struct_proc_taskallinfo) error {
	size := C.int(unsafe.Sizeof(*info))
	ptr := unsafe.Pointer(info)

	n := C.proc_pidinfo(C.int(pid), C.PROC_PIDTASKALLINFO, 0, ptr, size)
	if n != size {
		return fmt.Errorf("Could not read process info for pid %d", pid)
	}

	return nil
}
<|endoftext|>


<commit_before>package db

import (
	"database/sql"
	"errors"
	"reflect"

	"golang.org/x/net/context"

	_ "github.com/lib/pq" // allow postgres sql connections
	"github.com/rcrowley/go-metrics"
)

var ErrDestinationNotPointer = errors.New("dest is not a pointer")
var ErrDestinationNotSlice = errors.New("dest is not a slice")
var ErrDestinationNil = errors.New("dest is nil")
var ErrDestinationIncompatible = errors.New("Retrieved results' type is not compatible with dest")

type Query interface {
	Get(context.Context) ([]interface{}, error)
	IsComplete(context.Context, int) bool
}

type Pageable interface {
	PagingToken() string
}

type Record interface{}

// Open the postgres database at the provided url and performing an initial
// ping to ensure we can connect to it.
func Open(url string) (*sql.DB, error) {

	db, err := sql.Open("postgres", url)

	if err != nil {
		return db, err
	}

	err = db.Ping()

	if err != nil {
		return db, err
	}

	return db, nil
}

// Results runs the provided query, returning all found results
func Results(ctx context.Context, query Query) ([]interface{}, error) {
	return query.Get(ctx)
}

// Select runs the provided query, appending all results into dest.  Dest must
// be a pointer to a slice of a type compatible with the records returned from
// the query.
//
// NOTE:  At present this method is much more expensive than it should be
// because it does a lot of casting and reflection throughout its call graph (
// any given record ends up going from Record to interface{} and back again at
// least once, unnecessarily). this current implementation is a stopgap on the
// way to removing the Results() and First() functions, which return interface{}
// values.  We cannot yet remove those due to how intertwined they are with the
// SSE system.
func Select(ctx context.Context, query Query, dest interface{}) error {
	records, err := Results(ctx, query)

	if err != nil {
		return err
	}

	if dest == nil {
		return ErrDestinationNil
	}

	dvp := reflect.ValueOf(dest)

	if dvp.Kind() != reflect.Ptr {
		return ErrDestinationNotPointer
	}

	dv := reflect.Indirect(dvp)

	if dv.Kind() != reflect.Slice {
		return ErrDestinationNotSlice
	}

	rvp := reflect.New(dv.Type())
	rv := reflect.Indirect(rvp)
	slicet := dv.Type().Elem()

	for _, record := range records {
		recordv := reflect.ValueOf(record)

		if !recordv.Type().AssignableTo(slicet) {
			return ErrDestinationIncompatible
		}

		rv = reflect.Append(rv, recordv)
	}

	dv.Set(rv)
	return nil
}

// First runs the provided query, returning the first result if found,
// otherwise nil
func First(ctx context.Context, query Query) (interface{}, error) {
	res, err := query.Get(ctx)

	switch {
	case err != nil:
		return nil, err
	case len(res) == 0:
		return nil, nil
	default:
		return res[0], nil
	}
}

func MustFirst(ctx context.Context, q Query) interface{} {
	result, err := First(ctx, q)

	if err != nil {
		panic(err)
	}

	return result
}

func MustResults(ctx context.Context, q Query) []interface{} {
	result, err := Results(ctx, q)

	if err != nil {
		panic(err)
	}

	return result
}

func QueryGauge() metrics.Gauge {
	return globalStreamManager.queryGauge
}

// helper method suited to confirm query validity.  checkOptions ensures
// that zero or one of the provided bools ares true, but will return an error
// if more than one clause is true.
func checkOptions(clauses ...bool) error {
	hasOneSet := false

	for _, isSet := range clauses {
		if !isSet {
			continue
		}

		if hasOneSet {
			return errors.New("Invalid options: multiple are set")
		}

		hasOneSet = true
	}

	return nil
}

// Converts a typed slice to a slice of interface{}, suitable
// for return through the Get() method of Query
func makeResult(src interface{}) []interface{} {
	srcValue := reflect.ValueOf(src)
	srcLen := srcValue.Len()
	result := make([]interface{}, srcLen)

	for i := 0; i < srcLen; i++ {
		result[i] = srcValue.Index(i).Interface()
	}
	return result
}
<commit_msg>Properly order error checks on db.Select() to avoid unnecessary queries<commit_after>package db

import (
	"database/sql"
	"errors"
	"reflect"

	"golang.org/x/net/context"

	_ "github.com/lib/pq" // allow postgres sql connections
	"github.com/rcrowley/go-metrics"
)

var ErrDestinationNotPointer = errors.New("dest is not a pointer")
var ErrDestinationNotSlice = errors.New("dest is not a slice")
var ErrDestinationNil = errors.New("dest is nil")
var ErrDestinationIncompatible = errors.New("Retrieved results' type is not compatible with dest")

type Query interface {
	Get(context.Context) ([]interface{}, error)
	IsComplete(context.Context, int) bool
}

type Pageable interface {
	PagingToken() string
}

type Record interface{}

// Open the postgres database at the provided url and performing an initial
// ping to ensure we can connect to it.
func Open(url string) (*sql.DB, error) {

	db, err := sql.Open("postgres", url)

	if err != nil {
		return db, err
	}

	err = db.Ping()

	if err != nil {
		return db, err
	}

	return db, nil
}

// Results runs the provided query, returning all found results
func Results(ctx context.Context, query Query) ([]interface{}, error) {
	return query.Get(ctx)
}

// Select runs the provided query, appending all results into dest.  Dest must
// be a pointer to a slice of a type compatible with the records returned from
// the query.
//
// NOTE:  At present this method is much more expensive than it should be
// because it does a lot of casting and reflection throughout its call graph (
// any given record ends up going from Record to interface{} and back again at
// least once, unnecessarily). this current implementation is a stopgap on the
// way to removing the Results() and First() functions, which return interface{}
// values.  We cannot yet remove those due to how intertwined they are with the
// SSE system.
func Select(ctx context.Context, query Query, dest interface{}) error {

	if dest == nil {
		return ErrDestinationNil
	}

	// run the query
	records, err := Results(ctx, query)
	if err != nil {
		return err
	}

	// validate destination
	dvp := reflect.ValueOf(dest)
	if dvp.Kind() != reflect.Ptr {
		return ErrDestinationNotPointer
	}

	dv := reflect.Indirect(dvp)
	if dv.Kind() != reflect.Slice {
		return ErrDestinationNotSlice
	}

	// create new slice of correct type to
	// populate with results
	rvp := reflect.New(dv.Type())
	rv := reflect.Indirect(rvp)

	slicet := dv.Type().Elem()

	for _, record := range records {
		recordv := reflect.ValueOf(record)

		if !recordv.Type().AssignableTo(slicet) {
			return ErrDestinationIncompatible
		}

		rv = reflect.Append(rv, recordv)
	}

	dv.Set(rv)
	return nil
}

// First runs the provided query, returning the first result if found,
// otherwise nil
func First(ctx context.Context, query Query) (interface{}, error) {
	res, err := query.Get(ctx)

	switch {
	case err != nil:
		return nil, err
	case len(res) == 0:
		return nil, nil
	default:
		return res[0], nil
	}
}

func MustFirst(ctx context.Context, q Query) interface{} {
	result, err := First(ctx, q)

	if err != nil {
		panic(err)
	}

	return result
}

func MustResults(ctx context.Context, q Query) []interface{} {
	result, err := Results(ctx, q)

	if err != nil {
		panic(err)
	}

	return result
}

func QueryGauge() metrics.Gauge {
	return globalStreamManager.queryGauge
}

// helper method suited to confirm query validity.  checkOptions ensures
// that zero or one of the provided bools ares true, but will return an error
// if more than one clause is true.
func checkOptions(clauses ...bool) error {
	hasOneSet := false

	for _, isSet := range clauses {
		if !isSet {
			continue
		}

		if hasOneSet {
			return errors.New("Invalid options: multiple are set")
		}

		hasOneSet = true
	}

	return nil
}

// Converts a typed slice to a slice of interface{}, suitable
// for return through the Get() method of Query
func makeResult(src interface{}) []interface{} {
	srcValue := reflect.ValueOf(src)
	srcLen := srcValue.Len()
	result := make([]interface{}, srcLen)

	for i := 0; i < srcLen; i++ {
		result[i] = srcValue.Index(i).Interface()
	}
	return result
}
<|endoftext|>


<commit_before>package client

import (
	"fmt"
	"io/ioutil"
	"os"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/protocol/go"
	"github.com/maxtaco/go-framed-msgpack-rpc/rpc2"
)

// CmdProve is the wrapper structure for the the `keybase prove` operation.
type CmdProve struct {
	arg    keybase1.StartProofArg
	output string
}

// ParseArgv parses arguments for the prove command.
func (p *CmdProve) ParseArgv(ctx *cli.Context) error {
	nargs := len(ctx.Args())
	var err error
	p.arg.Force = ctx.Bool("force")
	p.output = ctx.String("output")

	if nargs > 2 || nargs == 0 {
		err = fmt.Errorf("prove takes 1 or args: <service> [<username>]")
	} else {
		p.arg.Service = ctx.Args()[0]
		if nargs == 2 {
			p.arg.Username = ctx.Args()[1]
		}
	}
	return err
}

func (p *CmdProve) fileOutputHook(txt string) (err error) {
	G.Log.Info("Writing proof to file '" + p.output + "'...")
	err = ioutil.WriteFile(p.output, []byte(txt), os.FileMode(0644))
	G.Log.Info("Written.")
	return
}

func newProveUIProtocol(ui ProveUI) rpc2.Protocol {
	return keybase1.ProveUiProtocol(ui)
}

// RunClient runs the `keybase prove` subcommand in client/server mode.
func (p *CmdProve) Run() error {
	var cli keybase1.ProveClient

	proveUI := ProveUI{parent: GlobUI}
	p.installOutputHook(&proveUI)

	protocols := []rpc2.Protocol{
		newProveUIProtocol(proveUI),
		NewLoginUIProtocol(),
		NewSecretUIProtocol(),
		NewLogUIProtocol(),
	}

	cli, err := GetProveClient()
	if err != nil {
		return err
	}
	if err = RegisterProtocols(protocols); err != nil {
		return err
	}

	// command line interface wants the PromptPosted ui loop
	p.arg.PromptPosted = true

	_, err = cli.StartProof(p.arg)
	return err
}

func (p *CmdProve) installOutputHook(ui *ProveUI) {
	if len(p.output) > 0 {
		ui.outputHook = func(s string) error {
			return p.fileOutputHook(s)
		}
	}
}

// NewCmdProve makes a new prove command from the given CLI parameters.
func NewCmdProve(cl *libcmdline.CommandLine) cli.Command {
	return cli.Command{
		Name:         "prove",
		ArgumentHelp: "<service> [username]",
		Usage:        "Generate a new proof",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "output, o",
				Usage: "Output proof text to a file (rather than standard out).",
			},
			cli.BoolFlag{
				Name:  "force, f",
				Usage: "Don't prompt.",
			},
		},
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdProve{}, "prove", c)
		},
	}
}

// GetUsage specifics the library features that the prove command needs.
func (p *CmdProve) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config:    true,
		API:       true,
		KbKeyring: true,
	}
}
<commit_msg>Clarify prove argument help<commit_after>package client

import (
	"fmt"
	"io/ioutil"
	"os"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/protocol/go"
	"github.com/maxtaco/go-framed-msgpack-rpc/rpc2"
)

// CmdProve is the wrapper structure for the the `keybase prove` operation.
type CmdProve struct {
	arg    keybase1.StartProofArg
	output string
}

// ParseArgv parses arguments for the prove command.
func (p *CmdProve) ParseArgv(ctx *cli.Context) error {
	nargs := len(ctx.Args())
	var err error
	p.arg.Force = ctx.Bool("force")
	p.output = ctx.String("output")

	if nargs > 2 || nargs == 0 {
		err = fmt.Errorf("prove takes 1 or args: <service> [<username>]")
	} else {
		p.arg.Service = ctx.Args()[0]
		if nargs == 2 {
			p.arg.Username = ctx.Args()[1]
		}
	}
	return err
}

func (p *CmdProve) fileOutputHook(txt string) (err error) {
	G.Log.Info("Writing proof to file '" + p.output + "'...")
	err = ioutil.WriteFile(p.output, []byte(txt), os.FileMode(0644))
	G.Log.Info("Written.")
	return
}

func newProveUIProtocol(ui ProveUI) rpc2.Protocol {
	return keybase1.ProveUiProtocol(ui)
}

// RunClient runs the `keybase prove` subcommand in client/server mode.
func (p *CmdProve) Run() error {
	var cli keybase1.ProveClient

	proveUI := ProveUI{parent: GlobUI}
	p.installOutputHook(&proveUI)

	protocols := []rpc2.Protocol{
		newProveUIProtocol(proveUI),
		NewLoginUIProtocol(),
		NewSecretUIProtocol(),
		NewLogUIProtocol(),
	}

	cli, err := GetProveClient()
	if err != nil {
		return err
	}
	if err = RegisterProtocols(protocols); err != nil {
		return err
	}

	// command line interface wants the PromptPosted ui loop
	p.arg.PromptPosted = true

	_, err = cli.StartProof(p.arg)
	return err
}

func (p *CmdProve) installOutputHook(ui *ProveUI) {
	if len(p.output) > 0 {
		ui.outputHook = func(s string) error {
			return p.fileOutputHook(s)
		}
	}
}

// NewCmdProve makes a new prove command from the given CLI parameters.
func NewCmdProve(cl *libcmdline.CommandLine) cli.Command {
	return cli.Command{
		Name:         "prove",
		ArgumentHelp: "<service> [service username]",
		Usage:        "Generate a new proof",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "output, o",
				Usage: "Output proof text to a file (rather than standard out).",
			},
			cli.BoolFlag{
				Name:  "force, f",
				Usage: "Don't prompt.",
			},
		},
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdProve{}, "prove", c)
		},
	}
}

// GetUsage specifics the library features that the prove command needs.
func (p *CmdProve) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config:    true,
		API:       true,
		KbKeyring: true,
	}
}
<|endoftext|>


<commit_before>// Package client is a CT log client implementation and contains types and code
// for interacting with RFC6962-compliant CT Log instances.
// See http://tools.ietf.org/html/rfc6962 for details
package client

import (
	"bytes"
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"time"

	ct "github.com/google/certificate-transparency/go"
	"golang.org/x/net/context"
)

// URI paths for CT Log endpoints
const (
	AddChainPath    = "/ct/v1/add-chain"
	AddPreChainPath = "/ct/v1/add-pre-chain"
	AddJSONPath     = "/ct/v1/add-json"
	GetSTHPath      = "/ct/v1/get-sth"
	GetEntriesPath  = "/ct/v1/get-entries"
)

// LogClient represents a client for a given CT Log instance
type LogClient struct {
	uri        string       // the base URI of the log. e.g. http://ct.googleapis/pilot
	httpClient *http.Client // used to interact with the log via HTTP
}

//////////////////////////////////////////////////////////////////////////////////
// JSON structures follow.
// These represent the structures returned by the CT Log server.
//////////////////////////////////////////////////////////////////////////////////

// addChainRequest represents the JSON request body sent to the add-chain CT
// method.
type addChainRequest struct {
	Chain [][]byte `json:"chain"`
}

// addChainResponse represents the JSON response to the add-chain CT method.
// An SCT represents a Log's promise to integrate a [pre-]certificate into the
// log within a defined period of time.
type addChainResponse struct {
	SCTVersion ct.Version `json:"sct_version"` // SCT structure version
	ID         []byte     `json:"id"`          // Log ID
	Timestamp  uint64     `json:"timestamp"`   // Timestamp of issuance
	Extensions string     `json:"extensions"`  // Holder for any CT extensions
	Signature  []byte     `json:"signature"`   // Log signature for this SCT
}

// addJSONRequest represents the JSON request body sent ot the add-json CT
// method.
type addJSONRequest struct {
	Data interface{} `json:"data"`
}

// getSTHResponse respresents the JSON response to the get-sth CT method
type getSTHResponse struct {
	TreeSize          uint64 `json:"tree_size"`           // Number of certs in the current tree
	Timestamp         uint64 `json:"timestamp"`           // Time that the tree was created
	SHA256RootHash    []byte `json:"sha256_root_hash"`    // Root hash of the tree
	TreeHeadSignature []byte `json:"tree_head_signature"` // Log signature for this STH
}

// getConsistencyProofResponse represents the JSON response to the CT get-consistency-proof method
type getConsistencyProofResponse struct {
	Consistency []string `json:"consistency"`
}

// getAuditProofResponse represents the JSON response to the CT get-audit-proof method
type getAuditProofResponse struct {
	Hash     []string `json:"hash"`      // the hashes which make up the proof
	TreeSize uint64   `json:"tree_size"` // the tree size against which this proof is constructed
}

// getAcceptedRootsResponse represents the JSON response to the CT get-roots method.
type getAcceptedRootsResponse struct {
	Certificates []string `json:"certificates"`
}

// getEntryAndProodReponse represents the JSON response to the CT get-entry-and-proof method
type getEntryAndProofResponse struct {
	LeafInput string   `json:"leaf_input"` // the entry itself
	ExtraData string   `json:"extra_data"` // any chain provided when the entry was added to the log
	AuditPath []string `json:"audit_path"` // the corresponding proof
}

// New constructs a new LogClient instance.
// |uri| is the base URI of the CT log instance to interact with, e.g.
// http://ct.googleapis.com/pilot
// |hc| is the underlying client to be used for HTTP requests to the CT log.
func New(uri string, hc *http.Client) *LogClient {
	if hc == nil {
		hc = new(http.Client)
	}
	return &LogClient{uri: uri, httpClient: hc}
}

// Makes a HTTP call to |uri|, and attempts to parse the response as a
// JSON representation of the structure in |res|. Uses |ctx| to
// control the HTTP call (so it can have a timeout or be cancelled by
// the caller), and |httpClient| to make the actual HTTP call.
// Returns a non-nil |error| if there was a problem.
func fetchAndParse(ctx context.Context, httpClient *http.Client, uri string, res interface{}) error {
	req, err := http.NewRequest(http.MethodGet, uri, nil)
	if err != nil {
		return err
	}
	req.Cancel = ctx.Done()
	resp, err := httpClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	// Make sure everything is read, so http.Client can reuse the connection.
	defer ioutil.ReadAll(resp.Body)

	if resp.StatusCode != 200 {
		return fmt.Errorf("got HTTP Status %s", resp.Status)
	}

	if err := json.NewDecoder(resp.Body).Decode(res); err != nil {
		return err
	}

	return nil
}

// Makes a HTTP POST call to |uri|, and attempts to parse the response as a JSON
// representation of the structure in |res|.
// Returns a non-nil |error| if there was a problem.
func (c *LogClient) postAndParse(uri string, req interface{}, res interface{}) (*http.Response, string, error) {
	postBody, err := json.Marshal(req)
	if err != nil {
		return nil, "", err
	}
	httpReq, err := http.NewRequest(http.MethodPost, uri, bytes.NewReader(postBody))
	if err != nil {
		return nil, "", err
	}
	httpReq.Header.Set("Content-Type", "application/json")
	resp, err := c.httpClient.Do(httpReq)
	// Read all of the body, if there is one, so that the http.Client can do
	// Keep-Alive:
	var body []byte
	if resp != nil {
		body, err = ioutil.ReadAll(resp.Body)
		resp.Body.Close()
	}
	if err != nil {
		return resp, string(body), err
	}
	if resp.StatusCode == 200 {
		if err != nil {
			return resp, string(body), err
		}
		if err = json.Unmarshal(body, &res); err != nil {
			return resp, string(body), err
		}
	}
	return resp, string(body), nil
}

func backoffForRetry(ctx context.Context, d time.Duration) error {
	backoffTimer := time.NewTimer(d)
	if ctx != nil {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-backoffTimer.C:
		}
	} else {
		<-backoffTimer.C
	}
	return nil
}

// Attempts to add |chain| to the log, using the api end-point specified by
// |path|. If provided context expires before submission is complete an
// error will be returned.
func (c *LogClient) addChainWithRetry(ctx context.Context, path string, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	var resp addChainResponse
	var req addChainRequest
	for _, link := range chain {
		req.Chain = append(req.Chain, link)
	}
	httpStatus := "Unknown"
	backoffSeconds := 0
	done := false
	for !done {
		if backoffSeconds > 0 {
			log.Printf("Got %s, backing-off %d seconds", httpStatus, backoffSeconds)
		}
		err := backoffForRetry(ctx, time.Second*time.Duration(backoffSeconds))
		if err != nil {
			return nil, err
		}
		if backoffSeconds > 0 {
			backoffSeconds = 0
		}
		httpResp, errorBody, err := c.postAndParse(c.uri+path, &req, &resp)
		if err != nil {
			backoffSeconds = 10
			continue
		}
		switch {
		case httpResp.StatusCode == 200:
			done = true
		case httpResp.StatusCode == 408:
			// request timeout, retry immediately
		case httpResp.StatusCode == 503:
			// Retry
			backoffSeconds = 10
			if retryAfter := httpResp.Header.Get("Retry-After"); retryAfter != "" {
				if seconds, err := strconv.Atoi(retryAfter); err == nil {
					backoffSeconds = seconds
				}
			}
		default:
			return nil, fmt.Errorf("got HTTP Status %s: %s", httpResp.Status, errorBody)
		}
		httpStatus = httpResp.Status
	}

	ds, err := ct.UnmarshalDigitallySigned(bytes.NewReader(resp.Signature))
	if err != nil {
		return nil, err
	}
	var logID ct.SHA256Hash
	copy(logID[:], resp.ID)
	return &ct.SignedCertificateTimestamp{
		SCTVersion: resp.SCTVersion,
		LogID:      logID,
		Timestamp:  resp.Timestamp,
		Extensions: ct.CTExtensions(resp.Extensions),
		Signature:  *ds}, nil
}

// AddChain adds the (DER represented) X509 |chain| to the log.
func (c *LogClient) AddChain(chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	return c.addChainWithRetry(nil, AddChainPath, chain)
}

// AddPreChain adds the (DER represented) Precertificate |chain| to the log.
func (c *LogClient) AddPreChain(chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	return c.addChainWithRetry(nil, AddPreChainPath, chain)
}

// AddChainWithContext adds the (DER represented) X509 |chain| to the log and
// fails if the provided context expires before the chain is submitted.
func (c *LogClient) AddChainWithContext(ctx context.Context, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	return c.addChainWithRetry(ctx, AddChainPath, chain)
}

func (c *LogClient) AddJSON(data interface{}) (*ct.SignedCertificateTimestamp, error) {
	req := addJSONRequest{
		Data: data,
	}
	var resp addChainResponse
	_, _, err := c.postAndParse(c.uri+AddJSONPath, &req, &resp)
	if err != nil {
		return nil, err
	}
	ds, err := ct.UnmarshalDigitallySigned(bytes.NewReader(resp.Signature))
	if err != nil {
		return nil, err
	}
	var logID ct.SHA256Hash
	copy(logID[:], resp.ID)
	return &ct.SignedCertificateTimestamp{
		SCTVersion: resp.SCTVersion,
		LogID:      logID,
		Timestamp:  resp.Timestamp,
		Extensions: ct.CTExtensions(resp.Extensions),
		Signature:  *ds}, nil
}

// GetSTH retrieves the current STH from the log.
// Returns a populated SignedTreeHead, or a non-nil error.
func (c *LogClient) GetSTH() (sth *ct.SignedTreeHead, err error) {
	var resp getSTHResponse
	if err = fetchAndParse(context.TODO(), c.httpClient, c.uri+GetSTHPath, &resp); err != nil {
		return
	}
	sth = &ct.SignedTreeHead{
		TreeSize:  resp.TreeSize,
		Timestamp: resp.Timestamp,
	}

	if len(resp.SHA256RootHash) != sha256.Size {
		return nil, fmt.Errorf("sha256_root_hash is invalid length, expected %d got %d", sha256.Size, len(resp.SHA256RootHash))
	}
	copy(sth.SHA256RootHash[:], resp.SHA256RootHash)

	ds, err := ct.UnmarshalDigitallySigned(bytes.NewReader(resp.TreeHeadSignature))
	if err != nil {
		return nil, err
	}
	// TODO(alcutter): Verify signature
	sth.TreeHeadSignature = *ds
	return
}
<commit_msg>Remove HTTP body from error messages (#1275)<commit_after>// Package client is a CT log client implementation and contains types and code
// for interacting with RFC6962-compliant CT Log instances.
// See http://tools.ietf.org/html/rfc6962 for details
package client

import (
	"bytes"
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"time"

	ct "github.com/google/certificate-transparency/go"
	"golang.org/x/net/context"
)

// URI paths for CT Log endpoints
const (
	AddChainPath    = "/ct/v1/add-chain"
	AddPreChainPath = "/ct/v1/add-pre-chain"
	AddJSONPath     = "/ct/v1/add-json"
	GetSTHPath      = "/ct/v1/get-sth"
	GetEntriesPath  = "/ct/v1/get-entries"
)

// LogClient represents a client for a given CT Log instance
type LogClient struct {
	uri        string       // the base URI of the log. e.g. http://ct.googleapis/pilot
	httpClient *http.Client // used to interact with the log via HTTP
}

//////////////////////////////////////////////////////////////////////////////////
// JSON structures follow.
// These represent the structures returned by the CT Log server.
//////////////////////////////////////////////////////////////////////////////////

// addChainRequest represents the JSON request body sent to the add-chain CT
// method.
type addChainRequest struct {
	Chain [][]byte `json:"chain"`
}

// addChainResponse represents the JSON response to the add-chain CT method.
// An SCT represents a Log's promise to integrate a [pre-]certificate into the
// log within a defined period of time.
type addChainResponse struct {
	SCTVersion ct.Version `json:"sct_version"` // SCT structure version
	ID         []byte     `json:"id"`          // Log ID
	Timestamp  uint64     `json:"timestamp"`   // Timestamp of issuance
	Extensions string     `json:"extensions"`  // Holder for any CT extensions
	Signature  []byte     `json:"signature"`   // Log signature for this SCT
}

// addJSONRequest represents the JSON request body sent ot the add-json CT
// method.
type addJSONRequest struct {
	Data interface{} `json:"data"`
}

// getSTHResponse respresents the JSON response to the get-sth CT method
type getSTHResponse struct {
	TreeSize          uint64 `json:"tree_size"`           // Number of certs in the current tree
	Timestamp         uint64 `json:"timestamp"`           // Time that the tree was created
	SHA256RootHash    []byte `json:"sha256_root_hash"`    // Root hash of the tree
	TreeHeadSignature []byte `json:"tree_head_signature"` // Log signature for this STH
}

// getConsistencyProofResponse represents the JSON response to the CT get-consistency-proof method
type getConsistencyProofResponse struct {
	Consistency []string `json:"consistency"`
}

// getAuditProofResponse represents the JSON response to the CT get-audit-proof method
type getAuditProofResponse struct {
	Hash     []string `json:"hash"`      // the hashes which make up the proof
	TreeSize uint64   `json:"tree_size"` // the tree size against which this proof is constructed
}

// getAcceptedRootsResponse represents the JSON response to the CT get-roots method.
type getAcceptedRootsResponse struct {
	Certificates []string `json:"certificates"`
}

// getEntryAndProodReponse represents the JSON response to the CT get-entry-and-proof method
type getEntryAndProofResponse struct {
	LeafInput string   `json:"leaf_input"` // the entry itself
	ExtraData string   `json:"extra_data"` // any chain provided when the entry was added to the log
	AuditPath []string `json:"audit_path"` // the corresponding proof
}

// New constructs a new LogClient instance.
// |uri| is the base URI of the CT log instance to interact with, e.g.
// http://ct.googleapis.com/pilot
// |hc| is the underlying client to be used for HTTP requests to the CT log.
func New(uri string, hc *http.Client) *LogClient {
	if hc == nil {
		hc = new(http.Client)
	}
	return &LogClient{uri: uri, httpClient: hc}
}

// Makes a HTTP call to |uri|, and attempts to parse the response as a
// JSON representation of the structure in |res|. Uses |ctx| to
// control the HTTP call (so it can have a timeout or be cancelled by
// the caller), and |httpClient| to make the actual HTTP call.
// Returns a non-nil |error| if there was a problem.
func fetchAndParse(ctx context.Context, httpClient *http.Client, uri string, res interface{}) error {
	req, err := http.NewRequest(http.MethodGet, uri, nil)
	if err != nil {
		return err
	}
	req.Cancel = ctx.Done()
	resp, err := httpClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	// Make sure everything is read, so http.Client can reuse the connection.
	defer ioutil.ReadAll(resp.Body)

	if resp.StatusCode != 200 {
		return fmt.Errorf("got HTTP Status %s", resp.Status)
	}

	if err := json.NewDecoder(resp.Body).Decode(res); err != nil {
		return err
	}

	return nil
}

// Makes a HTTP POST call to |uri|, and attempts to parse the response as a JSON
// representation of the structure in |res|.
// Returns a non-nil |error| if there was a problem.
func (c *LogClient) postAndParse(uri string, req interface{}, res interface{}) (*http.Response, string, error) {
	postBody, err := json.Marshal(req)
	if err != nil {
		return nil, "", err
	}
	httpReq, err := http.NewRequest(http.MethodPost, uri, bytes.NewReader(postBody))
	if err != nil {
		return nil, "", err
	}
	httpReq.Header.Set("Content-Type", "application/json")
	resp, err := c.httpClient.Do(httpReq)
	// Read all of the body, if there is one, so that the http.Client can do
	// Keep-Alive:
	var body []byte
	if resp != nil {
		body, err = ioutil.ReadAll(resp.Body)
		resp.Body.Close()
	}
	if err != nil {
		return resp, string(body), err
	}
	if resp.StatusCode == 200 {
		if err != nil {
			return resp, string(body), err
		}
		if err = json.Unmarshal(body, &res); err != nil {
			return resp, string(body), err
		}
	}
	return resp, string(body), nil
}

func backoffForRetry(ctx context.Context, d time.Duration) error {
	backoffTimer := time.NewTimer(d)
	if ctx != nil {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-backoffTimer.C:
		}
	} else {
		<-backoffTimer.C
	}
	return nil
}

// Attempts to add |chain| to the log, using the api end-point specified by
// |path|. If provided context expires before submission is complete an
// error will be returned.
func (c *LogClient) addChainWithRetry(ctx context.Context, path string, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	var resp addChainResponse
	var req addChainRequest
	for _, link := range chain {
		req.Chain = append(req.Chain, link)
	}
	httpStatus := "Unknown"
	backoffSeconds := 0
	done := false
	for !done {
		if backoffSeconds > 0 {
			log.Printf("Got %s, backing-off %d seconds", httpStatus, backoffSeconds)
		}
		err := backoffForRetry(ctx, time.Second*time.Duration(backoffSeconds))
		if err != nil {
			return nil, err
		}
		if backoffSeconds > 0 {
			backoffSeconds = 0
		}
		httpResp, _, err := c.postAndParse(c.uri+path, &req, &resp)
		if err != nil {
			backoffSeconds = 10
			continue
		}
		switch {
		case httpResp.StatusCode == 200:
			done = true
		case httpResp.StatusCode == 408:
			// request timeout, retry immediately
		case httpResp.StatusCode == 503:
			// Retry
			backoffSeconds = 10
			if retryAfter := httpResp.Header.Get("Retry-After"); retryAfter != "" {
				if seconds, err := strconv.Atoi(retryAfter); err == nil {
					backoffSeconds = seconds
				}
			}
		default:
			return nil, fmt.Errorf("got HTTP Status %s", httpResp.Status)
		}
		httpStatus = httpResp.Status
	}

	ds, err := ct.UnmarshalDigitallySigned(bytes.NewReader(resp.Signature))
	if err != nil {
		return nil, err
	}
	var logID ct.SHA256Hash
	copy(logID[:], resp.ID)
	return &ct.SignedCertificateTimestamp{
		SCTVersion: resp.SCTVersion,
		LogID:      logID,
		Timestamp:  resp.Timestamp,
		Extensions: ct.CTExtensions(resp.Extensions),
		Signature:  *ds}, nil
}

// AddChain adds the (DER represented) X509 |chain| to the log.
func (c *LogClient) AddChain(chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	return c.addChainWithRetry(nil, AddChainPath, chain)
}

// AddPreChain adds the (DER represented) Precertificate |chain| to the log.
func (c *LogClient) AddPreChain(chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	return c.addChainWithRetry(nil, AddPreChainPath, chain)
}

// AddChainWithContext adds the (DER represented) X509 |chain| to the log and
// fails if the provided context expires before the chain is submitted.
func (c *LogClient) AddChainWithContext(ctx context.Context, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error) {
	return c.addChainWithRetry(ctx, AddChainPath, chain)
}

func (c *LogClient) AddJSON(data interface{}) (*ct.SignedCertificateTimestamp, error) {
	req := addJSONRequest{
		Data: data,
	}
	var resp addChainResponse
	_, _, err := c.postAndParse(c.uri+AddJSONPath, &req, &resp)
	if err != nil {
		return nil, err
	}
	ds, err := ct.UnmarshalDigitallySigned(bytes.NewReader(resp.Signature))
	if err != nil {
		return nil, err
	}
	var logID ct.SHA256Hash
	copy(logID[:], resp.ID)
	return &ct.SignedCertificateTimestamp{
		SCTVersion: resp.SCTVersion,
		LogID:      logID,
		Timestamp:  resp.Timestamp,
		Extensions: ct.CTExtensions(resp.Extensions),
		Signature:  *ds}, nil
}

// GetSTH retrieves the current STH from the log.
// Returns a populated SignedTreeHead, or a non-nil error.
func (c *LogClient) GetSTH() (sth *ct.SignedTreeHead, err error) {
	var resp getSTHResponse
	if err = fetchAndParse(context.TODO(), c.httpClient, c.uri+GetSTHPath, &resp); err != nil {
		return
	}
	sth = &ct.SignedTreeHead{
		TreeSize:  resp.TreeSize,
		Timestamp: resp.Timestamp,
	}

	if len(resp.SHA256RootHash) != sha256.Size {
		return nil, fmt.Errorf("sha256_root_hash is invalid length, expected %d got %d", sha256.Size, len(resp.SHA256RootHash))
	}
	copy(sth.SHA256RootHash[:], resp.SHA256RootHash)

	ds, err := ct.UnmarshalDigitallySigned(bytes.NewReader(resp.TreeHeadSignature))
	if err != nil {
		return nil, err
	}
	// TODO(alcutter): Verify signature
	sth.TreeHeadSignature = *ds
	return
}
<|endoftext|>


<commit_before>package google

import (
	"context"
	"fmt"
	"log"
	"sync"
	"time"

	"github.com/hashicorp/errwrap"
)

const defaultBatchSendIntervalSec = 3

// RequestBatcher is a global batcher object that keeps track of
// existing batches.
// In general, a batcher should be created per service that requires batching
// in order to prevent blocking batching for one service due to another,
// and to minimize the possibility of overlap in batchKey formats
// (see SendRequestWithTimeout)
type RequestBatcher struct {
	sync.Mutex

	*batchingConfig
	parentCtx context.Context
	batches   map[string]*startedBatch
	debugId   string
}

// BatchRequest represents a single request to a global batcher.
type BatchRequest struct {
	// ResourceName represents the underlying resource for which
	// a request is made. Its format is determined by what SendF expects, but
	// typically should be the name of the parent GCP resource being changed.
	ResourceName string

	// Body is this request's data to be passed to SendF, and may be combined
	// with other bodies using CombineF.
	Body interface{}

	// CombineF function determines how to combine bodies from two batches.
	CombineF batcherCombineFunc

	// SendF function determines how to actually send a batched request to a
	// third party service. The arguments given to this function are
	// (ResourceName, Body) where Body may have been combined with other request
	// Bodies.
	SendF batcherSendFunc

	// ID for debugging request. This should be specific to a single request
	// (i.e. per Terraform resource)
	DebugId string
}

// These types are meant to be the public interface to batchers. They define
// logic to manage batch data type and behavior, and require service-specific
// implementations per type of request per service.
// Function type for combine existing batches and additional batch data
type batcherCombineFunc func(body interface{}, toAdd interface{}) (interface{}, error)

// Function type for sending a batch request
type batcherSendFunc func(resourceName string, body interface{}) (interface{}, error)

// batchResponse bundles an API response (data, error) tuple.
type batchResponse struct {
	body interface{}
	err  error
}

// startedBatch refers to a processed batch whose timer to send the request has
// already been started. The responses for the request is sent to each listener
// channel, representing parallel callers that are waiting on requests
// combined into this batch.
type startedBatch struct {
	batchKey string
	*BatchRequest

	listeners []chan batchResponse
	timer     *time.Timer
}

// batchingConfig contains user configuration for controlling batch requests.
type batchingConfig struct {
	sendAfter      time.Duration
	enableBatching bool
}

// Initializes a new batcher.
func NewRequestBatcher(debugId string, ctx context.Context, config *batchingConfig) *RequestBatcher {
	batcher := &RequestBatcher{
		debugId:        debugId,
		parentCtx:      ctx,
		batchingConfig: config,
		batches:        make(map[string]*startedBatch),
	}

	go func(b *RequestBatcher) {
		<-ctx.Done()
		b.stop()
	}(batcher)

	return batcher
}

func (b *RequestBatcher) stop() {
	b.Lock()
	defer b.Unlock()

	log.Printf("[DEBUG] Stopping batcher %q", b.debugId)
	for batchKey, batch := range b.batches {
		log.Printf("[DEBUG] Cleaning up batch request %q", batchKey)
		batch.timer.Stop()
		for _, l := range batch.listeners {
			close(l)
		}
	}
}

// SendRequestWithTimeout is expected to be called per parallel call.
// It manages waiting on the result of a batch request.
//
// Batch requests are grouped by the given batchKey. batchKey
// should be unique to the API request being sent, most likely similar to
// the HTTP request URL with GCP resource ID included in the URL (the caller
// may choose to use a key with method if needed to diff GET/read and
// POST/create)
//
// As an example, for google_project_service, the
// batcher is called to batch services.batchEnable() calls for a project
// $PROJECT. The calling code uses the template
// "serviceusage:projects/$PROJECT/services:batchEnable", which mirrors the HTTP request:
// POST https://serviceusage.googleapis.com/v1/projects/$PROJECT/services:batchEnable
func (b *RequestBatcher) SendRequestWithTimeout(batchKey string, request *BatchRequest, timeout time.Duration) (interface{}, error) {
	if request == nil {
		return nil, fmt.Errorf("error, cannot request batching for nil BatchRequest")
	}
	if request.CombineF == nil {
		return nil, fmt.Errorf("error, cannot request batching for BatchRequest with nil CombineF")
	}
	if request.SendF == nil {
		return nil, fmt.Errorf("error, cannot request batching for BatchRequest with nil SendF")
	}
	if !b.enableBatching {
		log.Printf("[DEBUG] Batching is disabled, sending single request for %q", request.DebugId)
		return request.SendF(request.ResourceName, request.Body)
	}

	respCh, err := b.registerBatchRequest(batchKey, request)
	if err != nil {
		return nil, fmt.Errorf("error adding request to batch: %s", err)
	}

	ctx, cancel := context.WithTimeout(b.parentCtx, timeout)
	defer cancel()

	select {
	case resp := <-respCh:
		if resp.err != nil {
			// use wrapf so we can potentially extract the original error type
			return nil, errwrap.Wrapf(fmt.Sprintf("Batch %q for request %q returned error: {{err}}", batchKey, request.DebugId), resp.err)
		}
		return resp.body, nil
	case <-ctx.Done():
		break
	}
	return nil, fmt.Errorf("Request %s timed out after %v", batchKey, timeout)
}

// registerBatchRequest safely sees if an existing batch has been started
// with the given batchKey. If a batch exists, this will combine the new
// request into this existing batch. Else, this method manages starting a new
// batch and adding it to the RequestBatcher's started batches.
func (b *RequestBatcher) registerBatchRequest(batchKey string, newRequest *BatchRequest) (<-chan batchResponse, error) {
	b.Lock()
	defer b.Unlock()

	// If batch already exists, combine this request into existing request.
	if batch, ok := b.batches[batchKey]; ok {
		return batch.addRequest(newRequest)
	}

	log.Printf("[DEBUG] Creating new batch %q from request %q", newRequest.DebugId, batchKey)
	// The calling goroutine will need a channel to wait on for a response.
	respCh := make(chan batchResponse, 1)

	// Create a new batch.
	b.batches[batchKey] = &startedBatch{
		BatchRequest: newRequest,
		batchKey:     batchKey,
		listeners:    []chan batchResponse{respCh},
	}

	// Start a timer to send the request
	b.batches[batchKey].timer = time.AfterFunc(b.sendAfter, func() {
		batch := b.popBatch(batchKey)

		var resp batchResponse
		if batch == nil {
			log.Printf("[DEBUG] Batch not found in saved batches, running single request batch %q", batchKey)
			resp = newRequest.send()
		} else {
			log.Printf("[DEBUG] Sending batch %q combining %d requests)", batchKey, len(batch.listeners))
			resp = batch.send()
		}

		// Send message to all goroutines waiting on result.
		for _, ch := range batch.listeners {
			ch <- resp
			close(ch)
		}
	})

	return respCh, nil
}

// popBatch safely gets and removes a batch with given batchkey from the
// RequestBatcher's started batches.
func (b *RequestBatcher) popBatch(batchKey string) *startedBatch {
	b.Lock()
	defer b.Unlock()

	batch, ok := b.batches[batchKey]
	if !ok {
		log.Printf("[DEBUG] Batch with ID %q not found in batcher", batchKey)
		return nil
	}

	delete(b.batches, batchKey)
	return batch
}

func (batch *startedBatch) addRequest(newRequest *BatchRequest) (<-chan batchResponse, error) {
	log.Printf("[DEBUG] Adding batch request %q to existing batch %q", newRequest.DebugId, batch.batchKey)
	if batch.CombineF == nil {
		return nil, fmt.Errorf("Provider Error: unable to add request %q to batch %q with no CombineF", newRequest.DebugId, batch.batchKey)
	}
	newBody, err := batch.CombineF(batch.Body, newRequest.Body)
	if err != nil {
		return nil, fmt.Errorf("Provider Error: Unable to combine request %q data into existing batch %q: %v", newRequest.DebugId, batch.batchKey, err)
	}
	batch.Body = newBody

	log.Printf("[DEBUG] Added batch request %q to batch. New batch body: %v", newRequest.DebugId, batch.Body)

	respCh := make(chan batchResponse, 1)
	batch.listeners = append(batch.listeners, respCh)
	return respCh, nil
}

func (req *BatchRequest) send() batchResponse {
	if req.SendF == nil {
		return batchResponse{
			err: fmt.Errorf("provider error: Batch request has no SendBatch function"),
		}
	}
	v, err := req.SendF(req.ResourceName, req.Body)
	return batchResponse{v, err}
}
<commit_msg>Batch errors now indicate how to disable batching<commit_after>package google

import (
	"context"
	"fmt"
	"log"
	"sync"
	"time"

	"github.com/hashicorp/errwrap"
)

const defaultBatchSendIntervalSec = 3

// RequestBatcher is a global batcher object that keeps track of
// existing batches.
// In general, a batcher should be created per service that requires batching
// in order to prevent blocking batching for one service due to another,
// and to minimize the possibility of overlap in batchKey formats
// (see SendRequestWithTimeout)
type RequestBatcher struct {
	sync.Mutex

	*batchingConfig
	parentCtx context.Context
	batches   map[string]*startedBatch
	debugId   string
}

// BatchRequest represents a single request to a global batcher.
type BatchRequest struct {
	// ResourceName represents the underlying resource for which
	// a request is made. Its format is determined by what SendF expects, but
	// typically should be the name of the parent GCP resource being changed.
	ResourceName string

	// Body is this request's data to be passed to SendF, and may be combined
	// with other bodies using CombineF.
	Body interface{}

	// CombineF function determines how to combine bodies from two batches.
	CombineF batcherCombineFunc

	// SendF function determines how to actually send a batched request to a
	// third party service. The arguments given to this function are
	// (ResourceName, Body) where Body may have been combined with other request
	// Bodies.
	SendF batcherSendFunc

	// ID for debugging request. This should be specific to a single request
	// (i.e. per Terraform resource)
	DebugId string
}

// These types are meant to be the public interface to batchers. They define
// logic to manage batch data type and behavior, and require service-specific
// implementations per type of request per service.
// Function type for combine existing batches and additional batch data
type batcherCombineFunc func(body interface{}, toAdd interface{}) (interface{}, error)

// Function type for sending a batch request
type batcherSendFunc func(resourceName string, body interface{}) (interface{}, error)

// batchResponse bundles an API response (data, error) tuple.
type batchResponse struct {
	body interface{}
	err  error
}

// startedBatch refers to a processed batch whose timer to send the request has
// already been started. The responses for the request is sent to each listener
// channel, representing parallel callers that are waiting on requests
// combined into this batch.
type startedBatch struct {
	batchKey string
	*BatchRequest

	listeners []chan batchResponse
	timer     *time.Timer
}

// batchingConfig contains user configuration for controlling batch requests.
type batchingConfig struct {
	sendAfter      time.Duration
	enableBatching bool
}

// Initializes a new batcher.
func NewRequestBatcher(debugId string, ctx context.Context, config *batchingConfig) *RequestBatcher {
	batcher := &RequestBatcher{
		debugId:        debugId,
		parentCtx:      ctx,
		batchingConfig: config,
		batches:        make(map[string]*startedBatch),
	}

	go func(b *RequestBatcher) {
		<-ctx.Done()
		b.stop()
	}(batcher)

	return batcher
}

func (b *RequestBatcher) stop() {
	b.Lock()
	defer b.Unlock()

	log.Printf("[DEBUG] Stopping batcher %q", b.debugId)
	for batchKey, batch := range b.batches {
		log.Printf("[DEBUG] Cleaning up batch request %q", batchKey)
		batch.timer.Stop()
		for _, l := range batch.listeners {
			close(l)
		}
	}
}

// SendRequestWithTimeout is expected to be called per parallel call.
// It manages waiting on the result of a batch request.
//
// Batch requests are grouped by the given batchKey. batchKey
// should be unique to the API request being sent, most likely similar to
// the HTTP request URL with GCP resource ID included in the URL (the caller
// may choose to use a key with method if needed to diff GET/read and
// POST/create)
//
// As an example, for google_project_service, the
// batcher is called to batch services.batchEnable() calls for a project
// $PROJECT. The calling code uses the template
// "serviceusage:projects/$PROJECT/services:batchEnable", which mirrors the HTTP request:
// POST https://serviceusage.googleapis.com/v1/projects/$PROJECT/services:batchEnable
func (b *RequestBatcher) SendRequestWithTimeout(batchKey string, request *BatchRequest, timeout time.Duration) (interface{}, error) {
	if request == nil {
		return nil, fmt.Errorf("error, cannot request batching for nil BatchRequest")
	}
	if request.CombineF == nil {
		return nil, fmt.Errorf("error, cannot request batching for BatchRequest with nil CombineF")
	}
	if request.SendF == nil {
		return nil, fmt.Errorf("error, cannot request batching for BatchRequest with nil SendF")
	}
	if !b.enableBatching {
		log.Printf("[DEBUG] Batching is disabled, sending single request for %q", request.DebugId)
		return request.SendF(request.ResourceName, request.Body)
	}

	respCh, err := b.registerBatchRequest(batchKey, request)
	if err != nil {
		return nil, fmt.Errorf("error adding request to batch: %s", err)
	}

	ctx, cancel := context.WithTimeout(b.parentCtx, timeout)
	defer cancel()

	select {
	case resp := <-respCh:
		if resp.err != nil {
			// use wrapf so we can potentially extract the original error type
			errMsg := fmt.Sprintf(
				"Batch %q for request %q returned error: {{err}}. To debug individual requests, try disabling batching: https://www.terraform.io/docs/providers/google/guides/provider_reference.html#enable_batching",
				batchKey, request.DebugId)
			return nil, errwrap.Wrapf(errMsg, resp.err)
		}
		return resp.body, nil
	case <-ctx.Done():
		break
	}
	return nil, fmt.Errorf("Request %s timed out after %v", batchKey, timeout)
}

// registerBatchRequest safely sees if an existing batch has been started
// with the given batchKey. If a batch exists, this will combine the new
// request into this existing batch. Else, this method manages starting a new
// batch and adding it to the RequestBatcher's started batches.
func (b *RequestBatcher) registerBatchRequest(batchKey string, newRequest *BatchRequest) (<-chan batchResponse, error) {
	b.Lock()
	defer b.Unlock()

	// If batch already exists, combine this request into existing request.
	if batch, ok := b.batches[batchKey]; ok {
		return batch.addRequest(newRequest)
	}

	log.Printf("[DEBUG] Creating new batch %q from request %q", newRequest.DebugId, batchKey)
	// The calling goroutine will need a channel to wait on for a response.
	respCh := make(chan batchResponse, 1)

	// Create a new batch.
	b.batches[batchKey] = &startedBatch{
		BatchRequest: newRequest,
		batchKey:     batchKey,
		listeners:    []chan batchResponse{respCh},
	}

	// Start a timer to send the request
	b.batches[batchKey].timer = time.AfterFunc(b.sendAfter, func() {
		batch := b.popBatch(batchKey)

		var resp batchResponse
		if batch == nil {
			log.Printf("[DEBUG] Batch not found in saved batches, running single request batch %q", batchKey)
			resp = newRequest.send()
		} else {
			log.Printf("[DEBUG] Sending batch %q combining %d requests)", batchKey, len(batch.listeners))
			resp = batch.send()
		}

		// Send message to all goroutines waiting on result.
		for _, ch := range batch.listeners {
			ch <- resp
			close(ch)
		}
	})

	return respCh, nil
}

// popBatch safely gets and removes a batch with given batchkey from the
// RequestBatcher's started batches.
func (b *RequestBatcher) popBatch(batchKey string) *startedBatch {
	b.Lock()
	defer b.Unlock()

	batch, ok := b.batches[batchKey]
	if !ok {
		log.Printf("[DEBUG] Batch with ID %q not found in batcher", batchKey)
		return nil
	}

	delete(b.batches, batchKey)
	return batch
}

func (batch *startedBatch) addRequest(newRequest *BatchRequest) (<-chan batchResponse, error) {
	log.Printf("[DEBUG] Adding batch request %q to existing batch %q", newRequest.DebugId, batch.batchKey)
	if batch.CombineF == nil {
		return nil, fmt.Errorf("Provider Error: unable to add request %q to batch %q with no CombineF", newRequest.DebugId, batch.batchKey)
	}
	newBody, err := batch.CombineF(batch.Body, newRequest.Body)
	if err != nil {
		return nil, fmt.Errorf("Provider Error: Unable to combine request %q data into existing batch %q: %v", newRequest.DebugId, batch.batchKey, err)
	}
	batch.Body = newBody

	log.Printf("[DEBUG] Added batch request %q to batch. New batch body: %v", newRequest.DebugId, batch.Body)

	respCh := make(chan batchResponse, 1)
	batch.listeners = append(batch.listeners, respCh)
	return respCh, nil
}

func (req *BatchRequest) send() batchResponse {
	if req.SendF == nil {
		return batchResponse{
			err: fmt.Errorf("provider error: Batch request has no SendBatch function"),
		}
	}
	v, err := req.SendF(req.ResourceName, req.Body)
	return batchResponse{v, err}
}
<|endoftext|>


<commit_before>package app

import (
	"bytes"
	"compress/flate"
	crand "crypto/rand"
	"fmt"
	"io"
	"math/rand"
	"time"
)

const _letters string = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"

func init() {
	rand.Seed(time.Now().UnixNano())
}

// NewUUID
// Source: http://play.golang.org/p/4FkNSiUDMg
// Description:
//		Generates and returns a uuid
// @returns string
func NewUUID() (string, error) {
	uuid := make([]byte, 16)
	n, err := io.ReadFull(crand.Reader, uuid)
	if n != len(uuid) || err != nil {
		return "", err
	}
	// variant bits; see section 4.1.1
	uuid[8] = uuid[8]&^0xc0 | 0x80
	// version 4 (pseudo-random); see section 4.1.3
	uuid[6] = uuid[6]&^0xf0 | 0x40
	// return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]), nil
	return fmt.Sprintf("%x%x%x%x%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil
}

func NewUUID2() (string, error) {
	b := make([]byte, 16)
	n, err := io.ReadFull(crand.Reader, b)
	if n != len(b) || err != nil {
		return "", err
	}
	// variant bits; see section 4.1.1
	b[8] = b[8]&^0xc0 | 0x80
	// version 4 (pseudo-random); see section 4.1.3
	b[6] = b[6]&^0xf0 | 0x40
	return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]), nil
}

// NewAPIKey
// Description:
//		Generates apikey of desired length
// @param int length of apikey
// @returns string
func NewAPIKey(n int) string {
	s := ""
	for i := 1; i <= n; i++ {
		s += string(_letters[rand.Intn(len(_letters))])
	}
	return s
}

// stringInSlice
// Description:
//		Loops through array of strings
//		Checks each string in array for match
//		If string match occurs returns true
// @param a {string} string to find
// @param list {[]string} array of strings to search
// @returns bool
func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

// sliceIndex
// Description:
//		Loops through array of strings
//		Checks each string in array for match
//		If string match occurs returns index
// @param value {string} string to find
// @param slice {[]string} array of strings to search
// @returns int
func sliceIndex(value string, slice []string) int {
	for p, v := range slice {
		if v == value {
			return p
		}
	}
	return -1
}

// Compression
// Source: https://github.com/schollz/gofind/blob/master/utils.go#L146-L169
//         https://github.com/schollz/gofind/blob/master/fingerprint.go#L43-L54
// Description:
//		Compress and Decompress bytes
func compressByte(src []byte) []byte {
	compressedData := new(bytes.Buffer)
	compress(src, compressedData, 9)
	return compressedData.Bytes()
}

func decompressByte(src []byte) []byte {
	compressedData := bytes.NewBuffer(src)
	deCompressedData := new(bytes.Buffer)
	decompress(compressedData, deCompressedData)
	return deCompressedData.Bytes()
}

func compress(src []byte, dest io.Writer, level int) {
	compressor, _ := flate.NewWriter(dest, level)
	compressor.Write(src)
	compressor.Close()
}

func decompress(src io.Reader, dest io.Writer) {
	decompressor := flate.NewReader(src)
	io.Copy(dest, decompressor)
	decompressor.Close()
}
<commit_msg>goreport fix commenting<commit_after>package app

import (
	"bytes"
	"compress/flate"
	crand "crypto/rand"
	"fmt"
	"io"
	"math/rand"
	"time"
)

const _letters string = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"

func init() {
	rand.Seed(time.Now().UnixNano())
}

// NewUUID generates a url friendly uuid
// Source: http://play.golang.org/p/4FkNSiUDMg
// @returns string
func NewUUID() (string, error) {
	uuid := make([]byte, 16)
	n, err := io.ReadFull(crand.Reader, uuid)
	if n != len(uuid) || err != nil {
		return "", err
	}
	// variant bits; see section 4.1.1
	uuid[8] = uuid[8]&^0xc0 | 0x80
	// version 4 (pseudo-random); see section 4.1.3
	uuid[6] = uuid[6]&^0xf0 | 0x40
	// return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]), nil
	return fmt.Sprintf("%x%x%x%x%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil
}

func NewUUID2() (string, error) {
	b := make([]byte, 16)
	n, err := io.ReadFull(crand.Reader, b)
	if n != len(b) || err != nil {
		return "", err
	}
	// variant bits; see section 4.1.1
	b[8] = b[8]&^0xc0 | 0x80
	// version 4 (pseudo-random); see section 4.1.3
	b[6] = b[6]&^0xf0 | 0x40
	return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]), nil
}

// NewAPIKey generates apikey of desired length
// @param int length of apikey
// @returns string
func NewAPIKey(n int) string {
	s := ""
	for i := 1; i <= n; i++ {
		s += string(_letters[rand.Intn(len(_letters))])
	}
	return s
}

// stringInSlice loops through a []string and returns a bool if string is found
// @param a {string} string to find
// @param list {[]string} array of strings to search
// @returns bool
func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

// sliceIndex loops through a []string and returns the index of a string
// Description:
//		Loops through array of strings
//		Checks each string in array for match
//		If string match occurs returns index
// @param value {string} string to find
// @param slice {[]string} array of strings to search
// @returns int
func sliceIndex(value string, slice []string) int {
	for p, v := range slice {
		if v == value {
			return p
		}
	}
	return -1
}

// Compression
// Source: https://github.com/schollz/gofind/blob/master/utils.go#L146-L169
//         https://github.com/schollz/gofind/blob/master/fingerprint.go#L43-L54
// Description:
//		Compress and Decompress bytes
func compressByte(src []byte) []byte {
	compressedData := new(bytes.Buffer)
	compress(src, compressedData, 9)
	return compressedData.Bytes()
}

func decompressByte(src []byte) []byte {
	compressedData := bytes.NewBuffer(src)
	deCompressedData := new(bytes.Buffer)
	decompress(compressedData, deCompressedData)
	return deCompressedData.Bytes()
}

func compress(src []byte, dest io.Writer, level int) {
	compressor, _ := flate.NewWriter(dest, level)
	compressor.Write(src)
	compressor.Close()
}

func decompress(src io.Reader, dest io.Writer) {
	decompressor := flate.NewReader(src)
	io.Copy(dest, decompressor)
	decompressor.Close()
}
<|endoftext|>