<commit_before>package mock

import (
	"context"
	"fmt"
	"sync"
	"time"

	log "github.com/monzo/slog"
	uuid "github.com/nu7hatch/gouuid"

	"github.com/monzo/typhon/message"
	"github.com/monzo/typhon/transport"
)

const (
	timeout = time.Second
)

type mockListener struct {
	tomb    *tomb.Tomb
	reqChan chan<- message.Request
}

type mockTransport struct {
	sync.RWMutex
	tomb         *tomb.Tomb
	listeners    map[string]*mockListener
	ready        chan struct{}                      // closed immediately for this transport
	inflightReqs map[string]chan<- message.Response // correlation id: response chan
}

func (mt *mockTransport) run() error {
	close(mt.ready)
	<-mt.tomb.Dying()
	mt.killListeners()
	return tomb.ErrDying
}

func (t *mockTransport) Ready() <-chan struct{} {
	return t.ready
}

func (t *mockTransport) Tomb() *tomb.Tomb {
	return t.tomb
}

func (t *mockTransport) killListeners() {
	t.RLock()
	ts := make([]*tomb.Tomb, 0, len(t.listeners))
	for _, l := range t.listeners {
		l.tomb.Killf("Listeners killed")
		ts = append(ts, l.tomb)
	}
	t.RUnlock()
	for _, t := range ts {
		t.Wait()
	}
}

func (t *mockTransport) StopListening(serviceName string) bool {
	t.RLock()
	l, ok := t.listeners[serviceName]
	if ok {
		l.tomb.Killf("Stopped listening")
	}
	t.RUnlock()
	if ok {
		l.tomb.Wait()
		return true
	}
	return false
}

func (t *mockTransport) Listen(serviceName string, rc chan<- message.Request) error {
	ctx := context.Background()
	l := &mockListener{
		tomb:    new(tomb.Tomb),
		reqChan: rc,
	}
	t.Lock()
	if _, ok := t.listeners[serviceName]; ok {
		t.Unlock()
		return transport.ErrAlreadyListening
	} else {
		t.listeners[serviceName] = l
		t.Unlock()
	}

	stop := func() {
		log.Debug(ctx, "[Typhon:MockTransport] Listener %s stopped", serviceName)
		t.Lock()
		defer t.Unlock()
		delete(t.listeners, serviceName)
	}

	// Wait here, rather than inside the goroutine, because we want to be able to report an (un)successful connection
	tm := l.tomb
	select {
	case <-t.tomb.Dying():
		stop()
		return tomb.ErrDying
	case <-tm.Dying():
		stop()
		return tomb.ErrDying
	case <-time.After(timeout):
		stop()
		return transport.ErrTimeout
	case <-t.Ready():
		log.Debug(ctx, "[Typhon:MockTransport] Listener %s started", serviceName)
	}

	tm.Go(func() error {
		defer stop()
		<-tm.Dying()
		return tomb.ErrDying
	})
	return nil
}

func (t *mockTransport) Send(req message.Request, timeout time.Duration) (message.Response, error) {
	ctx := context.Background()
	id := req.Id()
	if id == "" {
		_uuid, err := uuid.NewV4()
		if err != nil {
			log.Error(ctx, "[Typhon:MockTransport] Failed to generate request uuid: %v", err)
			return nil, err
		}
		req.SetId(_uuid.String())
	}

	// Make a copy of the response that does not preserve the Body (this is not preserved over the wire)
	req = req.Copy()
	req.SetBody(nil)

	t.RLock()
	l, ok := t.listeners[req.Service()]
	t.RUnlock()
	if ok {
		responseChan := make(chan message.Response, 1)
		t.Lock()
		t.inflightReqs[req.Id()] = responseChan
		t.Unlock()
		defer func() {
			t.Lock()
			delete(t.inflightReqs, req.Id())
			t.Unlock()
		}()

		timer := time.NewTimer(timeout)
		defer timer.Stop()
		select {
		case <-timer.C:
			log.Debug(ctx, "[Typhon:MockTransport] Timed out after %v waiting for delivery of \"%s\"", timeout, req.Id())
			return nil, transport.ErrTimeout
		case l.reqChan <- req:
		}

		select {
		case rsp := <-responseChan:
			return rsp, nil
		case <-timer.C:
			log.Debug(ctx, "[Typhon:MockTransport] Timed out after %v waiting for response to \"%s\"", timeout, req.Id())
			return nil, transport.ErrTimeout
		}
	}

	log.Warn(ctx, "[Typhon:MockTransport] Failing request to missing %s as timeout", req.Service())
	return nil, transport.ErrTimeout // Don't bother waiting artificially
}

func (t *mockTransport) Respond(req message.Request, rsp message.Response) error {
	t.RLock()
	rspChan, ok := t.inflightReqs[req.Id()]
	t.RUnlock()

	// Make a copy of the response that does not preserve the Body (this is not preserved over the wire)
	rsp = rsp.Copy()
	rsp.SetBody(nil)

	if ok {
		select {
		case rspChan <- rsp:
			return nil
		case <-time.After(timeout):
			return transport.ErrTimeout
		}
	}
	return fmt.Errorf("No correlated request \"%s\" is in-flight", req.Id())
}

func NewTransport() transport.Transport {
	trans := &mockTransport{
		tomb:         new(tomb.Tomb),
		listeners:    make(map[string]*mockListener),
		ready:        make(chan struct{}),
		inflightReqs: make(map[string]chan<- message.Response),
	}
	trans.tomb.Go(trans.run)
	return trans
}
<commit_msg>Fix mock transport import<commit_after>package mock

import (
	"context"
	"fmt"
	"sync"
	"time"

	log "github.com/monzo/slog"
	uuid "github.com/nu7hatch/gouuid"
	"gopkg.in/tomb.v2"

	"github.com/monzo/typhon/message"
	"github.com/monzo/typhon/transport"
)

const (
	timeout = time.Second
)

type mockListener struct {
	tomb    *tomb.Tomb
	reqChan chan<- message.Request
}

type mockTransport struct {
	sync.RWMutex
	tomb         *tomb.Tomb
	listeners    map[string]*mockListener
	ready        chan struct{}                      // closed immediately for this transport
	inflightReqs map[string]chan<- message.Response // correlation id: response chan
}

func (mt *mockTransport) run() error {
	close(mt.ready)
	<-mt.tomb.Dying()
	mt.killListeners()
	return tomb.ErrDying
}

func (t *mockTransport) Ready() <-chan struct{} {
	return t.ready
}

func (t *mockTransport) Tomb() *tomb.Tomb {
	return t.tomb
}

func (t *mockTransport) killListeners() {
	t.RLock()
	ts := make([]*tomb.Tomb, 0, len(t.listeners))
	for _, l := range t.listeners {
		l.tomb.Killf("Listeners killed")
		ts = append(ts, l.tomb)
	}
	t.RUnlock()
	for _, t := range ts {
		t.Wait()
	}
}

func (t *mockTransport) StopListening(serviceName string) bool {
	t.RLock()
	l, ok := t.listeners[serviceName]
	if ok {
		l.tomb.Killf("Stopped listening")
	}
	t.RUnlock()
	if ok {
		l.tomb.Wait()
		return true
	}
	return false
}

func (t *mockTransport) Listen(serviceName string, rc chan<- message.Request) error {
	ctx := context.Background()
	l := &mockListener{
		tomb:    new(tomb.Tomb),
		reqChan: rc,
	}
	t.Lock()
	if _, ok := t.listeners[serviceName]; ok {
		t.Unlock()
		return transport.ErrAlreadyListening
	} else {
		t.listeners[serviceName] = l
		t.Unlock()
	}

	stop := func() {
		log.Debug(ctx, "[Typhon:MockTransport] Listener %s stopped", serviceName)
		t.Lock()
		defer t.Unlock()
		delete(t.listeners, serviceName)
	}

	// Wait here, rather than inside the goroutine, because we want to be able to report an (un)successful connection
	tm := l.tomb
	select {
	case <-t.tomb.Dying():
		stop()
		return tomb.ErrDying
	case <-tm.Dying():
		stop()
		return tomb.ErrDying
	case <-time.After(timeout):
		stop()
		return transport.ErrTimeout
	case <-t.Ready():
		log.Debug(ctx, "[Typhon:MockTransport] Listener %s started", serviceName)
	}

	tm.Go(func() error {
		defer stop()
		<-tm.Dying()
		return tomb.ErrDying
	})
	return nil
}

func (t *mockTransport) Send(req message.Request, timeout time.Duration) (message.Response, error) {
	ctx := context.Background()
	id := req.Id()
	if id == "" {
		_uuid, err := uuid.NewV4()
		if err != nil {
			log.Error(ctx, "[Typhon:MockTransport] Failed to generate request uuid: %v", err)
			return nil, err
		}
		req.SetId(_uuid.String())
	}

	// Make a copy of the response that does not preserve the Body (this is not preserved over the wire)
	req = req.Copy()
	req.SetBody(nil)

	t.RLock()
	l, ok := t.listeners[req.Service()]
	t.RUnlock()
	if ok {
		responseChan := make(chan message.Response, 1)
		t.Lock()
		t.inflightReqs[req.Id()] = responseChan
		t.Unlock()
		defer func() {
			t.Lock()
			delete(t.inflightReqs, req.Id())
			t.Unlock()
		}()

		timer := time.NewTimer(timeout)
		defer timer.Stop()
		select {
		case <-timer.C:
			log.Debug(ctx, "[Typhon:MockTransport] Timed out after %v waiting for delivery of \"%s\"", timeout, req.Id())
			return nil, transport.ErrTimeout
		case l.reqChan <- req:
		}

		select {
		case rsp := <-responseChan:
			return rsp, nil
		case <-timer.C:
			log.Debug(ctx, "[Typhon:MockTransport] Timed out after %v waiting for response to \"%s\"", timeout, req.Id())
			return nil, transport.ErrTimeout
		}
	}

	log.Warn(ctx, "[Typhon:MockTransport] Failing request to missing %s as timeout", req.Service())
	return nil, transport.ErrTimeout // Don't bother waiting artificially
}

func (t *mockTransport) Respond(req message.Request, rsp message.Response) error {
	t.RLock()
	rspChan, ok := t.inflightReqs[req.Id()]
	t.RUnlock()

	// Make a copy of the response that does not preserve the Body (this is not preserved over the wire)
	rsp = rsp.Copy()
	rsp.SetBody(nil)

	if ok {
		select {
		case rspChan <- rsp:
			return nil
		case <-time.After(timeout):
			return transport.ErrTimeout
		}
	}
	return fmt.Errorf("No correlated request \"%s\" is in-flight", req.Id())
}

func NewTransport() transport.Transport {
	trans := &mockTransport{
		tomb:         new(tomb.Tomb),
		listeners:    make(map[string]*mockListener),
		ready:        make(chan struct{}),
		inflightReqs: make(map[string]chan<- message.Response),
	}
	trans.tomb.Go(trans.run)
	return trans
}
<|endoftext|>


<commit_before>package models

import (
	"errors"
	"github.com/vivowares/octopus/Godeps/_workspace/src/gopkg.in/olivere/elastic.v3"
	. "github.com/vivowares/octopus/utils"
	"strconv"
	"strings"
	"time"
)

type StatsQuery struct {
	Channel   *Channel
	TimeStart time.Time
	TimeEnd   time.Time
}

func (q *StatsQuery) Parse(params map[string]string) error {
	if timeRange, found := params["time_range"]; !found {
		return errors.New("missing time_range for channel stats")
	} else {
		ranges := strings.Split(timeRange, ":")
		if len(ranges) != 2 || len(ranges[0]) == 0 {
			return errors.New("invalid time_range format: " + timeRange)
		}
		start, err := strconv.ParseInt(ranges[0], 10, 64)
		if err != nil {
			return errors.New("error parsing time_range: " + timeRange)
		}
		q.TimeStart = time.Unix(MilliSecToSec(start), MilliSecToNano(start)).UTC()

		if len(ranges[1]) > 0 {
			end, err := strconv.ParseInt(ranges[1], 10, 64)
			if err != nil {
				return errors.New("error parsing time_range: " + timeRange)
			}
			q.TimeEnd = time.Unix(MilliSecToSec(end), MilliSecToNano(end)).UTC()
		} else {
			q.TimeEnd = time.Now().UTC()
		}
	}

	if q.TimeStart.After(q.TimeEnd) {
		return errors.New("invalid time_range, start_time is later than end_time")
	}

	return nil
}

func (q *StatsQuery) QueryES() (interface{}, error) {
	filterAgg := elastic.NewFilterAggregation()

	boolQ := elastic.NewBoolQuery()
	rangeQ := elastic.NewRangeQuery("timestamp").
		From(NanoToMilli(q.TimeStart.UnixNano())).
		To(NanoToMilli(q.TimeEnd.UnixNano()))
	boolQ.Must(rangeQ)

	filterAgg.Filter(boolQ)

	for _, tag := range q.Channel.Tags {
		filterAgg.SubAggregation(tag, elastic.NewTermsAggregation().Field(tag).Size(0))
	}

	resp, err := IndexClient.Search().
		SearchType("count").
		Index(TimedIndices(q.Channel, q.TimeStart, q.TimeEnd)).
		Type(IndexType).
		Aggregation("name", filterAgg).
		Do()

	if err != nil {
		return nil, err
	} else {
		if agg, found := resp.Aggregations.Filter("name"); found {
			res := make(map[string]interface{})
			for _, tag := range q.Channel.Tags {
				tagStats, found := agg.Aggregations.Terms(tag)
				if found {
					values := []interface{}{}
					for _, bucket := range tagStats.Buckets {
						values = append(values, bucket.Key)
					}
					res[tag] = values
				}
			}
			return res, nil
		}
		return nil, nil
	}
}
<commit_msg>fixed a bug when indices are not created, tag_stats shouldn't not return error<commit_after>package models

import (
	"errors"
	"github.com/vivowares/octopus/Godeps/_workspace/src/gopkg.in/olivere/elastic.v3"
	. "github.com/vivowares/octopus/utils"
	"strconv"
	"strings"
	"time"
)

type StatsQuery struct {
	Channel   *Channel
	TimeStart time.Time
	TimeEnd   time.Time
}

func (q *StatsQuery) Parse(params map[string]string) error {
	if timeRange, found := params["time_range"]; !found {
		return errors.New("missing time_range for channel stats")
	} else {
		ranges := strings.Split(timeRange, ":")
		if len(ranges) != 2 || len(ranges[0]) == 0 {
			return errors.New("invalid time_range format: " + timeRange)
		}
		start, err := strconv.ParseInt(ranges[0], 10, 64)
		if err != nil {
			return errors.New("error parsing time_range: " + timeRange)
		}
		q.TimeStart = time.Unix(MilliSecToSec(start), MilliSecToNano(start)).UTC()

		if len(ranges[1]) > 0 {
			end, err := strconv.ParseInt(ranges[1], 10, 64)
			if err != nil {
				return errors.New("error parsing time_range: " + timeRange)
			}
			q.TimeEnd = time.Unix(MilliSecToSec(end), MilliSecToNano(end)).UTC()
		} else {
			q.TimeEnd = time.Now().UTC()
		}
	}

	if q.TimeStart.After(q.TimeEnd) {
		return errors.New("invalid time_range, start_time is later than end_time")
	}

	return nil
}

func (q *StatsQuery) QueryES() (interface{}, error) {
	filterAgg := elastic.NewFilterAggregation()

	boolQ := elastic.NewBoolQuery()
	rangeQ := elastic.NewRangeQuery("timestamp").
		From(NanoToMilli(q.TimeStart.UnixNano())).
		To(NanoToMilli(q.TimeEnd.UnixNano()))
	boolQ.Must(rangeQ)

	filterAgg.Filter(boolQ)

	for _, tag := range q.Channel.Tags {
		filterAgg.SubAggregation(tag, elastic.NewTermsAggregation().Field(tag).Size(0))
	}

	resp, err := IndexClient.Search().
		SearchType("count").
		Index(TimedIndices(q.Channel, q.TimeStart, q.TimeEnd)).
		Type(IndexType).
		Aggregation("name", filterAgg).
		Do()

	res := make(map[string]interface{})
	for _, tag := range q.Channel.Tags {
		res[tag] = []interface{}{}
	}

	if err != nil {
		return res, nil
	} else {
		if agg, found := resp.Aggregations.Filter("name"); found {
			for _, tag := range q.Channel.Tags {
				tagStats, found := agg.Aggregations.Terms(tag)
				if found {
					for _, bucket := range tagStats.Buckets {
						res[tag] = append(res[tag].([]interface{}), bucket.Key)
					}
				}
			}
			return res, nil
		}
		return nil, nil
	}
}
<|endoftext|>


<commit_before>package gouda

import (
	"xml"
	//	"strings"
	"fmt"
	"os"
	"bytes"
	"container/vector"
)

/** Value **/

type XMLConnector struct {
	db     *os.File
	tables map[string]XMLTable
}

type XMLTable map[string]int //Kind

func (e *XMLConnector) Close() {
	e.commit()
	e.db.Close()
}

func (e *XMLConnector) Open(connectionString string) bool {
	f, err := os.Open(connectionString, os.O_RDWR|os.O_CREAT, 0755)
	if err != nil {
		fmt.Fprintln(os.Stderr, "Error openning"+fmt.Sprint(err))
		os.Exit(1)
	}
	e.db = f
	e.init()
	return false
}

func xml_attribute(xel xml.StartElement, name string) (xml.Attr, bool) {
	for _, a := range xel.Attr {
		if a.Name.Local == name {
			return a, true
		}
	}
	return xml.Attr{}, false
}

func (e *XMLConnector) init() {
	parser := xml.NewParser(e.db)
	a, err := parser.Token()
	schema := false
	lastType := NullKind
	var lastTable XMLTable
	for ; err == nil; a, err = parser.Token() {
		switch a.(type) {
		case xml.StartElement:
			xel := a.(xml.StartElement)
			el := xel.Name.Local
			//fmt.Println(el)
			if el == "schema" {
				schema = true
			}
			if schema {
				switch el {
				case "table":
					if attr, found := xml_attribute(xel, "name"); found {
					//	fmt.Println("Found Table ! " + attr.Value)
						lastTable = e.CreateTableWithoutId(attr.Value)
					} else {
					//	fmt.Println(os.Stderr, "Missing required attribute name on
")
					}
				case "attribute":
					if attr, found := xml_attribute(xel, "type"); found {
					//	fmt.Println("Found Attribute ! on" + fmt.Sprint(lastTable) + " " + attr.Value)
						//fmt.Println(xel)
						lastType = typeFromName(attr.Value)
					} else {
						fmt.Println(os.Stderr, "Missing required attribute type on <attribute>")
					}

				}
			}
		case xml.CharData:
			if schema && lastType != NullKind {
				b := bytes.NewBuffer(a.(xml.CharData))
				lastTable.AddAttribute(b.String(), lastType)
				lastType = NullKind
			}
		case xml.EndElement:
			el := a.(xml.EndElement).Name.Local
			//fmt.Println("/" + el)
			if el == "schema" {
				schema = false
			}
		}
	}
	if _,oser:=err.(os.Error); err != nil && !oser  {
		fmt.Printf("%T ", err)
		fmt.Println(err)
	}
}

func (e *XMLConnector) commit() {
	e.db.Seek(0, 0)
	e.db.Truncate(0)
	fmt.Fprintln(e.db, "<?xml version=\"1.0\" ?>")
	fmt.Fprintln(e.db, "<db>")
	fmt.Fprintln(e.db, "<schema>")
	for table, tabledesc := range e.tables {
//		fmt.Println("Comitting table " + table + " struct " + fmt.Sprint(tabledesc))
		fmt.Fprintf(e.db, "
\n", table)
		for key, typ := range tabledesc {
			fmt.Fprintf(e.db, "<attribute type=\"%s\">%s</attribute>\n", typeName(typ), key)
		}
		fmt.Fprintln(e.db, "
")
	}
	fmt.Fprintln(e.db, "</schema>")
	fmt.Fprint(e.db, "</db>")

}

//TODO type ValueKind ??
func typeName(kind int) string {
	switch kind {
	case IntKind:
		return "int"
	case StringKind:
		return "string"
	}
	return "UNKNOWN"
}

func typeFromName(kind string) int {
	switch kind {
	case "int":
		return IntKind
	case "string":
		return StringKind
	}
	return NullKind
}

func (e *XMLConnector) CreateTableWithoutId(table string) XMLTable {
	e.tables[table] = make(XMLTable)
	return e.tables[table]
}

func (e *XMLConnector) CreateTable(table string) XMLTable {
	a := e.CreateTableWithoutId(table)
	a.AddAttribute("id", IntKind)
	return a
}

func (e *XMLConnector) Table(table string) XMLTable {
	tab, ok := e.tables[table]
	if ok {
		return tab
	}
	return nil
}

func (e *XMLTable) AddAttribute(name string, typ int) {
	(*e)[name] = typ
}
func (e XMLTable) Attributes() map[string]int { return e }

func (e *XMLConnector) Query(r *Relation) *vector.Vector {
	return new(vector.Vector)
}

func OpenXML(conStr string) Connection {
	db := (new(XMLConnector))
	db.tables = make(map[string]XMLTable)
	db.Open(conStr)
	return db
}
<commit_msg>data persistence in XMLConnection<commit_after>package gouda

import (
	"xml"
	"strconv"
	"fmt"
	"os"
	"bytes"
	"container/vector"
)

type XMLConnector struct {
	db     *os.File
	tables map[string]*XMLTable
}

type XMLTable struct {
	schema map[string]int
	data   *vector.Vector
}

func (e *XMLConnector) Close() {
	e.commit()
	e.db.Close()
}

func (e *XMLConnector) Open(connectionString string) bool {
	f, err := os.Open(connectionString, os.O_RDWR|os.O_CREAT, 0755)
	if err != nil {
		fmt.Fprintln(os.Stderr, "Error openning"+fmt.Sprint(err))
		os.Exit(1)
	}
	e.db = f
	e.init()
	return false
}

func xml_attribute(xel xml.StartElement, name string) (xml.Attr, bool) {
	for _, a := range xel.Attr {
		if a.Name.Local == name {
			return a, true
		}
	}
	return xml.Attr{}, false
}

func (e *XMLConnector) init() {
	parser := xml.NewParser(e.db)
	a, err := parser.Token()
	schema := false
	lastType := NullKind
	var lastAttr string=""
	var lastTable *XMLTable
	var lastItem map[string]Value
	for ; err == nil; a, err = parser.Token() {
		switch a.(type) {
		case xml.StartElement:
			xel := a.(xml.StartElement)
			el := xel.Name.Local
			//fmt.Println(el)
			if el == "schema" {
				schema = true
			}
			if schema {
				switch el {
				case "table":
					if attr, found := xml_attribute(xel, "name"); found {
						//	fmt.Println("Found Table ! " + attr.Value)
						lastTable = e.CreateTableWithoutId(attr.Value)
					} else {
						//	fmt.Println(os.Stderr, "Missing required attribute name on
")
					}
				case "attribute":
					if attr, found := xml_attribute(xel, "type"); found {
						//	fmt.Println("Found Attribute ! on" + fmt.Sprint(lastTable) + " " + attr.Value)
						//fmt.Println(xel)
						lastType = typeFromName(attr.Value)
					} else {
						fmt.Println(os.Stderr, "Missing required attribute type on <attribute>")
					}

				}
			} else {
				switch el {
				case "tabledata":
					if attr, found := xml_attribute(xel, "name"); found {
						lastTable = e.Table(attr.Value)
					} else {
						fmt.Println(os.Stderr, "Missing required attribute name on <tabledata>")
					}
				case "item":
					lastItem = make(map[string]Value)

				case "value":
					if attr, found := xml_attribute(xel, "name"); found {
						lastAttr=attr.Value
					} else {
						fmt.Println(os.Stderr, "Missing required attribute name on <tabledata>")
					}
}
			}
		case xml.CharData:
			if schema && lastType != NullKind {
				b := bytes.NewBuffer(a.(xml.CharData))
				lastTable.AddAttribute(b.String(), lastType)
				lastType = NullKind
			}else if lastAttr != "" {
				b := bytes.NewBuffer(a.(xml.CharData))
				switch lastTable.Attributes()[lastAttr] {
				case StringKind:
				lastItem[lastAttr]=SysString(b.String()).Value()
				case IntKind:
				i,_:=strconv.Atoi(b.String())
				lastItem[lastAttr]=SysInt(i).Value()

				}
				lastAttr = ""

			}
		case xml.EndElement:
			el := a.(xml.EndElement).Name.Local
			//fmt.Println("/" + el)
			if el == "schema" {
				schema = false
			}

			if schema {
				if el == "table" {
					lastTable = nil
				}
			} else {
				switch el {
				case "tabledata":
					lastTable = nil
				case "item":
					lastTable.data.Push(lastItem)
				}
			}
		}
	}
	if _, oser := err.(os.Error); err != nil && !oser {
		fmt.Printf("%T ", err)
		fmt.Println(err)
	}
}

func (e *XMLConnector) commit() {
	e.db.Seek(0, 0)
	e.db.Truncate(0)
	fmt.Fprintln(e.db, "<?xml version=\"1.0\" ?>")
	fmt.Fprintln(e.db, "<db>")
	fmt.Fprintln(e.db, "<schema>")
	for table, tabledesc := range e.tables {
		//		fmt.Println("Comitting table " + table + " struct " + fmt.Sprint(tabledesc))
		fmt.Fprintf(e.db, "
\n", table)
		for key, typ := range tabledesc.Attributes() {
			fmt.Fprintf(e.db, "<attribute type=\"%s\">%s</attribute>\n", typeName(typ), key)
		}
		fmt.Fprintln(e.db, "
")
	}
	fmt.Fprintln(e.db, "</schema>")
	fmt.Fprintln(e.db, "<data>")
	for table, tabledesc := range e.tables {
		fmt.Fprintf(e.db, "<tabledata name=\"%s\">\n", table)
		for i := 0; i < tabledesc.Data().Len(); i++ {
			values := tabledesc.Data().At(i).(map[string]Value)
			fmt.Fprintln(e.db, "<item>")

			for key, typ := range tabledesc.Attributes() {
				fmt.Fprintf(e.db, "<value name=\"%s\">", key)
				switch typ {
				case IntKind:
					fmt.Fprintf(e.db, "%d", int(values[key].Int()))
				case StringKind:
					xml.Escape(e.db, bytes.NewBufferString(string(values[key].String())).Bytes())
				}
				fmt.Fprintf(e.db, "</value>\n")
			}

			fmt.Fprintln(e.db, "</item>")
		}
		fmt.Fprintln(e.db, "</tabledata>")
	}
	fmt.Fprintln(e.db, "</data>")
	fmt.Fprint(e.db, "</db>")
}

//TODO type ValueKind ??
func typeName(kind int) string {
	switch kind {
	case IntKind:
		return "int"
	case StringKind:
		return "string"
	}
	return "UNKNOWN"
}

func typeFromName(kind string) int {
	switch kind {
	case "int":
		return IntKind
	case "string":
		return StringKind
	}
	return NullKind
}

func (e *XMLConnector) CreateTableWithoutId(table string) *XMLTable {
	tmp := new(XMLTable)
	e.tables[table] = tmp
	tmp.schema = make(map[string]int)
	tmp.data = new(vector.Vector)
	return tmp
}

func (e *XMLConnector) CreateTable(table string) *XMLTable {
	a := e.CreateTableWithoutId(table)
	a.AddAttribute("id", IntKind)
	return a
}

func (e *XMLConnector) Table(table string) *XMLTable {
	tab, ok := e.tables[table]
	if ok {
		return tab
	}
	return nil
}

func (e *XMLTable) AddAttribute(name string, typ int) {
	e.schema[name] = typ
}
func (e *XMLTable) Attributes() map[string]int {
	return e.schema
}
func (e *XMLTable) Data() *vector.Vector { return e.data }

//unprotected
func (e *XMLTable) Insert(val map[string]Value) {
	e.data.Push(val)
}

func (e *XMLConnector) Query(r *Relation) *vector.Vector {
	return new(vector.Vector)
}

func OpenXML(conStr string) Connection {
	db := (new(XMLConnector))
	db.tables = make(map[string]*XMLTable)
	db.Open(conStr)
	return db
}
<|endoftext|>


<commit_before>// Package middleware - cloudfront wrapper for storage libs
// N.B. currently only works with S3, not arbitrary sites
//
package middleware

import (
	"context"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"net/url"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/service/cloudfront/sign"
	dcontext "github.com/docker/distribution/context"
	storagedriver "github.com/docker/distribution/registry/storage/driver"
	"github.com/docker/distribution/registry/storage/driver/middleware"
)

// cloudFrontStorageMiddleware provides a simple implementation of layerHandler that
// constructs temporary signed CloudFront URLs from the storagedriver layer URL,
// then issues HTTP Temporary Redirects to this CloudFront content URL.
type cloudFrontStorageMiddleware struct {
	storagedriver.StorageDriver
	awsIPs    *awsIPs
	urlSigner *sign.URLSigner
	baseURL   string
	duration  time.Duration
}

var _ storagedriver.StorageDriver = &cloudFrontStorageMiddleware{}

// newCloudFrontLayerHandler constructs and returns a new CloudFront
// LayerHandler implementation.
// Required options: baseurl, privatekey, keypairid

// Optional options: ipFilteredBy, awsregion
// ipfilteredby: valid value "none|aws|awsregion". "none", do not filter any IP, default value. "aws", only aws IP goes
//               to S3 directly. "awsregion", only regions listed in awsregion options goes to S3 directly
// awsregion: a comma separated string of AWS regions.
func newCloudFrontStorageMiddleware(storageDriver storagedriver.StorageDriver, options map[string]interface{}) (storagedriver.StorageDriver, error) {
	// parse baseurl
	base, ok := options["baseurl"]
	if !ok {
		return nil, fmt.Errorf("no baseurl provided")
	}
	baseURL, ok := base.(string)
	if !ok {
		return nil, fmt.Errorf("baseurl must be a string")
	}
	if !strings.Contains(baseURL, "://") {
		baseURL = "https://" + baseURL
	}
	if !strings.HasSuffix(baseURL, "/") {
		baseURL += "/"
	}
	if _, err := url.Parse(baseURL); err != nil {
		return nil, fmt.Errorf("invalid baseurl: %v", err)
	}

	// parse privatekey to get pkPath
	pk, ok := options["privatekey"]
	if !ok {
		return nil, fmt.Errorf("no privatekey provided")
	}
	pkPath, ok := pk.(string)
	if !ok {
		return nil, fmt.Errorf("privatekey must be a string")
	}

	// parse keypairid
	kpid, ok := options["keypairid"]
	if !ok {
		return nil, fmt.Errorf("no keypairid provided")
	}
	keypairID, ok := kpid.(string)
	if !ok {
		return nil, fmt.Errorf("keypairid must be a string")
	}

	// get urlSigner from the file specified in pkPath
	pkBytes, err := ioutil.ReadFile(pkPath)
	if err != nil {
		return nil, fmt.Errorf("failed to read privatekey file: %s", err)
	}

	block, _ := pem.Decode(pkBytes)
	if block == nil {
		return nil, fmt.Errorf("failed to decode private key as an rsa private key")
	}
	privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
	if err != nil {
		return nil, err
	}
	urlSigner := sign.NewURLSigner(keypairID, privateKey)

	// parse duration
	duration := 20 * time.Minute
	if d, ok := options["duration"]; ok {
		switch d := d.(type) {
		case time.Duration:
			duration = d
		case string:
			dur, err := time.ParseDuration(d)
			if err != nil {
				return nil, fmt.Errorf("invalid duration: %s", err)
			}
			duration = dur
		}
	}

	// parse updatefrenquency
	updateFrequency := defaultUpdateFrequency
	if u, ok := options["updatefrenquency"]; ok {
		switch u := u.(type) {
		case time.Duration:
			updateFrequency = u
		case string:
			updateFreq, err := time.ParseDuration(u)
			if err != nil {
				return nil, fmt.Errorf("invalid updatefrenquency: %s", err)
			}
			duration = updateFreq
		}
	}

	// parse iprangesurl
	ipRangesURL := defaultIPRangesURL
	if i, ok := options["iprangesurl"]; ok {
		if iprangeurl, ok := i.(string); ok {
			ipRangesURL = iprangeurl
		} else {
			return nil, fmt.Errorf("iprangesurl must be a string")
		}
	}

	// parse ipfilteredby
	var awsIPs *awsIPs
	if ipFilteredBy := options["ipfilteredby"].(string); ok {
		switch strings.ToLower(strings.TrimSpace(ipFilteredBy)) {
		case "", "none":
			awsIPs = nil
		case "aws":
			newAWSIPs(ipRangesURL, updateFrequency, nil)
		case "awsregion":
			var awsRegion []string
			if regions, ok := options["awsregion"].(string); ok {
				for _, awsRegions := range strings.Split(regions, ",") {
					awsRegion = append(awsRegion, strings.ToLower(strings.TrimSpace(awsRegions)))
				}
				awsIPs = newAWSIPs(ipRangesURL, updateFrequency, awsRegion)
			} else {
				return nil, fmt.Errorf("awsRegion must be a comma separated string of valid aws regions")
			}
		default:
			return nil, fmt.Errorf("ipfilteredby only allows a string the following value: none|aws|awsregion")
		}
	} else {
		return nil, fmt.Errorf("ipfilteredby only allows a string with the following value: none|aws|awsregion")
	}

	return &cloudFrontStorageMiddleware{
		StorageDriver: storageDriver,
		urlSigner:     urlSigner,
		baseURL:       baseURL,
		duration:      duration,
		awsIPs:        awsIPs,
	}, nil
}

// S3BucketKeyer is any type that is capable of returning the S3 bucket key
// which should be cached by AWS CloudFront.
type S3BucketKeyer interface {
	S3BucketKey(path string) string
}

// URLFor attempts to find a url which may be used to retrieve the file at the given path.
// Returns an error if the file cannot be found.
func (lh *cloudFrontStorageMiddleware) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
	// TODO(endophage): currently only supports S3
	keyer, ok := lh.StorageDriver.(S3BucketKeyer)
	if !ok {
		dcontext.GetLogger(ctx).Warn("the CloudFront middleware does not support this backend storage driver")
		return lh.StorageDriver.URLFor(ctx, path, options)
	}

	if eligibleForS3(ctx, lh.awsIPs) {
		return lh.StorageDriver.URLFor(ctx, path, options)
	}

	// Get signed cloudfront url.
	cfURL, err := lh.urlSigner.Sign(lh.baseURL+keyer.S3BucketKey(path), time.Now().Add(lh.duration))
	if err != nil {
		return "", err
	}
	return cfURL, nil
}

// init registers the cloudfront layerHandler backend.
func init() {
	storagemiddleware.Register("cloudfront", storagemiddleware.InitFunc(newCloudFrontStorageMiddleware))
}
<commit_msg>Bugfix: Make ipfilteredby not required<commit_after>// Package middleware - cloudfront wrapper for storage libs
// N.B. currently only works with S3, not arbitrary sites
//
package middleware

import (
	"context"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"net/url"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/service/cloudfront/sign"
	dcontext "github.com/docker/distribution/context"
	storagedriver "github.com/docker/distribution/registry/storage/driver"
	"github.com/docker/distribution/registry/storage/driver/middleware"
)

// cloudFrontStorageMiddleware provides a simple implementation of layerHandler that
// constructs temporary signed CloudFront URLs from the storagedriver layer URL,
// then issues HTTP Temporary Redirects to this CloudFront content URL.
type cloudFrontStorageMiddleware struct {
	storagedriver.StorageDriver
	awsIPs    *awsIPs
	urlSigner *sign.URLSigner
	baseURL   string
	duration  time.Duration
}

var _ storagedriver.StorageDriver = &cloudFrontStorageMiddleware{}

// newCloudFrontLayerHandler constructs and returns a new CloudFront
// LayerHandler implementation.
// Required options: baseurl, privatekey, keypairid

// Optional options: ipFilteredBy, awsregion
// ipfilteredby: valid value "none|aws|awsregion". "none", do not filter any IP, default value. "aws", only aws IP goes
//               to S3 directly. "awsregion", only regions listed in awsregion options goes to S3 directly
// awsregion: a comma separated string of AWS regions.
func newCloudFrontStorageMiddleware(storageDriver storagedriver.StorageDriver, options map[string]interface{}) (storagedriver.StorageDriver, error) {
	// parse baseurl
	base, ok := options["baseurl"]
	if !ok {
		return nil, fmt.Errorf("no baseurl provided")
	}
	baseURL, ok := base.(string)
	if !ok {
		return nil, fmt.Errorf("baseurl must be a string")
	}
	if !strings.Contains(baseURL, "://") {
		baseURL = "https://" + baseURL
	}
	if !strings.HasSuffix(baseURL, "/") {
		baseURL += "/"
	}
	if _, err := url.Parse(baseURL); err != nil {
		return nil, fmt.Errorf("invalid baseurl: %v", err)
	}

	// parse privatekey to get pkPath
	pk, ok := options["privatekey"]
	if !ok {
		return nil, fmt.Errorf("no privatekey provided")
	}
	pkPath, ok := pk.(string)
	if !ok {
		return nil, fmt.Errorf("privatekey must be a string")
	}

	// parse keypairid
	kpid, ok := options["keypairid"]
	if !ok {
		return nil, fmt.Errorf("no keypairid provided")
	}
	keypairID, ok := kpid.(string)
	if !ok {
		return nil, fmt.Errorf("keypairid must be a string")
	}

	// get urlSigner from the file specified in pkPath
	pkBytes, err := ioutil.ReadFile(pkPath)
	if err != nil {
		return nil, fmt.Errorf("failed to read privatekey file: %s", err)
	}

	block, _ := pem.Decode(pkBytes)
	if block == nil {
		return nil, fmt.Errorf("failed to decode private key as an rsa private key")
	}
	privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
	if err != nil {
		return nil, err
	}
	urlSigner := sign.NewURLSigner(keypairID, privateKey)

	// parse duration
	duration := 20 * time.Minute
	if d, ok := options["duration"]; ok {
		switch d := d.(type) {
		case time.Duration:
			duration = d
		case string:
			dur, err := time.ParseDuration(d)
			if err != nil {
				return nil, fmt.Errorf("invalid duration: %s", err)
			}
			duration = dur
		}
	}

	// parse updatefrenquency
	updateFrequency := defaultUpdateFrequency
	if u, ok := options["updatefrenquency"]; ok {
		switch u := u.(type) {
		case time.Duration:
			updateFrequency = u
		case string:
			updateFreq, err := time.ParseDuration(u)
			if err != nil {
				return nil, fmt.Errorf("invalid updatefrenquency: %s", err)
			}
			duration = updateFreq
		}
	}

	// parse iprangesurl
	ipRangesURL := defaultIPRangesURL
	if i, ok := options["iprangesurl"]; ok {
		if iprangeurl, ok := i.(string); ok {
			ipRangesURL = iprangeurl
		} else {
			return nil, fmt.Errorf("iprangesurl must be a string")
		}
	}

	// parse ipfilteredby
	var awsIPs *awsIPs
	if i, ok := options["ipfilteredby"]; ok {
		if ipFilteredBy, ok := i.(string); ok {
			switch strings.ToLower(strings.TrimSpace(ipFilteredBy)) {
			case "", "none":
				awsIPs = nil
			case "aws":
				awsIPs = newAWSIPs(ipRangesURL, updateFrequency, nil)
			case "awsregion":
				var awsRegion []string
				if i, ok := options["awsregion"]; ok {
					if regions, ok := i.(string); ok {
						for _, awsRegions := range strings.Split(regions, ",") {
							awsRegion = append(awsRegion, strings.ToLower(strings.TrimSpace(awsRegions)))
						}
						awsIPs = newAWSIPs(ipRangesURL, updateFrequency, awsRegion)
					} else {
						return nil, fmt.Errorf("awsRegion must be a comma separated string of valid aws regions")
					}
				} else {
					return nil, fmt.Errorf("awsRegion is not defined")
				}
			default:
				return nil, fmt.Errorf("ipfilteredby only allows a string the following value: none|aws|awsregion")
			}
		} else {
			return nil, fmt.Errorf("ipfilteredby only allows a string with the following value: none|aws|awsregion")
		}
	}

	return &cloudFrontStorageMiddleware{
		StorageDriver: storageDriver,
		urlSigner:     urlSigner,
		baseURL:       baseURL,
		duration:      duration,
		awsIPs:        awsIPs,
	}, nil
}

// S3BucketKeyer is any type that is capable of returning the S3 bucket key
// which should be cached by AWS CloudFront.
type S3BucketKeyer interface {
	S3BucketKey(path string) string
}

// URLFor attempts to find a url which may be used to retrieve the file at the given path.
// Returns an error if the file cannot be found.
func (lh *cloudFrontStorageMiddleware) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
	// TODO(endophage): currently only supports S3
	keyer, ok := lh.StorageDriver.(S3BucketKeyer)
	if !ok {
		dcontext.GetLogger(ctx).Warn("the CloudFront middleware does not support this backend storage driver")
		return lh.StorageDriver.URLFor(ctx, path, options)
	}

	if eligibleForS3(ctx, lh.awsIPs) {
		return lh.StorageDriver.URLFor(ctx, path, options)
	}

	// Get signed cloudfront url.
	cfURL, err := lh.urlSigner.Sign(lh.baseURL+keyer.S3BucketKey(path), time.Now().Add(lh.duration))
	if err != nil {
		return "", err
	}
	return cfURL, nil
}

// init registers the cloudfront layerHandler backend.
func init() {
	storagemiddleware.Register("cloudfront", storagemiddleware.InitFunc(newCloudFrontStorageMiddleware))
}
<|endoftext|>


<commit_before>package profiles

import (
	"log"
	"os"
	"strconv"
	"sync"
	"sync/atomic"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

/*
	basic insert into capped collection only profile, insert a doc of 400 bytes
*/

type cappedCollInsertProfile struct {
	UID int64

	indexGroup bool

	initProfile sync.Once

	session *mgo.Session
}

var _cappedCollInsertProfile cappedCollInsertProfile

// func Int2ObjId(i int64) string {
// 	// return string represenation of UID
// }

func (i cappedCollInsertProfile) SendNext(s *mgo.Session, worker_id int) error {
	var err error
	c := s.DB(getDBName(default_db_name_prefix)).C(getCollectionName(default_col_name_prefix))

	_u := atomic.AddInt64(&_cappedCollInsertProfile.UID, 1) // to make this unique

	doc := bson.M{
		// "_id":      _u,
		"name":     _u,
		"group":    rands[worker_id].Int(),
		"payload":  &Payload,
		"payload1": &Payload1,
		"payload2": &Payload2,
		"payload3": &Payload3,
		"payload4": &Payload4,
		"payload5": &Payload5,
		"payload6": &Payload6}

	if _profile_use_legacy_write {
		err = c.Insert(doc)
	} else {
		var results interface{}

		err = c.Database.Run(bson.D{{"insert", c.Name},
			{"documents", []bson.M{doc}}}, results)
	}

	panicOnError(err)
	return err
}

func initCappedTest(session *mgo.Session, _initdb bool) {
	log.Println("Initialize simple DB. with initdb flag ", _initdb)

	ci := new(mgo.CollectionInfo)
	ci.Capped = true
	ci.MaxBytes = 1000000000

	_initdb = true

	log.Println(". Init DB, drop collections")
	session.DB(default_db_name_prefix).C(default_col_name_prefix).DropCollection()
	session.DB(default_db_name_prefix).C(default_col_name_prefix).Create(ci)

	for i := 1; i < _multi_db; i++ {
		for j := 1; j < _multi_col; j++ {
			collection := session.DB(default_db_name_prefix + strconv.Itoa(i)).C(default_col_name_prefix + strconv.Itoa(j))
			collection.Create(ci)
			err := collection.EnsureIndexKey("name")
			if err != nil {
				panic(err)
			}

			err = collection.EnsureIndexKey("uid")
			if err != nil {
				panic(err)
			}
		}
	}

}

func (i cappedCollInsertProfile) SetupTest(s *mgo.Session, _initdb bool) error {
	i.session = s

	f := func() {
		initCappedTest(s, _initdb)
	}

	_cappedCollInsertProfile.initProfile.Do(f)
	return nil
}

func (i cappedCollInsertProfile) CsvString(total_time float64) string {
	return ""
}

func (i cappedCollInsertProfile) CsvHeader() string {
	return ""
}

func init() {
	atomic.StoreInt64(&_cappedCollInsertProfile.UID, -1) // UID starts with 1

	registerProfile("CAPPED_COLL_INSERT", func() Profile {
		return Profile(_cappedCollInsertProfile) // use the same instance
	})

	s := os.Getenv("HT_INDEX_GROUP")
	if s == "" {
		_cappedCollInsertProfile.indexGroup = false
	} else {
		_cappedCollInsertProfile.indexGroup = true
	}
}
<commit_msg>remove collection if exists<commit_after>package profiles

import (
	"log"
	"os"
	"strconv"
	"sync"
	"sync/atomic"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

/*
	basic insert into capped collection only profile, insert a doc of 400 bytes
*/

type cappedCollInsertProfile struct {
	UID int64

	indexGroup bool

	initProfile sync.Once

	session *mgo.Session
}

var _cappedCollInsertProfile cappedCollInsertProfile

// func Int2ObjId(i int64) string {
// 	// return string represenation of UID
// }

func (i cappedCollInsertProfile) SendNext(s *mgo.Session, worker_id int) error {
	var err error
	c := s.DB(getDBName(default_db_name_prefix)).C(getCollectionName(default_col_name_prefix))

	_u := atomic.AddInt64(&_cappedCollInsertProfile.UID, 1) // to make this unique

	doc := bson.M{
		// "_id":      _u,
		"name":     _u,
		"group":    rands[worker_id].Int(),
		"payload":  &Payload,
		"payload1": &Payload1,
		"payload2": &Payload2,
		"payload3": &Payload3,
		"payload4": &Payload4,
		"payload5": &Payload5,
		"payload6": &Payload6}

	if _profile_use_legacy_write {
		err = c.Insert(doc)
	} else {
		var results interface{}

		err = c.Database.Run(bson.D{{"insert", c.Name},
			{"documents", []bson.M{doc}}}, results)
	}

	panicOnError(err)
	return err
}

func initCappedTest(session *mgo.Session, _initdb bool) {
	log.Println("Initialize simple DB. with initdb flag ", _initdb)

	ci := new(mgo.CollectionInfo)
	ci.Capped = true
	ci.MaxBytes = 1000000000

	_initdb = true

	log.Println(". Init DB, drop collections")
	session.DB(default_db_name_prefix).C(default_col_name_prefix).DropCollection()
	session.DB(default_db_name_prefix).C(default_col_name_prefix).Create(ci)

	for i := 1; i < _multi_db; i++ {
		for j := 1; j < _multi_col; j++ {
			collection := session.DB(default_db_name_prefix + strconv.Itoa(i)).C(default_col_name_prefix + strconv.Itoa(j))
			collection.DropCollection()
			collection.Create(ci)
			err := collection.EnsureIndexKey("name")
			if err != nil {
				panic(err)
			}

			err = collection.EnsureIndexKey("uid")
			if err != nil {
				panic(err)
			}
		}
	}

}

func (i cappedCollInsertProfile) SetupTest(s *mgo.Session, _initdb bool) error {
	i.session = s

	f := func() {
		initCappedTest(s, _initdb)
	}

	_cappedCollInsertProfile.initProfile.Do(f)
	return nil
}

func (i cappedCollInsertProfile) CsvString(total_time float64) string {
	return ""
}

func (i cappedCollInsertProfile) CsvHeader() string {
	return ""
}

func init() {
	atomic.StoreInt64(&_cappedCollInsertProfile.UID, -1) // UID starts with 1

	registerProfile("CAPPED_COLL_INSERT", func() Profile {
		return Profile(_cappedCollInsertProfile) // use the same instance
	})

	s := os.Getenv("HT_INDEX_GROUP")
	if s == "" {
		_cappedCollInsertProfile.indexGroup = false
	} else {
		_cappedCollInsertProfile.indexGroup = true
	}
}
<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package thrift

import (
	"crypto/tls"
	"net"
	"time"
)

type TSSLSocket struct {
	conn    net.Conn
	addr    net.Addr
	timeout time.Duration
	cfg     *tls.Config
}

// NewTSSLSocket creates a net.Conn-backed TTransport, given a host and port and tls Configuration
//
// Example:
// 	trans, err := thrift.NewTSocket("localhost:9090")
func NewTSSLSocket(hostPort string, cfg *tls.Config) (*TSSLSocket, error) {
	return NewTSSLSocketTimeout(hostPort, cfg, 0)
}

// NewTSSLSocketTimeout creates a net.Conn-backed TTransport, given a host and port
// it also accepts a tls Configuration and a timeout as a time.Duration
func NewTSSLSocketTimeout(hostPort string, cfg *tls.Config, timeout time.Duration) (*TSSLSocket, error) {
	//conn, err := net.DialTimeout(network, address, timeout)
	addr, err := net.ResolveTCPAddr("tcp", hostPort)
	if err != nil {
		return nil, err
	}
	return NewTSSLSocketFromAddrTimeout(addr, cfg, timeout), nil
}

// Creates a TSSLSocket from a net.Addr
func NewTSSLSocketFromAddrTimeout(addr net.Addr, cfg *tls.Config, timeout time.Duration) *TSSLSocket {
	return &TSSLSocket{addr: addr, timeout: timeout, cfg: cfg}
}

// Creates a TSSLSocket from an existing net.Conn
func NewTSSLSocketFromConnTimeout(conn net.Conn, cfg *tls.Config, timeout time.Duration) *TSSLSocket {
	return &TSSLSocket{conn: conn, addr: conn.RemoteAddr(), timeout: timeout, cfg: cfg}
}

// Sets the socket timeout
func (p *TSSLSocket) SetTimeout(timeout time.Duration) error {
	p.timeout = timeout
	return nil
}

func (p *TSSLSocket) pushDeadline(read, write bool) {
	var t time.Time
	if p.timeout > 0 {
		t = time.Now().Add(time.Duration(p.timeout))
	}
	if read && write {
		p.conn.SetDeadline(t)
	} else if read {
		p.conn.SetReadDeadline(t)
	} else if write {
		p.conn.SetWriteDeadline(t)
	}
}

// Connects the socket, creating a new socket object if necessary.
func (p *TSSLSocket) Open() error {
	if p.IsOpen() {
		return NewTTransportException(ALREADY_OPEN, "Socket already connected.")
	}
	if p.addr == nil {
		return NewTTransportException(NOT_OPEN, "Cannot open nil address.")
	}
	if len(p.addr.Network()) == 0 {
		return NewTTransportException(NOT_OPEN, "Cannot open bad network name.")
	}
	if len(p.addr.String()) == 0 {
		return NewTTransportException(NOT_OPEN, "Cannot open bad address.")
	}
	var err error
	if p.conn, err = tls.Dial(p.addr.Network(), p.addr.String(), p.cfg); err != nil {
		return NewTTransportException(NOT_OPEN, err.Error())
	}
	return nil
}

// Retrieve the underlying net.Conn
func (p *TSSLSocket) Conn() net.Conn {
	return p.conn
}

// Returns true if the connection is open
func (p *TSSLSocket) IsOpen() bool {
	if p.conn == nil {
		return false
	}
	return true
}

// Closes the socket.
func (p *TSSLSocket) Close() error {
	// Close the socket
	if p.conn != nil {
		err := p.conn.Close()
		if err != nil {
			return err
		}
		p.conn = nil
	}
	return nil
}

func (p *TSSLSocket) Read(buf []byte) (int, error) {
	if !p.IsOpen() {
		return 0, NewTTransportException(NOT_OPEN, "Connection not open")
	}
	p.pushDeadline(true, false)
	n, err := p.conn.Read(buf)
	return n, NewTTransportExceptionFromError(err)
}

func (p *TSSLSocket) Write(buf []byte) (int, error) {
	if !p.IsOpen() {
		return 0, NewTTransportException(NOT_OPEN, "Connection not open")
	}
	p.pushDeadline(false, true)
	return p.conn.Write(buf)
}

func (p *TSSLSocket) Flush() error {
	return nil
}

func (p *TSSLSocket) Interrupt() error {
	if !p.IsOpen() {
		return nil
	}
	return p.conn.Close()
}
<commit_msg>THRIFT-3009 TSSLSocket does not use the correct hostname (breaks certificate checks) Client: Go Patch: Mathias Gottschlag <mgottschlag@gmail.com><commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package thrift

import (
	"crypto/tls"
	"net"
	"time"
)

type TSSLSocket struct {
	conn net.Conn
	// hostPort contains host:port (e.g. "asdf.com:12345"). The field is
	// only valid if addr is nil.
	hostPort string
	// addr is nil when hostPort is not "", and is only used when the
	// TSSLSocket is constructed from a net.Addr.
	addr    net.Addr
	timeout time.Duration
	cfg     *tls.Config
}

// NewTSSLSocket creates a net.Conn-backed TTransport, given a host and port and tls Configuration
//
// Example:
// 	trans, err := thrift.NewTSSLSocket("localhost:9090", nil)
func NewTSSLSocket(hostPort string, cfg *tls.Config) (*TSSLSocket, error) {
	return NewTSSLSocketTimeout(hostPort, cfg, 0)
}

// NewTSSLSocketTimeout creates a net.Conn-backed TTransport, given a host and port
// it also accepts a tls Configuration and a timeout as a time.Duration
func NewTSSLSocketTimeout(hostPort string, cfg *tls.Config, timeout time.Duration) (*TSSLSocket, error) {
	return &TSSLSocket{hostPort: hostPort, timeout: timeout, cfg: cfg}, nil
}

// Creates a TSSLSocket from a net.Addr
func NewTSSLSocketFromAddrTimeout(addr net.Addr, cfg *tls.Config, timeout time.Duration) *TSSLSocket {
	return &TSSLSocket{addr: addr, timeout: timeout, cfg: cfg}
}

// Creates a TSSLSocket from an existing net.Conn
func NewTSSLSocketFromConnTimeout(conn net.Conn, cfg *tls.Config, timeout time.Duration) *TSSLSocket {
	return &TSSLSocket{conn: conn, addr: conn.RemoteAddr(), timeout: timeout, cfg: cfg}
}

// Sets the socket timeout
func (p *TSSLSocket) SetTimeout(timeout time.Duration) error {
	p.timeout = timeout
	return nil
}

func (p *TSSLSocket) pushDeadline(read, write bool) {
	var t time.Time
	if p.timeout > 0 {
		t = time.Now().Add(time.Duration(p.timeout))
	}
	if read && write {
		p.conn.SetDeadline(t)
	} else if read {
		p.conn.SetReadDeadline(t)
	} else if write {
		p.conn.SetWriteDeadline(t)
	}
}

// Connects the socket, creating a new socket object if necessary.
func (p *TSSLSocket) Open() error {
	var err error
	// If we have a hostname, we need to pass the hostname to tls.Dial for
	// certificate hostname checks.
	if p.hostPort != "" {
		if p.conn, err = tls.Dial("tcp", p.hostPort, p.cfg); err != nil {
			return NewTTransportException(NOT_OPEN, err.Error())
		}
	} else {
		if p.IsOpen() {
			return NewTTransportException(ALREADY_OPEN, "Socket already connected.")
		}
		if p.addr == nil {
			return NewTTransportException(NOT_OPEN, "Cannot open nil address.")
		}
		if len(p.addr.Network()) == 0 {
			return NewTTransportException(NOT_OPEN, "Cannot open bad network name.")
		}
		if len(p.addr.String()) == 0 {
			return NewTTransportException(NOT_OPEN, "Cannot open bad address.")
		}
		if p.conn, err = tls.Dial(p.addr.Network(), p.addr.String(), p.cfg); err != nil {
			return NewTTransportException(NOT_OPEN, err.Error())
		}
	}
	return nil
}

// Retrieve the underlying net.Conn
func (p *TSSLSocket) Conn() net.Conn {
	return p.conn
}

// Returns true if the connection is open
func (p *TSSLSocket) IsOpen() bool {
	if p.conn == nil {
		return false
	}
	return true
}

// Closes the socket.
func (p *TSSLSocket) Close() error {
	// Close the socket
	if p.conn != nil {
		err := p.conn.Close()
		if err != nil {
			return err
		}
		p.conn = nil
	}
	return nil
}

func (p *TSSLSocket) Read(buf []byte) (int, error) {
	if !p.IsOpen() {
		return 0, NewTTransportException(NOT_OPEN, "Connection not open")
	}
	p.pushDeadline(true, false)
	n, err := p.conn.Read(buf)
	return n, NewTTransportExceptionFromError(err)
}

func (p *TSSLSocket) Write(buf []byte) (int, error) {
	if !p.IsOpen() {
		return 0, NewTTransportException(NOT_OPEN, "Connection not open")
	}
	p.pushDeadline(false, true)
	return p.conn.Write(buf)
}

func (p *TSSLSocket) Flush() error {
	return nil
}

func (p *TSSLSocket) Interrupt() error {
	if !p.IsOpen() {
		return nil
	}
	return p.conn.Close()
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"archive/tar"
	"bufio"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/hyperledger/fabric/common/flogging"
)

var vmLogger = flogging.MustGetLogger("container")

// These filetypes are excluded while creating the tar package sent to Docker
// Generated .class and other temporary files can be excluded
var javaExcludeFileTypes = map[string]bool{
	".class": true,
}

func WriteFolderToTarPackage(tw *tar.Writer, srcPath string, excludeDir string, includeFileTypeMap map[string]bool, excludeFileTypeMap map[string]bool) error {
	rootDirectory := srcPath
	vmLogger.Infof("rootDirectory = %s", rootDirectory)

	//append "/" if necessary
	if excludeDir != "" && strings.LastIndex(excludeDir, "/") < len(excludeDir)-1 {
		excludeDir = excludeDir + "/"
	}

	rootDirLen := len(rootDirectory)
	walkFn := func(path string, info os.FileInfo, err error) error {

		// If path includes .git, ignore
		if strings.Contains(path, ".git") {
			return nil
		}

		if info.Mode().IsDir() {
			return nil
		}

		//exclude any files with excludeDir prefix. They should already be in the tar
		if excludeDir != "" && strings.Index(path, excludeDir) == rootDirLen+1 {
			//1 for "/"
			return nil
		}
		// Because of scoping we can reference the external rootDirectory variable
		if len(path[rootDirLen:]) == 0 {
			return nil
		}
		ext := filepath.Ext(path)

		if includeFileTypeMap != nil {
			// we only want 'fileTypes' source files at this point
			if _, ok := includeFileTypeMap[ext]; ok != true {
				return nil
			}
		}

		//exclude the given file types
		if excludeFileTypeMap != nil {
			if exclude, ok := excludeFileTypeMap[ext]; ok && exclude {
				return nil
			}
		}

		newPath := fmt.Sprintf("src%s", path[rootDirLen:])
		//newPath := path[len(rootDirectory):]

		err = WriteFileToPackage(path, newPath, tw)
		if err != nil {
			return fmt.Errorf("Error writing file to package: %s", err)
		}

		vmLogger.Debugf("Writing file %s to tar", newPath)

		return nil
	}

	if err := filepath.Walk(rootDirectory, walkFn); err != nil {
		vmLogger.Infof("Error walking rootDirectory: %s", err)
		return err
	}
	return nil
}

//Package Java project to tar file from the source path
func WriteJavaProjectToPackage(tw *tar.Writer, srcPath string) error {

	vmLogger.Debugf("Packaging Java project from path %s", srcPath)

	if err := WriteFolderToTarPackage(tw, srcPath, "", nil, javaExcludeFileTypes); err != nil {

		vmLogger.Errorf("Error writing folder to tar package %s", err)
		return err
	}
	// Write the tar file out
	if err := tw.Close(); err != nil {
		return err
	}
	return nil

}

//WriteFileToPackage writes a file to the tarball
func WriteFileToPackage(localpath string, packagepath string, tw *tar.Writer) error {
	fd, err := os.Open(localpath)
	if err != nil {
		return fmt.Errorf("%s: %s", localpath, err)
	}
	defer fd.Close()

	is := bufio.NewReader(fd)
	return WriteStreamToPackage(is, localpath, packagepath, tw)

}

//WriteStreamToPackage writes bytes (from a file reader) to the tarball
func WriteStreamToPackage(is io.Reader, localpath string, packagepath string, tw *tar.Writer) error {
	info, err := os.Stat(localpath)
	if err != nil {
		return fmt.Errorf("%s: %s", localpath, err)
	}
	header, err := tar.FileInfoHeader(info, localpath)
	if err != nil {
		return fmt.Errorf("Error getting FileInfoHeader: %s", err)
	}

	//Let's take the variance out of the tar, make headers identical by using zero time
	oldname := header.Name
	var zeroTime time.Time
	header.AccessTime = zeroTime
	header.ModTime = zeroTime
	header.ChangeTime = zeroTime
	header.Name = packagepath
	header.Mode = 0100644

	if err = tw.WriteHeader(header); err != nil {
		return fmt.Errorf("Error write header for (path: %s, oldname:%s,newname:%s,sz:%d) : %s", localpath, oldname, packagepath, header.Size, err)
	}
	if _, err := io.Copy(tw, is); err != nil {
		return fmt.Errorf("Error copy (path: %s, oldname:%s,newname:%s,sz:%d) : %s", localpath, oldname, packagepath, header.Size, err)
	}

	return nil
}

func WriteBytesToPackage(name string, payload []byte, tw *tar.Writer) error {
	//Make headers identical by using zero time
	var zeroTime time.Time
	tw.WriteHeader(&tar.Header{Name: name, Size: int64(len(payload)), ModTime: zeroTime, AccessTime: zeroTime, ChangeTime: zeroTime})
	tw.Write(payload)

	return nil
}
<commit_msg>[FAB-7434] Normalize UID/GID in cc packages<commit_after>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"archive/tar"
	"bufio"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/hyperledger/fabric/common/flogging"
)

var vmLogger = flogging.MustGetLogger("container")

// These filetypes are excluded while creating the tar package sent to Docker
// Generated .class and other temporary files can be excluded
var javaExcludeFileTypes = map[string]bool{
	".class": true,
}

func WriteFolderToTarPackage(tw *tar.Writer, srcPath string, excludeDir string, includeFileTypeMap map[string]bool, excludeFileTypeMap map[string]bool) error {
	rootDirectory := srcPath
	vmLogger.Infof("rootDirectory = %s", rootDirectory)

	//append "/" if necessary
	if excludeDir != "" && strings.LastIndex(excludeDir, "/") < len(excludeDir)-1 {
		excludeDir = excludeDir + "/"
	}

	rootDirLen := len(rootDirectory)
	walkFn := func(path string, info os.FileInfo, err error) error {

		// If path includes .git, ignore
		if strings.Contains(path, ".git") {
			return nil
		}

		if info.Mode().IsDir() {
			return nil
		}

		//exclude any files with excludeDir prefix. They should already be in the tar
		if excludeDir != "" && strings.Index(path, excludeDir) == rootDirLen+1 {
			//1 for "/"
			return nil
		}
		// Because of scoping we can reference the external rootDirectory variable
		if len(path[rootDirLen:]) == 0 {
			return nil
		}
		ext := filepath.Ext(path)

		if includeFileTypeMap != nil {
			// we only want 'fileTypes' source files at this point
			if _, ok := includeFileTypeMap[ext]; ok != true {
				return nil
			}
		}

		//exclude the given file types
		if excludeFileTypeMap != nil {
			if exclude, ok := excludeFileTypeMap[ext]; ok && exclude {
				return nil
			}
		}

		newPath := fmt.Sprintf("src%s", path[rootDirLen:])
		//newPath := path[len(rootDirectory):]

		err = WriteFileToPackage(path, newPath, tw)
		if err != nil {
			return fmt.Errorf("Error writing file to package: %s", err)
		}

		vmLogger.Debugf("Writing file %s to tar", newPath)

		return nil
	}

	if err := filepath.Walk(rootDirectory, walkFn); err != nil {
		vmLogger.Infof("Error walking rootDirectory: %s", err)
		return err
	}
	return nil
}

//Package Java project to tar file from the source path
func WriteJavaProjectToPackage(tw *tar.Writer, srcPath string) error {

	vmLogger.Debugf("Packaging Java project from path %s", srcPath)

	if err := WriteFolderToTarPackage(tw, srcPath, "", nil, javaExcludeFileTypes); err != nil {

		vmLogger.Errorf("Error writing folder to tar package %s", err)
		return err
	}
	// Write the tar file out
	if err := tw.Close(); err != nil {
		return err
	}
	return nil

}

//WriteFileToPackage writes a file to the tarball
func WriteFileToPackage(localpath string, packagepath string, tw *tar.Writer) error {
	fd, err := os.Open(localpath)
	if err != nil {
		return fmt.Errorf("%s: %s", localpath, err)
	}
	defer fd.Close()

	is := bufio.NewReader(fd)
	return WriteStreamToPackage(is, localpath, packagepath, tw)

}

//WriteStreamToPackage writes bytes (from a file reader) to the tarball
func WriteStreamToPackage(is io.Reader, localpath string, packagepath string, tw *tar.Writer) error {
	info, err := os.Stat(localpath)
	if err != nil {
		return fmt.Errorf("%s: %s", localpath, err)
	}
	header, err := tar.FileInfoHeader(info, localpath)
	if err != nil {
		return fmt.Errorf("Error getting FileInfoHeader: %s", err)
	}

	//Let's take the variance out of the tar, make headers identical by using zero time
	oldname := header.Name
	var zeroTime time.Time
	header.AccessTime = zeroTime
	header.ModTime = zeroTime
	header.ChangeTime = zeroTime
	header.Name = packagepath
	header.Mode = 0100644
	header.Uid = 500
	header.Gid = 500

	if err = tw.WriteHeader(header); err != nil {
		return fmt.Errorf("Error write header for (path: %s, oldname:%s,newname:%s,sz:%d) : %s", localpath, oldname, packagepath, header.Size, err)
	}
	if _, err := io.Copy(tw, is); err != nil {
		return fmt.Errorf("Error copy (path: %s, oldname:%s,newname:%s,sz:%d) : %s", localpath, oldname, packagepath, header.Size, err)
	}

	return nil
}

func WriteBytesToPackage(name string, payload []byte, tw *tar.Writer) error {
	//Make headers identical by using zero time
	var zeroTime time.Time
	tw.WriteHeader(&tar.Header{Name: name, Size: int64(len(payload)), ModTime: zeroTime, AccessTime: zeroTime, ChangeTime: zeroTime})
	tw.Write(payload)

	return nil
}
<|endoftext|>


<commit_before>package Mongo

import (
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// // Find - return all documents that match the query
// func Find(collection *mgo.Collection, query bson.M, queryOptions QueryOptions) ([]bson.M, error) {
// 	var m []bson.M
// 	if queryOptions.Sort == "" {
// 		err := collection.Find(query).Skip(queryOptions.Skip).Limit(queryOptions.Limit).Select(queryOptions.Projection).All(&m)
// 		return m, err
// 	}
//
// 	err := collection.Find(query).Sort(queryOptions.Sort).Skip(queryOptions.Skip).Limit(queryOptions.Limit).Select(queryOptions.Projection).All(&m)
// 	return m, err
// }

// Find - return all documents that match the query
func Find(collection *mgo.Collection, query bson.M, queryOptions QueryOptions) *mgo.Query {
	if queryOptions.Sort == "" {
		return collection.Find(query).Skip(queryOptions.Skip).Limit(queryOptions.Limit).Select(queryOptions.Projection)
		//	return m, err
	}

	return collection.Find(query).Sort(queryOptions.Sort).Skip(queryOptions.Skip).Limit(queryOptions.Limit).Select(queryOptions.Projection)
	//	return m, err
}

// // FindByID returns the document by it's bson id
// func FindByID(collection *mgo.Collection, documentID string) (bson.M, error) {
// 	var m bson.M
// 	err := collection.FindId(documentID).One(&m)
// 	return m, err
// }

// FindByID returns the document by it's bson id
func FindByID(collection *mgo.Collection, documentID string) *mgo.Query {
	//	var m bson.M
	return collection.FindId(documentID)
	//	return m, err
}

// // FindOne returns the first document found
// func FindOne(collection *mgo.Collection, query bson.M) (bson.M, error) {
// 	var m bson.M
// 	err := collection.Find(query).One(&m)
// 	return m, err
// }

// FindOne returns the first document found
func FindOne(collection *mgo.Collection, query bson.M) *mgo.Query {
	//	var m bson.M
	return collection.Find(query)
	//return m, err
}

// Count returns the count from a query
func Count(collection *mgo.Collection, query bson.M) (int, error) {
	return collection.Find(query).Count()
}

// Distinct returns list of unique results
func Distinct(collection *mgo.Collection, query bson.M, propertyName string, results interface{}) (interface{}, error) {
	err := collection.Find(query).Distinct(propertyName, &results)
	return results, err
}
<commit_msg>updated repo<commit_after>package Mongo

import (
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// Find - return all documents that match the query
func Find(collection *mgo.Collection, query bson.M, queryOptions QueryOptions) *mgo.Query {
	if queryOptions.Sort == "" {
		return collection.Find(query).Skip(queryOptions.Skip).Limit(queryOptions.Limit).Select(queryOptions.Projection)
	}

	return collection.Find(query).Sort(queryOptions.Sort).Skip(queryOptions.Skip).Limit(queryOptions.Limit).Select(queryOptions.Projection)
}

// FindByID returns the document by it's bson id
func FindByID(collection *mgo.Collection, documentID string) *mgo.Query {
	return collection.FindId(documentID)
}

// FindOne returns the first document found
func FindOne(collection *mgo.Collection, query bson.M) *mgo.Query {
	return collection.Find(query)
}

// Count returns the count from a query
func Count(collection *mgo.Collection, query bson.M) (int, error) {
	return collection.Find(query).Count()
}

// Distinct returns list of unique results
func Distinct(collection *mgo.Collection, query bson.M, propertyName string, results interface{}) (interface{}, error) {
	err := collection.Find(query).Distinct(propertyName, &results)
	return results, err
}
<|endoftext|>


<commit_before>// dsrt.go -- directoy sort in reverse date order.  IE, newest is first.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"runtime"
	"sort"
	"strconv"
	"strings"
)

const lastCompiled = "21 May 17"

/*
Revision History
----------------
20 Apr 17 -- Started writing dsize rtn, based on dirlist.go
21 Apr 17 -- Now tweaking the output format.  And used flag package.  One as a pointer and one as a value, just to learn them.
22 Apr 17 -- Coded the use of the first non flag commandline param,  which is all I need.  Note that the flag must appear before the non-flag param, else the flag is ignored.
22 Apr 17 -- Now writing dsrt, to function similarly to dsort.
24 Apr 17 -- Now adding file matching, like "dir" or "ls" does.
25 Apr 17 -- Now adding sort by size as an option, like -s, and commas
26 Apr 17 -- Noticed that the match routine is case sensitive.  I don't like that.
27 Apr 17 -- commandline now allows a file spec.  I intend this for Windows.  I'll see how it goes.
19 May 19 -- Will now show the uid:gid for linux.
20 May 19 -- Turns out that (*syscall.Stat_t) only compiles on linux. Time for platform specific code.
21 May 19 -- Cross compiling to GOARCH=386, and the uid and User routines won't work. So I'm dealing with that.
*/

// FIS is a FileInfo slice, as in os.FileInfo
type FISlice []os.FileInfo
type FISliceDate []os.FileInfo
type FISliceSize []os.FileInfo

func (f FISliceDate) Less(i, j int) bool {
	return f[i].ModTime().UnixNano() > f[j].ModTime().UnixNano() // I want a reverse sort, newest first
}

func (f FISliceDate) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (f FISliceDate) Len() int {
	return len(f)
}

func (f FISliceSize) Less(i, j int) bool {
	return f[i].Size() > f[j].Size() // I want a reverse sort, largest first
}

func (f FISliceSize) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (f FISliceSize) Len() int {
	return len(f)
}

func main() {
	const numlines = 50
	var userptr *user.User
	var files FISlice
	var filesDate FISliceDate
	var filesSize FISliceSize
	var err error
	var count int

	var revflag = flag.Bool("r", false, "reverse the sort, ie, oldest or smallest is first") // Ptr

	var RevFlag bool
	flag.BoolVar(&RevFlag, "R", false, "Reverse the sort, ie, oldest or smallest is first") // Value

	var nlines = flag.Int("n", numlines, "number of lines to display") // Ptr

	var NLines int
	flag.IntVar(&NLines, "N", numlines, "number of lines to display") // Value

	var helpflag = flag.Bool("h", false, "print help message") // pointer
	var HelpFlag bool
	flag.BoolVar(&HelpFlag, "H", false, "print help message")

	var sizeflag = flag.Bool("s", false, "sort by size instead of by date") // pointer
	var SizeFlag bool
	flag.BoolVar(&SizeFlag, "S", false, "sort by size instead of by date")
	flag.Parse()

	fmt.Println(" dsrt will display sorted by date or size.  Written in Go.  LastCompiled ", lastCompiled)
	fmt.Println()

	uid := 0
	gid := 0
	systemStr := ""
	linuxflag := runtime.GOOS == "linux"
	if linuxflag {
		systemStr = "Linux"
	} else if runtime.GOOS == "windows" {
		systemStr = "Windows"
	} else {
		systemStr = "Mac, maybe"
	}

	if runtime.GOARCH == "amd64" {
		uid = os.Getuid() // int
		gid = os.Getgid() // int
		userptr, err = user.Current()
		if err != nil {
			fmt.Println(" user.Current error is ", err, "Exiting.")
			os.Exit(1)
		}
	}

	if *helpflag || HelpFlag {
		flag.PrintDefaults()
		if runtime.GOARCH == "amd64" {
			fmt.Printf("uid=%d, gid=%d, on a computer running %s for %s:%s Username %s, Name %s, HomeDir %s \n",
				uid, gid, systemStr, userptr.Uid, userptr.Gid, userptr.Username, userptr.Name, userptr.HomeDir)
		}

	}

	Reverse := *revflag || RevFlag
	SizeSort := *sizeflag || SizeFlag

	NumLines := numlines
	if *nlines != numlines {
		NumLines = *nlines
	} else if NLines != numlines {
		NumLines = NLines
	}
	askforinput := true

	CleanDirName := "." + string(filepath.Separator)
	CleanFileName := ""
	commandline := flag.Arg(0) // this only gets the first non flag argument.  That's all I want
	if len(commandline) > 0 {
		//		CleanDirName = filepath.Clean(commandline)
		CleanDirName, CleanFileName = filepath.Split(commandline)
		CleanFileName = strings.ToUpper(CleanFileName)
		askforinput = false
	}

	if askforinput {
		// Asking for input so don't have to worry about command line globbing
		fmt.Print(" Enter input for globbing: ")
		scanner := bufio.NewScanner(os.Stdin)
		scanner.Scan()
		newtext := scanner.Text()
		if err = scanner.Err(); err != nil {
			fmt.Fprintln(os.Stderr, " reading std input: ", err)
			os.Exit(1)
		}
		if len(newtext) > 0 {
			// time to do the stuff I'm writing this pgm for
			CleanDirName, CleanFileName = filepath.Split(newtext)
			CleanFileName = strings.ToUpper(CleanFileName)
		}

	}

	if len(CleanDirName) == 0 {
		CleanDirName = "." + string(filepath.Separator)
	}

	if len(CleanFileName) == 0 {
		CleanFileName = "*"
	}

	if SizeSort {
		filesSize, err = ioutil.ReadDir(CleanDirName)
		if err != nil {
			log.Fatal(err)
		}
		if Reverse {
			sort.Sort(sort.Reverse(filesSize))
		} else {
			sort.Sort(filesSize)
		}
		files = FISlice(filesSize)
	} else {
		filesDate, err = ioutil.ReadDir(CleanDirName)
		if err != nil {
			log.Fatal(err)
		}
		if Reverse {
			sort.Sort(sort.Reverse(filesDate))
		} else {
			sort.Sort(filesDate)
		}
		files = FISlice(filesDate)
	}

	fmt.Println(" Dirname is", CleanDirName)

	for _, f := range files {
		NAME := strings.ToUpper(f.Name())
		if BOOL, _ := filepath.Match(CleanFileName, NAME); BOOL {
			s := f.ModTime().Format("Jan-02-2006 15:04:05")
			sizeint := int(f.Size())
			sizestr := strconv.Itoa(sizeint)
			if sizeint > 100000 {
				sizestr = AddCommas(sizestr)
			}

			usernameStr, groupnameStr := "", ""
			if runtime.GOARCH == "amd64" {
				usernameStr, groupnameStr = GetUserGroupStr(f)
			}

			if linuxflag {
				if f.IsDir() {
					fmt.Printf("%10v %s:%s %15s %s <%s>\n", f.Mode(), usernameStr, groupnameStr, sizestr, s, f.Name())
				} else if f.Mode().IsRegular() {
					fmt.Printf("%10v %s:%s %15s %s %s\n", f.Mode(), usernameStr, groupnameStr, sizestr, s, f.Name())
				} else { // it's a symlink
					fmt.Printf("%10v %s:%s %15s %s (%s)\n", f.Mode(), usernameStr, groupnameStr, sizestr, s, f.Name())
				}
			} else { // must be windows because this won't compile on Mac.  And I'm ignoring symlinks
				if f.IsDir() {
					fmt.Printf("%15s %s <%s>\n", sizestr, s, f.Name())
				} else {
					fmt.Printf("%15s %s %s\n", sizestr, s, f.Name())
				}
			}
			count++
			if count > NumLines {
				break
			}
		}
	}

} // end main dsrt

//-------------------------------------------------------------------- InsertByteSlice
func InsertIntoByteSlice(slice, insertion []byte, index int) []byte {
	return append(slice[:index], append(insertion, slice[index:]...)...)
}

//---------------------------------------------------------------------- AddCommas
func AddCommas(instr string) string {
	var Comma []byte = []byte{','}

	BS := make([]byte, 0, 15)
	BS = append(BS, instr...)

	i := len(BS)

	for NumberOfCommas := i / 3; (NumberOfCommas > 0) && (i > 3); NumberOfCommas-- {
		i -= 3
		BS = InsertIntoByteSlice(BS, Comma, i)
	}
	return string(BS)
} // AddCommas
//---------------------------------------------------------------------------------------------------

// ---------------------------- GetIDname -----------------------------------------------------------
func GetIDname(uidStr string) string {

	if len(uidStr) == 0 {
		return ""
	}
	ptrToUser, err := user.LookupId(uidStr)
	if err != nil {
		panic("uid not found")
	}

	idname := ptrToUser.Username
	return idname

} // GetIDname

/*
package path
func Match

func Match(pattern, name string) (matched bool, err error)

Match reports whether name matches the shell file name pattern.  The pattern syntax is:

pattern:
	{ term }
term:
	'*'         matches any sequence of non-/ characters
	'?'         matches any single non-/ character
	'[' [ '^' ] { character-range } ']'
	            character class (must be non-empty)
	c           matches character c (c != '*', '?', '\\', '[')
	'\\' c      matches character c

character-range:
	c           matches character c (c != '\\', '-', ']')
	'\\' c      matches character c
	lo '-' hi   matches character c for lo <= c <= hi

Match requires pattern to match all of name, not just a substring.  The only possible returned error is ErrBadPattern, when pattern is malformed.

package os
type FileInfo

type FileInfo interface {
        Name() string       // base name of the file
        Size() int64        // length in bytes for regular files; system-dependent for others
        Mode() FileMode     // file mode bits
        ModTime() time.Time // modification time
        IsDir() bool        // abbreviation for Mode().IsDir()
        Sys() interface{}   // underlying data source (can return nil)
}

A FileInfo describes a file and is returned by Stat and Lstat.

func Lstat

func Lstat(name string) (FileInfo, error)

Lstat returns a FileInfo describing the named file.  If the file is a symbolic link, the returned FileInfo describes the symbolic link.  Lstat makes no attempt to follow the link.
If there is an error, it will be of type *PathError.

func Stat

func Stat(name string) (FileInfo, error)

Stat returns a FileInfo describing the named file.  If there is an error, it will be of type *PathError.

The insight I had with my append troubles that the 1 slice entries were empty, is that when I used append, it would do just that to the end of the slice, and ignore the empty slices.
I needed to make the slice as empty for this to work.  So I am directly assigning the DirEntries slice, and appending the FileNames slice, to make sure that these both are doing what I want.
This code is now doing exactly what I want.  I guess there is no substitute for playing with myself.  Wait, that didn't come out right.  Or did it.

package os/user
type User struct {
  Uid string
  Gid string
  Username string // login name
  Name string     // full or display name.  It may be blank.
  HomeDir string
}
*/
<commit_msg>modified:   dsrt/dsrt.go -- added exec timestamp detection code<commit_after>// dsrt.go -- directoy sort in reverse date order.  IE, newest is first.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"runtime"
	"sort"
	"strconv"
	"strings"
)

const LastAltered = "2 Sep 17"

/*
Revision History
----------------
20 Apr 17 -- Started writing dsize rtn, based on dirlist.go
21 Apr 17 -- Now tweaking the output format.  And used flag package.  One as a pointer and one as a value, just to learn them.
22 Apr 17 -- Coded the use of the first non flag commandline param,  which is all I need.  Note that the flag must appear before the non-flag param, else the flag is ignored.
22 Apr 17 -- Now writing dsrt, to function similarly to dsort.
24 Apr 17 -- Now adding file matching, like "dir" or "ls" does.
25 Apr 17 -- Now adding sort by size as an option, like -s, and commas
26 Apr 17 -- Noticed that the match routine is case sensitive.  I don't like that.
27 Apr 17 -- commandline now allows a file spec.  I intend this for Windows.  I'll see how it goes.
19 May 17 -- Will now show the uid:gid for linux.
20 May 17 -- Turns out that (*syscall.Stat_t) only compiles on linux.  Time for platform specific code.
21 May 17 -- Cross compiling to GOARCH=386, and the uid and User routines won't work.
 2 Sep 17 -- Added timestamp detection code I first wrote for gastricgo.
*/

// FIS is a FileInfo slice, as in os.FileInfo
type FISlice []os.FileInfo
type FISliceDate []os.FileInfo
type FISliceSize []os.FileInfo

func (f FISliceDate) Less(i, j int) bool {
	return f[i].ModTime().UnixNano() > f[j].ModTime().UnixNano() // I want a reverse sort, newest first
}

func (f FISliceDate) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (f FISliceDate) Len() int {
	return len(f)
}

func (f FISliceSize) Less(i, j int) bool {
	return f[i].Size() > f[j].Size() // I want a reverse sort, largest first
}

func (f FISliceSize) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (f FISliceSize) Len() int {
	return len(f)
}

func main() {
	const numlines = 50
	var userptr *user.User
	var files FISlice
	var filesDate FISliceDate
	var filesSize FISliceSize
	var err error
	var count int

	var revflag = flag.Bool("r", false, "reverse the sort, ie, oldest or smallest is first") // Ptr

	var RevFlag bool
	flag.BoolVar(&RevFlag, "R", false, "Reverse the sort, ie, oldest or smallest is first") // Value

	var nlines = flag.Int("n", numlines, "number of lines to display") // Ptr

	var NLines int
	flag.IntVar(&NLines, "N", numlines, "number of lines to display") // Value

	var helpflag = flag.Bool("h", false, "print help message") // pointer
	var HelpFlag bool
	flag.BoolVar(&HelpFlag, "H", false, "print help message")

	var sizeflag = flag.Bool("s", false, "sort by size instead of by date") // pointer
	var SizeFlag bool
	flag.BoolVar(&SizeFlag, "S", false, "sort by size instead of by date")
	flag.Parse()

	fmt.Println(" dsrt will display sorted by date or size.  Written in Go.  LastAltered ", LastAltered)
	execname, _ := os.Executable()
	ExecFI, _ := os.Stat(execname)
	ExecTimeStamp := ExecFI.ModTime().Format("Mon Jan 2 2006 15:04:05 MST")
	fmt.Println(ExecFI.Name(), "timestamp is", ExecTimeStamp, ".  Full exec is", execname)
	fmt.Println()

	uid := 0
	gid := 0
	systemStr := ""
	linuxflag := runtime.GOOS == "linux"
	if linuxflag {
		systemStr = "Linux"
	} else if runtime.GOOS == "windows" {
		systemStr = "Windows"
	} else {
		systemStr = "Mac, maybe"
	}

	if runtime.GOARCH == "amd64" {
		uid = os.Getuid() // int
		gid = os.Getgid() // int
		userptr, err = user.Current()
		if err != nil {
			fmt.Println(" user.Current error is ", err, "Exiting.")
			os.Exit(1)
		}
	}

	if *helpflag || HelpFlag {
		flag.PrintDefaults()
		if runtime.GOARCH == "amd64" {
			fmt.Printf("uid=%d, gid=%d, on a computer running %s for %s:%s Username %s, Name %s, HomeDir %s \n",
				uid, gid, systemStr, userptr.Uid, userptr.Gid, userptr.Username, userptr.Name, userptr.HomeDir)
		}

	}

	Reverse := *revflag || RevFlag
	SizeSort := *sizeflag || SizeFlag

	NumLines := numlines
	if *nlines != numlines {
		NumLines = *nlines
	} else if NLines != numlines {
		NumLines = NLines
	}
	askforinput := true

	CleanDirName := "." + string(filepath.Separator)
	CleanFileName := ""
	commandline := flag.Arg(0) // this only gets the first non flag argument.  That's all I want
	if len(commandline) > 0 {
		//		CleanDirName = filepath.Clean(commandline)
		CleanDirName, CleanFileName = filepath.Split(commandline)
		CleanFileName = strings.ToUpper(CleanFileName)
		askforinput = false
	}

	if askforinput {
		// Asking for input so don't have to worry about command line globbing
		fmt.Print(" Enter input for globbing: ")
		scanner := bufio.NewScanner(os.Stdin)
		scanner.Scan()
		newtext := scanner.Text()
		if err = scanner.Err(); err != nil {
			fmt.Fprintln(os.Stderr, " reading std input: ", err)
			os.Exit(1)
		}
		if len(newtext) > 0 {
			// time to do the stuff I'm writing this pgm for
			CleanDirName, CleanFileName = filepath.Split(newtext)
			CleanFileName = strings.ToUpper(CleanFileName)
		}

	}

	if len(CleanDirName) == 0 {
		CleanDirName = "." + string(filepath.Separator)
	}

	if len(CleanFileName) == 0 {
		CleanFileName = "*"
	}

	if SizeSort {
		filesSize, err = ioutil.ReadDir(CleanDirName)
		if err != nil {
			log.Fatal(err)
		}
		if Reverse {
			sort.Sort(sort.Reverse(filesSize))
		} else {
			sort.Sort(filesSize)
		}
		files = FISlice(filesSize)
	} else {
		filesDate, err = ioutil.ReadDir(CleanDirName)
		if err != nil {
			log.Fatal(err)
		}
		if Reverse {
			sort.Sort(sort.Reverse(filesDate))
		} else {
			sort.Sort(filesDate)
		}
		files = FISlice(filesDate)
	}

	fmt.Println(" Dirname is", CleanDirName)

	for _, f := range files {
		NAME := strings.ToUpper(f.Name())
		if BOOL, _ := filepath.Match(CleanFileName, NAME); BOOL {
			s := f.ModTime().Format("Jan-02-2006 15:04:05")
			sizeint := int(f.Size())
			sizestr := strconv.Itoa(sizeint)
			if sizeint > 100000 {
				sizestr = AddCommas(sizestr)
			}

			usernameStr, groupnameStr := "", ""
			if runtime.GOARCH == "amd64" {
				usernameStr, groupnameStr = GetUserGroupStr(f)
			}

			if linuxflag {
				if f.IsDir() {
					fmt.Printf("%10v %s:%s %15s %s <%s>\n", f.Mode(), usernameStr, groupnameStr, sizestr, s, f.Name())
				} else if f.Mode().IsRegular() {
					fmt.Printf("%10v %s:%s %15s %s %s\n", f.Mode(), usernameStr, groupnameStr, sizestr, s, f.Name())
				} else { // it's a symlink
					fmt.Printf("%10v %s:%s %15s %s (%s)\n", f.Mode(), usernameStr, groupnameStr, sizestr, s, f.Name())
				}
			} else { // must be windows because this won't compile on Mac.  And I'm ignoring symlinks
				if f.IsDir() {
					fmt.Printf("%15s %s <%s>\n", sizestr, s, f.Name())
				} else {
					fmt.Printf("%15s %s %s\n", sizestr, s, f.Name())
				}
			}
			count++
			if count > NumLines {
				break
			}
		}
	}

} // end main dsrt

//-------------------------------------------------------------------- InsertByteSlice
func InsertIntoByteSlice(slice, insertion []byte, index int) []byte {
	return append(slice[:index], append(insertion, slice[index:]...)...)
}

//---------------------------------------------------------------------- AddCommas
func AddCommas(instr string) string {
	var Comma []byte = []byte{','}

	BS := make([]byte, 0, 15)
	BS = append(BS, instr...)

	i := len(BS)

	for NumberOfCommas := i / 3; (NumberOfCommas > 0) && (i > 3); NumberOfCommas-- {
		i -= 3
		BS = InsertIntoByteSlice(BS, Comma, i)
	}
	return string(BS)
} // AddCommas
//---------------------------------------------------------------------------------------------------

// ---------------------------- GetIDname -----------------------------------------------------------
func GetIDname(uidStr string) string {

	if len(uidStr) == 0 {
		return ""
	}
	ptrToUser, err := user.LookupId(uidStr)
	if err != nil {
		panic("uid not found")
	}

	idname := ptrToUser.Username
	return idname

} // GetIDname

/*
package path
func Match

func Match(pattern, name string) (matched bool, err error)

Match reports whether name matches the shell file name pattern.  The pattern syntax is:

pattern:
	{ term }
term:
	'*'         matches any sequence of non-/ characters
	'?'         matches any single non-/ character
	'[' [ '^' ] { character-range } ']'
	            character class (must be non-empty)
	c           matches character c (c != '*', '?', '\\', '[')
	'\\' c      matches character c

character-range:
	c           matches character c (c != '\\', '-', ']')
	'\\' c      matches character c
	lo '-' hi   matches character c for lo <= c <= hi

Match requires pattern to match all of name, not just a substring.  The only possible returned error is ErrBadPattern, when pattern is malformed.

package os
type FileInfo

type FileInfo interface {
        Name() string       // base name of the file
        Size() int64        // length in bytes for regular files; system-dependent for others
        Mode() FileMode     // file mode bits
        ModTime() time.Time // modification time
        IsDir() bool        // abbreviation for Mode().IsDir()
        Sys() interface{}   // underlying data source (can return nil)
}

A FileInfo describes a file and is returned by Stat and Lstat.

func Lstat

func Lstat(name string) (FileInfo, error)

Lstat returns a FileInfo describing the named file.  If the file is a symbolic link, the returned FileInfo describes the symbolic link.  Lstat makes no attempt to follow the link.
If there is an error, it will be of type *PathError.

func Stat

func Stat(name string) (FileInfo, error)

Stat returns a FileInfo describing the named file.  If there is an error, it will be of type *PathError.

The insight I had with my append troubles that the 1 slice entries were empty, is that when I used append, it would do just that to the end of the slice, and ignore the empty slices.
I needed to make the slice as empty for this to work.  So I am directly assigning the DirEntries slice, and appending the FileNames slice, to make sure that these both are doing what I want.
This code is now doing exactly what I want.  I guess there is no substitute for playing with myself.  Wait, that didn't come out right.  Or did it.

package os/user
type User struct {
  Uid string
  Gid string
  Username string // login name
  Name string     // full or display name.  It may be blank.
  HomeDir string
}
*/
<|endoftext|>


<commit_before>// Written in 2014 by Philipp Stephani <p.stephani2@gmail.com>.
//
// To the extent possible under law, the author has dedicated all copyright and
// related and neighboring rights to this software to the public domain worldwide.
// This software is distributed without any warranty.
//
// You should have received a copy of the CC0 Public Domain Dedication along with
// this software.  If not, see http://creativecommons.org/publicdomain/zero/1.0/.

package main

import (
	"flag"
	"os"

	"github.com/golang/glog"

	"github.com/phst/dupremove/dup"
	"github.com/phst/dupremove/filter"
	"github.com/phst/dupremove/rdfind"
)

var dryRun = flag.Bool("n", false, "dry-run mode: don't remove any files")

func main() {
	flag.Parse()
	keep := []string{}
	dirs := []string{}
	mode := ""
	for _, arg := range flag.Args() {
		if arg == "keep" || arg == "remove" {
			mode = arg
		} else {
			if mode == "" {
				glog.Fatal("command line arguments need to start with 'keep' or 'remove'")
			} else {
				dirs = append(dirs, arg)
				if mode == "keep" {
					keep = append(keep, arg)
				}
			}
		}
	}
	if len(dirs) == 0 {
		glog.Fatal("no directories specified")
	}

	groups, err := rdfind.Run(dirs)
	if err != nil {
		glog.Fatalf("error running rdfind: %s", err)
	}
	glog.Infof("found %d file groups", len(groups))

	removed := 0
	for _, group := range groups {
		files := filter.RemovableFiles(group, keep)
		for _, file := range files {
			if err := remove(file); err != nil {
				glog.Errorf("could not remove file %s: %s", file, err)
			} else {
				glog.V(1).Infof("removed file %s", file)
				removed++
			}
		}
	}
	glog.Infof("removed %d files", removed)
}

func remove(f dup.FileName) error {
	if *dryRun {
		return nil
	}
	return os.Remove(string(f))
}
<commit_msg>Exit with a nonzero status if errors occured<commit_after>// Written in 2014 by Philipp Stephani <p.stephani2@gmail.com>.
//
// To the extent possible under law, the author has dedicated all copyright and
// related and neighboring rights to this software to the public domain worldwide.
// This software is distributed without any warranty.
//
// You should have received a copy of the CC0 Public Domain Dedication along with
// this software.  If not, see http://creativecommons.org/publicdomain/zero/1.0/.

package main

import (
	"flag"
	"os"

	"github.com/golang/glog"

	"github.com/phst/dupremove/dup"
	"github.com/phst/dupremove/filter"
	"github.com/phst/dupremove/rdfind"
)

var dryRun = flag.Bool("n", false, "dry-run mode: don't remove any files")

func main() {
	flag.Parse()
	keep := []string{}
	dirs := []string{}
	mode := ""
	for _, arg := range flag.Args() {
		if arg == "keep" || arg == "remove" {
			mode = arg
		} else {
			if mode == "" {
				glog.Fatal("command line arguments need to start with 'keep' or 'remove'")
			} else {
				dirs = append(dirs, arg)
				if mode == "keep" {
					keep = append(keep, arg)
				}
			}
		}
	}
	if len(dirs) == 0 {
		glog.Fatal("no directories specified")
	}

	groups, err := rdfind.Run(dirs)
	if err != nil {
		glog.Fatalf("error running rdfind: %s", err)
	}
	glog.Infof("found %d file groups", len(groups))

	removed := 0
	errors := 0
	for _, group := range groups {
		files := filter.RemovableFiles(group, keep)
		for _, file := range files {
			if err := remove(file); err != nil {
				glog.Errorf("could not remove file %s: %s", file, err)
				errors++
			} else {
				glog.V(1).Infof("removed file %s", file)
				removed++
			}
		}
	}
	glog.Infof("removed %d files", removed)
	if errors > 0 {
		glog.Errorf("could not remove %d files due to errors", errors)
		os.Exit(1)
	}
}

func remove(f dup.FileName) error {
	if *dryRun {
		return nil
	}
	return os.Remove(string(f))
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
)

var (
	dataFile = flag.String("file", "store.json", "data store file name")
	status   = flag.String("status", "stopped|pending|terminated", "the status you would like to poll")
)

var store *StatusStore

type Conn struct {
	aw2  *ec2.EC2
	data map[string]string
}

// create a struct to map toml config file
// TODO: add this as an switch in init()
type AwsConfig struct {
	AwsSecretKey string `toml:"AWS_ACCESS_KEY_ID"`
	AwsAccessKey string `toml:"AWS_SECRET_ACCESS_KEY"`
	Region       string `toml:"AWS_REGION"`
}

func recieveStatus(dataMap map[string]string) <-chan string {
	c := make(chan string)
	go func() {
		for _, v := range dataMap {
			c <- fmt.Sprintf("%s", v)
			//			time.Sleep(time.Duration(rand.Intn(1e3)) * time.Millisecond)
		}
	}()
	return c

}

<<<<<<< HEAD
func (c *Conn) iterateResToMap(resp *ec2.DescribeInstancesOutput) {
=======
func (c *Conn) iterateResToMap(resp *ec2.DescribeInstancesOutput, s *StatusStore) {
>>>>>>> 2e8ab3efb73a74c3b1055bab321e737a63f27390
	insMap := make(map[string]string)
	for idx, _ := range resp.Reservations {
		for _, inst := range resp.Reservations[idx].Instances {
			// fmt.Printf("   Instance State: %v InstanceID: %v \n", *inst.State.Name, *inst.InstanceID)
			// dereference pointer
			var id, state string
			id = *inst.PrivateDNSName
			state = *inst.State.Name
			insMap[id] = state
		}
		s.newWorld = insMap
	}
<<<<<<< HEAD
	c.data = insMap
}

func (c *Conn) GetEc2Data() {
=======

}

func (c *Conn) GetEc2Data(s *StatusStore) {
>>>>>>> 2e8ab3efb73a74c3b1055bab321e737a63f27390

	resp, err := c.aw2.DescribeInstances(nil)
	if err != nil {
		log.Fatal(err)
	}

<<<<<<< HEAD
	c.iterateResToMap(resp)
=======
	newMap := c.iterateResToMap(resp)
	if s.isFileLoaded {
		fmt.Println("We have a database file not reloading")
	} else {
		s.status = newMap
	}
}

func (c *Conn) startLoop(status string) bool {
	// Call the DescribeInstances Operation
	resp, err := c.aw2.DescribeInstances(nil)
	if err != nil {
		log.Fatal(err)
	}
	// re-format to method call
	newMap := c.iterateResToMap(resp)

	r := recieveStatus(newMap)

	// set a timeout for the channel
	timeout := time.After(5 * time.Second)

	// begin channel operations
	// TODO: this should have interfaces and structs and a poller
	for {
		select {
		case result := <-r:
			matched, _ := regexp.MatchString(status, result)
			if matched {
				res := strings.Split(result, ":")
				status, ip := res[0], res[1]
				fmt.Printf("Status:  %v PrivateIP:  %v  \n", status, ip)
			}
		case <-timeout:
			return false
		}
	}
>>>>>>> 2e8ab3efb73a74c3b1055bab321e737a63f27390

}

func NewEc2() *Conn {
	c := new(Conn)
	c.aw2 = ec2.New(&aws.Config{Region: "us-west-2"})

	return c
}

func (d *StatusStore) AddDataToFile(status string, c *Conn) {
	fmt.Println("im in adddatatofile")

	fmt.Println(d.status)

	for k, v := range c.data {
		if v == status {
			err := d.save(k, v)
			if err != nil {
				log.Printf("something went wrong save %s", k)
			}
		}
	}
}

func main() {
	// Create an EC2 service object in the "us-west-2" region
	// Note that you can also configure your region globally by
	// exporting the AWS_REGION environment variable
	flag.Parse()

	// instantiate new ec2 "object"
	c := NewEc2()

	d := NewStatusStore(*dataFile)

	c.GetEc2Data(d)

	// Get a data set to work with
<<<<<<< HEAD
	c.GetEc2Data()
=======

	// set the status map
>>>>>>> 2e8ab3efb73a74c3b1055bab321e737a63f27390

	// lets save some data
	d.AddDataToFile(*status, c)

}

func Add(value string) {
	key := store.Put(value)
	fmt.Println(key)
}
<commit_msg>removing this bs<commit_after><|endoftext|>


<commit_before>package bitrange

func rangeBitwiseAnd(m, n int) int {
    return 0
}
<commit_msg>solve 201 by finding the common prefix of m and n<commit_after>package bitrange

func rangeBitwiseAnd(m, n int) int {
    return commonPrefix(m, n)
}

// commonPrefix time complexity O(1), space complexity O(1)
func commonPrefix(m, n int) int {
    var shift int
    for m < n {
        m >>= 1
        n >>= 1
        shift++
    }
    return m << shift
}
<|endoftext|>


<commit_before>package config

import (
	"flag"
	"io"

	"github.com/BurntSushi/toml"
)

var configfile = flag.String("f", "Config filename", "autobot.json")

type Config struct {
}

func LoadConfig(r io.Reader) (c Config, err error) {
	_, err = toml.DecodeReader(r, &c)
	return
}
<commit_msg>removed legacy flag dependency from config package<commit_after>package config

import (
	"io"

	"github.com/BurntSushi/toml"
)

type Config struct {
}

func LoadConfig(r io.Reader) (c Config, err error) {
	_, err = toml.DecodeReader(r, &c)
	return
}
<|endoftext|>


<commit_before>package config
<commit_msg>Delete config package<commit_after><|endoftext|>


<commit_before>/*
    Quick JSON configuration parser
*/
package config

import (
  "github.com/stretchr/objx"
  "github.com/southern/logger"
  "io/ioutil"
  "log"
  "os"
  "path"
)

var (
  Global objx.Map
  Logger *logger.Logger
  GoPath = os.Getenv("GOPATH")
  Dir string
)

func init() {
  if len(Dir) == 0 && len(GoPath) > 0 {
    Dir = path.Join(
      GoPath,
      "src",
      "github.com",
      "yawnt",
      os.Args[0],
      "config",
    )
  }
}

// Load takes an environment, loads the JSON file associated with the
// environment, and returns an instance of objx.Map for accessing the
// properties.
func Load(env string) (Global objx.Map) {
  // Get current directory
  pwd, err := os.Getwd()
  if err != nil {
    log.Fatalln(err)
  }

  // Try reading locally first
  data, localerr := ioutil.ReadFile(path.Join(pwd, "config",
    env + ".config.json"))
  if localerr != nil {
    // Try reading GOPATH next.
    if len(Dir) > 0 {
      data, err = ioutil.ReadFile(path.Join(
        Dir,
        env + ".config.json",
      ))
      if err != nil {
        log.Fatalln(err.Error() + "; " + localerr.Error())
      }
    } else {
      log.Fatalln(localerr.Error() +
        ", and $GOPATH is not defined to determine the config directory.")
    }
  }

  // Convert from JSON to objx.Map
  Global, err = objx.FromJSON(string(data))
  if err != nil {
    log.Fatalln(err)
  }

  return
}
<commit_msg>If we can get the last caller in config, use its directory.<commit_after>/*
    Quick JSON configuration parser
*/
package config

import (
  "github.com/stretchr/objx"
  "github.com/southern/logger"
  "io/ioutil"
  "log"
  "os"
  "path"
  "runtime"
)

var (
  Global objx.Map
  Logger *logger.Logger
  GoPath = os.Getenv("GOPATH")
  Dir string
)

// Load takes an environment, loads the JSON file associated with the
// environment, and returns an instance of objx.Map for accessing the
// properties.
func Load(env string) (Global objx.Map) {
  _, file, _, ok := runtime.Caller(0)
  if ok {
    Dir = path.Dir(file)
  } else {
    if len(GoPath) > 0 {
      Dir = path.Join(
        GoPath,
        "src",
        "github.com",
        "spacedock-io",
        os.Args[0],
        "config",
      )
    }
  }

  // Get current directory
  pwd, err := os.Getwd()
  if err != nil {
    log.Fatalln(err)
  }

  // Try reading locally first
  data, localerr := ioutil.ReadFile(path.Join(pwd, "config",
    env + ".config.json"))
  if localerr != nil {
    // Try reading GOPATH next.
    if len(Dir) > 0 {
      data, err = ioutil.ReadFile(path.Join(
        Dir,
        env + ".config.json",
      ))
      if err != nil {
        log.Fatalln(err.Error() + "; " + localerr.Error())
      }
    } else {
      log.Fatalln(localerr.Error() +
        ", and $GOPATH is not defined to determine the config directory.")
    }
  }

  // Convert from JSON to objx.Map
  Global, err = objx.FromJSON(string(data))
  if err != nil {
    log.Fatalln(err)
  }

  return
}
<|endoftext|>