<commit_before>package datastore

import (
	"io"
	"log"

	dsq "github.com/ipfs/go-datastore/query"
)

// Here are some basic datastore implementations.

type keyMap map[Key]interface{}

// MapDatastore uses a standard Go map for internal storage.
type MapDatastore struct {
	values keyMap
}

// NewMapDatastore constructs a MapDatastore
func NewMapDatastore() (d *MapDatastore) {
	return &MapDatastore{
		values: keyMap{},
	}
}

// Put implements Datastore.Put
func (d *MapDatastore) Put(key Key, value interface{}) (err error) {
	d.values[key] = value
	return nil
}

// Get implements Datastore.Get
func (d *MapDatastore) Get(key Key) (value interface{}, err error) {
	val, found := d.values[key]
	if !found {
		return nil, ErrNotFound
	}
	return val, nil
}

// Has implements Datastore.Has
func (d *MapDatastore) Has(key Key) (exists bool, err error) {
	_, found := d.values[key]
	return found, nil
}

// Delete implements Datastore.Delete
func (d *MapDatastore) Delete(key Key) (err error) {
	if _, found := d.values[key]; !found {
		return ErrNotFound
	}
	delete(d.values, key)
	return nil
}

// Query implements Datastore.Query
func (d *MapDatastore) Query(q dsq.Query) (dsq.Results, error) {
	re := make([]dsq.Entry, 0, len(d.values))
	for k, v := range d.values {
		re = append(re, dsq.Entry{Key: k.String(), Value: v})
	}
	r := dsq.ResultsWithEntries(q, re)
	r = dsq.NaiveQueryApply(q, r)
	return r, nil
}

func (d *MapDatastore) Batch() (Batch, error) {
	return NewBasicBatch(d), nil
}

func (d *MapDatastore) Close() error {
	return nil
}

// NullDatastore stores nothing, but conforms to the API.
// Useful to test with.
type NullDatastore struct {
}

// NewNullDatastore constructs a null datastoe
func NewNullDatastore() *NullDatastore {
	return &NullDatastore{}
}

// Put implements Datastore.Put
func (d *NullDatastore) Put(key Key, value interface{}) (err error) {
	return nil
}

// Get implements Datastore.Get
func (d *NullDatastore) Get(key Key) (value interface{}, err error) {
	return nil, nil
}

// Has implements Datastore.Has
func (d *NullDatastore) Has(key Key) (exists bool, err error) {
	return false, nil
}

// Delete implements Datastore.Delete
func (d *NullDatastore) Delete(key Key) (err error) {
	return nil
}

// Query implements Datastore.Query
func (d *NullDatastore) Query(q dsq.Query) (dsq.Results, error) {
	return dsq.ResultsWithEntries(q, nil), nil
}

func (d *NullDatastore) Batch() (Batch, error) {
	return NewBasicBatch(d), nil
}

func (d *NullDatastore) Close() error {
	return nil
}

// LogDatastore logs all accesses through the datastore.
type LogDatastore struct {
	Name  string
	child Datastore
}

// Shim is a datastore which has a child.
type Shim interface {
	Datastore

	Children() []Datastore
}

// NewLogDatastore constructs a log datastore.
func NewLogDatastore(ds Datastore, name string) *LogDatastore {
	if len(name) < 1 {
		name = "LogDatastore"
	}
	return &LogDatastore{Name: name, child: ds}
}

// Children implements Shim
func (d *LogDatastore) Children() []Datastore {
	return []Datastore{d.child}
}

// Put implements Datastore.Put
func (d *LogDatastore) Put(key Key, value interface{}) (err error) {
	log.Printf("%s: Put %s\n", d.Name, key)
	// log.Printf("%s: Put %s ```%s```", d.Name, key, value)
	return d.child.Put(key, value)
}

// Get implements Datastore.Get
func (d *LogDatastore) Get(key Key) (value interface{}, err error) {
	log.Printf("%s: Get %s\n", d.Name, key)
	return d.child.Get(key)
}

// Has implements Datastore.Has
func (d *LogDatastore) Has(key Key) (exists bool, err error) {
	log.Printf("%s: Has %s\n", d.Name, key)
	return d.child.Has(key)
}

// Delete implements Datastore.Delete
func (d *LogDatastore) Delete(key Key) (err error) {
	log.Printf("%s: Delete %s\n", d.Name, key)
	return d.child.Delete(key)
}

// DiskUsage implements the PersistentDatastore interface.
func (d *LogDatastore) DiskUsage() (uint64, error) {
	du, err := DiskUsage(d.child)
	log.Printf("%s: DiskUsage: %d\n", d.Name, du)
	return du, err
}

// Query implements Datastore.Query
func (d *LogDatastore) Query(q dsq.Query) (dsq.Results, error) {
	log.Printf("%s: Query\n", d.Name)
	log.Printf("%s: q.Prefix: %s\n", d.Name, q.Prefix)
	log.Printf("%s: q.KeysOnly: %v\n", d.Name, q.KeysOnly)
	log.Printf("%s: q.Filters: %d\n", d.Name, len(q.Filters))
	log.Printf("%s: q.Orders: %d\n", d.Name, len(q.Orders))
	log.Printf("%s: q.Offset: %d\n", d.Name, q.Offset)

	return d.child.Query(q)
}

// LogBatch logs all accesses through the batch.
type LogBatch struct {
	Name  string
	child Batch
}

func (d *LogDatastore) Batch() (Batch, error) {
	log.Printf("%s: Batch\n", d.Name)
	if bds, ok := d.child.(Batching); ok {
		b, err := bds.Batch()

		if err != nil {
			return nil, err
		}
		return &LogBatch{
			Name:  d.Name,
			child: b,
		}, nil
	}
	return nil, ErrBatchUnsupported
}

// Put implements Batch.Put
func (d *LogBatch) Put(key Key, value interface{}) (err error) {
	log.Printf("%s: BatchPut %s\n", d.Name, key)
	// log.Printf("%s: Put %s ```%s```", d.Name, key, value)
	return d.child.Put(key, value)
}

// Delete implements Batch.Delete
func (d *LogBatch) Delete(key Key) (err error) {
	log.Printf("%s: BatchDelete %s\n", d.Name, key)
	return d.child.Delete(key)
}

// Commit implements Batch.Commit
func (d *LogBatch) Commit() (err error) {
	log.Printf("%s: BatchCommit\n", d.Name)
	return d.child.Commit()
}

func (d *LogDatastore) Close() error {
	log.Printf("%s: Close\n", d.Name)
	if cds, ok := d.child.(io.Closer); ok {
		return cds.Close()
	}
	return nil
}

func (d *LogDatastore) Check() error {
	if c, ok := d.child.(CheckedDatastore); ok {
		return c.Check()
	}
	return nil
}

func (d *LogDatastore) Scrub() error {
	if c, ok := d.child.(ScrubbedDatastore); ok {
		return c.Scrub()
	}
	return nil
}

func (d *LogDatastore) CollectGarbage() error {
	if c, ok := d.child.(GCDatastore); ok {
		return c.CollectGarbage()
	}
	return nil
}
<commit_msg>Feat #74: LogDatastore: log DiskUsage before it happens<commit_after>package datastore

import (
	"io"
	"log"

	dsq "github.com/ipfs/go-datastore/query"
)

// Here are some basic datastore implementations.

type keyMap map[Key]interface{}

// MapDatastore uses a standard Go map for internal storage.
type MapDatastore struct {
	values keyMap
}

// NewMapDatastore constructs a MapDatastore
func NewMapDatastore() (d *MapDatastore) {
	return &MapDatastore{
		values: keyMap{},
	}
}

// Put implements Datastore.Put
func (d *MapDatastore) Put(key Key, value interface{}) (err error) {
	d.values[key] = value
	return nil
}

// Get implements Datastore.Get
func (d *MapDatastore) Get(key Key) (value interface{}, err error) {
	val, found := d.values[key]
	if !found {
		return nil, ErrNotFound
	}
	return val, nil
}

// Has implements Datastore.Has
func (d *MapDatastore) Has(key Key) (exists bool, err error) {
	_, found := d.values[key]
	return found, nil
}

// Delete implements Datastore.Delete
func (d *MapDatastore) Delete(key Key) (err error) {
	if _, found := d.values[key]; !found {
		return ErrNotFound
	}
	delete(d.values, key)
	return nil
}

// Query implements Datastore.Query
func (d *MapDatastore) Query(q dsq.Query) (dsq.Results, error) {
	re := make([]dsq.Entry, 0, len(d.values))
	for k, v := range d.values {
		re = append(re, dsq.Entry{Key: k.String(), Value: v})
	}
	r := dsq.ResultsWithEntries(q, re)
	r = dsq.NaiveQueryApply(q, r)
	return r, nil
}

func (d *MapDatastore) Batch() (Batch, error) {
	return NewBasicBatch(d), nil
}

func (d *MapDatastore) Close() error {
	return nil
}

// NullDatastore stores nothing, but conforms to the API.
// Useful to test with.
type NullDatastore struct {
}

// NewNullDatastore constructs a null datastoe
func NewNullDatastore() *NullDatastore {
	return &NullDatastore{}
}

// Put implements Datastore.Put
func (d *NullDatastore) Put(key Key, value interface{}) (err error) {
	return nil
}

// Get implements Datastore.Get
func (d *NullDatastore) Get(key Key) (value interface{}, err error) {
	return nil, nil
}

// Has implements Datastore.Has
func (d *NullDatastore) Has(key Key) (exists bool, err error) {
	return false, nil
}

// Delete implements Datastore.Delete
func (d *NullDatastore) Delete(key Key) (err error) {
	return nil
}

// Query implements Datastore.Query
func (d *NullDatastore) Query(q dsq.Query) (dsq.Results, error) {
	return dsq.ResultsWithEntries(q, nil), nil
}

func (d *NullDatastore) Batch() (Batch, error) {
	return NewBasicBatch(d), nil
}

func (d *NullDatastore) Close() error {
	return nil
}

// LogDatastore logs all accesses through the datastore.
type LogDatastore struct {
	Name  string
	child Datastore
}

// Shim is a datastore which has a child.
type Shim interface {
	Datastore

	Children() []Datastore
}

// NewLogDatastore constructs a log datastore.
func NewLogDatastore(ds Datastore, name string) *LogDatastore {
	if len(name) < 1 {
		name = "LogDatastore"
	}
	return &LogDatastore{Name: name, child: ds}
}

// Children implements Shim
func (d *LogDatastore) Children() []Datastore {
	return []Datastore{d.child}
}

// Put implements Datastore.Put
func (d *LogDatastore) Put(key Key, value interface{}) (err error) {
	log.Printf("%s: Put %s\n", d.Name, key)
	// log.Printf("%s: Put %s ```%s```", d.Name, key, value)
	return d.child.Put(key, value)
}

// Get implements Datastore.Get
func (d *LogDatastore) Get(key Key) (value interface{}, err error) {
	log.Printf("%s: Get %s\n", d.Name, key)
	return d.child.Get(key)
}

// Has implements Datastore.Has
func (d *LogDatastore) Has(key Key) (exists bool, err error) {
	log.Printf("%s: Has %s\n", d.Name, key)
	return d.child.Has(key)
}

// Delete implements Datastore.Delete
func (d *LogDatastore) Delete(key Key) (err error) {
	log.Printf("%s: Delete %s\n", d.Name, key)
	return d.child.Delete(key)
}

// DiskUsage implements the PersistentDatastore interface.
func (d *LogDatastore) DiskUsage() (uint64, error) {
	log.Printf("%s: DiskUsage\n", d.Name)
	return DiskUsage(d.child)
}

// Query implements Datastore.Query
func (d *LogDatastore) Query(q dsq.Query) (dsq.Results, error) {
	log.Printf("%s: Query\n", d.Name)
	log.Printf("%s: q.Prefix: %s\n", d.Name, q.Prefix)
	log.Printf("%s: q.KeysOnly: %v\n", d.Name, q.KeysOnly)
	log.Printf("%s: q.Filters: %d\n", d.Name, len(q.Filters))
	log.Printf("%s: q.Orders: %d\n", d.Name, len(q.Orders))
	log.Printf("%s: q.Offset: %d\n", d.Name, q.Offset)

	return d.child.Query(q)
}

// LogBatch logs all accesses through the batch.
type LogBatch struct {
	Name  string
	child Batch
}

func (d *LogDatastore) Batch() (Batch, error) {
	log.Printf("%s: Batch\n", d.Name)
	if bds, ok := d.child.(Batching); ok {
		b, err := bds.Batch()

		if err != nil {
			return nil, err
		}
		return &LogBatch{
			Name:  d.Name,
			child: b,
		}, nil
	}
	return nil, ErrBatchUnsupported
}

// Put implements Batch.Put
func (d *LogBatch) Put(key Key, value interface{}) (err error) {
	log.Printf("%s: BatchPut %s\n", d.Name, key)
	// log.Printf("%s: Put %s ```%s```", d.Name, key, value)
	return d.child.Put(key, value)
}

// Delete implements Batch.Delete
func (d *LogBatch) Delete(key Key) (err error) {
	log.Printf("%s: BatchDelete %s\n", d.Name, key)
	return d.child.Delete(key)
}

// Commit implements Batch.Commit
func (d *LogBatch) Commit() (err error) {
	log.Printf("%s: BatchCommit\n", d.Name)
	return d.child.Commit()
}

func (d *LogDatastore) Close() error {
	log.Printf("%s: Close\n", d.Name)
	if cds, ok := d.child.(io.Closer); ok {
		return cds.Close()
	}
	return nil
}

func (d *LogDatastore) Check() error {
	if c, ok := d.child.(CheckedDatastore); ok {
		return c.Check()
	}
	return nil
}

func (d *LogDatastore) Scrub() error {
	if c, ok := d.child.(ScrubbedDatastore); ok {
		return c.Scrub()
	}
	return nil
}

func (d *LogDatastore) CollectGarbage() error {
	if c, ok := d.child.(GCDatastore); ok {
		return c.CollectGarbage()
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// bbramfs builds a simple initramfs given an existing built bb; see bb.go
// You have to run bb first, which creates cmds/bb/bbsh. cd to that directory,
// and run bbramfs, and you have a single binary which does all u-root commands.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/u-root/u-root/pkg/cpio"
	_ "github.com/u-root/u-root/pkg/cpio/newc"
	"github.com/u-root/u-root/pkg/ldd"
)

var (
	// Paths contains the paths to put into the initramfs.
	// The index is a root directory, and the value is the place from which to
	// walk. The only required root is the bbsh dir itself, and the starting
	// walk is init -- i.e. we grab only one file. Should you wish to bring in,
	// e.g., /lib/modules/4.04, you would do add the root as / and the
	// starting point for the walk as lib/modules/4.04. That way we only preserve
	// as much of the path as we need, but we can preserve it all.
	paths      = map[string][]string{}
	extraPaths = flag.String("extra", "", "Extra paths to add in the form root:start, e.g. /:etc/hosts")
	extraCmds  = flag.String("cmds", "", "Extra commands to add (full path, comma-separated string)")
)

func sanity() {
	goBinGo := filepath.Join(config.Goroot, "bin/go")
	_, err := os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	// but does the one in go/bin/OS_ARCH exist too?
	goBinGo = filepath.Join(config.Goroot, fmt.Sprintf("bin/%s_%s/go", config.Goos, config.Arch))
	_, err = os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	if config.Go == "" {
		log.Fatalf("Can't find a go binary! Is GOROOT set correctly?")
	}
}

// copyCommands takes a list of commands, generates the list of libs,
// and creates cpio records, including directory records.
func copyCommands(w cpio.Writer, cmds []string) {
	debug("copyCommands: start with %v", cmds)
	var recs []cpio.Record
	libs, err := uroot.LddList(cmds)
	if err != nil {
		log.Fatalf("%v\n", err)
	}
	cmds = append(cmds, libs...)
	for _, n := range cmds {
		debug("copyCommands: file %v", n)
		var dirlist []string
		for d := filepath.Dir(n); d != "/"; d = filepath.Dir(d) {
			dirlist = append([]string{d}, dirlist...)
		}
		dirlist = append(dirlist, n)
		for _, n := range dirlist {
			debug("copyCommands: %v", n)
			r, err := cpio.GetRecord(n)
			if err != nil {
				log.Fatalf("%v: %v", n, err)
			}
			recs = append(recs, r)
		}
	}
	cpio.MakeReproducible(recs)
	if err := w.WriteRecords(recs); err != nil {
		log.Fatalf("%v\n", err)
	}
}

func ramfs() {
	archiver, err := cpio.Format("newc")
	if err != nil {
		log.Fatalf("Creating newc archiver: %v", err)
	}

	oname := fmt.Sprintf("/tmp/initramfs.%v_%v.cpio", config.Goos, config.Arch)
	f, err := os.Create(oname)
	if err != nil {
		log.Fatalf("%v\n", err)
	}

	w := archiver.Writer(f)
	cpio.MakeReproducible(devCPIO[:])
	if err := w.WriteRecords(devCPIO[:]); err != nil {
		log.Fatalf("%v\n", err)
	}

	paths[filepath.Join(config.Gopath, "src/github.com/u-root/u-root/bb/bbsh")] = []string{"init", "ubin"}

	if *extraPaths != "" {
		extras := strings.Split(*extraPaths, " ")
		for _, x := range extras {
			paths["/"] = append(paths["/"], x)
		}
	}

	if *extraCmds != "" {
		copyCommands(w, strings.Split(*extraCmds, " "))
	}

	// For all the 'roots' in paths, start walking at the name.
	debug("PATHS: %v", paths)
	for r, list := range paths {
		for _, n := range list {
			if err := filepath.Walk(filepath.Join(r, n), func(name string, fi os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				cn, err := filepath.Rel(r, name)
				if err != nil {
					log.Fatalf("filepath.Rel(%v, %v): %v", r, name, err)
				}
				debug("%v\n", cn)
				rec, err := cpio.GetRecord(name)
				if err != nil {
					log.Fatalf("Getting record of %q failed: %v", cn, err)
				}
				// the name in the cpio is relative to our starting point.
				rec.Name = cn
				recs := []cpio.Record{rec}
				cpio.MakeReproducible(recs)
				if err := w.WriteRecords(recs); err != nil {
					log.Fatalf("%v\n", err)
				}
				return nil
			}); err != nil {
				log.Fatalf("bbsh walk failed: %v", err)
			}
		}
	}

	if err := w.WriteTrailer(); err != nil {
		log.Fatalf("Error writing trailer record: %v", err)
	}
	fmt.Printf("Output file is in %v\n", oname)
}
<commit_msg>bb: create dircomponents of extra paths<commit_after>// Copyright 2015-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// bbramfs builds a simple initramfs given an existing built bb; see bb.go
// You have to run bb first, which creates cmds/bb/bbsh. cd to that directory,
// and run bbramfs, and you have a single binary which does all u-root commands.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/u-root/u-root/pkg/cpio"
	_ "github.com/u-root/u-root/pkg/cpio/newc"
	"github.com/u-root/u-root/pkg/ldd"
)

var (
	// Paths contains the paths to put into the initramfs.
	// The index is a root directory, and the value is the place from which to
	// walk. The only required root is the bbsh dir itself, and the starting
	// walk is init -- i.e. we grab only one file. Should you wish to bring in,
	// e.g., /lib/modules/4.04, you would do add the root as / and the
	// starting point for the walk as lib/modules/4.04. That way we only preserve
	// as much of the path as we need, but we can preserve it all.
	paths      = map[string][]string{}
	extraPaths = flag.String("extra", "", "Extra paths to add in the form root:start, e.g. /:etc/hosts")
	extraCmds  = flag.String("cmds", "", "Extra commands to add (full path, comma-separated string)")
)

func sanity() {
	goBinGo := filepath.Join(config.Goroot, "bin/go")
	_, err := os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	// but does the one in go/bin/OS_ARCH exist too?
	goBinGo = filepath.Join(config.Goroot, fmt.Sprintf("bin/%s_%s/go", config.Goos, config.Arch))
	_, err = os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	if config.Go == "" {
		log.Fatalf("Can't find a go binary! Is GOROOT set correctly?")
	}
}

// dirComponents takes a string and returns an array of strings,
// such that we can create the directory records for intermediate
// directories.
func dirComponents(dir string) []string {
	var dirlist []string
	if filepath.Dir(dir) == "." {
		return []string{}
	}
	for d := filepath.Dir(dir); d != "/"; d = filepath.Dir(d) {
		dirlist = append([]string{d}, dirlist...)
	}
	dirlist = append(dirlist, dir)
	return dirlist
}

// copyCommands takes a list of commands, generates the list of libs,
// and creates cpio records, including directory records.
func copyCommands(w cpio.Writer, cmds []string) {
	debug("copyCommands: start with %v", cmds)
	var recs []cpio.Record
	libs, err := uroot.LddList(cmds)
	if err != nil {
		log.Fatalf("%v\n", err)
	}
	cmds = append(cmds, libs...)
	for _, n := range cmds {
		debug("copyCommands: file %v", n)
		for _, n := range dirComponents(n) {
			debug("copyCommands: %v", n)
			r, err := cpio.GetRecord(n)
			if err != nil {
				log.Fatalf("%v: %v", n, err)
			}
			recs = append(recs, r)
		}
	}
	cpio.MakeReproducible(recs)
	if err := w.WriteRecords(recs); err != nil {
		log.Fatalf("%v\n", err)
	}
}

func ramfs() {
	archiver, err := cpio.Format("newc")
	if err != nil {
		log.Fatalf("Creating newc archiver: %v", err)
	}

	oname := fmt.Sprintf("/tmp/initramfs.%v_%v.cpio", config.Goos, config.Arch)
	f, err := os.Create(oname)
	if err != nil {
		log.Fatalf("%v\n", err)
	}

	w := archiver.Writer(f)
	cpio.MakeReproducible(devCPIO[:])
	if err := w.WriteRecords(devCPIO[:]); err != nil {
		log.Fatalf("%v\n", err)
	}

	paths[filepath.Join(config.Gopath, "src/github.com/u-root/u-root/bb/bbsh")] = []string{"init", "ubin"}

	if *extraPaths != "" {
		extras := strings.Split(*extraPaths, " ")
		for _, x := range extras {
			paths["/"] = append(paths["/"], x)
		}
	}

	if *extraCmds != "" {
		copyCommands(w, strings.Split(*extraCmds, " "))
	}

	// For all the 'roots' in paths, start walking at the name.
	debug("PATHS: %v", paths)
	for r, list := range paths {
		debug("PATHS: root %v", r)
		// we need to make all the path prefix directories.
		for _, n := range list {
			debug("\troot %v, name %v", r, n)
			for _, d := range dirComponents(n) {
				debug("\t\troot %v, name %v, component %v", r, n, d)
				rec, err := cpio.GetRecord(d)
				if err != nil {
					log.Fatalf("Getting record of %q failed: %v", d, err)
				}
				recs := []cpio.Record{rec}
				cpio.MakeReproducible(recs)
				if err := w.WriteRecords(recs); err != nil {
					log.Fatalf("%v\n", err)
				}
			}
		}
		for _, n := range list {
			if err := filepath.Walk(filepath.Join(r, n), func(name string, fi os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				cn, err := filepath.Rel(r, name)
				if err != nil {
					log.Fatalf("filepath.Rel(%v, %v): %v", r, name, err)
				}
				debug("%v\n", cn)
				rec, err := cpio.GetRecord(name)
				if err != nil {
					log.Fatalf("Getting record of %q failed: %v", cn, err)
				}
				// the name in the cpio is relative to our starting point.
				rec.Name = cn
				recs := []cpio.Record{rec}
				cpio.MakeReproducible(recs)
				if err := w.WriteRecords(recs); err != nil {
					log.Fatalf("%v\n", err)
				}
				return nil
			}); err != nil {
				log.Fatalf("bbsh walk failed: %v", err)
			}
		}
	}

	if err := w.WriteTrailer(); err != nil {
		log.Fatalf("Error writing trailer record: %v", err)
	}
	fmt.Printf("Output file is in %v\n", oname)
}
<|endoftext|>


<commit_before>// Copyright © 2016 NAME HERE <EMAIL ADDRESS>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"path/filepath"

	homedir "github.com/mitchellh/go-homedir"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// listCmd represents the list command
var listCmd = &cobra.Command{
	Use:   "list",
	Short: "List all VMs",
	Run: func(cmd *cobra.Command, args []string) {
		// d := virtualbox.NewDriver("", "")
		// outList, err := d.List()
		// if err != nil {
		// 	log.Fatal(err)
		// }
		// fmt.Print(outList)

		host := viper.GetString("server.host")
		port := viper.GetString("server.port")

		home, err := homedir.Dir()
		if err != nil {
			log.Fatal(errors.Wrap(err, "could not detect users home directory"))
		}
		// Create client
		caCert, err := ioutil.ReadFile(filepath.Join(home, ".vmproxy", "cert.pem"))
		if err != nil {
			log.Fatal(err)
		}
		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)

		// cert, err := tls.LoadX509KeyPair("client.crt", "client.key")
		// if err != nil {
		// 	log.Fatal(err)
		// }

		client := &http.Client{
			Transport: &http.Transport{
				TLSClientConfig: &tls.Config{
					RootCAs: caCertPool,
					// Certificates: []tls.Certificate{cert},
				},
			},
		}

		// Create request
		req, err := http.NewRequest("GET", "https://"+host+":"+port+"/virtualbox/list", nil)

		// Fetch Request
		resp, err := client.Do(req)
		assert(err)

		// Read Response Body
		respBody, _ := ioutil.ReadAll(resp.Body)

		// Display Results
		fmt.Print(string(respBody))
	},
}

func init() {
	RootCmd.AddCommand(listCmd)

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// listCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// listCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")

}
<commit_msg>fix url path<commit_after>// Copyright © 2016 NAME HERE <EMAIL ADDRESS>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"path/filepath"

	homedir "github.com/mitchellh/go-homedir"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// listCmd represents the list command
var listCmd = &cobra.Command{
	Use:   "list",
	Short: "List all VMs",
	Run: func(cmd *cobra.Command, args []string) {
		// d := virtualbox.NewDriver("", "")
		// outList, err := d.List()
		// if err != nil {
		// 	log.Fatal(err)
		// }
		// fmt.Print(outList)

		host := viper.GetString("server.host")
		port := viper.GetString("server.port")

		home, err := homedir.Dir()
		if err != nil {
			log.Fatal(errors.Wrap(err, "could not detect users home directory"))
		}
		// Create client
		caCert, err := ioutil.ReadFile(filepath.Join(home, ".vmproxy", "cert.pem"))
		if err != nil {
			log.Fatal(err)
		}
		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)

		// cert, err := tls.LoadX509KeyPair("client.crt", "client.key")
		// if err != nil {
		// 	log.Fatal(err)
		// }

		client := &http.Client{
			Transport: &http.Transport{
				TLSClientConfig: &tls.Config{
					RootCAs: caCertPool,
					// Certificates: []tls.Certificate{cert},
				},
			},
		}

		// Create request
		req, err := http.NewRequest("GET", "https://"+host+":"+port+"/vbox/list", nil)

		// Fetch Request
		resp, err := client.Do(req)
		assert(err)

		// Read Response Body
		respBody, _ := ioutil.ReadAll(resp.Body)

		// Display Results
		fmt.Print(string(respBody))
	},
}

func init() {
	RootCmd.AddCommand(listCmd)

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// listCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// listCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")

}
<|endoftext|>


<commit_before>package sync

import (
  "ghighlighter/models"
  "ghighlighter/readmill/readmillreadings"
  "ghighlighter/readmill/readmillhighlights"
)

func Sync() {
  config := models.Config()

  syncHighlights(config.AccessToken)
  syncReadings(config.UserId, config.AccessToken)
}

func syncHighlights(accessToken string) {
  highlights := models.Highlights()

  tmpItems := make([]models.GhHighlight, len(highlights.Items))
  copy(tmpItems, highlights.Items)

  for _, highlight := range tmpItems {
    var highlightComment *readmillhighlights.HighlightComment
    if highlight.Comment != "" {
      highlightComment = &readmillhighlights.HighlightComment{highlight.Comment}
    }

    readmillHighlight := readmillhighlights.Highlight{highlight.Content,
      highlight.Position, highlight.Timestamp, readmillhighlights.HighlightLocators{},
      highlightComment}

    success := readmillhighlights.PostHighlight(readmillHighlight, highlight.ReadingReadmillId, accessToken)

    if success {
      highlights.Delete(highlight)
    }
  }
}

func syncReadings(userId int, accessToken string) {
  readings := models.Readings()
  readmillReadings := readmillreadings.GetReadings(userId, accessToken)

  for _, readmillReading := range readmillReadings {
    title := readmillReading.Book.Title + " - " + readmillReading.Book.Author
    reading := models.GhReading{title, readmillReading.Id, 1}
    readings.Add(reading)
  }
}

<commit_msg>Pass highlight comment to PostHighlight<commit_after>package sync

import (
  "ghighlighter/models"
  "ghighlighter/readmill/readmillreadings"
  "ghighlighter/readmill/readmillhighlights"
)

func Sync() {
  config := models.Config()

  syncHighlights(config.AccessToken)
  syncReadings(config.UserId, config.AccessToken)
}

func syncHighlights(accessToken string) {
  highlights := models.Highlights()

  tmpItems := make([]models.GhHighlight, len(highlights.Items))
  copy(tmpItems, highlights.Items)

  for _, highlight := range tmpItems {
    var highlightComment *readmillhighlights.HighlightComment
    if highlight.Comment != "" {
      highlightComment = &readmillhighlights.HighlightComment{highlight.Comment}
    }

    readmillHighlight := readmillhighlights.Highlight{highlight.Content,
      highlight.Position, highlight.Timestamp, readmillhighlights.HighlightLocators{}}

    success := readmillhighlights.PostHighlight(readmillHighlight, highlightComment, highlight.ReadingReadmillId, accessToken)

    if success {
      highlights.Delete(highlight)
    }
  }
}

func syncReadings(userId int, accessToken string) {
  readings := models.Readings()
  readmillReadings := readmillreadings.GetReadings(userId, accessToken)

  for _, readmillReading := range readmillReadings {
    title := readmillReading.Book.Title + " - " + readmillReading.Book.Author
    reading := models.GhReading{title, readmillReading.Id, 1}
    readings.Add(reading)
  }
}

<|endoftext|>


<commit_before>package main

import (
	"io"
	"log"
	"fmt"
	"time"
	"reflect"
	"encoding/json"

	"code.google.com/p/go.net/websocket"
)

type Conn struct {
	ws *websocket.Conn
}

func NewConn(ws *websocket.Conn) *Conn {
	c := new(Conn)
	c.ws = ws
	return c
}

func (c *Conn) Send(m Message) error {
	var err error

	cm := new(ClientMessage)
	cm.Kind = typeToKind(m)

	cm.Payload, err = json.Marshal(m)
	if err != nil {
		return fmt.Errorf("marshalling websocket message: %s", err)
	}

	start := time.Now().UnixNano() / 1e6

	err = websocket.JSON.Send(c.ws, cm)
	if err != nil {
		return fmt.Errorf("sending websocket message: %s", err)
	}

	end := time.Now().UnixNano() / 1e6 - start
	if end > 10 {
		log.Println("That took", end, "ms to send")
	}

	return nil
}

func (c *Conn) Recv() (Message, error) {
	cm := new(ClientMessage)
	err := websocket.JSON.Receive(c.ws, cm)
	if err != nil {
		if err != io.EOF {
			log.Println("Reading websocket message:", err)
		}
		return nil, err
	}

	m := kindToType(cm.Kind)
	err = json.Unmarshal(cm.Payload, &m)
	if err != nil {
		log.Println("Unmarshalling websocket message:", err)
		return nil, err
	}

	return m, nil
}

func kindToType(kind MessageKind) Message {
	switch kind {
		case MSG_ENTITY_CREATE:
			return &MsgEntityCreate{}
		case MSG_ENTITY_POSITION:
			return &MsgEntityPosition{}
		case MSG_ENTITY_REMOVE:
			return &MsgEntityRemove{}
		case MSG_BLOCK:
			return &MsgBlock{}
		case MSG_CONTROLS_STATE:
			return &MsgControlsState{}
		case MSG_CHAT:
			return &MsgChat{}
		case MSG_PLAYER_STATE:
			return &MsgPlayerState{}
		case MSG_DEBUG_RAY:
			return &MsgDebugRay{}
		case MSG_NTP_SYNC:
			return &MsgNtpSync{}
		case MSG_INVENTORY_STATE:
			return &MsgInventoryState{}
	}
	panic("Unknown message recieved from client: " + string(kind))
}

func typeToKind(m Message) MessageKind {
	switch m.(type) {
		case *MsgEntityCreate:
			return MSG_ENTITY_CREATE
		case *MsgEntityPosition:
			return MSG_ENTITY_POSITION
		case *MsgEntityRemove:
			return MSG_ENTITY_REMOVE
		case *MsgChunk:
			return MSG_CHUNK
		case *MsgBlock:
			return MSG_BLOCK
		case *MsgControlsState:
			return MSG_CONTROLS_STATE
		case *MsgChat:
			return MSG_CHAT
		case *MsgPlayerState:
			return MSG_PLAYER_STATE
		case *MsgDebugRay:
			return MSG_DEBUG_RAY
		case *MsgNtpSync:
			return MSG_NTP_SYNC
		case *MsgInventoryState:
			return MSG_INVENTORY_STATE
	}
	panic("Attempted to send unknown message to client: " + reflect.TypeOf(m).String())
}
<commit_msg>Don't print ANY silly errors in conn.go.<commit_after>package main

import (
	"io"
	"log"
	"fmt"
	"time"
	"reflect"
	"encoding/json"

	"code.google.com/p/go.net/websocket"
)

type Conn struct {
	ws *websocket.Conn
}

func NewConn(ws *websocket.Conn) *Conn {
	c := new(Conn)
	c.ws = ws
	return c
}

func (c *Conn) Send(m Message) error {
	var err error

	cm := new(ClientMessage)
	cm.Kind = typeToKind(m)

	cm.Payload, err = json.Marshal(m)
	if err != nil {
		return fmt.Errorf("marshalling websocket message: %s", err)
	}

	start := time.Now().UnixNano() / 1e6

	err = websocket.JSON.Send(c.ws, cm)
	if err != nil {
		return fmt.Errorf("sending websocket message: %s", err)
	}

	end := time.Now().UnixNano() / 1e6 - start
	if end > 10 {
		log.Println("That took", end, "ms to send")
	}

	return nil
}

func (c *Conn) Recv() (Message, error) {
	cm := new(ClientMessage)
	err := websocket.JSON.Receive(c.ws, cm)
	if err != nil {
		if err == io.EOF {
			return nil, err
		}
		return nil, fmt.Errorf("reading websocket message: %s", err)
	}

	m := kindToType(cm.Kind)
	err = json.Unmarshal(cm.Payload, &m)
	if err != nil {
		return nil, fmt.Errorf("unmarshalling websocket message: %s", err)
	}

	return m, nil
}

func kindToType(kind MessageKind) Message {
	switch kind {
		case MSG_ENTITY_CREATE:
			return &MsgEntityCreate{}
		case MSG_ENTITY_POSITION:
			return &MsgEntityPosition{}
		case MSG_ENTITY_REMOVE:
			return &MsgEntityRemove{}
		case MSG_BLOCK:
			return &MsgBlock{}
		case MSG_CONTROLS_STATE:
			return &MsgControlsState{}
		case MSG_CHAT:
			return &MsgChat{}
		case MSG_PLAYER_STATE:
			return &MsgPlayerState{}
		case MSG_DEBUG_RAY:
			return &MsgDebugRay{}
		case MSG_NTP_SYNC:
			return &MsgNtpSync{}
		case MSG_INVENTORY_STATE:
			return &MsgInventoryState{}
	}
	panic("Unknown message recieved from client: " + string(kind))
}

func typeToKind(m Message) MessageKind {
	switch m.(type) {
		case *MsgEntityCreate:
			return MSG_ENTITY_CREATE
		case *MsgEntityPosition:
			return MSG_ENTITY_POSITION
		case *MsgEntityRemove:
			return MSG_ENTITY_REMOVE
		case *MsgChunk:
			return MSG_CHUNK
		case *MsgBlock:
			return MSG_BLOCK
		case *MsgControlsState:
			return MSG_CONTROLS_STATE
		case *MsgChat:
			return MSG_CHAT
		case *MsgPlayerState:
			return MSG_PLAYER_STATE
		case *MsgDebugRay:
			return MSG_DEBUG_RAY
		case *MsgNtpSync:
			return MSG_NTP_SYNC
		case *MsgInventoryState:
			return MSG_INVENTORY_STATE
	}
	panic("Attempted to send unknown message to client: " + reflect.TypeOf(m).String())
}
<|endoftext|>


<commit_before>package main

import "flag"
import "fmt"
import "sort"
import (
	"bufio"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"time"
)

func Usage() string {
	return fmt.Sprintf(
		`
    %sUsage of %s:%s

        %sfindref%s %s[options]%s %smatch_regex%s %s[start_dir]%s %s[filename_regex]%s

    %sArguments:%s

        %smatch_regex:  This is an RE2 regular expression that will be matched against lines
                      in each file, with matches being displayed to the user.%s

        %sstart_dir:  This optional argument sets the starting directory to crawl looking
                    for eligible files with lines matching match_regex.  Default value
                    is the current working directory, AKA $PWD or '.'%s

        %sfilename_regex:  This optional argument restricts the set of files checked for
                         matching lines.  Eligible files must match this expression.
                         Default value matches all files%s

    %sOptions:%s
        %s
        -d | --debug
              Enable debug mode
        -f | --filename-only
              Display only filenames with matches, not the matches themselves
        -h | --hidden
              Include hidden files and files in hidden directories
        -i | --ignore-case
              Ignore case in regex (overrides smart-case)
        -m | --match-case
              Match regex case (if unset smart-case is used)
        -n | --no-color
              Disable colorized output
        -s | --stats
              Track basic statistics and print them on exit
        -v | --version
              Print current version and exit
%s
`,
		colors.Red, versionString(false), colors.Restore,
		colors.Brown, colors.Restore,
		colors.Green, colors.Restore,
		colors.Cyan, colors.Restore,
		colors.Blue, colors.Restore,
		colors.Purple, colors.Restore,
		colors.Red, colors.Restore,
		colors.Cyan, colors.Restore,
		colors.Blue, colors.Restore,
		colors.Purple, colors.Restore,
		colors.Red, colors.Restore,
		colors.Green, colors.Restore,
	)
}

const Version = "0.1.0"
const Date = "2017-04-03"

var FILE_PROCESSING_COMPLETE error = nil

var settings *Settings = NewSettings()
var statistics *Statistics = NewStatistics()
var colors *Colors = NewColors()

var filenameOnlyFiles []string = make([]string, 0, 100)
var filesToScan []FileToScan = make([]FileToScan, 0, 100)

func usageAndExit() {
	flag.Usage()
	os.Exit(1)
}

func debug(a ...interface{}) {
	if settings.Debug {
		fmt.Println(a...)
	}
}

func containsNullByte(line []byte) bool {
	for _, el := range line {
		if el == 0 {
			return true
		}
	}
	return false
}

func checkForMatches(path string) []Match {
	debug(colors.Blue+"Checking file for matches:"+colors.Restore, path)
	file, err := os.Open(path)
	if err != nil {
		fmt.Println(colors.Red+"Error opening file at '"+path+"'.  It might be a directory.  Err: "+colors.Restore, err)
		debug(colors.Red+"Error opening file at '"+path+"'.  It might be a directory.  Err: "+colors.Restore, err)
		return []Match{Match{path, 0, []byte{}, []int{}}}
	}
	defer func() {
		if path == "src/main/java/com/canopy/service/EFileService.java" {
			fmt.Println("Closing the file: " + path)
		}
		file.Close()
	}()

	retval := make([]Match, 50)

	scanner := bufio.NewScanner(file)
	var lineNumber int = 0
	for scanner.Scan() {
		lineNumber += 1
		line := scanner.Bytes()
		statistics.IncrLineCount()
		if containsNullByte(line) {
			// This is a binary file.  Skip it!
			debug(colors.Blue+"Not processing binary file:"+colors.Restore, path)
			return retval
		}
		if matchIndex := settings.MatchRegex.FindIndex(line); matchIndex != nil {
			// we have a match! loc == nil means no match so just ignore that case
			statistics.IncrMatchCount()
			if settings.FilenameOnly {
				filenameOnlyFiles = append(filenameOnlyFiles, path)
			} else {
				m := Match{path, lineNumber, line, matchIndex}
				m.printMatch()
				retval = append(retval, Match{path, lineNumber, line, matchIndex})
			}
		}
	}

	if err := scanner.Err(); err != nil {
		debug(colors.Red+"Error scanning line from file '"+path+"'. File will be skipped.  Err: "+colors.Restore, err)
	}
	return retval
}

func processFile(path string, info os.FileInfo, err error) error {
	if err != nil {
		debug("filepath.Walk encountered error with path '"+path+"'", err)
		return FILE_PROCESSING_COMPLETE
	}

	if info.IsDir() {
		if settings.IsHidden(path) {
			debug(colors.Blue, "Directory", path, "is hidden and will be pruned", colors.Restore)
			return filepath.SkipDir // skip the whole sub-contents of this hidden directory
		} else {
			return FILE_PROCESSING_COMPLETE
		}
	}

	if settings.PassesFileFilter(path) {
		debug(colors.Blue+"Passes file filter:", path)
		if settings.IsHidden(path) {
			debug(colors.Blue + "Hidden file '" + colors.Restore + path + colors.Blue + "' not processed")
			return FILE_PROCESSING_COMPLETE
		}
		statistics.IncrFilesToScan()
		defer statistics.IncrFileCount()

		filesToScan = append(filesToScan, FileToScan{Path: path, Info: info, Err: err})
	} else {
		debug(colors.Blue + "Ignoring file cause it doesn't match filter: " + colors.Restore + path)
	}
	return FILE_PROCESSING_COMPLETE
}

func getMatchRegex(ignoreCase bool, matchCase bool, usersRegex string) *regexp.Regexp {
	// If ignore case is set, ignore the case of the regex.
	// if match-case is not set, use smart case which means if it's all lower case be case-insensitive,
	// but if there's capitals then be case-sensitive
	if ignoreCase || (!matchCase && !regexp.MustCompile("[A-Z]").MatchString(usersRegex)) {
		debug(colors.Blue, "Match regex will be case-insensitive", colors.Restore)
		return regexp.MustCompile("(?i)" + usersRegex)
	} else {
		debug(colors.Blue, "Match regex will be exactly as user provided", colors.Restore)
		return regexp.MustCompile(usersRegex)
	}
}

func versionString(color bool) string {
	if color {
		return fmt.Sprintf("%s%s%s%s%s%s%s", colors.Cyan, "findref (version ", Version, " released on ", Date, ")", colors.Restore)
	} else {
		return fmt.Sprintf("%s%s%s%s%s", "findref (version ", Version, " released on ", Date, ")")
	}
}

func printVersion() {
	fmt.Println(versionString(true))
}

func uniq(stringSlice []string) []string {
	/* There is no built-in uniq function for slices, so we will use a map */
	stringMap := make(map[string]bool)
	for _, v := range stringSlice {
		stringMap[v] = true
	}
	retval := make([]string, len(stringMap), len(stringMap))
	i := 0
	for key := range stringMap {
		retval[i] = key
		i++
	}
	return retval
}

func finishAndExit() {
	if settings.FilenameOnly {
		filenames := uniq(filenameOnlyFiles)
		sort.Strings(filenames)
		for _, filename := range filenames {
			fmt.Printf("%s%s%s\n", colors.Purple, filename, colors.Restore)
		}
	}

	if settings.TrackStats {
		fmt.Printf("%sElapsed time:%s  %s\n", colors.Cyan, colors.Restore, statistics.ElapsedTime().String())
		fmt.Printf("%sLines scanned:%s %d\n", colors.Cyan, colors.Restore, statistics.LineCount())
		fmt.Printf("%sFiles scanned:%s %d\n", colors.Cyan, colors.Restore, statistics.FileCount())
		fmt.Printf("%sMatches found:%s %d\n", colors.Cyan, colors.Restore, statistics.MatchCount())
	}
}

func worker(id int, jobs <-chan string, results chan<- []Match) {
	for file := range jobs {
		debug(colors.Blue, "Worker number", id, "started file", file, colors.Restore)
		results <- checkForMatches(file)
		debug(colors.Blue, "Worker number", id, "finished file", file, colors.Restore)
	}
}

func main() {
	sPtr := flag.Bool("s", false, "Alias for --stats")
	dPtr := flag.Bool("d", false, "Alias for --debug")
	hPtr := flag.Bool("h", false, "Alias for --hidden")
	vPtr := flag.Bool("v", false, "Alias for --version")
	nPtr := flag.Bool("n", false, "Alias for --no-color")
	mPtr := flag.Bool("m", false, "Alias for --match-case")
	iPtr := flag.Bool("i", false, "Alias for --ignore-case")
	fPtr := flag.Bool("f", false, "Alias for --filename-only")
	helpPtr := flag.Bool("help", false, "Show usage")
	statsPtr := flag.Bool("stats", false, "Track and display statistics")
	debugPtr := flag.Bool("debug", false, "Enable debug mode")
	hiddenPtr := flag.Bool("hidden", false, "Include hidden files and files in hidden directories")
	versionPtr := flag.Bool("version", false, "Print current version and exit")
	nocolorPtr := flag.Bool("no-color", false, "Don't use color in output")
	matchCasePtr := flag.Bool("match-case", false, "Match regex case (if unset smart-case is used)")
	ignoreCasePtr := flag.Bool("ignore-case", false, "Ignore case in regex (overrides smart-case)")
	filenameOnlyPtr := flag.Bool("filename-only", false, "Display only filenames with matches")

	flag.Usage = func() {
		fmt.Print(Usage())
	}
	flag.Parse()

	if *vPtr || *versionPtr {
		printVersion()
		os.Exit(0)
	}

	if *helpPtr {
		usageAndExit()
	}

	if *nPtr || *nocolorPtr {
		debug("Color output is disabled")
		colors.ZeroColors()
	}

	settings.Debug = *debugPtr || *dPtr
	settings.TrackStats = *statsPtr || *sPtr
	settings.FilenameOnly = *filenameOnlyPtr || *fPtr
	settings.IncludeHidden = *hiddenPtr || *hPtr
	*matchCasePtr = *matchCasePtr || *mPtr
	*ignoreCasePtr = *ignoreCasePtr || *iPtr

	if settings.TrackStats {
		statistics.startTime = time.Now()
		debug(colors.Blue, "Start time is:", colors.Restore, statistics.startTime.String())
	}

	debug(colors.Blue, "stats enabled: ", colors.Restore, settings.TrackStats)
	debug(colors.Blue, "match-case enabled: ", colors.Restore, *matchCasePtr)
	debug(colors.Blue, "ignore-case enabled: ", colors.Restore, *ignoreCasePtr)
	debug(colors.Blue, "include hidden files: ", colors.Restore, settings.IncludeHidden)
	debug(colors.Blue, "debug mode: ", colors.Restore, settings.Debug)
	debug(colors.Blue, "filename only: ", colors.Restore, settings.FilenameOnly)

	rootDir := "."

	if len(flag.Args()) < 1 {
		fmt.Errorf("%s", "Must specify regex to match against files")
		usageAndExit()
	} else if len(flag.Args()) > 3 {
		fmt.Errorf("%s", "Too many args (expected 1 <= 3)")
		usageAndExit()
	} else {
		settings.MatchRegex = getMatchRegex(*ignoreCasePtr, *matchCasePtr, flag.Args()[0])

		if len(flag.Args()) >= 2 {
			rootDir = flag.Args()[1]
		}
		if len(flag.Args()) == 3 {
			settings.FilenameRegex = regexp.MustCompile(flag.Args()[2])
		}
	}

	debug(colors.Blue, "matchRegex: ", colors.Restore, settings.MatchRegex.String())
	debug(colors.Blue, "rootDir: ", colors.Restore, rootDir)
	debug(colors.Blue, "fileRegex: ", colors.Restore, settings.FilenameRegex.String())

	runtime.GOMAXPROCS(runtime.NumCPU())
	filepath.Walk(rootDir, processFile)

	// TODO: set niceness value to low

	jobs := make(chan string, len(filesToScan))
	results := make(chan []Match, 100)

	// two workers for each core
	numWorkers := runtime.NumCPU() * 1
	for w := 0; w < numWorkers; w++ {
		go worker(w, jobs, results)
	}

	// create a job for each file to scan
	for _, val := range filesToScan {
		jobs <- val.Path
	}
	close(jobs)

	for r := 0; r < len(filesToScan); r++ {
		result := <-results
		for _, res := range result {
			if res.hasMatch() {
				//res.printMatch()
			}
		}
	}

	finishAndExit()
}
<commit_msg>Release version 1.0.0<commit_after>package main

import "flag"
import "fmt"
import "sort"
import (
	"bufio"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"time"
)

func Usage() string {
	return fmt.Sprintf(
		`
    %sUsage of %s:%s

        %sfindref%s %s[options]%s %smatch_regex%s %s[start_dir]%s %s[filename_regex]%s

    %sArguments:%s

        %smatch_regex:  This is an RE2 regular expression that will be matched against lines
                      in each file, with matches being displayed to the user.%s

        %sstart_dir:  This optional argument sets the starting directory to crawl looking
                    for eligible files with lines matching match_regex.  Default value
                    is the current working directory, AKA $PWD or '.'%s

        %sfilename_regex:  This optional argument restricts the set of files checked for
                         matching lines.  Eligible files must match this expression.
                         Default value matches all files%s

    %sOptions:%s
        %s
        -d | --debug
              Enable debug mode
        -f | --filename-only
              Display only filenames with matches, not the matches themselves
        -h | --hidden
              Include hidden files and files in hidden directories
        -i | --ignore-case
              Ignore case in regex (overrides smart-case)
        -m | --match-case
              Match regex case (if unset smart-case is used)
        -n | --no-color
              Disable colorized output
        -s | --stats
              Track basic statistics and print them on exit
        -v | --version
              Print current version and exit
%s
`,
		colors.Red, versionString(false), colors.Restore,
		colors.Brown, colors.Restore,
		colors.Green, colors.Restore,
		colors.Cyan, colors.Restore,
		colors.Blue, colors.Restore,
		colors.Purple, colors.Restore,
		colors.Red, colors.Restore,
		colors.Cyan, colors.Restore,
		colors.Blue, colors.Restore,
		colors.Purple, colors.Restore,
		colors.Red, colors.Restore,
		colors.Green, colors.Restore,
	)
}

const Version = "1.0.0"
const Date = "2020-04-21"

var FILE_PROCESSING_COMPLETE error = nil

var settings *Settings = NewSettings()
var statistics *Statistics = NewStatistics()
var colors *Colors = NewColors()

var filenameOnlyFiles []string = make([]string, 0, 100)
var filesToScan []FileToScan = make([]FileToScan, 0, 100)

func usageAndExit() {
	flag.Usage()
	os.Exit(1)
}

func debug(a ...interface{}) {
	if settings.Debug {
		fmt.Println(a...)
	}
}

func containsNullByte(line []byte) bool {
	for _, el := range line {
		if el == 0 {
			return true
		}
	}
	return false
}

func checkForMatches(path string) []Match {
	debug(colors.Blue+"Checking file for matches:"+colors.Restore, path)
	file, err := os.Open(path)
	if err != nil {
		fmt.Println(colors.Red+"Error opening file at '"+path+"'.  It might be a directory.  Err: "+colors.Restore, err)
		debug(colors.Red+"Error opening file at '"+path+"'.  It might be a directory.  Err: "+colors.Restore, err)
		return []Match{Match{path, 0, []byte{}, []int{}}}
	}
	defer func() {
		if path == "src/main/java/com/canopy/service/EFileService.java" {
			fmt.Println("Closing the file: " + path)
		}
		file.Close()
	}()

	retval := make([]Match, 50)

	scanner := bufio.NewScanner(file)
	var lineNumber int = 0
	for scanner.Scan() {
		lineNumber += 1
		line := scanner.Bytes()
		statistics.IncrLineCount()
		if containsNullByte(line) {
			// This is a binary file.  Skip it!
			debug(colors.Blue+"Not processing binary file:"+colors.Restore, path)
			return retval
		}
		if matchIndex := settings.MatchRegex.FindIndex(line); matchIndex != nil {
			// we have a match! loc == nil means no match so just ignore that case
			statistics.IncrMatchCount()
			if settings.FilenameOnly {
				filenameOnlyFiles = append(filenameOnlyFiles, path)
			} else {
				m := Match{path, lineNumber, line, matchIndex}
				m.printMatch()
				retval = append(retval, Match{path, lineNumber, line, matchIndex})
			}
		}
	}

	if err := scanner.Err(); err != nil {
		debug(colors.Red+"Error scanning line from file '"+path+"'. File will be skipped.  Err: "+colors.Restore, err)
	}
	return retval
}

func processFile(path string, info os.FileInfo, err error) error {
	if err != nil {
		debug("filepath.Walk encountered error with path '"+path+"'", err)
		return FILE_PROCESSING_COMPLETE
	}

	if info.IsDir() {
		if settings.IsHidden(path) {
			debug(colors.Blue, "Directory", path, "is hidden and will be pruned", colors.Restore)
			return filepath.SkipDir // skip the whole sub-contents of this hidden directory
		} else {
			return FILE_PROCESSING_COMPLETE
		}
	}

	if settings.PassesFileFilter(path) {
		debug(colors.Blue+"Passes file filter:", path)
		if settings.IsHidden(path) {
			debug(colors.Blue + "Hidden file '" + colors.Restore + path + colors.Blue + "' not processed")
			return FILE_PROCESSING_COMPLETE
		}
		statistics.IncrFilesToScan()
		defer statistics.IncrFileCount()

		filesToScan = append(filesToScan, FileToScan{Path: path, Info: info, Err: err})
	} else {
		debug(colors.Blue + "Ignoring file cause it doesn't match filter: " + colors.Restore + path)
	}
	return FILE_PROCESSING_COMPLETE
}

func getMatchRegex(ignoreCase bool, matchCase bool, usersRegex string) *regexp.Regexp {
	// If ignore case is set, ignore the case of the regex.
	// if match-case is not set, use smart case which means if it's all lower case be case-insensitive,
	// but if there's capitals then be case-sensitive
	if ignoreCase || (!matchCase && !regexp.MustCompile("[A-Z]").MatchString(usersRegex)) {
		debug(colors.Blue, "Match regex will be case-insensitive", colors.Restore)
		return regexp.MustCompile("(?i)" + usersRegex)
	} else {
		debug(colors.Blue, "Match regex will be exactly as user provided", colors.Restore)
		return regexp.MustCompile(usersRegex)
	}
}

func versionString(color bool) string {
	if color {
		return fmt.Sprintf("%s%s%s%s%s%s%s", colors.Cyan, "findref (version ", Version, " released on ", Date, ")", colors.Restore)
	} else {
		return fmt.Sprintf("%s%s%s%s%s", "findref (version ", Version, " released on ", Date, ")")
	}
}

func printVersion() {
	fmt.Println(versionString(true))
}

func uniq(stringSlice []string) []string {
	/* There is no built-in uniq function for slices, so we will use a map */
	stringMap := make(map[string]bool)
	for _, v := range stringSlice {
		stringMap[v] = true
	}
	retval := make([]string, len(stringMap), len(stringMap))
	i := 0
	for key := range stringMap {
		retval[i] = key
		i++
	}
	return retval
}

func finishAndExit() {
	if settings.FilenameOnly {
		filenames := uniq(filenameOnlyFiles)
		sort.Strings(filenames)
		for _, filename := range filenames {
			fmt.Printf("%s%s%s\n", colors.Purple, filename, colors.Restore)
		}
	}

	if settings.TrackStats {
		fmt.Printf("%sElapsed time:%s  %s\n", colors.Cyan, colors.Restore, statistics.ElapsedTime().String())
		fmt.Printf("%sLines scanned:%s %d\n", colors.Cyan, colors.Restore, statistics.LineCount())
		fmt.Printf("%sFiles scanned:%s %d\n", colors.Cyan, colors.Restore, statistics.FileCount())
		fmt.Printf("%sMatches found:%s %d\n", colors.Cyan, colors.Restore, statistics.MatchCount())
	}
}

func worker(id int, jobs <-chan string, results chan<- []Match) {
	for file := range jobs {
		debug(colors.Blue, "Worker number", id, "started file", file, colors.Restore)
		results <- checkForMatches(file)
		debug(colors.Blue, "Worker number", id, "finished file", file, colors.Restore)
	}
}

func main() {
	sPtr := flag.Bool("s", false, "Alias for --stats")
	dPtr := flag.Bool("d", false, "Alias for --debug")
	hPtr := flag.Bool("h", false, "Alias for --hidden")
	vPtr := flag.Bool("v", false, "Alias for --version")
	nPtr := flag.Bool("n", false, "Alias for --no-color")
	mPtr := flag.Bool("m", false, "Alias for --match-case")
	iPtr := flag.Bool("i", false, "Alias for --ignore-case")
	fPtr := flag.Bool("f", false, "Alias for --filename-only")
	helpPtr := flag.Bool("help", false, "Show usage")
	statsPtr := flag.Bool("stats", false, "Track and display statistics")
	debugPtr := flag.Bool("debug", false, "Enable debug mode")
	hiddenPtr := flag.Bool("hidden", false, "Include hidden files and files in hidden directories")
	versionPtr := flag.Bool("version", false, "Print current version and exit")
	nocolorPtr := flag.Bool("no-color", false, "Don't use color in output")
	matchCasePtr := flag.Bool("match-case", false, "Match regex case (if unset smart-case is used)")
	ignoreCasePtr := flag.Bool("ignore-case", false, "Ignore case in regex (overrides smart-case)")
	filenameOnlyPtr := flag.Bool("filename-only", false, "Display only filenames with matches")

	flag.Usage = func() {
		fmt.Print(Usage())
	}
	flag.Parse()

	if *vPtr || *versionPtr {
		printVersion()
		os.Exit(0)
	}

	if *helpPtr {
		usageAndExit()
	}

	if *nPtr || *nocolorPtr {
		debug("Color output is disabled")
		colors.ZeroColors()
	}

	settings.Debug = *debugPtr || *dPtr
	settings.TrackStats = *statsPtr || *sPtr
	settings.FilenameOnly = *filenameOnlyPtr || *fPtr
	settings.IncludeHidden = *hiddenPtr || *hPtr
	*matchCasePtr = *matchCasePtr || *mPtr
	*ignoreCasePtr = *ignoreCasePtr || *iPtr

	if settings.TrackStats {
		statistics.startTime = time.Now()
		debug(colors.Blue, "Start time is:", colors.Restore, statistics.startTime.String())
	}

	debug(colors.Blue, "stats enabled: ", colors.Restore, settings.TrackStats)
	debug(colors.Blue, "match-case enabled: ", colors.Restore, *matchCasePtr)
	debug(colors.Blue, "ignore-case enabled: ", colors.Restore, *ignoreCasePtr)
	debug(colors.Blue, "include hidden files: ", colors.Restore, settings.IncludeHidden)
	debug(colors.Blue, "debug mode: ", colors.Restore, settings.Debug)
	debug(colors.Blue, "filename only: ", colors.Restore, settings.FilenameOnly)

	rootDir := "."

	if len(flag.Args()) < 1 {
		fmt.Errorf("%s", "Must specify regex to match against files")
		usageAndExit()
	} else if len(flag.Args()) > 3 {
		fmt.Errorf("%s", "Too many args (expected 1 <= 3)")
		usageAndExit()
	} else {
		settings.MatchRegex = getMatchRegex(*ignoreCasePtr, *matchCasePtr, flag.Args()[0])

		if len(flag.Args()) >= 2 {
			rootDir = flag.Args()[1]
		}
		if len(flag.Args()) == 3 {
			settings.FilenameRegex = regexp.MustCompile(flag.Args()[2])
		}
	}

	debug(colors.Blue, "matchRegex: ", colors.Restore, settings.MatchRegex.String())
	debug(colors.Blue, "rootDir: ", colors.Restore, rootDir)
	debug(colors.Blue, "fileRegex: ", colors.Restore, settings.FilenameRegex.String())

	runtime.GOMAXPROCS(runtime.NumCPU())
	filepath.Walk(rootDir, processFile)

	// TODO: set niceness value to low

	jobs := make(chan string, len(filesToScan))
	results := make(chan []Match, 100)

	// two workers for each core
	numWorkers := runtime.NumCPU() * 1
	for w := 0; w < numWorkers; w++ {
		go worker(w, jobs, results)
	}

	// create a job for each file to scan
	for _, val := range filesToScan {
		jobs <- val.Path
	}
	close(jobs)

	for r := 0; r < len(filesToScan); r++ {
		result := <-results
		for _, res := range result {
			if res.hasMatch() {
				//res.printMatch()
			}
		}
	}

	finishAndExit()
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package cat adds cat images to an issue or PR in response to a /meow comment
package cat

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/pluginhelp"
	"k8s.io/test-infra/prow/plugins"
)

var (
	match = regexp.MustCompile(`(?mi)^/meow(vie)?(?: (.+))?\s*$`)
	grumpyKeywords = regexp.MustCompile(`(?mi)^(no|grumpy)\s*$`)
	meow  = &realClowder{
		url: "https://api.thecatapi.com/v1/images/search?format=json&results_per_page=1",
	}
)

const (
	pluginName = "cat"
	grumpyURL  = "https://upload.wikimedia.org/wikipedia/commons/e/ee/Grumpy_Cat_by_Gage_Skidmore.jpg"
)

func init() {
	plugins.RegisterGenericCommentHandler(pluginName, handleGenericComment, helpProvider)
}

func helpProvider(config *plugins.Configuration, enabledRepos []string) (*pluginhelp.PluginHelp, error) {
	pluginHelp := &pluginhelp.PluginHelp{
		Description: "The cat plugin adds a cat image to an issue or PR in response to the `/meow` command.",
		Config: map[string]string{
			"": fmt.Sprintf("The cat plugin uses an api key for thecatapi.com stored in %s.", config.Cat.KeyPath),
		},
	}
	pluginHelp.AddCommand(pluginhelp.Command{
		Usage:       "/meow(vie) [CATegory]",
		Description: "Add a cat image to the issue or PR",
		Featured:    false,
		WhoCanUse:   "Anyone",
		Examples:    []string{"/meow", "/meow caturday", "/meowvie clothes"},
	})
	return pluginHelp, nil
}

type githubClient interface {
	CreateComment(owner, repo string, number int, comment string) error
}

type clowder interface {
	readCat(string, bool) (string, error)
}

type realClowder struct {
	url     string
	lock    sync.RWMutex
	update  time.Time
	key     string
	keyPath string
}

func (c *realClowder) setKey(keyPath string, log *logrus.Entry) {
	c.lock.Lock()
	defer c.lock.Unlock()
	if !time.Now().After(c.update) {
		return
	}
	c.update = time.Now().Add(1 * time.Minute)
	if keyPath == "" {
		c.key = ""
		return
	}
	b, err := ioutil.ReadFile(keyPath)
	if err == nil {
		c.key = strings.TrimSpace(string(b))
		return
	}
	log.WithError(err).Errorf("failed to read key at %s", keyPath)
	c.key = ""
}

type catResult struct {
	Image string `json:"url"`
}

func (cr catResult) Format() (string, error) {
	if cr.Image == "" {
		return "", errors.New("empty image url")
	}
	img, err := url.Parse(cr.Image)
	if err != nil {
		return "", fmt.Errorf("invalid image url %s: %v", cr.Image, err)
	}

	return fmt.Sprintf("![cat image](%s)", img), nil
}

func (c *realClowder) URL(category string, movieCat bool) string {
	c.lock.RLock()
	defer c.lock.RUnlock()
	uri := string(c.url)
	if category != "" {
		uri += "&category=" + url.QueryEscape(category)
	}
	if c.key != "" {
		uri += "&api_key=" + url.QueryEscape(c.key)
	}
	if movieCat {
		uri += "&mime_types=gif"
	}
	return uri
}

func (c *realClowder) readCat(category string, movieCat bool) (string, error) {
	cats := make([]catResult, 0)
	uri := c.URL(category, movieCat)
	//if strings.TrimSpace(category) == "no" {
	if grumpyKeywords.FindStringSubmatch(category) != nil {
		cats = append(cats, catResult{grumpyURL})
	} else {
		resp, err := http.Get(uri)
		if err != nil {
			return "", fmt.Errorf("could not read cat from %s: %v", uri, err)
		}
		defer resp.Body.Close()
		if sc := resp.StatusCode; sc > 299 || sc < 200 {
			return "", fmt.Errorf("failing %d response from %s", sc, uri)
		}
		if err = json.NewDecoder(resp.Body).Decode(&cats); err != nil {
			return "", err
		}
		if len(cats) < 1 {
			return "", fmt.Errorf("no cats in response from %s", uri)
		}
	}
	a := cats[0]
	if a.Image == "" {
		return "", fmt.Errorf("no image url in response from %s", uri)
	}
	// checking size, GitHub doesn't support big images
	toobig, err := github.ImageTooBig(a.Image)
	if err != nil {
		return "", fmt.Errorf("could not validate image size %s: %v", a.Image, err)
	} else if toobig {
		return "", fmt.Errorf("longcat is too long: %s", a.Image)
	}
	return a.Format()
}

func handleGenericComment(pc plugins.Agent, e github.GenericCommentEvent) error {
	return handle(
		pc.GitHubClient,
		pc.Logger,
		&e,
		meow,
		func() { meow.setKey(pc.PluginConfig.Cat.KeyPath, pc.Logger) },
	)
}

func handle(gc githubClient, log *logrus.Entry, e *github.GenericCommentEvent, c clowder, setKey func()) error {
	// Only consider new comments.
	if e.Action != github.GenericCommentActionCreated {
		return nil
	}
	// Make sure they are requesting a cat
	mat := match.FindStringSubmatch(e.Body)
	if mat == nil {
		return nil
	}

	category, movieCat, err := parseMatch(mat)
	if err != nil {
		return err
	}

	// Now that we know this is a relevant event we can set the key.
	setKey()

	org := e.Repo.Owner.Login
	repo := e.Repo.Name
	number := e.Number

	for i := 0; i < 3; i++ {
		resp, err := c.readCat(category, movieCat)
		if err != nil {
			log.WithError(err).Error("Failed to get cat img")
			continue
		}
		return gc.CreateComment(org, repo, number, plugins.FormatResponseRaw(e.Body, e.HTMLURL, e.User.Login, resp))
	}

	var msg string
	if category != "" {
		msg = "Bad category. Please see https://api.thecatapi.com/api/categories/list"
	} else {
		msg = "https://thecatapi.com appears to be down"
	}
	if err := gc.CreateComment(org, repo, number, plugins.FormatResponseRaw(e.Body, e.HTMLURL, e.User.Login, msg)); err != nil {
		log.WithError(err).Error("Failed to leave comment")
	}

	return errors.New("could not find a valid cat image")
}

func parseMatch(mat []string) (string, bool, error) {
	if len(mat) != 3 {
		err := fmt.Errorf("expected 3 capture groups in regexp match, but got %d", len(mat))
		return "", false, err
	}
	category := strings.TrimSpace(mat[2])
	movieCat := len(mat[1]) > 0 // "vie" suffix is present.
	return category, movieCat, nil
}
<commit_msg>Fix with go-fmt<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package cat adds cat images to an issue or PR in response to a /meow comment
package cat

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/sirupsen/logrus"

	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/pluginhelp"
	"k8s.io/test-infra/prow/plugins"
)

var (
	match          = regexp.MustCompile(`(?mi)^/meow(vie)?(?: (.+))?\s*$`)
	grumpyKeywords = regexp.MustCompile(`(?mi)^(no|grumpy)\s*$`)
	meow           = &realClowder{
		url: "https://api.thecatapi.com/v1/images/search?format=json&results_per_page=1",
	}
)

const (
	pluginName = "cat"
	grumpyURL  = "https://upload.wikimedia.org/wikipedia/commons/e/ee/Grumpy_Cat_by_Gage_Skidmore.jpg"
)

func init() {
	plugins.RegisterGenericCommentHandler(pluginName, handleGenericComment, helpProvider)
}

func helpProvider(config *plugins.Configuration, enabledRepos []string) (*pluginhelp.PluginHelp, error) {
	pluginHelp := &pluginhelp.PluginHelp{
		Description: "The cat plugin adds a cat image to an issue or PR in response to the `/meow` command.",
		Config: map[string]string{
			"": fmt.Sprintf("The cat plugin uses an api key for thecatapi.com stored in %s.", config.Cat.KeyPath),
		},
	}
	pluginHelp.AddCommand(pluginhelp.Command{
		Usage:       "/meow(vie) [CATegory]",
		Description: "Add a cat image to the issue or PR",
		Featured:    false,
		WhoCanUse:   "Anyone",
		Examples:    []string{"/meow", "/meow caturday", "/meowvie clothes"},
	})
	return pluginHelp, nil
}

type githubClient interface {
	CreateComment(owner, repo string, number int, comment string) error
}

type clowder interface {
	readCat(string, bool) (string, error)
}

type realClowder struct {
	url     string
	lock    sync.RWMutex
	update  time.Time
	key     string
	keyPath string
}

func (c *realClowder) setKey(keyPath string, log *logrus.Entry) {
	c.lock.Lock()
	defer c.lock.Unlock()
	if !time.Now().After(c.update) {
		return
	}
	c.update = time.Now().Add(1 * time.Minute)
	if keyPath == "" {
		c.key = ""
		return
	}
	b, err := ioutil.ReadFile(keyPath)
	if err == nil {
		c.key = strings.TrimSpace(string(b))
		return
	}
	log.WithError(err).Errorf("failed to read key at %s", keyPath)
	c.key = ""
}

type catResult struct {
	Image string `json:"url"`
}

func (cr catResult) Format() (string, error) {
	if cr.Image == "" {
		return "", errors.New("empty image url")
	}
	img, err := url.Parse(cr.Image)
	if err != nil {
		return "", fmt.Errorf("invalid image url %s: %v", cr.Image, err)
	}

	return fmt.Sprintf("![cat image](%s)", img), nil
}

func (c *realClowder) URL(category string, movieCat bool) string {
	c.lock.RLock()
	defer c.lock.RUnlock()
	uri := string(c.url)
	if category != "" {
		uri += "&category=" + url.QueryEscape(category)
	}
	if c.key != "" {
		uri += "&api_key=" + url.QueryEscape(c.key)
	}
	if movieCat {
		uri += "&mime_types=gif"
	}
	return uri
}

func (c *realClowder) readCat(category string, movieCat bool) (string, error) {
	cats := make([]catResult, 0)
	uri := c.URL(category, movieCat)
	//if strings.TrimSpace(category) == "no" {
	if grumpyKeywords.FindStringSubmatch(category) != nil {
		cats = append(cats, catResult{grumpyURL})
	} else {
		resp, err := http.Get(uri)
		if err != nil {
			return "", fmt.Errorf("could not read cat from %s: %v", uri, err)
		}
		defer resp.Body.Close()
		if sc := resp.StatusCode; sc > 299 || sc < 200 {
			return "", fmt.Errorf("failing %d response from %s", sc, uri)
		}
		if err = json.NewDecoder(resp.Body).Decode(&cats); err != nil {
			return "", err
		}
		if len(cats) < 1 {
			return "", fmt.Errorf("no cats in response from %s", uri)
		}
	}
	a := cats[0]
	if a.Image == "" {
		return "", fmt.Errorf("no image url in response from %s", uri)
	}
	// checking size, GitHub doesn't support big images
	toobig, err := github.ImageTooBig(a.Image)
	if err != nil {
		return "", fmt.Errorf("could not validate image size %s: %v", a.Image, err)
	} else if toobig {
		return "", fmt.Errorf("longcat is too long: %s", a.Image)
	}
	return a.Format()
}

func handleGenericComment(pc plugins.Agent, e github.GenericCommentEvent) error {
	return handle(
		pc.GitHubClient,
		pc.Logger,
		&e,
		meow,
		func() { meow.setKey(pc.PluginConfig.Cat.KeyPath, pc.Logger) },
	)
}

func handle(gc githubClient, log *logrus.Entry, e *github.GenericCommentEvent, c clowder, setKey func()) error {
	// Only consider new comments.
	if e.Action != github.GenericCommentActionCreated {
		return nil
	}
	// Make sure they are requesting a cat
	mat := match.FindStringSubmatch(e.Body)
	if mat == nil {
		return nil
	}

	category, movieCat, err := parseMatch(mat)
	if err != nil {
		return err
	}

	// Now that we know this is a relevant event we can set the key.
	setKey()

	org := e.Repo.Owner.Login
	repo := e.Repo.Name
	number := e.Number

	for i := 0; i < 3; i++ {
		resp, err := c.readCat(category, movieCat)
		if err != nil {
			log.WithError(err).Error("Failed to get cat img")
			continue
		}
		return gc.CreateComment(org, repo, number, plugins.FormatResponseRaw(e.Body, e.HTMLURL, e.User.Login, resp))
	}

	var msg string
	if category != "" {
		msg = "Bad category. Please see https://api.thecatapi.com/api/categories/list"
	} else {
		msg = "https://thecatapi.com appears to be down"
	}
	if err := gc.CreateComment(org, repo, number, plugins.FormatResponseRaw(e.Body, e.HTMLURL, e.User.Login, msg)); err != nil {
		log.WithError(err).Error("Failed to leave comment")
	}

	return errors.New("could not find a valid cat image")
}

func parseMatch(mat []string) (string, bool, error) {
	if len(mat) != 3 {
		err := fmt.Errorf("expected 3 capture groups in regexp match, but got %d", len(mat))
		return "", false, err
	}
	category := strings.TrimSpace(mat[2])
	movieCat := len(mat[1]) > 0 // "vie" suffix is present.
	return category, movieCat, nil
}
<|endoftext|>


<commit_before>package kdtree

import (
	"encoding/json"
	"fmt"
	"sort"
)

// Datapoint stores a set of floating-point values and a pointer to any other
// structure or type which you may wish to associate with the Datapoint.
type Datapoint struct {
	data interface{} // ideally a pointer to some other associated thing
	set  []float64
}

// Datapoints is a slice multiple of pointers to individual Datapoints
type Datapoints []*Datapoint

// NewDatapoint is an explicit constructor as an alternative to manual declaration
func NewDatapoint(data interface{}, points []float64) (*Datapoint, error) {
	if points == nil {
		return nil, fmt.Errorf("passed nil points slice to Datapoint constructor")
	}
	f := make([]float64, len(points), len(points))
	copy(f, points)
	d := Datapoint{
		data: data,
		set:  f,
	}
	return &d, nil
}

// Data returns the interface value of the object that the Datapoint is linked with.
func (d *Datapoint) Data() interface{} {
	return d.data
}

// Set returns a copy of the slice of floating-point values
func (d *Datapoint) Set() []float64 {
	var export = make([]float64, len(d.set), len(d.set))
	copy(export, d.set)
	return export
}

// Dimensionality returns spatial dimensions the Datapoint fits over.
func (d *Datapoint) Dimensionality() int {
	return len(d.set)
}

// RandomDatapoint will produce a 'free' PRNG Datapoint in n dimensions
// where all values in the set lie in [0,1).
// Useful for testing or adding noise to a dataset.
func RandomDatapoint(n uint) *Datapoint {
	return RandomDatapointInRange(n, 0, 1)
}

// RandomDatapointInRange will produce a 'free' PRNG Datapoint in n dimensions
// where all values in the set lie in [min,max).
// Useful for testing or adding noise to a dataset.
func RandomDatapointInRange(n uint, min, max float64) *Datapoint {
	f := make([]float64, n, n)
	for i := range f {
		f[i] = randomFloatInRange(min, max)
	}
	d := Datapoint{
		data: nil,
		set:  f,
	}
	return &d
}

// String returns a formatted string presentation of the Datapoint object,
// implementing Stringer interface
func (d *Datapoint) String() string {
	var present string
	present += fmt.Sprintf("{data: %v}\n", d.data)
	present += "{set: ["
	for i := range d.set {
		present += fmt.Sprintf("%d:{%v}, ", i, d.set[i])
	}
	present = present[0 : len(present)-2]
	present += "]}"
	return present
}

// By is the function signature required to wrap a given Less method as closure
type By func(p, q *Datapoint) bool

type datapointSorter struct {
	Datapoints
	by By // closure used in the Less method.
}

// Sort acts as interface implementation wrapper on a collection of Datapoints,
// called by functions with the By signature
func (by By) Sort(d Datapoints) {
	ds := &datapointSorter{
		Datapoints: d,
		by:         by,
	}
	sort.Sort(ds)
}

func (s *datapointSorter) Len() int {
	return len(s.Datapoints)
}

func (s *datapointSorter) Swap(i, j int) {
	s.Datapoints[i], s.Datapoints[j] = s.Datapoints[j], s.Datapoints[i]
}

func (s *datapointSorter) Less(i, j int) bool {
	return s.by(s.Datapoints[i], s.Datapoints[j])
}

// Comparator returns a dynamic "By" function on the specified plane,
// which gets passed to the Sort implementation's Less method.
func Comparator(plane int) By {
	return func(p, q *Datapoint) bool {
		return p.set[plane] < q.set[plane]
	}
}

// EqualTo provides a direct equality comparison between two Datapoints
func (d *Datapoint) EqualTo(q *Datapoint) bool {
	if len(d.set) != len(q.set) {
		return false
	}
	for i := range d.set {
		if d.set[i] != q.set[i] {
			return false
		}
	}
	return true
}

// EqualTo provides an equality comparison between each Datapoint in a set of Datapoints.
func (ds Datapoints) EqualTo(qs Datapoints) bool {
	if len(ds) != len(qs) {
		return false
	}
	for i := range ds {
		if !ds[i].EqualTo(qs[i]) {
			return false
		}
	}
	return true
}

// Importable is the interface implemented by types who can be directly converted into a valid Datapoint.
type Importable interface {
	ToDatapoint() *Datapoint
}

// Exportable is the interface implemented by types which can be take a Datapoint and use the set of floating-point values to update the calling object's data members.
type Exportable interface {
	FromDatapoint(*Datapoint)
}

// Import uses the Importable interface to cleanly append a single Datapoint to a the end of a set (slice) of Datapoints
func (ds Datapoints) Import(I Importable) {
	ds = append(ds, I.ToDatapoint())
}

// Convert uses the Importable interface to cleanly produce a kdtree
// from a slice of some type which has implented ToDataPoint(), and
// according to the pivot algorithm (PivotFunc).
func Convert(c []Importable, pivotDef PivotFunc) (*Branch, error) {
	var points = make(Datapoints, len(c), len(c))
	// basedim := len(c[0].ToDatapoint().set)
	for i := range c {
		points[i] = c[i].ToDatapoint()
		// TODO:
		// Implement build error types.
		// if points[i].Dimensionality() != basedim {
		// 	return nil, DimClashError
		// }
	}

	if pivotDef == nil {
		pivotDef = LazyAverage
	}

	b := Build(points, 0, pivotDef)
	return b, nil
}

// MarshalJSON implements encoding/json Marshaler interface
func (d *Datapoint) MarshalJSON() ([]byte, error) {
	return json.Marshal(map[string]interface{}{
		"data": d.data,
		"set":  d.set,
	})
}

// TODO: Implement encoding/json Unmarshaler interface method
// func (d *Datapoint) UnmarshalJSON([]byte) error {
// }
<commit_msg>Added points==nil check in NewDatapoint(data, points)<commit_after>package kdtree

import (
	"encoding/json"
	"fmt"
	"sort"
)

// Datapoint stores a set of floating-point values and a pointer to any other
// structure or type which you may wish to associate with the Datapoint.
type Datapoint struct {
	data interface{} // ideally a pointer to some other associated thing
	set  []float64
}

// Datapoints is a slice multiple of pointers to individual Datapoints
type Datapoints []*Datapoint

// NewDatapoint is an explicit constructor as an alternative to manual declaration
func NewDatapoint(data interface{}, points []float64) *Datapoint {
	if points == nil {
		points = []float64{}
	}
	f := make([]float64, len(points), len(points))
	copy(f, points)
	d := Datapoint{
		data: data,
		set:  f,
	}
	return &d
}

// Data returns the interface value of the object that the Datapoint is linked with.
func (d *Datapoint) Data() interface{} {
	return d.data
}

// Set returns a copy of the slice of floating-point values
func (d *Datapoint) Set() []float64 {
	var export = make([]float64, len(d.set), len(d.set))
	copy(export, d.set)
	return export
}

// Dimensionality returns spatial dimensions the Datapoint fits over.
func (d *Datapoint) Dimensionality() int {
	return len(d.set)
}

// RandomDatapoint will produce a 'free' PRNG Datapoint in n dimensions
// where all values in the set lie in [0,1).
// Useful for testing or adding noise to a dataset.
func RandomDatapoint(n uint) *Datapoint {
	return RandomDatapointInRange(n, 0, 1)
}

// RandomDatapointInRange will produce a 'free' PRNG Datapoint in n dimensions
// where all values in the set lie in [min,max).
// Useful for testing or adding noise to a dataset.
func RandomDatapointInRange(n uint, min, max float64) *Datapoint {
	f := make([]float64, n, n)
	for i := range f {
		f[i] = randomFloatInRange(min, max)
	}
	d := Datapoint{
		data: nil,
		set:  f,
	}
	return &d
}

// String returns a formatted string presentation of the Datapoint object,
// implementing Stringer interface
func (d *Datapoint) String() string {
	var present string
	present += fmt.Sprintf("{data: %v}\n", d.data)
	present += "{set: ["
	for i := range d.set {
		present += fmt.Sprintf("%d:{%v}, ", i, d.set[i])
	}
	present = present[0 : len(present)-2]
	present += "]}"
	return present
}

// By is the function signature required to wrap a given Less method as closure
type By func(p, q *Datapoint) bool

type datapointSorter struct {
	Datapoints
	by By // closure used in the Less method.
}

// Sort acts as interface implementation wrapper on a collection of Datapoints,
// called by functions with the By signature
func (by By) Sort(d Datapoints) {
	ds := &datapointSorter{
		Datapoints: d,
		by:         by,
	}
	sort.Sort(ds)
}

func (s *datapointSorter) Len() int {
	return len(s.Datapoints)
}

func (s *datapointSorter) Swap(i, j int) {
	s.Datapoints[i], s.Datapoints[j] = s.Datapoints[j], s.Datapoints[i]
}

func (s *datapointSorter) Less(i, j int) bool {
	return s.by(s.Datapoints[i], s.Datapoints[j])
}

// Comparator returns a dynamic "By" function on the specified plane,
// which gets passed to the Sort implementation's Less method.
func Comparator(plane int) By {
	return func(p, q *Datapoint) bool {
		return p.set[plane] < q.set[plane]
	}
}

// EqualTo provides a direct equality comparison between two Datapoints
func (d *Datapoint) EqualTo(q *Datapoint) bool {
	if len(d.set) != len(q.set) {
		return false
	}
	for i := range d.set {
		if d.set[i] != q.set[i] {
			return false
		}
	}
	return true
}

// EqualTo provides an equality comparison between each Datapoint in a set of Datapoints.
func (ds Datapoints) EqualTo(qs Datapoints) bool {
	if len(ds) != len(qs) {
		return false
	}
	for i := range ds {
		if !ds[i].EqualTo(qs[i]) {
			return false
		}
	}
	return true
}

// Importable is the interface implemented by types who can be directly converted into a valid Datapoint.
type Importable interface {
	ToDatapoint() *Datapoint
}

// Exportable is the interface implemented by types which can be take a Datapoint and use the set of floating-point values to update the calling object's data members.
type Exportable interface {
	FromDatapoint(*Datapoint)
}

// Import uses the Importable interface to cleanly append a single Datapoint to a the end of a set (slice) of Datapoints
func (ds Datapoints) Import(I Importable) {
	ds = append(ds, I.ToDatapoint())
}

// Convert uses the Importable interface to cleanly produce a kdtree
// from a slice of some type which has implented ToDataPoint(), and
// according to the pivot algorithm (PivotFunc).
func Convert(c []Importable, pivotDef PivotFunc) (*Branch, error) {
	var points = make(Datapoints, len(c), len(c))
	// basedim := len(c[0].ToDatapoint().set)
	for i := range c {
		points[i] = c[i].ToDatapoint()
		// TODO:
		// Implement build error types.
		// if points[i].Dimensionality() != basedim {
		// 	return nil, DimClashError
		// }
	}

	if pivotDef == nil {
		pivotDef = LazyAverage
	}

	b := Build(points, 0, pivotDef)
	return b, nil
}

// MarshalJSON implements encoding/json Marshaler interface
func (d *Datapoint) MarshalJSON() ([]byte, error) {
	return json.Marshal(map[string]interface{}{
		"data": d.data,
		"set":  d.set,
	})
}

// TODO: Implement encoding/json Unmarshaler interface method
// func (d *Datapoint) UnmarshalJSON([]byte) error {
// }
<|endoftext|>


<commit_before>// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gremlin

import (
	"reflect"
	"sort"
	"testing"

	"github.com/google/cayley/graph"
	"github.com/google/cayley/quad"

	_ "github.com/google/cayley/graph/memstore"
	_ "github.com/google/cayley/writer"
)

// This is a simple test graph.
//
//    +---+                        +---+
//    | A |-------               ->| F |<--
//    +---+       \------>+---+-/  +---+   \--+---+
//                 ------>|#B#|      |        | E |
//    +---+-------/      >+---+      |        +---+
//    | C |             /            v
//    +---+           -/           +---+
//      ----    +---+/             |#G#|
//          \-->|#D#|------------->+---+
//              +---+
//
var simpleGraph = []quad.Quad{
	{"A", "follows", "B", ""},
	{"C", "follows", "B", ""},
	{"C", "follows", "D", ""},
	{"D", "follows", "B", ""},
	{"B", "follows", "F", ""},
	{"F", "follows", "G", ""},
	{"D", "follows", "G", ""},
	{"E", "follows", "F", ""},
	{"B", "status", "cool", "status_graph"},
	{"D", "status", "cool", "status_graph"},
	{"G", "status", "cool", "status_graph"},
}

func makeTestSession(data []quad.Quad) *Session {
	ts, _ := graph.NewTripleStore("memstore", "", nil)
	w, _ := graph.NewQuadWriter("single", ts, nil)
	for _, t := range data {
		w.AddQuad(t)
	}
	return NewSession(ts, -1, false)
}

var testQueries = []struct {
	message string
	query   string
	tag     string
	expect  []string
}{
	// Simple query tests.
	{
		message: "get a single vertex",
		query: `
			g.V("A").All()
		`,
		expect: []string{"A"},
	},
	{
		message: "use .Out()",
		query: `
			g.V("A").Out("follows").All()
		`,
		expect: []string{"B"},
	},
	{
		message: "use .In()",
		query: `
			g.V("B").In("follows").All()
		`,
		expect: []string{"A", "C", "D"},
	},
	{
		message: "use .Both()",
		query: `
			g.V("F").Both("follows").All()
		`,
		expect: []string{"B", "G", "E"},
	},
	{
		message: "use .Tag()-.Is()-.Back()",
		query: `
			g.V("B").In("follows").Tag("foo").Out("status").Is("cool").Back("foo").All()
		`,
		expect: []string{"D"},
	},
	{
		message: "separate .Tag()-.Is()-.Back()",
		query: `
			x = g.V("C").Out("follows").Tag("foo").Out("status").Is("cool").Back("foo")
			x.In("follows").Is("D").Back("foo").All()
		`,
		expect: []string{"B"},
	},
	{
		message: "do multiple .Back()s",
		query: `
			g.V("E").Out("follows").As("f").Out("follows").Out("status").Is("cool").Back("f").In("follows").In("follows").As("acd").Out("status").Is("cool").Back("f").All()
		`,
		tag:    "acd",
		expect: []string{"D"},
	},

	// Morphism tests.
	{
		message: "show simple morphism",
		query: `
			grandfollows = g.M().Out("follows").Out("follows")
			g.V("C").Follow(grandfollows).All()
		`,
		expect: []string{"G", "F", "B"},
	},
	{
		message: "show reverse morphism",
		query: `
			grandfollows = g.M().Out("follows").Out("follows")
			g.V("F").FollowR(grandfollows).All()
		`,
		expect: []string{"A", "C", "D"},
	},

	// Intersection tests.
	{
		message: "show simple intersection",
		query: `
			function follows(x) { return g.V(x).Out("follows") }
			follows("D").And(follows("C")).All()
		`,
		expect: []string{"B"},
	},
	{
		message: "show simple morphism intersection",
		query: `
			grandfollows = g.M().Out("follows").Out("follows")
			function gfollows(x) { return g.V(x).Follow(grandfollows) }
			gfollows("A").And(gfollows("C")).All()
		`,
		expect: []string{"F"},
	},
	{
		message: "show double morphism intersection",
		query: `
			grandfollows = g.M().Out("follows").Out("follows")
			function gfollows(x) { return g.V(x).Follow(grandfollows) }
			gfollows("E").And(gfollows("C")).And(gfollows("B")).All()
		`,
		expect: []string{"G"},
	},
	{
		message: "show reverse intersection",
		query: `
			grandfollows = g.M().Out("follows").Out("follows")
			g.V("G").FollowR(grandfollows).Intersect(g.V("F").FollowR(grandfollows)).All()
		`,
		expect: []string{"C"},
	},
	{
		message: "show standard sort of morphism intersection, continue follow",
		query: `gfollowers = g.M().In("follows").In("follows")
			function cool(x) { return g.V(x).As("a").Out("status").Is("cool").Back("a") }
			cool("G").Follow(gfollowers).Intersect(cool("B").Follow(gfollowers)).All()
		`,
		expect: []string{"C"},
	},

	// Gremlin Has tests.
	{
		message: "show a simple Has",
		query: `
				g.V().Has("status", "cool").All()
		`,
		expect: []string{"G", "D", "B"},
	},
	{
		message: "show a double Has",
		query: `
				g.V().Has("status", "cool").Has("follows", "F").All()
		`,
		expect: []string{"B"},
	},

	// Tag tests.
	{
		message: "show a simple save",
		query: `
			g.V().Save("status", "somecool").All()
		`,
		tag:    "somecool",
		expect: []string{"cool", "cool", "cool"},
	},
	{
		message: "show a simple saveR",
		query: `
			g.V("cool").SaveR("status", "who").All()
		`,
		tag:    "who",
		expect: []string{"G", "D", "B"},
	},
	{
		message: "show an out save",
		query: `
			g.V("D").Out(null, "pred").All()
		`,
		tag:    "pred",
		expect: []string{"follows", "follows", "status"},
	},
	{
		message: "show a tag list",
		query: `
			g.V("D").Out(null, ["pred", "foo", "bar"]).All()
		`,
		tag:    "foo",
		expect: []string{"follows", "follows", "status"},
	},
	{
		message: "show a pred list",
		query: `
			g.V("D").Out(["follows", "status"]).All()
		`,
		expect: []string{"B", "G", "cool"},
	},
	{
		message: "show a predicate path",
		query: `
			g.V("D").Out(g.V("follows"), "pred").All()
		`,
		expect: []string{"B", "G"},
	},
}

func runQueryGetTag(g []quad.Quad, query string, tag string) []string {
	js := makeTestSession(g)
	c := make(chan interface{}, 5)
	js.ExecInput(query, c, -1)

	var results []string
	for res := range c {
		data := res.(*Result)
		if data.val == nil {
			val := (*data.actualResults)[tag]
			if val != nil {
				results = append(results, js.ts.NameOf(val))
			}
		}
	}

	return results
}

func TestGremlin(t *testing.T) {
	for _, test := range testQueries {
		if test.tag == "" {
			test.tag = TopResultTag
		}
		got := runQueryGetTag(simpleGraph, test.query, test.tag)
		sort.Strings(got)
		sort.Strings(test.expect)
		if !reflect.DeepEqual(got, test.expect) {
			t.Errorf("Failed to %s, got: %v expected: %v", test.message, got, test.expect)
		}
	}
}
<commit_msg>Fix build<commit_after>// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gremlin

import (
	"reflect"
	"sort"
	"testing"

	"github.com/google/cayley/graph"
	"github.com/google/cayley/quad"

	_ "github.com/google/cayley/graph/memstore"
	_ "github.com/google/cayley/writer"
)

// This is a simple test graph.
//
//    +---+                        +---+
//    | A |-------               ->| F |<--
//    +---+       \------>+---+-/  +---+   \--+---+
//                 ------>|#B#|      |        | E |
//    +---+-------/      >+---+      |        +---+
//    | C |             /            v
//    +---+           -/           +---+
//      ----    +---+/             |#G#|
//          \-->|#D#|------------->+---+
//              +---+
//
var simpleGraph = []quad.Quad{
	{"A", "follows", "B", ""},
	{"C", "follows", "B", ""},
	{"C", "follows", "D", ""},
	{"D", "follows", "B", ""},
	{"B", "follows", "F", ""},
	{"F", "follows", "G", ""},
	{"D", "follows", "G", ""},
	{"E", "follows", "F", ""},
	{"B", "status", "cool", "status_graph"},
	{"D", "status", "cool", "status_graph"},
	{"G", "status", "cool", "status_graph"},
}

func makeTestSession(data []quad.Quad) *Session {
	ts, _ := graph.NewTripleStore("memstore", "", nil)
	w, _ := graph.NewQuadWriter("single", ts, nil)
	for _, t := range data {
		w.AddQuad(t)
	}
	return NewSession(ts, -1, false)
}

var testQueries = []struct {
	message string
	query   string
	tag     string
	expect  []string
}{
	// Simple query tests.
	{
		message: "get a single vertex",
		query: `
			g.V("A").All()
		`,
		expect: []string{"A"},
	},
	{
		message: "use .Out()",
		query: `
			g.V("A").Out("follows").All()
		`,
		expect: []string{"B"},
	},
	{
		message: "use .In()",
		query: `
			g.V("B").In("follows").All()
		`,
		expect: []string{"A", "C", "D"},
	},
	{
		message: "use .Both()",
		query: `
			g.V("F").Both("follows").All()
		`,
		expect: []string{"B", "G", "E"},
	},
	{
		message: "use .Tag()-.Is()-.Back()",
		query: `
			g.V("B").In("follows").Tag("foo").Out("status").Is("cool").Back("foo").All()
		`,
		expect: []string{"D"},
	},
	{
		message: "separate .Tag()-.Is()-.Back()",
		query: `
			x = g.V("C").Out("follows").Tag("foo").Out("status").Is("cool").Back("foo")
			x.In("follows").Is("D").Back("foo").All()
		`,
		expect: []string{"B"},
	},
	{
		message: "do multiple .Back()s",
		query: `
			g.V("E").Out("follows").As("f").Out("follows").Out("status").Is("cool").Back("f").In("follows").In("follows").As("acd").Out("status").Is("cool").Back("f").All()
		`,
		tag:    "acd",
		expect: []string{"D"},
	},

	// Morphism tests.
	{
		message: "show simple morphism",
		query: `
			grandfollows = g.M().Out("follows").Out("follows")
			g.V("C").Follow(grandfollows).All()
		`,
		expect: []string{"G", "F", "B"},
	},
	{
		message: "show reverse morphism",
		query: `
			grandfollows = g.M().Out("follows").Out("follows")
			g.V("F").FollowR(grandfollows).All()
		`,
		expect: []string{"A", "C", "D"},
	},

	// Intersection tests.
	{
		message: "show simple intersection",
		query: `
			function follows(x) { return g.V(x).Out("follows") }
			follows("D").And(follows("C")).All()
		`,
		expect: []string{"B"},
	},
	{
		message: "show simple morphism intersection",
		query: `
			grandfollows = g.M().Out("follows").Out("follows")
			function gfollows(x) { return g.V(x).Follow(grandfollows) }
			gfollows("A").And(gfollows("C")).All()
		`,
		expect: []string{"F"},
	},
	{
		message: "show double morphism intersection",
		query: `
			grandfollows = g.M().Out("follows").Out("follows")
			function gfollows(x) { return g.V(x).Follow(grandfollows) }
			gfollows("E").And(gfollows("C")).And(gfollows("B")).All()
		`,
		expect: []string{"G"},
	},
	{
		message: "show reverse intersection",
		query: `
			grandfollows = g.M().Out("follows").Out("follows")
			g.V("G").FollowR(grandfollows).Intersect(g.V("F").FollowR(grandfollows)).All()
		`,
		expect: []string{"C"},
	},
	{
		message: "show standard sort of morphism intersection, continue follow",
		query: `gfollowers = g.M().In("follows").In("follows")
			function cool(x) { return g.V(x).As("a").Out("status").Is("cool").Back("a") }
			cool("G").Follow(gfollowers).Intersect(cool("B").Follow(gfollowers)).All()
		`,
		expect: []string{"C"},
	},

	// Gremlin Has tests.
	{
		message: "show a simple Has",
		query: `
				g.V().Has("status", "cool").All()
		`,
		expect: []string{"G", "D", "B"},
	},
	{
		message: "show a double Has",
		query: `
				g.V().Has("status", "cool").Has("follows", "F").All()
		`,
		expect: []string{"B"},
	},

	// Tag tests.
	{
		message: "show a simple save",
		query: `
			g.V().Save("status", "somecool").All()
		`,
		tag:    "somecool",
		expect: []string{"cool", "cool", "cool"},
	},
	{
		message: "show a simple saveR",
		query: `
			g.V("cool").SaveR("status", "who").All()
		`,
		tag:    "who",
		expect: []string{"G", "D", "B"},
	},
	{
		message: "show an out save",
		query: `
			g.V("D").Out(null, "pred").All()
		`,
		tag:    "pred",
		expect: []string{"follows", "follows", "status"},
	},
	{
		message: "show a tag list",
		query: `
			g.V("D").Out(null, ["pred", "foo", "bar"]).All()
		`,
		tag:    "foo",
		expect: []string{"follows", "follows", "status"},
	},
	{
		message: "show a pred list",
		query: `
			g.V("D").Out(["follows", "status"]).All()
		`,
		expect: []string{"B", "G", "cool"},
	},
	{
		message: "show a predicate path",
		query: `
			g.V("D").Out(g.V("follows"), "pred").All()
		`,
		expect: []string{"B", "G"},
	},
}

func runQueryGetTag(g []quad.Quad, query string, tag string) []string {
	js := makeTestSession(g)
	c := make(chan interface{}, 5)
	js.ExecInput(query, c, -1)

	var results []string
	for res := range c {
		data := res.(*Result)
		if data.val == nil {
			val := data.actualResults[tag]
			if val != nil {
				results = append(results, js.ts.NameOf(val))
			}
		}
	}

	return results
}

func TestGremlin(t *testing.T) {
	for _, test := range testQueries {
		if test.tag == "" {
			test.tag = TopResultTag
		}
		got := runQueryGetTag(simpleGraph, test.query, test.tag)
		sort.Strings(got)
		sort.Strings(test.expect)
		if !reflect.DeepEqual(got, test.expect) {
			t.Errorf("Failed to %s, got: %v expected: %v", test.message, got, test.expect)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Ole Krüger.
// Licensed under the MIT license which can be found in the LICENSE file.

package cemi

import (
	"errors"
	"fmt"
)

// IndividualAddr is an address for a KNX device.
type IndividualAddr uint16

// NewIndividualAddr3 generates a group address with format a/b/c.
func NewIndividualAddr3(a, b, c uint8) IndividualAddr {
	return IndividualAddr(a&15)<<12 | IndividualAddr(b&15)<<8 | IndividualAddr(c)
}

// NewIndividualAddr2 generates a group address with format a/b.
func NewIndividualAddr2(a, b uint8) IndividualAddr {
	return IndividualAddr(a)<<8 | IndividualAddr(b)
}

// NewIndividualAddrString parses the given string as a individual address. Supported formats are
// %d.%d.%d, %d.%d and %d.
func NewIndividualAddrString(addr string) (IndividualAddr, error) {
	var a, b, c uint
	num, _ := fmt.Sscanf(addr, "%d.%d.%d", &a, &b, &c)

	switch {
	case num > 2:
		return NewIndividualAddr3(uint8(a), uint8(b), uint8(c)), nil

	case num > 1:
		return NewIndividualAddr2(uint8(a), uint8(b)), nil

	case num > 0:
		return IndividualAddr(a), nil
	}

	return 0, errors.New("Input is not an individual address")
}

// String generates a string representation.
func (addr IndividualAddr) String() string {
	return fmt.Sprintf("%d.%d.%d", uint8(addr>>12)&15, uint8(addr>>8)&15, uint8(addr))
}

// GroupAddr is an address for a KNX group object.
type GroupAddr uint16

// NewGroupAddr3 generates a group address with format a/b/c.
func NewGroupAddr3(a, b, c uint8) GroupAddr {
	return GroupAddr(a&31)<<11 | GroupAddr(b&7)<<8 | GroupAddr(c)
}

// NewGroupAddr2 generates a group address with format a/b.
func NewGroupAddr2(a, b uint8) GroupAddr {
	return GroupAddr(a)<<8 | GroupAddr(b)
}

// NewGroupAddrString parses the given string as a group address. Supported formats are %d/%d/%d,
// %d/%d and %d.
func NewGroupAddrString(addr string) (GroupAddr, error) {
	var a, b, c uint
	num, _ := fmt.Sscanf(addr, "%d/%d/%d", &a, &b, &c)

	switch {
	case num > 2:
		return NewGroupAddr3(uint8(a), uint8(b), uint8(c)), nil

	case num > 1:
		return NewGroupAddr2(uint8(a), uint8(b)), nil

	case num > 0:
		return GroupAddr(b), nil
	}

	return 0, errors.New("Input is not a group address")
}

// String generates a string representation.
func (addr GroupAddr) String() string {
	return fmt.Sprintf("%d/%d/%d", uint8(addr>>11)&31, uint8(addr>>8)&7, uint8(addr))
}
<commit_msg>Fix parsing for 1-component group addresses<commit_after>// Copyright 2017 Ole Krüger.
// Licensed under the MIT license which can be found in the LICENSE file.

package cemi

import (
	"errors"
	"fmt"
)

// IndividualAddr is an address for a KNX device.
type IndividualAddr uint16

// NewIndividualAddr3 generates a group address with format a/b/c.
func NewIndividualAddr3(a, b, c uint8) IndividualAddr {
	return IndividualAddr(a&15)<<12 | IndividualAddr(b&15)<<8 | IndividualAddr(c)
}

// NewIndividualAddr2 generates a group address with format a/b.
func NewIndividualAddr2(a, b uint8) IndividualAddr {
	return IndividualAddr(a)<<8 | IndividualAddr(b)
}

// NewIndividualAddrString parses the given string as a individual address. Supported formats are
// %d.%d.%d, %d.%d and %d.
func NewIndividualAddrString(addr string) (IndividualAddr, error) {
	var a, b, c uint
	num, _ := fmt.Sscanf(addr, "%d.%d.%d", &a, &b, &c)

	switch {
	case num > 2:
		return NewIndividualAddr3(uint8(a), uint8(b), uint8(c)), nil

	case num > 1:
		return NewIndividualAddr2(uint8(a), uint8(b)), nil

	case num > 0:
		return IndividualAddr(a), nil
	}

	return 0, errors.New("Input is not an individual address")
}

// String generates a string representation.
func (addr IndividualAddr) String() string {
	return fmt.Sprintf("%d.%d.%d", uint8(addr>>12)&15, uint8(addr>>8)&15, uint8(addr))
}

// GroupAddr is an address for a KNX group object.
type GroupAddr uint16

// NewGroupAddr3 generates a group address with format a/b/c.
func NewGroupAddr3(a, b, c uint8) GroupAddr {
	return GroupAddr(a&31)<<11 | GroupAddr(b&7)<<8 | GroupAddr(c)
}

// NewGroupAddr2 generates a group address with format a/b.
func NewGroupAddr2(a, b uint8) GroupAddr {
	return GroupAddr(a)<<8 | GroupAddr(b)
}

// NewGroupAddrString parses the given string as a group address. Supported formats are %d/%d/%d,
// %d/%d and %d.
func NewGroupAddrString(addr string) (GroupAddr, error) {
	var a, b, c uint
	num, _ := fmt.Sscanf(addr, "%d/%d/%d", &a, &b, &c)

	switch {
	case num > 2:
		return NewGroupAddr3(uint8(a), uint8(b), uint8(c)), nil

	case num > 1:
		return NewGroupAddr2(uint8(a), uint8(b)), nil

	case num > 0:
		return GroupAddr(a), nil
	}

	return 0, errors.New("Input is not a group address")
}

// String generates a string representation.
func (addr GroupAddr) String() string {
	return fmt.Sprintf("%d/%d/%d", uint8(addr>>11)&31, uint8(addr>>8)&7, uint8(addr))
}
<|endoftext|>