<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/influxdata/telegraf/agent"
	"github.com/influxdata/telegraf/internal/config"

	_ "github.com/influxdata/telegraf/plugins/inputs/all"
	_ "github.com/influxdata/telegraf/plugins/outputs/all"
)

var fDebug = flag.Bool("debug", false,
	"show metrics as they're generated to stdout")
var fQuiet = flag.Bool("quiet", false,
	"run in quiet mode")
var fTest = flag.Bool("test", false, "gather metrics, print them out, and exit")
var fConfig = flag.String("config", "", "configuration file to load")
var fConfigDirectory = flag.String("config-directory", "",
	"directory containing additional *.conf files")
var fVersion = flag.Bool("version", false, "display the version")
var fSampleConfig = flag.Bool("sample-config", false,
	"print out full sample configuration")
var fPidfile = flag.String("pidfile", "", "file to write our pid to")
var fInputFilters = flag.String("input-filter", "",
	"filter the inputs to enable, separator is :")
var fOutputFilters = flag.String("output-filter", "",
	"filter the outputs to enable, separator is :")
var fUsage = flag.String("usage", "",
	"print usage for a plugin, ie, 'telegraf -usage mysql'")

var fInputFiltersLegacy = flag.String("filter", "",
	"filter the inputs to enable, separator is :")
var fOutputFiltersLegacy = flag.String("outputfilter", "",
	"filter the outputs to enable, separator is :")
var fConfigDirectoryLegacy = flag.String("configdirectory", "",
	"directory containing additional *.conf files")

// Telegraf version
//	-ldflags "-X main.Version=`git describe --always --tags`"
var Version string

const usage = `Telegraf, The plugin-driven server agent for collecting and reporting metrics.

Usage:

  telegraf <flags>

The flags are:

  -config <file>     configuration file to load
  -test              gather metrics once, print them to stdout, and exit
  -sample-config     print out full sample configuration to stdout
  -config-directory  directory containing additional *.conf files
  -input-filter      filter the input plugins to enable, separator is :
  -output-filter     filter the output plugins to enable, separator is :
  -usage             print usage for a plugin, ie, 'telegraf -usage mysql'
  -debug             print metrics as they're generated to stdout
  -quiet             run in quiet mode
  -version           print the version to stdout

Examples:

  # generate a telegraf config file:
  telegraf -sample-config > telegraf.conf

  # generate config with only cpu input & influxdb output plugins defined
  telegraf -sample-config -input-filter cpu -output-filter influxdb

  # run a single telegraf collection, outputing metrics to stdout
  telegraf -config telegraf.conf -test

  # run telegraf with all plugins defined in config file
  telegraf -config telegraf.conf

  # run telegraf, enabling the cpu & memory input, and influxdb output plugins
  telegraf -config telegraf.conf -input-filter cpu:mem -output-filter influxdb
`

func main() {
	reload := make(chan bool, 1)
	reload <- true
	for <-reload {
		reload <- false
		flag.Usage = func() { usageExit(0) }
		flag.Parse()

		if flag.NFlag() == 0 {
			usageExit(0)
		}

		var inputFilters []string
		if *fInputFiltersLegacy != "" {
			inputFilter := strings.TrimSpace(*fInputFiltersLegacy)
			inputFilters = strings.Split(":"+inputFilter+":", ":")
		}
		if *fInputFilters != "" {
			inputFilter := strings.TrimSpace(*fInputFilters)
			inputFilters = strings.Split(":"+inputFilter+":", ":")
		}

		var outputFilters []string
		if *fOutputFiltersLegacy != "" {
			outputFilter := strings.TrimSpace(*fOutputFiltersLegacy)
			outputFilters = strings.Split(":"+outputFilter+":", ":")
		}
		if *fOutputFilters != "" {
			outputFilter := strings.TrimSpace(*fOutputFilters)
			outputFilters = strings.Split(":"+outputFilter+":", ":")
		}

		if *fVersion {
			v := fmt.Sprintf("Telegraf - Version %s", Version)
			fmt.Println(v)
			return
		}

		if *fSampleConfig {
			config.PrintSampleConfig(inputFilters, outputFilters)
			return
		}

		if *fUsage != "" {
			if err := config.PrintInputConfig(*fUsage); err != nil {
				if err2 := config.PrintOutputConfig(*fUsage); err2 != nil {
					log.Fatalf("%s and %s", err, err2)
				}
			}
			return
		}

		var (
			c   *config.Config
			err error
		)

		if *fConfig != "" {
			c = config.NewConfig()
			c.OutputFilters = outputFilters
			c.InputFilters = inputFilters
			err = c.LoadConfig(*fConfig)
			if err != nil {
				log.Fatal(err)
			}
		} else {
			fmt.Println("You must specify a config file. See telegraf --help")
			os.Exit(1)
		}

		if *fConfigDirectoryLegacy != "" {
			err = c.LoadDirectory(*fConfigDirectoryLegacy)
			if err != nil {
				log.Fatal(err)
			}
		}

		if *fConfigDirectory != "" {
			err = c.LoadDirectory(*fConfigDirectory)
			if err != nil {
				log.Fatal(err)
			}
		}
		if len(c.Outputs) == 0 {
			log.Fatalf("Error: no outputs found, did you provide a valid config file?")
		}
		if len(c.Inputs) == 0 {
			log.Fatalf("Error: no inputs found, did you provide a valid config file?")
		}

		ag, err := agent.NewAgent(c)
		if err != nil {
			log.Fatal(err)
		}

		if *fDebug {
			ag.Config.Agent.Debug = true
		}

		if *fQuiet {
			ag.Config.Agent.Quiet = true
		}

		if *fTest {
			err = ag.Test()
			if err != nil {
				log.Fatal(err)
			}
			return
		}

		err = ag.Connect()
		if err != nil {
			log.Fatal(err)
		}

		shutdown := make(chan struct{})
		signals := make(chan os.Signal)
		signal.Notify(signals, os.Interrupt, syscall.SIGHUP)
		go func() {
			sig := <-signals
			if sig == os.Interrupt {
				close(shutdown)
			}
			if sig == syscall.SIGHUP {
				log.Printf("Reloading Telegraf config\n")
				<-reload
				reload <- true
				close(shutdown)
			}
		}()

		log.Printf("Starting Telegraf (version %s)\n", Version)
		log.Printf("Loaded outputs: %s", strings.Join(c.OutputNames(), " "))
		log.Printf("Loaded inputs: %s", strings.Join(c.InputNames(), " "))
		log.Printf("Tags enabled: %s", c.ListTags())

		if *fPidfile != "" {
			f, err := os.Create(*fPidfile)
			if err != nil {
				log.Fatalf("Unable to create pidfile: %s", err)
			}

			fmt.Fprintf(f, "%d\n", os.Getpid())

			f.Close()
		}

		ag.Run(shutdown)
	}
}

func usageExit(rc int) {
	fmt.Println(usage)
	os.Exit(rc)
}
<commit_msg>Add flags -usage-list to print all plugins inputs for telegraf<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/influxdata/telegraf/agent"
	"github.com/influxdata/telegraf/internal/config"
	"github.com/influxdata/telegraf/plugins/inputs"
	_ "github.com/influxdata/telegraf/plugins/inputs/all"
	_ "github.com/influxdata/telegraf/plugins/outputs/all"
)

var fDebug = flag.Bool("debug", false,
	"show metrics as they're generated to stdout")
var fQuiet = flag.Bool("quiet", false,
	"run in quiet mode")
var fTest = flag.Bool("test", false, "gather metrics, print them out, and exit")
var fConfig = flag.String("config", "", "configuration file to load")
var fConfigDirectory = flag.String("config-directory", "",
	"directory containing additional *.conf files")
var fVersion = flag.Bool("version", false, "display the version")
var fSampleConfig = flag.Bool("sample-config", false,
	"print out full sample configuration")
var fPidfile = flag.String("pidfile", "", "file to write our pid to")
var fInputFilters = flag.String("input-filter", "",
	"filter the inputs to enable, separator is :")
var fOutputFilters = flag.String("output-filter", "",
	"filter the outputs to enable, separator is :")
var fUsage = flag.String("usage", "",
	"print usage for a plugin, ie, 'telegraf -usage mysql'")
var fUsageList = flag.Bool("usage-list", false, "print all the plugins inputs")

var fInputFiltersLegacy = flag.String("filter", "",
	"filter the inputs to enable, separator is :")
var fOutputFiltersLegacy = flag.String("outputfilter", "",
	"filter the outputs to enable, separator is :")
var fConfigDirectoryLegacy = flag.String("configdirectory", "",
	"directory containing additional *.conf files")

// Telegraf version
//	-ldflags "-X main.Version=`git describe --always --tags`"
var Version string

const usage = `Telegraf, The plugin-driven server agent for collecting and reporting metrics.

Usage:

  telegraf <flags>

The flags are:

  -config <file>     configuration file to load
  -test              gather metrics once, print them to stdout, and exit
  -sample-config     print out full sample configuration to stdout
  -config-directory  directory containing additional *.conf files
  -input-filter      filter the input plugins to enable, separator is :
  -output-filter     filter the output plugins to enable, separator is :
  -usage             print usage for a plugin, ie, 'telegraf -usage mysql'
  -usage-list	     print all the plugins input
  -debug             print metrics as they're generated to stdout
  -quiet             run in quiet mode
  -version           print the version to stdout

Examples:

  # generate a telegraf config file:
  telegraf -sample-config > telegraf.conf

  # generate config with only cpu input & influxdb output plugins defined
  telegraf -sample-config -input-filter cpu -output-filter influxdb

  # run a single telegraf collection, outputing metrics to stdout
  telegraf -config telegraf.conf -test

  # run telegraf with all plugins defined in config file
  telegraf -config telegraf.conf

  # run telegraf, enabling the cpu & memory input, and influxdb output plugins
  telegraf -config telegraf.conf -input-filter cpu:mem -output-filter influxdb
`

func main() {
	reload := make(chan bool, 1)
	reload <- true
	for <-reload {
		reload <- false
		flag.Usage = func() { usageExit(0) }
		flag.Parse()

		if flag.NFlag() == 0 {
			usageExit(0)
		}

		var inputFilters []string
		if *fInputFiltersLegacy != "" {
			inputFilter := strings.TrimSpace(*fInputFiltersLegacy)
			inputFilters = strings.Split(":"+inputFilter+":", ":")
		}
		if *fInputFilters != "" {
			inputFilter := strings.TrimSpace(*fInputFilters)
			inputFilters = strings.Split(":"+inputFilter+":", ":")
		}

		var outputFilters []string
		if *fOutputFiltersLegacy != "" {
			outputFilter := strings.TrimSpace(*fOutputFiltersLegacy)
			outputFilters = strings.Split(":"+outputFilter+":", ":")
		}
		if *fOutputFilters != "" {
			outputFilter := strings.TrimSpace(*fOutputFilters)
			outputFilters = strings.Split(":"+outputFilter+":", ":")
		}

		if *fVersion {
			v := fmt.Sprintf("Telegraf - Version %s", Version)
			fmt.Println(v)
			return
		}

		if *fSampleConfig {
			config.PrintSampleConfig(inputFilters, outputFilters)
			return
		}

		if *fUsage != "" {
			if err := config.PrintInputConfig(*fUsage); err != nil {
				if err2 := config.PrintOutputConfig(*fUsage); err2 != nil {
					log.Fatalf("%s and %s", err, err2)
				}
			}
			return
		}

		if *fUsageList {
			fmt.Println("The plugin inputs avaiable:")
			for k, _ := range inputs.Inputs {
				fmt.Printf("  %s\n", k)
			}
		}

		var (
			c   *config.Config
			err error
		)

		if *fConfig != "" {
			c = config.NewConfig()
			c.OutputFilters = outputFilters
			c.InputFilters = inputFilters
			err = c.LoadConfig(*fConfig)
			if err != nil {
				log.Fatal(err)
			}
		} else {
			fmt.Println("You must specify a config file. See telegraf --help")
			os.Exit(1)
		}

		if *fConfigDirectoryLegacy != "" {
			err = c.LoadDirectory(*fConfigDirectoryLegacy)
			if err != nil {
				log.Fatal(err)
			}
		}

		if *fConfigDirectory != "" {
			err = c.LoadDirectory(*fConfigDirectory)
			if err != nil {
				log.Fatal(err)
			}
		}
		if len(c.Outputs) == 0 {
			log.Fatalf("Error: no outputs found, did you provide a valid config file?")
		}
		if len(c.Inputs) == 0 {
			log.Fatalf("Error: no inputs found, did you provide a valid config file?")
		}

		ag, err := agent.NewAgent(c)
		if err != nil {
			log.Fatal(err)
		}

		if *fDebug {
			ag.Config.Agent.Debug = true
		}

		if *fQuiet {
			ag.Config.Agent.Quiet = true
		}

		if *fTest {
			err = ag.Test()
			if err != nil {
				log.Fatal(err)
			}
			return
		}

		err = ag.Connect()
		if err != nil {
			log.Fatal(err)
		}

		shutdown := make(chan struct{})
		signals := make(chan os.Signal)
		signal.Notify(signals, os.Interrupt, syscall.SIGHUP)
		go func() {
			sig := <-signals
			if sig == os.Interrupt {
				close(shutdown)
			}
			if sig == syscall.SIGHUP {
				log.Printf("Reloading Telegraf config\n")
				<-reload
				reload <- true
				close(shutdown)
			}
		}()

		log.Printf("Starting Telegraf (version %s)\n", Version)
		log.Printf("Loaded outputs: %s", strings.Join(c.OutputNames(), " "))
		log.Printf("Loaded inputs: %s", strings.Join(c.InputNames(), " "))
		log.Printf("Tags enabled: %s", c.ListTags())

		if *fPidfile != "" {
			f, err := os.Create(*fPidfile)
			if err != nil {
				log.Fatalf("Unable to create pidfile: %s", err)
			}

			fmt.Fprintf(f, "%d\n", os.Getpid())

			f.Close()
		}

		ag.Run(shutdown)
	}
}

func usageExit(rc int) {
	fmt.Println(usage)
	os.Exit(rc)
}
<|endoftext|>


<commit_before>package filesystem

import (
	"io"
	"os"

	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/plumbing/cache"
	"gopkg.in/src-d/go-git.v4/plumbing/format/idxfile"
	"gopkg.in/src-d/go-git.v4/plumbing/format/objfile"
	"gopkg.in/src-d/go-git.v4/plumbing/format/packfile"
	"gopkg.in/src-d/go-git.v4/plumbing/storer"
	"gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit"
	"gopkg.in/src-d/go-git.v4/storage/memory"
	"gopkg.in/src-d/go-git.v4/utils/ioutil"

	"gopkg.in/src-d/go-billy.v3"
)

const DefaultMaxDeltaBaseCacheSize = 92 * cache.MiByte

type ObjectStorage struct {
	// DeltaBaseCache is an object cache uses to cache delta's bases when
	DeltaBaseCache cache.Object

	dir   *dotgit.DotGit
	index map[plumbing.Hash]*packfile.Index
}

func newObjectStorage(dir *dotgit.DotGit) (ObjectStorage, error) {
	s := ObjectStorage{
		DeltaBaseCache: cache.NewObjectLRU(DefaultMaxDeltaBaseCacheSize),
		dir:            dir,
	}

	return s, nil
}

func (s *ObjectStorage) requireIndex() error {
	if s.index != nil {
		return nil
	}

	s.index = make(map[plumbing.Hash]*packfile.Index, 0)
	packs, err := s.dir.ObjectPacks()
	if err != nil {
		return err
	}

	for _, h := range packs {
		if err := s.loadIdxFile(h); err != nil {
			return err
		}
	}

	return nil
}

func (s *ObjectStorage) loadIdxFile(h plumbing.Hash) error {
	f, err := s.dir.ObjectPackIdx(h)
	if err != nil {
		return err
	}

	defer ioutil.CheckClose(f, &err)
	idxf := idxfile.NewIdxfile()
	d := idxfile.NewDecoder(f)
	if err = d.Decode(idxf); err != nil {
		return err
	}

	s.index[h] = packfile.NewIndexFromIdxFile(idxf)
	return err
}

func (s *ObjectStorage) NewEncodedObject() plumbing.EncodedObject {
	return &plumbing.MemoryObject{}
}

func (s *ObjectStorage) PackfileWriter() (io.WriteCloser, error) {
	if err := s.requireIndex(); err != nil {
		return nil, err
	}

	w, err := s.dir.NewObjectPack()
	if err != nil {
		return nil, err
	}

	w.Notify = func(h plumbing.Hash, idx *packfile.Index) {
		s.index[h] = idx
	}

	return w, nil
}

// SetEncodedObject adds a new object to the storage.
func (s *ObjectStorage) SetEncodedObject(o plumbing.EncodedObject) (plumbing.Hash, error) {
	if o.Type() == plumbing.OFSDeltaObject || o.Type() == plumbing.REFDeltaObject {
		return plumbing.ZeroHash, plumbing.ErrInvalidType
	}

	ow, err := s.dir.NewObject()
	if err != nil {
		return plumbing.ZeroHash, err
	}

	defer ioutil.CheckClose(ow, &err)

	or, err := o.Reader()
	if err != nil {
		return plumbing.ZeroHash, err
	}

	defer ioutil.CheckClose(or, &err)

	if err := ow.WriteHeader(o.Type(), o.Size()); err != nil {
		return plumbing.ZeroHash, err
	}

	if _, err := io.Copy(ow, or); err != nil {
		return plumbing.ZeroHash, err
	}

	return o.Hash(), err
}

// EncodedObject returns the object with the given hash, by searching for it in
// the packfile and the git object directories.
func (s *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (plumbing.EncodedObject, error) {
	obj, err := s.getFromUnpacked(h)
	if err == plumbing.ErrObjectNotFound {
		obj, err = s.getFromPackfile(h)
	}

	if err != nil {
		return nil, err
	}

	if plumbing.AnyObject != t && obj.Type() != t {
		return nil, plumbing.ErrObjectNotFound
	}

	return obj, nil
}

func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedObject, err error) {
	f, err := s.dir.Object(h)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, plumbing.ErrObjectNotFound
		}

		return nil, err
	}

	defer ioutil.CheckClose(f, &err)

	obj = s.NewEncodedObject()
	r, err := objfile.NewReader(f)
	if err != nil {
		return nil, err
	}

	defer ioutil.CheckClose(r, &err)

	t, size, err := r.Header()
	if err != nil {
		return nil, err
	}

	obj.SetType(t)
	obj.SetSize(size)
	w, err := obj.Writer()
	if err != nil {
		return nil, err
	}

	_, err = io.Copy(w, r)
	return obj, err
}

// Get returns the object with the given hash, by searching for it in
// the packfile.
func (s *ObjectStorage) getFromPackfile(h plumbing.Hash) (plumbing.EncodedObject, error) {
	if err := s.requireIndex(); err != nil {
		return nil, err
	}

	pack, offset := s.findObjectInPackfile(h)
	if offset == -1 {
		return nil, plumbing.ErrObjectNotFound
	}

	f, err := s.dir.ObjectPack(pack)
	if err != nil {
		return nil, err
	}

	defer ioutil.CheckClose(f, &err)

	p := packfile.NewScanner(f)
	d, err := packfile.NewDecoder(p, memory.NewStorage())
	if err != nil {
		return nil, err
	}

	d.DeltaBaseCache = s.DeltaBaseCache
	d.SetIndex(s.index[pack])
	obj, err := d.DecodeObjectAt(offset)
	return obj, err
}

func (s *ObjectStorage) findObjectInPackfile(h plumbing.Hash) (plumbing.Hash, int64) {
	for packfile, index := range s.index {
		if e, ok := index.LookupHash(h); ok {
			return packfile, int64(e.Offset)
		}
	}

	return plumbing.ZeroHash, -1
}

// IterEncodedObjects returns an iterator for all the objects in the packfile
// with the given type.
func (s *ObjectStorage) IterEncodedObjects(t plumbing.ObjectType) (storer.EncodedObjectIter, error) {
	objects, err := s.dir.Objects()
	if err != nil {
		return nil, err
	}

	seen := make(map[plumbing.Hash]bool, 0)
	var iters []storer.EncodedObjectIter
	if len(objects) != 0 {
		iters = append(iters, &objectsIter{s: s, t: t, h: objects})
		seen = hashListAsMap(objects)
	}

	packi, err := s.buildPackfileIters(t, seen)
	if err != nil {
		return nil, err
	}

	iters = append(iters, packi...)
	return storer.NewMultiEncodedObjectIter(iters), nil
}

func (s *ObjectStorage) buildPackfileIters(t plumbing.ObjectType, seen map[plumbing.Hash]bool) ([]storer.EncodedObjectIter, error) {
	if err := s.requireIndex(); err != nil {
		return nil, err
	}

	packs, err := s.dir.ObjectPacks()
	if err != nil {
		return nil, err
	}

	var iters []storer.EncodedObjectIter
	for _, h := range packs {
		pack, err := s.dir.ObjectPack(h)
		if err != nil {
			return nil, err
		}

		iter, err := newPackfileIter(pack, t, seen, s.index[h])
		if err != nil {
			return nil, err
		}

		iters = append(iters, iter)
	}

	return iters, nil
}

type packfileIter struct {
	f billy.File
	d *packfile.Decoder
	t plumbing.ObjectType

	seen     map[plumbing.Hash]bool
	position uint32
	total    uint32
}

func NewPackfileIter(f billy.File, t plumbing.ObjectType) (storer.EncodedObjectIter, error) {
	return newPackfileIter(f, t, make(map[plumbing.Hash]bool), nil)
}

func newPackfileIter(f billy.File, t plumbing.ObjectType, seen map[plumbing.Hash]bool,
	index *packfile.Index) (storer.EncodedObjectIter, error) {
	s := packfile.NewScanner(f)
	_, total, err := s.Header()
	if err != nil {
		return nil, err
	}

	d, err := packfile.NewDecoderForType(s, memory.NewStorage(), t)
	if err != nil {
		return nil, err
	}

	d.SetIndex(index)

	return &packfileIter{
		f: f,
		d: d,
		t: t,

		total: total,
		seen:  seen,
	}, nil
}

func (iter *packfileIter) Next() (plumbing.EncodedObject, error) {
	for {
		if iter.position >= iter.total {
			return nil, io.EOF
		}

		obj, err := iter.d.DecodeObject()
		if err != nil {
			return nil, err
		}

		iter.position++
		if obj == nil {
			continue
		}

		if iter.seen[obj.Hash()] {
			return iter.Next()
		}

		return obj, nil
	}
}

// ForEach is never called since is used inside of a MultiObjectIterator
func (iter *packfileIter) ForEach(cb func(plumbing.EncodedObject) error) error {
	return nil
}

func (iter *packfileIter) Close() {
	iter.f.Close()
	iter.d.Close()
}

type objectsIter struct {
	s *ObjectStorage
	t plumbing.ObjectType
	h []plumbing.Hash
}

func (iter *objectsIter) Next() (plumbing.EncodedObject, error) {
	if len(iter.h) == 0 {
		return nil, io.EOF
	}

	obj, err := iter.s.getFromUnpacked(iter.h[0])
	iter.h = iter.h[1:]

	if err != nil {
		return nil, err
	}

	if iter.t != plumbing.AnyObject && iter.t != obj.Type() {
		return iter.Next()
	}

	return obj, err
}

// ForEach is never called since is used inside of a MultiObjectIterator
func (iter *objectsIter) ForEach(cb func(plumbing.EncodedObject) error) error {
	return nil
}

func (iter *objectsIter) Close() {
	iter.h = []plumbing.Hash{}
}

func hashListAsMap(l []plumbing.Hash) map[plumbing.Hash]bool {
	m := make(map[plumbing.Hash]bool, len(l))
	for _, h := range l {
		m[h] = true
	}

	return m
}
<commit_msg>filesystem: reuse cache for packfile iterator<commit_after>package filesystem

import (
	"io"
	"os"

	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/plumbing/cache"
	"gopkg.in/src-d/go-git.v4/plumbing/format/idxfile"
	"gopkg.in/src-d/go-git.v4/plumbing/format/objfile"
	"gopkg.in/src-d/go-git.v4/plumbing/format/packfile"
	"gopkg.in/src-d/go-git.v4/plumbing/storer"
	"gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit"
	"gopkg.in/src-d/go-git.v4/storage/memory"
	"gopkg.in/src-d/go-git.v4/utils/ioutil"

	"gopkg.in/src-d/go-billy.v3"
)

const DefaultMaxDeltaBaseCacheSize = 92 * cache.MiByte

type ObjectStorage struct {
	// DeltaBaseCache is an object cache uses to cache delta's bases when
	DeltaBaseCache cache.Object

	dir   *dotgit.DotGit
	index map[plumbing.Hash]*packfile.Index
}

func newObjectStorage(dir *dotgit.DotGit) (ObjectStorage, error) {
	s := ObjectStorage{
		DeltaBaseCache: cache.NewObjectLRU(DefaultMaxDeltaBaseCacheSize),
		dir:            dir,
	}

	return s, nil
}

func (s *ObjectStorage) requireIndex() error {
	if s.index != nil {
		return nil
	}

	s.index = make(map[plumbing.Hash]*packfile.Index, 0)
	packs, err := s.dir.ObjectPacks()
	if err != nil {
		return err
	}

	for _, h := range packs {
		if err := s.loadIdxFile(h); err != nil {
			return err
		}
	}

	return nil
}

func (s *ObjectStorage) loadIdxFile(h plumbing.Hash) error {
	f, err := s.dir.ObjectPackIdx(h)
	if err != nil {
		return err
	}

	defer ioutil.CheckClose(f, &err)
	idxf := idxfile.NewIdxfile()
	d := idxfile.NewDecoder(f)
	if err = d.Decode(idxf); err != nil {
		return err
	}

	s.index[h] = packfile.NewIndexFromIdxFile(idxf)
	return err
}

func (s *ObjectStorage) NewEncodedObject() plumbing.EncodedObject {
	return &plumbing.MemoryObject{}
}

func (s *ObjectStorage) PackfileWriter() (io.WriteCloser, error) {
	if err := s.requireIndex(); err != nil {
		return nil, err
	}

	w, err := s.dir.NewObjectPack()
	if err != nil {
		return nil, err
	}

	w.Notify = func(h plumbing.Hash, idx *packfile.Index) {
		s.index[h] = idx
	}

	return w, nil
}

// SetEncodedObject adds a new object to the storage.
func (s *ObjectStorage) SetEncodedObject(o plumbing.EncodedObject) (plumbing.Hash, error) {
	if o.Type() == plumbing.OFSDeltaObject || o.Type() == plumbing.REFDeltaObject {
		return plumbing.ZeroHash, plumbing.ErrInvalidType
	}

	ow, err := s.dir.NewObject()
	if err != nil {
		return plumbing.ZeroHash, err
	}

	defer ioutil.CheckClose(ow, &err)

	or, err := o.Reader()
	if err != nil {
		return plumbing.ZeroHash, err
	}

	defer ioutil.CheckClose(or, &err)

	if err := ow.WriteHeader(o.Type(), o.Size()); err != nil {
		return plumbing.ZeroHash, err
	}

	if _, err := io.Copy(ow, or); err != nil {
		return plumbing.ZeroHash, err
	}

	return o.Hash(), err
}

// EncodedObject returns the object with the given hash, by searching for it in
// the packfile and the git object directories.
func (s *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (plumbing.EncodedObject, error) {
	obj, err := s.getFromUnpacked(h)
	if err == plumbing.ErrObjectNotFound {
		obj, err = s.getFromPackfile(h)
	}

	if err != nil {
		return nil, err
	}

	if plumbing.AnyObject != t && obj.Type() != t {
		return nil, plumbing.ErrObjectNotFound
	}

	return obj, nil
}

func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedObject, err error) {
	f, err := s.dir.Object(h)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, plumbing.ErrObjectNotFound
		}

		return nil, err
	}

	defer ioutil.CheckClose(f, &err)

	obj = s.NewEncodedObject()
	r, err := objfile.NewReader(f)
	if err != nil {
		return nil, err
	}

	defer ioutil.CheckClose(r, &err)

	t, size, err := r.Header()
	if err != nil {
		return nil, err
	}

	obj.SetType(t)
	obj.SetSize(size)
	w, err := obj.Writer()
	if err != nil {
		return nil, err
	}

	_, err = io.Copy(w, r)
	return obj, err
}

// Get returns the object with the given hash, by searching for it in
// the packfile.
func (s *ObjectStorage) getFromPackfile(h plumbing.Hash) (plumbing.EncodedObject, error) {
	if err := s.requireIndex(); err != nil {
		return nil, err
	}

	pack, offset := s.findObjectInPackfile(h)
	if offset == -1 {
		return nil, plumbing.ErrObjectNotFound
	}

	f, err := s.dir.ObjectPack(pack)
	if err != nil {
		return nil, err
	}

	defer ioutil.CheckClose(f, &err)

	p := packfile.NewScanner(f)
	d, err := packfile.NewDecoder(p, memory.NewStorage())
	if err != nil {
		return nil, err
	}

	d.DeltaBaseCache = s.DeltaBaseCache
	d.SetIndex(s.index[pack])
	obj, err := d.DecodeObjectAt(offset)
	return obj, err
}

func (s *ObjectStorage) findObjectInPackfile(h plumbing.Hash) (plumbing.Hash, int64) {
	for packfile, index := range s.index {
		if e, ok := index.LookupHash(h); ok {
			return packfile, int64(e.Offset)
		}
	}

	return plumbing.ZeroHash, -1
}

// IterEncodedObjects returns an iterator for all the objects in the packfile
// with the given type.
func (s *ObjectStorage) IterEncodedObjects(t plumbing.ObjectType) (storer.EncodedObjectIter, error) {
	objects, err := s.dir.Objects()
	if err != nil {
		return nil, err
	}

	seen := make(map[plumbing.Hash]bool, 0)
	var iters []storer.EncodedObjectIter
	if len(objects) != 0 {
		iters = append(iters, &objectsIter{s: s, t: t, h: objects})
		seen = hashListAsMap(objects)
	}

	packi, err := s.buildPackfileIters(t, seen)
	if err != nil {
		return nil, err
	}

	iters = append(iters, packi...)
	return storer.NewMultiEncodedObjectIter(iters), nil
}

func (s *ObjectStorage) buildPackfileIters(t plumbing.ObjectType, seen map[plumbing.Hash]bool) ([]storer.EncodedObjectIter, error) {
	if err := s.requireIndex(); err != nil {
		return nil, err
	}

	packs, err := s.dir.ObjectPacks()
	if err != nil {
		return nil, err
	}

	var iters []storer.EncodedObjectIter
	for _, h := range packs {
		pack, err := s.dir.ObjectPack(h)
		if err != nil {
			return nil, err
		}

		iter, err := newPackfileIter(pack, t, seen, s.index[h], s.DeltaBaseCache)
		if err != nil {
			return nil, err
		}

		iters = append(iters, iter)
	}

	return iters, nil
}

type packfileIter struct {
	f billy.File
	d *packfile.Decoder
	t plumbing.ObjectType

	seen     map[plumbing.Hash]bool
	position uint32
	total    uint32
}

func NewPackfileIter(f billy.File, t plumbing.ObjectType) (storer.EncodedObjectIter, error) {
	return newPackfileIter(f, t, make(map[plumbing.Hash]bool), nil, nil)
}

func newPackfileIter(f billy.File, t plumbing.ObjectType, seen map[plumbing.Hash]bool,
	index *packfile.Index, cache cache.Object) (storer.EncodedObjectIter, error) {
	s := packfile.NewScanner(f)
	_, total, err := s.Header()
	if err != nil {
		return nil, err
	}

	d, err := packfile.NewDecoderForType(s, memory.NewStorage(), t)
	if err != nil {
		return nil, err
	}

	d.SetIndex(index)
	d.DeltaBaseCache = cache

	return &packfileIter{
		f: f,
		d: d,
		t: t,

		total: total,
		seen:  seen,
	}, nil
}

func (iter *packfileIter) Next() (plumbing.EncodedObject, error) {
	for {
		if iter.position >= iter.total {
			return nil, io.EOF
		}

		obj, err := iter.d.DecodeObject()
		if err != nil {
			return nil, err
		}

		iter.position++
		if obj == nil {
			continue
		}

		if iter.seen[obj.Hash()] {
			return iter.Next()
		}

		return obj, nil
	}
}

// ForEach is never called since is used inside of a MultiObjectIterator
func (iter *packfileIter) ForEach(cb func(plumbing.EncodedObject) error) error {
	return nil
}

func (iter *packfileIter) Close() {
	iter.f.Close()
	iter.d.Close()
}

type objectsIter struct {
	s *ObjectStorage
	t plumbing.ObjectType
	h []plumbing.Hash
}

func (iter *objectsIter) Next() (plumbing.EncodedObject, error) {
	if len(iter.h) == 0 {
		return nil, io.EOF
	}

	obj, err := iter.s.getFromUnpacked(iter.h[0])
	iter.h = iter.h[1:]

	if err != nil {
		return nil, err
	}

	if iter.t != plumbing.AnyObject && iter.t != obj.Type() {
		return iter.Next()
	}

	return obj, err
}

// ForEach is never called since is used inside of a MultiObjectIterator
func (iter *objectsIter) ForEach(cb func(plumbing.EncodedObject) error) error {
	return nil
}

func (iter *objectsIter) Close() {
	iter.h = []plumbing.Hash{}
}

func hashListAsMap(l []plumbing.Hash) map[plumbing.Hash]bool {
	m := make(map[plumbing.Hash]bool, len(l))
	for _, h := range l {
		m[h] = true
	}

	return m
}
<|endoftext|>


<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build freebsd dragonfly
// +build !nomeminfo

package collector

import (
	"fmt"

	"golang.org/x/sys/unix"
)

func (c *meminfoCollector) getMemInfo() (map[string]float64, error) {
	info := make(map[string]float64)

	size, err := unix.SysctlUint32("vm.stats.vm.v_page_size")
	if err != nil {
		return nil, fmt.Errorf("sysctl(vm.stats.vm.v_page_size) failed: %s", err)
	}

	for key, v := range map[string]string{
		"active":     "vm.stats.vm.v_active_count",
		"inactive":   "vm.stats.vm.v_inactive_count",
		"wire":       "vm.stats.vm.v_wire_count",
		"cache":      "vm.stats.vm.v_cache_count",
		"free":       "vm.stats.vm.v_free_count",
		"swappgsin":  "vm.stats.vm.v_swappgsin",
		"swappgsout": "vm.stats.vm.v_swappgsout",
		"total":      "vm.stats.vm.v_page_count",
	} {
		value, err := unix.SysctlUint32(v)
		if err != nil {
			return nil, err
		}
		// Convert metrics to kB (same as Linux meminfo).
		info[key] = float64(value) * float64(size)
	}
	return info, nil
}
<commit_msg>Added node_memory_buffer, node_memory_swaptotal to meminfo_bsd (#451)<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build freebsd dragonfly
// +build !nomeminfo

package collector

import (
	"fmt"

	"golang.org/x/sys/unix"
)

type sysctlType uint8

const (
	sysctlTypeUint32 sysctlType = iota
	sysctlTypeUint64
)

type meminfoSysctl struct {
	name       string
	dataType   sysctlType
	conversion func(uint64) uint64
}

func (c *meminfoCollector) getMemInfo() (map[string]float64, error) {
	info := make(map[string]float64)

	tmp32, err := unix.SysctlUint32("vm.stats.vm.v_page_size")
	if err != nil {
		return nil, fmt.Errorf("sysctl(vm.stats.vm.v_page_size) failed: %s", err)
	}
	size := uint64(tmp32)
	fromPage := func(v uint64) uint64 {
		return v * size
	}

	for key, v := range map[string]meminfoSysctl{
		"active_bytes":         {"vm.stats.vm.v_active_count", sysctlTypeUint32, fromPage},
		"inactive_bytes":       {"vm.stats.vm.v_inactive_count", sysctlTypeUint32, fromPage},
		"wired_bytes":          {"vm.stats.vm.v_wire_count", sysctlTypeUint32, fromPage},
		"cache_bytes":          {"vm.stats.vm.v_cache_count", sysctlTypeUint32, fromPage},
		"buffer_bytes":         {"vfs.bufspace", sysctlTypeUint32, nil},
		"free_bytes":           {"vm.stats.vm.v_free_count", sysctlTypeUint32, fromPage},
		"size_bytes":           {"vm.stats.vm.v_page_count", sysctlTypeUint32, fromPage},
		"swap_in_bytes_total":  {"vm.stats.vm.v_swappgsin", sysctlTypeUint32, fromPage},
		"swap_out_bytes_total": {"vm.stats.vm.v_swappgsout", sysctlTypeUint32, fromPage},
		"swap_size_bytes":      {"vm.swap_total", sysctlTypeUint64, nil},
	} {
		var tmp64 uint64
		switch v.dataType {
		case sysctlTypeUint32:
			tmp32, err = unix.SysctlUint32(v.name)
			tmp64 = uint64(tmp32)
		case sysctlTypeUint64:
			tmp64, err = unix.SysctlUint64(v.name)
		}
		if err != nil {
			return nil, err
		}

		if v.conversion != nil {
			// Convert to bytes.
			info[key] = float64(v.conversion(tmp64))
			continue
		}

		info[key] = float64(tmp64)
	}

	return info, nil
}
<|endoftext|>


<commit_before>package command

import (
	"os"

	"github.com/mitchellh/cli"
	"github.com/olekukonko/tablewriter"
	"github.com/pkg/errors"
)

//WorkloadList command
type WorkloadList struct {
	*command
}

//WorkloadListFactory returns a factory method for the join command
func WorkloadListFactory() (cli.Command, error) {
	comm, err := newCommand("nerd workload list", "show a list of all workloads in the current project", "", nil)
	if err != nil {
		return nil, errors.Wrap(err, "failed to create command")
	}
	cmd := &WorkloadList{
		command: comm,
	}
	cmd.runFunc = cmd.DoRun

	return cmd, nil
}

//DoRun is called by run and allows an error to be returned
func (cmd *WorkloadList) DoRun(args []string) (err error) {
	bclient, err := NewClient(cmd.config, cmd.session)
	if err != nil {
		HandleError(err)
	}

	ss, err := cmd.session.Read()
	if err != nil {
		HandleError(err)
	}

	out, err := bclient.ListWorkloads(ss.Project.Name)
	if err != nil {
		HandleError(err)
	}

	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"ProjectID", "WorkloadID"})
	for _, t := range out.Workloads {
		row := []string{}
		row = append(row, t.ProjectID)
		row = append(row, t.WorkloadID)
		table.Append(row)
	}

	table.Render()
	return nil
}
<commit_msg>List workload info<commit_after>package command

import (
	"fmt"
	"os"

	"github.com/mitchellh/cli"
	"github.com/olekukonko/tablewriter"
	"github.com/pkg/errors"
)

//WorkloadList command
type WorkloadList struct {
	*command
}

//WorkloadListFactory returns a factory method for the join command
func WorkloadListFactory() (cli.Command, error) {
	comm, err := newCommand("nerd workload list", "show a list of all workloads in the current project", "", nil)
	if err != nil {
		return nil, errors.Wrap(err, "failed to create command")
	}
	cmd := &WorkloadList{
		command: comm,
	}
	cmd.runFunc = cmd.DoRun

	return cmd, nil
}

//DoRun is called by run and allows an error to be returned
func (cmd *WorkloadList) DoRun(args []string) (err error) {
	bclient, err := NewClient(cmd.config, cmd.session)
	if err != nil {
		HandleError(err)
	}

	ss, err := cmd.session.Read()
	if err != nil {
		HandleError(err)
	}

	out, err := bclient.ListWorkloads(ss.Project.Name)
	if err != nil {
		HandleError(err)
	}

	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"ProjectID", "WorkloadID", "Image", "Created"})
	for _, t := range out.Workloads {
		row := []string{}
		row = append(row, t.ProjectID)
		row = append(row, t.WorkloadID)
		row = append(row, t.Image)
		row = append(row, fmt.Sprintf("%v", t.CreatedAt))
		table.Append(row)
	}

	table.Render()
	return nil
}
<|endoftext|>


<commit_before>package renders

import (
	"fmt"
	"html/template"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
)

var (
	cache               []*namedTemplate
	regularTemplateDefs []string
	basePath            string
	exts                []string
	lock                sync.Mutex
	re_defineTag        *regexp.Regexp = regexp.MustCompile("{{ ?define \"([^\"]*)\" ?\"?([a-zA-Z0-9]*)?\"? ?}}")
	re_templateTag      *regexp.Regexp = regexp.MustCompile("{{ ?template \"([^\"]*)\" ?([^ ]*)? ?}}")
)

type namedTemplate struct {
	Name string
	Src  string
}

// Load prepares and parses all templates from the passed basePath
func Load(opt Options) (map[string]*template.Template, error) {
	basePath = opt.Directory
	exts = opt.Extensions
	return loadTemplates(nil)
}

// LoadWithFuncMap prepares and parses all templates from the passed basePath and injects
// a custom template.FuncMap into each template
func LoadWithFuncMap(opt Options) (map[string]*template.Template, error) {
	basePath = opt.Directory
	exts = opt.Extensions
	return loadTemplates(opt.Funcs)
}

func loadTemplates(funcMap template.FuncMap) (map[string]*template.Template, error) {
	lock.Lock()
	defer lock.Unlock()

	templates := make(map[string]*template.Template)

	err := filepath.Walk(basePath, func(path string, fi os.FileInfo, err error) error {
		r, err := filepath.Rel(basePath, path)
		if err != nil {
			return err
		}

		ext := getExt(r)

		for _, extension := range exts {
			if ext == extension {
				if err := add(path); err != nil {
					panic(err)
				}

				// Now we find all regular template definitions and check for the most recent definiton
				for _, t := range regularTemplateDefs {
					found := false
					defineIdx := 0
					// From the beginning (which should) most specifc we look for definitions
					for _, nt := range cache {
						nt.Src = re_defineTag.ReplaceAllStringFunc(nt.Src, func(raw string) string {
							parsed := re_defineTag.FindStringSubmatch(raw)
							name := parsed[1]
							if name != t {
								return raw
							}
							// Don't touch the first definition
							if !found {
								found = true
								return raw
							}

							defineIdx += 1

							return fmt.Sprintf("{{ define \"%s_invalidated_#%d\" }}", name, defineIdx)
						})
					}
				}

				var (
					baseTmpl *template.Template
					i        int
				)

				for _, nt := range cache {
					var currentTmpl *template.Template
					if i == 0 {
						baseTmpl = template.New(nt.Name)
						currentTmpl = baseTmpl
					} else {
						currentTmpl = baseTmpl.New(nt.Name)
					}

					//template.Must(currentTmpl.Funcs(funcMap).Parse(nt.Src))
					currentTmpl.Funcs(funcMap)
					if _, err := currentTmpl.Parse(nt.Src); err != nil {
						return err
					}
					i++
				}
				tname := generateTemplateName(basePath, path)
				templates[tname] = baseTmpl

				// Make sure we empty the cache between runs
				cache = cache[0:0]

				break
				//return nil
			}
		}
		return nil

	})

	return templates, err
}

func add(path string) error {
	// Get file content
	tplSrc, err := file_content(path)
	if err != nil {
		return err
	}

	tplName := generateTemplateName(basePath, path)

	// Make sure template is not already included
	alreadyIncluded := false
	for _, nt := range cache {
		if nt.Name == tplName {
			alreadyIncluded = true
			break
		}
	}
	if alreadyIncluded {
		return nil
	}

	// Add to the cache
	nt := &namedTemplate{
		Name: tplName,
		Src:  tplSrc,
	}
	cache = append(cache, nt)

	// Check for any template block
	for _, raw := range re_templateTag.FindAllString(nt.Src, -1) {
		parsed := re_templateTag.FindStringSubmatch(raw)
		templatePath := parsed[1]
		ext := getExt(templatePath)
		if !strings.Contains(templatePath, ext) {
			regularTemplateDefs = append(regularTemplateDefs, templatePath)
			continue
		}

		// Add this template and continue looking for more template blocks
		add(filepath.Join(basePath, templatePath))
	}

	return nil
}
<commit_msg>revise template must bug<commit_after>package renders

import (
	"fmt"
	"html/template"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
)

var (
	cache               []*namedTemplate
	regularTemplateDefs []string
	basePath            string
	exts                []string
	lock                sync.Mutex
	re_defineTag        *regexp.Regexp = regexp.MustCompile("{{ ?define \"([^\"]*)\" ?\"?([a-zA-Z0-9]*)?\"? ?}}")
	re_templateTag      *regexp.Regexp = regexp.MustCompile("{{ ?template \"([^\"]*)\" ?([^ ]*)? ?}}")
)

type namedTemplate struct {
	Name string
	Src  string
}

// Load prepares and parses all templates from the passed basePath
func Load(opt Options) (map[string]*template.Template, error) {
	basePath = opt.Directory
	exts = opt.Extensions
	return loadTemplates(nil)
}

// LoadWithFuncMap prepares and parses all templates from the passed basePath and injects
// a custom template.FuncMap into each template
func LoadWithFuncMap(opt Options) (map[string]*template.Template, error) {
	basePath = opt.Directory
	exts = opt.Extensions
	return loadTemplates(opt.Funcs)
}

func loadTemplates(funcMap template.FuncMap) (map[string]*template.Template, error) {
	lock.Lock()
	defer lock.Unlock()

	templates := make(map[string]*template.Template)

	err := filepath.Walk(basePath, func(path string, fi os.FileInfo, err error) error {
		r, err := filepath.Rel(basePath, path)
		if err != nil {
			return err
		}

		ext := getExt(r)

		for _, extension := range exts {
			if ext == extension {
				if err := add(path); err != nil {
					panic(err)
				}

				// Now we find all regular template definitions and check for the most recent definiton
				for _, t := range regularTemplateDefs {
					found := false
					defineIdx := 0
					// From the beginning (which should) most specifc we look for definitions
					for _, nt := range cache {
						nt.Src = re_defineTag.ReplaceAllStringFunc(nt.Src, func(raw string) string {
							parsed := re_defineTag.FindStringSubmatch(raw)
							name := parsed[1]
							if name != t {
								return raw
							}
							// Don't touch the first definition
							if !found {
								found = true
								return raw
							}

							defineIdx += 1

							return fmt.Sprintf("{{ define \"%s_invalidated_#%d\" }}", name, defineIdx)
						})
					}
				}

				var (
					baseTmpl *template.Template
					i        int
				)

				for _, nt := range cache {
					var currentTmpl *template.Template
					if i == 0 {
						baseTmpl = template.New(nt.Name)
						currentTmpl = baseTmpl
					} else {
						currentTmpl = baseTmpl.New(nt.Name)
					}

					template.Must(currentTmpl.Funcs(funcMap).Parse(nt.Src))
					i++
				}
				tname := generateTemplateName(basePath, path)
				templates[tname] = baseTmpl

				// Make sure we empty the cache between runs
				cache = cache[0:0]

				break
				//return nil
			}
		}
		return nil

	})

	return templates, err
}

func add(path string) error {
	// Get file content
	tplSrc, err := file_content(path)
	if err != nil {
		return err
	}

	tplName := generateTemplateName(basePath, path)

	// Make sure template is not already included
	alreadyIncluded := false
	for _, nt := range cache {
		if nt.Name == tplName {
			alreadyIncluded = true
			break
		}
	}
	if alreadyIncluded {
		return nil
	}

	// Add to the cache
	nt := &namedTemplate{
		Name: tplName,
		Src:  tplSrc,
	}
	cache = append(cache, nt)

	// Check for any template block
	for _, raw := range re_templateTag.FindAllString(nt.Src, -1) {
		parsed := re_templateTag.FindStringSubmatch(raw)
		templatePath := parsed[1]
		ext := getExt(templatePath)
		if !strings.Contains(templatePath, ext) {
			regularTemplateDefs = append(regularTemplateDefs, templatePath)
			continue
		}

		// Add this template and continue looking for more template blocks
		add(filepath.Join(basePath, templatePath))
	}

	return nil
}
<|endoftext|>


<commit_before>package restful

// Copyright 2013 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"io/ioutil"
	"net/http"
	"strings"
)

// Request is a wrapper for a http Request that provides convenience methods
type Request struct {
	Request        *http.Request
	pathParameters map[string]string
	attributes     map[string]interface{} // for storing request-scoped values
}

func newRequest(httpRequest *http.Request) *Request {
	return &Request{httpRequest, map[string]string{}, map[string]interface{}{}} // empty parameters, attributes
}

// PathParameter accesses the Path parameter value by its name
func (r *Request) PathParameter(name string) string {
	return r.pathParameters[name]
}

// PathParameters accesses the Path parameter values
func (r *Request) PathParameters() map[string]string {
	return r.pathParameters
}

// QueryParameter returns the (first) Query parameter value by its name
func (r *Request) QueryParameter(name string) string {
	return r.Request.FormValue(name)
}

func (r *Request) BodyParameter(name string) (string, error) {
	attr := r.Attribute("x-restful-parse-form")
	if attr == nil || (attr != nil && !attr.(bool)) {
		err := r.Request.ParseForm()
		if err != nil {
			return "", err
		}
		r.SetAttribute("x-restful-parse-form", true)
	}
	return r.Request.PostFormValue(name), nil
}

// HeaderParameter returns the HTTP Header value of a Header name or empty if missing
func (r *Request) HeaderParameter(name string) string {
	return r.Request.Header.Get(name)
}

// ReadEntity checks the Accept header and reads the content into the entityPointer
func (r *Request) ReadEntity(entityPointer interface{}) error {
	contentType := r.Request.Header.Get(HEADER_ContentType)
	buffer, err := ioutil.ReadAll(r.Request.Body)
	if err != nil {
		return err
	}
	if strings.Contains(contentType, MIME_XML) {
		err = xml.Unmarshal(buffer, entityPointer)
	} else {
		if strings.Contains(contentType, MIME_JSON) {
			err = json.Unmarshal(buffer, entityPointer)
		} else {
			err = errors.New("[restful] Unable to unmarshal content of type:" + contentType)
		}
	}
	return err
}

// SetAttribute adds or replaces the attribute with the given value.
func (r *Request) SetAttribute(name string, value interface{}) {
	r.attributes[name] = value
}

// Attribute returns the value associated to the given name. Returns nil if absent.
func (r Request) Attribute(name string) interface{} {
	return r.attributes[name]
}
<commit_msg>Request.ParseForm is idempotent, so to check if it's called is unnecessary.<commit_after>package restful

// Copyright 2013 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"io/ioutil"
	"net/http"
	"strings"
)

// Request is a wrapper for a http Request that provides convenience methods
type Request struct {
	Request        *http.Request
	pathParameters map[string]string
	attributes     map[string]interface{} // for storing request-scoped values
}

func newRequest(httpRequest *http.Request) *Request {
	return &Request{httpRequest, map[string]string{}, map[string]interface{}{}} // empty parameters, attributes
}

// PathParameter accesses the Path parameter value by its name
func (r *Request) PathParameter(name string) string {
	return r.pathParameters[name]
}

// PathParameters accesses the Path parameter values
func (r *Request) PathParameters() map[string]string {
	return r.pathParameters
}

// QueryParameter returns the (first) Query parameter value by its name
func (r *Request) QueryParameter(name string) string {
	return r.Request.FormValue(name)
}

func (r *Request) BodyParameter(name string) (string, error) {
	err := r.Request.ParseForm()
	if err != nil {
		return "", err
	}
	return r.Request.PostFormValue(name), nil
}

// HeaderParameter returns the HTTP Header value of a Header name or empty if missing
func (r *Request) HeaderParameter(name string) string {
	return r.Request.Header.Get(name)
}

// ReadEntity checks the Accept header and reads the content into the entityPointer
func (r *Request) ReadEntity(entityPointer interface{}) error {
	contentType := r.Request.Header.Get(HEADER_ContentType)
	buffer, err := ioutil.ReadAll(r.Request.Body)
	if err != nil {
		return err
	}
	if strings.Contains(contentType, MIME_XML) {
		err = xml.Unmarshal(buffer, entityPointer)
	} else {
		if strings.Contains(contentType, MIME_JSON) {
			err = json.Unmarshal(buffer, entityPointer)
		} else {
			err = errors.New("[restful] Unable to unmarshal content of type:" + contentType)
		}
	}
	return err
}

// SetAttribute adds or replaces the attribute with the given value.
func (r *Request) SetAttribute(name string, value interface{}) {
	r.attributes[name] = value
}

// Attribute returns the value associated to the given name. Returns nil if absent.
func (r Request) Attribute(name string) interface{} {
	return r.attributes[name]
}
<|endoftext|>


<commit_before>package f

import(
    "strings"
    "github.com/ricallinson/stackr"
)

/*
    A Request represents an HTTP request received by the server.
*/
type Request struct {

    // The stackr.Request type.
    *stackr.Request

    // The application server.
    app *Server

    // This property is a slice containing properties mapped to the named route "parameters".
    // For example if you have the route "/user/:name", then the "name" property is available
    // to you as req.params["name"]. This object defaults to {}.
    Params map[string]string

    // The currently matched Route containing several properties such as the
    // route's original path string, the regexp generated, and so on.
    Route interface{}
}

/*
    Returns a new Request.
*/

func createRequest(req *stackr.Request, app *Server) (*Request) {

    this := &Request{}

    this.Request = req

    this.app = app

    this.Params = map[string]string{}

    // Could have been set by middleware.
    if this.Body == nil {
        this.Body = map[string]string{}
    }

    // Could have been set by middleware.
    if this.Query == nil {
        this.Query = map[string]string{}
    }

    // Could have been set by middleware.
    if this.Files == nil {
        this.Files = map[string]interface{}{}
    }

    return this
}

/*
    Return the value of param "name" when present. Lookup is performed in the following order:

    * Params
    * Body
    * Query

    Direct access to req.body, req.params, and req.query should be favoured for clarity -
    unless you truly accept input from each object.
*/
func (this *Request) Param(n string) (string) {
    var v string
    var ok bool
    v, ok = this.Params[n]
    if ok {
        return v
    }
    v, ok = this.Body[n]
    if ok {
        return v
    }
    v, ok = this.Query[n]
    if ok {
        return v
    }
    return ""
}

/*
    Get the case-insensitive request header field. The Referrer and Referer fields are interchangeable.
*/
func (this *Request) Get(f string) (string) {

    /*
        Possible future bug.
        http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
        Message headers are case-insensitive.
    */

    return this.Header.Get(f)
}

/*
    Check if the given types are acceptable, returning the best match when true,
    otherwise undefined - in which case you should respond with 406 "Not Acceptable".
*/
func (this *Request) Accepts(t string) (bool) {
    for _, v := range this.Accepted {
        if strings.ToLower(t) == v {
            return true
        }
    }
    return false
}

/*
    Check if the incoming request contains the "Content-Type" header field, and it matches the give mime "type".
*/
func (this *Request) Is(t string) (bool) {
    h := this.Get("Content-Type")
    return strings.ToLower(h) == strings.ToLower(t)
}

/*
    Check if the given "charset" is acceptable.
*/
func (this *Request) AcceptsCharset(c string) {
    panic(halt)
}

/*
    Check if the given "lang" is acceptable.
*/
func (this *Request) AcceptsLanguage(l string) {
    panic(halt)
}<commit_msg>Added cookie stuff again<commit_after>package f

import(
    "strings"
    "github.com/ricallinson/stackr"
)

/*
    A Request represents an HTTP request received by the server.
*/
type Request struct {

    // The stackr.Request type.
    *stackr.Request

    // The application server.
    app *Server

    // This property is a slice containing properties mapped to the named route "parameters".
    // For example if you have the route "/user/:name", then the "name" property is available
    // to you as req.params["name"]. This object defaults to {}.
    Params map[string]string

    // The currently matched Route containing several properties such as the
    // route's original path string, the regexp generated, and so on.
    Route interface{}
}

/*
    Returns a new Request.
*/

func createRequest(req *stackr.Request, app *Server) (*Request) {

    this := &Request{}

    this.Request = req

    this.app = app

    this.Params = map[string]string{}

    // Could have been set by middleware.
    if this.Body == nil {
        this.Body = map[string]string{}
    }

    // Could have been set by middleware.
    if this.Query == nil {
        this.Query = map[string]string{}
    }

    // Could have been set by middleware.
    if this.Files == nil {
        this.Files = map[string]interface{}{}
    }

    return this
}

/*
    Contains the cookies sent by the user-agent.
*/
func (this *Request) Cookie(n string, i *interface{}) {
    //
}

/*
    Contains the signed cookies sent by the user-agent, unsigned and ready for use.
    Signed cookies are accessed by a different function to show developer intent, otherwise a
    malicious attack could be placed on `req.Cookie` values which are easy to spoof.
    Note that signing a cookie does not mean it is "hidden" nor encrypted, this simply
    prevents tampering as the secret used to sign is private.
*/
func (this *Request) SignedCookie(n string, i *interface{}) {
    //
}

/*
    Return the value of param "name" when present. Lookup is performed in the following order:

    * Params
    * Body
    * Query

    Direct access to req.body, req.params, and req.query should be favoured for clarity -
    unless you truly accept input from each object.
*/
func (this *Request) Param(n string) (string) {
    var v string
    var ok bool
    v, ok = this.Params[n]
    if ok {
        return v
    }
    v, ok = this.Body[n]
    if ok {
        return v
    }
    v, ok = this.Query[n]
    if ok {
        return v
    }
    return ""
}

/*
    Get the case-insensitive request header field. The Referrer and Referer fields are interchangeable.
*/
func (this *Request) Get(f string) (string) {

    /*
        Possible future bug.
        http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
        Message headers are case-insensitive.
    */

    return this.Header.Get(f)
}

/*
    Check if the given types are acceptable, returning the best match when true,
    otherwise undefined - in which case you should respond with 406 "Not Acceptable".
*/
func (this *Request) Accepts(t string) (bool) {
    for _, v := range this.Accepted {
        if strings.ToLower(t) == v {
            return true
        }
    }
    return false
}

/*
    Check if the incoming request contains the "Content-Type" header field, and it matches the give mime "type".
*/
func (this *Request) Is(t string) (bool) {
    h := this.Get("Content-Type")
    return strings.ToLower(h) == strings.ToLower(t)
}

/*
    Check if the given "charset" is acceptable.
*/
func (this *Request) AcceptsCharset(c string) {
    panic(halt)
}

/*
    Check if the given "lang" is acceptable.
*/
func (this *Request) AcceptsLanguage(l string) {
    panic(halt)
}<|endoftext|>


<commit_before>package cachetree

import (
	"bytes"
	"encoding/binary"
	"io"
	"net"
)

func AskFileFromMembers(filename string) (data []byte) {
	conns := clientConnections[:]
	var err error
	for host, conn := range conns {
		err = requestFile(conn, filename)
		if err != nil {
			closeConn(host)
			go HandleMemberConnectionFail(err, host)
			continue
		}

		data, err = readFileIfExists(conn, filename)
		if err != nil {
			closeConn(host)
			go HandleMemberConnectionFail(err, host)
			continue
		}

		if data != nil {
			break
		}
	}

	return data
}

func DeleteFileFromMembers(filename string) {
	conns := clientConnections[:]
	for host, conn := range conns {
		err := requestFileDelete(conn, filename)
		if err != nil {
			closeConn(host)
			go HandleMemberConnectionFail(err, host)
			continue
		}

		file_name_data, err := readData(conn)
		if err != nil ||
			// Got wrong API
			string(file_name_data) != filename {
			closeConn(host)
			go HandleMemberConnectionFail(err, host)
			continue
		}
	}
}

func requestFile(conn *net.TCPConn, filename string) error {
	filename_len_bytes := make([]byte, 4)
	filename_bytes := []byte(filename)
	binary.BigEndian.PutUint32(filename_len_bytes, uint32(len(filename_bytes)))

	send_bytes := bytes.NewBuffer([]byte{})
	send_bytes.Write([]byte{CMD_REQUEST_FILE})
	send_bytes.Write(filename_len_bytes)
	send_bytes.Write(filename_bytes)

	_, err := io.Copy(conn, send_bytes)
	return err
}

func requestFileDelete(conn *net.TCPConn, filename string) error {
	filename_len_bytes := make([]byte, 4)
	filename_bytes := []byte(filename)
	binary.BigEndian.PutUint32(filename_len_bytes, uint32(len(filename_bytes)))

	send_bytes := bytes.NewBuffer([]byte{})
	send_bytes.Write([]byte{CMD_DELETE_FILE})
	send_bytes.Write(filename_len_bytes)
	send_bytes.Write(filename_bytes)

	_, err := io.Copy(conn, send_bytes)
	return err
}

func readFileIfExists(conn *net.TCPConn, filename string) ([]byte, error) {
	data, err := readData(conn)
	if err != nil {
		return nil, err
	}

	// if data length is 1, then this means connection don't have requested cache file
	if len(data) == 1 {
		return nil, nil
	}

	return data, nil
}
<commit_msg>fixed map copy bug<commit_after>package cachetree

import (
	"bytes"
	"encoding/binary"
	"io"
	"net"
)

func AskFileFromMembers(filename string) (data []byte) {
	conns := make(map[string]*net.TCPConn)
	for k, v :=range clientConnections {
		conns[k] = v
	}

	var err error
	for host, conn := range conns {
		err = requestFile(conn, filename)
		if err != nil {
			closeConn(host)
			go HandleMemberConnectionFail(err, host)
			continue
		}

		data, err = readFileIfExists(conn, filename)
		if err != nil {
			closeConn(host)
			go HandleMemberConnectionFail(err, host)
			continue
		}

		if data != nil {
			break
		}
	}

	return data
}

func DeleteFileFromMembers(filename string) {
	conns := make(map[string]*net.TCPConn)
	for k, v :=range clientConnections {
		conns[k] = v
	}

	for host, conn := range conns {
		err := requestFileDelete(conn, filename)
		if err != nil {
			closeConn(host)
			go HandleMemberConnectionFail(err, host)
			continue
		}

		file_name_data, err := readData(conn)
		if err != nil ||
			// Got wrong API
			string(file_name_data) != filename {
			closeConn(host)
			go HandleMemberConnectionFail(err, host)
			continue
		}
	}
}

func requestFile(conn *net.TCPConn, filename string) error {
	filename_len_bytes := make([]byte, 4)
	filename_bytes := []byte(filename)
	binary.BigEndian.PutUint32(filename_len_bytes, uint32(len(filename_bytes)))

	send_bytes := bytes.NewBuffer([]byte{})
	send_bytes.Write([]byte{CMD_REQUEST_FILE})
	send_bytes.Write(filename_len_bytes)
	send_bytes.Write(filename_bytes)

	_, err := io.Copy(conn, send_bytes)
	return err
}

func requestFileDelete(conn *net.TCPConn, filename string) error {
	filename_len_bytes := make([]byte, 4)
	filename_bytes := []byte(filename)
	binary.BigEndian.PutUint32(filename_len_bytes, uint32(len(filename_bytes)))

	send_bytes := bytes.NewBuffer([]byte{})
	send_bytes.Write([]byte{CMD_DELETE_FILE})
	send_bytes.Write(filename_len_bytes)
	send_bytes.Write(filename_bytes)

	_, err := io.Copy(conn, send_bytes)
	return err
}

func readFileIfExists(conn *net.TCPConn, filename string) ([]byte, error) {
	data, err := readData(conn)
	if err != nil {
		return nil, err
	}

	// if data length is 1, then this means connection don't have requested cache file
	if len(data) == 1 {
		return nil, nil
	}

	return data, nil
}
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package operator

import (
	"encoding/json"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"

	apiv1 "cuttlefish/liboperator/api/v1"

	"github.com/gorilla/mux"
)

// Sets up a unix socket for devices to connect to and returns a function that listens on the
// socket until an error occurrs.
func SetupDeviceEndpoint(pool *DevicePool, config apiv1.InfraConfig, path string) func() error {
	if err := os.RemoveAll(path); err != nil {
		return func() error { return fmt.Errorf("Failed to clean previous socket: %w", err) }
	}
	addr, err := net.ResolveUnixAddr("unixpacket", path)
	if err != nil {
		// Returns a loop function that will immediately return an error when invoked
		return func() error { return fmt.Errorf("Failed to create unix address from path: %w", err) }
	}
	sock, err := net.ListenUnix("unixpacket", addr)
	if err != nil {
		return func() error { return fmt.Errorf("Failed to create unix socket: %w", err) }
	}
	// Make sure the socket is only accessible by owner and group
	if err := os.Chmod(path, 0770); err != nil {
		// This shouldn't happen since the creation of the socket just succeeded
		log.Println("Failed to change permissions on socket file: ", err)
	}
	log.Println("Device endpoint created")
	// Serve the register_device endpoint in a background thread
	return func() error {
		defer sock.Close()
		for {
			c, err := sock.AcceptUnix()
			if err != nil {
				return err
			}
			go deviceEndpoint(NewJSONUnix(c), pool, config)
		}
	}
}

// Creates a router with handlers for the following endpoints:
// GET  /infra_config
// GET  /devices
// GET  /devices/{deviceId}/files/{path}
// GET  /polled_connections
// GET  /polled_connections/{connId}/messages
// POST /polled_connections/{connId}/:forward
// The maybeIntercept parameter is a function that accepts the
// requested device file and returns a path to a file to be returned instead or
// nil if the request should be allowed to proceed to the device.
func CreateHttpHandlers(
	pool *DevicePool,
	polledSet *PolledSet,
	config apiv1.InfraConfig,
	maybeIntercept func(string) *string) *mux.Router {
	router := mux.NewRouter()
	// The path parameter needs to include the leading '/'
	router.HandleFunc("/devices/{deviceId}/files{path:/.+}", func(w http.ResponseWriter, r *http.Request) {
		deviceFiles(w, r, pool, maybeIntercept)
	}).Methods("GET")
	router.HandleFunc("/devices", func(w http.ResponseWriter, r *http.Request) {
		listDevices(w, r, pool)
	}).Methods("GET")
	router.HandleFunc("/polled_connections/{connId}/:forward", func(w http.ResponseWriter, r *http.Request) {
		forward(w, r, polledSet)
	}).Methods("POST")
	router.HandleFunc("/polled_connections/{connId}/messages", func(w http.ResponseWriter, r *http.Request) {
		messages(w, r, polledSet)
	}).Methods("GET")
	router.HandleFunc("/polled_connections", func(w http.ResponseWriter, r *http.Request) {
		createPolledConnection(w, r, pool, polledSet)
	}).Methods("POST")
	router.HandleFunc("/infra_config", func(w http.ResponseWriter, r *http.Request) {
		ReplyJSONOK(w, config)
	}).Methods("GET")
	return router
}

// Device endpoint
func deviceEndpoint(c *JSONUnix, pool *DevicePool, config apiv1.InfraConfig) {
	log.Println("Device connected")
	defer c.Close()
	var msg apiv1.RegisterMsg
	if err := c.Recv(&msg); err != nil {
		log.Println("Error reading from device: ", err)
		return
	}
	if msg.Type != "register" {
		ReplyError(c, "First device message must be the registration")
		return
	}
	id := msg.DeviceId
	if id == "" {
		ReplyError(c, "Missing device_id")
		return
	}
	info := msg.Info
	if info == nil {
		log.Println("No device info provided by: ", id)
		info = make(map[string]interface{})
	}
	port := msg.Port
	device := NewDevice(c, port, info)
	if !pool.Register(device, id) {
		ReplyError(c, fmt.Sprintln("Device id already taken: ", id))
		return
	}
	defer pool.Unregister(id)
	if err := device.Send(config); err != nil {
		log.Println("Failed to send config to device: ", err)
		return
	}
	for {
		var msg apiv1.ForwardMsg
		if err := c.Recv(&msg); err != nil {
			log.Println("Error reading from device: ", err)
			return
		}
		if msg.Type != "forward" {
			ReplyError(c, fmt.Sprintln("Unrecognized message type: ", msg.Type))
			return
		}
		clientId := msg.ClientId
		if clientId == 0 {
			ReplyError(c, "Device forward message missing client id")
			return
		}
		payload := msg.Payload
		if payload == nil {
			ReplyError(c, "Device forward message missing payload")
			return
		}
		dMsg := map[string]interface{}{
			"message_type": "device_msg",
			"payload":      payload,
		}
		if err := device.ToClient(clientId, dMsg); err != nil {
			log.Println("Device: ", id, " failed to send message to client: ", err)
			ReplyError(c, fmt.Sprintln("Client disconnected: ", clientId))
		}
	}
}

// General client endpoints

func listDevices(w http.ResponseWriter, r *http.Request, pool *DevicePool) {
	if err := ReplyJSONOK(w, pool.DeviceIds()); err != nil {
		log.Println(err)
	}
}

func deviceFiles(w http.ResponseWriter, r *http.Request, pool *DevicePool, maybeIntercept func(string) *string) {
	vars := mux.Vars(r)
	devId := vars["deviceId"]
	dev := pool.GetDevice(devId)
	if dev == nil {
		http.NotFound(w, r)
		return
	}
	path := vars["path"]
	if alt := maybeIntercept(path); alt != nil {
		http.ServeFile(w, r, *alt)
	} else {
		r.URL.Path = path
		dev.Proxy.ServeHTTP(w, r)
	}
}

// Http long polling client endpoints

func createPolledConnection(w http.ResponseWriter, r *http.Request, pool *DevicePool, polledSet *PolledSet) {
	var msg apiv1.NewConnMsg
	err := json.NewDecoder(r.Body).Decode(&msg)
	if err != nil {
		log.Println("Failed to parse json from client: ", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	log.Println("id: ", msg.DeviceId)
	device := pool.GetDevice(msg.DeviceId)
	if device == nil {
		http.Error(w, "Device not found", http.StatusNotFound)
		return
	}
	conn := polledSet.NewConnection(device)
	reply := apiv1.NewConnReply{ConnId: conn.Id(), DeviceInfo: device.info}
	ReplyJSONOK(w, reply)
}

func forward(w http.ResponseWriter, r *http.Request, polledSet *PolledSet) {
	id := mux.Vars(r)["connId"]
	conn := polledSet.GetConnection(id)
	if conn == nil {
		http.NotFound(w, r)
		return
	}
	var msg apiv1.ForwardMsg
	err := json.NewDecoder(r.Body).Decode(&msg)
	if err != nil {
		log.Println("Failed to parse json from client: ", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	cMsg := apiv1.ClientMsg{
		Type:     "client_msg",
		ClientId: conn.ClientId(),
		Payload:  msg.Payload,
	}
	if err := conn.ToDevice(cMsg); err != nil {
		log.Println("Failed to send message to device: ", err)
		http.Error(w, "Device disconnected", http.StatusNotFound)
	}
	ReplyJSONOK(w, "ok")
}

func messages(w http.ResponseWriter, r *http.Request, polledSet *PolledSet) {
	id := mux.Vars(r)["connId"]
	conn := polledSet.GetConnection(id)
	if conn == nil {
		http.NotFound(w, r)
		return
	}
	start := 0
	count := -1 // All messages
	if sStr := r.FormValue("start"); sStr != "" {
		i, err := strconv.Atoi(sStr)
		if err != nil {
			log.Println("Invalid start value: ", sStr)
			http.Error(w, "Invalid value for start field", http.StatusBadRequest)
			return
		}
		start = i
	}
	if cStr := r.FormValue("count"); cStr != "" {
		i, err := strconv.Atoi(cStr)
		if err != nil {
			log.Println("Invalid count value: ", cStr)
			http.Error(w, "Invalid value for count field", http.StatusBadRequest)
			return
		}
		count = i
	}
	ReplyJSONOK(w, conn.GetMessages(start, count))
}
<commit_msg>Add API for getting device info<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package operator

import (
	"encoding/json"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"

	apiv1 "cuttlefish/liboperator/api/v1"

	"github.com/gorilla/mux"
)

// Sets up a unix socket for devices to connect to and returns a function that listens on the
// socket until an error occurrs.
func SetupDeviceEndpoint(pool *DevicePool, config apiv1.InfraConfig, path string) func() error {
	if err := os.RemoveAll(path); err != nil {
		return func() error { return fmt.Errorf("Failed to clean previous socket: %w", err) }
	}
	addr, err := net.ResolveUnixAddr("unixpacket", path)
	if err != nil {
		// Returns a loop function that will immediately return an error when invoked
		return func() error { return fmt.Errorf("Failed to create unix address from path: %w", err) }
	}
	sock, err := net.ListenUnix("unixpacket", addr)
	if err != nil {
		return func() error { return fmt.Errorf("Failed to create unix socket: %w", err) }
	}
	// Make sure the socket is only accessible by owner and group
	if err := os.Chmod(path, 0770); err != nil {
		// This shouldn't happen since the creation of the socket just succeeded
		log.Println("Failed to change permissions on socket file: ", err)
	}
	log.Println("Device endpoint created")
	// Serve the register_device endpoint in a background thread
	return func() error {
		defer sock.Close()
		for {
			c, err := sock.AcceptUnix()
			if err != nil {
				return err
			}
			go deviceEndpoint(NewJSONUnix(c), pool, config)
		}
	}
}

// Creates a router with handlers for the following endpoints:
// GET  /infra_config
// GET  /devices
// GET  /devices/{deviceId}
// GET  /devices/{deviceId}/files/{path}
// GET  /polled_connections
// GET  /polled_connections/{connId}/messages
// POST /polled_connections/{connId}/:forward
// The maybeIntercept parameter is a function that accepts the
// requested device file and returns a path to a file to be returned instead or
// nil if the request should be allowed to proceed to the device.
func CreateHttpHandlers(
	pool *DevicePool,
	polledSet *PolledSet,
	config apiv1.InfraConfig,
	maybeIntercept func(string) *string) *mux.Router {
	router := mux.NewRouter()
	// The path parameter needs to include the leading '/'
	router.HandleFunc("/devices/{deviceId}/files{path:/.+}", func(w http.ResponseWriter, r *http.Request) {
		deviceFiles(w, r, pool, maybeIntercept)
	}).Methods("GET")
	router.HandleFunc("/devices", func(w http.ResponseWriter, r *http.Request) {
		listDevices(w, r, pool)
	}).Methods("GET")
	router.HandleFunc("/devices/{deviceId}", func (w http.ResponseWriter, r *http.Request) {
		deviceInfo(w, r, pool);
	}).Methods("GET")
	router.HandleFunc("/polled_connections/{connId}/:forward", func(w http.ResponseWriter, r *http.Request) {
		forward(w, r, polledSet)
	}).Methods("POST")
	router.HandleFunc("/polled_connections/{connId}/messages", func(w http.ResponseWriter, r *http.Request) {
		messages(w, r, polledSet)
	}).Methods("GET")
	router.HandleFunc("/polled_connections", func(w http.ResponseWriter, r *http.Request) {
		createPolledConnection(w, r, pool, polledSet)
	}).Methods("POST")
	router.HandleFunc("/infra_config", func(w http.ResponseWriter, r *http.Request) {
		ReplyJSONOK(w, config)
	}).Methods("GET")
	return router
}

// Device endpoint
func deviceEndpoint(c *JSONUnix, pool *DevicePool, config apiv1.InfraConfig) {
	log.Println("Device connected")
	defer c.Close()
	var msg apiv1.RegisterMsg
	if err := c.Recv(&msg); err != nil {
		log.Println("Error reading from device: ", err)
		return
	}
	if msg.Type != "register" {
		ReplyError(c, "First device message must be the registration")
		return
	}
	id := msg.DeviceId
	if id == "" {
		ReplyError(c, "Missing device_id")
		return
	}
	info := msg.Info
	if info == nil {
		log.Println("No device info provided by: ", id)
		info = make(map[string]interface{})
	}
	port := msg.Port
	device := NewDevice(c, port, info)
	if !pool.Register(device, id) {
		ReplyError(c, fmt.Sprintln("Device id already taken: ", id))
		return
	}
	defer pool.Unregister(id)
	if err := device.Send(config); err != nil {
		log.Println("Failed to send config to device: ", err)
		return
	}
	for {
		var msg apiv1.ForwardMsg
		if err := c.Recv(&msg); err != nil {
			log.Println("Error reading from device: ", err)
			return
		}
		if msg.Type != "forward" {
			ReplyError(c, fmt.Sprintln("Unrecognized message type: ", msg.Type))
			return
		}
		clientId := msg.ClientId
		if clientId == 0 {
			ReplyError(c, "Device forward message missing client id")
			return
		}
		payload := msg.Payload
		if payload == nil {
			ReplyError(c, "Device forward message missing payload")
			return
		}
		dMsg := map[string]interface{}{
			"message_type": "device_msg",
			"payload":      payload,
		}
		if err := device.ToClient(clientId, dMsg); err != nil {
			log.Println("Device: ", id, " failed to send message to client: ", err)
			ReplyError(c, fmt.Sprintln("Client disconnected: ", clientId))
		}
	}
}

// General client endpoints

func listDevices(w http.ResponseWriter, r *http.Request, pool *DevicePool) {
	if err := ReplyJSONOK(w, pool.DeviceIds()); err != nil {
		log.Println(err)
	}
}

// Get device info

func deviceInfo(w http.ResponseWriter, r *http.Request, pool *DevicePool) {
	vars := mux.Vars(r)
	devId := vars["deviceId"]
	dev := pool.GetDevice(devId)
	if dev == nil {
		http.NotFound(w, r)
		return
	}
	ReplyJSONOK(w, dev.info)
}

func deviceFiles(w http.ResponseWriter, r *http.Request, pool *DevicePool, maybeIntercept func(string) *string) {
	vars := mux.Vars(r)
	devId := vars["deviceId"]
	dev := pool.GetDevice(devId)
	if dev == nil {
		http.NotFound(w, r)
		return
	}
	path := vars["path"]
	if alt := maybeIntercept(path); alt != nil {
		http.ServeFile(w, r, *alt)
	} else {
		r.URL.Path = path
		dev.Proxy.ServeHTTP(w, r)
	}
}

// Http long polling client endpoints

func createPolledConnection(w http.ResponseWriter, r *http.Request, pool *DevicePool, polledSet *PolledSet) {
	var msg apiv1.NewConnMsg
	err := json.NewDecoder(r.Body).Decode(&msg)
	if err != nil {
		log.Println("Failed to parse json from client: ", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	log.Println("id: ", msg.DeviceId)
	device := pool.GetDevice(msg.DeviceId)
	if device == nil {
		http.Error(w, "Device not found", http.StatusNotFound)
		return
	}
	conn := polledSet.NewConnection(device)
	reply := apiv1.NewConnReply{ConnId: conn.Id(), DeviceInfo: device.info}
	ReplyJSONOK(w, reply)
}

func forward(w http.ResponseWriter, r *http.Request, polledSet *PolledSet) {
	id := mux.Vars(r)["connId"]
	conn := polledSet.GetConnection(id)
	if conn == nil {
		http.NotFound(w, r)
		return
	}
	var msg apiv1.ForwardMsg
	err := json.NewDecoder(r.Body).Decode(&msg)
	if err != nil {
		log.Println("Failed to parse json from client: ", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	cMsg := apiv1.ClientMsg{
		Type:     "client_msg",
		ClientId: conn.ClientId(),
		Payload:  msg.Payload,
	}
	if err := conn.ToDevice(cMsg); err != nil {
		log.Println("Failed to send message to device: ", err)
		http.Error(w, "Device disconnected", http.StatusNotFound)
	}
	ReplyJSONOK(w, "ok")
}

func messages(w http.ResponseWriter, r *http.Request, polledSet *PolledSet) {
	id := mux.Vars(r)["connId"]
	conn := polledSet.GetConnection(id)
	if conn == nil {
		http.NotFound(w, r)
		return
	}
	start := 0
	count := -1 // All messages
	if sStr := r.FormValue("start"); sStr != "" {
		i, err := strconv.Atoi(sStr)
		if err != nil {
			log.Println("Invalid start value: ", sStr)
			http.Error(w, "Invalid value for start field", http.StatusBadRequest)
			return
		}
		start = i
	}
	if cStr := r.FormValue("count"); cStr != "" {
		i, err := strconv.Atoi(cStr)
		if err != nil {
			log.Println("Invalid count value: ", cStr)
			http.Error(w, "Invalid value for count field", http.StatusBadRequest)
			return
		}
		count = i
	}
	ReplyJSONOK(w, conn.GetMessages(start, count))
}
<|endoftext|>


<commit_before>// Package protocol provides core interfaces for protocol routing and negotiation in libp2p.
package protocol

import (
	"io"
)

// HandlerFunc is a user-provided function used by the Router to
// handle a protocol/stream.
//
// Will be invoked with the protocol ID string as the first argument,
// which may differ from the ID used for registration if the handler
// was registered using a match function.
type HandlerFunc = func(protocol string, rwc io.ReadWriteCloser) error

// Router is an interface that allows users to add and remove protocol handlers,
// which will be invoked when incoming stream requests for registered protocols
// are accepted.
//
// Upon receiving an incoming stream request, the Router will check all registered
// protocol handlers to determine which (if any) is capable of handling the stream.
// The handlers are checked in order of registration; if multiple handlers are
// eligible, only the first to be registered will be invoked.
type Router interface {

	// AddHandler registers the given handler to be invoked for
	// an exact literal match of the given protocol ID string.
	AddHandler(protocol string, handler HandlerFunc)

	// AddHandlerWithFunc registers the given handler to be invoked
	// when the provided match function returns true.
	//
	// The match function will be invoked with an incoming protocol
	// ID string, and should return true if the handler supports
	// the protocol. Note that the protocol ID argument is not
	// used for matching; if you want to match the protocol ID
	// string exactly, you must check for it in your match function.
	AddHandlerWithFunc(protocol string, match func(string) bool, handler HandlerFunc)

	// RemoveHandler removes the registered handler (if any) for the
	// given protocol ID string.
	RemoveHandler(protocol string)

	// Protocols returns a list of all registered protocol ID strings.
	// Note that the Router may be able to handle protocol IDs not
	// included in this list if handlers were added with match functions
	// using AddHandlerWithFunc.
	Protocols() []string
}

// Negotiator is a component capable of reaching agreement over what protocols
// to use for inbound streams of communication.
type Negotiator interface {
	// NegotiateLazy will return the registered protocol handler to use
	// for a given inbound stream, returning as soon as the protocol has been
	// determined. Returns an error if negotiation fails.
	//
	// NegotiateLazy may return before all protocol negotiation responses have been
	// written to the stream. This is in contrast to Negotiate, which will block until
	// the Negotiator is finished with the stream.
	//
	// Deprecated: use Negotiate instead.
	NegotiateLazy(rwc io.ReadWriteCloser) (io.ReadWriteCloser, string, HandlerFunc, error)

	// Negotiate will return the registered protocol handler to use for a given
	// inbound stream, returning after the protocol has been determined and the
	// Negotiator has finished using the stream for negotiation. Returns an
	// error if negotiation fails.
	Negotiate(rwc io.ReadWriteCloser) (string, HandlerFunc, error)

	// Handle calls Negotiate to determine which protocol handler to use for an
	// inbound stream, then invokes the protocol handler function, passing it
	// the protocol ID and the stream. Returns an error if negotiation fails.
	Handle(rwc io.ReadWriteCloser) error
}

// Switch is the component responsible for "dispatching" incoming stream requests to
// their corresponding stream handlers. It is both a Negotiator and a Router.
type Switch interface {
	Router
	Negotiator
}
<commit_msg>core/protocol: remove deprecated Negotiator.NegotiateLazy (#1869)<commit_after>// Package protocol provides core interfaces for protocol routing and negotiation in libp2p.
package protocol

import (
	"io"
)

// HandlerFunc is a user-provided function used by the Router to
// handle a protocol/stream.
//
// Will be invoked with the protocol ID string as the first argument,
// which may differ from the ID used for registration if the handler
// was registered using a match function.
type HandlerFunc = func(protocol string, rwc io.ReadWriteCloser) error

// Router is an interface that allows users to add and remove protocol handlers,
// which will be invoked when incoming stream requests for registered protocols
// are accepted.
//
// Upon receiving an incoming stream request, the Router will check all registered
// protocol handlers to determine which (if any) is capable of handling the stream.
// The handlers are checked in order of registration; if multiple handlers are
// eligible, only the first to be registered will be invoked.
type Router interface {

	// AddHandler registers the given handler to be invoked for
	// an exact literal match of the given protocol ID string.
	AddHandler(protocol string, handler HandlerFunc)

	// AddHandlerWithFunc registers the given handler to be invoked
	// when the provided match function returns true.
	//
	// The match function will be invoked with an incoming protocol
	// ID string, and should return true if the handler supports
	// the protocol. Note that the protocol ID argument is not
	// used for matching; if you want to match the protocol ID
	// string exactly, you must check for it in your match function.
	AddHandlerWithFunc(protocol string, match func(string) bool, handler HandlerFunc)

	// RemoveHandler removes the registered handler (if any) for the
	// given protocol ID string.
	RemoveHandler(protocol string)

	// Protocols returns a list of all registered protocol ID strings.
	// Note that the Router may be able to handle protocol IDs not
	// included in this list if handlers were added with match functions
	// using AddHandlerWithFunc.
	Protocols() []string
}

// Negotiator is a component capable of reaching agreement over what protocols
// to use for inbound streams of communication.
type Negotiator interface {
	// Negotiate will return the registered protocol handler to use for a given
	// inbound stream, returning after the protocol has been determined and the
	// Negotiator has finished using the stream for negotiation. Returns an
	// error if negotiation fails.
	Negotiate(rwc io.ReadWriteCloser) (string, HandlerFunc, error)

	// Handle calls Negotiate to determine which protocol handler to use for an
	// inbound stream, then invokes the protocol handler function, passing it
	// the protocol ID and the stream. Returns an error if negotiation fails.
	Handle(rwc io.ReadWriteCloser) error
}

// Switch is the component responsible for "dispatching" incoming stream requests to
// their corresponding stream handlers. It is both a Negotiator and a Router.
type Switch interface {
	Router
	Negotiator
}
<|endoftext|>


<commit_before>package processorcommand

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"

	"github.com/trustmaster/go-aspell"
)

type OCRResult struct {
	Type string
	Text string
}

func newOCRResult(ocrType string, result string) *OCRResult {
	return &OCRResult{
		ocrType,
		result,
	}
}

func (this *OCRResult) removeNonWords(blob string) string {
	speller, err := aspell.NewSpeller(map[string]string{
		"lang": "en_US",
	})
	if err != nil {
		fmt.Printf("Error: %s", err.Error())
		return ""
	}
	defer speller.Delete()

	singleCharWords := regexp.MustCompile("(a|i)")
	numberRegex := regexp.MustCompile("\\d{3,}")
	wordRegexp := regexp.MustCompile("\\b(\\w+)\\b")
	words := wordRegexp.FindAllString(blob, -1)

	str := ""

	for _, word := range words {
		if numberRegex.MatchString(word) {
			str += " " + word
		} else if len(word) == 1 {
			if singleCharWords.MatchString(word) {
				str += " " + word
			}
		} else if speller.Check(word) {
			str += " " + word
		}
	}

	return strings.TrimSpace(str)
}

func (this *OCRResult) getBestString() string {
	cleanString := this.removeNonWords(this.Text)
	ocrWordCount := this.wordCount(this.Text)
	cleanWordCount := this.wordCount(cleanString)

	percentKept := float64(cleanWordCount) / float64(ocrWordCount)

	if percentKept > .90 {
		return this.Text
	}

	return cleanString
}

func (this *OCRResult) wordCount(blob string) int {
	word_regexp := regexp.MustCompile("\\b(\\w+)\\b")
	words := word_regexp.FindAllString(blob, -1)

	// don't let single char words count towards the overal word count. Gets thrown off by poor OCR results
	count := 0
	for _, word := range words {
		if len(word) > 1 {
			count++
		}
	}

	return count
}

type MultiOCRCommand []OCRCommand

func (this MultiOCRCommand) Run(image string) (*OCRResult, error) {
	results := make(chan *OCRResult, len(this))
	errs := make(chan error, len(this))

	for _, command := range this {
		go func(c OCRCommand) {
			k, err := c.Run(image)
			if err != nil {
				errs <- err
				return
			}
			results <- k
		}(command)
	}

	max := -1
	var best *OCRResult

	for i := 0; i < len(this); i++ {
		select {
		case result := <-results:
			blob := result.getBestString()
			count := result.wordCount(blob)

			if count > max {
				best = result
				max = count
			}

		case err := <-errs:
			return nil, err
		}
	}

	// Return the average, same as before.
	return best, nil
}

type OCRCommand interface {
	Run(image string) (*OCRResult, error)
}

type MemeOCR struct {
	name string
}

func NewMemeOCR() *MemeOCR {
	return &MemeOCR{
		"MemeOCR",
	}
}

func (this *MemeOCR) Run(image string) (*OCRResult, error) {
	imageTif := fmt.Sprintf("%s_meme.tif", image)
	outText := fmt.Sprintf("%s_meme", image)
	preprocessingArgs := []string{image, "-resize", "400%", "-fill", "black", "-fuzz", "10%", "+opaque", "#FFFFFF", imageTif}
	tesseractArgs := []string{"-l", "meme", imageTif, outText}

	err := runProcessorCommand(GM_COMMAND, preprocessingArgs)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Meme preprocessing command failed with error = %v", err))
	}
	defer os.Remove(imageTif)

	err = runProcessorCommand("tesseract", tesseractArgs)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Meme tesseract command failed with error = %v", err))
	}
	defer os.Remove(outText)

	text, err := ioutil.ReadFile(outText + ".txt")
	if err != nil {
		return nil, err
	}

	result := strings.ToLower(strings.TrimSpace(string(text[:])))

	return newOCRResult(this.name, result), nil
}

type StandardOCR struct {
	name string
}

func NewStandardOCR() *StandardOCR {
	return &StandardOCR{
		"StandardOCR",
	}
}

func (this *StandardOCR) Run(image string) (*OCRResult, error) {
	imageTif := fmt.Sprintf("%s_standard.tif", image)
	outText := fmt.Sprintf("%s_standard", image)

	preprocessingArgs := []string{image, "-resize", "400%", "-type", "Grayscale", imageTif}
	tesseractArgs := []string{"-l", "eng", imageTif, outText}

	err := runProcessorCommand(GM_COMMAND, preprocessingArgs)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Standard preprocessing command failed with error = %v", err))
	}
	defer os.Remove(imageTif)

	err = runProcessorCommand("tesseract", tesseractArgs)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Standard tesseract command failed with error = %v", err))
	}
	defer os.Remove(outText)

	text, err := ioutil.ReadFile(outText + ".txt")
	if err != nil {
		return nil, err
	}

	result := strings.ToLower(strings.TrimSpace(string(text[:])))

	return newOCRResult(this.name, result), nil
}
<commit_msg>clean up the text files from OCR correctly<commit_after>package processorcommand

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"

	"github.com/trustmaster/go-aspell"
)

type OCRResult struct {
	Type string
	Text string
}

func newOCRResult(ocrType string, result string) *OCRResult {
	return &OCRResult{
		ocrType,
		result,
	}
}

func (this *OCRResult) removeNonWords(blob string) string {
	speller, err := aspell.NewSpeller(map[string]string{
		"lang": "en_US",
	})
	if err != nil {
		fmt.Printf("Error: %s", err.Error())
		return ""
	}
	defer speller.Delete()

	singleCharWords := regexp.MustCompile("(a|i)")
	numberRegex := regexp.MustCompile("\\d{3,}")
	wordRegexp := regexp.MustCompile("\\b(\\w+)\\b")
	words := wordRegexp.FindAllString(blob, -1)

	str := ""

	for _, word := range words {
		if numberRegex.MatchString(word) {
			str += " " + word
		} else if len(word) == 1 {
			if singleCharWords.MatchString(word) {
				str += " " + word
			}
		} else if speller.Check(word) {
			str += " " + word
		}
	}

	return strings.TrimSpace(str)
}

func (this *OCRResult) getBestString() string {
	cleanString := this.removeNonWords(this.Text)
	ocrWordCount := this.wordCount(this.Text)
	cleanWordCount := this.wordCount(cleanString)

	percentKept := float64(cleanWordCount) / float64(ocrWordCount)

	if percentKept > .90 {
		return this.Text
	}

	return cleanString
}

func (this *OCRResult) wordCount(blob string) int {
	word_regexp := regexp.MustCompile("\\b(\\w+)\\b")
	words := word_regexp.FindAllString(blob, -1)

	// don't let single char words count towards the overal word count. Gets thrown off by poor OCR results
	count := 0
	for _, word := range words {
		if len(word) > 1 {
			count++
		}
	}

	return count
}

type MultiOCRCommand []OCRCommand

func (this MultiOCRCommand) Run(image string) (*OCRResult, error) {
	results := make(chan *OCRResult, len(this))
	errs := make(chan error, len(this))

	for _, command := range this {
		go func(c OCRCommand) {
			k, err := c.Run(image)
			if err != nil {
				errs <- err
				return
			}
			results <- k
		}(command)
	}

	max := -1
	var best *OCRResult

	for i := 0; i < len(this); i++ {
		select {
		case result := <-results:
			blob := result.getBestString()
			count := result.wordCount(blob)

			if count > max {
				best = result
				max = count
			}

		case err := <-errs:
			return nil, err
		}
	}

	// Return the average, same as before.
	return best, nil
}

type OCRCommand interface {
	Run(image string) (*OCRResult, error)
}

type MemeOCR struct {
	name string
}

func NewMemeOCR() *MemeOCR {
	return &MemeOCR{
		"MemeOCR",
	}
}

func (this *MemeOCR) Run(image string) (*OCRResult, error) {
	imageTif := fmt.Sprintf("%s_meme.tif", image)
	outText := fmt.Sprintf("%s_meme", image)
	preprocessingArgs := []string{image, "-resize", "400%", "-fill", "black", "-fuzz", "10%", "+opaque", "#FFFFFF", imageTif}
	tesseractArgs := []string{"-l", "meme", imageTif, outText}

	err := runProcessorCommand(GM_COMMAND, preprocessingArgs)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Meme preprocessing command failed with error = %v", err))
	}
	defer os.Remove(imageTif)

	err = runProcessorCommand("tesseract", tesseractArgs)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Meme tesseract command failed with error = %v", err))
	}
	defer os.Remove(outText + ".txt")

	text, err := ioutil.ReadFile(outText + ".txt")
	if err != nil {
		return nil, err
	}

	result := strings.ToLower(strings.TrimSpace(string(text[:])))

	return newOCRResult(this.name, result), nil
}

type StandardOCR struct {
	name string
}

func NewStandardOCR() *StandardOCR {
	return &StandardOCR{
		"StandardOCR",
	}
}

func (this *StandardOCR) Run(image string) (*OCRResult, error) {
	imageTif := fmt.Sprintf("%s_standard.tif", image)
	outText := fmt.Sprintf("%s_standard", image)

	preprocessingArgs := []string{image, "-resize", "400%", "-type", "Grayscale", imageTif}
	tesseractArgs := []string{"-l", "eng", imageTif, outText}

	err := runProcessorCommand(GM_COMMAND, preprocessingArgs)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Standard preprocessing command failed with error = %v", err))
	}
	defer os.Remove(imageTif)

	err = runProcessorCommand("tesseract", tesseractArgs)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("Standard tesseract command failed with error = %v", err))
	}
	defer os.Remove(outText + ".txt")

	text, err := ioutil.ReadFile(outText + ".txt")
	if err != nil {
		return nil, err
	}

	result := strings.ToLower(strings.TrimSpace(string(text[:])))

	return newOCRResult(this.name, result), nil
}
<|endoftext|>