<commit_before>package main

import (
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
)

var (
	flagOut    = flag.String("o", "", "Output file, else stdout.")
	flagPkg    = flag.String("pkg", "main", "Package.")
	flagPrefix = flag.String("prefix", "", "Prefix to strip from filesnames.")
	flagIgnore = flag.String("ignore", "", "Regexp for files we should ignore (for example \\\\.DS_Store).")
)

type _escFile struct {
	data  []byte
	local string
}

func main() {
	flag.Parse()
	var err error
	var fnames, dirnames []string
	content := make(map[string]_escFile)
	prefix := filepath.ToSlash(*flagPrefix)
	var ignoreRegexp *regexp.Regexp
	if *flagIgnore != "" {
		ignoreRegexp, err = regexp.Compile(*flagIgnore)
		if err != nil {
			log.Fatal(err)
		}
	}
	for _, base := range flag.Args() {
		files := []string{base}
		for len(files) > 0 {
			fname := files[0]
			files = files[1:]
			if ignoreRegexp != nil && ignoreRegexp.MatchString(fname) {
				continue
			}
			f, err := os.Open(fname)
			if err != nil {
				log.Fatal(err)
			}
			fi, err := f.Stat()
			if err != nil {
				log.Fatal(err)
			}
			if fi.IsDir() {
				fis, err := f.Readdir(0)
				if err != nil {
					log.Fatal(err)
				}
				for _, fi := range fis {
					files = append(files, filepath.Join(fname, fi.Name()))
				}
			} else {
				b, err := ioutil.ReadAll(f)
				if err != nil {
					log.Fatal(err)
				}
				fpath := filepath.ToSlash(fname)
				n := strings.TrimPrefix(fpath, prefix)
				n = path.Join("/", n)
				content[n] = _escFile{data: b, local: fpath}
				fnames = append(fnames, n)
			}
			f.Close()
		}
	}
	sort.Strings(fnames)
	w := os.Stdout
	if *flagOut != "" {
		if w, err = os.Create(*flagOut); err != nil {
			log.Fatal(err)
		}
		defer w.Close()
	}
	fmt.Fprintf(w, header, *flagPkg)
	dirs := map[string]bool{"/": true}
	for _, fname := range fnames {
		f := content[fname]
		for b := path.Dir(fname); b != "/"; b = path.Dir(b) {
			dirs[b] = true
		}
		var buf bytes.Buffer
		gw := gzip.NewWriter(&buf)
		if _, err := gw.Write(f.data); err != nil {
			log.Fatal(err)
		}
		if err := gw.Close(); err != nil {
			log.Fatal(err)
		}
		fmt.Fprintf(w, `
	%q: {
		local: %q,
		size:  %v,
		compressed: %s,
	},%s`, fname, f.local, len(f.data), segment(&buf), "\n")
	}
	for d := range dirs {
		dirnames = append(dirnames, d)
	}
	sort.Strings(dirnames)
	for _, dir := range dirnames {
		local := path.Join(prefix, dir)
		if len(local) == 0 {
			local = "."
		}
		fmt.Fprintf(w, `
	%q: {
		isDir: true,
		local: %q,
	},%s`, dir, local, "\n")
	}
	fmt.Fprint(w, footer)
}

func segment(s *bytes.Buffer) string {
	var b bytes.Buffer
	b64 := base64.NewEncoder(base64.StdEncoding, &b)
	b64.Write(s.Bytes())
	b64.Close()
	res := "`\n"
	chunk := make([]byte, 76)
	for n, _ := b.Read(chunk); n > 0; n, _ = b.Read(chunk) {
		res += string(chunk[0:n]) + "\n"
	}
	return res + "`"
}

const (
	header = `package %s

import (
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"sync"
	"time"
)

type _escLocalFS struct{}

var _escLocal _escLocalFS

type _escStaticFS struct{}

var _escStatic _escStaticFS

type _escDir struct {
	fs http.FileSystem
	name string
}

type _escFile struct {
	compressed string
	size       int64
	local      string
	isDir      bool

	data []byte
	once sync.Once
	name string
}

func (_escLocalFS) Open(name string) (http.File, error) {
	f, present := _escData[path.Clean(name)]
	if !present {
		return nil, os.ErrNotExist
	}
	return os.Open(f.local)
}

func (_escStaticFS) prepare(name string) (*_escFile, error) {
	f, present := _escData[path.Clean(name)]
	if !present {
		return nil, os.ErrNotExist
	}
	var err error
	f.once.Do(func() {
		f.name = path.Base(name)
		if f.size == 0 {
			return
		}
		var gr *gzip.Reader
		b64 := base64.NewDecoder(base64.StdEncoding, bytes.NewBufferString(f.compressed))
		gr, err = gzip.NewReader(b64)
		if err != nil {
			return
		}
		f.data, err = ioutil.ReadAll(gr)
	})
	if err != nil {
		return nil, err
	}
	return f, nil
}

func (fs _escStaticFS) Open(name string) (http.File, error) {
	f, err := fs.prepare(name)
	if err != nil {
		return nil, err
	}
	return f.File()
}

func (dir _escDir) Open(name string) (http.File, error) {
	return dir.fs.Open(dir.name + name)
}

func (f *_escFile) File() (http.File, error) {
	type httpFile struct {
		*bytes.Reader
		*_escFile
	}
	return &httpFile{
		Reader:   bytes.NewReader(f.data),
		_escFile: f,
	}, nil
}

func (f *_escFile) Close() error {
	return nil
}

func (f *_escFile) Readdir(count int) ([]os.FileInfo, error) {
	return nil, nil
}

func (f *_escFile) Stat() (os.FileInfo, error) {
	return f, nil
}

func (f *_escFile) Name() string {
	return f.name
}

func (f *_escFile) Size() int64 {
	return f.size
}

func (f *_escFile) Mode() os.FileMode {
	return 0
}

func (f *_escFile) ModTime() time.Time {
	return time.Time{}
}

func (f *_escFile) IsDir() bool {
	return f.isDir
}

func (f *_escFile) Sys() interface{} {
	return f
}

// FS returns a http.Filesystem for the embedded assets. If useLocal is true,
// the filesystem's contents are instead used.
func FS(useLocal bool) http.FileSystem {
	if useLocal {
		return _escLocal
	}
	return _escStatic
}

// Dir returns a http.Filesystem for the embedded assets on a given prefix dir.
// If useLocal is true, the filesystem's contents are instead used.
func Dir(useLocal bool, name string) http.FileSystem {
	if useLocal {
		return _escDir{fs: _escLocal, name: name}
	}
	return _escDir{fs: _escStatic, name:name}
}

// FSByte returns the named file from the embedded assets. If useLocal is
// true, the filesystem's contents are instead used.
func FSByte(useLocal bool, name string) ([]byte, error) {
	if useLocal {
		f, err := _escLocal.Open(name)
		if err != nil {
			return nil, err
		}
		return ioutil.ReadAll(f)
	}
	f, err := _escStatic.prepare(name)
	if err != nil {
		return nil, err
	}
	return f.data, nil
}

// FSMustByte is the same as FSByte, but panics if name is not present.
func FSMustByte(useLocal bool, name string) []byte {
	b, err := FSByte(useLocal, name)
	if err != nil {
		panic(err)
	}
	return b
}

// FSString is the string version of FSByte.
func FSString(useLocal bool, name string) (string, error) {
	b, err := FSByte(useLocal, name)
	return string(b), err
}

// FSMustString is the string version of FSMustByte.
func FSMustString(useLocal bool, name string) string {
	return string(FSMustByte(useLocal, name))
}

var _escData = map[string]*_escFile{
`
	footer = `}
`
)
<commit_msg>Fixed minor gofmt issue<commit_after>package main

import (
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
)

var (
	flagOut    = flag.String("o", "", "Output file, else stdout.")
	flagPkg    = flag.String("pkg", "main", "Package.")
	flagPrefix = flag.String("prefix", "", "Prefix to strip from filesnames.")
	flagIgnore = flag.String("ignore", "", "Regexp for files we should ignore (for example \\\\.DS_Store).")
)

type _escFile struct {
	data  []byte
	local string
}

func main() {
	flag.Parse()
	var err error
	var fnames, dirnames []string
	content := make(map[string]_escFile)
	prefix := filepath.ToSlash(*flagPrefix)
	var ignoreRegexp *regexp.Regexp
	if *flagIgnore != "" {
		ignoreRegexp, err = regexp.Compile(*flagIgnore)
		if err != nil {
			log.Fatal(err)
		}
	}
	for _, base := range flag.Args() {
		files := []string{base}
		for len(files) > 0 {
			fname := files[0]
			files = files[1:]
			if ignoreRegexp != nil && ignoreRegexp.MatchString(fname) {
				continue
			}
			f, err := os.Open(fname)
			if err != nil {
				log.Fatal(err)
			}
			fi, err := f.Stat()
			if err != nil {
				log.Fatal(err)
			}
			if fi.IsDir() {
				fis, err := f.Readdir(0)
				if err != nil {
					log.Fatal(err)
				}
				for _, fi := range fis {
					files = append(files, filepath.Join(fname, fi.Name()))
				}
			} else {
				b, err := ioutil.ReadAll(f)
				if err != nil {
					log.Fatal(err)
				}
				fpath := filepath.ToSlash(fname)
				n := strings.TrimPrefix(fpath, prefix)
				n = path.Join("/", n)
				content[n] = _escFile{data: b, local: fpath}
				fnames = append(fnames, n)
			}
			f.Close()
		}
	}
	sort.Strings(fnames)
	w := os.Stdout
	if *flagOut != "" {
		if w, err = os.Create(*flagOut); err != nil {
			log.Fatal(err)
		}
		defer w.Close()
	}
	fmt.Fprintf(w, header, *flagPkg)
	dirs := map[string]bool{"/": true}
	for _, fname := range fnames {
		f := content[fname]
		for b := path.Dir(fname); b != "/"; b = path.Dir(b) {
			dirs[b] = true
		}
		var buf bytes.Buffer
		gw := gzip.NewWriter(&buf)
		if _, err := gw.Write(f.data); err != nil {
			log.Fatal(err)
		}
		if err := gw.Close(); err != nil {
			log.Fatal(err)
		}
		fmt.Fprintf(w, `
	%q: {
		local: %q,
		size:  %v,
		compressed: %s,
	},%s`, fname, f.local, len(f.data), segment(&buf), "\n")
	}
	for d := range dirs {
		dirnames = append(dirnames, d)
	}
	sort.Strings(dirnames)
	for _, dir := range dirnames {
		local := path.Join(prefix, dir)
		if len(local) == 0 {
			local = "."
		}
		fmt.Fprintf(w, `
	%q: {
		isDir: true,
		local: %q,
	},%s`, dir, local, "\n")
	}
	fmt.Fprint(w, footer)
}

func segment(s *bytes.Buffer) string {
	var b bytes.Buffer
	b64 := base64.NewEncoder(base64.StdEncoding, &b)
	b64.Write(s.Bytes())
	b64.Close()
	res := "`\n"
	chunk := make([]byte, 76)
	for n, _ := b.Read(chunk); n > 0; n, _ = b.Read(chunk) {
		res += string(chunk[0:n]) + "\n"
	}
	return res + "`"
}

const (
	header = `package %s

import (
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"sync"
	"time"
)

type _escLocalFS struct{}

var _escLocal _escLocalFS

type _escStaticFS struct{}

var _escStatic _escStaticFS

type _escDir struct {
	fs   http.FileSystem
	name string
}

type _escFile struct {
	compressed string
	size       int64
	local      string
	isDir      bool

	data []byte
	once sync.Once
	name string
}

func (_escLocalFS) Open(name string) (http.File, error) {
	f, present := _escData[path.Clean(name)]
	if !present {
		return nil, os.ErrNotExist
	}
	return os.Open(f.local)
}

func (_escStaticFS) prepare(name string) (*_escFile, error) {
	f, present := _escData[path.Clean(name)]
	if !present {
		return nil, os.ErrNotExist
	}
	var err error
	f.once.Do(func() {
		f.name = path.Base(name)
		if f.size == 0 {
			return
		}
		var gr *gzip.Reader
		b64 := base64.NewDecoder(base64.StdEncoding, bytes.NewBufferString(f.compressed))
		gr, err = gzip.NewReader(b64)
		if err != nil {
			return
		}
		f.data, err = ioutil.ReadAll(gr)
	})
	if err != nil {
		return nil, err
	}
	return f, nil
}

func (fs _escStaticFS) Open(name string) (http.File, error) {
	f, err := fs.prepare(name)
	if err != nil {
		return nil, err
	}
	return f.File()
}

func (dir _escDir) Open(name string) (http.File, error) {
	return dir.fs.Open(dir.name + name)
}

func (f *_escFile) File() (http.File, error) {
	type httpFile struct {
		*bytes.Reader
		*_escFile
	}
	return &httpFile{
		Reader:   bytes.NewReader(f.data),
		_escFile: f,
	}, nil
}

func (f *_escFile) Close() error {
	return nil
}

func (f *_escFile) Readdir(count int) ([]os.FileInfo, error) {
	return nil, nil
}

func (f *_escFile) Stat() (os.FileInfo, error) {
	return f, nil
}

func (f *_escFile) Name() string {
	return f.name
}

func (f *_escFile) Size() int64 {
	return f.size
}

func (f *_escFile) Mode() os.FileMode {
	return 0
}

func (f *_escFile) ModTime() time.Time {
	return time.Time{}
}

func (f *_escFile) IsDir() bool {
	return f.isDir
}

func (f *_escFile) Sys() interface{} {
	return f
}

// FS returns a http.Filesystem for the embedded assets. If useLocal is true,
// the filesystem's contents are instead used.
func FS(useLocal bool) http.FileSystem {
	if useLocal {
		return _escLocal
	}
	return _escStatic
}

// Dir returns a http.Filesystem for the embedded assets on a given prefix dir.
// If useLocal is true, the filesystem's contents are instead used.
func Dir(useLocal bool, name string) http.FileSystem {
	if useLocal {
		return _escDir{fs: _escLocal, name: name}
	}
	return _escDir{fs: _escStatic, name: name}
}

// FSByte returns the named file from the embedded assets. If useLocal is
// true, the filesystem's contents are instead used.
func FSByte(useLocal bool, name string) ([]byte, error) {
	if useLocal {
		f, err := _escLocal.Open(name)
		if err != nil {
			return nil, err
		}
		return ioutil.ReadAll(f)
	}
	f, err := _escStatic.prepare(name)
	if err != nil {
		return nil, err
	}
	return f.data, nil
}

// FSMustByte is the same as FSByte, but panics if name is not present.
func FSMustByte(useLocal bool, name string) []byte {
	b, err := FSByte(useLocal, name)
	if err != nil {
		panic(err)
	}
	return b
}

// FSString is the string version of FSByte.
func FSString(useLocal bool, name string) (string, error) {
	b, err := FSByte(useLocal, name)
	return string(b), err
}

// FSMustString is the string version of FSMustByte.
func FSMustString(useLocal bool, name string) string {
	return string(FSMustByte(useLocal, name))
}

var _escData = map[string]*_escFile{
`
	footer = `}
`
)
<|endoftext|>


<commit_before>// Copyright (c) 2022, the Drone Plugins project authors.
// Please see the AUTHORS file for details. All rights reserved.
// Use of this source code is governed by an Apache 2.0 license that can be
// found in the LICENSE file.

package main

import (
	"context"

	"github.com/drone-plugins/drone-sftp/plugin"

	"github.com/kelseyhightower/envconfig"
	"github.com/sirupsen/logrus"
)

func main() {
	logrus.SetFormatter(new(formatter))

	var args plugin.Args
	if err := envconfig.Process("", &args); err != nil {
		logrus.Fatalln(err)
	}

	switch args.Level {
	case "debug":
		logrus.SetFormatter(textFormatter)
		logrus.SetLevel(logrus.DebugLevel)
	case "trace":
		logrus.SetFormatter(textFormatter)
		logrus.SetLevel(logrus.TraceLevel)
	}

	if err := plugin.Exec(context.Background(), &args); err != nil {
		logrus.Fatalln(err)
	}
}

// default formatter that writes logs without including timestamp or level information.
type formatter struct{}

func (*formatter) Format(entry *logrus.Entry) ([]byte, error) {
	return []byte(entry.Message), nil
}

// text formatter that writes logs with level information
var textFormatter = &logrus.TextFormatter{
	DisableTimestamp: true,
}
<commit_msg>Handle --help in the cli<commit_after>// Copyright (c) 2022, the Drone Plugins project authors.
// Please see the AUTHORS file for details. All rights reserved.
// Use of this source code is governed by an Apache 2.0 license that can be
// found in the LICENSE file.

package main

import (
	"context"
	"os"

	"github.com/drone-plugins/drone-sftp/plugin"

	"github.com/kelseyhightower/envconfig"
	"github.com/sirupsen/logrus"
)

func main() {
	// TODO: Remove when docker runner works on Windows
	argCount := len(os.Args)
	if argCount != 1 {
		if argCount == 2 && os.Args[1] == "--help" {
			os.Exit(0)
		}
		os.Exit(1)
	}

	logrus.SetFormatter(new(formatter))

	var args plugin.Args
	if err := envconfig.Process("", &args); err != nil {
		logrus.Fatalln(err)
	}

	switch args.Level {
	case "debug":
		logrus.SetFormatter(textFormatter)
		logrus.SetLevel(logrus.DebugLevel)
	case "trace":
		logrus.SetFormatter(textFormatter)
		logrus.SetLevel(logrus.TraceLevel)
	}

	if err := plugin.Exec(context.Background(), &args); err != nil {
		logrus.Fatalln(err)
	}
}

// default formatter that writes logs without including timestamp or level information.
type formatter struct{}

func (*formatter) Format(entry *logrus.Entry) ([]byte, error) {
	return []byte(entry.Message), nil
}

// text formatter that writes logs with level information
var textFormatter = &logrus.TextFormatter{
	DisableTimestamp: true,
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
)

func showOnsenList(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Onsen List")
}

func showOnsenList2(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Onsen List2")
}

func main() {
	http.HandleFunc("/onsen/list", showOnsenList)
	http.HandleFunc("/onsen/list2", showOnsenList2)
	http.ListenAndServe(":8080", nil)
}
<commit_msg>add kika-pc<commit_after>package main

import (
	"fmt"
	"net/http"
)

func showOnsenList(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Onsen List")
}

func showKika(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "pc")
}

func main() {
	http.HandleFunc("/onsen/list", showOnsenList)
	http.HandleFunc("/kika", showKika)
	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/opencontainers/runtime-spec/specs-go"
)

// gitCommit will be the hash that the binary was built from
// and will be populated by the Makefile
var gitCommit = ""

const (
	version    = "0.1.1"
	specConfig = "config.json"
	usage      = `Open Container Initiative runtime

runc is a command line client for running applications packaged according to
the Open Container Initiative (OCI) format and is a compliant implementation of the
Open Container Initiative specification.

runc integrates well with existing process supervisors to provide a production
container runtime environment for applications. It can be used with your
existing process monitoring tools and the container will be spawned as a
direct child of the process supervisor.

Containers are configured using bundles. A bundle for a container is a directory
that includes a specification file named "` + specConfig + `" and a root filesystem.
The root filesystem contains the contents of the container.

To start a new instance of a container:

    # runc start [ -b bundle ] <container-id>

Where "<container-id>" is your name for the instance of the container that you
are starting. The name you provide for the container instance must be unique on
your host. Providing the bundle directory using "-b" is optional. The default
value for "bundle" is the current directory.`
)

func main() {
	app := cli.NewApp()
	app.Name = "runc"
	app.Usage = usage
	v := []string{
		version,
	}
	if gitCommit != "" {
		v = append(v, fmt.Sprintf("commit: %s", gitCommit))
	}
	v = append(v, fmt.Sprintf("spec: %s", specs.Version))
	app.Version = strings.Join(v, "\n")
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "debug",
			Usage: "enable debug output for logging",
		},
		cli.StringFlag{
			Name:  "log",
			Value: "/dev/null",
			Usage: "set the log file path where internal debug information is written",
		},
		cli.StringFlag{
			Name:  "log-format",
			Value: "text",
			Usage: "set the format used by logs ('text' (default), or 'json')",
		},
		cli.StringFlag{
			Name:  "root",
			Value: "/run/runc",
			Usage: "root directory for storage of container state (this should be located in tmpfs)",
		},
		cli.StringFlag{
			Name:  "criu",
			Value: "criu",
			Usage: "path to the criu binary used for checkpoint and restore",
		},
		cli.BoolFlag{
			Name:  "systemd-cgroup",
			Usage: "enable systemd cgroup support, expects cgroupsPath to be of form \"slice:prefix:name\" for e.g. \"system.slice:runc:434234\"",
		},
	}
	app.Commands = []cli.Command{
		checkpointCommand,
		deleteCommand,
		eventsCommand,
		execCommand,
		initCommand,
		killCommand,
		listCommand,
		pauseCommand,
		psCommand,
		restoreCommand,
		resumeCommand,
		specCommand,
		startCommand,
		stateCommand,
		updateCommand,
	}
	app.Before = func(context *cli.Context) error {
		if context.GlobalBool("debug") {
			logrus.SetLevel(logrus.DebugLevel)
		}
		if path := context.GlobalString("log"); path != "" {
			f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0666)
			if err != nil {
				return err
			}
			logrus.SetOutput(f)
		}
		switch context.GlobalString("log-format") {
		case "text":
			// retain logrus's default.
		case "json":
			logrus.SetFormatter(new(logrus.JSONFormatter))
		default:
			logrus.Fatalf("unknown log-format %q", context.GlobalString("log-format"))
		}
		return nil
	}
	if err := app.Run(os.Args); err != nil {
		fatal(err)
	}
}
<commit_msg>Unify log setting's error output<commit_after>package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/opencontainers/runtime-spec/specs-go"
)

// gitCommit will be the hash that the binary was built from
// and will be populated by the Makefile
var gitCommit = ""

const (
	version    = "0.1.1"
	specConfig = "config.json"
	usage      = `Open Container Initiative runtime

runc is a command line client for running applications packaged according to
the Open Container Initiative (OCI) format and is a compliant implementation of the
Open Container Initiative specification.

runc integrates well with existing process supervisors to provide a production
container runtime environment for applications. It can be used with your
existing process monitoring tools and the container will be spawned as a
direct child of the process supervisor.

Containers are configured using bundles. A bundle for a container is a directory
that includes a specification file named "` + specConfig + `" and a root filesystem.
The root filesystem contains the contents of the container.

To start a new instance of a container:

    # runc start [ -b bundle ] <container-id>

Where "<container-id>" is your name for the instance of the container that you
are starting. The name you provide for the container instance must be unique on
your host. Providing the bundle directory using "-b" is optional. The default
value for "bundle" is the current directory.`
)

func main() {
	app := cli.NewApp()
	app.Name = "runc"
	app.Usage = usage
	v := []string{
		version,
	}
	if gitCommit != "" {
		v = append(v, fmt.Sprintf("commit: %s", gitCommit))
	}
	v = append(v, fmt.Sprintf("spec: %s", specs.Version))
	app.Version = strings.Join(v, "\n")
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "debug",
			Usage: "enable debug output for logging",
		},
		cli.StringFlag{
			Name:  "log",
			Value: "/dev/null",
			Usage: "set the log file path where internal debug information is written",
		},
		cli.StringFlag{
			Name:  "log-format",
			Value: "text",
			Usage: "set the format used by logs ('text' (default), or 'json')",
		},
		cli.StringFlag{
			Name:  "root",
			Value: "/run/runc",
			Usage: "root directory for storage of container state (this should be located in tmpfs)",
		},
		cli.StringFlag{
			Name:  "criu",
			Value: "criu",
			Usage: "path to the criu binary used for checkpoint and restore",
		},
		cli.BoolFlag{
			Name:  "systemd-cgroup",
			Usage: "enable systemd cgroup support, expects cgroupsPath to be of form \"slice:prefix:name\" for e.g. \"system.slice:runc:434234\"",
		},
	}
	app.Commands = []cli.Command{
		checkpointCommand,
		deleteCommand,
		eventsCommand,
		execCommand,
		initCommand,
		killCommand,
		listCommand,
		pauseCommand,
		psCommand,
		restoreCommand,
		resumeCommand,
		specCommand,
		startCommand,
		stateCommand,
		updateCommand,
	}
	app.Before = func(context *cli.Context) error {
		if context.GlobalBool("debug") {
			logrus.SetLevel(logrus.DebugLevel)
		}
		if path := context.GlobalString("log"); path != "" {
			f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0666)
			if err != nil {
				return err
			}
			logrus.SetOutput(f)
		}
		switch context.GlobalString("log-format") {
		case "text":
			// retain logrus's default.
		case "json":
			logrus.SetFormatter(new(logrus.JSONFormatter))
		default:
			return fmt.Errorf("unknown log-format %q", context.GlobalString("log-format"))
		}
		return nil
	}
	if err := app.Run(os.Args); err != nil {
		fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "time"
    "strconv"

    "github.com/nsf/termbox-go"

    "./board"
)

// Board size
const (
    X = 4
    Y = 4
)

var (
    COLORS = []termbox.Attribute {
        termbox.ColorWhite,
        termbox.ColorGreen,
        termbox.ColorBlue,
        termbox.ColorCyan,
        termbox.ColorMagenta,
        termbox.ColorRed,
    }
)

func pad(str string, pad string, length int) string {
    for  {
        str = pad + str
        if len(str) > length-1 {
            return str[0:length]
        }
    }
 }

func cell_color(power int)termbox.Attribute {
    return COLORS[power % len(COLORS)]
}

func iPow(x int, n int) int {
    accu := 1
    for i := 0; i < n; i++ {
        accu *= x
    }

    return accu
}

func cell_str(power int)string {
    if(power == 0) {
        return ".";
    }
    // Convert to power of two and then string
    return strconv.Itoa(iPow(2, power))
}

func draw_cell(x int, y int, power int) {
    var tx = x*4;
    var color = cell_color(power)
    var str = pad(cell_str(power), " ", 4)
    //fmt.Printf("out str = %#v\nsf", str)

    for i, c := range str {
        //fmt.Println(i, c)
        //if(c == ' ' && tx != 999 && color != termbox.ColorYellow && i != 99) {
        if(c == ' ') {
            continue
        }
        termbox.SetCell(4+tx+i, 4+y, c, color, termbox.ColorDefault)
    }
}

func draw_score(b board.Board) {
    score := 0
    x := 24
    y := 4

    // Compute score
    for _, v := range b.Values() {
        for i := 1; i < v+1; i++ {
            score = score + i * iPow(2, i-1)
        }
    }

    str := "Score: "+ strconv.Itoa(score)

    for i, c := range str {
        termbox.SetCell(x+i, y, c, termbox.ColorDefault, termbox.ColorDefault)
    }
}

func draw_board(b board.Board) {
    termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)

    // Draw the cells
    for y := 0; y < Y; y++ {
    for x := 0; x < X; x++ {
        draw_cell(x, y, b.Cells[y][x])
    }}

    draw_score(b)

    termbox.Flush()
}

func main() {
    err := termbox.Init()
    if err != nil {
        panic(err)
    }

    // Our Game board
    var _board = board.New()

    // Cleanup on exit
    defer termbox.Close()

    // Keyboard only
    termbox.SetInputMode(termbox.InputEsc)

    // Clear empty
    termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
    termbox.Flush()

    draw_board(_board)

    // Event queue
    event_queue := make(chan termbox.Event)
    go func() {
        for {
            event_queue <- termbox.PollEvent()
        }
    }()

    // Event loop
loop:
    for {
        select {
        case ev := <-event_queue:
            switch ev.Type {
            case termbox.EventKey:
                // Exit
                if ev.Key == termbox.KeyCtrlC || ev.Key == termbox.KeyEsc {
                    break loop
                }

                switch ev.Key {
                    case termbox.KeyArrowLeft:
                        _board.Move(board.LEFT)
                    case termbox.KeyArrowRight:
                        _board.Move(board.RIGHT)
                    case termbox.KeyArrowUp:
                        _board.Move(board.UP)
                    case termbox.KeyArrowDown:
                        _board.Move(board.DOWN)
                }

                draw_board(_board)

                // Can no longer play
                if !_board.Playable() {
                    fmt.Println("\n\n\n\n\nLOST !")
                    break loop
                }

                draw_board(_board)
            case termbox.EventResize:
                draw_board(_board)
            }

        default:
            time.Sleep(10 * time.Millisecond)
        }
    }
}

<commit_msg>Add extra comments to draw_score<commit_after>package main

import (
    "fmt"
    "time"
    "strconv"

    "github.com/nsf/termbox-go"

    "./board"
)

// Board size
const (
    X = 4
    Y = 4
)

var (
    COLORS = []termbox.Attribute {
        termbox.ColorWhite,
        termbox.ColorGreen,
        termbox.ColorBlue,
        termbox.ColorCyan,
        termbox.ColorMagenta,
        termbox.ColorRed,
    }
)

func pad(str string, pad string, length int) string {
    for  {
        str = pad + str
        if len(str) > length-1 {
            return str[0:length]
        }
    }
 }

func cell_color(power int)termbox.Attribute {
    return COLORS[power % len(COLORS)]
}

func iPow(x int, n int) int {
    accu := 1
    for i := 0; i < n; i++ {
        accu *= x
    }

    return accu
}

func cell_str(power int)string {
    if(power == 0) {
        return ".";
    }
    // Convert to power of two and then string
    return strconv.Itoa(iPow(2, power))
}

func draw_cell(x int, y int, power int) {
    var tx = x*4;
    var color = cell_color(power)
    var str = pad(cell_str(power), " ", 4)
    //fmt.Printf("out str = %#v\nsf", str)

    for i, c := range str {
        //fmt.Println(i, c)
        //if(c == ' ' && tx != 999 && color != termbox.ColorYellow && i != 99) {
        if(c == ' ') {
            continue
        }
        termbox.SetCell(4+tx+i, 4+y, c, color, termbox.ColorDefault)
    }
}

func draw_score(b board.Board) {
    // Accu
    score := 0

    // Display offsets
    x := 24
    y := 4

    // Compute score
    for _, v := range b.Values() {
        for i := 1; i < v+1; i++ {
            score = score + i * iPow(2, i-1)
        }
    }

    // Build score string
    str := "Score: "+ strconv.Itoa(score)

    // Draw Score string
    for i, c := range str {
        termbox.SetCell(x+i, y, c, termbox.ColorDefault, termbox.ColorDefault)
    }
}

func draw_board(b board.Board) {
    termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)

    // Draw the cells
    for y := 0; y < Y; y++ {
    for x := 0; x < X; x++ {
        draw_cell(x, y, b.Cells[y][x])
    }}

    draw_score(b)

    termbox.Flush()
}

func main() {
    err := termbox.Init()
    if err != nil {
        panic(err)
    }

    // Our Game board
    var _board = board.New()

    // Cleanup on exit
    defer termbox.Close()

    // Keyboard only
    termbox.SetInputMode(termbox.InputEsc)

    // Clear empty
    termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
    termbox.Flush()

    draw_board(_board)

    // Event queue
    event_queue := make(chan termbox.Event)
    go func() {
        for {
            event_queue <- termbox.PollEvent()
        }
    }()

    // Event loop
loop:
    for {
        select {
        case ev := <-event_queue:
            switch ev.Type {
            case termbox.EventKey:
                // Exit
                if ev.Key == termbox.KeyCtrlC || ev.Key == termbox.KeyEsc {
                    break loop
                }

                switch ev.Key {
                    case termbox.KeyArrowLeft:
                        _board.Move(board.LEFT)
                    case termbox.KeyArrowRight:
                        _board.Move(board.RIGHT)
                    case termbox.KeyArrowUp:
                        _board.Move(board.UP)
                    case termbox.KeyArrowDown:
                        _board.Move(board.DOWN)
                }

                draw_board(_board)

                // Can no longer play
                if !_board.Playable() {
                    fmt.Println("\n\n\n\n\nLOST !")
                    break loop
                }

                draw_board(_board)
            case termbox.EventResize:
                draw_board(_board)
            }

        default:
            time.Sleep(10 * time.Millisecond)
        }
    }
}

<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"regexp"

	"github.com/PuerkitoBio/goquery"
)

var teaSites = [1]string{
	"https://verdanttea.com/",
}

var tags = [...]string{
	"span",
	"div",
	"p",
	"text",
	"li",
	"button",
	"a",
	"select",
	"option",
	"h1",
	"h2",
	"h3",
	"h4",
}

func ExampleScrape() {
	doc, err := goquery.NewDocument(teaSites[0])
	if err != nil {
		log.Fatal(err)
	}

	// Find the review items
	found := doc.Find("span").FilterFunction(func(i int, node *goquery.Selection) bool {
		text := node.Text()

		matched, err := regexp.MatchString("oolong", text)
		if err != nil {
			log.Fatal(err)
		}

		return matched
	})

	fmt.Printf("%s", found)

	found.Each(func(i int, s *goquery.Selection) {
		fmt.Printf(s.Text())
	})

}

func main() {
	ExampleScrape()
}
<commit_msg>can get hrefs<commit_after>package main

import (
	"fmt"
	"log"
	"regexp"

	"github.com/PuerkitoBio/goquery"
)

var teaSites = [...]string{
	"https://verdanttea.com/",
}

var teaTypes = [...]string{
	"oolong",
	"black",
	"white",
	"grean",
	"herbal",
	"yellow",
	"fermented",
}

var tags = [...]string{
	"span",
	"div",
	"p",
	"text",
	"li",
	"button",
	"a",
	"select",
	"option",
	"h1",
	"h2",
	"h3",
	"h4",
	"article",
}

/*
	visit site
	look for teaTypes
	if hyperlink, crawl if not seen
	if not hyperlink, save for language processing
*/

func ExampleScrape() {
	doc, err := goquery.NewDocument(teaSites[0])
	if err != nil {
		log.Fatal(err)
	}

	// Find the review items
	found := doc.Find("a").FilterFunction(func(i int, node *goquery.Selection) bool {
		text := node.Text()

		matched, err := regexp.MatchString("Oolong", text)
		if err != nil {
			log.Fatal(err)
		}

		return matched
	})

	found.Each(func(i int, s *goquery.Selection) {
		href, _ := s.Attr("href")
		fmt.Printf("href: %s, text: %s ", href, s.Text())
	})

}

func main() {
	ExampleScrape()
}
<|endoftext|>


<commit_before>package main

import (
	"expvar"
	"flag"
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"regexp"
	"syscall"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/marpaia/graphite-golang"
)

const (
	masterCheckInterval    int = 1
	connectionLostInterval int = 1
	fetchMetricsInterval   int = 1
)

func keyspaceEnable(pool *redis.Pool, port string) {
	c := pool.Get()
	defer c.Close()

	// check if notify-keyspace-events are enabled
	notify, err := redis.StringMap(c.Do("CONFIG", "GET", "notify-keyspace-events"))
	if err != nil {
		log.Printf("[keyspace-check] %s\n", err)
		return
	}

	for _, v := range notify {
		if keyspaceConfigRegex.FindString(v) != "" {
			// already enabled, we can already listen for LIST events
			log.Printf("[keyspace-check-%s] LIST keyspace-notifications already enabled\n", port)
		} else {
			// we need to enable notify-keyspace-events for LIST operations (also do not override previous config if any)
			if v == "" {
				// no previous config was set
				log.Printf("[keyspace-check-%s] Enabling LIST keyspace-notifications (no previous config found)\n", port)
				_, err := redis.String(c.Do("CONFIG", "SET", "notify-keyspace-events", "lK"))
				if err != nil {
					log.Println(err)
					return
				}
			} else {
				// previous config found, do not override
				log.Printf("[keyspace-check-%s] Enabling LIST keyspace-notifications (previous config found)\n", port)
				_, err := redis.String(c.Do("CONFIG", "SET", "notify-keyspace-events", v+"lK"))
				if err != nil {
					log.Println(err)
					return
				}
			}
		}
	}
}

func instanceAlive(pool *redis.Pool) bool {
	c := pool.Get()
	defer c.Close()
	_, err := c.Do("PING")
	if err != nil {
		return false
	}
	return true
}

func instanceIsMaster(pool *redis.Pool, port string) {
	c := pool.Get()
	defer c.Close()

	for {
		master, err := redis.StringMap(c.Do("CONFIG", "GET", "slaveof"))
		if err != nil {
			log.Println(err)
			// Retry connection to Redis until it is back
			defer c.Close()
			time.Sleep(time.Second * time.Duration(connectionLostInterval))
			c = pool.Get()
			continue
		}
		for _, value := range master {
			if value != "" {
				// instance is now a slave, notify
				if fetchPossible[port] {
					c.Do("PUBLISH", "redis-scouter", "stop")
					fetchPossible[port] = false
				}
			} else {
				fetchPossible[port] = true
			}
		}
		time.Sleep(time.Second * time.Duration(masterCheckInterval))
	}
}

func queueStats(port string) {
	// connect to redis
	pool := newPool(port)
	c := pool.Get()
	if !instanceAlive(pool) {
		log.Printf("error: no redis instance listening on port %s, aborting\n", port)
		return
	}

	// subscribe to the keyspace notifications
	c.Send("PSUBSCRIBE", "__keyspace@*", "redis-scouter")
	c.Flush()
	// ignore first two ACKs when subscribing
	c.Receive()
	c.Receive()

	go instanceIsMaster(pool, port)
	go keyspaceEnable(pool, port)

	for {
		if fetchPossible[port] {
			reply, err := redis.StringMap(c.Receive())
			if err != nil {
				// Retry connection to Redis until it is back
				defer c.Close()
				log.Printf("connection to redis lost. retry in 1s\n")
				time.Sleep(time.Second * time.Duration(connectionLostInterval))
				c = pool.Get()
				go keyspaceEnable(pool, port)
				c.Send("PSUBSCRIBE", "__keyspace@*", "redis-scouter")
				c.Flush()
				c.Receive()
				c.Receive()
				continue
			}
			// match for a LIST keyspace event
			for k, v := range reply {
				if v == "stop" {
					// break loop if we get a message on redis-scouter pubsub
					continue
				}
				operation := listOperationsRegex.FindString(v)
				queue := keyspaceRegex.FindStringSubmatch(k)
				if len(queue) == 2 && operation != "" {
					Stats.Add(fmt.Sprintf("%s.%s.%s", port, queue[1], operation), 1)
				}
			}
		} else {
			// do not fetch stats for now
			time.Sleep(time.Second * time.Duration(fetchMetricsInterval))
		}
	}
}

var Stats = expvar.NewMap("stats").Init()
var listOperationsRegex = regexp.MustCompile("^(lpush|lpushx|rpush|rpushx|lpop|blpop|rpop|brpop)$")
var keyspaceRegex = regexp.MustCompile("^__keyspace.*__:(?P<queue_name>.*)$")
var keyspaceConfigRegex = regexp.MustCompile("^(AK.*|.*l.*K.*)$")
var ports redisPorts
var graph *graphite.Graphite
var fetchPossible = make(map[string]bool)

func main() {
	flag.Var(&ports, "ports", "comma-separated list of redis ports")
	graphiteHost := flag.String("graphite-host", "localhost", "graphite hostname")
	graphitePort := flag.Int("graphite-port", 2003, "graphite port")
	interval := flag.Int("interval", 60, "interval for sending graphite metrics")
	simulate := flag.Bool("simulate", false, "simulate sending to graphite via stdout")
	profile := flag.Bool("profile", false, "enable pprof features for cpu/heap/goroutine")
	flag.Parse()

	// flag checks
	if len(ports) == 0 {
		log.Println("no redis instances defined, aborting")
		return
	}

	// simulate graphite sending via stdout
	if *simulate {
		graph = graphite.NewGraphiteNop(*graphiteHost, *graphitePort)
	} else {
		var err error
		graph, err = graphite.NewGraphite(*graphiteHost, *graphitePort)
		if err != nil {
			log.Println(err)
			return
		}
	}

	// check for enabled profiling flag
	if *profile {
		go http.ListenAndServe(":8888", nil)
	}

	hostname := hostnameGraphite()
	ticker := time.NewTicker(time.Second * time.Duration(*interval)).C

	for _, port := range ports {
		fetchPossible[port] = true
		log.Printf("[instance-%s] starting collector\n", port)
		go queueStats(port)
	}

	sig := make(chan os.Signal, 1)
	done := make(chan bool, 1)
	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)

	go func() {
		<-sig
		done <- true
	}()

	for {
		select {
		case <-ticker:
			Stats.Do(func(kv expvar.KeyValue) {
				graph.SimpleSend(fmt.Sprintf("scouter.%s.%s", hostname, kv.Key), kv.Value.String())
			})
		case <-done:
			log.Printf("[main] user aborted execution\n")
			return
		}
	}
}
<commit_msg>fix: enable metric collection only if flag is disabled, otherwise not needed<commit_after>package main

import (
	"expvar"
	"flag"
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"regexp"
	"syscall"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/marpaia/graphite-golang"
)

const (
	masterCheckInterval    int = 1
	connectionLostInterval int = 1
	fetchMetricsInterval   int = 1
)

func keyspaceEnable(pool *redis.Pool, port string) {
	c := pool.Get()
	defer c.Close()

	// check if notify-keyspace-events are enabled
	notify, err := redis.StringMap(c.Do("CONFIG", "GET", "notify-keyspace-events"))
	if err != nil {
		log.Printf("[keyspace-check] %s\n", err)
		return
	}

	for _, v := range notify {
		if keyspaceConfigRegex.FindString(v) != "" {
			// already enabled, we can already listen for LIST events
			log.Printf("[keyspace-check-%s] LIST keyspace-notifications already enabled\n", port)
		} else {
			// we need to enable notify-keyspace-events for LIST operations (also do not override previous config if any)
			if v == "" {
				// no previous config was set
				log.Printf("[keyspace-check-%s] Enabling LIST keyspace-notifications (no previous config found)\n", port)
				_, err := redis.String(c.Do("CONFIG", "SET", "notify-keyspace-events", "lK"))
				if err != nil {
					log.Println(err)
					return
				}
			} else {
				// previous config found, do not override
				log.Printf("[keyspace-check-%s] Enabling LIST keyspace-notifications (previous config found)\n", port)
				_, err := redis.String(c.Do("CONFIG", "SET", "notify-keyspace-events", v+"lK"))
				if err != nil {
					log.Println(err)
					return
				}
			}
		}
	}
}

func instanceAlive(pool *redis.Pool) bool {
	c := pool.Get()
	defer c.Close()
	_, err := c.Do("PING")
	if err != nil {
		return false
	}
	return true
}

func instanceIsMaster(pool *redis.Pool, port string) {
	c := pool.Get()
	defer c.Close()

	for {
		master, err := redis.StringMap(c.Do("CONFIG", "GET", "slaveof"))
		if err != nil {
			log.Println(err)
			// Retry connection to Redis until it is back
			defer c.Close()
			time.Sleep(time.Second * time.Duration(connectionLostInterval))
			c = pool.Get()
			continue
		}
		for _, value := range master {
			if value != "" {
				// instance is now a slave, notify
				if fetchPossible[port] {
					c.Do("PUBLISH", "redis-scouter", "stop")
					fetchPossible[port] = false
				}
			} else {
				// re-enable metrics
				if !fetchPossible[port] {
					fetchPossible[port] = true
				}
			}
		}
		time.Sleep(time.Second * time.Duration(masterCheckInterval))
	}
}

func queueStats(port string) {
	// connect to redis
	pool := newPool(port)
	c := pool.Get()
	if !instanceAlive(pool) {
		log.Printf("error: no redis instance listening on port %s, aborting\n", port)
		return
	}

	// subscribe to the keyspace notifications
	c.Send("PSUBSCRIBE", "__keyspace@*", "redis-scouter")
	c.Flush()
	// ignore first two ACKs when subscribing
	c.Receive()
	c.Receive()

	go instanceIsMaster(pool, port)
	go keyspaceEnable(pool, port)

	for {
		if fetchPossible[port] {
			reply, err := redis.StringMap(c.Receive())
			if err != nil {
				// Retry connection to Redis until it is back
				defer c.Close()
				log.Printf("connection to redis lost. retry in 1s\n")
				time.Sleep(time.Second * time.Duration(connectionLostInterval))
				c = pool.Get()
				go keyspaceEnable(pool, port)
				c.Send("PSUBSCRIBE", "__keyspace@*", "redis-scouter")
				c.Flush()
				c.Receive()
				c.Receive()
				continue
			}
			// match for a LIST keyspace event
			for k, v := range reply {
				if v == "stop" {
					// break loop if we get a message on redis-scouter pubsub
					continue
				}
				operation := listOperationsRegex.FindString(v)
				queue := keyspaceRegex.FindStringSubmatch(k)
				if len(queue) == 2 && operation != "" {
					Stats.Add(fmt.Sprintf("%s.%s.%s", port, queue[1], operation), 1)
				}
			}
		} else {
			// do not fetch stats for now
			time.Sleep(time.Second * time.Duration(fetchMetricsInterval))
		}
	}
}

var Stats = expvar.NewMap("stats").Init()
var listOperationsRegex = regexp.MustCompile("^(lpush|lpushx|rpush|rpushx|lpop|blpop|rpop|brpop)$")
var keyspaceRegex = regexp.MustCompile("^__keyspace.*__:(?P<queue_name>.*)$")
var keyspaceConfigRegex = regexp.MustCompile("^(AK.*|.*l.*K.*)$")
var ports redisPorts
var graph *graphite.Graphite
var fetchPossible = make(map[string]bool)

func main() {
	flag.Var(&ports, "ports", "comma-separated list of redis ports")
	graphiteHost := flag.String("graphite-host", "localhost", "graphite hostname")
	graphitePort := flag.Int("graphite-port", 2003, "graphite port")
	interval := flag.Int("interval", 60, "interval for sending graphite metrics")
	simulate := flag.Bool("simulate", false, "simulate sending to graphite via stdout")
	profile := flag.Bool("profile", false, "enable pprof features for cpu/heap/goroutine")
	flag.Parse()

	// flag checks
	if len(ports) == 0 {
		log.Println("no redis instances defined, aborting")
		return
	}

	// simulate graphite sending via stdout
	if *simulate {
		graph = graphite.NewGraphiteNop(*graphiteHost, *graphitePort)
	} else {
		var err error
		graph, err = graphite.NewGraphite(*graphiteHost, *graphitePort)
		if err != nil {
			log.Println(err)
			return
		}
	}

	// check for enabled profiling flag
	if *profile {
		go http.ListenAndServe(":8888", nil)
	}

	hostname := hostnameGraphite()
	ticker := time.NewTicker(time.Second * time.Duration(*interval)).C

	for _, port := range ports {
		fetchPossible[port] = true
		log.Printf("[instance-%s] starting collector\n", port)
		go queueStats(port)
	}

	sig := make(chan os.Signal, 1)
	done := make(chan bool, 1)
	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)

	go func() {
		<-sig
		done <- true
	}()

	for {
		select {
		case <-ticker:
			Stats.Do(func(kv expvar.KeyValue) {
				graph.SimpleSend(fmt.Sprintf("scouter.%s.%s", hostname, kv.Key), kv.Value.String())
			})
		case <-done:
			log.Printf("[main] user aborted execution\n")
			return
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"image"
	"image/png"
	"io/ioutil"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/bwmarrin/discordgo"
	"github.com/flamingyawn/discryptord/history"
	"github.com/flamingyawn/discryptord/types"
	"github.com/wcharczuk/go-chart"
	"github.com/wcharczuk/go-chart/drawing"
)

// Variables used for command line parameters
var (
	Token string
)

func init() {
	flag.StringVar(&Token, "t", "", "Bot Token")
	flag.Parse()
}

func main() {

	// Create a new Discord session using the provided bot token.
	dg, err := discordgo.New("Bot " + Token)
	if err != nil {
		fmt.Println("error creating Discord session,", err)
		return
	}

	// Register the messageCreate func as a callback for MessageCreate events.
	dg.AddHandler(messageCreate)

	// Open a websocket connection to Discord and begin listening.
	err = dg.Open()
	if err != nil {
		fmt.Println("error opening connection,", err)
		return
	}

	// Wait here until CTRL-C or other term signal is received.
	fmt.Println("Bot is now running.  Press CTRL-C to exit.")
	sc := make(chan os.Signal, 1)
	signal.Notify(sc, syscall.SIGINT, syscall.SIGTERM, os.Interrupt, os.Kill)
	<-sc

	// Cleanly close down the Discord session.
	dg.Close()
}

// This function will be called (due to AddHandler above) every time a new
// message is created on any channel that the autenticated bot has access to.
func messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) {

	// Make sure cryptograph isn't responding to any seedy bots or females
	if len(m.Content) <= 2 {
		return
	}
	authorIsHuman := (m.Author.ID != s.State.User.ID)
	hasAPenis := (m.Content[:2] == "# ")

	if authorIsHuman && hasAPenis {
		// Split the command to separate ticker from penis
		splitCommand := strings.Split(m.Content, " ")

		if len(splitCommand) == 2 || len(splitCommand) == 3 {
			var histoData types.HistoResponse
			var base string
			coin := splitCommand[1]

			// build uri
			if len(splitCommand) == 3 {
				base = splitCommand[2]
			} else {
				base = "usd"
			}
			resp, err := http.Get(history.HistoMinuteFor(coin, base))
			if err != nil {
				fmt.Println(err)
				return
			}
			defer resp.Body.Close()

			//
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				fmt.Println(err)
				return
			}

			//
			err = json.Unmarshal(body, &histoData)
			if err != nil {
				fmt.Println(err)
				return
			}

			////
			var xv []time.Time
			var yv, vol []float64
			var ymin, ymax, volmin, volmax float64 = 1000000, 0, 1000000000, 0

			for _, m := range histoData.Data {
				xv = append(xv, time.Unix(m.Time, 0))
				yv = append(yv, m.Close)
				vol = append(vol, m.Volumeto)

				if m.Close < ymin {
					ymin = m.Close
				}
				if m.Close > ymax {
					ymax = m.Close
				}
				if m.Volumeto < volmin {
					volmin = m.Volumeto
				}
				if m.Volumeto > volmax {
					volmax = m.Volumeto
				}
			}

			for i, v := range vol {
				vol[i] = ((v-volmin)/(volmax-volmin))*(ymax-ymin) + ymin
			}

			priceSeries := chart.TimeSeries{
				Name: "SPY",
				Style: chart.Style{
					Show:        true,
					StrokeColor: drawing.ColorFromHex("4DE786"),
				},
				XValues: xv,
				YValues: yv,
			}

			volumeSeries := chart.TimeSeries{
				Name: "SPY - VOL",
				Style: chart.Style{
					Show:        true,
					StrokeColor: drawing.ColorFromHex("00A1E7").WithAlpha(70),
				},
				XValues: xv,
				YValues: vol,
			}

			smaSeries := chart.SMASeries{
				Name: "SPY - SMA",
				Style: chart.Style{
					Show:            true,
					StrokeColor:     drawing.ColorFromHex("AE73FF"),
					StrokeDashArray: []float64{5.0, 5.0},
				},
				InnerSeries: priceSeries,
			}

			// bbSeries := &chart.BollingerBandsSeries{
			// 	Name: "SPY - Bol. Bands",
			// 	Style: chart.Style{
			// 		Show:        true,
			// 		StrokeColor: drawing.ColorFromHex("ffffff").WithAlpha(30),
			// 		FillColor:   drawing.ColorFromHex("ffffff").WithAlpha(1),
			// 	},
			// 	InnerSeries: priceSeries,
			// }

			graph := chart.Chart{
				Canvas: chart.Style{
					FillColor: drawing.ColorFromHex("36393E"),
				},
				Background: chart.Style{
					FillColor: drawing.ColorFromHex("36393E"),
				},
				XAxis: chart.XAxis{
					Style: chart.Style{
						StrokeColor: drawing.ColorFromHex("ffffff"),
						Show:        false,
					},
					TickPosition: chart.TickPositionBetweenTicks,
				},
				YAxis: chart.YAxis{
					Style: chart.Style{Show: false},
					Range: &chart.ContinuousRange{
						Max: ymax * 1.005,
						Min: ymin * 0.995,
					},
				},
				Series: []chart.Series{
					volumeSeries,
					// bbSeries,
					priceSeries,
					smaSeries,
				},
			}
			buffer := bytes.NewBuffer([]byte{})

			// render and save chart
			err = graph.Render(chart.PNG, buffer)
			img, _, _ := image.Decode(bytes.NewReader(buffer.Bytes()))
			out, err3 := os.Create("./img/graph.png")
			if err3 != nil {
				fmt.Println(err3)
			}
			err = png.Encode(out, img)

			// Read image
			finalImg, err4 := os.Open("./img/graph.png")
			defer finalImg.Close()
			if err4 != nil {
				fmt.Println(err4)
			}

			sym := ""

			if base == "usd" {
				sym = "$"
			} else if base == "btc" {
				sym = "Ƀ"
			} else if base == "eth" {
				sym = "Ξ"
			}

			price := yv[len(yv)-1]
			pairing := strings.ToUpper(coin) + "/" + strings.ToUpper(base)
			msg := "`" + pairing + " (Last 24h) :: " + sym + fmt.Sprintf("%f`", price)

			// Send image
			s.ChannelFileSendWithMessage(m.ChannelID, msg, splitCommand[1]+"usd.png", finalImg)

		}
	}
}
<commit_msg>fix hotfix<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"image"
	"image/png"
	"io/ioutil"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/bwmarrin/discordgo"
	"github.com/flamingyawn/discryptord/history"
	"github.com/flamingyawn/discryptord/types"
	"github.com/wcharczuk/go-chart"
	"github.com/wcharczuk/go-chart/drawing"
)

// Variables used for command line parameters
var (
	Token string
)

func init() {
	flag.StringVar(&Token, "t", "", "Bot Token")
	flag.Parse()
}

func main() {

	// Create a new Discord session using the provided bot token.
	dg, err := discordgo.New("Bot " + Token)
	if err != nil {
		fmt.Println("error creating Discord session,", err)
		return
	}

	// Register the messageCreate func as a callback for MessageCreate events.
	dg.AddHandler(messageCreate)

	// Open a websocket connection to Discord and begin listening.
	err = dg.Open()
	if err != nil {
		fmt.Println("error opening connection,", err)
		return
	}

	// Wait here until CTRL-C or other term signal is received.
	fmt.Println("Bot is now running.  Press CTRL-C to exit.")
	sc := make(chan os.Signal, 1)
	signal.Notify(sc, syscall.SIGINT, syscall.SIGTERM, os.Interrupt, os.Kill)
	<-sc

	// Cleanly close down the Discord session.
	dg.Close()
}

// This function will be called (due to AddHandler above) every time a new
// message is created on any channel that the autenticated bot has access to.
func messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) {

	// Make sure cryptograph isn't responding to any seedy bots or females
	if len(m.Content) <= 2 {
		return
	}
	authorIsHuman := (m.Author.ID != s.State.User.ID)
	hasAPenis := strings.HasPrefix(m.Content, "!")

	if authorIsHuman && hasAPenis {
		// Split the command to separate ticker from penis
		splitCommand := strings.Split(m.Content, " ")

		if len(splitCommand) == 1 || len(splitCommand) == 2 {
			var histoData types.HistoResponse
			var base string
			coin := splitCommand[0][1:]

			// build uri
			if len(splitCommand) == 2 {
				base = splitCommand[1]
			} else {
				base = "usd"
			}
			resp, err := http.Get(history.HistoMinuteFor(coin, base))
			if err != nil {
				fmt.Println(err)
				return
			}
			defer resp.Body.Close()

			//
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				fmt.Println(err)
				return
			}

			//
			err = json.Unmarshal(body, &histoData)
			if err != nil {
				fmt.Println(err)
				return
			}

			////
			var xv []time.Time
			var yv, vol []float64
			var ymin, ymax, volmin, volmax float64 = 1000000, 0, 1000000000, 0

			for _, m := range histoData.Data {
				xv = append(xv, time.Unix(m.Time, 0))
				yv = append(yv, m.Close)
				vol = append(vol, m.Volumeto)

				if m.Close < ymin {
					ymin = m.Close
				}
				if m.Close > ymax {
					ymax = m.Close
				}
				if m.Volumeto < volmin {
					volmin = m.Volumeto
				}
				if m.Volumeto > volmax {
					volmax = m.Volumeto
				}
			}

			for i, v := range vol {
				vol[i] = ((v-volmin)/(volmax-volmin))*(ymax-ymin) + ymin
			}

			priceSeries := chart.TimeSeries{
				Name: "SPY",
				Style: chart.Style{
					Show:        true,
					StrokeColor: drawing.ColorFromHex("4DE786"),
				},
				XValues: xv,
				YValues: yv,
			}

			volumeSeries := chart.TimeSeries{
				Name: "SPY - VOL",
				Style: chart.Style{
					Show:        true,
					StrokeColor: drawing.ColorFromHex("00A1E7").WithAlpha(70),
				},
				XValues: xv,
				YValues: vol,
			}

			smaSeries := chart.SMASeries{
				Name: "SPY - SMA",
				Style: chart.Style{
					Show:            true,
					StrokeColor:     drawing.ColorFromHex("AE73FF"),
					StrokeDashArray: []float64{5.0, 5.0},
				},
				InnerSeries: priceSeries,
			}

			// bbSeries := &chart.BollingerBandsSeries{
			// 	Name: "SPY - Bol. Bands",
			// 	Style: chart.Style{
			// 		Show:        true,
			// 		StrokeColor: drawing.ColorFromHex("ffffff").WithAlpha(30),
			// 		FillColor:   drawing.ColorFromHex("ffffff").WithAlpha(1),
			// 	},
			// 	InnerSeries: priceSeries,
			// }

			graph := chart.Chart{
				Canvas: chart.Style{
					FillColor: drawing.ColorFromHex("36393E"),
				},
				Background: chart.Style{
					FillColor: drawing.ColorFromHex("36393E"),
				},
				XAxis: chart.XAxis{
					Style: chart.Style{
						StrokeColor: drawing.ColorFromHex("ffffff"),
						Show:        false,
					},
					TickPosition: chart.TickPositionBetweenTicks,
				},
				YAxis: chart.YAxis{
					Style: chart.Style{Show: false},
					Range: &chart.ContinuousRange{
						Max: ymax * 1.005,
						Min: ymin * 0.995,
					},
				},
				Series: []chart.Series{
					volumeSeries,
					// bbSeries,
					priceSeries,
					smaSeries,
				},
			}
			buffer := bytes.NewBuffer([]byte{})

			// render and save chart
			err = graph.Render(chart.PNG, buffer)
			img, _, _ := image.Decode(bytes.NewReader(buffer.Bytes()))
			out, err3 := os.Create("./img/graph.png")
			if err3 != nil {
				fmt.Println(err3)
			}
			err = png.Encode(out, img)

			// Read image
			finalImg, err4 := os.Open("./img/graph.png")
			defer finalImg.Close()
			if err4 != nil {
				fmt.Println(err4)
			}

			sym := ""

			if base == "usd" {
				sym = "$"
			} else if base == "btc" {
				sym = "Ƀ"
			} else if base == "eth" {
				sym = "Ξ"
			}

			price := yv[len(yv)-1]
			pairing := strings.ToUpper(coin) + "/" + strings.ToUpper(base)
			msg := "`" + pairing + " (Last 24h) :: " + sym + fmt.Sprintf("%f`", price)

			// Send image
			s.ChannelFileSendWithMessage(m.ChannelID, msg, coin+base+".png", finalImg)

		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"fmt"
	"github.com/CiscoCloud/mantl-api/api"
	"github.com/CiscoCloud/mantl-api/install"
	"github.com/CiscoCloud/mantl-api/marathon"
	"github.com/CiscoCloud/mantl-api/mesos"
	"github.com/CiscoCloud/mantl-api/utils/http"
	"github.com/CiscoCloud/mantl-api/zookeeper"
	log "github.com/Sirupsen/logrus"
	consul "github.com/hashicorp/consul/api"
	"github.com/hashicorp/go-cleanhttp"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
	"os"
	"strings"
)

const Name = "mantl-api"
const Version = "0.1.4"

func main() {
	rootCmd := &cobra.Command{
		Use:   "mantl-api",
		Short: "runs the mantl-api",
		Run: func(cmd *cobra.Command, args []string) {
			start()
		},
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			readConfigFile()
			setupLogging()
		},
	}

	rootCmd.PersistentFlags().String("log-level", "info", "one of debug, info, warn, error, or fatal")
	rootCmd.PersistentFlags().String("log-format", "text", "specify output (text or json)")
	rootCmd.PersistentFlags().String("consul", "http://localhost:8500", "Consul Api address")
	rootCmd.PersistentFlags().Bool("consul-no-verify-ssl", false, "Consul SSL verification")
	rootCmd.PersistentFlags().String("marathon", "", "Marathon Api address")
	rootCmd.PersistentFlags().String("marathon-user", "", "Marathon Api user")
	rootCmd.PersistentFlags().String("marathon-password", "", "Marathon Api password")
	rootCmd.PersistentFlags().Bool("marathon-no-verify-ssl", false, "Marathon SSL verification")
	rootCmd.PersistentFlags().String("mesos", "", "Mesos Api address")
	rootCmd.PersistentFlags().String("mesos-principal", "", "Mesos principal for framework authentication")
	rootCmd.PersistentFlags().String("mesos-secret-path", "", "Path to a file on host sytem that contains the mesos secret for framework authentication")
	rootCmd.PersistentFlags().Bool("mesos-no-verify-ssl", false, "Mesos SSL verification")
	rootCmd.PersistentFlags().String("listen", ":4001", "mantl-api listen address")
	rootCmd.PersistentFlags().String("zookeeper", "", "Comma-delimited list of zookeeper servers")
	rootCmd.PersistentFlags().Bool("force-sync", false, "Force a synchronization of all sources")
	rootCmd.PersistentFlags().String("config-file", "", "The path to a configuration file")

	for _, flags := range []*pflag.FlagSet{rootCmd.PersistentFlags()} {
		err := viper.BindPFlags(flags)
		if err != nil {
			log.WithField("error", err).Fatal("could not bind flags")
		}
	}

	viper.SetEnvPrefix("mantl_api")
	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
	viper.AutomaticEnv()

	syncCommand := &cobra.Command{
		Use:   "sync",
		Short: "Synchronize universe repositories",
		Long:  "Forces a synchronization of all configured sources",
		Run: func(cmd *cobra.Command, args []string) {
			sync(nil, true)
		},
	}
	rootCmd.AddCommand(syncCommand)

	versionCommand := &cobra.Command{
		Use:   "version",
		Short: fmt.Sprintf("Print the version number of %s", Name),
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Printf("%s v%s\n", Name, Version)
		},
	}
	rootCmd.AddCommand(versionCommand)

	rootCmd.Execute()
}

func start() {
	log.Infof("Starting %s v%s", Name, Version)
	client := consulClient()

	marathonUrl := viper.GetString("marathon")
	if marathonUrl == "" {
		marathonUrl = NewDiscovery(client, "marathon", "", "http", "http://localhost:8080").discoveredUrl
	}
	marathonClient, err := marathon.NewMarathon(
		marathonUrl,
		viper.GetString("marathon-user"),
		viper.GetString("marathon-password"),
		viper.GetBool("marathon-no-verify-ssl"),
	)
	if err != nil {
		log.Fatalf("Could not create marathon client: %v", err)
	}

	mesosUrl := viper.GetString("mesos")
	if mesosUrl == "" {
		mesosUrl = NewDiscovery(client, "mesos", "leader", "http", "http://localhost:5050").discoveredUrl
	}
	mesosClient, err := mesos.NewMesos(
		mesosUrl,
		viper.GetString("mesos-principal"),
		viper.GetString("mesos-secret-path"),
		viper.GetBool("mesos-no-verify-ssl"),
	)
	if err != nil {
		log.Fatalf("Could not create mesos client: %v", err)
	}

	zkUrls := viper.GetString("zookeeper")
	if zkUrls == "" {
		zkUrls = NewDiscovery(client, "zookeeper", "", "", "localhost:2181").discoveredUrl
	}
	zkServers := strings.Split(zkUrls, ",")
	zk := zookeeper.NewZookeeper(zkServers)

	inst, err := install.NewInstall(client, marathonClient, mesosClient, zk)
	if err != nil {
		log.Fatalf("Could not create install client: %v", err)
	}

	// sync sources to consul
	sync(inst, viper.GetBool("force-sync"))

	// start listener
	api.NewApi(Name, viper.GetString("listen"), inst, mesosClient).Start()
}

func consulClient() *consul.Client {
	consulConfig := consul.DefaultConfig()
	scheme, address, _, err := http.ParseUrl(viper.GetString("consul"))
	if err != nil {
		log.Fatalf("Could not create consul client: %v", err)
	}
	consulConfig.Scheme = scheme
	consulConfig.Address = address

	log.Debugf("Using Consul at %s over %s", consulConfig.Address, consulConfig.Scheme)

	if viper.GetBool("consul-no-verify-ssl") {
		transport := cleanhttp.DefaultTransport()
		transport.TLSClientConfig = &tls.Config{
			InsecureSkipVerify: true,
		}
		consulConfig.HttpClient.Transport = transport
	}

	client, err := consul.NewClient(consulConfig)
	if err != nil {
		log.Fatalf("Could not create consul client: %v", err)
	}

	// abort if we cannot connect to consul
	err = testConsul(client)
	if err != nil {
		log.Fatalf("Could not connect to consul: %v", err)
	}

	return client
}

func testConsul(client *consul.Client) error {
	kv := client.KV()
	_, _, err := kv.Get("mantl-install", nil)
	return err
}

func sync(inst *install.Install, force bool) {
	var err error
	if inst == nil {
		client := consulClient()
		inst, err = install.NewInstall(client, nil, nil, nil)
		if err != nil {
			log.Fatalf("Could not create install client: %v", err)
		}
	}

	defaultSources := []*install.Source{
		&install.Source{
			Name:       "mantl",
			Path:       "https://github.com/CiscoCloud/mantl-universe.git",
			SourceType: install.Git,
			Branch:     "version-0.6",
			Index:      0,
		},
	}

	sources := []*install.Source{}

	configuredSources := viper.GetStringMap("sources")

	if len(configuredSources) > 0 {
		for name, val := range configuredSources {
			source := &install.Source{Name: name, SourceType: install.FileSystem}
			sourceConfig := val.(map[string]interface{})

			if path, ok := sourceConfig["path"].(string); ok {
				source.Path = path
			}

			if index, ok := sourceConfig["index"].(int64); ok {
				source.Index = int(index)
			}

			if sourceType, ok := sourceConfig["type"].(string); ok {
				if strings.EqualFold(sourceType, "git") {
					source.SourceType = install.Git
				}
			}

			if branch, ok := sourceConfig["branch"].(string); ok {
				source.Branch = branch
			}

			if source.IsValid() {
				sources = append(sources, source)
			} else {
				log.Warnf("Invalid source configuration for %s", name)
			}
		}
	}

	if len(sources) == 0 {
		sources = defaultSources
	}

	inst.SyncSources(sources, force)
}

func readConfigFile() {
	// read configuration file if specified
	configFile := viper.GetString("config-file")
	if configFile != "" {
		configFile = os.ExpandEnv(configFile)
		if _, err := os.Stat(configFile); err == nil {
			viper.SetConfigFile(configFile)
			err = viper.ReadInConfig()
			if err != nil {
				log.Warnf("Could not read configuration file: %v", err)
			}
		} else {
			log.Warnf("Could not find configuration file: %s", configFile)
		}
	}
}

func setupLogging() {
	switch viper.GetString("log-level") {
	case "debug":
		log.SetLevel(log.DebugLevel)
	case "info":
		log.SetLevel(log.InfoLevel)
	case "warn":
		log.SetLevel(log.WarnLevel)
	case "error":
		log.SetLevel(log.ErrorLevel)
	case "fatal":
		log.SetLevel(log.FatalLevel)
	default:
		log.WithField("log-level", viper.GetString("log-level")).Warning("invalid log level. defaulting to info.")
		log.SetLevel(log.InfoLevel)
	}

	switch viper.GetString("log-format") {
	case "text":
		log.SetFormatter(new(log.TextFormatter))
	case "json":
		log.SetFormatter(new(log.JSONFormatter))
	default:
		log.WithField("log-format", viper.GetString("log-format")).Warning("invalid log format. defaulting to text.")
		log.SetFormatter(new(log.TextFormatter))
	}
}
<commit_msg>deprecate mesos-secret<commit_after>package main

import (
	"crypto/tls"
	"fmt"
	"github.com/CiscoCloud/mantl-api/api"
	"github.com/CiscoCloud/mantl-api/install"
	"github.com/CiscoCloud/mantl-api/marathon"
	"github.com/CiscoCloud/mantl-api/mesos"
	"github.com/CiscoCloud/mantl-api/utils/http"
	"github.com/CiscoCloud/mantl-api/zookeeper"
	log "github.com/Sirupsen/logrus"
	consul "github.com/hashicorp/consul/api"
	"github.com/hashicorp/go-cleanhttp"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
	"os"
	"strings"
)

const Name = "mantl-api"
const Version = "0.1.4"

func main() {
	rootCmd := &cobra.Command{
		Use:   "mantl-api",
		Short: "runs the mantl-api",
		Run: func(cmd *cobra.Command, args []string) {
			start()
		},
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			readConfigFile()
			setupLogging()
		},
	}

	rootCmd.PersistentFlags().String("log-level", "info", "one of debug, info, warn, error, or fatal")
	rootCmd.PersistentFlags().String("log-format", "text", "specify output (text or json)")
	rootCmd.PersistentFlags().String("consul", "http://localhost:8500", "Consul Api address")
	rootCmd.PersistentFlags().Bool("consul-no-verify-ssl", false, "Consul SSL verification")
	rootCmd.PersistentFlags().String("marathon", "", "Marathon Api address")
	rootCmd.PersistentFlags().String("marathon-user", "", "Marathon Api user")
	rootCmd.PersistentFlags().String("marathon-password", "", "Marathon Api password")
	rootCmd.PersistentFlags().Bool("marathon-no-verify-ssl", false, "Marathon SSL verification")
	rootCmd.PersistentFlags().String("mesos", "", "Mesos Api address")
	rootCmd.PersistentFlags().String("mesos-principal", "", "Mesos principal for framework authentication")
	rootCmd.PersistentFlags().String("mesos-secret", "", "Deprecated. Use mesos-secret-path instead")
	rootCmd.PersistentFlags().String("mesos-secret-path", "/etc/sysconfig/mantl-api", "Path to a file on host sytem that contains the mesos secret for framework authentication")
	rootCmd.PersistentFlags().Bool("mesos-no-verify-ssl", false, "Mesos SSL verification")
	rootCmd.PersistentFlags().String("listen", ":4001", "mantl-api listen address")
	rootCmd.PersistentFlags().String("zookeeper", "", "Comma-delimited list of zookeeper servers")
	rootCmd.PersistentFlags().Bool("force-sync", false, "Force a synchronization of all sources")
	rootCmd.PersistentFlags().String("config-file", "", "The path to a configuration file")

	for _, flags := range []*pflag.FlagSet{rootCmd.PersistentFlags()} {
		err := viper.BindPFlags(flags)
		if err != nil {
			log.WithField("error", err).Fatal("could not bind flags")
		}
	}

	viper.SetEnvPrefix("mantl_api")
	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
	viper.AutomaticEnv()

	syncCommand := &cobra.Command{
		Use:   "sync",
		Short: "Synchronize universe repositories",
		Long:  "Forces a synchronization of all configured sources",
		Run: func(cmd *cobra.Command, args []string) {
			sync(nil, true)
		},
	}
	rootCmd.AddCommand(syncCommand)

	versionCommand := &cobra.Command{
		Use:   "version",
		Short: fmt.Sprintf("Print the version number of %s", Name),
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Printf("%s v%s\n", Name, Version)
		},
	}
	rootCmd.AddCommand(versionCommand)

	rootCmd.Execute()
}

func start() {
	log.Infof("Starting %s v%s", Name, Version)
	client := consulClient()

	marathonUrl := viper.GetString("marathon")
	if marathonUrl == "" {
		marathonUrl = NewDiscovery(client, "marathon", "", "http", "http://localhost:8080").discoveredUrl
	}
	marathonClient, err := marathon.NewMarathon(
		marathonUrl,
		viper.GetString("marathon-user"),
		viper.GetString("marathon-password"),
		viper.GetBool("marathon-no-verify-ssl"),
	)
	if err != nil {
		log.Fatalf("Could not create marathon client: %v", err)
	}

	mesosUrl := viper.GetString("mesos")
	if mesosUrl == "" {
		mesosUrl = NewDiscovery(client, "mesos", "leader", "http", "http://localhost:5050").discoveredUrl
	}
	mesosClient, err := mesos.NewMesos(
		mesosUrl,
		viper.GetString("mesos-principal"),
		viper.GetString("mesos-secret-path"),
		viper.GetBool("mesos-no-verify-ssl"),
	)
	if err != nil {
		log.Fatalf("Could not create mesos client: %v", err)
	}

	zkUrls := viper.GetString("zookeeper")
	if zkUrls == "" {
		zkUrls = NewDiscovery(client, "zookeeper", "", "", "localhost:2181").discoveredUrl
	}
	zkServers := strings.Split(zkUrls, ",")
	zk := zookeeper.NewZookeeper(zkServers)

	inst, err := install.NewInstall(client, marathonClient, mesosClient, zk)
	if err != nil {
		log.Fatalf("Could not create install client: %v", err)
	}

	// sync sources to consul
	sync(inst, viper.GetBool("force-sync"))

	// start listener
	api.NewApi(Name, viper.GetString("listen"), inst, mesosClient).Start()
}

func consulClient() *consul.Client {
	consulConfig := consul.DefaultConfig()
	scheme, address, _, err := http.ParseUrl(viper.GetString("consul"))
	if err != nil {
		log.Fatalf("Could not create consul client: %v", err)
	}
	consulConfig.Scheme = scheme
	consulConfig.Address = address

	log.Debugf("Using Consul at %s over %s", consulConfig.Address, consulConfig.Scheme)

	if viper.GetBool("consul-no-verify-ssl") {
		transport := cleanhttp.DefaultTransport()
		transport.TLSClientConfig = &tls.Config{
			InsecureSkipVerify: true,
		}
		consulConfig.HttpClient.Transport = transport
	}

	client, err := consul.NewClient(consulConfig)
	if err != nil {
		log.Fatalf("Could not create consul client: %v", err)
	}

	// abort if we cannot connect to consul
	err = testConsul(client)
	if err != nil {
		log.Fatalf("Could not connect to consul: %v", err)
	}

	return client
}

func testConsul(client *consul.Client) error {
	kv := client.KV()
	_, _, err := kv.Get("mantl-install", nil)
	return err
}

func sync(inst *install.Install, force bool) {
	var err error
	if inst == nil {
		client := consulClient()
		inst, err = install.NewInstall(client, nil, nil, nil)
		if err != nil {
			log.Fatalf("Could not create install client: %v", err)
		}
	}

	defaultSources := []*install.Source{
		&install.Source{
			Name:       "mantl",
			Path:       "https://github.com/CiscoCloud/mantl-universe.git",
			SourceType: install.Git,
			Branch:     "version-0.6",
			Index:      0,
		},
	}

	sources := []*install.Source{}

	configuredSources := viper.GetStringMap("sources")

	if len(configuredSources) > 0 {
		for name, val := range configuredSources {
			source := &install.Source{Name: name, SourceType: install.FileSystem}
			sourceConfig := val.(map[string]interface{})

			if path, ok := sourceConfig["path"].(string); ok {
				source.Path = path
			}

			if index, ok := sourceConfig["index"].(int64); ok {
				source.Index = int(index)
			}

			if sourceType, ok := sourceConfig["type"].(string); ok {
				if strings.EqualFold(sourceType, "git") {
					source.SourceType = install.Git
				}
			}

			if branch, ok := sourceConfig["branch"].(string); ok {
				source.Branch = branch
			}

			if source.IsValid() {
				sources = append(sources, source)
			} else {
				log.Warnf("Invalid source configuration for %s", name)
			}
		}
	}

	if len(sources) == 0 {
		sources = defaultSources
	}

	inst.SyncSources(sources, force)
}

func readConfigFile() {
	// read configuration file if specified
	configFile := viper.GetString("config-file")
	if configFile != "" {
		configFile = os.ExpandEnv(configFile)
		if _, err := os.Stat(configFile); err == nil {
			viper.SetConfigFile(configFile)
			err = viper.ReadInConfig()
			if err != nil {
				log.Warnf("Could not read configuration file: %v", err)
			}
		} else {
			log.Warnf("Could not find configuration file: %s", configFile)
		}
	}
}

func setupLogging() {
	switch viper.GetString("log-level") {
	case "debug":
		log.SetLevel(log.DebugLevel)
	case "info":
		log.SetLevel(log.InfoLevel)
	case "warn":
		log.SetLevel(log.WarnLevel)
	case "error":
		log.SetLevel(log.ErrorLevel)
	case "fatal":
		log.SetLevel(log.FatalLevel)
	default:
		log.WithField("log-level", viper.GetString("log-level")).Warning("invalid log level. defaulting to info.")
		log.SetLevel(log.InfoLevel)
	}

	switch viper.GetString("log-format") {
	case "text":
		log.SetFormatter(new(log.TextFormatter))
	case "json":
		log.SetFormatter(new(log.JSONFormatter))
	default:
		log.WithField("log-format", viper.GetString("log-format")).Warning("invalid log format. defaulting to text.")
		log.SetFormatter(new(log.TextFormatter))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"

	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/endpoints"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/aws/aws-sdk-go/service/elb"
)

func main() {
	awsRegion := endpoints.UsEast1RegionID
	tagName := "KubernetesCluster"
	tagValue := "myCluster"

	sess := session.Must(session.NewSession(&aws.Config{
		Region: aws.String(awsRegion),
	}))

	// get load balancer
	elbClient := elb.New(sess)

	loadBalancers, err := elbClient.DescribeLoadBalancers(nil)

	if err != nil {
		log.Fatalf("describeLoadBalancers %v", err)
	}

	elbNames := make([]*string, 0)

	for _, elbDesc := range loadBalancers.LoadBalancerDescriptions {
		elbNames = append(elbNames, elbDesc.LoadBalancerName)
	}

	// get tags
	loadBalancerTags, err := elbClient.DescribeTags(&elb.DescribeTagsInput{
		LoadBalancerNames: elbNames,
	})

	if err != nil {
		log.Fatalf("describeTags %v", err)
	}

	// filter to only names that belong to the cluster
	elbNames = make([]*string, 0)
	fmt.Println("In Cluster:")

	for _, elbTags := range loadBalancerTags.TagDescriptions {
		inCluster := false

		for _, kvp := range elbTags.Tags {
			if *kvp.Key == tagName && *kvp.Value == tagValue {
				inCluster = true
				break
			}
		}

		if inCluster {
			fmt.Printf("%v\n", *elbTags.LoadBalancerName)
			elbNames = append(elbNames, elbTags.LoadBalancerName)
		}
	}

	// query metrics
	cwClient := cloudwatch.New(sess)

	now := time.Now()
	then := now.Add(-60 * time.Minute)
	metricName := "RequestCount"
	period := int64(60 * 60)
	statistic := "Sum"
	namespace := "AWS/ELB"
	dimension := "LoadBalancerName"

	for _, elbName := range elbNames {
		log.Printf("Getting stats for %v", *elbName)

		metricStats, err := cwClient.GetMetricStatistics(&cloudwatch.GetMetricStatisticsInput{
			Dimensions: []*cloudwatch.Dimension{&cloudwatch.Dimension{
				Name:  &dimension,
				Value: elbName,
			}},
			StartTime:          &then,
			EndTime:            &now,
			ExtendedStatistics: nil,
			MetricName:         &metricName,
			Namespace:          &namespace,
			Period:             &period,
			Statistics:         []*string{&statistic},
			Unit:               nil,
		})

		if err != nil {
			log.Fatalf("getMetricStatistics %v", err)
		}

		fmt.Printf("metricStats %v", *metricStats)
	}
}
<commit_msg>Respect 20 elb limit for DescribeTags<commit_after>package main

import (
	"fmt"
	"log"

	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/endpoints"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/aws/aws-sdk-go/service/elb"
)

func main() {
	awsRegion := endpoints.UsEast1RegionID
	tagName := "KubernetesCluster"
	tagValue := "myCluster"

	sess := session.Must(session.NewSession(&aws.Config{
		Region: aws.String(awsRegion),
	}))

	// get load balancer
	elbClient := elb.New(sess)

	loadBalancers, err := elbClient.DescribeLoadBalancers(nil)

	if err != nil {
		log.Fatalf("describeLoadBalancers %v", err)
	}

	elbNames := make([]*string, 0)
	elbNamesInCluster := make([]*string, 0)

	for _, elbDesc := range loadBalancers.LoadBalancerDescriptions {
		elbNames = append(elbNames, elbDesc.LoadBalancerName)
	}

	for i := 0; i < (len(elbNames)/20)+1; i++ {

		startSlice := i * 20
		endSlice := (i + 1) * 20

		if endSlice > len(elbNames) {
			endSlice = len(elbNames)
		}

		// get tags
		loadBalancerTags, err := elbClient.DescribeTags(&elb.DescribeTagsInput{
			LoadBalancerNames: elbNames[startSlice:endSlice],
		})

		if err != nil {
			log.Fatalf("describeTags %v", err)
		}

		// filter to only names that belong to the cluster
		fmt.Println("In Cluster:")

		for _, elbTags := range loadBalancerTags.TagDescriptions {
			inCluster := false

			for _, kvp := range elbTags.Tags {
				if *kvp.Key == tagName && *kvp.Value == tagValue {
					inCluster = true
					break
				}
			}

			if inCluster {
				fmt.Printf("%v\n", *elbTags.LoadBalancerName)
				elbNamesInCluster = append(elbNamesInCluster, elbTags.LoadBalancerName)
			}
		}
	}

	fmt.Printf("Found %d load balancers\n", len(elbNamesInCluster))

	// query metrics
	cwClient := cloudwatch.New(sess)

	now := time.Now()
	then := now.Add(-60 * time.Second)
	metricName := "RequestCount"
	period := int64(60)
	statistic := "Sum"
	namespace := "AWS/ELB"
	dimension := "LoadBalancerName"

	for _, elbName := range elbNamesInCluster {
		log.Printf("Getting stats for %v", *elbName)

		metricStats, err := cwClient.GetMetricStatistics(&cloudwatch.GetMetricStatisticsInput{
			Dimensions: []*cloudwatch.Dimension{&cloudwatch.Dimension{
				Name:  &dimension,
				Value: elbName,
			}},
			StartTime:          &then,
			EndTime:            &now,
			ExtendedStatistics: nil,
			MetricName:         &metricName,
			Namespace:          &namespace,
			Period:             &period,
			Statistics:         []*string{&statistic},
			Unit:               nil,
		})

		if err != nil {
			log.Fatalf("getMetricStatistics %v", err)
		}

		fmt.Printf("metricStats %v", *metricStats)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"runtime"
	"time"
)

var dbRoot = flag.String("root", "db", "Root directory for database files.")
var flushTime = flag.Duration("flushDelay", time.Second*5,
	"Maximum amount of time to wait before flushing")
var maxOpQueue = flag.Int("maxOpQueue", 1000,
	"Maximum number of queued items before flushing")
var staticPath = flag.String("static", "static", "Path to static data")
var queryTimeout = flag.Duration("maxQueryTime", time.Minute*5,
	"Maximum amount of time a query is allowed to process.")
var queryBacklog = flag.Int("queryBacklog", 0, "Query scan/group backlog size")
var docBacklog = flag.Int("docBacklog", 0, "MR group request backlog size")
var cacheAddr = flag.String("memcache", "", "Memcached server to connect to")
var cacheBacklog = flag.Int("cacheBacklog", 1000, "Cache backlog size")
var cacheWorkers = flag.Int("cacheWorkers", 4, "Number of cache workers")

type routeHandler func(parts []string, w http.ResponseWriter, req *http.Request)

type routingEntry struct {
	Method   string
	Path     *regexp.Regexp
	Handler  routeHandler
	Deadline time.Duration
}

const dbMatch = "[-%+()$_a-zA-Z0-9]+"

var defaultDeadline = time.Millisecond * 50

var routingTable []routingEntry = []routingEntry{
	routingEntry{"GET", regexp.MustCompile("^/$"),
		serverInfo, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/_static/(.*)"),
		staticHandler, defaultDeadline},
	// Database stuff
	routingEntry{"GET", regexp.MustCompile("^/_all_dbs$"),
		listDatabases, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/_(.*)"),
		reservedHandler, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		dbInfo, defaultDeadline},
	routingEntry{"HEAD", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		checkDB, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/_changes$"),
		dbChanges, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/_query$"),
		query, *queryTimeout},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/_all"),
		allDocs, *queryTimeout},
	routingEntry{"POST", regexp.MustCompile("^/(" + dbMatch + ")/_compact"),
		compact, time.Second * 30},
	routingEntry{"PUT", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		createDB, defaultDeadline},
	routingEntry{"DELETE", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		deleteDB, defaultDeadline},
	routingEntry{"POST", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		newDocument, defaultDeadline},
	// Document stuff
	routingEntry{"PUT", regexp.MustCompile("^/(" + dbMatch + ")/([^/]+)$"),
		putDocument, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/([^/]+)$"),
		getDocument, defaultDeadline},
	routingEntry{"DELETE", regexp.MustCompile("^/(" + dbMatch + ")/([^/]+)$"),
		rmDocument, defaultDeadline},
}

func mustEncode(status int, w http.ResponseWriter, ob interface{}) {
	b, err := json.Marshal(ob)
	if err != nil {
		log.Fatalf("Error encoding %v.", ob)
	}
	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(b)))
	w.WriteHeader(status)
	w.Write(b)
}

func emitError(status int, w http.ResponseWriter, e, reason string) {
	m := map[string]string{"error": e, "reason": reason}
	mustEncode(status, w, m)
}

func staticHandler(parts []string, w http.ResponseWriter, req *http.Request) {
	w.Header().Del("Content-type")
	http.StripPrefix("/_static/",
		http.FileServer(http.Dir(*staticPath))).ServeHTTP(w, req)
}

func reservedHandler(parts []string, w http.ResponseWriter, req *http.Request) {
	emitError(400,
		w, "illegal_database_name",
		"Only lowercase characters (a-z), digits (0-9), "+
			"and any of the characters _, $, (, ), +, -, and / are allowed. "+
			"Must begin with a letter.")

}

func defaultHandler(parts []string, w http.ResponseWriter, req *http.Request) {
	emitError(400,

		w, "no_handler",
		fmt.Sprintf("Can't handle %v to %v\n", req.Method, req.URL.Path))

}

func findHandler(method, path string) (routingEntry, []string) {
	for _, r := range routingTable {
		if r.Method == method {
			matches := r.Path.FindAllStringSubmatch(path, 1)
			if len(matches) > 0 {
				return r, matches[0][1:]
			}
		}
	}
	return routingEntry{"DEFAULT", nil, defaultHandler, defaultDeadline},
		[]string{}
}

func handler(w http.ResponseWriter, req *http.Request) {
	defer req.Body.Close()
	start := time.Now()
	route, hparts := findHandler(req.Method, req.URL.Path)
	wd := time.AfterFunc(route.Deadline, func() {
		log.Printf("%v:%v is taking longer than %v",
			req.Method, req.URL.Path, route.Deadline)
	})

	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Content-type", "application/json")
	route.Handler(hparts, w, req)

	if !wd.Stop() {
		log.Printf("%v:%v eventually finished in %v",
			req.Method, req.URL.Path, time.Since(start))
	}
}

func main() {
	halfProcs := runtime.GOMAXPROCS(0) / 2
	if halfProcs < 1 {
		halfProcs = 1
	}
	queryWorkers := flag.Int("queryWorkers", halfProcs,
		"Number of query tree walkers.")
	docWorkers := flag.Int("docWorkers", halfProcs,
		"Number of document mapreduce workers.")

	addr := flag.String("addr", ":3133", "Address to bind to")
	flag.Parse()

	// Update the query handler deadline to the query timeout
	found := false
	for i := range routingTable {
		matches := routingTable[i].Path.FindAllStringSubmatch("/x/_query", 1)
		if len(matches) > 0 {
			routingTable[i].Deadline = *queryTimeout
			found = true
			break
		}
	}
	if !found {
		log.Fatalf("Programming error:  Could not find query handler")
	}

	processorInput = make(chan *processIn, *docBacklog)
	for i := 0; i < *docWorkers; i++ {
		go docProcessor(processorInput)
	}

	if *cacheAddr == "" {
		cacheInput = processorInput
		// Note: cacheInputSet will be null here, there should be no caching
	} else {
		cacheInput = make(chan *processIn, *cacheBacklog)
		cacheInputSet = make(chan *processOut, *cacheBacklog)
		for i := 0; i < *cacheWorkers; i++ {
			go cacheProcessor(cacheInput, cacheInputSet)
		}
	}

	queryInput = make(chan *queryIn, *queryBacklog)
	for i := 0; i < *queryWorkers; i++ {
		go queryExecutor(queryInput)
	}

	s := &http.Server{
		Addr:        *addr,
		Handler:     http.HandlerFunc(handler),
		ReadTimeout: 5 * time.Second,
	}
	log.Printf("Listening to web requests on %s", *addr)
	log.Fatal(s.ListenAndServe())
}
<commit_msg>Convenient way to dynamically profile.<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"regexp"
	"runtime"
	"runtime/pprof"
	"time"
)

var dbRoot = flag.String("root", "db", "Root directory for database files.")
var flushTime = flag.Duration("flushDelay", time.Second*5,
	"Maximum amount of time to wait before flushing")
var maxOpQueue = flag.Int("maxOpQueue", 1000,
	"Maximum number of queued items before flushing")
var staticPath = flag.String("static", "static", "Path to static data")
var queryTimeout = flag.Duration("maxQueryTime", time.Minute*5,
	"Maximum amount of time a query is allowed to process.")
var queryBacklog = flag.Int("queryBacklog", 0, "Query scan/group backlog size")
var docBacklog = flag.Int("docBacklog", 0, "MR group request backlog size")
var cacheAddr = flag.String("memcache", "", "Memcached server to connect to")
var cacheBacklog = flag.Int("cacheBacklog", 1000, "Cache backlog size")
var cacheWorkers = flag.Int("cacheWorkers", 4, "Number of cache workers")

// Profiling
var pprofFile = flag.String("proFile", "", "File to write profiling info into")
var pprofStart = flag.Duration("proStart", 5*time.Second,
	"How long after startup to start profiling")
var pprofDuration = flag.Duration("proDuration", 5*time.Minute,
	"How long to run the profiler before shutting it down")

type routeHandler func(parts []string, w http.ResponseWriter, req *http.Request)

type routingEntry struct {
	Method   string
	Path     *regexp.Regexp
	Handler  routeHandler
	Deadline time.Duration
}

const dbMatch = "[-%+()$_a-zA-Z0-9]+"

var defaultDeadline = time.Millisecond * 50

var routingTable []routingEntry = []routingEntry{
	routingEntry{"GET", regexp.MustCompile("^/$"),
		serverInfo, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/_static/(.*)"),
		staticHandler, defaultDeadline},
	// Database stuff
	routingEntry{"GET", regexp.MustCompile("^/_all_dbs$"),
		listDatabases, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/_(.*)"),
		reservedHandler, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		dbInfo, defaultDeadline},
	routingEntry{"HEAD", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		checkDB, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/_changes$"),
		dbChanges, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/_query$"),
		query, *queryTimeout},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/_all"),
		allDocs, *queryTimeout},
	routingEntry{"POST", regexp.MustCompile("^/(" + dbMatch + ")/_compact"),
		compact, time.Second * 30},
	routingEntry{"PUT", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		createDB, defaultDeadline},
	routingEntry{"DELETE", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		deleteDB, defaultDeadline},
	routingEntry{"POST", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		newDocument, defaultDeadline},
	// Document stuff
	routingEntry{"PUT", regexp.MustCompile("^/(" + dbMatch + ")/([^/]+)$"),
		putDocument, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/([^/]+)$"),
		getDocument, defaultDeadline},
	routingEntry{"DELETE", regexp.MustCompile("^/(" + dbMatch + ")/([^/]+)$"),
		rmDocument, defaultDeadline},
}

func mustEncode(status int, w http.ResponseWriter, ob interface{}) {
	b, err := json.Marshal(ob)
	if err != nil {
		log.Fatalf("Error encoding %v.", ob)
	}
	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(b)))
	w.WriteHeader(status)
	w.Write(b)
}

func emitError(status int, w http.ResponseWriter, e, reason string) {
	m := map[string]string{"error": e, "reason": reason}
	mustEncode(status, w, m)
}

func staticHandler(parts []string, w http.ResponseWriter, req *http.Request) {
	w.Header().Del("Content-type")
	http.StripPrefix("/_static/",
		http.FileServer(http.Dir(*staticPath))).ServeHTTP(w, req)
}

func reservedHandler(parts []string, w http.ResponseWriter, req *http.Request) {
	emitError(400,
		w, "illegal_database_name",
		"Only lowercase characters (a-z), digits (0-9), "+
			"and any of the characters _, $, (, ), +, -, and / are allowed. "+
			"Must begin with a letter.")

}

func defaultHandler(parts []string, w http.ResponseWriter, req *http.Request) {
	emitError(400,

		w, "no_handler",
		fmt.Sprintf("Can't handle %v to %v\n", req.Method, req.URL.Path))

}

func findHandler(method, path string) (routingEntry, []string) {
	for _, r := range routingTable {
		if r.Method == method {
			matches := r.Path.FindAllStringSubmatch(path, 1)
			if len(matches) > 0 {
				return r, matches[0][1:]
			}
		}
	}
	return routingEntry{"DEFAULT", nil, defaultHandler, defaultDeadline},
		[]string{}
}

func handler(w http.ResponseWriter, req *http.Request) {
	defer req.Body.Close()
	start := time.Now()
	route, hparts := findHandler(req.Method, req.URL.Path)
	wd := time.AfterFunc(route.Deadline, func() {
		log.Printf("%v:%v is taking longer than %v",
			req.Method, req.URL.Path, route.Deadline)
	})

	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Content-type", "application/json")
	route.Handler(hparts, w, req)

	if !wd.Stop() {
		log.Printf("%v:%v eventually finished in %v",
			req.Method, req.URL.Path, time.Since(start))
	}
}

func startProfiler() {
	time.Sleep(*pprofStart)
	log.Printf("Starting profiler")
	f, err := os.OpenFile(*pprofFile, os.O_WRONLY|os.O_CREATE, 0666)
	if err == nil {
		err = pprof.StartCPUProfile(f)
		if err != nil {
			log.Fatalf("Can't start profiler")
		}
		time.AfterFunc(*pprofDuration, func() {
			log.Printf("Shutting down profiler")
			pprof.StopCPUProfile()
			f.Close()
		})
	} else {
		log.Printf("Can't open profilefile")
	}

}

func main() {
	halfProcs := runtime.GOMAXPROCS(0) / 2
	if halfProcs < 1 {
		halfProcs = 1
	}
	queryWorkers := flag.Int("queryWorkers", halfProcs,
		"Number of query tree walkers.")
	docWorkers := flag.Int("docWorkers", halfProcs,
		"Number of document mapreduce workers.")

	addr := flag.String("addr", ":3133", "Address to bind to")
	flag.Parse()

	// Update the query handler deadline to the query timeout
	found := false
	for i := range routingTable {
		matches := routingTable[i].Path.FindAllStringSubmatch("/x/_query", 1)
		if len(matches) > 0 {
			routingTable[i].Deadline = *queryTimeout
			found = true
			break
		}
	}
	if !found {
		log.Fatalf("Programming error:  Could not find query handler")
	}

	processorInput = make(chan *processIn, *docBacklog)
	for i := 0; i < *docWorkers; i++ {
		go docProcessor(processorInput)
	}

	if *cacheAddr == "" {
		cacheInput = processorInput
		// Note: cacheInputSet will be null here, there should be no caching
	} else {
		cacheInput = make(chan *processIn, *cacheBacklog)
		cacheInputSet = make(chan *processOut, *cacheBacklog)
		for i := 0; i < *cacheWorkers; i++ {
			go cacheProcessor(cacheInput, cacheInputSet)
		}
	}

	queryInput = make(chan *queryIn, *queryBacklog)
	for i := 0; i < *queryWorkers; i++ {
		go queryExecutor(queryInput)
	}

	if *pprofFile != "" {
		go startProfiler()
	}

	s := &http.Server{
		Addr:        *addr,
		Handler:     http.HandlerFunc(handler),
		ReadTimeout: 5 * time.Second,
	}
	log.Printf("Listening to web requests on %s", *addr)
	log.Fatal(s.ListenAndServe())
}
<|endoftext|>