<commit_before>package static

import (
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"time"
)

// DirFile contains the static directory and file content info
type DirFile struct {
	Path       string
	Name       string
	Size       int64
	Mode       os.FileMode
	ModTime    int64
	IsDir      bool
	Compressed string
	Files      []*DirFile
}

// Files contains a full instance of a static file collection
type Files struct {
	dir Dir
}

// File contains the static FileInfo
type File struct {
	data         []byte
	path         string
	name         string
	size         int64
	mode         os.FileMode
	modTime      int64
	isDir        bool
	files        []*File
	lastDirIndex int
}

// Dir implements the FileSystem interface
type Dir struct {
	name             string
	isProductionMode bool
	files            map[string]*File
}

type httpFile struct {
	*bytes.Reader
	*File
}

// Config contains information about how extracting the data should behave
type Config struct {
	IsProductionMode bool
	Name             string
}

// Open returns the FileSystem DIR
func (dir Dir) Open(name string) (http.File, error) {

	if dir.isProductionMode {
		f, found := dir.files[path.Clean(name)]
		if !found {
			return nil, os.ErrNotExist
		}

		return f.File()
	}

	return os.Open(name)
}

// File returns an http.File or error
func (f File) File() (http.File, error) {

	// if production read filesystem file
	return &httpFile{
		Reader: bytes.NewReader(f.data),
		File:   &f,
	}, nil
}

// Close closes the File, rendering it unusable for I/O. It returns an error, if any.
func (f File) Close() error {
	return nil
}

// Readdir returns nil fileinfo and an error because the static FileSystem does not store directories
func (f File) Readdir(count int) ([]os.FileInfo, error) {

	if !f.IsDir() {
		return nil, errors.New("not a directory")
	}

	var files []os.FileInfo

	if count <= 0 {
		files = make([]os.FileInfo, len(f.files))
		count = len(f.files)
		f.lastDirIndex = 0
	} else {
		files = make([]os.FileInfo, count)
	}

	if f.lastDirIndex >= len(f.files) {
		return nil, io.EOF
	}

	if count+f.lastDirIndex >= len(f.files) {
		count = len(f.files)
	}

	for i := f.lastDirIndex; i < count; i++ {
		files = append(files, *f.files[i])
	}

	return files, nil
}

// Stat returns the FileInfo structure describing file. If there is an error, it will be of type *PathError.
func (f File) Stat() (os.FileInfo, error) {
	return f, nil
}

// Name returns the name of the file as presented to Open.
func (f File) Name() string {
	return f.name
}

// Size length in bytes for regular files; system-dependent for others
func (f File) Size() int64 {
	return f.size
}

// Mode returns file mode bits
func (f File) Mode() os.FileMode {
	mode := os.FileMode(0644)
	if f.IsDir() {
		return mode | os.ModeDir
	}
	return mode
}

// ModTime returns the files modification time
func (f File) ModTime() time.Time {
	return time.Unix(f.modTime, 0)
}

// IsDir reports whether f describes a directory.
func (f File) IsDir() bool {
	return f.isDir
}

// Sys returns the underlying data source (can return nil)
func (f File) Sys() interface{} {
	return f
}

// New create a new static file instance.
func New(config *Config, file *DirFile) (*Files, error) {
	files := map[string]*File{}

	if config.IsProductionMode {
		processFiles(files, file)
	}

	return &Files{
		dir: Dir{
			name:             config.Name,
			isProductionMode: config.IsProductionMode,
			files:            files,
		},
	}, nil
}

func processFiles(files map[string]*File, file *DirFile) *File {

	f := &File{
		path:    file.Path,
		name:    file.Name,
		size:    file.Size,
		mode:    file.Mode,
		modTime: file.ModTime,
		isDir:   file.IsDir,
		files:   []*File{},
	}

	files[f.path] = f

	if file.IsDir {
		for _, dirFile := range file.Files {
			resultFile := processFiles(files, dirFile)
			f.files = append(f.files, resultFile)
		}

		return f
	}

	// decompress file contents
	b64 := base64.NewDecoder(base64.StdEncoding, bytes.NewBufferString(file.Compressed))
	reader, err := gzip.NewReader(b64)
	if err != nil {
		log.Fatal(err)
	}

	f.data, err = ioutil.ReadAll(reader)
	if err != nil {
		log.Fatal(err)
	}

	return f
}

// FS returns an http.FileSystem object for serving files over http
func (s *Files) FS() http.FileSystem {
	return s.dir
}

// GetFile returns an http.File object
func (s *Files) GetFile(name string) (http.File, error) {
	return s.dir.Open(name)
}

// GetFileBytes return a files contents as []byte from the filesystem, static or local
func (s *Files) GetFileBytes(name string) ([]byte, error) {
	f, err := s.GetFile(name)
	if err != nil {
		return nil, err
	}

	return ioutil.ReadAll(f)
}

// GetFileString return a files contents as string from the filesystem, static or local
func (s *Files) GetFileString(name string) (string, error) {

	b, err := s.GetFileBytes(name)
	if err != nil {
		return "", err
	}

	return string(b), nil
}
<commit_msg>Update static file flag name<commit_after>package static

import (
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"time"
)

// DirFile contains the static directory and file content info
type DirFile struct {
	Path       string
	Name       string
	Size       int64
	Mode       os.FileMode
	ModTime    int64
	IsDir      bool
	Compressed string
	Files      []*DirFile
}

// Files contains a full instance of a static file collection
type Files struct {
	dir Dir
}

// File contains the static FileInfo
type File struct {
	data         []byte
	path         string
	name         string
	size         int64
	mode         os.FileMode
	modTime      int64
	isDir        bool
	files        []*File
	lastDirIndex int
}

// Dir implements the FileSystem interface
type Dir struct {
	useStaticFiles bool
	files          map[string]*File
}

type httpFile struct {
	*bytes.Reader
	*File
}

// Config contains information about how extracting the data should behave
type Config struct {
	useStaticFiles bool
}

// Open returns the FileSystem DIR
func (dir Dir) Open(name string) (http.File, error) {

	if dir.useStaticFiles {
		f, found := dir.files[path.Clean(name)]
		if !found {
			return nil, os.ErrNotExist
		}

		return f.File()
	}

	return os.Open(name)
}

// File returns an http.File or error
func (f File) File() (http.File, error) {

	// if production read filesystem file
	return &httpFile{
		Reader: bytes.NewReader(f.data),
		File:   &f,
	}, nil
}

// Close closes the File, rendering it unusable for I/O. It returns an error, if any.
func (f File) Close() error {
	return nil
}

// Readdir returns nil fileinfo and an error because the static FileSystem does not store directories
func (f File) Readdir(count int) ([]os.FileInfo, error) {

	if !f.IsDir() {
		return nil, errors.New("not a directory")
	}

	var files []os.FileInfo

	if count <= 0 {
		files = make([]os.FileInfo, len(f.files))
		count = len(f.files)
		f.lastDirIndex = 0
	} else {
		files = make([]os.FileInfo, count)
	}

	if f.lastDirIndex >= len(f.files) {
		return nil, io.EOF
	}

	if count+f.lastDirIndex >= len(f.files) {
		count = len(f.files)
	}

	for i := f.lastDirIndex; i < count; i++ {
		files = append(files, *f.files[i])
	}

	return files, nil
}

// Stat returns the FileInfo structure describing file. If there is an error, it will be of type *PathError.
func (f File) Stat() (os.FileInfo, error) {
	return f, nil
}

// Name returns the name of the file as presented to Open.
func (f File) Name() string {
	return f.name
}

// Size length in bytes for regular files; system-dependent for others
func (f File) Size() int64 {
	return f.size
}

// Mode returns file mode bits
func (f File) Mode() os.FileMode {
	mode := os.FileMode(0644)
	if f.IsDir() {
		return mode | os.ModeDir
	}
	return mode
}

// ModTime returns the files modification time
func (f File) ModTime() time.Time {
	return time.Unix(f.modTime, 0)
}

// IsDir reports whether f describes a directory.
func (f File) IsDir() bool {
	return f.isDir
}

// Sys returns the underlying data source (can return nil)
func (f File) Sys() interface{} {
	return f
}

// New create a new static file instance.
func New(config *Config, file *DirFile) (*Files, error) {
	files := map[string]*File{}

	if config.useStaticFiles {
		processFiles(files, file)
	}

	return &Files{
		dir: Dir{
			useStaticFiles: config.useStaticFiles,
			files:          files,
		},
	}, nil
}

func processFiles(files map[string]*File, file *DirFile) *File {

	f := &File{
		path:    file.Path,
		name:    file.Name,
		size:    file.Size,
		mode:    file.Mode,
		modTime: file.ModTime,
		isDir:   file.IsDir,
		files:   []*File{},
	}

	files[f.path] = f

	if file.IsDir {
		for _, dirFile := range file.Files {
			resultFile := processFiles(files, dirFile)
			f.files = append(f.files, resultFile)
		}

		return f
	}

	// decompress file contents
	b64 := base64.NewDecoder(base64.StdEncoding, bytes.NewBufferString(file.Compressed))
	reader, err := gzip.NewReader(b64)
	if err != nil {
		log.Fatal(err)
	}

	f.data, err = ioutil.ReadAll(reader)
	if err != nil {
		log.Fatal(err)
	}

	return f
}

// FS returns an http.FileSystem object for serving files over http
func (s *Files) FS() http.FileSystem {
	return s.dir
}

// GetFile returns an http.File object
func (s *Files) GetFile(name string) (http.File, error) {
	return s.dir.Open(name)
}

// GetFileBytes return a files contents as []byte from the filesystem, static or local
func (s *Files) GetFileBytes(name string) ([]byte, error) {
	f, err := s.GetFile(name)
	if err != nil {
		return nil, err
	}

	return ioutil.ReadAll(f)
}

// GetFileString return a files contents as string from the filesystem, static or local
func (s *Files) GetFileString(name string) (string, error) {

	b, err := s.GetFileBytes(name)
	if err != nil {
		return "", err
	}

	return string(b), nil
}
<|endoftext|>


<commit_before>// Usage: (uh, eventually, once all of this is implemented...)
// $ googlecl help  # prints help
// $ googlecl list  # lists all available APIs
// $ googlecl describe calendar                          # describes all methods available in Calendar API
// $ googlecl describe calendar calendars.get            # describes one method in Calendar API
//
// $ googlecl calendar calendars.get --calendarId=12345  # prints JSON API response
//
// $ cat someEvent.json | googlecl calendar events.insert --calendarId=12345 --in  # inserts an event
// $ googlecl calendar events.insert --calendarId=12345 --inFile=someEvent.json    # equivalent to above
//
// TODO: Handle auth somehow.

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"strings"
)

var cmds = map[string]func(){
	"help":     func() { log.Fatal("TODO: implement help command") },
	"list":     func() { log.Fatal("TODO: implement list command") },
	"describe": func() { log.Fatal("TODO: implement describe command") },
}

func parseArgs(args []string) map[string]string {
	m := make(map[string]string)
	for _, a := range args {
		if strings.HasPrefix(a, "--") {
			a = a[2:]
		} else if strings.HasPrefix(a, "-") {
			a = a[1:]
		} else {
			log.Fatalf("Invalid flag format %s", a)
		}

		if !strings.Contains(a, "=") {
			m[a] = "true"
		} else {
			parts := strings.SplitN(a, "=", 2)
			m[parts[0]] = parts[1]
		}
	}
	return m
}

func main() {
	if len(os.Args) == 1 {
		cmds["help"]()
		return
	}

	cmd := os.Args[1]
	if cmd == "" {
		log.Fatal("Must specify command or API name")
	}
	if cmdFn, found := cmds[cmd]; found {
		cmdFn()
		return
	}

	method := os.Args[2]
	if method == "" {
		log.Fatal("Must specify API method to call")
	}

	apiName := cmd
	fs := parseArgs(os.Args[3:])
	v := flagValue(fs, "v")
	if v == "" {
		// Look up preferred version in Directory
		var err error
		v, err = getPreferredVersion(apiName)
		if err != nil {
			log.Fatal(err)
		}
	}
	api, err := loadApi(apiName, v)
	if err != nil {
		log.Fatal(err)
	}
	if api == nil || (len(api.Resources) == 0 && len(api.Methods) == 0) {
		log.Fatalf("Couldn't load API %s %s", apiName, v)
	}

	m := findMethod(method, *api)
	if m == nil {
		log.Fatalf("Can't find requested method %s", method)
	}

	m.call(fs, apiName, v)
}

func findMethod(method string, api Api) *Method {
	parts := strings.Split(method, ".")
	var ms map[string]Method
	rs := api.Resources
	for i := 0; i < len(parts)-1; i++ {
		r, found := rs[parts[i]]
		if !found {
			return nil
		}
		rs = r.Resources
		ms = r.Methods
	}
	lp := parts[len(parts)-1:][0]
	m, found := ms[lp]
	if !found {
		return nil
	}
	return &m
}

func flagValue(fs map[string]string, k string) string {
	v, found := fs[k]
	if !found {
		return ""
	}
	return v
}

func getPreferredVersion(api string) (string, error) {
	var d struct {
		Items []struct {
			Version string
		}
	}
	err := getAndParse(fmt.Sprintf("https://www.googleapis.com/discovery/v1/apis?preferred=true&name=%s&fields=items/version", api), &d)
	if err != nil {
		return "", err
	}
	return d.Items[0].Version, nil
}

func loadApi(api, version string) (*Api, error) {
	var a Api
	err := getAndParse(fmt.Sprintf("https://www.googleapis.com/discovery/v1/apis/%s/%s/rest", api, version), &a)
	if err != nil {
		return nil, err
	}
	return &a, nil
}

func getAndParse(url string, v interface{}) error {
	r, err := http.Get(url)
	if err != nil {
		return err
	}
	defer r.Body.Close()
	err = json.NewDecoder(r.Body).Decode(v)
	if err != nil {
		return err
	}
	return nil
}

type Api struct {
	Resources map[string]Resource
	Methods   map[string]Method
}

type Resource struct {
	Resources map[string]Resource
	Methods   map[string]Method
}

type Method struct {
	Id, Path, HttpMethod string
	Parameters           map[string]Parameter
}

func (m Method) call(fs map[string]string, apiName, version string) {
	url := fmt.Sprintf("https://www.googleapis.com/%s/%s/%s", apiName, version, m.Path)
	for k, p := range m.Parameters {
		v := flagValue(fs, k)
		if v == "" {
			v = p.Default
		}
		if v == "" {
			continue
		}
		if p.Location == "path" {
			t := fmt.Sprintf("{%s}", k)
			if p.Required && v == "" {
				log.Printf("Missing required parameter %s", k)
			}
			url = strings.Replace(url, t, v, -1)
		} else if p.Location == "query" {
			if !strings.Contains(url, "?") {
				url += "?"
			}
			url += fmt.Sprintf("&%s=%s", k, v)
		}
	}

	var body io.Reader
	if v, found := fs["in"]; found && v == "true" {
		// If user passes the --in flag, use stdin as the request body
		body = os.Stdin
	} else if v, found := fs["inFile"]; found {
		// If user passes --inFile flag, open that file and use its content as request body
		var err error
		body, err = os.Open(v)
		if err != nil {
			log.Fatal(err)
		}
	}

	r, err := http.NewRequest(m.HttpMethod, url, body)
	if err != nil {
		log.Fatal(err)
	}

	client := &http.Client{}
	resp, err := client.Do(r)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		io.Copy(os.Stderr, resp.Body)
		os.Exit(1)
	} else {
		io.Copy(os.Stdout, resp.Body)
	}
}

type Parameter struct {
	Type, Description, Location, Default string
	Required                             bool
}
<commit_msg>Support common parameters<commit_after>// Usage: (uh, eventually, once all of this is implemented...)
// $ googlecl help  # prints help
// $ googlecl list  # lists all available APIs
// $ googlecl describe calendar                          # describes all methods available in Calendar API
// $ googlecl describe calendar calendars.get            # describes one method in Calendar API
//
// $ googlecl calendar calendars.get --calendarId=12345  # prints JSON API response
//
// $ cat someEvent.json | googlecl calendar events.insert --calendarId=12345 --in  # inserts an event
// $ googlecl calendar events.insert --calendarId=12345 --inFile=someEvent.json    # equivalent to above
//
// TODO: Handle auth somehow.

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"strings"
)

var cmds = map[string]func(){
	"help":     func() { log.Fatal("TODO: implement help command") },
	"list":     func() { log.Fatal("TODO: implement list command") },
	"describe": func() { log.Fatal("TODO: implement describe command") },
}

func parseArgs(args []string) map[string]string {
	m := make(map[string]string)
	for _, a := range args {
		if strings.HasPrefix(a, "--") {
			a = a[2:]
		} else if strings.HasPrefix(a, "-") {
			a = a[1:]
		} else {
			log.Fatalf("Invalid flag format %s", a)
		}

		if !strings.Contains(a, "=") {
			m[a] = "true"
		} else {
			parts := strings.SplitN(a, "=", 2)
			m[parts[0]] = parts[1]
		}
	}
	return m
}

func main() {
	if len(os.Args) == 1 {
		cmds["help"]()
		return
	}

	cmd := os.Args[1]
	if cmd == "" {
		log.Fatal("Must specify command or API name")
	}
	if cmdFn, found := cmds[cmd]; found {
		cmdFn()
		return
	}

	method := os.Args[2]
	if method == "" {
		log.Fatal("Must specify API method to call")
	}

	apiName := cmd
	fs := parseArgs(os.Args[3:])
	v := flagValue(fs, "v")
	if v == "" {
		// Look up preferred version in Directory
		var err error
		v, err = getPreferredVersion(apiName)
		if err != nil {
			log.Fatal(err)
		}
	}
	api, err := loadApi(apiName, v)
	if err != nil {
		log.Fatal(err)
	}
	if api == nil || (len(api.Resources) == 0 && len(api.Methods) == 0) {
		log.Fatalf("Couldn't load API %s %s", apiName, v)
	}

	m := findMethod(method, *api)
	if m == nil {
		log.Fatalf("Can't find requested method %s", method)
	}

	m.call(fs, api, apiName, v)
}

func findMethod(method string, api Api) *Method {
	parts := strings.Split(method, ".")
	var ms map[string]Method
	rs := api.Resources
	for i := 0; i < len(parts)-1; i++ {
		r, found := rs[parts[i]]
		if !found {
			return nil
		}
		rs = r.Resources
		ms = r.Methods
	}
	lp := parts[len(parts)-1:][0]
	m, found := ms[lp]
	if !found {
		return nil
	}
	return &m
}

func flagValue(fs map[string]string, k string) string {
	v, found := fs[k]
	if !found {
		return ""
	}
	return v
}

func getPreferredVersion(api string) (string, error) {
	var d struct {
		Items []struct {
			Version string
		}
	}
	err := getAndParse(fmt.Sprintf("https://www.googleapis.com/discovery/v1/apis?preferred=true&name=%s&fields=items/version", api), &d)
	if err != nil {
		return "", err
	}
	return d.Items[0].Version, nil
}

func loadApi(api, version string) (*Api, error) {
	var a Api
	err := getAndParse(fmt.Sprintf("https://www.googleapis.com/discovery/v1/apis/%s/%s/rest", api, version), &a)
	if err != nil {
		return nil, err
	}
	return &a, nil
}

func getAndParse(url string, v interface{}) error {
	r, err := http.Get(url)
	if err != nil {
		return err
	}
	defer r.Body.Close()
	err = json.NewDecoder(r.Body).Decode(v)
	if err != nil {
		return err
	}
	return nil
}

type Api struct {
	Resources  map[string]Resource
	Methods    map[string]Method
	Parameters map[string]Parameter
}

type Resource struct {
	Resources map[string]Resource
	Methods   map[string]Method
}

type Method struct {
	Id, Path, HttpMethod string
	Parameters           map[string]Parameter
}

func (m Method) call(fs map[string]string, api *Api, apiName, version string) {
	url := fmt.Sprintf("https://www.googleapis.com/%s/%s/%s", apiName, version, m.Path)
	for k, p := range m.Parameters {
		url = p.process(k, fs, url)
	}
	for k, p := range api.Parameters {
		url = p.process(k, fs, url)
	}

	var body io.Reader
	if v, found := fs["in"]; found && v == "true" {
		// If user passes the --in flag, use stdin as the request body
		body = os.Stdin
	} else if v, found := fs["inFile"]; found {
		// If user passes --inFile flag, open that file and use its content as request body
		var err error
		body, err = os.Open(v)
		if err != nil {
			log.Fatal(err)
		}
	}

	r, err := http.NewRequest(m.HttpMethod, url, body)
	if err != nil {
		log.Fatal(err)
	}

	client := &http.Client{}
	resp, err := client.Do(r)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		io.Copy(os.Stderr, resp.Body)
		os.Exit(1)
	} else {
		io.Copy(os.Stdout, resp.Body)
	}
}

type Parameter struct {
	Type, Description, Location, Default string
	Required                             bool
}

func (p Parameter) process(k string, fs map[string]string, url string) string {
	v := flagValue(fs, k)
	if v == "" {
		v = p.Default
	}
	if v == "" {
		return url
	}
	if p.Location == "path" {
		t := fmt.Sprintf("{%s}", k)
		if p.Required && v == "" {
			log.Printf("Missing required parameter %s", k)
		}
		return strings.Replace(url, t, v, -1)
	} else if p.Location == "query" {
		if !strings.Contains(url, "?") {
			url += "?"
		}
		return url + fmt.Sprintf("&%s=%s", k, v)
	}
	return url
}
<|endoftext|>


<commit_before>/*

Package graceful simplifies graceful shutdown of HTTP servers (Go 1.8+)

Installation

Just go get the package:

    go get -u github.com/TV4/graceful

Usage

A small usage example

    package main

    import (
        "context"
        "log"
        "net/http"
        "os"
        "time"

        "github.com/TV4/graceful"
    )

    type server struct {
        logger *log.Logger
    }

    func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
        time.Sleep(5 * time.Second)
        w.Write([]byte("Hello!"))
    }

    func (s *server) Shutdown(ctx context.Context) error {
        time.Sleep(2 * time.Second)
        s.logger.Println("Shutdown finished")
        return nil
    }

    func main() {
        graceful.LogListenAndServe(setup(":2017"))
    }

    func setup(addr string) (*http.Server, *log.Logger) {
        s := &server{logger: log.New(os.Stdout, "", 0)}
        return &http.Server{Addr: addr, Handler: s}, s.logger
    }

*/
package graceful

import (
	"context"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"
)

// Server is implemented by *http.Server
type Server interface {
	ListenAndServe() error
	Shutdowner
}

// TLSServer is implemented by *http.Server
type TLSServer interface {
	ListenAndServeTLS(string, string) error
	Shutdowner
}

// Shutdowner is implemented by *http.Server, and optionally by *http.Server.Handler
type Shutdowner interface {
	Shutdown(ctx context.Context) error
}

// Logger is implemented by *log.Logger
type Logger interface {
	Printf(format string, v ...interface{})
	Fatal(...interface{})
}

// logger is the logger used by the shutdown function
// (defaults to logging to ioutil.Discard)
var logger Logger = log.New(ioutil.Discard, "", 0)

// signals is the channel used to signal shutdown
var signals chan os.Signal

// Timeout for context used in call to *http.Server.Shutdown
var Timeout = 15 * time.Second

// Format strings used by the logger
var (
	ListeningFormat       = "Listening on http://0.0.0.0%s\n"
	ShutdownFormat        = "\nServer shutdown with timeout: %s\n"
	ErrorFormat           = "Error: %v\n"
	FinishedFormat        = "Shutdown finished %ds before deadline\n"
	FinishedHTTP          = "Finished all in-flight HTTP requests\n"
	HandlerShutdownFormat = "Shutting down handler with timeout: %ds\n"
)

// LogListenAndServe logs using the logger and then calls ListenAndServe
func LogListenAndServe(s Server, loggers ...Logger) {
	if hs, ok := s.(*http.Server); ok {
		logger := getLogger(loggers...)
		logger.Printf(ListeningFormat, hs.Addr)
	}

	ListenAndServe(s)
}

// ListenAndServe starts the server in a goroutine and then calls Shutdown
func ListenAndServe(s Server) {
	go func() {
		if err := s.ListenAndServe(); err != http.ErrServerClosed {
			logger.Fatal(err)
		}
	}()

	Shutdown(s)
}

// ListenAndServeTLS starts the server in a goroutine and then calls Shutdown
func ListenAndServeTLS(s TLSServer, certFile, keyFile string) {
	go func() {
		if err := s.ListenAndServeTLS(certFile, keyFile); err != http.ErrServerClosed {
			logger.Fatal(err)
		}
	}()

	Shutdown(s)
}

// Shutdown blocks until os.Interrupt or syscall.SIGTERM received, then
// running *http.Server.Shutdown with a context having a timeout
func Shutdown(s Shutdowner) {
	signals = make(chan os.Signal, 1)

	signal.Notify(signals, os.Interrupt, syscall.SIGTERM)

	<-signals

	shutdown(s, logger)
}

func shutdown(s Shutdowner, logger Logger) {
	if s == nil {
		return
	}

	if logger == nil {
		logger = log.New(ioutil.Discard, "", 0)
	}

	ctx, cancel := context.WithTimeout(context.Background(), Timeout)
	defer cancel()

	logger.Printf(ShutdownFormat, Timeout)

	if err := s.Shutdown(ctx); err != nil {
		logger.Printf(ErrorFormat, err)
	} else {
		if hs, ok := s.(*http.Server); ok {
			logger.Printf(FinishedHTTP)

			if hss, ok := hs.Handler.(Shutdowner); ok {
				select {
				case <-ctx.Done():
					if err := ctx.Err(); err != nil {
						logger.Printf(ErrorFormat, err)
						return
					}
				default:
					if deadline, ok := ctx.Deadline(); ok {
						secs := (time.Until(deadline) + time.Second/2) / time.Second
						logger.Printf(HandlerShutdownFormat, secs)
					}

					done := make(chan error)

					go func() {
						<-ctx.Done()
						done <- ctx.Err()
					}()

					go func() {
						done <- hss.Shutdown(ctx)
					}()

					if err := <-done; err != nil {
						logger.Printf(ErrorFormat, err)
						return
					}
				}
			}
		}

		if deadline, ok := ctx.Deadline(); ok {
			secs := (time.Until(deadline) + time.Second/2) / time.Second
			logger.Printf(FinishedFormat, secs)
		}
	}
}

func getLogger(loggers ...Logger) Logger {
	if len(loggers) > 0 {
		if logger = loggers[0]; logger != nil {
			return logger
		}

		return log.New(ioutil.Discard, "", 0)
	}

	return log.New(os.Stdout, "", 0)
}
<commit_msg>Set the logger globally in LogListenAndServe func<commit_after>/*

Package graceful simplifies graceful shutdown of HTTP servers (Go 1.8+)

Installation

Just go get the package:

    go get -u github.com/TV4/graceful

Usage

A small usage example

    package main

    import (
        "context"
        "log"
        "net/http"
        "os"
        "time"

        "github.com/TV4/graceful"
    )

    type server struct {
        logger *log.Logger
    }

    func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
        time.Sleep(5 * time.Second)
        w.Write([]byte("Hello!"))
    }

    func (s *server) Shutdown(ctx context.Context) error {
        time.Sleep(2 * time.Second)
        s.logger.Println("Shutdown finished")
        return nil
    }

    func main() {
        graceful.LogListenAndServe(setup(":2017"))
    }

    func setup(addr string) (*http.Server, *log.Logger) {
        s := &server{logger: log.New(os.Stdout, "", 0)}
        return &http.Server{Addr: addr, Handler: s}, s.logger
    }

*/
package graceful

import (
	"context"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"
)

// Server is implemented by *http.Server
type Server interface {
	ListenAndServe() error
	Shutdowner
}

// TLSServer is implemented by *http.Server
type TLSServer interface {
	ListenAndServeTLS(string, string) error
	Shutdowner
}

// Shutdowner is implemented by *http.Server, and optionally by *http.Server.Handler
type Shutdowner interface {
	Shutdown(ctx context.Context) error
}

// Logger is implemented by *log.Logger
type Logger interface {
	Printf(format string, v ...interface{})
	Fatal(...interface{})
}

// logger is the logger used by the shutdown function
// (defaults to logging to ioutil.Discard)
var logger Logger = log.New(ioutil.Discard, "", 0)

// signals is the channel used to signal shutdown
var signals chan os.Signal

// Timeout for context used in call to *http.Server.Shutdown
var Timeout = 15 * time.Second

// Format strings used by the logger
var (
	ListeningFormat       = "Listening on http://0.0.0.0%s\n"
	ShutdownFormat        = "\nServer shutdown with timeout: %s\n"
	ErrorFormat           = "Error: %v\n"
	FinishedFormat        = "Shutdown finished %ds before deadline\n"
	FinishedHTTP          = "Finished all in-flight HTTP requests\n"
	HandlerShutdownFormat = "Shutting down handler with timeout: %ds\n"
)

// LogListenAndServe logs using the logger and then calls ListenAndServe
func LogListenAndServe(s Server, loggers ...Logger) {
	if hs, ok := s.(*http.Server); ok {
		logger = getLogger(loggers...)
		logger.Printf(ListeningFormat, hs.Addr)
	}

	ListenAndServe(s)
}

// ListenAndServe starts the server in a goroutine and then calls Shutdown
func ListenAndServe(s Server) {
	go func() {
		if err := s.ListenAndServe(); err != http.ErrServerClosed {
			logger.Fatal(err)
		}
	}()

	Shutdown(s)
}

// ListenAndServeTLS starts the server in a goroutine and then calls Shutdown
func ListenAndServeTLS(s TLSServer, certFile, keyFile string) {
	go func() {
		if err := s.ListenAndServeTLS(certFile, keyFile); err != http.ErrServerClosed {
			logger.Fatal(err)
		}
	}()

	Shutdown(s)
}

// Shutdown blocks until os.Interrupt or syscall.SIGTERM received, then
// running *http.Server.Shutdown with a context having a timeout
func Shutdown(s Shutdowner) {
	signals = make(chan os.Signal, 1)

	signal.Notify(signals, os.Interrupt, syscall.SIGTERM)

	<-signals

	shutdown(s, logger)
}

func shutdown(s Shutdowner, logger Logger) {
	if s == nil {
		return
	}

	if logger == nil {
		logger = log.New(ioutil.Discard, "", 0)
	}

	ctx, cancel := context.WithTimeout(context.Background(), Timeout)
	defer cancel()

	logger.Printf(ShutdownFormat, Timeout)

	if err := s.Shutdown(ctx); err != nil {
		logger.Printf(ErrorFormat, err)
	} else {
		if hs, ok := s.(*http.Server); ok {
			logger.Printf(FinishedHTTP)

			if hss, ok := hs.Handler.(Shutdowner); ok {
				select {
				case <-ctx.Done():
					if err := ctx.Err(); err != nil {
						logger.Printf(ErrorFormat, err)
						return
					}
				default:
					if deadline, ok := ctx.Deadline(); ok {
						secs := (time.Until(deadline) + time.Second/2) / time.Second
						logger.Printf(HandlerShutdownFormat, secs)
					}

					done := make(chan error)

					go func() {
						<-ctx.Done()
						done <- ctx.Err()
					}()

					go func() {
						done <- hss.Shutdown(ctx)
					}()

					if err := <-done; err != nil {
						logger.Printf(ErrorFormat, err)
						return
					}
				}
			}
		}

		if deadline, ok := ctx.Deadline(); ok {
			secs := (time.Until(deadline) + time.Second/2) / time.Second
			logger.Printf(FinishedFormat, secs)
		}
	}
}

func getLogger(loggers ...Logger) Logger {
	if len(loggers) > 0 {
		if logger = loggers[0]; logger != nil {
			return logger
		}

		return log.New(ioutil.Discard, "", 0)
	}

	return log.New(os.Stdout, "", 0)
}
<|endoftext|>


<commit_before>package pixel

// Sprite is a drawable Picture. It's always anchored by the center of it's Picture.
type Sprite struct {
	tri    *TrianglesData
	bounds Rect
	d      Drawer
}

// NewSprite creates a Sprite from the supplied Picture.
func NewSprite(pic Picture) *Sprite {
	tri := MakeTrianglesData(6)
	s := &Sprite{
		tri: tri,
		d:   Drawer{Triangles: tri},
	}
	s.SetPicture(pic)
	return s
}

// SetPicture changes the Sprite's Picture. The new Picture may have a different size, everything
// works.
func (s *Sprite) SetPicture(pic Picture) {
	s.d.Picture = pic

	if s.bounds == pic.Bounds() {
		return
	}
	s.bounds = pic.Bounds()

	var (
		center     = s.bounds.Center()
		horizontal = V(s.bounds.W()/2, 0)
		vertical   = V(0, s.bounds.H()/2)
	)

	(*s.tri)[0].Position = -horizontal - vertical
	(*s.tri)[1].Position = +horizontal - vertical
	(*s.tri)[2].Position = +horizontal + vertical
	(*s.tri)[3].Position = -horizontal - vertical
	(*s.tri)[4].Position = +horizontal + vertical
	(*s.tri)[5].Position = -horizontal + vertical

	for i := range *s.tri {
		(*s.tri)[i].Color = NRGBA{1, 1, 1, 1}
		(*s.tri)[i].Picture = center + (*s.tri)[i].Position
		(*s.tri)[i].Intensity = 1
	}

	s.d.Dirty()
}

// Picture returns the current Sprite's Picture.
func (s *Sprite) Picture() Picture {
	return s.d.Picture
}

// Draw draws the Sprite onto the provided Target.
func (s *Sprite) Draw(t Target) {
	s.d.Draw(t)
}
<commit_msg>add IM with several methods<commit_after>package pixel

import (
	"image/color"
	"math"
)

// Sprite is a drawable Picture. It's always anchored by the center of it's Picture.
type Sprite struct {
	tri    *TrianglesData
	bounds Rect
	d      Drawer
}

// NewSprite creates a Sprite from the supplied Picture.
func NewSprite(pic Picture) *Sprite {
	tri := MakeTrianglesData(6)
	s := &Sprite{
		tri: tri,
		d:   Drawer{Triangles: tri},
	}
	s.SetPicture(pic)
	return s
}

// SetPicture changes the Sprite's Picture. The new Picture may have a different size, everything
// works.
func (s *Sprite) SetPicture(pic Picture) {
	s.d.Picture = pic

	if s.bounds == pic.Bounds() {
		return
	}
	s.bounds = pic.Bounds()

	var (
		center     = s.bounds.Center()
		horizontal = V(s.bounds.W()/2, 0)
		vertical   = V(0, s.bounds.H()/2)
	)

	(*s.tri)[0].Position = -horizontal - vertical
	(*s.tri)[1].Position = +horizontal - vertical
	(*s.tri)[2].Position = +horizontal + vertical
	(*s.tri)[3].Position = -horizontal - vertical
	(*s.tri)[4].Position = +horizontal + vertical
	(*s.tri)[5].Position = -horizontal + vertical

	for i := range *s.tri {
		(*s.tri)[i].Color = NRGBA{1, 1, 1, 1}
		(*s.tri)[i].Picture = center + (*s.tri)[i].Position
		(*s.tri)[i].Intensity = 1
	}

	s.d.Dirty()
}

// Picture returns the current Sprite's Picture.
func (s *Sprite) Picture() Picture {
	return s.d.Picture
}

// Draw draws the Sprite onto the provided Target.
func (s *Sprite) Draw(t Target) {
	s.d.Draw(t)
}

// IM is an immediate-like-mode shape drawer.
//
// TODO: mode doc
type IM struct {
	points []point
	opts   point
	matrix Matrix
	mask   NRGBA
	tri    *TrianglesData
	d      Drawer
	tmp    []Vec
}

type point struct {
	position  Vec
	color     NRGBA
	picture   Vec
	intensity float64
	width     float64
	precision int
	endshape  EndShape
}

// EndShape specifies the shape of an end of a line or a curve.
type EndShape int

const (
	// RoundEndShape is a circular end shape.
	RoundEndShape EndShape = iota

	// SharpEndShape is a square end shape.
	SharpEndShape
)

// NewIM creates a new empty IM. An optional Picture can be used to draw with a Picture.
//
// If you just want to draw primitive shapes, pass nil as the Picture.
func NewIM(pic Picture) *IM {
	tri := &TrianglesData{}
	im := &IM{
		tri: tri,
		d:   Drawer{Triangles: tri, Picture: pic},
	}
	im.Precision(64)
	im.SetMatrix(ZM)
	im.SetColorMask(NRGBA{1, 1, 1, 1})
	return im
}

// Clear removes all drawn shapes from the IM. This does not remove Pushed points.
func (im *IM) Clear() {
	im.tri.SetLen(0)
	im.d.Dirty()
}

// Draw draws all currently drawn shapes inside the IM onto another Target.
func (im *IM) Draw(t Target) {
	im.d.Draw(t)
}

// Push adds some points to the IM queue. All Pushed points will have the same properties except for
// the position.
func (im *IM) Push(pts ...Vec) {
	point := im.opts
	for _, pt := range pts {
		point.position = im.matrix.Project(pt)
		point.color = im.mask.Mul(im.opts.color)
		im.points = append(im.points, point)
	}
}

// Color sets the color of the next Pushed points.
func (im *IM) Color(color color.Color) {
	im.opts.color = NRGBAModel.Convert(color).(NRGBA)
}

// Picture sets the Picture coordinates of the next Pushed points.
func (im *IM) Picture(pic Vec) {
	im.opts.picture = pic
}

// Intensity sets the picture Intensity of the next Pushed points.
func (im *IM) Intensity(in float64) {
	im.opts.intensity = in
}

// Width sets the with property of the next Pushed points.
//
// Note that this property does not apply to filled shapes.
func (im *IM) Width(w float64) {
	im.opts.width = w
}

// Precision sets the curve/circle drawing precision of the next Pushed points.
//
// It is the number of segments per 360 degrees.
func (im *IM) Precision(p int) {
	im.opts.precision = p
	if p+1 > len(im.tmp) {
		im.tmp = append(im.tmp, make([]Vec, p+1-len(im.tmp))...)
	}
	if p+1 < len(im.tmp) {
		im.tmp = im.tmp[:p+1]
	}
}

// EndShape sets the endshape of the next Pushed points.
func (im *IM) EndShape(es EndShape) {
	im.opts.endshape = es
}

// SetMatrix sets a Matrix that all further points will be transformed by.
func (im *IM) SetMatrix(m Matrix) {
	im.matrix = m
}

// SetColorMask sets a color that all futher point's color will be multiplied by.
func (im *IM) SetColorMask(color color.Color) {
	im.mask = NRGBAModel.Convert(color).(NRGBA)
}

// FillConvexPolygon takes all points Pushed into the IM's queue and fills the convex polygon formed
// by them.
//
// It empties the queue after.
func (im *IM) FillConvexPolygon() {
	points := im.points
	im.points = nil

	if len(points) < 3 {
		return
	}

	i := im.tri.Len()
	im.tri.SetLen(im.tri.Len() + 3*(len(points)-2))

	for j := 1; j+1 < len(points); j++ {
		(*im.tri)[i].Position = points[0].position
		(*im.tri)[i].Color = points[0].color
		(*im.tri)[i].Picture = points[0].picture
		(*im.tri)[i].Intensity = points[0].intensity

		(*im.tri)[i+1].Position = points[j].position
		(*im.tri)[i+1].Color = points[j].color
		(*im.tri)[i+1].Picture = points[j].picture
		(*im.tri)[i+1].Intensity = points[j].intensity

		(*im.tri)[i+2].Position = points[j+1].position
		(*im.tri)[i+2].Color = points[j+1].color
		(*im.tri)[i+2].Picture = points[j+1].picture
		(*im.tri)[i+2].Intensity = points[j+1].intensity

		i += 3
	}

	im.d.Dirty()
}

// FillEllipseArc draws an ellipse arc around each point in the IM's queue. Low and high angles are
// in radians.
//
// It empties the queue after.
func (im *IM) FillEllipseArc(radius Vec, low, high float64) {
	points := im.points
	im.points = nil

	// normalize high
	if math.Abs(high-low) > 2*math.Pi {
		high = low + math.Mod(high-low, 2*math.Pi)
	}

	for _, pt := range points {
		im.Push(pt.position) // center

		num := math.Ceil(math.Abs(high-low) / (2 * math.Pi) * float64(pt.precision))
		delta := (high - low) / num
		for i := range im.tmp[:int(num)+1] {
			angle := low + float64(i)*delta
			sin, cos := math.Sincos(angle)
			im.tmp[i] = pt.position + V(
				radius.X()*cos,
				radius.Y()*sin,
			)
		}

		im.Push(im.tmp[:int(num)+1]...)
		im.FillConvexPolygon()
	}
}
<|endoftext|>


<commit_before>package csi

import (
	"context"
	"errors"
	"fmt"

	csipbv1 "github.com/container-storage-interface/spec/lib/go/csi"
	"github.com/hashicorp/nomad/plugins/base"
)

// CSIPlugin implements a lightweight abstraction layer around a CSI Plugin.
// It validates that responses from storage providers (SP's), correctly conform
// to the specification before returning response data or erroring.
type CSIPlugin interface {
	base.BasePlugin

	// PluginProbe is used to verify that the plugin is in a healthy state
	PluginProbe(ctx context.Context) (bool, error)

	// PluginGetInfo is used to return semantic data about the plugin.
	// Response:
	//  - string: name, the name of the plugin in domain notation format.
	PluginGetInfo(ctx context.Context) (string, error)

	// PluginGetCapabilities is used to return the available capabilities from the
	// identity service. This currently only looks for the CONTROLLER_SERVICE and
	// Accessible Topology Support
	PluginGetCapabilities(ctx context.Context) (*PluginCapabilitySet, error)

	// GetControllerCapabilities is used to get controller-specific capabilities
	// for a plugin.
	ControllerGetCapabilities(ctx context.Context) (*ControllerCapabilitySet, error)

	// ControllerPublishVolume is used to attach a remote volume to a cluster node.
	ControllerPublishVolume(ctx context.Context, req *ControllerPublishVolumeRequest) (*ControllerPublishVolumeResponse, error)

	// NodeGetCapabilities is used to return the available capabilities from the
	// Node Service.
	NodeGetCapabilities(ctx context.Context) (*NodeCapabilitySet, error)

	// NodeGetInfo is used to return semantic data about the current node in
	// respect to the SP.
	NodeGetInfo(ctx context.Context) (*NodeGetInfoResponse, error)

	// NodeStageVolume is used when a plugin has the STAGE_UNSTAGE volume capability
	// to prepare a volume for usage on a host. If err == nil, the response should
	// be assumed to be successful.
	NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability) error

	// NodeUnstageVolume is used when a plugin has the STAGE_UNSTAGE volume capability
	// to undo the work performed by NodeStageVolume. If a volume has been staged,
	// this RPC must be called before freeing the volume.
	//
	// If err == nil, the response should be assumed to be successful.
	NodeUnstageVolume(ctx context.Context, volumeID string, stagingTargetPath string) error

	// NodePublishVolume is used to prepare a volume for use by an allocation.
	// if err == nil the response should be assumed to be successful.
	NodePublishVolume(ctx context.Context, req *NodePublishVolumeRequest) error

	// Shutdown the client and ensure any connections are cleaned up.
	Close() error
}

type NodePublishVolumeRequest struct {
	// The ID of the volume to publish.
	VolumeID string

	// If the volume was attached via a call to `ControllerPublishVolume` then
	// we need to provide the returned PublishContext here.
	PublishContext map[string]string

	// The path to which the volume was staged by `NodeStageVolume`.
	// It MUST be an absolute path in the root filesystem of the process
	// serving this request.
	// E.g {the plugins internal mount path}/staging/volumeid/...
	//
	// It MUST be set if the Node Plugin implements the
	// `STAGE_UNSTAGE_VOLUME` node capability.
	StagingTargetPath string

	// The path to which the volume will be published.
	// It MUST be an absolute path in the root filesystem of the process serving this
	// request.
	// E.g {the plugins internal mount path}/per-alloc/allocid/volumeid/...
	//
	// The CO SHALL ensure uniqueness of target_path per volume.
	// The CO SHALL ensure that the parent directory of this path exists
	// and that the process serving the request has `read` and `write`
	// permissions to that parent directory.
	TargetPath string

	// Volume capability describing how the CO intends to use this volume.
	VolumeCapability *VolumeCapability

	Readonly bool

	// Reserved for future use.
	Secrets map[string]string
}

func (r *NodePublishVolumeRequest) ToCSIRepresentation() *csipbv1.NodePublishVolumeRequest {
	return &csipbv1.NodePublishVolumeRequest{
		VolumeId:          r.VolumeID,
		PublishContext:    r.PublishContext,
		StagingTargetPath: r.StagingTargetPath,
		TargetPath:        r.TargetPath,
		VolumeCapability:  r.VolumeCapability.ToCSIRepresentation(),
		Readonly:          r.Readonly,
		Secrets:           r.Secrets,
	}
}

func (r *NodePublishVolumeRequest) Validate() error {
	if r.VolumeID == "" {
		return errors.New("missing VolumeID")
	}

	if r.TargetPath == "" {
		return errors.New("missing TargetPath")
	}

	if r.VolumeCapability == nil {
		return errors.New("missing VolumeCapabilities")
	}

	return nil
}

type PluginCapabilitySet struct {
	hasControllerService bool
	hasTopologies        bool
}

func (p *PluginCapabilitySet) HasControllerService() bool {
	return p.hasControllerService
}

// HasTopologies indicates whether the volumes for this plugin are equally
// accessible by all nodes in the cluster.
// If true, we MUST use the topology information when scheduling workloads.
func (p *PluginCapabilitySet) HasToplogies() bool {
	return p.hasTopologies
}

func (p *PluginCapabilitySet) IsEqual(o *PluginCapabilitySet) bool {
	return p.hasControllerService == o.hasControllerService && p.hasTopologies == o.hasTopologies
}

func NewTestPluginCapabilitySet(topologies, controller bool) *PluginCapabilitySet {
	return &PluginCapabilitySet{
		hasTopologies:        topologies,
		hasControllerService: controller,
	}
}

func NewPluginCapabilitySet(capabilities *csipbv1.GetPluginCapabilitiesResponse) *PluginCapabilitySet {
	cs := &PluginCapabilitySet{}

	pluginCapabilities := capabilities.GetCapabilities()

	for _, pcap := range pluginCapabilities {
		if svcCap := pcap.GetService(); svcCap != nil {
			switch svcCap.Type {
			case csipbv1.PluginCapability_Service_UNKNOWN:
				continue
			case csipbv1.PluginCapability_Service_CONTROLLER_SERVICE:
				cs.hasControllerService = true
			case csipbv1.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS:
				cs.hasTopologies = true
			default:
				continue
			}
		}
	}

	return cs
}

type ControllerCapabilitySet struct {
	HasPublishUnpublishVolume    bool
	HasPublishReadonly           bool
	HasListVolumes               bool
	HasListVolumesPublishedNodes bool
}

func NewControllerCapabilitySet(resp *csipbv1.ControllerGetCapabilitiesResponse) *ControllerCapabilitySet {
	cs := &ControllerCapabilitySet{}

	pluginCapabilities := resp.GetCapabilities()
	for _, pcap := range pluginCapabilities {
		if c := pcap.GetRpc(); c != nil {
			switch c.Type {
			case csipbv1.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME:
				cs.HasPublishUnpublishVolume = true
			case csipbv1.ControllerServiceCapability_RPC_PUBLISH_READONLY:
				cs.HasPublishReadonly = true
			case csipbv1.ControllerServiceCapability_RPC_LIST_VOLUMES:
				cs.HasListVolumes = true
			case csipbv1.ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES:
				cs.HasListVolumesPublishedNodes = true
			default:
				continue
			}
		}
	}

	return cs
}

type ControllerPublishVolumeRequest struct {
	VolumeID string
	NodeID   string
	ReadOnly bool

	//TODO: Add Capabilities
}

type ControllerPublishVolumeResponse struct {
	PublishContext map[string]string
}

type NodeCapabilitySet struct {
	HasStageUnstageVolume bool
}

func NewNodeCapabilitySet(resp *csipbv1.NodeGetCapabilitiesResponse) *NodeCapabilitySet {
	cs := &NodeCapabilitySet{}
	pluginCapabilities := resp.GetCapabilities()
	for _, pcap := range pluginCapabilities {
		if c := pcap.GetRpc(); c != nil {
			switch c.Type {
			case csipbv1.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME:
				cs.HasStageUnstageVolume = true
			default:
				continue
			}
		}
	}

	return cs
}

// VolumeAccessMode represents the desired access mode of the CSI Volume
type VolumeAccessMode csipbv1.VolumeCapability_AccessMode_Mode

var _ fmt.Stringer = VolumeAccessModeUnknown

var (
	VolumeAccessModeUnknown               = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_UNKNOWN)
	VolumeAccessModeSingleNodeWriter      = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_SINGLE_NODE_WRITER)
	VolumeAccessModeSingleNodeReaderOnly  = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY)
	VolumeAccessModeMultiNodeReaderOnly   = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY)
	VolumeAccessModeMultiNodeSingleWriter = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_MULTI_NODE_SINGLE_WRITER)
	VolumeAccessModeMultiNodeMultiWriter  = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER)
)

func (a VolumeAccessMode) String() string {
	return a.ToCSIRepresentation().String()
}

func (a VolumeAccessMode) ToCSIRepresentation() csipbv1.VolumeCapability_AccessMode_Mode {
	return csipbv1.VolumeCapability_AccessMode_Mode(a)
}

// VolumeAccessType represents the filesystem apis that the user intends to use
// with the volume. E.g whether it will be used as a block device or if they wish
// to have a mounted filesystem.
type VolumeAccessType int32

var _ fmt.Stringer = VolumeAccessTypeBlock

var (
	VolumeAccessTypeBlock VolumeAccessType = 1
	VolumeAccessTypeMount VolumeAccessType = 2
)

func (v VolumeAccessType) String() string {
	if v == VolumeAccessTypeBlock {
		return "VolumeAccessType.Block"
	} else if v == VolumeAccessTypeMount {
		return "VolumeAccessType.Mount"
	} else {
		return "VolumeAccessType.Unspecified"
	}
}

// VolumeMountOptions contain optional additional configuration that can be used
// when specifying that a Volume should be used with VolumeAccessTypeMount.
type VolumeMountOptions struct {
	// FSType is an optional field that allows an operator to specify the type
	// of the filesystem.
	FSType string

	// MountFlags contains additional options that may be used when mounting the
	// volume by the plugin. This may contain sensitive data and should not be
	// leaked.
	MountFlags []string
}

// VolumeMountOptions implements the Stringer and GoStringer interfaces to prevent
// accidental leakage of sensitive mount flags via logs.
var _ fmt.Stringer = &VolumeMountOptions{}
var _ fmt.GoStringer = &VolumeMountOptions{}

func (v *VolumeMountOptions) String() string {
	mountFlagsString := "nil"
	if len(v.MountFlags) != 0 {
		mountFlagsString = "[REDACTED]"
	}

	return fmt.Sprintf("csi.VolumeMountOptions(FSType: %s, MountFlags: %s)", v.FSType, mountFlagsString)
}

func (v *VolumeMountOptions) GoString() string {
	return v.String()
}

// VolumeCapability describes the overall usage requirements for a given CSI Volume
type VolumeCapability struct {
	AccessType         VolumeAccessType
	AccessMode         VolumeAccessMode
	VolumeMountOptions *VolumeMountOptions
}

func (c *VolumeCapability) ToCSIRepresentation() *csipbv1.VolumeCapability {
	vc := &csipbv1.VolumeCapability{
		AccessMode: &csipbv1.VolumeCapability_AccessMode{
			Mode: c.AccessMode.ToCSIRepresentation(),
		},
	}

	if c.AccessType == VolumeAccessTypeMount {
		vc.AccessType = &csipbv1.VolumeCapability_Mount{
			Mount: &csipbv1.VolumeCapability_MountVolume{
				FsType:     c.VolumeMountOptions.FSType,
				MountFlags: c.VolumeMountOptions.MountFlags,
			},
		}
	} else {
		vc.AccessType = &csipbv1.VolumeCapability_Block{Block: &csipbv1.VolumeCapability_BlockVolume{}}
	}

	return vc
}
<commit_msg>csi: Nil check ToCSIRepresentation implementations<commit_after>package csi

import (
	"context"
	"errors"
	"fmt"

	csipbv1 "github.com/container-storage-interface/spec/lib/go/csi"
	"github.com/hashicorp/nomad/plugins/base"
)

// CSIPlugin implements a lightweight abstraction layer around a CSI Plugin.
// It validates that responses from storage providers (SP's), correctly conform
// to the specification before returning response data or erroring.
type CSIPlugin interface {
	base.BasePlugin

	// PluginProbe is used to verify that the plugin is in a healthy state
	PluginProbe(ctx context.Context) (bool, error)

	// PluginGetInfo is used to return semantic data about the plugin.
	// Response:
	//  - string: name, the name of the plugin in domain notation format.
	PluginGetInfo(ctx context.Context) (string, error)

	// PluginGetCapabilities is used to return the available capabilities from the
	// identity service. This currently only looks for the CONTROLLER_SERVICE and
	// Accessible Topology Support
	PluginGetCapabilities(ctx context.Context) (*PluginCapabilitySet, error)

	// GetControllerCapabilities is used to get controller-specific capabilities
	// for a plugin.
	ControllerGetCapabilities(ctx context.Context) (*ControllerCapabilitySet, error)

	// ControllerPublishVolume is used to attach a remote volume to a cluster node.
	ControllerPublishVolume(ctx context.Context, req *ControllerPublishVolumeRequest) (*ControllerPublishVolumeResponse, error)

	// NodeGetCapabilities is used to return the available capabilities from the
	// Node Service.
	NodeGetCapabilities(ctx context.Context) (*NodeCapabilitySet, error)

	// NodeGetInfo is used to return semantic data about the current node in
	// respect to the SP.
	NodeGetInfo(ctx context.Context) (*NodeGetInfoResponse, error)

	// NodeStageVolume is used when a plugin has the STAGE_UNSTAGE volume capability
	// to prepare a volume for usage on a host. If err == nil, the response should
	// be assumed to be successful.
	NodeStageVolume(ctx context.Context, volumeID string, publishContext map[string]string, stagingTargetPath string, capabilities *VolumeCapability) error

	// NodeUnstageVolume is used when a plugin has the STAGE_UNSTAGE volume capability
	// to undo the work performed by NodeStageVolume. If a volume has been staged,
	// this RPC must be called before freeing the volume.
	//
	// If err == nil, the response should be assumed to be successful.
	NodeUnstageVolume(ctx context.Context, volumeID string, stagingTargetPath string) error

	// NodePublishVolume is used to prepare a volume for use by an allocation.
	// if err == nil the response should be assumed to be successful.
	NodePublishVolume(ctx context.Context, req *NodePublishVolumeRequest) error

	// Shutdown the client and ensure any connections are cleaned up.
	Close() error
}

type NodePublishVolumeRequest struct {
	// The ID of the volume to publish.
	VolumeID string

	// If the volume was attached via a call to `ControllerPublishVolume` then
	// we need to provide the returned PublishContext here.
	PublishContext map[string]string

	// The path to which the volume was staged by `NodeStageVolume`.
	// It MUST be an absolute path in the root filesystem of the process
	// serving this request.
	// E.g {the plugins internal mount path}/staging/volumeid/...
	//
	// It MUST be set if the Node Plugin implements the
	// `STAGE_UNSTAGE_VOLUME` node capability.
	StagingTargetPath string

	// The path to which the volume will be published.
	// It MUST be an absolute path in the root filesystem of the process serving this
	// request.
	// E.g {the plugins internal mount path}/per-alloc/allocid/volumeid/...
	//
	// The CO SHALL ensure uniqueness of target_path per volume.
	// The CO SHALL ensure that the parent directory of this path exists
	// and that the process serving the request has `read` and `write`
	// permissions to that parent directory.
	TargetPath string

	// Volume capability describing how the CO intends to use this volume.
	VolumeCapability *VolumeCapability

	Readonly bool

	// Reserved for future use.
	Secrets map[string]string
}

func (r *NodePublishVolumeRequest) ToCSIRepresentation() *csipbv1.NodePublishVolumeRequest {
	if r == nil {
		return nil
	}

	return &csipbv1.NodePublishVolumeRequest{
		VolumeId:          r.VolumeID,
		PublishContext:    r.PublishContext,
		StagingTargetPath: r.StagingTargetPath,
		TargetPath:        r.TargetPath,
		VolumeCapability:  r.VolumeCapability.ToCSIRepresentation(),
		Readonly:          r.Readonly,
		Secrets:           r.Secrets,
	}
}

func (r *NodePublishVolumeRequest) Validate() error {
	if r.VolumeID == "" {
		return errors.New("missing VolumeID")
	}

	if r.TargetPath == "" {
		return errors.New("missing TargetPath")
	}

	if r.VolumeCapability == nil {
		return errors.New("missing VolumeCapabilities")
	}

	return nil
}

type PluginCapabilitySet struct {
	hasControllerService bool
	hasTopologies        bool
}

func (p *PluginCapabilitySet) HasControllerService() bool {
	return p.hasControllerService
}

// HasTopologies indicates whether the volumes for this plugin are equally
// accessible by all nodes in the cluster.
// If true, we MUST use the topology information when scheduling workloads.
func (p *PluginCapabilitySet) HasToplogies() bool {
	return p.hasTopologies
}

func (p *PluginCapabilitySet) IsEqual(o *PluginCapabilitySet) bool {
	return p.hasControllerService == o.hasControllerService && p.hasTopologies == o.hasTopologies
}

func NewTestPluginCapabilitySet(topologies, controller bool) *PluginCapabilitySet {
	return &PluginCapabilitySet{
		hasTopologies:        topologies,
		hasControllerService: controller,
	}
}

func NewPluginCapabilitySet(capabilities *csipbv1.GetPluginCapabilitiesResponse) *PluginCapabilitySet {
	cs := &PluginCapabilitySet{}

	pluginCapabilities := capabilities.GetCapabilities()

	for _, pcap := range pluginCapabilities {
		if svcCap := pcap.GetService(); svcCap != nil {
			switch svcCap.Type {
			case csipbv1.PluginCapability_Service_UNKNOWN:
				continue
			case csipbv1.PluginCapability_Service_CONTROLLER_SERVICE:
				cs.hasControllerService = true
			case csipbv1.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS:
				cs.hasTopologies = true
			default:
				continue
			}
		}
	}

	return cs
}

type ControllerCapabilitySet struct {
	HasPublishUnpublishVolume    bool
	HasPublishReadonly           bool
	HasListVolumes               bool
	HasListVolumesPublishedNodes bool
}

func NewControllerCapabilitySet(resp *csipbv1.ControllerGetCapabilitiesResponse) *ControllerCapabilitySet {
	cs := &ControllerCapabilitySet{}

	pluginCapabilities := resp.GetCapabilities()
	for _, pcap := range pluginCapabilities {
		if c := pcap.GetRpc(); c != nil {
			switch c.Type {
			case csipbv1.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME:
				cs.HasPublishUnpublishVolume = true
			case csipbv1.ControllerServiceCapability_RPC_PUBLISH_READONLY:
				cs.HasPublishReadonly = true
			case csipbv1.ControllerServiceCapability_RPC_LIST_VOLUMES:
				cs.HasListVolumes = true
			case csipbv1.ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES:
				cs.HasListVolumesPublishedNodes = true
			default:
				continue
			}
		}
	}

	return cs
}

type ControllerPublishVolumeRequest struct {
	VolumeID string
	NodeID   string
	ReadOnly bool

	//TODO: Add Capabilities
}

type ControllerPublishVolumeResponse struct {
	PublishContext map[string]string
}

type NodeCapabilitySet struct {
	HasStageUnstageVolume bool
}

func NewNodeCapabilitySet(resp *csipbv1.NodeGetCapabilitiesResponse) *NodeCapabilitySet {
	cs := &NodeCapabilitySet{}
	pluginCapabilities := resp.GetCapabilities()
	for _, pcap := range pluginCapabilities {
		if c := pcap.GetRpc(); c != nil {
			switch c.Type {
			case csipbv1.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME:
				cs.HasStageUnstageVolume = true
			default:
				continue
			}
		}
	}

	return cs
}

// VolumeAccessMode represents the desired access mode of the CSI Volume
type VolumeAccessMode csipbv1.VolumeCapability_AccessMode_Mode

var _ fmt.Stringer = VolumeAccessModeUnknown

var (
	VolumeAccessModeUnknown               = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_UNKNOWN)
	VolumeAccessModeSingleNodeWriter      = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_SINGLE_NODE_WRITER)
	VolumeAccessModeSingleNodeReaderOnly  = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY)
	VolumeAccessModeMultiNodeReaderOnly   = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY)
	VolumeAccessModeMultiNodeSingleWriter = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_MULTI_NODE_SINGLE_WRITER)
	VolumeAccessModeMultiNodeMultiWriter  = VolumeAccessMode(csipbv1.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER)
)

func (a VolumeAccessMode) String() string {
	return a.ToCSIRepresentation().String()
}

func (a VolumeAccessMode) ToCSIRepresentation() csipbv1.VolumeCapability_AccessMode_Mode {
	return csipbv1.VolumeCapability_AccessMode_Mode(a)
}

// VolumeAccessType represents the filesystem apis that the user intends to use
// with the volume. E.g whether it will be used as a block device or if they wish
// to have a mounted filesystem.
type VolumeAccessType int32

var _ fmt.Stringer = VolumeAccessTypeBlock

var (
	VolumeAccessTypeBlock VolumeAccessType = 1
	VolumeAccessTypeMount VolumeAccessType = 2
)

func (v VolumeAccessType) String() string {
	if v == VolumeAccessTypeBlock {
		return "VolumeAccessType.Block"
	} else if v == VolumeAccessTypeMount {
		return "VolumeAccessType.Mount"
	} else {
		return "VolumeAccessType.Unspecified"
	}
}

// VolumeMountOptions contain optional additional configuration that can be used
// when specifying that a Volume should be used with VolumeAccessTypeMount.
type VolumeMountOptions struct {
	// FSType is an optional field that allows an operator to specify the type
	// of the filesystem.
	FSType string

	// MountFlags contains additional options that may be used when mounting the
	// volume by the plugin. This may contain sensitive data and should not be
	// leaked.
	MountFlags []string
}

// VolumeMountOptions implements the Stringer and GoStringer interfaces to prevent
// accidental leakage of sensitive mount flags via logs.
var _ fmt.Stringer = &VolumeMountOptions{}
var _ fmt.GoStringer = &VolumeMountOptions{}

func (v *VolumeMountOptions) String() string {
	mountFlagsString := "nil"
	if len(v.MountFlags) != 0 {
		mountFlagsString = "[REDACTED]"
	}

	return fmt.Sprintf("csi.VolumeMountOptions(FSType: %s, MountFlags: %s)", v.FSType, mountFlagsString)
}

func (v *VolumeMountOptions) GoString() string {
	return v.String()
}

// VolumeCapability describes the overall usage requirements for a given CSI Volume
type VolumeCapability struct {
	AccessType         VolumeAccessType
	AccessMode         VolumeAccessMode
	VolumeMountOptions *VolumeMountOptions
}

func (c *VolumeCapability) ToCSIRepresentation() *csipbv1.VolumeCapability {
	if c == nil {
		return nil
	}

	vc := &csipbv1.VolumeCapability{
		AccessMode: &csipbv1.VolumeCapability_AccessMode{
			Mode: c.AccessMode.ToCSIRepresentation(),
		},
	}

	if c.AccessType == VolumeAccessTypeMount {
		vc.AccessType = &csipbv1.VolumeCapability_Mount{
			Mount: &csipbv1.VolumeCapability_MountVolume{
				FsType:     c.VolumeMountOptions.FSType,
				MountFlags: c.VolumeMountOptions.MountFlags,
			},
		}
	} else {
		vc.AccessType = &csipbv1.VolumeCapability_Block{Block: &csipbv1.VolumeCapability_BlockVolume{}}
	}

	return vc
}
<|endoftext|>


<commit_before>package core

import (
	"bufio"
	"context"
	"fmt"
	"github.com/grandcat/zeroconf"
	"gopkg.in/cheggaaa/pb.v1"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"time"
)

func LookupZeroconf(id string, entries chan *zeroconf.ServiceEntry, stop chan bool) {
	resolver, err := zeroconf.NewResolver(nil)
	if err != nil {
		log.Fatalln("Failed to initialize resolver:", err.Error())
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	// err = resolver.Lookup(ctx, id, "_figo._http._tcp", "local.", entries)
	err = resolver.Lookup(ctx, id, "_figo._http._tcp", "local.", entries)
	if err != nil {
		panic(err)
	}

	<-stop

	<-ctx.Done()
	return
}

func Figor(name string, out string) {
	log.SetOutput(ioutil.Discard)

	var entry *zeroconf.ServiceEntry

	{
		entries := make(chan *zeroconf.ServiceEntry)

		stopLookup := make(chan bool)
		go LookupZeroconf(GetHash(name), entries, stopLookup)

		{
			timer := time.NewTimer(time.Millisecond * 500)
			for i := 0; i < 2; i++ {
				loopDone := false
				select {
				case <-timer.C:
					fmt.Println("couldn't find anything!")
					fmt.Println("figos "+name, "or", "figos <filename> "+name)
				case entry = <-entries:
					timer.Stop()
					stopLookup <- true
					loopDone = true
				}
				if loopDone {
					break
				}
			}
		}
	}

	addr := entry.AddrIPv4[0]
	port := entry.Port

	urlPath := fmt.Sprintf("http://%v.%v.%v.%v:%d/%s", addr[0], addr[1], addr[2], addr[3], port, name)
	if resp, err := http.Get(urlPath); err != nil {
		panic(err)
	} else if resp.StatusCode != 200 {
		fmt.Println("Not correct file/code")
		os.Exit(1)
	} else {
		if out == "" {
			rgx := regexp.MustCompile("inline; filename=\"(.*)\"")
			out = rgx.FindStringSubmatch(resp.Header.Get("Content-Disposition"))[1]
		}
		if of, err := os.Create(out); err != nil {
			panic(err)
		} else {
			var datalen int
			fmt.Sscanf(resp.Header.Get("Content-Length"), "%d", &datalen)

			resp_buf := bufio.NewReader(resp.Body)
			of_buf := bufio.NewWriter(of)

			bar := pb.New(datalen).SetUnits(pb.U_BYTES)
			bar.Start()
			bar_writer := io.MultiWriter(of_buf, bar)
			io.Copy(bar_writer, resp_buf)
			bar.Finish()
		}
	}
}
<commit_msg>show filename<commit_after>package core

import (
	"bufio"
	"context"
	"fmt"
	"github.com/grandcat/zeroconf"
	"gopkg.in/cheggaaa/pb.v1"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"time"
)

func LookupZeroconf(id string, entries chan *zeroconf.ServiceEntry, stop chan bool) {
	resolver, err := zeroconf.NewResolver(nil)
	if err != nil {
		log.Fatalln("Failed to initialize resolver:", err.Error())
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	// err = resolver.Lookup(ctx, id, "_figo._http._tcp", "local.", entries)
	err = resolver.Lookup(ctx, id, "_figo._http._tcp", "local.", entries)
	if err != nil {
		panic(err)
	}

	<-stop

	<-ctx.Done()
	return
}

func Figor(name string, out string) {
	log.SetOutput(ioutil.Discard)

	var entry *zeroconf.ServiceEntry

	{
		entries := make(chan *zeroconf.ServiceEntry)

		stopLookup := make(chan bool)
		go LookupZeroconf(GetHash(name), entries, stopLookup)

		{
			timer := time.NewTimer(time.Millisecond * 500)
			for i := 0; i < 2; i++ {
				loopDone := false
				select {
				case <-timer.C:
					fmt.Println("couldn't find anything!")
					fmt.Println("figos "+name, "or", "figos <filename> "+name)
				case entry = <-entries:
					timer.Stop()
					stopLookup <- true
					loopDone = true
				}
				if loopDone {
					break
				}
			}
		}
	}

	addr := entry.AddrIPv4[0]
	port := entry.Port

	urlPath := fmt.Sprintf("http://%v.%v.%v.%v:%d/%s", addr[0], addr[1], addr[2], addr[3], port, name)
	if resp, err := http.Get(urlPath); err != nil {
		panic(err)
	} else if resp.StatusCode != 200 {
		fmt.Println("Not correct file/code")
		os.Exit(1)
	} else {
		if out == "" {
			rgx := regexp.MustCompile("inline; filename=\"(.*)\"")
			out = rgx.FindStringSubmatch(resp.Header.Get("Content-Disposition"))[1]
		}
		if of, err := os.Create(out); err != nil {
			panic(err)
		} else {
			var datalen int
			fmt.Sscanf(resp.Header.Get("Content-Length"), "%d", &datalen)

			resp_buf := bufio.NewReader(resp.Body)
			of_buf := bufio.NewWriter(of)

			bar := pb.New(datalen).SetUnits(pb.U_BYTES).Prefix(out)
			bar.Start()
			bar_writer := io.MultiWriter(of_buf, bar)
			io.Copy(bar_writer, resp_buf)
			bar.Finish()
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package control

import (
	"errors"

	"fmt"

	clientv2 "github.com/coreos/etcd/client"
	"github.com/coreos/etcd/clientv3"
	consulapi "github.com/hashicorp/consul/api"
	"github.com/samuel/go-zookeeper/zk"
	"golang.org/x/net/context"
)

type ReqHandler func(ctx context.Context, req *request) error

func newPutEtcd2(conn clientv2.KeysAPI) ReqHandler {
	return func(ctx context.Context, req *request) error {
		op := req.etcdv2Op
		_, err := conn.Set(context.Background(), op.key, op.value, nil)
		return err
	}
}

func newPutEtcd3(conn clientv3.KV) ReqHandler {
	return func(ctx context.Context, req *request) error {
		_, err := conn.Do(ctx, req.etcdv3Op)
		return err
	}
}

func newPutOverwriteZK(conn *zk.Conn) ReqHandler {
	return func(ctx context.Context, req *request) error {
		op := req.zkOp
		_, err := conn.Set(op.key, op.value, int32(-1))
		return err
	}
}

func newPutCreateZK(conn *zk.Conn) ReqHandler {
	// samekey
	return func(ctx context.Context, req *request) error {
		op := req.zkOp
		_, err := conn.Create(op.key, op.value, zkCreateFlags, zkCreateAcl)
		return err
	}
}

func newPutConsul(conn *consulapi.KV) ReqHandler {
	return func(ctx context.Context, req *request) error {
		op := req.consulOp
		_, err := conn.Put(&consulapi.KVPair{Key: op.key, Value: op.value}, nil)
		return err
	}
}

func newGetEtcd2(conn clientv2.KeysAPI) ReqHandler {
	return func(ctx context.Context, req *request) error {
		_, err := conn.Get(ctx, req.etcdv2Op.key, nil)
		return err
	}
}

func newGetEtcd3(conn clientv3.KV) ReqHandler {
	return func(ctx context.Context, req *request) error {
		_, err := conn.Do(ctx, req.etcdv3Op)
		return err
	}
}

func newGetZK(conn *zk.Conn) ReqHandler {
	return func(ctx context.Context, req *request) error {
		errt := ""
		if !req.zkOp.staleRead {
			_, err := conn.Sync("/" + req.zkOp.key)
			if err != nil {
				errt += err.Error()
			}
		}
		_, _, err := conn.Get("/" + req.zkOp.key)
		if err != nil {
			if errt != "" {
				errt += "; "
			}
			errt += fmt.Sprintf("%q while getting %q", err.Error(), "/"+req.zkOp.key)
		}
		if errt != "" {
			return errors.New(errt)
		}
		return nil
	}
}

func newGetConsul(conn *consulapi.KV) ReqHandler {
	return func(ctx context.Context, req *request) error {
		opt := &consulapi.QueryOptions{}
		if req.consulOp.staleRead {
			opt.AllowStale = true
			opt.RequireConsistent = false
		}
		if !req.consulOp.staleRead {
			opt.AllowStale = false
			opt.RequireConsistent = true
		}
		_, _, err := conn.Get(req.consulOp.key, opt)
		return err
	}
}
<commit_msg>control: fix wrong comment<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package control

import (
	"errors"

	"fmt"

	clientv2 "github.com/coreos/etcd/client"
	"github.com/coreos/etcd/clientv3"
	consulapi "github.com/hashicorp/consul/api"
	"github.com/samuel/go-zookeeper/zk"
	"golang.org/x/net/context"
)

type ReqHandler func(ctx context.Context, req *request) error

func newPutEtcd2(conn clientv2.KeysAPI) ReqHandler {
	return func(ctx context.Context, req *request) error {
		op := req.etcdv2Op
		_, err := conn.Set(context.Background(), op.key, op.value, nil)
		return err
	}
}

func newPutEtcd3(conn clientv3.KV) ReqHandler {
	return func(ctx context.Context, req *request) error {
		_, err := conn.Do(ctx, req.etcdv3Op)
		return err
	}
}

func newPutOverwriteZK(conn *zk.Conn) ReqHandler {
	// samekey
	return func(ctx context.Context, req *request) error {
		op := req.zkOp
		_, err := conn.Set(op.key, op.value, int32(-1))
		return err
	}
}

func newPutCreateZK(conn *zk.Conn) ReqHandler {
	return func(ctx context.Context, req *request) error {
		op := req.zkOp
		_, err := conn.Create(op.key, op.value, zkCreateFlags, zkCreateAcl)
		return err
	}
}

func newPutConsul(conn *consulapi.KV) ReqHandler {
	return func(ctx context.Context, req *request) error {
		op := req.consulOp
		_, err := conn.Put(&consulapi.KVPair{Key: op.key, Value: op.value}, nil)
		return err
	}
}

func newGetEtcd2(conn clientv2.KeysAPI) ReqHandler {
	return func(ctx context.Context, req *request) error {
		_, err := conn.Get(ctx, req.etcdv2Op.key, nil)
		return err
	}
}

func newGetEtcd3(conn clientv3.KV) ReqHandler {
	return func(ctx context.Context, req *request) error {
		_, err := conn.Do(ctx, req.etcdv3Op)
		return err
	}
}

func newGetZK(conn *zk.Conn) ReqHandler {
	return func(ctx context.Context, req *request) error {
		errt := ""
		if !req.zkOp.staleRead {
			_, err := conn.Sync("/" + req.zkOp.key)
			if err != nil {
				errt += err.Error()
			}
		}
		_, _, err := conn.Get("/" + req.zkOp.key)
		if err != nil {
			if errt != "" {
				errt += "; "
			}
			errt += fmt.Sprintf("%q while getting %q", err.Error(), "/"+req.zkOp.key)
		}
		if errt != "" {
			return errors.New(errt)
		}
		return nil
	}
}

func newGetConsul(conn *consulapi.KV) ReqHandler {
	return func(ctx context.Context, req *request) error {
		opt := &consulapi.QueryOptions{}
		if req.consulOp.staleRead {
			opt.AllowStale = true
			opt.RequireConsistent = false
		}
		if !req.consulOp.staleRead {
			opt.AllowStale = false
			opt.RequireConsistent = true
		}
		_, _, err := conn.Get(req.consulOp.key, opt)
		return err
	}
}
<|endoftext|>


<commit_before>package poll

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/venicegeo/pzsvc-exec/dispatcher/cfwrapper"
	"github.com/venicegeo/pzsvc-exec/dispatcher/model"
	"github.com/venicegeo/pzsvc-exec/pzsvc"
)

const defaultTaskDiskMB = 6142
const defaultTaskMemoryMB = 3072

var pzsvcGetS3FileSizeInMegabytes = pzsvc.GetS3FileSizeInMegabytes
var pzsvcRequestKnownJSON = pzsvc.RequestKnownJSON
var pzsvcSendExecResultNoData = pzsvc.SendExecResultNoData

// Loop is an encapsulation of configuration and functionality needed for a job polling loop
type Loop struct {
	PzSession     *pzsvc.Session
	PzConfig      pzsvc.Config
	SvcID         string
	ConfigPath    string
	ClientFactory cfwrapper.Factory
	vcapID        string
	taskLimit     int
	intervalTick  time.Duration

	stopChan         chan bool
	runIterationFunc func(l Loop) error
}

// NewLoop creates a Loop and does starting configuration based on the given parameters
func NewLoop(s *pzsvc.Session, configObj pzsvc.Config, svcID string, configPath string, clientFactory cfwrapper.Factory) (*Loop, error) {
	pzsvc.LogInfo(*s, "Initializing polling loop object")

	appID, err := getVCAPApplicationID()
	if err != nil {
		return nil, err
	}
	pzsvc.LogInfo(*s, "Found application name from VCAP Tree: "+appID)

	// Read the # of simultaneous Tasks that are allowed to be run by the Dispatcher
	taskLimit := 3
	if envTaskLimit := os.Getenv("TASK_LIMIT"); envTaskLimit != "" {
		taskLimit, _ = strconv.Atoi(envTaskLimit)
	}

	return &Loop{
		PzSession:        s,
		PzConfig:         configObj,
		SvcID:            svcID,
		ConfigPath:       configPath,
		ClientFactory:    clientFactory,
		vcapID:           appID,
		taskLimit:        taskLimit,
		intervalTick:     5 * time.Second,
		stopChan:         nil, // initialized when loop starts
		runIterationFunc: runIteration,
	}, nil
}

// Start begins the polling interval loop and returns a channel that feeds
// through any errors encountered in each interval
func (l *Loop) Start() <-chan error {
	errChan := make(chan error)
	l.stopChan = make(chan bool)
	go func() {
		ticker := time.Tick(l.intervalTick)
		for {
			select {
			case <-ticker:
				err := l.runIterationFunc(*l)
				if err != nil {
					errChan <- err
				}
			case <-l.stopChan:
				close(errChan)
				return
			}
		}
	}()
	return errChan
}

// Stop halts the loop's iteration
func (l Loop) Stop() {
	l.stopChan <- true
	close(l.stopChan)
}

func runIteration(l Loop) error {
	pzsvc.LogInfo(*l.PzSession, "Starting polling loop iteration")

	cfSession, err := l.ClientFactory.GetSession()
	if err != nil {
		pzsvc.LogSimpleErr(*l.PzSession, "Error generating valid CF Client", err)
		return err
	}

	numTasks, err := cfSession.CountTasksForApp(l.vcapID)
	if err != nil {
		pzsvc.LogSimpleErr(*l.PzSession, "Error checking running tasks. ", err)
		return err
	}
	if numTasks >= l.taskLimit {
		pzsvc.LogInfo(*l.PzSession, "Too many tasks already running, aborting polling loop iteration")
		return nil
	}

	taskItem, err := l.getPzTaskItem()
	if err != nil {
		return err
	}

	jobID := taskItem.Data.SvcData.JobID
	jobData := taskItem.Data.SvcData.Data.DataInputs.Body.Content
	if jobData == "" {
		message := fmt.Sprintf("Received job with empty data, aborting polling loop iteration; jobID=%s", jobID)
		pzsvc.LogWarn(*l.PzSession, message)
		return nil
	}
	pzsvc.LogInfo(*l.PzSession, "New Task Grabbed.  JobID: "+jobID)

	jobInput, err := l.parseJobInput(jobData)
	if err != nil {
		return err
	}

	workerCommand, err := l.buildWorkerCommand(jobInput, jobID)
	if err != nil {
		return err
	}

	diskMB, memoryMB := l.calculateDiskAndMemoryLimits(jobInput)

	taskRequest := cfwrapper.TaskRequest{
		Command:          workerCommand,
		Name:             jobID,
		DropletGUID:      l.vcapID,
		DiskInMegabyte:   diskMB,
		MemoryInMegabyte: memoryMB,
	}

	serializedInput, _ := json.Marshal(jobInput)
	pzsvc.LogAudit(*l.PzSession, l.PzSession.UserID, "Creating CF Task for Job "+jobID+" : "+workerCommand, l.PzSession.AppName, string(serializedInput), pzsvc.INFO)

	if err = cfSession.CreateTask(taskRequest); err != nil {
		if cfwrapper.IsMemoryLimitError(err) {
			pzsvc.LogAudit(*l.PzSession, l.PzSession.UserID, "Audit failure", l.PzSession.AppName, "The Memory limit of CF Org has been exceeded. No further jobs can be created.", pzsvc.ERROR)
			return errors.New("CF memory limit hit, will retry job later")
		}
		// General error - fail the job.
		pzsvc.LogAudit(*l.PzSession, l.PzSession.UserID, "Audit failure", l.PzSession.AppName, "Could not Create PCF Task for Job. Job Failed: "+err.Error(), pzsvc.ERROR)
		pzsvcSendExecResultNoData(*l.PzSession, l.PzSession.PzAddr, l.SvcID, jobID, pzsvc.PiazzaStatusFail)
		return err
	}

	return nil
}

func (l Loop) getPzTaskItem() (*model.PzTaskItem, error) {
	var pzTaskItem model.PzTaskItem
	url := fmt.Sprintf("%s/service/%s/task", l.PzSession.PzAddr, l.SvcID)

	byts, err := pzsvcRequestKnownJSON("POST", "", url, l.PzSession.PzAuth, &pzTaskItem)
	if err != nil {
		err.Log(*l.PzSession, "Dispatcher: error getting new task:"+string(byts))
		return nil, err
	}
	return &pzTaskItem, nil
}

func (l Loop) parseJobInput(jobInputStr string) (*pzsvc.InpStruct, error) {
	var err error
	var jobInputContent pzsvc.InpStruct

	if err = json.Unmarshal([]byte(jobInputStr), &jobInputContent); err != nil {
		pzsvc.LogSimpleErr(*l.PzSession, "Error decoding job input body", err)
		return nil, err
	}

	if jobInputContent.ExtAuth != "" {
		jobInputContent.ExtAuth = "*****"
	}
	if jobInputContent.PzAuth != "" {
		jobInputContent.PzAuth = "*****"
	}

	return &jobInputContent, nil
}

func (l Loop) buildWorkerCommand(jobInput *pzsvc.InpStruct, jobID string) (string, error) {
	workerCommand := fmt.Sprintf("worker --cliExtra '%s' --userID '%s' --config '%s' --serviceID '%s' --jobID '%s'",
		jobInput.Command, jobInput.UserID, l.ConfigPath, l.SvcID, jobID)

	if len(jobInput.InExtFiles) != len(jobInput.InExtNames) {
		return "", errors.New("Number of input file names and URLs did not match")
	}

	commandParts := []string{workerCommand}

	for i := range jobInput.InExtNames {
		inputPair := fmt.Sprintf("%s:%s", jobInput.InExtNames[i], jobInput.InExtFiles[i])
		commandParts = append(commandParts, "-i", inputPair)
	}

	for _, outputFile := range jobInput.OutGeoJs { // TODO: non-geojson outputs?
		commandParts = append(commandParts, "-o", outputFile)
	}

	return strings.Join(commandParts, " "), nil
}

func (l Loop) calculateAWSInputFileSizeMB(jobInput *pzsvc.InpStruct) (total int) {
	for _, url := range jobInput.InExtFiles {
		if strings.Contains(url, "amazonaws") {
			fileSize, err := pzsvcGetS3FileSizeInMegabytes(url)
			if err == nil {
				pzsvc.LogInfo(*l.PzSession, fmt.Sprintf("S3 File Size for %s found to be %d", url, fileSize))
				total += fileSize
			} else {
				err.Log(*l.PzSession, "Tried to get File Size from S3 File "+url+" but encountered an error; giving up on calculating input sizes")
				return 0
			}
		} else {
			pzsvc.LogInfo(*l.PzSession, fmt.Sprintf("Input file %s is not AWS; giving up on calculating input sizes", url))
			return 0
		}
	}
	return
}

func (l Loop) calculateDiskAndMemoryLimits(jobInput *pzsvc.InpStruct) (diskMB int, memoryMB int) {
	diskMB = defaultTaskDiskMB
	memoryMB = defaultTaskMemoryMB

	if inputSize := l.calculateAWSInputFileSizeMB(jobInput); inputSize > 0 {
		// Allocate 2G for the filesystem and executables (with some buffer), then add the image sizes
		diskMB = 2048 + (inputSize * 2)
		memoryMB = memoryMB + (inputSize * 5)
		pzsvc.LogInfo(*l.PzSession, fmt.Sprintf("Obtained S3 File Sizes for input files; will use Dynamic Disk Space of %d in Task container and Dynamic Memory Size of %d", diskMB, memoryMB))
	} else {
		pzsvc.LogInfo(*l.PzSession, "Could not get the S3 File Sizes for input files. Will use the default Disk and Memory Space when running Task.")
	}
	return
}
<commit_msg>Make chatty logs more accurate<commit_after>package poll

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/venicegeo/pzsvc-exec/dispatcher/cfwrapper"
	"github.com/venicegeo/pzsvc-exec/dispatcher/model"
	"github.com/venicegeo/pzsvc-exec/pzsvc"
)

const defaultTaskDiskMB = 6142
const defaultTaskMemoryMB = 3072

var pzsvcGetS3FileSizeInMegabytes = pzsvc.GetS3FileSizeInMegabytes
var pzsvcRequestKnownJSON = pzsvc.RequestKnownJSON
var pzsvcSendExecResultNoData = pzsvc.SendExecResultNoData

// Loop is an encapsulation of configuration and functionality needed for a job polling loop
type Loop struct {
	PzSession     *pzsvc.Session
	PzConfig      pzsvc.Config
	SvcID         string
	ConfigPath    string
	ClientFactory cfwrapper.Factory
	vcapID        string
	taskLimit     int
	intervalTick  time.Duration

	stopChan         chan bool
	runIterationFunc func(l Loop) error
}

// NewLoop creates a Loop and does starting configuration based on the given parameters
func NewLoop(s *pzsvc.Session, configObj pzsvc.Config, svcID string, configPath string, clientFactory cfwrapper.Factory) (*Loop, error) {
	pzsvc.LogInfo(*s, "Initializing polling loop object")

	appID, err := getVCAPApplicationID()
	if err != nil {
		return nil, err
	}
	pzsvc.LogInfo(*s, "Found application name from VCAP Tree: "+appID)

	// Read the # of simultaneous Tasks that are allowed to be run by the Dispatcher
	taskLimit := 3
	if envTaskLimit := os.Getenv("TASK_LIMIT"); envTaskLimit != "" {
		taskLimit, _ = strconv.Atoi(envTaskLimit)
	}

	return &Loop{
		PzSession:        s,
		PzConfig:         configObj,
		SvcID:            svcID,
		ConfigPath:       configPath,
		ClientFactory:    clientFactory,
		vcapID:           appID,
		taskLimit:        taskLimit,
		intervalTick:     5 * time.Second,
		stopChan:         nil, // initialized when loop starts
		runIterationFunc: runIteration,
	}, nil
}

// Start begins the polling interval loop and returns a channel that feeds
// through any errors encountered in each interval
func (l *Loop) Start() <-chan error {
	errChan := make(chan error)
	l.stopChan = make(chan bool)
	go func() {
		ticker := time.Tick(l.intervalTick)
		for {
			select {
			case <-ticker:
				err := l.runIterationFunc(*l)
				if err != nil {
					errChan <- err
				}
			case <-l.stopChan:
				close(errChan)
				return
			}
		}
	}()
	return errChan
}

// Stop halts the loop's iteration
func (l Loop) Stop() {
	l.stopChan <- true
	close(l.stopChan)
}

func runIteration(l Loop) error {
	pzsvc.LogInfo(*l.PzSession, "Starting polling loop iteration")

	cfSession, err := l.ClientFactory.GetSession()
	if err != nil {
		pzsvc.LogSimpleErr(*l.PzSession, "Error generating valid CF Client", err)
		return err
	}

	numTasks, err := cfSession.CountTasksForApp(l.vcapID)
	if err != nil {
		pzsvc.LogSimpleErr(*l.PzSession, "Error checking running tasks. ", err)
		return err
	}
	if numTasks >= l.taskLimit {
		pzsvc.LogInfo(*l.PzSession, "Too many tasks already running, skipping this iteration cycle")
		return nil
	}

	taskItem, err := l.getPzTaskItem()
	if err != nil {
		return err
	}

	jobID := taskItem.Data.SvcData.JobID
	jobData := taskItem.Data.SvcData.Data.DataInputs.Body.Content
	if jobData == "" {
		message := fmt.Sprintf("Received job with empty data, skipping this iteration cycle; (jobID='%s')", jobID)
		pzsvc.LogInfo(*l.PzSession, message)
		return nil
	}
	pzsvc.LogInfo(*l.PzSession, "New Task Grabbed.  JobID: "+jobID)

	jobInput, err := l.parseJobInput(jobData)
	if err != nil {
		return err
	}

	workerCommand, err := l.buildWorkerCommand(jobInput, jobID)
	if err != nil {
		return err
	}

	diskMB, memoryMB := l.calculateDiskAndMemoryLimits(jobInput)

	taskRequest := cfwrapper.TaskRequest{
		Command:          workerCommand,
		Name:             jobID,
		DropletGUID:      l.vcapID,
		DiskInMegabyte:   diskMB,
		MemoryInMegabyte: memoryMB,
	}

	serializedInput, _ := json.Marshal(jobInput)
	pzsvc.LogAudit(*l.PzSession, l.PzSession.UserID, "Creating CF Task for Job "+jobID+" : "+workerCommand, l.PzSession.AppName, string(serializedInput), pzsvc.INFO)

	if err = cfSession.CreateTask(taskRequest); err != nil {
		if cfwrapper.IsMemoryLimitError(err) {
			pzsvc.LogAudit(*l.PzSession, l.PzSession.UserID, "Audit failure", l.PzSession.AppName, "The Memory limit of CF Org has been exceeded. No further jobs can be created.", pzsvc.ERROR)
			return errors.New("CF memory limit hit, will retry job later")
		}
		// General error - fail the job.
		pzsvc.LogAudit(*l.PzSession, l.PzSession.UserID, "Audit failure", l.PzSession.AppName, "Could not Create PCF Task for Job. Job Failed: "+err.Error(), pzsvc.ERROR)
		pzsvcSendExecResultNoData(*l.PzSession, l.PzSession.PzAddr, l.SvcID, jobID, pzsvc.PiazzaStatusFail)
		return err
	}

	return nil
}

func (l Loop) getPzTaskItem() (*model.PzTaskItem, error) {
	var pzTaskItem model.PzTaskItem
	url := fmt.Sprintf("%s/service/%s/task", l.PzSession.PzAddr, l.SvcID)

	byts, err := pzsvcRequestKnownJSON("POST", "", url, l.PzSession.PzAuth, &pzTaskItem)
	if err != nil {
		err.Log(*l.PzSession, "Dispatcher: error getting new task:"+string(byts))
		return nil, err
	}
	return &pzTaskItem, nil
}

func (l Loop) parseJobInput(jobInputStr string) (*pzsvc.InpStruct, error) {
	var err error
	var jobInputContent pzsvc.InpStruct

	if err = json.Unmarshal([]byte(jobInputStr), &jobInputContent); err != nil {
		pzsvc.LogSimpleErr(*l.PzSession, "Error decoding job input body", err)
		return nil, err
	}

	if jobInputContent.ExtAuth != "" {
		jobInputContent.ExtAuth = "*****"
	}
	if jobInputContent.PzAuth != "" {
		jobInputContent.PzAuth = "*****"
	}

	return &jobInputContent, nil
}

func (l Loop) buildWorkerCommand(jobInput *pzsvc.InpStruct, jobID string) (string, error) {
	workerCommand := fmt.Sprintf("worker --cliExtra '%s' --userID '%s' --config '%s' --serviceID '%s' --jobID '%s'",
		jobInput.Command, jobInput.UserID, l.ConfigPath, l.SvcID, jobID)

	if len(jobInput.InExtFiles) != len(jobInput.InExtNames) {
		return "", errors.New("Number of input file names and URLs did not match")
	}

	commandParts := []string{workerCommand}

	for i := range jobInput.InExtNames {
		inputPair := fmt.Sprintf("%s:%s", jobInput.InExtNames[i], jobInput.InExtFiles[i])
		commandParts = append(commandParts, "-i", inputPair)
	}

	for _, outputFile := range jobInput.OutGeoJs { // TODO: non-geojson outputs?
		commandParts = append(commandParts, "-o", outputFile)
	}

	return strings.Join(commandParts, " "), nil
}

func (l Loop) calculateAWSInputFileSizeMB(jobInput *pzsvc.InpStruct) (total int) {
	for _, url := range jobInput.InExtFiles {
		if strings.Contains(url, "amazonaws") {
			fileSize, err := pzsvcGetS3FileSizeInMegabytes(url)
			if err == nil {
				pzsvc.LogInfo(*l.PzSession, fmt.Sprintf("S3 File Size for %s found to be %d", url, fileSize))
				total += fileSize
			} else {
				err.Log(*l.PzSession, "Tried to get File Size from S3 File "+url+" but encountered an error; giving up on calculating input sizes")
				return 0
			}
		} else {
			pzsvc.LogInfo(*l.PzSession, fmt.Sprintf("Input file %s is not AWS; giving up on calculating input sizes", url))
			return 0
		}
	}
	return
}

func (l Loop) calculateDiskAndMemoryLimits(jobInput *pzsvc.InpStruct) (diskMB int, memoryMB int) {
	diskMB = defaultTaskDiskMB
	memoryMB = defaultTaskMemoryMB

	if inputSize := l.calculateAWSInputFileSizeMB(jobInput); inputSize > 0 {
		// Allocate 2G for the filesystem and executables (with some buffer), then add the image sizes
		diskMB = 2048 + (inputSize * 2)
		memoryMB = memoryMB + (inputSize * 5)
		pzsvc.LogInfo(*l.PzSession, fmt.Sprintf("Obtained S3 File Sizes for input files; will use Dynamic Disk Space of %d in Task container and Dynamic Memory Size of %d", diskMB, memoryMB))
	} else {
		pzsvc.LogInfo(*l.PzSession, "Could not get the S3 File Sizes for input files. Will use the default Disk and Memory Space when running Task.")
	}
	return
}
<|endoftext|>


<commit_before>// -*- tab-width: 4; -*-

package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"regexp"
	"sort"
	"strings"
	"time"
)

func timeline_command(args []string) {
	fs := flag.NewFlagSet("timeline", flag.ExitOnError)
	durationFlag := fs.Duration("d", 0, "only show tweets created at most `duration` back in time. Example: -d 12h")
	fs.Usage = func() {
		fmt.Printf("usage: %s timeline [arguments]\n\nDisplays the timeline.\n\n", progname)
		fs.PrintDefaults()
	}
	fs.Parse(args) // currently using flag.ExitOnError, so we won't get an error on -h
	if fs.NArg() > 0 {
		fmt.Printf("Too many arguments given.\n")
		os.Exit(2)
	}

	if *durationFlag < 0 {
		fmt.Printf("Negative duration doesn't make sense.\n")
		os.Exit(2)
	}

	if len(conf.Following) == 0 {
		fmt.Printf("You're not following anyone.\n")
		os.Exit(0)
	}

	cache := Loadcache(configpath)

	now := time.Now().Round(time.Second)

	alltweets := get_tweets(cache)
	sort.Sort(alltweets)
	for _, tweet := range alltweets {
		if *durationFlag == 0 || (now.Sub(tweet.Created)) <= *durationFlag {
			print_tweet(tweet, now)
			fmt.Println()
		}
	}

	cache.Store(configpath)
}

func tweet_command(args []string) error {
	fs := flag.NewFlagSet("tweet", flag.ExitOnError)
	fs.Usage = func() {
		fmt.Printf("usage: %s tweet [words]\n   or: %s twet [words]\n\nAdds a new tweet to your twtfile (words joined together with a single space).\n", progname, progname)
		fs.PrintDefaults()
	}
	fs.Parse(args) // currently using flag.ExitOnError, so we won't get an error on -h

	twtfile := conf.Twtfile
	if len(twtfile) == 0 {
		return errors.New("cannot tweet without twtfile set in config")
	}
	// We don't support shell style ~user/foo.txt :P
	if strings.HasPrefix(twtfile, "~/") {
		twtfile = strings.Replace(twtfile, "~", homedir, 1)
	}

	text := strings.TrimSpace(strings.Join(fs.Args(), " "))
	if len(text) == 0 {
		return errors.New("cowardly refusing to tweet empty text, or only spaces")
	}
	text = fmt.Sprintf("%s\t%s\n", time.Now().Format(time.RFC3339), expand_mentions(text))

	f, err := os.OpenFile(twtfile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer f.Close()

	if _, err := f.WriteString(text); err != nil {
		return err
	}

	return nil
}

// Turns "@nick" into "@<nick URL>" if we're following nick.
func expand_mentions(text string) string {
	re := regexp.MustCompile(`@([_a-zA-Z0-9]+)`)
	return re.ReplaceAllStringFunc(text, func(match string) string {
		parts := re.FindStringSubmatch(match)
		mentionednick := parts[1]

		for followednick, followedurl := range conf.Following {
			if mentionednick == followednick {
				return fmt.Sprintf("@<%s %s>", followednick, followedurl)
			}
		}
		// Not expanding if we're not following
		return match
	})
}
<commit_msg>If no words given, readline the tweet text<commit_after>// -*- tab-width: 4; -*-

package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"regexp"
	"sort"
	"strings"
	"time"

	"gopkg.in/readline.v1"
)

func timeline_command(args []string) {
	fs := flag.NewFlagSet("timeline", flag.ExitOnError)
	durationFlag := fs.Duration("d", 0, "only show tweets created at most `duration` back in time. Example: -d 12h")
	fs.Usage = func() {
		fmt.Printf("usage: %s timeline [arguments]\n\nDisplays the timeline.\n\n", progname)
		fs.PrintDefaults()
	}
	fs.Parse(args) // currently using flag.ExitOnError, so we won't get an error on -h
	if fs.NArg() > 0 {
		fmt.Printf("Too many arguments given.\n")
		os.Exit(2)
	}

	if *durationFlag < 0 {
		fmt.Printf("Negative duration doesn't make sense.\n")
		os.Exit(2)
	}

	if len(conf.Following) == 0 {
		fmt.Printf("You're not following anyone.\n")
		os.Exit(0)
	}

	cache := Loadcache(configpath)

	now := time.Now().Round(time.Second)

	alltweets := get_tweets(cache)
	sort.Sort(alltweets)
	for _, tweet := range alltweets {
		if *durationFlag == 0 || (now.Sub(tweet.Created)) <= *durationFlag {
			print_tweet(tweet, now)
			fmt.Println()
		}
	}

	cache.Store(configpath)
}

func getline() (string, error) {
	rl, err := readline.New("> ")
	if err != nil {
		panic(err)
	}
	defer rl.Close()

	line, err := rl.Readline()
	if err != nil { // io.EOF, readline.ErrInterrupt
		return "", err
	}
	return line, nil
}

func tweet_command(args []string) error {
	fs := flag.NewFlagSet("tweet", flag.ExitOnError)
	fs.Usage = func() {
		fmt.Printf(`usage: %s tweet [words]
   or: %s twet [words]

Adds a new tweet to your twtfile. Words are joined together with a single
space. If no words are given, user will be prompted to input the text
interactively.
`, progname, progname)
		fs.PrintDefaults()
	}
	fs.Parse(args) // currently using flag.ExitOnError, so we won't get an error on -h

	twtfile := conf.Twtfile
	if len(twtfile) == 0 {
		return errors.New("cannot tweet without twtfile set in config")
	}
	// We don't support shell style ~user/foo.txt :P
	if strings.HasPrefix(twtfile, "~/") {
		twtfile = strings.Replace(twtfile, "~", homedir, 1)
	}

	var text string
	if fs.NArg() == 0 {
		var err error
		if text, err = getline(); err != nil {
			return fmt.Errorf("readline: %v", err)
		}
	} else {
		text = strings.TrimSpace(strings.Join(fs.Args(), " "))
		if len(text) == 0 {
			return errors.New("cowardly refusing to tweet empty text, or only spaces")
		}
	}
	text = fmt.Sprintf("%s\t%s\n", time.Now().Format(time.RFC3339), expand_mentions(text))
	f, err := os.OpenFile(twtfile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer f.Close()

	var n int
	if n, err = f.WriteString(text); err != nil {
		return err
	}
	fmt.Printf("appended %d bytes to %s:\n%s", n, conf.Twtfile, text)

	return nil
}

// Turns "@nick" into "@<nick URL>" if we're following nick.
func expand_mentions(text string) string {
	re := regexp.MustCompile(`@([_a-zA-Z0-9]+)`)
	return re.ReplaceAllStringFunc(text, func(match string) string {
		parts := re.FindStringSubmatch(match)
		mentionednick := parts[1]

		for followednick, followedurl := range conf.Following {
			if mentionednick == followednick {
				return fmt.Sprintf("@<%s %s>", followednick, followedurl)
			}
		}
		// Not expanding if we're not following
		return match
	})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/url"
	"os"

	"github.com/Dataman-Cloud/swan/api"
	"github.com/Dataman-Cloud/swan/api/router"
	"github.com/Dataman-Cloud/swan/api/router/application"
	ipamapi "github.com/Dataman-Cloud/swan/api/router/ipam"
	"github.com/Dataman-Cloud/swan/backend"
	"github.com/Dataman-Cloud/swan/health"
	"github.com/Dataman-Cloud/swan/ipam"
	"github.com/Dataman-Cloud/swan/mesosproto/mesos"
	"github.com/Dataman-Cloud/swan/scheduler"
	. "github.com/Dataman-Cloud/swan/store/local"

	"github.com/Dataman-Cloud/swan/types"
	"github.com/Sirupsen/logrus"
	"github.com/andygrunwald/megos"
	"github.com/golang/protobuf/proto"
	"github.com/urfave/cli"
)

var Commands = []cli.Command{
	{
		Name:    "server",
		Aliases: []string{"s"},
		Usage:   "spawn swan server",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "addr",
				Usage: "API Server address <ip:port>",
				Value: "0.0.0.0:9999",
			},

			cli.StringFlag{
				Name:  "masters",
				Usage: "masters address <ip:port>",
				Value: "127.0.0.0:5050",
			},

			cli.StringFlag{
				Name:  "user",
				Usage: "mesos user",
				Value: "root",
			},
		},
		Action: func(c *cli.Context) error {
			ServerCommand(c)
			return nil
		},
	},
}

func ServerCommand(c *cli.Context) {
	hostname, err := os.Hostname()
	if err != nil {
		hostname = "UNKNOWN"
	}

	fw := &mesos.FrameworkInfo{
		User:            proto.String(c.String("user")),
		Name:            proto.String("swan"),
		Hostname:        proto.String(hostname),
		FailoverTimeout: proto.Float64(60 * 60 * 24 * 7),
	}

	store, err := NewBoltStore(".bolt.db")
	if err != nil {
		logrus.Errorf("Init store engine failed:%s", err)
		return
	}

	frameworkId, err := store.FetchFrameworkID()
	if err != nil {
		logrus.Errorf("Fetch framework id failed: %s", err)
		return
	}

	if frameworkId != "" {
		fw.Id = &mesos.FrameworkID{
			Value: proto.String(frameworkId),
		}
	}

	msgQueue := make(chan types.ReschedulerMsg, 1)

	masters := []string{c.String("masters")}
	masterUrls := make([]*url.URL, 0)
	for _, master := range masters {
		masterUrl, _ := url.Parse(fmt.Sprintf("http://%s", master))
		masterUrls = append(masterUrls, masterUrl)
	}

	mesos := megos.NewClient(masterUrls, nil)
	state, err := mesos.GetStateFromCluster()
	if err != nil {
		panic(err)
	}

	cluster := state.Cluster
	if cluster == "" {
		cluster = "Unnamed"
	}

	sched := scheduler.NewScheduler(
		state.Leader,
		fw,
		store,
		cluster,
		health.NewHealthCheckManager(store, msgQueue),
		msgQueue,
	)

	backend := backend.NewBackend(sched, store)

	ipamStore, err := ipam.NewBoltStore(".bolt-foobar.db")
	if err != nil {
		logrus.Errorf("Init store engine failed:%s", err)
		return
	}
	ipamager := ipam.NewIPAM(ipamStore)

	srv := api.NewServer(c.String("addr"))

	routers := []router.Router{
		application.NewRouter(backend),
		ipamapi.NewRouter(ipamager),
	}

	srv.InitRouter(routers...)

	go func() {
		srv.ListenAndServe()
	}()

	<-sched.Start()
}
<commit_msg>remove unused commands.go<commit_after><|endoftext|>