<commit_before>package watcher

import (
	"bufio"
	"crypto/md5"
	"io"
	"log"
	"os"
	"regexp"
	"strings"

	"github.com/smotti/tad/config"
)

const FILE_TYPES = ".*\\.(json|txt|csv|log)"

type (
	// File is represents a file in the dir watched by a Watcher.
	File struct {
		Name     string
		Checksum string // MD5 checksum of the file's content.
	}

	// Watcher keeps an eye on files within a specified dir.
	Watcher struct {
		Dir   string  // The directory to watch.
		Files []*File // A list of files within the directory.
	}
)

func init() {
	files, err := createFileList(*config.Watch)
	if err != nil {
		log.Fatalln("Error:", err)
	}

	for _, v := range files {
		sum, err := calcHashSum(v)
		if err != nil {
			log.Println("Error:", err)
		}
		log.Printf("%x", sum)
	}
}

// createFileList creates a list of files within the given directory and returns
// a string slice.
func createFileList(s string) ([]string, error) {
	// Open the dir so we actually can get any info about it and its childs.
	dir, err := os.Open(strings.Trim(s, "/"))
	if err != nil {
		return nil, err
	}
	defer dir.Close()

	// Get FileInfo about the dir.
	di, err := dir.Stat()
	if err != nil {
		return nil, err
	}

	// Read the dirs content.
	var fi []os.FileInfo
	if di.IsDir() {
		fi, err = dir.Readdir(-1)
		if err != nil {
			return nil, err
		}
	}

	// Create the list of filenames (full path to file).
	files := make([]string, 0)
	for _, f := range fi {
		if !f.IsDir() {
			matched, err := regexp.MatchString(FILE_TYPES, f.Name())
			if err != nil {
				log.Println("Error:", err)
				continue
			}
			if matched {
				files = append(files, dir.Name()+"/"+f.Name())
				log.Println(dir.Name() + "/" + f.Name())
			}
		}
	}

	return files, nil
}

// calcHashSum uses the MD5 hash algorithm to calculate a hash sum of the
// content for the given file. It returns the sum as a byte slice.
func calcHashSum(s string) ([]byte, error) {
	// Open the file.
	file, err := os.Open(s)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	// Get a new hash value.
	h := md5.New()

	// Read line by line and add to the hash sum.
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		if err := scanner.Err(); err != nil {
			return nil, err
		}
		io.WriteString(h, scanner.Text())
	}

	return h.Sum(nil), nil
}

// NewWatcher create a new watcher for the given directory.
//func NewWatcher(dir string) *Watcher {
//}
<commit_msg>Add NewWatcher function to create a new directory watcher<commit_after>package watcher

import (
	"bufio"
	"crypto/md5"
	"io"
	"log"
	"os"
	"regexp"
	"strings"

	"github.com/smotti/tad/config"
)

type (
	// File is represents a file in the dir watched by a Watcher.
	File struct {
		Name     string
		Checksum []byte // MD5 checksum of the file's content.
	}

	// Watcher keeps an eye on files within a specified dir.
	Watcher struct {
		Dir   string  // The directory to watch.
		Files []*File // A list of files within the directory.
	}
)

const FILE_TYPES = ".*\\.(json|txt|csv|log)"

func init() {
	watcher := NewWatcher(*config.Watch)
}

// createFileList creates a list of files within the given directory and returns
// a string slice.
func createFileList(s string) ([]string, error) {
	// Open the dir so we actually can get any info about it and its childs.
	dir, err := os.Open(strings.Trim(s, "/"))
	if err != nil {
		return nil, err
	}
	defer dir.Close()

	// Get FileInfo about the dir.
	di, err := dir.Stat()
	if err != nil {
		return nil, err
	}

	// Read the dirs content.
	var fi []os.FileInfo
	if di.IsDir() {
		fi, err = dir.Readdir(-1)
		if err != nil {
			return nil, err
		}
	}

	// Create the list of filenames (full path to file).
	files := make([]string, 0)
	for _, f := range fi {
		if !f.IsDir() {
			matched, err := regexp.MatchString(FILE_TYPES, f.Name())
			if err != nil {
				log.Println("Error:", err)
				continue
			}
			if matched {
				files = append(files, dir.Name()+"/"+f.Name())
			}
		}
	}

	return files, nil
}

// calcHashSum uses the MD5 hash algorithm to calculate a hash sum of the
// content for the given file. It returns the sum as a byte slice.
func calcHashSum(s string) ([]byte, error) {
	// Open the file.
	file, err := os.Open(s)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	// Get a new hash value.
	h := md5.New()

	// Read line by line and add to the hash sum.
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		if err := scanner.Err(); err != nil {
			return nil, err
		}
		io.WriteString(h, scanner.Text())
	}

	return h.Sum(nil), nil
}

// NewWatcher create a new watcher for the given directory.
func NewWatcher(dir string) *Watcher {
	files, err := createFileList(dir)
	if err != nil {
		log.Fatalln("Error:", err)
	}

	watcher := &Watcher{Dir: dir}
	for _, f := range files {
		h, err := calcHashSum(f)
		if err != nil {
			log.Println("Error:", err)
		}
		watcher.Files = append(watcher.Files, &File{Name: f, Checksum: h})
	}

	return watcher
}
<|endoftext|>


<commit_before>// Package watcher is used for watching files and directories
// for automatic recompilation and restart of app on change
// when in development mode.
package watcher

import (
	"errors"
	"os"
	"path/filepath"
	"sync"

	"github.com/anonx/sunplate/log"

	"gopkg.in/fsnotify.v1"
)

// Type is a watcher type that allows registering new
// pattern - actions pairs.
type Type struct {
	mu       sync.Mutex
	watchers []*fsnotify.Watcher
}

// NewType allocates and returns a new instance of watcher Type.
func NewType() *Type {
	return &Type{}
}

// Listen gets a pattern and a function. The function will be executed
// when files matching the pattern will be modified.
func (t *Type) Listen(pattern string, fn func()) {
	// Create a new watcher.
	w, err := fsnotify.NewWatcher()
	log.AssertNil(err)

	// Find directories matching the pattern.
	ds := glob(pattern)
	if err != nil {
		log.Error.Panicf("Pattern `%s` is malformed. Error: %v.", pattern, err)
	}

	// Add the files to the watcher.
	for i := range ds {
		log.Trace.Printf(`Adding "%s" to the list of watched directories...`, ds[i])
		err := w.Add(ds[i])
		if err != nil {
			log.Warn.Println(err)
		}
	}

	// Add watcher to the list.
	i := len(t.watchers)
	t.watchers = append(t.watchers, w)

	// Start watching process.
	go t.NotifyOnUpdate(i, fn)
}

// NotifyOnUpdate starts the function every time a file change
// event is received. Start it as a goroutine.
func (t *Type) NotifyOnUpdate(watcherIndex int, fn func()) {
	for {
		select {
		case ev := <-t.watchers[watcherIndex].Events:
			if restartRequired(ev) {
				t.mu.Lock()
				log.Warn.Println("Cought event...")
				fn()
				t.mu.Unlock()
			}
		case err := <-t.watchers[watcherIndex].Errors:
			log.Warn.Println(err)
		}
	}
}

// restartRequired checks whether event indicates a file
// has been modified. If so, it returns true.
func restartRequired(event fsnotify.Event) bool {
	if event.Op&fsnotify.Chmod == fsnotify.Chmod {
		return true
	}
	return false
}

// glob returns names of all directories matching pattern or nil.
// The only supported special character is an asterisk at the end.
// It means that the directory is expected to be scanned recursively.
// There is no way for fsnotify to watch individual files (see #17),
// do we support only directories.
// File system errors such as I/O reading are ignored.
func glob(pattern string) (ds []string) {
	// Check whether recursive scan is expected.
	l := len(pattern)
	if l == 0 || pattern[l-1] != '*' {
		ds = append(ds, pattern)
		return // Return as is.
	}

	// Otherwise, trim the asterisk at the end.
	pattern = pattern[:l-1]

	// Start searching directories recursively.
	filepath.Walk(pattern, func(path string, info os.FileInfo, err error) error {
		// Make sure there are no any errors.
		if err != nil {
			return err
		}

		// Make sure the path represents a directory.
		if info.IsDir() {
			ds = append(ds, path) // Add current path to the list.
			return nil
		}

		// Otherwise, return an error.
		return errors.New("not a directory")
	})
	return
}
<commit_msg>Do not store a list of watchers<commit_after>// Package watcher is used for watching files and directories
// for automatic recompilation and restart of app on change
// when in development mode.
package watcher

import (
	"errors"
	"os"
	"path/filepath"
	"sync"

	"github.com/anonx/sunplate/log"

	"gopkg.in/fsnotify.v1"
)

// Type is a watcher type that allows registering new
// pattern - actions pairs.
type Type struct {
	mu sync.Mutex
}

// NewType allocates and returns a new instance of watcher Type.
func NewType() *Type {
	return &Type{}
}

// Listen gets a pattern and a function. The function will be executed
// when files matching the pattern will be modified.
func (t *Type) Listen(pattern string, fn func()) {
	// Create a new watcher.
	w, err := fsnotify.NewWatcher()
	log.AssertNil(err)

	// Find directories matching the pattern.
	ds := glob(pattern)
	if err != nil {
		log.Error.Panicf("Pattern `%s` is malformed. Error: %v.", pattern, err)
	}

	// Add the files to the watcher.
	for i := range ds {
		log.Trace.Printf(`Adding "%s" to the list of watched directories...`, ds[i])
		err := w.Add(ds[i])
		if err != nil {
			log.Warn.Println(err)
		}
	}

	// Start watching process.
	go t.NotifyOnUpdate(w, fn)
}

// NotifyOnUpdate starts the function every time a file change
// event is received. Start it as a goroutine.
func (t *Type) NotifyOnUpdate(watcher *fsnotify.Watcher, fn func()) {
	for {
		select {
		case ev := <-watcher.Events:
			if restartRequired(ev) {
				t.mu.Lock()
				log.Warn.Println("Cought event...")
				fn()
				t.mu.Unlock()
			}
		case err := <-watcher.Errors:
			log.Warn.Println(err)
		}
	}
}

// restartRequired checks whether event indicates a file
// has been modified. If so, it returns true.
func restartRequired(event fsnotify.Event) bool {
	if event.Op&fsnotify.Chmod == fsnotify.Chmod {
		return true
	}
	return false
}

// glob returns names of all directories matching pattern or nil.
// The only supported special character is an asterisk at the end.
// It means that the directory is expected to be scanned recursively.
// There is no way for fsnotify to watch individual files (see #17),
// do we support only directories.
// File system errors such as I/O reading are ignored.
func glob(pattern string) (ds []string) {
	// Check whether recursive scan is expected.
	l := len(pattern)
	if l == 0 || pattern[l-1] != '*' {
		ds = append(ds, pattern)
		return // Return as is.
	}

	// Otherwise, trim the asterisk at the end.
	pattern = pattern[:l-1]

	// Start searching directories recursively.
	filepath.Walk(pattern, func(path string, info os.FileInfo, err error) error {
		// Make sure there are no any errors.
		if err != nil {
			return err
		}

		// Make sure the path represents a directory.
		if info.IsDir() {
			ds = append(ds, path) // Add current path to the list.
			return nil
		}

		// Otherwise, return an error.
		return errors.New("not a directory")
	})
	return
}
<|endoftext|>


<commit_before>// Package app provides application support for context and MongoDB access.
// Current Status Codes:
//		200 OK           : StatusOK                  : Call is success and returning data.
//		204 No Content   : StatusNoContent           : Call is success and returns no data.
//		400 Bad Request  : StatusBadRequest          : Invalid post data (syntax or semantics).
//		401 Unauthorized : StatusUnauthorized        : Authentication failure.
//		404 Not Found    : StatusNotFound            : Invalid URL or identifier.
//		500 Internal     : StatusInternalServerError : Application specific beyond scope of user.
package app

import (
	"encoding/json"
	"net/http"

	"github.com/ardanlabs/kit/auth"
	"github.com/ardanlabs/kit/db"
	"github.com/ardanlabs/kit/log"
)

// Invalid describes a validation error belonging to a specific field.
type Invalid struct {
	Fld string `json:"field_name"`
	Err string `json:"error"`
}

// jsonError is the response for errors that occur within the API.
type jsonError struct {
	Error  string    `json:"error"`
	Fields []Invalid `json:"fields,omitempty"`
}

//==============================================================================

// Context contains data associated with a single request.
type Context struct {
	DB *db.DB
	http.ResponseWriter
	Request   *http.Request
	Params    map[string]string
	SessionID string
	User      *auth.User
	Status    int
}

// Error handles all error responses for the API.
func (c *Context) Error(err error) {
	switch err {
	case ErrNotFound:
		c.RespondError(err.Error(), http.StatusNotFound)
	case ErrInvalidID:
		c.RespondError(err.Error(), http.StatusBadRequest)
	case ErrValidation:
		c.RespondError(err.Error(), http.StatusBadRequest)
	default:
		c.RespondError(err.Error(), http.StatusInternalServerError)
	}
}

// Respond sends JSON to the client.
// If code is StatusNoContent, v is expected to be nil.
func (c *Context) Respond(data interface{}, code int) {
	log.User(c.SessionID, "api : Respond", "Started : Code[%d]", code)

	c.Status = code

	if code == http.StatusNoContent {
		c.WriteHeader(http.StatusNoContent)
		return
	}

	c.Header().Set("Content-Type", "application/json")
	c.WriteHeader(code)
	json.NewEncoder(c).Encode(data)

	log.User(c.SessionID, "api : Respond", "Completed")
}

// RespondInvalid sends JSON describing field validation errors.
func (c *Context) RespondInvalid(fields []Invalid) {
	v := jsonError{
		Error:  "field validation failure",
		Fields: fields,
	}
	c.Respond(v, http.StatusBadRequest)
}

// RespondError sends JSON describing the error
func (c *Context) RespondError(error string, code int) {
	c.Respond(jsonError{Error: error}, code)
}
<commit_msg>Support for JSONP<commit_after>// Package app provides application support for context and MongoDB access.
// Current Status Codes:
//		200 OK           : StatusOK                  : Call is success and returning data.
//		204 No Content   : StatusNoContent           : Call is success and returns no data.
//		400 Bad Request  : StatusBadRequest          : Invalid post data (syntax or semantics).
//		401 Unauthorized : StatusUnauthorized        : Authentication failure.
//		404 Not Found    : StatusNotFound            : Invalid URL or identifier.
//		500 Internal     : StatusInternalServerError : Application specific beyond scope of user.
package app

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/ardanlabs/kit/auth"
	"github.com/ardanlabs/kit/db"
	"github.com/ardanlabs/kit/log"
)

// Invalid describes a validation error belonging to a specific field.
type Invalid struct {
	Fld string `json:"field_name"`
	Err string `json:"error"`
}

// jsonError is the response for errors that occur within the API.
type jsonError struct {
	Error  string    `json:"error"`
	Fields []Invalid `json:"fields,omitempty"`
}

//==============================================================================

// Context contains data associated with a single request.
type Context struct {
	DB *db.DB
	http.ResponseWriter
	Request   *http.Request
	Params    map[string]string
	SessionID string
	User      *auth.User
	Status    int
}

// Error handles all error responses for the API.
func (c *Context) Error(err error) {
	switch err {
	case ErrNotFound:
		c.RespondError(err.Error(), http.StatusNotFound)
	case ErrInvalidID:
		c.RespondError(err.Error(), http.StatusBadRequest)
	case ErrValidation:
		c.RespondError(err.Error(), http.StatusBadRequest)
	default:
		c.RespondError(err.Error(), http.StatusInternalServerError)
	}
}

// Respond sends JSON to the client.
// If code is StatusNoContent, v is expected to be nil.
func (c *Context) Respond(data interface{}, code int) {
	log.User(c.SessionID, "api : Respond", "Started : Code[%d]", code)

	c.Status = code

	if code == http.StatusNoContent {
		c.WriteHeader(http.StatusNoContent)
		return
	}

	c.Header().Set("Content-Type", "application/json")
	c.WriteHeader(code)

	cb, exists := c.Params["callback"]
	if exists {
		b := bytes.NewBufferString(cb + "(")
		json.NewEncoder(b).Encode(data)
		b.WriteString(")")
		fmt.Fprintf(c, b.String())
	} else {
		json.NewEncoder(c).Encode(data)
	}

	log.User(c.SessionID, "api : Respond", "Completed")
}

// RespondInvalid sends JSON describing field validation errors.
func (c *Context) RespondInvalid(fields []Invalid) {
	v := jsonError{
		Error:  "field validation failure",
		Fields: fields,
	}
	c.Respond(v, http.StatusBadRequest)
}

// RespondError sends JSON describing the error
func (c *Context) RespondError(error string, code int) {
	c.Respond(jsonError{Error: error}, code)
}
<|endoftext|>


<commit_before>// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package generate

import (
	"log"

	"github.com/go-swagger/go-swagger/generator"
)

// Client the command to generate a swagger client
type Client struct {
	shared
	Name            string   `long:"name" short:"A" description:"the name of the application, defaults to a mangled value of info.title"`
	Operations      []string `long:"operation" short:"O" description:"specify an operation to include, repeat for multiple"`
	Tags            []string `long:"tags" description:"the tags to include, if not specified defaults to all"`
	Principal       string   `long:"principal" short:"P" description:"the model to use for the security principal"`
	Models          []string `long:"model" short:"M" description:"specify a model to include, repeat for multiple"`
	DefaultScheme   string   `long:"default-scheme" description:"the default scheme for this client" default:"http"`
	DefaultProduces string   `long:"default-produces" description:"the default mime type that API operations produce" default:"application/json"`
	SkipModels      bool     `long:"skip-models" description:"no models will be generated when this flag is specified"`
	SkipOperations  bool     `long:"skip-operations" description:"no operations will be generated when this flag is specified"`
	DumpData        bool     `long:"dump-data" description:"when present dumps the json for the template generator instead of generating files"`
	SkipValidation  bool     `long:"skip-validation" description:"skips validation of spec prior to generation"`
}

func (c *Client) getOpts() (*generator.GenOpts, error) {
	return &generator.GenOpts{
		Spec: string(c.Spec),

		Target:            string(c.Target),
		APIPackage:        c.APIPackage,
		ModelPackage:      c.ModelPackage,
		ServerPackage:     c.ServerPackage,
		ClientPackage:     c.ClientPackage,
		Principal:         c.Principal,
		DefaultScheme:     c.DefaultScheme,
		DefaultProduces:   c.DefaultProduces,
		IncludeModel:      !c.SkipModels,
		IncludeValidator:  !c.SkipModels,
		IncludeHandler:    !c.SkipOperations,
		IncludeParameters: !c.SkipOperations,
		IncludeResponses:  !c.SkipOperations,
		ValidateSpec:      !c.SkipValidation,
		Tags:              c.Tags,
		IncludeSupport:    true,
		Template:          c.Template,
		TemplateDir:       string(c.TemplateDir),
		DumpData:          c.DumpData,
		ExistingModels:    c.ExistingModels,
		IsClient:          true,
	}, nil
}

func (c *Client) getShared() *shared {
	return &c.shared
}

func (c *Client) generate(opts *generator.GenOpts) error {
	return generator.GenerateClient(c.Name, c.Models, c.Operations, opts)
}

func (c *Client) log(rp string) {
	log.Printf(`Generation completed!

For this generation to compile you need to have some packages in your GOPATH:

	* github.com/go-openapi/errors
	* github.com/go-openapi/runtime
	* github.com/go-openapi/runtime/client
	* github.com/go-openapi/strfmt

You can get these now with: go get -u -f %s/...
`, rp)
}

// Execute runs this command
func (c *Client) Execute(args []string) error {
	return createSwagger(c)
}
<commit_msg>Support AllowTemplateOverride in `generate client`<commit_after>// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package generate

import (
	"log"

	"github.com/go-swagger/go-swagger/generator"
)

// Client the command to generate a swagger client
type Client struct {
	shared
	Name            string   `long:"name" short:"A" description:"the name of the application, defaults to a mangled value of info.title"`
	Operations      []string `long:"operation" short:"O" description:"specify an operation to include, repeat for multiple"`
	Tags            []string `long:"tags" description:"the tags to include, if not specified defaults to all"`
	Principal       string   `long:"principal" short:"P" description:"the model to use for the security principal"`
	Models          []string `long:"model" short:"M" description:"specify a model to include, repeat for multiple"`
	DefaultScheme   string   `long:"default-scheme" description:"the default scheme for this client" default:"http"`
	DefaultProduces string   `long:"default-produces" description:"the default mime type that API operations produce" default:"application/json"`
	SkipModels      bool     `long:"skip-models" description:"no models will be generated when this flag is specified"`
	SkipOperations  bool     `long:"skip-operations" description:"no operations will be generated when this flag is specified"`
	DumpData        bool     `long:"dump-data" description:"when present dumps the json for the template generator instead of generating files"`
	SkipValidation  bool     `long:"skip-validation" description:"skips validation of spec prior to generation"`
}

func (c *Client) getOpts() (*generator.GenOpts, error) {
	return &generator.GenOpts{
		Spec: string(c.Spec),

		Target:                string(c.Target),
		APIPackage:            c.APIPackage,
		ModelPackage:          c.ModelPackage,
		ServerPackage:         c.ServerPackage,
		ClientPackage:         c.ClientPackage,
		Principal:             c.Principal,
		DefaultScheme:         c.DefaultScheme,
		DefaultProduces:       c.DefaultProduces,
		IncludeModel:          !c.SkipModels,
		IncludeValidator:      !c.SkipModels,
		IncludeHandler:        !c.SkipOperations,
		IncludeParameters:     !c.SkipOperations,
		IncludeResponses:      !c.SkipOperations,
		ValidateSpec:          !c.SkipValidation,
		Tags:                  c.Tags,
		IncludeSupport:        true,
		Template:              c.Template,
		TemplateDir:           string(c.TemplateDir),
		DumpData:              c.DumpData,
		ExistingModels:        c.ExistingModels,
		AllowTemplateOverride: c.AllowTemplateOverride,
		IsClient:              true,
	}, nil
}

func (c *Client) getShared() *shared {
	return &c.shared
}

func (c *Client) generate(opts *generator.GenOpts) error {
	return generator.GenerateClient(c.Name, c.Models, c.Operations, opts)
}

func (c *Client) log(rp string) {
	log.Printf(`Generation completed!

For this generation to compile you need to have some packages in your GOPATH:

	* github.com/go-openapi/errors
	* github.com/go-openapi/runtime
	* github.com/go-openapi/runtime/client
	* github.com/go-openapi/strfmt

You can get these now with: go get -u -f %s/...
`, rp)
}

// Execute runs this command
func (c *Client) Execute(args []string) error {
	return createSwagger(c)
}
<|endoftext|>


<commit_before>package vm

import "fmt"

type Error struct {
	name, message string
	child         *Thunk
}

func NewError(n, m string, xs ...interface{}) *Thunk {
	return ChainError(NewNil(), n, m, xs...)
}

func ChainError(e *Thunk, n, m string, xs ...interface{}) *Thunk {
	return Normal(Error{
		name:    n,
		message: fmt.Sprintf(m, xs...),
		child:   e,
	})
}

func NotCallableError(o Object) *Thunk {
	return TypeError(o, "Callable")
}

func TypeError(o Object, typ string) *Thunk {
	n, m := "TypeError", "%#v is not %s"

	if e, ok := o.(Error); ok {
		return ChainError(Normal(e), n, m, o, typ)
	}

	return NewError(n, m, o, typ)
}

func NumArgsError(f, condition string) *Thunk {
	return NewError(
		"NumArgsError",
		"Number of arguments to %s must be %s.", f, condition)
}

func ValueError(m string) *Thunk {
	return NewError("ValueError", m)
}
<commit_msg>Comment about lazy Error<commit_after>package vm

import "fmt"

type Error struct {
	// Errors should be lazy maybe.
	name, message string
	child         *Thunk
}

func NewError(n, m string, xs ...interface{}) *Thunk {
	return ChainError(NewNil(), n, m, xs...)
}

func ChainError(e *Thunk, n, m string, xs ...interface{}) *Thunk {
	return Normal(Error{
		name:    n,
		message: fmt.Sprintf(m, xs...),
		child:   e,
	})
}

func NotCallableError(o Object) *Thunk {
	return TypeError(o, "Callable")
}

func TypeError(o Object, typ string) *Thunk {
	n, m := "TypeError", "%#v is not %s"

	if e, ok := o.(Error); ok {
		return ChainError(Normal(e), n, m, o, typ)
	}

	return NewError(n, m, o, typ)
}

func NumArgsError(f, condition string) *Thunk {
	return NewError(
		"NumArgsError",
		"Number of arguments to %s must be %s.", f, condition)
}

func ValueError(m string) *Thunk {
	return NewError("ValueError", m)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

type SupervisorCon struct {
	filtersCache    []*Filter
	filtersCacheMux sync.RWMutex
}

type Filter struct {
	Regex      string `json:"regex"`
	Name       string `json:"name"`
	ClientHost string `json:"client_host"`
	Id         string `json:"id"`
}

// Returns a map of metricId => timestamp => count
func (f *Filter) GetStats(window int64, rollup int64) (map[int]map[int64]int64, error) {
	// Request
	uri := fmt.Sprintf("filter/%s/stats", f.Id)
	data, err := supervisorCon._get(uri)
	if err != nil {
		return nil, err
	}

	// Parse JSON
	var d map[string]interface{}
	je := json.Unmarshal([]byte(data), &d)
	if je != nil {
		return nil, je
	}

	// Now
	var nowUnix int64 = time.Now().Unix()
	var minTs int64 = nowUnix - window

	// To map + rollup
	var res map[int]map[int64]int64 = make(map[int]map[int64]int64)
	for metricId, data := range d["stats"].(map[string]interface{}) {
		// Convert metric to integer
		i, _ := strconv.ParseInt(metricId, 10, 0)
		metric := int(i)
		if res[metric] == nil {
			res[metric] = make(map[int64]int64)
		}

		// Iterate timestamp-value pairs
		var lowestBucket int64 = math.MaxInt64
		var highestBucket int64 = math.MinInt64
		for tsStr, val := range data.(map[string]interface{}) {
			// Convert to integer
			tsI, _ := strconv.ParseInt(tsStr, 10, 64)
			ts := int64(tsI)

			// Honor window
			if ts < minTs {
				continue
			}

			// Determine bucket
			bucket := ts
			if rollup != -1 {
				bucket = ts - (ts % rollup)
			}

			// Store first bucket
			if bucket < lowestBucket {
				lowestBucket = bucket
			}
			if bucket > highestBucket {
				highestBucket = bucket
			}

			// Put in resultset
			res[metric][bucket] += int64(val.(float64))
		}

		// Fill any gaps
		// log.Printf("first key %d", lowestBucket)
		// log.Printf("last key %d", highestBucket)
		// log.Printf("step size %d", rollup)
		// log.Printf("len before %d", len(res[metric]))
		for i := lowestBucket; i < highestBucket; i += rollup {
			// Explictly set the value
			if res[metric][i] == 0 {
				res[metric][i] = 0
			}
		}
		// log.Printf("len after %d", len(res[metric]))

	}

	return res, nil
}

func (s *SupervisorCon) Search(q string) (string, error) {
	// @todo setting to configure default search backend
	data, err := supervisorCon._postData("bigquery/query", q)
	return data, err
}

func (s *SupervisorCon) Connect() bool {
	if verbose {
		log.Printf("Connecting to %s", session["supervisor_uri"])
	}
	_, err := s._get("filter")
	if err == nil {
		if !silent {
			fmt.Printf("Connected to %s\n", session["supervisor_uri"])
		}
	} else {
		fmt.Printf("Failed to connect: %s", err)
		return false
	}
	return true
}

func (s *SupervisorCon) SetSupervisorConfig(k string, v string) (bool, error) {
	_, err := s._put(fmt.Sprintf("admin/config?key=%s&value=%s", url.QueryEscape(k), url.QueryEscape(v)))
	if err == nil {
		return false, err
	}
	return true, nil
}

func (s *SupervisorCon) Ping() {
	start := time.Now()
	_, err := s._get("ping")
	if err == nil {
		duration := time.Now().Sub(start)
		fmt.Printf("Pong, took %s\n", duration.String())
	}
}

func (s *SupervisorCon) CreateFilter(name string, regex string) (*Filter, error) {
	if verbose {
		log.Printf("Creating filter '%s' with regex '%s'", name, regex)
	}
	// Create
	s._post(fmt.Sprintf("filter?name=%s&regex=%s", url.QueryEscape(name), url.QueryEscape(regex)))

	// Clear cache
	s.filtersCacheMux.Lock()
	s.filtersCache = nil
	s.filtersCacheMux.Unlock()

	// Return instance
	return s.FilterByName(name)
}

func (s *SupervisorCon) RemoveFilter(name string) bool {
	if verbose {
		log.Printf("Deleting filter '%s'", name)
	}
	filter, e := s.FilterByName(name)
	if e != nil {
		return false
	}
	s._delete(fmt.Sprintf("filter/%s", url.QueryEscape(filter.Id)))

	// Clear cache
	s.filtersCacheMux.Lock()
	s.filtersCache = nil
	s.filtersCacheMux.Unlock()

	// Verify
	verify, _ := s.FilterByName(name)
	return verify == nil
}

func isUuid(in string) bool {
	isUuid, _ := regexp.MatchString("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", in)
	return isUuid
}

func (s *SupervisorCon) FilterByName(name string) (*Filter, error) {
	if isUuid(name) {
		return s.FilterById(name)
	}
	filters, err := s.Filters()
	if err != nil {
		return nil, err
	}
	for _, filter := range filters {
		if strings.ToLower(filter.Name) == strings.ToLower(name) {
			return filter, nil
		}
	}
	return nil, errors.New(fmt.Sprintf("Filter '%s' not found", name))
}

func (s *SupervisorCon) FilterById(id string) (*Filter, error) {
	filters, err := s.Filters()
	if err != nil {
		return nil, err
	}
	for _, filter := range filters {
		if id == filter.Id {
			return filter, nil
		}
	}
	return nil, errors.New(fmt.Sprintf("Filter '%s' not found", id))
}

func (s *SupervisorCon) Filters() ([]*Filter, error) {
	// Update function
	fetchData := func() ([]*Filter, error) {
		// List holder
		list := make([]*Filter, 0)

		// Fetch API
		data, err := s._get("filter")
		if err != nil {
			return nil, err
		}
		// Parse and create list
		var resp map[string]interface{}
		jErr := json.Unmarshal([]byte(data), &resp)
		if jErr != nil {
			return nil, jErr
		}
		for _, v := range resp["filters"].([]interface{}) {
			elm := v.(map[string]interface{})
			filter := newFilter()
			filter.Regex = fmt.Sprintf("%s", elm["regex"])
			filter.Name = fmt.Sprintf("%s", elm["name"])
			filter.ClientHost = fmt.Sprintf("%s", elm["client_host"])
			filter.Id = fmt.Sprintf("%s", elm["id"])

			// Tmp?
			if strings.HasPrefix(filter.Name, TMP_FILTER_PREFIX) {
				// @todo Remove if they are older than x days
				// go func(id string) {
				// 	s._delete(fmt.Sprintf("filter/%s", url.QueryEscape(id)))
				// }(filter.Id)
				continue
			}

			// Append
			list = append(list, filter)
		}

		// Put in cache
		if len(list) > 0 {
			s.filtersCacheMux.Lock()
			s.filtersCache = list
			s.filtersCacheMux.Unlock()
		}

		// Return
		return list, nil
	}

	// From cache?
	s.filtersCacheMux.RLock()
	if s.filtersCache != nil {
		s.filtersCacheMux.RUnlock()
		go fetchData() // Async update
		return s.filtersCache, nil
	}
	s.filtersCacheMux.RUnlock()

	// Not from cache
	return fetchData()
}

func (s *SupervisorCon) _get(uri string) (string, error) {
	return s._doRequest("GET", uri, "")
}

func (s *SupervisorCon) _put(uri string) (string, error) {
	return s._doRequest("PUT", uri, "")
}

func (s *SupervisorCon) _post(uri string) (string, error) {
	return s._doRequest("POST", uri, "")
}

func (s *SupervisorCon) _putData(uri string, data string) (string, error) {
	return s._doRequest("PUT", uri, data)
}

func (s *SupervisorCon) _postData(uri string, data string) (string, error) {
	return s._doRequest("POST", uri, data)
}

func (s *SupervisorCon) _delete(uri string) (string, error) {
	return s._doRequest("DELETE", uri, "")
}

func (s *SupervisorCon) _doRequest(method string, uri string, data string) (string, error) {
	// Client
	client := s._getHttpClient()

	// Request
	var reqBody *bytes.Buffer
	if len(data) > 0 {
		reqBody = bytes.NewBuffer([]byte(data))
	} else {
		reqBody = bytes.NewBuffer(make([]byte, 0))
	}
	req, err := http.NewRequest(method, fmt.Sprintf("%s%s", session["supervisor_uri"], uri), reqBody)
	if err != nil {
		return "", err
	}

	// Auth header
	req.Header.Add("Authorization", fmt.Sprintf("Basic %s", s._getBasicAuthToken()))

	// Execute
	resp, respErr := client.Do(req)
	if respErr != nil {
		return "", respErr
	}

	// Status
	if resp.StatusCode >= 400 {
		return "", errors.New(fmt.Sprintf("Status %d", resp.StatusCode))
	}

	// Read body
	defer resp.Body.Close()
	contents, readErr := ioutil.ReadAll(resp.Body)
	if readErr != nil {
		return "", readErr
	}
	str := string(contents)
	if verbose {
		log.Printf("Received body %s", str)
	}
	return str, nil
}

func (s *SupervisorCon) _getBasicAuthToken() string {
	return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", session["supervisor_username"], session["supervisor_password"])))
}

func (s *SupervisorCon) _getHttpClient() *http.Client {
	client := &http.Client{}
	return client
}

func NewSupervisorCon() *SupervisorCon {
	return &SupervisorCon{}
}

func newFilter() *Filter {
	return &Filter{}
}
<commit_msg>Auto-remove filters after a while<commit_after>package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

type SupervisorCon struct {
	filtersCache    []*Filter
	filtersCacheMux sync.RWMutex
}

type Filter struct {
	Regex      string `json:"regex"`
	Name       string `json:"name"`
	ClientHost string `json:"client_host"`
	Id         string `json:"id"`
}

// Returns a map of metricId => timestamp => count
func (f *Filter) GetStats(window int64, rollup int64) (map[int]map[int64]int64, error) {
	// Request
	uri := fmt.Sprintf("filter/%s/stats", f.Id)
	data, err := supervisorCon._get(uri)
	if err != nil {
		return nil, err
	}

	// Parse JSON
	var d map[string]interface{}
	je := json.Unmarshal([]byte(data), &d)
	if je != nil {
		return nil, je
	}

	// Now
	var nowUnix int64 = time.Now().Unix()
	var minTs int64 = nowUnix - window

	// To map + rollup
	var res map[int]map[int64]int64 = make(map[int]map[int64]int64)
	for metricId, data := range d["stats"].(map[string]interface{}) {
		// Convert metric to integer
		i, _ := strconv.ParseInt(metricId, 10, 0)
		metric := int(i)
		if res[metric] == nil {
			res[metric] = make(map[int64]int64)
		}

		// Iterate timestamp-value pairs
		var lowestBucket int64 = math.MaxInt64
		var highestBucket int64 = math.MinInt64
		for tsStr, val := range data.(map[string]interface{}) {
			// Convert to integer
			tsI, _ := strconv.ParseInt(tsStr, 10, 64)
			ts := int64(tsI)

			// Honor window
			if ts < minTs {
				continue
			}

			// Determine bucket
			bucket := ts
			if rollup != -1 {
				bucket = ts - (ts % rollup)
			}

			// Store first bucket
			if bucket < lowestBucket {
				lowestBucket = bucket
			}
			if bucket > highestBucket {
				highestBucket = bucket
			}

			// Put in resultset
			res[metric][bucket] += int64(val.(float64))
		}

		// Fill any gaps
		// log.Printf("first key %d", lowestBucket)
		// log.Printf("last key %d", highestBucket)
		// log.Printf("step size %d", rollup)
		// log.Printf("len before %d", len(res[metric]))
		for i := lowestBucket; i < highestBucket; i += rollup {
			// Explictly set the value
			if res[metric][i] == 0 {
				res[metric][i] = 0
			}
		}
		// log.Printf("len after %d", len(res[metric]))

	}

	return res, nil
}

func (s *SupervisorCon) Search(q string) (string, error) {
	// @todo setting to configure default search backend
	data, err := supervisorCon._postData("bigquery/query", q)
	return data, err
}

func (s *SupervisorCon) Connect() bool {
	if verbose {
		log.Printf("Connecting to %s", session["supervisor_uri"])
	}
	_, err := s._get("filter")
	if err == nil {
		if !silent {
			fmt.Printf("Connected to %s\n", session["supervisor_uri"])
		}
	} else {
		fmt.Printf("Failed to connect: %s", err)
		return false
	}
	return true
}

func (s *SupervisorCon) SetSupervisorConfig(k string, v string) (bool, error) {
	_, err := s._put(fmt.Sprintf("admin/config?key=%s&value=%s", url.QueryEscape(k), url.QueryEscape(v)))
	if err == nil {
		return false, err
	}
	return true, nil
}

func (s *SupervisorCon) Ping() {
	start := time.Now()
	_, err := s._get("ping")
	if err == nil {
		duration := time.Now().Sub(start)
		fmt.Printf("Pong, took %s\n", duration.String())
	}
}

func (s *SupervisorCon) CreateFilter(name string, regex string) (*Filter, error) {
	if verbose {
		log.Printf("Creating filter '%s' with regex '%s'", name, regex)
	}
	// Create
	s._post(fmt.Sprintf("filter?name=%s&regex=%s", url.QueryEscape(name), url.QueryEscape(regex)))

	// Clear cache
	s.filtersCacheMux.Lock()
	s.filtersCache = nil
	s.filtersCacheMux.Unlock()

	// Return instance
	return s.FilterByName(name)
}

func (s *SupervisorCon) RemoveFilter(name string) bool {
	if verbose {
		log.Printf("Deleting filter '%s'", name)
	}
	filter, e := s.FilterByName(name)
	if e != nil {
		return false
	}
	s._delete(fmt.Sprintf("filter/%s", url.QueryEscape(filter.Id)))

	// Clear cache
	s.filtersCacheMux.Lock()
	s.filtersCache = nil
	s.filtersCacheMux.Unlock()

	// Verify
	verify, _ := s.FilterByName(name)
	return verify == nil
}

func isUuid(in string) bool {
	isUuid, _ := regexp.MatchString("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", in)
	return isUuid
}

func (s *SupervisorCon) FilterByName(name string) (*Filter, error) {
	if isUuid(name) {
		return s.FilterById(name)
	}
	filters, err := s.Filters()
	if err != nil {
		return nil, err
	}
	for _, filter := range filters {
		if strings.ToLower(filter.Name) == strings.ToLower(name) {
			return filter, nil
		}
	}
	return nil, errors.New(fmt.Sprintf("Filter '%s' not found", name))
}

func (s *SupervisorCon) FilterById(id string) (*Filter, error) {
	filters, err := s.Filters()
	if err != nil {
		return nil, err
	}
	for _, filter := range filters {
		if id == filter.Id {
			return filter, nil
		}
	}
	return nil, errors.New(fmt.Sprintf("Filter '%s' not found", id))
}

func (s *SupervisorCon) Filters() ([]*Filter, error) {
	// Update function
	fetchData := func() ([]*Filter, error) {
		// List holder
		list := make([]*Filter, 0)

		// Fetch API
		data, err := s._get("filter")
		if err != nil {
			return nil, err
		}
		// Parse and create list
		var resp map[string]interface{}
		jErr := json.Unmarshal([]byte(data), &resp)
		if jErr != nil {
			return nil, jErr
		}
		for _, v := range resp["filters"].([]interface{}) {
			elm := v.(map[string]interface{})
			filter := newFilter()
			filter.Regex = fmt.Sprintf("%s", elm["regex"])
			filter.Name = fmt.Sprintf("%s", elm["name"])
			filter.ClientHost = fmt.Sprintf("%s", elm["client_host"])
			filter.Id = fmt.Sprintf("%s", elm["id"])

			// Tmp?
			if strings.HasPrefix(filter.Name, TMP_FILTER_PREFIX) {
				// Remove if they are older than x hours
				tsStr := filter.Name[len(TMP_FILTER_PREFIX):]
				tsVal, tsE := strconv.ParseInt(tsStr, 10, 64)
				if tsE != nil {
					tsVal = 0
				}
				tsNow := time.Now().Unix()
				maxHours := int64(1)
				tsMin := tsNow - (maxHours * 3600)
				if tsVal < tsMin {
					if verbose {
						log.Println(fmt.Sprintf("Removing stale filter %s", filter.Id))
					}
					go func(id string) {
						s._delete(fmt.Sprintf("filter/%s", url.QueryEscape(id)))
					}(filter.Id)
				}

				continue
			}

			// Append
			list = append(list, filter)
		}

		// Put in cache
		if len(list) > 0 {
			s.filtersCacheMux.Lock()
			s.filtersCache = list
			s.filtersCacheMux.Unlock()
		}

		// Return
		return list, nil
	}

	// From cache?
	s.filtersCacheMux.RLock()
	if s.filtersCache != nil {
		s.filtersCacheMux.RUnlock()
		go fetchData() // Async update
		return s.filtersCache, nil
	}
	s.filtersCacheMux.RUnlock()

	// Not from cache
	return fetchData()
}

func (s *SupervisorCon) _get(uri string) (string, error) {
	return s._doRequest("GET", uri, "")
}

func (s *SupervisorCon) _put(uri string) (string, error) {
	return s._doRequest("PUT", uri, "")
}

func (s *SupervisorCon) _post(uri string) (string, error) {
	return s._doRequest("POST", uri, "")
}

func (s *SupervisorCon) _putData(uri string, data string) (string, error) {
	return s._doRequest("PUT", uri, data)
}

func (s *SupervisorCon) _postData(uri string, data string) (string, error) {
	return s._doRequest("POST", uri, data)
}

func (s *SupervisorCon) _delete(uri string) (string, error) {
	return s._doRequest("DELETE", uri, "")
}

func (s *SupervisorCon) _doRequest(method string, uri string, data string) (string, error) {
	// Client
	client := s._getHttpClient()

	// Request
	var reqBody *bytes.Buffer
	if len(data) > 0 {
		reqBody = bytes.NewBuffer([]byte(data))
	} else {
		reqBody = bytes.NewBuffer(make([]byte, 0))
	}
	req, err := http.NewRequest(method, fmt.Sprintf("%s%s", session["supervisor_uri"], uri), reqBody)
	if err != nil {
		return "", err
	}

	// Auth header
	req.Header.Add("Authorization", fmt.Sprintf("Basic %s", s._getBasicAuthToken()))

	// Execute
	resp, respErr := client.Do(req)
	if respErr != nil {
		return "", respErr
	}

	// Status
	if resp.StatusCode >= 400 {
		return "", errors.New(fmt.Sprintf("Status %d", resp.StatusCode))
	}

	// Read body
	defer resp.Body.Close()
	contents, readErr := ioutil.ReadAll(resp.Body)
	if readErr != nil {
		return "", readErr
	}
	str := string(contents)
	if verbose {
		log.Printf("Received body %s", str)
	}
	return str, nil
}

func (s *SupervisorCon) _getBasicAuthToken() string {
	return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", session["supervisor_username"], session["supervisor_password"])))
}

func (s *SupervisorCon) _getHttpClient() *http.Client {
	client := &http.Client{}
	return client
}

func NewSupervisorCon() *SupervisorCon {
	return &SupervisorCon{}
}

func newFilter() *Filter {
	return &Filter{}
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package client

import (
	"errors"
	"fmt"
	"math"
	"sync"
	"time"

	"github.com/m3db/m3db/interfaces/m3db"
	"github.com/m3db/m3db/network/server/tchannelthrift/thrift/gen-go/rpc"

	"github.com/uber/tchannel-go/thrift"
)

var (
	errQueueNotOpen          = errors.New("host operation queue not open")
	errQueueUnknownOperation = errors.New("unknown operation")
)

type hostQueue interface {
	// Open the host queue
	Open()

	// Len returns the length of the queue
	Len() int

	// Enqueue an operation
	Enqueue(op m3db.Op) error

	// GetConnectionCount gets the current open connection count
	GetConnectionCount() int

	// Close the host queue, will flush any operations still pending
	Close()
}

type queue struct {
	sync.RWMutex

	opts                  m3db.ClientOptions
	host                  m3db.Host
	connPool              connectionPool
	writeBatchRequestPool writeBatchRequestPool
	writeRequestArrayPool writeRequestArrayPool
	size                  int
	ops                   []m3db.Op
	opsArrayPool          opArrayPool
	drainIn               chan []m3db.Op
	state                 state
}

func newHostQueue(
	host m3db.Host,
	writeBatchRequestPool writeBatchRequestPool,
	writeRequestArrayPool writeRequestArrayPool,
	opts m3db.ClientOptions,
) hostQueue {
	size := opts.GetHostQueueOpsFlushSize()

	opArrayPoolCapacity := int(math.Max(float64(size), float64(opts.GetWriteBatchSize())))
	opArrayPool := newOpArrayPool(opts.GetHostQueueOpsArrayPoolSize(), opArrayPoolCapacity)

	return &queue{
		opts:                  opts,
		host:                  host,
		connPool:              newConnectionPool(host, opts),
		writeBatchRequestPool: writeBatchRequestPool,
		writeRequestArrayPool: writeRequestArrayPool,
		size:         size,
		ops:          opArrayPool.Get(),
		opsArrayPool: opArrayPool,
		// NB(r): specifically use non-buffered queue for single flush at a time
		drainIn: make(chan []m3db.Op),
	}
}

func (q *queue) Open() {
	q.Lock()
	defer q.Unlock()

	if q.state != stateNotOpen {
		return
	}

	q.state = stateOpen

	// Open the connection pool
	q.connPool.Open()

	// Continually drain the queue until closed
	go q.drain()

	flushInterval := q.opts.GetHostQueueOpsFlushInterval()
	if flushInterval > 0 {
		// Continually flush the queue at given interval if set
		go q.flushEvery(flushInterval)
	}
}

func (q *queue) flushEvery(interval time.Duration) {
	for {
		q.RLock()
		state := q.state
		q.RUnlock()
		if state != stateOpen {
			return
		}

		time.Sleep(interval)

		q.Lock()
		if state != stateOpen {
			q.Unlock()
			return
		}
		q.flushWithLock()
		q.Unlock()
	}
}

func (q *queue) flushWithLock() {
	// Pass the current ops to drain
	q.drainIn <- q.ops

	// Reset ops
	q.ops = q.opsArrayPool.Get()
}

func (q *queue) drain() {
	wgAll := &sync.WaitGroup{}
	for {
		ops := <-q.drainIn

		var (
			currWriteOps      []m3db.Op
			currWriteRequests []*rpc.WriteRequest
			writeBatchSize    = q.opts.GetWriteBatchSize()
			opsLen            = len(ops)
		)
		for i := 0; i < opsLen; i++ {
			switch v := ops[i].(type) {
			case *writeOp:
				if currWriteOps == nil {
					currWriteOps = q.opsArrayPool.Get()
					currWriteRequests = q.writeRequestArrayPool.Get()
				}

				currWriteOps = append(currWriteOps, ops[i])
				currWriteRequests = append(currWriteRequests, &v.request)

				if len(currWriteOps) == writeBatchSize {
					// Reached write batch limit, write async and reset
					q.asyncWrite(wgAll, currWriteOps, currWriteRequests)
					currWriteOps = nil
					currWriteRequests = nil
				}
			default:
				completionFn := ops[i].GetCompletionFn()
				completionFn(nil, errQueueUnknownOperation)
			}
		}

		// If any outstanding write ops, async write
		if len(currWriteOps) > 0 {
			q.asyncWrite(wgAll, currWriteOps, currWriteRequests)
		}

		q.opsArrayPool.Put(ops)

		q.RLock()
		state := q.state
		q.RUnlock()

		if state != stateOpen {
			// Final drain, close the connection pool after all requests done
			wgAll.Wait()
			q.connPool.Close()
			return
		}
	}
}

func (q *queue) asyncWrite(wg *sync.WaitGroup, ops []m3db.Op, elems []*rpc.WriteRequest) {
	wg.Add(1)
	// TODO(r): Use a worker pool to avoid creating new go routines for async writes
	go func() {
		req := q.writeBatchRequestPool.Get()
		req.Elements = elems

		// NB(r): Defer is slow in the hot path unfortunately
		cleanup := func() {
			q.writeBatchRequestPool.Put(req)
			q.writeRequestArrayPool.Put(elems)
			q.opsArrayPool.Put(ops)
			wg.Done()
		}

		client, err := q.connPool.NextClient()
		if err != nil {
			// No client available
			callAllCompletionFns(ops, nil, err)
			cleanup()
			return
		}

		ctx, _ := thrift.NewContext(q.opts.GetWriteRequestTimeout())
		err = client.WriteBatch(ctx, req)
		if err == nil {
			// All succeeded
			callAllCompletionFns(ops, nil, nil)
			cleanup()
			return
		}

		if batchErrs, ok := err.(*rpc.WriteBatchErrors); ok {
			// Callback all writes with errors
			hasErr := make(map[int]struct{})
			for _, batchErr := range batchErrs.Errors {
				op := ops[batchErr.ElementErrorIndex]
				op.GetCompletionFn()(nil, fmt.Errorf(batchErr.Error.Message))
				hasErr[int(batchErr.ElementErrorIndex)] = struct{}{}
			}
			// Callback all writes with no errors
			for i := range ops {
				if _, ok := hasErr[i]; !ok {
					// No error
					ops[i].GetCompletionFn()(nil, nil)
				}
			}
			cleanup()
			return
		}

		// Entire batch failed
		callAllCompletionFns(ops, nil, err)
		cleanup()
	}()
}

func (q *queue) Len() int {
	q.RLock()
	v := len(q.ops)
	q.RUnlock()
	return v
}

func (q *queue) Enqueue(o m3db.Op) error {
	q.Lock()
	if q.state != stateOpen {
		q.Unlock()
		return errQueueNotOpen
	}
	q.ops = append(q.ops, o)
	// If queue is full flush
	if len(q.ops) == q.size {
		q.flushWithLock()
	}
	q.Unlock()
	return nil
}

func (q *queue) GetConnectionCount() int {
	return q.connPool.GetConnectionCount()
}

func (q *queue) Close() {
	q.Lock()
	defer q.Unlock()

	if q.state != stateOpen {
		return
	}

	q.state = stateClosed

	// Flush any remaining ops and stop the drain cycle
	q.flushWithLock()
}
<commit_msg>Avoid deadlock scenario from host queue when drain queue checks state (#11)<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package client

import (
	"errors"
	"fmt"
	"math"
	"sync"
	"time"

	"github.com/m3db/m3db/interfaces/m3db"
	"github.com/m3db/m3db/network/server/tchannelthrift/thrift/gen-go/rpc"

	"github.com/uber/tchannel-go/thrift"
)

var (
	errQueueNotOpen          = errors.New("host operation queue not open")
	errQueueUnknownOperation = errors.New("unknown operation")
)

type hostQueue interface {
	// Open the host queue
	Open()

	// Len returns the length of the queue
	Len() int

	// Enqueue an operation
	Enqueue(op m3db.Op) error

	// GetConnectionCount gets the current open connection count
	GetConnectionCount() int

	// Close the host queue, will flush any operations still pending
	Close()
}

type queue struct {
	sync.RWMutex

	opts                  m3db.ClientOptions
	host                  m3db.Host
	connPool              connectionPool
	writeBatchRequestPool writeBatchRequestPool
	writeRequestArrayPool writeRequestArrayPool
	size                  int
	ops                   []m3db.Op
	opsArrayPool          opArrayPool
	drainIn               chan []m3db.Op
	state                 state
}

func newHostQueue(
	host m3db.Host,
	writeBatchRequestPool writeBatchRequestPool,
	writeRequestArrayPool writeRequestArrayPool,
	opts m3db.ClientOptions,
) hostQueue {
	size := opts.GetHostQueueOpsFlushSize()

	opArrayPoolCapacity := int(math.Max(float64(size), float64(opts.GetWriteBatchSize())))
	opArrayPool := newOpArrayPool(opts.GetHostQueueOpsArrayPoolSize(), opArrayPoolCapacity)

	return &queue{
		opts:                  opts,
		host:                  host,
		connPool:              newConnectionPool(host, opts),
		writeBatchRequestPool: writeBatchRequestPool,
		writeRequestArrayPool: writeRequestArrayPool,
		size:         size,
		ops:          opArrayPool.Get(),
		opsArrayPool: opArrayPool,
		// NB(r): specifically use non-buffered queue for single flush at a time
		drainIn: make(chan []m3db.Op),
	}
}

func (q *queue) Open() {
	q.Lock()
	defer q.Unlock()

	if q.state != stateNotOpen {
		return
	}

	q.state = stateOpen

	// Open the connection pool
	q.connPool.Open()

	// Continually drain the queue until closed
	go q.drain()

	flushInterval := q.opts.GetHostQueueOpsFlushInterval()
	if flushInterval > 0 {
		// Continually flush the queue at given interval if set
		go q.flushEvery(flushInterval)
	}
}

func (q *queue) flushEvery(interval time.Duration) {
	for {
		q.RLock()
		state := q.state
		q.RUnlock()
		if state != stateOpen {
			return
		}

		time.Sleep(interval)

		q.Lock()
		if state != stateOpen {
			q.Unlock()
			return
		}
		needsDrain := q.rotateOpsWithLock()
		q.Unlock()

		if len(needsDrain) != 0 {
			q.drainIn <- needsDrain
		}
	}
}

func (q *queue) rotateOpsWithLock() []m3db.Op {
	if len(q.ops) == 0 {
		// No need to rotate as queue is empty
		return nil
	}

	needsDrain := q.ops

	// Reset ops
	q.ops = q.opsArrayPool.Get()

	return needsDrain
}

func (q *queue) drain() {
	wgAll := &sync.WaitGroup{}
	for {
		ops, ok := <-q.drainIn
		if !ok {
			break
		}
		var (
			currWriteOps      []m3db.Op
			currWriteRequests []*rpc.WriteRequest
			writeBatchSize    = q.opts.GetWriteBatchSize()
			opsLen            = len(ops)
		)
		for i := 0; i < opsLen; i++ {
			switch v := ops[i].(type) {
			case *writeOp:
				if currWriteOps == nil {
					currWriteOps = q.opsArrayPool.Get()
					currWriteRequests = q.writeRequestArrayPool.Get()
				}

				currWriteOps = append(currWriteOps, ops[i])
				currWriteRequests = append(currWriteRequests, &v.request)

				if len(currWriteOps) == writeBatchSize {
					// Reached write batch limit, write async and reset
					q.asyncWrite(wgAll, currWriteOps, currWriteRequests)
					currWriteOps = nil
					currWriteRequests = nil
				}
			default:
				completionFn := ops[i].GetCompletionFn()
				completionFn(nil, errQueueUnknownOperation)
			}
		}

		// If any outstanding write ops, async write
		if len(currWriteOps) > 0 {
			q.asyncWrite(wgAll, currWriteOps, currWriteRequests)
		}

		if ops != nil {
			q.opsArrayPool.Put(ops)
		}
	}

	// Close the connection pool after all requests done
	wgAll.Wait()
	q.connPool.Close()
}

func (q *queue) asyncWrite(wg *sync.WaitGroup, ops []m3db.Op, elems []*rpc.WriteRequest) {
	wg.Add(1)
	// TODO(r): Use a worker pool to avoid creating new go routines for async writes
	go func() {
		req := q.writeBatchRequestPool.Get()
		req.Elements = elems

		// NB(r): Defer is slow in the hot path unfortunately
		cleanup := func() {
			q.writeBatchRequestPool.Put(req)
			q.writeRequestArrayPool.Put(elems)
			q.opsArrayPool.Put(ops)
			wg.Done()
		}

		client, err := q.connPool.NextClient()
		if err != nil {
			// No client available
			callAllCompletionFns(ops, nil, err)
			cleanup()
			return
		}

		ctx, _ := thrift.NewContext(q.opts.GetWriteRequestTimeout())
		err = client.WriteBatch(ctx, req)
		if err == nil {
			// All succeeded
			callAllCompletionFns(ops, nil, nil)
			cleanup()
			return
		}

		if batchErrs, ok := err.(*rpc.WriteBatchErrors); ok {
			// Callback all writes with errors
			hasErr := make(map[int]struct{})
			for _, batchErr := range batchErrs.Errors {
				op := ops[batchErr.ElementErrorIndex]
				op.GetCompletionFn()(nil, fmt.Errorf(batchErr.Error.Message))
				hasErr[int(batchErr.ElementErrorIndex)] = struct{}{}
			}
			// Callback all writes with no errors
			for i := range ops {
				if _, ok := hasErr[i]; !ok {
					// No error
					ops[i].GetCompletionFn()(nil, nil)
				}
			}
			cleanup()
			return
		}

		// Entire batch failed
		callAllCompletionFns(ops, nil, err)
		cleanup()
	}()
}

func (q *queue) Len() int {
	q.RLock()
	v := len(q.ops)
	q.RUnlock()
	return v
}

func (q *queue) Enqueue(o m3db.Op) error {
	var needsDrain []m3db.Op
	q.Lock()
	if q.state != stateOpen {
		q.Unlock()
		return errQueueNotOpen
	}
	q.ops = append(q.ops, o)
	// If queue is full flush
	if len(q.ops) >= q.size {
		needsDrain = q.rotateOpsWithLock()
	}
	q.Unlock()

	if len(needsDrain) != 0 {
		q.drainIn <- needsDrain
	}
	return nil
}

func (q *queue) GetConnectionCount() int {
	return q.connPool.GetConnectionCount()
}

func (q *queue) Close() {
	q.Lock()
	if q.state != stateOpen {
		q.Unlock()
		return
	}

	q.state = stateClosed
	q.Unlock()

	// Now we're closed its safe to close the drainIn channel
	close(q.drainIn)
}
<|endoftext|>


<commit_before>package webhook

import (
	"encoding/json"
	"github.com/Comcast/webpa-common/httperror"
	AWS "github.com/Comcast/webpa-common/webhook/aws"
	"github.com/spf13/viper"
	"net/http"
	"time"
)

const (
	DefaultUndertakerInterval time.Duration = time.Minute
)

// Factory is a classic Factory Object for various webhook things.
type Factory struct {
	// Other configuration stuff can go here

	// Tick is an optional function that produces a channel for time ticks.
	// Test code can set this field to something that returns a channel under the control of the test.
	Tick func(time.Duration) <-chan time.Time `json:"-"`

	// UndertakerInterval is how often the Undertaker is invoked
	UndertakerInterval time.Duration `json:"undertakerInterval"`

	// Undertaker is set by clients after reading in a Factory from some external source.
	// The associated Undertaker is immutable after construction.
	Undertaker func([]W) []W `json:"-"`

	// internal handler for webhook
	m *monitor `json:"-"`

	// internal handler for AWS SNS Server
	AWS.Notifier `json:"-"`

	// StartConfig is the contains the data need to obtain the current system's listeners
	Start *StartConfig `json:"start"`
}

// NewFactory creates a Factory from a Viper environment.  This function always returns
// a non-nil Factory instance.
//
// This example uses Viper, which I highly recommend.  You could just pass an io.Reader too, and use
// the encoding/json package.  In any case, allowing the configuration source to be nil makes a lot
// of things easier on clients, like creating a test Factory for tests in client code.
func NewFactory(v *viper.Viper) (f *Factory, err error) {
	f = &Factory{
	/* put in any system defaults here.  they won't be overridden by Viper unless they are present in external configuration */
	}

	// allowing the viper instance to be nil allows a client to do
	// NewFactory(nil) to get a default Factory instance
	if v != nil {
		err = v.Unmarshal(f)
	}

	f.Start = NewStartFactory(v)
	f.Notifier, err = AWS.NewNotifier(v)

	return
}

// NewListAndHandler returns a List instance for accessing webhooks and an HTTP handler
// which can receive updates from external systems.
func (f *Factory) NewListAndHandler() (List, http.Handler) {
	tick := f.Tick
	if tick == nil {
		tick = time.Tick
	}

	monitor := &monitor{
		list:             NewRegistry(nil, f.Notifier.PublishMessage),
		undertaker:       f.Undertaker,
		changes:          make(chan []W, 10),
		undertakerTicker: tick(f.UndertakerInterval),
	}
	f.m = monitor

	f.m.Notifier = f.Notifier

	go monitor.listen()
	return monitor.list, monitor
}

// monitor is an internal type that listens for webhook updates, invokes
// the undertaker at specified intervals, and responds to HTTP requests.
type monitor struct {
	list             UpdatableList
	undertaker       func([]W) []W
	changes          chan []W
	undertakerTicker <-chan time.Time
	AWS.Notifier
}

func (f *Factory) SetList(ul UpdatableList) {
	f.m.list = ul
}

func (m *monitor) listen() {
	for {
		select {
		case update := <-m.changes:
			m.list.Update(update)
		case <-m.undertakerTicker:
			m.list.Filter(m.undertaker)
		}
	}
}

// ServeHTTP is used as POST handler for AWS SNS
// It transforms the message containing webhook to []W and updates the webhook list
func (m *monitor) ServeHTTP(response http.ResponseWriter, request *http.Request) {
	// transform a request into a []byte
	message := m.NotificationHandle(response, request)
	if message == nil {
		return
	}

	// transform message to W
	var newHook W
	if err := json.Unmarshal(message, &newHook); err != nil {
		httperror.Format(response, http.StatusBadRequest, "Notification Message JSON unmarshall failed")
		return
	}

	select {
	case m.changes <- []W{newHook}:
	default:
	}
}
<commit_msg>fix to pass correct viper instance to NewStartFactory<commit_after>package webhook

import (
	"encoding/json"
	"github.com/Comcast/webpa-common/httperror"
	AWS "github.com/Comcast/webpa-common/webhook/aws"
	"github.com/spf13/viper"
	"net/http"
	"time"
)

const (
	DefaultUndertakerInterval time.Duration = time.Minute
)

// Factory is a classic Factory Object for various webhook things.
type Factory struct {
	// Other configuration stuff can go here

	// Tick is an optional function that produces a channel for time ticks.
	// Test code can set this field to something that returns a channel under the control of the test.
	Tick func(time.Duration) <-chan time.Time `json:"-"`

	// UndertakerInterval is how often the Undertaker is invoked
	UndertakerInterval time.Duration `json:"undertakerInterval"`

	// Undertaker is set by clients after reading in a Factory from some external source.
	// The associated Undertaker is immutable after construction.
	Undertaker func([]W) []W `json:"-"`

	// internal handler for webhook
	m *monitor `json:"-"`

	// internal handler for AWS SNS Server
	AWS.Notifier `json:"-"`

	// StartConfig is the contains the data need to obtain the current system's listeners
	Start *StartConfig `json:"start"`
}

// NewFactory creates a Factory from a Viper environment.  This function always returns
// a non-nil Factory instance.
//
// This example uses Viper, which I highly recommend.  You could just pass an io.Reader too, and use
// the encoding/json package.  In any case, allowing the configuration source to be nil makes a lot
// of things easier on clients, like creating a test Factory for tests in client code.
func NewFactory(v *viper.Viper) (f *Factory, err error) {
	f = &Factory{
	/* put in any system defaults here.  they won't be overridden by Viper unless they are present in external configuration */
	}

	// allowing the viper instance to be nil allows a client to do
	// NewFactory(nil) to get a default Factory instance
	if v != nil {
		err = v.Unmarshal(f)
	}

	f.Start = NewStartFactory(v.Sub("start"))
	f.Notifier, err = AWS.NewNotifier(v)

	return
}

// NewListAndHandler returns a List instance for accessing webhooks and an HTTP handler
// which can receive updates from external systems.
func (f *Factory) NewListAndHandler() (List, http.Handler) {
	tick := f.Tick
	if tick == nil {
		tick = time.Tick
	}

	monitor := &monitor{
		list:             NewRegistry(nil, f.Notifier.PublishMessage),
		undertaker:       f.Undertaker,
		changes:          make(chan []W, 10),
		undertakerTicker: tick(f.UndertakerInterval),
	}
	f.m = monitor

	f.m.Notifier = f.Notifier

	go monitor.listen()
	return monitor.list, monitor
}

// monitor is an internal type that listens for webhook updates, invokes
// the undertaker at specified intervals, and responds to HTTP requests.
type monitor struct {
	list             UpdatableList
	undertaker       func([]W) []W
	changes          chan []W
	undertakerTicker <-chan time.Time
	AWS.Notifier
}

func (f *Factory) SetList(ul UpdatableList) {
	f.m.list = ul
}

func (m *monitor) listen() {
	for {
		select {
		case update := <-m.changes:
			m.list.Update(update)
		case <-m.undertakerTicker:
			m.list.Filter(m.undertaker)
		}
	}
}

// ServeHTTP is used as POST handler for AWS SNS
// It transforms the message containing webhook to []W and updates the webhook list
func (m *monitor) ServeHTTP(response http.ResponseWriter, request *http.Request) {
	// transform a request into a []byte
	message := m.NotificationHandle(response, request)
	if message == nil {
		return
	}

	// transform message to W
	var newHook W
	if err := json.Unmarshal(message, &newHook); err != nil {
		httperror.Format(response, http.StatusBadRequest, "Notification Message JSON unmarshall failed")
		return
	}

	select {
	case m.changes <- []W{newHook}:
	default:
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package workload

import (
	"fmt"

	"github.com/juju/errors"
	"github.com/juju/utils/set"
)

// The Juju-recognized states in which a workload might be.
const (
	StateUndefined = ""
	//TODO(wwitzel3) remove defined throughout
	StateDefined  = "defined"
	StateStarting = "starting"
	StateRunning  = "running"
	StateStopping = "stopping"
	StateStopped  = "stopped"
)

var okayStates = set.NewStrings(
	StateStarting,
	StateRunning,
	StateStopping,
	StateStopped,
)

// The different kinds of status blockers.
const (
	NotBlocked = ""
	// BlockerError indicates that the plugin reported a problem with
	// the workload.
	BlockerError = "error"
	// BlockerFailed indicates that Juju got a failure while trying
	// to interact with the workload (via its plugin).
	BlockerFailed = "failed"
)

// TODO(ericsnow) Use a separate StatusInfo and keep Status (quasi-)immutable?

// Status is the Juju-level status of a workload.
type Status struct {
	// State is which state the workload is in relative to Juju.
	State string
	// Blocker identifies the kind of blocker preventing interaction
	// with the workload.
	Blocker string
	// Message is a human-readable message describing the current status
	// of the workload, why it is in the current state, or what Juju is
	// doing right now relative to the workload. There may be no message.
	Message string
}

// String returns a string representing the status of the workload.
func (s Status) String() string {
	message := s.Message
	if s.IsBlocked() {
		if message == "" {
			message = "<no message>"
		}
		return fmt.Sprintf("(%s) %s", s.Blocker, message)
	}
	return message
}

// IsBlocked indicates whether or not the workload may workload
// to the next state.
func (s *Status) IsBlocked() bool {
	return s.Blocker != NotBlocked
}

// Advance updates the state of the Status to the next appropriate one.
// If a message is provided, it is set. Otherwise the current message
// will be cleared.
func (s *Status) Advance(message string) error {
	if s.IsBlocked() {
		reason := s.Blocker
		return errors.Errorf("cannot advance state (" + reason + ")")
	}
	switch s.State {
	case StateUndefined:
		s.State = StateDefined
	case StateDefined:
		s.State = StateStarting
	case StateStarting:
		s.State = StateRunning
	case StateRunning:
		s.State = StateStopping
	case StateStopping:
		s.State = StateStopped
	case StateStopped:
		return errors.Errorf("cannot advance from a final state")
	default:
		return errors.NotValidf("unrecognized state %q", s.State)
	}
	s.Message = message
	return nil
}

// SetFailed records that Juju encountered a problem when trying to
// interact with the workload. If Status.Blocker is already failed then
// the message is updated. If the workload is in an initial or final
// state then an error is returned.
func (s *Status) SetFailed(message string) error {
	switch s.State {
	case StateUndefined:
		return errors.Errorf("cannot fail while in an initial state")
	case StateDefined:
		return errors.Errorf("cannot fail while in an initial state")
	case StateStopped:
		return errors.Errorf("cannot fail while in a final state")
	}

	if message == "" {
		message = "problem while interacting with workload"
	}
	s.Blocker = BlockerFailed
	s.Message = message
	return nil
}

// SetError records that the workload isn't working correctly,
// as reported by the plugin. If already in an error state then the
// message is updated. The workload must be in a running state for there
// to be an error. problems during starting and stopping are recorded
// as failures rather than errors.
func (s *Status) SetError(message string) error {
	// TODO(ericsnow) Allow errors in other states?
	switch s.State {
	case StateRunning:
	default:
		return errors.Errorf("can error only while running")
	}

	if message == "" {
		message = "the workload has an error"
	}
	s.Blocker = BlockerError
	s.Message = message
	return nil
}

// Resolve clears any existing error or failure status for the workload.
// If a message is provided then it is set. Otherwise a message
// describing what was resolved will be set. If the workload is both
// failed and in an error state then both will be resolved at once.
// If the workload isn't currently blocked then an error is returned.
func (s *Status) Resolve(message string) error {
	if !s.IsBlocked() {
		// TODO(ericsnow) Do nothing?
		return errors.Errorf("not in an error or failed state")
	}

	defaultMessage := fmt.Sprintf("resolved blocker %q", s.Blocker)
	if message == "" {
		// TODO(ericsnow) Add in the current message?
		message = defaultMessage
	}

	s.Blocker = NotBlocked
	s.Message = message
	return nil
}

// Validate checkes the status to ensure it contains valid data.
func (s Status) Validate() error {
	if !okayStates.Contains(s.State) {
		return errors.NotValidf("Status.State (%q)", s.State)
	}

	switch s.Blocker {
	case BlockerFailed:
		switch s.State {
		case StateUndefined:
			return errors.NotValidf("failure in an initial state")
		case StateDefined:
			return errors.NotValidf("failure in an initial state")
		case StateStopped:
			return errors.NotValidf("failure in a final state")
		}
	case BlockerError:
		if s.State != StateRunning {
			return errors.NotValidf("error outside running state")
		}
	}

	return nil
}

// ValidateState verifies the state passed in is a valid okayState.
func ValidateState(state string) error {
	if !okayStates.Contains(state) {
		return errors.NotValidf("state (%q)", state)
	}
	return nil
}

// PluginStatus represents the data returned from the Plugin.Status call.
type PluginStatus struct {
	// State is the human-readable label returned by the plugin
	// that represents the status of the workload.
	State string `json:"state"`
}

// Validate returns nil if this value is valid, and an error that satisfies
// IsValid if it is not.
func (s PluginStatus) Validate() error {
	if s.State == "" {
		return errors.NotValidf("State cannot be empty")
	}
	return nil
}

// CombinedStatus holds the status information for a workload,
// from all sources.
type CombinedStatus struct {
	// Status is the Juju-level status information.
	Status Status
	// PluginStatus is the plugin-defined status information.
	PluginStatus PluginStatus
}

// Validate returns nil if this value is valid, and an error that satisfies
// IsValid if it is not.
func (s CombinedStatus) Validate() error {
	if err := s.Status.Validate(); err != nil {
		return errors.Trace(err)
	}
	if err := s.PluginStatus.Validate(); err != nil {
		return errors.Trace(err)
	}
	return nil
}
<commit_msg>update validation usage<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package workload

import (
	"fmt"

	"github.com/juju/errors"
	"github.com/juju/utils/set"
)

// The Juju-recognized states in which a workload might be.
const (
	StateUndefined = ""
	//TODO(wwitzel3) remove defined throughout
	StateDefined  = "defined"
	StateStarting = "starting"
	StateRunning  = "running"
	StateStopping = "stopping"
	StateStopped  = "stopped"
)

var okayStates = set.NewStrings(
	StateStarting,
	StateRunning,
	StateStopping,
	StateStopped,
)

// The different kinds of status blockers.
const (
	NotBlocked = ""
	// BlockerError indicates that the plugin reported a problem with
	// the workload.
	BlockerError = "error"
	// BlockerFailed indicates that Juju got a failure while trying
	// to interact with the workload (via its plugin).
	BlockerFailed = "failed"
)

// TODO(ericsnow) Use a separate StatusInfo and keep Status (quasi-)immutable?

// Status is the Juju-level status of a workload.
type Status struct {
	// State is which state the workload is in relative to Juju.
	State string
	// Blocker identifies the kind of blocker preventing interaction
	// with the workload.
	Blocker string
	// Message is a human-readable message describing the current status
	// of the workload, why it is in the current state, or what Juju is
	// doing right now relative to the workload. There may be no message.
	Message string
}

// String returns a string representing the status of the workload.
func (s Status) String() string {
	message := s.Message
	if s.IsBlocked() {
		if message == "" {
			message = "<no message>"
		}
		return fmt.Sprintf("(%s) %s", s.Blocker, message)
	}
	return message
}

// IsBlocked indicates whether or not the workload may workload
// to the next state.
func (s *Status) IsBlocked() bool {
	return s.Blocker != NotBlocked
}

// Advance updates the state of the Status to the next appropriate one.
// If a message is provided, it is set. Otherwise the current message
// will be cleared.
func (s *Status) Advance(message string) error {
	if s.IsBlocked() {
		reason := s.Blocker
		return errors.Errorf("cannot advance state (" + reason + ")")
	}
	switch s.State {
	case StateUndefined:
		s.State = StateDefined
	case StateDefined:
		s.State = StateStarting
	case StateStarting:
		s.State = StateRunning
	case StateRunning:
		s.State = StateStopping
	case StateStopping:
		s.State = StateStopped
	case StateStopped:
		return errors.Errorf("cannot advance from a final state")
	default:
		return errors.NotValidf("unrecognized state %q", s.State)
	}
	s.Message = message
	return nil
}

// SetFailed records that Juju encountered a problem when trying to
// interact with the workload. If Status.Blocker is already failed then
// the message is updated. If the workload is in an initial or final
// state then an error is returned.
func (s *Status) SetFailed(message string) error {
	switch s.State {
	case StateUndefined:
		return errors.Errorf("cannot fail while in an initial state")
	case StateDefined:
		return errors.Errorf("cannot fail while in an initial state")
	case StateStopped:
		return errors.Errorf("cannot fail while in a final state")
	}

	if message == "" {
		message = "problem while interacting with workload"
	}
	s.Blocker = BlockerFailed
	s.Message = message
	return nil
}

// SetError records that the workload isn't working correctly,
// as reported by the plugin. If already in an error state then the
// message is updated. The workload must be in a running state for there
// to be an error. problems during starting and stopping are recorded
// as failures rather than errors.
func (s *Status) SetError(message string) error {
	// TODO(ericsnow) Allow errors in other states?
	switch s.State {
	case StateRunning:
	default:
		return errors.Errorf("can error only while running")
	}

	if message == "" {
		message = "the workload has an error"
	}
	s.Blocker = BlockerError
	s.Message = message
	return nil
}

// Resolve clears any existing error or failure status for the workload.
// If a message is provided then it is set. Otherwise a message
// describing what was resolved will be set. If the workload is both
// failed and in an error state then both will be resolved at once.
// If the workload isn't currently blocked then an error is returned.
func (s *Status) Resolve(message string) error {
	if !s.IsBlocked() {
		// TODO(ericsnow) Do nothing?
		return errors.Errorf("not in an error or failed state")
	}

	defaultMessage := fmt.Sprintf("resolved blocker %q", s.Blocker)
	if message == "" {
		// TODO(ericsnow) Add in the current message?
		message = defaultMessage
	}

	s.Blocker = NotBlocked
	s.Message = message
	return nil
}

// Validate checkes the status to ensure it contains valid data.
func (s Status) Validate() error {
	if err := ValidateState(s.State); err != nil {
		return errors.NotValidf("Status.State (%q)", s.State)
	}

	switch s.Blocker {
	case BlockerFailed:
		switch s.State {
		case StateUndefined:
			return errors.NotValidf("failure in an initial state")
		case StateDefined:
			return errors.NotValidf("failure in an initial state")
		case StateStopped:
			return errors.NotValidf("failure in a final state")
		}
	case BlockerError:
		if s.State != StateRunning {
			return errors.NotValidf("error outside running state")
		}
	}

	return nil
}

// ValidateState verifies the state passed in is a valid okayState.
func ValidateState(state string) error {
	if !okayStates.Contains(state) {
		return errors.NotValidf("state (%q)", state)
	}
	return nil
}

// PluginStatus represents the data returned from the Plugin.Status call.
type PluginStatus struct {
	// State is the human-readable label returned by the plugin
	// that represents the status of the workload.
	State string `json:"state"`
}

// Validate returns nil if this value is valid, and an error that satisfies
// IsValid if it is not.
func (s PluginStatus) Validate() error {
	if s.State == "" {
		return errors.NotValidf("State cannot be empty")
	}
	return nil
}

// CombinedStatus holds the status information for a workload,
// from all sources.
type CombinedStatus struct {
	// Status is the Juju-level status information.
	Status Status
	// PluginStatus is the plugin-defined status information.
	PluginStatus PluginStatus
}

// Validate returns nil if this value is valid, and an error that satisfies
// IsValid if it is not.
func (s CombinedStatus) Validate() error {
	if err := s.Status.Validate(); err != nil {
		return errors.Trace(err)
	}
	if err := s.PluginStatus.Validate(); err != nil {
		return errors.Trace(err)
	}
	return nil
}
<|endoftext|>


<commit_before>package configuration

import (
	"fmt"
	"github.com/PaluMacil/dwn/database/store"
	"golang.org/x/oauth2"
	"os"
	"time"
)

const (
	CredentialPrefix = "CREDENTIAL:"
)

type FSNames []string

func (fs FSNames) Includes(fsName string) bool {
	for _, n := range fs {
		if n == fsName {
			return true
		}
	}
	return false
}

var (
	WellKnownAuthProviderIDs = FSNames{"GOOGLE"}
	WellKnownSMTPProviderIDs = FSNames{"SENDGRID"}
)

type ForeignSystemType string

func (t ForeignSystemType) Bytes() []byte {
	return []byte(t)
}

const (
	ForeignSystemTypeAuth ForeignSystemType = "AUTH"
	ForeignSystemTypeSMTP ForeignSystemType = "SMTP"
)

var ValidForeignSystemTypes = ForeignSystemTypes{
	ForeignSystemTypeAuth, ForeignSystemTypeSMTP,
}

type ForeignSystemTypes []ForeignSystemType

func (t ForeignSystemTypes) Includes(fsType ForeignSystemType) bool {
	for _, n := range t {
		if n == fsType {
			return true
		}
	}
	return false
}

type SetupConfiguration struct {
	InitialAdmin    string `json:"initialAdmin"`
	InitialPassword string `json:"-"`
}

type Configuration struct {
	WebServer WebServerConfiguration     `json:"webServer"`
	Setup     SetupConfiguration         `json:"setup"`
	Database  DatabaseConfiguration      `json:"database"`
	FS        ForeignSystemConfiguration `json:"fs"`
	Prod      bool                       `json:"prod"`
}

type WebServerConfiguration struct {
	Protocol    string `json:"protocol"`
	Host        string `json:"host"`
	Port        string `json:"port"`
	UIProxyPort string `json:"uiProxyPort"`
	ContentRoot string `json:"contentRoot"`
}

type DatabaseConfiguration struct {
	DataDir       string `json:"dataDir"`
	EncryptionKey []byte `json:"-"`
}

func (ws WebServerConfiguration) HomePage() string {
	port := ws.UIProxyPort
	if port == "" {
		port = ws.Port
	}
	return fmt.Sprintf("%s://%s:%s",
		ws.Protocol, ws.Host, port)
}

type AuthConfiguration struct {
	Google *oauth2.Config `json:"google,omitempty"`
}

type Credential struct {
	Name        string            `json:"name"`
	Type        ForeignSystemType `json:"type"`
	ID          string            `json:"id"`
	Secret      string            `json:"-"`
	FromEnv     bool              `json:"fromEnv"`
	CreatedBy   store.Identity    `json:"createdBy"`
	CreatedDate time.Time         `json:"createdDate"`
}

func (c Credential) Key() []byte {
	key := append(c.Prefix(), []byte(c.Type+":")...)
	return append(key, []byte(c.Name+":")...)
}

func (c Credential) Prefix() []byte {
	return []byte(CredentialPrefix)
}

type SMTPConfiguration struct {
	SendGrid *Credential `json:"sendGrid,omitempty"`
}

type ForeignSystemConfiguration struct {
	Auth AuthConfiguration `json:"auth"`
	SMTP SMTPConfiguration `json:"smtp"`
}

type Mode struct {
	Prod bool
}

func (mode Mode) Coalesce(envKey, prodDefault, devDefault string) string {
	value := os.Getenv(envKey)
	if value == "" {
		if mode.Prod {
			return prodDefault
		} else {
			return devDefault
		}
	}
	return value
}
<commit_msg>remove field<commit_after>package configuration

import (
	"fmt"
	"github.com/PaluMacil/dwn/database/store"
	"golang.org/x/oauth2"
	"os"
	"time"
)

const (
	CredentialPrefix = "CREDENTIAL:"
)

type FSNames []string

func (fs FSNames) Includes(fsName string) bool {
	for _, n := range fs {
		if n == fsName {
			return true
		}
	}
	return false
}

var (
	WellKnownAuthProviderIDs = FSNames{"GOOGLE"}
	WellKnownSMTPProviderIDs = FSNames{"SENDGRID"}
)

type ForeignSystemType string

func (t ForeignSystemType) Bytes() []byte {
	return []byte(t)
}

const (
	ForeignSystemTypeAuth ForeignSystemType = "AUTH"
	ForeignSystemTypeSMTP ForeignSystemType = "SMTP"
)

var ValidForeignSystemTypes = ForeignSystemTypes{
	ForeignSystemTypeAuth, ForeignSystemTypeSMTP,
}

type ForeignSystemTypes []ForeignSystemType

func (t ForeignSystemTypes) Includes(fsType ForeignSystemType) bool {
	for _, n := range t {
		if n == fsType {
			return true
		}
	}
	return false
}

type SetupConfiguration struct {
	InitialAdmin    string `json:"initialAdmin"`
	InitialPassword string `json:"-"`
}

type Configuration struct {
	WebServer WebServerConfiguration     `json:"webServer"`
	Setup     SetupConfiguration         `json:"setup"`
	Database  DatabaseConfiguration      `json:"database"`
	FS        ForeignSystemConfiguration `json:"fs"`
	Prod      bool                       `json:"prod"`
}

type WebServerConfiguration struct {
	Protocol    string `json:"protocol"`
	Host        string `json:"host"`
	Port        string `json:"port"`
	UIProxyPort string `json:"uiProxyPort"`
	ContentRoot string `json:"contentRoot"`
}

type DatabaseConfiguration struct {
	DataDir       string `json:"dataDir"`
	EncryptionKey []byte `json:"-"`
}

func (ws WebServerConfiguration) HomePage() string {
	port := ws.UIProxyPort
	if port == "" {
		port = ws.Port
	}
	return fmt.Sprintf("%s://%s:%s",
		ws.Protocol, ws.Host, port)
}

type AuthConfiguration struct {
	Google *oauth2.Config `json:"google,omitempty"`
}

type Credential struct {
	Name        string            `json:"name"`
	Type        ForeignSystemType `json:"type"`
	ID          string            `json:"id"`
	Secret      string            `json:"-"`
	CreatedBy   store.Identity    `json:"createdBy"`
	CreatedDate time.Time         `json:"createdDate"`
}

func (c Credential) Key() []byte {
	key := append(c.Prefix(), []byte(c.Type+":")...)
	return append(key, []byte(c.Name+":")...)
}

func (c Credential) Prefix() []byte {
	return []byte(CredentialPrefix)
}

type SMTPConfiguration struct {
	SendGrid *Credential `json:"sendGrid,omitempty"`
}

type ForeignSystemConfiguration struct {
	Auth AuthConfiguration `json:"auth"`
	SMTP SMTPConfiguration `json:"smtp"`
}

type Mode struct {
	Prod bool
}

func (mode Mode) Coalesce(envKey, prodDefault, devDefault string) string {
	value := os.Getenv(envKey)
	if value == "" {
		if mode.Prod {
			return prodDefault
		} else {
			return devDefault
		}
	}
	return value
}
<|endoftext|>