<commit_before>package main

import (
	"encoding/json"
	"net/http"
)

func (o *Orderup) getAPIv1() *API {
	return &API{
		Routes: []Route{
			Route{
				Path:        API_PREFIX + "/v1/queues/orders/list",
				HandlerFunc: o.listAPIHandler,
				Methods:     []string{"GET"},
			},
			Route{
				Path:        API_PREFIX + "/v1/queues/orders/history",
				HandlerFunc: o.historyAPIHandler,
				Methods:     []string{"GET"},
			},
		},
	}
}

type listRequest struct {
	Name string `json:name`
}

type listResponse struct {
	Response string   `json:"response"`
	Orders   *[]Order `json:"orders"`
}

func (o *Orderup) writeAPIResponse(w http.ResponseWriter, response []byte) {
	w.Header().Set("Content-type", "application/json")
	w.Write(response)
}

// Encode and return error response to user.
func (o *Orderup) writeAPIErrorResponse(w http.ResponseWriter, apiErr error) {
	apiResponse := &APIErrorResponse{
		Response: ERROR_RESPONSE,
		Errors:   []string{apiErr.Error()},
	}

	response, err := json.Marshal(apiResponse)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-type", "application/json")
	w.Write(response)
}

// list command.
func (o *Orderup) listAPIHandler(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)

	req := &listRequest{}

	if err := decoder.Decode(&req); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if req.Name == "" {
		o.writeAPIErrorResponse(w, WrongArgsError())
		return
	}

	orders, cmdErr := o.getPendingOrderList([]byte(req.Name))

	if cmdErr != nil {
		o.writeAPIErrorResponse(w, cmdErr)
		return
	}

	apiResponse := &listResponse{
		Response: "success",
		Orders:   orders,
	}

	response, err := json.Marshal(apiResponse)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	o.writeAPIResponse(w, response)
}

// history command.
func (o *Orderup) historyAPIHandler(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)

	req := &listRequest{}

	if err := decoder.Decode(&req); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if req.Name == "" {
		o.writeAPIErrorResponse(w, WrongArgsError())
		return
	}

	orders, cmdErr := o.getHistoryList([]byte(req.Name))

	if cmdErr != nil {
		o.writeAPIErrorResponse(w, cmdErr)
		return
	}

	apiResponse := &listResponse{
		Response: "success",
		Orders:   orders,
	}

	response, err := json.Marshal(apiResponse)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	o.writeAPIResponse(w, response)
}
<commit_msg>Implement create-q REST API command<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
)

func (o *Orderup) getAPIv1() *API {
	return &API{
		Routes: []Route{
			Route{
				Path:        API_PREFIX + "/v1/queues/orders/list",
				HandlerFunc: o.listAPIHandler,
				Methods:     []string{"GET"},
			},
			Route{
				Path:        API_PREFIX + "/v1/queues/orders/history",
				HandlerFunc: o.historyAPIHandler,
				Methods:     []string{"GET"},
			},
			Route{
				Path:        API_PREFIX + "/v1/queues",
				HandlerFunc: o.createQueueAPIHandler,
				Methods:     []string{"POST"},
			},
		},
	}
}

// Sturctures for encoding/decoding API calls.

type listRequest struct {
	Name string `json:name`
}

type listResponse struct {
	Response string   `json:"response"`
	Orders   *[]Order `json:"orders"`
}

type queueRequest struct {
	Name string `json:name`
}

type queueResponse struct {
	Response string `json:"response"`
}

func (o *Orderup) writeAPIResponse(w http.ResponseWriter, response []byte) {
	w.Header().Set("Content-type", "application/json")
	w.Write(response)
}

// Encode and return error response to user.
func (o *Orderup) writeAPIErrorResponse(w http.ResponseWriter, apiErr error) {
	apiResponse := &APIErrorResponse{
		Response: ERROR_RESPONSE,
		Errors:   []string{apiErr.Error()},
	}

	response, err := json.Marshal(apiResponse)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-type", "application/json")
	w.Write(response)
}

// list command.
func (o *Orderup) listAPIHandler(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)

	req := &listRequest{}

	if err := decoder.Decode(&req); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if req.Name == "" {
		o.writeAPIErrorResponse(w, WrongArgsError())
		return
	}

	orders, cmdErr := o.getPendingOrderList([]byte(req.Name))

	if cmdErr != nil {
		o.writeAPIErrorResponse(w, cmdErr)
		return
	}

	apiResponse := &listResponse{
		Response: "success",
		Orders:   orders,
	}

	response, err := json.Marshal(apiResponse)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	o.writeAPIResponse(w, response)
}

// history command.
func (o *Orderup) historyAPIHandler(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)

	req := &listRequest{}

	if err := decoder.Decode(&req); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if req.Name == "" {
		o.writeAPIErrorResponse(w, WrongArgsError())
		return
	}

	orders, cmdErr := o.getHistoryList([]byte(req.Name))

	if cmdErr != nil {
		o.writeAPIErrorResponse(w, cmdErr)
		return
	}

	apiResponse := &listResponse{
		Response: "success",
		Orders:   orders,
	}

	response, err := json.Marshal(apiResponse)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	o.writeAPIResponse(w, response)
}

// create-q command.
func (o *Orderup) createQueueAPIHandler(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)

	req := &queueRequest{}

	if err := decoder.Decode(&req); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if req.Name == "" {
		o.writeAPIErrorResponse(w, WrongArgsError())
		return
	}

	cmdErr := o.createQueue([]byte(req.Name))

	if cmdErr != nil {
		o.writeAPIErrorResponse(w, cmdErr)
		return
	}

	apiResponse := &listResponse{
		Response: fmt.Sprintf("Queue %s created.", req.Name),
	}

	response, err := json.Marshal(apiResponse)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	o.writeAPIResponse(w, response)
}
<|endoftext|>


<commit_before>package event_delete

import (
	"regexp"

	"github.com/karimra/gnmic/formatters"
)

// Delete, deletes the tags or values matching one of the regexes
type Delete struct {
	Type   string   `json:"type,omitempty"`
	Tags   []string `json:"tags,omitempty"`
	Values []string `json:"values,omitempty"`
	tags   []*regexp.Regexp
	values []*regexp.Regexp
}

func init() {
	formatters.Register("event_delete", func() formatters.EventProcessor {
		return &Delete{
			Type: "event_delete",
		}
	})
}

func (d *Delete) Init(cfg interface{}) error {
	err := formatters.DecodeConfig(cfg, d)
	if err != nil {
		return err
	}
	d.tags = make([]*regexp.Regexp, 0, len(d.Tags))
	for _, reg := range d.Tags {
		re, err := regexp.Compile(reg)
		if err != nil {
			return err
		}
		d.tags = append(d.tags, re)
	}
	//
	d.values = make([]*regexp.Regexp, 0, len(d.values))
	for _, reg := range d.Values {
		re, err := regexp.Compile(reg)
		if err != nil {
			return err
		}
		d.values = append(d.values, re)
	}
	return nil
}

func (d *Delete) Apply(e *formatters.EventMsg) {
	if e == nil {
		return
	}
	for k := range e.Values {
		for _, re := range d.values {
			if re.MatchString(k) {
				delete(e.Values, k)
				break
			}
		}
	}
	for k := range e.Tags {
		for _, re := range d.tags {
			if re.MatchString(k) {
				delete(e.Tags, k)
				break
			}
		}
	}
}
<commit_msg>implement delete<commit_after>package event_delete

import (
	"regexp"

	"github.com/karimra/gnmic/formatters"
)

// Delete, deletes ALL the tags or values matching one of the regexes
type Delete struct {
	Type   string   `json:"type,omitempty"`
	Tags   []string `json:"tags,omitempty"`
	Values []string `json:"values,omitempty"`
	tags   []*regexp.Regexp
	values []*regexp.Regexp
}

func init() {
	formatters.Register("event_delete", func() formatters.EventProcessor {
		return &Delete{}
	})
}

func (d *Delete) Init(cfg interface{}) error {
	err := formatters.DecodeConfig(cfg, d)
	if err != nil {
		return err
	}
	d.tags = make([]*regexp.Regexp, 0, len(d.Tags))
	for _, reg := range d.Tags {
		re, err := regexp.Compile(reg)
		if err != nil {
			return err
		}
		d.tags = append(d.tags, re)
	}
	//
	d.values = make([]*regexp.Regexp, 0, len(d.values))
	for _, reg := range d.Values {
		re, err := regexp.Compile(reg)
		if err != nil {
			return err
		}
		d.values = append(d.values, re)
	}
	return nil
}

func (d *Delete) Apply(e *formatters.EventMsg) {
	if e == nil {
		return
	}
	for k := range e.Values {
		for _, re := range d.values {
			if re.MatchString(k) {
				delete(e.Values, k)
			}
		}
	}
	for k := range e.Tags {
		for _, re := range d.tags {
			if re.MatchString(k) {
				delete(e.Tags, k)
			}
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>fixing build err<commit_after><|endoftext|>


<commit_before><commit_msg>Removed superfluous comment.<commit_after><|endoftext|>


<commit_before>// Copyright 2015 Brian J. Downs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package openweathermap

const unitError = "unit unavailable"
const langError = "language unavailable"

var DataUnits = map[string]string{"C": "metric", "F": "imperial", "K": "internal"}
var (
	baseURL      = "http://api.openweathermap.org/data/2.5/weather?%s"
	iconURL      = "http://openweathermap.org/img/w/%s"
	stationURL   = "http://api.openweathermap.org/data/2.5/station?id=%d"
	forecastBase = "http://api.openweathermap.org/data/2.5/forecast/daily?%s=%s&mode=json&units=%s&lang=%s&cnt=%d"
	historyURL   = "http://api.openweathermap.org/data/2.5/history/%s"
	dataPostURL  = "http://openweathermap.org/data/post"
)

// LangCodes holds all supported languages to be used
// inspried from and sourced from @bambocher (github.com/bambocher)
var LangCodes = map[string]string{
	"EN":    "English",
	"RU":    "Russian",
	"IT":    "Italian",
	"ES":    "Spanish",
	"SP":    "Spanish",
	"UK":    "Ukrainian",
	"UA":    "Ukrainian",
	"DE":    "German",
	"PT":    "Portuguese",
	"RO":    "Romanian",
	"PL":    "Polish",
	"FI":    "Finnish",
	"NL":    "Dutch",
	"FR":    "French",
	"BG":    "Bulgarian",
	"SV":    "Swedish",
	"SE":    "Swedish",
	"TR":    "Turkish",
	"HR":    "Croatian",
	"CA":    "Catalan",
	"ZH_TW": "Chinese Traditional",
	"ZH":    "Chinese Simplified",
	"ZH_CN": "Chinese Simplified",
}

// Config will hold default settings to be passed into the
// "New{current, forecast, etc}" function.
type Config struct {
	Mode     string // User choice of JSON or XML
	Unit     string // F, C, or K
	Lang     string // should reference a key in the LangCodes map
	APIKey   string // API Key for connecting to the OWM
	Username string // Username for posting data
	Password string // Pasword for posting data
}

// APIError returned on failed API calls.
type APIError struct {
	Message string `json:"message"`
	COD     string `json:"cod"`
}

// Coordinates struct holds longitude and latitude data in returned
// JSON or as parameter data for requests using longitude and latitude.
type Coordinates struct {
	Longitude float64 `json:"lon"`
	Latitude  float64 `json:"lat"`
}

// Sys struct contains general information about the request
// and the surrounding area for where the request was made.
type Sys struct {
	Type    int     `json:"type"`
	ID      int     `json:"id"`
	Message float64 `json:"message"`
	Country string  `json:"country"`
	Sunrise int     `json:"sunrise"`
	Sunset  int     `json:"sunset"`
}

// Wind struct contains the speed and degree of the wind.
type Wind struct {
	Speed float64 `json:"speed"`
	Deg   float64 `json:"deg"`
}

// Weather struct holds high-level, basic info on the returned
// data.
type Weather struct {
	ID          int    `json:"id"`
	Main        string `json:"main"`
	Description string `json:"description"`
	Icon        string `json:"icon"`
}

// Main struct contains the temperates, humidity, pressure for the request.
type Main struct {
	Temp       float64 `json:"temp"`
	TempMin    float64 `json:"temp_min"`
	TempMax    float64 `json:"temp_max"`
	Pressure   float64 `json:"pressure"`
	Sea_level  float64 `json:"sea_level"`
	Grnd_level float64 `json:"grnd_level"`
	Humidity   int     `json:"humidity"`
}

// Clouds struct holds data regarding cloud cover.
type Clouds struct {
	All int `json:"all"`
}

// ValidDataUnit makes sure the string passed in is an accepted
// unit of measure to be used for the return data.
func ValidDataUnit(u string) bool {
	for d := range DataUnits {
		if u == d {
			return true
		}
	}
	return false
}

// ValidLangCode makes sure the string passed in is an
// acceptable lang code.
func ValidLangCode(c string) bool {
	for d := range LangCodes {
		if c == d {
			return true
		}
	}
	return false
}

// ValidDataUnitSymbol makes sure the string passed in is an
// acceptable data unit symbol.
func ValidDataUnitSymbol(u string) bool {
	for _, d := range DataUnits {
		if u == d {
			return true
		}
	}
	return false
}

// CheckAPIKeyExists will see if an API key has been set.
func (c *Config) CheckAPIKeyExists() bool { return len(c.APIKey) > 1 }
<commit_msg>comment update<commit_after>// Copyright 2015 Brian J. Downs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package openweathermap

const unitError = "unit unavailable"
const langError = "language unavailable"

var DataUnits = map[string]string{"C": "metric", "F": "imperial", "K": "internal"}
var (
	baseURL      = "http://api.openweathermap.org/data/2.5/weather?%s"
	iconURL      = "http://openweathermap.org/img/w/%s"
	stationURL   = "http://api.openweathermap.org/data/2.5/station?id=%d"
	forecastBase = "http://api.openweathermap.org/data/2.5/forecast/daily?%s=%s&mode=json&units=%s&lang=%s&cnt=%d"
	historyURL   = "http://api.openweathermap.org/data/2.5/history/%s"
	dataPostURL  = "http://openweathermap.org/data/post"
)

// LangCodes holds all supported languages to be used
// inspried and sourced from @bambocher (github.com/bambocher)
var LangCodes = map[string]string{
	"EN":    "English",
	"RU":    "Russian",
	"IT":    "Italian",
	"ES":    "Spanish",
	"SP":    "Spanish",
	"UK":    "Ukrainian",
	"UA":    "Ukrainian",
	"DE":    "German",
	"PT":    "Portuguese",
	"RO":    "Romanian",
	"PL":    "Polish",
	"FI":    "Finnish",
	"NL":    "Dutch",
	"FR":    "French",
	"BG":    "Bulgarian",
	"SV":    "Swedish",
	"SE":    "Swedish",
	"TR":    "Turkish",
	"HR":    "Croatian",
	"CA":    "Catalan",
	"ZH_TW": "Chinese Traditional",
	"ZH":    "Chinese Simplified",
	"ZH_CN": "Chinese Simplified",
}

// Config will hold default settings to be passed into the
// "NewCurrent, NewForecast, etc}" functions.
type Config struct {
	Mode     string // user choice of JSON or XML
	Unit     string // measurement for results to be displayed.  F, C, or K
	Lang     string // should reference a key in the LangCodes map
	APIKey   string // API Key for connecting to the OWM
	Username string // Username for posting data
	Password string // Pasword for posting data
}

// APIError returned on failed API calls.
type APIError struct {
	Message string `json:"message"`
	COD     string `json:"cod"`
}

// Coordinates struct holds longitude and latitude data in returned
// JSON or as parameter data for requests using longitude and latitude.
type Coordinates struct {
	Longitude float64 `json:"lon"`
	Latitude  float64 `json:"lat"`
}

// Sys struct contains general information about the request
// and the surrounding area for where the request was made.
type Sys struct {
	Type    int     `json:"type"`
	ID      int     `json:"id"`
	Message float64 `json:"message"`
	Country string  `json:"country"`
	Sunrise int     `json:"sunrise"`
	Sunset  int     `json:"sunset"`
}

// Wind struct contains the speed and degree of the wind.
type Wind struct {
	Speed float64 `json:"speed"`
	Deg   float64 `json:"deg"`
}

// Weather struct holds high-level, basic info on the returned
// data.
type Weather struct {
	ID          int    `json:"id"`
	Main        string `json:"main"`
	Description string `json:"description"`
	Icon        string `json:"icon"`
}

// Main struct contains the temperates, humidity, pressure for the request.
type Main struct {
	Temp       float64 `json:"temp"`
	TempMin    float64 `json:"temp_min"`
	TempMax    float64 `json:"temp_max"`
	Pressure   float64 `json:"pressure"`
	Sea_level  float64 `json:"sea_level"`
	Grnd_level float64 `json:"grnd_level"`
	Humidity   int     `json:"humidity"`
}

// Clouds struct holds data regarding cloud cover.
type Clouds struct {
	All int `json:"all"`
}

// ValidDataUnit makes sure the string passed in is an accepted
// unit of measure to be used for the return data.
func ValidDataUnit(u string) bool {
	for d := range DataUnits {
		if u == d {
			return true
		}
	}
	return false
}

// ValidLangCode makes sure the string passed in is an
// acceptable lang code.
func ValidLangCode(c string) bool {
	for d := range LangCodes {
		if c == d {
			return true
		}
	}
	return false
}

// ValidDataUnitSymbol makes sure the string passed in is an
// acceptable data unit symbol.
func ValidDataUnitSymbol(u string) bool {
	for _, d := range DataUnits {
		if u == d {
			return true
		}
	}
	return false
}

// CheckAPIKeyExists will see if an API key has been set.
func (c *Config) CheckAPIKeyExists() bool { return len(c.APIKey) > 1 }
<|endoftext|>


<commit_before>package handlers

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"net/http"

	"github.com/arschles/gbs/log"
	docker "github.com/fsouza/go-dockerclient"
	"github.com/gorilla/mux"
)

const (
	defaultBuildImg = "quay.io/arschles/gbs-env:0.0.1"
)

func BuildURL() string {
	return fmt.Sprintf("/{%s}/{%s}/{%s}", site, org, repo)
}

type startBuildReq struct {
	BuildEnv string `json:"build_env"`
}

type startBuildResp struct {
	StatusURL string `json:"status_url"`
}

func Build(workdir string, dockerCl *docker.Client) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		flusher, ok := w.(http.Flusher)
		if !ok {
			httpErrf(w, http.StatusInternalServerError, "server doesn't support flushing output")
			return
		}

		site, ok := mux.Vars(r)[site]
		if !ok {
			httpErrf(w, http.StatusBadRequest, "missing site in path")
			return
		}

		org, ok := mux.Vars(r)[org]
		if !ok {
			httpErrf(w, http.StatusBadRequest, "missing org in path")
			return
		}

		repo, ok := mux.Vars(r)[repo]
		if !ok {
			httpErrf(w, http.StatusBadRequest, "missing repo in path")
			return
		}

		buildImg := defaultBuildImg
		req := new(startBuildReq)
		if err := json.NewDecoder(r.Body).Decode(req); err == nil {
			buildImg = req.BuildEnv
		}
		defer r.Body.Close()

		containerOpts := createContainerOpts(buildImg, workdir, site, org, repo)
		container, err := dockerCl.CreateContainer(containerOpts)
		if err != nil {
			log.Errf("creating container [%s]", err)
			httpErrf(w, http.StatusInternalServerError, "error creating container [%s]", err)
			return
		}

		hostConfig := &docker.HostConfig{Binds: []string{fmt.Sprintf("%s:%s", workdir, absPwd)}}
		if err := dockerCl.StartContainer(container.ID, hostConfig); err != nil {
			log.Errf("starting container [%s]", err)
			httpErrf(w, http.StatusInternalServerError, "error starting container [%s]", err)
			return
		}

		attachOpts, outputReader := attachToContainerOpts(container.ID)
		errCh := make(chan error)
		go func() {
			if err := dockerCl.AttachToContainer(attachOpts); err != nil {
				errCh <- err
			}
		}()

		w.WriteHeader(http.StatusCreated)
		go func(reader io.Reader) {
			scanner := bufio.NewScanner(reader)
			for scanner.Scan() {
				fmt.Fprintf(w, "%s\n", scanner.Text())
				flusher.Flush()
			}
			if err := scanner.Err(); err != nil {
				fmt.Fprintf(w, "error with scanner in attached container [%s]\n", err)
			}
		}(outputReader)

		code, err := dockerCl.WaitContainer(container.ID)
		if err != nil {
			log.Errf("waiting for container %s [%s]", container.ID, err)
			return
		}
		w.Write([]byte(fmt.Sprintf("exited with error code %d\n", code)))

		removeOpts := docker.RemoveContainerOptions{
			ID:            container.ID,
			RemoveVolumes: true,
			Force:         true,
		}
		if err := dockerCl.RemoveContainer(removeOpts); err != nil {
			log.Errf("removing container %s [%s]", container.ID, err)
		}
	})
}
<commit_msg>only set the build env if it's nonempty<commit_after>package handlers

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"net/http"

	"github.com/arschles/gbs/log"
	docker "github.com/fsouza/go-dockerclient"
	"github.com/gorilla/mux"
)

const (
	defaultBuildImg = "quay.io/arschles/gbs-env:0.0.1"
)

func BuildURL() string {
	return fmt.Sprintf("/{%s}/{%s}/{%s}", site, org, repo)
}

type startBuildReq struct {
	BuildEnv string `json:"build_env"`
}

type startBuildResp struct {
	StatusURL string `json:"status_url"`
}

func Build(workdir string, dockerCl *docker.Client) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		flusher, ok := w.(http.Flusher)
		if !ok {
			httpErrf(w, http.StatusInternalServerError, "server doesn't support flushing output")
			return
		}

		site, ok := mux.Vars(r)[site]
		if !ok {
			httpErrf(w, http.StatusBadRequest, "missing site in path")
			return
		}

		org, ok := mux.Vars(r)[org]
		if !ok {
			httpErrf(w, http.StatusBadRequest, "missing org in path")
			return
		}

		repo, ok := mux.Vars(r)[repo]
		if !ok {
			httpErrf(w, http.StatusBadRequest, "missing repo in path")
			return
		}

		buildImg := defaultBuildImg
		req := new(startBuildReq)
		if err := json.NewDecoder(r.Body).Decode(req); err == nil {
			if req.BuildEnv != "" {
				buildImg = req.BuildEnv
			}
		}
		defer r.Body.Close()

		containerOpts := createContainerOpts(buildImg, workdir, site, org, repo)
		container, err := dockerCl.CreateContainer(containerOpts)
		if err != nil {
			log.Errf("creating container [%s]", err)
			httpErrf(w, http.StatusInternalServerError, "error creating container [%s]", err)
			return
		}

		hostConfig := &docker.HostConfig{Binds: []string{fmt.Sprintf("%s:%s", workdir, absPwd)}}
		if err := dockerCl.StartContainer(container.ID, hostConfig); err != nil {
			log.Errf("starting container [%s]", err)
			httpErrf(w, http.StatusInternalServerError, "error starting container [%s]", err)
			return
		}

		attachOpts, outputReader := attachToContainerOpts(container.ID)
		errCh := make(chan error)
		go func() {
			if err := dockerCl.AttachToContainer(attachOpts); err != nil {
				errCh <- err
			}
		}()

		w.WriteHeader(http.StatusCreated)
		go func(reader io.Reader) {
			scanner := bufio.NewScanner(reader)
			for scanner.Scan() {
				fmt.Fprintf(w, "%s\n", scanner.Text())
				flusher.Flush()
			}
			if err := scanner.Err(); err != nil {
				fmt.Fprintf(w, "error with scanner in attached container [%s]\n", err)
			}
		}(outputReader)

		code, err := dockerCl.WaitContainer(container.ID)
		if err != nil {
			log.Errf("waiting for container %s [%s]", container.ID, err)
			return
		}
		w.Write([]byte(fmt.Sprintf("exited with error code %d\n", code)))

		removeOpts := docker.RemoveContainerOptions{
			ID:            container.ID,
			RemoveVolumes: true,
			Force:         true,
		}
		if err := dockerCl.RemoveContainer(removeOpts); err != nil {
			log.Errf("removing container %s [%s]", container.ID, err)
		}
	})
}
<|endoftext|>


<commit_before>package discovery

import (
	"fmt"
	"time"

	"github.com/sayden/docker-commander/Godeps/_workspace/src/github.com/sayden/etcd/client"
	"github.com/sayden/docker-commander/Godeps/_workspace/src/golang.org/x/net/context"
)

// EtcdClient struct to use with ETCD clusters
// TODO Make it work with an ETCD cluster instead of only one node
type EtcdClient struct {
	Host string
}

// WatchHosts TODO use some goroutines here
func (e *EtcdClient) WatchHosts() {
	kapi, err := e.getClient()

	if err != nil {
		panic("")
	}

	o := client.WatcherOptions{Recursive: true}
	w := kapi.Watcher("/test", &o)

	r, err := w.Next(context.Background())

	//TODO DELETE
	opGet := client.GetOptions{Recursive: true}
	respNodes, err := kapi.Get(context.Background(), "/test", &opGet)
	ns := respNodes.Node.Nodes

	// TODO Make a break action to stop recursivity
	if err == nil {
		if len(ns) == 0 {
			fmt.Println("Exiting app")
		} else {
			log.Printf("%q action triggered on %q to set it to %q\n", r.Action, r.Node.Key, r.Node.Value)
			log.Println(len(ns))
			e.WatchHosts()
		}
	} else {
		log.Fatal(err)
	}
}

// ListHosts returns connected swarm nodes
func (e *EtcdClient) ListHosts() ([]Node, error) {
	kapi, err := e.getClient()
	if err != nil {
		log.Fatal(err)
		return nil, err
	}

	// get "/docker/swarm/nodes" list
	o := client.GetOptions{Recursive: true}
	resp, err := kapi.Get(context.Background(), "/docker/swarm/nodes", &o)
	if err != nil {
		log.Fatal(err)
		return nil, err
	}

	if resp.Node.Dir == false {
		log.Fatal("Node not recognized as directory")
	}

	nodesArray := make([]Node, len(resp.Node.Nodes))
	for i, n := range resp.Node.Nodes {
		nodesArray[i] = Node{IP: n.Value}
	}

	return nodesArray, nil
}

func (e *EtcdClient) getClient() (client.KeysAPI, error) {
	cfg := client.Config{
		Endpoints: []string{e.Host},
		Transport: client.DefaultTransport,
		// set timeout per request to fail fast when the target endpoint is unavailable
		HeaderTimeoutPerRequest: time.Second,
	}
	c, err := client.New(cfg)
	if err != nil {
		log.Fatal(err)
		return nil, err
	}

	kapi := client.NewKeysAPI(c)
	return kapi, nil
}
<commit_msg>etcd import bug fixed<commit_after>package discovery

import (
	"fmt"
	"log"
	"time"

	"github.com/coreos/etcd/client"
	"github.com/sayden/docker-commander/Godeps/_workspace/src/golang.org/x/net/context"
)

// EtcdClient struct to use with ETCD clusters
// TODO Make it work with an ETCD cluster instead of only one node
type EtcdClient struct {
	Host string
}

// WatchHosts TODO use some goroutines here
func (e *EtcdClient) WatchHosts() {
	kapi, err := e.getClient()

	if err != nil {
		panic("")
	}

	o := client.WatcherOptions{Recursive: true}
	w := kapi.Watcher("/test", &o)

	r, err := w.Next(context.Background())

	//TODO DELETE
	opGet := client.GetOptions{Recursive: true}
	respNodes, err := kapi.Get(context.Background(), "/test", &opGet)
	ns := respNodes.Node.Nodes

	// TODO Make a break action to stop recursivity
	if err == nil {
		if len(ns) == 0 {
			fmt.Println("Exiting app")
		} else {
			log.Printf("%q action triggered on %q to set it to %q\n", r.Action, r.Node.Key, r.Node.Value)
			log.Println(len(ns))
			e.WatchHosts()
		}
	} else {
		log.Fatal(err)
	}
}

// ListHosts returns connected swarm nodes
func (e *EtcdClient) ListHosts() ([]Node, error) {
	kapi, err := e.getClient()
	if err != nil {
		log.Fatal(err)
		return nil, err
	}

	// get "/docker/swarm/nodes" list
	o := client.GetOptions{Recursive: true}
	resp, err := kapi.Get(context.Background(), "/docker/swarm/nodes", &o)
	if err != nil {
		log.Fatal(err)
		return nil, err
	}

	if resp.Node.Dir == false {
		log.Fatal("Node not recognized as directory")
	}

	nodesArray := make([]Node, len(resp.Node.Nodes))
	for i, n := range resp.Node.Nodes {
		nodesArray[i] = Node{IP: n.Value}
	}

	return nodesArray, nil
}

func (e *EtcdClient) getClient() (client.KeysAPI, error) {
	cfg := client.Config{
		Endpoints: []string{e.Host},
		Transport: client.DefaultTransport,
		// set timeout per request to fail fast when the target endpoint is unavailable
		HeaderTimeoutPerRequest: time.Second,
	}
	c, err := client.New(cfg)
	if err != nil {
		log.Fatal(err)
		return nil, err
	}

	kapi := client.NewKeysAPI(c)
	return kapi, nil
}
<|endoftext|>


<commit_before>package gherkin

import (
	"fmt"
	"github.com/cucumber/cucumber-messages-go"
	gio "github.com/gogo/protobuf/io"
	"io"
	"io/ioutil"
	"strings"
)

func GherkinMessages(paths []string, sourceStream io.Reader, language string, includeSource bool, includeGherkinDocument bool, includePickles bool) ([]messages.Wrapper, error) {
	var result []messages.Wrapper

	processSource := func(source *messages.Source) error {
		if includeSource {
			result = append(result, messages.Wrapper{
				Message: &messages.Wrapper_Source{
					Source: source,
				},
			})
		}

		doc, err := ParseGherkinDocumentForLanguage(strings.NewReader(source.Data), language)
		if errs, ok := err.(parseErrors); ok {
			// expected parse errors
			for _, err := range errs {
				if pe, ok := err.(*parseError); ok {
					result = append(result, pe.asAttachment(source.Uri))
				} else {
					return fmt.Errorf("parse feature file: %s, unexpected error: %+v\n", source.Uri, err)
				}
			}
			return nil
		}

		if includeGherkinDocument {
			doc.Uri = source.Uri
			result = append(result, messages.Wrapper{
				Message: &messages.Wrapper_GherkinDocument{
					GherkinDocument: doc,
				},
			})
		}

		if includePickles {
			for _, pickle := range Pickles(*doc, source.Uri) {
				result = append(result, messages.Wrapper{
					Message: &messages.Wrapper_Pickle{
						Pickle: pickle,
					},
				})
			}
		}
		return nil
	}

	if len(paths) == 0 {
		reader := gio.NewDelimitedReader(sourceStream, 2048)
		for {
			source := &messages.Source{}
			if err := reader.ReadMsg(source); err != nil {
				break
			}
			processSource(source)
		}
	} else {
		for _, path := range paths {
			in, err := ioutil.ReadFile(path)
			if err != nil {
				return result, fmt.Errorf("read feature file: %s - %+v", path, err)
			}
			source := &messages.Source{
				Uri:  path,
				Data: string(in),
				Media: &messages.Media{
					Encoding:    "UTF-8",
					ContentType: "text/x.cucumber.gherkin+plain",
				},
			}
			processSource(source)
		}
	}

	return result, nil
}

func (a *parseError) asAttachment(uri string) messages.Wrapper {
	return messages.Wrapper{
		Message: &messages.Wrapper_Attachment{
			Attachment: &messages.Attachment{
				Data: a.Error(),
				Source: &messages.SourceReference{
					Uri: uri,
					Location: &messages.Location{
						Line:   uint32(a.loc.Line),
						Column: uint32(a.loc.Column),
					},
				},
			}},
	}
}
<commit_msg>gherkin: go: Set max size to something big. Closes #449<commit_after>package gherkin

import (
	"fmt"
	"github.com/cucumber/cucumber-messages-go"
	gio "github.com/gogo/protobuf/io"
	"io"
	"io/ioutil"
	"math"
	"strings"
)

func GherkinMessages(paths []string, sourceStream io.Reader, language string, includeSource bool, includeGherkinDocument bool, includePickles bool) ([]messages.Wrapper, error) {
	var result []messages.Wrapper

	processSource := func(source *messages.Source) error {
		if includeSource {
			result = append(result, messages.Wrapper{
				Message: &messages.Wrapper_Source{
					Source: source,
				},
			})
		}

		doc, err := ParseGherkinDocumentForLanguage(strings.NewReader(source.Data), language)
		if errs, ok := err.(parseErrors); ok {
			// expected parse errors
			for _, err := range errs {
				if pe, ok := err.(*parseError); ok {
					result = append(result, pe.asAttachment(source.Uri))
				} else {
					return fmt.Errorf("parse feature file: %s, unexpected error: %+v\n", source.Uri, err)
				}
			}
			return nil
		}

		if includeGherkinDocument {
			doc.Uri = source.Uri
			result = append(result, messages.Wrapper{
				Message: &messages.Wrapper_GherkinDocument{
					GherkinDocument: doc,
				},
			})
		}

		if includePickles {
			for _, pickle := range Pickles(*doc, source.Uri) {
				result = append(result, messages.Wrapper{
					Message: &messages.Wrapper_Pickle{
						Pickle: pickle,
					},
				})
			}
		}
		return nil
	}

	if len(paths) == 0 {
		reader := gio.NewDelimitedReader(sourceStream, math.MaxInt32)
		for {
			source := &messages.Source{}
			if err := reader.ReadMsg(source); err != nil {
				break
			}
			processSource(source)
		}
	} else {
		for _, path := range paths {
			in, err := ioutil.ReadFile(path)
			if err != nil {
				return result, fmt.Errorf("read feature file: %s - %+v", path, err)
			}
			source := &messages.Source{
				Uri:  path,
				Data: string(in),
				Media: &messages.Media{
					Encoding:    "UTF-8",
					ContentType: "text/x.cucumber.gherkin+plain",
				},
			}
			processSource(source)
		}
	}

	return result, nil
}

func (a *parseError) asAttachment(uri string) messages.Wrapper {
	return messages.Wrapper{
		Message: &messages.Wrapper_Attachment{
			Attachment: &messages.Attachment{
				Data: a.Error(),
				Source: &messages.SourceReference{
					Uri: uri,
					Location: &messages.Location{
						Line:   uint32(a.loc.Line),
						Column: uint32(a.loc.Column),
					},
				},
			}},
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

// +build !windows
// socket_nix.go

package libkb

import (
	"fmt"
	"net"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/keybase/client/go/logger"
)

// Though I've never seen this come up in production, it definitely comes up
// in systests that multiple go-routines might race over the current
// working directory as they do the (chdir && dial) dance below. Make sure
// a lock is held whenever operating on sockets, so that two racing goroutines
// can't conflict here.
var bindLock sync.Mutex

func (s SocketInfo) BindToSocket() (ret net.Listener, err error) {

	// Lock so that multiple goroutines can't race over current working dir.
	// See note above.
	bindLock.Lock()
	defer bindLock.Unlock()

	bindFile := s.bindFile
	what := fmt.Sprintf("SocketInfo#BindToSocket(unix:%s)", bindFile)
	defer Trace(s.log, what, func() error { return err })()

	if err := MakeParentDirs(s.log, bindFile); err != nil {
		return nil, err
	}

	// Path can't be longer than N characters.
	// In this case Chdir to the file directory first and use a local path.
	// On many linuxes, N=108, on some N=106, and on macOS N=104.
	// N=104 is the lowest I know of.
	// It's the length of the path buffer in sockaddr_un.
	// And there may be a null-terminator in there, not sure, so make it 103 for good luck.
	// https://github.com/golang/go/issues/6895#issuecomment-98006662
	// https://gist.github.com/mlsteele/16dc5b6eb3d112b914183928c9af71b8#file-un-h-L79
	// We could always Chdir, but then this function would be non-threadsafe more of the time.
	// Pick your poison.
	if len(bindFile) >= 103 {
		prevWd, err := os.Getwd()
		if err != nil {
			return nil, fmt.Errorf("Error getting working directory: %s", err)
		}
		s.log.Warning("| Changing current working directory because path for binding is too long")
		dir := filepath.Dir(bindFile)
		s.log.Debug("| Chdir(%s)", dir)
		if err := os.Chdir(dir); err != nil {
			return nil, fmt.Errorf("Path can't be longer than 108 characters (failed to chdir): %s", err)
		}

		defer func() {
			s.log.Debug("| Chdir(%s)", prevWd)
			os.Chdir(prevWd)
		}()

		bindFile = filepath.Base(bindFile)
	}

	s.log.Info("| net.Listen on unix:%s", bindFile)
	ret, err = net.Listen("unix", bindFile)
	if err != nil {
		s.log.Warning("net.Listen failed with: %s", err.Error())
	}
	return ret, err
}

func (s SocketInfo) DialSocket() (net.Conn, error) {
	errs := []error{}
	for _, file := range s.dialFiles {
		ret, err := s.dialSocket(file)
		if err == nil {
			return ret, nil
		}
		errs = append(errs, err)
	}
	return nil, CombineErrors(errs...)
}

func (s SocketInfo) dialSocket(dialFile string) (ret net.Conn, err error) {

	// Lock so that multiple goroutines can't race over current working dir.
	// See note above.
	bindLock.Lock()
	defer bindLock.Unlock()

	what := fmt.Sprintf("SocketInfo#dialSocket(unix:%s)", dialFile)
	defer Trace(s.log, what, func() error { return err })()

	if dialFile == "" {
		return nil, fmt.Errorf("Can't dial empty path")
	}

	// Path can't be longer than 108 characters.
	// In this case Chdir to the file directory first.
	// https://github.com/golang/go/issues/6895#issuecomment-98006662
	if len(dialFile) >= 108 {
		prevWd, err := os.Getwd()
		if err != nil {
			return nil, fmt.Errorf("Error getting working directory: %s", err)
		}
		s.log.Warning("| Changing current working directory because path for dialing is too long")
		dir := filepath.Dir(dialFile)
		s.log.Debug("| os.Chdir(%s)", dir)
		if err := os.Chdir(dir); err != nil {
			return nil, fmt.Errorf("Path can't be longer than 108 characters (failed to chdir): %s", err)
		}
		defer os.Chdir(prevWd)
		dialFile = filepath.Base(dialFile)
	}

	s.log.Debug("| net.Dial(unix:%s)", dialFile)
	return net.Dial("unix", dialFile)
}

func NewSocket(g *GlobalContext) (ret Socket, err error) {
	var dialFiles []string
	dialFiles, err = g.Env.GetSocketDialFiles()
	if err != nil {
		return
	}
	var bindFile string
	bindFile, err = g.Env.GetSocketBindFile()
	if err != nil {
		return
	}
	log := g.Log
	if log == nil {
		log = logger.NewNull()
	}
	ret = SocketInfo{
		log:       log,
		dialFiles: dialFiles,
		bindFile:  bindFile,
	}
	return
}

func NewSocketWithFiles(
	log logger.Logger, bindFile string, dialFiles []string) Socket {
	return SocketInfo{
		log:       log,
		bindFile:  bindFile,
		dialFiles: dialFiles,
	}
}

// net.errClosing isn't exported, so do this.. UGLY!
func IsSocketClosedError(e error) bool {
	return strings.HasSuffix(e.Error(), "use of closed network connection")
}
<commit_msg>turn a socket bind warning into a debug<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

// +build !windows
// socket_nix.go

package libkb

import (
	"fmt"
	"net"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/keybase/client/go/logger"
)

// Though I've never seen this come up in production, it definitely comes up
// in systests that multiple go-routines might race over the current
// working directory as they do the (chdir && dial) dance below. Make sure
// a lock is held whenever operating on sockets, so that two racing goroutines
// can't conflict here.
var bindLock sync.Mutex

func (s SocketInfo) BindToSocket() (ret net.Listener, err error) {

	// Lock so that multiple goroutines can't race over current working dir.
	// See note above.
	bindLock.Lock()
	defer bindLock.Unlock()

	bindFile := s.bindFile
	what := fmt.Sprintf("SocketInfo#BindToSocket(unix:%s)", bindFile)
	defer Trace(s.log, what, func() error { return err })()

	if err := MakeParentDirs(s.log, bindFile); err != nil {
		return nil, err
	}

	// Path can't be longer than N characters.
	// In this case Chdir to the file directory first and use a local path.
	// On many linuxes, N=108, on some N=106, and on macOS N=104.
	// N=104 is the lowest I know of.
	// It's the length of the path buffer in sockaddr_un.
	// And there may be a null-terminator in there, not sure, so make it 103 for good luck.
	// https://github.com/golang/go/issues/6895#issuecomment-98006662
	// https://gist.github.com/mlsteele/16dc5b6eb3d112b914183928c9af71b8#file-un-h-L79
	// We could always Chdir, but then this function would be non-threadsafe more of the time.
	// Pick your poison.
	if len(bindFile) >= 103 {
		prevWd, err := os.Getwd()
		if err != nil {
			return nil, fmt.Errorf("Error getting working directory: %s", err)
		}
		s.log.Debug("| Changing current working directory because path for binding is too long")
		dir := filepath.Dir(bindFile)
		s.log.Debug("| Chdir(%s)", dir)
		if err := os.Chdir(dir); err != nil {
			return nil, fmt.Errorf("Path can't be longer than 108 characters (failed to chdir): %s", err)
		}

		defer func() {
			s.log.Debug("| Chdir(%s)", prevWd)
			os.Chdir(prevWd)
		}()

		bindFile = filepath.Base(bindFile)
	}

	s.log.Info("| net.Listen on unix:%s", bindFile)
	ret, err = net.Listen("unix", bindFile)
	if err != nil {
		s.log.Warning("net.Listen failed with: %s", err.Error())
	}
	return ret, err
}

func (s SocketInfo) DialSocket() (net.Conn, error) {
	errs := []error{}
	for _, file := range s.dialFiles {
		ret, err := s.dialSocket(file)
		if err == nil {
			return ret, nil
		}
		errs = append(errs, err)
	}
	return nil, CombineErrors(errs...)
}

func (s SocketInfo) dialSocket(dialFile string) (ret net.Conn, err error) {

	// Lock so that multiple goroutines can't race over current working dir.
	// See note above.
	bindLock.Lock()
	defer bindLock.Unlock()

	what := fmt.Sprintf("SocketInfo#dialSocket(unix:%s)", dialFile)
	defer Trace(s.log, what, func() error { return err })()

	if dialFile == "" {
		return nil, fmt.Errorf("Can't dial empty path")
	}

	// Path can't be longer than 108 characters.
	// In this case Chdir to the file directory first.
	// https://github.com/golang/go/issues/6895#issuecomment-98006662
	if len(dialFile) >= 108 {
		prevWd, err := os.Getwd()
		if err != nil {
			return nil, fmt.Errorf("Error getting working directory: %s", err)
		}
		s.log.Warning("| Changing current working directory because path for dialing is too long")
		dir := filepath.Dir(dialFile)
		s.log.Debug("| os.Chdir(%s)", dir)
		if err := os.Chdir(dir); err != nil {
			return nil, fmt.Errorf("Path can't be longer than 108 characters (failed to chdir): %s", err)
		}
		defer os.Chdir(prevWd)
		dialFile = filepath.Base(dialFile)
	}

	s.log.Debug("| net.Dial(unix:%s)", dialFile)
	return net.Dial("unix", dialFile)
}

func NewSocket(g *GlobalContext) (ret Socket, err error) {
	var dialFiles []string
	dialFiles, err = g.Env.GetSocketDialFiles()
	if err != nil {
		return
	}
	var bindFile string
	bindFile, err = g.Env.GetSocketBindFile()
	if err != nil {
		return
	}
	log := g.Log
	if log == nil {
		log = logger.NewNull()
	}
	ret = SocketInfo{
		log:       log,
		dialFiles: dialFiles,
		bindFile:  bindFile,
	}
	return
}

func NewSocketWithFiles(
	log logger.Logger, bindFile string, dialFiles []string) Socket {
	return SocketInfo{
		log:       log,
		bindFile:  bindFile,
		dialFiles: dialFiles,
	}
}

// net.errClosing isn't exported, so do this.. UGLY!
func IsSocketClosedError(e error) bool {
	return strings.HasSuffix(e.Error(), "use of closed network connection")
}
<|endoftext|>


<commit_before>// Copyright 2015-2016 John Weldon. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE.md file.

/*
Package dispatcher provides the mechanism for watching and processing
emails that appear in a designated pickup folder, and removes them
after successfully sending them, or else moves the failed messages to
the configured badmail folder.
*/
package dispatcher

import (
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/golang/glog"
)

type folderQueue struct {
	mailqueue string
	badmail   string
}

// NewPickupFolderQueue returns an implementation of MailQueueDispatcher
// that watches a mailqueue folder and consumes messages that are left
// there and if it fails to consume the message, moves the message to a
// badmail folder.
func NewPickupFolderQueue(mailqueue, badmail string) MailQueueDispatcher {
	return &folderQueue{mailqueue: mailqueue, badmail: badmail}
}

// Process implements the MailQueueDispatcher interface, and walks the
// mailqueue folder and sends the found messages to the callbackFn.
func (q *folderQueue) Process(callbackFn MailQueueCallbackFn) error {
	return filepath.Walk(q.mailqueue, q.processItem(callbackFn))
}

func (q *folderQueue) processItem(fn MailQueueCallbackFn) filepath.WalkFunc {
	return func(path string, info os.FileInfo, err2 error) error {
		glog.Infof("processing %q", path)
		if info.IsDir() {
			if path == q.mailqueue {
				return nil
			}
			return filepath.SkipDir
		}

		raw, err := ioutil.ReadFile(path)
		if err != nil {
			q.markBad(path, info)
			return nil
		}

		if fn(raw) {
			q.markComplete(path)
		} else {
			q.markBad(path, info)
		}
		return nil
	}
}

func (q *folderQueue) markBad(path string, info os.FileInfo) {
	glog.Errorf("processing %q was unsuccessful", path)
	target := filepath.Join(q.badmail, info.Name())
	err := os.Rename(path, target)
	if err != nil {
		glog.Errorf("moving %q to %q: %q", path, target, err)
	}
}

func (q *folderQueue) markComplete(path string) {
	if err := os.Remove(path); err != nil {
		glog.Errorf("removing file %q: %q", path, err)
	}
}
<commit_msg>Decrease logging slightly<commit_after>// Copyright 2015-2016 John Weldon. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE.md file.

/*
Package dispatcher provides the mechanism for watching and processing
emails that appear in a designated pickup folder, and removes them
after successfully sending them, or else moves the failed messages to
the configured badmail folder.
*/
package dispatcher

import (
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/golang/glog"
)

type folderQueue struct {
	mailqueue string
	badmail   string
}

// NewPickupFolderQueue returns an implementation of MailQueueDispatcher
// that watches a mailqueue folder and consumes messages that are left
// there and if it fails to consume the message, moves the message to a
// badmail folder.
func NewPickupFolderQueue(mailqueue, badmail string) MailQueueDispatcher {
	return &folderQueue{mailqueue: mailqueue, badmail: badmail}
}

// Process implements the MailQueueDispatcher interface, and walks the
// mailqueue folder and sends the found messages to the callbackFn.
func (q *folderQueue) Process(callbackFn MailQueueCallbackFn) error {
	return filepath.Walk(q.mailqueue, q.processItem(callbackFn))
}

func (q *folderQueue) processItem(fn MailQueueCallbackFn) filepath.WalkFunc {
	return func(path string, info os.FileInfo, err2 error) error {
		glog.V(2).Infof("processing %q", path)
		if info.IsDir() {
			if path == q.mailqueue {
				return nil
			}
			return filepath.SkipDir
		}

		raw, err := ioutil.ReadFile(path)
		if err != nil {
			q.markBad(path, info)
			return nil
		}

		if fn(raw) {
			q.markComplete(path)
		} else {
			q.markBad(path, info)
		}
		return nil
	}
}

func (q *folderQueue) markBad(path string, info os.FileInfo) {
	glog.Errorf("processing %q was unsuccessful", path)
	target := filepath.Join(q.badmail, info.Name())
	err := os.Rename(path, target)
	if err != nil {
		glog.Errorf("moving %q to %q: %q", path, target, err)
	}
}

func (q *folderQueue) markComplete(path string) {
	if err := os.Remove(path); err != nil {
		glog.Errorf("removing file %q: %q", path, err)
	}
}
<|endoftext|>


<commit_before>// package assert provides convenience assert methods to complement
// the built in go testing library. It's intended to add onto standard
// Go tests. Example usage:
//	func TestSomething(t *testing.T) {
//		i, err := doSomething()
//		assert.NoErr(err)
//		assert.Equal(i, 123, "returned integer")
//	}
package assert

import (
	"fmt"
	"runtime"
	"testing"
	"reflect"
)

// callerStr returns a string representation of the code numFrames stack
// frames above the code that called callerStr
func callerStr(numFrames int) string {
	_, file, line, _ := runtime.Caller(1 + numFrames)
	return fmt.Sprintf("%s:%d", file, line)
}

// callerStrErrorf calls t.Errorf with fmtStr and vals in it, prefixed
// by a callerStr representation of the code numFrames above the caller of
// this function
func callerStrf(numFrames int, fmtStr string, vals ...interface{}) string {
	origStr := fmt.Sprintf(fmtStr, vals...)
	return fmt.Sprintf("%s: %s", callerStr(1+numFrames), origStr)
}

// True fails the test if b is false. on failure, it calls
// t.Errorf(fmtStr, vals...)
func True(t *testing.T, b bool, fmtStr string, vals ...interface{}) {
	if !b {
		t.Errorf(callerStrf(1, fmtStr, vals...))
	}
}

// False is the equivalent of True(t, !b, fmtStr, vals...).
func False(t *testing.T, b bool, fmtStr string, vals ...interface{}) {
	if b {
		t.Errorf(callerStrf(1, fmtStr, vals...))
	}
}

// Nil uses reflect.DeepEqual(i, nil) to determine if i is nil.
// if it's not, Nil calls t.Errorf explaining that the noun i is nil when it
// should have been
func Nil(t *testing.T, i interface{}, noun string) {
	if !reflect.DeepEqual(i, nil) {
		t.Errorf(callerStrf(1, "the given %s [%+v] was not nil when it should have been", noun, i))
	}
}

// NotNil uses reflect.DeepEqual(i, nil) to determine if i is nil.
// if it is, NotNil calls t.Errorf explaining that the noun i is not nil when
// it shouldn't have been.
func NotNil(t *testing.T, i interface{}, noun string) {
	if reflect.DeepEqual(i, nil) {
		t.Errorf(callerStrf(1, "the given %s was nil when it shouldn't have been", noun))
	}
}

// Err calls t.Errorf if expected is not equal to actual.
// it uses reflect.DeepEqual to determine if the errors are equal
func Err(t *testing.T, expected error, actual error) {
	if !reflect.DeepEqual(expected, actual) {
		t.Errorf(callerStrf(1, "expected error %s but got %s", expected, actual))
	}
}

// Equal ensures that the actual value returned from a test was equal to an
// expected. it uses reflect.DeepEqual to do so.
// name is the name used to describe the values being compared.
// it's used in the error string if actual != expected.
func Equal(t *testing.T, actual, expected interface{}, noun string) {
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf(callerStrf(1, "actual %s [%+v] != expected %s [%+v]", noun, actual, noun, expected))
	}
}
<commit_msg>adding ExistsErr and NoErr<commit_after>// package assert provides convenience assert methods to complement
// the built in go testing library. It's intended to add onto standard
// Go tests. Example usage:
//	func TestSomething(t *testing.T) {
//		i, err := doSomething()
//		assert.NoErr(err)
//		assert.Equal(i, 123, "returned integer")
//	}
package assert

import (
	"fmt"
	"runtime"
	"testing"
	"reflect"
)

// callerStr returns a string representation of the code numFrames stack
// frames above the code that called callerStr
func callerStr(numFrames int) string {
	_, file, line, _ := runtime.Caller(1 + numFrames)
	return fmt.Sprintf("%s:%d", file, line)
}

// callerStrErrorf calls t.Errorf with fmtStr and vals in it, prefixed
// by a callerStr representation of the code numFrames above the caller of
// this function
func callerStrf(numFrames int, fmtStr string, vals ...interface{}) string {
	origStr := fmt.Sprintf(fmtStr, vals...)
	return fmt.Sprintf("%s: %s", callerStr(1+numFrames), origStr)
}

// True fails the test if b is false. on failure, it calls
// t.Errorf(fmtStr, vals...)
func True(t *testing.T, b bool, fmtStr string, vals ...interface{}) {
	if !b {
		t.Errorf(callerStrf(1, fmtStr, vals...))
	}
}

// False is the equivalent of True(t, !b, fmtStr, vals...).
func False(t *testing.T, b bool, fmtStr string, vals ...interface{}) {
	if b {
		t.Errorf(callerStrf(1, fmtStr, vals...))
	}
}

// Nil uses reflect.DeepEqual(i, nil) to determine if i is nil.
// if it's not, Nil calls t.Errorf explaining that the noun i is nil when it
// should have been
func Nil(t *testing.T, i interface{}, noun string) {
	if !reflect.DeepEqual(i, nil) {
		t.Errorf(callerStrf(1, "the given %s [%+v] was not nil when it should have been", noun, i))
	}
}

// NotNil uses reflect.DeepEqual(i, nil) to determine if i is nil.
// if it is, NotNil calls t.Errorf explaining that the noun i is not nil when
// it shouldn't have been.
func NotNil(t *testing.T, i interface{}, noun string) {
	if reflect.DeepEqual(i, nil) {
		t.Errorf(callerStrf(1, "the given %s was nil when it shouldn't have been", noun))
	}
}

// Err calls t.Errorf if expected is not equal to actual.
// it uses reflect.DeepEqual to determine if the errors are equal
func Err(t *testing.T, expected error, actual error) {
	if !reflect.DeepEqual(expected, actual) {
		t.Errorf(callerStrf(1, "expected error %s but got %s", expected, actual))
	}
}

// if err == nil, ExistsErr calls t.Errorf explaining that the error described by noun was
// nil when it shouldn't have been
func ExistsErr(t *testing.T, err error, noun string) {
	if err == nil {
		t.Errorf(callerStrf(1, "given error for %s was nil when it shouldn't have been", noun))
	}
}

// NoErr calls t.Errorf if e is not nil.
func NoErr(t *testing.T, e error) {
	if e != nil {
		t.Errorf(callerStrf(1, "expected no error but got %s", e))
	}
}

// Equal ensures that the actual value returned from a test was equal to an
// expected. it uses reflect.DeepEqual to do so.
// name is the name used to describe the values being compared.
// it's used in the error string if actual != expected.
func Equal(t *testing.T, actual, expected interface{}, noun string) {
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf(callerStrf(1, "actual %s [%+v] != expected %s [%+v]", noun, actual, noun, expected))
	}
}
<|endoftext|>


<commit_before>package ast

import (
	"fmt"
	"regexp"
	"time"

	"github.com/influxdata/influxdb/influxql"
)

// NodeTypeOf is used by all Node to identify the node duration Marshal and Unmarshal
const NodeTypeOf = "typeOf"

// JSONNode is the intermediate type between Node and JSON serialization
type JSONNode map[string]interface{}

// Type adds the Node type information
func (j JSONNode) Type(typ string) JSONNode {
	j[NodeTypeOf] = typ
	return j
}

// Set adds the key/value to the JSONNode
func (j JSONNode) Set(key string, value interface{}) JSONNode {
	j[key] = value
	return j
}

// SetDuration adds key to the JSONNode but formats the duration in InfluxQL style
func (j JSONNode) SetDuration(key string, value time.Duration) JSONNode {
	return j.Set(key, influxql.FormatDuration(value))
}

// SetRegex adds key to the JSONNode but formats the regex as a string
func (j JSONNode) SetRegex(key string, value *regexp.Regexp) JSONNode {
	if value == nil {
		return j.Set(key, nil)
	}
	return j.Set(key, value.String())
}

// SetOperator adds key to JSONNode but formats the operator as a string
func (j JSONNode) SetOperator(key string, op TokenType) JSONNode {
	return j.Set(key, op.String())
}

// SetFunctionType adds key to JSONNode but formats the function type as a string
func (j JSONNode) SetFunctionType(key string, fn FuncType) JSONNode {
	return j.Set(key, fn.String())
}

// TypeOf returns the type of the node
func (j JSONNode) TypeOf() (string, error) {
	return j.String(NodeTypeOf)
}

// CheckTypeOf tests that the typeOf field is correctly set to typ.
func (j JSONNode) CheckTypeOf(typ string) error {
	t, ok := j[NodeTypeOf]
	if !ok {
		return fmt.Errorf("missing typeOf field")
	}

	if t != typ {
		return fmt.Errorf("error unmarshaling node type %s; received %s", typ, t)
	}
	return nil
}

// Has returns true if field exists
func (j JSONNode) Has(field string) bool {
	_, ok := j[field]
	return ok
}

// Field returns expected field or error if field doesn't exist
func (j JSONNode) Field(field string) (interface{}, error) {
	fld, ok := j[field]
	if !ok {
		return nil, fmt.Errorf("missing expected field %s", field)
	}
	return fld, nil
}

// String reads the field for a string value
func (j JSONNode) String(field string) (string, error) {
	s, err := j.Field(field)
	if err != nil {
		return "", err
	}

	str, ok := s.(string)
	if !ok {
		return "", fmt.Errorf("field %s is not a string value but is %T", field, s)
	}
	return str, nil
}

// Int64 reads the field for a int64 value
func (j JSONNode) Int64(field string) (int64, error) {
	n, err := j.Field(field)
	if err != nil {
		return 0, err
	}

	num, ok := n.(int64)
	if !ok {
		flt, ok := n.(float64)
		if !ok {
			return 0, fmt.Errorf("field %s is not an integer value but is %T", field, n)
		}
		num = int64(flt)
	}
	return num, nil
}

// Float64 reads the field for a float64 value
func (j JSONNode) Float64(field string) (float64, error) {
	n, err := j.Field(field)
	if err != nil {
		return 0, err
	}

	num, ok := n.(float64)
	if !ok {
		integer, ok := n.(int64)
		if !ok {
			return 0, fmt.Errorf("field %s is not a floating point value but is %T", field, n)
		}
		num = float64(integer)
	}
	return num, nil
}

// Strings reads the field an array of strings
func (j JSONNode) Strings(field string) ([]string, error) {
	s, err := j.Field(field)
	if err != nil {
		return nil, err
	}

	strs, ok := s.([]string)
	if !ok {
		return nil, fmt.Errorf("field %s is not an array of strings but is %T", field, s)
	}
	return strs, nil
}

// Duration reads the field and assumes the string is in InfluxQL Duration format.
func (j JSONNode) Duration(field string) (time.Duration, error) {
	d, err := j.Field(field)
	if err != nil {
		return 0, err
	}

	dur, ok := d.(string)
	if !ok {
		return 0, fmt.Errorf("field %s is not a string duration value but is %T", field, d)
	}

	return influxql.ParseDuration(dur)
}

// Regex reads the field and assumes the string is a regular expression.
func (j JSONNode) Regex(field string) (*regexp.Regexp, error) {
	r, err := j.Field(field)
	if err != nil {
		return nil, err
	}

	re, ok := r.(string)
	if !ok {
		return nil, fmt.Errorf("field %s is not a string regex value but is %T", field, r)
	}

	return regexp.Compile(re)
}

// Bool reads the field for a boolean value
func (j JSONNode) Bool(field string) (bool, error) {
	b, err := j.Field(field)
	if err != nil {
		return false, err
	}

	boolean, ok := b.(bool)
	if !ok {
		return false, fmt.Errorf("field %s is not a bool value but is %T", field, b)
	}
	return boolean, nil
}

// Operator reads the field for an TokenType operator value
func (j JSONNode) Operator(field string) (TokenType, error) {
	o, err := j.Field(field)
	if err != nil {
		return TokenError, err
	}

	op, ok := o.(string)
	if !ok {
		return TokenError, fmt.Errorf("field %s is not an operator value but is %T", field, o)
	}

	return NewTokenType(op)
}

// FunctionType reads the field for an FuncType value
func (j JSONNode) FunctionType(field string) (FuncType, error) {
	f, err := j.Field(field)
	if err != nil {
		return -1, err
	}

	fn, ok := f.(string)
	if !ok {
		return -1, fmt.Errorf("field %s is not a function type value but is %T", field, f)
	}

	return NewFuncType(fn)
}

// NodeList reads the field for a list of nodes
func (j JSONNode) NodeList(field string) ([]Node, error) {
	l, err := j.Field(field)
	if err != nil {
		return nil, err
	}
	list, ok := l.([]map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("field %s is not a list of values but is %T", field, l)
	}
	nodes := make([]Node, len(list))
	for i, lst := range list {
		nodes[i], err = j.getNode(lst)
		if err != nil {
			return nil, err
		}
	}

	return nodes, nil
}

// Node reads the field for a node
func (j JSONNode) Node(field string) (Node, error) {
	nn, err := j.Field(field)
	if err != nil {
		return nil, err
	}
	return j.getNode(nn)
}

// IDNode reads an IdentifierNode from the field
func (j JSONNode) IDNode(field string) (*IdentifierNode, error) {
	n, err := j.Node(field)
	if err != nil {
		return nil, err
	}
	id, ok := n.(*IdentifierNode)
	if !ok {
		return nil, fmt.Errorf("field %s is not an identifier node but is %T", field, n)
	}
	return id, nil
}

// RefNode reads a ReferenceNode from the field
func (j JSONNode) RefNode(field string) (*ReferenceNode, error) {
	n, err := j.Node(field)
	if err != nil {
		return nil, err
	}
	id, ok := n.(*ReferenceNode)
	if !ok {
		return nil, fmt.Errorf("field %s is not a reference node but is %T", field, n)
	}
	return id, nil
}

func (j JSONNode) getNode(nn interface{}) (Node, error) {
	nd, ok := nn.(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("expected node type but is %T", nn)
	}

	node := JSONNode(nd)
	typ, err := node.TypeOf()
	if err != nil {
		return nil, err
	}

	var n Node
	switch typ {
	case "number":
		n = &NumberNode{}
	case "dbrp":
		n = &DBRPNode{}
	case "duration":
		n = &DurationNode{}
	case "bool":
		n = &BoolNode{}
	case "unary":
		n = &UnaryNode{}
	case "binary":
		n = &BinaryNode{}
	case "declaration":
		n = &DeclarationNode{}
	case "typeDeclaration":
		n = &TypeDeclarationNode{}
	case "identifier":
		n = &IdentifierNode{}
	case "reference":
		n = &ReferenceNode{}
	case "string":
		n = &StringNode{}
	case "list":
		n = &ListNode{}
	case "regex":
		n = &RegexNode{}
	case "star":
		n = &StarNode{}
	case "func":
		n = &FunctionNode{}
	case "lambda":
		n = &LambdaNode{}
	case "program":
		n = &ProgramNode{}
	case "comment":
		n = &CommentNode{}
	}
	err = n.unmarshal(node)
	return n, err
}
<commit_msg>Fix unmarshaling of AST node lists<commit_after>package ast

import (
	"fmt"
	"regexp"
	"time"

	"github.com/influxdata/influxdb/influxql"
)

// NodeTypeOf is used by all Node to identify the node duration Marshal and Unmarshal
const NodeTypeOf = "typeOf"

// JSONNode is the intermediate type between Node and JSON serialization
type JSONNode map[string]interface{}

// Type adds the Node type information
func (j JSONNode) Type(typ string) JSONNode {
	j[NodeTypeOf] = typ
	return j
}

// Set adds the key/value to the JSONNode
func (j JSONNode) Set(key string, value interface{}) JSONNode {
	j[key] = value
	return j
}

// SetDuration adds key to the JSONNode but formats the duration in InfluxQL style
func (j JSONNode) SetDuration(key string, value time.Duration) JSONNode {
	return j.Set(key, influxql.FormatDuration(value))
}

// SetRegex adds key to the JSONNode but formats the regex as a string
func (j JSONNode) SetRegex(key string, value *regexp.Regexp) JSONNode {
	if value == nil {
		return j.Set(key, nil)
	}
	return j.Set(key, value.String())
}

// SetOperator adds key to JSONNode but formats the operator as a string
func (j JSONNode) SetOperator(key string, op TokenType) JSONNode {
	return j.Set(key, op.String())
}

// SetFunctionType adds key to JSONNode but formats the function type as a string
func (j JSONNode) SetFunctionType(key string, fn FuncType) JSONNode {
	return j.Set(key, fn.String())
}

// TypeOf returns the type of the node
func (j JSONNode) TypeOf() (string, error) {
	return j.String(NodeTypeOf)
}

// CheckTypeOf tests that the typeOf field is correctly set to typ.
func (j JSONNode) CheckTypeOf(typ string) error {
	t, ok := j[NodeTypeOf]
	if !ok {
		return fmt.Errorf("missing typeOf field")
	}

	if t != typ {
		return fmt.Errorf("error unmarshaling node type %s; received %s", typ, t)
	}
	return nil
}

// Has returns true if field exists
func (j JSONNode) Has(field string) bool {
	_, ok := j[field]
	return ok
}

// Field returns expected field or error if field doesn't exist
func (j JSONNode) Field(field string) (interface{}, error) {
	fld, ok := j[field]
	if !ok {
		return nil, fmt.Errorf("missing expected field %s", field)
	}
	return fld, nil
}

// String reads the field for a string value
func (j JSONNode) String(field string) (string, error) {
	s, err := j.Field(field)
	if err != nil {
		return "", err
	}

	str, ok := s.(string)
	if !ok {
		return "", fmt.Errorf("field %s is not a string value but is %T", field, s)
	}
	return str, nil
}

// Int64 reads the field for a int64 value
func (j JSONNode) Int64(field string) (int64, error) {
	n, err := j.Field(field)
	if err != nil {
		return 0, err
	}

	num, ok := n.(int64)
	if !ok {
		flt, ok := n.(float64)
		if !ok {
			return 0, fmt.Errorf("field %s is not an integer value but is %T", field, n)
		}
		num = int64(flt)
	}
	return num, nil
}

// Float64 reads the field for a float64 value
func (j JSONNode) Float64(field string) (float64, error) {
	n, err := j.Field(field)
	if err != nil {
		return 0, err
	}

	num, ok := n.(float64)
	if !ok {
		integer, ok := n.(int64)
		if !ok {
			return 0, fmt.Errorf("field %s is not a floating point value but is %T", field, n)
		}
		num = float64(integer)
	}
	return num, nil
}

// Strings reads the field an array of strings
func (j JSONNode) Strings(field string) ([]string, error) {
	s, err := j.Field(field)
	if err != nil {
		return nil, err
	}

	strs, ok := s.([]string)
	if !ok {
		return nil, fmt.Errorf("field %s is not an array of strings but is %T", field, s)
	}
	return strs, nil
}

// Duration reads the field and assumes the string is in InfluxQL Duration format.
func (j JSONNode) Duration(field string) (time.Duration, error) {
	d, err := j.Field(field)
	if err != nil {
		return 0, err
	}

	dur, ok := d.(string)
	if !ok {
		return 0, fmt.Errorf("field %s is not a string duration value but is %T", field, d)
	}

	return influxql.ParseDuration(dur)
}

// Regex reads the field and assumes the string is a regular expression.
func (j JSONNode) Regex(field string) (*regexp.Regexp, error) {
	r, err := j.Field(field)
	if err != nil {
		return nil, err
	}

	re, ok := r.(string)
	if !ok {
		return nil, fmt.Errorf("field %s is not a string regex value but is %T", field, r)
	}

	return regexp.Compile(re)
}

// Bool reads the field for a boolean value
func (j JSONNode) Bool(field string) (bool, error) {
	b, err := j.Field(field)
	if err != nil {
		return false, err
	}

	boolean, ok := b.(bool)
	if !ok {
		return false, fmt.Errorf("field %s is not a bool value but is %T", field, b)
	}
	return boolean, nil
}

// Operator reads the field for an TokenType operator value
func (j JSONNode) Operator(field string) (TokenType, error) {
	o, err := j.Field(field)
	if err != nil {
		return TokenError, err
	}

	op, ok := o.(string)
	if !ok {
		return TokenError, fmt.Errorf("field %s is not an operator value but is %T", field, o)
	}

	return NewTokenType(op)
}

// FunctionType reads the field for an FuncType value
func (j JSONNode) FunctionType(field string) (FuncType, error) {
	f, err := j.Field(field)
	if err != nil {
		return -1, err
	}

	fn, ok := f.(string)
	if !ok {
		return -1, fmt.Errorf("field %s is not a function type value but is %T", field, f)
	}

	return NewFuncType(fn)
}

// NodeList reads the field for a list of nodes
func (j JSONNode) NodeList(field string) ([]Node, error) {
	l, err := j.Field(field)
	if err != nil {
		return nil, err
	}
	list, ok := l.([]interface{})
	if !ok {
		return nil, fmt.Errorf("field %s is not a list of values but is %T", field, l)
	}
	nodes := make([]Node, len(list))
	for i, lst := range list {
		nodes[i], err = j.getNode(lst)
		if err != nil {
			return nil, err
		}
	}

	return nodes, nil
}

// Node reads the field for a node
func (j JSONNode) Node(field string) (Node, error) {
	nn, err := j.Field(field)
	if err != nil {
		return nil, err
	}
	return j.getNode(nn)
}

// IDNode reads an IdentifierNode from the field
func (j JSONNode) IDNode(field string) (*IdentifierNode, error) {
	n, err := j.Node(field)
	if err != nil {
		return nil, err
	}
	id, ok := n.(*IdentifierNode)
	if !ok {
		return nil, fmt.Errorf("field %s is not an identifier node but is %T", field, n)
	}
	return id, nil
}

// RefNode reads a ReferenceNode from the field
func (j JSONNode) RefNode(field string) (*ReferenceNode, error) {
	n, err := j.Node(field)
	if err != nil {
		return nil, err
	}
	id, ok := n.(*ReferenceNode)
	if !ok {
		return nil, fmt.Errorf("field %s is not a reference node but is %T", field, n)
	}
	return id, nil
}

func (j JSONNode) getNode(nn interface{}) (Node, error) {
	nd, ok := nn.(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("expected node type but is %T", nn)
	}

	node := JSONNode(nd)
	typ, err := node.TypeOf()
	if err != nil {
		return nil, err
	}

	var n Node
	switch typ {
	case "number":
		n = &NumberNode{}
	case "dbrp":
		n = &DBRPNode{}
	case "duration":
		n = &DurationNode{}
	case "bool":
		n = &BoolNode{}
	case "unary":
		n = &UnaryNode{}
	case "binary":
		n = &BinaryNode{}
	case "declaration":
		n = &DeclarationNode{}
	case "typeDeclaration":
		n = &TypeDeclarationNode{}
	case "identifier":
		n = &IdentifierNode{}
	case "reference":
		n = &ReferenceNode{}
	case "string":
		n = &StringNode{}
	case "list":
		n = &ListNode{}
	case "regex":
		n = &RegexNode{}
	case "star":
		n = &StarNode{}
	case "func":
		n = &FunctionNode{}
	case "lambda":
		n = &LambdaNode{}
	case "program":
		n = &ProgramNode{}
	case "comment":
		n = &CommentNode{}
	}
	err = n.unmarshal(node)
	return n, err
}
<|endoftext|>


<commit_before>package proxy

import (
	"fmt"
	"net/http"
	"net/url"
	"sort"

	"github.com/mailgun/vulcand/Godeps/_workspace/src/github.com/mailgun/log"
	"github.com/mailgun/vulcand/Godeps/_workspace/src/github.com/mailgun/oxy/forward"
	"github.com/mailgun/vulcand/Godeps/_workspace/src/github.com/mailgun/oxy/roundrobin"
	"github.com/mailgun/vulcand/Godeps/_workspace/src/github.com/mailgun/oxy/stream"
	"github.com/mailgun/vulcand/Godeps/_workspace/src/github.com/mailgun/oxy/utils"

	"github.com/mailgun/vulcand/engine"
)

type frontend struct {
	key         engine.FrontendKey
	mux         *mux
	frontend    engine.Frontend
	lb          *roundrobin.Rebalancer
	handler     http.Handler
	watcher     *RTWatcher
	backend     *backend
	middlewares map[engine.MiddlewareKey]engine.Middleware
	log         utils.Logger
}

func newFrontend(m *mux, f engine.Frontend, b *backend) (*frontend, error) {
	fr := &frontend{
		key:         engine.FrontendKey{Id: f.Id},
		frontend:    f,
		mux:         m,
		backend:     b,
		middlewares: make(map[engine.MiddlewareKey]engine.Middleware),
		log:         log.GetLogger(),
	}

	if err := fr.rebuild(); err != nil {
		return nil, err
	}
	b.linkFrontend(engine.FrontendKey{f.Id}, fr)
	return fr, nil
}

func (f *frontend) String() string {
	return fmt.Sprintf("%v frontend(wrap=%v)", f.mux, &f.frontend)
}

// syncs backend servers and rebalancer state
func syncServers(m *mux, rb *roundrobin.Rebalancer, backend *backend, w *RTWatcher) error {
	// First, collect and parse servers to add
	newServers := map[string]*url.URL{}
	for _, s := range backend.servers {
		u, err := url.Parse(s.URL)
		if err != nil {
			return fmt.Errorf("failed to parse url %v", s.URL)
		}
		newServers[s.URL] = u
	}

	// Memorize what endpoints exist in load balancer at the moment
	existingServers := map[string]*url.URL{}
	for _, s := range rb.Servers() {
		existingServers[s.String()] = s
	}

	// First, add endpoints, that should be added and are not in lb
	for _, s := range newServers {
		if _, exists := existingServers[s.String()]; !exists {
			if err := rb.UpsertServer(s); err != nil {
				log.Errorf("%v failed to add %v, err: %s", m, s, err)
			} else {
				log.Infof("%v add %v", m, s)
			}
			w.upsertServer(s)
		}
	}

	// Second, remove endpoints that should not be there any more
	for k, v := range existingServers {
		if _, exists := newServers[k]; !exists {
			if err := rb.RemoveServer(v); err != nil {
				log.Errorf("%v failed to remove %v, err: %v", m, v, err)
			} else {
				log.Infof("%v removed %v", m, v)
			}
			w.removeServer(v)
		}
	}
	return nil
}

func (f *frontend) updateTransport(t *http.Transport) error {
	return f.rebuild()
}

func (f *frontend) sortedMiddlewares() []engine.Middleware {
	vals := make([]engine.Middleware, 0, len(f.middlewares))
	for _, m := range f.middlewares {
		vals = append(vals, m)
	}
	sort.Sort(sort.Reverse(&middlewareSorter{ms: vals}))
	return vals
}

func (f *frontend) rebuild() error {
	settings := f.frontend.HTTPSettings()

	// set up forwarder
	fwd, err := forward.New(
		forward.Logger(f.log),
		forward.RoundTripper(f.backend.transport),
		forward.Rewriter(
			&forward.HeaderRewriter{
				Hostname:           settings.Hostname,
				TrustForwardHeader: settings.TrustForwardHeader,
			}))

	// reporter and rtwatcher will be observing metrics
	rp := NewReporter(fwd, f.mux.options.MetricsClient, engine.FrontendKey{Id: f.frontend.Id})
	watcher, err := NewWatcher(rp)
	if err != nil {
		return err
	}

	// Create a load balancer
	rr, err := roundrobin.New(rp)
	if err != nil {
		return err
	}

	// Rebalancer will readjust load balancer weights based on error ratios
	rb, err := roundrobin.NewRebalancer(rr, roundrobin.RebalancerLogger(f.log))
	if err != nil {
		return err
	}

	// create middlewares sorted by priority and chain them
	middlewares := f.sortedMiddlewares()
	handlers := make([]http.Handler, len(middlewares))
	for i, m := range middlewares {
		var prev http.Handler
		if i == 0 {
			prev = rb
		} else {
			prev = handlers[i-1]
		}
		h, err := m.Middleware.NewHandler(prev)
		if err != nil {
			return err
		}
		handlers[i] = h
	}

	var next http.Handler
	if len(handlers) != 0 {
		next = handlers[len(handlers)-1]
	} else {
		next = rb
	}

	// stream will retry and replay requests, fix encodings
	if settings.FailoverPredicate == "" {
		settings.FailoverPredicate = `IsNetworkError() && RequestMethod() == "GET" && Attempts() < 2`
	}
	str, err := stream.New(next,
		stream.Logger(f.log),
		stream.Retry(settings.FailoverPredicate),
		stream.MaxRequestBodyBytes(settings.Limits.MaxBodyBytes),
		stream.MemRequestBodyBytes(settings.Limits.MaxMemBodyBytes))
	if err != nil {
		return err
	}

	if err := syncServers(f.mux, rb, f.backend, watcher); err != nil {
		return err
	}

	// Add the frontend to the router
	if err := f.mux.router.Handle(f.frontend.Route, str); err != nil {
		return err
	}

	f.lb = rb
	f.handler = str
	f.watcher = watcher
	return nil
}

func (f *frontend) upsertMiddleware(fk engine.FrontendKey, mi engine.Middleware) error {
	f.middlewares[engine.MiddlewareKey{FrontendKey: fk, Id: mi.Id}] = mi
	return f.rebuild()
}

func (f *frontend) deleteMiddleware(mk engine.MiddlewareKey) error {
	delete(f.middlewares, mk)
	return f.rebuild()
}

func (f *frontend) updateBackend(b *backend) error {
	oldb := f.backend
	f.backend = b

	// Switching backends, set the new transport and perform switch
	if b.backend.Id != oldb.backend.Id {
		log.Infof("%v updating backend from %v to %v", f, &oldb, &f.backend)
		oldb.unlinkFrontend(f.key)
		b.linkFrontend(f.key, f)
		return f.rebuild()
	}
	return syncServers(f.mux, f.lb, f.backend, f.watcher)
}

// TODO: implement rollback in case of suboperation failure
func (f *frontend) update(ef engine.Frontend, b *backend) error {
	oldf := f.frontend
	f.frontend = ef

	if err := f.updateBackend(b); err != nil {
		return err
	}

	if oldf.Route != ef.Route {
		log.Infof("%v updating route from %v to %v", oldf.Route, ef.Route)
		if err := f.mux.router.Handle(ef.Route, f.handler); err != nil {
			return err
		}
		if err := f.mux.router.Remove(oldf.Route); err != nil {
			return err
		}
	}

	olds := oldf.HTTPSettings()
	news := ef.HTTPSettings()
	if !olds.Equals(news) {
		if err := f.rebuild(); err != nil {
			return err
		}
	}

	return nil
}

func (f *frontend) remove() error {
	f.backend.unlinkFrontend(f.key)
	return f.mux.router.Remove(f.frontend.Route)
}

type middlewareSorter struct {
	ms []engine.Middleware
}

func (s *middlewareSorter) Len() int {
	return len(s.ms)
}

func (s *middlewareSorter) Swap(i, j int) {
	s.ms[i], s.ms[j] = s.ms[j], s.ms[i]
}

func (s *middlewareSorter) Less(i, j int) bool {
	return s.ms[i].Priority < s.ms[j].Priority
}
<commit_msg>Fix metrics<commit_after>package proxy

import (
	"fmt"
	"net/http"
	"net/url"
	"sort"

	"github.com/mailgun/vulcand/Godeps/_workspace/src/github.com/mailgun/log"
	"github.com/mailgun/vulcand/Godeps/_workspace/src/github.com/mailgun/oxy/forward"
	"github.com/mailgun/vulcand/Godeps/_workspace/src/github.com/mailgun/oxy/roundrobin"
	"github.com/mailgun/vulcand/Godeps/_workspace/src/github.com/mailgun/oxy/stream"
	"github.com/mailgun/vulcand/Godeps/_workspace/src/github.com/mailgun/oxy/utils"

	"github.com/mailgun/vulcand/engine"
)

type frontend struct {
	key         engine.FrontendKey
	mux         *mux
	frontend    engine.Frontend
	lb          *roundrobin.Rebalancer
	handler     http.Handler
	watcher     *RTWatcher
	backend     *backend
	middlewares map[engine.MiddlewareKey]engine.Middleware
	log         utils.Logger
}

func newFrontend(m *mux, f engine.Frontend, b *backend) (*frontend, error) {
	fr := &frontend{
		key:         engine.FrontendKey{Id: f.Id},
		frontend:    f,
		mux:         m,
		backend:     b,
		middlewares: make(map[engine.MiddlewareKey]engine.Middleware),
		log:         log.GetLogger(),
	}

	if err := fr.rebuild(); err != nil {
		return nil, err
	}
	b.linkFrontend(engine.FrontendKey{f.Id}, fr)
	return fr, nil
}

func (f *frontend) String() string {
	return fmt.Sprintf("%v frontend(wrap=%v)", f.mux, &f.frontend)
}

// syncs backend servers and rebalancer state
func syncServers(m *mux, rb *roundrobin.Rebalancer, backend *backend, w *RTWatcher) error {
	// First, collect and parse servers to add
	newServers := map[string]*url.URL{}
	for _, s := range backend.servers {
		u, err := url.Parse(s.URL)
		if err != nil {
			return fmt.Errorf("failed to parse url %v", s.URL)
		}
		newServers[s.URL] = u
	}

	// Memorize what endpoints exist in load balancer at the moment
	existingServers := map[string]*url.URL{}
	for _, s := range rb.Servers() {
		existingServers[s.String()] = s
	}

	// First, add endpoints, that should be added and are not in lb
	for _, s := range newServers {
		if _, exists := existingServers[s.String()]; !exists {
			if err := rb.UpsertServer(s); err != nil {
				log.Errorf("%v failed to add %v, err: %s", m, s, err)
			} else {
				log.Infof("%v add %v", m, s)
			}
			w.upsertServer(s)
		}
	}

	// Second, remove endpoints that should not be there any more
	for k, v := range existingServers {
		if _, exists := newServers[k]; !exists {
			if err := rb.RemoveServer(v); err != nil {
				log.Errorf("%v failed to remove %v, err: %v", m, v, err)
			} else {
				log.Infof("%v removed %v", m, v)
			}
			w.removeServer(v)
		}
	}
	return nil
}

func (f *frontend) updateTransport(t *http.Transport) error {
	return f.rebuild()
}

func (f *frontend) sortedMiddlewares() []engine.Middleware {
	vals := make([]engine.Middleware, 0, len(f.middlewares))
	for _, m := range f.middlewares {
		vals = append(vals, m)
	}
	sort.Sort(sort.Reverse(&middlewareSorter{ms: vals}))
	return vals
}

func (f *frontend) rebuild() error {
	settings := f.frontend.HTTPSettings()

	// set up forwarder
	fwd, err := forward.New(
		forward.Logger(f.log),
		forward.RoundTripper(f.backend.transport),
		forward.Rewriter(
			&forward.HeaderRewriter{
				Hostname:           settings.Hostname,
				TrustForwardHeader: settings.TrustForwardHeader,
			}))

	// reporter and rtwatcher will be observing metrics
	rp := NewReporter(fwd, f.mux.options.MetricsClient, engine.FrontendKey{Id: f.frontend.Id})
	watcher, err := NewWatcher(rp)
	if err != nil {
		return err
	}

	// Create a load balancer
	rr, err := roundrobin.New(watcher)
	if err != nil {
		return err
	}

	// Rebalancer will readjust load balancer weights based on error ratios
	rb, err := roundrobin.NewRebalancer(rr, roundrobin.RebalancerLogger(f.log))
	if err != nil {
		return err
	}

	// create middlewares sorted by priority and chain them
	middlewares := f.sortedMiddlewares()
	handlers := make([]http.Handler, len(middlewares))
	for i, m := range middlewares {
		var prev http.Handler
		if i == 0 {
			prev = rb
		} else {
			prev = handlers[i-1]
		}
		h, err := m.Middleware.NewHandler(prev)
		if err != nil {
			return err
		}
		handlers[i] = h
	}

	var next http.Handler
	if len(handlers) != 0 {
		next = handlers[len(handlers)-1]
	} else {
		next = rb
	}

	// stream will retry and replay requests, fix encodings
	if settings.FailoverPredicate == "" {
		settings.FailoverPredicate = `IsNetworkError() && RequestMethod() == "GET" && Attempts() < 2`
	}
	str, err := stream.New(next,
		stream.Logger(f.log),
		stream.Retry(settings.FailoverPredicate),
		stream.MaxRequestBodyBytes(settings.Limits.MaxBodyBytes),
		stream.MemRequestBodyBytes(settings.Limits.MaxMemBodyBytes))
	if err != nil {
		return err
	}

	if err := syncServers(f.mux, rb, f.backend, watcher); err != nil {
		return err
	}

	// Add the frontend to the router
	if err := f.mux.router.Handle(f.frontend.Route, str); err != nil {
		return err
	}

	f.lb = rb
	f.handler = str
	f.watcher = watcher
	return nil
}

func (f *frontend) upsertMiddleware(fk engine.FrontendKey, mi engine.Middleware) error {
	f.middlewares[engine.MiddlewareKey{FrontendKey: fk, Id: mi.Id}] = mi
	return f.rebuild()
}

func (f *frontend) deleteMiddleware(mk engine.MiddlewareKey) error {
	delete(f.middlewares, mk)
	return f.rebuild()
}

func (f *frontend) updateBackend(b *backend) error {
	oldb := f.backend
	f.backend = b

	// Switching backends, set the new transport and perform switch
	if b.backend.Id != oldb.backend.Id {
		log.Infof("%v updating backend from %v to %v", f, &oldb, &f.backend)
		oldb.unlinkFrontend(f.key)
		b.linkFrontend(f.key, f)
		return f.rebuild()
	}
	return syncServers(f.mux, f.lb, f.backend, f.watcher)
}

// TODO: implement rollback in case of suboperation failure
func (f *frontend) update(ef engine.Frontend, b *backend) error {
	oldf := f.frontend
	f.frontend = ef

	if err := f.updateBackend(b); err != nil {
		return err
	}

	if oldf.Route != ef.Route {
		log.Infof("%v updating route from %v to %v", oldf.Route, ef.Route)
		if err := f.mux.router.Handle(ef.Route, f.handler); err != nil {
			return err
		}
		if err := f.mux.router.Remove(oldf.Route); err != nil {
			return err
		}
	}

	olds := oldf.HTTPSettings()
	news := ef.HTTPSettings()
	if !olds.Equals(news) {
		if err := f.rebuild(); err != nil {
			return err
		}
	}

	return nil
}

func (f *frontend) remove() error {
	f.backend.unlinkFrontend(f.key)
	return f.mux.router.Remove(f.frontend.Route)
}

type middlewareSorter struct {
	ms []engine.Middleware
}

func (s *middlewareSorter) Len() int {
	return len(s.ms)
}

func (s *middlewareSorter) Swap(i, j int) {
	s.ms[i], s.ms[j] = s.ms[j], s.ms[i]
}

func (s *middlewareSorter) Less(i, j int) bool {
	return s.ms[i].Priority < s.ms[j].Priority
}
<|endoftext|>


<commit_before><commit_msg>Update<commit_after><|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"
	"text/tabwriter"
	"time"

	"github.com/go-kit/kit/log/term"
	metrics "github.com/rcrowley/go-metrics"

	tmtypes "github.com/tendermint/tendermint/types"
	"github.com/tendermint/tmlibs/log"
	"github.com/tendermint/tools/tm-monitor/monitor"
)

var version = "0.3.0"

var logger = log.NewNopLogger()

type statistics struct {
	BlockTimeSample    metrics.Histogram
	TxThroughputSample metrics.Histogram
	BlockLatency       metrics.Histogram
}

func main() {
	var duration, txsRate, connections int
	var verbose bool

	flag.IntVar(&connections, "c", 1, "Connections to keep open per endpoint")
	flag.IntVar(&duration, "T", 10, "Exit after the specified amount of time in seconds")
	flag.IntVar(&txsRate, "r", 1000, "Txs per second to send in a connection")
	flag.BoolVar(&verbose, "v", false, "Verbose output")

	flag.Usage = func() {
		fmt.Println(`Tendermint blockchain benchmarking tool.

Usage:
	tm-bench [-c 1] [-T 10] [-r 1000] [endpoints]

Examples:
	tm-bench localhost:46657`)
		fmt.Println("Flags:")
		flag.PrintDefaults()
	}

	flag.Parse()

	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(1)
	}

	if verbose {
		// Color errors red
		colorFn := func(keyvals ...interface{}) term.FgBgColor {
			for i := 1; i < len(keyvals); i += 2 {
				if _, ok := keyvals[i].(error); ok {
					return term.FgBgColor{Fg: term.White, Bg: term.Red}
				}
			}
			return term.FgBgColor{}
		}
		logger = log.NewTMLoggerWithColorFn(log.NewSyncWriter(os.Stdout), colorFn)
	}

	fmt.Printf("Running %ds test @ %s\n", duration, flag.Arg(0))

	endpoints := strings.Split(flag.Arg(0), ",")

	blockCh := make(chan tmtypes.Header, 100)
	blockLatencyCh := make(chan float64, 100)

	nodes := startNodes(endpoints, blockCh, blockLatencyCh)

	transacters := startTransacters(endpoints, connections, txsRate)

	stats := &statistics{
		BlockTimeSample:    metrics.NewHistogram(metrics.NewUniformSample(1000)),
		TxThroughputSample: metrics.NewHistogram(metrics.NewUniformSample(1000)),
		BlockLatency:       metrics.NewHistogram(metrics.NewUniformSample(1000)),
	}

	lastBlockHeight := int64(-1)

	durationTimer := time.After(time.Duration(duration) * time.Second)
	ticker := time.NewTicker(1 * time.Second)
	var blocks int
	var txs int64
	for {
		select {
		case b := <-blockCh:
			if lastBlockHeight < b.Height {
				blocks++
				txs += b.NumTxs
				lastBlockHeight = b.Height
			}
		case l := <-blockLatencyCh:
			stats.BlockLatency.Update(int64(l))
		case <-ticker.C:
			stats.BlockTimeSample.Update(int64(blocks))
			stats.TxThroughputSample.Update(txs)
			blocks = 0
			txs = 0
		case <-durationTimer:
			for _, t := range transacters {
				t.Stop()
			}

			printStatistics(stats)

			for _, n := range nodes {
				n.Stop()
			}
			return
		}
	}
}

func startNodes(endpoints []string, blockCh chan<- tmtypes.Header, blockLatencyCh chan<- float64) []*monitor.Node {
	nodes := make([]*monitor.Node, len(endpoints))

	for i, e := range endpoints {
		n := monitor.NewNode(e)
		n.SetLogger(logger.With("node", e))
		n.SendBlocksTo(blockCh)
		n.SendBlockLatenciesTo(blockLatencyCh)
		if err := n.Start(); err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		nodes[i] = n
	}

	return nodes
}

func startTransacters(endpoints []string, connections int, txsRate int) []*transacter {
	transacters := make([]*transacter, len(endpoints))

	for i, e := range endpoints {
		t := newTransacter(e, connections, txsRate)
		t.SetLogger(logger)
		if err := t.Start(); err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		transacters[i] = t
	}

	return transacters
}

func printStatistics(stats *statistics) {
	w := tabwriter.NewWriter(os.Stdout, 0, 0, 5, ' ', 0)
	fmt.Fprintln(w, "Stats\tAvg\tStdev\tMax\t")
	fmt.Fprintln(w, fmt.Sprintf("Block latency\t%.2fms\t%.2fms\t%dms\t",
		stats.BlockLatency.Mean()/1000000.0,
		stats.BlockLatency.StdDev()/1000000.0,
		stats.BlockLatency.Max()/1000000))
	fmt.Fprintln(w, fmt.Sprintf("Blocks/sec\t%.3f\t%.3f\t%d\t",
		stats.BlockTimeSample.Mean(),
		stats.BlockTimeSample.StdDev(),
		stats.BlockTimeSample.Max()))
	fmt.Fprintln(w, fmt.Sprintf("Txs/sec\t%.0f\t%.0f\t%d\t",
		stats.TxThroughputSample.Mean(),
		stats.TxThroughputSample.StdDev(),
		stats.TxThroughputSample.Max()))
	w.Flush()
}
<commit_msg> Changed tm-bench output to json (#83)<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/go-kit/kit/log/term"
	metrics "github.com/rcrowley/go-metrics"

	"text/tabwriter"

	tmtypes "github.com/tendermint/tendermint/types"
	"github.com/tendermint/tmlibs/log"
	"github.com/tendermint/tools/tm-monitor/monitor"
)

var version = "0.3.0"

var logger = log.NewNopLogger()

type statistics struct {
	BlockTimeSample    metrics.Histogram
	TxThroughputSample metrics.Histogram
	BlockLatency       metrics.Histogram
}

func main() {
	var duration, txsRate, connections int
	var verbose bool
	var outputFormat string

	flag.IntVar(&connections, "c", 1, "Connections to keep open per endpoint")
	flag.IntVar(&duration, "T", 10, "Exit after the specified amount of time in seconds")
	flag.IntVar(&txsRate, "r", 1000, "Txs per second to send in a connection")
	flag.StringVar(&outputFormat, "output-format", "plain", "Output format: plain or json")
	flag.BoolVar(&verbose, "v", false, "Verbose output")

	flag.Usage = func() {
		fmt.Println(`Tendermint blockchain benchmarking tool.

Usage:
	tm-bench [-c 1] [-T 10] [-r 1000] [endpoints] [-output-format <plain|json>]

Examples:
	tm-bench localhost:46657`)
		fmt.Println("Flags:")
		flag.PrintDefaults()
	}

	flag.Parse()

	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(1)
	}

	if verbose {
		if outputFormat == "json" {
			fmt.Println("Verbose mode not supported with json output.")
			os.Exit(1)
		}
		// Color errors red
		colorFn := func(keyvals ...interface{}) term.FgBgColor {
			for i := 1; i < len(keyvals); i += 2 {
				if _, ok := keyvals[i].(error); ok {
					return term.FgBgColor{Fg: term.White, Bg: term.Red}
				}
			}
			return term.FgBgColor{}
		}
		logger = log.NewTMLoggerWithColorFn(log.NewSyncWriter(os.Stdout), colorFn)

		fmt.Printf("Running %ds test @ %s\n", duration, flag.Arg(0))
	}

	endpoints := strings.Split(flag.Arg(0), ",")

	blockCh := make(chan tmtypes.Header, 100)
	blockLatencyCh := make(chan float64, 100)

	nodes := startNodes(endpoints, blockCh, blockLatencyCh)

	transacters := startTransacters(endpoints, connections, txsRate)

	stats := &statistics{
		BlockTimeSample:    metrics.NewHistogram(metrics.NewUniformSample(1000)),
		TxThroughputSample: metrics.NewHistogram(metrics.NewUniformSample(1000)),
		BlockLatency:       metrics.NewHistogram(metrics.NewUniformSample(1000)),
	}

	lastBlockHeight := int64(-1)

	durationTimer := time.After(time.Duration(duration) * time.Second)
	ticker := time.NewTicker(1 * time.Second)
	var blocks int
	var txs int64
	for {
		select {
		case b := <-blockCh:
			if lastBlockHeight < b.Height {
				blocks++
				txs += b.NumTxs
				lastBlockHeight = b.Height
			}
		case l := <-blockLatencyCh:
			stats.BlockLatency.Update(int64(l))
		case <-ticker.C:
			stats.BlockTimeSample.Update(int64(blocks))
			stats.TxThroughputSample.Update(txs)
			blocks = 0
			txs = 0
		case <-durationTimer:
			for _, t := range transacters {
				t.Stop()
			}

			printStatistics(stats, outputFormat)

			for _, n := range nodes {
				n.Stop()
			}
			return
		}
	}
}

func startNodes(endpoints []string, blockCh chan<- tmtypes.Header, blockLatencyCh chan<- float64) []*monitor.Node {
	nodes := make([]*monitor.Node, len(endpoints))

	for i, e := range endpoints {
		n := monitor.NewNode(e)
		n.SetLogger(logger.With("node", e))
		n.SendBlocksTo(blockCh)
		n.SendBlockLatenciesTo(blockLatencyCh)
		if err := n.Start(); err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		nodes[i] = n
	}

	return nodes
}

func startTransacters(endpoints []string, connections int, txsRate int) []*transacter {
	transacters := make([]*transacter, len(endpoints))

	for i, e := range endpoints {
		t := newTransacter(e, connections, txsRate)
		t.SetLogger(logger)
		if err := t.Start(); err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		transacters[i] = t
	}

	return transacters
}

type Results struct {
	BlockLatencyMean   float64
	BlockLatencyMax    int64
	BlockLatencyStdDev float64
	BlockTimeMean      float64
	BlockTimeMax       int64
	BlockTimeStdDev    float64
	TxThroughputMean   float64
	TxThroughputMax    int64
	TxThroughputStdDev float64
}

func printStatistics(stats *statistics, outputFormat string) {
	if outputFormat == "json" {
		result, _ := json.Marshal(Results{
			stats.BlockLatency.Mean() / 1000000.0,
			stats.BlockLatency.Max() / 1000000.0,
			stats.BlockLatency.StdDev() / 1000000.0,
			stats.BlockTimeSample.Mean(),
			stats.BlockTimeSample.Max(),
			stats.BlockTimeSample.StdDev(),
			stats.TxThroughputSample.Mean(),
			stats.TxThroughputSample.Max(),
			stats.TxThroughputSample.StdDev(),
		})
		fmt.Println(string(result))
	} else {
		w := tabwriter.NewWriter(os.Stdout, 0, 0, 5, ' ', 0)
		fmt.Fprintln(w, "Stats\tAvg\tStdev\tMax\t")
		fmt.Fprintln(w, fmt.Sprintf("Block latency\t%.2fms\t%.2fms\t%dms\t",
			stats.BlockLatency.Mean()/1000000.0,
			stats.BlockLatency.StdDev()/1000000.0,
			stats.BlockLatency.Max()/1000000))
		fmt.Fprintln(w, fmt.Sprintf("Blocks/sec\t%.3f\t%.3f\t%d\t",
			stats.BlockTimeSample.Mean(),
			stats.BlockTimeSample.StdDev(),
			stats.BlockTimeSample.Max()))
		fmt.Fprintln(w, fmt.Sprintf("Txs/sec\t%.0f\t%.0f\t%d\t",
			stats.TxThroughputSample.Mean(),
			stats.TxThroughputSample.StdDev(),
			stats.TxThroughputSample.Max()))
		w.Flush()
	}
}
<|endoftext|>


<commit_before>package confoo

import (
	"fmt"
	"io/ioutil"
	"os"
	"reflect"
	"strings"

	"gopkg.in/yaml.v2"
)

const confVar = "CONFOO_CONFIG_FILE"

func errorPanic(format string, a ...interface{}) {
	m := fmt.Sprintf(format, a...)
	m2 := "CONFOO - " + m
	panic(m2)
}

var confData map[interface{}]interface{}

func init() {
	confFile := os.Getenv("CONFOO_CONFIG_FILE")
	if confFile == "" {
		errorPanic(confVar + " is not set")
	}
	data, err := ioutil.ReadFile(confFile)
	if err != nil {
		errorPanic(err.Error())
		panic(err)
	}

	confData = make(map[interface{}]interface{})
	err = yaml.Unmarshal(data, &confData)
	if err != nil {
		errorPanic("cannot decode yaml data")
	}
}

func Configure(path string, target interface{}) {
	subConf := getSubConf(path, confData)
	if subConf != nil {
		configPath(path, reflect.ValueOf(target), subConf)
	}
}

func getSubConf(path string, conf interface{}) interface{} {
	subConf := conf
	for _, p := range strings.Split(path, ".") {
		m, ok := subConf.(map[interface{}]interface{})
		if !ok {
			errorPanic("%s: path not found", path)
		}

		subConf, ok = m[p]
		if !ok {
			return nil
		}
	}
	return subConf
}

func normalizeKey(s string) string {
	parts := strings.Split(s, "_")
	for i, p := range parts {
		parts[i] = strings.Title(p)
	}
	return strings.Join(parts, "")
}

func configStruct(path string, dest reflect.Value, conf interface{}) {
	if conf == nil {
		return
	}
	confMap, ok := conf.(map[interface{}]interface{})
	if !ok {
		errorPanic("%s: expected map not found", path)
	}
	for k, subConf := range confMap {
		kk, ok := k.(string)
		if !ok {
			//FIXME log event if CONFOO_DEBUG is set
			//errorPanic("%s.%v: map key is not a string", path, k)
			continue
		}
		fieldName := normalizeKey(kk)
		fieldVal := dest.FieldByName(fieldName)
		if fieldVal.Kind() == reflect.Invalid {
			//FIXME log event if CONFOO_DEBUG is set
			//errorPanic("%s.%v: field not present in target struct", path, k)
			continue
		}
		configPath(path+"."+kk, dest.FieldByName(fieldName), subConf)
	}
}

func configPath(path string, dest reflect.Value, conf interface{}) {
	destKind := dest.Kind()
	switch destKind {
	case reflect.Ptr:
		configPath(path, dest.Elem(), conf)
	case reflect.Struct:
		configStruct(path, dest, conf)
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
		reflect.String, reflect.Bool, reflect.Map:
		confValue := reflect.ValueOf(conf)
		confKind := confValue.Kind()
		if confKind != destKind {
			dType := dest.Type()
			cType := confValue.Type()
			errorPanic("%s: target type %v != conf type %v", path, dType, cType)
		}
		dest.Set(confValue)
	default:
		errorPanic("%s: conf type %v not handled", path, dest.Type())
	}
}
<commit_msg>removed useless panic()<commit_after>package confoo

import (
	"fmt"
	"io/ioutil"
	"os"
	"reflect"
	"strings"

	"gopkg.in/yaml.v2"
)

const confVar = "CONFOO_CONFIG_FILE"

func errorPanic(format string, a ...interface{}) {
	m := fmt.Sprintf(format, a...)
	m2 := "CONFOO - " + m
	panic(m2)
}

var confData map[interface{}]interface{}

func init() {
	confFile := os.Getenv("CONFOO_CONFIG_FILE")
	if confFile == "" {
		errorPanic(confVar + " is not set")
	}
	data, err := ioutil.ReadFile(confFile)
	if err != nil {
		errorPanic(err.Error())
	}

	confData = make(map[interface{}]interface{})
	err = yaml.Unmarshal(data, &confData)
	if err != nil {
		errorPanic("cannot decode yaml data")
	}
}

func Configure(path string, target interface{}) {
	subConf := getSubConf(path, confData)
	if subConf != nil {
		configPath(path, reflect.ValueOf(target), subConf)
	}
}

func getSubConf(path string, conf interface{}) interface{} {
	subConf := conf
	for _, p := range strings.Split(path, ".") {
		m, ok := subConf.(map[interface{}]interface{})
		if !ok {
			errorPanic("%s: path not found", path)
		}

		subConf, ok = m[p]
		if !ok {
			return nil
		}
	}
	return subConf
}

func normalizeKey(s string) string {
	parts := strings.Split(s, "_")
	for i, p := range parts {
		parts[i] = strings.Title(p)
	}
	return strings.Join(parts, "")
}

func configStruct(path string, dest reflect.Value, conf interface{}) {
	if conf == nil {
		return
	}
	confMap, ok := conf.(map[interface{}]interface{})
	if !ok {
		errorPanic("%s: expected map not found", path)
	}
	for k, subConf := range confMap {
		kk, ok := k.(string)
		if !ok {
			//FIXME log event if CONFOO_DEBUG is set
			//errorPanic("%s.%v: map key is not a string", path, k)
			continue
		}
		fieldName := normalizeKey(kk)
		fieldVal := dest.FieldByName(fieldName)
		if fieldVal.Kind() == reflect.Invalid {
			//FIXME log event if CONFOO_DEBUG is set
			//errorPanic("%s.%v: field not present in target struct", path, k)
			continue
		}
		configPath(path+"."+kk, dest.FieldByName(fieldName), subConf)
	}
}

func configPath(path string, dest reflect.Value, conf interface{}) {
	destKind := dest.Kind()
	switch destKind {
	case reflect.Ptr:
		configPath(path, dest.Elem(), conf)
	case reflect.Struct:
		configStruct(path, dest, conf)
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
		reflect.String, reflect.Bool, reflect.Map:
		confValue := reflect.ValueOf(conf)
		confKind := confValue.Kind()
		if confKind != destKind {
			dType := dest.Type()
			cType := confValue.Type()
			errorPanic("%s: target type %v != conf type %v", path, dType, cType)
		}
		dest.Set(confValue)
	default:
		errorPanic("%s: conf type %v not handled", path, dest.Type())
	}
}
<|endoftext|>