<commit_before>package infrastructure

import (
  "fmt"
  "time"
  httpurl "net/url"
  "errors"
  "encoding/json"
  "net/http"
  "io/ioutil"
  "bytes"
)

type Client struct{
  url *httpurl.URL
}

func New(url string) (*Client, error) {
  u, err := httpurl.Parse(url)
  if err != nil {
    return nil, err
  }
  return &Client{u}, nil
}

func NewFromURL(url httpurl.URL) *Client {
  return &Client{&url}
}

type TimeInterval struct {
  From time.Time
  To time.Time
}

func (t *TimeInterval) Check() error {
  if t.From.After(t.To) {
    return errors.New("From must be before To.")
  }
  return nil
}

func graphiteDateFormat(t time.Time) string {
  return fmt.Sprintf("%02d:%02d_%d%02d%02d", t.Hour(), t.Minute(), t.Year(), t.Month(), t.Day())
}

type FloatDatapoint struct {
  Time time.Time
  Value *float64
}

type IntDatapoint struct {
  Time time.Time
  Value *int64
}

type Datapoints struct {
  // Previous error to make API nicer.
  err error
  points [][]interface{}
}

func (d Datapoints) AsInts() ([]IntDatapoint, error) {
  if d.err != nil {
    return nil, d.err
  }

  points := make([]IntDatapoint, 0, len(d.points))
  for _, point := range d.points {
    jsonUnixTime, ok := point[1].(json.Number)
    if !ok {
      return nil, errors.New("Unix timestamp not number.")
    }
    unixTime, err := jsonUnixTime.Int64()
    if err != nil {
      return nil, errors.New("Unix time not proper number.")
    }

    var value *int64
    if point[0] != nil {
      jsonValue, ok := point[0].(json.Number)
      if !ok {
        return nil, errors.New("Value not a number.")
      }
      value = new(int64)
      *value, err = jsonValue.Int64()
      if err != nil {
        return nil, errors.New("Value not proper number.")
      }
    }
    points = append(points, IntDatapoint{time.Unix(unixTime, 0), value})
  }

  return points, nil
}

func (d Datapoints) AsFloats() ([]FloatDatapoint, error) {
  if d.err != nil {
    return nil, d.err
  }

  points := make([]FloatDatapoint, 0, len(d.points))
  for _, point := range d.points {
    jsonUnixTime, ok := point[1].(json.Number)
    if !ok {
      return nil, errors.New("Unix timestamp not number.")
    }
    unixTime, err := jsonUnixTime.Int64()
    if err != nil {
      return nil, errors.New("Unix time not proper number.")
    }

    var value *float64
    if point[0] != nil {
      jsonValue, ok := point[0].(json.Number)
      if !ok {
        return nil, errors.New("Value not a number.")
      }
      value = new(float64)
      *value, err = jsonValue.Float64()
      if err != nil {
        return nil, errors.New("Value not proper number.")
      }
    }
    points = append(points, FloatDatapoint{time.Unix(unixTime, 0), value})
  }

  return points, nil
}

func constructQueryPart(q string) httpurl.Values {
  query := make(httpurl.Values)
  query.Add("target", q)
  query.Add("format", "json")
  return query
}

// Helper method to make it easier to create an interface for Client.
func (g *Client) QueryInts(q string, interval TimeInterval) ([]IntDatapoint, error) {
  return g.Query(q, interval).AsInts()
}

// Helper method to make it easier to create an interface for Client.
func (g *Client) QueryFloats(q string, interval TimeInterval) ([]FloatDatapoint, error) {
  return g.Query(q, interval).AsFloats()
}

// Helper method to make it easier to create an interface for Client.
func (g *Client) QueryIntsSince(q string, ago time.Duration) ([]IntDatapoint, error) {
  return g.QuerySince(q, ago).AsInts()
}

// Helper method to make it easier to create an interface for Client.
func (g *Client) QueryFloatsSince(q string, ago time.Duration) ([]FloatDatapoint, error) {
  return g.QuerySince(q, ago).AsFloats()
}

// Fetches a Graphite result. Deferring identifying whether the result are ints
// of floats to later. Useful in clients that executes adhoc queries.
func (g *Client) Query(q string, interval TimeInterval) Datapoints {
  if err := interval.Check(); err != nil {
    return Datapoints{err, nil}
  }

  // Cloning to be able to modify.
  url := g.url

  queryPart := constructQueryPart(q)
  queryPart.Add("from", graphiteDateFormat(interval.From))
  queryPart.Add("until", graphiteDateFormat(interval.To))
  url.RawQuery = queryPart.Encode()

  resp, err := http.Get(url.String())
  if err != nil {
    return Datapoints{err, nil}
  }
  defer resp.Body.Close()

  body, err := ioutil.ReadAll(resp.Body)
  if err != nil {
    return Datapoints{err, nil}
  }

  return parseGraphiteResponse(body)
}

func (g *Client) QuerySince(q string, ago time.Duration) Datapoints {
  if ago.Nanoseconds() > 0 {
    return Datapoints{errors.New("Duration is expected to be positive."), nil}
  }

  // Cloning to be able to modify.
  url := g.url

  queryPart := constructQueryPart(q)
  queryPart.Add("from", fmt.Sprintf("%dminutes", ago.Minutes()))
  url.RawQuery = queryPart.Encode()

  resp, err := http.Get(url.String())
  if err != nil {
    return Datapoints{err, nil}
  }
  defer resp.Body.Close()

  body, err := ioutil.ReadAll(resp.Body)
  if err != nil {
    return Datapoints{err, nil}
  }

  return parseGraphiteResponse(body)
}

func parseGraphiteResponse(body []byte) Datapoints {
  var dps Datapoints
  var res []target

  decoder := json.NewDecoder(bytes.NewBuffer(body))

  // Important to distinguish between ints and floats.
  decoder.UseNumber()

  dps.err = decoder.Decode(&res)
  if len(res) == 0 {
    dps.err = errors.New("Unexpected Graphite response. No targets were returned.")
  }
  if len(res) > 1 {
    dps.err = errors.New("Unexpected Graphite response. More than one target was returned.")
  }
  if dps.err != nil {
    return dps
  }

  // TODO: Check the query is the same as the target name.

  dps.points = res[0].Datapoints
  return dps
}

type queryResult []target

type target struct {
  Target string `json:"target"`

  // Datapoints are either
  //
  //     [[FLOAT, INT], ..., [FLOAT, INT]]  (type []intDatapoint).
  //
  // or
  //
  //     [[FLOAT, FLOAT], ..., [FLOAT, FLOAT]] (type []floatDatapoint).
  Datapoints [][]interface{} `json:"datapoints"`
}
<commit_msg>Make it possible to inject http.Client<commit_after>package infrastructure

import (
  "fmt"
  "time"
  httpurl "net/url"
  "errors"
  "encoding/json"
  "net/http"
  "io/ioutil"
  "bytes"
)

type Client struct{
  url *httpurl.URL
  Client *http.Client
}

func New(url string) (*Client, error) {
  u, err := httpurl.Parse(url)
  if err != nil {
    return nil, err
  }
  return &Client{
    url: u,
    Client: &http.Client{},
  }, nil
}

func NewFromURL(url httpurl.URL) *Client {
  return &Client{&url, &http.Client{}}
}

type TimeInterval struct {
  From time.Time
  To time.Time
}

func (t *TimeInterval) Check() error {
  if t.From.After(t.To) {
    return errors.New("From must be before To.")
  }
  return nil
}

func graphiteDateFormat(t time.Time) string {
  return fmt.Sprintf("%02d:%02d_%d%02d%02d", t.Hour(), t.Minute(), t.Year(), t.Month(), t.Day())
}

type FloatDatapoint struct {
  Time time.Time
  Value *float64
}

type IntDatapoint struct {
  Time time.Time
  Value *int64
}

type Datapoints struct {
  // Previous error to make API nicer.
  err error
  points [][]interface{}
}

func (d Datapoints) AsInts() ([]IntDatapoint, error) {
  if d.err != nil {
    return nil, d.err
  }

  points := make([]IntDatapoint, 0, len(d.points))
  for _, point := range d.points {
    jsonUnixTime, ok := point[1].(json.Number)
    if !ok {
      return nil, errors.New("Unix timestamp not number.")
    }
    unixTime, err := jsonUnixTime.Int64()
    if err != nil {
      return nil, errors.New("Unix time not proper number.")
    }

    var value *int64
    if point[0] != nil {
      jsonValue, ok := point[0].(json.Number)
      if !ok {
        return nil, errors.New("Value not a number.")
      }
      value = new(int64)
      *value, err = jsonValue.Int64()
      if err != nil {
        return nil, errors.New("Value not proper number.")
      }
    }
    points = append(points, IntDatapoint{time.Unix(unixTime, 0), value})
  }

  return points, nil
}

func (d Datapoints) AsFloats() ([]FloatDatapoint, error) {
  if d.err != nil {
    return nil, d.err
  }

  points := make([]FloatDatapoint, 0, len(d.points))
  for _, point := range d.points {
    jsonUnixTime, ok := point[1].(json.Number)
    if !ok {
      return nil, errors.New("Unix timestamp not number.")
    }
    unixTime, err := jsonUnixTime.Int64()
    if err != nil {
      return nil, errors.New("Unix time not proper number.")
    }

    var value *float64
    if point[0] != nil {
      jsonValue, ok := point[0].(json.Number)
      if !ok {
        return nil, errors.New("Value not a number.")
      }
      value = new(float64)
      *value, err = jsonValue.Float64()
      if err != nil {
        return nil, errors.New("Value not proper number.")
      }
    }
    points = append(points, FloatDatapoint{time.Unix(unixTime, 0), value})
  }

  return points, nil
}

func constructQueryPart(q string) httpurl.Values {
  query := make(httpurl.Values)
  query.Add("target", q)
  query.Add("format", "json")
  return query
}

// Helper method to make it easier to create an interface for Client.
func (g *Client) QueryInts(q string, interval TimeInterval) ([]IntDatapoint, error) {
  return g.Query(q, interval).AsInts()
}

// Helper method to make it easier to create an interface for Client.
func (g *Client) QueryFloats(q string, interval TimeInterval) ([]FloatDatapoint, error) {
  return g.Query(q, interval).AsFloats()
}

// Helper method to make it easier to create an interface for Client.
func (g *Client) QueryIntsSince(q string, ago time.Duration) ([]IntDatapoint, error) {
  return g.QuerySince(q, ago).AsInts()
}

// Helper method to make it easier to create an interface for Client.
func (g *Client) QueryFloatsSince(q string, ago time.Duration) ([]FloatDatapoint, error) {
  return g.QuerySince(q, ago).AsFloats()
}

// Fetches a Graphite result. Deferring identifying whether the result are ints
// of floats to later. Useful in clients that executes adhoc queries.
func (g *Client) Query(q string, interval TimeInterval) Datapoints {
  if err := interval.Check(); err != nil {
    return Datapoints{err, nil}
  }

  // Cloning to be able to modify.
  url := g.url

  queryPart := constructQueryPart(q)
  queryPart.Add("from", graphiteDateFormat(interval.From))
  queryPart.Add("until", graphiteDateFormat(interval.To))
  url.RawQuery = queryPart.Encode()

  resp, err := g.Client.Get(url.String())
  if err != nil {
    return Datapoints{err, nil}
  }
  defer resp.Body.Close()

  body, err := ioutil.ReadAll(resp.Body)
  if err != nil {
    return Datapoints{err, nil}
  }

  return parseGraphiteResponse(body)
}

func (g *Client) QuerySince(q string, ago time.Duration) Datapoints {
  if ago.Nanoseconds() > 0 {
    return Datapoints{errors.New("Duration is expected to be positive."), nil}
  }

  // Cloning to be able to modify.
  url := g.url

  queryPart := constructQueryPart(q)
  queryPart.Add("from", fmt.Sprintf("%dminutes", ago.Minutes()))
  url.RawQuery = queryPart.Encode()

  resp, err := http.Get(url.String())
  if err != nil {
    return Datapoints{err, nil}
  }
  defer resp.Body.Close()

  body, err := ioutil.ReadAll(resp.Body)
  if err != nil {
    return Datapoints{err, nil}
  }

  return parseGraphiteResponse(body)
}

func parseGraphiteResponse(body []byte) Datapoints {
  var dps Datapoints
  var res []target

  decoder := json.NewDecoder(bytes.NewBuffer(body))

  // Important to distinguish between ints and floats.
  decoder.UseNumber()

  dps.err = decoder.Decode(&res)
  if len(res) == 0 {
    dps.err = errors.New("Unexpected Graphite response. No targets were returned.")
  }
  if len(res) > 1 {
    dps.err = errors.New("Unexpected Graphite response. More than one target was returned.")
  }
  if dps.err != nil {
    return dps
  }

  // TODO: Check the query is the same as the target name.

  dps.points = res[0].Datapoints
  return dps
}

type queryResult []target

type target struct {
  Target string `json:"target"`

  // Datapoints are either
  //
  //     [[FLOAT, INT], ..., [FLOAT, INT]]  (type []intDatapoint).
  //
  // or
  //
  //     [[FLOAT, FLOAT], ..., [FLOAT, FLOAT]] (type []floatDatapoint).
  Datapoints [][]interface{} `json:"datapoints"`
}
<|endoftext|>


<commit_before>package main

/*
	Wgetpipe takes a list of fully-qualified URLs over STDIN and Gets them,
	outputting the code, url and elapsed fetch time.

	scans stdin in a goro, spawning up to MAX getters at a time, which stream their
	responses over a channel back to main which formats the output
*/

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/cheggaaa/pb"
	"github.com/fatih/color"
	"github.com/viki-org/dnscache"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

var (
	MAX        int           // maximum number of outstanding HTTP get requests allowed
	SleepTime  time.Duration // Duration to sleep between GETter spawns
	ErrOnly    bool          // Quiet unless 0 == Code >= 400
	NoColor    bool          // Disable colorizing
	NoDnsCache bool          // Disable DNS caching
	Summary    bool          // Output final stats
	useBar     bool          // Use progress bar
	debug      bool          // Enable debugging

	OutFormat int         = log.Ldate | log.Ltime | log.Lshortfile
	DebugOut  *log.Logger = log.New(ioutil.Discard, "[DEBUG] ", OutFormat)
)

type urlCode struct {
	Url  string
	Code int
	Dur  time.Duration
	Err  error
}

func init() {
	flag.IntVar(&MAX, "max", 5, "Maximium in-flight GET requests at a time")
	flag.BoolVar(&ErrOnly, "errorsonly", false, "Only output errors (HTTP Codes >= 400)")
	flag.BoolVar(&NoColor, "nocolor", false, "Don't colorize the output")
	flag.BoolVar(&Summary, "stats", false, "Output stats at the end")
	flag.DurationVar(&SleepTime, "sleep", 0, "Amount of time to sleep between spawning a GETter (e.g. 1ms, 10s)")
	flag.BoolVar(&debug, "debug", false, "Enable debug output")
	flag.BoolVar(&NoDnsCache, "nodnscache", false, "Disable DNS caching")
	flag.BoolVar(&useBar, "bar", false, "Use progress bar instead of printing lines, can still use -stats")
	flag.Parse()

	// Handle boring people
	if NoColor {
		color.NoColor = true
	}

	// Handle debug
	if debug {
		DebugOut = log.New(os.Stderr, "[DEBUG] ", OutFormat)
	}

	// Sets the default http client to use dnscache, because duh
	if NoDnsCache == false {
		res := dnscache.New(1 * time.Hour)
		http.DefaultClient.Transport = &http.Transport{
			MaxIdleConnsPerHost: 64,
			Dial: func(network string, address string) (net.Conn, error) {
				separator := strings.LastIndex(address, ":")
				ip, _ := res.FetchOneString(address[:separator])
				return net.Dial("tcp", ip+address[separator:])
			},
		}
	}
}

func main() {

	var bar *pb.ProgressBar
	getChan := make(chan string)       // Channel to stream URLs to get
	rChan := make(chan urlCode)        // Channel to stream responses from the Gets
	doneChan := make(chan bool)        // Channel to signal a getter is done
	sigChan := make(chan os.Signal, 1) // Channel to stream signals
	abortChan := make(chan bool)       // Channel to tell the getters to abort
	count := 0
	error4s := 0
	error5s := 0
	errors := 0

	// Set up the progress bar
	if useBar {
		bar = pb.New(0)
	}

	// Stream the signals we care about
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

	// Signal handler
	go func() {
		<-sigChan
		DebugOut.Println("Signal seen, sending abort!")

		close(abortChan)
	}()

	// Spawn off the getters
	for g := 0; g < MAX; g++ {
		go getter(getChan, rChan, doneChan, abortChan)
	}

	// Block until all the getters are done, and then close rChan
	go func() {
		defer close(rChan)

		for c := 0; c < MAX; c++ {
			<-doneChan
			DebugOut.Printf("Done %d/%d\n", c+1, MAX)
		}
	}()

	// spawn off the scanner
	start := time.Now()
	go scanStdIn(getChan, abortChan, bar)

	if useBar {
		bar.Start()
	}
	// Collate the results
	for i := range rChan {
		count++

		if useBar {
			bar.Increment()
		}
		if i.Code == 0 {
			errors++
			if useBar {
				continue
			}
			color.Red("%d %s %s (%s)\n", i.Code, i.Url, i.Dur.String(), i.Err)
		} else if i.Code < 400 {
			if ErrOnly || useBar {
				// skip
				continue
			}
			color.Green("%d %s %s\n", i.Code, i.Url, i.Dur.String())
		} else if i.Code < 500 {
			error4s++
			if useBar {
				continue
			}
			color.Yellow("%d %s %s\n", i.Code, i.Url, i.Dur.String())
		} else {
			error5s++
			if useBar {
				continue
			}
			color.Red("%d %s %s\n", i.Code, i.Url, i.Dur.String())
		}
	}

	if useBar {
		bar.Finish()
	}
	elapsed := time.Since(start)

	if Summary {
		e := color.RedString("%d", errors)
		e4 := color.YellowString("%d", error4s)
		e5 := color.RedString("%d", error5s)
		fmt.Printf("\n\nGETs: %d\nErrors: %s\n500 Errors: %s\n400 Errors: %s\nElapsed Time: %s\n", count, e, e5, e4, elapsed.String())
	}
}

// scanStdIn takes a channel to pass inputted strings to,
// and does so until EOF, whereafter it closes the channel
func scanStdIn(getChan chan string, abortChan chan bool, bar *pb.ProgressBar) {
	defer close(getChan)

	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		select {
		case <-abortChan:
			DebugOut.Println("scanner abort seen!")
			return
		default:
		}
		DebugOut.Println("scanner sending...")

		getChan <- scanner.Text()
		if bar != nil {
			bar.Total += 1
		}

	}
	// POST: we've seen EOF
	DebugOut.Println("EOF seen")

}

// getter takes a receive channel, send channel, and done channel,
// running HTTP GETs for anything in the receive channel, returning
// formatted responses to the send channel, and signalling completion
// via the done channel
func getter(getChan chan string, rChan chan urlCode, doneChan chan bool, abortChan chan bool) {
	defer func() { doneChan <- true }()

	for {
		select {
		case <-abortChan:
			DebugOut.Println("getter abort seen!")
			return
		case url := <-getChan:
			if url == "" {
				// We assume an empty request is a closer
				// as that simplifies our for{select{}} loop
				// considerably
				DebugOut.Println("getter empty request seen!")
				return
			}
			DebugOut.Printf("getter getting %s\n", url)

			s := time.Now()
			response, err := http.Get(url)
			d := time.Since(s)
			if err != nil {
				// We assume code 0 to be a non-HTTP error
				rChan <- urlCode{url, 0, d, err}
			} else {
				response.Body.Close() // else leak
				rChan <- urlCode{url, response.StatusCode, d, nil}
			}

			if SleepTime > 0 {
				// Zzzzzzz
				time.Sleep(SleepTime)
			}
		}
	}
}
<commit_msg>Adds --guess to seed the progress bar start value<commit_after>package main

/*
	Wgetpipe takes a list of fully-qualified URLs over STDIN and Gets them,
	outputting the code, url and elapsed fetch time.

	scans stdin in a goro, spawning up to MAX getters at a time, which stream their
	responses over a channel back to main which formats the output
*/

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/cheggaaa/pb"
	"github.com/fatih/color"
	"github.com/viki-org/dnscache"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

var (
	MAX        int           // maximum number of outstanding HTTP get requests allowed
	SleepTime  time.Duration // Duration to sleep between GETter spawns
	ErrOnly    bool          // Quiet unless 0 == Code >= 400
	NoColor    bool          // Disable colorizing
	NoDnsCache bool          // Disable DNS caching
	Summary    bool          // Output final stats
	useBar     bool          // Use progress bar
	totalGuess int           // Guesstimate of number of GETs (useful with -bar)
	debug      bool          // Enable debugging

	OutFormat int         = log.Ldate | log.Ltime | log.Lshortfile
	DebugOut  *log.Logger = log.New(ioutil.Discard, "[DEBUG] ", OutFormat)
)

type urlCode struct {
	Url  string
	Code int
	Dur  time.Duration
	Err  error
}

func init() {
	flag.IntVar(&MAX, "max", 5, "Maximium in-flight GET requests at a time")
	flag.BoolVar(&ErrOnly, "errorsonly", false, "Only output errors (HTTP Codes >= 400)")
	flag.BoolVar(&NoColor, "nocolor", false, "Don't colorize the output")
	flag.BoolVar(&Summary, "stats", false, "Output stats at the end")
	flag.DurationVar(&SleepTime, "sleep", 0, "Amount of time to sleep between spawning a GETter (e.g. 1ms, 10s)")
	flag.BoolVar(&debug, "debug", false, "Enable debug output")
	flag.BoolVar(&NoDnsCache, "nodnscache", false, "Disable DNS caching")
	flag.BoolVar(&useBar, "bar", false, "Use progress bar instead of printing lines, can still use -stats")
	flag.IntVar(&totalGuess, "guess", 0, "Rough guess of how many GETs will be coming for -bar to start at. It will adjust")
	flag.Parse()

	// Handle boring people
	if NoColor {
		color.NoColor = true
	}

	// Handle debug
	if debug {
		DebugOut = log.New(os.Stderr, "[DEBUG] ", OutFormat)
	}

	// Sets the default http client to use dnscache, because duh
	if NoDnsCache == false {
		res := dnscache.New(1 * time.Hour)
		http.DefaultClient.Transport = &http.Transport{
			MaxIdleConnsPerHost: 64,
			Dial: func(network string, address string) (net.Conn, error) {
				separator := strings.LastIndex(address, ":")
				ip, _ := res.FetchOneString(address[:separator])
				return net.Dial("tcp", ip+address[separator:])
			},
		}
	}
}

func main() {

	var bar *pb.ProgressBar
	getChan := make(chan string, MAX*10) // Channel to stream URLs to get
	rChan := make(chan urlCode)          // Channel to stream responses from the Gets
	doneChan := make(chan bool)          // Channel to signal a getter is done
	sigChan := make(chan os.Signal, 1)   // Channel to stream signals
	abortChan := make(chan bool)         // Channel to tell the getters to abort
	count := 0
	error4s := 0
	error5s := 0
	errors := 0

	// Set up the progress bar
	if useBar {
		bar = pb.New(totalGuess)
	}

	// Stream the signals we care about
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

	// Signal handler
	go func() {
		<-sigChan
		DebugOut.Println("Signal seen, sending abort!")

		close(abortChan)
	}()

	// Spawn off the getters
	for g := 0; g < MAX; g++ {
		go getter(getChan, rChan, doneChan, abortChan)
	}

	// Block until all the getters are done, and then close rChan
	go func() {
		defer close(rChan)

		for c := 0; c < MAX; c++ {
			<-doneChan
			DebugOut.Printf("Done %d/%d\n", c+1, MAX)
		}
	}()

	// spawn off the scanner
	start := time.Now()
	go scanStdIn(getChan, abortChan, bar)

	if useBar {
		bar.Start()
	}
	// Collate the results
	for i := range rChan {
		count++

		if useBar {
			bar.Increment()
		}
		if i.Code == 0 {
			errors++
			if useBar {
				continue
			}
			color.Red("%d %s %s (%s)\n", i.Code, i.Url, i.Dur.String(), i.Err)
		} else if i.Code < 400 {
			if ErrOnly || useBar {
				// skip
				continue
			}
			color.Green("%d %s %s\n", i.Code, i.Url, i.Dur.String())
		} else if i.Code < 500 {
			error4s++
			if useBar {
				continue
			}
			color.Yellow("%d %s %s\n", i.Code, i.Url, i.Dur.String())
		} else {
			error5s++
			if useBar {
				continue
			}
			color.Red("%d %s %s\n", i.Code, i.Url, i.Dur.String())
		}
	}

	if useBar {
		bar.Finish()
	}
	elapsed := time.Since(start)

	if Summary {
		e := color.RedString("%d", errors)
		e4 := color.YellowString("%d", error4s)
		e5 := color.RedString("%d", error5s)
		fmt.Printf("\n\nGETs: %d\nErrors: %s\n500 Errors: %s\n400 Errors: %s\nElapsed Time: %s\n", count, e, e5, e4, elapsed.String())
	}
}

// scanStdIn takes a channel to pass inputted strings to,
// and does so until EOF, whereafter it closes the channel
func scanStdIn(getChan chan string, abortChan chan bool, bar *pb.ProgressBar) {
	defer close(getChan)

	scanner := bufio.NewScanner(os.Stdin)
	count := int64(0)
	for scanner.Scan() {
		select {
		case <-abortChan:
			DebugOut.Println("scanner abort seen!")
			return
		default:
		}
		DebugOut.Println("scanner sending...")

		getChan <- scanner.Text()
		if bar != nil {
			count++
			if bar.Total < count {
				bar.Total += 1
			}
		}

	}
	// POST: we've seen EOF
	DebugOut.Println("EOF seen")

}

// getter takes a receive channel, send channel, and done channel,
// running HTTP GETs for anything in the receive channel, returning
// formatted responses to the send channel, and signalling completion
// via the done channel
func getter(getChan chan string, rChan chan urlCode, doneChan chan bool, abortChan chan bool) {
	defer func() { doneChan <- true }()

	for {
		select {
		case <-abortChan:
			DebugOut.Println("getter abort seen!")
			return
		case url := <-getChan:
			if url == "" {
				// We assume an empty request is a closer
				// as that simplifies our for{select{}} loop
				// considerably
				DebugOut.Println("getter empty request seen!")
				return
			}
			DebugOut.Printf("getter getting %s\n", url)

			s := time.Now()
			response, err := http.Get(url)
			d := time.Since(s)
			if err != nil {
				// We assume code 0 to be a non-HTTP error
				rChan <- urlCode{url, 0, d, err}
			} else {
				response.Body.Close() // else leak
				rChan <- urlCode{url, response.StatusCode, d, nil}
			}

			if SleepTime > 0 {
				// Zzzzzzz
				time.Sleep(SleepTime)
			}
		}
	}
}
<|endoftext|>


<commit_before>package wkb

import (
	"encoding/binary"
	"fmt"
	"io"
)

type wkbReader func(io.Reader, binary.ByteOrder) (Geom, error)

var wkbReaders = map[uint32]wkbReader{
	wkbPoint:        pointReader,
	wkbPointZ:       pointZReader,
	wkbPointM:       pointMReader,
	wkbPointZM:      pointZMReader,
	wkbLineString:   lineStringReader,
	wkbLineStringZ:  lineStringZReader,
	wkbLineStringM:  lineStringMReader,
	wkbLineStringZM: lineStringZMReader,
	wkbPolygon:      polygonReader,
	wkbPolygonZ:     polygonZReader,
	wkbPolygonM:     polygonMReader,
	wkbPolygonZM:    polygonZMReader,
}

func readLinearRing(r io.Reader, byteOrder binary.ByteOrder) ([]Point, error) {
	var numPoints uint32
	if err := binary.Read(r, byteOrder, &numPoints); err != nil {
		return nil, err
	}
	points := make([]Point, numPoints)
	if err := binary.Read(r, byteOrder, &points); err != nil {
		return nil, err
	}
	return points, nil
}

func readLinearRingZ(r io.Reader, byteOrder binary.ByteOrder) ([]PointZ, error) {
	var numPoints uint32
	if err := binary.Read(r, byteOrder, &numPoints); err != nil {
		return nil, err
	}
	pointZs := make([]PointZ, numPoints)
	if err := binary.Read(r, byteOrder, &pointZs); err != nil {
		return nil, err
	}
	return pointZs, nil
}

func readLinearRingM(r io.Reader, byteOrder binary.ByteOrder) ([]PointM, error) {
	var numPoints uint32
	if err := binary.Read(r, byteOrder, &numPoints); err != nil {
		return nil, err
	}
	pointMs := make([]PointM, numPoints)
	if err := binary.Read(r, byteOrder, &pointMs); err != nil {
		return nil, err
	}
	return pointMs, nil
}

func readLinearRingZM(r io.Reader, byteOrder binary.ByteOrder) ([]PointZM, error) {
	var numPoints uint32
	if err := binary.Read(r, byteOrder, &numPoints); err != nil {
		return nil, err
	}
	pointZMs := make([]PointZM, numPoints)
	if err := binary.Read(r, byteOrder, &pointZMs); err != nil {
		return nil, err
	}
	return pointZMs, nil
}

func pointReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	point := Point{}
	if err := binary.Read(r, byteOrder, &point); err != nil {
		return nil, err
	}
	return point, nil
}

func pointZReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	pointZ := PointZ{}
	if err := binary.Read(r, byteOrder, &pointZ); err != nil {
		return nil, err
	}
	return pointZ, nil
}

func pointMReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	pointM := PointM{}
	if err := binary.Read(r, byteOrder, &pointM); err != nil {
		return nil, err
	}
	return pointM, nil
}

func pointZMReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	pointZM := PointZM{}
	if err := binary.Read(r, byteOrder, &pointZM); err != nil {
		return nil, err
	}
	return pointZM, nil
}

func lineStringReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	points, err := readLinearRing(r, byteOrder)
	if err != nil {
		return nil, err
	}
	return LineString{points}, nil
}

func lineStringZReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	pointZs, err := readLinearRingZ(r, byteOrder)
	if err != nil {
		return nil, err
	}
	return LineStringZ{pointZs}, nil
}

func lineStringMReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	pointMs, err := readLinearRingM(r, byteOrder)
	if err != nil {
		return nil, err
	}
	return LineStringM{pointMs}, nil
}

func lineStringZMReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	pointZMs, err := readLinearRingZM(r, byteOrder)
	if err != nil {
		return nil, err
	}
	return LineStringZM{pointZMs}, nil
}

func polygonReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	var numRings uint32
	if err := binary.Read(r, byteOrder, &numRings); err != nil {
		return nil, err
	}
	rings := make([]LinearRing, numRings)
	for i := uint32(0); i < numRings; i++ {
		points, err := readLinearRing(r, byteOrder)
		if err != nil {
			return nil, err
		}
		rings[i] = points
	}
	return Polygon{rings}, nil
}

func polygonZReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	var numRings uint32
	if err := binary.Read(r, byteOrder, &numRings); err != nil {
		return nil, err
	}
	rings := make([]LinearRingZ, numRings)
	for i := uint32(0); i < numRings; i++ {
		pointZs, err := readLinearRingZ(r, byteOrder)
		if err != nil {
			return nil, err
		}
		rings[i] = pointZs
	}
	return PolygonZ{rings}, nil
}

func polygonMReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	var numRings uint32
	if err := binary.Read(r, byteOrder, &numRings); err != nil {
		return nil, err
	}
	rings := make([]LinearRingM, numRings)
	for i := uint32(0); i < numRings; i++ {
		pointMs, err := readLinearRingM(r, byteOrder)
		if err != nil {
			return nil, err
		}
		rings[i] = pointMs
	}
	return PolygonM{rings}, nil
}

func polygonZMReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	var numRings uint32
	if err := binary.Read(r, byteOrder, &numRings); err != nil {
		return nil, err
	}
	rings := make([]LinearRingZM, numRings)
	for i := uint32(0); i < numRings; i++ {
		pointZMs, err := readLinearRingZM(r, byteOrder)
		if err != nil {
			return nil, err
		}
		rings[i] = pointZMs
	}
	return PolygonZM{rings}, nil
}

func Read(r io.Reader) (Geom, error) {

	var wkbByteOrder uint8
	if err := binary.Read(r, binary.LittleEndian, &wkbByteOrder); err != nil {
		return nil, err
	}
	var byteOrder binary.ByteOrder
	switch wkbByteOrder {
	case wkbXDR:
		byteOrder = binary.BigEndian
	case wkbNDR:
		byteOrder = binary.LittleEndian
	default:
		return nil, fmt.Errorf("invalid byte order %u", wkbByteOrder)
	}

	var wkbGeometryType uint32
	if err := binary.Read(r, byteOrder, &wkbGeometryType); err != nil {
		return nil, err
	}

	if reader, ok := wkbReaders[wkbGeometryType]; ok {
		return reader(r, byteOrder)
	} else {
		return nil, fmt.Errorf("unsupported geometry type %u", wkbGeometryType)
	}

}
<commit_msg>Improve coding style<commit_after>package wkb

import (
	"encoding/binary"
	"fmt"
	"io"
)

type wkbReader func(io.Reader, binary.ByteOrder) (Geom, error)

var wkbReaders = map[uint32]wkbReader{
	wkbPoint:        pointReader,
	wkbPointZ:       pointZReader,
	wkbPointM:       pointMReader,
	wkbPointZM:      pointZMReader,
	wkbLineString:   lineStringReader,
	wkbLineStringZ:  lineStringZReader,
	wkbLineStringM:  lineStringMReader,
	wkbLineStringZM: lineStringZMReader,
	wkbPolygon:      polygonReader,
	wkbPolygonZ:     polygonZReader,
	wkbPolygonM:     polygonMReader,
	wkbPolygonZM:    polygonZMReader,
}

func readLinearRing(r io.Reader, byteOrder binary.ByteOrder) ([]Point, error) {
	var numPoints uint32
	if err := binary.Read(r, byteOrder, &numPoints); err != nil {
		return nil, err
	}
	points := make([]Point, numPoints)
	if err := binary.Read(r, byteOrder, &points); err != nil {
		return nil, err
	}
	return points, nil
}

func readLinearRingZ(r io.Reader, byteOrder binary.ByteOrder) ([]PointZ, error) {
	var numPoints uint32
	if err := binary.Read(r, byteOrder, &numPoints); err != nil {
		return nil, err
	}
	pointZs := make([]PointZ, numPoints)
	if err := binary.Read(r, byteOrder, &pointZs); err != nil {
		return nil, err
	}
	return pointZs, nil
}

func readLinearRingM(r io.Reader, byteOrder binary.ByteOrder) ([]PointM, error) {
	var numPoints uint32
	if err := binary.Read(r, byteOrder, &numPoints); err != nil {
		return nil, err
	}
	pointMs := make([]PointM, numPoints)
	if err := binary.Read(r, byteOrder, &pointMs); err != nil {
		return nil, err
	}
	return pointMs, nil
}

func readLinearRingZM(r io.Reader, byteOrder binary.ByteOrder) ([]PointZM, error) {
	var numPoints uint32
	if err := binary.Read(r, byteOrder, &numPoints); err != nil {
		return nil, err
	}
	pointZMs := make([]PointZM, numPoints)
	if err := binary.Read(r, byteOrder, &pointZMs); err != nil {
		return nil, err
	}
	return pointZMs, nil
}

func pointReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	point := Point{}
	if err := binary.Read(r, byteOrder, &point); err != nil {
		return nil, err
	}
	return point, nil
}

func pointZReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	pointZ := PointZ{}
	if err := binary.Read(r, byteOrder, &pointZ); err != nil {
		return nil, err
	}
	return pointZ, nil
}

func pointMReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	pointM := PointM{}
	if err := binary.Read(r, byteOrder, &pointM); err != nil {
		return nil, err
	}
	return pointM, nil
}

func pointZMReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	pointZM := PointZM{}
	if err := binary.Read(r, byteOrder, &pointZM); err != nil {
		return nil, err
	}
	return pointZM, nil
}

func lineStringReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	points, err := readLinearRing(r, byteOrder)
	if err != nil {
		return nil, err
	}
	return LineString{points}, nil
}

func lineStringZReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	pointZs, err := readLinearRingZ(r, byteOrder)
	if err != nil {
		return nil, err
	}
	return LineStringZ{pointZs}, nil
}

func lineStringMReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	pointMs, err := readLinearRingM(r, byteOrder)
	if err != nil {
		return nil, err
	}
	return LineStringM{pointMs}, nil
}

func lineStringZMReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	pointZMs, err := readLinearRingZM(r, byteOrder)
	if err != nil {
		return nil, err
	}
	return LineStringZM{pointZMs}, nil
}

func polygonReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	var numRings uint32
	if err := binary.Read(r, byteOrder, &numRings); err != nil {
		return nil, err
	}
	rings := make([]LinearRing, numRings)
	for i := uint32(0); i < numRings; i++ {
		if points, err := readLinearRing(r, byteOrder); err != nil {
			return nil, err
		} else {
			rings[i] = points
		}
	}
	return Polygon{rings}, nil
}

func polygonZReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	var numRings uint32
	if err := binary.Read(r, byteOrder, &numRings); err != nil {
		return nil, err
	}
	rings := make([]LinearRingZ, numRings)
	for i := uint32(0); i < numRings; i++ {
		if pointZs, err := readLinearRingZ(r, byteOrder); err != nil {
			return nil, err
		} else {
			rings[i] = pointZs
		}
	}
	return PolygonZ{rings}, nil
}

func polygonMReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	var numRings uint32
	if err := binary.Read(r, byteOrder, &numRings); err != nil {
		return nil, err
	}
	rings := make([]LinearRingM, numRings)
	for i := uint32(0); i < numRings; i++ {
		if pointMs, err := readLinearRingM(r, byteOrder); err != nil {
			return nil, err
		} else {
			rings[i] = pointMs
		}
	}
	return PolygonM{rings}, nil
}

func polygonZMReader(r io.Reader, byteOrder binary.ByteOrder) (Geom, error) {
	var numRings uint32
	if err := binary.Read(r, byteOrder, &numRings); err != nil {
		return nil, err
	}
	rings := make([]LinearRingZM, numRings)
	for i := uint32(0); i < numRings; i++ {
		if pointZMs, err := readLinearRingZM(r, byteOrder); err != nil {
			return nil, err
		} else {
			rings[i] = pointZMs
		}
	}
	return PolygonZM{rings}, nil
}

func Read(r io.Reader) (Geom, error) {

	var wkbByteOrder uint8
	if err := binary.Read(r, binary.LittleEndian, &wkbByteOrder); err != nil {
		return nil, err
	}
	var byteOrder binary.ByteOrder
	switch wkbByteOrder {
	case wkbXDR:
		byteOrder = binary.BigEndian
	case wkbNDR:
		byteOrder = binary.LittleEndian
	default:
		return nil, fmt.Errorf("invalid byte order %u", wkbByteOrder)
	}

	var wkbGeometryType uint32
	if err := binary.Read(r, byteOrder, &wkbGeometryType); err != nil {
		return nil, err
	}

	if reader, ok := wkbReaders[wkbGeometryType]; ok {
		return reader(r, byteOrder)
	} else {
		return nil, fmt.Errorf("unsupported geometry type %u", wkbGeometryType)
	}

}
<|endoftext|>


<commit_before>package config

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"path"
	"path/filepath"
	"strings"

	docker "github.com/fsouza/go-dockerclient"
)

// Config represents the configuration for a worker server.
type Config struct {
	path         string // where was config file loaded from?
	Registry     string `json:"registry"`
	Sandbox      string `json:"sandbox"`
	Pool         string `json:"pool"`
	Reg_dir      string `json:"reg_dir"` // store local copies of handler code
	Cluster_name string `json:"cluster_name"`

	// pool options
	Pool_dir        string `json:"pool_dir"`
	Num_forkservers int    `json:"num_forkservers"`

	// olregistry
	Reg_cluster []string `json:"reg_cluster"`

	// sandbox
	Worker_dir       string `json:"worker_dir"`
	Cgroup_init_path string `json: "cgroup_init_path"`
	Cgroup_base      string `json: "cgroup_base"`
	Worker_port      string `json:"worker_port"`
	Docker_host      string `json:"docker_host"`

	// sandbox factory
	Sandbox_buffer int `json:"sandbox_buffer"`

	// for unit testing to skip pull path
	Skip_pull_existing bool `json:"Skip_pull_existing"`

	// pass through to sandbox envirenment variable
	Sandbox_config interface{} `json:"sandbox_config"`

	// write benchmark times to separate log file
	Benchmark_file string `json:"benchmark_log"`
}

// SandboxConfJson marshals the Sandbox_config of the Config into a JSON string.
func (c *Config) SandboxConfJson() string {
	s, err := json.Marshal(c.Sandbox_config)
	if err != nil {
		panic(err)
	}
	return string(s)
}

// Dump prints the Config as a JSON string.
func (c *Config) Dump() {
	s, err := json.Marshal(c)
	if err != nil {
		panic(err)
	}
	log.Printf("CONFIG = %v\n", string(s))
}

// DumpStr returns the Config as an indented JSON string.
func (c *Config) DumpStr() string {
	s, err := json.MarshalIndent(c, "", "\t")
	if err != nil {
		panic(err)
	}
	return string(s)
}

// Save writes the Config as an indented JSON to path with 644 mode.
func (c *Config) Save(path string) error {
	s, err := json.MarshalIndent(c, "", "\t")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(path, s, 0644)
}

// Defaults verifies the fields of Config are correct, and initializes some
// if they are empty.
func (c *Config) Defaults() error {
	if c.Cluster_name == "" {
		c.Cluster_name = "default"
	}

	if c.Worker_port == "" {
		c.Worker_port = "8080"
	}

	if c.Num_forkservers == 0 {
		c.Num_forkservers = 1
	}

	if c.Registry == "olregistry" && len(c.Reg_cluster) == 0 {
		return fmt.Errorf("must specify reg_cluster")
	}

	if c.Reg_dir == "" {
		return fmt.Errorf("must specify local registry directory")
	}

	if !path.IsAbs(c.Reg_dir) {
		if c.path == "" {
			return fmt.Errorf("Reg_dir cannot be relative, unless config is loaded from file")
		}
		path, err := filepath.Abs(path.Join(path.Dir(c.path), c.Reg_dir))
		if err != nil {
			return err
		}
		c.Reg_dir = path
	}

	// worker dir
	if c.Worker_dir == "" {
		return fmt.Errorf("must specify local worker directory")
	}

	if !path.IsAbs(c.Worker_dir) {
		if c.path == "" {
			return fmt.Errorf("Worker_dir cannot be relative, unless config is loaded from file")
		}
		path, err := filepath.Abs(path.Join(path.Dir(c.path), c.Worker_dir))
		if err != nil {
			return err
		}
		c.Worker_dir = path
	}

	// cgroup sandboxes require some extra settings
	if c.Sandbox == "cgroup" {
		// cgroup_init path
		if c.Cgroup_init_path == "" {
			return fmt.Errorf("must specify Cgroup_init_path")
		}

		if !path.IsAbs(c.Cgroup_init_path) {
			if c.path == "" {
				return fmt.Errorf("Cgroup_init_path cannot be relative, unless config is loaded from file")
			}
			path, err := filepath.Abs(path.Join(path.Dir(c.path), c.Cgroup_init_path))
			if err != nil {
				return err
			}
			c.Cgroup_init_path = path
		}

		// cgroup base path
		if c.Cgroup_base == "" {
			return fmt.Errorf("must specify Cgroup_base")
		}

		if !path.IsAbs(c.Cgroup_base) {
			if c.path == "" {
				return fmt.Errorf("Cgroup_base cannot be relative, unless config is loaded from file")
			}
			path, err := filepath.Abs(path.Join(path.Dir(c.path), c.Cgroup_base))
			if err != nil {
				return err
			}
			c.Cgroup_base = path
		}
	}

	// pool dir
	if c.Pool != "" {
		if c.Pool_dir == "" {
			return fmt.Errorf("must specify local pool directory if using interpreter pool")
		}

		if !path.IsAbs(c.Pool_dir) {
			if c.path == "" {
				return fmt.Errorf("Pool_dir cannot be relative, unless config is loaded from file")
			}
			path, err := filepath.Abs(path.Join(path.Dir(c.path), c.Pool_dir))
			if err != nil {
				return err
			}
			c.Pool_dir = path
		}
	}

	// daemon
	if c.Docker_host == "" {
		client, err := docker.NewClientFromEnv()
		if err != nil {
			return fmt.Errorf("failed to get docker client: ", err)
		}

		endpoint := client.Endpoint()
		local := "unix://"
		nonLocal := "https://"
		if strings.HasPrefix(endpoint, local) {
			c.Docker_host = "localhost"
		} else if strings.HasPrefix(endpoint, nonLocal) {
			start := strings.Index(endpoint, nonLocal) + len([]rune(nonLocal))
			end := strings.LastIndex(endpoint, ":")
			c.Docker_host = endpoint[start:end]
		} else {
			return fmt.Errorf("please specify a valid docker host!")
		}
	}

	return nil
}

// ParseConfig reads a file and tries to parse it as a JSON string to a Config
// instance.
func ParseConfig(path string) (*Config, error) {
	config_raw, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("could not open config (%v): %v\n", path, err.Error())
	}
	var config Config

	if err := json.Unmarshal(config_raw, &config); err != nil {
		log.Printf("FILE: %v\n", config_raw)
		return nil, fmt.Errorf("could not parse config (%v): %v\n", path, err.Error())
	}

	config.path = path
	if err := config.Defaults(); err != nil {
		return nil, err
	}

	return &config, nil
}
<commit_msg>Debugging<commit_after>package config

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"path"
	"path/filepath"
	"strings"

	docker "github.com/fsouza/go-dockerclient"
)

// Config represents the configuration for a worker server.
type Config struct {
	path         string // where was config file loaded from?
	Registry     string `json:"registry"`
	Sandbox      string `json:"sandbox"`
	Pool         string `json:"pool"`
	Reg_dir      string `json:"reg_dir"` // store local copies of handler code
	Cluster_name string `json:"cluster_name"`

	// pool options
	Pool_dir        string `json:"pool_dir"`
	Num_forkservers int    `json:"num_forkservers"`

	// olregistry
	Reg_cluster []string `json:"reg_cluster"`

	// sandbox
	Worker_dir       string `json:"worker_dir"`
	Cgroup_init_path string `json: "cgroup_init_path"`
	Cgroup_base      string `json: "cgroup_base"`
	Worker_port      string `json:"worker_port"`
	Docker_host      string `json:"docker_host"`

	// sandbox factory
	Sandbox_buffer int `json:"sandbox_buffer"`

	// for unit testing to skip pull path
	Skip_pull_existing bool `json:"Skip_pull_existing"`

	// pass through to sandbox envirenment variable
	Sandbox_config interface{} `json:"sandbox_config"`

	// write benchmark times to separate log file
	Benchmark_file string `json:"benchmark_log"`
}

// SandboxConfJson marshals the Sandbox_config of the Config into a JSON string.
func (c *Config) SandboxConfJson() string {
	s, err := json.Marshal(c.Sandbox_config)
	if err != nil {
		panic(err)
	}
	return string(s)
}

// Dump prints the Config as a JSON string.
func (c *Config) Dump() {
	s, err := json.Marshal(c)
	if err != nil {
		panic(err)
	}
	log.Printf("CONFIG = %v\n", string(s))
}

// DumpStr returns the Config as an indented JSON string.
func (c *Config) DumpStr() string {
	s, err := json.MarshalIndent(c, "", "\t")
	if err != nil {
		panic(err)
	}
	return string(s)
}

// Save writes the Config as an indented JSON to path with 644 mode.
func (c *Config) Save(path string) error {
	s, err := json.MarshalIndent(c, "", "\t")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(path, s, 0644)
}

// Defaults verifies the fields of Config are correct, and initializes some
// if they are empty.
func (c *Config) Defaults() error {
	if c.Cluster_name == "" {
		c.Cluster_name = "default"
	}

	if c.Worker_port == "" {
		c.Worker_port = "8080"
	}

	if c.Num_forkservers == 0 {
		c.Num_forkservers = 1
	}

	if c.Registry == "olregistry" && len(c.Reg_cluster) == 0 {
		return fmt.Errorf("must specify reg_cluster")
	}

	if c.Reg_dir == "" {
		return fmt.Errorf("must specify local registry directory")
	}

	if !path.IsAbs(c.Reg_dir) {
		if c.path == "" {
			return fmt.Errorf("Reg_dir cannot be relative, unless config is loaded from file")
		}
		path, err := filepath.Abs(path.Join(path.Dir(c.path), c.Reg_dir))
		if err != nil {
			return err
		}
		c.Reg_dir = path
	}

	// worker dir
	if c.Worker_dir == "" {
		return fmt.Errorf("must specify local worker directory")
	}

	if !path.IsAbs(c.Worker_dir) {
		if c.path == "" {
			return fmt.Errorf("Worker_dir cannot be relative, unless config is loaded from file")
		}
		path, err := filepath.Abs(path.Join(path.Dir(c.path), c.Worker_dir))
		if err != nil {
			return err
		}
		c.Worker_dir = path
	}

	// cgroup sandboxes require some extra settings
	if c.Sandbox == "cgroup" {
		// cgroup_init path
		if c.Cgroup_init_path == "" {
			return fmt.Errorf("must specify Cgroup_init_path")
		}

		if !path.IsAbs(c.Cgroup_init_path) {
			if c.path == "" {
				return fmt.Errorf("Cgroup_init_path cannot be relative, unless config is loaded from file")
			}
			path, err := filepath.Abs(path.Join(path.Dir(c.path), c.Cgroup_init_path))
			if err != nil {
				return err
			}
			c.Cgroup_init_path = path
		}

		// cgroup base path
		if c.Cgroup_base == "" {
			return fmt.Errorf("must specify Cgroup_base")
		}

		if !path.IsAbs(c.Cgroup_base) {
			if c.path == "" {
				return fmt.Errorf("Cgroup_base cannot be relative, unless config is loaded from file")
			}
			path, err := filepath.Abs(path.Join(path.Dir(c.path), c.Cgroup_base))
			if err != nil {
				return err
			}
			c.Cgroup_base = path
		}
	}

	// pool dir
	if c.Pool != "" {
		if c.Pool_dir == "" {
			return fmt.Errorf("must specify local pool directory if using interpreter pool")
		}

		if !path.IsAbs(c.Pool_dir) {
			if c.path == "" {
				return fmt.Errorf("Pool_dir cannot be relative, unless config is loaded from file")
			}
			path, err := filepath.Abs(path.Join(path.Dir(c.path), c.Pool_dir))
			if err != nil {
				return err
			}
			c.Pool_dir = path
		}
	}

	// daemon
	if c.Docker_host == "" {
		client, err := docker.NewClientFromEnv()
		if err != nil {
			return fmt.Errorf("failed to get docker client: ", err)
		}

		endpoint := client.Endpoint()
		local := "unix://"
		nonLocal := "https://"
		if strings.HasPrefix(endpoint, local) {
			c.Docker_host = "localhost"
		} else if strings.HasPrefix(endpoint, nonLocal) {
			start := strings.Index(endpoint, nonLocal) + len([]rune(nonLocal))
			end := strings.LastIndex(endpoint, ":")
			c.Docker_host = endpoint[start:end]
		} else {
			return fmt.Errorf("please specify a valid docker host!")
		}
	}

	if c.Sandbox == "" {
		c.Sandbox = "docker"
	}

	return nil
}

// ParseConfig reads a file and tries to parse it as a JSON string to a Config
// instance.
func ParseConfig(path string) (*Config, error) {
	config_raw, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("could not open config (%v): %v\n", path, err.Error())
	}
	var config Config

	if err := json.Unmarshal(config_raw, &config); err != nil {
		log.Printf("FILE: %v\n", config_raw)
		return nil, fmt.Errorf("could not parse config (%v): %v\n", path, err.Error())
	}

	config.path = path
	if err := config.Defaults(); err != nil {
		return nil, err
	}

	return &config, nil
}
<|endoftext|>


<commit_before>package workflow

import (
	"fmt"
	"testing"
)

func TestNewRunner(t *testing.T) {
	uuid := "some-uuid"

	varsWF := Workflow{"Some workflow", []Step{Step{Name: "a-step", Type: "none", Context: map[string]string{}}}, map[string]string{"a": "b"}}
	noVarsWF := Workflow{"Some workflow", []Step{Step{Name: "a-step", Type: "none", Context: map[string]string{}}}, map[string]string{}}

	type args struct {
		uuid string
		wf   Workflow
	}
	tests := []struct {
		name    string
		args    args
		hasVars bool
	}{
		{"creates a runner with vars", args{uuid, varsWF}, true},
		{"creates a runner with no vars", args{uuid, noVarsWF}, false},
	}
	for _, tt := range tests {
		var r interface{}
		r = NewRunner(tt.args.uuid, tt.args.wf)

		t.Run(tt.name, func(t *testing.T) {
			switch r.(type) {
			case Runner:
			default:
				t.Errorf("NewRunner() error: expected Runner, got %T", r)
			}
		})

		t.Run(fmt.Sprintf("%s - UUID", tt.name), func(t *testing.T) {
			if r.(Runner).UUID != uuid {
				t.Errorf("NewRunner() UUID error: expected %q, got %q", uuid, r.(Runner).UUID)
			}
		})

		t.Run(fmt.Sprintf("%s - Variables", tt.name), func(t *testing.T) {
			if _, ok := r.(Runner).Variables["Defaults"]; !ok {
				t.Errorf("NewRunner() Variables error: expected key Defaults, got %V", r.(Runner).Variables)
			}
		})

		if tt.hasVars {
			t.Run(fmt.Sprintf("%s - vars", tt.name), func(t *testing.T) {
				if _, ok := r.(Runner).Variables["Defaults"].(map[string]string)["a"]; !ok {
					t.Errorf("NewRunner() Default -> Variables error: expected key 'a', got %V", r.(Runner).Variables["Defaults"])
				}
			})
		}
	}
}

func TestParseRunner(t *testing.T) {
	wfrJSON := `
{
  "UUID": "some-uuid",
  "Last": "a step",
  "Variables": {
    "Defaults": {
      "foo": "bar"
    },
    "Something": {
      "baz": "quux"
    }
  },
  "Workflow": {
    "Name": "A Workflow",
    "Steps": [
      {
        "Name": "a step",
        "Type": "none"
      }
    ]
  }
}
`
	type args struct {
		data string
	}
	tests := []struct {
		name    string
		args    args
		wantErr bool
	}{
		{"simple runner", args{wfrJSON}, false},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			_, err := ParseRunner(tt.args.data)
			if (err != nil) != tt.wantErr {
				t.Errorf("ParseRunner() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
		})
	}
}

func TestRunner_Next(t *testing.T) {
	wf := Workflow{"Some workflow", []Step{Step{Name: "a step", Type: "none"}, Step{Name: "final step", Type: "none"}}, map[string]string{}}

	type fields struct {
		Last     string
		Workflow Workflow
	}
	tests := []struct {
		name     string
		fields   fields
		stepName string
		wantDone bool
	}{
		{"workflow with non-final step", fields{"a step", wf}, "final step", false},
		{"workflow with final step", fields{"final step", wf}, "final step", true},
	}
	for _, tt := range tests {
		wfr := &Runner{
			Last:     tt.fields.Last,
			Workflow: tt.fields.Workflow,
		}
		gotStep, gotDone := wfr.Next()

		t.Run(fmt.Sprintf("%s - done status", tt.name), func(t *testing.T) {
			if gotDone != tt.wantDone {
				t.Errorf("Runner.Next() gotDone = %v, want %v", gotDone, tt.wantDone)
			}
		})

		t.Run(fmt.Sprintf("%s - next step name", tt.name), func(t *testing.T) {
			if gotStep.Name != tt.stepName {
				t.Errorf("Runner.Next() gotStep.Name = %v, want %v", gotStep.Name, tt.stepName)
			}
		})

	}
}

func TestRunner_Current(t *testing.T) {
	wf := Workflow{"Some workflow", []Step{Step{Name: "a step", Type: "none"}, Step{Name: "final step", Type: "none"}}, map[string]string{}}

	type fields struct {
		Last     string
		Workflow Workflow
	}
	tests := []struct {
		name     string
		fields   fields
		stepName string
	}{
		{"No set step", fields{"", wf}, "a step"},
		{"First step", fields{"a step", wf}, "a step"},
		{"Last step", fields{"final step", wf}, "final step"},
	}
	for _, tt := range tests {
		wfr := &Runner{
			Last:     tt.fields.Last,
			Workflow: tt.fields.Workflow,
		}
		_, s := wfr.Current()

		t.Run(tt.name, func(t *testing.T) {
			if s.Name != tt.stepName {
				t.Errorf("Runner.Current() Step name = %v, want %v", s.Name, tt.stepName)
			}
		})
	}
}

func TestRunner_End(t *testing.T) {
	wfr := &Runner{}
	t.Run("set end state", func(t *testing.T) {
		if wfr.State == "ended" {
			t.Errorf("Runner.End() error: step already set to ended, %V", wfr)
		}

		wfr.End()

		if wfr.State != "ended" {
			t.Errorf("Runner.End() state change did not persist to ended, %V", wfr)
		}
	})
}

func TestRunner_Fail(t *testing.T) {
	wfr := &Runner{}
	failMsg := "a failure occurred"

	t.Run("set fail state", func(t *testing.T) {
		if wfr.State == "failed" {
			t.Errorf("Runner.Fail() error: step already set to failed, %V", wfr)
		}

		wfr.Fail(failMsg)

		if wfr.State != "failed" {
			t.Errorf("Runner.Fail() state change did not persist to failed, %V", wfr)
		}

		if wfr.ErrorMessage != failMsg {
			t.Errorf("Runner.Fail() state change did not persist error message, %V", wfr)
		}
	})
}

func TestRunner_Start(t *testing.T) {
	wfr := &Runner{}
	t.Run("set start state", func(t *testing.T) {
		if wfr.State == "started" {
			t.Errorf("Runner.Start() error: step already set to started, %V", wfr)
		}

		wfr.Start()

		if wfr.State != "started" {
			t.Errorf("Runner.Start() state change did not persist to started, %V", wfr)
		}
	})
}
<commit_msg>go fmt simplify<commit_after>package workflow

import (
	"fmt"
	"testing"
)

func TestNewRunner(t *testing.T) {
	uuid := "some-uuid"

	varsWF := Workflow{"Some workflow", []Step{{Name: "a-step", Type: "none", Context: map[string]string{}}}, map[string]string{"a": "b"}}
	noVarsWF := Workflow{"Some workflow", []Step{{Name: "a-step", Type: "none", Context: map[string]string{}}}, map[string]string{}}

	type args struct {
		uuid string
		wf   Workflow
	}
	tests := []struct {
		name    string
		args    args
		hasVars bool
	}{
		{"creates a runner with vars", args{uuid, varsWF}, true},
		{"creates a runner with no vars", args{uuid, noVarsWF}, false},
	}
	for _, tt := range tests {
		var r interface{}
		r = NewRunner(tt.args.uuid, tt.args.wf)

		t.Run(tt.name, func(t *testing.T) {
			switch r.(type) {
			case Runner:
			default:
				t.Errorf("NewRunner() error: expected Runner, got %T", r)
			}
		})

		t.Run(fmt.Sprintf("%s - UUID", tt.name), func(t *testing.T) {
			if r.(Runner).UUID != uuid {
				t.Errorf("NewRunner() UUID error: expected %q, got %q", uuid, r.(Runner).UUID)
			}
		})

		t.Run(fmt.Sprintf("%s - Variables", tt.name), func(t *testing.T) {
			if _, ok := r.(Runner).Variables["Defaults"]; !ok {
				t.Errorf("NewRunner() Variables error: expected key Defaults, got %V", r.(Runner).Variables)
			}
		})

		if tt.hasVars {
			t.Run(fmt.Sprintf("%s - vars", tt.name), func(t *testing.T) {
				if _, ok := r.(Runner).Variables["Defaults"].(map[string]string)["a"]; !ok {
					t.Errorf("NewRunner() Default -> Variables error: expected key 'a', got %V", r.(Runner).Variables["Defaults"])
				}
			})
		}
	}
}

func TestParseRunner(t *testing.T) {
	wfrJSON := `
{
  "UUID": "some-uuid",
  "Last": "a step",
  "Variables": {
    "Defaults": {
      "foo": "bar"
    },
    "Something": {
      "baz": "quux"
    }
  },
  "Workflow": {
    "Name": "A Workflow",
    "Steps": [
      {
        "Name": "a step",
        "Type": "none"
      }
    ]
  }
}
`
	type args struct {
		data string
	}
	tests := []struct {
		name    string
		args    args
		wantErr bool
	}{
		{"simple runner", args{wfrJSON}, false},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			_, err := ParseRunner(tt.args.data)
			if (err != nil) != tt.wantErr {
				t.Errorf("ParseRunner() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
		})
	}
}

func TestRunner_Next(t *testing.T) {
	wf := Workflow{"Some workflow", []Step{{Name: "a step", Type: "none"}, {Name: "final step", Type: "none"}}, map[string]string{}}

	type fields struct {
		Last     string
		Workflow Workflow
	}
	tests := []struct {
		name     string
		fields   fields
		stepName string
		wantDone bool
	}{
		{"workflow with non-final step", fields{"a step", wf}, "final step", false},
		{"workflow with final step", fields{"final step", wf}, "final step", true},
	}
	for _, tt := range tests {
		wfr := &Runner{
			Last:     tt.fields.Last,
			Workflow: tt.fields.Workflow,
		}
		gotStep, gotDone := wfr.Next()

		t.Run(fmt.Sprintf("%s - done status", tt.name), func(t *testing.T) {
			if gotDone != tt.wantDone {
				t.Errorf("Runner.Next() gotDone = %v, want %v", gotDone, tt.wantDone)
			}
		})

		t.Run(fmt.Sprintf("%s - next step name", tt.name), func(t *testing.T) {
			if gotStep.Name != tt.stepName {
				t.Errorf("Runner.Next() gotStep.Name = %v, want %v", gotStep.Name, tt.stepName)
			}
		})

	}
}

func TestRunner_Current(t *testing.T) {
	wf := Workflow{"Some workflow", []Step{{Name: "a step", Type: "none"}, {Name: "final step", Type: "none"}}, map[string]string{}}

	type fields struct {
		Last     string
		Workflow Workflow
	}
	tests := []struct {
		name     string
		fields   fields
		stepName string
	}{
		{"No set step", fields{"", wf}, "a step"},
		{"First step", fields{"a step", wf}, "a step"},
		{"Last step", fields{"final step", wf}, "final step"},
	}
	for _, tt := range tests {
		wfr := &Runner{
			Last:     tt.fields.Last,
			Workflow: tt.fields.Workflow,
		}
		_, s := wfr.Current()

		t.Run(tt.name, func(t *testing.T) {
			if s.Name != tt.stepName {
				t.Errorf("Runner.Current() Step name = %v, want %v", s.Name, tt.stepName)
			}
		})
	}
}

func TestRunner_End(t *testing.T) {
	wfr := &Runner{}
	t.Run("set end state", func(t *testing.T) {
		if wfr.State == "ended" {
			t.Errorf("Runner.End() error: step already set to ended, %V", wfr)
		}

		wfr.End()

		if wfr.State != "ended" {
			t.Errorf("Runner.End() state change did not persist to ended, %V", wfr)
		}
	})
}

func TestRunner_Fail(t *testing.T) {
	wfr := &Runner{}
	failMsg := "a failure occurred"

	t.Run("set fail state", func(t *testing.T) {
		if wfr.State == "failed" {
			t.Errorf("Runner.Fail() error: step already set to failed, %V", wfr)
		}

		wfr.Fail(failMsg)

		if wfr.State != "failed" {
			t.Errorf("Runner.Fail() state change did not persist to failed, %V", wfr)
		}

		if wfr.ErrorMessage != failMsg {
			t.Errorf("Runner.Fail() state change did not persist error message, %V", wfr)
		}
	})
}

func TestRunner_Start(t *testing.T) {
	wfr := &Runner{}
	t.Run("set start state", func(t *testing.T) {
		if wfr.State == "started" {
			t.Errorf("Runner.Start() error: step already set to started, %V", wfr)
		}

		wfr.Start()

		if wfr.State != "started" {
			t.Errorf("Runner.Start() state change did not persist to started, %V", wfr)
		}
	})
}
<|endoftext|>


<commit_before>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package id3v2

import (
	"bufio"
	"bytes"
	"io/ioutil"
	"os"
	"testing"

	"github.com/bogem/id3v2/util"
)

const (
	mp3Name        = "testdata/test.mp3"
	frontCoverName = "testdata/front_cover.jpg"
	backCoverName  = "testdata/back_cover.jpg"
	framesSize     = 222524
	tagSize        = tagHeaderSize + framesSize
	musicSize      = 4557971
	countOfFrames  = 12
)

var (
	frontCover = PictureFrame{
		Encoding:    ENUTF8,
		MimeType:    "image/jpeg",
		PictureType: PTFrontCover,
		Description: "Front cover",
	}
	backCover = PictureFrame{
		Encoding:    ENUTF8,
		MimeType:    "image/jpeg",
		PictureType: PTBackCover,
		Description: "Back cover",
	}

	engUSLF = UnsynchronisedLyricsFrame{
		Encoding:          ENUTF8,
		Language:          "eng",
		ContentDescriptor: "Content descriptor",
		Lyrics:            "bogem/id3v2",
	}
	gerUSLF = UnsynchronisedLyricsFrame{
		Encoding:          ENUTF8,
		Language:          "ger",
		ContentDescriptor: "Inhaltsdeskriptor",
		Lyrics:            "Einigkeit und Recht und Freiheit",
	}

	engComm = CommentFrame{
		Encoding:    ENUTF8,
		Language:    "eng",
		Description: "Short description",
		Text:        "The actual text",
	}
	gerComm = CommentFrame{
		Encoding:    ENUTF8,
		Language:    "ger",
		Description: "Kurze Beschreibung",
		Text:        "Der eigentliche Text",
	}

	unknownFrameID = "WPUB"
	unknownFrame   = UnknownFrame{
		body: []byte("https://soundcloud.com/suicidepart2"),
	}
)

func init() {
	var err error

	// Set covers' picture.
	frontCover.Picture, err = ioutil.ReadFile(frontCoverName)
	if err != nil {
		panic("Error while reading front cover file: " + err.Error())
	}
	backCover.Picture, err = ioutil.ReadFile(backCoverName)
	if err != nil {
		panic("Error while reading back cover file: " + err.Error())
	}
	if err := resetMP3Tag(); err != nil {
		panic("Error while reseting mp3 file: " + err.Error())
	}
}

// resetMP3Tag sets the default frames to mp3Name.
func resetMP3Tag() error {
	tag, err := Open(mp3Name)
	if tag == nil || err != nil {
		return err
	}

	tag.SetTitle("Title")
	tag.SetArtist("Artist")
	tag.SetAlbum("Album")
	tag.SetYear("2016")
	tag.SetGenre("Genre")

	// Set picture frames
	tag.AddAttachedPicture(frontCover)
	tag.AddAttachedPicture(backCover)

	// Set USLTs
	tag.AddUnsynchronisedLyricsFrame(engUSLF)
	tag.AddUnsynchronisedLyricsFrame(gerUSLF)

	// Set comments
	tag.AddCommentFrame(engComm)
	tag.AddCommentFrame(gerComm)

	// Set unknown frame
	tag.AddFrame(unknownFrameID, unknownFrame)

	if err = tag.Save(); err != nil {
		return err
	}

	if err = tag.Close(); err != nil {
		return err
	}

	return nil
}

func TestCountLenSize(t *testing.T) {
	tag, err := Open(mp3Name)
	if tag == nil || err != nil {
		t.Fatal("Error while opening mp3 file:", err)
	}

	// Check count
	if tag.Count() != countOfFrames {
		t.Errorf("Expected frames: %v, got: %v", countOfFrames, tag.Count())
	}

	// Check len of tag.AllFrames()
	if len(tag.AllFrames()) != 9 {
		t.Errorf("Expected: %v, got: %v", 9, len(tag.AllFrames()))
	}

	// Check saved tag size by reading the 6:10 bytes of mp3 file
	mp3, err := os.Open(mp3Name)
	if err != nil {
		t.Fatal("Error while opening mp3 file:", err)
	}
	tagHeader := make([]byte, tagHeaderSize)
	n, err := mp3.Read(tagHeader)
	if n != tagHeaderSize {
		t.Errorf("Expected length of header %v, got %v", tagHeaderSize, n)
	}
	if err != nil {
		t.Error("Error while reading a tag header:", err)
	}

	size, err := util.ParseSize(tagHeader[6:10])
	if err != nil {
		t.Error("Error while parsing a tag header size:", err)
	}

	if framesSize != size {
		t.Errorf("Expected size of frames: %v, got: %v", framesSize, size)
	}

	// Check tag.Size
	tagSize := tagHeaderSize + framesSize
	if tag.Size() != tagSize {
		t.Errorf("Expected tag.Size(): %v, got: %v", tagSize, tag.Size())
	}

	if err = tag.Close(); err != nil {
		t.Error("Error while closing a tag:", err)
	}
}

// TestIntegrityOfMusicAtTheBeginning checks
// if tag.Save doesn't truncate or add some extra bytes at the beginning
// of music part.
func TestIntegrityOfMusicAtTheBeginning(t *testing.T) {
	mp3, err := os.Open(mp3Name)
	if err != nil {
		t.Fatal("Error while opening mp3 file:", err)
	}

	rd := bufio.NewReader(mp3)
	n, err := rd.Discard(tagSize)
	if n != tagSize {
		t.Errorf("Expected length of discarded bytes %v, got %v", tagSize, n)
	}
	if err != nil {
		t.Fatal("Error while reading mp3 file:", err)
	}

	expected := []byte{255, 251, 144, 68, 0, 0, 0}
	got := make([]byte, len(expected))
	n, err = rd.Read(got)
	if n != len(expected) {
		t.Errorf("Expected length of read bytes %v, got %v", len(expected), n)
	}
	if err != nil {
		t.Fatal("Error while reading mp3 file:", err)
	}

	if !bytes.Equal(expected, got) {
		t.Fail()
	}

	if err = mp3.Close(); err != nil {
		t.Error("Error while closing a tag:", err)
	}
}

// TestIntegrityOfMusicAtTheEnd checks
// if tag.Save doesn't truncate music part or add some extra bytes at the end
// of music part.
func TestIntegrityOfMusicAtTheEnd(t *testing.T) {
	mp3, err := os.Open(mp3Name)
	if err != nil {
		t.Fatal("Error while opening mp3 file:", err)
	}

	rd := bufio.NewReader(mp3)
	expected := []byte{0, 0, 0, 0, 0, 0, 255}
	toDiscard := tagSize + musicSize - len(expected)
	n, err := rd.Discard(toDiscard)
	if n != toDiscard {
		t.Errorf("Expected length of discarded bytes %v, got %v", toDiscard, n)
	}
	if err != nil {
		t.Fatal("Error while discarding:", err)
	}

	got := make([]byte, len(expected))
	n, err = rd.Read(got)
	if n != len(expected) {
		t.Errorf("Expected length of read bytes %v, got %v", len(expected), n)
	}
	if err != nil {
		t.Fatal("Error while reading mp3 file:", err)
	}

	if !bytes.Equal(expected, got) {
		t.Fail()
	}

	if err = mp3.Close(); err != nil {
		t.Error("Error while closing a tag:", err)
	}
}

// TestCheckPermissions checks
// if tag.Save creates file with the same permissions of original file.
func TestCheckPermissions(t *testing.T) {
	originalFile, err := os.Open(mp3Name)
	if err != nil {
		t.Fatal("Error while opening mp3 file:", err)
	}
	originalStat, err := originalFile.Stat()
	if err != nil {
		t.Fatal("Error while getting mp3 file stat:", err)
	}
	originalMode := originalStat.Mode()
	originalFile.Close()

	tag, err := Open(mp3Name)
	if err != nil {
		t.Fatal("Error while parsing a tag:", err)
	}
	if err = tag.Save(); err != nil {
		t.Error("Error while saving a tag:", err)
	}
	if err = tag.Close(); err != nil {
		t.Error("Error while closing a tag:", err)
	}

	newFile, err := os.Open(mp3Name)
	if err != nil {
		t.Fatal("Error while opening mp3 file:", err)
	}
	newStat, err := newFile.Stat()
	if err != nil {
		t.Fatal("Error while getting mp3 file stats:", err)
	}
	newMode := newStat.Mode()

	if originalMode != newMode {
		t.Errorf("Expected permissions: %v, got %v", originalMode, newMode)
	}
}

// TestBlankID deletes all frames in tag, adds frame with blank id and checks
// if no tag is written by tag.Size (tag.WriteTo must not write tag to file
// if there are 0 frames).
func TestBlankID(t *testing.T) {
	tag, err := Open(mp3Name)
	if tag == nil || err != nil {
		t.Fatal("Error while opening mp3 file:", err)
	}

	tag.DeleteAllFrames()
	tag.AddFrame("", unknownFrame)

	if tag.Count() > 0 {
		t.Error("There should be no frames in tag, but there are", tag.Count())
	}

	if tag.HasAnyFrames() {
		t.Error("tag.HasAnyFrames should return false, but it returns true")
	}

	if tag.Size() != 0 {
		t.Error("Size of tag should be 0. Actual tag size:", tag.Size())
	}

	// tag.Save should write no frames to file
	if err = tag.Save(); err != nil {
		t.Error("Error while saving a tag:", err)
	}

	if err = tag.Close(); err != nil {
		t.Error("Error while closing a tag:", err)
	}

	// Parse tag. It should be no frames
	parsedTag, err := Open(mp3Name)
	if parsedTag == nil || err != nil {
		t.Fatal("Error while opening mp3 file:", err)
	}

	if tag.Count() > 0 {
		t.Error("There should be no frames in parsed tag, but there are", tag.Count())
	}

	if tag.HasAnyFrames() {
		t.Error("Parsed tag.HasAnyFrames should return false, but it returns true")
	}

	if tag.Size() != 0 {
		t.Error("Size of parsed tag should be 0. Actual tag size:", tag.Size())
	}
}
<commit_msg>Show expected and got in TestIntegrity if it fails<commit_after>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package id3v2

import (
	"bufio"
	"bytes"
	"io/ioutil"
	"os"
	"testing"

	"github.com/bogem/id3v2/util"
)

const (
	mp3Name        = "testdata/test.mp3"
	frontCoverName = "testdata/front_cover.jpg"
	backCoverName  = "testdata/back_cover.jpg"
	framesSize     = 222524
	tagSize        = tagHeaderSize + framesSize
	musicSize      = 4557971
	countOfFrames  = 12
)

var (
	frontCover = PictureFrame{
		Encoding:    ENUTF8,
		MimeType:    "image/jpeg",
		PictureType: PTFrontCover,
		Description: "Front cover",
	}
	backCover = PictureFrame{
		Encoding:    ENUTF8,
		MimeType:    "image/jpeg",
		PictureType: PTBackCover,
		Description: "Back cover",
	}

	engUSLF = UnsynchronisedLyricsFrame{
		Encoding:          ENUTF8,
		Language:          "eng",
		ContentDescriptor: "Content descriptor",
		Lyrics:            "bogem/id3v2",
	}
	gerUSLF = UnsynchronisedLyricsFrame{
		Encoding:          ENUTF8,
		Language:          "ger",
		ContentDescriptor: "Inhaltsdeskriptor",
		Lyrics:            "Einigkeit und Recht und Freiheit",
	}

	engComm = CommentFrame{
		Encoding:    ENUTF8,
		Language:    "eng",
		Description: "Short description",
		Text:        "The actual text",
	}
	gerComm = CommentFrame{
		Encoding:    ENUTF8,
		Language:    "ger",
		Description: "Kurze Beschreibung",
		Text:        "Der eigentliche Text",
	}

	unknownFrameID = "WPUB"
	unknownFrame   = UnknownFrame{
		body: []byte("https://soundcloud.com/suicidepart2"),
	}
)

func init() {
	var err error

	// Set covers' picture.
	frontCover.Picture, err = ioutil.ReadFile(frontCoverName)
	if err != nil {
		panic("Error while reading front cover file: " + err.Error())
	}
	backCover.Picture, err = ioutil.ReadFile(backCoverName)
	if err != nil {
		panic("Error while reading back cover file: " + err.Error())
	}
	if err := resetMP3Tag(); err != nil {
		panic("Error while reseting mp3 file: " + err.Error())
	}
}

// resetMP3Tag sets the default frames to mp3Name.
func resetMP3Tag() error {
	tag, err := Open(mp3Name)
	if tag == nil || err != nil {
		return err
	}

	tag.SetTitle("Title")
	tag.SetArtist("Artist")
	tag.SetAlbum("Album")
	tag.SetYear("2016")
	tag.SetGenre("Genre")

	// Set picture frames
	tag.AddAttachedPicture(frontCover)
	tag.AddAttachedPicture(backCover)

	// Set USLTs
	tag.AddUnsynchronisedLyricsFrame(engUSLF)
	tag.AddUnsynchronisedLyricsFrame(gerUSLF)

	// Set comments
	tag.AddCommentFrame(engComm)
	tag.AddCommentFrame(gerComm)

	// Set unknown frame
	tag.AddFrame(unknownFrameID, unknownFrame)

	if err = tag.Save(); err != nil {
		return err
	}

	if err = tag.Close(); err != nil {
		return err
	}

	return nil
}

func TestCountLenSize(t *testing.T) {
	tag, err := Open(mp3Name)
	if tag == nil || err != nil {
		t.Fatal("Error while opening mp3 file:", err)
	}

	// Check count
	if tag.Count() != countOfFrames {
		t.Errorf("Expected frames: %v, got: %v", countOfFrames, tag.Count())
	}

	// Check len of tag.AllFrames()
	if len(tag.AllFrames()) != 9 {
		t.Errorf("Expected: %v, got: %v", 9, len(tag.AllFrames()))
	}

	// Check saved tag size by reading the 6:10 bytes of mp3 file
	mp3, err := os.Open(mp3Name)
	if err != nil {
		t.Fatal("Error while opening mp3 file:", err)
	}
	tagHeader := make([]byte, tagHeaderSize)
	n, err := mp3.Read(tagHeader)
	if n != tagHeaderSize {
		t.Errorf("Expected length of header %v, got %v", tagHeaderSize, n)
	}
	if err != nil {
		t.Error("Error while reading a tag header:", err)
	}

	size, err := util.ParseSize(tagHeader[6:10])
	if err != nil {
		t.Error("Error while parsing a tag header size:", err)
	}

	if framesSize != size {
		t.Errorf("Expected size of frames: %v, got: %v", framesSize, size)
	}

	// Check tag.Size
	tagSize := tagHeaderSize + framesSize
	if tag.Size() != tagSize {
		t.Errorf("Expected tag.Size(): %v, got: %v", tagSize, tag.Size())
	}

	if err = tag.Close(); err != nil {
		t.Error("Error while closing a tag:", err)
	}
}

// TestIntegrityOfMusicAtTheBeginning checks
// if tag.Save doesn't truncate or add some extra bytes at the beginning
// of music part.
func TestIntegrityOfMusicAtTheBeginning(t *testing.T) {
	mp3, err := os.Open(mp3Name)
	if err != nil {
		t.Fatal("Error while opening mp3 file:", err)
	}

	rd := bufio.NewReader(mp3)
	n, err := rd.Discard(tagSize)
	if n != tagSize {
		t.Errorf("Expected length of discarded bytes %v, got %v", tagSize, n)
	}
	if err != nil {
		t.Fatal("Error while reading mp3 file:", err)
	}

	expected := []byte{255, 251, 144, 68, 0, 0, 0}
	got := make([]byte, len(expected))
	n, err = rd.Read(got)
	if n != len(expected) {
		t.Errorf("Expected length of read bytes %v, got %v", len(expected), n)
	}
	if err != nil {
		t.Fatal("Error while reading mp3 file:", err)
	}

	if !bytes.Equal(expected, got) {
		t.Fatalf("Expected %v, got %v", expected, got)
	}

	if err = mp3.Close(); err != nil {
		t.Error("Error while closing a tag:", err)
	}
}

// TestIntegrityOfMusicAtTheEnd checks
// if tag.Save doesn't truncate music part or add some extra bytes at the end
// of music part.
func TestIntegrityOfMusicAtTheEnd(t *testing.T) {
	mp3, err := os.Open(mp3Name)
	if err != nil {
		t.Fatal("Error while opening mp3 file:", err)
	}

	rd := bufio.NewReader(mp3)
	expected := []byte{0, 0, 0, 0, 0, 0, 255}
	toDiscard := tagSize + musicSize - len(expected)
	n, err := rd.Discard(toDiscard)
	if n != toDiscard {
		t.Errorf("Expected length of discarded bytes %v, got %v", toDiscard, n)
	}
	if err != nil {
		t.Fatal("Error while discarding:", err)
	}

	got := make([]byte, len(expected))
	n, err = rd.Read(got)
	if n != len(expected) {
		t.Errorf("Expected length of read bytes %v, got %v", len(expected), n)
	}
	if err != nil {
		t.Fatal("Error while reading mp3 file:", err)
	}

	if !bytes.Equal(expected, got) {
		t.Fatalf("Expected %v, got %v", expected, got)
	}

	if err = mp3.Close(); err != nil {
		t.Error("Error while closing a tag:", err)
	}
}

// TestCheckPermissions checks
// if tag.Save creates file with the same permissions of original file.
func TestCheckPermissions(t *testing.T) {
	originalFile, err := os.Open(mp3Name)
	if err != nil {
		t.Fatal("Error while opening mp3 file:", err)
	}
	originalStat, err := originalFile.Stat()
	if err != nil {
		t.Fatal("Error while getting mp3 file stat:", err)
	}
	originalMode := originalStat.Mode()
	originalFile.Close()

	tag, err := Open(mp3Name)
	if err != nil {
		t.Fatal("Error while parsing a tag:", err)
	}
	if err = tag.Save(); err != nil {
		t.Error("Error while saving a tag:", err)
	}
	if err = tag.Close(); err != nil {
		t.Error("Error while closing a tag:", err)
	}

	newFile, err := os.Open(mp3Name)
	if err != nil {
		t.Fatal("Error while opening mp3 file:", err)
	}
	newStat, err := newFile.Stat()
	if err != nil {
		t.Fatal("Error while getting mp3 file stats:", err)
	}
	newMode := newStat.Mode()

	if originalMode != newMode {
		t.Errorf("Expected permissions: %v, got %v", originalMode, newMode)
	}
}

// TestBlankID deletes all frames in tag, adds frame with blank id and checks
// if no tag is written by tag.Size (tag.WriteTo must not write tag to file
// if there are 0 frames).
func TestBlankID(t *testing.T) {
	tag, err := Open(mp3Name)
	if tag == nil || err != nil {
		t.Fatal("Error while opening mp3 file:", err)
	}

	tag.DeleteAllFrames()
	tag.AddFrame("", unknownFrame)

	if tag.Count() > 0 {
		t.Error("There should be no frames in tag, but there are", tag.Count())
	}

	if tag.HasAnyFrames() {
		t.Error("tag.HasAnyFrames should return false, but it returns true")
	}

	if tag.Size() != 0 {
		t.Error("Size of tag should be 0. Actual tag size:", tag.Size())
	}

	// tag.Save should write no frames to file
	if err = tag.Save(); err != nil {
		t.Error("Error while saving a tag:", err)
	}

	if err = tag.Close(); err != nil {
		t.Error("Error while closing a tag:", err)
	}

	// Parse tag. It should be no frames
	parsedTag, err := Open(mp3Name)
	if parsedTag == nil || err != nil {
		t.Fatal("Error while opening mp3 file:", err)
	}

	if tag.Count() > 0 {
		t.Error("There should be no frames in parsed tag, but there are", tag.Count())
	}

	if tag.HasAnyFrames() {
		t.Error("Parsed tag.HasAnyFrames should return false, but it returns true")
	}

	if tag.Size() != 0 {
		t.Error("Size of parsed tag should be 0. Actual tag size:", tag.Size())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	c "github.com/flynn/flynn/Godeps/_workspace/src/gopkg.in/check.v1"
	"github.com/flynn/flynn/pkg/attempt"
	"github.com/flynn/flynn/pkg/random"
)

func initApp(t *c.C, app string) string {
	dir := filepath.Join(t.MkDir(), "app")
	t.Assert(run(exec.Command("cp", "-r", filepath.Join("apps", app), dir)), Succeeds)
	t.Assert(git(dir, "init"), Succeeds)
	t.Assert(git(dir, "add", "."), Succeeds)
	t.Assert(git(dir, "commit", "-am", "init"), Succeeds)
	return dir
}

type appSuite struct {
	appDir string
}

func (s *appSuite) Flynn(args ...string) *CmdResult {
	return flynn(s.appDir, args...)
}

func (s *appSuite) Git(args ...string) *CmdResult {
	return git(s.appDir, args...)
}

type BasicSuite struct {
	appSuite
}

var _ = c.Suite(&BasicSuite{})

func (s *BasicSuite) SetUpSuite(t *c.C) {
	s.appDir = initApp(t, "basic")
}

var Attempts = attempt.Strategy{
	Min:   5,
	Total: 10 * time.Second,
	Delay: 500 * time.Millisecond,
}

func (s *BasicSuite) TestBasic(t *c.C) {
	name := random.String(30)
	t.Assert(s.Flynn("create", name), Outputs, fmt.Sprintf("Created %s\n", name))

	push := s.Git("push", "flynn", "master")
	t.Assert(push, OutputContains, "Node.js app detected")
	t.Assert(push, OutputContains, "Downloading and installing node")
	t.Assert(push, OutputContains, "Installing dependencies")
	t.Assert(push, OutputContains, "Procfile declares types -> web")
	t.Assert(push, OutputContains, "Creating release")
	t.Assert(push, OutputContains, "Application deployed")
	t.Assert(push, OutputContains, "* [new branch]      master -> master")

	t.Assert(s.Flynn("scale", "web=3"), Succeeds)

	route := random.String(32) + ".dev"
	newRoute := s.Flynn("route", "add", "-t", "http", route)
	t.Assert(newRoute, Succeeds)

	t.Assert(s.Flynn("route"), OutputContains, strings.TrimSpace(newRoute.Output))

	// use Attempts to give the processes time to start
	if err := Attempts.Run(func() error {
		ps := s.Flynn("ps")
		if ps.Err != nil {
			return ps.Err
		}
		psLines := strings.Split(strings.TrimSpace(ps.Output), "\n")
		if len(psLines) != 4 {
			return fmt.Errorf("Expected 4 ps lines, got %d", len(psLines))
		}

		for _, l := range psLines[1:] {
			idType := regexp.MustCompile(`\s+`).Split(l, 2)
			if idType[1] != "web" {
				return fmt.Errorf("Expected web type, got %s", idType[1])
			}
			log := s.Flynn("log", idType[0])
			if !strings.Contains(log.Output, "Listening on ") {
				return fmt.Errorf("Expected \"%s\" to contain \"Listening on \"", log.Output)
			}
		}
		return nil
	}); err != nil {
		t.Error(err)
	}

	// Make HTTP requests
	client := &http.Client{}
	req, err := http.NewRequest("GET", "http://"+routerIP, nil)
	if err != nil {
		t.Error(err)
	}
	req.Host = route
	res, err := client.Do(req)
	if err != nil {
		t.Error(err)
	}
	defer res.Body.Close()
	contents, err := ioutil.ReadAll(res.Body)
	if err != nil {
		t.Error(err)
	}
	t.Assert(res.StatusCode, c.Equals, 200)
	t.Assert(string(contents), Matches, `Hello to Yahoo from Flynn on port \d+`)
}
<commit_msg>test: Give the basic test more time to pass<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	c "github.com/flynn/flynn/Godeps/_workspace/src/gopkg.in/check.v1"
	"github.com/flynn/flynn/pkg/attempt"
	"github.com/flynn/flynn/pkg/random"
)

func initApp(t *c.C, app string) string {
	dir := filepath.Join(t.MkDir(), "app")
	t.Assert(run(exec.Command("cp", "-r", filepath.Join("apps", app), dir)), Succeeds)
	t.Assert(git(dir, "init"), Succeeds)
	t.Assert(git(dir, "add", "."), Succeeds)
	t.Assert(git(dir, "commit", "-am", "init"), Succeeds)
	return dir
}

type appSuite struct {
	appDir string
}

func (s *appSuite) Flynn(args ...string) *CmdResult {
	return flynn(s.appDir, args...)
}

func (s *appSuite) Git(args ...string) *CmdResult {
	return git(s.appDir, args...)
}

type BasicSuite struct {
	appSuite
}

var _ = c.Suite(&BasicSuite{})

func (s *BasicSuite) SetUpSuite(t *c.C) {
	s.appDir = initApp(t, "basic")
}

var Attempts = attempt.Strategy{
	Total: 20 * time.Second,
	Delay: 500 * time.Millisecond,
}

func (s *BasicSuite) TestBasic(t *c.C) {
	name := random.String(30)
	t.Assert(s.Flynn("create", name), Outputs, fmt.Sprintf("Created %s\n", name))

	push := s.Git("push", "flynn", "master")
	t.Assert(push, OutputContains, "Node.js app detected")
	t.Assert(push, OutputContains, "Downloading and installing node")
	t.Assert(push, OutputContains, "Installing dependencies")
	t.Assert(push, OutputContains, "Procfile declares types -> web")
	t.Assert(push, OutputContains, "Creating release")
	t.Assert(push, OutputContains, "Application deployed")
	t.Assert(push, OutputContains, "* [new branch]      master -> master")

	t.Assert(s.Flynn("scale", "web=3"), Succeeds)

	route := random.String(32) + ".dev"
	newRoute := s.Flynn("route", "add", "-t", "http", route)
	t.Assert(newRoute, Succeeds)

	t.Assert(s.Flynn("route"), OutputContains, strings.TrimSpace(newRoute.Output))

	// use Attempts to give the processes time to start
	if err := Attempts.Run(func() error {
		ps := s.Flynn("ps")
		if ps.Err != nil {
			return ps.Err
		}
		psLines := strings.Split(strings.TrimSpace(ps.Output), "\n")
		if len(psLines) != 4 {
			return fmt.Errorf("Expected 4 ps lines, got %d", len(psLines))
		}

		for _, l := range psLines[1:] {
			idType := regexp.MustCompile(`\s+`).Split(l, 2)
			if idType[1] != "web" {
				return fmt.Errorf("Expected web type, got %s", idType[1])
			}
			log := s.Flynn("log", idType[0])
			if !strings.Contains(log.Output, "Listening on ") {
				return fmt.Errorf("Expected \"%s\" to contain \"Listening on \"", log.Output)
			}
		}
		return nil
	}); err != nil {
		t.Error(err)
	}

	// Make HTTP requests
	client := &http.Client{}
	req, err := http.NewRequest("GET", "http://"+routerIP, nil)
	if err != nil {
		t.Error(err)
	}
	req.Host = route
	res, err := client.Do(req)
	if err != nil {
		t.Error(err)
	}
	defer res.Body.Close()
	contents, err := ioutil.ReadAll(res.Body)
	if err != nil {
		t.Error(err)
	}
	t.Assert(res.StatusCode, c.Equals, 200)
	t.Assert(string(contents), Matches, `Hello to Yahoo from Flynn on port \d+`)
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2014 The pblcache Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package cache

import (
	"fmt"
	"github.com/lpabon/tm"
	"github.com/pblcache/pblcache/src/message"
	"github.com/pblcache/pblcache/src/tests"
	"math/rand"
	"os"
	"sync"
	"testing"
	"time"
)

func TestNewLog(t *testing.T) {

	// Simple log
	testcachefile := tests.Tempfile()
	l, blocks := NewLog(testcachefile, 16, 4096, 4, 4096*2)
	tests.Assert(t, l != nil)
	tests.Assert(t, blocks == 16)
	l.Close()

	// Check the log correctly return maximum number of
	// blocks that are aligned to the segments.
	// 17 blocks are not aligned to a segment with 4 blocks
	// per segment
	l, blocks = NewLog(testcachefile, 17, 4096, 4, 4096*2)
	tests.Assert(t, l != nil)
	tests.Assert(t, blocks == 16)
	l.Close()

	// Cleanup
	os.Remove(testcachefile)
}

// Should wrap four times
func TestWrapPut(t *testing.T) {
	// Simple log
	blocks := uint64(16)
	testcachefile := tests.Tempfile()
	l, logblocks := NewLog(testcachefile, blocks, 4096, 2, 4096*2)
	tests.Assert(t, l != nil)
	tests.Assert(t, blocks == logblocks)

	here := make(chan *message.Message)
	wraps := uint64(4)

	// Write enough blocks to wrap around the log
	// as many times as determined by the value in 'wraps'
	for io := uint8(0); io < uint8(blocks*wraps); io++ {
		buf := make([]byte, 4096)
		buf[0] = byte(io)

		msg := message.NewMsgPut()
		msg.RetChan = here

		iopkt := msg.IoPkt()
		iopkt.Buffer = buf
		iopkt.BlockNum = uint64(io % uint8(blocks))

		l.Msgchan <- msg
		<-here
	}

	// Check that we have wrapped the correct number of times
	tests.Assert(t, l.stats.wraps == wraps)

	// Cleanup
	l.Close()
	os.Remove(testcachefile)
}

func TestReadCorrectness(t *testing.T) {
	// Simple log
	blocks := uint64(240)
	bs := uint64(4096)
	blocks_per_segment := uint64(2)
	buffercache := uint64(4096 * 10)
	testcachefile := tests.Tempfile()
	l, logblocks := NewLog(testcachefile,
		blocks,
		bs,
		blocks_per_segment,
		buffercache)
	tests.Assert(t, l != nil)
	tests.Assert(t, blocks == logblocks)

	here := make(chan *message.Message)

	// Write enough blocks in the log to reach
	// the end.
	for io := uint8(0); io < uint8(blocks); io++ {
		buf := make([]byte, 4096)

		// Save the block number in the buffer
		// so that we can check it later.  For simplicity
		// we have made sure the block number is only
		// one byte.
		buf[0] = byte(io)

		msg := message.NewMsgPut()
		msg.RetChan = here

		iopkt := msg.IoPkt()
		iopkt.Buffer = buf
		iopkt.BlockNum = uint64(io)

		l.Msgchan <- msg
		<-here
	}
	buf := make([]byte, 4096)
	msg := message.NewMsgGet()
	msg.RetChan = here

	iopkt := msg.IoPkt()
	iopkt.Buffer = buf
	iopkt.BlockNum = blocks - 1

	l.Msgchan <- msg
	<-here

	tests.Assert(t, buf[0] == uint8(blocks-1))

	for io := uint8(0); io < uint8(blocks); io++ {
		buf := make([]byte, 4096)
		msg := message.NewMsgGet()
		msg.RetChan = here

		iopkt := msg.IoPkt()
		iopkt.Buffer = buf
		iopkt.BlockNum = uint64(io)
		l.Msgchan <- msg

		// Wait here for the response
		<-here

		// Check the block number is correct
		tests.Assert(t, buf[0] == uint8(io))
	}

	l.Close()
	os.Remove(testcachefile)
}

func logtest_response_handler(
	t *testing.T,
	wg *sync.WaitGroup,
	quit chan struct{},
	m chan *message.Message) {

	var (
		gets, puts int
		tg, tp     tm.TimeDuration
	)

	defer wg.Done()

	emptychan := false
	for {

		// Check if we have been signaled through <-quit
		// If we have, we now know that as soon as the
		// message channel is empty, we can quit.
		if emptychan {
			if len(m) == 0 {
				break
			}
		}

		// Check incoming channels
		select {
		case msg := <-m:
			// Collect stats
			switch msg.Type {
			case message.MsgGet:
				gets++
				tg.Add(msg.TimeElapsed())
			case message.MsgPut:
				puts++
				tp.Add(msg.TimeElapsed())
			}

		case <-quit:
			emptychan = true
		}
	}
	fmt.Printf("Gets:%d, Puts:%d\n"+
		"Mean times in usecs: Gets:%.2f, Puts:%.2f\n",
		gets, puts, tg.MeanTimeUsecs(), tp.MeanTimeUsecs())
}

func TestLogConcurrency(t *testing.T) {
	// Simple log
	blocks := uint64(240)
	bs := uint64(4096)
	blocks_per_segment := uint64(2)
	buffercache := uint64(4096 * 24)
	testcachefile := tests.Tempfile()
	l, logblocks := NewLog(testcachefile,
		blocks,
		bs,
		blocks_per_segment,
		buffercache)
	tests.Assert(t, l != nil)
	tests.Assert(t, blocks == logblocks)

	here := make(chan *message.Message)

	// Fill the log
	for io := uint8(0); io < uint8(blocks); io++ {
		buf := make([]byte, 4096)
		buf[0] = byte(io)

		msg := message.NewMsgPut()
		msg.RetChan = here

		iopkt := msg.IoPkt()
		iopkt.Buffer = buf
		iopkt.BlockNum = uint64(io)

		l.Msgchan <- msg
		<-here
	}

	var wgIo, wgRet sync.WaitGroup

	// Start up response server
	returnch := make(chan *message.Message, 100)
	quit := make(chan struct{})
	wgRet.Add(1)
	go logtest_response_handler(t, &wgRet, quit, returnch)

	// Create 100 readers
	for i := 0; i < 100; i++ {
		wgIo.Add(1)
		go func() {
			defer wgIo.Done()
			r := rand.New(rand.NewSource(time.Now().UnixNano()))

			// Each client to send 1k IOs
			for io := 0; io < 1000; io++ {
				msg := message.NewMsgGet()
				iopkt := msg.IoPkt()
				iopkt.Buffer = make([]byte, bs)

				// Maximum "disk" size is 10 times bigger than cache
				iopkt.BlockNum = uint64(r.Int63n(int64(blocks)))
				msg.RetChan = returnch

				// Send request
				msg.TimeStart()
				l.Msgchan <- msg

				// Simulate waiting for more work by sleeping
				// anywhere from 100usecs to 10ms
				time.Sleep(time.Microsecond * time.Duration((r.Intn(10000) + 100)))
			}
		}()
	}

	// Write to the log while the readers are reading
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	for wrap := 0; wrap < 30; wrap++ {
		for io := uint8(0); io < uint8(blocks); io++ {
			buf := make([]byte, 4096)
			buf[0] = byte(io)

			msg := message.NewMsgPut()
			msg.RetChan = returnch

			iopkt := msg.IoPkt()
			iopkt.Buffer = buf
			iopkt.BlockNum = uint64(io)

			msg.TimeStart()
			l.Msgchan <- msg
			time.Sleep(time.Microsecond * time.Duration((r.Intn(1000) + 100)))
		}
	}

	// Wait for all clients to finish
	wgIo.Wait()

	// Send receiver a message that all clients have shut down
	close(quit)

	// Wait for receiver to finish emptying its channel
	wgRet.Wait()

	// Cleanup
	fmt.Print(l)
	l.Close()
	os.Remove(testcachefile)

}
<commit_msg>Race condition fixed accessing stats<commit_after>//
// Copyright (c) 2014 The pblcache Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package cache

import (
	"fmt"
	"github.com/lpabon/tm"
	"github.com/pblcache/pblcache/src/message"
	"github.com/pblcache/pblcache/src/tests"
	"math/rand"
	"os"
	"sync"
	"testing"
	"time"
)

func TestNewLog(t *testing.T) {

	// Simple log
	testcachefile := tests.Tempfile()
	l, blocks := NewLog(testcachefile, 16, 4096, 4, 4096*2)
	tests.Assert(t, l != nil)
	tests.Assert(t, blocks == 16)
	l.Close()

	// Check the log correctly return maximum number of
	// blocks that are aligned to the segments.
	// 17 blocks are not aligned to a segment with 4 blocks
	// per segment
	l, blocks = NewLog(testcachefile, 17, 4096, 4, 4096*2)
	tests.Assert(t, l != nil)
	tests.Assert(t, blocks == 16)
	l.Close()

	// Cleanup
	os.Remove(testcachefile)
}

// Should wrap four times
func TestWrapPut(t *testing.T) {
	// Simple log
	blocks := uint64(16)
	testcachefile := tests.Tempfile()
	l, logblocks := NewLog(testcachefile, blocks, 4096, 2, 4096*2)
	tests.Assert(t, l != nil)
	tests.Assert(t, blocks == logblocks)

	here := make(chan *message.Message)
	wraps := uint64(4)

	// Write enough blocks to wrap around the log
	// as many times as determined by the value in 'wraps'
	for io := uint8(0); io < uint8(blocks*wraps); io++ {
		buf := make([]byte, 4096)
		buf[0] = byte(io)

		msg := message.NewMsgPut()
		msg.RetChan = here

		iopkt := msg.IoPkt()
		iopkt.Buffer = buf
		iopkt.BlockNum = uint64(io % uint8(blocks))

		l.Msgchan <- msg
		<-here
	}

	// Close will also empty all the channels
	l.Close()

	// Check that we have wrapped the correct number of times
	tests.Assert(t, l.stats.wraps == wraps)

	// Cleanup
	os.Remove(testcachefile)
}

func TestReadCorrectness(t *testing.T) {
	// Simple log
	blocks := uint64(240)
	bs := uint64(4096)
	blocks_per_segment := uint64(2)
	buffercache := uint64(4096 * 10)
	testcachefile := tests.Tempfile()
	l, logblocks := NewLog(testcachefile,
		blocks,
		bs,
		blocks_per_segment,
		buffercache)
	tests.Assert(t, l != nil)
	tests.Assert(t, blocks == logblocks)

	here := make(chan *message.Message)

	// Write enough blocks in the log to reach
	// the end.
	for io := uint8(0); io < uint8(blocks); io++ {
		buf := make([]byte, 4096)

		// Save the block number in the buffer
		// so that we can check it later.  For simplicity
		// we have made sure the block number is only
		// one byte.
		buf[0] = byte(io)

		msg := message.NewMsgPut()
		msg.RetChan = here

		iopkt := msg.IoPkt()
		iopkt.Buffer = buf
		iopkt.BlockNum = uint64(io)

		l.Msgchan <- msg
		<-here
	}
	buf := make([]byte, 4096)
	msg := message.NewMsgGet()
	msg.RetChan = here

	iopkt := msg.IoPkt()
	iopkt.Buffer = buf
	iopkt.BlockNum = blocks - 1

	l.Msgchan <- msg
	<-here

	tests.Assert(t, buf[0] == uint8(blocks-1))

	for io := uint8(0); io < uint8(blocks); io++ {
		buf := make([]byte, 4096)
		msg := message.NewMsgGet()
		msg.RetChan = here

		iopkt := msg.IoPkt()
		iopkt.Buffer = buf
		iopkt.BlockNum = uint64(io)
		l.Msgchan <- msg

		// Wait here for the response
		<-here

		// Check the block number is correct
		tests.Assert(t, buf[0] == uint8(io))
	}

	l.Close()
	os.Remove(testcachefile)
}

func logtest_response_handler(
	t *testing.T,
	wg *sync.WaitGroup,
	quit chan struct{},
	m chan *message.Message) {

	var (
		gets, puts int
		tg, tp     tm.TimeDuration
	)

	defer wg.Done()

	emptychan := false
	for {

		// Check if we have been signaled through <-quit
		// If we have, we now know that as soon as the
		// message channel is empty, we can quit.
		if emptychan {
			if len(m) == 0 {
				break
			}
		}

		// Check incoming channels
		select {
		case msg := <-m:
			// Collect stats
			switch msg.Type {
			case message.MsgGet:
				gets++
				tg.Add(msg.TimeElapsed())
			case message.MsgPut:
				puts++
				tp.Add(msg.TimeElapsed())
			}

		case <-quit:
			emptychan = true
		}
	}
	fmt.Printf("Gets:%d, Puts:%d\n"+
		"Mean times in usecs: Gets:%.2f, Puts:%.2f\n",
		gets, puts, tg.MeanTimeUsecs(), tp.MeanTimeUsecs())
}

func TestLogConcurrency(t *testing.T) {
	// Simple log
	blocks := uint64(240)
	bs := uint64(4096)
	blocks_per_segment := uint64(2)
	buffercache := uint64(4096 * 24)
	testcachefile := tests.Tempfile()
	l, logblocks := NewLog(testcachefile,
		blocks,
		bs,
		blocks_per_segment,
		buffercache)
	tests.Assert(t, l != nil)
	tests.Assert(t, blocks == logblocks)

	here := make(chan *message.Message)

	// Fill the log
	for io := uint8(0); io < uint8(blocks); io++ {
		buf := make([]byte, 4096)
		buf[0] = byte(io)

		msg := message.NewMsgPut()
		msg.RetChan = here

		iopkt := msg.IoPkt()
		iopkt.Buffer = buf
		iopkt.BlockNum = uint64(io)

		l.Msgchan <- msg
		<-here
	}

	var wgIo, wgRet sync.WaitGroup

	// Start up response server
	returnch := make(chan *message.Message, 100)
	quit := make(chan struct{})
	wgRet.Add(1)
	go logtest_response_handler(t, &wgRet, quit, returnch)

	// Create 100 readers
	for i := 0; i < 100; i++ {
		wgIo.Add(1)
		go func() {
			defer wgIo.Done()
			r := rand.New(rand.NewSource(time.Now().UnixNano()))

			// Each client to send 1k IOs
			for io := 0; io < 1000; io++ {
				msg := message.NewMsgGet()
				iopkt := msg.IoPkt()
				iopkt.Buffer = make([]byte, bs)

				// Maximum "disk" size is 10 times bigger than cache
				iopkt.BlockNum = uint64(r.Int63n(int64(blocks)))
				msg.RetChan = returnch

				// Send request
				msg.TimeStart()
				l.Msgchan <- msg

				// Simulate waiting for more work by sleeping
				// anywhere from 100usecs to 10ms
				time.Sleep(time.Microsecond * time.Duration((r.Intn(10000) + 100)))
			}
		}()
	}

	// Write to the log while the readers are reading
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	for wrap := 0; wrap < 30; wrap++ {
		for io := uint8(0); io < uint8(blocks); io++ {
			buf := make([]byte, 4096)
			buf[0] = byte(io)

			msg := message.NewMsgPut()
			msg.RetChan = returnch

			iopkt := msg.IoPkt()
			iopkt.Buffer = buf
			iopkt.BlockNum = uint64(io)

			msg.TimeStart()
			l.Msgchan <- msg
			time.Sleep(time.Microsecond * time.Duration((r.Intn(1000) + 100)))
		}
	}

	// Wait for all clients to finish
	wgIo.Wait()

	// Send receiver a message that all clients have shut down
	close(quit)

	// Wait for receiver to finish emptying its channel
	wgRet.Wait()

	// Cleanup
	fmt.Print(l)
	l.Close()
	os.Remove(testcachefile)

}
<|endoftext|>