<commit_before>package main

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"errors"
	"flag"
	"fmt"
	"github.com/bmizerany/perks/quantile"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"runtime"
	"strings"
	"sync"
	"syscall"
	"time"
)

type Configuration struct {
	concurrentClients    int
	hosts                []string
	hostsString          string
	httpConnectTimeout   int
	httpKeepAlive        bool
	httpMethod           string
	httpPostData         []byte
	httpPostDataFilePath string
	httpReadTimeout      int
	httpWriteTimeout     int
	reportInterval       int64
	testDuration         int64
	urlString            string
	urls                 []string
	urlsFilePath         string
}

type Result struct {
	host             string
	url              string
	start            time.Time
	duration         time.Duration
	httpStatus       int
	responseBodySize int64
	requestBodySize  int64
	networkFailed    bool
	error            error
}

type Metrics struct {
	startTime         time.Time
	total             int64
	success           int64
	httpFailed        int64
	networkFailed     int64
	totalResponseSize int64
	totalRequestSize  int64
	quantiles         *quantile.Stream
}

func NewMetrics() *Metrics {
	return &Metrics{
		startTime:         time.Now(),
		total:             0,
		success:           0,
		httpFailed:        0,
		networkFailed:     0,
		totalResponseSize: 0,
		totalRequestSize:  0,
		quantiles:         quantile.NewTargeted(0.50, 0.95, 0.99),
	}
}

func (this *Metrics) AddResult(result *Result) {
	this.total++

	if result.networkFailed {
		this.networkFailed++
	} else if result.httpStatus != http.StatusOK {
		this.httpFailed++
	} else {
		this.totalResponseSize += result.responseBodySize
		this.totalRequestSize += result.requestBodySize
		this.success++
	}

	this.quantiles.Insert(float64(result.duration.Nanoseconds() / int64(time.Microsecond)))
}

func (this *Metrics) P50() float64 {
	return this.quantiles.Query(0.50)
}

func (this *Metrics) P95() float64 {
	return this.quantiles.Query(0.95)
}

func (this *Metrics) P99() float64 {
	return this.quantiles.Query(0.99)
}

func (this *Metrics) ResetQuantiles() {
	this.quantiles.Reset()
}

func metricsUpdater(config *Configuration, metrics *Metrics, quit chan bool, results chan Result) {
	tick := time.NewTicker(time.Duration(config.reportInterval) * time.Second)

	for {
		select {
		case <-quit:
			printStatus(config, metrics)
			break
		case <-tick.C:
			printStatus(config, metrics)
			// metrics.ResetQuantiles()
		case r := <-results:
			metrics.AddResult(&r)
		}
	}
}

func (this *Result) Reset() {
	this.host = ""
	this.url = ""
	this.start = time.Unix(0, 0)
	this.duration = 0
	this.httpStatus = 0
	this.responseBodySize = 0
	this.requestBodySize = 0
	this.networkFailed = false
	this.error = nil
}

type ConnectionWrapper struct {
	net.Conn
	readTimeout  time.Duration
	writeTimeout time.Duration
	result       *Result
}

func (this *ConnectionWrapper) Read(b []byte) (n int, err error) {
	len, err := this.Conn.Read(b)

	if err == nil {
		this.result.responseBodySize += int64(len)
		this.Conn.SetReadDeadline(time.Now().Add(this.readTimeout))
	}

	return len, err
}

func (this *ConnectionWrapper) Write(b []byte) (n int, err error) {
	len, err := this.Conn.Write(b)

	if err == nil {
		this.result.requestBodySize += int64(len)
		this.Conn.SetWriteDeadline(time.Now().Add(this.writeTimeout))
	}

	return len, err
}

func NewConfiguration() *Configuration {
	return &Configuration{
		concurrentClients:    10,
		hosts:                make([]string, 0),
		httpConnectTimeout:   5000,
		httpKeepAlive:        true,
		httpMethod:           "GET",
		httpPostData:         make([]byte, 0),
		httpPostDataFilePath: "",
		httpReadTimeout:      5000,
		httpWriteTimeout:     5000,
		reportInterval:       5,
		testDuration:         60000,
		urlString:            "",
		urls:                 make([]string, 0),
		urlsFilePath:         ""}
}

func parseArgs(config *Configuration) {
	flag.Int64Var(&config.reportInterval, "r", 5, "How often to report statistics in seconds")
	flag.IntVar(&config.concurrentClients, "c", 10, "Number of concurrent clients")
	flag.StringVar(&config.hostsString, "h", "", "Optional comma delimited list of HTTP hosts which will be prepended to URLs")
	flag.StringVar(&config.urlsFilePath, "f", "", "URLs file path (line seperated)")
	flag.StringVar(&config.urlString, "u", "", "URL to use for requests in lieu of an input file")
	flag.BoolVar(&config.httpKeepAlive, "k", true, "Do HTTP keep-alive")
	flag.StringVar(&config.httpPostDataFilePath, "d", "", "HTTP POST data file path")
	flag.Int64Var(&config.testDuration, "t", -1, "Test duration")
	flag.IntVar(&config.httpConnectTimeout, "tc", 5000, "Connect timeout (in milliseconds)")
	flag.IntVar(&config.httpWriteTimeout, "tw", 5000, "Write timeout (in milliseconds)")
	flag.IntVar(&config.httpReadTimeout, "tr", 5000, "Read timeout (in milliseconds)")
	flag.Parse()

	config.testDuration = config.testDuration * time.Second.Nanoseconds()

	if config.hostsString != "" {
		config.hosts = strings.Split(config.hostsString, ",")
	}

	if config.urlsFilePath != "" {
		fileLines, err := readLines(config.urlsFilePath)

		if err != nil {
			log.Fatalf("Error in ioutil.ReadFile for file: %s Error: ", config.urlsFilePath, err)
		}

		config.urls = fileLines
	}

	if config.urlString != "" {
		config.urls = append(config.urls, config.urlString)
	}

	if config.httpPostDataFilePath != "" {
		config.httpMethod = "POST"
		data, err := ioutil.ReadFile(config.httpPostDataFilePath)
		if err != nil {
			log.Fatalf("Error reading post data file: %s", err)
		}

		config.httpPostData = data
	}
}

func checkConfiguration(config *Configuration) error {
	if len(config.urls) == 0 {
		return errors.New("A URL was not specified via the commandline, nor was an input file provided")
	}

	if config.testDuration == -1 {
		return errors.New("Required param test duration was not provided")
	}

	return nil
}

func readLines(path string) (lines []string, err error) {

	var file *os.File
	var part []byte
	var prefix bool

	if file, err = os.Open(path); err != nil {
		return
	}
	defer file.Close()

	reader := bufio.NewReader(file)
	buffer := bytes.NewBuffer(make([]byte, 0))
	for {
		if part, prefix, err = reader.ReadLine(); err != nil {
			break
		}
		buffer.Write(part)
		if !prefix {
			lines = append(lines, buffer.String())
			buffer.Reset()
		}
	}

	if err == io.EOF {
		err = nil
	}
	return
}

func random(min, max int) int {
	rand.Seed(time.Now().Unix())
	return rand.Intn(max-min) + min
}

func HTTPClient(result *Result, connectTimeout, readTimeout, writeTimeout time.Duration) *http.Client {

	return &http.Client{
		Transport: &http.Transport{
			Dial:            TimeoutDialer(result, connectTimeout, readTimeout, writeTimeout),
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		},
	}
}

func TimeoutDialer(result *Result, connectTimeout, readTimeout, writeTimeout time.Duration) func(net, address string) (conn net.Conn, err error) {
	return func(mynet, address string) (net.Conn, error) {
		conn, err := net.DialTimeout(mynet, address, connectTimeout)
		if err != nil {
			return nil, err
		}

		conn.SetReadDeadline(time.Now().Add(readTimeout))
		conn.SetWriteDeadline(time.Now().Add(writeTimeout))

		return &ConnectionWrapper{Conn: conn, readTimeout: readTimeout, writeTimeout: writeTimeout, result: result}, nil
	}
}

func client(config *Configuration, quit chan bool, results chan Result, done *sync.WaitGroup) {
	defer func() {
		if r := recover(); r != nil {
			fmt.Println("caught recover: ", r)
			os.Exit(1)
		}
	}()

	defer done.Done()

	var result Result
	httpclient := HTTPClient(&result, time.Duration(config.httpConnectTimeout)*time.Millisecond,
		time.Duration(config.httpReadTimeout)*time.Millisecond,
		time.Duration(config.httpWriteTimeout)*time.Millisecond)

	hostsLength := len(config.hosts)
	urlsLength := len(config.urls)

	for {
		if len(quit) > 0 {
			break
		}
		tmpUrl := config.urls[random(0, urlsLength)]
		result.Reset()

		if hostsLength > 0 {
			result.host = config.hosts[random(0, hostsLength)]
			tmpUrl = result.host + tmpUrl
		}
		req, _ := http.NewRequest(config.httpMethod, tmpUrl, bytes.NewReader(config.httpPostData))
		result.url = tmpUrl
		result.host = req.Host

		if config.httpKeepAlive == true {
			req.Header.Add("Connection", "keep-alive")
		} else {
			req.Header.Add("Connection", "close")
		}

		startTime := time.Now()
		resp, err := httpclient.Do(req)
		result.duration = time.Now().Sub(startTime)
		result.start = startTime

		if err != nil {
			result.networkFailed = true
			result.error = err
			results <- result
			continue
		}

		_, errRead := ioutil.ReadAll(resp.Body)

		if errRead != nil {
			result.networkFailed = true
			result.error = errRead
			results <- result
			continue
		}

		result.httpStatus = resp.StatusCode
		resp.Body.Close()
		results <- result
	}
}

func printStatus(config *Configuration, metrics *Metrics) {
	successRate := 0.0
	httpFailRate := 0.0
	netFailRate := 0.0
	sendThroughput := 0.0
	receiveThroughput := 0.0

	if metrics.total != 0 {
		successRate = float64(metrics.success / metrics.total)
		httpFailRate = float64(metrics.httpFailed / metrics.total)
		netFailRate = float64(metrics.networkFailed / metrics.total)
		sendThroughput = float64(metrics.totalResponseSize/config.reportInterval) / 1024.0
		receiveThroughput = float64(metrics.totalResponseSize/config.reportInterval) / 1024.0
	}

	now := time.Now()

	fmt.Printf("%s | seconds : %08.0f | total : %010d | succes : %1.2f | http failure : %1.2f | network failure : %1.2f | receive : %4.2f kB/s | send : %4.2f kB/s | P50 : %2.2f µs | P95 : %2.2f µs | P99 : %2.2f µs\n",
		now.Format("02 Jan 2006 15:04:05"),
		now.Sub(metrics.startTime).Seconds(),
		metrics.total,
		successRate,
		httpFailRate,
		netFailRate,
		receiveThroughput,
		sendThroughput,
		metrics.P50(),
		metrics.P95(),
		metrics.P99())
}

func main() {
	config := NewConfiguration()
	parseArgs(config)
	err := checkConfiguration(config)
	if err != nil {
		log.Println(err)
		flag.Usage()
		os.Exit(1)
	}

	startTime := time.Now()
	metrics := NewMetrics()
	quit := make(chan bool)

	// start the profiler
	// http.ListenAndServe("localhost:6060", nil)

	signalChannel := make(chan os.Signal, 2)
	signal.Notify(signalChannel, os.Interrupt, syscall.SIGTERM)
	go func() {
		_ = <-signalChannel
		quit <- true
		printStatus(config, metrics)
		os.Exit(0)
	}()

	// set the max number of process
	goMaxProcs := os.Getenv("GOMAXPROCS")
	if goMaxProcs == "" {
		runtime.GOMAXPROCS(runtime.NumCPU())
	}

	var clientGroup sync.WaitGroup

	// spawn the metric updater
	results := make(chan Result)
	go metricsUpdater(config, metrics, quit, results)

	// spawn the clients
	fmt.Printf("Dispatching %d clients\n", config.concurrentClients)
	clientGroup.Add(config.concurrentClients)
	for i := 0; i < config.concurrentClients; i++ {
		go client(config, quit, results, &clientGroup)
	}

	// wait for test duration
	for {
		if time.Now().Sub(startTime).Nanoseconds() > config.testDuration {
			quit <- true
			break
		}
		time.Sleep(time.Second)
	}

	clientGroup.Wait()
}
<commit_msg>fixed some of the reporting and quit behavior<commit_after>package main

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"errors"
	"flag"
	"fmt"
	"github.com/bmizerany/perks/quantile"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"runtime"
	"strings"
	"sync"
	"syscall"
	"time"
)

type Configuration struct {
	concurrentClients    int
	hosts                []string
	hostsString          string
	httpConnectTimeout   int
	httpKeepAlive        bool
	httpMethod           string
	httpPostData         []byte
	httpPostDataFilePath string
	httpReadTimeout      int
	httpWriteTimeout     int
	reportInterval       int64
	testDuration         int64
	urlString            string
	urls                 []string
	urlsFilePath         string
}

type Result struct {
	host             string
	url              string
	start            time.Time
	duration         time.Duration
	httpStatus       int
	responseBodySize int64
	requestBodySize  int64
	networkFailed    bool
	error            error
}

type Metrics struct {
	startTime         time.Time
	total             int64
	success           int64
	httpFailed        int64
	networkFailed     int64
	totalResponseSize int64
	totalRequestSize  int64
	quantiles         *quantile.Stream
}

func NewMetrics() *Metrics {
	return &Metrics{
		startTime:         time.Now(),
		total:             0,
		success:           0,
		httpFailed:        0,
		networkFailed:     0,
		totalResponseSize: 0,
		totalRequestSize:  0,
		quantiles:         quantile.NewTargeted(0.50, 0.95, 0.99),
	}
}

func (this *Metrics) AddResult(result *Result) {
	this.total++

	if result.networkFailed {
		this.networkFailed++
	} else if result.httpStatus != http.StatusOK {
		this.httpFailed++
	} else {
		this.totalResponseSize += result.responseBodySize
		this.totalRequestSize += result.requestBodySize
		this.success++
	}

	this.quantiles.Insert(float64(result.duration.Nanoseconds() / int64(time.Microsecond)))
}

func (this *Metrics) P50() float64 {
	return this.quantiles.Query(0.50)
}

func (this *Metrics) P95() float64 {
	return this.quantiles.Query(0.95)
}

func (this *Metrics) P99() float64 {
	return this.quantiles.Query(0.99)
}

func (this *Metrics) ResetQuantiles() {
	this.quantiles.Reset()
}

func metricsUpdater(config *Configuration, metrics *Metrics, quit chan bool, results chan Result) {
	tick := time.NewTicker(time.Duration(config.reportInterval) * time.Second)

	for {
		select {
		case <-tick.C:
			printStatus(config, metrics)
			metrics.ResetQuantiles()
		case <-quit:
			return
		case r := <-results:
			metrics.AddResult(&r)
		}
	}
}

func (this *Result) Reset() {
	this.host = ""
	this.url = ""
	this.start = time.Unix(0, 0)
	this.duration = 0
	this.httpStatus = 0
	this.responseBodySize = 0
	this.requestBodySize = 0
	this.networkFailed = false
	this.error = nil
}

type ConnectionWrapper struct {
	net.Conn
	readTimeout  time.Duration
	writeTimeout time.Duration
	result       *Result
}

func (this *ConnectionWrapper) Read(b []byte) (n int, err error) {
	len, err := this.Conn.Read(b)

	if err == nil {
		this.result.responseBodySize += int64(len)
		this.Conn.SetReadDeadline(time.Now().Add(this.readTimeout))
	}

	return len, err
}

func (this *ConnectionWrapper) Write(b []byte) (n int, err error) {
	len, err := this.Conn.Write(b)

	if err == nil {
		this.result.requestBodySize += int64(len)
		this.Conn.SetWriteDeadline(time.Now().Add(this.writeTimeout))
	}

	return len, err
}

func NewConfiguration() *Configuration {
	return &Configuration{
		concurrentClients:    10,
		hosts:                make([]string, 0),
		httpConnectTimeout:   5000,
		httpKeepAlive:        true,
		httpMethod:           "GET",
		httpPostData:         make([]byte, 0),
		httpPostDataFilePath: "",
		httpReadTimeout:      5000,
		httpWriteTimeout:     5000,
		reportInterval:       5,
		testDuration:         60000,
		urlString:            "",
		urls:                 make([]string, 0),
		urlsFilePath:         ""}
}

func parseArgs(config *Configuration) {
	flag.Int64Var(&config.reportInterval, "r", 5, "How often to report statistics in seconds")
	flag.IntVar(&config.concurrentClients, "c", 10, "Number of concurrent clients")
	flag.StringVar(&config.hostsString, "h", "", "Optional comma delimited list of HTTP hosts which will be prepended to URLs")
	flag.StringVar(&config.urlsFilePath, "f", "", "URLs file path (line seperated)")
	flag.StringVar(&config.urlString, "u", "", "URL to use for requests in lieu of an input file")
	flag.BoolVar(&config.httpKeepAlive, "k", true, "Do HTTP keep-alive")
	flag.StringVar(&config.httpPostDataFilePath, "d", "", "HTTP POST data file path")
	flag.Int64Var(&config.testDuration, "t", -1, "Test duration")
	flag.IntVar(&config.httpConnectTimeout, "tc", 5000, "Connect timeout (in milliseconds)")
	flag.IntVar(&config.httpWriteTimeout, "tw", 5000, "Write timeout (in milliseconds)")
	flag.IntVar(&config.httpReadTimeout, "tr", 5000, "Read timeout (in milliseconds)")
	flag.Parse()

	config.testDuration = config.testDuration * time.Second.Nanoseconds()

	if config.hostsString != "" {
		config.hosts = strings.Split(config.hostsString, ",")
	}

	if config.urlsFilePath != "" {
		fileLines, err := readLines(config.urlsFilePath)

		if err != nil {
			log.Fatalf("Error in ioutil.ReadFile for file: %s Error: ", config.urlsFilePath, err)
		}

		config.urls = fileLines
	}

	if config.urlString != "" {
		config.urls = append(config.urls, config.urlString)
	}

	if config.httpPostDataFilePath != "" {
		config.httpMethod = "POST"
		data, err := ioutil.ReadFile(config.httpPostDataFilePath)
		if err != nil {
			log.Fatalf("Error reading post data file: %s", err)
		}

		config.httpPostData = data
	}
}

func checkConfiguration(config *Configuration) error {
	if len(config.urls) == 0 {
		return errors.New("A URL was not specified via the commandline, nor was an input file provided")
	}

	if config.testDuration == -1 {
		return errors.New("Required param test duration was not provided")
	}

	return nil
}

func readLines(path string) (lines []string, err error) {
	var file *os.File
	var part []byte
	var prefix bool

	if file, err = os.Open(path); err != nil {
		return
	}
	defer file.Close()

	reader := bufio.NewReader(file)
	buffer := bytes.NewBuffer(make([]byte, 0))
	for {
		if part, prefix, err = reader.ReadLine(); err != nil {
			break
		}
		buffer.Write(part)
		if !prefix {
			lines = append(lines, buffer.String())
			buffer.Reset()
		}
	}

	if err == io.EOF {
		err = nil
	}
	return
}

func random(min, max int) int {
	rand.Seed(time.Now().Unix())
	return rand.Intn(max-min) + min
}

func HTTPClient(result *Result, connectTimeout, readTimeout, writeTimeout time.Duration) *http.Client {

	return &http.Client{
		Transport: &http.Transport{
			Dial:            TimeoutDialer(result, connectTimeout, readTimeout, writeTimeout),
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		},
	}
}

func TimeoutDialer(result *Result, connectTimeout, readTimeout, writeTimeout time.Duration) func(net, address string) (conn net.Conn, err error) {
	return func(mynet, address string) (net.Conn, error) {
		conn, err := net.DialTimeout(mynet, address, connectTimeout)
		if err != nil {
			return nil, err
		}

		conn.SetReadDeadline(time.Now().Add(readTimeout))
		conn.SetWriteDeadline(time.Now().Add(writeTimeout))

		return &ConnectionWrapper{Conn: conn, readTimeout: readTimeout, writeTimeout: writeTimeout, result: result}, nil
	}
}

func client(config *Configuration, quit chan bool, results chan Result, done *sync.WaitGroup) {
	defer func() {
		if r := recover(); r != nil {
			fmt.Println("caught recover: ", r)
			os.Exit(1)
		}
	}()

	defer func() {
		done.Done()
	}()

	var result Result
	httpclient := HTTPClient(&result, time.Duration(config.httpConnectTimeout)*time.Millisecond,
		time.Duration(config.httpReadTimeout)*time.Millisecond,
		time.Duration(config.httpWriteTimeout)*time.Millisecond)

	hostsLength := len(config.hosts)
	urlsLength := len(config.urls)

	for {
		select {
		case <-quit:
			return
		default:
		}

		tmpUrl := config.urls[random(0, urlsLength)]
		result.Reset()

		if hostsLength > 0 {
			result.host = config.hosts[random(0, hostsLength)]
			tmpUrl = result.host + tmpUrl
		}
		req, _ := http.NewRequest(config.httpMethod, tmpUrl, bytes.NewReader(config.httpPostData))
		result.url = tmpUrl
		result.host = req.Host

		if config.httpKeepAlive == true {
			req.Header.Add("Connection", "keep-alive")
		} else {
			req.Header.Add("Connection", "close")
		}

		startTime := time.Now()
		resp, err := httpclient.Do(req)
		result.duration = time.Now().Sub(startTime)
		result.start = startTime

		if err != nil {
			result.networkFailed = true
			result.error = err
			results <- result
			continue
		}

		_, errRead := ioutil.ReadAll(resp.Body)

		if errRead != nil {
			result.networkFailed = true
			result.error = errRead
			results <- result
			continue
		}

		result.httpStatus = resp.StatusCode
		resp.Body.Close()
		results <- result
	}
}

func printStatus(config *Configuration, metrics *Metrics) {
	successRate := 0.0
	httpFailRate := 0.0
	netFailRate := 0.0
	sendThroughput := 0.0
	receiveThroughput := 0.0

	if metrics.total != 0 {
		successRate = float64(metrics.success / metrics.total)
		httpFailRate = float64(metrics.httpFailed / metrics.total)
		netFailRate = float64(metrics.networkFailed / metrics.total)
		sendThroughput = float64(metrics.totalResponseSize/config.reportInterval) / 1024.0
		receiveThroughput = float64(metrics.totalResponseSize/config.reportInterval) / 1024.0
	}

	now := time.Now()

	fmt.Printf("%s | seconds : %08.0f | total : %010d | succes : %1.2f | http failure : %1.2f | network failure : %1.2f | receive : %4.2f kB/s | send : %4.2f kB/s | P50 : %2.2f µs | P95 : %2.2f µs | P99 : %2.2f µs\n",
		now.Format("02 Jan 2006 15:04:05"),
		now.Sub(metrics.startTime).Seconds(),
		metrics.total,
		successRate,
		httpFailRate,
		netFailRate,
		receiveThroughput,
		sendThroughput,
		metrics.P50(),
		metrics.P95(),
		metrics.P99())
}

func main() {
	config := NewConfiguration()
	parseArgs(config)
	err := checkConfiguration(config)
	if err != nil {
		log.Println(err)
		flag.Usage()
		os.Exit(1)
	}

	startTime := time.Now()
	metrics := NewMetrics()
	quit := make(chan bool)

	// start the profiler
	// http.ListenAndServe("localhost:6060", nil)

	signalChannel := make(chan os.Signal, 2)
	signal.Notify(signalChannel, os.Interrupt, syscall.SIGTERM)
	go func() {
		_ = <-signalChannel
		os.Exit(0)
	}()

	// set the max number of process
	goMaxProcs := os.Getenv("GOMAXPROCS")
	if goMaxProcs == "" {
		runtime.GOMAXPROCS(runtime.NumCPU())
	}

	var clientGroup sync.WaitGroup

	// spawn the metric updater
	results := make(chan Result, 100000)
	go metricsUpdater(config, metrics, quit, results)

	// spawn the clients
	clientGroup.Add(config.concurrentClients)
	for i := 0; i < config.concurrentClients; i++ {
		go client(config, quit, results, &clientGroup)
	}

	// wait for test duration
	for {
		if time.Now().Sub(startTime).Nanoseconds() > config.testDuration {
			close(quit)
			break
		}
		time.Sleep(1)
	}

	clientGroup.Wait()
	printStatus(config, metrics)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fstesting

import (
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/user"
	"strconv"
	"strings"
	"time"

	"github.com/googlecloudplatform/gcsfuse/fs"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	"github.com/jacobsa/oglematchers"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

// A struct that can be embedded to inherit common file system test behaviors.
type fsTest struct {
	ctx    context.Context
	clock  timeutil.Clock
	bucket gcs.Bucket
	mfs    *fuse.MountedFileSystem

	// Files to close when tearing down. Nil entries are skipped.
	toClose []io.Closer
}

var _ fsTestInterface = &fsTest{}

func (t *fsTest) setUpFsTest(deps FSTestDeps) {
	t.ctx = context.Background()
	t.clock = deps.Clock
	t.bucket = deps.Bucket

	// Set up a temporary directory for mounting.
	mountPoint, err := ioutil.TempDir("", "fs_test")
	if err != nil {
		panic("ioutil.TempDir: " + err.Error())
	}

	// Mount a file system.
	server, err := fs.NewServer(t.clock, t.bucket)
	if err != nil {
		panic("NewServer: " + err.Error())
	}

	t.mfs, err = fuse.Mount(mountPoint, server, &fuse.MountConfig{})
	if err != nil {
		panic("Mount: " + err.Error())
	}
}

func (t *fsTest) tearDownFsTest() {
	// Close any files we opened.
	for _, c := range t.toClose {
		if c == nil {
			continue
		}

		err := c.Close()
		if err != nil {
			panic(err)
		}
	}

	// Unmount the file system. Try again on "resource busy" errors.
	delay := 10 * time.Millisecond
	for {
		err := t.mfs.Unmount()
		if err == nil {
			break
		}

		if strings.Contains(err.Error(), "resource busy") {
			log.Println("Resource busy error while unmounting; trying again")
			time.Sleep(delay)
			delay = time.Duration(1.3 * float64(delay))
			continue
		}

		panic("MountedFileSystem.Unmount: " + err.Error())
	}

	if err := t.mfs.Join(t.ctx); err != nil {
		panic("MountedFileSystem.Join: " + err.Error())
	}
}

func (t *fsTest) createWithContents(name string, contents string) error {
	return t.createObjects(
		[]*gcsutil.ObjectInfo{
			&gcsutil.ObjectInfo{
				Attrs: storage.ObjectAttrs{
					Name: name,
				},
				Contents: contents,
			},
		})
}

func (t *fsTest) createObjects(objects []*gcsutil.ObjectInfo) error {
	_, err := gcsutil.CreateObjects(t.ctx, t.bucket, objects)
	return err
}

func (t *fsTest) createEmptyObjects(names []string) error {
	_, err := gcsutil.CreateEmptyObjects(t.ctx, t.bucket, names)
	return err
}

// Ensure that the clock will report a different time after returning.
func (t *fsTest) advanceTime() {
	// For simulated clocks, we can just advance the time.
	if c, ok := t.clock.(*timeutil.SimulatedClock); ok {
		c.AdvanceTime(time.Second)
		return
	}

	// Otherwise, sleep a moment.
	time.Sleep(time.Millisecond)
}

// Return a matcher that matches event times as reported by the bucket
// corresponding to the supplied start time as measured by the test.
func (t *fsTest) matchesStartTime(start time.Time) oglematchers.Matcher {
	// For simulated clocks we can use exact equality.
	if _, ok := t.clock.(*timeutil.SimulatedClock); ok {
		return timeutil.TimeEq(start)
	}

	// Otherwise, we need to take into account latency between the start of our
	// call and the time the server actually executed the operation.
	const slop = 60 * time.Second
	return timeutil.TimeNear(start, slop)
}

////////////////////////////////////////////////////////////////////////
// Common helpers
////////////////////////////////////////////////////////////////////////

func getFileNames(entries []os.FileInfo) (names []string) {
	for _, e := range entries {
		names = append(names, e.Name())
	}

	return
}

// REQUIRES: n % 4 == 0
func randString(n int) string {
	bytes := make([]byte, n)
	for i := 0; i < n; i += 4 {
		u32 := rand.Uint32()
		bytes[i] = byte(u32 >> 0)
		bytes[i+1] = byte(u32 >> 8)
		bytes[i+2] = byte(u32 >> 16)
		bytes[i+3] = byte(u32 >> 24)
	}

	return string(bytes)
}

func readRange(r io.ReadSeeker, offset int64, n int) (s string, err error) {
	if _, err = r.Seek(offset, 0); err != nil {
		return
	}

	bytes := make([]byte, n)
	if _, err = io.ReadFull(r, bytes); err != nil {
		return
	}

	s = string(bytes)
	return
}

func currentUid() uint32 {
	user, err := user.Current()
	if err != nil {
		panic(err)
	}

	uid, err := strconv.ParseUint(user.Uid, 10, 32)
	if err != nil {
		panic(err)
	}

	return uint32(uid)
}

func currentGid() uint32 {
	user, err := user.Current()
	if err != nil {
		panic(err)
	}

	gid, err := strconv.ParseUint(user.Gid, 10, 32)
	if err != nil {
		panic(err)
	}

	return uint32(gid)
}
<commit_msg>Use the new fuse.Unmount function.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fstesting

import (
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/user"
	"strconv"
	"strings"
	"time"

	"github.com/googlecloudplatform/gcsfuse/fs"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	"github.com/jacobsa/oglematchers"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

// A struct that can be embedded to inherit common file system test behaviors.
type fsTest struct {
	ctx    context.Context
	clock  timeutil.Clock
	bucket gcs.Bucket
	mfs    *fuse.MountedFileSystem

	// Files to close when tearing down. Nil entries are skipped.
	toClose []io.Closer
}

var _ fsTestInterface = &fsTest{}

func (t *fsTest) setUpFsTest(deps FSTestDeps) {
	t.ctx = context.Background()
	t.clock = deps.Clock
	t.bucket = deps.Bucket

	// Set up a temporary directory for mounting.
	mountPoint, err := ioutil.TempDir("", "fs_test")
	if err != nil {
		panic("ioutil.TempDir: " + err.Error())
	}

	// Mount a file system.
	server, err := fs.NewServer(t.clock, t.bucket)
	if err != nil {
		panic("NewServer: " + err.Error())
	}

	t.mfs, err = fuse.Mount(mountPoint, server, &fuse.MountConfig{})
	if err != nil {
		panic("Mount: " + err.Error())
	}
}

func (t *fsTest) tearDownFsTest() {
	// Close any files we opened.
	for _, c := range t.toClose {
		if c == nil {
			continue
		}

		err := c.Close()
		if err != nil {
			panic(err)
		}
	}

	// Unmount the file system. Try again on "resource busy" errors.
	delay := 10 * time.Millisecond
	for {
		err := fuse.Unmount(t.mfs.Dir())
		if err == nil {
			break
		}

		if strings.Contains(err.Error(), "resource busy") {
			log.Println("Resource busy error while unmounting; trying again")
			time.Sleep(delay)
			delay = time.Duration(1.3 * float64(delay))
			continue
		}

		panic("MountedFileSystem.Unmount: " + err.Error())
	}

	if err := t.mfs.Join(t.ctx); err != nil {
		panic("MountedFileSystem.Join: " + err.Error())
	}
}

func (t *fsTest) createWithContents(name string, contents string) error {
	return t.createObjects(
		[]*gcsutil.ObjectInfo{
			&gcsutil.ObjectInfo{
				Attrs: storage.ObjectAttrs{
					Name: name,
				},
				Contents: contents,
			},
		})
}

func (t *fsTest) createObjects(objects []*gcsutil.ObjectInfo) error {
	_, err := gcsutil.CreateObjects(t.ctx, t.bucket, objects)
	return err
}

func (t *fsTest) createEmptyObjects(names []string) error {
	_, err := gcsutil.CreateEmptyObjects(t.ctx, t.bucket, names)
	return err
}

// Ensure that the clock will report a different time after returning.
func (t *fsTest) advanceTime() {
	// For simulated clocks, we can just advance the time.
	if c, ok := t.clock.(*timeutil.SimulatedClock); ok {
		c.AdvanceTime(time.Second)
		return
	}

	// Otherwise, sleep a moment.
	time.Sleep(time.Millisecond)
}

// Return a matcher that matches event times as reported by the bucket
// corresponding to the supplied start time as measured by the test.
func (t *fsTest) matchesStartTime(start time.Time) oglematchers.Matcher {
	// For simulated clocks we can use exact equality.
	if _, ok := t.clock.(*timeutil.SimulatedClock); ok {
		return timeutil.TimeEq(start)
	}

	// Otherwise, we need to take into account latency between the start of our
	// call and the time the server actually executed the operation.
	const slop = 60 * time.Second
	return timeutil.TimeNear(start, slop)
}

////////////////////////////////////////////////////////////////////////
// Common helpers
////////////////////////////////////////////////////////////////////////

func getFileNames(entries []os.FileInfo) (names []string) {
	for _, e := range entries {
		names = append(names, e.Name())
	}

	return
}

// REQUIRES: n % 4 == 0
func randString(n int) string {
	bytes := make([]byte, n)
	for i := 0; i < n; i += 4 {
		u32 := rand.Uint32()
		bytes[i] = byte(u32 >> 0)
		bytes[i+1] = byte(u32 >> 8)
		bytes[i+2] = byte(u32 >> 16)
		bytes[i+3] = byte(u32 >> 24)
	}

	return string(bytes)
}

func readRange(r io.ReadSeeker, offset int64, n int) (s string, err error) {
	if _, err = r.Seek(offset, 0); err != nil {
		return
	}

	bytes := make([]byte, n)
	if _, err = io.ReadFull(r, bytes); err != nil {
		return
	}

	s = string(bytes)
	return
}

func currentUid() uint32 {
	user, err := user.Current()
	if err != nil {
		panic(err)
	}

	uid, err := strconv.ParseUint(user.Uid, 10, 32)
	if err != nil {
		panic(err)
	}

	return uint32(uid)
}

func currentGid() uint32 {
	user, err := user.Current()
	if err != nil {
		panic(err)
	}

	gid, err := strconv.ParseUint(user.Gid, 10, 32)
	if err != nil {
		panic(err)
	}

	return uint32(gid)
}
<|endoftext|>


<commit_before>package app

import (
	"github.com/prometheus/client_golang/prometheus"
)

var (
	// HTTPResponseCtr allows the counting of Http Responses and their status codes
	HTTPResponseCtr = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "http_resp_total",
			Help: "Number of http responses",
		},
		[]string{"code"},
	)
	// BackendActive allows the counting of active (registered) backends
	BackendActive = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "backend_active_total",
			Help: "Number of active backends",
		},
	)
	// BackendsRegistered allows the counting of backends that have been registered
	BackendsRegistered = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "backend_registered_total",
			Help: "Number of backends that have been registered",
		},
	)
	// BackendsStarted allows the counting of backends that have been started
	BackendsStarted = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "backend_started_total",
			Help: "Number of backends that have been started",
		},
	)
	// BackendsUnregistered allows the counting of backends that have been unregistered
	BackendsUnregistered = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "backend_unregistered_total",
			Help: "Number of backends that have been unregistered",
		},
	)
	// BackendFailure allows the counting of backends and the corresponding failure reason
	BackendFailure = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "backend_failure_total",
			Help: "Number of backends that have failed",
		},
		[]string{"reason"},
	)
	// BackendReconnectSSH allows the counting of backends that have reconnected to the SSH server
	BackendReconnectSSH = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "backend_reconnect_ssh_total",
			Help: "Number of backends that have reconnected to SSH",
		},
	)
	// BackendProvisioningDuration allows the histogram of provisioning durations
	BackendProvisioningDuration = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Name:    "backend_provisioning_seconds",
			Help:    "Provisioning duration for backends",
			Buckets: []float64{1, 5, 10, 30, 1 * 60, 2 * 60, 3 * 60, 4 * 60, 5 * 60, 10 * 60, 20 * 60},
		},
	)
	// BackendConnectSSHDuration allows the histogram of provisioning durations
	BackendConnectSSHDuration = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Name:    "backend_connect_ssh_seconds",
			Help:    "SSH connection duration for backends",
			Buckets: []float64{1, 5, 10, 30, 1 * 60, 2 * 60, 3 * 60, 4 * 60, 5 * 60},
		},
	)
	// BackendBootstrapDuration allows the histogram of provisioning durations
	BackendBootstrapDuration = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Name:    "backend_bootstrap_seconds",
			Help:    "SSH bootstrap duration for backends",
			Buckets: []float64{1, 5, 10, 30, 1 * 60, 2 * 60, 3 * 60, 4 * 60, 5 * 60, 10 * 60, 20 * 60},
		},
	)
)

func init() {
	prometheus.MustRegister(HTTPResponseCtr)
}
<commit_msg>Prometheus metrics weren't registered<commit_after>package app

import (
	"github.com/prometheus/client_golang/prometheus"
)

var (
	// HTTPResponseCtr allows the counting of Http Responses and their status codes
	HTTPResponseCtr = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "http_resp_total",
			Help: "Number of http responses",
		},
		[]string{"code"},
	)
	// BackendActive allows the counting of active (registered) backends
	BackendActive = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "backend_active_total",
			Help: "Number of active backends",
		},
	)
	// BackendsRegistered allows the counting of backends that have been registered
	BackendsRegistered = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "backend_registered_total",
			Help: "Number of backends that have been registered",
		},
	)
	// BackendsStarted allows the counting of backends that have been started
	BackendsStarted = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "backend_started_total",
			Help: "Number of backends that have been started",
		},
	)
	// BackendsUnregistered allows the counting of backends that have been unregistered
	BackendsUnregistered = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "backend_unregistered_total",
			Help: "Number of backends that have been unregistered",
		},
	)
	// BackendFailure allows the counting of backends and the corresponding failure reason
	BackendFailure = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "backend_failure_total",
			Help: "Number of backends that have failed",
		},
		[]string{"reason"},
	)
	// BackendReconnectSSH allows the counting of backends that have reconnected to the SSH server
	BackendReconnectSSH = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "backend_reconnect_ssh_total",
			Help: "Number of backends that have reconnected to SSH",
		},
	)
	// BackendProvisioningDuration allows the histogram of provisioning durations
	BackendProvisioningDuration = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Name:    "backend_provisioning_seconds",
			Help:    "Provisioning duration for backends",
			Buckets: []float64{1, 5, 10, 30, 1 * 60, 2 * 60, 3 * 60, 4 * 60, 5 * 60, 10 * 60, 20 * 60},
		},
	)
	// BackendConnectSSHDuration allows the histogram of provisioning durations
	BackendConnectSSHDuration = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Name:    "backend_connect_ssh_seconds",
			Help:    "SSH connection duration for backends",
			Buckets: []float64{1, 5, 10, 30, 1 * 60, 2 * 60, 3 * 60, 4 * 60, 5 * 60},
		},
	)
	// BackendBootstrapDuration allows the histogram of provisioning durations
	BackendBootstrapDuration = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Name:    "backend_bootstrap_seconds",
			Help:    "SSH bootstrap duration for backends",
			Buckets: []float64{1, 5, 10, 30, 1 * 60, 2 * 60, 3 * 60, 4 * 60, 5 * 60, 10 * 60, 20 * 60},
		},
	)
)

func init() {
	prometheus.MustRegister(HTTPResponseCtr)
	prometheus.MustRegister(BackendActive)
	prometheus.MustRegister(BackendsRegistered)
	prometheus.MustRegister(BackendsStarted)
	prometheus.MustRegister(BackendsUnregistered)
	prometheus.MustRegister(BackendFailure)
	prometheus.MustRegister(BackendReconnectSSH)
	prometheus.MustRegister(BackendProvisioningDuration)
	prometheus.MustRegister(BackendConnectSSHDuration)
	prometheus.MustRegister(BackendBootstrapDuration)
}
<|endoftext|>


<commit_before>package functions

import (
	"sync"

	"honnef.co/go/tools/callgraph"
	"honnef.co/go/tools/callgraph/static"
	"honnef.co/go/tools/ssa"
	"honnef.co/go/tools/staticcheck/vrp"
)

var stdlibDescs = map[string]Description{
	"strings.Map":            Description{Pure: true},
	"strings.Repeat":         Description{Pure: true},
	"strings.Replace":        Description{Pure: true},
	"strings.Title":          Description{Pure: true},
	"strings.ToLower":        Description{Pure: true},
	"strings.ToLowerSpecial": Description{Pure: true},
	"strings.ToTitle":        Description{Pure: true},
	"strings.ToTitleSpecial": Description{Pure: true},
	"strings.ToUpper":        Description{Pure: true},
	"strings.ToUpperSpecial": Description{Pure: true},
	"strings.Trim":           Description{Pure: true},
	"strings.TrimFunc":       Description{Pure: true},
	"strings.TrimLeft":       Description{Pure: true},
	"strings.TrimLeftFunc":   Description{Pure: true},
	"strings.TrimPrefix":     Description{Pure: true},
	"strings.TrimRight":      Description{Pure: true},
	"strings.TrimRightFunc":  Description{Pure: true},
	"strings.TrimSpace":      Description{Pure: true},
	"strings.TrimSuffix":     Description{Pure: true},

	"(*net/http.Request).WithContext": Description{Pure: true},
}

type Description struct {
	// The function is known to be pure
	Pure bool
	// The function is known to never return (panics notwithstanding)
	Infinite bool
	// Variable ranges
	Ranges vrp.Ranges
	Loops  []Loop
}

type descriptionEntry struct {
	ready  chan struct{}
	result Description
}

type Descriptions struct {
	CallGraph *callgraph.Graph
	mu        sync.Mutex
	cache     map[*ssa.Function]*descriptionEntry
}

func NewDescriptions(prog *ssa.Program) *Descriptions {
	return &Descriptions{
		CallGraph: static.CallGraph(prog),
		cache:     map[*ssa.Function]*descriptionEntry{},
	}
}

func (d *Descriptions) Get(fn *ssa.Function) Description {
	d.mu.Lock()
	fd := d.cache[fn]
	if fd == nil {
		fd = &descriptionEntry{
			ready: make(chan struct{}),
		}
		d.cache[fn] = fd
		d.mu.Unlock()

		{
			fd.result = stdlibDescs[fn.RelString(nil)]
			fd.result.Pure = fd.result.Pure || d.IsPure(fn)
			fd.result.Infinite = fd.result.Infinite || !terminates(fn)
			fd.result.Ranges = vrp.BuildGraph(fn).Solve()
			fd.result.Loops = findLoops(fn)
		}

		close(fd.ready)
	} else {
		d.mu.Unlock()
		<-fd.ready
	}
	return fd.result
}
<commit_msg>functions: detect functions that return nil errors<commit_after>package functions

import (
	"go/types"
	"sync"

	"honnef.co/go/tools/callgraph"
	"honnef.co/go/tools/callgraph/static"
	"honnef.co/go/tools/ssa"
	"honnef.co/go/tools/staticcheck/vrp"
)

var stdlibDescs = map[string]Description{
	"strings.Map":            Description{Pure: true},
	"strings.Repeat":         Description{Pure: true},
	"strings.Replace":        Description{Pure: true},
	"strings.Title":          Description{Pure: true},
	"strings.ToLower":        Description{Pure: true},
	"strings.ToLowerSpecial": Description{Pure: true},
	"strings.ToTitle":        Description{Pure: true},
	"strings.ToTitleSpecial": Description{Pure: true},
	"strings.ToUpper":        Description{Pure: true},
	"strings.ToUpperSpecial": Description{Pure: true},
	"strings.Trim":           Description{Pure: true},
	"strings.TrimFunc":       Description{Pure: true},
	"strings.TrimLeft":       Description{Pure: true},
	"strings.TrimLeftFunc":   Description{Pure: true},
	"strings.TrimPrefix":     Description{Pure: true},
	"strings.TrimRight":      Description{Pure: true},
	"strings.TrimRightFunc":  Description{Pure: true},
	"strings.TrimSpace":      Description{Pure: true},
	"strings.TrimSuffix":     Description{Pure: true},

	"(*net/http.Request).WithContext": Description{Pure: true},

	"math/rand.Read":         Description{NilError: true},
	"(*math/rand.Rand).Read": Description{NilError: true},
}

type Description struct {
	// The function is known to be pure
	Pure bool
	// The function is known to never return (panics notwithstanding)
	Infinite bool
	// Variable ranges
	Ranges vrp.Ranges
	Loops  []Loop
	// Function returns an error as its last argument, but it is
	// always nil
	NilError bool
}

type descriptionEntry struct {
	ready  chan struct{}
	result Description
}

type Descriptions struct {
	CallGraph *callgraph.Graph
	mu        sync.Mutex
	cache     map[*ssa.Function]*descriptionEntry
}

func NewDescriptions(prog *ssa.Program) *Descriptions {
	return &Descriptions{
		CallGraph: static.CallGraph(prog),
		cache:     map[*ssa.Function]*descriptionEntry{},
	}
}

func (d *Descriptions) Get(fn *ssa.Function) Description {
	d.mu.Lock()
	fd := d.cache[fn]
	if fd == nil {
		fd = &descriptionEntry{
			ready: make(chan struct{}),
		}
		d.cache[fn] = fd
		d.mu.Unlock()

		{
			fd.result = stdlibDescs[fn.RelString(nil)]
			fd.result.Pure = fd.result.Pure || d.IsPure(fn)
			fd.result.Infinite = fd.result.Infinite || !terminates(fn)
			fd.result.Ranges = vrp.BuildGraph(fn).Solve()
			fd.result.Loops = findLoops(fn)
			fd.result.NilError = fd.result.NilError || IsNilError(fn)
		}

		close(fd.ready)
	} else {
		d.mu.Unlock()
		<-fd.ready
	}
	return fd.result
}

func IsNilError(fn *ssa.Function) bool {
	// TODO(dh): This is very simplistic, as we only look for constant
	// nil returns. A more advanced approach would work transitively.
	// An even more advanced approach would be context-aware and
	// determine nil errors based on inputs (e.g. io.WriteString to a
	// bytes.Buffer will always return nil, but an io.WriteString to
	// an os.File might not). Similarly, an os.File opened for reading
	// won't error on Close, but other files will.
	res := fn.Signature.Results()
	if res.Len() == 0 {
		return false
	}
	last := res.At(res.Len() - 1)
	if types.TypeString(last.Type(), nil) != "error" {
		return false
	}

	if fn.Blocks == nil {
		return false
	}
	for _, block := range fn.Blocks {
		if len(block.Instrs) == 0 {
			continue
		}
		ins := block.Instrs[len(block.Instrs)-1]
		ret, ok := ins.(*ssa.Return)
		if !ok {
			continue
		}
		v := ret.Results[len(ret.Results)-1]
		c, ok := v.(*ssa.Const)
		if !ok {
			return false
		}
		if !c.IsNil() {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package db
<commit_msg>Add some docs for db package<commit_after>// This package provides machinery for the database subsystem of
// horizon.
//
// The crux of this package is `db.Query`.  Various structs implement this
// interface, where the struct usually represent the arguments to the query.
// For example the following would represent a query to find a single account
// by address:
//
//    type AccountByAddress struct {
//      GormQuery
//      Address string
//    }
//
// You would then implement the query's execution like so:
//
//    func (q AccountByAddress) Get() []interface{} {
//      var account Account
//      err := q.GormQuery.DB.Where("address = ?", q.Address).First(&account).Error
//      if err != nil {
//        return nil, err
//      }
//
//      return []interface{}{account}, nil
//    }
//
// Executing queries happens through `db.Results()`, `db.First()`, and
// `db.Stream()`.
//
package db
<|endoftext|>


<commit_before>package scrape

import (
	"io"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
)

// setup a test HTTP server along with a scrape.Client that is configured to
// talk to that test server. Tests should register handlers on the mux which
// provide mock responses for the GitHub pages being tested.
func setup() (client *Client, mux *http.ServeMux, cleanup func()) {
	mux = http.NewServeMux()
	server := httptest.NewServer(mux)

	client = NewClient(nil)
	client.baseURL, _ = url.Parse(server.URL + "/")

	return client, mux, server.Close
}

func copyTestFile(w io.Writer, filename string) error {
	f, err := os.Open("testdata/" + filename)
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = io.Copy(w, f)
	return err
}
<commit_msg>scrape: define testdata directory in package var<commit_after>package scrape

import (
	"io"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
)

// setup a test HTTP server along with a scrape.Client that is configured to
// talk to that test server. Tests should register handlers on the mux which
// provide mock responses for the GitHub pages being tested.
func setup() (client *Client, mux *http.ServeMux, cleanup func()) {
	mux = http.NewServeMux()
	server := httptest.NewServer(mux)

	client = NewClient(nil)
	client.baseURL, _ = url.Parse(server.URL + "/")

	return client, mux, server.Close
}

var testDataDir = "testdata/"

func copyTestFile(w io.Writer, filename string) error {
	f, err := os.Open(testDataDir + filename)
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = io.Copy(w, f)
	return err
}
<|endoftext|>


<commit_before>package restoreintegration_test

import (
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-cf/cf-redis-broker/restoreconfig"
)

var _ = Describe("restore", func() {
	var restoreCommand *exec.Cmd

	var instanceID string
	var sourceRdbPath string
	var testDataDir string
	var redisSession *gexec.Session

	var config restoreconfig.Config

	BeforeEach(func() {
		instanceID = "test_instance"
		err := copyFile(filepath.Join("..", "brokerintegration", "assets", "redis.conf"), "/tmp/redis.conf")
		Ω(err).ShouldNot(HaveOccurred())
		err = copyFile(filepath.Join("assets", "monit"), "/tmp/monit")
		Ω(err).ShouldNot(HaveOccurred())
		err = os.Chmod("/tmp/monit", 0755)
		Ω(err).ShouldNot(HaveOccurred())

	})

	AfterEach(func() {
		pid, err := config.InstancePid(instanceID)
		if err == nil {
			syscall.Kill(pid, syscall.SIGKILL)
		}

		Eventually(redisSession, "20s").Should(gexec.Exit(0))
	})

	Describe("common to plans", func() {
		BeforeEach(func() {
			config, testDataDir, sourceRdbPath, restoreCommand, redisSession = setupRedisTestForPlan(instanceID, false)
		})

		It("exits with a non zero status if no arguments are provided", func() {
			restoreCommand.Args = []string{}
			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session, "20s").Should(gexec.Exit(1))
			Eventually(session.Err).Should(gbytes.Say("usage: restore <instance_id> <rdb_path>"))
		})

		It("exits with a non zero status if the RDB file does not exist", func() {
			restoreCommand.Args = []string{restoreCommand.Args[0], instanceID, "bar"}
			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session.Err).Should(gbytes.Say("RDB file not found"))
			Eventually(session, "20s").Should(gexec.Exit(1))
		})

		It("exits with a non zero status if the config cannot be loaded", func() {
			restoreCommand.Env = []string{"RESTORE_CONFIG_PATH=foo"}
			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session.Err).Should(gbytes.Say("Could not load config"))
			Eventually(session, "20s").Should(gexec.Exit(1))
		})

		// 	It("stops and then starts the process-watcher", func() {
		// 		session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
		// 		Expect(err).NotTo(HaveOccurred())

		// 		Eventually(session, "20s").Should(gexec.Exit(0))

		// 		monitLogBytes, err := ioutil.ReadFile(monitLogFile)
		// 		Expect(err).ToNot(HaveOccurred())

		// 		Expect(string(monitLogBytes)).To(ContainSubstring("stopping process-watcher"))
		// 		Expect(string(monitLogBytes)).To(ContainSubstring("starting process-watcher"))
		// 	})

		It("stops redis", func() {
			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(redisSession, "20s").Should(gexec.Exit(0))
			Eventually(session, "20s").Should(gexec.Exit(0))
		})

		It("exits successfully if the instance and the RDB file exist", func() {
			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session, "20s").Should(gexec.Exit(0))
		})

		It("does not leave redis running", func() {
			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session, "20s").Should(gexec.Exit(0))

			pkillCommand := exec.Command("pkill", "redis-server")
			pkillSession, err := gexec.Start(pkillCommand, GinkgoWriter, GinkgoWriter)
			// pkill returns 1 if there is nothing for it to kill
			Eventually(pkillSession).Should(gexec.Exit(1))
		})
	})

	Describe("shared plan", func() {
		BeforeEach(func() {
			config, testDataDir, sourceRdbPath, restoreCommand, redisSession = setupRedisTestForPlan(instanceID, false)
		})

		It("exits with a non zero status if the instance directory does not exist", func() {
			restoreCommand.Args = []string{restoreCommand.Args[0], "foo", sourceRdbPath}
			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session.Err).Should(gbytes.Say("Instance not found"))
			Eventually(session, "20s").Should(gexec.Exit(1))
		})

		It("creates a new RDB file in the instance directory", func() {
			newRdbPath := filepath.Join(testDataDir, "dump.rdb")

			_, err := os.Stat(newRdbPath)
			Expect(os.IsNotExist(err)).To(BeTrue())

			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session, "20s").Should(gexec.Exit(0))

			copiedFileContents, err := ioutil.ReadFile(newRdbPath)
			Expect(err).NotTo(HaveOccurred())
			sourceFileContents, err := ioutil.ReadFile(sourceRdbPath)
			Expect(err).NotTo(HaveOccurred())

			Expect(copiedFileContents).To(Equal(sourceFileContents))
		})

		It("creates a new AOF file in the instance directory", func() {
			aofPath := filepath.Join(testDataDir, "appendonly.aof")

			_, err := os.Stat(aofPath)
			Expect(os.IsNotExist(err)).To(BeTrue())

			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session, "20s").Should(gexec.Exit(0))

			fileContents, err := ioutil.ReadFile(aofPath)
			Expect(err).NotTo(HaveOccurred())
			Expect(string(fileContents)).To(ContainSubstring("TEST_KEY"))
		})

	})

	Describe("dedicated plan", func() {
		BeforeEach(func() {
			config, testDataDir, sourceRdbPath, restoreCommand, redisSession = setupRedisTestForPlan(instanceID, true)
		})

		It("creates a new RDB file in the instance directory", func() {
			newRdbPath := filepath.Join(testDataDir, "dump.rdb")

			_, err := os.Stat(newRdbPath)
			Expect(os.IsNotExist(err)).To(BeTrue())

			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session, "20s").Should(gexec.Exit(0))

			copiedFileContents, err := ioutil.ReadFile(newRdbPath)
			Expect(err).NotTo(HaveOccurred())
			sourceFileContents, err := ioutil.ReadFile(sourceRdbPath)
			Expect(err).NotTo(HaveOccurred())

			Expect(copiedFileContents).To(Equal(sourceFileContents))
		})

		It("creates a new AOF file in the instance directory", func() {
			aofPath := filepath.Join(testDataDir, "appendonly.aof")

			_, err := os.Stat(aofPath)
			Expect(os.IsNotExist(err)).To(BeTrue())

			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session, "20s").Should(gexec.Exit(0))

			fileContents, err := ioutil.ReadFile(aofPath)
			Expect(err).NotTo(HaveOccurred())
			Expect(string(fileContents)).To(ContainSubstring("TEST_KEY"))
		})
	})
})

func setupRedisTestForPlan(instanceID string, dedicated bool) (config restoreconfig.Config, testDataDir, sourceRdbPath string, restoreCommand *exec.Cmd, redisSession *gexec.Session) {
	var configPath string
	if dedicated {
		configPath = filepath.Join("assets", "restore-dedicated.yml")
	} else {
		configPath = filepath.Join("assets", "restore-shared.yml")
	}

	config, _ = restoreconfig.Load(configPath)

	var testInstanceDir string
	if dedicated {
		testDataDir = filepath.Join(config.RedisDataDirectory)
		testInstanceDir = testDataDir
		os.RemoveAll(testDataDir)
	} else {
		testInstanceDir = filepath.Join(config.RedisDataDirectory, instanceID)
		testDataDir = filepath.Join(testInstanceDir, "db")
		os.RemoveAll(testInstanceDir)
	}
	os.MkdirAll(testDataDir, 0777)

	monitLogDir, err := ioutil.TempDir("", "monit-test-logs")
	Expect(err).NotTo(HaveOccurred())

	monitLogFile := filepath.Join(monitLogDir, "monit.log")
	sourceRdbPath = filepath.Join("assets", "dump.rdb")
	restoreCommand = exec.Command(restoreExecutablePath, instanceID, sourceRdbPath)
	restoreCommand.Env = append(os.Environ(), "RESTORE_CONFIG_PATH="+configPath)
	restoreCommand.Env = append(restoreCommand.Env, "MONIT_LOG_FILE="+monitLogFile)

	fakeChownPath := "assets"
	for i, envVar := range restoreCommand.Env {
		parts := strings.Split(envVar, "=")
		if parts[0] == "PATH" {
			path := fakeChownPath + ":" + parts[1]

			restoreCommand.Env[i] = "PATH=" + path
		}
	}

	pidfilePath := filepath.Join(testInstanceDir, "redis-server.pid")
	redisCmd := exec.Command("redis-server",
		"--pidfile", pidfilePath,
		"--daemonize", "yes",
	)

	redisSession, err = gexec.Start(redisCmd, GinkgoWriter, GinkgoWriter)
	Expect(err).ToNot(HaveOccurred())

	pidFileWritten := make(chan bool)
	go func(c chan<- bool) {
		for {
			if _, err := os.Stat(pidfilePath); !os.IsNotExist(err) {
				c <- true
				break
			}
			time.Sleep(50 * time.Millisecond)
		}
	}(pidFileWritten)

	// wait for redis to write pid file
	select {
	case <-pidFileWritten:
		break
	case <-time.After(30 * time.Second):
		Fail("Test timed out waiting for redis to write PID file.")
	}

	return config, testDataDir, sourceRdbPath, restoreCommand, redisSession
}

func copyFile(sourcePath, destinationPath string) error {
	source, err := os.Open(sourcePath)
	if err != nil {
		return err
	}
	defer source.Close()

	destination, err := os.Create(destinationPath)
	if err != nil {
		return err
	}
	defer destination.Close()

	_, err = io.Copy(destination, source)
	return err
}
<commit_msg>Refactor of the restoreintegration tests<commit_after>package restoreintegration_test

import (
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-cf/cf-redis-broker/restoreconfig"
)

const (
	sharedPlan    string = "shared"
	dedicatedPlan string = "dedicated"
)

var _ = Describe("restore", func() {
	var restoreCommand *exec.Cmd

	var instanceID string
	var sourceRdbPath string
	var redisSession *gexec.Session

	var config restoreconfig.Config

	BeforeEach(func() {
		instanceID = "test_instance"
		sourceRdbPath = filepath.Join("assets", "dump.rdb")
		err := copyFile(filepath.Join("..", "brokerintegration", "assets", "redis.conf"), "/tmp/redis.conf")
		Ω(err).ShouldNot(HaveOccurred())
		err = copyFile(filepath.Join("assets", "monit"), "/tmp/monit")
		Ω(err).ShouldNot(HaveOccurred())
		err = os.Chmod("/tmp/monit", 0755)
		Ω(err).ShouldNot(HaveOccurred())

	})

	AfterEach(func() {
		pid, err := config.InstancePid(instanceID)
		if err == nil {
			syscall.Kill(pid, syscall.SIGKILL)
		}

		Eventually(redisSession, "20s").Should(gexec.Exit(0))
	})

	Describe("common to plans", func() {
		BeforeEach(func() {
			config = loadRestoreConfig(sharedPlan)
			redisSession = startRedisSession(config, instanceID, sharedPlan)
			restoreCommand = buildRestoreCommand(sourceRdbPath, instanceID, sharedPlan)
		})

		It("exits with a non zero status if no arguments are provided", func() {
			restoreCommand.Args = []string{}
			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session, "20s").Should(gexec.Exit(1))
			Eventually(session.Err).Should(gbytes.Say("usage: restore <instance_id> <rdb_path>"))
		})

		It("exits with a non zero status if the RDB file does not exist", func() {
			restoreCommand.Args = []string{restoreCommand.Args[0], instanceID, "bar"}
			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session.Err).Should(gbytes.Say("RDB file not found"))
			Eventually(session, "20s").Should(gexec.Exit(1))
		})

		It("exits with a non zero status if the config cannot be loaded", func() {
			restoreCommand.Env = []string{"RESTORE_CONFIG_PATH=foo"}
			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session.Err).Should(gbytes.Say("Could not load config"))
			Eventually(session, "20s").Should(gexec.Exit(1))
		})

		// 	It("stops and then starts the process-watcher", func() {
		// 		session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
		// 		Expect(err).NotTo(HaveOccurred())

		// 		Eventually(session, "20s").Should(gexec.Exit(0))

		// 		monitLogBytes, err := ioutil.ReadFile(monitLogFile)
		// 		Expect(err).ToNot(HaveOccurred())

		// 		Expect(string(monitLogBytes)).To(ContainSubstring("stopping process-watcher"))
		// 		Expect(string(monitLogBytes)).To(ContainSubstring("starting process-watcher"))
		// 	})

		It("stops redis", func() {
			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(redisSession, "20s").Should(gexec.Exit(0))
			Eventually(session, "20s").Should(gexec.Exit(0))
		})

		It("exits successfully if the instance and the RDB file exist", func() {
			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session, "20s").Should(gexec.Exit(0))
		})

		It("does not leave redis running", func() {
			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session, "20s").Should(gexec.Exit(0))

			pkillCommand := exec.Command("pkill", "redis-server")
			pkillSession, err := gexec.Start(pkillCommand, GinkgoWriter, GinkgoWriter)
			// pkill returns 1 if there is nothing for it to kill
			Eventually(pkillSession).Should(gexec.Exit(1))
		})
	})

	Describe("shared plan", func() {
		BeforeEach(func() {
			config = loadRestoreConfig(sharedPlan)
			redisSession = startRedisSession(config, instanceID, sharedPlan)
			restoreCommand = buildRestoreCommand(sourceRdbPath, instanceID, sharedPlan)
		})

		It("exits with a non zero status if the instance directory does not exist", func() {
			restoreCommand.Args = []string{restoreCommand.Args[0], "foo", sourceRdbPath}
			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session.Err).Should(gbytes.Say("Instance not found"))
			Eventually(session, "20s").Should(gexec.Exit(1))
		})

		It("creates a new RDB file in the instance directory", func() {
			newRdbPath := filepath.Join(config.InstanceDataDir(instanceID), "dump.rdb")

			_, err := os.Stat(newRdbPath)
			Expect(os.IsNotExist(err)).To(BeTrue())

			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session, "20s").Should(gexec.Exit(0))

			copiedFileContents, err := ioutil.ReadFile(newRdbPath)
			Expect(err).NotTo(HaveOccurred())
			sourceFileContents, err := ioutil.ReadFile(sourceRdbPath)
			Expect(err).NotTo(HaveOccurred())

			Expect(copiedFileContents).To(Equal(sourceFileContents))
		})

		It("creates a new AOF file in the instance directory", func() {
			aofPath := filepath.Join(config.InstanceDataDir(instanceID), "appendonly.aof")

			_, err := os.Stat(aofPath)
			Expect(os.IsNotExist(err)).To(BeTrue())

			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session, "20s").Should(gexec.Exit(0))

			fileContents, err := ioutil.ReadFile(aofPath)
			Expect(err).NotTo(HaveOccurred())
			Expect(string(fileContents)).To(ContainSubstring("TEST_KEY"))
		})

	})

	Describe("dedicated plan", func() {
		BeforeEach(func() {
			config = loadRestoreConfig(dedicatedPlan)
			redisSession = startRedisSession(config, instanceID, dedicatedPlan)
			restoreCommand = buildRestoreCommand(sourceRdbPath, instanceID, dedicatedPlan)
		})

		It("creates a new RDB file in the instance directory", func() {
			newRdbPath := filepath.Join(config.InstanceDataDir(instanceID), "dump.rdb")

			_, err := os.Stat(newRdbPath)
			Expect(os.IsNotExist(err)).To(BeTrue())

			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session, "20s").Should(gexec.Exit(0))

			copiedFileContents, err := ioutil.ReadFile(newRdbPath)
			Expect(err).NotTo(HaveOccurred())
			sourceFileContents, err := ioutil.ReadFile(sourceRdbPath)
			Expect(err).NotTo(HaveOccurred())

			Expect(copiedFileContents).To(Equal(sourceFileContents))
		})

		It("creates a new AOF file in the instance directory", func() {
			aofPath := filepath.Join(config.InstanceDataDir(instanceID), "appendonly.aof")

			_, err := os.Stat(aofPath)
			Expect(os.IsNotExist(err)).To(BeTrue())

			session, err := gexec.Start(restoreCommand, GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(session, "20s").Should(gexec.Exit(0))

			fileContents, err := ioutil.ReadFile(aofPath)
			Expect(err).NotTo(HaveOccurred())
			Expect(string(fileContents)).To(ContainSubstring("TEST_KEY"))
		})
	})
})

func startRedisSession(config restoreconfig.Config, instanceID, planName string) (redisSession *gexec.Session) {
	var testInstanceDir string
	testDataDir := config.InstanceDataDir(instanceID)
	if planName == dedicatedPlan {
		testInstanceDir = testDataDir
		os.RemoveAll(testDataDir)
	} else {
		testInstanceDir = filepath.Join(config.RedisDataDirectory, instanceID)
		os.RemoveAll(testInstanceDir)
	}
	os.MkdirAll(testDataDir, 0777)

	pidfilePath := filepath.Join(testInstanceDir, "redis-server.pid")
	redisCmd := exec.Command("redis-server",
		"--pidfile", pidfilePath,
		"--daemonize", "yes",
	)
	var err error
	redisSession, err = gexec.Start(redisCmd, GinkgoWriter, GinkgoWriter)
	Expect(err).ToNot(HaveOccurred())

	pidFileWritten := make(chan bool)
	go func(c chan<- bool) {
		for {
			if _, err := os.Stat(pidfilePath); !os.IsNotExist(err) {
				c <- true
				break
			}
			time.Sleep(50 * time.Millisecond)
		}
	}(pidFileWritten)

	// wait for redis to write pid file
	select {
	case <-pidFileWritten:
		break
	case <-time.After(30 * time.Second):
		Fail("Test timed out waiting for redis to write PID file.")
	}

	return redisSession
}

func loadRestoreConfig(planName string) restoreconfig.Config {
	configPath := filepath.Join("assets", "restore-"+planName+".yml")

	config, err := restoreconfig.Load(configPath)
	Expect(err).ToNot(HaveOccurred())
	return config
}

func buildRestoreCommand(sourceRdbPath, instanceID, planName string) *exec.Cmd {
	configPath := filepath.Join("assets", "restore-"+planName+".yml")
	monitLogDir, err := ioutil.TempDir("", "monit-test-logs")
	Expect(err).NotTo(HaveOccurred())

	monitLogFile := filepath.Join(monitLogDir, "monit.log")
	restoreCommand := exec.Command(restoreExecutablePath, instanceID, sourceRdbPath)
	restoreCommand.Env = append(os.Environ(), "RESTORE_CONFIG_PATH="+configPath)
	restoreCommand.Env = append(restoreCommand.Env, "MONIT_LOG_FILE="+monitLogFile)

	fakeChownPath := "assets"
	for i, envVar := range restoreCommand.Env {
		parts := strings.Split(envVar, "=")
		if parts[0] == "PATH" {
			path := fakeChownPath + ":" + parts[1]

			restoreCommand.Env[i] = "PATH=" + path
		}
	}
	return restoreCommand
}

func copyFile(sourcePath, destinationPath string) error {
	source, err := os.Open(sourcePath)
	if err != nil {
		return err
	}
	defer source.Close()

	destination, err := os.Create(destinationPath)
	if err != nil {
		return err
	}
	defer destination.Close()

	_, err = io.Copy(destination, source)
	return err
}
<|endoftext|>


<commit_before>package openstack

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccOpenStackComputeV2FlavorDataSource_basic(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccOpenStackComputeV2FlavorDataSource_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckComputeV2FlavorDataSourceID("data.openstack_compute_flavor_v2.flavor_1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "name", "m1.acctest"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "ram", "512"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "disk", "5"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "vcpus", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "rx_tx_factor", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "is_public", "true"),
				),
			},
		},
	})
}

func TestAccOpenStackComputeV2FlavorDataSource_testQueries(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccOpenStackComputeV2FlavorDataSource_queryDisk,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckComputeV2FlavorDataSourceID("data.openstack_compute_flavor_v2.flavor_1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "name", "m1.acctest"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "ram", "512"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "disk", "5"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "vcpus", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "rx_tx_factor", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "is_public", "true"),
				),
			},
			resource.TestStep{
				Config: testAccOpenStackComputeV2FlavorDataSource_queryMinDisk,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckComputeV2FlavorDataSourceID("data.openstack_compute_flavor_v2.flavor_1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "name", "m1.acctest"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "ram", "512"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "disk", "5"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "vcpus", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "rx_tx_factor", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "is_public", "true"),
				),
			},
			resource.TestStep{
				Config: testAccOpenStackComputeV2FlavorDataSource_queryMinRAM,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckComputeV2FlavorDataSourceID("data.openstack_compute_flavor_v2.flavor_1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "name", "m1.acctest"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "ram", "512"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "disk", "5"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "vcpus", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "rx_tx_factor", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "is_public", "true"),
				),
			},
			resource.TestStep{
				Config: testAccOpenStackComputeV2FlavorDataSource_queryVCPUs,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckComputeV2FlavorDataSourceID("data.openstack_compute_flavor_v2.flavor_1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "name", "m1.acctest"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "ram", "512"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "disk", "5"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "vcpus", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "rx_tx_factor", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "is_public", "true"),
				),
			},
		},
	})
}

func testAccCheckComputeV2FlavorDataSourceID(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Can't find flavor data source: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("Flavor data source ID not set")
		}

		return nil
	}
}

const testAccOpenStackComputeV2FlavorDataSource_basic = `
data "openstack_compute_flavor_v2" "flavor_1" {
  name = "m1.acctest"
}
`

const testAccOpenStackComputeV2FlavorDataSource_queryDisk = `
data "openstack_compute_flavor_v2" "flavor_1" {
  disk = 5
}
`

const testAccOpenStackComputeV2FlavorDataSource_queryMinDisk = `
data "openstack_compute_flavor_v2" "flavor_1" {
  name = "m1.acctest"
  min_disk = 5
}
`

const testAccOpenStackComputeV2FlavorDataSource_queryMinRAM = `
data "openstack_compute_flavor_v2" "flavor_1" {
  name = "m1.acctest"
  min_ram = 512
}
`

const testAccOpenStackComputeV2FlavorDataSource_queryVCPUs = `
data "openstack_compute_flavor_v2" "flavor_1" {
  name = "m1.acctest"
  vcpus = 1
}
`
<commit_msg>Compute v2: Fixing flavor data source acceptance test<commit_after>package openstack

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccOpenStackComputeV2FlavorDataSource_basic(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccOpenStackComputeV2FlavorDataSource_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckComputeV2FlavorDataSourceID("data.openstack_compute_flavor_v2.flavor_1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "name", "m1.acctest"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "ram", "512"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "disk", "5"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "vcpus", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "rx_tx_factor", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "is_public", "true"),
				),
			},
		},
	})
}

func TestAccOpenStackComputeV2FlavorDataSource_testQueries(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccOpenStackComputeV2FlavorDataSource_queryDisk,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckComputeV2FlavorDataSourceID("data.openstack_compute_flavor_v2.flavor_1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "name", "m1.resize"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "ram", "512"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "disk", "6"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "vcpus", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "rx_tx_factor", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "is_public", "true"),
				),
			},
			resource.TestStep{
				Config: testAccOpenStackComputeV2FlavorDataSource_queryMinDisk,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckComputeV2FlavorDataSourceID("data.openstack_compute_flavor_v2.flavor_1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "name", "m1.acctest"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "ram", "512"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "disk", "5"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "vcpus", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "rx_tx_factor", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "is_public", "true"),
				),
			},
			resource.TestStep{
				Config: testAccOpenStackComputeV2FlavorDataSource_queryMinRAM,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckComputeV2FlavorDataSourceID("data.openstack_compute_flavor_v2.flavor_1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "name", "m1.acctest"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "ram", "512"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "disk", "5"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "vcpus", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "rx_tx_factor", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "is_public", "true"),
				),
			},
			resource.TestStep{
				Config: testAccOpenStackComputeV2FlavorDataSource_queryVCPUs,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckComputeV2FlavorDataSourceID("data.openstack_compute_flavor_v2.flavor_1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "name", "m1.acctest"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "ram", "512"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "disk", "5"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "vcpus", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "rx_tx_factor", "1"),
					resource.TestCheckResourceAttr(
						"data.openstack_compute_flavor_v2.flavor_1", "is_public", "true"),
				),
			},
		},
	})
}

func testAccCheckComputeV2FlavorDataSourceID(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Can't find flavor data source: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("Flavor data source ID not set")
		}

		return nil
	}
}

const testAccOpenStackComputeV2FlavorDataSource_basic = `
data "openstack_compute_flavor_v2" "flavor_1" {
  name = "m1.acctest"
}
`

const testAccOpenStackComputeV2FlavorDataSource_queryDisk = `
data "openstack_compute_flavor_v2" "flavor_1" {
  disk = 6
}
`

const testAccOpenStackComputeV2FlavorDataSource_queryMinDisk = `
data "openstack_compute_flavor_v2" "flavor_1" {
  name = "m1.acctest"
  min_disk = 5
}
`

const testAccOpenStackComputeV2FlavorDataSource_queryMinRAM = `
data "openstack_compute_flavor_v2" "flavor_1" {
  name = "m1.acctest"
  min_ram = 512
}
`

const testAccOpenStackComputeV2FlavorDataSource_queryVCPUs = `
data "openstack_compute_flavor_v2" "flavor_1" {
  name = "m1.acctest"
  vcpus = 1
}
`
<|endoftext|>


<commit_before>// Copyright 2015 Yoshi Yamaguchi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shortener

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"path"
	"time"

	"appengine"
	"appengine/datastore"
)

const (
	Version            = "0.1.0"
	starttime          = int64(1437625938157481) // around 2015 Jul 23 13:34
	timestampLeftShift = 22
)

var chars []byte // used for unique id generation.

// initChars initialize chars as sequence of 0-9A-Za-z
func initChars() {
	chars = make([]byte, 62)
	for i := 0; i < 10; i++ { // 0-9
		chars[i] = byte(48 + i)
	}
	for i := 0; i < 26; i++ { // A-Z
		chars[i+10] = byte(65 + i)
	}
	for i := 0; i < 26; i++ { // a-z
		chars[i+36] = byte(97 + i)
	}
}

// init setup chars and URL routers.
func init() {
	initChars()
	router := &RegexpHandler{}
	router.HandleFunc(`/`, top)
	router.HandleFunc(`/[0-9A-Za-z_\-]{10,}`, redirect)
	router.HandleFunc(`/version`, version)
	router.HandleFunc(`/shortener/v1`, shortener)
	http.Handle("/", router)
}

// version returns application version.
func version(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, Version)
}

// top returns UI front page.
func top(w http.ResponseWriter, r *http.Request) {

	fmt.Fprintf(w, "hello")
}

// shortener
func shortener(w http.ResponseWriter, r *http.Request) {
	req := URLRequest{}
	if r.Method != "POST" {
		http.Error(w, fmt.Sprintf("Methods but for POST are not allowed"), http.StatusMethodNotAllowed)
	}
	data, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, fmt.Sprintf("Unexpected payload: %v", err), http.StatusInternalServerError)
	}
	err = json.Unmarshal(data, &req)
	if err != nil {
		http.Error(w, fmt.Sprintf("JSON decode error: %v", err), http.StatusInternalServerError)
	}

	c := appengine.NewContext(r)
	id := uniqueid()
	e := &URLEntity{
		ID:  id,
		URL: req.URL,
	}
	key := datastore.NewIncompleteKey(c, "URL", nil)
	_, err = datastore.Put(c, key, e)
	if err != nil {
		http.Error(w, fmt.Sprintf("datastore put error: %v", err), http.StatusInternalServerError)
	}
	entry, err := json.Marshal(e)
	if err != nil {
		http.Error(w, fmt.Sprintf("JSON encode error: %v", err), http.StatusInternalServerError)
	}
	fmt.Fprintf(w, string(entry))
}

// redirect find specified shortened URL path from datastore and redirect to original URL.
func redirect(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	id := path.Base(r.URL.Path)
	es := []URLEntity{}
	keys, err := datastore.NewQuery("URL").Filter("ID=", id).GetAll(c, &es)
	if err != nil {
		http.Error(w, fmt.Sprintf("datastore error: %v", err), http.StatusInternalServerError)
	}
	if len(es) == 0 {
		http.Redirect(w, r, "/", http.StatusNotFound)
		return
	}
	original := es[0].URL
	es[0].Count += 1
	key, err := datastore.Put(c, key[0], es[0])
	if err != nil {
		http.Error(w, fmt.Sprintf("datastore error: %v", err), http.StatusInternalServerError)
	}
	http.Redirect(w, r, original, http.StatusFound)
}

// uniqueid generates unique id from unix time in microsecond and randomnumber based on it,
// then convert it to base 62 number
func uniqueid() string {
	now := time.Now().UnixNano() / 1000
	delta := now - starttime
	rand.Seed(delta)
	n := rand.Intn(2 ^ timestampLeftShift)
	id := delta<<timestampLeftShift | int64(n)

	size := int64(len(chars))
	result := make([]byte, 36)
	i := 0
	for id > 0 {
		rem := id % size
		id = id / size
		result[i] = chars[rem]
		i++
	}
	return string(result[:i])
}
<commit_msg>add URL validator<commit_after>// Copyright 2015 Yoshi Yamaguchi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shortener

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"path"
	"regexp"
	"time"

	"appengine"
	"appengine/datastore"
)

const (
	Version            = "0.1.0"
	starttime          = int64(1437625938157481) // around 2015 Jul 23 13:34
	timestampLeftShift = 22
)

var chars []byte // used for unique id generation.

var URLpattern = regexp.MustCompile(`(https?|ftp)://([\w-]+\.)+[\w-]+(/[\w- ./?%&=]*)?`)

// initChars initialize chars as sequence of 0-9A-Za-z
func initChars() {
	chars = make([]byte, 62)
	for i := 0; i < 10; i++ { // 0-9
		chars[i] = byte(48 + i)
	}
	for i := 0; i < 26; i++ { // A-Z
		chars[i+10] = byte(65 + i)
	}
	for i := 0; i < 26; i++ { // a-z
		chars[i+36] = byte(97 + i)
	}
}

// init setup chars and URL routers.
func init() {
	initChars()
	router := &RegexpHandler{}
	router.HandleFunc(`/`, top)
	router.HandleFunc(`/[0-9A-Za-z_\-]{10,}`, redirect)
	router.HandleFunc(`/version`, version)
	router.HandleFunc(`/shortener/v1`, shortener)
	http.Handle("/", router)
}

// version returns application version.
func version(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, Version)
}

// top returns UI front page.
func top(w http.ResponseWriter, r *http.Request) {

	fmt.Fprintf(w, "hello")
}

// shortener
func shortener(w http.ResponseWriter, r *http.Request) {
	req := URLRequest{}
	if r.Method != "POST" {
		http.Error(w, fmt.Sprintf("Methods but for POST are not allowed"), http.StatusMethodNotAllowed)
	}
	data, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, fmt.Sprintf("Unexpected payload: %v", err), http.StatusInternalServerError)
	}
	err = json.Unmarshal(data, &req)
	if err != nil {
		http.Error(w, fmt.Sprintf("JSON decode error: %v", err), http.StatusInternalServerError)
	}

	if !URLpattern.MatchString(req.URL) {
		http.Error(w, fmt.Sprintf("parameter must be valid web URI"), http.StatusBadRequest)
	}

	c := appengine.NewContext(r)
	id := uniqueid()
	e := &URLEntity{
		ID:  id,
		URL: req.URL,
	}
	key := datastore.NewIncompleteKey(c, "URL", nil)
	_, err = datastore.Put(c, key, e)
	if err != nil {
		http.Error(w, fmt.Sprintf("datastore put error: %v", err), http.StatusInternalServerError)
	}
	entry, err := json.Marshal(e)
	if err != nil {
		http.Error(w, fmt.Sprintf("JSON encode error: %v", err), http.StatusInternalServerError)
	}
	fmt.Fprintf(w, string(entry))
}

// redirect find specified shortened URL path from datastore and redirect to original URL.
func redirect(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	id := path.Base(r.URL.Path)
	es := []URLEntity{}
	keys, err := datastore.NewQuery("URL").Filter("ID=", id).GetAll(c, &es)
	if err != nil {
		http.Error(w, fmt.Sprintf("datastore error: %v", err), http.StatusInternalServerError)
	}
	if len(es) == 0 {
		http.Redirect(w, r, "/", http.StatusNotFound)
		return
	}
	original := es[0].URL
	es[0].Count += 1
	_, err = datastore.Put(c, keys[0], es[0])
	if err != nil {
		http.Error(w, fmt.Sprintf("datastore error: %v", err), http.StatusInternalServerError)
	}
	http.Redirect(w, r, original, http.StatusFound)
}

// uniqueid generates unique id from unix time in microsecond and randomnumber based on it,
// then convert it to base 62 number
func uniqueid() string {
	now := time.Now().UnixNano() / 1000
	delta := now - starttime
	rand.Seed(delta)
	n := rand.Intn(2 ^ timestampLeftShift)
	id := delta<<timestampLeftShift | int64(n)

	size := int64(len(chars))
	result := make([]byte, 36)
	i := 0
	for id > 0 {
		rem := id % size
		id = id / size
		result[i] = chars[rem]
		i++
	}
	return string(result[:i])
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"os"

	"code.google.com/p/gcfg"
	"github.com/mitchellh/go-homedir"
	"github.com/spf13/cobra"
)

var (
	home   string
	siaDir string
	config Config
)

type Config struct {
	Siad struct {
		ApiPort           uint16
		RpcPort           uint16
		NoBootstrap       bool
		ConfigFilename    string
		HostDirectory     string
		StyleDirectory    string
		DownloadDirectory string
	}
}

// checkSiaDir verifies that a sia directory exists.
func checkSiaDir() (err error) {
	home, err := homedir.Dir()
	if err != nil {
		return
	}

	// Check that ~/.config/sia exists
	dirname := home + "/.config/sia/"
	if _, err = os.Stat(dirname); err == nil {
		siaDir = dirname
		return
	}

	err = errors.New("No sia directory found, please create and populate the sia directory (instructions found in README.md)")
	return
}

// confilgFilenameDefault checks multiple directories for a config file and
// loads the first one it finds. "" is returned if no config file is found.
func configFilenameDefault() string {
	// Try siaDir/config
	filename := siaDir + "config"
	if _, err := os.Stat(filename); err == nil {
		return filename
	}
	return ""
}

// startEnvironment calls createEnvironment(), which will handle everything
// else.
func startEnvironment(cmd *cobra.Command, args []string) {
	_, err := CreateEnvironment(config)
	if err != nil {
		fmt.Println(err)
	}
}

/*
// install moves the given folder to the $home/.config/sia/style folder. Isn't
// guaranteed to work correctly.
func install(cmd *cobra.Command, args []string) {
	if len(args) != 1 {
		fmt.Println("Incorrect use of install - must supply filepath to the styles folder.")
		return
	}

	home, err := homedir.Dir()
	if err != nil {
		fmt.Println("Error:", err)
		return
	}

	configDir := home + "/.config/sia/style"
	err = os.MkdirAll(configDir, os.ModeDir|os.ModePerm)
	if err != nil {
		fmt.Println("Error:", err)
		return
	}

	err = os.Rename(args[0], configDir)
	if err != nil {
		fmt.Println("Error:", err)
		return
	}
	fmt.Println("Moved styles folder to", configDir)
}
*/

// Prints version information about Sia Daemon.
func version(cmd *cobra.Command, args []string) {
	fmt.Println("Sia Daemon v0.1.0")
}

func main() {
	// Check that the sia directory exists and can be found.
	err := checkSiaDir()
	if err != nil {
		fmt.Println(err)
		return
	}
	home, err = homedir.Dir()
	if err != nil {
		fmt.Println(err)
		return
	}

	root := &cobra.Command{
		Use:   os.Args[0],
		Short: "Sia Daemon v0.1.0",
		Long:  "Sia Daemon v0.1.0",
		Run:   startEnvironment,
	}

	/*
	root.AddCommand(&cobra.Command{
		Use:   "install",
		Short: "Install siad",
		Long:  "Install siad by creating a config folder and copying in the styles folder",
		Run:   install,
	})
	*/

	root.AddCommand(&cobra.Command{
		Use:   "version",
		Short: "Print version information",
		Long:  "Print version information about the Sia Daemon",
		Run:   version,
	})

	// Add flag defaults, which have the lowest priority. Every value will be
	// set.
	defaultConfigFile := configFilenameDefault()
	defaultHostDir := siaDir + "host/"
	defaultStyleDir := siaDir + "style/"
	defaultDownloadDir := home + "/Desktop/Downloads/"
	root.PersistentFlags().Uint16VarP(&config.Siad.ApiPort, "api-port", "a", 9980, "which port is used to communicate with the user")
	root.PersistentFlags().Uint16VarP(&config.Siad.RpcPort, "rpc-port", "r", 9988, "which port is used when talking to other nodes on the network")
	root.PersistentFlags().BoolVarP(&config.Siad.NoBootstrap, "no-bootstrap", "n", false, "disable bootstrapping on this run.")
	root.PersistentFlags().StringVarP(&config.Siad.ConfigFilename, "config-file", "c", defaultConfigFile, "tell siad where to load the config file")
	root.PersistentFlags().StringVarP(&config.Siad.HostDirectory, "host-dir", "H", defaultHostDir, "where the host puts all uploaded files")
	root.PersistentFlags().StringVarP(&config.Siad.StyleDirectory, "style-dir", "s", defaultStyleDir, "where to find the files that compose the frontend")
	root.PersistentFlags().StringVarP(&config.Siad.DownloadDirectory, "download-dir", "d", defaultDownloadDir, "where to download files")

	// Load the config file, which has the middle priorty. Only values defined
	// in the config file will be set.
	if config.Siad.ConfigFilename != "" {
		err := gcfg.ReadFileInto(&config, config.Siad.ConfigFilename)
		if err != nil {
			fmt.Println(err)
			return
		}
	}

	// Execute wil over-write any flags set by the config file, but only if the
	// user specified them manually.
	root.Execute()
}
<commit_msg>better behavior at startup for bad configurations<commit_after>package main

import (
	"fmt"
	"os"

	"code.google.com/p/gcfg"
	"github.com/mitchellh/go-homedir"
	"github.com/spf13/cobra"
)

var (
	config Config
)

type Config struct {
	Siad struct {
		ApiPort           uint16
		RpcPort           uint16
		NoBootstrap       bool
		ConfigFilename    string
		HostDirectory     string
		StyleDirectory    string
		DownloadDirectory string
	}
}

// findSiaDir first checks the current directory, then checks ~/.config/sia,
// looking for the html template file that's needed to run the web app.
// findSiaDir will return home="" if it can't find the home dir, but it won't
// report an error for that. It'll only report an error if it can't find
// template.html.
func findSiaDir() (home, siaDir string, err error) {
	// Check the current directory for the template file.
	var found bool
	if _, err = os.Stat("style/template.html"); err == nil {
		found = true
	}

	// Check ~/.config/sia for the template file.
	home, err = homedir.Dir()
	if err == nil && !found {
		dirname := home + "/.config/sia/style/template.html"
		if _, err = os.Stat(dirname); err == nil {
			siaDir = dirname
			return
		}
	}

	// This is the only error that can be returned.
	if !found {
		err = fmt.Errorf("Style folder not found, please put the 'style/' folder in the current directory")
	} else {
		err = nil
	}
	return
}

// startEnvironment calls createEnvironment(), which will handle everything
// else.
func startEnvironment(cmd *cobra.Command, args []string) {
	_, err := CreateEnvironment(config)
	if err != nil {
		fmt.Println(err)
	}
}

// Prints version information about Sia Daemon.
func version(cmd *cobra.Command, args []string) {
	fmt.Println("Sia Daemon v0.1.0")
}

func main() {
	root := &cobra.Command{
		Use:   os.Args[0],
		Short: "Sia Daemon v0.1.0",
		Long:  "Sia Daemon v0.1.0",
		Run:   startEnvironment,
	}

	root.AddCommand(&cobra.Command{
		Use:   "version",
		Short: "Print version information",
		Long:  "Print version information about the Sia Daemon",
		Run:   version,
	})

	// Add flag defaults, which have the lowest priority.
	home, siaDir, err := findSiaDir()
	if err != nil {
		// Print the error as a warning
		fmt.Println(err)
	}
	defaultConfigFile := siaDir + "config"
	defaultHostDir := siaDir + "host/"
	defaultStyleDir := siaDir + "style/"
	defaultDownloadDir := home + "/Downloads/"
	root.PersistentFlags().Uint16VarP(&config.Siad.ApiPort, "api-port", "a", 9980, "which port is used to communicate with the user")
	root.PersistentFlags().Uint16VarP(&config.Siad.RpcPort, "rpc-port", "r", 9988, "which port is used when talking to other nodes on the network")
	root.PersistentFlags().BoolVarP(&config.Siad.NoBootstrap, "no-bootstrap", "n", false, "disable bootstrapping on this run.")
	root.PersistentFlags().StringVarP(&config.Siad.ConfigFilename, "config-file", "c", defaultConfigFile, "tell siad where to load the config file")
	root.PersistentFlags().StringVarP(&config.Siad.HostDirectory, "host-dir", "H", defaultHostDir, "where the host puts all uploaded files")
	root.PersistentFlags().StringVarP(&config.Siad.StyleDirectory, "style-dir", "s", defaultStyleDir, "where to find the files that compose the frontend")
	root.PersistentFlags().StringVarP(&config.Siad.DownloadDirectory, "download-dir", "d", defaultDownloadDir, "where to download files")

	// Load the config file, which has the middle priorty. Only values defined
	// in the config file will be set.
	if _, err = os.Stat(config.Siad.ConfigFilename); err == nil {
		err := gcfg.ReadFileInto(&config, config.Siad.ConfigFilename)
		if err != nil {
			fmt.Println("Error reading config file:", err)
		}
	}

	// Execute wil over-write any flags set by the config file, but only if the
	// user specified them manually.
	root.Execute()
}
<|endoftext|>