<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state_test

import (
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/state"
)

func (s *StorageStateSuite) setupMultipleStoragesForAdd(c *gc.C) (*state.Charm, *state.Unit, int) {
	storageCons := map[string]state.StorageConstraints{
		"multi1to10": makeStorageCons("loop", 0, 3),
	}
	ch := s.AddTestingCharm(c, "storage-block2")
	service, err := s.State.AddService("storage-block2", "user-test-admin@local", ch, nil, storageCons)
	c.Assert(err, jc.ErrorIsNil)
	u, err := service.AddUnit()
	c.Assert(err, jc.ErrorIsNil)

	original, err := s.State.AllStorageInstances()
	c.Assert(err, jc.ErrorIsNil)
	return ch, u, len(original)
}

func (s *StorageStateSuite) assertStorageAddedDynamically(c *gc.C, original, expected int) {
	final, err := s.State.AllStorageInstances()
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(len(final)-original, gc.Equals, expected)
}

func (s *StorageStateSuite) TestAddStorageToUnit(c *gc.C) {
	ch, u, original := s.setupMultipleStoragesForAdd(c)

	err := s.State.AddStorageForUnit(ch.Meta(), u, "multi1to10", makeStorageCons("loop-pool", 1024, 1))
	c.Assert(err, jc.ErrorIsNil)
	s.assertStorageAddedDynamically(c, original, 1)
}

func (s *StorageStateSuite) TestAddStorageWithCount(c *gc.C) {
	ch, u, original := s.setupMultipleStoragesForAdd(c)

	err := s.State.AddStorageForUnit(ch.Meta(), u, "multi1to10", makeStorageCons("loop-pool", 1024, 2))
	c.Assert(err, jc.ErrorIsNil)
	s.assertStorageAddedDynamically(c, original, 2)
}

func (s *StorageStateSuite) TestAddStorageMultipleCalls(c *gc.C) {
	ch, u, original := s.setupMultipleStoragesForAdd(c)

	err := s.State.AddStorageForUnit(ch.Meta(), u, "multi1to10", makeStorageCons("loop-pool", 1024, 2))
	c.Assert(err, jc.ErrorIsNil)
	s.assertStorageAddedDynamically(c, original, 2)

	// Should not succeed as the number of storages after
	// this call would be 11 whereas our upper limit is 10 here.
	err = s.State.AddStorageForUnit(ch.Meta(), u, "multi1to10", makeStorageCons("loop-pool", 1024, 6))
	c.Assert(err, gc.ErrorMatches, `.*charm "storage-block2" store "multi1to10": at most 10 instances supported, 11 specified.*`)
	s.assertStorageAddedDynamically(c, original+2, 0)
}

func (s *StorageStateSuite) TestAddStorageExceedCount(c *gc.C) {
	service, u, _ := s.setupSingleStorage(c, "block", "loop-pool")
	s.assertStorageAddedDynamically(c, 0, 1)

	ch, _, err := service.Charm()
	c.Assert(err, jc.ErrorIsNil)

	err = s.State.AddStorageForUnit(ch.Meta(), u, "data", makeStorageCons("loop-pool", 1024, 1))
	c.Assert(err, gc.ErrorMatches, `.*charm "storage-block" store "data": at most 1 instances supported, 2 specified.*`)
	s.assertStorageAddedDynamically(c, 1, 0)
}

func (s *StorageStateSuite) TestAddStorageTriggerDefaultPopulated(c *gc.C) {
	ch, u, original := s.setupMultipleStoragesForAdd(c)

	err := s.State.AddStorageForUnit(ch.Meta(), u, "multi1to10", state.StorageConstraints{})
	c.Assert(err, jc.ErrorIsNil)
	s.assertStorageAddedDynamically(c, original, 1)
}

func (s *StorageStateSuite) TestAddStorageDiffPool(c *gc.C) {
	ch, u, original := s.setupMultipleStoragesForAdd(c)

	err := s.State.AddStorageForUnit(ch.Meta(), u, "multi1to10", state.StorageConstraints{Pool: "loop-pool"})
	c.Assert(err, jc.ErrorIsNil)
	s.assertStorageAddedDynamically(c, original, 1)
}

func (s *StorageStateSuite) TestAddStorageDiffSize(c *gc.C) {
	ch, u, original := s.setupMultipleStoragesForAdd(c)

	err := s.State.AddStorageForUnit(ch.Meta(), u, "multi1to10", state.StorageConstraints{Size: 2048})
	c.Assert(err, jc.ErrorIsNil)
	s.assertStorageAddedDynamically(c, original, 1)
}

func (s *StorageStateSuite) TestAddStorageLessMinSize(c *gc.C) {
	ch, u, original := s.setupMultipleStoragesForAdd(c)

	err := s.State.AddStorageForUnit(ch.Meta(), u, "multi2up", state.StorageConstraints{Size: 2})
	c.Assert(err, gc.ErrorMatches, `.*charm "storage-block2" store "multi2up": minimum storage size is 2.0GB, 2.0MB specified.*`)
	s.assertStorageAddedDynamically(c, original, 0)
}

func (s *StorageStateSuite) TestAddStorageWrongName(c *gc.C) {
	ch, u, original := s.setupMultipleStoragesForAdd(c)

	err := s.State.AddStorageForUnit(ch.Meta(), u, "furball", state.StorageConstraints{Size: 2})
	c.Assert(err, gc.ErrorMatches, `.*charm storage "furball" not found.*`)
	s.assertStorageAddedDynamically(c, original, 0)
}

func (s *StorageStateSuite) TestAddStorageConcurrently(c *gc.C) {
	ch, u, original := s.setupMultipleStoragesForAdd(c)
	addStorage := func() {
		s.State.AddStorageForUnit(ch.Meta(), u, "multi1to10", state.StorageConstraints{})
	}
	defer state.SetBeforeHooks(c, s.State, addStorage).Check()
	addStorage()
	// Only 1 should have been added
	s.assertStorageAddedDynamically(c, original, 1)
}

func (s *StorageStateSuite) TestAddStorageConcurrentlyExceedCount(c *gc.C) {
	ch, u, original := s.setupMultipleStoragesForAdd(c)

	count := 6
	addStorage := func() {
		s.State.AddStorageForUnit(ch.Meta(), u, "multi1to10", state.StorageConstraints{Count: uint64(count)})
	}
	defer state.SetBeforeHooks(c, s.State, addStorage).Check()
	addStorage()
	// Only "count" number of instances should have been added.
	s.assertStorageAddedDynamically(c, original, count)
}
<commit_msg>Cleared implementation based on review.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state_test

import (
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/state"
)

type StorageAddSuite struct {
	StorageStateSuiteBase

	charm                *state.Charm
	unit                 *state.Unit
	originalStorageCount int
}

var _ = gc.Suite(&StorageAddSuite{})

func (s *StorageAddSuite) setupMultipleStoragesForAdd(c *gc.C) {
	storageCons := map[string]state.StorageConstraints{
		"multi1to10": makeStorageCons("loop", 0, 3),
	}
	s.charm = s.AddTestingCharm(c, "storage-block2")
	service, err := s.State.AddService("storage-block2", "user-test-admin@local", s.charm, nil, storageCons)
	c.Assert(err, jc.ErrorIsNil)
	s.unit, err = service.AddUnit()
	c.Assert(err, jc.ErrorIsNil)

	all, err := s.State.AllStorageInstances()
	c.Assert(err, jc.ErrorIsNil)
	s.originalStorageCount = len(all)
}

func (s *StorageAddSuite) assertStorageCount(c *gc.C, expected int) {
	all, err := s.State.AllStorageInstances()
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(len(all), gc.Equals, expected)
}

func (s *StorageAddSuite) TestAddStorageToUnit(c *gc.C) {
	s.setupMultipleStoragesForAdd(c)

	err := s.State.AddStorageForUnit(s.charm.Meta(), s.unit, "multi1to10", makeStorageCons("loop-pool", 1024, 1))
	c.Assert(err, jc.ErrorIsNil)
	s.assertStorageCount(c, s.originalStorageCount+1)
}

func (s *StorageAddSuite) TestAddStorageWithCount(c *gc.C) {
	s.setupMultipleStoragesForAdd(c)
	err := s.State.AddStorageForUnit(s.charm.Meta(), s.unit, "multi1to10", makeStorageCons("loop-pool", 1024, 2))
	c.Assert(err, jc.ErrorIsNil)
	s.assertStorageCount(c, s.originalStorageCount+2)
}

func (s *StorageAddSuite) TestAddStorageMultipleCalls(c *gc.C) {
	s.setupMultipleStoragesForAdd(c)

	err := s.State.AddStorageForUnit(s.charm.Meta(), s.unit, "multi1to10", makeStorageCons("loop-pool", 1024, 2))
	c.Assert(err, jc.ErrorIsNil)
	s.assertStorageCount(c, s.originalStorageCount+2)

	// Should not succeed as the number of storages after
	// this call would be 11 whereas our upper limit is 10 here.
	err = s.State.AddStorageForUnit(s.charm.Meta(), s.unit, "multi1to10", makeStorageCons("loop-pool", 1024, 6))
	c.Assert(err, gc.ErrorMatches, `.*charm "storage-block2" store "multi1to10": at most 10 instances supported, 11 specified.*`)
	s.assertStorageCount(c, s.originalStorageCount+2)
}

func (s *StorageAddSuite) TestAddStorageExceedCount(c *gc.C) {
	service, u, _ := s.setupSingleStorage(c, "block", "loop-pool")
	s.assertStorageCount(c, 1)

	ch, _, err := service.Charm()
	c.Assert(err, jc.ErrorIsNil)

	err = s.State.AddStorageForUnit(ch.Meta(), u, "data", makeStorageCons("loop-pool", 1024, 1))
	c.Assert(err, gc.ErrorMatches, `.*charm "storage-block" store "data": at most 1 instances supported, 2 specified.*`)
	s.assertStorageCount(c, 1)
}

func (s *StorageAddSuite) TestAddStorageTriggerDefaultPopulated(c *gc.C) {
	s.setupMultipleStoragesForAdd(c)

	err := s.State.AddStorageForUnit(s.charm.Meta(), s.unit, "multi1to10", state.StorageConstraints{})
	c.Assert(err, jc.ErrorIsNil)
	s.assertStorageCount(c, s.originalStorageCount+1)
}

func (s *StorageAddSuite) TestAddStorageDiffPool(c *gc.C) {
	s.setupMultipleStoragesForAdd(c)

	err := s.State.AddStorageForUnit(s.charm.Meta(), s.unit, "multi1to10", state.StorageConstraints{Pool: "loop-pool"})
	c.Assert(err, jc.ErrorIsNil)
	s.assertStorageCount(c, s.originalStorageCount+1)
}

func (s *StorageAddSuite) TestAddStorageDiffSize(c *gc.C) {
	s.setupMultipleStoragesForAdd(c)

	err := s.State.AddStorageForUnit(s.charm.Meta(), s.unit, "multi1to10", state.StorageConstraints{Size: 2048})
	c.Assert(err, jc.ErrorIsNil)
	s.assertStorageCount(c, s.originalStorageCount+1)
}

func (s *StorageAddSuite) TestAddStorageLessMinSize(c *gc.C) {
	s.setupMultipleStoragesForAdd(c)

	err := s.State.AddStorageForUnit(s.charm.Meta(), s.unit, "multi2up", state.StorageConstraints{Size: 2})
	c.Assert(err, gc.ErrorMatches, `.*charm "storage-block2" store "multi2up": minimum storage size is 2.0GB, 2.0MB specified.*`)
	s.assertStorageCount(c, s.originalStorageCount)
}

func (s *StorageAddSuite) TestAddStorageWrongName(c *gc.C) {
	s.setupMultipleStoragesForAdd(c)

	err := s.State.AddStorageForUnit(s.charm.Meta(), s.unit, "furball", state.StorageConstraints{Size: 2})
	c.Assert(err, gc.ErrorMatches, `.*charm storage "furball" not found.*`)
	s.assertStorageCount(c, s.originalStorageCount)
}

func (s *StorageAddSuite) TestAddStorageConcurrently(c *gc.C) {
	s.setupMultipleStoragesForAdd(c)
	addStorage := func() {
		s.State.AddStorageForUnit(s.charm.Meta(), s.unit, "multi1to10", state.StorageConstraints{})
	}
	defer state.SetBeforeHooks(c, s.State, addStorage).Check()
	addStorage()
	// Only 1 should have been added
	s.assertStorageCount(c, s.originalStorageCount+1)
}

func (s *StorageAddSuite) TestAddStorageConcurrentlyExceedCount(c *gc.C) {
	s.setupMultipleStoragesForAdd(c)

	count := 6
	addStorage := func() {
		s.State.AddStorageForUnit(s.charm.Meta(), s.unit, "multi1to10", state.StorageConstraints{Count: uint64(count)})
	}
	defer state.SetBeforeHooks(c, s.State, addStorage).Check()
	addStorage()
	// Only "count" number of instances should have been added.
	s.assertStorageCount(c, s.originalStorageCount+count)
}
<|endoftext|>


<commit_before>package filters

import (
	"fmt"
	"net/http"

	"github.com/dgrijalva/jwt-go"
	"github.com/emicklei/go-restful"
)

// jwtFilter implements a filter for JWT.
type jwtFilter struct {
	publicKey []byte // The key to use
	loginPath string // The login path to ignore (users acquire their token here)
}

// NewJWTFilter creates a new jwtFilter
func NewJWTFilter(publicKey []byte, loginPath string) *jwtFilter {
	return &jwtFilter{
		publicKey: publicKey,
		loginPath: loginPath,
	}
}

// Filter implements the logic of validating REST end points against the JWT token.
func (f *jwtFilter) Filter(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
	// if the user is logging in for the first time then the
	// path will be /login. If that is the case then we just
	// go to the next filter because there is no token to
	// authenticate against.
	if req.Request.URL.Path != f.loginPath {

		token, err := jwt.ParseFromRequest(req.Request, f.getKey)
		if err != nil || !token.Valid {
			fmt.Printf("invalid token for url %s: %s\n ", req.Request.URL.Path, err)
			resp.WriteErrorString(http.StatusUnauthorized, "Not authorized")
			return
		}
	}
	chain.ProcessFilter(req, resp)
}

// Return the key jwt uses to validate a token.
func (f *jwtFilter) getKey(token *jwt.Token) (interface{}, error) {
	return f.publicKey, nil
}
<commit_msg>Fix comment to reflect the path we check.<commit_after>package filters

import (
	"fmt"
	"net/http"

	"github.com/dgrijalva/jwt-go"
	"github.com/emicklei/go-restful"
)

// jwtFilter implements a filter for JWT.
type jwtFilter struct {
	publicKey []byte // The key to use
	loginPath string // The login path to ignore (users acquire their token here)
}

// NewJWTFilter creates a new jwtFilter
func NewJWTFilter(publicKey []byte, loginPath string) *jwtFilter {
	return &jwtFilter{
		publicKey: publicKey,
		loginPath: loginPath,
	}
}

// Filter implements the logic of validating REST end points against the JWT token.
func (f *jwtFilter) Filter(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
	// if the user is logging in for the first time then the
	// path will be f.loginPath. If that is the case then we just
	// go to the next filter because there is no token to
	// authenticate against.
	if req.Request.URL.Path != f.loginPath {

		token, err := jwt.ParseFromRequest(req.Request, f.getKey)
		if err != nil || !token.Valid {
			fmt.Printf("invalid token for url %s: %s\n ", req.Request.URL.Path, err)
			resp.WriteErrorString(http.StatusUnauthorized, "Not authorized")
			return
		}
	}
	chain.ProcessFilter(req, resp)
}

// Return the key jwt uses to validate a token.
func (f *jwtFilter) getKey(token *jwt.Token) (interface{}, error) {
	return f.publicKey, nil
}
<|endoftext|>


<commit_before>// TODO Change random number generation, and decrease number generation on succesive calls
// TODO Do no overwrite already existing values
// TODO Check not for odd numbers but for 2 pow n + 1
package main

import (
	"errors"
	"fmt"
	"math/rand"
	"time"
)

const CornerStartingHeight = 50
const Roughness = 4

type FractalParams struct {
	LowX         int
	LowY         int
	HighX        int
	HighY        int
	RandomFactor float64
}

// Generate an height map using a fractal algorithm
func generate(size int) ([][]float64, error) {
	// check if size is odd
	if size%2 == 0 {
		return nil, errors.New("Size must be odd.")
	}

	var x, y = size, size

	// Generate our slice to hold the height map data
	heightMap := make([][]float64, y, y)

	for i := range heightMap {
		heightMap[i] = make([]float64, x, x)
	}

	// Assign a basic value to each of the four corners
	heightMap[0][0] = CornerStartingHeight
	heightMap[0][y-1] = CornerStartingHeight
	heightMap[x-1][0] = CornerStartingHeight
	heightMap[x-1][y-1] = CornerStartingHeight

	randomGen := rand.New(rand.NewSource(time.Now().UnixNano()))
	randomFactor := float64(Roughness * x / 10)
	fractalGeneration(heightMap, FractalParams{0, 0, x - 1, y - 1, randomFactor}, randomGen)

	return heightMap, nil
}

// Applies a fractal algorithm to a sub-section of the map
func fractalGeneration(heightMap [][]float64, params FractalParams,
	randomGen *rand.Rand) {
	// check for end of recursion
	fmt.Printf("%d %d %d %d \n", params.LowX, params.LowY, params.HighX, params.HighY)
	// if params.LowX >= params.HighX-1 || params.LowY >= params.HighY-1 {
	// 	fmt.Printf("Exit\n")
	// 	time.Sleep(1000 * time.Millisecond)
	// 	return
	// }

	// assign center value step
	var averageCenter = (heightMap[params.LowX][params.LowY] +
		heightMap[params.LowX][params.HighY] +
		heightMap[params.HighX][params.LowY] +
		heightMap[params.HighX][params.HighY]) / 4

	// we multiply the RandomFactor by 2 when we assign the center point because
	// the center points needs to be more randomized than the corner midpoints
	heightMap[(params.LowX+params.HighX)/2][(params.LowY+params.HighY)/2] = averageCenter +
		0 //(randomGen.NormFloat64() * params.RandomFactor * 2)

	xMidPoint := (params.LowX + params.HighX) / 2
	yMidPoint := (params.LowY + params.HighY) / 2

	// assign corner midpoints step
	heightMap[params.LowX][yMidPoint] = (heightMap[params.LowX][params.LowY]+
		heightMap[params.LowX][params.HighY])/2 + 0 //randomGen.NormFloat64()*params.RandomFactor

	heightMap[params.HighX][yMidPoint] = (heightMap[params.HighX][params.LowY]+
		heightMap[params.HighX][params.HighY])/2 + 0 //randomGen.NormFloat64()*params.RandomFactor

	heightMap[xMidPoint][params.LowY] = (heightMap[params.LowX][params.LowY]+
		heightMap[params.HighX][params.LowY])/2 + 0 //randomGen.NormFloat64()*params.RandomFactor

	heightMap[xMidPoint][params.HighY] = (heightMap[params.LowX][params.HighY]+
		heightMap[params.HighX][params.HighY])/2 + 0 //randomGen.NormFloat64()*params.RandomFactor

	// check for end of recursion
	// if params.HighX-params.LowX <= 2 {
	// 	return
	// }

	// Recalculate RandomFactor so it gets lower with each iteration
	// var newRandomFactor = params.RandomFactor

	// Recursively call fractal generation
	if params.HighX-params.LowX > 2 {
		fractalGeneration(heightMap, FractalParams{params.LowX, params.LowY,
			xMidPoint, yMidPoint,
			params.RandomFactor}, randomGen)

		fractalGeneration(heightMap, FractalParams{xMidPoint, params.LowY,
			params.HighX, yMidPoint,
			params.RandomFactor}, randomGen)

		fractalGeneration(heightMap, FractalParams{params.LowX, yMidPoint,
			xMidPoint, params.HighY,
			params.RandomFactor}, randomGen)

		fractalGeneration(heightMap, FractalParams{xMidPoint, yMidPoint,
			params.HighX, params.HighY,
			params.RandomFactor}, randomGen)
	}
}
<commit_msg>Remove unnecessary comments<commit_after>// TODO Change random number generation, and decrease number generation on succesive calls
// TODO Do no overwrite already existing values
// TODO Check not for odd numbers but for 2 pow n + 1
package main

import (
	"errors"
	"fmt"
	"math/rand"
	"time"
)

const CornerStartingHeight = 50
const Roughness = 4

type FractalParams struct {
	LowX         int
	LowY         int
	HighX        int
	HighY        int
	RandomFactor float64
}

// Generate an height map using a fractal algorithm
func generate(size int) ([][]float64, error) {
	// check if size is odd
	if size%2 == 0 {
		return nil, errors.New("Size must be odd.")
	}

	var x, y = size, size

	// Generate our slice to hold the height map data
	heightMap := make([][]float64, y, y)

	for i := range heightMap {
		heightMap[i] = make([]float64, x, x)
	}

	// Assign a basic value to each of the four corners
	heightMap[0][0] = CornerStartingHeight
	heightMap[0][y-1] = CornerStartingHeight
	heightMap[x-1][0] = CornerStartingHeight
	heightMap[x-1][y-1] = CornerStartingHeight

	randomGen := rand.New(rand.NewSource(time.Now().UnixNano()))
	randomFactor := float64(Roughness * x / 10)
	fractalGeneration(heightMap, FractalParams{0, 0, x - 1, y - 1, randomFactor}, randomGen)

	return heightMap, nil
}

// Applies a fractal algorithm to a sub-section of the map
func fractalGeneration(heightMap [][]float64, params FractalParams,
	randomGen *rand.Rand) {
	fmt.Printf("%d %d %d %d \n", params.LowX, params.LowY, params.HighX, params.HighY)

	// assign center value step
	var averageCenter = (heightMap[params.LowX][params.LowY] +
		heightMap[params.LowX][params.HighY] +
		heightMap[params.HighX][params.LowY] +
		heightMap[params.HighX][params.HighY]) / 4

	// we multiply the RandomFactor by 2 when we assign the center point because
	// the center points needs to be more randomized than the corner midpoints
	heightMap[(params.LowX+params.HighX)/2][(params.LowY+params.HighY)/2] = averageCenter +
		0 //(randomGen.NormFloat64() * params.RandomFactor * 2)

	xMidPoint := (params.LowX + params.HighX) / 2
	yMidPoint := (params.LowY + params.HighY) / 2

	// assign corner midpoints step
	heightMap[params.LowX][yMidPoint] = (heightMap[params.LowX][params.LowY]+
		heightMap[params.LowX][params.HighY])/2 + 0 //randomGen.NormFloat64()*params.RandomFactor

	heightMap[params.HighX][yMidPoint] = (heightMap[params.HighX][params.LowY]+
		heightMap[params.HighX][params.HighY])/2 + 0 //randomGen.NormFloat64()*params.RandomFactor

	heightMap[xMidPoint][params.LowY] = (heightMap[params.LowX][params.LowY]+
		heightMap[params.HighX][params.LowY])/2 + 0 //randomGen.NormFloat64()*params.RandomFactor

	heightMap[xMidPoint][params.HighY] = (heightMap[params.LowX][params.HighY]+
		heightMap[params.HighX][params.HighY])/2 + 0 //randomGen.NormFloat64()*params.RandomFactor

	// check for end of recursion
	// if params.HighX-params.LowX <= 2 {
	// 	return
	// }

	// Recalculate RandomFactor so it gets lower with each iteration
	// var newRandomFactor = params.RandomFactor

	// Recursively call fractal generation
	if params.HighX-params.LowX > 2 {
		fractalGeneration(heightMap, FractalParams{params.LowX, params.LowY,
			xMidPoint, yMidPoint,
			params.RandomFactor}, randomGen)

		fractalGeneration(heightMap, FractalParams{xMidPoint, params.LowY,
			params.HighX, yMidPoint,
			params.RandomFactor}, randomGen)

		fractalGeneration(heightMap, FractalParams{params.LowX, yMidPoint,
			xMidPoint, params.HighY,
			params.RandomFactor}, randomGen)

		fractalGeneration(heightMap, FractalParams{xMidPoint, yMidPoint,
			params.HighX, params.HighY,
			params.RandomFactor}, randomGen)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/kardianos/osext"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strings"
)

const app = "BrewTroller Build Bot"
const version = "0.1.0"

const SourceDir = "/BrewTroller"
const OptionsFileName = "./BrewTroller/options.json"

// Look for a run in debug mode flag, default to off
var debugMode = flag.Bool("debug", false, "Enables server debug mode")

func makeErrorResonse(code string, err error, context ...string) []byte {
	em := make(map[string]string)

	em["code"] = code

	//If we are running in debug mode use the actual error as the message
	if *debugMode {
		em["message"] = err.Error()
	} else {
		//Not in debug mode, use generic response
		switch code {
		case "500":
			em["message"] = "Internal Server Error"
		case "400":
			em["message"] = "Bad Request"
		}
	}

	if *debugMode {
		for i, v := range context {
			em[fmt.Sprintf("context%i", i)] = v
		}
	}

	//Encode the error reponse for transmission
	enc, _ := json.Marshal(em)

	return enc
}

func main() {
	flag.Parse()
	if *debugMode {
		fmt.Println("Debug mode enabled")
	}
	router := mux.NewRouter()
	router.HandleFunc("/", HomeHandler).Methods("GET")
	router.HandleFunc("/options", OptionsHandler).Methods("GET")
	router.HandleFunc("/build", BuildHandler).Methods("POST")
	http.ListenAndServe(":8080", router)
}

func HomeHandler(rw http.ResponseWriter, req *http.Request) {
	info := make(map[string]string)
	info["app"] = app
	info["version"] = version
	if *debugMode {
		c := exec.Command("uname", "-a")
		uname, _ := c.Output()
		info["host"] = string(uname)
	}
	rw.Header().Add("Content-Type", "application/json")
	encRes, _ := json.Marshal(info)
	rw.Write(encRes)
}

func OptionsHandler(rw http.ResponseWriter, req *http.Request) {
	//Read options file
	var opts, err = ioutil.ReadFile(OptionsFileName)
	if err != nil {
		rw.WriteHeader(http.StatusInternalServerError)
		errResp := makeErrorResonse("500", err)
		rw.Write(errResp)
	}
	rw.Header().Add("Content-Type", "application/json")
	rw.Write(opts)
}

func BuildHandler(rw http.ResponseWriter, req *http.Request) {
	//Generate a unique folder name to execute the build in
	// create a temp prefix with the requester addr, with '.' and ':' subbed
	reqID := strings.Replace(req.RemoteAddr, ".", "_", -1)
	reqID = strings.Replace(reqID, ":", "-", -1) + "-"
	tempDir, err := ioutil.TempDir("", reqID)

	//Handle error making temp build directory
	if err != nil {
		errResp := makeErrorResonse("500", err)
		rw.WriteHeader(http.StatusInternalServerError)
		rw.Write(errResp)
		return
	}
	//Clean-up the temp dir
	defer os.RemoveAll(tempDir)

	//Get request data
	reqData, err := ioutil.ReadAll(req.Body)

	//Handle error reading POST data
	if err != nil {
		errResp := makeErrorResonse("500", err)
		rw.WriteHeader(http.StatusInternalServerError)
		rw.Write(errResp)
		return
	}

	//Convert the post data to a map
	optsMap := make(map[string]string)
	err = json.Unmarshal(reqData, &optsMap)

	//Handle errors unmarshalling build options
	if err != nil {
		errResp := makeErrorResonse("400", err)
		rw.WriteHeader(http.StatusBadRequest)
		rw.Write(errResp)
		return
	}

	//Ensure we have a board option
	board, found := optsMap["board"]
	if !found {
		err := errors.New("Board Option Must be Supplied!")
		errResp := makeErrorResonse("400", err)
		rw.WriteHeader(http.StatusBadRequest)
		rw.Write(errResp)
		return
	}

	//Make a slice to hold the options, with an init len of 0 and a capacity of 20
	//   we start with a capacity of 20 to prevent having to initialize a new slice after every append
	cmakeOpts := make([]string, 0, 20)
	//iterate through the build options requested and make a slice to pass to cmake
	for k, v := range optsMap {
		opt := fmt.Sprintf("-D%s=%s", k, v)
		cmakeOpts = append(cmakeOpts, opt)
	}
	//Append the absolute path to the brewtroller source directory
	currDir, _ := osext.ExecutableFolder()
	pathToSource := currDir + SourceDir
	cmakeOpts = append(cmakeOpts, pathToSource)

	//Attempt to setup Cmake build dir
	cmakeCmd := exec.Command("cmake", cmakeOpts...)
	cmakeCmd.Dir = tempDir

	cmakeOut, err := cmakeCmd.CombinedOutput()
	//Handle cmake setup error
	if err != nil {
		errResp := makeErrorResonse("500", err, string(cmakeOut))
		rw.WriteHeader(http.StatusInternalServerError)
		rw.Write(errResp)
		return
	}

	//build the image(s) -- in the future we will build an eeprom image to upload
	makeCmd := exec.Command("make")
	makeCmd.Dir = tempDir
	makeOut, err := makeCmd.CombinedOutput()
	//Handle any errors from make
	if err != nil {
		errResp := makeErrorResonse("500", err, string(makeOut))
		rw.WriteHeader(http.StatusInternalServerError)
		rw.Write(errResp)
		return
	}

	//Grab the binary and read it
	binary, err := ioutil.ReadFile(tempDir + "/src/BrewTroller-" + board + ".hex")
	if err != nil {
		errResp := makeErrorResonse("500", err)
		rw.WriteHeader(http.StatusInternalServerError)
		rw.Write(errResp)
		return
	}

	//Create response map
	resp := make(map[string]string)

	if *debugMode {
		resp["reqID"] = reqID
		resp["buildLocation"] = tempDir
		resp["reqDat"] = string(reqData)
		resp["cmake-output"] = string(cmakeOut)
		resp["make-output"] = string(makeOut)
	}

	resp["binary"] = string(binary)

	enc, _ := json.Marshal(resp)
	rw.Header().Add("Content-Type", "application/json")
	rw.Write(enc)
}
<commit_msg>Make sure we are reading the options file from the path relative to the executable's location<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/kardianos/osext"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strings"
)

const app = "BrewTroller Build Bot"
const version = "0.1.0"

const SourceDir = "/BrewTroller"
const OptionsFileName = "/BrewTroller/options.json"

// Look for a run in debug mode flag, default to off
var debugMode = flag.Bool("debug", false, "Enables server debug mode")

func makeErrorResonse(code string, err error, context ...string) []byte {
	em := make(map[string]string)

	em["code"] = code

	//If we are running in debug mode use the actual error as the message
	if *debugMode {
		em["message"] = err.Error()
	} else {
		//Not in debug mode, use generic response
		switch code {
		case "500":
			em["message"] = "Internal Server Error"
		case "400":
			em["message"] = "Bad Request"
		}
	}

	if *debugMode {
		for i, v := range context {
			em[fmt.Sprintf("context%i", i)] = v
		}
	}

	//Encode the error reponse for transmission
	enc, _ := json.Marshal(em)

	return enc
}

func main() {
	flag.Parse()
	if *debugMode {
		fmt.Println("Debug mode enabled")
	}
	router := mux.NewRouter()
	router.HandleFunc("/", HomeHandler).Methods("GET")
	router.HandleFunc("/options", OptionsHandler).Methods("GET")
	router.HandleFunc("/build", BuildHandler).Methods("POST")
	http.ListenAndServe(":8080", router)
}

func HomeHandler(rw http.ResponseWriter, req *http.Request) {
	info := make(map[string]string)
	info["app"] = app
	info["version"] = version
	if *debugMode {
		c := exec.Command("uname", "-a")
		uname, _ := c.Output()
		info["host"] = string(uname)
	}
	rw.Header().Add("Content-Type", "application/json")
	encRes, _ := json.Marshal(info)
	rw.Write(encRes)
}

func OptionsHandler(rw http.ResponseWriter, req *http.Request) {
	//Read options file
	currDir, _ := osext.ExecutableFolder()
	var opts, err = ioutil.ReadFile(currDir + OptionsFileName)
	if err != nil {
		rw.WriteHeader(http.StatusInternalServerError)
		errResp := makeErrorResonse("500", err)
		rw.Write(errResp)
	}
	rw.Header().Add("Content-Type", "application/json")
	rw.Write(opts)
}

func BuildHandler(rw http.ResponseWriter, req *http.Request) {
	//Generate a unique folder name to execute the build in
	// create a temp prefix with the requester addr, with '.' and ':' subbed
	reqID := strings.Replace(req.RemoteAddr, ".", "_", -1)
	reqID = strings.Replace(reqID, ":", "-", -1) + "-"
	tempDir, err := ioutil.TempDir("", reqID)

	//Handle error making temp build directory
	if err != nil {
		errResp := makeErrorResonse("500", err)
		rw.WriteHeader(http.StatusInternalServerError)
		rw.Write(errResp)
		return
	}
	//Clean-up the temp dir
	defer os.RemoveAll(tempDir)

	//Get request data
	reqData, err := ioutil.ReadAll(req.Body)

	//Handle error reading POST data
	if err != nil {
		errResp := makeErrorResonse("500", err)
		rw.WriteHeader(http.StatusInternalServerError)
		rw.Write(errResp)
		return
	}

	//Convert the post data to a map
	optsMap := make(map[string]string)
	err = json.Unmarshal(reqData, &optsMap)

	//Handle errors unmarshalling build options
	if err != nil {
		errResp := makeErrorResonse("400", err)
		rw.WriteHeader(http.StatusBadRequest)
		rw.Write(errResp)
		return
	}

	//Ensure we have a board option
	board, found := optsMap["board"]
	if !found {
		err := errors.New("Board Option Must be Supplied!")
		errResp := makeErrorResonse("400", err)
		rw.WriteHeader(http.StatusBadRequest)
		rw.Write(errResp)
		return
	}

	//Make a slice to hold the options, with an init len of 0 and a capacity of 20
	//   we start with a capacity of 20 to prevent having to initialize a new slice after every append
	cmakeOpts := make([]string, 0, 20)
	//iterate through the build options requested and make a slice to pass to cmake
	for k, v := range optsMap {
		opt := fmt.Sprintf("-D%s=%s", k, v)
		cmakeOpts = append(cmakeOpts, opt)
	}
	//Append the absolute path to the brewtroller source directory
	currDir, _ := osext.ExecutableFolder()
	pathToSource := currDir + SourceDir
	cmakeOpts = append(cmakeOpts, pathToSource)

	//Attempt to setup Cmake build dir
	cmakeCmd := exec.Command("cmake", cmakeOpts...)
	cmakeCmd.Dir = tempDir

	cmakeOut, err := cmakeCmd.CombinedOutput()
	//Handle cmake setup error
	if err != nil {
		errResp := makeErrorResonse("500", err, string(cmakeOut))
		rw.WriteHeader(http.StatusInternalServerError)
		rw.Write(errResp)
		return
	}

	//build the image(s) -- in the future we will build an eeprom image to upload
	makeCmd := exec.Command("make")
	makeCmd.Dir = tempDir
	makeOut, err := makeCmd.CombinedOutput()
	//Handle any errors from make
	if err != nil {
		errResp := makeErrorResonse("500", err, string(makeOut))
		rw.WriteHeader(http.StatusInternalServerError)
		rw.Write(errResp)
		return
	}

	//Grab the binary and read it
	binary, err := ioutil.ReadFile(tempDir + "/src/BrewTroller-" + board + ".hex")
	if err != nil {
		errResp := makeErrorResonse("500", err)
		rw.WriteHeader(http.StatusInternalServerError)
		rw.Write(errResp)
		return
	}

	//Create response map
	resp := make(map[string]string)

	if *debugMode {
		resp["reqID"] = reqID
		resp["buildLocation"] = tempDir
		resp["reqDat"] = string(reqData)
		resp["cmake-output"] = string(cmakeOut)
		resp["make-output"] = string(makeOut)
	}

	resp["binary"] = string(binary)

	enc, _ := json.Marshal(resp)
	rw.Header().Add("Content-Type", "application/json")
	rw.Write(enc)
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main_test

import (
	"bytes"
	"errors"
	"fmt"
	"internal/testenv"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"testing"
)

const (
	dataDir = "testdata"
	binary  = "./testvet.exe"
)

// We implement TestMain so remove the test binary when all is done.
func TestMain(m *testing.M) {
	result := m.Run()
	os.Remove(binary)
	os.Exit(result)
}

var (
	buildMu sync.Mutex // guards following
	built   = false    // We have built the binary.
	failed  = false    // We have failed to build the binary, don't try again.
)

func Build(t *testing.T) {
	buildMu.Lock()
	defer buildMu.Unlock()
	if built {
		return
	}
	if failed {
		t.Skip("cannot run on this environment")
	}
	testenv.MustHaveGoBuild(t)
	cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", binary)
	output, err := cmd.CombinedOutput()
	if err != nil {
		failed = true
		fmt.Fprintf(os.Stderr, "%s\n", output)
		t.Fatal(err)
	}
	built = true
}

func vetCmd(t *testing.T, args ...string) *exec.Cmd {
	cmd := exec.Command(testenv.GoToolPath(t), "vet", "-vettool="+binary)
	cmd.Args = append(cmd.Args, args...)
	testdata, err := filepath.Abs("testdata")
	if err != nil {
		t.Fatal(err)
	}
	cmd.Env = append(os.Environ(), "GOPATH="+testdata)
	return cmd
}

func TestVet(t *testing.T) {
	t.Parallel()
	Build(t)
	for _, pkg := range []string{
		"asm",
		"assign",
		"atomic",
		"bool",
		"buildtag",
		"cgo",
		"composite",
		"copylock",
		"deadcode",
		"httpresponse",
		"lostcancel",
		"method",
		"nilfunc",
		"print",
		"rangeloop",
		"shift",
		"structtag",
		"testingpkg",
		// "testtag" has its own test
		"unmarshal",
		"unsafeptr",
		"unused",
	} {
		pkg := pkg
		t.Run(pkg, func(t *testing.T) {
			t.Parallel()

			// Skip cgo test on platforms without cgo.
			if pkg == "cgo" && !cgoEnabled(t) {
				return
			}

			cmd := vetCmd(t, "-printfuncs=Warn,Warnf", pkg)

			// The asm test assumes amd64.
			if pkg == "asm" {
				cmd.Env = append(cmd.Env, "GOOS=linux", "GOARCH=amd64")
			}

			dir := filepath.Join("testdata/src", pkg)
			gos, err := filepath.Glob(filepath.Join(dir, "*.go"))
			if err != nil {
				t.Fatal(err)
			}
			asms, err := filepath.Glob(filepath.Join(dir, "*.s"))
			if err != nil {
				t.Fatal(err)
			}
			var files []string
			files = append(files, gos...)
			files = append(files, asms...)

			errchk(cmd, files, t)
		})
	}
}

func cgoEnabled(t *testing.T) bool {
	// Don't trust build.Default.CgoEnabled as it is false for
	// cross-builds unless CGO_ENABLED is explicitly specified.
	// That's fine for the builders, but causes commands like
	// 'GOARCH=386 go test .' to fail.
	// Instead, we ask the go command.
	cmd := exec.Command(testenv.GoToolPath(t), "list", "-f", "{{context.CgoEnabled}}")
	out, _ := cmd.CombinedOutput()
	return string(out) == "true\n"
}

func errchk(c *exec.Cmd, files []string, t *testing.T) {
	output, err := c.CombinedOutput()
	if _, ok := err.(*exec.ExitError); !ok {
		t.Logf("vet output:\n%s", output)
		t.Fatal(err)
	}
	fullshort := make([]string, 0, len(files)*2)
	for _, f := range files {
		fullshort = append(fullshort, f, filepath.Base(f))
	}
	err = errorCheck(string(output), false, fullshort...)
	if err != nil {
		t.Errorf("error check failed: %s", err)
	}
}

// TestTags verifies that the -tags argument controls which files to check.
func TestTags(t *testing.T) {
	t.Parallel()
	Build(t)
	for tag, wantFile := range map[string]int{
		"testtag":     1, // file1
		"x testtag y": 1,
		"othertag":    2,
	} {
		tag, wantFile := tag, wantFile
		t.Run(tag, func(t *testing.T) {
			t.Parallel()
			t.Logf("-tags=%s", tag)
			cmd := vetCmd(t, "-tags="+tag, "tagtest")
			output, err := cmd.CombinedOutput()

			want := fmt.Sprintf("file%d.go", wantFile)
			dontwant := fmt.Sprintf("file%d.go", 3-wantFile)

			// file1 has testtag and file2 has !testtag.
			if !bytes.Contains(output, []byte(filepath.Join("tagtest", want))) {
				t.Errorf("%s: %s was excluded, should be included", tag, want)
			}
			if bytes.Contains(output, []byte(filepath.Join("tagtest", dontwant))) {
				t.Errorf("%s: %s was included, should be excluded", tag, dontwant)
			}
			if t.Failed() {
				t.Logf("err=%s, output=<<%s>>", err, output)
			}
		})
	}
}

// All declarations below were adapted from test/run.go.

// errorCheck matches errors in outStr against comments in source files.
// For each line of the source files which should generate an error,
// there should be a comment of the form // ERROR "regexp".
// If outStr has an error for a line which has no such comment,
// this function will report an error.
// Likewise if outStr does not have an error for a line which has a comment,
// or if the error message does not match the <regexp>.
// The <regexp> syntax is Perl but it's best to stick to egrep.
//
// Sources files are supplied as fullshort slice.
// It consists of pairs: full path to source file and its base name.
func errorCheck(outStr string, wantAuto bool, fullshort ...string) (err error) {
	var errs []error
	out := splitOutput(outStr, wantAuto)
	// Cut directory name.
	for i := range out {
		for j := 0; j < len(fullshort); j += 2 {
			full, short := fullshort[j], fullshort[j+1]
			out[i] = strings.ReplaceAll(out[i], full, short)
		}
	}

	var want []wantedError
	for j := 0; j < len(fullshort); j += 2 {
		full, short := fullshort[j], fullshort[j+1]
		want = append(want, wantedErrors(full, short)...)
	}
	for _, we := range want {
		var errmsgs []string
		if we.auto {
			errmsgs, out = partitionStrings("<autogenerated>", out)
		} else {
			errmsgs, out = partitionStrings(we.prefix, out)
		}
		if len(errmsgs) == 0 {
			errs = append(errs, fmt.Errorf("%s:%d: missing error %q", we.file, we.lineNum, we.reStr))
			continue
		}
		matched := false
		n := len(out)
		for _, errmsg := range errmsgs {
			// Assume errmsg says "file:line: foo".
			// Cut leading "file:line: " to avoid accidental matching of file name instead of message.
			text := errmsg
			if i := strings.Index(text, " "); i >= 0 {
				text = text[i+1:]
			}
			if we.re.MatchString(text) {
				matched = true
			} else {
				out = append(out, errmsg)
			}
		}
		if !matched {
			errs = append(errs, fmt.Errorf("%s:%d: no match for %#q in:\n\t%s", we.file, we.lineNum, we.reStr, strings.Join(out[n:], "\n\t")))
			continue
		}
	}

	if len(out) > 0 {
		errs = append(errs, fmt.Errorf("Unmatched Errors:"))
		for _, errLine := range out {
			errs = append(errs, fmt.Errorf("%s", errLine))
		}
	}

	if len(errs) == 0 {
		return nil
	}
	if len(errs) == 1 {
		return errs[0]
	}
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "\n")
	for _, err := range errs {
		fmt.Fprintf(&buf, "%s\n", err.Error())
	}
	return errors.New(buf.String())
}

func splitOutput(out string, wantAuto bool) []string {
	// gc error messages continue onto additional lines with leading tabs.
	// Split the output at the beginning of each line that doesn't begin with a tab.
	// <autogenerated> lines are impossible to match so those are filtered out.
	var res []string
	for _, line := range strings.Split(out, "\n") {
		line = strings.TrimSuffix(line, "\r") // normalize Windows output
		if strings.HasPrefix(line, "\t") {
			res[len(res)-1] += "\n" + line
		} else if strings.HasPrefix(line, "go tool") || strings.HasPrefix(line, "#") || !wantAuto && strings.HasPrefix(line, "<autogenerated>") {
			continue
		} else if strings.TrimSpace(line) != "" {
			res = append(res, line)
		}
	}
	return res
}

// matchPrefix reports whether s starts with file name prefix followed by a :,
// and possibly preceded by a directory name.
func matchPrefix(s, prefix string) bool {
	i := strings.Index(s, ":")
	if i < 0 {
		return false
	}
	j := strings.LastIndex(s[:i], "/")
	s = s[j+1:]
	if len(s) <= len(prefix) || s[:len(prefix)] != prefix {
		return false
	}
	if s[len(prefix)] == ':' {
		return true
	}
	return false
}

func partitionStrings(prefix string, strs []string) (matched, unmatched []string) {
	for _, s := range strs {
		if matchPrefix(s, prefix) {
			matched = append(matched, s)
		} else {
			unmatched = append(unmatched, s)
		}
	}
	return
}

type wantedError struct {
	reStr   string
	re      *regexp.Regexp
	lineNum int
	auto    bool // match <autogenerated> line
	file    string
	prefix  string
}

var (
	errRx       = regexp.MustCompile(`// (?:GC_)?ERROR (.*)`)
	errAutoRx   = regexp.MustCompile(`// (?:GC_)?ERRORAUTO (.*)`)
	errQuotesRx = regexp.MustCompile(`"([^"]*)"`)
	lineRx      = regexp.MustCompile(`LINE(([+-])([0-9]+))?`)
)

// wantedErrors parses expected errors from comments in a file.
func wantedErrors(file, short string) (errs []wantedError) {
	cache := make(map[string]*regexp.Regexp)

	src, err := ioutil.ReadFile(file)
	if err != nil {
		log.Fatal(err)
	}
	for i, line := range strings.Split(string(src), "\n") {
		lineNum := i + 1
		if strings.Contains(line, "////") {
			// double comment disables ERROR
			continue
		}
		var auto bool
		m := errAutoRx.FindStringSubmatch(line)
		if m != nil {
			auto = true
		} else {
			m = errRx.FindStringSubmatch(line)
		}
		if m == nil {
			continue
		}
		all := m[1]
		mm := errQuotesRx.FindAllStringSubmatch(all, -1)
		if mm == nil {
			log.Fatalf("%s:%d: invalid errchk line: %s", file, lineNum, line)
		}
		for _, m := range mm {
			replacedOnce := false
			rx := lineRx.ReplaceAllStringFunc(m[1], func(m string) string {
				if replacedOnce {
					return m
				}
				replacedOnce = true
				n := lineNum
				if strings.HasPrefix(m, "LINE+") {
					delta, _ := strconv.Atoi(m[5:])
					n += delta
				} else if strings.HasPrefix(m, "LINE-") {
					delta, _ := strconv.Atoi(m[5:])
					n -= delta
				}
				return fmt.Sprintf("%s:%d", short, n)
			})
			re := cache[rx]
			if re == nil {
				var err error
				re, err = regexp.Compile(rx)
				if err != nil {
					log.Fatalf("%s:%d: invalid regexp \"%#q\" in ERROR line: %v", file, lineNum, rx, err)
				}
				cache[rx] = re
			}
			prefix := fmt.Sprintf("%s:%d", short, lineNum)
			errs = append(errs, wantedError{
				reStr:   rx,
				re:      re,
				prefix:  prefix,
				auto:    auto,
				lineNum: lineNum,
				file:    short,
			})
		}
	}

	return
}
<commit_msg>cmd/vet: do not write test vet binary to GOROOT<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main_test

import (
	"bytes"
	"errors"
	"fmt"
	"internal/testenv"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"testing"
)

const dataDir = "testdata"

var binary string

// We implement TestMain so remove the test binary when all is done.
func TestMain(m *testing.M) {
	dir, err := ioutil.TempDir("", "vet_test")
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	defer os.RemoveAll(dir)
	binary = filepath.Join(dir, "testvet.exe")

	result := m.Run()
	os.Exit(result)
}

var (
	buildMu sync.Mutex // guards following
	built   = false    // We have built the binary.
	failed  = false    // We have failed to build the binary, don't try again.
)

func Build(t *testing.T) {
	buildMu.Lock()
	defer buildMu.Unlock()
	if built {
		return
	}
	if failed {
		t.Skip("cannot run on this environment")
	}
	testenv.MustHaveGoBuild(t)
	cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", binary)
	output, err := cmd.CombinedOutput()
	if err != nil {
		failed = true
		fmt.Fprintf(os.Stderr, "%s\n", output)
		t.Fatal(err)
	}
	built = true
}

func vetCmd(t *testing.T, args ...string) *exec.Cmd {
	cmd := exec.Command(testenv.GoToolPath(t), "vet", "-vettool="+binary)
	cmd.Args = append(cmd.Args, args...)
	testdata, err := filepath.Abs("testdata")
	if err != nil {
		t.Fatal(err)
	}
	cmd.Env = append(os.Environ(), "GOPATH="+testdata)
	return cmd
}

func TestVet(t *testing.T) {
	t.Parallel()
	Build(t)
	for _, pkg := range []string{
		"asm",
		"assign",
		"atomic",
		"bool",
		"buildtag",
		"cgo",
		"composite",
		"copylock",
		"deadcode",
		"httpresponse",
		"lostcancel",
		"method",
		"nilfunc",
		"print",
		"rangeloop",
		"shift",
		"structtag",
		"testingpkg",
		// "testtag" has its own test
		"unmarshal",
		"unsafeptr",
		"unused",
	} {
		pkg := pkg
		t.Run(pkg, func(t *testing.T) {
			t.Parallel()

			// Skip cgo test on platforms without cgo.
			if pkg == "cgo" && !cgoEnabled(t) {
				return
			}

			cmd := vetCmd(t, "-printfuncs=Warn,Warnf", pkg)

			// The asm test assumes amd64.
			if pkg == "asm" {
				cmd.Env = append(cmd.Env, "GOOS=linux", "GOARCH=amd64")
			}

			dir := filepath.Join("testdata/src", pkg)
			gos, err := filepath.Glob(filepath.Join(dir, "*.go"))
			if err != nil {
				t.Fatal(err)
			}
			asms, err := filepath.Glob(filepath.Join(dir, "*.s"))
			if err != nil {
				t.Fatal(err)
			}
			var files []string
			files = append(files, gos...)
			files = append(files, asms...)

			errchk(cmd, files, t)
		})
	}
}

func cgoEnabled(t *testing.T) bool {
	// Don't trust build.Default.CgoEnabled as it is false for
	// cross-builds unless CGO_ENABLED is explicitly specified.
	// That's fine for the builders, but causes commands like
	// 'GOARCH=386 go test .' to fail.
	// Instead, we ask the go command.
	cmd := exec.Command(testenv.GoToolPath(t), "list", "-f", "{{context.CgoEnabled}}")
	out, _ := cmd.CombinedOutput()
	return string(out) == "true\n"
}

func errchk(c *exec.Cmd, files []string, t *testing.T) {
	output, err := c.CombinedOutput()
	if _, ok := err.(*exec.ExitError); !ok {
		t.Logf("vet output:\n%s", output)
		t.Fatal(err)
	}
	fullshort := make([]string, 0, len(files)*2)
	for _, f := range files {
		fullshort = append(fullshort, f, filepath.Base(f))
	}
	err = errorCheck(string(output), false, fullshort...)
	if err != nil {
		t.Errorf("error check failed: %s", err)
	}
}

// TestTags verifies that the -tags argument controls which files to check.
func TestTags(t *testing.T) {
	t.Parallel()
	Build(t)
	for tag, wantFile := range map[string]int{
		"testtag":     1, // file1
		"x testtag y": 1,
		"othertag":    2,
	} {
		tag, wantFile := tag, wantFile
		t.Run(tag, func(t *testing.T) {
			t.Parallel()
			t.Logf("-tags=%s", tag)
			cmd := vetCmd(t, "-tags="+tag, "tagtest")
			output, err := cmd.CombinedOutput()

			want := fmt.Sprintf("file%d.go", wantFile)
			dontwant := fmt.Sprintf("file%d.go", 3-wantFile)

			// file1 has testtag and file2 has !testtag.
			if !bytes.Contains(output, []byte(filepath.Join("tagtest", want))) {
				t.Errorf("%s: %s was excluded, should be included", tag, want)
			}
			if bytes.Contains(output, []byte(filepath.Join("tagtest", dontwant))) {
				t.Errorf("%s: %s was included, should be excluded", tag, dontwant)
			}
			if t.Failed() {
				t.Logf("err=%s, output=<<%s>>", err, output)
			}
		})
	}
}

// All declarations below were adapted from test/run.go.

// errorCheck matches errors in outStr against comments in source files.
// For each line of the source files which should generate an error,
// there should be a comment of the form // ERROR "regexp".
// If outStr has an error for a line which has no such comment,
// this function will report an error.
// Likewise if outStr does not have an error for a line which has a comment,
// or if the error message does not match the <regexp>.
// The <regexp> syntax is Perl but it's best to stick to egrep.
//
// Sources files are supplied as fullshort slice.
// It consists of pairs: full path to source file and its base name.
func errorCheck(outStr string, wantAuto bool, fullshort ...string) (err error) {
	var errs []error
	out := splitOutput(outStr, wantAuto)
	// Cut directory name.
	for i := range out {
		for j := 0; j < len(fullshort); j += 2 {
			full, short := fullshort[j], fullshort[j+1]
			out[i] = strings.ReplaceAll(out[i], full, short)
		}
	}

	var want []wantedError
	for j := 0; j < len(fullshort); j += 2 {
		full, short := fullshort[j], fullshort[j+1]
		want = append(want, wantedErrors(full, short)...)
	}
	for _, we := range want {
		var errmsgs []string
		if we.auto {
			errmsgs, out = partitionStrings("<autogenerated>", out)
		} else {
			errmsgs, out = partitionStrings(we.prefix, out)
		}
		if len(errmsgs) == 0 {
			errs = append(errs, fmt.Errorf("%s:%d: missing error %q", we.file, we.lineNum, we.reStr))
			continue
		}
		matched := false
		n := len(out)
		for _, errmsg := range errmsgs {
			// Assume errmsg says "file:line: foo".
			// Cut leading "file:line: " to avoid accidental matching of file name instead of message.
			text := errmsg
			if i := strings.Index(text, " "); i >= 0 {
				text = text[i+1:]
			}
			if we.re.MatchString(text) {
				matched = true
			} else {
				out = append(out, errmsg)
			}
		}
		if !matched {
			errs = append(errs, fmt.Errorf("%s:%d: no match for %#q in:\n\t%s", we.file, we.lineNum, we.reStr, strings.Join(out[n:], "\n\t")))
			continue
		}
	}

	if len(out) > 0 {
		errs = append(errs, fmt.Errorf("Unmatched Errors:"))
		for _, errLine := range out {
			errs = append(errs, fmt.Errorf("%s", errLine))
		}
	}

	if len(errs) == 0 {
		return nil
	}
	if len(errs) == 1 {
		return errs[0]
	}
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "\n")
	for _, err := range errs {
		fmt.Fprintf(&buf, "%s\n", err.Error())
	}
	return errors.New(buf.String())
}

func splitOutput(out string, wantAuto bool) []string {
	// gc error messages continue onto additional lines with leading tabs.
	// Split the output at the beginning of each line that doesn't begin with a tab.
	// <autogenerated> lines are impossible to match so those are filtered out.
	var res []string
	for _, line := range strings.Split(out, "\n") {
		line = strings.TrimSuffix(line, "\r") // normalize Windows output
		if strings.HasPrefix(line, "\t") {
			res[len(res)-1] += "\n" + line
		} else if strings.HasPrefix(line, "go tool") || strings.HasPrefix(line, "#") || !wantAuto && strings.HasPrefix(line, "<autogenerated>") {
			continue
		} else if strings.TrimSpace(line) != "" {
			res = append(res, line)
		}
	}
	return res
}

// matchPrefix reports whether s starts with file name prefix followed by a :,
// and possibly preceded by a directory name.
func matchPrefix(s, prefix string) bool {
	i := strings.Index(s, ":")
	if i < 0 {
		return false
	}
	j := strings.LastIndex(s[:i], "/")
	s = s[j+1:]
	if len(s) <= len(prefix) || s[:len(prefix)] != prefix {
		return false
	}
	if s[len(prefix)] == ':' {
		return true
	}
	return false
}

func partitionStrings(prefix string, strs []string) (matched, unmatched []string) {
	for _, s := range strs {
		if matchPrefix(s, prefix) {
			matched = append(matched, s)
		} else {
			unmatched = append(unmatched, s)
		}
	}
	return
}

type wantedError struct {
	reStr   string
	re      *regexp.Regexp
	lineNum int
	auto    bool // match <autogenerated> line
	file    string
	prefix  string
}

var (
	errRx       = regexp.MustCompile(`// (?:GC_)?ERROR (.*)`)
	errAutoRx   = regexp.MustCompile(`// (?:GC_)?ERRORAUTO (.*)`)
	errQuotesRx = regexp.MustCompile(`"([^"]*)"`)
	lineRx      = regexp.MustCompile(`LINE(([+-])([0-9]+))?`)
)

// wantedErrors parses expected errors from comments in a file.
func wantedErrors(file, short string) (errs []wantedError) {
	cache := make(map[string]*regexp.Regexp)

	src, err := ioutil.ReadFile(file)
	if err != nil {
		log.Fatal(err)
	}
	for i, line := range strings.Split(string(src), "\n") {
		lineNum := i + 1
		if strings.Contains(line, "////") {
			// double comment disables ERROR
			continue
		}
		var auto bool
		m := errAutoRx.FindStringSubmatch(line)
		if m != nil {
			auto = true
		} else {
			m = errRx.FindStringSubmatch(line)
		}
		if m == nil {
			continue
		}
		all := m[1]
		mm := errQuotesRx.FindAllStringSubmatch(all, -1)
		if mm == nil {
			log.Fatalf("%s:%d: invalid errchk line: %s", file, lineNum, line)
		}
		for _, m := range mm {
			replacedOnce := false
			rx := lineRx.ReplaceAllStringFunc(m[1], func(m string) string {
				if replacedOnce {
					return m
				}
				replacedOnce = true
				n := lineNum
				if strings.HasPrefix(m, "LINE+") {
					delta, _ := strconv.Atoi(m[5:])
					n += delta
				} else if strings.HasPrefix(m, "LINE-") {
					delta, _ := strconv.Atoi(m[5:])
					n -= delta
				}
				return fmt.Sprintf("%s:%d", short, n)
			})
			re := cache[rx]
			if re == nil {
				var err error
				re, err = regexp.Compile(rx)
				if err != nil {
					log.Fatalf("%s:%d: invalid regexp \"%#q\" in ERROR line: %v", file, lineNum, rx, err)
				}
				cache[rx] = re
			}
			prefix := fmt.Sprintf("%s:%d", short, lineNum)
			errs = append(errs, wantedError{
				reStr:   rx,
				re:      re,
				prefix:  prefix,
				auto:    auto,
				lineNum: lineNum,
				file:    short,
			})
		}
	}

	return
}
<|endoftext|>


<commit_before>// +build ignore

// Run tests for all the remotes
//
// Run with go run test_all.go
package main

import (
	"flag"
	"log"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strings"
	"time"

	"github.com/ncw/rclone/fs"
	_ "github.com/ncw/rclone/fs/all" // import all fs
	"github.com/ncw/rclone/fstest"
)

type remoteConfig struct {
	Name     string
	SubDir   bool
	FastList bool
}

var (
	remotes = []remoteConfig{
		{
			Name:     "TestAmazonCloudDrive:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestB2:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestCryptDrive:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestCryptSwift:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestDrive:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestDropbox:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestGoogleCloudStorage:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestHubic:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestOneDrive:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestS3:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestSftp:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestSwift:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestYandex:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestFTP:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestBox:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestQingStor:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestAzureBlob:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestPcloud:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestWebdav:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestCache:",
			SubDir:   true,
			FastList: true,
		},
	}
	binary = "fs.test"
	// Flags
	maxTries = flag.Int("maxtries", 5, "Number of times to try each test")
	runTests = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'")
	clean    = flag.Bool("clean", false, "Instead of testing, clean all left over test directories")
	runOnly  = flag.String("run-only", "", "Run only those tests matching the regexp supplied")
)

// test holds info about a running test
type test struct {
	remote      string
	subdir      bool
	cmdLine     []string
	cmdString   string
	try         int
	err         error
	output      []byte
	failedTests []string
	runFlag     string
}

// newTest creates a new test
func newTest(remote string, subdir bool, fastlist bool) *test {
	t := &test{
		remote:  remote,
		subdir:  subdir,
		cmdLine: []string{"./" + binary, "-remote", remote},
		try:     1,
	}
	if *fstest.Verbose {
		t.cmdLine = append(t.cmdLine, "-test.v")
		fs.Config.LogLevel = fs.LogLevelDebug
	}
	if *runOnly != "" {
		t.cmdLine = append(t.cmdLine, "-test.run", *runOnly)
	}
	if subdir {
		t.cmdLine = append(t.cmdLine, "-subdir")
	}
	if fastlist {
		t.cmdLine = append(t.cmdLine, "-fast-list")
	}
	t.cmdString = toShell(t.cmdLine)
	return t
}

// dumpOutput prints the error output
func (t *test) dumpOutput() {
	log.Println("------------------------------------------------------------")
	log.Printf("---- %q ----", t.cmdString)
	log.Println(string(t.output))
	log.Println("------------------------------------------------------------")
}

var failRe = regexp.MustCompile(`(?m)^--- FAIL: (Test\w*) \(`)

// findFailures looks for all the tests which failed
func (t *test) findFailures() {
	oldFailedTests := t.failedTests
	t.failedTests = nil
	for _, matches := range failRe.FindAllSubmatch(t.output, -1) {
		t.failedTests = append(t.failedTests, string(matches[1]))
	}
	if len(t.failedTests) != 0 {
		t.runFlag = "^(" + strings.Join(t.failedTests, "|") + ")$"
	} else {
		t.runFlag = ""
	}
	if t.passed() && len(t.failedTests) != 0 {
		log.Printf("%q - Expecting no errors but got: %v", t.cmdString, t.failedTests)
		t.dumpOutput()
	} else if !t.passed() && len(t.failedTests) == 0 {
		log.Printf("%q - Expecting errors but got none: %v", t.cmdString, t.failedTests)
		t.dumpOutput()
		t.failedTests = oldFailedTests
	}
}

// nextCmdLine returns the next command line
func (t *test) nextCmdLine() []string {
	cmdLine := t.cmdLine[:]
	if t.runFlag != "" {
		cmdLine = append(cmdLine, "-test.run", t.runFlag)
	}
	return cmdLine
}

// if matches then is definitely OK in the shell
var shellOK = regexp.MustCompile("^[A-Za-z0-9./_:-]+$")

// converts a argv style input into a shell command
func toShell(args []string) (result string) {
	for _, arg := range args {
		if result != "" {
			result += " "
		}
		if shellOK.MatchString(arg) {
			result += arg
		} else {
			result += "'" + arg + "'"
		}
	}
	return result
}

// trial runs a single test
func (t *test) trial() {
	cmdLine := t.nextCmdLine()
	cmdString := toShell(cmdLine)
	log.Printf("%q - Starting (try %d/%d)", cmdString, t.try, *maxTries)
	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
	start := time.Now()
	t.output, t.err = cmd.CombinedOutput()
	duration := time.Since(start)
	t.findFailures()
	if t.passed() {
		log.Printf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, t.try, *maxTries)
	} else {
		log.Printf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, t.try, *maxTries, t.err, t.failedTests)
	}
}

// cleanFs runs a single clean fs for left over directories
func (t *test) cleanFs() error {
	f, err := fs.NewFs(t.remote)
	if err != nil {
		return err
	}
	entries, err := fs.ListDirSorted(f, true, "")
	if err != nil {
		return err
	}
	return entries.ForDirError(func(dir fs.Directory) error {
		remote := dir.Remote()
		if fstest.MatchTestRemote.MatchString(remote) {
			log.Printf("Purging %s%s", t.remote, remote)
			dir, err := fs.NewFs(t.remote + remote)
			if err != nil {
				return err
			}
			return fs.Purge(dir)
		}
		return nil
	})
}

// clean runs a single clean on a fs for left over directories
func (t *test) clean() {
	log.Printf("%q - Starting clean (try %d/%d)", t.remote, t.try, *maxTries)
	start := time.Now()
	t.err = t.cleanFs()
	if t.err != nil {
		log.Printf("%q - Failed to purge %v", t.remote, t.err)
	}
	duration := time.Since(start)
	if t.passed() {
		log.Printf("%q - Finished OK in %v (try %d/%d)", t.cmdString, duration, t.try, *maxTries)
	} else {
		log.Printf("%q - Finished ERROR in %v (try %d/%d): %v", t.cmdString, duration, t.try, *maxTries, t.err)
	}
}

// passed returns true if the test passed
func (t *test) passed() bool {
	return t.err == nil
}

// run runs all the trials for this test
func (t *test) run(result chan<- *test) {
	for t.try = 1; t.try <= *maxTries; t.try++ {
		if *clean {
			if !t.subdir {
				t.clean()
			}
		} else {
			t.trial()
		}
		if t.passed() {
			break
		}
	}
	if !t.passed() {
		t.dumpOutput()
	}
	result <- t
}

// makeTestBinary makes the binary we will run
func makeTestBinary() {
	if runtime.GOOS == "windows" {
		binary += ".exe"
	}
	log.Printf("Making test binary %q", binary)
	err := exec.Command("go", "test", "-c", "-o", binary).Run()
	if err != nil {
		log.Fatalf("Failed to make test binary: %v", err)
	}
	if _, err := os.Stat(binary); err != nil {
		log.Fatalf("Couldn't find test binary %q", binary)
	}
}

// removeTestBinary removes the binary made in makeTestBinary
func removeTestBinary() {
	err := os.Remove(binary) // Delete the binary when finished
	if err != nil {
		log.Printf("Error removing test binary %q: %v", binary, err)
	}
}

func main() {
	flag.Parse()
	if *runTests != "" {
		newRemotes := []remoteConfig{}
		for _, name := range strings.Split(*runTests, ",") {
			for i := range remotes {
				if remotes[i].Name == name {
					newRemotes = append(newRemotes, remotes[i])
					goto found
				}
			}
			log.Printf("Remote %q not found - inserting with default flags", name)
			newRemotes = append(newRemotes, remoteConfig{Name: name})
		found:
		}
		remotes = newRemotes
	}
	var names []string
	for _, remote := range remotes {
		names = append(names, remote.Name)
	}
	log.Printf("Testing remotes: %s", strings.Join(names, ", "))

	start := time.Now()
	if *clean {
		fs.LoadConfig()
	} else {
		makeTestBinary()
		defer removeTestBinary()
	}

	// start the tests
	results := make(chan *test, 8)
	awaiting := 0
	bools := []bool{false, true}
	if *clean {
		// Don't run -subdir and -fast-list if -clean
		bools = bools[:1]
	}
	for _, remote := range remotes {
		for _, subdir := range bools {
			for _, fastlist := range bools {
				if (!subdir || subdir && remote.SubDir) && (!fastlist || fastlist && remote.FastList) {
					go newTest(remote.Name, subdir, fastlist).run(results)
					awaiting++
				}
			}
		}
	}

	// Wait for the tests to finish
	var failed []*test
	for ; awaiting > 0; awaiting-- {
		t := <-results
		if !t.passed() {
			failed = append(failed, t)
		}
	}
	duration := time.Since(start)

	// Summarise results
	if len(failed) == 0 {
		log.Printf("PASS: All tests finished OK in %v", duration)
	} else {
		log.Printf("FAIL: %d tests failed in %v", len(failed), duration)
		for _, t := range failed {
			log.Printf("  * %s", toShell(t.nextCmdLine()))
			log.Printf("    * Failed tests: %v", t.failedTests)
		}
		os.Exit(1)
	}
}
<commit_msg>fs: integration tests: add 30 minute timeout per test<commit_after>// +build ignore

// Run tests for all the remotes
//
// Run with go run test_all.go
package main

import (
	"flag"
	"log"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strings"
	"time"

	"github.com/ncw/rclone/fs"
	_ "github.com/ncw/rclone/fs/all" // import all fs
	"github.com/ncw/rclone/fstest"
)

type remoteConfig struct {
	Name     string
	SubDir   bool
	FastList bool
}

var (
	remotes = []remoteConfig{
		{
			Name:     "TestAmazonCloudDrive:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestB2:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestCryptDrive:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestCryptSwift:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestDrive:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestDropbox:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestGoogleCloudStorage:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestHubic:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestOneDrive:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestS3:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestSftp:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestSwift:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestYandex:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestFTP:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestBox:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestQingStor:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestAzureBlob:",
			SubDir:   true,
			FastList: true,
		},
		{
			Name:     "TestPcloud:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestWebdav:",
			SubDir:   false,
			FastList: false,
		},
		{
			Name:     "TestCache:",
			SubDir:   true,
			FastList: true,
		},
	}
	binary = "fs.test"
	// Flags
	maxTries = flag.Int("maxtries", 5, "Number of times to try each test")
	runTests = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'")
	clean    = flag.Bool("clean", false, "Instead of testing, clean all left over test directories")
	runOnly  = flag.String("run-only", "", "Run only those tests matching the regexp supplied")
	timeout  = flag.Duration("timeout", 30*time.Minute, "Maximum time to run each test for before giving up")
)

// test holds info about a running test
type test struct {
	remote      string
	subdir      bool
	cmdLine     []string
	cmdString   string
	try         int
	err         error
	output      []byte
	failedTests []string
	runFlag     string
}

// newTest creates a new test
func newTest(remote string, subdir bool, fastlist bool) *test {
	t := &test{
		remote:  remote,
		subdir:  subdir,
		cmdLine: []string{"./" + binary, "-test.timeout", (*timeout).String(), "-remote", remote},
		try:     1,
	}
	if *fstest.Verbose {
		t.cmdLine = append(t.cmdLine, "-test.v")
		fs.Config.LogLevel = fs.LogLevelDebug
	}
	if *runOnly != "" {
		t.cmdLine = append(t.cmdLine, "-test.run", *runOnly)
	}
	if subdir {
		t.cmdLine = append(t.cmdLine, "-subdir")
	}
	if fastlist {
		t.cmdLine = append(t.cmdLine, "-fast-list")
	}
	t.cmdString = toShell(t.cmdLine)
	return t
}

// dumpOutput prints the error output
func (t *test) dumpOutput() {
	log.Println("------------------------------------------------------------")
	log.Printf("---- %q ----", t.cmdString)
	log.Println(string(t.output))
	log.Println("------------------------------------------------------------")
}

var failRe = regexp.MustCompile(`(?m)^--- FAIL: (Test\w*) \(`)

// findFailures looks for all the tests which failed
func (t *test) findFailures() {
	oldFailedTests := t.failedTests
	t.failedTests = nil
	for _, matches := range failRe.FindAllSubmatch(t.output, -1) {
		t.failedTests = append(t.failedTests, string(matches[1]))
	}
	if len(t.failedTests) != 0 {
		t.runFlag = "^(" + strings.Join(t.failedTests, "|") + ")$"
	} else {
		t.runFlag = ""
	}
	if t.passed() && len(t.failedTests) != 0 {
		log.Printf("%q - Expecting no errors but got: %v", t.cmdString, t.failedTests)
		t.dumpOutput()
	} else if !t.passed() && len(t.failedTests) == 0 {
		log.Printf("%q - Expecting errors but got none: %v", t.cmdString, t.failedTests)
		t.dumpOutput()
		t.failedTests = oldFailedTests
	}
}

// nextCmdLine returns the next command line
func (t *test) nextCmdLine() []string {
	cmdLine := t.cmdLine[:]
	if t.runFlag != "" {
		cmdLine = append(cmdLine, "-test.run", t.runFlag)
	}
	return cmdLine
}

// if matches then is definitely OK in the shell
var shellOK = regexp.MustCompile("^[A-Za-z0-9./_:-]+$")

// converts a argv style input into a shell command
func toShell(args []string) (result string) {
	for _, arg := range args {
		if result != "" {
			result += " "
		}
		if shellOK.MatchString(arg) {
			result += arg
		} else {
			result += "'" + arg + "'"
		}
	}
	return result
}

// trial runs a single test
func (t *test) trial() {
	cmdLine := t.nextCmdLine()
	cmdString := toShell(cmdLine)
	log.Printf("%q - Starting (try %d/%d)", cmdString, t.try, *maxTries)
	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
	start := time.Now()
	t.output, t.err = cmd.CombinedOutput()
	duration := time.Since(start)
	t.findFailures()
	if t.passed() {
		log.Printf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, t.try, *maxTries)
	} else {
		log.Printf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, t.try, *maxTries, t.err, t.failedTests)
	}
}

// cleanFs runs a single clean fs for left over directories
func (t *test) cleanFs() error {
	f, err := fs.NewFs(t.remote)
	if err != nil {
		return err
	}
	entries, err := fs.ListDirSorted(f, true, "")
	if err != nil {
		return err
	}
	return entries.ForDirError(func(dir fs.Directory) error {
		remote := dir.Remote()
		if fstest.MatchTestRemote.MatchString(remote) {
			log.Printf("Purging %s%s", t.remote, remote)
			dir, err := fs.NewFs(t.remote + remote)
			if err != nil {
				return err
			}
			return fs.Purge(dir)
		}
		return nil
	})
}

// clean runs a single clean on a fs for left over directories
func (t *test) clean() {
	log.Printf("%q - Starting clean (try %d/%d)", t.remote, t.try, *maxTries)
	start := time.Now()
	t.err = t.cleanFs()
	if t.err != nil {
		log.Printf("%q - Failed to purge %v", t.remote, t.err)
	}
	duration := time.Since(start)
	if t.passed() {
		log.Printf("%q - Finished OK in %v (try %d/%d)", t.cmdString, duration, t.try, *maxTries)
	} else {
		log.Printf("%q - Finished ERROR in %v (try %d/%d): %v", t.cmdString, duration, t.try, *maxTries, t.err)
	}
}

// passed returns true if the test passed
func (t *test) passed() bool {
	return t.err == nil
}

// run runs all the trials for this test
func (t *test) run(result chan<- *test) {
	for t.try = 1; t.try <= *maxTries; t.try++ {
		if *clean {
			if !t.subdir {
				t.clean()
			}
		} else {
			t.trial()
		}
		if t.passed() {
			break
		}
	}
	if !t.passed() {
		t.dumpOutput()
	}
	result <- t
}

// makeTestBinary makes the binary we will run
func makeTestBinary() {
	if runtime.GOOS == "windows" {
		binary += ".exe"
	}
	log.Printf("Making test binary %q", binary)
	err := exec.Command("go", "test", "-c", "-o", binary).Run()
	if err != nil {
		log.Fatalf("Failed to make test binary: %v", err)
	}
	if _, err := os.Stat(binary); err != nil {
		log.Fatalf("Couldn't find test binary %q", binary)
	}
}

// removeTestBinary removes the binary made in makeTestBinary
func removeTestBinary() {
	err := os.Remove(binary) // Delete the binary when finished
	if err != nil {
		log.Printf("Error removing test binary %q: %v", binary, err)
	}
}

func main() {
	flag.Parse()
	if *runTests != "" {
		newRemotes := []remoteConfig{}
		for _, name := range strings.Split(*runTests, ",") {
			for i := range remotes {
				if remotes[i].Name == name {
					newRemotes = append(newRemotes, remotes[i])
					goto found
				}
			}
			log.Printf("Remote %q not found - inserting with default flags", name)
			newRemotes = append(newRemotes, remoteConfig{Name: name})
		found:
		}
		remotes = newRemotes
	}
	var names []string
	for _, remote := range remotes {
		names = append(names, remote.Name)
	}
	log.Printf("Testing remotes: %s", strings.Join(names, ", "))

	start := time.Now()
	if *clean {
		fs.LoadConfig()
	} else {
		makeTestBinary()
		defer removeTestBinary()
	}

	// start the tests
	results := make(chan *test, 8)
	awaiting := 0
	bools := []bool{false, true}
	if *clean {
		// Don't run -subdir and -fast-list if -clean
		bools = bools[:1]
	}
	for _, remote := range remotes {
		for _, subdir := range bools {
			for _, fastlist := range bools {
				if (!subdir || subdir && remote.SubDir) && (!fastlist || fastlist && remote.FastList) {
					go newTest(remote.Name, subdir, fastlist).run(results)
					awaiting++
				}
			}
		}
	}

	// Wait for the tests to finish
	var failed []*test
	for ; awaiting > 0; awaiting-- {
		t := <-results
		if !t.passed() {
			failed = append(failed, t)
		}
	}
	duration := time.Since(start)

	// Summarise results
	if len(failed) == 0 {
		log.Printf("PASS: All tests finished OK in %v", duration)
	} else {
		log.Printf("FAIL: %d tests failed in %v", len(failed), duration)
		for _, t := range failed {
			log.Printf("  * %s", toShell(t.nextCmdLine()))
			log.Printf("    * Failed tests: %v", t.failedTests)
		}
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Package watchfiles provides a reusable utility to watch files and get notified of changes
package watchfiles

import (
	"bufio"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sync"
	"time"

	"gopkg.in/fsnotify.v1"
)

var logger = log.New(os.Stderr, "watchfiles: ", log.LstdFlags|log.Lmicroseconds)

// SetLog allows customizing the logger
func SetLog(l *log.Logger) {
	logger = l
}

// WatchFiles allows loading, and optionally watching files for changes.
type WatchFiles struct {
	path        string
	filePattern *regexp.Regexp
	updateCb    UpdateCallback
	removeCb    RemoveCallback

	eventsLock *sync.Mutex
	eventsMap  map[string]fsnotify.Event
}

// UpdateCallback gets invoked when a watched file is modified,
// or when a new file is added to a watched directory
type UpdateCallback func(filePath string, reader *bufio.Reader)

// RemoveCallback gets invoked when a watched file is modified
type RemoveCallback func(filePath string)

// NewWatchFiles is the constructor function for a Config instance
//
//	The following parameters are supported:
//		filePath: the directory or file to open and watch. If a directory is used, all files in it are loaded
//		updateCb: (optional) a callback that is invoked when config data files are updated, or added
//		removeCb: (optional) a callback that is invoked when config data files are removed
func NewWatchFiles(filePath string, filePattern *regexp.Regexp, updateCb UpdateCallback,
	removeCb RemoveCallback) (*WatchFiles, error) {

	c := WatchFiles{path: filePath, filePattern: filePattern}
	c.updateCb = updateCb
	c.removeCb = removeCb

	c.eventsLock = &sync.Mutex{}
	c.eventsMap = make(map[string]fsnotify.Event)

	var err = c.loadFiles(filePath)
	if err != nil {
		return nil, err
	}

	c.watchFile(filePath)
	return &c, nil
}

func (wf *WatchFiles) loadFiles(filePath string) error {
	f, err := os.Open(filePath)
	if err != nil {
		logger.Printf("ERROR - Unable to open: %s", filePath)
		return err
	}

	defer f.Close()
	fi, err := f.Stat()
	if err != nil {
		logger.Printf("ERROR - Unable to stat: %s", filePath)
		return err
	}

	if fi.IsDir() {
		list, err := f.Readdir(-1)
		if err != nil {
			logger.Printf("ERROR - Unable to read directory: %s", fi.Name())
			return err
		}

		for _, dirFile := range list {
			if dirFile.IsDir() {
				logger.Printf("INFO - Skipping sub-directory %s ", fi.Name())
				continue
			}

			var fileName = filepath.Join(f.Name(), dirFile.Name())
			wf.openAndloadFile(fileName)
		}
	} else {
		wf.loadFile(f)
	}

	return nil
}

func (wf *WatchFiles) openAndloadFile(filePath string) {
	logger.Printf("INFO - Processsing file: %s", filePath)

	configFile, err := os.Open(filePath)
	if err != nil {
		logger.Printf("ERROR - %v", err)
		return
	}

	defer configFile.Close()
	wf.loadFile(configFile)
}

func (wf *WatchFiles) loadFile(configFile *os.File) {
	rdr := bufio.NewReader(configFile)
	logger.Printf("loaded data from: %s", configFile.Name())

	if wf.updateCb != nil {
		wf.updateCb(configFile.Name(), rdr)
	}
}

func (wf *WatchFiles) watchFile(filePath string) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		logger.Printf("ERROR - %v", err)
	}

	go func() {
		for {
			timer := time.NewTimer(1 * time.Second)
			select {
			case event := <-watcher.Events:
				fn := filepath.Base(event.Name)
				if !wf.filePattern.MatchString(fn) {
					logger.Printf("INFO - ignoring file %s as it does not match name pattern", event.Name)
					continue
				}

				logger.Printf("INFO - Received file watch event: %s", event.String())
				wf.storeEvent(event)
				timer.Stop()

			case <-timer.C:
				wf.processEvents()

			case err := <-watcher.Errors:
				logger.Printf("error: %v from file watcher: %s", err, filePath)
			}
		}
	}()

	err = watcher.Add(filePath)
	if err != nil {
		logger.Printf("ERROR - %v", err)
		return
	}

	logger.Printf("INFO - Watching file: %s", filePath)
}

func (wf *WatchFiles) storeEvent(event fsnotify.Event) {
	wf.eventsLock.Lock()
	defer wf.eventsLock.Unlock()

	wf.eventsMap[event.Name] = event
	logger.Printf("INFO - %d pending events", len(wf.eventsMap))
}

func (wf *WatchFiles) processEvents() {
	wf.eventsLock.Lock()
	defer wf.eventsLock.Unlock()

	if len(wf.eventsMap) == 0 {
		return
	}

	logger.Printf("INFO - Processing %d events", len(wf.eventsMap))

	for _, event := range wf.eventsMap {
		delete(wf.eventsMap, event.Name)
		if ((event.Op & fsnotify.Write) == fsnotify.Write) || ((event.Op & fsnotify.Chmod) == fsnotify.Chmod) {
			logger.Printf("INFO - Modified file: %s", event.Name)
			wf.openAndloadFile(event.Name)
		} else if (event.Op & fsnotify.Create) == fsnotify.Create {
			logger.Printf("INFO - Created file: %s", event.Name)
			wf.openAndloadFile(event.Name)
		} else if (event.Op & fsnotify.Remove) == fsnotify.Remove {
			logger.Printf("INFO - Deleted file: %s", event.Name)
			if wf.removeCb != nil {
				wf.removeCb(event.Name)
			}
		} else if (event.Op & fsnotify.Rename) == fsnotify.Rename {
			logger.Printf("INFO - Renamed file: %s", event.Name)
			wf.openAndloadFile(event.Name)
		}
	}
}
<commit_msg>add log<commit_after>// Package watchfiles provides a reusable utility to watch files and get notified of changes
package watchfiles

import (
	"bufio"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sync"
	"time"

	"gopkg.in/fsnotify.v1"
)

var logger = log.New(os.Stderr, "watchfiles: ", log.LstdFlags|log.Lmicroseconds)

// SetLog allows customizing the logger
func SetLog(l *log.Logger) {
	logger = l
}

// WatchFiles allows loading, and optionally watching files for changes.
type WatchFiles struct {
	path        string
	filePattern *regexp.Regexp
	updateCb    UpdateCallback
	removeCb    RemoveCallback

	eventsLock *sync.Mutex
	eventsMap  map[string]fsnotify.Event
}

// UpdateCallback gets invoked when a watched file is modified,
// or when a new file is added to a watched directory
type UpdateCallback func(filePath string, reader *bufio.Reader)

// RemoveCallback gets invoked when a watched file is modified
type RemoveCallback func(filePath string)

// NewWatchFiles is the constructor function for a Config instance
//
//	The following parameters are supported:
//		filePath: the directory or file to open and watch. If a directory is used, all files in it are loaded
//		updateCb: (optional) a callback that is invoked when config data files are updated, or added
//		removeCb: (optional) a callback that is invoked when config data files are removed
func NewWatchFiles(filePath string, filePattern *regexp.Regexp, updateCb UpdateCallback,
	removeCb RemoveCallback) (*WatchFiles, error) {

	c := WatchFiles{path: filePath, filePattern: filePattern}
	c.updateCb = updateCb
	c.removeCb = removeCb

	c.eventsLock = &sync.Mutex{}
	c.eventsMap = make(map[string]fsnotify.Event)

	var err = c.loadFiles(filePath)
	if err != nil {
		return nil, err
	}

	c.watchFile(filePath)
	return &c, nil
}

func (wf *WatchFiles) loadFiles(filePath string) error {
	f, err := os.Open(filePath)
	if err != nil {
		logger.Printf("ERROR - Unable to open: %s", filePath)
		return err
	}

	defer f.Close()
	fi, err := f.Stat()
	if err != nil {
		logger.Printf("ERROR - Unable to stat: %s", filePath)
		return err
	}

	if fi.IsDir() {
		list, err := f.Readdir(-1)
		if err != nil {
			logger.Printf("ERROR - Unable to read directory: %s", fi.Name())
			return err
		}

		for _, dirFile := range list {
			if dirFile.IsDir() {
				logger.Printf("INFO - Skipping sub-directory %s ", fi.Name())
				continue
			}

			var fileName = filepath.Join(f.Name(), dirFile.Name())
			wf.openAndloadFile(fileName)
		}
	} else {
		wf.loadFile(f)
	}

	return nil
}

func (wf *WatchFiles) openAndloadFile(filePath string) {
	logger.Printf("INFO - Processsing file: %s", filePath)

	configFile, err := os.Open(filePath)
	if err != nil {
		logger.Printf("ERROR - %v", err)
		return
	}

	defer configFile.Close()
	wf.loadFile(configFile)
}

func (wf *WatchFiles) loadFile(configFile *os.File) {
	rdr := bufio.NewReader(configFile)
	logger.Printf("loaded data from: %s", configFile.Name())

	if wf.updateCb != nil {
		wf.updateCb(configFile.Name(), rdr)
	}
}

func (wf *WatchFiles) watchFile(filePath string) {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		logger.Printf("ERROR - %v", err)
	}

	go func() {
		for {
			timer := time.NewTimer(1 * time.Second)
			select {
			case event := <-watcher.Events:
				logger.Printf("INFO - Received file watch event: %s", event.String())
				fn := filepath.Base(event.Name)
				if !wf.filePattern.MatchString(fn) {
					logger.Printf("INFO - ignoring file %s as it does not match name pattern", event.Name)
					continue
				}

				wf.storeEvent(event)
				timer.Stop()

			case <-timer.C:
				wf.processEvents()

			case err := <-watcher.Errors:
				logger.Printf("error: %v from file watcher: %s", err, filePath)
			}
		}
	}()

	err = watcher.Add(filePath)
	if err != nil {
		logger.Printf("ERROR - %v", err)
		return
	}

	logger.Printf("INFO - Watching file: %s", filePath)
}

func (wf *WatchFiles) storeEvent(event fsnotify.Event) {
	wf.eventsLock.Lock()
	defer wf.eventsLock.Unlock()

	wf.eventsMap[event.Name] = event
	logger.Printf("INFO - %d pending events", len(wf.eventsMap))
}

func (wf *WatchFiles) processEvents() {
	wf.eventsLock.Lock()
	defer wf.eventsLock.Unlock()

	if len(wf.eventsMap) == 0 {
		return
	}

	logger.Printf("INFO - Processing %d events", len(wf.eventsMap))

	for _, event := range wf.eventsMap {
		delete(wf.eventsMap, event.Name)
		if ((event.Op & fsnotify.Write) == fsnotify.Write) || ((event.Op & fsnotify.Chmod) == fsnotify.Chmod) {
			logger.Printf("INFO - Modified file: %s", event.Name)
			wf.openAndloadFile(event.Name)
		} else if (event.Op & fsnotify.Create) == fsnotify.Create {
			logger.Printf("INFO - Created file: %s", event.Name)
			wf.openAndloadFile(event.Name)
		} else if (event.Op & fsnotify.Remove) == fsnotify.Remove {
			logger.Printf("INFO - Deleted file: %s", event.Name)
			if wf.removeCb != nil {
				wf.removeCb(event.Name)
			}
		} else if (event.Op & fsnotify.Rename) == fsnotify.Rename {
			logger.Printf("INFO - Renamed file: %s", event.Name)
			wf.openAndloadFile(event.Name)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package index

import (
	"log"
	"os"
	"syscall"
)

func mmapFile(f *os.File) mmapData {
	st, err := f.Stat()
	if err != nil {
		log.Fatal(err)
	}
	size := st.Size()
	if int64(int(size+4095)) != size+4095 {
		log.Fatalf("%s: too large for mmap", f.Name())
	}
	n := int(size)
	if n == 0 {
		return mmapData{f, nil, 0}
	}
	data, err := syscall.Mmap(int(f.Fd()), 0, (n+4095)&^4095, syscall.PROT_READ, syscall.MAP_SHARED)
	if err != nil {
		log.Fatalf("mmap %s: %v", f.Name(), err)
	}
	return mmapData{f, data[:n], 0}
}

func unmmapFile(mm *mmapData) {
  err := syscall.Munmap(mm.d)
  if err != nil {
  	log.Println("unmmapFile:", err)
  }
}

<commit_msg>mmap_linux: fix unmmap to work<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package index

import (
	"log"
	"os"
	"syscall"
)

func mmapFile(f *os.File) mmapData {
	st, err := f.Stat()
	if err != nil {
		log.Fatal(err)
	}
	size := st.Size()
	if int64(int(size+4095)) != size+4095 {
		log.Fatalf("%s: too large for mmap", f.Name())
	}
	n := int(size)
	if n == 0 {
		return mmapData{f, nil, 0}
	}
	data, err := syscall.Mmap(int(f.Fd()), 0, (n+4095)&^4095, syscall.PROT_READ, syscall.MAP_SHARED)
	if err != nil {
		log.Fatalf("mmap %s: %v", f.Name(), err)
	}
	return mmapData{f, data[:n], 0}
}

func unmmapFile(mm *mmapData) {
	len_equal_cap := mm.d[0:cap(mm.d)] // else get EINVAL
	err := syscall.Munmap(len_equal_cap)
	if err != nil {
		log.Println("unmmapFile:", err)
	}
}

<|endoftext|>