<commit_before>package script

import (
	"bytes"
	"os"
	"text/template"

	"github.com/spf13/cobra"
)

type scriptConfig struct {
	DBName      string
	BucketName  string
	LogPrefix   string
	ClientPort  string
	ProjectName string
	KeyPath     string
	Conns       int
	Clients     int
	ValSize     int
	Total       int
}

var (
	Command = &cobra.Command{
		Use:        "script",
		Short:      "Generates cloud provisioning script.",
		SuggestFor: []string{"scrt"},
		RunE:       scriptCommandFunc,
	}

	outputPath string
	cfg        = scriptConfig{}
)

func init() {
	Command.PersistentFlags().StringVarP(&outputPath, "output", "o", "script.sh", "File path to store script.")
	Command.PersistentFlags().StringVarP(&cfg.DBName, "db-name", "d", "etcdv3", "Name of database (etcdv2, etcdv3, zookeeper, zk, consul).")
	Command.PersistentFlags().StringVarP(&cfg.BucketName, "bucket-name", "b", "", "Name of bucket to store results.")
	Command.PersistentFlags().StringVarP(&cfg.LogPrefix, "log-prefix", "p", "bench-01", "Prefix to name instances, logs files.")
	Command.PersistentFlags().StringVarP(&cfg.ClientPort, "client-port", "c", "2379", "2379 for etcd, 2181 for Zookeeper, 8500 for Consul.")
	Command.PersistentFlags().StringVarP(&cfg.ProjectName, "project-name", "n", "etcd-development", "Project name.")
	Command.PersistentFlags().StringVarP(&cfg.KeyPath, "key-path", "k", "$HOME/key.json", "Key path.")
	Command.PersistentFlags().IntVar(&cfg.Conns, "conns", 1, "conns.")
	Command.PersistentFlags().IntVar(&cfg.Clients, "clients", 1, "clients.")
	Command.PersistentFlags().IntVar(&cfg.ValSize, "val-size", 256, "val-size.")
	Command.PersistentFlags().IntVar(&cfg.Total, "total", 3000000, "total.")
}

func scriptCommandFunc(cmd *cobra.Command, args []string) error {
	tpl := template.Must(template.New("scriptTemplate").Parse(scriptTemplate))
	buf := new(bytes.Buffer)
	if err := tpl.Execute(buf, cfg); err != nil {
		return err
	}
	return toFile(buf.String(), outputPath)
}

const scriptTemplate = `
#!/usr/bin/env bash
set -e

gcloud compute instances list
gcloud compute instances create {{.LogPrefix}}-{{.DBName}}-1 --custom-cpu=8 --custom-memory=16 --image="ubuntu-15-10" --boot-disk-size=50 --boot-disk-type="pd-ssd" --local-ssd interface=SCSI --zone us-central1-a
gcloud compute instances create {{.LogPrefix}}-{{.DBName}}-2 --custom-cpu=8 --custom-memory=16 --image="ubuntu-15-10" --boot-disk-size=50 --boot-disk-type="pd-ssd" --local-ssd interface=SCSI --zone us-central1-a
gcloud compute instances create {{.LogPrefix}}-{{.DBName}}-3 --custom-cpu=8 --custom-memory=16 --image="ubuntu-15-10" --boot-disk-size=50 --boot-disk-type="pd-ssd" --local-ssd interface=SCSI --zone us-central1-a
gcloud compute instances create {{.LogPrefix}}-{{.DBName}}-tester --custom-cpu=16 --custom-memory=30 --image="ubuntu-15-10" --boot-disk-size=50 --boot-disk-type="pd-ssd" --zone us-central1-a
gcloud compute instances list

gcloud compute ssh {{.LogPrefix}}-{{.DBName}}-1
gcloud compute ssh {{.LogPrefix}}-{{.DBName}}-2
gcloud compute ssh {{.LogPrefix}}-{{.DBName}}-3
gcloud compute ssh {{.LogPrefix}}-{{.DBName}}-tester

#########
# agent #
#########
GO_VERSION="1.6" && cd /usr/local && sudo rm -rf ./go && sudo curl -s https://storage.googleapis.com/golang/go$GO_VERSION.linux-amd64.tar.gz | sudo tar -v -C /usr/local/ -xz && cd $HOME;
echo "export GOPATH=$(echo $HOME)/go" >> $HOME/.bashrc
PATH_VAR=$PATH":/usr/local/go/bin:$(echo $HOME)/go/bin"
echo "export PATH=$(echo $PATH_VAR)" >> $HOME/.bashrc
export GOPATH=$(echo $HOME)/go
PATH_VAR=$PATH":/usr/local/go/bin:$(echo $HOME)/go/bin"
export PATH=$(echo $PATH_VAR)
go get -v -u -f github.com/coreos/dbtester
curl https://storage.googleapis.com/etcd/dbtester_agent.sh | bash -s f /mnt/ssd0

cd /mnt/ssd0
ls /mnt/ssd0
cat /mnt/ssd0/agent.log

##########
# tester #
##########
ulimit -n 3000
ulimit -n
GO_VERSION="1.6" && cd /usr/local && sudo rm -rf ./go && sudo curl -s https://storage.googleapis.com/golang/go$GO_VERSION.linux-amd64.tar.gz | sudo tar -v -C /usr/local/ -xz && cd $HOME;
echo "export GOPATH=$(echo $HOME)/go" >> $HOME/.bashrc
PATH_VAR=$PATH":/usr/local/go/bin:$(echo $HOME)/go/bin"
echo "export PATH=$(echo $PATH_VAR)" >> $HOME/.bashrc
export GOPATH=$(echo $HOME)/go
PATH_VAR=$PATH":/usr/local/go/bin:$(echo $HOME)/go/bin"
export PATH=$(echo $PATH_VAR)
psn ps-kill --force -s dbtester
go get -v -u -f github.com/coreos/dbtester

# start test
AGENT_ENDPOINTS='___IP_ADDR_1___:3500,___IP_ADDR_2___:3500,___IP_ADDR_3___:3500'
DATABASE_ENDPOINTS='___IP_ADDR_1___:{{.ClientPort}},___IP_ADDR_2___:{{.ClientPort}},___IP_ADDR_3___:{{.ClientPort}}'

# start database
dbtester start --agent-endpoints=$(echo $AGENT_ENDPOINTS) --database={{.DBName}} --database-log-path=database.log --log-prefix={{.LogPrefix}}-{{.DBName}} --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}} --monitor-result-path=monitor.csv;

cat /mnt/ssd0/agent.log
cat /mnt/ssd0/database.log

# start benchmark
nohup dbtester bench --database={{.DBName}} --sample --no-histogram --csv-result-path={{.LogPrefix}}-{{.DBName}}-timeseries.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}} --endpoints=$DATABASE_ENDPOINTS --conns={{.Conns}} --clients={{.Clients}} put --key-size=64 --val-size={{.ValSize}} --total={{.Total}} > {{.LogPrefix}}-{{.DBName}}-result.txt 2>&1 &

cat {{.LogPrefix}}-{{.DBName}}-result.txt

# benchmark done!
# stop database to trigger uploading in remote machines
dbtester stop --agent-endpoints=$(echo $AGENT_ENDPOINTS)

dbtester upload --from={{.LogPrefix}}-{{.DBName}}-timeseries.csv --to={{.LogPrefix}}-{{.DBName}}-timeseries.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}

####################
# in case of panic #
####################
dbtester upload --from=/mnt/ssd0/agent.log --to={{.LogPrefix}}-{{.DBName}}-1-agent.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/database.log --to={{.LogPrefix}}-{{.DBName}}-1-database.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/monitor.csv --to={{.LogPrefix}}-{{.DBName}}-1-monitor.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}

dbtester upload --from=/mnt/ssd0/agent.log --to={{.LogPrefix}}-{{.DBName}}-2-agent.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/database.log --to={{.LogPrefix}}-{{.DBName}}-2-database.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/monitor.csv --to={{.LogPrefix}}-{{.DBName}}-2-monitor.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}

dbtester upload --from=/mnt/ssd0/agent.log --to={{.LogPrefix}}-{{.DBName}}-3-agent.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/database.log --to={{.LogPrefix}}-{{.DBName}}-3-database.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/monitor.csv --to={{.LogPrefix}}-{{.DBName}}-3-monitor.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}

`

func toFile(txt, fpath string) error {
	f, err := os.OpenFile(fpath, os.O_RDWR|os.O_TRUNC, 0777)
	if err != nil {
		f, err = os.Create(fpath)
		if err != nil {
			return err
		}
	}
	defer f.Close()
	if _, err := f.WriteString(txt); err != nil {
		return err
	}
	return nil
}
<commit_msg>script: rename<commit_after>package script

import (
	"bytes"
	"os"
	"text/template"

	"github.com/spf13/cobra"
)

type scriptConfig struct {
	DBName      string
	BucketName  string
	LogPrefix   string
	ClientPort  string
	ProjectName string
	KeyPath     string
	BenchType   string
	Conns       int
	Clients     int
	ValSize     int
	Total       int
}

var (
	Command = &cobra.Command{
		Use:        "script",
		Short:      "Generates cloud provisioning script.",
		SuggestFor: []string{"scrt"},
		RunE:       scriptCommandFunc,
	}

	outputPath string
	cfg        = scriptConfig{}
)

func init() {
	Command.PersistentFlags().StringVarP(&outputPath, "output", "o", "script.sh", "File path to store script.")
	Command.PersistentFlags().StringVarP(&cfg.DBName, "db-name", "d", "etcdv3", "Name of database (etcdv2, etcdv3, zookeeper, zk, consul).")
	Command.PersistentFlags().StringVarP(&cfg.BucketName, "bucket-name", "b", "", "Name of bucket to store results.")
	Command.PersistentFlags().StringVarP(&cfg.LogPrefix, "log-prefix", "p", "bench-01", "Prefix to name instances, logs files.")
	Command.PersistentFlags().StringVarP(&cfg.ClientPort, "client-port", "c", "2379", "2379 for etcd, 2181 for Zookeeper, 8500 for Consul.")
	Command.PersistentFlags().StringVarP(&cfg.ProjectName, "project-name", "n", "etcd-development", "Project name.")
	Command.PersistentFlags().StringVarP(&cfg.KeyPath, "key-path", "k", "$HOME/key.json", "Key path.")
	Command.PersistentFlags().StringVarP(&cfg.BenchType, "bench-type", "t", "put", "'put' or 'range'.")
	Command.PersistentFlags().IntVar(&cfg.Conns, "conns", 1, "conns.")
	Command.PersistentFlags().IntVar(&cfg.Clients, "clients", 1, "clients.")
	Command.PersistentFlags().IntVar(&cfg.ValSize, "val-size", 256, "val-size.")
	Command.PersistentFlags().IntVar(&cfg.Total, "total", 3000000, "total.")
}

func scriptCommandFunc(cmd *cobra.Command, args []string) error {
	if cfg.BenchType == "range" {
		cfg.BenchType = "range --consistency s --single-key"
	}
	tpl := template.Must(template.New("scriptTemplate").Parse(scriptTemplate))
	buf := new(bytes.Buffer)
	if err := tpl.Execute(buf, cfg); err != nil {
		return err
	}
	return toFile(buf.String(), outputPath)
}

const scriptTemplate = `
#!/usr/bin/env bash
set -e

gcloud compute instances list
gcloud compute instances create {{.LogPrefix}}-{{.DBName}}-1 --custom-cpu=8 --custom-memory=16 --image="ubuntu-15-10" --boot-disk-size=50 --boot-disk-type="pd-ssd" --local-ssd interface=SCSI --zone us-central1-a
gcloud compute instances create {{.LogPrefix}}-{{.DBName}}-2 --custom-cpu=8 --custom-memory=16 --image="ubuntu-15-10" --boot-disk-size=50 --boot-disk-type="pd-ssd" --local-ssd interface=SCSI --zone us-central1-a
gcloud compute instances create {{.LogPrefix}}-{{.DBName}}-3 --custom-cpu=8 --custom-memory=16 --image="ubuntu-15-10" --boot-disk-size=50 --boot-disk-type="pd-ssd" --local-ssd interface=SCSI --zone us-central1-a
gcloud compute instances create {{.LogPrefix}}-{{.DBName}}-tester --custom-cpu=16 --custom-memory=30 --image="ubuntu-15-10" --boot-disk-size=50 --boot-disk-type="pd-ssd" --zone us-central1-a
gcloud compute instances list

gcloud compute ssh {{.LogPrefix}}-{{.DBName}}-1
gcloud compute ssh {{.LogPrefix}}-{{.DBName}}-2
gcloud compute ssh {{.LogPrefix}}-{{.DBName}}-3
gcloud compute ssh {{.LogPrefix}}-{{.DBName}}-tester

#########
# agent #
#########
GO_VERSION="1.6" && cd /usr/local && sudo rm -rf ./go && sudo curl -s https://storage.googleapis.com/golang/go$GO_VERSION.linux-amd64.tar.gz | sudo tar -v -C /usr/local/ -xz && cd $HOME;
echo "export GOPATH=$(echo $HOME)/go" >> $HOME/.bashrc
PATH_VAR=$PATH":/usr/local/go/bin:$(echo $HOME)/go/bin"
echo "export PATH=$(echo $PATH_VAR)" >> $HOME/.bashrc
export GOPATH=$(echo $HOME)/go
PATH_VAR=$PATH":/usr/local/go/bin:$(echo $HOME)/go/bin"
export PATH=$(echo $PATH_VAR)
go get -v -u -f github.com/coreos/dbtester
curl https://storage.googleapis.com/etcd/dbtester_agent.sh | bash -s f /mnt/ssd0

cd /mnt/ssd0
ls /mnt/ssd0
cat /mnt/ssd0/agent.log

##########
# tester #
##########
ulimit -n 3000
ulimit -n
GO_VERSION="1.6" && cd /usr/local && sudo rm -rf ./go && sudo curl -s https://storage.googleapis.com/golang/go$GO_VERSION.linux-amd64.tar.gz | sudo tar -v -C /usr/local/ -xz && cd $HOME;
echo "export GOPATH=$(echo $HOME)/go" >> $HOME/.bashrc
PATH_VAR=$PATH":/usr/local/go/bin:$(echo $HOME)/go/bin"
echo "export PATH=$(echo $PATH_VAR)" >> $HOME/.bashrc
export GOPATH=$(echo $HOME)/go
PATH_VAR=$PATH":/usr/local/go/bin:$(echo $HOME)/go/bin"
export PATH=$(echo $PATH_VAR)
psn ps-kill --force -s dbtester
go get -v -u -f github.com/coreos/dbtester

# start test
AGENT_ENDPOINTS='___IP_ADDR_1___:3500,___IP_ADDR_2___:3500,___IP_ADDR_3___:3500'
DATABASE_ENDPOINTS='___IP_ADDR_1___:{{.ClientPort}},___IP_ADDR_2___:{{.ClientPort}},___IP_ADDR_3___:{{.ClientPort}}'

# start database
dbtester start --agent-endpoints=$(echo $AGENT_ENDPOINTS) --database={{.DBName}} --database-log-path=database.log --log-prefix={{.LogPrefix}}-{{.DBName}} --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}} --monitor-result-path=monitor.csv;

cat /mnt/ssd0/agent.log
cat /mnt/ssd0/database.log

# start benchmark
nohup dbtester bench --database={{.DBName}} --sample --no-histogram --csv-result-path={{.LogPrefix}}-{{.DBName}}-timeseries.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}} --endpoints=$DATABASE_ENDPOINTS --conns={{.Conns}} --clients={{.Clients}} {{.BenchType}} --key-size=64 --val-size={{.ValSize}} --total={{.Total}} > {{.LogPrefix}}-{{.DBName}}-result.txt 2>&1 &

cat {{.LogPrefix}}-{{.DBName}}-result.txt

# benchmark done!
# stop database to trigger uploading in remote machines
dbtester stop --agent-endpoints=$(echo $AGENT_ENDPOINTS)

dbtester upload --from={{.LogPrefix}}-{{.DBName}}-timeseries.csv --to={{.LogPrefix}}-{{.DBName}}-timeseries.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}

####################
# in case of panic #
####################
dbtester upload --from=/mnt/ssd0/agent.log --to={{.LogPrefix}}-{{.DBName}}-1-agent.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/database.log --to={{.LogPrefix}}-{{.DBName}}-1-database.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/monitor.csv --to={{.LogPrefix}}-{{.DBName}}-1-monitor.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}

dbtester upload --from=/mnt/ssd0/agent.log --to={{.LogPrefix}}-{{.DBName}}-2-agent.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/database.log --to={{.LogPrefix}}-{{.DBName}}-2-database.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/monitor.csv --to={{.LogPrefix}}-{{.DBName}}-2-monitor.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}

dbtester upload --from=/mnt/ssd0/agent.log --to={{.LogPrefix}}-{{.DBName}}-3-agent.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/database.log --to={{.LogPrefix}}-{{.DBName}}-3-database.log --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}
dbtester upload --from=/mnt/ssd0/monitor.csv --to={{.LogPrefix}}-{{.DBName}}-3-monitor.csv --google-cloud-project-name={{.ProjectName}} --key-path={{.KeyPath}} --bucket={{.BucketName}}

`

func toFile(txt, fpath string) error {
	f, err := os.OpenFile(fpath, os.O_RDWR|os.O_TRUNC, 0777)
	if err != nil {
		f, err = os.Create(fpath)
		if err != nil {
			return err
		}
	}
	defer f.Close()
	if _, err := f.WriteString(txt); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package neurgo

import (
	"log"
	"math"
	"math/rand"
)

type StochasticHillClimber struct {
	currentCandidate *NeuralNetwork
	currentOptimal   *NeuralNetwork
}

const MAX_ITERATIONS_BEFORE_RESTART = 10000

func (shc *StochasticHillClimber) Train(neuralNet *NeuralNetwork, examples []*TrainingSample) *NeuralNetwork {

	fittestNeuralNet := neuralNet

	// Apply NN to problem and save fitness
	fitness := fittestNeuralNet.Fitness(examples)

	if fitness > FITNESS_THRESHOLD {
		return fittestNeuralNet
	}

	for i := 0; ; i++ {

		// Save the genotype
		candidateNeuralNet := fittestNeuralNet.Copy()

		// Perturb synaptic weights and biases
		shc.perturbParameters(candidateNeuralNet)

		// Re-Apply NN to problem
		candidateFitness := candidateNeuralNet.Fitness(examples)
		// log.Printf("fitness: %f", candidateFitness)

		// If the fitness of the perturbed NN is higher, discard original NN and keep
		// the new.  If the fitness of original is higher, discard perturbed and keep
		// the old.
		if candidateFitness > fitness {
			fittestNeuralNet = candidateNeuralNet
			fitness = candidateFitness
		}

		if IntModuloProper(i, MAX_ITERATIONS_BEFORE_RESTART) {
			log.Printf("reset params to random")
			shc.resetParametersToRandom(fittestNeuralNet)
		}

		if candidateFitness > FITNESS_THRESHOLD {
			break
		}

	}

	return fittestNeuralNet

}

// 1. Each neuron in the neural net (weight or bias) will be chosen for perturbation
//    with a probability of 1/sqrt(nn_size)
// 2. Within the chosen neuron, the weights which will be perturbed will be chosen
//    with probability of 1/sqrt(parameters_size)
// 3. The intensity of the parameter perturbation will chosen with uniform distribution
//    of -pi and pi
func (shc *StochasticHillClimber) perturbParameters(neuralNet *NeuralNetwork) {

	// pick the neurons to perturb (at least one)
	neurons := shc.chooseNeuronsToPerturb(neuralNet)

	for _, neuron := range neurons {
		shc.perturbNeuron(neuron)
	}

}

func (shc *StochasticHillClimber) resetParametersToRandom(neuralNet *NeuralNetwork) {

	neurons := neuralNet.neurons()
	for _, neuronNode := range neurons {
		for _, cxn := range neuronNode.inbound {
			for j, _ := range cxn.weights {
				cxn.weights[j] = RandomInRange(-1*math.Pi, math.Pi)
			}
		}
		neuronNode.processor.setBias(RandomInRange(-1*math.Pi, math.Pi))
	}

}

func (shc *StochasticHillClimber) chooseNeuronsToPerturb(neuralNet *NeuralNetwork) []*Node {

	neuronsToPerturb := make([]*Node, 0)

	// choose some random neurons to perturb.  we need at least one, so
	// keep looping until we've chosen at least one
	didChooseNeuron := false
	for {

		probability := shc.nodePerturbProbability(neuralNet)
		neurons := neuralNet.neurons()
		for _, neuronNode := range neurons {
			if rand.Float64() < probability {
				neuronsToPerturb = append(neuronsToPerturb, neuronNode)
				didChooseNeuron = true
			}
		}

		if didChooseNeuron {
			break
		}

	}
	return neuronsToPerturb

}

func (shc *StochasticHillClimber) nodePerturbProbability(neuralNet *NeuralNetwork) float64 {
	neurons := neuralNet.neurons()
	numNeurons := len(neurons)
	return 1 / math.Sqrt(float64(numNeurons))
}

func (shc *StochasticHillClimber) perturbNeuron(node *Node) {

	probability := shc.parameterPerturbProbability(node)

	// keep trying until we've perturbed at least one parameter
	for {
		didPerturbWeight := false
		for _, cxn := range node.inbound {
			didPerturbWeight = shc.possiblyPerturbConnection(cxn, probability)
		}

		didPerturbBias := shc.possiblyPerturbBias(node, probability)

		// did we perturb anything?  if so, we're done
		if didPerturbWeight || didPerturbBias {
			break
		}

	}

}

func (shc *StochasticHillClimber) parameterPerturbProbability(node *Node) float64 {
	numWeights := 0
	for _, connection := range node.inbound {
		numWeights += len(connection.weights)
	}
	return 1 / math.Sqrt(float64(numWeights))
}

func (shc *StochasticHillClimber) possiblyPerturbConnection(cxn *connection, probability float64) bool {

	didPerturb := false
	for j, weight := range cxn.weights {
		if rand.Float64() < probability {
			perturbedWeight := shc.perturbParameter(weight)
			cxn.weights[j] = perturbedWeight
			didPerturb = true
		}
	}
	return didPerturb

}

func (shc *StochasticHillClimber) possiblyPerturbBias(node *Node, probability float64) bool {
	didPerturb := false
	hasBias := node.processor.hasBias()
	if hasBias && rand.Float64() < probability {
		bias := node.processor.bias()
		perturbedBias := shc.perturbParameter(bias)
		node.processor.setBias(perturbedBias)
		didPerturb = true
	}
	return didPerturb
}

func (shc *StochasticHillClimber) perturbParameter(parameter float64) float64 {

	parameter += RandomInRange(-1*math.Pi, math.Pi)
	return parameter

}
<commit_msg>Remove debug logs<commit_after>package neurgo

import (
	"math"
	"math/rand"
)

type StochasticHillClimber struct {
	currentCandidate *NeuralNetwork
	currentOptimal   *NeuralNetwork
}

const MAX_ITERATIONS_BEFORE_RESTART = 10000

func (shc *StochasticHillClimber) Train(neuralNet *NeuralNetwork, examples []*TrainingSample) *NeuralNetwork {

	fittestNeuralNet := neuralNet

	// Apply NN to problem and save fitness
	fitness := fittestNeuralNet.Fitness(examples)

	if fitness > FITNESS_THRESHOLD {
		return fittestNeuralNet
	}

	for i := 0; ; i++ {

		// Save the genotype
		candidateNeuralNet := fittestNeuralNet.Copy()

		// Perturb synaptic weights and biases
		shc.perturbParameters(candidateNeuralNet)

		// Re-Apply NN to problem
		candidateFitness := candidateNeuralNet.Fitness(examples)

		// If fitness of perturbed NN is higher, discard original NN and keep new
		// If fitness of original is higher, discard perturbed and keep old.
		if candidateFitness > fitness {
			fittestNeuralNet = candidateNeuralNet
			fitness = candidateFitness
		}

		if IntModuloProper(i, MAX_ITERATIONS_BEFORE_RESTART) {
			shc.resetParametersToRandom(fittestNeuralNet)
		}

		if candidateFitness > FITNESS_THRESHOLD {
			break
		}

	}

	return fittestNeuralNet

}

// 1. Each neuron in the neural net (weight or bias) will be chosen for perturbation
//    with a probability of 1/sqrt(nn_size)
// 2. Within the chosen neuron, the weights which will be perturbed will be chosen
//    with probability of 1/sqrt(parameters_size)
// 3. The intensity of the parameter perturbation will chosen with uniform distribution
//    of -pi and pi
func (shc *StochasticHillClimber) perturbParameters(neuralNet *NeuralNetwork) {

	// pick the neurons to perturb (at least one)
	neurons := shc.chooseNeuronsToPerturb(neuralNet)

	for _, neuron := range neurons {
		shc.perturbNeuron(neuron)
	}

}

func (shc *StochasticHillClimber) resetParametersToRandom(neuralNet *NeuralNetwork) {

	neurons := neuralNet.neurons()
	for _, neuronNode := range neurons {
		for _, cxn := range neuronNode.inbound {
			for j, _ := range cxn.weights {
				cxn.weights[j] = RandomInRange(-1*math.Pi, math.Pi)
			}
		}
		neuronNode.processor.setBias(RandomInRange(-1*math.Pi, math.Pi))
	}

}

func (shc *StochasticHillClimber) chooseNeuronsToPerturb(neuralNet *NeuralNetwork) []*Node {

	neuronsToPerturb := make([]*Node, 0)

	// choose some random neurons to perturb.  we need at least one, so
	// keep looping until we've chosen at least one
	didChooseNeuron := false
	for {

		probability := shc.nodePerturbProbability(neuralNet)
		neurons := neuralNet.neurons()
		for _, neuronNode := range neurons {
			if rand.Float64() < probability {
				neuronsToPerturb = append(neuronsToPerturb, neuronNode)
				didChooseNeuron = true
			}
		}

		if didChooseNeuron {
			break
		}

	}
	return neuronsToPerturb

}

func (shc *StochasticHillClimber) nodePerturbProbability(neuralNet *NeuralNetwork) float64 {
	neurons := neuralNet.neurons()
	numNeurons := len(neurons)
	return 1 / math.Sqrt(float64(numNeurons))
}

func (shc *StochasticHillClimber) perturbNeuron(node *Node) {

	probability := shc.parameterPerturbProbability(node)

	// keep trying until we've perturbed at least one parameter
	for {
		didPerturbWeight := false
		for _, cxn := range node.inbound {
			didPerturbWeight = shc.possiblyPerturbConnection(cxn, probability)
		}

		didPerturbBias := shc.possiblyPerturbBias(node, probability)

		// did we perturb anything?  if so, we're done
		if didPerturbWeight || didPerturbBias {
			break
		}

	}

}

func (shc *StochasticHillClimber) parameterPerturbProbability(node *Node) float64 {
	numWeights := 0
	for _, connection := range node.inbound {
		numWeights += len(connection.weights)
	}
	return 1 / math.Sqrt(float64(numWeights))
}

func (shc *StochasticHillClimber) possiblyPerturbConnection(cxn *connection, probability float64) bool {

	didPerturb := false
	for j, weight := range cxn.weights {
		if rand.Float64() < probability {
			perturbedWeight := shc.perturbParameter(weight)
			cxn.weights[j] = perturbedWeight
			didPerturb = true
		}
	}
	return didPerturb

}

func (shc *StochasticHillClimber) possiblyPerturbBias(node *Node, probability float64) bool {
	didPerturb := false
	hasBias := node.processor.hasBias()
	if hasBias && rand.Float64() < probability {
		bias := node.processor.bias()
		perturbedBias := shc.perturbParameter(bias)
		node.processor.setBias(perturbedBias)
		didPerturb = true
	}
	return didPerturb
}

func (shc *StochasticHillClimber) perturbParameter(parameter float64) float64 {

	parameter += RandomInRange(-1*math.Pi, math.Pi)
	return parameter

}
<|endoftext|>


<commit_before>package sweetiebot

import (
  "github.com/bwmarrin/discordgo"
  "strings"
)

type HelpCommand struct {
}

func (c *HelpCommand) Name() string {
  return "Help";
}
func (c *HelpCommand) Process(args []string, msg *discordgo.Message) (string, bool) {
  if len(args) == 0 {
    s := []string{"Sweetie Bot knows the following commands. For more information on a specific command, type !help [command].\n"}
    for k, v := range sb.commands {
      s = append(s, k + ": " + v.c.UsageShort())
    }

    return "```" + strings.Join(s, "\n") + "```", true
  }
  v, ok := sb.commands[strings.ToLower(args[0])]
  if !ok {
    return "``` Sweetie Bot doesn't recognize the '" + args[0] + "' command. You can check what commands Sweetie Bot knows by typing !help.```", false
  }
  return "```> !" + v.c.Name() + " " + v.c.Usage() + "```", true
}
func (c *HelpCommand) Usage() string {
  return FormatUsage(c, "[command]", "Lists all available commands Sweetie Bot knows, or gives information about the given command. Of course, you should have figured this out by now, since you just typed !help help for some reason.")
}
func (c *HelpCommand) UsageShort() string { return "[PM Only] Generates the list you are looking at right now." }
func (c *HelpCommand) Roles() []string { return []string{} }
func (c *HelpCommand) Channels() []string { return []string{} }

type AboutCommand struct {
}

func (c *AboutCommand) Name() string {
  return "About";
}
func (c *AboutCommand) Process(args []string, msg *discordgo.Message) (string, bool) {
  s := "```Sweetie Bot version " + sb.version
  if sb.config.Debug {
    return s + " [debug]```", false
  }
  return s + " [release]```", false
}
func (c *AboutCommand) Usage() string {
  return FormatUsage(c, "", "Displays information about Sweetie Bot. What, did you think it would do something else?")
}
func (c *AboutCommand) UsageShort() string { return "Displays information about Sweetie Bot." }
func (c *AboutCommand) Roles() []string { return []string{} }
func (c *AboutCommand) Channels() []string { return []string{} }<commit_msg>Just in case, plug another hole<commit_after>package sweetiebot

import (
  "github.com/bwmarrin/discordgo"
  "strings"
)

type HelpCommand struct {
}

func (c *HelpCommand) Name() string {
  return "Help";
}
func (c *HelpCommand) Process(args []string, msg *discordgo.Message) (string, bool) {
  if len(args) == 0 {
    s := []string{"Sweetie Bot knows the following commands. For more information on a specific command, type !help [command].\n"}
    for k, v := range sb.commands {
      s = append(s, k + ": " + v.c.UsageShort())
    }

    return "```" + strings.Join(s, "\n") + "```", true
  }
  v, ok := sb.commands[strings.ToLower(args[0])]
  if !ok {
    return "``` Sweetie Bot doesn't recognize that command. You can check what commands Sweetie Bot knows by typing !help.```", false
  }
  return "```> !" + v.c.Name() + " " + v.c.Usage() + "```", true
}
func (c *HelpCommand) Usage() string {
  return FormatUsage(c, "[command]", "Lists all available commands Sweetie Bot knows, or gives information about the given command. Of course, you should have figured this out by now, since you just typed !help help for some reason.")
}
func (c *HelpCommand) UsageShort() string { return "[PM Only] Generates the list you are looking at right now." }
func (c *HelpCommand) Roles() []string { return []string{} }
func (c *HelpCommand) Channels() []string { return []string{} }

type AboutCommand struct {
}

func (c *AboutCommand) Name() string {
  return "About";
}
func (c *AboutCommand) Process(args []string, msg *discordgo.Message) (string, bool) {
  s := "```Sweetie Bot version " + sb.version
  if sb.config.Debug {
    return s + " [debug]```", false
  }
  return s + " [release]```", false
}
func (c *AboutCommand) Usage() string {
  return FormatUsage(c, "", "Displays information about Sweetie Bot. What, did you think it would do something else?")
}
func (c *AboutCommand) UsageShort() string { return "Displays information about Sweetie Bot." }
func (c *AboutCommand) Roles() []string { return []string{} }
func (c *AboutCommand) Channels() []string { return []string{} }<|endoftext|>


<commit_before>package tasks

import (
	"context"
	"time"

	"pixur.org/pixur/schema"
	"pixur.org/pixur/schema/db"
	tab "pixur.org/pixur/schema/tables"
	"pixur.org/pixur/status"
)

type AddPicTagsTask struct {
	// Deps
	DB  db.DB
	Now func() time.Time

	// Inputs
	PicID    int64
	TagNames []string
	Ctx      context.Context
}

// TODO: add tests
func (t *AddPicTagsTask) Run() (errCap status.S) {
	j, err := tab.NewJob(t.DB)
	if err != nil {
		return status.InternalError(err, "can't create job")
	}
	defer cleanUp(j, &errCap)

	u, sts := requireCapability(t.Ctx, j, schema.User_PIC_TAG_CREATE)
	if sts != nil {
		return sts
	}

	pics, err := j.FindPics(db.Opts{
		Prefix: tab.PicsPrimary{&t.PicID},
		Lock:   db.LockWrite,
	})
	if err != nil {
		return status.InternalError(err, "can't lookup pic")
	}
	if len(pics) != 1 {
		return status.NotFound(err, "can't find pic")
	}
	p := pics[0]

	if err := upsertTags(j, t.TagNames, p.PicId, t.Now(), u.UserId); err != nil {
		return err
	}

	if err := j.Commit(); err != nil {
		return status.InternalError(err, "can't commit job")
	}
	return nil
}
<commit_msg>tasks: don't allow tagging of deleted pics<commit_after>package tasks

import (
	"context"
	"time"

	"pixur.org/pixur/schema"
	"pixur.org/pixur/schema/db"
	tab "pixur.org/pixur/schema/tables"
	"pixur.org/pixur/status"
)

type AddPicTagsTask struct {
	// Deps
	DB  db.DB
	Now func() time.Time

	// Inputs
	PicID    int64
	TagNames []string
	Ctx      context.Context
}

// TODO: add tests
func (t *AddPicTagsTask) Run() (errCap status.S) {
	j, err := tab.NewJob(t.DB)
	if err != nil {
		return status.InternalError(err, "can't create job")
	}
	defer cleanUp(j, &errCap)

	u, sts := requireCapability(t.Ctx, j, schema.User_PIC_TAG_CREATE)
	if sts != nil {
		return sts
	}

	pics, err := j.FindPics(db.Opts{
		Prefix: tab.PicsPrimary{&t.PicID},
		Lock:   db.LockWrite,
	})
	if err != nil {
		return status.InternalError(err, "can't lookup pic")
	}
	if len(pics) != 1 {
		return status.NotFound(err, "can't find pic")
	}
	p := pics[0]

	if p.HardDeleted() {
		return status.InvalidArgument(nil, "can't tag deleted pic")
	}

	if err := upsertTags(j, t.TagNames, p.PicId, t.Now(), u.UserId); err != nil {
		return err
	}

	if err := j.Commit(); err != nil {
		return status.InternalError(err, "can't commit job")
	}
	return nil
}
<|endoftext|>


<commit_before>package cli_test

import (
	"testing"

	"github.com/stretchr/testify/require"

	"github.com/kopia/kopia/internal/blobtesting"
	"github.com/kopia/kopia/internal/repotesting"
	"github.com/kopia/kopia/repo/blob"
	"github.com/kopia/kopia/repo/content"
	"github.com/kopia/kopia/tests/testenv"
)

func (s *formatSpecificTestSuite) setupInMemoryRepo(t *testing.T) *testenv.CLITest {
	t.Helper()

	runner := testenv.NewInProcRunner(t)
	env := testenv.NewCLITest(t, s.formatFlags, runner)
	st := repotesting.NewReconnectableStorage(t, blobtesting.NewVersionedMapStorage(nil))

	env.RunAndExpectSuccess(t, "repo", "create", "in-memory", "--uuid",
		st.ConnectionInfo().Config.(*repotesting.ReconnectableStorageOptions).UUID)

	return env
}

func (s *formatSpecificTestSuite) TestRepositorySetParameters(t *testing.T) {
	env := s.setupInMemoryRepo(t)
	out := env.RunAndExpectSuccess(t, "repository", "status")

	// default values
	require.Contains(t, out, "Max pack length:     20 MiB")

	if s.formatVersion == content.FormatVersion1 {
		require.Contains(t, out, "Format version:      1")
	} else {
		require.Contains(t, out, "Format version:      2")
	}

	// failure cases
	env.RunAndExpectFailure(t, "repository", "set-parameters")
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--index-version=33")
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--max-pack-size-mb=9")
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--max-pack-size-mb=121")

	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--index-version=2", "--max-pack-size-mb=33")
	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Max pack length:     33 MiB")

	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--max-pack-size-mb=44")
	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Max pack length:     44 MiB")
}

func (s *formatSpecificTestSuite) TestRepositorySetParametersRetention(t *testing.T) {
	env := s.setupInMemoryRepo(t)

	// set retention
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--retention-mode", blob.Compliance.String(),
		"--retention-period", "24h")

	out := env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Blob retention mode:     COMPLIANCE")
	require.Contains(t, out, "Blob retention period:   24h0m0s")

	// update retention
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--retention-mode", blob.Governance.String(),
		"--retention-period", "24h1m")

	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Blob retention mode:     GOVERNANCE")
	require.Contains(t, out, "Blob retention period:   24h1m0s")

	// clear retention settings
	_, out = env.RunAndExpectSuccessWithErrOut(t, "repository", "set-parameters", "--retention-mode", "none")
	require.Contains(t, out, "disabling blob retention")

	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.NotContains(t, out, "Blob retention mode")
	require.NotContains(t, out, "Blob retention period")

	// invalid retention settings
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--retention-mode", "invalid-mode")
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--retention-mode", "COMPLIANCE", "--retention-period", "0h")
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--retention-mode", "COMPLIANCE", "--retention-period", "6h") // less than 24hr

	// set retention again after clear
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--retention-mode", "COMPLIANCE", "--retention-period", "24h")
	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Blob retention mode:     COMPLIANCE")
	require.Contains(t, out, "Blob retention period:   24h0m0s")

	// update without period
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--retention-period", "25h")
	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Blob retention mode:     COMPLIANCE")
	require.Contains(t, out, "Blob retention period:   25h0m0s")

	// update without mode
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--retention-mode", "GOVERNANCE")
	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Blob retention mode:     GOVERNANCE")
	require.Contains(t, out, "Blob retention period:   25h0m0s")

	// update retention (use days, weeks, nanoseconds)
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--retention-mode", blob.Compliance.String(),
		"--retention-period", "1w2d6h3ns")

	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Blob retention mode:     COMPLIANCE")
	require.Contains(t, out, "Blob retention period:   222h0m0.000000003s")
}

func (s *formatSpecificTestSuite) TestRepositorySetParametersUpgrade(t *testing.T) {
	env := s.setupInMemoryRepo(t)
	out := env.RunAndExpectSuccess(t, "repository", "status")

	// default values
	require.Contains(t, out, "Max pack length:     20 MiB")

	if s.formatVersion == content.FormatVersion1 {
		require.Contains(t, out, "Format version:      1")
		require.Contains(t, out, "Epoch Manager:       disabled")
		env.RunAndExpectFailure(t, "index", "epoch", "list")
	} else {
		require.Contains(t, out, "Format version:      2")
		require.Contains(t, out, "Epoch Manager:       enabled")
		env.RunAndExpectSuccess(t, "index", "epoch", "list")
	}

	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--upgrade")
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--epoch-min-duration", "3h")
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--epoch-cleanup-safety-margin", "23h")
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--epoch-advance-on-size-mb", "77")
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--epoch-advance-on-count", "22")
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--epoch-checkpoint-frequency", "9")

	env.RunAndExpectFailure(t, "repository", "set-parameters", "--epoch-min-duration", "1s")
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--epoch-refresh-frequency", "10h")
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--epoch-checkpoint-frequency", "-10")
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--epoch-cleanup-safety-margin", "10s")
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--epoch-advance-on-count", "1")

	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Epoch Manager:       enabled")
	require.Contains(t, out, "Index Format:        v2")
	require.Contains(t, out, "Format version:      2")
	require.Contains(t, out, "Epoch cleanup margin:    23h0m0s")
	require.Contains(t, out, "Epoch advance on:        22 blobs or 77 MiB, minimum 3h0m0s")
	require.Contains(t, out, "Epoch checkpoint every:  9 epochs")

	env.RunAndExpectSuccess(t, "index", "epoch", "list")
}
<commit_msg>test(cli): add separate test case for days in --retention-period flag (#2057)<commit_after>package cli_test

import (
	"testing"

	"github.com/stretchr/testify/require"

	"github.com/kopia/kopia/internal/blobtesting"
	"github.com/kopia/kopia/internal/repotesting"
	"github.com/kopia/kopia/repo/blob"
	"github.com/kopia/kopia/repo/content"
	"github.com/kopia/kopia/tests/testenv"
)

func (s *formatSpecificTestSuite) setupInMemoryRepo(t *testing.T) *testenv.CLITest {
	t.Helper()

	runner := testenv.NewInProcRunner(t)
	env := testenv.NewCLITest(t, s.formatFlags, runner)
	st := repotesting.NewReconnectableStorage(t, blobtesting.NewVersionedMapStorage(nil))

	env.RunAndExpectSuccess(t, "repo", "create", "in-memory", "--uuid",
		st.ConnectionInfo().Config.(*repotesting.ReconnectableStorageOptions).UUID)

	return env
}

func (s *formatSpecificTestSuite) TestRepositorySetParameters(t *testing.T) {
	env := s.setupInMemoryRepo(t)
	out := env.RunAndExpectSuccess(t, "repository", "status")

	// default values
	require.Contains(t, out, "Max pack length:     20 MiB")

	if s.formatVersion == content.FormatVersion1 {
		require.Contains(t, out, "Format version:      1")
	} else {
		require.Contains(t, out, "Format version:      2")
	}

	// failure cases
	env.RunAndExpectFailure(t, "repository", "set-parameters")
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--index-version=33")
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--max-pack-size-mb=9")
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--max-pack-size-mb=121")

	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--index-version=2", "--max-pack-size-mb=33")
	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Max pack length:     33 MiB")

	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--max-pack-size-mb=44")
	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Max pack length:     44 MiB")
}

func (s *formatSpecificTestSuite) TestRepositorySetParametersRetention(t *testing.T) {
	env := s.setupInMemoryRepo(t)

	// set retention
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--retention-mode", blob.Compliance.String(),
		"--retention-period", "24h")

	out := env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Blob retention mode:     COMPLIANCE")
	require.Contains(t, out, "Blob retention period:   24h0m0s")

	// update retention
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--retention-mode", blob.Governance.String(),
		"--retention-period", "24h1m")

	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Blob retention mode:     GOVERNANCE")
	require.Contains(t, out, "Blob retention period:   24h1m0s")

	// clear retention settings
	_, out = env.RunAndExpectSuccessWithErrOut(t, "repository", "set-parameters", "--retention-mode", "none")
	require.Contains(t, out, "disabling blob retention")

	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.NotContains(t, out, "Blob retention mode")
	require.NotContains(t, out, "Blob retention period")

	// invalid retention settings
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--retention-mode", "invalid-mode")
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--retention-mode", "COMPLIANCE", "--retention-period", "0h")
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--retention-mode", "COMPLIANCE", "--retention-period", "6h") // less than 24hr

	// set retention again after clear
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--retention-mode", "COMPLIANCE", "--retention-period", "24h")
	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Blob retention mode:     COMPLIANCE")
	require.Contains(t, out, "Blob retention period:   24h0m0s")

	// update without period
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--retention-period", "25h")
	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Blob retention mode:     COMPLIANCE")
	require.Contains(t, out, "Blob retention period:   25h0m0s")

	// update without mode
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--retention-mode", "GOVERNANCE")
	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Blob retention mode:     GOVERNANCE")
	require.Contains(t, out, "Blob retention period:   25h0m0s")

	// update retention (use days, weeks, nanoseconds)
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--retention-mode", blob.Compliance.String(),
		"--retention-period", "1w2d6h3ns")

	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Blob retention mode:     COMPLIANCE")
	require.Contains(t, out, "Blob retention period:   222h0m0.000000003s")

	// update retention (use days)
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--retention-mode", blob.Compliance.String(),
		"--retention-period", "7d")

	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Blob retention mode:     COMPLIANCE")
	require.Contains(t, out, "Blob retention period:   168h0m0s")
}

func (s *formatSpecificTestSuite) TestRepositorySetParametersUpgrade(t *testing.T) {
	env := s.setupInMemoryRepo(t)
	out := env.RunAndExpectSuccess(t, "repository", "status")

	// default values
	require.Contains(t, out, "Max pack length:     20 MiB")

	if s.formatVersion == content.FormatVersion1 {
		require.Contains(t, out, "Format version:      1")
		require.Contains(t, out, "Epoch Manager:       disabled")
		env.RunAndExpectFailure(t, "index", "epoch", "list")
	} else {
		require.Contains(t, out, "Format version:      2")
		require.Contains(t, out, "Epoch Manager:       enabled")
		env.RunAndExpectSuccess(t, "index", "epoch", "list")
	}

	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--upgrade")
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--epoch-min-duration", "3h")
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--epoch-cleanup-safety-margin", "23h")
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--epoch-advance-on-size-mb", "77")
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--epoch-advance-on-count", "22")
	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--epoch-checkpoint-frequency", "9")

	env.RunAndExpectFailure(t, "repository", "set-parameters", "--epoch-min-duration", "1s")
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--epoch-refresh-frequency", "10h")
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--epoch-checkpoint-frequency", "-10")
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--epoch-cleanup-safety-margin", "10s")
	env.RunAndExpectFailure(t, "repository", "set-parameters", "--epoch-advance-on-count", "1")

	out = env.RunAndExpectSuccess(t, "repository", "status")
	require.Contains(t, out, "Epoch Manager:       enabled")
	require.Contains(t, out, "Index Format:        v2")
	require.Contains(t, out, "Format version:      2")
	require.Contains(t, out, "Epoch cleanup margin:    23h0m0s")
	require.Contains(t, out, "Epoch advance on:        22 blobs or 77 MiB, minimum 3h0m0s")
	require.Contains(t, out, "Epoch checkpoint every:  9 epochs")

	env.RunAndExpectSuccess(t, "index", "epoch", "list")
}
<|endoftext|>


<commit_before>package profile_test

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"path/filepath"
	"testing"

	"github.com/cespare/xxhash"
	"github.com/tormoder/fit/cmd/fitgen/internal/profile"
)

const (
	testdata      = "testdata"
	fileExt       = ".xlsx"
	goldenSuffix  = ".golden"
	currentSuffix = ".current"
)

var update = flag.Bool("update", false, "update .golden output files")

func init() { flag.Parse() }

var currentSDK = sdks[0]

var defGenOpts = []profile.GeneratorOption{
	profile.WithGenerationTimestamp(false),
}

func relPath(sdkVersion string) string {
	return filepath.Join(testdata, sdkVersion+fileExt)
}

func writeProfile(p *profile.Profile, w io.Writer) error {
	var err error
	write := func(buf []byte) {
		if err != nil {
			return
		}
		_, err = w.Write(buf)
	}
	write([]byte("// TYPES\n"))
	write(p.TypesSource)
	write([]byte("// MESSAGES\n"))
	write(p.MessagesSource)
	write([]byte("// PROFILE\n"))
	write(p.ProfileSource)
	write([]byte("// STRINGER TYPE INPUT\n"))
	write([]byte(p.StringerInput))
	write([]byte("\n// MESSAGE NUMS WITHOUT MESSAGE\n"))
	for _, mn := range p.MesgNumsWithoutMessage {
		write([]byte(mn))
		write([]byte{'\n'})
	}
	return err
}

func writeProfileToFile(p *profile.Profile, path string) error {
	buf := new(bytes.Buffer)
	err := writeProfile(p, buf)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(path, buf.Bytes(), 0644)
}

func profileFingerprint(p *profile.Profile) uint64 {
	h := xxhash.New()
	_ = writeProfile(p, h)
	return h.Sum64()
}

type sdk struct {
	majVer, minVer    int
	goldenFingerprint uint64
}

var sdks = []sdk{
	{16, 20, 4359339570427813792},
	{20, 14, 15171043981936645962},
}

func TestGenerator(t *testing.T) {
	for _, sdk := range sdks {
		sdkFullVer := fmt.Sprintf("%d.%d", sdk.majVer, sdk.minVer)
		t.Run(sdkFullVer, func(t *testing.T) {
			if sdk == currentSDK && testing.Short() {
				t.Skip("skipping test in short mode")
			}
			path := relPath(sdkFullVer)
			data, err := ioutil.ReadFile(path)
			if err != nil {
				t.Fatal(err)
			}
			g, err := profile.NewGenerator(sdk.majVer, sdk.minVer, data, defGenOpts...)
			if err != nil {
				t.Fatal(err)
			}
			p, err := g.GenerateProfile()
			if err != nil {
				t.Fatal(err)
			}
			gotFP := profileFingerprint(p)
			if gotFP == sdk.goldenFingerprint {
				return
			}
			t.Errorf("profile fingerprint differs: got: %d, want %d", gotFP, sdk.goldenFingerprint)
			if !*update {
				path = path + currentSuffix
			} else {
				path = path + goldenSuffix
			}
			err = writeProfileToFile(p, path)
			if err != nil {
				t.Fatalf("error writing output: %v", err)
			}
			if !*update {
				t.Logf("current output written to: %s", path)
			} else {
				t.Logf("%q has been updated", path)
				t.Logf("new fingerprint is: %d", gotFP)
			}
		})
	}
}

var profileSink *profile.Profile

func BenchmarkGenerator(b *testing.B) {
	for _, sdk := range sdks {
		sdkFullVer := fmt.Sprintf("%d.%d", sdk.majVer, sdk.minVer)
		b.Run(sdkFullVer, func(b *testing.B) {
			path := relPath(sdkFullVer)
			data, err := ioutil.ReadFile(path)
			if err != nil {
				b.Fatalf("error reading profile workbook: %v", err)
			}
			b.ReportAllocs()
			b.ResetTimer()
			for i := 0; i < b.N; i++ {
				g, err := profile.NewGenerator(sdk.majVer, sdk.minVer, data, defGenOpts...)
				if err != nil {
					b.Fatal(err)
				}
				profileSink, err = g.GenerateProfile()
				if err != nil {
					b.Fatal(err)
				}
			}
		})
	}
}
<commit_msg>internal/profile: make TestGenerator parallel<commit_after>package profile_test

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"path/filepath"
	"testing"

	"github.com/cespare/xxhash"
	"github.com/tormoder/fit/cmd/fitgen/internal/profile"
)

const (
	testdata      = "testdata"
	fileExt       = ".xlsx"
	goldenSuffix  = ".golden"
	currentSuffix = ".current"
)

var update = flag.Bool("update", false, "update .golden output files")

func init() { flag.Parse() }

var currentSDK = sdks[0]

var defGenOpts = []profile.GeneratorOption{
	profile.WithGenerationTimestamp(false),
}

func relPath(sdkVersion string) string {
	return filepath.Join(testdata, sdkVersion+fileExt)
}

func writeProfile(p *profile.Profile, w io.Writer) error {
	var err error
	write := func(buf []byte) {
		if err != nil {
			return
		}
		_, err = w.Write(buf)
	}
	write([]byte("// TYPES\n"))
	write(p.TypesSource)
	write([]byte("// MESSAGES\n"))
	write(p.MessagesSource)
	write([]byte("// PROFILE\n"))
	write(p.ProfileSource)
	write([]byte("// STRINGER TYPE INPUT\n"))
	write([]byte(p.StringerInput))
	write([]byte("\n// MESSAGE NUMS WITHOUT MESSAGE\n"))
	for _, mn := range p.MesgNumsWithoutMessage {
		write([]byte(mn))
		write([]byte{'\n'})
	}
	return err
}

func writeProfileToFile(p *profile.Profile, path string) error {
	buf := new(bytes.Buffer)
	err := writeProfile(p, buf)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(path, buf.Bytes(), 0644)
}

func profileFingerprint(p *profile.Profile) uint64 {
	h := xxhash.New()
	_ = writeProfile(p, h)
	return h.Sum64()
}

type sdk struct {
	majVer, minVer    int
	goldenFingerprint uint64
}

var sdks = []sdk{
	{16, 20, 4359339570427813792},
	{20, 14, 15171043981936645962},
}

func TestGenerator(t *testing.T) {
	for _, sdk := range sdks {
		sdk := sdk // Capture range variable.
		sdkFullVer := fmt.Sprintf("%d.%d", sdk.majVer, sdk.minVer)
		t.Run(sdkFullVer, func(t *testing.T) {
			t.Parallel()
			if sdk == currentSDK && testing.Short() {
				t.Skip("skipping test in short mode")
			}
			path := relPath(sdkFullVer)
			data, err := ioutil.ReadFile(path)
			if err != nil {
				t.Fatal(err)
			}
			g, err := profile.NewGenerator(sdk.majVer, sdk.minVer, data, defGenOpts...)
			if err != nil {
				t.Fatal(err)
			}
			p, err := g.GenerateProfile()
			if err != nil {
				t.Fatal(err)
			}
			gotFP := profileFingerprint(p)
			if gotFP == sdk.goldenFingerprint {
				return
			}
			t.Errorf("profile fingerprint differs: got: %d, want %d", gotFP, sdk.goldenFingerprint)
			if !*update {
				path = path + currentSuffix
			} else {
				path = path + goldenSuffix
			}
			err = writeProfileToFile(p, path)
			if err != nil {
				t.Fatalf("error writing output: %v", err)
			}
			if !*update {
				t.Logf("current output written to: %s", path)
			} else {
				t.Logf("%q has been updated", path)
				t.Logf("new fingerprint is: %d", gotFP)
			}
		})
	}
}

var profileSink *profile.Profile

func BenchmarkGenerator(b *testing.B) {
	for _, sdk := range sdks {
		sdkFullVer := fmt.Sprintf("%d.%d", sdk.majVer, sdk.minVer)
		b.Run(sdkFullVer, func(b *testing.B) {
			path := relPath(sdkFullVer)
			data, err := ioutil.ReadFile(path)
			if err != nil {
				b.Fatalf("error reading profile workbook: %v", err)
			}
			b.ReportAllocs()
			b.ResetTimer()
			for i := 0; i < b.N; i++ {
				g, err := profile.NewGenerator(sdk.majVer, sdk.minVer, data, defGenOpts...)
				if err != nil {
					b.Fatal(err)
				}
				profileSink, err = g.GenerateProfile()
				if err != nil {
					b.Fatal(err)
				}
			}
		})
	}
}
<|endoftext|>


<commit_before>package fetcheddit

import (
	"code.leeclagett.com/grokeddit"
	"errors"
)

type thingIterater interface {
	setArray([]grokeddit.Thing)
	hasNext() bool
	getNext() (grokeddit.Thing, error)
}

type nextChunk struct {
	nextThings     []grokeddit.Thing
	moreThings     bool
	retrievalError error
}

type thingList struct {
	fetchMoreThings   func() ([]grokeddit.Thing, bool, error)
	currentThings     thingIterater // An abstraction for iterating over the current slice of Thing objects. Implementations can change the order of iterating.
	moreThings        bool          // The last value returned by fetchMoreThings
	chunkChannel chan nextChunk
}

func fetchThingList(fetchGroked func(anchor *AnchorPoint) (grokeddit.Groked, error), anchor *AnchorPoint) (thingList, error) {
	var currentThingIterater thingIterater
	var fetchMore func() ([]grokeddit.Thing, bool, error)

	// make a copy of the anchor, since it will be stored in a
	// closure thats invoked asynchronously
	if anchor != nil {
		newAnchor := new(AnchorPoint)
		*newAnchor = *anchor
		anchor = newAnchor
	}

	if anchor == nil || anchor.Direction == Next {
		currentThingIterater = &forwardThingIterate{}
		fetchMore = func() ([]grokeddit.Thing, bool, error) {
			newGroked, error := fetchGroked(anchor)
			if error != nil {
				return nil, false, error
			}

			if anchor == nil {
				anchor = new(AnchorPoint)
				anchor.Direction = Next
			}

			if newGroked.ListingNext != nil {
				anchor.Anchor = *newGroked.ListingNext
			}

			return newGroked.Children, newGroked.ListingNext != nil, nil
		}
	} else {
		currentThingIterater = &reverseThingIterate{}
		fetchMore = func() ([]grokeddit.Thing, bool, error) {
			newGroked, error := fetchGroked(anchor)
			if error != nil {
				return nil, false, error
			}

			// There is a bug on the reddit side, the "before"
			// field is always NULL with a before request. This is
			// solved with a hack that will do 1 extra request
			// than necessary, the additional traffic is on you
			// reddit.
			more := false
			if newGroked.Children != nil && len(newGroked.Children) > 0 {
				anchor.Anchor = newGroked.Children[0].Id
				more = true
			}

			return newGroked.Children, more, nil
		}
	}

	firstThings, moreThings, error := fetchMore()
	if error != nil {
		return thingList{}, errors.New("No things in first request")
	}

	currentThingIterater.setArray(firstThings)
	newThingList := thingList{fetchMore, currentThingIterater, moreThings, make(chan nextChunk)}
	newThingList.fetchNextBlockAsync()
	return newThingList, nil
}

func (list *thingList) fetchNextBlockAsync() {
	go func() {
		nextThings, moreThings, error := list.fetchMoreThings()
		list.chunkChannel <- nextChunk{nextThings, moreThings, error}
	}()
}

func (list *thingList) hasNext() bool {
	return list.moreThings || list.currentThings.hasNext()
}

func (list *thingList) getNext() (grokeddit.Thing, error) {
	if !list.hasNext() {
		return grokeddit.Thing{}, errors.New("No more things to iterate")
	}

	if !list.currentThings.hasNext() {
		nextChunk := <- list.chunkChannel

		if nextChunk.moreThings {
			list.fetchNextBlockAsync()
		}

		if nextChunk.retrievalError != nil {
			return grokeddit.Thing{}, errors.New("Unable to fetch more things: " + nextChunk.retrievalError.Error())
		}

		list.moreThings = nextChunk.moreThings
		list.currentThings.setArray(nextChunk.nextThings)
	}

	return list.currentThings.getNext()
}
<commit_msg>Fixed an obvious bug. How about those unit tests?<commit_after>package fetcheddit

import (
	"code.leeclagett.com/grokeddit"
	"errors"
)

type thingIterater interface {
	setArray([]grokeddit.Thing)
	hasNext() bool
	getNext() (grokeddit.Thing, error)
}

type nextChunk struct {
	nextThings     []grokeddit.Thing
	moreThings     bool
	retrievalError error
}

type thingList struct {
	fetchMoreThings   func() ([]grokeddit.Thing, bool, error)
	currentThings     thingIterater // An abstraction for iterating over the current slice of Thing objects. Implementations can change the order of iterating.
	moreThings        bool          // The last value returned by fetchMoreThings
	chunkChannel chan nextChunk
}

func fetchThingList(fetchGroked func(anchor *AnchorPoint) (grokeddit.Groked, error), anchor *AnchorPoint) (thingList, error) {
	var currentThingIterater thingIterater
	var fetchMore func() ([]grokeddit.Thing, bool, error)

	// make a copy of the anchor, since it will be stored in a
	// closure thats invoked asynchronously
	if anchor != nil {
		newAnchor := new(AnchorPoint)
		*newAnchor = *anchor
		anchor = newAnchor
	}

	if anchor == nil || anchor.Direction == Next {
		currentThingIterater = &forwardThingIterate{}
		fetchMore = func() ([]grokeddit.Thing, bool, error) {
			newGroked, error := fetchGroked(anchor)
			if error != nil {
				return nil, false, error
			}

			if anchor == nil {
				anchor = new(AnchorPoint)
				anchor.Direction = Next
			}

			if newGroked.ListingNext != nil {
				anchor.Anchor = *newGroked.ListingNext
			}

			return newGroked.Children, newGroked.ListingNext != nil, nil
		}
	} else {
		currentThingIterater = &reverseThingIterate{}
		fetchMore = func() ([]grokeddit.Thing, bool, error) {
			newGroked, error := fetchGroked(anchor)
			if error != nil {
				return nil, false, error
			}

			// There is a bug on the reddit side, the "before"
			// field is always NULL with a before request. This is
			// solved with a hack that will do 1 extra request
			// than necessary, the additional traffic is on you
			// reddit.
			more := false
			if newGroked.Children != nil && len(newGroked.Children) > 0 {
				anchor.Anchor = newGroked.Children[0].Id
				more = true
			}

			return newGroked.Children, more, nil
		}
	}

	firstThings, moreThings, error := fetchMore()
	if error != nil {
		return thingList{}, errors.New("No things in first request")
	}

	currentThingIterater.setArray(firstThings)
	newThingList := thingList{fetchMore, currentThingIterater, moreThings, make(chan nextChunk)}

	if moreThings {
		newThingList.fetchNextBlockAsync()
	}
	return newThingList, nil
}

func (list *thingList) fetchNextBlockAsync() {
	go func() {
		nextThings, moreThings, error := list.fetchMoreThings()
		list.chunkChannel <- nextChunk{nextThings, moreThings, error}
	}()
}

func (list *thingList) hasNext() bool {
	return list.moreThings || list.currentThings.hasNext()
}

func (list *thingList) getNext() (grokeddit.Thing, error) {
	if !list.hasNext() {
		return grokeddit.Thing{}, errors.New("No more things to iterate")
	}

	if !list.currentThings.hasNext() {
		nextChunk := <- list.chunkChannel

		if nextChunk.moreThings {
			list.fetchNextBlockAsync()
		}

		if nextChunk.retrievalError != nil {
			return grokeddit.Thing{}, errors.New("Unable to fetch more things: " + nextChunk.retrievalError.Error())
		}

		list.moreThings = nextChunk.moreThings
		list.currentThings.setArray(nextChunk.nextThings)
	}

	return list.currentThings.getNext()
}
<|endoftext|>


<commit_before>package sgload

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"path"
	"regexp"
	"strings"
	"time"

	sgreplicate "github.com/couchbaselabs/sg-replicate"
	"github.com/peterbourgon/g2s"
)

var (
	// The sample rate -- set this to 0.1 if you only want
	// 10% of the samples to be pushed to statds, or .01 if you only
	// want 1% of the samples pushed to statsd.  Useful for
	// not overwhelming stats if you have too many samples.
	statsdSampleRate float32 = 1.0
)

type SGDataStore struct {
	SyncGatewayUrl       string
	SyncGatewayAdminPort int
	UserCreds            UserCred
	StatsdClient         *g2s.Statsd
	CompressionEnabled   bool
}

func NewSGDataStore(sgUrl string, sgAdminPort int, statsdClient *g2s.Statsd, compressionEnabled bool) *SGDataStore {
	return &SGDataStore{
		SyncGatewayUrl:       sgUrl,
		SyncGatewayAdminPort: sgAdminPort,
		StatsdClient:         statsdClient,
		CompressionEnabled:   compressionEnabled,
	}
}

func (s *SGDataStore) SetUserCreds(u UserCred) {
	s.UserCreds = u
}

func (s SGDataStore) CreateUser(u UserCred, channelNames []string) error {

	adminUrl, err := s.sgAdminURL()
	if err != nil {
		return err
	}

	adminUrlUserEndpoint, err := addEndpointToUrl(adminUrl, "_user")
	if err != nil {
		return err
	}

	adminUrlUserEndpoint = addTrailingSlash(adminUrlUserEndpoint)

	userDoc := map[string]interface{}{}
	userDoc["name"] = u.Username
	userDoc["password"] = u.Password
	userDoc["admin_channels"] = channelNames

	docBytes, err := json.Marshal(userDoc)
	if err != nil {
		return err
	}
	buf := bytes.NewBuffer(docBytes)

	req, err := http.NewRequest("POST", adminUrlUserEndpoint, buf)

	req.Header.Set("Content-Type", "application/json")

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	s.pushTimingStat("create_user", time.Since(startTime))

	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	io.Copy(ioutil.Discard, resp.Body)

	return nil
}

func (s SGDataStore) sgAdminURL() (string, error) {

	sgAdminURL := s.SyncGatewayUrl

	parsedSgUrl, err := url.Parse(s.SyncGatewayUrl)
	if err != nil {
		log.Printf("Error parsing url: %v", err)
		return sgAdminURL, err
	}

	// find the port from the url
	host, port, err := splitHostPortWrapper(parsedSgUrl.Host)
	if err != nil {
		return sgAdminURL, err
	}

	if port != "" {
		// is there a port?
		// do a regex replace on :port on the original url
		r := regexp.MustCompile(port)
		sgAdminURL = r.ReplaceAllString(
			sgAdminURL,
			fmt.Sprintf("%d", s.SyncGatewayAdminPort),
		)

	} else {
		// is there no port?
		// do a regex replace of host with host:port
		r := regexp.MustCompile(host)
		hostWithAdminPort := fmt.Sprintf("%v:%v", host, s.SyncGatewayAdminPort)
		sgAdminURL = r.ReplaceAllString(
			sgAdminURL,
			hostWithAdminPort,
		)

	}

	// return it
	return sgAdminURL, nil

}

func (s SGDataStore) Changes(sinceVal Sincer, limit int) (changes sgreplicate.Changes, newSinceVal Sincer, err error) {

	changesFeedEndpoint, err := addEndpointToUrl(s.SyncGatewayUrl, "_changes")
	if err != nil {
		return sgreplicate.Changes{}, sinceVal, err
	}

	changesFeedParams := NewChangesFeedParams(sinceVal, limit)

	changesFeedUrl := fmt.Sprintf(
		"%s?%s",
		changesFeedEndpoint,
		changesFeedParams,
	)

	req, err := http.NewRequest("GET", changesFeedUrl, nil)
	s.addAuthIfNeeded(req)

	req.Header.Set("Content-Type", "application/json")

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return sgreplicate.Changes{}, sinceVal, err
	}
	defer resp.Body.Close()

	s.pushTimingStat("changes_feed", time.Since(startTime))
	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return sgreplicate.Changes{}, sinceVal, fmt.Errorf("Unexpected response status for changes_feed GET request: %d", resp.StatusCode)
	}

	decoder := json.NewDecoder(resp.Body)
	err = decoder.Decode(&changes)
	if err != nil {
		return sgreplicate.Changes{}, sinceVal, err
	}
	lastSequenceStr, ok := changes.LastSequence.(string)
	if !ok {
		return sgreplicate.Changes{}, sinceVal, fmt.Errorf("Could not convert changes.LastSequence to string")
	}
	lastSequenceSincer := StringSincer{
		Since: lastSequenceStr,
	}

	return changes, lastSequenceSincer, nil
}

// Create a single document in Sync Gateway
func (s SGDataStore) CreateDocument(d Document) (sgreplicate.DocumentRevisionPair, error) {

	docRevisionPairs, err := s.BulkCreateDocuments([]Document{d}, true)
	if err != nil {
		return sgreplicate.DocumentRevisionPair{}, err
	}
	if len(docRevisionPairs) == 0 {
		return sgreplicate.DocumentRevisionPair{}, fmt.Errorf("Unexpected response")
	}
	return docRevisionPairs[0], nil

}

// Bulk create a set of documents in Sync Gateway
func (s SGDataStore) BulkCreateDocuments(docs []Document, newEdits bool) ([]sgreplicate.DocumentRevisionPair, error) {

	defer s.pushCounter("create_document_counter", len(docs))

	// Set the "created_at" timestamp which is used to calculate the
	// gateload roundtrip time
	updateCreatedAtTimestamp(docs)

	bulkDocsResponse := []sgreplicate.DocumentRevisionPair{}

	bulkDocsEndpoint, err := addEndpointToUrl(s.SyncGatewayUrl, "_bulk_docs")
	if err != nil {
		return bulkDocsResponse, err
	}

	bulkDocs := BulkDocs{
		Documents: docs,
		NewEdits:  newEdits,
	}

	docBytes, err := json.Marshal(bulkDocs)
	if err != nil {
		return bulkDocsResponse, err
	}
	var buf *bytes.Buffer
	if s.CompressionEnabled {
		buf = &bytes.Buffer{}
		gzipWriter := gzip.NewWriter(buf)
		if _, err := gzipWriter.Write(docBytes); err != nil {
			return bulkDocsResponse, err
		}
		if err = gzipWriter.Close(); err != nil {
			return bulkDocsResponse, err
		}
	} else {
		buf = bytes.NewBuffer(docBytes)
	}

	req, err := http.NewRequest("POST", bulkDocsEndpoint, buf)
	s.addAuthIfNeeded(req)

	req.Header.Set("Content-Type", "application/json")
	if s.CompressionEnabled {
		req.Header.Set("Content-Encoding", "gzip")
	}

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return bulkDocsResponse, err
	}
	defer resp.Body.Close()

	s.pushTimingStat("create_document", timeDeltaPerDocument(len(docs), time.Since(startTime)))
	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return bulkDocsResponse, fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	decoder := json.NewDecoder(resp.Body)
	if err = decoder.Decode(&bulkDocsResponse); err != nil {
		return bulkDocsResponse, err
	}

	// If any of the bulk docs had errors, return an error
	for _, docRevisionPair := range bulkDocsResponse {
		if docRevisionPair.Error != "" {
			return bulkDocsResponse, fmt.Errorf("%v", docRevisionPair.Error)
		}
	}

	return bulkDocsResponse, nil

}

func (s SGDataStore) BulkGetDocuments(r sgreplicate.BulkGetRequest) ([]sgreplicate.Document, error) {

	defer s.pushCounter("get_document_counter", len(r.Docs))

	bulkGetEndpoint, err := addEndpointToUrl(s.SyncGatewayUrl, "_bulk_get")
	if err != nil {
		return nil, err
	}

	bulkGetBytes, err := json.Marshal(r)
	if err != nil {
		return nil, fmt.Errorf("BulkGetDocuemnts failed to marshal request: %v", err)
	}

	buf := bytes.NewBuffer(bulkGetBytes)

	req, err := http.NewRequest("POST", bulkGetEndpoint, buf)
	s.addAuthIfNeeded(req)

	req.Header.Set("Content-Type", "application/json")

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	s.pushTimingStat("get_document", timeDeltaPerDocument(len(r.Docs), time.Since(startTime)))
	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return nil, fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	// Parse the response and make sure that we got all the docs we requested

	// Logger function that ignores any logging from the ReadBulkGetResponse method
	loggerFunc := func(key string, format string, args ...interface{}) {}
	documents, err := sgreplicate.ReadBulkGetResponse(resp, loggerFunc)
	if len(documents) != len(r.Docs) {
		return nil, fmt.Errorf("Expected %d docs, got %d docs", len(r.Docs), len(documents))
	}

	for _, doc := range documents {
		createAtRFC3339NanoIface, ok := doc.Body["created_at"]
		if !ok {
			logger.Warn("Document missing created_at field", "doc.Body", doc.Body)
			continue
		}
		createAtRFC3339NanoStr, ok := createAtRFC3339NanoIface.(string)
		if !ok {
			logger.Warn("Document created_at not a string", "doc.Body", doc.Body)
			continue
		}
		createAtRFC3339Nano, err := time.Parse(
			time.RFC3339Nano,
			createAtRFC3339NanoStr,
		)
		if err != nil {
			logger.Warn("Could not parse doc.created_at field into time", "createAtRFC3339Nano", createAtRFC3339Nano)
			continue
		}
		delta := time.Since(createAtRFC3339Nano)
		s.pushTimingStat("gateload_roundtrip", delta)

		possiblyLogVerboseWarning(delta, doc)

	}

	return documents, nil

}

// If the round trip time is over a certain threshold, log a verbose
// warning.  Trying to debug https://github.com/couchbaselabs/sgload/issues/12
func possiblyLogVerboseWarning(delta time.Duration, doc sgreplicate.Document) {

	maxSecondsExpected := time.Duration(60 * time.Second)
	if delta > time.Duration(time.Second*maxSecondsExpected) {

		createAtRFC3339NanoIface, ok := doc.Body["created_at"]
		if !ok {
			logger.Warn("Document missing created_at field", "doc.Body", doc.Body)
		}
		createAtRFC3339NanoStr, ok := createAtRFC3339NanoIface.(string)
		if !ok {
			logger.Warn("Document created_at not a string", "doc.Body", doc.Body)
		}
		createAtRFC3339Nano, err := time.Parse(
			time.RFC3339Nano,
			createAtRFC3339NanoStr,
		)
		if err != nil {
			logger.Warn("Could not parse doc.created_at field into time", "createAtRFC3339Nano", createAtRFC3339Nano)
		}

		logger.Warn(
			"Gateload roundtrip time exceeeded expected time",
			"maxSecondsExpected",
			maxSecondsExpected,
			"delta",
			delta,
			"createAtRFC3339NanoStr",
			createAtRFC3339NanoStr,
			"createAtRFC3339Nano",
			createAtRFC3339Nano,
			"time.Since(createAtRFC3339Nano)",
			time.Since(createAtRFC3339Nano),
			"now",
			time.Now(),
			"now RFC3339Nano",
			time.Now().Format(time.RFC3339Nano),
		)

	}

}

// add BasicAuth header for user if needed
func (s SGDataStore) addAuthIfNeeded(req *http.Request) {
	if !s.UserCreds.Empty() {
		req.SetBasicAuth(s.UserCreds.Username, s.UserCreds.Password)
		req.Header.Set("X-sgload-username", s.UserCreds.Username)
	}
}

func (s SGDataStore) pushTimingStat(key string, delta time.Duration) {
	if s.StatsdClient == nil {
		return
	}
	s.StatsdClient.Timing(
		statsdSampleRate,
		key,
		delta,
	)
}

func (s SGDataStore) pushCounter(key string, n int) {
	if s.StatsdClient == nil {
		return
	}
	s.StatsdClient.Counter(
		statsdSampleRate,
		key,
		n,
	)
}

func splitHostPortWrapper(host string) (string, string, error) {
	if !strings.Contains(host, ":") {
		return host, "", nil
	}

	return net.SplitHostPort(host)
}

func addEndpointToUrl(urlStr, endpoint string) (string, error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return "", err
	}
	u.Path = path.Join(u.Path, endpoint)
	return u.String(), nil
}

func addTrailingSlash(urlStr string) string {
	return fmt.Sprintf("%v/", urlStr)
}

func timeDeltaPerDocument(numDocs int, timeDeltaAllDocs time.Duration) time.Duration {
	if numDocs == 0 {
		return timeDeltaAllDocs
	}
	return time.Duration(int64(timeDeltaAllDocs) / int64(numDocs))
}
<commit_msg>Fixes #23 by bumping the max idle connections per host<commit_after>package sgload

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"path"
	"regexp"
	"strings"
	"time"

	sgreplicate "github.com/couchbaselabs/sg-replicate"
	"github.com/peterbourgon/g2s"
)

var (
	// The sample rate -- set this to 0.1 if you only want
	// 10% of the samples to be pushed to statds, or .01 if you only
	// want 1% of the samples pushed to statsd.  Useful for
	// not overwhelming stats if you have too many samples.
	statsdSampleRate float32 = 1.0

	sgClient *http.Client
)

func init() {
	customDefaultTransport := customDefaultTransportWithConnPool(1000)
	sgClient = &http.Client{Transport: customDefaultTransport}
}

type SGDataStore struct {
	SyncGatewayUrl       string
	SyncGatewayAdminPort int
	UserCreds            UserCred
	StatsdClient         *g2s.Statsd
	CompressionEnabled   bool
}

func NewSGDataStore(sgUrl string, sgAdminPort int, statsdClient *g2s.Statsd, compressionEnabled bool) *SGDataStore {
	return &SGDataStore{
		SyncGatewayUrl:       sgUrl,
		SyncGatewayAdminPort: sgAdminPort,
		StatsdClient:         statsdClient,
		CompressionEnabled:   compressionEnabled,
	}
}

func (s *SGDataStore) SetUserCreds(u UserCred) {
	s.UserCreds = u
}

func (s SGDataStore) CreateUser(u UserCred, channelNames []string) error {

	adminUrl, err := s.sgAdminURL()
	if err != nil {
		return err
	}

	adminUrlUserEndpoint, err := addEndpointToUrl(adminUrl, "_user")
	if err != nil {
		return err
	}

	adminUrlUserEndpoint = addTrailingSlash(adminUrlUserEndpoint)

	userDoc := map[string]interface{}{}
	userDoc["name"] = u.Username
	userDoc["password"] = u.Password
	userDoc["admin_channels"] = channelNames

	docBytes, err := json.Marshal(userDoc)
	if err != nil {
		return err
	}
	buf := bytes.NewBuffer(docBytes)

	req, err := http.NewRequest("POST", adminUrlUserEndpoint, buf)

	req.Header.Set("Content-Type", "application/json")

	client := getHttpClient()

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	s.pushTimingStat("create_user", time.Since(startTime))

	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	io.Copy(ioutil.Discard, resp.Body)

	return nil
}

func (s SGDataStore) sgAdminURL() (string, error) {

	sgAdminURL := s.SyncGatewayUrl

	parsedSgUrl, err := url.Parse(s.SyncGatewayUrl)
	if err != nil {
		log.Printf("Error parsing url: %v", err)
		return sgAdminURL, err
	}

	// find the port from the url
	host, port, err := splitHostPortWrapper(parsedSgUrl.Host)
	if err != nil {
		return sgAdminURL, err
	}

	if port != "" {
		// is there a port?
		// do a regex replace on :port on the original url
		r := regexp.MustCompile(port)
		sgAdminURL = r.ReplaceAllString(
			sgAdminURL,
			fmt.Sprintf("%d", s.SyncGatewayAdminPort),
		)

	} else {
		// is there no port?
		// do a regex replace of host with host:port
		r := regexp.MustCompile(host)
		hostWithAdminPort := fmt.Sprintf("%v:%v", host, s.SyncGatewayAdminPort)
		sgAdminURL = r.ReplaceAllString(
			sgAdminURL,
			hostWithAdminPort,
		)

	}

	// return it
	return sgAdminURL, nil

}

func (s SGDataStore) Changes(sinceVal Sincer, limit int) (changes sgreplicate.Changes, newSinceVal Sincer, err error) {

	changesFeedEndpoint, err := addEndpointToUrl(s.SyncGatewayUrl, "_changes")
	if err != nil {
		return sgreplicate.Changes{}, sinceVal, err
	}

	changesFeedParams := NewChangesFeedParams(sinceVal, limit)

	changesFeedUrl := fmt.Sprintf(
		"%s?%s",
		changesFeedEndpoint,
		changesFeedParams,
	)

	req, err := http.NewRequest("GET", changesFeedUrl, nil)
	s.addAuthIfNeeded(req)

	req.Header.Set("Content-Type", "application/json")

	client := getHttpClient()

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return sgreplicate.Changes{}, sinceVal, err
	}
	defer resp.Body.Close()

	s.pushTimingStat("changes_feed", time.Since(startTime))
	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return sgreplicate.Changes{}, sinceVal, fmt.Errorf("Unexpected response status for changes_feed GET request: %d", resp.StatusCode)
	}

	decoder := json.NewDecoder(resp.Body)
	err = decoder.Decode(&changes)
	if err != nil {
		return sgreplicate.Changes{}, sinceVal, err
	}
	lastSequenceStr, ok := changes.LastSequence.(string)
	if !ok {
		return sgreplicate.Changes{}, sinceVal, fmt.Errorf("Could not convert changes.LastSequence to string")
	}
	lastSequenceSincer := StringSincer{
		Since: lastSequenceStr,
	}

	return changes, lastSequenceSincer, nil
}

// Create a single document in Sync Gateway
func (s SGDataStore) CreateDocument(d Document) (sgreplicate.DocumentRevisionPair, error) {

	docRevisionPairs, err := s.BulkCreateDocuments([]Document{d}, true)
	if err != nil {
		return sgreplicate.DocumentRevisionPair{}, err
	}
	if len(docRevisionPairs) == 0 {
		return sgreplicate.DocumentRevisionPair{}, fmt.Errorf("Unexpected response")
	}
	return docRevisionPairs[0], nil

}

// Bulk create a set of documents in Sync Gateway
func (s SGDataStore) BulkCreateDocuments(docs []Document, newEdits bool) ([]sgreplicate.DocumentRevisionPair, error) {

	defer s.pushCounter("create_document_counter", len(docs))

	// Set the "created_at" timestamp which is used to calculate the
	// gateload roundtrip time
	updateCreatedAtTimestamp(docs)

	bulkDocsResponse := []sgreplicate.DocumentRevisionPair{}

	bulkDocsEndpoint, err := addEndpointToUrl(s.SyncGatewayUrl, "_bulk_docs")
	if err != nil {
		return bulkDocsResponse, err
	}

	bulkDocs := BulkDocs{
		Documents: docs,
		NewEdits:  newEdits,
	}

	docBytes, err := json.Marshal(bulkDocs)
	if err != nil {
		return bulkDocsResponse, err
	}
	var buf *bytes.Buffer
	if s.CompressionEnabled {
		buf = &bytes.Buffer{}
		gzipWriter := gzip.NewWriter(buf)
		if _, err := gzipWriter.Write(docBytes); err != nil {
			return bulkDocsResponse, err
		}
		if err = gzipWriter.Close(); err != nil {
			return bulkDocsResponse, err
		}
	} else {
		buf = bytes.NewBuffer(docBytes)
	}

	req, err := http.NewRequest("POST", bulkDocsEndpoint, buf)
	s.addAuthIfNeeded(req)

	req.Header.Set("Content-Type", "application/json")
	if s.CompressionEnabled {
		req.Header.Set("Content-Encoding", "gzip")
	}

	client := getHttpClient()

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return bulkDocsResponse, err
	}
	defer resp.Body.Close()

	s.pushTimingStat("create_document", timeDeltaPerDocument(len(docs), time.Since(startTime)))
	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return bulkDocsResponse, fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	decoder := json.NewDecoder(resp.Body)
	if err = decoder.Decode(&bulkDocsResponse); err != nil {
		return bulkDocsResponse, err
	}

	// If any of the bulk docs had errors, return an error
	for _, docRevisionPair := range bulkDocsResponse {
		if docRevisionPair.Error != "" {
			return bulkDocsResponse, fmt.Errorf("%v", docRevisionPair.Error)
		}
	}

	return bulkDocsResponse, nil

}

func (s SGDataStore) BulkGetDocuments(r sgreplicate.BulkGetRequest) ([]sgreplicate.Document, error) {

	defer s.pushCounter("get_document_counter", len(r.Docs))

	bulkGetEndpoint, err := addEndpointToUrl(s.SyncGatewayUrl, "_bulk_get")
	if err != nil {
		return nil, err
	}

	bulkGetBytes, err := json.Marshal(r)
	if err != nil {
		return nil, fmt.Errorf("BulkGetDocuemnts failed to marshal request: %v", err)
	}

	buf := bytes.NewBuffer(bulkGetBytes)

	req, err := http.NewRequest("POST", bulkGetEndpoint, buf)
	s.addAuthIfNeeded(req)

	req.Header.Set("Content-Type", "application/json")

	client := getHttpClient()

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	s.pushTimingStat("get_document", timeDeltaPerDocument(len(r.Docs), time.Since(startTime)))
	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return nil, fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	// Parse the response and make sure that we got all the docs we requested

	// Logger function that ignores any logging from the ReadBulkGetResponse method
	loggerFunc := func(key string, format string, args ...interface{}) {}
	documents, err := sgreplicate.ReadBulkGetResponse(resp, loggerFunc)
	if len(documents) != len(r.Docs) {
		return nil, fmt.Errorf("Expected %d docs, got %d docs", len(r.Docs), len(documents))
	}

	for _, doc := range documents {
		createAtRFC3339NanoIface, ok := doc.Body["created_at"]
		if !ok {
			logger.Warn("Document missing created_at field", "doc.Body", doc.Body)
			continue
		}
		createAtRFC3339NanoStr, ok := createAtRFC3339NanoIface.(string)
		if !ok {
			logger.Warn("Document created_at not a string", "doc.Body", doc.Body)
			continue
		}
		createAtRFC3339Nano, err := time.Parse(
			time.RFC3339Nano,
			createAtRFC3339NanoStr,
		)
		if err != nil {
			logger.Warn("Could not parse doc.created_at field into time", "createAtRFC3339Nano", createAtRFC3339Nano)
			continue
		}
		delta := time.Since(createAtRFC3339Nano)
		s.pushTimingStat("gateload_roundtrip", delta)

		possiblyLogVerboseWarning(delta, doc)

	}

	return documents, nil

}

// If the round trip time is over a certain threshold, log a verbose
// warning.  Trying to debug https://github.com/couchbaselabs/sgload/issues/12
func possiblyLogVerboseWarning(delta time.Duration, doc sgreplicate.Document) {

	maxSecondsExpected := time.Duration(60 * time.Second)
	if delta > time.Duration(time.Second*maxSecondsExpected) {

		createAtRFC3339NanoIface, ok := doc.Body["created_at"]
		if !ok {
			logger.Warn("Document missing created_at field", "doc.Body", doc.Body)
		}
		createAtRFC3339NanoStr, ok := createAtRFC3339NanoIface.(string)
		if !ok {
			logger.Warn("Document created_at not a string", "doc.Body", doc.Body)
		}
		createAtRFC3339Nano, err := time.Parse(
			time.RFC3339Nano,
			createAtRFC3339NanoStr,
		)
		if err != nil {
			logger.Warn("Could not parse doc.created_at field into time", "createAtRFC3339Nano", createAtRFC3339Nano)
		}

		logger.Warn(
			"Gateload roundtrip time exceeeded expected time",
			"maxSecondsExpected",
			maxSecondsExpected,
			"delta",
			delta,
			"createAtRFC3339NanoStr",
			createAtRFC3339NanoStr,
			"createAtRFC3339Nano",
			createAtRFC3339Nano,
			"time.Since(createAtRFC3339Nano)",
			time.Since(createAtRFC3339Nano),
			"now",
			time.Now(),
			"now RFC3339Nano",
			time.Now().Format(time.RFC3339Nano),
		)

	}

}

// add BasicAuth header for user if needed
func (s SGDataStore) addAuthIfNeeded(req *http.Request) {
	if !s.UserCreds.Empty() {
		req.SetBasicAuth(s.UserCreds.Username, s.UserCreds.Password)
		req.Header.Set("X-sgload-username", s.UserCreds.Username)
	}
}

func (s SGDataStore) pushTimingStat(key string, delta time.Duration) {
	if s.StatsdClient == nil {
		return
	}
	s.StatsdClient.Timing(
		statsdSampleRate,
		key,
		delta,
	)
}

func (s SGDataStore) pushCounter(key string, n int) {
	if s.StatsdClient == nil {
		return
	}
	s.StatsdClient.Counter(
		statsdSampleRate,
		key,
		n,
	)
}

func splitHostPortWrapper(host string) (string, string, error) {
	if !strings.Contains(host, ":") {
		return host, "", nil
	}

	return net.SplitHostPort(host)
}

func addEndpointToUrl(urlStr, endpoint string) (string, error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return "", err
	}
	u.Path = path.Join(u.Path, endpoint)
	return u.String(), nil
}

func addTrailingSlash(urlStr string) string {
	return fmt.Sprintf("%v/", urlStr)
}

func timeDeltaPerDocument(numDocs int, timeDeltaAllDocs time.Duration) time.Duration {
	if numDocs == 0 {
		return timeDeltaAllDocs
	}
	return time.Duration(int64(timeDeltaAllDocs) / int64(numDocs))
}

func getHttpClient() *http.Client {
	// return http.DefaultClient
	return sgClient
}

func customDefaultTransportWithConnPool(numConnections int) *http.Transport {
	// Customize the Transport to have larger connection pool
	defaultRoundTripper := http.DefaultTransport
	defaultTransportPointer, ok := defaultRoundTripper.(*http.Transport)
	if !ok {
		panic(fmt.Sprintf("defaultRoundTripper not an *http.Transport"))
	}
	defaultTransport := *defaultTransportPointer // dereference it to get a copy of the struct that the pointer points to
	defaultTransport.MaxIdleConns = numConnections
	defaultTransport.MaxIdleConnsPerHost = numConnections
	return &defaultTransport

}
<|endoftext|>


<commit_before>package phonenumber

// Source: exercism/problem-specifications
// Commit: 39cba0d phone-number: Remove test using malformed input. (#772)
// Problem Specifications Version: 1.2.0

// Cleanup user-entered phone numbers
var numberTests = []struct {
	description string
	input       string
	expectErr   bool
	number      string
	areaCode    string
	formatted   string
}{
	{
		description: "cleans the number",
		input:       "(223) 456-7890",
		number:      "2234567890",
		areaCode:    "223",
		formatted:   "(223) 456-7890",
	},
	{
		description: "cleans numbers with dots",
		input:       "223.456.7890",
		number:      "2234567890",
		areaCode:    "223",
		formatted:   "(223) 456-7890",
	},
	{
		description: "cleans numbers with multiple spaces",
		input:       "223 456   7890   ",
		number:      "2234567890",
		areaCode:    "223",
		formatted:   "(223) 456-7890",
	},
	{
		description: "invalid when 9 digits",
		input:       "123456789",
		expectErr:   true,
	},
	{
		description: "invalid when 11 digits does not start with a 1",
		input:       "22234567890",
		expectErr:   true,
	},
	{
		description: "valid when 11 digits and starting with 1",
		input:       "12234567890",
		number:      "2234567890",
		areaCode:    "223",
		formatted:   "(223) 456-7890",
	},
	{
		description: "valid when 11 digits and starting with 1 even with punctuation",
		input:       "+1 (223) 456-7890",
		number:      "2234567890",
		areaCode:    "223",
		formatted:   "(223) 456-7890",
	},
	{
		description: "invalid when more than 11 digits",
		input:       "321234567890",
		expectErr:   true,
	},
	{
		description: "invalid with letters",
		input:       "123-abc-7890",
		expectErr:   true,
	},
	{
		description: "invalid with punctuations",
		input:       "123-@:!-7890",
		expectErr:   true,
	},
	{
		description: "invalid if area code does not start with 2-9",
		input:       "(123) 456-7890",
		expectErr:   true,
	},
	{
		description: "invalid if exchange code does not start with 2-9",
		input:       "(223) 056-7890",
		expectErr:   true,
	},
}
<commit_msg>phone-number: new test cases (#1015)<commit_after>package phonenumber

// Source: exercism/problem-specifications
// Commit: 8380763 phone-number: add missing edge cases (#1090)
// Problem Specifications Version: 1.3.0

// Cleanup user-entered phone numbers
var numberTests = []struct {
	description string
	input       string
	expectErr   bool
	number      string
	areaCode    string
	formatted   string
}{
	{
		description: "cleans the number",
		input:       "(223) 456-7890",
		number:      "2234567890",
		areaCode:    "223",
		formatted:   "(223) 456-7890",
	},
	{
		description: "cleans numbers with dots",
		input:       "223.456.7890",
		number:      "2234567890",
		areaCode:    "223",
		formatted:   "(223) 456-7890",
	},
	{
		description: "cleans numbers with multiple spaces",
		input:       "223 456   7890   ",
		number:      "2234567890",
		areaCode:    "223",
		formatted:   "(223) 456-7890",
	},
	{
		description: "invalid when 9 digits",
		input:       "123456789",
		expectErr:   true,
	},
	{
		description: "invalid when 11 digits does not start with a 1",
		input:       "22234567890",
		expectErr:   true,
	},
	{
		description: "valid when 11 digits and starting with 1",
		input:       "12234567890",
		number:      "2234567890",
		areaCode:    "223",
		formatted:   "(223) 456-7890",
	},
	{
		description: "valid when 11 digits and starting with 1 even with punctuation",
		input:       "+1 (223) 456-7890",
		number:      "2234567890",
		areaCode:    "223",
		formatted:   "(223) 456-7890",
	},
	{
		description: "invalid when more than 11 digits",
		input:       "321234567890",
		expectErr:   true,
	},
	{
		description: "invalid with letters",
		input:       "123-abc-7890",
		expectErr:   true,
	},
	{
		description: "invalid with punctuations",
		input:       "123-@:!-7890",
		expectErr:   true,
	},
	{
		description: "invalid if area code starts with 0",
		input:       "(023) 456-7890",
		expectErr:   true,
	},
	{
		description: "invalid if area code starts with 1",
		input:       "(123) 456-7890",
		expectErr:   true,
	},
	{
		description: "invalid if exchange code starts with 0",
		input:       "(223) 056-7890",
		expectErr:   true,
	},
	{
		description: "invalid if exchange code starts with 1",
		input:       "(223) 156-7890",
		expectErr:   true,
	},
}
<|endoftext|>