<commit_before>package engine

import (
	"crypto/tls"
	"io/ioutil"
	"log"
	"net"

	"github.com/imdario/mergo"
	"github.com/influx6/relay"
	"gopkg.in/yaml.v2"
)

// DefaultConfig provides a default configuration for the app
var DefaultConfig = Config{
	Addr:   ":8080",
	UseTLS: false,
	Certs:  TLSConfig{},
}

//TLSConfig provides a base config for tls configuration
type TLSConfig struct {
	conf *tls.Config
	key  string
	cert string
}

// UnmarshalYaml unmarshalls the incoming data for use
func (t *TLSConfig) UnmarshalYaml(unmarshal func(interface{}) error) error {
	// var k, c string

	// if err := unmarshal()
	return nil
}

// Config provides configuration for Afro
type Config struct {
	Addr   string
	UseTLS bool
	Certs  TLSConfig
}

// NewConfig returns a new configuration file
func NewConfig() *Config {
	c := DefaultConfig
	return &c
}

// Load loads the configuration from a yaml file
func (c *Config) Load(file string) error {
	data, err := ioutil.ReadFile(file)

	if err != nil {
		return err
	}

	conf := Config{}
	err = yaml.Unmarshal(data, &conf)

	if err != nil {
		return err
	}

	return mergo.MergeWithOverwrite(c, conf)
}

// UnmarshalYAML unmarshals and sets the configuration options
func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {

	return nil
}

//Engine provides a base luncher for a service
type Engine struct {
	*relay.Routes
	li     net.Listener
	config *Config
}

//NewEngine returns a new app configuration
func NewEngine(c *Config) *Engine {
	return &Engine{
		Routes: relay.NewRoutes(),
		config: c,
	}
}

// Serve serves the app and configuration and loads the routes and serivices settings
func (a *Engine) Serve() {
	var err error
	var li net.Listener

	if a.config.UseTLS && a.config.Certs.conf != nil {
		_, li, err = relay.CreateTLS(a.config.Addr, a.config.Certs.conf, a)
	} else {
		_, li, err = relay.CreateHTTP(a.config.Addr, a)
	}

	if err != nil {
		log.Fatalf("Server failed to start: %+s", err.Error())
		return
	}

	a.li = li
}
<commit_msg>updating engine<commit_after>package engine

import (
	"crypto/tls"
	"io/ioutil"
	"log"
	"net"

	"github.com/imdario/mergo"
	"github.com/influx6/relay"
	"gopkg.in/yaml.v2"
)

// DefaultConfig provides a default configuration for the app
var DefaultConfig = Config{
	Addr:   ":8080",
	UseTLS: false,
}

//TLSConfig provides a base config for tls configuration
type TLSConfig struct {
	Cert *tls.Config
}

type tlsconf struct {
	Key  string
	Cert string
}

// UnmarshalYaml unmarshalls the incoming data for use
func (t *TLSConfig) UnmarshalYaml(unmarshal func(interface{}) error) error {
	to := tlsconf{}

	if err := unmarshal(&to); err != nil {
		return err
	}

	co, err := relay.LoadTLS(to.Cert, to.Key)

	if err != nil {
		return err
	}

	t.Cert = co
	return nil
}

// Config provides configuration for Afro
type Config struct {
	Addr   string
	UseTLS bool
	C      TLSConfig
}

// NewConfig returns a new configuration file
func NewConfig() *Config {
	c := DefaultConfig
	return &c
}

// Load loads the configuration from a yaml file
func (c *Config) Load(file string) error {
	data, err := ioutil.ReadFile(file)

	if err != nil {
		return err
	}

	conf := Config{}
	err = yaml.Unmarshal(data, &conf)

	if err != nil {
		return err
	}

	return mergo.MergeWithOverwrite(c, conf)
}

// UnmarshalYAML unmarshals and sets the configuration options
func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {

	return nil
}

//Engine provides a base luncher for a service
type Engine struct {
	*relay.Routes
	li     net.Listener
	config *Config
}

//NewEngine returns a new app configuration
func NewEngine(c *Config) *Engine {
	return &Engine{
		Routes: relay.NewRoutes(),
		config: c,
	}
}

// Serve serves the app and configuration and loads the routes and serivices settings
func (a *Engine) Serve() error {
	var err error
	var li net.Listener

	if a.config.UseTLS && a.config.C.Cert != nil {
		_, li, err = relay.CreateTLS(a.config.Addr, a.config.C.Cert, a)
	} else {
		_, li, err = relay.CreateHTTP(a.config.Addr, a)
	}

	if err != nil {
		log.Fatalf("Server failed to start: %+s", err.Error())
		return err
	}

	a.li = li

	return nil
}

// Addr returns the address of the app
func (a *Engine) Addr() net.Addr {
	return a.li.Addr()
}

// Close closes and returns an error of the internal listener
func (a *Engine) Close() error {
	return a.li.Close()
}
<|endoftext|>


<commit_before>package cmdutil

import (
	"bytes"
	"io"
	"io/ioutil"
	"os/exec"
	"strings"

	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
)

// IO defines the inputs and outputs for a command.
type IO struct {
	Stdin  io.Reader
	Stdout io.Writer
	Stderr io.Writer
}

// RunIO runs the command with the given IO and arguments.
func RunIO(ioObj IO, args ...string) error {
	return RunIODirPath(ioObj, "", args...)
}

// RunStdin runs the command with the given stdin and arguments.
func RunStdin(stdin io.Reader, args ...string) error {
	return RunIO(IO{Stdin: stdin}, args...)
}

// RunIODirPath runs the command with the given IO and arguments in the given directory specified by dirPath.
func RunIODirPath(ioObj IO, dirPath string, args ...string) error {
	var debugStderr io.ReadWriter = bytes.NewBuffer(nil)
	var stderr io.Writer = debugStderr
	if ioObj.Stderr != nil {
		stderr = io.MultiWriter(debugStderr, ioObj.Stderr)
	}
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Stdin = ioObj.Stdin
	cmd.Stdout = ioObj.Stdout
	cmd.Stderr = stderr
	cmd.Dir = dirPath
	if err := cmd.Run(); err != nil {
		stderrContent, _ := ioutil.ReadAll(debugStderr)
		if len(stderrContent) > 0 {
			return errors.Wrapf(err, "%s\nStderr\nError%s", strings.Join(args, " "), err.Error(), string(stderrContent))
		}
		return errors.Wrapf(err, "%s", strings.Join(args, " "))
	}
	return nil
}
<commit_msg>Fix an error formatter I messed up.<commit_after>package cmdutil

import (
	"bytes"
	"io"
	"io/ioutil"
	"os/exec"
	"strings"

	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
)

// IO defines the inputs and outputs for a command.
type IO struct {
	Stdin  io.Reader
	Stdout io.Writer
	Stderr io.Writer
}

// RunIO runs the command with the given IO and arguments.
func RunIO(ioObj IO, args ...string) error {
	return RunIODirPath(ioObj, "", args...)
}

// RunStdin runs the command with the given stdin and arguments.
func RunStdin(stdin io.Reader, args ...string) error {
	return RunIO(IO{Stdin: stdin}, args...)
}

// RunIODirPath runs the command with the given IO and arguments in the given directory specified by dirPath.
func RunIODirPath(ioObj IO, dirPath string, args ...string) error {
	var debugStderr io.ReadWriter = bytes.NewBuffer(nil)
	var stderr io.Writer = debugStderr
	if ioObj.Stderr != nil {
		stderr = io.MultiWriter(debugStderr, ioObj.Stderr)
	}
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Stdin = ioObj.Stdin
	cmd.Stdout = ioObj.Stdout
	cmd.Stderr = stderr
	cmd.Dir = dirPath
	if err := cmd.Run(); err != nil {
		stderrContent, _ := ioutil.ReadAll(debugStderr)
		if len(stderrContent) > 0 {
			return errors.Wrapf(err, "%s\nStderr: %s\nError", strings.Join(args, " "), string(stderrContent))
		}
		return errors.Wrapf(err, "%s", strings.Join(args, " "))
	}
	return nil
}
<|endoftext|>


<commit_before>package errors

import (
	"errors"
	"fmt"
	"runtime/debug"
)

func New(text string) error {
	return errors.New(text)
}

type exError struct {
	Base      error
	callstack string
}

func (self exError) Error() string {
	return self.Base.Error() + "\n" + self.callstack
}

func (self exError) Reason() error {
	return self.Base
}

func Ex(s string) error {
	return exError{errors.New(s), string(debug.Stack())}
}

func AsEx(err error) error {
	if err == nil {
		return nil
	}
	if _, ok := err.(exError); ok {
		return err
	}
	return exError{err, string(debug.Stack())}
}

type errorList struct {
	errs []error
}

func (self *errorList) Add(err error) *errorList {
	if err == nil {
		return self
	}

	if v, ok := err.(*errorList); ok {
		return self.AddAll(v.errs)
	}

	if self.errs == nil {
		self.errs = []error{err}
	} else {
		self.errs = append(self.errs, err)
	}
	return self
}

func (self *errorList) AddAll(errs []error) *errorList {
	if errs == nil {
		return self
	}

	for _, e := range errs {
		self.Add(e)
	}
	return self
}

func (self *errorList) Err() error {
	if self.errs == nil {
		return nil
	} else {
		l := len(self.errs)
		if l == 0 {
			return nil
		}
		if l == 1 {
			return self.errs[0]
		}
		return self
	}
}

func (self *errorList) Error() string {
	return fmt.Sprintf("%v", self.errs)
}

func List() *errorList {
	return &errorList{}
}
<commit_msg>added documentation for errors library<commit_after>// Package errors provides helper functions and structures for more natural error handling in some situations.
package errors

import (
	"errors"
	"fmt"
	"runtime/debug"
)

// Convenience funciton to call errors.New() from the standart library.
func New(text string) error {
	return errors.New(text)
}

// An exception: an error with the stack trace, similar to the exceptions in other languages like Java.
type ExError struct {
	Base      error
	callstack string
}

// Implementation of an error interface for exception.
func (self ExError) Error() string {
	return self.Base.Error() + "\n" + self.callstack
}

// Get the underlying error from the exception.
func (self ExError) Reason() error {
	return self.Base
}

// A function to create an exception
func Ex(s string) ExError {
	return ExError{errors.New(s), string(debug.Stack())}
}

// Convert an error to an exception.
// This function doesn't do anythins with nil errors and errors which are already exceptions.
func AsEx(err error) error {
	if err == nil {
		return nil
	}
	if _, ok := err.(ExError); ok {
		return err
	}
	return ExError{err, string(debug.Stack())}
}

/*
An error which is a list of errors.
It's intended to be used locally in a function to aggregate errors from different calls.
Here is a typical use case:

		type aggregator struct {
			val1 io.Closer
			val2 io.Closer
			vals []io.Closer
		}

		func (self *aggregator) CLose() error {
			errs := errors.List().
				Add(self.val1.Close()).
				Add(self.val2.Close())
			for _, v := range self.vals {
				errs.Add(v.Close())
			}
			return errs.Err()
		}
*/
type ErrorList struct {
	errs []error
}

// Add an error to the list.
// This function ignores nil errors
// and inlines other error lists so that you don't have error lists of error lists.
func (self *ErrorList) Add(err error) *ErrorList {
	if err == nil {
		return self
	}

	if v, ok := err.(*ErrorList); ok {
		return self.AddAll(v.errs)
	}

	if self.errs == nil {
		self.errs = []error{err}
	} else {
		self.errs = append(self.errs, err)
	}
	return self
}

// Add all errors from the array to the error list.
// Nil errors and other error lists are added as in Add.
func (self *ErrorList) AddAll(errs []error) *ErrorList {
	if errs == nil {
		return self
	}

	for _, e := range errs {
		self.Add(e)
	}
	return self
}

// Simplify the error list.
// This function gives nil for empty error list
// and a single error for the error list of length one.
func (self *ErrorList) Err() error {
	if self.errs == nil {
		return nil
	} else {
		l := len(self.errs)
		if l == 0 {
			return nil
		}
		if l == 1 {
			return self.errs[0]
		}
		return self
	}
}

// Implementation of an error interface for error list.
func (self *ErrorList) Error() string {
	return fmt.Sprintf("%v", self.Err())
}

// Create an empty error list.
func List() *ErrorList {
	return &ErrorList{}
}
<|endoftext|>


<commit_before>package sudoku

import (
	"encoding/csv"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

type difficultySignals map[string]float64

//A difficulty signal generator can return more than one difficutly signal, so it doesn't just return float64
//Each signal generator should always return a map with the SAME keys--so if you've called it once you know what the
//next calls will have as keys.
type difficultySignalGenerator func(directions SolveDirections) difficultySignals

const _DIFFICULTY_WEIGHT_FILENAME = "difficulties.csv"

var difficultySignalGenerators []difficultySignalGenerator
var difficultySignalWeights map[string]float64

func init() {
	difficultySignalGenerators = []difficultySignalGenerator{
		signalTechnique,
		signalNumberOfSteps,
		signalTechniquePercentage,
		signalPercentageFilledSteps,
		signalNumberUnfilled,
		signalStepsUntilNonFill,
	}

	//TODO: set reasonable DifficultySignalWeights here after we have training data we feel confident in.
	worked := false
	difficultyFile := _DIFFICULTY_WEIGHT_FILENAME

	//For now, just search upwards until we find a difficulty CSV.
	for !worked {
		worked = loadDifficultyWeights(difficultyFile)
		if worked {
			break
		}
		difficultyFile = "../" + difficultyFile
		//We're just making an ever-longer ../../../ ... FILENAME, but if we absolutized it now, we couldn't easily continue
		//prepending ../ . So just test the absFile, but still operate on difficultyFile.
		absFile, _ := filepath.Abs(difficultyFile)
		if absFile == "/"+_DIFFICULTY_WEIGHT_FILENAME {
			//We're already at the top.
			log.Println("Couldn't find a difficulty weights file.")
			break
		}
	}
}

func loadDifficultyWeights(fileName string) bool {

	//TODO: test that this loading works.

	inputFile, err := os.Open(fileName)
	if err != nil {
		//This error will be common because we'll be calling into this repeatedly in init with filenames that we don't know are valid.
		return false
	}
	defer inputFile.Close()
	csvIn := csv.NewReader(inputFile)
	records, csvErr := csvIn.ReadAll()
	if csvErr != nil {
		log.Println("The provided CSV could not be parsed.")
		return false
	}

	//Load up the weights into a map.
	difficultySignalWeights = make(map[string]float64)
	for i, record := range records {
		if len(record) != 2 {
			log.Fatalln("Record in weights csv wasn't right size: ", i, record)
		}
		theFloat, err := strconv.ParseFloat(record[1], 64)
		if err != nil {
			log.Fatalln("Record in weights had an invalid float: ", i, record, err)
		}
		difficultySignalWeights[record[0]] = theFloat
	}

	return true
}

func (self SolveDirections) Stats() []string {
	//TODO: test this.
	techniqueCount := make(map[string]int)
	var lastStep *SolveStep
	dissimilarityAccum := 0.0
	for _, step := range self {
		if lastStep != nil {
			dissimilarityAccum += step.TargetCells.chainDissimilarity(lastStep.TargetCells)
		}
		techniqueCount[step.Technique.Name()] += 1
		lastStep = step
	}
	dissimilarityAccum /= float64(len(self))

	var result []string

	//TODO: use a standard divider across the codebase
	divider := "-------------------------"

	result = append(result, divider)
	//TODO: we shouldn't even include this... it's not meaningful to report the difficulty of a single solve.
	result = append(result, fmt.Sprintf("Difficulty : %f", self.signals().Difficulty()))
	result = append(result, divider)
	result = append(result, fmt.Sprintf("Step count: %d", len(self)))
	result = append(result, divider)
	result = append(result, fmt.Sprintf("Avg Dissimilarity: %f", dissimilarityAccum))
	result = append(result, divider)

	//We want a stable ordering for technique counts.
	for _, technique := range AllTechniques {
		//TODO: pad the technique name with enough spaces so the colon lines up.
		result = append(result, fmt.Sprintf("%s : %d", technique.Name(), techniqueCount[technique.Name()]))
	}

	result = append(result, divider)

	return result
}

func (self SolveDirections) Description() []string {

	if len(self) == 0 {
		return []string{""}
	}

	descriptions := make([]string, len(self))

	for i, step := range self {
		intro := ""
		switch i {
		case 0:
			intro = "First, "
		case len(self) - 1:
			intro = "Finally, "
		default:
			//TODO: switch between "then" and "next" randomly.
			intro = "Next, "
		}
		descriptions[i] = intro + strings.ToLower(step.Description())

	}
	return descriptions
}

func (self SolveDirections) Walkthrough(grid *Grid) string {

	//TODO: test this.

	if len(self) == 0 {
		return "The puzzle could not be solved with any of the techniques we're aware of."
	}

	clone := grid.Copy()
	defer clone.Done()

	DIVIDER := "\n\n--------------------------------------------\n\n"

	intro := fmt.Sprintf("This will take %d steps to solve.", len(self))

	intro += "\nWhen you start, your grid looks like this:\n"

	intro += clone.Diagram()

	intro += "\n"

	intro += DIVIDER

	descriptions := self.Description()

	results := make([]string, len(self))

	for i, description := range descriptions {

		result := description + "\n"
		result += "After doing that, your grid will look like: \n\n"

		self[i].Apply(clone)

		result += clone.Diagram()

		results[i] = result
	}

	return intro + strings.Join(results, DIVIDER) + DIVIDER + "Now the puzzle is solved."
}

//Because of the contract of a DifficultySignalGenerator (that it always returns the same keys), as long as DifficultySignalGenerators stays constant
//it's reasonable for callers to assume that one call to Signals() will return all of the string keys you'll see any time you call Signals()
func (self SolveDirections) signals() difficultySignals {
	result := difficultySignals{}
	for _, generator := range difficultySignalGenerators {
		result.Add(generator(self))
	}
	return result
}

//This will overwrite colliding values
//TODO: this is confusingly named
func (self difficultySignals) Add(other difficultySignals) {
	for key, val := range other {
		self[key] = val
	}
}

//For keys in both, will sum them together.
//TODO: this is confusingly named (compared to Add)
// Do we really need both Sum and Add?
func (self difficultySignals) Sum(other difficultySignals) {
	for key, val := range other {
		self[key] += val
	}
}

func (self difficultySignals) Difficulty() float64 {
	accum := 0.0

	if constant, ok := difficultySignalWeights["Constant"]; ok {
		accum = constant
	} else {
		log.Println("Didn't have the constant term loaded.")
	}

	for signal, val := range self {
		//We can discard the OK because 0 is a reasonable thing to do with weights we aren't aware of.
		weight, _ := difficultySignalWeights[signal]

		accum += val * weight
	}

	if accum < 0.0 {
		log.Println("Accumuldated difficulty snapped to 0.0:", accum)
		accum = 0.0
	}

	if accum > 1.0 {
		log.Println("Accumulated difficulty snapped to 1.0:", accum)
		accum = 1.0
	}

	return accum
}

//Rest of file is different Signals

//This technique returns a count of how many each type of technique is seen.
//Different techniques are different "difficulties" so seeing more of a hard technique will
//Lead to a higher overall difficulty.
func signalTechnique(directions SolveDirections) difficultySignals {
	//Our contract is to always return every signal name, even if it's 0.0.
	result := difficultySignals{}
	for _, technique := range AllTechniques {
		result[technique.Name()+" Count"] = 0.0
	}
	for _, step := range directions {
		result[step.Technique.Name()+" Count"]++
	}
	return result
}

//This signal is just number of steps. More steps is PROBABLY a harder puzzle.
func signalNumberOfSteps(directions SolveDirections) difficultySignals {
	return difficultySignals{
		"Number of Steps": float64(len(directions)),
	}
}

//This signal is like signalTechnique, except it returns the count divided by the TOTAL number of steps.
func signalTechniquePercentage(directions SolveDirections) difficultySignals {
	//Our contract is to always return every signal name, even if it's 0.0.
	result := difficultySignals{}
	for _, technique := range AllTechniques {
		result[technique.Name()+" Percentage"] = 0.0
	}

	count := len(directions)

	if count == 0 {
		return result
	}

	for _, step := range directions {
		result[step.Technique.Name()+" Percentage"]++
	}

	//Now normalize all of them
	for name, _ := range result {
		result[name] /= float64(count)
	}

	return result
}

//This signal is how many steps are filled out of all steps. Presumably harder puzzles will have more non-fill steps.
func signalPercentageFilledSteps(directions SolveDirections) difficultySignals {
	numerator := 0.0
	denominator := float64(len(directions))

	for _, step := range directions {
		if step.Technique.IsFill() {
			numerator += 1.0
		}
	}

	return difficultySignals{
		"Percentage Fill Steps": numerator / denominator,
	}
}

//This signal is how many cells are unfilled at the beginning. Presumably harder puzzles will have fewer cells filled (although obviously this isn't necessarily true)
func signalNumberUnfilled(directions SolveDirections) difficultySignals {

	//We don't have access to the underlying grid, so we'll just count how many fill steps (since each can only add one number, and no numbers are ever unfilled)

	count := 0.0
	for _, step := range directions {
		if step.Technique.IsFill() {
			count++
		}
	}

	return difficultySignals{
		"Number Unfilled Cells": count,
	}
}

//This signal is how many steps into the solve directions before you encounter your first non-fill step. Non-fill steps are harder, so this signal
//captures how easy the start of the puzzle is.
func signalStepsUntilNonFill(directions SolveDirections) difficultySignals {
	count := 0.0
	for _, step := range directions {
		if !step.Technique.IsFill() {
			break
		}
		count++
	}

	return difficultySignals{
		"Steps Until Nonfill": count,
	}

}
<commit_msg>GoDoc for all of SolveDirections' methods.<commit_after>package sudoku

import (
	"encoding/csv"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

type difficultySignals map[string]float64

//A difficulty signal generator can return more than one difficutly signal, so it doesn't just return float64
//Each signal generator should always return a map with the SAME keys--so if you've called it once you know what the
//next calls will have as keys.
type difficultySignalGenerator func(directions SolveDirections) difficultySignals

const _DIFFICULTY_WEIGHT_FILENAME = "difficulties.csv"

var difficultySignalGenerators []difficultySignalGenerator
var difficultySignalWeights map[string]float64

func init() {
	difficultySignalGenerators = []difficultySignalGenerator{
		signalTechnique,
		signalNumberOfSteps,
		signalTechniquePercentage,
		signalPercentageFilledSteps,
		signalNumberUnfilled,
		signalStepsUntilNonFill,
	}

	//TODO: set reasonable DifficultySignalWeights here after we have training data we feel confident in.
	worked := false
	difficultyFile := _DIFFICULTY_WEIGHT_FILENAME

	//For now, just search upwards until we find a difficulty CSV.
	for !worked {
		worked = loadDifficultyWeights(difficultyFile)
		if worked {
			break
		}
		difficultyFile = "../" + difficultyFile
		//We're just making an ever-longer ../../../ ... FILENAME, but if we absolutized it now, we couldn't easily continue
		//prepending ../ . So just test the absFile, but still operate on difficultyFile.
		absFile, _ := filepath.Abs(difficultyFile)
		if absFile == "/"+_DIFFICULTY_WEIGHT_FILENAME {
			//We're already at the top.
			log.Println("Couldn't find a difficulty weights file.")
			break
		}
	}
}

func loadDifficultyWeights(fileName string) bool {

	//TODO: test that this loading works.

	inputFile, err := os.Open(fileName)
	if err != nil {
		//This error will be common because we'll be calling into this repeatedly in init with filenames that we don't know are valid.
		return false
	}
	defer inputFile.Close()
	csvIn := csv.NewReader(inputFile)
	records, csvErr := csvIn.ReadAll()
	if csvErr != nil {
		log.Println("The provided CSV could not be parsed.")
		return false
	}

	//Load up the weights into a map.
	difficultySignalWeights = make(map[string]float64)
	for i, record := range records {
		if len(record) != 2 {
			log.Fatalln("Record in weights csv wasn't right size: ", i, record)
		}
		theFloat, err := strconv.ParseFloat(record[1], 64)
		if err != nil {
			log.Fatalln("Record in weights had an invalid float: ", i, record, err)
		}
		difficultySignalWeights[record[0]] = theFloat
	}

	return true
}

//Stats returns a printout of interesting statistics about the SolveDirections, including number of steps,
//difficulty (based on this solve description alone), how unrelated the cells in subsequent steps are,
//and the values of all of the signals used to generate the difficulty.
func (self SolveDirections) Stats() []string {
	//TODO: test this.
	techniqueCount := make(map[string]int)
	var lastStep *SolveStep
	dissimilarityAccum := 0.0
	for _, step := range self {
		if lastStep != nil {
			dissimilarityAccum += step.TargetCells.chainDissimilarity(lastStep.TargetCells)
		}
		techniqueCount[step.Technique.Name()] += 1
		lastStep = step
	}
	dissimilarityAccum /= float64(len(self))

	var result []string

	//TODO: use a standard divider across the codebase
	divider := "-------------------------"

	result = append(result, divider)
	//TODO: we shouldn't even include this... it's not meaningful to report the difficulty of a single solve.
	result = append(result, fmt.Sprintf("Difficulty : %f", self.signals().Difficulty()))
	result = append(result, divider)
	result = append(result, fmt.Sprintf("Step count: %d", len(self)))
	result = append(result, divider)
	result = append(result, fmt.Sprintf("Avg Dissimilarity: %f", dissimilarityAccum))
	result = append(result, divider)

	//We want a stable ordering for technique counts.
	for _, technique := range AllTechniques {
		//TODO: pad the technique name with enough spaces so the colon lines up.
		result = append(result, fmt.Sprintf("%s : %d", technique.Name(), techniqueCount[technique.Name()]))
	}

	result = append(result, divider)

	return result
}

//Description returns a comprehensive prose description of the SolveDirections, including reasoning for each step, that
//if followed would lead to the grid being solved. Unlike Walkthrough, Description() does not include diagrams
//for each step.
func (self SolveDirections) Description() []string {

	if len(self) == 0 {
		return []string{""}
	}

	descriptions := make([]string, len(self))

	for i, step := range self {
		intro := ""
		switch i {
		case 0:
			intro = "First, "
		case len(self) - 1:
			intro = "Finally, "
		default:
			//TODO: switch between "then" and "next" randomly.
			intro = "Next, "
		}
		descriptions[i] = intro + strings.ToLower(step.Description())

	}
	return descriptions
}

//Walkthrough prints an exhaustive set of human-readable directions that includes diagrams at each
//step to make it easier to follow.
func (self SolveDirections) Walkthrough(grid *Grid) string {

	//TODO: test this.

	if len(self) == 0 {
		return "The puzzle could not be solved with any of the techniques we're aware of."
	}

	clone := grid.Copy()
	defer clone.Done()

	DIVIDER := "\n\n--------------------------------------------\n\n"

	intro := fmt.Sprintf("This will take %d steps to solve.", len(self))

	intro += "\nWhen you start, your grid looks like this:\n"

	intro += clone.Diagram()

	intro += "\n"

	intro += DIVIDER

	descriptions := self.Description()

	results := make([]string, len(self))

	for i, description := range descriptions {

		result := description + "\n"
		result += "After doing that, your grid will look like: \n\n"

		self[i].Apply(clone)

		result += clone.Diagram()

		results[i] = result
	}

	return intro + strings.Join(results, DIVIDER) + DIVIDER + "Now the puzzle is solved."
}

//Because of the contract of a DifficultySignalGenerator (that it always returns the same keys), as long as DifficultySignalGenerators stays constant
//it's reasonable for callers to assume that one call to Signals() will return all of the string keys you'll see any time you call Signals()
func (self SolveDirections) signals() difficultySignals {
	result := difficultySignals{}
	for _, generator := range difficultySignalGenerators {
		result.Add(generator(self))
	}
	return result
}

//This will overwrite colliding values
//TODO: this is confusingly named
func (self difficultySignals) Add(other difficultySignals) {
	for key, val := range other {
		self[key] = val
	}
}

//For keys in both, will sum them together.
//TODO: this is confusingly named (compared to Add)
// Do we really need both Sum and Add?
func (self difficultySignals) Sum(other difficultySignals) {
	for key, val := range other {
		self[key] += val
	}
}

func (self difficultySignals) Difficulty() float64 {
	accum := 0.0

	if constant, ok := difficultySignalWeights["Constant"]; ok {
		accum = constant
	} else {
		log.Println("Didn't have the constant term loaded.")
	}

	for signal, val := range self {
		//We can discard the OK because 0 is a reasonable thing to do with weights we aren't aware of.
		weight, _ := difficultySignalWeights[signal]

		accum += val * weight
	}

	if accum < 0.0 {
		log.Println("Accumuldated difficulty snapped to 0.0:", accum)
		accum = 0.0
	}

	if accum > 1.0 {
		log.Println("Accumulated difficulty snapped to 1.0:", accum)
		accum = 1.0
	}

	return accum
}

//Rest of file is different Signals

//This technique returns a count of how many each type of technique is seen.
//Different techniques are different "difficulties" so seeing more of a hard technique will
//Lead to a higher overall difficulty.
func signalTechnique(directions SolveDirections) difficultySignals {
	//Our contract is to always return every signal name, even if it's 0.0.
	result := difficultySignals{}
	for _, technique := range AllTechniques {
		result[technique.Name()+" Count"] = 0.0
	}
	for _, step := range directions {
		result[step.Technique.Name()+" Count"]++
	}
	return result
}

//This signal is just number of steps. More steps is PROBABLY a harder puzzle.
func signalNumberOfSteps(directions SolveDirections) difficultySignals {
	return difficultySignals{
		"Number of Steps": float64(len(directions)),
	}
}

//This signal is like signalTechnique, except it returns the count divided by the TOTAL number of steps.
func signalTechniquePercentage(directions SolveDirections) difficultySignals {
	//Our contract is to always return every signal name, even if it's 0.0.
	result := difficultySignals{}
	for _, technique := range AllTechniques {
		result[technique.Name()+" Percentage"] = 0.0
	}

	count := len(directions)

	if count == 0 {
		return result
	}

	for _, step := range directions {
		result[step.Technique.Name()+" Percentage"]++
	}

	//Now normalize all of them
	for name, _ := range result {
		result[name] /= float64(count)
	}

	return result
}

//This signal is how many steps are filled out of all steps. Presumably harder puzzles will have more non-fill steps.
func signalPercentageFilledSteps(directions SolveDirections) difficultySignals {
	numerator := 0.0
	denominator := float64(len(directions))

	for _, step := range directions {
		if step.Technique.IsFill() {
			numerator += 1.0
		}
	}

	return difficultySignals{
		"Percentage Fill Steps": numerator / denominator,
	}
}

//This signal is how many cells are unfilled at the beginning. Presumably harder puzzles will have fewer cells filled (although obviously this isn't necessarily true)
func signalNumberUnfilled(directions SolveDirections) difficultySignals {

	//We don't have access to the underlying grid, so we'll just count how many fill steps (since each can only add one number, and no numbers are ever unfilled)

	count := 0.0
	for _, step := range directions {
		if step.Technique.IsFill() {
			count++
		}
	}

	return difficultySignals{
		"Number Unfilled Cells": count,
	}
}

//This signal is how many steps into the solve directions before you encounter your first non-fill step. Non-fill steps are harder, so this signal
//captures how easy the start of the puzzle is.
func signalStepsUntilNonFill(directions SolveDirections) difficultySignals {
	count := 0.0
	for _, step := range directions {
		if !step.Technique.IsFill() {
			break
		}
		count++
	}

	return difficultySignals{
		"Steps Until Nonfill": count,
	}

}
<|endoftext|>


<commit_before>// Package htcmap implements a basic hot to color ramp functions.
// With givin value, value minimum and value maximum (given range) returns
// RGB color from Blue for minimum value to Red for maximum value.
// Intermediate colors such as green, yellow and more might also be returned for
// values between minimum and maximum (within range).
package htcmap

import "fmt"

type Range struct {
	VMin, VMax float64
}

func (r *Range) AsFloat(val float64) (float64, float64, float64) {
	return AsFloat(val, r.VMin, r.VMax)
}

func (r *Range) AsUInt8(val float64) (uint8, uint8, uint8) {
	return AsUInt8(val, r.VMin, r.VMax)
}

func (r *Range) AsStr(val float64) string {
	return AsStr(val, r.VMin, r.VMax)
}

// AsFloat returns red, green, blue values each as float64 in range [0..1].
func AsFloat(val, vmin, vmax float64) (float64, float64, float64) {
	return htc(val, vmin, vmax)
}

// AsUInt8 returns red, green, blue values each as uint8 in range [0..255].
func AsUInt8(val, vmin, vmax float64) (uint8, uint8, uint8) {
	const m = 0xFF
	r, g, b := AsFloat(val, vmin, vmax)
	return uint8(r * m), uint8(g * m), uint8(b * m)
}

// AsStr returns red, green, blue values represented as six hex digits string
// which begins with hash (#). It looks like a HTML RGB color #00FF00.
func AsStr(val, vmin, vmax float64) string {
	r, g, b := AsUInt8(val, vmin, vmax)
	return fmt.Sprintf("#%02X%02X%02X", r, g, b)
}

// Stop points can be modified/adjusted for better correlation.
func htc(val, vmin, vmax float64) (float64, float64, float64) {

	var (
		vrange  float64 = vmax - vmin
		r, g, b float64 = 1, 1, 1
	)

	if val < vmin {
		val = vmin
	}
	if val > vmax {
		val = vmax
	}

	if val < (vmin + 0.25*vrange) {
		r = 0
		g = 4 * (val - vmin) / vrange
	} else if val < (vmin + 0.5*vrange) {
		r = 0
		b = 1 + 4*(vmin+0.25*vrange)/vrange
	} else if val < (vmin+0.75*vrange)/vrange {
		b = 0
		r = 4 * (val - vmin - 0.5*vrange) / vrange
	} else {
		b = 0
		g = 1 + 4*(vmin+0.75*vrange-val)/vrange
	}
	return r, g, b
}
<commit_msg>Added docstring for methods.<commit_after>// Package htcmap implements a basic hot to color ramp functions.
// With givin value, value minimum and value maximum (given range) returns
// RGB color from Blue for minimum value to Red for maximum value.
// Intermediate colors such as green, yellow and more might also be returned for
// values between minimum and maximum (within range).
package htcmap

import "fmt"

// Range hold value minimum and value maximum (range) and provides the similar
// functions as a module.
type Range struct {
	VMin, VMax float64
}

// AsFloat returns red, green, blue values each as float64 in range [0..1].
func (r *Range) AsFloat(val float64) (float64, float64, float64) {
	return AsFloat(val, r.VMin, r.VMax)
}

// AsUInt8 returns red, green, blue values each as uint8 in range [0..255].
func (r *Range) AsUInt8(val float64) (uint8, uint8, uint8) {
	return AsUInt8(val, r.VMin, r.VMax)
}

// AsStr returns red, green, blue values represented as six hex digits string
// which begins with hash (#). It looks like a HTML RGB color #00FF00.
func (r *Range) AsStr(val float64) string {
	return AsStr(val, r.VMin, r.VMax)
}

// AsFloat returns red, green, blue values each as float64 in range [0..1].
func AsFloat(val, vmin, vmax float64) (float64, float64, float64) {
	return htc(val, vmin, vmax)
}

// AsUInt8 returns red, green, blue values each as uint8 in range [0..255].
func AsUInt8(val, vmin, vmax float64) (uint8, uint8, uint8) {
	const m = 0xFF
	r, g, b := AsFloat(val, vmin, vmax)
	return uint8(r * m), uint8(g * m), uint8(b * m)
}

// AsStr returns red, green, blue values represented as six hex digits string
// which begins with hash (#). It looks like a HTML RGB color #00FF00.
func AsStr(val, vmin, vmax float64) string {
	r, g, b := AsUInt8(val, vmin, vmax)
	return fmt.Sprintf("#%02X%02X%02X", r, g, b)
}

// Stop points can be modified/adjusted for better correlation.
func htc(val, vmin, vmax float64) (float64, float64, float64) {

	var (
		vrange  float64 = vmax - vmin
		r, g, b float64 = 1, 1, 1
	)

	if val < vmin {
		val = vmin
	}
	if val > vmax {
		val = vmax
	}

	if val < (vmin + 0.25*vrange) {
		r = 0
		g = 4 * (val - vmin) / vrange
	} else if val < (vmin + 0.5*vrange) {
		r = 0
		b = 1 + 4*(vmin+0.25*vrange)/vrange
	} else if val < (vmin+0.75*vrange)/vrange {
		b = 0
		r = 4 * (val - vmin - 0.5*vrange) / vrange
	} else {
		b = 0
		g = 1 + 4*(vmin+0.75*vrange-val)/vrange
	}
	return r, g, b
}
<|endoftext|>


<commit_before>package cassandra

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"sync"
	"sync/atomic"
	"time"

	"github.com/gocql/gocql"
	"github.com/uluyol/fabbench/db"
	"github.com/uluyol/fabbench/recorders"
)

type conf struct {
	ClientRetries      *int   `json:"clientRetries",omitempty`
	NumRetries         *int   `json:"numRetries,omitempty"`
	NumConns           *int   `json:"numConns,omitempty"`
	ReadConsistency    string `json:"readConsistency"`
	WriteConsistency   string `json:"writeConsistency"`
	Keyspace           string `json:"keyspace"`
	Table              string `json:"table"`
	ReplicationFactor  *int   `json:"replicationFactor,omitempty"`
	KeyCaching         string `json:"keyCaching"`
	CompactionStrategy string `json:"compactionStrategy"`
	Timeout            string `json:"timeout"`

	TraceData *string `json:"traceData",omitempty`
	TraceRate *int32  `json:"traceRate",omitempty`
}

func newInt(v int) *int { return &v }

func (c *conf) fillDefaults() {
	intFields := []struct {
		val **int
		def int
	}{
		{&c.ClientRetries, 10},
		{&c.NumRetries, 4},
		{&c.ReplicationFactor, 3},
		{&c.NumConns, 4},
	}
	for _, f := range intFields {
		if *f.val == nil {
			*f.val = newInt(f.def)
		}
	}

	strFields := []struct {
		val *string
		def string
	}{
		{&c.ReadConsistency, "ONE"},
		{&c.WriteConsistency, "ONE"},
		{&c.Keyspace, "fabbench"},
		{&c.Table, "udata"},
		{&c.KeyCaching, "ALL"},
		{&c.CompactionStrategy, "LeveledCompactionStrategy"},
		{&c.Timeout, "5s"},
	}
	for _, f := range strFields {
		if *f.val == "" {
			*f.val = f.def
		}
	}
}

func parseConsistency(s string) (c gocql.Consistency, err error) {
	defer func() {
		if e := recover(); e != nil {
			if er, ok := e.(error); ok {
				err = er
				return
			}
			panic(e)
		}
	}()
	return gocql.ParseConsistency(s), nil
}

type client struct {
	cluster          *gocql.ClusterConfig
	readConsistency  gocql.Consistency
	writeConsistency gocql.Consistency
	conf             *conf

	session struct {
		once sync.Once
		s    *gocql.Session
		err  error
	}

	at     *recorders.AsyncTrace
	tracer gocql.Tracer

	getQPool sync.Pool
	putQPool sync.Pool

	opCount int32
}

func (c *client) getSession() (*gocql.Session, error) {
	c.session.once.Do(func() {
		c.session.s, c.session.err = c.cluster.CreateSession()
		c.tracer = gocql.NewTraceWriter(c.session.s, recorders.NewTraceConsumer(c.at.C))
	})

	return c.session.s, c.session.err
}

func (c *client) traceQuery(q *gocql.Query) *gocql.Query {
	if c.tracer != nil {
		if atomic.AddInt32(&c.opCount, 1)%*c.conf.TraceRate == 0 {
			return q.Trace(c.tracer)
		}
	}
	return q
}

func (c *client) getQuery(s *gocql.Session) *gocql.Query {
	t := c.getQPool.Get()
	if t != nil {
		if q, ok := t.(*gocql.Query); ok {
			return q
		}
	}
	q := s.Query("SELECT vval FROM " + c.conf.Table + " WHERE vkey = ?")
	q.Consistency(c.readConsistency)
	return q
}

func (c *client) cacheGetQuery(q *gocql.Query) { c.getQPool.Put(q) }

func (c *client) putQuery(s *gocql.Session) *gocql.Query {
	t := c.putQPool.Get()
	if t != nil {
		if q, ok := t.(*gocql.Query); ok {
			return q
		}
	}
	q := s.Query("INSERT INTO " + c.conf.Table + " (vkey, vval) VALUES (?, ?)")
	q.Consistency(c.writeConsistency)
	return q
}

func (c *client) cachePutQuery(q *gocql.Query) { c.putQPool.Put(q) }

func newClient(hosts []string, cfg *conf) (db.DB, error) {
	timeout, err := time.ParseDuration(cfg.Timeout)
	if err != nil {
		return nil, fmt.Errorf("error parsing timeout as time.Duration: %v")
	}

	readConsistency, err := parseConsistency(cfg.ReadConsistency)
	if err != nil {
		return nil, fmt.Errorf("invalid read consistency: %v")
	}
	writeConsistency, err := parseConsistency(cfg.WriteConsistency)
	if err != nil {
		return nil, fmt.Errorf("invalid write consistency: %v")
	}

	cluster := gocql.NewCluster(hosts...)
	cluster.ProtoVersion = 4
	cluster.RetryPolicy = &gocql.SimpleRetryPolicy{NumRetries: *cfg.NumRetries}
	cluster.Timeout = timeout
	cluster.SocketKeepalive = 30 * time.Second
	cluster.NumConns = *cfg.NumConns

	// Test that we can create a session.
	// We don't know if the Keyspace exists yet,
	// so we can't actually create the session used for gets/puts now.
	// We'll create it when the first request starts executing.
	//
	// For now, just make sure we can connect so we can give sane errors.
	s, err := cluster.CreateSession()
	if err != nil {
		return nil, fmt.Errorf("unable to connect to db: %v", err)
	}
	s.Close()

	cluster.Keyspace = cfg.Keyspace

	var at *recorders.AsyncTrace
	if cfg.TraceRate != nil && cfg.TraceData != nil {
		f, err := os.Create(*cfg.TraceData)
		if err != nil {
			return nil, fmt.Errorf("unable to open trace file: %v", err)
		}
		at = recorders.NewAsyncTrace(f)
		go at.Consume()
	}

	c := &client{
		cluster:          cluster,
		readConsistency:  readConsistency,
		writeConsistency: writeConsistency,
		conf:             cfg,
		at:               at,
	}
	return c, nil
}

func (c *client) Init(ctx context.Context) error {
	ks := c.cluster.Keyspace
	c.cluster.Keyspace = ""

	defer func() {
		c.cluster.Keyspace = ks
	}()

	session, err := c.cluster.CreateSession()
	if err != nil {
		return fmt.Errorf("unable to connect to db to make table: %v", err)
	}
	defer session.Close()

	q := session.Query(fmt.Sprintf(
		"CREATE KEYSPACE %s WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': %d}",
		ks, *c.conf.ReplicationFactor))
	if err := q.WithContext(ctx).Exec(); err != nil {
		return fmt.Errorf("unable to create keyspace %s: %v", ks, err)
	}
	q = session.Query(fmt.Sprintf(
		"CREATE TABLE %s.%s (vkey varchar primary key, vval varchar)",
		ks, c.conf.Table))
	if err := q.WithContext(ctx).Exec(); err != nil {
		return fmt.Errorf("unable to create table %s: %v", c.conf.Table, err)
	}
	return nil
}

func (c *client) Get(ctx context.Context, key string) (string, error) {
	s, err := c.getSession()
	if err != nil {
		return "", fmt.Errorf("unable to connect to db: %v", err)
	}
	q := c.getQuery(s)
	var v string
	for retry := 0; retry < *c.conf.ClientRetries; retry++ {
		err := c.traceQuery(q.Bind(key).WithContext(ctx)).Scan(&v)
		switch err {
		case gocql.ErrNoConnections:
			// retry
		default:
			c.cacheGetQuery(q)
			return v, err
		}
	}
	c.cacheGetQuery(q)
	return "", gocql.ErrNoConnections
}

func (c *client) Put(ctx context.Context, key, val string) error {
	s, err := c.getSession()
	if err != nil {
		return fmt.Errorf("unable to connect to db: %v", err)
	}
	q := c.putQuery(s)

	for retry := 0; retry < *c.conf.ClientRetries; retry++ {
		err := c.traceQuery(q.Bind(key, val).WithContext(ctx)).Exec()
		switch err {
		case gocql.ErrNoConnections:
			// retry
		default:
			c.cachePutQuery(q)
			return err
		}
	}
	c.cachePutQuery(q)
	return gocql.ErrNoConnections
}

func (c *client) Close() error {
	c.at.Close()
	return nil
}

func makeDB(hosts []string, data []byte) (db.DB, error) {
	var cfg conf
	if err := json.Unmarshal(data, &cfg); err != nil {
		return nil, fmt.Errorf("invalid cassandra config: %v", err)
	}
	cfg.fillDefaults()
	return newClient(hosts, &cfg)
}

func init() {
	db.Register("cassandra", makeDB)
}
<commit_msg>db/cassandra: separate traces for reads, writes<commit_after>package cassandra

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"sync"
	"sync/atomic"
	"time"

	"github.com/gocql/gocql"
	"github.com/uluyol/fabbench/db"
	"github.com/uluyol/fabbench/recorders"
)

type conf struct {
	ClientRetries      *int   `json:"clientRetries",omitempty`
	NumRetries         *int   `json:"numRetries,omitempty"`
	NumConns           *int   `json:"numConns,omitempty"`
	ReadConsistency    string `json:"readConsistency"`
	WriteConsistency   string `json:"writeConsistency"`
	Keyspace           string `json:"keyspace"`
	Table              string `json:"table"`
	ReplicationFactor  *int   `json:"replicationFactor,omitempty"`
	KeyCaching         string `json:"keyCaching"`
	CompactionStrategy string `json:"compactionStrategy"`
	Timeout            string `json:"timeout"`

	TraceData *string `json:"traceData",omitempty`
	TraceRate *int32  `json:"traceRate",omitempty`
}

func newInt(v int) *int { return &v }

func (c *conf) fillDefaults() {
	intFields := []struct {
		val **int
		def int
	}{
		{&c.ClientRetries, 10},
		{&c.NumRetries, 4},
		{&c.ReplicationFactor, 3},
		{&c.NumConns, 4},
	}
	for _, f := range intFields {
		if *f.val == nil {
			*f.val = newInt(f.def)
		}
	}

	strFields := []struct {
		val *string
		def string
	}{
		{&c.ReadConsistency, "ONE"},
		{&c.WriteConsistency, "ONE"},
		{&c.Keyspace, "fabbench"},
		{&c.Table, "udata"},
		{&c.KeyCaching, "ALL"},
		{&c.CompactionStrategy, "LeveledCompactionStrategy"},
		{&c.Timeout, "5s"},
	}
	for _, f := range strFields {
		if *f.val == "" {
			*f.val = f.def
		}
	}
}

func parseConsistency(s string) (c gocql.Consistency, err error) {
	defer func() {
		if e := recover(); e != nil {
			if er, ok := e.(error); ok {
				err = er
				return
			}
			panic(e)
		}
	}()
	return gocql.ParseConsistency(s), nil
}

type client struct {
	cluster          *gocql.ClusterConfig
	readConsistency  gocql.Consistency
	writeConsistency gocql.Consistency
	conf             *conf

	session struct {
		once sync.Once
		s    *gocql.Session
		err  error
	}

	rat     *recorders.AsyncTrace
	wat *recorders.AsyncTrace
	rtracer gocql.Tracer
	wtracer gocql.Tracer

	getQPool sync.Pool
	putQPool sync.Pool

	opCount int32
}

func (c *client) getSession() (*gocql.Session, error) {
	c.session.once.Do(func() {
		c.session.s, c.session.err = c.cluster.CreateSession()
		c.rtracer = gocql.NewTraceWriter(c.session.s, recorders.NewTraceConsumer(c.rat.C))
		c.wtracer = gocql.NewTraceWriter(c.session.s, recorders.NewTraceConsumer(c.wat.C))
	})

	return c.session.s, c.session.err
}

func (c *client) traceQuery(q *gocql.Query, read bool) *gocql.Query {
	if c.rtracer != nil && c.wtracer != nil {
		tracer := c.rtracer
		if !read {
			tracer = c.wtracer
		}
		if atomic.AddInt32(&c.opCount, 1)%*c.conf.TraceRate == 0 {
			return q.Trace(tracer)
		}
	}
	return q
}

func (c *client) getQuery(s *gocql.Session) *gocql.Query {
	t := c.getQPool.Get()
	if t != nil {
		if q, ok := t.(*gocql.Query); ok {
			return q
		}
	}
	q := s.Query("SELECT vval FROM " + c.conf.Table + " WHERE vkey = ?")
	q.Consistency(c.readConsistency)
	return q
}

func (c *client) cacheGetQuery(q *gocql.Query) { c.getQPool.Put(q) }

func (c *client) putQuery(s *gocql.Session) *gocql.Query {
	t := c.putQPool.Get()
	if t != nil {
		if q, ok := t.(*gocql.Query); ok {
			return q
		}
	}
	q := s.Query("INSERT INTO " + c.conf.Table + " (vkey, vval) VALUES (?, ?)")
	q.Consistency(c.writeConsistency)
	return q
}

func (c *client) cachePutQuery(q *gocql.Query) { c.putQPool.Put(q) }

func newClient(hosts []string, cfg *conf) (db.DB, error) {
	timeout, err := time.ParseDuration(cfg.Timeout)
	if err != nil {
		return nil, fmt.Errorf("error parsing timeout as time.Duration: %v")
	}

	readConsistency, err := parseConsistency(cfg.ReadConsistency)
	if err != nil {
		return nil, fmt.Errorf("invalid read consistency: %v")
	}
	writeConsistency, err := parseConsistency(cfg.WriteConsistency)
	if err != nil {
		return nil, fmt.Errorf("invalid write consistency: %v")
	}

	cluster := gocql.NewCluster(hosts...)
	cluster.ProtoVersion = 4
	cluster.RetryPolicy = &gocql.SimpleRetryPolicy{NumRetries: *cfg.NumRetries}
	cluster.Timeout = timeout
	cluster.SocketKeepalive = 30 * time.Second
	cluster.NumConns = *cfg.NumConns

	// Test that we can create a session.
	// We don't know if the Keyspace exists yet,
	// so we can't actually create the session used for gets/puts now.
	// We'll create it when the first request starts executing.
	//
	// For now, just make sure we can connect so we can give sane errors.
	s, err := cluster.CreateSession()
	if err != nil {
		return nil, fmt.Errorf("unable to connect to db: %v", err)
	}
	s.Close()

	cluster.Keyspace = cfg.Keyspace

	var rat, wat *recorders.AsyncTrace
	if cfg.TraceRate != nil && cfg.TraceData != nil {
		rf, err := os.Create(*cfg.TraceData + "-ro.gz")
		if err != nil {
			return nil, fmt.Errorf("unable to open ro trace file: %v", err)
		}
		wf, err := os.Create(*cfg.TraceData + "-wo.gz")
		if err != nil {
			return nil, fmt.Errorf("unable to open wo trace file: %v", err)
		}
		rat = recorders.NewAsyncTrace(rf)
		wat = recorders.NewAsyncTrace(wf)
		go rat.Consume()
		go wat.Consume()
	}

	c := &client{
		cluster:          cluster,
		readConsistency:  readConsistency,
		writeConsistency: writeConsistency,
		conf:             cfg,
		rat:               rat,
		wat: wat,
	}
	return c, nil
}

func (c *client) Init(ctx context.Context) error {
	ks := c.cluster.Keyspace
	c.cluster.Keyspace = ""

	defer func() {
		c.cluster.Keyspace = ks
	}()

	session, err := c.cluster.CreateSession()
	if err != nil {
		return fmt.Errorf("unable to connect to db to make table: %v", err)
	}
	defer session.Close()

	q := session.Query(fmt.Sprintf(
		"CREATE KEYSPACE %s WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': %d}",
		ks, *c.conf.ReplicationFactor))
	if err := q.WithContext(ctx).Exec(); err != nil {
		return fmt.Errorf("unable to create keyspace %s: %v", ks, err)
	}
	q = session.Query(fmt.Sprintf(
		"CREATE TABLE %s.%s (vkey varchar primary key, vval varchar)",
		ks, c.conf.Table))
	if err := q.WithContext(ctx).Exec(); err != nil {
		return fmt.Errorf("unable to create table %s: %v", c.conf.Table, err)
	}
	return nil
}

func (c *client) Get(ctx context.Context, key string) (string, error) {
	s, err := c.getSession()
	if err != nil {
		return "", fmt.Errorf("unable to connect to db: %v", err)
	}
	q := c.getQuery(s)
	var v string
	for retry := 0; retry < *c.conf.ClientRetries; retry++ {
		err := c.traceQuery(q.Bind(key).WithContext(ctx), true).Scan(&v)
		switch err {
		case gocql.ErrNoConnections:
			// retry
		default:
			c.cacheGetQuery(q)
			return v, err
		}
	}
	c.cacheGetQuery(q)
	return "", gocql.ErrNoConnections
}

func (c *client) Put(ctx context.Context, key, val string) error {
	s, err := c.getSession()
	if err != nil {
		return fmt.Errorf("unable to connect to db: %v", err)
	}
	q := c.putQuery(s)

	for retry := 0; retry < *c.conf.ClientRetries; retry++ {
		err := c.traceQuery(q.Bind(key, val).WithContext(ctx), false).Exec()
		switch err {
		case gocql.ErrNoConnections:
			// retry
		default:
			c.cachePutQuery(q)
			return err
		}
	}
	c.cachePutQuery(q)
	return gocql.ErrNoConnections
}

func (c *client) Close() error {
	c.rat.Close()
	c.wat.Close()
	return nil
}

func makeDB(hosts []string, data []byte) (db.DB, error) {
	var cfg conf
	if err := json.Unmarshal(data, &cfg); err != nil {
		return nil, fmt.Errorf("invalid cassandra config: %v", err)
	}
	cfg.fillDefaults()
	return newClient(hosts, &cfg)
}

func init() {
	db.Register("cassandra", makeDB)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"strings"
	"time"

	tgCfgUtil "github.com/GoogleCloudPlatform/testgrid/config"
	"github.com/GoogleCloudPlatform/testgrid/config/yamlcfg"
	"github.com/GoogleCloudPlatform/testgrid/util/gcs"
	prowConfig "k8s.io/test-infra/prow/config"
	configflagutil "k8s.io/test-infra/prow/flagutil/config"

	"cloud.google.com/go/storage"
	"github.com/sirupsen/logrus"
	"sigs.k8s.io/yaml"
)

type multiString []string

func (m multiString) String() string {
	return strings.Join(m, ",")
}

func (m *multiString) Set(v string) error {
	*m = strings.Split(v, ",")
	return nil
}

// How long Configurator waits between file checks in polling mode
const pollingTime = time.Second

type options struct {
	creds              string
	inputs             multiString
	oneshot            bool
	output             string
	printText          bool
	validateConfigFile bool
	worldReadable      bool
	writeYAML          bool
	prowConfig         configflagutil.ConfigOptions
	defaultYAML        string
	updateDescription  bool
	prowJobURLPrefix   string
	strictUnmarshal    bool
}

func (o *options) gatherOptions(fs *flag.FlagSet, args []string) error {
	fs.StringVar(&o.creds, "gcp-service-account", "", "/path/to/gcp/creds (use local creds if empty)")
	fs.BoolVar(&o.oneshot, "oneshot", false, "Write proto once and exit instead of monitoring --yaml files for changes")
	fs.StringVar(&o.output, "output", "", "write proto to gs://bucket/obj or /local/path")
	fs.BoolVar(&o.printText, "print-text", false, "print generated info in text format to stdout")
	fs.BoolVar(&o.validateConfigFile, "validate-config-file", false, "validate that the given config files are syntactically correct and exit (proto is not written anywhere)")
	fs.BoolVar(&o.worldReadable, "world-readable", false, "when uploading the proto to GCS, makes it world readable. Has no effect on writing to the local filesystem.")
	fs.BoolVar(&o.writeYAML, "output-yaml", false, "Output to TestGrid YAML instead of config proto")
	fs.Var(&o.inputs, "yaml", "comma-separated list of input YAML files or directories")
	o.prowConfig.ConfigPathFlagName = "prow-config"
	o.prowConfig.JobConfigPathFlagName = "prow-job-config"
	o.prowConfig.AddFlags(fs)
	fs.StringVar(&o.defaultYAML, "default", "", "path to default settings; required for proto outputs")
	fs.BoolVar(&o.updateDescription, "update-description", false, "add prowjob info to description even if non-empty")
	fs.StringVar(&o.prowJobURLPrefix, "prowjob-url-prefix", "", "for prowjob_config_url in descriptions: {prowjob-url-prefix}/{prowjob.sourcepath}")
	fs.BoolVar(&o.strictUnmarshal, "strict-unmarshal", false, "whether or not we want to be strict when unmarshalling configs")

	if err := fs.Parse(args); err != nil {
		return err
	}

	if len(o.inputs) == 0 || o.inputs[0] == "" {
		return errors.New("--yaml must include at least one file")
	}

	if !o.printText && !o.validateConfigFile && o.output == "" {
		return errors.New("--print-text, --validate-config-file, or --output required")
	}
	if o.validateConfigFile && o.output != "" {
		return errors.New("--validate-config-file doesn't write the proto anywhere")
	}
	if err := o.prowConfig.ValidateConfigOptional(); err != nil {
		return err
	}
	if o.defaultYAML == "" && !o.writeYAML {
		logrus.Warnf("--default not explicitly specified; assuming %s", o.inputs[0])
		o.defaultYAML = o.inputs[0]
	}
	return nil
}

// announceChanges watches for changes in "paths" and writes them to the channel
func announceChanges(ctx context.Context, paths []string, channel chan []string) {
	defer close(channel)
	modified := map[string]time.Time{}

	// TODO(fejta): consider waiting for a notification rather than polling
	// but performance isn't that big a deal here.
	for {
		// Terminate
		select {
		case <-ctx.Done():
			return
		default:
		}

		var changed []string

		// Check known files for deletions
		for p := range modified {
			_, accessErr := os.Stat(p)
			if os.IsNotExist(accessErr) {
				changed = append(changed, p)
				delete(modified, p)
			}
		}

		// Check given locations for new or modified files
		err := yamlcfg.SeekYAMLFiles(paths, func(path string, info os.FileInfo) error {
			lastModTime, present := modified[path]

			if t := info.ModTime(); !present || t.After(lastModTime) {
				changed = append(changed, path)
				modified[path] = t
			}

			return nil
		})

		if err != nil {
			logrus.WithError(err).Error("walk issue in announcer")
			return
		}

		if len(changed) > 0 {
			select {
			case <-ctx.Done():
				return
			case channel <- changed:
			}
		} else {
			time.Sleep(pollingTime)
		}
	}
}

func announceProwChanges(ctx context.Context, pca *prowConfig.Agent, channel chan []string) {
	pch := make(chan prowConfig.Delta)
	pca.Subscribe(pch)
	for {
		<-pch
		select {
		case <-ctx.Done():
			return
		case channel <- []string{"prow config"}:
		}
	}
}

func write(ctx context.Context, client *storage.Client, path string, bytes []byte, worldReadable bool, cacheControl string) error {

	u, err := url.Parse(path)
	if err != nil {
		return fmt.Errorf("invalid url %s: %v", path, err)
	}
	if u.Scheme != "gs" {
		return ioutil.WriteFile(path, bytes, 0644)
	}
	var p gcs.Path
	if err = p.SetURL(u); err != nil {
		return err
	}
	return gcs.Upload(ctx, client, p, bytes, worldReadable, cacheControl)
}

// Ignores what changed for now and recomputes everything
func doOneshot(ctx context.Context, client *storage.Client, opt options, prowConfigAgent *prowConfig.Agent) error {

	// Read Data Sources: Default, YAML configs, Prow Annotations
	c, err := yamlcfg.ReadConfig(opt.inputs, opt.defaultYAML, opt.strictUnmarshal)
	if err != nil {
		return fmt.Errorf("could not read testgrid config: %v", err)
	}

	// Remains nil if no default YAML
	var d *yamlcfg.DefaultConfiguration
	if opt.defaultYAML != "" {
		b, err := ioutil.ReadFile(opt.defaultYAML)
		if err != nil {
			return err
		}
		val, err := yamlcfg.LoadDefaults(b)
		if err != nil {
			return err
		}
		d = &val

	}

	pac := prowAwareConfigurator{
		defaultTestgridConfig: d,
		prowConfig:            prowConfigAgent.Config(),
		updateDescription:     opt.updateDescription,
		prowJobConfigPath:     opt.prowConfig.JobConfigPath,
		prowJobURLPrefix:      opt.prowJobURLPrefix,
	}

	if err := pac.applyProwjobAnnotations(&c); err != nil {
		return fmt.Errorf("could not apply prowjob annotations: %v", err)
	}

	// Print proto if requested
	if opt.printText {
		if opt.writeYAML {
			b, err := yaml.Marshal(&c)
			if err != nil {
				return fmt.Errorf("could not print yaml config: %v", err)
			}
			os.Stdout.Write(b)
		} else if err := tgCfgUtil.MarshalText(&c, os.Stdout); err != nil {
			return fmt.Errorf("could not print config: %v", err)
		}
	}

	// Write proto if requested
	if opt.output != "" {
		var b []byte
		var err error
		if opt.writeYAML {
			b, err = yaml.Marshal(&c)
		} else {
			b, err = tgCfgUtil.MarshalBytes(&c)
		}
		if err == nil {
			err = write(ctx, client, opt.output, b, opt.worldReadable, "")
		}
		if err != nil {
			return fmt.Errorf("could not write config: %v", err)
		}
	}
	return nil
}

func main() {
	// Parse flags
	var opt options
	if err := opt.gatherOptions(flag.CommandLine, os.Args[1:]); err != nil {
		log.Fatalf("Bad flags: %v", err)
	}

	ctx := context.Background()

	var prowConfigAgent *prowConfig.Agent
	if opt.prowConfig.ConfigPath != "" && opt.prowConfig.JobConfigPath != "" {
		agent, err := opt.prowConfig.ConfigAgent()
		if err != nil {
			log.Fatalf("FAIL: couldn't load prow config: %v", err)
		}
		prowConfigAgent = agent
	}

	// Config file validation only
	if opt.validateConfigFile {
		if err := doOneshot(ctx, nil, opt, prowConfigAgent); err != nil {
			log.Fatalf("FAIL: %v", err)
		}
		log.Println("Config validated successfully")
		return
	}

	// Set up GCS client if output is to GCS
	var client *storage.Client
	if strings.HasPrefix(opt.output, "gs://") {
		var err error
		var creds []string
		if opt.creds != "" {
			creds = append(creds, opt.creds)
		}
		client, err = gcs.ClientWithCreds(ctx, creds...)
		if err != nil {
			log.Fatalf("failed to create gcs client: %v", err)
		}
	}

	// Oneshot mode, write config and exit
	if opt.oneshot {
		if err := doOneshot(ctx, client, opt, prowConfigAgent); err != nil {
			log.Fatalf("FAIL: %v", err)
		}
		return
	}

	// Service mode, monitor input files for changes
	channel := make(chan []string)
	// Monitor files for changes
	go announceChanges(ctx, opt.inputs, channel)
	go announceProwChanges(ctx, prowConfigAgent, channel)

	// Wait for changed files
	for changes := range channel {
		log.Printf("Changed: %v", changes)
		log.Println("Writing config...")
		if err := doOneshot(ctx, client, opt, prowConfigAgent); err != nil {
			log.Printf("FAIL: %v", err)
			continue
		}
		log.Printf("Wrote config to %s", opt.output)
	}
}
<commit_msg>Add TG Validation to Configurator's validate-only mode<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"strings"
	"time"

	tgCfgUtil "github.com/GoogleCloudPlatform/testgrid/config"
	"github.com/GoogleCloudPlatform/testgrid/config/yamlcfg"
	"github.com/GoogleCloudPlatform/testgrid/util/gcs"
	prowConfig "k8s.io/test-infra/prow/config"
	configflagutil "k8s.io/test-infra/prow/flagutil/config"

	"cloud.google.com/go/storage"
	"github.com/sirupsen/logrus"
	"sigs.k8s.io/yaml"
)

type multiString []string

func (m multiString) String() string {
	return strings.Join(m, ",")
}

func (m *multiString) Set(v string) error {
	*m = strings.Split(v, ",")
	return nil
}

// How long Configurator waits between file checks in polling mode
const pollingTime = time.Second

type options struct {
	creds              string
	inputs             multiString
	oneshot            bool
	output             string
	printText          bool
	validateConfigFile bool
	worldReadable      bool
	writeYAML          bool
	prowConfig         configflagutil.ConfigOptions
	defaultYAML        string
	updateDescription  bool
	prowJobURLPrefix   string
	strictUnmarshal    bool
}

func (o *options) gatherOptions(fs *flag.FlagSet, args []string) error {
	fs.StringVar(&o.creds, "gcp-service-account", "", "/path/to/gcp/creds (use local creds if empty)")
	fs.BoolVar(&o.oneshot, "oneshot", false, "Write proto once and exit instead of monitoring --yaml files for changes")
	fs.StringVar(&o.output, "output", "", "write proto to gs://bucket/obj or /local/path")
	fs.BoolVar(&o.printText, "print-text", false, "print generated info in text format to stdout")
	fs.BoolVar(&o.validateConfigFile, "validate-config-file", false, "validate that the given config files are syntactically correct and exit (proto is not written anywhere)")
	fs.BoolVar(&o.worldReadable, "world-readable", false, "when uploading the proto to GCS, makes it world readable. Has no effect on writing to the local filesystem.")
	fs.BoolVar(&o.writeYAML, "output-yaml", false, "Output to TestGrid YAML instead of config proto")
	fs.Var(&o.inputs, "yaml", "comma-separated list of input YAML files or directories")
	o.prowConfig.ConfigPathFlagName = "prow-config"
	o.prowConfig.JobConfigPathFlagName = "prow-job-config"
	o.prowConfig.AddFlags(fs)
	fs.StringVar(&o.defaultYAML, "default", "", "path to default settings; required for proto outputs")
	fs.BoolVar(&o.updateDescription, "update-description", false, "add prowjob info to description even if non-empty")
	fs.StringVar(&o.prowJobURLPrefix, "prowjob-url-prefix", "", "for prowjob_config_url in descriptions: {prowjob-url-prefix}/{prowjob.sourcepath}")
	fs.BoolVar(&o.strictUnmarshal, "strict-unmarshal", false, "whether or not we want to be strict when unmarshalling configs")

	if err := fs.Parse(args); err != nil {
		return err
	}

	if len(o.inputs) == 0 || o.inputs[0] == "" {
		return errors.New("--yaml must include at least one file")
	}

	if !o.printText && !o.validateConfigFile && o.output == "" {
		return errors.New("--print-text, --validate-config-file, or --output required")
	}
	if o.validateConfigFile && o.output != "" {
		return errors.New("--validate-config-file doesn't write the proto anywhere")
	}
	if err := o.prowConfig.ValidateConfigOptional(); err != nil {
		return err
	}
	if o.defaultYAML == "" && !o.writeYAML {
		logrus.Warnf("--default not explicitly specified; assuming %s", o.inputs[0])
		o.defaultYAML = o.inputs[0]
	}
	return nil
}

// announceChanges watches for changes in "paths" and writes them to the channel
func announceChanges(ctx context.Context, paths []string, channel chan []string) {
	defer close(channel)
	modified := map[string]time.Time{}

	// TODO(fejta): consider waiting for a notification rather than polling
	// but performance isn't that big a deal here.
	for {
		// Terminate
		select {
		case <-ctx.Done():
			return
		default:
		}

		var changed []string

		// Check known files for deletions
		for p := range modified {
			_, accessErr := os.Stat(p)
			if os.IsNotExist(accessErr) {
				changed = append(changed, p)
				delete(modified, p)
			}
		}

		// Check given locations for new or modified files
		err := yamlcfg.SeekYAMLFiles(paths, func(path string, info os.FileInfo) error {
			lastModTime, present := modified[path]

			if t := info.ModTime(); !present || t.After(lastModTime) {
				changed = append(changed, path)
				modified[path] = t
			}

			return nil
		})

		if err != nil {
			logrus.WithError(err).Error("walk issue in announcer")
			return
		}

		if len(changed) > 0 {
			select {
			case <-ctx.Done():
				return
			case channel <- changed:
			}
		} else {
			time.Sleep(pollingTime)
		}
	}
}

func announceProwChanges(ctx context.Context, pca *prowConfig.Agent, channel chan []string) {
	pch := make(chan prowConfig.Delta)
	pca.Subscribe(pch)
	for {
		<-pch
		select {
		case <-ctx.Done():
			return
		case channel <- []string{"prow config"}:
		}
	}
}

func write(ctx context.Context, client *storage.Client, path string, bytes []byte, worldReadable bool, cacheControl string) error {

	u, err := url.Parse(path)
	if err != nil {
		return fmt.Errorf("invalid url %s: %v", path, err)
	}
	if u.Scheme != "gs" {
		return ioutil.WriteFile(path, bytes, 0644)
	}
	var p gcs.Path
	if err = p.SetURL(u); err != nil {
		return err
	}
	return gcs.Upload(ctx, client, p, bytes, worldReadable, cacheControl)
}

// Ignores what changed for now and recomputes everything
func doOneshot(ctx context.Context, client *storage.Client, opt options, prowConfigAgent *prowConfig.Agent) error {

	// Read Data Sources: Default, YAML configs, Prow Annotations
	c, err := yamlcfg.ReadConfig(opt.inputs, opt.defaultYAML, opt.strictUnmarshal)
	if err != nil {
		return fmt.Errorf("could not read testgrid config: %v", err)
	}

	// Remains nil if no default YAML
	var d *yamlcfg.DefaultConfiguration
	if opt.defaultYAML != "" {
		b, err := ioutil.ReadFile(opt.defaultYAML)
		if err != nil {
			return err
		}
		val, err := yamlcfg.LoadDefaults(b)
		if err != nil {
			return err
		}
		d = &val

	}

	pac := prowAwareConfigurator{
		defaultTestgridConfig: d,
		prowConfig:            prowConfigAgent.Config(),
		updateDescription:     opt.updateDescription,
		prowJobConfigPath:     opt.prowConfig.JobConfigPath,
		prowJobURLPrefix:      opt.prowJobURLPrefix,
	}

	if err := pac.applyProwjobAnnotations(&c); err != nil {
		return fmt.Errorf("could not apply prowjob annotations: %v", err)
	}

	if opt.validateConfigFile {
		return tgCfgUtil.Validate(&c)
	}

	// Print proto if requested
	if opt.printText {
		if opt.writeYAML {
			b, err := yaml.Marshal(&c)
			if err != nil {
				return fmt.Errorf("could not print yaml config: %v", err)
			}
			os.Stdout.Write(b)
		} else if err := tgCfgUtil.MarshalText(&c, os.Stdout); err != nil {
			return fmt.Errorf("could not print config: %v", err)
		}
	}

	// Write proto if requested
	if opt.output != "" {
		var b []byte
		var err error
		if opt.writeYAML {
			b, err = yaml.Marshal(&c)
		} else {
			b, err = tgCfgUtil.MarshalBytes(&c)
		}
		if err == nil {
			err = write(ctx, client, opt.output, b, opt.worldReadable, "")
		}
		if err != nil {
			return fmt.Errorf("could not write config: %v", err)
		}
	}
	return nil
}

func main() {
	// Parse flags
	var opt options
	if err := opt.gatherOptions(flag.CommandLine, os.Args[1:]); err != nil {
		log.Fatalf("Bad flags: %v", err)
	}

	ctx := context.Background()

	var prowConfigAgent *prowConfig.Agent
	if opt.prowConfig.ConfigPath != "" && opt.prowConfig.JobConfigPath != "" {
		agent, err := opt.prowConfig.ConfigAgent()
		if err != nil {
			log.Fatalf("FAIL: couldn't load prow config: %v", err)
		}
		prowConfigAgent = agent
	}

	// Config file validation only
	if opt.validateConfigFile {
		if err := doOneshot(ctx, nil, opt, prowConfigAgent); err != nil {
			log.Fatalf("FAIL: %v", err)
		}
		log.Println("Config validated successfully")
		return
	}

	// Set up GCS client if output is to GCS
	var client *storage.Client
	if strings.HasPrefix(opt.output, "gs://") {
		var err error
		var creds []string
		if opt.creds != "" {
			creds = append(creds, opt.creds)
		}
		client, err = gcs.ClientWithCreds(ctx, creds...)
		if err != nil {
			log.Fatalf("failed to create gcs client: %v", err)
		}
	}

	// Oneshot mode, write config and exit
	if opt.oneshot {
		if err := doOneshot(ctx, client, opt, prowConfigAgent); err != nil {
			log.Fatalf("FAIL: %v", err)
		}
		return
	}

	// Service mode, monitor input files for changes
	channel := make(chan []string)
	// Monitor files for changes
	go announceChanges(ctx, opt.inputs, channel)
	go announceProwChanges(ctx, prowConfigAgent, channel)

	// Wait for changed files
	for changes := range channel {
		log.Printf("Changed: %v", changes)
		log.Println("Writing config...")
		if err := doOneshot(ctx, client, opt, prowConfigAgent); err != nil {
			log.Printf("FAIL: %v", err)
			continue
		}
		log.Printf("Wrote config to %s", opt.output)
	}
}
<|endoftext|>


<commit_before>package eval

import "os"

// Variable represents an elvish variable.
type Variable interface {
	Set(v Value)
	Get() Value
}

type ptrVariable struct {
	valuePtr *Value
}

func NewPtrVariable(v Value) Variable {
	return ptrVariable{&v}
}

func (iv ptrVariable) Set(val Value) {
	*iv.valuePtr = val
}

func (iv ptrVariable) Get() Value {
	return *iv.valuePtr
}

// elemVariable is an element of a IndexSetter.
type elemVariable struct {
	container IndexSetter
	index     Value
}

func (ev elemVariable) Set(val Value) {
	ev.container.IndexSet(ev.index, val)
}

func (ev elemVariable) Get() Value {
	return ev.container.IndexOne(ev.index)
}

type envVariable struct {
	name string
}

func newEnvVariable(name string) envVariable {
	return envVariable{name}
}

func (ev envVariable) Set(val Value) {
	os.Setenv(ev.name, ToString(val))
}

func (ev envVariable) Get() Value {
	return String(os.Getenv(ev.name))
}
<commit_msg>eval: add roVariable.<commit_after>package eval

import (
	"errors"
	"os"
)

var (
	ErrRoVariable = errors.New("read-only; cannot be set")
)

// Variable represents an elvish variable.
type Variable interface {
	Set(v Value)
	Get() Value
}

type ptrVariable struct {
	valuePtr *Value
}

func NewPtrVariable(v Value) Variable {
	return ptrVariable{&v}
}

func (iv ptrVariable) Set(val Value) {
	*iv.valuePtr = val
}

func (iv ptrVariable) Get() Value {
	return *iv.valuePtr
}

type roVariable struct {
	value Value
}

func NewRoVariable(v Value) Variable {
	return roVariable{v}
}

func (rv roVariable) Set(val Value) {
	throw(ErrRoVariable)
}

func (rv roVariable) Get() Value {
	return rv.value
}

// elemVariable is an element of a IndexSetter.
type elemVariable struct {
	container IndexSetter
	index     Value
}

func (ev elemVariable) Set(val Value) {
	ev.container.IndexSet(ev.index, val)
}

func (ev elemVariable) Get() Value {
	return ev.container.IndexOne(ev.index)
}

type envVariable struct {
	name string
}

func newEnvVariable(name string) envVariable {
	return envVariable{name}
}

func (ev envVariable) Set(val Value) {
	os.Setenv(ev.name, ToString(val))
}

func (ev envVariable) Get() Value {
	return String(os.Getenv(ev.name))
}
<|endoftext|>


<commit_before>// Copyright 2017 The go-hep Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ntcsv_test

import (
	"fmt"
	"log"
	"reflect"
	"testing"

	"go-hep.org/x/hep/hbook/ntup/ntcsv"
)

func TestOpen(t *testing.T) {
	for _, test := range []struct {
		name  string
		query string
		opts  []ntcsv.Option
	}{
		{"testdata/simple.csv", `var1, var2, var3`,
			[]ntcsv.Option{
				ntcsv.Comma(';'),
				ntcsv.Comment('#'),
			},
		},
		{"testdata/simple.csv", `var1, var2, var3`,
			[]ntcsv.Option{
				ntcsv.Comma(';'),
				ntcsv.Comment('#'),
				ntcsv.Columns("var1", "var2", "var3"),
			},
		},
		{"testdata/simple.csv", `v1, v2, v3`,
			[]ntcsv.Option{
				ntcsv.Comma(';'),
				ntcsv.Comment('#'),
				ntcsv.Columns("v1", "v2", "v3"),
			},
		},
		{"testdata/simple-comma.csv", `var1, var2, var3`,
			[]ntcsv.Option{
				ntcsv.Comma(','),
				ntcsv.Comment('#'),
			},
		},
		{"testdata/simple-with-comment.csv", `var1, var2, var3`,
			[]ntcsv.Option{
				ntcsv.Comma(';'),
				ntcsv.Comment('#'),
			},
		},
		{"testdata/simple-with-comment.csv", `v1, v2, v3`,
			[]ntcsv.Option{
				ntcsv.Comma(';'),
				ntcsv.Comment('#'),
				ntcsv.Columns("v1", "v2", "v3"),
			},
		},
		{"testdata/simple-with-header.csv", `i, f, str`,
			[]ntcsv.Option{
				ntcsv.Header(),
				ntcsv.Comma(';'),
				ntcsv.Comment('#'),
			},
		},
		{"testdata/simple-with-header.csv", `i64, f64, str`,
			[]ntcsv.Option{
				ntcsv.Header(),
				ntcsv.Columns("i64", "f64", "str"),
				ntcsv.Comma(';'),
				ntcsv.Comment('#'),
			},
		},
	} {
		testCSV(t, test.name, test.query, test.opts...)
	}
}

func testCSV(t *testing.T, name, query string, opts ...ntcsv.Option) {
	nt, err := ntcsv.Open(name, opts...)
	if err != nil {
		t.Errorf("%s: error opening n-tuple: %v", name, err)
		return
	}
	defer nt.DB().Close()

	type dataType struct {
		i int64
		f float64
		s string
	}

	var got []dataType
	err = nt.Scan(
		query,
		func(i int64, f float64, s string) error {
			got = append(got, dataType{i, f, s})
			return nil
		},
	)
	if err != nil {
		t.Errorf("%s: error scanning: %v", name, err)
		return
	}

	want := []dataType{
		{0, 0, "str-0"},
		{1, 1, "str-1"},
		{2, 2, "str-2"},
		{3, 3, "str-3"},
		{4, 4, "str-4"},
		{5, 5, "str-5"},
		{6, 6, "str-6"},
		{7, 7, "str-7"},
		{8, 8, "str-8"},
		{9, 9, "str-9"},
	}
	if !reflect.DeepEqual(got, want) {
		t.Errorf("%s: got=\n%v\nwant=\n%v\n", name, got, want)
		return
	}
}

func ExampleOpen() {
	// Open a new n-tuple pointing at a CSV file "testdata/simple.csv"
	// whose field separator is ';'.
	// We rename the columns v1, v2 and v3.
	nt, err := ntcsv.Open(
		"testdata/simple.csv",
		ntcsv.Comma(';'),
		ntcsv.Columns("v1", "v2", "v3"),
	)
	if err != nil {
		log.Fatal(err)
	}

	defer nt.DB().Close()

	err = nt.Scan("v1, v2, v3", func(i int64, f float64, s string) error {
		fmt.Printf("%d %f %q\n", i, f, s)
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}

	// Output:
	// 0 0.000000 "str-0"
	// 1 1.000000 "str-1"
	// 2 2.000000 "str-2"
	// 3 3.000000 "str-3"
	// 4 4.000000 "str-4"
	// 5 5.000000 "str-5"
	// 6 6.000000 "str-6"
	// 7 7.000000 "str-7"
	// 8 8.000000 "str-8"
	// 9 9.000000 "str-9"
}

func ExampleOpen_withDefaultVarNames() {
	// Open a new n-tuple pointing at a CSV file "testdata/simple.csv"
	// whose field separator is ';'.
	// We use the default column names: var1, var2, var3, ...
	nt, err := ntcsv.Open(
		"testdata/simple.csv",
		ntcsv.Comma(';'),
	)
	if err != nil {
		log.Fatal(err)
	}

	defer nt.DB().Close()

	err = nt.Scan("var1, var2, var3", func(i int64, f float64, s string) error {
		fmt.Printf("%d %f %q\n", i, f, s)
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}

	// Output:
	// 0 0.000000 "str-0"
	// 1 1.000000 "str-1"
	// 2 2.000000 "str-2"
	// 3 3.000000 "str-3"
	// 4 4.000000 "str-4"
	// 5 5.000000 "str-5"
	// 6 6.000000 "str-6"
	// 7 7.000000 "str-7"
	// 8 8.000000 "str-8"
	// 9 9.000000 "str-9"
}

func ExampleOpen_withHeader() {
	// Open a new n-tuple pointing at a CSV file "testdata/simple.csv"
	// whose field separator is ';'.
	// We rename the columns v1, v2 and v3.
	// We tell the CSV driver to handle the CSV header.
	nt, err := ntcsv.Open(
		"testdata/simple-with-header.csv",
		ntcsv.Comma(';'),
		ntcsv.Header(),
		ntcsv.Columns("v1", "v2", "v3"),
	)
	if err != nil {
		log.Fatal(err)
	}

	defer nt.DB().Close()

	err = nt.Scan("v1, v2, v3", func(i int64, f float64, s string) error {
		fmt.Printf("%d %f %q\n", i, f, s)
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}

	// Output:
	// 0 0.000000 "str-0"
	// 1 1.000000 "str-1"
	// 2 2.000000 "str-2"
	// 3 3.000000 "str-3"
	// 4 4.000000 "str-4"
	// 5 5.000000 "str-5"
	// 6 6.000000 "str-6"
	// 7 7.000000 "str-7"
	// 8 8.000000 "str-8"
	// 9 9.000000 "str-9"
}

func ExampleOpen_withHeaderAndImplicitColumns() {
	// Open a new n-tuple pointing at a CSV file "testdata/simple.csv"
	// whose field separator is ';'.
	// We tell the CSV driver to handle the CSV header.
	// And we implicitly use the column names for the queries.
	nt, err := ntcsv.Open(
		"testdata/simple-with-header.csv",
		ntcsv.Comma(';'),
		ntcsv.Header(),
	)
	if err != nil {
		log.Fatal(err)
	}

	defer nt.DB().Close()

	err = nt.Scan("i, f, str", func(i int64, f float64, s string) error {
		fmt.Printf("%d %f %q\n", i, f, s)
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}

	// Output:
	// 0 0.000000 "str-0"
	// 1 1.000000 "str-1"
	// 2 2.000000 "str-2"
	// 3 3.000000 "str-3"
	// 4 4.000000 "str-4"
	// 5 5.000000 "str-5"
	// 6 6.000000 "str-6"
	// 7 7.000000 "str-7"
	// 8 8.000000 "str-8"
	// 9 9.000000 "str-9"
}

func ExampleOpen_withHeaderAndExlicitColumns() {
	// Open a new n-tuple pointing at a CSV file "testdata/simple.csv"
	// whose field separator is ';'.
	// We tell the CSV driver to handle the CSV header.
	// And we explicitly use our column names for the queries.
	nt, err := ntcsv.Open(
		"testdata/simple-with-header.csv",
		ntcsv.Comma(';'),
		ntcsv.Header(),
		ntcsv.Columns("v1", "v2", "v3"),
	)
	if err != nil {
		log.Fatal(err)
	}

	defer nt.DB().Close()

	err = nt.Scan("v1, v2, v3", func(i int64, f float64, s string) error {
		fmt.Printf("%d %f %q\n", i, f, s)
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}

	// Output:
	// 0 0.000000 "str-0"
	// 1 1.000000 "str-1"
	// 2 2.000000 "str-2"
	// 3 3.000000 "str-3"
	// 4 4.000000 "str-4"
	// 5 5.000000 "str-5"
	// 6 6.000000 "str-6"
	// 7 7.000000 "str-7"
	// 8 8.000000 "str-8"
	// 9 9.000000 "str-9"
}
<commit_msg>hbook/ntup/ntcsv: use sub tests<commit_after>// Copyright 2017 The go-hep Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ntcsv_test

import (
	"fmt"
	"log"
	"reflect"
	"testing"

	"go-hep.org/x/hep/hbook/ntup/ntcsv"
)

func TestOpen(t *testing.T) {
	for _, test := range []struct {
		name  string
		query string
		opts  []ntcsv.Option
	}{
		{"testdata/simple.csv", `var1, var2, var3`,
			[]ntcsv.Option{
				ntcsv.Comma(';'),
				ntcsv.Comment('#'),
			},
		},
		{"testdata/simple.csv", `var1, var2, var3`,
			[]ntcsv.Option{
				ntcsv.Comma(';'),
				ntcsv.Comment('#'),
				ntcsv.Columns("var1", "var2", "var3"),
			},
		},
		{"testdata/simple.csv", `v1, v2, v3`,
			[]ntcsv.Option{
				ntcsv.Comma(';'),
				ntcsv.Comment('#'),
				ntcsv.Columns("v1", "v2", "v3"),
			},
		},
		{"testdata/simple-comma.csv", `var1, var2, var3`,
			[]ntcsv.Option{
				ntcsv.Comma(','),
				ntcsv.Comment('#'),
			},
		},
		{"testdata/simple-with-comment.csv", `var1, var2, var3`,
			[]ntcsv.Option{
				ntcsv.Comma(';'),
				ntcsv.Comment('#'),
			},
		},
		{"testdata/simple-with-comment.csv", `v1, v2, v3`,
			[]ntcsv.Option{
				ntcsv.Comma(';'),
				ntcsv.Comment('#'),
				ntcsv.Columns("v1", "v2", "v3"),
			},
		},
		{"testdata/simple-with-header.csv", `i, f, str`,
			[]ntcsv.Option{
				ntcsv.Header(),
				ntcsv.Comma(';'),
				ntcsv.Comment('#'),
			},
		},
		{"testdata/simple-with-header.csv", `i64, f64, str`,
			[]ntcsv.Option{
				ntcsv.Header(),
				ntcsv.Columns("i64", "f64", "str"),
				ntcsv.Comma(';'),
				ntcsv.Comment('#'),
			},
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			testCSV(t, test.name, test.query, test.opts...)
		})
	}
}

func testCSV(t *testing.T, name, query string, opts ...ntcsv.Option) {
	nt, err := ntcsv.Open(name, opts...)
	if err != nil {
		t.Fatalf("%s: error opening n-tuple: %v", name, err)
	}
	defer nt.DB().Close()

	type dataType struct {
		i int64
		f float64
		s string
	}

	var got []dataType
	err = nt.Scan(
		query,
		func(i int64, f float64, s string) error {
			got = append(got, dataType{i, f, s})
			return nil
		},
	)
	if err != nil {
		t.Fatalf("%s: error scanning: %v", name, err)
	}

	want := []dataType{
		{0, 0, "str-0"},
		{1, 1, "str-1"},
		{2, 2, "str-2"},
		{3, 3, "str-3"},
		{4, 4, "str-4"},
		{5, 5, "str-5"},
		{6, 6, "str-6"},
		{7, 7, "str-7"},
		{8, 8, "str-8"},
		{9, 9, "str-9"},
	}
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("%s: got=\n%v\nwant=\n%v\n", name, got, want)
	}
}

func ExampleOpen() {
	// Open a new n-tuple pointing at a CSV file "testdata/simple.csv"
	// whose field separator is ';'.
	// We rename the columns v1, v2 and v3.
	nt, err := ntcsv.Open(
		"testdata/simple.csv",
		ntcsv.Comma(';'),
		ntcsv.Columns("v1", "v2", "v3"),
	)
	if err != nil {
		log.Fatal(err)
	}

	defer nt.DB().Close()

	err = nt.Scan("v1, v2, v3", func(i int64, f float64, s string) error {
		fmt.Printf("%d %f %q\n", i, f, s)
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}

	// Output:
	// 0 0.000000 "str-0"
	// 1 1.000000 "str-1"
	// 2 2.000000 "str-2"
	// 3 3.000000 "str-3"
	// 4 4.000000 "str-4"
	// 5 5.000000 "str-5"
	// 6 6.000000 "str-6"
	// 7 7.000000 "str-7"
	// 8 8.000000 "str-8"
	// 9 9.000000 "str-9"
}

func ExampleOpen_withDefaultVarNames() {
	// Open a new n-tuple pointing at a CSV file "testdata/simple.csv"
	// whose field separator is ';'.
	// We use the default column names: var1, var2, var3, ...
	nt, err := ntcsv.Open(
		"testdata/simple.csv",
		ntcsv.Comma(';'),
	)
	if err != nil {
		log.Fatal(err)
	}

	defer nt.DB().Close()

	err = nt.Scan("var1, var2, var3", func(i int64, f float64, s string) error {
		fmt.Printf("%d %f %q\n", i, f, s)
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}

	// Output:
	// 0 0.000000 "str-0"
	// 1 1.000000 "str-1"
	// 2 2.000000 "str-2"
	// 3 3.000000 "str-3"
	// 4 4.000000 "str-4"
	// 5 5.000000 "str-5"
	// 6 6.000000 "str-6"
	// 7 7.000000 "str-7"
	// 8 8.000000 "str-8"
	// 9 9.000000 "str-9"
}

func ExampleOpen_withHeader() {
	// Open a new n-tuple pointing at a CSV file "testdata/simple.csv"
	// whose field separator is ';'.
	// We rename the columns v1, v2 and v3.
	// We tell the CSV driver to handle the CSV header.
	nt, err := ntcsv.Open(
		"testdata/simple-with-header.csv",
		ntcsv.Comma(';'),
		ntcsv.Header(),
		ntcsv.Columns("v1", "v2", "v3"),
	)
	if err != nil {
		log.Fatal(err)
	}

	defer nt.DB().Close()

	err = nt.Scan("v1, v2, v3", func(i int64, f float64, s string) error {
		fmt.Printf("%d %f %q\n", i, f, s)
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}

	// Output:
	// 0 0.000000 "str-0"
	// 1 1.000000 "str-1"
	// 2 2.000000 "str-2"
	// 3 3.000000 "str-3"
	// 4 4.000000 "str-4"
	// 5 5.000000 "str-5"
	// 6 6.000000 "str-6"
	// 7 7.000000 "str-7"
	// 8 8.000000 "str-8"
	// 9 9.000000 "str-9"
}

func ExampleOpen_withHeaderAndImplicitColumns() {
	// Open a new n-tuple pointing at a CSV file "testdata/simple.csv"
	// whose field separator is ';'.
	// We tell the CSV driver to handle the CSV header.
	// And we implicitly use the column names for the queries.
	nt, err := ntcsv.Open(
		"testdata/simple-with-header.csv",
		ntcsv.Comma(';'),
		ntcsv.Header(),
	)
	if err != nil {
		log.Fatal(err)
	}

	defer nt.DB().Close()

	err = nt.Scan("i, f, str", func(i int64, f float64, s string) error {
		fmt.Printf("%d %f %q\n", i, f, s)
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}

	// Output:
	// 0 0.000000 "str-0"
	// 1 1.000000 "str-1"
	// 2 2.000000 "str-2"
	// 3 3.000000 "str-3"
	// 4 4.000000 "str-4"
	// 5 5.000000 "str-5"
	// 6 6.000000 "str-6"
	// 7 7.000000 "str-7"
	// 8 8.000000 "str-8"
	// 9 9.000000 "str-9"
}

func ExampleOpen_withHeaderAndExlicitColumns() {
	// Open a new n-tuple pointing at a CSV file "testdata/simple.csv"
	// whose field separator is ';'.
	// We tell the CSV driver to handle the CSV header.
	// And we explicitly use our column names for the queries.
	nt, err := ntcsv.Open(
		"testdata/simple-with-header.csv",
		ntcsv.Comma(';'),
		ntcsv.Header(),
		ntcsv.Columns("v1", "v2", "v3"),
	)
	if err != nil {
		log.Fatal(err)
	}

	defer nt.DB().Close()

	err = nt.Scan("v1, v2, v3", func(i int64, f float64, s string) error {
		fmt.Printf("%d %f %q\n", i, f, s)
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}

	// Output:
	// 0 0.000000 "str-0"
	// 1 1.000000 "str-1"
	// 2 2.000000 "str-2"
	// 3 3.000000 "str-3"
	// 4 4.000000 "str-4"
	// 5 5.000000 "str-5"
	// 6 6.000000 "str-6"
	// 7 7.000000 "str-7"
	// 8 8.000000 "str-8"
	// 9 9.000000 "str-9"
}
<|endoftext|>