<commit_before>// tinystat is used to compare two or more sets of measurements (e.g., runs of a
// multiple runs of benchmarks of two possible implementations) and determine if
// they are statistically different.
//
// Imagine we have the results of different animals' SAT scores. Each animal
// took the SATs multiple times, and we're assuming that differences between
// each animal's attempts are measurement error (i.e., normally distributed). We
// can test for differences as follows:
//
//     $ tinystat examples/iguana examples/chameleon examples/leopard
//
//     Filename            N  Min         Max         Median      Mean        StdDev
//     examples/iguana     7  50.000000   750.000000  200.000000  300.000000  238.047614
//     examples/chameleon  5  150.000000  930.000000  500.000000  540.000000  299.081929
//                         No difference proven at 95% confidence.
//     examples/leopard    6  700.000000  700.000000  700.000000  700.000000  0.000000
//                         Difference at 95% confidence!
//                            400.000000 +/- 215.283224
//                            57.142857% +/- 30.754746%
//                            (Student's t, pooled s = 175.809815)
//
// As you can see, despite the superficial differences between the iguana's
// scores and the chameleon's scores, there is no statistically significant
// difference between the two at a 95% confidence level. The leopard, on the
// other hand, has statistically significantly different scores.
package main

import (
	"encoding/csv"
	"fmt"
	"os"
	"strconv"
	"text/tabwriter"

	"github.com/codahale/tinystat"
	"github.com/docopt/docopt-go"
)

func main() {
	usage := `tinystat helps you conduct experiments.

Usage:
  tinystat [options] <control> [<experiment>...]

Arguments:
  <control>     Input file containing control measurements.
  <experiment>  Input file containing experimental measurements.

Options:
  -h --help               Display this usage information.
  -c --confidence=<pct>   Specify confidence level for analysis. [default: 95]
  -C --column=<num>       The column to analyze. [default: 0]
  -d --delimiter=(t|s|c)  Tab, space, or comma delimited data. [default: t]
`

	args, err := docopt.Parse(usage, nil, true, "", true)
	if err != nil {
		panic(err)
	}

	confidence, err := strconv.ParseFloat(args["--confidence"].(string), 64)
	if err != nil {
		panic(err)
	}

	column, err := strconv.Atoi(args["--column"].(string))
	if err != nil {
		panic(err)
	}

	var delimiter rune
	switch d := args["--delimiter"]; d {
	case "t":
		delimiter = '\t'
	case "c":
		delimiter = ','
	case "s":
		delimiter = ' '
	default:
		panic(fmt.Errorf("bad delimiter: %#v", d))
	}

	controlFilename := args["<control>"].(string)
	experimentFilenames := args["<experiment>"].([]string)

	w := new(tabwriter.Writer)
	w.Init(os.Stdout, 2, 0, 2, ' ', 0)
	fmt.Fprintln(w, "Filename\tN\tMin\tMax\tMedian\tMean\tStdDev\t")

	control, err := readFile(controlFilename, column, delimiter)
	if err != nil {
		panic(err)
	}
	fmt.Fprintf(w,
		"%s\t%.0f\t%f\t%f\t%f\t%f\t%f\t\n",
		controlFilename,
		control.N,
		control.Min,
		control.Max,
		control.Median,
		control.Mean,
		control.StdDev,
	)

	for _, filename := range experimentFilenames {
		experimental, err := readFile(filename, column, delimiter)
		if err != nil {
			panic(err)
		}

		fmt.Fprintf(w,
			"%s\t%.0f\t%f\t%f\t%f\t%f\t%f\t\n",
			filename,
			experimental.N,
			experimental.Min,
			experimental.Max,
			experimental.Median,
			experimental.Mean,
			experimental.StdDev,
		)

		d := tinystat.Compare(control, experimental, confidence)
		if d.Significant() {
			fmt.Fprintf(w, "\tDifference at %v%% confidence!\n", confidence)
			fmt.Fprintf(w, "\t\t%10f +/- %10f\n", d.Delta, d.Error)
			fmt.Fprintf(w, "\t\t%9f%% +/- %9f%%\n", d.PctDelta, d.PctError)
			fmt.Fprintf(w, "\t\t(Student's t, pooled s = %f)\n", d.PooledStdDev)
		} else {
			fmt.Fprintf(w, "\tNo difference proven at %v%% confidence.\n", confidence)
		}
	}
	_ = w.Flush()
}

func readFile(filename string, col int, del rune) (tinystat.Summary, error) {
	f, err := os.Open(filename)
	if err != nil {
		return tinystat.Summary{}, err
	}
	defer f.Close()

	r := csv.NewReader(f)
	r.Comma = del
	records, err := r.ReadAll()
	if err != nil {
		return tinystat.Summary{}, err
	}

	data := make([]float64, len(records))
	for i, s := range records {
		n, err := strconv.ParseFloat(s[col], 64)
		if err != nil {
			return tinystat.Summary{}, err
		}
		data[i] = n
	}

	return tinystat.Summarize(data), nil
}
<commit_msg>Add box charts.<commit_after>// tinystat is used to compare two or more sets of measurements (e.g., runs of a
// multiple runs of benchmarks of two possible implementations) and determine if
// they are statistically different.
//
// Imagine we have the results of different animals' SAT scores. Each animal
// took the SATs multiple times, and we're assuming that differences between
// each animal's attempts are measurement error (i.e., normally distributed). We
// can test for differences as follows:
//
//     $ tinystat iguana chameleon leopard
//
//       1000  +
//             |
//             |                              |
//             |                        +-----------+
//             |                        |           |
//             |              |         |           |
//             |              |         |           |  +-----*-----+
//             |              |         |           |
//             |              |         |     *     |
//        500  +              |         +-----------+
//             |              |         |           |
//             |        +-----------+   |           |
//             |        |     *     |   |           |
//             |        |           |   +-----------+
//             |        +-----------+         |
//             |        +-----------+         |
//             |              |
//          0  +--------------|-----------------------------------------------
//                         iguana         chameleon       leopard
//
//     Filename   N  Min         Max         Median      Mean        StdDev
//     iguana     7  50.000000   750.000000  200.000000  300.000000  238.047614
//     chameleon  5  150.000000  930.000000  500.000000  540.000000  299.081929
//                No difference proven at 95% confidence.
//     leopard    6  700.000000  700.000000  700.000000  700.000000  0.000000
//                Difference at 95% confidence!
//                  400.000000 +/- 215.281773
//                  57.142857% +/- 30.754539%
//                  (Student's t, pooled s = 175.809815)
//
// As you can see, despite the superficial differences between the iguana's
// scores and the chameleon's scores, there is no statistically significant
// difference between the two at a 95% confidence level. The leopard, on the
// other hand, has statistically significantly different scores.
package main

import (
	"bytes"
	"encoding/csv"
	"fmt"
	"os"
	"strconv"
	"text/tabwriter"

	"github.com/codahale/tinystat"
	"github.com/docopt/docopt-go"
	"github.com/vdobler/chart"
	"github.com/vdobler/chart/txtg"
)

func main() {
	usage := `tinystat helps you conduct experiments.

Usage:
  tinystat [options] <control> [<experiment>...]

Arguments:
  <control>     Input file containing control measurements.
  <experiment>  Input file containing experimental measurements.

Options:
  -h --help               Display this usage information.
  -c --confidence=<pct>   Specify confidence level for analysis. [default: 95]
  -C --column=<num>       The column to analyze. [default: 0]
  -d --delimiter=(t|s|c)  Tab, space, or comma delimited data. [default: t]
  -n --no-chart           Don't display the box chart.
  -W --width=<chars>      Width of box chart, in characters. [default: 74]
  -H --height=<chars>     Height of box chart, in characters. [default: 20]
`

	args, err := docopt.Parse(usage, nil, true, "", true)
	if err != nil {
		panic(err)
	}

	confidence, err := strconv.ParseFloat(args["--confidence"].(string), 64)
	if err != nil {
		panic(err)
	}

	column, err := strconv.Atoi(args["--column"].(string))
	if err != nil {
		panic(err)
	}

	width, err := strconv.Atoi(args["--width"].(string))
	if err != nil {
		panic(err)
	}

	height, err := strconv.Atoi(args["--height"].(string))
	if err != nil {
		panic(err)
	}

	var delimiter rune
	switch d := args["--delimiter"]; d {
	case "t":
		delimiter = '\t'
	case "c":
		delimiter = ','
	case "s":
		delimiter = ' '
	default:
		panic(fmt.Errorf("bad delimiter: %#v", d))
	}

	controlFilename := args["<control>"].(string)
	experimentFilenames := args["<experiment>"].([]string)

	c := chart.BoxChart{}
	c.XRange.Fixed(-1, 3, 1)
	c.XRange.Category = append([]string{controlFilename}, experimentFilenames...)

	table := bytes.NewBuffer(nil)
	w := new(tabwriter.Writer)
	w.Init(table, 2, 0, 2, ' ', 0)
	fmt.Fprintln(w, "Filename\tN\tMin\tMax\tMedian\tMean\tStdDev\t")

	control, controlData, err := readFile(controlFilename, column, delimiter)
	if err != nil {
		panic(err)
	}
	c.AddSet(0, controlData, true)
	fmt.Fprintf(w,
		"%s\t%.0f\t%f\t%f\t%f\t%f\t%f\t\n",
		controlFilename,
		control.N,
		control.Min,
		control.Max,
		control.Median,
		control.Mean,
		control.StdDev,
	)

	for i, filename := range experimentFilenames {
		experimental, expData, err := readFile(filename, column, delimiter)
		if err != nil {
			panic(err)
		}

		c.AddSet(float64(i+1), expData, true)

		fmt.Fprintf(w,
			"%s\t%.0f\t%f\t%f\t%f\t%f\t%f\t\n",
			filename,
			experimental.N,
			experimental.Min,
			experimental.Max,
			experimental.Median,
			experimental.Mean,
			experimental.StdDev,
		)

		d := tinystat.Compare(control, experimental, confidence)
		if d.Significant() {
			fmt.Fprintf(w, "\tDifference at %v%% confidence!\n", confidence)
			fmt.Fprintf(w, "\t\t%10f +/- %10f\n", d.Delta, d.Error)
			fmt.Fprintf(w, "\t\t%9f%% +/- %9f%%\n", d.PctDelta, d.PctError)
			fmt.Fprintf(w, "\t\t(Student's t, pooled s = %f)\n", d.PooledStdDev)
		} else {
			fmt.Fprintf(w, "\tNo difference proven at %v%% confidence.\n", confidence)
		}
	}
	_ = w.Flush()

	if args["--no-chart"] != true {
		txt := txtg.New(width, height)
		c.Plot(txt)
		fmt.Println(txt)
	}

	fmt.Println(table.String())
}

func readFile(filename string, col int, del rune) (tinystat.Summary, []float64, error) {
	f, err := os.Open(filename)
	if err != nil {
		return tinystat.Summary{}, nil, err
	}
	defer f.Close()

	r := csv.NewReader(f)
	r.Comma = del
	records, err := r.ReadAll()
	if err != nil {
		return tinystat.Summary{}, nil, err
	}

	data := make([]float64, len(records))
	for i, s := range records {
		n, err := strconv.ParseFloat(s[col], 64)
		if err != nil {
			return tinystat.Summary{}, nil, err
		}
		data[i] = n
	}

	return tinystat.Summarize(data), data, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/moov-io/ach"
	"log"
	"os"
	"path/filepath"
	"runtime/pprof"
	"time"
)

var (
	fPath      = flag.String("fPath", "", "File Path")
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
)

// main creates an ACH File with 4 batches of SEC Code PPD.
// Each batch contains an EntryAddendaCount of 2500.
func main() {
	flag.Parse()

	path := filepath.Join(*fPath, time.Now().UTC().Format("200601021504")+".ach")
	write(path)
}

func write(path string) {
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	f, err := os.Create(path)
	if err != nil {
		fmt.Printf("%T: %s", err, err)
	}

	// To create a file
	fh := ach.NewFileHeader()
	fh.ImmediateDestination = "231380104"
	fh.ImmediateOrigin = "121042882"
	fh.FileCreationDate = time.Now()
	fh.ImmediateDestinationName = "Citadel"
	fh.ImmediateOriginName = "Wells Fargo"
	file := ach.NewFile()
	file.SetHeader(fh)

	// Create 4 Batches of SEC Code PPD
	for i := 0; i < 4; i++ {
		bh := ach.NewBatchHeader()
		bh.ServiceClassCode = 200
		bh.CompanyName = "Wells Fargo"
		bh.CompanyIdentification = "121042882"
		bh.StandardEntryClassCode = "PPD"
		bh.CompanyEntryDescription = "Trans. Description"
		bh.EffectiveEntryDate = time.Now().AddDate(0, 0, 1)
		bh.ODFIIdentification = "121042882"

		batch, _ := ach.NewBatch(bh)

		// Create Entry
		entrySeq := 0
		for i := 0; i < 1250; i++ {
			entrySeq = entrySeq + 1

			entryEntrySeq := ach.NewEntryDetail()
			entryEntrySeq.TransactionCode = 22
			entryEntrySeq.SetRDFI("231380104")
			entryEntrySeq.DFIAccountNumber = "81967038518"
			entryEntrySeq.Amount = 100000
			//entryEntrySeq.IndividualName = randomdata.FullName(randomdata.RandomGender)
			entryEntrySeq.IndividualName = "Steven Tander"
			entryEntrySeq.SetTraceNumber(bh.ODFIIdentification, entrySeq)
			//entryEntrySeq.IdentificationNumber = "#" + randomdata.RandStringRunes(13) + "#"
			entryEntrySeq.IdentificationNumber = "#83738AB#"
			entryEntrySeq.Category = ach.CategoryForward

			// Add addenda record for an entry
			addendaEntrySeq := ach.NewAddenda05()
			addendaEntrySeq.PaymentRelatedInformation = "bonus pay for amazing work on #OSS"
			entryEntrySeq.AddAddenda(addendaEntrySeq)

			// Add entries
			batch.AddEntry(entryEntrySeq)

		}

		// Create the batch.
		if err := batch.Create(); err != nil {
			fmt.Printf("%T: %s", err, err)
		}

		// Add batch to the file
		file.AddBatch(batch)
	}

	// Create the file
	if err := file.Create(); err != nil {
		fmt.Printf("%T: %s", err, err)
	}

	// Write to a file
	w := ach.NewWriter(f)
	if err := w.Write(file); err != nil {
		fmt.Printf("%T: %s", err, err)
	}
	if err := w.Flush(); err != nil {
		fmt.Println(err.Error())
	}
	if err := f.Close(); err != nil {
		fmt.Println(err.Error())
	}
}
<commit_msg>cmd/writeACH: fix compile (for now)<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/moov-io/ach"
	"log"
	"os"
	"path/filepath"
	"runtime/pprof"
	"time"
)

var (
	fPath      = flag.String("fPath", "", "File Path")
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
)

// main creates an ACH File with 4 batches of SEC Code PPD.
// Each batch contains an EntryAddendaCount of 2500.
func main() {
	flag.Parse()

	path := filepath.Join(*fPath, time.Now().UTC().Format("200601021504")+".ach")
	write(path)
}

func write(path string) {
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	f, err := os.Create(path)
	if err != nil {
		fmt.Printf("%T: %s", err, err)
	}

	// To create a file
	fh := ach.NewFileHeader()
	fh.ImmediateDestination = "231380104"
	fh.ImmediateOrigin = "121042882"
	fh.FileCreationDate = time.Now()
	fh.ImmediateDestinationName = "Citadel"
	fh.ImmediateOriginName = "Wells Fargo"
	file := ach.NewFile()
	file.SetHeader(fh)

	// Create 4 Batches of SEC Code PPD
	for i := 0; i < 4; i++ {
		bh := ach.NewBatchHeader()
		bh.ServiceClassCode = 200
		bh.CompanyName = "Wells Fargo"
		bh.CompanyIdentification = "121042882"
		bh.StandardEntryClassCode = "PPD"
		bh.CompanyEntryDescription = "Trans. Description"
		bh.EffectiveEntryDate = time.Now().AddDate(0, 0, 1)
		bh.ODFIIdentification = "121042882"

		batch, _ := ach.NewBatch(bh)

		// Create Entry
		entrySeq := 0
		for i := 0; i < 1250; i++ {
			entrySeq = entrySeq + 1

			entryEntrySeq := ach.NewEntryDetail()
			entryEntrySeq.TransactionCode = 22
			entryEntrySeq.SetRDFI("231380104")
			entryEntrySeq.DFIAccountNumber = "81967038518"
			entryEntrySeq.Amount = 100000
			//entryEntrySeq.IndividualName = randomdata.FullName(randomdata.RandomGender)
			entryEntrySeq.IndividualName = "Steven Tander"
			entryEntrySeq.SetTraceNumber(bh.ODFIIdentification, entrySeq)
			//entryEntrySeq.IdentificationNumber = "#" + randomdata.RandStringRunes(13) + "#"
			entryEntrySeq.IdentificationNumber = "#83738AB#"
			entryEntrySeq.Category = ach.CategoryForward

			// Add addenda record for an entry
			addendaEntrySeq := ach.NewAddenda05()
			addendaEntrySeq.PaymentRelatedInformation = "bonus pay for amazing work on #OSS"
			entryEntrySeq.AddAddenda(addendaEntrySeq)

			// Add entries
			batch.AddEntry(entryEntrySeq)

		}

		// Create the batch.
		if err := batch.Create(); err != nil {
			fmt.Printf("%T: %s", err, err)
		}

		// Add batch to the file
		file.AddBatch(batch)
	}

	// Create the file
	if err := file.Create(); err != nil {
		fmt.Printf("%T: %s", err, err)
	}

	// Write to a file
	w := ach.NewWriter(f)
	if err := w.Write(file); err != nil {
		fmt.Printf("%T: %s", err, err)
	}
	w.Flush()
	if err := f.Close(); err != nil {
		fmt.Println(err.Error())
	}
}
<|endoftext|>


<commit_before>package cmd_runner

import (
	"fmt"

	"github.com/centurylinkcloud/clc-go-cli/auth"
	"github.com/centurylinkcloud/clc-go-cli/command_loader"
	"github.com/centurylinkcloud/clc-go-cli/commands"
	"github.com/centurylinkcloud/clc-go-cli/config"
	"github.com/centurylinkcloud/clc-go-cli/formatter_provider"
	"github.com/centurylinkcloud/clc-go-cli/model_adjuster"
	"github.com/centurylinkcloud/clc-go-cli/model_loader"
	"github.com/centurylinkcloud/clc-go-cli/model_validator"
	"github.com/centurylinkcloud/clc-go-cli/models/datacenter"
	"github.com/centurylinkcloud/clc-go-cli/options"
	"github.com/centurylinkcloud/clc-go-cli/parser"
	"github.com/centurylinkcloud/clc-go-cli/state"
)

func Run(args []string) string {
	if len(args) == 0 {
		return help()
	}
	if len(args) == 1 && args[0] == "--help" {
		return help()
	}
	cmdArg := ""
	optionArgs := args[1:]
	if len(args) >= 2 {
		cmdArg = args[1]
		optionArgs = args[2:]
	}
	resource, err := command_loader.LoadResource(args[0])
	if err != nil {
		return err.Error()
	}
	cmd, err := command_loader.LoadCommand(resource, cmdArg)
	if err != nil {
		if cmdArg == "--help" {
			return command_loader.GetCommandsWithDescriptions(resource)
		}
		return err.Error()
	}
	if cmd.Command() == "" {
		optionArgs = args[1:]
	}
	parsedArgs, err := parser.ParseArguments(optionArgs)
	if err != nil {
		return err.Error()
	}
	yes, filename, err := options.AreToBeTakenFromFile(parsedArgs)
	if err != nil {
		return err.Error()
	}
	if yes {
		parsedArgs, err = state.ArgumentsFromJSON(filename)
		if err != nil {
			return err.Error()
		}
	}
	yes, err = options.AreToBeSaved(parsedArgs)
	if err != nil {
		return err.Error()
	}
	if yes {
		output, err := state.ArgumentsToJSON(parsedArgs, cmd.InputModel())
		if err != nil {
			return err.Error()
		}
		return output
	}
	options, err := options.ExtractFrom(parsedArgs)
	if err != nil {
		return err.Error()
	}
	if options.Help {
		return cmd.ShowHelp()
	}
	conf, err := config.LoadConfig()
	if err != nil {
		return err.Error()
	}
	if cmd.Resource() == "login" {
		return cmd.(*commands.Login).Login(options, conf)
	}
	err = model_loader.LoadModel(parsedArgs, cmd.InputModel())
	if err != nil {
		return err.Error()
	}
	datacenter.ApplyDefault(cmd.InputModel(), conf)
	err = model_validator.ValidateModel(cmd.InputModel())
	if err != nil {
		return err.Error()
	}
	err = model_adjuster.ApplyDefaultBehaviour(cmd.InputModel())
	if err != nil {
		return err.Error()
	}
	if cmd.IsOffline() {
		res, err := cmd.ExecuteOffline()
		if err != nil {
			return err.Error()
		}
		return res
	}
	cn, err := auth.AuthenticateCommand(options, conf)
	if err != nil {
		return err.Error()
	}
	err = model_adjuster.InferID(cmd.InputModel(), cn)
	if err != nil {
		return err.Error()
	}
	err = cmd.Execute(cn)
	if err != nil {
		return err.Error()
	}
	err = state.SaveLastResult(cmd.OutputModel())
	if err != nil {
		return err.Error()
	}
	f, err := formatter_provider.GetOutputFormatter(options, conf)
	if err != nil {
		return err.Error()
	}
	outputModel := cmd.OutputModel()
	if messagePtr, ok := outputModel.(*string); ok {
		return *messagePtr
	}
	detyped, err := parser.ConvertToMapOrSlice(outputModel)
	if err != nil {
		return err.Error()
	}
	if options.Filter != "" {
		filtered, err := parser.ParseFilter(detyped, options.Filter)
		if err != nil {
			return err.Error()
		} else if filtered == nil {
			return "No results found for the given filter."
		} else {
			detyped = filtered
		}
	}
	if options.Query != "" {
		queried, err := parser.ParseQuery(detyped, options.Query)
		if err != nil {
			return err.Error()
		} else if queried == nil {
			return "No results found for the given query."
		} else {
			detyped = queried
		}
	}
	output, err := f.FormatOutput(detyped)
	if err != nil {
		return err.Error()
	}
	return output
}

func help() string {
	res := "Usage: clc <resource> [<command>] [options and parameters].\n\n"
	res += "To get a list of all available commands for the given resource if any or to get a direct resource description use 'clc <resource> --help'.\n"
	res += "To get a command description and a list of all available parameters for the given command use 'clc <resource> <command> --help'.\n\n"
	res += "Available resources:\n\n"
	resources := command_loader.GetResources()
	for _, rsr := range resources {
		res += fmt.Sprintf("\t%v\n", rsr)
	}
	return res
}
<commit_msg>Make multiline string for usage help<commit_after>package cmd_runner

import (
	"fmt"

	"github.com/centurylinkcloud/clc-go-cli/auth"
	"github.com/centurylinkcloud/clc-go-cli/command_loader"
	"github.com/centurylinkcloud/clc-go-cli/commands"
	"github.com/centurylinkcloud/clc-go-cli/config"
	"github.com/centurylinkcloud/clc-go-cli/formatter_provider"
	"github.com/centurylinkcloud/clc-go-cli/model_adjuster"
	"github.com/centurylinkcloud/clc-go-cli/model_loader"
	"github.com/centurylinkcloud/clc-go-cli/model_validator"
	"github.com/centurylinkcloud/clc-go-cli/models/datacenter"
	"github.com/centurylinkcloud/clc-go-cli/options"
	"github.com/centurylinkcloud/clc-go-cli/parser"
	"github.com/centurylinkcloud/clc-go-cli/state"
)

func Run(args []string) string {
	if len(args) == 0 {
		return help()
	}
	if len(args) == 1 && args[0] == "--help" {
		return help()
	}
	cmdArg := ""
	optionArgs := args[1:]
	if len(args) >= 2 {
		cmdArg = args[1]
		optionArgs = args[2:]
	}
	resource, err := command_loader.LoadResource(args[0])
	if err != nil {
		return err.Error()
	}
	cmd, err := command_loader.LoadCommand(resource, cmdArg)
	if err != nil {
		if cmdArg == "--help" {
			return command_loader.GetCommandsWithDescriptions(resource)
		}
		return err.Error()
	}
	if cmd.Command() == "" {
		optionArgs = args[1:]
	}
	parsedArgs, err := parser.ParseArguments(optionArgs)
	if err != nil {
		return err.Error()
	}
	yes, filename, err := options.AreToBeTakenFromFile(parsedArgs)
	if err != nil {
		return err.Error()
	}
	if yes {
		parsedArgs, err = state.ArgumentsFromJSON(filename)
		if err != nil {
			return err.Error()
		}
	}
	yes, err = options.AreToBeSaved(parsedArgs)
	if err != nil {
		return err.Error()
	}
	if yes {
		output, err := state.ArgumentsToJSON(parsedArgs, cmd.InputModel())
		if err != nil {
			return err.Error()
		}
		return output
	}
	options, err := options.ExtractFrom(parsedArgs)
	if err != nil {
		return err.Error()
	}
	if options.Help {
		return cmd.ShowHelp()
	}
	conf, err := config.LoadConfig()
	if err != nil {
		return err.Error()
	}
	if cmd.Resource() == "login" {
		return cmd.(*commands.Login).Login(options, conf)
	}
	err = model_loader.LoadModel(parsedArgs, cmd.InputModel())
	if err != nil {
		return err.Error()
	}
	datacenter.ApplyDefault(cmd.InputModel(), conf)
	err = model_validator.ValidateModel(cmd.InputModel())
	if err != nil {
		return err.Error()
	}
	err = model_adjuster.ApplyDefaultBehaviour(cmd.InputModel())
	if err != nil {
		return err.Error()
	}
	if cmd.IsOffline() {
		res, err := cmd.ExecuteOffline()
		if err != nil {
			return err.Error()
		}
		return res
	}
	cn, err := auth.AuthenticateCommand(options, conf)
	if err != nil {
		return err.Error()
	}
	err = model_adjuster.InferID(cmd.InputModel(), cn)
	if err != nil {
		return err.Error()
	}
	err = cmd.Execute(cn)
	if err != nil {
		return err.Error()
	}
	err = state.SaveLastResult(cmd.OutputModel())
	if err != nil {
		return err.Error()
	}
	f, err := formatter_provider.GetOutputFormatter(options, conf)
	if err != nil {
		return err.Error()
	}
	outputModel := cmd.OutputModel()
	if messagePtr, ok := outputModel.(*string); ok {
		return *messagePtr
	}
	detyped, err := parser.ConvertToMapOrSlice(outputModel)
	if err != nil {
		return err.Error()
	}
	if options.Filter != "" {
		filtered, err := parser.ParseFilter(detyped, options.Filter)
		if err != nil {
			return err.Error()
		} else if filtered == nil {
			return "No results found for the given filter."
		} else {
			detyped = filtered
		}
	}
	if options.Query != "" {
		queried, err := parser.ParseQuery(detyped, options.Query)
		if err != nil {
			return err.Error()
		} else if queried == nil {
			return "No results found for the given query."
		} else {
			detyped = queried
		}
	}
	output, err := f.FormatOutput(detyped)
	if err != nil {
		return err.Error()
	}
	return output
}

func help() string {
	res := `Usage: clc <resource> [<command>] [options and parameters].

To get a list of all available commands for the given resource if any or to get a direct resource description use 'clc <resource> --help'.
To get a command description and a list of all available parameters for the given command use 'clc <resource> <command> --help'.

Available resources:

`
	resources := command_loader.GetResources()
	for _, rsr := range resources {
		res += fmt.Sprintf("\t%v\n", rsr)
	}
	return res
}
<|endoftext|>


<commit_before>package lb

import (
	"bytes"
	"encoding/json"
	"github.com/xlvector/dlog"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strconv"
	"time"
)

type DownStream struct {
	up        *UpstreamConfig
	used      bool
	sid       string
	startTime time.Time
}

type ServiceHandler struct {
	srv           *ServiceConfig
	freeUpstreams chan *UpstreamConfig
	cache         map[string]*DownStream
	ticker        *time.Ticker
}

func NewServiceHandler(srv *ServiceConfig) *ServiceHandler {
	ret := &ServiceHandler{
		srv:           srv,
		freeUpstreams: make(chan *UpstreamConfig, 100),
		cache:         make(map[string]*DownStream),
		ticker:        time.NewTicker(time.Second * 3),
	}
	for _, up := range srv.Upstreams {
		dlog.Println("append up: ", up.Addr)
		ret.freeUpstreams <- up
	}
	go func() {
		for t := range ret.ticker.C {
			dlog.Warn("close timeout upstream: %d", t.Unix())
			del := []*DownStream{}
			for _, d := range ret.cache {
				if d.used && time.Now().Sub(d.startTime).Seconds() > 300 {
					d.used = false
					del = append(del, d)
					continue
				}
				if !d.used && time.Now().Sub(d.startTime).Seconds() > 120 {
					del = append(del, d)
					continue
				}
			}

			for _, d := range del {
				dlog.Println("free upstream: ", d.up.Addr)
				delete(ret.cache, d.sid)
				ret.freeUpstreams <- d.up
			}
		}
	}()
	return ret
}

func randString() string {
	str := strconv.FormatInt(time.Now().UnixNano(), 10)
	str += "_"
	str += strconv.FormatInt(rand.Int63(), 10)
	return str
}

func (p *ServiceHandler) closeDownstream(down *DownStream) {
	delete(p.cache, down.sid)
	p.freeUpstreams <- down.up
}

func (p *ServiceHandler) createDownstream(req *http.Request) *DownStream {
	if sid, err := req.Cookie("lbc_session_id"); err == nil && sid != nil {
		dlog.Println("recv sid: ", sid.Value)
		if down, ok := p.cache[sid.Value]; ok {
			if isClose, err2 := req.Cookie("lbc_close"); err2 == nil && isClose != nil {
				p.closeDownstream(down)
				return nil
			}
			return down
		}
		dlog.Println("fail to find sid: ", sid.Value, p.cache)
		return nil
	} else {
		down := &DownStream{
			up:        <-p.freeUpstreams,
			sid:       randString(),
			startTime: time.Now(),
		}
		dlog.Println("use up: ", down.up.Addr)
		p.cache[down.sid] = down
		return down
	}
}

type LoadBalancer struct {
	services map[string]*ServiceHandler
	client   *http.Client
}

func NewLoadBalancer(conf string) *LoadBalancer {
	b, err := ioutil.ReadFile(conf)
	if err != nil {
		dlog.Error("fail to read config: %v", err)
		return nil
	}
	srvs := make(map[string]*ServiceConfig)
	err = json.Unmarshal(b, &srvs)
	if err != nil {
		dlog.Error("fail to decode json: %v", err)
	}

	ret := &LoadBalancer{
		services: make(map[string]*ServiceHandler),
		client:   &http.Client{},
	}

	for k, v := range srvs {
		ret.services[k] = NewServiceHandler(v)
	}
	return ret
}

func (p *LoadBalancer) printError(rw http.ResponseWriter, down *DownStream, msg string) {
	dlog.Warn(msg)
	rw.WriteHeader(500)
	rw.Write([]byte(msg))
	if down != nil {
		down.used = false
	}
}

func (p *LoadBalancer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	srv, ok := p.services[req.Host]
	if !ok {
		p.printError(rw, nil, "fail to get service: "+req.Host)
		return
	}

	down := srv.createDownstream(req)
	if nil == down {
		dlog.Warn("fail to find downstream")
		rw.WriteHeader(500)
		p.printError(rw, nil, "fail to find downstream")
		return
	}
	down.used = true

	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		p.printError(rw, down, "fail to read body: "+err.Error())
		return
	}
	req2, err := http.NewRequest(req.Method, "http://"+down.up.Addr+req.RequestURI, bytes.NewReader(body))
	dlog.Println(req2.Method, " ", req2.URL.String())
	if err != nil {
		p.printError(rw, down, "new req2 fail: "+err.Error())
		return
	}
	resp, err := p.client.Do(req2)
	if err != nil {
		p.printError(rw, down, "call upstream failed: "+err.Error())
		return
	}

	if resp.Body == nil {
		p.printError(rw, down, "upstream resp body is nil")
		return
	}

	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		p.printError(rw, down, "read upstream body fail: "+err.Error())
		return
	}
	http.SetCookie(rw, &http.Cookie{
		Name:  "lbc_session_id",
		Value: down.sid,
	})
	rw.WriteHeader(200)
	rw.Write(b)
	down.used = false
}
<commit_msg>reset start time<commit_after>package lb

import (
	"bytes"
	"encoding/json"
	"github.com/xlvector/dlog"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strconv"
	"time"
)

type DownStream struct {
	up        *UpstreamConfig
	used      bool
	sid       string
	startTime time.Time
}

type ServiceHandler struct {
	srv           *ServiceConfig
	freeUpstreams chan *UpstreamConfig
	cache         map[string]*DownStream
	ticker        *time.Ticker
}

func NewServiceHandler(srv *ServiceConfig) *ServiceHandler {
	ret := &ServiceHandler{
		srv:           srv,
		freeUpstreams: make(chan *UpstreamConfig, 100),
		cache:         make(map[string]*DownStream),
		ticker:        time.NewTicker(time.Second * 3),
	}
	for _, up := range srv.Upstreams {
		dlog.Println("append up: ", up.Addr)
		ret.freeUpstreams <- up
	}
	go func() {
		for t := range ret.ticker.C {
			dlog.Warn("close timeout upstream: %d", t.Unix())
			del := []*DownStream{}
			for _, d := range ret.cache {
				if d.used && time.Now().Sub(d.startTime).Seconds() > 300 {
					d.used = false
					del = append(del, d)
					continue
				}
				if !d.used && time.Now().Sub(d.startTime).Seconds() > 120 {
					del = append(del, d)
					continue
				}
			}

			for _, d := range del {
				dlog.Println("free upstream: ", d.up.Addr)
				delete(ret.cache, d.sid)
				ret.freeUpstreams <- d.up
			}
		}
	}()
	return ret
}

func randString() string {
	str := strconv.FormatInt(time.Now().UnixNano(), 10)
	str += "_"
	str += strconv.FormatInt(rand.Int63(), 10)
	return str
}

func (p *ServiceHandler) closeDownstream(down *DownStream) {
	delete(p.cache, down.sid)
	p.freeUpstreams <- down.up
}

func (p *ServiceHandler) createDownstream(req *http.Request) *DownStream {
	if sid, err := req.Cookie("lbc_session_id"); err == nil && sid != nil {
		dlog.Println("recv sid: ", sid.Value)
		if down, ok := p.cache[sid.Value]; ok {
			if isClose, err2 := req.Cookie("lbc_close"); err2 == nil && isClose != nil {
				p.closeDownstream(down)
				return nil
			}
			down.startTime = time.Now()
			return down
		}
		dlog.Println("fail to find sid: ", sid.Value, p.cache)
		return nil
	} else {
		down := &DownStream{
			up:        <-p.freeUpstreams,
			sid:       randString(),
			startTime: time.Now(),
		}
		dlog.Println("use up: ", down.up.Addr)
		p.cache[down.sid] = down
		return down
	}
}

type LoadBalancer struct {
	services map[string]*ServiceHandler
	client   *http.Client
}

func NewLoadBalancer(conf string) *LoadBalancer {
	b, err := ioutil.ReadFile(conf)
	if err != nil {
		dlog.Error("fail to read config: %v", err)
		return nil
	}
	srvs := make(map[string]*ServiceConfig)
	err = json.Unmarshal(b, &srvs)
	if err != nil {
		dlog.Error("fail to decode json: %v", err)
	}

	ret := &LoadBalancer{
		services: make(map[string]*ServiceHandler),
		client:   &http.Client{},
	}

	for k, v := range srvs {
		ret.services[k] = NewServiceHandler(v)
	}
	return ret
}

func (p *LoadBalancer) printError(rw http.ResponseWriter, down *DownStream, msg string) {
	dlog.Warn(msg)
	rw.WriteHeader(500)
	rw.Write([]byte(msg))
	if down != nil {
		down.used = false
	}
}

func (p *LoadBalancer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	srv, ok := p.services[req.Host]
	if !ok {
		p.printError(rw, nil, "fail to get service: "+req.Host)
		return
	}

	down := srv.createDownstream(req)
	if nil == down {
		dlog.Warn("fail to find downstream")
		rw.WriteHeader(500)
		p.printError(rw, nil, "fail to find downstream")
		return
	}
	down.used = true

	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		p.printError(rw, down, "fail to read body: "+err.Error())
		return
	}
	req2, err := http.NewRequest(req.Method, "http://"+down.up.Addr+req.RequestURI, bytes.NewReader(body))
	dlog.Println(req2.Method, " ", req2.URL.String())
	if err != nil {
		p.printError(rw, down, "new req2 fail: "+err.Error())
		return
	}
	resp, err := p.client.Do(req2)
	if err != nil {
		p.printError(rw, down, "call upstream failed: "+err.Error())
		return
	}

	if resp.Body == nil {
		p.printError(rw, down, "upstream resp body is nil")
		return
	}

	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		p.printError(rw, down, "read upstream body fail: "+err.Error())
		return
	}
	http.SetCookie(rw, &http.Cookie{
		Name:  "lbc_session_id",
		Value: down.sid,
	})
	rw.WriteHeader(200)
	rw.Write(b)
	down.used = false
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package ticketbuyer

import (
	"github.com/decred/dcrwallet/wallet"
)

// PurchaseManager is the main handler of websocket notifications to
// pass to the purchaser and internal quit notifications.
type PurchaseManager struct {
	w         *wallet.Wallet
	purchaser *TicketPurchaser
	ntfnChan  <-chan *wallet.TransactionNotifications
	quit      chan struct{}
}

// NewPurchaseManager creates a new PurchaseManager.
func NewPurchaseManager(w *wallet.Wallet, purchaser *TicketPurchaser,
	ntfnChan <-chan *wallet.TransactionNotifications,
	quit chan struct{}) *PurchaseManager {
	return &PurchaseManager{
		w:         w,
		purchaser: purchaser,
		ntfnChan:  ntfnChan,
		quit:      quit,
	}
}

// purchase purchases the tickets for the given block height.
func (p *PurchaseManager) purchase(height int64) {
	log.Infof("Block height %v connected", height)
	purchaseInfo, err := p.purchaser.Purchase(height)
	if err != nil {
		log.Errorf("Failed to purchase tickets this round: %v", err)
		return
	}
	log.Debugf("Purchased %v tickets this round", purchaseInfo.Purchased)
}

// NotificationHandler handles notifications, which trigger ticket purchases.
func (p *PurchaseManager) NotificationHandler() {
out:
	for {
		select {
		case v := <-p.ntfnChan:
			if v != nil {
				for _, block := range v.AttachedBlocks {
					go p.purchase(int64(block.Height))
				}
			}
		case <-p.quit:
			break out
		}
	}
}
<commit_msg>Serialize calls to ticketbuyer Purchase. (#472)<commit_after>// Copyright (c) 2016 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package ticketbuyer

import (
	"github.com/decred/dcrwallet/wallet"
)

// PurchaseManager is the main handler of websocket notifications to
// pass to the purchaser and internal quit notifications.
type PurchaseManager struct {
	w         *wallet.Wallet
	purchaser *TicketPurchaser
	ntfnChan  <-chan *wallet.TransactionNotifications
	quit      chan struct{}
}

// NewPurchaseManager creates a new PurchaseManager.
func NewPurchaseManager(w *wallet.Wallet, purchaser *TicketPurchaser,
	ntfnChan <-chan *wallet.TransactionNotifications,
	quit chan struct{}) *PurchaseManager {
	return &PurchaseManager{
		w:         w,
		purchaser: purchaser,
		ntfnChan:  ntfnChan,
		quit:      quit,
	}
}

// purchase purchases the tickets for the given block height.
func (p *PurchaseManager) purchase(height int64) {
	log.Infof("Block height %v connected", height)
	purchaseInfo, err := p.purchaser.Purchase(height)
	if err != nil {
		log.Errorf("Failed to purchase tickets this round: %v", err)
		return
	}
	log.Debugf("Purchased %v tickets this round", purchaseInfo.Purchased)
}

// NotificationHandler handles notifications, which trigger ticket purchases.
func (p *PurchaseManager) NotificationHandler() {
	s1, s2 := make(chan struct{}), make(chan struct{})
	close(s1) // unblock first worker
out:
	for {
		select {
		case v := <-p.ntfnChan:
			go func(s1, s2 chan struct{}) {
				<-s1 // wait for previous worker to finish
				for _, block := range v.AttachedBlocks {
					p.purchase(int64(block.Height))
				}
				close(s2) // unblock next worker
			}(s1, s2)
			s1, s2 = s2, make(chan struct{})
		case <-p.quit:
			break out
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Steven Oud. All rights reserved.
// Use of this source code is governed by a MIT-style license that can be found
// in the LICENSE file.

package mathcat

import (
	"errors"
	"unicode"
)

const eol rune = -1

// lexer holds the lexer's state while scanning an expression. If any error
// occurs, the scanning stops immediatly and returns the error.
type lexer struct {
	expr   []rune   // the input expression
	ch     rune     // current character
	pos    int      // current character position
	start  int      // current read offset
	tokens []*Token // tokenized lexemes
}

// https://en.wikipedia.org/wiki/Whitespace_character
var whitespaceChars = []rune{
	'\u0009', '\u000A', '\u000B', '\u000C', '\u000D', '\u0020', '\u0085',
	'\u00A0', '\u1680', '\u2000', '\u2001', '\u2002', '\u2003', '\u2004',
	'\u2005', '\u2006', '\u2007', '\u2008', '\u2009', '\u200A', '\u2028',
	'\u2029', '\u202F', '\u205F', '\u3000', '\u180E', '\u200B', '\u200C',
	'\u200D', '\u2060', '\uFEFF',
}

func isIdent(c rune) bool {
	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= 0x80 && unicode.IsLetter(c))
}

func isNumber(c rune) bool {
	return (c >= '0' && c <= '9') || c == '.'
}

func isHex(c rune) bool {
	return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}

func isBinary(c rune) bool {
	return c == '0' || c == '1'
}

func isOctal(c rune) bool {
	return c >= '0' && c <= '7'
}

func isWhitespace(c rune) bool {
	if c == '\t' || c == ' ' || c == '\r' {
		return true
	}

	for _, v := range whitespaceChars {
		if v == c {
			return true
		}
	}

	return false
}

// Lex starts lexing an expression. We keep reading until EOL is found, which
// we add because we need a padding of 1 to always be able to peek().
//
// Returns the generated tokens and any error found.
func Lex(expr string) ([]*Token, error) {
	l := &lexer{
		expr:  append([]rune(expr), eol), // add eol as padding
		pos:   0,
		start: 0,
	}

	return l.lex()
}

func (l *lexer) lex() ([]*Token, error) {
loop:
	for l.ch != eol {
		l.start = l.pos

		l.eat()

		switch {
		case isIdent(l.ch):
			l.readIdent()
		case isNumber(l.ch):
			l.readNumber()
		case isWhitespace(l.ch):
			l.skipWhitespace()
		default:
			switch l.ch {
			case '+':
				l.switchEq(ADD, ADD_EQ)
			case '-':
				if l.isNegation() && l.peek() != '=' {
					l.emit(UNARY_MIN)
					break
				}
				l.switchEq(SUB, SUB_EQ)
			case '/':
				l.switchEq(DIV, DIV_EQ)
			case '*':
				if l.peek() == '*' {
					l.eat()
					l.switchEq(POW, POW_EQ)
				} else {
					l.switchEq(MUL, MUL_EQ)
				}
			case '%':
				l.switchEq(REM, REM_EQ)
			case '&':
				l.switchEq(AND, AND_EQ)
			case '|':
				l.switchEq(OR, OR_EQ)
			case '^':
				l.switchEq(XOR, XOR_EQ)
			case '<':
				if l.peek() == '<' {
					l.eat()
					l.switchEq(LSH, LSH_EQ)
				} else {
					l.switchEq(LT, LT_EQ)
				}
			case '>':
				if l.peek() == '>' {
					l.eat()
					l.switchEq(RSH, RSH_EQ)
				} else {
					l.switchEq(GT, GT_EQ)
				}
			case '~':
				l.emit(NOT)
			case '=':
				l.switchEq(EQ, EQ_EQ)
			case '!':
				if l.peek() != '=' {
					return nil, errors.New("Invalid operation " + string(l.ch))
				}
				l.eat()
				l.emit(BANG_EQ)
			case '(':
				l.emit(LPAREN)
			case ')':
				l.emit(RPAREN)
			case ',':
				l.emit(COMMA)
			case '#':
				// Comment, stop scanning for tokens
				l.emit(EOL)
				break loop
			case eol:
				l.emit(EOL)
			default:
				l.emit(ILLEGAL)
				return nil, errors.New("Invalid token " + string(l.ch))
			}
		}
	}

	return l.tokens, nil
}

func (l *lexer) peek() rune {
	return l.expr[l.pos]
}

func (l *lexer) prev() *Token {
	return l.tokens[len(l.tokens)-1]
}

func (l *lexer) eat() rune {
	l.ch = l.peek()
	l.pos++
	return l.ch
}

func (l *lexer) emit(toktype tokenType) {
	l.tokens = append(l.tokens, newToken(toktype, string(l.expr[l.start:l.pos]), l.start))
}

func (l *lexer) skipWhitespace() {
	for isWhitespace(l.peek()) {
		l.eat()
	}
}

func (l *lexer) readIdent() {
	for isIdent(l.peek()) || isNumber(l.peek()) {
		l.eat()
	}

	l.emit(IDENT)
}

func (l *lexer) readNumber() {
	// Hex literals
	if l.ch == '0' && (l.peek() == 'x' || l.peek() == 'X') {
		l.eat()

		for isHex(l.peek()) {
			l.eat()
		}

		l.emit(HEX)
		return
	}

	// Binary literals
	if l.ch == '0' && (l.peek() == 'b' || l.peek() == 'B') {
		l.eat()

		for isBinary(l.peek()) {
			l.eat()
		}

		l.emit(BINARY)
		return
	}

	// Octal literals
	if l.ch == '0' && (l.peek() == 'o' || l.peek() == 'O') {
		l.eat()

		for isOctal(l.peek()) {
			l.eat()
		}

		l.emit(OCTAL)
		return
	}

	// Normal literals
	for isNumber(l.peek()) || l.peek() == 'e' || l.peek() == 'E' {
		l.eat()
		if (l.ch == 'e' || l.ch == 'E') && l.peek() == '-' {
			l.eat()
		}
	}

	l.emit(NUMBER)
}

func (l *lexer) isNegation() bool {
	return l.tokens == nil || l.prev().Type == LPAREN || l.prev().IsOperator()
}

func (l *lexer) switchEq(tokA, tokB tokenType) {
	if l.peek() == '=' {
		l.eat()
		l.emit(tokB)
	} else {
		l.emit(tokA)
	}
}
<commit_msg>Move all whitespace characters into slice<commit_after>// Copyright 2016 Steven Oud. All rights reserved.
// Use of this source code is governed by a MIT-style license that can be found
// in the LICENSE file.

package mathcat

import (
	"errors"
	"unicode"
)

const eol rune = -1

// lexer holds the lexer's state while scanning an expression. If any error
// occurs, the scanning stops immediatly and returns the error.
type lexer struct {
	expr   []rune   // the input expression
	ch     rune     // current character
	pos    int      // current character position
	start  int      // current read offset
	tokens []*Token // tokenized lexemes
}

// https://en.wikipedia.org/wiki/Whitespace_character
var whitespaceChars = []rune{
	'\t', ' ', '\r', '\n', '\u0009', '\u000A', '\u000B', '\u000C', '\u000D',
	'\u0020', '\u0085', '\u00A0', '\u1680', '\u2000', '\u2001', '\u2002',
	'\u2003', '\u2004', '\u2005', '\u2006', '\u2007', '\u2008', '\u2009',
	'\u200A', '\u2028', '\u2029', '\u202F', '\u205F', '\u3000', '\u180E',
	'\u200B', '\u200C', '\u200D', '\u2060', '\uFEFF',
}

func isIdent(c rune) bool {
	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= 0x80 && unicode.IsLetter(c))
}

func isNumber(c rune) bool {
	return (c >= '0' && c <= '9') || c == '.'
}

func isHex(c rune) bool {
	return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}

func isBinary(c rune) bool {
	return c == '0' || c == '1'
}

func isOctal(c rune) bool {
	return c >= '0' && c <= '7'
}

func isWhitespace(c rune) bool {
	for _, v := range whitespaceChars {
		if v == c {
			return true
		}
	}

	return false
}

// Lex starts lexing an expression. We keep reading until EOL is found, which
// we add because we need a padding of 1 to always be able to peek().
//
// Returns the generated tokens and any error found.
func Lex(expr string) ([]*Token, error) {
	l := &lexer{
		expr:  append([]rune(expr), eol), // add eol as padding
		pos:   0,
		start: 0,
	}

	return l.lex()
}

func (l *lexer) lex() ([]*Token, error) {
loop:
	for l.ch != eol {
		l.start = l.pos

		l.eat()

		switch {
		case isIdent(l.ch):
			l.readIdent()
		case isNumber(l.ch):
			l.readNumber()
		case isWhitespace(l.ch):
			l.skipWhitespace()
		default:
			switch l.ch {
			case '+':
				l.switchEq(ADD, ADD_EQ)
			case '-':
				if l.isNegation() && l.peek() != '=' {
					l.emit(UNARY_MIN)
					break
				}
				l.switchEq(SUB, SUB_EQ)
			case '/':
				l.switchEq(DIV, DIV_EQ)
			case '*':
				if l.peek() == '*' {
					l.eat()
					l.switchEq(POW, POW_EQ)
				} else {
					l.switchEq(MUL, MUL_EQ)
				}
			case '%':
				l.switchEq(REM, REM_EQ)
			case '&':
				l.switchEq(AND, AND_EQ)
			case '|':
				l.switchEq(OR, OR_EQ)
			case '^':
				l.switchEq(XOR, XOR_EQ)
			case '<':
				if l.peek() == '<' {
					l.eat()
					l.switchEq(LSH, LSH_EQ)
				} else {
					l.switchEq(LT, LT_EQ)
				}
			case '>':
				if l.peek() == '>' {
					l.eat()
					l.switchEq(RSH, RSH_EQ)
				} else {
					l.switchEq(GT, GT_EQ)
				}
			case '~':
				l.emit(NOT)
			case '=':
				l.switchEq(EQ, EQ_EQ)
			case '!':
				if l.peek() != '=' {
					return nil, errors.New("Invalid operation " + string(l.ch))
				}
				l.eat()
				l.emit(BANG_EQ)
			case '(':
				l.emit(LPAREN)
			case ')':
				l.emit(RPAREN)
			case ',':
				l.emit(COMMA)
			case '#':
				// Comment, stop scanning for tokens
				l.emit(EOL)
				break loop
			case eol:
				l.emit(EOL)
			default:
				l.emit(ILLEGAL)
				return nil, errors.New("Invalid token " + string(l.ch))
			}
		}
	}

	return l.tokens, nil
}

func (l *lexer) peek() rune {
	return l.expr[l.pos]
}

func (l *lexer) prev() *Token {
	return l.tokens[len(l.tokens)-1]
}

func (l *lexer) eat() rune {
	l.ch = l.peek()
	l.pos++
	return l.ch
}

func (l *lexer) emit(toktype tokenType) {
	l.tokens = append(l.tokens, newToken(toktype, string(l.expr[l.start:l.pos]), l.start))
}

func (l *lexer) skipWhitespace() {
	for isWhitespace(l.peek()) {
		l.eat()
	}
}

func (l *lexer) readIdent() {
	for isIdent(l.peek()) || isNumber(l.peek()) {
		l.eat()
	}

	l.emit(IDENT)
}

func (l *lexer) readNumber() {
	// Hex literals
	if l.ch == '0' && (l.peek() == 'x' || l.peek() == 'X') {
		l.eat()

		for isHex(l.peek()) {
			l.eat()
		}

		l.emit(HEX)
		return
	}

	// Binary literals
	if l.ch == '0' && (l.peek() == 'b' || l.peek() == 'B') {
		l.eat()

		for isBinary(l.peek()) {
			l.eat()
		}

		l.emit(BINARY)
		return
	}

	// Octal literals
	if l.ch == '0' && (l.peek() == 'o' || l.peek() == 'O') {
		l.eat()

		for isOctal(l.peek()) {
			l.eat()
		}

		l.emit(OCTAL)
		return
	}

	// Normal literals
	for isNumber(l.peek()) || l.peek() == 'e' || l.peek() == 'E' {
		l.eat()
		if (l.ch == 'e' || l.ch == 'E') && l.peek() == '-' {
			l.eat()
		}
	}

	l.emit(NUMBER)
}

func (l *lexer) isNegation() bool {
	return l.tokens == nil || l.prev().Type == LPAREN || l.prev().IsOperator()
}

func (l *lexer) switchEq(tokA, tokB tokenType) {
	if l.peek() == '=' {
		l.eat()
		l.emit(tokB)
	} else {
		l.emit(tokA)
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Ben Morgan. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.

// The code and API of this file is based on text/template/parse/lex.go
// from the Go standard library:
//
// Copyright (c) 2012 The Go Authors. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//    * Redistributions of source code must retain the above copyright
//      notice, this list of conditions and the following disclaimer.
//    * Redistributions in binary form must reproduce the above
//      copyright notice, this list of conditions and the following
//      disclaimer in the documentation and/or other materials provided
//      with the distribution.
//    * Neither the name of Google Inc. nor the names of its
//      contributors may be used to endorse or promote products derived
//      from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// Package lex provides a base lexer to help you implement your own.
//
// Feel free to copy this into your library (according to the license)
// as well as use it as a library. Both methods are possible.
//
// Code and API based off standard library text/template/parse.
package lex

import (
	"fmt"
	"strings"
	"unicode/utf8"
)

// EOF is returned by Lexer.Next upon reaching end-of-file.
const EOF = -1

// A Type is the type of a token.
//
// In this package, only several types are predefined. The rest you
// can set yourself like so:
//
//  const (
//      TypeSpace = (1+lex.TypeEOF)+iota // continue where lex left off
//      TypeNumber
//      TypeIdent
//      ...
//  )
//
type Type int

const (
	TypeError Type = iota // string is error text
	TypeEOF               // end-of-file, last reserved type
)

type Token struct {
	Type
	Pos   int
	Value string
}

type StateFn func(*Lexer) StateFn

type Lexer struct {
	name    string
	input   string
	width   int
	base    int
	pos     int
	lastPos int
	tokens  chan Token
}

// New creates a new Lexer and returns it.
//
// Before calling NextToken, it should be run in a separate goroutine:
//
//  l := lex.New(name, input)
//  go l.Run(sf)
//  ...
//  t := l.NextToken()
func New(name, input string) *Lexer {
	l := &Lexer{
		name:   name,
		input:  input,
		tokens: make(chan Token),
	}
	return l
}

// Lex creates a new Lexer and starts running it with sf.
func Lex(name, input string, sf StateFn) *Lexer {
	l := New(name, input)
	go l.Run(sf)
	return l
}

// Run starts the lexer with the given StateFn.
// After receiving a nil StateFn, it closes the tokens channel.
func (l *Lexer) Run(fn StateFn) {
	for state := fn; state != nil; {
		state = state(l)
	}
	close(l.tokens)
}

// NextToken returns the next token from the input.
// Called by the parser, not in the lexing goroutine.
//
// Note: if l.Run has not been called, NextToken will block.
func (l *Lexer) NextToken() Token {
	t := <-l.tokens
	l.lastPos = t.Pos
	return t
}

// Drain drains the output so the lexing goroutine will exit.
// Called by the parser, not in the lexing goroutine.
func (l *Lexer) Drain() {
	for range l.tokens {
	}
}

// LineNumber reports the line of the last token returned by NextToken.
func (l *Lexer) LineNumber() int {
	return 1 + strings.Count(l.input[:l.lastPos], "\n")
}

// ColumnNumber reports the column of the last token returned by NextToken.
func (l *Lexer) ColumnNumber() int {
	code := l.input[:l.lastPos]
	if i := strings.LastIndex(code, "\n"); i >= 0 {
		return l.lastPos - i
	} else {
		return 1 + len(code)
	}
}

// Name returns the name of the input.
func (l *Lexer) Name() string { return l.name }

// Value returns the current token value, essentially the part of input
// from l.base to l.pos.
func (l *Lexer) Value() string { return l.input[l.base:l.pos] }

// Len returns the size of the current read token.
func (l *Lexer) Len() int { return l.base - l.pos }

// Inc increments the position by n.
func (l *Lexer) Inc(n int) { l.pos += n }

// Dec decrements the position by n.
func (l *Lexer) Dec(n int) { l.pos -= n }

// Pos returns the current position in the input.
func (l *Lexer) Pos() int { return l.pos }

// Input returns a slice of the current position plus n.
func (l *Lexer) Input(n int) string {
	return l.input[l.pos+n:]
}

// Next returns the next rune in the input.
// If there is no more input left to read, EOF is returned.
func (l *Lexer) Next() rune {
	if int(l.pos) >= len(l.input) {
		l.width = 0
		return EOF
	}
	r, w := utf8.DecodeRuneInString(l.input[l.pos:])
	l.width = w
	l.pos += l.width
	return r
}

// Peek returns but does not consume the next rune in the input.
func (l *Lexer) Peek() rune {
	r := l.Next()
	l.Backup()
	return r
}

// Emit passes a token back to the client.
func (l *Lexer) Emit(t Type) {
	l.tokens <- Token{t, l.base, l.input[l.base:l.pos]}
	l.base = l.pos
}

// Ignore skips over the pending input before this point.
func (l *Lexer) Ignore() {
	l.base = l.pos
}

// Backup steps back one rune. Can only be called once per call of Next.
func (l *Lexer) Backup() {
	l.pos -= l.width
}

// Consume tries to consume exactly the string s.
func (l *Lexer) Consume(s string) bool {
	ok := l.HasPrefix(s)
	if ok {
		l.pos += len(s)
	}
	return ok
}

// Accept consumes the next rune if it is from the valid set.
func (l *Lexer) Accept(valid string) bool {
	if strings.IndexRune(valid, l.Next()) >= 0 {
		return true
	}
	l.Backup()
	return false
}

// AcceptRun consumes a run of runes from the valid set.
// The number of bytes advanced is returned.
func (l *Lexer) AcceptRun(valid string) int {
	var n int
	for strings.IndexRune(valid, l.Next()) >= 0 {
		n += l.width
	}
	l.Backup()
	return n
}

// AcceptFunc consumes the next rune if f returns true.
func (l *Lexer) AcceptFunc(f func(r rune) bool) bool {
	if f(l.Next()) {
		return true
	}
	l.Backup()
	return false
}

// AcceptFunc consumes a run of runes as long as f returns true.
// The number of bytes advanced is returned.
func (l *Lexer) AcceptFuncRun(f func(r rune) bool) int {
	var n int
	for f(l.Next()) {
		n += l.width
	}
	l.Backup()
	return n
}

// AcceptBut consumes a rune if it is not from the invalid set.
func (l *Lexer) AcceptBut(invalid string) bool {
	if strings.IndexRune(invalid, l.Next()) < 0 {
		return true
	}
	l.Backup()
	return false
}

// AcceptButRun consumes runes as long as they are not in the invalid set.
// The number of bytes advanced is returned.
func (l *Lexer) AcceptButRun(invalid string) int {
	var n int
	for strings.IndexRune(invalid, l.Next()) < 0 {
		n += l.width
	}
	l.Backup()
	return n
}

// HasPrefix returns true if the input from the current position
// has the prefix s. It does not consume the prefix.
func (l *Lexer) HasPrefix(s string) bool {
	return strings.HasPrefix(l.input[l.pos:], s)
}

// HasPrefixAfter returns true if the input from the current position
// plus after bytes has the prefix s. It does not consume the prefix.
func (l *Lexer) HasPrefixAfter(after int, s string) bool {
	return strings.HasPrefix(l.input[l.pos+after:], s)
}

// Errorf returns an error token and terminates the scan by passing
// back a nil pointer that will be the next state, terminating l.NextToken.
func (l *Lexer) Errorf(format string, args ...interface{}) StateFn {
	l.tokens <- Token{TypeError, l.base, fmt.Sprintf(format, args...)}
	return nil
}
<commit_msg>Bugfix: Len returned wrong value<commit_after>// Copyright (c) 2015, Ben Morgan. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.

// The code and API of this file is based on text/template/parse/lex.go
// from the Go standard library:
//
// Copyright (c) 2012 The Go Authors. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//    * Redistributions of source code must retain the above copyright
//      notice, this list of conditions and the following disclaimer.
//    * Redistributions in binary form must reproduce the above
//      copyright notice, this list of conditions and the following
//      disclaimer in the documentation and/or other materials provided
//      with the distribution.
//    * Neither the name of Google Inc. nor the names of its
//      contributors may be used to endorse or promote products derived
//      from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// Package lex provides a base lexer to help you implement your own.
//
// Feel free to copy this into your library (according to the license)
// as well as use it as a library. Both methods are possible.
//
// Code and API based off standard library text/template/parse.
package lex

import (
	"fmt"
	"strings"
	"unicode/utf8"
)

// EOF is returned by Lexer.Next upon reaching end-of-file.
const EOF = -1

// A Type is the type of a token.
//
// In this package, only several types are predefined. The rest you
// can set yourself like so:
//
//  const (
//      TypeSpace = (1+lex.TypeEOF)+iota // continue where lex left off
//      TypeNumber
//      TypeIdent
//      ...
//  )
//
type Type int

const (
	TypeError Type = iota // string is error text
	TypeEOF               // end-of-file, last reserved type
)

type Token struct {
	Type
	Pos   int
	Value string
}

type StateFn func(*Lexer) StateFn

type Lexer struct {
	name    string
	input   string
	width   int
	base    int
	pos     int
	lastPos int
	tokens  chan Token
}

// New creates a new Lexer and returns it.
//
// Before calling NextToken, it should be run in a separate goroutine:
//
//  l := lex.New(name, input)
//  go l.Run(sf)
//  ...
//  t := l.NextToken()
func New(name, input string) *Lexer {
	l := &Lexer{
		name:   name,
		input:  input,
		tokens: make(chan Token),
	}
	return l
}

// Lex creates a new Lexer and starts running it with sf.
func Lex(name, input string, sf StateFn) *Lexer {
	l := New(name, input)
	go l.Run(sf)
	return l
}

// Run starts the lexer with the given StateFn.
// After receiving a nil StateFn, it closes the tokens channel.
func (l *Lexer) Run(fn StateFn) {
	for state := fn; state != nil; {
		state = state(l)
	}
	close(l.tokens)
}

// NextToken returns the next token from the input.
// Called by the parser, not in the lexing goroutine.
//
// Note: if l.Run has not been called, NextToken will block.
func (l *Lexer) NextToken() Token {
	t := <-l.tokens
	l.lastPos = t.Pos
	return t
}

// Drain drains the output so the lexing goroutine will exit.
// Called by the parser, not in the lexing goroutine.
func (l *Lexer) Drain() {
	for range l.tokens {
	}
}

// LineNumber reports the line of the last token returned by NextToken.
func (l *Lexer) LineNumber() int {
	return 1 + strings.Count(l.input[:l.lastPos], "\n")
}

// ColumnNumber reports the column of the last token returned by NextToken.
func (l *Lexer) ColumnNumber() int {
	code := l.input[:l.lastPos]
	if i := strings.LastIndex(code, "\n"); i >= 0 {
		return l.lastPos - i
	} else {
		return 1 + len(code)
	}
}

// Name returns the name of the input.
func (l *Lexer) Name() string { return l.name }

// Value returns the current token value, essentially the part of input
// from l.base to l.pos.
func (l *Lexer) Value() string { return l.input[l.base:l.pos] }

// Len returns the size of the current read token.
func (l *Lexer) Len() int { return l.pos - l.base }

// Inc increments the position by n.
func (l *Lexer) Inc(n int) { l.pos += n }

// Dec decrements the position by n.
func (l *Lexer) Dec(n int) { l.pos -= n }

// Pos returns the current position in the input.
func (l *Lexer) Pos() int { return l.pos }

// Input returns a slice of the current position plus n.
func (l *Lexer) Input(n int) string {
	return l.input[l.pos+n:]
}

// Next returns the next rune in the input.
// If there is no more input left to read, EOF is returned.
func (l *Lexer) Next() rune {
	if int(l.pos) >= len(l.input) {
		l.width = 0
		return EOF
	}
	r, w := utf8.DecodeRuneInString(l.input[l.pos:])
	l.width = w
	l.pos += l.width
	return r
}

// Peek returns but does not consume the next rune in the input.
func (l *Lexer) Peek() rune {
	r := l.Next()
	l.Backup()
	return r
}

// Emit passes a token back to the client.
func (l *Lexer) Emit(t Type) {
	l.tokens <- Token{t, l.base, l.input[l.base:l.pos]}
	l.base = l.pos
}

// Ignore skips over the pending input before this point.
func (l *Lexer) Ignore() {
	l.base = l.pos
}

// Backup steps back one rune. Can only be called once per call of Next.
func (l *Lexer) Backup() {
	l.pos -= l.width
}

// Consume tries to consume exactly the string s.
func (l *Lexer) Consume(s string) bool {
	ok := l.HasPrefix(s)
	if ok {
		l.pos += len(s)
	}
	return ok
}

// Accept consumes the next rune if it is from the valid set.
func (l *Lexer) Accept(valid string) bool {
	if strings.IndexRune(valid, l.Next()) >= 0 {
		return true
	}
	l.Backup()
	return false
}

// AcceptRun consumes a run of runes from the valid set.
// The number of bytes advanced is returned.
func (l *Lexer) AcceptRun(valid string) int {
	var n int
	for strings.IndexRune(valid, l.Next()) >= 0 {
		n += l.width
	}
	l.Backup()
	return n
}

// AcceptFunc consumes the next rune if f returns true.
func (l *Lexer) AcceptFunc(f func(r rune) bool) bool {
	if f(l.Next()) {
		return true
	}
	l.Backup()
	return false
}

// AcceptFunc consumes a run of runes as long as f returns true.
// The number of bytes advanced is returned.
func (l *Lexer) AcceptFuncRun(f func(r rune) bool) int {
	var n int
	for f(l.Next()) {
		n += l.width
	}
	l.Backup()
	return n
}

// AcceptBut consumes a rune if it is not from the invalid set.
func (l *Lexer) AcceptBut(invalid string) bool {
	if strings.IndexRune(invalid, l.Next()) < 0 {
		return true
	}
	l.Backup()
	return false
}

// AcceptButRun consumes runes as long as they are not in the invalid set.
// The number of bytes advanced is returned.
func (l *Lexer) AcceptButRun(invalid string) int {
	var n int
	for strings.IndexRune(invalid, l.Next()) < 0 {
		n += l.width
	}
	l.Backup()
	return n
}

// HasPrefix returns true if the input from the current position
// has the prefix s. It does not consume the prefix.
func (l *Lexer) HasPrefix(s string) bool {
	return strings.HasPrefix(l.input[l.pos:], s)
}

// HasPrefixAfter returns true if the input from the current position
// plus after bytes has the prefix s. It does not consume the prefix.
func (l *Lexer) HasPrefixAfter(after int, s string) bool {
	return strings.HasPrefix(l.input[l.pos+after:], s)
}

// Errorf returns an error token and terminates the scan by passing
// back a nil pointer that will be the next state, terminating l.NextToken.
func (l *Lexer) Errorf(format string, args ...interface{}) StateFn {
	l.tokens <- Token{TypeError, l.base, fmt.Sprintf(format, args...)}
	return nil
}
<|endoftext|>


<commit_before>package php

import (
	"fmt"
	"regexp"
	"strings"
	"unicode/utf8"
)

// Lexer represents the state of "lexing" items from a source string.
// The idea is derived from a Rob Pike talk:
// http://www.youtube.com/watch?v=HxaD_trXwRE
type lexer struct {
	// start stores the start position of the currently lexing item.
	start int
	// lastStart stores the start position of the previously lexed item.
	lastStart int
	// lastPos stores the position of the previous lexed element.
	lastPos int

	// pos is the current position of the lexer in the input, as an index
	// of the input string.
	pos  int
	line int
	// width is the length of the current rune
	width int
	items chan Item // channel of scanned items.

	// input is the full input string.
	input string

	// file is the filename of the input, used to print errors.
	file string
}

func newLexer(input string) *lexer {
	l := &lexer{
		line:  1,
		input: input,
		items: make(chan Item),
	}
	go l.run()
	return l
}

// stateFn represents the state of the scanner
// as a function that returns the next state.
type stateFn func(*lexer) stateFn

// run lexes the input by executing state functions until
// the state is nil.
func (l *lexer) run() {
	for state := lexHTML; state != nil; {
		state = state(l)
	}
	close(l.items) // No more tokens will be delivered.
}

// emit gets the current item, sends it on the item channel
// and prepares for lexing the next item
func (l *lexer) emit(t ItemType) {
	i := Item{t, l.currentLocation(), l.input[l.start:l.pos]}
	l.incrementLines()
	l.items <- i
	l.start = l.pos
}

func (l *lexer) currentLocation() Location {
	return Location{Pos: l.start, Line: l.line, File: l.file}
}

// nextItem returns the next item from the input.
func (l *lexer) nextItem() Item {
	Item := <-l.items
	l.lastPos = Item.pos.Pos
	return Item
}

// peek returns but does not consume the next rune in the input.
func (l *lexer) peek() rune {
	r := l.next()
	l.backup()
	return r
}

// backup steps back one rune. Can only be called once per call of next.
func (l *lexer) backup() {
	l.pos -= l.width
}

func (l *lexer) accept(valid string) bool {
	if strings.IndexRune(valid, l.next()) >= 0 {
		return true
	}
	l.backup()
	return false
}

// acceptRun consumes a run of runes from the valid set.
func (l *lexer) acceptRun(valid string) {
	for strings.IndexRune(valid, l.next()) >= 0 {
	}
	l.backup()
}

func (l *lexer) next() rune {
	if int(l.pos) >= len(l.input) {
		l.width = 0
		return eof
	}
	r, w := utf8.DecodeRuneInString(l.input[l.pos:])
	l.width = w
	l.pos += l.width
	return r
}

// ignore skips over the pending input before this point.
func (l *lexer) ignore() {
	l.start = l.pos
	l.incrementLines()
}

func (l *lexer) skipSpace() {
	r := l.next()
	for isSpace(r) {
		r = l.next()
	}
	l.backup()
	l.ignore()
}

func (l *lexer) errorf(format string, args ...interface{}) stateFn {
	i := Item{itemError, l.currentLocation(), fmt.Sprintf(format, args...)}
	l.incrementLines()
	l.items <- i
	return nil
}

func (l *lexer) incrementLines() {
	l.line += strings.Count(l.input[l.lastStart:l.pos], "\n")
	l.lastStart = l.pos
}

// isSpace reports whether r is a space character.
func isSpace(r rune) bool {
	return r == ' ' || r == '\t' || r == '\n'
}

func isKeyword(i ItemType) bool {
	return keywordMap[i]
}

// keywordMap lists all keywords that should be ignored as a prefix to a longer
// identifier.
var keywordMap = map[ItemType]bool{}

func init() {
	re := regexp.MustCompile("^[a-zA-Z]+")
	for keyword, item := range tokenMap {
		if re.MatchString(keyword) {
			keywordMap[item] = true
		}
	}
}
<commit_msg>Supporting all unicode whitespace as defined in unicode package<commit_after>package php

import (
	"fmt"
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

// Lexer represents the state of "lexing" items from a source string.
// The idea is derived from a Rob Pike talk:
// http://www.youtube.com/watch?v=HxaD_trXwRE
type lexer struct {
	// start stores the start position of the currently lexing item.
	start int
	// lastStart stores the start position of the previously lexed item.
	lastStart int
	// lastPos stores the position of the previous lexed element.
	lastPos int

	// pos is the current position of the lexer in the input, as an index
	// of the input string.
	pos  int
	line int
	// width is the length of the current rune
	width int
	items chan Item // channel of scanned items.

	// input is the full input string.
	input string

	// file is the filename of the input, used to print errors.
	file string
}

func newLexer(input string) *lexer {
	l := &lexer{
		line:  1,
		input: input,
		items: make(chan Item),
	}
	go l.run()
	return l
}

// stateFn represents the state of the scanner
// as a function that returns the next state.
type stateFn func(*lexer) stateFn

// run lexes the input by executing state functions until
// the state is nil.
func (l *lexer) run() {
	for state := lexHTML; state != nil; {
		state = state(l)
	}
	close(l.items) // No more tokens will be delivered.
}

// emit gets the current item, sends it on the item channel
// and prepares for lexing the next item
func (l *lexer) emit(t ItemType) {
	i := Item{t, l.currentLocation(), l.input[l.start:l.pos]}
	l.incrementLines()
	l.items <- i
	l.start = l.pos
}

func (l *lexer) currentLocation() Location {
	return Location{Pos: l.start, Line: l.line, File: l.file}
}

// nextItem returns the next item from the input.
func (l *lexer) nextItem() Item {
	Item := <-l.items
	l.lastPos = Item.pos.Pos
	return Item
}

// peek returns but does not consume the next rune in the input.
func (l *lexer) peek() rune {
	r := l.next()
	l.backup()
	return r
}

// backup steps back one rune. Can only be called once per call of next.
func (l *lexer) backup() {
	l.pos -= l.width
}

func (l *lexer) accept(valid string) bool {
	if strings.IndexRune(valid, l.next()) >= 0 {
		return true
	}
	l.backup()
	return false
}

// acceptRun consumes a run of runes from the valid set.
func (l *lexer) acceptRun(valid string) {
	for strings.IndexRune(valid, l.next()) >= 0 {
	}
	l.backup()
}

func (l *lexer) next() rune {
	if int(l.pos) >= len(l.input) {
		l.width = 0
		return eof
	}
	r, w := utf8.DecodeRuneInString(l.input[l.pos:])
	l.width = w
	l.pos += l.width
	return r
}

// ignore skips over the pending input before this point.
func (l *lexer) ignore() {
	l.start = l.pos
	l.incrementLines()
}

func (l *lexer) skipSpace() {
	r := l.next()
	for isSpace(r) {
		r = l.next()
	}
	l.backup()
	l.ignore()
}

func (l *lexer) errorf(format string, args ...interface{}) stateFn {
	i := Item{itemError, l.currentLocation(), fmt.Sprintf(format, args...)}
	l.incrementLines()
	l.items <- i
	return nil
}

func (l *lexer) incrementLines() {
	l.line += strings.Count(l.input[l.lastStart:l.pos], "\n")
	l.lastStart = l.pos
}

// isSpace reports whether r is a space character.
func isSpace(r rune) bool {
	return unicode.IsSpace(r)
}

func isKeyword(i ItemType) bool {
	return keywordMap[i]
}

// keywordMap lists all keywords that should be ignored as a prefix to a longer
// identifier.
var keywordMap = map[ItemType]bool{}

func init() {
	re := regexp.MustCompile("^[a-zA-Z]+")
	for keyword, item := range tokenMap {
		if re.MatchString(keyword) {
			keywordMap[item] = true
		}
	}
}
<|endoftext|>


<commit_before>package tripeg

import (
	"fmt"
	"strings"
	"testing"
)

func TestValidJumpVertical(t *testing.T) {
	b, _ := BuildBoard(1)
	h, err1 := b.GetHole(3, 3)
	if err1 != nil {
		t.Errorf("Can't find hole 3,3")
	}
	o, err2 := b.GetHole(2, 4)
	if err2 != nil {
		t.Errorf("Can't find hole, 2,4")
	}
	_, _, err := b.Jump(h, o)
	if err != nil {
		t.Fatal("Should have been successful.")
	}
}

func TestValidJumpHorizontal(t *testing.T) {
	b, _ := BuildBoard(6)
	h, err1 := b.GetHole(3, 3)
	if err1 != nil {
		t.Errorf("Can't find hole 3,3")
	}
	o, err2 := b.GetHole(3, 5)
	if err2 != nil {
		t.Errorf("Can't find hole, 3,5")
	}
	_, _, err := b.Jump(h, o)
	if err != nil {
		t.Fatal("Should have been successful.")
	}
}

func TestInvavidJumpOverHasNoPeg(t *testing.T) {
	b, _ := BuildBoard(6)
	h, err1 := b.GetHole(2, 6)
	if err1 != nil {
		t.Errorf("Can't find hole 2,6")
	}
	o, err2 := b.GetHole(3, 7)
	if err2 != nil {
		t.Errorf("Can't find hole, 3,7")
	}
	_, _, err := b.Jump(h, o)
	if err != nil {
		if !strings.HasPrefix(err.Error(), "No Peg in over hole") {
			fmt.Println(err)
			t.Fatal("Should have failed with No Peg in over hole.")
		}
	}
}

func TestInvavidTargetPegFull(t *testing.T) {
	b, _ := BuildBoard(6)
	h, err1 := b.GetHole(3, 3)
	if err1 != nil {
		t.Errorf("Can't find hole 3,3")
	}
	o, err2 := b.GetHole(2, 4)
	if err2 != nil {
		t.Errorf("Can't find hole, 2,4")
	}
	_, _, err := b.Jump(h, o)
	if err != nil {
		if err.Error() != "Target hole(1,5) has a peg in it\n" {
			t.Fatal("Should have failed Target hole(1,5) has a peg in it")
		}
	}
}

func TestInvalidHolePegEmpty(t *testing.T) {
	b, _ := BuildBoard(6)
	h, err1 := b.GetHole(3, 7)
	if err1 != nil {
		t.Errorf("Can't find hole 3,7")
	}
	o, err2 := b.GetHole(2, 6)
	if err2 != nil {
		t.Errorf("Can't find hole, 2,6")
	}
	_, _, err := b.Jump(h, o)
	if err != nil {
		if err.Error() != "No Peg in move hole 2,6\n" {
			t.Fatal("Should have failed with No Peg in move hole 2,6")
		}
	}
}

func TestErrorsTwo(t *testing.T) {
	ea := ErrorArray{}
	for r := 0; r < 3; r++ {
		ea.Add(fmt.Errorf("Error %d", r+1))
	}
	expected := "Error 1\nError 2\nError 3"
	received := ea.Error()
	if received != expected {
		t.Fatalf("Expected:\n%v\nReceived:\n%v\n", expected, received)
	}
}

func TestErrorsTwelve(t *testing.T) {
	ea := ErrorArray{}
	for r := 0; r < 13; r++ {
		ea.Add(fmt.Errorf("Error %d", r+1))
	}
	expected := "Error 1\nError 2\nError 3\nError 4\nError 5\nError 6\nError 7\nError 8\nError 9\nError 10\nToo many errors! Count: 12"
	received := ea.Error()
	if received != expected {
		t.Fatalf("Expected:\n%v\nReceived:\n%v\n", expected, received)
	}
}
<commit_msg>that upped the test coverage...not sure how legit this is<commit_after>package tripeg

import (
	"fmt"
	"strings"
	"testing"
)

func TestValidJumpVertical(t *testing.T) {
	b, _ := BuildBoard(1)
	h, err1 := b.GetHole(3, 3)
	if err1 != nil {
		t.Errorf("Can't find hole 3,3")
	}
	o, err2 := b.GetHole(2, 4)
	if err2 != nil {
		t.Errorf("Can't find hole, 2,4")
	}
	_, _, err := b.Jump(h, o)
	if err != nil {
		t.Fatal("Should have been successful.")
	}
}

func TestValidJumpHorizontal(t *testing.T) {
	b, _ := BuildBoard(6)
	h, err1 := b.GetHole(3, 3)
	if err1 != nil {
		t.Errorf("Can't find hole 3,3")
	}
	o, err2 := b.GetHole(3, 5)
	if err2 != nil {
		t.Errorf("Can't find hole, 3,5")
	}
	_, _, err := b.Jump(h, o)
	if err != nil {
		t.Fatal("Should have been successful.")
	}
}

func TestInvavidJumpOverHasNoPeg(t *testing.T) {
	b, _ := BuildBoard(6)
	h, err1 := b.GetHole(2, 6)
	if err1 != nil {
		t.Errorf("Can't find hole 2,6")
	}
	o, err2 := b.GetHole(3, 7)
	if err2 != nil {
		t.Errorf("Can't find hole, 3,7")
	}
	_, _, err := b.Jump(h, o)
	if err != nil {
		if !strings.HasPrefix(err.Error(), "No Peg in over hole") {
			fmt.Println(err)
			t.Fatal("Should have failed with No Peg in over hole.")
		}
	}
}

func TestInvavidTargetPegFull(t *testing.T) {
	b, _ := BuildBoard(6)
	h, err1 := b.GetHole(3, 3)
	if err1 != nil {
		t.Errorf("Can't find hole 3,3")
	}
	o, err2 := b.GetHole(2, 4)
	if err2 != nil {
		t.Errorf("Can't find hole, 2,4")
	}
	_, _, err := b.Jump(h, o)
	if err != nil {
		if err.Error() != "Target hole(1,5) has a peg in it\n" {
			t.Fatal("Should have failed Target hole(1,5) has a peg in it")
		}
	}
}

func TestInvalidHolePegEmpty(t *testing.T) {
	b, _ := BuildBoard(6)
	h, err1 := b.GetHole(3, 7)
	if err1 != nil {
		t.Errorf("Can't find hole 3,7")
	}
	o, err2 := b.GetHole(2, 6)
	if err2 != nil {
		t.Errorf("Can't find hole, 2,6")
	}
	_, _, err := b.Jump(h, o)
	if err != nil {
		if err.Error() != "No Peg in move hole 2,6\n" {
			t.Fatal("Should have failed with No Peg in move hole 2,6")
		}
	}
}

func TestErrorsTwo(t *testing.T) {
	ea := ErrorArray{}
	for r := 0; r < 3; r++ {
		ea.Add(fmt.Errorf("Error %d", r+1))
	}
	expected := "Error 1\nError 2\nError 3"
	received := ea.Error()
	if received != expected {
		t.Fatalf("Expected:\n%v\nReceived:\n%v\n", expected, received)
	}
}

func TestErrorsTwelve(t *testing.T) {
	ea := ErrorArray{}
	for r := 0; r < 13; r++ {
		ea.Add(fmt.Errorf("Error %d", r+1))
	}
	expected := "Error 1\nError 2\nError 3\nError 4\nError 5\nError 6\nError 7\nError 8\nError 9\nError 10\nToo many errors! Count: 12"
	received := ea.Error()
	if received != expected {
		t.Fatalf("Expected:\n%v\nReceived:\n%v\n", expected, received)
	}
}

func TestValidSolve(t *testing.T) {
	b, _ := BuildBoard(6)
	b.Solve()
	mlen := len(b.MoveChart)
	if mlen != 13 {
		t.Fatalf("Expected 13 moves, got %d\n", mlen)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 The Trickuri Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Trickuri generates test cases for URL spoofing vulnerabilities.
package main

import (
	"crypto"
	"crypto/rand"
	"crypto/rsa"
	"crypto/sha256"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"flag"
	"fmt"
	"io"
	"log"
	"math/big"
	"net"
	"net/http"
	"os"
	"path"
	"runtime"
	"strconv"
	"strings"
	"time"
)

var (
	port      = flag.Int("p", 1270, "port on which to listen")
	httpsPort = flag.Int("h", 8443, "port on which the HTTPS proxy will listen")
	directory = flag.String("d", userHomeDir()+"/.config/trickuri", "default directory in which to save certificates")
	testcaseDirectory = flag.String("t", "", "default file location containing index.html and a web-feature-tests directory")
)

var (
	caKey   crypto.PrivateKey
	caCert  *x509.Certificate
	certMap = make(map[string]*tls.Certificate)

	// TODO: fix the data race around this variable, either with a lock on this
	// variable, or (possibly better) by using a channel.
	lastTunneledHost string
)

// userHomeDir returns a suitable directory for the default certificate storage path.
func userHomeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}
		return home
	}
	return os.Getenv("HOME")
}

// newCertificate returns an initialized certificate with usable values.
func newCertificate() (*x509.Certificate, error) {
	maxSerial := new(big.Int).Lsh(big.NewInt(1), 128)
	serial, err := rand.Int(rand.Reader, maxSerial)
	if err != nil {
		return nil, err
	}

	return &x509.Certificate{
		SerialNumber: serial,
		Subject: pkix.Name{
			Organization: []string{"TrickUri Interception Certificate"},
		},
		NotBefore:             time.Now().AddDate(0, 0, -7), // mitigate clock-skew
		NotAfter:              time.Now().AddDate(1, 0, 7),
		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
	}, nil
}

// newRootCertificate returns a root certificate and its key.
func newRootCertificate() ([]byte, *rsa.PrivateKey, error) {
	private, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, nil, err
	}

	cert, err := newCertificate()
	if err != nil {
		return nil, nil, err
	}
	cert.IsCA = true
	cert.MaxPathLen = 0
	cert.MaxPathLenZero = true
	cert.Subject = pkix.Name{Organization: []string{"TrickUri Root"}}
	hash := sha256.Sum256(private.PublicKey.N.Bytes())
	cert.SubjectKeyId = hash[:]
	cert.KeyUsage |= x509.KeyUsageCertSign

	certBytes, err := x509.CreateCertificate(rand.Reader, cert, cert, &private.PublicKey, private)
	if err != nil {
		return nil, nil, err
	}

	return certBytes, private, nil
}

// writeCertificate writes the provided certificate and key to files in the specified paths.
func writeCertificate(cert []byte, key *rsa.PrivateKey, certPath, keyPath string) error {
	certFile, err := os.Create(certPath)
	if err != nil {
		return err
	}
	if err := pem.Encode(certFile, &pem.Block{Type: "CERTIFICATE", Bytes: cert}); err != nil {
		return err
	}
	if err := certFile.Close(); err != nil {
		return err
	}

	keyFile, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}
	if err := pem.Encode(keyFile, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}); err != nil {
		return err
	}
	if err := keyFile.Close(); err != nil {
		return err
	}

	return nil
}

// rootCertificate returns a root certificate, either loaded from the given
// directory, or created and saved.
func rootCertificate(directory string) (tls.Certificate, error) {
	certPath := path.Join(directory, "root.cer")
	keyPath := path.Join(directory, "root.pem")

	rootKeys, err := tls.LoadX509KeyPair(certPath, keyPath)
	if err != nil {
		log.Println("Failed to load root certificate. Recreating root certificate.")
		rootCert, rootKey, err := newRootCertificate()
		if err != nil {
			return tls.Certificate{}, err
		}
		if err := writeCertificate(rootCert, rootKey, certPath, keyPath); err != nil {
			return tls.Certificate{}, err
		}
		return tls.LoadX509KeyPair(certPath, keyPath)
	}
	return rootKeys, err
}

// newHostCertificate returns a certificate for the given hostname.
func newHostCertificate(hostname string) (*tls.Certificate, error) {
	priv, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, err
	}

	cert, err := newCertificate()
	if err != nil {
		return nil, err
	}
	if ip := net.ParseIP(hostname); ip != nil {
		cert.IPAddresses = append(cert.IPAddresses, ip)
	} else {
		cert.DNSNames = append(cert.DNSNames, hostname)
	}

	certBytes, err := x509.CreateCertificate(rand.Reader, cert, caCert, &priv.PublicKey, caKey)
	if err != nil {
		return nil, err
	}

	return &tls.Certificate{
		Certificate: [][]byte{certBytes},
		PrivateKey:  priv,
	}, nil
}

// certificate returns a matching certificate, either from a cache or by generating a new one.
func certificate(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
	var hostname string
	if len(info.ServerName) > 0 {
		hostname = info.ServerName
	} else {
		// If the SNI hostname is empty (e.g., for IP addresses), create a certificate for the last tunneled hostname.
		hostname = lastTunneledHost
	}

	if cert, ok := certMap[hostname]; ok {
		return cert, nil
	}

	cert, err := newHostCertificate(hostname)
	if err != nil {
		return nil, err
	}
	certMap[hostname] = cert

	return cert, nil
}

const pacFmt = `
function FindProxyForURL(url, host) {
        // Bypass list. See https://findproxyforurl.com/pac-functions/ for functions.
        if (shExpMatch(host, "*.google.com")) return "DIRECT";
        if (shExpMatch(host, "*.gstatic.com")) return "DIRECT";
        if (shExpMatch(host, "*.googleusercontent.com")) return "DIRECT";
        if (shExpMatch(host, "*.googleapis.com")) return "DIRECT";
        if (shExpMatch(host, "*.microsoft.com")) return "DIRECT";
        if (dnsDomainLevels(host) < 1) return "DIRECT";

        // Return a response from TrickUri.
        return "PROXY localhost:%d";
}`

// servePAC serves a proxy auto configuration script that directs the client to
// go direct for responses that should not be generated from this tool. This helps
// ensure that Chrome/Windows/etc are not unduly affected by our HTTP(S) interceptions.
func servePAC(w http.ResponseWriter, _ *http.Request) {
	w.Header().Set("Content-Type", "application/x-ns-proxy-autoconfig; charset=utf-8")
	w.WriteHeader(http.StatusOK)
	fmt.Fprintf(w, pacFmt, *port)
}

func serveEcho(w http.ResponseWriter, r *http.Request) {
	if err := r.Write(w); err != nil {
		log.Println(err)
	}
}

func serveRootCert(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, *directory+"/root.cer")
}

// serveHttpAuth returns whether the request contains HTTP Auth headers.
// When it does, it also writes a 401 Unauthorized response.
func serveHttpAuth(w http.ResponseWriter, r *http.Request) bool {
	if _, _, ok := r.BasicAuth(); ok {
		return true
	}
	w.Header().Set("WWW-Authenticate", "Basic realm=test")
	w.WriteHeader(401)
	w.Write([]byte("Unauthorized"))
	return false
}

func tunnelHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodConnect {
		// This is a request that doesn't require tunneling, use the default handler.
		httpHandler(w, r)
		return
	}

	// All https requests will be forwarded to the HTTPS proxy running on the set HTTPS port.
	httpsHost := "localhost:" + strconv.Itoa(*httpsPort)
	dst, err := net.DialTimeout("tcp", httpsHost, 3*time.Second)
	if err != nil {
		http.Error(w, "forwarding to "+httpsHost+" failed", http.StatusInternalServerError)
		return
	}
	w.WriteHeader(http.StatusOK)
	hj, ok := w.(http.Hijacker)
	if !ok {
		http.Error(w, "hijacking is not supported", http.StatusInternalServerError)
		return
	}
	hijackedConn, _, err := hj.Hijack()
	if err != nil {
		http.Error(w, "error hijacking connection", http.StatusInternalServerError)
		return
	}

	// Save for certificate generation if SNI is not available.
	lastTunneledHost = r.URL.Hostname()

	go forward(dst, hijackedConn)
	go forward(hijackedConn, dst)
}

func forward(dst io.WriteCloser, src io.ReadCloser) {
	defer dst.Close()
	defer src.Close()
	io.Copy(dst, src)
}

func httpHandler(w http.ResponseWriter, r *http.Request) {
	switch r.URL.EscapedPath() {
	case "/proxy.pac":
		servePAC(w, r)
		return
	case "/echo":
		serveEcho(w, r)
		return
	case "/root.cer":
		serveRootCert(w, r)
		return
	case "/web-feature-tests/http-auth/":
		if !serveHttpAuth(w, r) {
			return
		}
	}

	if strings.HasPrefix(r.URL.EscapedPath(), "/web-feature-tests") {
		tc := http.FileServer(http.Dir("."))
		tc.ServeHTTP(w, r)
		return
	}

	// An index explaining how to use the different testcases.
	http.ServeFile(w, r, *testcaseDirectory+"index.html")
}

const welcomeFmt = `

----Welcome to Trickuri!----

This tool facilitates testing of applications that displays URLs to users.
Trickuri is ready to receive requests.

1.) Download the root certificate at http://localhost:%d/root.cer and import it
into your browser/OS certificate store. See README.md for instructions on how to
import a root certificate.

2.) Set the proxy server of the application under test to
http://localhost:%[1]d/proxy.pac or localhost:%[1]d. Using the PAC file will
pass through requests to google.com and microsoft.com for common Chrome/Windows
requests. See https://www.chromium.org/developers/design-documents/network-settings
for instructions on configuring Chrome's proxy server, if you are testing Chrome.

3.) Visit https://example.com/ (or any other URL) to see a list of test cases.

`

func main() {
	flag.Parse()

	if err := os.MkdirAll(*directory, os.ModePerm); err != nil {
		log.Fatal(err)
	}

	rootCert, err := rootCertificate(*directory)
	if err != nil {
		log.Fatal(err)
	}
	caCert, err = x509.ParseCertificate(rootCert.Certificate[0])
	if err != nil {
		log.Fatal(err)
	}
	caKey = rootCert.PrivateKey

	fmt.Printf(welcomeFmt, *port)

	httpsServer := http.Server{
		Addr: "localhost:" + strconv.Itoa(*httpsPort),
		TLSConfig: &tls.Config{
			CipherSuites: []uint16{
				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			},
			MinVersion:               tls.VersionTLS12,
			PreferServerCipherSuites: true,
			GetCertificate:           certificate,
		},
		Handler: http.HandlerFunc(httpHandler),
	}
	go func() {
		if err := httpsServer.ListenAndServeTLS("", ""); err != nil {
			log.Fatal(err)
		}
	}()

	httpServer := http.Server{
		Addr:    "localhost:" + strconv.Itoa(*port),
		Handler: http.HandlerFunc(tunnelHandler),
	}
	if err := httpServer.ListenAndServe(); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Added file path to web tests<commit_after>// Copyright 2018 The Trickuri Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Trickuri generates test cases for URL spoofing vulnerabilities.
package main

import (
	"crypto"
	"crypto/rand"
	"crypto/rsa"
	"crypto/sha256"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"flag"
	"fmt"
	"io"
	"log"
	"math/big"
	"net"
	"net/http"
	"os"
	"path"
	"runtime"
	"strconv"
	"strings"
	"time"
)

var (
	port      = flag.Int("p", 1270, "port on which to listen")
	httpsPort = flag.Int("h", 8443, "port on which the HTTPS proxy will listen")
	directory = flag.String("d", userHomeDir()+"/.config/trickuri", "default directory in which to save certificates")
	testcaseDirectory = flag.String("t", "./", "default file location containing index.html and a web-feature-tests directory")
)

var (
	caKey   crypto.PrivateKey
	caCert  *x509.Certificate
	certMap = make(map[string]*tls.Certificate)

	// TODO: fix the data race around this variable, either with a lock on this
	// variable, or (possibly better) by using a channel.
	lastTunneledHost string
)

// userHomeDir returns a suitable directory for the default certificate storage path.
func userHomeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}
		return home
	}
	return os.Getenv("HOME")
}

// newCertificate returns an initialized certificate with usable values.
func newCertificate() (*x509.Certificate, error) {
	maxSerial := new(big.Int).Lsh(big.NewInt(1), 128)
	serial, err := rand.Int(rand.Reader, maxSerial)
	if err != nil {
		return nil, err
	}

	return &x509.Certificate{
		SerialNumber: serial,
		Subject: pkix.Name{
			Organization: []string{"TrickUri Interception Certificate"},
		},
		NotBefore:             time.Now().AddDate(0, 0, -7), // mitigate clock-skew
		NotAfter:              time.Now().AddDate(1, 0, 7),
		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
	}, nil
}

// newRootCertificate returns a root certificate and its key.
func newRootCertificate() ([]byte, *rsa.PrivateKey, error) {
	private, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, nil, err
	}

	cert, err := newCertificate()
	if err != nil {
		return nil, nil, err
	}
	cert.IsCA = true
	cert.MaxPathLen = 0
	cert.MaxPathLenZero = true
	cert.Subject = pkix.Name{Organization: []string{"TrickUri Root"}}
	hash := sha256.Sum256(private.PublicKey.N.Bytes())
	cert.SubjectKeyId = hash[:]
	cert.KeyUsage |= x509.KeyUsageCertSign

	certBytes, err := x509.CreateCertificate(rand.Reader, cert, cert, &private.PublicKey, private)
	if err != nil {
		return nil, nil, err
	}

	return certBytes, private, nil
}

// writeCertificate writes the provided certificate and key to files in the specified paths.
func writeCertificate(cert []byte, key *rsa.PrivateKey, certPath, keyPath string) error {
	certFile, err := os.Create(certPath)
	if err != nil {
		return err
	}
	if err := pem.Encode(certFile, &pem.Block{Type: "CERTIFICATE", Bytes: cert}); err != nil {
		return err
	}
	if err := certFile.Close(); err != nil {
		return err
	}

	keyFile, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}
	if err := pem.Encode(keyFile, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}); err != nil {
		return err
	}
	if err := keyFile.Close(); err != nil {
		return err
	}

	return nil
}

// rootCertificate returns a root certificate, either loaded from the given
// directory, or created and saved.
func rootCertificate(directory string) (tls.Certificate, error) {
	certPath := path.Join(directory, "root.cer")
	keyPath := path.Join(directory, "root.pem")

	rootKeys, err := tls.LoadX509KeyPair(certPath, keyPath)
	if err != nil {
		log.Println("Failed to load root certificate. Recreating root certificate.")
		rootCert, rootKey, err := newRootCertificate()
		if err != nil {
			return tls.Certificate{}, err
		}
		if err := writeCertificate(rootCert, rootKey, certPath, keyPath); err != nil {
			return tls.Certificate{}, err
		}
		return tls.LoadX509KeyPair(certPath, keyPath)
	}
	return rootKeys, err
}

// newHostCertificate returns a certificate for the given hostname.
func newHostCertificate(hostname string) (*tls.Certificate, error) {
	priv, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, err
	}

	cert, err := newCertificate()
	if err != nil {
		return nil, err
	}
	if ip := net.ParseIP(hostname); ip != nil {
		cert.IPAddresses = append(cert.IPAddresses, ip)
	} else {
		cert.DNSNames = append(cert.DNSNames, hostname)
	}

	certBytes, err := x509.CreateCertificate(rand.Reader, cert, caCert, &priv.PublicKey, caKey)
	if err != nil {
		return nil, err
	}

	return &tls.Certificate{
		Certificate: [][]byte{certBytes},
		PrivateKey:  priv,
	}, nil
}

// certificate returns a matching certificate, either from a cache or by generating a new one.
func certificate(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
	var hostname string
	if len(info.ServerName) > 0 {
		hostname = info.ServerName
	} else {
		// If the SNI hostname is empty (e.g., for IP addresses), create a certificate for the last tunneled hostname.
		hostname = lastTunneledHost
	}

	if cert, ok := certMap[hostname]; ok {
		return cert, nil
	}

	cert, err := newHostCertificate(hostname)
	if err != nil {
		return nil, err
	}
	certMap[hostname] = cert

	return cert, nil
}

const pacFmt = `
function FindProxyForURL(url, host) {
        // Bypass list. See https://findproxyforurl.com/pac-functions/ for functions.
        if (shExpMatch(host, "*.google.com")) return "DIRECT";
        if (shExpMatch(host, "*.gstatic.com")) return "DIRECT";
        if (shExpMatch(host, "*.googleusercontent.com")) return "DIRECT";
        if (shExpMatch(host, "*.googleapis.com")) return "DIRECT";
        if (shExpMatch(host, "*.microsoft.com")) return "DIRECT";
        if (dnsDomainLevels(host) < 1) return "DIRECT";

        // Return a response from TrickUri.
        return "PROXY localhost:%d";
}`

// servePAC serves a proxy auto configuration script that directs the client to
// go direct for responses that should not be generated from this tool. This helps
// ensure that Chrome/Windows/etc are not unduly affected by our HTTP(S) interceptions.
func servePAC(w http.ResponseWriter, _ *http.Request) {
	w.Header().Set("Content-Type", "application/x-ns-proxy-autoconfig; charset=utf-8")
	w.WriteHeader(http.StatusOK)
	fmt.Fprintf(w, pacFmt, *port)
}

func serveEcho(w http.ResponseWriter, r *http.Request) {
	if err := r.Write(w); err != nil {
		log.Println(err)
	}
}

func serveRootCert(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, *directory+"/root.cer")
}

// serveHttpAuth returns whether the request contains HTTP Auth headers.
// When it does, it also writes a 401 Unauthorized response.
func serveHttpAuth(w http.ResponseWriter, r *http.Request) bool {
	if _, _, ok := r.BasicAuth(); ok {
		return true
	}
	w.Header().Set("WWW-Authenticate", "Basic realm=test")
	w.WriteHeader(401)
	w.Write([]byte("Unauthorized"))
	return false
}

func tunnelHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodConnect {
		// This is a request that doesn't require tunneling, use the default handler.
		httpHandler(w, r)
		return
	}

	// All https requests will be forwarded to the HTTPS proxy running on the set HTTPS port.
	httpsHost := "localhost:" + strconv.Itoa(*httpsPort)
	dst, err := net.DialTimeout("tcp", httpsHost, 3*time.Second)
	if err != nil {
		http.Error(w, "forwarding to "+httpsHost+" failed", http.StatusInternalServerError)
		return
	}
	w.WriteHeader(http.StatusOK)
	hj, ok := w.(http.Hijacker)
	if !ok {
		http.Error(w, "hijacking is not supported", http.StatusInternalServerError)
		return
	}
	hijackedConn, _, err := hj.Hijack()
	if err != nil {
		http.Error(w, "error hijacking connection", http.StatusInternalServerError)
		return
	}

	// Save for certificate generation if SNI is not available.
	lastTunneledHost = r.URL.Hostname()

	go forward(dst, hijackedConn)
	go forward(hijackedConn, dst)
}

func forward(dst io.WriteCloser, src io.ReadCloser) {
	defer dst.Close()
	defer src.Close()
	io.Copy(dst, src)
}

func httpHandler(w http.ResponseWriter, r *http.Request) {
	switch r.URL.EscapedPath() {
	case "/proxy.pac":
		servePAC(w, r)
		return
	case "/echo":
		serveEcho(w, r)
		return
	case "/root.cer":
		serveRootCert(w, r)
		return
	case "/web-feature-tests/http-auth/":
		if !serveHttpAuth(w, r) {
			return
		}
	}

	if strings.HasPrefix(r.URL.EscapedPath(), "/web-feature-tests") {
		tc := http.FileServer(http.Dir(*testcaseDirectory))
		tc.ServeHTTP(w, r)
		return
	}

	// An index explaining how to use the different testcases.
	http.ServeFile(w, r, *testcaseDirectory+"index.html")
}

const welcomeFmt = `

----Welcome to Trickuri!----

This tool facilitates testing of applications that displays URLs to users.
Trickuri is ready to receive requests.

1.) Download the root certificate at http://localhost:%d/root.cer and import it
into your browser/OS certificate store. See README.md for instructions on how to
import a root certificate.

2.) Set the proxy server of the application under test to
http://localhost:%[1]d/proxy.pac or localhost:%[1]d. Using the PAC file will
pass through requests to google.com and microsoft.com for common Chrome/Windows
requests. See https://www.chromium.org/developers/design-documents/network-settings
for instructions on configuring Chrome's proxy server, if you are testing Chrome.

3.) Visit https://example.com/ (or any other URL) to see a list of test cases.

`

func main() {
	flag.Parse()

	if err := os.MkdirAll(*directory, os.ModePerm); err != nil {
		log.Fatal(err)
	}

	rootCert, err := rootCertificate(*directory)
	if err != nil {
		log.Fatal(err)
	}
	caCert, err = x509.ParseCertificate(rootCert.Certificate[0])
	if err != nil {
		log.Fatal(err)
	}
	caKey = rootCert.PrivateKey

	fmt.Printf(welcomeFmt, *port)

	httpsServer := http.Server{
		Addr: "localhost:" + strconv.Itoa(*httpsPort),
		TLSConfig: &tls.Config{
			CipherSuites: []uint16{
				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			},
			MinVersion:               tls.VersionTLS12,
			PreferServerCipherSuites: true,
			GetCertificate:           certificate,
		},
		Handler: http.HandlerFunc(httpHandler),
	}
	go func() {
		if err := httpsServer.ListenAndServeTLS("", ""); err != nil {
			log.Fatal(err)
		}
	}()

	httpServer := http.Server{
		Addr:    "localhost:" + strconv.Itoa(*port),
		Handler: http.HandlerFunc(tunnelHandler),
	}
	if err := httpServer.ListenAndServe(); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>