<commit_before>package main

import (
	"bytes"
	"fmt"
	"github.com/nu7hatch/gouuid"
	"io/ioutil"
	"log"
	"net/http"
	"runtime"
	"time"
	"flag"
)

type TargetList map[string][]EventTarget

// why not just []string of urls? In case we need meta data for these later on.
type EventTarget struct {
	Url string
}

// Our object use to repeat events.
// todo: should I just pass the http.Request? Is that thread safe?
type RequestMessage struct {
	URL     string
	Method  string
	Source  string
	Headers http.Header
	Body    []byte
}

func handler(w http.ResponseWriter, r *http.Request) {

	body, _ := ioutil.ReadAll(r.Body)

	requestObj := RequestMessage{
		URL:     r.RequestURI,
		Method:  r.Method,
		Source:  r.RemoteAddr,
		Headers: r.Header,
		Body:    body,
	}

	// get a UUID for this transaction
	u5, err := uuid.NewV4()
	if err != nil {
		fmt.Println("error:", err)
		return
	}

	queue := requestObj.URL[1:]

	fmt.Fprintf(w, "{ \"id\":\"%s\" }\n", u5) // to lazy to do a real json.Marshal, etc

	if eventTargets, ok := targets[queue]; ok {
		for _, eventTarget := range eventTargets {
			//log.Printf("id=%s queue=%s msg=%s url=%s\n", u5, queue, "sendingto", eventTarget.Url)
			addchan <- CounterKey{queue, eventTarget.Url}
			go sendEvent(u5, queue, eventTarget, requestObj)
		}
	} else {
		log.Printf("id=%s queue=%s msg=%s\n", u5, queue, "no such queue")
	}
}

func sendEvent(u5 *uuid.UUID, queue string, eventTarget EventTarget, req RequestMessage) {
	start := time.Now()
	var sent bool
	sent = false
	attempts := 0
	sleepDuration := time.Millisecond * 100
	for {
		attempts++

		client := &http.Client{
			Timeout: 10 * time.Second,
		}
		httpReq, _ := http.NewRequest(req.Method, eventTarget.Url, bytes.NewBuffer(req.Body))
		//httpReq.Header = req.Headers
		for headerName, values := range req.Headers {
			for _, value := range values {
				httpReq.Header.Add(headerName, value)
			}
		}
		httpReq.Header.Set("X-Wsq-Id", (*u5).String())
		resp, err := client.Do(httpReq)

		if err == nil && resp.StatusCode == 200 {
			sent = true
			break
		}

		// max duration, ever
		if time.Since(start) > time.Second*60 {
			break
		}

		// oops, didn't work; have a pause and try again in a bit
		time.Sleep(sleepDuration)
		// slowly ramp up our sleep interval, shall we?
		// todo: if sleepDuration > N value, don't increase it again -- apply a cap
		sleepDuration = time.Duration(float64(sleepDuration) * 1.5)
	}
	elapsed := time.Since(start)

	if sent {
		deltchan <- CounterKey{queue, eventTarget.Url}
	} else {
		delfchan <- CounterKey{queue, eventTarget.Url}
	}

	log.Printf("id=%s queue=%s msg=%s url=%s attempts=%v sent=%v duration=%.3f\n",
		u5, queue, "endsend", eventTarget.Url, attempts, sent, elapsed.Seconds()*1e3)
}

type CounterKey struct {
	Queue string
	Url   string
}

type CounterVals struct {
	Current uint64
	Total   uint64
	Success uint64
	Failure uint64
}

var counters = make(map[CounterKey]CounterVals)
var addchan = make(chan CounterKey, 100)
var deltchan = make(chan CounterKey, 100)
var delfchan = make(chan CounterKey, 100)

var targets TargetList

func main() {
	log.SetFlags(log.LstdFlags | log.Lmicroseconds)

	configId := flag.String("config", "default", "Which stanza of the config to use")
	flag.Parse()

	targets, ok := allTargets[*configId]
	if !ok {
		panic("Could not load expected configuration")
	}

	// initialize counters to zero
	// You don't _have_ to do this, but I like having all the counters
	// reporting 0 immediately for stat collection purposes.
	for queue, eventTargets := range targets {
		for _, eventTarget := range eventTargets {
			counters[CounterKey{queue, eventTarget.Url}] = CounterVals{0,0,0,0}
		}
	}

	// goroutine to keep the counters up-to-date
	go func() {
		for {
			// watch each channel as items rolls in and modify the counters as needed
			select {
			// you can't do counters[control].Current++ in go, so this mess is what results
			case control := <-addchan:
				tmp := counters[control]
				tmp.Current++
				tmp.Total++
				counters[control] = tmp
			case control := <-deltchan:
				tmp := counters[control]
				tmp.Current--
				tmp.Success++
				counters[control] = tmp
			case control := <-delfchan:
				tmp := counters[control]
				tmp.Current--
				tmp.Failure++
				counters[control] = tmp
			}
		}
	}()

	// A dumb goroutine to watch memory usage and counter metrics
	go func() {
		var mem runtime.MemStats
		for {
			runtime.ReadMemStats(&mem)
			log.Printf("metrics=ram alloc=%v totalalloc=%v heapalloc=%v heapsys=%v routines=%v\n",
				mem.Alloc, mem.TotalAlloc, mem.HeapAlloc, mem.HeapSys, runtime.NumGoroutine())
			for cKeys, cVals:= range counters {
				log.Printf("metrics=queues queue=%s endpoint=%s current=%d total=%d success=%d failure=%d\n",
					cKeys.Queue, cKeys.Url, cVals.Current, cVals.Total, cVals.Success, cVals.Failure)
			}
			time.Sleep(time.Second * 5)
		}
	}()

	// Oh, hey, there's the webserver!
	fmt.Println("Starting server")
	http.HandleFunc("/", handler)
	err := http.ListenAndServe(":8000", nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}

}
<commit_msg>Playing with a conversion to a goroutine per endpoint<commit_after>package main

import (
	"bytes"
	"fmt"
	"github.com/nu7hatch/gouuid"
	"io/ioutil"
	"log"
	"net/http"
	"runtime"
	"time"
	"flag"
)

type TargetList map[string][]EventTarget

// why not just []string of urls? In case we need meta data for these later on.
type EventTarget struct {
	Url string
}

// Our object use to repeat events.
// todo: should I just pass the http.Request? Is that thread safe?
type RequestMessage struct {
	UUID    string
	URL     string
	Method  string
	Source  string
	Headers http.Header
	Body    []byte
}

func handler(w http.ResponseWriter, r *http.Request) {

	// get a UUID for this transaction
	u5, err := uuid.NewV4()
	if err != nil {
		fmt.Println("error:", err)
		return
	}

	body, _ := ioutil.ReadAll(r.Body)

	requestObj := RequestMessage{
		UUID:    u5.String(),
		URL:     r.RequestURI,
		Method:  r.Method,
		Source:  r.RemoteAddr,
		Headers: r.Header,
		Body:    body,
	}

	queue := requestObj.URL[1:]

	fmt.Fprintf(w, "{ \"id\":\"%s\" }\n", u5) // to lazy to do a real json.Marshal, etc

	if eventTargets, ok := targets[queue]; ok {
		for _, eventTarget := range eventTargets {
			qu := QueueUrl{queue, eventTarget.Url}
			addchan <- qu
			// this select/case/default is a non-blocking chan push
			select {
			// todo: maybe circular chans here? IE: throw away oldest items when the chan is full.
			case sendPool[qu].RequestChan <- requestObj:
			default:
				log.Printf("id=%s queue=%s msg=%s\n", u5, queue, "queue full!")
			}
		}
	} else {
		log.Printf("id=%s queue=%s msg=%s\n", u5, queue, "no such queue")
		log.Printf("queues=%v\n", targets)
	}
}

func sendEvent(qu QueueUrl, req RequestMessage) {
	start := time.Now()
	var sent bool
	sent = false
	attempts := 0
	sleepDuration := time.Millisecond * 100
	for {
		attempts++

		client := &http.Client{
			Timeout: 10 * time.Second,
		}
		httpReq, _ := http.NewRequest(req.Method, qu.Url, bytes.NewBuffer(req.Body))
		//httpReq.Header = req.Headers
		for headerName, values := range req.Headers {
			for _, value := range values {
				httpReq.Header.Add(headerName, value)
			}
		}
		httpReq.Header.Set("X-Wsq-Id", req.UUID)
		resp, err := client.Do(httpReq)

		if err == nil && resp.StatusCode == 200 {
			sent = true
			break
		}

		// max duration, ever
		if time.Since(start) > time.Second*60 {
			break
		}

		// oops, didn't work; have a pause and try again in a bit
		time.Sleep(sleepDuration)
		// slowly ramp up our sleep interval, shall we?
		// todo: if sleepDuration > N value, don't increase it again -- apply a cap
		sleepDuration = time.Duration(float64(sleepDuration) * 1.5)
	}
	elapsed := time.Since(start)

	if sent {
		deltchan <- qu
	} else {
		delfchan <- qu
	}

	log.Printf("id=%s queue=%s msg=%s url=%s attempts=%v sent=%v duration=%.3f\n",
		req.UUID, qu.Queue, "endsend", qu.Url, attempts, sent, elapsed.Seconds()*1e3)
}

type QueueUrl struct {
	Queue string
	Url   string
}

type CounterVals struct {
	Current uint64
	Total   uint64
	Success uint64
	Failure uint64
}

var counters = make(map[QueueUrl]CounterVals)
var addchan = make(chan QueueUrl, 100)
var deltchan = make(chan QueueUrl, 100)
var delfchan = make(chan QueueUrl, 100)

type Worker struct {
  QueueUrl    QueueUrl
  RequestChan chan RequestMessage
  QuitChan    chan bool
}

func (w Worker) Start() {
	go func() {
		for {
			select {
			case work := <-w.RequestChan:
				// Receive a work request.
				fmt.Printf("start worker %v got %v\n", w.QueueUrl, work.UUID)
				sendEvent(w.QueueUrl, work)
				fmt.Printf(" done worker %v got %v\n", w.QueueUrl, work.UUID)
			case <-w.QuitChan:
				// We have been asked to stop.
				fmt.Printf("worker %v stopping\n", w.QueueUrl)
				return
			}
		}
	}()
}

var sendPool = make(map[QueueUrl]Worker)

var targets TargetList

func main() {
	log.SetFlags(log.LstdFlags | log.Lmicroseconds)

	configId := flag.String("config", "default", "Which stanza of the config to use")
	flag.Parse()

	var ok bool
	targets, ok = allTargets[*configId]
	if !ok {
		panic("Could not load expected configuration")
	}

	// initialize counters to zero
	// You don't _have_ to do this, but I like having all the counters
	// reporting 0 immediately for stat collection purposes.
	for queue, eventTargets := range targets {
		for _, eventTarget := range eventTargets {
			qu := QueueUrl{queue, eventTarget.Url}
			counters[qu] = CounterVals{0,0,0,0}
			sendPool[qu] = Worker{
				QueueUrl: qu,
				RequestChan: make(chan RequestMessage, 10000),
				QuitChan: make(chan bool),
			}
			sendPool[qu].Start()
		}
	}

	// goroutine to keep the counters up-to-date
	go func() {
		for {
			// watch each channel as items rolls in and modify the counters as needed
			select {
			// you can't do counters[control].Current++ in go, so this mess is what results
			case control := <-addchan:
				tmp := counters[control]
				tmp.Current++
				tmp.Total++
				counters[control] = tmp
			case control := <-deltchan:
				tmp := counters[control]
				tmp.Current--
				tmp.Success++
				counters[control] = tmp
			case control := <-delfchan:
				tmp := counters[control]
				tmp.Current--
				tmp.Failure++
				counters[control] = tmp
			}
		}
	}()

	// A dumb goroutine to watch memory usage and counter metrics
	go func() {
		var mem runtime.MemStats
		for {
			runtime.ReadMemStats(&mem)
			log.Printf("metrics=ram alloc=%v totalalloc=%v heapalloc=%v heapsys=%v routines=%v\n",
				mem.Alloc, mem.TotalAlloc, mem.HeapAlloc, mem.HeapSys, runtime.NumGoroutine())
			for cKeys, cVals:= range counters {
				log.Printf("metrics=queues queue=%s endpoint=%s current=%d total=%d success=%d failure=%d\n",
					cKeys.Queue, cKeys.Url, cVals.Current, cVals.Total, cVals.Success, cVals.Failure)
			}
			time.Sleep(time.Second * 5)
		}
	}()

	// Oh, hey, there's the webserver!
	fmt.Println("Starting server")
	http.HandleFunc("/", handler)
	err := http.ListenAndServe(":8000", nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"strings"

	"github.com/drone-plugins/drone-git-push/repo"
	"github.com/drone/drone-go/drone"
	"github.com/drone/drone-go/plugin"
)

var (
	build          string
	buildDate      string
	privateKeyPath string = "/root/.ssh/id_rsa"
)

type DeployWorkspace struct {
	Workspace drone.Workspace
}

func main() {
	fmt.Printf("Drone Capistrano Plugin built at %s\n", buildDate)

	workspace := drone.Workspace{}
	vargs := Params{}

	dw := DeployWorkspace{workspace}

	plugin.Param("workspace", &workspace)
	plugin.Param("vargs", &vargs)
	plugin.MustParse()

	log("Installing Drone's ssh key")
	if err := repo.WriteKey(&workspace); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	os.Setenv("BUILD_PATH", workspace.Path)
	os.Setenv("GIT_SSH_KEY", privateKeyPath)

	tasks := strings.Fields(vargs.Tasks)

	if len(tasks) == 0 {
		fmt.Println("Please provide Capistrano tasks to execute")
		os.Exit(1)
		return
	}

	log("Running Bundler")
	bundle_args := []string{"install"}
	if ! vargs.Debug {
		bundle_args = append(bundle_args, "--quiet")
	}
	if len(vargs.BundlePath) > 0 {
		bundle_args = append(bundle_args, "--path", vargs.BundlePath)
	}
	bundle := dw.bundle(bundle_args...)
	if err := bundle.Run(); err != nil {
		fmt.Println(err)
		os.Exit(1)
		return
	}

	log("Running Capistrano")
	capistrano := dw.cap(tasks...)
	if err := capistrano.Run(); err != nil {
		fmt.Println(err)
		os.Exit(1)
		return
	}
}

func (w *DeployWorkspace) cap(tasks ...string) *exec.Cmd {
	args := append([]string{"exec", "cap"}, tasks...)
	return w.bundle(args...)
}

func (w *DeployWorkspace) bundle(args ...string) *exec.Cmd {
	return w.command("/bundle.sh", args...)
}

func (w *DeployWorkspace) command(cmd string, args ...string) *exec.Cmd {
	c := exec.Command(cmd, args...)
	c.Dir = w.Workspace.Path
	c.Env = os.Environ()
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr
	return c
}

func log(message string, a ...interface{}) {
	fmt.Printf("=> %s\n", fmt.Sprintf(message, a...))
}
<commit_msg>separate function for generating bundler arguments<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"strings"

	"github.com/drone-plugins/drone-git-push/repo"
	"github.com/drone/drone-go/drone"
	"github.com/drone/drone-go/plugin"
)

var (
	build          string
	buildDate      string
	privateKeyPath string = "/root/.ssh/id_rsa"
)

type DeployWorkspace struct {
	Workspace drone.Workspace
}

func main() {
	fmt.Printf("Drone Capistrano Plugin built at %s\n", buildDate)

	workspace := drone.Workspace{}
	vargs := Params{}

	dw := DeployWorkspace{workspace}

	plugin.Param("workspace", &workspace)
	plugin.Param("vargs", &vargs)
	plugin.MustParse()

	log("Installing Drone's ssh key")
	if err := repo.WriteKey(&workspace); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	os.Setenv("BUILD_PATH", workspace.Path)
	os.Setenv("GIT_SSH_KEY", privateKeyPath)

	tasks := strings.Fields(vargs.Tasks)

	if len(tasks) == 0 {
		fmt.Println("Please provide Capistrano tasks to execute")
		os.Exit(1)
		return
	}

	log("Running Bundler")
	bundle := dw.bundle(bundlerArgs(vargs)...)
	if err := bundle.Run(); err != nil {
		fmt.Println(err)
		os.Exit(1)
		return
	}

	log("Running Capistrano")
	capistrano := dw.cap(tasks...)
	if err := capistrano.Run(); err != nil {
		fmt.Println(err)
		os.Exit(1)
		return
	}
}

func bundlerArgs(vargs Params) []string {
	args := []string{"install"}
	if ! vargs.Debug {
		args = append(args, "--quiet")
	}
	if len(vargs.BundlePath) > 0 {
		args = append(args, "--path", vargs.BundlePath)
	}
	return args
}

func (w *DeployWorkspace) cap(tasks ...string) *exec.Cmd {
	args := append([]string{"exec", "cap"}, tasks...)
	return w.bundle(args...)
}

func (w *DeployWorkspace) bundle(args ...string) *exec.Cmd {
	return w.command("/bundle.sh", args...)
}

func (w *DeployWorkspace) command(cmd string, args ...string) *exec.Cmd {
	c := exec.Command(cmd, args...)
	c.Dir = w.Workspace.Path
	c.Env = os.Environ()
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr
	return c
}

func log(message string, a ...interface{}) {
	fmt.Printf("=> %s\n", fmt.Sprintf(message, a...))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"evac/server"
	"evac/filterlist"
)

/* TODO: 1 - Read incoming DNS request */
/* TODO: 2 - Check cache for request response */
/* TODO: 3 - Check blacklist for request domain */
/* TODO: 4 - Request from remote DNS server */
/* TODO: 5 - Serve DNS response to client */

func main() {
	filterlist.NewCache(200)
	listener := server.DnsServer{make(chan server.Request)}
	go func () {
		err := listener.Start(":53")
		if err != nil {
			fmt.Print(err)
		}
	}()

	<-listener.IncomingRequests
	fmt.Print("Done")
}
<commit_msg>Use DnsServer constructor<commit_after>package main

import (
	"fmt"
	"evac/server"
	"evac/filterlist"
)

/* TODO: 1 - Read incoming DNS request */
/* TODO: 2 - Check cache for request response */
/* TODO: 3 - Check blacklist for request domain */
/* TODO: 4 - Request from remote DNS server */
/* TODO: 5 - Serve DNS response to client */

func main() {
	filterlist.NewCache(200)
	listener := server.NewServer(50)
	go func () {
		err := listener.Start(":53")
		if err != nil {
			fmt.Print(err)
		}
	}()

	<-listener.IncomingRequests
	fmt.Print("Done")
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"go/format"
	"os"
	"os/exec"
	"path"
	"strings"
	"text/template"
)

const (
	defaultfilename = "buildconstants_generated.go"

	templatePrelude = `// Generated by running
//      buildconstants
// DO NOT EDIT

package {{.GOPKG}}

const (
	MustRunBuildConstants = 0
`

	templatePostlude = ")"
)

type cmd struct {
	Var  string
	Line string
	echo bool // get value from env
}

// read commands from a text file.  Each command has its own line
//   and the lines are of the form `VAR = echo $FOO`
//   where $VAR is the env variable and everything after '=' will be evaluated for the value
func cmdRead(cmdfile string) ([]cmd, error) {

	f, err := os.Open(cmdfile)
	if err != nil {
		return []cmd{}, err
	}
	defer f.Close()
	var cmds []cmd
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		splits := strings.Split(scanner.Text(), "=")
		if len(splits) < 2 {
			continue
		}
		line := strings.TrimSpace(strings.Join(splits[1:], " "))
		c := cmd{
			Var:  strings.TrimSpace(splits[0]),
			Line: line,
			echo: strings.HasPrefix(line, "$"),
		}
		cmds = append(cmds, c)
	}
	return cmds, nil
}

// operates on inputs since it needs to be in the same order
func envTemplate(ins []cmd) string {
	var s string
	for _, in := range ins {
		s = s + " " + in.Var + " = \"{{." + in.Var + "}}\"\n"
	}
	return s
}

func do(command cmd) (string, error) {
	expanded := os.Expand(command.Line, os.Getenv)
	var out string
	if command.echo {
		out = expanded
	} else {
		split := strings.Split(expanded, " ")
		cmd := exec.Command(split[0], split[1:]...)
		bout, err := cmd.Output()
		if err != nil {
			return "", err
		}
		out = string(bout)
	}

	val := strings.TrimSpace(out)
	err := os.Setenv(command.Var, val)
	if err != nil {
		return "", err
	}
	return val, nil
}

func pkg() string {
	pkgCmd := exec.Command("go", "list", ".")
	out, err := pkgCmd.Output()
	if err != nil {
		return "main"
	}
	_, packageName := path.Split(strings.TrimSpace(string(out)))
	return packageName
}

func main() {

	cmdfile := flag.String("i", "commands.txt", "output file")
	fname := flag.String("o", defaultfilename, "output file")
	packageName := flag.String("package", pkg(), "package the generated file will be in.")
	flag.Parse()

	// incmds := []cmd{
	// 	cmd{Var: "GITVERSION", Line: "git rev-list --tags --max-count=1"},
	// 	cmd{Var: "GITTAG", Line: "git describe --always --tags ${GITVERSION}"},
	// 	cmd{Var: "GOVERSION", Line: "go version"},
	// 	cmd{Var: "BUILD_NUMBER", echo: true},
	// 	cmd{Var: "BRANCH_NAME", echo: true},
	// }
	incmds, err := cmdRead(*cmdfile)
	if err != nil {
		os.Exit(1)
	}

	outcmds := make(map[string]string, len(incmds)+1) //since we add GOPKG
	outcmds["GOPKG"] = *packageName
	if err != nil {
		os.Exit(1)
	}

	for _, cmd := range incmds {
		val, err := do(cmd)
		if err != nil {
			val = ""
			// os.Exit(1)
		}
		outcmds[cmd.Var] = val
	}

	templ := templatePrelude + envTemplate(incmds) + templatePostlude

	t := template.Must(template.New("templ").Parse(templ))

	var buf bytes.Buffer
	err = t.Execute(&buf, outcmds)
	if err != nil {
		os.Exit(1)
	}
	fmted, err := format.Source(buf.Bytes())
	if err != nil {
		os.Exit(1)
	}

	f, err := os.Create(*fname)
	if err != nil {
		os.Exit(1)
	}
	defer f.Close()
	f.Write(fmted)
}
<commit_msg>Clean up comments<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"go/format"
	"os"
	"os/exec"
	"path"
	"strings"
	"text/template"
)

const (
	defaultfilename = "buildconstants_generated.go"

	templatePrelude = `// Generated by running
//      buildconstants
// DO NOT EDIT

package {{.GOPKG}}

const (
	MustRunBuildConstants = 0
`

	templatePostlude = ")"
)

type cmd struct {
	Var  string
	Line string
	echo bool // get value from env
}

// read commands from a text file.  Each command has its own line
//   and the lines are of the form `VAR = echo $FOO`
//   where $VAR is the env variable and everything after '=' will be evaluated for the value
func cmdRead(cmdfile string) ([]cmd, error) {

	f, err := os.Open(cmdfile)
	if err != nil {
		return []cmd{}, err
	}
	defer f.Close()
	var cmds []cmd
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		splits := strings.Split(scanner.Text(), "=")
		if len(splits) < 2 {
			continue
		}
		line := strings.TrimSpace(strings.Join(splits[1:], " "))
		c := cmd{
			Var:  strings.TrimSpace(splits[0]),
			Line: line,
			echo: strings.HasPrefix(line, "$"),
		}
		cmds = append(cmds, c)
	}
	return cmds, nil
}

// operates on inputs since it needs to be in the same order
func envTemplate(ins []cmd) string {
	var s string
	for _, in := range ins {
		s = s + " " + in.Var + " = \"{{." + in.Var + "}}\"\n"
	}
	return s
}

func do(command cmd) (string, error) {
	expanded := os.Expand(command.Line, os.Getenv)
	var out string
	if command.echo {
		out = expanded
	} else {
		split := strings.Split(expanded, " ")
		cmd := exec.Command(split[0], split[1:]...)
		bout, err := cmd.Output()
		if err != nil {
			return "", err
		}
		out = string(bout)
	}

	val := strings.TrimSpace(out)
	err := os.Setenv(command.Var, val)
	if err != nil {
		return "", err
	}
	return val, nil
}

// get current package's name
func pkg() string {
	pkgCmd := exec.Command("go", "list", ".")
	out, err := pkgCmd.Output()
	if err != nil {
		return "main"
	}
	_, packageName := path.Split(strings.TrimSpace(string(out)))
	return packageName
}

func main() {

	cmdfile := flag.String("i", "commands.txt", "output file")
	fname := flag.String("o", defaultfilename, "output file")
	packageName := flag.String("package", pkg(), "package the generated file will be in.")
	flag.Parse()

	incmds, err := cmdRead(*cmdfile)
	if err != nil {
		os.Exit(1)
	}

	outcmds := make(map[string]string, len(incmds)+1) //since we add GOPKG
	outcmds["GOPKG"] = *packageName

	for _, cmd := range incmds {
		val, err := do(cmd)
		if err != nil {
			val = ""
		}
		outcmds[cmd.Var] = val
	}

	templ := templatePrelude + envTemplate(incmds) + templatePostlude

	t := template.Must(template.New("templ").Parse(templ))

	var buf bytes.Buffer
	err = t.Execute(&buf, outcmds)
	if err != nil {
		os.Exit(1)
	}
	fmted, err := format.Source(buf.Bytes())
	if err != nil {
		os.Exit(1)
	}

	f, err := os.Create(*fname)
	if err != nil {
		os.Exit(1)
	}
	defer f.Close()
	f.Write(fmted)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"os"
	"strconv"
	"strings"
	"text/template"

	"github.com/fatih/color"
)

var REQ = `http://www.omdbapi.com/?t={{.Title}}&y=&plot=short&r=json`

type Movie struct {
	Title      string
	Year       string
	ImdbRating string
	Metascore  string
	Runtime    string
	Genre      string
	Director   string
	Writer     string
	Actors     string
	Plot       string
}

func colorsForRating(rating float64) (*color.Color, *color.Color) {
	var c *color.Color
	if rating < 50 {
		c = color.New(color.FgRed)
	} else if rating < 70 {
		c = color.New(color.FgYellow)
	} else {
		c = color.New(color.FgGreen)
	}
	return c, c.Add(color.Bold)
}

func printMetascore(score string) {
	rating, _ := strconv.Atoi(score)
	color, boldColor := colorsForRating(float64(rating))
	boldColor.Printf("Metascore   : %.0d%% ", rating)
	printRatingBar(float64(rating), color)
	fmt.Println("")
}

func printIMDBRating(r string) {
	rating, _ := strconv.ParseFloat(r, 64)
	rating = rating * 10
	color, boldColor := colorsForRating(rating)
	boldColor.Printf("IMDB Rating : %.0f%% ", rating)
	printRatingBar(rating, color)
	fmt.Println("")
}

func printRatingBar(rating float64, color *color.Color) {
	color.Printf("[")
	for i := 0; float64(i) < math.Floor(rating); i++ {
		color.Printf("=")
	}
	for i := math.Ceil(rating); i < 100; i++ {
		color.Printf(" ")
	}
	color.Printf("]")
}

func printValue(title, value string) {
	boldCyan := color.New(color.FgCyan).Add(color.Bold)
	boldCyan.Printf("%11s : ", title)
	color.Cyan(value)
}

func getMovie(title string) (*Movie, error) {
	data := struct {
		Title string
	}{
		title,
	}

	var URL bytes.Buffer
	tpl, _ := template.New("req").Parse(REQ)
	_ = tpl.Execute(&URL, data)

	resp, err := http.Get(URL.String())
	if err != nil {
		return nil, err
	}
	body, _ := ioutil.ReadAll(resp.Body)

	var movie Movie
	if err = json.Unmarshal(body, &movie); err != nil {
		return nil, err
	}

	return &movie, nil
}

func printMovieInformation(movie *Movie) {
	printValue("Title", movie.Title)
	printValue("Director", movie.Director)
	printValue("Year", movie.Year)
	printValue("Genre", movie.Genre)
	printValue("Actors", movie.Actors)
	printValue("Writer(s)", movie.Writer)

	printIMDBRating(movie.ImdbRating)
	printMetascore(movie.Metascore)

	fmt.Println("\n", movie.Plot)
}

func main() {
	title := strings.Join(os.Args[1:], "+")
	movie, err := getMovie(title)
	if err != nil {
		log.Fatal(err)
	}

	if movie.Title == "" {
		color.Red("Could not find movie titled : %s", title)
		return
	}

	printMovieInformation(movie)
}
<commit_msg>Replace for loop with more concise strings.Repeat<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"os"
	"strconv"
	"strings"
	"text/template"

	"github.com/fatih/color"
)

var REQ = `http://www.omdbapi.com/?t={{.Title}}&y=&plot=short&r=json`

type Movie struct {
	Title      string
	Year       string
	ImdbRating string
	Metascore  string
	Runtime    string
	Genre      string
	Director   string
	Writer     string
	Actors     string
	Plot       string
}

func colorsForRating(rating float64) (*color.Color, *color.Color) {
	var c *color.Color
	if rating < 50 {
		c = color.New(color.FgRed)
	} else if rating < 70 {
		c = color.New(color.FgYellow)
	} else {
		c = color.New(color.FgGreen)
	}
	return c, c.Add(color.Bold)
}

func printMetascore(score string) {
	rating, _ := strconv.Atoi(score)
	color, boldColor := colorsForRating(float64(rating))
	boldColor.Printf("Metascore   : %.0d%% ", rating)
	printRatingBar(float64(rating), color)
	fmt.Println("")
}

func printIMDBRating(r string) {
	rating, _ := strconv.ParseFloat(r, 64)
	rating = rating * 10
	color, boldColor := colorsForRating(rating)
	boldColor.Printf("IMDB Rating : %.0f%% ", rating)
	printRatingBar(rating, color)
	fmt.Println("")
}

func printRatingBar(rating float64, color *color.Color) {
	color.Printf("[")
	color.Printf("%s", strings.Repeat("=", int(math.Floor(rating))))
	color.Printf("%s", strings.Repeat(" ", int(100-math.Floor(rating))))
	color.Printf("]")
}

func printValue(title, value string) {
	boldCyan := color.New(color.FgCyan).Add(color.Bold)
	boldCyan.Printf("%11s : ", title)
	color.Cyan(value)
}

func getMovie(title string) (*Movie, error) {
	data := struct {
		Title string
	}{
		title,
	}

	var URL bytes.Buffer
	tpl, _ := template.New("req").Parse(REQ)
	_ = tpl.Execute(&URL, data)

	resp, err := http.Get(URL.String())
	if err != nil {
		return nil, err
	}
	body, _ := ioutil.ReadAll(resp.Body)

	var movie Movie
	if err = json.Unmarshal(body, &movie); err != nil {
		return nil, err
	}

	return &movie, nil
}

func printMovieInformation(movie *Movie) {
	printValue("Title", movie.Title)
	printValue("Director", movie.Director)
	printValue("Year", movie.Year)
	printValue("Genre", movie.Genre)
	printValue("Actors", movie.Actors)
	printValue("Writer(s)", movie.Writer)

	printIMDBRating(movie.ImdbRating)
	printMetascore(movie.Metascore)

	fmt.Println("\n", movie.Plot)
}

func main() {
	title := strings.Join(os.Args[1:], "+")
	movie, err := getMovie(title)
	if err != nil {
		log.Fatal(err)
	}

	if movie.Title == "" {
		color.Red("Could not find movie titled : %s", title)
		return
	}

	printMovieInformation(movie)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"io"
	"log"
	"net/http"
	"os"
	"strconv"
	"sync"
	"sync/atomic"
	"time"
)

const (
	rootPath = "/"
	userPath = "/user/"
)

type serverContext struct {
	tickets    int64
	dbreads    int32
	computes   int32
	dbmutex    sync.RWMutex
	cachemutex sync.RWMutex
	db         map[string]string
	cache      map[string]string
	mdb        *sql.DB
	realDB     bool
}

func (s *serverContext) getTicket(user string) (string, int, error) {
	if user == "" || user == "errorc" {
		return "", http.StatusNotFound, fmt.Errorf("getTicket(errorc)")
	}
	if user == "errors" {
		return "", http.StatusInternalServerError, fmt.Errorf("getTicket(errors)")
	}

	// try cache
	t1, errCache := s.cacheRead(user)
	if errCache == nil {
		return t1, http.StatusOK, nil
	}

	// try DB
	t2, errDB := s.dbRead(user)
	if errDB == nil {
		s.cacheWrite(user, t2)
		return t2, http.StatusOK, nil
	}

	log.Printf("dbread failure: %v", errDB)

	// try compute
	t3, errCompute := s.compute(user)
	if errCompute == nil {
		s.dbWrite(user, t3)
		return t3, http.StatusOK, nil
	}

	return "", http.StatusInternalServerError, fmt.Errorf("getTicket() failure: %v", errCompute)
}

func (s *serverContext) cacheRead(user string) (string, error) {
	defer s.cachemutex.RUnlock()
	s.cachemutex.RLock()

	t, found := s.cache[user]
	if found {
		return t, nil
	}

	return "", fmt.Errorf("cacheread not found")
}

func (s *serverContext) cacheWrite(user, ticket string) {
	defer s.cachemutex.Unlock()
	s.cachemutex.Lock()

	s.cache[user] = ticket
}

func (s *serverContext) dbRead(user string) (string, error) {
	defer atomic.AddInt32(&s.dbreads, -1)
	r := atomic.AddInt32(&s.dbreads, 1)
	delay := time.Duration(r) * 200 * time.Millisecond

	log.Printf("dbreads=%d delay=%v", r, delay)

	timeout := 2000 * time.Millisecond
	if delay > timeout {
		time.Sleep(timeout)
		return "", fmt.Errorf("dbread timeout: %v", timeout)
	}

	time.Sleep(delay)

	defer s.dbmutex.RUnlock()
	s.dbmutex.RLock()

	if s.realDB {

		rows, errQuery := s.mdb.Query("select ticket from ticket_table where user = ?", user)
		if errQuery != nil {
			return "", fmt.Errorf("mysql dbread query: %v", errQuery)
		}

		defer rows.Close()

		rows.Next()
		var t string
		if errScan := rows.Scan(&t); errScan != nil {
			return "", fmt.Errorf("mysql dbread not found: %v", errScan)
		}

		log.Printf("mysql dbread: user=%s ticket=%s", user, t)
		return t, nil
	}

	t, found := s.db[user]
	if found {
		return t, nil
	}

	return "", fmt.Errorf("dbread not found")
}

func (s *serverContext) dbWrite(user, ticket string) {
	defer s.dbmutex.Unlock()
	s.dbmutex.Lock()

	if s.realDB {

		rows, errQuery := s.mdb.Query("insert into ticket_table (user, ticket) values(?,?) on duplicate key update ticket=?", user, ticket, ticket)
		if errQuery != nil {
			log.Printf("mysql dbwrite query: %v", errQuery)
			return
		}

		defer rows.Close()

		log.Printf("mysql dbwrite: user=%s ticket=%s", user, ticket)

		return
	}

	s.db[user] = ticket
}

func (s *serverContext) compute(user string) (string, error) {

	defer atomic.AddInt32(&s.computes, -1)
	c := atomic.AddInt32(&s.computes, 1)
	delay := time.Duration(c) * 1000 * time.Millisecond

	log.Printf("computes=%d delay=%v", c, delay)

	timeout := 10000 * time.Millisecond
	if delay > timeout {
		time.Sleep(timeout)
		return "", fmt.Errorf("compute timeout: %v", timeout)
	}

	time.Sleep(delay)

	n := atomic.AddInt64(&s.tickets, 1)
	t := strconv.FormatInt(n, 16)
	return t, nil
}

func main() {

	s := &serverContext{
		db:    map[string]string{},
		cache: map[string]string{},
	}

	realdb := os.Getenv("DB_REAL")
	s.realDB = realdb != ""

	if s.realDB {
		user := os.Getenv("DB_USER")
		pass := os.Getenv("DB_PASS")
		host := os.Getenv("DB_HOST")
		dbname := os.Getenv("DB_NAME")

		msg := fmt.Sprintf("DB_REAL='%s' DB_USER='%s' DB_PASS='%s' DB_HOST='%s' DB_NAME='%s'", realdb, user, pass, host, dbname)

		if user == "" || pass == "" || host == "" || dbname == "" {
			log.Fatalf("missing parameter: %s", msg)
		}

		log.Print(msg)

		// username:password@protocol(address)/dbname?param=value
		dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s", user, pass, host, dbname)

		mdb, errDB := sql.Open("mysql", dsn)
		if errDB != nil {
			mdb.Close()
			log.Fatalf("sql open(%s): %v", dsn, errDB)
		}

		s.mdb = mdb
	}

	http.HandleFunc(rootPath, func(w http.ResponseWriter, r *http.Request) { contextHandle(w, r, s, rootHandler) })
	http.HandleFunc(userPath, func(w http.ResponseWriter, r *http.Request) { contextHandle(w, r, s, userHandler) })

	//registerStatic("/www/", currDir)

	addr := ":8080"

	if len(os.Args) > 1 {
		addr = os.Args[1]
	}

	log.Printf("serving on port TCP %s", addr)

	if err := http.ListenAndServe(addr, nil); err != nil {
		log.Panicf("ListenAndServe: %s: %s", addr, err)
	}
}

/*
type staticHandler struct {
	innerHandler http.Handler
}

func registerStatic(path, dir string) {
	http.Handle(path, staticHandler{http.StripPrefix(path, http.FileServer(http.Dir(dir)))})
	log.Printf("registering static directory %s as www path %s", dir, path)
}

func (handler staticHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	log.Printf("staticHandler.ServeHTTP url=%s from=%s", r.URL.Path, r.RemoteAddr)
	handler.innerHandler.ServeHTTP(w, r)
}
*/

func contextHandle(w http.ResponseWriter, r *http.Request, s *serverContext, handler func(http.ResponseWriter, *http.Request, *serverContext)) {
	handler(w, r, s)
}

func rootHandler(w http.ResponseWriter, r *http.Request, s *serverContext) {
	me := "rootHandler"
	msg := fmt.Sprintf("%s: url=%s from=%s", me, r.URL.Path, r.RemoteAddr)
	log.Print(msg)

	code := http.StatusNotFound
	http.Error(w, strconv.Itoa(code)+" - "+http.StatusText(code)+" - "+msg, code)

	//io.WriteString(w, msg)
}

func userHandler(w http.ResponseWriter, r *http.Request, s *serverContext) {
	me := "userHandler"
	msg := fmt.Sprintf("%s: url=%s from=%s", me, r.URL.Path, r.RemoteAddr)
	log.Print(msg)

	user := r.URL.Path[len(userPath):]

	begin := time.Now()

	ticket, code, err := s.getTicket(user)

	elapsed := time.Since(begin)
	e := fmt.Sprintf(" (elapsed=%v) ", elapsed)

	log.Printf("%s: ticket=%s code=%d err=%v"+e, me, ticket, code, err)

	if err != nil {
		http.Error(w, me+": "+strconv.Itoa(code)+" - "+http.StatusText(code)+": "+err.Error()+e, code)
		return
	}

	io.WriteString(w, msg+" ticket="+ticket+e)
}
<commit_msg>Cache skeleton.<commit_after>package main

import (
	"database/sql"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"io"
	"log"
	"net/http"
	"os"
	"strconv"
	"sync"
	"sync/atomic"
	"time"
)

const (
	rootPath = "/"
	userPath = "/user/"
)

type serverContext struct {
	tickets    int64
	dbreads    int32
	computes   int32
	dbmutex    sync.RWMutex
	cachemutex sync.RWMutex
	db         map[string]string
	cache      map[string]string
	mdb        *sql.DB // MySQL
	dynamo     int     // dynamoDB cache
	realDB     bool
	realCache  bool
}

func (s *serverContext) openDB() {

	dbreal := os.Getenv("DB_REAL")
	s.realDB = dbreal != ""

	if !s.realDB {
		return
	}

	user := os.Getenv("DB_USER")
	pass := os.Getenv("DB_PASS")
	host := os.Getenv("DB_HOST")
	dbname := os.Getenv("DB_NAME")

	msg := fmt.Sprintf("DB_REAL='%s' DB_USER='%s' DB_PASS='%s' DB_HOST='%s' DB_NAME='%s'", dbreal, user, pass, host, dbname)

	if user == "" || pass == "" || host == "" || dbname == "" {
		log.Fatalf("missing parameter: %s", msg)
	}

	log.Print(msg)

	// username:password@protocol(address)/dbname?param=value
	dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s", user, pass, host, dbname)

	mdb, errDB := sql.Open("mysql", dsn)
	if errDB != nil {
		mdb.Close()
		log.Fatalf("sql open(%s): %v", dsn, errDB)
	}

	s.mdb = mdb
}

func (s *serverContext) openCache() {

	cachereal := os.Getenv("CACHE_REAL")
	s.realCache = cachereal != ""

	if !s.realCache {
		return
	}

	table := os.Getenv("CACHE_TABLE")

	msg := fmt.Sprintf("CACHE_REAL='%s' CACHE_TABLE='%s'", cachereal, table)

	if table == "" {
		log.Fatalf("missing parameter: %s", msg)
	}

	log.Print(msg)

	/*
		dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s", user, pass, host, dbname)

		mdb, errDB := sql.Open("mysql", dsn)
		if errDB != nil {
			mdb.Close()
			log.Fatalf("sql open(%s): %v", dsn, errDB)
		}

		s.mdb = mdb
	*/
	s.dynamo = 1
}

func (s *serverContext) getTicket(user string) (string, int, error) {
	if user == "" || user == "errorc" {
		return "", http.StatusNotFound, fmt.Errorf("getTicket(errorc)")
	}
	if user == "errors" {
		return "", http.StatusInternalServerError, fmt.Errorf("getTicket(errors)")
	}

	// try cache
	t1, errCache := s.cacheRead(user)
	if errCache == nil {
		return t1, http.StatusOK, nil
	}

	log.Printf("cacheread failure: %v", errCache)

	// try DB
	t2, errDB := s.dbRead(user)
	if errDB == nil {
		s.cacheWrite(user, t2)
		return t2, http.StatusOK, nil
	}

	log.Printf("dbread failure: %v", errDB)

	// try compute
	t3, errCompute := s.compute(user)
	if errCompute == nil {
		s.dbWrite(user, t3)
		return t3, http.StatusOK, nil
	}

	return "", http.StatusInternalServerError, fmt.Errorf("getTicket() failure: %v", errCompute)
}

func (s *serverContext) cacheRead(user string) (string, error) {
	defer s.cachemutex.RUnlock()
	s.cachemutex.RLock()

	if s.realCache {
		return "", fmt.Errorf("dynamoDB cacheread: FIXME WRITEME")
	}

	t, found := s.cache[user]
	if found {
		return t, nil
	}

	return "", fmt.Errorf("cacheread: not found")
}

func (s *serverContext) cacheWrite(user, ticket string) {
	defer s.cachemutex.Unlock()
	s.cachemutex.Lock()

	if s.realCache {
		log.Printf("dynamoDB cachewrite: FIXME WRITEME")
		return
	}

	s.cache[user] = ticket
}

func (s *serverContext) dbRead(user string) (string, error) {
	defer atomic.AddInt32(&s.dbreads, -1)
	r := atomic.AddInt32(&s.dbreads, 1)
	delay := time.Duration(r) * 200 * time.Millisecond

	log.Printf("dbreads=%d delay=%v", r, delay)

	timeout := 2000 * time.Millisecond
	if delay > timeout {
		time.Sleep(timeout)
		return "", fmt.Errorf("dbread timeout: %v", timeout)
	}

	time.Sleep(delay)

	defer s.dbmutex.RUnlock()
	s.dbmutex.RLock()

	if s.realDB {

		rows, errQuery := s.mdb.Query("select ticket from ticket_table where user = ?", user)
		if errQuery != nil {
			return "", fmt.Errorf("mysql dbread query: %v", errQuery)
		}

		defer rows.Close()

		rows.Next()
		var t string
		if errScan := rows.Scan(&t); errScan != nil {
			return "", fmt.Errorf("mysql dbread not found: %v", errScan)
		}

		log.Printf("mysql dbread: user=%s ticket=%s", user, t)
		return t, nil
	}

	t, found := s.db[user]
	if found {
		return t, nil
	}

	return "", fmt.Errorf("dbread not found")
}

func (s *serverContext) dbWrite(user, ticket string) {
	defer s.dbmutex.Unlock()
	s.dbmutex.Lock()

	if s.realDB {

		rows, errQuery := s.mdb.Query("insert into ticket_table (user, ticket) values(?,?) on duplicate key update ticket=?", user, ticket, ticket)
		if errQuery != nil {
			log.Printf("mysql dbwrite query: %v", errQuery)
			return
		}

		defer rows.Close()

		log.Printf("mysql dbwrite: user=%s ticket=%s", user, ticket)

		return
	}

	s.db[user] = ticket
}

func (s *serverContext) compute(user string) (string, error) {

	defer atomic.AddInt32(&s.computes, -1)
	c := atomic.AddInt32(&s.computes, 1)
	delay := time.Duration(c) * 1000 * time.Millisecond

	log.Printf("computes=%d delay=%v", c, delay)

	timeout := 10000 * time.Millisecond
	if delay > timeout {
		time.Sleep(timeout)
		return "", fmt.Errorf("compute timeout: %v", timeout)
	}

	time.Sleep(delay)

	n := atomic.AddInt64(&s.tickets, 1)
	t := strconv.FormatInt(n, 16)
	return t, nil
}

func main() {

	s := &serverContext{
		db:    map[string]string{},
		cache: map[string]string{},
	}

	s.openDB()
	s.openCache()

	http.HandleFunc(rootPath, func(w http.ResponseWriter, r *http.Request) { contextHandle(w, r, s, rootHandler) })
	http.HandleFunc(userPath, func(w http.ResponseWriter, r *http.Request) { contextHandle(w, r, s, userHandler) })

	//registerStatic("/www/", currDir)

	addr := ":8080"

	if len(os.Args) > 1 {
		addr = os.Args[1]
	}

	log.Printf("serving on port TCP %s", addr)

	if err := http.ListenAndServe(addr, nil); err != nil {
		log.Panicf("ListenAndServe: %s: %s", addr, err)
	}
}

/*
type staticHandler struct {
	innerHandler http.Handler
}

func registerStatic(path, dir string) {
	http.Handle(path, staticHandler{http.StripPrefix(path, http.FileServer(http.Dir(dir)))})
	log.Printf("registering static directory %s as www path %s", dir, path)
}

func (handler staticHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	log.Printf("staticHandler.ServeHTTP url=%s from=%s", r.URL.Path, r.RemoteAddr)
	handler.innerHandler.ServeHTTP(w, r)
}
*/

func contextHandle(w http.ResponseWriter, r *http.Request, s *serverContext, handler func(http.ResponseWriter, *http.Request, *serverContext)) {
	handler(w, r, s)
}

func rootHandler(w http.ResponseWriter, r *http.Request, s *serverContext) {
	me := "rootHandler"
	msg := fmt.Sprintf("%s: url=%s from=%s", me, r.URL.Path, r.RemoteAddr)
	log.Print(msg)

	code := http.StatusNotFound
	http.Error(w, strconv.Itoa(code)+" - "+http.StatusText(code)+" - "+msg, code)

	//io.WriteString(w, msg)
}

func userHandler(w http.ResponseWriter, r *http.Request, s *serverContext) {
	me := "userHandler"
	msg := fmt.Sprintf("%s: url=%s from=%s", me, r.URL.Path, r.RemoteAddr)
	log.Print(msg)

	user := r.URL.Path[len(userPath):]

	begin := time.Now()

	ticket, code, err := s.getTicket(user)

	elapsed := time.Since(begin)
	e := fmt.Sprintf(" (elapsed=%v) ", elapsed)

	log.Printf("%s: ticket=%s code=%d err=%v"+e, me, ticket, code, err)

	if err != nil {
		http.Error(w, me+": "+strconv.Itoa(code)+" - "+http.StatusText(code)+": "+err.Error()+e, code)
		return
	}

	io.WriteString(w, msg+" ticket="+ticket+e)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"
	"strconv"
	"strings"

	"github.com/bcollazo/pokenalysis/poke"
	"github.com/bcollazo/pokenalysis/serve"
)

var command string
var gens string
var sort int
var host string
var port int
var machines string

var GEN_BOUNDS = map[string][]int{
	"1": []int{1, 151},
	"2": []int{152, 251},
	"3": []int{252, 386},
	"4": []int{387, 494},
	"5": []int{495, 649},
	"6": []int{650, 721},
	"7": []int{722, 802},
}

func idsFromGens(gens string) []int {
	genKeys := strings.Split(gens, ",")
	ids := []int{}
	for _, k := range genKeys {
		genIds := poke.IntRange(GEN_BOUNDS[k][0], GEN_BOUNDS[k][1])
		ids = append(ids, genIds...)
	}
	return ids
}

func main() {
	flag.StringVar(&command, "command", "histo", "one of either 'histo', 'superhisto', 'goodratio', 'bestpoke', 'work'")
	flag.StringVar(&gens, "gens", "1", "comma-separated generations to include")
	flag.IntVar(&sort, "sort", 0, "sort direction. -1, 0, or 1")

	flag.StringVar(&host, "host", "localhost", "host where this code is running.  Used when command is 'master'")
	flag.IntVar(&port, "port", 3000, "port to use if command is 'master' or 'serve'")
	flag.StringVar(&machines, "machines", "localhost:3000", "comma-separated hostnames")
	flag.Parse()

	isValid := map[string]bool{
		"clean":      true,
		"histo":      true,
		"superhisto": true,
		"goodratio":  true,
		"typecomb":   true,
		"bestpoke":   true,
		"serve":      true,
		"master":     true,
	}
	if !isValid[command] {
		panic("Bad Command")
	}

	if command == "clean" {
		os.RemoveAll(poke.DATA_DIR)
		return
	}

	ids := idsFromGens(gens)
	var stringPort = ":" + strconv.Itoa(port)
	if command == "master" {
		parsed := strings.Split(machines, ",")
		serve.StartMaster(ids, host, stringPort, parsed)
		return
	}

	// Commands that need data ready.
	poke.MaybeDownloadData(ids)
	list := poke.ReadDataFromLocal(ids)
	if command == "histo" {
		poke.Histo(list, sort)
	} else if command == "superhisto" {
		poke.SuperEffectiveHisto(list, sort)
	} else if command == "goodratio" {
		poke.GoodRatios(list, sort)
	} else if command == "typecomb" {
		poke.BestTypeComb(list, sort)
	} else if command == "bestpoke" {
		poke.BestPokemons(list, sort)
	} else if command == "serve" {
		serve.StartWorker(stringPort)
	}
}
<commit_msg>Cleanup comments<commit_after>package main

import (
	"flag"
	"os"
	"strconv"
	"strings"

	"github.com/bcollazo/pokenalysis/poke"
	"github.com/bcollazo/pokenalysis/serve"
)

var command string
var gens string
var sort int
var host string
var port int
var machines string

var GEN_BOUNDS = map[string][]int{
	"1": []int{1, 151},
	"2": []int{152, 251},
	"3": []int{252, 386},
	"4": []int{387, 494},
	"5": []int{495, 649},
	"6": []int{650, 721},
	"7": []int{722, 802},
}

func idsFromGens(gens string) []int {
	genKeys := strings.Split(gens, ",")
	ids := []int{}
	for _, k := range genKeys {
		genIds := poke.IntRange(GEN_BOUNDS[k][0], GEN_BOUNDS[k][1])
		ids = append(ids, genIds...)
	}
	return ids
}

func main() {
	flag.StringVar(&command, "command", "histo", "one of either 'histo', 'superhisto', 'goodratio', 'bestpoke', 'work'")
	flag.StringVar(&gens, "gens", "1", "comma-separated generations to include")
	flag.IntVar(&sort, "sort", 0, "sort direction. -1, 0, or 1")

	flag.StringVar(&host, "host", "localhost", "host where this code is running.  Used when command is 'master'")
	flag.IntVar(&port, "port", 3000, "port to use if command is 'master' or 'serve'")
	flag.StringVar(&machines, "machines", "localhost:3000", "comma-separated hostnames")
	flag.Parse()

	// Validate flags.
	isValid := map[string]bool{
		"clean":      true,
		"histo":      true,
		"superhisto": true,
		"goodratio":  true,
		"typecomb":   true,
		"bestpoke":   true,
		"serve":      true,
		"master":     true,
	}
	if !isValid[command] {
		panic("Bad Command")
	}

	// ===== clean command
	if command == "clean" {
		os.RemoveAll(poke.DATA_DIR)
		return
	}

	// ===== mater command
	ids := idsFromGens(gens)
	var stringPort = ":" + strconv.Itoa(port)
	if command == "master" {
		parsed := strings.Split(machines, ",")
		serve.StartMaster(ids, host, stringPort, parsed)
		return
	}

	// ===== the rest of the commands (require data)
	poke.MaybeDownloadData(ids)
	list := poke.ReadDataFromLocal(ids)
	if command == "histo" {
		poke.Histo(list, sort)
	} else if command == "superhisto" {
		poke.SuperEffectiveHisto(list, sort)
	} else if command == "goodratio" {
		poke.GoodRatios(list, sort)
	} else if command == "typecomb" {
		poke.BestTypeComb(list, sort)
	} else if command == "bestpoke" {
		poke.BestPokemons(list, sort)
	} else if command == "serve" {
		serve.StartWorker(stringPort)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/zshift/goplay/controllers"
	"log"
	"net/http"

	"github.com/emicklei/go-restful"
)

func main() {
	ws := new(restful.WebService)
	registerRoutes(ws)
	restful.Add(ws)

	log.Println("Starting server")
	http.ListenAndServe(":8080", nil)
}

func registerRoutes(ws *restful.WebService) {
	log.Println("Registering routes")
	healthController := controllers.NewHealthController()
	healthController.RegisterRoutes(ws)
}
<commit_msg>Logging when server fails to start<commit_after>package main

import (
	"github.com/zshift/goplay/controllers"
	"log"
	"net/http"

	"github.com/emicklei/go-restful"
)

func main() {
	ws := new(restful.WebService)
	registerRoutes(ws)
	restful.Add(ws)

	log.Println("Starting server")

	err := http.ListenAndServe(":8080", nil)

	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}

func registerRoutes(ws *restful.WebService) {
	log.Println("Registering routes")

	healthController := controllers.NewHealthController()
	healthController.RegisterRoutes(ws)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
	Package fmt implements formatted I/O with functions analogous
	to C's printf and scanf.  The format 'verbs' are derived from C's but
	are simpler.

	Printing

	The verbs:

	General:
		%v	the value in a default format.
			when printing structs, the plus flag (%+v) adds field names
		%#v	a Go-syntax representation of the value
		%T	a Go-syntax representation of the type of the value
		%%	a literal percent sign; consumes no value

	Boolean:
		%t	the word true or false
	Integer:
		%b	base 2
		%c	the character represented by the corresponding Unicode code point
		%d	base 10
		%o	base 8
		%q	a single-quoted character literal safely escaped with Go syntax.
		%x	base 16, with lower-case letters for a-f
		%X	base 16, with upper-case letters for A-F
		%U	Unicode format: U+1234; same as "U+%04X"
	Floating-point and complex constituents:
		%b	decimalless scientific notation with exponent a power of two,
			in the manner of strconv.FormatFloat with the 'b' format,
			e.g. -123456p-78
		%e	scientific notation, e.g. -1234.456e+78
		%E	scientific notation, e.g. -1234.456E+78
		%f	decimal point but no exponent, e.g. 123.456
		%F	synonym for %f
		%g	whichever of %e or %f produces more compact output
		%G	whichever of %E or %f produces more compact output
	String and slice of bytes:
		%s	the uninterpreted bytes of the string or slice
		%q	a double-quoted string safely escaped with Go syntax
		%x	base 16, lower-case, two characters per byte
		%X	base 16, upper-case, two characters per byte
	Pointer:
		%p	base 16 notation, with leading 0x

	There is no 'u' flag.  Integers are printed unsigned if they have unsigned type.
	Similarly, there is no need to specify the size of the operand (int8, int64).

	Width is specified by an optional decimal number immediately following the verb.
	If absent, the width is whatever is necessary to represent the value.
	Precision is specified after the (optional) width by a period followed by a
	decimal number. If no period is present, a default precision is used.
	A period with no following number specifies a precision of zero.
	Examples:
		%f:    default width, default precision
		%9f    width 9, default precision
		%.2f   default width, precision 2
		%9.2f  width 9, precision 2
		%9.f   width 9, precision 0

	Width and precision are measured in units of Unicode code points.
	(This differs from C's printf where the units are numbers
	of bytes.) Either or both of the flags may be replaced with the
	character '*', causing their values to be obtained from the next
	operand, which must be of type int.

	For most values, width is the minimum number of characters to output,
	padding the formatted form with spaces if necessary.
	For strings, precision is the maximum number of characters to output,
	truncating if necessary.

	For floating-point values, width sets the minimum width of the field and
	precision sets the number of places after the decimal, if appropriate,
	except that for %g/%G it sets the total number of digits. For example,
	given 123.45 the format %6.2f prints 123.45 while %.4g prints 123.5.
	The default precision for %e and %f is 6; for %g it is the smallest
	number of digits necessary to identify the value uniquely.

	For complex numbers, the width and precision apply to the two
	components independently and the result is parenthsized, so %f applied
	to 1.2+3.4i produces (1.200000+3.400000i).

	Other flags:
		+	always print a sign for numeric values;
			guarantee ASCII-only output for %q (%+q)
		-	pad with spaces on the right rather than the left (left-justify the field)
		#	alternate format: add leading 0 for octal (%#o), 0x for hex (%#x);
			0X for hex (%#X); suppress 0x for %p (%#p);
			for %q, print a raw (backquoted) string if strconv.CanBackquote
			returns true;
			write e.g. U+0078 'x' if the character is printable for %U (%#U).
		' '	(space) leave a space for elided sign in numbers (% d);
			put spaces between bytes printing strings or slices in hex (% x, % X)
		0	pad with leading zeros rather than spaces;
			for numbers, this moves the padding after the sign

	Flags are ignored by verbs that do not expect them.
	For example there is no alternate decimal format, so %#d and %d
	behave identically.

	For each Printf-like function, there is also a Print function
	that takes no format and is equivalent to saying %v for every
	operand.  Another variant Println inserts blanks between
	operands and appends a newline.

	Regardless of the verb, if an operand is an interface value,
	the internal concrete value is used, not the interface itself.
	Thus:
		var i interface{} = 23
		fmt.Printf("%v\n", i)
	will print 23.

	Except when printed using the the verbs %T and %p, special
	formatting considerations apply for operands that implement
	certain interfaces. In order of application:

	1. If an operand implements the Formatter interface, it will
	be invoked. Formatter provides fine control of formatting.

	2. If the %v verb is used with the # flag (%#v) and the operand
	implements the GoStringer interface, that will be invoked.

	If the format (which is implicitly %v for Println etc.) is valid
	for a string (%s %q %v %x %X), the following two rules apply:

	3. If an operand implements the error interface, the Error method
	will be invoked to convert the object to a string, which will then
	be formatted as required by the verb (if any).

	4. If an operand implements method String() string, that method
	will be invoked to convert the object to a string, which will then
	be formatted as required by the verb (if any).

	For compound operands such as slices and structs, the format
	applies to the elements of each operand, recursively, not to the
	operand as a whole. Thus %q will quote each element of a slice
	of strings, and %6.2f will control formatting for each element
	of a floating-point array.

	To avoid recursion in cases such as
		type X string
		func (x X) String() string { return Sprintf("<%s>", x) }
	convert the value before recurring:
		func (x X) String() string { return Sprintf("<%s>", string(x)) }

	Explicit argument indexes:

	In Printf, Sprintf, and Fprintf, the default behavior is for each
	formatting verb to format successive arguments passed in the call.
	However, the notation [n] immediately before the verb indicates that the
	nth one-indexed argument is to be formatted instead. The same notation
	before a '*' for a width or precision selects the argument index holding
	the value. After processing a bracketed expression [n], arguments n+1,
	n+2, etc. will be processed unless otherwise directed.

	For example,
		fmt.Sprintf("%[2]d %[1]d\n", 11, 22)
	will yield "22, 11", while
		fmt.Sprintf("%[3]*.[2]*[1]f", 12.0, 2, 6),
	equivalent to
		fmt.Sprintf("%6.2f", 12.0),
	will yield " 12.00". Because an explicit index affects subsequent verbs,
	this notation can be used to print the same values multiple times
	by resetting the index for the first argument to be repeated:
		fmt.Sprintf("%d %d %#[1]x %#x", 16, 17)
	will yield "16 17 0x10 0x11".

	Format errors:

	If an invalid argument is given for a verb, such as providing
	a string to %d, the generated string will contain a
	description of the problem, as in these examples:

		Wrong type or unknown verb: %!verb(type=value)
			Printf("%d", hi):          %!d(string=hi)
		Too many arguments: %!(EXTRA type=value)
			Printf("hi", "guys"):      hi%!(EXTRA string=guys)
		Too few arguments: %!verb(MISSING)
			Printf("hi%d"):            hi %!d(MISSING)
		Non-int for width or precision: %!(BADWIDTH) or %!(BADPREC)
			Printf("%*s", 4.5, "hi"):  %!(BADWIDTH)hi
			Printf("%.*s", 4.5, "hi"): %!(BADPREC)hi
		Invalid or invalid use of argument index: %!(BADINDEX)
			Printf("%*[2]d", 7):       %!d(BADINDEX)
			Printf("%.[2]d", 7):       %!d(BADINDEX)

	All errors begin with the string "%!" followed sometimes
	by a single character (the verb) and end with a parenthesized
	description.

	If an Error or String method triggers a panic when called by a
	print routine, the fmt package reformats the error message
	from the panic, decorating it with an indication that it came
	through the fmt package.  For example, if a String method
	calls panic("bad"), the resulting formatted message will look
	like
		%!s(PANIC=bad)

	The %!s just shows the print verb in use when the failure
	occurred.

	Scanning

	An analogous set of functions scans formatted text to yield
	values.  Scan, Scanf and Scanln read from os.Stdin; Fscan,
	Fscanf and Fscanln read from a specified io.Reader; Sscan,
	Sscanf and Sscanln read from an argument string.  Scanln,
	Fscanln and Sscanln stop scanning at a newline and require that
	the items be followed by one; Scanf, Fscanf and Sscanf require
	newlines in the input to match newlines in the format; the other
	routines treat newlines as spaces.

	Scanf, Fscanf, and Sscanf parse the arguments according to a
	format string, analogous to that of Printf.  For example, %x
	will scan an integer as a hexadecimal number, and %v will scan
	the default representation format for the value.

	The formats behave analogously to those of Printf with the
	following exceptions:

		%p is not implemented
		%T is not implemented
		%e %E %f %F %g %G are all equivalent and scan any floating point or complex value
		%s and %v on strings scan a space-delimited token
		Flags # and + are not implemented.

	The familiar base-setting prefixes 0 (octal) and 0x
	(hexadecimal) are accepted when scanning integers without a
	format or with the %v verb.

	Width is interpreted in the input text (%5s means at most
	five runes of input will be read to scan a string) but there
	is no syntax for scanning with a precision (no %5.2f, just
	%5f).

	When scanning with a format, all non-empty runs of space
	characters (except newline) are equivalent to a single
	space in both the format and the input.  With that proviso,
	text in the format string must match the input text; scanning
	stops if it does not, with the return value of the function
	indicating the number of arguments scanned.

	In all the scanning functions, a carriage return followed
	immediately by a newline is treated as a plain newline
	(\r\n means the same as \n).

	In all the scanning functions, if an operand implements method
	Scan (that is, it implements the Scanner interface) that
	method will be used to scan the text for that operand.  Also,
	if the number of arguments scanned is less than the number of
	arguments provided, an error is returned.

	All arguments to be scanned must be either pointers to basic
	types or implementations of the Scanner interface.

	Note: Fscan etc. can read one character (rune) past the input
	they return, which means that a loop calling a scan routine
	may skip some of the input.  This is usually a problem only
	when there is no space between input values.  If the reader
	provided to Fscan implements ReadRune, that method will be used
	to read characters.  If the reader also implements UnreadRune,
	that method will be used to save the character and successive
	calls will not lose data.  To attach ReadRune and UnreadRune
	methods to a reader without that capability, use
	bufio.NewReader.
*/
package fmt
<commit_msg>fmt: fix typo in help doc<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
	Package fmt implements formatted I/O with functions analogous
	to C's printf and scanf.  The format 'verbs' are derived from C's but
	are simpler.

	Printing

	The verbs:

	General:
		%v	the value in a default format.
			when printing structs, the plus flag (%+v) adds field names
		%#v	a Go-syntax representation of the value
		%T	a Go-syntax representation of the type of the value
		%%	a literal percent sign; consumes no value

	Boolean:
		%t	the word true or false
	Integer:
		%b	base 2
		%c	the character represented by the corresponding Unicode code point
		%d	base 10
		%o	base 8
		%q	a single-quoted character literal safely escaped with Go syntax.
		%x	base 16, with lower-case letters for a-f
		%X	base 16, with upper-case letters for A-F
		%U	Unicode format: U+1234; same as "U+%04X"
	Floating-point and complex constituents:
		%b	decimalless scientific notation with exponent a power of two,
			in the manner of strconv.FormatFloat with the 'b' format,
			e.g. -123456p-78
		%e	scientific notation, e.g. -1234.456e+78
		%E	scientific notation, e.g. -1234.456E+78
		%f	decimal point but no exponent, e.g. 123.456
		%F	synonym for %f
		%g	whichever of %e or %f produces more compact output
		%G	whichever of %E or %f produces more compact output
	String and slice of bytes:
		%s	the uninterpreted bytes of the string or slice
		%q	a double-quoted string safely escaped with Go syntax
		%x	base 16, lower-case, two characters per byte
		%X	base 16, upper-case, two characters per byte
	Pointer:
		%p	base 16 notation, with leading 0x

	There is no 'u' flag.  Integers are printed unsigned if they have unsigned type.
	Similarly, there is no need to specify the size of the operand (int8, int64).

	Width is specified by an optional decimal number immediately following the verb.
	If absent, the width is whatever is necessary to represent the value.
	Precision is specified after the (optional) width by a period followed by a
	decimal number. If no period is present, a default precision is used.
	A period with no following number specifies a precision of zero.
	Examples:
		%f:    default width, default precision
		%9f    width 9, default precision
		%.2f   default width, precision 2
		%9.2f  width 9, precision 2
		%9.f   width 9, precision 0

	Width and precision are measured in units of Unicode code points.
	(This differs from C's printf where the units are numbers
	of bytes.) Either or both of the flags may be replaced with the
	character '*', causing their values to be obtained from the next
	operand, which must be of type int.

	For most values, width is the minimum number of characters to output,
	padding the formatted form with spaces if necessary.
	For strings, precision is the maximum number of characters to output,
	truncating if necessary.

	For floating-point values, width sets the minimum width of the field and
	precision sets the number of places after the decimal, if appropriate,
	except that for %g/%G it sets the total number of digits. For example,
	given 123.45 the format %6.2f prints 123.45 while %.4g prints 123.5.
	The default precision for %e and %f is 6; for %g it is the smallest
	number of digits necessary to identify the value uniquely.

	For complex numbers, the width and precision apply to the two
	components independently and the result is parenthsized, so %f applied
	to 1.2+3.4i produces (1.200000+3.400000i).

	Other flags:
		+	always print a sign for numeric values;
			guarantee ASCII-only output for %q (%+q)
		-	pad with spaces on the right rather than the left (left-justify the field)
		#	alternate format: add leading 0 for octal (%#o), 0x for hex (%#x);
			0X for hex (%#X); suppress 0x for %p (%#p);
			for %q, print a raw (backquoted) string if strconv.CanBackquote
			returns true;
			write e.g. U+0078 'x' if the character is printable for %U (%#U).
		' '	(space) leave a space for elided sign in numbers (% d);
			put spaces between bytes printing strings or slices in hex (% x, % X)
		0	pad with leading zeros rather than spaces;
			for numbers, this moves the padding after the sign

	Flags are ignored by verbs that do not expect them.
	For example there is no alternate decimal format, so %#d and %d
	behave identically.

	For each Printf-like function, there is also a Print function
	that takes no format and is equivalent to saying %v for every
	operand.  Another variant Println inserts blanks between
	operands and appends a newline.

	Regardless of the verb, if an operand is an interface value,
	the internal concrete value is used, not the interface itself.
	Thus:
		var i interface{} = 23
		fmt.Printf("%v\n", i)
	will print 23.

	Except when printed using the verbs %T and %p, special
	formatting considerations apply for operands that implement
	certain interfaces. In order of application:

	1. If an operand implements the Formatter interface, it will
	be invoked. Formatter provides fine control of formatting.

	2. If the %v verb is used with the # flag (%#v) and the operand
	implements the GoStringer interface, that will be invoked.

	If the format (which is implicitly %v for Println etc.) is valid
	for a string (%s %q %v %x %X), the following two rules apply:

	3. If an operand implements the error interface, the Error method
	will be invoked to convert the object to a string, which will then
	be formatted as required by the verb (if any).

	4. If an operand implements method String() string, that method
	will be invoked to convert the object to a string, which will then
	be formatted as required by the verb (if any).

	For compound operands such as slices and structs, the format
	applies to the elements of each operand, recursively, not to the
	operand as a whole. Thus %q will quote each element of a slice
	of strings, and %6.2f will control formatting for each element
	of a floating-point array.

	To avoid recursion in cases such as
		type X string
		func (x X) String() string { return Sprintf("<%s>", x) }
	convert the value before recurring:
		func (x X) String() string { return Sprintf("<%s>", string(x)) }

	Explicit argument indexes:

	In Printf, Sprintf, and Fprintf, the default behavior is for each
	formatting verb to format successive arguments passed in the call.
	However, the notation [n] immediately before the verb indicates that the
	nth one-indexed argument is to be formatted instead. The same notation
	before a '*' for a width or precision selects the argument index holding
	the value. After processing a bracketed expression [n], arguments n+1,
	n+2, etc. will be processed unless otherwise directed.

	For example,
		fmt.Sprintf("%[2]d %[1]d\n", 11, 22)
	will yield "22, 11", while
		fmt.Sprintf("%[3]*.[2]*[1]f", 12.0, 2, 6),
	equivalent to
		fmt.Sprintf("%6.2f", 12.0),
	will yield " 12.00". Because an explicit index affects subsequent verbs,
	this notation can be used to print the same values multiple times
	by resetting the index for the first argument to be repeated:
		fmt.Sprintf("%d %d %#[1]x %#x", 16, 17)
	will yield "16 17 0x10 0x11".

	Format errors:

	If an invalid argument is given for a verb, such as providing
	a string to %d, the generated string will contain a
	description of the problem, as in these examples:

		Wrong type or unknown verb: %!verb(type=value)
			Printf("%d", hi):          %!d(string=hi)
		Too many arguments: %!(EXTRA type=value)
			Printf("hi", "guys"):      hi%!(EXTRA string=guys)
		Too few arguments: %!verb(MISSING)
			Printf("hi%d"):            hi %!d(MISSING)
		Non-int for width or precision: %!(BADWIDTH) or %!(BADPREC)
			Printf("%*s", 4.5, "hi"):  %!(BADWIDTH)hi
			Printf("%.*s", 4.5, "hi"): %!(BADPREC)hi
		Invalid or invalid use of argument index: %!(BADINDEX)
			Printf("%*[2]d", 7):       %!d(BADINDEX)
			Printf("%.[2]d", 7):       %!d(BADINDEX)

	All errors begin with the string "%!" followed sometimes
	by a single character (the verb) and end with a parenthesized
	description.

	If an Error or String method triggers a panic when called by a
	print routine, the fmt package reformats the error message
	from the panic, decorating it with an indication that it came
	through the fmt package.  For example, if a String method
	calls panic("bad"), the resulting formatted message will look
	like
		%!s(PANIC=bad)

	The %!s just shows the print verb in use when the failure
	occurred.

	Scanning

	An analogous set of functions scans formatted text to yield
	values.  Scan, Scanf and Scanln read from os.Stdin; Fscan,
	Fscanf and Fscanln read from a specified io.Reader; Sscan,
	Sscanf and Sscanln read from an argument string.  Scanln,
	Fscanln and Sscanln stop scanning at a newline and require that
	the items be followed by one; Scanf, Fscanf and Sscanf require
	newlines in the input to match newlines in the format; the other
	routines treat newlines as spaces.

	Scanf, Fscanf, and Sscanf parse the arguments according to a
	format string, analogous to that of Printf.  For example, %x
	will scan an integer as a hexadecimal number, and %v will scan
	the default representation format for the value.

	The formats behave analogously to those of Printf with the
	following exceptions:

		%p is not implemented
		%T is not implemented
		%e %E %f %F %g %G are all equivalent and scan any floating point or complex value
		%s and %v on strings scan a space-delimited token
		Flags # and + are not implemented.

	The familiar base-setting prefixes 0 (octal) and 0x
	(hexadecimal) are accepted when scanning integers without a
	format or with the %v verb.

	Width is interpreted in the input text (%5s means at most
	five runes of input will be read to scan a string) but there
	is no syntax for scanning with a precision (no %5.2f, just
	%5f).

	When scanning with a format, all non-empty runs of space
	characters (except newline) are equivalent to a single
	space in both the format and the input.  With that proviso,
	text in the format string must match the input text; scanning
	stops if it does not, with the return value of the function
	indicating the number of arguments scanned.

	In all the scanning functions, a carriage return followed
	immediately by a newline is treated as a plain newline
	(\r\n means the same as \n).

	In all the scanning functions, if an operand implements method
	Scan (that is, it implements the Scanner interface) that
	method will be used to scan the text for that operand.  Also,
	if the number of arguments scanned is less than the number of
	arguments provided, an error is returned.

	All arguments to be scanned must be either pointers to basic
	types or implementations of the Scanner interface.

	Note: Fscan etc. can read one character (rune) past the input
	they return, which means that a loop calling a scan routine
	may skip some of the input.  This is usually a problem only
	when there is no space between input values.  If the reader
	provided to Fscan implements ReadRune, that method will be used
	to read characters.  If the reader also implements UnreadRune,
	that method will be used to save the character and successive
	calls will not lose data.  To attach ReadRune and UnreadRune
	methods to a reader without that capability, use
	bufio.NewReader.
*/
package fmt
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"sync"
	"sync/atomic"

	"github.com/caelifer/dups/balancer"
	"github.com/caelifer/dups/fstree"
	"github.com/caelifer/dups/mapreduce"
)

// dup type describes found duplicate file
type dup struct {
	Count int    // Number of identical copies for the hash
	Size  int64  // File size
	Hash  string // Crypto signature
	Path  string // Paths with matching signatures
}

// Value implements mapreduce.Value interface
func (d dup) Value() interface{} {
	return d
}

func (d dup) String() string {
	return fmt.Sprintf("%s:%d:%d:%q", d.Hash, d.Count, d.Size, d.Path)
}

// Global stats for activity report
var stats struct {
	TotlalNodes      uint64
	TotalCopies      uint64
	TotalWastedSpace uint64
}

// Flags
var (
	cpuprofile      = flag.String("cpuprofile", "", "write cpu profile to file")
	memprofile      = flag.String("memprofile", "", "write memory profile to this file")
	maxWorkerNumber = flag.Int("jobs", runtime.NumCPU(), "Number of parallel jobs")
)

// Global pool manager interfaced via WorkQueue
var WorkQueue = balancer.NewWorkQueue(*maxWorkerNumber)

func main() {
	// First parse flags
	flag.Parse()

	// Prep runtime to use the maxWorkerNumber real threads
	runtime.GOMAXPROCS(*maxWorkerNumber)

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal(err)
		}
		defer func() {
			pprof.WriteHeapProfile(f)
			f.Close()
		}()
	}

	// Process command line params
	paths := flag.Args()

	if len(paths) == 0 {
		// Default is current directory
		paths = []string{"."}
	}

	// Channel interfaces between Map() and Reduce() functions
	var keyValChan <-chan mapreduce.KeyValue
	var valChan <-chan mapreduce.Value

	// Start map-reduce and remove duplicate path nodes
	keyValChan = mapreduce.Map(makeNodeMapFnWithPaths(paths))
	valChan = mapreduce.Reduce(keyValChan, reduceByFileName)

	// Map by filesize
	keyValChan = mapreduce.Map(makeFileSizeMapFnFrom(valChan))
	valChan = mapreduce.Reduce(keyValChan, reduceByFileSize)

	// Map by fast SHA1 hash (first 1024 bytes)
	keyValChan = mapreduce.Map(makeFileHashMapFnFrom(valChan, true))
	valChan = mapreduce.Reduce(keyValChan, reduceByHash)

	// Map by SHA1 hash (full file hashing)
	keyValChan = mapreduce.Map(makeFileHashMapFnFrom(valChan, false))
	valChan = mapreduce.Reduce(keyValChan, reduceByHash)

	// Final reduce before reporting
	dups := make(chan dup)
	go func(out chan<- dup) {
		byHash := make(map[string][]*Node)

		f := true
		for x := range valChan {
			if f {
				log.Println("Started final reduce before reporting stage")
				f = false
			}
			n := x.Value().(*Node) // Type assert

			// Aggregate
			if v, ok := byHash[n.Hash]; ok {
				// Found node with the same file size
				byHash[n.Hash] = append(v, n)
			} else {
				byHash[n.Hash] = []*Node{n}
			}
		}

		// Reduce
		for hash, nodes := range byHash {
			count := len(nodes)
			if count > 1 {
				// Update free size stats
				atomic.AddUint64(&stats.TotalWastedSpace, uint64(nodes[0].Size*int64(count-1)))

				for _, node := range nodes {
					// Update dups number stats
					atomic.AddUint64(&stats.TotalCopies, 1)
					out <- dup{Count: count, Size: node.Size, Hash: hash, Path: node.Path}
				}
			}
		}
		close(out)
	}(dups)

	// Report
	for d := range dups {
		fmt.Println(d)
	}
	// Stats report
	log.Printf("Examined %d files, found %d dups, total wasted space %.2fGB\n",
		stats.TotlalNodes, stats.TotalCopies, float64(stats.TotalWastedSpace)/(1024*1024*1024))
}

// makeNodeMapFnWithPaths
func makeNodeMapFnWithPaths(paths []string) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue) {
		// Process all command line paths
		f := true
		for _, path_ := range paths {
			if f {
				log.Println("Started Node mapping stage")
				f = false
			}
			// err := filepath.Walk(path_, func(path string, info os.FileInfo, err error) error {
			err := fstree.Walk(WorkQueue, path_, func(path string, info os.FileInfo, err error) error {
				// Handle passthrough error
				if err != nil {
					log.Println("WARN", err)
					return nil
				}

				// Only process simple files
				if IsFile(info) {
					size := info.Size()
					out <- mapreduce.NewKVType(
						mapreduce.KeyTypeFromString(path),
						&Node{Path: path, Size: size},
					)
					// Increase seen files counter
					atomic.AddUint64(&stats.TotlalNodes, 1)
				}
				return nil
			})

			if err != nil {
				log.Fatal(err)
			}
		}
	}
}

// reduceByFileName custom function remove nodes with duplicate paths
func reduceByFileName(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
	byName := make(map[mapreduce.KeyType]*Node)

	f := true
	for x := range in {
		if f {
			log.Println("Started reduce by file name stage")
			f = false
		}
		path := x.Key()           // Get key
		node := x.Value().(*Node) // Assert type

		// Add values to the map for aggregation, skip nodes with the same path
		if _, ok := byName[path]; !ok {
			byName[path] = node
			out <- node // send first copy
		}
	}
}

// Very simple function to map nodes by size
func makeFileSizeMapFnFrom(in <-chan mapreduce.Value) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue) {
		f := true
		for x := range in {
			if f {
				log.Println("Started maping Node size stage")
				f = false
			}
			node := x.Value().(*Node) // Assert type

			out <- mapreduce.NewKVType(
				mapreduce.KeyTypeFromInt64(node.Size),
				node,
			)
		}
	}
}

// reduceByFileSize custom function to filter files by size
func reduceByFileSize(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
	bySize := make(map[mapreduce.KeyType][]*Node)

	f := true
	for x := range in {
		if f {
			log.Println("Started reducing by Node size stage")
			f = false
		}
		size := x.Key()           // Get key
		node := x.Value().(*Node) // Assert type

		// Add values to the map for aggregation
		if v, ok := bySize[size]; ok {
			// Found node with the same file size
			if len(v) == 1 {
				// First time we found duplicate, send first node too
				out <- v[0]
			}
			bySize[size] = append(v, node)
			// Send duplicate downstream
			out <- node
		} else {
			// Store first copy
			bySize[size] = []*Node{node}
		}
	}
}

func makeFileHashMapFnFrom(in <-chan mapreduce.Value, fast bool) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue) {
		hashType := "full"
		if fast {
			hashType = "fast"
		}
		var wg sync.WaitGroup

		f := true
		for x := range in {
			if f {
				log.Printf("Started Node mapping by %s SHA1 hash stage\n", hashType)
				f = false
			}
			node := x.Value().(*Node) // Assert type

			// Add to wait group
			wg.Add(1)

			// Calculate hash using balancer
			go func(n *Node) {
				WorkQueue <- func() {
					defer wg.Done() // Signal done

					// Default hash value for files < 1024 if fast is true
					hash := "0h"

					// Little optimization to avoid calculating fast hash on small files
					if !fast || node.Size > blockSize {
						// Always calculate hash if fast == false or file size > blockSize
						hash = n.calculateHash(fast) // Fast hash calculation - SHA1 of first 1024 bytes
					}

					// Don't process files for which we failed to calculate SHA1 hash
					if hash == "" {
						// log.Printf("WARN Unable calculate SHA1 hash for %q\n", node.Path)
						return
					}

					out <- mapreduce.NewKVType(
						mapreduce.KeyTypeFromString(hash),
						n,
					)
				}
			}(node)
		}
		// Wait for all results be submitted
		wg.Wait()
	}
}

func reduceByHash(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
	byHash := make(map[mapreduce.KeyType][]*Node)

	f := true
	for x := range in {
		if f {
			log.Println("Started reducing by Node SHA1 hash stage")
			f = false
		}
		hash := x.Key()
		node := x.Value().(*Node) // Assert type

		// Add hash value to a node
		node.Hash = hash.String()

		if v, ok := byHash[hash]; ok {
			// Found node with the same SHA1 hash
			// Send out aggregeted results
			if len(v) == 1 {
				// First time we found duplicate, send first node too
				out <- v[0]
			}
			byHash[hash] = append(v, node)
			out <- node
		} else {
			byHash[hash] = []*Node{node}
		}
	}
}

func IsFile(fi os.FileInfo) bool {
	return fi.Mode()&os.ModeType == 0
}
<commit_msg> - Removed extra debug logging<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"sync"
	"sync/atomic"

	"github.com/caelifer/dups/balancer"
	"github.com/caelifer/dups/fstree"
	"github.com/caelifer/dups/mapreduce"
)

// dup type describes found duplicate file
type dup struct {
	Count int    // Number of identical copies for the hash
	Size  int64  // File size
	Hash  string // Crypto signature
	Path  string // Paths with matching signatures
}

// Value implements mapreduce.Value interface
func (d dup) Value() interface{} {
	return d
}

func (d dup) String() string {
	return fmt.Sprintf("%s:%d:%d:%q", d.Hash, d.Count, d.Size, d.Path)
}

// Global stats for activity report
var stats struct {
	TotlalNodes      uint64
	TotalCopies      uint64
	TotalWastedSpace uint64
}

// Flags
var (
	cpuprofile      = flag.String("cpuprofile", "", "write cpu profile to file")
	memprofile      = flag.String("memprofile", "", "write memory profile to this file")
	maxWorkerNumber = flag.Int("jobs", runtime.NumCPU(), "Number of parallel jobs")
)

// Global pool manager interfaced via WorkQueue
var WorkQueue = balancer.NewWorkQueue(*maxWorkerNumber)

func main() {
	// First parse flags
	flag.Parse()

	// Prep runtime to use the maxWorkerNumber real threads
	runtime.GOMAXPROCS(*maxWorkerNumber)

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal(err)
		}
		defer func() {
			pprof.WriteHeapProfile(f)
			f.Close()
		}()
	}

	// Process command line params
	paths := flag.Args()

	if len(paths) == 0 {
		// Default is current directory
		paths = []string{"."}
	}

	// Channel interfaces between Map() and Reduce() functions
	var keyValChan <-chan mapreduce.KeyValue
	var valChan <-chan mapreduce.Value

	// Start map-reduce and remove duplicate path nodes
	keyValChan = mapreduce.Map(makeNodeMapFnWithPaths(paths))
	valChan = mapreduce.Reduce(keyValChan, reduceByFileName)

	// Map by filesize
	keyValChan = mapreduce.Map(makeFileSizeMapFnFrom(valChan))
	valChan = mapreduce.Reduce(keyValChan, reduceByFileSize)

	// Map by fast SHA1 hash (first 1024 bytes)
	keyValChan = mapreduce.Map(makeFileHashMapFnFrom(valChan, true))
	valChan = mapreduce.Reduce(keyValChan, reduceByHash)

	// Map by SHA1 hash (full file hashing)
	keyValChan = mapreduce.Map(makeFileHashMapFnFrom(valChan, false))
	valChan = mapreduce.Reduce(keyValChan, reduceByHash)

	// Final reduce before reporting
	dups := make(chan dup)
	go func(out chan<- dup) {
		byHash := make(map[string][]*Node)

		for x := range valChan {
			n := x.Value().(*Node) // Type assert

			// Aggregate
			if v, ok := byHash[n.Hash]; ok {
				// Found node with the same file size
				byHash[n.Hash] = append(v, n)
			} else {
				byHash[n.Hash] = []*Node{n}
			}
		}

		// Reduce
		for hash, nodes := range byHash {
			count := len(nodes)
			if count > 1 {
				// Update free size stats
				atomic.AddUint64(&stats.TotalWastedSpace, uint64(nodes[0].Size*int64(count-1)))

				for _, node := range nodes {
					// Update dups number stats
					atomic.AddUint64(&stats.TotalCopies, 1)
					out <- dup{Count: count, Size: node.Size, Hash: hash, Path: node.Path}
				}
			}
		}
		close(out)
	}(dups)

	// Report
	for d := range dups {
		fmt.Println(d)
	}
	// Stats report
	log.Printf("Examined %d files, found %d dups, total wasted space %.2fGB\n",
		stats.TotlalNodes, stats.TotalCopies, float64(stats.TotalWastedSpace)/(1024*1024*1024))
}

// makeNodeMapFnWithPaths
func makeNodeMapFnWithPaths(paths []string) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue) {
		// Process all command line paths
		for _, path_ := range paths {
			// err := filepath.Walk(path_, func(path string, info os.FileInfo, err error) error {
			err := fstree.Walk(WorkQueue, path_, func(path string, info os.FileInfo, err error) error {
				// Handle passthrough error
				if err != nil {
					log.Println("WARN", err)
					return nil
				}

				// Only process simple files
				if IsFile(info) {
					size := info.Size()
					out <- mapreduce.NewKVType(
						mapreduce.KeyTypeFromString(path),
						&Node{Path: path, Size: size},
					)
					// Increase seen files counter
					atomic.AddUint64(&stats.TotlalNodes, 1)
				}
				return nil
			})

			if err != nil {
				log.Fatal(err)
			}
		}
	}
}

// reduceByFileName custom function remove nodes with duplicate paths
func reduceByFileName(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
	byName := make(map[mapreduce.KeyType]*Node)

	for x := range in {
		path := x.Key()           // Get key
		node := x.Value().(*Node) // Assert type

		// Add values to the map for aggregation, skip nodes with the same path
		if _, ok := byName[path]; !ok {
			byName[path] = node
			out <- node // send first copy
		}
	}
}

// Very simple function to map nodes by size
func makeFileSizeMapFnFrom(in <-chan mapreduce.Value) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue) {
		for x := range in {
			node := x.Value().(*Node) // Assert type

			out <- mapreduce.NewKVType(
				mapreduce.KeyTypeFromInt64(node.Size),
				node,
			)
		}
	}
}

// reduceByFileSize custom function to filter files by size
func reduceByFileSize(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
	bySize := make(map[mapreduce.KeyType][]*Node)

	for x := range in {
		size := x.Key()           // Get key
		node := x.Value().(*Node) // Assert type

		// Add values to the map for aggregation
		if v, ok := bySize[size]; ok {
			// Found node with the same file size
			if len(v) == 1 {
				// First time we found duplicate, send first node too
				out <- v[0]
			}
			bySize[size] = append(v, node)
			// Send duplicate downstream
			out <- node
		} else {
			// Store first copy
			bySize[size] = []*Node{node}
		}
	}
}

func makeFileHashMapFnFrom(in <-chan mapreduce.Value, fast bool) mapreduce.MapFn {
	return func(out chan<- mapreduce.KeyValue) {
		var wg sync.WaitGroup
		for x := range in {
			node := x.Value().(*Node) // Assert type

			// Add to wait group
			wg.Add(1)

			// Calculate hash using balancer
			go func(n *Node) {
				WorkQueue <- func() {
					defer wg.Done() // Signal done

					// Default hash value for files < 1024 if fast is true
					hash := "0h"

					// Little optimization to avoid calculating fast hash on small files
					if !fast || node.Size > blockSize {
						// Always calculate hash if fast == false or file size > blockSize
						hash = n.calculateHash(fast) // Fast hash calculation - SHA1 of first 1024 bytes
					}

					// Don't process files for which we failed to calculate SHA1 hash
					if hash == "" {
						// log.Printf("WARN Unable calculate SHA1 hash for %q\n", node.Path)
						return
					}

					out <- mapreduce.NewKVType(
						mapreduce.KeyTypeFromString(hash),
						n,
					)
				}
			}(node)
		}
		// Wait for all results be submitted
		wg.Wait()
	}
}

func reduceByHash(out chan<- mapreduce.Value, in <-chan mapreduce.KeyValue) {
	byHash := make(map[mapreduce.KeyType][]*Node)

	for x := range in {
		hash := x.Key()
		node := x.Value().(*Node) // Assert type

		// Add hash value to a node
		node.Hash = hash.String()

		if v, ok := byHash[hash]; ok {
			// Found node with the same SHA1 hash
			// Send out aggregeted results
			if len(v) == 1 {
				// First time we found duplicate, send first node too
				out <- v[0]
			}
			byHash[hash] = append(v, node)
			out <- node
		} else {
			byHash[hash] = []*Node{node}
		}
	}
}

func IsFile(fi os.FileInfo) bool {
	return fi.Mode()&os.ModeType == 0
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// HTTP file system request handler

package http

import (
	"fmt"
	"io"
	"mime"
	"os"
	"path"
	"strconv"
	"strings"
	"time"
	"utf8"
)

// Heuristic: b is text if it is valid UTF-8 and doesn't
// contain any unprintable ASCII or Unicode characters.
func isText(b []byte) bool {
	for len(b) > 0 && utf8.FullRune(b) {
		rune, size := utf8.DecodeRune(b)
		if size == 1 && rune == utf8.RuneError {
			// decoding error
			return false
		}
		if 0x80 <= rune && rune <= 0x9F {
			return false
		}
		if rune < ' ' {
			switch rune {
			case '\n', '\r', '\t':
				// okay
			default:
				// binary garbage
				return false
			}
		}
		b = b[size:]
	}
	return true
}

func dirList(w ResponseWriter, f *os.File) {
	fmt.Fprintf(w, "
\n")
	for {
		dirs, err := f.Readdir(100)
		if err != nil || len(dirs) == 0 {
			break
		}
		for _, d := range dirs {
			name := d.Name
			if d.IsDirectory() {
				name += "/"
			}
			// TODO htmlescape
			fmt.Fprintf(w, "%s\n", name, name)
		}
	}
	fmt.Fprintf(w, "
\n")
}

func serveFile(w ResponseWriter, r *Request, name string, redirect bool) {
	const indexPage = "/index.html"

	// redirect .../index.html to .../
	if strings.HasSuffix(r.URL.Path, indexPage) {
		Redirect(w, r, r.URL.Path[0:len(r.URL.Path)-len(indexPage)+1], StatusMovedPermanently)
		return
	}

	f, err := os.Open(name, os.O_RDONLY, 0)
	if err != nil {
		// TODO expose actual error?
		NotFound(w, r)
		return
	}
	defer f.Close()

	d, err1 := f.Stat()
	if err1 != nil {
		// TODO expose actual error?
		NotFound(w, r)
		return
	}

	if redirect {
		// redirect to canonical path: / at end of directory url
		// r.URL.Path always begins with /
		url := r.URL.Path
		if d.IsDirectory() {
			if url[len(url)-1] != '/' {
				Redirect(w, r, url+"/", StatusMovedPermanently)
				return
			}
		} else {
			if url[len(url)-1] == '/' {
				Redirect(w, r, url[0:len(url)-1], StatusMovedPermanently)
				return
			}
		}
	}

	if t, _ := time.Parse(TimeFormat, r.Header["If-Modified-Since"]); t != nil && d.Mtime_ns/1e9 <= t.Seconds() {
		w.WriteHeader(StatusNotModified)
		return
	}
	w.SetHeader("Last-Modified", time.SecondsToUTC(d.Mtime_ns/1e9).Format(TimeFormat))

	// use contents of index.html for directory, if present
	if d.IsDirectory() {
		index := name + indexPage
		ff, err := os.Open(index, os.O_RDONLY, 0)
		if err == nil {
			defer ff.Close()
			dd, err := ff.Stat()
			if err == nil {
				name = index
				d = dd
				f = ff
			}
		}
	}

	if d.IsDirectory() {
		dirList(w, f)
		return
	}

	// serve file
	size := d.Size
	code := StatusOK

	// use extension to find content type.
	ext := path.Ext(name)
	if ctype := mime.TypeByExtension(ext); ctype != "" {
		w.SetHeader("Content-Type", ctype)
	} else {
		// read first chunk to decide between utf-8 text and binary
		var buf [1024]byte
		n, _ := io.ReadFull(f, buf[:])
		b := buf[:n]
		if isText(b) {
			w.SetHeader("Content-Type", "text-plain; charset=utf-8")
		} else {
			w.SetHeader("Content-Type", "application/octet-stream") // generic binary
		}
		f.Seek(0, 0) // rewind to output whole file
	}

	// handle Content-Range header.
	// TODO(adg): handle multiple ranges
	ranges, err := parseRange(r.Header["Range"], size)
	if err != nil || len(ranges) > 1 {
		Error(w, err.String(), StatusRequestedRangeNotSatisfiable)
		return
	}
	if len(ranges) == 1 {
		ra := ranges[0]
		if _, err := f.Seek(ra.start, 0); err != nil {
			Error(w, err.String(), StatusRequestedRangeNotSatisfiable)
			return
		}
		size = ra.length
		code = StatusPartialContent
		w.SetHeader("Content-Range", fmt.Sprintf("%d-%d/%d", ra.start, ra.start+ra.length, d.Size))
	}

	w.SetHeader("Accept-Ranges", "bytes")
	w.SetHeader("Content-Length", strconv.Itoa64(size))

	w.WriteHeader(code)

	io.Copyn(w, f, size)
}

// ServeFile replies to the request with the contents of the named file or directory.
func ServeFile(w ResponseWriter, r *Request, name string) {
	serveFile(w, r, name, false)
}

type fileHandler struct {
	root   string
	prefix string
}

// FileServer returns a handler that serves HTTP requests
// with the contents of the file system rooted at root.
// It strips prefix from the incoming requests before
// looking up the file name in the file system.
func FileServer(root, prefix string) Handler { return &fileHandler{root, prefix} }

func (f *fileHandler) ServeHTTP(w ResponseWriter, r *Request) {
	path := r.URL.Path
	if !strings.HasPrefix(path, f.prefix) {
		NotFound(w, r)
		return
	}
	path = path[len(f.prefix):]
	serveFile(w, r, f.root+"/"+path, true)
}

// httpRange specifies the byte range to be sent to the client.
type httpRange struct {
	start, length int64
}

// parseRange parses a Range header string as per RFC 2616.
func parseRange(s string, size int64) ([]httpRange, os.Error) {
	if s == "" {
		return nil, nil // header not present
	}
	const b = "bytes="
	if !strings.HasPrefix(s, b) {
		return nil, os.NewError("invalid range")
	}
	var ranges []httpRange
	for _, ra := range strings.Split(s[len(b):], ",", -1) {
		i := strings.Index(ra, "-")
		if i < 0 {
			return nil, os.NewError("invalid range")
		}
		start, end := ra[:i], ra[i+1:]
		var r httpRange
		if start == "" {
			// If no start is specified, end specifies the
			// range start relative to the end of the file.
			i, err := strconv.Atoi64(end)
			if err != nil {
				return nil, os.NewError("invalid range")
			}
			if i > size {
				i = size
			}
			r.start = size - i
			r.length = size - r.start
		} else {
			i, err := strconv.Atoi64(start)
			if err != nil || i > size || i < 0 {
				return nil, os.NewError("invalid range")
			}
			r.start = i
			if end == "" {
				// If no end is specified, range extends to end of the file.
				r.length = size - r.start
			} else {
				i, err := strconv.Atoi64(end)
				if err != nil || r.start > i {
					return nil, os.NewError("invalid range")
				}
				if i >= size {
					i = size - 1
				}
				r.length = i - r.start + 1
			}
		}
		ranges = append(ranges, r)
	}
	return ranges, nil
}
<commit_msg>http: include DEL in the test for unprintable chars<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// HTTP file system request handler

package http

import (
	"fmt"
	"io"
	"mime"
	"os"
	"path"
	"strconv"
	"strings"
	"time"
	"utf8"
)

// Heuristic: b is text if it is valid UTF-8 and doesn't
// contain any unprintable ASCII or Unicode characters.
func isText(b []byte) bool {
	for len(b) > 0 && utf8.FullRune(b) {
		rune, size := utf8.DecodeRune(b)
		if size == 1 && rune == utf8.RuneError {
			// decoding error
			return false
		}
		if 0x7F <= rune && rune <= 0x9F {
			return false
		}
		if rune < ' ' {
			switch rune {
			case '\n', '\r', '\t':
				// okay
			default:
				// binary garbage
				return false
			}
		}
		b = b[size:]
	}
	return true
}

func dirList(w ResponseWriter, f *os.File) {
	fmt.Fprintf(w, "
\n")
	for {
		dirs, err := f.Readdir(100)
		if err != nil || len(dirs) == 0 {
			break
		}
		for _, d := range dirs {
			name := d.Name
			if d.IsDirectory() {
				name += "/"
			}
			// TODO htmlescape
			fmt.Fprintf(w, "%s\n", name, name)
		}
	}
	fmt.Fprintf(w, "
\n")
}

func serveFile(w ResponseWriter, r *Request, name string, redirect bool) {
	const indexPage = "/index.html"

	// redirect .../index.html to .../
	if strings.HasSuffix(r.URL.Path, indexPage) {
		Redirect(w, r, r.URL.Path[0:len(r.URL.Path)-len(indexPage)+1], StatusMovedPermanently)
		return
	}

	f, err := os.Open(name, os.O_RDONLY, 0)
	if err != nil {
		// TODO expose actual error?
		NotFound(w, r)
		return
	}
	defer f.Close()

	d, err1 := f.Stat()
	if err1 != nil {
		// TODO expose actual error?
		NotFound(w, r)
		return
	}

	if redirect {
		// redirect to canonical path: / at end of directory url
		// r.URL.Path always begins with /
		url := r.URL.Path
		if d.IsDirectory() {
			if url[len(url)-1] != '/' {
				Redirect(w, r, url+"/", StatusMovedPermanently)
				return
			}
		} else {
			if url[len(url)-1] == '/' {
				Redirect(w, r, url[0:len(url)-1], StatusMovedPermanently)
				return
			}
		}
	}

	if t, _ := time.Parse(TimeFormat, r.Header["If-Modified-Since"]); t != nil && d.Mtime_ns/1e9 <= t.Seconds() {
		w.WriteHeader(StatusNotModified)
		return
	}
	w.SetHeader("Last-Modified", time.SecondsToUTC(d.Mtime_ns/1e9).Format(TimeFormat))

	// use contents of index.html for directory, if present
	if d.IsDirectory() {
		index := name + indexPage
		ff, err := os.Open(index, os.O_RDONLY, 0)
		if err == nil {
			defer ff.Close()
			dd, err := ff.Stat()
			if err == nil {
				name = index
				d = dd
				f = ff
			}
		}
	}

	if d.IsDirectory() {
		dirList(w, f)
		return
	}

	// serve file
	size := d.Size
	code := StatusOK

	// use extension to find content type.
	ext := path.Ext(name)
	if ctype := mime.TypeByExtension(ext); ctype != "" {
		w.SetHeader("Content-Type", ctype)
	} else {
		// read first chunk to decide between utf-8 text and binary
		var buf [1024]byte
		n, _ := io.ReadFull(f, buf[:])
		b := buf[:n]
		if isText(b) {
			w.SetHeader("Content-Type", "text-plain; charset=utf-8")
		} else {
			w.SetHeader("Content-Type", "application/octet-stream") // generic binary
		}
		f.Seek(0, 0) // rewind to output whole file
	}

	// handle Content-Range header.
	// TODO(adg): handle multiple ranges
	ranges, err := parseRange(r.Header["Range"], size)
	if err != nil || len(ranges) > 1 {
		Error(w, err.String(), StatusRequestedRangeNotSatisfiable)
		return
	}
	if len(ranges) == 1 {
		ra := ranges[0]
		if _, err := f.Seek(ra.start, 0); err != nil {
			Error(w, err.String(), StatusRequestedRangeNotSatisfiable)
			return
		}
		size = ra.length
		code = StatusPartialContent
		w.SetHeader("Content-Range", fmt.Sprintf("%d-%d/%d", ra.start, ra.start+ra.length, d.Size))
	}

	w.SetHeader("Accept-Ranges", "bytes")
	w.SetHeader("Content-Length", strconv.Itoa64(size))

	w.WriteHeader(code)

	io.Copyn(w, f, size)
}

// ServeFile replies to the request with the contents of the named file or directory.
func ServeFile(w ResponseWriter, r *Request, name string) {
	serveFile(w, r, name, false)
}

type fileHandler struct {
	root   string
	prefix string
}

// FileServer returns a handler that serves HTTP requests
// with the contents of the file system rooted at root.
// It strips prefix from the incoming requests before
// looking up the file name in the file system.
func FileServer(root, prefix string) Handler { return &fileHandler{root, prefix} }

func (f *fileHandler) ServeHTTP(w ResponseWriter, r *Request) {
	path := r.URL.Path
	if !strings.HasPrefix(path, f.prefix) {
		NotFound(w, r)
		return
	}
	path = path[len(f.prefix):]
	serveFile(w, r, f.root+"/"+path, true)
}

// httpRange specifies the byte range to be sent to the client.
type httpRange struct {
	start, length int64
}

// parseRange parses a Range header string as per RFC 2616.
func parseRange(s string, size int64) ([]httpRange, os.Error) {
	if s == "" {
		return nil, nil // header not present
	}
	const b = "bytes="
	if !strings.HasPrefix(s, b) {
		return nil, os.NewError("invalid range")
	}
	var ranges []httpRange
	for _, ra := range strings.Split(s[len(b):], ",", -1) {
		i := strings.Index(ra, "-")
		if i < 0 {
			return nil, os.NewError("invalid range")
		}
		start, end := ra[:i], ra[i+1:]
		var r httpRange
		if start == "" {
			// If no start is specified, end specifies the
			// range start relative to the end of the file.
			i, err := strconv.Atoi64(end)
			if err != nil {
				return nil, os.NewError("invalid range")
			}
			if i > size {
				i = size
			}
			r.start = size - i
			r.length = size - r.start
		} else {
			i, err := strconv.Atoi64(start)
			if err != nil || i > size || i < 0 {
				return nil, os.NewError("invalid range")
			}
			r.start = i
			if end == "" {
				// If no end is specified, range extends to end of the file.
				r.length = size - r.start
			} else {
				i, err := strconv.Atoi64(end)
				if err != nil || r.start > i {
					return nil, os.NewError("invalid range")
				}
				if i >= size {
					i = size - 1
				}
				r.length = i - r.start + 1
			}
		}
		ranges = append(ranges, r)
	}
	return ranges, nil
}
<|endoftext|>