<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/couchbaselabs/go-couchbase"

	"github.com/couchbaselabs/cbfs/config"
)

var workers = flag.Int("workers", 4, "Number of upload workers")
var couchbaseServer = flag.String("couchbase", "", "Couchbase URL")
var couchbaseBucket = flag.String("bucket", "default", "Couchbase bucket")

var cb *couchbase.Bucket

var commands = map[string]struct {
	nargs  int
	f      func(args []string)
	argstr string
}{
	"upload":  {2, uploadCommand, "/src/dir http://cbfs:8484/path/"},
	"getconf": {0, getConfCommand, ""},
	"setconf": {2, setConfCommand, "prop value"},
}

func init() {
	log.SetFlags(log.Lmicroseconds)

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr,
			"Usage of %s [-flags] cmd cmdargs\n",
			os.Args[0])

		fmt.Fprintf(os.Stderr, "\nCommands:\n")

		for k, v := range commands {
			fmt.Fprintf(os.Stderr, "  %s %s\n", k, v.argstr)
		}

		fmt.Fprintf(os.Stderr, "\nFlags:\n")
		flag.PrintDefaults()
		os.Exit(1)
	}

}

var wg = sync.WaitGroup{}

type uploadReq struct {
	src  string
	dest string
}

func recognizeTypeByName(n, def string) string {
	byname := mime.TypeByExtension(n)
	switch {
	case byname != "":
		return byname
	case strings.HasSuffix(n, ".js"):
		return "application/javascript"
	}
	return def
}

func uploadFile(src, dest string) error {
	f, err := os.Open(src)
	if err != nil {
		return err
	}
	defer f.Close()

	someBytes := make([]byte, 512)
	n, err := f.Read(someBytes)
	if err != nil && err != io.EOF {
		return err
	}
	someBytes = someBytes[:n]
	_, err = f.Seek(0, 0)
	if err != nil {
		return err
	}

	preq, err := http.NewRequest("PUT", dest, f)
	if err != nil {
		return err
	}

	ctype := http.DetectContentType(someBytes)
	if strings.HasPrefix(ctype, "text/plain") ||
		strings.HasPrefix(ctype, "application/octet-stream") {
		ctype = recognizeTypeByName(src, ctype)
	}

	preq.Header.Set("Content-Type", ctype)

	resp, err := http.DefaultClient.Do(preq)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		return fmt.Errorf("HTTP Error:  %v", resp.Status)
	}
	return nil
}

func uploader(ch chan uploadReq) {
	defer wg.Done()
	for req := range ch {
		log.Printf("%v -> %v", req.src, req.dest)
		err := uploadFile(req.src, req.dest)
		if err != nil {
			log.Fatalf("Error uploading file: %v", err)
		}
	}
}

func syncUp(src, u string, ch chan<- uploadReq) {
	err := filepath.Walk(src,
		func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if !info.IsDir() {
				shortPath := path[len(src):]
				ch <- uploadReq{path, u + shortPath}
			}

			return err
		})
	if err != nil {
		log.Fatalf("Traversal error: %v", err)
	}
}

func uploadCommand(args []string) {
	ch := make(chan uploadReq)

	for i := 0; i < *workers; i++ {
		wg.Add(1)
		go uploader(ch)
	}

	syncUp(args[0], args[1], ch)

	close(ch)
	wg.Wait()
}

func getConfCommand(args []string) {
	if cb == nil {
		log.Fatalf("No couchbase bucket specified")
	}
	conf := cbfsconfig.CBFSConfig{}
	err := conf.RetrieveConfig(cb)
	if err != nil {
		log.Fatalf("Error getting config: %v", err)
	}

	conf.Dump(os.Stdout)
}

func parseDuration(s string) time.Duration {
	d, err := time.ParseDuration(s)
	if err != nil {
		log.Fatalf("Unable to parse duration: %v", err)
	}
	return d
}

func parseInt(s string) int {
	i, err := strconv.Atoi(s)
	if err != nil {
		log.Fatalf("Error parsing int: %v", err)
	}
	return i
}

func setConfCommand(args []string) {
	if cb == nil {
		log.Fatalf("No couchbase bucket specified")
	}
	conf := cbfsconfig.DefaultConfig()
	err := conf.RetrieveConfig(cb)
	if err != nil {
		log.Printf("Error getting config: %v, using default", err)
	}

	switch args[0] {
	default:
		log.Fatalf("Unhandled property: %v (try running getconf)",
			args[0])
	case "gcfreq":
		conf.GCFreq = parseDuration(args[1])
	case "hash":
		conf.Hash = args[1]
	case "hbfreq":
		conf.HeartbeatFreq = parseDuration(args[1])
	case "minrepl":
		conf.MinReplicas = parseInt(args[1])
	case "cleanCount":
		conf.NodeCleanCount = parseInt(args[1])
	case "reconcileFreq":
		conf.ReconcileFreq = parseDuration(args[1])
	case "nodeCheckFreq":
		conf.StaleNodeCheckFreq = parseDuration(args[1])
	case "staleLimit":
		conf.StaleNodeLimit = parseDuration(args[1])
	}

	err = conf.StoreConfig(cb)
	if err != nil {
		log.Fatalf("Error updating config: %v", err)
	}
}

func main() {
	flag.Parse()

	if flag.NArg() < 1 {
		flag.Usage()
	}

	if *couchbaseServer != "" {
		var err error
		cb, err = couchbase.GetBucket(*couchbaseServer,
			"default", *couchbaseBucket)
		if err != nil {
			log.Fatalf("Error connecting to couchbase: %v", err)
		}
	}

	cmdName := flag.Arg(0)
	cmd, ok := commands[cmdName]
	if !ok {
		fmt.Fprintf(os.Stderr, "Unknown command: %v\n", cmdName)
		flag.Usage()
	}
	if flag.NArg()-1 != cmd.nargs {
		fmt.Fprintf(os.Stderr, "Incorrect arguments for %v\n", cmdName)
		flag.Usage()
	}

	cmd.f(flag.Args()[1:])
}
<commit_msg>Be smarter about file types in upload.<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/couchbaselabs/go-couchbase"

	"github.com/couchbaselabs/cbfs/config"
)

var workers = flag.Int("workers", 4, "Number of upload workers")
var couchbaseServer = flag.String("couchbase", "", "Couchbase URL")
var couchbaseBucket = flag.String("bucket", "default", "Couchbase bucket")

var cb *couchbase.Bucket

var commands = map[string]struct {
	nargs  int
	f      func(args []string)
	argstr string
}{
	"upload":  {2, uploadCommand, "/src/dir http://cbfs:8484/path/"},
	"getconf": {0, getConfCommand, ""},
	"setconf": {2, setConfCommand, "prop value"},
}

func init() {
	log.SetFlags(log.Lmicroseconds)

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr,
			"Usage of %s [-flags] cmd cmdargs\n",
			os.Args[0])

		fmt.Fprintf(os.Stderr, "\nCommands:\n")

		for k, v := range commands {
			fmt.Fprintf(os.Stderr, "  %s %s\n", k, v.argstr)
		}

		fmt.Fprintf(os.Stderr, "\nFlags:\n")
		flag.PrintDefaults()
		os.Exit(1)
	}

}

var wg = sync.WaitGroup{}

type uploadReq struct {
	src  string
	dest string
}

func recognizeTypeByName(n, def string) string {
	byname := mime.TypeByExtension(n)
	switch {
	case byname != "":
		return byname
	case strings.HasSuffix(n, ".js"):
		return "application/javascript"
	}
	return def
}

func uploadFile(src, dest string) error {
	f, err := os.Open(src)
	if err != nil {
		return err
	}
	defer f.Close()

	someBytes := make([]byte, 512)
	n, err := f.Read(someBytes)
	if err != nil && err != io.EOF {
		return err
	}
	someBytes = someBytes[:n]
	_, err = f.Seek(0, 0)
	if err != nil {
		return err
	}

	preq, err := http.NewRequest("PUT", dest, f)
	if err != nil {
		return err
	}

	ctype := http.DetectContentType(someBytes)
	if strings.HasPrefix(ctype, "text/plain") ||
		strings.HasPrefix(ctype, "application/octet-stream") {
		ctype = recognizeTypeByName(src, ctype)
	}

	preq.Header.Set("Content-Type", ctype)

	resp, err := http.DefaultClient.Do(preq)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		return fmt.Errorf("HTTP Error:  %v", resp.Status)
	}
	return nil
}

func uploader(ch chan uploadReq) {
	defer wg.Done()
	for req := range ch {
		log.Printf("%v -> %v", req.src, req.dest)
		err := uploadFile(req.src, req.dest)
		if err != nil {
			log.Fatalf("Error uploading file: %v", err)
		}
	}
}

func syncUp(src, u string, ch chan<- uploadReq) {
	err := filepath.Walk(src,
		func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			switch info.Mode() & os.ModeType {
			case os.ModeDir:
				// ignoring quietly
			case os.ModeCharDevice, os.ModeDevice,
				os.ModeNamedPipe, os.ModeSocket, os.ModeSymlink:

				log.Printf("Ignoring special file: %v", path)
			default:
				shortPath := path[len(src):]
				ch <- uploadReq{path, u + shortPath}
			}
			return err
		})
	if err != nil {
		log.Fatalf("Traversal error: %v", err)
	}
}

func uploadCommand(args []string) {
	ch := make(chan uploadReq)

	for i := 0; i < *workers; i++ {
		wg.Add(1)
		go uploader(ch)
	}

	syncUp(args[0], args[1], ch)

	close(ch)
	wg.Wait()
}

func getConfCommand(args []string) {
	if cb == nil {
		log.Fatalf("No couchbase bucket specified")
	}
	conf := cbfsconfig.CBFSConfig{}
	err := conf.RetrieveConfig(cb)
	if err != nil {
		log.Fatalf("Error getting config: %v", err)
	}

	conf.Dump(os.Stdout)
}

func parseDuration(s string) time.Duration {
	d, err := time.ParseDuration(s)
	if err != nil {
		log.Fatalf("Unable to parse duration: %v", err)
	}
	return d
}

func parseInt(s string) int {
	i, err := strconv.Atoi(s)
	if err != nil {
		log.Fatalf("Error parsing int: %v", err)
	}
	return i
}

func setConfCommand(args []string) {
	if cb == nil {
		log.Fatalf("No couchbase bucket specified")
	}
	conf := cbfsconfig.DefaultConfig()
	err := conf.RetrieveConfig(cb)
	if err != nil {
		log.Printf("Error getting config: %v, using default", err)
	}

	switch args[0] {
	default:
		log.Fatalf("Unhandled property: %v (try running getconf)",
			args[0])
	case "gcfreq":
		conf.GCFreq = parseDuration(args[1])
	case "hash":
		conf.Hash = args[1]
	case "hbfreq":
		conf.HeartbeatFreq = parseDuration(args[1])
	case "minrepl":
		conf.MinReplicas = parseInt(args[1])
	case "cleanCount":
		conf.NodeCleanCount = parseInt(args[1])
	case "reconcileFreq":
		conf.ReconcileFreq = parseDuration(args[1])
	case "nodeCheckFreq":
		conf.StaleNodeCheckFreq = parseDuration(args[1])
	case "staleLimit":
		conf.StaleNodeLimit = parseDuration(args[1])
	}

	err = conf.StoreConfig(cb)
	if err != nil {
		log.Fatalf("Error updating config: %v", err)
	}
}

func main() {
	flag.Parse()

	if flag.NArg() < 1 {
		flag.Usage()
	}

	if *couchbaseServer != "" {
		var err error
		cb, err = couchbase.GetBucket(*couchbaseServer,
			"default", *couchbaseBucket)
		if err != nil {
			log.Fatalf("Error connecting to couchbase: %v", err)
		}
	}

	cmdName := flag.Arg(0)
	cmd, ok := commands[cmdName]
	if !ok {
		fmt.Fprintf(os.Stderr, "Unknown command: %v\n", cmdName)
		flag.Usage()
	}
	if flag.NArg()-1 != cmd.nargs {
		fmt.Fprintf(os.Stderr, "Incorrect arguments for %v\n", cmdName)
		flag.Usage()
	}

	cmd.f(flag.Args()[1:])
}
<|endoftext|>


<commit_before>package cryptex

import (
	"encoding/json"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/ieee0824/cryptex/kms"
	"log"
)

type V struct {
	Value interface{}
}

type Cryptex struct {
	msg map[string]interface{}
	kms *kms.KMS
}

func New(m map[string]interface{}, sess *session.Session, keyID string) *Cryptex {
	c := &Cryptex{
		msg: m,
		kms: kms.New(sess),
	}
	c.kms.SetKey(keyID)
	return c
}

func (c *Cryptex) encrypt(obj interface{}) (interface{}, error) {
	m, ok := obj.(map[string]interface{})
	if !ok {
		v := &V{
			Value: obj,
		}
		bin, err := json.Marshal(v)
		if err != nil {
			return nil, err
		}

		result, err := c.kms.Encrypt(bin)
		if err != nil {
			return nil, err
		}

		return result, nil

	}

	for k, v := range m {
		o, err := c.encrypt(v)
		if err != nil {
			return nil, err
		}
		obj.(map[string]interface{})[k] = o
	}
	return obj, nil
}

func (c *Cryptex) decrypt(obj interface{}) (interface{}, error) {
	m, ok := obj.(map[string]interface{})
	if !ok {
		p, err := c.kms.Decrypt(obj.([]byte))
		if err != nil {
			log.Fatalln(err)
			return nil, err
		}

		v := V{}

		if err := json.Unmarshal(p, &v); err != nil {
			return nil, err
		}

		return v.Value, nil
	}

	for k, v := range m {
		o, err := c.decrypt(v)
		if err != nil {
			return nil, err
		}
		obj.(map[string]interface{})[k] = o
	}
	return obj, nil
}

func (c *Cryptex) Encrypt() (interface{}, error) {
	o, err := c.encrypt(c.msg)
	if err != nil {
		return nil, err
	}
	return o, nil
}

func (c *Cryptex) Decrypt(d interface{}) (interface{}, error) {
	o, err := c.decrypt(d)
	if err != nil {
		return nil, err
	}

	return o, nil
}
<commit_msg>edit機能の実装<commit_after>package cryptex

import (
	"encoding/json"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/ieee0824/cryptex/kms"
	"log"
	"os"
	"os/exec"
	"io/ioutil"
)

func getEditor() string {
	if e := os.Getenv("PNZR_EDITOR"); e != "" {
		return e
	}

	if e := os.Getenv("EDITOR"); e != "" {
		return e
	}

	return "nano"
}

type V struct {
	Value interface{}
}

type Cryptex struct {
	msg map[string]interface{}
	kms *kms.KMS
}

func New(m map[string]interface{}, sess *session.Session, keyID string) *Cryptex {
	c := &Cryptex{
		msg: m,
		kms: kms.New(sess),
	}
	c.kms.SetKey(keyID)
	return c
}

func (c *Cryptex) encrypt(obj interface{}) (interface{}, error) {
	m, ok := obj.(map[string]interface{})
	if !ok {
		v := &V{
			Value: obj,
		}
		bin, err := json.Marshal(v)
		if err != nil {
			return nil, err
		}

		result, err := c.kms.Encrypt(bin)
		if err != nil {
			return nil, err
		}

		return result, nil

	}

	for k, v := range m {
		o, err := c.encrypt(v)
		if err != nil {
			return nil, err
		}
		obj.(map[string]interface{})[k] = o
	}
	return obj, nil
}

func (c *Cryptex) decrypt(obj interface{}) (interface{}, error) {
	m, ok := obj.(map[string]interface{})
	if !ok {
		p, err := c.kms.Decrypt(obj.([]byte))
		if err != nil {
			log.Fatalln(err)
			return nil, err
		}

		v := V{}

		if err := json.Unmarshal(p, &v); err != nil {
			return nil, err
		}

		return v.Value, nil
	}

	for k, v := range m {
		o, err := c.decrypt(v)
		if err != nil {
			return nil, err
		}
		obj.(map[string]interface{})[k] = o
	}
	return obj, nil
}

func (c *Cryptex) Encrypt() (interface{}, error) {
	o, err := c.encrypt(c.msg)
	if err != nil {
		return nil, err
	}
	return o, nil
}

func (c *Cryptex) Decrypt(d interface{}) (interface{}, error) {
	o, err := c.decrypt(d)
	if err != nil {
		return nil, err
	}

	return o, nil
}

func (c *Cryptex) Edit(i interface{}) (interface{}, error) {
	p, err := c.decrypt(i)
	if err != nil {
		return nil, err
	}
	bin, err := json.MarshalIndent(p, "", "    ")
	if err != nil {
		return nil, err
	}

	file, err := ioutil.TempFile(os.TempDir(), "cryptex")
	if err != nil {
		return nil, err
	}
	defer os.Remove(file.Name())
	if _, err := file.Write(bin); err != nil {
		return nil, err
	}

	cmd := exec.Command(getEditor(), file.Name())
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	if err := cmd.Run(); err != nil {
		return nil, err
	}

	pbin, err := ioutil.ReadFile(file.Name())
	if err != nil {
		return nil, err
	}

	m := map[string]interface{}{}

	if err := json.Unmarshal(pbin, &m); err != nil {
		return nil, err
	}

	result, err := c.encrypt(m)
	if err != nil {
		return nil, err
	}

	return result, nil
}
<|endoftext|>


<commit_before>// Package whoami implements a plugin that returns details about the resolving
// querying it.
package whoami

import (
	"net"
	"strconv"

	"github.com/coredns/coredns/request"

	"github.com/miekg/dns"
	"golang.org/x/net/context"
)

// Whoami is a plugin that returns your IP address, port and the protocol used for connecting
// to CoreDNS.
type Whoami struct{}

// ServeDNS implements the plugin.Handler interface.
func (wh Whoami) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) {
	state := request.Request{W: w, Req: r}

	a := new(dns.Msg)
	a.SetReply(r)
	a.Compress = true
	a.Authoritative = true

	ip := state.IP()
	var rr dns.RR

	switch state.Family() {
	case 1:
		rr = new(dns.A)
		rr.(*dns.A).Hdr = dns.RR_Header{Name: state.QName(), Rrtype: dns.TypeA, Class: state.QClass()}
		rr.(*dns.A).A = net.ParseIP(ip).To4()
	case 2:
		rr = new(dns.AAAA)
		rr.(*dns.AAAA).Hdr = dns.RR_Header{Name: state.QName(), Rrtype: dns.TypeAAAA, Class: state.QClass()}
		rr.(*dns.AAAA).AAAA = net.ParseIP(ip)
	}

	srv := new(dns.SRV)
	srv.Hdr = dns.RR_Header{Name: "_" + state.Proto() + "." + state.QName(), Rrtype: dns.TypeSRV, Class: state.QClass()}
	port, _ := strconv.Atoi(state.Port())
	srv.Port = uint16(port)
	srv.Target = "."

	a.Extra = []dns.RR{rr, srv}

	state.SizeAndDo(a)
	w.WriteMsg(a)

	return 0, nil
}

// Name implements the Handler interface.
func (wh Whoami) Name() string { return "whoami" }
<commit_msg>plugin/whoami (#1132)<commit_after>// Package whoami implements a plugin that returns details about the resolving
// querying it.
package whoami

import (
	"net"
	"strconv"

	"github.com/coredns/coredns/request"

	"github.com/miekg/dns"
	"golang.org/x/net/context"
)

// Whoami is a plugin that returns your IP address, port and the protocol used for connecting
// to CoreDNS.
type Whoami struct{}

// ServeDNS implements the plugin.Handler interface.
func (wh Whoami) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) {
	state := request.Request{W: w, Req: r}

	a := new(dns.Msg)
	a.SetReply(r)
	a.Compress = true
	a.Authoritative = true

	ip := state.IP()
	var rr dns.RR

	switch state.Family() {
	case 1:
		rr = new(dns.A)
		rr.(*dns.A).Hdr = dns.RR_Header{Name: state.QName(), Rrtype: dns.TypeA, Class: state.QClass()}
		rr.(*dns.A).A = net.ParseIP(ip).To4()
	case 2:
		rr = new(dns.AAAA)
		rr.(*dns.AAAA).Hdr = dns.RR_Header{Name: state.QName(), Rrtype: dns.TypeAAAA, Class: state.QClass()}
		rr.(*dns.AAAA).AAAA = net.ParseIP(ip)
	}

	srv := new(dns.SRV)
	srv.Hdr = dns.RR_Header{Name: "_" + state.Proto() + "." + state.QName(), Rrtype: dns.TypeSRV, Class: state.QClass()}
	if state.QName() == "." {
		srv.Hdr.Name = "_" + state.Proto() + state.QName()
	}
	port, _ := strconv.Atoi(state.Port())
	srv.Port = uint16(port)
	srv.Target = "."

	a.Extra = []dns.RR{rr, srv}

	state.SizeAndDo(a)
	w.WriteMsg(a)

	return 0, nil
}

// Name implements the Handler interface.
func (wh Whoami) Name() string { return "whoami" }
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"regexp"
	"strconv"
	"testing"
	"time"
)

// Test that useful common cache-related parameters are sent to the
// client by this CDN provider.

// Should propagate an Age header from origin and then increment it for the
// time it's in cache.
func TestRespHeaderAge(t *testing.T) {
	ResetBackends(backendsByPriority)

	const originAgeInSeconds = 100
	const secondsToWaitBetweenRequests = 5
	const expectedAgeInSeconds = originAgeInSeconds + secondsToWaitBetweenRequests
	requestReceivedCount := 0

	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if requestReceivedCount == 0 {
			w.Header().Set("Cache-Control", "max-age=1800, public")
			w.Header().Set("Age", fmt.Sprintf("%d", originAgeInSeconds))
			w.Write([]byte("cacheable request"))
		} else {
			t.Error("Unexpected subsequent request received at Origin")
		}
		requestReceivedCount++
	})

	req := NewUniqueEdgeGET(t)
	resp := RoundTripCheckError(t, req)
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		t.Fatalf("Edge returned an unexpected status: %q", resp.Status)
	}

	// wait a little bit. Edge should update the Age header, we know Origin will not
	time.Sleep(time.Duration(secondsToWaitBetweenRequests) * time.Second)
	resp = RoundTripCheckError(t, req)
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		t.Fatalf("Edge returned an unexpected status: %q", resp.Status)
	}

	edgeAgeHeader := resp.Header.Get("Age")
	if edgeAgeHeader == "" {
		t.Fatal("Age Header is not set")
	}

	edgeAgeInSeconds, convErr := strconv.Atoi(edgeAgeHeader)
	if convErr != nil {
		t.Fatal(convErr)
	}

	if edgeAgeInSeconds != expectedAgeInSeconds {
		t.Errorf(
			"Age header from Edge is not as expected. Got %q, expected '%d'",
			edgeAgeHeader,
			expectedAgeInSeconds,
		)
	}
}

// Should set an X-Cache header containing HIT/MISS from 'origin, itself'
func TestRespHeaderXCacheAppend(t *testing.T) {
	ResetBackends(backendsByPriority)

	if vendorCloudflare {
		t.Skip(notSupportedByVendor)
	}

	const originXCache = "HIT"

	var (
		xCache         string
		expectedXCache string
	)

	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("X-Cache", originXCache)
	})

	// Get first request, will come from origin, cannot be cached - hence cache MISS
	req := NewUniqueEdgeGET(t)
	resp := RoundTripCheckError(t, req)
	defer resp.Body.Close()

	xCache = resp.Header.Get("X-Cache")
	expectedXCache = fmt.Sprintf("%s, MISS", originXCache)
	if xCache != expectedXCache {
		t.Errorf(
			"X-Cache on initial hit is wrong: expected %q, got %q",
			expectedXCache,
			xCache,
		)
	}

}

// Should set a header containing 'MISS' if request is not cached
func TestRespHeaderCacheMiss(t *testing.T) {
	ResetBackends(backendsByPriority)

	const expectedVal = "MISS"

	var (
		headerName string
		headerVal  string
	)

	switch {
	case vendorCloudflare:
		headerName = "CF-Cache-Status"
	case vendorFastly:
		headerName = "X-Cache"
	default:
		t.Fatal(notImplementedForVendor)
	}

	// Get first request, will come from origin, cannot be cached - hence cache MISS
	req := NewUniqueEdgeGET(t)
	resp := RoundTripCheckError(t, req)
	defer resp.Body.Close()

	headerVal = resp.Header.Get(headerName)

	if headerVal != expectedVal {
		t.Errorf(
			"%s on initial hit is wrong: expected %q, got %q",
			headerName,
			expectedVal,
			headerVal,
		)
	}

}

// Should set an 'Served-By' header giving information on the edge node and location served from.
func TestRespHeaderServedBy(t *testing.T) {
	ResetBackends(backendsByPriority)

	var expectedServedByRegexp *regexp.Regexp
	var headerName string

	switch {
	case vendorCloudflare:
		headerName = "CF-RAY"
		expectedServedByRegexp = regexp.MustCompile("^[a-z0-9]{16}-[A-Z]{3}$")
	case vendorFastly:
		headerName = "X-Served-By"
		expectedServedByRegexp = regexp.MustCompile("^cache-[a-z0-9]+-[A-Z]{3}$")
	default:
		t.Fatal(notImplementedForVendor)
	}

	req := NewUniqueEdgeGET(t)
	resp := RoundTripCheckError(t, req)
	defer resp.Body.Close()

	actualHeader := resp.Header.Get(headerName)

	if actualHeader == "" {
		t.Error(headerName + " header has not been set by Edge")
	}

	if expectedServedByRegexp.FindString(actualHeader) != actualHeader {
		t.Errorf("%s is not as expected: got %q", headerName, actualHeader)
	}

}

// Should set an X-Cache-Hits header containing hit count for this object,
// from the Edge AND the Origin, assuming Origin sets one.
// This is in the format "{origin-hit-count}, {edge-hit-count}"
func TestRespHeaderXCacheHitsAppend(t *testing.T) {
	ResetBackends(backendsByPriority)

	if vendorCloudflare {
		t.Skip(notSupportedByVendor)
	}

	const originXCacheHits = "53"

	var (
		xCacheHits         string
		expectedXCacheHits string
	)

	uuid := NewUUID()

	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "GET" && r.URL.Path == fmt.Sprintf("/%s", uuid) {
			w.Header().Set("X-Cache-Hits", originXCacheHits)
		}
	})

	sourceUrl := fmt.Sprintf("https://%s/%s", *edgeHost, uuid)

	// Get first request, will come from origin. Edge Hit Count 0
	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp := RoundTripCheckError(t, req)
	defer resp.Body.Close()

	xCacheHits = resp.Header.Get("X-Cache-Hits")
	expectedXCacheHits = fmt.Sprintf("%s, 0", originXCacheHits)
	if xCacheHits != expectedXCacheHits {
		t.Errorf(
			"X-Cache-Hits on initial hit is wrong: expected %q, got %q",
			expectedXCacheHits,
			xCacheHits,
		)
	}

	// Get request again. Should come from Edge now, hit count 1
	resp = RoundTripCheckError(t, req)
	defer resp.Body.Close()

	xCacheHits = resp.Header.Get("X-Cache-Hits")
	expectedXCacheHits = fmt.Sprintf("%s, 1", originXCacheHits)
	if xCacheHits != expectedXCacheHits {
		t.Errorf(
			"X-Cache-Hits on second hit is wrong: expected %q, got %q",
			expectedXCacheHits,
			xCacheHits,
		)
	}
}
<commit_msg>Test for header containing 'HIT' on cache hit<commit_after>package main

import (
	"fmt"
	"net/http"
	"regexp"
	"strconv"
	"testing"
	"time"
)

// Test that useful common cache-related parameters are sent to the
// client by this CDN provider.

// Should propagate an Age header from origin and then increment it for the
// time it's in cache.
func TestRespHeaderAge(t *testing.T) {
	ResetBackends(backendsByPriority)

	const originAgeInSeconds = 100
	const secondsToWaitBetweenRequests = 5
	const expectedAgeInSeconds = originAgeInSeconds + secondsToWaitBetweenRequests
	requestReceivedCount := 0

	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if requestReceivedCount == 0 {
			w.Header().Set("Cache-Control", "max-age=1800, public")
			w.Header().Set("Age", fmt.Sprintf("%d", originAgeInSeconds))
			w.Write([]byte("cacheable request"))
		} else {
			t.Error("Unexpected subsequent request received at Origin")
		}
		requestReceivedCount++
	})

	req := NewUniqueEdgeGET(t)
	resp := RoundTripCheckError(t, req)
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		t.Fatalf("Edge returned an unexpected status: %q", resp.Status)
	}

	// wait a little bit. Edge should update the Age header, we know Origin will not
	time.Sleep(time.Duration(secondsToWaitBetweenRequests) * time.Second)
	resp = RoundTripCheckError(t, req)
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		t.Fatalf("Edge returned an unexpected status: %q", resp.Status)
	}

	edgeAgeHeader := resp.Header.Get("Age")
	if edgeAgeHeader == "" {
		t.Fatal("Age Header is not set")
	}

	edgeAgeInSeconds, convErr := strconv.Atoi(edgeAgeHeader)
	if convErr != nil {
		t.Fatal(convErr)
	}

	if edgeAgeInSeconds != expectedAgeInSeconds {
		t.Errorf(
			"Age header from Edge is not as expected. Got %q, expected '%d'",
			edgeAgeHeader,
			expectedAgeInSeconds,
		)
	}
}

// Should set an X-Cache header containing HIT/MISS from 'origin, itself'
func TestRespHeaderXCacheAppend(t *testing.T) {
	ResetBackends(backendsByPriority)

	if vendorCloudflare {
		t.Skip(notSupportedByVendor)
	}

	const originXCache = "HIT"

	var (
		xCache         string
		expectedXCache string
	)

	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("X-Cache", originXCache)
	})

	// Get first request, will come from origin, cannot be cached - hence cache MISS
	req := NewUniqueEdgeGET(t)
	resp := RoundTripCheckError(t, req)
	defer resp.Body.Close()

	xCache = resp.Header.Get("X-Cache")
	expectedXCache = fmt.Sprintf("%s, MISS", originXCache)
	if xCache != expectedXCache {
		t.Errorf(
			"X-Cache on initial hit is wrong: expected %q, got %q",
			expectedXCache,
			xCache,
		)
	}

}

// Should set a header containing 'HIT' or 'MISS' depending on whether request is cached
func TestRespHeaderCacheHitMiss(t *testing.T) {
	ResetBackends(backendsByPriority)

	var (
		expectedVal string
		headerName  string
		headerVal   string
	)

	switch {
	case vendorCloudflare:
		headerName = "CF-Cache-Status"
	case vendorFastly:
		headerName = "X-Cache"
	default:
		t.Fatal(notImplementedForVendor)
	}

	// Get first request, will come from origin, cannot be cached - hence cache MISS
	req := NewUniqueEdgeGET(t)
	resp := RoundTripCheckError(t, req)
	defer resp.Body.Close()

	expectedVal = "MISS"
	headerVal = resp.Header.Get(headerName)

	if headerVal != expectedVal {
		t.Errorf(
			"%s on initial hit is wrong: expected %q, got %q",
			headerName,
			expectedVal,
			headerVal,
		)
	}

	// Get request again. Should come from Edge now and result in a cache HIT
	resp = RoundTripCheckError(t, req)
	defer resp.Body.Close()

	expectedVal = "HIT"
	headerVal = resp.Header.Get(headerName)

	if headerVal != expectedVal {
		t.Errorf(
			"%s on second request is wrong: expected %q, got %q",
			headerName,
			expectedVal,
			headerVal,
		)
	}
}

// Should set an 'Served-By' header giving information on the edge node and location served from.
func TestRespHeaderServedBy(t *testing.T) {
	ResetBackends(backendsByPriority)

	var expectedServedByRegexp *regexp.Regexp
	var headerName string

	switch {
	case vendorCloudflare:
		headerName = "CF-RAY"
		expectedServedByRegexp = regexp.MustCompile("^[a-z0-9]{16}-[A-Z]{3}$")
	case vendorFastly:
		headerName = "X-Served-By"
		expectedServedByRegexp = regexp.MustCompile("^cache-[a-z0-9]+-[A-Z]{3}$")
	default:
		t.Fatal(notImplementedForVendor)
	}

	req := NewUniqueEdgeGET(t)
	resp := RoundTripCheckError(t, req)
	defer resp.Body.Close()

	actualHeader := resp.Header.Get(headerName)

	if actualHeader == "" {
		t.Error(headerName + " header has not been set by Edge")
	}

	if expectedServedByRegexp.FindString(actualHeader) != actualHeader {
		t.Errorf("%s is not as expected: got %q", headerName, actualHeader)
	}

}

// Should set an X-Cache-Hits header containing hit count for this object,
// from the Edge AND the Origin, assuming Origin sets one.
// This is in the format "{origin-hit-count}, {edge-hit-count}"
func TestRespHeaderXCacheHitsAppend(t *testing.T) {
	ResetBackends(backendsByPriority)

	if vendorCloudflare {
		t.Skip(notSupportedByVendor)
	}

	const originXCacheHits = "53"

	var (
		xCacheHits         string
		expectedXCacheHits string
	)

	uuid := NewUUID()

	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "GET" && r.URL.Path == fmt.Sprintf("/%s", uuid) {
			w.Header().Set("X-Cache-Hits", originXCacheHits)
		}
	})

	sourceUrl := fmt.Sprintf("https://%s/%s", *edgeHost, uuid)

	// Get first request, will come from origin. Edge Hit Count 0
	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp := RoundTripCheckError(t, req)
	defer resp.Body.Close()

	xCacheHits = resp.Header.Get("X-Cache-Hits")
	expectedXCacheHits = fmt.Sprintf("%s, 0", originXCacheHits)
	if xCacheHits != expectedXCacheHits {
		t.Errorf(
			"X-Cache-Hits on initial hit is wrong: expected %q, got %q",
			expectedXCacheHits,
			xCacheHits,
		)
	}

	// Get request again. Should come from Edge now, hit count 1
	resp = RoundTripCheckError(t, req)
	defer resp.Body.Close()

	xCacheHits = resp.Header.Get("X-Cache-Hits")
	expectedXCacheHits = fmt.Sprintf("%s, 1", originXCacheHits)
	if xCacheHits != expectedXCacheHits {
		t.Errorf(
			"X-Cache-Hits on second hit is wrong: expected %q, got %q",
			expectedXCacheHits,
			xCacheHits,
		)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"math"
	"odf/ods"
	"regexp"
	"strconv"
	"strings"
)

const (
	odsPath = "/Users/carl/Downloads/bills.ods"
)

var (
	moneyCleanRegex = regexp.MustCompile(`[$,]`)
)

// Print out the data in a table.
func Print(table ods.Table) {
	for _, row := range table.Strings() {
		if len(row) == 0 || row[0] == "" {
			continue
		}
		sep := ""
		for _, field := range row {
			fmt.Print(sep, strconv.Quote(field))
			sep = "\t"
		}
		fmt.Print("\n")
	}
}

func ProcessNewFormat(table ods.Table) {
	for _, row := range table.Strings() {
		if len(row) < 2 || row[0] == "" || strings.Index(row[0], "Balance") == 0 {
			continue
		}
		if strings.Index(row[0], "Billing Cycle") == 0 {
			log.Printf("*** %s\n", row[0])
			continue
		}
		biller := row[0]
		date := row[1]
		if date == "" {
			continue
		}
		debit := 0.0
		credit := 0.0
		if len(row) >= 3 {
			var err error
			debit, err = parseMoney(row[2])
			printErr(err)
		}
		if len(row) >= 4 {
			var err error
			credit, err = parseMoney(row[3])
			printErr(err)
		}
		log.Printf("%35s; %11s; %10.2f; %5.2f\n", biller, date, debit, credit)
	}
}

func ProcessOldFormat(table ods.Table) {
	for _, row := range table.Strings() {
		if row[0] == "Leftover" {
			continue
		}
		if strings.Index(row[0], "Billing Cycle") == 0 {
			log.Printf("*** %s\n", row[0])
			continue
		}
		biller := row[0]
		date := row[1]
		if date == "" {
			continue
		}
		var amount float64
		var err error
		if len(row) >= 3 {
			amount, err = parseMoney(row[2])
			if printErr(err) {
				continue
			}
		}
		credit := 0.0
		debit := 0.0
		if amount < 0 || biller != "Paycheck" {
			credit = math.Abs(amount)
		} else {
			debit = math.Abs(amount)
		}

		log.Printf("%35s; %11s; %10.2f; %5.2f\n", biller, date, debit, credit)
	}
}

// Parse a string into a float. This removes non-numeric characters.
func parseMoney(data string) (float64, error) {
	var amount float64
	var err error
	if len(data) > 0 {
		amount, err = strconv.ParseFloat(moneyCleanRegex.ReplaceAllString(data, ""), 32)
	}
	return amount, err
}

func printErr(err error) (bool) {
	if err != nil {
		log.Printf("[ERROR] %s", err)
		return true
	}
	return false
}

func main() {
	var doc ods.Doc

	f, err := ods.Open(odsPath)
	if err != nil {
		log.Fatal(err)
		return
	}
	defer f.Close()

	if err := f.ParseContent(&doc); err != nil {
		log.Fatal(err)
		return
	}

	// Dump the first table one line per row, writing
	// tab separated, quoted fields.
	for _, table := range doc.Table {
		if year, err := strconv.Atoi(table.Name); err == nil {
			if year > 2008 {
				log.Printf("***** Sheet: %s [new] *****", table.Name)
					ProcessNewFormat(table)
			} else {
				log.Printf("***** Sheet: %s [old] *****", table.Name)
					ProcessOldFormat(table)
			}
		}
	}
}
<commit_msg>Update logging to use new logger. Fix up last bits of parsing wonkiness.<commit_after>package main

import (
	"math"
	"regexp"
	"sort"
	"strconv"
	"strings"

	//"database/sql"
	//_ "github.com/lib/pq"
	"odf/ods"

	"broke/logger"
)

const (
	ODS_PATH = "/Users/carl/Downloads/bills.ods"
	DEBUG    = true
)

var (
	moneyCleanRegex = regexp.MustCompile(`[$,]`)
	newSkipTerms    = []string{"Allowance", "Auto Insurance", "Auto Tags", "Balance", "Bills", "Income", "Savings"}
	oldSkipTerms    = []string{"Carry over", "Leftover"}
)

func ProcessNewFormat(table ods.Table) {
	for _, row := range table.Strings() {
		if len(row) < 2 || row[0] == "" || shouldSkip(newSkipTerms, row[0]) {
			continue
		}
		if strings.Index(row[0], "Billing Cycle") == 0 {
			logger.Info(row[0])
			continue
		}
		biller := row[0]
		date := row[1]
		if date == "" {
			continue
		}
		var debit, credit float64
		if len(row) >= 3 {
			var err error
			debit, err = parseMoney(row[2])
			printErr(err)
		}
		if len(row) >= 4 {
			var err error
			credit, err = parseMoney(row[3])
			printErr(err)
		}

		if credit != 0 || debit != 0 {
			if DEBUG {
				logger.Debugf("%35s; %11s; %10.2f; %5.2f", biller, date, debit, credit)
			}
		} else {
				logger.Debugf("********** %s [%s] had no values", biller, date)
		}
	}
}

func ProcessOldFormat(table ods.Table) {
	for _, row := range table.Strings() {
		if shouldSkip(oldSkipTerms, row[0]) {
			continue
		}
		if strings.Index(row[0], "Billing Cycle") == 0 {
			logger.Infof("*** %s", row[0])
			continue
		}
		biller := row[0]
		date := row[1]
		if date == "" {
			continue
		}
		var amount float64
		var err error
		if len(row) >= 3 {
			amount, err = parseMoney(row[2])
			if printErr(err) {
				continue
			}
		}
		var credit, debit float64
		if amount < 0 || biller != "Paycheck" {
			credit = math.Abs(amount)
		} else {
			debit = math.Abs(amount)
		}

		if credit != 0 || debit != 0 {
			if DEBUG {
				logger.Debugf("%35s; %11s; %10.2f; %5.2f", biller, date, debit, credit)
			}
		} else {
			if DEBUG {
				logger.Debugf("%s [%s] had no values", biller, date)
			}
		}
	}
}

// Parse a string into a float. This removes non-numeric characters.
func parseMoney(data string) (float64, error) {
	var amount float64
	var err error
	if len(data) > 0 {
		amount, err = strconv.ParseFloat(moneyCleanRegex.ReplaceAllString(data, ""), 32)
	}
	return amount, err
}

func printErr(err error) bool {
	if err != nil {
		logger.Error(err)
		return true
	}
	return false
}

func shouldSkip(skips []string, val string) (bool) {
	return sort.SearchStrings(skips, strings.Trim(val, " ")) < 0
}

func main() {
	var doc ods.Doc

	f, err := ods.Open(ODS_PATH)
	if err != nil {
		logger.Fatal(err)
		return
	}
	defer f.Close()

	if err := f.ParseContent(&doc); err != nil {
		logger.Fatal(err)
		return
	}

	// Dump the first table one line per row, writing
	// tab separated, quoted fields.
	for _, table := range doc.Table {
		if year, err := strconv.Atoi(table.Name); err == nil {
			if year >= 2008 {
				logger.Infof("***** Sheet: %s [new] *****", table.Name)
				ProcessNewFormat(table)
			} else {
				logger.Infof("***** Sheet: %s [old] *****", table.Name)
				ProcessOldFormat(table)
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
    "bufio"
    "encoding/csv"
    "flag"
    "fmt"
    "io"
    "os"
    "strconv"
    "strings"
)

const nilCommentRune = "TOTALLYNOTACOMMENTCHAR"

// TODO: Add support for specifying fields by field header name instead of just number
// TODO: check ReadRuneFromString instead of existing technique
// TODO: Print version
// TODO: Publish binaries
// TODO: Include test suite
// TODO: Accept filename without --filename flag

var inSep string
var outSep string
var commentRune string

var filename string

var fieldNumsRaw string
var noRFC = flag.Bool("no-rfc", false, "Program defaults to printing RFC 4180-compliant, quoted, well-formatted CSV. If this flag is supplied, output is returned as a string naively joined by --out-sep. --no-rfc is assumed to imply you want to pass the output to naive tools like cut or awk, and in that case, it is recommended that you select an --out-sep that is unlikely to be in youc content, such as a pipe or a backtick.")

func init() {
    const (
        filenameFlagLong = "file"
        filenameFlagShort = "f"
        filenameDesc = "File to read from. If not specified, program reads from stdin."

        fieldNumsRawFlagLong = "fields"
        fieldNumsRawFlagShort = "F"
        fieldNumsRawDesc =  "Comma-separated list of field indexes (starting at 0) to print to the command line"

        inSepFlagLong = "in-sep"
        inSepFlagShort = "i"
        inSepDesc = "Single character field separator used by your input"
        inSepDefault = ","

        outSepFlagLong = "out-sep"
        outSepFlagShort = "o"
        outSepDesc = "Single-character field separator to use when printing multiple columns in your output. Only valid if outputting something meant to be passed to cut/awk, and not a properly-formatted, quoted CSV file."
        outSepDefault = ","

        commentRuneFlagLong = "comment-char"
        commentRuneFlagShort = "c"
        commentRuneDesc = "Single-character field separator to use when printing multiple columns in your output. Only valid if outputting something meant to be passed to cut/awk, and not a properly-formatted, quoted CSV file."
    )

    flag.StringVar(&filename, filenameFlagLong, "", filenameDesc)
    flag.StringVar(&filename, filenameFlagShort, "", filenameDesc)

    flag.StringVar(&fieldNumsRaw, fieldNumsRawFlagLong, "", fieldNumsRawDesc)
    flag.StringVar(&fieldNumsRaw, fieldNumsRawFlagShort, "", fieldNumsRawDesc)

    flag.StringVar(&inSep, inSepFlagLong, inSepDefault, inSepDesc)
    flag.StringVar(&inSep, inSepFlagShort, inSepDefault, inSepDesc)

    flag.StringVar(&outSep, outSepFlagLong, outSepDefault, outSepDesc)
    flag.StringVar(&outSep, outSepFlagShort, outSepDefault, outSepDesc)

    flag.StringVar(&commentRune, commentRuneFlagLong, nilCommentRune, commentRuneDesc)
    flag.StringVar(&commentRune, commentRuneFlagShort, nilCommentRune, commentRuneDesc)
}

func main() {
    flag.Parse()

    var fieldNums []int

    // Identify the numbers you want to print out
    fieldNumsRaw = strings.Trim(fieldNumsRaw, ",")
    if fieldNumsRaw != "" {
        for _, numStr := range strings.Split(fieldNumsRaw, ",") {
            numStr := strings.TrimSpace(numStr)
            numInt, err := strconv.Atoi(numStr)
            if err != nil {
                panic(err)
            }
            fieldNums = append(fieldNums, numInt)
        }
    }

    var reader *bufio.Reader
    if filename == "" {
        reader = bufio.NewReader(os.Stdin)
    } else {
        f, err := os.Open(filename)
        if err != nil {
            panic(err)
        }

        reader = bufio.NewReader(f)
    }

    csvReader := csv.NewReader(reader)
    csvReader.LazyQuotes = true
    csvReader.TrailingComma = true
    csvReader.Comma = getSeparator(inSep)
    csvReader.FieldsPerRecord = -1
    if commentRune != nilCommentRune {
        csvReader.Comment = ([]rune(commentRune))[0]
    }

    csvWriter := csv.NewWriter(os.Stdout)
    csvWriter.Comma = getSeparator(outSep)

    for {
        fields, err := csvReader.Read()
        if err != nil {
            if err == io.EOF {
                break
            }
            panic(err)
        }

        var toPrint []string
        if fieldNumsRaw == "" {  // Print all fields
            toPrint = fields
        } else {
            for _, num := range fieldNums {
                if num > len(fields) - 1 {
                    toPrint = append(toPrint, "")  // Append _something_, so printing columns out of order preserves the column index:value mapping in all columns. I.e., --field-nums=1,2,0 on a 2-column line will print "a,,b" so you always know your knew third column has certain content
                } else {
                    toPrint = append(toPrint, fields[num])
                }
            }
        }

        if *noRFC == false {
            csvWriter.Write(toPrint)
        } else {
            fmt.Println(strings.Join(toPrint, outSep))
        }
    }
    if *noRFC == false {
        csvWriter.Flush()
    }
}

func getSeparator(sepString string) (sepRune rune) {
    sepString = `'` + sepString + `'`
    sepRunes, err := strconv.Unquote(sepString)
    if err != nil {
        if err.Error() == "invalid syntax" {  // Single quote was used as separator. No idea why someone would want this, but it doesn't hurt to support it
            sepString = `"` + sepString + `"`
            sepRunes, err = strconv.Unquote(sepString)
            if err != nil {
                panic(err)
            }

        } else {
            panic(err)
        }
    }
    sepRune = ([]rune(sepRunes))[0]

    return sepRune
}
<commit_msg>Added ability to use short flags. Yay short flags! Happy Flag Day, America!<commit_after>package main

import (
    "bufio"
    "encoding/csv"
    "flag"
    "fmt"
    "io"
    "os"
    "strconv"
    "strings"
)

const NILCOMMENTRUNE = "TOTALLYNOTACOMMENTCHAR"
const VERSION = 1.1

// TODO: Add support for specifying fields by field header name instead of just number
// TODO: check ReadRuneFromString instead of existing technique
// TODO: Print version
// TODO: Publish binaries
// TODO: Include test suite
// TODO: Accept filename without --filename flag

var inSep string
var outSep string
var commentRune string

var filename string

var fieldNumsRaw string
var noRFC = flag.Bool("no-rfc", false, "Program defaults to printing RFC 4180-compliant, quoted, well-formatted CSV. If this flag is supplied, output is returned as a string naively joined by --out-sep. --no-rfc is assumed to imply you want to pass the output to naive tools like cut or awk, and in that case, it is recommended that you select an --out-sep that is unlikely to be in youc content, such as a pipe or a backtick.")

var printVersion bool

func init() {
    const (
        filenameFlagLong = "file"
        filenameFlagShort = "f"
        filenameDesc = "File to read from. If not specified, program reads from stdin."

        fieldNumsRawFlagLong = "fields"
        fieldNumsRawFlagShort = "F"
        fieldNumsRawDesc =  "Comma-separated list of field indexes (starting at 0) to print to the command line"

        inSepFlagLong = "in-sep"
        inSepFlagShort = "i"
        inSepDesc = "Single character field separator used by your input"
        inSepDefault = ","

        outSepFlagLong = "out-sep"
        outSepFlagShort = "o"
        outSepDesc = "Single-character field separator to use when printing multiple columns in your output. Only valid if outputting something meant to be passed to cut/awk, and not a properly-formatted, quoted CSV file."
        outSepDefault = ","

        commentRuneFlagLong = "comment-char"
        commentRuneFlagShort = "c"
        commentRuneDesc = "Single-character field separator to use when printing multiple columns in your output. Only valid if outputting something meant to be passed to cut/awk, and not a properly-formatted, quoted CSV file."

        printVersionFlagLong = "version"
        printVersionFlagShort = "v"
        printVersionDesc = "Print program version"
        printVersionDefault = false
    )

    flag.StringVar(&filename, filenameFlagLong, "", filenameDesc)
    flag.StringVar(&filename, filenameFlagShort, "", filenameDesc)

    flag.StringVar(&fieldNumsRaw, fieldNumsRawFlagLong, "", fieldNumsRawDesc)
    flag.StringVar(&fieldNumsRaw, fieldNumsRawFlagShort, "", fieldNumsRawDesc)

    flag.StringVar(&inSep, inSepFlagLong, inSepDefault, inSepDesc)
    flag.StringVar(&inSep, inSepFlagShort, inSepDefault, inSepDesc)

    flag.StringVar(&outSep, outSepFlagLong, outSepDefault, outSepDesc)
    flag.StringVar(&outSep, outSepFlagShort, outSepDefault, outSepDesc)

    flag.StringVar(&commentRune, commentRuneFlagLong, NILCOMMENTRUNE, commentRuneDesc)
    flag.StringVar(&commentRune, commentRuneFlagShort, NILCOMMENTRUNE, commentRuneDesc)

    flag.BoolVar(&printVersion, printVersionFlagLong, printVersionDefault, printVersionDesc)
    flag.BoolVar(&printVersion, printVersionFlagShort, printVersionDefault, printVersionDesc)
}

func main() {
    flag.Parse()

    if printVersion == true {
        fmt.Fprintf(os.Stdout, "csvmaster version %d, made by spiffytech. http://github.com/spiffytech/csvmaster", VERSION)
        return
    }

    var fieldNums []int

    // Identify the numbers you want to print out
    fieldNumsRaw = strings.Trim(fieldNumsRaw, ",")
    if fieldNumsRaw != "" {
        for _, numStr := range strings.Split(fieldNumsRaw, ",") {
            numStr := strings.TrimSpace(numStr)
            numInt, err := strconv.Atoi(numStr)
            if err != nil {
                panic(err)
            }
            fieldNums = append(fieldNums, numInt)
        }
    }

    var reader *bufio.Reader
    if filename == "" {
        reader = bufio.NewReader(os.Stdin)
    } else {
        f, err := os.Open(filename)
        if err != nil {
            panic(err)
        }

        reader = bufio.NewReader(f)
    }

    csvReader := csv.NewReader(reader)
    csvReader.LazyQuotes = true
    csvReader.TrailingComma = true
    csvReader.Comma = getSeparator(inSep)
    csvReader.FieldsPerRecord = -1
    if commentRune != NILCOMMENTRUNE {
        csvReader.Comment = ([]rune(commentRune))[0]
    }

    csvWriter := csv.NewWriter(os.Stdout)
    csvWriter.Comma = getSeparator(outSep)

    for {
        fields, err := csvReader.Read()
        if err != nil {
            if err == io.EOF {
                break
            }
            panic(err)
        }

        var toPrint []string
        if fieldNumsRaw == "" {  // Print all fields
            toPrint = fields
        } else {
            for _, num := range fieldNums {
                if num > len(fields) - 1 {
                    toPrint = append(toPrint, "")  // Append _something_, so printing columns out of order preserves the column index:value mapping in all columns. I.e., --field-nums=1,2,0 on a 2-column line will print "a,,b" so you always know your knew third column has certain content
                } else {
                    toPrint = append(toPrint, fields[num])
                }
            }
        }

        if *noRFC == false {
            csvWriter.Write(toPrint)
        } else {
            fmt.Println(strings.Join(toPrint, outSep))
        }
    }
    if *noRFC == false {
        csvWriter.Flush()
    }
}

func getSeparator(sepString string) (sepRune rune) {
    sepString = `'` + sepString + `'`
    sepRunes, err := strconv.Unquote(sepString)
    if err != nil {
        if err.Error() == "invalid syntax" {  // Single quote was used as separator. No idea why someone would want this, but it doesn't hurt to support it
            sepString = `"` + sepString + `"`
            sepRunes, err = strconv.Unquote(sepString)
            if err != nil {
                panic(err)
            }

        } else {
            panic(err)
        }
    }
    sepRune = ([]rune(sepRunes))[0]

    return sepRune
}
<|endoftext|>


<commit_before>package openapi

import (
	"testing"
)

const (
	exampleCom  = "https://example.com"
	exampleMail = "foo@example.com"
)

type candidate struct {
	label  string
	in     validater
	hasErr bool
}

func testValidater(t *testing.T, candidates []candidate) {
	t.Helper()
	for _, c := range candidates {
		if err := c.in.Validate(); (err != nil) != c.hasErr {
			if c.hasErr {
				t.Error("error should be occurred, but not")
				continue
			}
			t.Errorf("error is occurred: %s", err)
		}
	}
}

func TestHasDuplicatedParameter(t *testing.T) {
	t.Run("no duplicated param", testHasDuplicatedParameterFalse)
	t.Run("there's duplicated param", testHasDuplicatedParameterTrue)
}

func testHasDuplicatedParameterFalse(t *testing.T) {
	params := []*Parameter{
		&Parameter{Name: "foo", In: "header"},
		&Parameter{Name: "foo", In: "path", Required: true},
		&Parameter{Name: "bar", In: "path", Required: true},
	}
	if hasDuplicatedParameter(params) {
		t.Error("should return false")
	}
}

func testHasDuplicatedParameterTrue(t *testing.T) {
	params := []*Parameter{
		&Parameter{Name: "foo", In: "header"},
		&Parameter{Name: "foo", In: "header"},
	}
	if !hasDuplicatedParameter(params) {
		t.Error("should return true")
	}
}

func TestMustURL(t *testing.T) {
	candidates := []struct {
		label  string
		in     string
		hasErr bool
	}{
		{"empty", "", true},
		{"valid HTTP url", "http://example.com", false},
		{"allowed relative path", "foo/bar/baz", true},
		{"absolute path", "/foo/bar/baz", false},
		{"plain string", "foobarbaz", true},
	}
	for _, c := range candidates {
		if err := mustURL(c.label, c.in); (err != nil) != c.hasErr {
			t.Logf("error occured at %s", c.label)
			if c.hasErr {
				t.Error("error should occured, but not")
				return
			}
			t.Error("error should not occurred, but occurred")
			return
		}
	}
}

func TestDocumentValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Document{}, true},
		{"withInvalidVersion", Document{Version: "1.0"}, true},
		{"withVersion", Document{Version: "3.0.0"}, true},
		{"valid", Document{Version: "3.0.0", Info: &Info{Title: "foo", TermsOfService: exampleCom, Version: "1.0"}, Paths: Paths{}}, false},
	}
	testValidater(t, candidates)
}

func TestValidateOASVersion(t *testing.T) {
	candidates := []struct {
		label  string
		in     string
		hasErr bool
	}{
		{"empty", "", true},
		{"invalidVersion", "foobar", true},
		{"swagger", "2.0", true},
		{"valid", "3.0.0", false},
	}
	for _, c := range candidates {
		if err := validateOASVersion(c.in); (err != nil) != c.hasErr {
			t.Log(c.label)
			if c.hasErr {
				t.Error("error should be occurred, but not")
				continue
			}
			t.Errorf("error should not be occured: %s", err)
		}
	}
}

func TestContactValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Contact{}, true},
		{"withURL", Contact{URL: exampleCom}, false},
		{"invalidURL", Contact{URL: "foobar"}, true},
		{"withEmail", Contact{Email: exampleMail}, true},
		{"valid", Contact{URL: exampleCom, Email: exampleMail}, false},
		{"invalidEmail", Contact{URL: exampleCom, Email: "foobar"}, true},
	}

	testValidater(t, candidates)
}

func TestLicenseValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", License{}, true},
		{"withName", License{Name: "foobar"}, true},
		{"withURL", License{URL: exampleCom}, true},
		{"invalidURL", License{Name: "foobar", URL: "foobar"}, true},
		{"valid", License{Name: "foobar", URL: exampleCom}, false},
	}
	testValidater(t, candidates)
}

func TestServerValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Server{}, true},
		{"invalidURL", Server{URL: "foobar%"}, true},
		{"withURL", Server{URL: exampleCom}, false},
	}
	testValidater(t, candidates)
}

func TestServerVariableValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", ServerVariable{}, true},
		{"withDefault", ServerVariable{Default: "default"}, false},
	}
	testValidater(t, candidates)
}

func TestPathsValidate(t *testing.T) {
	t.Run("duplicate pathItem", testPathItemDuplicate)
}

func getPaths(id1, id2 string) Paths {
	return Paths{
		"/foo/bar": &PathItem{
			Get:  &Operation{OperationID: id1, Responses: Responses{"200": &Response{Description: "foo"}}},
			Post: &Operation{OperationID: id2, Responses: Responses{"200": &Response{Description: "foo"}}},
		},
	}
}

func testPathItemDuplicate(t *testing.T) {
	candidates := []candidate{
		{"invalid", getPaths("foobar", "foobar"), true},
		{"valid", getPaths("foo", "bar"), false},
	}
	testValidater(t, candidates)
}

func TestExternalDocumentationValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", ExternalDocumentation{}, true},
		{"invalidURL", ExternalDocumentation{URL: "foobar"}, true},
		{"valid", ExternalDocumentation{URL: exampleCom}, false},
	}
	testValidater(t, candidates)
}

func TestTagValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Tag{}, true},
		{"withEmptyExternalDocs", Tag{ExternalDocs: &ExternalDocumentation{}}, true},
		{"withValidExternalDocs", Tag{ExternalDocs: &ExternalDocumentation{URL: exampleCom}}, true},

		{"withName", Tag{Name: "foo"}, false},
	}
	testValidater(t, candidates)
}

func TestDiscriminatorValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Discriminator{}, true},
		{"withPropertyName", Discriminator{PropertyName: "foobar"}, false},
	}
	testValidater(t, candidates)
}

func TestXMLValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", XML{}, true},
		{"invalidURLNamespace", XML{Namespace: "foobar"}, true},
		{"withNamespace", XML{Namespace: exampleCom}, false},
	}
	testValidater(t, candidates)
}

func TestOAuthFlowValidate(t *testing.T) {
	mockScopes := map[string]string{"foo": "bar"}

	empty := OAuthFlow{}
	aURL := OAuthFlow{AuthorizationURL: exampleCom}
	tURL := OAuthFlow{TokenURL: exampleCom}
	rURL := OAuthFlow{RefreshURL: exampleCom}
	scopes := OAuthFlow{Scopes: mockScopes}
	atURL := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom}
	arURL := OAuthFlow{AuthorizationURL: exampleCom, RefreshURL: exampleCom}
	aURLscopes := OAuthFlow{AuthorizationURL: exampleCom, Scopes: mockScopes}
	trURL := OAuthFlow{TokenURL: exampleCom, RefreshURL: exampleCom}
	tURLscopes := OAuthFlow{TokenURL: exampleCom, Scopes: mockScopes}
	rURLscopes := OAuthFlow{RefreshURL: exampleCom, Scopes: mockScopes}
	atrURL := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom, RefreshURL: exampleCom}
	atURLscopes := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom, Scopes: mockScopes}
	arURLscopes := OAuthFlow{AuthorizationURL: exampleCom, RefreshURL: exampleCom, Scopes: mockScopes}
	trURLscopes := OAuthFlow{TokenURL: exampleCom, RefreshURL: exampleCom, Scopes: mockScopes}
	atrURLscopes := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom, RefreshURL: exampleCom, Scopes: mockScopes}
	invalidURL := OAuthFlow{AuthorizationURL: "foobar", TokenURL: "foobar", RefreshURL: "foobar", Scopes: mockScopes}
	zeroMap := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom, RefreshURL: exampleCom, Scopes: map[string]string{}}

	candidates := []struct {
		label   string
		in      OAuthFlow
		haveErr [4]bool
	}{
		{"empty", empty, [4]bool{true, true, true, true}},
		{"aURL", aURL, [4]bool{true, true, true, true}},
		{"tURL", tURL, [4]bool{true, true, true, true}},
		{"rURL", rURL, [4]bool{true, true, true, true}},
		{"scopes", scopes, [4]bool{true, true, true, true}},
		{"aURL/tURL", atURL, [4]bool{true, true, true, true}},
		{"aURL/rURL", arURL, [4]bool{true, true, true, true}},
		{"aURL/scopes", aURLscopes, [4]bool{false, true, true, true}},
		{"tURL/rURL", trURL, [4]bool{true, true, true, true}},
		{"tURL/scopes", tURLscopes, [4]bool{true, false, false, true}},
		{"rURL/scopes", rURLscopes, [4]bool{true, true, true, true}},
		{"aURL/tURL/rURL", atrURL, [4]bool{true, true, true, true}},
		{"aURL/tURL/scopes", atURLscopes, [4]bool{false, false, false, false}},
		{"aURL/rURL/scopes", arURLscopes, [4]bool{false, true, true, true}},
		{"tURL/rURL/scopes", trURLscopes, [4]bool{true, false, false, true}},
		{"aURL/tURL/rURL/scopes", atrURLscopes, [4]bool{false, false, false, false}},

		{"invalidURL", invalidURL, [4]bool{true, true, true, true}},
		{"zero length map", zeroMap, [4]bool{true, true, true, true}},
	}
	for _, c := range candidates {
		testOAuthFlowValidate(t, c.label, c.in, c.haveErr)
	}
}

var flowTypes = []string{"implicit", "password", "clientCredentials", "authorizationCode"}

func testOAuthFlowValidate(t *testing.T, label string, oauthFlow OAuthFlow, haveErr [4]bool) {
	if err := oauthFlow.Validate(""); err == nil {
		t.Logf("%s-empty", label)
		t.Error("error should be occurred, but not")
	}
	if err := oauthFlow.Validate("foobar"); err == nil {
		t.Logf("%s-wrongtype", label)
		t.Error("error should be occurred, but not")
	}
	for i, flowType := range flowTypes {
		if err := oauthFlow.Validate(flowType); (err != nil) != haveErr[i] {
			t.Logf("%s-%s", label, flowType)
			if haveErr[i] {
				t.Error("error should be occurred, but not")
				continue
			}
			t.Error("error should not be occurred, but occurred")
			t.Log(err)
		}
	}
}
<commit_msg>add empty test for schema<commit_after>package openapi

import (
	"testing"
)

const (
	exampleCom  = "https://example.com"
	exampleMail = "foo@example.com"
)

type candidate struct {
	label  string
	in     validater
	hasErr bool
}

func testValidater(t *testing.T, candidates []candidate) {
	t.Helper()
	for _, c := range candidates {
		if err := c.in.Validate(); (err != nil) != c.hasErr {
			if c.hasErr {
				t.Error("error should be occurred, but not")
				continue
			}
			t.Errorf("error is occurred: %s", err)
		}
	}
}

func TestHasDuplicatedParameter(t *testing.T) {
	t.Run("no duplicated param", testHasDuplicatedParameterFalse)
	t.Run("there's duplicated param", testHasDuplicatedParameterTrue)
}

func testHasDuplicatedParameterFalse(t *testing.T) {
	params := []*Parameter{
		&Parameter{Name: "foo", In: "header"},
		&Parameter{Name: "foo", In: "path", Required: true},
		&Parameter{Name: "bar", In: "path", Required: true},
	}
	if hasDuplicatedParameter(params) {
		t.Error("should return false")
	}
}

func testHasDuplicatedParameterTrue(t *testing.T) {
	params := []*Parameter{
		&Parameter{Name: "foo", In: "header"},
		&Parameter{Name: "foo", In: "header"},
	}
	if !hasDuplicatedParameter(params) {
		t.Error("should return true")
	}
}

func TestMustURL(t *testing.T) {
	candidates := []struct {
		label  string
		in     string
		hasErr bool
	}{
		{"empty", "", true},
		{"valid HTTP url", "http://example.com", false},
		{"allowed relative path", "foo/bar/baz", true},
		{"absolute path", "/foo/bar/baz", false},
		{"plain string", "foobarbaz", true},
	}
	for _, c := range candidates {
		if err := mustURL(c.label, c.in); (err != nil) != c.hasErr {
			t.Logf("error occured at %s", c.label)
			if c.hasErr {
				t.Error("error should occured, but not")
				return
			}
			t.Error("error should not occurred, but occurred")
			return
		}
	}
}

func TestDocumentValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Document{}, true},
		{"withInvalidVersion", Document{Version: "1.0"}, true},
		{"withVersion", Document{Version: "3.0.0"}, true},
		{"valid", Document{Version: "3.0.0", Info: &Info{Title: "foo", TermsOfService: exampleCom, Version: "1.0"}, Paths: Paths{}}, false},
	}
	testValidater(t, candidates)
}

func TestValidateOASVersion(t *testing.T) {
	candidates := []struct {
		label  string
		in     string
		hasErr bool
	}{
		{"empty", "", true},
		{"invalidVersion", "foobar", true},
		{"swagger", "2.0", true},
		{"valid", "3.0.0", false},
	}
	for _, c := range candidates {
		if err := validateOASVersion(c.in); (err != nil) != c.hasErr {
			t.Log(c.label)
			if c.hasErr {
				t.Error("error should be occurred, but not")
				continue
			}
			t.Errorf("error should not be occured: %s", err)
		}
	}
}

func TestContactValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Contact{}, true},
		{"withURL", Contact{URL: exampleCom}, false},
		{"invalidURL", Contact{URL: "foobar"}, true},
		{"withEmail", Contact{Email: exampleMail}, true},
		{"valid", Contact{URL: exampleCom, Email: exampleMail}, false},
		{"invalidEmail", Contact{URL: exampleCom, Email: "foobar"}, true},
	}

	testValidater(t, candidates)
}

func TestLicenseValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", License{}, true},
		{"withName", License{Name: "foobar"}, true},
		{"withURL", License{URL: exampleCom}, true},
		{"invalidURL", License{Name: "foobar", URL: "foobar"}, true},
		{"valid", License{Name: "foobar", URL: exampleCom}, false},
	}
	testValidater(t, candidates)
}

func TestServerValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Server{}, true},
		{"invalidURL", Server{URL: "foobar%"}, true},
		{"withURL", Server{URL: exampleCom}, false},
	}
	testValidater(t, candidates)
}

func TestServerVariableValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", ServerVariable{}, true},
		{"withDefault", ServerVariable{Default: "default"}, false},
	}
	testValidater(t, candidates)
}

func TestPathsValidate(t *testing.T) {
	t.Run("duplicate pathItem", testPathItemDuplicate)
}

func getPaths(id1, id2 string) Paths {
	return Paths{
		"/foo/bar": &PathItem{
			Get:  &Operation{OperationID: id1, Responses: Responses{"200": &Response{Description: "foo"}}},
			Post: &Operation{OperationID: id2, Responses: Responses{"200": &Response{Description: "foo"}}},
		},
	}
}

func testPathItemDuplicate(t *testing.T) {
	candidates := []candidate{
		{"invalid", getPaths("foobar", "foobar"), true},
		{"valid", getPaths("foo", "bar"), false},
	}
	testValidater(t, candidates)
}

func TestExternalDocumentationValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", ExternalDocumentation{}, true},
		{"invalidURL", ExternalDocumentation{URL: "foobar"}, true},
		{"valid", ExternalDocumentation{URL: exampleCom}, false},
	}
	testValidater(t, candidates)
}

func TestTagValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Tag{}, true},
		{"withEmptyExternalDocs", Tag{ExternalDocs: &ExternalDocumentation{}}, true},
		{"withValidExternalDocs", Tag{ExternalDocs: &ExternalDocumentation{URL: exampleCom}}, true},

		{"withName", Tag{Name: "foo"}, false},
	}
	testValidater(t, candidates)
}

func TestSchemaValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Schema{}, false},
	}
	testValidater(t, candidates)
}

func TestDiscriminatorValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Discriminator{}, true},
		{"withPropertyName", Discriminator{PropertyName: "foobar"}, false},
	}
	testValidater(t, candidates)
}

func TestXMLValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", XML{}, true},
		{"invalidURLNamespace", XML{Namespace: "foobar"}, true},
		{"withNamespace", XML{Namespace: exampleCom}, false},
	}
	testValidater(t, candidates)
}

func TestOAuthFlowValidate(t *testing.T) {
	mockScopes := map[string]string{"foo": "bar"}

	empty := OAuthFlow{}
	aURL := OAuthFlow{AuthorizationURL: exampleCom}
	tURL := OAuthFlow{TokenURL: exampleCom}
	rURL := OAuthFlow{RefreshURL: exampleCom}
	scopes := OAuthFlow{Scopes: mockScopes}
	atURL := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom}
	arURL := OAuthFlow{AuthorizationURL: exampleCom, RefreshURL: exampleCom}
	aURLscopes := OAuthFlow{AuthorizationURL: exampleCom, Scopes: mockScopes}
	trURL := OAuthFlow{TokenURL: exampleCom, RefreshURL: exampleCom}
	tURLscopes := OAuthFlow{TokenURL: exampleCom, Scopes: mockScopes}
	rURLscopes := OAuthFlow{RefreshURL: exampleCom, Scopes: mockScopes}
	atrURL := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom, RefreshURL: exampleCom}
	atURLscopes := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom, Scopes: mockScopes}
	arURLscopes := OAuthFlow{AuthorizationURL: exampleCom, RefreshURL: exampleCom, Scopes: mockScopes}
	trURLscopes := OAuthFlow{TokenURL: exampleCom, RefreshURL: exampleCom, Scopes: mockScopes}
	atrURLscopes := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom, RefreshURL: exampleCom, Scopes: mockScopes}
	invalidURL := OAuthFlow{AuthorizationURL: "foobar", TokenURL: "foobar", RefreshURL: "foobar", Scopes: mockScopes}
	zeroMap := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom, RefreshURL: exampleCom, Scopes: map[string]string{}}

	candidates := []struct {
		label   string
		in      OAuthFlow
		haveErr [4]bool
	}{
		{"empty", empty, [4]bool{true, true, true, true}},
		{"aURL", aURL, [4]bool{true, true, true, true}},
		{"tURL", tURL, [4]bool{true, true, true, true}},
		{"rURL", rURL, [4]bool{true, true, true, true}},
		{"scopes", scopes, [4]bool{true, true, true, true}},
		{"aURL/tURL", atURL, [4]bool{true, true, true, true}},
		{"aURL/rURL", arURL, [4]bool{true, true, true, true}},
		{"aURL/scopes", aURLscopes, [4]bool{false, true, true, true}},
		{"tURL/rURL", trURL, [4]bool{true, true, true, true}},
		{"tURL/scopes", tURLscopes, [4]bool{true, false, false, true}},
		{"rURL/scopes", rURLscopes, [4]bool{true, true, true, true}},
		{"aURL/tURL/rURL", atrURL, [4]bool{true, true, true, true}},
		{"aURL/tURL/scopes", atURLscopes, [4]bool{false, false, false, false}},
		{"aURL/rURL/scopes", arURLscopes, [4]bool{false, true, true, true}},
		{"tURL/rURL/scopes", trURLscopes, [4]bool{true, false, false, true}},
		{"aURL/tURL/rURL/scopes", atrURLscopes, [4]bool{false, false, false, false}},

		{"invalidURL", invalidURL, [4]bool{true, true, true, true}},
		{"zero length map", zeroMap, [4]bool{true, true, true, true}},
	}
	for _, c := range candidates {
		testOAuthFlowValidate(t, c.label, c.in, c.haveErr)
	}
}

var flowTypes = []string{"implicit", "password", "clientCredentials", "authorizationCode"}

func testOAuthFlowValidate(t *testing.T, label string, oauthFlow OAuthFlow, haveErr [4]bool) {
	if err := oauthFlow.Validate(""); err == nil {
		t.Logf("%s-empty", label)
		t.Error("error should be occurred, but not")
	}
	if err := oauthFlow.Validate("foobar"); err == nil {
		t.Logf("%s-wrongtype", label)
		t.Error("error should be occurred, but not")
	}
	for i, flowType := range flowTypes {
		if err := oauthFlow.Validate(flowType); (err != nil) != haveErr[i] {
			t.Logf("%s-%s", label, flowType)
			if haveErr[i] {
				t.Error("error should be occurred, but not")
				continue
			}
			t.Error("error should not be occurred, but occurred")
			t.Log(err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"strings"
	"time"
)

type Crawler interface {
	Crawl()
}

func NewCrawler(travis *Travis, db *DB) []Crawler {
	return []Crawler{
		&NewBuildCrawler{travis, db, log.New(os.Stderr, "[NewBuildCrawler] ", log.LstdFlags)},
		&FinishedBuildCrawler{travis, db, log.New(os.Stderr, "[FinishedBuildCrawler] ", log.LstdFlags)},
	}
}

type NewBuildCrawler struct {
	Travis *Travis
	DB     *DB
	Logger *log.Logger
}

func (c *NewBuildCrawler) Crawl() {
	ch := time.Tick(10 * time.Second)
	for _ = range ch {
		c.Logger.Println("crawling for new builds...")
		c.crawlNewBuilds()
	}
}

func (c *NewBuildCrawler) crawlNewBuilds() {
	repos, err := c.Travis.Repos()
	if err != nil {
		c.Logger.Println(err)
		return
	}

	newBuilds := []string{}
	for _, repo := range repos {
		updated, err := c.DB.Upsert("new_builds", Query{"lastbuildid": repo.LastBuildId}, repo)
		if err != nil {
			c.Logger.Println(err)
			continue
		}

		if updated {
			newBuilds = append(newBuilds, repo.Slug)
		}
	}

	c.Logger.Printf("harvested %d builds with %d new builds: %s\n", len(repos), len(newBuilds), strings.Join(newBuilds, ", "))
}

type FinishedBuildCrawler struct {
	Travis *Travis
	DB     *DB
	Logger *log.Logger
}

func (c *FinishedBuildCrawler) Crawl() {
	ch := time.Tick(30 * time.Second)
	for _ = range ch {
		c.Logger.Println("crawling for finsihed builds...")
		c.crawlFinishedBuilds()
	}
}

func (c *FinishedBuildCrawler) crawlFinishedBuilds() {
	colNames, finishedBuilds := c.doCrawlFinishedBuilds()
	c.Logger.Printf("harvested %d finsihed builds: %s\n", len(finishedBuilds), strings.Join(finishedBuilds, ", "))

	err := c.ensureColIndexes(colNames)
	if err != nil {
		c.Logger.Println(err)
	}
}

func (c *FinishedBuildCrawler) doCrawlFinishedBuilds() (colNames map[string]string, finishedBuilds []string) {
	colNames = make(map[string]string)

	var repo *Repo

	iter := c.DB.C("new_builds").Find(nil).Iter()
	for iter.Next(&repo) {
		build, err := c.crawlFinsihedBuild(repo)
		if err != nil {
			c.Logger.Println(err)
			continue
		}

		colName, updated, err := c.upsertBuild(build)
		if err != nil {
			c.Logger.Println(err)
			continue
		}

		colNames[colName] = colName

		if updated {
			finishedBuilds = append(finishedBuilds, repo.Slug)
		}
	}

	return
}

func (c *FinishedBuildCrawler) crawlFinsihedBuild(repo *Repo) (build *Build, err error) {
	build, err = c.Travis.Build(repo.LastBuildId)
	if err != nil {
		return
	}

	isFinished := !(build.FinishedAt == nil || build.StartedAt == nil)
	if !isFinished {
		err = fmt.Errorf("skipping build: %s - %d\n", repo.Slug, repo.LastBuildId)
		return
	}

	build.Repository = repo

	return
}

func (c *FinishedBuildCrawler) upsertBuild(build *Build) (colName string, updated bool, err error) {
	colName = buildColName(build.StartedAt)

	updated, err = c.DB.Upsert(colName, Query{"id": build.Id}, build)
	if err != nil {
		return
	}

	err = c.DB.C("new_builds").Remove(Query{"lastbuildid": build.Repository.LastBuildId})
	if err != nil {
		return
	}

	return
}

func (c *FinishedBuildCrawler) ensureColIndexes(colNames map[string]string) error {
	for _, colName := range colNames {
		c.Logger.Printf("ensuring index for collection %s\n", colName)
		err := c.DB.EnsureIndexKey(colName, "id")
		if err != nil {
			return err
		}
	}

	return nil
}

func buildColName(date *time.Time) string {
	buildDate := date.UTC().Format("2006_01_02")
	return fmt.Sprintf("builds_%s", buildDate)
}
<commit_msg>Only fetch finished builds every 60 sec<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"strings"
	"time"
)

type Crawler interface {
	Crawl()
}

func NewCrawler(travis *Travis, db *DB) []Crawler {
	return []Crawler{
		&NewBuildCrawler{travis, db, log.New(os.Stderr, "[NewBuildCrawler] ", log.LstdFlags)},
		&FinishedBuildCrawler{travis, db, log.New(os.Stderr, "[FinishedBuildCrawler] ", log.LstdFlags)},
	}
}

type NewBuildCrawler struct {
	Travis *Travis
	DB     *DB
	Logger *log.Logger
}

func (c *NewBuildCrawler) Crawl() {
	ch := time.Tick(10 * time.Second)
	for _ = range ch {
		c.Logger.Println("crawling for new builds...")
		c.crawlNewBuilds()
	}
}

func (c *NewBuildCrawler) crawlNewBuilds() {
	repos, err := c.Travis.Repos()
	if err != nil {
		c.Logger.Println(err)
		return
	}

	newBuilds := []string{}
	for _, repo := range repos {
		updated, err := c.DB.Upsert("new_builds", Query{"lastbuildid": repo.LastBuildId}, repo)
		if err != nil {
			c.Logger.Println(err)
			continue
		}

		if updated {
			newBuilds = append(newBuilds, repo.Slug)
		}
	}

	c.Logger.Printf("harvested %d builds with %d new builds: %s\n", len(repos), len(newBuilds), strings.Join(newBuilds, ", "))
}

type FinishedBuildCrawler struct {
	Travis *Travis
	DB     *DB
	Logger *log.Logger
}

func (c *FinishedBuildCrawler) Crawl() {
	ch := time.Tick(60 * time.Second)
	for _ = range ch {
		c.Logger.Println("crawling for finsihed builds...")
		c.crawlFinishedBuilds()
	}
}

func (c *FinishedBuildCrawler) crawlFinishedBuilds() {
	colNames, finishedBuilds := c.doCrawlFinishedBuilds()
	c.Logger.Printf("harvested %d finsihed builds: %s\n", len(finishedBuilds), strings.Join(finishedBuilds, ", "))

	err := c.ensureColIndexes(colNames)
	if err != nil {
		c.Logger.Println(err)
	}
}

func (c *FinishedBuildCrawler) doCrawlFinishedBuilds() (colNames map[string]string, finishedBuilds []string) {
	colNames = make(map[string]string)

	var repo *Repo

	iter := c.DB.C("new_builds").Find(nil).Iter()
	for iter.Next(&repo) {
		build, err := c.crawlFinsihedBuild(repo)
		if err != nil {
			c.Logger.Println(err)
			continue
		}

		colName, updated, err := c.upsertBuild(build)
		if err != nil {
			c.Logger.Println(err)
			continue
		}

		colNames[colName] = colName

		if updated {
			finishedBuilds = append(finishedBuilds, repo.Slug)
		}
	}

	return
}

func (c *FinishedBuildCrawler) crawlFinsihedBuild(repo *Repo) (build *Build, err error) {
	build, err = c.Travis.Build(repo.LastBuildId)
	if err != nil {
		return
	}

	isFinished := !(build.FinishedAt == nil || build.StartedAt == nil)
	if !isFinished {
		err = fmt.Errorf("skipping build: %s - %d\n", repo.Slug, repo.LastBuildId)
		return
	}

	build.Repository = repo

	return
}

func (c *FinishedBuildCrawler) upsertBuild(build *Build) (colName string, updated bool, err error) {
	colName = buildColName(build.StartedAt)

	updated, err = c.DB.Upsert(colName, Query{"id": build.Id}, build)
	if err != nil {
		return
	}

	err = c.DB.C("new_builds").Remove(Query{"lastbuildid": build.Repository.LastBuildId})
	if err != nil {
		return
	}

	return
}

func (c *FinishedBuildCrawler) ensureColIndexes(colNames map[string]string) error {
	for _, colName := range colNames {
		c.Logger.Printf("ensuring index for collection %s\n", colName)
		err := c.DB.EnsureIndexKey(colName, "id")
		if err != nil {
			return err
		}
	}

	return nil
}

func buildColName(date *time.Time) string {
	buildDate := date.UTC().Format("2006_01_02")
	return fmt.Sprintf("builds_%s", buildDate)
}
<|endoftext|>


<commit_before>package crawler

import (
	"../log"
	"github.com/PuerkitoBio/goquery"
)
func Crawl() {
	doc, err := goquery.NewDocument("https://blog.golang.org")
	if err != nil {
		log.Fatal(err)
	}

	titles := make([]string, 10)
	doc.Find(".article").Each(func(i int, s *goquery.Selection) {
		title := s.Find("h3").Text()
		titles = append(titles, title)
	})
	log.Info(titles)
	log.Flush()
}
<commit_msg>deferred flush(), return in case of fatal<commit_after>package crawler

import (
	"../log"
	"github.com/PuerkitoBio/goquery"
)
func Crawl() {
	defer log.Flush()
	log.Info("Started crawl")

	doc, err := goquery.NewDocument("https://blog.golang.org")
	if err != nil {
		log.Fatal(err)
		return
	}

	titles := make([]string, 10)
	doc.Find(".article").Each(func(i int, s *goquery.Selection) {
		title := s.Find("h3").Text()
		titles = append(titles, title)
	})
	log.Info(titles)
	log.Info("Finished crawl")
}
<|endoftext|>


<commit_before>package fasthttp

import (
	"net"
	"runtime"
	"runtime/debug"
	"strings"
	"sync"
	"time"
)

// workerPool serves incoming connections via a pool of workers
// in FILO order, i.e. the most recently stopped worker will serve the next
// incoming connection.
//
// Such a scheme keeps CPU caches hot (in theory).
type workerPool struct {
	// Function for serving server connections.
	// It must leave c unclosed.
	WorkerFunc func(c net.Conn) error

	MaxWorkersCount int

	LogAllErrors bool

	Logger Logger

	lock         sync.Mutex
	workersCount int
	mustStop     bool

	ready []*workerChan

	stopCh chan struct{}

	workerChanPool sync.Pool
}

type workerChan struct {
	lastUseTime time.Time
	ch          chan net.Conn
}

func (wp *workerPool) Start() {
	if wp.stopCh != nil {
		panic("BUG: workerPool already started")
	}
	wp.stopCh = make(chan struct{})
	stopCh := wp.stopCh
	go func() {
		var scratch []*workerChan
		for {
			select {
			case <-stopCh:
				return
			default:
				time.Sleep(10 * time.Second)
			}
			wp.clean(&scratch)
		}
	}()
}

func (wp *workerPool) Stop() {
	if wp.stopCh == nil {
		panic("BUG: workerPool wasn't started")
	}
	close(wp.stopCh)
	wp.stopCh = nil

	// Stop all the workers waiting for incoming connections.
	// Do not wait for busy workers - they will stop after
	// serving the connection and noticing wp.mustStop = true.
	wp.lock.Lock()
	for _, ch := range wp.ready {
		ch.ch <- nil
	}
	wp.ready = nil
	wp.mustStop = true
	wp.lock.Unlock()
}

const maxIdleWorkerDuration = 60 * time.Second

func (wp *workerPool) clean(scratch *[]*workerChan) {
	// Clean least recently used workers if they didn't serve connections
	// for more than maxIdleWorkerDuration.
	currentTime := time.Now()

	wp.lock.Lock()
	ready := wp.ready
	n := len(ready)
	i := 0
	for i < n && currentTime.Sub(ready[i].lastUseTime) > maxIdleWorkerDuration {
		i++
	}
	if i > 0 {
		*scratch = append((*scratch)[:0], ready[:i]...)
		m := copy(ready, ready[i:])
		for i = m; i < n; i++ {
			ready[i] = nil
		}
		wp.ready = ready[:m]
	}
	wp.lock.Unlock()

	// Notify obsolete workers to stop.
	// This notification must be outside the wp.lock, since ch.ch
	// may be blocking and may consume a lot of time if many workers
	// are located on non-local CPUs.
	tmp := *scratch
	for i, ch := range tmp {
		ch.ch <- nil
		tmp[i] = nil
	}
}

func (wp *workerPool) Serve(c net.Conn) bool {
	ch := wp.getCh()
	if ch == nil {
		return false
	}
	ch.ch <- c
	return true
}

var workerChanCap = func() int {
	// Use blocking workerChan if GOMAXPROCS=1.
	// This immediately switches Serve to WorkerFunc, which results
	// in higher performance (under go1.5 at least).
	if runtime.GOMAXPROCS(0) == 1 {
		return 0
	}

	// Use non-blocking workerChan if GOMAXPROCS>1,
	// since otherwise the Serve caller (Acceptor) may lag accepting
	// new connections if WorkerFunc is CPU-bound.
	return 1
}()

func (wp *workerPool) getCh() *workerChan {
	var ch *workerChan
	createWorker := false

	wp.lock.Lock()
	ready := wp.ready
	n := len(ready) - 1
	if n < 0 {
		if wp.workersCount < wp.MaxWorkersCount {
			createWorker = true
			wp.workersCount++
		}
	} else {
		ch = ready[n]
		ready[n] = nil
		wp.ready = ready[:n]
	}
	wp.lock.Unlock()

	if ch == nil {
		if !createWorker {
			return nil
		}
		vch := wp.workerChanPool.Get()
		if vch == nil {
			vch = &workerChan{
				ch: make(chan net.Conn, workerChanCap),
			}
		}
		ch = vch.(*workerChan)
		go func() {
			wp.workerFunc(ch)
			wp.workerChanPool.Put(vch)
		}()
	}
	return ch
}

func (wp *workerPool) release(ch *workerChan) bool {
	ch.lastUseTime = time.Now()
	wp.lock.Lock()
	if wp.mustStop {
		wp.lock.Unlock()
		return false
	}
	wp.ready = append(wp.ready, ch)
	wp.lock.Unlock()
	return true
}

func (wp *workerPool) workerFunc(ch *workerChan) {
	var c net.Conn
	var err error

	defer func() {
		if r := recover(); r != nil {
			wp.Logger.Printf("panic: %s\nStack trace:\n%s", r, debug.Stack())
		}

		if c != nil {
			c.Close()
			wp.release(ch)
		}
		wp.lock.Lock()
		wp.workersCount--
		wp.lock.Unlock()
	}()

	for c = range ch.ch {
		if c == nil {
			break
		}
		if err = wp.WorkerFunc(c); err != nil && err != errHijacked {
			errStr := err.Error()
			if wp.LogAllErrors || !(strings.Contains(errStr, "broken pipe") ||
				strings.Contains(errStr, "reset by peer") ||
				strings.Contains(errStr, "i/o timeout")) {
				wp.Logger.Printf("error when serving connection %q<->%q: %s", c.LocalAddr(), c.RemoteAddr(), err)
			}
		}
		if err != errHijacked {
			c.Close()
		}
		c = nil

		if !wp.release(ch) {
			break
		}
	}
}
<commit_msg>workerpool: properly clean scratch buffer for workerChans<commit_after>package fasthttp

import (
	"net"
	"runtime"
	"runtime/debug"
	"strings"
	"sync"
	"time"
)

// workerPool serves incoming connections via a pool of workers
// in FILO order, i.e. the most recently stopped worker will serve the next
// incoming connection.
//
// Such a scheme keeps CPU caches hot (in theory).
type workerPool struct {
	// Function for serving server connections.
	// It must leave c unclosed.
	WorkerFunc func(c net.Conn) error

	MaxWorkersCount int

	LogAllErrors bool

	Logger Logger

	lock         sync.Mutex
	workersCount int
	mustStop     bool

	ready []*workerChan

	stopCh chan struct{}

	workerChanPool sync.Pool
}

type workerChan struct {
	lastUseTime time.Time
	ch          chan net.Conn
}

func (wp *workerPool) Start() {
	if wp.stopCh != nil {
		panic("BUG: workerPool already started")
	}
	wp.stopCh = make(chan struct{})
	stopCh := wp.stopCh
	go func() {
		var scratch []*workerChan
		for {
			select {
			case <-stopCh:
				return
			default:
				time.Sleep(10 * time.Second)
			}
			wp.clean(&scratch)
		}
	}()
}

func (wp *workerPool) Stop() {
	if wp.stopCh == nil {
		panic("BUG: workerPool wasn't started")
	}
	close(wp.stopCh)
	wp.stopCh = nil

	// Stop all the workers waiting for incoming connections.
	// Do not wait for busy workers - they will stop after
	// serving the connection and noticing wp.mustStop = true.
	wp.lock.Lock()
	for _, ch := range wp.ready {
		ch.ch <- nil
	}
	wp.ready = nil
	wp.mustStop = true
	wp.lock.Unlock()
}

const maxIdleWorkerDuration = 60 * time.Second

func (wp *workerPool) clean(scratch *[]*workerChan) {
	// Clean least recently used workers if they didn't serve connections
	// for more than maxIdleWorkerDuration.
	currentTime := time.Now()

	wp.lock.Lock()
	ready := wp.ready
	n := len(ready)
	i := 0
	for i < n && currentTime.Sub(ready[i].lastUseTime) > maxIdleWorkerDuration {
		i++
	}
	*scratch = append((*scratch)[:0], ready[:i]...)
	if i > 0 {
		m := copy(ready, ready[i:])
		for i = m; i < n; i++ {
			ready[i] = nil
		}
		wp.ready = ready[:m]
	}
	wp.lock.Unlock()

	// Notify obsolete workers to stop.
	// This notification must be outside the wp.lock, since ch.ch
	// may be blocking and may consume a lot of time if many workers
	// are located on non-local CPUs.
	tmp := *scratch
	for i, ch := range tmp {
		ch.ch <- nil
		tmp[i] = nil
	}
}

func (wp *workerPool) Serve(c net.Conn) bool {
	ch := wp.getCh()
	if ch == nil {
		return false
	}
	ch.ch <- c
	return true
}

var workerChanCap = func() int {
	// Use blocking workerChan if GOMAXPROCS=1.
	// This immediately switches Serve to WorkerFunc, which results
	// in higher performance (under go1.5 at least).
	if runtime.GOMAXPROCS(0) == 1 {
		return 0
	}

	// Use non-blocking workerChan if GOMAXPROCS>1,
	// since otherwise the Serve caller (Acceptor) may lag accepting
	// new connections if WorkerFunc is CPU-bound.
	return 1
}()

func (wp *workerPool) getCh() *workerChan {
	var ch *workerChan
	createWorker := false

	wp.lock.Lock()
	ready := wp.ready
	n := len(ready) - 1
	if n < 0 {
		if wp.workersCount < wp.MaxWorkersCount {
			createWorker = true
			wp.workersCount++
		}
	} else {
		ch = ready[n]
		ready[n] = nil
		wp.ready = ready[:n]
	}
	wp.lock.Unlock()

	if ch == nil {
		if !createWorker {
			return nil
		}
		vch := wp.workerChanPool.Get()
		if vch == nil {
			vch = &workerChan{
				ch: make(chan net.Conn, workerChanCap),
			}
		}
		ch = vch.(*workerChan)
		go func() {
			wp.workerFunc(ch)
			wp.workerChanPool.Put(vch)
		}()
	}
	return ch
}

func (wp *workerPool) release(ch *workerChan) bool {
	ch.lastUseTime = time.Now()
	wp.lock.Lock()
	if wp.mustStop {
		wp.lock.Unlock()
		return false
	}
	wp.ready = append(wp.ready, ch)
	wp.lock.Unlock()
	return true
}

func (wp *workerPool) workerFunc(ch *workerChan) {
	var c net.Conn
	var err error

	defer func() {
		if r := recover(); r != nil {
			wp.Logger.Printf("panic: %s\nStack trace:\n%s", r, debug.Stack())
		}

		if c != nil {
			c.Close()
			wp.release(ch)
		}
		wp.lock.Lock()
		wp.workersCount--
		wp.lock.Unlock()
	}()

	for c = range ch.ch {
		if c == nil {
			break
		}
		if err = wp.WorkerFunc(c); err != nil && err != errHijacked {
			errStr := err.Error()
			if wp.LogAllErrors || !(strings.Contains(errStr, "broken pipe") ||
				strings.Contains(errStr, "reset by peer") ||
				strings.Contains(errStr, "i/o timeout")) {
				wp.Logger.Printf("error when serving connection %q<->%q: %s", c.LocalAddr(), c.RemoteAddr(), err)
			}
		}
		if err != errHijacked {
			c.Close()
		}
		c = nil

		if !wp.release(ch) {
			break
		}
	}
}
<|endoftext|>


<commit_before>package strategies

import (
	"log"

	"github.com/lean-poker/poker-player-go/leanpoker"
	"github.com/lean-poker/poker-player-go/rank_api"
)

const (
	BaseRank = 10
)

func Default(game *leanpoker.Game) int {
	if rank, ok := getRank(game); ok {
		log.Printf("Make bet = %v", rank)
		return rank
	}

	if checkBigBed(game) {
		log.Printf("Got big bet")
		return 0
	}

	log.Printf("No strategy. Calling.")
	return game.Call()
}

func getRank(game *leanpoker.Game) (int, bool) {
	cards := game.Cards()

	if len(cards) < 5 {
		log.Printf("Less 5 cards")
		return 0, false
	}

	rank := rank_api.GetRank(cards)

	b := game.SmallBlind

	switch rank.Rank {
	case 1:
		return BaseRank, true
	case 2:
		return raiseOrCall(2*b, game, 0.1), true
	case 3:
		return raiseOrCall(3*b, game, 0.3), true
	case 4:
		return raiseOrCall(4*b, game, 0.5), true
	case 5:
		return raiseOrCall(5*b, game, 0.5), true
	case 6:
		return raiseOrCall(6*b, game, 1), true
	case 7:
		return raiseOrCall(7*b, game, 1), true
	case 8:
		return raiseOrCall(8*b, game, 1), true
	default:
		return 0, true
	}
}

func raiseOrCall(bet int, game *leanpoker.Game, max float64) int {
	if game.CurrentBuyIn > bet {
		return game.CanCall(game.CurrentBuyIn+game.SmallBlind, max)
	}

	return bet
}
<commit_msg>Add logging<commit_after>package strategies

import (
	"log"

	"github.com/lean-poker/poker-player-go/leanpoker"
	"github.com/lean-poker/poker-player-go/rank_api"
)

const (
	BaseRank = 10
)

func Default(game *leanpoker.Game) int {
	if rank, ok := getRank(game); ok {
		log.Printf("Make bet = %v", rank)
		return rank
	}

	if checkBigBed(game) {
		log.Printf("Got big bet")
		return 0
	}

	log.Printf("No strategy. Calling.")
	return game.Call()
}

func getRank(game *leanpoker.Game) (int, bool) {
	cards := game.Cards()

	if len(cards) < 5 {
		log.Printf("Less 5 cards")
		return 0, false
	}

	rank := rank_api.GetRank(cards)

	b := game.SmallBlind

	switch rank.Rank {
	case 1:
		log.Printf("1 %v %v", game.Cards())
		return raiseOrCall(b, game, 0.2), true
	case 2:
		log.Printf("2 %v %v", game.Cards())
		return raiseOrCall(2*b, game, 0.2), true
	case 3:
		log.Printf("3 %v %v", game.Cards())
		return raiseOrCall(3*b, game, 0.3), true
	case 4:
		log.Printf("4 %v %v", game.Cards())
		return raiseOrCall(4*b, game, 0.5), true
	case 5:
		log.Printf("5 %v %v", game.Cards())
		return raiseOrCall(5*b, game, 0.5), true
	case 6:
		log.Printf("6 %v %v", game.Cards())
		return raiseOrCall(6*b, game, 1), true
	case 7:
		log.Printf("7 %v %v", game.Cards())
		return raiseOrCall(7*b, game, 1), true
	case 8:
		log.Printf("8 %v %v", game.Cards())
		return raiseOrCall(8*b, game, 1), true
	default:
		return 0, true
	}
}

func raiseOrCall(bet int, game *leanpoker.Game, max float64) int {
	if game.CurrentBuyIn > bet {
		log.Printf("raiseOrCall game.CurrentBuyIn > bet, %v, %v", bet, max)
		return game.CanCall(game.CurrentBuyIn+game.SmallBlind, max)
	}

	return bet
}
<|endoftext|>


<commit_before>// Copyright 2013, Michiel Buddingh, All rights reserved.
// Use of this code is governed by version 2.0 or later of the Apache
// License, available at http://www.apache.org/licenses/LICENSE-2.0

package spamsum

import (
	"math"
)

const (
	insCost    = 1
	delCost    = 1
	changeCost = 3
)

// Compare two SpamSums, returning a value between 0 and 100.
// This method is currently not bug-for-bug compatible with the
// original spamsum.
func (from SpamSum) Compare(to SpamSum) (similarity uint32) {
	q := float32(from.blocksize) / float32(to.blocksize)
	if q == 1 {
		similarity = uint32(max(
			score(from.leftPart[:from.leftIndex],
				to.leftPart[:to.leftIndex],
				int(from.blocksize)),
			score(from.rightPart[:from.rightIndex],
				to.rightPart[:to.rightIndex],
				int(to.blocksize))))

	} else if q == 2 {
		similarity = uint32(score(
			from.leftPart[:from.leftIndex],
			to.rightPart[:to.rightIndex],
			int(from.blocksize)))
	} else if q == 0.5 {
		similarity = uint32(score(
			from.rightPart[:from.rightIndex],
			to.leftPart[:to.leftIndex],
			int(to.blocksize)))
	} else {
		similarity = 0
	}
	return
}

func score(from, to []byte, blocksize int) (score int) {
	if !hasCommonSubstring(from, to) {
		return 0
	}

	from = eliminateRepetition(from)
	to = eliminateRepetition(to)

	score = editDistance(from, to)

	score *= SpamsumLength
	score /= len(from) + len(to)

	score = (score * 100) / 64

	if score > 100 {
		score = 0
	}
	score = 100 - score

	maxscore := blocksize / minBlockSize * min(len(from), len(to))
	score = min(score, maxscore)

	return score
}

func editDistance(from, to []byte) int {
	// memoize turns a recursive levenshtein function into one that uses an
	// array to cache results.  Uses |from| * |to| ints of memory.
	memoize := func(calculate func(a, b []byte) int) func(a, b []byte) int {
		var memo []int
		ffl, ttl := len(from), len(to)
		memo = make([]int, ffl*ttl)

		return func(from, to []byte) int {
			fl, tl := len(from), len(to)

			if fl == 0 {
				return tl
			}
			if tl == 0 {
				return fl
			}

			index := ((tl - 1) * ffl) + fl - 1
			if memo[index] == 0 {
				memo[index] = calculate(from, to)

			}
			return memo[index]
		}
	}

	var levenshteinRecursive func(from, to []byte) int

	// to see uncached results, just remove the memoize()
	levenshteinRecursive = memoize(func(from, to []byte) (distance int) {
		// This algorithm is not tuned for anything but legibility, complexity
		// is O(|from| * |to|).  The original code has the option of swapping
		// adjacent characters; as far as I can deduce, this is never used due
		// to the cost penalty, so it is omitted here.
		fl, tl := len(from), len(to)

		if fl == 0 {
			return tl
		}
		if tl == 0 {
			return fl
		}

		var cost = changeCost

		if from[fl-1] == to[tl-1] {
			cost = 0
		}

		return min(
			levenshteinRecursive(from[:fl-1], to)+delCost,
			levenshteinRecursive(from, to[:tl-1])+delCost,
			levenshteinRecursive(from[:fl-1], to[:tl-1])+cost)
	})

	return levenshteinRecursive(from, to)
}

// eliminateRepetition reduces sequences of repeating bytes
// longer than 3 bytes to length 3.
func eliminateRepetition(from []byte) (to []byte) {
	to = make([]byte, len(from))
	copy(to, from[:3])

	i, j := 3, 3
	for ; i < len(from); i++ {
		if from[i-3] != from[i] ||
			from[i-2] != from[i] ||
			from[i-1] != from[i] {
			to[j] = from[i]
			j++
		}
	}

	return to[:j]
}

// hasCommonSubstring returns true if the two byte slices
// passed have a common substring of at least seven bytes.
func hasCommonSubstring(seq1, seq2 []byte) (found bool) {
shift_offset:
	for shift := len(seq1) - 7; shift >= 7-len(seq2); shift-- {
		firstbound, secondbound := max(0, shift), max(0, -shift)
		common := 0
		for i, j := firstbound, secondbound; j < len(seq2) && i < len(seq1); i++ {
			if seq1[i] != seq2[j] {
				common = 0
			} else if common == 6 {
				found = true
				break shift_offset
			} else {
				common++
			}
			j++
		}
	}
	return
}

// min returns the minimum of its arguments
func min(args ...int) int {
	min := int(math.MaxInt32)
	for _, m := range args {
		if m < min {
			min = m
		}
	}
	return min
}

// max returns the maximum of its arguments
func max(args ...int) int {
	max := int(-math.MaxInt32)
	for _, m := range args {
		if m > max {
			max = m
		}
	}
	return max
}
<commit_msg>Remove provably redundant clause<commit_after>// Copyright 2013, Michiel Buddingh, All rights reserved.
// Use of this code is governed by version 2.0 or later of the Apache
// License, available at http://www.apache.org/licenses/LICENSE-2.0

package spamsum

import (
	"math"
)

const (
	insCost    = 1
	delCost    = 1
	changeCost = 3
)

// Compare two SpamSums, returning a value between 0 and 100.
// This method is currently not bug-for-bug compatible with the
// original spamsum.
func (from SpamSum) Compare(to SpamSum) (similarity uint32) {
	q := float32(from.blocksize) / float32(to.blocksize)
	if q == 1 {
		similarity = uint32(max(
			score(from.leftPart[:from.leftIndex],
				to.leftPart[:to.leftIndex],
				int(from.blocksize)),
			score(from.rightPart[:from.rightIndex],
				to.rightPart[:to.rightIndex],
				int(to.blocksize))))

	} else if q == 2 {
		similarity = uint32(score(
			from.leftPart[:from.leftIndex],
			to.rightPart[:to.rightIndex],
			int(from.blocksize)))
	} else if q == 0.5 {
		similarity = uint32(score(
			from.rightPart[:from.rightIndex],
			to.leftPart[:to.leftIndex],
			int(to.blocksize)))
	} else {
		similarity = 0
	}
	return
}

func score(from, to []byte, blocksize int) (score int) {
	if !hasCommonSubstring(from, to) {
		return 0
	}

	from = eliminateRepetition(from)
	to = eliminateRepetition(to)

	score = editDistance(from, to)

	score *= SpamsumLength
	score /= len(from) + len(to)

	score = (score * 100) / 64

	score = 100 - score

	maxscore := blocksize / minBlockSize * min(len(from), len(to))
	score = min(score, maxscore)

	return score
}

func editDistance(from, to []byte) int {
	// memoize turns a recursive levenshtein function into one that uses an
	// array to cache results.  Uses |from| * |to| ints of memory.
	memoize := func(calculate func(a, b []byte) int) func(a, b []byte) int {
		var memo []int
		ffl, ttl := len(from), len(to)
		memo = make([]int, ffl*ttl)

		return func(from, to []byte) int {
			fl, tl := len(from), len(to)

			if fl == 0 {
				return tl
			}
			if tl == 0 {
				return fl
			}

			index := ((tl - 1) * ffl) + fl - 1
			if memo[index] == 0 {
				memo[index] = calculate(from, to)

			}
			return memo[index]
		}
	}

	var levenshteinRecursive func(from, to []byte) int

	// to see uncached results, just remove the memoize()
	levenshteinRecursive = memoize(func(from, to []byte) (distance int) {
		// This algorithm is not tuned for anything but legibility, complexity
		// is O(|from| * |to|).  The original code has the option of swapping
		// adjacent characters; as far as I can deduce, this is never used due
		// to the cost penalty, so it is omitted here.
		fl, tl := len(from), len(to)

		if fl == 0 {
			return tl
		}
		if tl == 0 {
			return fl
		}

		var cost = changeCost

		if from[fl-1] == to[tl-1] {
			cost = 0
		}

		return min(
			levenshteinRecursive(from[:fl-1], to)+delCost,
			levenshteinRecursive(from, to[:tl-1])+delCost,
			levenshteinRecursive(from[:fl-1], to[:tl-1])+cost)
	})

	return levenshteinRecursive(from, to)
}

// eliminateRepetition reduces sequences of repeating bytes
// longer than 3 bytes to length 3.
func eliminateRepetition(from []byte) (to []byte) {
	to = make([]byte, len(from))
	copy(to, from[:3])

	i, j := 3, 3
	for ; i < len(from); i++ {
		if from[i-3] != from[i] ||
			from[i-2] != from[i] ||
			from[i-1] != from[i] {
			to[j] = from[i]
			j++
		}
	}

	return to[:j]
}

// hasCommonSubstring returns true if the two byte slices
// passed have a common substring of at least seven bytes.
func hasCommonSubstring(seq1, seq2 []byte) (found bool) {
shift_offset:
	for shift := len(seq1) - 7; shift >= 7-len(seq2); shift-- {
		firstbound, secondbound := max(0, shift), max(0, -shift)
		common := 0
		for i, j := firstbound, secondbound; j < len(seq2) && i < len(seq1); i++ {
			if seq1[i] != seq2[j] {
				common = 0
			} else if common == 6 {
				found = true
				break shift_offset
			} else {
				common++
			}
			j++
		}
	}
	return
}

// min returns the minimum of its arguments
func min(args ...int) int {
	min := int(math.MaxInt32)
	for _, m := range args {
		if m < min {
			min = m
		}
	}
	return min
}

// max returns the maximum of its arguments
func max(args ...int) int {
	max := int(-math.MaxInt32)
	for _, m := range args {
		if m > max {
			max = m
		}
	}
	return max
}
<|endoftext|>


<commit_before>// Copyright 2015 CloudAwan LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package etcd

import (
	"github.com/cloudawan/cloudone_utility/logger"
	"github.com/coreos/etcd/client"
	"time"
)

var log = logger.GetLog("utility")

type EtcdClient struct {
	KeysAPI                     client.KeysAPI
	EtcdEndpoints               []string
	EtcdHeaderTimeoutPerRequest time.Duration
}

func CreateEtcdClient(etcdEndpoints []string, etcdHeaderTimeoutPerRequest time.Duration) *EtcdClient {
	etcdClient := &EtcdClient{nil, etcdEndpoints, etcdHeaderTimeoutPerRequest}
	etcdClient.GetKeysAPI()
	return etcdClient
}

func (etcdClient *EtcdClient) GetKeysAPI() (returnedKeysAPI client.KeysAPI, returnedError error) {
	defer func() {
		if err := recover(); err != nil {
			log.Error("GetKeysAPI Error: %s", err)
			log.Error(logger.GetStackTrace(4096, false))
			etcdClient.KeysAPI = nil
			returnedKeysAPI = nil
			returnedError = err.(error)
		}
	}()

	if etcdClient.KeysAPI != nil {
		return etcdClient.KeysAPI, nil
	} else {
		config := client.Config{
			Endpoints:               etcdClient.EtcdEndpoints,
			Transport:               client.DefaultTransport,
			HeaderTimeoutPerRequest: etcdClient.EtcdHeaderTimeoutPerRequest,
		}
		configuredClient, err := client.New(config)
		if err != nil {
			etcdClient.KeysAPI = nil
			log.Error(err)
			return nil, err
		}
		keysAPI := client.NewKeysAPI(configuredClient)

		etcdClient.KeysAPI = keysAPI
		return keysAPI, nil
	}
}
<commit_msg>Add create etcd directory if not existing<commit_after>// Copyright 2015 CloudAwan LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package etcd

import (
	"github.com/cloudawan/cloudone_utility/logger"
	"github.com/coreos/etcd/client"
	"golang.org/x/net/context"
	"time"
)

var log = logger.GetLog("utility")

type EtcdClient struct {
	KeysAPI                     client.KeysAPI
	EtcdEndpoints               []string
	EtcdHeaderTimeoutPerRequest time.Duration
	EtcdBasePath                string
}

func CreateEtcdClient(etcdEndpoints []string, etcdHeaderTimeoutPerRequest time.Duration, etcdBasePath string) *EtcdClient {
	etcdClient := &EtcdClient{nil, etcdEndpoints, etcdHeaderTimeoutPerRequest, etcdBasePath}
	etcdClient.GetKeysAPI()
	return etcdClient
}

func (etcdClient *EtcdClient) GetKeysAPI() (returnedKeysAPI client.KeysAPI, returnedError error) {
	defer func() {
		if err := recover(); err != nil {
			log.Error("GetKeysAPI Error: %s", err)
			log.Error(logger.GetStackTrace(4096, false))
			etcdClient.KeysAPI = nil
			returnedKeysAPI = nil
			returnedError = err.(error)
		}
	}()

	if etcdClient.KeysAPI != nil {
		return etcdClient.KeysAPI, nil
	} else {
		config := client.Config{
			Endpoints:               etcdClient.EtcdEndpoints,
			Transport:               client.DefaultTransport,
			HeaderTimeoutPerRequest: etcdClient.EtcdHeaderTimeoutPerRequest,
		}
		configuredClient, err := client.New(config)
		if err != nil {
			etcdClient.KeysAPI = nil
			log.Error(err)
			return nil, err
		}
		keysAPI := client.NewKeysAPI(configuredClient)

		etcdClient.KeysAPI = keysAPI
		return keysAPI, nil
	}
}

func (etcdClient *EtcdClient) CreateDirectoryIfNotExist(key string) error {
	keysAPI, err := etcdClient.GetKeysAPI()
	if err != nil {
		log.Error(err)
		return err
	}

	_, err = keysAPI.Get(context.Background(), key, nil)
	errorData, ok := err.(client.Error)
	if ok == false {
		log.Error("Fail to convert error: %v", err)
		return err
	}
	// Not existing, create the directory
	if errorData.Code == client.ErrorCodeKeyNotFound {
		_, err = keysAPI.Set(context.Background(), key, "", &client.SetOptions{Dir: true})
		if err != nil {
			log.Error(err)
			return err
		}
	}
	return nil
}
<|endoftext|>