<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Usage:
//
// Writes to stdout by default:
//   camget BLOBREF
//
// Like curl, lets you set output file/directory with -o:
//   camget -o dir BLOBREF     (if dir exists and is directory, BLOBREF must be a directory, and -f to overwrite any files)
//   camget -o file  BLOBREF
//
// Should be possible to get a directory JSON blob without recursively
// fetching an entire directory.  Likewise with files.  But default
// should be sensitive on the type of the listed blob.  Maybe --blob
// just to get the blob?  Seems consistent.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"camli/blobref"
	"camli/client"
	"camli/index"
)

var (
	flagVerbose = flag.Bool("verbose", false, "be verbose")
	flagCheck   = flag.Bool("check", false, "just check for the existence of listed blobs; returning 0 if all our present")
	flagOutput  = flag.String("o", "-", "Output file/directory to create.  Use -f to overwrite.")
	flagVia     = flag.String("via", "", "Fetch the blob via the given comma-separated sharerefs (dev only).")
)

var viaRefs []*blobref.BlobRef

func main() {
	client.AddFlags()
	flag.Parse()

	if len(*flagVia) > 0 {
		vs := strings.Split(*flagVia, ",")
		viaRefs = make([]*blobref.BlobRef, len(vs))
		for i, sbr := range vs {
			viaRefs[i] = blobref.Parse(sbr)
			if viaRefs[i] == nil {
				log.Fatalf("Invalid -via blobref: %q", sbr)
			}
			if *flagVerbose {
				log.Printf("via: %s", sbr)
			}
		}
	}

	cl := client.NewOrFail()

	for n := 0; n < flag.NArg(); n++ {
		arg := flag.Arg(n)
		br := blobref.Parse(arg)
		if br == nil {
			log.Fatalf("Failed to parse argument %q as a blobref.", arg)
		}
		if *flagCheck {
			// TODO: do HEAD requests checking if the blobs exists.
			log.Fatal("not implemented")
			return
		}
		if *flagOutput == "-" {
			rc, err := fetch(cl, br)
			if err != nil {
				log.Fatal(err)
			}
			defer rc.Close()
			if _, err := io.Copy(os.Stdout, rc); err != nil {
				log.Fatalf("Failed reading %q: %v", br, err)
			}
			return
		}
		if err := smartFetch(cl, *flagOutput, br); err != nil {
			log.Fatal(err)
		}
	}
}

func fetch(cl *client.Client, br *blobref.BlobRef) (r io.ReadCloser, err os.Error) {
	if *flagVerbose {
		log.Printf("Fetching %s", br.String())
	}
	if len(viaRefs) > 0 {
		r, _, err = cl.FetchVia(br, viaRefs)
	} else {
		r, _, err = cl.FetchStreaming(br)
	}
	if err != nil {
		return nil, fmt.Errorf("Failed to fetch %q: %s", br, err)
	}
	return r, err
}

const sniffSize = 10 * 1024

// smartFetch the things that blobs point to, not just blobs. (wow)
func smartFetch(cl *client.Client, out string, br *blobref.BlobRef) os.Error {
	rc, err := fetch(cl, br)
	if err != nil {
		return err
	}
	defer rc.Close()

	sniffer := new(index.BlobSniffer)
	_, err = io.Copyn(sniffer, rc, sniffSize)
	if err != nil && err != os.EOF {
		return err
	}

	sniffer.Parse()
	_, ok := sniffer.Superset()

	if !ok {
		// opaque data - put it in a file
		f, err := os.Create(out)
		if err != nil {
			return err
		}
		defer f.Close()
		body, _ := sniffer.Body()
		r := io.MultiReader(bytes.NewBuffer(body), rc)
		_, err = io.Copy(f, r)
		return err
	}

	log.Fatal("camli blob parsing not implemented")

	return nil
}
<commit_msg>implement recursive fetch<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Usage:
//
// Writes to stdout by default:
//   camget BLOBREF
//
// Like curl, lets you set output file/directory with -o:
//   camget -o dir BLOBREF     (if dir exists and is directory, BLOBREF must be a directory, and -f to overwrite any files)
//   camget -o file  BLOBREF
//
// Should be possible to get a directory JSON blob without recursively
// fetching an entire directory.  Likewise with files.  But default
// should be sensitive on the type of the listed blob.  Maybe --blob
// just to get the blob?  Seems consistent.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"camli/blobref"
	"camli/client"
	"camli/index"
	"camli/schema"
)

var (
	flagVerbose = flag.Bool("verbose", false, "be verbose")
	flagCheck   = flag.Bool("check", false, "just check for the existence of listed blobs; returning 0 if all our present")
	flagOutput  = flag.String("o", "-", "Output file/directory to create.  Use -f to overwrite.")
	flagVia     = flag.String("via", "", "Fetch the blob via the given comma-separated sharerefs (dev only).")
)

var viaRefs []*blobref.BlobRef

func main() {
	client.AddFlags()
	flag.Parse()

	if len(*flagVia) > 0 {
		vs := strings.Split(*flagVia, ",")
		viaRefs = make([]*blobref.BlobRef, len(vs))
		for i, sbr := range vs {
			viaRefs[i] = blobref.Parse(sbr)
			if viaRefs[i] == nil {
				log.Fatalf("Invalid -via blobref: %q", sbr)
			}
			if *flagVerbose {
				log.Printf("via: %s", sbr)
			}
		}
	}

	cl := client.NewOrFail()

	for n := 0; n < flag.NArg(); n++ {
		arg := flag.Arg(n)
		br := blobref.Parse(arg)
		if br == nil {
			log.Fatalf("Failed to parse argument %q as a blobref.", arg)
		}
		if *flagCheck {
			// TODO: do HEAD requests checking if the blobs exists.
			log.Fatal("not implemented")
			return
		}
		if *flagOutput == "-" {
			rc, err := fetch(cl, br)
			if err != nil {
				log.Fatal(err)
			}
			defer rc.Close()
			if _, err := io.Copy(os.Stdout, rc); err != nil {
				log.Fatalf("Failed reading %q: %v", br, err)
			}
			return
		}
		if err := smartFetch(cl, *flagOutput, br); err != nil {
			log.Fatal(err)
		}
	}
}

func fetch(cl *client.Client, br *blobref.BlobRef) (r io.ReadCloser, err os.Error) {
	if *flagVerbose {
		log.Printf("Fetching %s", br.String())
	}
	if len(viaRefs) > 0 {
		r, _, err = cl.FetchVia(br, viaRefs)
	} else {
		r, _, err = cl.FetchStreaming(br)
	}
	if err != nil {
		return nil, fmt.Errorf("Failed to fetch %q: %s", br, err)
	}
	return r, err
}

// A little less than the sniffer will take, so we don't truncate.
const sniffSize = 900 * 1024

// smartFetch the things that blobs point to, not just blobs. (wow)
func smartFetch(cl *client.Client, targ string, br *blobref.BlobRef) os.Error {
	if *flagVerbose {
		log.Printf("Fetching %v into %q", br, targ)
	}

	rc, err := fetch(cl, br)
	if err != nil {
		return err
	}
	defer rc.Close()

	sniffer := new(index.BlobSniffer)
	_, err = io.Copyn(sniffer, rc, sniffSize)
	if err != nil && err != os.EOF {
		return err
	}

	sniffer.Parse()
	sc, ok := sniffer.Superset()

	if !ok {
		// opaque data - put it in a file
		f, err := os.Create(targ)
		if err != nil {
			return fmt.Errorf("opaque: %v", err)
		}
		defer f.Close()
		body, _ := sniffer.Body()
		r := io.MultiReader(bytes.NewBuffer(body), rc)
		_, err = io.Copy(f, r)
		return err
	}

	sc.BlobRef = br

	switch sc.Type {
	case "directory":
		dir := filepath.Join(targ, sc.FileName)
		if err := os.MkdirAll(dir, sc.UnixMode()); err != nil {
			return err
		}
		if err := setFileMeta(dir, sc); err != nil {
			log.Print(err)
		}
		entries := blobref.Parse(sc.Entries)
		if entries == nil {
			return fmt.Errorf("bad entries blobref: %v", sc.Entries)
		}
		return smartFetch(cl, dir, entries)
	case "static-set":
		// directory entries
		for _, m := range sc.Members {
			dref := blobref.Parse(m)
			if dref == nil {
				return fmt.Errorf("bad member blobref: %v", m)
			}
			if err := smartFetch(cl, targ, dref); err != nil {
				return err
			}
		}
		return nil
	case "file":
		name := filepath.Join(targ, sc.FileName)
		f, err := os.Create(name)
		if err != nil {
			return fmt.Errorf("file type: %v", err)
		}
		defer f.Close()
		for _, p := range sc.Parts {
			if p.BytesRef != nil {
				panic("don't know how to handle BytesRef")
			}
			rc, err := fetch(cl, p.BlobRef)
			if err != nil {
				return err
			}
			_, err = io.Copy(f, rc)
			rc.Close()
			if err != nil {
				return err
			}
		}
		if err := setFileMeta(name, sc); err != nil {
			log.Print(err)
		}
		return nil
	default:
		return os.NewError("unknown blob type: " + sc.Type)
	}
	panic("unreachable")
}

func setFileMeta(name string, sc *schema.Superset) os.Error {
	if err := os.Chmod(name, sc.UnixMode()); err != nil {
		return err
	}
	if err := os.Chown(name, sc.UnixOwnerId, sc.UnixGroupId); err != nil {
		return err
	}
	t, err := time.Parse(time.RFC3339, sc.UnixMtime)
	if err != nil {
		return nil
	}
	return os.Chtimes(name, 0, t.Nanoseconds())
}
<|endoftext|>


<commit_before>package sql

import (
	"database/sql"
	"errors"
	"fmt"
	"reflect"
	"strconv"
)

func NewPool(sourceStrings []string) (pool Conn, err error) {
	queue := make(chan *sql.DB, len(sourceStrings))
	var conn *sql.DB
	for _, sourceStr := range sourceStrings {
		conn, err = sql.Open("mysql", sourceStr)
		if err != nil {
			return
		}
		err = conn.Ping()
		if err != nil {
			return
		}

		queue <- conn
	}
	pool = &Pool{queue}
	return
}

type Pool struct {
	queue chan *sql.DB
}

func (p *Pool) Autocommit(acFun ConnFun) (err error) {
	conn := <-p.queue
	defer func() { p.queue <- conn }()

	acPool := &Pool{queue: make(chan *sql.DB, 1)}
	acPool.queue <- conn

	return acFun(acPool)
}

func (p *Pool) Transact(txFun ConnFun) (err error) {
	conn := <-p.queue
	defer func() { p.queue <- conn }()

	txPool := &Pool{queue: make(chan *sql.DB, 1)}
	txPool.queue <- conn

	_, err = conn.Exec("START TRANSACTION")
	if err != nil {
		return
	}

	err = txFun(txPool)
	if err != nil {
		return rollback(conn, err)
	}

	_, err = conn.Exec("COMMIT")
	if err != nil {
		return
	}

	return
}

func rollback(conn *sql.DB, err error) error {
	_, rbErr := conn.Exec("ROLLBACK")
	if rbErr != nil {
		err = errors.New("Rollback error: " + rbErr.Error() + " Query error: " + err.Error())
	}
	return err
}

// Query with fixed args
func (p *Pool) Query(query string, args ...interface{}) (rows *sql.Rows, err error) {
	conn := <-p.queue
	defer func() { p.queue <- conn }()

	FixArgs(args)
	rows, err = conn.Query(query, args...)
	if err != nil {
		err = errors.New("sql.Query Error: " + err.Error() + ". Query: " + query + " Args: " + fmt.Sprint(args))
	}
	return
}

// Execute with fixed args
func (p *Pool) Exec(query string, args ...interface{}) (res sql.Result, err error) {
	conn := <-p.queue
	defer func() { p.queue <- conn }()

	FixArgs(args)
	res, err = conn.Exec(query, args...)
	if err != nil {
		err = errors.New("sql.Exec Error: " + err.Error() + ". Query: " + query + " Args: " + fmt.Sprint(args))
	}
	return
}

/*
Fix args by converting them to values of their underlying kind.
This avoids problems in database/sql with e.g custom string types.
Without FixArgs, the following code:

	type Foo string
	...
	pool.Query("SELECT * WHERE Foo=?", Foo("bar"))

would give you the error:

	sql: converting Exec argument #1's type: unsupported type Foo, a string
*/
func FixArgs(args []interface{}) {
	for i, arg := range args {
		vArg := reflect.ValueOf(arg)
		if vArg.Kind() == reflect.String {
			args[i] = vArg.String()
		}
	}
}

func (p *Pool) SelectInt(query string, args ...interface{}) (num int, err error) {
	found, err := p.queryOne(query, args, &num)
	if !found {
		err = errors.New("Query returned no rows")
	}
	return
}

func (p *Pool) SelectString(query string, args ...interface{}) (str string, err error) {
	found, err := p.queryOne(query, args, &str)
	if !found {
		err = errors.New("Query returned no rows")
	}
	return
}

func (p *Pool) SelectUInt(query string, args ...interface{}) (num uint, err error) {
	found, err := p.queryOne(query, args, &num)
	if !found {
		err = errors.New("Query returned no rows")
	}
	return
}

func (p *Pool) SelectIntForce(query string, args ...interface{}) (num int, err error) {
	found, err := p.queryOne(query, args, &num)
	if !found {
		panic("Query returned no rows")
	}
	return
}

func (p *Pool) SelectStringForce(query string, args ...interface{}) (str string, err error) {
	found, err := p.queryOne(query, args, &str)
	if !found {
		panic("Query returned no rows")
	}
	return
}

func (p *Pool) SelectUIntForce(query string, args ...interface{}) (num uint, err error) {
	found, err := p.queryOne(query, args, &num)
	if !found {
		panic("Query returned no rows")
	}
	return
}

func (p *Pool) SelectIntMaybe(query string, args ...interface{}) (num int, found bool, err error) {
	found, err = p.queryOne(query, args, &num)
	return
}

func (p *Pool) SelectStringMaybe(query string, args ...interface{}) (str string, found bool, err error) {
	found, err = p.queryOne(query, args, &str)
	return
}

func (p *Pool) SelectUIntMaybe(query string, args ...interface{}) (num uint, found bool, err error) {
	found, err = p.queryOne(query, args, &num)
	return
}

func (p *Pool) queryOne(query string, args []interface{}, out interface{}) (found bool, err error) {
	rows, err := p.Query(query, args...)
	if err != nil {
		return
	}

	if rows.Next() {
		err = rows.Scan(out)
		if err != nil {
			return
		}
		if rows.Next() {
			err = errors.New("Query returned too many rows")
			return
		}
		found = true
	} else {
		found = false
	}

	return
}

// type TransactFunc func() error

// func (p *Pool) Transact(f TransactFunc) (err error) {
// 	conn := <-p.queue
// 	defer func() { p.queue <- conn }()

// 	_, err = conn.Exec("START TRANSACTION")
// 	if err != nil {
// 		return
// 	}

// 	err = f(conn)
// 	if err != nil {
// 		_, rollbackError := conn.Exec("ROLLBACK")
// 		if rollbackError != nil {
// 			panic("Could not rollback transaction. TransactionFunc error: " + err.Error() + ". Rollback error:" + rollbackError.Error())
// 		}

// 	} else {
// 		_, err = conn.Exec("COMMIT")
// 	}

// 	return
// }

func (p *Pool) UpdateOne(query string, args ...interface{}) (err error) {
	rowsAffected, err := p.Update(query, args...)
	if err != nil {
		return
	}
	if rowsAffected != 1 {
		return errors.New(fmt.Sprintf("UpdateOne affected %d rows. Query: %q Args: %q", rowsAffected, query, args))
	}
	return
}

func (p *Pool) Update(query string, args ...interface{}) (rowsAffected int64, err error) {
	res, err := p.Exec(query, args...)
	if err != nil {
		return
	}

	rowsAffected, err = res.RowsAffected()
	return
}

func (p *Pool) InsertIgnoreId(query string, args ...interface{}) (err error) {
	_, err = p.Insert(query, args...)
	return
}

func (p *Pool) Insert(query string, args ...interface{}) (id int64, err error) {
	res, err := p.Exec(query, args...)
	if err != nil {
		return
	}
	id, err = res.LastInsertId()
	return
}

func (p *Pool) Select(output interface{}, sql string, args ...interface{}) error {
	// Check types
	var outputPtr = reflect.ValueOf(output)
	if outputPtr.Kind() != reflect.Ptr {
		return errors.New("fun/sql.Select: expects a pointer to a slice of items")
	}
	var outputReflection = reflect.Indirect(outputPtr)
	if outputReflection.Kind() != reflect.Slice {
		return errors.New("fun/sql.Select: expects items to be a slice")
	}
	if outputReflection.Len() != 0 {
		return errors.New("fun/sql.Select: expects items to be empty")
	}
	outputItemType := outputReflection.Type().Elem().Elem()
	if outputItemType.Kind() != reflect.Struct {
		return errors.New("fun/sql.Select: expects items to be a slice of structs")
	}

	// Query DB
	var rows, err = p.Query(sql, args...)
	if err != nil {
		return err
	}

	// Reflect onto structs
	columns, err := rows.Columns()
	if err != nil {
		return err
	}

	structType := outputReflection.Type().Elem()
	for rows.Next() {
		structPtrVal := reflect.New(structType.Elem())
		outputItemStructVal := structPtrVal.Elem()
		err = structFromRow(outputItemStructVal, columns, rows)
		if err != nil {
			return err
		}

		outputReflection.Set(reflect.Append(outputReflection, structPtrVal))
	}
	return nil
}

const selectOneTypeError = "fun/sql.SelectOne: expects a **struct, e.g var person *Person; c.SelectOne(&person, sql)"

func (p *Pool) SelectOne(output interface{}, query string, args ...interface{}) error {
	return p.selectOne(output, query, true, args...)
}
func (p *Pool) SelectMaybe(output interface{}, query string, args ...interface{}) error {
	return p.selectOne(output, query, false, args...)
}
func (p *Pool) selectOne(output interface{}, query string, required bool, args ...interface{}) error {
	// Check types
	var outputReflectionPtr = reflect.ValueOf(output)
	if !outputReflectionPtr.IsValid() {
		panic(selectOneTypeError)
	}
	if outputReflectionPtr.Kind() != reflect.Ptr {
		panic(selectOneTypeError)
	}
	var outputReflection = outputReflectionPtr.Elem()
	if outputReflection.Kind() != reflect.Ptr {
		panic(selectOneTypeError)
	}

	// Query DB
	var rows, err = p.Query(query, args...)
	if err != nil {
		return err
	}

	// Reflect onto struct
	columns, err := rows.Columns()
	if err != nil {
		return err
	}
	if !rows.Next() {
		if required {
			return errors.New("SelectOne got 0 rows. Query: " + query)
		} else {
			return nil
		}
	}

	var vStruct reflect.Value
	if outputReflection.IsNil() {
		structPtrVal := reflect.New(outputReflection.Type().Elem())
		outputReflection.Set(structPtrVal)
		vStruct = structPtrVal.Elem()
	} else {
		vStruct = outputReflection.Elem()
	}

	err = structFromRow(vStruct, columns, rows)
	if err != nil {
		return err
	}

	if rows.Next() {
		return errors.New("fun/sql.SelectOne: got multiple rows. Query: " + query + " Args: " + fmt.Sprint(args))
	}

	return nil
}

func structFromRow(outputItemStructVal reflect.Value, columns []string, rows *sql.Rows) (err error) {
	vals := make([]interface{}, len(columns))
	for i, _ := range columns {
		vals[i] = &sql.RawBytes{}
		// vals[i] = &[]byte{}
	}
	err = rows.Scan(vals...)
	if err != nil {
		return
	}

	var uintVal uint64
	var intVal int64
	for i, column := range columns {
		bytes := []byte(*vals[i].(*sql.RawBytes))
		// bytes := []byte(*vals[i].(*[]byte))
		if bytes == nil {
			continue // Leave struct field empty
		}
		var outputItemField = outputItemStructVal.FieldByName(column)
		switch outputItemField.Kind() {
		case reflect.String:
			outputItemField.SetString(string(bytes))
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			uintVal, err = strconv.ParseUint(string(bytes), 10, 64)
			if err != nil {
				return
			}
			outputItemField.SetUint(reflect.ValueOf(uintVal).Uint())
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			intVal, err = strconv.ParseInt(string(bytes), 10, 64)
			if err != nil {
				return
			}
			outputItemField.SetInt(reflect.ValueOf(intVal).Int())
		default:
			err = errors.New("fun/sql: Bad row value for column " + column + ": " + outputItemField.Kind().String())
			return
		}
	}

	return
}
<commit_msg>Make empty string nil, to respect NOT NULL restrictions in schema. Not pretty, and does not do anything for non-string NOT NULL restrictions, but it is a work around for now...<commit_after>package sql

import (
	"database/sql"
	"errors"
	"fmt"
	"reflect"
	"strconv"
)

func NewPool(sourceStrings []string) (pool Conn, err error) {
	queue := make(chan *sql.DB, len(sourceStrings))
	var conn *sql.DB
	for _, sourceStr := range sourceStrings {
		conn, err = sql.Open("mysql", sourceStr)
		if err != nil {
			return
		}
		err = conn.Ping()
		if err != nil {
			return
		}

		queue <- conn
	}
	pool = &Pool{queue}
	return
}

type Pool struct {
	queue chan *sql.DB
}

func (p *Pool) Autocommit(acFun ConnFun) (err error) {
	conn := <-p.queue
	defer func() { p.queue <- conn }()

	acPool := &Pool{queue: make(chan *sql.DB, 1)}
	acPool.queue <- conn

	return acFun(acPool)
}

func (p *Pool) Transact(txFun ConnFun) (err error) {
	conn := <-p.queue
	defer func() { p.queue <- conn }()

	txPool := &Pool{queue: make(chan *sql.DB, 1)}
	txPool.queue <- conn

	_, err = conn.Exec("START TRANSACTION")
	if err != nil {
		return
	}

	err = txFun(txPool)
	if err != nil {
		return rollback(conn, err)
	}

	_, err = conn.Exec("COMMIT")
	if err != nil {
		return
	}

	return
}

func rollback(conn *sql.DB, err error) error {
	_, rbErr := conn.Exec("ROLLBACK")
	if rbErr != nil {
		err = errors.New("Rollback error: " + rbErr.Error() + " Query error: " + err.Error())
	}
	return err
}

// Query with fixed args
func (p *Pool) Query(query string, args ...interface{}) (rows *sql.Rows, err error) {
	conn := <-p.queue
	defer func() { p.queue <- conn }()

	FixArgs(args)
	rows, err = conn.Query(query, args...)
	if err != nil {
		err = errors.New("sql.Query Error: " + err.Error() + ". Query: " + query + " Args: " + fmt.Sprint(args))
	}
	return
}

// Execute with fixed args
func (p *Pool) Exec(query string, args ...interface{}) (res sql.Result, err error) {
	conn := <-p.queue
	defer func() { p.queue <- conn }()

	FixArgs(args)
	res, err = conn.Exec(query, args...)
	if err != nil {
		err = errors.New("sql.Exec Error: " + err.Error() + ". Query: " + query + " Args: " + fmt.Sprint(args))
	}
	return
}

/*
Fix args by converting them to values of their underlying kind.
This avoids problems in database/sql with e.g custom string types.
Without FixArgs, the following code:

	type Foo string
	...
	pool.Query("SELECT * WHERE Foo=?", Foo("bar"))

would give you the error:

	sql: converting Exec argument #1's type: unsupported type Foo, a string
*/
func FixArgs(args []interface{}) {
	for i, arg := range args {
		vArg := reflect.ValueOf(arg)
		switch vArg.Kind() {
		case reflect.String:
			args[i] = vArg.String()
			if args[i] == "" {
				args[i] = nil
			}
		}
	}
}

func (p *Pool) SelectInt(query string, args ...interface{}) (num int, err error) {
	found, err := p.queryOne(query, args, &num)
	if !found {
		err = errors.New("Query returned no rows")
	}
	return
}

func (p *Pool) SelectString(query string, args ...interface{}) (str string, err error) {
	found, err := p.queryOne(query, args, &str)
	if !found {
		err = errors.New("Query returned no rows")
	}
	return
}

func (p *Pool) SelectUInt(query string, args ...interface{}) (num uint, err error) {
	found, err := p.queryOne(query, args, &num)
	if !found {
		err = errors.New("Query returned no rows")
	}
	return
}

func (p *Pool) SelectIntForce(query string, args ...interface{}) (num int, err error) {
	found, err := p.queryOne(query, args, &num)
	if !found {
		panic("Query returned no rows")
	}
	return
}

func (p *Pool) SelectStringForce(query string, args ...interface{}) (str string, err error) {
	found, err := p.queryOne(query, args, &str)
	if !found {
		panic("Query returned no rows")
	}
	return
}

func (p *Pool) SelectUIntForce(query string, args ...interface{}) (num uint, err error) {
	found, err := p.queryOne(query, args, &num)
	if !found {
		panic("Query returned no rows")
	}
	return
}

func (p *Pool) SelectIntMaybe(query string, args ...interface{}) (num int, found bool, err error) {
	found, err = p.queryOne(query, args, &num)
	return
}

func (p *Pool) SelectStringMaybe(query string, args ...interface{}) (str string, found bool, err error) {
	found, err = p.queryOne(query, args, &str)
	return
}

func (p *Pool) SelectUIntMaybe(query string, args ...interface{}) (num uint, found bool, err error) {
	found, err = p.queryOne(query, args, &num)
	return
}

func (p *Pool) queryOne(query string, args []interface{}, out interface{}) (found bool, err error) {
	rows, err := p.Query(query, args...)
	if err != nil {
		return
	}

	if rows.Next() {
		err = rows.Scan(out)
		if err != nil {
			return
		}
		if rows.Next() {
			err = errors.New("Query returned too many rows")
			return
		}
		found = true
	} else {
		found = false
	}

	return
}

// type TransactFunc func() error

// func (p *Pool) Transact(f TransactFunc) (err error) {
// 	conn := <-p.queue
// 	defer func() { p.queue <- conn }()

// 	_, err = conn.Exec("START TRANSACTION")
// 	if err != nil {
// 		return
// 	}

// 	err = f(conn)
// 	if err != nil {
// 		_, rollbackError := conn.Exec("ROLLBACK")
// 		if rollbackError != nil {
// 			panic("Could not rollback transaction. TransactionFunc error: " + err.Error() + ". Rollback error:" + rollbackError.Error())
// 		}

// 	} else {
// 		_, err = conn.Exec("COMMIT")
// 	}

// 	return
// }

func (p *Pool) UpdateOne(query string, args ...interface{}) (err error) {
	rowsAffected, err := p.Update(query, args...)
	if err != nil {
		return
	}
	if rowsAffected != 1 {
		return errors.New(fmt.Sprintf("UpdateOne affected %d rows. Query: %q Args: %q", rowsAffected, query, args))
	}
	return
}

func (p *Pool) Update(query string, args ...interface{}) (rowsAffected int64, err error) {
	res, err := p.Exec(query, args...)
	if err != nil {
		return
	}

	rowsAffected, err = res.RowsAffected()
	return
}

func (p *Pool) InsertIgnoreId(query string, args ...interface{}) (err error) {
	_, err = p.Insert(query, args...)
	return
}

func (p *Pool) Insert(query string, args ...interface{}) (id int64, err error) {
	res, err := p.Exec(query, args...)
	if err != nil {
		return
	}
	id, err = res.LastInsertId()
	return
}

func (p *Pool) Select(output interface{}, sql string, args ...interface{}) error {
	// Check types
	var outputPtr = reflect.ValueOf(output)
	if outputPtr.Kind() != reflect.Ptr {
		return errors.New("fun/sql.Select: expects a pointer to a slice of items")
	}
	var outputReflection = reflect.Indirect(outputPtr)
	if outputReflection.Kind() != reflect.Slice {
		return errors.New("fun/sql.Select: expects items to be a slice")
	}
	if outputReflection.Len() != 0 {
		return errors.New("fun/sql.Select: expects items to be empty")
	}
	outputItemType := outputReflection.Type().Elem().Elem()
	if outputItemType.Kind() != reflect.Struct {
		return errors.New("fun/sql.Select: expects items to be a slice of structs")
	}

	// Query DB
	var rows, err = p.Query(sql, args...)
	if err != nil {
		return err
	}

	// Reflect onto structs
	columns, err := rows.Columns()
	if err != nil {
		return err
	}

	structType := outputReflection.Type().Elem()
	for rows.Next() {
		structPtrVal := reflect.New(structType.Elem())
		outputItemStructVal := structPtrVal.Elem()
		err = structFromRow(outputItemStructVal, columns, rows)
		if err != nil {
			return err
		}

		outputReflection.Set(reflect.Append(outputReflection, structPtrVal))
	}
	return nil
}

const selectOneTypeError = "fun/sql.SelectOne: expects a **struct, e.g var person *Person; c.SelectOne(&person, sql)"

func (p *Pool) SelectOne(output interface{}, query string, args ...interface{}) error {
	return p.selectOne(output, query, true, args...)
}
func (p *Pool) SelectMaybe(output interface{}, query string, args ...interface{}) error {
	return p.selectOne(output, query, false, args...)
}
func (p *Pool) selectOne(output interface{}, query string, required bool, args ...interface{}) error {
	// Check types
	var outputReflectionPtr = reflect.ValueOf(output)
	if !outputReflectionPtr.IsValid() {
		panic(selectOneTypeError)
	}
	if outputReflectionPtr.Kind() != reflect.Ptr {
		panic(selectOneTypeError)
	}
	var outputReflection = outputReflectionPtr.Elem()
	if outputReflection.Kind() != reflect.Ptr {
		panic(selectOneTypeError)
	}

	// Query DB
	var rows, err = p.Query(query, args...)
	if err != nil {
		return err
	}

	// Reflect onto struct
	columns, err := rows.Columns()
	if err != nil {
		return err
	}
	if !rows.Next() {
		if required {
			return errors.New("SelectOne got 0 rows. Query: " + query)
		} else {
			return nil
		}
	}

	var vStruct reflect.Value
	if outputReflection.IsNil() {
		structPtrVal := reflect.New(outputReflection.Type().Elem())
		outputReflection.Set(structPtrVal)
		vStruct = structPtrVal.Elem()
	} else {
		vStruct = outputReflection.Elem()
	}

	err = structFromRow(vStruct, columns, rows)
	if err != nil {
		return err
	}

	if rows.Next() {
		return errors.New("fun/sql.SelectOne: got multiple rows. Query: " + query + " Args: " + fmt.Sprint(args))
	}

	return nil
}

func structFromRow(outputItemStructVal reflect.Value, columns []string, rows *sql.Rows) (err error) {
	vals := make([]interface{}, len(columns))
	for i, _ := range columns {
		vals[i] = &sql.RawBytes{}
		// vals[i] = &[]byte{}
	}
	err = rows.Scan(vals...)
	if err != nil {
		return
	}

	var uintVal uint64
	var intVal int64
	for i, column := range columns {
		bytes := []byte(*vals[i].(*sql.RawBytes))
		// bytes := []byte(*vals[i].(*[]byte))
		if bytes == nil {
			continue // Leave struct field empty
		}
		var outputItemField = outputItemStructVal.FieldByName(column)
		switch outputItemField.Kind() {
		case reflect.String:
			outputItemField.SetString(string(bytes))
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			uintVal, err = strconv.ParseUint(string(bytes), 10, 64)
			if err != nil {
				return
			}
			outputItemField.SetUint(reflect.ValueOf(uintVal).Uint())
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			intVal, err = strconv.ParseInt(string(bytes), 10, 64)
			if err != nil {
				return
			}
			outputItemField.SetInt(reflect.ValueOf(intVal).Int())
		default:
			err = errors.New("fun/sql: Bad row value for column " + column + ": " + outputItemField.Kind().String())
			return
		}
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/abbot/go-http-auth"
	"github.com/andelf/go-curl"
	"github.com/nu7hatch/gouuid"
)

var config Config

type Config struct {
	HTTPBasicUsername string
	HTTPBasicPassword string
	HTTPBasicRealm    string
	Port              string
	Location          string
	ChecksURL         string
	MeasurerCount     int
}

type Check struct {
	ID  string `json:"id"`
	URL string `json:"url"`
}

type Measurement struct {
	Check             Check   `json:"check"`
	ID                string  `json:"id"`
	Location          string  `json:"location"`
	T                 int     `json:"t"`
	ExitStatus        int     `json:"exit_status"`
	ConnectTime       float64 `json:"connectTime,omitempty"`
	StartTransferTime float64 `json:"starttransferTime,omitempty"`
	LocalIP           string  `json:"localIP,omitempty"`
	PrimaryIP         string  `json:"primaryIP,omitempty"`
	TotalTime         float64 `json:"totalTime,omitempty"`
	HTTPStatus        int     `json:"httpStatus,omitempty"`
	NameLookupTime    float64 `json:"namelookupTime,omitempty"`
	SizeDownload      float64 `json:"sizeDownload,omitempty"`
}

func (c *Check) Measure(config Config) Measurement {
	var m Measurement

	id, _ := uuid.NewV4()
	m.ID = id.String()
	m.Check = *c
	m.Location = config.Location

	easy := curl.EasyInit()
	defer easy.Cleanup()

	easy.Setopt(curl.OPT_URL, c.URL)

	// dummy func for curl output
	noOut := func(buf []byte, userdata interface{}) bool {
		return true
	}

	easy.Setopt(curl.OPT_WRITEFUNCTION, noOut)
	easy.Setopt(curl.OPT_CONNECTTIMEOUT, 10)
	easy.Setopt(curl.OPT_TIMEOUT, 10)

	now := time.Now()
	m.T = int(now.Unix())

	if err := easy.Perform(); err != nil {
		if e, ok := err.(curl.CurlError); ok {
			m.ExitStatus = (int(e))
			return m
		}
		os.Exit(1)
	}

	m.ExitStatus = 0
	httpStatus, _ := easy.Getinfo(curl.INFO_RESPONSE_CODE)
	m.HTTPStatus = httpStatus.(int)

	connectTime, _ := easy.Getinfo(curl.INFO_CONNECT_TIME)
	m.ConnectTime = connectTime.(float64)

	namelookupTime, _ := easy.Getinfo(curl.INFO_NAMELOOKUP_TIME)
	m.NameLookupTime = namelookupTime.(float64)

	starttransferTime, _ := easy.Getinfo(curl.INFO_STARTTRANSFER_TIME)
	m.StartTransferTime = starttransferTime.(float64)

	totalTime, _ := easy.Getinfo(curl.INFO_TOTAL_TIME)
	m.TotalTime = totalTime.(float64)

	localIP, _ := easy.Getinfo(curl.INFO_LOCAL_IP)
	m.LocalIP = localIP.(string)

	primaryIP, _ := easy.Getinfo(curl.INFO_PRIMARY_IP)
	m.PrimaryIP = primaryIP.(string)

	sizeDownload, _ := easy.Getinfo(curl.INFO_SIZE_DOWNLOAD)
	m.SizeDownload = sizeDownload.(float64)

	return m
}

func measurer(config Config, toMeasurer chan Check, toStreamer chan Measurement) {
	for {
		c := <-toMeasurer
		m := c.Measure(config)

		toStreamer <- m
	}
}

func streamer(config Config, toStreamer chan Measurement) {
	a := func(user, realm string) string {
		if user == config.HTTPBasicUsername {
			return config.HTTPBasicPassword
		}
		return ""
	}

	h := func(w http.ResponseWriter, r *auth.AuthenticatedRequest) {
		w.Header().Set("Content-Type", "application/json")
		enc := json.NewEncoder(w)
		for {
			err := enc.Encode(<-toStreamer)
			if err != nil {
				return
			}

			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			}
		}
	}

	authenticator := auth.NewBasicAuthenticator(config.HTTPBasicRealm, a)

	http.HandleFunc("/measurements", authenticator.Wrap(h))

	log.Printf("fn=streamer listening=true port=%s\n", config.Port)
	err := http.ListenAndServe(":"+config.Port, nil)
	if err != nil {
		log.Fatal(err)
	}
}

func getChecks(config Config) []Check {
	url := config.ChecksURL

	res, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}

	var checks []Check
	err = json.Unmarshal(body, &checks)
	if err != nil {
		log.Fatal(err)
	}

	return checks
}

func scheduler(check Check, toMeasurer chan Check) {
	for {
		toMeasurer <- check
		time.Sleep(1000 * time.Millisecond)
	}
}

func init() {
	flag.StringVar(&config.HTTPBasicUsername, "http_basic_username", "", "HTTP basic authentication username")
	flag.StringVar(&config.HTTPBasicPassword, "http_basic_password", "", "HTTP basic authentication password")
	flag.StringVar(&config.HTTPBasicRealm, "http_basic_realm", "", "HTTP basic authentication realm")
	flag.StringVar(&config.Port, "port", "5000", "port the HTTP server should listen on")
	flag.StringVar(&config.Location, "location", "undefined", "location of this sensor")
	flag.StringVar(&config.ChecksURL, "checks_url", "https://s3.amazonaws.com/canary-public-data/checks.json", "URL for check data")
	flag.IntVar(&config.MeasurerCount, "measurer_count", 1, "number of measurers to run")
}

func main() {
	flag.Parse()

	if len(config.HTTPBasicUsername) == 0 && len(config.HTTPBasicPassword) == 0 {
		log.Fatal("fatal - HTTP basic auth not set correctly")
	}

	checkList := getChecks(config)

	toMeasurer := make(chan Check)
	toStreamer := make(chan Measurement)

	// spawn one scheduler per check
	for _, c := range checkList {
		go scheduler(c, toMeasurer)
	}

	// spawn N measurers
	for i := 0; i < config.MeasurerCount; i++ {
		go measurer(config, toMeasurer, toStreamer)
	}

	// stream measurements to clients over HTTP
	go streamer(config, toStreamer)

	select {}
}
<commit_msg>fix json fields<commit_after>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/abbot/go-http-auth"
	"github.com/andelf/go-curl"
	"github.com/nu7hatch/gouuid"
)

var config Config

type Config struct {
	HTTPBasicUsername string
	HTTPBasicPassword string
	HTTPBasicRealm    string
	Port              string
	Location          string
	ChecksURL         string
	MeasurerCount     int
}

type Check struct {
	ID  string `json:"id"`
	URL string `json:"url"`
}

type Measurement struct {
	Check             Check   `json:"check"`
	ID                string  `json:"id"`
	Location          string  `json:"location"`
	T                 int     `json:"t"`
	ExitStatus        int     `json:"exit_status"`
	ConnectTime       float64 `json:"connect_time,omitempty"`
	StartTransferTime float64 `json:"starttransfer_time,omitempty"`
	LocalIP           string  `json:"local_ip,omitempty"`
	PrimaryIP         string  `json:"primary_ip,omitempty"`
	TotalTime         float64 `json:"total_time,omitempty"`
	HTTPStatus        int     `json:"http_status,omitempty"`
	NameLookupTime    float64 `json:"namelookup_time,omitempty"`
	SizeDownload      float64 `json:"size_download,omitempty"`
}

func (c *Check) Measure(config Config) Measurement {
	var m Measurement

	id, _ := uuid.NewV4()
	m.ID = id.String()
	m.Check = *c
	m.Location = config.Location

	easy := curl.EasyInit()
	defer easy.Cleanup()

	easy.Setopt(curl.OPT_URL, c.URL)

	// dummy func for curl output
	noOut := func(buf []byte, userdata interface{}) bool {
		return true
	}

	easy.Setopt(curl.OPT_WRITEFUNCTION, noOut)
	easy.Setopt(curl.OPT_CONNECTTIMEOUT, 10)
	easy.Setopt(curl.OPT_TIMEOUT, 10)

	now := time.Now()
	m.T = int(now.Unix())

	if err := easy.Perform(); err != nil {
		if e, ok := err.(curl.CurlError); ok {
			m.ExitStatus = (int(e))
			return m
		}
		os.Exit(1)
	}

	m.ExitStatus = 0
	httpStatus, _ := easy.Getinfo(curl.INFO_RESPONSE_CODE)
	m.HTTPStatus = httpStatus.(int)

	connectTime, _ := easy.Getinfo(curl.INFO_CONNECT_TIME)
	m.ConnectTime = connectTime.(float64)

	namelookupTime, _ := easy.Getinfo(curl.INFO_NAMELOOKUP_TIME)
	m.NameLookupTime = namelookupTime.(float64)

	starttransferTime, _ := easy.Getinfo(curl.INFO_STARTTRANSFER_TIME)
	m.StartTransferTime = starttransferTime.(float64)

	totalTime, _ := easy.Getinfo(curl.INFO_TOTAL_TIME)
	m.TotalTime = totalTime.(float64)

	localIP, _ := easy.Getinfo(curl.INFO_LOCAL_IP)
	m.LocalIP = localIP.(string)

	primaryIP, _ := easy.Getinfo(curl.INFO_PRIMARY_IP)
	m.PrimaryIP = primaryIP.(string)

	sizeDownload, _ := easy.Getinfo(curl.INFO_SIZE_DOWNLOAD)
	m.SizeDownload = sizeDownload.(float64)

	return m
}

func measurer(config Config, toMeasurer chan Check, toStreamer chan Measurement) {
	for {
		c := <-toMeasurer
		m := c.Measure(config)

		toStreamer <- m
	}
}

func streamer(config Config, toStreamer chan Measurement) {
	a := func(user, realm string) string {
		if user == config.HTTPBasicUsername {
			return config.HTTPBasicPassword
		}
		return ""
	}

	h := func(w http.ResponseWriter, r *auth.AuthenticatedRequest) {
		w.Header().Set("Content-Type", "application/json")
		enc := json.NewEncoder(w)
		for {
			err := enc.Encode(<-toStreamer)
			if err != nil {
				return
			}

			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			}
		}
	}

	authenticator := auth.NewBasicAuthenticator(config.HTTPBasicRealm, a)

	http.HandleFunc("/measurements", authenticator.Wrap(h))

	log.Printf("fn=streamer listening=true port=%s\n", config.Port)
	err := http.ListenAndServe(":"+config.Port, nil)
	if err != nil {
		log.Fatal(err)
	}
}

func getChecks(config Config) []Check {
	url := config.ChecksURL

	res, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}

	var checks []Check
	err = json.Unmarshal(body, &checks)
	if err != nil {
		log.Fatal(err)
	}

	return checks
}

func scheduler(check Check, toMeasurer chan Check) {
	for {
		toMeasurer <- check
		time.Sleep(1000 * time.Millisecond)
	}
}

func init() {
	flag.StringVar(&config.HTTPBasicUsername, "http_basic_username", "", "HTTP basic authentication username")
	flag.StringVar(&config.HTTPBasicPassword, "http_basic_password", "", "HTTP basic authentication password")
	flag.StringVar(&config.HTTPBasicRealm, "http_basic_realm", "", "HTTP basic authentication realm")
	flag.StringVar(&config.Port, "port", "5000", "port the HTTP server should listen on")
	flag.StringVar(&config.Location, "location", "undefined", "location of this sensor")
	flag.StringVar(&config.ChecksURL, "checks_url", "https://s3.amazonaws.com/canary-public-data/checks.json", "URL for check data")
	flag.IntVar(&config.MeasurerCount, "measurer_count", 1, "number of measurers to run")
}

func main() {
	flag.Parse()

	if len(config.HTTPBasicUsername) == 0 && len(config.HTTPBasicPassword) == 0 {
		log.Fatal("fatal - HTTP basic auth not set correctly")
	}

	checkList := getChecks(config)

	toMeasurer := make(chan Check)
	toStreamer := make(chan Measurement)

	// spawn one scheduler per check
	for _, c := range checkList {
		go scheduler(c, toMeasurer)
	}

	// spawn N measurers
	for i := 0; i < config.MeasurerCount; i++ {
		go measurer(config, toMeasurer, toStreamer)
	}

	// stream measurements to clients over HTTP
	go streamer(config, toStreamer)

	select {}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package rewrite contains commands for writing the altered import statements.
package rewrite

import (
	"errors"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"sort"
	"time"

	"github.com/kardianos/vendor/vendorfile"
)

// ListStatus indicates the status of the import.
type ListStatus byte

func (ls ListStatus) String() string {
	switch ls {
	case StatusUnknown:
		return "?"
	case StatusMissing:
		return "m"
	case StatusStd:
		return "s"
	case StatusLocal:
		return "l"
	case StatusExternal:
		return "e"
	case StatusInternal:
		return "i"
	case StatusUnused:
		return "u"
	}
	return ""
}

const (
	// StatusUnknown indicates the status was unable to be obtained.
	StatusUnknown ListStatus = iota
	// StatusMissing indicates import not found in GOROOT or GOPATH.
	StatusMissing
	// StatusStd indicates import found in GOROOT.
	StatusStd
	// StatusLocal indicates import is part of the local project.
	StatusLocal
	// StatusExternal indicates import is found in GOPATH and not copied.
	StatusExternal
	// StatusInternal indicates import has been copied locally under internal.
	StatusInternal
	// StatusUnused indicates import has been copied, but is no longer used.
	StatusUnused
)

// ListItem represents a package in the current project.
type ListItem struct {
	Status     ListStatus
	Path       string
	VendorPath string
}

func (li ListItem) String() string {
	return li.Status.String() + " " + li.Path
}

type listItemSort []ListItem

func (li listItemSort) Len() int      { return len(li) }
func (li listItemSort) Swap(i, j int) { li[i], li[j] = li[j], li[i] }
func (li listItemSort) Less(i, j int) bool {
	if li[i].Status == li[j].Status {
		return li[i].Path < li[j].Path
	}
	return li[i].Status > li[j].Status
}

const (
	vendorFilename = "vendor.json"
	internalFolder = "internal"

	looplimit = 10000
)

var (
	internalVendor = filepath.Join(internalFolder, vendorFilename)
)

var (
	ErrVendorFileExists  = errors.New(internalVendor + " file already exists.")
	ErrMissingVendorFile = errors.New("Unable to find internal folder with vendor file.")
	ErrMissingGOROOT     = errors.New("Unable to determine GOROOT.")
	ErrMissingGOPATH     = errors.New("Missing GOPATH.")
	ErrVendorExists      = errors.New("Package already exists as a vendor package.")
	ErrLocalPackage      = errors.New("Cannot vendor a local package.")
	ErrImportExists      = errors.New("Import exists. To update use update command.")
	ErrImportNotExists   = errors.New("Import does not exist.")
	ErrNoLocalPath       = errors.New("Import is present in vendor file, but is missing local path.")
	ErrFilesExists       = errors.New("Files exists at destination of internal vendor path.")
)

type ErrNotInGOPATH struct {
	Missing string
}

func (err ErrNotInGOPATH) Error() string {
	return fmt.Sprintf("Package %q not a go package or not in GOPATH.", err.Missing)
}

type ErrDirtyPackage struct {
	ImportPath string
}

func (err ErrDirtyPackage) Error() string {
	return fmt.Sprintf("Package %q has uncommited changes in the vcs.", err.ImportPath)
}

type ErrLoopLimit struct {
	Loop string
}

func (err ErrLoopLimit) Error() string {
	return fmt.Sprintf("BUG: Loop limit of %d was reached for loop %s. Please report this bug.", looplimit, err.Loop)
}

func CmdInit() error {
	/*
		1. Determine if CWD contains "internal/vendor.json".
		2. If exists, return error.
		3. Create directory if it doesn't exist.
		4. Create "internal/vendor.json" file.
	*/
	wd, err := os.Getwd()
	if err != nil {
		return err
	}
	_, err = os.Stat(filepath.Join(wd, internalVendor))
	if os.IsNotExist(err) == false {
		return ErrVendorFileExists
	}
	err = os.MkdirAll(filepath.Join(wd, internalFolder), 0777)
	if err != nil {
		return err
	}
	vf := &vendorfile.File{}
	return writeVendorFile(wd, vf)
}

func CmdList() ([]ListItem, error) {
	/*
		1. Find vendor root.
		2. Find vendor root import path via GOPATH.
		3. Walk directory, find all directories with go files.
		4. Parse imports for all go files.
		5. Determine the status of all imports.
		  * Std
		  * Local
		  * External Vendor
		  * Internal Vendor
		  * Unused Vendor
		6. Return Vendor import paths.
	*/
	ctx, err := NewContextWD()
	if err != nil {
		return nil, err
	}

	err = ctx.LoadPackage()
	if err != nil {
		return nil, err
	}

	list := make([]ListItem, 0, len(ctx.Package))
	for _, pkg := range ctx.Package {
		li := ListItem{
			Status:     pkg.Status,
			Path:       pkg.ImportPath,
			VendorPath: pkg.VendorPath,
		}
		if vp, found := ctx.vendorFileLocal[pkg.ImportPath]; found {
			li.VendorPath = vp.Canonical
		}
		list = append(list, li)
	}
	// Sort li by Status, then Path.
	sort.Sort(listItemSort(list))

	return list, nil
}

/*
	Add, Update, and Remove will start with the same steps as List.
	Rather then returning the results, it will find any affected files,
	alter their imports, then write the files back out. Also copy or remove
	files and folders as needed.
*/

func CmdAdd(importPath string) error {
	return addUpdateImportPath(importPath, verifyAdd)
}

func CmdUpdate(importPath string) error {
	return addUpdateImportPath(importPath, verifyUpdate)
}

func verifyAdd(ctx *Context, importPath, local string) (string, error) {
	for _, pkg := range ctx.VendorFile.Package {
		if pkg.Canonical == importPath {
			return importPath, ErrImportExists
		}
	}
	// Check fo existing internal folders present.
	dirPath := filepath.Join(ctx.RootGopath, local)
	dir, err := os.Open(dirPath)
	if err != nil {
		if os.IsNotExist(err) {
			// No folder present, no need to check for files.
			return importPath, nil
		}
		return importPath, err
	}
	fl, err := dir.Readdir(-1)
	dir.Close()
	if err != nil {
		return importPath, err
	}
	for _, fi := range fl {
		if fi.IsDir() == false {
			return importPath, ErrFilesExists
		}
	}
	return importPath, nil
}
func verifyUpdate(ctx *Context, importPath, local string) (string, error) {
	for _, pkg := range ctx.VendorFile.Package {
		if pkg.Canonical == importPath {
			return importPath, nil
		}
	}
	for _, pkg := range ctx.VendorFile.Package {
		if pkg.Local == importPath {
			return pkg.Canonical, nil
		}
	}
	return importPath, ErrImportNotExists
}

func addUpdateImportPath(importPath string, verify func(ctx *Context, importPath, local string) (string, error)) error {
	importPath = slashToImportPath(importPath)
	ctx, err := NewContextWD()
	if err != nil {
		return err
	}

	err = ctx.LoadPackage(importPath)
	if err != nil {
		return err
	}

	localImportPath, err := findLocalImportPath(ctx, importPath)
	if err != nil {
		return err
	}
	// Adjust relative local path to GOPATH import path.
	localImportPath = path.Join(ctx.RootImportPath, internalFolder, localImportPath)

	importPath, err = verify(ctx, importPath, localImportPath)
	if err != nil {
		return err
	}

	err = ctx.AddImports(importPath)
	if err != nil {
		return err
	}

	pkg, foundPkg := ctx.Package[importPath]
	if !foundPkg {
		return ErrNotInGOPATH{importPath}
	}
	if pkg.Status != StatusExternal {
		if pkg.Status == StatusInternal {
			return ErrVendorExists
		}
		if pkg.Status == StatusLocal {
			return ErrLocalPackage
		}
		return ErrNotInGOPATH{importPath}
	}

	// Update vendor file with correct Local field.
	var vp *vendorfile.Package
	for _, vpkg := range ctx.VendorFile.Package {
		if vpkg.Canonical == importPath {
			vp = vpkg
			break
		}
	}
	if vp == nil {
		vp = &vendorfile.Package{
			Add:       true,
			Canonical: importPath,
			Local:     localImportPath,
		}
		ctx.VendorFile.Package = append(ctx.VendorFile.Package, vp)
		ctx.vendorFileLocal[vp.Local] = vp
	}
	// Find the VCS information.
	vcs, err := FindVcs(pkg.Gopath, pkg.Dir)
	if err != nil {
		return err
	}
	if vcs.Dirty {
		return ErrDirtyPackage{pkg.ImportPath}
	}
	vp.Revision = vcs.Revision
	if vcs.RevisionTime != nil {
		vp.RevisionTime = vcs.RevisionTime.Format(time.RFC3339)
	}

	// Write the vendor file.
	err = writeVendorFile(ctx.RootDir, ctx.VendorFile)
	if err != nil {
		return err
	}

	// Copy the package locally.
	err = CopyPackage(filepath.Join(ctx.RootGopath, slashToFilepath(localImportPath)), pkg.Dir)
	if err != nil {
		return err
	}

	err = ctx.AddImports(importPath, localImportPath)
	if err != nil {
		return err
	}

	// Determine which files to touch.
	files := make(map[string]*File, len(ctx.VendorFile.Package)*3)

	// Rules are all lines in the vendor file.
	rules := make([]Rule, 0, len(ctx.VendorFile.Package))
	for _, vp := range ctx.VendorFile.Package {
		for fpath, f := range ctx.fileImports[vp.Canonical] {
			files[fpath] = f
		}
		rules = append(rules, Rule{From: vp.Canonical, To: vp.Local})
	}
	// Add local package files.
	if localPkg, found := ctx.Package[localImportPath]; found {
		for _, f := range localPkg.Files {
			files[f.Path] = f
		}
	}
	// Rewrite any external package where the local path is different then the vendor path.
	for _, pkg := range ctx.Package {
		if pkg.Status != StatusExternal {
			continue
		}
		if pkg.ImportPath == pkg.VendorPath {
			continue
		}
		for _, otherPkg := range ctx.Package {
			if pkg == otherPkg {
				continue
			}
			if otherPkg.Status != StatusInternal {
				continue
			}
			if otherPkg.VendorPath != pkg.VendorPath {
				continue
			}

			for fpath, f := range ctx.fileImports[pkg.ImportPath] {
				files[fpath] = f
			}
			rules = append(rules, Rule{From: pkg.ImportPath, To: otherPkg.ImportPath})
			break
		}
	}

	return ctx.RewriteFiles(files, rules)
}
func CmdRemove(importPath string) error {
	importPath = slashToImportPath(importPath)
	ctx, err := NewContextWD()
	if err != nil {
		return err
	}

	err = ctx.LoadPackage(importPath)
	if err != nil {
		return err
	}

	localPath := ""
	vendorPath := importPath
	localFound := false
	vendorFileIndex := 0
	for i, pkg := range ctx.VendorFile.Package {
		if pkg.Canonical == importPath {
			localPath = pkg.Local
			localFound = true
			vendorFileIndex = i
			break
		}
	}
	if !localFound {
		for i, pkg := range ctx.VendorFile.Package {
			if pkg.Local == importPath {
				localPath = pkg.Local
				vendorPath = pkg.Canonical
				localFound = true
				vendorFileIndex = i
				break
			}
		}
	}
	if !localFound {
		return ErrImportNotExists
	}
	if localPath == "" {
		return ErrNoLocalPath
	}

	files := ctx.fileImports[localPath]

	err = ctx.RewriteFiles(files, []Rule{{From: localPath, To: vendorPath}})
	if err != nil {
		return err
	}

	err = RemovePackage(filepath.Join(ctx.RootGopath, slashToFilepath(localPath)))
	if err != nil {
		return err
	}
	for i, pkg := range ctx.VendorFile.Package {
		if i == vendorFileIndex {
			pkg.Remove = true
		}
	}

	return writeVendorFile(ctx.RootDir, ctx.VendorFile)
}
<commit_msg>rewrite: continue to work even when no vcs is found.<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package rewrite contains commands for writing the altered import statements.
package rewrite

import (
	"errors"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"sort"
	"time"

	"github.com/kardianos/vendor/vendorfile"
)

// ListStatus indicates the status of the import.
type ListStatus byte

func (ls ListStatus) String() string {
	switch ls {
	case StatusUnknown:
		return "?"
	case StatusMissing:
		return "m"
	case StatusStd:
		return "s"
	case StatusLocal:
		return "l"
	case StatusExternal:
		return "e"
	case StatusInternal:
		return "i"
	case StatusUnused:
		return "u"
	}
	return ""
}

const (
	// StatusUnknown indicates the status was unable to be obtained.
	StatusUnknown ListStatus = iota
	// StatusMissing indicates import not found in GOROOT or GOPATH.
	StatusMissing
	// StatusStd indicates import found in GOROOT.
	StatusStd
	// StatusLocal indicates import is part of the local project.
	StatusLocal
	// StatusExternal indicates import is found in GOPATH and not copied.
	StatusExternal
	// StatusInternal indicates import has been copied locally under internal.
	StatusInternal
	// StatusUnused indicates import has been copied, but is no longer used.
	StatusUnused
)

// ListItem represents a package in the current project.
type ListItem struct {
	Status     ListStatus
	Path       string
	VendorPath string
}

func (li ListItem) String() string {
	return li.Status.String() + " " + li.Path
}

type listItemSort []ListItem

func (li listItemSort) Len() int      { return len(li) }
func (li listItemSort) Swap(i, j int) { li[i], li[j] = li[j], li[i] }
func (li listItemSort) Less(i, j int) bool {
	if li[i].Status == li[j].Status {
		return li[i].Path < li[j].Path
	}
	return li[i].Status > li[j].Status
}

const (
	vendorFilename = "vendor.json"
	internalFolder = "internal"

	looplimit = 10000
)

var (
	internalVendor = filepath.Join(internalFolder, vendorFilename)
)

var (
	ErrVendorFileExists  = errors.New(internalVendor + " file already exists.")
	ErrMissingVendorFile = errors.New("Unable to find internal folder with vendor file.")
	ErrMissingGOROOT     = errors.New("Unable to determine GOROOT.")
	ErrMissingGOPATH     = errors.New("Missing GOPATH.")
	ErrVendorExists      = errors.New("Package already exists as a vendor package.")
	ErrLocalPackage      = errors.New("Cannot vendor a local package.")
	ErrImportExists      = errors.New("Import exists. To update use update command.")
	ErrImportNotExists   = errors.New("Import does not exist.")
	ErrNoLocalPath       = errors.New("Import is present in vendor file, but is missing local path.")
	ErrFilesExists       = errors.New("Files exists at destination of internal vendor path.")
)

type ErrNotInGOPATH struct {
	Missing string
}

func (err ErrNotInGOPATH) Error() string {
	return fmt.Sprintf("Package %q not a go package or not in GOPATH.", err.Missing)
}

type ErrDirtyPackage struct {
	ImportPath string
}

func (err ErrDirtyPackage) Error() string {
	return fmt.Sprintf("Package %q has uncommited changes in the vcs.", err.ImportPath)
}

type ErrLoopLimit struct {
	Loop string
}

func (err ErrLoopLimit) Error() string {
	return fmt.Sprintf("BUG: Loop limit of %d was reached for loop %s. Please report this bug.", looplimit, err.Loop)
}

func CmdInit() error {
	/*
		1. Determine if CWD contains "internal/vendor.json".
		2. If exists, return error.
		3. Create directory if it doesn't exist.
		4. Create "internal/vendor.json" file.
	*/
	wd, err := os.Getwd()
	if err != nil {
		return err
	}
	_, err = os.Stat(filepath.Join(wd, internalVendor))
	if os.IsNotExist(err) == false {
		return ErrVendorFileExists
	}
	err = os.MkdirAll(filepath.Join(wd, internalFolder), 0777)
	if err != nil {
		return err
	}
	vf := &vendorfile.File{}
	return writeVendorFile(wd, vf)
}

func CmdList() ([]ListItem, error) {
	/*
		1. Find vendor root.
		2. Find vendor root import path via GOPATH.
		3. Walk directory, find all directories with go files.
		4. Parse imports for all go files.
		5. Determine the status of all imports.
		  * Std
		  * Local
		  * External Vendor
		  * Internal Vendor
		  * Unused Vendor
		6. Return Vendor import paths.
	*/
	ctx, err := NewContextWD()
	if err != nil {
		return nil, err
	}

	err = ctx.LoadPackage()
	if err != nil {
		return nil, err
	}

	list := make([]ListItem, 0, len(ctx.Package))
	for _, pkg := range ctx.Package {
		li := ListItem{
			Status:     pkg.Status,
			Path:       pkg.ImportPath,
			VendorPath: pkg.VendorPath,
		}
		if vp, found := ctx.vendorFileLocal[pkg.ImportPath]; found {
			li.VendorPath = vp.Canonical
		}
		list = append(list, li)
	}
	// Sort li by Status, then Path.
	sort.Sort(listItemSort(list))

	return list, nil
}

/*
	Add, Update, and Remove will start with the same steps as List.
	Rather then returning the results, it will find any affected files,
	alter their imports, then write the files back out. Also copy or remove
	files and folders as needed.
*/

func CmdAdd(importPath string) error {
	return addUpdateImportPath(importPath, verifyAdd)
}

func CmdUpdate(importPath string) error {
	return addUpdateImportPath(importPath, verifyUpdate)
}

func verifyAdd(ctx *Context, importPath, local string) (string, error) {
	for _, pkg := range ctx.VendorFile.Package {
		if pkg.Canonical == importPath {
			return importPath, ErrImportExists
		}
	}
	// Check fo existing internal folders present.
	dirPath := filepath.Join(ctx.RootGopath, local)
	dir, err := os.Open(dirPath)
	if err != nil {
		if os.IsNotExist(err) {
			// No folder present, no need to check for files.
			return importPath, nil
		}
		return importPath, err
	}
	fl, err := dir.Readdir(-1)
	dir.Close()
	if err != nil {
		return importPath, err
	}
	for _, fi := range fl {
		if fi.IsDir() == false {
			return importPath, ErrFilesExists
		}
	}
	return importPath, nil
}
func verifyUpdate(ctx *Context, importPath, local string) (string, error) {
	for _, pkg := range ctx.VendorFile.Package {
		if pkg.Canonical == importPath {
			return importPath, nil
		}
	}
	for _, pkg := range ctx.VendorFile.Package {
		if pkg.Local == importPath {
			return pkg.Canonical, nil
		}
	}
	return importPath, ErrImportNotExists
}

func addUpdateImportPath(importPath string, verify func(ctx *Context, importPath, local string) (string, error)) error {
	importPath = slashToImportPath(importPath)
	ctx, err := NewContextWD()
	if err != nil {
		return err
	}

	err = ctx.LoadPackage(importPath)
	if err != nil {
		return err
	}

	localImportPath, err := findLocalImportPath(ctx, importPath)
	if err != nil {
		return err
	}
	// Adjust relative local path to GOPATH import path.
	localImportPath = path.Join(ctx.RootImportPath, internalFolder, localImportPath)

	importPath, err = verify(ctx, importPath, localImportPath)
	if err != nil {
		return err
	}

	err = ctx.AddImports(importPath)
	if err != nil {
		return err
	}

	pkg, foundPkg := ctx.Package[importPath]
	if !foundPkg {
		return ErrNotInGOPATH{importPath}
	}
	if pkg.Status != StatusExternal {
		if pkg.Status == StatusInternal {
			return ErrVendorExists
		}
		if pkg.Status == StatusLocal {
			return ErrLocalPackage
		}
		return ErrNotInGOPATH{importPath}
	}

	// Update vendor file with correct Local field.
	var vp *vendorfile.Package
	for _, vpkg := range ctx.VendorFile.Package {
		if vpkg.Canonical == importPath {
			vp = vpkg
			break
		}
	}
	if vp == nil {
		vp = &vendorfile.Package{
			Add:       true,
			Canonical: importPath,
			Local:     localImportPath,
		}
		ctx.VendorFile.Package = append(ctx.VendorFile.Package, vp)
		ctx.vendorFileLocal[vp.Local] = vp
	}
	// Find the VCS information.
	vcs, err := FindVcs(pkg.Gopath, pkg.Dir)
	if err != nil {
		return err
	}
	if vcs != nil {
		if vcs.Dirty {
			return ErrDirtyPackage{pkg.ImportPath}
		}
		vp.Revision = vcs.Revision
		if vcs.RevisionTime != nil {
			vp.RevisionTime = vcs.RevisionTime.Format(time.RFC3339)
		}
	}

	// Write the vendor file.
	err = writeVendorFile(ctx.RootDir, ctx.VendorFile)
	if err != nil {
		return err
	}

	// Copy the package locally.
	err = CopyPackage(filepath.Join(ctx.RootGopath, slashToFilepath(localImportPath)), pkg.Dir)
	if err != nil {
		return err
	}

	err = ctx.AddImports(importPath, localImportPath)
	if err != nil {
		return err
	}

	// Determine which files to touch.
	files := make(map[string]*File, len(ctx.VendorFile.Package)*3)

	// Rules are all lines in the vendor file.
	rules := make([]Rule, 0, len(ctx.VendorFile.Package))
	for _, vp := range ctx.VendorFile.Package {
		for fpath, f := range ctx.fileImports[vp.Canonical] {
			files[fpath] = f
		}
		rules = append(rules, Rule{From: vp.Canonical, To: vp.Local})
	}
	// Add local package files.
	if localPkg, found := ctx.Package[localImportPath]; found {
		for _, f := range localPkg.Files {
			files[f.Path] = f
		}
	}
	// Rewrite any external package where the local path is different then the vendor path.
	for _, pkg := range ctx.Package {
		if pkg.Status != StatusExternal {
			continue
		}
		if pkg.ImportPath == pkg.VendorPath {
			continue
		}
		for _, otherPkg := range ctx.Package {
			if pkg == otherPkg {
				continue
			}
			if otherPkg.Status != StatusInternal {
				continue
			}
			if otherPkg.VendorPath != pkg.VendorPath {
				continue
			}

			for fpath, f := range ctx.fileImports[pkg.ImportPath] {
				files[fpath] = f
			}
			rules = append(rules, Rule{From: pkg.ImportPath, To: otherPkg.ImportPath})
			break
		}
	}

	return ctx.RewriteFiles(files, rules)
}
func CmdRemove(importPath string) error {
	importPath = slashToImportPath(importPath)
	ctx, err := NewContextWD()
	if err != nil {
		return err
	}

	err = ctx.LoadPackage(importPath)
	if err != nil {
		return err
	}

	localPath := ""
	vendorPath := importPath
	localFound := false
	vendorFileIndex := 0
	for i, pkg := range ctx.VendorFile.Package {
		if pkg.Canonical == importPath {
			localPath = pkg.Local
			localFound = true
			vendorFileIndex = i
			break
		}
	}
	if !localFound {
		for i, pkg := range ctx.VendorFile.Package {
			if pkg.Local == importPath {
				localPath = pkg.Local
				vendorPath = pkg.Canonical
				localFound = true
				vendorFileIndex = i
				break
			}
		}
	}
	if !localFound {
		return ErrImportNotExists
	}
	if localPath == "" {
		return ErrNoLocalPath
	}

	files := ctx.fileImports[localPath]

	err = ctx.RewriteFiles(files, []Rule{{From: localPath, To: vendorPath}})
	if err != nil {
		return err
	}

	err = RemovePackage(filepath.Join(ctx.RootGopath, slashToFilepath(localPath)))
	if err != nil {
		return err
	}
	for i, pkg := range ctx.VendorFile.Package {
		if i == vendorFileIndex {
			pkg.Remove = true
		}
	}

	return writeVendorFile(ctx.RootDir, ctx.VendorFile)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/abbot/go-http-auth"
	"github.com/andelf/go-curl"
	"github.com/nu7hatch/gouuid"
	"github.com/rcrowley/go-metrics"
	"github.com/rcrowley/go-metrics/librato"
)

var config Config

type Config struct {
	HTTPBasicUsername string
	HTTPBasicPassword string
	HTTPBasicRealm    string
	Port              string
	Location          string
	ChecksURL         string
	MeasurerCount     int
	LibratoEmail      string
	LibratoToken      string
	ScheduleTimer     metrics.Timer
}

type Check struct {
	ID  string `json:"id"`
	URL string `json:"url"`
}

type Measurement struct {
	Check             Check   `json:"check"`
	ID                string  `json:"id"`
	Location          string  `json:"location"`
	T                 int     `json:"t"`
	ExitStatus        int     `json:"exit_status"`
	ConnectTime       float64 `json:"connect_time,omitempty"`
	StartTransferTime float64 `json:"starttransfer_time,omitempty"`
	LocalIP           string  `json:"local_ip,omitempty"`
	PrimaryIP         string  `json:"primary_ip,omitempty"`
	TotalTime         float64 `json:"total_time,omitempty"`
	HTTPStatus        int     `json:"http_status,omitempty"`
	NameLookupTime    float64 `json:"namelookup_time,omitempty"`
	SizeDownload      float64 `json:"size_download,omitempty"`
}

func (c *Check) Measure(config Config) Measurement {
	var m Measurement

	id, _ := uuid.NewV4()
	m.ID = id.String()
	m.Check = *c
	m.Location = config.Location

	easy := curl.EasyInit()
	defer easy.Cleanup()

	easy.Setopt(curl.OPT_URL, c.URL)

	// dummy func for curl output
	noOut := func(buf []byte, userdata interface{}) bool {
		return true
	}

	easy.Setopt(curl.OPT_WRITEFUNCTION, noOut)
	easy.Setopt(curl.OPT_CONNECTTIMEOUT, 10)
	easy.Setopt(curl.OPT_TIMEOUT, 10)

	now := time.Now()
	m.T = int(now.Unix())

	if err := easy.Perform(); err != nil {
		if e, ok := err.(curl.CurlError); ok {
			m.ExitStatus = (int(e))
			return m
		}
		os.Exit(1)
	}

	m.ExitStatus = 0
	httpStatus, _ := easy.Getinfo(curl.INFO_RESPONSE_CODE)
	m.HTTPStatus = httpStatus.(int)

	connectTime, _ := easy.Getinfo(curl.INFO_CONNECT_TIME)
	m.ConnectTime = connectTime.(float64)

	namelookupTime, _ := easy.Getinfo(curl.INFO_NAMELOOKUP_TIME)
	m.NameLookupTime = namelookupTime.(float64)

	starttransferTime, _ := easy.Getinfo(curl.INFO_STARTTRANSFER_TIME)
	m.StartTransferTime = starttransferTime.(float64)

	totalTime, _ := easy.Getinfo(curl.INFO_TOTAL_TIME)
	m.TotalTime = totalTime.(float64)

	localIP, _ := easy.Getinfo(curl.INFO_LOCAL_IP)
	m.LocalIP = localIP.(string)

	primaryIP, _ := easy.Getinfo(curl.INFO_PRIMARY_IP)
	m.PrimaryIP = primaryIP.(string)

	sizeDownload, _ := easy.Getinfo(curl.INFO_SIZE_DOWNLOAD)
	m.SizeDownload = sizeDownload.(float64)

	return m
}

func measurer(config Config, toMeasurer chan Check, toStreamer chan Measurement) {
	for {
		c := <-toMeasurer
		m := c.Measure(config)

		toStreamer <- m
	}
}

func streamer(config Config, toStreamer chan Measurement) {
	a := func(user, realm string) string {
		if user == config.HTTPBasicUsername {
			return config.HTTPBasicPassword
		}
		return ""
	}

	h := func(w http.ResponseWriter, r *auth.AuthenticatedRequest) {
		w.Header().Set("Content-Type", "application/json")
		enc := json.NewEncoder(w)
		for {
			err := enc.Encode(<-toStreamer)
			if err != nil {
				return
			}

			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			}
		}
	}

	authenticator := auth.NewBasicAuthenticator(config.HTTPBasicRealm, a)

	http.HandleFunc("/measurements", authenticator.Wrap(h))

	log.Printf("fn=streamer listening=true port=%s\n", config.Port)
	err := http.ListenAndServe(":"+config.Port, nil)
	if err != nil {
		log.Fatal(err)
	}
}

func getChecks(config Config) []Check {
	url := config.ChecksURL

	res, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}

	var checks []Check
	err = json.Unmarshal(body, &checks)
	if err != nil {
		log.Fatal(err)
	}

	return checks
}

func scheduler(config Config, check Check, toMeasurer chan Check) {
	for {
		config.ScheduleTimer.Time(func() { toMeasurer <- check })
		time.Sleep(1000 * time.Millisecond)
	}
}

func init() {
	flag.StringVar(&config.HTTPBasicUsername, "http_basic_username", "", "HTTP basic authentication username")
	flag.StringVar(&config.HTTPBasicPassword, "http_basic_password", "", "HTTP basic authentication password")
	flag.StringVar(&config.HTTPBasicRealm, "http_basic_realm", "", "HTTP basic authentication realm")
	flag.StringVar(&config.Port, "port", "5000", "port the HTTP server should listen on")
	flag.StringVar(&config.Location, "location", "undefined", "location of this sensor")
	flag.StringVar(&config.ChecksURL, "checks_url", "https://s3.amazonaws.com/canary-public-data/checks.json", "URL for check data")
	flag.IntVar(&config.MeasurerCount, "measurer_count", 1, "number of measurers to run")

	config.LibratoEmail = os.Getenv("LIBRATO_EMAIL")
	config.LibratoToken = os.Getenv("LIBRATO_TOKEN")

	config.ScheduleTimer = metrics.NewTimer()
	metrics.Register("schedule", config.ScheduleTimer)
}

func main() {
	flag.Parse()

	if len(config.HTTPBasicUsername) == 0 && len(config.HTTPBasicPassword) == 0 {
		log.Fatal("fatal - HTTP basic auth not set correctly")
	}

	if config.LibratoEmail != "" && config.LibratoToken != "" {
		log.Println("fn=main metircs=librato")
		go librato.Librato(metrics.DefaultRegistry,
			10e9,                  // interval
			config.LibratoEmail,   // account owner email address
			config.LibratoToken,   // Librato API token
			config.Location,       // source
			[]float64{50, 95, 99}, // precentiles to send
			time.Millisecond,      // time unit
		)
	}

	checkList := getChecks(config)

	toMeasurer := make(chan Check)
	toStreamer := make(chan Measurement)

	// spawn one scheduler per check
	for _, c := range checkList {
		go scheduler(config, c, toMeasurer)
	}

	// spawn N measurers
	for i := 0; i < config.MeasurerCount; i++ {
		go measurer(config, toMeasurer, toStreamer)
	}

	// stream measurements to clients over HTTP
	go streamer(config, toStreamer)

	select {}
}
<commit_msg>capture some metrics around how long it takes to push a measurement to the streamer<commit_after>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/abbot/go-http-auth"
	"github.com/andelf/go-curl"
	"github.com/nu7hatch/gouuid"
	"github.com/rcrowley/go-metrics"
	"github.com/rcrowley/go-metrics/librato"
)

var config Config

type Config struct {
	HTTPBasicUsername string
	HTTPBasicPassword string
	HTTPBasicRealm    string
	Port              string
	Location          string
	ChecksURL         string
	MeasurerCount     int
	LibratoEmail      string
	LibratoToken      string
	ScheduleTimer     metrics.Timer
	MeasureTimer      metrics.Timer
}

type Check struct {
	ID  string `json:"id"`
	URL string `json:"url"`
}

type Measurement struct {
	Check             Check   `json:"check"`
	ID                string  `json:"id"`
	Location          string  `json:"location"`
	T                 int     `json:"t"`
	ExitStatus        int     `json:"exit_status"`
	ConnectTime       float64 `json:"connect_time,omitempty"`
	StartTransferTime float64 `json:"starttransfer_time,omitempty"`
	LocalIP           string  `json:"local_ip,omitempty"`
	PrimaryIP         string  `json:"primary_ip,omitempty"`
	TotalTime         float64 `json:"total_time,omitempty"`
	HTTPStatus        int     `json:"http_status,omitempty"`
	NameLookupTime    float64 `json:"namelookup_time,omitempty"`
	SizeDownload      float64 `json:"size_download,omitempty"`
}

func (c *Check) Measure(config Config) Measurement {
	var m Measurement

	id, _ := uuid.NewV4()
	m.ID = id.String()
	m.Check = *c
	m.Location = config.Location

	easy := curl.EasyInit()
	defer easy.Cleanup()

	easy.Setopt(curl.OPT_URL, c.URL)

	// dummy func for curl output
	noOut := func(buf []byte, userdata interface{}) bool {
		return true
	}

	easy.Setopt(curl.OPT_WRITEFUNCTION, noOut)
	easy.Setopt(curl.OPT_CONNECTTIMEOUT, 10)
	easy.Setopt(curl.OPT_TIMEOUT, 10)

	now := time.Now()
	m.T = int(now.Unix())

	if err := easy.Perform(); err != nil {
		if e, ok := err.(curl.CurlError); ok {
			m.ExitStatus = (int(e))
			return m
		}
		os.Exit(1)
	}

	m.ExitStatus = 0
	httpStatus, _ := easy.Getinfo(curl.INFO_RESPONSE_CODE)
	m.HTTPStatus = httpStatus.(int)

	connectTime, _ := easy.Getinfo(curl.INFO_CONNECT_TIME)
	m.ConnectTime = connectTime.(float64)

	namelookupTime, _ := easy.Getinfo(curl.INFO_NAMELOOKUP_TIME)
	m.NameLookupTime = namelookupTime.(float64)

	starttransferTime, _ := easy.Getinfo(curl.INFO_STARTTRANSFER_TIME)
	m.StartTransferTime = starttransferTime.(float64)

	totalTime, _ := easy.Getinfo(curl.INFO_TOTAL_TIME)
	m.TotalTime = totalTime.(float64)

	localIP, _ := easy.Getinfo(curl.INFO_LOCAL_IP)
	m.LocalIP = localIP.(string)

	primaryIP, _ := easy.Getinfo(curl.INFO_PRIMARY_IP)
	m.PrimaryIP = primaryIP.(string)

	sizeDownload, _ := easy.Getinfo(curl.INFO_SIZE_DOWNLOAD)
	m.SizeDownload = sizeDownload.(float64)

	return m
}

func measurer(config Config, toMeasurer chan Check, toStreamer chan Measurement) {
	for {
		c := <-toMeasurer
		m := c.Measure(config)

		config.MeasureTimer.Time(func() { toStreamer <- m })
	}
}

func streamer(config Config, toStreamer chan Measurement) {
	a := func(user, realm string) string {
		if user == config.HTTPBasicUsername {
			return config.HTTPBasicPassword
		}
		return ""
	}

	h := func(w http.ResponseWriter, r *auth.AuthenticatedRequest) {
		w.Header().Set("Content-Type", "application/json")
		enc := json.NewEncoder(w)
		for {
			err := enc.Encode(<-toStreamer)
			if err != nil {
				return
			}

			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			}
		}
	}

	authenticator := auth.NewBasicAuthenticator(config.HTTPBasicRealm, a)

	http.HandleFunc("/measurements", authenticator.Wrap(h))

	log.Printf("fn=streamer listening=true port=%s\n", config.Port)
	err := http.ListenAndServe(":"+config.Port, nil)
	if err != nil {
		log.Fatal(err)
	}
}

func getChecks(config Config) []Check {
	url := config.ChecksURL

	res, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}

	var checks []Check
	err = json.Unmarshal(body, &checks)
	if err != nil {
		log.Fatal(err)
	}

	return checks
}

func scheduler(config Config, check Check, toMeasurer chan Check) {
	for {
		config.ScheduleTimer.Time(func() { toMeasurer <- check })
		time.Sleep(1000 * time.Millisecond)
	}
}

func init() {
	flag.StringVar(&config.HTTPBasicUsername, "http_basic_username", "", "HTTP basic authentication username")
	flag.StringVar(&config.HTTPBasicPassword, "http_basic_password", "", "HTTP basic authentication password")
	flag.StringVar(&config.HTTPBasicRealm, "http_basic_realm", "", "HTTP basic authentication realm")
	flag.StringVar(&config.Port, "port", "5000", "port the HTTP server should listen on")
	flag.StringVar(&config.Location, "location", "undefined", "location of this sensor")
	flag.StringVar(&config.ChecksURL, "checks_url", "https://s3.amazonaws.com/canary-public-data/checks.json", "URL for check data")
	flag.IntVar(&config.MeasurerCount, "measurer_count", 1, "number of measurers to run")

	config.LibratoEmail = os.Getenv("LIBRATO_EMAIL")
	config.LibratoToken = os.Getenv("LIBRATO_TOKEN")

	config.ScheduleTimer = metrics.NewTimer()
	metrics.Register("sensord.schedule", config.ScheduleTimer)

	config.MeasureTimer = metrics.NewTimer()
	metrics.Register("sensord.measure", config.MeasureTimer)
}

func main() {
	flag.Parse()

	if len(config.HTTPBasicUsername) == 0 && len(config.HTTPBasicPassword) == 0 {
		log.Fatal("fatal - HTTP basic auth not set correctly")
	}

	if config.LibratoEmail != "" && config.LibratoToken != "" {
		log.Println("fn=main metircs=librato")
		go librato.Librato(metrics.DefaultRegistry,
			10e9,                  // interval
			config.LibratoEmail,   // account owner email address
			config.LibratoToken,   // Librato API token
			config.Location,       // source
			[]float64{50, 95, 99}, // precentiles to send
			time.Millisecond,      // time unit
		)
	}

	checkList := getChecks(config)

	toMeasurer := make(chan Check)
	toStreamer := make(chan Measurement)

	// spawn one scheduler per check
	for _, c := range checkList {
		go scheduler(config, c, toMeasurer)
	}

	// spawn N measurers
	for i := 0; i < config.MeasurerCount; i++ {
		go measurer(config, toMeasurer, toStreamer)
	}

	// stream measurements to clients over HTTP
	go streamer(config, toStreamer)

	select {}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/tylerb/graceful"

	"github.com/stripe/sequins/backend"
)

type sequins struct {
	config  sequinsConfig
	http    *http.Server
	backend backend.Backend

	dbs     map[string]*db
	dbsLock sync.RWMutex

	peers     *peers
	zkWatcher *zkWatcher

	refreshLock    sync.Mutex
	refreshWorkers chan bool
	refreshTicker  *time.Ticker
	sighups        chan os.Signal
}

type status struct {
	DBs map[string]dbStatus `json:"dbs"`
}

func newSequins(backend backend.Backend, config sequinsConfig) *sequins {
	return &sequins{
		config:      config,
		backend:     backend,
		refreshLock: sync.Mutex{},
	}
}

func (s *sequins) init() error {
	if s.config.ZK.Servers != nil {
		err := s.initCluster()
		if err != nil {
			return err
		}
	}

	// Create local directories, and load any cached versions we have.
	err := s.initLocalStore()
	if err != nil {
		return nil
	}

	// To limit the number of parallel loads, create a full buffered channel.
	// Workers grab one from the channel when they trigger a load, and put it
	// back when they're done.
	maxLoads := s.config.MaxParallelLoads
	if maxLoads != 0 {
		s.refreshWorkers = make(chan bool, maxLoads)
		for i := 0; i < maxLoads; i++ {
			s.refreshWorkers <- true
		}
	}

	// Trigger loads before we start up.
	s.refreshAll()

	// Automatically refresh, if configured to do so.
	refresh := s.config.RefreshPeriod.Duration
	if refresh != 0 {
		s.refreshTicker = time.NewTicker(refresh)
		go func() {
			log.Println("Automatically checking for new versions every", refresh.String())
			for range s.refreshTicker.C {
				s.refreshAll()
			}
		}()
	}

	// Refresh on SIGHUP.
	sighups := make(chan os.Signal)
	signal.Notify(sighups, syscall.SIGHUP)
	go func() {
		for range sighups {
			s.refreshAll()
		}
	}()

	s.sighups = sighups
	return nil
}

func (s *sequins) initCluster() error {
	prefix := path.Join("/", s.config.ZK.ClusterName)
	zkWatcher, err := connectZookeeper(s.config.ZK.Servers, prefix)
	if err != nil {
		return err
	}

	hostname := s.config.ZK.AdvertisedHostname
	if hostname == "" {
		hostname, err = os.Hostname()
		if err != nil {
			return err
		}
	}

	_, port, err := net.SplitHostPort(s.config.Bind)
	if err != nil {
		return err
	}

	routableAddress := net.JoinHostPort(hostname, port)
	shardID := s.config.ZK.ShardID
	if shardID == "" {
		shardID = routableAddress
	}

	peers := watchPeers(zkWatcher, shardID, routableAddress)
	peers.waitToConverge(s.config.ZK.TimeToConverge.Duration)

	s.zkWatcher = zkWatcher
	s.peers = peers
	return nil
}

func (s *sequins) initLocalStore() error {
	// TODO: lock local filestore
	dataPath := filepath.Join(s.config.LocalStore, "data")
	os.MkdirAll(dataPath, 0755|os.ModeDir)

	// Attempt to load any versions we have cached locally, so that we can start
	// quickly even if there are newer versions to download.
	res, err := ioutil.ReadDir(dataPath)
	if err != nil {
		return err
	}

	localDirs := make(map[string]bool)
	for _, dir := range res {
		localDirs[dir.Name()] = true
	}

	dbs, err := s.backend.ListDBs()
	if err != nil {
		log.Println("Error listing DBs from %s: %s", s.backend.DisplayPath(""), err)
		return err
	}

	built := make(chan *db, len(dbs))
	for _, dbName := range dbs {
		if !localDirs[dbName] {
			built <- nil
			continue
		}

		go func(name string) {
			db := newDB(s, name)
			err := db.loadLatestLocalVersion()
			if err == nil {
				built <- db
			} else {
				built <- nil
			}
		}(dbName)
	}

	localDBs := make(map[string]*db)
	for i := 0; i < len(dbs); i++ {
		db := <-built
		if db != nil {
			localDBs[db.name] = db
		}
	}

	// TODO: clean up any local data we have that doesn't have a corresponding
	// remote DB or version

	s.dbsLock.Lock()
	s.dbs = localDBs
	s.dbsLock.Unlock()
	return nil
}

func (s *sequins) start() {
	defer s.shutdown()

	log.Println("Listening on", s.config.Bind)
	graceful.Run(s.config.Bind, time.Second, s)
}

func (s *sequins) shutdown() {
	log.Println("Shutting down...")
	signal.Stop(s.sighups)

	if s.refreshTicker != nil {
		s.refreshTicker.Stop()
	}

	zk := s.zkWatcher
	if zk != nil {
		zk.close()
	}

	// TODO: figure out how to cancel in-progress downloads
	// s.dbsLock.Lock()
	// defer s.dbsLock.Unlock()

	// for _, db := range s.dbs {
	// 	db.close()
	// }
}

func (s *sequins) refreshAll() {
	s.refreshLock.Lock()
	defer s.refreshLock.Unlock()

	dbs, err := s.backend.ListDBs()
	if err != nil {
		log.Println("Error listing DBs from %s: %s", s.backend.DisplayPath(""), err)
		return
	}

	// Add any new DBS, and trigger refreshes for existing ones. This
	// Lock pattern is a bit goofy, since the lock is on the request
	// path and we want to hold it for as short a time as possible.
	s.dbsLock.RLock()

	newDBs := make(map[string]*db)
	for _, name := range dbs {
		db := s.dbs[name]
		if db == nil {
			db = newDB(s, name)
			// TODO: load the latest valid cached version
		}

		go s.refresh(db)
		newDBs[name] = db
	}

	// Now, grab the full lock to switch the new map in.
	s.dbsLock.RUnlock()
	s.dbsLock.Lock()

	oldDBs := s.dbs
	s.dbs = newDBs

	// Finally, close any dbs that we're removing.
	s.dbsLock.Unlock()
	s.dbsLock.RLock()

	for name, db := range oldDBs {
		if s.dbs[name] == nil {
			db.close()
			db.delete()
		}
	}

	s.dbsLock.RUnlock()
}

func (s *sequins) refresh(db *db) {
	if s.refreshWorkers != nil {
		<-s.refreshWorkers
		defer func() {
			s.refreshWorkers <- true
		}()
	}

	err := db.refresh()
	if err != nil {
		log.Printf("Error refreshing %s: %s", db.name, err)
	}
}

func (s *sequins) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if r.URL.Path == "/" {
		s.serveStatus(w, r)
		return
	}

	var dbName, key string
	path := strings.TrimPrefix(r.URL.Path, "/")
	split := strings.Index(path, "/")
	if split == -1 {
		dbName = path
		key = ""
	} else {
		dbName = path[:split]
		key = path[split+1:]
	}

	if dbName == "" {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	s.dbsLock.RLock()
	db := s.dbs[dbName]
	s.dbsLock.RUnlock()

	// If this is a proxy request, we don't want to confuse "we don't have this
	// db" with "we don't have this key"; if this is a proxied request, then the
	// peer apparently does have the db, and thinks we do too (which should never
	// happen). So we use 501 Not Implemented to indicate the former. To users,
	// we present a uniform 404.
	if db == nil {
		if r.URL.Query().Get("proxy") != "" {
			w.WriteHeader(http.StatusNotImplemented)
		} else {
			w.WriteHeader(http.StatusNotFound)
		}

		return
	}

	db.serveKey(w, r, key)
}

func (s *sequins) serveStatus(w http.ResponseWriter, r *http.Request) {
	s.dbsLock.RLock()

	status := status{DBs: make(map[string]dbStatus)}
	for name, db := range s.dbs {
		status.DBs[name] = db.status()
	}

	s.dbsLock.RUnlock()

	jsonBytes, err := json.Marshal(status)
	if err != nil {
		log.Println("Error serving status:", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.Header()["Content-Type"] = []string{"application/json"}
	w.Write(jsonBytes)
}
<commit_msg>Shutdown is overrated<commit_after>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/tylerb/graceful"

	"github.com/stripe/sequins/backend"
)

type sequins struct {
	config  sequinsConfig
	http    *http.Server
	backend backend.Backend

	dbs     map[string]*db
	dbsLock sync.RWMutex

	peers     *peers
	zkWatcher *zkWatcher

	refreshLock    sync.Mutex
	refreshWorkers chan bool
	refreshTicker  *time.Ticker
	sighups        chan os.Signal
}

type status struct {
	DBs map[string]dbStatus `json:"dbs"`
}

func newSequins(backend backend.Backend, config sequinsConfig) *sequins {
	return &sequins{
		config:      config,
		backend:     backend,
		refreshLock: sync.Mutex{},
	}
}

func (s *sequins) init() error {
	if s.config.ZK.Servers != nil {
		err := s.initCluster()
		if err != nil {
			return err
		}
	}

	// Create local directories, and load any cached versions we have.
	err := s.initLocalStore()
	if err != nil {
		return nil
	}

	// To limit the number of parallel loads, create a full buffered channel.
	// Workers grab one from the channel when they trigger a load, and put it
	// back when they're done.
	maxLoads := s.config.MaxParallelLoads
	if maxLoads != 0 {
		s.refreshWorkers = make(chan bool, maxLoads)
		for i := 0; i < maxLoads; i++ {
			s.refreshWorkers <- true
		}
	}

	// Trigger loads before we start up.
	s.refreshAll()

	// Automatically refresh, if configured to do so.
	refresh := s.config.RefreshPeriod.Duration
	if refresh != 0 {
		s.refreshTicker = time.NewTicker(refresh)
		go func() {
			log.Println("Automatically checking for new versions every", refresh.String())
			for range s.refreshTicker.C {
				s.refreshAll()
			}
		}()
	}

	// Refresh on SIGHUP.
	sighups := make(chan os.Signal)
	signal.Notify(sighups, syscall.SIGHUP)
	go func() {
		for range sighups {
			s.refreshAll()
		}
	}()

	s.sighups = sighups
	return nil
}

func (s *sequins) initCluster() error {
	prefix := path.Join("/", s.config.ZK.ClusterName)
	zkWatcher, err := connectZookeeper(s.config.ZK.Servers, prefix)
	if err != nil {
		return err
	}

	hostname := s.config.ZK.AdvertisedHostname
	if hostname == "" {
		hostname, err = os.Hostname()
		if err != nil {
			return err
		}
	}

	_, port, err := net.SplitHostPort(s.config.Bind)
	if err != nil {
		return err
	}

	routableAddress := net.JoinHostPort(hostname, port)
	shardID := s.config.ZK.ShardID
	if shardID == "" {
		shardID = routableAddress
	}

	peers := watchPeers(zkWatcher, shardID, routableAddress)
	peers.waitToConverge(s.config.ZK.TimeToConverge.Duration)

	s.zkWatcher = zkWatcher
	s.peers = peers
	return nil
}

func (s *sequins) initLocalStore() error {
	// TODO: lock local filestore
	dataPath := filepath.Join(s.config.LocalStore, "data")
	os.MkdirAll(dataPath, 0755|os.ModeDir)

	// Attempt to load any versions we have cached locally, so that we can start
	// quickly even if there are newer versions to download.
	res, err := ioutil.ReadDir(dataPath)
	if err != nil {
		return err
	}

	localDirs := make(map[string]bool)
	for _, dir := range res {
		localDirs[dir.Name()] = true
	}

	dbs, err := s.backend.ListDBs()
	if err != nil {
		log.Println("Error listing DBs from %s: %s", s.backend.DisplayPath(""), err)
		return err
	}

	built := make(chan *db, len(dbs))
	for _, dbName := range dbs {
		if !localDirs[dbName] {
			built <- nil
			continue
		}

		go func(name string) {
			db := newDB(s, name)
			err := db.loadLatestLocalVersion()
			if err == nil {
				built <- db
			} else {
				built <- nil
			}
		}(dbName)
	}

	localDBs := make(map[string]*db)
	for i := 0; i < len(dbs); i++ {
		db := <-built
		if db != nil {
			localDBs[db.name] = db
		}
	}

	// TODO: clean up any local data we have that doesn't have a corresponding
	// remote DB or version

	s.dbsLock.Lock()
	s.dbs = localDBs
	s.dbsLock.Unlock()
	return nil
}

func (s *sequins) start() {
	defer s.shutdown()

	log.Println("Listening on", s.config.Bind)
	graceful.Run(s.config.Bind, time.Second, s)
}

func (s *sequins) shutdown() {
	log.Println("Shutting down...")
	signal.Stop(s.sighups)

	if s.refreshTicker != nil {
		s.refreshTicker.Stop()
	}

	// zk := s.zkWatcher
	// if zk != nil {
	// 	zk.close()
	// }

	// TODO: figure out how to cancel in-progress downloads
	// s.dbsLock.Lock()
	// defer s.dbsLock.Unlock()

	// for _, db := range s.dbs {
	// 	db.close()
	// }
}

func (s *sequins) refreshAll() {
	s.refreshLock.Lock()
	defer s.refreshLock.Unlock()

	dbs, err := s.backend.ListDBs()
	if err != nil {
		log.Println("Error listing DBs from %s: %s", s.backend.DisplayPath(""), err)
		return
	}

	// Add any new DBS, and trigger refreshes for existing ones. This
	// Lock pattern is a bit goofy, since the lock is on the request
	// path and we want to hold it for as short a time as possible.
	s.dbsLock.RLock()

	newDBs := make(map[string]*db)
	for _, name := range dbs {
		db := s.dbs[name]
		if db == nil {
			db = newDB(s, name)
			// TODO: load the latest valid cached version
		}

		go s.refresh(db)
		newDBs[name] = db
	}

	// Now, grab the full lock to switch the new map in.
	s.dbsLock.RUnlock()
	s.dbsLock.Lock()

	oldDBs := s.dbs
	s.dbs = newDBs

	// Finally, close any dbs that we're removing.
	s.dbsLock.Unlock()
	s.dbsLock.RLock()

	for name, db := range oldDBs {
		if s.dbs[name] == nil {
			db.close()
			db.delete()
		}
	}

	s.dbsLock.RUnlock()
}

func (s *sequins) refresh(db *db) {
	if s.refreshWorkers != nil {
		<-s.refreshWorkers
		defer func() {
			s.refreshWorkers <- true
		}()
	}

	err := db.refresh()
	if err != nil {
		log.Printf("Error refreshing %s: %s", db.name, err)
	}
}

func (s *sequins) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if r.URL.Path == "/" {
		s.serveStatus(w, r)
		return
	}

	var dbName, key string
	path := strings.TrimPrefix(r.URL.Path, "/")
	split := strings.Index(path, "/")
	if split == -1 {
		dbName = path
		key = ""
	} else {
		dbName = path[:split]
		key = path[split+1:]
	}

	if dbName == "" {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	s.dbsLock.RLock()
	db := s.dbs[dbName]
	s.dbsLock.RUnlock()

	// If this is a proxy request, we don't want to confuse "we don't have this
	// db" with "we don't have this key"; if this is a proxied request, then the
	// peer apparently does have the db, and thinks we do too (which should never
	// happen). So we use 501 Not Implemented to indicate the former. To users,
	// we present a uniform 404.
	if db == nil {
		if r.URL.Query().Get("proxy") != "" {
			w.WriteHeader(http.StatusNotImplemented)
		} else {
			w.WriteHeader(http.StatusNotFound)
		}

		return
	}

	db.serveKey(w, r, key)
}

func (s *sequins) serveStatus(w http.ResponseWriter, r *http.Request) {
	s.dbsLock.RLock()

	status := status{DBs: make(map[string]dbStatus)}
	for name, db := range s.dbs {
		status.DBs[name] = db.status()
	}

	s.dbsLock.RUnlock()

	jsonBytes, err := json.Marshal(status)
	if err != nil {
		log.Println("Error serving status:", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.Header()["Content-Type"] = []string{"application/json"}
	w.Write(jsonBytes)
}
<|endoftext|>


<commit_before>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file tests function evaluation.

package golisp

import (
	. "launchpad.net/gocheck"
)

type EvalSuite struct {
}

var _ = Suite(&EvalSuite{})

func (s *EvalSuite) SetUpSuite(c *C) {
	Global = NewSymbolTableFrameBelow(nil)
	InitBuiltins()
}

func (s *EvalSuite) TestEvalWithNilFunction(c *C) {
	code, _ := Parse("(no-function-named-this)")
	result, err := Eval(code, Global)
	c.Assert(err, NotNil)
	c.Assert(result, IsNil)
}

func (s *EvalSuite) TestEvalWithOverridingSearchEnv(c *C) {
	newEnv := NewSymbolTableFrameBelow(Global)
	sym := SymbolWithName("test")
	code, _ := Parse("test")
	Global.BindLocallyTo(sym, IntegerWithValue(10))
	newEnv.BindLocallyTo(sym, IntegerWithValue(42))
	globalVal, _ := Eval(code, Global)
	c.Assert(IntegerValue(globalVal), Equals, int64(10))
	newEnvVal, _ := Eval(code, Global, newEnv)
	c.Assert(IntegerValue(newEnvVal), Equals, int64(42))
}
<commit_msg>Add a test for indirect evaluation<commit_after>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file tests function evaluation.

package golisp

import (
	. "launchpad.net/gocheck"
)

type EvalSuite struct {
}

var _ = Suite(&EvalSuite{})

func (s *EvalSuite) SetUpSuite(c *C) {
	Global = NewSymbolTableFrameBelow(nil)
	InitBuiltins()
}

func (s *EvalSuite) TestEvalWithNilFunction(c *C) {
	code, _ := Parse("(no-function-named-this)")
	result, err := Eval(code, Global)
	c.Assert(err, NotNil)
	c.Assert(result, IsNil)
}

func (s *EvalSuite) TestEvalWithOverridingSearchEnv(c *C) {
	newEnv := NewSymbolTableFrameBelow(Global)
	sym := SymbolWithName("test")
	code, _ := Parse("test")
	Global.BindLocallyTo(sym, IntegerWithValue(10))
	newEnv.BindLocallyTo(sym, IntegerWithValue(42))
	globalVal, _ := Eval(code, Global)
	c.Assert(IntegerValue(globalVal), Equals, int64(10))
	newEnvVal, _ := Eval(code, Global, newEnv)
	c.Assert(IntegerValue(newEnvVal), Equals, int64(42))
}

func (s *EvalSuite) TestIndirectEvalWithOverridingSearchEnv(c *C) {
	newEnv := NewSymbolTableFrameBelow(Global)
	sym := SymbolWithName("test")
	code, _ := Parse("(+ (* test 2) 1)")
	Global.BindLocallyTo(sym, IntegerWithValue(10))
	newEnv.BindLocallyTo(sym, IntegerWithValue(42))
	globalVal, _ := Eval(code, Global)
	c.Assert(IntegerValue(globalVal), Equals, int64(21))
	newEnvVal, _ := Eval(code, Global, newEnv)
	c.Assert(IntegerValue(newEnvVal), Equals, int64(85))
}
<|endoftext|>