<commit_before>package light

import (
	"bytes"
    "errors"
    "net/http"
    "io/ioutil"
	"encoding/json"
	"github.com/kju2/buildbulb/util"
)

type color struct {
	Hue float32
	Kelvin float32
	Saturation float32
	Brightness float32
}

type lightHttp struct {
	Id string
	Label string
	apiKey string
	Color color
	Power string
	Duration int
}

func newLightHttp(bulbName string, apiKey string) (*lightHttp, error) {
	util.Log.Info("Using LIFX HTTP REST interface")
	req, err := http.NewRequest("GET", "https://api.lifx.com/v1/lights/label:" + bulbName, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Authorization", "Bearer " + apiKey)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusNotFound {
		return nil, errors.New("No bulb found with label " + bulbName)
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var lights []lightHttp
	err = json.Unmarshal(body, &lights)
	if err != nil {
		return nil, err
	}

	if len(lights)> 1 {
		return nil, errors.New("Bulb name is ambiguous.")
	}
	lights[0].apiKey = apiKey
	lights[0].Duration = 2
	lights[0].Color.Brightness = 1
	return &lights[0], nil
}

func (l *lightHttp) setColor(c Color) {
	l.Color.Hue = float32(c)
	l.update()
}

func (l *lightHttp) setPower(p bool) {
	if p {
		l.Power = "on"
	} else {
		l.Power = "off"
	}
	l.update()
}

func (l *lightHttp) turnOff() {
	l.setPower(false)
}

func (l *lightHttp) turnOn() {
	l.setPower(true)
}

func (l *lightHttp) update() {
	url := "https://api.lifx.com/v1/lights/label:" + l.Label + "/state"
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(l)
	req, _ := http.NewRequest("PUT", url, b)
	req.Header.Set("Authorization", "Bearer " + l.apiKey)
	req.Header.Set("Content-Type", "application/json")
	_, err := http.DefaultClient.Do(req)
	if err != nil {
		util.Log.Error("Could not update light! ", err)
	}
}<commit_msg>Fix http connection resource leak<commit_after>package light

import (
	"bytes"
    "errors"
    "net/http"
    "io/ioutil"
	"encoding/json"
	"github.com/kju2/buildbulb/util"
)

type color struct {
	Hue float32
	Kelvin float32
	Saturation float32
	Brightness float32
}

type lightHttp struct {
	Id string
	Label string
	apiKey string
	Color color
	Power string
	Duration int
}

func newLightHttp(bulbName string, apiKey string) (*lightHttp, error) {
	util.Log.Info("Using LIFX HTTP REST interface")
	req, err := http.NewRequest("GET", "https://api.lifx.com/v1/lights/label:" + bulbName, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Authorization", "Bearer " + apiKey)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusNotFound {
		return nil, errors.New("No bulb found with label " + bulbName)
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var lights []lightHttp
	err = json.Unmarshal(body, &lights)
	if err != nil {
		return nil, err
	}

	if len(lights)> 1 {
		return nil, errors.New("Bulb name is ambiguous.")
	}
	lights[0].apiKey = apiKey
	lights[0].Duration = 2
	lights[0].Color.Brightness = 1
	return &lights[0], nil
}

func (l *lightHttp) setColor(c Color) {
	l.Color.Hue = float32(c)
	l.update()
}

func (l *lightHttp) setPower(p bool) {
	if p {
		l.Power = "on"
	} else {
		l.Power = "off"
	}
	l.update()
}

func (l *lightHttp) turnOff() {
	l.setPower(false)
}

func (l *lightHttp) turnOn() {
	l.setPower(true)
}

func (l *lightHttp) update() {
	url := "https://api.lifx.com/v1/lights/label:" + l.Label + "/state"
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(l)
	req, _ := http.NewRequest("PUT", url, b)
	req.Header.Set("Authorization", "Bearer " + l.apiKey)
	req.Header.Set("Content-Type", "application/json")
        resp, err := http.DefaultClient.Do(req)
        defer resp.Body.Close()
	if err != nil {
		util.Log.Error("Could not update light! ", err)
	}
}
<|endoftext|>


<commit_before>package lmdb

import (
	crand "crypto/rand"
	"io/ioutil"
	"math/rand"
	"os"
	"sync/atomic"
	"testing"
)

// repeatedly put (overwrite) keys.
func BenchmarkTxn_Put(b *testing.B) {
	initRandSource(b)
	env, path := setupBenchDB(b)
	defer teardownBenchDB(b, env, path)

	dbi := openBenchDBI(b, env)

	rc := newRandSourceCursor()
	ps, err := populateBenchmarkDB(env, dbi, &rc)
	if err != nil {
		b.Errorf("populate db: %v", err)
		return
	}

	err = env.Update(func(txn *Txn) (err error) {
		b.ResetTimer()
		defer b.StopTimer()
		for i := 0; i < b.N; i++ {
			k := ps[rand.Intn(len(ps)/2)*2]
			v := makeBenchDBVal(&rc)
			err := txn.Put(dbi, k, v, 0)
			if err != nil {
				return err
			}
		}
		return nil
	})
	if err != nil {
		b.Error(err)
		return
	}
}

// repeatedly put (overwrite) keys using the PutReserve method.
func BenchmarkTxn_PutReserve(b *testing.B) {
	initRandSource(b)
	env, path := setupBenchDB(b)
	defer teardownBenchDB(b, env, path)

	dbi := openBenchDBI(b, env)

	rc := newRandSourceCursor()
	ps, err := populateBenchmarkDB(env, dbi, &rc)
	if err != nil {
		b.Errorf("populate db: %v", err)
		return
	}

	err = env.Update(func(txn *Txn) (err error) {
		b.ResetTimer()
		defer b.StopTimer()
		for i := 0; i < b.N; i++ {
			k := ps[rand.Intn(len(ps)/2)*2]
			v := makeBenchDBVal(&rc)
			buf, err := txn.PutReserve(dbi, k, len(v), 0)
			if err != nil {
				return err
			}
			copy(buf, v)
		}
		return nil
	})
	if err != nil {
		b.Error(err)
		return
	}
}

// repeatedly put (overwrite) keys using the PutReserve method on an
// environment with WriteMap.
func BenchmarkTxn_PutReserve_writemap(b *testing.B) {
	initRandSource(b)
	env, path := setupBenchDBFlags(b, WriteMap)
	defer teardownBenchDB(b, env, path)

	dbi := openBenchDBI(b, env)

	rc := newRandSourceCursor()
	ps, err := populateBenchmarkDB(env, dbi, &rc)
	if err != nil {
		b.Errorf("populate db: %v", err)
		return
	}

	err = env.Update(func(txn *Txn) (err error) {
		b.ResetTimer()
		defer b.StopTimer()
		for i := 0; i < b.N; i++ {
			k := ps[rand.Intn(len(ps)/2)*2]
			v := makeBenchDBVal(&rc)
			buf, err := txn.PutReserve(dbi, k, len(v), 0)
			if err != nil {
				return err
			}
			copy(buf, v)
		}
		return nil
	})
	if err != nil {
		b.Error(err)
		return
	}
}

// repeatedly put (overwrite) keys.
func BenchmarkTxn_Put_writemap(b *testing.B) {
	initRandSource(b)
	env, path := setupBenchDBFlags(b, WriteMap)
	defer teardownBenchDB(b, env, path)

	dbi := openBenchDBI(b, env)

	var ps [][]byte

	rc := newRandSourceCursor()
	ps, err := populateBenchmarkDB(env, dbi, &rc)
	if err != nil {
		b.Errorf("populate db: %v", err)
		return
	}

	err = env.Update(func(txn *Txn) (err error) {
		b.ResetTimer()
		defer b.StopTimer()
		for i := 0; i < b.N; i++ {
			k := ps[rand.Intn(len(ps)/2)*2]
			v := makeBenchDBVal(&rc)
			err := txn.Put(dbi, k, v, 0)
			bTxnMust(b, txn, err, "putting data")
		}

		return nil
	})
}

// repeatedly get random keys.
func BenchmarkTxn_Get_ro(b *testing.B) {
	initRandSource(b)
	env, path := setupBenchDB(b)
	defer teardownBenchDB(b, env, path)

	dbi := openBenchDBI(b, env)

	rc := newRandSourceCursor()
	ps, err := populateBenchmarkDB(env, dbi, &rc)
	if err != nil {
		b.Errorf("populate db: %v", err)
		return
	}

	err = env.View(func(txn *Txn) (err error) {
		b.ResetTimer()
		defer b.StopTimer()
		for i := 0; i < b.N; i++ {
			_, err := txn.Get(dbi, ps[rand.Intn(len(ps))])
			if IsNotFound(err) {
				continue
			}
			if err != nil {
				b.Fatalf("error getting data: %v", err)
			}
		}

		return nil
	})
	if err != nil {
		b.Error(err)
	}
}

// like BenchmarkTxnGetReadonly but txn.RawRead is set to true.
func BenchmarkTxn_Get_raw_ro(b *testing.B) {
	initRandSource(b)
	env, path := setupBenchDB(b)
	defer teardownBenchDB(b, env, path)

	dbi := openBenchDBI(b, env)

	rc := newRandSourceCursor()
	ps, err := populateBenchmarkDB(env, dbi, &rc)
	if err != nil {
		b.Errorf("populate db: %v", err)
		return
	}

	err = env.View(func(txn *Txn) (err error) {
		txn.RawRead = true
		b.ResetTimer()
		defer b.StopTimer()
		for i := 0; i < b.N; i++ {
			_, err := txn.Get(dbi, ps[rand.Intn(len(ps))])
			if IsNotFound(err) {
				continue
			}
			if err != nil {
				b.Fatalf("error getting data: %v", err)
			}
		}
		return nil
	})
	if err != nil {
		b.Error(err)
		return
	}
}

// repeatedly scan all the values in a database.
func BenchmarkScan_ro(b *testing.B) {
	initRandSource(b)
	env, path := setupBenchDB(b)
	defer teardownBenchDB(b, env, path)

	dbi := openBenchDBI(b, env)

	rc := newRandSourceCursor()
	_, err := populateBenchmarkDB(env, dbi, &rc)
	if err != nil {
		b.Errorf("populate db: %v", err)
		return
	}

	err = env.View(func(txn *Txn) (err error) {
		b.ResetTimer()
		defer b.StopTimer()
		for i := 0; i < b.N; i++ {
			err := benchmarkScanDBI(txn, dbi)
			if err != nil {
				return err
			}
		}

		return nil
	})
	if err != nil {
		b.Error(err)
		return
	}
}

// like BenchmarkCursoreScanReadonly but txn.RawRead is set to true.
func BenchmarkScan_raw_ro(b *testing.B) {
	initRandSource(b)
	env, path := setupBenchDB(b)
	defer teardownBenchDB(b, env, path)

	dbi := openBenchDBI(b, env)

	rc := newRandSourceCursor()
	_, err := populateBenchmarkDB(env, dbi, &rc)
	if err != nil {
		b.Errorf("populate db: %v", err)
		return
	}

	err = env.View(func(txn *Txn) (err error) {
		txn.RawRead = true

		b.ResetTimer()
		defer b.StopTimer()
		for i := 0; i < b.N; i++ {
			err := benchmarkScanDBI(txn, dbi)
			if err != nil {
				return err
			}
		}

		return nil
	})
	if err != nil {
		b.Errorf("benchmark: %v", err)
		return
	}
}

func populateBenchmarkDB(env *Env, dbi DBI, rc *randSourceCursor) ([][]byte, error) {
	var ps [][]byte

	err := env.Update(func(txn *Txn) (err error) {
		for i := 0; i < benchDBNumKeys; i++ {
			k := makeBenchDBKey(rc)
			v := makeBenchDBVal(rc)
			err := txn.Put(dbi, k, v, 0)
			ps = append(ps, k, v)
			if err != nil {
				return err
			}
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	return ps, nil
}

func benchmarkScanDBI(txn *Txn, dbi DBI) error {
	cur, err := txn.OpenCursor(dbi)
	if err != nil {
		return err
	}
	defer cur.Close()

	var count int64
	for {
		_, _, err := cur.Get(nil, nil, Next)
		if IsNotFound(err) {
			return nil
		}
		if err != nil {
			return err
		}
		count++
	}
}

func setupBenchDB(b *testing.B) (*Env, string) {
	return setupBenchDBFlags(b, 0)

}
func setupBenchDBFlags(b *testing.B, flags uint) (*Env, string) {
	env, err := NewEnv()
	bMust(b, err, "creating env")
	err = env.SetMaxDBs(26)
	bMust(b, err, "setting max dbs")
	err = env.SetMapSize(1 << 30) // 1GB
	bMust(b, err, "sizing env")
	path, err := ioutil.TempDir("", "mdb_test-bench-")
	bMust(b, err, "creating temp directory")
	err = env.Open(path, flags, 0644)
	if err != nil {
		teardownBenchDB(b, env, path)
	}
	bMust(b, err, "opening database")
	return env, path
}

func openBenchDBI(b *testing.B, env *Env) DBI {
	txn, err := env.BeginTxn(nil, 0)
	bMust(b, err, "starting transaction")
	dbi, err := txn.OpenDBI("benchmark", Create)
	if err != nil {
		txn.Abort()
		b.Fatalf("error opening dbi: %v", err)
	}
	err = txn.Commit()
	bMust(b, err, "commiting transaction")
	return dbi
}

func teardownBenchDB(b *testing.B, env *Env, path string) {
	env.Close()
	os.RemoveAll(path)
}

func randBytes(n int) []byte {
	p := make([]byte, n)
	crand.Read(p)
	return p
}

func bMust(b *testing.B, err error, action string) {
	if err != nil {
		b.Fatalf("error %s: %v", action, err)
	}
}

func bTxnMust(b *testing.B, txn *Txn, err error, action string) {
	if err != nil {
		txn.Abort()
		b.Fatalf("error %s: %v", action, err)
	}
}

const randSourceSize = 10 << 20 // size of the 'entropy pool' for random byte generation.
const benchDBNumKeys = 100000   // number of keys to store in benchmark databases
const benchDBMaxKeyLen = 30     // maximum length for database keys (size is limited by MDB)
const benchDBMaxValLen = 2000   // maximum lengh for database values

func makeBenchDBKey(c *randSourceCursor) []byte {
	return c.NBytes(rand.Intn(benchDBMaxKeyLen) + 1)
}

func makeBenchDBVal(c *randSourceCursor) []byte {
	return c.NBytes(rand.Intn(benchDBMaxValLen) + 1)
}

// holds a bunch of random bytes so repeated generation of 'random' slices is
// cheap.  acts as a ring which can be read from (although doesn't implement io.Reader).
var _initRand int32
var randSource [randSourceSize]byte

func initRandSource(b *testing.B) {
	if atomic.AddInt32(&_initRand, 1) > 1 {
		return
	}
	b.Logf("initializing random source data")
	n, err := crand.Read(randSource[:])
	bMust(b, err, "initializing random source")
	if n < len(randSource) {
		b.Fatalf("unable to read enough random source data %d", n)
	}
}

// acts as a simple byte slice generator.
type randSourceCursor int

func newRandSourceCursor() randSourceCursor {
	i := rand.Intn(randSourceSize)
	return randSourceCursor(i)
}

func (c *randSourceCursor) NBytes(n int) []byte {
	i := int(*c)
	if n >= randSourceSize {
		panic("rand size too big")
	}
	*c = (*c + randSourceCursor(n)) % randSourceSize
	_n := i + n - randSourceSize
	if _n > 0 {
		p := make([]byte, n)
		m := copy(p, randSource[i:])
		copy(p[m:], randSource[:])
		return p
	}
	return randSource[i : i+n]
}
<commit_msg>clean up benchmarks<commit_after>package lmdb

import (
	crand "crypto/rand"
	"math/rand"
	"os"
	"sync/atomic"
	"testing"
)

// repeatedly put (overwrite) keys.
func BenchmarkTxn_Put(b *testing.B) {
	initRandSource(b)
	env := setup(b)
	defer clean(env, b)

	dbi := openBenchDBI(b, env)

	rc := newRandSourceCursor()
	ps, err := populateBenchmarkDB(env, dbi, &rc)
	if err != nil {
		b.Errorf("populate db: %v", err)
		return
	}

	err = env.Update(func(txn *Txn) (err error) {
		b.ResetTimer()
		defer b.StopTimer()
		for i := 0; i < b.N; i++ {
			k := ps[rand.Intn(len(ps)/2)*2]
			v := makeBenchDBVal(&rc)
			err := txn.Put(dbi, k, v, 0)
			if err != nil {
				return err
			}
		}
		return nil
	})
	if err != nil {
		b.Error(err)
		return
	}
}

// repeatedly put (overwrite) keys using the PutReserve method.
func BenchmarkTxn_PutReserve(b *testing.B) {
	initRandSource(b)
	env := setup(b)
	defer clean(env, b)

	dbi := openBenchDBI(b, env)

	rc := newRandSourceCursor()
	ps, err := populateBenchmarkDB(env, dbi, &rc)
	if err != nil {
		b.Errorf("populate db: %v", err)
		return
	}

	err = env.Update(func(txn *Txn) (err error) {
		b.ResetTimer()
		defer b.StopTimer()
		for i := 0; i < b.N; i++ {
			k := ps[rand.Intn(len(ps)/2)*2]
			v := makeBenchDBVal(&rc)
			buf, err := txn.PutReserve(dbi, k, len(v), 0)
			if err != nil {
				return err
			}
			copy(buf, v)
		}
		return nil
	})
	if err != nil {
		b.Error(err)
		return
	}
}

// repeatedly put (overwrite) keys using the PutReserve method on an
// environment with WriteMap.
func BenchmarkTxn_PutReserve_writemap(b *testing.B) {
	initRandSource(b)
	env := setupFlags(b, WriteMap)
	defer clean(env, b)

	dbi := openBenchDBI(b, env)

	rc := newRandSourceCursor()
	ps, err := populateBenchmarkDB(env, dbi, &rc)
	if err != nil {
		b.Errorf("populate db: %v", err)
		return
	}

	err = env.Update(func(txn *Txn) (err error) {
		b.ResetTimer()
		defer b.StopTimer()
		for i := 0; i < b.N; i++ {
			k := ps[rand.Intn(len(ps)/2)*2]
			v := makeBenchDBVal(&rc)
			buf, err := txn.PutReserve(dbi, k, len(v), 0)
			if err != nil {
				return err
			}
			copy(buf, v)
		}
		return nil
	})
	if err != nil {
		b.Error(err)
		return
	}
}

// repeatedly put (overwrite) keys.
func BenchmarkTxn_Put_writemap(b *testing.B) {
	initRandSource(b)
	env := setupFlags(b, WriteMap)
	defer clean(env, b)

	dbi := openBenchDBI(b, env)

	var ps [][]byte

	rc := newRandSourceCursor()
	ps, err := populateBenchmarkDB(env, dbi, &rc)
	if err != nil {
		b.Errorf("populate db: %v", err)
		return
	}

	err = env.Update(func(txn *Txn) (err error) {
		b.ResetTimer()
		defer b.StopTimer()
		for i := 0; i < b.N; i++ {
			k := ps[rand.Intn(len(ps)/2)*2]
			v := makeBenchDBVal(&rc)
			err := txn.Put(dbi, k, v, 0)
			if err != nil {
				return err
			}
		}

		return nil
	})
	if err != nil {
		b.Error(err)
	}
}

// repeatedly get random keys.
func BenchmarkTxn_Get_ro(b *testing.B) {
	initRandSource(b)
	env := setup(b)
	defer clean(env, b)

	dbi := openBenchDBI(b, env)

	rc := newRandSourceCursor()
	ps, err := populateBenchmarkDB(env, dbi, &rc)
	if err != nil {
		b.Errorf("populate db: %v", err)
		return
	}

	err = env.View(func(txn *Txn) (err error) {
		b.ResetTimer()
		defer b.StopTimer()
		for i := 0; i < b.N; i++ {
			_, err := txn.Get(dbi, ps[rand.Intn(len(ps))])
			if IsNotFound(err) {
				continue
			}
			if err != nil {
				b.Fatalf("error getting data: %v", err)
			}
		}

		return nil
	})
	if err != nil {
		b.Error(err)
	}
}

// like BenchmarkTxnGetReadonly but txn.RawRead is set to true.
func BenchmarkTxn_Get_raw_ro(b *testing.B) {
	initRandSource(b)
	env := setup(b)
	defer clean(env, b)

	dbi := openBenchDBI(b, env)

	rc := newRandSourceCursor()
	ps, err := populateBenchmarkDB(env, dbi, &rc)
	if err != nil {
		b.Errorf("populate db: %v", err)
		return
	}

	err = env.View(func(txn *Txn) (err error) {
		txn.RawRead = true
		b.ResetTimer()
		defer b.StopTimer()
		for i := 0; i < b.N; i++ {
			_, err := txn.Get(dbi, ps[rand.Intn(len(ps))])
			if IsNotFound(err) {
				continue
			}
			if err != nil {
				b.Fatalf("error getting data: %v", err)
			}
		}
		return nil
	})
	if err != nil {
		b.Error(err)
		return
	}
}

// repeatedly scan all the values in a database.
func BenchmarkScan_ro(b *testing.B) {
	initRandSource(b)
	env := setup(b)
	defer clean(env, b)

	dbi := openBenchDBI(b, env)

	rc := newRandSourceCursor()
	_, err := populateBenchmarkDB(env, dbi, &rc)
	if err != nil {
		b.Errorf("populate db: %v", err)
		return
	}

	err = env.View(func(txn *Txn) (err error) {
		b.ResetTimer()
		defer b.StopTimer()
		for i := 0; i < b.N; i++ {
			err := benchmarkScanDBI(txn, dbi)
			if err != nil {
				return err
			}
		}

		return nil
	})
	if err != nil {
		b.Error(err)
		return
	}
}

// like BenchmarkCursoreScanReadonly but txn.RawRead is set to true.
func BenchmarkScan_raw_ro(b *testing.B) {
	initRandSource(b)
	env := setup(b)
	defer clean(env, b)

	dbi := openBenchDBI(b, env)

	rc := newRandSourceCursor()
	_, err := populateBenchmarkDB(env, dbi, &rc)
	if err != nil {
		b.Errorf("populate db: %v", err)
		return
	}

	err = env.View(func(txn *Txn) (err error) {
		txn.RawRead = true

		b.ResetTimer()
		defer b.StopTimer()
		for i := 0; i < b.N; i++ {
			err := benchmarkScanDBI(txn, dbi)
			if err != nil {
				return err
			}
		}

		return nil
	})
	if err != nil {
		b.Errorf("benchmark: %v", err)
		return
	}
}

// populateBenchmarkDB fills env with data.
//
// populateBenchmarkDB calls env.SetMapSize and must not be called concurrent
// with other transactions.
func populateBenchmarkDB(env *Env, dbi DBI, rc *randSourceCursor) ([][]byte, error) {
	var ps [][]byte

	err := env.SetMapSize(benchDBMapSize)
	if err != nil {
		return nil, err
	}

	err = env.Update(func(txn *Txn) (err error) {
		for i := 0; i < benchDBNumKeys; i++ {
			k := makeBenchDBKey(rc)
			v := makeBenchDBVal(rc)
			err := txn.Put(dbi, k, v, 0)
			ps = append(ps, k, v)
			if err != nil {
				return err
			}
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	return ps, nil
}

func benchmarkScanDBI(txn *Txn, dbi DBI) error {
	cur, err := txn.OpenCursor(dbi)
	if err != nil {
		return err
	}
	defer cur.Close()

	var count int64
	for {
		_, _, err := cur.Get(nil, nil, Next)
		if IsNotFound(err) {
			return nil
		}
		if err != nil {
			return err
		}
		count++
	}
}

func openBenchDBI(b *testing.B, env *Env) DBI {
	var dbi DBI
	err := env.Update(func(txn *Txn) (err error) {
		dbi, err = txn.OpenDBI("benchmark", Create)
		return err
	})
	if err != nil {
		b.Errorf("unable to open benchmark database")
	}
	return dbi
}

func teardownBenchDB(b *testing.B, env *Env, path string) {
	env.Close()
	os.RemoveAll(path)
}

func randBytes(n int) []byte {
	p := make([]byte, n)
	crand.Read(p)
	return p
}

func bMust(b *testing.B, err error, action string) {
	if err != nil {
		b.Fatalf("error %s: %v", action, err)
	}
}

const randSourceSize = 10 << 20  // size of the 'entropy pool' for random byte generation.
const benchDBMapSize = 100 << 20 // size of a benchmark db memory map
const benchDBNumKeys = 1 << 12   // number of keys to store in benchmark databases
const benchDBMaxKeyLen = 30      // maximum length for database keys (size is limited by MDB)
const benchDBMaxValLen = 4096    // maximum lengh for database values

func makeBenchDBKey(c *randSourceCursor) []byte {
	return c.NBytes(rand.Intn(benchDBMaxKeyLen) + 1)
}

func makeBenchDBVal(c *randSourceCursor) []byte {
	return c.NBytes(rand.Intn(benchDBMaxValLen) + 1)
}

// holds a bunch of random bytes so repeated generation of 'random' slices is
// cheap.  acts as a ring which can be read from (although doesn't implement io.Reader).
var _initRand int32
var randSource [randSourceSize]byte

func initRandSource(b *testing.B) {
	if atomic.AddInt32(&_initRand, 1) > 1 {
		return
	}
	b.Logf("initializing random source data")
	n, err := crand.Read(randSource[:])
	bMust(b, err, "initializing random source")
	if n < len(randSource) {
		b.Fatalf("unable to read enough random source data %d", n)
	}
}

// acts as a simple byte slice generator.
type randSourceCursor int

func newRandSourceCursor() randSourceCursor {
	i := rand.Intn(randSourceSize)
	return randSourceCursor(i)
}

func (c *randSourceCursor) NBytes(n int) []byte {
	i := int(*c)
	if n >= randSourceSize {
		panic("rand size too big")
	}
	*c = (*c + randSourceCursor(n)) % randSourceSize
	_n := i + n - randSourceSize
	if _n > 0 {
		p := make([]byte, n)
		m := copy(p, randSource[i:])
		copy(p[m:], randSource[:])
		return p
	}
	return randSource[i : i+n]
}
<|endoftext|>


<commit_before>package logger

import (
	"github.com/revel/log15"
	"gopkg.in/natefinch/lumberjack.v2"
	"io"
)

// Filters out records which do not match the level
// Uses the `log15.FilterHandler` to perform this task
func LevelHandler(lvl LogLevel, h LogHandler) LogHandler {
	l15Lvl := log15.Lvl(lvl)
	return log15.FilterHandler(func(r *log15.Record) (pass bool) {
		return r.Lvl == l15Lvl
	}, h)
}

// Filters out records which do not match the level
// Uses the `log15.FilterHandler` to perform this task
func MinLevelHandler(lvl LogLevel, h LogHandler) LogHandler {
	l15Lvl := log15.Lvl(lvl)
	return log15.FilterHandler(func(r *log15.Record) (pass bool) {
		return r.Lvl <= l15Lvl
	}, h)
}

// Filters out records which match the level
// Uses the `log15.FilterHandler` to perform this task
func NotLevelHandler(lvl LogLevel, h LogHandler) LogHandler {
	l15Lvl := log15.Lvl(lvl)
	return log15.FilterHandler(func(r *log15.Record) (pass bool) {
		return r.Lvl != l15Lvl
	}, h)
}

// Adds in a context called `caller` to the record (contains file name and line number like `foo.go:12`)
// Uses the `log15.CallerFileHandler` to perform this task
func CallerFileHandler(h LogHandler) LogHandler {
	return log15.CallerFileHandler(h)
}

// Adds in a context called `caller` to the record (contains file name and line number like `foo.go:12`)
// Uses the `log15.CallerFuncHandler` to perform this task
func CallerFuncHandler(h LogHandler) LogHandler {
	return log15.CallerFuncHandler(h)
}

// Filters out records which match the key value pair
// Uses the `log15.MatchFilterHandler` to perform this task
func MatchHandler(key string, value interface{}, h LogHandler) LogHandler {
	return log15.MatchFilterHandler(key, value, h)
}

// If match then A handler is called otherwise B handler is called
func MatchAbHandler(key string, value interface{}, a, b LogHandler) LogHandler {
	return log15.FuncHandler(func(r *log15.Record) error {
		for i := 0; i < len(r.Ctx); i += 2 {
			if r.Ctx[i] == key {
				if r.Ctx[i+1] == value {
					if a != nil {
						return a.Log(r)
					}
					return nil
				}
			}
		}
		if b != nil {
			return b.Log(r)
		}
		return nil
	})
}

// The nil handler is used if logging for a specific request needs to be turned off
func NilHandler() LogHandler {
	return log15.FuncHandler(func(r *log15.Record) error {
		return nil
	})
}

// Match all values in map to log
func MatchMapHandler(matchMap map[string]interface{}, a LogHandler) LogHandler {
	return matchMapHandler(matchMap, false, a)
}

// Match !(Match all values in map to log) The inverse of MatchMapHandler
func NotMatchMapHandler(matchMap map[string]interface{}, a LogHandler) LogHandler {
	return matchMapHandler(matchMap, true, a)
}

// Rather then chaining multiple filter handlers, process all here
func matchMapHandler(matchMap map[string]interface{}, inverse bool, a LogHandler) LogHandler {
	return log15.FuncHandler(func(r *log15.Record) error {
		checkMap := map[string]bool{}
		// Copy the map to a bool
		for i := 0; i < len(r.Ctx); i += 2 {
			if value, found := matchMap[r.Ctx[i].(string)]; found && value == r.Ctx[i+1] {
				checkMap[r.Ctx[i].(string)] = true
			}
		}
		if len(checkMap) == len(matchMap) {
			if !inverse {
				return a.Log(r)
			}
		} else if inverse {
			return a.Log(r)
		}
		return nil
	})
}

// Filters out records which do not match the key value pair
// Uses the `log15.FilterHandler` to perform this task
func NotMatchHandler(key string, value interface{}, h LogHandler) LogHandler {
	return log15.FilterHandler(func(r *log15.Record) (pass bool) {
		switch key {
		case r.KeyNames.Lvl:
			return r.Lvl != value
		case r.KeyNames.Time:
			return r.Time != value
		case r.KeyNames.Msg:
			return r.Msg != value
		}

		for i := 0; i < len(r.Ctx); i += 2 {
			if r.Ctx[i] == key {
				return r.Ctx[i+1] == value
			}
		}
		return true
	}, h)
}

func MultiHandler(hs ...LogHandler) LogHandler {
	// Convert the log handlers to log15.Handlers
	handlers := []log15.Handler{}
	for _, h := range hs {
		if h != nil {
			handlers = append(handlers, h)
		}
	}

	return log15.MultiHandler(handlers...)
}

// Outputs the records to the passed in stream
// Uses the `log15.StreamHandler` to perform this task
func StreamHandler(wr io.Writer, fmtr LogFormat) LogHandler {
	return log15.StreamHandler(wr, fmtr)
}

// Filter handler, this is the only
// Uses the `log15.FilterHandler` to perform this task
func FilterHandler(fn func(r *log15.Record) bool, h LogHandler) LogHandler {
	return log15.FilterHandler(fn, h)
}

type ListLogHandler struct {
	handlers []LogHandler
}

func NewListLogHandler(h1, h2 LogHandler) *ListLogHandler {
	ll := &ListLogHandler{handlers: []LogHandler{h1, h2}}
	return ll
}
func (ll *ListLogHandler) Log(r *log15.Record) (err error) {
	for _, handler := range ll.handlers {
		if err == nil {
			err = handler.Log(r)
		} else {
			handler.Log(r)
		}
	}
	return
}
func (ll *ListLogHandler) Add(h LogHandler) {
	if h != nil {
		ll.handlers = append(ll.handlers, h)
	}
}
func (ll *ListLogHandler) Del(h LogHandler) {
	if h != nil {
		for i, handler := range ll.handlers {
			if handler == h {
				ll.handlers = append(ll.handlers[:i], ll.handlers[i+1:]...)
			}
		}
	}
}

type CompositeMultiHandler struct {
	DebugHandler    LogHandler
	InfoHandler     LogHandler
	WarnHandler     LogHandler
	ErrorHandler    LogHandler
	CriticalHandler LogHandler
}

func NewCompositeMultiHandler() (*CompositeMultiHandler, LogHandler) {
	cw := &CompositeMultiHandler{}
	return cw, cw
}
func (h *CompositeMultiHandler) Log(r *log15.Record) (err error) {

	var handler LogHandler
	switch r.Lvl {
	case log15.LvlInfo:
		handler = h.InfoHandler
	case log15.LvlDebug:
		handler = h.DebugHandler
	case log15.LvlWarn:
		handler = h.WarnHandler
	case log15.LvlError:
		handler = h.ErrorHandler
	case log15.LvlCrit:
		handler = h.CriticalHandler
	}

	// Embed the caller function in the context
	if handler != nil {
		handler.Log(r)
	}
	return
}
func (h *CompositeMultiHandler) SetHandler(handler LogHandler, replace bool, level LogLevel) {
	if handler == nil {
		// Ignore empty handler
		return
	}
	source := &h.DebugHandler
	switch level {
	case LvlDebug:
		source = &h.DebugHandler
	case LvlInfo:
		source = &h.InfoHandler
	case LvlWarn:
		source = &h.WarnHandler
	case LvlError:
		source = &h.ErrorHandler
	case LvlCrit:
		source = &h.CriticalHandler
	}

	if !replace && *source != nil {
		// If this already was a list add a new logger to it
		if ll, found := (*source).(*ListLogHandler); found {
			ll.Add(handler)
		} else {
			*source = NewListLogHandler(*source, handler)
		}
	} else {
		*source = handler
	}
}

func (h *CompositeMultiHandler) SetHandlers(handler LogHandler, options *LogOptions) {
	if len(options.Levels) == 0 {
		options.Levels = LvlAllList
	}
	// Set all levels
	for _, lvl := range options.Levels {
		h.SetHandler(handler, options.ReplaceExistingHandler, lvl)
	}

}
func (h *CompositeMultiHandler) SetJson(writer io.Writer, options *LogOptions) {
	handler := CallerFileHandler(StreamHandler(writer, log15.JsonFormatEx(
		options.GetBoolDefault("pretty", false),
		options.GetBoolDefault("lineSeparated", true),
	)))
	if options.HandlerWrap != nil {
		handler = options.HandlerWrap.SetChild(handler)
	}
	h.SetHandlers(handler, options)
}

// Use built in rolling function
func (h *CompositeMultiHandler) SetJsonFile(filePath string, options *LogOptions) {
	writer := &lumberjack.Logger{
		Filename:   filePath,
		MaxSize:    options.GetIntDefault("maxSizeMB", 1024), // megabytes
		MaxAge:     options.GetIntDefault("maxAgeDays", 7),   //days
		MaxBackups: options.GetIntDefault("maxBackups", 7),
		Compress:   options.GetBoolDefault("compress", true),
	}
	h.SetJson(writer, options)
}

func (h *CompositeMultiHandler) SetTerminal(writer io.Writer, options *LogOptions) {
	streamHandler := StreamHandler(
		writer,
		TerminalFormatHandler(
			options.GetBoolDefault("noColor", false),
			options.GetBoolDefault("smallDate", true)))

	if os.Stdout == writer {
		streamHandler = StreamHandler(
			colorable.NewColorableStdout(),
			TerminalFormatHandler(
				options.GetBoolDefault("noColor", false),
				options.GetBoolDefault("smallDate", true)))
	} else if os.Stderr == writer {
		streamHandler = StreamHandler(
			colorable.NewColorableStderr(),
			TerminalFormatHandler(
				options.GetBoolDefault("noColor", false),
				options.GetBoolDefault("smallDate", true)))
	}

	handler := CallerFileHandler(streamHandler)
	if options.HandlerWrap != nil {
		handler = options.HandlerWrap.SetChild(handler)
	}
	h.SetHandlers(handler, options)
}

// Use built in rolling function
func (h *CompositeMultiHandler) SetTerminalFile(filePath string, options *LogOptions) {
	writer := &lumberjack.Logger{
		Filename:   filePath,
		MaxSize:    options.GetIntDefault("maxSizeMB", 1024), // megabytes
		MaxAge:     options.GetIntDefault("maxAgeDays", 7),   //days
		MaxBackups: options.GetIntDefault("maxBackups", 7),
		Compress:   options.GetBoolDefault("compress", true),
	}
	h.SetTerminal(writer, options)
}

func (h *CompositeMultiHandler) Disable(levels ...LogLevel) {
	if len(levels) == 0 {
		levels = LvlAllList
	}
	for _, level := range levels {
		switch level {
		case LvlDebug:
			h.DebugHandler = nil
		case LvlInfo:
			h.InfoHandler = nil
		case LvlWarn:
			h.WarnHandler = nil
		case LvlError:
			h.ErrorHandler = nil
		case LvlCrit:
			h.CriticalHandler = nil
		}
	}
}
<commit_msg>fix import errors<commit_after>package logger

import (
	"io"
	"os"

	colorable "github.com/mattn/go-colorable"
	"github.com/revel/log15"
	"gopkg.in/natefinch/lumberjack.v2"
)

// Filters out records which do not match the level
// Uses the `log15.FilterHandler` to perform this task
func LevelHandler(lvl LogLevel, h LogHandler) LogHandler {
	l15Lvl := log15.Lvl(lvl)
	return log15.FilterHandler(func(r *log15.Record) (pass bool) {
		return r.Lvl == l15Lvl
	}, h)
}

// Filters out records which do not match the level
// Uses the `log15.FilterHandler` to perform this task
func MinLevelHandler(lvl LogLevel, h LogHandler) LogHandler {
	l15Lvl := log15.Lvl(lvl)
	return log15.FilterHandler(func(r *log15.Record) (pass bool) {
		return r.Lvl <= l15Lvl
	}, h)
}

// Filters out records which match the level
// Uses the `log15.FilterHandler` to perform this task
func NotLevelHandler(lvl LogLevel, h LogHandler) LogHandler {
	l15Lvl := log15.Lvl(lvl)
	return log15.FilterHandler(func(r *log15.Record) (pass bool) {
		return r.Lvl != l15Lvl
	}, h)
}

// Adds in a context called `caller` to the record (contains file name and line number like `foo.go:12`)
// Uses the `log15.CallerFileHandler` to perform this task
func CallerFileHandler(h LogHandler) LogHandler {
	return log15.CallerFileHandler(h)
}

// Adds in a context called `caller` to the record (contains file name and line number like `foo.go:12`)
// Uses the `log15.CallerFuncHandler` to perform this task
func CallerFuncHandler(h LogHandler) LogHandler {
	return log15.CallerFuncHandler(h)
}

// Filters out records which match the key value pair
// Uses the `log15.MatchFilterHandler` to perform this task
func MatchHandler(key string, value interface{}, h LogHandler) LogHandler {
	return log15.MatchFilterHandler(key, value, h)
}

// If match then A handler is called otherwise B handler is called
func MatchAbHandler(key string, value interface{}, a, b LogHandler) LogHandler {
	return log15.FuncHandler(func(r *log15.Record) error {
		for i := 0; i < len(r.Ctx); i += 2 {
			if r.Ctx[i] == key {
				if r.Ctx[i+1] == value {
					if a != nil {
						return a.Log(r)
					}
					return nil
				}
			}
		}
		if b != nil {
			return b.Log(r)
		}
		return nil
	})
}

// The nil handler is used if logging for a specific request needs to be turned off
func NilHandler() LogHandler {
	return log15.FuncHandler(func(r *log15.Record) error {
		return nil
	})
}

// Match all values in map to log
func MatchMapHandler(matchMap map[string]interface{}, a LogHandler) LogHandler {
	return matchMapHandler(matchMap, false, a)
}

// Match !(Match all values in map to log) The inverse of MatchMapHandler
func NotMatchMapHandler(matchMap map[string]interface{}, a LogHandler) LogHandler {
	return matchMapHandler(matchMap, true, a)
}

// Rather then chaining multiple filter handlers, process all here
func matchMapHandler(matchMap map[string]interface{}, inverse bool, a LogHandler) LogHandler {
	return log15.FuncHandler(func(r *log15.Record) error {
		checkMap := map[string]bool{}
		// Copy the map to a bool
		for i := 0; i < len(r.Ctx); i += 2 {
			if value, found := matchMap[r.Ctx[i].(string)]; found && value == r.Ctx[i+1] {
				checkMap[r.Ctx[i].(string)] = true
			}
		}
		if len(checkMap) == len(matchMap) {
			if !inverse {
				return a.Log(r)
			}
		} else if inverse {
			return a.Log(r)
		}
		return nil
	})
}

// Filters out records which do not match the key value pair
// Uses the `log15.FilterHandler` to perform this task
func NotMatchHandler(key string, value interface{}, h LogHandler) LogHandler {
	return log15.FilterHandler(func(r *log15.Record) (pass bool) {
		switch key {
		case r.KeyNames.Lvl:
			return r.Lvl != value
		case r.KeyNames.Time:
			return r.Time != value
		case r.KeyNames.Msg:
			return r.Msg != value
		}

		for i := 0; i < len(r.Ctx); i += 2 {
			if r.Ctx[i] == key {
				return r.Ctx[i+1] == value
			}
		}
		return true
	}, h)
}

func MultiHandler(hs ...LogHandler) LogHandler {
	// Convert the log handlers to log15.Handlers
	handlers := []log15.Handler{}
	for _, h := range hs {
		if h != nil {
			handlers = append(handlers, h)
		}
	}

	return log15.MultiHandler(handlers...)
}

// Outputs the records to the passed in stream
// Uses the `log15.StreamHandler` to perform this task
func StreamHandler(wr io.Writer, fmtr LogFormat) LogHandler {
	return log15.StreamHandler(wr, fmtr)
}

// Filter handler, this is the only
// Uses the `log15.FilterHandler` to perform this task
func FilterHandler(fn func(r *log15.Record) bool, h LogHandler) LogHandler {
	return log15.FilterHandler(fn, h)
}

type ListLogHandler struct {
	handlers []LogHandler
}

func NewListLogHandler(h1, h2 LogHandler) *ListLogHandler {
	ll := &ListLogHandler{handlers: []LogHandler{h1, h2}}
	return ll
}
func (ll *ListLogHandler) Log(r *log15.Record) (err error) {
	for _, handler := range ll.handlers {
		if err == nil {
			err = handler.Log(r)
		} else {
			handler.Log(r)
		}
	}
	return
}
func (ll *ListLogHandler) Add(h LogHandler) {
	if h != nil {
		ll.handlers = append(ll.handlers, h)
	}
}
func (ll *ListLogHandler) Del(h LogHandler) {
	if h != nil {
		for i, handler := range ll.handlers {
			if handler == h {
				ll.handlers = append(ll.handlers[:i], ll.handlers[i+1:]...)
			}
		}
	}
}

type CompositeMultiHandler struct {
	DebugHandler    LogHandler
	InfoHandler     LogHandler
	WarnHandler     LogHandler
	ErrorHandler    LogHandler
	CriticalHandler LogHandler
}

func NewCompositeMultiHandler() (*CompositeMultiHandler, LogHandler) {
	cw := &CompositeMultiHandler{}
	return cw, cw
}
func (h *CompositeMultiHandler) Log(r *log15.Record) (err error) {

	var handler LogHandler
	switch r.Lvl {
	case log15.LvlInfo:
		handler = h.InfoHandler
	case log15.LvlDebug:
		handler = h.DebugHandler
	case log15.LvlWarn:
		handler = h.WarnHandler
	case log15.LvlError:
		handler = h.ErrorHandler
	case log15.LvlCrit:
		handler = h.CriticalHandler
	}

	// Embed the caller function in the context
	if handler != nil {
		handler.Log(r)
	}
	return
}
func (h *CompositeMultiHandler) SetHandler(handler LogHandler, replace bool, level LogLevel) {
	if handler == nil {
		// Ignore empty handler
		return
	}
	source := &h.DebugHandler
	switch level {
	case LvlDebug:
		source = &h.DebugHandler
	case LvlInfo:
		source = &h.InfoHandler
	case LvlWarn:
		source = &h.WarnHandler
	case LvlError:
		source = &h.ErrorHandler
	case LvlCrit:
		source = &h.CriticalHandler
	}

	if !replace && *source != nil {
		// If this already was a list add a new logger to it
		if ll, found := (*source).(*ListLogHandler); found {
			ll.Add(handler)
		} else {
			*source = NewListLogHandler(*source, handler)
		}
	} else {
		*source = handler
	}
}

func (h *CompositeMultiHandler) SetHandlers(handler LogHandler, options *LogOptions) {
	if len(options.Levels) == 0 {
		options.Levels = LvlAllList
	}
	// Set all levels
	for _, lvl := range options.Levels {
		h.SetHandler(handler, options.ReplaceExistingHandler, lvl)
	}

}
func (h *CompositeMultiHandler) SetJson(writer io.Writer, options *LogOptions) {
	handler := CallerFileHandler(StreamHandler(writer, log15.JsonFormatEx(
		options.GetBoolDefault("pretty", false),
		options.GetBoolDefault("lineSeparated", true),
	)))
	if options.HandlerWrap != nil {
		handler = options.HandlerWrap.SetChild(handler)
	}
	h.SetHandlers(handler, options)
}

// Use built in rolling function
func (h *CompositeMultiHandler) SetJsonFile(filePath string, options *LogOptions) {
	writer := &lumberjack.Logger{
		Filename:   filePath,
		MaxSize:    options.GetIntDefault("maxSizeMB", 1024), // megabytes
		MaxAge:     options.GetIntDefault("maxAgeDays", 7),   //days
		MaxBackups: options.GetIntDefault("maxBackups", 7),
		Compress:   options.GetBoolDefault("compress", true),
	}
	h.SetJson(writer, options)
}

func (h *CompositeMultiHandler) SetTerminal(writer io.Writer, options *LogOptions) {
	streamHandler := StreamHandler(
		writer,
		TerminalFormatHandler(
			options.GetBoolDefault("noColor", false),
			options.GetBoolDefault("smallDate", true)))

	if os.Stdout == writer {
		streamHandler = StreamHandler(
			colorable.NewColorableStdout(),
			TerminalFormatHandler(
				options.GetBoolDefault("noColor", false),
				options.GetBoolDefault("smallDate", true)))
	} else if os.Stderr == writer {
		streamHandler = StreamHandler(
			colorable.NewColorableStderr(),
			TerminalFormatHandler(
				options.GetBoolDefault("noColor", false),
				options.GetBoolDefault("smallDate", true)))
	}

	handler := CallerFileHandler(streamHandler)
	if options.HandlerWrap != nil {
		handler = options.HandlerWrap.SetChild(handler)
	}
	h.SetHandlers(handler, options)
}

// Use built in rolling function
func (h *CompositeMultiHandler) SetTerminalFile(filePath string, options *LogOptions) {
	writer := &lumberjack.Logger{
		Filename:   filePath,
		MaxSize:    options.GetIntDefault("maxSizeMB", 1024), // megabytes
		MaxAge:     options.GetIntDefault("maxAgeDays", 7),   //days
		MaxBackups: options.GetIntDefault("maxBackups", 7),
		Compress:   options.GetBoolDefault("compress", true),
	}
	h.SetTerminal(writer, options)
}

func (h *CompositeMultiHandler) Disable(levels ...LogLevel) {
	if len(levels) == 0 {
		levels = LvlAllList
	}
	for _, level := range levels {
		switch level {
		case LvlDebug:
			h.DebugHandler = nil
		case LvlInfo:
			h.InfoHandler = nil
		case LvlWarn:
			h.WarnHandler = nil
		case LvlError:
			h.ErrorHandler = nil
		case LvlCrit:
			h.CriticalHandler = nil
		}
	}
}
<|endoftext|>


<commit_before>package consul

import (
	"fmt"
	"log"
	"runtime"
	"sort"
	"strings"
	"time"

	"github.com/eBay/fabio/_third_party/github.com/hashicorp/consul/api"
)

// watchAutoConfig monitors the consul health checks and creates a new configuration
// on every change.
func watchAutoConfig(client *api.Client, tagPrefix string, config chan []string) {
	var lastIndex uint64

	for {
		q := &api.QueryOptions{RequireConsistent: true, WaitIndex: lastIndex}
		checks, meta, err := client.Health().State("any", q)
		if err != nil {
			log.Printf("[WARN] Error fetching health state. %v", err)
			time.Sleep(time.Second)
			continue
		}

		log.Printf("[INFO] Health changed to #%d", meta.LastIndex)
		config <- servicesConfig(client, passingServices(checks), tagPrefix)
		lastIndex = meta.LastIndex
	}
}

// servicesConfig determines which service instances have passing health checks
// and then finds the ones which have tags with the right prefix to build the config from.
func servicesConfig(client *api.Client, checks []*api.HealthCheck, tagPrefix string) []string {
	// map service name to list of service passing for which the health check is ok
	m := map[string]map[string]bool{}
	for _, check := range checks {
		name, id := check.ServiceName, check.ServiceID

		if _, ok := m[name]; !ok {
			m[name] = map[string]bool{}
		}
		m[name][id] = true
	}

	var config []string
	for name, passing := range m {
		cfg := serviceConfig(client, name, passing, tagPrefix)
		config = append(config, cfg...)
	}

	// sort config in reverse order to sort most specific config to the top
	sort.Sort(sort.Reverse(sort.StringSlice(config)))

	return config
}

// serviceConfig constructs the config for all good instances of a single service.
func serviceConfig(client *api.Client, name string, passing map[string]bool, tagPrefix string) (config []string) {
	if name == "" || len(passing) == 0 {
		return nil
	}

	q := &api.QueryOptions{RequireConsistent: true}
	svcs, _, err := client.Catalog().Service(name, "", q)
	if err != nil {
		log.Printf("[WARN] [%s] Error getting catalog service %s. %v", name, err)
		return nil
	}

	for _, svc := range svcs {
		// check if the instance is in the list of instances
		// which passed the health check
		if _, ok := passing[svc.ServiceID]; !ok {
			continue
		}

		for _, tag := range svc.ServiceTags {
			if host, path, ok := parseURLPrefixTag(tag, tagPrefix); ok {
				name, addr, port := svc.ServiceName, svc.ServiceAddress, svc.ServicePort
				if runtime.GOOS == "darwin" && !strings.Contains(addr, ".") {
					addr += ".local"
				}
				config = append(config, fmt.Sprintf("route add %s %s%s http://%s:%d/ tags %q", name, host, path, addr, port, strings.Join(svc.ServiceTags, ",")))
			}
		}
	}
	return config
}
<commit_msg>fix vet warning<commit_after>package consul

import (
	"fmt"
	"log"
	"runtime"
	"sort"
	"strings"
	"time"

	"github.com/eBay/fabio/_third_party/github.com/hashicorp/consul/api"
)

// watchAutoConfig monitors the consul health checks and creates a new configuration
// on every change.
func watchAutoConfig(client *api.Client, tagPrefix string, config chan []string) {
	var lastIndex uint64

	for {
		q := &api.QueryOptions{RequireConsistent: true, WaitIndex: lastIndex}
		checks, meta, err := client.Health().State("any", q)
		if err != nil {
			log.Printf("[WARN] Error fetching health state. %v", err)
			time.Sleep(time.Second)
			continue
		}

		log.Printf("[INFO] Health changed to #%d", meta.LastIndex)
		config <- servicesConfig(client, passingServices(checks), tagPrefix)
		lastIndex = meta.LastIndex
	}
}

// servicesConfig determines which service instances have passing health checks
// and then finds the ones which have tags with the right prefix to build the config from.
func servicesConfig(client *api.Client, checks []*api.HealthCheck, tagPrefix string) []string {
	// map service name to list of service passing for which the health check is ok
	m := map[string]map[string]bool{}
	for _, check := range checks {
		name, id := check.ServiceName, check.ServiceID

		if _, ok := m[name]; !ok {
			m[name] = map[string]bool{}
		}
		m[name][id] = true
	}

	var config []string
	for name, passing := range m {
		cfg := serviceConfig(client, name, passing, tagPrefix)
		config = append(config, cfg...)
	}

	// sort config in reverse order to sort most specific config to the top
	sort.Sort(sort.Reverse(sort.StringSlice(config)))

	return config
}

// serviceConfig constructs the config for all good instances of a single service.
func serviceConfig(client *api.Client, name string, passing map[string]bool, tagPrefix string) (config []string) {
	if name == "" || len(passing) == 0 {
		return nil
	}

	q := &api.QueryOptions{RequireConsistent: true}
	svcs, _, err := client.Catalog().Service(name, "", q)
	if err != nil {
		log.Printf("[WARN] Error getting catalog service %s. %v", name, err)
		return nil
	}

	for _, svc := range svcs {
		// check if the instance is in the list of instances
		// which passed the health check
		if _, ok := passing[svc.ServiceID]; !ok {
			continue
		}

		for _, tag := range svc.ServiceTags {
			if host, path, ok := parseURLPrefixTag(tag, tagPrefix); ok {
				name, addr, port := svc.ServiceName, svc.ServiceAddress, svc.ServicePort
				if runtime.GOOS == "darwin" && !strings.Contains(addr, ".") {
					addr += ".local"
				}
				config = append(config, fmt.Sprintf("route add %s %s%s http://%s:%d/ tags %q", name, host, path, addr, port, strings.Join(svc.ServiceTags, ",")))
			}
		}
	}
	return config
}
<|endoftext|>


<commit_before>package common

import (
	"bufio"
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"github.com/mholt/archiver"
	"github.com/mitchellh/go-homedir"
	"gopkg.in/yaml.v2"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
)

// ExtensionImpl provides API for an extension
type ExtensionImpl interface {
	ID() string
	DecorateStackTemplate(assetName string, stackName string, templateBody io.Reader) (io.Reader, error)
	DecorateStackParameters(stackName string, stackParameters map[string]string) (map[string]string, error)
	DecorateStackTags(stackName string, stackTags map[string]string) (map[string]string, error)
}

// BaseExtensionImpl basic no-op extension
type BaseExtensionImpl struct {
	id string
}

// DecorateStackTemplate don't decorate, just return
func (ext *BaseExtensionImpl) DecorateStackTemplate(assetName string, stackName string, inTemplate io.Reader) (io.Reader, error) {
	return inTemplate, nil
}

// DecorateStackParameters don't decorate, just return
func (ext *BaseExtensionImpl) DecorateStackParameters(stackName string, stackParameters map[string]string) (map[string]string, error) {
	return stackParameters, nil
}

// DecorateStackTags don't decorate, just return
func (ext *BaseExtensionImpl) DecorateStackTags(stackName string, stackTags map[string]string) (map[string]string, error) {
	return stackTags, nil
}

// ID returns unique id for extension
func (ext *BaseExtensionImpl) ID() string {
	return ext.id
}

// ExtensionsManager provides API for running extensions
type ExtensionsManager interface {
	ExtensionImpl
	AddExtension(extension ExtensionImpl) error
}

// Implementation of ExtensionsManager
type extensionsManager struct {
	BaseExtensionImpl
	extensions []ExtensionImpl
}

// Create a new extensionsManager
func newExtensionsManager() (ExtensionsManager, error) {
	extMgr := &extensionsManager{
		BaseExtensionImpl{""},
		make([]ExtensionImpl, 0),
	}
	return extMgr, nil
}
func (extMgr *extensionsManager) AddExtension(extension ExtensionImpl) error {
	if extension == nil {
		return fmt.Errorf("extension was nil")
	}
	// ensure extension isn't already loaded
	for _, existingExt := range extMgr.extensions {
		if existingExt.ID() == extension.ID() {
			return fmt.Errorf("extension '%s' already loaded...skipping", extension.ID())
		}
	}
	extMgr.extensions = append(extMgr.extensions, extension)
	return nil
}

// DecorateStackTemplate for all extensions
func (extMgr *extensionsManager) DecorateStackTemplate(assetName string, stackName string, inTemplate io.Reader) (io.Reader, error) {
	outTemplate := inTemplate
	for _, ext := range extMgr.extensions {
		var err error
		outTemplate, err = ext.DecorateStackTemplate(assetName, stackName, outTemplate)
		if err != nil {
			return nil, err
		}
	}
	return outTemplate, nil
}

// DecorateStackParameters for all extensions
func (extMgr *extensionsManager) DecorateStackParameters(stackName string, stackParameters map[string]string) (map[string]string, error) {
	outParams := stackParameters
	for _, ext := range extMgr.extensions {
		var err error
		outParams, err = ext.DecorateStackParameters(stackName, outParams)
		if err != nil {
			return nil, err
		}
	}
	return outParams, nil
}

// DecorateStackTags for all extensions
func (extMgr *extensionsManager) DecorateStackTags(stackName string, stackTags map[string]string) (map[string]string, error) {
	outTags := stackTags
	for _, ext := range extMgr.extensions {
		var err error
		outTags, err = ext.DecorateStackTags(stackName, outTags)
		if err != nil {
			return nil, err
		}
	}
	return outTags, nil
}

// Extension for template overrides in mu.yml
type templateOverrideExtension struct {
	BaseExtensionImpl
	stackNameMatcher *regexp.Regexp
	decoration       interface{}
}

func newTemplateOverrideExtension(stackNamePattern string, template interface{}) ExtensionImpl {
	id := fmt.Sprintf("templateOverride:%s", stackNamePattern)
	ext := &templateOverrideExtension{
		BaseExtensionImpl{id},
		regexp.MustCompile(fmt.Sprintf("^%s$", stackNamePattern)),
		template,
	}
	return ext
}

// DecorateStackTemplate from overrides in mu.yml
func (ext *templateOverrideExtension) DecorateStackTemplate(assetName string, stackName string, inTemplate io.Reader) (io.Reader, error) {
	if ext.stackNameMatcher.MatchString(stackName) {
		return decorateTemplate(inTemplate, ext.decoration)
	}
	return inTemplate, nil
}

// Extension for tag overrides in mu.yml
type tagOverrideExtension struct {
	BaseExtensionImpl
	stackNameMatcher *regexp.Regexp
	tags             map[string]string
}

func newTagOverrideExtension(stackNamePattern string, tags map[string]string) ExtensionImpl {
	id := fmt.Sprintf("tagOverride:%s", stackNamePattern)
	ext := &tagOverrideExtension{
		BaseExtensionImpl{id},
		regexp.MustCompile(fmt.Sprintf("^%s$", stackNamePattern)),
		tags,
	}
	return ext
}

// DecorateStackTags from overrides in mu.yml
func (ext *tagOverrideExtension) DecorateStackTags(stackName string, stackTags map[string]string) (map[string]string, error) {
	if ext.stackNameMatcher.MatchString(stackName) {
		for k, v := range ext.tags {
			stackTags[k] = v
		}
	}
	return stackTags, nil
}

// Extension for archives of templates
type templateArchiveExtension struct {
	BaseExtensionImpl
	path string
	mode bool
}

// Extension for param overrides in mu.yml
type paramOverrideExtension struct {
	BaseExtensionImpl
	stackNameMatcher *regexp.Regexp
	params           map[string]string
}

func newParameterOverrideExtension(stackNamePattern string, params map[string]string) ExtensionImpl {
	id := fmt.Sprintf("paramOverride:%s", stackNamePattern)
	ext := &paramOverrideExtension{
		BaseExtensionImpl{id},
		regexp.MustCompile(fmt.Sprintf("^%s$", stackNamePattern)),
		params,
	}
	return ext
}

// DecorateStackParameters from overrides in mu.yml
func (ext *paramOverrideExtension) DecorateStackParameters(stackName string, stackParams map[string]string) (map[string]string, error) {
	if ext.stackNameMatcher.MatchString(stackName) {
		for k, v := range ext.params {
			stackParams[k] = v
		}
	}
	return stackParams, nil
}

func newTemplateArchiveExtension(u *url.URL, artifactManager ArtifactManager) (ExtensionImpl, error) {
	log.Debugf("Loading extension from '%s'", u)

	userdir, err := homedir.Dir()
	if err != nil {
		return nil, err
	}
	extensionsDirectory := filepath.Join(userdir, ".mu", "extensions")

	extID := urlToID(u)
	ext := &templateArchiveExtension{
		BaseExtensionImpl{u.String()},
		filepath.Join(extensionsDirectory, extID),
	}

	if fi, err := os.Stat(u.Path); u.Scheme == "file" && err == nil && fi.IsDir() {
		ext.path = u.Path
		log.Debugf("Loaded extension from '%s'", u.Path)
	} else {
		// check for existing etag
		etag := ""
		etagBytes, err := ioutil.ReadFile(filepath.Join(ext.path, ".etag"))
		if err == nil {
			etag = string(etagBytes)
		}

		body, etag, err := artifactManager.GetArtifact(u.String(), etag)
		if err != nil {
			return nil, err
		}

		if body != nil {
			defer body.Close()

			// empty dir
			os.RemoveAll(ext.path)
			os.MkdirAll(ext.path, 0700)

			// write out archive to dir
			err = extractArchive(ext.path, body)
			if err != nil {
				return nil, err
			}

			// write new etag
			err = ioutil.WriteFile(filepath.Join(ext.path, ".etag"), []byte(etag), 0644)
			if err != nil {
				return nil, err
			}
			log.Debugf("Loaded extension from '%s' [id=%s]", u, ext.id)
		} else {
			log.Debugf("Loaded extension from cache [id=%s]", ext.id)
		}

	}

	// try loading the extension manifest
	extManifest := make(map[interface{}]interface{})
	extManifestFile, err := ioutil.ReadFile(filepath.Join(ext.path, "mu-extension.yml"))
	if err == nil {
		err = yaml.Unmarshal(extManifestFile, extManifest)
		if err != nil {
			log.Debugf("error unmarshalling mu-extension.yml: %s", err)
		}
	} else {
		log.Debugf("error reading mu-extension.yml: %s", err)
	}

	// log info about the new extension
	if name, ok := extManifest["name"]; ok {
		if version, ok := extManifest["version"]; ok {
			log.Infof("Loaded extension %s (version=%v)", name, version)
		} else {
			log.Infof("Loaded extension %s", name)
		}
	} else {
		log.Infof("Loaded extension %s", u)
	}

	return ext, nil
}

// DecorateStackTemplate from template files in archive
func (ext *templateArchiveExtension) DecorateStackTemplate(assetName string, stackName string, inTemplate io.Reader) (io.Reader, error) {
	// TODO: handle replacement of inTemplate
	outTemplate := inTemplate
	assetPath := filepath.Join(ext.path, assetName)
	yamlFile, err := ioutil.ReadFile(assetPath)
	if err != nil {
		log.Debugf("Unable to find asset '%s' in extension '%s': %s", assetName, ext.id, err)
	} else {
		decoration := make(map[interface{}]interface{})
		err = yaml.Unmarshal(yamlFile, decoration)
		if err != nil {
			log.Warningf("Unable to parse asset '%s' in extension '%s': %s", assetName, ext.id, err)
		} else {
			return decorateTemplate(inTemplate, decoration)
		}
	}
	return outTemplate, nil
}

func urlToID(u *url.URL) string {
	h := sha1.New()
	h.Write([]byte(u.String()))
	return hex.EncodeToString(h.Sum(nil))
}

func extractArchive(destPath string, archive io.ReadCloser) error {
	reader := bufio.NewReader(archive)
	headBytes, err := reader.Peek(512)
	if err != nil {
		return err
	}
	contentType := http.DetectContentType(headBytes)
	log.Debugf("Extracting type '%s'", contentType)

	switch contentType {
	case "application/x-gzip":
		return archiver.TarGz.Read(reader, destPath)
	case "application/zip":
		return archiver.Zip.Read(reader, destPath)
	default:
		return fmt.Errorf("unable to handle archive of content-type '%s'", contentType)
	}
}
<commit_msg>remove extra field<commit_after>package common

import (
	"bufio"
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"github.com/mholt/archiver"
	"github.com/mitchellh/go-homedir"
	"gopkg.in/yaml.v2"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
)

// ExtensionImpl provides API for an extension
type ExtensionImpl interface {
	ID() string
	DecorateStackTemplate(assetName string, stackName string, templateBody io.Reader) (io.Reader, error)
	DecorateStackParameters(stackName string, stackParameters map[string]string) (map[string]string, error)
	DecorateStackTags(stackName string, stackTags map[string]string) (map[string]string, error)
}

// BaseExtensionImpl basic no-op extension
type BaseExtensionImpl struct {
	id string
}

// DecorateStackTemplate don't decorate, just return
func (ext *BaseExtensionImpl) DecorateStackTemplate(assetName string, stackName string, inTemplate io.Reader) (io.Reader, error) {
	return inTemplate, nil
}

// DecorateStackParameters don't decorate, just return
func (ext *BaseExtensionImpl) DecorateStackParameters(stackName string, stackParameters map[string]string) (map[string]string, error) {
	return stackParameters, nil
}

// DecorateStackTags don't decorate, just return
func (ext *BaseExtensionImpl) DecorateStackTags(stackName string, stackTags map[string]string) (map[string]string, error) {
	return stackTags, nil
}

// ID returns unique id for extension
func (ext *BaseExtensionImpl) ID() string {
	return ext.id
}

// ExtensionsManager provides API for running extensions
type ExtensionsManager interface {
	ExtensionImpl
	AddExtension(extension ExtensionImpl) error
}

// Implementation of ExtensionsManager
type extensionsManager struct {
	BaseExtensionImpl
	extensions []ExtensionImpl
}

// Create a new extensionsManager
func newExtensionsManager() (ExtensionsManager, error) {
	extMgr := &extensionsManager{
		BaseExtensionImpl{""},
		make([]ExtensionImpl, 0),
	}
	return extMgr, nil
}
func (extMgr *extensionsManager) AddExtension(extension ExtensionImpl) error {
	if extension == nil {
		return fmt.Errorf("extension was nil")
	}
	// ensure extension isn't already loaded
	for _, existingExt := range extMgr.extensions {
		if existingExt.ID() == extension.ID() {
			return fmt.Errorf("extension '%s' already loaded...skipping", extension.ID())
		}
	}
	extMgr.extensions = append(extMgr.extensions, extension)
	return nil
}

// DecorateStackTemplate for all extensions
func (extMgr *extensionsManager) DecorateStackTemplate(assetName string, stackName string, inTemplate io.Reader) (io.Reader, error) {
	outTemplate := inTemplate
	for _, ext := range extMgr.extensions {
		var err error
		outTemplate, err = ext.DecorateStackTemplate(assetName, stackName, outTemplate)
		if err != nil {
			return nil, err
		}
	}
	return outTemplate, nil
}

// DecorateStackParameters for all extensions
func (extMgr *extensionsManager) DecorateStackParameters(stackName string, stackParameters map[string]string) (map[string]string, error) {
	outParams := stackParameters
	for _, ext := range extMgr.extensions {
		var err error
		outParams, err = ext.DecorateStackParameters(stackName, outParams)
		if err != nil {
			return nil, err
		}
	}
	return outParams, nil
}

// DecorateStackTags for all extensions
func (extMgr *extensionsManager) DecorateStackTags(stackName string, stackTags map[string]string) (map[string]string, error) {
	outTags := stackTags
	for _, ext := range extMgr.extensions {
		var err error
		outTags, err = ext.DecorateStackTags(stackName, outTags)
		if err != nil {
			return nil, err
		}
	}
	return outTags, nil
}

// Extension for template overrides in mu.yml
type templateOverrideExtension struct {
	BaseExtensionImpl
	stackNameMatcher *regexp.Regexp
	decoration       interface{}
}

func newTemplateOverrideExtension(stackNamePattern string, template interface{}) ExtensionImpl {
	id := fmt.Sprintf("templateOverride:%s", stackNamePattern)
	ext := &templateOverrideExtension{
		BaseExtensionImpl{id},
		regexp.MustCompile(fmt.Sprintf("^%s$", stackNamePattern)),
		template,
	}
	return ext
}

// DecorateStackTemplate from overrides in mu.yml
func (ext *templateOverrideExtension) DecorateStackTemplate(assetName string, stackName string, inTemplate io.Reader) (io.Reader, error) {
	if ext.stackNameMatcher.MatchString(stackName) {
		return decorateTemplate(inTemplate, ext.decoration)
	}
	return inTemplate, nil
}

// Extension for tag overrides in mu.yml
type tagOverrideExtension struct {
	BaseExtensionImpl
	stackNameMatcher *regexp.Regexp
	tags             map[string]string
}

func newTagOverrideExtension(stackNamePattern string, tags map[string]string) ExtensionImpl {
	id := fmt.Sprintf("tagOverride:%s", stackNamePattern)
	ext := &tagOverrideExtension{
		BaseExtensionImpl{id},
		regexp.MustCompile(fmt.Sprintf("^%s$", stackNamePattern)),
		tags,
	}
	return ext
}

// DecorateStackTags from overrides in mu.yml
func (ext *tagOverrideExtension) DecorateStackTags(stackName string, stackTags map[string]string) (map[string]string, error) {
	if ext.stackNameMatcher.MatchString(stackName) {
		for k, v := range ext.tags {
			stackTags[k] = v
		}
	}
	return stackTags, nil
}

// Extension for archives of templates
type templateArchiveExtension struct {
	BaseExtensionImpl
	path string
}

// Extension for param overrides in mu.yml
type paramOverrideExtension struct {
	BaseExtensionImpl
	stackNameMatcher *regexp.Regexp
	params           map[string]string
}

func newParameterOverrideExtension(stackNamePattern string, params map[string]string) ExtensionImpl {
	id := fmt.Sprintf("paramOverride:%s", stackNamePattern)
	ext := &paramOverrideExtension{
		BaseExtensionImpl{id},
		regexp.MustCompile(fmt.Sprintf("^%s$", stackNamePattern)),
		params,
	}
	return ext
}

// DecorateStackParameters from overrides in mu.yml
func (ext *paramOverrideExtension) DecorateStackParameters(stackName string, stackParams map[string]string) (map[string]string, error) {
	if ext.stackNameMatcher.MatchString(stackName) {
		for k, v := range ext.params {
			stackParams[k] = v
		}
	}
	return stackParams, nil
}

func newTemplateArchiveExtension(u *url.URL, artifactManager ArtifactManager) (ExtensionImpl, error) {
	log.Debugf("Loading extension from '%s'", u)

	userdir, err := homedir.Dir()
	if err != nil {
		return nil, err
	}
	extensionsDirectory := filepath.Join(userdir, ".mu", "extensions")

	extID := urlToID(u)
	ext := &templateArchiveExtension{
		BaseExtensionImpl{u.String()},
		filepath.Join(extensionsDirectory, extID),
	}

	if fi, err := os.Stat(u.Path); u.Scheme == "file" && err == nil && fi.IsDir() {
		ext.path = u.Path
		log.Debugf("Loaded extension from '%s'", u.Path)
	} else {
		// check for existing etag
		etag := ""
		etagBytes, err := ioutil.ReadFile(filepath.Join(ext.path, ".etag"))
		if err == nil {
			etag = string(etagBytes)
		}

		body, etag, err := artifactManager.GetArtifact(u.String(), etag)
		if err != nil {
			return nil, err
		}

		if body != nil {
			defer body.Close()

			// empty dir
			os.RemoveAll(ext.path)
			os.MkdirAll(ext.path, 0700)

			// write out archive to dir
			err = extractArchive(ext.path, body)
			if err != nil {
				return nil, err
			}

			// write new etag
			err = ioutil.WriteFile(filepath.Join(ext.path, ".etag"), []byte(etag), 0644)
			if err != nil {
				return nil, err
			}
			log.Debugf("Loaded extension from '%s' [id=%s]", u, ext.id)
		} else {
			log.Debugf("Loaded extension from cache [id=%s]", ext.id)
		}

	}

	// try loading the extension manifest
	extManifest := make(map[interface{}]interface{})
	extManifestFile, err := ioutil.ReadFile(filepath.Join(ext.path, "mu-extension.yml"))
	if err == nil {
		err = yaml.Unmarshal(extManifestFile, extManifest)
		if err != nil {
			log.Debugf("error unmarshalling mu-extension.yml: %s", err)
		}
	} else {
		log.Debugf("error reading mu-extension.yml: %s", err)
	}

	// log info about the new extension
	if name, ok := extManifest["name"]; ok {
		if version, ok := extManifest["version"]; ok {
			log.Infof("Loaded extension %s (version=%v)", name, version)
		} else {
			log.Infof("Loaded extension %s", name)
		}
	} else {
		log.Infof("Loaded extension %s", u)
	}

	return ext, nil
}

// DecorateStackTemplate from template files in archive
func (ext *templateArchiveExtension) DecorateStackTemplate(assetName string, stackName string, inTemplate io.Reader) (io.Reader, error) {
	// TODO: handle replacement of inTemplate
	outTemplate := inTemplate
	assetPath := filepath.Join(ext.path, assetName)
	yamlFile, err := ioutil.ReadFile(assetPath)
	if err != nil {
		log.Debugf("Unable to find asset '%s' in extension '%s': %s", assetName, ext.id, err)
	} else {
		decoration := make(map[interface{}]interface{})
		err = yaml.Unmarshal(yamlFile, decoration)
		if err != nil {
			log.Warningf("Unable to parse asset '%s' in extension '%s': %s", assetName, ext.id, err)
		} else {
			return decorateTemplate(inTemplate, decoration)
		}
	}
	return outTemplate, nil
}

func urlToID(u *url.URL) string {
	h := sha1.New()
	h.Write([]byte(u.String()))
	return hex.EncodeToString(h.Sum(nil))
}

func extractArchive(destPath string, archive io.ReadCloser) error {
	reader := bufio.NewReader(archive)
	headBytes, err := reader.Peek(512)
	if err != nil {
		return err
	}
	contentType := http.DetectContentType(headBytes)
	log.Debugf("Extracting type '%s'", contentType)

	switch contentType {
	case "application/x-gzip":
		return archiver.TarGz.Read(reader, destPath)
	case "application/zip":
		return archiver.Zip.Read(reader, destPath)
	default:
		return fmt.Errorf("unable to handle archive of content-type '%s'", contentType)
	}
}
<|endoftext|>


<commit_before>// Copyright © 2016 Jip J. Dekker <jip@dekker.li>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package compiler

import (
	"os"
	"path/filepath"

	log "github.com/Sirupsen/logrus"
	"github.com/jjdekker/ponder/settings"
)

// CompileDir compiles all lilypond files and makes all
// sheet music available in the OutputDir
func CompileDir(path string, opts *settings.Settings) {

}

func generateScores() ([]*settings.Score, func(string, os.FileInfo) error) {
	scores := make([]*settings.Score)
	return scores, func(path string, file os.FileInfo) error {
		switch filepath.Ext(path) {
		case ".ly":
			log.WithFields(log.Fields{"path": path}).Info("adding lilypond file")
			append(scores, &settings.Score{Path: path})

		case ".json":
			if filepath.Base(path) != "ponder.json" {
				log.WithFields(log.Fields{"path": path}).Info("adding json file")
				if score, err := fromJSON(path); err != nil {
					log.WithFields(log.Fields{
						"error": err,
						"path":  path,
					}).Warning("unable to parse score settings, skipping...")
				} else {
					append(scores, score)
				}
			}

		default:
			log.WithFields(log.Fields{"path": path}).Debug("ignoring file")
		}
	}
}
<commit_msg>Adds filewalker calls to CompileDir<commit_after>// Copyright © 2016 Jip J. Dekker <jip@dekker.li>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package compiler

import (
	"os"
	"path/filepath"

	log "github.com/Sirupsen/logrus"
	"github.com/jjdekker/ponder/settings"
)

// CompileDir compiles all lilypond files and makes all
// sheet music available in the OutputDir
func CompileDir(path string, opts *settings.Settings) {
	// Find all scores
	scores, collector := generateScores()
	filepath.Walk(path, compilePath(path, opts, collector))
}

func generateScores() ([]*settings.Score, func(string, os.FileInfo) error) {
	scores := make([]*settings.Score)
	return scores, func(path string, file os.FileInfo) error {
		switch filepath.Ext(path) {
		case ".ly":
			log.WithFields(log.Fields{"path": path}).Info("adding lilypond file")
			append(scores, &settings.Score{Path: path})

		case ".json":
			if filepath.Base(path) != "ponder.json" {
				log.WithFields(log.Fields{"path": path}).Info("adding json file")
				if score, err := fromJSON(path); err != nil {
					log.WithFields(log.Fields{
						"error": err,
						"path":  path,
					}).Warning("unable to parse score settings, skipping...")
				} else {
					append(scores, score)
				}
			}

		default:
			log.WithFields(log.Fields{"path": path}).Debug("ignoring file")
		}
	}
}
<|endoftext|>


<commit_before>// © 2014 Tajti Attila

package basedir

import (
	"os"
	"path/filepath"
	"syscall"
)

const FileModeDir os.FileMode = 0700 // Permission mode to create missing base directories

// Dir represents a base directory
type Dir struct {
	dirs []string
}

// Open the file subpath inside a directory. The search is started in
// the most important directory, which is specified by the HOME variable.
// If no file is found, the error for the first directory is returned.
func (d *Dir) Open(subpath string) (f *os.File, err error) {
	for _, path := range d.dirs {
		var e error
		if f, e = os.Open(filepath.Join(path, subpath)); e == nil {
			return f, nil
		}
		if err == nil {
			err = e // keep first error
		}
	}
	return
}

// Opens all files matching subpath from all directories.
// If no file is found, the error for the first directory is returned.
func (d *Dir) OpenAll(subpath string) (files []*os.File, err error) {
	for _, path := range d.dirs {
		var f *os.File
		if f, err = os.Open(filepath.Join(path, subpath)); err == nil {
			files = append(files, f)
		}
	}
	if len(files) != 0 {
		err = nil
	}
	return
}

// Create a new file in the "home" directory.
// If the "home" directory does not exist, it is created with FileModeDir.
func (d *Dir) Create(subpath string) (*os.File, error) {
	path := filepath.Join(d.dirs[0], subpath)
	if err := os.MkdirAll(filepath.Dir(path), FileModeDir); err != nil {
		return nil, err
	}
	return os.Create(path)
}

// Mkdir creates a new directory inside the "home" directory.
// If the "home" directory does not exist, it is created with FileModeDir.
// The subdirectory will be created using the permission bits.
func (d *Dir) Mkdir(subpath string, perm os.FileMode) error {
	if err := os.MkdirAll(d.dirs[0], FileModeDir); err != nil {
		return err
	}
	return os.Mkdir(filepath.Join(d.dirs[0], subpath), perm)
}

// MkdirAll creates a new directory, along with any necessary parents inside the "home" directory.
// If the "home" directory does not exist, it is created with FileModeDir.
// The subdirectories will be created using the permission bits.
func (d *Dir) MkdirAll(subpath string, perm os.FileMode) error {
	if err := os.MkdirAll(d.dirs[0], FileModeDir); err != nil {
		return err
	}
	return os.MkdirAll(filepath.Join(d.dirs[0], subpath), perm)
}

// Remove removes the named file or directory inside the "home" directory.
func (d *Dir) Remove(subpath string) error {
	return os.Remove(filepath.Join(d.dirs[0], subpath))
}

// RemoveAll removes the named file or directory and any children it contains inside the "home" directory.
func (d *Dir) RemoveAll(subpath string) error {
	return os.RemoveAll(filepath.Join(d.dirs[0], subpath))
}

// Dir returns the absolute path if subpath exists in any of the base directories.
// The absolute path returned always contain a path separator at the end if err is nil.
func (d *Dir) Dir(subpath string) (path string, err error) {
	for _, path := range d.dirs {
		path = filepath.Join(path, subpath)
		fi, err2 := os.Stat(path)
		if err2 == nil {
			if fi.IsDir() {
				return addsep(path), nil
			} else {
				err2 = &os.PathError{"Opendir", path, syscall.ENOTDIR}
			}
		}
		if err == nil {
			err = err2
		}
	}
	return "", err
}

// EnsureDir returns the absolute path if subpath exists in any of the base directories.
// If subpath does not exist in any of the base directories,
// it is created in the "home" directory with FileModeDir.
//
// It is equivalent to calling Dir() and then MkdirAll() if the first call has failed.
func (d *Dir) EnsureDir(subpath string, perm os.FileMode) (path string, err error) {
	path, err = d.Dir(subpath)
	if err != nil {
		err = d.MkdirAll(subpath, perm)
		if err == nil {
			path = addsep(filepath.Join(d.dirs[0], subpath))
		}
	}
	return
}

// envHome: environment variable name for base directory
// defHome: environment variable name for list of additional directories list
// envDirs: default base directory inside $HOME, if not provided in environment variable
// defDirs: default optional
func newDir(envHome, defHome, envDirs string, defDirs ...string) *Dir {
	base := os.Getenv(envHome)
	if base == "" {
		base = defHome
	}
	d := &Dir{[]string{expandTilde(base)}}
	if dirs := os.Getenv(envDirs); dirs != "" {
		s := 0
		for p, ch := range dirs {
			if ch == os.PathListSeparator {
				d.add(dirs[s:p])
				s = p + 1
			}
		}
		d.add(dirs[s:])
	} else {
		d.dirs = append(d.dirs, defDirs...)
	}
	return d
}

func (d *Dir) add(path string) {
	if path != "" {
		d.dirs = append(d.dirs, path)
	}
}
<commit_msg>add Gopath<commit_after>// © 2014 Tajti Attila

package basedir

import (
	"os"
	"path/filepath"
	"syscall"
)

const FileModeDir os.FileMode = 0700 // Permission mode to create missing base directories

// Dir represents a base directory
type Dir struct {
	dirs []string
}

// Gopath is a Dir for accessing files within GOROOT and GOPATH.
var Gopath = newDir("GOROOT", "", "GOPATH", "")

// Open the file subpath inside a directory. The search is started in
// the most important directory, which is specified by the HOME variable.
// If no file is found, the error for the first directory is returned.
func (d *Dir) Open(subpath string) (f *os.File, err error) {
	for _, path := range d.dirs {
		var e error
		if f, e = os.Open(filepath.Join(path, subpath)); e == nil {
			return f, nil
		}
		if err == nil {
			err = e // keep first error
		}
	}
	return
}

// Opens all files matching subpath from all directories.
// If no file is found, the error for the first directory is returned.
func (d *Dir) OpenAll(subpath string) (files []*os.File, err error) {
	for _, path := range d.dirs {
		var f *os.File
		if f, err = os.Open(filepath.Join(path, subpath)); err == nil {
			files = append(files, f)
		}
	}
	if len(files) != 0 {
		err = nil
	}
	return
}

// Create a new file in the "home" directory.
// If the "home" directory does not exist, it is created with FileModeDir.
func (d *Dir) Create(subpath string) (*os.File, error) {
	path := filepath.Join(d.dirs[0], subpath)
	if err := os.MkdirAll(filepath.Dir(path), FileModeDir); err != nil {
		return nil, err
	}
	return os.Create(path)
}

// Mkdir creates a new directory inside the "home" directory.
// If the "home" directory does not exist, it is created with FileModeDir.
// The subdirectory will be created using the permission bits.
func (d *Dir) Mkdir(subpath string, perm os.FileMode) error {
	if err := os.MkdirAll(d.dirs[0], FileModeDir); err != nil {
		return err
	}
	return os.Mkdir(filepath.Join(d.dirs[0], subpath), perm)
}

// MkdirAll creates a new directory, along with any necessary parents inside the "home" directory.
// If the "home" directory does not exist, it is created with FileModeDir.
// The subdirectories will be created using the permission bits.
func (d *Dir) MkdirAll(subpath string, perm os.FileMode) error {
	if err := os.MkdirAll(d.dirs[0], FileModeDir); err != nil {
		return err
	}
	return os.MkdirAll(filepath.Join(d.dirs[0], subpath), perm)
}

// Remove removes the named file or directory inside the "home" directory.
func (d *Dir) Remove(subpath string) error {
	return os.Remove(filepath.Join(d.dirs[0], subpath))
}

// RemoveAll removes the named file or directory and any children it contains inside the "home" directory.
func (d *Dir) RemoveAll(subpath string) error {
	return os.RemoveAll(filepath.Join(d.dirs[0], subpath))
}

// Dir returns the absolute path if subpath exists in any of the base directories.
// The absolute path returned always contain a path separator at the end if err is nil.
func (d *Dir) Dir(subpath string) (path string, err error) {
	for _, path := range d.dirs {
		path = filepath.Join(path, subpath)
		fi, err2 := os.Stat(path)
		if err2 == nil {
			if fi.IsDir() {
				return addsep(path), nil
			} else {
				err2 = &os.PathError{"Opendir", path, syscall.ENOTDIR}
			}
		}
		if err == nil {
			err = err2
		}
	}
	return "", err
}

// EnsureDir returns the absolute path if subpath exists in any of the base directories.
// If subpath does not exist in any of the base directories,
// it is created in the "home" directory with FileModeDir.
//
// It is equivalent to calling Dir() and then MkdirAll() if the first call has failed.
func (d *Dir) EnsureDir(subpath string, perm os.FileMode) (path string, err error) {
	path, err = d.Dir(subpath)
	if err != nil {
		err = d.MkdirAll(subpath, perm)
		if err == nil {
			path = addsep(filepath.Join(d.dirs[0], subpath))
		}
	}
	return
}

// envHome: environment variable name for base directory
// defHome: environment variable name for list of additional directories list
// envDirs: default base directory inside $HOME, if not provided in environment variable
// defDirs: default optional
func newDir(envHome, defHome, envDirs string, defDirs ...string) *Dir {
	base := os.Getenv(envHome)
	if base == "" {
		base = expandTilde(defHome)
	}
	d := new(Dir)
	d.add(base)
	if dirs := os.Getenv(envDirs); dirs != "" {
		s := 0
		for p, ch := range dirs {
			if ch == os.PathListSeparator {
				d.add(dirs[s:p])
				s = p + 1
			}
		}
		d.add(dirs[s:])
	} else {
		d.dirs = append(d.dirs, defDirs...)
	}
	if len(d.dirs) == 0 {
		p, err := os.Getwd()
		if err != nil {
			p = "."
		}
		d.add(p)
	}
	return d
}

func (d *Dir) add(path string) {
	if path != "" {
		d.dirs = append(d.dirs, path)
	}
}
<|endoftext|>


<commit_before>package kamino

import (
	"errors"
	"fmt"
	"strconv"
)

type genome struct {
	APIToken string
	Account  string
	Depth    string
	Ref      string
	Repo     string
	UseCache string
}

/*
NewGenome creates a new genome type, which is the options struct for a CloneFactory.

Valid fields for opts are as follows:

	* "account"
		- purpose: GitHub account
		- required: true
	* "cache"
		- purpose: whether or not to use the cached / previously cloned version of the repo
		- required: false
		- default: "no"
		- valid options:
			* "no" - do not use cache, create a uniquely named directory
			* "if_available" - use cache if already created, otherwise create a uniquely named directory
			* "create" - use cache if already created, create cache if not present
			* "force" - use cache if already created, fail if cache not present
	* "depth"
		- purpose: git clone `--depth` option
		- required: false
		- default: "50"
		- validation: must be empty string or parsable as a base 10 integer
	* "repo"
		- purpose: GitHub repo
		- required: true
	* "token"
		- purpose: GitHub API token for private repos
		- required: false (functionally required if your repo is private)
		- default: (not sent with request if empty)

*/
func NewGenome(opts map[string]string) (*genome, error) {
	g := &genome{}

	if depth, ok := opts["depth"]; ok && depth != "" {
		if _, err := strconv.Atoi(depth); err == nil {
			g.Depth = depth
		} else {
			return nil, fmt.Errorf("%q is not a valid clone depth", depth)
		}
	} else {
		g.Depth = "50"
	}

	if token, ok := opts["token"]; ok {
		g.APIToken = token
	}

	if account, ok := opts["account"]; ok && account != "" {
		g.Account = account
	} else {
		return nil, errors.New("account must be provided")
	}

	if ref, ok := opts["ref"]; ok && ref != "" {
		g.Ref = ref
	} else {
		return nil, errors.New("ref must be provided")
	}

	if repo, ok := opts["repo"]; ok && repo != "" {
		g.Repo = repo
	} else {
		return nil, errors.New("repo must be provided")
	}

	if cache, ok := opts["cache"]; ok && cache != "" {
		if validCacheOption(cache) {
			g.UseCache = cache
		} else {
			return nil, fmt.Errorf("%q is not a valid cache option", cache)
		}
	} else {
		g.UseCache = "no"
	}

	return g, nil
}

func validCacheOptions() []string {
	return []string{
		"create",
		"force",
		"if_available",
		"no",
	}
}

func validCacheOption(opt string) bool {
	for _, v := range validCacheOptions() {
		if v == opt {
			return true
		}
	}

	return false
}
<commit_msg>Adding ref to genome documentation<commit_after>package kamino

import (
	"errors"
	"fmt"
	"strconv"
)

type genome struct {
	APIToken string
	Account  string
	Depth    string
	Ref      string
	Repo     string
	UseCache string
}

/*
NewGenome creates a new genome type, which is the options struct for a CloneFactory.

Valid fields for opts are as follows:

	* "account"
		- purpose: GitHub account
		- required: true
	* "cache"
		- purpose: whether or not to use the cached / previously cloned version of the repo
		- required: false
		- default: "no"
		- valid options:
			* "no" - do not use cache, create a uniquely named directory
			* "if_available" - use cache if already created, otherwise create a uniquely named directory
			* "create" - use cache if already created, create cache if not present
			* "force" - use cache if already created, fail if cache not present
	* "depth"
		- purpose: git clone `--depth` option
		- required: false
		- default: "50"
		- validation: must be empty string or parsable as a base 10 integer
	* "ref"
		- purpose: the git SHA to check out in the cloned repo
		- required: true
	* "repo"
		- purpose: GitHub repo
		- required: true
	* "token"
		- purpose: GitHub API token for private repos
		- required: false (functionally required if your repo is private)
		- default: (not sent with request if empty)

*/
func NewGenome(opts map[string]string) (*genome, error) {
	g := &genome{}

	if depth, ok := opts["depth"]; ok && depth != "" {
		if _, err := strconv.Atoi(depth); err == nil {
			g.Depth = depth
		} else {
			return nil, fmt.Errorf("%q is not a valid clone depth", depth)
		}
	} else {
		g.Depth = "50"
	}

	if token, ok := opts["token"]; ok {
		g.APIToken = token
	}

	if account, ok := opts["account"]; ok && account != "" {
		g.Account = account
	} else {
		return nil, errors.New("account must be provided")
	}

	if ref, ok := opts["ref"]; ok && ref != "" {
		g.Ref = ref
	} else {
		return nil, errors.New("ref must be provided")
	}

	if repo, ok := opts["repo"]; ok && repo != "" {
		g.Repo = repo
	} else {
		return nil, errors.New("repo must be provided")
	}

	if cache, ok := opts["cache"]; ok && cache != "" {
		if validCacheOption(cache) {
			g.UseCache = cache
		} else {
			return nil, fmt.Errorf("%q is not a valid cache option", cache)
		}
	} else {
		g.UseCache = "no"
	}

	return g, nil
}

func validCacheOptions() []string {
	return []string{
		"create",
		"force",
		"if_available",
		"no",
	}
}

func validCacheOption(opt string) bool {
	for _, v := range validCacheOptions() {
		if v == opt {
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>// Copyright (C) 2015 Thomas de Zeeuw.
//
// Licensed under the MIT license that can be found in the LICENSE file.

// Package logger provides asynchronous logging. It is build for customisation
// and speed. It uses a custom EventWriter so any custom backend can be used to
// store the logs. Logger provides multiple ways to log information with
// different levels of importance.
//
// Each logging operation makes a single call to the EventWriter's Write method,
// but not necessarily at the same time as Log operation is called, since the
// logging is done asynchronously. The logger package can used simultaneously
// from multiple goroutines.
//
// Because the logger package is asynchronous Close musted be called before the
// program exits, this way logger will make sure all log event will be written.
//
// By default there are six different event types (from lower to higher): debug,
// info, warn, error, fatal and thumb. But new event types can be created, to be
// used in the custom EventWriter.
package logger
<commit_msg>Document panic of log operations<commit_after>// Copyright (C) 2015 Thomas de Zeeuw.
//
// Licensed under the MIT license that can be found in the LICENSE file.

// Package logger provides asynchronous logging. It is build for customisation
// and speed. It uses a custom EventWriter so any custom backend can be used to
// store the logs. Logger provides multiple ways to log information with
// different levels of importance.
//
// Each logging operation makes a single call to the EventWriter's Write method,
// but not necessarily at the same time as Log operation is called, since the
// logging is done asynchronously. The logger package can used simultaneously
// from multiple goroutines.
//
// Because the logger package is asynchronous Close musted be called before the
// program exits, this way logger will make sure all log event will be written.
// After Close is called all calls to any log operation will panic. This is
// because internally the logger package uses a channel to make the logging
// asynchronous and sending to a closed channel will panic.
//
// By default there are six different event types (from lower to higher): debug,
// info, warn, error, fatal and thumb. But new event types can be created, to be
// used in the custom EventWriter.
package logger
<|endoftext|>


<commit_before>// Copyright 2013 - 2014 The XORM Authors. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

/*

Package xorm is a simple and powerful ORM for Go.

Installation

Make sure you have installed Go 1.1+ and then:

    go get github.com/go-xorm/xorm

Create Engine

Firstly, we should new an engine for a database

    engine, err := xorm.NewEngine(driverName, dataSourceName)

Method NewEngine's parameters is the same as sql.Open. It depends
drivers' implementation.
Generally, one engine for an application is enough. You can set it as package variable.

Raw Methods

Xorm also support raw sql execution:

1. query a SQL string, the returned results is []map[string][]byte

    results, err := engine.Query("select * from user")

2. execute a SQL string, the returned results

    affected, err := engine.Exec("update user set .... where ...")

ORM Methods

There are 7 major ORM methods and many helpful methods to use to operate database.

1. Insert one or multipe records to database

    affected, err := engine.Insert(&struct)
    // INSERT INTO struct () values ()
    affected, err := engine.Insert(&struct1, &struct2)
    // INSERT INTO struct1 () values ()
    // INSERT INTO struct2 () values ()
    affected, err := engine.Insert(&sliceOfStruct)
    // INSERT INTO struct () values (),(),()
    affected, err := engine.Insert(&struct1, &sliceOfStruct2)
    // INSERT INTO struct1 () values ()
    // INSERT INTO struct2 () values (),(),()

2. Query one record from database

    has, err := engine.Get(&user)
    // SELECT * FROM user LIMIT 1

3. Query multiple records from database

    sliceOfStructs := new(Struct)
    err := engine.Find(sliceOfStructs)
    // SELECT * FROM user

4. Query multiple records and record by record handle, there two methods, one is Iterate,
another is Raws

    err := engine.Iterate(...)
    // SELECT * FROM user

    raws, err := engine.Raws(...)
    // SELECT * FROM user
    bean := new(Struct)
    for raws.Next() {
        err = raws.Scan(bean)
    }

5. Update one or more records

    affected, err := engine.Update(&user)
    // UPDATE user SET ...

6. Delete one or more records, Delete MUST has conditon

    affected, err := engine.Where(...).Delete(&user)
    // DELETE FROM user Where ...

7. Count records

    counts, err := engine.Count(&user)
    // SELECT count(*) AS total FROM user

Conditions

The above 7 methods could use with condition methods chainable.
Attention: the above 7 methods should be the last chainable method.

1. Id, In

    engine.Id(1).Get(&user) // for single primary key
    // SELECT * FROM user WHERE id = 1
    engine.Id(core.PK{1, 2}).Get(&user) // for composite primary keys
    // SELECT * FROM user WHERE id1 = 1 AND id2 = 2
    engine.In("id", 1, 2, 3).Find(&users)
    // SELECT * FROM user WHERE id IN (1, 2, 3)
    engine.In("id", []int{1, 2, 3})
    // SELECT * FROM user WHERE id IN (1, 2, 3)

2. Where, And, Or

    engine.Where().And().Or().Find()
    // SELECT * FROM user WHERE (.. AND ..) OR ...

3. OrderBy, Asc, Desc

    engine.Asc().Desc().Find()
    // SELECT * FROM user ORDER BY .. ASC, .. DESC
    engine.OrderBy().Find()
    // SELECT * FROM user ORDER BY ..

4. Limit, Top

    engine.Limit().Find()
    // SELECT * FROM user LIMIT .. OFFSET ..
    engine.Top(5).Find()
    // SELECT TOP 5 * FROM user // for mssql
    // SELECT * FROM user LIMIT .. OFFSET 0 //for other databases

5. Sql, let you custom SQL

    engine.Sql("select * from user").Find()

6. Cols, Omit, Distinct

    engine.Cols("col1, col2").Find()
    // SELECT col1, col2 FROM user
    engine.Cols("col1", "col2").Where().Update(user)
    // UPDATE user set col1 = ?, col2 = ? Where ...
    engine.Omit("col1").Find()
    // SELECT col2, col3 FROM user
    engine.Omit("col1").Insert()
    // INSERT INTO table (non-col1) VALUES ()
    engine.Distinct("col1").Find()
    // SELECT DISTINCT col1 FROM user

7. Join, GroupBy, Having

    engine.GroupBy("name").Having("name='xlw'").Find()
    //SELECT * FROM user GROUP BY name HAVING name='xlw'
    engine.Join("LEFT", "userdetail", "user.id=userdetail.id").Find()
    //SELECT * FROM user LEFT JOIN userdetail ON user.id=userdetail.id

More usage, please visit https://github.com/go-xorm/xorm/blob/master/docs/QuickStartEn.md
*/
package xorm
<commit_msg>improved doc.go<commit_after>// Copyright 2013 - 2014 The XORM Authors. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

/*

Package xorm is a simple and powerful ORM for Go.

Installation

Make sure you have installed Go 1.1+ and then:

    go get github.com/go-xorm/xorm

Create Engine

Firstly, we should new an engine for a database

    engine, err := xorm.NewEngine(driverName, dataSourceName)

Method NewEngine's parameters is the same as sql.Open. It depends
drivers' implementation.
Generally, one engine for an application is enough. You can set it as package variable.

Raw Methods

Xorm also support raw sql execution:

1. query a SQL string, the returned results is []map[string][]byte

    results, err := engine.Query("select * from user")

2. execute a SQL string, the returned results

    affected, err := engine.Exec("update user set .... where ...")

ORM Methods

There are 7 major ORM methods and many helpful methods to use to operate database.

1. Insert one or multipe records to database

    affected, err := engine.Insert(&struct)
    // INSERT INTO struct () values ()
    affected, err := engine.Insert(&struct1, &struct2)
    // INSERT INTO struct1 () values ()
    // INSERT INTO struct2 () values ()
    affected, err := engine.Insert(&sliceOfStruct)
    // INSERT INTO struct () values (),(),()
    affected, err := engine.Insert(&struct1, &sliceOfStruct2)
    // INSERT INTO struct1 () values ()
    // INSERT INTO struct2 () values (),(),()

2. Query one record from database

    has, err := engine.Get(&user)
    // SELECT * FROM user LIMIT 1

3. Query multiple records from database

    sliceOfStructs := new(Struct)
    err := engine.Find(sliceOfStructs)
    // SELECT * FROM user

4. Query multiple records and record by record handle, there two methods, one is Iterate,
another is Rows

    err := engine.Iterate(...)
    // SELECT * FROM user

    rows, err := engine.Rows(...)
    // SELECT * FROM user
    defer rows.Close()
    bean := new(Struct)
    for rows.Next() {
        err = rows.Scan(bean)
    }

5. Update one or more records

    affected, err := engine.Id(...).Update(&user)
    // UPDATE user SET ...

6. Delete one or more records, Delete MUST has conditon

    affected, err := engine.Where(...).Delete(&user)
    // DELETE FROM user Where ...

7. Count records

    counts, err := engine.Count(&user)
    // SELECT count(*) AS total FROM user

Conditions

The above 7 methods could use with condition methods chainable.
Attention: the above 7 methods should be the last chainable method.

1. Id, In

    engine.Id(1).Get(&user) // for single primary key
    // SELECT * FROM user WHERE id = 1
    engine.Id(core.PK{1, 2}).Get(&user) // for composite primary keys
    // SELECT * FROM user WHERE id1 = 1 AND id2 = 2
    engine.In("id", 1, 2, 3).Find(&users)
    // SELECT * FROM user WHERE id IN (1, 2, 3)
    engine.In("id", []int{1, 2, 3})
    // SELECT * FROM user WHERE id IN (1, 2, 3)

2. Where, And, Or

    engine.Where().And().Or().Find()
    // SELECT * FROM user WHERE (.. AND ..) OR ...

3. OrderBy, Asc, Desc

    engine.Asc().Desc().Find()
    // SELECT * FROM user ORDER BY .. ASC, .. DESC
    engine.OrderBy().Find()
    // SELECT * FROM user ORDER BY ..

4. Limit, Top

    engine.Limit().Find()
    // SELECT * FROM user LIMIT .. OFFSET ..
    engine.Top(5).Find()
    // SELECT TOP 5 * FROM user // for mssql
    // SELECT * FROM user LIMIT .. OFFSET 0 //for other databases

5. Sql, let you custom SQL

    engine.Sql("select * from user").Find()

6. Cols, Omit, Distinct

    engine.Cols("col1, col2").Find()
    // SELECT col1, col2 FROM user
    engine.Cols("col1", "col2").Where().Update(user)
    // UPDATE user set col1 = ?, col2 = ? Where ...
    engine.Omit("col1").Find()
    // SELECT col2, col3 FROM user
    engine.Omit("col1").Insert()
    // INSERT INTO table (non-col1) VALUES ()
    engine.Distinct("col1").Find()
    // SELECT DISTINCT col1 FROM user

7. Join, GroupBy, Having

    engine.GroupBy("name").Having("name='xlw'").Find()
    //SELECT * FROM user GROUP BY name HAVING name='xlw'
    engine.Join("LEFT", "userdetail", "user.id=userdetail.id").Find()
    //SELECT * FROM user LEFT JOIN userdetail ON user.id=userdetail.id

More usage, please visit http://xorm.io/docs
*/
package xorm
<|endoftext|>


<commit_before>/*
Package goon provides an autocaching interface to the app engine datastore
similar to the python NDB package.

Usage:

	type Group struct {
		Name string
	}

	func Test(w http.ResponseWriter, r *http.Request) {
		g := Group{
			Name: "test",
		}

		n := goon.NewGoon(r)

		// Create a new entity with an incomplete key and no parent.
		e, _ := n.NewEntity(nil, &g)
		fmt.Fprintln(w, "e with incomplete key:", e)

		// The kind name "Group" is fetched by reflecting on g.
		_ = n.Put(e)
		fmt.Fprintln(w, "e with key:", e)

		var g2 Group
		// Fetch it back.
		e2, _ := n.KeyGet(&g2, e.Key)
		fmt.Fprintln(w, "e2:", e2)
	}
*/
package goon
<commit_msg>More docs<commit_after>/*
Package goon provides an autocaching interface to the app engine datastore
similar to the python NDB package.

Usage:

	type Group struct {
		Name string
	}

	func Test(w http.ResponseWriter, r *http.Request) {
		g := Group{
			Name: "test",
		}

		n := goon.NewGoon(r)

		// Create a new entity with an incomplete key and no parent.
		e, _ := n.NewEntity(nil, &g)
		fmt.Fprintln(w, "e with incomplete key:", e)

		// The kind name "Group" is fetched by reflecting on g.
		_ = n.Put(e)
		fmt.Fprintln(w, "e with key:", e)

		var g2 Group
		// Fetch it back.
		e2, _ := n.KeyGet(&g2, e.Key)
		fmt.Fprintln(w, "e2:", e2)
	}

Currently in early development: do not use in production.

Submit bugs and code to: https://github.com/mjibson/goon

*/
package goon
<|endoftext|>