<commit_before>package expr

import (
	"encoding/json"
	"fmt"
	"math"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/GaryBoone/GoStats/stats"
	"github.com/MiniProfiler/go/miniprofiler"
	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/tsaf/expr/parse"
	"github.com/StackExchange/tsaf/search"
)

var Builtins = map[string]parse.Func{
	"avg": {
		[]parse.FuncType{parse.TYPE_SERIES},
		parse.TYPE_NUMBER,
		Avg,
	},
	"band": {
		[]parse.FuncType{parse.TYPE_STRING, parse.TYPE_STRING, parse.TYPE_STRING, parse.TYPE_NUMBER},
		parse.TYPE_SERIES,
		Band,
	},
	"dev": {
		[]parse.FuncType{parse.TYPE_SERIES},
		parse.TYPE_NUMBER,
		Dev,
	},
	"recent": {
		[]parse.FuncType{parse.TYPE_SERIES},
		parse.TYPE_NUMBER,
		Recent,
	},
	"since": {
		[]parse.FuncType{parse.TYPE_SERIES},
		parse.TYPE_NUMBER,
		Since,
	},
	"forecastlr": {
		[]parse.FuncType{parse.TYPE_SERIES, parse.TYPE_NUMBER},
		parse.TYPE_NUMBER,
		Forecast_lr,
	},
	"percentile": {
		[]parse.FuncType{parse.TYPE_SERIES, parse.TYPE_NUMBER},
		parse.TYPE_NUMBER,
		Percentile,
	},
	"q": {
		[]parse.FuncType{parse.TYPE_STRING, parse.TYPE_STRING},
		parse.TYPE_SERIES,
		Query,
	},
	"ungroup": {
		[]parse.FuncType{parse.TYPE_NUMBER},
		parse.TYPE_NUMBER,
		Ungroup,
	},
}

const tsdbFmt = "2006/01/02 15:04:05"

func Band(e *state, T miniprofiler.Timer, query, duration, period string, num float64) (r []*Result, err error) {
	T.Step("band", func(T miniprofiler.Timer) {
		var d, p opentsdb.Duration
		d, err = opentsdb.ParseDuration(duration)
		if err != nil {
			return
		}
		p, err = opentsdb.ParseDuration(period)
		if err != nil {
			return
		}
		if num < 1 || num > 100 {
			err = fmt.Errorf("expr: Band: num out of bounds")
		}
		q, err := opentsdb.ParseQuery(query)
		if err != nil {
			return
		}
		if err = ExpandSearch(q); err != nil {
			return
		}
		req := opentsdb.Request{
			Queries: []*opentsdb.Query{q},
		}
		now := time.Now().UTC()
		for i := 0; i < int(num); i++ {
			now = now.Add(time.Duration(-p))
			req.End = now.Unix()
			req.Start = now.Add(time.Duration(-d)).Unix()
			b, _ := json.MarshalIndent(&req, "", "  ")
			var s opentsdb.ResponseSet
			T.StepCustomTiming("tsdb", "query", string(b), func() {
				s, err = e.context.Query(req)
			})
			if err != nil {
				return
			}
			for _, res := range s {
				newarr := true
				for _, a := range r {
					if !a.Group.Equal(res.Tags) {
						continue
					}
					newarr = false
					values := a.Value.(Series)
					for k, v := range res.DPS {
						values[k] = v
					}
				}
				if newarr {
					values := make(Series)
					a := &Result{Group: res.Tags}
					for k, v := range res.DPS {
						values[k] = v
					}
					a.Value = values
					r = append(r, a)
				}
			}
		}
	})
	return
}

func Query(e *state, T miniprofiler.Timer, query, duration string) (r []*Result, err error) {
	q, err := opentsdb.ParseQuery(query)
	if err != nil {
		return
	}
	if err = ExpandSearch(q); err != nil {
		return
	}
	d, err := opentsdb.ParseDuration(duration)
	if err != nil {
		return
	}
	req := opentsdb.Request{
		Queries: []*opentsdb.Query{q},
		Start:   fmt.Sprintf("%s-ago", d),
	}
	b, _ := json.MarshalIndent(&req, "", "  ")
	var s opentsdb.ResponseSet
	T.StepCustomTiming("tsdb", "query", string(b), func() {
		s, err = e.context.Query(req)
	})
	if err != nil {
		return
	}
	for _, res := range s {
		r = append(r, &Result{
			Value: Series(res.DPS),
			Group: res.Tags,
		})
	}
	return
}

func reduce(e *state, T miniprofiler.Timer, series []*Result, F func(Series, ...float64) float64, args ...float64) (r []*Result, err error) {
	for _, s := range series {
		switch t := s.Value.(type) {
		case Series:
			if len(t) == 0 {
				// do something here?
				continue
			}
			r = append(r, &Result{
				Value: Number(F(t, args...)),
				Group: s.Group,
			})
		default:
			panic(fmt.Errorf("expr: expected a series"))
		}
	}
	return
}

func ExpandSearch(q *opentsdb.Query) error {
	for k, ov := range q.Tags {
		v := ov
		if v == "*" || !strings.Contains(v, "*") || strings.Contains(v, "|") {
			continue
		}
		v = strings.Replace(v, ".", `\.`, -1)
		v = strings.Replace(v, "*", ".*", -1)
		v = "^" + v + "$"
		re := regexp.MustCompile(v)
		var nvs []string
		vs := search.TagValuesByMetricTagKey(q.Metric, k)
		for _, nv := range vs {
			if re.MatchString(nv) {
				nvs = append(nvs, nv)
			}
		}
		if len(nvs) == 0 {
			return fmt.Errorf("expr: no tags matching %s=%s", k, ov)
		}
		q.Tags[k] = strings.Join(nvs, "|")
	}
	return nil
}

func Avg(e *state, T miniprofiler.Timer, series []*Result) ([]*Result, error) {
	return reduce(e, T, series, avg)
}

// avg returns the mean of x.
func avg(dps Series, args ...float64) (a float64) {
	for _, v := range dps {
		a += float64(v)
	}
	a /= float64(len(dps))
	return
}

func Dev(e *state, T miniprofiler.Timer, series []*Result) ([]*Result, error) {
	return reduce(e, T, series, dev)
}

// dev returns the sample standard deviation of x.
func dev(dps Series, args ...float64) (d float64) {
	a := avg(dps)
	for _, v := range dps {
		d += math.Pow(float64(v)-a, 2)
	}
	// how should we handle len(x) == 1?
	d /= float64(len(dps) - 1)
	return math.Sqrt(d)
}

func Recent(e *state, T miniprofiler.Timer, series []*Result) ([]*Result, error) {
	return reduce(e, T, series, recent)
}

func recent(dps Series, args ...float64) (a float64) {
	last := -1
	for k, v := range dps {
		d, err := strconv.Atoi(k)
		if err != nil {
			panic(err)
		}
		if d > last {
			a = float64(v)
		}
	}
	return
}

func Since(e *state, T miniprofiler.Timer, series []*Result) ([]*Result, error) {
	return reduce(e, T, series, since)
}

func since(dps Series, args ...float64) (a float64) {
	var last time.Time
	for k := range dps {
		d, err := strconv.ParseInt(k, 10, 64)
		if err != nil {
			panic(err)
		}
		t := time.Unix(d, 0)
		if t.After(last) {
			last = t
		}
	}
	s := time.Since(last)
	return s.Seconds()
}

func Forecast_lr(e *state, T miniprofiler.Timer, series []*Result, y float64) (r []*Result, err error) {
	return reduce(e, T, series, forecast_lr, y)
}

// forecast_lr returns the number of seconds a linear regression predicts the
// series will take to reach y_val.
func forecast_lr(dps Series, args ...float64) float64 {
	y_val := args[0]
	var x []float64
	var y []float64
	for k, v := range dps {
		d, err := strconv.ParseInt(k, 10, 64)
		if err != nil {
			panic(err)
		}
		x = append(x, float64(d))
		y = append(y, float64(v))
	}
	var slope, intercept, _, _, _, _ = stats.LinearRegression(x, y)
	// If the slope is basically 0, return -1 since forecast alerts wouldn't care about things that
	// "already happened". There might be a better way to handle this, but this works for now
	if int64(slope) == 0 {
		return -1
	}
	intercept_time := (y_val - intercept) / slope
	t := time.Unix(int64(intercept_time), 0)
	s := time.Since(t)
	return -s.Seconds()
}

func Percentile(e *state, T miniprofiler.Timer, series []*Result, p float64) (r []*Result, err error) {
	return reduce(e, T, series, percentile, p)
}

// percentile returns the value at the corresponding percentile between 0 and 1.
// Min and Max can be simulated using p <= 0 and p >= 1, respectively.
func percentile(dps Series, args ...float64) (a float64) {
	p := args[0]
	var x []float64
	for _, v := range dps {
		x = append(x, float64(v))
	}
	sort.Float64s(x)
	if p <= 0 {
		return x[0]
	}
	if p >= 1 {
		return x[len(x)-1]
	}
	i := p * float64(len(x)-1)
	i = math.Ceil(i)
	return x[int(i)]
}

func Ungroup(e *state, T miniprofiler.Timer, d []*Result) ([]*Result, error) {
	if len(d) > 1 {
		return nil, fmt.Errorf("ungroup: more than 1 group not supported")
	}
	for _, v := range d {
		v.Group = nil
	}
	return d, nil
}
<commit_msg>Add a sum reduction function<commit_after>package expr

import (
	"encoding/json"
	"fmt"
	"math"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/GaryBoone/GoStats/stats"
	"github.com/MiniProfiler/go/miniprofiler"
	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/tsaf/expr/parse"
	"github.com/StackExchange/tsaf/search"
)

var Builtins = map[string]parse.Func{
	"avg": {
		[]parse.FuncType{parse.TYPE_SERIES},
		parse.TYPE_NUMBER,
		Avg,
	},
	"sum": {
		[]parse.FuncType{parse.TYPE_SERIES},
		parse.TYPE_NUMBER,
		Sum,
	},
	"band": {
		[]parse.FuncType{parse.TYPE_STRING, parse.TYPE_STRING, parse.TYPE_STRING, parse.TYPE_NUMBER},
		parse.TYPE_SERIES,
		Band,
	},
	"dev": {
		[]parse.FuncType{parse.TYPE_SERIES},
		parse.TYPE_NUMBER,
		Dev,
	},
	"recent": {
		[]parse.FuncType{parse.TYPE_SERIES},
		parse.TYPE_NUMBER,
		Recent,
	},
	"since": {
		[]parse.FuncType{parse.TYPE_SERIES},
		parse.TYPE_NUMBER,
		Since,
	},
	"forecastlr": {
		[]parse.FuncType{parse.TYPE_SERIES, parse.TYPE_NUMBER},
		parse.TYPE_NUMBER,
		Forecast_lr,
	},
	"percentile": {
		[]parse.FuncType{parse.TYPE_SERIES, parse.TYPE_NUMBER},
		parse.TYPE_NUMBER,
		Percentile,
	},
	"q": {
		[]parse.FuncType{parse.TYPE_STRING, parse.TYPE_STRING},
		parse.TYPE_SERIES,
		Query,
	},
	"ungroup": {
		[]parse.FuncType{parse.TYPE_NUMBER},
		parse.TYPE_NUMBER,
		Ungroup,
	},
}

const tsdbFmt = "2006/01/02 15:04:05"

func Band(e *state, T miniprofiler.Timer, query, duration, period string, num float64) (r []*Result, err error) {
	T.Step("band", func(T miniprofiler.Timer) {
		var d, p opentsdb.Duration
		d, err = opentsdb.ParseDuration(duration)
		if err != nil {
			return
		}
		p, err = opentsdb.ParseDuration(period)
		if err != nil {
			return
		}
		if num < 1 || num > 100 {
			err = fmt.Errorf("expr: Band: num out of bounds")
		}
		q, err := opentsdb.ParseQuery(query)
		if err != nil {
			return
		}
		if err = ExpandSearch(q); err != nil {
			return
		}
		req := opentsdb.Request{
			Queries: []*opentsdb.Query{q},
		}
		now := time.Now().UTC()
		for i := 0; i < int(num); i++ {
			now = now.Add(time.Duration(-p))
			req.End = now.Unix()
			req.Start = now.Add(time.Duration(-d)).Unix()
			b, _ := json.MarshalIndent(&req, "", "  ")
			var s opentsdb.ResponseSet
			T.StepCustomTiming("tsdb", "query", string(b), func() {
				s, err = e.context.Query(req)
			})
			if err != nil {
				return
			}
			for _, res := range s {
				newarr := true
				for _, a := range r {
					if !a.Group.Equal(res.Tags) {
						continue
					}
					newarr = false
					values := a.Value.(Series)
					for k, v := range res.DPS {
						values[k] = v
					}
				}
				if newarr {
					values := make(Series)
					a := &Result{Group: res.Tags}
					for k, v := range res.DPS {
						values[k] = v
					}
					a.Value = values
					r = append(r, a)
				}
			}
		}
	})
	return
}

func Query(e *state, T miniprofiler.Timer, query, duration string) (r []*Result, err error) {
	q, err := opentsdb.ParseQuery(query)
	if err != nil {
		return
	}
	if err = ExpandSearch(q); err != nil {
		return
	}
	d, err := opentsdb.ParseDuration(duration)
	if err != nil {
		return
	}
	req := opentsdb.Request{
		Queries: []*opentsdb.Query{q},
		Start:   fmt.Sprintf("%s-ago", d),
	}
	b, _ := json.MarshalIndent(&req, "", "  ")
	var s opentsdb.ResponseSet
	T.StepCustomTiming("tsdb", "query", string(b), func() {
		s, err = e.context.Query(req)
	})
	if err != nil {
		return
	}
	for _, res := range s {
		r = append(r, &Result{
			Value: Series(res.DPS),
			Group: res.Tags,
		})
	}
	return
}

func reduce(e *state, T miniprofiler.Timer, series []*Result, F func(Series, ...float64) float64, args ...float64) (r []*Result, err error) {
	for _, s := range series {
		switch t := s.Value.(type) {
		case Series:
			if len(t) == 0 {
				// do something here?
				continue
			}
			r = append(r, &Result{
				Value: Number(F(t, args...)),
				Group: s.Group,
			})
		default:
			panic(fmt.Errorf("expr: expected a series"))
		}
	}
	return
}

func ExpandSearch(q *opentsdb.Query) error {
	for k, ov := range q.Tags {
		v := ov
		if v == "*" || !strings.Contains(v, "*") || strings.Contains(v, "|") {
			continue
		}
		v = strings.Replace(v, ".", `\.`, -1)
		v = strings.Replace(v, "*", ".*", -1)
		v = "^" + v + "$"
		re := regexp.MustCompile(v)
		var nvs []string
		vs := search.TagValuesByMetricTagKey(q.Metric, k)
		for _, nv := range vs {
			if re.MatchString(nv) {
				nvs = append(nvs, nv)
			}
		}
		if len(nvs) == 0 {
			return fmt.Errorf("expr: no tags matching %s=%s", k, ov)
		}
		q.Tags[k] = strings.Join(nvs, "|")
	}
	return nil
}

func Avg(e *state, T miniprofiler.Timer, series []*Result) ([]*Result, error) {
	return reduce(e, T, series, avg)
}

// avg returns the mean of x.
func avg(dps Series, args ...float64) (a float64) {
	for _, v := range dps {
		a += float64(v)
	}
	a /= float64(len(dps))
	return
}

func Sum(e *state, T miniprofiler.Timer, series []*Result) ([]*Result, error) {
	return reduce(e, T, series, sum)
}

func sum(dps Series, args ...float64) (a float64) {
	for _, v := range dps {
		a += float64(v)
	}
	return
}

func Dev(e *state, T miniprofiler.Timer, series []*Result) ([]*Result, error) {
	return reduce(e, T, series, dev)
}

// dev returns the sample standard deviation of x.
func dev(dps Series, args ...float64) (d float64) {
	a := avg(dps)
	for _, v := range dps {
		d += math.Pow(float64(v)-a, 2)
	}
	// how should we handle len(x) == 1?
	d /= float64(len(dps) - 1)
	return math.Sqrt(d)
}

func Recent(e *state, T miniprofiler.Timer, series []*Result) ([]*Result, error) {
	return reduce(e, T, series, recent)
}

func recent(dps Series, args ...float64) (a float64) {
	last := -1
	for k, v := range dps {
		d, err := strconv.Atoi(k)
		if err != nil {
			panic(err)
		}
		if d > last {
			a = float64(v)
		}
	}
	return
}

func Since(e *state, T miniprofiler.Timer, series []*Result) ([]*Result, error) {
	return reduce(e, T, series, since)
}

func since(dps Series, args ...float64) (a float64) {
	var last time.Time
	for k := range dps {
		d, err := strconv.ParseInt(k, 10, 64)
		if err != nil {
			panic(err)
		}
		t := time.Unix(d, 0)
		if t.After(last) {
			last = t
		}
	}
	s := time.Since(last)
	return s.Seconds()
}

func Forecast_lr(e *state, T miniprofiler.Timer, series []*Result, y float64) (r []*Result, err error) {
	return reduce(e, T, series, forecast_lr, y)
}

// forecast_lr returns the number of seconds a linear regression predicts the
// series will take to reach y_val.
func forecast_lr(dps Series, args ...float64) float64 {
	y_val := args[0]
	var x []float64
	var y []float64
	for k, v := range dps {
		d, err := strconv.ParseInt(k, 10, 64)
		if err != nil {
			panic(err)
		}
		x = append(x, float64(d))
		y = append(y, float64(v))
	}
	var slope, intercept, _, _, _, _ = stats.LinearRegression(x, y)
	// If the slope is basically 0, return -1 since forecast alerts wouldn't care about things that
	// "already happened". There might be a better way to handle this, but this works for now
	if int64(slope) == 0 {
		return -1
	}
	intercept_time := (y_val - intercept) / slope
	t := time.Unix(int64(intercept_time), 0)
	s := time.Since(t)
	return -s.Seconds()
}

func Percentile(e *state, T miniprofiler.Timer, series []*Result, p float64) (r []*Result, err error) {
	return reduce(e, T, series, percentile, p)
}

// percentile returns the value at the corresponding percentile between 0 and 1.
// Min and Max can be simulated using p <= 0 and p >= 1, respectively.
func percentile(dps Series, args ...float64) (a float64) {
	p := args[0]
	var x []float64
	for _, v := range dps {
		x = append(x, float64(v))
	}
	sort.Float64s(x)
	if p <= 0 {
		return x[0]
	}
	if p >= 1 {
		return x[len(x)-1]
	}
	i := p * float64(len(x)-1)
	i = math.Ceil(i)
	return x[int(i)]
}

func Ungroup(e *state, T miniprofiler.Timer, d []*Result) ([]*Result, error) {
	if len(d) > 1 {
		return nil, fmt.Errorf("ungroup: more than 1 group not supported")
	}
	for _, v := range d {
		v.Group = nil
	}
	return d, nil
}
<|endoftext|>


<commit_before>package gitbackend

import (
	"bytes"
	"fmt"
	"github.com/libgit2/git2go"
	"io"
	"io/ioutil"
	"strings"
)

type FileStore struct {
	repo *git.Repository
}

func NewFileStore(path string, isBare bool) (fs FileStore, err error) {
	repo, err := git.InitRepository(path, isBare)
	if err != nil {
		return
	}
	fs.repo = repo
	return
}

func (f *FileStore) ReadDir(path string) (list []FileInfo, err error) {
	if strings.Trim(path, "/ ") == "" {
		return f.readRootDir()
	} else {
		return f.readSubDir(path)
	}
}

func (f *FileStore) readRootDir() (list []FileInfo, err error) {
	headCommitTree, err, noHead := f.headCommitTree()
	if err != nil {
		// return empty list for newly initialized repository without proper HEAD
		// usually the first commit sets a proper HEAD
		// this is only necessary for the root directory since there are no files after init
		if noHead {
			err = nil
		}
		return
	}
	list = f.listTree(headCommitTree)
	return
}

func (f *FileStore) readSubDir(path string) (list []FileInfo, err error) {
	headCommitTree, err, _ := f.headCommitTree()
	if err != nil {
		return
	}

	entry, err := headCommitTree.EntryByPath(path)
	if err != nil {
		return
	}
	tree, err := f.repo.LookupTree(entry.Id)
	if err != nil {
		return
	}

	list = f.listTree(tree)
	return
}

func (f *FileStore) listTree(tree *git.Tree) (list []FileInfo) {
	var i uint64
	for i = 0; i < tree.EntryCount(); i++ {
		entry := tree.EntryByIndex(i)
		isDir := entry.Type == git.ObjectTree
		list = append(list, FileInfo{entry.Name, isDir})
	}
	return
}

func (f *FileStore) Checksum(path string) (hexdigest string, err error) {
	headCommitTree, err, _ := f.headCommitTree()
	if err != nil {
		fmt.Println(err)
		return
	}
	entry, err := headCommitTree.EntryByPath(path)
	if err != nil {
		return
	}
	hexdigest = entry.Id.String()
	return
}

func (f *FileStore) ReadFile(path string) (reader io.Reader, err error) {
	headCommitTree, err, _ := f.headCommitTree()
	if err != nil {
		fmt.Println(err)
		return
	}
	entry, err := headCommitTree.EntryByPath(path)
	if err != nil {
		return
	}
	blob, err := f.repo.LookupBlob(entry.Id)
	if err != nil {
		return
	}
	reader = bytes.NewBuffer(blob.Contents())
	return
}

func (f *FileStore) WriteFile(path string, reader io.Reader, commitInfo CommitInfo) (err error) {
	blobOid, err := f.writeData(reader)

	if err != nil {
		fmt.Println(err)
		return
	}

	oldTree, err, _ := f.headCommitTree()
	if err != nil {
		fmt.Println(err)
		return
	}

	newTreeId, err := f.updateTree(oldTree, path, blobOid)
	if err != nil {
		fmt.Println(err)
		return
	}

	tree, err := f.repo.LookupTree(newTreeId)
	if err != nil {
		fmt.Println(err)
		return
	}

	sig := &git.Signature{
		Name:  commitInfo.AuthorName(),
		Email: commitInfo.AuthorEmail(),
		When:  commitInfo.Time(),
	}

	commit, err, _ := f.headCommit()
	if err != nil {
		return
	}

	_, err = f.repo.CreateCommit("HEAD", sig, sig, commitInfo.Message(), tree, commit)
	if err != nil {
		fmt.Println(err)
		return
	}
	return
}

func (f *FileStore) writeData(reader io.Reader) (blobOid *git.Oid, err error) {
	odb, err := f.repo.Odb()
	if err != nil {
		fmt.Println(err)
		return
	}

	data, err := ioutil.ReadAll(reader)
	if err != nil {
		fmt.Println(err)
		return
	}

	blobOid, err = odb.Write(data, git.ObjectBlob)
	return
}

func (f *FileStore) updateTree(oldParentTree *git.Tree, path string, blobOid *git.Oid) (oid *git.Oid, err error) {
	treebuilder, err := f.repo.TreeBuilderFromTree(oldParentTree)
	if err != nil {
		fmt.Println(err)
		return
	}
	parts := strings.SplitN(path, "/", 2)
	if len(parts) == 1 {
		err = treebuilder.Insert(parts[0], blobOid, int(git.FilemodeBlob))
		if err != nil {
			fmt.Println(err)
			return
		}
		return treebuilder.Write()
	}

	newTreeOid, err := treebuilder.Write()
	if err != nil {
		fmt.Println(err)
		return
	}
	newTree, err := f.repo.LookupTree(newTreeOid)
	if err != nil {
		fmt.Println(err)
		return
	}

	oldChildTreeTreeEntry := newTree.EntryByName(parts[0])
	if oldChildTreeTreeEntry == nil {
		err = fmt.Errorf("Could not find Entry by Name %s", parts[0])
		return
	}
	oldChildTree, err2 := f.repo.LookupTree(oldChildTreeTreeEntry.Id)
	if err2 != nil {
		fmt.Println(err2)
		err = err2
		return
	}
	childTreeOid, err2 := f.updateTree(oldChildTree, parts[1], blobOid)
	if err2 != nil {
		fmt.Println(err2)
		err = err2
		return
	}
	err = treebuilder.Insert(parts[0], childTreeOid, int(git.FilemodeTree))
	if err != nil {
		fmt.Println(err)
		return
	}

	oid, err = treebuilder.Write()
	return
}

func (f *FileStore) headCommitTree() (tree *git.Tree, err error, noHead bool) {
	commit, err, noHead := f.headCommit()
	if err != nil {
		return
	}
	tree, err = commit.Tree()
	return
}

func (f *FileStore) headCommit() (commit *git.Commit, err error, noHead bool) {
	oid, err, noHead := f.headCommitId()
	if err != nil {
		return
	}
	commit, err = f.repo.LookupCommit(oid)
	return
}

func (f *FileStore) headCommitId() (oid *git.Oid, err error, noHead bool) {
	headRef, err := f.repo.LookupReference("HEAD")
	if err != nil {
		return
	}
	ref, err := headRef.Resolve()
	if err != nil {
		noHead = true
		return
	}
	oid = ref.Target()
	if oid == nil {
		err = fmt.Errorf("Could not get Target for HEAD(%s)\n", oid.String())
	}
	return
}
<commit_msg>Refactor variable names.<commit_after>package gitbackend

import (
	"bytes"
	"fmt"
	"github.com/libgit2/git2go"
	"io"
	"io/ioutil"
	"strings"
)

type FileStore struct {
	repo *git.Repository
}

func NewFileStore(path string, isBare bool) (fileStore FileStore, err error) {
	repo, err := git.InitRepository(path, isBare)
	if err != nil {
		return
	}
	fileStore.repo = repo
	return
}

func (this *FileStore) ReadDir(path string) (list []FileInfo, err error) {
	if strings.Trim(path, "/ ") == "" {
		return this.readRootDir()
	} else {
		return this.readSubDir(path)
	}
}

func (this *FileStore) readRootDir() (list []FileInfo, err error) {
	headCommitTree, err, noHead := this.headCommitTree()
	if err != nil {
		// return empty list for newly initialized repository without proper HEAD
		// usually the first commit sets a proper HEAD
		// this is only necessary for the root directory since there are no files after init
		if noHead {
			err = nil
		}
		return
	}
	list = this.listTree(headCommitTree)
	return
}

func (this *FileStore) readSubDir(path string) (list []FileInfo, err error) {
	headCommitTree, err, _ := this.headCommitTree()
	if err != nil {
		return
	}

	entry, err := headCommitTree.EntryByPath(path)
	if err != nil {
		return
	}
	tree, err := this.repo.LookupTree(entry.Id)
	if err != nil {
		return
	}

	list = this.listTree(tree)
	return
}

func (this *FileStore) listTree(tree *git.Tree) (list []FileInfo) {
	var i uint64
	for i = 0; i < tree.EntryCount(); i++ {
		entry := tree.EntryByIndex(i)
		isDir := entry.Type == git.ObjectTree
		list = append(list, FileInfo{entry.Name, isDir})
	}
	return
}

func (this *FileStore) Checksum(path string) (hexdigest string, err error) {
	headCommitTree, err, _ := this.headCommitTree()
	if err != nil {
		fmt.Println(err)
		return
	}
	entry, err := headCommitTree.EntryByPath(path)
	if err != nil {
		return
	}
	hexdigest = entry.Id.String()
	return
}

func (this *FileStore) ReadFile(path string) (reader io.Reader, err error) {
	headCommitTree, err, _ := this.headCommitTree()
	if err != nil {
		fmt.Println(err)
		return
	}
	entry, err := headCommitTree.EntryByPath(path)
	if err != nil {
		return
	}
	blob, err := this.repo.LookupBlob(entry.Id)
	if err != nil {
		return
	}
	reader = bytes.NewBuffer(blob.Contents())
	return
}

func (this *FileStore) WriteFile(path string, reader io.Reader, commitInfo CommitInfo) (err error) {
	blobOid, err := this.writeData(reader)

	if err != nil {
		fmt.Println(err)
		return
	}

	oldTree, err, _ := this.headCommitTree()
	if err != nil {
		fmt.Println(err)
		return
	}

	newTreeId, err := this.updateTree(oldTree, path, blobOid)
	if err != nil {
		fmt.Println(err)
		return
	}

	tree, err := this.repo.LookupTree(newTreeId)
	if err != nil {
		fmt.Println(err)
		return
	}

	sig := &git.Signature{
		Name:  commitInfo.AuthorName(),
		Email: commitInfo.AuthorEmail(),
		When:  commitInfo.Time(),
	}

	commit, err, _ := this.headCommit()
	if err != nil {
		return
	}

	_, err = this.repo.CreateCommit("HEAD", sig, sig, commitInfo.Message(), tree, commit)
	if err != nil {
		fmt.Println(err)
		return
	}
	return
}

func (this *FileStore) writeData(reader io.Reader) (blobOid *git.Oid, err error) {
	odb, err := this.repo.Odb()
	if err != nil {
		fmt.Println(err)
		return
	}

	data, err := ioutil.ReadAll(reader)
	if err != nil {
		fmt.Println(err)
		return
	}

	blobOid, err = odb.Write(data, git.ObjectBlob)
	return
}

func (this *FileStore) updateTree(oldParentTree *git.Tree, path string, blobOid *git.Oid) (oid *git.Oid, err error) {
	treebuilder, err := this.repo.TreeBuilderFromTree(oldParentTree)
	if err != nil {
		fmt.Println(err)
		return
	}
	parts := strings.SplitN(path, "/", 2)
	if len(parts) == 1 {
		err = treebuilder.Insert(parts[0], blobOid, int(git.FilemodeBlob))
		if err != nil {
			fmt.Println(err)
			return
		}
		return treebuilder.Write()
	}

	newTreeOid, err := treebuilder.Write()
	if err != nil {
		fmt.Println(err)
		return
	}
	newTree, err := this.repo.LookupTree(newTreeOid)
	if err != nil {
		fmt.Println(err)
		return
	}

	oldChildTreeTreeEntry := newTree.EntryByName(parts[0])
	if oldChildTreeTreeEntry == nil {
		err = fmt.Errorf("Could not find Entry by Name %s", parts[0])
		return
	}
	oldChildTree, err2 := this.repo.LookupTree(oldChildTreeTreeEntry.Id)
	if err2 != nil {
		fmt.Println(err2)
		err = err2
		return
	}
	childTreeOid, err2 := this.updateTree(oldChildTree, parts[1], blobOid)
	if err2 != nil {
		fmt.Println(err2)
		err = err2
		return
	}
	err = treebuilder.Insert(parts[0], childTreeOid, int(git.FilemodeTree))
	if err != nil {
		fmt.Println(err)
		return
	}

	oid, err = treebuilder.Write()
	return
}

func (this *FileStore) headCommitTree() (tree *git.Tree, err error, noHead bool) {
	commit, err, noHead := this.headCommit()
	if err != nil {
		return
	}
	tree, err = commit.Tree()
	return
}

func (this *FileStore) headCommit() (commit *git.Commit, err error, noHead bool) {
	oid, err, noHead := this.headCommitId()
	if err != nil {
		return
	}
	commit, err = this.repo.LookupCommit(oid)
	return
}

func (this *FileStore) headCommitId() (oid *git.Oid, err error, noHead bool) {
	headRef, err := this.repo.LookupReference("HEAD")
	if err != nil {
		return
	}
	ref, err := headRef.Resolve()
	if err != nil {
		noHead = true
		return
	}
	oid = ref.Target()
	if oid == nil {
		err = fmt.Errorf("Could not get Target for HEAD(%s)\n", oid.String())
	}
	return
}
<|endoftext|>


<commit_before>package ics

func escape(s string) []byte {
	p := make([]byte, 0, len(s))
	for i := 0; i < len(s); i++ {
		switch s[i] {
		case '\\':
			p = append(p, '\\', '\\')
		case ';':
			p = append(p, '\\', ';')
		case ',':
			p = append(p, '\\', ',')
		case '\n':
			p = append(p, '\\', 'n')
		default:
			p = append(p, s[i])
		}
	}
	return p
}

func unescape(p []byte) []byte {
	u := p[:0]
	for i := 0; i < len(p); i++ {
		if p[i] == '\\' && i+1 < len(p) {
			i++
			switch p[i] {
			case '\\':
				u = append(u, '\\')
			case ';':
				u = append(u, ';')
			case ',':
				u = append(u, ',')
			case 'N', 'n':
				u = append(u, '\n')
			default:
				u = append(u, p[i])
			}
		} else {
			u = append(u, p[i])
		}
	}
}

func escape6868(s []byte) []byte {
	p := make([]byte, 0, len(s))
	for i := 0; i < len(s); i++ {
		switch s[i] {
		case '\n':
			p = append(p, '^', 'n')
		case '^':
			p = append(p, '^', '^')
		case '"':
			p = append(p, '^', '\'')
		default:
			p = append(p, s[i])
		}
	}
	return p
}

func unescape6868(p []byte) []byte {
	u := p[:0]
	for i := 0; i < len(p); i++ {
		if p[i] == '^' && i+1 < len(p) {
			i++
			switch p[i] {
			case 'n':
				u = append(u, '\n') //crlf on windows?
			case '^':
				u = append(u, '^')
			case '\'':
				u = append(u, '"')
			default:
				u = append(u, '^', p[i])
			}
		} else {
			u = append(u, p[i])
		}
	}
	return u
}
<commit_msg>changed []byte -> string<commit_after>package ics

func escape(s string) []byte {
	p := make([]byte, 0, len(s))
	for i := 0; i < len(s); i++ {
		switch s[i] {
		case '\\':
			p = append(p, '\\', '\\')
		case ';':
			p = append(p, '\\', ';')
		case ',':
			p = append(p, '\\', ',')
		case '\n':
			p = append(p, '\\', 'n')
		default:
			p = append(p, s[i])
		}
	}
	return p
}

func unescape(p []byte) []byte {
	u := p[:0]
	for i := 0; i < len(p); i++ {
		if p[i] == '\\' && i+1 < len(p) {
			i++
			switch p[i] {
			case '\\':
				u = append(u, '\\')
			case ';':
				u = append(u, ';')
			case ',':
				u = append(u, ',')
			case 'N', 'n':
				u = append(u, '\n')
			default:
				u = append(u, p[i])
			}
		} else {
			u = append(u, p[i])
		}
	}
}

func escape6868(s string) []byte {
	p := make([]byte, 0, len(s))
	for i := 0; i < len(s); i++ {
		switch s[i] {
		case '\n':
			p = append(p, '^', 'n')
		case '^':
			p = append(p, '^', '^')
		case '"':
			p = append(p, '^', '\'')
		default:
			p = append(p, s[i])
		}
	}
	return p
}

func unescape6868(p []byte) []byte {
	u := p[:0]
	for i := 0; i < len(p); i++ {
		if p[i] == '^' && i+1 < len(p) {
			i++
			switch p[i] {
			case 'n':
				u = append(u, '\n') //crlf on windows?
			case '^':
				u = append(u, '^')
			case '\'':
				u = append(u, '"')
			default:
				u = append(u, '^', p[i])
			}
		} else {
			u = append(u, p[i])
		}
	}
	return u
}
<|endoftext|>


<commit_before>// +build linux

package main

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"syscall"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/opencontainers/runc/libcontainer"
	"github.com/opencontainers/runc/libcontainer/configs"
	"github.com/opencontainers/specs/specs-go"
)

const wildcard = -1

var errEmptyID = errors.New("container id cannot be empty")

var allowedDevices = []*configs.Device{
	// allow mknod for any device
	{
		Type:        'c',
		Major:       wildcard,
		Minor:       wildcard,
		Permissions: "m",
		Allow:       true,
	},
	{
		Type:        'b',
		Major:       wildcard,
		Minor:       wildcard,
		Permissions: "m",
		Allow:       true,
	},
	{
		Type:        'c',
		Path:        "/dev/null",
		Major:       1,
		Minor:       3,
		Permissions: "rwm",
		Allow:       true,
	},
	{
		Type:        'c',
		Path:        "/dev/random",
		Major:       1,
		Minor:       8,
		Permissions: "rwm",
		Allow:       true,
	},
	{
		Type:        'c',
		Path:        "/dev/full",
		Major:       1,
		Minor:       7,
		Permissions: "rwm",
		Allow:       true,
	},
	{
		Type:        'c',
		Path:        "/dev/tty",
		Major:       5,
		Minor:       0,
		Permissions: "rwm",
		Allow:       true,
	},
	{
		Type:        'c',
		Path:        "/dev/zero",
		Major:       1,
		Minor:       5,
		Permissions: "rwm",
		Allow:       true,
	},
	{
		Type:        'c',
		Path:        "/dev/urandom",
		Major:       1,
		Minor:       9,
		Permissions: "rwm",
		Allow:       true,
	},
	{
		Path:        "/dev/console",
		Type:        'c',
		Major:       5,
		Minor:       1,
		Permissions: "rwm",
		Allow:       true,
	},
	// /dev/pts/ - pts namespaces are "coming soon"
	{
		Path:        "",
		Type:        'c',
		Major:       136,
		Minor:       wildcard,
		Permissions: "rwm",
		Allow:       true,
	},
	{
		Path:        "",
		Type:        'c',
		Major:       5,
		Minor:       2,
		Permissions: "rwm",
		Allow:       true,
	},
	// tuntap
	{
		Path:        "",
		Type:        'c',
		Major:       10,
		Minor:       200,
		Permissions: "rwm",
		Allow:       true,
	},
}

var (
	maskedPaths = []string{
		"/proc/kcore",
		"/proc/latency_stats",
		"/proc/timer_stats",
		"/proc/sched_debug",
	}
	readonlyPaths = []string{
		"/proc/asound",
		"/proc/bus",
		"/proc/fs",
		"/proc/irq",
		"/proc/sys",
		"/proc/sysrq-trigger",
	}
)

var container libcontainer.Container

func containerPreload(context *cli.Context) error {
	c, err := getContainer(context)
	if err != nil {
		return err
	}
	container = c
	return nil
}

// loadFactory returns the configured factory instance for execing containers.
func loadFactory(context *cli.Context) (libcontainer.Factory, error) {
	root := context.GlobalString("root")
	abs, err := filepath.Abs(root)
	if err != nil {
		return nil, err
	}
	return libcontainer.New(abs, libcontainer.Cgroupfs, func(l *libcontainer.LinuxFactory) error {
		l.CriuPath = context.GlobalString("criu")
		return nil
	})
}

// getContainer returns the specified container instance by loading it from state
// with the default factory.
func getContainer(context *cli.Context) (libcontainer.Container, error) {
	id := context.Args().First()
	if id == "" {
		return nil, errEmptyID
	}
	factory, err := loadFactory(context)
	if err != nil {
		return nil, err
	}
	return factory.Load(id)
}

// fatal prints the error's details if it is a libcontainer specific error type
// then exits the program with an exit status of 1.
func fatal(err error) {
	// make sure the error is written to the logger
	logrus.Error(err)
	fmt.Fprintln(os.Stderr, err)
	os.Exit(1)
}

func fatalf(t string, v ...interface{}) {
	fatal(fmt.Errorf(t, v...))
}

func getDefaultImagePath(context *cli.Context) string {
	cwd, err := os.Getwd()
	if err != nil {
		panic(err)
	}
	return filepath.Join(cwd, "checkpoint")
}

// newProcess returns a new libcontainer Process with the arguments from the
// spec and stdio from the current process.
func newProcess(p specs.Process) (*libcontainer.Process, error) {
	lp := &libcontainer.Process{
		Args: p.Args,
		Env:  p.Env,
		// TODO: fix libcontainer's API to better support uid/gid in a typesafe way.
		User:            fmt.Sprintf("%d:%d", p.User.UID, p.User.GID),
		Cwd:             p.Cwd,
		Capabilities:    p.Capabilities,
		Label:           p.SelinuxLabel,
		NoNewPrivileges: &p.NoNewPrivileges,
		AppArmorProfile: p.ApparmorProfile,
	}
	for _, rlimit := range p.Rlimits {
		rl, err := createLibContainerRlimit(rlimit)
		if err != nil {
			return nil, err
		}
		lp.Rlimits = append(lp.Rlimits, rl)
	}
	return lp, nil
}

func dupStdio(process *libcontainer.Process, rootuid int) error {
	process.Stdin = os.Stdin
	process.Stdout = os.Stdout
	process.Stderr = os.Stderr
	for _, fd := range []uintptr{
		os.Stdin.Fd(),
		os.Stdout.Fd(),
		os.Stderr.Fd(),
	} {
		if err := syscall.Fchown(int(fd), rootuid, rootuid); err != nil {
			return err
		}
	}
	return nil
}

// If systemd is supporting sd_notify protocol, this function will add support
// for sd_notify protocol from within the container.
func setupSdNotify(spec *specs.Spec, notifySocket string) {
	spec.Mounts = append(spec.Mounts, specs.Mount{Destination: notifySocket, Type: "bind", Source: notifySocket, Options: []string{"bind"}})
	spec.Process.Env = append(spec.Process.Env, fmt.Sprintf("NOTIFY_SOCKET=%s", notifySocket))
}

func destroy(container libcontainer.Container) {
	if err := container.Destroy(); err != nil {
		logrus.Error(err)
	}
}

// setupIO sets the proper IO on the process depending on the configuration
// If there is a nil error then there must be a non nil tty returned
func setupIO(process *libcontainer.Process, rootuid int, console string, createTTY, detach bool) (*tty, error) {
	// detach and createTty will not work unless a console path is passed
	// so error out here before changing any terminal settings
	if createTTY && detach && console == "" {
		return nil, fmt.Errorf("cannot allocate tty if runc will detach")
	}
	if createTTY {
		return createTty(process, rootuid, console)
	}
	if detach {
		if err := dupStdio(process, rootuid); err != nil {
			return nil, err
		}
		return &tty{}, nil
	}
	return createStdioPipes(process, rootuid)
}

// createPidFile creates a file with the processes pid inside it atomically
// it creates a temp file with the paths filename + '.' infront of it
// then renames the file
func createPidFile(path string, process *libcontainer.Process) error {
	pid, err := process.Pid()
	if err != nil {
		return err
	}
	var (
		tmpDir  = filepath.Dir(path)
		tmpName = filepath.Join(tmpDir, fmt.Sprintf(".%s", filepath.Base(path)))
	)
	f, err := os.OpenFile(tmpName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
	if err != nil {
		return err
	}
	_, err = fmt.Fprintf(f, "%d", pid)
	f.Close()
	if err != nil {
		return err
	}
	return os.Rename(tmpName, path)
}

func createContainer(context *cli.Context, id string, spec *specs.Spec) (libcontainer.Container, error) {
	config, err := createLibcontainerConfig(id, spec)
	if err != nil {
		return nil, err
	}

	if _, err := os.Stat(config.Rootfs); err != nil {
		if os.IsNotExist(err) {
			return nil, fmt.Errorf("rootfs (%q) does not exist", config.Rootfs)
		}
		return nil, err
	}

	factory, err := loadFactory(context)
	if err != nil {
		return nil, err
	}
	return factory.Create(id, config)
}

type runner struct {
	enableSubreaper bool
	shouldDestroy   bool
	detach          bool
	listenFDs       []*os.File
	pidFile         string
	console         string
	container       libcontainer.Container
}

func (r *runner) run(config *specs.Process) (int, error) {
	process, err := newProcess(*config)
	if err != nil {
		r.destroy()
		return -1, err
	}
	if len(r.listenFDs) > 0 {
		process.Env = append(process.Env, fmt.Sprintf("LISTEN_FDS=%d", len(r.listenFDs)), "LISTEN_PID=1")
		process.ExtraFiles = append(process.ExtraFiles, r.listenFDs...)
	}
	rootuid, err := r.container.Config().HostUID()
	if err != nil {
		r.destroy()
		return -1, err
	}
	tty, err := setupIO(process, rootuid, r.console, config.Terminal, r.detach)
	if err != nil {
		r.destroy()
		return -1, err
	}
	handler := newSignalHandler(tty, r.enableSubreaper)
	if err := r.container.Start(process); err != nil {
		r.destroy()
		tty.Close()
		return -1, err
	}
	if err := tty.ClosePostStart(); err != nil {
		r.terminate(process)
		r.destroy()
		tty.Close()
		return -1, err
	}
	if r.pidFile != "" {
		if err := createPidFile(r.pidFile, process); err != nil {
			r.terminate(process)
			r.destroy()
			tty.Close()
			return -1, err
		}
	}
	if r.detach {
		tty.Close()
		return 0, nil
	}
	status, err := handler.forward(process)
	if err != nil {
		r.terminate(process)
	}
	r.destroy()
	tty.Close()
	return status, err
}

func (r *runner) destroy() {
	if r.shouldDestroy {
		destroy(r.container)
	}
}

func (r *runner) terminate(p *libcontainer.Process) {
	p.Signal(syscall.SIGKILL)
	p.Wait()
}

func validateProcessSpec(spec *specs.Process) error {
	if spec.Cwd == "" {
		return fmt.Errorf("Cwd property must not be empty")
	}
	if !filepath.IsAbs(spec.Cwd) {
		return fmt.Errorf("Cwd must be an absolute path")
	}
	if len(spec.Args) == 0 {
		return fmt.Errorf("args must not be empty")
	}
	return nil
}
<commit_msg>Sync on the pid file to ensure the write is persisted<commit_after>// +build linux

package main

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"syscall"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/opencontainers/runc/libcontainer"
	"github.com/opencontainers/runc/libcontainer/configs"
	"github.com/opencontainers/specs/specs-go"
)

const wildcard = -1

var errEmptyID = errors.New("container id cannot be empty")

var allowedDevices = []*configs.Device{
	// allow mknod for any device
	{
		Type:        'c',
		Major:       wildcard,
		Minor:       wildcard,
		Permissions: "m",
		Allow:       true,
	},
	{
		Type:        'b',
		Major:       wildcard,
		Minor:       wildcard,
		Permissions: "m",
		Allow:       true,
	},
	{
		Type:        'c',
		Path:        "/dev/null",
		Major:       1,
		Minor:       3,
		Permissions: "rwm",
		Allow:       true,
	},
	{
		Type:        'c',
		Path:        "/dev/random",
		Major:       1,
		Minor:       8,
		Permissions: "rwm",
		Allow:       true,
	},
	{
		Type:        'c',
		Path:        "/dev/full",
		Major:       1,
		Minor:       7,
		Permissions: "rwm",
		Allow:       true,
	},
	{
		Type:        'c',
		Path:        "/dev/tty",
		Major:       5,
		Minor:       0,
		Permissions: "rwm",
		Allow:       true,
	},
	{
		Type:        'c',
		Path:        "/dev/zero",
		Major:       1,
		Minor:       5,
		Permissions: "rwm",
		Allow:       true,
	},
	{
		Type:        'c',
		Path:        "/dev/urandom",
		Major:       1,
		Minor:       9,
		Permissions: "rwm",
		Allow:       true,
	},
	{
		Path:        "/dev/console",
		Type:        'c',
		Major:       5,
		Minor:       1,
		Permissions: "rwm",
		Allow:       true,
	},
	// /dev/pts/ - pts namespaces are "coming soon"
	{
		Path:        "",
		Type:        'c',
		Major:       136,
		Minor:       wildcard,
		Permissions: "rwm",
		Allow:       true,
	},
	{
		Path:        "",
		Type:        'c',
		Major:       5,
		Minor:       2,
		Permissions: "rwm",
		Allow:       true,
	},
	// tuntap
	{
		Path:        "",
		Type:        'c',
		Major:       10,
		Minor:       200,
		Permissions: "rwm",
		Allow:       true,
	},
}

var (
	maskedPaths = []string{
		"/proc/kcore",
		"/proc/latency_stats",
		"/proc/timer_stats",
		"/proc/sched_debug",
	}
	readonlyPaths = []string{
		"/proc/asound",
		"/proc/bus",
		"/proc/fs",
		"/proc/irq",
		"/proc/sys",
		"/proc/sysrq-trigger",
	}
)

var container libcontainer.Container

func containerPreload(context *cli.Context) error {
	c, err := getContainer(context)
	if err != nil {
		return err
	}
	container = c
	return nil
}

// loadFactory returns the configured factory instance for execing containers.
func loadFactory(context *cli.Context) (libcontainer.Factory, error) {
	root := context.GlobalString("root")
	abs, err := filepath.Abs(root)
	if err != nil {
		return nil, err
	}
	return libcontainer.New(abs, libcontainer.Cgroupfs, func(l *libcontainer.LinuxFactory) error {
		l.CriuPath = context.GlobalString("criu")
		return nil
	})
}

// getContainer returns the specified container instance by loading it from state
// with the default factory.
func getContainer(context *cli.Context) (libcontainer.Container, error) {
	id := context.Args().First()
	if id == "" {
		return nil, errEmptyID
	}
	factory, err := loadFactory(context)
	if err != nil {
		return nil, err
	}
	return factory.Load(id)
}

// fatal prints the error's details if it is a libcontainer specific error type
// then exits the program with an exit status of 1.
func fatal(err error) {
	// make sure the error is written to the logger
	logrus.Error(err)
	fmt.Fprintln(os.Stderr, err)
	os.Exit(1)
}

func fatalf(t string, v ...interface{}) {
	fatal(fmt.Errorf(t, v...))
}

func getDefaultImagePath(context *cli.Context) string {
	cwd, err := os.Getwd()
	if err != nil {
		panic(err)
	}
	return filepath.Join(cwd, "checkpoint")
}

// newProcess returns a new libcontainer Process with the arguments from the
// spec and stdio from the current process.
func newProcess(p specs.Process) (*libcontainer.Process, error) {
	lp := &libcontainer.Process{
		Args: p.Args,
		Env:  p.Env,
		// TODO: fix libcontainer's API to better support uid/gid in a typesafe way.
		User:            fmt.Sprintf("%d:%d", p.User.UID, p.User.GID),
		Cwd:             p.Cwd,
		Capabilities:    p.Capabilities,
		Label:           p.SelinuxLabel,
		NoNewPrivileges: &p.NoNewPrivileges,
		AppArmorProfile: p.ApparmorProfile,
	}
	for _, rlimit := range p.Rlimits {
		rl, err := createLibContainerRlimit(rlimit)
		if err != nil {
			return nil, err
		}
		lp.Rlimits = append(lp.Rlimits, rl)
	}
	return lp, nil
}

func dupStdio(process *libcontainer.Process, rootuid int) error {
	process.Stdin = os.Stdin
	process.Stdout = os.Stdout
	process.Stderr = os.Stderr
	for _, fd := range []uintptr{
		os.Stdin.Fd(),
		os.Stdout.Fd(),
		os.Stderr.Fd(),
	} {
		if err := syscall.Fchown(int(fd), rootuid, rootuid); err != nil {
			return err
		}
	}
	return nil
}

// If systemd is supporting sd_notify protocol, this function will add support
// for sd_notify protocol from within the container.
func setupSdNotify(spec *specs.Spec, notifySocket string) {
	spec.Mounts = append(spec.Mounts, specs.Mount{Destination: notifySocket, Type: "bind", Source: notifySocket, Options: []string{"bind"}})
	spec.Process.Env = append(spec.Process.Env, fmt.Sprintf("NOTIFY_SOCKET=%s", notifySocket))
}

func destroy(container libcontainer.Container) {
	if err := container.Destroy(); err != nil {
		logrus.Error(err)
	}
}

// setupIO sets the proper IO on the process depending on the configuration
// If there is a nil error then there must be a non nil tty returned
func setupIO(process *libcontainer.Process, rootuid int, console string, createTTY, detach bool) (*tty, error) {
	// detach and createTty will not work unless a console path is passed
	// so error out here before changing any terminal settings
	if createTTY && detach && console == "" {
		return nil, fmt.Errorf("cannot allocate tty if runc will detach")
	}
	if createTTY {
		return createTty(process, rootuid, console)
	}
	if detach {
		if err := dupStdio(process, rootuid); err != nil {
			return nil, err
		}
		return &tty{}, nil
	}
	return createStdioPipes(process, rootuid)
}

// createPidFile creates a file with the processes pid inside it atomically
// it creates a temp file with the paths filename + '.' infront of it
// then renames the file
func createPidFile(path string, process *libcontainer.Process) error {
	pid, err := process.Pid()
	if err != nil {
		return err
	}
	var (
		tmpDir  = filepath.Dir(path)
		tmpName = filepath.Join(tmpDir, fmt.Sprintf(".%s", filepath.Base(path)))
	)
	f, err := os.OpenFile(tmpName, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, 0666)
	if err != nil {
		return err
	}
	_, err = fmt.Fprintf(f, "%d", pid)
	f.Close()
	if err != nil {
		return err
	}
	return os.Rename(tmpName, path)
}

func createContainer(context *cli.Context, id string, spec *specs.Spec) (libcontainer.Container, error) {
	config, err := createLibcontainerConfig(id, spec)
	if err != nil {
		return nil, err
	}

	if _, err := os.Stat(config.Rootfs); err != nil {
		if os.IsNotExist(err) {
			return nil, fmt.Errorf("rootfs (%q) does not exist", config.Rootfs)
		}
		return nil, err
	}

	factory, err := loadFactory(context)
	if err != nil {
		return nil, err
	}
	return factory.Create(id, config)
}

type runner struct {
	enableSubreaper bool
	shouldDestroy   bool
	detach          bool
	listenFDs       []*os.File
	pidFile         string
	console         string
	container       libcontainer.Container
}

func (r *runner) run(config *specs.Process) (int, error) {
	process, err := newProcess(*config)
	if err != nil {
		r.destroy()
		return -1, err
	}
	if len(r.listenFDs) > 0 {
		process.Env = append(process.Env, fmt.Sprintf("LISTEN_FDS=%d", len(r.listenFDs)), "LISTEN_PID=1")
		process.ExtraFiles = append(process.ExtraFiles, r.listenFDs...)
	}
	rootuid, err := r.container.Config().HostUID()
	if err != nil {
		r.destroy()
		return -1, err
	}
	tty, err := setupIO(process, rootuid, r.console, config.Terminal, r.detach)
	if err != nil {
		r.destroy()
		return -1, err
	}
	handler := newSignalHandler(tty, r.enableSubreaper)
	if err := r.container.Start(process); err != nil {
		r.destroy()
		tty.Close()
		return -1, err
	}
	if err := tty.ClosePostStart(); err != nil {
		r.terminate(process)
		r.destroy()
		tty.Close()
		return -1, err
	}
	if r.pidFile != "" {
		if err := createPidFile(r.pidFile, process); err != nil {
			r.terminate(process)
			r.destroy()
			tty.Close()
			return -1, err
		}
	}
	if r.detach {
		tty.Close()
		return 0, nil
	}
	status, err := handler.forward(process)
	if err != nil {
		r.terminate(process)
	}
	r.destroy()
	tty.Close()
	return status, err
}

func (r *runner) destroy() {
	if r.shouldDestroy {
		destroy(r.container)
	}
}

func (r *runner) terminate(p *libcontainer.Process) {
	p.Signal(syscall.SIGKILL)
	p.Wait()
}

func validateProcessSpec(spec *specs.Process) error {
	if spec.Cwd == "" {
		return fmt.Errorf("Cwd property must not be empty")
	}
	if !filepath.IsAbs(spec.Cwd) {
		return fmt.Errorf("Cwd must be an absolute path")
	}
	if len(spec.Args) == 0 {
		return fmt.Errorf("args must not be empty")
	}
	return nil
}
<|endoftext|>


<commit_before>package eneru

import (
	"encoding/json"
	"github.com/plimble/utils/strings2"
	"net/http"
	"strings"
)

func buildUrl(url, path string, query *Query) string {
	if query == nil {
		return string2.ConcatBase(url, path)
	}

	return string2.ConcatBase(url, path, query.String())
}

func addTailingSlash(url string) string {
	if !strings.HasSuffix(url, "/") {
		return string2.ConcatBase(url, "/")
	}

	return url
}

func decodeResp(resp *http.Response, v interface{}) error {
	defer resp.Body.Close()
	return json.NewDecoder(resp.Body).Decode(v)
}

func encodeResp(w http.ResponseWriter, v interface{}) {
	json.NewEncoder(w).Encode(v)
}

func buildPathIndexType(index, ty string) string {
	switch {
	case index != "" && ty == "" && action == "":
		return string2.Concat(index)
	case index != "" && ty != "" && action == "":
		return string2.Concat(index, "/", ty)
	}

	return string2.Concat(index, "/", ty)
}

func buildPathIndexTypeAction(index, ty, action string) string {
	switch {
	case index == "" && ty == "" && action != "":
		return action
	case index != "" && ty == "" && action != "":
		return string2.Concat(index, "/", action)
	}

	return string2.Concat(index, "/", ty, "/", action)
}
<commit_msg>fix buildPathIndexType no action variable<commit_after>package eneru

import (
	"encoding/json"
	"github.com/plimble/utils/strings2"
	"net/http"
	"strings"
)

func buildUrl(url, path string, query *Query) string {
	if query == nil {
		return string2.ConcatBase(url, path)
	}

	return string2.ConcatBase(url, path, query.String())
}

func addTailingSlash(url string) string {
	if !strings.HasSuffix(url, "/") {
		return string2.ConcatBase(url, "/")
	}

	return url
}

func decodeResp(resp *http.Response, v interface{}) error {
	defer resp.Body.Close()
	return json.NewDecoder(resp.Body).Decode(v)
}

func encodeResp(w http.ResponseWriter, v interface{}) {
	json.NewEncoder(w).Encode(v)
}

func buildPathIndexType(index, ty string) string {
	switch {
	case index != "" && ty == "":
		return string2.Concat(index)
	case index != "" && ty != "":
		return string2.Concat(index, "/", ty)
	}

	return string2.Concat(index, "/", ty)
}

func buildPathIndexTypeAction(index, ty, action string) string {
	switch {
	case index == "" && ty == "" && action != "":
		return action
	case index != "" && ty == "" && action != "":
		return string2.Concat(index, "/", action)
	}

	return string2.Concat(index, "/", ty, "/", action)
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"net/http"
	"os"
	"strings"
)

func downloadFile(url string, path string) (string, error) {
	tokens := strings.Split(url, "/")
	fileName := tokens[len(tokens)-1]
	fullPath := path + string(os.PathSeparator) + fileName

	if _, err := os.Stat(fullPath); err == nil {
		logger.Noticef("%s exists", fileName)
		return fileName, nil
	}

	output, err := os.Create(fullPath)
	if err != nil {
		logger.Error(err)
		return "", err
	}
	defer output.Close()

	logger.Debugf("--> Downloading %s", url)
	response, err := http.Get(url)
	if err != nil {
		logger.Error(err)
		return "", err
	}
	defer response.Body.Close()

	n, err := io.Copy(output, response.Body)
	if err != nil {
		logger.Error(err)
		return "", err
	}
	logger.Debugf("%s: %d bytes", fileName, n)
	return fileName, nil
}

func removeFile(url string, path string) error {
	tokens := strings.Split(url, "/")
	fileName := tokens[len(tokens)-1]
	logger.Debugf("--> Deleting %s", fileName)
	fullPath := path + string(os.PathSeparator) + fileName
	err := os.Remove(fullPath)
	if err != nil {
		logger.Error(err)
		return err
	}
	logger.Debugf("--> Deleted %s", fileName)
	return nil
}
<commit_msg>log full path<commit_after>package main

import (
	"io"
	"net/http"
	"os"
	"strings"
)

func downloadFile(url string, path string) (string, error) {
	tokens := strings.Split(url, "/")
	fileName := tokens[len(tokens)-1]
	fullPath := path + string(os.PathSeparator) + fileName

	if _, err := os.Stat(fullPath); err == nil {
		logger.Noticef("%s exists", fileName)
		return fileName, nil
	}

	output, err := os.Create(fullPath)
	if err != nil {
		logger.Error(err)
		return "", err
	}
	defer output.Close()

	logger.Debugf("--> Downloading %s", url)
	response, err := http.Get(url)
	if err != nil {
		logger.Error(err)
		return "", err
	}
	defer response.Body.Close()

	n, err := io.Copy(output, response.Body)
	if err != nil {
		logger.Error(err)
		return "", err
	}
	logger.Debugf("%s: %d bytes", fullPath, n)
	return fileName, nil
}

func removeFile(url string, path string) error {
	tokens := strings.Split(url, "/")
	fileName := tokens[len(tokens)-1]
	logger.Debugf("--> Deleting %s", fileName)
	fullPath := path + string(os.PathSeparator) + fileName
	err := os.Remove(fullPath)
	if err != nil {
		logger.Error(err)
		return err
	}
	logger.Debugf("--> Deleted %s", fullPath)
	return nil
}
<|endoftext|>


<commit_before>package turbolib

import "fmt"

func Lib() {
	fmt.Println("Lib")
}
<commit_msg>lib 2<commit_after>package turbolib

import "fmt"

func Lib() {
	fmt.Println("Lib 2")
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/tsileo/rawgeo"
)

func parseLocation(db *rawgeo.RawGeo) error {
	file, err := os.Open("cities1000.txt")
	if err != nil {
		panic(err)
	}
	defer file.Close()
	scan := bufio.NewScanner(file)
	for scan.Scan() {
		line := scan.Text()
		s := strings.Split(line, "\t")
		if len(s) < 19 {
			continue
		}
		lat, err := strconv.ParseFloat(s[4], 64)
		if err != nil {
			return err
		}
		lon, err := strconv.ParseFloat(s[5], 64)
		if err != nil {
			return err
		}
		p := &rawgeo.Point{
			ID:  fmt.Sprintf("%s-%s", s[8], s[1]),
			Lat: lat,
			Lng: lon,
		}
		if err := db.Index(p); err != nil {
			if err == rawgeo.ErrInvalidLatLong {
				continue
			}
			return err
		}
	}
	return nil
}

func main() {
	db, err := rawgeo.New("cities.db")
	if err != nil {
		panic(err)
	}
	defer db.Close()
	if err := parseLocation(db); err != nil {
		panic(err)
	}
	n := time.Now()
	// Query should return Austin TX
	res, err := db.Query(30.26715, -97.74306, 40) // 40m
	took := time.Since(n)
	fmt.Printf("query took %v\n", took)
	if res != nil && len(res) > 0 {
		fmt.Printf("res=%+v", res[0])
	} else {
		fmt.Printf("res=%q\nerr=%v", res, err)
	}
}
<commit_msg>update example<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/pariz/gountries"

	"a4.io/rawgeo"
)

var data = gountries.New()

type Location struct {
	ID                 string  `json:"id"`          // 0
	CityName           string  `json:"city_name"`   // 1
	Lat                float64 `json:"lat"`         // 4
	Lon                float64 `json:"lon"`         // 5
	CountryCode        string  `json:"cc"`          // 8
	AdminCode          string  `json:"admin"`       // 10
	CountrySubdivision string  `json:"subdivision"` // 11

	CountryName     string `json:"country_name"`
	SubdivisionName string `json:"subdivison_name"`
	Name            string `json:"name"`
}

// https://github.com/hexorx/countries/tree/master/lib
// source => https://anonscm.debian.org/cgit/pkg-isocodes/iso-codes.git/tree/iso_3166-2

// XXX(tsileo): filter by feature class/feature code to only restrict to cities

var locations = map[string]*Location{}

func parseLocation(db *rawgeo.RawGeo) error {
	file, err := os.Open("cities1000.txt")
	if err != nil {
		panic(err)
	}
	defer file.Close()
	scan := bufio.NewScanner(file)
	for scan.Scan() {
		line := scan.Text()
		s := strings.Split(line, "\t")
		if len(s) < 19 {
			continue
		}
		lat, err := strconv.ParseFloat(s[4], 64)
		if err != nil {
			return err
		}
		lon, err := strconv.ParseFloat(s[5], 64)
		if err != nil {
			return err
		}
		loc := &Location{
			ID:                 s[0],
			CityName:           s[1],
			Lat:                lat,
			Lon:                lon,
			CountryCode:        s[8],
			AdminCode:          s[10], // admin
			CountrySubdivision: s[11], // admin2
		}
		country, _ := data.FindCountryByAlpha(loc.CountryCode)

		loc.CountryName = country.Name.Common
		for _, subdiv := range country.SubDivisions() {
			if subdiv.Code == loc.CountrySubdivision {
				loc.SubdivisionName = subdiv.Name
			}
		}

		if loc.SubdivisionName != "" {
			loc.Name = fmt.Sprintf("%s, %s, %s", loc.CityName, loc.SubdivisionName, loc.CountryName)
		} else {
			loc.Name = fmt.Sprintf("%s, %s", loc.CityName, loc.CountryName)
		}

		locations[loc.ID] = loc
		p := &rawgeo.Point{
			ID:  loc.ID,
			Lat: lat,
			Lng: lon,
		}
		if err := db.Index(p); err != nil {
			if err == rawgeo.ErrInvalidLatLong {
				continue
			}
			return err
		}
	}
	return nil
}

func main() {
	n := time.Now()
	db, err := rawgeo.New("cities2.db")
	if err != nil {
		panic(err)
	}
	defer db.Close()
	if err := parseLocation(db); err != nil {
		panic(err)
	}
	took := time.Since(n)
	fmt.Printf("loading took %v\n", took)

	n = time.Now()
	// Query should return Austin TX
	res, err := db.Query(48.26127189, 4.0871129, 1000)
	// res, err := db.Query(30.26715, -97.74306, 40) // 40m
	took = time.Since(n)
	fmt.Printf("query took %v\n", took)
	if res != nil && len(res) > 0 {
		fmt.Printf("res=%+v", locations[res[0].ID])
	} else {
		fmt.Printf("res=%q\nerr=%v", res, err)
	}
}
<|endoftext|>


<commit_before>package widget

import (
	"fmt"
	"os"
	"path"

	"github.com/jinzhu/gorm"
	"github.com/qor/admin"
	"github.com/qor/qor"
	"github.com/qor/qor/resource"
	"github.com/qor/roles"
)

var (
	root, _                = os.Getwd()
	viewPaths              []string
	registeredWidgets      []*Widget
	registeredWidgetsGroup []*WidgetsGroup
)

// Config widget config
type Config struct {
	DB    *gorm.DB
	Admin *admin.Admin
}

func init() {
	if path := os.Getenv("WEB_ROOT"); path != "" {
		root = path
	}
	registerViewPath(path.Join(root, "app/views/widgets"))
}

// New new widgets container
func New(config *Config) *Widgets {
	widgets := &Widgets{Config: config}
	widgets.RegisterViewPath("app/views/widgets")
	return widgets
}

// Widgets widgets container
type Widgets struct {
	Config                *Config
	Resource              *admin.Resource
	WidgetSettingResource *admin.Resource
}

// RegisterWidget register a new widget
func (widgets *Widgets) RegisterWidget(w *Widget) {
	registeredWidgets = append(registeredWidgets, w)
}

// RegisterWidgetGroup register widgets group
func (widgets *Widgets) RegisterWidgetsGroup(group *WidgetsGroup) {
	registeredWidgetsGroup = append(registeredWidgetsGroup, group)
}

// ConfigureQorResource a method used to config Widget for qor admin
func (widgets *Widgets) ConfigureQorResource(res resource.Resourcer) {
	if res, ok := res.(*admin.Resource); ok {
		// register view paths
		res.GetAdmin().RegisterViewPath("github.com/qor/widget/views")

		// set resources
		res.Name = "Widget"
		widgets.Resource = res

		// set setting resource
		if widgets.WidgetSettingResource == nil {
			widgets.WidgetSettingResource = res.GetAdmin().NewResource(&QorWidgetSetting{}, &admin.Config{Name: res.Name, Permission: roles.Deny(roles.Create, roles.Anyone)})

			widgets.WidgetSettingResource.Meta(&admin.Meta{Name: "Name", Permission: roles.Deny(roles.Update, roles.Anyone)})
			widgets.WidgetSettingResource.Meta(&admin.Meta{
				Name: "Scope",
				Type: "hidden",
				Valuer: func(result interface{}, context *qor.Context) interface{} {
					if scope := context.Request.URL.Query().Get("widget_scope"); scope != "" {
						return scope
					}

					if setting, ok := result.(*QorWidgetSetting); ok {
						if setting.Scope != "" {
							return setting.Scope
						}
					}

					return "default"
				},
			})
			widgets.WidgetSettingResource.Meta(&admin.Meta{
				Name: "Widgets",
				Type: "select_one",
				Valuer: func(result interface{}, context *qor.Context) interface{} {
					if setting, ok := result.(*QorWidgetSetting); ok {
						return GetWidget(setting.Kind).Name
					}
					return ""
				},
				Collection: func(result interface{}, context *qor.Context) (results [][]string) {
					if setting, ok := result.(*QorWidgetSetting); ok {
						for _, group := range registeredWidgetsGroup {
							if group.Name == setting.Kind {
								for _, widget := range group.Widgets {
									results = append(results, []string{widget, widget})
								}
							}
						}
					}
					return
				},
			})
			widgets.WidgetSettingResource.Meta(&admin.Meta{
				Name: "Template",
				Type: "select_one",
				Valuer: func(result interface{}, context *qor.Context) interface{} {
					if setting, ok := result.(*QorWidgetSetting); ok {
						return setting.GetTemplate()
					}
					return ""
				},
				Collection: func(result interface{}, context *qor.Context) (results [][]string) {
					if setting, ok := result.(*QorWidgetSetting); ok {
						if widget := GetWidget(setting.Kind); widget != nil {
							for _, value := range widget.Templates {
								results = append(results, []string{value, value})
							}
						}
					}
					return
				},
			})

			widgets.WidgetSettingResource.IndexAttrs("ID", "Name", "Widgets", "Template", "Kind", "CreatedAt", "UpdatedAt")
			widgets.WidgetSettingResource.EditAttrs(
				"ID", "Scope", "Widgets", "Template",
				&admin.Section{
					Title: "Settings",
					Rows:  [][]string{{"Kind"}, {"SerializableMeta"}},
				},
			)
		}

		// use widget theme
		res.UseTheme("widget")
		widgets.WidgetSettingResource.UseTheme("widget")

		for funcName, fc := range funcMap {
			res.GetAdmin().RegisterFuncMap(funcName, fc)
		}

		// configure routes
		controller := widgetController{Widgets: widgets}
		router := res.GetAdmin().GetRouter()
		router.Get(res.ToParam(), controller.Index)
		router.Get(fmt.Sprintf("%v/inline-edit", res.ToParam()), controller.InlineEdit)
		router.Get(fmt.Sprintf("%v/%v", res.ToParam(), res.ParamIDName()), controller.Edit)
		router.Get(fmt.Sprintf("%v/%v/edit", res.ToParam(), res.ParamIDName()), controller.Edit)
		router.Put(fmt.Sprintf("%v/%v", res.ToParam(), res.ParamIDName()), controller.Update)
	}
}

// Widget widget struct
type Widget struct {
	Name      string
	Templates []string
	Setting   *admin.Resource
	Context   func(context *Context, setting interface{}) *Context
}

// WidgetsGroup widgets Group
type WidgetsGroup struct {
	Name    string
	Widgets []string
}

// GetWidget get widget by name
func GetWidget(name string) *Widget {
	for _, w := range registeredWidgets {
		if w.Name == name {
			return w
		}
	}

	for _, g := range registeredWidgetsGroup {
		if g.Name == name {
			for _, widgetName := range g.Widgets {
				return GetWidget(widgetName)
			}
		}
	}
	return nil
}
<commit_msg>Return widget type as Kind<commit_after>package widget

import (
	"fmt"
	"os"
	"path"

	"github.com/jinzhu/gorm"
	"github.com/qor/admin"
	"github.com/qor/qor"
	"github.com/qor/qor/resource"
	"github.com/qor/roles"
)

var (
	root, _                = os.Getwd()
	viewPaths              []string
	registeredWidgets      []*Widget
	registeredWidgetsGroup []*WidgetsGroup
)

// Config widget config
type Config struct {
	DB    *gorm.DB
	Admin *admin.Admin
}

func init() {
	if path := os.Getenv("WEB_ROOT"); path != "" {
		root = path
	}
	registerViewPath(path.Join(root, "app/views/widgets"))
}

// New new widgets container
func New(config *Config) *Widgets {
	widgets := &Widgets{Config: config}
	widgets.RegisterViewPath("app/views/widgets")
	return widgets
}

// Widgets widgets container
type Widgets struct {
	Config                *Config
	Resource              *admin.Resource
	WidgetSettingResource *admin.Resource
}

// RegisterWidget register a new widget
func (widgets *Widgets) RegisterWidget(w *Widget) {
	registeredWidgets = append(registeredWidgets, w)
}

// RegisterWidgetGroup register widgets group
func (widgets *Widgets) RegisterWidgetsGroup(group *WidgetsGroup) {
	registeredWidgetsGroup = append(registeredWidgetsGroup, group)
}

// ConfigureQorResource a method used to config Widget for qor admin
func (widgets *Widgets) ConfigureQorResource(res resource.Resourcer) {
	if res, ok := res.(*admin.Resource); ok {
		// register view paths
		res.GetAdmin().RegisterViewPath("github.com/qor/widget/views")

		// set resources
		res.Name = "Widget"
		widgets.Resource = res

		// set setting resource
		if widgets.WidgetSettingResource == nil {
			widgets.WidgetSettingResource = res.GetAdmin().NewResource(&QorWidgetSetting{}, &admin.Config{Name: res.Name, Permission: roles.Deny(roles.Create, roles.Anyone)})

			widgets.WidgetSettingResource.Meta(&admin.Meta{Name: "Name", Permission: roles.Deny(roles.Update, roles.Anyone)})
			widgets.WidgetSettingResource.Meta(&admin.Meta{
				Name: "Scope",
				Type: "hidden",
				Valuer: func(result interface{}, context *qor.Context) interface{} {
					if scope := context.Request.URL.Query().Get("widget_scope"); scope != "" {
						return scope
					}

					if setting, ok := result.(*QorWidgetSetting); ok {
						if setting.Scope != "" {
							return setting.Scope
						}
					}

					return "default"
				},
			})
			widgets.WidgetSettingResource.Meta(&admin.Meta{
				Name: "Widgets",
				Type: "select_one",
				Valuer: func(result interface{}, context *qor.Context) interface{} {
					if typ := context.Request.URL.Query().Get("widget_type"); typ != "" {
						return typ
					}

					if setting, ok := result.(*QorWidgetSetting); ok {
						return GetWidget(setting.Kind).Name
					}

					return ""
				},
				Collection: func(result interface{}, context *qor.Context) (results [][]string) {
					if setting, ok := result.(*QorWidgetSetting); ok {
						for _, group := range registeredWidgetsGroup {
							if group.Name == setting.Kind {
								for _, widget := range group.Widgets {
									results = append(results, []string{widget, widget})
								}
							}
						}

						if len(results) == 0 {
							results = append(results, []string{setting.Kind, setting.Kind})
						}
					}
					return
				},
			})
			widgets.WidgetSettingResource.Meta(&admin.Meta{
				Name: "Template",
				Type: "select_one",
				Valuer: func(result interface{}, context *qor.Context) interface{} {
					if setting, ok := result.(*QorWidgetSetting); ok {
						return setting.GetTemplate()
					}
					return ""
				},
				Collection: func(result interface{}, context *qor.Context) (results [][]string) {
					if setting, ok := result.(*QorWidgetSetting); ok {
						if widget := GetWidget(setting.Kind); widget != nil {
							for _, value := range widget.Templates {
								results = append(results, []string{value, value})
							}
						}
					}
					return
				},
			})

			widgets.WidgetSettingResource.IndexAttrs("ID", "Name", "Widgets", "Template", "Kind", "CreatedAt", "UpdatedAt")
			widgets.WidgetSettingResource.EditAttrs(
				"ID", "Scope", "Widgets", "Template",
				&admin.Section{
					Title: "Settings",
					Rows:  [][]string{{"Kind"}, {"SerializableMeta"}},
				},
			)
		}

		// use widget theme
		res.UseTheme("widget")
		widgets.WidgetSettingResource.UseTheme("widget")

		for funcName, fc := range funcMap {
			res.GetAdmin().RegisterFuncMap(funcName, fc)
		}

		// configure routes
		controller := widgetController{Widgets: widgets}
		router := res.GetAdmin().GetRouter()
		router.Get(res.ToParam(), controller.Index)
		router.Get(fmt.Sprintf("%v/inline-edit", res.ToParam()), controller.InlineEdit)
		router.Get(fmt.Sprintf("%v/%v", res.ToParam(), res.ParamIDName()), controller.Edit)
		router.Get(fmt.Sprintf("%v/%v/edit", res.ToParam(), res.ParamIDName()), controller.Edit)
		router.Put(fmt.Sprintf("%v/%v", res.ToParam(), res.ParamIDName()), controller.Update)
	}
}

// Widget widget struct
type Widget struct {
	Name      string
	Templates []string
	Setting   *admin.Resource
	Context   func(context *Context, setting interface{}) *Context
}

// WidgetsGroup widgets Group
type WidgetsGroup struct {
	Name    string
	Widgets []string
}

// GetWidget get widget by name
func GetWidget(name string) *Widget {
	for _, w := range registeredWidgets {
		if w.Name == name {
			return w
		}
	}

	for _, g := range registeredWidgetsGroup {
		if g.Name == name {
			for _, widgetName := range g.Widgets {
				return GetWidget(widgetName)
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*-
 * Copyright 2015 Square Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"crypto/tls"
	"crypto/x509"
	"io"
	"net"
	"sync"
)

// Accept incoming connections and spawn Go routines to handle them.
func accept(listener net.Listener, wg *sync.WaitGroup, stopper chan bool, leaf *x509.Certificate) {
	defer wg.Done()
	defer listener.Close()

	for {
		// Wait for new connection
		conn, err := listener.Accept()

		// Check if we're supposed to stop
		select {
		case _ = <-stopper:
			logger.Printf("closing socket with cert serial no. %d (expiring %s)", leaf.SerialNumber, leaf.NotAfter.String())
			return
		default:
		}

		if err != nil {
			logger.Printf("error accepting connection: %s", err)
			continue
		}

		tlsConn, ok := conn.(*tls.Conn)
		if !ok {
			logger.Printf("received non-TLS connection from %s? Ignoring", conn.RemoteAddr())
			conn.Close()
			continue
		}

		// Force handshake. Handshake usually happens on first read/write, but
		// we want to authenticate before reading/writing so we need to force
		// the handshake to get the client cert.
		err = tlsConn.Handshake()
		if err != nil {
			logger.Printf("failed TLS handshake on %s: %s", conn.RemoteAddr(), err)
			conn.Close()
			continue
		}

		if !authorized(tlsConn.ConnectionState()) {
			logger.Printf("rejecting connection from %s: bad client certificate", conn.RemoteAddr())
			conn.Close()
			continue
		}

		wg.Add(1)
		go handle(conn, wg)
	}
}

// Handle incoming connection by opening new connection to our backend service
// and fusing them together.
func handle(conn net.Conn, wg *sync.WaitGroup) {
	defer wg.Done()

	logger.Printf("incoming connection: %s", conn.RemoteAddr())

	backend, err := net.Dial((*forwardAddress).Network(), (*forwardAddress).String())

	if err != nil {
		logger.Printf("failed to dial backend: %s", err)
		return
	}

	fuse(conn, backend)
}

// Fuse connections together
func fuse(client, backend net.Conn) {
	// Copy from client -> backend, and from backend -> client
	go func() { copyData(client, backend) }()
	copyData(backend, client)
}

// Copy data between two connections
func copyData(dst net.Conn, src net.Conn) {
	defer dst.Close()
	defer src.Close()
	defer logger.Printf("closed pipe: %s <- %s", dst.RemoteAddr(), src.RemoteAddr())
	logger.Printf("opening pipe: %s <- %s", dst.RemoteAddr(), src.RemoteAddr())

	_, err := io.Copy(dst, src)

	if err != nil {
		logger.Printf("%s", err)
	}
}

// Helper function to decode a *net.TCPAddr into a tuple of network and
// address. Must use this since kavu/so_reuseport does not currently
// support passing "tcp" to support for IPv4 and IPv6. We must pass "tcp4"
// or "tcp6" explicitly.
func decodeAddress(tuple *net.TCPAddr) (network, address string) {
	if tuple.IP.To4() != nil {
		network = "tcp4"
	} else {
		network = "tcp6"
	}

	address = tuple.String()
	return
}
<commit_msg>Make sure to close client conn if backend connection fails<commit_after>/*-
 * Copyright 2015 Square Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"crypto/tls"
	"crypto/x509"
	"io"
	"net"
	"sync"
)

// Accept incoming connections and spawn Go routines to handle them.
func accept(listener net.Listener, wg *sync.WaitGroup, stopper chan bool, leaf *x509.Certificate) {
	defer wg.Done()
	defer listener.Close()

	for {
		// Wait for new connection
		conn, err := listener.Accept()

		// Check if we're supposed to stop
		select {
		case _ = <-stopper:
			logger.Printf("closing socket with cert serial no. %d (expiring %s)", leaf.SerialNumber, leaf.NotAfter.String())
			return
		default:
		}

		if err != nil {
			logger.Printf("error accepting connection: %s", err)
			continue
		}

		tlsConn, ok := conn.(*tls.Conn)
		if !ok {
			logger.Printf("received non-TLS connection from %s? Ignoring", conn.RemoteAddr())
			conn.Close()
			continue
		}

		// Force handshake. Handshake usually happens on first read/write, but
		// we want to authenticate before reading/writing so we need to force
		// the handshake to get the client cert.
		err = tlsConn.Handshake()
		if err != nil {
			logger.Printf("failed TLS handshake on %s: %s", conn.RemoteAddr(), err)
			conn.Close()
			continue
		}

		if !authorized(tlsConn.ConnectionState()) {
			logger.Printf("rejecting connection from %s: bad client certificate", conn.RemoteAddr())
			conn.Close()
			continue
		}

		wg.Add(1)
		go handle(conn, wg)
	}
}

// Handle incoming connection by opening new connection to our backend service
// and fusing them together.
func handle(conn net.Conn, wg *sync.WaitGroup) {
	defer wg.Done()
	defer conn.Close()

	logger.Printf("incoming connection: %s", conn.RemoteAddr())

	backend, err := net.Dial((*forwardAddress).Network(), (*forwardAddress).String())

	if err != nil {
		logger.Printf("failed to dial backend: %s", err)
		return
	}

	fuse(conn, backend)
}

// Fuse connections together
func fuse(client, backend net.Conn) {
	// Copy from client -> backend, and from backend -> client
	go func() { copyData(client, backend) }()
	copyData(backend, client)
}

// Copy data between two connections
func copyData(dst net.Conn, src net.Conn) {
	defer dst.Close()
	defer src.Close()
	defer logger.Printf("closed pipe: %s <- %s", dst.RemoteAddr(), src.RemoteAddr())
	logger.Printf("opening pipe: %s <- %s", dst.RemoteAddr(), src.RemoteAddr())

	_, err := io.Copy(dst, src)

	if err != nil {
		logger.Printf("%s", err)
	}
}

// Helper function to decode a *net.TCPAddr into a tuple of network and
// address. Must use this since kavu/so_reuseport does not currently
// support passing "tcp" to support for IPv4 and IPv6. We must pass "tcp4"
// or "tcp6" explicitly.
func decodeAddress(tuple *net.TCPAddr) (network, address string) {
	if tuple.IP.To4() != nil {
		network = "tcp4"
	} else {
		network = "tcp6"
	}

	address = tuple.String()
	return
}
<|endoftext|>


<commit_before>package server

import (
	"errors"
	"github.com/jzipfler/htw-ava/client"
	"strconv"
)

const (
	MINIMUM_PORT = 0
	MAXIMUM_PORT = 65535
)

// A server includes a client.NetworkClient type
// and adds a port where a client can connect to
type NetworkServer struct {
	client.NetworkClient
	port int
}

// New returns a NetworkServer type.
func New() NetworkServer {
	return NetworkServer{client.New(), 0}
}

func (networkServer *NetworkServer) SetPort(port int) error {
	if port < MINIMUM_PORT || port > MAXIMUM_PORT {
		return errors.New("Port must be between " + strconv.Itoa(MINIMUM_PORT) + " and " + strconv.Itoa(MAXIMUM_PORT))
	}
	networkServer.port = port
	return nil
}

// Returns the port as its integer value.
func (networkServer NetworkServer) Port() int {
	return networkServer.port
}

// String gives a string object which contains a representation of a NetworkServer type.
func (networkServer NetworkServer) String() string {
	return networkServer.IpAddressAsString() + ":" + strconv.Itoa(networkServer.Port())
}
<commit_msg>Added a function that returns the ip address and port as string as following: "ip:port". I added this because the String() method is not the right method for some other functions like net.Listen to return such a string.<commit_after>package server

import (
	"errors"
	"github.com/jzipfler/htw-ava/client"
	"strconv"
)

const (
	MINIMUM_PORT = 0
	MAXIMUM_PORT = 65535
)

// A server includes a client.NetworkClient type
// and adds a port where a client can connect to
type NetworkServer struct {
	client.NetworkClient
	port int
}

// New returns a NetworkServer type.
func New() NetworkServer {
	return NetworkServer{client.New(), 0}
}

func (networkServer *NetworkServer) SetPort(port int) error {
	if port < MINIMUM_PORT || port > MAXIMUM_PORT {
		return errors.New("Port must be between " + strconv.Itoa(MINIMUM_PORT) + " and " + strconv.Itoa(MAXIMUM_PORT))
	}
	networkServer.port = port
	return nil
}

// Returns the port as its integer value.
func (networkServer NetworkServer) Port() int {
	return networkServer.port
}

// IpAndPortAsString returns the content of ipAddress and port in the following
// format: "IP_ADDRESS:PORT".
// (The " are used to visualize the beginning and end of the format.)
func (networkServer NetworkServer) IpAndPortAsString() string {
	return networkServer.IpAddressAsString() + ":" + strconv.Itoa(networkServer.Port())
}

// String gives a string object which contains a representation of a NetworkServer type.
func (networkServer NetworkServer) String() string {
	return networkServer.IpAddressAsString() + ":" + strconv.Itoa(networkServer.Port())
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package xwatch

import (
	"errors"
	"sync"

	"github.com/m3db/m3x/close"
	"github.com/m3db/m3x/log"
)

// ErrSourceClosed could be thrown from SourceInput to indicate that the Source should be closed
var ErrSourceClosed = errors.New("source closed")

// SourceInput provides data for Source
type SourceInput interface {
	// Poll will be called by Source for data, any backoff/jitter logic should be handled here
	Poll() (interface{}, error)
}

// Source polls data by calling SourcePollFn and notifies its watches on updates
type Source interface {
	xclose.SimpleCloser

	// WaitInit returns a channel that blocks until the Source was initialized
	WaitInit() <-chan struct{}
	// Get returns the latest value
	Get() interface{}
	// Watch returns the value and an Watch
	Watch() (interface{}, Watch, error)
}

// NewSource returns a Source
func NewSource(input SourceInput, logger xlog.Logger) Source {
	s := &source{
		input:  input,
		w:      NewWatchable(),
		logger: logger,
		initCh:     make(chan struct{}),
	}

	go s.run()
	return s
}

type source struct {
	sync.RWMutex

	input       SourceInput
	w           Watchable
	closed      bool
	initialized bool
	initCh      chan struct{}
	logger      xlog.Logger
}

func (s *source) run() {
	for !s.isClosed() {
		data, err := s.input.Poll()
		if err == ErrSourceClosed {
			s.logger.Errorf("Upstream source is closed")
			s.Close()
			return
		}
		if err != nil {
			s.logger.Errorf("Error polling input source: %v", err)
			continue
		}

		err = s.w.Update(data)

		if err == nil && !s.initialized {
			close(s.initCh)
			s.initialized = true
		}
	}
}

func (s *source) isClosed() bool {
	s.RLock()
	defer s.RUnlock()
	return s.closed
}

func (s *source) WaitInit() <-chan struct{} {
	return s.initCh
}

func (s *source) Close() {
	s.Lock()
	defer s.Unlock()
	if s.closed {
		return
	}
	s.closed = true
	s.w.Close()
}

func (s *source) Get() interface{} {
	return s.w.Get()
}

func (s *source) Watch() (interface{}, Watch, error) {
	return s.w.Watch()
}
<commit_msg>Use lower case log messages for consistency in xwatch<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package xwatch

import (
	"errors"
	"sync"

	"github.com/m3db/m3x/close"
	"github.com/m3db/m3x/log"
)

// ErrSourceClosed could be thrown from SourceInput to indicate that the Source should be closed
var ErrSourceClosed = errors.New("source closed")

// SourceInput provides data for Source
type SourceInput interface {
	// Poll will be called by Source for data, any backoff/jitter logic should be handled here
	Poll() (interface{}, error)
}

// Source polls data by calling SourcePollFn and notifies its watches on updates
type Source interface {
	xclose.SimpleCloser

	// WaitInit returns a channel that blocks until the Source was initialized
	WaitInit() <-chan struct{}
	// Get returns the latest value
	Get() interface{}
	// Watch returns the value and an Watch
	Watch() (interface{}, Watch, error)
}

// NewSource returns a Source
func NewSource(input SourceInput, logger xlog.Logger) Source {
	s := &source{
		input:  input,
		w:      NewWatchable(),
		logger: logger,
		initCh: make(chan struct{}),
	}

	go s.run()
	return s
}

type source struct {
	sync.RWMutex

	input       SourceInput
	w           Watchable
	closed      bool
	initialized bool
	initCh      chan struct{}
	logger      xlog.Logger
}

func (s *source) run() {
	for !s.isClosed() {
		data, err := s.input.Poll()
		if err == ErrSourceClosed {
			s.logger.Errorf("watch source upstream is closed")
			s.Close()
			return
		}
		if err != nil {
			s.logger.Errorf("watch source poll error: %v", err)
			continue
		}

		err = s.w.Update(data)

		if err == nil && !s.initialized {
			close(s.initCh)
			s.initialized = true
		}
	}
}

func (s *source) isClosed() bool {
	s.RLock()
	defer s.RUnlock()
	return s.closed
}

func (s *source) WaitInit() <-chan struct{} {
	return s.initCh
}

func (s *source) Close() {
	s.Lock()
	defer s.Unlock()
	if s.closed {
		return
	}
	s.closed = true
	s.w.Close()
}

func (s *source) Get() interface{} {
	return s.w.Get()
}

func (s *source) Watch() (interface{}, Watch, error) {
	return s.w.Watch()
}
<|endoftext|>


<commit_before>package watcher_test

import (
	"io/ioutil"
	"path/filepath"
	"testing"
	"time"

	"gopkg.in/fsnotify.v1"

	"github.com/minodisk/go-watcher"
)

type Task struct {
	Text     string
	Filename string
	Duration time.Duration
}

func TestWatch(t *testing.T) {
	tasks := []*Task{
		&Task{"foo", "fixtures/foo", time.Second * 1},
		&Task{"baz", "fixtures/bar/baz", time.Second * 2},
		&Task{"quux", "fixtures/bar/qux/quux", time.Second * 3},
	}

	for _, task := range tasks {
		filename, err := filepath.Abs(task.Filename)
		if err != nil {
			t.Fatal(err)
		}
		task.Filename = filename

		go func(task *Task) {
			time.Sleep(task.Duration)
			err := ioutil.WriteFile(task.Filename, []byte(task.Text), 0644)
			if err != nil {
				t.Fatal(err)
			}
		}(task)
	}

	done := make(chan bool)
	w := watcher.NewWatcher()
	go func() {
		i := 0
		for {
			select {
			case event := <-w.Events:
				if event.Op&fsnotify.Write == fsnotify.Write {
					task := tasks[i]
					i++

					actual := event.Name
					expected := task.Filename
					if actual == expected {
						if i == 3 {
							w.Close()
							done <- true
							return
						}
					} else {
						t.Errorf("filename is expected %s, but actual %s", expected, actual)
					}
				}
			}
		}
	}()
	w.Watch([]string{"fixtures"})
	<-done
}
<commit_msg>Trace log<commit_after>package watcher_test

import (
	"io/ioutil"
	"log"
	"path/filepath"
	"testing"
	"time"

	"gopkg.in/fsnotify.v1"

	"github.com/minodisk/go-watcher"
)

type Task struct {
	Text     string
	Filename string
	Duration time.Duration
}

func TestWatch(t *testing.T) {
	tasks := []*Task{
		&Task{"foo", "fixtures/foo", time.Second * 1},
		&Task{"baz", "fixtures/bar/baz", time.Second * 2},
		&Task{"quux", "fixtures/bar/qux/quux", time.Second * 3},
	}

	for _, task := range tasks {
		filename, err := filepath.Abs(task.Filename)
		if err != nil {
			t.Fatal(err)
		}
		task.Filename = filename

		go func(task *Task) {
			time.Sleep(task.Duration)
			err := ioutil.WriteFile(task.Filename, []byte(task.Text), 0644)
			if err != nil {
				t.Fatal(err)
			}
		}(task)
	}

	done := make(chan bool)
	w := watcher.NewWatcher()
	go func() {
		i := 0
		for {
			select {
			case event := <-w.Events:
				log.Println(event)
				if event.Op&fsnotify.Write == fsnotify.Write {
					if i > len(tasks) {
						t.Fatal("should be notified less than %d", len(tasks))
					}

					task := tasks[i]
					i++

					actual := event.Name
					expected := task.Filename
					if actual == expected {
						if i == 3 {
							w.Close()
							done <- true
							return
						}
					} else {
						t.Errorf("filename is expected %s, but actual %s", expected, actual)
					}
				}
			}
		}
	}()
	w.Watch([]string{"fixtures"})
	<-done
}
<|endoftext|>