<commit_before>package tabular

import (
	"fmt"
	"io"
)

// ErrInvalidRowWidth is error returned when adding new row with invalid width.
type ErrInvalidRowWidth struct {
	actual   int
	expected int
}

func (e ErrInvalidRowWidth) Error() string {
	return fmt.Sprintf("Invalid row width = %d, expected = %d.", e.actual, e.expected)
}

// ErrHeadersRequired is error returned when writer is missing the headers.
type ErrHeadersRequired struct {
	w Writer
}

func (e ErrHeadersRequired) Error() string {
	return fmt.Sprintf("Writer %s needs headers.", e.w.Name())
}

// NewDataSet creates new dataset.
func NewDataSet() *Dataset {
	d := &Dataset{
		headers: &Headers{},
	}
	return d
}

// Dataset represents set of data.
type Dataset struct {
	headers *Headers
	rows    []*Row

	columns int
	widths  map[int]int
}

// AddHeader adds new header.
func (d *Dataset) AddHeader(key string, title string) {
	d.headers.Add(key, title)
	d.updateHeaders()
}

// GetHeader returns header by its index.
func (d *Dataset) GetHeader(idx int) (*Header, bool) {
	return d.headers.Get(idx)
}

// HeaderCount returns header count.
func (d *Dataset) HeaderCount() int {
	return d.headers.Len()
}

// Headers returns a slice of headers.
func (d *Dataset) Headers() []*Header {
	return d.headers.Items()
}

// HasHeaders returns true if headers was set.
func (d *Dataset) HasHeaders() bool {
	return d.headers.Len() > 0
}

// Append appends new rows to the dataset.
func (d *Dataset) Append(rows ...*Row) error {
	for _, r := range rows {
		if err := d.validateRow(r); err != nil {
			return err
		}
		d.rows = append(d.rows, r)
		d.updateWidths(r)
	}
	return nil
}

// HasCol checks the presence of given column.
func (d *Dataset) HasCol(key string) bool {
	_, ok := d.getColumnIndex(key)
	return ok
}

// GetColValues returns values of given column.
func (d *Dataset) GetColValues(key string) []string {
	if idx, ok := d.getColumnIndex(key); ok {
		var col []string
		for _, row := range d.rows {
			col = append(col, row.Get(idx))
		}
		return col
	}
	return nil
}

// GetColsValues returns values of given columns.
func (d *Dataset) GetColsValues(keys []string) map[string][]string {
	data := make(map[string][]string)
	for _, key := range keys {
		if cols := d.GetColValues(key); cols != nil {
			data[key] = cols
		}
	}
	return data
}

// GetKeyWidth returns maximum column width.
func (d *Dataset) GetKeyWidth(key string) int {
	if idx, ok := d.getColumnIndex(key); ok {
		return d.getIndexWidth(idx)
	}
	return 0
}

// GetIdxWidth returns maximum column width.
func (d *Dataset) GetIdxWidth(idx int) int {
	return d.getIndexWidth(idx)
}

// Find filters dataset for tag.
func (d *Dataset) Find(tag string) *Dataset {
	var rows []*Row
	for _, row := range d.rows {
		if row.HasTag(tag) {
			rows = append(rows, row)
		}
	}
	d.rows = rows
	return d
}

// FindAny filters dataset for any tags.
func (d *Dataset) FindAny(tags ...string) *Dataset {
	var rows []*Row
	for _, row := range d.rows {
		if row.HasAnyTags(tags...) {
			rows = append(rows, row)
		}
	}
	d.rows = rows
	return d
}

// FindAll filters dataset for all tags.
func (d *Dataset) FindAll(tags ...string) *Dataset {
	var rows []*Row
	for _, row := range d.rows {
		if row.HasAllTags(tags...) {
			rows = append(rows, row)
		}
	}
	d.rows = rows
	return d
}

// Slice returns sliced dataset.
func (d *Dataset) Slice(start int, end int) *Dataset {
	d.rows = d.rows[start:end]
	return d
}

// Sort sorts dataset by key, set reverse to inverse the order direction.
func (d *Dataset) Sort(key string, reverse bool) *Dataset {
	if idx, ok := d.getColumnIndex(key); ok {
		sorter := &RowSorter{
			rows:    d.rows,
			idx:     idx,
			reverse: reverse,
		}
		d.rows = sorter.Sort()
	}
	return d
}

// Write writes dataset using dataset writer to writer.
func (d *Dataset) Write(dw Writer, w io.Writer) error {
	if dw.NeedsHeaders() && d.headers.Empty() {
		return ErrHeadersRequired{dw}
	}
	return dw.Write(d, w)
}

// Get returns a row on given index.
func (d *Dataset) Get(idx int) (*Row, bool) {
	if d.isValidIndex(idx) {
		return d.rows[idx], true
	}
	return nil, false
}

// Rows returns all rows of dataset.
func (d *Dataset) Rows() []*Row {
	return d.rows
}

// Len returns the row count.
func (d *Dataset) Len() int {
	return len(d.rows)
}

func (d *Dataset) validateRow(r *Row) error {
	columns := r.Len()
	if columns < 1 {
		return ErrInvalidRowWidth{
			actual:   columns,
			expected: 1,
		}
	}

	if d.Len() > 0 && d.columns != columns {
		return ErrInvalidRowWidth{
			actual:   columns,
			expected: d.columns,
		}
	}

	return nil
}

func (d *Dataset) updateHeaders() {
	d.columns = d.HeaderCount()
}

func (d *Dataset) updateWidths(r *Row) {
	if !d.HasHeaders() {
		d.columns = r.Len()
	}

	if d.widths == nil {
		d.widths = make(map[int]int)
	}

	for idx, item := range r.Items() {
		l := len(item)
		if l > d.widths[idx] {
			d.widths[idx] = l
		}
	}
}

func (d *Dataset) getColumnIndex(key string) (int, bool) {
	for i, col := range d.headers.Items() {
		if col.Key == key {
			return i, true
		}
	}
	return 0, false
}

func (d *Dataset) getIndexWidth(idx int) int {
	if d.isValidIndex(idx) {
		width := d.widths[idx]
		if d.HasHeaders() {
			if h, ok := d.GetHeader(idx); ok {
				hdrWidth := len(h.Title)
				if hdrWidth > width {
					return hdrWidth
				}
			}
		}
		return width
	}
	return d.widths[idx]
}

func (d *Dataset) isValidIndex(idx int) bool {
	if d.Len() == 0 {
		return false
	}
	return idx >= 0 && idx <= d.Len()
}
<commit_msg>remove GetColsValues method<commit_after>package tabular

import (
	"fmt"
	"io"
)

// ErrInvalidRowWidth is error returned when adding new row with invalid width.
type ErrInvalidRowWidth struct {
	actual   int
	expected int
}

func (e ErrInvalidRowWidth) Error() string {
	return fmt.Sprintf("Invalid row width = %d, expected = %d.", e.actual, e.expected)
}

// ErrHeadersRequired is error returned when writer is missing the headers.
type ErrHeadersRequired struct {
	w Writer
}

func (e ErrHeadersRequired) Error() string {
	return fmt.Sprintf("Writer %s needs headers.", e.w.Name())
}

// NewDataSet creates new dataset.
func NewDataSet() *Dataset {
	d := &Dataset{
		headers: &Headers{},
	}
	return d
}

// Dataset represents a set of data.
type Dataset struct {
	headers *Headers
	rows    []*Row

	columns int
	lengths map[int]int
}

// AddHeader adds new header.
func (d *Dataset) AddHeader(key string, title string) {
	d.headers.Add(key, title)
	d.updateHeaders()
}

// GetHeader returns header by its index.
func (d *Dataset) GetHeader(idx int) (*Header, bool) {
	return d.headers.Get(idx)
}

// HeaderCount returns header count.
func (d *Dataset) HeaderCount() int {
	return d.headers.Len()
}

// Headers returns a slice of headers.
func (d *Dataset) Headers() []*Header {
	return d.headers.Items()
}

// HasHeaders returns true if headers was set.
func (d *Dataset) HasHeaders() bool {
	return d.headers.Len() > 0
}

// Append appends new rows to the dataset.
func (d *Dataset) Append(rows ...*Row) error {
	for _, r := range rows {
		if err := d.validateRow(r); err != nil {
			return err
		}
		d.rows = append(d.rows, r)
		d.updateLengths(r)
	}
	return nil
}

// HasCol checks the presence of given column.
func (d *Dataset) HasCol(key string) bool {
	_, ok := d.getColumnIndex(key)
	return ok
}

// GetColValues returns values of given column.
func (d *Dataset) GetColValues(key string) []string {
	if idx, ok := d.getColumnIndex(key); ok {
		var col []string
		for _, row := range d.rows {
			col = append(col, row.Get(idx))
		}
		return col
	}
	return nil
}

// GetKeyWidth returns maximum column width.
func (d *Dataset) GetKeyWidth(key string) int {
	if idx, ok := d.getColumnIndex(key); ok {
		return d.getIndexWidth(idx)
	}
	return 0
}

// GetIdxWidth returns maximum column width.
func (d *Dataset) GetIdxWidth(idx int) int {
	return d.getIndexWidth(idx)
}

// Find filters dataset for tag.
func (d *Dataset) Find(tag string) *Dataset {
	var rows []*Row
	for _, row := range d.rows {
		if row.HasTag(tag) {
			rows = append(rows, row)
		}
	}
	d.rows = rows
	return d
}

// FindAny filters dataset for any tags.
func (d *Dataset) FindAny(tags ...string) *Dataset {
	var rows []*Row
	for _, row := range d.rows {
		if row.HasAnyTags(tags...) {
			rows = append(rows, row)
		}
	}
	d.rows = rows
	return d
}

// FindAll filters dataset for all tags.
func (d *Dataset) FindAll(tags ...string) *Dataset {
	var rows []*Row
	for _, row := range d.rows {
		if row.HasAllTags(tags...) {
			rows = append(rows, row)
		}
	}
	d.rows = rows
	return d
}

// Slice returns sliced dataset.
func (d *Dataset) Slice(start int, end int) *Dataset {
	d.rows = d.rows[start:end]
	return d
}

// Sort sorts dataset by key, set reverse to inverse the order direction.
func (d *Dataset) Sort(key string, reverse bool) *Dataset {
	if idx, ok := d.getColumnIndex(key); ok {
		sorter := &RowSorter{
			rows:    d.rows,
			idx:     idx,
			reverse: reverse,
		}
		d.rows = sorter.Sort()
	}
	return d
}

// Write writes dataset using dataset writer to writer.
func (d *Dataset) Write(dw Writer, w io.Writer) error {
	if dw.NeedsHeaders() && d.headers.Empty() {
		return ErrHeadersRequired{dw}
	}
	return dw.Write(d, w)
}

// Get returns a row on given index.
func (d *Dataset) Get(idx int) (*Row, bool) {
	if d.isValidIndex(idx) {
		return d.rows[idx], true
	}
	return nil, false
}

// Rows returns all rows of dataset.
func (d *Dataset) Rows() []*Row {
	return d.rows
}

// Len returns the row count.
func (d *Dataset) Len() int {
	return len(d.rows)
}

func (d *Dataset) validateRow(r *Row) error {
	columns := r.Len()
	if columns < 1 {
		return ErrInvalidRowWidth{
			actual:   columns,
			expected: 1,
		}
	}

	if d.Len() > 0 && d.columns != columns {
		return ErrInvalidRowWidth{
			actual:   columns,
			expected: d.columns,
		}
	}

	return nil
}

func (d *Dataset) updateHeaders() {
	d.columns = d.HeaderCount()
}

func (d *Dataset) updateLengths(r *Row) {
	if !d.HasHeaders() {
		d.columns = r.Len()
	}

	if d.lengths == nil {
		d.lengths = make(map[int]int)
	}

	for idx, item := range r.Items() {
		l := len(item)
		if l > d.lengths[idx] {
			d.lengths[idx] = l
		}
	}
}

func (d *Dataset) getColumnIndex(key string) (int, bool) {
	for i, col := range d.headers.Items() {
		if col.Key == key {
			return i, true
		}
	}
	return 0, false
}

func (d *Dataset) getIndexWidth(idx int) int {
	if d.isValidIndex(idx) {
		length := d.lengths[idx]
		if d.HasHeaders() {
			if h, ok := d.GetHeader(idx); ok {
				hdrWidth := len(h.Title)
				if hdrWidth > length {
					return hdrWidth
				}
			}
		}
		return length
	}
	return d.lengths[idx]
}

func (d *Dataset) isValidIndex(idx int) bool {
	if d.Len() == 0 {
		return false
	}
	return idx >= 0 && idx <= d.Len()
}
<|endoftext|>


<commit_before>package main

import (
    "github.com/Lupino/huabot-brain/models"
    "github.com/mikespook/gearman-go/client"
    "mime/multipart"
    "crypto/sha1"
    "io"
    "os"
    "encoding/hex"
    "encoding/json"
    "fmt"
    "sync"
    "image"
    "log"
    _ "image/png"
    _ "image/jpeg"
)

type PredictTag struct {
    Id int         `json:"id,omitempty"`
    Score float64  `json:"score,omitempty"`
    Tag models.Tag `json:"tag,omitempty"`
}
type PredictResult struct {
    BetResult []PredictTag `json:"bet_result,omitempty"`
    Time  float64          `json:"time,omitempty"`
    Error string           `json:"err,omitempty"`
}

func uploadFile(realFile *multipart.FileHeader) (file *models.File, err error) {
    var source multipart.File
    if source, err = realFile.Open(); err != nil {
        return
    }
    defer source.Close()
    var img image.Config
    if img, _, err = image.DecodeConfig(source); err != nil {
        return
    }
    source.Seek(0, 0)
    hasher := sha1.New()
    io.Copy(hasher, source)
    fileKey := hex.EncodeToString(hasher.Sum(nil))
    file = &models.File{Key: fileKey}
    var engine = models.GetEngine()
    has, _ := engine.Get(file)
    if !has {
        var dst *os.File
        if dst, err = os.Create(*UPLOADPATH + fileKey); err != nil {
            return
        }
        defer dst.Close()
        source.Seek(0, 0)
        if _, err = io.Copy(dst, source); err != nil {
            return
        }

        file.Width = img.Width
        file.Height = img.Height

        if _, err = engine.Insert(file); err != nil {
            return
        }
    }
    return
}

func saveTag(realTag string) (tag *models.Tag, err error) {
    tag = &models.Tag{Name: realTag}
    var engine = models.GetEngine()
    has, _ := engine.Get(tag)
    if !has {
        if _, err = engine.Insert(tag); err != nil {
            return
        }
    }
    return
}

func saveDataset(file *models.File, tag *models.Tag, dataType uint, desc string) (dataset *models.Dataset, err error) {
    dataset = &models.Dataset{FileId: file.Id, TagId: tag.Id}
    var engine = models.GetEngine()
    has, _ := engine.Get(dataset)
    if !has {
        dataset.DataType = dataType
        dataset.Description = desc
        if _, err = engine.Insert(dataset); err != nil {
            return
        }
        var sql string
        if dataType == models.TRAIN {
          sql = "update `tag` set `train_count` = `train_count` + 1 where `id` = ?"
        } else if dataType == models.VAL {
          sql = "update `tag` set `test_count` = `test_count` + 1 where `id` = ?"
        }
        engine.Exec(sql, tag.Id)
    }
    dataset.File = file
    dataset.Tag = tag
    return
}

func loadDataset(dataType uint) (text string, err error) {
    var engine = models.GetEngine()
    err = engine.Where("data_type=?", dataType).Iterate(new(models.Dataset), func(i int, bean interface{}) error {
        dataset := bean.(*models.Dataset)
        dataset.FillObject()
        text = fmt.Sprintf("%s%s %d\n", text, dataset.File.Key, dataset.TagId)
        return nil
    })
    return
}

func submit(funcName string, workload []byte) ([]byte, error) {
    var mutex sync.Mutex
    var result []byte
    var errResult error
    c, err := client.New("tcp4", *GEARMAND)
    if err != nil {
        return nil, err
    }
    defer c.Close()
    c.ErrorHandler = func(e error) {
        log.Println(e)
    }
    jobHandler := func(resp *client.Response) {
        if resp.DataType == client.WorkComplate {
            result, errResult = resp.Result()
            mutex.Unlock()
        }
    }
    _, err = c.Do(funcName, workload, client.JobNormal, jobHandler)
    if err != nil {
        log.Printf("gearman Do %s Error: %s\n", funcName, err)
        return nil, err
    }
    mutex.Lock()
    mutex.Lock()
    return result, errResult
}

func caffeTrain() (string, error) {
    result, err := submit("CAFFE:TRAIN", nil)
    if err != nil {
        return "", err
    }
    return string(result), nil
}

func caffeTrainStop() (string, error) {
    result, err := submit("CAFFE:TRAIN:STOP", nil)
    if err != nil {
        return "", err
    }
    return string(result), nil
}

func caffeTrainStatus() ([]byte, error) {
    result, err := submit("CAFFE:TRAIN:STATUS", nil)
    if err != nil {
        return nil, err
    }
    return result, nil
}

func caffeTrainPlot(plotType string) ([]byte, error) {
    return submit("CAFFE:TRAIN:PLOT", []byte(plotType))
}

func caffePredict(url string) (result PredictResult, err error) {
    var data []byte
    if data, err = submit("CAFFE:PREDICT:URL", []byte(url)); err != nil {
        return
    }
    if err = json.Unmarshal(data, &result); err != nil {
        return
    }

    var engine = models.GetEngine()
    for i, ptag := range result.BetResult {
        ptag.Tag.Id = ptag.Id
        engine.Get(&ptag.Tag)
        result.BetResult[i].Tag = ptag.Tag
    }
    return
}
<commit_msg>Fixed. gearman client error padding<commit_after>package main

import (
    "github.com/Lupino/huabot-brain/models"
    "github.com/mikespook/gearman-go/client"
    "mime/multipart"
    "crypto/sha1"
    "io"
    "os"
    "encoding/hex"
    "encoding/json"
    "fmt"
    "sync"
    "image"
    "log"
    "bytes"
    "errors"
    _ "image/png"
    _ "image/jpeg"
)

type PredictTag struct {
    Id int         `json:"id,omitempty"`
    Score float64  `json:"score,omitempty"`
    Tag models.Tag `json:"tag,omitempty"`
}
type PredictResult struct {
    BetResult []PredictTag `json:"bet_result,omitempty"`
    Time  float64          `json:"time,omitempty"`
    Error string           `json:"err,omitempty"`
}

func uploadFile(realFile *multipart.FileHeader) (file *models.File, err error) {
    var source multipart.File
    if source, err = realFile.Open(); err != nil {
        return
    }
    defer source.Close()
    var img image.Config
    if img, _, err = image.DecodeConfig(source); err != nil {
        return
    }
    source.Seek(0, 0)
    hasher := sha1.New()
    io.Copy(hasher, source)
    fileKey := hex.EncodeToString(hasher.Sum(nil))
    file = &models.File{Key: fileKey}
    var engine = models.GetEngine()
    has, _ := engine.Get(file)
    if !has {
        var dst *os.File
        if dst, err = os.Create(*UPLOADPATH + fileKey); err != nil {
            return
        }
        defer dst.Close()
        source.Seek(0, 0)
        if _, err = io.Copy(dst, source); err != nil {
            return
        }

        file.Width = img.Width
        file.Height = img.Height

        if _, err = engine.Insert(file); err != nil {
            return
        }
    }
    return
}

func saveTag(realTag string) (tag *models.Tag, err error) {
    tag = &models.Tag{Name: realTag}
    var engine = models.GetEngine()
    has, _ := engine.Get(tag)
    if !has {
        if _, err = engine.Insert(tag); err != nil {
            return
        }
    }
    return
}

func saveDataset(file *models.File, tag *models.Tag, dataType uint, desc string) (dataset *models.Dataset, err error) {
    dataset = &models.Dataset{FileId: file.Id, TagId: tag.Id}
    var engine = models.GetEngine()
    has, _ := engine.Get(dataset)
    if !has {
        dataset.DataType = dataType
        dataset.Description = desc
        if _, err = engine.Insert(dataset); err != nil {
            return
        }
        var sql string
        if dataType == models.TRAIN {
          sql = "update `tag` set `train_count` = `train_count` + 1 where `id` = ?"
        } else if dataType == models.VAL {
          sql = "update `tag` set `test_count` = `test_count` + 1 where `id` = ?"
        }
        engine.Exec(sql, tag.Id)
    }
    dataset.File = file
    dataset.Tag = tag
    return
}

func loadDataset(dataType uint) (text string, err error) {
    var engine = models.GetEngine()
    err = engine.Where("data_type=?", dataType).Iterate(new(models.Dataset), func(i int, bean interface{}) error {
        dataset := bean.(*models.Dataset)
        dataset.FillObject()
        text = fmt.Sprintf("%s%s %d\n", text, dataset.File.Key, dataset.TagId)
        return nil
    })
    return
}

func submit(funcName string, workload []byte) ([]byte, error) {
    var mutex sync.Mutex
    var result []byte
    var errResult error
    c, err := client.New("tcp4", *GEARMAND)
    if err != nil {
        return nil, err
    }
    defer c.Close()
    c.ErrorHandler = func(e error) {
        log.Println(e)
    }
    jobHandler := func(resp *client.Response) {
        result, errResult = resp.Result()
        mutex.Unlock()
    }
    _, err = c.Do(funcName, workload, client.JobNormal, jobHandler)
    if err != nil {
        log.Printf("Gearman Doing %s Error: %s\n", funcName, err)
        return nil, err
    }
    mutex.Lock()
    mutex.Lock()
    if bytes.Equal(result, []byte("error")) {
        errResult = errors.New("Gearman doing " + funcName + " Error")
    }
    return result, errResult
}

func caffeTrain() (string, error) {
    result, err := submit("CAFFE:TRAIN", nil)
    if err != nil {
        return "", err
    }
    return string(result), nil
}

func caffeTrainStop() (string, error) {
    result, err := submit("CAFFE:TRAIN:STOP", nil)
    if err != nil {
        return "", err
    }
    return string(result), nil
}

func caffeTrainStatus() ([]byte, error) {
    result, err := submit("CAFFE:TRAIN:STATUS", nil)
    if err != nil {
        return nil, err
    }
    return result, nil
}

func caffeTrainPlot(plotType string) ([]byte, error) {
    return submit("CAFFE:TRAIN:PLOT", []byte(plotType))
}

func caffePredict(url string) (result PredictResult, err error) {
    var data []byte
    if data, err = submit("CAFFE:PREDICT:URL", []byte(url)); err != nil {
        return
    }
    if err = json.Unmarshal(data, &result); err != nil {
        return
    }

    var engine = models.GetEngine()
    for i, ptag := range result.BetResult {
        ptag.Tag.Id = ptag.Id
        engine.Get(&ptag.Tag)
        result.BetResult[i].Tag = ptag.Tag
    }
    return
}
<|endoftext|>


<commit_before>package effio

import (
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
)

func (cmd *Cmd) SummarizeCSV() {
	var hbktFlag int
	var inFlag, outFlag string
	var jsonFlag bool

	cmd.DefaultFlags()
	cmd.FlagSet.StringVar(&inFlag, "in", "", "CSV file to load")
	cmd.FlagSet.IntVar(&hbktFlag, "hbkt", 10, "data bin width")
	cmd.FlagSet.StringVar(&outFlag, "out", "", "CSV file to write")
	cmd.FlagSet.BoolVar(&jsonFlag, "json", false, "Print JSON instead of human-readable text.")
	cmd.ParseArgs()

	recs := LoadFioLog(inFlag)
	smry := recs.Summarize(hbktFlag)
	AppendMetadata(inFlag, &smry)

	if jsonFlag {
		os.Stdout.Write(toJson(smry))
	} else {
		printSummary(smry)
	}
}

// effio summarize-all -path suites -out public/data
func (cmd *Cmd) SummarizeAll() {
	var hbktFlag int
	var outFlag string

	cmd.DefaultFlags()
	cmd.FlagSet.IntVar(&hbktFlag, "hbkt", 10, "data bin width")
	cmd.FlagSet.StringVar(&outFlag, "out", "public/data", "directory to write summaries to")
	cmd.ParseArgs()

	fi, err := os.Stat(outFlag)
	if err != nil {
		log.Fatalf("Could not stat '%s': %s\n", outFlag, err)
	}
	if !fi.IsDir() {
		log.Fatalf("'%s' must be a directory!\n", outFlag)
	}

	files := InventoryCSVFiles(cmd.PathFlag)

	for _, file := range files {
		fi, err := os.Stat(file)
		if err != nil {
			log.Fatalf("Could not stat CSV file '%s': %s\n", file, err)
		}
		// avoid tiny files, not enough data for summarize to work
		if fi.Size() < 5000 {
			log.Printf("Skipping file %q because it is only %d bytes.\n", file, fi.Size())
			continue
		}

		recs := LoadFioLog(file)
		smry := recs.Summarize(hbktFlag)
		smry.Name = path.Base(file)
		smry.Path = file

		parts := strings.Split(smry.Name, ".")
		switch parts[0] {
		case "bw_bw":
			smry.LogType = "bw"
		case "lat_lat":
			smry.LogType = "lat"
		case "lat_slat":
			smry.LogType = "slat"
		case "lat_clat":
			smry.LogType = "clat"
		case "iops_iops":
			smry.LogType = "iops"
		}

		AppendMetadata(file, &smry)

		// output filename is SHA1 of the source file
		sha1sum := sha1file(file)
		outpath := path.Join(outFlag, fmt.Sprintf("%s-%s.json", sha1sum, smry.LogType))

		out, err := os.OpenFile(outpath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
		if err != nil {
			log.Fatalf("Could not open file '%s' for write: %s\n", outpath, err)
		}
		out.Write(toJson(smry))
	}
}

func InventoryCSVFiles(dpath string) []string {
	out := make([]string, 0)
	// WARNING: using assumptions based on effio conventions
	roots := []string{"bw_bw", "lat_lat", "lat_slat", "lat_clat", "iops_iops"}
	wanted := make([]string, 30)

	// WARNING: generate all the names I've seen. This is a big fragile. For example,
	// it will ignore logs > 5 e.g. bw_bw.8.log would be ignored.
	count := 0
	for i, top := range roots {
		wanted[count] = fmt.Sprintf("%s.log", top)
		count++
		for _, name := range roots {
			wanted[count] = fmt.Sprintf("%s.%d.log", name, i)
			count++
		}
	}

	visitor := func(dpath string, f os.FileInfo, err error) error {
		if err != nil {
			log.Fatalf("Encountered an error while inventorying files in '%s': %s", dpath, err)
		}

		fi, err := os.Stat(dpath)
		if err != nil {
			log.Fatalf("Could not stat '%s': %s\n", dpath, err)
		}

		// skip empty and tiny files
		if fi.Size() < 100 {
			return nil
		}

		for _, want := range wanted {
			if path.Base(dpath) == want {
				out = append(out, dpath)
			}
		}

		return nil
	}

	err := filepath.Walk(dpath, visitor)
	if err != nil {
		log.Fatalf("Could not inventory files in '%s': %s", dpath, err)
	}

	return out
}

func toJson(smry LogSummaries) []byte {
	js, err := json.Marshal(smry)
	if err != nil {
		fmt.Printf("Failed to encode summary data as JSON: %s\n", err)
		os.Exit(1)
	}
	return append(js, byte('\n'))
}

func printSummary(smry LogSummaries) {
	fmt.Printf("Min:                %d\n", smry.Summary.Min)
	fmt.Printf("Max:                %d\n", smry.Summary.Max)
	fmt.Printf("Count:              %d\n", smry.Summary.Count)
	fmt.Printf("Sum:                %d\n", smry.Summary.Sum)
	fmt.Printf("Average:            %g\n", smry.Summary.Average)
	fmt.Printf("Standard Deviation: %g\n", smry.Summary.Stdev)
	fmt.Printf("Begin Timestamp:    %d\n", smry.Summary.MinTs)
	fmt.Printf("End Timestamp:      %d\n", smry.Summary.MaxTs)
	fmt.Printf("Elapsed Time:       %d\n", smry.Summary.Elapsed)
	fmt.Printf("\n")
	fmt.Printf("P1:    % 8d P5:     % 8d P10:     % 8d\n", smry.Pcntl[1].Val, smry.Pcntl[5].Val, smry.Pcntl[10].Val)
	fmt.Printf("P25:   % 8d P50:    % 8d P75:     % 8d\n", smry.Pcntl[25].Val, smry.Pcntl[50].Val, smry.Pcntl[75].Val)
	fmt.Printf("P90:   % 8d P95:    % 8d P99:     % 8d\n", smry.Pcntl[90].Val, smry.Pcntl[95].Val, smry.Pcntl[99].Val)
	fmt.Printf("P99.9: % 8d P99.99: % 8d P99.999: % 8d\n", smry.Pcntl[99.9].Val, smry.Pcntl[99.99].Val, smry.Pcntl[99.999].Val)

	fmt.Printf("\nAll Binned Data[% 4d]:   ", len(smry.Bin))
	for _, bkt := range smry.Bin {
		fmt.Printf("% 7.3f ", bkt.Average)
	}
	fmt.Printf("\nRead Binned Data[% 4d]:  ", len(smry.RBin))
	for _, bkt := range smry.RBin {
		fmt.Printf("% 7.3f ", bkt.Average)
	}
	fmt.Printf("\nWrite Binned Data[% 4d]: ", len(smry.WBin))
	for _, bkt := range smry.WBin {
		fmt.Printf("% 7.3f ", bkt.Average)
	}
	fmt.Printf("\nTrim Binned Data[% 4d]:  ", len(smry.TBin))
	for _, bkt := range smry.TBin {
		fmt.Printf("% 7.3f ", bkt.Average)
	}
	fmt.Printf("\n")
}

func AppendMetadata(dpath string, smry *LogSummaries) {
	fcmd_filenames := []string{"command.json", "test.json"}
	dir := path.Dir(dpath)

	for _, name := range fcmd_filenames {
		fpath := path.Join(dir, name)
		if fi, err := os.Stat(fpath); err == nil {
			if fi.Size() > 0 {
				smry.FioCommand = LoadFioCommandJson(fpath)
			}
		}
	}

	fpath := path.Join(dir, "output.json")
	if fi, err := os.Stat(fpath); err == nil {
		if fi.Size() > 0 {
			smry.FioJsonData = LoadFioJsonData(fpath)
		}
	}
}

func sha1file(file string) string {
	hasher := sha1.New()

	f, err := os.Open(file)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	if _, err := io.Copy(hasher, f); err != nil {
		log.Fatal(err)
	}

	return fmt.Sprintf("%x", hasher.Sum(nil))
}
<commit_msg>simplify file filtering with a regex<commit_after>package effio

import (
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"
)

func (cmd *Cmd) SummarizeCSV() {
	var hbktFlag int
	var inFlag, outFlag string
	var jsonFlag bool

	cmd.DefaultFlags()
	cmd.FlagSet.StringVar(&inFlag, "in", "", "CSV file to load")
	cmd.FlagSet.IntVar(&hbktFlag, "hbkt", 10, "data bin width")
	cmd.FlagSet.StringVar(&outFlag, "out", "", "CSV file to write")
	cmd.FlagSet.BoolVar(&jsonFlag, "json", false, "Print JSON instead of human-readable text.")
	cmd.ParseArgs()

	recs := LoadFioLog(inFlag)
	smry := recs.Summarize(hbktFlag)
	AppendMetadata(inFlag, &smry)

	if jsonFlag {
		os.Stdout.Write(toJson(smry))
	} else {
		printSummary(smry)
	}
}

// effio summarize-all -path suites -out public/data
func (cmd *Cmd) SummarizeAll() {
	var hbktFlag int
	var outFlag string

	cmd.DefaultFlags()
	cmd.FlagSet.IntVar(&hbktFlag, "hbkt", 10, "data bin width")
	cmd.FlagSet.StringVar(&outFlag, "out", "public/data", "directory to write summaries to")
	cmd.ParseArgs()

	fi, err := os.Stat(outFlag)
	if err != nil {
		log.Fatalf("Could not stat '%s': %s\n", outFlag, err)
	}
	if !fi.IsDir() {
		log.Fatalf("'%s' must be a directory!\n", outFlag)
	}

	files := InventoryCSVFiles(cmd.PathFlag)

	for _, file := range files {
		fi, err := os.Stat(file)
		if err != nil {
			log.Fatalf("Could not stat CSV file '%s': %s\n", file, err)
		}
		// avoid tiny files, not enough data for summarize to work
		if fi.Size() < 5000 {
			log.Printf("Skipping file %q because it is only %d bytes.\n", file, fi.Size())
			continue
		}

		recs := LoadFioLog(file)
		smry := recs.Summarize(hbktFlag)
		smry.Name = path.Base(file)
		smry.Path = file

		parts := strings.Split(smry.Name, ".")
		switch parts[0] {
		case "bw_bw":
			smry.LogType = "bw"
		case "lat_lat":
			smry.LogType = "lat"
		case "lat_slat":
			smry.LogType = "slat"
		case "lat_clat":
			smry.LogType = "clat"
		case "iops_iops":
			smry.LogType = "iops"
		}

		AppendMetadata(file, &smry)

		// output filename is SHA1 of the source file
		sha1sum := sha1file(file)
		outpath := path.Join(outFlag, fmt.Sprintf("%s-%s.json", sha1sum, smry.LogType))

		out, err := os.OpenFile(outpath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
		if err != nil {
			log.Fatalf("Could not open file '%s' for write: %s\n", outpath, err)
		}
		out.Write(toJson(smry))
	}
}

func InventoryCSVFiles(dpath string) []string {
	out := make([]string, 0)
	re := "(bw_bw|lat_lat|lat_slat|lat_clat|iops_iops)\\.?\\d*\\.log$"

	visitor := func(dpath string, f os.FileInfo, err error) error {
		if err != nil {
			log.Fatalf("Encountered an error while inventorying files in '%s': %s", dpath, err)
		}

		fi, err := os.Stat(dpath)
		if err != nil {
			log.Fatalf("Could not stat '%s': %s\n", dpath, err)
		}

		// skip empty and tiny files
		if fi.Size() < 100 {
			return nil
		}

		// WARNING: using assumptions based on effio conventions
		matched, err := regexp.MatchString(re, dpath)
		if err != nil {
			panic(err)
		}
		if matched {
			out = append(out, dpath)
		}

		return nil
	}

	err := filepath.Walk(dpath, visitor)
	if err != nil {
		log.Fatalf("Could not inventory files in '%s': %s", dpath, err)
	}

	return out
}

func toJson(smry LogSummaries) []byte {
	js, err := json.Marshal(smry)
	if err != nil {
		fmt.Printf("Failed to encode summary data as JSON: %s\n", err)
		os.Exit(1)
	}
	return append(js, byte('\n'))
}

func printSummary(smry LogSummaries) {
	fmt.Printf("Min:                %d\n", smry.Summary.Min)
	fmt.Printf("Max:                %d\n", smry.Summary.Max)
	fmt.Printf("Count:              %d\n", smry.Summary.Count)
	fmt.Printf("Sum:                %d\n", smry.Summary.Sum)
	fmt.Printf("Average:            %g\n", smry.Summary.Average)
	fmt.Printf("Standard Deviation: %g\n", smry.Summary.Stdev)
	fmt.Printf("Begin Timestamp:    %d\n", smry.Summary.MinTs)
	fmt.Printf("End Timestamp:      %d\n", smry.Summary.MaxTs)
	fmt.Printf("Elapsed Time:       %d\n", smry.Summary.Elapsed)
	fmt.Printf("\n")
	fmt.Printf("P1:    % 8d P5:     % 8d P10:     % 8d\n", smry.Pcntl[1].Val, smry.Pcntl[5].Val, smry.Pcntl[10].Val)
	fmt.Printf("P25:   % 8d P50:    % 8d P75:     % 8d\n", smry.Pcntl[25].Val, smry.Pcntl[50].Val, smry.Pcntl[75].Val)
	fmt.Printf("P90:   % 8d P95:    % 8d P99:     % 8d\n", smry.Pcntl[90].Val, smry.Pcntl[95].Val, smry.Pcntl[99].Val)
	fmt.Printf("P99.9: % 8d P99.99: % 8d P99.999: % 8d\n", smry.Pcntl[99.9].Val, smry.Pcntl[99.99].Val, smry.Pcntl[99.999].Val)

	fmt.Printf("\nAll Binned Data[% 4d]:   ", len(smry.Bin))
	for _, bkt := range smry.Bin {
		fmt.Printf("% 7.3f ", bkt.Average)
	}
	fmt.Printf("\nRead Binned Data[% 4d]:  ", len(smry.RBin))
	for _, bkt := range smry.RBin {
		fmt.Printf("% 7.3f ", bkt.Average)
	}
	fmt.Printf("\nWrite Binned Data[% 4d]: ", len(smry.WBin))
	for _, bkt := range smry.WBin {
		fmt.Printf("% 7.3f ", bkt.Average)
	}
	fmt.Printf("\nTrim Binned Data[% 4d]:  ", len(smry.TBin))
	for _, bkt := range smry.TBin {
		fmt.Printf("% 7.3f ", bkt.Average)
	}
	fmt.Printf("\n")
}

func AppendMetadata(dpath string, smry *LogSummaries) {
	fcmd_filenames := []string{"command.json", "test.json"}
	dir := path.Dir(dpath)

	for _, name := range fcmd_filenames {
		fpath := path.Join(dir, name)
		if fi, err := os.Stat(fpath); err == nil {
			if fi.Size() > 0 {
				smry.FioCommand = LoadFioCommandJson(fpath)
			}
		}
	}

	fpath := path.Join(dir, "output.json")
	if fi, err := os.Stat(fpath); err == nil {
		if fi.Size() > 0 {
			smry.FioJsonData = LoadFioJsonData(fpath)
		}
	}
}

func sha1file(file string) string {
	hasher := sha1.New()

	f, err := os.Open(file)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	if _, err := io.Copy(hasher, f); err != nil {
		log.Fatal(err)
	}

	return fmt.Sprintf("%x", hasher.Sum(nil))
}
<|endoftext|>


<commit_before>package qbt

import (
	"testing"
)

func TestCommandsFromJsonFile_FindOne(t *testing.T) {
	commands := NewCommandsFromJsonFile("/Users/marcotroisi/gocode/src/github.com/marcotroisi/qbt/file.json")
	result := commands.FindOne("test", "")

	for i, command := range result {
		if i == 0 && command != "vendor/bin/phpspec" {
			t.Error("expected \"vendor/bin/phpspec\", got ", command)
		}
		if i == 1 && command != "vendor/bin/phpunit -c phpunit" {
			t.Error("expected \"vendor/bin/phpunit -c phpunit\", got ", command)
		}
	}
}

func TestCommandsFromJsonFile_FindOneWithBlock(t *testing.T) {
	commands := NewCommandsFromJsonFile("/Users/marcotroisi/gocode/src/github.com/marcotroisi/qbt/file.json")
	result := commands.FindOne("test", "unit")

	for i, command := range result {
		if i == 0 && command != "vendor/bin/phpunit -c phpunit" {
			t.Error("expected \"vendor/bin/phpunit -c phpunit\", got ", command)
		}
	}
}<commit_msg>absolute path for file.json<commit_after>package qbt

import (
	"testing"
	"path/filepath"
)

func TestCommandsFromJsonFile_FindOne(t *testing.T) {
	absPath, _ := filepath.Abs("../file.json")
	commands := NewCommandsFromJsonFile(absPath)
	result := commands.FindOne("test", "")

	for i, command := range result {
		if i == 0 && command != "vendor/bin/phpspec" {
			t.Error("expected \"vendor/bin/phpspec\", got ", command)
		}
		if i == 1 && command != "vendor/bin/phpunit -c phpunit" {
			t.Error("expected \"vendor/bin/phpunit -c phpunit\", got ", command)
		}
	}
}

func TestCommandsFromJsonFile_FindOneWithBlock(t *testing.T) {
	commands := NewCommandsFromJsonFile("/Users/marcotroisi/gocode/src/github.com/marcotroisi/qbt/file.json")
	result := commands.FindOne("test", "unit")

	for i, command := range result {
		if i == 0 && command != "vendor/bin/phpunit -c phpunit" {
			t.Error("expected \"vendor/bin/phpunit -c phpunit\", got ", command)
		}
	}
}<|endoftext|>


<commit_before>package tasks

// NOTE we do not log anything in this file! Leave it to the http handler to handle errors.

import (
	"database/sql"
	"time"
)

// TODO add done, done_by, c_time (maybe m_time?)
type Task struct {
	Key        int64     `json:"key"`
	ShortDesc  string    `json:"short_desc"`
	LongDesc   string    `json:"long_desc"`
	Done       bool      `json:"done"`
	DoneBy     string    `json:"done_by"`
	CreateTime time.Time `json:"c_time"`
	ChoreId    int64     `json:"chore_id"`
}

func setChoresNowStmt(db *sql.DB) (*sql.Stmt, error) {
	return db.Prepare("INSERT INTO tasks(chore_id, c_time) SELECT chore_id, NULL from chores where (morning = ? OR night = ?) AND ((dwm = 'd') OR (dwm = 'w' AND day = ?) OR (dwm = 'm' AND date = ?))")
}

func (h *handler) setChoresNow(t time.Time) error {
	var (
		isMorning int
		isNight   int
	)
	if t.Hour() >= 5 || t.Hour() < 17 {
		isMorning = 1
		isNight = -1
	} else {
		isNight = -1
		isMorning = 1
	}
	_, err := h.setChoresNowStmt.Exec(isMorning, isNight, t.Weekday(), t.Day())
	return err
}

func getAllTasksStmt(db *sql.DB) (*sql.Stmt, error) {
	return db.Prepare("SELECT task_id, chores.short_desc, chores.long_desc, done, done_by, tasks.c_time, tasks.chore_id from tasks left join chores on (tasks.chore_id = chores.chore_id)")
}

func (h *handler) getAllTasks() ([]*Task, error) {
	rows, err := h.getAllTasksStmt.Query()
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var (
		taskList    []*Task
		task_id     int64
		short_desc  string
		long_desc   string
		done        bool
		done_by_raw sql.NullString
		done_by     string
		chore_id    int64
		c_time      time.Time
	)
	for rows.Next() {
		err := rows.Scan(&task_id, &short_desc, &long_desc, &done, &done_by_raw, &c_time, &chore_id)
		if err != nil {
			return nil, err
		}

		if done_by_raw.Valid {
			done_by = done_by_raw.String
		} else {
			done_by = ""
		}

		t := &Task{task_id, short_desc, long_desc, done, done_by, c_time, chore_id}
		taskList = append(taskList, t)
	}
	if err = rows.Err(); err != nil {
		return nil, err
	}
	return taskList, nil
}

func modifyTaskStmt(db *sql.DB) (*sql.Stmt, error) {
	return db.Prepare("UPDATE tasks set done = ?, done_by = ? where task_id = ?")
}

func (h *handler) modifyTask(task *Task) error {
	_, err := h.modifyTaskStmt.Exec(
		task.Done,
		task.DoneBy,
	)
	return err
}
<commit_msg>whoops<commit_after>package tasks

// NOTE we do not log anything in this file! Leave it to the http handler to handle errors.

import (
	"database/sql"
	"time"
)

// TODO add done, done_by, c_time (maybe m_time?)
type Task struct {
	Key        int64     `json:"key"`
	ShortDesc  string    `json:"short_desc"`
	LongDesc   string    `json:"long_desc"`
	Done       bool      `json:"done"`
	DoneBy     string    `json:"done_by"`
	CreateTime time.Time `json:"c_time"`
	ChoreId    int64     `json:"chore_id"`
}

func setChoresNowStmt(db *sql.DB) (*sql.Stmt, error) {
	return db.Prepare("INSERT INTO tasks(chore_id, c_time) SELECT chore_id, NULL from chores where (morning = ? OR night = ?) AND ((dwm = 'd') OR (dwm = 'w' AND day = ?) OR (dwm = 'm' AND date = ?))")
}

func (h *handler) setChoresNow(t time.Time) error {
	var (
		isMorning int
		isNight   int
	)
	if t.Hour() >= 5 || t.Hour() < 17 {
		isMorning = 1
		isNight = -1
	} else {
		isMorning = -1
		isNight = 1
	}
	_, err := h.setChoresNowStmt.Exec(isMorning, isNight, t.Weekday(), t.Day())
	return err
}

func getAllTasksStmt(db *sql.DB) (*sql.Stmt, error) {
	return db.Prepare("SELECT task_id, chores.short_desc, chores.long_desc, done, done_by, tasks.c_time, tasks.chore_id from tasks left join chores on (tasks.chore_id = chores.chore_id)")
}

func (h *handler) getAllTasks() ([]*Task, error) {
	rows, err := h.getAllTasksStmt.Query()
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var (
		taskList    []*Task
		task_id     int64
		short_desc  string
		long_desc   string
		done        bool
		done_by_raw sql.NullString
		done_by     string
		chore_id    int64
		c_time      time.Time
	)
	for rows.Next() {
		err := rows.Scan(&task_id, &short_desc, &long_desc, &done, &done_by_raw, &c_time, &chore_id)
		if err != nil {
			return nil, err
		}

		if done_by_raw.Valid {
			done_by = done_by_raw.String
		} else {
			done_by = ""
		}

		t := &Task{task_id, short_desc, long_desc, done, done_by, c_time, chore_id}
		taskList = append(taskList, t)
	}
	if err = rows.Err(); err != nil {
		return nil, err
	}
	return taskList, nil
}

func modifyTaskStmt(db *sql.DB) (*sql.Stmt, error) {
	return db.Prepare("UPDATE tasks set done = ?, done_by = ? where task_id = ?")
}

func (h *handler) modifyTask(task *Task) error {
	_, err := h.modifyTaskStmt.Exec(
		task.Done,
		task.DoneBy,
	)
	return err
}
<|endoftext|>


<commit_before>package fdrpc

import (
	"bufio"
	"encoding/gob"
	"log"
	"net"
	"syscall"

	"github.com/flynn/rpcplus"
)

type FDWriter struct {
	conn    *net.UnixConn
	fds     []int
	fdCount int
}

func NewFDWriter(conn *net.UnixConn) *FDWriter {
	return &FDWriter{conn: conn}
}

func (w *FDWriter) Close() error {
	return w.conn.Close()
}

func (w *FDWriter) Read(b []byte) (int, error) {
	return w.conn.Read(b)
}

func (w *FDWriter) Write(b []byte) (int, error) {
	if len(w.fds) == 0 {
		return w.conn.Write(b)
	} else {
		rights := syscall.UnixRights(w.fds...)
		n, _, err := w.conn.WriteMsgUnix(b, rights, nil)
		w.fds = nil
		return n, err
	}
}

func (w *FDWriter) AddFD(fd int) int {
	w.fds = append(w.fds, fd)
	res := w.fdCount
	w.fdCount++
	return res
}

type gobServerCodec struct {
	fdWriter *FDWriter
	dec      *gob.Decoder
	enc      *gob.Encoder
	encBuf   *bufio.Writer
}

func (c *gobServerCodec) ReadRequestHeader(r *rpcplus.Request) error {
	return c.dec.Decode(r)
}

func (c *gobServerCodec) ReadRequestBody(body interface{}) error {
	return c.dec.Decode(body)
}

func (c *gobServerCodec) WriteResponse(r *rpcplus.Response, body interface{}, last bool) (err error) {
	switch f := body.(type) {
	case *FD:
		f.FD = c.fdWriter.AddFD(f.FD)
	case *[]FD:
		for i, fd := range *f {
			(*f)[i].FD = c.fdWriter.AddFD(fd.FD)
		}
	}

	if err = c.enc.Encode(r); err != nil {
		return
	}
	if err = c.enc.Encode(body); err != nil {
		return
	}
	return c.encBuf.Flush()
}

func (c *gobServerCodec) Close() error {
	return c.fdWriter.Close()
}

func ListenAndServe(path string) error {
	listener, err := net.ListenUnix("unix", &net.UnixAddr{Net: "unix", Name: path})
	if err != nil {
		return err
	}
	for {
		conn, err := listener.AcceptUnix()
		if err != nil {
			log.Printf("rpc socket accept error: %s", err)
		}
		go func() {
			defer conn.Close()
			ServeConn(conn)
		}()
	}
}

func ServeConn(conn *net.UnixConn) {
	fdWriter := NewFDWriter(conn)
	buf := bufio.NewWriter(fdWriter)
	srv := &gobServerCodec{fdWriter, gob.NewDecoder(fdWriter), gob.NewEncoder(buf), buf}
	rpcplus.ServeCodec(srv)
}
<commit_msg>fdrpc: Add ClosingFD type that closes after sending<commit_after>package fdrpc

import (
	"bufio"
	"encoding/gob"
	"log"
	"net"
	"syscall"

	"github.com/flynn/rpcplus"
)

type ClosingFD FD

type FDWriter struct {
	conn    *net.UnixConn
	fds     []int
	fdCount int
}

func NewFDWriter(conn *net.UnixConn) *FDWriter {
	return &FDWriter{conn: conn}
}

func (w *FDWriter) Close() error {
	return w.conn.Close()
}

func (w *FDWriter) Read(b []byte) (int, error) {
	return w.conn.Read(b)
}

func (w *FDWriter) Write(b []byte) (int, error) {
	if len(w.fds) == 0 {
		return w.conn.Write(b)
	} else {
		rights := syscall.UnixRights(w.fds...)
		n, _, err := w.conn.WriteMsgUnix(b, rights, nil)
		w.fds = nil
		return n, err
	}
}

func (w *FDWriter) AddFD(fd int) int {
	w.fds = append(w.fds, fd)
	res := w.fdCount
	w.fdCount++
	return res
}

type gobServerCodec struct {
	fdWriter *FDWriter
	dec      *gob.Decoder
	enc      *gob.Encoder
	encBuf   *bufio.Writer
}

func (c *gobServerCodec) ReadRequestHeader(r *rpcplus.Request) error {
	return c.dec.Decode(r)
}

func (c *gobServerCodec) ReadRequestBody(body interface{}) error {
	return c.dec.Decode(body)
}

func (c *gobServerCodec) WriteResponse(r *rpcplus.Response, body interface{}, last bool) (err error) {
	switch f := body.(type) {
	case *FD:
		f.FD = c.fdWriter.AddFD(f.FD)
	case *[]FD:
		for i, fd := range *f {
			(*f)[i].FD = c.fdWriter.AddFD(fd.FD)
		}
	case *ClosingFD:
		defer syscall.Close(f.FD)
		body = &FD{c.fdWriter.AddFD(f.FD)}
	}

	if err = c.enc.Encode(r); err != nil {
		return
	}
	if err = c.enc.Encode(body); err != nil {
		return
	}
	return c.encBuf.Flush()
}

func (c *gobServerCodec) Close() error {
	return c.fdWriter.Close()
}

func ListenAndServe(path string) error {
	listener, err := net.ListenUnix("unix", &net.UnixAddr{Net: "unix", Name: path})
	if err != nil {
		return err
	}
	for {
		conn, err := listener.AcceptUnix()
		if err != nil {
			log.Printf("rpc socket accept error: %s", err)
		}
		go func() {
			defer conn.Close()
			ServeConn(conn)
		}()
	}
}

func ServeConn(conn *net.UnixConn) {
	fdWriter := NewFDWriter(conn)
	buf := bufio.NewWriter(fdWriter)
	srv := &gobServerCodec{fdWriter, gob.NewDecoder(fdWriter), gob.NewEncoder(buf), buf}
	rpcplus.ServeCodec(srv)
}
<|endoftext|>


<commit_before>package features

import (
	"errors"
	"os"
	"sync"
	"unsafe"

	"github.com/cilium/ebpf"
	"github.com/cilium/ebpf/internal/sys"
	"github.com/cilium/ebpf/internal/unix"
)

func init() {
	mc.mapTypes = make(map[ebpf.MapType]error)
	mc.mapFlags = make(map[MapFlags]error)
}

var (
	mc mapCache
)

type mapCache struct {
	sync.Mutex
	mapTypes map[ebpf.MapType]error
	mapFlags map[MapFlags]error
}

func createMapTypeAttr(mt ebpf.MapType) *sys.MapCreateAttr {
	a := &sys.MapCreateAttr{
		MapType:    sys.MapType(mt),
		KeySize:    4,
		ValueSize:  4,
		MaxEntries: 1,
	}

	// switch on map types to generate correct MapCreateAttr
	switch mt {
	case ebpf.StackTrace:
		// valueSize needs to be sizeof(uint64)
		a.ValueSize = 8
	case ebpf.LPMTrie:
		// keySize and valueSize need to be sizeof(struct{u32 + u8}) + 1 + padding = 8
		// BPF_F_NO_PREALLOC needs to be set
		// checked at allocation time for lpm_trie maps
		a.KeySize = 8
		a.ValueSize = 8
		a.MapFlags = unix.BPF_F_NO_PREALLOC
	case ebpf.ArrayOfMaps, ebpf.HashOfMaps:
		// assign invalid innerMapFd to pass validation check
		// will return EBADF
		a.InnerMapFd = ^uint32(0)
	case ebpf.CGroupStorage, ebpf.PerCPUCGroupStorage:
		// keySize needs to be sizeof(struct{u32 + u64}) = 12 (+ padding = 16)
		// by using unsafe.Sizeof(int) we are making sure that this works on 32bit and 64bit archs
		// checked at allocation time
		var align int
		a.KeySize = uint32(8 + unsafe.Sizeof(align))
		a.MaxEntries = 0
	case ebpf.Queue, ebpf.Stack:
		// keySize needs to be 0, see alloc_check for queue and stack maps
		a.KeySize = 0
	case ebpf.RingBuf:
		// keySize and valueSize need to be 0
		// maxEntries needs to be power of 2 and PAGE_ALIGNED
		// checked at allocation time
		a.KeySize = 0
		a.ValueSize = 0
		a.MaxEntries = uint32(os.Getpagesize())
	case ebpf.SkStorage, ebpf.InodeStorage, ebpf.TaskStorage:
		// maxEntries needs to be 0
		// BPF_F_NO_PREALLOC needs to be set
		// btf* fields need to be set
		// see alloc_check for local_storage map types
		a.MaxEntries = 0
		a.MapFlags = unix.BPF_F_NO_PREALLOC
		a.BtfKeyTypeId = 1   // BTF_KIND_INT
		a.BtfValueTypeId = 3 // BTF_KIND_ARRAY
		a.BtfFd = ^uint32(0)
	case ebpf.StructOpsMap:
		// StructOps requires setting a vmlinux type id, but id 1 will always
		// resolve to some type of integer. This will cause ENOTSUPP.
		a.BtfVmlinuxValueTypeId = 1
	}

	return a
}

// HaveMapType probes the running kernel for the availability of the specified map type.
//
// See the package documentation for the meaning of the error return value.
func HaveMapType(mt ebpf.MapType) (err error) {
	defer func() {
		// This closure modifies a named return variable.
		err = wrapProbeErrors(err)
	}()

	if err := validateMaptype(mt); err != nil {
		return err
	}

	return haveMapType(mt)
}

func validateMaptype(mt ebpf.MapType) error {
	if mt > mt.Max() {
		return os.ErrInvalid
	}
	return nil
}

func haveMapType(mt ebpf.MapType) error {
	mc.Lock()
	defer mc.Unlock()
	err, ok := mc.mapTypes[mt]
	if ok {
		return err
	}

	fd, err := sys.MapCreate(createMapTypeAttr(mt))
	if err == nil {
		fd.Close()
	}

	switch {
	// For nested and storage map types we accept EBADF as indicator that these maps are supported
	case errors.Is(err, unix.EBADF):
		if isMapOfMaps(mt) || isStorageMap(mt) {
			err = nil
		}

	// ENOTSUPP means the map type is at least known to the kernel.
	case errors.Is(err, sys.ENOTSUPP):
		if mt == ebpf.StructOpsMap {
			err = nil
		}

	// EINVAL occurs when attempting to create a map with an unknown type.
	// E2BIG occurs when MapCreateAttr contains non-zero bytes past the end
	// of the struct known by the running kernel, meaning the kernel is too old
	// to support the given map type.
	case errors.Is(err, unix.EINVAL), errors.Is(err, unix.E2BIG):
		err = ebpf.ErrNotSupported
	}

	mc.mapTypes[mt] = err

	return err
}

func isMapOfMaps(mt ebpf.MapType) bool {
	switch mt {
	case ebpf.ArrayOfMaps, ebpf.HashOfMaps:
		return true
	}
	return false
}

func isStorageMap(mt ebpf.MapType) bool {
	switch mt {
	case ebpf.SkStorage, ebpf.InodeStorage, ebpf.TaskStorage:
		return true
	}
	return false
}

// MapFlags indicates the flags passed to the kernel during map creation
type MapFlags = sys.MapFlags

const (
	BPF_F_NO_PREALLOC MapFlags = unix.BPF_F_NO_PREALLOC
	BPF_F_RDONLY_PROG MapFlags = unix.BPF_F_RDONLY_PROG
	BPF_F_WRONLY_PROG MapFlags = unix.BPF_F_WRONLY_PROG
	BPF_F_MMAPABLE    MapFlags = unix.BPF_F_MMAPABLE
	BPF_F_INNER_MAP   MapFlags = unix.BPF_F_INNER_MAP
)

// HaveMapFlag probes the running kernel for the availability of the specified map flag.
//
// See the package documentation for the meaning of the error return value.
func HaveMapFlag(flag MapFlags) (err error) {
	defer func() {
		// This closure modifies a named return variable.
		err = wrapProbeErrors(err)
	}()

	return haveMapFlag(flag)
}

func haveMapFlag(flag MapFlags) error {
	mc.Lock()
	defer mc.Unlock()
	err, ok := mc.mapFlags[flag]
	if ok {
		return err
	}

	attr, err := createMapFlagTypeAttr(flag)
	if err != nil {
		return err
	}

	fd, err := sys.MapCreate(attr)
	if err == nil {
		fd.Close()
	}

	// EINVAL occurs when attempting to create a map with an unknown type or an unknown flag.
	if errors.Is(err, unix.EINVAL) {
		err = ebpf.ErrNotSupported
	}

	mc.mapFlags[flag] = err

	return err
}

func createMapFlagTypeAttr(flag MapFlags) (*sys.MapCreateAttr, error) {
	a := &sys.MapCreateAttr{
		KeySize:    4,
		ValueSize:  4,
		MaxEntries: 1,
		MapFlags:   flag,
	}

	// For now, we do not check if the map type is supported because we only support
	// probing for flags defined on arrays and hashs that are always supported.
	// In the future, if we allow probing on flags defined on newer types, checking for map type
	// support will be required.

	switch flag {
	case unix.BPF_F_MMAPABLE, unix.BPF_F_INNER_MAP, unix.BPF_F_RDONLY_PROG, unix.BPF_F_WRONLY_PROG:
		a.MapType = sys.MapType(ebpf.Array)
		return a, nil
	case unix.BPF_F_NO_PREALLOC:
		a.MapType = sys.MapType(ebpf.Hash)
		return a, nil
	}

	return nil, errors.New("probe not implemented")
}
<commit_msg>features: document MapFlags<commit_after>package features

import (
	"errors"
	"os"
	"sync"
	"unsafe"

	"github.com/cilium/ebpf"
	"github.com/cilium/ebpf/internal/sys"
	"github.com/cilium/ebpf/internal/unix"
)

func init() {
	mc.mapTypes = make(map[ebpf.MapType]error)
	mc.mapFlags = make(map[MapFlags]error)
}

var (
	mc mapCache
)

type mapCache struct {
	sync.Mutex
	mapTypes map[ebpf.MapType]error
	mapFlags map[MapFlags]error
}

func createMapTypeAttr(mt ebpf.MapType) *sys.MapCreateAttr {
	a := &sys.MapCreateAttr{
		MapType:    sys.MapType(mt),
		KeySize:    4,
		ValueSize:  4,
		MaxEntries: 1,
	}

	// switch on map types to generate correct MapCreateAttr
	switch mt {
	case ebpf.StackTrace:
		// valueSize needs to be sizeof(uint64)
		a.ValueSize = 8
	case ebpf.LPMTrie:
		// keySize and valueSize need to be sizeof(struct{u32 + u8}) + 1 + padding = 8
		// BPF_F_NO_PREALLOC needs to be set
		// checked at allocation time for lpm_trie maps
		a.KeySize = 8
		a.ValueSize = 8
		a.MapFlags = unix.BPF_F_NO_PREALLOC
	case ebpf.ArrayOfMaps, ebpf.HashOfMaps:
		// assign invalid innerMapFd to pass validation check
		// will return EBADF
		a.InnerMapFd = ^uint32(0)
	case ebpf.CGroupStorage, ebpf.PerCPUCGroupStorage:
		// keySize needs to be sizeof(struct{u32 + u64}) = 12 (+ padding = 16)
		// by using unsafe.Sizeof(int) we are making sure that this works on 32bit and 64bit archs
		// checked at allocation time
		var align int
		a.KeySize = uint32(8 + unsafe.Sizeof(align))
		a.MaxEntries = 0
	case ebpf.Queue, ebpf.Stack:
		// keySize needs to be 0, see alloc_check for queue and stack maps
		a.KeySize = 0
	case ebpf.RingBuf:
		// keySize and valueSize need to be 0
		// maxEntries needs to be power of 2 and PAGE_ALIGNED
		// checked at allocation time
		a.KeySize = 0
		a.ValueSize = 0
		a.MaxEntries = uint32(os.Getpagesize())
	case ebpf.SkStorage, ebpf.InodeStorage, ebpf.TaskStorage:
		// maxEntries needs to be 0
		// BPF_F_NO_PREALLOC needs to be set
		// btf* fields need to be set
		// see alloc_check for local_storage map types
		a.MaxEntries = 0
		a.MapFlags = unix.BPF_F_NO_PREALLOC
		a.BtfKeyTypeId = 1   // BTF_KIND_INT
		a.BtfValueTypeId = 3 // BTF_KIND_ARRAY
		a.BtfFd = ^uint32(0)
	case ebpf.StructOpsMap:
		// StructOps requires setting a vmlinux type id, but id 1 will always
		// resolve to some type of integer. This will cause ENOTSUPP.
		a.BtfVmlinuxValueTypeId = 1
	}

	return a
}

// HaveMapType probes the running kernel for the availability of the specified map type.
//
// See the package documentation for the meaning of the error return value.
func HaveMapType(mt ebpf.MapType) (err error) {
	defer func() {
		// This closure modifies a named return variable.
		err = wrapProbeErrors(err)
	}()

	if err := validateMaptype(mt); err != nil {
		return err
	}

	return haveMapType(mt)
}

func validateMaptype(mt ebpf.MapType) error {
	if mt > mt.Max() {
		return os.ErrInvalid
	}
	return nil
}

func haveMapType(mt ebpf.MapType) error {
	mc.Lock()
	defer mc.Unlock()
	err, ok := mc.mapTypes[mt]
	if ok {
		return err
	}

	fd, err := sys.MapCreate(createMapTypeAttr(mt))
	if err == nil {
		fd.Close()
	}

	switch {
	// For nested and storage map types we accept EBADF as indicator that these maps are supported
	case errors.Is(err, unix.EBADF):
		if isMapOfMaps(mt) || isStorageMap(mt) {
			err = nil
		}

	// ENOTSUPP means the map type is at least known to the kernel.
	case errors.Is(err, sys.ENOTSUPP):
		if mt == ebpf.StructOpsMap {
			err = nil
		}

	// EINVAL occurs when attempting to create a map with an unknown type.
	// E2BIG occurs when MapCreateAttr contains non-zero bytes past the end
	// of the struct known by the running kernel, meaning the kernel is too old
	// to support the given map type.
	case errors.Is(err, unix.EINVAL), errors.Is(err, unix.E2BIG):
		err = ebpf.ErrNotSupported
	}

	mc.mapTypes[mt] = err

	return err
}

func isMapOfMaps(mt ebpf.MapType) bool {
	switch mt {
	case ebpf.ArrayOfMaps, ebpf.HashOfMaps:
		return true
	}
	return false
}

func isStorageMap(mt ebpf.MapType) bool {
	switch mt {
	case ebpf.SkStorage, ebpf.InodeStorage, ebpf.TaskStorage:
		return true
	}
	return false
}

// MapFlags document which flags may be feature probed.
type MapFlags = sys.MapFlags

// Flags which may be feature probed.
const (
	BPF_F_NO_PREALLOC MapFlags = unix.BPF_F_NO_PREALLOC
	BPF_F_RDONLY_PROG MapFlags = unix.BPF_F_RDONLY_PROG
	BPF_F_WRONLY_PROG MapFlags = unix.BPF_F_WRONLY_PROG
	BPF_F_MMAPABLE    MapFlags = unix.BPF_F_MMAPABLE
	BPF_F_INNER_MAP   MapFlags = unix.BPF_F_INNER_MAP
)

// HaveMapFlag probes the running kernel for the availability of the specified map flag.
//
// Returns an error if flag is not one of the flags declared in this package.
// See the package documentation for the meaning of the error return value.
func HaveMapFlag(flag MapFlags) (err error) {
	defer func() {
		// This closure modifies a named return variable.
		err = wrapProbeErrors(err)
	}()

	return haveMapFlag(flag)
}

func haveMapFlag(flag MapFlags) error {
	mc.Lock()
	defer mc.Unlock()
	err, ok := mc.mapFlags[flag]
	if ok {
		return err
	}

	attr, err := createMapFlagTypeAttr(flag)
	if err != nil {
		return err
	}

	fd, err := sys.MapCreate(attr)
	if err == nil {
		fd.Close()
	}

	// EINVAL occurs when attempting to create a map with an unknown type or an unknown flag.
	if errors.Is(err, unix.EINVAL) {
		err = ebpf.ErrNotSupported
	}

	mc.mapFlags[flag] = err

	return err
}

func createMapFlagTypeAttr(flag MapFlags) (*sys.MapCreateAttr, error) {
	a := &sys.MapCreateAttr{
		KeySize:    4,
		ValueSize:  4,
		MaxEntries: 1,
		MapFlags:   flag,
	}

	// For now, we do not check if the map type is supported because we only support
	// probing for flags defined on arrays and hashs that are always supported.
	// In the future, if we allow probing on flags defined on newer types, checking for map type
	// support will be required.

	switch flag {
	case unix.BPF_F_MMAPABLE, unix.BPF_F_INNER_MAP, unix.BPF_F_RDONLY_PROG, unix.BPF_F_WRONLY_PROG:
		a.MapType = sys.MapType(ebpf.Array)
		return a, nil
	case unix.BPF_F_NO_PREALLOC:
		a.MapType = sys.MapType(ebpf.Hash)
		return a, nil
	}

	return nil, errors.New("probe not implemented")
}
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import (
	"encoding/json"
	"fmt"
)

//{"took":9,"_shards":{"total":3,"successful":3,"failed":0},"total":3,
//"matches":[{"_index":"test-geoevents-20150706-115234","_id":"nyc,West Village"},{"_index":"test-geoevents-20150706-115234","_id":"nyc,Manhattan"},{"_index":"test-geoevents-20150706-115234","_id":"nyc,Greenwich Village"}]}

type PercolatorResult struct {
	SearchResult
	Matches []PercolatorMatch `json:"matches"`
}

type PercolatorMatch struct {
	Index string `json:"_index"`
	Id    string `json:"_id"`
}

// RegisterPercolate allows the caller to register queries against an index,
// and then send percolate requests which include a doc, and getting back the
// queries that match on that doc out of the set of registered queries.  Think
// of it as the reverse operation of indexing and then searching. Instead of
// sending docs, indexing them, and then running queries. One sends queries,
// registers them, and then sends docs and finds out which queries match that
// doc.
// see http://www.elasticsearch.org/guide/reference/api/percolate.html
func (c *Conn) RegisterPercolate(index string, name string, args map[string]interface{}, query OneTermQuery) (BaseResponse, error) {
	var url string
	var retval BaseResponse
	url = fmt.Sprintf("/_percolator/%s/%s", index, name)
	body, err := c.DoCommand("PUT", url, args, query)
	if err != nil {
		return retval, err
	}
	if err == nil {
		// marshall into json
		jsonErr := json.Unmarshal(body, &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	return retval, err
}

func (c *Conn) Percolate(index string, _type string, name string, args map[string]interface{}, doc string) (PercolatorResult, error) {
	var url string
	var retval PercolatorResult
	url = fmt.Sprintf("/%s/%s/_percolate", index, _type)
	body, err := c.DoCommand("GET", url, args, doc)
	if err != nil {
		return retval, err
	}
	if err == nil {
		// marshall into json
		jsonErr := json.Unmarshal(body, &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	return retval, err
}
<commit_msg>Fix RegisterPercolate to actually work with ES 1.6+<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import (
	"encoding/json"
	"fmt"
)

type PercolatorResult struct {
	SearchResult
	Matches []PercolatorMatch `json:"matches"`
}

type PercolatorMatch struct {
	Index string `json:"_index"`
	Id    string `json:"_id"`
}

// See http://www.elasticsearch.org/guide/reference/api/percolate.html
func (c *Conn) RegisterPercolate(index string, id string, data interface{}) (BaseResponse, error) {
	var url string
	var retval BaseResponse
	url = fmt.Sprintf("/%s/.percolator/%s", index, id)
	body, err := c.DoCommand("PUT", url, nil, data)
	if err != nil {
		return retval, err
	}
	if err == nil {
		// marshall into json
		jsonErr := json.Unmarshal(body, &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	return retval, err
}

func (c *Conn) Percolate(index string, _type string, name string, args map[string]interface{}, doc string) (PercolatorResult, error) {
	var url string
	var retval PercolatorResult
	url = fmt.Sprintf("/%s/%s/_percolate", index, _type)
	body, err := c.DoCommand("GET", url, args, doc)
	if err != nil {
		return retval, err
	}
	if err == nil {
		// marshall into json
		jsonErr := json.Unmarshal(body, &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	return retval, err
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package health

import (
	"fmt"
	"net/http"
	"sync"
	"time"

	log "github.com/sirupsen/logrus"
)

// The HealthReport struct has slots for the levels of health that we monitor and aggregate.
type HealthReport struct {
	Live  bool
	Ready bool
}

type reporterState struct {
	// The health indicators that this reporter reports.
	reports HealthReport

	// Expiry time for this reporter's reports.  Zero means that reports never expire.
	timeout time.Duration

	// The most recent report.
	latest HealthReport

	// Time of that most recent report.
	timestamp time.Time
}

// A HealthAggregator receives health reports from individual reporters (which are typically
// components of a particular daemon or application) and aggregates them into an overall health
// summary.  For each monitored kind of health, all of the reporters that report that need to say
// that it is good; for example, to be 'ready' overall, all of the reporters that report readiness
// need to have recently said 'Ready: true'.
type HealthAggregator struct {
	// Mutex to protect concurrent access to this health aggregator.
	mutex *sync.Mutex

	// Map from reporter name to corresponding state.
	reporters map[string]*reporterState
}

// RegisterReporter registers a reporter with a HealthAggregator.  The aggregator uses NAME to
// identify the reporter.  REPORTS indicates the kinds of health that this reporter will report.
// TIMEOUT is the expiry time for this reporter's reports; the implication of which is that the
// reporter should normally refresh its reports well before this time has expired.
func (aggregator *HealthAggregator) RegisterReporter(name string, reports *HealthReport, timeout time.Duration) {
	aggregator.mutex.Lock()
	defer aggregator.mutex.Unlock()
	aggregator.reporters[name] = &reporterState{
		reports:   *reports,
		timeout:   timeout,
		latest:    HealthReport{Live: true},
		timestamp: time.Now(),
	}
	return
}

// Report reports current health from a reporter to a HealthAggregator.  NAME is the reporter's name
// and REPORTS conveys the current status, for each kind of health that the reporter said it was
// going to report when it called RegisterReporter.
func (aggregator *HealthAggregator) Report(name string, report *HealthReport) {
	aggregator.mutex.Lock()
	defer aggregator.mutex.Unlock()
	reporter := aggregator.reporters[name]
	reporter.latest = *report
	reporter.timestamp = time.Now()
	return
}

func NewHealthAggregator() *HealthAggregator {
	return &HealthAggregator{mutex: &sync.Mutex{}, reporters: map[string]*reporterState{}}
}

// Summary calculates the current overall health for a HealthAggregator.
func (aggregator *HealthAggregator) Summary() *HealthReport {
	aggregator.mutex.Lock()
	defer aggregator.mutex.Unlock()

	// In the absence of any reporters, default to indicating that we are both live and ready.
	summary := &HealthReport{Live: true, Ready: true}

	// Now for each reporter...
	for name, reporter := range aggregator.reporters {
		log.WithFields(log.Fields{
			"name":           name,
			"reporter-state": reporter,
		}).Debug("Detailed health state")

		// Reset Live to false if that reporter is registered to report liveness and hasn't
		// recently said that it is live.
		if summary.Live && reporter.reports.Live && (!reporter.latest.Live ||
			(reporter.timeout != 0 && time.Since(reporter.timestamp) > reporter.timeout)) {
			summary.Live = false
		}

		// Reset Ready to false if that reporter is registered to report readiness and
		// hasn't recently said that it is ready.
		if summary.Ready && reporter.reports.Ready && (!reporter.latest.Ready ||
			(reporter.timeout != 0 && time.Since(reporter.timestamp) > reporter.timeout)) {
			summary.Ready = false
		}
	}

	log.WithField("summary", summary).Info("Overall health")
	return summary
}

const (
	// The HTTP status that we use for 'ready' or 'live'.  204 means "No Content: The server
	// successfully processed the request and is not returning any content."  (Kubernetes
	// interpets any 200<=status<400 as 'good'.)
	StatusGood = 204

	// The HTTP status that we use for 'not ready' or 'not live'.  503 means "Service
	// Unavailable: The server is currently unavailable (because it is overloaded or down for
	// maintenance). Generally, this is a temporary state."  (Kubernetes interpets any
	// status>=400 as 'bad'.)
	StatusBad = 503
)

// ServeHTTP publishes the current overall liveness and readiness at http://*:PORT/liveness and
// http://*:PORT/readiness respectively.  A GET request on those URLs returns StatusGood or
// StatusBad, according to the current overall liveness or readiness.  These endpoints are designed
// for use by Kubernetes liveness and readiness probes.
func (aggregator *HealthAggregator) ServeHTTP(port int) {

	log.WithField("port", port).Info("Starting health endpoints")
	http.HandleFunc("/readiness", func(rsp http.ResponseWriter, req *http.Request) {
		log.Debug("GET /readiness")
		status := StatusBad
		if aggregator.Summary().Ready {
			log.Debug("Felix is ready")
			status = StatusGood
		}
		rsp.WriteHeader(status)
	})
	http.HandleFunc("/liveness", func(rsp http.ResponseWriter, req *http.Request) {
		log.Debug("GET /liveness")
		status := StatusBad
		if aggregator.Summary().Live {
			log.Debug("Felix is live")
			status = StatusGood
		}
		rsp.WriteHeader(status)
	})
	for {
		err := http.ListenAndServe(fmt.Sprintf(":%v", port), nil)
		log.WithError(err).Error(
			"Readiness endpoint failed, trying to restart it...")
		time.Sleep(1 * time.Second)
	}
}
<commit_msg>Support stopping and starting health HTTP server<commit_after>// Copyright (c) 2017 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package health

import (
	"fmt"
	"net/http"
	"sync"
	"time"

	log "github.com/sirupsen/logrus"
)

// The HealthReport struct has slots for the levels of health that we monitor and aggregate.
type HealthReport struct {
	Live  bool
	Ready bool
}

type reporterState struct {
	// The health indicators that this reporter reports.
	reports HealthReport

	// Expiry time for this reporter's reports.  Zero means that reports never expire.
	timeout time.Duration

	// The most recent report.
	latest HealthReport

	// Time of that most recent report.
	timestamp time.Time
}

// A HealthAggregator receives health reports from individual reporters (which are typically
// components of a particular daemon or application) and aggregates them into an overall health
// summary.  For each monitored kind of health, all of the reporters that report that need to say
// that it is good; for example, to be 'ready' overall, all of the reporters that report readiness
// need to have recently said 'Ready: true'.
type HealthAggregator struct {
	// Mutex to protect concurrent access to this health aggregator.
	mutex *sync.Mutex

	// Map from reporter name to corresponding state.
	reporters map[string]*reporterState

	// HTTP server mux.  This is where we register handlers for particular URLs.
	httpServeMux *http.ServeMux

	// HTTP server.  Non-nil when there should be a server running.
	httpServer *http.Server
}

// RegisterReporter registers a reporter with a HealthAggregator.  The aggregator uses NAME to
// identify the reporter.  REPORTS indicates the kinds of health that this reporter will report.
// TIMEOUT is the expiry time for this reporter's reports; the implication of which is that the
// reporter should normally refresh its reports well before this time has expired.
func (aggregator *HealthAggregator) RegisterReporter(name string, reports *HealthReport, timeout time.Duration) {
	aggregator.mutex.Lock()
	defer aggregator.mutex.Unlock()
	aggregator.reporters[name] = &reporterState{
		reports:   *reports,
		timeout:   timeout,
		latest:    HealthReport{Live: true},
		timestamp: time.Now(),
	}
	return
}

// Report reports current health from a reporter to a HealthAggregator.  NAME is the reporter's name
// and REPORTS conveys the current status, for each kind of health that the reporter said it was
// going to report when it called RegisterReporter.
func (aggregator *HealthAggregator) Report(name string, report *HealthReport) {
	aggregator.mutex.Lock()
	defer aggregator.mutex.Unlock()
	reporter := aggregator.reporters[name]
	reporter.latest = *report
	reporter.timestamp = time.Now()
	return
}

func NewHealthAggregator() *HealthAggregator {
	aggregator := &HealthAggregator{
		mutex:        &sync.Mutex{},
		reporters:    map[string]*reporterState{},
		httpServeMux: http.NewServeMux(),
	}
	aggregator.httpServeMux.HandleFunc("/readiness", func(rsp http.ResponseWriter, req *http.Request) {
		log.Debug("GET /readiness")
		status := StatusBad
		if aggregator.Summary().Ready {
			log.Debug("Felix is ready")
			status = StatusGood
		}
		rsp.WriteHeader(status)
	})
	aggregator.httpServeMux.HandleFunc("/liveness", func(rsp http.ResponseWriter, req *http.Request) {
		log.Debug("GET /liveness")
		status := StatusBad
		if aggregator.Summary().Live {
			log.Debug("Felix is live")
			status = StatusGood
		}
		rsp.WriteHeader(status)
	})
	return aggregator
}

// Summary calculates the current overall health for a HealthAggregator.
func (aggregator *HealthAggregator) Summary() *HealthReport {
	aggregator.mutex.Lock()
	defer aggregator.mutex.Unlock()

	// In the absence of any reporters, default to indicating that we are both live and ready.
	summary := &HealthReport{Live: true, Ready: true}

	// Now for each reporter...
	for name, reporter := range aggregator.reporters {
		log.WithFields(log.Fields{
			"name":           name,
			"reporter-state": reporter,
		}).Debug("Detailed health state")

		// Reset Live to false if that reporter is registered to report liveness and hasn't
		// recently said that it is live.
		if summary.Live && reporter.reports.Live && (!reporter.latest.Live ||
			(reporter.timeout != 0 && time.Since(reporter.timestamp) > reporter.timeout)) {
			summary.Live = false
		}

		// Reset Ready to false if that reporter is registered to report readiness and
		// hasn't recently said that it is ready.
		if summary.Ready && reporter.reports.Ready && (!reporter.latest.Ready ||
			(reporter.timeout != 0 && time.Since(reporter.timestamp) > reporter.timeout)) {
			summary.Ready = false
		}
	}

	log.WithField("summary", summary).Info("Overall health")
	return summary
}

const (
	// The HTTP status that we use for 'ready' or 'live'.  204 means "No Content: The server
	// successfully processed the request and is not returning any content."  (Kubernetes
	// interpets any 200<=status<400 as 'good'.)
	StatusGood = 204

	// The HTTP status that we use for 'not ready' or 'not live'.  503 means "Service
	// Unavailable: The server is currently unavailable (because it is overloaded or down for
	// maintenance). Generally, this is a temporary state."  (Kubernetes interpets any
	// status>=400 as 'bad'.)
	StatusBad = 503
)

// ServeHTTP publishes the current overall liveness and readiness at http://*:PORT/liveness and
// http://*:PORT/readiness respectively.  A GET request on those URLs returns StatusGood or
// StatusBad, according to the current overall liveness or readiness.  These endpoints are designed
// for use by Kubernetes liveness and readiness probes.
func (aggregator *HealthAggregator) ServeHTTP(enabled bool, port int) {
	aggregator.mutex.Lock()
	defer aggregator.mutex.Unlock()
	if enabled {
		if aggregator.httpServer != nil {
			log.WithField("port", port).Info("Health enabled.  Server is already running.")
			return
		}
		log.WithField("port", port).Info("Health enabled.  Starting server.")
		aggregator.httpServer = &http.Server{
			Addr:    fmt.Sprintf(":%v", port),
			Handler: aggregator.httpServeMux,
		}
		go func() {
			for {
				server := aggregator.getHTTPServer()
				if server == nil {
					// HTTP serving is now disabled.
					break
				}
				err := server.ListenAndServe()
				log.WithError(err).Error(
					"Health endpoint failed, trying to restart it...")
				time.Sleep(1 * time.Second)
			}
		}()
	} else {
		if aggregator.httpServer != nil {
			log.Info("Health disabled.  Stopping server.")
			aggregator.httpServer.Close()
			aggregator.httpServer = nil
		}
	}
}

func (aggregator *HealthAggregator) getHTTPServer() *http.Server {
	aggregator.mutex.Lock()
	defer aggregator.mutex.Unlock()
	return aggregator.httpServer
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("E2E Test", func() {
	It("Schedule QueueJob with SchedulerName", func() {
		context := initTestContext()
		defer cleanupTestContext(context)
		rep := clusterSize(context, oneCPU)
		queueJob := createQueueJobWithScheduler(context, "kar-scheduler", "qj-1", 2, rep, "busybox", oneCPU)
		err := waitJobReady(context, queueJob.Name)
		Expect(err).NotTo(HaveOccurred())
	})

	It("Schedule QueueJob", func() {
		context := initTestContext()
		defer cleanupTestContext(context)
		rep := clusterSize(context, oneCPU)
		queueJob := createQueueJob(context, "qj-1", 2, rep, "busybox", oneCPU)
		err := waitJobReady(context, queueJob.Name)
		Expect(err).NotTo(HaveOccurred())
	})

	It("Gang scheduling", func() {
		context := initTestContext()
		defer cleanupTestContext(context)
		rep := clusterSize(context, oneCPU)/2 + 1

		replicaset := createReplicaSet(context, "rs-1", rep, "nginx", oneCPU)
		err := waitReplicaSetReady(context, replicaset.Name)
		Expect(err).NotTo(HaveOccurred())

		queueJob := createQueueJob(context, "gang-qj", rep, rep, "busybox", oneCPU)
		err = waitJobNotReady(context, queueJob.Name)
		Expect(err).NotTo(HaveOccurred())

		err = deleteReplicaSet(context, replicaset.Name)
		Expect(err).NotTo(HaveOccurred())

		err = waitJobReady(context, queueJob.Name)
		Expect(err).NotTo(HaveOccurred())
	})

	It("Preemption", func() {
		context := initTestContext()
		defer cleanupTestContext(context)

		slot := oneCPU
		rep := clusterSize(context, slot)

		qj1 := createQueueJob(context, "preemptee-qj", 1, rep, "nginx", slot)
		err := waitTasksReady(context, qj1.Name, int(rep))
		Expect(err).NotTo(HaveOccurred())

		qj2 := createQueueJob(context, "preemptor-qj", 1, rep, "nginx", slot)
		err = waitTasksReady(context, qj2.Name, int(rep)/2)
		Expect(err).NotTo(HaveOccurred())

		err = waitTasksReady(context, qj1.Name, int(rep)/2)
		Expect(err).NotTo(HaveOccurred())
	})

	It("TaskPriority", func() {
		context := initTestContext()
		defer cleanupTestContext(context)

		slot := oneCPU
		rep := clusterSize(context, slot)

		replicaset := createReplicaSet(context, "rs-1", rep/2, "nginx", slot)
		err := waitReplicaSetReady(context, replicaset.Name)
		Expect(err).NotTo(HaveOccurred())

		ts := []taskSpec{
			{
				name: "worker",
				img:  "nginx",
				pri:  workerPriority,
				rep:  rep,
				req:  slot,
			},
			{
				name: "master",
				img:  "nginx",
				pri:  masterPriority,
				rep:  1,
				req:  slot,
			},
		}

		qj := createQueueJobEx(context, "multi-pod-qj", rep/2, ts)

		expectedTasks := map[string]int32{"master": 1, "worker": rep/2 - 1}
		err = waitTasksReadyEx(context, qj.Name, expectedTasks)
		Expect(err).NotTo(HaveOccurred())
	})
})
<commit_msg>Skip TaskPriority e2e test.<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("E2E Test", func() {
	It("Schedule QueueJob with SchedulerName", func() {
		context := initTestContext()
		defer cleanupTestContext(context)
		rep := clusterSize(context, oneCPU)
		queueJob := createQueueJobWithScheduler(context, "kar-scheduler", "qj-1", 2, rep, "busybox", oneCPU)
		err := waitJobReady(context, queueJob.Name)
		Expect(err).NotTo(HaveOccurred())
	})

	It("Schedule QueueJob", func() {
		context := initTestContext()
		defer cleanupTestContext(context)
		rep := clusterSize(context, oneCPU)
		queueJob := createQueueJob(context, "qj-1", 2, rep, "busybox", oneCPU)
		err := waitJobReady(context, queueJob.Name)
		Expect(err).NotTo(HaveOccurred())
	})

	It("Gang scheduling", func() {
		context := initTestContext()
		defer cleanupTestContext(context)
		rep := clusterSize(context, oneCPU)/2 + 1

		replicaset := createReplicaSet(context, "rs-1", rep, "nginx", oneCPU)
		err := waitReplicaSetReady(context, replicaset.Name)
		Expect(err).NotTo(HaveOccurred())

		queueJob := createQueueJob(context, "gang-qj", rep, rep, "busybox", oneCPU)
		err = waitJobNotReady(context, queueJob.Name)
		Expect(err).NotTo(HaveOccurred())

		err = deleteReplicaSet(context, replicaset.Name)
		Expect(err).NotTo(HaveOccurred())

		err = waitJobReady(context, queueJob.Name)
		Expect(err).NotTo(HaveOccurred())
	})

	It("Preemption", func() {
		context := initTestContext()
		defer cleanupTestContext(context)

		slot := oneCPU
		rep := clusterSize(context, slot)

		qj1 := createQueueJob(context, "preemptee-qj", 1, rep, "nginx", slot)
		err := waitTasksReady(context, qj1.Name, int(rep))
		Expect(err).NotTo(HaveOccurred())

		qj2 := createQueueJob(context, "preemptor-qj", 1, rep, "nginx", slot)
		err = waitTasksReady(context, qj2.Name, int(rep)/2)
		Expect(err).NotTo(HaveOccurred())

		err = waitTasksReady(context, qj1.Name, int(rep)/2)
		Expect(err).NotTo(HaveOccurred())
	})

	It("TaskPriority", func() {
		Skip("Skip TaskPriority test because of dind setting.")
		context := initTestContext()
		defer cleanupTestContext(context)

		slot := oneCPU
		rep := clusterSize(context, slot)

		replicaset := createReplicaSet(context, "rs-1", rep/2, "nginx", slot)
		err := waitReplicaSetReady(context, replicaset.Name)
		Expect(err).NotTo(HaveOccurred())

		ts := []taskSpec{
			{
				name: "worker",
				img:  "nginx",
				pri:  workerPriority,
				rep:  rep,
				req:  slot,
			},
			{
				name: "master",
				img:  "nginx",
				pri:  masterPriority,
				rep:  1,
				req:  slot,
			},
		}

		qj := createQueueJobEx(context, "multi-pod-qj", rep/2, ts)

		expectedTasks := map[string]int32{"master": 1, "worker": rep/2 - 1}
		err = waitTasksReadyEx(context, qj.Name, expectedTasks)
		Expect(err).NotTo(HaveOccurred())
	})
})
<|endoftext|>


<commit_before>// $G $F.go && $L $F.$A && ./$A.out

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

type T struct {
	i int
}

type IN interface{}

func main() {
	var i *int
	var f *float32
	var s *string
	var m map[float32]*int
	var c chan int
	var t *T
	var in IN
	var ta []IN

	i = nil
	f = nil
	s = nil
	m = nil
	c = nil
	t = nil
	i = nil
	ta = make([]IN, 1)
	ta[0] = nil

	_, _, _, _, _, _, _, _ = i, f, s, m, c, t, in, ta
}
<commit_msg>test: new nil semantics<commit_after>// $G $F.go && $L $F.$A && ./$A.out

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"time"
)

type T struct {
	i int
}

type IN interface{}

func main() {
	var i *int
	var f *float32
	var s *string
	var m map[float32]*int
	var c chan int
	var t *T
	var in IN
	var ta []IN

	i = nil
	f = nil
	s = nil
	m = nil
	c = nil
	t = nil
	i = nil
	ta = make([]IN, 1)
	ta[0] = nil

	_, _, _, _, _, _, _, _ = i, f, s, m, c, t, in, ta

	arraytest()
	chantest()
	maptest()
	slicetest()
}

func shouldPanic(f func()) {
	defer func() {
		if recover() == nil {
			panic("not panicking")
		}
	}()
	f()
}

func shouldBlock(f func()) {
	go func() {
		f()
		panic("did not block")
	}()
	time.Sleep(1e7)
}

// nil array pointer

func arraytest() {
	var p *[10]int

	// Looping over indices is fine.
	s := 0
	for i := range p {
		s += i
	}
	if s != 45 {
		panic(s)
	}

	s = 0
	for i := 0; i < len(p); i++ {
		s += i
	}
	if s != 45 {
		panic(s)
	}

	// Looping over values is not.
	shouldPanic(func() {
		for i, v := range p {
			s += i + v
		}
	})

	shouldPanic(func() {
		for i := 0; i < len(p); i++ {
			s += p[i]
		}
	})
}

// nil channel
// select tests already handle select on nil channel

func chantest() {
	var ch chan int

	// nil channel is never ready
	shouldBlock(func() {
		ch <- 1
	})
	shouldBlock(func() {
		<-ch
	})
	shouldBlock(func() {
		x, ok := <-ch
		println(x, ok)
	})

	if len(ch) != 0 {
		panic(len(ch))
	}
	if cap(ch) != 0 {
		panic(cap(ch))
	}
}

// nil map

func maptest() {
	var m map[int]int

	// nil map appears empty
	if len(m) != 0 {
		panic(len(m))
	}
	if m[1] != 0 {
		panic(m[1])
	}
	if x, ok := m[1]; x != 0 || ok {
		panic(fmt.Sprint(x, ok))
	}

	for k, v := range m {
		panic(k)
		panic(v)
	}

	// but cannot be written to
	shouldPanic(func() {
		m[2] = 3
	})
	shouldPanic(func() {
		m[2] = 0, false
	})
}

// nil slice

func slicetest() {
	var x []int

	// nil slice is just a 0-element slice.
	if len(x) != 0 {
		panic(len(x))
	}
	if cap(x) != 0 {
		panic(cap(x))
	}

	// no 0-element slices can be read from or written to
	var s int
	shouldPanic(func() {
		s += x[1]
	})
	shouldPanic(func() {
		x[2] = s
	})
}
<|endoftext|>