<commit_before>// Package task provides the tracking of state for a single task pushed by the
// external task queue.
//
// The Task type ...
// TODO(dev) Improve comments and header before merging to dev.
package task

import (
	"archive/tar"
	"compress/gzip"
	"github.com/m-lab/etl/parser"
	"io"
	"io/ioutil"
	"strings"
)

type Task struct {
	table string        // The table to insert rows into, INCLUDING the partition!
	rdr   *tar.Reader   // Tar reader from which to read tests.
	prsr  parser.Parser // Parser to parse the tests.
}

// NewTask constructs a task, injecting the tar reader and the parser.
func NewTask(rdr *tar.Reader, prsr parser.Parser, table string) *Task {
	t := new(Task)
	t.table = table
	t.rdr = rdr
	t.prsr = prsr
	return t
}

// Next reads the next test object from the tar file.
// Returns io.EOF when there are no more tests.
func (tt *Task) Next() (string, []byte, error) {
	h, err := tt.rdr.Next()
	if err != nil {
		return "", nil, err
	}
	if h.Typeflag != tar.TypeReg {
		return h.Name, nil, nil
	}
	var data []byte
	if strings.HasSuffix(strings.ToLower(h.Name), ".gz") {
		// TODO add unit test
		zipReader, err := gzip.NewReader(tt.rdr)
		if err != nil {
			return h.Name, nil, err
		}
		defer zipReader.Close()
		data, err = ioutil.ReadAll(zipReader)
	} else {
		data, err = ioutil.ReadAll(tt.rdr)
	}
	if err != nil {
		return h.Name, nil, err
	}
	return h.Name, data, nil
}

// ProcessAllTests loops through all the tests in a tar file, calls the
// injected parser to parse them, and inserts them into bigquery (not yet implemented).
func (tt *Task) ProcessAllTests() {
	// Read each file from the tar
	for fn, data, err := tt.Next(); err != io.EOF; fn, data, err = tt.Next() {
		if err != nil {
			if err == io.EOF {
				return
			}
			// TODO(dev) add error handling
			continue
		}
		if data == nil {
			// If verbose, log the filename that is skipped.
			continue
		}

		// TODO update table name
		_, err := tt.prsr.HandleTest(fn, tt.table, data)
		// TODO handle insertion into BQ.
		if err != nil {
			continue
		}
	}
	return
}
<commit_msg>Embed tar.Reader and parser.Parser<commit_after>// Package task provides the tracking of state for a single task pushed by the
// external task queue.
//
// The Task type ...
// TODO(dev) Improve comments and header before merging to dev.
package task

import (
	"archive/tar"
	"compress/gzip"
	"github.com/m-lab/etl/parser"
	"io"
	"io/ioutil"
	"strings"
)

type Task struct {
	table         string // The table to insert rows into, INCLUDING the partition!
	*tar.Reader          // Tar reader from which to read tests.
	parser.Parser        // Parser to parse the tests.
}

// NewTask constructs a task, injecting the tar reader and the parser.
func NewTask(rdr *tar.Reader, prsr parser.Parser, table string) *Task {
	t := new(Task)
	t.table = table
	t.Reader = rdr
	t.Parser = prsr
	return t
}

// Next reads the next test object from the tar file.
// Returns io.EOF when there are no more tests.
func (tt *Task) Next() (string, []byte, error) {
	h, err := tt.Reader.Next()
	if err != nil {
		return "", nil, err
	}
	if h.Typeflag != tar.TypeReg {
		return h.Name, nil, nil
	}
	var data []byte
	if strings.HasSuffix(strings.ToLower(h.Name), ".gz") {
		// TODO add unit test
		zipReader, err := gzip.NewReader(tt.Reader)
		if err != nil {
			return h.Name, nil, err
		}
		defer zipReader.Close()
		data, err = ioutil.ReadAll(zipReader)
	} else {
		data, err = ioutil.ReadAll(tt.Reader)
	}
	if err != nil {
		return h.Name, nil, err
	}
	return h.Name, data, nil
}

// ProcessAllTests loops through all the tests in a tar file, calls the
// injected parser to parse them, and inserts them into bigquery (not yet implemented).
func (tt *Task) ProcessAllTests() {
	// Read each file from the tar
	for fn, data, err := tt.Next(); err != io.EOF; fn, data, err = tt.Next() {
		if err != nil {
			if err == io.EOF {
				return
			}
			// TODO(dev) add error handling
			continue
		}
		if data == nil {
			// If verbose, log the filename that is skipped.
			continue
		}

		// TODO update table name
		_, err := tt.Parser.HandleTest(fn, tt.table, data)
		// TODO handle insertion into BQ.
		if err != nil {
			continue
		}
	}
	return
}
<|endoftext|>


<commit_before>package task

import "github.com/pborman/uuid"

// Task states
const (
	// Unknown state of the task
	// This is the default state of a task
	// when new task is initially created
	TaskStateUnknown = "unknown"

	// Task has been received by the
	// minion and is queued for execution
	TaskStateQueued = "queued"

	// Task is being processed
	TaskStateProcessing = "processing"

	// Task has been processed by the
	// minion and was flagged as successful
	TaskStateSuccess = "success"

	// Task has been processed by the
	// minion and was flagged as failed
	TaskStateFailed = "failed"

	// Task has been skipped
	TaskStateSkipped = "skipped"
)

// Task type represents a task that is processed by minions
type Task struct {
	// Environment to use for this task
	Environment string `json:"environment"`

	// Command to be processed
	Command string `json:"command"`

	// Time when the command was sent for processing
	TimeReceived int64 `json:"timeReceived"`

	// Time when the command was processed
	TimeProcessed int64 `json:"timeProcessed"`

	// Task unique id
	ID uuid.UUID `json:"id"`

	// Result of task after processing
	Result string `json:"result"`

	// Task state
	State string `json:"state"`
}

// New creates a new task
func New(command, environment string) *Task {
	t := &Task{
		Command:     command,
		Environment: environment,
		ID:          uuid.NewRandom(),
		State:       TaskStateUnknown,
	}

	return t
}
<commit_msg>task: add a dry-run field to the task.Task type<commit_after>package task

import "github.com/pborman/uuid"

// Task states
const (
	// Unknown state of the task
	// This is the default state of a task
	// when new task is initially created
	TaskStateUnknown = "unknown"

	// Task has been received by the
	// minion and is queued for execution
	TaskStateQueued = "queued"

	// Task is being processed
	TaskStateProcessing = "processing"

	// Task has been processed by the
	// minion and was flagged as successful
	TaskStateSuccess = "success"

	// Task has been processed by the
	// minion and was flagged as failed
	TaskStateFailed = "failed"

	// Task has been skipped
	TaskStateSkipped = "skipped"
)

// Task type represents a task that is processed by minions
type Task struct {
	// Do not take any actions, just report what would be done
	DryRun bool `json:"dryRun"`

	// Environment to use for this task
	Environment string `json:"environment"`

	// Command to be processed
	Command string `json:"command"`

	// Time when the command was sent for processing
	TimeReceived int64 `json:"timeReceived"`

	// Time when the command was processed
	TimeProcessed int64 `json:"timeProcessed"`

	// Task unique id
	ID uuid.UUID `json:"id"`

	// Result of task after processing
	Result string `json:"result"`

	// Task state
	State string `json:"state"`
}

// New creates a new task
func New(command, environment string) *Task {
	t := &Task{
		Command:     command,
		Environment: environment,
		ID:          uuid.NewRandom(),
		State:       TaskStateUnknown,
	}

	return t
}
<|endoftext|>


<commit_before>package bulk_test

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/kusubooru/local-tagger/bulk"
)

var errInvalidFormat = errors.New("invalid csv file format")
var loadCSVTests = []struct {
	in   string
	out  []bulk.Image
	oerr error
}{
	{"", []bulk.Image{}, nil},
	{",,,,", []bulk.Image{}, nil},
	{",,,,,,,,,,,,,,,,,,,", []bulk.Image(nil), errInvalidFormat},
	{"invalid format", []bulk.Image(nil), errInvalidFormat},
	{",,,,\n,,,,\n,,,,", []bulk.Image{}, nil},
	{
		"/server/path/img1,tag1 tag2,source,s,",
		[]bulk.Image{
			{Name: "img1", Tags: []string{"tag1", "tag2"}, Source: "source", Rating: "s"},
		},
		nil,
	},
	{
		"/server/path/img1,tag1 tag2,source1,s,\n" +
			",,,,\n" +
			"/server/path/img2,tag1 tag2,source2,q,\n",
		[]bulk.Image{
			{Name: "img1", Tags: []string{"tag1", "tag2"}, Source: "source1", Rating: "s"},
			{Name: "img2", Tags: []string{"tag1", "tag2"}, Source: "source2", Rating: "q"},
		},
		nil,
	},
}

func TestLoadCSV(t *testing.T) {
	for _, tt := range loadCSVTests {
		got, err := bulk.LoadCSV(strings.NewReader(tt.in))
		if want := tt.oerr; !reflect.DeepEqual(err, want) {
			t.Errorf("LoadCSV(%q) returned err %q, want %q", tt.in, err, want)
		}
		if want := tt.out; !reflect.DeepEqual(got, want) {
			t.Errorf("LoadCSV(%q) => %q, want %q", tt.in, got, want)
		}
	}
}

func TestLoadCSV_readFail(t *testing.T) {
	r := strings.NewReader("")
	in := ErrReader(r, fmt.Errorf("read fail"))
	got, err := bulk.LoadCSV(in)
	if err == nil {
		t.Errorf("LoadCSV with read failure must return err but returned %q, %q", got, err)
	}
}

var currentPrefixTests = []struct {
	indir  string
	infile string
	out    string
	oerr   error
}{
	{
		filepath.Join("/", "local", "path", "dir"),
		",,,,",
		filepath.Join("/"),
		nil,
	},
	{
		filepath.Join("/", "local", "path", "dir"),
		filepath.Join("/", "server", "path", "dir") + ",,,,",
		filepath.Join("/", "server", "path"),
		nil,
	},
	{
		filepath.Join("/", "local", "path", "dir"),
		"",
		"",
		nil,
	},
}

func TestCurrentPrefix(t *testing.T) {
	for _, tt := range currentPrefixTests {
		got, err := bulk.CurrentPrefix(tt.indir, strings.NewReader(tt.infile))
		if want := tt.oerr; !reflect.DeepEqual(err, want) {
			t.Errorf("CurrentPrefix(%q, %q) returned err %q, want %q", tt.indir, tt.infile, err, want)
		}
		if want := tt.out; !reflect.DeepEqual(got, want) {
			t.Errorf("CurrentPrefix(%q, %q) => %q, want %q", tt.indir, tt.infile, got, want)
		}
	}
}

func TestCurrentPrefix_readFail(t *testing.T) {
	r := strings.NewReader("")
	in := ErrReader(r, fmt.Errorf("read fail"))
	got, err := bulk.CurrentPrefix("", in)
	if err == nil {
		t.Errorf("CurrentPrefix with read failure must return err but returned %q, %q", got, err)
	}
}

var findByIDTests = []struct {
	images []bulk.Image
	id     int
	out    *bulk.Image
}{
	{nil, 0, nil},
	{[]bulk.Image{}, 0, nil},
	{[]bulk.Image{{ID: 1}}, 0, nil},
	{[]bulk.Image{{ID: 1}}, 1, &bulk.Image{ID: 1}},
	{[]bulk.Image{{ID: 2}, {ID: 1}}, 1, &bulk.Image{ID: 1}},
	{[]bulk.Image{{ID: 2}, {ID: 3}, {ID: 1}}, 3, &bulk.Image{ID: 3}},
	{[]bulk.Image{{ID: 2}, {ID: 3}, {ID: 1}}, 5, nil},
}

func TestFindByID(t *testing.T) {
	for _, tt := range findByIDTests {
		got := bulk.FindByID(tt.images, tt.id)
		if want := tt.out; !reflect.DeepEqual(got, want) {
			t.Errorf("FindByID(%q, %q) => %q, want %q", tt.images, tt.id, got, want)
		}
	}
}

var combineTests = []struct {
	images   []bulk.Image
	metadata []bulk.Image
	out      []bulk.Image
}{
	{
		[]bulk.Image{{ID: 0}},
		[]bulk.Image{{ID: 0, Source: "source"}},
		[]bulk.Image{{ID: 0}},
	},
	{
		nil,
		[]bulk.Image{{ID: 0, Source: "source"}},
		nil,
	},
	{
		[]bulk.Image{{ID: 0}},
		nil,
		[]bulk.Image{{ID: 0}},
	},
	{
		[]bulk.Image{{Name: "img1"}},
		[]bulk.Image{{Name: "img1", Source: "source"}},
		[]bulk.Image{{Name: "img1", Source: "source"}},
	},
	{
		[]bulk.Image{{Name: "img1"}, {Name: "img1"}},
		[]bulk.Image{{Name: "img1", Source: "source"}},
		[]bulk.Image{{Name: "img1", Source: "source"}, {Name: "img1"}},
	},
	{
		[]bulk.Image{{Name: "img1"}, {Name: "img1"}},
		[]bulk.Image{{Name: "img1", Source: "source"}, {Name: "img1", Rating: "q"}},
		[]bulk.Image{{Name: "img1", Rating: "q"}, {Name: "img1"}},
	},
	{
		[]bulk.Image{{Name: "img1"}, {Name: "img2"}},
		[]bulk.Image{{Name: "img1", Source: "source1"}, {Name: "img2", Source: "source2"}, {Name: "img3", Source: "source3"}},
		[]bulk.Image{{Name: "img1", Source: "source1"}, {Name: "img2", Source: "source2"}},
	},
	{
		[]bulk.Image{{Name: "img2"}, {Name: "img1"}},
		[]bulk.Image{{Name: "img2", Source: "source2"}, {Name: "img1", Source: "source1"}},
		[]bulk.Image{{Name: "img1", Source: "source1"}, {Name: "img2", Source: "source2"}},
	},
}

func TestCombine(t *testing.T) {
	for _, tt := range combineTests {
		got := bulk.Combine(tt.images, tt.metadata)
		if want := tt.out; !reflect.DeepEqual(got, want) {
			t.Errorf("Combine(%q, %q) => %q, want %q", tt.images, tt.metadata, got, want)
		}
	}
}

type FileInfoMock struct {
	name  string
	isDir bool
}

func (fim FileInfoMock) Name() string       { return fim.name }
func (fim FileInfoMock) Size() int64        { return 0 }
func (fim FileInfoMock) Mode() os.FileMode  { return 0 }
func (fim FileInfoMock) ModTime() time.Time { return time.Now() }
func (fim FileInfoMock) IsDir() bool        { return fim.isDir }
func (fim FileInfoMock) Sys() interface{}   { return nil }

var loadImagesTests = []struct {
	in  []os.FileInfo
	out []bulk.Image
}{
	// Empty case.
	{
		[]os.FileInfo{},
		[]bulk.Image{},
	},
	// Ignoring .csv and folders.
	{
		[]os.FileInfo{
			FileInfoMock{name: "bulk.csv"},
			FileInfoMock{name: "folder1", isDir: true},
			FileInfoMock{name: "folder2", isDir: true},
		},
		[]bulk.Image{},
	},
	// Ignoring .ico and folder.
	{
		[]os.FileInfo{
			FileInfoMock{name: "a.jpg"},
			FileInfoMock{name: "b.ico"},
			FileInfoMock{name: "folder", isDir: true},
		},
		[]bulk.Image{
			{ID: 0, Name: "a.jpg"},
		},
	},
	// All supported types.
	{
		[]os.FileInfo{
			FileInfoMock{name: "a.gif"},
			FileInfoMock{name: "b.jpeg"},
			FileInfoMock{name: "c.jpg"},
			FileInfoMock{name: "e.png"},
			FileInfoMock{name: "f.swf"},
		},
		[]bulk.Image{
			{ID: 0, Name: "a.gif"},
			{ID: 1, Name: "b.jpeg"},
			{ID: 2, Name: "c.jpg"},
			{ID: 3, Name: "e.png"},
			{ID: 4, Name: "f.swf"},
		},
	},
	// We  depend on iotuil.ReadDir to sort the dir entries.  If for some
	// reason we get unsorted dir entries we treat that order as the correct
	// one.
	{
		[]os.FileInfo{
			FileInfoMock{name: "zzz.jpg"},
			FileInfoMock{name: "bbb.jpg"},
			FileInfoMock{name: "aaa.jpg"},
		},
		[]bulk.Image{
			{ID: 0, Name: "zzz.jpg"},
			{ID: 1, Name: "bbb.jpg"},
			{ID: 2, Name: "aaa.jpg"},
		},
	},
}

func TestLoadImages(t *testing.T) {
	for _, tt := range loadImagesTests {
		got := bulk.LoadImages(tt.in)
		if want := tt.out; !reflect.DeepEqual(got, want) {
			t.Errorf("LoadImages(%q) => %q, want %q", tt.in, got, want)
		}
	}
}

var saveTests = []struct {
	images   []bulk.Image
	dir      string
	prefix   string
	useLinux bool
	out      string
}{
	{
		[]bulk.Image{{ID: 0, Name: "img1", Source: "source1", Rating: "s"}},
		filepath.Join("/", "local", "path", "dir"),
		filepath.Join("/", "server", "path"),
		false,
		filepath.Join("/", "server", "path", "dir", "img1") + ",,source1,s,\n",
	},
	{
		[]bulk.Image{{ID: 0, Name: "img1", Source: "source1", Rating: "s"}},
		filepath.Join("/", "local", "path", "dir"),
		filepath.Join("/", "server", "path"),
		true,
		"/server/path/dir/img1,,source1,s,\n",
	},
	{
		[]bulk.Image{
			{ID: 0, Name: "img1", Source: "source1", Rating: "s"},
			{ID: 1, Name: "img2", Source: "source2", Rating: "q"},
		},
		filepath.Join("/", "local", "path", "dir"),
		filepath.Join("/", "server", "path"),
		false,
		filepath.Join("/", "server", "path", "dir", "img1") + ",,source1,s,\n" + filepath.Join("/", "server", "path", "dir", "img2") + ",,source2,q,\n",
	},
}

func TestSave(t *testing.T) {
	for _, tt := range saveTests {
		var b bytes.Buffer
		err := bulk.Save(&b, tt.images, tt.dir, tt.prefix, tt.useLinux)
		if err != nil {
			t.Errorf("Save(%q, %q, %q) returned err %q", tt.images, tt.dir, tt.prefix, err)
		}
		if got, want := b.String(), tt.out; got != want {
			t.Errorf("Save(%q, %q, %q) => %q, want %q", tt.images, tt.dir, tt.prefix, got, want)
		}
	}
}

func TestSave_writeFail(t *testing.T) {
	images := []bulk.Image{{ID: 0}, {ID: 1}}
	var b bytes.Buffer
	in := ErrWriter(&b, fmt.Errorf("write fail"))
	err := bulk.Save(in, images, "", "", false)
	if err == nil {
		t.Errorf("Save with write failure must return err but returned %q", err)
	}
}

// ErrWriter returns a writer that always fails with the provided error.
func ErrWriter(w io.Writer, err error) io.Writer {
	return &errWriter{w, err}
}

type errWriter struct {
	w   io.Writer
	err error
}

func (e *errWriter) Write(p []byte) (n int, err error) {
	err = e.err
	return
}

// ErrReader returns a reader that always fails with the provided error.
func ErrReader(r io.Reader, err error) io.Reader {
	return &errReader{r, err}
}

type errReader struct {
	r   io.Reader
	err error
}

func (e *errReader) Read(p []byte) (n int, err error) {
	err = e.err
	return
}
<commit_msg>Fix local-tagger import & add CurrentPrefix test<commit_after>package bulk_test

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/kusubooru/tagaa/bulk"
)

var errInvalidFormat = errors.New("invalid csv file format")
var loadCSVTests = []struct {
	in   string
	out  []bulk.Image
	oerr error
}{
	{"", []bulk.Image{}, nil},
	{",,,,", []bulk.Image{}, nil},
	{",,,,,,,,,,,,,,,,,,,", []bulk.Image(nil), errInvalidFormat},
	{"invalid format", []bulk.Image(nil), errInvalidFormat},
	{",,,,\n,,,,\n,,,,", []bulk.Image{}, nil},
	{
		"/server/path/img1,tag1 tag2,source,s,",
		[]bulk.Image{
			{Name: "img1", Tags: []string{"tag1", "tag2"}, Source: "source", Rating: "s"},
		},
		nil,
	},
	{
		"/server/path/img1,tag1 tag2,source1,s,\n" +
			",,,,\n" +
			"/server/path/img2,tag1 tag2,source2,q,\n",
		[]bulk.Image{
			{Name: "img1", Tags: []string{"tag1", "tag2"}, Source: "source1", Rating: "s"},
			{Name: "img2", Tags: []string{"tag1", "tag2"}, Source: "source2", Rating: "q"},
		},
		nil,
	},
}

func TestLoadCSV(t *testing.T) {
	for _, tt := range loadCSVTests {
		got, err := bulk.LoadCSV(strings.NewReader(tt.in))
		if want := tt.oerr; !reflect.DeepEqual(err, want) {
			t.Errorf("LoadCSV(%q) returned err %q, want %q", tt.in, err, want)
		}
		if want := tt.out; !reflect.DeepEqual(got, want) {
			t.Errorf("LoadCSV(%q) => %q, want %q", tt.in, got, want)
		}
	}
}

func TestLoadCSV_readFail(t *testing.T) {
	r := strings.NewReader("")
	in := ErrReader(r, fmt.Errorf("read fail"))
	got, err := bulk.LoadCSV(in)
	if err == nil {
		t.Errorf("LoadCSV with read failure must return err but returned %q, %q", got, err)
	}
}

var currentPrefixTests = []struct {
	indir  string
	infile string
	out    string
	oerr   error
}{
	{
		filepath.Join("/", "local", "path", "dir"),
		",,,,",
		filepath.Join("/"),
		nil,
	},
	{
		filepath.Join("/", "local", "path", "dir"),
		filepath.Join("/", "server", "path", "dir") + ",,,,",
		filepath.Join("/", "server", "path"),
		nil,
	},
	{
		filepath.Join("/", "local", "path", "dir"),
		"",
		"",
		nil,
	},
	{
		filepath.Join("/", "local", "path", "dir"),
		filepath.Join("/", "server", "path", "dir", "somepic.jpg") + ",,,,",
		filepath.Join("/", "server", "path"),
		nil,
	},
}

func TestCurrentPrefix(t *testing.T) {
	for _, tt := range currentPrefixTests {
		got, err := bulk.CurrentPrefix(tt.indir, strings.NewReader(tt.infile))
		if want := tt.oerr; !reflect.DeepEqual(err, want) {
			t.Errorf("CurrentPrefix(%q, %q) returned err %q, want %q", tt.indir, tt.infile, err, want)
		}
		if want := tt.out; !reflect.DeepEqual(got, want) {
			t.Errorf("CurrentPrefix(%q, %q) => %q, want %q", tt.indir, tt.infile, got, want)
		}
	}
}

func TestCurrentPrefix_readFail(t *testing.T) {
	r := strings.NewReader("")
	in := ErrReader(r, fmt.Errorf("read fail"))
	got, err := bulk.CurrentPrefix("", in)
	if err == nil {
		t.Errorf("CurrentPrefix with read failure must return err but returned %q, %q", got, err)
	}
}

var findByIDTests = []struct {
	images []bulk.Image
	id     int
	out    *bulk.Image
}{
	{nil, 0, nil},
	{[]bulk.Image{}, 0, nil},
	{[]bulk.Image{{ID: 1}}, 0, nil},
	{[]bulk.Image{{ID: 1}}, 1, &bulk.Image{ID: 1}},
	{[]bulk.Image{{ID: 2}, {ID: 1}}, 1, &bulk.Image{ID: 1}},
	{[]bulk.Image{{ID: 2}, {ID: 3}, {ID: 1}}, 3, &bulk.Image{ID: 3}},
	{[]bulk.Image{{ID: 2}, {ID: 3}, {ID: 1}}, 5, nil},
}

func TestFindByID(t *testing.T) {
	for _, tt := range findByIDTests {
		got := bulk.FindByID(tt.images, tt.id)
		if want := tt.out; !reflect.DeepEqual(got, want) {
			t.Errorf("FindByID(%q, %q) => %q, want %q", tt.images, tt.id, got, want)
		}
	}
}

var combineTests = []struct {
	images   []bulk.Image
	metadata []bulk.Image
	out      []bulk.Image
}{
	{
		[]bulk.Image{{ID: 0}},
		[]bulk.Image{{ID: 0, Source: "source"}},
		[]bulk.Image{{ID: 0}},
	},
	{
		nil,
		[]bulk.Image{{ID: 0, Source: "source"}},
		nil,
	},
	{
		[]bulk.Image{{ID: 0}},
		nil,
		[]bulk.Image{{ID: 0}},
	},
	{
		[]bulk.Image{{Name: "img1"}},
		[]bulk.Image{{Name: "img1", Source: "source"}},
		[]bulk.Image{{Name: "img1", Source: "source"}},
	},
	{
		[]bulk.Image{{Name: "img1"}, {Name: "img1"}},
		[]bulk.Image{{Name: "img1", Source: "source"}},
		[]bulk.Image{{Name: "img1", Source: "source"}, {Name: "img1"}},
	},
	{
		[]bulk.Image{{Name: "img1"}, {Name: "img1"}},
		[]bulk.Image{{Name: "img1", Source: "source"}, {Name: "img1", Rating: "q"}},
		[]bulk.Image{{Name: "img1", Rating: "q"}, {Name: "img1"}},
	},
	{
		[]bulk.Image{{Name: "img1"}, {Name: "img2"}},
		[]bulk.Image{{Name: "img1", Source: "source1"}, {Name: "img2", Source: "source2"}, {Name: "img3", Source: "source3"}},
		[]bulk.Image{{Name: "img1", Source: "source1"}, {Name: "img2", Source: "source2"}},
	},
	{
		[]bulk.Image{{Name: "img2"}, {Name: "img1"}},
		[]bulk.Image{{Name: "img2", Source: "source2"}, {Name: "img1", Source: "source1"}},
		[]bulk.Image{{Name: "img1", Source: "source1"}, {Name: "img2", Source: "source2"}},
	},
}

func TestCombine(t *testing.T) {
	for _, tt := range combineTests {
		got := bulk.Combine(tt.images, tt.metadata)
		if want := tt.out; !reflect.DeepEqual(got, want) {
			t.Errorf("Combine(%q, %q) => %q, want %q", tt.images, tt.metadata, got, want)
		}
	}
}

type FileInfoMock struct {
	name  string
	isDir bool
}

func (fim FileInfoMock) Name() string       { return fim.name }
func (fim FileInfoMock) Size() int64        { return 0 }
func (fim FileInfoMock) Mode() os.FileMode  { return 0 }
func (fim FileInfoMock) ModTime() time.Time { return time.Now() }
func (fim FileInfoMock) IsDir() bool        { return fim.isDir }
func (fim FileInfoMock) Sys() interface{}   { return nil }

var loadImagesTests = []struct {
	in  []os.FileInfo
	out []bulk.Image
}{
	// Empty case.
	{
		[]os.FileInfo{},
		[]bulk.Image{},
	},
	// Ignoring .csv and folders.
	{
		[]os.FileInfo{
			FileInfoMock{name: "bulk.csv"},
			FileInfoMock{name: "folder1", isDir: true},
			FileInfoMock{name: "folder2", isDir: true},
		},
		[]bulk.Image{},
	},
	// Ignoring .ico and folder.
	{
		[]os.FileInfo{
			FileInfoMock{name: "a.jpg"},
			FileInfoMock{name: "b.ico"},
			FileInfoMock{name: "folder", isDir: true},
		},
		[]bulk.Image{
			{ID: 0, Name: "a.jpg"},
		},
	},
	// All supported types.
	{
		[]os.FileInfo{
			FileInfoMock{name: "a.gif"},
			FileInfoMock{name: "b.jpeg"},
			FileInfoMock{name: "c.jpg"},
			FileInfoMock{name: "e.png"},
			FileInfoMock{name: "f.swf"},
		},
		[]bulk.Image{
			{ID: 0, Name: "a.gif"},
			{ID: 1, Name: "b.jpeg"},
			{ID: 2, Name: "c.jpg"},
			{ID: 3, Name: "e.png"},
			{ID: 4, Name: "f.swf"},
		},
	},
	// We  depend on iotuil.ReadDir to sort the dir entries.  If for some
	// reason we get unsorted dir entries we treat that order as the correct
	// one.
	{
		[]os.FileInfo{
			FileInfoMock{name: "zzz.jpg"},
			FileInfoMock{name: "bbb.jpg"},
			FileInfoMock{name: "aaa.jpg"},
		},
		[]bulk.Image{
			{ID: 0, Name: "zzz.jpg"},
			{ID: 1, Name: "bbb.jpg"},
			{ID: 2, Name: "aaa.jpg"},
		},
	},
}

func TestLoadImages(t *testing.T) {
	for _, tt := range loadImagesTests {
		got := bulk.LoadImages(tt.in)
		if want := tt.out; !reflect.DeepEqual(got, want) {
			t.Errorf("LoadImages(%q) => %q, want %q", tt.in, got, want)
		}
	}
}

var saveTests = []struct {
	images   []bulk.Image
	dir      string
	prefix   string
	useLinux bool
	out      string
}{
	{
		[]bulk.Image{{ID: 0, Name: "img1", Source: "source1", Rating: "s"}},
		filepath.Join("/", "local", "path", "dir"),
		filepath.Join("/", "server", "path"),
		false,
		filepath.Join("/", "server", "path", "dir", "img1") + ",,source1,s,\n",
	},
	{
		[]bulk.Image{{ID: 0, Name: "img1", Source: "source1", Rating: "s"}},
		filepath.Join("/", "local", "path", "dir"),
		filepath.Join("/", "server", "path"),
		true,
		"/server/path/dir/img1,,source1,s,\n",
	},
	{
		[]bulk.Image{
			{ID: 0, Name: "img1", Source: "source1", Rating: "s"},
			{ID: 1, Name: "img2", Source: "source2", Rating: "q"},
		},
		filepath.Join("/", "local", "path", "dir"),
		filepath.Join("/", "server", "path"),
		false,
		filepath.Join("/", "server", "path", "dir", "img1") + ",,source1,s,\n" + filepath.Join("/", "server", "path", "dir", "img2") + ",,source2,q,\n",
	},
}

func TestSave(t *testing.T) {
	for _, tt := range saveTests {
		var b bytes.Buffer
		err := bulk.Save(&b, tt.images, tt.dir, tt.prefix, tt.useLinux)
		if err != nil {
			t.Errorf("Save(%q, %q, %q) returned err %q", tt.images, tt.dir, tt.prefix, err)
		}
		if got, want := b.String(), tt.out; got != want {
			t.Errorf("Save(%q, %q, %q) => %q, want %q", tt.images, tt.dir, tt.prefix, got, want)
		}
	}
}

func TestSave_writeFail(t *testing.T) {
	images := []bulk.Image{{ID: 0}, {ID: 1}}
	var b bytes.Buffer
	in := ErrWriter(&b, fmt.Errorf("write fail"))
	err := bulk.Save(in, images, "", "", false)
	if err == nil {
		t.Errorf("Save with write failure must return err but returned %q", err)
	}
}

// ErrWriter returns a writer that always fails with the provided error.
func ErrWriter(w io.Writer, err error) io.Writer {
	return &errWriter{w, err}
}

type errWriter struct {
	w   io.Writer
	err error
}

func (e *errWriter) Write(p []byte) (n int, err error) {
	err = e.err
	return
}

// ErrReader returns a reader that always fails with the provided error.
func ErrReader(r io.Reader, err error) io.Reader {
	return &errReader{r, err}
}

type errReader struct {
	r   io.Reader
	err error
}

func (e *errReader) Read(p []byte) (n int, err error) {
	err = e.err
	return
}
<|endoftext|>


<commit_before>package bulk

import (
	"crypto/tls"
	"net/http"
	"os"
	"time"

	"github.com/cloudfoundry-incubator/nsync/recipebuilder"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/cc_messages"
	"github.com/cloudfoundry-incubator/runtime-schema/metric"
	"github.com/cloudfoundry/gunk/timeprovider"
	"github.com/pivotal-golang/lager"
)

const (
	syncDesiredLRPsDuration = metric.Duration("DesiredLRPSyncDuration")
)

type Processor struct {
	bbs             bbs.NsyncBBS
	pollingInterval time.Duration
	ccFetchTimeout  time.Duration
	freshnessTTL    time.Duration
	bulkBatchSize   uint
	skipCertVerify  bool
	logger          lager.Logger
	fetcher         Fetcher
	differ          Differ
	timeProvider    timeprovider.TimeProvider
}

func NewProcessor(
	bbs bbs.NsyncBBS,
	pollingInterval time.Duration,
	ccFetchTimeout time.Duration,
	freshnessTTL time.Duration,
	bulkBatchSize uint,
	skipCertVerify bool,
	logger lager.Logger,
	fetcher Fetcher,
	differ Differ,
	timeProvider timeprovider.TimeProvider,
) *Processor {
	return &Processor{
		bbs:             bbs,
		pollingInterval: pollingInterval,
		ccFetchTimeout:  ccFetchTimeout,
		freshnessTTL:    freshnessTTL,
		bulkBatchSize:   bulkBatchSize,
		skipCertVerify:  skipCertVerify,
		logger:          logger,
		fetcher:         fetcher,
		differ:          differ,
		timeProvider:    timeProvider,
	}
}

func (p *Processor) Run(signals <-chan os.Signal, ready chan<- struct{}) error {
	close(ready)

	stop := p.sync(signals)

	for {
		if stop {
			return nil
		}
		select {
		case <-signals:
			return nil
		case <-time.After(p.pollingInterval):
			stop = p.sync(signals)
		}
	}
}

func (p *Processor) sync(signals <-chan os.Signal) bool {
	start := p.timeProvider.Time()
	duration := time.Duration(0)
	defer func() {
		syncDesiredLRPsDuration.Send(duration)
	}()

	processLog := p.logger.Session("processor")

	processLog.Info("getting-desired-lrps-from-bbs")

	existing, err := p.bbs.GetAllDesiredLRPsByDomain(recipebuilder.LRPDomain)
	if err != nil {
		p.logger.Error("failed-to-get-desired-lrps", err)
		return false
	}

	fromCC := make(chan cc_messages.DesireAppRequestFromCC)

	httpClient := &http.Client{
		Timeout: p.ccFetchTimeout,
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: p.skipCertVerify,
			},
		},
	}

	processLog.Info("fetching-desired-from-cc")

	fetchErrs := make(chan error)

	go func() {
		fetchErrs <- p.fetcher.Fetch(fromCC, httpClient)
	}()

	changes := p.differ.Diff(existing, fromCC)
	fetchErr := <-fetchErrs
	if fetchErr != nil {
		processLog.Error("failed-to-fetch", fetchErr)
		return false
	}

	for _, change := range changes {
		select {
		case <-signals:
			// allow interruption while processing changes
			return true
		default:
			p.bbs.ChangeDesiredLRP(change)
		}
	}

	err = p.bbs.BumpFreshness(recipebuilder.LRPDomain, p.freshnessTTL)
	if err != nil {
		processLog.Error("failed-to-bump-freshness", err)
	}

	duration = p.timeProvider.Time().Sub(start)
	return false
}
<commit_msg>Set up the Transport properly for timeouts and SSL<commit_after>package bulk

import (
	"crypto/tls"
	"net"
	"net/http"
	"os"
	"time"

	"github.com/cloudfoundry-incubator/nsync/recipebuilder"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/cc_messages"
	"github.com/cloudfoundry-incubator/runtime-schema/metric"
	"github.com/cloudfoundry/gunk/timeprovider"
	"github.com/pivotal-golang/lager"
)

const (
	syncDesiredLRPsDuration = metric.Duration("DesiredLRPSyncDuration")
)

type Processor struct {
	bbs             bbs.NsyncBBS
	pollingInterval time.Duration
	ccFetchTimeout  time.Duration
	freshnessTTL    time.Duration
	bulkBatchSize   uint
	skipCertVerify  bool
	logger          lager.Logger
	fetcher         Fetcher
	differ          Differ
	timeProvider    timeprovider.TimeProvider
}

func NewProcessor(
	bbs bbs.NsyncBBS,
	pollingInterval time.Duration,
	ccFetchTimeout time.Duration,
	freshnessTTL time.Duration,
	bulkBatchSize uint,
	skipCertVerify bool,
	logger lager.Logger,
	fetcher Fetcher,
	differ Differ,
	timeProvider timeprovider.TimeProvider,
) *Processor {
	return &Processor{
		bbs:             bbs,
		pollingInterval: pollingInterval,
		ccFetchTimeout:  ccFetchTimeout,
		freshnessTTL:    freshnessTTL,
		bulkBatchSize:   bulkBatchSize,
		skipCertVerify:  skipCertVerify,
		logger:          logger,
		fetcher:         fetcher,
		differ:          differ,
		timeProvider:    timeProvider,
	}
}

func (p *Processor) Run(signals <-chan os.Signal, ready chan<- struct{}) error {
	close(ready)

	stop := p.sync(signals)

	for {
		if stop {
			return nil
		}
		select {
		case <-signals:
			return nil
		case <-time.After(p.pollingInterval):
			stop = p.sync(signals)
		}
	}
}

func (p *Processor) sync(signals <-chan os.Signal) bool {
	start := p.timeProvider.Time()
	duration := time.Duration(0)
	defer func() {
		syncDesiredLRPsDuration.Send(duration)
	}()

	processLog := p.logger.Session("processor")

	processLog.Info("getting-desired-lrps-from-bbs")

	existing, err := p.bbs.GetAllDesiredLRPsByDomain(recipebuilder.LRPDomain)
	if err != nil {
		p.logger.Error("failed-to-get-desired-lrps", err)
		return false
	}

	fromCC := make(chan cc_messages.DesireAppRequestFromCC)

	httpClient := &http.Client{
		Timeout: p.ccFetchTimeout,
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   10 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			TLSHandshakeTimeout: 10 * time.Second,
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: p.skipCertVerify,
				MinVersion:         tls.VersionTLS10,
			},
		},
	}

	processLog.Info("fetching-desired-from-cc")

	fetchErrs := make(chan error)

	go func() {
		fetchErrs <- p.fetcher.Fetch(fromCC, httpClient)
	}()

	changes := p.differ.Diff(existing, fromCC)
	fetchErr := <-fetchErrs
	if fetchErr != nil {
		processLog.Error("failed-to-fetch", fetchErr)
		return false
	}

	for _, change := range changes {
		select {
		case <-signals:
			// allow interruption while processing changes
			return true
		default:
			p.bbs.ChangeDesiredLRP(change)
		}
	}

	err = p.bbs.BumpFreshness(recipebuilder.LRPDomain, p.freshnessTTL)
	if err != nil {
		processLog.Error("failed-to-bump-freshness", err)
	}

	duration = p.timeProvider.Time().Sub(start)
	return false
}
<|endoftext|>


<commit_before><commit_msg>running make generate<commit_after><|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logic

import (
	"github.com/golang/glog"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/labels"

	vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/poc.autoscaling.k8s.io/v1alpha1"
	vpa_lister "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/listers/poc.autoscaling.k8s.io/v1alpha1"
	vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa"
)

// ContainerResources holds resources request for container
type ContainerResources struct {
	Requests v1.ResourceList
}

func newContainerResources() ContainerResources {
	return ContainerResources{Requests: v1.ResourceList{}}
}

// RecommendationProvider gets current recommendation, annotations and vpaName for the given pod.
type RecommendationProvider interface {
	GetContainersResourcesForPod(pod *v1.Pod) ([]ContainerResources, vpa_api_util.ContainerToAnnotationsMap, string, error)
}

type recommendationProvider struct {
	vpaLister               vpa_lister.VerticalPodAutoscalerLister
	recommendationProcessor vpa_api_util.RecommendationProcessor
}

// NewRecommendationProvider constructs the recommendation provider that list VPAs and can be used to determine recommendations for pods.
func NewRecommendationProvider(vpaLister vpa_lister.VerticalPodAutoscalerLister, recommendationProcessor vpa_api_util.RecommendationProcessor) *recommendationProvider {
	return &recommendationProvider{vpaLister: vpaLister,
		recommendationProcessor: recommendationProcessor}
}

// getContainersResources returns the recommended resources for each container in the given pod in the same order they are specified in the pod.Spec.
func getContainersResources(pod *v1.Pod, podRecommendation vpa_types.RecommendedPodResources) []ContainerResources {
	resources := make([]ContainerResources, len(pod.Spec.Containers))
	for i, container := range pod.Spec.Containers {
		resources[i] = newContainerResources()

		recommendation := vpa_api_util.GetRecommendationForContainer(container.Name, &podRecommendation)
		if recommendation == nil {
			glog.V(2).Infof("no matching recommendation found for container %s", container.Name)
			continue
		}
		resources[i].Requests = recommendation.Target
	}
	return resources
}

func (p *recommendationProvider) getMatchingVPA(pod *v1.Pod) *vpa_types.VerticalPodAutoscaler {
	configs, err := p.vpaLister.VerticalPodAutoscalers(pod.Namespace).List(labels.Everything())
	if err != nil {
		glog.Error("failed to get vpa configs: %v", err)
		return nil
	}
	onConfigs := make([]*vpa_types.VerticalPodAutoscaler, 0)
	for _, vpaConfig := range configs {
		if vpa_api_util.GetUpdateMode(vpaConfig) == vpa_types.UpdateModeOff {
			continue
		}
		onConfigs = append(onConfigs, vpaConfig)
	}
	glog.Infof("Let's choose from %d configs", len(onConfigs))
	return vpa_api_util.GetControllingVPAForPod(pod, onConfigs)
}

// GetContainersResourcesForPod returns recommended request for a given pod, annotations and name of controlling VPA.
// The returned slice corresponds 1-1 to containers in the Pod.
func (p *recommendationProvider) GetContainersResourcesForPod(pod *v1.Pod) ([]ContainerResources, vpa_api_util.ContainerToAnnotationsMap, string, error) {
	glog.V(2).Infof("updating requirements for pod %s.", pod.Name)
	vpaConfig := p.getMatchingVPA(pod)
	if vpaConfig == nil {
		glog.V(2).Infof("no matching VPA found for pod %s", pod.Name)
		return nil, nil, "", nil
	}

	var annotations vpa_api_util.ContainerToAnnotationsMap
	recommendedPodResources := &vpa_types.RecommendedPodResources{}

	if vpaConfig.Status.Recommendation != nil {
		var err error
		recommendedPodResources, annotations, err = p.recommendationProcessor.Apply(vpaConfig.Status.Recommendation, vpaConfig.Spec.ResourcePolicy, vpaConfig.Status.Conditions, pod)
		if err != nil {
			glog.V(2).Infof("cannot process recommendation for pod %s", pod.Name)
			return nil, annotations, vpaConfig.Name, err
		}
	}
	containerResources := getContainersResources(pod, *recommendedPodResources)
	return containerResources, annotations, vpaConfig.Name, nil
}
<commit_msg>Improve logging to be more clear (with pod name) and using logging level<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logic

import (
	"github.com/golang/glog"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/labels"

	vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/poc.autoscaling.k8s.io/v1alpha1"
	vpa_lister "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/listers/poc.autoscaling.k8s.io/v1alpha1"
	vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa"
)

// ContainerResources holds resources request for container
type ContainerResources struct {
	Requests v1.ResourceList
}

func newContainerResources() ContainerResources {
	return ContainerResources{Requests: v1.ResourceList{}}
}

// RecommendationProvider gets current recommendation, annotations and vpaName for the given pod.
type RecommendationProvider interface {
	GetContainersResourcesForPod(pod *v1.Pod) ([]ContainerResources, vpa_api_util.ContainerToAnnotationsMap, string, error)
}

type recommendationProvider struct {
	vpaLister               vpa_lister.VerticalPodAutoscalerLister
	recommendationProcessor vpa_api_util.RecommendationProcessor
}

// NewRecommendationProvider constructs the recommendation provider that list VPAs and can be used to determine recommendations for pods.
func NewRecommendationProvider(vpaLister vpa_lister.VerticalPodAutoscalerLister, recommendationProcessor vpa_api_util.RecommendationProcessor) *recommendationProvider {
	return &recommendationProvider{vpaLister: vpaLister,
		recommendationProcessor: recommendationProcessor}
}

// getContainersResources returns the recommended resources for each container in the given pod in the same order they are specified in the pod.Spec.
func getContainersResources(pod *v1.Pod, podRecommendation vpa_types.RecommendedPodResources) []ContainerResources {
	resources := make([]ContainerResources, len(pod.Spec.Containers))
	for i, container := range pod.Spec.Containers {
		resources[i] = newContainerResources()

		recommendation := vpa_api_util.GetRecommendationForContainer(container.Name, &podRecommendation)
		if recommendation == nil {
			glog.V(2).Infof("no matching recommendation found for container %s", container.Name)
			continue
		}
		resources[i].Requests = recommendation.Target
	}
	return resources
}

func (p *recommendationProvider) getMatchingVPA(pod *v1.Pod) *vpa_types.VerticalPodAutoscaler {
	configs, err := p.vpaLister.VerticalPodAutoscalers(pod.Namespace).List(labels.Everything())
	if err != nil {
		glog.Error("failed to get vpa configs: %v", err)
		return nil
	}
	onConfigs := make([]*vpa_types.VerticalPodAutoscaler, 0)
	for _, vpaConfig := range configs {
		if vpa_api_util.GetUpdateMode(vpaConfig) == vpa_types.UpdateModeOff {
			continue
		}
		onConfigs = append(onConfigs, vpaConfig)
	}
	glog.V(2).Infof("Let's choose from %d configs for pod %s/%s", len(onConfigs), pod.Namespace, pod.Name)
	return vpa_api_util.GetControllingVPAForPod(pod, onConfigs)
}

// GetContainersResourcesForPod returns recommended request for a given pod, annotations and name of controlling VPA.
// The returned slice corresponds 1-1 to containers in the Pod.
func (p *recommendationProvider) GetContainersResourcesForPod(pod *v1.Pod) ([]ContainerResources, vpa_api_util.ContainerToAnnotationsMap, string, error) {
	glog.V(2).Infof("updating requirements for pod %s.", pod.Name)
	vpaConfig := p.getMatchingVPA(pod)
	if vpaConfig == nil {
		glog.V(2).Infof("no matching VPA found for pod %s", pod.Name)
		return nil, nil, "", nil
	}

	var annotations vpa_api_util.ContainerToAnnotationsMap
	recommendedPodResources := &vpa_types.RecommendedPodResources{}

	if vpaConfig.Status.Recommendation != nil {
		var err error
		recommendedPodResources, annotations, err = p.recommendationProcessor.Apply(vpaConfig.Status.Recommendation, vpaConfig.Spec.ResourcePolicy, vpaConfig.Status.Conditions, pod)
		if err != nil {
			glog.V(2).Infof("cannot process recommendation for pod %s", pod.Name)
			return nil, annotations, vpaConfig.Name, err
		}
	}
	containerResources := getContainersResources(pod, *recommendedPodResources)
	return containerResources, annotations, vpaConfig.Name, nil
}
<|endoftext|>


<commit_before>package bytecode

import "fmt"

func typenameString(c *CpoolInfo, info MultinameInfo) string {
	var str string
	name := c.Multinames[info.Name]
	str += c.Strings[name.Name]
	str += "<"
	for i, p := range info.Params {
		if i > 0 {
			str += ", "
		}
		str += c.MultinameString(p)
	}
	str += ">"
	return str
}

// MultinameString converts a multiname to a string
func (c *CpoolInfo) MultinameString(m uint32) string {
	info := c.Multinames[m]
	switch info.Kind {
	case MultinameKindQName, MultinameKindQNameA:
		return c.Strings[info.Name]
	case MultinameKindRTQName, MultinameKindRTQNameA:
		return fmt.Sprintf("[*].%v", c.Strings[info.Name])
	case MultinameKindRTQNameL, MultinameKindRTQNameLA:
		return fmt.Sprint("[*].[*]")
	case MultinameKindTypename:
		return typenameString(c, info)
	default:
		return fmt.Sprint(c.Strings[info.Name])
	}
}

// NamespaceString converts a namespace to a string
func (c *CpoolInfo) NamespaceString(n uint32) string {
	info := c.Namespaces[n]
	return c.Strings[info.Name]
}
<commit_msg>avoid crash on malformed multiname<commit_after>package bytecode

import "fmt"

func typenameString(c *CpoolInfo, info MultinameInfo) string {
	var str string
	name := c.Multinames[info.Name]
	str += c.Strings[name.Name]
	str += "<"
	for i, p := range info.Params {
		if i > 0 {
			str += ", "
		}
		str += c.MultinameString(p)
	}
	str += ">"
	return str
}

// MultinameString converts a multiname to a string
func (c *CpoolInfo) MultinameString(m uint32) string {
	if int(m) >= len(c.Multinames) {
		return ""
	}
	info := c.Multinames[m]
	switch info.Kind {
	case MultinameKindQName, MultinameKindQNameA:
		return c.Strings[info.Name]
	case MultinameKindRTQName, MultinameKindRTQNameA:
		return fmt.Sprintf("[*].%v", c.Strings[info.Name])
	case MultinameKindRTQNameL, MultinameKindRTQNameLA:
		return fmt.Sprint("[*].[*]")
	case MultinameKindTypename:
		return typenameString(c, info)
	default:
		return fmt.Sprint(c.Strings[info.Name])
	}
}

// NamespaceString converts a namespace to a string
func (c *CpoolInfo) NamespaceString(n uint32) string {
	info := c.Namespaces[n]
	return c.Strings[info.Name]
}
<|endoftext|>


<commit_before>/**
 * [x]remove multiple keywords at once
 * [x]add multiple keywords at once
 **/

package line

import (
	"net/http"
	"regexp"

	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/liam-lai/ptt-alertor/models/subscription"
	user "github.com/liam-lai/ptt-alertor/models/user/redis"
	"github.com/liam-lai/ptt-alertor/myutil"
	"github.com/line/line-bot-sdk-go/linebot"
)

var bot *linebot.Client
var err error
var commands = map[string]string{
	"指令": "可使用的指令清單",
	"清單": "目前追蹤的看板與關鍵字",
	"新增": "新增看板關鍵字。範例：\n\t\t新增 gossiping 爆卦\n\t\t新增 gossiping 爆卦,問卦\n\t\t新增 gossiping 爆卦，問卦",
	"刪除": "刪除看板關鍵字。範例：\n\t\t刪除 gossiping 爆卦\n\t\t刪除 gossiping 爆卦,問卦\n\t\t刪除 gossiping 爆卦，問卦",
}

func init() {
	config := myutil.Config("line")
	bot, err = linebot.New(config["channelSecret"], config["channelAccessToken"])
	if err != nil {
		log.Fatal(err)
	}
}

func HandleRequest(r *http.Request) {
	events, err := bot.ParseRequest(r)
	if err != nil {
		log.WithError(err).Error("Line ParseRequest Error")
	}
	for _, event := range events {
		switch event.Type {
		case linebot.EventTypeMessage:
			handleMessage(event)
		case linebot.EventTypeFollow:
			handleFollow(event)
		case linebot.EventTypeUnfollow:
			handleUnfollow(event)
		}
	}
}

/**
 * TODO: check board exist or not
 * 1. hotboard
 * 2. allboard
 **/
func handleMessage(event *linebot.Event) {
	var responseText string
	userID := event.Source.UserID
	switch message := event.Message.(type) {
	case *linebot.TextMessage:
		responseText = handleCommand(message.Text, userID)
	}
	_, err := bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(responseText)).Do()
	if err != nil {
		log.WithError(err).Error("Line Reply Message Failed")
	}
}

func handleCommand(text string, userID string) string {
	args := strings.Fields(text)
	command, args := args[0], args[1:]
	switch command {
	case "清單":
		return new(user.User).Find(userID).Subscribes.String()
	case "指令":
		return stringCommands()
	case "新增", "刪除":
		matched, err := regexp.MatchString("^(新增|刪除)(\\s+)([\\w\\d]+)(\\s+)([^\\s]+)$", text)
		if err != nil {
			log.WithError(err).Error("Line Check Command Failed")
		}
		if !matched {
			return "指令格式錯誤。關鍵字與逗號間不可有空格。範例：\n" + command + " gossiping 問卦,爆卦"
		}
		board := args[0]
		keywords := splitKeywords(args[1])
		if command == "新增" {
			err := subscribe(userID, board, keywords)
			if err != nil {
				return "新增失敗，請等待修復。"
			}
			return "新增成功"
		}
		if command == "刪除" {
			err := unsubscribe(userID, board, keywords)
			if err != nil {
				return "刪除失敗，請等待修復。"
			}
			return "刪除成功"
		}
	}
	return "無此指令，請打「指令」查看指令清單"
}

func stringCommands() string {
	str := ""
	for key, val := range commands {
		str += key + "：" + val + "\n"
	}
	return str
}

func splitKeywords(keywords string) []string {
	if strings.Contains(keywords, ",") {
		return strings.Split(keywords, ",")
	}

	if strings.Contains(keywords, "，") {
		return strings.Split(keywords, "，")
	}

	return []string{keywords}
}

func subscribe(account string, board string, keywords []string) error {
	u := new(user.User).Find(account)
	sub := subscription.Subscribe{
		Board:    board,
		Keywords: keywords,
	}
	u.Subscribes.Add(sub)
	err := u.Update()
	if err != nil {
		log.WithError(err).Error("Line Subscribe Update Error")
	}
	return err
}

func unsubscribe(account string, board string, keywords []string) error {
	u := new(user.User).Find(account)
	sub := subscription.Subscribe{
		Board:    board,
		Keywords: keywords,
	}
	u.Subscribes.Remove(sub)
	err := u.Update()
	if err != nil {
		log.WithError(err).Error("Line UnSubscribe Update Error")
	}
	return err
}

func handleFollow(event *linebot.Event) {
	userID := event.Source.UserID
	profile, err := bot.GetProfile(userID).Do()
	if err != nil {
		log.WithError(err).Error("")
	}

	log.WithFields(log.Fields{
		"ID": profile.UserID,
	}).Info("Line Follow")

	u := new(user.User).Find(profile.UserID)

	if u.Profile.Account != "" {
		log.WithFields(log.Fields{
			"ID": profile.UserID,
		}).Info("Line ReFollow")
		u.Enable = true
		u.Update()
	} else {
		log.WithFields(log.Fields{
			"ID": profile.UserID,
		}).Info("Line Follow")
		u.Profile.Account = profile.UserID
		u.Profile.Line = profile.UserID
		u.Enable = true
		err = u.Save()
		if err != nil {
			log.WithError(err).Error("Line Follow Save User Failed")
		}
	}

	_, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(profile.DisplayName+" 歡迎使用 PTT Alertor\n輸入「指令」查看相關功能。")).Do()
	if err != nil {
		log.WithError(err).Error("Line Follow Replay Message Failed")
	}
}

func handleUnfollow(event *linebot.Event) {
	userID := event.Source.UserID
	log.WithFields(log.Fields{
		"ID": userID,
	}).Info("Line Unfollow")
	u := new(user.User).Find(userID)
	u.Enable = false
	u.Update()
}

// useless
func handleLeave() {

}

// useless
func handleJoin() {

}

// useless
func handlePostback() {

}

// useless
func handleBeacon() {

}

func PushTextMessage(id string, message string) {
	_, err := bot.PushMessage(id, linebot.NewTextMessage(message)).Do()
	if err != nil {
		log.WithError(err).Error("Line Push Message Failed")
	} else {
		log.WithFields(log.Fields{
			"ID": id,
		}).Info("Line Push Message")
	}
}

func BroadcastTextMessage(ids []string, message string) {
	_, err := bot.Multicast(ids, linebot.NewTextMessage(message)).Do()
	if err != nil {
		log.WithError(err).Error("Line Broadcast Message Failed")
	} else {
		log.WithFields(log.Fields{
			"IDs": ids,
		}).Info("Line BroadCast Message")
	}
}
<commit_msg>allow hyphen and underscore in board name, fix #49<commit_after>/**
 * [x]remove multiple keywords at once
 * [x]add multiple keywords at once
 **/

package line

import (
	"net/http"
	"regexp"

	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/liam-lai/ptt-alertor/models/subscription"
	user "github.com/liam-lai/ptt-alertor/models/user/redis"
	"github.com/liam-lai/ptt-alertor/myutil"
	"github.com/line/line-bot-sdk-go/linebot"
)

var bot *linebot.Client
var err error
var commands = map[string]string{
	"指令": "可使用的指令清單",
	"清單": "目前追蹤的看板與關鍵字",
	"新增": "新增看板關鍵字。範例：\n\t\t新增 gossiping 爆卦\n\t\t新增 gossiping 爆卦,問卦\n\t\t新增 gossiping 爆卦，問卦",
	"刪除": "刪除看板關鍵字。範例：\n\t\t刪除 gossiping 爆卦\n\t\t刪除 gossiping 爆卦,問卦\n\t\t刪除 gossiping 爆卦，問卦",
}

func init() {
	config := myutil.Config("line")
	bot, err = linebot.New(config["channelSecret"], config["channelAccessToken"])
	if err != nil {
		log.Fatal(err)
	}
}

func HandleRequest(r *http.Request) {
	events, err := bot.ParseRequest(r)
	if err != nil {
		log.WithError(err).Error("Line ParseRequest Error")
	}
	for _, event := range events {
		switch event.Type {
		case linebot.EventTypeMessage:
			handleMessage(event)
		case linebot.EventTypeFollow:
			handleFollow(event)
		case linebot.EventTypeUnfollow:
			handleUnfollow(event)
		}
	}
}

/**
 * TODO: check board exist or not
 * 1. hotboard
 * 2. allboard
 **/
func handleMessage(event *linebot.Event) {
	var responseText string
	userID := event.Source.UserID
	switch message := event.Message.(type) {
	case *linebot.TextMessage:
		responseText = handleCommand(message.Text, userID)
	}
	_, err := bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(responseText)).Do()
	if err != nil {
		log.WithError(err).Error("Line Reply Message Failed")
	}
}

func handleCommand(text string, userID string) string {
	args := strings.Fields(text)
	command, args := args[0], args[1:]
	switch command {
	case "清單":
		return new(user.User).Find(userID).Subscribes.String()
	case "指令":
		return stringCommands()
	case "新增", "刪除":
		matched, err := regexp.MatchString("^(新增|刪除)(\\s+)([\\w\\d-_]+)(\\s+)([^\\s]+)$", text)
		if err != nil {
			log.WithError(err).Error("Line Check Command Failed")
		}
		if !matched {
			return "指令格式錯誤。關鍵字與逗號間不可有空格。範例：\n" + command + " gossiping 問卦,爆卦"
		}
		board := args[0]
		keywords := splitKeywords(args[1])
		if command == "新增" {
			err := subscribe(userID, board, keywords)
			if err != nil {
				return "新增失敗，請等待修復。"
			}
			return "新增成功"
		}
		if command == "刪除" {
			err := unsubscribe(userID, board, keywords)
			if err != nil {
				return "刪除失敗，請等待修復。"
			}
			return "刪除成功"
		}
	}
	return "無此指令，請打「指令」查看指令清單"
}

func stringCommands() string {
	str := ""
	for key, val := range commands {
		str += key + "：" + val + "\n"
	}
	return str
}

func splitKeywords(keywords string) []string {
	if strings.Contains(keywords, ",") {
		return strings.Split(keywords, ",")
	}

	if strings.Contains(keywords, "，") {
		return strings.Split(keywords, "，")
	}

	return []string{keywords}
}

func subscribe(account string, board string, keywords []string) error {
	u := new(user.User).Find(account)
	sub := subscription.Subscribe{
		Board:    board,
		Keywords: keywords,
	}
	u.Subscribes.Add(sub)
	err := u.Update()
	if err != nil {
		log.WithError(err).Error("Line Subscribe Update Error")
	}
	return err
}

func unsubscribe(account string, board string, keywords []string) error {
	u := new(user.User).Find(account)
	sub := subscription.Subscribe{
		Board:    board,
		Keywords: keywords,
	}
	u.Subscribes.Remove(sub)
	err := u.Update()
	if err != nil {
		log.WithError(err).Error("Line UnSubscribe Update Error")
	}
	return err
}

func handleFollow(event *linebot.Event) {
	userID := event.Source.UserID
	profile, err := bot.GetProfile(userID).Do()
	if err != nil {
		log.WithError(err).Error("")
	}

	log.WithFields(log.Fields{
		"ID": profile.UserID,
	}).Info("Line Follow")

	u := new(user.User).Find(profile.UserID)

	if u.Profile.Account != "" {
		log.WithFields(log.Fields{
			"ID": profile.UserID,
		}).Info("Line ReFollow")
		u.Enable = true
		u.Update()
	} else {
		log.WithFields(log.Fields{
			"ID": profile.UserID,
		}).Info("Line Follow")
		u.Profile.Account = profile.UserID
		u.Profile.Line = profile.UserID
		u.Enable = true
		err = u.Save()
		if err != nil {
			log.WithError(err).Error("Line Follow Save User Failed")
		}
	}

	_, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(profile.DisplayName+" 歡迎使用 PTT Alertor\n輸入「指令」查看相關功能。")).Do()
	if err != nil {
		log.WithError(err).Error("Line Follow Replay Message Failed")
	}
}

func handleUnfollow(event *linebot.Event) {
	userID := event.Source.UserID
	log.WithFields(log.Fields{
		"ID": userID,
	}).Info("Line Unfollow")
	u := new(user.User).Find(userID)
	u.Enable = false
	u.Update()
}

// useless
func handleLeave() {

}

// useless
func handleJoin() {

}

// useless
func handlePostback() {

}

// useless
func handleBeacon() {

}

func PushTextMessage(id string, message string) {
	_, err := bot.PushMessage(id, linebot.NewTextMessage(message)).Do()
	if err != nil {
		log.WithError(err).Error("Line Push Message Failed")
	} else {
		log.WithFields(log.Fields{
			"ID": id,
		}).Info("Line Push Message")
	}
}

func BroadcastTextMessage(ids []string, message string) {
	_, err := bot.Multicast(ids, linebot.NewTextMessage(message)).Do()
	if err != nil {
		log.WithError(err).Error("Line Broadcast Message Failed")
	} else {
		log.WithFields(log.Fields{
			"IDs": ids,
		}).Info("Line BroadCast Message")
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Zack Guo <gizak@icloud.com>. All rights reserved.
// Use of this source code is governed by a MIT license that can
// be found in the LICENSE file.

package termui

import (
	"fmt"
	"math"
)

// only 16 possible combinations, why bother
var braillePatterns = map[[2]int]rune{
	[2]int{0, 0}: '⣀',
	[2]int{0, 1}: '⡠',
	[2]int{0, 2}: '⡐',
	[2]int{0, 3}: '⡈',

	[2]int{1, 0}: '⢄',
	[2]int{1, 1}: '⠤',
	[2]int{1, 2}: '⠔',
	[2]int{1, 3}: '⠌',

	[2]int{2, 0}: '⢂',
	[2]int{2, 1}: '⠢',
	[2]int{2, 2}: '⠒',
	[2]int{2, 3}: '⠊',

	[2]int{3, 0}: '⢁',
	[2]int{3, 1}: '⠡',
	[2]int{3, 2}: '⠑',
	[2]int{3, 3}: '⠉',
}

var lSingleBraille = [4]rune{'\u2840', '⠄', '⠂', '⠁'}
var rSingleBraille = [4]rune{'\u2880', '⠠', '⠐', '⠈'}

// LineChart has two modes: braille(default) and dot. Using braille gives 2x capicity as dot mode,
// because one braille char can represent two data points.
/*
  lc := termui.NewLineChart()
  lc.Border.Label = "braille-mode Line Chart"
  lc.Data = [1.2, 1.3, 1.5, 1.7, 1.5, 1.6, 1.8, 2.0]
  lc.Width = 50
  lc.Height = 12
  lc.AxesColor = termui.ColorWhite
  lc.LineColor = termui.ColorGreen | termui.AttrBold
  // termui.Render(lc)...
*/
type LineChart struct {
	Block
	Data          []float64
	DataLabels    []string // if unset, the data indices will be used
	Mode          string   // braille | dot
	DotStyle      rune
	LineColor     Attribute
	scale         float64 // data span per cell on y-axis
	AxesColor     Attribute
	drawingX      int
	drawingY      int
	axisYHeight   int
	axisXWidth    int
	axisYLabelGap int
	axisXLabelGap int
	topValue      float64
	bottomValue   float64
	labelX        [][]rune
	labelY        [][]rune
	labelYSpace   int
	maxY          float64
	minY          float64
}

// NewLineChart returns a new LineChart with current theme.
func NewLineChart() *LineChart {
	lc := &LineChart{Block: *NewBlock()}
	lc.AxesColor = ThemeAttr("linechart.axes.fg")
	lc.LineColor = ThemeAttr("linechart.line.fg")
	lc.Mode = "braille"
	lc.DotStyle = '•'
	lc.axisXLabelGap = 2
	lc.axisYLabelGap = 1
	lc.bottomValue = math.Inf(1)
	lc.topValue = math.Inf(-1)
	return lc
}

// one cell contains two data points
// so the capicity is 2x as dot-mode
func (lc *LineChart) renderBraille() Buffer {
	buf := NewBuffer()

	// return: b -> which cell should the point be in
	//         m -> in the cell, divided into 4 equal height levels, which subcell?
	getPos := func(d float64) (b, m int) {
		cnt4 := int((d-lc.bottomValue)/(lc.scale/4) + 0.5)
		b = cnt4 / 4
		m = cnt4 % 4
		return
	}
	// plot points
	for i := 0; 2*i+1 < len(lc.Data) && i < lc.axisXWidth; i++ {
		b0, m0 := getPos(lc.Data[2*i])
		b1, m1 := getPos(lc.Data[2*i+1])

		if b0 == b1 {
			c := Cell{
				Ch: braillePatterns[[2]int{m0, m1}],
				Bg: lc.Bg,
				Fg: lc.LineColor,
			}
			y := lc.innerArea.Min.Y + lc.innerArea.Dy() - 3 - b0
			x := lc.innerArea.Min.X + lc.labelYSpace + 1 + i
			buf.Set(x, y, c)
		} else {
			c0 := Cell{Ch: lSingleBraille[m0],
				Fg: lc.LineColor,
				Bg: lc.Bg}
			x0 := lc.innerArea.Min.X + lc.labelYSpace + 1 + i
			y0 := lc.innerArea.Min.Y + lc.innerArea.Dy() - 3 - b0
			buf.Set(x0, y0, c0)

			c1 := Cell{Ch: rSingleBraille[m1],
				Fg: lc.LineColor,
				Bg: lc.Bg}
			x1 := lc.innerArea.Min.X + lc.labelYSpace + 1 + i
			y1 := lc.innerArea.Min.Y + lc.innerArea.Dy() - 3 - b1
			buf.Set(x1, y1, c1)
		}

	}
	return buf
}

func (lc *LineChart) renderDot() Buffer {
	buf := NewBuffer()
	for i := 0; i < len(lc.Data) && i < lc.axisXWidth; i++ {
		c := Cell{
			Ch: lc.DotStyle,
			Fg: lc.LineColor,
			Bg: lc.Bg,
		}
		x := lc.innerArea.Min.X + lc.labelYSpace + 1 + i
		y := lc.innerArea.Min.Y + lc.innerArea.Dy() - 3 - int((lc.Data[i]-lc.bottomValue)/lc.scale+0.5)
		buf.Set(x, y, c)
	}

	return buf
}

func (lc *LineChart) calcLabelX() {
	lc.labelX = [][]rune{}

	for i, l := 0, 0; i < len(lc.DataLabels) && l < lc.axisXWidth; i++ {
		if lc.Mode == "dot" {
			if l >= len(lc.DataLabels) {
				break
			}

			s := str2runes(lc.DataLabels[l])
			w := strWidth(lc.DataLabels[l])
			if l+w <= lc.axisXWidth {
				lc.labelX = append(lc.labelX, s)
			}
			l += w + lc.axisXLabelGap
		} else { // braille
			if 2*l >= len(lc.DataLabels) {
				break
			}

			s := str2runes(lc.DataLabels[2*l])
			w := strWidth(lc.DataLabels[2*l])
			if l+w <= lc.axisXWidth {
				lc.labelX = append(lc.labelX, s)
			}
			l += w + lc.axisXLabelGap

		}
	}
}

func shortenFloatVal(x float64) string {
	s := fmt.Sprintf("%.2f", x)
	if len(s)-3 > 3 {
		s = fmt.Sprintf("%.2e", x)
	}

	if x < 0 {
		s = fmt.Sprintf("%.2f", x)
	}
	return s
}

func (lc *LineChart) calcLabelY() {
	span := lc.topValue - lc.bottomValue
	lc.scale = span / float64(lc.axisYHeight)

	n := (1 + lc.axisYHeight) / (lc.axisYLabelGap + 1)
	lc.labelY = make([][]rune, n)
	maxLen := 0
	for i := 0; i < n; i++ {
		s := str2runes(shortenFloatVal(lc.bottomValue + float64(i)*span/float64(n)))
		if len(s) > maxLen {
			maxLen = len(s)
		}
		lc.labelY[i] = s
	}

	lc.labelYSpace = maxLen
}

func (lc *LineChart) calcLayout() {
	// set datalabels if it is not provided
	if lc.DataLabels == nil || len(lc.DataLabels) == 0 {
		lc.DataLabels = make([]string, len(lc.Data))
		for i := range lc.Data {
			lc.DataLabels[i] = fmt.Sprint(i)
		}
	}

	// lazy increase, to avoid y shaking frequently
	// update bound Y when drawing is gonna overflow
	lc.minY = lc.Data[0]
	lc.maxY = lc.Data[0]

	// valid visible range
	vrange := lc.innerArea.Dx()
	if lc.Mode == "braille" {
		vrange = 2 * lc.innerArea.Dx()
	}
	if vrange > len(lc.Data) {
		vrange = len(lc.Data)
	}

	for _, v := range lc.Data[:vrange] {
		if v > lc.maxY {
			lc.maxY = v
		}
		if v < lc.minY {
			lc.minY = v
		}
	}

	span := lc.maxY - lc.minY

	if lc.minY < lc.bottomValue {
		lc.bottomValue = lc.minY - 0.2*span
	}

	if lc.maxY > lc.topValue {
		lc.topValue = lc.maxY + 0.2*span
	}

	lc.axisYHeight = lc.innerArea.Dy() - 2
	lc.calcLabelY()

	lc.axisXWidth = lc.innerArea.Dx() - 1 - lc.labelYSpace
	lc.calcLabelX()

	lc.drawingX = lc.innerArea.Min.X + 1 + lc.labelYSpace
	lc.drawingY = lc.innerArea.Min.Y
}

func (lc *LineChart) plotAxes() Buffer {
	buf := NewBuffer()

	origY := lc.innerArea.Min.Y + lc.innerArea.Dy() - 2
	origX := lc.innerArea.Min.X + lc.labelYSpace

	buf.Set(origX, origY, Cell{Ch: ORIGIN, Fg: lc.AxesColor, Bg: lc.Bg})

	for x := origX + 1; x < origX+lc.axisXWidth; x++ {
		buf.Set(x, origY, Cell{Ch: HDASH, Fg: lc.AxesColor, Bg: lc.Bg})
	}

	for dy := 1; dy <= lc.axisYHeight; dy++ {
		buf.Set(origX, origY-dy, Cell{Ch: VDASH, Fg: lc.AxesColor, Bg: lc.Bg})
	}

	// x label
	oft := 0
	for _, rs := range lc.labelX {
		if oft+len(rs) > lc.axisXWidth {
			break
		}
		for j, r := range rs {
			c := Cell{
				Ch: r,
				Fg: lc.AxesColor,
				Bg: lc.Bg,
			}
			x := origX + oft + j
			y := lc.innerArea.Min.Y + lc.innerArea.Dy() - 1
			buf.Set(x, y, c)
		}
		oft += len(rs) + lc.axisXLabelGap
	}

	// y labels
	for i, rs := range lc.labelY {
		for j, r := range rs {
			buf.Set(
				lc.innerArea.Min.X+j,
				origY-i*(lc.axisYLabelGap+1),
				Cell{Ch: r, Fg: lc.AxesColor, Bg: lc.Bg})
		}
	}

	return buf
}

// Buffer implements Bufferer interface.
func (lc *LineChart) Buffer() Buffer {
	buf := lc.Block.Buffer()

	if lc.Data == nil || len(lc.Data) == 0 {
		return buf
	}
	lc.calcLayout()
	buf.Merge(lc.plotAxes())

	if lc.Mode == "dot" {
		buf.Merge(lc.renderDot())
	} else {
		buf.Merge(lc.renderBraille())
	}

	return buf
}
<commit_msg>linechart: reset bottomValue and topValue on each calculation, to only consider current data<commit_after>// Copyright 2016 Zack Guo <gizak@icloud.com>. All rights reserved.
// Use of this source code is governed by a MIT license that can
// be found in the LICENSE file.

package termui

import (
	"fmt"
	"math"
)

// only 16 possible combinations, why bother
var braillePatterns = map[[2]int]rune{
	[2]int{0, 0}: '⣀',
	[2]int{0, 1}: '⡠',
	[2]int{0, 2}: '⡐',
	[2]int{0, 3}: '⡈',

	[2]int{1, 0}: '⢄',
	[2]int{1, 1}: '⠤',
	[2]int{1, 2}: '⠔',
	[2]int{1, 3}: '⠌',

	[2]int{2, 0}: '⢂',
	[2]int{2, 1}: '⠢',
	[2]int{2, 2}: '⠒',
	[2]int{2, 3}: '⠊',

	[2]int{3, 0}: '⢁',
	[2]int{3, 1}: '⠡',
	[2]int{3, 2}: '⠑',
	[2]int{3, 3}: '⠉',
}

var lSingleBraille = [4]rune{'\u2840', '⠄', '⠂', '⠁'}
var rSingleBraille = [4]rune{'\u2880', '⠠', '⠐', '⠈'}

// LineChart has two modes: braille(default) and dot. Using braille gives 2x capicity as dot mode,
// because one braille char can represent two data points.
/*
  lc := termui.NewLineChart()
  lc.Border.Label = "braille-mode Line Chart"
  lc.Data = [1.2, 1.3, 1.5, 1.7, 1.5, 1.6, 1.8, 2.0]
  lc.Width = 50
  lc.Height = 12
  lc.AxesColor = termui.ColorWhite
  lc.LineColor = termui.ColorGreen | termui.AttrBold
  // termui.Render(lc)...
*/
type LineChart struct {
	Block
	Data          []float64
	DataLabels    []string // if unset, the data indices will be used
	Mode          string   // braille | dot
	DotStyle      rune
	LineColor     Attribute
	scale         float64 // data span per cell on y-axis
	AxesColor     Attribute
	drawingX      int
	drawingY      int
	axisYHeight   int
	axisXWidth    int
	axisYLabelGap int
	axisXLabelGap int
	topValue      float64
	bottomValue   float64
	labelX        [][]rune
	labelY        [][]rune
	labelYSpace   int
	maxY          float64
	minY          float64
}

// NewLineChart returns a new LineChart with current theme.
func NewLineChart() *LineChart {
	lc := &LineChart{Block: *NewBlock()}
	lc.AxesColor = ThemeAttr("linechart.axes.fg")
	lc.LineColor = ThemeAttr("linechart.line.fg")
	lc.Mode = "braille"
	lc.DotStyle = '•'
	lc.axisXLabelGap = 2
	lc.axisYLabelGap = 1
	lc.bottomValue = math.Inf(1)
	lc.topValue = math.Inf(-1)
	return lc
}

// one cell contains two data points
// so the capicity is 2x as dot-mode
func (lc *LineChart) renderBraille() Buffer {
	buf := NewBuffer()

	// return: b -> which cell should the point be in
	//         m -> in the cell, divided into 4 equal height levels, which subcell?
	getPos := func(d float64) (b, m int) {
		cnt4 := int((d-lc.bottomValue)/(lc.scale/4) + 0.5)
		b = cnt4 / 4
		m = cnt4 % 4
		return
	}
	// plot points
	for i := 0; 2*i+1 < len(lc.Data) && i < lc.axisXWidth; i++ {
		b0, m0 := getPos(lc.Data[2*i])
		b1, m1 := getPos(lc.Data[2*i+1])

		if b0 == b1 {
			c := Cell{
				Ch: braillePatterns[[2]int{m0, m1}],
				Bg: lc.Bg,
				Fg: lc.LineColor,
			}
			y := lc.innerArea.Min.Y + lc.innerArea.Dy() - 3 - b0
			x := lc.innerArea.Min.X + lc.labelYSpace + 1 + i
			buf.Set(x, y, c)
		} else {
			c0 := Cell{Ch: lSingleBraille[m0],
				Fg: lc.LineColor,
				Bg: lc.Bg}
			x0 := lc.innerArea.Min.X + lc.labelYSpace + 1 + i
			y0 := lc.innerArea.Min.Y + lc.innerArea.Dy() - 3 - b0
			buf.Set(x0, y0, c0)

			c1 := Cell{Ch: rSingleBraille[m1],
				Fg: lc.LineColor,
				Bg: lc.Bg}
			x1 := lc.innerArea.Min.X + lc.labelYSpace + 1 + i
			y1 := lc.innerArea.Min.Y + lc.innerArea.Dy() - 3 - b1
			buf.Set(x1, y1, c1)
		}

	}
	return buf
}

func (lc *LineChart) renderDot() Buffer {
	buf := NewBuffer()
	for i := 0; i < len(lc.Data) && i < lc.axisXWidth; i++ {
		c := Cell{
			Ch: lc.DotStyle,
			Fg: lc.LineColor,
			Bg: lc.Bg,
		}
		x := lc.innerArea.Min.X + lc.labelYSpace + 1 + i
		y := lc.innerArea.Min.Y + lc.innerArea.Dy() - 3 - int((lc.Data[i]-lc.bottomValue)/lc.scale+0.5)
		buf.Set(x, y, c)
	}

	return buf
}

func (lc *LineChart) calcLabelX() {
	lc.labelX = [][]rune{}

	for i, l := 0, 0; i < len(lc.DataLabels) && l < lc.axisXWidth; i++ {
		if lc.Mode == "dot" {
			if l >= len(lc.DataLabels) {
				break
			}

			s := str2runes(lc.DataLabels[l])
			w := strWidth(lc.DataLabels[l])
			if l+w <= lc.axisXWidth {
				lc.labelX = append(lc.labelX, s)
			}
			l += w + lc.axisXLabelGap
		} else { // braille
			if 2*l >= len(lc.DataLabels) {
				break
			}

			s := str2runes(lc.DataLabels[2*l])
			w := strWidth(lc.DataLabels[2*l])
			if l+w <= lc.axisXWidth {
				lc.labelX = append(lc.labelX, s)
			}
			l += w + lc.axisXLabelGap

		}
	}
}

func shortenFloatVal(x float64) string {
	s := fmt.Sprintf("%.2f", x)
	if len(s)-3 > 3 {
		s = fmt.Sprintf("%.2e", x)
	}

	if x < 0 {
		s = fmt.Sprintf("%.2f", x)
	}
	return s
}

func (lc *LineChart) calcLabelY() {
	span := lc.topValue - lc.bottomValue
	lc.scale = span / float64(lc.axisYHeight)

	n := (1 + lc.axisYHeight) / (lc.axisYLabelGap + 1)
	lc.labelY = make([][]rune, n)
	maxLen := 0
	for i := 0; i < n; i++ {
		s := str2runes(shortenFloatVal(lc.bottomValue + float64(i)*span/float64(n)))
		if len(s) > maxLen {
			maxLen = len(s)
		}
		lc.labelY[i] = s
	}

	lc.labelYSpace = maxLen
}

func (lc *LineChart) calcLayout() {
	// set datalabels if it is not provided
	if lc.DataLabels == nil || len(lc.DataLabels) == 0 {
		lc.DataLabels = make([]string, len(lc.Data))
		for i := range lc.Data {
			lc.DataLabels[i] = fmt.Sprint(i)
		}
	}

	// lazy increase, to avoid y shaking frequently
	// update bound Y when drawing is gonna overflow
	lc.minY = lc.Data[0]
	lc.maxY = lc.Data[0]

	lc.bottomValue = lc.minY
	lc.topValue = lc.maxY

	// valid visible range
	vrange := lc.innerArea.Dx()
	if lc.Mode == "braille" {
		vrange = 2 * lc.innerArea.Dx()
	}
	if vrange > len(lc.Data) {
		vrange = len(lc.Data)
	}

	for _, v := range lc.Data[:vrange] {
		if v > lc.maxY {
			lc.maxY = v
		}
		if v < lc.minY {
			lc.minY = v
		}
	}

	span := lc.maxY - lc.minY

	if lc.minY < lc.bottomValue {
		lc.bottomValue = lc.minY - 0.2*span
	}

	if lc.maxY > lc.topValue {
		lc.topValue = lc.maxY + 0.2*span
	}

	lc.axisYHeight = lc.innerArea.Dy() - 2
	lc.calcLabelY()

	lc.axisXWidth = lc.innerArea.Dx() - 1 - lc.labelYSpace
	lc.calcLabelX()

	lc.drawingX = lc.innerArea.Min.X + 1 + lc.labelYSpace
	lc.drawingY = lc.innerArea.Min.Y
}

func (lc *LineChart) plotAxes() Buffer {
	buf := NewBuffer()

	origY := lc.innerArea.Min.Y + lc.innerArea.Dy() - 2
	origX := lc.innerArea.Min.X + lc.labelYSpace

	buf.Set(origX, origY, Cell{Ch: ORIGIN, Fg: lc.AxesColor, Bg: lc.Bg})

	for x := origX + 1; x < origX+lc.axisXWidth; x++ {
		buf.Set(x, origY, Cell{Ch: HDASH, Fg: lc.AxesColor, Bg: lc.Bg})
	}

	for dy := 1; dy <= lc.axisYHeight; dy++ {
		buf.Set(origX, origY-dy, Cell{Ch: VDASH, Fg: lc.AxesColor, Bg: lc.Bg})
	}

	// x label
	oft := 0
	for _, rs := range lc.labelX {
		if oft+len(rs) > lc.axisXWidth {
			break
		}
		for j, r := range rs {
			c := Cell{
				Ch: r,
				Fg: lc.AxesColor,
				Bg: lc.Bg,
			}
			x := origX + oft + j
			y := lc.innerArea.Min.Y + lc.innerArea.Dy() - 1
			buf.Set(x, y, c)
		}
		oft += len(rs) + lc.axisXLabelGap
	}

	// y labels
	for i, rs := range lc.labelY {
		for j, r := range rs {
			buf.Set(
				lc.innerArea.Min.X+j,
				origY-i*(lc.axisYLabelGap+1),
				Cell{Ch: r, Fg: lc.AxesColor, Bg: lc.Bg})
		}
	}

	return buf
}

// Buffer implements Bufferer interface.
func (lc *LineChart) Buffer() Buffer {
	buf := lc.Block.Buffer()

	if lc.Data == nil || len(lc.Data) == 0 {
		return buf
	}
	lc.calcLayout()
	buf.Merge(lc.plotAxes())

	if lc.Mode == "dot" {
		buf.Merge(lc.renderDot())
	} else {
		buf.Merge(lc.renderBraille())
	}

	return buf
}
<|endoftext|>


<commit_before>package test

import (
	"encoding/xml"
	"github.com/masci/flick-rsync/flickr"
	"io/ioutil"
)

type LoginResponse struct {
	flickr.FlickrResponse
	User struct {
		XMLName  xml.Name `xml:"user"`
		ID       string   `xml:"id,attr"`
		Username string   `xml:"username"`
	}
}

func Login(client *flickr.FlickrClient, token *flickr.OAuthToken) (*LoginResponse, error) {
	client.EndpointUrl = flickr.API_ENDPOINT

	client.SetDefaultArgs()
	client.Args.Set("method", "flickr.test.login")
	client.Args.Set("oauth_token", token.OAuthToken)
	client.Args.Set("oauth_consumer_key", client.ApiKey)

	client.Sign(token.OAuthTokenSecret)

	res, err := client.HTTPClient.Get(client.GetUrl())
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	loginResponse := LoginResponse{}
	err = xml.Unmarshal([]byte(body), &loginResponse)
	if err != nil {
		return nil, err
	}

	// TODO parse flickr errors

	return &loginResponse, nil
}
<commit_msg>no more need to pass in the oauth token<commit_after>package test

import (
	"encoding/xml"
	"github.com/masci/flick-rsync/flickr"
	"io/ioutil"
)

type LoginResponse struct {
	flickr.FlickrResponse
	User struct {
		XMLName  xml.Name `xml:"user"`
		ID       string   `xml:"id,attr"`
		Username string   `xml:"username"`
	}
}

func Login(client *flickr.FlickrClient) (*LoginResponse, error) {
	client.EndpointUrl = flickr.API_ENDPOINT

	client.SetDefaultArgs()
	client.Args.Set("method", "flickr.test.login")
	client.Args.Set("oauth_token", client.OAuthToken)
	client.Args.Set("oauth_consumer_key", client.ApiKey)

	client.Sign(client.OAuthTokenSecret)

	res, err := client.HTTPClient.Get(client.GetUrl())
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	loginResponse := LoginResponse{}
	err = xml.Unmarshal([]byte(body), &loginResponse)
	if err != nil {
		return nil, err
	}

	// TODO parse flickr errors

	return &loginResponse, nil
}
<|endoftext|>


<commit_before>package sched

import (
	"bytes"
	"log"
	"time"

	"github.com/StackExchange/tsaf/_third_party/github.com/StackExchange/scollector/collect"
	"github.com/StackExchange/tsaf/_third_party/github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/tsaf/conf"
	"github.com/StackExchange/tsaf/expr"
)

func (s *Schedule) Status(ak AlertKey) *State {
	s.Lock()
	st := s.status[ak]
	s.Unlock()
	return st
}

func (s *Schedule) Check() {
	s.runStates = make(map[AlertKey]Status)
	s.cache = opentsdb.NewCache(s.Conf.TsdbHost, s.Conf.ResponseLimit)
	for _, a := range s.Conf.Alerts {
		s.CheckAlert(a)
	}
	s.CheckUnknown()
	checkNotify := false
	silenced := s.Silenced()
	for ak, status := range s.runStates {
		state := s.Status(ak)
		last := state.Append(status)
		a := s.Conf.Alerts[ak.Name()]
		if status > stNormal {
			var subject = new(bytes.Buffer)
			if err := s.ExecuteSubject(subject, a, state); err != nil {
				log.Println(err)
			}
			state.Subject = subject.String()
			state.Open = true
		}
		// On state increase, clear old notifications and notify current.
		// On state decrease, and if the old alert was already acknowledged, notify current.
		// If the old alert was not acknowledged, do nothing.
		// Do nothing if state did not change.
		notify := func(notifications map[string]*conf.Notification) {
			for _, n := range notifications {
				s.Notify(state, n)
				checkNotify = true
			}
		}
		notifyCurrent := func() {
			state.NeedAck = true
			if _, present := silenced[ak]; present {
				return
			}
			switch status {
			case stCritical, stUnknown:
				notify(a.CritNotification)
			case stWarning:
				notify(a.WarnNotification)
			}
		}
		clearOld := func() {
			state.NeedAck = false
			s.Lock()
			delete(s.Notifications, ak)
			s.Unlock()
		}
		if status > last {
			clearOld()
			notifyCurrent()
		} else if status < last {
			if _, hasOld := s.Notifications[ak]; hasOld {
				notifyCurrent()
			}
		}
	}
	if checkNotify {
		s.nc <- true
	}
	s.Save()
}

func (s *Schedule) CheckUnknown() {
	s.Lock()
	for ak, st := range s.status {
		if st.Forgotten {
			continue
		}
		t := s.Conf.Alerts[ak.Name()].Unknown
		if t == 0 {
			t = s.Conf.Unknown
		}
		if t == 0 {
			continue
		}
		if time.Since(st.Touched) < t {
			continue
		}
		s.runStates[ak] = stUnknown
	}
	s.Unlock()
}

func (s *Schedule) CheckAlert(a *conf.Alert) {
	log.Printf("checking alert %v", a.Name)
	start := time.Now()
	crits := s.CheckExpr(a, a.Crit, stCritical, nil)
	warns := s.CheckExpr(a, a.Warn, stWarning, crits)
	log.Printf("done checking alert %v (%s): %v crits, %v warns", a.Name, time.Since(start), len(crits), len(warns))
}

func (s *Schedule) CheckExpr(a *conf.Alert, e *expr.Expr, checkStatus Status, ignore AlertKeys) (alerts AlertKeys) {
	if e == nil {
		return
	}
	results, _, err := e.Execute(s.cache, nil)
	if err != nil {
		collect.Add("check.errs", opentsdb.TagSet{"metric": a.Name}, 1)
		log.Println(err)
		return
	}
Loop:
	for _, r := range results {
		if a.Squelched(r.Group) {
			continue
		}
		ak := NewAlertKey(a.Name, r.Group)
		for _, v := range ignore {
			if ak == v {
				continue Loop
			}
		}
		state := s.Status(ak)
		if state == nil {
			state = &State{
				Alert: ak.Name(),
				Tags:  r.Group.Tags(),
				Group: r.Group,
			}
			s.Lock()
			s.status[ak] = state
			s.Unlock()
		}
		state.Touch()
		status := checkStatus
		state.Computations = r.Computations
		var n float64
		switch v := r.Value.(type) {
		case expr.Number:
			n = float64(v)
		case expr.Scalar:
			n = float64(v)
		default:
			panic("expected number or scalar")
		}
		if n != 0 {
			state.Expr = e.String()
			alerts = append(alerts, ak)
		} else {
			status = stNormal
		}
		if status > s.runStates[ak] {
			s.runStates[ak] = status
		}
	}
	return
}
<commit_msg>Don't check warnings on crit error<commit_after>package sched

import (
	"bytes"
	"log"
	"time"

	"github.com/StackExchange/tsaf/_third_party/github.com/StackExchange/scollector/collect"
	"github.com/StackExchange/tsaf/_third_party/github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/tsaf/conf"
	"github.com/StackExchange/tsaf/expr"
)

func (s *Schedule) Status(ak AlertKey) *State {
	s.Lock()
	st := s.status[ak]
	s.Unlock()
	return st
}

func (s *Schedule) Check() {
	s.runStates = make(map[AlertKey]Status)
	s.cache = opentsdb.NewCache(s.Conf.TsdbHost, s.Conf.ResponseLimit)
	for _, a := range s.Conf.Alerts {
		s.CheckAlert(a)
	}
	s.CheckUnknown()
	checkNotify := false
	silenced := s.Silenced()
	for ak, status := range s.runStates {
		state := s.Status(ak)
		last := state.Append(status)
		a := s.Conf.Alerts[ak.Name()]
		if status > stNormal {
			var subject = new(bytes.Buffer)
			if err := s.ExecuteSubject(subject, a, state); err != nil {
				log.Println(err)
			}
			state.Subject = subject.String()
			state.Open = true
		}
		// On state increase, clear old notifications and notify current.
		// On state decrease, and if the old alert was already acknowledged, notify current.
		// If the old alert was not acknowledged, do nothing.
		// Do nothing if state did not change.
		notify := func(notifications map[string]*conf.Notification) {
			for _, n := range notifications {
				s.Notify(state, n)
				checkNotify = true
			}
		}
		notifyCurrent := func() {
			state.NeedAck = true
			if _, present := silenced[ak]; present {
				return
			}
			switch status {
			case stCritical, stUnknown:
				notify(a.CritNotification)
			case stWarning:
				notify(a.WarnNotification)
			}
		}
		clearOld := func() {
			state.NeedAck = false
			s.Lock()
			delete(s.Notifications, ak)
			s.Unlock()
		}
		if status > last {
			clearOld()
			notifyCurrent()
		} else if status < last {
			if _, hasOld := s.Notifications[ak]; hasOld {
				notifyCurrent()
			}
		}
	}
	if checkNotify {
		s.nc <- true
	}
	s.Save()
}

func (s *Schedule) CheckUnknown() {
	s.Lock()
	for ak, st := range s.status {
		if st.Forgotten {
			continue
		}
		t := s.Conf.Alerts[ak.Name()].Unknown
		if t == 0 {
			t = s.Conf.Unknown
		}
		if t == 0 {
			continue
		}
		if time.Since(st.Touched) < t {
			continue
		}
		s.runStates[ak] = stUnknown
	}
	s.Unlock()
}

func (s *Schedule) CheckAlert(a *conf.Alert) {
	log.Printf("checking alert %v", a.Name)
	start := time.Now()
	crits, err := s.CheckExpr(a, a.Crit, stCritical, nil)
	var warns AlertKeys
	if err == nil {
		warns, _ = s.CheckExpr(a, a.Warn, stWarning, crits)
	}
	log.Printf("done checking alert %v (%s): %v crits, %v warns", a.Name, time.Since(start), len(crits), len(warns))
}

func (s *Schedule) CheckExpr(a *conf.Alert, e *expr.Expr, checkStatus Status, ignore AlertKeys) (alerts AlertKeys, err error) {
	if e == nil {
		return
	}
	defer func() {
		if err == nil {
			return
		}
		collect.Add("check.errs", opentsdb.TagSet{"metric": a.Name}, 1)
		log.Println(err)
	}()
	results, _, err := e.Execute(s.cache, nil)
	if err != nil {
		return
	}
Loop:
	for _, r := range results {
		if a.Squelched(r.Group) {
			continue
		}
		ak := NewAlertKey(a.Name, r.Group)
		for _, v := range ignore {
			if ak == v {
				continue Loop
			}
		}
		state := s.Status(ak)
		if state == nil {
			state = &State{
				Alert: ak.Name(),
				Tags:  r.Group.Tags(),
				Group: r.Group,
			}
			s.Lock()
			s.status[ak] = state
			s.Unlock()
		}
		state.Touch()
		status := checkStatus
		state.Computations = r.Computations
		var n float64
		switch v := r.Value.(type) {
		case expr.Number:
			n = float64(v)
		case expr.Scalar:
			n = float64(v)
		default:
			panic("expected number or scalar")
		}
		if n != 0 {
			state.Expr = e.String()
			alerts = append(alerts, ak)
		} else {
			status = stNormal
		}
		if status > s.runStates[ak] {
			s.runStates[ak] = status
		}
	}
	return
}
<|endoftext|>