<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package time_test

import (
	"fmt"
	"os"
	"reflect"
	"testing"
	"time"
)

func init() {
	if time.ZoneinfoForTesting() != nil {
		panic(fmt.Errorf("zoneinfo initialized before first LoadLocation"))
	}
}

func TestEnvVarUsage(t *testing.T) {
	time.ResetZoneinfoForTesting()

	const testZoneinfo = "foo.zip"
	const env = "ZONEINFO"

	defer os.Setenv(env, os.Getenv(env))
	os.Setenv(env, testZoneinfo)

	// Result isn't important, we're testing the side effect of this command
	time.LoadLocation("Asia/Jerusalem")
	defer time.ResetZoneinfoForTesting()

	if zoneinfo := time.ZoneinfoForTesting(); testZoneinfo != *zoneinfo {
		t.Errorf("zoneinfo does not match env variable: got %q want %q", zoneinfo, testZoneinfo)
	}
}

func TestLoadLocationValidatesNames(t *testing.T) {
	time.ResetZoneinfoForTesting()
	const env = "ZONEINFO"
	defer os.Setenv(env, os.Getenv(env))
	os.Setenv(env, "")

	bad := []string{
		"/usr/foo/Foo",
		"\\UNC\foo",
		"..",
		"a..",
	}
	for _, v := range bad {
		_, err := time.LoadLocation(v)
		if err != time.ErrLocation {
			t.Errorf("LoadLocation(%q) error = %v; want ErrLocation", v, err)
		}
	}
}

func TestVersion3(t *testing.T) {
	time.ForceZipFileForTesting(true)
	defer time.ForceZipFileForTesting(false)
	_, err := time.LoadLocation("Asia/Jerusalem")
	if err != nil {
		t.Fatal(err)
	}
}

// Test that we get the correct results for times before the first
// transition time. To do this we explicitly check early dates in a
// couple of specific timezones.
func TestFirstZone(t *testing.T) {
	time.ForceZipFileForTesting(true)
	defer time.ForceZipFileForTesting(false)

	const format = "Mon, 02 Jan 2006 15:04:05 -0700 (MST)"
	var tests = []struct {
		zone  string
		unix  int64
		want1 string
		want2 string
	}{
		{
			"PST8PDT",
			-1633269601,
			"Sun, 31 Mar 1918 01:59:59 -0800 (PST)",
			"Sun, 31 Mar 1918 03:00:00 -0700 (PDT)",
		},
		{
			"Pacific/Fakaofo",
			1325242799,
			"Thu, 29 Dec 2011 23:59:59 -1100 (-11)",
			"Sat, 31 Dec 2011 00:00:00 +1300 (+13)",
		},
	}

	for _, test := range tests {
		z, err := time.LoadLocation(test.zone)
		if err != nil {
			t.Fatal(err)
		}
		s := time.Unix(test.unix, 0).In(z).Format(format)
		if s != test.want1 {
			t.Errorf("for %s %d got %q want %q", test.zone, test.unix, s, test.want1)
		}
		s = time.Unix(test.unix+1, 0).In(z).Format(format)
		if s != test.want2 {
			t.Errorf("for %s %d got %q want %q", test.zone, test.unix, s, test.want2)
		}
	}
}

func TestLocationNames(t *testing.T) {
	if time.Local.String() != "Local" {
		t.Errorf(`invalid Local location name: got %q want "Local"`, time.Local)
	}
	if time.UTC.String() != "UTC" {
		t.Errorf(`invalid UTC location name: got %q want "UTC"`, time.UTC)
	}
}

func TestLoadLocationFromTzinfo(t *testing.T) {
	time.ForceZipFileForTesting(true)
	defer time.ForceZipFileForTesting(false)

	const locationName = "Asia/Jerusalem"
	reference, err := time.LoadLocation(locationName)
	if err != nil {
		t.Fatal(err)
	}

	tzinfo, err := time.LoadTzinfo(locationName, time.OrigZoneSources[len(time.OrigZoneSources)-1])
	if err != nil {
		t.Fatal(err)
	}
	sample, err := time.LoadLocationFromTZData(locationName, tzinfo)
	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(reference, sample) {
		t.Errorf("return values of LoadLocationFromTZData and LoadLocation don't match")
	}
}
<commit_msg>time: rename TestLoadLocationFromTzinfo to TestLoadLocationFromTZData<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package time_test

import (
	"fmt"
	"os"
	"reflect"
	"testing"
	"time"
)

func init() {
	if time.ZoneinfoForTesting() != nil {
		panic(fmt.Errorf("zoneinfo initialized before first LoadLocation"))
	}
}

func TestEnvVarUsage(t *testing.T) {
	time.ResetZoneinfoForTesting()

	const testZoneinfo = "foo.zip"
	const env = "ZONEINFO"

	defer os.Setenv(env, os.Getenv(env))
	os.Setenv(env, testZoneinfo)

	// Result isn't important, we're testing the side effect of this command
	time.LoadLocation("Asia/Jerusalem")
	defer time.ResetZoneinfoForTesting()

	if zoneinfo := time.ZoneinfoForTesting(); testZoneinfo != *zoneinfo {
		t.Errorf("zoneinfo does not match env variable: got %q want %q", zoneinfo, testZoneinfo)
	}
}

func TestLoadLocationValidatesNames(t *testing.T) {
	time.ResetZoneinfoForTesting()
	const env = "ZONEINFO"
	defer os.Setenv(env, os.Getenv(env))
	os.Setenv(env, "")

	bad := []string{
		"/usr/foo/Foo",
		"\\UNC\foo",
		"..",
		"a..",
	}
	for _, v := range bad {
		_, err := time.LoadLocation(v)
		if err != time.ErrLocation {
			t.Errorf("LoadLocation(%q) error = %v; want ErrLocation", v, err)
		}
	}
}

func TestVersion3(t *testing.T) {
	time.ForceZipFileForTesting(true)
	defer time.ForceZipFileForTesting(false)
	_, err := time.LoadLocation("Asia/Jerusalem")
	if err != nil {
		t.Fatal(err)
	}
}

// Test that we get the correct results for times before the first
// transition time. To do this we explicitly check early dates in a
// couple of specific timezones.
func TestFirstZone(t *testing.T) {
	time.ForceZipFileForTesting(true)
	defer time.ForceZipFileForTesting(false)

	const format = "Mon, 02 Jan 2006 15:04:05 -0700 (MST)"
	var tests = []struct {
		zone  string
		unix  int64
		want1 string
		want2 string
	}{
		{
			"PST8PDT",
			-1633269601,
			"Sun, 31 Mar 1918 01:59:59 -0800 (PST)",
			"Sun, 31 Mar 1918 03:00:00 -0700 (PDT)",
		},
		{
			"Pacific/Fakaofo",
			1325242799,
			"Thu, 29 Dec 2011 23:59:59 -1100 (-11)",
			"Sat, 31 Dec 2011 00:00:00 +1300 (+13)",
		},
	}

	for _, test := range tests {
		z, err := time.LoadLocation(test.zone)
		if err != nil {
			t.Fatal(err)
		}
		s := time.Unix(test.unix, 0).In(z).Format(format)
		if s != test.want1 {
			t.Errorf("for %s %d got %q want %q", test.zone, test.unix, s, test.want1)
		}
		s = time.Unix(test.unix+1, 0).In(z).Format(format)
		if s != test.want2 {
			t.Errorf("for %s %d got %q want %q", test.zone, test.unix, s, test.want2)
		}
	}
}

func TestLocationNames(t *testing.T) {
	if time.Local.String() != "Local" {
		t.Errorf(`invalid Local location name: got %q want "Local"`, time.Local)
	}
	if time.UTC.String() != "UTC" {
		t.Errorf(`invalid UTC location name: got %q want "UTC"`, time.UTC)
	}
}

func TestLoadLocationFromTZData(t *testing.T) {
	time.ForceZipFileForTesting(true)
	defer time.ForceZipFileForTesting(false)

	const locationName = "Asia/Jerusalem"
	reference, err := time.LoadLocation(locationName)
	if err != nil {
		t.Fatal(err)
	}

	tzinfo, err := time.LoadTzinfo(locationName, time.OrigZoneSources[len(time.OrigZoneSources)-1])
	if err != nil {
		t.Fatal(err)
	}
	sample, err := time.LoadLocationFromTZData(locationName, tzinfo)
	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(reference, sample) {
		t.Errorf("return values of LoadLocationFromTZData and LoadLocation don't match")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"os/exec"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestSystem(t *testing.T) {

	testCases := []struct {
		name          string
		command       []string
		expected      string
		errorExpected bool
	}{
		{
			name:          "Shows usage if no option is defined",
			command:       []string{""},
			expected:      usage + "\n",
			errorExpected: true,
		},
		{
			name:          "Shows usage if option is '-h'",
			command:       []string{"-h"},
			expected:      intro + usage + options + "\n",
			errorExpected: false,
		},
		{
			name:          "Shows report from bank records file",
			command:       []string{"report", "--input", "./tests/fixtures/sample_records_load.csv"},
			expected:      "77.52 COMPRA CONTINENTE MAI \n95.09 COMPRA FARMACIA SAO J ",
			errorExpected: false,
		},
		{
			name:          "Create category",
			command:       []string{"category", "new", "ThisIsACategoryNameForTesting"},
			expected:      "Created category 'ThisIsACategoryNameForTesting'",
			errorExpected: false,
		},
		{
			name:     "Show Category",
			command:  []string{"category", "show"},
			expected: "ThisIsACategoryNameForTesting\n",
		},
		{
			name:     "Loads records from file",
			command:  []string{"load", "--input", "./tests/fixtures/sample_records_load.csv"},
			expected: "",
		},
		{
			name:     "Shows sellers loaded by the run report",
			command:  []string{"seller", "show"},
			expected: "COMPRA CONTINENTE MAI \nCOMPRA FARMACIA SAO J \n",
		},
		{
			name:     "Adds pretty name to seller",
			command:  []string{"seller", "change", "COMPRA CONTINENTE MAI ", "--pretty", "Continente"},
			expected: "",
		},
		{
			name:     "Show seller changed",
			command:  []string{"seller", "show"},
			expected: "Continente\nCOMPRA FARMACIA SAO J \n",
		},
		{
			name:          "Shows report from bank records file, with sellers name instead of slug",
			command:       []string{"report", "--input", "./tests/fixtures/sample_records_load.csv"},
			expected:      "77.52 Continente\n95.09 COMPRA FARMACIA SAO J ",
			errorExpected: false,
		},
	}

	for _, tc := range testCases {
		cmd := exec.Command("../../bankcli", tc.command...)
		stdoutStderr, err := cmd.CombinedOutput()

		if tc.errorExpected {
			assert.Error(t, err)
		} else {
			assert.NoError(t, err)
		}
		assert.Equal(t, tc.expected, string(stdoutStderr), tc.name)
	}

	// Remove any test files
	if err := os.RemoveAll(DatabasePath + "/" + DatabaseName + ".db"); err != nil {
		t.Error(err)
	}
}
<commit_msg>New system test just to validate system status<commit_after>package main

import (
	"os"
	"os/exec"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestSystem(t *testing.T) {

	testCases := []struct {
		name          string
		command       []string
		expected      string
		errorExpected bool
	}{
		{
			name:          "Shows usage if no option is defined",
			command:       []string{""},
			expected:      usage + "\n",
			errorExpected: true,
		},
		{
			name:          "Shows usage if option is '-h'",
			command:       []string{"-h"},
			expected:      intro + usage + options + "\n",
			errorExpected: false,
		},
		{
			name:          "Shows report from bank records file",
			command:       []string{"report", "--input", "./tests/fixtures/sample_records_load.csv"},
			expected:      "77.52 COMPRA CONTINENTE MAI \n95.09 COMPRA FARMACIA SAO J ",
			errorExpected: false,
		},
		{
			name:     "No sellers should be available here",
			command:  []string{"seller", "show"},
			expected: "",
		},
		{
			name:          "Create category",
			command:       []string{"category", "new", "ThisIsACategoryNameForTesting"},
			expected:      "Created category 'ThisIsACategoryNameForTesting'",
			errorExpected: false,
		},
		{
			name:     "Show Category",
			command:  []string{"category", "show"},
			expected: "ThisIsACategoryNameForTesting\n",
		},
		{
			name:     "Loads records from file",
			command:  []string{"load", "--input", "./tests/fixtures/sample_records_load.csv"},
			expected: "",
		},
		{
			name:     "Shows sellers loaded by the run report",
			command:  []string{"seller", "show"},
			expected: "COMPRA CONTINENTE MAI \nCOMPRA FARMACIA SAO J \n",
		},
		{
			name:     "Adds pretty name to seller",
			command:  []string{"seller", "change", "COMPRA CONTINENTE MAI ", "--pretty", "Continente"},
			expected: "",
		},
		{
			name:     "Show seller changed",
			command:  []string{"seller", "show"},
			expected: "Continente\nCOMPRA FARMACIA SAO J \n",
		},
		{
			name:          "Shows report from bank records file, with sellers name instead of slug",
			command:       []string{"report", "--input", "./tests/fixtures/sample_records_load.csv"},
			expected:      "77.52 Continente\n95.09 COMPRA FARMACIA SAO J ",
			errorExpected: false,
		},
	}

	for _, tc := range testCases {
		cmd := exec.Command("../../bankcli", tc.command...)
		stdoutStderr, err := cmd.CombinedOutput()

		if tc.errorExpected {
			assert.Error(t, err)
		} else {
			assert.NoError(t, err)
		}
		assert.Equal(t, tc.expected, string(stdoutStderr), tc.name)
	}

	// Remove any test files
	if err := os.RemoveAll(DatabasePath + "/" + DatabaseName + ".db"); err != nil {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>package disk

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"sync"
	"time"

	log "github.com/funkygao/log4go"
)

// queue is a bounded, disk-backed, append-only type that combines queue and
// log semantics.
// key/value byte slices can be appended and read back in order through
// cursor.
//
// Internally, the queue writes key/value byte slices to multiple segment files so
// that disk space can be reclaimed. When a segment file is larger than
// the max segment size, a new file is created.   Segments are removed
// after cursor has advanced past the last entry.  The first
// segment is the head, and the last segment is the tail.  Reads are from
// the head segment and writes tail segment.
//
// queues can have a max size configured such that when the size of all
// segments on disk exceeds the size, write will fail.
//
// ┌─────┐
// │head │
// ├─────┘
// │
// ▼
// ┌─────────────────┐ ┌─────────────────┐┌─────────────────┐
// │segment 1 - 10MB │ │segment 2 - 10MB ││segment 3 - 10MB │
// └─────────────────┘ └─────────────────┘└─────────────────┘
//                          ▲                               ▲
//                          │                               │
//                          │                               │
//                       ┌───────┐                     ┌─────┐
//                       │cursor │                     │tail │
//                       └───────┘                     └─────┘
type queue struct {
	mu sync.RWMutex
	wg sync.WaitGroup

	dir          string // Directory to create segments
	clusterTopic clusterTopic

	// The maximum size in bytes of a segment file before a new one should be created
	maxSegmentSize int64

	// The maximum size allowed in bytes of all segments before writes will return an error
	// -1 means unlimited
	maxSize int64

	purgeInterval time.Duration

	cursor     *cursor
	head, tail *segment
	segments   segments

	quit          chan struct{}
	emptyInflight bool // FIXME
}

// newQueue create a queue that will store segments in dir and that will
// consume more than maxSize on disk.
func newQueue(ct clusterTopic, dir string, maxSize int64, purgeInterval time.Duration) *queue {
	q := &queue{
		clusterTopic:   ct,
		dir:            dir,
		quit:           make(chan struct{}),
		maxSegmentSize: defaultSegmentSize,
		maxSize:        maxSize,
		purgeInterval:  purgeInterval,
		segments:       segments{},
	}
	q.cursor = newCursor(q)
	return q
}

// Open opens the queue for reading and writing
func (l *queue) Open() error {
	l.mu.Lock()
	defer l.mu.Unlock()

	if err := mkdirIfNotExist(l.dir); err != nil {
		return err
	}

	segments, err := l.loadSegments()
	if err != nil {
		return err
	}
	l.segments = segments

	if len(l.segments) == 0 {
		// create the 1st segment
		if _, err = l.addSegment(); err != nil {
			return err
		}
	}

	l.head = l.segments[0]
	l.tail = l.segments[len(l.segments)-1]

	// cursor open must be placed below queue open
	if err = l.cursor.open(); err != nil {
		return err
	}

	l.wg.Add(1)
	go l.housekeeping()

	l.wg.Add(1)
	go l.pump()

	return nil
}

// Close stops the queue for reading and writing
func (l *queue) Close() error {
	l.mu.Lock()
	defer l.mu.Unlock()

	for _, s := range l.segments {
		if err := s.Close(); err != nil {
			return err
		}
	}

	l.head = nil
	l.tail = nil
	l.segments = nil
	close(l.quit)

	l.wg.Wait()
	if err := l.cursor.dump(); err != nil {
		return err
	}
	l.cursor = nil
	return nil
}

// Remove removes all underlying file-based resources for the queue.
// It is an error to call this on an open queue.
func (l *queue) Remove() error {
	l.mu.Lock()
	defer l.mu.Unlock()

	if l.head != nil || l.tail != nil || l.segments != nil {
		return ErrQueueOpen
	}

	return os.RemoveAll(l.dir)
}

// Purge garbage collects the segments that are behind cursor.
func (l *queue) Purge() error {
	l.mu.Lock()
	defer l.mu.Unlock()

	if len(l.segments) <= 1 {
		// head, curror, tail are in the same segment
		return nil
	}

	for {
		if l.cursor.pos.SegmentID > l.head.id {
			l.trimHead()
		} else {
			return nil
		}

	}
}

// LastModified returns the last time the queue was modified.
func (l *queue) LastModified() (time.Time, error) {
	l.mu.RLock()
	defer l.mu.RUnlock()

	if l.tail != nil {
		return l.tail.LastModified()
	}
	return time.Time{}.UTC(), nil
}

// Append appends a block to the end of the queue
func (l *queue) Append(b *block) error {
	l.mu.Lock()
	defer l.mu.Unlock()

	if l.tail == nil {
		return ErrNotOpen
	}

	if l.maxSize > 0 && l.diskUsage()+b.size() > l.maxSize {
		return ErrQueueFull
	}

	// Append the entry to the tail, if the segment is full,
	// try to create new segment and retry the append
	if err := l.tail.Append(b); err == ErrSegmentFull {
		segment, err := l.addSegment()
		if err != nil {
			return err
		}

		l.tail = segment
		return l.tail.Append(b)
	} else if err != nil {
		return err
	}
	return nil
}

func (q *queue) Next(b *block) (err error) {
	q.mu.RLock()
	defer q.mu.RUnlock()

	c := q.cursor
	if c == nil {
		return ErrNotOpen
	}
	err = c.seg.ReadOne(b)
	switch err {
	case nil:
		c.advanceOffset(b.size())
		return

	case io.EOF:
		// cursor might have:
		// 1. reached end of the current segment: will advance to next segment
		// 2. reached end of tail
		if ok := c.advanceSegment(); !ok {
			return ErrEOQ
		}

		// advanced to next segment, read one block
		err = c.seg.ReadOne(b)
		switch err {
		case nil:
			// bingo!
			c.advanceOffset(b.size())
			return

		case io.EOF:
			// tail is empty
			return ErrEOQ

		default:
			return
		}

	default:
		return
	}
}

func (l *queue) EmptyInflight() bool {
	return l.emptyInflight
}

// diskUsage returns the total size on disk used by the queue
func (l *queue) diskUsage() int64 {
	var size int64
	for _, s := range l.segments {
		size += s.DiskUsage()
	}
	return size
}

// loadSegments loads all segments on disk
func (l *queue) loadSegments() (segments, error) {
	segments := []*segment{}

	files, err := ioutil.ReadDir(l.dir)
	if err != nil {
		return segments, err
	}

	for _, segment := range files {
		if segment.IsDir() || segment.Name() == cursorFile {
			continue
		}

		// Segments file names are all numeric
		id, err := strconv.ParseUint(segment.Name(), 10, 64)
		if err != nil {
			continue
		}

		segment, err := newSegment(id, filepath.Join(l.dir, segment.Name()), l.maxSegmentSize)
		if err != nil {
			return segments, err
		}

		segments = append(segments, segment)
	}
	return segments, nil
}

// addSegment creates a new empty segment file
// caller is responsible for the lock
func (l *queue) addSegment() (*segment, error) {
	nextID, err := l.nextSegmentID()
	if err != nil {
		return nil, err
	}

	path := filepath.Join(l.dir, fmt.Sprintf("%020d", nextID))
	segment, err := newSegment(nextID, path, l.maxSegmentSize)
	if err != nil {
		return nil, err
	}

	l.segments = append(l.segments, segment)
	return segment, nil
}

// nextSegmentID returns the next segment ID that is free
func (l *queue) nextSegmentID() (uint64, error) {
	segments, err := ioutil.ReadDir(l.dir)
	if err != nil {
		return 0, err
	}

	var maxID uint64
	for _, segment := range segments {
		// Segments should be files.  Skip anything that is not a dir.
		if segment.IsDir() {
			continue
		}

		// Segments file names are all numeric
		segmentID, err := strconv.ParseUint(segment.Name(), 10, 64)
		if err != nil {
			log.Warn("unexpected segment file: %s", segment.Name())
			continue
		}

		if segmentID > maxID {
			maxID = segmentID
		}
	}

	return maxID + 1, nil
}

func (l *queue) trimHead() (err error) {
	l.segments = l.segments[1:]

	if err = l.head.Remove(); err != nil {
		return
	}

	l.head = l.segments[0]
	return
}

func (l *queue) nextDir() string {
	// find least loaded dir
	return ""
}
<commit_msg>fix dead lock issue<commit_after>package disk

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"sync"
	"time"

	log "github.com/funkygao/log4go"
)

// queue is a bounded, disk-backed, append-only type that combines queue and
// log semantics.
// key/value byte slices can be appended and read back in order through
// cursor.
//
// Internally, the queue writes key/value byte slices to multiple segment files so
// that disk space can be reclaimed. When a segment file is larger than
// the max segment size, a new file is created.   Segments are removed
// after cursor has advanced past the last entry.  The first
// segment is the head, and the last segment is the tail.  Reads are from
// the head segment and writes tail segment.
//
// queues can have a max size configured such that when the size of all
// segments on disk exceeds the size, write will fail.
//
// ┌─────┐
// │head │
// ├─────┘
// │
// ▼
// ┌─────────────────┐ ┌─────────────────┐┌─────────────────┐
// │segment 1 - 10MB │ │segment 2 - 10MB ││segment 3 - 10MB │
// └─────────────────┘ └─────────────────┘└─────────────────┘
//                          ▲                               ▲
//                          │                               │
//                          │                               │
//                       ┌───────┐                     ┌─────┐
//                       │cursor │                     │tail │
//                       └───────┘                     └─────┘
type queue struct {
	mu sync.RWMutex
	wg sync.WaitGroup

	dir          string // Directory to create segments
	clusterTopic clusterTopic

	// The maximum size in bytes of a segment file before a new one should be created
	maxSegmentSize int64

	// The maximum size allowed in bytes of all segments before writes will return an error
	// -1 means unlimited
	maxSize int64

	purgeInterval time.Duration

	cursor     *cursor
	head, tail *segment
	segments   segments

	quit          chan struct{}
	emptyInflight bool // FIXME
}

// newQueue create a queue that will store segments in dir and that will
// consume more than maxSize on disk.
func newQueue(ct clusterTopic, dir string, maxSize int64, purgeInterval time.Duration) *queue {
	q := &queue{
		clusterTopic:   ct,
		dir:            dir,
		quit:           make(chan struct{}),
		maxSegmentSize: defaultSegmentSize,
		maxSize:        maxSize,
		purgeInterval:  purgeInterval,
		segments:       segments{},
	}
	q.cursor = newCursor(q)
	return q
}

// Open opens the queue for reading and writing
func (l *queue) Open() error {
	l.mu.Lock()
	defer l.mu.Unlock()

	if err := mkdirIfNotExist(l.dir); err != nil {
		return err
	}

	segments, err := l.loadSegments()
	if err != nil {
		return err
	}
	l.segments = segments

	if len(l.segments) == 0 {
		// create the 1st segment
		if _, err = l.addSegment(); err != nil {
			return err
		}
	}

	l.head = l.segments[0]
	l.tail = l.segments[len(l.segments)-1]

	// cursor open must be placed below queue open
	if err = l.cursor.open(); err != nil {
		return err
	}

	l.wg.Add(1)
	go l.housekeeping()

	l.wg.Add(1)
	go l.pump()

	return nil
}

// Close stops the queue for reading and writing
func (l *queue) Close() error {
	close(l.quit)

	l.mu.Lock()
	defer l.mu.Unlock()

	for _, s := range l.segments {
		if err := s.Close(); err != nil {
			return err
		}
	}

	l.head = nil
	l.tail = nil
	l.segments = nil

	l.wg.Wait()
	if err := l.cursor.dump(); err != nil {
		return err
	}
	l.cursor = nil
	return nil
}

// Remove removes all underlying file-based resources for the queue.
// It is an error to call this on an open queue.
func (l *queue) Remove() error {
	l.mu.Lock()
	defer l.mu.Unlock()

	if l.head != nil || l.tail != nil || l.segments != nil {
		return ErrQueueOpen
	}

	return os.RemoveAll(l.dir)
}

// Purge garbage collects the segments that are behind cursor.
func (l *queue) Purge() error {
	l.mu.Lock()
	defer l.mu.Unlock()

	if len(l.segments) <= 1 {
		// head, curror, tail are in the same segment
		return nil
	}

	for {
		if l.cursor.pos.SegmentID > l.head.id {
			l.trimHead()
		} else {
			return nil
		}

	}
}

// LastModified returns the last time the queue was modified.
func (l *queue) LastModified() (time.Time, error) {
	l.mu.RLock()
	defer l.mu.RUnlock()

	if l.tail != nil {
		return l.tail.LastModified()
	}
	return time.Time{}.UTC(), nil
}

// Append appends a block to the end of the queue
func (l *queue) Append(b *block) error {
	l.mu.Lock()
	defer l.mu.Unlock()

	if l.tail == nil {
		return ErrNotOpen
	}

	if l.maxSize > 0 && l.diskUsage()+b.size() > l.maxSize {
		return ErrQueueFull
	}

	// Append the entry to the tail, if the segment is full,
	// try to create new segment and retry the append
	if err := l.tail.Append(b); err == ErrSegmentFull {
		segment, err := l.addSegment()
		if err != nil {
			return err
		}

		l.tail = segment
		return l.tail.Append(b)
	} else if err != nil {
		return err
	}
	return nil
}

func (q *queue) Next(b *block) (err error) {
	q.mu.RLock()
	defer q.mu.RUnlock()

	c := q.cursor
	if c == nil {
		return ErrNotOpen
	}
	err = c.seg.ReadOne(b)
	switch err {
	case nil:
		c.advanceOffset(b.size())
		return

	case io.EOF:
		// cursor might have:
		// 1. reached end of the current segment: will advance to next segment
		// 2. reached end of tail
		if ok := c.advanceSegment(); !ok {
			return ErrEOQ
		}

		// advanced to next segment, read one block
		err = c.seg.ReadOne(b)
		switch err {
		case nil:
			// bingo!
			c.advanceOffset(b.size())
			return

		case io.EOF:
			// tail is empty
			return ErrEOQ

		default:
			return
		}

	default:
		return
	}
}

func (l *queue) EmptyInflight() bool {
	return l.emptyInflight
}

// diskUsage returns the total size on disk used by the queue
func (l *queue) diskUsage() int64 {
	var size int64
	for _, s := range l.segments {
		size += s.DiskUsage()
	}
	return size
}

// loadSegments loads all segments on disk
func (l *queue) loadSegments() (segments, error) {
	segments := []*segment{}

	files, err := ioutil.ReadDir(l.dir)
	if err != nil {
		return segments, err
	}

	for _, segment := range files {
		if segment.IsDir() || segment.Name() == cursorFile {
			continue
		}

		// Segments file names are all numeric
		id, err := strconv.ParseUint(segment.Name(), 10, 64)
		if err != nil {
			continue
		}

		segment, err := newSegment(id, filepath.Join(l.dir, segment.Name()), l.maxSegmentSize)
		if err != nil {
			return segments, err
		}

		segments = append(segments, segment)
	}
	return segments, nil
}

// addSegment creates a new empty segment file
// caller is responsible for the lock
func (l *queue) addSegment() (*segment, error) {
	nextID, err := l.nextSegmentID()
	if err != nil {
		return nil, err
	}

	path := filepath.Join(l.dir, fmt.Sprintf("%020d", nextID))
	segment, err := newSegment(nextID, path, l.maxSegmentSize)
	if err != nil {
		return nil, err
	}

	l.segments = append(l.segments, segment)
	return segment, nil
}

// nextSegmentID returns the next segment ID that is free
func (l *queue) nextSegmentID() (uint64, error) {
	segments, err := ioutil.ReadDir(l.dir)
	if err != nil {
		return 0, err
	}

	var maxID uint64
	for _, segment := range segments {
		// Segments should be files.  Skip anything that is not a dir.
		if segment.IsDir() {
			continue
		}

		// Segments file names are all numeric
		segmentID, err := strconv.ParseUint(segment.Name(), 10, 64)
		if err != nil {
			log.Warn("unexpected segment file: %s", segment.Name())
			continue
		}

		if segmentID > maxID {
			maxID = segmentID
		}
	}

	return maxID + 1, nil
}

func (l *queue) trimHead() (err error) {
	l.segments = l.segments[1:]

	if err = l.head.Remove(); err != nil {
		return
	}

	l.head = l.segments[0]
	return
}

func (l *queue) nextDir() string {
	// find least loaded dir
	return ""
}
<|endoftext|>


<commit_before>package auth

import (
	"crypto/rsa"
	"fmt"

	jwt "github.com/dgrijalva/jwt-go"
)

const (
	errorMessageMalformed     = "token malformed"
	errorMessageExpired       = "token expired or not yet valid"
	errorMessageInvalid       = "invalid token"
	errorMessageClaimsInvalid = "invalid token claims"
)

// ConsumerFor derives the Consumer from the JWT claims
func ConsumerFor(token *jwt.Token) (*Consumer, error) {
	if claims, ok := token.Claims.(*JWTClaims); ok {
		return &claims.Consumer, nil
	}
	return nil, fmt.Errorf("cannot assert claims for type %T", token.Claims)
}

// JWTClaims represents the claims within the JWT.
type JWTClaims struct {
	Consumer Consumer `json:"consumer"`
	jwt.StandardClaims
}

// ParseJWT parses a JWT string and checks its signature validity
func ParseJWT(pk *rsa.PublicKey, raw string) (*jwt.Token, error) {
	// Parse the JWT token
	token, err := jwt.ParseWithClaims(raw, &JWTClaims{}, checkSignatureFunc(pk))

	// Bail out if the token could not be parsed
	if err != nil {
		if _, ok := err.(*jwt.ValidationError); ok {
			// Handle any token specific errors
			var errorMessage string
			if err.(*jwt.ValidationError).Errors&jwt.ValidationErrorMalformed != 0 {
				errorMessage = errorMessageMalformed
			} else if err.(*jwt.ValidationError).Errors&(jwt.ValidationErrorExpired|jwt.ValidationErrorNotValidYet) != 0 {
				errorMessage = errorMessageExpired
			} else {
				errorMessage = errorMessageInvalid
			}
			return nil, fmt.Errorf(errorMessage)
		}
		return nil, fmt.Errorf(errorMessageInvalid)
	}

	// Check the claims and token are valid
	if _, ok := token.Claims.(*JWTClaims); !ok || !token.Valid {
		return nil, fmt.Errorf(errorMessageClaimsInvalid)
	}

	return token, nil
}

func checkSignatureFunc(pk *rsa.PublicKey) func(t *jwt.Token) (interface{}, error) {
	return func(t *jwt.Token) (interface{}, error) {
		// Ensure the signing method was not changed
		if _, ok := t.Method.(*jwt.SigningMethodRSA); !ok {
			return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"])
		}
		return pk, nil
	}
}
<commit_msg>refactor: checkSignatureFunc now uses new error types<commit_after>package auth

import (
	"crypto/rsa"
	"fmt"

	jwt "github.com/dgrijalva/jwt-go"
)

const (
	errorMessageMalformed     = "token malformed"
	errorMessageExpired       = "token expired or not yet valid"
	errorMessageInvalid       = "invalid token"
	errorMessageClaimsInvalid = "invalid token claims"
)

// ConsumerFor derives the Consumer from the JWT claims
func ConsumerFor(token *jwt.Token) (*Consumer, error) {
	if claims, ok := token.Claims.(*JWTClaims); ok {
		return &claims.Consumer, nil
	}
	return nil, fmt.Errorf("cannot assert claims for type %T", token.Claims)
}

// JWTClaims represents the claims within the JWT.
type JWTClaims struct {
	Consumer Consumer `json:"consumer"`
	jwt.StandardClaims
}

// ParseJWT parses a JWT string and checks its signature validity
func ParseJWT(pk *rsa.PublicKey, raw string) (*jwt.Token, error) {
	// Parse the JWT token
	token, err := jwt.ParseWithClaims(raw, &JWTClaims{}, checkSignatureFunc(pk))

	// Bail out if the token could not be parsed
	if err != nil {
		if _, ok := err.(*jwt.ValidationError); ok {
			// Handle any token specific errors
			var errorMessage string
			if err.(*jwt.ValidationError).Errors&jwt.ValidationErrorMalformed != 0 {
				errorMessage = errorMessageMalformed
			} else if err.(*jwt.ValidationError).Errors&(jwt.ValidationErrorExpired|jwt.ValidationErrorNotValidYet) != 0 {
				errorMessage = errorMessageExpired
			} else {
				errorMessage = errorMessageInvalid
			}
			return nil, fmt.Errorf(errorMessage)
		}
		return nil, fmt.Errorf(errorMessageInvalid)
	}

	// Check the claims and token are valid
	if _, ok := token.Claims.(*JWTClaims); !ok || !token.Valid {
		return nil, fmt.Errorf(errorMessageClaimsInvalid)
	}

	return token, nil
}

func checkSignatureFunc(pk *rsa.PublicKey) func(token *jwt.Token) (interface{}, error) {
	return func(token *jwt.Token) (interface{}, error) {
		// Ensure the signing method was not changed
		if _, ok := token.Method.(*jwt.SigningMethodRSA); !ok {
			return nil, &ErrUnexpectedSigningMethod{token.Header["alg"]}
		}
		return pk, nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/gocql/gocql"
	"github.com/raintank/metrictank/cluster"
	"github.com/raintank/metrictank/idx/cassandra"
	"github.com/raintank/worldping-api/pkg/log"
	"gopkg.in/raintank/schema.v1"
)

var (
	dryRun          = flag.Bool("dry-run", true, "run in dry-run mode. No changes will be made.")
	logLevel        = flag.Int("log-level", 2, "log level. 0=TRACE|1=DEBUG|2=INFO|3=WARN|4=ERROR|5=CRITICAL|6=FATAL")
	srcCassAddr     = flag.String("src-cass-addr", "localhost", "Address of cassandra host to migrate from.")
	dstCassAddr     = flag.String("dst-cass-addr", "localhost", "Address of cassandra host to migrate to.")
	srcKeyspace     = flag.String("src-keyspace", "raintank", "Cassandra keyspace in use on source.")
	dstKeyspace     = flag.String("dst-keyspace", "raintank", "Cassandra keyspace in use on destination.")
	partitionScheme = flag.String("partition-scheme", "byOrg", "method used for partitioning metrics. (byOrg|bySeries)")
	numPartitions   = flag.Int("num-partitions", 1, "number of partitions in cluster")

	wg sync.WaitGroup
)

func main() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "mt-index-migrate")
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "Migrate metric index from one cassandra keyspace to another.")
		fmt.Fprintln(os.Stderr, "This tool can be used for moving data to a different keyspace or cassandra cluster")
		fmt.Fprintln(os.Stderr, "or for resetting partition information when the number of paritions being used has changed.")
		fmt.Fprintf(os.Stderr, "\nFlags:\n\n")
		flag.PrintDefaults()
	}
	flag.Parse()
	log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":false}`, *logLevel))

	defsChan := make(chan *schema.MetricDefinition, 100)

	srcCluster := gocql.NewCluster(*srcCassAddr)
	srcCluster.Consistency = gocql.ParseConsistency("one")
	srcCluster.Timeout = time.Second
	srcCluster.NumConns = 2
	srcCluster.ProtoVersion = 4
	srcCluster.Keyspace = *srcKeyspace
	srcSession, err := srcCluster.CreateSession()
	if err != nil {
		log.Fatal(4, "failed to create cql session for source cassandra. %s", err)
	}
	dstCluster := gocql.NewCluster(*dstCassAddr)
	dstCluster.Consistency = gocql.ParseConsistency("one")
	dstCluster.Timeout = time.Second
	dstCluster.NumConns = 2
	dstCluster.ProtoVersion = 4
	dstCluster.Keyspace = *dstKeyspace
	dstSession, err := dstCluster.CreateSession()
	if err != nil {
		log.Fatal(4, "failed to create cql session for destination cassandra. %s", err)
	}

	// ensure the dest table exists.
	err = dstSession.Query(fmt.Sprintf(cassandra.TableSchema, *dstKeyspace)).Exec()
	if err != nil {
		log.Fatal(4, "cassandra-idx failed to initialize cassandra table. %s", err)
	}

	wg.Add(1)
	go writeDefs(dstSession, defsChan)
	wg.Add(1)
	go getDefs(srcSession, defsChan)

	wg.Wait()

}

func writeDefs(session *gocql.Session, defsChan chan *schema.MetricDefinition) {
	log.Info("starting write thread")
	defer wg.Done()
	counter := 0
	pre := time.Now()
	for def := range defsChan {
		qry := `INSERT INTO metric_idx (id, orgid, partition, name, metric, interval, unit, mtype, tags, lastupdate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
		if *dryRun {
			fmt.Printf(
				"INSERT INTO metric_idx (id, orgid, partition, name, metric, interval, unit, mtype, tags, lastupdate) VALUES ('%s', '%d', '%d','%s', '%s','%d', '%s','%s', '%v', '%d')\n",
				def.Id,
				def.OrgId,
				def.Partition,
				def.Name,
				def.Metric,
				def.Interval,
				def.Unit,
				def.Mtype,
				def.Tags,
				def.LastUpdate)
			continue
		}
		success := false
		attempts := 0
		for !success {
			if err := session.Query(
				qry,
				def.Id,
				def.OrgId,
				def.Partition,
				def.Name,
				def.Metric,
				def.Interval,
				def.Unit,
				def.Mtype,
				def.Tags,
				def.LastUpdate).Exec(); err != nil {

				if (attempts % 20) == 0 {
					log.Warn("cassandra-idx Failed to write def to cassandra. it will be retried. %s", err)
				}
				sleepTime := 100 * attempts
				if sleepTime > 2000 {
					sleepTime = 2000
				}
				time.Sleep(time.Duration(sleepTime) * time.Millisecond)
				attempts++
			} else {
				success = true
				log.Debug("cassandra-idx metricDef saved to cassandra. %s", def.Id)
				counter++
			}
		}
	}
	log.Info("Inserted %d metricDefs in %s", counter, time.Since(pre).String())
}

func getDefs(session *gocql.Session, defsChan chan *schema.MetricDefinition) {
	log.Info("starting read thread")
	defer wg.Done()
	defer close(defsChan)
	partitioner, err := cluster.NewKafkaPartitioner(*partitionScheme)
	if err != nil {
		log.Fatal(4, "failed to initialize partitioner. %s", err)
	}
	iter := session.Query("SELECT id, orgid, partition, name, metric, interval, unit, mtype, tags, lastupdate from metric_idx").Iter()

	var id, name, metric, unit, mtype string
	var orgId, interval int
	var partition int32
	var lastupdate int64
	var tags []string
	for iter.Scan(&id, &orgId, &partition, &name, &metric, &interval, &unit, &mtype, &tags, &lastupdate) {
		mdef := schema.MetricDefinition{
			Id:         id,
			OrgId:      orgId,
			Partition:  partition,
			Name:       name,
			Metric:     metric,
			Interval:   interval,
			Unit:       unit,
			Mtype:      mtype,
			Tags:       tags,
			LastUpdate: lastupdate,
		}
		log.Debug("retrieved %s from old index.", mdef.Id)
		if *numPartitions == 1 {
			mdef.Partition = 0
		} else {
			p, err := partitioner.Partition(&mdef, int32(*numPartitions))
			if err != nil {
				log.Fatal(4, "failed to get partition id of metric. %s", err)
			} else {
				mdef.Partition = p
			}
		}
		defsChan <- &mdef
	}
}
<commit_msg>use new partitioner package<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/gocql/gocql"
	"github.com/raintank/metrictank/cluster/partitioner"
	"github.com/raintank/metrictank/idx/cassandra"
	"github.com/raintank/worldping-api/pkg/log"
	"gopkg.in/raintank/schema.v1"
)

var (
	dryRun          = flag.Bool("dry-run", true, "run in dry-run mode. No changes will be made.")
	logLevel        = flag.Int("log-level", 2, "log level. 0=TRACE|1=DEBUG|2=INFO|3=WARN|4=ERROR|5=CRITICAL|6=FATAL")
	srcCassAddr     = flag.String("src-cass-addr", "localhost", "Address of cassandra host to migrate from.")
	dstCassAddr     = flag.String("dst-cass-addr", "localhost", "Address of cassandra host to migrate to.")
	srcKeyspace     = flag.String("src-keyspace", "raintank", "Cassandra keyspace in use on source.")
	dstKeyspace     = flag.String("dst-keyspace", "raintank", "Cassandra keyspace in use on destination.")
	partitionScheme = flag.String("partition-scheme", "byOrg", "method used for partitioning metrics. (byOrg|bySeries)")
	numPartitions   = flag.Int("num-partitions", 1, "number of partitions in cluster")

	wg sync.WaitGroup
)

func main() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "mt-index-migrate")
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "Migrate metric index from one cassandra keyspace to another.")
		fmt.Fprintln(os.Stderr, "This tool can be used for moving data to a different keyspace or cassandra cluster")
		fmt.Fprintln(os.Stderr, "or for resetting partition information when the number of paritions being used has changed.")
		fmt.Fprintf(os.Stderr, "\nFlags:\n\n")
		flag.PrintDefaults()
	}
	flag.Parse()
	log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":false}`, *logLevel))

	defsChan := make(chan *schema.MetricDefinition, 100)

	srcCluster := gocql.NewCluster(*srcCassAddr)
	srcCluster.Consistency = gocql.ParseConsistency("one")
	srcCluster.Timeout = time.Second
	srcCluster.NumConns = 2
	srcCluster.ProtoVersion = 4
	srcCluster.Keyspace = *srcKeyspace
	srcSession, err := srcCluster.CreateSession()
	if err != nil {
		log.Fatal(4, "failed to create cql session for source cassandra. %s", err)
	}
	dstCluster := gocql.NewCluster(*dstCassAddr)
	dstCluster.Consistency = gocql.ParseConsistency("one")
	dstCluster.Timeout = time.Second
	dstCluster.NumConns = 2
	dstCluster.ProtoVersion = 4
	dstCluster.Keyspace = *dstKeyspace
	dstSession, err := dstCluster.CreateSession()
	if err != nil {
		log.Fatal(4, "failed to create cql session for destination cassandra. %s", err)
	}

	// ensure the dest table exists.
	err = dstSession.Query(fmt.Sprintf(cassandra.TableSchema, *dstKeyspace)).Exec()
	if err != nil {
		log.Fatal(4, "cassandra-idx failed to initialize cassandra table. %s", err)
	}

	wg.Add(1)
	go writeDefs(dstSession, defsChan)
	wg.Add(1)
	go getDefs(srcSession, defsChan)

	wg.Wait()

}

func writeDefs(session *gocql.Session, defsChan chan *schema.MetricDefinition) {
	log.Info("starting write thread")
	defer wg.Done()
	counter := 0
	pre := time.Now()
	for def := range defsChan {
		qry := `INSERT INTO metric_idx (id, orgid, partition, name, metric, interval, unit, mtype, tags, lastupdate) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
		if *dryRun {
			fmt.Printf(
				"INSERT INTO metric_idx (id, orgid, partition, name, metric, interval, unit, mtype, tags, lastupdate) VALUES ('%s', '%d', '%d','%s', '%s','%d', '%s','%s', '%v', '%d')\n",
				def.Id,
				def.OrgId,
				def.Partition,
				def.Name,
				def.Metric,
				def.Interval,
				def.Unit,
				def.Mtype,
				def.Tags,
				def.LastUpdate)
			continue
		}
		success := false
		attempts := 0
		for !success {
			if err := session.Query(
				qry,
				def.Id,
				def.OrgId,
				def.Partition,
				def.Name,
				def.Metric,
				def.Interval,
				def.Unit,
				def.Mtype,
				def.Tags,
				def.LastUpdate).Exec(); err != nil {

				if (attempts % 20) == 0 {
					log.Warn("cassandra-idx Failed to write def to cassandra. it will be retried. %s", err)
				}
				sleepTime := 100 * attempts
				if sleepTime > 2000 {
					sleepTime = 2000
				}
				time.Sleep(time.Duration(sleepTime) * time.Millisecond)
				attempts++
			} else {
				success = true
				log.Debug("cassandra-idx metricDef saved to cassandra. %s", def.Id)
				counter++
			}
		}
	}
	log.Info("Inserted %d metricDefs in %s", counter, time.Since(pre).String())
}

func getDefs(session *gocql.Session, defsChan chan *schema.MetricDefinition) {
	log.Info("starting read thread")
	defer wg.Done()
	defer close(defsChan)
	partitioner, err := partitioner.NewKafka(*partitionScheme)
	if err != nil {
		log.Fatal(4, "failed to initialize partitioner. %s", err)
	}
	iter := session.Query("SELECT id, orgid, partition, name, metric, interval, unit, mtype, tags, lastupdate from metric_idx").Iter()

	var id, name, metric, unit, mtype string
	var orgId, interval int
	var partition int32
	var lastupdate int64
	var tags []string
	for iter.Scan(&id, &orgId, &partition, &name, &metric, &interval, &unit, &mtype, &tags, &lastupdate) {
		mdef := schema.MetricDefinition{
			Id:         id,
			OrgId:      orgId,
			Partition:  partition,
			Name:       name,
			Metric:     metric,
			Interval:   interval,
			Unit:       unit,
			Mtype:      mtype,
			Tags:       tags,
			LastUpdate: lastupdate,
		}
		log.Debug("retrieved %s from old index.", mdef.Id)
		if *numPartitions == 1 {
			mdef.Partition = 0
		} else {
			p, err := partitioner.Partition(&mdef, int32(*numPartitions))
			if err != nil {
				log.Fatal(4, "failed to get partition id of metric. %s", err)
			} else {
				mdef.Partition = p
			}
		}
		defsChan <- &mdef
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package references

import (
	"github.com/spf13/cobra"
)

//Cmd to manage references
var Cmd = &cobra.Command{
	Use:   "ref",
	Aliases: []string{"references"},
	Short: "Manage References",
	Long:  "Manage References. References must refer to a keystore that also exists in the env.",
}

var org, env, name, description, refers, resourceType string

func init() {

	Cmd.PersistentFlags().StringVarP(&org, "org", "o",
		"", "Apigee organization name")

	Cmd.PersistentFlags().StringVarP(&env, "env", "e",
		"", "Apigee environment name")

	_ = Cmd.MarkPersistentFlagRequired("org")
	_ = Cmd.MarkPersistentFlagRequired("env")

	Cmd.AddCommand(ListCmd)
	Cmd.AddCommand(UpdateCmd)
	Cmd.AddCommand(DelCmd)
	Cmd.AddCommand(CreateCmd)
}
<commit_msg>reference cli help<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package references

import (
	"github.com/spf13/cobra"
)

//Cmd to manage references
var Cmd = &cobra.Command{
	Use:   "references",
	Aliases: []string{"refs"},
	Short: "Manage References within an Apigee environment",
	Long:  "Manage References. References must refer to a keystore that also exists in the env.",
}

var org, env, name, description, refers, resourceType string

func init() {

	Cmd.PersistentFlags().StringVarP(&org, "org", "o",
		"", "Apigee organization name")

	Cmd.PersistentFlags().StringVarP(&env, "env", "e",
		"", "Apigee environment name")

	_ = Cmd.MarkPersistentFlagRequired("org")
	_ = Cmd.MarkPersistentFlagRequired("env")

	Cmd.AddCommand(ListCmd)
	Cmd.AddCommand(UpdateCmd)
	Cmd.AddCommand(DelCmd)
	Cmd.AddCommand(CreateCmd)
}
<|endoftext|>


<commit_before>package main_test

import (
	"fmt"
	"net"
	"net/http"
	"os/exec"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

func startSwitchboard(args ...string) ifrit.Process {
	command := exec.Command(switchboardBinPath, args...)
	runner := ginkgomon.New(ginkgomon.Config{
		Command:    command,
		Name:       fmt.Sprintf("switchboard"),
		StartCheck: "started",
	})
	return ginkgomon.Invoke(runner)
}

func startBackend(port uint) ifrit.Process {
	command := exec.Command(dummyBackendBinPath, fmt.Sprintf("-port=%d", port))
	runner := ginkgomon.New(ginkgomon.Config{
		Command:    command,
		Name:       fmt.Sprintf("fake-backend:%d", port),
		StartCheck: "Backend listening on",
	})
	return ginkgomon.Invoke(runner)
}

func startHealthcheck(port uint) ifrit.Process {
	command := exec.Command(dummyHealthcheckBinPath, fmt.Sprintf("-port=%d", port))
	runner := ginkgomon.New(ginkgomon.Config{
		Command:    command,
		Name:       fmt.Sprintf("fake-healthcheck:%d", port),
		StartCheck: "Healthcheck listening on",
	})
	return ginkgomon.Invoke(runner)
}

func sendData(conn net.Conn, data string) (string, error) {
	conn.Write([]byte(data))
	buffer := make([]byte, 1024)
	_, err := conn.Read(buffer)
	if err != nil {
		return "", err
	} else {
		return string(buffer), nil
	}
}

var _ = Describe("Switchboard", func() {
	var (
		backendProcess      ifrit.Process
		backendProcess2     ifrit.Process
		healthcheckProcess  ifrit.Process
		healthcheckProcess2 ifrit.Process
		switchboardProcess  ifrit.Process
	)

	BeforeEach(func() {
		backendProcess = startBackend(backendPort)
		backendProcess2 = startBackend(backendPort2)

		healthcheckProcess = startHealthcheck(dummyHealthcheckPort)
		healthcheckProcess2 = startHealthcheck(dummyHealthcheckPort2)

		switchboardProcess = startSwitchboard(
			fmt.Sprintf("-config=%s", proxyConfigFile),
			fmt.Sprintf("-pidFile=%s", pidFile),
		)
	})

	AfterEach(func() {
		ginkgomon.Kill(switchboardProcess)
		ginkgomon.Kill(healthcheckProcess2)
		ginkgomon.Kill(healthcheckProcess)
		ginkgomon.Kill(backendProcess2)
		ginkgomon.Kill(backendProcess)
	})

	Context("when there are multiple concurrent clients", func() {
		var conn1, conn2, conn3 net.Conn
		var data1, data2, data3 string

		It("proxies all the connections to the backend", func() {
			done1 := make(chan interface{})
			go func() {
				defer GinkgoRecover()
				defer close(done1)

				var err error
				Eventually(func() error {
					conn1, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data1, err = sendData(conn1, "test1")
				Expect(err).ToNot(HaveOccurred())
			}()

			done2 := make(chan interface{})
			go func() {
				defer GinkgoRecover()
				defer close(done2)

				var err error
				Eventually(func() error {
					conn2, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data2, err = sendData(conn2, "test2")
				Expect(err).ToNot(HaveOccurred())
			}()

			done3 := make(chan interface{})
			go func() {
				defer GinkgoRecover()
				defer close(done3)

				var err error
				Eventually(func() error {
					conn3, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data3, err = sendData(conn3, "test3")
				Expect(err).ToNot(HaveOccurred())
			}()

			<-done1
			<-done2
			<-done3

			Expect(data1).Should(ContainSubstring(fmt.Sprintf("Echo from port %d: test1", backendPort)))
			Expect(data2).Should(ContainSubstring(fmt.Sprintf("Echo from port %d: test2", backendPort)))
			Expect(data3).Should(ContainSubstring(fmt.Sprintf("Echo from port %d: test3", backendPort)))
		})
	})

	Context("when other clients disconnect", func() {
		var conn net.Conn
		var connToDisconnect net.Conn

		It("maintains a long-lived connection when other clients disconnect", func() {
			Eventually(func() error {
				var err error
				conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
				return err
			}).ShouldNot(HaveOccurred())

			Eventually(func() error {
				var err error
				connToDisconnect, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
				return err
			}).ShouldNot(HaveOccurred())

			dataBeforeDisconnect, err := sendData(conn, "data before disconnect")
			Expect(err).ToNot(HaveOccurred())
			Expect(dataBeforeDisconnect).Should(ContainSubstring("data before disconnect"))

			connToDisconnect.Close()

			dataAfterDisconnect, err := sendData(conn, "data after disconnect")
			Expect(err).ToNot(HaveOccurred())
			Expect(dataAfterDisconnect).Should(ContainSubstring("data after disconnect"))
		})
	})

	Context("when the healthcheck succeeds", func() {
		var client net.Conn

		It("checks health again after the specified interval", func() {
			Eventually(func() error {
				var err error
				client, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
				return err
			}).ShouldNot(HaveOccurred())

			buffer := make([]byte, 1024)

			client.Write([]byte("data around first healthcheck"))
			n, err := client.Read(buffer)
			Expect(err).ToNot(HaveOccurred())
			Expect(string(buffer[:n])).Should(ContainSubstring("data around first healthcheck"))

			Consistently(func() error {
				client.Write([]byte("data around subsequent healthcheck"))
				_, err = client.Read(buffer)
				return err
			}, 3*time.Second).ShouldNot(HaveOccurred())
		})
	})

	Context("when the cluster is down", func() {
		Context("when the healthcheck reports a 503", func() {
			It("disconnects client connections", func() {
				var conn net.Conn
				Eventually(func() error {
					var err error
					conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				dataWhileHealthy, err := sendData(conn, "data while healthy")
				Expect(err).ToNot(HaveOccurred())
				Expect(dataWhileHealthy).Should(ContainSubstring("data while healthy"))

				resp, httpErr := http.Get(fmt.Sprintf("http://localhost:%d/set503", dummyHealthcheckPort))
				Expect(httpErr).NotTo(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(http.StatusOK))

				resp, httpErr = http.Get(fmt.Sprintf("http://localhost:%d/", dummyHealthcheckPort))
				Expect(resp.StatusCode).To(Equal(http.StatusServiceUnavailable))
				Expect(httpErr).NotTo(HaveOccurred())

				Eventually(func() error {
					_, err := sendData(conn, "data when unhealthy")
					return err
				}, 2*time.Second).Should(HaveOccurred())
			})
		})

		Context("when a backend goes down", func() {
			var conn net.Conn
			var data string

			BeforeEach(func() {
				Eventually(func() (err error) {
					conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data, err := sendData(conn, "data before hang")
				Expect(err).ShouldNot(HaveOccurred())
				Expect(data).Should(ContainSubstring("data before hang"))

				resp, httpErr := http.Get(fmt.Sprintf("http://localhost:%d/setHang", dummyHealthcheckPort))
				Expect(httpErr).NotTo(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(http.StatusOK))
			})

			It("disconnects existing client connections", func(done Done) {
				defer close(done)

				Eventually(func() error {
					_, err := sendData(conn, "data after hang")
					return err
				}, proxyConfig.HealthcheckTimeout()*4).Should(HaveOccurred())
			}, 5)

			It("proxies new connections to another backend", func(done Done) {
				defer close(done)

				time.Sleep(3 * proxyConfig.HealthcheckTimeout()) // wait for failover

				var err error
				Eventually(func() error {
					conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data, err = sendData(conn, "test")
				Expect(err).ToNot(HaveOccurred())
				Expect(data).Should(ContainSubstring(fmt.Sprintf("Echo from port %d: test", backendPort2)))
			}, 5)
		})

		Context("when all backends are down", func() {
			BeforeEach(func() {
				resp, httpErr := http.Get(fmt.Sprintf("http://localhost:%d/setHang", dummyHealthcheckPort))
				Expect(httpErr).NotTo(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(http.StatusOK))

				resp, httpErr = http.Get(fmt.Sprintf("http://localhost:%d/setHang", dummyHealthcheckPort2))
				Expect(httpErr).NotTo(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(http.StatusOK))
			})

			It("rejects any new connections that are attempted", func(done Done) {
				defer close(done)

				time.Sleep(3 * proxyConfig.HealthcheckTimeout()) // wait for failover

				var conn net.Conn
				Eventually(func() (err error) {
					conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}, 1*time.Second).ShouldNot(HaveOccurred())

				Eventually(func() error {
					_, err := sendData(conn, "write that should fail")
					return err
				}, proxyConfig.HealthcheckTimeout()*4).Should(HaveOccurred())

			}, 20)
		})
	})
})
<commit_msg>Do no expect particular backend since there are 2<commit_after>package main_test

import (
	"fmt"
	"net"
	"net/http"
	"os/exec"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

func startSwitchboard(args ...string) ifrit.Process {
	command := exec.Command(switchboardBinPath, args...)
	runner := ginkgomon.New(ginkgomon.Config{
		Command:    command,
		Name:       fmt.Sprintf("switchboard"),
		StartCheck: "started",
	})
	return ginkgomon.Invoke(runner)
}

func startBackend(port uint) ifrit.Process {
	command := exec.Command(dummyBackendBinPath, fmt.Sprintf("-port=%d", port))
	runner := ginkgomon.New(ginkgomon.Config{
		Command:    command,
		Name:       fmt.Sprintf("fake-backend:%d", port),
		StartCheck: "Backend listening on",
	})
	return ginkgomon.Invoke(runner)
}

func startHealthcheck(port uint) ifrit.Process {
	command := exec.Command(dummyHealthcheckBinPath, fmt.Sprintf("-port=%d", port))
	runner := ginkgomon.New(ginkgomon.Config{
		Command:    command,
		Name:       fmt.Sprintf("fake-healthcheck:%d", port),
		StartCheck: "Healthcheck listening on",
	})
	return ginkgomon.Invoke(runner)
}

func sendData(conn net.Conn, data string) (string, error) {
	conn.Write([]byte(data))
	buffer := make([]byte, 1024)
	_, err := conn.Read(buffer)
	if err != nil {
		return "", err
	} else {
		return string(buffer), nil
	}
}

var _ = Describe("Switchboard", func() {
	var (
		backendProcess      ifrit.Process
		backendProcess2     ifrit.Process
		healthcheckProcess  ifrit.Process
		healthcheckProcess2 ifrit.Process
		switchboardProcess  ifrit.Process
	)

	BeforeEach(func() {
		backendProcess = startBackend(backendPort)
		backendProcess2 = startBackend(backendPort2)

		healthcheckProcess = startHealthcheck(dummyHealthcheckPort)
		healthcheckProcess2 = startHealthcheck(dummyHealthcheckPort2)

		switchboardProcess = startSwitchboard(
			fmt.Sprintf("-config=%s", proxyConfigFile),
			fmt.Sprintf("-pidFile=%s", pidFile),
		)
	})

	AfterEach(func() {
		ginkgomon.Kill(switchboardProcess)
		ginkgomon.Kill(healthcheckProcess2)
		ginkgomon.Kill(healthcheckProcess)
		ginkgomon.Kill(backendProcess2)
		ginkgomon.Kill(backendProcess)
	})

	Context("when there are multiple concurrent clients", func() {
		var conn1, conn2, conn3 net.Conn
		var data1, data2, data3 string

		It("proxies all the connections to the backend", func() {
			done1 := make(chan interface{})
			go func() {
				defer GinkgoRecover()
				defer close(done1)

				var err error
				Eventually(func() error {
					conn1, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data1, err = sendData(conn1, "test1")
				Expect(err).ToNot(HaveOccurred())
			}()

			done2 := make(chan interface{})
			go func() {
				defer GinkgoRecover()
				defer close(done2)

				var err error
				Eventually(func() error {
					conn2, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data2, err = sendData(conn2, "test2")
				Expect(err).ToNot(HaveOccurred())
			}()

			done3 := make(chan interface{})
			go func() {
				defer GinkgoRecover()
				defer close(done3)

				var err error
				Eventually(func() error {
					conn3, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data3, err = sendData(conn3, "test3")
				Expect(err).ToNot(HaveOccurred())
			}()

			<-done1
			<-done2
			<-done3

			Expect(data1).Should(ContainSubstring("test1"))
			Expect(data2).Should(ContainSubstring("test2"))
			Expect(data3).Should(ContainSubstring("test3"))
		})
	})

	Context("when other clients disconnect", func() {
		var conn net.Conn
		var connToDisconnect net.Conn

		It("maintains a long-lived connection when other clients disconnect", func() {
			Eventually(func() error {
				var err error
				conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
				return err
			}).ShouldNot(HaveOccurred())

			Eventually(func() error {
				var err error
				connToDisconnect, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
				return err
			}).ShouldNot(HaveOccurred())

			dataBeforeDisconnect, err := sendData(conn, "data before disconnect")
			Expect(err).ToNot(HaveOccurred())
			Expect(dataBeforeDisconnect).Should(ContainSubstring("data before disconnect"))

			connToDisconnect.Close()

			dataAfterDisconnect, err := sendData(conn, "data after disconnect")
			Expect(err).ToNot(HaveOccurred())
			Expect(dataAfterDisconnect).Should(ContainSubstring("data after disconnect"))
		})
	})

	Context("when the healthcheck succeeds", func() {
		var client net.Conn

		It("checks health again after the specified interval", func() {
			Eventually(func() error {
				var err error
				client, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
				return err
			}).ShouldNot(HaveOccurred())

			buffer := make([]byte, 1024)

			client.Write([]byte("data around first healthcheck"))
			n, err := client.Read(buffer)
			Expect(err).ToNot(HaveOccurred())
			Expect(string(buffer[:n])).Should(ContainSubstring("data around first healthcheck"))

			Consistently(func() error {
				client.Write([]byte("data around subsequent healthcheck"))
				_, err = client.Read(buffer)
				return err
			}, 3*time.Second).ShouldNot(HaveOccurred())
		})
	})

	Context("when the cluster is down", func() {
		Context("when the healthcheck reports a 503", func() {
			It("disconnects client connections", func() {
				var conn net.Conn
				Eventually(func() error {
					var err error
					conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				dataWhileHealthy, err := sendData(conn, "data while healthy")
				Expect(err).ToNot(HaveOccurred())
				Expect(dataWhileHealthy).Should(ContainSubstring("data while healthy"))

				resp, httpErr := http.Get(fmt.Sprintf("http://localhost:%d/set503", dummyHealthcheckPort))
				Expect(httpErr).NotTo(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(http.StatusOK))

				resp, httpErr = http.Get(fmt.Sprintf("http://localhost:%d/", dummyHealthcheckPort))
				Expect(resp.StatusCode).To(Equal(http.StatusServiceUnavailable))
				Expect(httpErr).NotTo(HaveOccurred())

				Eventually(func() error {
					_, err := sendData(conn, "data when unhealthy")
					return err
				}, 2*time.Second).Should(HaveOccurred())
			})
		})

		Context("when a backend goes down", func() {
			var conn net.Conn
			var data string

			BeforeEach(func() {
				Eventually(func() (err error) {
					conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data, err := sendData(conn, "data before hang")
				Expect(err).ShouldNot(HaveOccurred())
				Expect(data).Should(ContainSubstring("data before hang"))

				resp, httpErr := http.Get(fmt.Sprintf("http://localhost:%d/setHang", dummyHealthcheckPort))
				Expect(httpErr).NotTo(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(http.StatusOK))
			})

			It("disconnects existing client connections", func(done Done) {
				defer close(done)

				Eventually(func() error {
					_, err := sendData(conn, "data after hang")
					return err
				}, proxyConfig.HealthcheckTimeout()*4).Should(HaveOccurred())
			}, 5)

			It("proxies new connections to another backend", func(done Done) {
				defer close(done)

				time.Sleep(3 * proxyConfig.HealthcheckTimeout()) // wait for failover

				var err error
				Eventually(func() error {
					conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data, err = sendData(conn, "test")
				Expect(err).ToNot(HaveOccurred())
				Expect(data).Should(ContainSubstring(fmt.Sprintf("Echo from port %d: test", backendPort2)))
			}, 5)
		})

		Context("when all backends are down", func() {
			BeforeEach(func() {
				resp, httpErr := http.Get(fmt.Sprintf("http://localhost:%d/setHang", dummyHealthcheckPort))
				Expect(httpErr).NotTo(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(http.StatusOK))

				resp, httpErr = http.Get(fmt.Sprintf("http://localhost:%d/setHang", dummyHealthcheckPort2))
				Expect(httpErr).NotTo(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(http.StatusOK))
			})

			It("rejects any new connections that are attempted", func(done Done) {
				defer close(done)

				time.Sleep(3 * proxyConfig.HealthcheckTimeout()) // wait for failover

				var conn net.Conn
				Eventually(func() (err error) {
					conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}, 1*time.Second).ShouldNot(HaveOccurred())

				Eventually(func() error {
					_, err := sendData(conn, "write that should fail")
					return err
				}, proxyConfig.HealthcheckTimeout()*4).Should(HaveOccurred())

			}, 20)
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tsuru

import (
	"bytes"
	"github.com/tsuru/tsuru/cmd"
	tTesting "github.com/tsuru/tsuru/testing"
	"launchpad.net/gocheck"
	"os"
	"testing"
)

type S struct {
	recover []string
}

func (s *S) SetUpSuite(c *gocheck.C) {
	s.recover = tTesting.SetTargetFile(c)
}

func (s *S) TearDownSuite(c *gocheck.C) {
	tTesting.RollbackTargetFile(s.recover)
}

var _ = gocheck.Suite(&S{})
var manager *cmd.Manager

func Test(t *testing.T) { gocheck.TestingT(t) }

func (s *S) SetUpTest(c *gocheck.C) {
	var stdout, stderr bytes.Buffer
	manager = cmd.NewManager("glb", "0.x", "Foo-Tsuru", &stdout, &stderr, os.Stdin, nil)
}
<commit_msg>cmd/tsuru-base: fix call to testing.SetTargetFile<commit_after>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tsuru

import (
	"bytes"
	"github.com/tsuru/tsuru/cmd"
	tTesting "github.com/tsuru/tsuru/testing"
	"launchpad.net/gocheck"
	"os"
	"testing"
)

type S struct {
	recover []string
}

func (s *S) SetUpSuite(c *gocheck.C) {
	s.recover = tTesting.SetTargetFile(c, []byte("http://localhost"))
}

func (s *S) TearDownSuite(c *gocheck.C) {
	tTesting.RollbackTargetFile(s.recover)
}

var _ = gocheck.Suite(&S{})
var manager *cmd.Manager

func Test(t *testing.T) { gocheck.TestingT(t) }

func (s *S) SetUpTest(c *gocheck.C) {
	var stdout, stderr bytes.Buffer
	manager = cmd.NewManager("glb", "0.x", "Foo-Tsuru", &stdout, &stderr, os.Stdin, nil)
}
<|endoftext|>


<commit_before>package treacy

import (
	"github.com/gin-gonic/gin"
	"github.com/onsi/gomega/format"
	"github.com/onsi/gomega/matchers"
	"reflect"
	"runtime"
)

type (
	EngineMock struct {
		running bool
		routes  []RouteInfo
		port    string
	}

	RouteInfo struct {
		Method  string
		Path    string
		Handler string
	}
)

func (e *EngineMock) Run(port string) error {
	e.running, e.port = true, port
	return nil
}

func (e *EngineMock) POST(relativePath string, handlers ...gin.HandlerFunc) {
	for _, handler := range handlers {
		e.routes = append(e.routes, RouteInfo{Method: "POST", Path: relativePath, Handler: funcName(handler)})
	}
}

func funcName(f interface{}) string {
	return runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
}

func (e *EngineMock) IsRunning() bool {
	return e.running
}

func (e *EngineMock) Port() string {
	return e.port
}

func (e *EngineMock) Routes() []RouteInfo {
	return e.routes
}

// BeRunning matcher
func BeRunning() *isRunningMatcher {
	return &isRunningMatcher{}
}

type isRunningMatcher struct{}

func (matcher *isRunningMatcher) Match(actual interface{}) (success bool, err error) {
	return (&matchers.BeTrueMatcher{}).Match(actual.(*EngineMock).IsRunning())
}

func (matcher *isRunningMatcher) FailureMessage(actual interface{}) (message string) {
	return format.Message(actual, "to be running")
}

func (matcher *isRunningMatcher) NegatedFailureMessage(actual interface{}) (message string) {
	return format.Message(actual, "not to be running")
}

// Handle route matcher
func Handle(method string) *routeMatcher {
	return &routeMatcher{Method: method}
}

type routeMatcher struct {
	Method  string
	Path    string
	Handler string
}

func (matcher *routeMatcher) On(path string) *routeMatcher {
	matcher.Path = path
	return matcher
}

func (matcher *routeMatcher) By(handler string) *routeMatcher {
	matcher.Handler = handler
	return matcher
}

func (matcher *routeMatcher) Expected() *RouteInfo {
	return &RouteInfo{
		Method:  matcher.Method,
		Path:    matcher.Path,
		Handler: matcher.Handler}
}

func (matcher *routeMatcher) Match(actual interface{}) (success bool, err error) {
	containElementMatcher := &matchers.ContainElementMatcher{
		Element: RouteInfo{
			Method:  matcher.Method,
			Path:    matcher.Path,
			Handler: matcher.Handler}}
	return (containElementMatcher).Match(actual.(*EngineMock).Routes())
}

func (matcher *routeMatcher) FailureMessage(actual interface{}) (message string) {
	return format.Message(actual.(*EngineMock).Routes(), "to include", matcher.Expected())
}

func (matcher *routeMatcher) NegatedFailureMessage(actual interface{}) (message string) {
	return format.Message(actual.(*EngineMock).Routes(), "not to include", matcher.Expected())
}
<commit_msg>RouteInfo replaced by string in route handler and EngineMock<commit_after>package treacy

import (
	"fmt"
	"github.com/gin-gonic/gin"
	"github.com/onsi/gomega/format"
	"github.com/onsi/gomega/matchers"
	"reflect"
	"runtime"
)

type (
	EngineMock struct {
		running bool
		routes  []string
		port    string
	}
)

func (e *EngineMock) Run(port string) error {
	e.running, e.port = true, port
	return nil
}

func (e *EngineMock) POST(relativePath string, handlers ...gin.HandlerFunc) {
	for _, handler := range handlers {
		e.routes = append(e.routes, fmt.Sprintf("%s %s -> %s", "POST", relativePath, funcName(handler)))
	}
}

func funcName(f interface{}) string {
	return runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
}

func (e *EngineMock) IsRunning() bool {
	return e.running
}

func (e *EngineMock) Port() string {
	return e.port
}

func (e *EngineMock) Routes() []string {
	return e.routes
}

// BeRunning matcher
func BeRunning() *isRunningMatcher {
	return &isRunningMatcher{}
}

type isRunningMatcher struct{}

func (matcher *isRunningMatcher) Match(actual interface{}) (success bool, err error) {
	return (&matchers.BeTrueMatcher{}).Match(actual.(*EngineMock).IsRunning())
}

func (matcher *isRunningMatcher) FailureMessage(actual interface{}) (message string) {
	return format.Message(actual, "to be running")
}

func (matcher *isRunningMatcher) NegatedFailureMessage(actual interface{}) (message string) {
	return format.Message(actual, "not to be running")
}

// Handle route matcher
func Handle(method string) *routeMatcher {
	return &routeMatcher{Method: method}
}

type routeMatcher struct {
	Method  string
	Path    string
	Handler string
}

func (matcher *routeMatcher) On(path string) *routeMatcher {
	matcher.Path = path
	return matcher
}

func (matcher *routeMatcher) By(handler string) *routeMatcher {
	matcher.Handler = handler
	return matcher
}

func (matcher *routeMatcher) ToString() string {
	return fmt.Sprintf("%s %s -> %s", matcher.Method, matcher.Path, matcher.Handler)
}

func (matcher *routeMatcher) Match(actual interface{}) (success bool, err error) {
	containElementMatcher := &matchers.ContainElementMatcher{Element: matcher.ToString()}
	return (containElementMatcher).Match(actual.(*EngineMock).Routes())
}

func (matcher *routeMatcher) FailureMessage(actual interface{}) (message string) {
	return format.Message(actual.(*EngineMock).Routes(), "to include", matcher.ToString())
}

func (matcher *routeMatcher) NegatedFailureMessage(actual interface{}) (message string) {
	return format.Message(actual.(*EngineMock).Routes(), "not to include", matcher.ToString())
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Doctl Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package do

import (
	"fmt"
	"net/url"
	"strconv"
	"sync"

	"github.com/digitalocean/godo"
)

const maxFetchPages = 5

var perPage = 200

var fetchFn = fetchPage

type paginatedList struct {
	list []interface{}
	mu   sync.Mutex
}

func (pl *paginatedList) append(items ...interface{}) {
	pl.mu.Lock()
	defer pl.mu.Unlock()

	pl.list = append(pl.list, items...)
}

// Generator is a function that generates the list to be paginated.
type Generator func(*godo.ListOptions) ([]interface{}, *godo.Response, error)

// PaginateResp paginates a Response.
func PaginateResp(gen Generator) ([]interface{}, error) {
	opt := &godo.ListOptions{Page: 1, PerPage: perPage}

	l := paginatedList{}

	fetchChan := make(chan int, maxFetchPages)

	var wg sync.WaitGroup
	for i := 0; i < maxFetchPages-1; i++ {
		wg.Add(1)
		go func() {
			for page := range fetchChan {
				items, err := fetchFn(gen, page)
				if err == nil {
					l.append(items...)
				}
			}
			wg.Done()
		}()
	}

	// fetch first page to get page count (x)
	items, resp, err := gen(opt)
	if err != nil {
		return nil, err
	}

	l.append(items...)

	// find last page
	lp, err := lastPage(resp)
	if err != nil {
		return nil, err
	}

	// start with second page
	opt.Page++
	for ; opt.Page <= lp; opt.Page++ {
		fetchChan <- opt.Page
	}
	close(fetchChan)

	wg.Wait()

	return l.list, nil
}

func fetchPage(gen Generator, page int) ([]interface{}, error) {
	opt := &godo.ListOptions{Page: page, PerPage: perPage}
	items, _, err := gen(opt)
	return items, err
}

func lastPage(resp *godo.Response) (int, error) {
	if resp.Links == nil || resp.Links.Pages == nil {
		// no other pages
		return 1, nil
	}

	uStr := resp.Links.Pages.Last
	if uStr == "" {
		return 1, nil
	}

	u, err := url.Parse(uStr)
	if err != nil {
		return 0, fmt.Errorf("could not parse last page: %v", err)
	}

	pageStr := u.Query().Get("page")
	page, err := strconv.Atoi(pageStr)
	if err != nil {
		return 0, fmt.Errorf("could not find page param: %v", err)
	}

	return page, err
}
<commit_msg>pagination: updated pagination to give deterministic and natural ordering (#1070)<commit_after>/*
Copyright 2018 The Doctl Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package do

import (
	"fmt"
	"net/url"
	"strconv"
	"sync"

	"github.com/digitalocean/godo"
)

const maxFetchPages = 5

var perPage = 200

var fetchFn = fetchPage

type paginatedList struct {
	list  [][]interface{}
	total int
	mu    sync.Mutex
}

func (pl *paginatedList) set(page int, items []interface{}) {
	pl.mu.Lock()
	defer pl.mu.Unlock()
	pl.total += len(items)
	pl.list[page-1] = items
}

// Generator is a function that generates the list to be paginated.
type Generator func(*godo.ListOptions) ([]interface{}, *godo.Response, error)

// PaginateResp paginates a Response.
func PaginateResp(gen Generator) ([]interface{}, error) {
	opt := &godo.ListOptions{Page: 1, PerPage: perPage}

	// fetch first page to get page count (x)
	firstPage, resp, err := gen(opt)
	if err != nil {
		return nil, err
	}

	// find last page
	lp, err := lastPage(resp)
	if err != nil {
		return nil, err
	}

	l := paginatedList{
		list: make([][]interface{}, lp),
	}

	// set results from the first page
	l.set(1, firstPage)

	fetchChan := make(chan int, maxFetchPages)

	var wg sync.WaitGroup
	for i := 0; i < maxFetchPages-1; i++ {
		wg.Add(1)
		go func() {
			for page := range fetchChan {
				items, err := fetchFn(gen, page)
				if err == nil {
					l.set(page, items)
				}
			}
			wg.Done()
		}()
	}

	// start with second page
	opt.Page++
	for ; opt.Page <= lp; opt.Page++ {
		fetchChan <- opt.Page
	}
	close(fetchChan)

	wg.Wait()

	// flatten paginated list
	items := make([]interface{}, l.total)[:0]
	for _, page := range l.list {
		if page == nil {
			// must have been an error getting page results
			continue
		}
		for _, item := range page {
			items = append(items, item)
		}
	}

	return items, nil
}

func fetchPage(gen Generator, page int) ([]interface{}, error) {
	opt := &godo.ListOptions{Page: page, PerPage: perPage}
	items, _, err := gen(opt)
	return items, err
}

func lastPage(resp *godo.Response) (int, error) {
	if resp.Links == nil || resp.Links.Pages == nil {
		// no other pages
		return 1, nil
	}

	uStr := resp.Links.Pages.Last
	if uStr == "" {
		return 1, nil
	}

	u, err := url.Parse(uStr)
	if err != nil {
		return 0, fmt.Errorf("could not parse last page: %v", err)
	}

	pageStr := u.Query().Get("page")
	page, err := strconv.Atoi(pageStr)
	if err != nil {
		return 0, fmt.Errorf("could not find page param: %v", err)
	}

	return page, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strings"

	"github.com/Smartling/api-sdk-go"
	"github.com/reconquest/hierr-go"
)

func doFilesPush(
	client *smartling.Client,
	config Config,
	args map[string]interface{},
) error {
	var (
		project       = config.ProjectID
		file, _       = args["<file>"].(string)
		uri, useURI   = args["<uri>"].(string)
		branch, _     = args["--branch"].(string)
		locales, _    = args["--locale"].([]string)
		authorize     = args["--authorize"].(bool)
		directory     = args["--directory"].(string)
		fileType, _   = args["--type"].(string)
		directives, _ = args["--directive"].([]string)
	)

	if branch == "@auto" {
		var err error

		branch, err = getGitBranch()
		if err != nil {
			return hierr.Errorf(
				err,
				"unable to autodetect branch name",
			)
		}

		logger.Infof("autodetected branch name: %s", branch)
	}

	if branch != "" {
		branch = strings.TrimSuffix(branch, "/") + "/"
	}

	patterns := []string{}

	if file != "" {
		patterns = append(patterns, file)
	} else {
		for pattern, section := range config.Files {
			if section.Push.Type != "" {
				patterns = append(patterns, pattern)
			}
		}
	}

	files := []string{}

	for _, pattern := range patterns {
		base, pattern := getDirectoryFromPattern(pattern)
		chunk, err := globFilesLocally(
			directory,
			base,
			pattern,
		)
		if err != nil {
			return NewError(
				hierr.Errorf(
					err,
					`unable to find matching files to upload`,
				),

				`Check, that specified pattern is valid and refer to help for`+
					` more information about glob patterns.`,
			)
		}

		files = append(files, chunk...)
	}

	if len(files) == 0 {
		return NewError(
			fmt.Errorf(`no files found by specified patterns`),

			`Check command line pattern if any and configuration file for`+
				` more patterns to search for.`,
		)
	}

	if uri != "" && len(files) > 1 {
		return NewError(
			fmt.Errorf(
				`more than one file is matching speciifed pattern and <uri>`+
					` is specified too`,
			),

			`Either remove <uri> argument or make sure that only one file`+
				` is matching mask.`,
		)
	}

	for _, file := range files {
		if !useURI {
			uri = file
		}

		fileConfig, err := config.GetFileConfig(file)
		if err != nil {
			return NewError(
				hierr.Errorf(
					err,
					`unable to retrieve file specific configuration`,
				),

				``,
			)
		}

		contents, err := ioutil.ReadFile(file)
		if err != nil {
			return NewError(
				hierr.Errorf(
					err,
					`unable to read file contents "%s"`,
					file,
				),

				`Check that file exists and readable by current user.`,
			)
		}

		request := smartling.FileUploadRequest{
			File:               contents,
			Authorize:          authorize,
			LocalesToAuthorize: locales,
		}

		request.FileURI = branch + uri

		if fileConfig.Push.Type == "" {
			if fileType == "" {
				request.FileType = smartling.GetFileTypeByExtension(
					filepath.Ext(file),
				)

				if request.FileType == smartling.FileTypeUnknown {
					return NewError(
						fmt.Errorf(
							"unable to deduce file type from extension: %q",
							filepath.Ext(file),
						),

						`You need to specify file type via --type option.`,
					)
				}
			} else {
				request.FileType = smartling.FileType(fileType)
			}
		} else {
			request.FileType = smartling.FileType(fileConfig.Push.Type)
		}

		request.Smartling.Directives = fileConfig.Push.Directives

		for _, directive := range directives {
			spec := strings.SplitN(directive, "=", 2)
			if len(spec) != 2 {
				return NewError(
					fmt.Errorf(
						"invalid directive specification: %q",
						directive,
					),

					`Should be in the form of <name>=<value>.`,
				)
			}

			if request.Smartling.Directives == nil {
				request.Smartling.Directives = map[string]string{}
			}

			request.Smartling.Directives[spec[0]] = spec[1]
		}

		response, err := client.UploadFile(project, request)

		if err != nil {
			return NewError(
				hierr.Errorf(
					err,
					`unable to upload file "%s"`,
					file,
				),

				`Check, that you have enough permissions to upload file to`+
					` the specified project`,
			)
		}

		status := "new"
		if response.Overwritten {
			status = "overwritten"
		}

		fmt.Printf(
			"%s (%s) %s [%d strings %d words]\n",
			branch+file,
			request.FileType,
			status,
			response.StringCount,
			response.WordCount,
		)
	}

	return nil
}
<commit_msg>#39: strip base dir prefix from file path<commit_after>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strings"

	"github.com/Smartling/api-sdk-go"
	"github.com/reconquest/hierr-go"
)

func doFilesPush(
	client *smartling.Client,
	config Config,
	args map[string]interface{},
) error {
	var (
		project       = config.ProjectID
		file, _       = args["<file>"].(string)
		uri, useURI   = args["<uri>"].(string)
		branch, _     = args["--branch"].(string)
		locales, _    = args["--locale"].([]string)
		authorize     = args["--authorize"].(bool)
		directory     = args["--directory"].(string)
		fileType, _   = args["--type"].(string)
		directives, _ = args["--directive"].([]string)
	)

	if branch == "@auto" {
		var err error

		branch, err = getGitBranch()
		if err != nil {
			return hierr.Errorf(
				err,
				"unable to autodetect branch name",
			)
		}

		logger.Infof("autodetected branch name: %s", branch)
	}

	if branch != "" {
		branch = strings.TrimSuffix(branch, "/") + "/"
	}

	patterns := []string{}

	if file != "" {
		patterns = append(patterns, file)
	} else {
		for pattern, section := range config.Files {
			if section.Push.Type != "" {
				patterns = append(patterns, pattern)
			}
		}
	}

	files := []string{}

	for _, pattern := range patterns {
		base, pattern := getDirectoryFromPattern(pattern)
		chunk, err := globFilesLocally(
			directory,
			base,
			pattern,
		)
		if err != nil {
			return NewError(
				hierr.Errorf(
					err,
					`unable to find matching files to upload`,
				),

				`Check, that specified pattern is valid and refer to help for`+
					` more information about glob patterns.`,
			)
		}

		files = append(files, chunk...)
	}

	if len(files) == 0 {
		return NewError(
			fmt.Errorf(`no files found by specified patterns`),

			`Check command line pattern if any and configuration file for`+
				` more patterns to search for.`,
		)
	}

	if uri != "" && len(files) > 1 {
		return NewError(
			fmt.Errorf(
				`more than one file is matching speciifed pattern and <uri>`+
					` is specified too`,
			),

			`Either remove <uri> argument or make sure that only one file`+
				` is matching mask.`,
		)
	}

	base, err := filepath.Abs(config.path)
	if err != nil {
		return NewError(
			hierr.Errorf(
				err,
				`unable to resolve absolute path to config`,
			),

			`It's internal error, please, contact developer for more info`,
		)
	}

	base = filepath.Dir(base)

	for _, file := range files {
		name, err := filepath.Abs(file)
		if err != nil {
			return NewError(
				hierr.Errorf(
					err,
					`unable to resolve absolute path to file: %q`,
					file,
				),

				`Check, that file exists and you have proper permissions `+
					`to access it.`,
			)
		}

		if !filepath.HasPrefix(name, base) {
			return NewError(
				errors.New(
					`you are trying to push file outside project directory`,
				),

				`Check file path and path to configuration file and try again.`,
			)
		}

		name, err = filepath.Rel(base, name)
		if err != nil {
			return NewError(
				hierr.Errorf(
					err,
					`unable to resolve relative path to file: %q`,
					file,
				),

				`Check, that file exists and you have proper permissions `+
					`to access it.`,
			)
		}

		if !useURI {
			uri = name
		}

		fileConfig, err := config.GetFileConfig(file)
		if err != nil {
			return NewError(
				hierr.Errorf(
					err,
					`unable to retrieve file specific configuration`,
				),

				``,
			)
		}

		contents, err := ioutil.ReadFile(file)
		if err != nil {
			return NewError(
				hierr.Errorf(
					err,
					`unable to read file contents "%s"`,
					file,
				),

				`Check that file exists and readable by current user.`,
			)
		}

		request := smartling.FileUploadRequest{
			File:               contents,
			Authorize:          authorize,
			LocalesToAuthorize: locales,
		}

		request.FileURI = branch + uri

		if fileConfig.Push.Type == "" {
			if fileType == "" {
				request.FileType = smartling.GetFileTypeByExtension(
					filepath.Ext(file),
				)

				if request.FileType == smartling.FileTypeUnknown {
					return NewError(
						fmt.Errorf(
							"unable to deduce file type from extension: %q",
							filepath.Ext(file),
						),

						`You need to specify file type via --type option.`,
					)
				}
			} else {
				request.FileType = smartling.FileType(fileType)
			}
		} else {
			request.FileType = smartling.FileType(fileConfig.Push.Type)
		}

		request.Smartling.Directives = fileConfig.Push.Directives

		for _, directive := range directives {
			spec := strings.SplitN(directive, "=", 2)
			if len(spec) != 2 {
				return NewError(
					fmt.Errorf(
						"invalid directive specification: %q",
						directive,
					),

					`Should be in the form of <name>=<value>.`,
				)
			}

			if request.Smartling.Directives == nil {
				request.Smartling.Directives = map[string]string{}
			}

			request.Smartling.Directives[spec[0]] = spec[1]
		}

		response, err := client.UploadFile(project, request)

		if err != nil {
			return NewError(
				hierr.Errorf(
					err,
					`unable to upload file "%s"`,
					file,
				),

				`Check, that you have enough permissions to upload file to`+
					` the specified project`,
			)
		}

		status := "new"
		if response.Overwritten {
			status = "overwritten"
		}

		fmt.Printf(
			"%s (%s) %s [%d strings %d words]\n",
			uri,
			request.FileType,
			status,
			response.StringCount,
			response.WordCount,
		)
	}

	return nil
}
<|endoftext|>