<commit_before>package ingestion

import (
	"fmt"
	"io"
	"path/filepath"
	"time"

	"github.com/boltdb/bolt"
	"github.com/skia-dev/glog"
	"go.skia.org/infra/go/fileutil"
	"go.skia.org/infra/go/sharedconfig"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/go/vcsinfo"
)

// BoltDB bucket where MD5 hashes of processed files are stored.
const PROCESSED_FILES_BUCKET = "processed_files"

// Source defines an ingestion source that returns lists of result files
// either through polling or in an event driven mode.
type Source interface {
	// Return a list of result files that originated between the given
	// timestamps in milliseconds.
	Poll(startTime, endTime int64) ([]ResultFileLocation, error)

	// EventChan returns a channel that sends lists of result files when they
	// are ready for processing. If this source does not support events it should
	// return nil.
	EventChan() <-chan []ResultFileLocation

	// ID returns a unique identifier for this source.
	ID() string
}

// ResultFileLocation is an abstract interface to a file like object that
// contains results that need to be ingested.
type ResultFileLocation interface {
	// Open returns a reader that allows to read the content of the file.
	Open() (io.ReadCloser, error)

	// Name returns the full path of the file. The last segment is usually the
	// the file name.
	Name() string

	// MD5 returns the MD5 hash of the content of the file.
	MD5() string
}

// Processor is the core of an ingester. It takes instances of ResultFileLocation
// and ingests them. It is responsible for the storage of ingested data.
type Processor interface {
	// Process ingests a single result file. It is either stores the file
	// immediately or updates the internal state of the processor and writes
	// data during the BatchFinished call.
	Process(resultsFile ResultFileLocation) error

	// BatchFinished is called when the current batch is finished. This is
	// to cover the case when ingestion is better done for the whole batch
	// This should reset the internal state of the Processor instance.
	BatchFinished() error
}

// Ingester is the main type that drives ingestion for a single type.
type Ingester struct {
	id           string
	vcs          vcsinfo.VCS
	nCommits     int
	minDuration  time.Duration
	runEvery     time.Duration
	sources      []Source
	processor    Processor
	stopChannels []chan<- bool
	statusDB     *bolt.DB
}

// NewIngester creates a new ingester with the given id and configuration around
// the supplied vcs (version control system), input sources and Processor instance.
func NewIngester(ingesterID string, ingesterConf *sharedconfig.IngesterConfig, vcs vcsinfo.VCS, sources []Source, processor Processor) (*Ingester, error) {
	statusDir := fileutil.Must(fileutil.EnsureDirExists(filepath.Join(ingesterConf.StatusDir, ingesterID)))
	dbName := filepath.Join(statusDir, fmt.Sprintf("%s-status.db", ingesterID))
	statusDB, err := bolt.Open(dbName, 0600, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		return nil, fmt.Errorf("Unable to open db at %s. Got error: %s", dbName, err)
	}

	ret := &Ingester{
		id:          ingesterID,
		vcs:         vcs,
		nCommits:    ingesterConf.NCommits,
		minDuration: time.Duration(ingesterConf.MinDays) * time.Hour * 24,
		runEvery:    ingesterConf.RunEvery.Duration,
		sources:     sources,
		processor:   processor,
		statusDB:    statusDB,
	}

	return ret, nil
}

// Start starts the ingester in a new goroutine.
func (i *Ingester) Start() {
	pollChan, eventChan := i.getInputChannels()
	stopCh := make(chan bool)
	i.stopChannels = append(i.stopChannels, stopCh)

	go func(stopCh <-chan bool) {
		var resultFiles []ResultFileLocation = nil
		var fromPolling bool

	MainLoop:
		for {
			select {
			case resultFiles = <-pollChan:
				fromPolling = true
			case resultFiles = <-eventChan:
				fromPolling = false
			case <-stopCh:
				break MainLoop
			}
			i.processResults(resultFiles, fromPolling)
		}
	}(stopCh)
}

// stop stops the ingestion process. Currently only used for testing.
func (i *Ingester) stop() {
	for _, ch := range i.stopChannels {
		ch <- true
	}
	util.Close(i.statusDB)
}

func (i *Ingester) getInputChannels() (<-chan []ResultFileLocation, <-chan []ResultFileLocation) {
	pollChan := make(chan []ResultFileLocation)
	eventChan := make(chan []ResultFileLocation)
	i.stopChannels = make([]chan<- bool, 0, len(i.sources))

	for _, source := range i.sources {
		stopCh := make(chan bool)
		go func(source Source, stopCh <-chan bool) {
			util.Repeat(i.runEvery, stopCh, func() {
				var startTime, endTime int64 = 0, 0
				startTime, endTime, err := i.getCommitRangeOfInterest()
				if err != nil {
					glog.Errorf("Unable to retrieve the start and end time. Got error: %s", err)
					return
				}

				resultFiles, err := source.Poll(startTime, endTime)
				if err != nil {
					glog.Errorf("Error polling data source '%s': %s", source.ID(), err)
					return
				}
				pollChan <- resultFiles
			})
		}(source, stopCh)
		i.stopChannels = append(i.stopChannels, stopCh)

		if ch := source.EventChan(); ch != nil {
			stopCh := make(chan bool)
			go func(ch <-chan []ResultFileLocation, stopCh <-chan bool) {
			MainLoop:
				for {
					select {
					case eventChan <- (<-ch):
					case <-stopCh:
						break MainLoop
					}
				}
			}(ch, stopCh)
			i.stopChannels = append(i.stopChannels, stopCh)
		}
	}
	return pollChan, eventChan
}

func (i *Ingester) inProcessedFiles(md5 string) bool {
	ret := false
	getFn := func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte(PROCESSED_FILES_BUCKET))
		if bucket == nil {
			return nil
		}

		ret = bucket.Get([]byte(md5)) != nil
		return nil
	}

	if err := i.statusDB.View(getFn); err != nil {
		glog.Errorf("Error reading from bucket %s: %s", PROCESSED_FILES_BUCKET, err)
	}
	return ret
}

func (i *Ingester) addToProcessedFiles(md5s []string) {
	updateFn := func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists([]byte(PROCESSED_FILES_BUCKET))
		if err != nil {
			return err
		}

		for _, md5 := range md5s {
			if err := bucket.Put([]byte(md5), []byte{}); err != nil {
				return err
			}
		}
		return nil
	}

	if err := i.statusDB.Update(updateFn); err != nil {
		glog.Errorf("Error writing to bucket %s/%v: %s", PROCESSED_FILES_BUCKET, md5s, err)
	}
}

// processFiles ingests a set of result files.
func (i *Ingester) processResults(resultFiles []ResultFileLocation, fromPolling bool) {
	glog.Infof("Start ingester: %s", i.id)

	processedMD5s := make([]string, 0, len(resultFiles))
	for _, resultLocation := range resultFiles {
		if !i.inProcessedFiles(resultLocation.MD5()) {
			if err := i.processor.Process(resultLocation); err != nil {
				glog.Errorf("Failed to ingest %s: %s", resultLocation.Name(), err)
				continue
			}

			// Gather all successfully processed MD5s
			processedMD5s = append(processedMD5s, resultLocation.MD5())
		}
		// TODO(stephana): Add a metrics to capture how often we skip files
		// because we have processed them already. Including a metric to capture
		// the percent of processed files that come from polling vs events.

	}

	// Notify the ingester that the batch has finished and cause it to reset its
	// state and do any pending ingestion.
	if err := i.processor.BatchFinished(); err != nil {
		glog.Errorf("Batchfinished failed: %s", err)
	} else {
		i.addToProcessedFiles(processedMD5s)
	}

	glog.Infof("Finish ingester: %s", i.id)
}

// getCommitRangeOfInterest returns the time range (start, end) that
// we are interested in. This method assumes that UpdateCommitInfo
// has been called and therefore reading the tile should not fail.
func (i *Ingester) getCommitRangeOfInterest() (int64, int64, error) {
	// Make sure the VCS is up to date.
	if err := i.vcs.Update(true, false); err != nil {
		return 0, 0, err
	}

	// Get the desired numbr of commits in the desired time frame.
	delta := -i.minDuration
	hashes := i.vcs.From(time.Now().Add(delta))
	if len(hashes) == 0 {
		return 0, 0, fmt.Errorf("No commits found.")
	}

	for len(hashes) < i.nCommits {
		delta *= 2
		moreHashes := i.vcs.From(time.Now().Add(delta))
		if len(moreHashes) == len(hashes) {
			hashes = moreHashes
			break
		}
		hashes = moreHashes
	}

	if len(hashes) > i.nCommits {
		hashes = hashes[len(hashes)-i.nCommits:]
	}

	// Get the commit time of the first commit of interest.
	detail, err := i.vcs.Details(hashes[0])
	if err != nil {
		return 0, 0, err
	}

	return detail.Timestamp.Unix(), time.Now().Unix(), nil
}
<commit_msg>This is adds metrics to the generic ingestion module. It captures metrics about all stages of the ingestion process and allows to distinct between results being ingested by the polling process and event driven results.<commit_after>package ingestion

import (
	"fmt"
	"io"
	"path/filepath"
	"time"

	"github.com/boltdb/bolt"
	"github.com/rcrowley/go-metrics"
	"github.com/skia-dev/glog"
	"go.skia.org/infra/go/fileutil"
	smetrics "go.skia.org/infra/go/metrics"
	"go.skia.org/infra/go/sharedconfig"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/go/vcsinfo"
)

// BoltDB bucket where MD5 hashes of processed files are stored.
const PROCESSED_FILES_BUCKET = "processed_files"

// Source defines an ingestion source that returns lists of result files
// either through polling or in an event driven mode.
type Source interface {
	// Return a list of result files that originated between the given
	// timestamps in milliseconds.
	Poll(startTime, endTime int64) ([]ResultFileLocation, error)

	// EventChan returns a channel that sends lists of result files when they
	// are ready for processing. If this source does not support events it should
	// return nil.
	EventChan() <-chan []ResultFileLocation

	// ID returns a unique identifier for this source.
	ID() string
}

// ResultFileLocation is an abstract interface to a file like object that
// contains results that need to be ingested.
type ResultFileLocation interface {
	// Open returns a reader that allows to read the content of the file.
	Open() (io.ReadCloser, error)

	// Name returns the full path of the file. The last segment is usually the
	// the file name.
	Name() string

	// MD5 returns the MD5 hash of the content of the file.
	MD5() string
}

// Processor is the core of an ingester. It takes instances of ResultFileLocation
// and ingests them. It is responsible for the storage of ingested data.
type Processor interface {
	// Process ingests a single result file. It is either stores the file
	// immediately or updates the internal state of the processor and writes
	// data during the BatchFinished call.
	Process(resultsFile ResultFileLocation) error

	// BatchFinished is called when the current batch is finished. This is
	// to cover the case when ingestion is better done for the whole batch
	// This should reset the internal state of the Processor instance.
	BatchFinished() error
}

// Ingester is the main type that drives ingestion for a single type.
type Ingester struct {
	id           string
	vcs          vcsinfo.VCS
	nCommits     int
	minDuration  time.Duration
	runEvery     time.Duration
	sources      []Source
	processor    Processor
	stopChannels []chan<- bool
	statusDB     *bolt.DB

	// srcMetrics capture a set of metrics for each input source.
	srcMetrics []*sourceMetrics

	// pollProcessMetrics capture metrics from processing polled result files.
	pollProcessMetrics *processMetrics

	// eventProcessMetrics capture metrics from processing result files delivered by events from sources.
	eventProcessMetrics *processMetrics

	// processTimer measure the overall time it takes to process a set of files.
	processTimer metrics.Timer

	// processFileTimer measures how long it takes to process an individual file.
	processFileTimer metrics.Timer
}

// NewIngester creates a new ingester with the given id and configuration around
// the supplied vcs (version control system), input sources and Processor instance.
func NewIngester(ingesterID string, ingesterConf *sharedconfig.IngesterConfig, vcs vcsinfo.VCS, sources []Source, processor Processor) (*Ingester, error) {
	statusDir := fileutil.Must(fileutil.EnsureDirExists(filepath.Join(ingesterConf.StatusDir, ingesterID)))
	dbName := filepath.Join(statusDir, fmt.Sprintf("%s-status.db", ingesterID))
	statusDB, err := bolt.Open(dbName, 0600, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		return nil, fmt.Errorf("Unable to open db at %s. Got error: %s", dbName, err)
	}

	ret := &Ingester{
		id:          ingesterID,
		vcs:         vcs,
		nCommits:    ingesterConf.NCommits,
		minDuration: time.Duration(ingesterConf.MinDays) * time.Hour * 24,
		runEvery:    ingesterConf.RunEvery.Duration,
		sources:     sources,
		processor:   processor,
		statusDB:    statusDB,
	}
	ret.setupMetrics()
	return ret, nil
}

// setupMetrics instantiates and registers the metrics instances used by the Ingester.
func (i *Ingester) setupMetrics() {
	i.pollProcessMetrics = newProcessMetrics(i.id, "poll")
	i.eventProcessMetrics = newProcessMetrics(i.id, "event")
	i.srcMetrics = newSourceMetrics(i.id, i.sources)
	i.processTimer = metrics.NewRegisteredTimer(fmt.Sprintf("%s.process", i.id), metrics.DefaultRegistry)
	i.processFileTimer = metrics.NewRegisteredTimer(fmt.Sprintf("%s.process-file", i.id), metrics.DefaultRegistry)
}

// Start starts the ingester in a new goroutine.
func (i *Ingester) Start() {
	pollChan, eventChan := i.getInputChannels()
	stopCh := make(chan bool)
	i.stopChannels = append(i.stopChannels, stopCh)

	go func(stopCh <-chan bool) {
		var resultFiles []ResultFileLocation = nil
		var useMetrics *processMetrics

	MainLoop:
		for {
			select {
			case resultFiles = <-pollChan:
				useMetrics = i.pollProcessMetrics
			case resultFiles = <-eventChan:
				useMetrics = i.eventProcessMetrics
			case <-stopCh:
				break MainLoop
			}
			i.processResults(resultFiles, useMetrics)
		}
	}(stopCh)
}

// stop stops the ingestion process. Currently only used for testing.
func (i *Ingester) stop() {
	for _, ch := range i.stopChannels {
		ch <- true
	}
	util.Close(i.statusDB)
}

func (i *Ingester) getInputChannels() (<-chan []ResultFileLocation, <-chan []ResultFileLocation) {
	pollChan := make(chan []ResultFileLocation)
	eventChan := make(chan []ResultFileLocation)
	i.stopChannels = make([]chan<- bool, 0, len(i.sources))

	for idx, source := range i.sources {
		stopCh := make(chan bool)
		go func(source Source, srcMetrics *sourceMetrics, stopCh <-chan bool) {
			util.Repeat(i.runEvery, stopCh, func() {
				var startTime, endTime int64 = 0, 0
				startTime, endTime, err := i.getCommitRangeOfInterest()
				if err != nil {
					glog.Errorf("Unable to retrieve the start and end time. Got error: %s", err)
					return
				}

				// measure how long the polling takes.
				pollStart := time.Now()
				resultFiles, err := source.Poll(startTime, endTime)
				srcMetrics.pollTimer.UpdateSince(pollStart)
				if err != nil {
					// Indicate that there was an error in polling the source.
					srcMetrics.pollError.Update(1)
					glog.Errorf("Error polling data source '%s': %s", source.ID(), err)
					return
				}

				// Indicate that the polling was successful.
				srcMetrics.pollError.Update(0)
				pollChan <- resultFiles
				srcMetrics.liveness.Update()
			})
		}(source, i.srcMetrics[idx], stopCh)
		i.stopChannels = append(i.stopChannels, stopCh)

		if ch := source.EventChan(); ch != nil {
			stopCh := make(chan bool)
			go func(ch <-chan []ResultFileLocation, stopCh <-chan bool) {
			MainLoop:
				for {
					select {
					case eventChan <- (<-ch):
					case <-stopCh:
						break MainLoop
					}
				}
			}(ch, stopCh)
			i.stopChannels = append(i.stopChannels, stopCh)
		}
	}
	return pollChan, eventChan
}

// inProcessedFiles returns true if the given md5 hash is in the list of
// already processed files.
func (i *Ingester) inProcessedFiles(md5 string) bool {
	ret := false
	getFn := func(tx *bolt.Tx) error {
		bucket := tx.Bucket([]byte(PROCESSED_FILES_BUCKET))
		if bucket == nil {
			return nil
		}

		ret = bucket.Get([]byte(md5)) != nil
		return nil
	}

	if err := i.statusDB.View(getFn); err != nil {
		glog.Errorf("Error reading from bucket %s: %s", PROCESSED_FILES_BUCKET, err)
	}
	return ret
}

// addToProcessedFiles adds the given list of md5 hashes to the list of
// file that have been already processed.
func (i *Ingester) addToProcessedFiles(md5s []string) {
	updateFn := func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists([]byte(PROCESSED_FILES_BUCKET))
		if err != nil {
			return err
		}

		for _, md5 := range md5s {
			if err := bucket.Put([]byte(md5), []byte{}); err != nil {
				return err
			}
		}
		return nil
	}

	if err := i.statusDB.Update(updateFn); err != nil {
		glog.Errorf("Error writing to bucket %s/%v: %s", PROCESSED_FILES_BUCKET, md5s, err)
	}
}

// processResults ingests a set of result files.
func (i *Ingester) processResults(resultFiles []ResultFileLocation, targetMetrics *processMetrics) {
	glog.Infof("Start ingester: %s", i.id)

	processedMD5s := make([]string, 0, len(resultFiles))
	processedCounter, ignoredCounter, errorCounter := 0, 0, 0

	// time how long the overall process takes.
	processStart := time.Now()
	for _, resultLocation := range resultFiles {
		if !i.inProcessedFiles(resultLocation.MD5()) {
			// time how long it takes to process a file.
			processFileStart := time.Now()
			err := i.processor.Process(resultLocation)
			i.processFileTimer.UpdateSince(processFileStart)

			if err != nil {
				errorCounter++
				glog.Errorf("Failed to ingest %s: %s", resultLocation.Name(), err)
				continue
			}

			// Gather all successfully processed MD5s
			processedCounter++
			processedMD5s = append(processedMD5s, resultLocation.MD5())
		} else {
			ignoredCounter++
		}
	}

	// Update the timer and the gauges that measure how the ingestion works
	// for the input type.
	i.processTimer.UpdateSince(processStart)
	targetMetrics.totalFilesGauge.Update(int64(len(resultFiles)))
	targetMetrics.processedGauge.Update(int64(processedCounter))
	targetMetrics.ignoredGauge.Update(int64(ignoredCounter))
	targetMetrics.errorGauge.Update(int64(errorCounter))

	// Notify the ingester that the batch has finished and cause it to reset its
	// state and do any pending ingestion.
	if err := i.processor.BatchFinished(); err != nil {
		glog.Errorf("Batchfinished failed: %s", err)
	} else {
		i.addToProcessedFiles(processedMD5s)
	}

	glog.Infof("Finish ingester: %s", i.id)
}

// getCommitRangeOfInterest returns the time range (start, end) that
// we are interested in. This method assumes that UpdateCommitInfo
// has been called and therefore reading the tile should not fail.
func (i *Ingester) getCommitRangeOfInterest() (int64, int64, error) {
	// Make sure the VCS is up to date.
	if err := i.vcs.Update(true, false); err != nil {
		return 0, 0, err
	}

	// Get the desired numbr of commits in the desired time frame.
	delta := -i.minDuration
	hashes := i.vcs.From(time.Now().Add(delta))
	if len(hashes) == 0 {
		return 0, 0, fmt.Errorf("No commits found.")
	}

	for len(hashes) < i.nCommits {
		delta *= 2
		moreHashes := i.vcs.From(time.Now().Add(delta))
		if len(moreHashes) == len(hashes) {
			hashes = moreHashes
			break
		}
		hashes = moreHashes
	}

	if len(hashes) > i.nCommits {
		hashes = hashes[len(hashes)-i.nCommits:]
	}

	// Get the commit time of the first commit of interest.
	detail, err := i.vcs.Details(hashes[0])
	if err != nil {
		return 0, 0, err
	}

	return detail.Timestamp.Unix(), time.Now().Unix(), nil
}

// processMetrics contains the metrics we are interested for processing results.
// We have one instance for polled result files and one for files that were
// delievered via events.
type processMetrics struct {
	totalFilesGauge metrics.Gauge
	processedGauge  metrics.Gauge
	ignoredGauge    metrics.Gauge
	errorGauge      metrics.Gauge
}

// newProcessMetrics instantiates the metrics to track processing and registers them
// with the metrics package.
func newProcessMetrics(id, subtype string) *processMetrics {
	prefix := fmt.Sprintf("%s.%s", id, subtype)
	return &processMetrics{
		totalFilesGauge: metrics.NewRegisteredGauge(prefix+".total", metrics.DefaultRegistry),
		processedGauge:  metrics.NewRegisteredGauge(prefix+".processed", metrics.DefaultRegistry),
		ignoredGauge:    metrics.NewRegisteredGauge(prefix+".ignored", metrics.DefaultRegistry),
		errorGauge:      metrics.NewRegisteredGauge(prefix+".errors", metrics.DefaultRegistry),
	}
}

// sourceMetrics tracks metrics for one input source.
type sourceMetrics struct {
	liveness       *smetrics.Liveness
	pollTimer      metrics.Timer
	pollError      metrics.Gauge
	eventsReceived metrics.Meter
}

// newSourceMetrics instantiates a set of metrics for an input source.
func newSourceMetrics(id string, sources []Source) []*sourceMetrics {
	ret := make([]*sourceMetrics, len(sources))
	for idx, source := range sources {
		prefix := fmt.Sprintf("%s.%s", id, source.ID())
		ret[idx] = &sourceMetrics{
			liveness:       smetrics.NewLiveness(prefix + ".poll-liveness"),
			pollTimer:      metrics.NewRegisteredTimer(prefix+".poll-timer", metrics.DefaultRegistry),
			pollError:      metrics.NewRegisteredGauge(prefix+".poll-error", metrics.DefaultRegistry),
			eventsReceived: metrics.NewRegisteredMeter(prefix+".events-received", metrics.DefaultRegistry),
		}
	}
	return ret
}
<|endoftext|>


<commit_before>// Convenience utilities for testing.
package testutils

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"reflect"
	"runtime"
	"strings"
	"testing"

	assert "github.com/stretchr/testify/require"
	"go.skia.org/infra/go/repo_root"
)

const (
	SMALL_TEST  = "small"
	MEDIUM_TEST = "medium"
	LARGE_TEST  = "large"
)

var (
	small         = flag.Bool(SMALL_TEST, false, "Whether or not to run small tests.")
	medium        = flag.Bool(MEDIUM_TEST, false, "Whether or not to run medium tests.")
	large         = flag.Bool(LARGE_TEST, false, "Whether or not to run large tests.")
	uncategorized = flag.Bool("uncategorized", false, "Only run uncategorized tests.")

	// DEFAULT_RUN indicates whether the given test type runs by default
	// when no filter flag is specified.
	DEFAULT_RUN = map[string]bool{
		SMALL_TEST:  true,
		MEDIUM_TEST: true,
		LARGE_TEST:  true,
	}

	TIMEOUT_SMALL  = "4s"
	TIMEOUT_MEDIUM = "15s"
	TIMEOUT_LARGE  = "2m"

	TIMEOUT_RACE = "5m"

	// TEST_TYPES lists all of the types of tests.
	TEST_TYPES = []string{
		SMALL_TEST,
		MEDIUM_TEST,
		LARGE_TEST,
	}
)

// ShouldRun determines whether the test should run based on the provided flags.
func ShouldRun(testType string) bool {
	if *uncategorized {
		return false
	}

	// Fallback if no test filter is specified.
	if !*small && !*medium && !*large {
		return DEFAULT_RUN[testType]
	}

	switch testType {
	case SMALL_TEST:
		return *small
	case MEDIUM_TEST:
		return *medium
	case LARGE_TEST:
		return *large
	}
	return false
}

// SmallTest is a function which should be called at the beginning of a small
// test: A test (under 2 seconds) with no dependencies on external databases,
// networks, etc.
func SmallTest(t *testing.T) {
	if !ShouldRun(SMALL_TEST) {
		t.Skip("Not running small tests.")
	}
}

// MediumTest is a function which should be called at the beginning of an
// medium-sized test: a test (2-15 seconds) which has dependencies on external
// databases, networks, etc.
func MediumTest(t *testing.T) {
	if !ShouldRun(MEDIUM_TEST) || testing.Short() {
		t.Skip("Not running medium tests.")
	}
}

// LargeTest is a function which should be called at the beginning of a large
// test: a test (> 15 seconds) with significant reliance on external
// dependencies which makes it too slow or flaky to run as part of the normal
// test suite.
func LargeTest(t *testing.T) {
	if !ShouldRun(LARGE_TEST) || testing.Short() {
		t.Skip("Not running large tests.")
	}
}

// AssertCopy is AssertDeepEqual but also checks that none of the direct fields
// have a zero value and none of the direct fields point to the same object.
// This catches regressions where a new field is added without adding that field
// to the Copy method. Arguments must be structs.
func AssertCopy(t *testing.T, a, b interface{}) {
	AssertDeepEqual(t, a, b)

	// Check that all fields are non-zero.
	va := reflect.ValueOf(a)
	vb := reflect.ValueOf(b)
	assert.Equal(t, va.Type(), vb.Type(), "Arguments are different types.")
	for va.Kind() == reflect.Ptr {
		assert.Equal(t, reflect.Ptr, vb.Kind(), "Arguments are different types (pointer vs. non-pointer)")
		va = va.Elem()
		vb = vb.Elem()
	}
	assert.Equal(t, reflect.Struct, va.Kind(), "Not a struct or pointer to struct.")
	assert.Equal(t, reflect.Struct, vb.Kind(), "Arguments are different types (pointer vs. non-pointer)")
	for i := 0; i < va.NumField(); i++ {
		fa := va.Field(i)
		z := reflect.Zero(fa.Type())
		if reflect.DeepEqual(fa.Interface(), z.Interface()) {
			assert.FailNow(t, fmt.Sprintf("Missing field %q (or set to zero value).", va.Type().Field(i).Name))
		}
		if fa.Kind() == reflect.Map || fa.Kind() == reflect.Ptr || fa.Kind() == reflect.Slice {
			fb := vb.Field(i)
			assert.NotEqual(t, fa.Pointer(), fb.Pointer(), "Field %q not deep-copied.", va.Type().Field(i).Name)
		}
	}
}

// TestDataDir returns the path to the caller's testdata directory, which
// is assumed to be "<path to caller dir>/testdata".
func TestDataDir() (string, error) {
	_, thisFile, _, ok := runtime.Caller(0)
	if !ok {
		return "", fmt.Errorf("Could not find test data dir: runtime.Caller() failed.")
	}
	for skip := 0; ; skip++ {
		_, file, _, ok := runtime.Caller(skip)
		if !ok {
			return "", fmt.Errorf("Could not find test data dir: runtime.Caller() failed.")
		}
		if file != thisFile {
			return path.Join(path.Dir(file), "testdata"), nil
		}
	}
}

func readFile(filename string) (io.Reader, error) {
	dir, err := TestDataDir()
	if err != nil {
		return nil, fmt.Errorf("Could not read %s: %v", filename, err)
	}
	f, err := os.Open(path.Join(dir, filename))
	if err != nil {
		return nil, fmt.Errorf("Could not read %s: %v", filename, err)
	}
	return f, nil
}

// ReadFile reads a file from the caller's testdata directory.
func ReadFile(filename string) (string, error) {
	f, err := readFile(filename)
	if err != nil {
		return "", err
	}
	b, err := ioutil.ReadAll(f)
	if err != nil {
		return "", fmt.Errorf("Could not read %s: %v", filename, err)
	}
	return string(b), nil
}

// MustReadFile reads a file from the caller's testdata directory and panics on
// error.
func MustReadFile(filename string) string {
	s, err := ReadFile(filename)
	if err != nil {
		panic(err)
	}
	return s
}

// ReadJsonFile reads a JSON file from the caller's testdata directory into the
// given interface.
func ReadJsonFile(filename string, dest interface{}) error {
	f, err := readFile(filename)
	if err != nil {
		return err
	}
	return json.NewDecoder(f).Decode(dest)
}

// MustReadJsonFile reads a JSON file from the caller's testdata directory into
// the given interface and panics on error.
func MustReadJsonFile(filename string, dest interface{}) {
	if err := ReadJsonFile(filename, dest); err != nil {
		panic(err)
	}
}

// WriteFile writes the given contents to the given file path, reporting any
// error.
func WriteFile(t assert.TestingT, filename, contents string) {
	assert.NoErrorf(t, ioutil.WriteFile(filename, []byte(contents), os.ModePerm), "Unable to write to file %s", filename)
}

// CloseInTest takes an ioutil.Closer and Closes it, reporting any error.
func CloseInTest(t assert.TestingT, c io.Closer) {
	if err := c.Close(); err != nil {
		t.Errorf("Failed to Close(): %v", err)
	}
}

// AssertCloses takes an ioutil.Closer and asserts that it closes.
func AssertCloses(t assert.TestingT, c io.Closer) {
	assert.NoError(t, c.Close())
}

// Remove attempts to remove the given file and asserts that no error is returned.
func Remove(t assert.TestingT, fp string) {
	assert.NoError(t, os.Remove(fp))
}

// RemoveAll attempts to remove the given directory and asserts that no error is returned.
func RemoveAll(t assert.TestingT, fp string) {
	assert.NoError(t, os.RemoveAll(fp))
}

// TempDir is a wrapper for ioutil.TempDir. Returns the path to the directory and a cleanup
// function to defer.
func TempDir(t assert.TestingT) (string, func()) {
	d, err := ioutil.TempDir("", "testutils")
	assert.NoError(t, err)
	return d, func() {
		RemoveAll(t, d)
	}
}

// MarshalJSON encodes the given interface to a JSON string.
func MarshalJSON(t *testing.T, i interface{}) string {
	b, err := json.Marshal(i)
	assert.NoError(t, err)
	return string(b)
}

// MarshalIndentJSON encodes the given interface to an indented JSON string.
func MarshalIndentJSON(t *testing.T, i interface{}) string {
	b, err := json.MarshalIndent(i, "", "  ")
	assert.NoError(t, err)
	return string(b)
}

// AssertErrorContains asserts that the given error contains the given string.
func AssertErrorContains(t *testing.T, err error, substr string) {
	assert.NotNil(t, err)
	assert.True(t, strings.Contains(err.Error(), substr))
}

// Return the path to the root of the checkout.
func GetRepoRoot(t *testing.T) string {
	root, err := repo_root.Get()
	assert.NoError(t, err)
	return root
}
<commit_msg>Increase timeout for Large tests to 4m<commit_after>// Convenience utilities for testing.
package testutils

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"reflect"
	"runtime"
	"strings"
	"testing"

	assert "github.com/stretchr/testify/require"
	"go.skia.org/infra/go/repo_root"
)

const (
	SMALL_TEST  = "small"
	MEDIUM_TEST = "medium"
	LARGE_TEST  = "large"
)

var (
	small         = flag.Bool(SMALL_TEST, false, "Whether or not to run small tests.")
	medium        = flag.Bool(MEDIUM_TEST, false, "Whether or not to run medium tests.")
	large         = flag.Bool(LARGE_TEST, false, "Whether or not to run large tests.")
	uncategorized = flag.Bool("uncategorized", false, "Only run uncategorized tests.")

	// DEFAULT_RUN indicates whether the given test type runs by default
	// when no filter flag is specified.
	DEFAULT_RUN = map[string]bool{
		SMALL_TEST:  true,
		MEDIUM_TEST: true,
		LARGE_TEST:  true,
	}

	TIMEOUT_SMALL  = "4s"
	TIMEOUT_MEDIUM = "15s"
	TIMEOUT_LARGE  = "4m"

	TIMEOUT_RACE = "5m"

	// TEST_TYPES lists all of the types of tests.
	TEST_TYPES = []string{
		SMALL_TEST,
		MEDIUM_TEST,
		LARGE_TEST,
	}
)

// ShouldRun determines whether the test should run based on the provided flags.
func ShouldRun(testType string) bool {
	if *uncategorized {
		return false
	}

	// Fallback if no test filter is specified.
	if !*small && !*medium && !*large {
		return DEFAULT_RUN[testType]
	}

	switch testType {
	case SMALL_TEST:
		return *small
	case MEDIUM_TEST:
		return *medium
	case LARGE_TEST:
		return *large
	}
	return false
}

// SmallTest is a function which should be called at the beginning of a small
// test: A test (under 2 seconds) with no dependencies on external databases,
// networks, etc.
func SmallTest(t *testing.T) {
	if !ShouldRun(SMALL_TEST) {
		t.Skip("Not running small tests.")
	}
}

// MediumTest is a function which should be called at the beginning of an
// medium-sized test: a test (2-15 seconds) which has dependencies on external
// databases, networks, etc.
func MediumTest(t *testing.T) {
	if !ShouldRun(MEDIUM_TEST) || testing.Short() {
		t.Skip("Not running medium tests.")
	}
}

// LargeTest is a function which should be called at the beginning of a large
// test: a test (> 15 seconds) with significant reliance on external
// dependencies which makes it too slow or flaky to run as part of the normal
// test suite.
func LargeTest(t *testing.T) {
	if !ShouldRun(LARGE_TEST) || testing.Short() {
		t.Skip("Not running large tests.")
	}
}

// AssertCopy is AssertDeepEqual but also checks that none of the direct fields
// have a zero value and none of the direct fields point to the same object.
// This catches regressions where a new field is added without adding that field
// to the Copy method. Arguments must be structs.
func AssertCopy(t *testing.T, a, b interface{}) {
	AssertDeepEqual(t, a, b)

	// Check that all fields are non-zero.
	va := reflect.ValueOf(a)
	vb := reflect.ValueOf(b)
	assert.Equal(t, va.Type(), vb.Type(), "Arguments are different types.")
	for va.Kind() == reflect.Ptr {
		assert.Equal(t, reflect.Ptr, vb.Kind(), "Arguments are different types (pointer vs. non-pointer)")
		va = va.Elem()
		vb = vb.Elem()
	}
	assert.Equal(t, reflect.Struct, va.Kind(), "Not a struct or pointer to struct.")
	assert.Equal(t, reflect.Struct, vb.Kind(), "Arguments are different types (pointer vs. non-pointer)")
	for i := 0; i < va.NumField(); i++ {
		fa := va.Field(i)
		z := reflect.Zero(fa.Type())
		if reflect.DeepEqual(fa.Interface(), z.Interface()) {
			assert.FailNow(t, fmt.Sprintf("Missing field %q (or set to zero value).", va.Type().Field(i).Name))
		}
		if fa.Kind() == reflect.Map || fa.Kind() == reflect.Ptr || fa.Kind() == reflect.Slice {
			fb := vb.Field(i)
			assert.NotEqual(t, fa.Pointer(), fb.Pointer(), "Field %q not deep-copied.", va.Type().Field(i).Name)
		}
	}
}

// TestDataDir returns the path to the caller's testdata directory, which
// is assumed to be "<path to caller dir>/testdata".
func TestDataDir() (string, error) {
	_, thisFile, _, ok := runtime.Caller(0)
	if !ok {
		return "", fmt.Errorf("Could not find test data dir: runtime.Caller() failed.")
	}
	for skip := 0; ; skip++ {
		_, file, _, ok := runtime.Caller(skip)
		if !ok {
			return "", fmt.Errorf("Could not find test data dir: runtime.Caller() failed.")
		}
		if file != thisFile {
			return path.Join(path.Dir(file), "testdata"), nil
		}
	}
}

func readFile(filename string) (io.Reader, error) {
	dir, err := TestDataDir()
	if err != nil {
		return nil, fmt.Errorf("Could not read %s: %v", filename, err)
	}
	f, err := os.Open(path.Join(dir, filename))
	if err != nil {
		return nil, fmt.Errorf("Could not read %s: %v", filename, err)
	}
	return f, nil
}

// ReadFile reads a file from the caller's testdata directory.
func ReadFile(filename string) (string, error) {
	f, err := readFile(filename)
	if err != nil {
		return "", err
	}
	b, err := ioutil.ReadAll(f)
	if err != nil {
		return "", fmt.Errorf("Could not read %s: %v", filename, err)
	}
	return string(b), nil
}

// MustReadFile reads a file from the caller's testdata directory and panics on
// error.
func MustReadFile(filename string) string {
	s, err := ReadFile(filename)
	if err != nil {
		panic(err)
	}
	return s
}

// ReadJsonFile reads a JSON file from the caller's testdata directory into the
// given interface.
func ReadJsonFile(filename string, dest interface{}) error {
	f, err := readFile(filename)
	if err != nil {
		return err
	}
	return json.NewDecoder(f).Decode(dest)
}

// MustReadJsonFile reads a JSON file from the caller's testdata directory into
// the given interface and panics on error.
func MustReadJsonFile(filename string, dest interface{}) {
	if err := ReadJsonFile(filename, dest); err != nil {
		panic(err)
	}
}

// WriteFile writes the given contents to the given file path, reporting any
// error.
func WriteFile(t assert.TestingT, filename, contents string) {
	assert.NoErrorf(t, ioutil.WriteFile(filename, []byte(contents), os.ModePerm), "Unable to write to file %s", filename)
}

// CloseInTest takes an ioutil.Closer and Closes it, reporting any error.
func CloseInTest(t assert.TestingT, c io.Closer) {
	if err := c.Close(); err != nil {
		t.Errorf("Failed to Close(): %v", err)
	}
}

// AssertCloses takes an ioutil.Closer and asserts that it closes.
func AssertCloses(t assert.TestingT, c io.Closer) {
	assert.NoError(t, c.Close())
}

// Remove attempts to remove the given file and asserts that no error is returned.
func Remove(t assert.TestingT, fp string) {
	assert.NoError(t, os.Remove(fp))
}

// RemoveAll attempts to remove the given directory and asserts that no error is returned.
func RemoveAll(t assert.TestingT, fp string) {
	assert.NoError(t, os.RemoveAll(fp))
}

// TempDir is a wrapper for ioutil.TempDir. Returns the path to the directory and a cleanup
// function to defer.
func TempDir(t assert.TestingT) (string, func()) {
	d, err := ioutil.TempDir("", "testutils")
	assert.NoError(t, err)
	return d, func() {
		RemoveAll(t, d)
	}
}

// MarshalJSON encodes the given interface to a JSON string.
func MarshalJSON(t *testing.T, i interface{}) string {
	b, err := json.Marshal(i)
	assert.NoError(t, err)
	return string(b)
}

// MarshalIndentJSON encodes the given interface to an indented JSON string.
func MarshalIndentJSON(t *testing.T, i interface{}) string {
	b, err := json.MarshalIndent(i, "", "  ")
	assert.NoError(t, err)
	return string(b)
}

// AssertErrorContains asserts that the given error contains the given string.
func AssertErrorContains(t *testing.T, err error, substr string) {
	assert.NotNil(t, err)
	assert.True(t, strings.Contains(err.Error(), substr))
}

// Return the path to the root of the checkout.
func GetRepoRoot(t *testing.T) string {
	root, err := repo_root.Get()
	assert.NoError(t, err)
	return root
}
<|endoftext|>


<commit_before>package internal

import (
	"math"
)

// Threshold is an adaptive error threshold.
type Threshold struct {
	values []float64
	lower  []float64
	upper  []float64

	no uint
	εa float64
	εr float64
}

// NewThreshold creates a Threshold.
func NewThreshold(outputs uint, absolute, relative float64) *Threshold {
	return &Threshold{
		values: make([]float64, outputs),
		lower:  repeat(math.Inf(1.0), outputs),
		upper:  repeat(math.Inf(-1.0), outputs),

		no: outputs,
		εa: absolute,
		εr: relative,
	}
}

// Check checks if the threshold is satisfied.
func (self *Threshold) Check(error []float64) bool {
	for i, no := uint(0), self.no; i < no; i++ {
		if error[i] > self.values[i] {
			return false
		}
	}
	return true
}

// Compress compresses multiple errors into a single one so that it can later on
// be tested against the threshold.
func (self *Threshold) Compress(error, errors []float64) {
	no := self.no
	nn := uint(len(errors)) / no
	for i := uint(0); i < nn; i++ {
		for j := uint(0); j < no; j++ {
			error[j] = math.Max(error[j], math.Abs(errors[i*no+j]))
		}
	}
}

// Update updates the threshold.
func (self *Threshold) Update(values []float64) {
	no := self.no
	for i, m := uint(0), uint(len(values)); i < m; i++ {
		j := i % no
		self.lower[j] = math.Min(self.lower[j], values[i])
		self.upper[j] = math.Max(self.upper[j], values[i])
	}
	for i := uint(0); i < no; i++ {
		self.values[i] = math.Max(self.εa, self.εr*(self.upper[i]-self.lower[i]))
	}
}

func repeat(value float64, times uint) []float64 {
	values := make([]float64, times)
	for i := uint(0); i < times; i++ {
		values[i] = value
	}
	return values
}
<commit_msg>a/internal: flatten Threshold.Compress<commit_after>package internal

import (
	"math"
)

// Threshold is an adaptive error threshold.
type Threshold struct {
	values []float64
	lower  []float64
	upper  []float64

	no uint
	εa float64
	εr float64
}

// NewThreshold creates a Threshold.
func NewThreshold(outputs uint, absolute, relative float64) *Threshold {
	return &Threshold{
		values: make([]float64, outputs),
		lower:  repeat(math.Inf(1.0), outputs),
		upper:  repeat(math.Inf(-1.0), outputs),

		no: outputs,
		εa: absolute,
		εr: relative,
	}
}

// Check checks if the threshold is satisfied.
func (self *Threshold) Check(error []float64) bool {
	for i, no := uint(0), self.no; i < no; i++ {
		if error[i] > self.values[i] {
			return false
		}
	}
	return true
}

// Compress compresses multiple errors into a single one so that it can later on
// be tested against the threshold.
func (self *Threshold) Compress(error, errors []float64) {
	no := self.no
	for i, m := uint(0), uint(len(errors)); i < m; i++ {
		j := i % no
		error[j] = math.Max(error[j], math.Abs(errors[i*no+j]))
	}
}

// Update updates the threshold.
func (self *Threshold) Update(values []float64) {
	no := self.no
	for i, m := uint(0), uint(len(values)); i < m; i++ {
		j := i % no
		self.lower[j] = math.Min(self.lower[j], values[i])
		self.upper[j] = math.Max(self.upper[j], values[i])
	}
	for i := uint(0); i < no; i++ {
		self.values[i] = math.Max(self.εa, self.εr*(self.upper[i]-self.lower[i]))
	}
}

func repeat(value float64, times uint) []float64 {
	values := make([]float64, times)
	for i := uint(0); i < times; i++ {
		values[i] = value
	}
	return values
}
<|endoftext|>


<commit_before>// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/pprof"

	"github.com/gorilla/mux"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/mysql"
	"github.com/pingcap/tidb/terror"
	"github.com/pingcap/tidb/util/printer"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	log "github.com/sirupsen/logrus"
)

const defaultStatusAddr = ":10080"

func (s *Server) startStatusHTTP() {
	go s.startHTTPServer()
}

func (s *Server) startHTTPServer() {
	router := mux.NewRouter()
	router.HandleFunc("/status", s.handleStatus)
	// HTTP path for prometheus.
	router.Handle("/metrics", prometheus.Handler())

	// HTTP path for dump statistics.
	router.Handle("/stats/dump/{db}/{table}", s.newStatsHandler())

	router.Handle("/settings", settingsHandler{})
	router.Handle("/binlog/recover", binlogRecover{})

	tikvHandlerTool := s.newTikvHandlerTool()
	router.Handle("/schema", schemaHandler{tikvHandlerTool})
	router.Handle("/schema/{db}", schemaHandler{tikvHandlerTool})
	router.Handle("/schema/{db}/{table}", schemaHandler{tikvHandlerTool})
	router.Handle("/tables/{colID}/{colTp}/{colFlag}/{colLen}", valueHandler{})
	router.Handle("/ddl/history", ddlHistoryJobHandler{tikvHandlerTool})
	router.Handle("/ddl/owner/resign", ddlResignOwnerHandler{tikvHandlerTool.store.(kv.Storage)})

	// HTTP path for get server info.
	router.Handle("/info", serverInfoHandler{tikvHandlerTool})
	router.Handle("/info/all", allServerInfoHandler{tikvHandlerTool})
	if s.cfg.Store == "tikv" {
		// HTTP path for tikv.
		router.Handle("/tables/{db}/{table}/regions", tableHandler{tikvHandlerTool, opTableRegions})
		router.Handle("/tables/{db}/{table}/scatter", tableHandler{tikvHandlerTool, opTableScatter})
		router.Handle("/tables/{db}/{table}/stop-scatter", tableHandler{tikvHandlerTool, opStopTableScatter})
		router.Handle("/tables/{db}/{table}/disk-usage", tableHandler{tikvHandlerTool, opTableDiskUsage})
		router.Handle("/regions/meta", regionHandler{tikvHandlerTool})
		router.Handle("/regions/{regionID}", regionHandler{tikvHandlerTool})
		router.Handle("/mvcc/key/{db}/{table}/{handle}", mvccTxnHandler{tikvHandlerTool, opMvccGetByKey})
		router.Handle("/mvcc/txn/{startTS}/{db}/{table}", mvccTxnHandler{tikvHandlerTool, opMvccGetByTxn})
		router.Handle("/mvcc/hex/{hexKey}", mvccTxnHandler{tikvHandlerTool, opMvccGetByHex})
		router.Handle("/mvcc/index/{db}/{table}/{index}/{handle}", mvccTxnHandler{tikvHandlerTool, opMvccGetByIdx})
	}
	addr := fmt.Sprintf(":%d", s.cfg.Status.StatusPort)
	if s.cfg.Status.StatusPort == 0 {
		addr = defaultStatusAddr
	}

	serverMux := http.NewServeMux()
	serverMux.Handle("/", router)

	serverMux.HandleFunc("/debug/pprof/", pprof.Index)
	serverMux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
	serverMux.HandleFunc("/debug/pprof/profile", pprof.Profile)
	serverMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
	serverMux.HandleFunc("/debug/pprof/trace", pprof.Trace)

	log.Infof("Listening on %v for status and metrics report.", addr)
	s.statusServer = &http.Server{Addr: addr, Handler: serverMux}
	var err error
	if len(s.cfg.Security.ClusterSSLCA) != 0 {
		err = s.statusServer.ListenAndServeTLS(s.cfg.Security.ClusterSSLCert, s.cfg.Security.ClusterSSLKey)
	} else {
		err = s.statusServer.ListenAndServe()
	}

	if err != nil {
		log.Info(err)
	}
}

// status of TiDB.
type status struct {
	Connections int    `json:"connections"`
	Version     string `json:"version"`
	GitHash     string `json:"git_hash"`
}

func (s *Server) handleStatus(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Type", "application/json")

	st := status{
		Connections: s.ConnectionCount(),
		Version:     mysql.ServerVersion,
		GitHash:     printer.TiDBGitHash,
	}
	js, err := json.Marshal(st)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		log.Error("Encode json error", err)
	} else {
		_, err = w.Write(js)
		terror.Log(errors.Trace(err))
	}
}
<commit_msg>Add index page on HTTP Server (#7769)<commit_after>// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/pprof"

	"github.com/gorilla/mux"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/mysql"
	"github.com/pingcap/tidb/terror"
	"github.com/pingcap/tidb/util/printer"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	log "github.com/sirupsen/logrus"
)

const defaultStatusAddr = ":10080"

func (s *Server) startStatusHTTP() {
	go s.startHTTPServer()
}

func (s *Server) startHTTPServer() {
	router := mux.NewRouter()

	router.HandleFunc("/status", s.handleStatus).Name("Status")
	// HTTP path for prometheus.
	router.Handle("/metrics", prometheus.Handler()).Name("Metrics")

	// HTTP path for dump statistics.
	router.Handle("/stats/dump/{db}/{table}", s.newStatsHandler()).Name("StatsDump")

	router.Handle("/settings", settingsHandler{}).Name("Settings")
	router.Handle("/binlog/recover", binlogRecover{}).Name("BinlogRecover")

	tikvHandlerTool := s.newTikvHandlerTool()
	router.Handle("/schema", schemaHandler{tikvHandlerTool}).Name("Schema")
	router.Handle("/schema/{db}", schemaHandler{tikvHandlerTool})
	router.Handle("/schema/{db}/{table}", schemaHandler{tikvHandlerTool})
	router.Handle("/tables/{colID}/{colTp}/{colFlag}/{colLen}", valueHandler{})
	router.Handle("/ddl/history", ddlHistoryJobHandler{tikvHandlerTool}).Name("DDL_History")
	router.Handle("/ddl/owner/resign", ddlResignOwnerHandler{tikvHandlerTool.store.(kv.Storage)}).Name("DDL_Owner_Resign")

	// HTTP path for get server info.
	router.Handle("/info", serverInfoHandler{tikvHandlerTool}).Name("Info")
	router.Handle("/info/all", allServerInfoHandler{tikvHandlerTool}).Name("InfoALL")
	if s.cfg.Store == "tikv" {
		// HTTP path for tikv.
		router.Handle("/tables/{db}/{table}/regions", tableHandler{tikvHandlerTool, opTableRegions})
		router.Handle("/tables/{db}/{table}/scatter", tableHandler{tikvHandlerTool, opTableScatter})
		router.Handle("/tables/{db}/{table}/stop-scatter", tableHandler{tikvHandlerTool, opStopTableScatter})
		router.Handle("/tables/{db}/{table}/disk-usage", tableHandler{tikvHandlerTool, opTableDiskUsage})
		router.Handle("/regions/meta", regionHandler{tikvHandlerTool}).Name("RegionsMeta")
		router.Handle("/regions/{regionID}", regionHandler{tikvHandlerTool})
		router.Handle("/mvcc/key/{db}/{table}/{handle}", mvccTxnHandler{tikvHandlerTool, opMvccGetByKey})
		router.Handle("/mvcc/txn/{startTS}/{db}/{table}", mvccTxnHandler{tikvHandlerTool, opMvccGetByTxn})
		router.Handle("/mvcc/hex/{hexKey}", mvccTxnHandler{tikvHandlerTool, opMvccGetByHex})
		router.Handle("/mvcc/index/{db}/{table}/{index}/{handle}", mvccTxnHandler{tikvHandlerTool, opMvccGetByIdx})
	}
	addr := fmt.Sprintf(":%d", s.cfg.Status.StatusPort)
	if s.cfg.Status.StatusPort == 0 {
		addr = defaultStatusAddr
	}

	serverMux := http.NewServeMux()
	serverMux.Handle("/", router)

	serverMux.HandleFunc("/debug/pprof/", pprof.Index)
	serverMux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
	serverMux.HandleFunc("/debug/pprof/profile", pprof.Profile)
	serverMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
	serverMux.HandleFunc("/debug/pprof/trace", pprof.Trace)

	var (
		err            error
		httpRouterPage bytes.Buffer
		pathTemplate   string
	)
	httpRouterPage.WriteString("<title>TiDB Status and Metrics Report</title>
TiDB Status and Metrics Report

")
	err = router.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error {
		pathTemplate, err = route.GetPathTemplate()
		if err != nil {
			log.Error("Get http router path error ", err)
		}
		name := route.GetName() //If the name attribute is not set, GetName returns ""
		if name != "" && err == nil {
			httpRouterPage.WriteString("
" + name + "
")
		}
		return nil
	})
	if err != nil {
		log.Error("Generate root error ", err)
	}
	httpRouterPage.WriteString("
Debug
")
	httpRouterPage.WriteString("
")
	router.HandleFunc("/", func(responseWriter http.ResponseWriter, request *http.Request) {
		_, err = responseWriter.Write([]byte(httpRouterPage.String()))
		if err != nil {
			log.Error("Http index page error ", err)
		}
	})

	log.Infof("Listening on %v for status and metrics report.", addr)
	s.statusServer = &http.Server{Addr: addr, Handler: serverMux}

	if len(s.cfg.Security.ClusterSSLCA) != 0 {
		err = s.statusServer.ListenAndServeTLS(s.cfg.Security.ClusterSSLCert, s.cfg.Security.ClusterSSLKey)
	} else {
		err = s.statusServer.ListenAndServe()
	}

	if err != nil {
		log.Info(err)
	}
}

// status of TiDB.
type status struct {
	Connections int    `json:"connections"`
	Version     string `json:"version"`
	GitHash     string `json:"git_hash"`
}

func (s *Server) handleStatus(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Type", "application/json")

	st := status{
		Connections: s.ConnectionCount(),
		Version:     mysql.ServerVersion,
		GitHash:     printer.TiDBGitHash,
	}
	js, err := json.Marshal(st)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		log.Error("Encode json error", err)
	} else {
		_, err = w.Write(js)
		terror.Log(errors.Trace(err))
	}
}
<|endoftext|>


<commit_before>package server

import (
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	"testing"
)

var (
	handler     = new(CombinerServer)
	validParams = "?docs=1.pdf&callback=http://google.com?_test=true"
	invalidMsg  = "Need some docs and a callback url\n"
	validMsg    = "Started combination on [1.pdf]\n"
)

func TestPing(t *testing.T) {
	recorder := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "http://example.com/health_check.html", nil)
	handler.Ping(recorder, req)
	assert.Equal(t, 200, recorder.Code)
	assert.Equal(t, "", recorder.Body.String())
}

//TODO broken by change to JSON POST
func TestProcessCombineRequestRejectsInvalidRequest(t *testing.T) {
	recorder := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "http://example.com/", nil) // nil ref here
	handler.ProcessJob(recorder, req)
	assert.Equal(t, 400, recorder.Code)
	assert.Equal(t, invalidMsg, recorder.Body.String())
}

//TODO broken by change to JSON POST
func TestProcessCombineRequestAcceptsValidRequest(t *testing.T) {
	recorder := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "http://example.com/"+validParams, nil)
	handler.ProcessJob(recorder, req)
	assert.Equal(t, 200, recorder.Code)
	assert.Equal(t, validMsg, recorder.Body.String())
}
<commit_msg>Fix server tests<commit_after>package server

import (
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
)

var (
	handler        = new(CombinerServer)
	invalidJson    = "{"
	incompleteJson = "{}"
	validJson      = "{\"bucket_name\":\"a\",\"employer_id\":1,\"doc_list\":[\"1.pdf\"], \"callback\":\"http://a.com\"}"
)

func TestPing(t *testing.T) {
	recorder := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "http://example.com/health_check.html", nil)
	handler.Ping(recorder, req)
	assert.Equal(t, 200, recorder.Code)
	assert.Equal(t, "", recorder.Body.String())
}

func TestRejectsInvalidRequest(t *testing.T) {
	recorder := httptest.NewRecorder()
	postBody := strings.NewReader(invalidJson)
	req, _ := http.NewRequest("POST", "http://example.com/", postBody)
	handler.ProcessJob(recorder, req)
	assert.Equal(t, 400, recorder.Code)
	assert.Equal(t, recorder.Body.String(), string(invalidMessage)+"\n")
}

func TestRejectsIncompleteRequest(t *testing.T) {
	recorder := httptest.NewRecorder()
	postBody := strings.NewReader(incompleteJson)
	req, _ := http.NewRequest("POST", "http://example.com/", postBody)
	handler.ProcessJob(recorder, req)
	assert.Equal(t, 400, recorder.Code)
	assert.Equal(t, recorder.Body.String(), string(invalidMessage)+"\n")
}

func TestAcceptsValidRequest(t *testing.T) {
	recorder := httptest.NewRecorder()
	postBody := strings.NewReader(validJson)
	req, _ := http.NewRequest("POST", "http://example.com/", postBody)
	handler.ProcessJob(recorder, req)
	assert.Equal(t, 200, recorder.Code)
	assert.Equal(t, recorder.Body.String(), string(okMessage))
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/raintank/raintank-metric/metricdef"
	"github.com/raintank/raintank-metric/schema"
	"log"
	"time"
)

type Hit struct {
	Index  string                  `json:"_index"`
	Type   string                  `json:"_type"`
	Id     string                  `json:"_id"`
	Score  float64                 `json:"_score"`
	Source schema.MetricDefinition `json:"_source"`
}

type EsResult struct {
	Took     int
	TimedOut bool
	_shards  struct {
		total      int
		successful int
		failed     int
	}
	Hits struct {
		Total    int
		MaxScore int
		Hits     []Hit
	}
}

func perror(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

var esAddr = flag.String("es-addr", "localhost:9200", "elasticsearch address")
var esIndex = flag.String("es-index", "metrictank", "elasticsearch index to query")
var format = flag.String("format", "list", "format: list|vegeta-graphite|vegeta-mt")
var maxAge = flag.Int("max-age", 3600, "max age (last update diff with now) of metricdefs")
var from = flag.String("from", "30min", "from. eg '30min', '5h', '14d', etc")
var fromS uint32

func showList(ds []*schema.MetricDefinition) {
	for _, d := range ds {
		fmt.Println(d.OrgId, d.Name)
	}
}
func showVegetaGraphite(ds []*schema.MetricDefinition) {
	for _, d := range ds {
		fmt.Printf("GET http://localhost:8888/render?target=%s&from=-%s\nX-Org-Id: %d\n", d.Name, *from, d.OrgId)
	}
}
func showVegetaMT(ds []*schema.MetricDefinition) {
	from := time.Now().Add(-time.Duration(fromS) * time.Second)
	for _, d := range ds {
		if d.LastUpdate > time.Now().Unix()-int64(*maxAge) {
			fmt.Printf("GET http://localhost:18763/get?target=%s&from=%d\n", d.Id, from.Unix())
		}
	}
}

func main() {
	flag.Parse()
	var show func(ds []*schema.MetricDefinition)
	switch *format {
	case "list":
		show = showList
	case "vegeta-graphite":
		show = showVegetaGraphite
	case "vegeta-mt":
		show = showVegetaMT
	default:
		log.Fatal("invalid format")
	}
	var err error
	fromS, err = inSeconds(*from)
	perror(err)
	defs, err := metricdef.NewDefsEs(*esAddr, "", "", *esIndex)
	perror(err)
	met, scroll_id, err := defs.GetMetrics("")
	perror(err)
	show(met)
	for scroll_id != "" {
		met, scroll_id, err = defs.GetMetrics(scroll_id)
		perror(err)
		show(met)
	}
}
<commit_msg>print number of metrics to stderr<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/raintank/raintank-metric/metricdef"
	"github.com/raintank/raintank-metric/schema"
	"log"
	"os"
	"time"
)

type Hit struct {
	Index  string                  `json:"_index"`
	Type   string                  `json:"_type"`
	Id     string                  `json:"_id"`
	Score  float64                 `json:"_score"`
	Source schema.MetricDefinition `json:"_source"`
}

type EsResult struct {
	Took     int
	TimedOut bool
	_shards  struct {
		total      int
		successful int
		failed     int
	}
	Hits struct {
		Total    int
		MaxScore int
		Hits     []Hit
	}
}

func perror(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

var esAddr = flag.String("es-addr", "localhost:9200", "elasticsearch address")
var esIndex = flag.String("es-index", "metrictank", "elasticsearch index to query")
var format = flag.String("format", "list", "format: list|vegeta-graphite|vegeta-mt")
var maxAge = flag.Int("max-age", 3600, "max age (last update diff with now) of metricdefs")
var from = flag.String("from", "30min", "from. eg '30min', '5h', '14d', etc")
var silent = flag.Bool("silent", false, "silent mode (don't print number of metrics loaded to stderr)")
var fromS uint32

func showList(ds []*schema.MetricDefinition) {
	for _, d := range ds {
		fmt.Println(d.OrgId, d.Name)
	}
}
func showVegetaGraphite(ds []*schema.MetricDefinition) {
	for _, d := range ds {
		fmt.Printf("GET http://localhost:8888/render?target=%s&from=-%s\nX-Org-Id: %d\n", d.Name, *from, d.OrgId)
	}
}
func showVegetaMT(ds []*schema.MetricDefinition) {
	from := time.Now().Add(-time.Duration(fromS) * time.Second)
	for _, d := range ds {
		if d.LastUpdate > time.Now().Unix()-int64(*maxAge) {
			fmt.Printf("GET http://localhost:18763/get?target=%s&from=%d\n", d.Id, from.Unix())
		}
	}
}

func main() {
	flag.Parse()
	var show func(ds []*schema.MetricDefinition)
	switch *format {
	case "list":
		show = showList
	case "vegeta-graphite":
		show = showVegetaGraphite
	case "vegeta-mt":
		show = showVegetaMT
	default:
		log.Fatal("invalid format")
	}
	var err error
	fromS, err = inSeconds(*from)
	perror(err)
	defs, err := metricdef.NewDefsEs(*esAddr, "", "", *esIndex)
	perror(err)
	met, scroll_id, err := defs.GetMetrics("")
	perror(err)
	total := len(met)
	show(met)
	for scroll_id != "" {
		met, scroll_id, err = defs.GetMetrics(scroll_id)
		perror(err)
		show(met)
		total += len(met)
	}
	if !*silent {
		fmt.Fprintf(os.Stderr, "listed %d metrics\n", total)
	}
}
<|endoftext|>


<commit_before>//
// media.go
// Copyright 2017 Konstantin Dovnar
//
// Licensed under the Apache License, Version 2.0 (the "License");
// http://www.apache.org/licenses/LICENSE-2.0
//

package instagram

import (
	"strconv"
	"encoding/json"
)

// TypeImage is a string that define image type for media.
const TypeImage = "image"

// TypeVideo is a string that define video type for media.
const TypeVideo = "video"

const (
	image   = "GraphImage"
	video   = "GraphVideo"
	sidebar = "GraphSidecar"
)

// A Media describes an Instagram media info.
type Media struct {
	Caption       string
	Code          string
	CommentsCount uint32
	Date          uint64
	ID            string
	AD            bool
	LikesCount    uint32
	Type          string
	MediaURL      string
	Owner         Account
	MediaList     []mediaItem
}

type mediaItem struct {
	Type string
	URL  string
	Code string
}

// Update try to update media data
func (m *Media) Update() {
	media, err := GetMediaByCode(m.Code)
	if err == nil {
		*m = media
	}
}

func getFromMediaPage(data []byte) (Media, error) {
	var mediaJSON struct {
		Graphql struct {
			ShortcodeMedia struct {
				Typename   string `json:"__typename"`
				ID         string `json:"id"`
				Shortcode  string `json:"shortcode"`
				DisplayURL string `json:"display_url"`
				VideoURL   string `json:"video_url"`
				IsVideo    bool `json:"is_video"`
				EdgeMediaToCaption struct {
					Edges []struct {
						Node struct {
							Text string `json:"text"`
						} `json:"node"`
					} `json:"edges"`
				} `json:"edge_media_to_caption"`
				EdgeMediaToComment struct {
					Count int `json:"count"`
				} `json:"edge_media_to_comment"`
				TakenAtTimestamp int `json:"taken_at_timestamp"`
				EdgeMediaPreviewLike struct {
					Count int `json:"count"`
				} `json:"edge_media_preview_like"`
				Owner struct {
					ID            string `json:"id"`
					ProfilePicURL string `json:"profile_pic_url"`
					Username      string `json:"username"`
					FullName      string `json:"full_name"`
					IsPrivate     bool `json:"is_private"`
				} `json:"owner"`
				IsAd bool `json:"is_ad"`
				EdgeSidecarToChildren struct {
					Edges []struct {
						Node struct {
							Typename   string `json:"__typename"`
							ID         string `json:"id"`
							Shortcode  string `json:"shortcode"`
							DisplayURL string `json:"display_url"`
							VideoURL   string `json:"video_url"`
							IsVideo    bool `json:"is_video"`
						} `json:"node"`
					} `json:"edges"`
				} `json:"edge_sidecar_to_children"`
			} `json:"shortcode_media"`
		} `json:"graphql"`
	}

	err := json.Unmarshal(data, &mediaJSON)
	if err != nil {
		return Media{}, err
	}

	media := Media{}
	media.Code = mediaJSON.Graphql.ShortcodeMedia.Shortcode
	media.ID = mediaJSON.Graphql.ShortcodeMedia.ID
	media.AD = mediaJSON.Graphql.ShortcodeMedia.IsAd
	media.Date = uint64(mediaJSON.Graphql.ShortcodeMedia.TakenAtTimestamp)
	media.CommentsCount = uint32(mediaJSON.Graphql.ShortcodeMedia.EdgeMediaToComment.Count)
	media.LikesCount = uint32(mediaJSON.Graphql.ShortcodeMedia.EdgeMediaPreviewLike.Count)
	media.Caption = mediaJSON.Graphql.ShortcodeMedia.EdgeMediaToCaption.Edges[0].Node.Text

	var mediaType = mediaJSON.Graphql.ShortcodeMedia.Typename
	if mediaType == sidebar {
		for _, mediaItemJSON := range mediaJSON.Graphql.ShortcodeMedia.EdgeSidecarToChildren.Edges {
			var item mediaItem
			item.Code = mediaItemJSON.Node.Shortcode
			if mediaItemJSON.Node.IsVideo {
				item.URL = mediaItemJSON.Node.VideoURL
				item.Type = TypeVideo
			} else {
				item.URL = mediaItemJSON.Node.DisplayURL
				item.Type = TypeImage
			}
			media.MediaList = append(media.MediaList, item)
		}
	} else {
		if mediaType == video {
			media.Type = TypeVideo
			media.MediaURL = mediaJSON.Graphql.ShortcodeMedia.VideoURL
		} else {
			media.Type = TypeImage
			media.MediaURL = mediaJSON.Graphql.ShortcodeMedia.DisplayURL
		}
		var item mediaItem
		item.Code = media.Code
		item.Type = media.Type
		item.URL = media.MediaURL
		media.MediaList = append(media.MediaList, item)
	}

	media.Owner.ID = mediaJSON.Graphql.ShortcodeMedia.Owner.ID
	media.Owner.ProfilePicURL = mediaJSON.Graphql.ShortcodeMedia.Owner.ProfilePicURL
	media.Owner.Username = mediaJSON.Graphql.ShortcodeMedia.Owner.Username
	media.Owner.FullName = mediaJSON.Graphql.ShortcodeMedia.Owner.FullName
	media.Owner.Private = mediaJSON.Graphql.ShortcodeMedia.Owner.IsPrivate

	return media, nil
}

func getFromAccountMediaList(data []byte) (Media, error) {
	var mediaJSON struct {
		ID   string `json:"id"`
		Code string `json:"code"`
		User struct {
			ID             string `json:"id"`
			FullName       string `json:"full_name"`
			ProfilePicture string `json:"profile_picture"`
			Username       string `json:"username"`
		} `json:"user"`
		Images struct {
			StandardResolution struct {
				Width  int `json:"width"`
				Height int `json:"height"`
				URL    string `json:"url"`
			} `json:"standard_resolution"`
		} `json:"images"`
		CreatedTime string `json:"created_time"`
		Caption struct {
			Text string `json:"text"`
		} `json:"caption"`
		Likes struct {
			Count float64 `json:"count"`
		} `json:"likes"`
		Comments struct {
			Count float64 `json:"count"`
		} `json:"comments"`
		Type string `json:"type"`
		Videos struct {
			StandardResolution struct {
				Width  int `json:"width"`
				Height int `json:"height"`
				URL    string `json:"url"`
			} `json:"standard_resolution"`
		} `json:"videos"`
	}

	err := json.Unmarshal(data, &mediaJSON)
	if err != nil {
		return Media{}, err
	}

	media := Media{}
	media.Code = mediaJSON.Code
	media.ID = mediaJSON.ID
	media.Type = mediaJSON.Type
	media.Caption = mediaJSON.Caption.Text
	media.LikesCount = uint32(mediaJSON.Likes.Count)
	media.CommentsCount = uint32(mediaJSON.Comments.Count)

	date, err := strconv.ParseUint(mediaJSON.CreatedTime, 10, 64)
	if err == nil {
		media.Date = date
	}

	if media.Type == TypeVideo {
		media.MediaURL = mediaJSON.Videos.StandardResolution.URL
	} else {
		media.MediaURL = mediaJSON.Images.StandardResolution.URL
	}

	media.Owner.Username = mediaJSON.User.Username
	media.Owner.FullName = mediaJSON.User.FullName
	media.Owner.ID = mediaJSON.User.ID
	media.Owner.ProfilePicURL = mediaJSON.User.ProfilePicture

	return media, nil
}

func getFromSearchMediaList(data []byte) (Media, error) {
	var mediaJSON struct {
		CommentsDisabled bool `json:"comments_disabled"`
		ID               string `json:"id"`
		Owner struct {
			ID string `json:"id"`
		} `json:"owner"`
		ThumbnailSrc string `json:"thumbnail_src"`
		IsVideo      bool `json:"is_video"`
		Code         string `json:"code"`
		Date         float64 `json:"date"`
		DisplaySrc   string `json:"display_src"`
		Caption      string `json:"caption"`
		Comments struct {
			Count float64 `json:"count"`
		} `json:"comments"`
		Likes struct {
			Count float64 `json:"count"`
		} `json:"likes"`
	}

	err := json.Unmarshal(data, &mediaJSON)
	if err != nil {
		return Media{}, err
	}

	media := Media{}
	media.ID = mediaJSON.ID
	media.Code = mediaJSON.Code
	media.MediaURL = mediaJSON.DisplaySrc
	media.Caption = mediaJSON.Caption
	media.Date = uint64(mediaJSON.Date)
	media.LikesCount = uint32(mediaJSON.Likes.Count)
	media.CommentsCount = uint32(mediaJSON.Comments.Count)
	media.Owner.ID = mediaJSON.Owner.ID

	if mediaJSON.IsVideo {
		media.Type = TypeVideo
	} else {
		media.Type = TypeImage
	}

	return media, nil
}
<commit_msg>Get media collection from account media page<commit_after>//
// media.go
// Copyright 2017 Konstantin Dovnar
//
// Licensed under the Apache License, Version 2.0 (the "License");
// http://www.apache.org/licenses/LICENSE-2.0
//

package instagram

import (
	"strconv"
	"encoding/json"
)

// TypeImage is a string that define image type for media.
const TypeImage = "image"

// TypeVideo is a string that define video type for media.
const TypeVideo = "video"

// TypeCarousel is a string that define carousel (collection of media) type for media.
const TypeCarousel = "carousel"

const (
	video   = "GraphVideo"
	sidebar = "GraphSidecar"

	carousel = "carousel"
)

// A Media describes an Instagram media info.
type Media struct {
	Caption       string
	Code          string
	CommentsCount uint32
	Date          uint64
	ID            string
	AD            bool
	LikesCount    uint32
	Type          string
	MediaURL      string
	Owner         Account
	MediaList     []mediaItem
}

type mediaItem struct {
	Type string
	URL  string
	Code string
}

// Update try to update media data
func (m *Media) Update() {
	media, err := GetMediaByCode(m.Code)
	if err == nil {
		*m = media
	}
}

func getFromMediaPage(data []byte) (Media, error) {
	var mediaJSON struct {
		Graphql struct {
			ShortcodeMedia struct {
				Typename   string `json:"__typename"`
				ID         string `json:"id"`
				Shortcode  string `json:"shortcode"`
				DisplayURL string `json:"display_url"`
				VideoURL   string `json:"video_url"`
				IsVideo    bool `json:"is_video"`
				EdgeMediaToCaption struct {
					Edges []struct {
						Node struct {
							Text string `json:"text"`
						} `json:"node"`
					} `json:"edges"`
				} `json:"edge_media_to_caption"`
				EdgeMediaToComment struct {
					Count int `json:"count"`
				} `json:"edge_media_to_comment"`
				TakenAtTimestamp int `json:"taken_at_timestamp"`
				EdgeMediaPreviewLike struct {
					Count int `json:"count"`
				} `json:"edge_media_preview_like"`
				Owner struct {
					ID            string `json:"id"`
					ProfilePicURL string `json:"profile_pic_url"`
					Username      string `json:"username"`
					FullName      string `json:"full_name"`
					IsPrivate     bool `json:"is_private"`
				} `json:"owner"`
				IsAd bool `json:"is_ad"`
				EdgeSidecarToChildren struct {
					Edges []struct {
						Node struct {
							Typename   string `json:"__typename"`
							ID         string `json:"id"`
							Shortcode  string `json:"shortcode"`
							DisplayURL string `json:"display_url"`
							VideoURL   string `json:"video_url"`
							IsVideo    bool `json:"is_video"`
						} `json:"node"`
					} `json:"edges"`
				} `json:"edge_sidecar_to_children"`
			} `json:"shortcode_media"`
		} `json:"graphql"`
	}

	err := json.Unmarshal(data, &mediaJSON)
	if err != nil {
		return Media{}, err
	}

	media := Media{}
	media.Code = mediaJSON.Graphql.ShortcodeMedia.Shortcode
	media.ID = mediaJSON.Graphql.ShortcodeMedia.ID
	media.AD = mediaJSON.Graphql.ShortcodeMedia.IsAd
	media.Date = uint64(mediaJSON.Graphql.ShortcodeMedia.TakenAtTimestamp)
	media.CommentsCount = uint32(mediaJSON.Graphql.ShortcodeMedia.EdgeMediaToComment.Count)
	media.LikesCount = uint32(mediaJSON.Graphql.ShortcodeMedia.EdgeMediaPreviewLike.Count)
	media.Caption = mediaJSON.Graphql.ShortcodeMedia.EdgeMediaToCaption.Edges[0].Node.Text

	var mediaType = mediaJSON.Graphql.ShortcodeMedia.Typename
	if mediaType == sidebar {
		for _, mediaItemJSON := range mediaJSON.Graphql.ShortcodeMedia.EdgeSidecarToChildren.Edges {
			var item mediaItem
			item.Code = mediaItemJSON.Node.Shortcode
			if mediaItemJSON.Node.IsVideo {
				item.URL = mediaItemJSON.Node.VideoURL
				item.Type = TypeVideo
			} else {
				item.URL = mediaItemJSON.Node.DisplayURL
				item.Type = TypeImage
			}
			media.MediaList = append(media.MediaList, item)
		}
		media.Type = TypeCarousel
	} else {
		if mediaType == video {
			media.Type = TypeVideo
			media.MediaURL = mediaJSON.Graphql.ShortcodeMedia.VideoURL
		} else {
			media.Type = TypeImage
			media.MediaURL = mediaJSON.Graphql.ShortcodeMedia.DisplayURL
		}
		var item mediaItem
		item.Code = media.Code
		item.Type = media.Type
		item.URL = media.MediaURL
		media.MediaList = append(media.MediaList, item)
	}

	media.Owner.ID = mediaJSON.Graphql.ShortcodeMedia.Owner.ID
	media.Owner.ProfilePicURL = mediaJSON.Graphql.ShortcodeMedia.Owner.ProfilePicURL
	media.Owner.Username = mediaJSON.Graphql.ShortcodeMedia.Owner.Username
	media.Owner.FullName = mediaJSON.Graphql.ShortcodeMedia.Owner.FullName
	media.Owner.Private = mediaJSON.Graphql.ShortcodeMedia.Owner.IsPrivate

	return media, nil
}

func getFromAccountMediaList(data []byte) (Media, error) {
	var mediaJSON struct {
		ID   string `json:"id"`
		Code string `json:"code"`
		User struct {
			ID             string `json:"id"`
			FullName       string `json:"full_name"`
			ProfilePicture string `json:"profile_picture"`
			Username       string `json:"username"`
		} `json:"user"`
		Images struct {
			StandardResolution struct {
				Width  int `json:"width"`
				Height int `json:"height"`
				URL    string `json:"url"`
			} `json:"standard_resolution"`
		} `json:"images"`
		CreatedTime string `json:"created_time"`
		Caption struct {
			Text string `json:"text"`
		} `json:"caption"`
		Likes struct {
			Count float64 `json:"count"`
		} `json:"likes"`
		Comments struct {
			Count float64 `json:"count"`
		} `json:"comments"`
		Type string `json:"type"`
		Videos struct {
			StandardResolution struct {
				Width  int `json:"width"`
				Height int `json:"height"`
				URL    string `json:"url"`
			} `json:"standard_resolution"`
		} `json:"videos"`
		CarouselMedia []struct {
			Images struct {
				StandardResolution struct {
					URL string `json:"url"`
				} `json:"standard_resolution"`
			} `json:"images"`
			UsersInPhoto []interface{} `json:"users_in_photo"`
			Type         string `json:"type"`
		} `json:"carousel_media"`
	}

	err := json.Unmarshal(data, &mediaJSON)
	if err != nil {
		return Media{}, err
	}

	media := Media{}
	media.Code = mediaJSON.Code
	media.ID = mediaJSON.ID
	media.Caption = mediaJSON.Caption.Text
	media.LikesCount = uint32(mediaJSON.Likes.Count)
	media.CommentsCount = uint32(mediaJSON.Comments.Count)

	date, err := strconv.ParseUint(mediaJSON.CreatedTime, 10, 64)
	if err == nil {
		media.Date = date
	}

	if mediaJSON.Type == carousel {
		media.Type = TypeCarousel
		for _, itemJSOM := range mediaJSON.CarouselMedia {
			var item mediaItem
			item.Type = itemJSOM.Type
			item.URL = itemJSOM.Images.StandardResolution.URL + ".jpg"
			media.MediaList = append(media.MediaList, item)
		}
	} else {
		if mediaJSON.Type == TypeVideo {
			media.MediaURL = mediaJSON.Videos.StandardResolution.URL
			media.Type = TypeVideo
		} else {
			media.MediaURL = mediaJSON.Images.StandardResolution.URL
			media.Type = TypeImage
		}
		var item mediaItem
		item.Type = media.Type
		item.URL = media.MediaURL
		item.Code = media.Code
		media.MediaList = append(media.MediaList, item)
	}

	media.Owner.Username = mediaJSON.User.Username
	media.Owner.FullName = mediaJSON.User.FullName
	media.Owner.ID = mediaJSON.User.ID
	media.Owner.ProfilePicURL = mediaJSON.User.ProfilePicture

	return media, nil
}

func getFromSearchMediaList(data []byte) (Media, error) {
	var mediaJSON struct {
		CommentsDisabled bool `json:"comments_disabled"`
		ID               string `json:"id"`
		Owner struct {
			ID string `json:"id"`
		} `json:"owner"`
		ThumbnailSrc string `json:"thumbnail_src"`
		IsVideo      bool `json:"is_video"`
		Code         string `json:"code"`
		Date         float64 `json:"date"`
		DisplaySrc   string `json:"display_src"`
		Caption      string `json:"caption"`
		Comments struct {
			Count float64 `json:"count"`
		} `json:"comments"`
		Likes struct {
			Count float64 `json:"count"`
		} `json:"likes"`
	}

	err := json.Unmarshal(data, &mediaJSON)
	if err != nil {
		return Media{}, err
	}

	media := Media{}
	media.ID = mediaJSON.ID
	media.Code = mediaJSON.Code
	media.MediaURL = mediaJSON.DisplaySrc
	media.Caption = mediaJSON.Caption
	media.Date = uint64(mediaJSON.Date)
	media.LikesCount = uint32(mediaJSON.Likes.Count)
	media.CommentsCount = uint32(mediaJSON.Comments.Count)
	media.Owner.ID = mediaJSON.Owner.ID

	if mediaJSON.IsVideo {
		media.Type = TypeVideo
	} else {
		media.Type = TypeImage
	}

	return media, nil
}
<|endoftext|>


<commit_before>package adapters

import (
	"fmt"
	"html/template"
	"os"
	"path/filepath"
	"strings"

	"github.com/gosimple/slug"
	"github.com/russross/blackfriday"
	"github.com/termie/go-shutil"

	"mirovarga.com/litepub/application"
	"mirovarga.com/litepub/domain"
)

// ProgressFunc is used to monitor progress of generating a Blog. It is called
// before a file generation is started.
type ProgressFunc func(path string)

// StaticBlogGenerator generates Blogs to static HTML files.
// TODO add docs from README
type StaticBlogGenerator struct {
	id            string
	templatesDir  string
	outputDir     string
	progressFunc  ProgressFunc
	readers       application.Readers
	indexTemplate *template.Template
	postTemplate  *template.Template
	tagTemplate   *template.Template
	posts         []domain.Post
	postsByTag    map[string][]domain.Post
}

// Generate generates a Blog to static HTML files.
func (g StaticBlogGenerator) Generate() error {
	err := g.prepareOutputDir()
	if err != nil {
		return fmt.Errorf("Failed to prepare output directory: %s", err)
	}

	err = g.readPosts()
	if err != nil {
		return fmt.Errorf("Failed to read posts: %s", err)
	}

	err = g.generateIndex()
	if err != nil {
		return fmt.Errorf("Failed to generate index: %s", err)
	}

	err = g.generateTags()
	if err != nil {
		return fmt.Errorf("Failed to generate tags: %s", err)
	}

	err = g.generatePosts()
	if err != nil {
		return fmt.Errorf("Failed to generate posts: %s", err)
	}

	return nil
}

func (g StaticBlogGenerator) prepareOutputDir() error {
	os.RemoveAll(g.outputDir)

	err := shutil.CopyTree(g.templatesDir, g.outputDir,
		&shutil.CopyTreeOptions{
			Symlinks: true,
			Ignore: func(string, []os.FileInfo) []string {
				return []string{"layout.tmpl", "index.tmpl", "post.tmpl", "tag.tmpl"}
			},
			CopyFunction:           shutil.Copy,
			IgnoreDanglingSymlinks: false,
		})
	if err != nil {
		return err
	}

	os.Mkdir(filepath.Join(g.outputDir, "tags"), 0700)

	return nil
}

func (g *StaticBlogGenerator) readPosts() error {
	blog, err := g.readers.GetBlog(g.id)
	if err != nil {
		return err
	}

	g.posts = blog.PostsByDate(false, false)

	for _, tag := range blog.Tags(false) {
		g.postsByTag[tag] = blog.PostsByDate(false, false, tag)
	}

	return nil
}

func (g StaticBlogGenerator) generateIndex() error {
	return g.generatePage(g.indexTemplate, "index.html", g.posts)
}

func (g StaticBlogGenerator) generatePosts() error {
	for _, post := range g.posts {
		err := g.generatePage(g.postTemplate, slug.Make(post.Title)+".html", post)
		if err != nil {
			return err
		}
	}

	return nil
}

func (g StaticBlogGenerator) generateTags() error {
	for tag, posts := range g.postsByTag {
		err := g.generatePage(g.tagTemplate,
			filepath.Join("tags", slug.Make(tag)+".html"), struct {
				Name  string
				Posts []domain.Post
			}{tag, posts})
		if err != nil {
			return err
		}
	}

	return nil
}

func (g StaticBlogGenerator) generatePage(template *template.Template,
	path string, data interface{}) error {
	g.progressFunc(path)

	pageFile, err := os.OpenFile(filepath.Join(g.outputDir, path),
		os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		return err
	}
	defer pageFile.Close()

	return template.Execute(pageFile, data)
}

// NewStaticBlogGenerator creates a StaticBlogGenerator that generates the Blog
// with the ID to static HTML files in the outputDir using templates from
// the templatesDir.
func NewStaticBlogGenerator(id, templatesDir, outputDir string,
	readers application.Readers) (StaticBlogGenerator, error) {
	return NewStaticBlogGeneratorWithProgress(id, templatesDir, outputDir, nil, readers)
}

// NewStaticBlogGeneratorWithProgress creates a StaticBlogGenerator that
// generates the Blog with the ID to static HTML files in the outputDir using
// templates from the templatesDir. It calls the progressFunc before generating
// each file.
func NewStaticBlogGeneratorWithProgress(id, templatesDir, outputDir string,
	progressFunc ProgressFunc, readers application.Readers) (StaticBlogGenerator, error) {
	if _, err := os.Stat(templatesDir); err != nil {
		return StaticBlogGenerator{},
			fmt.Errorf("Templates directory not found: %s", templatesDir)
	}

	indexTemplate, err := createTemplate(templatesDir, "index.tmpl")
	if err != nil {
		return StaticBlogGenerator{}, err
	}

	postTemplate, err := createTemplate(templatesDir, "post.tmpl")
	if err != nil {
		return StaticBlogGenerator{}, err
	}

	tagTemplate, err := createTemplate(templatesDir, "tag.tmpl")
	if err != nil {
		return StaticBlogGenerator{}, err
	}

	return StaticBlogGenerator{id, templatesDir, outputDir, progressFunc,
		readers, indexTemplate, postTemplate, tagTemplate, []domain.Post{},
		make(map[string][]domain.Post)}, nil
}

func createTemplate(dir, name string) (*template.Template, error) {
	return template.New("layout.tmpl").Funcs(templateFuncs).ParseFiles(
		filepath.Join(dir, "layout.tmpl"),
		filepath.Join(dir, name))
}

var templateFuncs = template.FuncMap{
	"html":    html,
	"summary": summary,
	"even":    even,
	"inc":     inc,
	"slug":    slugify,
}

func html(markdown string) template.HTML {
	html := blackfriday.MarkdownCommon([]byte(markdown))
	return template.HTML(html)
}

func summary(content string) string {
	lines := strings.Split(content, "\n\n")
	for _, line := range lines {
		if !strings.HasPrefix(line, "#") {
			return line
		}
	}
	return content
}

func even(integer int) bool {
	return integer%2 == 0
}

func inc(integer int) int {
	return integer + 1
}

func slugify(str string) string {
	return slug.Make(str)
}
<commit_msg>Updated a TODO<commit_after>package adapters

import (
	"fmt"
	"html/template"
	"os"
	"path/filepath"
	"strings"

	"github.com/gosimple/slug"
	"github.com/russross/blackfriday"
	"github.com/termie/go-shutil"

	"mirovarga.com/litepub/application"
	"mirovarga.com/litepub/domain"
)

// ProgressFunc is used to monitor progress of generating a Blog. It is called
// before a file generation is started.
type ProgressFunc func(path string)

// StaticBlogGenerator generates Blogs to static HTML files.
// TODO add docs from litepub.com
type StaticBlogGenerator struct {
	id            string
	templatesDir  string
	outputDir     string
	progressFunc  ProgressFunc
	readers       application.Readers
	indexTemplate *template.Template
	postTemplate  *template.Template
	tagTemplate   *template.Template
	posts         []domain.Post
	postsByTag    map[string][]domain.Post
}

// Generate generates a Blog to static HTML files.
func (g StaticBlogGenerator) Generate() error {
	err := g.prepareOutputDir()
	if err != nil {
		return fmt.Errorf("Failed to prepare output directory: %s", err)
	}

	err = g.readPosts()
	if err != nil {
		return fmt.Errorf("Failed to read posts: %s", err)
	}

	err = g.generateIndex()
	if err != nil {
		return fmt.Errorf("Failed to generate index: %s", err)
	}

	err = g.generateTags()
	if err != nil {
		return fmt.Errorf("Failed to generate tags: %s", err)
	}

	err = g.generatePosts()
	if err != nil {
		return fmt.Errorf("Failed to generate posts: %s", err)
	}

	return nil
}

func (g StaticBlogGenerator) prepareOutputDir() error {
	os.RemoveAll(g.outputDir)

	err := shutil.CopyTree(g.templatesDir, g.outputDir,
		&shutil.CopyTreeOptions{
			Symlinks: true,
			Ignore: func(string, []os.FileInfo) []string {
				return []string{"layout.tmpl", "index.tmpl", "post.tmpl", "tag.tmpl"}
			},
			CopyFunction:           shutil.Copy,
			IgnoreDanglingSymlinks: false,
		})
	if err != nil {
		return err
	}

	os.Mkdir(filepath.Join(g.outputDir, "tags"), 0700)

	return nil
}

func (g *StaticBlogGenerator) readPosts() error {
	blog, err := g.readers.GetBlog(g.id)
	if err != nil {
		return err
	}

	g.posts = blog.PostsByDate(false, false)

	for _, tag := range blog.Tags(false) {
		g.postsByTag[tag] = blog.PostsByDate(false, false, tag)
	}

	return nil
}

func (g StaticBlogGenerator) generateIndex() error {
	return g.generatePage(g.indexTemplate, "index.html", g.posts)
}

func (g StaticBlogGenerator) generatePosts() error {
	for _, post := range g.posts {
		err := g.generatePage(g.postTemplate, slug.Make(post.Title)+".html", post)
		if err != nil {
			return err
		}
	}

	return nil
}

func (g StaticBlogGenerator) generateTags() error {
	for tag, posts := range g.postsByTag {
		err := g.generatePage(g.tagTemplate,
			filepath.Join("tags", slug.Make(tag)+".html"), struct {
				Name  string
				Posts []domain.Post
			}{tag, posts})
		if err != nil {
			return err
		}
	}

	return nil
}

func (g StaticBlogGenerator) generatePage(template *template.Template,
	path string, data interface{}) error {
	g.progressFunc(path)

	pageFile, err := os.OpenFile(filepath.Join(g.outputDir, path),
		os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		return err
	}
	defer pageFile.Close()

	return template.Execute(pageFile, data)
}

// NewStaticBlogGenerator creates a StaticBlogGenerator that generates the Blog
// with the ID to static HTML files in the outputDir using templates from
// the templatesDir.
func NewStaticBlogGenerator(id, templatesDir, outputDir string,
	readers application.Readers) (StaticBlogGenerator, error) {
	return NewStaticBlogGeneratorWithProgress(id, templatesDir, outputDir, nil, readers)
}

// NewStaticBlogGeneratorWithProgress creates a StaticBlogGenerator that
// generates the Blog with the ID to static HTML files in the outputDir using
// templates from the templatesDir. It calls the progressFunc before generating
// each file.
func NewStaticBlogGeneratorWithProgress(id, templatesDir, outputDir string,
	progressFunc ProgressFunc, readers application.Readers) (StaticBlogGenerator, error) {
	if _, err := os.Stat(templatesDir); err != nil {
		return StaticBlogGenerator{},
			fmt.Errorf("Templates directory not found: %s", templatesDir)
	}

	indexTemplate, err := createTemplate(templatesDir, "index.tmpl")
	if err != nil {
		return StaticBlogGenerator{}, err
	}

	postTemplate, err := createTemplate(templatesDir, "post.tmpl")
	if err != nil {
		return StaticBlogGenerator{}, err
	}

	tagTemplate, err := createTemplate(templatesDir, "tag.tmpl")
	if err != nil {
		return StaticBlogGenerator{}, err
	}

	return StaticBlogGenerator{id, templatesDir, outputDir, progressFunc,
		readers, indexTemplate, postTemplate, tagTemplate, []domain.Post{},
		make(map[string][]domain.Post)}, nil
}

func createTemplate(dir, name string) (*template.Template, error) {
	return template.New("layout.tmpl").Funcs(templateFuncs).ParseFiles(
		filepath.Join(dir, "layout.tmpl"),
		filepath.Join(dir, name))
}

var templateFuncs = template.FuncMap{
	"html":    html,
	"summary": summary,
	"even":    even,
	"inc":     inc,
	"slug":    slugify,
}

func html(markdown string) template.HTML {
	html := blackfriday.MarkdownCommon([]byte(markdown))
	return template.HTML(html)
}

func summary(content string) string {
	lines := strings.Split(content, "\n\n")
	for _, line := range lines {
		if !strings.HasPrefix(line, "#") {
			return line
		}
	}
	return content
}

func even(integer int) bool {
	return integer%2 == 0
}

func inc(integer int) int {
	return integer + 1
}

func slugify(str string) string {
	return slug.Make(str)
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package interfacer

import (
	"bytes"
	"flag"
	"go/build"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

var (
	name = flag.String("name", "", "name of the test to run")
)

func want(t *testing.T, p string) (string, bool) {
	if !strings.HasPrefix(p, "./") && !strings.HasSuffix(p, ".go") {
		p = filepath.Join("src", p)
	}
	if strings.HasSuffix(p, "/...") {
		p = p[:len(p)-4]
	}
	outBytes, err := ioutil.ReadFile(p + ".out")
	if err == nil {
		return string(outBytes), false
	}
	if !os.IsNotExist(err) {
		t.Fatal(err)
	}
	errBytes, err := ioutil.ReadFile(p + ".err")
	if err == nil {
		return string(errBytes), true
	}
	if !os.IsNotExist(err) {
		t.Fatal(err)
	}
	t.Fatalf("Output file not found for %s", p)
	return "", false
}

func doTest(t *testing.T, p string) {
	exp, wantErr := want(t, p)
	if strings.HasPrefix(exp, "/") {
		exp = build.Default.GOPATH + exp
	}
	doTestWant(t, p, exp, wantErr, p)
}

func endNewline(s string) string {
	if s[len(s)-1] == '\n' {
		return s
	}
	return s + "\n"
}

func doTestWant(t *testing.T, name, exp string, wantErr bool, args ...string) {
	var b bytes.Buffer
	err := CheckArgs(args, &b, true)
	exp = endNewline(exp)
	if wantErr {
		if err == nil {
			t.Fatalf("Wanted error in %s, but none found.", name)
		}
		got := endNewline(err.Error())
		if exp != got {
			t.Fatalf("Error mismatch in %s:\nExpected:\n%sGot:\n%s",
				name, exp, got)
		}
		return
	}
	if err != nil {
		t.Fatalf("Did not want error in %s:\n%v", name, err)
	}
	got := endNewline(b.String())
	if exp != got {
		t.Fatalf("Output mismatch in %s:\nExpected:\n%sGot:\n%s",
			name, exp, got)
	}
}

func TestAll(t *testing.T) {
	if err := os.Chdir("testdata"); err != nil {
		t.Fatal(err)
	}
	defer func() {
		if err := os.Chdir(".."); err != nil {
			t.Fatal(err)
		}
	}()
	wd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	build.Default.GOPATH = wd
	if *name != "" {
		doTest(t, *name)
		return
	}
	paths, err := filepath.Glob("*")
	if err != nil {
		t.Fatal(err)
	}
	for _, p := range paths {
		if strings.HasSuffix(p, ".out") || strings.HasSuffix(p, ".err") {
			continue
		}
		if p == "src" {
			continue
		}
		if strings.HasSuffix(p, ".go") {
			// Go file
			doTest(t, p)
		} else {
			// local recursive
			doTest(t, "./"+p+"/...")
		}
	}
	dirs, err := filepath.Glob("src/*")
	if err != nil {
		t.Fatal(err)
	}
	for _, d := range dirs {
		if strings.HasSuffix(d, ".out") || strings.HasSuffix(d, ".err") {
			continue
		}
		// non-local recursive
		doTest(t, d[4:]+"/...")
		// local recursive
		doTest(t, "./"+d+"/...")
	}
	// local non-recursive
	doTest(t, "./single")
	// non-local non-recursive
	doTest(t, "single")
	// non-existent Go file
	doTest(t, "missing.go")
	// local non-existent non-recursive
	doTest(t, "./missing")
	// non-local non-existent non-recursive
	doTest(t, "missing")
	// local non-existent recursive
	doTest(t, "./missing-rec/...")
	doTestWant(t, "wrong-args", "named files must be .go files: bar", true, "foo.go", "bar")
}
<commit_msg>Don't crash on test cases with empty output<commit_after>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package interfacer

import (
	"bytes"
	"flag"
	"go/build"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

var (
	name = flag.String("name", "", "name of the test to run")
)

func want(t *testing.T, p string) (string, bool) {
	if !strings.HasPrefix(p, "./") && !strings.HasSuffix(p, ".go") {
		p = filepath.Join("src", p)
	}
	if strings.HasSuffix(p, "/...") {
		p = p[:len(p)-4]
	}
	outBytes, err := ioutil.ReadFile(p + ".out")
	if err == nil {
		return string(outBytes), false
	}
	if !os.IsNotExist(err) {
		t.Fatal(err)
	}
	errBytes, err := ioutil.ReadFile(p + ".err")
	if err == nil {
		return string(errBytes), true
	}
	if !os.IsNotExist(err) {
		t.Fatal(err)
	}
	t.Fatalf("Output file not found for %s", p)
	return "", false
}

func doTest(t *testing.T, p string) {
	exp, wantErr := want(t, p)
	if strings.HasPrefix(exp, "/") {
		exp = build.Default.GOPATH + exp
	}
	doTestWant(t, p, exp, wantErr, p)
}

func endNewline(s string) string {
	if strings.HasSuffix(s, "\n") {
		return s
	}
	return s + "\n"
}

func doTestWant(t *testing.T, name, exp string, wantErr bool, args ...string) {
	var b bytes.Buffer
	err := CheckArgs(args, &b, true)
	exp = endNewline(exp)
	if wantErr {
		if err == nil {
			t.Fatalf("Wanted error in %s, but none found.", name)
		}
		got := endNewline(err.Error())
		if exp != got {
			t.Fatalf("Error mismatch in %s:\nExpected:\n%sGot:\n%s",
				name, exp, got)
		}
		return
	}
	if err != nil {
		t.Fatalf("Did not want error in %s:\n%v", name, err)
	}
	got := endNewline(b.String())
	if exp != got {
		t.Fatalf("Output mismatch in %s:\nExpected:\n%sGot:\n%s",
			name, exp, got)
	}
}

func TestAll(t *testing.T) {
	if err := os.Chdir("testdata"); err != nil {
		t.Fatal(err)
	}
	defer func() {
		if err := os.Chdir(".."); err != nil {
			t.Fatal(err)
		}
	}()
	wd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	build.Default.GOPATH = wd
	if *name != "" {
		doTest(t, *name)
		return
	}
	paths, err := filepath.Glob("*")
	if err != nil {
		t.Fatal(err)
	}
	for _, p := range paths {
		if strings.HasSuffix(p, ".out") || strings.HasSuffix(p, ".err") {
			continue
		}
		if p == "src" {
			continue
		}
		if strings.HasSuffix(p, ".go") {
			// Go file
			doTest(t, p)
		} else {
			// local recursive
			doTest(t, "./"+p+"/...")
		}
	}
	dirs, err := filepath.Glob("src/*")
	if err != nil {
		t.Fatal(err)
	}
	for _, d := range dirs {
		if strings.HasSuffix(d, ".out") || strings.HasSuffix(d, ".err") {
			continue
		}
		// non-local recursive
		doTest(t, d[4:]+"/...")
		// local recursive
		doTest(t, "./"+d+"/...")
	}
	// local non-recursive
	doTest(t, "./single")
	// non-local non-recursive
	doTest(t, "single")
	// non-existent Go file
	doTest(t, "missing.go")
	// local non-existent non-recursive
	doTest(t, "./missing")
	// non-local non-existent non-recursive
	doTest(t, "missing")
	// local non-existent recursive
	doTest(t, "./missing-rec/...")
	doTestWant(t, "wrong-args", "named files must be .go files: bar", true, "foo.go", "bar")
}
<|endoftext|>


<commit_before>package intervals

import mt "github.com/brettbuddin/musictheory"

// Scales
var (
	Chromatic,
	Major,
	Minor,
	MajorPentatonic,
	MinorPentatonic,
	Ionian,
	Dorian,
	Phrygian,
	Aeolian,
	Lydian,
	Mixolydian,
	Locrian []mt.Interval
)

func init() {
	P1 := mt.Perfect(1)
	P4 := mt.Perfect(4)
	P5 := mt.Perfect(5)

	M2 := mt.Major(2)
	M3 := mt.Major(3)
	M6 := mt.Major(6)
	M7 := mt.Major(7)

	m2 := mt.Minor(2)
	m3 := mt.Minor(3)
	m6 := mt.Minor(6)
	m7 := mt.Minor(7)

	A4 := mt.Augmented(4)
	d5 := mt.Diminished(5)

	Chromatic = []mt.Interval{P1, m2, M2, m3, M3, P4, A4, P5, m6, M6, m7, M7}

	Ionian = []mt.Interval{P1, M2, M3, P4, P5, M6, M7}
	Major = Ionian

	Dorian = []mt.Interval{P1, M2, m3, P4, P5, M6, m7}
	Phrygian = []mt.Interval{P1, m2, m3, P4, P5, m6, m7}
	Lydian = []mt.Interval{P1, M2, M3, A4, P5, M6, M7}
	Mixolydian = []mt.Interval{P1, M2, M3, P4, P5, M6, m7}

	Aeolian = []mt.Interval{P1, M2, m3, P4, P5, m6, m7}
	Minor = Aeolian

	MajorPentatonic = []mt.Interval{P1, M2, M3, P5, M6}
	MinorPentatonic = []mt.Interval{P1, m3, P4, P5, m7}

	Locrian = []mt.Interval{P1, m2, m3, P4, d5, m6, m7}
}
<commit_msg>Additional scales.<commit_after>package intervals

import mt "github.com/brettbuddin/musictheory"

// Scales
var (
	Aeolian,
	Chromatic,
	DominantBebop,
	Dorian,
	DoubleHarmonic,
	HarmonicMinor,
	HarmonicMinorBebop,
	InSen,
	Ionian,
	Lydian,
	Major,
	MajorBebop,
	MajorPentatonic,
	MelodicMinorBebop,
	Minor,
	MinorPentatonic,
	Mixolydian,
	Phrygian,
	Locrian,
	WholeTone []mt.Interval
)

func init() {
	var (
		A4 = mt.Augmented(4)
		A5 = mt.Augmented(5)
		M2 = mt.Major(2)
		M3 = mt.Major(3)
		M6 = mt.Major(6)
		M7 = mt.Major(7)
		P1 = mt.Perfect(1)
		P4 = mt.Perfect(4)
		P5 = mt.Perfect(5)
		d5 = mt.Diminished(5)
		d7 = mt.Diminished(7)
		m2 = mt.Minor(2)
		m3 = mt.Minor(3)
		m6 = mt.Minor(6)
		m7 = mt.Minor(7)
	)

	Aeolian = []mt.Interval{P1, M2, m3, P4, P5, m6, m7}
	Chromatic = []mt.Interval{P1, m2, M2, m3, M3, P4, A4, P5, m6, M6, m7, M7}
	DominantBebop = []mt.Interval{P1, M2, M3, P4, P5, M6, m7, M7}
	Dorian = []mt.Interval{P1, M2, m3, P4, P5, M6, m7}
	DoubleHarmonic = []mt.Interval{m2, M3, P4, P5, m6, M7}
	HarmonicMinor = []mt.Interval{P1, M2, m3, P4, P5, m6, M7}
	HarmonicMinorBebop = []mt.Interval{P1, M2, m3, P4, P5, M6, d7, m7}
	InSen = []mt.Interval{P1, m2, P4, P5, m7}
	Ionian = []mt.Interval{P1, M2, M3, P4, P5, M6, M7}
	Locrian = []mt.Interval{P1, m2, m3, P4, d5, m6, m7}
	Lydian = []mt.Interval{P1, M2, M3, A4, P5, M6, M7}
	MajorBebop = []mt.Interval{P1, M2, M3, P4, P5, A5, M6, M7}
	MajorPentatonic = []mt.Interval{P1, M2, M3, P5, M6}
	MelodicMinorBebop = []mt.Interval{P1, M2, m3, P4, P5, m6, M6, M7}
	MinorPentatonic = []mt.Interval{P1, m3, P4, P5, m7}
	Mixolydian = []mt.Interval{P1, M2, M3, P4, P5, M6, m7}
	Phrygian = []mt.Interval{P1, m2, m3, P4, P5, m6, m7}
	WholeTone = []mt.Interval{P1, M2, M3, d5, m6, m7}

	Major = Ionian
	Minor = Aeolian
}
<|endoftext|>