<commit_before>// Application that does metrics analysis as described in the design doc:
// go/ct_metrics_analysis
//
// Can be tested locally with:
// $ go run go/worker_scripts/metrics_analysis/main.go --start_range=1 --num=3 --run_id=rmistry-test1 --benchmark_extra_args="--output-format=csv" --metric_name="loadingMetric" --logtostderr=true --local
//
package main

import (
	"bytes"
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/ct/go/worker_scripts/worker_common"
	"go.skia.org/infra/go/sklog"
	skutil "go.skia.org/infra/go/util"
)

const (
	// The number of goroutines that will run in parallel to download traces and
	// run metrics analysis.
	WORKER_POOL_SIZE = 5

	METRICS_BENCHMARK_TIMEOUT_SECS = 300

	TRACE_OUTPUT_BUCKET = "chrome-telemetry-output"
)

var (
	startRange         = flag.Int("start_range", 1, "The number this worker will run metrics analysis from.")
	num                = flag.Int("num", 100, "The total number of traces to run metrics analysis from the start_range.")
	runID              = flag.String("run_id", "", "The unique run id (typically requester + timestamp).")
	benchmarkExtraArgs = flag.String("benchmark_extra_args", "", "The extra arguments that are passed to the specified benchmark.")
	metricName         = flag.String("metric_name", "", "The metric to parse the traces with. Eg: loadingMetric")
	valueColumnName    = flag.String("value_column_name", "", "Which column's entries to use as field values when combining CSVs.")
)

func metricsAnalysis() error {
	ctx := context.Background()
	worker_common.Init(ctx)
	if !*worker_common.Local {
		defer util.CleanTmpDir()
	}
	defer util.TimeTrack(time.Now(), "Metrics Analysis")
	defer sklog.Flush()

	// Validate required arguments.
	if *runID == "" {
		return errors.New("Must specify --run_id")
	}
	if *metricName == "" {
		return errors.New("Must specify --metric_name")
	}

	// Use defaults.
	if *valueColumnName == "" {
		*valueColumnName = util.DEFAULT_VALUE_COLUMN_NAME
	}

	// Instantiate GcsUtil object.
	gs, err := util.NewGcsUtil(nil)
	if err != nil {
		return fmt.Errorf("GcsUtil instantiation failed: %s", err)
	}

	// Download the trace URLs for this run from Google storage.
	tracesFilename := *runID + ".traces.csv"
	skutil.MkdirAll(util.PagesetsDir, 0700)
	tmpDir, err := ioutil.TempDir(util.PagesetsDir, "traces")
	if err != nil {
		return fmt.Errorf("Could not create tmpdir: %s", err)
	}
	defer skutil.RemoveAll(tmpDir)
	remotePatchesDir := filepath.Join(util.BenchmarkRunsDir, *runID)

	// Download traces.
	if _, err := util.DownloadPatch(filepath.Join(tmpDir, tracesFilename), filepath.Join(remotePatchesDir, tracesFilename), gs); err != nil {
		return fmt.Errorf("Could not download %s: %s", tracesFilename, err)
	}
	traces, err := util.GetCustomPages(filepath.Join(tmpDir, tracesFilename))
	if err != nil {
		return fmt.Errorf("Could not read custom traces file %s: %s", tracesFilename, err)
	}
	if len(traces) == 0 {
		return errors.New(fmt.Sprintf("No traces found in %s", tracesFilename))
	}
	traces = util.GetCustomPagesWithinRange(*startRange, *num, traces)
	sklog.Infof("Using %d traces", len(traces))

	// Establish output paths for trace downloads and metrics.
	traceDownloadDir := filepath.Join(util.StorageDir, util.TraceDownloadsDir, *runID)
	skutil.RemoveAll(traceDownloadDir)
	skutil.MkdirAll(traceDownloadDir, 0700)
	defer skutil.RemoveAll(traceDownloadDir)

	localOutputDir := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runID)
	skutil.RemoveAll(localOutputDir)
	skutil.MkdirAll(localOutputDir, 0700)
	defer skutil.RemoveAll(localOutputDir)
	remoteDir := filepath.Join(util.BenchmarkRunsDir, *runID)

	sklog.Infof("===== Going to run the task with %d parallel goroutines =====", WORKER_POOL_SIZE)
	// Create channel that contains all trace ULs. This channel will
	// be consumed by the worker pool.
	traceRequests := getClosedChannelOfTraces(traces)

	var wg sync.WaitGroup

	// If not a single benchmark run succeeds then throw at error at the end.
	atleastOneBenchmarkSucceeded := false
	// Gather traceURLs that could not be downloaded.
	erroredTraces := []string{}
	// Mutex to control access to the above slice.
	var erroredTracesMutex sync.Mutex

	// Loop through workers in the worker pool.
	for i := 0; i < WORKER_POOL_SIZE; i++ {
		// Increment the WaitGroup counter.
		wg.Add(1)

		// Create and run a goroutine closure that runs the analysis benchmark.
		go func() {
			// Decrement the WaitGroup counter when the goroutine completes.
			defer wg.Done()

			for t := range traceRequests {
				sklog.Infof("========== Downloading trace %s ==========", t)
				downloadedTrace, err := downloadTrace(t, traceDownloadDir, gs)
				if err != nil {
					sklog.Errorf("Could not download %s: %s", t, err)
					erroredTracesMutex.Lock()
					erroredTraces = append(erroredTraces, t)
					erroredTracesMutex.Unlock()
					continue
				}

				sklog.Infof("========== Processing %s ==========", t)
				if err := runMetricsAnalysisBenchmark(ctx, localOutputDir, downloadedTrace, t); err != nil {
					sklog.Errorf("Error during run_benchmark: %s", err)
				} else {
					atleastOneBenchmarkSucceeded = true
				}
			}
		}()
	}

	// Wait for all spawned goroutines to complete.
	wg.Wait()

	// Summarize errors.
	if len(erroredTraces) > 0 {
		sklog.Error("The following traces could not be downloaded:")
		for _, erroredTrace := range erroredTraces {
			sklog.Errorf("\t%s", erroredTrace)
		}
	}
	if !atleastOneBenchmarkSucceeded {
		return errors.New("Not a single benchmark run was successful. Something is wrong.")
	}

	// If "--output-format=csv" was specified then merge all CSV files and upload.
	if strings.Contains(*benchmarkExtraArgs, "--output-format=csv") {
		// Construct path to CT's python scripts.
		pathToPyFiles := util.GetPathToPyFiles(*worker_common.Local, false /* runOnMaster */)
		if err := util.MergeUploadCSVFilesOnWorkers(ctx, localOutputDir, pathToPyFiles, *runID, remoteDir, *valueColumnName, gs, *startRange, true /* handleStrings */, false /* addRanks */, map[string]map[string]string{} /* pageRankToAdditionalFields */); err != nil {
			return fmt.Errorf("Error while processing withpatch CSV files: %s", err)
		}
	}

	return nil
}

// runMetricsAnalysisBenchmark runs the analysis_metrics_ct benchmark on the provided trace.
func runMetricsAnalysisBenchmark(ctx context.Context, outputPath, downloadedTrace, cloudTraceLink string) error {
	args := []string{
		filepath.Join(util.GetPathToTelemetryCTBinaries(*worker_common.Local), util.BINARY_ANALYZE_METRICS),
		"--local-trace-path", downloadedTrace,
		"--cloud-trace-link", cloudTraceLink,
		"--metric-name", *metricName,
		"--output-csv", filepath.Join(outputPath, getTraceName(downloadedTrace), "result.csv"),
	}
	// Calculate what timeout should be used when executing run_benchmark.
	timeoutSecs := util.GetRunBenchmarkTimeoutValue(*benchmarkExtraArgs, METRICS_BENCHMARK_TIMEOUT_SECS)
	sklog.Infof("Using %d seconds for timeout", timeoutSecs)
	// Remove from benchmarkExtraArgs "special" flags that are recognized by CT but not
	// by the run_benchmark script.
	extraArgs := util.RemoveFlagsFromArgs(*benchmarkExtraArgs, util.RUN_BENCHMARK_TIMEOUT_FLAG, util.MAX_PAGES_PER_BOT)
	// Split extraArgs if not empty and append to args.
	if extraArgs != "" {
		args = append(args, strings.Fields(extraArgs)...)
	}
	// Set the DISPLAY.
	env := []string{
		"DISPLAY=:0",
	}
	// Append the original environment as well.
	for _, e := range os.Environ() {
		env = append(env, e)
	}

	// Create buffer for capturing the stdout and stderr of the benchmark run.
	var b bytes.Buffer
	if _, err := b.WriteString(fmt.Sprintf("========== Stdout and stderr for %s ==========\n", downloadedTrace)); err != nil {
		return fmt.Errorf("Error writing to output buffer: %s", err)
	}
	if err := util.ExecuteCmdWithConfigurableLogging(ctx, "python", args, env, time.Duration(timeoutSecs)*time.Second, &b, &b, false, false); err != nil {
		output, getErr := util.GetRunBenchmarkOutput(b)
		skutil.LogErr(getErr)
		fmt.Println(output)
		return fmt.Errorf("Run benchmark command failed with: %s", err)
	}
	output, err := util.GetRunBenchmarkOutput(b)
	if err != nil {
		return fmt.Errorf("Could not get run benchmark output: %s", err)
	}
	// Print the output and return.
	fmt.Println(output)
	return nil
}

// downloadTrace downloads the traceURL from google storage into the specified
// destDir.
func downloadTrace(traceURL, destDir string, gs *util.GcsUtil) (string, error) {
	traceName := getTraceName(traceURL)
	traceDest := filepath.Join(destDir, traceName)
	if err := gs.DownloadRemoteFileFromBucket(TRACE_OUTPUT_BUCKET, traceName, traceDest); err != nil {
		return "", fmt.Errorf("Error downloading %s from %s to %s: %s", traceName, TRACE_OUTPUT_BUCKET, traceDest, err)
	}
	return traceDest, nil
}

// getTraceName parses the provided traceURI and returns the name of the trace.
// traceURI could be a file path or a URL.
func getTraceName(traceURI string) string {
	traceTokens := strings.Split(traceURI, "/")
	return traceTokens[len(traceTokens)-1]
}

// getClosedChannelOfTraces returns a channel that contains all trace URLs.
func getClosedChannelOfTraces(traces []string) chan string {
	tracesChannel := make(chan string, len(traces))
	for _, t := range traces {
		tracesChannel <- t
	}
	close(tracesChannel)
	return tracesChannel
}

func main() {
	retCode := 0
	if err := metricsAnalysis(); err != nil {
		sklog.Errorf("Error while running metrics analysis: %s", err)
		retCode = 255
	}
	os.Exit(retCode)
}
<commit_msg>[CT] Fixes for the new metrics analysis binary<commit_after>// Application that does metrics analysis as described in the design doc:
// go/ct_metrics_analysis
//
// Can be tested locally with:
// $ go run go/worker_scripts/metrics_analysis/main.go --start_range=1 --num=3 --run_id=rmistry-test1 --benchmark_extra_args="--output-format=csv" --metric_name="loadingMetric" --logtostderr=true --local
//
package main

import (
	"bytes"
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/ct/go/worker_scripts/worker_common"
	"go.skia.org/infra/go/sklog"
	skutil "go.skia.org/infra/go/util"
)

const (
	// The number of goroutines that will run in parallel to download traces and
	// run metrics analysis.
	WORKER_POOL_SIZE = 5

	METRICS_BENCHMARK_TIMEOUT_SECS = 300

	TRACE_OUTPUT_BUCKET = "chrome-telemetry-output"
)

var (
	startRange         = flag.Int("start_range", 1, "The number this worker will run metrics analysis from.")
	num                = flag.Int("num", 100, "The total number of traces to run metrics analysis from the start_range.")
	runID              = flag.String("run_id", "", "The unique run id (typically requester + timestamp).")
	benchmarkExtraArgs = flag.String("benchmark_extra_args", "", "The extra arguments that are passed to the specified benchmark.")
	metricName         = flag.String("metric_name", "", "The metric to parse the traces with. Eg: loadingMetric")
	valueColumnName    = flag.String("value_column_name", "", "Which column's entries to use as field values when combining CSVs.")
)

func metricsAnalysis() error {
	ctx := context.Background()
	worker_common.Init(ctx)
	if !*worker_common.Local {
		defer util.CleanTmpDir()
	}
	defer util.TimeTrack(time.Now(), "Metrics Analysis")
	defer sklog.Flush()

	// Validate required arguments.
	if *runID == "" {
		return errors.New("Must specify --run_id")
	}
	if *metricName == "" {
		return errors.New("Must specify --metric_name")
	}

	// Use defaults.
	if *valueColumnName == "" {
		*valueColumnName = util.DEFAULT_VALUE_COLUMN_NAME
	}

	// Instantiate GcsUtil object.
	gs, err := util.NewGcsUtil(nil)
	if err != nil {
		return fmt.Errorf("GcsUtil instantiation failed: %s", err)
	}

	// Download the trace URLs for this run from Google storage.
	tracesFilename := *runID + ".traces.csv"
	skutil.MkdirAll(util.PagesetsDir, 0700)
	tmpDir, err := ioutil.TempDir(util.PagesetsDir, "traces")
	if err != nil {
		return fmt.Errorf("Could not create tmpdir: %s", err)
	}
	defer skutil.RemoveAll(tmpDir)
	remotePatchesDir := filepath.Join(util.BenchmarkRunsDir, *runID)

	// Download traces.
	if _, err := util.DownloadPatch(filepath.Join(tmpDir, tracesFilename), filepath.Join(remotePatchesDir, tracesFilename), gs); err != nil {
		return fmt.Errorf("Could not download %s: %s", tracesFilename, err)
	}
	traces, err := util.GetCustomPages(filepath.Join(tmpDir, tracesFilename))
	if err != nil {
		return fmt.Errorf("Could not read custom traces file %s: %s", tracesFilename, err)
	}
	if len(traces) == 0 {
		return errors.New(fmt.Sprintf("No traces found in %s", tracesFilename))
	}
	traces = util.GetCustomPagesWithinRange(*startRange, *num, traces)
	sklog.Infof("Using %d traces", len(traces))

	// Establish output paths for trace downloads and metrics.
	traceDownloadDir := filepath.Join(util.StorageDir, util.TraceDownloadsDir, *runID)
	skutil.RemoveAll(traceDownloadDir)
	skutil.MkdirAll(traceDownloadDir, 0700)
	defer skutil.RemoveAll(traceDownloadDir)

	localOutputDir := filepath.Join(util.StorageDir, util.BenchmarkRunsDir, *runID)
	skutil.RemoveAll(localOutputDir)
	skutil.MkdirAll(localOutputDir, 0700)
	defer skutil.RemoveAll(localOutputDir)
	remoteDir := filepath.Join(util.BenchmarkRunsDir, *runID)

	sklog.Infof("===== Going to run the task with %d parallel goroutines =====", WORKER_POOL_SIZE)
	// Create channel that contains all trace ULs. This channel will
	// be consumed by the worker pool.
	traceRequests := getClosedChannelOfTraces(traces)

	var wg sync.WaitGroup

	// If not a single benchmark run succeeds then throw at error at the end.
	atleastOneBenchmarkSucceeded := false
	// Gather traceURLs that could not be downloaded.
	erroredTraces := []string{}
	// Mutex to control access to the above slice.
	var erroredTracesMutex sync.Mutex

	// Loop through workers in the worker pool.
	for i := 0; i < WORKER_POOL_SIZE; i++ {
		// Increment the WaitGroup counter.
		wg.Add(1)

		// Create and run a goroutine closure that runs the analysis benchmark.
		go func() {
			// Decrement the WaitGroup counter when the goroutine completes.
			defer wg.Done()

			for t := range traceRequests {
				sklog.Infof("========== Downloading trace %s ==========", t)
				downloadedTrace, err := downloadTrace(t, traceDownloadDir, gs)
				if err != nil {
					sklog.Errorf("Could not download %s: %s", t, err)
					erroredTracesMutex.Lock()
					erroredTraces = append(erroredTraces, t)
					erroredTracesMutex.Unlock()
					continue
				}

				sklog.Infof("========== Processing %s ==========", t)
				if err := runMetricsAnalysisBenchmark(ctx, localOutputDir, downloadedTrace, t); err != nil {
					sklog.Errorf("Error during run_benchmark: %s", err)
				} else {
					atleastOneBenchmarkSucceeded = true
				}
			}
		}()
	}

	// Wait for all spawned goroutines to complete.
	wg.Wait()

	// Summarize errors.
	if len(erroredTraces) > 0 {
		sklog.Error("The following traces could not be downloaded:")
		for _, erroredTrace := range erroredTraces {
			sklog.Errorf("\t%s", erroredTrace)
		}
	}
	if !atleastOneBenchmarkSucceeded {
		return errors.New("Not a single benchmark run was successful. Something is wrong.")
	}

	// If "--output-format=csv" was specified then merge all CSV files and upload.
	if strings.Contains(*benchmarkExtraArgs, "--output-format=csv") {
		// Construct path to CT's python scripts.
		pathToPyFiles := util.GetPathToPyFiles(*worker_common.Local, false /* runOnMaster */)
		if err := util.MergeUploadCSVFilesOnWorkers(ctx, localOutputDir, pathToPyFiles, *runID, remoteDir, *valueColumnName, gs, *startRange, true /* handleStrings */, false /* addRanks */, map[string]map[string]string{} /* pageRankToAdditionalFields */); err != nil {
			return fmt.Errorf("Error while processing withpatch CSV files: %s", err)
		}
	}

	return nil
}

// runMetricsAnalysisBenchmark runs the analysis_metrics_ct benchmark on the provided trace.
func runMetricsAnalysisBenchmark(ctx context.Context, outputPath, downloadedTrace, cloudTraceLink string) error {
	outputCSVDir := filepath.Join(outputPath, getTraceName(downloadedTrace))
	if err := os.MkdirAll(outputCSVDir, 0700); err != nil {
		return fmt.Errorf("Could not create %s: %s", outputCSVDir, err)
	}

	args := []string{
		filepath.Join(util.GetPathToTelemetryCTBinaries(*worker_common.Local), util.BINARY_ANALYZE_METRICS),
		"--local-trace-path", downloadedTrace,
		"--cloud-trace-link", cloudTraceLink,
		"--metric-name", *metricName,
		"--output-csv", filepath.Join(outputCSVDir, "results.csv"),
	}
	// Calculate what timeout should be used when executing run_benchmark.
	timeoutSecs := util.GetRunBenchmarkTimeoutValue(*benchmarkExtraArgs, METRICS_BENCHMARK_TIMEOUT_SECS)
	sklog.Infof("Using %d seconds for timeout", timeoutSecs)
	// Set the DISPLAY.
	env := []string{
		"DISPLAY=:0",
	}
	// Append the original environment as well.
	for _, e := range os.Environ() {
		env = append(env, e)
	}

	// Create buffer for capturing the stdout and stderr of the benchmark run.
	var b bytes.Buffer
	if _, err := b.WriteString(fmt.Sprintf("========== Stdout and stderr for %s ==========\n", downloadedTrace)); err != nil {
		return fmt.Errorf("Error writing to output buffer: %s", err)
	}
	if err := util.ExecuteCmdWithConfigurableLogging(ctx, "python", args, env, time.Duration(timeoutSecs)*time.Second, &b, &b, false, false); err != nil {
		output, getErr := util.GetRunBenchmarkOutput(b)
		skutil.LogErr(getErr)
		fmt.Println(output)
		return fmt.Errorf("Run benchmark command failed with: %s", err)
	}
	output, err := util.GetRunBenchmarkOutput(b)
	if err != nil {
		return fmt.Errorf("Could not get run benchmark output: %s", err)
	}
	// Print the output and return.
	fmt.Println(output)
	return nil
}

// downloadTrace downloads the traceURL from google storage into the specified
// destDir.
func downloadTrace(traceURL, destDir string, gs *util.GcsUtil) (string, error) {
	traceName := getTraceName(traceURL)
	traceDest := filepath.Join(destDir, traceName)
	if err := gs.DownloadRemoteFileFromBucket(TRACE_OUTPUT_BUCKET, traceName, traceDest); err != nil {
		return "", fmt.Errorf("Error downloading %s from %s to %s: %s", traceName, TRACE_OUTPUT_BUCKET, traceDest, err)
	}
	return traceDest, nil
}

// getTraceName parses the provided traceURI and returns the name of the trace.
// traceURI could be a file path or a URL.
func getTraceName(traceURI string) string {
	traceTokens := strings.Split(traceURI, "/")
	return traceTokens[len(traceTokens)-1]
}

// getClosedChannelOfTraces returns a channel that contains all trace URLs.
func getClosedChannelOfTraces(traces []string) chan string {
	tracesChannel := make(chan string, len(traces))
	for _, t := range traces {
		tracesChannel <- t
	}
	close(tracesChannel)
	return tracesChannel
}

func main() {
	retCode := 0
	if err := metricsAnalysis(); err != nil {
		sklog.Errorf("Error while running metrics analysis: %s", err)
		retCode = 255
	}
	os.Exit(retCode)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sync"

	"github.com/andrewstuart/go-metio"
	"github.com/andrewstuart/go-nzb"
	"github.com/andrewstuart/nntp"
	"github.com/andrewstuart/yenc"
)

//Download will retrieve all the files for an NZB and extract them when
//finished.
func Download(nz *nzb.NZB, dir string) error {
	files := &sync.WaitGroup{}
	files.Add(len(nz.Files))

	var rarFiles []string

	tempDir := dir + "/temp"

	err := os.MkdirAll(tempDir, 0775)

	if err != nil {
		return err
	}

	group := metio.NewReaderGroup()
	go meter(nz, group)

	for n := range nz.Files {
		num := n
		file := nz.Files[n]

		fileSegs := &sync.WaitGroup{}
		fileSegs.Add(len(file.Segments))

		fileBufs := make([]string, len(file.Segments))

		name, err := file.Name()

		if err != nil {
			name = fmt.Sprintf("file-%d", num)
		}

		fName := path.Clean(fmt.Sprintf("%s/%s", dir, name))

		//Write to disk
		go func() {
			defer files.Done()

			fileSegs.Wait()

			if IsRar(fName) {
				rarFiles = append(rarFiles, fName)
			}

			toFile, err := os.Create(fName)
			defer toFile.Close()

			if err != nil {
				log.Println("Couldn't create file.")
				return
			}

			for i := range fileBufs {
				var f *os.File
				f, err = os.Open(fileBufs[i])
				defer f.Close()
				defer os.Remove(fileBufs[i])

				if err != nil {
					return
				}

				_, err = io.Copy(toFile, f)

				if err != nil {
					return
				}
			}
		}()

		//Get from network
		for i := range file.Segments {
			go func(i int) {
				defer fileSegs.Done()
				seg := file.Segments[i]

				tf := path.Clean(fmt.Sprintf("%s/temp/%s", dir, seg.Id))

				var f os.FileInfo
				//Check to see if file segment has been previously downloaded completely
				//That is, it exists and has the proper size.
				if f, err = os.Stat(tf); err == nil && f.Size() == int64(seg.Bytes) {
					// meter <- seg.Bytes
					fileBufs[i] = tf
					return
				}

				var art *nntp.Response
				art, err = use.GetArticle(file.Groups[0], html.UnescapeString(seg.Id))

				if err != nil {
					log.Printf("error downloading file %s: %v\n", file.Subject, err)
					return
				}

				if art.Body == nil {
					log.Printf("error getting article: no body - %+v\n", art)
					return
				}

				var r io.Reader = art.Body
				defer art.Body.Close()

				mr := metio.NewReader(art.Body)
				group.Add(mr)
				defer group.Remove(mr)

				r = yenc.NewReader(mr)

				var destF *os.File
				destF, err = os.Create(tf)
				defer destF.Close()

				if err != nil {
					return
				}

				fileBufs[i] = tf
				_, err = io.Copy(destF, r)
				// _, err = io.Copy(f, lr)

				if err != nil {
					log.Printf("There was an error reading the article body: %v\n", err)
				}
			}(i)
		}
	}

	files.Wait()

	if len(rarFiles) > 0 {
		log.Println("Unrarring")
	}

	for _, fName := range rarFiles {
		files, _ := ioutil.ReadDir(dir)

		rErr := Unrar(fName, dir)

		if rErr == nil {
			for fi := range files {
				fdir := dir + "/" + files[fi].Name()
				err := os.Remove(fdir)
				if err != nil {
					log.Println("Error removing file", fdir, err)
				}
			}
		}
	}

	os.RemoveAll(tempDir)

	return err
}
<commit_msg>Update ID field<commit_after>package main

import (
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sync"

	"github.com/andrewstuart/go-metio"
	"github.com/andrewstuart/go-nzb"
	"github.com/andrewstuart/nntp"
	"github.com/andrewstuart/yenc"
)

//Download will retrieve all the files for an NZB and extract them when
//finished.
func Download(nz *nzb.NZB, dir string) error {
	files := &sync.WaitGroup{}
	files.Add(len(nz.Files))

	var rarFiles []string

	tempDir := dir + "/temp"

	err := os.MkdirAll(tempDir, 0775)

	if err != nil {
		return err
	}

	group := metio.NewReaderGroup()
	go meter(nz, group)

	for n := range nz.Files {
		num := n
		file := nz.Files[n]

		fileSegs := &sync.WaitGroup{}
		fileSegs.Add(len(file.Segments))

		fileBufs := make([]string, len(file.Segments))

		name, err := file.Name()

		if err != nil {
			name = fmt.Sprintf("file-%d", num)
		}

		fName := path.Clean(fmt.Sprintf("%s/%s", dir, name))

		//Write to disk
		go func() {
			defer files.Done()

			fileSegs.Wait()

			if IsRar(fName) {
				rarFiles = append(rarFiles, fName)
			}

			toFile, err := os.Create(fName)
			defer toFile.Close()

			if err != nil {
				log.Println("Couldn't create file.")
				return
			}

			for i := range fileBufs {
				var f *os.File
				f, err = os.Open(fileBufs[i])
				defer f.Close()
				defer os.Remove(fileBufs[i])

				if err != nil {
					return
				}

				_, err = io.Copy(toFile, f)

				if err != nil {
					return
				}
			}
		}()

		//Get from network
		for i := range file.Segments {
			go func(i int) {
				defer fileSegs.Done()
				seg := file.Segments[i]

				tf := path.Clean(fmt.Sprintf("%s/temp/%s", dir, seg.ID))

				var f os.FileInfo
				//Check to see if file segment has been previously downloaded completely
				//That is, it exists and has the proper size.
				if f, err = os.Stat(tf); err == nil && f.Size() == int64(seg.Bytes) {
					// meter <- seg.Bytes
					fileBufs[i] = tf
					return
				}

				var art *nntp.Response
				art, err = use.GetArticle(file.Groups[0], html.UnescapeString(seg.ID))

				if err != nil {
					log.Printf("error downloading file %s: %v\n", file.Subject, err)
					return
				}

				if art.Body == nil {
					log.Printf("error getting article: no body - %+v\n", art)
					return
				}

				var r io.Reader = art.Body
				defer art.Body.Close()

				mr := metio.NewReader(art.Body)
				group.Add(mr)
				defer group.Remove(mr)

				r = yenc.NewReader(mr)

				var destF *os.File
				destF, err = os.Create(tf)
				defer destF.Close()

				if err != nil {
					return
				}

				fileBufs[i] = tf
				_, err = io.Copy(destF, r)
				// _, err = io.Copy(f, lr)

				if err != nil {
					log.Printf("There was an error reading the article body: %v\n", err)
				}
			}(i)
		}
	}

	files.Wait()

	if len(rarFiles) > 0 {
		log.Println("Unrarring")
	}

	for _, fName := range rarFiles {
		files, _ := ioutil.ReadDir(dir)

		rErr := Unrar(fName, dir)

		if rErr == nil {
			for fi := range files {
				fdir := dir + "/" + files[fi].Name()
				err := os.Remove(fdir)
				if err != nil {
					log.Println("Error removing file", fdir, err)
				}
			}
		}
	}

	os.RemoveAll(tempDir)

	return err
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"image"
	"io/ioutil"
	"net/http"
	"net/url"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

// When all else fails, Google it. Uses the regular web interface. There are
// two image search APIs, but one is deprecated and doesn't support exact size
// matching, and the other requires an API key limited to 100 searches a day.
const googleSearchFormat = `https://www.google.com.br/search?tbs=isz%3Aex%2Ciszw%3A%v%2Ciszh%3A%v&tbm=isch&num=5&q=`

// Possible Google result formats
var googleSearchResultPatterns = []string{`imgurl=(.+?\.(jpeg|jpg|png))&imgrefurl=`, `\"ou\":\"(.+?)\",\"`}

// Returns the first steam grid image URL found by Google search of a given
// game name.
func getGoogleImage(gameName string, artStyleExtensions []string) (string, error) {
	if gameName == "" {
		return "", nil
	}

	url := fmt.Sprintf(googleSearchFormat, artStyleExtensions[5], artStyleExtensions[6]) + url.QueryEscape(gameName)

	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return "", err
	}

	// If we don't set an user agent, Google will block us because we are a
	// bot. If we set something like "SteamGrid Image Search" it'll work, but
	// Google will serve a simple HTML page without direct image links.
	// So we have to lie.
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36")
	response, err := client.Do(req)
	if err != nil {
		return "", err
	}

	responseBytes, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return "", err
	}
	response.Body.Close()

	for _, googleSearchResultPattern := range googleSearchResultPatterns {
		pattern := regexp.MustCompile(googleSearchResultPattern)
		matches := pattern.FindStringSubmatch(string(responseBytes))

		if len(matches) >= 1 {
			return matches[1], nil
		}
	}
	return "", nil
}

// https://www.steamgriddb.com/api/v2
type SteamGridDBResponse struct {
	Success bool
	Data []struct {
		Id int
		Score int
		Style string
		Url string
		Thumb string
		Tags []string
		Author struct {
			Name string
			Steam64 string
			Avatar string
		}
	}
}

type SteamGridDBSearchResponse struct {
	Success bool
	Data []struct {
		Id int
		Name string
		Types []string
		Verified bool
	}
}

// Search SteamGridDB for cover image
const SteamGridDBBaseURL = "https://www.steamgriddb.com/api/v2"

func SteamGridDBGetRequest(url string, steamGridDBApiKey string) ([]byte, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	req.Header.Add("Authorization", "Bearer " + steamGridDBApiKey)
	if err != nil {
		return nil, err
	}

	response, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	if response.StatusCode == 401 {
		// Authorization token is missing or invalid
		return nil, errors.New("401")
	} else if response.StatusCode == 404 {
		// Could not find game with that id
		return nil, errors.New("404")
	}

	responseBytes, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}
	response.Body.Close()

	return responseBytes, nil
}

func getSteamGridDBImage(game *Game, artStyleExtensions []string, steamGridDBApiKey string) (string, error) {
	// Specify artType:
	// "alternate" "blurred" "white_logo" "material" "no_logo"
	artTypes := []string{"alternate"}
	filter := "?styles=" + strings.Join(artTypes, ",")

	// Try for HQ, then for LQ
	// It's possible to request both dimensions in one go but that'll give us scrambled results with no indicator which result has which size.
	for i := 0; i < 3; i += 2 {
		dimensions := filter + "&dimensions=" + artStyleExtensions[3 + i] + "x" + artStyleExtensions[4 + i]

		// Try with game.ID which is probably steams appID
		url := SteamGridDBBaseURL + "/grids/steam/" + game.ID + dimensions
		responseBytes, err := SteamGridDBGetRequest(url, steamGridDBApiKey)
		var jsonResponse SteamGridDBResponse

		// Authorization token is missing or invalid
	 	if err != nil && err.Error() == "401" {
			return "", errors.New("SteamGridDB authorization token is missing or invalid")
		// Could not find game with that id
		} else if err != nil && err.Error() == "404" {
			// Try searching for the name…
			url = SteamGridDBBaseURL + "/search/autocomplete/" + game.Name + dimensions
			responseBytes, err = SteamGridDBGetRequest(url, steamGridDBApiKey)
			if err != nil {
				return "", err
			}

			var jsonSearchResponse SteamGridDBSearchResponse
			err = json.Unmarshal(responseBytes, &jsonSearchResponse)
			if err != nil {
				return "", errors.New("Best search match doesn't has a " + strings.Join(artTypes, ",") + " type")
			}

			SteamGridDBGameId := -1
			if jsonSearchResponse.Success && len(jsonSearchResponse.Data) >= 1 {
				for _, n := range jsonSearchResponse.Data[0].Types {
					for _, m := range artTypes {
						if n == m {
							// This game has at least one of our requested artTypes
							SteamGridDBGameId = jsonSearchResponse.Data[0].Id
							break
						}
					}

					if SteamGridDBGameId != -1 {
						break
					}
				}
			}

			if SteamGridDBGameId == -1 {
				return "", nil
			}

			// …and get the url of the top result.
			url = SteamGridDBBaseURL + "/grids/game/" + strconv.Itoa(SteamGridDBGameId) + dimensions
			responseBytes, err = SteamGridDBGetRequest(url, steamGridDBApiKey)
			if err != nil {
				return "", err
			}
		} else if err != nil {
			return "", err
		}

		err = json.Unmarshal(responseBytes, &jsonResponse)
		if err != nil {
			return "", err
		}

		if jsonResponse.Success && len(jsonResponse.Data) >= 1 {
			return jsonResponse.Data[0].Url, nil
		}
	}

	return "", nil
}

const IGDBImageURL = "https://images.igdb.com/igdb/image/upload/t_720p/%v.jpg"
const IGDBGameURL = "https://api-v3.igdb.com/games"
const IGDBCoverURL = "https://api-v3.igdb.com/covers"
const IGDBGameBody = `fields name,cover; search "%v";`
const IGDBCoverBody = `fields image_id; where id = %v;`

type IGDBGame struct {
	Id int
	Cover int
	Name string
}

type IGDBCover struct {
	Id int
	Image_id string
}

func IGDBPostRequest(url string, body string, IGDBApiKey string) ([]byte, error) {
	client := &http.Client{}
	req, err := http.NewRequest("POST", url, strings.NewReader(body))
	req.Header.Add("user-key", IGDBApiKey)
	req.Header.Add("Accept", "application/json")
	if err != nil {
		return nil, err
	}

	response, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	responseBytes, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}
	response.Body.Close()

	return responseBytes, nil
}

func getIGDBImage(gameName string, IGDBApiKey string) (string, error) {
	responseBytes, err := IGDBPostRequest(IGDBGameURL, fmt.Sprintf(IGDBGameBody, gameName), IGDBApiKey)
	if err != nil {
		return "", err
	}

	var jsonGameResponse []IGDBGame
	err = json.Unmarshal(responseBytes, &jsonGameResponse)
	if err != nil {
		return "", nil
	}

	if len(jsonGameResponse) < 1 || jsonGameResponse[0].Cover == 0 {
		return "", nil
	}

	responseBytes, err = IGDBPostRequest(IGDBCoverURL, fmt.Sprintf(IGDBCoverBody, jsonGameResponse[0].Cover), IGDBApiKey)
	if err != nil {
		return "", err
	}

	var jsonCoverResponse []IGDBCover
	err = json.Unmarshal(responseBytes, &jsonCoverResponse)
	if err != nil {
		return "", nil
	}

	if len(jsonCoverResponse) >= 1 {
		return fmt.Sprintf(IGDBImageURL, jsonCoverResponse[0].Image_id), nil
	}

	return "", nil
}

// Tries to fetch a URL, returning the response only if it was positive.
func tryDownload(url string) (*http.Response, error) {
	response, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	if response.StatusCode == 404 {
		// Some apps don't have an image and there's nothing we can do.
		return nil, nil
	} else if response.StatusCode >= 400 {
		// Other errors should be reported, though.
		return nil, errors.New("Failed to download image " + url + ": " + response.Status)
	}

	return response, nil
}

// Primary URL for downloading grid images.
const akamaiURLFormat = `https://steamcdn-a.akamaihd.net/steam/apps/%v/`

// The subreddit mentions this as primary, but I've found Akamai to contain
// more images and answer faster.
const steamCdnURLFormat = `cdn.akamai.steamstatic.com/steam/apps/%v/`

// Tries to load the grid image for a game from a number of alternative
// sources. Returns the final response received and a flag indicating if it was
// from a Google search (useful because we want to log the lower quality
// images).
func getImageAlternatives(game *Game, artStyle string, artStyleExtensions []string, steamGridDBApiKey string, IGDBApiKey string) (response *http.Response, from string, err error) {
	from = "steam server"
	response, err = tryDownload(fmt.Sprintf(akamaiURLFormat + artStyleExtensions[2], game.ID))
	if err == nil && response != nil {
		return
	}

	response, err = tryDownload(fmt.Sprintf(steamCdnURLFormat + artStyleExtensions[2], game.ID))
	if err == nil && response != nil {
		return
	}

	url := ""
	if (artStyle == "Cover" || artStyle == "Banner") && steamGridDBApiKey != "" && url == "" {
		from = "SteamGridDB"
		url, err = getSteamGridDBImage(game, artStyleExtensions, steamGridDBApiKey)
		if err != nil {
			return
		}
	}

	// IGDB has mostly cover styles
	if artStyle == "Cover" && IGDBApiKey != "" && url == "" {
		from = "IGDB"
		url, err = getIGDBImage(game.Name, IGDBApiKey)
		if err != nil {
			return
		}
	}

	// Skip for Covers, bad results
	if artStyle == "Banner" && url == "" {
		from = "search"
		url, err = getGoogleImage(game.Name, artStyleExtensions)
		if err != nil {
			return
		}
	}

	response, err = tryDownload(url)
	if err == nil && response != nil {
		return
	}

	return nil, "", nil
}

// DownloadImage tries to download the game images, saving it in game.ImageBytes. Returns
// flags indicating if the operation succeeded and if the image downloaded was
// from a search.
func DownloadImage(gridDir string, game *Game, artStyle string, artStyleExtensions []string, steamGridDBApiKey string, IGDBApiKey string) (string, error) {
	response, from, err := getImageAlternatives(game, artStyle, artStyleExtensions, steamGridDBApiKey, IGDBApiKey)
	if response == nil || err != nil {
		return "", err
	}

	contentType := response.Header.Get("Content-Type")
	urlExt := filepath.Ext(response.Request.URL.Path)
	if contentType != "" {
		game.ImageExt = "." + strings.Split(contentType, "/")[1]
	} else if urlExt != "" {
		game.ImageExt = urlExt
	} else {
		// Steam is forgiving on image extensions.
		game.ImageExt = "jpg"
	}

	if game.ImageExt == ".jpeg" {
		// The new library ignores .jpeg
		game.ImageExt = ".jpg"
	} else if game.ImageExt == ".octet-stream" {
		// Amazonaws (steamgriddb) gives us an .octet-stream
		game.ImageExt = ".png"
	}

	imageBytes, err := ioutil.ReadAll(response.Body)
	response.Body.Close()

	// catch false aspect ratios
	image, _, err := image.Decode(bytes.NewBuffer(imageBytes))
	if err != nil {
		return "", err
	}
	imageSize := image.Bounds().Max
	if (artStyle == "Banner" && imageSize.X < imageSize.Y) {
		return "", nil
	} else if (artStyle == "Cover" && imageSize.X > imageSize.Y) {
		return "", nil
	}

	game.ImageSource = from;

	game.CleanImageBytes = imageBytes
	return from, nil
}

// Get game name from SteamDB as last resort.
const steamDBFormat = `https://steamdb.info/app/%v`

func GetGameName(gameId string) string {
	response, err := tryDownload(fmt.Sprintf(steamDBFormat, gameId))
	if err != nil || response == nil {
		return ""
	}
	page, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return ""
	}
	response.Body.Close()

	pattern := regexp.MustCompile("
\nName	\\s*(.*?)	")
	match := pattern.FindStringSubmatch(string(page))
	if match == nil || len(match) == 0 {
		return ""
	} else {
		return match[1]
	}
}
<commit_msg>fixed googleSearchFormat string<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"image"
	"io/ioutil"
	"net/http"
	"net/url"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

// When all else fails, Google it. Uses the regular web interface. There are
// two image search APIs, but one is deprecated and doesn't support exact size
// matching, and the other requires an API key limited to 100 searches a day.
const googleSearchFormat = `https://www.google.com.br/search?tbs=isz%%3Aex%%2Ciszw%%3A%v%%2Ciszh%%3A%v&tbm=isch&num=5&q=`

// Possible Google result formats
var googleSearchResultPatterns = []string{`imgurl=(.+?\.(jpeg|jpg|png))&imgrefurl=`, `\"ou\":\"(.+?)\",\"`}

// Returns the first steam grid image URL found by Google search of a given
// game name.
func getGoogleImage(gameName string, artStyleExtensions []string) (string, error) {
	if gameName == "" {
		return "", nil
	}

	url := fmt.Sprintf(googleSearchFormat, artStyleExtensions[5], artStyleExtensions[6]) + url.QueryEscape(gameName)

	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return "", err
	}

	// If we don't set an user agent, Google will block us because we are a
	// bot. If we set something like "SteamGrid Image Search" it'll work, but
	// Google will serve a simple HTML page without direct image links.
	// So we have to lie.
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36")
	response, err := client.Do(req)
	if err != nil {
		return "", err
	}

	responseBytes, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return "", err
	}
	response.Body.Close()

	for _, googleSearchResultPattern := range googleSearchResultPatterns {
		pattern := regexp.MustCompile(googleSearchResultPattern)
		matches := pattern.FindStringSubmatch(string(responseBytes))

		if len(matches) >= 1 {
			return matches[1], nil
		}
	}
	return "", nil
}

// https://www.steamgriddb.com/api/v2
type SteamGridDBResponse struct {
	Success bool
	Data []struct {
		Id int
		Score int
		Style string
		Url string
		Thumb string
		Tags []string
		Author struct {
			Name string
			Steam64 string
			Avatar string
		}
	}
}

type SteamGridDBSearchResponse struct {
	Success bool
	Data []struct {
		Id int
		Name string
		Types []string
		Verified bool
	}
}

// Search SteamGridDB for cover image
const SteamGridDBBaseURL = "https://www.steamgriddb.com/api/v2"

func SteamGridDBGetRequest(url string, steamGridDBApiKey string) ([]byte, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	req.Header.Add("Authorization", "Bearer " + steamGridDBApiKey)
	if err != nil {
		return nil, err
	}

	response, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	if response.StatusCode == 401 {
		// Authorization token is missing or invalid
		return nil, errors.New("401")
	} else if response.StatusCode == 404 {
		// Could not find game with that id
		return nil, errors.New("404")
	}

	responseBytes, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}
	response.Body.Close()

	return responseBytes, nil
}

func getSteamGridDBImage(game *Game, artStyleExtensions []string, steamGridDBApiKey string) (string, error) {
	// Specify artType:
	// "alternate" "blurred" "white_logo" "material" "no_logo"
	artTypes := []string{"alternate"}
	filter := "?styles=" + strings.Join(artTypes, ",")

	// Try for HQ, then for LQ
	// It's possible to request both dimensions in one go but that'll give us scrambled results with no indicator which result has which size.
	for i := 0; i < 3; i += 2 {
		dimensions := filter + "&dimensions=" + artStyleExtensions[3 + i] + "x" + artStyleExtensions[4 + i]

		// Try with game.ID which is probably steams appID
		url := SteamGridDBBaseURL + "/grids/steam/" + game.ID + dimensions
		responseBytes, err := SteamGridDBGetRequest(url, steamGridDBApiKey)
		var jsonResponse SteamGridDBResponse

		// Authorization token is missing or invalid
	 	if err != nil && err.Error() == "401" {
			return "", errors.New("SteamGridDB authorization token is missing or invalid")
		// Could not find game with that id
		} else if err != nil && err.Error() == "404" {
			// Try searching for the name…
			url = SteamGridDBBaseURL + "/search/autocomplete/" + game.Name + dimensions
			responseBytes, err = SteamGridDBGetRequest(url, steamGridDBApiKey)
			if err != nil {
				return "", err
			}

			var jsonSearchResponse SteamGridDBSearchResponse
			err = json.Unmarshal(responseBytes, &jsonSearchResponse)
			if err != nil {
				return "", errors.New("Best search match doesn't has a " + strings.Join(artTypes, ",") + " type")
			}

			SteamGridDBGameId := -1
			if jsonSearchResponse.Success && len(jsonSearchResponse.Data) >= 1 {
				for _, n := range jsonSearchResponse.Data[0].Types {
					for _, m := range artTypes {
						if n == m {
							// This game has at least one of our requested artTypes
							SteamGridDBGameId = jsonSearchResponse.Data[0].Id
							break
						}
					}

					if SteamGridDBGameId != -1 {
						break
					}
				}
			}

			if SteamGridDBGameId == -1 {
				return "", nil
			}

			// …and get the url of the top result.
			url = SteamGridDBBaseURL + "/grids/game/" + strconv.Itoa(SteamGridDBGameId) + dimensions
			responseBytes, err = SteamGridDBGetRequest(url, steamGridDBApiKey)
			if err != nil {
				return "", err
			}
		} else if err != nil {
			return "", err
		}

		err = json.Unmarshal(responseBytes, &jsonResponse)
		if err != nil {
			return "", err
		}

		if jsonResponse.Success && len(jsonResponse.Data) >= 1 {
			return jsonResponse.Data[0].Url, nil
		}
	}

	return "", nil
}

const IGDBImageURL = "https://images.igdb.com/igdb/image/upload/t_720p/%v.jpg"
const IGDBGameURL = "https://api-v3.igdb.com/games"
const IGDBCoverURL = "https://api-v3.igdb.com/covers"
const IGDBGameBody = `fields name,cover; search "%v";`
const IGDBCoverBody = `fields image_id; where id = %v;`

type IGDBGame struct {
	Id int
	Cover int
	Name string
}

type IGDBCover struct {
	Id int
	Image_id string
}

func IGDBPostRequest(url string, body string, IGDBApiKey string) ([]byte, error) {
	client := &http.Client{}
	req, err := http.NewRequest("POST", url, strings.NewReader(body))
	req.Header.Add("user-key", IGDBApiKey)
	req.Header.Add("Accept", "application/json")
	if err != nil {
		return nil, err
	}

	response, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	responseBytes, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}
	response.Body.Close()

	return responseBytes, nil
}

func getIGDBImage(gameName string, IGDBApiKey string) (string, error) {
	responseBytes, err := IGDBPostRequest(IGDBGameURL, fmt.Sprintf(IGDBGameBody, gameName), IGDBApiKey)
	if err != nil {
		return "", err
	}

	var jsonGameResponse []IGDBGame
	err = json.Unmarshal(responseBytes, &jsonGameResponse)
	if err != nil {
		return "", nil
	}

	if len(jsonGameResponse) < 1 || jsonGameResponse[0].Cover == 0 {
		return "", nil
	}

	responseBytes, err = IGDBPostRequest(IGDBCoverURL, fmt.Sprintf(IGDBCoverBody, jsonGameResponse[0].Cover), IGDBApiKey)
	if err != nil {
		return "", err
	}

	var jsonCoverResponse []IGDBCover
	err = json.Unmarshal(responseBytes, &jsonCoverResponse)
	if err != nil {
		return "", nil
	}

	if len(jsonCoverResponse) >= 1 {
		return fmt.Sprintf(IGDBImageURL, jsonCoverResponse[0].Image_id), nil
	}

	return "", nil
}

// Tries to fetch a URL, returning the response only if it was positive.
func tryDownload(url string) (*http.Response, error) {
	response, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	if response.StatusCode == 404 {
		// Some apps don't have an image and there's nothing we can do.
		return nil, nil
	} else if response.StatusCode >= 400 {
		// Other errors should be reported, though.
		return nil, errors.New("Failed to download image " + url + ": " + response.Status)
	}

	return response, nil
}

// Primary URL for downloading grid images.
const akamaiURLFormat = `https://steamcdn-a.akamaihd.net/steam/apps/%v/`

// The subreddit mentions this as primary, but I've found Akamai to contain
// more images and answer faster.
const steamCdnURLFormat = `cdn.akamai.steamstatic.com/steam/apps/%v/`

// Tries to load the grid image for a game from a number of alternative
// sources. Returns the final response received and a flag indicating if it was
// from a Google search (useful because we want to log the lower quality
// images).
func getImageAlternatives(game *Game, artStyle string, artStyleExtensions []string, steamGridDBApiKey string, IGDBApiKey string) (response *http.Response, from string, err error) {
	from = "steam server"
	response, err = tryDownload(fmt.Sprintf(akamaiURLFormat + artStyleExtensions[2], game.ID))
	if err == nil && response != nil {
		return
	}

	response, err = tryDownload(fmt.Sprintf(steamCdnURLFormat + artStyleExtensions[2], game.ID))
	if err == nil && response != nil {
		return
	}

	url := ""
	if (artStyle == "Cover" || artStyle == "Banner") && steamGridDBApiKey != "" && url == "" {
		from = "SteamGridDB"
		url, err = getSteamGridDBImage(game, artStyleExtensions, steamGridDBApiKey)
		if err != nil {
			return
		}
	}

	// IGDB has mostly cover styles
	if artStyle == "Cover" && IGDBApiKey != "" && url == "" {
		from = "IGDB"
		url, err = getIGDBImage(game.Name, IGDBApiKey)
		if err != nil {
			return
		}
	}

	// Skip for Covers, bad results
	if artStyle == "Banner" && url == "" {
		from = "search"
		url, err = getGoogleImage(game.Name, artStyleExtensions)
		if err != nil {
			return
		}
	}

	response, err = tryDownload(url)
	if err == nil && response != nil {
		return
	}

	return nil, "", nil
}

// DownloadImage tries to download the game images, saving it in game.ImageBytes. Returns
// flags indicating if the operation succeeded and if the image downloaded was
// from a search.
func DownloadImage(gridDir string, game *Game, artStyle string, artStyleExtensions []string, steamGridDBApiKey string, IGDBApiKey string) (string, error) {
	response, from, err := getImageAlternatives(game, artStyle, artStyleExtensions, steamGridDBApiKey, IGDBApiKey)
	if response == nil || err != nil {
		return "", err
	}

	contentType := response.Header.Get("Content-Type")
	urlExt := filepath.Ext(response.Request.URL.Path)
	if contentType != "" {
		game.ImageExt = "." + strings.Split(contentType, "/")[1]
	} else if urlExt != "" {
		game.ImageExt = urlExt
	} else {
		// Steam is forgiving on image extensions.
		game.ImageExt = "jpg"
	}

	if game.ImageExt == ".jpeg" {
		// The new library ignores .jpeg
		game.ImageExt = ".jpg"
	} else if game.ImageExt == ".octet-stream" {
		// Amazonaws (steamgriddb) gives us an .octet-stream
		game.ImageExt = ".png"
	}

	imageBytes, err := ioutil.ReadAll(response.Body)
	response.Body.Close()

	// catch false aspect ratios
	image, _, err := image.Decode(bytes.NewBuffer(imageBytes))
	if err != nil {
		return "", err
	}
	imageSize := image.Bounds().Max
	if (artStyle == "Banner" && imageSize.X < imageSize.Y) {
		return "", nil
	} else if (artStyle == "Cover" && imageSize.X > imageSize.Y) {
		return "", nil
	}

	game.ImageSource = from;

	game.CleanImageBytes = imageBytes
	return from, nil
}

// Get game name from SteamDB as last resort.
const steamDBFormat = `https://steamdb.info/app/%v`

func GetGameName(gameId string) string {
	response, err := tryDownload(fmt.Sprintf(steamDBFormat, gameId))
	if err != nil || response == nil {
		return ""
	}
	page, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return ""
	}
	response.Body.Close()

	pattern := regexp.MustCompile("
\nName	\\s*(.*?)	")
	match := pattern.FindStringSubmatch(string(page))
	if match == nil || len(match) == 0 {
		return ""
	} else {
		return match[1]
	}
}
<|endoftext|>


<commit_before>package main

import (
	"code.tobolaski.com/btobolaski/terraform-linode"
	"github.com/hashicorp/terraform/plugin"
)

func main() {
	plugin.Serve(&plugin.ServeOpts{
		ProviderFunc: linode.Provider,
	})
}
<commit_msg>Rename the project for github<commit_after>package main

import (
	"github.com/btobolaskiterraform-linode"
	"github.com/hashicorp/terraform/plugin"
)

func main() {
	plugin.Serve(&plugin.ServeOpts{
		ProviderFunc: linode.Provider,
	})
}
<|endoftext|>


<commit_before>package nodejs

import (
  "net/http"
  "io/ioutil"
  "regexp"
  "runtime"
  "fmt"
  "os"

  "github.com/markelog/eclectica/variables"
)

var (
  client = &http.Client{}

  versionsLink = "https://nodejs.org/dist"
  home = fmt.Sprintf("%s/%s", variables.Home, "node")

  bins = [2]string{"node", "npm"}
  prefix = "/usr/local/bin"
)

func Keyword(keyword string) (map[string]string, error) {
  result := make(map[string]string)
  sumUrl := fmt.Sprintf("%s/%s/SHASUMS256.txt", versionsLink, keyword)
  sourcesUrl := fmt.Sprintf("%s/%s", versionsLink, keyword)
  file, err := info(sumUrl)

  if err != nil {
    return result, err
  }

  versionReg := regexp.MustCompile(`node-v(\d+\.\d+\.\d)`)

  version := versionReg.FindStringSubmatch(file)[1]
  result["name"] = "node"
  result["version"] = version
  result["filename"] = fmt.Sprintf("node-v%s-%s-x64", version, runtime.GOOS)
  result["url"] = fmt.Sprintf("%s/%s.tar.gz", sourcesUrl, result["filename"])

  return result, nil
}

func Version(version string) (map[string]string, error) {
  if version == "latest" || version == "lts" {
    return Keyword(version)
  }

  result := make(map[string]string)

  sourcesUrl := fmt.Sprintf("%s/v%s", versionsLink, version)

  result["name"] = "node"
  result["version"] = version
  result["filename"] = fmt.Sprintf("node-v%s-%s-x64", version, runtime.GOOS)
  result["url"] = fmt.Sprintf("%s/%s.tar.gz", sourcesUrl, result["filename"])

  return result, nil
}

func Remove(version string) error {
  var err error
  base := fmt.Sprintf("%s/%s", home, version)

  err = os.RemoveAll(base)

  if err != nil {
    return err
  }

  for _, bin := range bins {
    exec := fmt.Sprintf("%s/%s", prefix, bin)

    err = os.RemoveAll(exec)

    if err != nil {
      return err
    }
  }

  return nil
}

func Activate(data map[string]string) error {
  var err error

  base := fmt.Sprintf("%s/%s/bin", home, data["version"])

  for _, bin := range bins {
    from := fmt.Sprintf("%s/%s", base, bin)
    to := fmt.Sprintf("%s/%s", prefix, bin)

    os.Remove(to)

    err = os.Symlink(from, to)

    if err != nil {
      return err
    }
  }

  return nil
}

func info(url string) (file string, err error){
  response, err := client.Get(url)

  if err != nil {
    return "", err
  }

  defer response.Body.Close()
  contents, err := ioutil.ReadAll(response.Body)

  if err != nil {
    return "", err
  }

  return string(contents), nil
}
<commit_msg>Correctly copy node dists<commit_after>package nodejs

import (
  "net/http"
  "io/ioutil"
  "regexp"
  "runtime"
  "fmt"
  "os"

  "github.com/markelog/eclectica/variables"

  "github.com/markelog/cprf"
)

var (
  client = &http.Client{}

  versionsLink = "https://nodejs.org/dist"
  home = fmt.Sprintf("%s/%s", variables.Home, "node")

  files = [4]string{"bin", "lib", "include", "share"}
  prefix = "/usr/local"
)

func Keyword(keyword string) (map[string]string, error) {
  result := make(map[string]string)
  sumUrl := fmt.Sprintf("%s/%s/SHASUMS256.txt", versionsLink, keyword)
  sourcesUrl := fmt.Sprintf("%s/%s", versionsLink, keyword)
  file, err := info(sumUrl)

  if err != nil {
    return result, err
  }

  versionReg := regexp.MustCompile(`node-v(\d+\.\d+\.\d)`)

  version := versionReg.FindStringSubmatch(file)[1]
  result["name"] = "node"
  result["version"] = version
  result["filename"] = fmt.Sprintf("node-v%s-%s-x64", version, runtime.GOOS)
  result["url"] = fmt.Sprintf("%s/%s.tar.gz", sourcesUrl, result["filename"])

  return result, nil
}

func Version(version string) (map[string]string, error) {
  if version == "latest" || version == "lts" {
    return Keyword(version)
  }

  result := make(map[string]string)

  sourcesUrl := fmt.Sprintf("%s/v%s", versionsLink, version)

  result["name"] = "node"
  result["version"] = version
  result["filename"] = fmt.Sprintf("node-v%s-%s-x64", version, runtime.GOOS)
  result["url"] = fmt.Sprintf("%s/%s.tar.gz", sourcesUrl, result["filename"])

  return result, nil
}

func Remove(version string) error {
  var err error
  base := fmt.Sprintf("%s/%s", home, version)

  err = os.RemoveAll(base)

  if err != nil {
    return err
  }

  return nil
}

func Activate(data map[string]string) error {
  var err error

  base := fmt.Sprintf("%s/%s", home, data["version"])

  for _, file := range files {
    from := fmt.Sprintf("%s/%s", base, file)
    to := prefix

    // Older versions might not have certain files
    if _, err := os.Stat(from); os.IsNotExist(err) {
      continue
    }

    err = cprf.Copy(from, to)

    if err != nil {
      return err
    }
  }

  return nil
}

func info(url string) (file string, err error){
  response, err := client.Get(url)

  if err != nil {
    return "", err
  }

  defer response.Body.Close()
  contents, err := ioutil.ReadAll(response.Body)

  if err != nil {
    return "", err
  }

  return string(contents), nil
}
<|endoftext|>


<commit_before>// Daedalean-specific configuration file, defining a bunch of constants that are company-specific. Create your own and
// mark it with your build tag, then remove the !ddln tag below.
// +build ddln !ddln

package config

type RequirementLevel int

// Requirement levels according to DO-178C (do not change!)
const (
	SYSTEM RequirementLevel = iota
	HIGH
	LOW
)

// Document types:
// ORD - Overall (aka System) Requirement Document
// SRD - Software Requirements Data
// SDD - Software Design Description
// HRD - Hardware Requirements Data
// HDD - Hardware Design Description

// Requirement types:
// SYS - System/overall requirements (defined in ORD documents)
// SWH - Sofware high-level requirements (defined in SRD documents)
// SWL - Software low-level requirements (defined in SDD documents)
// HWH - Hardware high-level requirements (defined in HRD documents)
// HWL - Hardware low-level requirements (defined in HDD documents)

// Map from requirement type to requirement level.
var ReqTypeToReqLevel = map[string]RequirementLevel{
	"SYS": SYSTEM,
	"SWH": HIGH,
	"HWH": HIGH,
	"SWL": LOW,
	"HWL": LOW,
}

// Map from document type to requirement type.
var DocTypeToReqType = map[string]string{
	"ORD": "SYS",
	"SRD": "SWH",
	"HRD": "HWH",
	"SDD": "SWL",
	"HDD": "HWL",
}

// Map from requirement type to document type.
var ReqTypeToDocType = map[string]string{
	"SYS": "ORD",
	"SWH": "SRD",
	"SWL": "SDD",
	"HWH": "HRD",
	"HWL": "HDD",
}

// Map from document type to document ID.
// https://a.daedalean.ai/organisation-of-documentation
var DocTypeToDocId = map[string]string{
	"H":       "0",
	"DS":      "1",
	"CLSRS":   "5",
	"RS":      "6",
	"SDS":     "7",
	"CS":      "8",
	"HRS":     "9",
	"HCS":     "10",
	"HDS":     "11",
	"HVVS":    "12",
	"HAS":     "13",
	"HCMS":    "14",
	"PDAS":    "15",
	"CMP":     "20",
	"CLCMP":   "21",
	"PAP":     "22",
	"CLPAP":   "23",
	"CLTQP":   "25",
	"SCMP":    "26",
	"CLSCMP":  "27",
	"SQAP":    "28",
	"CLSQAP":  "29",
	"SDP":     "30",
	"HPAP":    "32",
	"CLHPAP":  "33",
	"TPPR":    "50",
	"TPSQAR":  "51",
	"ORD":     "100",
	"ICD":     "101",
	"CP":      "102",
	"DP":      "103",
	"DD":      "104",
	"VAP":     "105",
	"VEP":     "106",
	"CI":      "107",
	"FHA":     "108",
	"SFHA":    "109",
	"PSSA":    "110",
	"SSA":     "111",
	"CCA":     "112",
	"SPP":     "113",
	"VAD":     "114",
	"VED":     "115",
	"ECM":     "116",
	"EPA":     "117",
	"CSCR":    "118",
	"PSAC":    "134",
	"TQP":     "135",
	"SVP":     "136",
	"SRD":     "137",
	"SDD":     "138",
	"SLECI":   "141",
	"SCI":     "142",
	"SAS":     "143",
	"STD":     "144",
	"SQARI":   "145",
	"SPVP":    "146",
	"SVVDVP":  "147",
	"STCP":    "148",
	"SPVR":    "149",
	"SVVDVR":  "150",
	"STR":     "151",
	"PHAC":    "167",
	"HDP":     "168",
	"HVEP":    "169",
	"ECMP":    "170",
	"HVAP":    "171",
	"HCMP":    "172",
	"HECI":    "173",
	"HCI":     "174",
	"HRD":     "175",
	"HDD":     "176",
	"HTD":     "177",
	"HRAP":    "178",
	"HRAR":    "179",
	"HTP":     "180",
	"HTR":     "181",
	"HATC":    "182",
	"HACS":    "183",
	"FFPA":    "184",
	"TPORD":   "200",
	"TPICD":   "201",
	"TPCP":    "202",
	"TPDP":    "203",
	"TPDD":    "204",
	"TPVAP":   "205",
	"TPVEP":   "206",
	"TPCI":    "207",
	"TPFHA":   "208",
	"TPSFHA":  "209",
	"TPPSSA":  "210",
	"TPSSA":   "211",
	"TPCCA":   "212",
	"TPSPP":   "213",
	"TPVAD":   "214",
	"TPVED":   "215",
	"TPECM":   "216",
	"TPEPA":   "217",
	"TPCSCR":  "218",
	"TPPSAC":  "234",
	"TPSDP":   "235",
	"TPSVP":   "236",
	"TPSRD":   "237",
	"TPSDD":   "238",
	"TPSVCP":  "239",
	"TPSVR":   "240",
	"TPSLECI": "241",
	"TPSCI":   "242",
	"TPSAS":   "243",
	"TPSTD":   "244",
	"TPPHAC":  "267",
	"TPHDP":   "268",
	"TPHVEP":  "269",
	"TPECMP":  "270",
	"TPHVAP":  "271",
	"TPHCMP":  "272",
	"TPHECI":  "273",
	"TPHCI":   "274",
	"TPHRD":   "275",
	"TPHDD":   "276",
	"TPHTD":   "277",
	"TPHRAP":  "278",
	"TPHRAR":  "279",
	"TPHTP":   "280",
	"TPHTR":   "281",
	"TPHATC":  "282",
	"TPHACS":  "283",
	"TPFFPA":  "284",
	"CLORD":   "300",
	"CLICD":   "301",
	"CLCP":    "302",
	"CLDP":    "303",
	"CLDD":    "304",
	"CLVAP":   "305",
	"CLVEP":   "306",
	"CLCI":    "307",
	"CLFHA":   "308",
	"CLSFHA":  "309",
	"CLPSSA":  "310",
	"CLSSA":   "311",
	"CLCCA":   "312",
	"CLSPP":   "313",
	"CLVAD":   "314",
	"CLVED":   "315",
	"CLECM":   "316",
	"CLEPA":   "317",
	"CLCSCR":  "318",
	"CLPSAC":  "334",
	"CLSDP":   "335",
	"CLSVP":   "336",
	"CLSRD":   "337",
	"CLSDD":   "338",
	"CLSVCP":  "339",
	"CLSVR":   "340",
	"CLSLECI": "341",
	"CLSCI":   "342",
	"CLSAS":   "343",
	"CLSTD":   "344",
	"CLSOI1":  "345",
	"CLSOI2":  "346",
	"CLSOI3":  "347",
	"CLSOI4":  "348",
	"CLSPR":   "349",
	"CLRA":    "350",
	"CLSCR":   "351",
	"CLPHAC":  "367",
	"CLHDP":   "368",
	"CLHVEP":  "369",
	"CLECMP":  "370",
	"CLHVAP":  "371",
	"CLHCMP":  "372",
	"CLHECI":  "373",
	"CLHCI":   "374",
	"CLHRD":   "375",
	"CLHDD":   "376",
	"CLHTD":   "377",
	"CLHRAP":  "378",
	"CLHRAR":  "379",
	"CLHTP":   "380",
	"CLHTR":   "381",
	"CLHATC":  "382",
	"CLHACS":  "383",
	"CLFFPA":  "384",
}
<commit_msg>config: Update the doc types<commit_after>// Daedalean-specific configuration file, defining a bunch of constants that are company-specific. Create your own and
// mark it with your build tag, then remove the !ddln tag below.
// +build ddln !ddln

package config

type RequirementLevel int

// Requirement levels according to DO-178C (do not change!)
const (
	SYSTEM RequirementLevel = iota
	HIGH
	LOW
)

// Document types:
// ORD - Overall (aka System) Requirement Document
// SRD - Software Requirements Data
// SDD - Software Design Description
// HRD - Hardware Requirements Data
// HDD - Hardware Design Description

// Requirement types:
// SYS - System/overall requirements (defined in ORD documents)
// SWH - Sofware high-level requirements (defined in SRD documents)
// SWL - Software low-level requirements (defined in SDD documents)
// HWH - Hardware high-level requirements (defined in HRD documents)
// HWL - Hardware low-level requirements (defined in HDD documents)

// Map from requirement type to requirement level.
var ReqTypeToReqLevel = map[string]RequirementLevel{
	"SYS": SYSTEM,
	"SWH": HIGH,
	"HWH": HIGH,
	"SWL": LOW,
	"HWL": LOW,
}

// Map from document type to requirement type.
var DocTypeToReqType = map[string]string{
	"ORD": "SYS",
	"SRD": "SWH",
	"HRD": "HWH",
	"SDD": "SWL",
	"HDD": "HWL",
}

// Map from requirement type to document type.
var ReqTypeToDocType = map[string]string{
	"SYS": "ORD",
	"SWH": "SRD",
	"SWL": "SDD",
	"HWH": "HRD",
	"HWL": "HDD",
}

// Map from document type to document ID.
// https://a.daedalean.ai/organisation-of-documentation
var DocTypeToDocId = map[string]string{
	"H":       "0",
	"DS":      "1",
	"CLSRS":   "5",
	"RS":      "6",
	"SDS":     "7",
	"CS":      "8",
	"HRS":     "9",
	"HCS":     "10",
	"HDS":     "11",
	"HVVS":    "12",
	"HARS":    "13",
	"CLCMP":   "21",
	"CLPAP":   "23",
	"CLTQP":   "25",
	"CMP":     "26",
	"CLSCMP":  "27",
	"QAP":     "28",
	"CLSQAP":  "29",
	"SDP":     "30",
	"CLHPAP":  "33",
	"TPPR":    "50",
	"TPSQAR":  "51",
	"ORD":     "100",
	"ICD":     "101",
	"CP":      "102",
	"DP":      "103",
	"DD":      "104",
	"VAP":     "105",
	"VEP":     "106",
	"CI":      "107",
	"FHA":     "108",
	"SFHA":    "109",
	"PSSA":    "110",
	"SSA":     "111",
	"CCA":     "112",
	"SPP":     "113",
	"VAD":     "114",
	"VED":     "115",
	"ECM":     "116",
	"EPA":     "117",
	"CSCR":    "118",
	"PSAC":    "134",
	"TQP":     "135",
	"SVP":     "136",
	"SRD":     "137",
	"SDD":     "138",
	"SLECI":   "141",
	"SCI":     "142",
	"SAS":     "143",
	"STD":     "144",
	"SQARI":   "145",
	"SPVP":    "146",
	"SVVDVP":  "147",
	"STCP":    "148",
	"SPVR":    "149",
	"SVVDVR":  "150",
	"STR":     "151",
	"PHAC":    "167",
	"HDP":     "168",
	"HVVP":    "169",
	"ECMP":    "170",
	"HECI":    "173",
	"HCI":     "174",
	"HRD":     "175",
	"HDD":     "176",
	"HTD":     "177",
	"HRAP":    "178",
	"HRAR":    "179",
	"HTP":     "180",
	"HTR":     "181",
	"HATC":    "182",
	"HAS":     "183",
	"FFPA":    "184",
	"TPORD":   "200",
	"TPICD":   "201",
	"TPCP":    "202",
	"TPDP":    "203",
	"TPDD":    "204",
	"TPVAP":   "205",
	"TPVEP":   "206",
	"TPCI":    "207",
	"TPFHA":   "208",
	"TPSFHA":  "209",
	"TPPSSA":  "210",
	"TPSSA":   "211",
	"TPCCA":   "212",
	"TPSPP":   "213",
	"TPVAD":   "214",
	"TPVED":   "215",
	"TPECM":   "216",
	"TPEPA":   "217",
	"TPCSCR":  "218",
	"TPPSAC":  "234",
	"TPSDP":   "235",
	"TPSVP":   "236",
	"TPSRD":   "237",
	"TPSDD":   "238",
	"TPSVCP":  "239",
	"TPSVR":   "240",
	"TPSLECI": "241",
	"TPSCI":   "242",
	"TPSAS":   "243",
	"TPSTD":   "244",
	"TPPHAC":  "267",
	"TPHDP":   "268",
	"TPHVEP":  "269",
	"TPECMP":  "270",
	"TPHVAP":  "271",
	"TPHCMP":  "272",
	"TPHECI":  "273",
	"TPHCI":   "274",
	"TPHRD":   "275",
	"TPHDD":   "276",
	"TPHTD":   "277",
	"TPHRAP":  "278",
	"TPHRAR":  "279",
	"TPHTP":   "280",
	"TPHTR":   "281",
	"TPHATC":  "282",
	"TPHACS":  "283",
	"TPFFPA":  "284",
	"CLORD":   "300",
	"CLICD":   "301",
	"CLCP":    "302",
	"CLDP":    "303",
	"CLDD":    "304",
	"CLVAP":   "305",
	"CLVEP":   "306",
	"CLCI":    "307",
	"CLFHA":   "308",
	"CLSFHA":  "309",
	"CLPSSA":  "310",
	"CLSSA":   "311",
	"CLCCA":   "312",
	"CLSPP":   "313",
	"CLVAD":   "314",
	"CLVED":   "315",
	"CLECM":   "316",
	"CLEPA":   "317",
	"CLCSCR":  "318",
	"CLPSAC":  "334",
	"CLSDP":   "335",
	"CLSVP":   "336",
	"CLSRD":   "337",
	"CLSDD":   "338",
	"CLSVCP":  "339",
	"CLSVR":   "340",
	"CLSLECI": "341",
	"CLSCI":   "342",
	"CLSAS":   "343",
	"CLSTD":   "344",
	"CLSOI1":  "345",
	"CLSOI2":  "346",
	"CLSOI3":  "347",
	"CLSOI4":  "348",
	"CLSPR":   "349",
	"CLRA":    "350",
	"CLSCR":   "351",
	"CLPHAC":  "367",
	"CLHDP":   "368",
	"CLHVEP":  "369",
	"CLECMP":  "370",
	"CLHVAP":  "371",
	"CLHCMP":  "372",
	"CLHECI":  "373",
	"CLHCI":   "374",
	"CLHRD":   "375",
	"CLHDD":   "376",
	"CLHTD":   "377",
	"CLHRAP":  "378",
	"CLHRAR":  "379",
	"CLHTP":   "380",
	"CLHTR":   "381",
	"CLHATC":  "382",
	"CLHACS":  "383",
	"CLFFPA":  "384",
}
<|endoftext|>


<commit_before>package typhon

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"testing"
	"time"

	"github.com/facebookgo/httpcontrol"
	"github.com/fortytw2/leaktest"
	"github.com/monzo/terrors"
	"github.com/stretchr/testify/suite"
)

func TestE2E(t *testing.T) {
	t.Parallel()
	suite.Run(t, &e2eSuite{})
}

type e2eSuite struct {
	suite.Suite
}

func (suite *e2eSuite) SetupTest() {
	Client = Service(BareClient).Filter(ErrorFilter)
}

func (suite *e2eSuite) TearDownTest() {
	Client = BareClient
}

func (suite *e2eSuite) serve(svc Service) Server {
	s, err := Listen(svc, "localhost:0")
	suite.Require().NoError(err)
	return s
}

func (suite *e2eSuite) TestStraightforward() {
	defer leaktest.Check(suite.T())()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	svc := Service(func(req Request) Response {
		// Simple requests like this shouldn't be chunked
		suite.Assert().NotContains(req.TransferEncoding, "chunked")
		suite.Assert().True(req.ContentLength > 0)
		return req.Response(map[string]string{
			"b": "a"})
	})
	svc = svc.Filter(ErrorFilter)
	s := suite.serve(svc)
	defer s.Stop()

	req := NewRequest(ctx, "GET", fmt.Sprintf("http://%s", s.Listener().Addr()), map[string]string{
		"a": "b"})
	rsp := req.Send().Response()
	suite.Require().NoError(rsp.Error)
	suite.Assert().Equal(http.StatusOK, rsp.StatusCode)
	// The response is simple too; shouldn't be chunked
	suite.Assert().NotContains(rsp.TransferEncoding, "chunked")
	suite.Assert().True(rsp.ContentLength > 0)
}

func (suite *e2eSuite) TestDomainSocket() {
	defer leaktest.Check(suite.T())()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	svc := Service(func(req Request) Response {
		return NewResponse(req)
	})
	svc = svc.Filter(ErrorFilter)

	addr := &net.UnixAddr{
		Net:  "unix",
		Name: "/tmp/typhon-test.sock"}
	l, err := net.ListenUnix("unix", addr)
	suite.Require().NoError(err)
	defer l.Close()

	s, err := Serve(svc, l)
	suite.Require().NoError(err)
	defer s.Stop()

	sockTransport := &httpcontrol.Transport{
		Dial: func(network, address string) (net.Conn, error) {
			return net.DialUnix("unix", nil, addr)
		}}
	req := NewRequest(ctx, "GET", "http://localhost/foo", nil)
	rsp := req.SendVia(HttpService(sockTransport)).Response()
	suite.Require().NoError(rsp.Error)
	suite.Assert().Equal(http.StatusOK, rsp.StatusCode)
}

func (suite *e2eSuite) TestError() {
	defer leaktest.Check(suite.T())()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	expectedErr := terrors.Unauthorized("ah_ah_ah", "You didn't say the magic word!", map[string]string{
		"param": "value"})
	svc := Service(func(req Request) Response {
		rsp := Response{
			Error: expectedErr}
		rsp.Write([]byte("throwaway")) // should be removed
		return rsp
	})
	svc = svc.Filter(ErrorFilter)
	s := suite.serve(svc)
	defer s.Stop()

	req := NewRequest(ctx, "GET", fmt.Sprintf("http://%s", s.Listener().Addr()), nil)
	rsp := req.Send().Response()
	suite.Assert().Equal(http.StatusUnauthorized, rsp.StatusCode)

	b, _ := rsp.BodyBytes(false)
	suite.Assert().NotContains(string(b), "throwaway")

	suite.Require().Error(rsp.Error)
	terr := terrors.Wrap(rsp.Error, nil).(*terrors.Error)
	terrExpect := terrors.Unauthorized("ah_ah_ah", "You didn't say the magic word!", nil)
	suite.Assert().Equal(terrExpect.Message, terr.Message)
	suite.Assert().Equal(terrExpect.Code, terr.Code)
	suite.Assert().Equal("value", terr.Params["param"])
}

func (suite *e2eSuite) TestCancellation() {
	defer leaktest.Check(suite.T())()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	cancelled := make(chan struct{})
	svc := Service(func(req Request) Response {
		<-req.Done()
		close(cancelled)
		return req.Response("cancelled ok")
	})
	svc = svc.Filter(ErrorFilter)
	s := suite.serve(svc)
	defer s.Stop()

	req := NewRequest(ctx, "GET", fmt.Sprintf("http://%s/", s.Listener().Addr()), nil)
	f := req.Send()
	select {
	case <-cancelled:
		suite.Assert().Fail("cancellation propagated prematurely")
	case <-time.After(30 * time.Millisecond):
	}
	f.Cancel()
	select {
	case <-cancelled:
	case <-time.After(30 * time.Millisecond):
		suite.Assert().Fail("cancellation not propagated")
	}
}

func (suite *e2eSuite) TestNoFollowRedirect() {
	defer leaktest.Check(suite.T())()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	svc := Service(func(req Request) Response {
		if req.URL.Path == "/redirected" {
			return req.Response("😱")
		}

		rsp := req.Response(nil)
		dst := fmt.Sprintf("http://%s/redirected", req.Host)
		http.Redirect(rsp.Writer(), &req.Request, dst, http.StatusFound)
		return rsp
	})
	s := suite.serve(svc)
	defer s.Stop()
	req := NewRequest(ctx, "GET", fmt.Sprintf("http://%s/", s.Listener().Addr()), nil)
	rsp := req.Send().Response()
	suite.Assert().NoError(rsp.Error)
	suite.Assert().Equal(http.StatusFound, rsp.StatusCode)
}

func (suite *e2eSuite) TestProxiedStreamer() {
	defer leaktest.Check(suite.T())()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	chunks := make(chan bool, 2)
	chunks <- true
	downstream := Service(func(req Request) Response {
		rsp := req.Response(nil)
		rsp.Body = Streamer()
		go func() {
			defer rsp.Body.Close()
			n := 0
			for range chunks {
				rsp.Encode(map[string]int{
					"chunk": n})
				n++
			}
		}()
		return rsp
	})
	s := suite.serve(downstream)
	defer s.Stop()

	proxy := Service(func(req Request) Response {
		proxyReq := NewRequest(req, "GET", fmt.Sprintf("http://%s/", s.Listener().Addr()), nil)
		return proxyReq.Send().Response()
	})
	ps := suite.serve(proxy)
	defer ps.Stop()

	req := NewRequest(ctx, "GET", fmt.Sprintf("http://%s/", ps.Listener().Addr()), nil)
	rsp := req.Send().Response()
	suite.Assert().NoError(rsp.Error)
	suite.Assert().Equal(http.StatusOK, rsp.StatusCode)
	// The response is streaming; should be chunked
	suite.Assert().Contains(rsp.TransferEncoding, "chunked")
	suite.Assert().True(rsp.ContentLength < 0)
	for i := 0; i < 1000; i++ {
		b := make([]byte, 500)
		n, err := rsp.Body.Read(b)
		suite.Require().NoError(err)
		v := map[string]int{}
		suite.Require().NoError(json.Unmarshal(b[:n], &v))
		suite.Require().Equal(i, v["chunk"])
		chunks <- true
	}
	close(chunks)
}

// TestInfiniteContext verifies that Typhon does not leak Goroutines if an infinite context (one that's never calcelled)
// is used to make a request.
func (suite *e2eSuite) TestInfiniteContext() {
	defer leaktest.Check(suite.T())()
	ctx := context.Background()

	svc := Service(func(req Request) Response {
		return req.Response(map[string]string{
			"b": "a"})
	})
	svc = svc.Filter(ErrorFilter)
	s := suite.serve(svc)
	defer s.Stop()

	req := NewRequest(ctx, "GET", fmt.Sprintf("http://%s", s.Listener().Addr()), map[string]string{
		"a": "b"})
	rsp := req.Send().Response()
	suite.Require().NoError(rsp.Error)
	suite.Assert().Equal(http.StatusOK, rsp.StatusCode)

	ioutil.ReadAll(rsp.Body) // Consume the body (after which the request should be auto-closed)
}
<commit_msg>Typo 🙈<commit_after>package typhon

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"testing"
	"time"

	"github.com/facebookgo/httpcontrol"
	"github.com/fortytw2/leaktest"
	"github.com/monzo/terrors"
	"github.com/stretchr/testify/suite"
)

func TestE2E(t *testing.T) {
	t.Parallel()
	suite.Run(t, &e2eSuite{})
}

type e2eSuite struct {
	suite.Suite
}

func (suite *e2eSuite) SetupTest() {
	Client = Service(BareClient).Filter(ErrorFilter)
}

func (suite *e2eSuite) TearDownTest() {
	Client = BareClient
}

func (suite *e2eSuite) serve(svc Service) Server {
	s, err := Listen(svc, "localhost:0")
	suite.Require().NoError(err)
	return s
}

func (suite *e2eSuite) TestStraightforward() {
	defer leaktest.Check(suite.T())()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	svc := Service(func(req Request) Response {
		// Simple requests like this shouldn't be chunked
		suite.Assert().NotContains(req.TransferEncoding, "chunked")
		suite.Assert().True(req.ContentLength > 0)
		return req.Response(map[string]string{
			"b": "a"})
	})
	svc = svc.Filter(ErrorFilter)
	s := suite.serve(svc)
	defer s.Stop()

	req := NewRequest(ctx, "GET", fmt.Sprintf("http://%s", s.Listener().Addr()), map[string]string{
		"a": "b"})
	rsp := req.Send().Response()
	suite.Require().NoError(rsp.Error)
	suite.Assert().Equal(http.StatusOK, rsp.StatusCode)
	// The response is simple too; shouldn't be chunked
	suite.Assert().NotContains(rsp.TransferEncoding, "chunked")
	suite.Assert().True(rsp.ContentLength > 0)
}

func (suite *e2eSuite) TestDomainSocket() {
	defer leaktest.Check(suite.T())()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	svc := Service(func(req Request) Response {
		return NewResponse(req)
	})
	svc = svc.Filter(ErrorFilter)

	addr := &net.UnixAddr{
		Net:  "unix",
		Name: "/tmp/typhon-test.sock"}
	l, err := net.ListenUnix("unix", addr)
	suite.Require().NoError(err)
	defer l.Close()

	s, err := Serve(svc, l)
	suite.Require().NoError(err)
	defer s.Stop()

	sockTransport := &httpcontrol.Transport{
		Dial: func(network, address string) (net.Conn, error) {
			return net.DialUnix("unix", nil, addr)
		}}
	req := NewRequest(ctx, "GET", "http://localhost/foo", nil)
	rsp := req.SendVia(HttpService(sockTransport)).Response()
	suite.Require().NoError(rsp.Error)
	suite.Assert().Equal(http.StatusOK, rsp.StatusCode)
}

func (suite *e2eSuite) TestError() {
	defer leaktest.Check(suite.T())()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	expectedErr := terrors.Unauthorized("ah_ah_ah", "You didn't say the magic word!", map[string]string{
		"param": "value"})
	svc := Service(func(req Request) Response {
		rsp := Response{
			Error: expectedErr}
		rsp.Write([]byte("throwaway")) // should be removed
		return rsp
	})
	svc = svc.Filter(ErrorFilter)
	s := suite.serve(svc)
	defer s.Stop()

	req := NewRequest(ctx, "GET", fmt.Sprintf("http://%s", s.Listener().Addr()), nil)
	rsp := req.Send().Response()
	suite.Assert().Equal(http.StatusUnauthorized, rsp.StatusCode)

	b, _ := rsp.BodyBytes(false)
	suite.Assert().NotContains(string(b), "throwaway")

	suite.Require().Error(rsp.Error)
	terr := terrors.Wrap(rsp.Error, nil).(*terrors.Error)
	terrExpect := terrors.Unauthorized("ah_ah_ah", "You didn't say the magic word!", nil)
	suite.Assert().Equal(terrExpect.Message, terr.Message)
	suite.Assert().Equal(terrExpect.Code, terr.Code)
	suite.Assert().Equal("value", terr.Params["param"])
}

func (suite *e2eSuite) TestCancellation() {
	defer leaktest.Check(suite.T())()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	cancelled := make(chan struct{})
	svc := Service(func(req Request) Response {
		<-req.Done()
		close(cancelled)
		return req.Response("cancelled ok")
	})
	svc = svc.Filter(ErrorFilter)
	s := suite.serve(svc)
	defer s.Stop()

	req := NewRequest(ctx, "GET", fmt.Sprintf("http://%s/", s.Listener().Addr()), nil)
	f := req.Send()
	select {
	case <-cancelled:
		suite.Assert().Fail("cancellation propagated prematurely")
	case <-time.After(30 * time.Millisecond):
	}
	f.Cancel()
	select {
	case <-cancelled:
	case <-time.After(30 * time.Millisecond):
		suite.Assert().Fail("cancellation not propagated")
	}
}

func (suite *e2eSuite) TestNoFollowRedirect() {
	defer leaktest.Check(suite.T())()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	svc := Service(func(req Request) Response {
		if req.URL.Path == "/redirected" {
			return req.Response("😱")
		}

		rsp := req.Response(nil)
		dst := fmt.Sprintf("http://%s/redirected", req.Host)
		http.Redirect(rsp.Writer(), &req.Request, dst, http.StatusFound)
		return rsp
	})
	s := suite.serve(svc)
	defer s.Stop()
	req := NewRequest(ctx, "GET", fmt.Sprintf("http://%s/", s.Listener().Addr()), nil)
	rsp := req.Send().Response()
	suite.Assert().NoError(rsp.Error)
	suite.Assert().Equal(http.StatusFound, rsp.StatusCode)
}

func (suite *e2eSuite) TestProxiedStreamer() {
	defer leaktest.Check(suite.T())()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	chunks := make(chan bool, 2)
	chunks <- true
	downstream := Service(func(req Request) Response {
		rsp := req.Response(nil)
		rsp.Body = Streamer()
		go func() {
			defer rsp.Body.Close()
			n := 0
			for range chunks {
				rsp.Encode(map[string]int{
					"chunk": n})
				n++
			}
		}()
		return rsp
	})
	s := suite.serve(downstream)
	defer s.Stop()

	proxy := Service(func(req Request) Response {
		proxyReq := NewRequest(req, "GET", fmt.Sprintf("http://%s/", s.Listener().Addr()), nil)
		return proxyReq.Send().Response()
	})
	ps := suite.serve(proxy)
	defer ps.Stop()

	req := NewRequest(ctx, "GET", fmt.Sprintf("http://%s/", ps.Listener().Addr()), nil)
	rsp := req.Send().Response()
	suite.Assert().NoError(rsp.Error)
	suite.Assert().Equal(http.StatusOK, rsp.StatusCode)
	// The response is streaming; should be chunked
	suite.Assert().Contains(rsp.TransferEncoding, "chunked")
	suite.Assert().True(rsp.ContentLength < 0)
	for i := 0; i < 1000; i++ {
		b := make([]byte, 500)
		n, err := rsp.Body.Read(b)
		suite.Require().NoError(err)
		v := map[string]int{}
		suite.Require().NoError(json.Unmarshal(b[:n], &v))
		suite.Require().Equal(i, v["chunk"])
		chunks <- true
	}
	close(chunks)
}

// TestInfiniteContext verifies that Typhon does not leak Goroutines if an infinite context (one that's never cancelled)
// is used to make a request.
func (suite *e2eSuite) TestInfiniteContext() {
	defer leaktest.Check(suite.T())()
	ctx := context.Background()

	svc := Service(func(req Request) Response {
		return req.Response(map[string]string{
			"b": "a"})
	})
	svc = svc.Filter(ErrorFilter)
	s := suite.serve(svc)
	defer s.Stop()

	req := NewRequest(ctx, "GET", fmt.Sprintf("http://%s", s.Listener().Addr()), map[string]string{
		"a": "b"})
	rsp := req.Send().Response()
	suite.Require().NoError(rsp.Error)
	suite.Assert().Equal(http.StatusOK, rsp.StatusCode)

	ioutil.ReadAll(rsp.Body) // Consume the body (after which the request should be auto-closed)
}
<|endoftext|>


<commit_before>package system

import (
	"bufio"
	"bytes"
	"fmt"

	"github.com/shirou/gopsutil/host"
	"github.com/shirou/gopsutil/load"

	"github.com/influxdb/telegraf/plugins"
)

type SystemStats struct{}

func (_ *SystemStats) Description() string {
	return "Read metrics about system load & uptime"
}

func (_ *SystemStats) SampleConfig() string { return "" }

func (_ *SystemStats) add(acc plugins.Accumulator,
	name string, val float64, tags map[string]string) {
	if val >= 0 {
		acc.Add(name, val, tags)
	}
}

func (_ *SystemStats) Gather(acc plugins.Accumulator) error {
	loadavg, err := load.LoadAvg()
	if err != nil {
		return err
	}

	hostinfo, err := host.HostInfo()
	if err != nil {
		return err
	}

	acc.Add("load1", loadavg.Load1, nil)
	acc.Add("load5", loadavg.Load5, nil)
	acc.Add("load15", loadavg.Load15, nil)
	acc.Add("uptime", hostinfo.Uptime, nil)
	acc.Add("uptime_format", format_uptime(hostinfo.Uptime), nil)

	return nil
}

func format_uptime(uptime uint64) string {
	buf := new(bytes.Buffer)
	w := bufio.NewWriter(buf)

	days := uptime / (60 * 60 * 24)

	if days != 0 {
		s := ""
		if days > 1 {
			s = "s"
		}
		fmt.Fprintf(w, "%d day%s, ", days, s)
	}

	minutes := uptime / 60
	hours := minutes / 60
	hours %= 24
	minutes %= 60

	fmt.Fprintf(w, "%2d:%02d", hours, minutes)

	w.Flush()
	return buf.String()
}

func init() {
	plugins.Add("system", func() plugins.Plugin {
		return &SystemStats{}
	})
}
<commit_msg>Convert uptime to float64 for backwards compatability.<commit_after>package system

import (
	"bufio"
	"bytes"
	"fmt"

	"github.com/shirou/gopsutil/host"
	"github.com/shirou/gopsutil/load"

	"github.com/influxdb/telegraf/plugins"
)

type SystemStats struct{}

func (_ *SystemStats) Description() string {
	return "Read metrics about system load & uptime"
}

func (_ *SystemStats) SampleConfig() string { return "" }

func (_ *SystemStats) add(acc plugins.Accumulator,
	name string, val float64, tags map[string]string) {
	if val >= 0 {
		acc.Add(name, val, tags)
	}
}

func (_ *SystemStats) Gather(acc plugins.Accumulator) error {
	loadavg, err := load.LoadAvg()
	if err != nil {
		return err
	}

	hostinfo, err := host.HostInfo()
	if err != nil {
		return err
	}

	acc.Add("load1", loadavg.Load1, nil)
	acc.Add("load5", loadavg.Load5, nil)
	acc.Add("load15", loadavg.Load15, nil)
	acc.Add("uptime", float64(hostinfo.Uptime), nil)
	acc.Add("uptime_format", format_uptime(hostinfo.Uptime), nil)

	return nil
}

func format_uptime(uptime uint64) string {
	buf := new(bytes.Buffer)
	w := bufio.NewWriter(buf)

	days := uptime / (60 * 60 * 24)

	if days != 0 {
		s := ""
		if days > 1 {
			s = "s"
		}
		fmt.Fprintf(w, "%d day%s, ", days, s)
	}

	minutes := uptime / 60
	hours := minutes / 60
	hours %= 24
	minutes %= 60

	fmt.Fprintf(w, "%2d:%02d", hours, minutes)

	w.Flush()
	return buf.String()
}

func init() {
	plugins.Add("system", func() plugins.Plugin {
		return &SystemStats{}
	})
}
<|endoftext|>


<commit_before>package broker

import (
	grpc "google.golang.org/grpc"
	codes "google.golang.org/grpc/codes"
	emulators "google/emulators"
	"testing"
)

func TestCreateSpec(t *testing.T) {

	s := New()
	want := &emulators.EmulatorSpec{
		Id:            "foo",
		TargetPattern: []string{"foo*./", "bar*./"},
		CommandLine: &emulators.CommandLine{
			Path: "/exepath",
			Args: []string{"arg1", "arg2"},
		},
	}

	req := &emulators.CreateEmulatorSpecRequest{
		SpecId: "foo",
		Spec:   want}
	spec, err := s.CreateEmulatorSpec(nil, req)

	if err != nil {
		panic(err)
	}

	got, err := s.GetEmulatorSpec(nil, &emulators.SpecId{spec.Id})

	if err != nil {
		panic(err)
	}

	if got != want {
		t.Errorf("Failed to find back the same spec want = %v, got %v", want, got)
	}
}

func TestDoubleCreateSpec(t *testing.T) {

	s := New()
	want := &emulators.EmulatorSpec{
		Id:            "foo",
		TargetPattern: []string{"foo*./", "bar*./"},
		CommandLine: &emulators.CommandLine{
			Path: "/exepath",
			Args: []string{"arg1", "arg2"},
		},
	}

	req := &emulators.CreateEmulatorSpecRequest{
		SpecId: "foo",
		Spec:   want}
	_, err := s.CreateEmulatorSpec(nil, req)

	if err != nil {
		panic(err)
	}

	spec, err := s.CreateEmulatorSpec(nil, req)

	if err == nil {
		t.Errorf("This creation should have failed.")
	}

	if grpc.Code(err) != codes.AlreadyExists {
		t.Errorf("This creation should have failed with AlreadyExists.")
	}

	if spec != nil {
		t.Errorf("It should not have returned a spec %q.", spec)
	}
}
<commit_msg>test on non existent spec<commit_after>package broker

import (
	grpc "google.golang.org/grpc"
	codes "google.golang.org/grpc/codes"
	emulators "google/emulators"
	"testing"
)

func TestCreateSpec(t *testing.T) {

	s := New()
	want := &emulators.EmulatorSpec{
		Id:            "foo",
		TargetPattern: []string{"foo*./", "bar*./"},
		CommandLine: &emulators.CommandLine{
			Path: "/exepath",
			Args: []string{"arg1", "arg2"},
		},
	}

	req := &emulators.CreateEmulatorSpecRequest{
		SpecId: "foo",
		Spec:   want}
	spec, err := s.CreateEmulatorSpec(nil, req)

	if err != nil {
		panic(err)
	}

	got, err := s.GetEmulatorSpec(nil, &emulators.SpecId{spec.Id})

	if err != nil {
		panic(err)
	}

	if got != want {
		t.Errorf("Failed to find back the same spec want = %v, got %v", want, got)
	}
}

func TestDoubleCreateSpec(t *testing.T) {

	s := New()
	want := &emulators.EmulatorSpec{
		Id:            "foo",
		TargetPattern: []string{"foo*./", "bar*./"},
		CommandLine: &emulators.CommandLine{
			Path: "/exepath",
			Args: []string{"arg1", "arg2"},
		},
	}

	req := &emulators.CreateEmulatorSpecRequest{
		SpecId: "foo",
		Spec:   want}
	_, err := s.CreateEmulatorSpec(nil, req)

	if err != nil {
		panic(err)
	}

	spec, err := s.CreateEmulatorSpec(nil, req)

	if err == nil {
		t.Errorf("This creation should have failed.")
	}

	if grpc.Code(err) != codes.AlreadyExists {
		t.Errorf("This creation should have failed with AlreadyExists.")
	}

	if spec != nil {
		t.Errorf("It should not have returned a spec %q.", spec)
	}
}

func TestMissingSpec(t *testing.T) {
	s := New()
	_, err := s.GetEmulatorSpec(nil, &emulators.SpecId{"whatever"})

	if err == nil {
		t.Errorf("Get of a non existent spec should have failed.")
	}
	if grpc.Code(err) != codes.NotFound {
		t.Errorf("Get should return NotFound as error")
	}

}
<|endoftext|>