<commit_before>// Package gcloud implements remote storage of state on Google Cloud Storage (GCS).
package gcloud

import (
	"context"
	"fmt"
	"strings"

	"cloud.google.com/go/storage"
	"github.com/hashicorp/terraform/backend"
	"github.com/hashicorp/terraform/helper/pathorcontents"
	"github.com/hashicorp/terraform/helper/schema"
	googleContext "golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/option"
)

// gcsBackend implements "backend".Backend for GCS.
// Input(), Validate() and Configure() are implemented by embedding *schema.Backend.
// State(), DeleteState() and States() are implemented explicitly.
type gcsBackend struct {
	*schema.Backend

	storageClient  *storage.Client
	storageContext googleContext.Context

	bucketName string
	stateDir   string
}

func New() backend.Backend {
	be := &gcsBackend{}
	be.Backend = &schema.Backend{
		ConfigureFunc: be.configure,
		Schema: map[string]*schema.Schema{
			"bucket": {
				Type:        schema.TypeString,
				Required:    true,
				Description: "The name of the Google Cloud Storage bucket",
			},

			"state_dir": {
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The directory where state files will be saved inside the bucket",
			},

			"credentials": {
				Type:        schema.TypeString,
				Optional:    true,
				Description: "Google Cloud JSON Account Key",
				Default:     "",
			},
		},
	}

	return be
}

func (b *gcsBackend) configure(ctx context.Context) error {
	if b.storageClient != nil {
		return nil
	}

	// ctx is a background context with the backend config added.
	// Since no context is passed to remoteClient.Get(), .Lock(), etc. but
	// one is required for calling the GCP API, we're holding on to this
	// context here and re-use it later.
	b.storageContext = ctx

	data := schema.FromContextBackendConfig(b.storageContext)

	b.bucketName = data.Get("bucket").(string)
	b.stateDir = strings.TrimLeft(data.Get("state_dir").(string), "/")

	var tokenSource oauth2.TokenSource

	if credentials := data.Get("credentials").(string); credentials != "" {
		credentialsJson, _, err := pathorcontents.Read(data.Get("credentials").(string))
		if err != nil {
			return fmt.Errorf("Error loading credentials: %v", err)
		}

		jwtConfig, err := google.JWTConfigFromJSON([]byte(credentialsJson), storage.ScopeReadWrite)
		if err != nil {
			return fmt.Errorf("Failed to get Google OAuth2 token: %v", err)
		}

		tokenSource = jwtConfig.TokenSource(b.storageContext)
	} else {
		var err error
		tokenSource, err = google.DefaultTokenSource(b.storageContext, storage.ScopeReadWrite)
		if err != nil {
			return fmt.Errorf("Failed to get Google Application Default Credentials: %v", err)
		}
	}

	client, err := storage.NewClient(b.storageContext, option.WithTokenSource(tokenSource))
	if err != nil {
		return fmt.Errorf("Failed to create Google Storage client: %v", err)
	}

	b.storageClient = client

	return nil
}
<commit_msg>backend/remote-state/gcloud: Unify on the "context" package.<commit_after>// Package gcloud implements remote storage of state on Google Cloud Storage (GCS).
package gcloud

import (
	"context"
	"fmt"
	"strings"

	"cloud.google.com/go/storage"
	"github.com/hashicorp/terraform/backend"
	"github.com/hashicorp/terraform/helper/pathorcontents"
	"github.com/hashicorp/terraform/helper/schema"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/option"
)

// gcsBackend implements "backend".Backend for GCS.
// Input(), Validate() and Configure() are implemented by embedding *schema.Backend.
// State(), DeleteState() and States() are implemented explicitly.
type gcsBackend struct {
	*schema.Backend

	storageClient  *storage.Client
	storageContext context.Context

	bucketName string
	stateDir   string
}

func New() backend.Backend {
	be := &gcsBackend{}
	be.Backend = &schema.Backend{
		ConfigureFunc: be.configure,
		Schema: map[string]*schema.Schema{
			"bucket": {
				Type:        schema.TypeString,
				Required:    true,
				Description: "The name of the Google Cloud Storage bucket",
			},

			"state_dir": {
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The directory where state files will be saved inside the bucket",
			},

			"credentials": {
				Type:        schema.TypeString,
				Optional:    true,
				Description: "Google Cloud JSON Account Key",
				Default:     "",
			},
		},
	}

	return be
}

func (b *gcsBackend) configure(ctx context.Context) error {
	if b.storageClient != nil {
		return nil
	}

	// ctx is a background context with the backend config added.
	// Since no context is passed to remoteClient.Get(), .Lock(), etc. but
	// one is required for calling the GCP API, we're holding on to this
	// context here and re-use it later.
	b.storageContext = ctx

	data := schema.FromContextBackendConfig(b.storageContext)

	b.bucketName = data.Get("bucket").(string)
	b.stateDir = strings.TrimLeft(data.Get("state_dir").(string), "/")

	var tokenSource oauth2.TokenSource

	if credentials := data.Get("credentials").(string); credentials != "" {
		credentialsJson, _, err := pathorcontents.Read(data.Get("credentials").(string))
		if err != nil {
			return fmt.Errorf("Error loading credentials: %v", err)
		}

		jwtConfig, err := google.JWTConfigFromJSON([]byte(credentialsJson), storage.ScopeReadWrite)
		if err != nil {
			return fmt.Errorf("Failed to get Google OAuth2 token: %v", err)
		}

		tokenSource = jwtConfig.TokenSource(b.storageContext)
	} else {
		var err error
		tokenSource, err = google.DefaultTokenSource(b.storageContext, storage.ScopeReadWrite)
		if err != nil {
			return fmt.Errorf("Failed to get Google Application Default Credentials: %v", err)
		}
	}

	client, err := storage.NewClient(b.storageContext, option.WithTokenSource(tokenSource))
	if err != nil {
		return fmt.Errorf("Failed to create Google Storage client: %v", err)
	}

	b.storageClient = client

	return nil
}
<|endoftext|>


<commit_before>package importer

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"sync"

	"github.com/sgeisbacher/photogallery-api/media"
)

type ImportManager struct {
	MediaService   *media.MediaService
	GalleryService *media.GalleryService
}

func (mgr ImportManager) ScanFolder(path string) {
	var wg sync.WaitGroup
	imageFilesChan := make(chan string)

	// start worker threads
	go mgr.handleImageFile(imageFilesChan, &wg)
	go mgr.handleImageFile(imageFilesChan, &wg)

	files, err := ioutil.ReadDir(path)
	if err != nil {
		log.Fatalf("error while reading dir: %v", err)
	}
	for _, file := range files {
		wg.Add(1)
		if !file.IsDir() {
			imageFilesChan <- path + "/" + file.Name()
		}
	}

	close(imageFilesChan)
	wg.Wait()
}

func (mgr ImportManager) handleImageFile(imagesChan <-chan string, wg *sync.WaitGroup) {
	for filePath := range imagesChan {
		fileHash, err := hashFile(filePath)
		if err != nil {
			fmt.Printf("skipping file '%v' due to an error: %v\n", filePath, err)
			continue
		}
		fmt.Printf("md5ChkSum of '%v': %v\n", filePath, fileHash)
		wg.Done()
	}
}

func hashFile(filename string) (string, error) {
	var md5ChkSum string
	file, err := os.Open(filename)
	if err != nil {
		fmt.Println("could not open file:", filename)
		return md5ChkSum, err
	}
	defer file.Close()
	hash := md5.New()
	if _, err := io.Copy(hash, file); err != nil {
		fmt.Println("could not hash file:", filename)
		return md5ChkSum, err
	}
	md5ChkSum = hex.EncodeToString(hash.Sum(nil))
	return md5ChkSum, nil
}
<commit_msg>added gallery-detection to importer<commit_after>package importer

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"

	"github.com/sgeisbacher/photogallery-api/media"
)

type ImportMediaData struct {
	path        string
	galleryName string
}

type ImportManager struct {
	MediaService   *media.MediaService
	GalleryService *media.GalleryService
}

func (mgr ImportManager) ScanFolder(path string) {
	var wg sync.WaitGroup
	imagesChan := make(chan ImportMediaData)

	// start worker threads
	go mgr.handleImageFile(imagesChan, &wg)
	go mgr.handleImageFile(imagesChan, &wg)

	path = addSlash(path)

	files, err := ioutil.ReadDir(path)
	if err != nil {
		log.Fatalf("error while reading dir: %v", err)
	}
	for _, file := range files {
		if !file.IsDir() {
			fmt.Printf("skipping file '%v' (not allowed here), because its on gallery-folder-level\n", path+file.Name())
			continue
		}
		scanGalleryFolder(file.Name(), addSlash(path+file.Name()), imagesChan, &wg)
	}

	close(imagesChan)
	wg.Wait()
}

func scanGalleryFolder(galleryName, path string, imagesChan chan ImportMediaData, wg *sync.WaitGroup) {
	files, err := ioutil.ReadDir(path)
	if err != nil {
		log.Fatalf("error while reading dir: %v", err)
	}
	for _, file := range files {
		if file.IsDir() {
			fmt.Printf("skipping directory '%v' (not allowed here), because its on image-level\n", addSlash(path+file.Name()))
			continue
		}
		wg.Add(1)
		importMediaData := ImportMediaData{
			path:        path + file.Name(),
			galleryName: galleryName,
		}
		imagesChan <- importMediaData
	}
}

func (mgr ImportManager) handleImageFile(imagesChan <-chan ImportMediaData, wg *sync.WaitGroup) {
	for importMediaData := range imagesChan {
		fileHash, err := hashFile(importMediaData.path)
		if err != nil {
			fmt.Printf("skipping file '%v' due to an error: %v\n", importMediaData.path, err)
			continue
		}
		fmt.Printf("md5ChkSum of '%v' in gallery '%v': %v\n", importMediaData.path, importMediaData.galleryName, fileHash)
		wg.Done()
	}
}

func hashFile(filename string) (string, error) {
	var md5ChkSum string
	file, err := os.Open(filename)
	if err != nil {
		fmt.Println("could not open file:", filename)
		return md5ChkSum, err
	}
	defer file.Close()
	hash := md5.New()
	if _, err := io.Copy(hash, file); err != nil {
		fmt.Println("could not hash file:", filename)
		return md5ChkSum, err
	}
	md5ChkSum = hex.EncodeToString(hash.Sum(nil))
	return md5ChkSum, nil
}

func addSlash(path string) string {
	path = strings.TrimSpace(path)
	if !strings.HasSuffix(path, "/") {
		path = path + "/"
	}
	return path
}
<|endoftext|>


<commit_before>package lfs

import (
	"fmt"
	"github.com/github/git-lfs/git"
	"github.com/rubyist/tracerx"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

const Version = "0.5.1"

var (
	LargeSizeThreshold = 5 * 1024 * 1024
	TempDir            = filepath.Join(os.TempDir(), "git-lfs")
	UserAgent          string
	LocalWorkingDir    string
	LocalGitDir        string
	LocalMediaDir      string
	LocalLogDir        string
	checkedTempDir     string
)

func TempFile(prefix string) (*os.File, error) {
	if checkedTempDir != TempDir {
		if err := os.MkdirAll(TempDir, 0774); err != nil {
			return nil, err
		}
		checkedTempDir = TempDir
	}

	return ioutil.TempFile(TempDir, prefix)
}

func ResetTempDir() error {
	checkedTempDir = ""
	return os.RemoveAll(TempDir)
}

func LocalMediaPath(sha string) (string, error) {
	path := filepath.Join(LocalMediaDir, sha[0:2], sha[2:4])
	if err := os.MkdirAll(path, 0744); err != nil {
		return "", fmt.Errorf("Error trying to create local media directory in '%s': %s", path, err)
	}

	return filepath.Join(path, sha), nil
}

func Environ() []string {
	osEnviron := os.Environ()
	env := make([]string, 4, len(osEnviron)+4)
	env[0] = fmt.Sprintf("LocalWorkingDir=%s", LocalWorkingDir)
	env[1] = fmt.Sprintf("LocalGitDir=%s", LocalGitDir)
	env[2] = fmt.Sprintf("LocalMediaDir=%s", LocalMediaDir)
	env[3] = fmt.Sprintf("TempDir=%s", TempDir)

	for _, e := range osEnviron {
		if !strings.Contains(e, "GIT_") {
			continue
		}
		env = append(env, e)
	}

	return env
}

func InRepo() bool {
	return LocalWorkingDir != ""
}

func init() {
	var err error

	tracerx.DefaultKey = "GIT"
	tracerx.Prefix = "trace git-lfs: "

	LocalWorkingDir, LocalGitDir, err = resolveGitDir()
	if err == nil {
		LocalMediaDir = filepath.Join(LocalGitDir, "lfs", "objects")
		LocalLogDir = filepath.Join(LocalMediaDir, "logs")
		TempDir = filepath.Join(LocalGitDir, "lfs", "tmp")

		if err := os.MkdirAll(LocalMediaDir, 0744); err != nil {
			panic(fmt.Errorf("Error trying to create objects directory in '%s': %s", LocalMediaDir, err))
		}

		if err := os.MkdirAll(LocalLogDir, 0744); err != nil {
			panic(fmt.Errorf("Error trying to create log directory in '%s': %s", LocalLogDir, err))
		}

		if err := os.MkdirAll(TempDir, 0744); err != nil {
			panic(fmt.Errorf("Error trying to create temp directory in '%s': %s", TempDir, err))
		}

	}

	gitVersion, err := git.Config.Version()
	if err != nil {
		gitVersion = "unknown"
	}

	UserAgent = fmt.Sprintf("git-lfs/%s (GitHub; %s %s; git %s; go %s)", Version,
		runtime.GOOS,
		runtime.GOARCH,
		strings.Replace(gitVersion, "git version ", "", 1),
		strings.Replace(runtime.Version(), "go", "", 1))
}

func resolveGitDir() (string, string, error) {
	wd, err := os.Getwd()
	if err != nil {
		return "", "", err
	}

	return recursiveResolveGitDir(wd)
}

func recursiveResolveGitDir(dir string) (string, string, error) {
	var cleanDir = filepath.Clean(dir)
	if cleanDir[len(cleanDir)-1] == os.PathSeparator {
		return "", "", fmt.Errorf("Git repository not found")
	}

	if filepath.Base(dir) == gitExt {
		return filepath.Dir(dir), dir, nil
	}

	gitDir := filepath.Join(dir, gitExt)
	if info, err := os.Stat(gitDir); err == nil {
		if info.IsDir() {
			return dir, gitDir, nil
		} else {
			return processDotGitFile(gitDir)
		}
	}

	return recursiveResolveGitDir(filepath.Dir(dir))
}

func processDotGitFile(file string) (string, string, error) {
	f, err := os.Open(file)
	defer f.Close()

	if err != nil {
		return "", "", err
	}

	data := make([]byte, 512)
	n, err := f.Read(data)
	if err != nil {
		return "", "", err
	}

	contents := string(data[0:n])
	wd, _ := os.Getwd()
	if strings.HasPrefix(contents, gitPtrPrefix) {
		dir := strings.TrimSpace(strings.Split(contents, gitPtrPrefix)[1])
		absDir, _ := filepath.Abs(dir)
		return wd, absDir, nil
	}

	return wd, "", nil
}

const (
	gitExt       = ".git"
	gitPtrPrefix = "gitdir: "
)
<commit_msg>Checked for err != nil instead of == nil<commit_after>package lfs

import (
	"fmt"
	"github.com/github/git-lfs/git"
	"github.com/rubyist/tracerx"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

const Version = "0.5.1"

var (
	LargeSizeThreshold = 5 * 1024 * 1024
	TempDir            = filepath.Join(os.TempDir(), "git-lfs")
	UserAgent          string
	LocalWorkingDir    string
	LocalGitDir        string
	LocalMediaDir      string
	LocalLogDir        string
	checkedTempDir     string
)

func TempFile(prefix string) (*os.File, error) {
	if checkedTempDir != TempDir {
		if err := os.MkdirAll(TempDir, 0774); err != nil {
			return nil, err
		}
		checkedTempDir = TempDir
	}

	return ioutil.TempFile(TempDir, prefix)
}

func ResetTempDir() error {
	checkedTempDir = ""
	return os.RemoveAll(TempDir)
}

func LocalMediaPath(sha string) (string, error) {
	path := filepath.Join(LocalMediaDir, sha[0:2], sha[2:4])
	if err := os.MkdirAll(path, 0744); err != nil {
		return "", fmt.Errorf("Error trying to create local media directory in '%s': %s", path, err)
	}

	return filepath.Join(path, sha), nil
}

func Environ() []string {
	osEnviron := os.Environ()
	env := make([]string, 4, len(osEnviron)+4)
	env[0] = fmt.Sprintf("LocalWorkingDir=%s", LocalWorkingDir)
	env[1] = fmt.Sprintf("LocalGitDir=%s", LocalGitDir)
	env[2] = fmt.Sprintf("LocalMediaDir=%s", LocalMediaDir)
	env[3] = fmt.Sprintf("TempDir=%s", TempDir)

	for _, e := range osEnviron {
		if !strings.Contains(e, "GIT_") {
			continue
		}
		env = append(env, e)
	}

	return env
}

func InRepo() bool {
	return LocalWorkingDir != ""
}

func init() {
	var err error

	tracerx.DefaultKey = "GIT"
	tracerx.Prefix = "trace git-lfs: "

	LocalWorkingDir, LocalGitDir, err = resolveGitDir()
	if err == nil {
		LocalMediaDir = filepath.Join(LocalGitDir, "lfs", "objects")
		LocalLogDir = filepath.Join(LocalMediaDir, "logs")
		TempDir = filepath.Join(LocalGitDir, "lfs", "tmp")

		if err := os.MkdirAll(LocalMediaDir, 0744); err != nil {
			panic(fmt.Errorf("Error trying to create objects directory in '%s': %s", LocalMediaDir, err))
		}

		if err := os.MkdirAll(LocalLogDir, 0744); err != nil {
			panic(fmt.Errorf("Error trying to create log directory in '%s': %s", LocalLogDir, err))
		}

		if err := os.MkdirAll(TempDir, 0744); err != nil {
			panic(fmt.Errorf("Error trying to create temp directory in '%s': %s", TempDir, err))
		}

	}

	gitVersion, err := git.Config.Version()
	if err != nil {
		gitVersion = "unknown"
	}

	UserAgent = fmt.Sprintf("git-lfs/%s (GitHub; %s %s; git %s; go %s)", Version,
		runtime.GOOS,
		runtime.GOARCH,
		strings.Replace(gitVersion, "git version ", "", 1),
		strings.Replace(runtime.Version(), "go", "", 1))
}

func resolveGitDir() (string, string, error) {
	wd, err := os.Getwd()
	if err != nil {
		return "", "", err
	}

	return recursiveResolveGitDir(wd)
}

func recursiveResolveGitDir(dir string) (string, string, error) {
	var cleanDir = filepath.Clean(dir)
	if cleanDir[len(cleanDir)-1] == os.PathSeparator {
		return "", "", fmt.Errorf("Git repository not found")
	}

	if filepath.Base(dir) == gitExt {
		return filepath.Dir(dir), dir, nil
	}

	gitDir := filepath.Join(dir, gitExt)
	info, err := os.Stat(gitDir)
	if err != nil {
		return recursiveResolveGitDir(filepath.Dir(dir))
	}

	if info.IsDir() {
		return dir, gitDir, nil
	}

	return processDotGitFile(gitDir)
}

func processDotGitFile(file string) (string, string, error) {
	f, err := os.Open(file)
	defer f.Close()

	if err != nil {
		return "", "", err
	}

	data := make([]byte, 512)
	n, err := f.Read(data)
	if err != nil {
		return "", "", err
	}

	contents := string(data[0:n])
	wd, _ := os.Getwd()
	if strings.HasPrefix(contents, gitPtrPrefix) {
		dir := strings.TrimSpace(strings.Split(contents, gitPtrPrefix)[1])
		absDir, _ := filepath.Abs(dir)
		return wd, absDir, nil
	}

	return wd, "", nil
}

const (
	gitExt       = ".git"
	gitPtrPrefix = "gitdir: "
)
<|endoftext|>


<commit_before>package main

// ingest is the command line tool for pulling performance data from Google
// Storage and putting in Tiles. See the code in go/ingester for details on how
// ingestion is done.

import (
	"encoding/json"
	"flag"
	"net"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	"code.google.com/p/goauth2/compute/serviceaccount"
	"github.com/golang/glog"
	"github.com/rcrowley/go-metrics"
	"skia.googlesource.com/buildbot.git/perf/go/auth"
	"skia.googlesource.com/buildbot.git/perf/go/config"
	"skia.googlesource.com/buildbot.git/perf/go/flags"
	"skia.googlesource.com/buildbot.git/perf/go/gitinfo"
	"skia.googlesource.com/buildbot.git/perf/go/goldingester"
	"skia.googlesource.com/buildbot.git/perf/go/ingester"
	"skia.googlesource.com/buildbot.git/perf/go/trybot"
)

// flags
var (
	timestampFile  = flag.String("timestamp_file", "/tmp/timestamp.json", "File where timestamp data for ingester runs will be stored.")
	tileDir        = flag.String("tile_dir", "/tmp/tileStore2/", "Path where tiles will be placed.")
	gitRepoDir     = flag.String("git_repo_dir", "../../../skia", "Directory location for the Skia repo.")
	runEvery       = flag.Duration("run_every", 5*time.Minute, "How often the ingester should pull data from Google Storage.")
	runTrybotEvery = flag.Duration("run_trybot_every", 1*time.Minute, "How often the ingester to pull trybot data from Google Storage.")
	run            = flag.String("run", "nano,nano-trybot,golden", "A comma separated list of ingesters to run.")
	graphiteServer = flag.String("graphite_server", "skia-monitoring-b:2003", "Where is Graphite metrics ingestion server running.")
	doOauth        = flag.Bool("oauth", true, "Run through the OAuth 2.0 flow on startup, otherwise use a GCE service account.")
)

func Init() {
	metrics.RegisterRuntimeMemStats(metrics.DefaultRegistry)
	go metrics.CaptureRuntimeMemStats(metrics.DefaultRegistry, 1*time.Minute)
	addr, _ := net.ResolveTCPAddr("tcp", *graphiteServer)
	go metrics.Graphite(metrics.DefaultRegistry, 1*time.Minute, "ingest", addr)
}

// Timestamps is used to read and write the timestamp file, which records the time
// each ingestion last completed successfully.
//
// If an entry doesn't exist it returns BEGINNING_OF_TIME.
//
// Timestamp files look something like:
// {
//      "ingest":1445363563,
//      "trybot":1445363564,
//      "golden":1445363564,
// }
type Timestamps struct {
	Ingester map[string]int64 // Maps ingester name to its timestamp.

	filename string
	mutex    sync.Mutex
}

// NewTimestamp creates a new Timestamps that will read and write to the given
// filename.
func NewTimestamps(filename string) *Timestamps {
	return &Timestamps{
		Ingester: map[string]int64{
			"ingest": config.BEGINNING_OF_TIME.Unix(),
			"trybot": config.BEGINNING_OF_TIME.Unix(),
			"golden": config.BEGINNING_OF_TIME.Unix(),
		},
		filename: filename,
	}
}

// Read the timestamp data from the file.
func (t *Timestamps) Read() {
	t.mutex.Lock()
	defer t.mutex.Unlock()
	timestampFile, err := os.Open(t.filename)
	if err != nil {
		glog.Errorf("Error opening timestamp: %s", err)
		return
	}
	defer timestampFile.Close()
	err = json.NewDecoder(timestampFile).Decode(&t.Ingester)
	if err != nil {
		glog.Errorf("Failed to parse file %s: %s", t.filename, err)
	}
}

// Write the timestamp data to the file.
func (t *Timestamps) Write() {
	t.mutex.Lock()
	defer t.mutex.Unlock()
	writeTimestampFile, err := os.Create(t.filename)
	if err != nil {
		glog.Errorf("Write Timestamps: Failed to open file %s for writing: %s", t.filename, err)
		return
	}
	defer writeTimestampFile.Close()
	if err := json.NewEncoder(writeTimestampFile).Encode(t.Ingester); err != nil {
		glog.Errorf("Write Timestamps: Failed to encode timestamp file: %s", err)
	}
}

// Process is what each ingestion is wrapped up behind.
//
// A Process is expected to never return, and should be called as a Go routine.
type Process func()

// NewIngestionProcess creates a Process for ingesting data.
func NewIngestionProcess(ts *Timestamps, tsName string, git *gitinfo.GitInfo, tileDir, datasetName string, f ingester.IngestResultsFiles, gsDir string, every time.Duration) Process {
	i, err := ingester.NewIngester(git, tileDir, datasetName, f, gsDir)
	if err != nil {
		glog.Fatalf("Failed to create Ingester: %s", err)
	}

	// oneStep is a single round of ingestion.
	oneStep := func() {
		now := time.Now()
		err := i.Update(true, ts.Ingester[tsName])
		if err != nil {
			glog.Error(err)
		} else {
			ts.Ingester[tsName] = now.Unix()
			ts.Write()
		}
	}

	return func() {
		oneStep()
		for _ = range time.Tick(every) {
			oneStep()
		}
	}
}

func main() {
	flag.Parse()
	flags.Log()
	Init()

	var client *http.Client
	var err error
	if *doOauth {
		client, err = auth.RunFlow()
		if err != nil {
			glog.Fatalf("Failed to auth: %s", err)
		}
	} else {
		client, err = serviceaccount.NewClient(nil)
		if err != nil {
			glog.Fatalf("Failed to auth using a service account: %s", err)
		}
	}

	ingester.Init(client)
	trybot.Init()
	goldingester.Init()
	ts := NewTimestamps(*timestampFile)
	ts.Read()
	glog.Infof("Timestamps: %#v\n", ts.Ingester)

	git, err := gitinfo.NewGitInfo(*gitRepoDir, true)
	if err != nil {
		glog.Fatal("Failed loading Git info: %s\n", err)
	}

	// ingesters is a list of all the types of ingestion we can do.
	ingesters := map[string]Process{
		"nano":        NewIngestionProcess(ts, "ingest", git, *tileDir, config.DATASET_NANO, ingester.NanoBenchIngestion, "nano-json-v1", *runEvery),
		"nano-trybot": NewIngestionProcess(ts, "trybot", git, *tileDir, config.DATASET_NANO, trybot.TrybotIngestion, "trybot/nano-json-v1", *runTrybotEvery),
		"golden":      NewIngestionProcess(ts, "golden", git, *tileDir, config.DATASET_GOLDEN, goldingester.GoldenIngester, "dm-json-v1", *runEvery),
	}

	for _, name := range strings.Split(*run, ",") {
		glog.Infof("Process name: %s", name)
		if process, ok := ingesters[name]; ok {
			go process()
		} else {
			glog.Fatalf("Not a valid ingester name: %s", name)
		}
	}

	select {}
}
<commit_msg>Service accounts break nano ingestion.<commit_after>package main

// ingest is the command line tool for pulling performance data from Google
// Storage and putting in Tiles. See the code in go/ingester for details on how
// ingestion is done.

import (
	"encoding/json"
	"flag"
	"net"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/rcrowley/go-metrics"
	"skia.googlesource.com/buildbot.git/perf/go/auth"
	"skia.googlesource.com/buildbot.git/perf/go/config"
	"skia.googlesource.com/buildbot.git/perf/go/flags"
	"skia.googlesource.com/buildbot.git/perf/go/gitinfo"
	"skia.googlesource.com/buildbot.git/perf/go/goldingester"
	"skia.googlesource.com/buildbot.git/perf/go/ingester"
	"skia.googlesource.com/buildbot.git/perf/go/trybot"
)

// flags
var (
	timestampFile  = flag.String("timestamp_file", "/tmp/timestamp.json", "File where timestamp data for ingester runs will be stored.")
	tileDir        = flag.String("tile_dir", "/tmp/tileStore2/", "Path where tiles will be placed.")
	gitRepoDir     = flag.String("git_repo_dir", "../../../skia", "Directory location for the Skia repo.")
	runEvery       = flag.Duration("run_every", 5*time.Minute, "How often the ingester should pull data from Google Storage.")
	runTrybotEvery = flag.Duration("run_trybot_every", 1*time.Minute, "How often the ingester to pull trybot data from Google Storage.")
	run            = flag.String("run", "nano,nano-trybot,golden", "A comma separated list of ingesters to run.")
	graphiteServer = flag.String("graphite_server", "skia-monitoring-b:2003", "Where is Graphite metrics ingestion server running.")
	doOauth        = flag.Bool("oauth", true, "Run through the OAuth 2.0 flow on startup, otherwise use a GCE service account.")
)

func Init() {
	metrics.RegisterRuntimeMemStats(metrics.DefaultRegistry)
	go metrics.CaptureRuntimeMemStats(metrics.DefaultRegistry, 1*time.Minute)
	addr, _ := net.ResolveTCPAddr("tcp", *graphiteServer)
	go metrics.Graphite(metrics.DefaultRegistry, 1*time.Minute, "ingest", addr)
}

// Timestamps is used to read and write the timestamp file, which records the time
// each ingestion last completed successfully.
//
// If an entry doesn't exist it returns BEGINNING_OF_TIME.
//
// Timestamp files look something like:
// {
//      "ingest":1445363563,
//      "trybot":1445363564,
//      "golden":1445363564,
// }
type Timestamps struct {
	Ingester map[string]int64 // Maps ingester name to its timestamp.

	filename string
	mutex    sync.Mutex
}

// NewTimestamp creates a new Timestamps that will read and write to the given
// filename.
func NewTimestamps(filename string) *Timestamps {
	return &Timestamps{
		Ingester: map[string]int64{
			"ingest": config.BEGINNING_OF_TIME.Unix(),
			"trybot": config.BEGINNING_OF_TIME.Unix(),
			"golden": config.BEGINNING_OF_TIME.Unix(),
		},
		filename: filename,
	}
}

// Read the timestamp data from the file.
func (t *Timestamps) Read() {
	t.mutex.Lock()
	defer t.mutex.Unlock()
	timestampFile, err := os.Open(t.filename)
	if err != nil {
		glog.Errorf("Error opening timestamp: %s", err)
		return
	}
	defer timestampFile.Close()
	err = json.NewDecoder(timestampFile).Decode(&t.Ingester)
	if err != nil {
		glog.Errorf("Failed to parse file %s: %s", t.filename, err)
	}
}

// Write the timestamp data to the file.
func (t *Timestamps) Write() {
	t.mutex.Lock()
	defer t.mutex.Unlock()
	writeTimestampFile, err := os.Create(t.filename)
	if err != nil {
		glog.Errorf("Write Timestamps: Failed to open file %s for writing: %s", t.filename, err)
		return
	}
	defer writeTimestampFile.Close()
	if err := json.NewEncoder(writeTimestampFile).Encode(t.Ingester); err != nil {
		glog.Errorf("Write Timestamps: Failed to encode timestamp file: %s", err)
	}
}

// Process is what each ingestion is wrapped up behind.
//
// A Process is expected to never return, and should be called as a Go routine.
type Process func()

// NewIngestionProcess creates a Process for ingesting data.
func NewIngestionProcess(ts *Timestamps, tsName string, git *gitinfo.GitInfo, tileDir, datasetName string, f ingester.IngestResultsFiles, gsDir string, every time.Duration) Process {
	i, err := ingester.NewIngester(git, tileDir, datasetName, f, gsDir)
	if err != nil {
		glog.Fatalf("Failed to create Ingester: %s", err)
	}

	// oneStep is a single round of ingestion.
	oneStep := func() {
		now := time.Now()
		err := i.Update(true, ts.Ingester[tsName])
		if err != nil {
			glog.Error(err)
		} else {
			ts.Ingester[tsName] = now.Unix()
			ts.Write()
		}
	}

	return func() {
		oneStep()
		for _ = range time.Tick(every) {
			oneStep()
		}
	}
}

func main() {
	flag.Parse()
	flags.Log()
	Init()

	var client *http.Client
	var err error
	if *doOauth {
		client, err = auth.RunFlow()
		if err != nil {
			glog.Fatalf("Failed to auth: %s", err)
		}
	} else {
		client = nil
		// Add back service account access here when it's fixed.
	}

	ingester.Init(client)
	trybot.Init()
	goldingester.Init()
	ts := NewTimestamps(*timestampFile)
	ts.Read()
	glog.Infof("Timestamps: %#v\n", ts.Ingester)

	git, err := gitinfo.NewGitInfo(*gitRepoDir, true)
	if err != nil {
		glog.Fatal("Failed loading Git info: %s\n", err)
	}

	// ingesters is a list of all the types of ingestion we can do.
	ingesters := map[string]Process{
		"nano":        NewIngestionProcess(ts, "ingest", git, *tileDir, config.DATASET_NANO, ingester.NanoBenchIngestion, "nano-json-v1", *runEvery),
		"nano-trybot": NewIngestionProcess(ts, "trybot", git, *tileDir, config.DATASET_NANO, trybot.TrybotIngestion, "trybot/nano-json-v1", *runTrybotEvery),
		"golden":      NewIngestionProcess(ts, "golden", git, *tileDir, config.DATASET_GOLDEN, goldingester.GoldenIngester, "dm-json-v1", *runEvery),
	}

	for _, name := range strings.Split(*run, ",") {
		glog.Infof("Process name: %s", name)
		if process, ok := ingesters[name]; ok {
			go process()
		} else {
			glog.Fatalf("Not a valid ingester name: %s", name)
		}
	}

	select {}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package instances

import (
	"fmt"
	"net/http"
	"strings"

	compute "google.golang.org/api/compute/v1"
	"k8s.io/ingress/controllers/gce/storage"
	"k8s.io/ingress/controllers/gce/utils"
	"k8s.io/kubernetes/pkg/util/sets"

	"github.com/golang/glog"
)

const (
	// State string required by gce library to list all instances.
	allInstances = "ALL"
)

// Instances implements NodePool.
type Instances struct {
	cloud InstanceGroups
	// zones is a list of zones seeded by Kubernetes node zones.
	// TODO: we can figure this out.
	snapshotter storage.Snapshotter
	zoneLister
}

// NewNodePool creates a new node pool.
// - cloud: implements InstanceGroups, used to sync Kubernetes nodes with
//   members of the cloud InstanceGroup.
func NewNodePool(cloud InstanceGroups) NodePool {
	return &Instances{cloud, storage.NewInMemoryPool(), nil}
}

// Init initializes the instance pool. The given zoneLister is used to list
// all zones that require an instance group, and to lookup which zone a
// given Kubernetes node is in so we can add it to the right instance group.
func (i *Instances) Init(zl zoneLister) {
	i.zoneLister = zl
}

// AddInstanceGroup creates or gets an instance group if it doesn't exist
// and adds the given port to it. Returns a list of one instance group per zone,
// all of which have the exact same named port.
func (i *Instances) AddInstanceGroup(name string, port int64) ([]*compute.InstanceGroup, *compute.NamedPort, error) {
	igs := []*compute.InstanceGroup{}
	namedPort := &compute.NamedPort{}

	zones, err := i.ListZones()
	if err != nil {
		return igs, namedPort, err
	}

	for _, zone := range zones {
		ig, _ := i.Get(name, zone)
		var err error
		if ig == nil {
			glog.Infof("Creating instance group %v in zone %v", name, zone)
			ig, err = i.cloud.CreateInstanceGroup(name, zone)
			if err != nil {
				return nil, nil, err
			}
		} else {
			glog.V(3).Infof("Instance group %v already exists in zone %v, adding port %d to it", name, zone, port)
		}
		defer i.snapshotter.Add(name, struct{}{})
		namedPort, err = i.cloud.AddPortToInstanceGroup(ig, port)
		if err != nil {
			return nil, nil, err
		}
		igs = append(igs, ig)
	}
	return igs, namedPort, nil
}

// DeleteInstanceGroup deletes the given IG by name, from all zones.
func (i *Instances) DeleteInstanceGroup(name string) error {
	defer i.snapshotter.Delete(name)
	errs := []error{}

	zones, err := i.ListZones()
	if err != nil {
		return err
	}
	for _, zone := range zones {
		if err := i.cloud.DeleteInstanceGroup(name, zone); err != nil {
			if !utils.IsHTTPErrorCode(err, http.StatusNotFound) {
				errs = append(errs, err)
			}
		} else {
			glog.Infof("Deleted instance group %v in zone %v", name, zone)
		}
	}
	if len(errs) == 0 {
		return nil
	}
	return fmt.Errorf("%v", errs)
}

// list lists all instances in all zones.
func (i *Instances) list(name string) (sets.String, error) {
	nodeNames := sets.NewString()
	zones, err := i.ListZones()
	if err != nil {
		return nodeNames, err
	}

	for _, zone := range zones {
		instances, err := i.cloud.ListInstancesInInstanceGroup(
			name, zone, allInstances)
		if err != nil {
			return nodeNames, err
		}
		for _, ins := range instances.Items {
			// TODO: If round trips weren't so slow one would be inclided
			// to GetInstance using this url and get the name.
			parts := strings.Split(ins.Instance, "/")
			nodeNames.Insert(parts[len(parts)-1])
		}
	}
	return nodeNames, nil
}

// Get returns the Instance Group by name.
func (i *Instances) Get(name, zone string) (*compute.InstanceGroup, error) {
	ig, err := i.cloud.GetInstanceGroup(name, zone)
	if err != nil {
		return nil, err
	}
	i.snapshotter.Add(name, struct{}{})
	return ig, nil
}

// splitNodesByZones takes a list of node names and returns a map of zone:node names.
// It figures out the zones by asking the zoneLister.
func (i *Instances) splitNodesByZone(names []string) map[string][]string {
	nodesByZone := map[string][]string{}
	for _, name := range names {
		zone, err := i.GetZoneForNode(name)
		if err != nil {
			glog.Errorf("Failed to get zones for %v: %v, skipping", name, err)
			continue
		}
		if _, ok := nodesByZone[zone]; !ok {
			nodesByZone[zone] = []string{}
		}
		nodesByZone[zone] = append(nodesByZone[zone], name)
	}
	return nodesByZone
}

// Add adds the given instances to the appropriately zoned Instance Group.
func (i *Instances) Add(groupName string, names []string) error {
	errs := []error{}
	for zone, nodeNames := range i.splitNodesByZone(names) {
		glog.V(1).Infof("Adding nodes %v to %v in zone %v", nodeNames, groupName, zone)
		if err := i.cloud.AddInstancesToInstanceGroup(groupName, zone, nodeNames); err != nil {
			errs = append(errs, err)
		}
	}
	if len(errs) == 0 {
		return nil
	}
	return fmt.Errorf("%v", errs)
}

// Remove removes the given instances from the appropriately zoned Instance Group.
func (i *Instances) Remove(groupName string, names []string) error {
	errs := []error{}
	for zone, nodeNames := range i.splitNodesByZone(names) {
		glog.V(1).Infof("Adding nodes %v to %v in zone %v", nodeNames, groupName, zone)
		if err := i.cloud.RemoveInstancesFromInstanceGroup(groupName, zone, nodeNames); err != nil {
			errs = append(errs, err)
		}
	}
	if len(errs) == 0 {
		return nil
	}
	return fmt.Errorf("%v", errs)
}

// Sync syncs kubernetes instances with the instances in the instance group.
func (i *Instances) Sync(nodes []string) (err error) {
	glog.V(4).Infof("Syncing nodes %v", nodes)

	defer func() {
		// The node pool is only responsible for syncing nodes to instance
		// groups. It never creates/deletes, so if an instance groups is
		// not found there's nothing it can do about it anyway. Most cases
		// this will happen because the backend pool has deleted the instance
		// group, however if it happens because a user deletes the IG by mistake
		// we should just wait till the backend pool fixes it.
		if utils.IsHTTPErrorCode(err, http.StatusNotFound) {
			glog.Infof("Node pool encountered a 404, ignoring: %v", err)
			err = nil
		}
	}()

	pool := i.snapshotter.Snapshot()
	for igName := range pool {
		gceNodes := sets.NewString()
		gceNodes, err = i.list(igName)
		if err != nil {
			return err
		}
		kubeNodes := sets.NewString(nodes...)

		// A node deleted via kubernetes could still exist as a gce vm. We don't
		// want to route requests to it. Similarly, a node added to kubernetes
		// needs to get added to the instance group so we do route requests to it.

		removeNodes := gceNodes.Difference(kubeNodes).List()
		addNodes := kubeNodes.Difference(gceNodes).List()
		if len(removeNodes) != 0 {
			if err = i.Remove(
				igName, gceNodes.Difference(kubeNodes).List()); err != nil {
				return err
			}
		}

		if len(addNodes) != 0 {
			if err = i.Add(
				igName, kubeNodes.Difference(gceNodes).List()); err != nil {
				return err
			}
		}
	}
	return nil
}
<commit_msg>Log node-sync details<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package instances

import (
	"fmt"
	"net/http"
	"strings"

	compute "google.golang.org/api/compute/v1"
	"k8s.io/ingress/controllers/gce/storage"
	"k8s.io/ingress/controllers/gce/utils"
	"k8s.io/kubernetes/pkg/util/sets"

	"github.com/golang/glog"
)

const (
	// State string required by gce library to list all instances.
	allInstances = "ALL"
)

// Instances implements NodePool.
type Instances struct {
	cloud InstanceGroups
	// zones is a list of zones seeded by Kubernetes node zones.
	// TODO: we can figure this out.
	snapshotter storage.Snapshotter
	zoneLister
}

// NewNodePool creates a new node pool.
// - cloud: implements InstanceGroups, used to sync Kubernetes nodes with
//   members of the cloud InstanceGroup.
func NewNodePool(cloud InstanceGroups) NodePool {
	return &Instances{cloud, storage.NewInMemoryPool(), nil}
}

// Init initializes the instance pool. The given zoneLister is used to list
// all zones that require an instance group, and to lookup which zone a
// given Kubernetes node is in so we can add it to the right instance group.
func (i *Instances) Init(zl zoneLister) {
	i.zoneLister = zl
}

// AddInstanceGroup creates or gets an instance group if it doesn't exist
// and adds the given port to it. Returns a list of one instance group per zone,
// all of which have the exact same named port.
func (i *Instances) AddInstanceGroup(name string, port int64) ([]*compute.InstanceGroup, *compute.NamedPort, error) {
	igs := []*compute.InstanceGroup{}
	namedPort := &compute.NamedPort{}

	zones, err := i.ListZones()
	if err != nil {
		return igs, namedPort, err
	}

	for _, zone := range zones {
		ig, _ := i.Get(name, zone)
		var err error
		if ig == nil {
			glog.Infof("Creating instance group %v in zone %v", name, zone)
			ig, err = i.cloud.CreateInstanceGroup(name, zone)
			if err != nil {
				return nil, nil, err
			}
		} else {
			glog.V(3).Infof("Instance group %v already exists in zone %v, adding port %d to it", name, zone, port)
		}
		defer i.snapshotter.Add(name, struct{}{})
		namedPort, err = i.cloud.AddPortToInstanceGroup(ig, port)
		if err != nil {
			return nil, nil, err
		}
		igs = append(igs, ig)
	}
	return igs, namedPort, nil
}

// DeleteInstanceGroup deletes the given IG by name, from all zones.
func (i *Instances) DeleteInstanceGroup(name string) error {
	defer i.snapshotter.Delete(name)
	errs := []error{}

	zones, err := i.ListZones()
	if err != nil {
		return err
	}
	for _, zone := range zones {
		if err := i.cloud.DeleteInstanceGroup(name, zone); err != nil {
			if !utils.IsHTTPErrorCode(err, http.StatusNotFound) {
				errs = append(errs, err)
			}
		} else {
			glog.Infof("Deleted instance group %v in zone %v", name, zone)
		}
	}
	if len(errs) == 0 {
		return nil
	}
	return fmt.Errorf("%v", errs)
}

// list lists all instances in all zones.
func (i *Instances) list(name string) (sets.String, error) {
	nodeNames := sets.NewString()
	zones, err := i.ListZones()
	if err != nil {
		return nodeNames, err
	}

	for _, zone := range zones {
		instances, err := i.cloud.ListInstancesInInstanceGroup(
			name, zone, allInstances)
		if err != nil {
			return nodeNames, err
		}
		for _, ins := range instances.Items {
			// TODO: If round trips weren't so slow one would be inclided
			// to GetInstance using this url and get the name.
			parts := strings.Split(ins.Instance, "/")
			nodeNames.Insert(parts[len(parts)-1])
		}
	}
	return nodeNames, nil
}

// Get returns the Instance Group by name.
func (i *Instances) Get(name, zone string) (*compute.InstanceGroup, error) {
	ig, err := i.cloud.GetInstanceGroup(name, zone)
	if err != nil {
		return nil, err
	}
	i.snapshotter.Add(name, struct{}{})
	return ig, nil
}

// splitNodesByZones takes a list of node names and returns a map of zone:node names.
// It figures out the zones by asking the zoneLister.
func (i *Instances) splitNodesByZone(names []string) map[string][]string {
	nodesByZone := map[string][]string{}
	for _, name := range names {
		zone, err := i.GetZoneForNode(name)
		if err != nil {
			glog.Errorf("Failed to get zones for %v: %v, skipping", name, err)
			continue
		}
		if _, ok := nodesByZone[zone]; !ok {
			nodesByZone[zone] = []string{}
		}
		nodesByZone[zone] = append(nodesByZone[zone], name)
	}
	return nodesByZone
}

// Add adds the given instances to the appropriately zoned Instance Group.
func (i *Instances) Add(groupName string, names []string) error {
	errs := []error{}
	for zone, nodeNames := range i.splitNodesByZone(names) {
		glog.V(1).Infof("Adding nodes %v to %v in zone %v", nodeNames, groupName, zone)
		if err := i.cloud.AddInstancesToInstanceGroup(groupName, zone, nodeNames); err != nil {
			errs = append(errs, err)
		}
	}
	if len(errs) == 0 {
		return nil
	}
	return fmt.Errorf("%v", errs)
}

// Remove removes the given instances from the appropriately zoned Instance Group.
func (i *Instances) Remove(groupName string, names []string) error {
	errs := []error{}
	for zone, nodeNames := range i.splitNodesByZone(names) {
		glog.V(1).Infof("Adding nodes %v to %v in zone %v", nodeNames, groupName, zone)
		if err := i.cloud.RemoveInstancesFromInstanceGroup(groupName, zone, nodeNames); err != nil {
			errs = append(errs, err)
		}
	}
	if len(errs) == 0 {
		return nil
	}
	return fmt.Errorf("%v", errs)
}

// Sync syncs kubernetes instances with the instances in the instance group.
func (i *Instances) Sync(nodes []string) (err error) {
	glog.V(4).Infof("Syncing nodes %v", nodes)

	defer func() {
		// The node pool is only responsible for syncing nodes to instance
		// groups. It never creates/deletes, so if an instance groups is
		// not found there's nothing it can do about it anyway. Most cases
		// this will happen because the backend pool has deleted the instance
		// group, however if it happens because a user deletes the IG by mistake
		// we should just wait till the backend pool fixes it.
		if utils.IsHTTPErrorCode(err, http.StatusNotFound) {
			glog.Infof("Node pool encountered a 404, ignoring: %v", err)
			err = nil
		}
	}()

	pool := i.snapshotter.Snapshot()
	for igName := range pool {
		gceNodes := sets.NewString()
		gceNodes, err = i.list(igName)
		if err != nil {
			return err
		}
		kubeNodes := sets.NewString(nodes...)

		// A node deleted via kubernetes could still exist as a gce vm. We don't
		// want to route requests to it. Similarly, a node added to kubernetes
		// needs to get added to the instance group so we do route requests to it.

		removeNodes := gceNodes.Difference(kubeNodes).List()
		addNodes := kubeNodes.Difference(gceNodes).List()
		if len(removeNodes) != 0 {
			glog.V(4).Infof("Removing nodes from IG: %v", removeNodes)
			if err = i.Remove(igName, removeNodes); err != nil {
				return err
			}
		}

		if len(addNodes) != 0 {
			glog.V(4).Infof("Adding nodes to IG: %v", removeNodes)
			if err = i.Add(igName, addNodes); err != nil {
				return err
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package lxd

import (
	"fmt"
	"log"

	"strings"

	"github.com/hashicorp/terraform/helper/schema"
	lxd "github.com/lxc/lxd/client"
	"github.com/lxc/lxd/shared/api"
)

func resourceLxdCachedImage() *schema.Resource {
	return &schema.Resource{
		Create: resourceLxdCachedImageCreate,
		Update: resourceLxdCachedImageUpdate,
		Delete: resourceLxdCachedImageDelete,
		Exists: resourceLxdCachedImageExists,
		Read:   resourceLxdCachedImageRead,

		Schema: map[string]*schema.Schema{

			"aliases": {
				Type:     schema.TypeList,
				ForceNew: false,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
			},

			"copy_aliases": {
				Type:     schema.TypeBool,
				Default:  false,
				Optional: true,
				ForceNew: true,
			},

			"source_image": {
				Type:     schema.TypeString,
				ForceNew: true,
				Required: true,
			},

			"source_remote": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"remote": &schema.Schema{
				Type:     schema.TypeString,
				ForceNew: true,
				Optional: true,
				Default:  "",
			},

			// Computed attributes

			"architecture": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"created_at": {
				Type:     schema.TypeInt,
				Computed: true,
			},

			"fingerprint": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"copied_aliases": {
				Type:     schema.TypeList,
				Computed: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
			},
		},
	}
}

func resourceLxdCachedImageCreate(d *schema.ResourceData, meta interface{}) error {
	p := meta.(*LxdProvider)

	dstName := p.selectRemote(d)
	dstServer, err := p.GetContainerServer(dstName)
	if err != nil {
		return err
	}

	srcName := d.Get("source_remote").(string)
	imgServer, err := p.GetImageServer(srcName)
	if err != nil {
		return err
	}

	image := d.Get("source_image").(string)
	// has the user provided an fingerprint or alias?
	aliasTarget, _, _ := imgServer.GetImageAlias(image)
	if aliasTarget != nil {
		image = aliasTarget.Target
	}

	aliases := make([]api.ImageAlias, 0)
	if v, ok := d.GetOk("aliases"); ok {
		for _, alias := range v.([]interface{}) {
			// Check image alias doesn't already exist on destination
			dstAliasTarget, _, _ := dstServer.GetImageAlias(alias.(string))
			if dstAliasTarget != nil {
				return fmt.Errorf("Image alias already exists on destination: %s", alias.(string))
			}

			ia := api.ImageAlias{
				Name: alias.(string),
			}

			aliases = append(aliases, ia)
		}
	}

	// Get data about remote image, also checks it exists
	imgInfo, _, err := imgServer.GetImage(image)
	if err != nil {
		return err
	}

	copyAliases := d.Get("copy_aliases").(bool)

	// Execute the copy
	// Image copy arguments
	args := lxd.ImageCopyArgs{
		Aliases: aliases,
		Public:  false,
	}

	op, err := dstServer.CopyImage(imgServer, *imgInfo, &args)
	if err != nil {
		return err
	}

	// Wait for operation to finish
	err = op.Wait()
	if err != nil {
		return err
	}

	// Image was successfully copied, set resource ID
	id := newCachedImageId(dstName, imgInfo.Fingerprint)
	d.SetId(id.resourceId())

	// store remote aliases that we've copied, so we can filter them out later
	copied := make([]string, 0)
	if copyAliases {
		for _, a := range imgInfo.Aliases {
			copied = append(copied, a.Name)
		}
	}
	d.Set("copied_aliases", copied)

	return resourceLxdCachedImageRead(d, meta)
}

func resourceLxdCachedImageCopyProgressHandler(prog string) {
	log.Println("[DEBUG] - image copy progress: ", prog)
}

func resourceLxdCachedImageUpdate(d *schema.ResourceData, meta interface{}) error {
	p := meta.(*LxdProvider)
	remote := p.selectRemote(d)
	server, err := p.GetContainerServer(remote)
	if err != nil {
		return err
	}
	id := newCachedImageIdFromResourceId(d.Id())

	if d.HasChange("aliases") {
		old, new := d.GetChange("aliases")
		oldSet := schema.NewSet(schema.HashString, old.([]interface{}))
		newSet := schema.NewSet(schema.HashString, new.([]interface{}))
		aliasesToRemove := oldSet.Difference(newSet)
		aliasesToAdd := newSet.Difference(oldSet)

		// Delete removed
		for _, a := range aliasesToRemove.List() {
			alias := a.(string)
			err := server.DeleteImageAlias(alias)
			if err != nil {
				return err
			}
		}
		// Add new
		for _, a := range aliasesToAdd.List() {
			alias := a.(string)

			req := api.ImageAliasesPost{}
			req.Name = alias
			req.Target = id.fingerprint

			err := server.CreateImageAlias(req)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func resourceLxdCachedImageDelete(d *schema.ResourceData, meta interface{}) error {
	p := meta.(*LxdProvider)
	remote := p.selectRemote(d)
	server, err := p.GetContainerServer(remote)
	if err != nil {
		return err
	}

	id := newCachedImageIdFromResourceId(d.Id())

	op, err := server.DeleteImage(id.fingerprint)
	if err != nil {
		return err
	}

	return op.Wait()
}

func resourceLxdCachedImageExists(d *schema.ResourceData, meta interface{}) (bool, error) {
	p := meta.(*LxdProvider)
	remote := p.selectRemote(d)
	server, err := p.GetContainerServer(remote)
	if err != nil {
		return false, err
	}

	id := newCachedImageIdFromResourceId(d.Id())

	_, _, err = server.GetImage(id.fingerprint)
	if err != nil {
		if err.Error() == "not found" {
			return false, nil
		}
		return false, err
	}

	return true, nil
}

func resourceLxdCachedImageRead(d *schema.ResourceData, meta interface{}) error {
	p := meta.(*LxdProvider)
	remote := p.selectRemote(d)
	server, err := p.GetImageServer(remote)
	if err != nil {
		return err
	}

	id := newCachedImageIdFromResourceId(d.Id())

	img, _, err := server.GetImage(id.fingerprint)
	if err != nil {
		if err.Error() == "not found" {
			d.SetId("")
			return nil
		}
		return err
	}

	d.Set("fingerprint", id.fingerprint)
	d.Set("source_remote", d.Get("source_remote"))
	d.Set("copy_aliases", d.Get("copy_aliases"))
	d.Set("architecture", img.Architecture)
	d.Set("created_at", img.CreatedAt.Unix())

	// Read aliases from img and set in resource data
	// If the user has set 'copy_aliases' to true, then the
	// locally cached image will have aliases set that aren't
	// in the Terraform config.
	// These need to be filtered out here so not to cause a diff.
	var aliases []string
	copiedAliases := d.Get("copied_aliases").([]interface{})
	configAliases := d.Get("aliases").([]interface{})
	copiedSet := schema.NewSet(schema.HashString, copiedAliases)
	configSet := schema.NewSet(schema.HashString, configAliases)

	for _, a := range img.Aliases {
		if configSet.Contains(a.Name) || !copiedSet.Contains(a.Name) {
			aliases = append(aliases, a.Name)
		} else {
			log.Println("[DEBUG] filtered alias ", a)
		}
	}
	d.Set("aliases", aliases)

	return nil
}

type cachedImageId struct {
	remote      string
	fingerprint string
}

func newCachedImageId(remote, fingerprint string) cachedImageId {
	return cachedImageId{
		remote:      remote,
		fingerprint: fingerprint,
	}
}

func newCachedImageIdFromResourceId(id string) cachedImageId {
	parts := strings.SplitN(id, "/", 2)
	return cachedImageId{
		remote:      parts[0],
		fingerprint: parts[1],
	}
}

func (id cachedImageId) resourceId() string {
	return fmt.Sprintf("%s/%s", id.remote, id.fingerprint)
}
<commit_msg>golint: type cachedImageId should be cachedImageID<commit_after>package lxd

import (
	"fmt"
	"log"

	"strings"

	"github.com/hashicorp/terraform/helper/schema"
	lxd "github.com/lxc/lxd/client"
	"github.com/lxc/lxd/shared/api"
)

func resourceLxdCachedImage() *schema.Resource {
	return &schema.Resource{
		Create: resourceLxdCachedImageCreate,
		Update: resourceLxdCachedImageUpdate,
		Delete: resourceLxdCachedImageDelete,
		Exists: resourceLxdCachedImageExists,
		Read:   resourceLxdCachedImageRead,

		Schema: map[string]*schema.Schema{

			"aliases": {
				Type:     schema.TypeList,
				ForceNew: false,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
			},

			"copy_aliases": {
				Type:     schema.TypeBool,
				Default:  false,
				Optional: true,
				ForceNew: true,
			},

			"source_image": {
				Type:     schema.TypeString,
				ForceNew: true,
				Required: true,
			},

			"source_remote": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"remote": &schema.Schema{
				Type:     schema.TypeString,
				ForceNew: true,
				Optional: true,
				Default:  "",
			},

			// Computed attributes

			"architecture": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"created_at": {
				Type:     schema.TypeInt,
				Computed: true,
			},

			"fingerprint": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"copied_aliases": {
				Type:     schema.TypeList,
				Computed: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
			},
		},
	}
}

func resourceLxdCachedImageCreate(d *schema.ResourceData, meta interface{}) error {
	p := meta.(*LxdProvider)

	dstName := p.selectRemote(d)
	dstServer, err := p.GetContainerServer(dstName)
	if err != nil {
		return err
	}

	srcName := d.Get("source_remote").(string)
	imgServer, err := p.GetImageServer(srcName)
	if err != nil {
		return err
	}

	image := d.Get("source_image").(string)
	// has the user provided an fingerprint or alias?
	aliasTarget, _, _ := imgServer.GetImageAlias(image)
	if aliasTarget != nil {
		image = aliasTarget.Target
	}

	aliases := make([]api.ImageAlias, 0)
	if v, ok := d.GetOk("aliases"); ok {
		for _, alias := range v.([]interface{}) {
			// Check image alias doesn't already exist on destination
			dstAliasTarget, _, _ := dstServer.GetImageAlias(alias.(string))
			if dstAliasTarget != nil {
				return fmt.Errorf("Image alias already exists on destination: %s", alias.(string))
			}

			ia := api.ImageAlias{
				Name: alias.(string),
			}

			aliases = append(aliases, ia)
		}
	}

	// Get data about remote image, also checks it exists
	imgInfo, _, err := imgServer.GetImage(image)
	if err != nil {
		return err
	}

	copyAliases := d.Get("copy_aliases").(bool)

	// Execute the copy
	// Image copy arguments
	args := lxd.ImageCopyArgs{
		Aliases: aliases,
		Public:  false,
	}

	op, err := dstServer.CopyImage(imgServer, *imgInfo, &args)
	if err != nil {
		return err
	}

	// Wait for operation to finish
	err = op.Wait()
	if err != nil {
		return err
	}

	// Image was successfully copied, set resource ID
	id := newCachedImageId(dstName, imgInfo.Fingerprint)
	d.SetId(id.resourceId())

	// store remote aliases that we've copied, so we can filter them out later
	copied := make([]string, 0)
	if copyAliases {
		for _, a := range imgInfo.Aliases {
			copied = append(copied, a.Name)
		}
	}
	d.Set("copied_aliases", copied)

	return resourceLxdCachedImageRead(d, meta)
}

func resourceLxdCachedImageCopyProgressHandler(prog string) {
	log.Println("[DEBUG] - image copy progress: ", prog)
}

func resourceLxdCachedImageUpdate(d *schema.ResourceData, meta interface{}) error {
	p := meta.(*LxdProvider)
	remote := p.selectRemote(d)
	server, err := p.GetContainerServer(remote)
	if err != nil {
		return err
	}
	id := newCachedImageIdFromResourceId(d.Id())

	if d.HasChange("aliases") {
		old, new := d.GetChange("aliases")
		oldSet := schema.NewSet(schema.HashString, old.([]interface{}))
		newSet := schema.NewSet(schema.HashString, new.([]interface{}))
		aliasesToRemove := oldSet.Difference(newSet)
		aliasesToAdd := newSet.Difference(oldSet)

		// Delete removed
		for _, a := range aliasesToRemove.List() {
			alias := a.(string)
			err := server.DeleteImageAlias(alias)
			if err != nil {
				return err
			}
		}
		// Add new
		for _, a := range aliasesToAdd.List() {
			alias := a.(string)

			req := api.ImageAliasesPost{}
			req.Name = alias
			req.Target = id.fingerprint

			err := server.CreateImageAlias(req)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func resourceLxdCachedImageDelete(d *schema.ResourceData, meta interface{}) error {
	p := meta.(*LxdProvider)
	remote := p.selectRemote(d)
	server, err := p.GetContainerServer(remote)
	if err != nil {
		return err
	}

	id := newCachedImageIdFromResourceId(d.Id())

	op, err := server.DeleteImage(id.fingerprint)
	if err != nil {
		return err
	}

	return op.Wait()
}

func resourceLxdCachedImageExists(d *schema.ResourceData, meta interface{}) (bool, error) {
	p := meta.(*LxdProvider)
	remote := p.selectRemote(d)
	server, err := p.GetContainerServer(remote)
	if err != nil {
		return false, err
	}

	id := newCachedImageIdFromResourceId(d.Id())

	_, _, err = server.GetImage(id.fingerprint)
	if err != nil {
		if err.Error() == "not found" {
			return false, nil
		}
		return false, err
	}

	return true, nil
}

func resourceLxdCachedImageRead(d *schema.ResourceData, meta interface{}) error {
	p := meta.(*LxdProvider)
	remote := p.selectRemote(d)
	server, err := p.GetImageServer(remote)
	if err != nil {
		return err
	}

	id := newCachedImageIdFromResourceId(d.Id())

	img, _, err := server.GetImage(id.fingerprint)
	if err != nil {
		if err.Error() == "not found" {
			d.SetId("")
			return nil
		}
		return err
	}

	d.Set("fingerprint", id.fingerprint)
	d.Set("source_remote", d.Get("source_remote"))
	d.Set("copy_aliases", d.Get("copy_aliases"))
	d.Set("architecture", img.Architecture)
	d.Set("created_at", img.CreatedAt.Unix())

	// Read aliases from img and set in resource data
	// If the user has set 'copy_aliases' to true, then the
	// locally cached image will have aliases set that aren't
	// in the Terraform config.
	// These need to be filtered out here so not to cause a diff.
	var aliases []string
	copiedAliases := d.Get("copied_aliases").([]interface{})
	configAliases := d.Get("aliases").([]interface{})
	copiedSet := schema.NewSet(schema.HashString, copiedAliases)
	configSet := schema.NewSet(schema.HashString, configAliases)

	for _, a := range img.Aliases {
		if configSet.Contains(a.Name) || !copiedSet.Contains(a.Name) {
			aliases = append(aliases, a.Name)
		} else {
			log.Println("[DEBUG] filtered alias ", a)
		}
	}
	d.Set("aliases", aliases)

	return nil
}

type cachedImageID struct {
	remote      string
	fingerprint string
}

func newCachedImageId(remote, fingerprint string) cachedImageID {
	return cachedImageID{
		remote:      remote,
		fingerprint: fingerprint,
	}
}

func newCachedImageIdFromResourceId(id string) cachedImageID {
	parts := strings.SplitN(id, "/", 2)
	return cachedImageID{
		remote:      parts[0],
		fingerprint: parts[1],
	}
}

func (id cachedImageID) resourceId() string {
	return fmt.Sprintf("%s/%s", id.remote, id.fingerprint)
}
<|endoftext|>


<commit_before>package mcstore

import (
	"fmt"
	"net/http/httptest"
	"time"

	"github.com/emicklei/go-restful"
	"github.com/materials-commons/config"
	"github.com/materials-commons/gohandy/ezhttp"
	c "github.com/materials-commons/mcstore/cmd/pkg/client"
	"github.com/materials-commons/mcstore/pkg/db/dai"
	"github.com/materials-commons/mcstore/pkg/testdb"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/parnurzeal/gorequest"
	"github.com/willf/bitset"
	"net/http"
)

var _ = fmt.Println

var _ = Describe("UploadResource", func() {
	Describe("findStartingBlock method tests", func() {
		var (
			blocks *bitset.BitSet
		)

		BeforeEach(func() {
			blocks = bitset.New(10)
		})

		It("Should return 1 if no blocks have been set", func() {
			block := findStartingBlock(blocks)
			Expect(block).To(BeNumerically("==", 1))
		})

		It("Should return 2 if the first block has been uploaded", func() {
			// BitSet starts a zero. Flowjs starts at 1. So we have to adjust.
			blocks.Set(0)
			block := findStartingBlock(blocks)
			Expect(block).To(BeNumerically("==", 2))
		})

		It("Should return 1 if only the last block as been uploaded", func() {
			blocks.Set(9)
			block := findStartingBlock(blocks)
			Expect(block).To(BeNumerically("==", 1))
		})

		It("Should return 2 if only 2 has not been set (all others set)", func() {
			complement := blocks.Complement()
			complement.Clear(1) // second block
			block := findStartingBlock(complement)
			Expect(block).To(BeNumerically("==", 2))
		})
	})

	Describe("Upload REST API method tests", func() {
		var (
			client        *gorequest.SuperAgent
			server        *httptest.Server
			container     *restful.Container
			rr            *httptest.ResponseRecorder
			uploadRequest CreateUploadRequest
			uploads       dai.Uploads
		)

		BeforeEach(func() {
			client = c.NewGoRequest()
			container = NewServicesContainer(testdb.Sessions)
			server = httptest.NewServer(container)
			rr = httptest.NewRecorder()
			config.Set("mcurl", server.URL)
			uploadRequest = CreateUploadRequest{
				ProjectID:   "test",
				DirectoryID: "test",
				FileName:    "testreq.txt",
				FileSize:    4,
				ChunkSize:   2,
				FileMTime:   time.Now().Format(time.RFC1123),
				Checksum:    "abc123456",
			}
			uploads = dai.NewRUploads(testdb.RSessionMust())
		})

		var (
			createUploadRequest = func(req CreateUploadRequest) (*CreateUploadResponse, error) {
				r, body, errs := client.Post(Url("/upload")).Send(req).End()
				if err := ToError(r, errs); err != nil {
					return nil, err
				}

				var uploadResponse CreateUploadResponse
				if err := ToJSON(body, &uploadResponse); err != nil {
					return nil, err
				}
				return &uploadResponse, nil
			}
		)

		AfterEach(func() {
			server.Close()
		})

		Describe("create upload tests", func() {
			Context("No existing uploads that match request", func() {
				It("Should return an error when the user doesn't have permission", func() {
					// Set apikey for user who doesn't have permission
					config.Set("apikey", "test2")
					r, _, errs := client.Post(Url("/upload")).Send(uploadRequest).End()
					err := ToError(r, errs)
					Expect(err).NotTo(BeNil())
					Expect(r.StatusCode).To(BeNumerically("==", http.StatusUnauthorized))
				})

				It("Should return an error when the project doesn't exist", func() {
					config.Set("apikey", "test")
					uploadRequest.ProjectID = "does-not-exist"
					r, _, errs := client.Post(Url("/upload")).Send(uploadRequest).End()
					err := ToError(r, errs)
					Expect(err).NotTo(BeNil())
					Expect(r.StatusCode).To(BeNumerically("==", http.StatusNotFound))
				})

				It("Should return an error when the directory doesn't exist", func() {
					config.Set("apikey", "test")
					uploadRequest.DirectoryID = "does-not-exist"
					r, _, errs := client.Post(Url("/upload")).Send(uploadRequest).End()
					err := ToError(r, errs)
					Expect(err).NotTo(BeNil())
					Expect(r.StatusCode).To(BeNumerically("==", http.StatusNotFound))
				})

				It("Should return an error when the apikey doesn't exist", func() {
					config.Set("apikey", "does-not-exist")
					r, _, errs := client.Post(Url("/upload")).Send(uploadRequest).End()
					err := ToError(r, errs)
					Expect(err).NotTo(BeNil())
					Expect(r.StatusCode).To(BeNumerically("==", http.StatusUnauthorized))
				})

				It("Should create a new request for a valid submit", func() {
					config.Set("apikey", "test")
					r, body, errs := client.Post(Url("/upload")).Send(uploadRequest).End()
					err := ToError(r, errs)
					Expect(err).To(BeNil())
					Expect(r.StatusCode).To(BeNumerically("==", http.StatusOK))
					var uploadResponse CreateUploadResponse
					err = ToJSON(body, &uploadResponse)
					Expect(err).To(BeNil())
					Expect(uploadResponse.StartingBlock).To(BeNumerically("==", 1))

					uploadEntry, err := uploads.ByID(uploadResponse.RequestID)
					Expect(err).To(BeNil())
					Expect(uploadEntry.ID).To(Equal(uploadResponse.RequestID))
					err = uploads.Delete(uploadEntry.ID)
					Expect(err).To(BeNil())
				})
			})

			Context("Existing uploads that could match", func() {
				var (
					idsToDelete []string
				)

				BeforeEach(func() {
					idsToDelete = []string{}
				})

				AfterEach(func() {
					for _, id := range idsToDelete {
						uploads.Delete(id)
					}
				})

				var addID = func(id string) {
					idsToDelete = append(idsToDelete, id)
				}

				It("Should find an existing upload rather than create a new one", func() {
					config.Set("apikey", "test")
					r, body, errs := client.Post(Url("/upload")).Send(uploadRequest).End()
					err := ToError(r, errs)
					Expect(err).To(BeNil())
					var firstUploadResponse CreateUploadResponse
					err = ToJSON(body, &firstUploadResponse)
					Expect(err).To(BeNil())
					Expect(firstUploadResponse.StartingBlock).To(BeNumerically("==", 1))

					// Resend request - we should get the exact same request id back
					r, body, errs = client.Post(Url("/upload")).Send(uploadRequest).End()
					err = ToError(r, errs)
					Expect(err).To(BeNil())
					var secondUploadResponse CreateUploadResponse
					err = ToJSON(body, &secondUploadResponse)
					Expect(err).To(BeNil())
					Expect(secondUploadResponse.StartingBlock).To(BeNumerically("==", firstUploadResponse.StartingBlock))
					Expect(secondUploadResponse.RequestID).To(Equal(firstUploadResponse.RequestID))
					addID(firstUploadResponse.RequestID)
				})

				It("Should create a new upload when the request has a different checksum", func() {
					// Create two upload requests that are identical except for their checksums. This
					// should result in two different requests.

					config.Set("apikey", "test")
					r, body, errs := client.Post(Url("/upload")).Send(uploadRequest).End()
					err := ToError(r, errs)
					Expect(err).To(BeNil())
					var firstUploadResponse CreateUploadResponse
					err = ToJSON(body, &firstUploadResponse)
					Expect(err).To(BeNil())
					Expect(firstUploadResponse.StartingBlock).To(BeNumerically("==", 1))
					addID(firstUploadResponse.RequestID)

					// Send second request with a different checksum
					uploadRequest.Checksum = "def456"
					r, body, errs = client.Post(Url("/upload")).Send(uploadRequest).End()
					err = ToError(r, errs)
					Expect(err).To(BeNil())
					var secondUploadResponse CreateUploadResponse
					err = ToJSON(body, &secondUploadResponse)
					Expect(err).To(BeNil())
					Expect(secondUploadResponse.StartingBlock).To(BeNumerically("==", 1))
					Expect(secondUploadResponse.RequestID).NotTo(Equal(firstUploadResponse.RequestID))
					addID(secondUploadResponse.RequestID)
				})
			})

			Context("Restarting upload requests", func() {
				var (
					idsToDelete []string
				)

				BeforeEach(func() {
					idsToDelete = []string{}
				})

				AfterEach(func() {
					for _, id := range idsToDelete {
						uploads.Delete(id)
					}
				})

				var addID = func(id string) {
					idsToDelete = append(idsToDelete, id)
				}

				It("Should ask for second block after sending first block and then requesting upload again", func() {
					config.Set("apikey", "test")
					r, body, errs := client.Post(Url("/upload")).Send(uploadRequest).End()
					err := ToError(r, errs)
					Expect(err).To(BeNil())
					Expect(r.StatusCode).To(BeNumerically("==", http.StatusOK))
					var uploadResponse CreateUploadResponse
					err = ToJSON(body, &uploadResponse)
					Expect(err).To(BeNil())
					Expect(uploadResponse.StartingBlock).To(BeNumerically("==", 1))
					addID(uploadResponse.RequestID)

					// Second first block
					ezclient := ezhttp.NewClient()
					params := make(map[string]string)
					params["flowChunkNumber"] = "1"
					params["flowTotalChunks"] = "2"
					params["flowChunkSize"] = "2"
					params["flowTotalSize"] = "4"
					params["flowIdentifier"] = uploadResponse.RequestID
					params["flowFileName"] = "testreq.txt"
					params["flowRelativePath"] = "test/testreq.txt"
					params["projectID"] = "test"
					params["directoryID"] = "test"
					params["fileID"] = ""
					sc, err, body := ezclient.PostFileBytes(Url("/upload/chunk"), "/tmp/test.txt", "chunkData",
						[]byte("ab"), params)
					Expect(err).To(BeNil())
					Expect(sc).To(BeNumerically("==", http.StatusOK))
					var chunkResp UploadChunkResponse
					err = ToJSON(body, &chunkResp)
					Expect(err).To(BeNil())
					Expect(chunkResp.Done).To(BeFalse())

					// Now we will request this upload a second time.
					r, body, errs = client.Post(Url("/upload")).Send(uploadRequest).End()
					err = ToError(r, errs)
					Expect(err).To(BeNil())
					Expect(r.StatusCode).To(BeNumerically("==", http.StatusOK))
					var uploadResponse2 CreateUploadResponse
					err = ToJSON(body, &uploadResponse2)
					Expect(err).To(BeNil())
					Expect(uploadResponse2.StartingBlock).To(BeNumerically("==", 2))
					Expect(uploadResponse2.RequestID).To(Equal(uploadResponse.RequestID))
				})

				It("Should return an error when sending a bad id", func() {
					ezclient := ezhttp.NewClient()
					params := make(map[string]string)
					params["flowChunkNumber"] = "1"
					params["flowTotalChunks"] = "2"
					params["flowChunkSize"] = "2"
					params["flowTotalSize"] = "4"
					params["flowIdentifier"] = "i-dont-exist"
					params["flowFileName"] = "testreq.txt"
					params["flowRelativePath"] = "test/testreq.txt"
					params["projectID"] = "test"
					params["directoryID"] = "test"
					params["fileID"] = ""
					_, err, _ := ezclient.PostFileBytes(Url("/upload/chunk"), "/tmp/test.txt", "chunkData",
						[]byte("ab"), params)
					Expect(err).NotTo(BeNil())
				})
			})
		})

		Describe("get uploads tests", func() {
			It("Should return an error on a bad apikey", func() {
				config.Set("apikey", "test")
				resp, err := createUploadRequest(uploadRequest)
				Expect(err).To(BeNil())

				config.Set("apikey", "bad-key")
				r, _, errs := client.Get(Url("/upload/test")).End()
				err = ToError(r, errs)
				Expect(err).ToNot(BeNil())
				Expect(r.StatusCode).To(BeNumerically("==", http.StatusUnauthorized))

				err = uploads.Delete(resp.RequestID)
				Expect(err).To(BeNil())
			})

			It("Should return an error on a bad project", func() {
				config.Set("apikey", "test")
				r, _, errs := client.Get(Url("/upload/bad-project-id")).End()
				err := ToError(r, errs)
				Expect(err).ToNot(BeNil())
				Expect(r.StatusCode).To(BeNumerically("==", http.StatusBadRequest))
			})

			It("Should get existing upload requests for a project", func() {
				config.Set("apikey", "test")
				resp, err := createUploadRequest(uploadRequest)
				Expect(err).To(BeNil())
				r, body, errs := client.Get(Url("/upload/test")).End()
				err = ToError(r, errs)
				Expect(err).To(BeNil())
				Expect(r.StatusCode).To(BeNumerically("==", http.StatusOK))
				var entries []UploadEntry
				err = ToJSON(body, &entries)
				Expect(err).To(BeNil())
				Expect(entries).To(HaveLen(1))
				entry := entries[0]
				Expect(entry.RequestID).To(Equal(resp.RequestID))

				err = uploads.Delete(resp.RequestID)
				Expect(err).To(BeNil())
			})
		})
	})
})
<commit_msg>Fix tests and run formatter.<commit_after>package mcstore

import (
	"fmt"
	"net/http/httptest"
	"time"

	"net/http"

	"github.com/emicklei/go-restful"
	"github.com/materials-commons/config"
	"github.com/materials-commons/gohandy/ezhttp"
	c "github.com/materials-commons/mcstore/cmd/pkg/client"
	"github.com/materials-commons/mcstore/pkg/db/dai"
	"github.com/materials-commons/mcstore/pkg/testdb"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/parnurzeal/gorequest"
	"github.com/willf/bitset"
)

var _ = fmt.Println

var _ = Describe("UploadResource", func() {
	Describe("findStartingBlock method tests", func() {
		var (
			blocks *bitset.BitSet
		)

		BeforeEach(func() {
			blocks = bitset.New(10)
		})

		It("Should return 1 if no blocks have been set", func() {
			block := findStartingBlock(blocks)
			Expect(block).To(BeNumerically("==", 1))
		})

		It("Should return 2 if the first block has been uploaded", func() {
			// BitSet starts a zero. Flowjs starts at 1. So we have to adjust.
			blocks.Set(0)
			block := findStartingBlock(blocks)
			Expect(block).To(BeNumerically("==", 2))
		})

		It("Should return 1 if only the last block as been uploaded", func() {
			blocks.Set(9)
			block := findStartingBlock(blocks)
			Expect(block).To(BeNumerically("==", 1))
		})

		It("Should return 2 if only 2 has not been set (all others set)", func() {
			complement := blocks.Complement()
			complement.Clear(1) // second block
			block := findStartingBlock(complement)
			Expect(block).To(BeNumerically("==", 2))
		})
	})

	Describe("Upload REST API method tests", func() {
		var (
			client        *gorequest.SuperAgent
			server        *httptest.Server
			container     *restful.Container
			rr            *httptest.ResponseRecorder
			uploadRequest CreateUploadRequest
			uploads       dai.Uploads
		)

		BeforeEach(func() {
			client = c.NewGoRequest()
			container = NewServicesContainer(testdb.Sessions)
			server = httptest.NewServer(container)
			rr = httptest.NewRecorder()
			config.Set("mcurl", server.URL)
			uploadRequest = CreateUploadRequest{
				ProjectID:   "test",
				DirectoryID: "test",
				FileName:    "testreq.txt",
				FileSize:    4,
				ChunkSize:   2,
				FileMTime:   time.Now().Format(time.RFC1123),
				Checksum:    "abc123456",
			}
			uploads = dai.NewRUploads(testdb.RSessionMust())
		})

		var (
			createUploadRequest = func(req CreateUploadRequest) (*CreateUploadResponse, error) {
				r, body, errs := client.Post(Url("/upload")).Send(req).End()
				if err := ToError(r, errs); err != nil {
					return nil, err
				}

				var uploadResponse CreateUploadResponse
				if err := ToJSON(body, &uploadResponse); err != nil {
					return nil, err
				}
				return &uploadResponse, nil
			}
		)

		AfterEach(func() {
			server.Close()
		})

		Describe("create upload tests", func() {
			Context("No existing uploads that match request", func() {
				It("Should return an error when the user doesn't have permission", func() {
					// Set apikey for user who doesn't have permission
					config.Set("apikey", "test2")
					r, _, errs := client.Post(Url("/upload")).Send(uploadRequest).End()
					err := ToError(r, errs)
					Expect(err).NotTo(BeNil())
					Expect(r.StatusCode).To(BeNumerically("==", http.StatusUnauthorized))
				})

				It("Should return an error when the project doesn't exist", func() {
					config.Set("apikey", "test")
					uploadRequest.ProjectID = "does-not-exist"
					r, _, errs := client.Post(Url("/upload")).Send(uploadRequest).End()
					err := ToError(r, errs)
					Expect(err).NotTo(BeNil())
					Expect(r.StatusCode).To(BeNumerically("==", http.StatusNotFound))
				})

				It("Should return an error when the directory doesn't exist", func() {
					config.Set("apikey", "test")
					uploadRequest.DirectoryID = "does-not-exist"
					r, _, errs := client.Post(Url("/upload")).Send(uploadRequest).End()
					err := ToError(r, errs)
					Expect(err).NotTo(BeNil())
					Expect(r.StatusCode).To(BeNumerically("==", http.StatusNotFound))
				})

				It("Should return an error when the apikey doesn't exist", func() {
					config.Set("apikey", "does-not-exist")
					r, _, errs := client.Post(Url("/upload")).Send(uploadRequest).End()
					err := ToError(r, errs)
					Expect(err).NotTo(BeNil())
					Expect(r.StatusCode).To(BeNumerically("==", http.StatusUnauthorized))
				})

				It("Should create a new request for a valid submit", func() {
					config.Set("apikey", "test")
					r, body, errs := client.Post(Url("/upload")).Send(uploadRequest).End()
					err := ToError(r, errs)
					Expect(err).To(BeNil())
					Expect(r.StatusCode).To(BeNumerically("==", http.StatusOK))
					var uploadResponse CreateUploadResponse
					err = ToJSON(body, &uploadResponse)
					Expect(err).To(BeNil())
					Expect(uploadResponse.StartingBlock).To(BeNumerically("==", 1))

					uploadEntry, err := uploads.ByID(uploadResponse.RequestID)
					Expect(err).To(BeNil())
					Expect(uploadEntry.ID).To(Equal(uploadResponse.RequestID))
					err = uploads.Delete(uploadEntry.ID)
					Expect(err).To(BeNil())
				})
			})

			Context("Existing uploads that could match", func() {
				var (
					idsToDelete []string
				)

				BeforeEach(func() {
					idsToDelete = []string{}
				})

				AfterEach(func() {
					for _, id := range idsToDelete {
						uploads.Delete(id)
					}
				})

				var addID = func(id string) {
					idsToDelete = append(idsToDelete, id)
				}

				It("Should find an existing upload rather than create a new one", func() {
					config.Set("apikey", "test")
					r, body, errs := client.Post(Url("/upload")).Send(uploadRequest).End()
					err := ToError(r, errs)
					Expect(err).To(BeNil())
					var firstUploadResponse CreateUploadResponse
					err = ToJSON(body, &firstUploadResponse)
					Expect(err).To(BeNil())
					Expect(firstUploadResponse.StartingBlock).To(BeNumerically("==", 1))

					// Resend request - we should get the exact same request id back
					r, body, errs = client.Post(Url("/upload")).Send(uploadRequest).End()
					err = ToError(r, errs)
					Expect(err).To(BeNil())
					var secondUploadResponse CreateUploadResponse
					err = ToJSON(body, &secondUploadResponse)
					Expect(err).To(BeNil())
					Expect(secondUploadResponse.StartingBlock).To(BeNumerically("==", firstUploadResponse.StartingBlock))
					Expect(secondUploadResponse.RequestID).To(Equal(firstUploadResponse.RequestID))
					addID(firstUploadResponse.RequestID)
				})

				It("Should create a new upload when the request has a different checksum", func() {
					// Create two upload requests that are identical except for their checksums. This
					// should result in two different requests.

					config.Set("apikey", "test")
					r, body, errs := client.Post(Url("/upload")).Send(uploadRequest).End()
					err := ToError(r, errs)
					Expect(err).To(BeNil())
					var firstUploadResponse CreateUploadResponse
					err = ToJSON(body, &firstUploadResponse)
					Expect(err).To(BeNil())
					Expect(firstUploadResponse.StartingBlock).To(BeNumerically("==", 1))
					addID(firstUploadResponse.RequestID)

					// Send second request with a different checksum
					uploadRequest.Checksum = "def456"
					r, body, errs = client.Post(Url("/upload")).Send(uploadRequest).End()
					err = ToError(r, errs)
					Expect(err).To(BeNil())
					var secondUploadResponse CreateUploadResponse
					err = ToJSON(body, &secondUploadResponse)
					Expect(err).To(BeNil())
					Expect(secondUploadResponse.StartingBlock).To(BeNumerically("==", 1))
					Expect(secondUploadResponse.RequestID).NotTo(Equal(firstUploadResponse.RequestID))
					addID(secondUploadResponse.RequestID)
				})
			})

			Context("Restarting upload requests", func() {
				var (
					idsToDelete []string
				)

				BeforeEach(func() {
					idsToDelete = []string{}
				})

				AfterEach(func() {
					for _, id := range idsToDelete {
						uploads.Delete(id)
					}
				})

				var addID = func(id string) {
					idsToDelete = append(idsToDelete, id)
				}

				It("Should ask for second block after sending first block and then requesting upload again", func() {
					config.Set("apikey", "test")
					r, body, errs := client.Post(Url("/upload")).Send(uploadRequest).End()
					err := ToError(r, errs)
					Expect(err).To(BeNil())
					Expect(r.StatusCode).To(BeNumerically("==", http.StatusOK))
					var uploadResponse CreateUploadResponse
					err = ToJSON(body, &uploadResponse)
					Expect(err).To(BeNil())
					Expect(uploadResponse.StartingBlock).To(BeNumerically("==", 1))
					addID(uploadResponse.RequestID)

					// Second first block
					ezclient := ezhttp.NewClient()
					params := make(map[string]string)
					params["flowChunkNumber"] = "1"
					params["flowTotalChunks"] = "2"
					params["flowChunkSize"] = "2"
					params["flowTotalSize"] = "4"
					params["flowIdentifier"] = uploadResponse.RequestID
					params["flowFileName"] = "testreq.txt"
					params["flowRelativePath"] = "test/testreq.txt"
					params["projectID"] = "test"
					params["directoryID"] = "test"
					params["fileID"] = ""
					sc, err, body := ezclient.PostFileBytes(Url("/upload/chunk"), "/tmp/test.txt", "chunkData",
						[]byte("ab"), params)
					Expect(err).To(BeNil())
					Expect(sc).To(BeNumerically("==", http.StatusOK))
					var chunkResp UploadChunkResponse
					err = ToJSON(body, &chunkResp)
					Expect(err).To(BeNil())
					Expect(chunkResp.Done).To(BeFalse())

					// Now we will request this upload a second time.
					r, body, errs = client.Post(Url("/upload")).Send(uploadRequest).End()
					err = ToError(r, errs)
					Expect(err).To(BeNil())
					Expect(r.StatusCode).To(BeNumerically("==", http.StatusOK))
					var uploadResponse2 CreateUploadResponse
					err = ToJSON(body, &uploadResponse2)
					Expect(err).To(BeNil())
					Expect(uploadResponse2.StartingBlock).To(BeNumerically("==", 2))
					Expect(uploadResponse2.RequestID).To(Equal(uploadResponse.RequestID))
				})

				It("Should return an error when sending a bad id", func() {
					ezclient := ezhttp.NewClient()
					params := make(map[string]string)
					params["flowChunkNumber"] = "1"
					params["flowTotalChunks"] = "2"
					params["flowChunkSize"] = "2"
					params["flowTotalSize"] = "4"
					params["flowIdentifier"] = "i-dont-exist"
					params["flowFileName"] = "testreq.txt"
					params["flowRelativePath"] = "test/testreq.txt"
					params["projectID"] = "test"
					params["directoryID"] = "test"
					params["fileID"] = ""
					_, err, _ := ezclient.PostFileBytes(Url("/upload/chunk"), "/tmp/test.txt", "chunkData",
						[]byte("ab"), params)
					Expect(err).NotTo(BeNil())
				})
			})
		})

		Describe("get uploads tests", func() {
			It("Should return an error on a bad apikey", func() {
				config.Set("apikey", "test")
				resp, err := createUploadRequest(uploadRequest)
				Expect(err).To(BeNil())

				config.Set("apikey", "bad-key")
				r, _, errs := client.Get(Url("/upload/test")).End()
				err = ToError(r, errs)
				Expect(err).ToNot(BeNil())
				Expect(r.StatusCode).To(BeNumerically("==", http.StatusUnauthorized))

				err = uploads.Delete(resp.RequestID)
				Expect(err).To(BeNil())
			})

			It("Should return an error on a bad project", func() {
				config.Set("apikey", "test")
				r, _, errs := client.Get(Url("/upload/bad-project-id")).End()
				err := ToError(r, errs)
				Expect(err).ToNot(BeNil())
				Expect(r.StatusCode).To(BeNumerically("==", http.StatusNotFound))
			})

			It("Should get existing upload requests for a project", func() {
				config.Set("apikey", "test")
				resp, err := createUploadRequest(uploadRequest)
				Expect(err).To(BeNil())
				r, body, errs := client.Get(Url("/upload/test")).End()
				err = ToError(r, errs)
				Expect(err).To(BeNil())
				Expect(r.StatusCode).To(BeNumerically("==", http.StatusOK))
				var entries []UploadEntry
				err = ToJSON(body, &entries)
				Expect(err).To(BeNil())
				Expect(entries).To(HaveLen(1))
				entry := entries[0]
				Expect(entry.RequestID).To(Equal(resp.RequestID))

				err = uploads.Delete(resp.RequestID)
				Expect(err).To(BeNil())
			})
		})
	})
})
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kube

import (
	"errors"
	"fmt"
	"io/ioutil"

	"github.com/sirupsen/logrus"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
)

func localConfig() (*rest.Config, error) {
	return rest.InClusterConfig()
}

func kubeConfigs(kubeconfig string) (map[string]rest.Config, string, error) {
	// Attempt to load external clusters too
	var loader clientcmd.ClientConfigLoader
	if kubeconfig != "" { // load from --kubeconfig
		loader = &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}
	} else {
		loader = clientcmd.NewDefaultClientConfigLoadingRules()
	}

	cfg, err := loader.Load()
	if err != nil && kubeconfig != "" {
		return nil, "", fmt.Errorf("load: %v", err)
	}
	if err != nil {
		logrus.WithError(err).Warn("Cannot load kubecfg")
		return nil, "", nil
	}
	configs := map[string]rest.Config{}
	for context := range cfg.Contexts {
		logrus.Infof("* %s", context)
		contextCfg, err := clientcmd.NewNonInteractiveClientConfig(*cfg, context, &clientcmd.ConfigOverrides{}, loader).ClientConfig()
		if err != nil {
			return nil, "", fmt.Errorf("create %s client: %v", context, err)
		}
		configs[context] = *contextCfg
	}
	return configs, cfg.CurrentContext, nil
}

func buildConfigs(buildCluster string) (map[string]rest.Config, error) {
	if buildCluster == "" { // load from --build-cluster
		return nil, nil
	}
	data, err := ioutil.ReadFile(buildCluster)
	if err != nil {
		return nil, fmt.Errorf("read: %v", err)
	}
	raw, err := UnmarshalClusterMap(data)
	if err != nil {
		return nil, fmt.Errorf("unmarshal: %v", err)
	}
	cfg := &clientcmdapi.Config{
		Clusters:  map[string]*clientcmdapi.Cluster{},
		AuthInfos: map[string]*clientcmdapi.AuthInfo{},
		Contexts:  map[string]*clientcmdapi.Context{},
	}
	for alias, config := range raw {
		cfg.Clusters[alias] = &clientcmdapi.Cluster{
			Server:                   config.Endpoint,
			CertificateAuthorityData: config.ClusterCACertificate,
		}
		cfg.AuthInfos[alias] = &clientcmdapi.AuthInfo{
			ClientCertificateData: config.ClientCertificate,
			ClientKeyData:         config.ClientKey,
		}
		cfg.Contexts[alias] = &clientcmdapi.Context{
			Cluster:  alias,
			AuthInfo: alias,
			// TODO(fejta): Namespace?
		}
	}
	configs := map[string]rest.Config{}
	for context := range cfg.Contexts {
		logrus.Infof("* %s", context)
		contextCfg, err := clientcmd.NewNonInteractiveClientConfig(*cfg, context, &clientcmd.ConfigOverrides{}, nil).ClientConfig()
		if err != nil {
			return nil, fmt.Errorf("create %s client: %v", context, err)
		}
		configs[context] = *contextCfg
	}
	return configs, nil
}

func mergeConfigs(local *rest.Config, foreign map[string]rest.Config, currentContext string, buildClusters map[string]rest.Config) (map[string]rest.Config, error) {
	if buildClusters != nil {
		if _, ok := buildClusters[DefaultClusterAlias]; !ok {
			return nil, fmt.Errorf("build-cluster must have a %s context", DefaultClusterAlias)
		}
	}
	ret := map[string]rest.Config{}
	for ctx, cfg := range foreign {
		ret[ctx] = cfg
	}
	for ctx, cfg := range buildClusters {
		ret[ctx] = cfg
	}
	if local != nil {
		ret[InClusterContext] = *local
	} else if currentContext != "" {
		ret[InClusterContext] = ret[currentContext]
	} else {
		return nil, errors.New("no prow cluster access: in-cluster current kubecfg context required")
	}
	if len(ret) == 0 {
		return nil, errors.New("no client contexts found")
	}
	if _, ok := ret[DefaultClusterAlias]; !ok {
		ret[DefaultClusterAlias] = ret[InClusterContext]
	}
	return ret, nil
}

// LoadClusterConfigs loads rest.Configs for creation of clients, by using either a normal
// .kube/config file, a custom `Cluster` file, or both. The configs are returned in a mapping
// of context --> config. The default context is included in this mapping and specified as a
// return vaule. Errors are returned if .kube/config is specified and invalid or if no valid
// contexts are found.
func LoadClusterConfigs(kubeconfig, buildCluster string) (map[string]rest.Config, error) {

	logrus.Infof("Loading cluster contexts...")
	// This will work if we are running inside kubernetes
	localCfg, err := localConfig()
	if err != nil {
		logrus.WithError(err).Warn("Failed to create in-cluster config")
	}

	kubeCfgs, currentContext, err := kubeConfigs(kubeconfig)
	if err != nil {
		return nil, fmt.Errorf("kubecfg: %v", err)
	}

	// TODO(fejta): drop build-cluster support
	buildCfgs, err := buildConfigs(buildCluster)
	if err != nil {
		return nil, fmt.Errorf("build-cluster: %v", err)
	}

	return mergeConfigs(localCfg, kubeCfgs, currentContext, buildCfgs)
}
<commit_msg>Make kubeconfig loading logs less alarming.<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kube

import (
	"errors"
	"fmt"
	"io/ioutil"

	"github.com/sirupsen/logrus"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
)

func localConfig() (*rest.Config, error) {
	return rest.InClusterConfig()
}

func kubeConfigs(kubeconfig string) (map[string]rest.Config, string, error) {
	// Attempt to load external clusters too
	var loader clientcmd.ClientConfigLoader
	if kubeconfig != "" { // load from --kubeconfig
		loader = &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}
	} else {
		loader = clientcmd.NewDefaultClientConfigLoadingRules()
	}

	cfg, err := loader.Load()
	if err != nil && kubeconfig != "" {
		return nil, "", fmt.Errorf("load: %v", err)
	}
	if err != nil {
		logrus.WithError(err).Warn("Cannot load kubecfg")
		return nil, "", nil
	}
	configs := map[string]rest.Config{}
	for context := range cfg.Contexts {
		contextCfg, err := clientcmd.NewNonInteractiveClientConfig(*cfg, context, &clientcmd.ConfigOverrides{}, loader).ClientConfig()
		if err != nil {
			return nil, "", fmt.Errorf("create %s client: %v", context, err)
		}
		configs[context] = *contextCfg
		logrus.Infof("Parsed kubeconfig context: %s", context)
	}
	return configs, cfg.CurrentContext, nil
}

func buildConfigs(buildCluster string) (map[string]rest.Config, error) {
	if buildCluster == "" { // load from --build-cluster
		return nil, nil
	}
	data, err := ioutil.ReadFile(buildCluster)
	if err != nil {
		return nil, fmt.Errorf("read: %v", err)
	}
	raw, err := UnmarshalClusterMap(data)
	if err != nil {
		return nil, fmt.Errorf("unmarshal: %v", err)
	}
	cfg := &clientcmdapi.Config{
		Clusters:  map[string]*clientcmdapi.Cluster{},
		AuthInfos: map[string]*clientcmdapi.AuthInfo{},
		Contexts:  map[string]*clientcmdapi.Context{},
	}
	for alias, config := range raw {
		cfg.Clusters[alias] = &clientcmdapi.Cluster{
			Server:                   config.Endpoint,
			CertificateAuthorityData: config.ClusterCACertificate,
		}
		cfg.AuthInfos[alias] = &clientcmdapi.AuthInfo{
			ClientCertificateData: config.ClientCertificate,
			ClientKeyData:         config.ClientKey,
		}
		cfg.Contexts[alias] = &clientcmdapi.Context{
			Cluster:  alias,
			AuthInfo: alias,
			// TODO(fejta): Namespace?
		}
	}
	configs := map[string]rest.Config{}
	for context := range cfg.Contexts {
		logrus.Infof("* %s", context)
		contextCfg, err := clientcmd.NewNonInteractiveClientConfig(*cfg, context, &clientcmd.ConfigOverrides{}, nil).ClientConfig()
		if err != nil {
			return nil, fmt.Errorf("create %s client: %v", context, err)
		}
		configs[context] = *contextCfg
	}
	return configs, nil
}

func mergeConfigs(local *rest.Config, foreign map[string]rest.Config, currentContext string, buildClusters map[string]rest.Config) (map[string]rest.Config, error) {
	if buildClusters != nil {
		if _, ok := buildClusters[DefaultClusterAlias]; !ok {
			return nil, fmt.Errorf("build-cluster must have a %s context", DefaultClusterAlias)
		}
	}
	ret := map[string]rest.Config{}
	for ctx, cfg := range foreign {
		ret[ctx] = cfg
	}
	for ctx, cfg := range buildClusters {
		ret[ctx] = cfg
	}
	if local != nil {
		ret[InClusterContext] = *local
	} else if currentContext != "" {
		ret[InClusterContext] = ret[currentContext]
	} else {
		return nil, errors.New("no prow cluster access: in-cluster current kubecfg context required")
	}
	if len(ret) == 0 {
		return nil, errors.New("no client contexts found")
	}
	if _, ok := ret[DefaultClusterAlias]; !ok {
		ret[DefaultClusterAlias] = ret[InClusterContext]
	}
	return ret, nil
}

// LoadClusterConfigs loads rest.Configs for creation of clients, by using either a normal
// .kube/config file, a custom `Cluster` file, or both. The configs are returned in a mapping
// of context --> config. The default context is included in this mapping and specified as a
// return vaule. Errors are returned if .kube/config is specified and invalid or if no valid
// contexts are found.
func LoadClusterConfigs(kubeconfig, buildCluster string) (map[string]rest.Config, error) {

	logrus.Infof("Loading cluster contexts...")
	// This will work if we are running inside kubernetes
	localCfg, err := localConfig()
	if err != nil {
		logrus.WithError(err).Warn("Could not create in-cluster config (expected when running outside the cluster).")
	}

	kubeCfgs, currentContext, err := kubeConfigs(kubeconfig)
	if err != nil {
		return nil, fmt.Errorf("kubecfg: %v", err)
	}

	// TODO(fejta): drop build-cluster support
	buildCfgs, err := buildConfigs(buildCluster)
	if err != nil {
		return nil, fmt.Errorf("build-cluster: %v", err)
	}

	return mergeConfigs(localCfg, kubeCfgs, currentContext, buildCfgs)
}
<|endoftext|>


<commit_before>package cfgtest

import (
	cfg "github.com/mbrt/gmailctl/pkg/config/v1alpha3"
	"github.com/mbrt/gmailctl/pkg/parser"
)

func Test(rules []parser.Rule, tests []cfg.Test) error {
	return nil
}

func match(msg cfg.Message, c parser.CriteriaAST) (bool, error) {
	eval := ruleEvaluator{msg: msg}
	c.AcceptVisitor(&eval)
	return eval.Match, eval.Err
}

type ruleEvaluator struct {
	msg   cfg.Message
	Match bool
	Err   error
}

func (r *ruleEvaluator) VisitNode(n *parser.Node) {
	if n.Operation == parser.OperationNot {
		m, err := match(r.msg, n.Children[0])
		r.Match, r.Err = !m, err
		return
	}

	r.Match = accumulateInit(n.Operation)
	for _, child := range n.Children {
		m, err := match(r.msg, child)
		r.Match = accumulate(n.Operation, r.Match, m)
		if r.Err == nil && err != nil {
			// Never override the first error.
			r.Err = err
		}
	}
}

func (r *ruleEvaluator) VisitLeaf(n *parser.Leaf) {
	// TODO
}

func accumulateInit(op parser.OperationType) bool {
	// We need to start with a true only when we need to accumulate with
	// AND. OR has to start with false and we don't care about NOT.
	return op == parser.OperationAnd
}

func accumulate(op parser.OperationType, init, new bool) bool {
	if op == parser.OperationAnd {
		return init && new
	}
	return init || new
}
<commit_msg>Rewrite evaluators.<commit_after>package cfgtest

import (
	"fmt"
	"regexp"

	cfg "github.com/mbrt/gmailctl/pkg/config/v1alpha3"
	"github.com/mbrt/gmailctl/pkg/parser"
)

func Test(rules []parser.Rule, tests []cfg.Test) error {
	return nil
}

func NewEvaluator(criteria parser.CriteriaAST) (RuleEvaluator, error) {
	v := evalBuilder{}
	criteria.AcceptVisitor(&v)
	return v.Res, v.Err
}

type RuleEvaluator interface {
	Match(msg cfg.Message) bool
}

type andNode struct {
	children []RuleEvaluator
}

func (n andNode) Match(msg cfg.Message) bool {
	for _, c := range n.children {
		if !c.Match(msg) {
			return false
		}
	}
	return true
}

type orNode struct {
	children []RuleEvaluator
}

func (n orNode) Match(msg cfg.Message) bool {
	for _, c := range n.children {
		if c.Match(msg) {
			return true
		}
	}
	return false
}

type notNode struct {
	child RuleEvaluator
}

func (n notNode) Match(msg cfg.Message) bool {
	return !n.child.Match(msg)
}

type funcNode struct {
	op parser.FunctionType
	re *regexp.Regexp
}

func (n funcNode) Match(msg cfg.Message) bool {
	var fields []string

	switch n.op {
	case parser.FunctionFrom:
		fields = []string{msg.From}
	case parser.FunctionTo:
		fields = msg.To
	case parser.FunctionCc:
		fields = msg.Cc
	case parser.FunctionBcc:
		fields = msg.Bcc
	case parser.FunctionList:
		fields = msg.Lists
	case parser.FunctionSubject:
		fields = []string{msg.Subject}
	case parser.FunctionHas:
		fields = []string{msg.Body}
	}

	for _, f := range fields {
		if n.re.MatchString(f) {
			return true
		}
	}

	return false
}

type evalBuilder struct {
	Res RuleEvaluator
	Err error
}

func (r *evalBuilder) VisitNode(n *parser.Node) {
	var children []RuleEvaluator
	for _, c := range n.Children {
		ce, err := NewEvaluator(c)
		if err != nil {
			r.Err = err
			return
		}
		children = append(children, ce)
	}

	switch n.Operation {
	case parser.OperationAnd:
		r.Res = andNode{children}
	case parser.OperationOr:
		r.Res = orNode{children}
	case parser.OperationNot:
		if len(children) != 1 {
			r.Err = fmt.Errorf("unexpected children size for 'not' node: %d", len(children))
		}
		r.Res = notNode{children[0]}
	default:
		r.Err = fmt.Errorf("unsupported operation %s", n.Operation)
	}
}

func (r *evalBuilder) VisitLeaf(n *parser.Leaf) {
	// TODO
}
<|endoftext|>